@atproto/repo 0.2.0 → 0.3.1

package/tests/_util.ts

@@ -7,15 +7,15 @@ import { RepoStorage } from '../src/storage'
 import { MST } from '../src/mst'
 import {
   BlockMap,
-  collapseWriteLog,
   CollectionContents,
   RecordWriteOp,
   RepoContents,
   RecordPath,
-  WriteLog,
   WriteOpAction,
   RecordClaim,
   Commit,
+  DataDiff,
+  CommitData,
 } from '../src'
 import { Keypair, randomBytes } from '@atproto/crypto'
 
@@ -109,7 +109,7 @@ export const fillRepo = async (
   }
 }
 
-export const editRepo = async (
+export const formatEdit = async (
   repo: Repo,
   prevData: RepoContents,
   keypair: crypto.Keypair,
@@ -118,91 +118,58 @@ export const editRepo = async (
     updates?: number
     deletes?: number
   },
-): Promise<{ repo: Repo; data: RepoContents }> => {
+): Promise<{ commit: CommitData; data: RepoContents }> => {
   const { adds = 0, updates = 0, deletes = 0 } = params
   const repoData: RepoContents = {}
+  const writes: RecordWriteOp[] = []
   for (const collName of testCollections) {
-    const collData = prevData[collName]
+    const collData = { ...(prevData[collName] ?? {}) }
     const shuffled = shuffle(Object.entries(collData))
 
     for (let i = 0; i < adds; i++) {
       const object = generateObject()
       const rkey = TID.nextStr()
       collData[rkey] = object
-      repo = await repo.applyWrites(
-        {
-          action: WriteOpAction.Create,
-          collection: collName,
-          rkey,
-          record: object,
-        },
-        keypair,
-      )
+      writes.push({
+        action: WriteOpAction.Create,
+        collection: collName,
+        rkey,
+        record: object,
+      })
     }
 
     const toUpdate = shuffled.slice(0, updates)
     for (let i = 0; i < toUpdate.length; i++) {
       const object = generateObject()
       const rkey = toUpdate[i][0]
-      repo = await repo.applyWrites(
-        {
-          action: WriteOpAction.Update,
-          collection: collName,
-          rkey,
-          record: object,
-        },
-        keypair,
-      )
       collData[rkey] = object
+      writes.push({
+        action: WriteOpAction.Update,
+        collection: collName,
+        rkey,
+        record: object,
+      })
     }
 
     const toDelete = shuffled.slice(updates, deletes)
     for (let i = 0; i < toDelete.length; i++) {
       const rkey = toDelete[i][0]
-      repo = await repo.applyWrites(
-        {
-          action: WriteOpAction.Delete,
-          collection: collName,
-          rkey,
-        },
-        keypair,
-      )
       delete collData[rkey]
+      writes.push({
+        action: WriteOpAction.Delete,
+        collection: collName,
+        rkey,
+      })
     }
     repoData[collName] = collData
   }
+  const commit = await repo.formatCommit(writes, keypair)
   return {
-    repo,
+    commit,
     data: repoData,
   }
 }
 
-export const verifyRepoDiff = async (
-  writeLog: WriteLog,
-  before: RepoContents,
-  after: RepoContents,
-): Promise<void> => {
-  const getVal = (op: RecordWriteOp, data: RepoContents) => {
-    return (data[op.collection] || {})[op.rkey]
-  }
-  const ops = await collapseWriteLog(writeLog)
-
-  for (const op of ops) {
-    if (op.action === WriteOpAction.Create) {
-      expect(getVal(op, before)).toBeUndefined()
-      expect(getVal(op, after)).toEqual(op.record)
-    } else if (op.action === WriteOpAction.Update) {
-      expect(getVal(op, before)).toBeDefined()
-      expect(getVal(op, after)).toEqual(op.record)
-    } else if (op.action === WriteOpAction.Delete) {
-      expect(getVal(op, before)).toBeDefined()
-      expect(getVal(op, after)).toBeUndefined()
-    } else {
-      throw new Error('unexpected op type')
-    }
-  }
-}
-
 export const contentsToClaims = (contents: RepoContents): RecordClaim[] => {
   const claims: RecordClaim[] = []
   for (const coll of Object.keys(contents)) {
@@ -233,23 +200,27 @@ export const addBadCommit = async (
   keypair: Keypair,
 ): Promise<Repo> => {
   const obj = generateObject()
-  const blocks = new BlockMap()
-  const cid = await blocks.add(obj)
+  const newBlocks = new BlockMap()
+  const cid = await newBlocks.add(obj)
   const updatedData = await repo.data.add(`com.example.test/${TID.next()}`, cid)
-  const unstoredData = await updatedData.getUnstoredBlocks()
-  blocks.addMap(unstoredData.blocks)
+  const dataCid = await updatedData.getPointer()
+  const diff = await DataDiff.of(updatedData, repo.data)
+  newBlocks.addMap(diff.newMstBlocks)
   // we generate a bad sig by signing some other data
+  const rev = TID.nextStr(repo.commit.rev)
   const commit: Commit = {
     ...repo.commit,
-    prev: repo.cid,
-    data: unstoredData.root,
+    rev,
+    data: dataCid,
     sig: await keypair.sign(randomBytes(256)),
   }
-  const commitCid = await blocks.add(commit)
+  const commitCid = await newBlocks.add(commit)
   await repo.storage.applyCommit({
-    commit: commitCid,
+    cid: commitCid,
+    rev,
     prev: repo.cid,
-    blocks: blocks,
+    newBlocks,
+    removedCids: diff.removedCids,
   })
   return await Repo.load(repo.storage, commitCid)
 }

package/tests/mst.test.ts

@@ -150,7 +150,10 @@ describe('Merkle Search Tree', () => {
       } else {
         cid = entry.value
       }
-      const found = (await blockstore.has(cid)) || diff.newCids.has(cid)
+      const found =
+        (await blockstore.has(cid)) ||
+        diff.newMstBlocks.has(cid) ||
+        diff.newLeafCids.has(cid)
       expect(found).toBeTruthy()
     }
   })

package/tests/{sync/narrow.test.ts → proofs.test.ts}

@@ -1,13 +1,12 @@
 import { TID, streamToBuffer } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
-import { RecordClaim, Repo, RepoContents } from '../../src'
-import { MemoryBlockstore } from '../../src/storage'
-import * as verify from '../../src/verify'
-import * as sync from '../../src/sync'
+import { RecordClaim, Repo, RepoContents } from '../src'
+import { MemoryBlockstore } from '../src/storage'
+import * as sync from '../src/sync'
 
-import * as util from '../_util'
+import * as util from './_util'
 
-describe('Narrow Sync', () => {
+describe('Repo Proofs', () => {
   let storage: MemoryBlockstore
   let repo: Repo
   let keypair: crypto.Keypair
@@ -29,7 +28,7 @@ describe('Narrow Sync', () => {
   }
 
   const doVerify = (proofs: Uint8Array, claims: RecordClaim[]) => {
-    return verify.verifyProofs(proofs, claims, repoDid, keypair.did())
+    return sync.verifyProofs(proofs, claims, repoDid, keypair.did())
   }
 
   it('verifies valid records', async () => {
@@ -112,7 +111,7 @@ describe('Narrow Sync', () => {
       possible[8],
     ]
     const proofs = await getProofs(claims)
-    const records = await verify.verifyRecords(proofs, repoDid, keypair.did())
+    const records = await sync.verifyRecords(proofs, repoDid, keypair.did())
     for (const record of records) {
       const foundClaim = claims.find(
         (claim) =>
@@ -127,23 +126,13 @@ describe('Narrow Sync', () => {
     }
   })
 
-  it('verifyRecords throws on a bad signature', async () => {
-    const badRepo = await util.addBadCommit(repo, keypair)
-    const claims = util.contentsToClaims(repoData)
-    const proofs = await streamToBuffer(
-      sync.getRecords(storage, badRepo.cid, claims),
-    )
-    const fn = verify.verifyRecords(proofs, repoDid, keypair.did())
-    await expect(fn).rejects.toThrow(verify.RepoVerificationError)
-  })
-
   it('verifyProofs throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
     const claims = util.contentsToClaims(repoData)
     const proofs = await streamToBuffer(
       sync.getRecords(storage, badRepo.cid, claims),
     )
-    const fn = verify.verifyProofs(proofs, claims, repoDid, keypair.did())
-    await expect(fn).rejects.toThrow(verify.RepoVerificationError)
+    const fn = sync.verifyProofs(proofs, claims, repoDid, keypair.did())
+    await expect(fn).rejects.toThrow(sync.RepoVerificationError)
   })
 })

package/tests/repo.test.ts

@@ -22,7 +22,7 @@ describe('Repo', () => {
 
   it('has proper metadata', async () => {
     expect(repo.did).toEqual(keypair.did())
-    expect(repo.version).toBe(2)
+    expect(repo.version).toBe(3)
   })
 
   it('does basic operations', async () => {
@@ -75,12 +75,13 @@ describe('Repo', () => {
   })
 
   it('edits and deletes content', async () => {
-    const edited = await util.editRepo(repo, repoData, keypair, {
+    const edit = await util.formatEdit(repo, repoData, keypair, {
       adds: 20,
       updates: 20,
       deletes: 20,
     })
-    repo = edited.repo
+    repo = await repo.applyCommit(edit.commit)
+    repoData = edit.data
     const contents = await repo.getContents()
     expect(contents).toEqual(repoData)
   })
@@ -100,6 +101,6 @@ describe('Repo', () => {
     const contents = await reloadedRepo.getContents()
     expect(contents).toEqual(repoData)
     expect(repo.did).toEqual(keypair.did())
-    expect(repo.version).toBe(2)
+    expect(repo.version).toBe(3)
   })
 })

package/tests/sync.test.ts

@@ -0,0 +1,97 @@
+import * as crypto from '@atproto/crypto'
+import {
+  CidSet,
+  Repo,
+  RepoContents,
+  RepoVerificationError,
+  readCarWithRoot,
+} from '../src'
+import { MemoryBlockstore } from '../src/storage'
+import * as sync from '../src/sync'
+
+import * as util from './_util'
+import { streamToBuffer } from '@atproto/common'
+import { CarReader } from '@ipld/car/reader'
+
+describe('Repo Sync', () => {
+  let storage: MemoryBlockstore
+  let repo: Repo
+  let keypair: crypto.Keypair
+  let repoData: RepoContents
+
+  const repoDid = 'did:example:test'
+
+  beforeAll(async () => {
+    storage = new MemoryBlockstore()
+    keypair = await crypto.Secp256k1Keypair.create()
+    repo = await Repo.create(storage, repoDid, keypair)
+    const filled = await util.fillRepo(repo, keypair, 20)
+    repo = filled.repo
+    repoData = filled.data
+  })
+
+  it('sync a full repo', async () => {
+    const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const car = await readCarWithRoot(carBytes)
+    const verified = await sync.verifyRepo(
+      car.blocks,
+      car.root,
+      repoDid,
+      keypair.did(),
+    )
+    const syncStorage = new MemoryBlockstore()
+    await syncStorage.applyCommit(verified.commit)
+    const loadedRepo = await Repo.load(syncStorage, car.root)
+    const contents = await loadedRepo.getContents()
+    expect(contents).toEqual(repoData)
+    const contentsFromOps: RepoContents = {}
+    for (const write of verified.creates) {
+      contentsFromOps[write.collection] ??= {}
+      contentsFromOps[write.collection][write.rkey] = write.record
+    }
+    expect(contentsFromOps).toEqual(repoData)
+  })
+
+  it('does not sync duplicate blocks', async () => {
+    const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const car = await CarReader.fromBytes(carBytes)
+    const cids = new CidSet()
+    for await (const block of car.blocks()) {
+      if (cids.has(block.cid)) {
+        throw new Error(`duplicate block: :${block.cid.toString()}`)
+      }
+      cids.add(block.cid)
+    }
+  })
+
+  it('syncs a repo that is behind', async () => {
+    // add more to providers's repo & have consumer catch up
+    const edit = await util.formatEdit(repo, repoData, keypair, {
+      adds: 10,
+      updates: 10,
+      deletes: 10,
+    })
+    const verified = await sync.verifyDiff(
+      repo,
+      edit.commit.newBlocks,
+      edit.commit.cid,
+      repoDid,
+      keypair.did(),
+    )
+    await storage.applyCommit(verified.commit)
+    repo = await Repo.load(storage, verified.commit.cid)
+    const contents = await repo.getContents()
+    expect(contents).toEqual(edit.data)
+  })
+
+  it('throws on a bad signature', async () => {
+    const badRepo = await util.addBadCommit(repo, keypair)
+    const carBytes = await streamToBuffer(
+      sync.getFullRepo(storage, badRepo.cid),
+    )
+    const car = await readCarWithRoot(carBytes)
+    await expect(
+      sync.verifyRepo(car.blocks, car.root, repoDid, keypair.did()),
+    ).rejects.toThrow(RepoVerificationError)
+  })
+})

package/tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
   "exclude": ["**/*.spec.ts", "**/*.test.ts"]
-}
+}

package/tsconfig.json

@@ -5,11 +5,11 @@
     "outDir": "./dist", // Your outDir,
     "emitDeclarationOnly": true
   },
-  "include": ["./src","__tests__/**/**.ts"],
+  "include": ["./src", "__tests__/**/**.ts"],
   "references": [
     { "path": "../common/tsconfig.build.json" },
     { "path": "../crypto/tsconfig.build.json" },
     { "path": "../identity/tsconfig.build.json" },
-    { "path": "../nsid/tsconfig.build.json" },
+    { "path": "../nsid/tsconfig.build.json" }
   ]
-}
+}

package/dist/src/block-map.d.ts

@@ -1,23 +0,0 @@
-import { CID } from 'multiformats/cid';
-export declare class BlockMap {
-    private map;
-    add(value: unknown): Promise<CID>;
-    set(cid: CID, bytes: Uint8Array): void;
-    get(cid: CID): Uint8Array | undefined;
-    getMany(cids: CID[]): {
-        blocks: BlockMap;
-        missing: CID[];
-    };
-    has(cid: CID): boolean;
-    clear(): void;
-    forEach(cb: (bytes: Uint8Array, cid: CID) => void): void;
-    entries(): Entry[];
-    addMap(toAdd: BlockMap): void;
-    get size(): number;
-    equals(other: BlockMap): boolean;
-}
-declare type Entry = {
-    cid: CID;
-    bytes: Uint8Array;
-};
-export default BlockMap;

package/dist/src/blockstore/index.d.ts

@@ -1,2 +0,0 @@
-export * from './ipld-store';
-export * from './memory-blockstore';

package/dist/src/blockstore/ipld-store.d.ts

@@ -1,27 +0,0 @@
-import { CID } from 'multiformats/cid';
-import { BlockWriter } from '@ipld/car/writer';
-import { check } from '@atproto/common';
-import { BlockReader } from '@ipld/car/api';
-import CidSet from '../cid-set';
-export declare abstract class IpldStore {
-    staged: Map<string, Uint8Array>;
-    constructor();
-    abstract getSavedBytes(cid: CID): Promise<Uint8Array | null>;
-    abstract hasSavedBlock(cid: CID): Promise<boolean>;
-    abstract saveStaged(): Promise<void>;
-    abstract destroySaved(): Promise<void>;
-    stageBytes(k: CID, v: Uint8Array): Promise<void>;
-    stage(value: unknown): Promise<CID>;
-    getBytes(cid: CID): Promise<Uint8Array>;
-    get<T>(cid: CID, schema: check.Def<T>): Promise<T>;
-    getUnchecked(cid: CID): Promise<unknown>;
-    has(cid: CID): Promise<boolean>;
-    isMissing(cid: CID): Promise<boolean>;
-    checkMissing(cids: CidSet): Promise<CidSet>;
-    clearStaged(): Promise<void>;
-    destroy(): Promise<void>;
-    addToCar(car: BlockWriter, cid: CID): Promise<void>;
-    stageCar(buf: Uint8Array): Promise<CID>;
-    stageCarBlocks(car: BlockReader): Promise<void>;
-}
-export default IpldStore;

package/dist/src/blockstore/memory-blockstore.d.ts

@@ -1,13 +0,0 @@
-import { CID } from 'multiformats/cid';
-import IpldStore from './ipld-store';
-export declare class MemoryBlockstore extends IpldStore {
-    blocks: Map<string, Uint8Array>;
-    constructor();
-    getSavedBytes(cid: CID): Promise<Uint8Array | null>;
-    hasSavedBlock(cid: CID): Promise<boolean>;
-    saveStaged(): Promise<void>;
-    sizeInBytes(): Promise<number>;
-    destroySaved(): Promise<void>;
-    getContents(): Promise<Record<string, unknown>>;
-}
-export default MemoryBlockstore;

package/dist/src/blockstore/persistent-blockstore.d.ts

@@ -1,12 +0,0 @@
-import level from 'level';
-import { CID } from 'multiformats/cid';
-import IpldStore from './ipld-store';
-export declare class PersistentBlockstore extends IpldStore {
-    store: level.Level;
-    constructor(location?: string);
-    getBytes(cid: CID): Promise<Uint8Array>;
-    putBytes(cid: CID, bytes: Uint8Array): Promise<void>;
-    has(cid: CID): Promise<boolean>;
-    destroy(): Promise<void>;
-}
-export default PersistentBlockstore;

package/dist/src/cid-set.d.ts

@@ -1,14 +0,0 @@
-import { CID } from 'multiformats';
-export declare class CidSet {
-    private set;
-    constructor(arr?: CID[]);
-    add(cid: CID): CidSet;
-    addSet(toMerge: CidSet): CidSet;
-    subtractSet(toSubtract: CidSet): CidSet;
-    delete(cid: CID): this;
-    has(cid: CID): boolean;
-    size(): number;
-    clear(): CidSet;
-    toList(): CID[];
-}
-export default CidSet;

package/dist/src/collection.d.ts

@@ -1,22 +0,0 @@
-import { NSID } from '@atproto/nsid';
-import { CID } from 'multiformats/cid';
-import Repo from './repo';
-export declare class Collection {
-    repo: Repo;
-    nsid: NSID;
-    constructor(repo: Repo, nsid: NSID | string);
-    name(): string;
-    dataIdForRecord(key: string): string;
-    getRecord(key: string): Promise<unknown | null>;
-    listRecords(count?: number, after?: string, before?: string): Promise<{
-        key: string;
-        value: unknown;
-    }[]>;
-    createRecord(record: unknown, rkey?: string): Promise<{
-        rkey: string;
-        cid: CID;
-    }>;
-    updateRecord(rkey: string, record: unknown): Promise<CID>;
-    deleteRecord(rkey: string): Promise<void>;
-}
-export default Collection;

package/dist/src/data-diff.d.ts

@@ -1,34 +0,0 @@
-import { CID } from 'multiformats';
-import CidSet from './cid-set';
-import { DataStore } from './types';
-export declare class DataDiff {
-    adds: Record<string, DataAdd>;
-    updates: Record<string, DataUpdate>;
-    deletes: Record<string, DataDelete>;
-    newCids: CidSet;
-    static of(curr: DataStore, prev: DataStore | null): Promise<DataDiff>;
-    recordAdd(key: string, cid: CID): void;
-    recordUpdate(key: string, prev: CID, cid: CID): void;
-    recordDelete(key: string, cid: CID): void;
-    recordNewCid(cid: CID): void;
-    addDiff(diff: DataDiff): void;
-    addList(): DataAdd[];
-    updateList(): DataUpdate[];
-    deleteList(): DataDelete[];
-    newCidList(): CID[];
-    updatedKeys(): string[];
-}
-export declare type DataAdd = {
-    key: string;
-    cid: CID;
-};
-export declare type DataUpdate = {
-    key: string;
-    prev: CID;
-    cid: CID;
-};
-export declare type DataDelete = {
-    key: string;
-    cid: CID;
-};
-export default DataDiff;

package/dist/src/error.d.ts

@@ -1,21 +0,0 @@
-import { Def } from '@atproto/common/src/check';
-import { CID } from 'multiformats/cid';
-export declare class MissingBlockError extends Error {
-    cid: CID;
-    constructor(cid: CID, def?: Def<unknown>);
-}
-export declare class MissingBlocksError extends Error {
-    context: string;
-    cids: CID[];
-    constructor(context: string, cids: CID[]);
-}
-export declare class MissingCommitBlocksError extends Error {
-    commit: CID;
-    cids: CID[];
-    constructor(commit: CID, cids: CID[]);
-}
-export declare class UnexpectedObjectError extends Error {
-    cid: CID;
-    def: Def<unknown>;
-    constructor(cid: CID, def: Def<unknown>);
-}

package/dist/src/index.d.ts

@@ -1,7 +0,0 @@
-export * from './blockstore';
-export * from './repo';
-export * from './mst';
-export * from './storage';
-export * from './types';
-export * from './verify';
-export * from './util';

package/dist/src/logger.d.ts

@@ -1,2 +0,0 @@
-export declare const logger: import("pino").default.Logger<import("pino").default.LoggerOptions>;
-export default logger;

package/dist/src/mst/diff.d.ts

@@ -1,33 +0,0 @@
-import * as auth from '@atproto/auth';
-import { CID } from 'multiformats';
-import CidSet from '../cid-set';
-export declare class DataDiff {
-    adds: Record<string, DataAdd>;
-    updates: Record<string, DataUpdate>;
-    deletes: Record<string, DataDelete>;
-    newCids: CidSet;
-    recordAdd(key: string, cid: CID): void;
-    recordUpdate(key: string, prev: CID, cid: CID): void;
-    recordDelete(key: string, cid: CID): void;
-    recordNewCid(cid: CID): void;
-    addDiff(diff: DataDiff): void;
-    addList(): DataAdd[];
-    updateList(): DataUpdate[];
-    deleteList(): DataDelete[];
-    newCidList(): CID[];
-    updatedKeys(): string[];
-    neededCapabilities(rootDid: string): auth.ucans.Capability[];
-}
-export declare type DataAdd = {
-    key: string;
-    cid: CID;
-};
-export declare type DataUpdate = {
-    key: string;
-    prev: CID;
-    cid: CID;
-};
-export declare type DataDelete = {
-    key: string;
-    cid: CID;
-};

package/dist/src/mst/index.d.ts

@@ -1,4 +0,0 @@
-export * from './mst';
-export * from './diff';
-export * from './walker';
-export * as mstUtil from './util';