@atproto/repo 0.1.0 → 0.3.0

package/src/util.ts

@@ -4,36 +4,34 @@ import { CarReader } from '@ipld/car/reader'
 import { BlockWriter, CarWriter } from '@ipld/car/writer'
 import { Block as CarBlock } from '@ipld/car/api'
 import {
-  streamToArray,
+  streamToBuffer,
   verifyCidForBytes,
   cborDecode,
   check,
   schema,
   cidForCbor,
+  byteIterableToStream,
+  TID,
 } from '@atproto/common'
 import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon'
 
 import * as crypto from '@atproto/crypto'
-import Repo from './repo'
-import { MST } from './mst'
 import DataDiff from './data-diff'
-import { RepoStorage } from './storage'
 import {
   Commit,
-  DataStore,
+  LegacyV2Commit,
   RecordCreateDescript,
   RecordDeleteDescript,
   RecordPath,
   RecordUpdateDescript,
   RecordWriteDescript,
   UnsignedCommit,
-  WriteLog,
   WriteOpAction,
 } from './types'
 import BlockMap from './block-map'
-import { MissingBlocksError } from './error'
 import * as parse from './parse'
 import { Keypair } from '@atproto/crypto'
+import { Readable } from 'stream'
 
 export async function* verifyIncomingCarBlocks(
   car: AsyncIterable<CarBlock>,
@@ -44,25 +42,37 @@ export async function* verifyIncomingCarBlocks(
   }
 }
 
-export const writeCar = async (
+// we have to turn the car writer output into a stream in order to properly handle errors
+export function writeCarStream(
   root: CID | null,
   fn: (car: BlockWriter) => Promise<void>,
-): Promise<Uint8Array> => {
+): Readable {
   const { writer, out } =
     root !== null ? CarWriter.create(root) : CarWriter.create()
-  const bytes = streamToArray(out)
-  try {
-    await fn(writer)
-  } finally {
-    writer.close()
+
+  const stream = byteIterableToStream(out)
+  fn(writer)
+    .catch((err) => {
+      stream.destroy(err)
+    })
+    .finally(() => writer.close())
+  return stream
+}
+
+export async function* writeCar(
+  root: CID | null,
+  fn: (car: BlockWriter) => Promise<void>,
+): AsyncIterable<Uint8Array> {
+  const stream = writeCarStream(root, fn)
+  for await (const chunk of stream) {
+    yield chunk
   }
-  return bytes
 }
 
-export const blocksToCar = async (
+export const blocksToCarStream = (
   root: CID | null,
   blocks: BlockMap,
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return writeCar(root, async (writer) => {
     for (const entry of blocks.entries()) {
       await writer.put(entry)
@@ -70,6 +80,14 @@ export const blocksToCar = async (
   })
 }
 
+export const blocksToCarFile = (
+  root: CID | null,
+  blocks: BlockMap,
+): Promise<Uint8Array> => {
+  const carStream = blocksToCarStream(root, blocks)
+  return streamToBuffer(carStream)
+}
+
 export const readCar = async (
   bytes: Uint8Array,
 ): Promise<{ roots: CID[]; blocks: BlockMap }> => {
@@ -99,33 +117,6 @@ export const readCarWithRoot = async (
   }
 }
 
-export const getWriteLog = async (
-  storage: RepoStorage,
-  latest: CID,
-  earliest: CID | null,
-): Promise<WriteLog> => {
-  const commits = await storage.getCommitPath(latest, earliest)
-  if (!commits) throw new Error('Could not find shared history')
-  const heads = await Promise.all(commits.map((c) => Repo.load(storage, c)))
-  // Turn commit path into list of diffs
-  let prev: DataStore = await MST.create(storage) // Empty
-  const msts = heads.map((h) => h.data)
-  const diffs: DataDiff[] = []
-  for (const mst of msts) {
-    diffs.push(await DataDiff.of(mst, prev))
-    prev = mst
-  }
-  const fullDiff = collapseDiffs(diffs)
-  const diffBlocks = await storage.getBlocks(fullDiff.newCidList())
-  if (diffBlocks.missing.length > 0) {
-    throw new MissingBlocksError('write op log', diffBlocks.missing)
-  }
-  // Map MST diffs to write ops
-  return Promise.all(
-    diffs.map((diff) => diffToWriteDescripts(diff, diffBlocks.blocks)),
-  )
-}
-
 export const diffToWriteDescripts = (
   diff: DataDiff,
   blocks: BlockMap,
@@ -166,55 +157,18 @@ export const diffToWriteDescripts = (
   ])
 }
 
-export const collapseWriteLog = (log: WriteLog): RecordWriteDescript[] => {
-  const creates: Record<string, RecordCreateDescript> = {}
-  const updates: Record<string, RecordUpdateDescript> = {}
-  const deletes: Record<string, RecordDeleteDescript> = {}
-  for (const commit of log) {
-    for (const op of commit) {
-      const key = op.collection + '/' + op.rkey
-      if (op.action === WriteOpAction.Create) {
-        const del = deletes[key]
-        if (del) {
-          if (del.cid !== op.cid) {
-            updates[key] = {
-              ...op,
-              action: WriteOpAction.Update,
-              prev: del.cid,
-            }
-          }
-          delete deletes[key]
-        } else {
-          creates[key] = op
-        }
-      } else if (op.action === WriteOpAction.Update) {
-        updates[key] = op
-        delete creates[key]
-        delete deletes[key]
-      } else if (op.action === WriteOpAction.Delete) {
-        if (creates[key]) {
-          delete creates[key]
-        } else {
-          delete updates[key]
-          deletes[key] = op
-        }
-      } else {
-        throw new Error(`unknown action: ${op}`)
-      }
+export const ensureCreates = (
+  descripts: RecordWriteDescript[],
+): RecordCreateDescript[] => {
+  const creates: RecordCreateDescript[] = []
+  for (const descript of descripts) {
+    if (descript.action !== WriteOpAction.Create) {
+      throw new Error(`Unexpected action: ${descript.action}`)
+    } else {
+      creates.push(descript)
     }
   }
-  return [
-    ...Object.values(creates),
-    ...Object.values(updates),
-    ...Object.values(deletes),
-  ]
-}
-
-export const collapseDiffs = (diffs: DataDiff[]): DataDiff => {
-  return diffs.reduce((acc, cur) => {
-    acc.addDiff(cur)
-    return acc
-  }, new DataDiff())
+  return creates
 }
 
 export const parseDataKey = (key: string): RecordPath => {
@@ -267,3 +221,15 @@ export const cborToLexRecord = (val: Uint8Array): RepoRecord => {
 export const cidForRecord = async (val: LexValue) => {
   return cidForCbor(lexToIpld(val))
 }
+
+export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
+  if (commit.version === 3) {
+    return commit
+  } else {
+    return {
+      ...commit,
+      version: 3,
+      rev: commit.rev ?? TID.nextStr(),
+    }
+  }
+}

package/tests/_util.ts

@@ -7,15 +7,15 @@ import { RepoStorage } from '../src/storage'
 import { MST } from '../src/mst'
 import {
   BlockMap,
-  collapseWriteLog,
   CollectionContents,
   RecordWriteOp,
   RepoContents,
   RecordPath,
-  WriteLog,
   WriteOpAction,
   RecordClaim,
   Commit,
+  DataDiff,
+  CommitData,
 } from '../src'
 import { Keypair, randomBytes } from '@atproto/crypto'
 
@@ -109,7 +109,7 @@ export const fillRepo = async (
   }
 }
 
-export const editRepo = async (
+export const formatEdit = async (
   repo: Repo,
   prevData: RepoContents,
   keypair: crypto.Keypair,
@@ -118,91 +118,58 @@ export const editRepo = async (
     updates?: number
     deletes?: number
   },
-): Promise<{ repo: Repo; data: RepoContents }> => {
+): Promise<{ commit: CommitData; data: RepoContents }> => {
   const { adds = 0, updates = 0, deletes = 0 } = params
   const repoData: RepoContents = {}
+  const writes: RecordWriteOp[] = []
   for (const collName of testCollections) {
-    const collData = prevData[collName]
+    const collData = { ...(prevData[collName] ?? {}) }
     const shuffled = shuffle(Object.entries(collData))
 
     for (let i = 0; i < adds; i++) {
       const object = generateObject()
       const rkey = TID.nextStr()
       collData[rkey] = object
-      repo = await repo.applyWrites(
-        {
-          action: WriteOpAction.Create,
-          collection: collName,
-          rkey,
-          record: object,
-        },
-        keypair,
-      )
+      writes.push({
+        action: WriteOpAction.Create,
+        collection: collName,
+        rkey,
+        record: object,
+      })
     }
 
     const toUpdate = shuffled.slice(0, updates)
     for (let i = 0; i < toUpdate.length; i++) {
       const object = generateObject()
       const rkey = toUpdate[i][0]
-      repo = await repo.applyWrites(
-        {
-          action: WriteOpAction.Update,
-          collection: collName,
-          rkey,
-          record: object,
-        },
-        keypair,
-      )
       collData[rkey] = object
+      writes.push({
+        action: WriteOpAction.Update,
+        collection: collName,
+        rkey,
+        record: object,
+      })
     }
 
     const toDelete = shuffled.slice(updates, deletes)
     for (let i = 0; i < toDelete.length; i++) {
       const rkey = toDelete[i][0]
-      repo = await repo.applyWrites(
-        {
-          action: WriteOpAction.Delete,
-          collection: collName,
-          rkey,
-        },
-        keypair,
-      )
       delete collData[rkey]
+      writes.push({
+        action: WriteOpAction.Delete,
+        collection: collName,
+        rkey,
+      })
     }
     repoData[collName] = collData
   }
+  const commit = await repo.formatCommit(writes, keypair)
   return {
-    repo,
+    commit,
     data: repoData,
   }
 }
 
-export const verifyRepoDiff = async (
-  writeLog: WriteLog,
-  before: RepoContents,
-  after: RepoContents,
-): Promise<void> => {
-  const getVal = (op: RecordWriteOp, data: RepoContents) => {
-    return (data[op.collection] || {})[op.rkey]
-  }
-  const ops = await collapseWriteLog(writeLog)
-
-  for (const op of ops) {
-    if (op.action === WriteOpAction.Create) {
-      expect(getVal(op, before)).toBeUndefined()
-      expect(getVal(op, after)).toEqual(op.record)
-    } else if (op.action === WriteOpAction.Update) {
-      expect(getVal(op, before)).toBeDefined()
-      expect(getVal(op, after)).toEqual(op.record)
-    } else if (op.action === WriteOpAction.Delete) {
-      expect(getVal(op, before)).toBeDefined()
-      expect(getVal(op, after)).toBeUndefined()
-    } else {
-      throw new Error('unexpected op type')
-    }
-  }
-}
-
 export const contentsToClaims = (contents: RepoContents): RecordClaim[] => {
   const claims: RecordClaim[] = []
   for (const coll of Object.keys(contents)) {
@@ -233,23 +200,27 @@ export const addBadCommit = async (
   keypair: Keypair,
 ): Promise<Repo> => {
   const obj = generateObject()
-  const blocks = new BlockMap()
-  const cid = await blocks.add(obj)
+  const newBlocks = new BlockMap()
+  const cid = await newBlocks.add(obj)
   const updatedData = await repo.data.add(`com.example.test/${TID.next()}`, cid)
-  const unstoredData = await updatedData.getUnstoredBlocks()
-  blocks.addMap(unstoredData.blocks)
+  const dataCid = await updatedData.getPointer()
+  const diff = await DataDiff.of(updatedData, repo.data)
+  newBlocks.addMap(diff.newMstBlocks)
   // we generate a bad sig by signing some other data
+  const rev = TID.nextStr(repo.commit.rev)
   const commit: Commit = {
     ...repo.commit,
-    prev: repo.cid,
-    data: unstoredData.root,
+    rev,
+    data: dataCid,
     sig: await keypair.sign(randomBytes(256)),
   }
-  const commitCid = await blocks.add(commit)
+  const commitCid = await newBlocks.add(commit)
   await repo.storage.applyCommit({
-    commit: commitCid,
+    cid: commitCid,
+    rev,
     prev: repo.cid,
-    blocks: blocks,
+    newBlocks,
+    removedCids: diff.removedCids,
   })
   return await Repo.load(repo.storage, commitCid)
 }

package/tests/mst.test.ts

@@ -150,7 +150,10 @@ describe('Merkle Search Tree', () => {
       } else {
         cid = entry.value
       }
-      const found = (await blockstore.has(cid)) || diff.newCids.has(cid)
+      const found =
+        (await blockstore.has(cid)) ||
+        diff.newMstBlocks.has(cid) ||
+        diff.newLeafCids.has(cid)
       expect(found).toBeTruthy()
     }
   })

package/tests/{sync/narrow.test.ts → proofs.test.ts}

@@ -1,13 +1,12 @@
-import { TID } from '@atproto/common'
+import { TID, streamToBuffer } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
-import { RecordClaim, Repo, RepoContents } from '../../src'
-import { MemoryBlockstore } from '../../src/storage'
-import * as verify from '../../src/verify'
-import * as sync from '../../src/sync'
+import { RecordClaim, Repo, RepoContents } from '../src'
+import { MemoryBlockstore } from '../src/storage'
+import * as sync from '../src/sync'
 
-import * as util from '../_util'
+import * as util from './_util'
 
-describe('Narrow Sync', () => {
+describe('Repo Proofs', () => {
   let storage: MemoryBlockstore
   let repo: Repo
   let keypair: crypto.Keypair
@@ -25,11 +24,11 @@ describe('Narrow Sync', () => {
   })
 
   const getProofs = async (claims: RecordClaim[]) => {
-    return sync.getRecords(storage, repo.cid, claims)
+    return streamToBuffer(sync.getRecords(storage, repo.cid, claims))
   }
 
   const doVerify = (proofs: Uint8Array, claims: RecordClaim[]) => {
-    return verify.verifyProofs(proofs, claims, repoDid, keypair.did())
+    return sync.verifyProofs(proofs, claims, repoDid, keypair.did())
   }
 
   it('verifies valid records', async () => {
@@ -112,7 +111,7 @@ describe('Narrow Sync', () => {
       possible[8],
     ]
     const proofs = await getProofs(claims)
-    const records = await verify.verifyRecords(proofs, repoDid, keypair.did())
+    const records = await sync.verifyRecords(proofs, repoDid, keypair.did())
     for (const record of records) {
       const foundClaim = claims.find(
         (claim) =>
@@ -127,19 +126,13 @@ describe('Narrow Sync', () => {
     }
   })
 
-  it('verifyRecords throws on a bad signature', async () => {
-    const badRepo = await util.addBadCommit(repo, keypair)
-    const claims = util.contentsToClaims(repoData)
-    const proofs = await sync.getRecords(storage, badRepo.cid, claims)
-    const fn = verify.verifyRecords(proofs, repoDid, keypair.did())
-    await expect(fn).rejects.toThrow(verify.RepoVerificationError)
-  })
-
   it('verifyProofs throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
     const claims = util.contentsToClaims(repoData)
-    const proofs = await sync.getRecords(storage, badRepo.cid, claims)
-    const fn = verify.verifyProofs(proofs, claims, repoDid, keypair.did())
-    await expect(fn).rejects.toThrow(verify.RepoVerificationError)
+    const proofs = await streamToBuffer(
+      sync.getRecords(storage, badRepo.cid, claims),
+    )
+    const fn = sync.verifyProofs(proofs, claims, repoDid, keypair.did())
+    await expect(fn).rejects.toThrow(sync.RepoVerificationError)
   })
 })

package/tests/repo.test.ts

@@ -22,7 +22,7 @@ describe('Repo', () => {
 
   it('has proper metadata', async () => {
     expect(repo.did).toEqual(keypair.did())
-    expect(repo.version).toBe(2)
+    expect(repo.version).toBe(3)
   })
 
   it('does basic operations', async () => {
@@ -75,12 +75,13 @@ describe('Repo', () => {
   })
 
   it('edits and deletes content', async () => {
-    const edited = await util.editRepo(repo, repoData, keypair, {
+    const edit = await util.formatEdit(repo, repoData, keypair, {
       adds: 20,
       updates: 20,
       deletes: 20,
     })
-    repo = edited.repo
+    repo = await repo.applyCommit(edit.commit)
+    repoData = edit.data
     const contents = await repo.getContents()
     expect(contents).toEqual(repoData)
   })
@@ -100,6 +101,6 @@ describe('Repo', () => {
     const contents = await reloadedRepo.getContents()
     expect(contents).toEqual(repoData)
     expect(repo.did).toEqual(keypair.did())
-    expect(repo.version).toBe(2)
+    expect(repo.version).toBe(3)
   })
 })

package/tests/sync.test.ts

@@ -0,0 +1,97 @@
+import * as crypto from '@atproto/crypto'
+import {
+  CidSet,
+  Repo,
+  RepoContents,
+  RepoVerificationError,
+  readCarWithRoot,
+} from '../src'
+import { MemoryBlockstore } from '../src/storage'
+import * as sync from '../src/sync'
+
+import * as util from './_util'
+import { streamToBuffer } from '@atproto/common'
+import { CarReader } from '@ipld/car/reader'
+
+describe('Repo Sync', () => {
+  let storage: MemoryBlockstore
+  let repo: Repo
+  let keypair: crypto.Keypair
+  let repoData: RepoContents
+
+  const repoDid = 'did:example:test'
+
+  beforeAll(async () => {
+    storage = new MemoryBlockstore()
+    keypair = await crypto.Secp256k1Keypair.create()
+    repo = await Repo.create(storage, repoDid, keypair)
+    const filled = await util.fillRepo(repo, keypair, 20)
+    repo = filled.repo
+    repoData = filled.data
+  })
+
+  it('sync a full repo', async () => {
+    const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const car = await readCarWithRoot(carBytes)
+    const verified = await sync.verifyRepo(
+      car.blocks,
+      car.root,
+      repoDid,
+      keypair.did(),
+    )
+    const syncStorage = new MemoryBlockstore()
+    await syncStorage.applyCommit(verified.commit)
+    const loadedRepo = await Repo.load(syncStorage, car.root)
+    const contents = await loadedRepo.getContents()
+    expect(contents).toEqual(repoData)
+    const contentsFromOps: RepoContents = {}
+    for (const write of verified.creates) {
+      contentsFromOps[write.collection] ??= {}
+      contentsFromOps[write.collection][write.rkey] = write.record
+    }
+    expect(contentsFromOps).toEqual(repoData)
+  })
+
+  it('does not sync duplicate blocks', async () => {
+    const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const car = await CarReader.fromBytes(carBytes)
+    const cids = new CidSet()
+    for await (const block of car.blocks()) {
+      if (cids.has(block.cid)) {
+        throw new Error(`duplicate block: :${block.cid.toString()}`)
+      }
+      cids.add(block.cid)
+    }
+  })
+
+  it('syncs a repo that is behind', async () => {
+    // add more to providers's repo & have consumer catch up
+    const edit = await util.formatEdit(repo, repoData, keypair, {
+      adds: 10,
+      updates: 10,
+      deletes: 10,
+    })
+    const verified = await sync.verifyDiff(
+      repo,
+      edit.commit.newBlocks,
+      edit.commit.cid,
+      repoDid,
+      keypair.did(),
+    )
+    await storage.applyCommit(verified.commit)
+    repo = await Repo.load(storage, verified.commit.cid)
+    const contents = await repo.getContents()
+    expect(contents).toEqual(edit.data)
+  })
+
+  it('throws on a bad signature', async () => {
+    const badRepo = await util.addBadCommit(repo, keypair)
+    const carBytes = await streamToBuffer(
+      sync.getFullRepo(storage, badRepo.cid),
+    )
+    const car = await readCarWithRoot(carBytes)
+    await expect(
+      sync.verifyRepo(car.blocks, car.root, repoDid, keypair.did()),
+    ).rejects.toThrow(RepoVerificationError)
+  })
+})

package/tests/util.test.ts

@@ -0,0 +1,21 @@
+import { dataToCborBlock, wait } from '@atproto/common'
+import { writeCar } from '../src'
+
+describe('Utils', () => {
+  describe('writeCar()', () => {
+    it('propagates errors', async () => {
+      const iterate = async () => {
+        const iter = writeCar(null, async (car) => {
+          await wait(1)
+          const block = await dataToCborBlock({ test: 1 })
+          await car.put(block)
+          throw new Error('Oops!')
+        })
+        for await (const bytes of iter) {
+          // no-op
+        }
+      }
+      await expect(iterate).rejects.toThrow('Oops!')
+    })
+  })
+})