@atproto/repo 0.1.0 → 0.2.0

package/src/sync/provider.ts

@@ -1,4 +1,4 @@
-import { Commit, def, RecordPath } from '../types'
+import { def, RecordPath } from '../types'
 import { BlockWriter } from '@ipld/car/writer'
 import { CID } from 'multiformats/cid'
 import CidSet from '../cid-set'
@@ -10,10 +10,10 @@ import { MST } from '../mst'
 // Checkouts
 // -------------
 
-export const getCheckout = async (
+export const getCheckout = (
   storage: RepoStorage,
   commitCid: CID,
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return util.writeCar(commitCid, async (car: BlockWriter) => {
     const commit = await storage.readObjAndBytes(commitCid, def.commit)
     await car.put({ cid: commitCid, bytes: commit.bytes })
@@ -22,24 +22,24 @@ export const getCheckout = async (
   })
 }
 
-// Diffs
+// Commits
 // -------------
 
-export const getDiff = async (
+export const getCommits = (
   storage: RepoStorage,
   latest: CID,
   earliest: CID | null,
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return util.writeCar(latest, (car: BlockWriter) => {
     return writeCommitsToCarStream(storage, car, latest, earliest)
   })
 }
 
-export const getFullRepo = async (
+export const getFullRepo = (
   storage: RepoStorage,
   cid: CID,
-): Promise<Uint8Array> => {
-  return getDiff(storage, cid, null)
+): AsyncIterable<Uint8Array> => {
+  return getCommits(storage, cid, null)
 }
 
 export const writeCommitsToCarStream = async (
@@ -63,11 +63,11 @@ export const writeCommitsToCarStream = async (
 // Narrow slices
 // -------------
 
-export const getRecords = async (
+export const getRecords = (
   storage: RepoStorage,
   commitCid: CID,
   paths: RecordPath[],
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return util.writeCar(commitCid, async (car: BlockWriter) => {
     const commit = await storage.readObjAndBytes(commitCid, def.commit)
     await car.put({ cid: commitCid, bytes: commit.bytes })

package/src/types.ts

@@ -1,5 +1,4 @@
 import { z } from 'zod'
-import { BlockWriter } from '@ipld/car/writer'
 import { schema as common, def as commonDef } from '@atproto/common'
 import { CID } from 'multiformats'
 import BlockMap from './block-map'
@@ -101,6 +100,19 @@ export type CommitData = CommitBlockData & {
   prev: CID | null
 }
 
+export type RebaseData = {
+  commit: CID
+  rebased: CID
+  blocks: BlockMap
+  preservedCids: CID[]
+}
+
+export type CommitCidData = {
+  commit: CID
+  prev: CID | null
+  cids: CID[]
+}
+
 export type RepoUpdate = CommitData & {
   ops: RecordWriteOp[]
 }
@@ -108,6 +120,12 @@ export type RepoUpdate = CommitData & {
 export type CollectionContents = Record<string, RepoRecord>
 export type RepoContents = Record<string, CollectionContents>
 
+export type RepoRecordWithCid = { cid: CID; value: RepoRecord }
+export type CollectionContentsWithCids = Record<string, RepoRecordWithCid>
+export type RepoContentsWithCids = Record<string, CollectionContentsWithCids>
+
+export type DatastoreContents = Record<string, CID>
+
 export type RecordPath = {
   collection: string
   rkey: string
@@ -118,23 +136,3 @@ export type RecordClaim = {
   rkey: string
   record: RepoRecord | null
 }
-
-// DataStores
-// ---------------
-
-export type DataValue = {
-  key: string
-  value: CID
-}
-
-export interface DataStore {
-  add(key: string, value: CID): Promise<DataStore>
-  update(key: string, value: CID): Promise<DataStore>
-  delete(key: string): Promise<DataStore>
-  get(key: string): Promise<CID | null>
-  list(count?: number, after?: string, before?: string): Promise<DataValue[]>
-  listWithPrefix(prefix: string, count?: number): Promise<DataValue[]>
-  getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }>
-  writeToCarStream(car: BlockWriter): Promise<void>
-  cidsForPath(key: string): Promise<CID[]>
-}

package/src/util.ts

@@ -4,12 +4,13 @@ import { CarReader } from '@ipld/car/reader'
 import { BlockWriter, CarWriter } from '@ipld/car/writer'
 import { Block as CarBlock } from '@ipld/car/api'
 import {
-  streamToArray,
+  streamToBuffer,
   verifyCidForBytes,
   cborDecode,
   check,
   schema,
   cidForCbor,
+  byteIterableToStream,
 } from '@atproto/common'
 import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon'
 
@@ -20,7 +21,6 @@ import DataDiff from './data-diff'
 import { RepoStorage } from './storage'
 import {
   Commit,
-  DataStore,
   RecordCreateDescript,
   RecordDeleteDescript,
   RecordPath,
@@ -34,6 +34,7 @@ import BlockMap from './block-map'
 import { MissingBlocksError } from './error'
 import * as parse from './parse'
 import { Keypair } from '@atproto/crypto'
+import { Readable } from 'stream'
 
 export async function* verifyIncomingCarBlocks(
   car: AsyncIterable<CarBlock>,
@@ -44,25 +45,37 @@ export async function* verifyIncomingCarBlocks(
   }
 }
 
-export const writeCar = async (
+// we have to turn the car writer output into a stream in order to properly handle errors
+export function writeCarStream(
   root: CID | null,
   fn: (car: BlockWriter) => Promise<void>,
-): Promise<Uint8Array> => {
+): Readable {
   const { writer, out } =
     root !== null ? CarWriter.create(root) : CarWriter.create()
-  const bytes = streamToArray(out)
-  try {
-    await fn(writer)
-  } finally {
-    writer.close()
+
+  const stream = byteIterableToStream(out)
+  fn(writer)
+    .catch((err) => {
+      stream.destroy(err)
+    })
+    .finally(() => writer.close())
+  return stream
+}
+
+export async function* writeCar(
+  root: CID | null,
+  fn: (car: BlockWriter) => Promise<void>,
+): AsyncIterable<Uint8Array> {
+  const stream = writeCarStream(root, fn)
+  for await (const chunk of stream) {
+    yield chunk
   }
-  return bytes
 }
 
-export const blocksToCar = async (
+export const blocksToCarStream = (
   root: CID | null,
   blocks: BlockMap,
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return writeCar(root, async (writer) => {
     for (const entry of blocks.entries()) {
       await writer.put(entry)
@@ -70,6 +83,14 @@ export const blocksToCar = async (
   })
 }
 
+export const blocksToCarFile = (
+  root: CID | null,
+  blocks: BlockMap,
+): Promise<Uint8Array> => {
+  const carStream = blocksToCarStream(root, blocks)
+  return streamToBuffer(carStream)
+}
+
 export const readCar = async (
   bytes: Uint8Array,
 ): Promise<{ roots: CID[]; blocks: BlockMap }> => {
@@ -108,7 +129,7 @@ export const getWriteLog = async (
   if (!commits) throw new Error('Could not find shared history')
   const heads = await Promise.all(commits.map((c) => Repo.load(storage, c)))
   // Turn commit path into list of diffs
-  let prev: DataStore = await MST.create(storage) // Empty
+  let prev = await MST.create(storage) // Empty
   const msts = heads.map((h) => h.data)
   const diffs: DataDiff[] = []
   for (const mst of msts) {

package/src/verify.ts

@@ -6,7 +6,7 @@ import ReadableRepo from './readable-repo'
 import Repo from './repo'
 import CidSet from './cid-set'
 import * as util from './util'
-import { RecordClaim, RepoContents } from './types'
+import { RecordClaim, RepoContents, RepoContentsWithCids } from './types'
 import { def } from './types'
 import { MST } from './mst'
 import { cidForCbor } from '@atproto/common'
@@ -16,22 +16,18 @@ export type VerifiedCheckout = {
   newCids: CidSet
 }
 
+export type VerifiedCheckoutWithCids = {
+  commit: CID
+  contents: RepoContentsWithCids
+}
+
 export const verifyCheckout = async (
   storage: ReadableBlockstore,
   head: CID,
   did: string,
   signingKey: string,
 ): Promise<VerifiedCheckout> => {
-  const repo = await ReadableRepo.load(storage, head)
-  if (repo.did !== did) {
-    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
-  }
-  const validSig = await util.verifyCommitSig(repo.commit, signingKey)
-  if (!validSig) {
-    throw new RepoVerificationError(
-      `Invalid signature on commit: ${repo.cid.toString()}`,
-    )
-  }
+  const repo = await verifyRepoRoot(storage, head, did, signingKey)
   const diff = await DataDiff.of(repo.data, null)
   const newCids = new CidSet([repo.cid]).addSet(diff.newCids)
 
@@ -51,6 +47,51 @@ export const verifyCheckout = async (
   }
 }
 
+export const verifyCheckoutWithCids = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedCheckoutWithCids> => {
+  const repo = await verifyRepoRoot(storage, head, did, signingKey)
+  const diff = await DataDiff.of(repo.data, null)
+
+  const contents: RepoContentsWithCids = {}
+  for (const add of diff.addList()) {
+    const { collection, rkey } = util.parseDataKey(add.key)
+    contents[collection] ??= {}
+    contents[collection][rkey] = {
+      cid: add.cid,
+      value: await storage.readRecord(add.cid),
+    }
+  }
+
+  return {
+    commit: repo.cid,
+    contents,
+  }
+}
+
+// @NOTE only verifies the root, not the repo contents
+const verifyRepoRoot = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<ReadableRepo> => {
+  const repo = await ReadableRepo.load(storage, head)
+  if (repo.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
+  }
+  const validSig = await util.verifyCommitSig(repo.commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${repo.cid.toString()}`,
+    )
+  }
+  return repo
+}
+
 export type VerifiedUpdate = {
   commit: CID
   prev: CID | null

package/tests/rebase.test.ts

@@ -0,0 +1,37 @@
+import * as crypto from '@atproto/crypto'
+import { Repo } from '../src/repo'
+import { MemoryBlockstore } from '../src/storage'
+import * as util from './_util'
+import { Secp256k1Keypair } from '@atproto/crypto'
+
+describe('Rebases', () => {
+  let storage: MemoryBlockstore
+  let keypair: crypto.Keypair
+  let repo: Repo
+
+  it('fills a repo with data', async () => {
+    storage = new MemoryBlockstore()
+    keypair = await Secp256k1Keypair.create()
+    repo = await Repo.create(storage, keypair.did(), keypair)
+    const filled = await util.fillRepo(repo, keypair, 100)
+    repo = filled.repo
+  })
+
+  it('rebases the repo & preserves contents', async () => {
+    const dataCidBefore = await repo.data.getPointer()
+    const contents = await repo.getContents()
+    repo = await repo.rebase(keypair)
+    const rebasedContents = await repo.getContents()
+    expect(rebasedContents).toEqual(contents)
+    const dataCidAfter = await repo.data.getPointer()
+    expect(dataCidAfter.equals(dataCidBefore)).toBeTruthy()
+  })
+
+  it('only keeps around relevant cids', async () => {
+    const allCids = await repo.data.allCids()
+    allCids.add(repo.cid)
+    for (const cid of storage.blocks.cids()) {
+      expect(allCids.has(cid)).toBeTruthy()
+    }
+  })
+})

package/tests/sync/checkout.test.ts

@@ -1,9 +1,11 @@
 import * as crypto from '@atproto/crypto'
-import { Repo, RepoContents, RepoVerificationError } from '../../src'
+import { CidSet, Repo, RepoContents, RepoVerificationError } from '../../src'
 import { MemoryBlockstore } from '../../src/storage'
 import * as sync from '../../src/sync'
 
 import * as util from '../_util'
+import { streamToBuffer } from '@atproto/common'
+import { CarReader } from '@ipld/car/reader'
 
 describe('Checkout Sync', () => {
   let storage: MemoryBlockstore
@@ -25,7 +27,9 @@ describe('Checkout Sync', () => {
   })
 
   it('sync a non-historical repo checkout', async () => {
-    const checkoutCar = await sync.getCheckout(storage, repo.cid)
+    const checkoutCar = await streamToBuffer(
+      sync.getCheckout(storage, repo.cid),
+    )
     const checkout = await sync.loadCheckout(
       syncStorage,
       checkoutCar,
@@ -47,9 +51,23 @@ describe('Checkout Sync', () => {
     expect(hasGenesisCommit).toBeFalsy()
   })
 
+  it('does not sync duplicate blocks', async () => {
+    const carBytes = await streamToBuffer(sync.getCheckout(storage, repo.cid))
+    const car = await CarReader.fromBytes(carBytes)
+    const cids = new CidSet()
+    for await (const block of car.blocks()) {
+      if (cids.has(block.cid)) {
+        throw new Error(`duplicate block: :${block.cid.toString()}`)
+      }
+      cids.add(block.cid)
+    }
+  })
+
   it('throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
-    const checkoutCar = await sync.getCheckout(storage, badRepo.cid)
+    const checkoutCar = await streamToBuffer(
+      sync.getCheckout(storage, badRepo.cid),
+    )
     await expect(
       sync.loadCheckout(syncStorage, checkoutCar, repoDid, keypair.did()),
     ).rejects.toThrow(RepoVerificationError)

package/tests/sync/diff.test.ts

@@ -4,6 +4,7 @@ import { MemoryBlockstore } from '../../src/storage'
 import * as sync from '../../src/sync'
 
 import * as util from '../_util'
+import { streamToBuffer } from '@atproto/common'
 
 describe('Diff Sync', () => {
   let storage: MemoryBlockstore
@@ -24,7 +25,7 @@ describe('Diff Sync', () => {
   let syncRepo: Repo
 
   it('syncs an empty repo', async () => {
-    const car = await sync.getFullRepo(storage, repo.cid)
+    const car = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
     const loaded = await sync.loadFullRepo(
       syncStorage,
       car,
@@ -41,7 +42,7 @@ describe('Diff Sync', () => {
     repo = filled.repo
     repoData = filled.data
 
-    const car = await sync.getFullRepo(storage, repo.cid)
+    const car = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
     const loaded = await sync.loadFullRepo(
       syncStorage,
       car,
@@ -64,7 +65,9 @@ describe('Diff Sync', () => {
     })
     repo = edited.repo
     repoData = edited.data
-    const diffCar = await sync.getDiff(storage, repo.cid, syncRepo.cid)
+    const diffCar = await streamToBuffer(
+      sync.getCommits(storage, repo.cid, syncRepo.cid),
+    )
     const loaded = await sync.loadDiff(
       syncRepo,
       diffCar,
@@ -79,7 +82,9 @@ describe('Diff Sync', () => {
 
   it('throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
-    const diffCar = await sync.getDiff(storage, badRepo.cid, syncRepo.cid)
+    const diffCar = await streamToBuffer(
+      sync.getCommits(storage, badRepo.cid, syncRepo.cid),
+    )
     await expect(
       sync.loadDiff(syncRepo, diffCar, repoDid, keypair.did()),
     ).rejects.toThrow('Invalid signature on commit')

package/tests/sync/narrow.test.ts

@@ -1,4 +1,4 @@
-import { TID } from '@atproto/common'
+import { TID, streamToBuffer } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import { RecordClaim, Repo, RepoContents } from '../../src'
 import { MemoryBlockstore } from '../../src/storage'
@@ -25,7 +25,7 @@ describe('Narrow Sync', () => {
   })
 
   const getProofs = async (claims: RecordClaim[]) => {
-    return sync.getRecords(storage, repo.cid, claims)
+    return streamToBuffer(sync.getRecords(storage, repo.cid, claims))
   }
 
   const doVerify = (proofs: Uint8Array, claims: RecordClaim[]) => {
@@ -130,7 +130,9 @@ describe('Narrow Sync', () => {
   it('verifyRecords throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
     const claims = util.contentsToClaims(repoData)
-    const proofs = await sync.getRecords(storage, badRepo.cid, claims)
+    const proofs = await streamToBuffer(
+      sync.getRecords(storage, badRepo.cid, claims),
+    )
     const fn = verify.verifyRecords(proofs, repoDid, keypair.did())
     await expect(fn).rejects.toThrow(verify.RepoVerificationError)
   })
@@ -138,7 +140,9 @@ describe('Narrow Sync', () => {
   it('verifyProofs throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
     const claims = util.contentsToClaims(repoData)
-    const proofs = await sync.getRecords(storage, badRepo.cid, claims)
+    const proofs = await streamToBuffer(
+      sync.getRecords(storage, badRepo.cid, claims),
+    )
     const fn = verify.verifyProofs(proofs, claims, repoDid, keypair.did())
     await expect(fn).rejects.toThrow(verify.RepoVerificationError)
   })

package/tests/util.test.ts

@@ -0,0 +1,21 @@
+import { dataToCborBlock, wait } from '@atproto/common'
+import { writeCar } from '../src'
+
+describe('Utils', () => {
+  describe('writeCar()', () => {
+    it('propagates errors', async () => {
+      const iterate = async () => {
+        const iter = writeCar(null, async (car) => {
+          await wait(1)
+          const block = await dataToCborBlock({ test: 1 })
+          await car.put(block)
+          throw new Error('Oops!')
+        })
+        for await (const bytes of iter) {
+          // no-op
+        }
+      }
+      await expect(iterate).rejects.toThrow('Oops!')
+    })
+  })
+})