@atproto/repo 0.1.0 → 0.3.0

package/src/storage/memory-blockstore.ts

@@ -1,14 +1,15 @@
 import { CID } from 'multiformats/cid'
-import { CommitData, def } from '../types'
+import { CommitData } from '../types'
 import BlockMap from '../block-map'
-import { MST } from '../mst'
-import DataDiff from '../data-diff'
-import { MissingCommitBlocksError } from '../error'
-import RepoStorage from './repo-storage'
+import ReadableBlockstore from './readable-blockstore'
+import { RepoStorage } from './types'
 
-export class MemoryBlockstore extends RepoStorage {
+export class MemoryBlockstore
+  extends ReadableBlockstore
+  implements RepoStorage
+{
   blocks: BlockMap
-  head: CID | null = null
+  root: CID | null = null
 
   constructor(blocks?: BlockMap) {
     super()
@@ -18,8 +19,8 @@ export class MemoryBlockstore extends RepoStorage {
     }
   }
 
-  async getHead(): Promise<CID | null> {
-    return this.head
+  async getRoot(): Promise<CID | null> {
+    return this.root
   }
 
   async getBytes(cid: CID): Promise<Uint8Array | null> {
@@ -42,60 +43,19 @@ export class MemoryBlockstore extends RepoStorage {
     this.blocks.addMap(blocks)
   }
 
-  async indexCommits(commits: CommitData[]): Promise<void> {
-    commits.forEach((commit) => {
-      this.blocks.addMap(commit.blocks)
-    })
-  }
-
-  async updateHead(cid: CID, _prev: CID | null): Promise<void> {
-    this.head = cid
+  async updateRoot(cid: CID): Promise<void> {
+    this.root = cid
   }
 
   async applyCommit(commit: CommitData): Promise<void> {
-    this.blocks.addMap(commit.blocks)
-    this.head = commit.commit
-  }
-
-  async getCommitPath(
-    latest: CID,
-    earliest: CID | null,
-  ): Promise<CID[] | null> {
-    let curr: CID | null = latest
-    const path: CID[] = []
-    while (curr !== null) {
-      path.push(curr)
-      const commit = await this.readObj(curr, def.commit)
-      if (!earliest && commit.prev === null) {
-        return path.reverse()
-      } else if (earliest && commit.prev.equals(earliest)) {
-        return path.reverse()
-      }
-      curr = commit.prev
+    this.root = commit.cid
+    const rmCids = commit.removedCids.toList()
+    for (const cid of rmCids) {
+      this.blocks.delete(cid)
     }
-    return null
-  }
-
-  async getBlocksForCommits(
-    commits: CID[],
-  ): Promise<{ [commit: string]: BlockMap }> {
-    const commitData: { [commit: string]: BlockMap } = {}
-    let prevData: MST | null = null
-    for (const commitCid of commits) {
-      const commit = await this.readObj(commitCid, def.commit)
-      const data = await MST.load(this, commit.data)
-      const diff = await DataDiff.of(data, prevData)
-      const { blocks, missing } = await this.getBlocks([
-        commitCid,
-        ...diff.newCidList(),
-      ])
-      if (missing.length > 0) {
-        throw new MissingCommitBlocksError(commitCid, missing)
-      }
-      commitData[commitCid.toString()] = blocks
-      prevData = data
-    }
-    return commitData
+    commit.newBlocks.forEach((bytes, cid) => {
+      this.blocks.set(cid, bytes)
+    })
   }
 
   async sizeInBytes(): Promise<number> {

package/src/storage/types.ts

@@ -1,5 +1,34 @@
 import stream from 'stream'
 import { CID } from 'multiformats/cid'
+import { RepoRecord } from '@atproto/lexicon'
+import { check } from '@atproto/common'
+import BlockMap from '../block-map'
+import { CommitData } from '../types'
+
+export interface RepoStorage {
+  // Writable
+  getRoot(): Promise<CID | null>
+  putBlock(cid: CID, block: Uint8Array, rev: string): Promise<void>
+  putMany(blocks: BlockMap, rev: string): Promise<void>
+  updateRoot(cid: CID): Promise<void>
+  applyCommit(commit: CommitData)
+
+  // Readable
+  getBytes(cid: CID): Promise<Uint8Array | null>
+  has(cid: CID): Promise<boolean>
+  getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+  attemptRead<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array } | null>
+  readObjAndBytes<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array }>
+  readObj<T>(cid: CID, def: check.Def<T>): Promise<T>
+  attemptReadRecord(cid: CID): Promise<RepoRecord | null>
+  readRecord(cid: CID): Promise<RepoRecord>
+}
 
 export interface BlobStore {
   putTemp(bytes: Uint8Array | stream.Readable): Promise<string>
@@ -9,6 +38,7 @@ export interface BlobStore {
   unquarantine(cid: CID): Promise<void>
   getBytes(cid: CID): Promise<Uint8Array>
   getStream(cid: CID): Promise<stream.Readable>
+  hasStored(cid: CID): Promise<boolean>
   delete(cid: CID): Promise<void>
 }
 

package/src/sync/consumer.ts

@@ -1,137 +1,194 @@
 import { CID } from 'multiformats/cid'
-import { MemoryBlockstore, RepoStorage } from '../storage'
-import Repo from '../repo'
-import * as verify from '../verify'
+import { MemoryBlockstore, ReadableBlockstore, SyncStorage } from '../storage'
+import DataDiff from '../data-diff'
+import ReadableRepo from '../readable-repo'
 import * as util from '../util'
-import { CommitData, RepoContents, WriteLog } from '../types'
-import CidSet from '../cid-set'
-import { MissingBlocksError } from '../error'
-
-// Checkouts
-// -------------
-
-export const loadCheckout = async (
-  storage: RepoStorage,
-  repoCar: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<{ root: CID; contents: RepoContents }> => {
-  const { root, blocks } = await util.readCarWithRoot(repoCar)
-  const updateStorage = new MemoryBlockstore(blocks)
-  const checkout = await verify.verifyCheckout(
-    updateStorage,
-    root,
-    did,
-    signingKey,
-  )
-
-  const checkoutBlocks = await updateStorage.getBlocks(
-    checkout.newCids.toList(),
-  )
-  if (checkoutBlocks.missing.length > 0) {
-    throw new MissingBlocksError('sync', checkoutBlocks.missing)
-  }
-  await Promise.all([
-    storage.putMany(checkoutBlocks.blocks),
-    storage.updateHead(root, null),
-  ])
+import { RecordClaim, VerifiedDiff, VerifiedRepo } from '../types'
+import { def } from '../types'
+import { MST } from '../mst'
+import { cidForCbor } from '@atproto/common'
+import BlockMap from '../block-map'
+
+export const verifyRepoCar = async (
+  carBytes: Uint8Array,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedRepo> => {
+  const car = await util.readCarWithRoot(carBytes)
+  return verifyRepo(car.blocks, car.root, did, signingKey)
+}
 
+export const verifyRepo = async (
+  blocks: BlockMap,
+  head: CID,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedRepo> => {
+  const diff = await verifyDiff(null, blocks, head, did, signingKey)
+  const creates = util.ensureCreates(diff.writes)
   return {
-    root,
-    contents: checkout.contents,
+    creates,
+    commit: diff.commit,
   }
 }
 
-// Diffs
-// -------------
+export const verifyDiffCar = async (
+  repo: ReadableRepo | null,
+  carBytes: Uint8Array,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedDiff> => {
+  const car = await util.readCarWithRoot(carBytes)
+  return verifyDiff(repo, car.blocks, car.root, did, signingKey)
+}
 
-export const loadFullRepo = async (
-  storage: RepoStorage,
-  repoCar: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<{ root: CID; writeLog: WriteLog }> => {
-  const { root, blocks } = await util.readCarWithRoot(repoCar)
-  const updateStorage = new MemoryBlockstore(blocks)
-  const updates = await verify.verifyFullHistory(
+export const verifyDiff = async (
+  repo: ReadableRepo | null,
+  updateBlocks: BlockMap,
+  updateRoot: CID,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedDiff> => {
+  const stagedStorage = new MemoryBlockstore(updateBlocks)
+  const updateStorage = repo
+    ? new SyncStorage(stagedStorage, repo.storage)
+    : stagedStorage
+  const updated = await verifyRepoRoot(
     updateStorage,
-    root,
+    updateRoot,
     did,
     signingKey,
   )
-
-  const [writeLog] = await Promise.all([
-    persistUpdates(storage, updateStorage, updates),
-    storage.updateHead(root, null),
-  ])
-
+  const diff = await DataDiff.of(updated.data, repo?.data ?? null)
+  const writes = await util.diffToWriteDescripts(diff, updateBlocks)
+  const newBlocks = diff.newMstBlocks
+  const leaves = updateBlocks.getMany(diff.newLeafCids.toList())
+  if (leaves.missing.length > 0) {
+    throw new Error(`missing leaf blocks: ${leaves.missing}`)
+  }
+  newBlocks.addMap(leaves.blocks)
+  const removedCids = diff.removedCids
+  const commitCid = await newBlocks.add(updated.commit)
+  // ensure the commit cid actually changed
+  if (repo) {
+    if (commitCid.equals(repo.cid)) {
+      newBlocks.delete(commitCid)
+    } else {
+      removedCids.add(repo.cid)
+    }
+  }
   return {
-    root,
-    writeLog,
+    writes,
+    commit: {
+      cid: updated.cid,
+      rev: updated.commit.rev,
+      prev: repo?.cid ?? null,
+      since: repo?.commit.rev ?? null,
+      newBlocks,
+      removedCids,
+    },
   }
 }
 
-export const loadDiff = async (
-  repo: Repo,
-  diffCar: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<{ root: CID; writeLog: WriteLog }> => {
-  const { root, blocks } = await util.readCarWithRoot(diffCar)
-  const updateStorage = new MemoryBlockstore(blocks)
-  const updates = await verify.verifyUpdates(
-    repo,
-    updateStorage,
-    root,
-    did,
-    signingKey,
-  )
-
-  const [writeLog] = await Promise.all([
-    persistUpdates(repo.storage, updateStorage, updates),
-    repo.storage.updateHead(root, repo.cid),
-  ])
-
-  return {
-    root,
-    writeLog,
+// @NOTE only verifies the root, not the repo contents
+const verifyRepoRoot = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did?: string,
+  signingKey?: string,
+): Promise<ReadableRepo> => {
+  const repo = await ReadableRepo.load(storage, head)
+  if (did !== undefined && repo.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
   }
+  if (signingKey !== undefined) {
+    const validSig = await util.verifyCommitSig(repo.commit, signingKey)
+    if (!validSig) {
+      throw new RepoVerificationError(
+        `Invalid signature on commit: ${repo.cid.toString()}`,
+      )
+    }
+  }
+  return repo
 }
 
-// Helpers
-// -------------
-
-export const persistUpdates = async (
-  storage: RepoStorage,
-  updateStorage: RepoStorage,
-  updates: verify.VerifiedUpdate[],
-): Promise<WriteLog> => {
-  const newCids = new CidSet()
-  for (const update of updates) {
-    newCids.addSet(update.newCids)
+export const verifyProofs = async (
+  proofs: Uint8Array,
+  claims: RecordClaim[],
+  did: string,
+  didKey: string,
+): Promise<{ verified: RecordClaim[]; unverified: RecordClaim[] }> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
   }
-
-  const diffBlocks = await updateStorage.getBlocks(newCids.toList())
-  if (diffBlocks.missing.length > 0) {
-    throw new MissingBlocksError('sync', diffBlocks.missing)
+  const validSig = await util.verifyCommitSig(commit, didKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
+    )
   }
-  const commits: CommitData[] = updates.map((update) => {
-    const forCommit = diffBlocks.blocks.getMany(update.newCids.toList())
-    if (forCommit.missing.length > 0) {
-      throw new MissingBlocksError('sync', forCommit.missing)
-    }
-    return {
-      commit: update.commit,
-      prev: update.prev,
-      blocks: forCommit.blocks,
+  const mst = MST.load(blockstore, commit.data)
+  const verified: RecordClaim[] = []
+  const unverified: RecordClaim[] = []
+  for (const claim of claims) {
+    const found = await mst.get(
+      util.formatDataKey(claim.collection, claim.rkey),
+    )
+    const record = found ? await blockstore.readObj(found, def.map) : null
+    if (claim.record === null) {
+      if (record === null) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
+      }
+    } else {
+      const expected = await cidForCbor(claim.record)
+      if (expected.equals(found)) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
+      }
     }
-  })
-
-  await storage.indexCommits(commits)
+  }
+  return { verified, unverified }
+}
 
-  return Promise.all(
-    updates.map((upd) =>
-      util.diffToWriteDescripts(upd.diff, diffBlocks.blocks),
-    ),
-  )
+export const verifyRecords = async (
+  proofs: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<RecordClaim[]> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
+  }
+  const validSig = await util.verifyCommitSig(commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
+    )
+  }
+  const mst = MST.load(blockstore, commit.data)
+
+  const records: RecordClaim[] = []
+  const leaves = await mst.reachableLeaves()
+  for (const leaf of leaves) {
+    const { collection, rkey } = util.parseDataKey(leaf.key)
+    const record = await blockstore.attemptReadRecord(leaf.value)
+    if (record) {
+      records.push({
+        collection,
+        rkey,
+        record,
+      })
+    }
+  }
+  return records
 }
+
+export class RepoVerificationError extends Error {}

package/src/sync/provider.ts

@@ -1,4 +1,4 @@
-import { Commit, def, RecordPath } from '../types'
+import { def, RecordPath } from '../types'
 import { BlockWriter } from '@ipld/car/writer'
 import { CID } from 'multiformats/cid'
 import CidSet from '../cid-set'
@@ -7,13 +7,13 @@ import { RepoStorage } from '../storage'
 import * as util from '../util'
 import { MST } from '../mst'
 
-// Checkouts
+// Full Repo
 // -------------
 
-export const getCheckout = async (
+export const getFullRepo = (
   storage: RepoStorage,
   commitCid: CID,
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return util.writeCar(commitCid, async (car: BlockWriter) => {
     const commit = await storage.readObjAndBytes(commitCid, def.commit)
     await car.put({ cid: commitCid, bytes: commit.bytes })
@@ -22,52 +22,14 @@ export const getCheckout = async (
   })
 }
 
-// Diffs
-// -------------
-
-export const getDiff = async (
-  storage: RepoStorage,
-  latest: CID,
-  earliest: CID | null,
-): Promise<Uint8Array> => {
-  return util.writeCar(latest, (car: BlockWriter) => {
-    return writeCommitsToCarStream(storage, car, latest, earliest)
-  })
-}
-
-export const getFullRepo = async (
-  storage: RepoStorage,
-  cid: CID,
-): Promise<Uint8Array> => {
-  return getDiff(storage, cid, null)
-}
-
-export const writeCommitsToCarStream = async (
-  storage: RepoStorage,
-  car: BlockWriter,
-  latest: CID,
-  earliest: CID | null,
-): Promise<void> => {
-  const commits = await storage.getCommits(latest, earliest)
-  if (commits === null) {
-    throw new Error('Could not find shared history')
-  }
-  if (commits.length === 0) return
-  for (const commit of commits) {
-    for (const entry of commit.blocks.entries()) {
-      await car.put(entry)
-    }
-  }
-}
-
 // Narrow slices
 // -------------
 
-export const getRecords = async (
+export const getRecords = (
   storage: RepoStorage,
   commitCid: CID,
   paths: RecordPath[],
-): Promise<Uint8Array> => {
+): AsyncIterable<Uint8Array> => {
   return util.writeCar(commitCid, async (car: BlockWriter) => {
     const commit = await storage.readObjAndBytes(commitCid, def.commit)
     await car.put({ cid: commitCid, bytes: commit.bytes })

package/src/types.ts

@@ -1,33 +1,54 @@
 import { z } from 'zod'
-import { BlockWriter } from '@ipld/car/writer'
 import { schema as common, def as commonDef } from '@atproto/common'
 import { CID } from 'multiformats'
 import BlockMap from './block-map'
 import { RepoRecord } from '@atproto/lexicon'
+import CidSet from './cid-set'
 
 // Repo nodes
 // ---------------
 
 const unsignedCommit = z.object({
   did: z.string(),
-  version: z.number(),
-  prev: common.cid.nullable(),
+  version: z.literal(3),
   data: common.cid,
+  rev: z.string(),
+  // `prev` added for backwards compatibility with v2, no requirement of keeping around history
+  prev: common.cid.nullable().optional(),
 })
 export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
 const commit = z.object({
   did: z.string(),
-  version: z.number(),
-  prev: common.cid.nullable(),
+  version: z.literal(3),
   data: common.cid,
+  rev: z.string(),
+  prev: common.cid.nullable().optional(),
   sig: common.bytes,
 })
 export type Commit = z.infer<typeof commit>
 
+const legacyV2Commit = z.object({
+  did: z.string(),
+  version: z.literal(2),
+  data: common.cid,
+  rev: z.string().optional(),
+  prev: common.cid.nullable(),
+  sig: common.bytes,
+})
+export type LegacyV2Commit = z.infer<typeof legacyV2Commit>
+
+const versionedCommit = z.discriminatedUnion('version', [
+  commit,
+  legacyV2Commit,
+])
+export type VersionedCommit = z.infer<typeof versionedCommit>
+
 export const schema = {
   ...common,
   commit,
+  legacyV2Commit,
+  versionedCommit,
 }
 
 export const def = {
@@ -36,6 +57,10 @@ export const def = {
     name: 'commit',
     schema: schema.commit,
   },
+  versionedCommit: {
+    name: 'versioned_commit',
+    schema: schema.versionedCommit,
+  },
 }
 
 // Repo Operations
@@ -92,13 +117,13 @@ export type WriteLog = RecordWriteDescript[][]
 // Updates/Commits
 // ---------------
 
-export type CommitBlockData = {
-  commit: CID
-  blocks: BlockMap
-}
-
-export type CommitData = CommitBlockData & {
+export type CommitData = {
+  cid: CID
+  rev: string
+  since: string | null
   prev: CID | null
+  newBlocks: BlockMap
+  removedCids: CidSet
 }
 
 export type RepoUpdate = CommitData & {
@@ -108,6 +133,12 @@ export type RepoUpdate = CommitData & {
 export type CollectionContents = Record<string, RepoRecord>
 export type RepoContents = Record<string, CollectionContents>
 
+export type RepoRecordWithCid = { cid: CID; value: RepoRecord }
+export type CollectionContentsWithCids = Record<string, RepoRecordWithCid>
+export type RepoContentsWithCids = Record<string, CollectionContentsWithCids>
+
+export type DatastoreContents = Record<string, CID>
+
 export type RecordPath = {
   collection: string
   rkey: string
@@ -119,22 +150,15 @@ export type RecordClaim = {
   record: RepoRecord | null
 }
 
-// DataStores
+// Sync
 // ---------------
 
-export type DataValue = {
-  key: string
-  value: CID
+export type VerifiedDiff = {
+  writes: RecordWriteDescript[]
+  commit: CommitData
 }
 
-export interface DataStore {
-  add(key: string, value: CID): Promise<DataStore>
-  update(key: string, value: CID): Promise<DataStore>
-  delete(key: string): Promise<DataStore>
-  get(key: string): Promise<CID | null>
-  list(count?: number, after?: string, before?: string): Promise<DataValue[]>
-  listWithPrefix(prefix: string, count?: number): Promise<DataValue[]>
-  getUnstoredBlocks(): Promise<{ root: CID; blocks: BlockMap }>
-  writeToCarStream(car: BlockWriter): Promise<void>
-  cidsForPath(key: string): Promise<CID[]>
+export type VerifiedRepo = {
+  creates: RecordCreateDescript[]
+  commit: CommitData
 }