@atproto/repo 0.2.0 → 0.3.1

package/src/storage/types.ts

@@ -1,5 +1,34 @@
 import stream from 'stream'
 import { CID } from 'multiformats/cid'
+import { RepoRecord } from '@atproto/lexicon'
+import { check } from '@atproto/common'
+import BlockMap from '../block-map'
+import { CommitData } from '../types'
+
+export interface RepoStorage {
+  // Writable
+  getRoot(): Promise<CID | null>
+  putBlock(cid: CID, block: Uint8Array, rev: string): Promise<void>
+  putMany(blocks: BlockMap, rev: string): Promise<void>
+  updateRoot(cid: CID): Promise<void>
+  applyCommit(commit: CommitData)
+
+  // Readable
+  getBytes(cid: CID): Promise<Uint8Array | null>
+  has(cid: CID): Promise<boolean>
+  getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+  attemptRead<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array } | null>
+  readObjAndBytes<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array }>
+  readObj<T>(cid: CID, def: check.Def<T>): Promise<T>
+  attemptReadRecord(cid: CID): Promise<RepoRecord | null>
+  readRecord(cid: CID): Promise<RepoRecord>
+}
 
 export interface BlobStore {
   putTemp(bytes: Uint8Array | stream.Readable): Promise<string>

package/src/sync/consumer.ts

@@ -1,140 +1,194 @@
 import { CID } from 'multiformats/cid'
-import { MemoryBlockstore, RepoStorage } from '../storage'
-import Repo from '../repo'
-import * as verify from '../verify'
+import { MemoryBlockstore, ReadableBlockstore, SyncStorage } from '../storage'
+import DataDiff from '../data-diff'
+import ReadableRepo from '../readable-repo'
 import * as util from '../util'
-import { CommitData, RepoContents, WriteLog } from '../types'
-import CidSet from '../cid-set'
-import { MissingBlocksError } from '../error'
-
-// Checkouts
-// -------------
-
-export const loadCheckout = async (
-  storage: RepoStorage,
-  repoCar: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<{ root: CID; contents: RepoContents }> => {
-  const { root, blocks } = await util.readCarWithRoot(repoCar)
-  const updateStorage = new MemoryBlockstore(blocks)
-  const checkout = await verify.verifyCheckout(
-    updateStorage,
-    root,
-    did,
-    signingKey,
-  )
-
-  const checkoutBlocks = await updateStorage.getBlocks(
-    checkout.newCids.toList(),
-  )
-  if (checkoutBlocks.missing.length > 0) {
-    throw new MissingBlocksError('sync', checkoutBlocks.missing)
-  }
-  await Promise.all([
-    storage.putMany(checkoutBlocks.blocks),
-    storage.updateHead(root, null),
-  ])
+import { RecordClaim, VerifiedDiff, VerifiedRepo } from '../types'
+import { def } from '../types'
+import { MST } from '../mst'
+import { cidForCbor } from '@atproto/common'
+import BlockMap from '../block-map'
+
+export const verifyRepoCar = async (
+  carBytes: Uint8Array,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedRepo> => {
+  const car = await util.readCarWithRoot(carBytes)
+  return verifyRepo(car.blocks, car.root, did, signingKey)
+}
 
+export const verifyRepo = async (
+  blocks: BlockMap,
+  head: CID,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedRepo> => {
+  const diff = await verifyDiff(null, blocks, head, did, signingKey)
+  const creates = util.ensureCreates(diff.writes)
   return {
-    root,
-    contents: checkout.contents,
+    creates,
+    commit: diff.commit,
   }
 }
 
-// Diffs
-// -------------
+export const verifyDiffCar = async (
+  repo: ReadableRepo | null,
+  carBytes: Uint8Array,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedDiff> => {
+  const car = await util.readCarWithRoot(carBytes)
+  return verifyDiff(repo, car.blocks, car.root, did, signingKey)
+}
 
-export const loadFullRepo = async (
-  storage: RepoStorage,
-  repoCar: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<{ root: CID; writeLog: WriteLog; repo: Repo }> => {
-  const { root, blocks } = await util.readCarWithRoot(repoCar)
-  const updateStorage = new MemoryBlockstore(blocks)
-  const updates = await verify.verifyFullHistory(
+export const verifyDiff = async (
+  repo: ReadableRepo | null,
+  updateBlocks: BlockMap,
+  updateRoot: CID,
+  did?: string,
+  signingKey?: string,
+): Promise<VerifiedDiff> => {
+  const stagedStorage = new MemoryBlockstore(updateBlocks)
+  const updateStorage = repo
+    ? new SyncStorage(stagedStorage, repo.storage)
+    : stagedStorage
+  const updated = await verifyRepoRoot(
     updateStorage,
-    root,
+    updateRoot,
     did,
     signingKey,
   )
-
-  const [writeLog] = await Promise.all([
-    persistUpdates(storage, updateStorage, updates),
-    storage.updateHead(root, null),
-  ])
-
-  const repo = await Repo.load(storage, root)
-
+  const diff = await DataDiff.of(updated.data, repo?.data ?? null)
+  const writes = await util.diffToWriteDescripts(diff, updateBlocks)
+  const newBlocks = diff.newMstBlocks
+  const leaves = updateBlocks.getMany(diff.newLeafCids.toList())
+  if (leaves.missing.length > 0) {
+    throw new Error(`missing leaf blocks: ${leaves.missing}`)
+  }
+  newBlocks.addMap(leaves.blocks)
+  const removedCids = diff.removedCids
+  const commitCid = await newBlocks.add(updated.commit)
+  // ensure the commit cid actually changed
+  if (repo) {
+    if (commitCid.equals(repo.cid)) {
+      newBlocks.delete(commitCid)
+    } else {
+      removedCids.add(repo.cid)
+    }
+  }
   return {
-    root,
-    writeLog,
-    repo,
+    writes,
+    commit: {
+      cid: updated.cid,
+      rev: updated.commit.rev,
+      prev: repo?.cid ?? null,
+      since: repo?.commit.rev ?? null,
+      newBlocks,
+      removedCids,
+    },
   }
 }
 
-export const loadDiff = async (
-  repo: Repo,
-  diffCar: Uint8Array,
-  did: string,
-  signingKey: string,
-): Promise<{ root: CID; writeLog: WriteLog }> => {
-  const { root, blocks } = await util.readCarWithRoot(diffCar)
-  const updateStorage = new MemoryBlockstore(blocks)
-  const updates = await verify.verifyUpdates(
-    repo,
-    updateStorage,
-    root,
-    did,
-    signingKey,
-  )
-
-  const [writeLog] = await Promise.all([
-    persistUpdates(repo.storage, updateStorage, updates),
-    repo.storage.updateHead(root, repo.cid),
-  ])
-
-  return {
-    root,
-    writeLog,
+// @NOTE only verifies the root, not the repo contents
+const verifyRepoRoot = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did?: string,
+  signingKey?: string,
+): Promise<ReadableRepo> => {
+  const repo = await ReadableRepo.load(storage, head)
+  if (did !== undefined && repo.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
   }
+  if (signingKey !== undefined) {
+    const validSig = await util.verifyCommitSig(repo.commit, signingKey)
+    if (!validSig) {
+      throw new RepoVerificationError(
+        `Invalid signature on commit: ${repo.cid.toString()}`,
+      )
+    }
+  }
+  return repo
 }
 
-// Helpers
-// -------------
-
-export const persistUpdates = async (
-  storage: RepoStorage,
-  updateStorage: RepoStorage,
-  updates: verify.VerifiedUpdate[],
-): Promise<WriteLog> => {
-  const newCids = new CidSet()
-  for (const update of updates) {
-    newCids.addSet(update.newCids)
+export const verifyProofs = async (
+  proofs: Uint8Array,
+  claims: RecordClaim[],
+  did: string,
+  didKey: string,
+): Promise<{ verified: RecordClaim[]; unverified: RecordClaim[] }> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
   }
-
-  const diffBlocks = await updateStorage.getBlocks(newCids.toList())
-  if (diffBlocks.missing.length > 0) {
-    throw new MissingBlocksError('sync', diffBlocks.missing)
+  const validSig = await util.verifyCommitSig(commit, didKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
+    )
   }
-  const commits: CommitData[] = updates.map((update) => {
-    const forCommit = diffBlocks.blocks.getMany(update.newCids.toList())
-    if (forCommit.missing.length > 0) {
-      throw new MissingBlocksError('sync', forCommit.missing)
-    }
-    return {
-      commit: update.commit,
-      prev: update.prev,
-      blocks: forCommit.blocks,
+  const mst = MST.load(blockstore, commit.data)
+  const verified: RecordClaim[] = []
+  const unverified: RecordClaim[] = []
+  for (const claim of claims) {
+    const found = await mst.get(
+      util.formatDataKey(claim.collection, claim.rkey),
+    )
+    const record = found ? await blockstore.readObj(found, def.map) : null
+    if (claim.record === null) {
+      if (record === null) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
+      }
+    } else {
+      const expected = await cidForCbor(claim.record)
+      if (expected.equals(found)) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
+      }
     }
-  })
-
-  await storage.indexCommits(commits)
+  }
+  return { verified, unverified }
+}
 
-  return Promise.all(
-    updates.map((upd) =>
-      util.diffToWriteDescripts(upd.diff, diffBlocks.blocks),
-    ),
-  )
+export const verifyRecords = async (
+  proofs: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<RecordClaim[]> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
+  }
+  const validSig = await util.verifyCommitSig(commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
+    )
+  }
+  const mst = MST.load(blockstore, commit.data)
+
+  const records: RecordClaim[] = []
+  const leaves = await mst.reachableLeaves()
+  for (const leaf of leaves) {
+    const { collection, rkey } = util.parseDataKey(leaf.key)
+    const record = await blockstore.attemptReadRecord(leaf.value)
+    if (record) {
+      records.push({
+        collection,
+        rkey,
+        record,
+      })
+    }
+  }
+  return records
 }
+
+export class RepoVerificationError extends Error {}

package/src/sync/provider.ts

@@ -7,10 +7,10 @@ import { RepoStorage } from '../storage'
 import * as util from '../util'
 import { MST } from '../mst'
 
-// Checkouts
+// Full Repo
 // -------------
 
-export const getCheckout = (
+export const getFullRepo = (
   storage: RepoStorage,
   commitCid: CID,
 ): AsyncIterable<Uint8Array> => {
@@ -22,44 +22,6 @@ export const getCheckout = (
   })
 }
 
-// Commits
-// -------------
-
-export const getCommits = (
-  storage: RepoStorage,
-  latest: CID,
-  earliest: CID | null,
-): AsyncIterable<Uint8Array> => {
-  return util.writeCar(latest, (car: BlockWriter) => {
-    return writeCommitsToCarStream(storage, car, latest, earliest)
-  })
-}
-
-export const getFullRepo = (
-  storage: RepoStorage,
-  cid: CID,
-): AsyncIterable<Uint8Array> => {
-  return getCommits(storage, cid, null)
-}
-
-export const writeCommitsToCarStream = async (
-  storage: RepoStorage,
-  car: BlockWriter,
-  latest: CID,
-  earliest: CID | null,
-): Promise<void> => {
-  const commits = await storage.getCommits(latest, earliest)
-  if (commits === null) {
-    throw new Error('Could not find shared history')
-  }
-  if (commits.length === 0) return
-  for (const commit of commits) {
-    for (const entry of commit.blocks.entries()) {
-      await car.put(entry)
-    }
-  }
-}
-
 // Narrow slices
 // -------------
 

package/src/types.ts

@@ -1,32 +1,55 @@
 import { z } from 'zod'
-import { schema as common, def as commonDef } from '@atproto/common'
+import { def as commonDef } from '@atproto/common-web'
+import { schema as common } from '@atproto/common'
 import { CID } from 'multiformats'
 import BlockMap from './block-map'
 import { RepoRecord } from '@atproto/lexicon'
+import CidSet from './cid-set'
 
 // Repo nodes
 // ---------------
 
 const unsignedCommit = z.object({
   did: z.string(),
-  version: z.number(),
-  prev: common.cid.nullable(),
+  version: z.literal(3),
   data: common.cid,
+  rev: z.string(),
+  // `prev` added for backwards compatibility with v2, no requirement of keeping around history
+  prev: common.cid.nullable().optional(),
 })
 export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
 const commit = z.object({
   did: z.string(),
-  version: z.number(),
-  prev: common.cid.nullable(),
+  version: z.literal(3),
   data: common.cid,
+  rev: z.string(),
+  prev: common.cid.nullable().optional(),
   sig: common.bytes,
 })
 export type Commit = z.infer<typeof commit>
 
+const legacyV2Commit = z.object({
+  did: z.string(),
+  version: z.literal(2),
+  data: common.cid,
+  rev: z.string().optional(),
+  prev: common.cid.nullable(),
+  sig: common.bytes,
+})
+export type LegacyV2Commit = z.infer<typeof legacyV2Commit>
+
+const versionedCommit = z.discriminatedUnion('version', [
+  commit,
+  legacyV2Commit,
+])
+export type VersionedCommit = z.infer<typeof versionedCommit>
+
 export const schema = {
   ...common,
   commit,
+  legacyV2Commit,
+  versionedCommit,
 }
 
 export const def = {
@@ -35,6 +58,10 @@ export const def = {
     name: 'commit',
     schema: schema.commit,
   },
+  versionedCommit: {
+    name: 'versioned_commit',
+    schema: schema.versionedCommit,
+  },
 }
 
 // Repo Operations
@@ -91,26 +118,13 @@ export type WriteLog = RecordWriteDescript[][]
 // Updates/Commits
 // ---------------
 
-export type CommitBlockData = {
-  commit: CID
-  blocks: BlockMap
-}
-
-export type CommitData = CommitBlockData & {
-  prev: CID | null
-}
-
-export type RebaseData = {
-  commit: CID
-  rebased: CID
-  blocks: BlockMap
-  preservedCids: CID[]
-}
-
-export type CommitCidData = {
-  commit: CID
+export type CommitData = {
+  cid: CID
+  rev: string
+  since: string | null
   prev: CID | null
-  cids: CID[]
+  newBlocks: BlockMap
+  removedCids: CidSet
 }
 
 export type RepoUpdate = CommitData & {
@@ -136,3 +150,16 @@ export type RecordClaim = {
   rkey: string
   record: RepoRecord | null
 }
+
+// Sync
+// ---------------
+
+export type VerifiedDiff = {
+  writes: RecordWriteDescript[]
+  commit: CommitData
+}
+
+export type VerifiedRepo = {
+  creates: RecordCreateDescript[]
+  commit: CommitData
+}

package/src/util.ts

@@ -11,27 +11,24 @@ import {
   schema,
   cidForCbor,
   byteIterableToStream,
+  TID,
 } from '@atproto/common'
 import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon'
 
 import * as crypto from '@atproto/crypto'
-import Repo from './repo'
-import { MST } from './mst'
 import DataDiff from './data-diff'
-import { RepoStorage } from './storage'
 import {
   Commit,
+  LegacyV2Commit,
   RecordCreateDescript,
   RecordDeleteDescript,
   RecordPath,
   RecordUpdateDescript,
   RecordWriteDescript,
   UnsignedCommit,
-  WriteLog,
   WriteOpAction,
 } from './types'
 import BlockMap from './block-map'
-import { MissingBlocksError } from './error'
 import * as parse from './parse'
 import { Keypair } from '@atproto/crypto'
 import { Readable } from 'stream'
@@ -120,33 +117,6 @@ export const readCarWithRoot = async (
   }
 }
 
-export const getWriteLog = async (
-  storage: RepoStorage,
-  latest: CID,
-  earliest: CID | null,
-): Promise<WriteLog> => {
-  const commits = await storage.getCommitPath(latest, earliest)
-  if (!commits) throw new Error('Could not find shared history')
-  const heads = await Promise.all(commits.map((c) => Repo.load(storage, c)))
-  // Turn commit path into list of diffs
-  let prev = await MST.create(storage) // Empty
-  const msts = heads.map((h) => h.data)
-  const diffs: DataDiff[] = []
-  for (const mst of msts) {
-    diffs.push(await DataDiff.of(mst, prev))
-    prev = mst
-  }
-  const fullDiff = collapseDiffs(diffs)
-  const diffBlocks = await storage.getBlocks(fullDiff.newCidList())
-  if (diffBlocks.missing.length > 0) {
-    throw new MissingBlocksError('write op log', diffBlocks.missing)
-  }
-  // Map MST diffs to write ops
-  return Promise.all(
-    diffs.map((diff) => diffToWriteDescripts(diff, diffBlocks.blocks)),
-  )
-}
-
 export const diffToWriteDescripts = (
   diff: DataDiff,
   blocks: BlockMap,
@@ -187,55 +157,18 @@ export const diffToWriteDescripts = (
   ])
 }
 
-export const collapseWriteLog = (log: WriteLog): RecordWriteDescript[] => {
-  const creates: Record<string, RecordCreateDescript> = {}
-  const updates: Record<string, RecordUpdateDescript> = {}
-  const deletes: Record<string, RecordDeleteDescript> = {}
-  for (const commit of log) {
-    for (const op of commit) {
-      const key = op.collection + '/' + op.rkey
-      if (op.action === WriteOpAction.Create) {
-        const del = deletes[key]
-        if (del) {
-          if (del.cid !== op.cid) {
-            updates[key] = {
-              ...op,
-              action: WriteOpAction.Update,
-              prev: del.cid,
-            }
-          }
-          delete deletes[key]
-        } else {
-          creates[key] = op
-        }
-      } else if (op.action === WriteOpAction.Update) {
-        updates[key] = op
-        delete creates[key]
-        delete deletes[key]
-      } else if (op.action === WriteOpAction.Delete) {
-        if (creates[key]) {
-          delete creates[key]
-        } else {
-          delete updates[key]
-          deletes[key] = op
-        }
-      } else {
-        throw new Error(`unknown action: ${op}`)
-      }
+export const ensureCreates = (
+  descripts: RecordWriteDescript[],
+): RecordCreateDescript[] => {
+  const creates: RecordCreateDescript[] = []
+  for (const descript of descripts) {
+    if (descript.action !== WriteOpAction.Create) {
+      throw new Error(`Unexpected action: ${descript.action}`)
+    } else {
+      creates.push(descript)
     }
   }
-  return [
-    ...Object.values(creates),
-    ...Object.values(updates),
-    ...Object.values(deletes),
-  ]
-}
-
-export const collapseDiffs = (diffs: DataDiff[]): DataDiff => {
-  return diffs.reduce((acc, cur) => {
-    acc.addDiff(cur)
-    return acc
-  }, new DataDiff())
+  return creates
 }
 
 export const parseDataKey = (key: string): RecordPath => {
@@ -288,3 +221,15 @@ export const cborToLexRecord = (val: Uint8Array): RepoRecord => {
 export const cidForRecord = async (val: LexValue) => {
   return cidForCbor(lexToIpld(val))
 }
+
+export const ensureV3Commit = (commit: LegacyV2Commit | Commit): Commit => {
+  if (commit.version === 3) {
+    return commit
+  } else {
+    return {
+      ...commit,
+      version: 3,
+      rev: commit.rev ?? TID.nextStr(),
+    }
+  }
+}