@atproto/repo 0.0.1 → 0.2.0

package/src/verify.ts

@@ -1,62 +1,268 @@
 import { CID } from 'multiformats/cid'
-import * as auth from '@atproto/auth'
-import { IpldStore } from './blockstore'
+import { MemoryBlockstore, ReadableBlockstore, RepoStorage } from './storage'
+import DataDiff from './data-diff'
+import SyncStorage from './storage/sync-storage'
+import ReadableRepo from './readable-repo'
 import Repo from './repo'
-import { DataDiff } from './mst'
+import CidSet from './cid-set'
 import * as util from './util'
+import { RecordClaim, RepoContents, RepoContentsWithCids } from './types'
 import { def } from './types'
+import { MST } from './mst'
+import { cidForCbor } from '@atproto/common'
+
+export type VerifiedCheckout = {
+  contents: RepoContents
+  newCids: CidSet
+}
+
+export type VerifiedCheckoutWithCids = {
+  commit: CID
+  contents: RepoContentsWithCids
+}
+
+export const verifyCheckout = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedCheckout> => {
+  const repo = await verifyRepoRoot(storage, head, did, signingKey)
+  const diff = await DataDiff.of(repo.data, null)
+  const newCids = new CidSet([repo.cid]).addSet(diff.newCids)
+
+  const contents: RepoContents = {}
+  for (const add of diff.addList()) {
+    const { collection, rkey } = util.parseDataKey(add.key)
+    if (!contents[collection]) {
+      contents[collection] = {}
+    }
+    const record = await storage.readRecord(add.cid)
+    contents[collection][rkey] = record
+  }
+
+  return {
+    contents,
+    newCids,
+  }
+}
+
+export const verifyCheckoutWithCids = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedCheckoutWithCids> => {
+  const repo = await verifyRepoRoot(storage, head, did, signingKey)
+  const diff = await DataDiff.of(repo.data, null)
+
+  const contents: RepoContentsWithCids = {}
+  for (const add of diff.addList()) {
+    const { collection, rkey } = util.parseDataKey(add.key)
+    contents[collection] ??= {}
+    contents[collection][rkey] = {
+      cid: add.cid,
+      value: await storage.readRecord(add.cid),
+    }
+  }
+
+  return {
+    commit: repo.cid,
+    contents,
+  }
+}
+
+// @NOTE only verifies the root, not the repo contents
+const verifyRepoRoot = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<ReadableRepo> => {
+  const repo = await ReadableRepo.load(storage, head)
+  if (repo.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
+  }
+  const validSig = await util.verifyCommitSig(repo.commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${repo.cid.toString()}`,
+    )
+  }
+  return repo
+}
+
+export type VerifiedUpdate = {
+  commit: CID
+  prev: CID | null
+  diff: DataDiff
+  newCids: CidSet
+}
+
+export const verifyFullHistory = async (
+  storage: RepoStorage,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedUpdate[]> => {
+  const commitPath = await storage.getCommitPath(head, null)
+  if (commitPath === null) {
+    throw new RepoVerificationError('Could not find shared history')
+  } else if (commitPath.length < 1) {
+    throw new RepoVerificationError('Expected at least one commit')
+  }
+  const baseRepo = await Repo.load(storage, commitPath[0])
+  const baseDiff = await DataDiff.of(baseRepo.data, null)
+  const baseRepoCids = new CidSet([baseRepo.cid]).addSet(baseDiff.newCids)
+  const init: VerifiedUpdate = {
+    commit: baseRepo.cid,
+    prev: null,
+    diff: baseDiff,
+    newCids: baseRepoCids,
+  }
+  const updates = await verifyCommitPath(
+    baseRepo,
+    storage,
+    commitPath.slice(1),
+    did,
+    signingKey,
+  )
+  return [init, ...updates]
+}
 
 export const verifyUpdates = async (
-  blockstore: IpldStore,
-  earliest: CID | null,
-  latest: CID,
-  verifier: auth.Verifier,
-): Promise<DataDiff> => {
-  const commitPath = await util.getCommitPath(blockstore, earliest, latest)
+  repo: ReadableRepo,
+  updateStorage: RepoStorage,
+  updateRoot: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedUpdate[]> => {
+  const commitPath = await updateStorage.getCommitPath(updateRoot, repo.cid)
   if (commitPath === null) {
-    throw new Error('Could not find shared history')
+    throw new RepoVerificationError('Could not find shared history')
   }
-  const fullDiff = new DataDiff()
-  if (commitPath.length === 0) return fullDiff
-  let prevRepo = await Repo.load(blockstore, commitPath[0])
-  for (const commit of commitPath.slice(1)) {
-    const nextRepo = await Repo.load(blockstore, commit)
-    const diff = await prevRepo.data.diff(nextRepo.data)
+  const syncStorage = new SyncStorage(updateStorage, repo.storage)
+  return verifyCommitPath(repo, syncStorage, commitPath, did, signingKey)
+}
+
+export const verifyCommitPath = async (
+  baseRepo: ReadableRepo,
+  storage: ReadableBlockstore,
+  commitPath: CID[],
+  did: string,
+  signingKey: string,
+): Promise<VerifiedUpdate[]> => {
+  const updates: VerifiedUpdate[] = []
+  if (commitPath.length === 0) return updates
+  let prevRepo = baseRepo
+  for (const commit of commitPath) {
+    const nextRepo = await ReadableRepo.load(storage, commit)
+    const diff = await DataDiff.of(nextRepo.data, prevRepo.data)
 
-    if (!nextRepo.root.meta.equals(prevRepo.root.meta)) {
-      throw new Error('Not supported: repo metadata updated')
+    if (nextRepo.did !== did) {
+      throw new RepoVerificationError(`Invalid repo did: ${nextRepo.did}`)
+    }
+    if (!util.metaEqual(nextRepo.commit, prevRepo.commit)) {
+      throw new RepoVerificationError('Not supported: repo metadata updated')
     }
 
-    let didForSignature: string
-    if (nextRepo.root.auth_token) {
-      // verify auth token covers all necessary writes
-      const encodedToken = await blockstore.get(
-        nextRepo.root.auth_token,
-        def.string,
+    const validSig = await util.verifyCommitSig(nextRepo.commit, signingKey)
+    if (!validSig) {
+      throw new RepoVerificationError(
+        `Invalid signature on commit: ${nextRepo.cid.toString()}`,
       )
-      const token = await verifier.validateUcan(encodedToken)
-      const neededCaps = diff.neededCapabilities(prevRepo.did)
-      for (const cap of neededCaps) {
-        await verifier.verifyAtpUcan(token, prevRepo.did, cap)
+    }
+
+    const newCids = new CidSet([nextRepo.cid]).addSet(diff.newCids)
+
+    updates.push({
+      commit: nextRepo.cid,
+      prev: prevRepo.cid,
+      diff,
+      newCids,
+    })
+    prevRepo = nextRepo
+  }
+  return updates
+}
+
+export const verifyProofs = async (
+  proofs: Uint8Array,
+  claims: RecordClaim[],
+  did: string,
+  didKey: string,
+): Promise<{ verified: RecordClaim[]; unverified: RecordClaim[] }> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
+  }
+  const validSig = await util.verifyCommitSig(commit, didKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
+    )
+  }
+  const mst = MST.load(blockstore, commit.data)
+  const verified: RecordClaim[] = []
+  const unverified: RecordClaim[] = []
+  for (const claim of claims) {
+    const found = await mst.get(
+      util.formatDataKey(claim.collection, claim.rkey),
+    )
+    const record = found ? await blockstore.readObj(found, def.map) : null
+    if (claim.record === null) {
+      if (record === null) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
       }
-      didForSignature = token.payload.iss
     } else {
-      didForSignature = prevRepo.did
+      const expected = await cidForCbor(claim.record)
+      if (expected.equals(found)) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
+      }
     }
+  }
+  return { verified, unverified }
+}
 
-    // verify signature matches repo root + auth token
-    // const commit = await toRepo.getCommit()
-    const validSig = await verifier.verifySignature(
-      didForSignature,
-      nextRepo.commit.root.bytes,
-      nextRepo.commit.sig,
+export const verifyRecords = async (
+  proofs: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<RecordClaim[]> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
+  }
+  const validSig = await util.verifyCommitSig(commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
     )
-    if (!validSig) {
-      throw new Error(`Invalid signature on commit: ${nextRepo.cid.toString()}`)
-    }
+  }
+  const mst = MST.load(blockstore, commit.data)
 
-    fullDiff.addDiff(diff)
-    prevRepo = nextRepo
+  const records: RecordClaim[] = []
+  const leaves = await mst.reachableLeaves()
+  for (const leaf of leaves) {
+    const { collection, rkey } = util.parseDataKey(leaf.key)
+    const record = await blockstore.attemptReadRecord(leaf.value)
+    if (record) {
+      records.push({
+        collection,
+        rkey,
+        record,
+      })
+    }
   }
-  return fullDiff
+  return records
 }
+
+export class RepoVerificationError extends Error {}

package/tests/_util.ts

@@ -1,38 +1,42 @@
+import fs from 'fs'
 import { CID } from 'multiformats'
-import { TID } from '@atproto/common'
-import * as auth from '@atproto/auth'
-import IpldStore from '../src/blockstore/ipld-store'
+import { TID, dataToCborBlock } from '@atproto/common'
+import * as crypto from '@atproto/crypto'
 import { Repo } from '../src/repo'
-import { MemoryBlockstore } from '../src/blockstore'
-import { DataDiff, MST } from '../src/mst'
-import fs from 'fs'
-import { RecordWriteOp } from '../src'
+import { RepoStorage } from '../src/storage'
+import { MST } from '../src/mst'
+import {
+  BlockMap,
+  collapseWriteLog,
+  CollectionContents,
+  RecordWriteOp,
+  RepoContents,
+  RecordPath,
+  WriteLog,
+  WriteOpAction,
+  RecordClaim,
+  Commit,
+} from '../src'
+import { Keypair, randomBytes } from '@atproto/crypto'
 
 type IdMapping = Record<string, CID>
 
-const fakeStore = new MemoryBlockstore()
-
-export const randomCid = async (store: IpldStore = fakeStore): Promise<CID> => {
-  const str = randomStr(50)
-  return store.stage({ test: str })
-}
-
-export const generateBulkTids = (count: number): TID[] => {
-  const ids: TID[] = []
-  for (let i = 0; i < count; i++) {
-    ids.push(TID.next())
+export const randomCid = async (storage?: RepoStorage): Promise<CID> => {
+  const block = await dataToCborBlock({ test: randomStr(50) })
+  if (storage) {
+    await storage.putBlock(block.cid, block.bytes)
   }
-  return ids
+  return block.cid
 }
 
-export const generateBulkTidMapping = async (
+export const generateBulkDataKeys = async (
   count: number,
-  blockstore: IpldStore = fakeStore,
+  blockstore?: RepoStorage,
 ): Promise<IdMapping> => {
-  const ids = generateBulkTids(count)
   const obj: IdMapping = {}
-  for (const id of ids) {
-    obj[id.toString()] = await randomCid(blockstore)
+  for (let i = 0; i < count; i++) {
+    const key = `com.example.record/${TID.nextStr()}`
+    obj[key] = await randomCid(blockstore)
   }
   return obj
 }
@@ -76,32 +80,29 @@ export const generateObject = (): Record<string, string> => {
 
 export const testCollections = ['com.example.posts', 'com.example.likes']
 
-export type CollectionData = Record<string, unknown>
-export type RepoData = Record<string, CollectionData>
-
 export const fillRepo = async (
   repo: Repo,
-  authStore: auth.AuthStore,
+  keypair: crypto.Keypair,
   itemsPerCollection: number,
-): Promise<{ repo: Repo; data: RepoData }> => {
-  const repoData: RepoData = {}
+): Promise<{ repo: Repo; data: RepoContents }> => {
+  const repoData: RepoContents = {}
   const writes: RecordWriteOp[] = []
   for (const collName of testCollections) {
-    const collData: CollectionData = {}
+    const collData: CollectionContents = {}
     for (let i = 0; i < itemsPerCollection; i++) {
       const object = generateObject()
       const rkey = TID.nextStr()
       collData[rkey] = object
       writes.push({
-        action: 'create',
+        action: WriteOpAction.Create,
         collection: collName,
         rkey,
-        value: object,
+        record: object,
       })
     }
     repoData[collName] = collData
   }
-  const updated = await repo.stageUpdate(writes).createCommit(authStore)
+  const updated = await repo.applyWrites(writes, keypair)
   return {
     repo: updated,
     data: repoData,
@@ -110,17 +111,16 @@ export const fillRepo = async (
 
 export const editRepo = async (
   repo: Repo,
-  prevData: RepoData,
-  authStore: auth.AuthStore,
+  prevData: RepoContents,
+  keypair: crypto.Keypair,
   params: {
     adds?: number
     updates?: number
     deletes?: number
   },
-): Promise<{ repo: Repo; data: RepoData }> => {
+): Promise<{ repo: Repo; data: RepoContents }> => {
   const { adds = 0, updates = 0, deletes = 0 } = params
-  const repoData: RepoData = {}
-  const writes: RecordWriteOp[] = []
+  const repoData: RepoContents = {}
   for (const collName of testCollections) {
     const collData = prevData[collName]
     const shuffled = shuffle(Object.entries(collData))
@@ -129,94 +129,129 @@ export const editRepo = async (
       const object = generateObject()
       const rkey = TID.nextStr()
       collData[rkey] = object
-      writes.push({
-        action: 'create',
-        collection: collName,
-        rkey,
-        value: object,
-      })
+      repo = await repo.applyWrites(
+        {
+          action: WriteOpAction.Create,
+          collection: collName,
+          rkey,
+          record: object,
+        },
+        keypair,
+      )
     }
 
     const toUpdate = shuffled.slice(0, updates)
     for (let i = 0; i < toUpdate.length; i++) {
       const object = generateObject()
       const rkey = toUpdate[i][0]
-      writes.push({
-        action: 'update',
-        collection: collName,
-        rkey,
-        value: object,
-      })
+      repo = await repo.applyWrites(
+        {
+          action: WriteOpAction.Update,
+          collection: collName,
+          rkey,
+          record: object,
+        },
+        keypair,
+      )
       collData[rkey] = object
     }
 
     const toDelete = shuffled.slice(updates, deletes)
     for (let i = 0; i < toDelete.length; i++) {
       const rkey = toDelete[i][0]
-      writes.push({
-        action: 'delete',
-        collection: collName,
-        rkey,
-      })
+      repo = await repo.applyWrites(
+        {
+          action: WriteOpAction.Delete,
+          collection: collName,
+          rkey,
+        },
+        keypair,
+      )
       delete collData[rkey]
     }
     repoData[collName] = collData
   }
-  const updated = await repo.stageUpdate(writes).createCommit(authStore)
   return {
-    repo: updated,
+    repo,
     data: repoData,
   }
 }
 
-export const checkRepo = async (repo: Repo, data: RepoData): Promise<void> => {
-  for (const collName of Object.keys(data)) {
-    const collData = data[collName]
-    for (const rkey of Object.keys(collData)) {
-      const record = await repo.getRecord(collName, rkey)
-      expect(record).toEqual(collData[rkey])
-    }
-  }
-}
-
-export const checkRepoDiff = async (
-  diff: DataDiff,
-  before: RepoData,
-  after: RepoData,
+export const verifyRepoDiff = async (
+  writeLog: WriteLog,
+  before: RepoContents,
+  after: RepoContents,
 ): Promise<void> => {
-  const getObjectCid = async (
-    key: string,
-    data: RepoData,
-  ): Promise<CID | undefined> => {
-    const parts = key.split('/')
-    const collection = parts[0]
-    const obj = (data[collection] || {})[parts[1]]
-    return obj === undefined ? undefined : fakeStore.stage(obj)
+  const getVal = (op: RecordWriteOp, data: RepoContents) => {
+    return (data[op.collection] || {})[op.rkey]
   }
+  const ops = await collapseWriteLog(writeLog)
 
-  for (const add of diff.addList()) {
-    const beforeCid = await getObjectCid(add.key, before)
-    const afterCid = await getObjectCid(add.key, after)
-
-    expect(beforeCid).toBeUndefined()
-    expect(afterCid).toEqual(add.cid)
+  for (const op of ops) {
+    if (op.action === WriteOpAction.Create) {
+      expect(getVal(op, before)).toBeUndefined()
+      expect(getVal(op, after)).toEqual(op.record)
+    } else if (op.action === WriteOpAction.Update) {
+      expect(getVal(op, before)).toBeDefined()
+      expect(getVal(op, after)).toEqual(op.record)
+    } else if (op.action === WriteOpAction.Delete) {
+      expect(getVal(op, before)).toBeDefined()
+      expect(getVal(op, after)).toBeUndefined()
+    } else {
+      throw new Error('unexpected op type')
+    }
   }
+}
 
-  for (const update of diff.updateList()) {
-    const beforeCid = await getObjectCid(update.key, before)
-    const afterCid = await getObjectCid(update.key, after)
-
-    expect(beforeCid).toEqual(update.prev)
-    expect(afterCid).toEqual(update.cid)
+export const contentsToClaims = (contents: RepoContents): RecordClaim[] => {
+  const claims: RecordClaim[] = []
+  for (const coll of Object.keys(contents)) {
+    for (const rkey of Object.keys(contents[coll])) {
+      claims.push({
+        collection: coll,
+        rkey: rkey,
+        record: contents[coll][rkey],
+      })
+    }
   }
+  return claims
+}
+
+export const pathsForOps = (ops: RecordWriteOp[]): RecordPath[] =>
+  ops.map((op) => ({ collection: op.collection, rkey: op.rkey }))
 
-  for (const del of diff.deleteList()) {
-    const beforeCid = await getObjectCid(del.key, before)
-    const afterCid = await getObjectCid(del.key, after)
+export const saveMst = async (storage: RepoStorage, mst: MST): Promise<CID> => {
+  const diff = await mst.getUnstoredBlocks()
+  await storage.putMany(diff.blocks)
+  return diff.root
+}
 
-    expect(beforeCid).toEqual(del.cid)
-    expect(afterCid).toBeUndefined()
+// Creating repo
+// -------------------
+export const addBadCommit = async (
+  repo: Repo,
+  keypair: Keypair,
+): Promise<Repo> => {
+  const obj = generateObject()
+  const blocks = new BlockMap()
+  const cid = await blocks.add(obj)
+  const updatedData = await repo.data.add(`com.example.test/${TID.next()}`, cid)
+  const unstoredData = await updatedData.getUnstoredBlocks()
+  blocks.addMap(unstoredData.blocks)
+  // we generate a bad sig by signing some other data
+  const commit: Commit = {
+    ...repo.commit,
+    prev: repo.cid,
+    data: unstoredData.root,
+    sig: await keypair.sign(randomBytes(256)),
   }
+  const commitCid = await blocks.add(commit)
+  await repo.storage.applyCommit({
+    commit: commitCid,
+    prev: repo.cid,
+    blocks: blocks,
+  })
+  return await Repo.load(repo.storage, commitCid)
 }
 
 // Logging