@atproto/repo 0.0.1 → 0.1.0

package/src/util.ts

@@ -1,88 +1,269 @@
 import { CID } from 'multiformats/cid'
-import * as auth from '@atproto/auth'
+import * as cbor from '@ipld/dag-cbor'
+import { CarReader } from '@ipld/car/reader'
+import { BlockWriter, CarWriter } from '@ipld/car/writer'
+import { Block as CarBlock } from '@ipld/car/api'
+import {
+  streamToArray,
+  verifyCidForBytes,
+  cborDecode,
+  check,
+  schema,
+  cidForCbor,
+} from '@atproto/common'
+import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon'
+
+import * as crypto from '@atproto/crypto'
 import Repo from './repo'
-import { DataDiff, MST } from './mst'
-import { IpldStore } from './blockstore'
-import { DataStore, RecordWriteOp, def } from './types'
-
-export const ucanForOperation = async (
-  prevData: DataStore,
-  newData: DataStore,
-  rootDid: string,
-  authStore: auth.AuthStore,
-): Promise<string> => {
-  const diff = await prevData.diff(newData)
-  const neededCaps = diff.neededCapabilities(rootDid)
-  const ucanForOp = await authStore.createUcanForCaps(rootDid, neededCaps, 30)
-  return auth.encodeUcan(ucanForOp)
-}
-
-export const getCommitPath = async (
-  blockstore: IpldStore,
-  earliest: CID | null,
-  latest: CID,
-): Promise<CID[] | null> => {
-  let curr: CID | null = latest
-  const path: CID[] = []
-  while (curr !== null) {
-    path.push(curr)
-    const commit = await blockstore.get(curr, def.commit)
-    if (earliest && curr.equals(earliest)) {
-      return path.reverse()
-    }
-    const root = await blockstore.get(commit.root, def.repoRoot)
-    if (!earliest && root.prev === null) {
-      return path.reverse()
+import { MST } from './mst'
+import DataDiff from './data-diff'
+import { RepoStorage } from './storage'
+import {
+  Commit,
+  DataStore,
+  RecordCreateDescript,
+  RecordDeleteDescript,
+  RecordPath,
+  RecordUpdateDescript,
+  RecordWriteDescript,
+  UnsignedCommit,
+  WriteLog,
+  WriteOpAction,
+} from './types'
+import BlockMap from './block-map'
+import { MissingBlocksError } from './error'
+import * as parse from './parse'
+import { Keypair } from '@atproto/crypto'
+
+export async function* verifyIncomingCarBlocks(
+  car: AsyncIterable<CarBlock>,
+): AsyncIterable<CarBlock> {
+  for await (const block of car) {
+    await verifyCidForBytes(block.cid, block.bytes)
+    yield block
+  }
+}
+
+export const writeCar = async (
+  root: CID | null,
+  fn: (car: BlockWriter) => Promise<void>,
+): Promise<Uint8Array> => {
+  const { writer, out } =
+    root !== null ? CarWriter.create(root) : CarWriter.create()
+  const bytes = streamToArray(out)
+  try {
+    await fn(writer)
+  } finally {
+    writer.close()
+  }
+  return bytes
+}
+
+export const blocksToCar = async (
+  root: CID | null,
+  blocks: BlockMap,
+): Promise<Uint8Array> => {
+  return writeCar(root, async (writer) => {
+    for (const entry of blocks.entries()) {
+      await writer.put(entry)
     }
-    curr = root.prev
+  })
+}
+
+export const readCar = async (
+  bytes: Uint8Array,
+): Promise<{ roots: CID[]; blocks: BlockMap }> => {
+  const car = await CarReader.fromBytes(bytes)
+  const roots = await car.getRoots()
+  const blocks = new BlockMap()
+  for await (const block of verifyIncomingCarBlocks(car.blocks())) {
+    await blocks.set(block.cid, block.bytes)
+  }
+  return {
+    roots,
+    blocks,
   }
-  return null
 }
 
-export const getWriteOpLog = async (
-  blockstore: IpldStore,
-  earliest: CID | null,
+export const readCarWithRoot = async (
+  bytes: Uint8Array,
+): Promise<{ root: CID; blocks: BlockMap }> => {
+  const { roots, blocks } = await readCar(bytes)
+  if (roots.length !== 1) {
+    throw new Error(`Expected one root, got ${roots.length}`)
+  }
+  const root = roots[0]
+  return {
+    root,
+    blocks,
+  }
+}
+
+export const getWriteLog = async (
+  storage: RepoStorage,
   latest: CID,
-): Promise<RecordWriteOp[][]> => {
-  const commits = await getCommitPath(blockstore, earliest, latest)
+  earliest: CID | null,
+): Promise<WriteLog> => {
+  const commits = await storage.getCommitPath(latest, earliest)
   if (!commits) throw new Error('Could not find shared history')
-  const heads = await Promise.all(commits.map((c) => Repo.load(blockstore, c)))
+  const heads = await Promise.all(commits.map((c) => Repo.load(storage, c)))
   // Turn commit path into list of diffs
-  let prev: DataStore = await MST.create(blockstore) // Empty
+  let prev: DataStore = await MST.create(storage) // Empty
   const msts = heads.map((h) => h.data)
   const diffs: DataDiff[] = []
   for (const mst of msts) {
-    diffs.push(await prev.diff(mst))
+    diffs.push(await DataDiff.of(mst, prev))
     prev = mst
   }
+  const fullDiff = collapseDiffs(diffs)
+  const diffBlocks = await storage.getBlocks(fullDiff.newCidList())
+  if (diffBlocks.missing.length > 0) {
+    throw new MissingBlocksError('write op log', diffBlocks.missing)
+  }
   // Map MST diffs to write ops
-  return Promise.all(diffs.map((diff) => diffToWriteOps(blockstore, diff)))
+  return Promise.all(
+    diffs.map((diff) => diffToWriteDescripts(diff, diffBlocks.blocks)),
+  )
 }
 
-export const diffToWriteOps = (
-  blockstore: IpldStore,
+export const diffToWriteDescripts = (
   diff: DataDiff,
-): Promise<RecordWriteOp[]> => {
+  blocks: BlockMap,
+): Promise<RecordWriteDescript[]> => {
   return Promise.all([
     ...diff.addList().map(async (add) => {
-      const { collection, rkey } = parseRecordKey(add.key)
-      const value = await blockstore.getUnchecked(add.cid)
-      return { action: 'create' as const, collection, rkey, value }
+      const { collection, rkey } = parseDataKey(add.key)
+      const value = await parse.getAndParseRecord(blocks, add.cid)
+      return {
+        action: WriteOpAction.Create,
+        collection,
+        rkey,
+        cid: add.cid,
+        record: value.record,
+      } as RecordCreateDescript
     }),
     ...diff.updateList().map(async (upd) => {
-      const { collection, rkey } = parseRecordKey(upd.key)
-      const value = await blockstore.getUnchecked(upd.cid)
-      return { action: 'update' as const, collection, rkey, value }
+      const { collection, rkey } = parseDataKey(upd.key)
+      const value = await parse.getAndParseRecord(blocks, upd.cid)
+      return {
+        action: WriteOpAction.Update,
+        collection,
+        rkey,
+        cid: upd.cid,
+        prev: upd.prev,
+        record: value.record,
+      } as RecordUpdateDescript
     }),
     ...diff.deleteList().map((del) => {
-      const { collection, rkey } = parseRecordKey(del.key)
-      return { action: 'delete' as const, collection, rkey }
+      const { collection, rkey } = parseDataKey(del.key)
+      return {
+        action: WriteOpAction.Delete,
+        collection,
+        rkey,
+        cid: del.cid,
+      } as RecordDeleteDescript
     }),
   ])
 }
 
-export const parseRecordKey = (key: string) => {
+export const collapseWriteLog = (log: WriteLog): RecordWriteDescript[] => {
+  const creates: Record<string, RecordCreateDescript> = {}
+  const updates: Record<string, RecordUpdateDescript> = {}
+  const deletes: Record<string, RecordDeleteDescript> = {}
+  for (const commit of log) {
+    for (const op of commit) {
+      const key = op.collection + '/' + op.rkey
+      if (op.action === WriteOpAction.Create) {
+        const del = deletes[key]
+        if (del) {
+          if (del.cid !== op.cid) {
+            updates[key] = {
+              ...op,
+              action: WriteOpAction.Update,
+              prev: del.cid,
+            }
+          }
+          delete deletes[key]
+        } else {
+          creates[key] = op
+        }
+      } else if (op.action === WriteOpAction.Update) {
+        updates[key] = op
+        delete creates[key]
+        delete deletes[key]
+      } else if (op.action === WriteOpAction.Delete) {
+        if (creates[key]) {
+          delete creates[key]
+        } else {
+          delete updates[key]
+          deletes[key] = op
+        }
+      } else {
+        throw new Error(`unknown action: ${op}`)
+      }
+    }
+  }
+  return [
+    ...Object.values(creates),
+    ...Object.values(updates),
+    ...Object.values(deletes),
+  ]
+}
+
+export const collapseDiffs = (diffs: DataDiff[]): DataDiff => {
+  return diffs.reduce((acc, cur) => {
+    acc.addDiff(cur)
+    return acc
+  }, new DataDiff())
+}
+
+export const parseDataKey = (key: string): RecordPath => {
   const parts = key.split('/')
   if (parts.length !== 2) throw new Error(`Invalid record key: ${key}`)
   return { collection: parts[0], rkey: parts[1] }
 }
+
+export const formatDataKey = (collection: string, rkey: string): string => {
+  return collection + '/' + rkey
+}
+
+export const metaEqual = (a: Commit, b: Commit): boolean => {
+  return a.did === b.did && a.version === b.version
+}
+
+export const signCommit = async (
+  unsigned: UnsignedCommit,
+  keypair: Keypair,
+): Promise<Commit> => {
+  const encoded = cbor.encode(unsigned)
+  const sig = await keypair.sign(encoded)
+  return {
+    ...unsigned,
+    sig,
+  }
+}
+
+export const verifyCommitSig = async (
+  commit: Commit,
+  didKey: string,
+): Promise<boolean> => {
+  const { sig, ...rest } = commit
+  const encoded = cbor.encode(rest)
+  return crypto.verifySignature(didKey, encoded, sig)
+}
+
+export const cborToLex = (val: Uint8Array): LexValue => {
+  return ipldToLex(cborDecode(val))
+}
+
+export const cborToLexRecord = (val: Uint8Array): RepoRecord => {
+  const parsed = cborToLex(val)
+  if (!check.is(parsed, schema.map)) {
+    throw new Error('lexicon records be a json object')
+  }
+  return parsed
+}
+
+export const cidForRecord = async (val: LexValue) => {
+  return cidForCbor(lexToIpld(val))
+}

package/src/verify.ts

@@ -1,62 +1,227 @@
 import { CID } from 'multiformats/cid'
-import * as auth from '@atproto/auth'
-import { IpldStore } from './blockstore'
+import { MemoryBlockstore, ReadableBlockstore, RepoStorage } from './storage'
+import DataDiff from './data-diff'
+import SyncStorage from './storage/sync-storage'
+import ReadableRepo from './readable-repo'
 import Repo from './repo'
-import { DataDiff } from './mst'
+import CidSet from './cid-set'
 import * as util from './util'
+import { RecordClaim, RepoContents } from './types'
 import { def } from './types'
+import { MST } from './mst'
+import { cidForCbor } from '@atproto/common'
+
+export type VerifiedCheckout = {
+  contents: RepoContents
+  newCids: CidSet
+}
+
+export const verifyCheckout = async (
+  storage: ReadableBlockstore,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedCheckout> => {
+  const repo = await ReadableRepo.load(storage, head)
+  if (repo.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${repo.did}`)
+  }
+  const validSig = await util.verifyCommitSig(repo.commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${repo.cid.toString()}`,
+    )
+  }
+  const diff = await DataDiff.of(repo.data, null)
+  const newCids = new CidSet([repo.cid]).addSet(diff.newCids)
+
+  const contents: RepoContents = {}
+  for (const add of diff.addList()) {
+    const { collection, rkey } = util.parseDataKey(add.key)
+    if (!contents[collection]) {
+      contents[collection] = {}
+    }
+    const record = await storage.readRecord(add.cid)
+    contents[collection][rkey] = record
+  }
+
+  return {
+    contents,
+    newCids,
+  }
+}
+
+export type VerifiedUpdate = {
+  commit: CID
+  prev: CID | null
+  diff: DataDiff
+  newCids: CidSet
+}
+
+export const verifyFullHistory = async (
+  storage: RepoStorage,
+  head: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedUpdate[]> => {
+  const commitPath = await storage.getCommitPath(head, null)
+  if (commitPath === null) {
+    throw new RepoVerificationError('Could not find shared history')
+  } else if (commitPath.length < 1) {
+    throw new RepoVerificationError('Expected at least one commit')
+  }
+  const baseRepo = await Repo.load(storage, commitPath[0])
+  const baseDiff = await DataDiff.of(baseRepo.data, null)
+  const baseRepoCids = new CidSet([baseRepo.cid]).addSet(baseDiff.newCids)
+  const init: VerifiedUpdate = {
+    commit: baseRepo.cid,
+    prev: null,
+    diff: baseDiff,
+    newCids: baseRepoCids,
+  }
+  const updates = await verifyCommitPath(
+    baseRepo,
+    storage,
+    commitPath.slice(1),
+    did,
+    signingKey,
+  )
+  return [init, ...updates]
+}
 
 export const verifyUpdates = async (
-  blockstore: IpldStore,
-  earliest: CID | null,
-  latest: CID,
-  verifier: auth.Verifier,
-): Promise<DataDiff> => {
-  const commitPath = await util.getCommitPath(blockstore, earliest, latest)
+  repo: ReadableRepo,
+  updateStorage: RepoStorage,
+  updateRoot: CID,
+  did: string,
+  signingKey: string,
+): Promise<VerifiedUpdate[]> => {
+  const commitPath = await updateStorage.getCommitPath(updateRoot, repo.cid)
   if (commitPath === null) {
-    throw new Error('Could not find shared history')
+    throw new RepoVerificationError('Could not find shared history')
   }
-  const fullDiff = new DataDiff()
-  if (commitPath.length === 0) return fullDiff
-  let prevRepo = await Repo.load(blockstore, commitPath[0])
-  for (const commit of commitPath.slice(1)) {
-    const nextRepo = await Repo.load(blockstore, commit)
-    const diff = await prevRepo.data.diff(nextRepo.data)
+  const syncStorage = new SyncStorage(updateStorage, repo.storage)
+  return verifyCommitPath(repo, syncStorage, commitPath, did, signingKey)
+}
 
-    if (!nextRepo.root.meta.equals(prevRepo.root.meta)) {
-      throw new Error('Not supported: repo metadata updated')
+export const verifyCommitPath = async (
+  baseRepo: ReadableRepo,
+  storage: ReadableBlockstore,
+  commitPath: CID[],
+  did: string,
+  signingKey: string,
+): Promise<VerifiedUpdate[]> => {
+  const updates: VerifiedUpdate[] = []
+  if (commitPath.length === 0) return updates
+  let prevRepo = baseRepo
+  for (const commit of commitPath) {
+    const nextRepo = await ReadableRepo.load(storage, commit)
+    const diff = await DataDiff.of(nextRepo.data, prevRepo.data)
+
+    if (nextRepo.did !== did) {
+      throw new RepoVerificationError(`Invalid repo did: ${nextRepo.did}`)
+    }
+    if (!util.metaEqual(nextRepo.commit, prevRepo.commit)) {
+      throw new RepoVerificationError('Not supported: repo metadata updated')
     }
 
-    let didForSignature: string
-    if (nextRepo.root.auth_token) {
-      // verify auth token covers all necessary writes
-      const encodedToken = await blockstore.get(
-        nextRepo.root.auth_token,
-        def.string,
+    const validSig = await util.verifyCommitSig(nextRepo.commit, signingKey)
+    if (!validSig) {
+      throw new RepoVerificationError(
+        `Invalid signature on commit: ${nextRepo.cid.toString()}`,
       )
-      const token = await verifier.validateUcan(encodedToken)
-      const neededCaps = diff.neededCapabilities(prevRepo.did)
-      for (const cap of neededCaps) {
-        await verifier.verifyAtpUcan(token, prevRepo.did, cap)
+    }
+
+    const newCids = new CidSet([nextRepo.cid]).addSet(diff.newCids)
+
+    updates.push({
+      commit: nextRepo.cid,
+      prev: prevRepo.cid,
+      diff,
+      newCids,
+    })
+    prevRepo = nextRepo
+  }
+  return updates
+}
+
+export const verifyProofs = async (
+  proofs: Uint8Array,
+  claims: RecordClaim[],
+  did: string,
+  didKey: string,
+): Promise<{ verified: RecordClaim[]; unverified: RecordClaim[] }> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
+  }
+  const validSig = await util.verifyCommitSig(commit, didKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
+    )
+  }
+  const mst = MST.load(blockstore, commit.data)
+  const verified: RecordClaim[] = []
+  const unverified: RecordClaim[] = []
+  for (const claim of claims) {
+    const found = await mst.get(
+      util.formatDataKey(claim.collection, claim.rkey),
+    )
+    const record = found ? await blockstore.readObj(found, def.map) : null
+    if (claim.record === null) {
+      if (record === null) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
       }
-      didForSignature = token.payload.iss
     } else {
-      didForSignature = prevRepo.did
+      const expected = await cidForCbor(claim.record)
+      if (expected.equals(found)) {
+        verified.push(claim)
+      } else {
+        unverified.push(claim)
+      }
     }
+  }
+  return { verified, unverified }
+}
 
-    // verify signature matches repo root + auth token
-    // const commit = await toRepo.getCommit()
-    const validSig = await verifier.verifySignature(
-      didForSignature,
-      nextRepo.commit.root.bytes,
-      nextRepo.commit.sig,
+export const verifyRecords = async (
+  proofs: Uint8Array,
+  did: string,
+  signingKey: string,
+): Promise<RecordClaim[]> => {
+  const car = await util.readCarWithRoot(proofs)
+  const blockstore = new MemoryBlockstore(car.blocks)
+  const commit = await blockstore.readObj(car.root, def.commit)
+  if (commit.did !== did) {
+    throw new RepoVerificationError(`Invalid repo did: ${commit.did}`)
+  }
+  const validSig = await util.verifyCommitSig(commit, signingKey)
+  if (!validSig) {
+    throw new RepoVerificationError(
+      `Invalid signature on commit: ${car.root.toString()}`,
     )
-    if (!validSig) {
-      throw new Error(`Invalid signature on commit: ${nextRepo.cid.toString()}`)
-    }
+  }
+  const mst = MST.load(blockstore, commit.data)
 
-    fullDiff.addDiff(diff)
-    prevRepo = nextRepo
+  const records: RecordClaim[] = []
+  const leaves = await mst.reachableLeaves()
+  for (const leaf of leaves) {
+    const { collection, rkey } = util.parseDataKey(leaf.key)
+    const record = await blockstore.attemptReadRecord(leaf.value)
+    if (record) {
+      records.push({
+        collection,
+        rkey,
+        record,
+      })
+    }
   }
-  return fullDiff
+  return records
 }
+
+export class RepoVerificationError extends Error {}