@atproto/repo 0.0.1 → 0.2.0

package/src/types.ts

@@ -1,101 +1,138 @@
 import { z } from 'zod'
-import { BlockWriter } from '@ipld/car/writer'
-import { def as common } from '@atproto/common'
+import { schema as common, def as commonDef } from '@atproto/common'
 import { CID } from 'multiformats'
-import { DataDiff } from './mst'
+import BlockMap from './block-map'
+import { RepoRecord } from '@atproto/lexicon'
 
-const repoMeta = z.object({
+// Repo nodes
+// ---------------
+
+const unsignedCommit = z.object({
   did: z.string(),
   version: z.number(),
-  datastore: z.string(),
-})
-export type RepoMeta = z.infer<typeof repoMeta>
-
-const repoRoot = z.object({
-  meta: common.cid,
   prev: common.cid.nullable(),
-  auth_token: common.cid.nullable(),
   data: common.cid,
 })
-export type RepoRoot = z.infer<typeof repoRoot>
+export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
 const commit = z.object({
-  root: common.cid,
+  did: z.string(),
+  version: z.number(),
+  prev: common.cid.nullable(),
+  data: common.cid,
   sig: common.bytes,
 })
 export type Commit = z.infer<typeof commit>
 
-export const cidCreateOp = z.object({
-  action: z.literal('create'),
-  collection: z.string(),
-  rkey: z.string(),
-  cid: common.cid,
-})
-export type CidCreateOp = z.infer<typeof cidCreateOp>
+export const schema = {
+  ...common,
+  commit,
+}
 
-export const cidUpdateOp = z.object({
-  action: z.literal('update'),
-  collection: z.string(),
-  rkey: z.string(),
-  cid: common.cid,
-})
-export type CidUpdateOp = z.infer<typeof cidUpdateOp>
+export const def = {
+  ...commonDef,
+  commit: {
+    name: 'commit',
+    schema: schema.commit,
+  },
+}
 
-export const deleteOp = z.object({
-  action: z.literal('delete'),
-  collection: z.string(),
-  rkey: z.string(),
-})
-export type DeleteOp = z.infer<typeof deleteOp>
+// Repo Operations
+// ---------------
 
-export const cidWriteOp = z.union([cidCreateOp, cidUpdateOp, deleteOp])
-export type CidWriteOp = z.infer<typeof cidWriteOp>
+export enum WriteOpAction {
+  Create = 'create',
+  Update = 'update',
+  Delete = 'delete',
+}
 
-export const recordCreateOp = z.object({
-  action: z.literal('create'),
-  collection: z.string(),
-  rkey: z.string(),
-  value: z.any(),
-})
-export type RecordCreateOp = z.infer<typeof recordCreateOp>
+export type RecordCreateOp = {
+  action: WriteOpAction.Create
+  collection: string
+  rkey: string
+  record: RepoRecord
+}
 
-export const recordUpdateOp = z.object({
-  action: z.literal('update'),
-  collection: z.string(),
-  rkey: z.string(),
-  value: z.any(),
-})
-export type RecordUpdateOp = z.infer<typeof recordUpdateOp>
+export type RecordUpdateOp = {
+  action: WriteOpAction.Update
+  collection: string
+  rkey: string
+  record: RepoRecord
+}
 
-export const recordWriteOp = z.union([recordCreateOp, recordUpdateOp, deleteOp])
-export type RecordWriteOp = z.infer<typeof recordWriteOp>
+export type RecordDeleteOp = {
+  action: WriteOpAction.Delete
+  collection: string
+  rkey: string
+}
 
-export const def = {
-  ...common,
-  repoMeta,
-  repoRoot,
-  commit,
-  cidWriteOp,
-  recordWriteOp,
+export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp
+
+export type RecordCreateDescript = RecordCreateOp & {
+  cid: CID
 }
 
-export interface CarStreamable {
-  writeToCarStream(car: BlockWriter): Promise<void>
+export type RecordUpdateDescript = RecordUpdateOp & {
+  prev: CID
+  cid: CID
 }
 
-export type DataValue = {
-  key: string
-  value: CID
+export type RecordDeleteDescript = RecordDeleteOp & {
+  cid: CID
+}
+
+export type RecordWriteDescript =
+  | RecordCreateDescript
+  | RecordUpdateDescript
+  | RecordDeleteDescript
+
+export type WriteLog = RecordWriteDescript[][]
+
+// Updates/Commits
+// ---------------
+
+export type CommitBlockData = {
+  commit: CID
+  blocks: BlockMap
+}
+
+export type CommitData = CommitBlockData & {
+  prev: CID | null
+}
+
+export type RebaseData = {
+  commit: CID
+  rebased: CID
+  blocks: BlockMap
+  preservedCids: CID[]
+}
+
+export type CommitCidData = {
+  commit: CID
+  prev: CID | null
+  cids: CID[]
+}
+
+export type RepoUpdate = CommitData & {
+  ops: RecordWriteOp[]
+}
+
+export type CollectionContents = Record<string, RepoRecord>
+export type RepoContents = Record<string, CollectionContents>
+
+export type RepoRecordWithCid = { cid: CID; value: RepoRecord }
+export type CollectionContentsWithCids = Record<string, RepoRecordWithCid>
+export type RepoContentsWithCids = Record<string, CollectionContentsWithCids>
+
+export type DatastoreContents = Record<string, CID>
+
+export type RecordPath = {
+  collection: string
+  rkey: string
 }
 
-export interface DataStore {
-  add(key: string, value: CID): Promise<DataStore>
-  update(key: string, value: CID): Promise<DataStore>
-  delete(key: string): Promise<DataStore>
-  get(key: string): Promise<CID | null>
-  list(count: number, after?: string, before?: string): Promise<DataValue[]>
-  listWithPrefix(prefix: string, count?: number): Promise<DataValue[]>
-  diff(other: DataStore): Promise<DataDiff>
-  stage(): Promise<CID>
-  writeToCarStream(car: BlockWriter): Promise<void>
+export type RecordClaim = {
+  collection: string
+  rkey: string
+  record: RepoRecord | null
 }

package/src/util.ts

@@ -1,88 +1,290 @@
 import { CID } from 'multiformats/cid'
-import * as auth from '@atproto/auth'
+import * as cbor from '@ipld/dag-cbor'
+import { CarReader } from '@ipld/car/reader'
+import { BlockWriter, CarWriter } from '@ipld/car/writer'
+import { Block as CarBlock } from '@ipld/car/api'
+import {
+  streamToBuffer,
+  verifyCidForBytes,
+  cborDecode,
+  check,
+  schema,
+  cidForCbor,
+  byteIterableToStream,
+} from '@atproto/common'
+import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon'
+
+import * as crypto from '@atproto/crypto'
 import Repo from './repo'
-import { DataDiff, MST } from './mst'
-import { IpldStore } from './blockstore'
-import { DataStore, RecordWriteOp, def } from './types'
-
-export const ucanForOperation = async (
-  prevData: DataStore,
-  newData: DataStore,
-  rootDid: string,
-  authStore: auth.AuthStore,
-): Promise<string> => {
-  const diff = await prevData.diff(newData)
-  const neededCaps = diff.neededCapabilities(rootDid)
-  const ucanForOp = await authStore.createUcanForCaps(rootDid, neededCaps, 30)
-  return auth.encodeUcan(ucanForOp)
-}
-
-export const getCommitPath = async (
-  blockstore: IpldStore,
-  earliest: CID | null,
-  latest: CID,
-): Promise<CID[] | null> => {
-  let curr: CID | null = latest
-  const path: CID[] = []
-  while (curr !== null) {
-    path.push(curr)
-    const commit = await blockstore.get(curr, def.commit)
-    if (earliest && curr.equals(earliest)) {
-      return path.reverse()
-    }
-    const root = await blockstore.get(commit.root, def.repoRoot)
-    if (!earliest && root.prev === null) {
-      return path.reverse()
+import { MST } from './mst'
+import DataDiff from './data-diff'
+import { RepoStorage } from './storage'
+import {
+  Commit,
+  RecordCreateDescript,
+  RecordDeleteDescript,
+  RecordPath,
+  RecordUpdateDescript,
+  RecordWriteDescript,
+  UnsignedCommit,
+  WriteLog,
+  WriteOpAction,
+} from './types'
+import BlockMap from './block-map'
+import { MissingBlocksError } from './error'
+import * as parse from './parse'
+import { Keypair } from '@atproto/crypto'
+import { Readable } from 'stream'
+
+export async function* verifyIncomingCarBlocks(
+  car: AsyncIterable<CarBlock>,
+): AsyncIterable<CarBlock> {
+  for await (const block of car) {
+    await verifyCidForBytes(block.cid, block.bytes)
+    yield block
+  }
+}
+
+// we have to turn the car writer output into a stream in order to properly handle errors
+export function writeCarStream(
+  root: CID | null,
+  fn: (car: BlockWriter) => Promise<void>,
+): Readable {
+  const { writer, out } =
+    root !== null ? CarWriter.create(root) : CarWriter.create()
+
+  const stream = byteIterableToStream(out)
+  fn(writer)
+    .catch((err) => {
+      stream.destroy(err)
+    })
+    .finally(() => writer.close())
+  return stream
+}
+
+export async function* writeCar(
+  root: CID | null,
+  fn: (car: BlockWriter) => Promise<void>,
+): AsyncIterable<Uint8Array> {
+  const stream = writeCarStream(root, fn)
+  for await (const chunk of stream) {
+    yield chunk
+  }
+}
+
+export const blocksToCarStream = (
+  root: CID | null,
+  blocks: BlockMap,
+): AsyncIterable<Uint8Array> => {
+  return writeCar(root, async (writer) => {
+    for (const entry of blocks.entries()) {
+      await writer.put(entry)
     }
-    curr = root.prev
+  })
+}
+
+export const blocksToCarFile = (
+  root: CID | null,
+  blocks: BlockMap,
+): Promise<Uint8Array> => {
+  const carStream = blocksToCarStream(root, blocks)
+  return streamToBuffer(carStream)
+}
+
+export const readCar = async (
+  bytes: Uint8Array,
+): Promise<{ roots: CID[]; blocks: BlockMap }> => {
+  const car = await CarReader.fromBytes(bytes)
+  const roots = await car.getRoots()
+  const blocks = new BlockMap()
+  for await (const block of verifyIncomingCarBlocks(car.blocks())) {
+    await blocks.set(block.cid, block.bytes)
+  }
+  return {
+    roots,
+    blocks,
   }
-  return null
 }
 
-export const getWriteOpLog = async (
-  blockstore: IpldStore,
-  earliest: CID | null,
+export const readCarWithRoot = async (
+  bytes: Uint8Array,
+): Promise<{ root: CID; blocks: BlockMap }> => {
+  const { roots, blocks } = await readCar(bytes)
+  if (roots.length !== 1) {
+    throw new Error(`Expected one root, got ${roots.length}`)
+  }
+  const root = roots[0]
+  return {
+    root,
+    blocks,
+  }
+}
+
+export const getWriteLog = async (
+  storage: RepoStorage,
   latest: CID,
-): Promise<RecordWriteOp[][]> => {
-  const commits = await getCommitPath(blockstore, earliest, latest)
+  earliest: CID | null,
+): Promise<WriteLog> => {
+  const commits = await storage.getCommitPath(latest, earliest)
   if (!commits) throw new Error('Could not find shared history')
-  const heads = await Promise.all(commits.map((c) => Repo.load(blockstore, c)))
+  const heads = await Promise.all(commits.map((c) => Repo.load(storage, c)))
   // Turn commit path into list of diffs
-  let prev: DataStore = await MST.create(blockstore) // Empty
+  let prev = await MST.create(storage) // Empty
   const msts = heads.map((h) => h.data)
   const diffs: DataDiff[] = []
   for (const mst of msts) {
-    diffs.push(await prev.diff(mst))
+    diffs.push(await DataDiff.of(mst, prev))
     prev = mst
   }
+  const fullDiff = collapseDiffs(diffs)
+  const diffBlocks = await storage.getBlocks(fullDiff.newCidList())
+  if (diffBlocks.missing.length > 0) {
+    throw new MissingBlocksError('write op log', diffBlocks.missing)
+  }
   // Map MST diffs to write ops
-  return Promise.all(diffs.map((diff) => diffToWriteOps(blockstore, diff)))
+  return Promise.all(
+    diffs.map((diff) => diffToWriteDescripts(diff, diffBlocks.blocks)),
+  )
 }
 
-export const diffToWriteOps = (
-  blockstore: IpldStore,
+export const diffToWriteDescripts = (
   diff: DataDiff,
-): Promise<RecordWriteOp[]> => {
+  blocks: BlockMap,
+): Promise<RecordWriteDescript[]> => {
   return Promise.all([
     ...diff.addList().map(async (add) => {
-      const { collection, rkey } = parseRecordKey(add.key)
-      const value = await blockstore.getUnchecked(add.cid)
-      return { action: 'create' as const, collection, rkey, value }
+      const { collection, rkey } = parseDataKey(add.key)
+      const value = await parse.getAndParseRecord(blocks, add.cid)
+      return {
+        action: WriteOpAction.Create,
+        collection,
+        rkey,
+        cid: add.cid,
+        record: value.record,
+      } as RecordCreateDescript
     }),
     ...diff.updateList().map(async (upd) => {
-      const { collection, rkey } = parseRecordKey(upd.key)
-      const value = await blockstore.getUnchecked(upd.cid)
-      return { action: 'update' as const, collection, rkey, value }
+      const { collection, rkey } = parseDataKey(upd.key)
+      const value = await parse.getAndParseRecord(blocks, upd.cid)
+      return {
+        action: WriteOpAction.Update,
+        collection,
+        rkey,
+        cid: upd.cid,
+        prev: upd.prev,
+        record: value.record,
+      } as RecordUpdateDescript
     }),
     ...diff.deleteList().map((del) => {
-      const { collection, rkey } = parseRecordKey(del.key)
-      return { action: 'delete' as const, collection, rkey }
+      const { collection, rkey } = parseDataKey(del.key)
+      return {
+        action: WriteOpAction.Delete,
+        collection,
+        rkey,
+        cid: del.cid,
+      } as RecordDeleteDescript
     }),
   ])
 }
 
-export const parseRecordKey = (key: string) => {
+export const collapseWriteLog = (log: WriteLog): RecordWriteDescript[] => {
+  const creates: Record<string, RecordCreateDescript> = {}
+  const updates: Record<string, RecordUpdateDescript> = {}
+  const deletes: Record<string, RecordDeleteDescript> = {}
+  for (const commit of log) {
+    for (const op of commit) {
+      const key = op.collection + '/' + op.rkey
+      if (op.action === WriteOpAction.Create) {
+        const del = deletes[key]
+        if (del) {
+          if (del.cid !== op.cid) {
+            updates[key] = {
+              ...op,
+              action: WriteOpAction.Update,
+              prev: del.cid,
+            }
+          }
+          delete deletes[key]
+        } else {
+          creates[key] = op
+        }
+      } else if (op.action === WriteOpAction.Update) {
+        updates[key] = op
+        delete creates[key]
+        delete deletes[key]
+      } else if (op.action === WriteOpAction.Delete) {
+        if (creates[key]) {
+          delete creates[key]
+        } else {
+          delete updates[key]
+          deletes[key] = op
+        }
+      } else {
+        throw new Error(`unknown action: ${op}`)
+      }
+    }
+  }
+  return [
+    ...Object.values(creates),
+    ...Object.values(updates),
+    ...Object.values(deletes),
+  ]
+}
+
+export const collapseDiffs = (diffs: DataDiff[]): DataDiff => {
+  return diffs.reduce((acc, cur) => {
+    acc.addDiff(cur)
+    return acc
+  }, new DataDiff())
+}
+
+export const parseDataKey = (key: string): RecordPath => {
   const parts = key.split('/')
   if (parts.length !== 2) throw new Error(`Invalid record key: ${key}`)
   return { collection: parts[0], rkey: parts[1] }
 }
+
+export const formatDataKey = (collection: string, rkey: string): string => {
+  return collection + '/' + rkey
+}
+
+export const metaEqual = (a: Commit, b: Commit): boolean => {
+  return a.did === b.did && a.version === b.version
+}
+
+export const signCommit = async (
+  unsigned: UnsignedCommit,
+  keypair: Keypair,
+): Promise<Commit> => {
+  const encoded = cbor.encode(unsigned)
+  const sig = await keypair.sign(encoded)
+  return {
+    ...unsigned,
+    sig,
+  }
+}
+
+export const verifyCommitSig = async (
+  commit: Commit,
+  didKey: string,
+): Promise<boolean> => {
+  const { sig, ...rest } = commit
+  const encoded = cbor.encode(rest)
+  return crypto.verifySignature(didKey, encoded, sig)
+}
+
+export const cborToLex = (val: Uint8Array): LexValue => {
+  return ipldToLex(cborDecode(val))
+}
+
+export const cborToLexRecord = (val: Uint8Array): RepoRecord => {
+  const parsed = cborToLex(val)
+  if (!check.is(parsed, schema.map)) {
+    throw new Error('lexicon records be a json object')
+  }
+  return parsed
+}
+
+export const cidForRecord = async (val: LexValue) => {
+  return cidForCbor(lexToIpld(val))
+}