@atproto/repo 0.0.1 → 0.2.0

package/dist/util.d.ts

@@ -1,13 +1,36 @@
+/// <reference types="node" />
 import { CID } from 'multiformats/cid';
-import * as auth from '@atproto/auth';
-import { DataDiff } from './mst';
-import { IpldStore } from './blockstore';
-import { DataStore, RecordWriteOp } from './types';
-export declare const ucanForOperation: (prevData: DataStore, newData: DataStore, rootDid: string, authStore: auth.AuthStore) => Promise<string>;
-export declare const getCommitPath: (blockstore: IpldStore, earliest: CID | null, latest: CID) => Promise<CID[] | null>;
-export declare const getWriteOpLog: (blockstore: IpldStore, earliest: CID | null, latest: CID) => Promise<RecordWriteOp[][]>;
-export declare const diffToWriteOps: (blockstore: IpldStore, diff: DataDiff) => Promise<RecordWriteOp[]>;
-export declare const parseRecordKey: (key: string) => {
-    collection: string;
-    rkey: string;
-};
+import { BlockWriter } from '@ipld/car/writer';
+import { Block as CarBlock } from '@ipld/car/api';
+import { LexValue, RepoRecord } from '@atproto/lexicon';
+import DataDiff from './data-diff';
+import { RepoStorage } from './storage';
+import { Commit, RecordPath, RecordWriteDescript, UnsignedCommit, WriteLog } from './types';
+import BlockMap from './block-map';
+import { Keypair } from '@atproto/crypto';
+import { Readable } from 'stream';
+export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>): AsyncIterable<CarBlock>;
+export declare function writeCarStream(root: CID | null, fn: (car: BlockWriter) => Promise<void>): Readable;
+export declare function writeCar(root: CID | null, fn: (car: BlockWriter) => Promise<void>): AsyncIterable<Uint8Array>;
+export declare const blocksToCarStream: (root: CID | null, blocks: BlockMap) => AsyncIterable<Uint8Array>;
+export declare const blocksToCarFile: (root: CID | null, blocks: BlockMap) => Promise<Uint8Array>;
+export declare const readCar: (bytes: Uint8Array) => Promise<{
+    roots: CID[];
+    blocks: BlockMap;
+}>;
+export declare const readCarWithRoot: (bytes: Uint8Array) => Promise<{
+    root: CID;
+    blocks: BlockMap;
+}>;
+export declare const getWriteLog: (storage: RepoStorage, latest: CID, earliest: CID | null) => Promise<WriteLog>;
+export declare const diffToWriteDescripts: (diff: DataDiff, blocks: BlockMap) => Promise<RecordWriteDescript[]>;
+export declare const collapseWriteLog: (log: WriteLog) => RecordWriteDescript[];
+export declare const collapseDiffs: (diffs: DataDiff[]) => DataDiff;
+export declare const parseDataKey: (key: string) => RecordPath;
+export declare const formatDataKey: (collection: string, rkey: string) => string;
+export declare const metaEqual: (a: Commit, b: Commit) => boolean;
+export declare const signCommit: (unsigned: UnsignedCommit, keypair: Keypair) => Promise<Commit>;
+export declare const verifyCommitSig: (commit: Commit, didKey: string) => Promise<boolean>;
+export declare const cborToLex: (val: Uint8Array) => LexValue;
+export declare const cborToLexRecord: (val: Uint8Array) => RepoRecord;
+export declare const cidForRecord: (val: LexValue) => Promise<CID>;

package/dist/verify.d.ts

@@ -1,5 +1,32 @@
 import { CID } from 'multiformats/cid';
-import * as auth from '@atproto/auth';
-import { IpldStore } from './blockstore';
-import { DataDiff } from './mst';
-export declare const verifyUpdates: (blockstore: IpldStore, earliest: CID | null, latest: CID, verifier: auth.Verifier) => Promise<DataDiff>;
+import { ReadableBlockstore, RepoStorage } from './storage';
+import DataDiff from './data-diff';
+import ReadableRepo from './readable-repo';
+import CidSet from './cid-set';
+import { RecordClaim, RepoContents, RepoContentsWithCids } from './types';
+export declare type VerifiedCheckout = {
+    contents: RepoContents;
+    newCids: CidSet;
+};
+export declare type VerifiedCheckoutWithCids = {
+    commit: CID;
+    contents: RepoContentsWithCids;
+};
+export declare const verifyCheckout: (storage: ReadableBlockstore, head: CID, did: string, signingKey: string) => Promise<VerifiedCheckout>;
+export declare const verifyCheckoutWithCids: (storage: ReadableBlockstore, head: CID, did: string, signingKey: string) => Promise<VerifiedCheckoutWithCids>;
+export declare type VerifiedUpdate = {
+    commit: CID;
+    prev: CID | null;
+    diff: DataDiff;
+    newCids: CidSet;
+};
+export declare const verifyFullHistory: (storage: RepoStorage, head: CID, did: string, signingKey: string) => Promise<VerifiedUpdate[]>;
+export declare const verifyUpdates: (repo: ReadableRepo, updateStorage: RepoStorage, updateRoot: CID, did: string, signingKey: string) => Promise<VerifiedUpdate[]>;
+export declare const verifyCommitPath: (baseRepo: ReadableRepo, storage: ReadableBlockstore, commitPath: CID[], did: string, signingKey: string) => Promise<VerifiedUpdate[]>;
+export declare const verifyProofs: (proofs: Uint8Array, claims: RecordClaim[], did: string, didKey: string) => Promise<{
+    verified: RecordClaim[];
+    unverified: RecordClaim[];
+}>;
+export declare const verifyRecords: (proofs: Uint8Array, did: string, signingKey: string) => Promise<RecordClaim[]>;
+export declare class RepoVerificationError extends Error {
+}

package/jest.bench.config.js

@@ -2,6 +2,7 @@ const base = require('./jest.config')
 
 module.exports = {
   ...base,
-  testRegex: '(/bench/.*.bench)',
+  roots: ['<rootDir>/bench'],
+  testRegex: '(.*.bench)',
   testTimeout: 3000000,
 }

package/package.json

@@ -1,15 +1,20 @@
 {
   "name": "@atproto/repo",
-  "version": "0.0.1",
+  "version": "0.2.0",
   "main": "dist/index.js",
   "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bluesky-social/atproto.git",
+    "directory": "packages/repo"
+  },
   "scripts": {
     "test": "jest",
     "test:profile": "node --inspect ../../node_modules/.bin/jest",
     "bench": "jest --config jest.bench.config.js",
-    "bench:profile": "node --inspect ../../node_modules/.bin/jest --config jest.bench.config.js",
-    "prettier": "prettier --check src/",
-    "prettier:fix": "prettier --write src/",
+    "bench:profile": "node --inspect-brk ../../node_modules/.bin/jest --config jest.bench.config.js",
+    "prettier": "prettier --check src/ tests/",
+    "prettier:fix": "prettier --write src/ tests/",
     "lint": "eslint . --ext .ts,.tsx",
     "lint:fix": "yarn lint --fix",
     "verify": "run-p prettier lint",
@@ -22,13 +27,15 @@
     "postpublish": "npm run update-main-to-src"
   },
   "dependencies": {
-    "@atproto/auth": "*",
     "@atproto/common": "*",
+    "@atproto/crypto": "*",
+    "@atproto/identity": "*",
+    "@atproto/lexicon": "*",
     "@atproto/nsid": "*",
     "@ipld/car": "^3.2.3",
     "@ipld/dag-cbor": "^7.0.0",
     "multiformats": "^9.6.4",
     "uint8arrays": "3.0.0",
-    "zod": "^3.14.2"
+    "zod": "^3.21.4"
   }
 }

package/src/block-map.ts

@@ -0,0 +1,103 @@
+import { lexToIpld, LexValue } from '@atproto/lexicon'
+import { dataToCborBlock } from '@atproto/common'
+import { CID } from 'multiformats/cid'
+import * as uint8arrays from 'uint8arrays'
+
+export class BlockMap {
+  private map: Map<string, Uint8Array> = new Map()
+
+  async add(value: LexValue): Promise<CID> {
+    const block = await dataToCborBlock(lexToIpld(value))
+    this.set(block.cid, block.bytes)
+    return block.cid
+  }
+
+  set(cid: CID, bytes: Uint8Array) {
+    this.map.set(cid.toString(), bytes)
+  }
+
+  get(cid: CID): Uint8Array | undefined {
+    return this.map.get(cid.toString())
+  }
+
+  delete(cid: CID) {
+    this.map.delete(cid.toString())
+  }
+
+  getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } {
+    const missing: CID[] = []
+    const blocks = new BlockMap()
+    for (const cid of cids) {
+      const got = this.map.get(cid.toString())
+      if (got) {
+        blocks.set(cid, got)
+      } else {
+        missing.push(cid)
+      }
+    }
+    return { blocks, missing }
+  }
+
+  has(cid: CID): boolean {
+    return this.map.has(cid.toString())
+  }
+
+  clear(): void {
+    this.map.clear()
+  }
+
+  forEach(cb: (bytes: Uint8Array, cid: CID) => void): void {
+    this.map.forEach((val, key) => cb(val, CID.parse(key)))
+  }
+
+  entries(): Entry[] {
+    const entries: Entry[] = []
+    this.forEach((bytes, cid) => {
+      entries.push({ cid, bytes })
+    })
+    return entries
+  }
+
+  cids(): CID[] {
+    return this.entries().map((e) => e.cid)
+  }
+
+  addMap(toAdd: BlockMap) {
+    toAdd.forEach((bytes, cid) => {
+      this.set(cid, bytes)
+    })
+  }
+
+  get size(): number {
+    return this.map.size
+  }
+
+  get byteSize(): number {
+    let size = 0
+    this.forEach((bytes) => {
+      size += bytes.length
+    })
+    return size
+  }
+
+  equals(other: BlockMap): boolean {
+    if (this.size !== other.size) {
+      return false
+    }
+    for (const entry of this.entries()) {
+      const otherBytes = other.get(entry.cid)
+      if (!otherBytes) return false
+      if (!uint8arrays.equals(entry.bytes, otherBytes)) {
+        return false
+      }
+    }
+    return true
+  }
+}
+
+type Entry = {
+  cid: CID
+  bytes: Uint8Array
+}
+
+export default BlockMap

package/src/cid-set.ts

@@ -42,8 +42,7 @@ export class CidSet {
   }
 
   toList(): CID[] {
-    const arr = [...this.set]
-    return arr.map((c) => CID.parse(c))
+    return [...this.set].map((c) => CID.parse(c))
   }
 }
 

package/src/data-diff.ts

@@ -0,0 +1,117 @@
+import { CID } from 'multiformats'
+import CidSet from './cid-set'
+import { MST, mstDiff } from './mst'
+
+export class DataDiff {
+  adds: Record<string, DataAdd> = {}
+  updates: Record<string, DataUpdate> = {}
+  deletes: Record<string, DataDelete> = {}
+
+  newCids: CidSet = new CidSet()
+  removedCids: CidSet = new CidSet()
+
+  static async of(curr: MST, prev: MST | null): Promise<DataDiff> {
+    return mstDiff(curr, prev)
+  }
+
+  recordAdd(key: string, cid: CID): void {
+    this.adds[key] = { key, cid }
+    this.newCids.add(cid)
+  }
+
+  recordUpdate(key: string, prev: CID, cid: CID): void {
+    this.updates[key] = { key, prev, cid }
+    this.newCids.add(cid)
+  }
+
+  recordDelete(key: string, cid: CID): void {
+    this.deletes[key] = { key, cid }
+  }
+
+  recordNewCid(cid: CID): void {
+    if (this.removedCids.has(cid)) {
+      this.removedCids.delete(cid)
+    } else {
+      this.newCids.add(cid)
+    }
+  }
+
+  recordRemovedCid(cid: CID): void {
+    if (this.newCids.has(cid)) {
+      this.newCids.delete(cid)
+    } else {
+      this.removedCids.add(cid)
+    }
+  }
+
+  addDiff(diff: DataDiff) {
+    for (const add of diff.addList()) {
+      if (this.deletes[add.key]) {
+        const del = this.deletes[add.key]
+        if (del.cid !== add.cid) {
+          this.recordUpdate(add.key, del.cid, add.cid)
+        }
+        delete this.deletes[add.key]
+      } else {
+        this.recordAdd(add.key, add.cid)
+      }
+    }
+    for (const update of diff.updateList()) {
+      this.recordUpdate(update.key, update.prev, update.cid)
+      delete this.adds[update.key]
+      delete this.deletes[update.key]
+    }
+    for (const del of diff.deleteList()) {
+      if (this.adds[del.key]) {
+        delete this.adds[del.key]
+      } else {
+        delete this.updates[del.key]
+        this.recordDelete(del.key, del.cid)
+      }
+    }
+    this.newCids.addSet(diff.newCids)
+  }
+
+  addList(): DataAdd[] {
+    return Object.values(this.adds)
+  }
+
+  updateList(): DataUpdate[] {
+    return Object.values(this.updates)
+  }
+
+  deleteList(): DataDelete[] {
+    return Object.values(this.deletes)
+  }
+
+  newCidList(): CID[] {
+    return this.newCids.toList()
+  }
+
+  updatedKeys(): string[] {
+    const keys = [
+      ...Object.keys(this.adds),
+      ...Object.keys(this.updates),
+      ...Object.keys(this.deletes),
+    ]
+    return [...new Set(keys)]
+  }
+}
+
+export type DataAdd = {
+  key: string
+  cid: CID
+}
+
+export type DataUpdate = {
+  key: string
+  prev: CID
+  cid: CID
+}
+
+export type DataDelete = {
+  key: string
+  cid: CID
+}
+
+export default DataDiff

package/src/error.ts

@@ -0,0 +1,31 @@
+import { CID } from 'multiformats/cid'
+
+export class MissingBlockError extends Error {
+  constructor(public cid: CID, def?: string) {
+    let msg = `block not found: ${cid.toString()}`
+    if (def) {
+      msg += `, expected type: ${def}`
+    }
+    super(msg)
+  }
+}
+
+export class MissingBlocksError extends Error {
+  constructor(public context: string, public cids: CID[]) {
+    const cidStr = cids.map((c) => c.toString())
+    super(`missing ${context} blocks: ${cidStr}`)
+  }
+}
+
+export class MissingCommitBlocksError extends Error {
+  constructor(public commit: CID, public cids: CID[]) {
+    const cidStr = cids.map((c) => c.toString())
+    super(`missing blocks for commit ${commit.toString()}: ${cidStr}`)
+  }
+}
+
+export class UnexpectedObjectError extends Error {
+  constructor(public cid: CID, public def: string) {
+    super(`unexpected object at ${cid.toString()}, expected: ${def}`)
+  }
+}

package/src/index.ts

@@ -1,7 +1,10 @@
-export * from './blockstore'
+export * from './block-map'
+export * from './cid-set'
 export * from './repo'
 export * from './mst'
 export * from './storage'
+export * from './sync'
 export * from './types'
 export * from './verify'
+export * from './data-diff'
 export * from './util'

package/src/mst/diff.ts

@@ -1,106 +1,136 @@
-import * as auth from '@atproto/auth'
-import { CID } from 'multiformats'
-import CidSet from '../cid-set'
-import { parseRecordKey } from '../util'
-
-export class DataDiff {
-  adds: Record<string, DataAdd> = {}
-  updates: Record<string, DataUpdate> = {}
-  deletes: Record<string, DataDelete> = {}
-
-  newCids: CidSet = new CidSet()
-
-  recordAdd(key: string, cid: CID): void {
-    this.adds[key] = { key, cid }
-    this.newCids.add(cid)
-  }
-
-  recordUpdate(key: string, prev: CID, cid: CID): void {
-    this.updates[key] = { key, prev, cid }
-    this.newCids.add(cid)
-  }
-
-  recordDelete(key: string, cid: CID): void {
-    this.deletes[key] = { key, cid }
+import { DataDiff } from '../data-diff'
+import MST from './mst'
+import MstWalker from './walker'
+
+export const nullDiff = async (tree: MST): Promise<DataDiff> => {
+  const diff = new DataDiff()
+  for await (const entry of tree.walk()) {
+    if (entry.isLeaf()) {
+      diff.recordAdd(entry.key, entry.value)
+    } else {
+      diff.recordNewCid(entry.pointer)
+    }
   }
+  return diff
+}
 
-  recordNewCid(cid: CID): void {
-    this.newCids.add(cid)
+export const mstDiff = async (
+  curr: MST,
+  prev: MST | null,
+): Promise<DataDiff> => {
+  await curr.getPointer()
+  if (prev === null) {
+    return nullDiff(curr)
   }
 
-  addDiff(diff: DataDiff) {
-    for (const add of diff.addList()) {
-      if (this.deletes[add.key]) {
-        const del = this.deletes[add.key]
-        if (del.cid !== add.cid) {
-          this.recordUpdate(add.key, del.cid, add.cid)
-        }
-        delete this.deletes[add.key]
+  await prev.getPointer()
+  const diff = new DataDiff()
+
+  const leftWalker = new MstWalker(prev)
+  const rightWalker = new MstWalker(curr)
+  while (!leftWalker.status.done || !rightWalker.status.done) {
+    // if one walker is finished, continue walking the other & logging all nodes
+    if (leftWalker.status.done && !rightWalker.status.done) {
+      const node = rightWalker.status.curr
+      if (node.isLeaf()) {
+        diff.recordAdd(node.key, node.value)
       } else {
-        this.recordAdd(add.key, add.cid)
+        diff.recordNewCid(node.pointer)
       }
+      await rightWalker.advance()
+      continue
+    } else if (!leftWalker.status.done && rightWalker.status.done) {
+      const node = leftWalker.status.curr
+      if (node.isLeaf()) {
+        diff.recordDelete(node.key, node.value)
+      } else {
+        diff.recordRemovedCid(node.pointer)
+      }
+      await leftWalker.advance()
+      continue
     }
-    for (const update of diff.updateList()) {
-      this.recordUpdate(update.key, update.prev, update.cid)
-      delete this.adds[update.key]
-      delete this.deletes[update.key]
-    }
-    for (const del of diff.deleteList()) {
-      if (this.adds[del.key]) {
-        delete this.adds[del.key]
+    if (leftWalker.status.done || rightWalker.status.done) break
+    const left = leftWalker.status.curr
+    const right = rightWalker.status.curr
+    if (left === null || right === null) break
+
+    // if both pointers are leaves, record an update & advance both or record the lowest key and advance that pointer
+    if (left.isLeaf() && right.isLeaf()) {
+      if (left.key === right.key) {
+        if (!left.value.equals(right.value)) {
+          diff.recordUpdate(left.key, left.value, right.value)
+        }
+        await leftWalker.advance()
+        await rightWalker.advance()
+      } else if (left.key < right.key) {
+        diff.recordDelete(left.key, left.value)
+        await leftWalker.advance()
       } else {
-        delete this.updates[del.key]
-        this.recordDelete(del.key, del.cid)
+        diff.recordAdd(right.key, right.value)
+        await rightWalker.advance()
       }
+      continue
     }
-    this.newCids.addSet(diff.newCids)
-  }
 
-  addList(): DataAdd[] {
-    return Object.values(this.adds)
-  }
-
-  updateList(): DataUpdate[] {
-    return Object.values(this.updates)
-  }
-
-  deleteList(): DataDelete[] {
-    return Object.values(this.deletes)
-  }
+    // next, ensure that we're on the same layer
+    // if one walker is at a higher layer than the other, we need to do one of two things
+    // if the higher walker is pointed at a tree, step into that tree to try to catch up with the lower
+    // if the higher walker is pointed at a leaf, then advance the lower walker to try to catch up the higher
+    if (leftWalker.layer() > rightWalker.layer()) {
+      if (left.isLeaf()) {
+        if (right.isLeaf()) {
+          diff.recordAdd(right.key, right.value)
+        } else {
+          diff.recordNewCid(right.pointer)
+        }
+        await rightWalker.advance()
+      } else {
+        diff.recordRemovedCid(left.pointer)
+        await leftWalker.stepInto()
+      }
+      continue
+    } else if (leftWalker.layer() < rightWalker.layer()) {
+      if (right.isLeaf()) {
+        if (left.isLeaf()) {
+          diff.recordDelete(left.key, left.value)
+        } else {
+          diff.recordRemovedCid(left.pointer)
+        }
+        await leftWalker.advance()
+      } else {
+        diff.recordNewCid(right.pointer)
+        await rightWalker.stepInto()
+      }
+      continue
+    }
 
-  newCidList(): CID[] {
-    return this.newCids.toList()
-  }
+    // if we're on the same level, and both pointers are trees, do a comparison
+    // if they're the same, step over. if they're different, step in to find the subdiff
+    if (left.isTree() && right.isTree()) {
+      if (left.pointer.equals(right.pointer)) {
+        await leftWalker.stepOver()
+        await rightWalker.stepOver()
+      } else {
+        diff.recordNewCid(right.pointer)
+        diff.recordRemovedCid(left.pointer)
+        await leftWalker.stepInto()
+        await rightWalker.stepInto()
+      }
+      continue
+    }
 
-  updatedKeys(): string[] {
-    const keys = [
-      ...Object.keys(this.adds),
-      ...Object.keys(this.updates),
-      ...Object.keys(this.deletes),
-    ]
-    return [...new Set(keys)]
-  }
+    // finally, if one pointer is a tree and the other is a leaf, simply step into the tree
+    if (left.isLeaf() && right.isTree()) {
+      diff.recordNewCid(right.pointer)
+      await rightWalker.stepInto()
+      continue
+    } else if (left.isTree() && right.isLeaf()) {
+      diff.recordRemovedCid(left.pointer)
+      await leftWalker.stepInto()
+      continue
+    }
 
-  neededCapabilities(rootDid: string): auth.ucans.Capability[] {
-    return this.updatedKeys().map((key) => {
-      const { collection, rkey } = parseRecordKey(key)
-      return auth.writeCap(rootDid, collection, rkey)
-    })
+    throw new Error('Unidentifiable case in diff walk')
   }
-}
-
-export type DataAdd = {
-  key: string
-  cid: CID
-}
-
-export type DataUpdate = {
-  key: string
-  prev: CID
-  cid: CID
-}
-
-export type DataDelete = {
-  key: string
-  cid: CID
+  return diff
 }