@atproto/repo 0.1.0 → 0.3.0

package/jest.bench.config.js

@@ -2,6 +2,7 @@ const base = require('./jest.config')
 
 module.exports = {
   ...base,
-  testRegex: '(/bench/.*.bench)',
+  roots: ['<rootDir>/bench'],
+  testRegex: '(.*.bench)',
   testTimeout: 3000000,
 }

package/package.json

@@ -1,15 +1,20 @@
 {
   "name": "@atproto/repo",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "main": "dist/index.js",
   "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bluesky-social/atproto.git",
+    "directory": "packages/repo"
+  },
   "scripts": {
     "test": "jest",
     "test:profile": "node --inspect ../../node_modules/.bin/jest",
     "bench": "jest --config jest.bench.config.js",
-    "bench:profile": "node --inspect ../../node_modules/.bin/jest --config jest.bench.config.js",
-    "prettier": "prettier --check src/",
-    "prettier:fix": "prettier --write src/",
+    "bench:profile": "node --inspect-brk ../../node_modules/.bin/jest --config jest.bench.config.js",
+    "prettier": "prettier --check src/ tests/",
+    "prettier:fix": "prettier --write src/ tests/",
     "lint": "eslint . --ext .ts,.tsx",
     "lint:fix": "yarn lint --fix",
     "verify": "run-p prettier lint",
@@ -24,13 +29,13 @@
   "dependencies": {
     "@atproto/common": "*",
     "@atproto/crypto": "*",
-    "@atproto/did-resolver": "*",
+    "@atproto/identity": "*",
     "@atproto/lexicon": "*",
-    "@atproto/nsid": "*",
+    "@atproto/syntax": "*",
     "@ipld/car": "^3.2.3",
     "@ipld/dag-cbor": "^7.0.0",
     "multiformats": "^9.6.4",
     "uint8arrays": "3.0.0",
-    "zod": "^3.14.2"
+    "zod": "^3.21.4"
   }
 }

package/src/block-map.ts

@@ -20,6 +20,10 @@ export class BlockMap {
     return this.map.get(cid.toString())
   }
 
+  delete(cid: CID) {
+    this.map.delete(cid.toString())
+  }
+
   getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } {
     const missing: CID[] = []
     const blocks = new BlockMap()
@@ -54,6 +58,10 @@ export class BlockMap {
     return entries
   }
 
+  cids(): CID[] {
+    return this.entries().map((e) => e.cid)
+  }
+
   addMap(toAdd: BlockMap) {
     toAdd.forEach((bytes, cid) => {
       this.set(cid, bytes)

package/src/data-diff.ts

@@ -1,81 +1,83 @@
 import { CID } from 'multiformats'
 import CidSet from './cid-set'
-import { MST, mstDiff } from './mst'
-import { DataStore } from './types'
+import { MST, NodeEntry, mstDiff } from './mst'
+import BlockMap from './block-map'
 
 export class DataDiff {
   adds: Record<string, DataAdd> = {}
   updates: Record<string, DataUpdate> = {}
   deletes: Record<string, DataDelete> = {}
 
-  newCids: CidSet = new CidSet()
+  newMstBlocks: BlockMap = new BlockMap()
+  newLeafCids: CidSet = new CidSet()
   removedCids: CidSet = new CidSet()
 
-  static async of(curr: DataStore, prev: DataStore | null): Promise<DataDiff> {
-    if (curr instanceof MST && (prev === null || prev instanceof MST)) {
-      return mstDiff(curr, prev)
+  static async of(curr: MST, prev: MST | null): Promise<DataDiff> {
+    return mstDiff(curr, prev)
+  }
+
+  async nodeAdd(node: NodeEntry) {
+    if (node.isLeaf()) {
+      this.leafAdd(node.key, node.value)
+    } else {
+      const data = await node.serialize()
+      this.treeAdd(data.cid, data.bytes)
     }
-    throw new Error('Unsupported DataStore type for diff')
   }
 
-  recordAdd(key: string, cid: CID): void {
+  async nodeDelete(node: NodeEntry) {
+    if (node.isLeaf()) {
+      const key = node.key
+      const cid = node.value
+      this.deletes[key] = { key, cid }
+      this.removedCids.add(cid)
+    } else {
+      const cid = await node.getPointer()
+      this.treeDelete(cid)
+    }
+  }
+
+  leafAdd(key: string, cid: CID) {
     this.adds[key] = { key, cid }
-    this.newCids.add(cid)
+    if (this.removedCids.has(cid)) {
+      this.removedCids.delete(cid)
+    } else {
+      this.newLeafCids.add(cid)
+    }
   }
 
-  recordUpdate(key: string, prev: CID, cid: CID): void {
+  leafUpdate(key: string, prev: CID, cid: CID) {
+    if (prev.equals(cid)) return
     this.updates[key] = { key, prev, cid }
-    this.newCids.add(cid)
+    this.removedCids.add(prev)
+    this.newLeafCids.add(cid)
   }
 
-  recordDelete(key: string, cid: CID): void {
+  leafDelete(key: string, cid: CID) {
     this.deletes[key] = { key, cid }
+    if (this.newLeafCids.has(cid)) {
+      this.newLeafCids.delete(cid)
+    } else {
+      this.removedCids.add(cid)
+    }
   }
 
-  recordNewCid(cid: CID): void {
+  treeAdd(cid: CID, bytes: Uint8Array) {
     if (this.removedCids.has(cid)) {
       this.removedCids.delete(cid)
     } else {
-      this.newCids.add(cid)
+      this.newMstBlocks.set(cid, bytes)
     }
   }
 
-  recordRemovedCid(cid: CID): void {
-    if (this.newCids.has(cid)) {
-      this.newCids.delete(cid)
+  treeDelete(cid: CID) {
+    if (this.newMstBlocks.has(cid)) {
+      this.newMstBlocks.delete(cid)
     } else {
       this.removedCids.add(cid)
     }
   }
 
-  addDiff(diff: DataDiff) {
-    for (const add of diff.addList()) {
-      if (this.deletes[add.key]) {
-        const del = this.deletes[add.key]
-        if (del.cid !== add.cid) {
-          this.recordUpdate(add.key, del.cid, add.cid)
-        }
-        delete this.deletes[add.key]
-      } else {
-        this.recordAdd(add.key, add.cid)
-      }
-    }
-    for (const update of diff.updateList()) {
-      this.recordUpdate(update.key, update.prev, update.cid)
-      delete this.adds[update.key]
-      delete this.deletes[update.key]
-    }
-    for (const del of diff.deleteList()) {
-      if (this.adds[del.key]) {
-        delete this.adds[del.key]
-      } else {
-        delete this.updates[del.key]
-        this.recordDelete(del.key, del.cid)
-      }
-    }
-    this.newCids.addSet(diff.newCids)
-  }
-
   addList(): DataAdd[] {
     return Object.values(this.adds)
   }
@@ -88,10 +90,6 @@ export class DataDiff {
     return Object.values(this.deletes)
   }
 
-  newCidList(): CID[] {
-    return this.newCids.toList()
-  }
-
   updatedKeys(): string[] {
     const keys = [
       ...Object.keys(this.adds),

package/src/index.ts

@@ -5,5 +5,5 @@ export * from './mst'
 export * from './storage'
 export * from './sync'
 export * from './types'
-export * from './verify'
+export * from './data-diff'
 export * from './util'

package/src/mst/diff.ts

@@ -5,11 +5,7 @@ import MstWalker from './walker'
 export const nullDiff = async (tree: MST): Promise<DataDiff> => {
   const diff = new DataDiff()
   for await (const entry of tree.walk()) {
-    if (entry.isLeaf()) {
-      diff.recordAdd(entry.key, entry.value)
-    } else {
-      diff.recordNewCid(entry.pointer)
-    }
+    await diff.nodeAdd(entry)
   }
   return diff
 }
@@ -31,21 +27,11 @@ export const mstDiff = async (
   while (!leftWalker.status.done || !rightWalker.status.done) {
     // if one walker is finished, continue walking the other & logging all nodes
     if (leftWalker.status.done && !rightWalker.status.done) {
-      const node = rightWalker.status.curr
-      if (node.isLeaf()) {
-        diff.recordAdd(node.key, node.value)
-      } else {
-        diff.recordNewCid(node.pointer)
-      }
+      await diff.nodeAdd(rightWalker.status.curr)
       await rightWalker.advance()
       continue
     } else if (!leftWalker.status.done && rightWalker.status.done) {
-      const node = leftWalker.status.curr
-      if (node.isLeaf()) {
-        diff.recordDelete(node.key, node.value)
-      } else {
-        diff.recordRemovedCid(node.pointer)
-      }
+      await diff.nodeDelete(leftWalker.status.curr)
       await leftWalker.advance()
       continue
     }
@@ -58,15 +44,15 @@ export const mstDiff = async (
     if (left.isLeaf() && right.isLeaf()) {
       if (left.key === right.key) {
         if (!left.value.equals(right.value)) {
-          diff.recordUpdate(left.key, left.value, right.value)
+          diff.leafUpdate(left.key, left.value, right.value)
         }
         await leftWalker.advance()
         await rightWalker.advance()
       } else if (left.key < right.key) {
-        diff.recordDelete(left.key, left.value)
+        diff.leafDelete(left.key, left.value)
         await leftWalker.advance()
       } else {
-        diff.recordAdd(right.key, right.value)
+        diff.leafAdd(right.key, right.value)
         await rightWalker.advance()
       }
       continue
@@ -78,27 +64,19 @@ export const mstDiff = async (
     // if the higher walker is pointed at a leaf, then advance the lower walker to try to catch up the higher
     if (leftWalker.layer() > rightWalker.layer()) {
       if (left.isLeaf()) {
-        if (right.isLeaf()) {
-          diff.recordAdd(right.key, right.value)
-        } else {
-          diff.recordNewCid(right.pointer)
-        }
+        await diff.nodeAdd(right)
         await rightWalker.advance()
       } else {
-        diff.recordRemovedCid(left.pointer)
+        await diff.nodeDelete(left)
         await leftWalker.stepInto()
       }
       continue
     } else if (leftWalker.layer() < rightWalker.layer()) {
       if (right.isLeaf()) {
-        if (left.isLeaf()) {
-          diff.recordDelete(left.key, left.value)
-        } else {
-          diff.recordRemovedCid(left.pointer)
-        }
+        await diff.nodeDelete(left)
         await leftWalker.advance()
       } else {
-        diff.recordNewCid(right.pointer)
+        await diff.nodeAdd(right)
         await rightWalker.stepInto()
       }
       continue
@@ -111,8 +89,8 @@ export const mstDiff = async (
         await leftWalker.stepOver()
         await rightWalker.stepOver()
       } else {
-        diff.recordNewCid(right.pointer)
-        diff.recordRemovedCid(left.pointer)
+        await diff.nodeAdd(right)
+        await diff.nodeDelete(left)
         await leftWalker.stepInto()
         await rightWalker.stepInto()
       }
@@ -121,11 +99,11 @@ export const mstDiff = async (
 
     // finally, if one pointer is a tree and the other is a leaf, simply step into the tree
     if (left.isLeaf() && right.isTree()) {
-      diff.recordNewCid(right.pointer)
+      await diff.nodeAdd(right)
       await rightWalker.stepInto()
       continue
     } else if (left.isTree() && right.isLeaf()) {
-      diff.recordRemovedCid(left.pointer)
+      await diff.nodeDelete(left)
       await leftWalker.stepInto()
       continue
     }

package/src/mst/mst.ts

@@ -2,8 +2,7 @@ import z from 'zod'
 import { CID } from 'multiformats'
 
 import { ReadableBlockstore } from '../storage'
-import { schema as common, cidForCbor } from '@atproto/common'
-import { DataStore } from '../types'
+import { schema as common, cidForCbor, dataToCborBlock } from '@atproto/common'
 import { BlockWriter } from '@ipld/car/api'
 import * as util from './util'
 import BlockMap from '../block-map'
@@ -68,7 +67,7 @@ export type MstOpts = {
   layer: number
 }
 
-export class MST implements DataStore {
+export class MST {
   storage: ReadableBlockstore
   entries: NodeEntry[] | null
   layer: number | null
@@ -154,6 +153,13 @@ export class MST implements DataStore {
   // Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed
   async getPointer(): Promise<CID> {
     if (!this.outdatedPointer) return this.pointer
+    const { cid } = await this.serialize()
+    this.pointer = cid
+    this.outdatedPointer = false
+    return this.pointer
+  }
+
+  async serialize(): Promise<{ cid: CID; bytes: Uint8Array }> {
     let entries = await this.getEntries()
     const outdated = entries.filter(
       (e) => e.isTree() && e.outdatedPointer,
@@ -162,9 +168,12 @@ export class MST implements DataStore {
       await Promise.all(outdated.map((e) => e.getPointer()))
       entries = await this.getEntries()
     }
-    this.pointer = await util.cidForEntries(entries)
-    this.outdatedPointer = false
-    return this.pointer
+    const data = util.serializeNodeData(entries)
+    const block = await dataToCborBlock(data)
+    return {
+      cid: block.cid,
+      bytes: block.bytes,
+    }
   }
 
   // In most cases, we get the layer of a node from a hint on creation
@@ -695,17 +704,9 @@ export class MST implements DataStore {
   // Sync Protocol
 
   async writeToCarStream(car: BlockWriter): Promise<void> {
-    const entries = await this.getEntries()
     const leaves = new CidSet()
     let toFetch = new CidSet()
     toFetch.add(await this.getPointer())
-    for (const entry of entries) {
-      if (entry.isLeaf()) {
-        leaves.add(entry.value)
-      } else {
-        toFetch.add(await entry.getPointer())
-      }
-    }
     while (toFetch.size() > 0) {
       const nextLayer = new CidSet()
       const fetched = await this.storage.getBlocks(toFetch.toList())

package/src/readable-repo.ts

@@ -1,5 +1,5 @@
 import { CID } from 'multiformats/cid'
-import { Commit, def, DataStore, RepoContents } from './types'
+import { def, RepoContents, Commit } from './types'
 import { ReadableBlockstore } from './storage'
 import { MST } from './mst'
 import log from './logger'
@@ -9,14 +9,14 @@ import { MissingBlocksError } from './error'
 
 type Params = {
   storage: ReadableBlockstore
-  data: DataStore
+  data: MST
   commit: Commit
   cid: CID
 }
 
 export class ReadableRepo {
   storage: ReadableBlockstore
-  data: DataStore
+  data: MST
   commit: Commit
   cid: CID
 
@@ -28,13 +28,13 @@ export class ReadableRepo {
   }
 
   static async load(storage: ReadableBlockstore, commitCid: CID) {
-    const commit = await storage.readObj(commitCid, def.commit)
+    const commit = await storage.readObj(commitCid, def.versionedCommit)
     const data = await MST.load(storage, commit.data)
     log.info({ did: commit.did }, 'loaded repo for')
     return new ReadableRepo({
       storage,
       data,
-      commit,
+      commit: util.ensureV3Commit(commit),
       cid: commitCid,
     })
   }

package/src/repo.ts

@@ -1,12 +1,12 @@
 import { CID } from 'multiformats/cid'
+import { TID } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import {
   Commit,
+  CommitData,
   def,
-  DataStore,
   RecordCreateOp,
   RecordWriteOp,
-  CommitData,
   WriteOpAction,
 } from './types'
 import { RepoStorage } from './storage'
@@ -19,7 +19,7 @@ import * as util from './util'
 
 type Params = {
   storage: RepoStorage
-  data: DataStore
+  data: MST
   commit: Commit
   cid: CID
 }
@@ -46,25 +46,29 @@ export class Repo extends ReadableRepo {
       const dataKey = util.formatDataKey(record.collection, record.rkey)
       data = await data.add(dataKey, cid)
     }
+    const dataCid = await data.getPointer()
+    const diff = await DataDiff.of(data, null)
+    newBlocks.addMap(diff.newMstBlocks)
 
-    const unstoredData = await data.getUnstoredBlocks()
-    newBlocks.addMap(unstoredData.blocks)
-
+    const rev = TID.nextStr()
     const commit = await util.signCommit(
       {
         did,
-        version: 2,
-        prev: null,
-        data: unstoredData.root,
+        version: 3,
+        rev,
+        prev: null, // added for backwards compatibility with v2
+        data: dataCid,
       },
       keypair,
     )
     const commitCid = await newBlocks.add(commit)
-
     return {
-      commit: commitCid,
+      cid: commitCid,
+      rev,
+      since: null,
       prev: null,
-      blocks: newBlocks,
+      newBlocks,
+      removedCids: diff.removedCids,
     }
   }
 
@@ -73,7 +77,7 @@ export class Repo extends ReadableRepo {
     commit: CommitData,
   ): Promise<Repo> {
     await storage.applyCommit(commit)
-    return Repo.load(storage, commit.commit)
+    return Repo.load(storage, commit.cid)
   }
 
   static async create(
@@ -92,17 +96,17 @@ export class Repo extends ReadableRepo {
   }
 
   static async load(storage: RepoStorage, cid?: CID) {
-    const commitCid = cid || (await storage.getHead())
+    const commitCid = cid || (await storage.getRoot())
     if (!commitCid) {
       throw new Error('No cid provided and none in storage')
     }
-    const commit = await storage.readObj(commitCid, def.commit)
+    const commit = await storage.readObj(commitCid, def.versionedCommit)
     const data = await MST.load(storage, commit.data)
     log.info({ did: commit.did }, 'loaded repo for')
     return new Repo({
       storage,
       data,
-      commit,
+      commit: util.ensureV3Commit(commit),
       cid: commitCid,
     })
   }
@@ -112,16 +116,16 @@ export class Repo extends ReadableRepo {
     keypair: crypto.Keypair,
   ): Promise<CommitData> {
     const writes = Array.isArray(toWrite) ? toWrite : [toWrite]
-    const commitBlocks = new BlockMap()
+    const leaves = new BlockMap()
 
     let data = this.data
     for (const write of writes) {
       if (write.action === WriteOpAction.Create) {
-        const cid = await commitBlocks.add(write.record)
+        const cid = await leaves.add(write.record)
         const dataKey = write.collection + '/' + write.rkey
         data = await data.add(dataKey, cid)
       } else if (write.action === WriteOpAction.Update) {
-        const cid = await commitBlocks.add(write.record)
+        const cid = await leaves.add(write.record)
         const dataKey = write.collection + '/' + write.rkey
         data = await data.update(dataKey, cid)
       } else if (write.action === WriteOpAction.Delete) {
@@ -130,44 +134,50 @@ export class Repo extends ReadableRepo {
       }
     }
 
-    const unstoredData = await data.getUnstoredBlocks()
-    commitBlocks.addMap(unstoredData.blocks)
-
-    // ensure we're not missing any blocks that were removed and then readded in this commit
+    const dataCid = await data.getPointer()
     const diff = await DataDiff.of(data, this.data)
-    const found = commitBlocks.getMany(diff.newCidList())
-    if (found.missing.length > 0) {
-      const fromStorage = await this.storage.getBlocks(found.missing)
-      if (fromStorage.missing.length > 0) {
-        // this shouldn't ever happen
-        throw new Error(
-          'Could not find block for commit in Datastore or storage',
-        )
-      }
-      commitBlocks.addMap(fromStorage.blocks)
+    const newBlocks = diff.newMstBlocks
+    const removedCids = diff.removedCids
+
+    const addedLeaves = leaves.getMany(diff.newLeafCids.toList())
+    if (addedLeaves.missing.length > 0) {
+      throw new Error(`Missing leaf blocks: ${addedLeaves.missing}`)
     }
+    newBlocks.addMap(addedLeaves.blocks)
 
+    const rev = TID.nextStr(this.commit.rev)
     const commit = await util.signCommit(
       {
         did: this.did,
-        version: 2,
-        prev: this.cid,
-        data: unstoredData.root,
+        version: 3,
+        rev,
+        prev: null, // added for backwards compatibility with v2
+        data: dataCid,
       },
       keypair,
     )
-    const commitCid = await commitBlocks.add(commit)
+    const commitCid = await newBlocks.add(commit)
+
+    // ensure the commit cid actually changed
+    if (commitCid.equals(this.cid)) {
+      newBlocks.delete(commitCid)
+    } else {
+      removedCids.add(this.cid)
+    }
 
     return {
-      commit: commitCid,
+      cid: commitCid,
+      rev,
+      since: this.commit.rev,
       prev: this.cid,
-      blocks: commitBlocks,
+      newBlocks,
+      removedCids,
     }
   }
 
   async applyCommit(commitData: CommitData): Promise<Repo> {
     await this.storage.applyCommit(commitData)
-    return Repo.load(this.storage, commitData.commit)
+    return Repo.load(this.storage, commitData.cid)
   }
 
   async applyWrites(

package/src/storage/index.ts

@@ -1,5 +1,4 @@
 export * from './readable-blockstore'
-export * from './repo-storage'
 export * from './memory-blockstore'
 export * from './sync-storage'
 export * from './types'