@atproto/repo 0.2.0 → 0.3.0

package/src/data-diff.ts

@@ -1,77 +1,83 @@
 import { CID } from 'multiformats'
 import CidSet from './cid-set'
-import { MST, mstDiff } from './mst'
+import { MST, NodeEntry, mstDiff } from './mst'
+import BlockMap from './block-map'
 
 export class DataDiff {
   adds: Record<string, DataAdd> = {}
   updates: Record<string, DataUpdate> = {}
   deletes: Record<string, DataDelete> = {}
 
-  newCids: CidSet = new CidSet()
+  newMstBlocks: BlockMap = new BlockMap()
+  newLeafCids: CidSet = new CidSet()
   removedCids: CidSet = new CidSet()
 
   static async of(curr: MST, prev: MST | null): Promise<DataDiff> {
     return mstDiff(curr, prev)
   }
 
-  recordAdd(key: string, cid: CID): void {
+  async nodeAdd(node: NodeEntry) {
+    if (node.isLeaf()) {
+      this.leafAdd(node.key, node.value)
+    } else {
+      const data = await node.serialize()
+      this.treeAdd(data.cid, data.bytes)
+    }
+  }
+
+  async nodeDelete(node: NodeEntry) {
+    if (node.isLeaf()) {
+      const key = node.key
+      const cid = node.value
+      this.deletes[key] = { key, cid }
+      this.removedCids.add(cid)
+    } else {
+      const cid = await node.getPointer()
+      this.treeDelete(cid)
+    }
+  }
+
+  leafAdd(key: string, cid: CID) {
     this.adds[key] = { key, cid }
-    this.newCids.add(cid)
+    if (this.removedCids.has(cid)) {
+      this.removedCids.delete(cid)
+    } else {
+      this.newLeafCids.add(cid)
+    }
   }
 
-  recordUpdate(key: string, prev: CID, cid: CID): void {
+  leafUpdate(key: string, prev: CID, cid: CID) {
+    if (prev.equals(cid)) return
     this.updates[key] = { key, prev, cid }
-    this.newCids.add(cid)
+    this.removedCids.add(prev)
+    this.newLeafCids.add(cid)
   }
 
-  recordDelete(key: string, cid: CID): void {
+  leafDelete(key: string, cid: CID) {
     this.deletes[key] = { key, cid }
+    if (this.newLeafCids.has(cid)) {
+      this.newLeafCids.delete(cid)
+    } else {
+      this.removedCids.add(cid)
+    }
   }
 
-  recordNewCid(cid: CID): void {
+  treeAdd(cid: CID, bytes: Uint8Array) {
     if (this.removedCids.has(cid)) {
       this.removedCids.delete(cid)
     } else {
-      this.newCids.add(cid)
+      this.newMstBlocks.set(cid, bytes)
     }
   }
 
-  recordRemovedCid(cid: CID): void {
-    if (this.newCids.has(cid)) {
-      this.newCids.delete(cid)
+  treeDelete(cid: CID) {
+    if (this.newMstBlocks.has(cid)) {
+      this.newMstBlocks.delete(cid)
     } else {
       this.removedCids.add(cid)
     }
   }
 
-  addDiff(diff: DataDiff) {
-    for (const add of diff.addList()) {
-      if (this.deletes[add.key]) {
-        const del = this.deletes[add.key]
-        if (del.cid !== add.cid) {
-          this.recordUpdate(add.key, del.cid, add.cid)
-        }
-        delete this.deletes[add.key]
-      } else {
-        this.recordAdd(add.key, add.cid)
-      }
-    }
-    for (const update of diff.updateList()) {
-      this.recordUpdate(update.key, update.prev, update.cid)
-      delete this.adds[update.key]
-      delete this.deletes[update.key]
-    }
-    for (const del of diff.deleteList()) {
-      if (this.adds[del.key]) {
-        delete this.adds[del.key]
-      } else {
-        delete this.updates[del.key]
-        this.recordDelete(del.key, del.cid)
-      }
-    }
-    this.newCids.addSet(diff.newCids)
-  }
-
   addList(): DataAdd[] {
     return Object.values(this.adds)
   }
@@ -84,10 +90,6 @@ export class DataDiff {
     return Object.values(this.deletes)
   }
 
-  newCidList(): CID[] {
-    return this.newCids.toList()
-  }
-
   updatedKeys(): string[] {
     const keys = [
       ...Object.keys(this.adds),

package/src/index.ts

@@ -5,6 +5,5 @@ export * from './mst'
 export * from './storage'
 export * from './sync'
 export * from './types'
-export * from './verify'
 export * from './data-diff'
 export * from './util'

package/src/mst/diff.ts

@@ -5,11 +5,7 @@ import MstWalker from './walker'
 export const nullDiff = async (tree: MST): Promise<DataDiff> => {
   const diff = new DataDiff()
   for await (const entry of tree.walk()) {
-    if (entry.isLeaf()) {
-      diff.recordAdd(entry.key, entry.value)
-    } else {
-      diff.recordNewCid(entry.pointer)
-    }
+    await diff.nodeAdd(entry)
   }
   return diff
 }
@@ -31,21 +27,11 @@ export const mstDiff = async (
   while (!leftWalker.status.done || !rightWalker.status.done) {
     // if one walker is finished, continue walking the other & logging all nodes
     if (leftWalker.status.done && !rightWalker.status.done) {
-      const node = rightWalker.status.curr
-      if (node.isLeaf()) {
-        diff.recordAdd(node.key, node.value)
-      } else {
-        diff.recordNewCid(node.pointer)
-      }
+      await diff.nodeAdd(rightWalker.status.curr)
       await rightWalker.advance()
       continue
     } else if (!leftWalker.status.done && rightWalker.status.done) {
-      const node = leftWalker.status.curr
-      if (node.isLeaf()) {
-        diff.recordDelete(node.key, node.value)
-      } else {
-        diff.recordRemovedCid(node.pointer)
-      }
+      await diff.nodeDelete(leftWalker.status.curr)
       await leftWalker.advance()
       continue
     }
@@ -58,15 +44,15 @@ export const mstDiff = async (
     if (left.isLeaf() && right.isLeaf()) {
       if (left.key === right.key) {
         if (!left.value.equals(right.value)) {
-          diff.recordUpdate(left.key, left.value, right.value)
+          diff.leafUpdate(left.key, left.value, right.value)
         }
         await leftWalker.advance()
         await rightWalker.advance()
       } else if (left.key < right.key) {
-        diff.recordDelete(left.key, left.value)
+        diff.leafDelete(left.key, left.value)
         await leftWalker.advance()
       } else {
-        diff.recordAdd(right.key, right.value)
+        diff.leafAdd(right.key, right.value)
         await rightWalker.advance()
       }
       continue
@@ -78,27 +64,19 @@ export const mstDiff = async (
     // if the higher walker is pointed at a leaf, then advance the lower walker to try to catch up the higher
     if (leftWalker.layer() > rightWalker.layer()) {
       if (left.isLeaf()) {
-        if (right.isLeaf()) {
-          diff.recordAdd(right.key, right.value)
-        } else {
-          diff.recordNewCid(right.pointer)
-        }
+        await diff.nodeAdd(right)
         await rightWalker.advance()
       } else {
-        diff.recordRemovedCid(left.pointer)
+        await diff.nodeDelete(left)
         await leftWalker.stepInto()
       }
       continue
     } else if (leftWalker.layer() < rightWalker.layer()) {
       if (right.isLeaf()) {
-        if (left.isLeaf()) {
-          diff.recordDelete(left.key, left.value)
-        } else {
-          diff.recordRemovedCid(left.pointer)
-        }
+        await diff.nodeDelete(left)
         await leftWalker.advance()
       } else {
-        diff.recordNewCid(right.pointer)
+        await diff.nodeAdd(right)
         await rightWalker.stepInto()
       }
       continue
@@ -111,8 +89,8 @@ export const mstDiff = async (
         await leftWalker.stepOver()
         await rightWalker.stepOver()
       } else {
-        diff.recordNewCid(right.pointer)
-        diff.recordRemovedCid(left.pointer)
+        await diff.nodeAdd(right)
+        await diff.nodeDelete(left)
         await leftWalker.stepInto()
         await rightWalker.stepInto()
       }
@@ -121,11 +99,11 @@ export const mstDiff = async (
 
     // finally, if one pointer is a tree and the other is a leaf, simply step into the tree
     if (left.isLeaf() && right.isTree()) {
-      diff.recordNewCid(right.pointer)
+      await diff.nodeAdd(right)
       await rightWalker.stepInto()
       continue
     } else if (left.isTree() && right.isLeaf()) {
-      diff.recordRemovedCid(left.pointer)
+      await diff.nodeDelete(left)
       await leftWalker.stepInto()
       continue
     }

package/src/mst/mst.ts

@@ -2,7 +2,7 @@ import z from 'zod'
 import { CID } from 'multiformats'
 
 import { ReadableBlockstore } from '../storage'
-import { schema as common, cidForCbor } from '@atproto/common'
+import { schema as common, cidForCbor, dataToCborBlock } from '@atproto/common'
 import { BlockWriter } from '@ipld/car/api'
 import * as util from './util'
 import BlockMap from '../block-map'
@@ -153,6 +153,13 @@ export class MST {
   // Instead we keep track of whether the pointer is outdated and only (recursively) calculate when needed
   async getPointer(): Promise<CID> {
     if (!this.outdatedPointer) return this.pointer
+    const { cid } = await this.serialize()
+    this.pointer = cid
+    this.outdatedPointer = false
+    return this.pointer
+  }
+
+  async serialize(): Promise<{ cid: CID; bytes: Uint8Array }> {
     let entries = await this.getEntries()
     const outdated = entries.filter(
       (e) => e.isTree() && e.outdatedPointer,
@@ -161,9 +168,12 @@ export class MST {
       await Promise.all(outdated.map((e) => e.getPointer()))
       entries = await this.getEntries()
     }
-    this.pointer = await util.cidForEntries(entries)
-    this.outdatedPointer = false
-    return this.pointer
+    const data = util.serializeNodeData(entries)
+    const block = await dataToCborBlock(data)
+    return {
+      cid: block.cid,
+      bytes: block.bytes,
+    }
   }
 
   // In most cases, we get the layer of a node from a hint on creation

package/src/readable-repo.ts

@@ -1,5 +1,5 @@
 import { CID } from 'multiformats/cid'
-import { Commit, def, RepoContents } from './types'
+import { def, RepoContents, Commit } from './types'
 import { ReadableBlockstore } from './storage'
 import { MST } from './mst'
 import log from './logger'
@@ -28,13 +28,13 @@ export class ReadableRepo {
   }
 
   static async load(storage: ReadableBlockstore, commitCid: CID) {
-    const commit = await storage.readObj(commitCid, def.commit)
+    const commit = await storage.readObj(commitCid, def.versionedCommit)
     const data = await MST.load(storage, commit.data)
     log.info({ did: commit.did }, 'loaded repo for')
     return new ReadableRepo({
       storage,
       data,
-      commit,
+      commit: util.ensureV3Commit(commit),
       cid: commitCid,
     })
   }

package/src/repo.ts

@@ -1,13 +1,13 @@
 import { CID } from 'multiformats/cid'
+import { TID } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import {
   Commit,
+  CommitData,
   def,
   RecordCreateOp,
   RecordWriteOp,
-  CommitData,
   WriteOpAction,
-  RebaseData,
 } from './types'
 import { RepoStorage } from './storage'
 import { MST } from './mst'
@@ -46,25 +46,29 @@ export class Repo extends ReadableRepo {
       const dataKey = util.formatDataKey(record.collection, record.rkey)
       data = await data.add(dataKey, cid)
     }
+    const dataCid = await data.getPointer()
+    const diff = await DataDiff.of(data, null)
+    newBlocks.addMap(diff.newMstBlocks)
 
-    const unstoredData = await data.getUnstoredBlocks()
-    newBlocks.addMap(unstoredData.blocks)
-
+    const rev = TID.nextStr()
     const commit = await util.signCommit(
       {
         did,
-        version: 2,
-        prev: null,
-        data: unstoredData.root,
+        version: 3,
+        rev,
+        prev: null, // added for backwards compatibility with v2
+        data: dataCid,
       },
       keypair,
     )
     const commitCid = await newBlocks.add(commit)
-
     return {
-      commit: commitCid,
+      cid: commitCid,
+      rev,
+      since: null,
       prev: null,
-      blocks: newBlocks,
+      newBlocks,
+      removedCids: diff.removedCids,
     }
   }
 
@@ -73,7 +77,7 @@ export class Repo extends ReadableRepo {
     commit: CommitData,
   ): Promise<Repo> {
     await storage.applyCommit(commit)
-    return Repo.load(storage, commit.commit)
+    return Repo.load(storage, commit.cid)
   }
 
   static async create(
@@ -92,17 +96,17 @@ export class Repo extends ReadableRepo {
   }
 
   static async load(storage: RepoStorage, cid?: CID) {
-    const commitCid = cid || (await storage.getHead())
+    const commitCid = cid || (await storage.getRoot())
     if (!commitCid) {
       throw new Error('No cid provided and none in storage')
     }
-    const commit = await storage.readObj(commitCid, def.commit)
+    const commit = await storage.readObj(commitCid, def.versionedCommit)
     const data = await MST.load(storage, commit.data)
     log.info({ did: commit.did }, 'loaded repo for')
     return new Repo({
       storage,
       data,
-      commit,
+      commit: util.ensureV3Commit(commit),
       cid: commitCid,
     })
   }
@@ -112,16 +116,16 @@ export class Repo extends ReadableRepo {
     keypair: crypto.Keypair,
   ): Promise<CommitData> {
     const writes = Array.isArray(toWrite) ? toWrite : [toWrite]
-    const commitBlocks = new BlockMap()
+    const leaves = new BlockMap()
 
     let data = this.data
     for (const write of writes) {
       if (write.action === WriteOpAction.Create) {
-        const cid = await commitBlocks.add(write.record)
+        const cid = await leaves.add(write.record)
         const dataKey = write.collection + '/' + write.rkey
         data = await data.add(dataKey, cid)
       } else if (write.action === WriteOpAction.Update) {
-        const cid = await commitBlocks.add(write.record)
+        const cid = await leaves.add(write.record)
         const dataKey = write.collection + '/' + write.rkey
         data = await data.update(dataKey, cid)
       } else if (write.action === WriteOpAction.Delete) {
@@ -130,44 +134,50 @@ export class Repo extends ReadableRepo {
       }
     }
 
-    const unstoredData = await data.getUnstoredBlocks()
-    commitBlocks.addMap(unstoredData.blocks)
-
-    // ensure we're not missing any blocks that were removed and then readded in this commit
+    const dataCid = await data.getPointer()
     const diff = await DataDiff.of(data, this.data)
-    const found = commitBlocks.getMany(diff.newCidList())
-    if (found.missing.length > 0) {
-      const fromStorage = await this.storage.getBlocks(found.missing)
-      if (fromStorage.missing.length > 0) {
-        // this shouldn't ever happen
-        throw new Error(
-          'Could not find block for commit in Datastore or storage',
-        )
-      }
-      commitBlocks.addMap(fromStorage.blocks)
+    const newBlocks = diff.newMstBlocks
+    const removedCids = diff.removedCids
+
+    const addedLeaves = leaves.getMany(diff.newLeafCids.toList())
+    if (addedLeaves.missing.length > 0) {
+      throw new Error(`Missing leaf blocks: ${addedLeaves.missing}`)
     }
+    newBlocks.addMap(addedLeaves.blocks)
 
+    const rev = TID.nextStr(this.commit.rev)
     const commit = await util.signCommit(
       {
         did: this.did,
-        version: 2,
-        prev: this.cid,
-        data: unstoredData.root,
+        version: 3,
+        rev,
+        prev: null, // added for backwards compatibility with v2
+        data: dataCid,
       },
       keypair,
     )
-    const commitCid = await commitBlocks.add(commit)
+    const commitCid = await newBlocks.add(commit)
+
+    // ensure the commit cid actually changed
+    if (commitCid.equals(this.cid)) {
+      newBlocks.delete(commitCid)
+    } else {
+      removedCids.add(this.cid)
+    }
 
     return {
-      commit: commitCid,
+      cid: commitCid,
+      rev,
+      since: this.commit.rev,
       prev: this.cid,
-      blocks: commitBlocks,
+      newBlocks,
+      removedCids,
     }
   }
 
   async applyCommit(commitData: CommitData): Promise<Repo> {
     await this.storage.applyCommit(commitData)
-    return Repo.load(this.storage, commitData.commit)
+    return Repo.load(this.storage, commitData.cid)
   }
 
   async applyWrites(
@@ -177,37 +187,6 @@ export class Repo extends ReadableRepo {
     const commit = await this.formatCommit(toWrite, keypair)
     return this.applyCommit(commit)
   }
-
-  async formatRebase(keypair: crypto.Keypair): Promise<RebaseData> {
-    const preservedCids = await this.data.allCids()
-    const blocks = new BlockMap()
-    const commit = await util.signCommit(
-      {
-        did: this.did,
-        version: 2,
-        prev: null,
-        data: this.commit.data,
-      },
-      keypair,
-    )
-    const commitCid = await blocks.add(commit)
-    return {
-      commit: commitCid,
-      rebased: this.cid,
-      blocks,
-      preservedCids: preservedCids.toList(),
-    }
-  }
-
-  async applyRebase(rebase: RebaseData): Promise<Repo> {
-    await this.storage.applyRebase(rebase)
-    return Repo.load(this.storage, rebase.commit)
-  }
-
-  async rebase(keypair: crypto.Keypair): Promise<Repo> {
-    const rebaseData = await this.formatRebase(keypair)
-    return this.applyRebase(rebaseData)
-  }
 }
 
 export default Repo

package/src/storage/index.ts

@@ -1,5 +1,4 @@
 export * from './readable-blockstore'
-export * from './repo-storage'
 export * from './memory-blockstore'
 export * from './sync-storage'
 export * from './types'

package/src/storage/memory-blockstore.ts

@@ -1,15 +1,15 @@
 import { CID } from 'multiformats/cid'
-import { CommitData, def, RebaseData } from '../types'
+import { CommitData } from '../types'
 import BlockMap from '../block-map'
-import { MST } from '../mst'
-import DataDiff from '../data-diff'
-import { MissingCommitBlocksError } from '../error'
-import RepoStorage from './repo-storage'
-import CidSet from '../cid-set'
+import ReadableBlockstore from './readable-blockstore'
+import { RepoStorage } from './types'
 
-export class MemoryBlockstore extends RepoStorage {
+export class MemoryBlockstore
+  extends ReadableBlockstore
+  implements RepoStorage
+{
   blocks: BlockMap
-  head: CID | null = null
+  root: CID | null = null
 
   constructor(blocks?: BlockMap) {
     super()
@@ -19,8 +19,8 @@ export class MemoryBlockstore extends RepoStorage {
     }
   }
 
-  async getHead(): Promise<CID | null> {
-    return this.head
+  async getRoot(): Promise<CID | null> {
+    return this.root
   }
 
   async getBytes(cid: CID): Promise<Uint8Array | null> {
@@ -43,78 +43,19 @@ export class MemoryBlockstore extends RepoStorage {
     this.blocks.addMap(blocks)
   }
 
-  async indexCommits(commits: CommitData[]): Promise<void> {
-    commits.forEach((commit) => {
-      this.blocks.addMap(commit.blocks)
-    })
-  }
-
-  async updateHead(cid: CID, _prev: CID | null): Promise<void> {
-    this.head = cid
+  async updateRoot(cid: CID): Promise<void> {
+    this.root = cid
   }
 
   async applyCommit(commit: CommitData): Promise<void> {
-    this.blocks.addMap(commit.blocks)
-    this.head = commit.commit
-  }
-
-  async getCommitPath(
-    latest: CID,
-    earliest: CID | null,
-  ): Promise<CID[] | null> {
-    let curr: CID | null = latest
-    const path: CID[] = []
-    while (curr !== null) {
-      path.push(curr)
-      const commit = await this.readObj(curr, def.commit)
-      if (!earliest && commit.prev === null) {
-        return path.reverse()
-      } else if (earliest && commit.prev.equals(earliest)) {
-        return path.reverse()
-      }
-      curr = commit.prev
-    }
-    return null
-  }
-
-  async getBlocksForCommits(
-    commits: CID[],
-  ): Promise<{ [commit: string]: BlockMap }> {
-    const commitData: { [commit: string]: BlockMap } = {}
-    let prevData: MST | null = null
-    for (const commitCid of commits) {
-      const commit = await this.readObj(commitCid, def.commit)
-      const data = await MST.load(this, commit.data)
-      const diff = await DataDiff.of(data, prevData)
-      const { blocks, missing } = await this.getBlocks([
-        commitCid,
-        ...diff.newCidList(),
-      ])
-      if (missing.length > 0) {
-        throw new MissingCommitBlocksError(commitCid, missing)
-      }
-      commitData[commitCid.toString()] = blocks
-      prevData = data
-    }
-    return commitData
-  }
-
-  async applyRebase(rebase: RebaseData) {
-    this.putMany(rebase.blocks)
-    const allCids = new CidSet([
-      ...rebase.preservedCids,
-      ...rebase.blocks.cids(),
-    ])
-    const toDelete: CID[] = []
-    this.blocks.forEach((_bytes, cid) => {
-      if (!allCids.has(cid)) {
-        toDelete.push(cid)
-      }
-    })
-    for (const cid of toDelete) {
+    this.root = commit.cid
+    const rmCids = commit.removedCids.toList()
+    for (const cid of rmCids) {
       this.blocks.delete(cid)
     }
-    await this.updateHead(rebase.commit, null)
+    commit.newBlocks.forEach((bytes, cid) => {
+      this.blocks.set(cid, bytes)
+    })
   }
 
   async sizeInBytes(): Promise<number> {

package/src/storage/types.ts

@@ -1,5 +1,34 @@
 import stream from 'stream'
 import { CID } from 'multiformats/cid'
+import { RepoRecord } from '@atproto/lexicon'
+import { check } from '@atproto/common'
+import BlockMap from '../block-map'
+import { CommitData } from '../types'
+
+export interface RepoStorage {
+  // Writable
+  getRoot(): Promise<CID | null>
+  putBlock(cid: CID, block: Uint8Array, rev: string): Promise<void>
+  putMany(blocks: BlockMap, rev: string): Promise<void>
+  updateRoot(cid: CID): Promise<void>
+  applyCommit(commit: CommitData)
+
+  // Readable
+  getBytes(cid: CID): Promise<Uint8Array | null>
+  has(cid: CID): Promise<boolean>
+  getBlocks(cids: CID[]): Promise<{ blocks: BlockMap; missing: CID[] }>
+  attemptRead<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array } | null>
+  readObjAndBytes<T>(
+    cid: CID,
+    def: check.Def<T>,
+  ): Promise<{ obj: T; bytes: Uint8Array }>
+  readObj<T>(cid: CID, def: check.Def<T>): Promise<T>
+  attemptReadRecord(cid: CID): Promise<RepoRecord | null>
+  readRecord(cid: CID): Promise<RepoRecord>
+}
 
 export interface BlobStore {
   putTemp(bytes: Uint8Array | stream.Readable): Promise<string>