@atproto/repo 0.0.1 → 0.1.0

package/src/data-diff.ts

@@ -0,0 +1,121 @@
+import { CID } from 'multiformats'
+import CidSet from './cid-set'
+import { MST, mstDiff } from './mst'
+import { DataStore } from './types'
+
+export class DataDiff {
+  adds: Record<string, DataAdd> = {}
+  updates: Record<string, DataUpdate> = {}
+  deletes: Record<string, DataDelete> = {}
+
+  newCids: CidSet = new CidSet()
+  removedCids: CidSet = new CidSet()
+
+  static async of(curr: DataStore, prev: DataStore | null): Promise<DataDiff> {
+    if (curr instanceof MST && (prev === null || prev instanceof MST)) {
+      return mstDiff(curr, prev)
+    }
+    throw new Error('Unsupported DataStore type for diff')
+  }
+
+  recordAdd(key: string, cid: CID): void {
+    this.adds[key] = { key, cid }
+    this.newCids.add(cid)
+  }
+
+  recordUpdate(key: string, prev: CID, cid: CID): void {
+    this.updates[key] = { key, prev, cid }
+    this.newCids.add(cid)
+  }
+
+  recordDelete(key: string, cid: CID): void {
+    this.deletes[key] = { key, cid }
+  }
+
+  recordNewCid(cid: CID): void {
+    if (this.removedCids.has(cid)) {
+      this.removedCids.delete(cid)
+    } else {
+      this.newCids.add(cid)
+    }
+  }
+
+  recordRemovedCid(cid: CID): void {
+    if (this.newCids.has(cid)) {
+      this.newCids.delete(cid)
+    } else {
+      this.removedCids.add(cid)
+    }
+  }
+
+  addDiff(diff: DataDiff) {
+    for (const add of diff.addList()) {
+      if (this.deletes[add.key]) {
+        const del = this.deletes[add.key]
+        if (del.cid !== add.cid) {
+          this.recordUpdate(add.key, del.cid, add.cid)
+        }
+        delete this.deletes[add.key]
+      } else {
+        this.recordAdd(add.key, add.cid)
+      }
+    }
+    for (const update of diff.updateList()) {
+      this.recordUpdate(update.key, update.prev, update.cid)
+      delete this.adds[update.key]
+      delete this.deletes[update.key]
+    }
+    for (const del of diff.deleteList()) {
+      if (this.adds[del.key]) {
+        delete this.adds[del.key]
+      } else {
+        delete this.updates[del.key]
+        this.recordDelete(del.key, del.cid)
+      }
+    }
+    this.newCids.addSet(diff.newCids)
+  }
+
+  addList(): DataAdd[] {
+    return Object.values(this.adds)
+  }
+
+  updateList(): DataUpdate[] {
+    return Object.values(this.updates)
+  }
+
+  deleteList(): DataDelete[] {
+    return Object.values(this.deletes)
+  }
+
+  newCidList(): CID[] {
+    return this.newCids.toList()
+  }
+
+  updatedKeys(): string[] {
+    const keys = [
+      ...Object.keys(this.adds),
+      ...Object.keys(this.updates),
+      ...Object.keys(this.deletes),
+    ]
+    return [...new Set(keys)]
+  }
+}
+
+export type DataAdd = {
+  key: string
+  cid: CID
+}
+
+export type DataUpdate = {
+  key: string
+  prev: CID
+  cid: CID
+}
+
+export type DataDelete = {
+  key: string
+  cid: CID
+}
+
+export default DataDiff

package/src/error.ts

@@ -0,0 +1,31 @@
+import { CID } from 'multiformats/cid'
+
+export class MissingBlockError extends Error {
+  constructor(public cid: CID, def?: string) {
+    let msg = `block not found: ${cid.toString()}`
+    if (def) {
+      msg += `, expected type: ${def}`
+    }
+    super(msg)
+  }
+}
+
+export class MissingBlocksError extends Error {
+  constructor(public context: string, public cids: CID[]) {
+    const cidStr = cids.map((c) => c.toString())
+    super(`missing ${context} blocks: ${cidStr}`)
+  }
+}
+
+export class MissingCommitBlocksError extends Error {
+  constructor(public commit: CID, public cids: CID[]) {
+    const cidStr = cids.map((c) => c.toString())
+    super(`missing blocks for commit ${commit.toString()}: ${cidStr}`)
+  }
+}
+
+export class UnexpectedObjectError extends Error {
+  constructor(public cid: CID, public def: string) {
+    super(`unexpected object at ${cid.toString()}, expected: ${def}`)
+  }
+}

package/src/index.ts

@@ -1,7 +1,9 @@
-export * from './blockstore'
+export * from './block-map'
+export * from './cid-set'
 export * from './repo'
 export * from './mst'
 export * from './storage'
+export * from './sync'
 export * from './types'
 export * from './verify'
 export * from './util'

package/src/mst/diff.ts

@@ -1,106 +1,136 @@
-import * as auth from '@atproto/auth'
-import { CID } from 'multiformats'
-import CidSet from '../cid-set'
-import { parseRecordKey } from '../util'
-
-export class DataDiff {
-  adds: Record<string, DataAdd> = {}
-  updates: Record<string, DataUpdate> = {}
-  deletes: Record<string, DataDelete> = {}
-
-  newCids: CidSet = new CidSet()
-
-  recordAdd(key: string, cid: CID): void {
-    this.adds[key] = { key, cid }
-    this.newCids.add(cid)
-  }
-
-  recordUpdate(key: string, prev: CID, cid: CID): void {
-    this.updates[key] = { key, prev, cid }
-    this.newCids.add(cid)
-  }
-
-  recordDelete(key: string, cid: CID): void {
-    this.deletes[key] = { key, cid }
+import { DataDiff } from '../data-diff'
+import MST from './mst'
+import MstWalker from './walker'
+
+export const nullDiff = async (tree: MST): Promise<DataDiff> => {
+  const diff = new DataDiff()
+  for await (const entry of tree.walk()) {
+    if (entry.isLeaf()) {
+      diff.recordAdd(entry.key, entry.value)
+    } else {
+      diff.recordNewCid(entry.pointer)
+    }
   }
+  return diff
+}
 
-  recordNewCid(cid: CID): void {
-    this.newCids.add(cid)
+export const mstDiff = async (
+  curr: MST,
+  prev: MST | null,
+): Promise<DataDiff> => {
+  await curr.getPointer()
+  if (prev === null) {
+    return nullDiff(curr)
   }
 
-  addDiff(diff: DataDiff) {
-    for (const add of diff.addList()) {
-      if (this.deletes[add.key]) {
-        const del = this.deletes[add.key]
-        if (del.cid !== add.cid) {
-          this.recordUpdate(add.key, del.cid, add.cid)
-        }
-        delete this.deletes[add.key]
+  await prev.getPointer()
+  const diff = new DataDiff()
+
+  const leftWalker = new MstWalker(prev)
+  const rightWalker = new MstWalker(curr)
+  while (!leftWalker.status.done || !rightWalker.status.done) {
+    // if one walker is finished, continue walking the other & logging all nodes
+    if (leftWalker.status.done && !rightWalker.status.done) {
+      const node = rightWalker.status.curr
+      if (node.isLeaf()) {
+        diff.recordAdd(node.key, node.value)
       } else {
-        this.recordAdd(add.key, add.cid)
+        diff.recordNewCid(node.pointer)
       }
+      await rightWalker.advance()
+      continue
+    } else if (!leftWalker.status.done && rightWalker.status.done) {
+      const node = leftWalker.status.curr
+      if (node.isLeaf()) {
+        diff.recordDelete(node.key, node.value)
+      } else {
+        diff.recordRemovedCid(node.pointer)
+      }
+      await leftWalker.advance()
+      continue
     }
-    for (const update of diff.updateList()) {
-      this.recordUpdate(update.key, update.prev, update.cid)
-      delete this.adds[update.key]
-      delete this.deletes[update.key]
-    }
-    for (const del of diff.deleteList()) {
-      if (this.adds[del.key]) {
-        delete this.adds[del.key]
+    if (leftWalker.status.done || rightWalker.status.done) break
+    const left = leftWalker.status.curr
+    const right = rightWalker.status.curr
+    if (left === null || right === null) break
+
+    // if both pointers are leaves, record an update & advance both or record the lowest key and advance that pointer
+    if (left.isLeaf() && right.isLeaf()) {
+      if (left.key === right.key) {
+        if (!left.value.equals(right.value)) {
+          diff.recordUpdate(left.key, left.value, right.value)
+        }
+        await leftWalker.advance()
+        await rightWalker.advance()
+      } else if (left.key < right.key) {
+        diff.recordDelete(left.key, left.value)
+        await leftWalker.advance()
       } else {
-        delete this.updates[del.key]
-        this.recordDelete(del.key, del.cid)
+        diff.recordAdd(right.key, right.value)
+        await rightWalker.advance()
       }
+      continue
     }
-    this.newCids.addSet(diff.newCids)
-  }
 
-  addList(): DataAdd[] {
-    return Object.values(this.adds)
-  }
-
-  updateList(): DataUpdate[] {
-    return Object.values(this.updates)
-  }
-
-  deleteList(): DataDelete[] {
-    return Object.values(this.deletes)
-  }
+    // next, ensure that we're on the same layer
+    // if one walker is at a higher layer than the other, we need to do one of two things
+    // if the higher walker is pointed at a tree, step into that tree to try to catch up with the lower
+    // if the higher walker is pointed at a leaf, then advance the lower walker to try to catch up the higher
+    if (leftWalker.layer() > rightWalker.layer()) {
+      if (left.isLeaf()) {
+        if (right.isLeaf()) {
+          diff.recordAdd(right.key, right.value)
+        } else {
+          diff.recordNewCid(right.pointer)
+        }
+        await rightWalker.advance()
+      } else {
+        diff.recordRemovedCid(left.pointer)
+        await leftWalker.stepInto()
+      }
+      continue
+    } else if (leftWalker.layer() < rightWalker.layer()) {
+      if (right.isLeaf()) {
+        if (left.isLeaf()) {
+          diff.recordDelete(left.key, left.value)
+        } else {
+          diff.recordRemovedCid(left.pointer)
+        }
+        await leftWalker.advance()
+      } else {
+        diff.recordNewCid(right.pointer)
+        await rightWalker.stepInto()
+      }
+      continue
+    }
 
-  newCidList(): CID[] {
-    return this.newCids.toList()
-  }
+    // if we're on the same level, and both pointers are trees, do a comparison
+    // if they're the same, step over. if they're different, step in to find the subdiff
+    if (left.isTree() && right.isTree()) {
+      if (left.pointer.equals(right.pointer)) {
+        await leftWalker.stepOver()
+        await rightWalker.stepOver()
+      } else {
+        diff.recordNewCid(right.pointer)
+        diff.recordRemovedCid(left.pointer)
+        await leftWalker.stepInto()
+        await rightWalker.stepInto()
+      }
+      continue
+    }
 
-  updatedKeys(): string[] {
-    const keys = [
-      ...Object.keys(this.adds),
-      ...Object.keys(this.updates),
-      ...Object.keys(this.deletes),
-    ]
-    return [...new Set(keys)]
-  }
+    // finally, if one pointer is a tree and the other is a leaf, simply step into the tree
+    if (left.isLeaf() && right.isTree()) {
+      diff.recordNewCid(right.pointer)
+      await rightWalker.stepInto()
+      continue
+    } else if (left.isTree() && right.isLeaf()) {
+      diff.recordRemovedCid(left.pointer)
+      await leftWalker.stepInto()
+      continue
+    }
 
-  neededCapabilities(rootDid: string): auth.ucans.Capability[] {
-    return this.updatedKeys().map((key) => {
-      const { collection, rkey } = parseRecordKey(key)
-      return auth.writeCap(rootDid, collection, rkey)
-    })
+    throw new Error('Unidentifiable case in diff walk')
   }
-}
-
-export type DataAdd = {
-  key: string
-  cid: CID
-}
-
-export type DataUpdate = {
-  key: string
-  prev: CID
-  cid: CID
-}
-
-export type DataDelete = {
-  key: string
-  cid: CID
+  return diff
 }