@atproto/repo 0.0.1

package/src/mst/util.ts

@@ -0,0 +1,122 @@
+import { CID } from 'multiformats'
+import * as uint8arrays from 'uint8arrays'
+import IpldStore from '../blockstore/ipld-store'
+import { sha256 } from '@atproto/crypto'
+import { MST, Leaf, NodeEntry, NodeData, MstOpts, Fanout } from './mst'
+import { cidForData } from '@atproto/common'
+
+type SupportedBases = 'base2' | 'base8' | 'base16' | 'base32' | 'base64'
+
+export const leadingZerosOnHash = async (
+  key: string,
+  fanout: Fanout,
+): Promise<number> => {
+  if ([2, 8, 16, 32, 64].indexOf(fanout) < 0) {
+    throw new Error(`Not a valid fanout: ${fanout}`)
+  }
+  const base: SupportedBases = `base${fanout}`
+  const zeroChar = uint8arrays.toString(new Uint8Array(1), base)[0]
+  const hash = await sha256(key)
+  const encoded = uint8arrays.toString(hash, base)
+  let count = 0
+  for (const char of encoded) {
+    if (char === zeroChar) {
+      count++
+    } else {
+      break
+    }
+  }
+  return count
+}
+
+export const layerForEntries = async (
+  entries: NodeEntry[],
+  fanout: Fanout,
+): Promise<number | null> => {
+  const firstLeaf = entries.find((entry) => entry.isLeaf())
+  if (!firstLeaf || firstLeaf.isTree()) return null
+  return await leadingZerosOnHash(firstLeaf.key, fanout)
+}
+
+export const deserializeNodeData = async (
+  blockstore: IpldStore,
+  data: NodeData,
+  opts?: Partial<MstOpts>,
+): Promise<NodeEntry[]> => {
+  const { layer, fanout } = opts || {}
+  const entries: NodeEntry[] = []
+  if (data.l !== null) {
+    entries.push(
+      await MST.load(blockstore, data.l, {
+        layer: layer ? layer - 1 : undefined,
+        fanout,
+      }),
+    )
+  }
+  let lastKey = ''
+  for (const entry of data.e) {
+    const key = lastKey.slice(0, entry.p) + entry.k
+    entries.push(new Leaf(key, entry.v))
+    lastKey = key
+    if (entry.t !== null) {
+      entries.push(
+        await MST.load(blockstore, entry.t, {
+          layer: layer ? layer - 1 : undefined,
+          fanout,
+        }),
+      )
+    }
+  }
+  return entries
+}
+
+export const serializeNodeData = (entries: NodeEntry[]): NodeData => {
+  const data: NodeData = {
+    l: null,
+    e: [],
+  }
+  let i = 0
+  if (entries[0]?.isTree()) {
+    i++
+    data.l = entries[0].pointer
+  }
+  let lastKey = ''
+  while (i < entries.length) {
+    const leaf = entries[i]
+    const next = entries[i + 1]
+    if (!leaf.isLeaf()) {
+      throw new Error('Not a valid node: two subtrees next to each other')
+    }
+    i++
+    let subtree: CID | null = null
+    if (next?.isTree()) {
+      subtree = next.pointer
+      i++
+    }
+    const prefixLen = countPrefixLen(lastKey, leaf.key)
+    data.e.push({
+      p: prefixLen,
+      k: leaf.key.slice(prefixLen),
+      v: leaf.value,
+      t: subtree,
+    })
+
+    lastKey = leaf.key
+  }
+  return data
+}
+
+export const countPrefixLen = (a: string, b: string): number => {
+  let i
+  for (i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) {
+      break
+    }
+  }
+  return i
+}
+
+export const cidForEntries = async (entries: NodeEntry[]): Promise<CID> => {
+  const data = serializeNodeData(entries)
+  return cidForData(data)
+}

package/src/mst/walker.ts

@@ -0,0 +1,120 @@
+import { MST, NodeEntry } from './mst'
+
+type WalkerStatusDone = {
+  done: true
+}
+
+type WalkerStatusProgress = {
+  done: false
+  curr: NodeEntry
+  walking: MST | null // walking set to null if `curr` is the root of the tree
+  index: number
+}
+
+type WalkerStatus = WalkerStatusDone | WalkerStatusProgress
+
+export class MstWalker {
+  stack: WalkerStatus[] = []
+  status: WalkerStatus
+
+  constructor(public root: MST) {
+    this.status = {
+      done: false,
+      curr: root,
+      walking: null,
+      index: 0,
+    }
+  }
+
+  // return the current layer of the node you are walking
+  layer(): number {
+    if (this.status.done) {
+      throw new Error('Walk is done')
+    }
+    if (this.status.walking) {
+      return this.status.walking.layer ?? 0
+    }
+    // if curr is the root of the tree, add 1
+    if (this.status.curr.isTree()) {
+      return (this.status.curr.layer ?? 0) + 1
+    }
+    throw new Error('Could not identify layer of walk')
+  }
+
+  // move to the next node in the subtree, skipping over the subtree
+  async stepOver(): Promise<void> {
+    if (this.status.done) return
+    // if stepping over the root of the node, we're done
+    if (this.status.walking === null) {
+      this.status = { done: true }
+      return
+    }
+    const entries = await this.status.walking.getEntries()
+    this.status.index++
+    const next = entries[this.status.index]
+    if (!next) {
+      const popped = this.stack.pop()
+      if (!popped) {
+        this.status = { done: true }
+        return
+      } else {
+        this.status = popped
+        await this.stepOver()
+        return
+      }
+    } else {
+      this.status.curr = next
+    }
+  }
+
+  // step into a subtree, throws if currently pointed at a leaf
+  async stepInto(): Promise<void> {
+    if (this.status.done) return
+    // edge case for very start of walk
+    if (this.status.walking === null) {
+      if (!this.status.curr.isTree()) {
+        throw new Error('The root of the tree cannot be a leaf')
+      }
+      const next = await this.status.curr.atIndex(0)
+      if (!next) {
+        this.status = { done: true }
+      } else {
+        this.status = {
+          done: false,
+          walking: this.status.curr,
+          curr: next,
+          index: 0,
+        }
+      }
+      return
+    }
+    if (!this.status.curr.isTree()) {
+      throw new Error('No tree at pointer, cannot step into')
+    }
+
+    const next = await this.status.curr.atIndex(0)
+    if (!next) {
+      throw new Error(
+        'Tried to step into a node with 0 entries which is invalid',
+      )
+    }
+
+    this.stack.push({ ...this.status })
+    this.status.walking = this.status.curr
+    this.status.curr = next
+    this.status.index = 0
+  }
+
+  // advance the pointer to the next node in the tree,
+  // stepping into the current node if necessary
+  async advance(): Promise<void> {
+    if (this.status.done) return
+    if (this.status.curr.isLeaf()) {
+      await this.stepOver()
+    } else {
+      await this.stepInto()
+    }
+  }
+}
+
+export default MstWalker

package/src/repo.ts

@@ -0,0 +1,312 @@
+import { CID } from 'multiformats/cid'
+import { CarWriter } from '@ipld/car'
+import { BlockWriter } from '@ipld/car/writer'
+import {
+  RepoRoot,
+  Commit,
+  def,
+  DataStore,
+  RepoMeta,
+  RecordCreateOp,
+  RecordWriteOp,
+} from './types'
+import { streamToArray } from '@atproto/common'
+import IpldStore from './blockstore/ipld-store'
+import * as auth from '@atproto/auth'
+import { MST } from './mst'
+import log from './logger'
+import * as util from './util'
+
+type Params = {
+  blockstore: IpldStore
+  data: DataStore
+  commit: Commit
+  root: RepoRoot
+  meta: RepoMeta
+  cid: CID
+  stagedWrites: RecordWriteOp[]
+}
+
+export class Repo {
+  blockstore: IpldStore
+  data: DataStore
+  commit: Commit
+  root: RepoRoot
+  meta: RepoMeta
+  cid: CID
+  stagedWrites: RecordWriteOp[]
+
+  constructor(params: Params) {
+    this.blockstore = params.blockstore
+    this.data = params.data
+    this.commit = params.commit
+    this.root = params.root
+    this.meta = params.meta
+    this.cid = params.cid
+    this.stagedWrites = params.stagedWrites
+  }
+
+  static async create(
+    blockstore: IpldStore,
+    did: string,
+    authStore: auth.AuthStore,
+    initialRecords: RecordCreateOp[] = [],
+  ): Promise<Repo> {
+    let tokenCid: CID | null = null
+    if (!(await authStore.canSignForDid(did))) {
+      const foundUcan = await authStore.findUcan(auth.maintenanceCap(did))
+      if (foundUcan === null) {
+        throw new Error(`No valid Ucan for creating repo`)
+      }
+      tokenCid = await blockstore.stage(auth.encodeUcan(foundUcan))
+    }
+
+    let data = await MST.create(blockstore)
+    for (const write of initialRecords) {
+      const cid = await blockstore.stage(write.value)
+      const dataKey = write.collection + '/' + write.rkey
+      data = await data.add(dataKey, cid)
+    }
+    const dataCid = await data.stage()
+
+    const meta: RepoMeta = {
+      did,
+      version: 1,
+      datastore: 'mst',
+    }
+    const metaCid = await blockstore.stage(meta)
+
+    const root: RepoRoot = {
+      meta: metaCid,
+      prev: null,
+      auth_token: tokenCid,
+      data: dataCid,
+    }
+
+    const rootCid = await blockstore.stage(root)
+    const commit: Commit = {
+      root: rootCid,
+      sig: await authStore.sign(rootCid.bytes),
+    }
+
+    const cid = await blockstore.stage(commit)
+
+    await blockstore.saveStaged()
+
+    log.info({ did }, `created repo`)
+    return new Repo({
+      blockstore,
+      data,
+      commit,
+      root,
+      meta,
+      cid,
+      stagedWrites: [],
+    })
+  }
+
+  static async load(blockstore: IpldStore, cid: CID) {
+    const commit = await blockstore.get(cid, def.commit)
+    const root = await blockstore.get(commit.root, def.repoRoot)
+    const meta = await blockstore.get(root.meta, def.repoMeta)
+    const data = await MST.load(blockstore, root.data)
+    log.info({ did: meta.did }, 'loaded repo for')
+    return new Repo({
+      blockstore,
+      data,
+      commit,
+      root,
+      meta,
+      cid,
+      stagedWrites: [],
+    })
+  }
+
+  private updateRepo(params: Partial<Params>): Repo {
+    return new Repo({
+      blockstore: params.blockstore || this.blockstore,
+      data: params.data || this.data,
+      commit: params.commit || this.commit,
+      root: params.root || this.root,
+      meta: params.meta || this.meta,
+      cid: params.cid || this.cid,
+      stagedWrites: params.stagedWrites || this.stagedWrites,
+    })
+  }
+
+  get did(): string {
+    return this.meta.did
+  }
+
+  async getRecord(collection: string, rkey: string): Promise<unknown | null> {
+    const dataKey = collection + '/' + rkey
+    const cid = await this.data.get(dataKey)
+    if (!cid) return null
+    return this.blockstore.getUnchecked(cid)
+  }
+
+  stageUpdate(write: RecordWriteOp | RecordWriteOp[]): Repo {
+    const writeArr = Array.isArray(write) ? write : [write]
+    return this.updateRepo({
+      stagedWrites: [...this.stagedWrites, ...writeArr],
+    })
+  }
+
+  async createCommit(
+    authStore: auth.AuthStore,
+    performUpdate?: (prev: CID, curr: CID) => Promise<CID | null>,
+  ): Promise<Repo> {
+    let data = this.data
+    for (const write of this.stagedWrites) {
+      if (write.action === 'create') {
+        const cid = await this.blockstore.stage(write.value)
+        const dataKey = write.collection + '/' + write.rkey
+        data = await data.add(dataKey, cid)
+      } else if (write.action === 'update') {
+        const cid = await this.blockstore.stage(write.value)
+        const dataKey = write.collection + '/' + write.rkey
+        data = await data.update(dataKey, cid)
+      } else if (write.action === 'delete') {
+        const dataKey = write.collection + '/' + write.rkey
+        data = await data.delete(dataKey)
+      }
+    }
+    const token = (await authStore.canSignForDid(this.did))
+      ? null
+      : await util.ucanForOperation(this.data, data, this.did, authStore)
+    const tokenCid = token ? await this.blockstore.stage(token) : null
+
+    const dataCid = await data.stage()
+    const root: RepoRoot = {
+      meta: this.root.meta,
+      prev: this.cid,
+      auth_token: tokenCid,
+      data: dataCid,
+    }
+    const rootCid = await this.blockstore.stage(root)
+    const commit: Commit = {
+      root: rootCid,
+      sig: await authStore.sign(rootCid.bytes),
+    }
+    const commitCid = await this.blockstore.stage(commit)
+
+    if (performUpdate) {
+      const rebaseOn = await performUpdate(this.cid, commitCid)
+      if (rebaseOn) {
+        await this.blockstore.clearStaged()
+        const rebaseRepo = await Repo.load(this.blockstore, rebaseOn)
+        return rebaseRepo.createCommit(authStore, performUpdate)
+      } else {
+        await this.blockstore.saveStaged()
+      }
+    } else {
+      await this.blockstore.saveStaged()
+    }
+
+    return this.updateRepo({
+      cid: commitCid,
+      root,
+      commit,
+      data,
+      stagedWrites: [],
+    })
+  }
+
+  async revert(count: number): Promise<Repo> {
+    let revertTo = this.cid
+    for (let i = 0; i < count; i++) {
+      const commit = await this.blockstore.get(revertTo, def.commit)
+      const root = await this.blockstore.get(commit.root, def.repoRoot)
+      if (root.prev === null) {
+        throw new Error(`Could not revert ${count} commits`)
+      }
+      revertTo = root.prev
+    }
+    return Repo.load(this.blockstore, revertTo)
+  }
+
+  // CAR FILES
+  // -----------
+
+  async getCarNoHistory(): Promise<Uint8Array> {
+    return this.openCar((car: BlockWriter) => {
+      return this.writeCheckoutToCarStream(car)
+    })
+  }
+
+  async getDiffCar(to: CID | null): Promise<Uint8Array> {
+    return this.openCar((car: BlockWriter) => {
+      return this.writeCommitsToCarStream(car, to, this.cid)
+    })
+  }
+
+  async getFullHistory(): Promise<Uint8Array> {
+    return this.getDiffCar(null)
+  }
+
+  private async openCar(
+    fn: (car: BlockWriter) => Promise<void>,
+  ): Promise<Uint8Array> {
+    const { writer, out } = CarWriter.create([this.cid])
+    try {
+      await fn(writer)
+    } finally {
+      writer.close()
+    }
+    return streamToArray(out)
+  }
+
+  async writeCheckoutToCarStream(car: BlockWriter): Promise<void> {
+    const commit = await this.blockstore.get(this.cid, def.commit)
+    const root = await this.blockstore.get(commit.root, def.repoRoot)
+    await this.blockstore.addToCar(car, this.cid)
+    await this.blockstore.addToCar(car, commit.root)
+    await this.blockstore.addToCar(car, root.meta)
+    if (root.auth_token) {
+      await this.blockstore.addToCar(car, root.auth_token)
+    }
+    await this.data.writeToCarStream(car)
+  }
+
+  async writeCommitsToCarStream(
+    car: BlockWriter,
+    oldestCommit: CID | null,
+    recentCommit: CID,
+  ): Promise<void> {
+    const commitPath = await util.getCommitPath(
+      this.blockstore,
+      oldestCommit,
+      recentCommit,
+    )
+    if (commitPath === null) {
+      throw new Error('Could not find shared history')
+    }
+    if (commitPath.length === 0) return
+    const firstHeadInPath = await Repo.load(this.blockstore, commitPath[0])
+    // handle the first commit
+    let prevHead: Repo | null =
+      firstHeadInPath.root.prev !== null
+        ? await Repo.load(this.blockstore, firstHeadInPath.root.prev)
+        : null
+    for (const commit of commitPath) {
+      const nextHead = await Repo.load(this.blockstore, commit)
+      await this.blockstore.addToCar(car, nextHead.cid)
+      await this.blockstore.addToCar(car, nextHead.commit.root)
+      await this.blockstore.addToCar(car, nextHead.root.meta)
+      if (nextHead.root.auth_token) {
+        await this.blockstore.addToCar(car, nextHead.root.auth_token)
+      }
+      if (prevHead === null) {
+        await nextHead.data.writeToCarStream(car)
+      } else {
+        const diff = await prevHead.data.diff(nextHead.data)
+        await Promise.all(
+          diff.newCidList().map((cid) => this.blockstore.addToCar(car, cid)),
+        )
+      }
+      prevHead = nextHead
+    }
+  }
+}
+
+export default Repo

package/src/storage/index.ts

@@ -0,0 +1 @@
+export * from './types'

package/src/storage/types.ts

@@ -0,0 +1,12 @@
+import stream from 'stream'
+import { CID } from 'multiformats/cid'
+
+export interface BlobStore {
+  putTemp(bytes: Uint8Array | stream.Readable): Promise<string>
+  makePermanent(key: string, cid: CID): Promise<void>
+  putPermanent(cid: CID, bytes: Uint8Array | stream.Readable): Promise<void>
+  getBytes(cid: CID): Promise<Uint8Array>
+  getStream(cid: CID): Promise<stream.Readable>
+}
+
+export class BlobNotFoundError extends Error {}

package/src/sync.ts

@@ -0,0 +1,38 @@
+import * as auth from '@atproto/auth'
+import { IpldStore } from './blockstore'
+import { DataDiff } from './mst'
+import Repo from './repo'
+import * as verify from './verify'
+
+export const loadRepoFromCar = async (
+  carBytes: Uint8Array,
+  blockstore: IpldStore,
+  verifier: auth.Verifier,
+): Promise<Repo> => {
+  const root = await blockstore.stageCar(carBytes)
+  const repo = await Repo.load(blockstore, root)
+  await verify.verifyUpdates(blockstore, null, repo.cid, verifier)
+  await blockstore.saveStaged()
+  return repo
+}
+
+export const loadDiff = async (
+  repo: Repo,
+  diffCar: Uint8Array,
+  verifier: auth.Verifier,
+): Promise<{ repo: Repo; diff: DataDiff }> => {
+  const blockstore = repo.blockstore
+  const root = await blockstore.stageCar(diffCar)
+  const diff = await verify.verifyUpdates(
+    repo.blockstore,
+    repo.cid,
+    root,
+    verifier,
+  )
+  const updatedRepo = await Repo.load(blockstore, root)
+  await blockstore.saveStaged()
+  return {
+    repo: updatedRepo,
+    diff,
+  }
+}

package/src/types.ts

@@ -0,0 +1,101 @@
+import { z } from 'zod'
+import { BlockWriter } from '@ipld/car/writer'
+import { def as common } from '@atproto/common'
+import { CID } from 'multiformats'
+import { DataDiff } from './mst'
+
+const repoMeta = z.object({
+  did: z.string(),
+  version: z.number(),
+  datastore: z.string(),
+})
+export type RepoMeta = z.infer<typeof repoMeta>
+
+const repoRoot = z.object({
+  meta: common.cid,
+  prev: common.cid.nullable(),
+  auth_token: common.cid.nullable(),
+  data: common.cid,
+})
+export type RepoRoot = z.infer<typeof repoRoot>
+
+const commit = z.object({
+  root: common.cid,
+  sig: common.bytes,
+})
+export type Commit = z.infer<typeof commit>
+
+export const cidCreateOp = z.object({
+  action: z.literal('create'),
+  collection: z.string(),
+  rkey: z.string(),
+  cid: common.cid,
+})
+export type CidCreateOp = z.infer<typeof cidCreateOp>
+
+export const cidUpdateOp = z.object({
+  action: z.literal('update'),
+  collection: z.string(),
+  rkey: z.string(),
+  cid: common.cid,
+})
+export type CidUpdateOp = z.infer<typeof cidUpdateOp>
+
+export const deleteOp = z.object({
+  action: z.literal('delete'),
+  collection: z.string(),
+  rkey: z.string(),
+})
+export type DeleteOp = z.infer<typeof deleteOp>
+
+export const cidWriteOp = z.union([cidCreateOp, cidUpdateOp, deleteOp])
+export type CidWriteOp = z.infer<typeof cidWriteOp>
+
+export const recordCreateOp = z.object({
+  action: z.literal('create'),
+  collection: z.string(),
+  rkey: z.string(),
+  value: z.any(),
+})
+export type RecordCreateOp = z.infer<typeof recordCreateOp>
+
+export const recordUpdateOp = z.object({
+  action: z.literal('update'),
+  collection: z.string(),
+  rkey: z.string(),
+  value: z.any(),
+})
+export type RecordUpdateOp = z.infer<typeof recordUpdateOp>
+
+export const recordWriteOp = z.union([recordCreateOp, recordUpdateOp, deleteOp])
+export type RecordWriteOp = z.infer<typeof recordWriteOp>
+
+export const def = {
+  ...common,
+  repoMeta,
+  repoRoot,
+  commit,
+  cidWriteOp,
+  recordWriteOp,
+}
+
+export interface CarStreamable {
+  writeToCarStream(car: BlockWriter): Promise<void>
+}
+
+export type DataValue = {
+  key: string
+  value: CID
+}
+
+export interface DataStore {
+  add(key: string, value: CID): Promise<DataStore>
+  update(key: string, value: CID): Promise<DataStore>
+  delete(key: string): Promise<DataStore>
+  get(key: string): Promise<CID | null>
+  list(count: number, after?: string, before?: string): Promise<DataValue[]>
+  listWithPrefix(prefix: string, count?: number): Promise<DataValue[]>
+  diff(other: DataStore): Promise<DataDiff>
+  stage(): Promise<CID>
+  writeToCarStream(car: BlockWriter): Promise<void>
+}