@atproto/repo 0.0.1

package/src/blockstore/index.ts

@@ -0,0 +1,2 @@
+export * from './ipld-store'
+export * from './memory-blockstore'

package/src/blockstore/ipld-store.ts

@@ -0,0 +1,103 @@
+import { CID } from 'multiformats/cid'
+import { BlockWriter } from '@ipld/car/writer'
+
+import * as common from '@atproto/common'
+import { check, util, valueToIpldBlock } from '@atproto/common'
+import { BlockReader } from '@ipld/car/api'
+import CidSet from '../cid-set'
+import { CarReader } from '@ipld/car/reader'
+
+export abstract class IpldStore {
+  staged: Map<string, Uint8Array>
+
+  constructor() {
+    this.staged = new Map()
+  }
+
+  abstract getSavedBytes(cid: CID): Promise<Uint8Array | null>
+  abstract hasSavedBlock(cid: CID): Promise<boolean>
+  abstract saveStaged(): Promise<void>
+  abstract destroySaved(): Promise<void>
+
+  async stageBytes(k: CID, v: Uint8Array): Promise<void> {
+    this.staged.set(k.toString(), v)
+  }
+
+  async stage(value: unknown): Promise<CID> {
+    const block = await valueToIpldBlock(value)
+    await this.stageBytes(block.cid, block.bytes)
+    return block.cid
+  }
+
+  async getBytes(cid: CID): Promise<Uint8Array> {
+    const fromStaged = this.staged.get(cid.toString())
+    if (fromStaged) return fromStaged
+    const fromBlocks = await this.getSavedBytes(cid)
+    if (fromBlocks) return fromBlocks
+    throw new Error(`Not found: ${cid.toString()}`)
+  }
+
+  async get<T>(cid: CID, schema: check.Def<T>): Promise<T> {
+    const value = await this.getUnchecked(cid)
+    try {
+      return check.assure(schema, value)
+    } catch (err) {
+      throw new Error(
+        `Did not find expected object at ${cid.toString()}: ${err}`,
+      )
+    }
+  }
+
+  async getUnchecked(cid: CID): Promise<unknown> {
+    const bytes = await this.getBytes(cid)
+    return common.ipldBytesToValue(bytes)
+  }
+
+  async has(cid: CID): Promise<boolean> {
+    return this.staged.has(cid.toString()) || this.hasSavedBlock(cid)
+  }
+
+  async isMissing(cid: CID): Promise<boolean> {
+    const has = await this.has(cid)
+    return !has
+  }
+
+  async checkMissing(cids: CidSet): Promise<CidSet> {
+    const missing = await util.asyncFilter(cids.toList(), (c) => {
+      return this.isMissing(c)
+    })
+    return new CidSet(missing)
+  }
+
+  async clearStaged(): Promise<void> {
+    this.staged.clear()
+  }
+
+  async destroy(): Promise<void> {
+    this.clearStaged()
+    await this.destroySaved()
+  }
+
+  async addToCar(car: BlockWriter, cid: CID) {
+    car.put({ cid, bytes: await this.getBytes(cid) })
+  }
+
+  async stageCar(buf: Uint8Array): Promise<CID> {
+    const car = await CarReader.fromBytes(buf)
+    const roots = await car.getRoots()
+    if (roots.length !== 1) {
+      throw new Error(`Expected one root, got ${roots.length}`)
+    }
+    const rootCid = roots[0]
+    await this.stageCarBlocks(car)
+    return rootCid
+  }
+
+  async stageCarBlocks(car: BlockReader): Promise<void> {
+    for await (const block of car.blocks()) {
+      await this.stageBytes(block.cid, block.bytes)
+    }
+  }
+}
+
+export default IpldStore

package/src/blockstore/memory-blockstore.ts

@@ -0,0 +1,49 @@
+import { CID } from 'multiformats/cid'
+import IpldStore from './ipld-store'
+
+export class MemoryBlockstore extends IpldStore {
+  blocks: Map<string, Uint8Array>
+
+  constructor() {
+    super()
+    this.blocks = new Map()
+  }
+
+  async getSavedBytes(cid: CID): Promise<Uint8Array | null> {
+    return this.blocks.get(cid.toString()) || null
+  }
+
+  async hasSavedBlock(cid: CID): Promise<boolean> {
+    return this.blocks.has(cid.toString())
+  }
+
+  async saveStaged(): Promise<void> {
+    this.staged.forEach((val, key) => {
+      this.blocks.set(key, val)
+    })
+    this.clearStaged()
+  }
+
+  async sizeInBytes(): Promise<number> {
+    let total = 0
+    for (const val of this.blocks.values()) {
+      total += val.byteLength
+    }
+    return total
+  }
+
+  async destroySaved(): Promise<void> {
+    this.blocks.clear()
+  }
+
+  // Mainly for dev purposes
+  async getContents(): Promise<Record<string, unknown>> {
+    const contents: Record<string, unknown> = {}
+    for (const key of this.blocks.keys()) {
+      contents[key] = await this.getUnchecked(CID.parse(key))
+    }
+    return contents
+  }
+}
+
+export default MemoryBlockstore

package/src/cid-set.ts

@@ -0,0 +1,50 @@
+import { CID } from 'multiformats'
+
+export class CidSet {
+  private set: Set<string>
+
+  constructor(arr: CID[] = []) {
+    const strArr = arr.map((c) => c.toString())
+    this.set = new Set(strArr)
+  }
+
+  add(cid: CID): CidSet {
+    this.set.add(cid.toString())
+    return this
+  }
+
+  addSet(toMerge: CidSet): CidSet {
+    toMerge.toList().map((c) => this.add(c))
+    return this
+  }
+
+  subtractSet(toSubtract: CidSet): CidSet {
+    toSubtract.toList().map((c) => this.delete(c))
+    return this
+  }
+
+  delete(cid: CID) {
+    this.set.delete(cid.toString())
+    return this
+  }
+
+  has(cid: CID): boolean {
+    return this.set.has(cid.toString())
+  }
+
+  size(): number {
+    return this.set.size
+  }
+
+  clear(): CidSet {
+    this.set.clear()
+    return this
+  }
+
+  toList(): CID[] {
+    const arr = [...this.set]
+    return arr.map((c) => CID.parse(c))
+  }
+}
+
+export default CidSet

package/src/index.ts

@@ -0,0 +1,7 @@
+export * from './blockstore'
+export * from './repo'
+export * from './mst'
+export * from './storage'
+export * from './types'
+export * from './verify'
+export * from './util'

package/src/logger.ts

@@ -0,0 +1,5 @@
+import { subsystemLogger } from '@atproto/common'
+
+export const logger = subsystemLogger('repo')
+
+export default logger

package/src/mst/diff.ts

@@ -0,0 +1,106 @@
+import * as auth from '@atproto/auth'
+import { CID } from 'multiformats'
+import CidSet from '../cid-set'
+import { parseRecordKey } from '../util'
+
+export class DataDiff {
+  adds: Record<string, DataAdd> = {}
+  updates: Record<string, DataUpdate> = {}
+  deletes: Record<string, DataDelete> = {}
+
+  newCids: CidSet = new CidSet()
+
+  recordAdd(key: string, cid: CID): void {
+    this.adds[key] = { key, cid }
+    this.newCids.add(cid)
+  }
+
+  recordUpdate(key: string, prev: CID, cid: CID): void {
+    this.updates[key] = { key, prev, cid }
+    this.newCids.add(cid)
+  }
+
+  recordDelete(key: string, cid: CID): void {
+    this.deletes[key] = { key, cid }
+  }
+
+  recordNewCid(cid: CID): void {
+    this.newCids.add(cid)
+  }
+
+  addDiff(diff: DataDiff) {
+    for (const add of diff.addList()) {
+      if (this.deletes[add.key]) {
+        const del = this.deletes[add.key]
+        if (del.cid !== add.cid) {
+          this.recordUpdate(add.key, del.cid, add.cid)
+        }
+        delete this.deletes[add.key]
+      } else {
+        this.recordAdd(add.key, add.cid)
+      }
+    }
+    for (const update of diff.updateList()) {
+      this.recordUpdate(update.key, update.prev, update.cid)
+      delete this.adds[update.key]
+      delete this.deletes[update.key]
+    }
+    for (const del of diff.deleteList()) {
+      if (this.adds[del.key]) {
+        delete this.adds[del.key]
+      } else {
+        delete this.updates[del.key]
+        this.recordDelete(del.key, del.cid)
+      }
+    }
+    this.newCids.addSet(diff.newCids)
+  }
+
+  addList(): DataAdd[] {
+    return Object.values(this.adds)
+  }
+
+  updateList(): DataUpdate[] {
+    return Object.values(this.updates)
+  }
+
+  deleteList(): DataDelete[] {
+    return Object.values(this.deletes)
+  }
+
+  newCidList(): CID[] {
+    return this.newCids.toList()
+  }
+
+  updatedKeys(): string[] {
+    const keys = [
+      ...Object.keys(this.adds),
+      ...Object.keys(this.updates),
+      ...Object.keys(this.deletes),
+    ]
+    return [...new Set(keys)]
+  }
+
+  neededCapabilities(rootDid: string): auth.ucans.Capability[] {
+    return this.updatedKeys().map((key) => {
+      const { collection, rkey } = parseRecordKey(key)
+      return auth.writeCap(rootDid, collection, rkey)
+    })
+  }
+}
+
+export type DataAdd = {
+  key: string
+  cid: CID
+}
+
+export type DataUpdate = {
+  key: string
+  prev: CID
+  cid: CID
+}
+
+export type DataDelete = {
+  key: string
+  cid: CID
+}

package/src/mst/index.ts

@@ -0,0 +1,4 @@
+export * from './mst'
+export * from './diff'
+export * from './walker'
+export * as mstUtil from './util'