@atproto/repo 0.0.1

package/README.md

@@ -0,0 +1,3 @@
+# ATP Repo
+
+The "ATP repository" core implementation (a Merkle Search Tree).

package/babel.config.js

@@ -0,0 +1 @@
+module.exports = require('../../babel.config.js')

package/bench/mst.bench.ts

@@ -0,0 +1,162 @@
+import { CID } from 'multiformats'
+import { Fanout, MemoryBlockstore, MST, NodeEntry } from '../src'
+import * as util from '../tests/_util'
+import fs from 'fs'
+
+type BenchmarkData = {
+  fanout: number
+  size: number
+  addTime: string
+  saveTime: string
+  walkTime: string
+  depth: number
+  maxWidth: number
+  blockstoreSize: number
+  largestProofSize: number
+  avgProofSize: number
+  widths: Record<number, number>
+}
+
+describe('MST Benchmarks', () => {
+  let mapping: Record<string, CID>
+  let shuffled: [string, CID][]
+
+  const size = 500000
+
+  beforeAll(async () => {
+    mapping = await util.generateBulkTidMapping(size)
+    shuffled = util.shuffle(Object.entries(mapping))
+  })
+
+  // const fanouts: Fanout[] = [8, 16, 32]
+  const fanouts: Fanout[] = [16, 32]
+  it('benchmarks various fanouts', async () => {
+    let benches: BenchmarkData[] = []
+    for (const fanout of fanouts) {
+      const blockstore = new MemoryBlockstore()
+      let mst = await MST.create(blockstore, [], { fanout })
+
+      const start = Date.now()
+
+      for (const entry of shuffled) {
+        mst = await mst.add(entry[0], entry[1])
+      }
+
+      const doneAdding = Date.now()
+
+      const root = await mst.save()
+
+      const doneSaving = Date.now()
+
+      let reloaded = await MST.load(blockstore, root, { fanout })
+      const widthTracker = new NodeWidths()
+      for await (const entry of reloaded.walk()) {
+        await widthTracker.trackEntry(entry)
+      }
+
+      const doneWalking = Date.now()
+
+      const paths = await reloaded.paths()
+      let largestProof = 0
+      let combinedProofSizes = 0
+      for (const path of paths) {
+        let proofSize = 0
+        for (const entry of path) {
+          if (entry.isTree()) {
+            const bytes = await blockstore.getBytes(entry.pointer)
+            proofSize += bytes.byteLength
+          }
+        }
+        largestProof = Math.max(largestProof, proofSize)
+        combinedProofSizes += proofSize
+      }
+      const avgProofSize = Math.ceil(combinedProofSizes / paths.length)
+
+      const blockstoreSize = await blockstore.sizeInBytes()
+
+      benches.push({
+        fanout,
+        size,
+        addTime: secDiff(start, doneAdding),
+        saveTime: secDiff(doneAdding, doneSaving),
+        walkTime: secDiff(doneSaving, doneWalking),
+        depth: await mst.getLayer(),
+        blockstoreSize,
+        largestProofSize: largestProof,
+        avgProofSize: avgProofSize,
+        maxWidth: widthTracker.max,
+        widths: widthTracker.data,
+      })
+    }
+    writeBenchData(benches, 'mst-benchmarks')
+  })
+})
+
+const secDiff = (first: number, second: number): string => {
+  return ((second - first) / 1000).toFixed(3)
+}
+
+class NodeWidths {
+  data = {
+    0: 0,
+    16: 0,
+    32: 0,
+    48: 0,
+    64: 0,
+    96: 0,
+    128: 0,
+    160: 0,
+    192: 0,
+    224: 0,
+    256: 0,
+  }
+  max = 0
+
+  async trackEntry(entry: NodeEntry) {
+    if (!entry.isTree()) return
+    const entries = await entry.getEntries()
+    const width = entries.filter((e) => e.isLeaf()).length
+    this.max = Math.max(this.max, width)
+    if (width >= 0) this.data[0]++
+    if (width >= 16) this.data[16]++
+    if (width >= 32) this.data[32]++
+    if (width >= 48) this.data[48]++
+    if (width >= 64) this.data[64]++
+    if (width >= 96) this.data[96]++
+    if (width >= 128) this.data[128]++
+    if (width >= 160) this.data[160]++
+    if (width >= 192) this.data[192]++
+    if (width >= 224) this.data[224]++
+    if (width >= 256) this.data[256]++
+  }
+}
+
+const writeBenchData = (benches: BenchmarkData[], fileLoc: string) => {
+  let toWrite = ''
+  for (const bench of benches) {
+    toWrite += `Fanout: ${bench.fanout}
+----------------------
+Time to add ${bench.size} leaves: ${bench.addTime}s
+Time to save tree with ${bench.size} leaves: ${bench.saveTime}s
+Time to reconstruct & walk ${bench.size} leaves: ${bench.walkTime}s
+Tree depth: ${bench.depth}
+Max Node Width (only counting leaves): ${bench.maxWidth}
+The total blockstore size is: ${bench.blockstoreSize} bytes
+Largest proof size: ${bench.largestProofSize} bytes
+Average proof size: ${bench.avgProofSize} bytes
+Nodes with >= 0 leaves: ${bench.widths[0]}
+Nodes with >= 16 leaves: ${bench.widths[16]}
+Nodes with >= 32 leaves: ${bench.widths[32]}
+Nodes with >= 48 leaves: ${bench.widths[48]}
+Nodes with >= 64 leaves: ${bench.widths[64]}
+Nodes with >= 96 leaves: ${bench.widths[96]}
+Nodes with >= 128 leaves: ${bench.widths[128]}
+Nodes with >= 160 leaves: ${bench.widths[160]}
+Nodes with >= 192 leaves: ${bench.widths[192]}
+Nodes with >= 224 leaves: ${bench.widths[224]}
+Nodes with >= 256 leaves: ${bench.widths[256]}
+
+`
+  }
+  fs.writeFileSync(fileLoc, toWrite)
+}

package/bench/repo.bench.ts

@@ -0,0 +1,39 @@
+import * as auth from '@atproto/auth'
+import { MemoryBlockstore, Repo } from '../src'
+import * as util from '../tests/_util'
+
+describe('Repo Benchmarks', () => {
+  const verifier = new auth.Verifier()
+  const size = 10000
+
+  let blockstore: MemoryBlockstore
+  let authStore: auth.AuthStore
+  let repo: Repo
+
+  beforeAll(async () => {
+    blockstore = new MemoryBlockstore()
+    authStore = await verifier.createTempAuthStore()
+    await authStore.claimFull()
+    repo = await Repo.create(blockstore, await authStore.did(), authStore)
+  })
+
+  it('calculates size', async () => {
+    const posts = repo.getCollection('app.bsky.post')
+    for (let i = 0; i < size; i++) {
+      if (i % 500 === 0) {
+        console.log(i)
+      }
+      await posts.createRecord({
+        $type: 'app.bsky.post',
+        text: util.randomStr(150),
+        reply: {
+          root: 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345',
+          parent: 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345',
+        },
+        createdAt: new Date().toISOString(),
+      })
+    }
+
+    console.log('SIZE: ', await blockstore.sizeInBytes())
+  })
+})

package/build.js

@@ -0,0 +1,22 @@
+const pkgJson = require('@npmcli/package-json')
+const { nodeExternalsPlugin } = require('esbuild-node-externals')
+
+const buildShallow =
+  process.argv.includes('--shallow') || process.env.ATP_BUILD_SHALLOW === 'true'
+
+if (process.argv.includes('--update-main-to-dist')) {
+  return pkgJson
+    .load(__dirname)
+    .then((pkg) => pkg.update({ main: 'dist/index.js' }))
+    .then((pkg) => pkg.save())
+}
+
+require('esbuild').build({
+  logLevel: 'info',
+  entryPoints: ['src/index.ts'],
+  bundle: true,
+  sourcemap: true,
+  outdir: 'dist',
+  platform: 'node',
+  plugins: buildShallow ? [nodeExternalsPlugin()] : [],
+})

package/dist/blockstore/index.d.ts

@@ -0,0 +1,2 @@
+export * from './ipld-store';
+export * from './memory-blockstore';

package/dist/blockstore/ipld-store.d.ts

@@ -0,0 +1,27 @@
+import { CID } from 'multiformats/cid';
+import { BlockWriter } from '@ipld/car/writer';
+import { check } from '@atproto/common';
+import { BlockReader } from '@ipld/car/api';
+import CidSet from '../cid-set';
+export declare abstract class IpldStore {
+    staged: Map<string, Uint8Array>;
+    constructor();
+    abstract getSavedBytes(cid: CID): Promise<Uint8Array | null>;
+    abstract hasSavedBlock(cid: CID): Promise<boolean>;
+    abstract saveStaged(): Promise<void>;
+    abstract destroySaved(): Promise<void>;
+    stageBytes(k: CID, v: Uint8Array): Promise<void>;
+    stage(value: unknown): Promise<CID>;
+    getBytes(cid: CID): Promise<Uint8Array>;
+    get<T>(cid: CID, schema: check.Def<T>): Promise<T>;
+    getUnchecked(cid: CID): Promise<unknown>;
+    has(cid: CID): Promise<boolean>;
+    isMissing(cid: CID): Promise<boolean>;
+    checkMissing(cids: CidSet): Promise<CidSet>;
+    clearStaged(): Promise<void>;
+    destroy(): Promise<void>;
+    addToCar(car: BlockWriter, cid: CID): Promise<void>;
+    stageCar(buf: Uint8Array): Promise<CID>;
+    stageCarBlocks(car: BlockReader): Promise<void>;
+}
+export default IpldStore;

package/dist/blockstore/memory-blockstore.d.ts

@@ -0,0 +1,13 @@
+import { CID } from 'multiformats/cid';
+import IpldStore from './ipld-store';
+export declare class MemoryBlockstore extends IpldStore {
+    blocks: Map<string, Uint8Array>;
+    constructor();
+    getSavedBytes(cid: CID): Promise<Uint8Array | null>;
+    hasSavedBlock(cid: CID): Promise<boolean>;
+    saveStaged(): Promise<void>;
+    sizeInBytes(): Promise<number>;
+    destroySaved(): Promise<void>;
+    getContents(): Promise<Record<string, unknown>>;
+}
+export default MemoryBlockstore;

package/dist/cid-set.d.ts

@@ -0,0 +1,14 @@
+import { CID } from 'multiformats';
+export declare class CidSet {
+    private set;
+    constructor(arr?: CID[]);
+    add(cid: CID): CidSet;
+    addSet(toMerge: CidSet): CidSet;
+    subtractSet(toSubtract: CidSet): CidSet;
+    delete(cid: CID): this;
+    has(cid: CID): boolean;
+    size(): number;
+    clear(): CidSet;
+    toList(): CID[];
+}
+export default CidSet;

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export * from './blockstore';
+export * from './repo';
+export * from './mst';
+export * from './storage';
+export * from './types';
+export * from './verify';
+export * from './util';