@atproto/repo 0.8.13 → 0.9.0

package/tests/mst.test.ts

@@ -1,4 +1,5 @@
-import { CID } from 'multiformats'
+import assert from 'node:assert'
+import { Cid, parseCid } from '@atproto/lex-data'
 import { DataAdd, DataDelete, DataDiff, DataUpdate } from '../src/data-diff'
 import { MST } from '../src/mst'
 import { InvalidMstKeyError, countPrefixLen } from '../src/mst/util'
@@ -8,8 +9,8 @@ import * as util from './_util'
 describe('Merkle Search Tree', () => {
   let blockstore: MemoryBlockstore
   let mst: MST
-  let mapping: Record<string, CID>
-  let shuffled: [string, CID][]
+  let mapping: Record<string, Cid>
+  let shuffled: [string, Cid][]
 
   beforeAll(async () => {
     blockstore = new MemoryBlockstore()
@@ -24,7 +25,8 @@ describe('Merkle Search Tree', () => {
     }
     for (const entry of shuffled) {
       const got = await mst.get(entry[0])
-      expect(entry[1].equals(got)).toBeTruthy()
+      assert(got !== null)
+      expect(entry[1].equals(got)).toBe(true)
     }
 
     const totalSize = await mst.leafCount()
@@ -35,7 +37,7 @@ describe('Merkle Search Tree', () => {
     let editedMst = mst
     const toEdit = shuffled.slice(0, 100)
 
-    const edited: [string, CID][] = []
+    const edited: [string, Cid][] = []
     for (const entry of toEdit) {
       const newCid = await util.randomCid()
       editedMst = await editedMst.update(entry[0], newCid)
@@ -44,7 +46,8 @@ describe('Merkle Search Tree', () => {
 
     for (const entry of edited) {
       const got = await editedMst.get(entry[0])
-      expect(entry[1].equals(got)).toBeTruthy()
+      assert(got !== null)
+      expect(entry[1].equals(got)).toBe(true)
     }
 
     const totalSize = await editedMst.leafCount()
@@ -68,7 +71,8 @@ describe('Merkle Search Tree', () => {
     }
     for (const entry of theRest) {
       const got = await deletedMst.get(entry[0])
-      expect(entry[1].equals(got)).toBeTruthy()
+      assert(got !== null)
+      expect(entry[1].equals(got)).toBe(true)
     }
   })
 
@@ -142,12 +146,7 @@ describe('Merkle Search Tree', () => {
 
     // ensure we correctly report all added CIDs
     for await (const entry of toDiff.walk()) {
-      let cid: CID
-      if (entry.isTree()) {
-        cid = await entry.getPointer()
-      } else {
-        cid = entry.value
-      }
+      const cid = entry.isTree() ? await entry.getPointer() : entry.value
       const found =
         (await blockstore.has(cid)) ||
         diff.newMstBlocks.has(cid) ||
@@ -177,12 +176,12 @@ describe('Merkle Search Tree', () => {
   describe('MST Interop Allowable Keys', () => {
     let blockstore: MemoryBlockstore
     let mst: MST
-    let cid1: CID
+    let cid1: Cid
 
     beforeAll(async () => {
       blockstore = new MemoryBlockstore()
       mst = await MST.create(blockstore)
-      cid1 = CID.parse(
+      cid1 = parseCid(
         'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
       )
     })
@@ -257,11 +256,11 @@ describe('Merkle Search Tree', () => {
   describe('MST Interop Known Maps', () => {
     let blockstore: MemoryBlockstore
     let mst: MST
-    let cid1: CID
+    let cid1: Cid
 
     beforeAll(async () => {
       blockstore = new MemoryBlockstore()
-      cid1 = CID.parse(
+      cid1 = parseCid(
         'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
       )
     })
@@ -310,11 +309,11 @@ describe('Merkle Search Tree', () => {
   describe('MST Interop Edge Cases', () => {
     let blockstore: MemoryBlockstore
     let mst: MST
-    let cid1: CID
+    let cid1: Cid
 
     beforeAll(async () => {
       blockstore = new MemoryBlockstore()
-      cid1 = CID.parse(
+      cid1 = parseCid(
         'bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454',
       )
     })

package/tests/proofs.test.ts

@@ -1,5 +1,7 @@
-import { TID, cidForCbor, streamToBuffer } from '@atproto/common'
+import { TID } from '@atproto/common-web'
 import * as crypto from '@atproto/crypto'
+import { cidForLex } from '@atproto/lex-cbor'
+import { NsidString } from '@atproto/syntax'
 import { RecordCidClaim, RecordPath, Repo, RepoContents } from '../src'
 import { MemoryBlockstore } from '../src/storage'
 import * as sync from '../src/sync'
@@ -23,7 +25,7 @@ describe('Repo Proofs', () => {
   })
 
   const getProofs = async (claims: RecordPath[]) => {
-    return streamToBuffer(sync.getRecords(storage, repo.cid, claims))
+    return util.toBuffer(sync.getRecords(storage, repo.cid, claims))
   }
 
   const doVerify = (proofs: Uint8Array, claims: RecordCidClaim[]) => {
@@ -34,12 +36,12 @@ describe('Repo Proofs', () => {
     contents: RepoContents,
   ): Promise<RecordCidClaim[]> => {
     const claims: RecordCidClaim[] = []
-    for (const coll of Object.keys(contents)) {
+    for (const coll of Object.keys(contents) as NsidString[]) {
       for (const rkey of Object.keys(contents[coll])) {
         claims.push({
           collection: coll,
           rkey: rkey,
-          cid: await cidForCbor(contents[coll][rkey]),
+          cid: await cidForLex(contents[coll][rkey]),
         })
       }
     }
@@ -136,7 +138,7 @@ describe('Repo Proofs', () => {
         throw new Error('Could not find record for claim')
       }
       expect(foundClaim.cid).toEqual(
-        await cidForCbor(repoData[record.collection][record.rkey]),
+        await cidForLex(repoData[record.collection][record.rkey]),
       )
     }
   })
@@ -144,7 +146,7 @@ describe('Repo Proofs', () => {
   it('verifyProofs throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
     const claims = await contentsToClaims(repoData)
-    const proofs = await streamToBuffer(
+    const proofs = await util.toBuffer(
       sync.getRecords(storage, badRepo.cid, claims),
     )
     const fn = sync.verifyProofs(proofs, claims, repoDid, keypair.did())

package/tests/repo.test.ts

@@ -1,4 +1,4 @@
-import { TID } from '@atproto/common'
+import { TID } from '@atproto/common-web'
 import * as crypto from '@atproto/crypto'
 import { Secp256k1Keypair } from '@atproto/crypto'
 import { RepoContents, WriteOpAction, verifyCommitSig } from '../src'

package/tests/sync.test.ts

@@ -1,4 +1,3 @@
-import { streamToBuffer } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import {
   CidSet,
@@ -31,7 +30,7 @@ describe('Repo Sync', () => {
   })
 
   it('sync a full repo', async () => {
-    const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const carBytes = await util.toBuffer(sync.getFullRepo(storage, repo.cid))
     const car = await readCarWithRoot(carBytes)
     const verified = await sync.verifyRepo(
       car.blocks,
@@ -54,7 +53,7 @@ describe('Repo Sync', () => {
   })
 
   it('does not sync duplicate blocks', async () => {
-    const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
+    const carBytes = await util.toBuffer(sync.getFullRepo(storage, repo.cid))
     const car = await readCar(carBytes)
     const cids = new CidSet()
     car.blocks.forEach((_, cid) => {
@@ -87,9 +86,7 @@ describe('Repo Sync', () => {
 
   it('throws on a bad signature', async () => {
     const badRepo = await util.addBadCommit(repo, keypair)
-    const carBytes = await streamToBuffer(
-      sync.getFullRepo(storage, badRepo.cid),
-    )
+    const carBytes = await util.toBuffer(sync.getFullRepo(storage, badRepo.cid))
     const car = await readCarWithRoot(carBytes)
     await expect(
       sync.verifyRepo(car.blocks, car.root, repoDid, keypair.did()),

package/bench/mst.bench.ts

@@ -1,165 +0,0 @@
-import fs from 'node:fs'
-import { CID } from 'multiformats'
-import { Fanout, MST, MemoryBlockstore, NodeEntry } from '../src'
-import * as util from '../tests/_util'
-
-type BenchmarkData = {
-  fanout: number
-  size: number
-  addTime: string
-  saveTime: string
-  walkTime: string
-  depth: number
-  maxWidth: number
-  blockstoreSize: number
-  largestProofSize: number
-  avgProofSize: number
-  widths: Record<number, number>
-}
-
-describe('MST Benchmarks', () => {
-  let mapping: Record<string, CID>
-  let shuffled: [string, CID][]
-
-  const size = 500000
-
-  beforeAll(async () => {
-    mapping = await util.generateBulkDataKeys(size)
-    shuffled = util.shuffle(Object.entries(mapping))
-  })
-
-  // const fanouts: Fanout[] = [8, 16, 32]
-  const fanouts: Fanout[] = [16, 32]
-  it('benchmarks various fanouts', async () => {
-    const benches: BenchmarkData[] = []
-    for (const fanout of fanouts) {
-      const blockstore = new MemoryBlockstore()
-      let mst = await MST.create(blockstore, [], { fanout })
-
-      const start = Date.now()
-
-      for (const entry of shuffled) {
-        mst = await mst.add(entry[0], entry[1])
-      }
-
-      const doneAdding = Date.now()
-
-      const root = await util.saveMst(blockstore, mst)
-
-      const doneSaving = Date.now()
-
-      const reloaded = await MST.load(blockstore, root, { fanout })
-      const widthTracker = new NodeWidths()
-      for await (const entry of reloaded.walk()) {
-        await widthTracker.trackEntry(entry)
-      }
-
-      const doneWalking = Date.now()
-
-      const paths = await reloaded.paths()
-      let largestProof = 0
-      let combinedProofSizes = 0
-      for (const path of paths) {
-        let proofSize = 0
-        for (const entry of path) {
-          if (entry.isTree()) {
-            const bytes = await blockstore.getBytes(entry.pointer)
-            if (!bytes) {
-              throw new Error(`Bytes not found: ${entry.pointer}`)
-            }
-            proofSize += bytes.byteLength
-          }
-        }
-        largestProof = Math.max(largestProof, proofSize)
-        combinedProofSizes += proofSize
-      }
-      const avgProofSize = Math.ceil(combinedProofSizes / paths.length)
-
-      const blockstoreSize = await blockstore.sizeInBytes()
-
-      benches.push({
-        fanout,
-        size,
-        addTime: secDiff(start, doneAdding),
-        saveTime: secDiff(doneAdding, doneSaving),
-        walkTime: secDiff(doneSaving, doneWalking),
-        depth: await mst.getLayer(),
-        blockstoreSize,
-        largestProofSize: largestProof,
-        avgProofSize: avgProofSize,
-        maxWidth: widthTracker.max,
-        widths: widthTracker.data,
-      })
-    }
-    writeBenchData(benches, 'mst-benchmarks')
-  })
-})
-
-const secDiff = (first: number, second: number): string => {
-  return ((second - first) / 1000).toFixed(3)
-}
-
-class NodeWidths {
-  data = {
-    0: 0,
-    16: 0,
-    32: 0,
-    48: 0,
-    64: 0,
-    96: 0,
-    128: 0,
-    160: 0,
-    192: 0,
-    224: 0,
-    256: 0,
-  }
-  max = 0
-
-  async trackEntry(entry: NodeEntry) {
-    if (!entry.isTree()) return
-    const entries = await entry.getEntries()
-    const width = entries.filter((e) => e.isLeaf()).length
-    this.max = Math.max(this.max, width)
-    if (width >= 0) this.data[0]++
-    if (width >= 16) this.data[16]++
-    if (width >= 32) this.data[32]++
-    if (width >= 48) this.data[48]++
-    if (width >= 64) this.data[64]++
-    if (width >= 96) this.data[96]++
-    if (width >= 128) this.data[128]++
-    if (width >= 160) this.data[160]++
-    if (width >= 192) this.data[192]++
-    if (width >= 224) this.data[224]++
-    if (width >= 256) this.data[256]++
-  }
-}
-
-const writeBenchData = (benches: BenchmarkData[], fileLoc: string) => {
-  let toWrite = ''
-  for (const bench of benches) {
-    toWrite += `Fanout: ${bench.fanout}
-----------------------
-Time to add ${bench.size} leaves: ${bench.addTime}s
-Time to save tree with ${bench.size} leaves: ${bench.saveTime}s
-Time to reconstruct & walk ${bench.size} leaves: ${bench.walkTime}s
-Tree depth: ${bench.depth}
-Max Node Width (only counting leaves): ${bench.maxWidth}
-The total blockstore size is: ${bench.blockstoreSize} bytes
-Largest proof size: ${bench.largestProofSize} bytes
-Average proof size: ${bench.avgProofSize} bytes
-Nodes with >= 0 leaves: ${bench.widths[0]}
-Nodes with >= 16 leaves: ${bench.widths[16]}
-Nodes with >= 32 leaves: ${bench.widths[32]}
-Nodes with >= 48 leaves: ${bench.widths[48]}
-Nodes with >= 64 leaves: ${bench.widths[64]}
-Nodes with >= 96 leaves: ${bench.widths[96]}
-Nodes with >= 128 leaves: ${bench.widths[128]}
-Nodes with >= 160 leaves: ${bench.widths[160]}
-Nodes with >= 192 leaves: ${bench.widths[192]}
-Nodes with >= 224 leaves: ${bench.widths[224]}
-Nodes with >= 256 leaves: ${bench.widths[256]}
-
-`
-  }
-  fs.writeFileSync(fileLoc, toWrite)
-}

package/bench/repo.bench.ts

@@ -1,48 +0,0 @@
-import { TID } from '@atproto/common'
-import * as crypto from '@atproto/crypto'
-import { Secp256k1Keypair } from '@atproto/crypto'
-import { MemoryBlockstore, Repo, WriteOpAction } from '../src'
-import * as util from '../tests/_util'
-
-describe('Repo Benchmarks', () => {
-  const size = 10000
-
-  let blockstore: MemoryBlockstore
-  let keypair: crypto.Keypair
-  let repo: Repo
-
-  beforeAll(async () => {
-    blockstore = new MemoryBlockstore()
-    keypair = await Secp256k1Keypair.create()
-    repo = await Repo.create(blockstore, await keypair.did(), keypair)
-  })
-
-  it('calculates size', async () => {
-    for (let i = 0; i < size; i++) {
-      if (i % 500 === 0) {
-        console.log(i)
-      }
-
-      await repo.applyCommit(
-        {
-          action: WriteOpAction.Create,
-          collection: 'app.bsky.post',
-          rkey: TID.nextStr(),
-          value: {
-            $type: 'app.bsky.post',
-            text: util.randomStr(150),
-            reply: {
-              root: 'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345',
-              parent:
-                'at://did:plc:1234abdefeoi23/app.bsky.post/12345678912345',
-            },
-            createdAt: new Date().toISOString(),
-          },
-        },
-        keypair,
-      )
-    }
-
-    console.log('SIZE: ', await blockstore.sizeInBytes())
-  })
-})