@atproto/repo 0.7.3 → 0.8.0

package/src/sync/provider.ts

@@ -1,5 +1,5 @@
-import { BlockWriter } from '@ipld/car/writer'
 import { CID } from 'multiformats/cid'
+import { writeCarStream } from '../car'
 import { CidSet } from '../cid-set'
 import { MissingBlocksError } from '../error'
 import { MST } from '../mst'
@@ -14,12 +14,16 @@ export const getFullRepo = (
   storage: RepoStorage,
   commitCid: CID,
 ): AsyncIterable<Uint8Array> => {
-  return util.writeCar(commitCid, async (car: BlockWriter) => {
-    const commit = await storage.readObjAndBytes(commitCid, def.commit)
-    await car.put({ cid: commitCid, bytes: commit.bytes })
-    const mst = MST.load(storage, commit.obj.data)
-    await mst.writeToCarStream(car)
-  })
+  return writeCarStream(commitCid, iterateFullRepo(storage, commitCid))
+}
+
+async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
+  const commit = await storage.readObjAndBytes(commitCid, def.commit)
+  yield { cid: commitCid, bytes: commit.bytes }
+  const mst = MST.load(storage, commit.obj.data)
+  for await (const block of mst.carBlockStream()) {
+    yield block
+  }
 }
 
 // Narrow slices
@@ -30,24 +34,31 @@ export const getRecords = (
   commitCid: CID,
   paths: RecordPath[],
 ): AsyncIterable<Uint8Array> => {
-  return util.writeCar(commitCid, async (car: BlockWriter) => {
-    const commit = await storage.readObjAndBytes(commitCid, def.commit)
-    await car.put({ cid: commitCid, bytes: commit.bytes })
-    const mst = MST.load(storage, commit.obj.data)
-    const cidsForPaths = await Promise.all(
-      paths.map((p) =>
-        mst.cidsForPath(util.formatDataKey(p.collection, p.rkey)),
-      ),
-    )
-    const allCids = cidsForPaths.reduce((acc, cur) => {
-      return acc.addSet(new CidSet(cur))
-    }, new CidSet())
-    const found = await storage.getBlocks(allCids.toList())
-    if (found.missing.length > 0) {
-      throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
-    }
-    for (const block of found.blocks.entries()) {
-      await car.put(block)
-    }
-  })
+  return writeCarStream(
+    commitCid,
+    iterateRecordBlocks(storage, commitCid, paths),
+  )
+}
+
+async function* iterateRecordBlocks(
+  storage: ReadableBlockstore,
+  commitCid: CID,
+  paths: RecordPath[],
+) {
+  const commit = await storage.readObjAndBytes(commitCid, def.commit)
+  yield { cid: commitCid, bytes: commit.bytes }
+  const mst = MST.load(storage, commit.obj.data)
+  const cidsForPaths = await Promise.all(
+    paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))),
+  )
+  const allCids = cidsForPaths.reduce((acc, cur) => {
+    return acc.addSet(new CidSet(cur))
+  }, new CidSet())
+  const found = await storage.getBlocks(allCids.toList())
+  if (found.missing.length > 0) {
+    throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
+  }
+  for (const block of found.blocks.entries()) {
+    yield block
+  }
 }

package/src/util.ts

@@ -1,26 +1,10 @@
-import { Readable } from 'node:stream'
-import { setImmediate } from 'node:timers/promises'
-import { CarBlockIterator } from '@ipld/car/iterator'
-import { BlockWriter, CarWriter } from '@ipld/car/writer'
 import * as cbor from '@ipld/dag-cbor'
-import { CID } from 'multiformats/cid'
-import {
-  TID,
-  byteIterableToStream,
-  cborDecode,
-  check,
-  cidForCbor,
-  schema,
-  streamToBuffer,
-  verifyCidForBytes,
-} from '@atproto/common'
+import { TID, cborDecode, check, cidForCbor, schema } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import { Keypair } from '@atproto/crypto'
 import { LexValue, RepoRecord, ipldToLex, lexToIpld } from '@atproto/lexicon'
-import { BlockMap } from './block-map'
 import { DataDiff } from './data-diff'
 import {
-  CarBlock,
   Commit,
   LegacyV2Commit,
   RecordCreateDescript,
@@ -32,103 +16,6 @@ import {
   WriteOpAction,
 } from './types'
 
-export async function* verifyIncomingCarBlocks(
-  car: AsyncIterable<CarBlock>,
-): AsyncIterable<CarBlock> {
-  for await (const block of car) {
-    await verifyCidForBytes(block.cid, block.bytes)
-    yield block
-  }
-}
-
-// we have to turn the car writer output into a stream in order to properly handle errors
-export function writeCarStream(
-  root: CID | null,
-  fn: (car: BlockWriter) => Promise<void>,
-): Readable {
-  const { writer, out } =
-    root !== null ? CarWriter.create(root) : CarWriter.create()
-
-  const stream = byteIterableToStream(out)
-  fn(writer)
-    .catch((err) => {
-      stream.destroy(err)
-    })
-    .finally(() => writer.close())
-  return stream
-}
-
-export async function* writeCar(
-  root: CID | null,
-  fn: (car: BlockWriter) => Promise<void>,
-): AsyncIterable<Uint8Array> {
-  const stream = writeCarStream(root, fn)
-  for await (const chunk of stream) {
-    yield chunk
-  }
-}
-
-export const blocksToCarStream = (
-  root: CID | null,
-  blocks: BlockMap,
-): AsyncIterable<Uint8Array> => {
-  return writeCar(root, async (writer) => {
-    for (const entry of blocks.entries()) {
-      await writer.put(entry)
-    }
-  })
-}
-
-export const blocksToCarFile = (
-  root: CID | null,
-  blocks: BlockMap,
-): Promise<Uint8Array> => {
-  const carStream = blocksToCarStream(root, blocks)
-  return streamToBuffer(carStream)
-}
-
-export const carToBlocks = async (
-  car: CarBlockIterator,
-): Promise<{ roots: CID[]; blocks: BlockMap }> => {
-  const roots = await car.getRoots()
-  const blocks = new BlockMap()
-  for await (const block of verifyIncomingCarBlocks(car)) {
-    blocks.set(block.cid, block.bytes)
-    // break up otherwise "synchronous" work in car parsing
-    await setImmediate()
-  }
-  return {
-    roots,
-    blocks,
-  }
-}
-
-export const readCar = async (
-  bytes: Uint8Array,
-): Promise<{ roots: CID[]; blocks: BlockMap }> => {
-  const car = await CarBlockIterator.fromBytes(bytes)
-  return carToBlocks(car)
-}
-
-export const readCarStream = async (stream: AsyncIterable<Uint8Array>) => {
-  const car = await CarBlockIterator.fromIterable(stream)
-  return carToBlocks(car)
-}
-
-export const readCarWithRoot = async (
-  bytes: Uint8Array,
-): Promise<{ root: CID; blocks: BlockMap }> => {
-  const { roots, blocks } = await readCar(bytes)
-  if (roots.length !== 1) {
-    throw new Error(`Expected one root, got ${roots.length}`)
-  }
-  const root = roots[0]
-  return {
-    root,
-    blocks,
-  }
-}
-
 export const diffToWriteDescripts = (
   diff: DataDiff,
 ): Promise<RecordWriteDescript[]> => {

package/tests/car-file-fixtures.json

@@ -0,0 +1,28 @@
+[
+  {
+    "root": "bafyreiapldaco7m23c7qzc4w42r7kxmcswm64nkindtuh4vwztrpoe7m5m",
+    "blocks": [
+      {
+        "cid": "bafyreiapldaco7m23c7qzc4w42r7kxmcswm64nkindtuh4vwztrpoe7m5m",
+        "bytes": "oWR0ZXN0ZHJvb3Q"
+      },
+      {
+        "cid": "bafyreieteuyxvbvjbvuhsuo54qx6r3tnjxtp3ub6kb66rdjml3murxjcsy",
+        "bytes": "oWR0ZXN0WQEAM32TVA//xgHS9Gtukp0vw6whQ+TnlwF9czt5A7Dxz/URSbryc9Pdw7HESX+jC2oPI/6rwKbhSml2kxJo4MaUeIg/HWI9ixxALw5gIF/I0JC3ejXVAu1Pw6bI9RWa5TgIvAnSow0pJ6jbaaWHlxCpqqHCNHUYbIC14D9k+RK3yS0h2g+O+gRUETQt8t4jOKxEhD037cYEuJCD+fWzFoLkEkrPdUWeqlFQxGt6bflCYjZFgiZKFUo72afR3XM16+jOlhOl+EtuqFcijYJ6MIB53qI8P8HMC2RVH0Mv8UYWLcWatl+CNLykEjesnMar5CvZT8j4w5EyEiS09iD4r6bljg"
+      },
+      {
+        "cid": "bafyreiahvdwcywzm35id7hajal5l73p4bnoyaowgckeatd6sbxsacoo5jq",
+        "bytes": "oWR0ZXN0WID81CSnkdKi0OBexCStL5gex6Fkehtg7R9Jaww7LFuJC/6Or9Cdb58I+6T9Kjqhf1bf5t5WLbTGt8HOlf1Ysl3wqfdxdxnRZYjyng8YjEmH2Twkc//moluskzywxfOwhRXsXI7SYt36OUkS8AKDzmhTijOG2XWSa2QvU3iSSjP8zw"
+      },
+      {
+        "cid": "bafyreictawwxwjuto6csptbtktbbgwkrryqoakgyb3qovakgiqjzmvzi6a",
+        "bytes": "oWR0ZXN0RBnEJSE"
+      },
+      {
+        "cid": "bafyreierfkr6cwv2ux5hk6hp5qh5fhgzyr4yicad5m3pyu3ndaatyqucxu",
+        "bytes": "oWR0ZXN0WQH09xLXHen1UTIPEap18egWK6OiRVQ4oLBqjfBVQGdiSsmsIn8uXDM6wp63kCrYWLvXoI0dREVW6+RUjoQIbmDhiKbEUTuLbcFiJnwD5gSlkXdUEwO4UeLfvcAsDJXel5lLaOlOeRTfA3Wf+rEEpr/ccCuwOATBPZBjPrneFVQu8UdsvTu5W144tGxp/ptBsMBqM3yLYkYYHNRrjcAI7iFiszsfKzo28GyM199Jko11mcVhNQ8KHZS72jbsWW/HyfJsL1M/dn6sDCfIaro66mTLRddSQaheQL4NBW5FEgLUBO2VDuBT9fVIl2IwQcijZAOakjLkS1sY2SyUmdsicqGRalnOrVlC5iWQcwHDXLzm9GWz/vfuoF+jzWsdpo6cjT5MIN8uXpzoZRSjH1+UXFxCzFhXkDwL/xJUq8u/0OFGp0mzDc7RLO3gC7X/ENaVPtz/wQaZ3q00EeHvDuiaxHdKIesf/+IkbqS1XjKtaHemB511MFiVf7l2OcqsUU12A5VMreqWPwbAHLgFRDPWiLS0D7yEF3KJX1IupxBmidQMT+SlmCp7FZMdJeHJ3pzpbQv+EoKSXja7it2D2uYsMcTn2DGhlVYsCcQd8PVNKZmPXuC7D82N4sh8p2XVW4LbsDfNTOrgHLX7zRX31VNa47w5Uc03Wuk"
+      }
+    ],
+    "car": "OqJlcm9vdHOB2CpYJQABcRIgD1jAJ32a2L8Mi5bmo/VdgpWZ7jVIaOdD8rbM4vcT7OtndmVyc2lvbgEvAXESIA9YwCd9mti/DIuW5qP1XYKVme41SGjnQ/K2zOL3E+zroWR0ZXN0ZHJvb3StAgFxEiCTJTF6hqkNaHlR3eQv6O5tTeb90D5QfeiNLF7ZSN0ilqFkdGVzdFkBADN9k1QP/8YB0vRrbpKdL8OsIUPk55cBfXM7eQOw8c/1EUm68nPT3cOxxEl/owtqDyP+q8Cm4UppdpMSaODGlHiIPx1iPYscQC8OYCBfyNCQt3o11QLtT8OmyPUVmuU4CLwJ0qMNKSeo22mlh5cQqaqhwjR1GGyAteA/ZPkSt8ktIdoPjvoEVBE0LfLeIzisRIQ9N+3GBLiQg/n1sxaC5BJKz3VFnqpRUMRrem35QmI2RYImShVKO9mn0d1zNevozpYTpfhLbqhXIo2CejCAed6iPD/BzAtkVR9DL/FGFi3FmrZfgjS8pBI3rJzGq+Qr2U/I+MORMhIktPYg+K+m5Y6sAQFxEiAHqOwsWyzfUD+cCQL6v+38C12AOsYSiAmP0g3kATndTKFkdGVzdFiA/NQkp5HSotDgXsQkrS+YHsehZHobYO0fSWsMOyxbiQv+jq/QnW+fCPuk/So6oX9W3+beVi20xrfBzpX9WLJd8Kn3cXcZ0WWI8p4PGIxJh9k8JHP/5qJbrJM8sMXzsIUV7FyO0mLd+jlJEvACg85oU4ozhtl1kmtkL1N4kkoz/M8vAXESIFMFrXsmk3eFJ8wzVMITWVGOIOAo2A7g6oFGRBOWVyjwoWR0ZXN0RBnEJSGhBAFxEiCRKqPhWrql+nV47+wP0pzZxHmECAPrNvxTbRgBPEKCvaFkdGVzdFkB9PcS1x3p9VEyDxGqdfHoFiujokVUOKCwao3wVUBnYkrJrCJ/LlwzOsKet5Aq2Fi716CNHURFVuvkVI6ECG5g4YimxFE7i23BYiZ8A+YEpZF3VBMDuFHi373ALAyV3peZS2jpTnkU3wN1n/qxBKa/3HArsDgEwT2QYz653hVULvFHbL07uVteOLRsaf6bQbDAajN8i2JGGBzUa43ACO4hYrM7Hys6NvBsjNffSZKNdZnFYTUPCh2Uu9o27Flvx8nybC9TP3Z+rAwnyGq6Oupky0XXUkGoXkC+DQVuRRIC1ATtlQ7gU/X1SJdiMEHIo2QDmpIy5EtbGNkslJnbInKhkWpZzq1ZQuYlkHMBw1y85vRls/737qBfo81rHaaOnI0+TCDfLl6c6GUUox9flFxcQsxYV5A8C/8SVKvLv9DhRqdJsw3O0Szt4Au1/xDWlT7c/8EGmd6tNBHh7w7omsR3SiHrH//iJG6ktV4yrWh3pgeddTBYlX+5djnKrFFNdgOVTK3qlj8GwBy4BUQz1oi0tA+8hBdyiV9SLqcQZonUDE/kpZgqexWTHSXhyd6c6W0L/hKCkl42u4rdg9rmLDHE59gxoZVWLAnEHfD1TSmZj17guw/NjeLIfKdl1VuC27A3zUzq4By1+80V99VTWuO8OVHNN1rp"
+  }
+]

package/tests/car.test.ts

@@ -0,0 +1,58 @@
+import { CID } from 'multiformats/cid'
+import * as ui8 from 'uint8arrays'
+import { dataToCborBlock, streamToBytes, wait } from '@atproto/common'
+import { CarBlock, readCarStream, writeCarStream } from '../src'
+import fixtures from './car-file-fixtures.json'
+
+describe('car', () => {
+  for (const fixture of fixtures) {
+    it('correctly writes car files', async () => {
+      const root = CID.parse(fixture.root)
+      async function* blockIter() {
+        for (const block of fixture.blocks) {
+          const cid = CID.parse(block.cid)
+          const bytes = ui8.fromString(block.bytes, 'base64')
+          yield { cid, bytes }
+        }
+      }
+      const carStream = writeCarStream(root, blockIter())
+      const car = await streamToBytes(carStream)
+      const carB64 = ui8.toString(car, 'base64')
+      expect(carB64).toEqual(fixture.car)
+    })
+
+    it('correctly reads carfiles', async () => {
+      const carStream = [ui8.fromString(fixture.car, 'base64')]
+      const { roots, blocks } = await readCarStream(carStream)
+      expect(roots.length).toBe(1)
+      expect(roots[0].toString()).toEqual(fixture.root)
+      const carBlocks: CarBlock[] = []
+      for await (const block of blocks) {
+        carBlocks.push(block)
+      }
+      expect(carBlocks.length).toEqual(fixture.blocks.length)
+      for (let i = 0; i < carBlocks.length; i++) {
+        expect(carBlocks[i].cid.toString()).toEqual(fixture.blocks[i].cid)
+        expect(ui8.toString(carBlocks[i].bytes, 'base64')).toEqual(
+          fixture.blocks[i].bytes,
+        )
+      }
+    })
+  }
+
+  it('writeCar propagates errors', async () => {
+    const iterate = async () => {
+      async function* blockIterator() {
+        await wait(1)
+        const block = await dataToCborBlock({ test: 1 })
+        yield block
+        throw new Error('Oops!')
+      }
+      const iter = writeCarStream(null, blockIterator())
+      for await (const _bytes of iter) {
+        // no-op
+      }
+    }
+    await expect(iterate).rejects.toThrow('Oops!')
+  })
+})

package/tests/sync.test.ts

@@ -1,4 +1,3 @@
-import { CarReader } from '@ipld/car/reader'
 import { streamToBuffer } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import {
@@ -7,6 +6,7 @@ import {
   RepoContents,
   RepoVerificationError,
   getAndParseRecord,
+  readCar,
   readCarWithRoot,
 } from '../src'
 import { MemoryBlockstore } from '../src/storage'
@@ -55,14 +55,14 @@ describe('Repo Sync', () => {
 
   it('does not sync duplicate blocks', async () => {
     const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
-    const car = await CarReader.fromBytes(carBytes)
+    const car = await readCar(carBytes)
     const cids = new CidSet()
-    for await (const block of car.blocks()) {
-      if (cids.has(block.cid)) {
-        throw new Error(`duplicate block: :${block.cid.toString()}`)
+    car.blocks.forEach((_, cid) => {
+      if (cids.has(cid)) {
+        throw new Error(`duplicate block: :${cid.toString()}`)
       }
-      cids.add(block.cid)
-    }
+      cids.add(cid)
+    })
   })
 
   it('syncs a repo that is behind', async () => {

package/tsconfig.build.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/block-map.ts","./src/cid-set.ts","./src/data-diff.ts","./src/error.ts","./src/index.ts","./src/logger.ts","./src/parse.ts","./src/readable-repo.ts","./src/repo.ts","./src/types.ts","./src/util.ts","./src/mst/diff.ts","./src/mst/index.ts","./src/mst/mst.ts","./src/mst/util.ts","./src/mst/walker.ts","./src/storage/index.ts","./src/storage/memory-blockstore.ts","./src/storage/readable-blockstore.ts","./src/storage/sync-storage.ts","./src/storage/types.ts","./src/sync/consumer.ts","./src/sync/index.ts","./src/sync/provider.ts"],"version":"5.6.3"}
+{"root":["./src/block-map.ts","./src/car.ts","./src/cid-set.ts","./src/data-diff.ts","./src/error.ts","./src/index.ts","./src/logger.ts","./src/parse.ts","./src/readable-repo.ts","./src/repo.ts","./src/types.ts","./src/util.ts","./src/mst/diff.ts","./src/mst/index.ts","./src/mst/mst.ts","./src/mst/util.ts","./src/mst/walker.ts","./src/storage/index.ts","./src/storage/memory-blockstore.ts","./src/storage/readable-blockstore.ts","./src/storage/sync-storage.ts","./src/storage/types.ts","./src/sync/consumer.ts","./src/sync/index.ts","./src/sync/provider.ts"],"version":"5.6.3"}

package/tsconfig.tests.tsbuildinfo

@@ -1 +1 @@
-{"root":["./tests/_keys.ts","./tests/_util.ts","./tests/commit-data.test.ts","./tests/commit-proofs.test.ts","./tests/covering-proofs.test.ts","./tests/mst.test.ts","./tests/proofs.test.ts","./tests/repo.test.ts","./tests/sync.test.ts","./tests/util.test.ts"],"version":"5.6.3"}
+{"root":["./tests/_keys.ts","./tests/_util.ts","./tests/car.test.ts","./tests/commit-data.test.ts","./tests/commit-proofs.test.ts","./tests/covering-proofs.test.ts","./tests/mst.test.ts","./tests/proofs.test.ts","./tests/repo.test.ts","./tests/sync.test.ts"],"version":"5.6.3"}

package/tests/util.test.ts

@@ -1,21 +0,0 @@
-import { dataToCborBlock, wait } from '@atproto/common'
-import { writeCar } from '../src'
-
-describe('Utils', () => {
-  describe('writeCar()', () => {
-    it('propagates errors', async () => {
-      const iterate = async () => {
-        const iter = writeCar(null, async (car) => {
-          await wait(1)
-          const block = await dataToCborBlock({ test: 1 })
-          await car.put(block)
-          throw new Error('Oops!')
-        })
-        for await (const _bytes of iter) {
-          // no-op
-        }
-      }
-      await expect(iterate).rejects.toThrow('Oops!')
-    })
-  })
-})