@atproto/repo 0.7.3 → 0.8.1

package/src/car.ts

@@ -0,0 +1,219 @@
+import * as cbor from '@ipld/dag-cbor'
+import { CID } from 'multiformats/cid'
+import * as ui8 from 'uint8arrays'
+import * as varint from 'varint'
+import {
+  check,
+  parseCidFromBytes,
+  schema,
+  streamToBuffer,
+  verifyCidForBytes,
+} from '@atproto/common'
+import { BlockMap } from './block-map'
+import { CarBlock } from './types'
+
+export async function* writeCarStream(
+  root: CID | null,
+  blocks: AsyncIterable<CarBlock>,
+): AsyncIterable<Uint8Array> {
+  const header = new Uint8Array(
+    cbor.encode({
+      version: 1,
+      roots: root ? [root] : [],
+    }),
+  )
+  yield new Uint8Array(varint.encode(header.byteLength))
+  yield header
+  for await (const block of blocks) {
+    yield new Uint8Array(
+      varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength),
+    )
+    yield block.cid.bytes
+    yield block.bytes
+  }
+}
+
+export const blocksToCarFile = (
+  root: CID | null,
+  blocks: BlockMap,
+): Promise<Uint8Array> => {
+  const carStream = blocksToCarStream(root, blocks)
+  return streamToBuffer(carStream)
+}
+
+export const blocksToCarStream = (
+  root: CID | null,
+  blocks: BlockMap,
+): AsyncIterable<Uint8Array> => {
+  return writeCarStream(root, iterateBlocks(blocks))
+}
+
+async function* iterateBlocks(blocks: BlockMap) {
+  for (const entry of blocks.entries()) {
+    yield { cid: entry.cid, bytes: entry.bytes }
+  }
+}
+
+export const readCar = async (
+  bytes: Uint8Array,
+): Promise<{ roots: CID[]; blocks: BlockMap }> => {
+  const { roots, blocks } = await readCarStream([bytes])
+  const blockMap = new BlockMap()
+  for await (const block of blocks) {
+    blockMap.set(block.cid, block.bytes)
+  }
+  return { roots, blocks: blockMap }
+}
+
+export const readCarWithRoot = async (
+  bytes: Uint8Array,
+): Promise<{ root: CID; blocks: BlockMap }> => {
+  const { roots, blocks } = await readCar(bytes)
+  if (roots.length !== 1) {
+    throw new Error(`Expected one root, got ${roots.length}`)
+  }
+  const root = roots[0]
+  return {
+    root,
+    blocks,
+  }
+}
+export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
+  dump: () => Promise<void>
+}
+
+export const readCarStream = async (
+  car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+): Promise<{
+  roots: CID[]
+  blocks: CarBlockIterable
+}> => {
+  const reader = new BufferedReader(car)
+  try {
+    const headerSize = await reader.readVarint()
+    if (headerSize === null) {
+      throw new Error('Could not parse CAR header')
+    }
+    const headerBytes = await reader.read(headerSize)
+    const header = cbor.decode(headerBytes)
+    if (!check.is(header, schema.carHeader)) {
+      throw new Error('Could not parse CAR header')
+    }
+    return {
+      roots: header.roots,
+      blocks: readCarBlocksIter(reader),
+    }
+  } catch (err) {
+    await reader.close()
+    throw err
+  }
+}
+
+const readCarBlocksIter = (reader: BufferedReader) => {
+  const iter = readCarBlocksIterGenerator(reader) as CarBlockIterable
+
+  iter.dump = async () => {
+    // try/finally to ensure that reader.close is called even if blocks.return throws.
+    try {
+      // Prevent the iterator from being started after this method is called.
+      await iter.return()
+    } finally {
+      // @NOTE the "finally" block of the async generator won't be called
+      // if the iteration was never started so we need to manually close here.
+      await reader.close()
+    }
+  }
+
+  return iter
+}
+
+async function* readCarBlocksIterGenerator(
+  reader: BufferedReader,
+): AsyncIterable<CarBlock> {
+  try {
+    while (!reader.isDone) {
+      const blockSize = await reader.readVarint()
+      if (blockSize === null) {
+        break
+      }
+      const blockBytes = await reader.read(blockSize)
+      const cid = parseCidFromBytes(blockBytes.slice(0, 36))
+      const bytes = blockBytes.slice(36)
+      yield { cid, bytes }
+    }
+  } finally {
+    await reader.close()
+  }
+}
+
+export async function* verifyIncomingCarBlocks(
+  car: AsyncIterable<CarBlock>,
+): AsyncIterable<CarBlock> {
+  for await (const block of car) {
+    await verifyCidForBytes(block.cid, block.bytes)
+    yield block
+  }
+}
+
+class BufferedReader {
+  buffer: Uint8Array = new Uint8Array()
+  iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>
+  isDone = false
+
+  constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {
+    this.iterator =
+      Symbol.asyncIterator in stream
+        ? stream[Symbol.asyncIterator]()
+        : stream[Symbol.iterator]()
+  }
+
+  async read(bytesToRead: number): Promise<Uint8Array> {
+    await this.readUntilBuffered(bytesToRead)
+    const value = this.buffer.slice(0, bytesToRead)
+    this.buffer = this.buffer.slice(bytesToRead)
+    return value
+  }
+
+  async readVarint(): Promise<number | null> {
+    let done = false
+    const bytes: Uint8Array[] = []
+    while (!done) {
+      const byte = await this.read(1)
+      if (byte.byteLength === 0) {
+        if (bytes.length > 0) {
+          throw new Error('could not parse varint')
+        } else {
+          return null
+        }
+      }
+      bytes.push(byte)
+      if (byte[0] < 128) {
+        done = true
+      }
+    }
+    const concatted = ui8.concat(bytes)
+    return varint.decode(concatted)
+  }
+
+  private async readUntilBuffered(bytesToRead: number) {
+    if (this.isDone) {
+      return
+    }
+    while (this.buffer.length < bytesToRead) {
+      const next = await this.iterator.next()
+      if (next.done) {
+        this.isDone = true
+        return
+      }
+      this.buffer = ui8.concat([this.buffer, next.value])
+    }
+  }
+
+  async close(): Promise<void> {
+    if (!this.isDone && this.iterator.return) {
+      await this.iterator.return()
+    }
+    this.isDone = true
+    this.buffer = new Uint8Array()
+  }
+}

package/src/index.ts

@@ -7,4 +7,5 @@ export * from './storage'
 export * from './sync'
 export * from './types'
 export * from './data-diff'
+export * from './car'
 export * from './util'

package/src/mst/mst.ts

@@ -1,4 +1,3 @@
-import { BlockWriter } from '@ipld/car/writer'
 import { CID } from 'multiformats'
 import { z } from 'zod'
 import { cidForCbor, dataToCborBlock, schema as common } from '@atproto/common'
@@ -7,6 +6,7 @@ import { CidSet } from '../cid-set'
 import { MissingBlockError, MissingBlocksError } from '../error'
 import * as parse from '../parse'
 import { ReadableBlockstore } from '../storage'
+import { CarBlock } from '../types'
 import * as util from './util'
 
 /**
@@ -726,7 +726,7 @@ export class MST {
 
   // Sync Protocol
 
-  async writeToCarStream(car: BlockWriter): Promise<void> {
+  async *carBlockStream(): AsyncIterable<CarBlock> {
     const leaves = new CidSet()
     let toFetch = new CidSet()
     toFetch.add(await this.getPointer())
@@ -742,7 +742,7 @@ export class MST {
           cid,
           nodeDataDef,
         )
-        await car.put({ cid, bytes: found.bytes })
+        yield { cid, bytes: found.bytes }
         const entries = await util.deserializeNodeData(this.storage, found.obj)
 
         for (const entry of entries) {
@@ -761,7 +761,7 @@ export class MST {
     }
 
     for (const leaf of leafData.blocks.entries()) {
-      await car.put(leaf)
+      yield leaf
     }
   }
 

package/src/sync/consumer.ts

@@ -1,5 +1,6 @@
 import { CID } from 'multiformats/cid'
 import { BlockMap } from '../block-map'
+import { readCarWithRoot } from '../car'
 import { DataDiff } from '../data-diff'
 import { MST } from '../mst'
 import { ReadableRepo } from '../readable-repo'
@@ -18,7 +19,7 @@ export const verifyRepoCar = async (
   did?: string,
   signingKey?: string,
 ): Promise<VerifiedRepo> => {
-  const car = await util.readCarWithRoot(carBytes)
+  const car = await readCarWithRoot(carBytes)
   return verifyRepo(car.blocks, car.root, did, signingKey)
 }
 
@@ -44,7 +45,7 @@ export const verifyDiffCar = async (
   signingKey?: string,
   opts?: { ensureLeaves?: boolean },
 ): Promise<VerifiedDiff> => {
-  const car = await util.readCarWithRoot(carBytes)
+  const car = await readCarWithRoot(carBytes)
   return verifyDiff(repo, car.blocks, car.root, did, signingKey, opts)
 }
 
@@ -127,7 +128,7 @@ export const verifyProofs = async (
   did: string,
   didKey: string,
 ): Promise<{ verified: RecordCidClaim[]; unverified: RecordCidClaim[] }> => {
-  const car = await util.readCarWithRoot(proofs)
+  const car = await readCarWithRoot(proofs)
   const blockstore = new MemoryBlockstore(car.blocks)
   const commit = await blockstore.readObj(car.root, def.commit)
   if (commit.did !== did) {
@@ -169,7 +170,7 @@ export const verifyRecords = async (
   did: string,
   signingKey: string,
 ): Promise<RecordClaim[]> => {
-  const car = await util.readCarWithRoot(proofs)
+  const car = await readCarWithRoot(proofs)
   const blockstore = new MemoryBlockstore(car.blocks)
   const commit = await blockstore.readObj(car.root, def.commit)
   if (commit.did !== did) {

package/src/sync/provider.ts

@@ -1,5 +1,5 @@
-import { BlockWriter } from '@ipld/car/writer'
 import { CID } from 'multiformats/cid'
+import { writeCarStream } from '../car'
 import { CidSet } from '../cid-set'
 import { MissingBlocksError } from '../error'
 import { MST } from '../mst'
@@ -14,12 +14,16 @@ export const getFullRepo = (
   storage: RepoStorage,
   commitCid: CID,
 ): AsyncIterable<Uint8Array> => {
-  return util.writeCar(commitCid, async (car: BlockWriter) => {
-    const commit = await storage.readObjAndBytes(commitCid, def.commit)
-    await car.put({ cid: commitCid, bytes: commit.bytes })
-    const mst = MST.load(storage, commit.obj.data)
-    await mst.writeToCarStream(car)
-  })
+  return writeCarStream(commitCid, iterateFullRepo(storage, commitCid))
+}
+
+async function* iterateFullRepo(storage: RepoStorage, commitCid: CID) {
+  const commit = await storage.readObjAndBytes(commitCid, def.commit)
+  yield { cid: commitCid, bytes: commit.bytes }
+  const mst = MST.load(storage, commit.obj.data)
+  for await (const block of mst.carBlockStream()) {
+    yield block
+  }
 }
 
 // Narrow slices
@@ -30,24 +34,31 @@ export const getRecords = (
   commitCid: CID,
   paths: RecordPath[],
 ): AsyncIterable<Uint8Array> => {
-  return util.writeCar(commitCid, async (car: BlockWriter) => {
-    const commit = await storage.readObjAndBytes(commitCid, def.commit)
-    await car.put({ cid: commitCid, bytes: commit.bytes })
-    const mst = MST.load(storage, commit.obj.data)
-    const cidsForPaths = await Promise.all(
-      paths.map((p) =>
-        mst.cidsForPath(util.formatDataKey(p.collection, p.rkey)),
-      ),
-    )
-    const allCids = cidsForPaths.reduce((acc, cur) => {
-      return acc.addSet(new CidSet(cur))
-    }, new CidSet())
-    const found = await storage.getBlocks(allCids.toList())
-    if (found.missing.length > 0) {
-      throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
-    }
-    for (const block of found.blocks.entries()) {
-      await car.put(block)
-    }
-  })
+  return writeCarStream(
+    commitCid,
+    iterateRecordBlocks(storage, commitCid, paths),
+  )
+}
+
+async function* iterateRecordBlocks(
+  storage: ReadableBlockstore,
+  commitCid: CID,
+  paths: RecordPath[],
+) {
+  const commit = await storage.readObjAndBytes(commitCid, def.commit)
+  yield { cid: commitCid, bytes: commit.bytes }
+  const mst = MST.load(storage, commit.obj.data)
+  const cidsForPaths = await Promise.all(
+    paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))),
+  )
+  const allCids = cidsForPaths.reduce((acc, cur) => {
+    return acc.addSet(new CidSet(cur))
+  }, new CidSet())
+  const found = await storage.getBlocks(allCids.toList())
+  if (found.missing.length > 0) {
+    throw new MissingBlocksError('writeRecordsToCarStream', found.missing)
+  }
+  for (const block of found.blocks.entries()) {
+    yield block
+  }
 }

package/src/util.ts

@@ -1,26 +1,10 @@
-import { Readable } from 'node:stream'
-import { setImmediate } from 'node:timers/promises'
-import { CarBlockIterator } from '@ipld/car/iterator'
-import { BlockWriter, CarWriter } from '@ipld/car/writer'
 import * as cbor from '@ipld/dag-cbor'
-import { CID } from 'multiformats/cid'
-import {
-  TID,
-  byteIterableToStream,
-  cborDecode,
-  check,
-  cidForCbor,
-  schema,
-  streamToBuffer,
-  verifyCidForBytes,
-} from '@atproto/common'
+import { TID, cborDecode, check, cidForCbor, schema } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import { Keypair } from '@atproto/crypto'
 import { LexValue, RepoRecord, ipldToLex, lexToIpld } from '@atproto/lexicon'
-import { BlockMap } from './block-map'
 import { DataDiff } from './data-diff'
 import {
-  CarBlock,
   Commit,
   LegacyV2Commit,
   RecordCreateDescript,
@@ -32,103 +16,6 @@ import {
   WriteOpAction,
 } from './types'
 
-export async function* verifyIncomingCarBlocks(
-  car: AsyncIterable<CarBlock>,
-): AsyncIterable<CarBlock> {
-  for await (const block of car) {
-    await verifyCidForBytes(block.cid, block.bytes)
-    yield block
-  }
-}
-
-// we have to turn the car writer output into a stream in order to properly handle errors
-export function writeCarStream(
-  root: CID | null,
-  fn: (car: BlockWriter) => Promise<void>,
-): Readable {
-  const { writer, out } =
-    root !== null ? CarWriter.create(root) : CarWriter.create()
-
-  const stream = byteIterableToStream(out)
-  fn(writer)
-    .catch((err) => {
-      stream.destroy(err)
-    })
-    .finally(() => writer.close())
-  return stream
-}
-
-export async function* writeCar(
-  root: CID | null,
-  fn: (car: BlockWriter) => Promise<void>,
-): AsyncIterable<Uint8Array> {
-  const stream = writeCarStream(root, fn)
-  for await (const chunk of stream) {
-    yield chunk
-  }
-}
-
-export const blocksToCarStream = (
-  root: CID | null,
-  blocks: BlockMap,
-): AsyncIterable<Uint8Array> => {
-  return writeCar(root, async (writer) => {
-    for (const entry of blocks.entries()) {
-      await writer.put(entry)
-    }
-  })
-}
-
-export const blocksToCarFile = (
-  root: CID | null,
-  blocks: BlockMap,
-): Promise<Uint8Array> => {
-  const carStream = blocksToCarStream(root, blocks)
-  return streamToBuffer(carStream)
-}
-
-export const carToBlocks = async (
-  car: CarBlockIterator,
-): Promise<{ roots: CID[]; blocks: BlockMap }> => {
-  const roots = await car.getRoots()
-  const blocks = new BlockMap()
-  for await (const block of verifyIncomingCarBlocks(car)) {
-    blocks.set(block.cid, block.bytes)
-    // break up otherwise "synchronous" work in car parsing
-    await setImmediate()
-  }
-  return {
-    roots,
-    blocks,
-  }
-}
-
-export const readCar = async (
-  bytes: Uint8Array,
-): Promise<{ roots: CID[]; blocks: BlockMap }> => {
-  const car = await CarBlockIterator.fromBytes(bytes)
-  return carToBlocks(car)
-}
-
-export const readCarStream = async (stream: AsyncIterable<Uint8Array>) => {
-  const car = await CarBlockIterator.fromIterable(stream)
-  return carToBlocks(car)
-}
-
-export const readCarWithRoot = async (
-  bytes: Uint8Array,
-): Promise<{ root: CID; blocks: BlockMap }> => {
-  const { roots, blocks } = await readCar(bytes)
-  if (roots.length !== 1) {
-    throw new Error(`Expected one root, got ${roots.length}`)
-  }
-  const root = roots[0]
-  return {
-    root,
-    blocks,
-  }
-}
-
 export const diffToWriteDescripts = (
   diff: DataDiff,
 ): Promise<RecordWriteDescript[]> => {

package/tests/car-file-fixtures.json

@@ -0,0 +1,28 @@
+[
+  {
+    "root": "bafyreiapldaco7m23c7qzc4w42r7kxmcswm64nkindtuh4vwztrpoe7m5m",
+    "blocks": [
+      {
+        "cid": "bafyreiapldaco7m23c7qzc4w42r7kxmcswm64nkindtuh4vwztrpoe7m5m",
+        "bytes": "oWR0ZXN0ZHJvb3Q"
+      },
+      {
+        "cid": "bafyreieteuyxvbvjbvuhsuo54qx6r3tnjxtp3ub6kb66rdjml3murxjcsy",
+        "bytes": "oWR0ZXN0WQEAM32TVA//xgHS9Gtukp0vw6whQ+TnlwF9czt5A7Dxz/URSbryc9Pdw7HESX+jC2oPI/6rwKbhSml2kxJo4MaUeIg/HWI9ixxALw5gIF/I0JC3ejXVAu1Pw6bI9RWa5TgIvAnSow0pJ6jbaaWHlxCpqqHCNHUYbIC14D9k+RK3yS0h2g+O+gRUETQt8t4jOKxEhD037cYEuJCD+fWzFoLkEkrPdUWeqlFQxGt6bflCYjZFgiZKFUo72afR3XM16+jOlhOl+EtuqFcijYJ6MIB53qI8P8HMC2RVH0Mv8UYWLcWatl+CNLykEjesnMar5CvZT8j4w5EyEiS09iD4r6bljg"
+      },
+      {
+        "cid": "bafyreiahvdwcywzm35id7hajal5l73p4bnoyaowgckeatd6sbxsacoo5jq",
+        "bytes": "oWR0ZXN0WID81CSnkdKi0OBexCStL5gex6Fkehtg7R9Jaww7LFuJC/6Or9Cdb58I+6T9Kjqhf1bf5t5WLbTGt8HOlf1Ysl3wqfdxdxnRZYjyng8YjEmH2Twkc//moluskzywxfOwhRXsXI7SYt36OUkS8AKDzmhTijOG2XWSa2QvU3iSSjP8zw"
+      },
+      {
+        "cid": "bafyreictawwxwjuto6csptbtktbbgwkrryqoakgyb3qovakgiqjzmvzi6a",
+        "bytes": "oWR0ZXN0RBnEJSE"
+      },
+      {
+        "cid": "bafyreierfkr6cwv2ux5hk6hp5qh5fhgzyr4yicad5m3pyu3ndaatyqucxu",
+        "bytes": "oWR0ZXN0WQH09xLXHen1UTIPEap18egWK6OiRVQ4oLBqjfBVQGdiSsmsIn8uXDM6wp63kCrYWLvXoI0dREVW6+RUjoQIbmDhiKbEUTuLbcFiJnwD5gSlkXdUEwO4UeLfvcAsDJXel5lLaOlOeRTfA3Wf+rEEpr/ccCuwOATBPZBjPrneFVQu8UdsvTu5W144tGxp/ptBsMBqM3yLYkYYHNRrjcAI7iFiszsfKzo28GyM199Jko11mcVhNQ8KHZS72jbsWW/HyfJsL1M/dn6sDCfIaro66mTLRddSQaheQL4NBW5FEgLUBO2VDuBT9fVIl2IwQcijZAOakjLkS1sY2SyUmdsicqGRalnOrVlC5iWQcwHDXLzm9GWz/vfuoF+jzWsdpo6cjT5MIN8uXpzoZRSjH1+UXFxCzFhXkDwL/xJUq8u/0OFGp0mzDc7RLO3gC7X/ENaVPtz/wQaZ3q00EeHvDuiaxHdKIesf/+IkbqS1XjKtaHemB511MFiVf7l2OcqsUU12A5VMreqWPwbAHLgFRDPWiLS0D7yEF3KJX1IupxBmidQMT+SlmCp7FZMdJeHJ3pzpbQv+EoKSXja7it2D2uYsMcTn2DGhlVYsCcQd8PVNKZmPXuC7D82N4sh8p2XVW4LbsDfNTOrgHLX7zRX31VNa47w5Uc03Wuk"
+      }
+    ],
+    "car": "OqJlcm9vdHOB2CpYJQABcRIgD1jAJ32a2L8Mi5bmo/VdgpWZ7jVIaOdD8rbM4vcT7OtndmVyc2lvbgEvAXESIA9YwCd9mti/DIuW5qP1XYKVme41SGjnQ/K2zOL3E+zroWR0ZXN0ZHJvb3StAgFxEiCTJTF6hqkNaHlR3eQv6O5tTeb90D5QfeiNLF7ZSN0ilqFkdGVzdFkBADN9k1QP/8YB0vRrbpKdL8OsIUPk55cBfXM7eQOw8c/1EUm68nPT3cOxxEl/owtqDyP+q8Cm4UppdpMSaODGlHiIPx1iPYscQC8OYCBfyNCQt3o11QLtT8OmyPUVmuU4CLwJ0qMNKSeo22mlh5cQqaqhwjR1GGyAteA/ZPkSt8ktIdoPjvoEVBE0LfLeIzisRIQ9N+3GBLiQg/n1sxaC5BJKz3VFnqpRUMRrem35QmI2RYImShVKO9mn0d1zNevozpYTpfhLbqhXIo2CejCAed6iPD/BzAtkVR9DL/FGFi3FmrZfgjS8pBI3rJzGq+Qr2U/I+MORMhIktPYg+K+m5Y6sAQFxEiAHqOwsWyzfUD+cCQL6v+38C12AOsYSiAmP0g3kATndTKFkdGVzdFiA/NQkp5HSotDgXsQkrS+YHsehZHobYO0fSWsMOyxbiQv+jq/QnW+fCPuk/So6oX9W3+beVi20xrfBzpX9WLJd8Kn3cXcZ0WWI8p4PGIxJh9k8JHP/5qJbrJM8sMXzsIUV7FyO0mLd+jlJEvACg85oU4ozhtl1kmtkL1N4kkoz/M8vAXESIFMFrXsmk3eFJ8wzVMITWVGOIOAo2A7g6oFGRBOWVyjwoWR0ZXN0RBnEJSGhBAFxEiCRKqPhWrql+nV47+wP0pzZxHmECAPrNvxTbRgBPEKCvaFkdGVzdFkB9PcS1x3p9VEyDxGqdfHoFiujokVUOKCwao3wVUBnYkrJrCJ/LlwzOsKet5Aq2Fi716CNHURFVuvkVI6ECG5g4YimxFE7i23BYiZ8A+YEpZF3VBMDuFHi373ALAyV3peZS2jpTnkU3wN1n/qxBKa/3HArsDgEwT2QYz653hVULvFHbL07uVteOLRsaf6bQbDAajN8i2JGGBzUa43ACO4hYrM7Hys6NvBsjNffSZKNdZnFYTUPCh2Uu9o27Flvx8nybC9TP3Z+rAwnyGq6Oupky0XXUkGoXkC+DQVuRRIC1ATtlQ7gU/X1SJdiMEHIo2QDmpIy5EtbGNkslJnbInKhkWpZzq1ZQuYlkHMBw1y85vRls/737qBfo81rHaaOnI0+TCDfLl6c6GUUox9flFxcQsxYV5A8C/8SVKvLv9DhRqdJsw3O0Szt4Au1/xDWlT7c/8EGmd6tNBHh7w7omsR3SiHrH//iJG6ktV4yrWh3pgeddTBYlX+5djnKrFFNdgOVTK3qlj8GwBy4BUQz1oi0tA+8hBdyiV9SLqcQZonUDE/kpZgqexWTHSXhyd6c6W0L/hKCkl42u4rdg9rmLDHE59gxoZVWLAnEHfD1TSmZj17guw/NjeLIfKdl1VuC27A3zUzq4By1+80V99VTWuO8OVHNN1rp"
+  }
+]

package/tests/car.test.ts

@@ -0,0 +1,58 @@
+import { CID } from 'multiformats/cid'
+import * as ui8 from 'uint8arrays'
+import { dataToCborBlock, streamToBytes, wait } from '@atproto/common'
+import { CarBlock, readCarStream, writeCarStream } from '../src'
+import fixtures from './car-file-fixtures.json'
+
+describe('car', () => {
+  for (const fixture of fixtures) {
+    it('correctly writes car files', async () => {
+      const root = CID.parse(fixture.root)
+      async function* blockIter() {
+        for (const block of fixture.blocks) {
+          const cid = CID.parse(block.cid)
+          const bytes = ui8.fromString(block.bytes, 'base64')
+          yield { cid, bytes }
+        }
+      }
+      const carStream = writeCarStream(root, blockIter())
+      const car = await streamToBytes(carStream)
+      const carB64 = ui8.toString(car, 'base64')
+      expect(carB64).toEqual(fixture.car)
+    })
+
+    it('correctly reads carfiles', async () => {
+      const carStream = [ui8.fromString(fixture.car, 'base64')]
+      const { roots, blocks } = await readCarStream(carStream)
+      expect(roots.length).toBe(1)
+      expect(roots[0].toString()).toEqual(fixture.root)
+      const carBlocks: CarBlock[] = []
+      for await (const block of blocks) {
+        carBlocks.push(block)
+      }
+      expect(carBlocks.length).toEqual(fixture.blocks.length)
+      for (let i = 0; i < carBlocks.length; i++) {
+        expect(carBlocks[i].cid.toString()).toEqual(fixture.blocks[i].cid)
+        expect(ui8.toString(carBlocks[i].bytes, 'base64')).toEqual(
+          fixture.blocks[i].bytes,
+        )
+      }
+    })
+  }
+
+  it('writeCar propagates errors', async () => {
+    const iterate = async () => {
+      async function* blockIterator() {
+        await wait(1)
+        const block = await dataToCborBlock({ test: 1 })
+        yield block
+        throw new Error('Oops!')
+      }
+      const iter = writeCarStream(null, blockIterator())
+      for await (const _bytes of iter) {
+        // no-op
+      }
+    }
+    await expect(iterate).rejects.toThrow('Oops!')
+  })
+})

package/tests/sync.test.ts

@@ -1,4 +1,3 @@
-import { CarReader } from '@ipld/car/reader'
 import { streamToBuffer } from '@atproto/common'
 import * as crypto from '@atproto/crypto'
 import {
@@ -7,6 +6,7 @@ import {
   RepoContents,
   RepoVerificationError,
   getAndParseRecord,
+  readCar,
   readCarWithRoot,
 } from '../src'
 import { MemoryBlockstore } from '../src/storage'
@@ -55,14 +55,14 @@ describe('Repo Sync', () => {
 
   it('does not sync duplicate blocks', async () => {
     const carBytes = await streamToBuffer(sync.getFullRepo(storage, repo.cid))
-    const car = await CarReader.fromBytes(carBytes)
+    const car = await readCar(carBytes)
     const cids = new CidSet()
-    for await (const block of car.blocks()) {
-      if (cids.has(block.cid)) {
-        throw new Error(`duplicate block: :${block.cid.toString()}`)
+    car.blocks.forEach((_, cid) => {
+      if (cids.has(cid)) {
+        throw new Error(`duplicate block: :${cid.toString()}`)
       }
-      cids.add(block.cid)
-    }
+      cids.add(cid)
+    })
   })
 
   it('syncs a repo that is behind', async () => {

package/tsconfig.build.tsbuildinfo

@@ -1 +1 @@
-{"root":["./src/block-map.ts","./src/cid-set.ts","./src/data-diff.ts","./src/error.ts","./src/index.ts","./src/logger.ts","./src/parse.ts","./src/readable-repo.ts","./src/repo.ts","./src/types.ts","./src/util.ts","./src/mst/diff.ts","./src/mst/index.ts","./src/mst/mst.ts","./src/mst/util.ts","./src/mst/walker.ts","./src/storage/index.ts","./src/storage/memory-blockstore.ts","./src/storage/readable-blockstore.ts","./src/storage/sync-storage.ts","./src/storage/types.ts","./src/sync/consumer.ts","./src/sync/index.ts","./src/sync/provider.ts"],"version":"5.6.3"}
+{"root":["./src/block-map.ts","./src/car.ts","./src/cid-set.ts","./src/data-diff.ts","./src/error.ts","./src/index.ts","./src/logger.ts","./src/parse.ts","./src/readable-repo.ts","./src/repo.ts","./src/types.ts","./src/util.ts","./src/mst/diff.ts","./src/mst/index.ts","./src/mst/mst.ts","./src/mst/util.ts","./src/mst/walker.ts","./src/storage/index.ts","./src/storage/memory-blockstore.ts","./src/storage/readable-blockstore.ts","./src/storage/sync-storage.ts","./src/storage/types.ts","./src/sync/consumer.ts","./src/sync/index.ts","./src/sync/provider.ts"],"version":"5.8.2"}

package/tsconfig.tests.tsbuildinfo

@@ -1 +1 @@
-{"root":["./tests/_keys.ts","./tests/_util.ts","./tests/commit-data.test.ts","./tests/commit-proofs.test.ts","./tests/covering-proofs.test.ts","./tests/mst.test.ts","./tests/proofs.test.ts","./tests/repo.test.ts","./tests/sync.test.ts","./tests/util.test.ts"],"version":"5.6.3"}
+{"root":["./tests/_keys.ts","./tests/_util.ts","./tests/car.test.ts","./tests/commit-data.test.ts","./tests/commit-proofs.test.ts","./tests/covering-proofs.test.ts","./tests/mst.test.ts","./tests/proofs.test.ts","./tests/repo.test.ts","./tests/sync.test.ts"],"version":"5.8.2"}

package/tests/util.test.ts

@@ -1,21 +0,0 @@
-import { dataToCborBlock, wait } from '@atproto/common'
-import { writeCar } from '../src'
-
-describe('Utils', () => {
-  describe('writeCar()', () => {
-    it('propagates errors', async () => {
-      const iterate = async () => {
-        const iter = writeCar(null, async (car) => {
-          await wait(1)
-          const block = await dataToCborBlock({ test: 1 })
-          await car.put(block)
-          throw new Error('Oops!')
-        })
-        for await (const _bytes of iter) {
-          // no-op
-        }
-      }
-      await expect(iterate).rejects.toThrow('Oops!')
-    })
-  })
-})