@atproto/repo 0.3.5 → 0.3.7

package/src/types.ts

@@ -2,6 +2,7 @@ import { z } from 'zod'
 import { def as commonDef } from '@atproto/common-web'
 import { schema as common } from '@atproto/common'
 import { CID } from 'multiformats'
+import * as car from '@ipld/car/api'
 import BlockMap from './block-map'
 import { RepoRecord } from '@atproto/lexicon'
 import CidSet from './cid-set'
@@ -15,7 +16,7 @@ const unsignedCommit = z.object({
   data: common.cid,
   rev: z.string(),
   // `prev` added for backwards compatibility with v2, no requirement of keeping around history
-  prev: common.cid.nullable().optional(),
+  prev: common.cid.nullable(),
 })
 export type UnsignedCommit = z.infer<typeof unsignedCommit> & { sig?: never }
 
@@ -24,7 +25,7 @@ const commit = z.object({
   version: z.literal(3),
   data: common.cid,
   rev: z.string(),
-  prev: common.cid.nullable().optional(),
+  prev: common.cid.nullable(),
   sig: common.bytes,
 })
 export type Commit = z.infer<typeof commit>
@@ -95,16 +96,25 @@ export type RecordDeleteOp = {
 
 export type RecordWriteOp = RecordCreateOp | RecordUpdateOp | RecordDeleteOp
 
-export type RecordCreateDescript = RecordCreateOp & {
+export type RecordCreateDescript = {
+  action: WriteOpAction.Create
+  collection: string
+  rkey: string
   cid: CID
 }
 
-export type RecordUpdateDescript = RecordUpdateOp & {
+export type RecordUpdateDescript = {
+  action: WriteOpAction.Update
+  collection: string
+  rkey: string
   prev: CID
   cid: CID
 }
 
-export type RecordDeleteDescript = RecordDeleteOp & {
+export type RecordDeleteDescript = {
+  action: WriteOpAction.Delete
+  collection: string
+  rkey: string
   cid: CID
 }
 
@@ -163,3 +173,5 @@ export type VerifiedRepo = {
   creates: RecordCreateDescript[]
   commit: CommitData
 }
+
+export type CarBlock = car.Block

package/src/util.ts

@@ -1,8 +1,8 @@
+import { setImmediate } from 'node:timers/promises'
 import { CID } from 'multiformats/cid'
 import * as cbor from '@ipld/dag-cbor'
-import { CarReader } from '@ipld/car/reader'
+import { CarBlockIterator } from '@ipld/car'
 import { BlockWriter, CarWriter } from '@ipld/car/writer'
-import { Block as CarBlock } from '@ipld/car/api'
 import {
   streamToBuffer,
   verifyCidForBytes,
@@ -18,6 +18,7 @@ import { ipldToLex, lexToIpld, LexValue, RepoRecord } from '@atproto/lexicon'
 import * as crypto from '@atproto/crypto'
 import DataDiff from './data-diff'
 import {
+  CarBlock,
   Commit,
   LegacyV2Commit,
   RecordCreateDescript,
@@ -29,7 +30,6 @@ import {
   WriteOpAction,
 } from './types'
 import BlockMap from './block-map'
-import * as parse from './parse'
 import { Keypair } from '@atproto/crypto'
 import { Readable } from 'stream'
 
@@ -88,14 +88,15 @@ export const blocksToCarFile = (
   return streamToBuffer(carStream)
 }
 
-export const readCar = async (
-  bytes: Uint8Array,
+export const carToBlocks = async (
+  car: CarBlockIterator,
 ): Promise<{ roots: CID[]; blocks: BlockMap }> => {
-  const car = await CarReader.fromBytes(bytes)
   const roots = await car.getRoots()
   const blocks = new BlockMap()
-  for await (const block of verifyIncomingCarBlocks(car.blocks())) {
+  for await (const block of verifyIncomingCarBlocks(car)) {
     blocks.set(block.cid, block.bytes)
+    // break up otherwise "synchronous" work in car parsing
+    await setImmediate()
   }
   return {
     roots,
@@ -103,6 +104,18 @@ export const readCar = async (
   }
 }
 
+export const readCar = async (
+  bytes: Uint8Array,
+): Promise<{ roots: CID[]; blocks: BlockMap }> => {
+  const car = await CarBlockIterator.fromBytes(bytes)
+  return carToBlocks(car)
+}
+
+export const readCarStream = async (stream: AsyncIterable<Uint8Array>) => {
+  const car = await CarBlockIterator.fromIterable(stream)
+  return carToBlocks(car)
+}
+
 export const readCarWithRoot = async (
   bytes: Uint8Array,
 ): Promise<{ root: CID; blocks: BlockMap }> => {
@@ -119,30 +132,25 @@ export const readCarWithRoot = async (
 
 export const diffToWriteDescripts = (
   diff: DataDiff,
-  blocks: BlockMap,
 ): Promise<RecordWriteDescript[]> => {
   return Promise.all([
     ...diff.addList().map(async (add) => {
       const { collection, rkey } = parseDataKey(add.key)
-      const value = await parse.getAndParseRecord(blocks, add.cid)
       return {
         action: WriteOpAction.Create,
         collection,
         rkey,
         cid: add.cid,
-        record: value.record,
       } as RecordCreateDescript
     }),
     ...diff.updateList().map(async (upd) => {
       const { collection, rkey } = parseDataKey(upd.key)
-      const value = await parse.getAndParseRecord(blocks, upd.cid)
       return {
         action: WriteOpAction.Update,
         collection,
         rkey,
         cid: upd.cid,
         prev: upd.prev,
-        record: value.record,
       } as RecordUpdateDescript
     }),
     ...diff.deleteList().map((del) => {

package/tests/sync.test.ts

@@ -4,6 +4,7 @@ import {
   Repo,
   RepoContents,
   RepoVerificationError,
+  getAndParseRecord,
   readCarWithRoot,
 } from '../src'
 import { MemoryBlockstore } from '../src/storage'
@@ -47,7 +48,8 @@ describe('Repo Sync', () => {
     const contentsFromOps: RepoContents = {}
     for (const write of verified.creates) {
       contentsFromOps[write.collection] ??= {}
-      contentsFromOps[write.collection][write.rkey] = write.record
+      const parsed = await getAndParseRecord(car.blocks, write.cid)
+      contentsFromOps[write.collection][write.rkey] = parsed.record
     }
     expect(contentsFromOps).toEqual(repoData)
   })