@atproto/repo 0.7.3 → 0.8.0

package/dist/sync/provider.js

@@ -24,6 +24,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.getRecords = exports.getFullRepo = void 0;
+const car_1 = require("../car");
 const cid_set_1 = require("../cid-set");
 const error_1 = require("../error");
 const mst_1 = require("../mst");
@@ -32,33 +33,37 @@ const util = __importStar(require("../util"));
 // Full Repo
 // -------------
 const getFullRepo = (storage, commitCid) => {
-    return util.writeCar(commitCid, async (car) => {
-        const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
-        await car.put({ cid: commitCid, bytes: commit.bytes });
-        const mst = mst_1.MST.load(storage, commit.obj.data);
-        await mst.writeToCarStream(car);
-    });
+    return (0, car_1.writeCarStream)(commitCid, iterateFullRepo(storage, commitCid));
 };
 exports.getFullRepo = getFullRepo;
+async function* iterateFullRepo(storage, commitCid) {
+    const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
+    yield { cid: commitCid, bytes: commit.bytes };
+    const mst = mst_1.MST.load(storage, commit.obj.data);
+    for await (const block of mst.carBlockStream()) {
+        yield block;
+    }
+}
 // Narrow slices
 // -------------
 const getRecords = (storage, commitCid, paths) => {
-    return util.writeCar(commitCid, async (car) => {
-        const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
-        await car.put({ cid: commitCid, bytes: commit.bytes });
-        const mst = mst_1.MST.load(storage, commit.obj.data);
-        const cidsForPaths = await Promise.all(paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))));
-        const allCids = cidsForPaths.reduce((acc, cur) => {
-            return acc.addSet(new cid_set_1.CidSet(cur));
-        }, new cid_set_1.CidSet());
-        const found = await storage.getBlocks(allCids.toList());
-        if (found.missing.length > 0) {
-            throw new error_1.MissingBlocksError('writeRecordsToCarStream', found.missing);
-        }
-        for (const block of found.blocks.entries()) {
-            await car.put(block);
-        }
-    });
+    return (0, car_1.writeCarStream)(commitCid, iterateRecordBlocks(storage, commitCid, paths));
 };
 exports.getRecords = getRecords;
+async function* iterateRecordBlocks(storage, commitCid, paths) {
+    const commit = await storage.readObjAndBytes(commitCid, types_1.def.commit);
+    yield { cid: commitCid, bytes: commit.bytes };
+    const mst = mst_1.MST.load(storage, commit.obj.data);
+    const cidsForPaths = await Promise.all(paths.map((p) => mst.cidsForPath(util.formatDataKey(p.collection, p.rkey))));
+    const allCids = cidsForPaths.reduce((acc, cur) => {
+        return acc.addSet(new cid_set_1.CidSet(cur));
+    }, new cid_set_1.CidSet());
+    const found = await storage.getBlocks(allCids.toList());
+    if (found.missing.length > 0) {
+        throw new error_1.MissingBlocksError('writeRecordsToCarStream', found.missing);
+    }
+    for (const block of found.blocks.entries()) {
+        yield block;
+    }
+}
 //# sourceMappingURL=provider.js.map

package/dist/sync/provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/sync/provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,wCAAmC;AACnC,oCAA6C;AAC7C,gCAA4B;AAE5B,oCAA0C;AAC1C,8CAA+B;AAE/B,YAAY;AACZ,gBAAgB;AAET,MAAM,WAAW,GAAG,CACzB,OAAoB,EACpB,SAAc,EACa,EAAE;IAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,EAAE,GAAgB,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;QACnE,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;QACtD,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9C,MAAM,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAVY,QAAA,WAAW,eAUvB;AAED,gBAAgB;AAChB,gBAAgB;AAET,MAAM,UAAU,GAAG,CACxB,OAA2B,EAC3B,SAAc,EACd,KAAmB,EACQ,EAAE;IAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,EAAE,GAAgB,EAAE,EAAE;QACzD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;QACnE,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;QACtD,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACd,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAC1D,CACF,CAAA;QACD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAC/C,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,gBAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACpC,CAAC,EAAE,IAAI,gBAAM,EAAE,CAAC,CAAA;QAChB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,0BAAkB,CAAC,yBAAyB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;QACxE,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YAC3C,MAAM,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtB,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAzBY,QAAA,UAAU,cAyBtB"}
+{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../src/sync/provider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,gCAAuC;AACvC,wCAAmC;AACnC,oCAA6C;AAC7C,gCAA4B;AAE5B,oCAA0C;AAC1C,8CAA+B;AAE/B,YAAY;AACZ,gBAAgB;AAET,MAAM,WAAW,GAAG,CACzB,OAAoB,EACpB,SAAc,EACa,EAAE;IAC7B,OAAO,IAAA,oBAAc,EAAC,SAAS,EAAE,eAAe,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAA;AACvE,CAAC,CAAA;AALY,QAAA,WAAW,eAKvB;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAAC,OAAoB,EAAE,SAAc;IAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;IACnE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAA;IAC7C,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9C,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,CAAC,cAAc,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,gBAAgB;AAChB,gBAAgB;AAET,MAAM,UAAU,GAAG,CACxB,OAA2B,EAC3B,SAAc,EACd,KAAmB,EACQ,EAAE;IAC7B,OAAO,IAAA,oBAAc,EACnB,SAAS,EACT,mBAAmB,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAC/C,CAAA;AACH,CAAC,CAAA;AATY,QAAA,UAAU,cAStB;AAED,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACjC,OAA2B,EAC3B,SAAc,EACd,KAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,SAAS,EAAE,WAAG,CAAC,MAAM,CAAC,CAAA;IACnE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAA;IAC7C,MAAM,GAAG,GAAG,SAAG,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC5E,CAAA;IACD,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC/C,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,gBAAM,CAAC,GAAG,CAAC,CAAC,CAAA;IACpC,CAAC,EAAE,IAAI,gBAAM,EAAE,CAAC,CAAA;IAChB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IACvD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,0BAAkB,CAAC,yBAAyB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IACxE,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC"}

package/dist/types.d.ts

@@ -204,6 +204,16 @@ export declare const schema: {
         prev?: any;
     }>]>;
     cid: z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>;
+    carHeader: z.ZodObject<{
+        version: z.ZodLiteral<1>;
+        roots: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        version: 1;
+        roots: CID[];
+    }, {
+        version: 1;
+        roots: any[];
+    }>;
     bytes: z.ZodType<Uint8Array, z.ZodTypeDef, Uint8Array>;
     string: z.ZodString;
     array: z.ZodArray<z.ZodUnknown, "many">;
@@ -283,6 +293,7 @@ export declare const def: {
         }>]>;
     };
     cid: import("@atproto/common-web/dist/check").Def<CID>;
+    carHeader: import("@atproto/common-web/dist/check").Def<import("@atproto/common").CarHeader>;
     bytes: import("@atproto/common-web/dist/check").Def<Uint8Array>;
     string: import("@atproto/common-web/dist/check").Def<string>;
     map: import("@atproto/common-web/dist/check").Def<Record<string, unknown>>;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAKlC,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,KAAK,CAAA;CAAE,CAAA;AAE7E,QAAA,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;EAOV,CAAA;AACF,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAA;AAE3C,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAA;AAE3D,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAGnB,CAAA;AACF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAA;AAE7D,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA5CO,EAAE,UAAU,CAAC,EAAE,UAAU,CAAC,EACpD,MAAK;WAAkC,EAAG,OACvC,aAAY,EAAG,UAAU;YACZ,EAAE,SAAS;WAExB,EAAG,QACN,CAAC,EAAE,UAAU;SAED,EAAG,SAAS,CAAC,EAAE,SACzB,EAAC,EAAI,UAAS;aACJ,EAAG,UACf;CAsCC,CAAA;AAED,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAUf,CAAA;AAKD,oBAAY,aAAa;IACvB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,cAAc,CAAA;AAE5E,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,GAAG,CAAA;IACT,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,mBAAmB,GAC3B,oBAAoB,GACpB,oBAAoB,GACpB,oBAAoB,CAAA;AAExB,MAAM,MAAM,QAAQ,GAAG,mBAAmB,EAAE,EAAE,CAAA;AAK9C,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,GAAG,CAAA;IACR,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,IAAI,EAAE,GAAG,GAAG,IAAI,CAAA;IAChB,SAAS,EAAE,QAAQ,CAAA;IACnB,cAAc,EAAE,QAAQ,CAAA;IACxB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG;IACpC,GAAG,EAAE,aAAa,EAAE,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;AAC3D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAE7D,MAAM,MAAM,iBAAiB,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAAA;AAC/D,MAAM,MAAM,0BAA0B,GAAG,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAA;AAC1E,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAA;AAE7E,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AAEnD,MAAM,MAAM,UAAU,GAAG;IACvB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,GAAG,IAAI,CAAA;CAC1B,CAAA;AAKD,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,mBAAmB,EAAE,CAAA;IAC7B,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC/B,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,QAAQ,GAAG;IACrB,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,UAAU,CAAA;CAClB,CAAA"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAA;AAKlC,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,KAAK,CAAA;CAAE,CAAA;AAE7E,QAAA,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;EAOV,CAAA;AACF,MAAM,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,MAAM,CAAC,CAAA;AAE3C,QAAA,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;EAOlB,CAAA;AACF,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAA;AAE3D,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAGnB,CAAA;AACF,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAA;AAE7D,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;SA/BP,EAAG,UAAU,CAAC,EAAE,UAAU,CAAC,EAAE,MAAM;eAAsC,EAAG,SACvF;iBAAmB,EAAG,UACrB;eACgB,EAAG,QAAQ,CAAC,EAAE,UAAU,CAAE,EAAC,UAAU,CAAC,EAAE,MAAM;gBAGzD,EAAG,UACT;;;;;;;WAKD,EACA,OAAO,aAAY,EAAG,UAAU;YAEnB,EAAG,SAAS;WACtB,EAAI,QAAO,CAAC,EAAE,UACT;SACN,EAAG,SAAS,CAAC,EAAE,SACX,EAAC,EAAG,UAAU;aACf,EAAG,UAAU;CAgBjB,CAAA;AAED,eAAO,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAUf,CAAA;AAKD,oBAAY,aAAa;IACvB,MAAM,WAAW;IACjB,MAAM,WAAW;IACjB,MAAM,WAAW;CAClB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG,cAAc,GAAG,cAAc,CAAA;AAE5E,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,GAAG,CAAA;IACT,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAA;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,CAAA;CACT,CAAA;AAED,MAAM,MAAM,mBAAmB,GAC3B,oBAAoB,GACpB,oBAAoB,GACpB,oBAAoB,CAAA;AAExB,MAAM,MAAM,QAAQ,GAAG,mBAAmB,EAAE,EAAE,CAAA;AAK9C,MAAM,MAAM,UAAU,GAAG;IACvB,GAAG,EAAE,GAAG,CAAA;IACR,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,IAAI,EAAE,GAAG,GAAG,IAAI,CAAA;IAChB,SAAS,EAAE,QAAQ,CAAA;IACnB,cAAc,EAAE,QAAQ,CAAA;IACxB,WAAW,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG;IACpC,GAAG,EAAE,aAAa,EAAE,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;AAC3D,MAAM,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;AAE7D,MAAM,MAAM,iBAAiB,GAAG;IAAE,GAAG,EAAE,GAAG,CAAC;IAAC,KAAK,EAAE,UAAU,CAAA;CAAE,CAAA;AAC/D,MAAM,MAAM,0BAA0B,GAAG,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAA;AAC1E,MAAM,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAA;AAE7E,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;AAEnD,MAAM,MAAM,UAAU,GAAG;IACvB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,GAAG,GAAG,IAAI,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,GAAG,IAAI,CAAA;CAC1B,CAAA;AAKD,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,EAAE,mBAAmB,EAAE,CAAA;IAC7B,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,YAAY,GAAG;IACzB,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC/B,MAAM,EAAE,UAAU,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,QAAQ,GAAG;IACrB,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,UAAU,CAAA;CAClB,CAAA"}

package/dist/util.d.ts

@@ -1,33 +1,7 @@
-import { Readable } from 'node:stream';
-import { CarBlockIterator } from '@ipld/car/iterator';
-import { BlockWriter } from '@ipld/car/writer';
-import { CID } from 'multiformats/cid';
 import { Keypair } from '@atproto/crypto';
 import { LexValue, RepoRecord } from '@atproto/lexicon';
-import { BlockMap } from './block-map';
 import { DataDiff } from './data-diff';
-import { CarBlock, Commit, LegacyV2Commit, RecordCreateDescript, RecordPath, RecordWriteDescript, UnsignedCommit } from './types';
-export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>): AsyncIterable<CarBlock>;
-export declare function writeCarStream(root: CID | null, fn: (car: BlockWriter) => Promise<void>): Readable;
-export declare function writeCar(root: CID | null, fn: (car: BlockWriter) => Promise<void>): AsyncIterable<Uint8Array>;
-export declare const blocksToCarStream: (root: CID | null, blocks: BlockMap) => AsyncIterable<Uint8Array>;
-export declare const blocksToCarFile: (root: CID | null, blocks: BlockMap) => Promise<Uint8Array>;
-export declare const carToBlocks: (car: CarBlockIterator) => Promise<{
-    roots: CID[];
-    blocks: BlockMap;
-}>;
-export declare const readCar: (bytes: Uint8Array) => Promise<{
-    roots: CID[];
-    blocks: BlockMap;
-}>;
-export declare const readCarStream: (stream: AsyncIterable<Uint8Array>) => Promise<{
-    roots: CID[];
-    blocks: BlockMap;
-}>;
-export declare const readCarWithRoot: (bytes: Uint8Array) => Promise<{
-    root: CID;
-    blocks: BlockMap;
-}>;
+import { Commit, LegacyV2Commit, RecordCreateDescript, RecordPath, RecordWriteDescript, UnsignedCommit } from './types';
 export declare const diffToWriteDescripts: (diff: DataDiff) => Promise<RecordWriteDescript[]>;
 export declare const ensureCreates: (descripts: RecordWriteDescript[]) => RecordCreateDescript[];
 export declare const parseDataKey: (key: string) => RecordPath;
@@ -37,6 +11,6 @@ export declare const signCommit: (unsigned: UnsignedCommit, keypair: Keypair) =>
 export declare const verifyCommitSig: (commit: Commit, didKey: string) => Promise<boolean>;
 export declare const cborToLex: (val: Uint8Array) => LexValue;
 export declare const cborToLexRecord: (val: Uint8Array) => RepoRecord;
-export declare const cidForRecord: (val: LexValue) => Promise<CID>;
+export declare const cidForRecord: (val: LexValue) => Promise<import("multiformats/cid").CID>;
 export declare const ensureV3Commit: (commit: LegacyV2Commit | Commit) => Commit;
 //# sourceMappingURL=util.d.ts.map

package/dist/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAa,MAAM,kBAAkB,CAAA;AAEzD,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAYtC,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAwB,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EACL,QAAQ,EACR,MAAM,EACN,cAAc,EACd,oBAAoB,EAEpB,UAAU,EAEV,mBAAmB,EACnB,cAAc,EAEf,MAAM,SAAS,CAAA;AAEhB,wBAAuB,uBAAuB,CAC5C,GAAG,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC3B,aAAa,CAAC,QAAQ,CAAC,CAKzB;AAGD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,EAAE,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,GACtC,QAAQ,CAWV;AAED,wBAAuB,QAAQ,CAC7B,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,EAAE,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,GACtC,aAAa,CAAC,UAAU,CAAC,CAK3B;AAED,eAAO,MAAM,iBAAiB,SACtB,GAAG,GAAG,IAAI,UACR,QAAQ,KACf,aAAa,CAAC,UAAU,CAM1B,CAAA;AAED,eAAO,MAAM,eAAe,SACpB,GAAG,GAAG,IAAI,UACR,QAAQ,KACf,OAAO,CAAC,UAAU,CAGpB,CAAA;AAED,eAAO,MAAM,WAAW,QACjB,gBAAgB,KACpB,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAY5C,CAAA;AAED,eAAO,MAAM,OAAO,UACX,UAAU,KAChB,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAG5C,CAAA;AAED,eAAO,MAAM,aAAa,WAAkB,aAAa,CAAC,UAAU,CAAC;WArBjD,GAAG,EAAE;YAAU,QAAQ;EAwB1C,CAAA;AAED,eAAO,MAAM,eAAe,UACnB,UAAU,KAChB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAUzC,CAAA;AAED,eAAO,MAAM,oBAAoB,SACzB,QAAQ,KACb,OAAO,CAAC,mBAAmB,EAAE,CA+B/B,CAAA;AAED,eAAO,MAAM,aAAa,cACb,mBAAmB,EAAE,KAC/B,oBAAoB,EAUtB,CAAA;AAED,eAAO,MAAM,YAAY,QAAS,MAAM,KAAG,UAI1C,CAAA;AAED,eAAO,MAAM,aAAa,eAAgB,MAAM,QAAQ,MAAM,KAAG,MAEhE,CAAA;AAED,eAAO,MAAM,SAAS,MAAO,MAAM,KAAK,MAAM,KAAG,OAEhD,CAAA;AAED,eAAO,MAAM,UAAU,aACX,cAAc,WACf,OAAO,KACf,OAAO,CAAC,MAAM,CAOhB,CAAA;AAED,eAAO,MAAM,eAAe,WAClB,MAAM,UACN,MAAM,KACb,OAAO,CAAC,OAAO,CAIjB,CAAA;AAED,eAAO,MAAM,SAAS,QAAS,UAAU,KAAG,QAE3C,CAAA;AAED,eAAO,MAAM,eAAe,QAAS,UAAU,KAAG,UAMjD,CAAA;AAED,eAAO,MAAM,YAAY,QAAe,QAAQ,iBAE/C,CAAA;AAED,eAAO,MAAM,cAAc,WAAY,cAAc,GAAG,MAAM,KAAG,MAUhE,CAAA"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AACzC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAwB,MAAM,kBAAkB,CAAA;AAC7E,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EACL,MAAM,EACN,cAAc,EACd,oBAAoB,EAEpB,UAAU,EAEV,mBAAmB,EACnB,cAAc,EAEf,MAAM,SAAS,CAAA;AAEhB,eAAO,MAAM,oBAAoB,SACzB,QAAQ,KACb,OAAO,CAAC,mBAAmB,EAAE,CA+B/B,CAAA;AAED,eAAO,MAAM,aAAa,cACb,mBAAmB,EAAE,KAC/B,oBAAoB,EAUtB,CAAA;AAED,eAAO,MAAM,YAAY,QAAS,MAAM,KAAG,UAI1C,CAAA;AAED,eAAO,MAAM,aAAa,eAAgB,MAAM,QAAQ,MAAM,KAAG,MAEhE,CAAA;AAED,eAAO,MAAM,SAAS,MAAO,MAAM,KAAK,MAAM,KAAG,OAEhD,CAAA;AAED,eAAO,MAAM,UAAU,aACX,cAAc,WACf,OAAO,KACf,OAAO,CAAC,MAAM,CAOhB,CAAA;AAED,eAAO,MAAM,eAAe,WAClB,MAAM,UACN,MAAM,KACb,OAAO,CAAC,OAAO,CAIjB,CAAA;AAED,eAAO,MAAM,SAAS,QAAS,UAAU,KAAG,QAE3C,CAAA;AAED,eAAO,MAAM,eAAe,QAAS,UAAU,KAAG,UAMjD,CAAA;AAED,eAAO,MAAM,YAAY,QAAe,QAAQ,4CAE/C,CAAA;AAED,eAAO,MAAM,cAAc,WAAY,cAAc,GAAG,MAAM,KAAG,MAUhE,CAAA"}

package/dist/util.js

@@ -23,91 +23,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureV3Commit = exports.cidForRecord = exports.cborToLexRecord = exports.cborToLex = exports.verifyCommitSig = exports.signCommit = exports.metaEqual = exports.formatDataKey = exports.parseDataKey = exports.ensureCreates = exports.diffToWriteDescripts = exports.readCarWithRoot = exports.readCarStream = exports.readCar = exports.carToBlocks = exports.blocksToCarFile = exports.blocksToCarStream = void 0;
-exports.verifyIncomingCarBlocks = verifyIncomingCarBlocks;
-exports.writeCarStream = writeCarStream;
-exports.writeCar = writeCar;
-const promises_1 = require("node:timers/promises");
-const iterator_1 = require("@ipld/car/iterator");
-const writer_1 = require("@ipld/car/writer");
+exports.ensureV3Commit = exports.cidForRecord = exports.cborToLexRecord = exports.cborToLex = exports.verifyCommitSig = exports.signCommit = exports.metaEqual = exports.formatDataKey = exports.parseDataKey = exports.ensureCreates = exports.diffToWriteDescripts = void 0;
 const cbor = __importStar(require("@ipld/dag-cbor"));
 const common_1 = require("@atproto/common");
 const crypto = __importStar(require("@atproto/crypto"));
 const lexicon_1 = require("@atproto/lexicon");
-const block_map_1 = require("./block-map");
 const types_1 = require("./types");
-async function* verifyIncomingCarBlocks(car) {
-    for await (const block of car) {
-        await (0, common_1.verifyCidForBytes)(block.cid, block.bytes);
-        yield block;
-    }
-}
-// we have to turn the car writer output into a stream in order to properly handle errors
-function writeCarStream(root, fn) {
-    const { writer, out } = root !== null ? writer_1.CarWriter.create(root) : writer_1.CarWriter.create();
-    const stream = (0, common_1.byteIterableToStream)(out);
-    fn(writer)
-        .catch((err) => {
-        stream.destroy(err);
-    })
-        .finally(() => writer.close());
-    return stream;
-}
-async function* writeCar(root, fn) {
-    const stream = writeCarStream(root, fn);
-    for await (const chunk of stream) {
-        yield chunk;
-    }
-}
-const blocksToCarStream = (root, blocks) => {
-    return writeCar(root, async (writer) => {
-        for (const entry of blocks.entries()) {
-            await writer.put(entry);
-        }
-    });
-};
-exports.blocksToCarStream = blocksToCarStream;
-const blocksToCarFile = (root, blocks) => {
-    const carStream = (0, exports.blocksToCarStream)(root, blocks);
-    return (0, common_1.streamToBuffer)(carStream);
-};
-exports.blocksToCarFile = blocksToCarFile;
-const carToBlocks = async (car) => {
-    const roots = await car.getRoots();
-    const blocks = new block_map_1.BlockMap();
-    for await (const block of verifyIncomingCarBlocks(car)) {
-        blocks.set(block.cid, block.bytes);
-        // break up otherwise "synchronous" work in car parsing
-        await (0, promises_1.setImmediate)();
-    }
-    return {
-        roots,
-        blocks,
-    };
-};
-exports.carToBlocks = carToBlocks;
-const readCar = async (bytes) => {
-    const car = await iterator_1.CarBlockIterator.fromBytes(bytes);
-    return (0, exports.carToBlocks)(car);
-};
-exports.readCar = readCar;
-const readCarStream = async (stream) => {
-    const car = await iterator_1.CarBlockIterator.fromIterable(stream);
-    return (0, exports.carToBlocks)(car);
-};
-exports.readCarStream = readCarStream;
-const readCarWithRoot = async (bytes) => {
-    const { roots, blocks } = await (0, exports.readCar)(bytes);
-    if (roots.length !== 1) {
-        throw new Error(`Expected one root, got ${roots.length}`);
-    }
-    const root = roots[0];
-    return {
-        root,
-        blocks,
-    };
-};
-exports.readCarWithRoot = readCarWithRoot;
 const diffToWriteDescripts = (diff) => {
     return Promise.all([
         ...diff.addList().map(async (add) => {

package/dist/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAkCA,0DAOC;AAGD,wCAcC;AAED,4BAQC;AAnED,mDAAmD;AACnD,iDAAqD;AACrD,6CAAyD;AACzD,qDAAsC;AAEtC,4CASwB;AACxB,wDAAyC;AAEzC,8CAA6E;AAC7E,2CAAsC;AAEtC,mCAWgB;AAET,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,GAA4B;IAE5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,MAAM,IAAA,0BAAiB,EAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,yFAAyF;AACzF,SAAgB,cAAc,CAC5B,IAAgB,EAChB,EAAuC;IAEvC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GACnB,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,kBAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAS,CAAC,MAAM,EAAE,CAAA;IAE7D,MAAM,MAAM,GAAG,IAAA,6BAAoB,EAAC,GAAG,CAAC,CAAA;IACxC,EAAE,CAAC,MAAM,CAAC;SACP,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACb,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC,CAAC;SACD,OAAO,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;IAChC,OAAO,MAAM,CAAA;AACf,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,QAAQ,CAC7B,IAAgB,EAChB,EAAuC;IAEvC,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;IACvC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAEM,MAAM,iBAAiB,GAAG,CAC/B,IAAgB,EAChB,MAAgB,EACW,EAAE;IAC7B,OAAO,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;YACrC,MAAM,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACzB,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AATY,QAAA,iBAAiB,qBAS7B;AAEM,MAAM,eAAe,GAAG,CAC7B,IAAgB,EAChB,MAAgB,EACK,EAAE;IACvB,MAAM,SAAS,GAAG,IAAA,yBAAiB,EAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACjD,OAAO,IAAA,uBAAc,EAAC,SAAS,CAAC,CAAA;AAClC,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,WAAW,GAAG,KAAK,EAC9B,GAAqB,EACwB,EAAE;IAC/C,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,CAAA;IAClC,MAAM,MAAM,GAAG,IAAI,oBAAQ,EAAE,CAAA;IAC7B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,uBAAuB,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QAClC,uDAAuD;QACvD,MAAM,IAAA,uBAAY,GAAE,CAAA;IACtB,CAAC;IACD,OAAO;QACL,KAAK;QACL,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAdY,QAAA,WAAW,eAcvB;AAEM,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAiB,EAC4B,EAAE;IAC/C,MAAM,GAAG,GAAG,MAAM,2BAAgB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACnD,OAAO,IAAA,mBAAW,EAAC,GAAG,CAAC,CAAA;AACzB,CAAC,CAAA;AALY,QAAA,OAAO,WAKnB;AAEM,MAAM,aAAa,GAAG,KAAK,EAAE,MAAiC,EAAE,EAAE;IACvE,MAAM,GAAG,GAAG,MAAM,2BAAgB,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IACvD,OAAO,IAAA,mBAAW,EAAC,GAAG,CAAC,CAAA;AACzB,CAAC,CAAA;AAHY,QAAA,aAAa,iBAGzB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,KAAiB,EACyB,EAAE;IAC5C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,eAAO,EAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,OAAO;QACL,IAAI;QACJ,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAZY,QAAA,eAAe,mBAY3B;AAEM,MAAM,oBAAoB,GAAG,CAClC,IAAc,EACkB,EAAE;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC;QACjB,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAClC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAI;aACS,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAA;AAjCY,QAAA,oBAAoB,wBAiChC;AAEM,MAAM,aAAa,GAAG,CAC3B,SAAgC,EACR,EAAE;IAC1B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,qBAAa,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAZY,QAAA,aAAa,iBAYzB;AAEM,MAAM,YAAY,GAAG,CAAC,GAAW,EAAc,EAAE;IACtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;IACrE,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;AACjD,CAAC,CAAA;AAJY,QAAA,YAAY,gBAIxB;AAEM,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAU,EAAE;IACxE,OAAO,UAAU,GAAG,GAAG,GAAG,IAAI,CAAA;AAChC,CAAC,CAAA;AAFY,QAAA,aAAa,iBAEzB;AAEM,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;IACzD,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAA;AACnD,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAwB,EACxB,OAAgB,EACC,EAAE;IACnB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACvC,OAAO;QACL,GAAG,QAAQ;QACX,GAAG;KACJ,CAAA;AACH,CAAC,CAAA;AAVY,QAAA,UAAU,cAUtB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,MAAc,EACd,MAAc,EACI,EAAE;IACpB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAA;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;AACrD,CAAC,CAAA;AAPY,QAAA,eAAe,mBAO3B;AAEM,MAAM,SAAS,GAAG,CAAC,GAAe,EAAY,EAAE;IACrD,OAAO,IAAA,mBAAS,EAAC,IAAA,mBAAU,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,eAAe,GAAG,CAAC,GAAe,EAAc,EAAE;IAC7D,MAAM,MAAM,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,CAAA;IAC7B,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,YAAY,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;IAClD,OAAO,IAAA,mBAAU,EAAC,IAAA,mBAAS,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,YAAY,gBAExB;AAEM,MAAM,cAAc,GAAG,CAAC,MAA+B,EAAU,EAAE;IACxE,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,CAAA;IACf,CAAC;SAAM,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,YAAG,CAAC,OAAO,EAAE;SACjC,CAAA;IACH,CAAC;AACH,CAAC,CAAA;AAVY,QAAA,cAAc,kBAU1B"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qDAAsC;AACtC,4CAA4E;AAC5E,wDAAyC;AAEzC,8CAA6E;AAE7E,mCAUgB;AAET,MAAM,oBAAoB,GAAG,CAClC,IAAc,EACkB,EAAE;IAClC,OAAO,OAAO,CAAC,GAAG,CAAC;QACjB,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAClC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,IAAI,EAAE,GAAG,CAAC,IAAI;aACS,CAAA;QAC3B,CAAC,CAAC;QACF,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC/B,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,IAAA,oBAAY,EAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAClD,OAAO;gBACL,MAAM,EAAE,qBAAa,CAAC,MAAM;gBAC5B,UAAU;gBACV,IAAI;gBACJ,GAAG,EAAE,GAAG,CAAC,GAAG;aACW,CAAA;QAC3B,CAAC,CAAC;KACH,CAAC,CAAA;AACJ,CAAC,CAAA;AAjCY,QAAA,oBAAoB,wBAiChC;AAEM,MAAM,aAAa,GAAG,CAC3B,SAAgC,EACR,EAAE;IAC1B,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,KAAK,qBAAa,CAAC,MAAM,EAAE,CAAC;YAC7C,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC,CAAA;AAZY,QAAA,aAAa,iBAYzB;AAEM,MAAM,YAAY,GAAG,CAAC,GAAW,EAAc,EAAE;IACtD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;IAC5B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAA;IACrE,OAAO,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAA;AACjD,CAAC,CAAA;AAJY,QAAA,YAAY,gBAIxB;AAEM,MAAM,aAAa,GAAG,CAAC,UAAkB,EAAE,IAAY,EAAU,EAAE;IACxE,OAAO,UAAU,GAAG,GAAG,GAAG,IAAI,CAAA;AAChC,CAAC,CAAA;AAFY,QAAA,aAAa,iBAEzB;AAEM,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;IACzD,OAAO,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,CAAA;AACnD,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,UAAU,GAAG,KAAK,EAC7B,QAAwB,EACxB,OAAgB,EACC,EAAE;IACnB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACrC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACvC,OAAO;QACL,GAAG,QAAQ;QACX,GAAG;KACJ,CAAA;AACH,CAAC,CAAA;AAVY,QAAA,UAAU,cAUtB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,MAAc,EACd,MAAc,EACI,EAAE;IACpB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAA;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACjC,OAAO,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAA;AACrD,CAAC,CAAA;AAPY,QAAA,eAAe,mBAO3B;AAEM,MAAM,SAAS,GAAG,CAAC,GAAe,EAAY,EAAE;IACrD,OAAO,IAAA,mBAAS,EAAC,IAAA,mBAAU,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,SAAS,aAErB;AAEM,MAAM,eAAe,GAAG,CAAC,GAAe,EAAc,EAAE;IAC7D,MAAM,MAAM,GAAG,IAAA,iBAAS,EAAC,GAAG,CAAC,CAAA;IAC7B,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;IACrD,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,YAAY,GAAG,KAAK,EAAE,GAAa,EAAE,EAAE;IAClD,OAAO,IAAA,mBAAU,EAAC,IAAA,mBAAS,EAAC,GAAG,CAAC,CAAC,CAAA;AACnC,CAAC,CAAA;AAFY,QAAA,YAAY,gBAExB;AAEM,MAAM,cAAc,GAAG,CAAC,MAA+B,EAAU,EAAE;IACxE,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,MAAM,CAAA;IACf,CAAC;SAAM,CAAC;QACN,OAAO;YACL,GAAG,MAAM;YACT,OAAO,EAAE,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,YAAG,CAAC,OAAO,EAAE;SACjC,CAAA;IACH,CAAC;AACH,CAAC,CAAA;AAVY,QAAA,cAAc,kBAU1B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/repo",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "license": "MIT",
   "description": "atproto repo and MST implementation",
   "keywords": [
@@ -19,15 +19,15 @@
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "dependencies": {
-    "@ipld/car": "^3.2.3",
     "@ipld/dag-cbor": "^7.0.0",
     "multiformats": "^9.9.0",
     "uint8arrays": "3.0.0",
+    "varint": "^6.0.0",
     "zod": "^3.23.8",
-    "@atproto/common": "^0.4.9",
-    "@atproto/common-web": "^0.4.0",
+    "@atproto/common": "^0.4.10",
+    "@atproto/common-web": "^0.4.1",
     "@atproto/crypto": "^0.4.4",
-    "@atproto/lexicon": "^0.4.9"
+    "@atproto/lexicon": "^0.4.10"
   },
   "devDependencies": {
     "jest": "^28.1.2",

package/src/car.ts

@@ -0,0 +1,219 @@
+import * as cbor from '@ipld/dag-cbor'
+import { CID } from 'multiformats/cid'
+import * as ui8 from 'uint8arrays'
+import * as varint from 'varint'
+import {
+  check,
+  parseCidFromBytes,
+  schema,
+  streamToBuffer,
+  verifyCidForBytes,
+} from '@atproto/common'
+import { BlockMap } from './block-map'
+import { CarBlock } from './types'
+
+export async function* writeCarStream(
+  root: CID | null,
+  blocks: AsyncIterable<CarBlock>,
+): AsyncIterable<Uint8Array> {
+  const header = new Uint8Array(
+    cbor.encode({
+      version: 1,
+      roots: root ? [root] : [],
+    }),
+  )
+  yield new Uint8Array(varint.encode(header.byteLength))
+  yield header
+  for await (const block of blocks) {
+    yield new Uint8Array(
+      varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength),
+    )
+    yield block.cid.bytes
+    yield block.bytes
+  }
+}
+
+export const blocksToCarFile = (
+  root: CID | null,
+  blocks: BlockMap,
+): Promise<Uint8Array> => {
+  const carStream = blocksToCarStream(root, blocks)
+  return streamToBuffer(carStream)
+}
+
+export const blocksToCarStream = (
+  root: CID | null,
+  blocks: BlockMap,
+): AsyncIterable<Uint8Array> => {
+  return writeCarStream(root, iterateBlocks(blocks))
+}
+
+async function* iterateBlocks(blocks: BlockMap) {
+  for (const entry of blocks.entries()) {
+    yield { cid: entry.cid, bytes: entry.bytes }
+  }
+}
+
+export const readCar = async (
+  bytes: Uint8Array,
+): Promise<{ roots: CID[]; blocks: BlockMap }> => {
+  const { roots, blocks } = await readCarStream([bytes])
+  const blockMap = new BlockMap()
+  for await (const block of blocks) {
+    blockMap.set(block.cid, block.bytes)
+  }
+  return { roots, blocks: blockMap }
+}
+
+export const readCarWithRoot = async (
+  bytes: Uint8Array,
+): Promise<{ root: CID; blocks: BlockMap }> => {
+  const { roots, blocks } = await readCar(bytes)
+  if (roots.length !== 1) {
+    throw new Error(`Expected one root, got ${roots.length}`)
+  }
+  const root = roots[0]
+  return {
+    root,
+    blocks,
+  }
+}
+export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
+  dump: () => Promise<void>
+}
+
+export const readCarStream = async (
+  car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,
+): Promise<{
+  roots: CID[]
+  blocks: CarBlockIterable
+}> => {
+  const reader = new BufferedReader(car)
+  try {
+    const headerSize = await reader.readVarint()
+    if (headerSize === null) {
+      throw new Error('Could not parse CAR header')
+    }
+    const headerBytes = await reader.read(headerSize)
+    const header = cbor.decode(headerBytes)
+    if (!check.is(header, schema.carHeader)) {
+      throw new Error('Could not parse CAR header')
+    }
+    return {
+      roots: header.roots,
+      blocks: readCarBlocksIter(reader),
+    }
+  } catch (err) {
+    await reader.close()
+    throw err
+  }
+}
+
+const readCarBlocksIter = (reader: BufferedReader) => {
+  const iter = readCarBlocksIterGenerator(reader) as CarBlockIterable
+
+  iter.dump = async () => {
+    // try/finally to ensure that reader.close is called even if blocks.return throws.
+    try {
+      // Prevent the iterator from being started after this method is called.
+      await iter.return()
+    } finally {
+      // @NOTE the "finally" block of the async generator won't be called
+      // if the iteration was never started so we need to manually close here.
+      await reader.close()
+    }
+  }
+
+  return iter
+}
+
+async function* readCarBlocksIterGenerator(
+  reader: BufferedReader,
+): AsyncIterable<CarBlock> {
+  try {
+    while (!reader.isDone) {
+      const blockSize = await reader.readVarint()
+      if (blockSize === null) {
+        break
+      }
+      const blockBytes = await reader.read(blockSize)
+      const cid = parseCidFromBytes(blockBytes.slice(0, 36))
+      const bytes = blockBytes.slice(36)
+      yield { cid, bytes }
+    }
+  } finally {
+    await reader.close()
+  }
+}
+
+export async function* verifyIncomingCarBlocks(
+  car: AsyncIterable<CarBlock>,
+): AsyncIterable<CarBlock> {
+  for await (const block of car) {
+    await verifyCidForBytes(block.cid, block.bytes)
+    yield block
+  }
+}
+
+class BufferedReader {
+  buffer: Uint8Array = new Uint8Array()
+  iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>
+  isDone = false
+
+  constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {
+    this.iterator =
+      Symbol.asyncIterator in stream
+        ? stream[Symbol.asyncIterator]()
+        : stream[Symbol.iterator]()
+  }
+
+  async read(bytesToRead: number): Promise<Uint8Array> {
+    await this.readUntilBuffered(bytesToRead)
+    const value = this.buffer.slice(0, bytesToRead)
+    this.buffer = this.buffer.slice(bytesToRead)
+    return value
+  }
+
+  async readVarint(): Promise<number | null> {
+    let done = false
+    const bytes: Uint8Array[] = []
+    while (!done) {
+      const byte = await this.read(1)
+      if (byte.byteLength === 0) {
+        if (bytes.length > 0) {
+          throw new Error('could not parse varint')
+        } else {
+          return null
+        }
+      }
+      bytes.push(byte)
+      if (byte[0] < 128) {
+        done = true
+      }
+    }
+    const concatted = ui8.concat(bytes)
+    return varint.decode(concatted)
+  }
+
+  private async readUntilBuffered(bytesToRead: number) {
+    if (this.isDone) {
+      return
+    }
+    while (this.buffer.length < bytesToRead) {
+      const next = await this.iterator.next()
+      if (next.done) {
+        this.isDone = true
+        return
+      }
+      this.buffer = ui8.concat([this.buffer, next.value])
+    }
+  }
+
+  async close(): Promise<void> {
+    if (!this.isDone && this.iterator.return) {
+      await this.iterator.return()
+    }
+    this.isDone = true
+    this.buffer = new Uint8Array()
+  }
+}

package/src/index.ts

@@ -7,4 +7,5 @@ export * from './storage'
 export * from './sync'
 export * from './types'
 export * from './data-diff'
+export * from './car'
 export * from './util'

package/src/mst/mst.ts

@@ -1,4 +1,3 @@
-import { BlockWriter } from '@ipld/car/writer'
 import { CID } from 'multiformats'
 import { z } from 'zod'
 import { cidForCbor, dataToCborBlock, schema as common } from '@atproto/common'
@@ -7,6 +6,7 @@ import { CidSet } from '../cid-set'
 import { MissingBlockError, MissingBlocksError } from '../error'
 import * as parse from '../parse'
 import { ReadableBlockstore } from '../storage'
+import { CarBlock } from '../types'
 import * as util from './util'
 
 /**
@@ -726,7 +726,7 @@ export class MST {
 
   // Sync Protocol
 
-  async writeToCarStream(car: BlockWriter): Promise<void> {
+  async *carBlockStream(): AsyncIterable<CarBlock> {
     const leaves = new CidSet()
     let toFetch = new CidSet()
     toFetch.add(await this.getPointer())
@@ -742,7 +742,7 @@ export class MST {
           cid,
           nodeDataDef,
         )
-        await car.put({ cid, bytes: found.bytes })
+        yield { cid, bytes: found.bytes }
         const entries = await util.deserializeNodeData(this.storage, found.obj)
 
         for (const entry of entries) {
@@ -761,7 +761,7 @@ export class MST {
     }
 
     for (const leaf of leafData.blocks.entries()) {
-      await car.put(leaf)
+      yield leaf
     }
   }
 

package/src/sync/consumer.ts

@@ -1,5 +1,6 @@
 import { CID } from 'multiformats/cid'
 import { BlockMap } from '../block-map'
+import { readCarWithRoot } from '../car'
 import { DataDiff } from '../data-diff'
 import { MST } from '../mst'
 import { ReadableRepo } from '../readable-repo'
@@ -18,7 +19,7 @@ export const verifyRepoCar = async (
   did?: string,
   signingKey?: string,
 ): Promise<VerifiedRepo> => {
-  const car = await util.readCarWithRoot(carBytes)
+  const car = await readCarWithRoot(carBytes)
   return verifyRepo(car.blocks, car.root, did, signingKey)
 }
 
@@ -44,7 +45,7 @@ export const verifyDiffCar = async (
   signingKey?: string,
   opts?: { ensureLeaves?: boolean },
 ): Promise<VerifiedDiff> => {
-  const car = await util.readCarWithRoot(carBytes)
+  const car = await readCarWithRoot(carBytes)
   return verifyDiff(repo, car.blocks, car.root, did, signingKey, opts)
 }
 
@@ -127,7 +128,7 @@ export const verifyProofs = async (
   did: string,
   didKey: string,
 ): Promise<{ verified: RecordCidClaim[]; unverified: RecordCidClaim[] }> => {
-  const car = await util.readCarWithRoot(proofs)
+  const car = await readCarWithRoot(proofs)
   const blockstore = new MemoryBlockstore(car.blocks)
   const commit = await blockstore.readObj(car.root, def.commit)
   if (commit.did !== did) {
@@ -169,7 +170,7 @@ export const verifyRecords = async (
   did: string,
   signingKey: string,
 ): Promise<RecordClaim[]> => {
-  const car = await util.readCarWithRoot(proofs)
+  const car = await readCarWithRoot(proofs)
   const blockstore = new MemoryBlockstore(car.blocks)
   const commit = await blockstore.readObj(car.root, def.commit)
   if (commit.did !== did) {