@atproto/repo 0.8.12 → 0.9.0

package/CHANGELOG.md

@@ -1,5 +1,28 @@
 # @atproto/repo
 
+## 0.9.0
+
+### Minor Changes
+
+- [#4408](https://github.com/bluesky-social/atproto/pull/4408) [`d0c136c`](https://github.com/bluesky-social/atproto/commit/d0c136cba2ec8fa97017849b1023d5af5d2cc60c) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Use `Cid` from `@atproto/lex` instead of `CID` from `multiformats` to represent CIDs.
+
+- [#4408](https://github.com/bluesky-social/atproto/pull/4408) [`d0c136c`](https://github.com/bluesky-social/atproto/commit/d0c136cba2ec8fa97017849b1023d5af5d2cc60c) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Remove dependency on `multiformats`
+
+### Patch Changes
+
+- [#4408](https://github.com/bluesky-social/atproto/pull/4408) [`d0c136c`](https://github.com/bluesky-social/atproto/commit/d0c136cba2ec8fa97017849b1023d5af5d2cc60c) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Remove dependency on `uint8arrays`
+
+- [#4408](https://github.com/bluesky-social/atproto/pull/4408) [`d0c136c`](https://github.com/bluesky-social/atproto/commit/d0c136cba2ec8fa97017849b1023d5af5d2cc60c) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Makes `@atproto/repo` less dependent on NodeJS internal (to eventually allow this lib to work in other environments)
+
+- Updated dependencies [[`0dbea15`](https://github.com/bluesky-social/atproto/commit/0dbea15da48a6ca913cc3a3a2d8c0ffe64d7c69a), [`d0c136c`](https://github.com/bluesky-social/atproto/commit/d0c136cba2ec8fa97017849b1023d5af5d2cc60c), [`527f5d4`](https://github.com/bluesky-social/atproto/commit/527f5d4c5d0c9264c2ff6f23ad06a41163fc6809)]:
+  - @atproto/syntax@0.5.2
+
+## 0.8.13
+
+### Patch Changes
+
+- [#4729](https://github.com/bluesky-social/atproto/pull/4729) [`192685f`](https://github.com/bluesky-social/atproto/commit/192685fca75a68c9c50a94817d3f27da7fc02f56) Thanks [@matthieusieben](https://github.com/matthieusieben)! - Optimize BufferedReader implementation
+
 ## 0.8.12
 
 ### Patch Changes

package/LICENSE.txt

@@ -1,6 +1,6 @@
 Dual MIT/Apache-2.0 License
 
-Copyright (c) 2022-2025 Bluesky Social PBC, and Contributors
+Copyright (c) 2022-2026 Bluesky Social PBC, and Contributors
 
 Except as otherwise noted in individual files, this software is licensed under the MIT license (<http://opensource.org/licenses/MIT>), or the Apache License, Version 2.0 (<http://www.apache.org/licenses/LICENSE-2.0>).
 

package/dist/block-map.d.ts

@@ -1,31 +1,30 @@
-import { CID } from 'multiformats/cid';
-import { LexValue } from '@atproto/lexicon';
-export declare class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {
+import { Cid, LexValue } from '@atproto/lex-data';
+export declare class BlockMap implements Iterable<[cid: Cid, bytes: Uint8Array]> {
     private map;
-    constructor(entries?: Iterable<readonly [cid: CID, bytes: Uint8Array]>);
-    add(value: LexValue): Promise<CID>;
-    set(cid: CID, bytes: Uint8Array): BlockMap;
-    get(cid: CID): Uint8Array | undefined;
-    delete(cid: CID): BlockMap;
-    getMany(cids: CID[]): {
+    constructor(entries?: Iterable<readonly [cid: Cid, bytes: Uint8Array]>);
+    add(value: LexValue): Promise<Cid>;
+    set(cid: Cid, bytes: Uint8Array): BlockMap;
+    get(cid: Cid): Uint8Array | undefined;
+    delete(cid: Cid): BlockMap;
+    getMany(cids: Cid[]): {
         blocks: BlockMap;
-        missing: CID[];
+        missing: Cid[];
     };
-    has(cid: CID): boolean;
+    has(cid: Cid): boolean;
     clear(): void;
-    forEach(cb: (bytes: Uint8Array, cid: CID) => void): void;
+    forEach(cb: (bytes: Uint8Array, cid: Cid) => void): void;
     entries(): Entry[];
-    cids(): CID[];
+    cids(): Cid[];
     addMap(toAdd: BlockMap): BlockMap;
     get size(): number;
     get byteSize(): number;
     equals(other: BlockMap): boolean;
-    keys(): Generator<CID, void, unknown>;
+    keys(): Generator<Cid, void, unknown>;
     values(): Generator<Uint8Array, void, unknown>;
-    [Symbol.iterator](): Generator<[CID, Uint8Array], void, unknown>;
+    [Symbol.iterator](): Generator<[Cid, Uint8Array], void, unknown>;
 }
 type Entry = {
-    cid: CID;
+    cid: Cid;
     bytes: Uint8Array;
 };
 export default BlockMap;

package/dist/block-map.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"block-map.d.ts","sourceRoot":"","sources":["../src/block-map.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAGtC,OAAO,EAAE,QAAQ,EAAa,MAAM,kBAAkB,CAAA;AAEtD,qBAAa,QAAS,YAAW,QAAQ,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAqC;gBAEpC,OAAO,CAAC,EAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAQhE,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAMxC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,GAAG,QAAQ;IAK1C,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,GAAG,SAAS;IAIrC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,QAAQ;IAK1B,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG;QAAE,MAAM,EAAE,QAAQ,CAAC;QAAC,OAAO,EAAE,GAAG,EAAE,CAAA;KAAE;IAc1D,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO;IAItB,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI;IAIxD,OAAO,IAAI,KAAK,EAAE;IAIlB,IAAI,IAAI,GAAG,EAAE;IAIb,MAAM,CAAC,KAAK,EAAE,QAAQ,GAAG,QAAQ;IAKjC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,QAAQ,IAAI,MAAM,CAIrB;IAED,MAAM,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO;IAc/B,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC;IAMrC,MAAM,IAAI,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC;IAI9C,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;CAKlE;AAMD,KAAK,KAAK,GAAG;IACX,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,UAAU,CAAA;CAClB,CAAA;AAED,eAAe,QAAQ,CAAA"}
+{"version":3,"file":"block-map.d.ts","sourceRoot":"","sources":["../src/block-map.ts"],"names":[],"mappings":"AACA,OAAO,EACL,GAAG,EACH,QAAQ,EAIT,MAAM,mBAAmB,CAAA;AAE1B,qBAAa,QAAS,YAAW,QAAQ,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IACtE,OAAO,CAAC,GAAG,CAAqC;gBAEpC,OAAO,CAAC,EAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAQhE,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC;IAOxC,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,GAAG,QAAQ;IAK1C,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,GAAG,SAAS;IAIrC,MAAM,CAAC,GAAG,EAAE,GAAG,GAAG,QAAQ;IAK1B,OAAO,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG;QAAE,MAAM,EAAE,QAAQ,CAAC;QAAC,OAAO,EAAE,GAAG,EAAE,CAAA;KAAE;IAc1D,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO;IAItB,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,KAAK,IAAI,GAAG,IAAI;IAIxD,OAAO,IAAI,KAAK,EAAE;IAIlB,IAAI,IAAI,GAAG,EAAE;IAIb,MAAM,CAAC,KAAK,EAAE,QAAQ,GAAG,QAAQ;IAKjC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,IAAI,QAAQ,IAAI,MAAM,CAIrB;IAED,MAAM,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO;IAc/B,IAAI,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC;IAMrC,MAAM,IAAI,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC;IAI9C,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;CAKlE;AAMD,KAAK,KAAK,GAAG;IACX,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,UAAU,CAAA;CAClB,CAAA;AAED,eAAe,QAAQ,CAAA"}

package/dist/block-map.js

@@ -1,44 +1,8 @@
 "use strict";
-/* eslint-disable import/no-deprecated */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BlockMap = void 0;
-const cid_1 = require("multiformats/cid");
-const uint8arrays = __importStar(require("uint8arrays"));
-const common_1 = require("@atproto/common");
-const lexicon_1 = require("@atproto/lexicon");
+const lex_cbor_1 = require("@atproto/lex-cbor");
+const lex_data_1 = require("@atproto/lex-data");
 class BlockMap {
     constructor(entries) {
         Object.defineProperty(this, "map", {
@@ -54,9 +18,10 @@ class BlockMap {
         }
     }
     async add(value) {
-        const block = await (0, common_1.dataToCborBlock)((0, lexicon_1.lexToIpld)(value));
-        this.set(block.cid, block.bytes);
-        return block.cid;
+        const bytes = (0, lex_cbor_1.encode)(value);
+        const cid = await (0, lex_data_1.cidForCbor)(bytes);
+        this.set(cid, bytes);
+        return cid;
     }
     set(cid, bytes) {
         this.map.set(cid.toString(), bytes);
@@ -121,7 +86,7 @@ class BlockMap {
             const otherBytes = other.get(cid);
             if (!otherBytes)
                 return false;
-            if (!uint8arrays.equals(bytes, otherBytes)) {
+            if (!(0, lex_data_1.ui8Equals)(bytes, otherBytes)) {
                 return false;
             }
         }
@@ -129,7 +94,7 @@ class BlockMap {
     }
     *keys() {
         for (const key of this.map.keys()) {
-            yield cid_1.CID.parse(key);
+            yield (0, lex_data_1.parseCid)(key);
         }
     }
     *values() {
@@ -137,7 +102,7 @@ class BlockMap {
     }
     *[Symbol.iterator]() {
         for (const [key, value] of this.map) {
-            yield [cid_1.CID.parse(key), value];
+            yield [(0, lex_data_1.parseCid)(key), value];
         }
     }
 }

package/dist/block-map.js.map

@@ -1 +1 @@
-{"version":3,"file":"block-map.js","sourceRoot":"","sources":["../src/block-map.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEzC,0CAAsC;AACtC,yDAA0C;AAC1C,4CAAiD;AACjD,8CAAsD;AAEtD,MAAa,QAAQ;IAGnB,YAAY,OAA0D;QAF9D;;;;mBAA+B,IAAI,GAAG,EAAE;WAAA;QAG9C,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;gBACnC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAe;QACvB,MAAM,KAAK,GAAG,MAAM,IAAA,wBAAe,EAAC,IAAA,mBAAS,EAAC,KAAK,CAAC,CAAC,CAAA;QACrD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QAChC,OAAO,KAAK,CAAC,GAAG,CAAA;IAClB,CAAC;IAED,GAAG,CAAC,GAAQ,EAAE,KAAiB;QAC7B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QACnC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,GAAG,CAAC,GAAQ;QACV,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,GAAQ;QACb,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC/B,OAAO,IAAI,CAAA;IACb,CAAC;IAED,OAAO,CAAC,IAAW;QACjB,MAAM,OAAO,GAAU,EAAE,CAAA;QACzB,MAAM,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAA;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;YACxC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACnB,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;IAED,GAAG,CAAC,GAAQ;QACV,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAA;IAClB,CAAC;IAED,OAAO,CAAC,EAAyC;QAC/C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI;YAAE,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACjD,CAAC;IAED,OAAO;QACL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAClC,CAAC;IAED,IAAI;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,CAAC,KAAe;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACtD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAA;IACtB,CAAC;IAED,IAAI,QAAQ;QACV,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAAE,IAAI,IAAI,KAAK,CAAC,MAAM,CAAA;QACvD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAe;QACpB,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;YAChC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACjC,IAAI,CAAC,UAAU;gBAAE,OAAO,KAAK,CAAA;YAC7B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC3C,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,CAAC,IAAI;QACH,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,SAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IAED,CAAC,MAAM;QACL,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAA;IAC1B,CAAC;IAED,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QAChB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,SAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC/B,CAAC;IACH,CAAC;CACF;AA7GD,4BA6GC;AAED,SAAS,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,CAA6B;IACvD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;AACvB,CAAC;AAOD,kBAAe,QAAQ,CAAA","sourcesContent":["/* eslint-disable import/no-deprecated */\n\nimport { CID } from 'multiformats/cid'\nimport * as uint8arrays from 'uint8arrays'\nimport { dataToCborBlock } from '@atproto/common'\nimport { LexValue, lexToIpld } from '@atproto/lexicon'\n\nexport class BlockMap implements Iterable<[cid: CID, bytes: Uint8Array]> {\n  private map: Map<string, Uint8Array> = new Map()\n\n  constructor(entries?: Iterable<readonly [cid: CID, bytes: Uint8Array]>) {\n    if (entries) {\n      for (const [cid, bytes] of entries) {\n        this.set(cid, bytes)\n      }\n    }\n  }\n\n  async add(value: LexValue): Promise<CID> {\n    const block = await dataToCborBlock(lexToIpld(value))\n    this.set(block.cid, block.bytes)\n    return block.cid\n  }\n\n  set(cid: CID, bytes: Uint8Array): BlockMap {\n    this.map.set(cid.toString(), bytes)\n    return this\n  }\n\n  get(cid: CID): Uint8Array | undefined {\n    return this.map.get(cid.toString())\n  }\n\n  delete(cid: CID): BlockMap {\n    this.map.delete(cid.toString())\n    return this\n  }\n\n  getMany(cids: CID[]): { blocks: BlockMap; missing: CID[] } {\n    const missing: CID[] = []\n    const blocks = new BlockMap()\n    for (const cid of cids) {\n      const got = this.map.get(cid.toString())\n      if (got) {\n        blocks.set(cid, got)\n      } else {\n        missing.push(cid)\n      }\n    }\n    return { blocks, missing }\n  }\n\n  has(cid: CID): boolean {\n    return this.map.has(cid.toString())\n  }\n\n  clear(): void {\n    this.map.clear()\n  }\n\n  forEach(cb: (bytes: Uint8Array, cid: CID) => void): void {\n    for (const [cid, bytes] of this) cb(bytes, cid)\n  }\n\n  entries(): Entry[] {\n    return Array.from(this, toEntry)\n  }\n\n  cids(): CID[] {\n    return Array.from(this.keys())\n  }\n\n  addMap(toAdd: BlockMap): BlockMap {\n    for (const [cid, bytes] of toAdd) this.set(cid, bytes)\n    return this\n  }\n\n  get size(): number {\n    return this.map.size\n  }\n\n  get byteSize(): number {\n    let size = 0\n    for (const bytes of this.values()) size += bytes.length\n    return size\n  }\n\n  equals(other: BlockMap): boolean {\n    if (this.size !== other.size) {\n      return false\n    }\n    for (const [cid, bytes] of this) {\n      const otherBytes = other.get(cid)\n      if (!otherBytes) return false\n      if (!uint8arrays.equals(bytes, otherBytes)) {\n        return false\n      }\n    }\n    return true\n  }\n\n  *keys(): Generator<CID, void, unknown> {\n    for (const key of this.map.keys()) {\n      yield CID.parse(key)\n    }\n  }\n\n  *values(): Generator<Uint8Array, void, unknown> {\n    yield* this.map.values()\n  }\n\n  *[Symbol.iterator](): Generator<[CID, Uint8Array], void, unknown> {\n    for (const [key, value] of this.map) {\n      yield [CID.parse(key), value]\n    }\n  }\n}\n\nfunction toEntry([cid, bytes]: readonly [CID, Uint8Array]): Entry {\n  return { cid, bytes }\n}\n\ntype Entry = {\n  cid: CID\n  bytes: Uint8Array\n}\n\nexport default BlockMap\n"]}
+{"version":3,"file":"block-map.js","sourceRoot":"","sources":["../src/block-map.ts"],"names":[],"mappings":";;;AAAA,gDAA0C;AAC1C,gDAM0B;AAE1B,MAAa,QAAQ;IAGnB,YAAY,OAA0D;QAF9D;;;;mBAA+B,IAAI,GAAG,EAAE;WAAA;QAG9C,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;gBACnC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAe;QACvB,MAAM,KAAK,GAAG,IAAA,iBAAM,EAAC,KAAK,CAAC,CAAA;QAC3B,MAAM,GAAG,GAAG,MAAM,IAAA,qBAAU,EAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACpB,OAAO,GAAG,CAAA;IACZ,CAAC;IAED,GAAG,CAAC,GAAQ,EAAE,KAAiB;QAC7B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,KAAK,CAAC,CAAA;QACnC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,GAAG,CAAC,GAAQ;QACV,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,CAAC,GAAQ;QACb,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;QAC/B,OAAO,IAAI,CAAA;IACb,CAAC;IAED,OAAO,CAAC,IAAW;QACjB,MAAM,OAAO,GAAU,EAAE,CAAA;QACzB,MAAM,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAA;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;YACxC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACnB,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;IAED,GAAG,CAAC,GAAQ;QACV,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAA;IACrC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAA;IAClB,CAAC;IAED,OAAO,CAAC,EAAyC;QAC/C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI;YAAE,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACjD,CAAC;IAED,OAAO;QACL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;IAClC,CAAC;IAED,IAAI;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,CAAC,KAAe;QACpB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACtD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAA;IACtB,CAAC;IAED,IAAI,QAAQ;QACV,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAAE,IAAI,IAAI,KAAK,CAAC,MAAM,CAAA;QACvD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,MAAM,CAAC,KAAe;QACpB,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YAC7B,OAAO,KAAK,CAAA;QACd,CAAC;QACD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;YAChC,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACjC,IAAI,CAAC,UAAU;gBAAE,OAAO,KAAK,CAAA;YAC7B,IAAI,CAAC,IAAA,oBAAS,EAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;gBAClC,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,CAAC,IAAI;QACH,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAClC,MAAM,IAAA,mBAAQ,EAAC,GAAG,CAAC,CAAA;QACrB,CAAC;IACH,CAAC;IAED,CAAC,MAAM;QACL,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAA;IAC1B,CAAC;IAED,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;QAChB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,IAAA,mBAAQ,EAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAA;QAC9B,CAAC;IACH,CAAC;CACF;AA9GD,4BA8GC;AAED,SAAS,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,CAA6B;IACvD,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;AACvB,CAAC;AAOD,kBAAe,QAAQ,CAAA","sourcesContent":["import { encode } from '@atproto/lex-cbor'\nimport {\n  Cid,\n  LexValue,\n  cidForCbor,\n  parseCid,\n  ui8Equals,\n} from '@atproto/lex-data'\n\nexport class BlockMap implements Iterable<[cid: Cid, bytes: Uint8Array]> {\n  private map: Map<string, Uint8Array> = new Map()\n\n  constructor(entries?: Iterable<readonly [cid: Cid, bytes: Uint8Array]>) {\n    if (entries) {\n      for (const [cid, bytes] of entries) {\n        this.set(cid, bytes)\n      }\n    }\n  }\n\n  async add(value: LexValue): Promise<Cid> {\n    const bytes = encode(value)\n    const cid = await cidForCbor(bytes)\n    this.set(cid, bytes)\n    return cid\n  }\n\n  set(cid: Cid, bytes: Uint8Array): BlockMap {\n    this.map.set(cid.toString(), bytes)\n    return this\n  }\n\n  get(cid: Cid): Uint8Array | undefined {\n    return this.map.get(cid.toString())\n  }\n\n  delete(cid: Cid): BlockMap {\n    this.map.delete(cid.toString())\n    return this\n  }\n\n  getMany(cids: Cid[]): { blocks: BlockMap; missing: Cid[] } {\n    const missing: Cid[] = []\n    const blocks = new BlockMap()\n    for (const cid of cids) {\n      const got = this.map.get(cid.toString())\n      if (got) {\n        blocks.set(cid, got)\n      } else {\n        missing.push(cid)\n      }\n    }\n    return { blocks, missing }\n  }\n\n  has(cid: Cid): boolean {\n    return this.map.has(cid.toString())\n  }\n\n  clear(): void {\n    this.map.clear()\n  }\n\n  forEach(cb: (bytes: Uint8Array, cid: Cid) => void): void {\n    for (const [cid, bytes] of this) cb(bytes, cid)\n  }\n\n  entries(): Entry[] {\n    return Array.from(this, toEntry)\n  }\n\n  cids(): Cid[] {\n    return Array.from(this.keys())\n  }\n\n  addMap(toAdd: BlockMap): BlockMap {\n    for (const [cid, bytes] of toAdd) this.set(cid, bytes)\n    return this\n  }\n\n  get size(): number {\n    return this.map.size\n  }\n\n  get byteSize(): number {\n    let size = 0\n    for (const bytes of this.values()) size += bytes.length\n    return size\n  }\n\n  equals(other: BlockMap): boolean {\n    if (this.size !== other.size) {\n      return false\n    }\n    for (const [cid, bytes] of this) {\n      const otherBytes = other.get(cid)\n      if (!otherBytes) return false\n      if (!ui8Equals(bytes, otherBytes)) {\n        return false\n      }\n    }\n    return true\n  }\n\n  *keys(): Generator<Cid, void, unknown> {\n    for (const key of this.map.keys()) {\n      yield parseCid(key)\n    }\n  }\n\n  *values(): Generator<Uint8Array, void, unknown> {\n    yield* this.map.values()\n  }\n\n  *[Symbol.iterator](): Generator<[Cid, Uint8Array], void, unknown> {\n    for (const [key, value] of this.map) {\n      yield [parseCid(key), value]\n    }\n  }\n}\n\nfunction toEntry([cid, bytes]: readonly [Cid, Uint8Array]): Entry {\n  return { cid, bytes }\n}\n\ntype Entry = {\n  cid: Cid\n  bytes: Uint8Array\n}\n\nexport default BlockMap\n"]}

package/dist/car.d.ts

@@ -1,9 +1,9 @@
-import { CID } from 'multiformats/cid';
+import { Cid } from '@atproto/lex-data';
 import { BlockMap } from './block-map';
 import { CarBlock } from './types';
-export declare function writeCarStream(root: CID | null, blocks: AsyncIterable<CarBlock>): AsyncIterable<Uint8Array>;
-export declare const blocksToCarFile: (root: CID | null, blocks: BlockMap) => Promise<Uint8Array>;
-export declare const blocksToCarStream: (root: CID | null, blocks: BlockMap) => AsyncIterable<Uint8Array>;
+export declare function writeCarStream(root: Cid | null, blocks: AsyncIterable<CarBlock>): AsyncIterable<Uint8Array>;
+export declare function blocksToCarFile(root: Cid | null, blocks: BlockMap): Promise<Uint8Array>;
+export declare const blocksToCarStream: (root: Cid | null, blocks: BlockMap) => AsyncIterable<Uint8Array>;
 export type ReadCarOptions = {
     /**
      * When true, does not verify CID-to-content mapping within CAR.
@@ -11,22 +11,22 @@ export type ReadCarOptions = {
     skipCidVerification?: boolean;
 };
 export declare const readCar: (bytes: Uint8Array, opts?: ReadCarOptions) => Promise<{
-    roots: CID[];
+    roots: Cid[];
     blocks: BlockMap;
 }>;
 export declare const readCarWithRoot: (bytes: Uint8Array, opts?: ReadCarOptions) => Promise<{
-    root: CID;
+    root: Cid;
     blocks: BlockMap;
 }>;
 export type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {
     dump: () => Promise<void>;
 };
 export declare const readCarStream: (car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>, opts?: ReadCarOptions) => Promise<{
-    roots: CID[];
+    roots: Cid[];
     blocks: CarBlockIterable;
 }>;
 export declare const readCarReader: (reader: BytesReader, opts?: ReadCarOptions) => Promise<{
-    roots: CID[];
+    roots: Cid[];
     blocks: CarBlockIterable;
 }>;
 export declare function verifyIncomingCarBlocks(car: AsyncIterable<CarBlock>): AsyncGenerator<CarBlock, void, unknown>;

package/dist/car.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAUtC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAElC,wBAAuB,cAAc,CACnC,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC9B,aAAa,CAAC,UAAU,CAAC,CAgB3B;AAED,eAAO,MAAM,eAAe,GAC1B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,OAAO,CAAC,UAAU,CAGpB,CAAA;AAED,eAAO,MAAM,iBAAiB,GAC5B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,aAAa,CAAC,UAAU,CAE1B,CAAA;AAQD,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;CAC9B,CAAA;AAED,eAAO,MAAM,OAAO,GAClB,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAO5C,CAAA;AAED,eAAO,MAAM,eAAe,GAC1B,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAUzC,CAAA;AACD,MAAM,MAAM,gBAAgB,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG;IACvE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1B,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,KAAK,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACrD,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAEA,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,QAAQ,WAAW,EACnB,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAmBA,CAAA;AAoDD,wBAAuB,uBAAuB,CAC5C,GAAG,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC3B,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAKzC;AAuBD,UAAU,WAAW;IACnB,MAAM,EAAE,OAAO,CAAA;IACf,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IAC9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB"}
+{"version":3,"file":"car.d.ts","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,GAAG,EAA4B,MAAM,mBAAmB,CAAA;AACjE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAU,MAAM,SAAS,CAAA;AAG1C,wBAAuB,cAAc,CACnC,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC9B,aAAa,CAAC,UAAU,CAAC,CAgB3B;AAED,wBAAsB,eAAe,CACnC,IAAI,EAAE,GAAG,GAAG,IAAI,EAChB,MAAM,EAAE,QAAQ,GACf,OAAO,CAAC,UAAU,CAAC,CAErB;AAED,eAAO,MAAM,iBAAiB,GAC5B,MAAM,GAAG,GAAG,IAAI,EAChB,QAAQ,QAAQ,KACf,aAAa,CAAC,UAAU,CAE1B,CAAA;AAQD,MAAM,MAAM,cAAc,GAAG;IAC3B;;OAEG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAA;CAC9B,CAAA;AAED,eAAO,MAAM,OAAO,GAClB,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAO5C,CAAA;AAED,eAAO,MAAM,eAAe,GAC1B,OAAO,UAAU,EACjB,OAAO,cAAc,KACpB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAA;CAAE,CAUzC,CAAA;AACD,MAAM,MAAM,gBAAgB,GAAG,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG;IACvE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1B,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,KAAK,QAAQ,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,UAAU,CAAC,EACrD,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAEA,CAAA;AAED,eAAO,MAAM,aAAa,GACxB,QAAQ,WAAW,EACnB,OAAO,cAAc,KACpB,OAAO,CAAC;IACT,KAAK,EAAE,GAAG,EAAE,CAAA;IACZ,MAAM,EAAE,gBAAgB,CAAA;CACzB,CAoBA,CAAA;AAoDD,wBAAuB,uBAAuB,CAC5C,GAAG,EAAE,aAAa,CAAC,QAAQ,CAAC,GAC3B,cAAc,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAOzC;AAuBD,UAAU,WAAW;IACnB,MAAM,EAAE,OAAO,CAAA;IACf,IAAI,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IAC9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACvB"}

package/dist/car.js

@@ -1,5 +1,4 @@
 "use strict";
-/* eslint-disable import/no-deprecated */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -34,15 +33,17 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readCarReader = exports.readCarStream = exports.readCarWithRoot = exports.readCar = exports.blocksToCarStream = exports.blocksToCarFile = void 0;
+exports.readCarReader = exports.readCarStream = exports.readCarWithRoot = exports.readCar = exports.blocksToCarStream = void 0;
 exports.writeCarStream = writeCarStream;
+exports.blocksToCarFile = blocksToCarFile;
 exports.verifyIncomingCarBlocks = verifyIncomingCarBlocks;
 const promises_1 = require("node:timers/promises");
-const cbor = __importStar(require("@ipld/dag-cbor"));
-const ui8 = __importStar(require("uint8arrays"));
 const varint = __importStar(require("varint"));
-const common_1 = require("@atproto/common");
+const cbor = __importStar(require("@atproto/lex-cbor"));
+const lex_data_1 = require("@atproto/lex-data");
 const block_map_1 = require("./block-map");
+const types_1 = require("./types");
+const util_1 = require("./util");
 async function* writeCarStream(root, blocks) {
     const header = new Uint8Array(cbor.encode({
         version: 1,
@@ -56,11 +57,9 @@ async function* writeCarStream(root, blocks) {
         yield block.bytes;
     }
 }
-const blocksToCarFile = (root, blocks) => {
-    const carStream = (0, exports.blocksToCarStream)(root, blocks);
-    return (0, common_1.streamToBuffer)(carStream);
-};
-exports.blocksToCarFile = blocksToCarFile;
+async function blocksToCarFile(root, blocks) {
+    return (0, util_1.concatBytesAsync)((0, exports.blocksToCarStream)(root, blocks));
+}
 const blocksToCarStream = (root, blocks) => {
     return writeCarStream(root, iterateBlocks(blocks));
 };
@@ -103,11 +102,12 @@ const readCarReader = async (reader, opts) => {
         }
         const headerBytes = await reader.read(headerSize);
         const header = cbor.decode(headerBytes);
-        if (!common_1.check.is(header, common_1.schema.carHeader)) {
-            throw new Error('Could not parse CAR header');
+        const result = types_1.schema.carHeader.safeParse(header);
+        if (!result.success) {
+            throw new Error('Could not parse CAR header', { cause: result.error });
         }
         return {
-            roots: header.roots,
+            roots: result.data.roots,
             blocks: readCarBlocksIter(reader, opts),
         };
     }
@@ -146,7 +146,7 @@ async function* readCarBlocksIterGenerator(reader) {
                 break;
             }
             const blockBytes = await reader.read(blockSize);
-            const cid = (0, common_1.parseCidFromBytes)(blockBytes.subarray(0, 36));
+            const cid = (0, lex_data_1.decodeCid)(blockBytes.subarray(0, 36));
             const bytes = blockBytes.subarray(36);
             yield { cid, bytes };
             // yield to the event loop every 25 blocks
@@ -163,7 +163,9 @@ async function* readCarBlocksIterGenerator(reader) {
 }
 async function* verifyIncomingCarBlocks(car) {
     for await (const block of car) {
-        await (0, common_1.verifyCidForBytes)(block.cid, block.bytes);
+        if (!(await (0, lex_data_1.isCidForBytes)(block.cid, block.bytes))) {
+            throw new Error(`Not a valid CID for bytes (${block.cid.toString()})`);
+        }
         yield block;
     }
 }
@@ -185,7 +187,7 @@ const readVarint = async (reader) => {
             done = true;
         }
     }
-    const concatted = ui8.concat(bytes);
+    const concatted = Buffer.concat(bytes);
     return varint.decode(concatted);
 };
 class Ui8Reader {
@@ -219,14 +221,13 @@ class Ui8Reader {
     }
     async close() { }
 }
+/**
+ * This code was optimized for performance. See
+ * {@link https://github.com/bluesky-social/atproto/pull/4729 #4729} for more details
+ * and benchmarks.
+ */
 class BufferedReader {
     constructor(stream) {
-        Object.defineProperty(this, "buffer", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: new Uint8Array()
-        });
         Object.defineProperty(this, "iterator", {
             enumerable: true,
             configurable: true,
@@ -239,36 +240,105 @@ class BufferedReader {
             writable: true,
             value: false
         });
+        /** fifo list of chunks to consume */
+        Object.defineProperty(this, "chunks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
         this.iterator =
             Symbol.asyncIterator in stream
                 ? stream[Symbol.asyncIterator]()
                 : stream[Symbol.iterator]();
     }
-    async read(bytesToRead) {
-        await this.readUntilBuffered(bytesToRead);
-        const value = this.buffer.subarray(0, bytesToRead);
-        this.buffer = this.buffer.subarray(bytesToRead);
-        return value;
+    /** Number of bytes currently buffered and available for reading */
+    get bufferedByteLength() {
+        let total = 0;
+        for (let i = 0; i < this.chunks.length; i++) {
+            total += this.chunks[i].byteLength;
+        }
+        return total;
     }
-    async readUntilBuffered(bytesToRead) {
-        if (this.isDone) {
-            return;
+    /**
+     * @note concurrent reads are **NOT** supported by the current implementation
+     * and would require call to readUntilBuffered to be using a fifo lock for
+     * read()s to be processed in fifo order.
+     */
+    async read(bytesToRead) {
+        const bytesNeeded = bytesToRead - this.bufferedByteLength;
+        if (bytesNeeded > 0 && !this.isDone) {
+            await this.readUntilBuffered(bytesNeeded);
+        }
+        const resultLength = Math.min(bytesToRead, this.bufferedByteLength);
+        if (resultLength <= 0)
+            return new Uint8Array();
+        const firstChunk = this.consumeChunk(resultLength);
+        if (firstChunk.byteLength === resultLength) {
+            // If the data consumed from the first chunk contains all we need, return
+            // it as-is. This allows to avoid any copy operation.
+            return firstChunk;
         }
-        while (this.buffer.length < bytesToRead) {
+        // The first chunk does not have all the data we need. We have to copy
+        // multiple chunks into a larger buffer
+        const result = new Uint8Array(resultLength);
+        let resultWriteIndex = 0;
+        // Copy the first chunk into the result buffer
+        result.set(firstChunk, resultWriteIndex);
+        resultWriteIndex += firstChunk.byteLength;
+        // Copy more chunks as needed (we use do-while because we *know* we need
+        // more than one chunk)
+        do {
+            const missingLength = resultLength - resultWriteIndex;
+            const currentChunk = this.consumeChunk(missingLength);
+            result.set(currentChunk, resultWriteIndex);
+            resultWriteIndex += currentChunk.byteLength;
+        } while (resultWriteIndex < resultLength);
+        return result;
+    }
+    async readUntilBuffered(bytesNeeded) {
+        let bytesRead = 0;
+        while (bytesRead < bytesNeeded) {
             const next = await this.iterator.next();
             if (next.done) {
                 this.isDone = true;
-                return;
+                break;
             }
-            this.buffer = ui8.concat([this.buffer, next.value]);
+            else {
+                this.chunks.push(next.value);
+                bytesRead += next.value.byteLength;
+            }
+        }
+        return bytesRead;
+    }
+    consumeChunk(bytesToConsume) {
+        const firstChunk = this.chunks[0];
+        if (bytesToConsume < firstChunk.byteLength) {
+            // return a sub-view of the data being read and replace the first chunk
+            // with a sub-view that does not contain that data.
+            // @NOTE for some reason, subarray() revealed to be 7-8% slower in NodeJS
+            // benchmarks.
+            // this.chunks[0] = firstChunk.subarray(bytesToConsume)
+            // return firstChunk.subarray(0, bytesToConsume)
+            this.chunks[0] = new Uint8Array(firstChunk.buffer, firstChunk.byteOffset + bytesToConsume, firstChunk.byteLength - bytesToConsume);
+            return new Uint8Array(firstChunk.buffer, firstChunk.byteOffset, bytesToConsume);
+        }
+        else {
+            // First chunk is being read in full, discard it
+            this.chunks.shift();
+            return firstChunk;
         }
     }
     async close() {
-        if (!this.isDone && this.iterator.return) {
-            await this.iterator.return();
+        try {
+            if (!this.isDone && this.iterator.return) {
+                await this.iterator.return();
+            }
+        }
+        finally {
+            this.isDone = true;
+            this.chunks.length = 0;
         }
-        this.isDone = true;
-        this.buffer = new Uint8Array();
     }
 }
 //# sourceMappingURL=car.js.map

package/dist/car.js.map

@@ -1 +1 @@
-{"version":3,"file":"car.js","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":";AAAA,yCAAyC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAiBzC,wCAmBC;AAmJD,0DAOC;AA5LD,mDAAmD;AACnD,qDAAsC;AAEtC,iDAAkC;AAClC,+CAAgC;AAChC,4CAMwB;AACxB,2CAAsC;AAG/B,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,IAAgB,EAChB,MAA+B;IAE/B,MAAM,MAAM,GAAG,IAAI,UAAU,CAC3B,IAAI,CAAC,MAAM,CAAC;QACV,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;KAC1B,CAAC,CACH,CAAA;IACD,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,UAAU,CAClB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CACnE,CAAA;QACD,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAA;QACrB,MAAM,KAAK,CAAC,KAAK,CAAA;IACnB,CAAC;AACH,CAAC;AAEM,MAAM,eAAe,GAAG,CAC7B,IAAgB,EAChB,MAAgB,EACK,EAAE;IACvB,MAAM,SAAS,GAAG,IAAA,yBAAiB,EAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACjD,OAAO,IAAA,uBAAc,EAAC,SAAS,CAAC,CAAA;AAClC,CAAC,CAAA;AANY,QAAA,eAAe,mBAM3B;AAEM,MAAM,iBAAiB,GAAG,CAC/B,IAAgB,EAChB,MAAgB,EACW,EAAE;IAC7B,OAAO,cAAc,CAAC,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;AACpD,CAAC,CAAA;AALY,QAAA,iBAAiB,qBAK7B;AAED,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,MAAgB;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAA;IAC9C,CAAC;AACH,CAAC;AASM,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAiB,EACjB,IAAqB,EACwB,EAAE;IAC/C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,qBAAa,EAAC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,IAAI,oBAAQ,EAAE,CAAA;IAC/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACpC,CAAC,CAAA;AAVY,QAAA,OAAO,WAUnB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,KAAiB,EACjB,IAAqB,EACqB,EAAE;IAC5C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,eAAO,EAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,OAAO;QACL,IAAI;QACJ,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAbY,QAAA,eAAe,mBAa3B;AAKM,MAAM,aAAa,GAAG,KAAK,EAChC,GAAqD,EACrD,IAAqB,EAIpB,EAAE;IACH,OAAO,IAAA,qBAAa,EAAC,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;AACrD,CAAC,CAAA;AARY,QAAA,aAAa,iBAQzB;AAEM,MAAM,aAAa,GAAG,KAAK,EAChC,MAAmB,EACnB,IAAqB,EAIpB,EAAE;IACH,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;QAC3C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACvC,IAAI,CAAC,cAAK,CAAC,EAAE,CAAC,MAAM,EAAE,eAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC;SACxC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AAzBY,QAAA,aAAa,iBAyBzB;AAED,MAAM,iBAAiB,GAAG,CACxB,MAAmB,EACnB,IAAqB,EACH,EAAE;IACpB,IAAI,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAA;IAClD,IAAI,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,SAAS,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;QAC9B,KAAK,CAAC,IAAI;YACR,kFAAkF;YAClF,IAAI,CAAC;gBACH,uEAAuE;gBACvE,MAAM,SAAS,CAAC,MAAM,EAAE,CAAA;YAC1B,CAAC;oBAAS,CAAC;gBACT,mEAAmE;gBACnE,wEAAwE;gBACxE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACtB,CAAC;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,KAAK,SAAS,CAAC,CAAC,0BAA0B,CACxC,MAAmB;IAEnB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;YAC1C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,MAAK;YACP,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,IAAA,0BAAiB,EAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YACzD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;YAEpB,0CAA0C;YAC1C,qFAAqF;YACrF,MAAM,EAAE,CAAA;YACR,IAAI,MAAM,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAA,uBAAY,GAAE,CAAA;YACtB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,GAA4B;IAE5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,MAAM,IAAA,0BAAiB,EAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QAC/C,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,GAAG,KAAK,EAAE,MAAmB,EAA0B,EAAE;IACvE,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,OAAO,CAAC,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjC,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YAC3C,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;YAClB,IAAI,GAAG,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACnC,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACjC,CAAC,CAAA;AAQD,MAAM,SAAS;IAIb,YAAmB,KAAiB;QAAxB;;;;mBAAO,KAAK;WAAY;QAHpC;;;;mBAAM,CAAC;WAAA;QACP;;;;mBAAS,KAAK;WAAA;IAEyB,CAAC;IAExC,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,CAAA;QACnE,IAAI,CAAC,GAAG,IAAI,WAAW,CAAA;QACvB,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACpB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,KAAK,CAAC,KAAK,KAAmB,CAAC;CAChC;AAED,MAAM,cAAc;IAKlB,YAAY,MAAwD;QAJpE;;;;mBAAqB,IAAI,UAAU,EAAE;WAAA;QACrC;;;;;WAA0D;QAC1D;;;;mBAAS,KAAK;WAAA;QAGZ,IAAI,CAAC,QAAQ;YACX,MAAM,CAAC,aAAa,IAAI,MAAM;gBAC5B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;gBAChC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAA;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QAClD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAA;QAC/C,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACjD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAM;QACR,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;YACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,OAAM;YACR,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QACrD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC9B,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,UAAU,EAAE,CAAA;IAChC,CAAC;CACF","sourcesContent":["/* eslint-disable import/no-deprecated */\n\nimport { setImmediate } from 'node:timers/promises'\nimport * as cbor from '@ipld/dag-cbor'\nimport { CID } from 'multiformats/cid'\nimport * as ui8 from 'uint8arrays'\nimport * as varint from 'varint'\nimport {\n  check,\n  parseCidFromBytes,\n  schema,\n  streamToBuffer,\n  verifyCidForBytes,\n} from '@atproto/common'\nimport { BlockMap } from './block-map'\nimport { CarBlock } from './types'\n\nexport async function* writeCarStream(\n  root: CID | null,\n  blocks: AsyncIterable<CarBlock>,\n): AsyncIterable<Uint8Array> {\n  const header = new Uint8Array(\n    cbor.encode({\n      version: 1,\n      roots: root ? [root] : [],\n    }),\n  )\n  yield new Uint8Array(varint.encode(header.byteLength))\n  yield header\n  for await (const block of blocks) {\n    yield new Uint8Array(\n      varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength),\n    )\n    yield block.cid.bytes\n    yield block.bytes\n  }\n}\n\nexport const blocksToCarFile = (\n  root: CID | null,\n  blocks: BlockMap,\n): Promise<Uint8Array> => {\n  const carStream = blocksToCarStream(root, blocks)\n  return streamToBuffer(carStream)\n}\n\nexport const blocksToCarStream = (\n  root: CID | null,\n  blocks: BlockMap,\n): AsyncIterable<Uint8Array> => {\n  return writeCarStream(root, iterateBlocks(blocks))\n}\n\nasync function* iterateBlocks(blocks: BlockMap) {\n  for (const entry of blocks.entries()) {\n    yield { cid: entry.cid, bytes: entry.bytes }\n  }\n}\n\nexport type ReadCarOptions = {\n  /**\n   * When true, does not verify CID-to-content mapping within CAR.\n   */\n  skipCidVerification?: boolean\n}\n\nexport const readCar = async (\n  bytes: Uint8Array,\n  opts?: ReadCarOptions,\n): Promise<{ roots: CID[]; blocks: BlockMap }> => {\n  const { roots, blocks } = await readCarReader(new Ui8Reader(bytes), opts)\n  const blockMap = new BlockMap()\n  for await (const block of blocks) {\n    blockMap.set(block.cid, block.bytes)\n  }\n  return { roots, blocks: blockMap }\n}\n\nexport const readCarWithRoot = async (\n  bytes: Uint8Array,\n  opts?: ReadCarOptions,\n): Promise<{ root: CID; blocks: BlockMap }> => {\n  const { roots, blocks } = await readCar(bytes, opts)\n  if (roots.length !== 1) {\n    throw new Error(`Expected one root, got ${roots.length}`)\n  }\n  const root = roots[0]\n  return {\n    root,\n    blocks,\n  }\n}\nexport type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {\n  dump: () => Promise<void>\n}\n\nexport const readCarStream = async (\n  car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,\n  opts?: ReadCarOptions,\n): Promise<{\n  roots: CID[]\n  blocks: CarBlockIterable\n}> => {\n  return readCarReader(new BufferedReader(car), opts)\n}\n\nexport const readCarReader = async (\n  reader: BytesReader,\n  opts?: ReadCarOptions,\n): Promise<{\n  roots: CID[]\n  blocks: CarBlockIterable\n}> => {\n  try {\n    const headerSize = await readVarint(reader)\n    if (headerSize === null) {\n      throw new Error('Could not parse CAR header')\n    }\n    const headerBytes = await reader.read(headerSize)\n    const header = cbor.decode(headerBytes)\n    if (!check.is(header, schema.carHeader)) {\n      throw new Error('Could not parse CAR header')\n    }\n    return {\n      roots: header.roots,\n      blocks: readCarBlocksIter(reader, opts),\n    }\n  } catch (err) {\n    await reader.close()\n    throw err\n  }\n}\n\nconst readCarBlocksIter = (\n  reader: BytesReader,\n  opts?: ReadCarOptions,\n): CarBlockIterable => {\n  let generator = readCarBlocksIterGenerator(reader)\n  if (!opts?.skipCidVerification) {\n    generator = verifyIncomingCarBlocks(generator)\n  }\n  return Object.assign(generator, {\n    async dump() {\n      // try/finally to ensure that reader.close is called even if blocks.return throws.\n      try {\n        // Prevent the iterator from being started after this method is called.\n        await generator.return()\n      } finally {\n        // @NOTE the \"finally\" block of the async generator won't be called\n        // if the iteration was never started so we need to manually close here.\n        await reader.close()\n      }\n    },\n  })\n}\n\nasync function* readCarBlocksIterGenerator(\n  reader: BytesReader,\n): AsyncGenerator<CarBlock, void, unknown> {\n  let blocks = 0\n  try {\n    while (!reader.isDone) {\n      const blockSize = await readVarint(reader)\n      if (blockSize === null) {\n        break\n      }\n      const blockBytes = await reader.read(blockSize)\n      const cid = parseCidFromBytes(blockBytes.subarray(0, 36))\n      const bytes = blockBytes.subarray(36)\n      yield { cid, bytes }\n\n      // yield to the event loop every 25 blocks\n      // in the case the incoming CAR is synchronous, this can end up jamming up the thread\n      blocks++\n      if (blocks % 25 === 0) {\n        await setImmediate()\n      }\n    }\n  } finally {\n    await reader.close()\n  }\n}\n\nexport async function* verifyIncomingCarBlocks(\n  car: AsyncIterable<CarBlock>,\n): AsyncGenerator<CarBlock, void, unknown> {\n  for await (const block of car) {\n    await verifyCidForBytes(block.cid, block.bytes)\n    yield block\n  }\n}\n\nconst readVarint = async (reader: BytesReader): Promise<number | null> => {\n  let done = false\n  const bytes: Uint8Array[] = []\n  while (!done) {\n    const byte = await reader.read(1)\n    if (byte.byteLength === 0) {\n      if (bytes.length > 0) {\n        throw new Error('could not parse varint')\n      } else {\n        return null\n      }\n    }\n    bytes.push(byte)\n    if (byte[0] < 128) {\n      done = true\n    }\n  }\n  const concatted = ui8.concat(bytes)\n  return varint.decode(concatted)\n}\n\ninterface BytesReader {\n  isDone: boolean\n  read(bytesToRead: number): Promise<Uint8Array>\n  close(): Promise<void>\n}\n\nclass Ui8Reader implements BytesReader {\n  idx = 0\n  isDone = false\n\n  constructor(public bytes: Uint8Array) {}\n\n  async read(bytesToRead: number): Promise<Uint8Array> {\n    const value = this.bytes.subarray(this.idx, this.idx + bytesToRead)\n    this.idx += bytesToRead\n    if (this.idx >= this.bytes.length) {\n      this.isDone = true\n    }\n    return value\n  }\n\n  async close(): Promise<void> {}\n}\n\nclass BufferedReader implements BytesReader {\n  buffer: Uint8Array = new Uint8Array()\n  iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>\n  isDone = false\n\n  constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {\n    this.iterator =\n      Symbol.asyncIterator in stream\n        ? stream[Symbol.asyncIterator]()\n        : stream[Symbol.iterator]()\n  }\n\n  async read(bytesToRead: number): Promise<Uint8Array> {\n    await this.readUntilBuffered(bytesToRead)\n    const value = this.buffer.subarray(0, bytesToRead)\n    this.buffer = this.buffer.subarray(bytesToRead)\n    return value\n  }\n\n  private async readUntilBuffered(bytesToRead: number) {\n    if (this.isDone) {\n      return\n    }\n    while (this.buffer.length < bytesToRead) {\n      const next = await this.iterator.next()\n      if (next.done) {\n        this.isDone = true\n        return\n      }\n      this.buffer = ui8.concat([this.buffer, next.value])\n    }\n  }\n\n  async close(): Promise<void> {\n    if (!this.isDone && this.iterator.return) {\n      await this.iterator.return()\n    }\n    this.isDone = true\n    this.buffer = new Uint8Array()\n  }\n}\n"]}
+{"version":3,"file":"car.js","sourceRoot":"","sources":["../src/car.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQA,wCAmBC;AAED,0CAKC;AA4ID,0DASC;AAvLD,mDAAmD;AACnD,+CAAgC;AAChC,wDAAyC;AACzC,gDAAiE;AACjE,2CAAsC;AACtC,mCAA0C;AAC1C,iCAAyC;AAElC,KAAK,SAAS,CAAC,CAAC,cAAc,CACnC,IAAgB,EAChB,MAA+B;IAE/B,MAAM,MAAM,GAAG,IAAI,UAAU,CAC3B,IAAI,CAAC,MAAM,CAAC;QACV,OAAO,EAAE,CAAC;QACV,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;KAC1B,CAAC,CACH,CAAA;IACD,MAAM,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,CAAA;IACZ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,MAAM,IAAI,UAAU,CAClB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CACnE,CAAA;QACD,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,CAAA;QACrB,MAAM,KAAK,CAAC,KAAK,CAAA;IACnB,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CACnC,IAAgB,EAChB,MAAgB;IAEhB,OAAO,IAAA,uBAAgB,EAAC,IAAA,yBAAiB,EAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;AAC1D,CAAC;AAEM,MAAM,iBAAiB,GAAG,CAC/B,IAAgB,EAChB,MAAgB,EACW,EAAE;IAC7B,OAAO,cAAc,CAAC,IAAI,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAA;AACpD,CAAC,CAAA;AALY,QAAA,iBAAiB,qBAK7B;AAED,KAAK,SAAS,CAAC,CAAC,aAAa,CAAC,MAAgB;IAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAA;IAC9C,CAAC;AACH,CAAC;AASM,MAAM,OAAO,GAAG,KAAK,EAC1B,KAAiB,EACjB,IAAqB,EACwB,EAAE;IAC/C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,qBAAa,EAAC,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,QAAQ,GAAG,IAAI,oBAAQ,EAAE,CAAA;IAC/B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;IACtC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACpC,CAAC,CAAA;AAVY,QAAA,OAAO,WAUnB;AAEM,MAAM,eAAe,GAAG,KAAK,EAClC,KAAiB,EACjB,IAAqB,EACqB,EAAE;IAC5C,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,IAAA,eAAO,EAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,0BAA0B,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3D,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACrB,OAAO;QACL,IAAI;QACJ,MAAM;KACP,CAAA;AACH,CAAC,CAAA;AAbY,QAAA,eAAe,mBAa3B;AAKM,MAAM,aAAa,GAAG,KAAK,EAChC,GAAqD,EACrD,IAAqB,EAIpB,EAAE;IACH,OAAO,IAAA,qBAAa,EAAC,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;AACrD,CAAC,CAAA;AARY,QAAA,aAAa,iBAQzB;AAEM,MAAM,aAAa,GAAG,KAAK,EAChC,MAAmB,EACnB,IAAqB,EAIpB,EAAE;IACH,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;QAC3C,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QACvC,MAAM,MAAM,GAAG,cAAM,CAAC,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;QACjD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;QACxE,CAAC;QACD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK;YACxB,MAAM,EAAE,iBAAiB,CAAC,MAAM,EAAE,IAAI,CAAC;SACxC,CAAA;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,GAAG,CAAA;IACX,CAAC;AACH,CAAC,CAAA;AA1BY,QAAA,aAAa,iBA0BzB;AAED,MAAM,iBAAiB,GAAG,CACxB,MAAmB,EACnB,IAAqB,EACH,EAAE;IACpB,IAAI,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAA;IAClD,IAAI,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;QAC/B,SAAS,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAA;IAChD,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;QAC9B,KAAK,CAAC,IAAI;YACR,kFAAkF;YAClF,IAAI,CAAC;gBACH,uEAAuE;gBACvE,MAAM,SAAS,CAAC,MAAM,EAAE,CAAA;YAC1B,CAAC;oBAAS,CAAC;gBACT,mEAAmE;gBACnE,wEAAwE;gBACxE,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACtB,CAAC;QACH,CAAC;KACF,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,KAAK,SAAS,CAAC,CAAC,0BAA0B,CACxC,MAAmB;IAEnB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAA;YAC1C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;gBACvB,MAAK;YACP,CAAC;YACD,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,IAAA,oBAAS,EAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;YACrC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAA;YAEpB,0CAA0C;YAC1C,qFAAqF;YACrF,MAAM,EAAE,CAAA;YACR,IAAI,MAAM,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAA,uBAAY,GAAE,CAAA;YACtB,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC;AACH,CAAC;AAEM,KAAK,SAAS,CAAC,CAAC,uBAAuB,CAC5C,GAA4B;IAE5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,MAAM,IAAA,wBAAa,EAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;QACxE,CAAC;QACD,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,GAAG,KAAK,EAAE,MAAmB,EAA0B,EAAE;IACvE,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,OAAO,CAAC,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjC,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAA;YAC3C,CAAC;iBAAM,CAAC;gBACN,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChB,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;YAClB,IAAI,GAAG,IAAI,CAAA;QACb,CAAC;IACH,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACtC,OAAO,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;AACjC,CAAC,CAAA;AAQD,MAAM,SAAS;IAIb,YAAmB,KAAiB;QAAxB;;;;mBAAO,KAAK;WAAY;QAHpC;;;;mBAAM,CAAC;WAAA;QACP;;;;mBAAS,KAAK;WAAA;IAEyB,CAAC;IAExC,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,CAAA;QACnE,IAAI,CAAC,GAAG,IAAI,WAAW,CAAA;QACvB,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;QACpB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED,KAAK,CAAC,KAAK,KAAmB,CAAC;CAChC;AAED;;;;GAIG;AACH,MAAM,cAAc;IAOlB,YAAY,MAAwD;QANpE;;;;;WAA0D;QAC1D;;;;mBAAS,KAAK;WAAA;QAEd,qCAAqC;QAC7B;;;;mBAAuB,EAAE;WAAA;QAG/B,IAAI,CAAC,QAAQ;YACX,MAAM,CAAC,aAAa,IAAI,MAAM;gBAC5B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;gBAChC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAA;IACjC,CAAC;IAED,mEAAmE;IACnE,IAAI,kBAAkB;QACpB,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAA;QACpC,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI,CAAC,WAAmB;QAC5B,MAAM,WAAW,GAAG,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAA;QACzD,IAAI,WAAW,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAA;QAC3C,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAA;QACnE,IAAI,YAAY,IAAI,CAAC;YAAE,OAAO,IAAI,UAAU,EAAE,CAAA;QAE9C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;QAClD,IAAI,UAAU,CAAC,UAAU,KAAK,YAAY,EAAE,CAAC;YAC3C,yEAAyE;YACzE,qDAAqD;YACrD,OAAO,UAAU,CAAA;QACnB,CAAC;QAED,sEAAsE;QACtE,uCAAuC;QACvC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,YAAY,CAAC,CAAA;QAC3C,IAAI,gBAAgB,GAAG,CAAC,CAAA;QAExB,8CAA8C;QAC9C,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAA;QACxC,gBAAgB,IAAI,UAAU,CAAC,UAAU,CAAA;QAEzC,wEAAwE;QACxE,uBAAuB;QACvB,GAAG,CAAC;YACF,MAAM,aAAa,GAAG,YAAY,GAAG,gBAAgB,CAAA;YACrD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,CAAA;YAErD,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAA;YAC1C,gBAAgB,IAAI,YAAY,CAAC,UAAU,CAAA;QAC7C,CAAC,QAAQ,gBAAgB,GAAG,YAAY,EAAC;QAEzC,OAAO,MAAM,CAAA;IACf,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,WAAmB;QACjD,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,OAAO,SAAS,GAAG,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,MAAK;YACP,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC5B,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAA;YACpC,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,YAAY,CAAC,cAAsB;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAA;QAClC,IAAI,cAAc,GAAG,UAAU,CAAC,UAAU,EAAE,CAAC;YAC3C,uEAAuE;YACvE,mDAAmD;YAEnD,yEAAyE;YACzE,cAAc;YAEd,uDAAuD;YACvD,gDAAgD;YAEhD,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,UAAU,CAC7B,UAAU,CAAC,MAAM,EACjB,UAAU,CAAC,UAAU,GAAG,cAAc,EACtC,UAAU,CAAC,UAAU,GAAG,cAAc,CACvC,CAAA;YACD,OAAO,IAAI,UAAU,CACnB,UAAU,CAAC,MAAM,EACjB,UAAU,CAAC,UAAU,EACrB,cAAc,CACf,CAAA;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;YACnB,OAAO,UAAU,CAAA;QACnB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACzC,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAA;YAC9B,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;QACxB,CAAC;IACH,CAAC;CACF","sourcesContent":["import { setImmediate } from 'node:timers/promises'\nimport * as varint from 'varint'\nimport * as cbor from '@atproto/lex-cbor'\nimport { Cid, decodeCid, isCidForBytes } from '@atproto/lex-data'\nimport { BlockMap } from './block-map'\nimport { CarBlock, schema } from './types'\nimport { concatBytesAsync } from './util'\n\nexport async function* writeCarStream(\n  root: Cid | null,\n  blocks: AsyncIterable<CarBlock>,\n): AsyncIterable<Uint8Array> {\n  const header = new Uint8Array(\n    cbor.encode({\n      version: 1,\n      roots: root ? [root] : [],\n    }),\n  )\n  yield new Uint8Array(varint.encode(header.byteLength))\n  yield header\n  for await (const block of blocks) {\n    yield new Uint8Array(\n      varint.encode(block.cid.bytes.byteLength + block.bytes.byteLength),\n    )\n    yield block.cid.bytes\n    yield block.bytes\n  }\n}\n\nexport async function blocksToCarFile(\n  root: Cid | null,\n  blocks: BlockMap,\n): Promise<Uint8Array> {\n  return concatBytesAsync(blocksToCarStream(root, blocks))\n}\n\nexport const blocksToCarStream = (\n  root: Cid | null,\n  blocks: BlockMap,\n): AsyncIterable<Uint8Array> => {\n  return writeCarStream(root, iterateBlocks(blocks))\n}\n\nasync function* iterateBlocks(blocks: BlockMap) {\n  for (const entry of blocks.entries()) {\n    yield { cid: entry.cid, bytes: entry.bytes }\n  }\n}\n\nexport type ReadCarOptions = {\n  /**\n   * When true, does not verify CID-to-content mapping within CAR.\n   */\n  skipCidVerification?: boolean\n}\n\nexport const readCar = async (\n  bytes: Uint8Array,\n  opts?: ReadCarOptions,\n): Promise<{ roots: Cid[]; blocks: BlockMap }> => {\n  const { roots, blocks } = await readCarReader(new Ui8Reader(bytes), opts)\n  const blockMap = new BlockMap()\n  for await (const block of blocks) {\n    blockMap.set(block.cid, block.bytes)\n  }\n  return { roots, blocks: blockMap }\n}\n\nexport const readCarWithRoot = async (\n  bytes: Uint8Array,\n  opts?: ReadCarOptions,\n): Promise<{ root: Cid; blocks: BlockMap }> => {\n  const { roots, blocks } = await readCar(bytes, opts)\n  if (roots.length !== 1) {\n    throw new Error(`Expected one root, got ${roots.length}`)\n  }\n  const root = roots[0]\n  return {\n    root,\n    blocks,\n  }\n}\nexport type CarBlockIterable = AsyncGenerator<CarBlock, void, unknown> & {\n  dump: () => Promise<void>\n}\n\nexport const readCarStream = async (\n  car: Iterable<Uint8Array> | AsyncIterable<Uint8Array>,\n  opts?: ReadCarOptions,\n): Promise<{\n  roots: Cid[]\n  blocks: CarBlockIterable\n}> => {\n  return readCarReader(new BufferedReader(car), opts)\n}\n\nexport const readCarReader = async (\n  reader: BytesReader,\n  opts?: ReadCarOptions,\n): Promise<{\n  roots: Cid[]\n  blocks: CarBlockIterable\n}> => {\n  try {\n    const headerSize = await readVarint(reader)\n    if (headerSize === null) {\n      throw new Error('Could not parse CAR header')\n    }\n    const headerBytes = await reader.read(headerSize)\n    const header = cbor.decode(headerBytes)\n    const result = schema.carHeader.safeParse(header)\n    if (!result.success) {\n      throw new Error('Could not parse CAR header', { cause: result.error })\n    }\n    return {\n      roots: result.data.roots,\n      blocks: readCarBlocksIter(reader, opts),\n    }\n  } catch (err) {\n    await reader.close()\n    throw err\n  }\n}\n\nconst readCarBlocksIter = (\n  reader: BytesReader,\n  opts?: ReadCarOptions,\n): CarBlockIterable => {\n  let generator = readCarBlocksIterGenerator(reader)\n  if (!opts?.skipCidVerification) {\n    generator = verifyIncomingCarBlocks(generator)\n  }\n  return Object.assign(generator, {\n    async dump() {\n      // try/finally to ensure that reader.close is called even if blocks.return throws.\n      try {\n        // Prevent the iterator from being started after this method is called.\n        await generator.return()\n      } finally {\n        // @NOTE the \"finally\" block of the async generator won't be called\n        // if the iteration was never started so we need to manually close here.\n        await reader.close()\n      }\n    },\n  })\n}\n\nasync function* readCarBlocksIterGenerator(\n  reader: BytesReader,\n): AsyncGenerator<CarBlock, void, unknown> {\n  let blocks = 0\n  try {\n    while (!reader.isDone) {\n      const blockSize = await readVarint(reader)\n      if (blockSize === null) {\n        break\n      }\n      const blockBytes = await reader.read(blockSize)\n      const cid = decodeCid(blockBytes.subarray(0, 36))\n      const bytes = blockBytes.subarray(36)\n      yield { cid, bytes }\n\n      // yield to the event loop every 25 blocks\n      // in the case the incoming CAR is synchronous, this can end up jamming up the thread\n      blocks++\n      if (blocks % 25 === 0) {\n        await setImmediate()\n      }\n    }\n  } finally {\n    await reader.close()\n  }\n}\n\nexport async function* verifyIncomingCarBlocks(\n  car: AsyncIterable<CarBlock>,\n): AsyncGenerator<CarBlock, void, unknown> {\n  for await (const block of car) {\n    if (!(await isCidForBytes(block.cid, block.bytes))) {\n      throw new Error(`Not a valid CID for bytes (${block.cid.toString()})`)\n    }\n    yield block\n  }\n}\n\nconst readVarint = async (reader: BytesReader): Promise<number | null> => {\n  let done = false\n  const bytes: Uint8Array[] = []\n  while (!done) {\n    const byte = await reader.read(1)\n    if (byte.byteLength === 0) {\n      if (bytes.length > 0) {\n        throw new Error('could not parse varint')\n      } else {\n        return null\n      }\n    }\n    bytes.push(byte)\n    if (byte[0] < 128) {\n      done = true\n    }\n  }\n  const concatted = Buffer.concat(bytes)\n  return varint.decode(concatted)\n}\n\ninterface BytesReader {\n  isDone: boolean\n  read(bytesToRead: number): Promise<Uint8Array>\n  close(): Promise<void>\n}\n\nclass Ui8Reader implements BytesReader {\n  idx = 0\n  isDone = false\n\n  constructor(public bytes: Uint8Array) {}\n\n  async read(bytesToRead: number): Promise<Uint8Array> {\n    const value = this.bytes.subarray(this.idx, this.idx + bytesToRead)\n    this.idx += bytesToRead\n    if (this.idx >= this.bytes.length) {\n      this.isDone = true\n    }\n    return value\n  }\n\n  async close(): Promise<void> {}\n}\n\n/**\n * This code was optimized for performance. See\n * {@link https://github.com/bluesky-social/atproto/pull/4729 #4729} for more details\n * and benchmarks.\n */\nclass BufferedReader implements BytesReader {\n  iterator: Iterator<Uint8Array> | AsyncIterator<Uint8Array>\n  isDone = false\n\n  /** fifo list of chunks to consume */\n  private chunks: Uint8Array[] = []\n\n  constructor(stream: Iterable<Uint8Array> | AsyncIterable<Uint8Array>) {\n    this.iterator =\n      Symbol.asyncIterator in stream\n        ? stream[Symbol.asyncIterator]()\n        : stream[Symbol.iterator]()\n  }\n\n  /** Number of bytes currently buffered and available for reading */\n  get bufferedByteLength() {\n    let total = 0\n    for (let i = 0; i < this.chunks.length; i++) {\n      total += this.chunks[i].byteLength\n    }\n    return total\n  }\n\n  /**\n   * @note concurrent reads are **NOT** supported by the current implementation\n   * and would require call to readUntilBuffered to be using a fifo lock for\n   * read()s to be processed in fifo order.\n   */\n  async read(bytesToRead: number): Promise<Uint8Array> {\n    const bytesNeeded = bytesToRead - this.bufferedByteLength\n    if (bytesNeeded > 0 && !this.isDone) {\n      await this.readUntilBuffered(bytesNeeded)\n    }\n\n    const resultLength = Math.min(bytesToRead, this.bufferedByteLength)\n    if (resultLength <= 0) return new Uint8Array()\n\n    const firstChunk = this.consumeChunk(resultLength)\n    if (firstChunk.byteLength === resultLength) {\n      // If the data consumed from the first chunk contains all we need, return\n      // it as-is. This allows to avoid any copy operation.\n      return firstChunk\n    }\n\n    // The first chunk does not have all the data we need. We have to copy\n    // multiple chunks into a larger buffer\n    const result = new Uint8Array(resultLength)\n    let resultWriteIndex = 0\n\n    // Copy the first chunk into the result buffer\n    result.set(firstChunk, resultWriteIndex)\n    resultWriteIndex += firstChunk.byteLength\n\n    // Copy more chunks as needed (we use do-while because we *know* we need\n    // more than one chunk)\n    do {\n      const missingLength = resultLength - resultWriteIndex\n      const currentChunk = this.consumeChunk(missingLength)\n\n      result.set(currentChunk, resultWriteIndex)\n      resultWriteIndex += currentChunk.byteLength\n    } while (resultWriteIndex < resultLength)\n\n    return result\n  }\n\n  private async readUntilBuffered(bytesNeeded: number) {\n    let bytesRead = 0\n    while (bytesRead < bytesNeeded) {\n      const next = await this.iterator.next()\n      if (next.done) {\n        this.isDone = true\n        break\n      } else {\n        this.chunks.push(next.value)\n        bytesRead += next.value.byteLength\n      }\n    }\n    return bytesRead\n  }\n\n  private consumeChunk(bytesToConsume: number) {\n    const firstChunk = this.chunks[0]!\n    if (bytesToConsume < firstChunk.byteLength) {\n      // return a sub-view of the data being read and replace the first chunk\n      // with a sub-view that does not contain that data.\n\n      // @NOTE for some reason, subarray() revealed to be 7-8% slower in NodeJS\n      // benchmarks.\n\n      // this.chunks[0] = firstChunk.subarray(bytesToConsume)\n      // return firstChunk.subarray(0, bytesToConsume)\n\n      this.chunks[0] = new Uint8Array(\n        firstChunk.buffer,\n        firstChunk.byteOffset + bytesToConsume,\n        firstChunk.byteLength - bytesToConsume,\n      )\n      return new Uint8Array(\n        firstChunk.buffer,\n        firstChunk.byteOffset,\n        bytesToConsume,\n      )\n    } else {\n      // First chunk is being read in full, discard it\n      this.chunks.shift()\n      return firstChunk\n    }\n  }\n\n  async close(): Promise<void> {\n    try {\n      if (!this.isDone && this.iterator.return) {\n        await this.iterator.return()\n      }\n    } finally {\n      this.isDone = true\n      this.chunks.length = 0\n    }\n  }\n}\n"]}

package/dist/cid-set.d.ts

@@ -1,15 +1,16 @@
-import { CID } from 'multiformats';
-export declare class CidSet {
+import { Cid } from '@atproto/lex-data';
+export declare class CidSet implements Iterable<Cid> {
     private set;
-    constructor(arr?: CID[]);
-    add(cid: CID): CidSet;
+    constructor(arr?: Cid[]);
+    add(cid: Cid): CidSet;
     addSet(toMerge: CidSet): CidSet;
     subtractSet(toSubtract: CidSet): CidSet;
-    delete(cid: CID): this;
-    has(cid: CID): boolean;
+    delete(cid: Cid): this;
+    has(cid: Cid): boolean;
     size(): number;
     clear(): CidSet;
-    toList(): CID[];
+    toList(): Cid[];
+    [Symbol.iterator](): Generator<Cid, void, unknown>;
 }
 export default CidSet;
 //# sourceMappingURL=cid-set.d.ts.map

package/dist/cid-set.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cid-set.d.ts","sourceRoot":"","sources":["../src/cid-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAA;AAElC,qBAAa,MAAM;IACjB,OAAO,CAAC,GAAG,CAAa;gBAEZ,GAAG,GAAE,GAAG,EAAO;IAK3B,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM;IAKrB,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAK/B,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAKvC,MAAM,CAAC,GAAG,EAAE,GAAG;IAKf,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO;IAItB,IAAI,IAAI,MAAM;IAId,KAAK,IAAI,MAAM;IAKf,MAAM,IAAI,GAAG,EAAE;CAGhB;AAED,eAAe,MAAM,CAAA"}
+{"version":3,"file":"cid-set.d.ts","sourceRoot":"","sources":["../src/cid-set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAY,MAAM,mBAAmB,CAAA;AAEjD,qBAAa,MAAO,YAAW,QAAQ,CAAC,GAAG,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAa;gBAEZ,GAAG,GAAE,GAAG,EAAO;IAK3B,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM;IAKrB,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAK/B,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAKvC,MAAM,CAAC,GAAG,EAAE,GAAG;IAKf,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO;IAItB,IAAI,IAAI,MAAM;IAId,KAAK,IAAI,MAAM;IAKf,MAAM,IAAI,GAAG,EAAE;IAId,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC;CAKpD;AAED,eAAe,MAAM,CAAA"}