@atcute/car 3.0.5 → 3.1.1

package/dist/v4/repo-reader/stream-repo-reader.js

@@ -0,0 +1,143 @@
+import Queue from 'yocto-queue';
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { decodeUtf8From } from '@atcute/uint8array';
+import * as CarReader from '../car-reader/index.js';
+import { assert } from '../utils.js';
+import { isCommit, isMstNode } from './mst.js';
+import { RepoEntry } from './types.js';
+export const repoEntryTransform = () => {
+    const transform = new TransformStream();
+    let repo;
+    return {
+        readable: new ReadableStream({
+            async start(controller) {
+                repo = fromStream(transform.readable);
+                try {
+                    for await (const entry of repo) {
+                        controller.enqueue(entry);
+                    }
+                    await repo.dispose();
+                    controller.close();
+                }
+                catch (err) {
+                    controller.error(err);
+                }
+            },
+            async cancel() {
+                if (repo !== undefined) {
+                    await repo.dispose();
+                }
+            },
+        }),
+        writable: transform.writable,
+    };
+};
+export const fromStream = (stream) => {
+    let missingBlocks = [];
+    return {
+        get missingBlocks() {
+            return missingBlocks;
+        },
+        async dispose() {
+            // does nothing for now
+        },
+        [Symbol.asyncDispose]() {
+            return this.dispose();
+        },
+        async *[Symbol.asyncIterator]() {
+            // await using car = CarReader.fromStream(stream);
+            const car = CarReader.fromStream(stream);
+            try {
+                const pending = new Map();
+                const strays = new Map();
+                const queue = new Queue();
+                const request = (cid, meta) => {
+                    const entry = strays.get(cid);
+                    if (entry !== undefined) {
+                        strays.delete(cid);
+                        queue.enqueue({ c: cid, e: entry, m: meta });
+                    }
+                    else {
+                        pending.set(cid, meta);
+                    }
+                };
+                {
+                    const roots = await car.roots();
+                    assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+                    const rootCid = roots[0].$link;
+                    request(rootCid, { t: 0 });
+                }
+                for await (const entry of car) {
+                    const cid = CID.toString(entry.cid);
+                    {
+                        const meta = pending.get(cid);
+                        if (meta !== undefined) {
+                            pending.delete(cid);
+                            queue.enqueue({ c: cid, e: entry, m: meta });
+                        }
+                        else {
+                            strays.set(cid, entry);
+                        }
+                    }
+                    let task;
+                    while ((task = queue.dequeue())) {
+                        const { c: cid, e: entry, m: meta } = task;
+                        switch (meta.t) {
+                            case 0: {
+                                const commit = CBOR.decode(entry.bytes);
+                                assert(isCommit(commit), `expected commit block; cid=${cid}`);
+                                request(commit.data.$link, { t: 1 });
+                                break;
+                            }
+                            case 1: {
+                                const node = CBOR.decode(entry.bytes);
+                                assert(isMstNode(node), `expected mst node block; cid=${cid}`);
+                                const entries = node.e;
+                                const left = node.l;
+                                let lastKey = '';
+                                if (left !== null) {
+                                    request(left.$link, meta);
+                                }
+                                for (let i = 0, il = entries.length; i < il; i++) {
+                                    const entry = entries[i];
+                                    const next = entry.t;
+                                    const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
+                                    const key = lastKey.slice(0, entry.p) + key_str;
+                                    lastKey = key;
+                                    request(entry.v.$link, { t: 2, k: key });
+                                    if (next !== null) {
+                                        request(next.$link, { t: 1 });
+                                    }
+                                }
+                                break;
+                            }
+                            case 2: {
+                                const [collection, rkey] = meta.k.split('/');
+                                yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
+                                break;
+                            }
+                        }
+                    }
+                }
+                missingBlocks = Array.from(pending, ([cid, meta]) => {
+                    switch (meta.t) {
+                        case 0: {
+                            return { cid, type: 'commit' };
+                        }
+                        case 1: {
+                            return { cid, type: 'mst-node' };
+                        }
+                        case 2: {
+                            return { cid, type: 'record', key: meta.k };
+                        }
+                    }
+                });
+            }
+            finally {
+                await car.dispose();
+            }
+        },
+    };
+};
+//# sourceMappingURL=stream-repo-reader.js.map

package/dist/v4/repo-reader/stream-repo-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream-repo-reader.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/stream-repo-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,aAAa,CAAC;AAEhC,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,OAAO,KAAK,SAAS,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAsCvC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAgD,EAAE;IACnF,MAAM,SAAS,GAAG,IAAI,eAAe,EAA0B,CAAC;IAChE,IAAI,IAAoC,CAAC;IAEzC,OAAO;QACN,QAAQ,EAAE,IAAI,cAAc,CAAC;YAC5B,KAAK,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAI,GAAG,UAAU,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBAEtC,IAAI,CAAC;oBACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;wBAChC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBAC3B,CAAC;oBAED,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAErB,UAAU,CAAC,KAAK,EAAE,CAAC;gBACpB,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACd,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACvB,CAAC;YACF,CAAC;YACD,KAAK,CAAC,MAAM;gBACX,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACxB,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;gBACtB,CAAC;YACF,CAAC;SACD,CAAC;QACF,QAAQ,EAAE,SAAS,CAAC,QAAQ;KAC5B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAkC,EAAsB,EAAE;IACpF,IAAI,aAAa,GAAwB,EAAE,CAAC;IAE5C,OAAO;QACN,IAAI,aAAa;YAChB,OAAO,aAAa,CAAC;QACtB,CAAC;QAED,KAAK,CAAC,OAAO;YACZ,uBAAuB;QACxB,CAAC;QAED,CAAC,MAAM,CAAC,YAAY,CAAC;YACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;QACvB,CAAC;QACD,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;YAC5B,kDAAkD;YAClD,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAEzC,IAAI,CAAC;gBACJ,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;gBAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;gBAErD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAQ,CAAC;gBAEhC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAE,IAAe,EAAQ,EAAE;oBACtD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAE9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBACnB,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC9C,CAAC;yBAAM,CAAC;wBACP,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;oBACxB,CAAC;gBACF,CAAC,CAAC;gBAEF,CAAC;oBACA,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC;oBAChC,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,gDAAgD,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;oBAE3F,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;oBAC/B,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC5B,CAAC;gBAED,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;oBAC/B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;oBAEpC,CAAC;wBACA,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBAE9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;4BACxB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;4BACpB,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC9C,CAAC;6BAAM,CAAC;4BACP,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;wBACxB,CAAC;oBACF,CAAC;oBAED,IAAI,IAAsB,CAAC;oBAC3B,OAAO,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;wBACjC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;wBAE3C,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;4BAChB,KAAK,CAAC,CAAC,CAAC,CAAC;gCACR,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gCACxC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,8BAA8B,GAAG,EAAE,CAAC,CAAC;gCAE9D,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gCACrC,MAAM;4BACP,CAAC;4BACD,KAAK,CAAC,CAAC,CAAC,CAAC;gCACR,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gCACtC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,gCAAgC,GAAG,EAAE,CAAC,CAAC;gCAE/D,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;gCACvB,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;gCAEpB,IAAI,OAAO,GAAG,EAAE,CAAC;gCAEjB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;oCACnB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gCAC3B,CAAC;gCAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;oCAClD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oCACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC;oCAErB,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oCACxD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;oCAEhD,OAAO,GAAG,GAAG,CAAC;oCAEd,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oCAEzC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;wCACnB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oCAC/B,CAAC;gCACF,CAAC;gCAED,MAAM;4BACP,CAAC;4BACD,KAAK,CAAC,CAAC,CAAC,CAAC;gCACR,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gCAE7C,MAAM,IAAI,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;gCACvE,MAAM;4BACP,CAAC;wBACF,CAAC;oBACF,CAAC;gBACF,CAAC;gBAED,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,EAAqB,EAAE;oBACtE,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC;wBAChB,KAAK,CAAC,CAAC,CAAC,CAAC;4BACR,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;wBAChC,CAAC;wBACD,KAAK,CAAC,CAAC,CAAC,CAAC;4BACR,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC;wBAClC,CAAC;wBACD,KAAK,CAAC,CAAC,CAAC,CAAC;4BACR,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;wBAC7C,CAAC;oBACF,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACV,MAAM,GAAG,CAAC,OAAO,EAAE,CAAC;YACrB,CAAC;QACF,CAAC;KACD,CAAC;AACH,CAAC,CAAC"}

package/dist/v4/repo-reader/sync-blockmap.d.ts

@@ -0,0 +1,29 @@
+import * as CID from '@atcute/cid';
+import * as CarReader from '../car-reader/index.js';
+export type BlockMap = Map<string, CarReader.CarEntry>;
+/**
+ * collects entries from a CAR archive into a mapping of CID string -> actual bytes
+ * @param iterator a generator that yields objects with a `cid` and `bytes` property
+ * @returns a mapping of CID string -> actual bytes
+ */
+export declare const collectBlock: (iterator: Iterable<CarReader.CarEntry>) => BlockMap;
+/**
+ * reads a block from the blockmap and validates it against the provided validation function
+ * @param map a mapping of CID string -> actual bytes
+ * @param link a CID link to read
+ * @param validate a validation function to validate the decoded data
+ * @returns the decoded and validated data
+ */
+export declare const readBlock: <T>(map: BlockMap, link: CID.CidLink, validate: (value: unknown) => value is T) => T;
+/** node entry object */
+export interface NodeEntry {
+    key: string;
+    cid: CID.CidLink;
+}
+/**
+ * walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
+ * @param map a mapping of CID string -> actual bytes
+ * @param pointer a CID link to the root of the MST
+ * @returns a generator that yields the entries of the MST
+ */
+export declare function walkMstEntries(map: BlockMap, pointer: CID.CidLink): Generator<NodeEntry>;

package/dist/v4/repo-reader/sync-blockmap.js

@@ -0,0 +1,58 @@
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { decodeUtf8From } from '@atcute/uint8array';
+import * as CarReader from '../car-reader/index.js';
+import { assert } from '../utils.js';
+import { isMstNode } from './mst.js';
+/**
+ * collects entries from a CAR archive into a mapping of CID string -> actual bytes
+ * @param iterator a generator that yields objects with a `cid` and `bytes` property
+ * @returns a mapping of CID string -> actual bytes
+ */
+export const collectBlock = (iterator) => {
+    const blockmap = new Map();
+    for (const entry of iterator) {
+        blockmap.set(CID.toString(entry.cid), entry);
+    }
+    return blockmap;
+};
+/**
+ * reads a block from the blockmap and validates it against the provided validation function
+ * @param map a mapping of CID string -> actual bytes
+ * @param link a CID link to read
+ * @param validate a validation function to validate the decoded data
+ * @returns the decoded and validated data
+ */
+export const readBlock = (map, link, validate) => {
+    const cid = link.$link;
+    const entry = map.get(cid);
+    assert(entry != null, `cid not found in blockmap; cid=${cid}`);
+    const data = CBOR.decode(entry.bytes);
+    assert(validate(data), `validation failed for cid=${cid}`);
+    return data;
+};
+/**
+ * walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
+ * @param map a mapping of CID string -> actual bytes
+ * @param pointer a CID link to the root of the MST
+ * @returns a generator that yields the entries of the MST
+ */
+export function* walkMstEntries(map, pointer) {
+    const data = readBlock(map, pointer, isMstNode);
+    const entries = data.e;
+    let lastKey = '';
+    if (data.l !== null) {
+        yield* walkMstEntries(map, data.l);
+    }
+    for (let i = 0, il = entries.length; i < il; i++) {
+        const entry = entries[i];
+        const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
+        const key = lastKey.slice(0, entry.p) + key_str;
+        lastKey = key;
+        yield { key: key, cid: entry.v };
+        if (entry.t !== null) {
+            yield* walkMstEntries(map, entry.t);
+        }
+    }
+}
+//# sourceMappingURL=sync-blockmap.js.map

package/dist/v4/repo-reader/sync-blockmap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-blockmap.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/sync-blockmap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEpD,OAAO,KAAK,SAAS,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAIrC;;;;GAIG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,QAAsC,EAAY,EAAE;IAChF,MAAM,QAAQ,GAAa,IAAI,GAAG,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC9B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,QAAQ,CAAC;AACjB,CAAC,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CACxB,GAAa,EACb,IAAiB,EACjB,QAAwC,EACpC,EAAE;IACN,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC;IAEvB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,MAAM,CAAC,KAAK,IAAI,IAAI,EAAE,kCAAkC,GAAG,EAAE,CAAC,CAAC;IAE/D,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,6BAA6B,GAAG,EAAE,CAAC,CAAC;IAE3D,OAAO,IAAI,CAAC;AACb,CAAC,CAAC;AAQF;;;;;GAKG;AACH,MAAM,SAAS,CAAC,CAAC,cAAc,CAAC,GAAa,EAAE,OAAoB;IAClE,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC;IAEvB,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAEzB,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;QAEhD,OAAO,GAAG,GAAG,CAAC;QAEd,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QAEjC,IAAI,KAAK,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YACtB,KAAK,CAAC,CAAC,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC;IACF,CAAC;AACF,CAAC"}

package/dist/v4/repo-reader/sync-repo-reader.d.ts

@@ -0,0 +1,2 @@
+import { RepoEntry } from './types.js';
+export declare function fromUint8Array(buf: Uint8Array): Generator<RepoEntry>;

package/dist/v4/repo-reader/sync-repo-reader.js

@@ -0,0 +1,20 @@
+import * as CarReader from '../car-reader/index.js';
+import { assert } from '../utils.js';
+import { isCommit } from './mst.js';
+import { collectBlock, readBlock, walkMstEntries } from './sync-blockmap.js';
+import { RepoEntry } from './types.js';
+export function* fromUint8Array(buf) {
+    const car = CarReader.fromUint8Array(buf);
+    const roots = car.roots;
+    assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+    const blockmap = collectBlock(car);
+    assert(blockmap.size > 0, `expected at least 1 block in the archive; got=${blockmap.size}`);
+    const commit = readBlock(blockmap, roots[0], isCommit);
+    for (const { key, cid } of walkMstEntries(blockmap, commit.data)) {
+        const [collection, rkey] = key.split('/');
+        const carEntry = blockmap.get(cid.$link);
+        assert(carEntry != null, `cid not found in blockmap; cid=${cid}`);
+        yield new RepoEntry(collection, rkey, cid, carEntry);
+    }
+}
+//# sourceMappingURL=sync-repo-reader.js.map

package/dist/v4/repo-reader/sync-repo-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync-repo-reader.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/sync-repo-reader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,MAAM,SAAS,CAAC,CAAC,cAAc,CAAC,GAAe;IAC9C,MAAM,GAAG,GAAG,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;IAExB,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,gDAAgD,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAE3F,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,iDAAiD,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAE5F,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IAEvD,KAAK,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAClE,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,IAAI,IAAI,EAAE,kCAAkC,GAAG,EAAE,CAAC,CAAC;QAElE,MAAM,IAAI,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;IACtD,CAAC;AACF,CAAC"}

package/dist/v4/repo-reader/types.d.ts

@@ -0,0 +1,20 @@
+import * as CID from '@atcute/cid';
+import { type CarEntry } from '../car-reader/index.js';
+export declare class RepoEntry {
+    /** the collection this record belongs to */
+    readonly collection: string;
+    /** record key */
+    readonly rkey: string;
+    /** CID of this record */
+    readonly cid: CID.CidLink;
+    /** the associated CarEntry for this record */
+    readonly carEntry: CarEntry;
+    /**
+     * raw contents of this record
+     */
+    get bytes(): Uint8Array;
+    /**
+     * decoded contents of this record
+     */
+    get record(): unknown;
+}

package/dist/v4/repo-reader/types.js

@@ -0,0 +1,37 @@
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import {} from '../car-reader/index.js';
+export class RepoEntry {
+    collection;
+    rkey;
+    cid;
+    carEntry;
+    /** @internal */
+    constructor(
+    /** the collection this record belongs to */
+    collection, 
+    /** record key */
+    rkey, 
+    /** CID of this record */
+    cid, 
+    /** the associated CarEntry for this record */
+    carEntry) {
+        this.collection = collection;
+        this.rkey = rkey;
+        this.cid = cid;
+        this.carEntry = carEntry;
+    }
+    /**
+     * raw contents of this record
+     */
+    get bytes() {
+        return this.carEntry.bytes;
+    }
+    /**
+     * decoded contents of this record
+     */
+    get record() {
+        return CBOR.decode(this.bytes);
+    }
+}
+//# sourceMappingURL=types.js.map

package/dist/v4/repo-reader/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAiB,MAAM,wBAAwB,CAAC;AAEvD,MAAM,OAAO,SAAS;IAIJ;IAEA;IAEA;IAEA;IATjB,gBAAgB;IAChB;IACC,4CAA4C;IAC5B,UAAkB;IAClC,iBAAiB;IACD,IAAY;IAC5B,yBAAyB;IACT,GAAgB;IAChC,8CAA8C;IAC9B,QAAkB;QANlB,eAAU,GAAV,UAAU,CAAQ;QAElB,SAAI,GAAJ,IAAI,CAAQ;QAEZ,QAAG,GAAH,GAAG,CAAa;QAEhB,aAAQ,GAAR,QAAQ,CAAU;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CACD"}

package/dist/v4/utils.d.ts

@@ -0,0 +1,3 @@
+export declare const assert: {
+    (condition: boolean, message: string): asserts condition;
+};

package/dist/v4/utils.js

@@ -0,0 +1,6 @@
+export const assert = (condition, message) => {
+    if (!condition) {
+        throw new Error(message);
+    }
+};
+//# sourceMappingURL=utils.js.map

package/dist/v4/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../lib/v4/utils.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAEf,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE;IAC1B,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC,CAAC"}

package/lib/v3/atproto-repo.ts

@@ -0,0 +1,15 @@
+export {
+	collectBlock,
+	isCommit,
+	isMstNode,
+	isTreeEntry,
+	fromUint8Array as iterateAtpRepo,
+	readBlock,
+	RepoEntry,
+	walkMstEntries,
+	type BlockMap,
+	type Commit,
+	type MstNode,
+	type NodeEntry,
+	type TreeEntry,
+} from '../v4/repo-reader/index.js';

package/lib/v3/reader.ts

@@ -0,0 +1,9 @@
+export {
+	isCarV1Header,
+	fromUint8Array as readCar,
+	type CarEntry,
+	type CarHeader,
+	type CarV1Header,
+	type StreamedCarReader,
+	type SyncCarReader,
+} from '../v4/car-reader/index.js';

package/lib/v4/car-reader/index.ts

@@ -0,0 +1,4 @@
+export * from './types.js';
+
+export * from './stream-car-reader.js';
+export * from './sync-car-reader.js';

package/lib/v4/car-reader/stream-car-reader.ts

@@ -0,0 +1,240 @@
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { concat } from '@atcute/uint8array';
+
+import { isCarV1Header, type CarEntry, type CarHeader } from './types.js';
+
+export interface StreamedCarReader {
+	header(): Promise<CarHeader>;
+	roots(): Promise<CBOR.CidLink[]>;
+
+	dispose(): Promise<void>;
+
+	[Symbol.asyncDispose](): Promise<void>;
+	[Symbol.asyncIterator](): AsyncIterator<CarEntry>;
+}
+
+export const carEntryTransform = (): ReadableWritablePair<CarEntry, Uint8Array> => {
+	const transform = new TransformStream<Uint8Array, Uint8Array>();
+	let car: StreamedCarReader | undefined;
+
+	return {
+		readable: new ReadableStream({
+			async start(controller) {
+				car = fromStream(transform.readable);
+
+				try {
+					for await (const entry of car) {
+						controller.enqueue(entry);
+					}
+
+					await car.dispose();
+
+					controller.close();
+				} catch (err) {
+					controller.error(err);
+				}
+			},
+			async cancel() {
+				if (car !== undefined) {
+					await car.dispose();
+				}
+			},
+		}),
+		writable: transform.writable,
+	};
+};
+
+export const fromStream = (stream: ReadableStream<Uint8Array>): StreamedCarReader => {
+	let chunk = new Uint8Array(0) as Uint8Array; // annoying!
+	let offset = 0;
+
+	let _header: CarHeader | undefined;
+
+	const reader = stream.getReader();
+
+	const readVarint = async (): Promise<number> => {
+		let value = 0;
+		let shift = 0;
+
+		const MSB = 0x80;
+		const REST = 0x7f;
+
+		while (true) {
+			if (chunk.length === 0) {
+				const { value, done } = await reader.read();
+				if (done) {
+					throw new Error(`unexpected eof while decoding varint`);
+				}
+
+				chunk = value;
+			}
+
+			const byte = chunk[0];
+			chunk = chunk.subarray(1);
+
+			value += shift < 28 ? (byte & REST) << shift : (byte & REST) * 2 ** shift;
+			shift += 7;
+
+			offset++;
+
+			if ((byte & MSB) === 0) {
+				return value;
+			}
+		}
+	};
+
+	const readExact = async (n: number): Promise<Uint8Array> => {
+		const buffer = new Uint8Array(n);
+		let written = 0;
+
+		while (written < n) {
+			if (chunk.length === 0) {
+				const { value, done } = await reader.read();
+
+				if (done) {
+					throw new Error('unexpected eof while reading data');
+				}
+
+				chunk = value;
+			}
+
+			const taken = Math.min(n - written, chunk.length);
+			buffer.set(chunk.subarray(0, taken), written);
+
+			written += taken;
+			chunk = chunk.subarray(taken);
+		}
+
+		offset += n;
+		return buffer;
+	};
+
+	const readCid = async (): Promise<CID.Cid> => {
+		const head = await readExact(4);
+
+		const version = head[0];
+		const codec = head[1];
+		const digestType = head[2];
+		const digestSize = head[3];
+
+		if (version !== CID.CID_VERSION) {
+			throw new RangeError(`incorrect cid version (got v${version})`);
+		}
+
+		if (codec !== CID.CODEC_DCBOR && codec !== CID.CODEC_RAW) {
+			throw new RangeError(`incorrect cid codec (got 0x${codec.toString(16)})`);
+		}
+
+		if (digestType !== CID.HASH_SHA256) {
+			throw new RangeError(`incorrect cid digest type (got 0x${digestType.toString(16)})`);
+		}
+
+		if (digestSize !== 32 && digestSize !== 0) {
+			throw new RangeError(`incorrect cid digest size (got ${digestSize})`);
+		}
+
+		// concatenate and have digest refer back to this buffer
+		const bytes = concat([head, await readExact(digestSize)]);
+		const digest = bytes.subarray(4, 4 + digestSize);
+
+		const cid: CID.Cid = {
+			version: version,
+			codec: codec,
+			digest: {
+				codec: digestType,
+				contents: digest,
+			},
+			bytes: bytes,
+		};
+
+		return cid;
+	};
+
+	return {
+		[Symbol.asyncDispose]() {
+			return this.dispose();
+		},
+
+		async dispose() {
+			await reader.cancel();
+		},
+
+		async header(): Promise<CarHeader> {
+			if (_header !== undefined) {
+				return _header;
+			}
+
+			const headerStart = offset;
+
+			const headerSize = await readVarint();
+			if (headerSize === 0) {
+				throw new RangeError(`invalid car header; length=0`);
+			}
+
+			const dataStart = offset;
+
+			const raw = await readExact(headerSize);
+			const data = CBOR.decode(raw);
+			if (!isCarV1Header(data)) {
+				throw new TypeError(`expected a car v1 archive`);
+			}
+
+			const dataEnd = offset;
+			const headerEnd = offset;
+
+			return (_header = { data, headerStart, headerEnd, dataStart, dataEnd });
+		},
+
+		async roots(): Promise<CBOR.CidLink[]> {
+			const header = await this.header();
+
+			return header.data.roots;
+		},
+
+		async *[Symbol.asyncIterator](): AsyncGenerator<CarEntry> {
+			// ensure the header is read first
+			if (_header === undefined) {
+				await this.header();
+			}
+
+			while (true) {
+				if (chunk.length === 0) {
+					const { value, done } = await reader.read();
+
+					if (done) {
+						return;
+					}
+
+					chunk = value;
+				}
+
+				const entryStart = offset;
+				const entrySize = await readVarint();
+
+				const cidStart = offset;
+				const cid = await readCid();
+
+				const bytesStart = offset;
+				const bytesSize = entrySize - (bytesStart - cidStart);
+				const bytes = await readExact(bytesSize);
+
+				const cidEnd = bytesStart;
+				const bytesEnd = offset;
+				const entryEnd = bytesEnd;
+
+				yield {
+					cid,
+					bytes,
+
+					entryStart,
+					entryEnd,
+					cidStart,
+					cidEnd,
+					bytesStart,
+					bytesEnd,
+				};
+			}
+		},
+	};
+};