@atcute/repo 0.1.0

package/dist/verify.d.ts

@@ -0,0 +1,17 @@
+import type { PublicKey } from '@atcute/crypto';
+import type { AtprotoDid } from '@atcute/lexicons/syntax';
+export interface VerifiedRecord {
+    /** CID of the record */
+    cid: string;
+    /** Record data */
+    record: unknown;
+}
+export interface VerifyRecordOptions {
+    did?: AtprotoDid;
+    collection: string;
+    rkey: string;
+    publicKey?: PublicKey;
+    carBytes: Uint8Array;
+}
+export declare const verifyRecord: ({ did, collection, rkey, publicKey, carBytes, }: VerifyRecordOptions) => Promise<VerifiedRecord>;
+//# sourceMappingURL=verify.d.ts.map

package/dist/verify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.d.ts","sourceRoot":"","sources":["../lib/verify.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAQ1D,MAAM,WAAW,cAAc;IAC9B,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,kBAAkB;IAClB,MAAM,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IACnC,GAAG,CAAC,EAAE,UAAU,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,QAAQ,EAAE,UAAU,CAAC;CACrB;AAED,eAAO,MAAM,YAAY,GAAU,iDAMhC,mBAAmB,KAAG,OAAO,CAAC,cAAc,CA6D9C,CAAC"}

package/dist/verify.js

@@ -0,0 +1,166 @@
+import * as CAR from '@atcute/car';
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { isNodeData } from '@atcute/mst';
+import { decodeUtf8From, encodeUtf8, toSha256 } from '@atcute/uint8array';
+import { isCommit } from './types.js';
+export const verifyRecord = async ({ did, collection, rkey, publicKey, carBytes, }) => {
+    // read the car
+    let blockmap;
+    let commit;
+    {
+        const reader = CAR.fromUint8Array(carBytes);
+        if (reader.header.data.roots.length !== 1) {
+            throw new Error(`car must have exactly one root`);
+        }
+        blockmap = new Map();
+        for (const entry of reader) {
+            const cidString = CID.toString(entry.cid);
+            // Verify that `bytes` matches its associated CID
+            const expectedCid = CID.toString(await CID.create(entry.cid.codec, entry.bytes));
+            if (cidString !== expectedCid) {
+                throw new Error(`cid does not match bytes`);
+            }
+            blockmap.set(cidString, entry.bytes);
+        }
+        if (blockmap.size === 0) {
+            throw new Error(`car must have at least one block`);
+        }
+        commit = readBlock(blockmap, reader.header.data.roots[0].$link, isCommit);
+    }
+    // verify did in commit matches the did
+    if (did !== undefined && commit.did !== did) {
+        throw new Error(`did in commit does not match expected did`);
+    }
+    // verify signature contained in commit is valid (if publicKey provided)
+    if (publicKey) {
+        const { sig, ...unsigned } = commit;
+        const data = CBOR.encode(unsigned);
+        const valid = await publicKey.verify(CBOR.fromBytes(sig), data);
+        if (!valid) {
+            throw new Error(`signature verification failed`);
+        }
+    }
+    // find and verify the record in the commit
+    const targetKey = `${collection}/${rkey}`;
+    const { found } = await dfs(blockmap, commit.data.$link, targetKey);
+    if (!found) {
+        throw new Error(`could not find record in car`);
+    }
+    return {
+        cid: found.cid,
+        record: found.record,
+    };
+};
+const readBlock = (blockmap, cid, validate) => {
+    const bytes = blockmap.get(cid);
+    if (!bytes) {
+        throw new Error(`cid not found in blockmap; cid=${cid}`);
+    }
+    const decoded = CBOR.decode(bytes);
+    if (!validate(decoded)) {
+        throw new Error(`validation failed for cid=${cid}`);
+    }
+    return decoded;
+};
+const dfs = async (blockmap, from, targetKey, visited = new Set()) => {
+    // If there's no starting point, return empty state
+    if (from == null) {
+        return { found: false };
+    }
+    // Check for cycles
+    {
+        if (visited.has(from)) {
+            throw new Error(`cycle detected; cid=${from}`);
+        }
+        visited.add(from);
+    }
+    // Get the block data
+    let node;
+    {
+        const bytes = blockmap.get(from);
+        if (!bytes) {
+            return { found: false };
+        }
+        const decoded = CBOR.decode(bytes);
+        if (!isNodeData(decoded)) {
+            throw new Error(`invalid mst node; cid=${from}`);
+        }
+        node = decoded;
+    }
+    // Recursively process the left child
+    const left = await dfs(blockmap, node.l?.$link, targetKey, visited);
+    let key = '';
+    let found = left.found;
+    let depth;
+    let firstKey;
+    let lastKey;
+    // Process all entries in this node
+    for (const entry of node.e) {
+        // Construct the key by truncating and appending
+        key = key.substring(0, entry.p) + decodeUtf8From(CBOR.fromBytes(entry.k));
+        // Check if this is our target key
+        if (key === targetKey) {
+            const recordBytes = blockmap.get(entry.v.$link);
+            if (recordBytes) {
+                const record = CBOR.decode(recordBytes);
+                found = { cid: entry.v.$link, record };
+            }
+        }
+        // Calculate depth based on leading zeros in the hash
+        const keyDigest = await toSha256(encodeUtf8(key));
+        let zeroCount = 0;
+        outerLoop: for (const byte of keyDigest) {
+            for (let bit = 7; bit >= 0; bit--) {
+                if (((byte >> bit) & 1) !== 0) {
+                    break outerLoop;
+                }
+                zeroCount++;
+            }
+        }
+        const thisDepth = Math.floor(zeroCount / 2);
+        // Ensure consistent depth
+        if (depth === undefined) {
+            depth = thisDepth;
+        }
+        else if (depth !== thisDepth) {
+            throw new Error(`node has entries with different depths; cid=${from}`);
+        }
+        // Track first and last keys
+        if (lastKey === undefined) {
+            firstKey = key;
+            lastKey = key;
+        }
+        // Check key ordering
+        if (lastKey > key) {
+            throw new Error(`entries are out of order; cid=${from}`);
+        }
+        // Process right child
+        const right = await dfs(blockmap, entry.t?.$link, targetKey, visited);
+        // Check ordering with right subtree
+        if (right.min && right.min < lastKey) {
+            throw new Error(`entries are out of order; cid=${from}`);
+        }
+        found = found || right.found;
+        // Check depth ordering
+        if (left.depth !== undefined && left.depth >= thisDepth) {
+            throw new Error(`depths are out of order; cid=${from}`);
+        }
+        if (right.depth !== undefined && right.depth >= thisDepth) {
+            throw new Error(`depths are out of order; cid=${from}`);
+        }
+        // Update last key based on right subtree
+        lastKey = right.max ?? key;
+    }
+    // Check ordering with left subtree
+    if (left.max && firstKey && left.max > firstKey) {
+        throw new Error(`entries are out of order; cid=${from}`);
+    }
+    return {
+        found,
+        min: firstKey,
+        max: lastKey,
+        depth,
+    };
+};
+//# sourceMappingURL=verify.js.map

package/dist/verify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verify.js","sourceRoot":"","sources":["../lib/verify.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AACnC,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AAGnC,OAAO,EAAE,UAAU,EAAiB,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAE1E,OAAO,EAAE,QAAQ,EAAe,MAAM,YAAY,CAAC;AAmBnD,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,EAAE,EAClC,GAAG,EACH,UAAU,EACV,IAAI,EACJ,SAAS,EACT,QAAQ,GACa,EAA2B,EAAE;IAClD,eAAe;IACf,IAAI,QAAkB,CAAC;IACvB,IAAI,MAAc,CAAC;IACnB,CAAC;QACA,MAAM,MAAM,GAAG,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACnD,CAAC;QAED,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;QACrB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAE1C,iDAAiD;YACjD,MAAM,WAAW,GAAG,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAiB,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7F,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC7C,CAAC;YAED,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC3E,CAAC;IAED,uCAAuC;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC9D,CAAC;IAED,wEAAwE;IACxE,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;QAEpC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,CACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAA4B,EAC9C,IAA+B,CAC/B,CAAC;QAEF,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAClD,CAAC;IACF,CAAC;IAED,2CAA2C;IAC3C,MAAM,SAAS,GAAG,GAAG,UAAU,IAAI,IAAI,EAAE,CAAC;IAC1C,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACpE,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IACjD,CAAC;IAED,OAAO;QACN,GAAG,EAAE,KAAK,CAAC,GAAG;QACd,MAAM,EAAE,KAAK,CAAC,MAAM;KACpB,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAI,QAAkB,EAAE,GAAW,EAAE,QAAwC,EAAK,EAAE;IACrG,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,kCAAkC,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC,CAAC;AASF,MAAM,GAAG,GAAG,KAAK,EAChB,QAAkB,EAClB,IAAwB,EACxB,SAAiB,EACjB,UAAU,IAAI,GAAG,EAAU,EACN,EAAE;IACvB,mDAAmD;IACnD,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;QAClB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC;IAED,mBAAmB;IACnB,CAAC;QACA,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,qBAAqB;IACrB,IAAI,IAAc,CAAC;IACnB,CAAC;QACA,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,GAAG,OAAO,CAAC;IAChB,CAAC;IAED,qCAAqC;IACrC,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAEpE,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,KAAyB,CAAC;IAC9B,IAAI,QAA4B,CAAC;IACjC,IAAI,OAA2B,CAAC;IAEhC,mCAAmC;IACnC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5B,gDAAgD;QAChD,GAAG,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAE1E,kCAAkC;QAClC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAChD,IAAI,WAAW,EAAE,CAAC;gBACjB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACxC,KAAK,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;YACxC,CAAC;QACF,CAAC;QAED,qDAAqD;QACrD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,SAAS,EAAE,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YACzC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC;gBACnC,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/B,MAAM,SAAS,CAAC;gBACjB,CAAC;gBACD,SAAS,EAAE,CAAC;YACb,CAAC;QACF,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;QAE5C,0BAA0B;QAC1B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACzB,KAAK,GAAG,SAAS,CAAC;QACnB,CAAC;aAAM,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,4BAA4B;QAC5B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC3B,QAAQ,GAAG,GAAG,CAAC;YACf,OAAO,GAAG,GAAG,CAAC;QACf,CAAC;QAED,qBAAqB;QACrB,IAAI,OAAO,GAAG,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,sBAAsB;QACtB,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAEtE,oCAAoC;QACpC,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,GAAG,GAAG,OAAO,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;QAED,KAAK,GAAG,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC;QAE7B,uBAAuB;QACvB,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,yCAAyC;QACzC,OAAO,GAAG,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC;IAC5B,CAAC;IAED,mCAAmC;IACnC,IAAI,IAAI,CAAC,GAAG,IAAI,QAAQ,IAAI,IAAI,CAAC,GAAG,GAAG,QAAQ,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO;QACN,KAAK;QACL,GAAG,EAAE,QAAQ;QACb,GAAG,EAAE,OAAO;QACZ,KAAK;KACL,CAAC;AACH,CAAC,CAAC"}

package/lib/index.ts

@@ -0,0 +1,5 @@
+export type { RepoEntry } from './types.js';
+
+export * from './streamed-reader.js';
+export * from './reader.js';
+export * from './verify.js';

package/lib/reader.ts

@@ -0,0 +1,98 @@
+import type { CarEntry } from '@atcute/car';
+import * as CAR from '@atcute/car';
+import * as CBOR from '@atcute/cbor';
+import type { CidLink } from '@atcute/cid';
+import * as CID from '@atcute/cid';
+import { isNodeData } from '@atcute/mst';
+import { decodeUtf8From } from '@atcute/uint8array';
+
+import { isCommit, RepoEntry } from './types.js';
+import { assert } from './utils.js';
+
+/** @internal */
+type EntryMap = Map<string, CarEntry>;
+
+/** node entry object */
+interface NodeEntry {
+	key: string;
+	cid: CidLink;
+}
+
+export function* fromUint8Array(buf: Uint8Array): Generator<RepoEntry> {
+	const car = CAR.fromUint8Array(buf);
+	const roots = car.roots;
+
+	assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+
+	const map: EntryMap = new Map();
+	for (const entry of car) {
+		map.set(CID.toString(entry.cid), entry);
+	}
+
+	// [commit, mst node, record?]
+	assert(map.size >= 2, `expected at least 2 blocks in the archive; got=${map.size}`);
+
+	const commit = readEntry(map, roots[0], isCommit);
+
+	for (const { key, cid } of walkMstEntries(map, commit.data)) {
+		const [collection, rkey] = key.split('/');
+
+		const carEntry = map.get(cid.$link);
+		assert(carEntry != null, `cid not found in blockmap; cid=${cid}`);
+
+		yield new RepoEntry(collection, rkey, cid, carEntry);
+	}
+}
+
+/**
+ * reads a block from the blockmap and validates it against the provided validation function
+ * @internal
+ * @param map a mapping of CID string -> actual bytes
+ * @param link a CID link to read
+ * @param validate a validation function to validate the decoded data
+ * @returns the decoded and validated data
+ */
+export const readEntry = <T>(map: EntryMap, link: CidLink, validate: (value: unknown) => value is T): T => {
+	const cid = link.$link;
+
+	const entry = map.get(cid);
+	assert(entry != null, `cid not found in blockmap; cid=${cid}`);
+
+	const data = CBOR.decode(entry.bytes);
+	assert(validate(data), `validation failed for cid=${cid}`);
+
+	return data;
+};
+
+/**
+ * walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
+ * @internal
+ * @param map a mapping of CID string -> actual bytes
+ * @param pointer a CID link to the root of the MST
+ * @returns a generator that yields the entries of the MST
+ */
+export function* walkMstEntries(map: EntryMap, pointer: CidLink): Generator<NodeEntry> {
+	const data = readEntry(map, pointer, isNodeData);
+	const entries = data.e;
+
+	let lastKey = '';
+
+	if (data.l !== null) {
+		yield* walkMstEntries(map, data.l);
+	}
+
+	for (let i = 0, il = entries.length; i < il; i++) {
+		const entry = entries[i];
+
+		const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
+		const key = lastKey.slice(0, entry.p) + key_str;
+
+		lastKey = key;
+
+		yield { key: key, cid: entry.v };
+
+		if (entry.t !== null) {
+			yield* walkMstEntries(map, entry.t);
+		}
+	}
+}

package/lib/streamed-reader.ts

@@ -0,0 +1,208 @@
+import type { CarEntry } from '@atcute/car';
+import * as CAR from '@atcute/car';
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { isNodeData } from '@atcute/mst';
+import { decodeUtf8From } from '@atcute/uint8array';
+
+import { isCommit, RepoEntry } from './types.js';
+import { assert } from './utils.js';
+import Queue from './utils/queue.js';
+
+type EntryMeta = { t: 0 } | { t: 1 } | { t: 2; k: string };
+
+type Task = {
+	c: string;
+	e: CarEntry;
+	m: EntryMeta;
+};
+
+export type MissingBlockEntry =
+	| {
+			cid: string;
+			type: 'record';
+			key: string;
+	  }
+	| {
+			cid: string;
+			type: 'mst-node';
+	  }
+	| {
+			cid: string;
+			type: 'commit';
+	  };
+
+export interface StreamedRepoReader {
+	/**
+	 * list of blocks that were referenced but not found in the repository.
+	 * blocks may be reported as missing if multiple records share the same CID.
+	 */
+	readonly missingBlocks: readonly MissingBlockEntry[];
+
+	dispose(): Promise<void>;
+
+	[Symbol.asyncDispose](): Promise<void>;
+	[Symbol.asyncIterator](): AsyncIterator<RepoEntry>;
+}
+
+export const repoEntryTransform = (): ReadableWritablePair<RepoEntry, Uint8Array> => {
+	const transform = new TransformStream<Uint8Array, Uint8Array>();
+	let repo: StreamedRepoReader | undefined;
+
+	return {
+		readable: new ReadableStream({
+			async start(controller) {
+				repo = fromStream(transform.readable);
+
+				try {
+					for await (const entry of repo) {
+						controller.enqueue(entry);
+					}
+
+					await repo.dispose();
+
+					controller.close();
+				} catch (err) {
+					controller.error(err);
+				}
+			},
+			async cancel() {
+				if (repo !== undefined) {
+					await repo.dispose();
+				}
+			},
+		}),
+		writable: transform.writable,
+	};
+};
+
+export const fromStream = (stream: ReadableStream<Uint8Array>): StreamedRepoReader => {
+	let missingBlocks: MissingBlockEntry[] = [];
+
+	return {
+		get missingBlocks() {
+			return missingBlocks;
+		},
+
+		async dispose() {
+			// does nothing for now
+		},
+
+		[Symbol.asyncDispose]() {
+			return this.dispose();
+		},
+		async *[Symbol.asyncIterator]() {
+			// await using car = CarReader.fromStream(stream);
+			const car = CAR.fromStream(stream);
+
+			try {
+				const pending = new Map<string, EntryMeta>();
+				const strays = new Map<string, CarEntry>();
+
+				const queue = new Queue<Task>();
+
+				const request = (cid: string, meta: EntryMeta): void => {
+					const entry = strays.get(cid);
+
+					if (entry !== undefined) {
+						strays.delete(cid);
+						queue.enqueue({ c: cid, e: entry, m: meta });
+					} else {
+						pending.set(cid, meta);
+					}
+				};
+
+				{
+					const roots = await car.roots();
+					assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+
+					const rootCid = roots[0].$link;
+					request(rootCid, { t: 0 });
+				}
+
+				for await (const entry of car) {
+					const cid = CID.toString(entry.cid);
+
+					{
+						const meta = pending.get(cid);
+
+						if (meta !== undefined) {
+							pending.delete(cid);
+							queue.enqueue({ c: cid, e: entry, m: meta });
+						} else {
+							strays.set(cid, entry);
+						}
+					}
+
+					let task: Task | undefined;
+					while ((task = queue.dequeue())) {
+						const { c: cid, e: entry, m: meta } = task;
+
+						switch (meta.t) {
+							case 0: {
+								const commit = CBOR.decode(entry.bytes);
+								assert(isCommit(commit), `expected commit block; cid=${cid}`);
+
+								request(commit.data.$link, { t: 1 });
+								break;
+							}
+							case 1: {
+								const node = CBOR.decode(entry.bytes);
+								assert(isNodeData(node), `expected mst node block; cid=${cid}`);
+
+								const entries = node.e;
+								const left = node.l;
+
+								let lastKey = '';
+
+								if (left !== null) {
+									request(left.$link, meta);
+								}
+
+								for (let i = 0, il = entries.length; i < il; i++) {
+									const entry = entries[i];
+									const next = entry.t;
+
+									const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
+									const key = lastKey.slice(0, entry.p) + key_str;
+
+									lastKey = key;
+
+									request(entry.v.$link, { t: 2, k: key });
+
+									if (next !== null) {
+										request(next.$link, { t: 1 });
+									}
+								}
+
+								break;
+							}
+							case 2: {
+								const [collection, rkey] = meta.k.split('/');
+
+								yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
+								break;
+							}
+						}
+					}
+				}
+
+				missingBlocks = Array.from(pending, ([cid, meta]): MissingBlockEntry => {
+					switch (meta.t) {
+						case 0: {
+							return { cid, type: 'commit' };
+						}
+						case 1: {
+							return { cid, type: 'mst-node' };
+						}
+						case 2: {
+							return { cid, type: 'record', key: meta.k };
+						}
+					}
+				});
+			} finally {
+				await car.dispose();
+			}
+		},
+	};
+};

package/lib/types.ts

@@ -0,0 +1,64 @@
+import type { CarEntry } from '@atcute/car';
+import * as CBOR from '@atcute/cbor';
+import { isBytes, type Bytes } from '@atcute/cbor';
+import { isCidLink, type CidLink } from '@atcute/cid';
+
+export class RepoEntry {
+	/** @internal */
+	constructor(
+		/** the collection this record belongs to */
+		public readonly collection: string,
+		/** record key */
+		public readonly rkey: string,
+		/** CID of this record */
+		public readonly cid: CidLink,
+		/** the associated CarEntry for this record */
+		public readonly carEntry: CarEntry,
+	) {}
+
+	/**
+	 * raw contents of this record
+	 */
+	get bytes(): Uint8Array {
+		return this.carEntry.bytes;
+	}
+
+	/**
+	 * decoded contents of this record
+	 */
+	get record(): unknown {
+		return CBOR.decode(this.bytes);
+	}
+}
+
+/** commit object */
+export interface Commit {
+	version: 3;
+	did: string;
+	data: CidLink;
+	rev: string;
+	prev: CidLink | null;
+	sig: Bytes;
+}
+
+/**
+ * checks if value is a valid commit object
+ * @param value value to check
+ * @returns true if the value is a valid commit object, false otherwise
+ */
+export const isCommit = (value: unknown): value is Commit => {
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	const obj = value as Record<string, unknown>;
+
+	return (
+		obj.version === 3 &&
+		typeof obj.did === 'string' &&
+		isCidLink(obj.data) &&
+		typeof obj.rev === 'string' &&
+		(obj.prev === null || isCidLink(obj.prev)) &&
+		isBytes(obj.sig)
+	);
+};