@atcute/car 3.0.5 → 3.1.1

package/lib/{utilities → v4/car-reader}/sync-car-reader.ts

@@ -2,14 +2,101 @@ import * as CBOR from '@atcute/cbor';
 import * as CID from '@atcute/cid';
 import * as varint from '@atcute/varint';
 
-import { isCarV1Header, type CarEntry, type CarHeader } from './car.js';
-import type { SyncByteReader } from './sync-byte-reader.js';
+import { isCarV1Header, type CarEntry, type CarHeader } from './types.js';
+
+interface SyncByteReader {
+	readonly pos: number;
+	upto(size: number): Uint8Array;
+	exactly(size: number, seek: boolean): Uint8Array;
+	seek(size: number): void;
+}
 
 export interface SyncCarReader {
-	header: CarHeader;
+	readonly header: CarHeader;
+	readonly roots: CBOR.CidLink[];
+
+	/** @deprecated do for..of on the reader directly */
 	iterate(): Generator<CarEntry>;
+	[Symbol.iterator](): Iterator<CarEntry>;
 }
 
+export const fromUint8Array = (buffer: Uint8Array): SyncCarReader => {
+	const reader = createUint8Reader(buffer);
+	const header = readHeader(reader);
+
+	return {
+		header,
+		roots: header.data.roots,
+
+		*iterate() {
+			while (reader.upto(8 + 36).length > 0) {
+				const entryStart = reader.pos;
+				const entrySize = readVarint(reader, 8);
+
+				const cidStart = reader.pos;
+				const cid = readCid(reader);
+
+				const bytesStart = reader.pos;
+				const bytesSize = entrySize - (bytesStart - cidStart);
+				const bytes = reader.exactly(bytesSize, true);
+
+				const cidEnd = bytesStart;
+				const bytesEnd = reader.pos;
+				const entryEnd = bytesEnd;
+
+				yield {
+					cid,
+					bytes,
+
+					entryStart,
+					entryEnd,
+					cidStart,
+					cidEnd,
+					bytesStart,
+					bytesEnd,
+				};
+			}
+		},
+
+		[Symbol.iterator](): Iterator<CarEntry> {
+			return this.iterate();
+		},
+	};
+};
+
+const createUint8Reader = (buf: Uint8Array): SyncByteReader => {
+	let pos = 0;
+
+	return {
+		get pos() {
+			return pos;
+		},
+
+		seek(size) {
+			if (size > buf.length - pos) {
+				throw new RangeError('unexpected end of data');
+			}
+
+			pos += size;
+		},
+		upto(size) {
+			return buf.subarray(pos, pos + size);
+		},
+		exactly(size, seek) {
+			if (size > buf.length - pos) {
+				throw new RangeError('unexpected end of data');
+			}
+
+			const slice = buf.subarray(pos, pos + size);
+			if (seek) {
+				pos += size;
+			}
+
+			return slice;
+		},
+	};
+};
+
 const readVarint = (reader: SyncByteReader, size: number): number => {
 	const buf = reader.upto(size);
 	if (buf.length === 0) {
@@ -82,40 +169,3 @@ const readCid = (reader: SyncByteReader): CID.Cid => {
 
 	return cid;
 };
-
-export const createCarReader = (reader: SyncByteReader): SyncCarReader => {
-	const header = readHeader(reader);
-
-	return {
-		header,
-		*iterate() {
-			while (reader.upto(8 + 36).length > 0) {
-				const entryStart = reader.pos;
-				const entrySize = readVarint(reader, 8);
-
-				const cidStart = reader.pos;
-				const cid = readCid(reader);
-
-				const bytesStart = reader.pos;
-				const bytesSize = entrySize - (bytesStart - cidStart);
-				const bytes = reader.exactly(bytesSize, true);
-
-				const cidEnd = bytesStart;
-				const bytesEnd = reader.pos;
-				const entryEnd = bytesEnd;
-
-				yield {
-					cid,
-					bytes,
-
-					entryStart,
-					entryEnd,
-					cidStart,
-					cidEnd,
-					bytesStart,
-					bytesEnd,
-				};
-			}
-		},
-	};
-};

package/lib/v4/index.ts

@@ -0,0 +1,2 @@
+export * as CarReader from './car-reader/index.js';
+export * as RepoReader from './repo-reader/index.js';

package/lib/v4/repo-reader/index.ts

@@ -0,0 +1,6 @@
+export * from './types.js';
+
+export * from './mst.js';
+export * from './stream-repo-reader.js';
+export * from './sync-blockmap.js';
+export * from './sync-repo-reader.js';

package/lib/v4/repo-reader/mst.ts

@@ -0,0 +1,110 @@
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+
+const isCidLink = (value: unknown): value is CID.CidLink => {
+	if (value instanceof CID.CidLinkWrapper) {
+		return true;
+	}
+
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	return '$link' in value && typeof value.$link === 'string';
+};
+
+const isBytes = (value: unknown): value is CBOR.Bytes => {
+	if (value instanceof CBOR.BytesWrapper) {
+		return true;
+	}
+
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	return '$bytes' in value && typeof value.$bytes === 'string';
+};
+
+/** commit object */
+export interface Commit {
+	version: 3;
+	did: string;
+	data: CID.CidLink;
+	rev: string;
+	prev: CID.CidLink | null;
+	sig: CBOR.Bytes;
+}
+
+/**
+ * checks if a value is a valid commit object
+ * @param value the value to check
+ * @returns true if the value is a valid commit object, false otherwise
+ */
+export const isCommit = (value: unknown): value is Commit => {
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	const obj = value as Record<string, unknown>;
+
+	return (
+		obj.version === 3 &&
+		typeof obj.did === 'string' &&
+		isCidLink(obj.data) &&
+		typeof obj.rev === 'string' &&
+		(obj.prev === null || isCidLink(obj.prev)) &&
+		isBytes(obj.sig)
+	);
+};
+
+/** mst tree entry object */
+export interface TreeEntry {
+	/** count of bytes shared with previous TreeEntry in this Node (if any) */
+	p: number;
+	/** remainder of key for this TreeEntry, after "prefixlen" have been removed */
+	k: CBOR.Bytes;
+	/** link to a sub-tree Node at a lower level which has keys sorting after this TreeEntry's key (to the "right"), but before the next TreeEntry's key in this Node (if any) */
+	v: CID.CidLink;
+	/** next subtree (to the right of leaf) */
+	t: CID.CidLink | null;
+}
+
+/**
+ * checks if a value is a valid mst tree entry object
+ * @param value the value to check
+ * @returns true if the value is a valid mst tree entry object, false otherwise
+ */
+export const isTreeEntry = (value: unknown): value is TreeEntry => {
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	const obj = value as Record<string, unknown>;
+
+	return (
+		typeof obj.p === 'number' && isBytes(obj.k) && isCidLink(obj.v) && (obj.t === null || isCidLink(obj.t))
+	);
+};
+
+/** mst node object */
+export interface MstNode {
+	/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
+	l: CID.CidLink | null;
+	/** ordered list of TreeEntry objects */
+	e: TreeEntry[];
+}
+
+/**
+ * checks if a value is a valid mst node object
+ * @param value the value to check
+ * @returns true if the value is a valid mst node object, false otherwise
+ */
+export const isMstNode = (value: unknown): value is MstNode => {
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	const obj = value as Record<string, unknown>;
+
+	return (obj.l === null || isCidLink(obj.l)) && Array.isArray(obj.e) && obj.e.every(isTreeEntry);
+};

package/lib/v4/repo-reader/stream-repo-reader.ts

@@ -0,0 +1,209 @@
+import Queue from 'yocto-queue';
+
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { decodeUtf8From } from '@atcute/uint8array';
+
+import * as CarReader from '../car-reader/index.js';
+import { assert } from '../utils.js';
+
+import { isCommit, isMstNode } from './mst.js';
+import { RepoEntry } from './types.js';
+
+type EntryMeta = { t: 0 } | { t: 1 } | { t: 2; k: string };
+
+type Task = {
+	c: string;
+	e: CarReader.CarEntry;
+	m: EntryMeta;
+};
+
+export type MissingBlockEntry =
+	| {
+			cid: string;
+			type: 'record';
+			key: string;
+	  }
+	| {
+			cid: string;
+			type: 'mst-node';
+	  }
+	| {
+			cid: string;
+			type: 'commit';
+	  };
+
+export interface StreamedRepoReader {
+	/**
+	 * list of blocks that were referenced but not found in the repository.
+	 * blocks may be reported as missing if multiple records share the same CID.
+	 */
+	readonly missingBlocks: readonly MissingBlockEntry[];
+
+	dispose(): Promise<void>;
+
+	[Symbol.asyncDispose](): Promise<void>;
+	[Symbol.asyncIterator](): AsyncIterator<RepoEntry>;
+}
+
+export const repoEntryTransform = (): ReadableWritablePair<RepoEntry, Uint8Array> => {
+	const transform = new TransformStream<Uint8Array, Uint8Array>();
+	let repo: StreamedRepoReader | undefined;
+
+	return {
+		readable: new ReadableStream({
+			async start(controller) {
+				repo = fromStream(transform.readable);
+
+				try {
+					for await (const entry of repo) {
+						controller.enqueue(entry);
+					}
+
+					await repo.dispose();
+
+					controller.close();
+				} catch (err) {
+					controller.error(err);
+				}
+			},
+			async cancel() {
+				if (repo !== undefined) {
+					await repo.dispose();
+				}
+			},
+		}),
+		writable: transform.writable,
+	};
+};
+
+export const fromStream = (stream: ReadableStream<Uint8Array>): StreamedRepoReader => {
+	let missingBlocks: MissingBlockEntry[] = [];
+
+	return {
+		get missingBlocks() {
+			return missingBlocks;
+		},
+
+		async dispose() {
+			// does nothing for now
+		},
+
+		[Symbol.asyncDispose]() {
+			return this.dispose();
+		},
+		async *[Symbol.asyncIterator]() {
+			// await using car = CarReader.fromStream(stream);
+			const car = CarReader.fromStream(stream);
+
+			try {
+				const pending = new Map<string, EntryMeta>();
+				const strays = new Map<string, CarReader.CarEntry>();
+
+				const queue = new Queue<Task>();
+
+				const request = (cid: string, meta: EntryMeta): void => {
+					const entry = strays.get(cid);
+
+					if (entry !== undefined) {
+						strays.delete(cid);
+						queue.enqueue({ c: cid, e: entry, m: meta });
+					} else {
+						pending.set(cid, meta);
+					}
+				};
+
+				{
+					const roots = await car.roots();
+					assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+
+					const rootCid = roots[0].$link;
+					request(rootCid, { t: 0 });
+				}
+
+				for await (const entry of car) {
+					const cid = CID.toString(entry.cid);
+
+					{
+						const meta = pending.get(cid);
+
+						if (meta !== undefined) {
+							pending.delete(cid);
+							queue.enqueue({ c: cid, e: entry, m: meta });
+						} else {
+							strays.set(cid, entry);
+						}
+					}
+
+					let task: Task | undefined;
+					while ((task = queue.dequeue())) {
+						const { c: cid, e: entry, m: meta } = task;
+
+						switch (meta.t) {
+							case 0: {
+								const commit = CBOR.decode(entry.bytes);
+								assert(isCommit(commit), `expected commit block; cid=${cid}`);
+
+								request(commit.data.$link, { t: 1 });
+								break;
+							}
+							case 1: {
+								const node = CBOR.decode(entry.bytes);
+								assert(isMstNode(node), `expected mst node block; cid=${cid}`);
+
+								const entries = node.e;
+								const left = node.l;
+
+								let lastKey = '';
+
+								if (left !== null) {
+									request(left.$link, meta);
+								}
+
+								for (let i = 0, il = entries.length; i < il; i++) {
+									const entry = entries[i];
+									const next = entry.t;
+
+									const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
+									const key = lastKey.slice(0, entry.p) + key_str;
+
+									lastKey = key;
+
+									request(entry.v.$link, { t: 2, k: key });
+
+									if (next !== null) {
+										request(next.$link, { t: 1 });
+									}
+								}
+
+								break;
+							}
+							case 2: {
+								const [collection, rkey] = meta.k.split('/');
+
+								yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
+								break;
+							}
+						}
+					}
+				}
+
+				missingBlocks = Array.from(pending, ([cid, meta]): MissingBlockEntry => {
+					switch (meta.t) {
+						case 0: {
+							return { cid, type: 'commit' };
+						}
+						case 1: {
+							return { cid, type: 'mst-node' };
+						}
+						case 2: {
+							return { cid, type: 'record', key: meta.k };
+						}
+					}
+				});
+			} finally {
+				await car.dispose();
+			}
+		},
+	};
+};

package/lib/v4/repo-reader/sync-blockmap.ts

@@ -0,0 +1,85 @@
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+import { decodeUtf8From } from '@atcute/uint8array';
+
+import * as CarReader from '../car-reader/index.js';
+import { assert } from '../utils.js';
+
+import { isMstNode } from './mst.js';
+
+export type BlockMap = Map<string, CarReader.CarEntry>;
+
+/**
+ * collects entries from a CAR archive into a mapping of CID string -> actual bytes
+ * @param iterator a generator that yields objects with a `cid` and `bytes` property
+ * @returns a mapping of CID string -> actual bytes
+ */
+export const collectBlock = (iterator: Iterable<CarReader.CarEntry>): BlockMap => {
+	const blockmap: BlockMap = new Map();
+	for (const entry of iterator) {
+		blockmap.set(CID.toString(entry.cid), entry);
+	}
+
+	return blockmap;
+};
+
+/**
+ * reads a block from the blockmap and validates it against the provided validation function
+ * @param map a mapping of CID string -> actual bytes
+ * @param link a CID link to read
+ * @param validate a validation function to validate the decoded data
+ * @returns the decoded and validated data
+ */
+export const readBlock = <T>(
+	map: BlockMap,
+	link: CID.CidLink,
+	validate: (value: unknown) => value is T,
+): T => {
+	const cid = link.$link;
+
+	const entry = map.get(cid);
+	assert(entry != null, `cid not found in blockmap; cid=${cid}`);
+
+	const data = CBOR.decode(entry.bytes);
+	assert(validate(data), `validation failed for cid=${cid}`);
+
+	return data;
+};
+
+/** node entry object */
+export interface NodeEntry {
+	key: string;
+	cid: CID.CidLink;
+}
+
+/**
+ * walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
+ * @param map a mapping of CID string -> actual bytes
+ * @param pointer a CID link to the root of the MST
+ * @returns a generator that yields the entries of the MST
+ */
+export function* walkMstEntries(map: BlockMap, pointer: CID.CidLink): Generator<NodeEntry> {
+	const data = readBlock(map, pointer, isMstNode);
+	const entries = data.e;
+
+	let lastKey = '';
+
+	if (data.l !== null) {
+		yield* walkMstEntries(map, data.l);
+	}
+
+	for (let i = 0, il = entries.length; i < il; i++) {
+		const entry = entries[i];
+
+		const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
+		const key = lastKey.slice(0, entry.p) + key_str;
+
+		lastKey = key;
+
+		yield { key: key, cid: entry.v };
+
+		if (entry.t !== null) {
+			yield* walkMstEntries(map, entry.t);
+		}
+	}
+}

package/lib/v4/repo-reader/sync-repo-reader.ts

@@ -0,0 +1,27 @@
+import * as CarReader from '../car-reader/index.js';
+import { assert } from '../utils.js';
+
+import { isCommit } from './mst.js';
+import { collectBlock, readBlock, walkMstEntries } from './sync-blockmap.js';
+import { RepoEntry } from './types.js';
+
+export function* fromUint8Array(buf: Uint8Array): Generator<RepoEntry> {
+	const car = CarReader.fromUint8Array(buf);
+	const roots = car.roots;
+
+	assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+
+	const blockmap = collectBlock(car);
+	assert(blockmap.size > 0, `expected at least 1 block in the archive; got=${blockmap.size}`);
+
+	const commit = readBlock(blockmap, roots[0], isCommit);
+
+	for (const { key, cid } of walkMstEntries(blockmap, commit.data)) {
+		const [collection, rkey] = key.split('/');
+
+		const carEntry = blockmap.get(cid.$link);
+		assert(carEntry != null, `cid not found in blockmap; cid=${cid}`);
+
+		yield new RepoEntry(collection, rkey, cid, carEntry);
+	}
+}

package/lib/v4/repo-reader/types.ts

@@ -0,0 +1,32 @@
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+
+import { type CarEntry } from '../car-reader/index.js';
+
+export class RepoEntry {
+	/** @internal */
+	constructor(
+		/** the collection this record belongs to */
+		public readonly collection: string,
+		/** record key */
+		public readonly rkey: string,
+		/** CID of this record */
+		public readonly cid: CID.CidLink,
+		/** the associated CarEntry for this record */
+		public readonly carEntry: CarEntry,
+	) {}
+
+	/**
+	 * raw contents of this record
+	 */
+	get bytes(): Uint8Array {
+		return this.carEntry.bytes;
+	}
+
+	/**
+	 * decoded contents of this record
+	 */
+	get record(): unknown {
+		return CBOR.decode(this.bytes);
+	}
+}

package/lib/v4/utils.ts

@@ -0,0 +1,7 @@
+export const assert: {
+	(condition: boolean, message: string): asserts condition;
+} = (condition, message) => {
+	if (!condition) {
+		throw new Error(message);
+	}
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "@atcute/car",
-  "version": "3.0.5",
+  "version": "3.1.1",
   "description": "lightweight DASL CAR and atproto repository decoder for AT Protocol.",
   "keywords": [
     "atproto",
@@ -20,7 +20,10 @@
     "!lib/**/*.test.ts"
   ],
   "exports": {
-    ".": "./dist/index.js"
+    ".": "./dist/v3/index.js",
+    "./v4": "./dist/v4/index.js",
+    "./v4/car-reader": "./dist/v4/car-reader/index.js",
+    "./v4/repo-reader": "./dist/v4/repo-reader/index.js"
   },
   "sideEffects": false,
   "devDependencies": {
@@ -28,8 +31,10 @@
     "@atcute/multibase": "^1.1.4"
   },
   "dependencies": {
+    "yocto-queue": "^1.2.1",
     "@atcute/cbor": "^2.2.4",
     "@atcute/cid": "^2.2.3",
+    "@atcute/uint8array": "^1.0.2",
     "@atcute/varint": "^1.0.2"
   },
   "scripts": {