@atcute/car 3.1.2 → 5.0.0

package/dist/v4/repo-reader/types.js

@@ -1,37 +0,0 @@
-import * as CBOR from '@atcute/cbor';
-import * as CID from '@atcute/cid';
-import {} from '../car-reader/index.js';
-export class RepoEntry {
-    collection;
-    rkey;
-    cid;
-    carEntry;
-    /** @internal */
-    constructor(
-    /** the collection this record belongs to */
-    collection, 
-    /** record key */
-    rkey, 
-    /** CID of this record */
-    cid, 
-    /** the associated CarEntry for this record */
-    carEntry) {
-        this.collection = collection;
-        this.rkey = rkey;
-        this.cid = cid;
-        this.carEntry = carEntry;
-    }
-    /**
-     * raw contents of this record
-     */
-    get bytes() {
-        return this.carEntry.bytes;
-    }
-    /**
-     * decoded contents of this record
-     */
-    get record() {
-        return CBOR.decode(this.bytes);
-    }
-}
-//# sourceMappingURL=types.js.map

package/dist/v4/repo-reader/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../lib/v4/repo-reader/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,cAAc,CAAC;AACrC,OAAO,KAAK,GAAG,MAAM,aAAa,CAAC;AAEnC,OAAO,EAAiB,MAAM,wBAAwB,CAAC;AAEvD,MAAM,OAAO,SAAS;IAIJ;IAEA;IAEA;IAEA;IATjB,gBAAgB;IAChB;IACC,4CAA4C;IAC5B,UAAkB;IAClC,iBAAiB;IACD,IAAY;IAC5B,yBAAyB;IACT,GAAgB;IAChC,8CAA8C;IAC9B,QAAkB;QANlB,eAAU,GAAV,UAAU,CAAQ;QAElB,SAAI,GAAJ,IAAI,CAAQ;QAEZ,QAAG,GAAH,GAAG,CAAa;QAEhB,aAAQ,GAAR,QAAQ,CAAU;IAChC,CAAC;IAEJ;;OAEG;IACH,IAAI,KAAK;QACR,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,IAAI,MAAM;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;CACD"}

package/dist/v4/utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../lib/v4/utils.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,MAAM,EAAE;IACpB,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;CAKzD,CAAC"}

package/dist/v4/utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../lib/v4/utils.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,MAAM,GAEf,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE;IAC1B,IAAI,CAAC,SAAS,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;AACF,CAAC,CAAC"}

package/lib/v3/atproto-repo.ts

@@ -1,15 +0,0 @@
-export {
-	collectBlock,
-	isCommit,
-	isMstNode,
-	isTreeEntry,
-	fromUint8Array as iterateAtpRepo,
-	readBlock,
-	RepoEntry,
-	walkMstEntries,
-	type BlockMap,
-	type Commit,
-	type MstNode,
-	type NodeEntry,
-	type TreeEntry,
-} from '../v4/repo-reader/index.js';

package/lib/v3/index.ts

@@ -1,2 +0,0 @@
-export * from './atproto-repo.js';
-export * from './reader.js';

package/lib/v3/reader.ts

@@ -1,9 +0,0 @@
-export {
-	isCarV1Header,
-	fromUint8Array as readCar,
-	type CarEntry,
-	type CarHeader,
-	type CarV1Header,
-	type StreamedCarReader,
-	type SyncCarReader,
-} from '../v4/car-reader/index.js';

package/lib/v4/car-reader/index.ts

@@ -1,4 +0,0 @@
-export * from './types.js';
-
-export * from './stream-car-reader.js';
-export * from './sync-car-reader.js';

package/lib/v4/index.ts

@@ -1,2 +0,0 @@
-export * as CarReader from './car-reader/index.js';
-export * as RepoReader from './repo-reader/index.js';

package/lib/v4/repo-reader/index.ts

@@ -1,6 +0,0 @@
-export * from './types.js';
-
-export * from './mst.js';
-export * from './stream-repo-reader.js';
-export * from './sync-blockmap.js';
-export * from './sync-repo-reader.js';

package/lib/v4/repo-reader/mst.ts

@@ -1,110 +0,0 @@
-import * as CBOR from '@atcute/cbor';
-import * as CID from '@atcute/cid';
-
-const isCidLink = (value: unknown): value is CID.CidLink => {
-	if (value instanceof CID.CidLinkWrapper) {
-		return true;
-	}
-
-	if (value === null || typeof value !== 'object') {
-		return false;
-	}
-
-	return '$link' in value && typeof value.$link === 'string';
-};
-
-const isBytes = (value: unknown): value is CBOR.Bytes => {
-	if (value instanceof CBOR.BytesWrapper) {
-		return true;
-	}
-
-	if (value === null || typeof value !== 'object') {
-		return false;
-	}
-
-	return '$bytes' in value && typeof value.$bytes === 'string';
-};
-
-/** commit object */
-export interface Commit {
-	version: 3;
-	did: string;
-	data: CID.CidLink;
-	rev: string;
-	prev: CID.CidLink | null;
-	sig: CBOR.Bytes;
-}
-
-/**
- * checks if a value is a valid commit object
- * @param value the value to check
- * @returns true if the value is a valid commit object, false otherwise
- */
-export const isCommit = (value: unknown): value is Commit => {
-	if (value === null || typeof value !== 'object') {
-		return false;
-	}
-
-	const obj = value as Record<string, unknown>;
-
-	return (
-		obj.version === 3 &&
-		typeof obj.did === 'string' &&
-		isCidLink(obj.data) &&
-		typeof obj.rev === 'string' &&
-		(obj.prev === null || isCidLink(obj.prev)) &&
-		isBytes(obj.sig)
-	);
-};
-
-/** mst tree entry object */
-export interface TreeEntry {
-	/** count of bytes shared with previous TreeEntry in this Node (if any) */
-	p: number;
-	/** remainder of key for this TreeEntry, after "prefixlen" have been removed */
-	k: CBOR.Bytes;
-	/** link to a sub-tree Node at a lower level which has keys sorting after this TreeEntry's key (to the "right"), but before the next TreeEntry's key in this Node (if any) */
-	v: CID.CidLink;
-	/** next subtree (to the right of leaf) */
-	t: CID.CidLink | null;
-}
-
-/**
- * checks if a value is a valid mst tree entry object
- * @param value the value to check
- * @returns true if the value is a valid mst tree entry object, false otherwise
- */
-export const isTreeEntry = (value: unknown): value is TreeEntry => {
-	if (value === null || typeof value !== 'object') {
-		return false;
-	}
-
-	const obj = value as Record<string, unknown>;
-
-	return (
-		typeof obj.p === 'number' && isBytes(obj.k) && isCidLink(obj.v) && (obj.t === null || isCidLink(obj.t))
-	);
-};
-
-/** mst node object */
-export interface MstNode {
-	/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
-	l: CID.CidLink | null;
-	/** ordered list of TreeEntry objects */
-	e: TreeEntry[];
-}
-
-/**
- * checks if a value is a valid mst node object
- * @param value the value to check
- * @returns true if the value is a valid mst node object, false otherwise
- */
-export const isMstNode = (value: unknown): value is MstNode => {
-	if (value === null || typeof value !== 'object') {
-		return false;
-	}
-
-	const obj = value as Record<string, unknown>;
-
-	return (obj.l === null || isCidLink(obj.l)) && Array.isArray(obj.e) && obj.e.every(isTreeEntry);
-};

package/lib/v4/repo-reader/stream-repo-reader.ts

@@ -1,209 +0,0 @@
-import Queue from 'yocto-queue';
-
-import * as CBOR from '@atcute/cbor';
-import * as CID from '@atcute/cid';
-import { decodeUtf8From } from '@atcute/uint8array';
-
-import * as CarReader from '../car-reader/index.js';
-import { assert } from '../utils.js';
-
-import { isCommit, isMstNode } from './mst.js';
-import { RepoEntry } from './types.js';
-
-type EntryMeta = { t: 0 } | { t: 1 } | { t: 2; k: string };
-
-type Task = {
-	c: string;
-	e: CarReader.CarEntry;
-	m: EntryMeta;
-};
-
-export type MissingBlockEntry =
-	| {
-			cid: string;
-			type: 'record';
-			key: string;
-	  }
-	| {
-			cid: string;
-			type: 'mst-node';
-	  }
-	| {
-			cid: string;
-			type: 'commit';
-	  };
-
-export interface StreamedRepoReader {
-	/**
-	 * list of blocks that were referenced but not found in the repository.
-	 * blocks may be reported as missing if multiple records share the same CID.
-	 */
-	readonly missingBlocks: readonly MissingBlockEntry[];
-
-	dispose(): Promise<void>;
-
-	[Symbol.asyncDispose](): Promise<void>;
-	[Symbol.asyncIterator](): AsyncIterator<RepoEntry>;
-}
-
-export const repoEntryTransform = (): ReadableWritablePair<RepoEntry, Uint8Array> => {
-	const transform = new TransformStream<Uint8Array, Uint8Array>();
-	let repo: StreamedRepoReader | undefined;
-
-	return {
-		readable: new ReadableStream({
-			async start(controller) {
-				repo = fromStream(transform.readable);
-
-				try {
-					for await (const entry of repo) {
-						controller.enqueue(entry);
-					}
-
-					await repo.dispose();
-
-					controller.close();
-				} catch (err) {
-					controller.error(err);
-				}
-			},
-			async cancel() {
-				if (repo !== undefined) {
-					await repo.dispose();
-				}
-			},
-		}),
-		writable: transform.writable,
-	};
-};
-
-export const fromStream = (stream: ReadableStream<Uint8Array>): StreamedRepoReader => {
-	let missingBlocks: MissingBlockEntry[] = [];
-
-	return {
-		get missingBlocks() {
-			return missingBlocks;
-		},
-
-		async dispose() {
-			// does nothing for now
-		},
-
-		[Symbol.asyncDispose]() {
-			return this.dispose();
-		},
-		async *[Symbol.asyncIterator]() {
-			// await using car = CarReader.fromStream(stream);
-			const car = CarReader.fromStream(stream);
-
-			try {
-				const pending = new Map<string, EntryMeta>();
-				const strays = new Map<string, CarReader.CarEntry>();
-
-				const queue = new Queue<Task>();
-
-				const request = (cid: string, meta: EntryMeta): void => {
-					const entry = strays.get(cid);
-
-					if (entry !== undefined) {
-						strays.delete(cid);
-						queue.enqueue({ c: cid, e: entry, m: meta });
-					} else {
-						pending.set(cid, meta);
-					}
-				};
-
-				{
-					const roots = await car.roots();
-					assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
-
-					const rootCid = roots[0].$link;
-					request(rootCid, { t: 0 });
-				}
-
-				for await (const entry of car) {
-					const cid = CID.toString(entry.cid);
-
-					{
-						const meta = pending.get(cid);
-
-						if (meta !== undefined) {
-							pending.delete(cid);
-							queue.enqueue({ c: cid, e: entry, m: meta });
-						} else {
-							strays.set(cid, entry);
-						}
-					}
-
-					let task: Task | undefined;
-					while ((task = queue.dequeue())) {
-						const { c: cid, e: entry, m: meta } = task;
-
-						switch (meta.t) {
-							case 0: {
-								const commit = CBOR.decode(entry.bytes);
-								assert(isCommit(commit), `expected commit block; cid=${cid}`);
-
-								request(commit.data.$link, { t: 1 });
-								break;
-							}
-							case 1: {
-								const node = CBOR.decode(entry.bytes);
-								assert(isMstNode(node), `expected mst node block; cid=${cid}`);
-
-								const entries = node.e;
-								const left = node.l;
-
-								let lastKey = '';
-
-								if (left !== null) {
-									request(left.$link, meta);
-								}
-
-								for (let i = 0, il = entries.length; i < il; i++) {
-									const entry = entries[i];
-									const next = entry.t;
-
-									const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
-									const key = lastKey.slice(0, entry.p) + key_str;
-
-									lastKey = key;
-
-									request(entry.v.$link, { t: 2, k: key });
-
-									if (next !== null) {
-										request(next.$link, { t: 1 });
-									}
-								}
-
-								break;
-							}
-							case 2: {
-								const [collection, rkey] = meta.k.split('/');
-
-								yield new RepoEntry(collection, rkey, CID.toCidLink(entry.cid), entry);
-								break;
-							}
-						}
-					}
-				}
-
-				missingBlocks = Array.from(pending, ([cid, meta]): MissingBlockEntry => {
-					switch (meta.t) {
-						case 0: {
-							return { cid, type: 'commit' };
-						}
-						case 1: {
-							return { cid, type: 'mst-node' };
-						}
-						case 2: {
-							return { cid, type: 'record', key: meta.k };
-						}
-					}
-				});
-			} finally {
-				await car.dispose();
-			}
-		},
-	};
-};

package/lib/v4/repo-reader/sync-blockmap.ts

@@ -1,85 +0,0 @@
-import * as CBOR from '@atcute/cbor';
-import * as CID from '@atcute/cid';
-import { decodeUtf8From } from '@atcute/uint8array';
-
-import * as CarReader from '../car-reader/index.js';
-import { assert } from '../utils.js';
-
-import { isMstNode } from './mst.js';
-
-export type BlockMap = Map<string, CarReader.CarEntry>;
-
-/**
- * collects entries from a CAR archive into a mapping of CID string -> actual bytes
- * @param iterator a generator that yields objects with a `cid` and `bytes` property
- * @returns a mapping of CID string -> actual bytes
- */
-export const collectBlock = (iterator: Iterable<CarReader.CarEntry>): BlockMap => {
-	const blockmap: BlockMap = new Map();
-	for (const entry of iterator) {
-		blockmap.set(CID.toString(entry.cid), entry);
-	}
-
-	return blockmap;
-};
-
-/**
- * reads a block from the blockmap and validates it against the provided validation function
- * @param map a mapping of CID string -> actual bytes
- * @param link a CID link to read
- * @param validate a validation function to validate the decoded data
- * @returns the decoded and validated data
- */
-export const readBlock = <T>(
-	map: BlockMap,
-	link: CID.CidLink,
-	validate: (value: unknown) => value is T,
-): T => {
-	const cid = link.$link;
-
-	const entry = map.get(cid);
-	assert(entry != null, `cid not found in blockmap; cid=${cid}`);
-
-	const data = CBOR.decode(entry.bytes);
-	assert(validate(data), `validation failed for cid=${cid}`);
-
-	return data;
-};
-
-/** node entry object */
-export interface NodeEntry {
-	key: string;
-	cid: CID.CidLink;
-}
-
-/**
- * walks the entries of a Merkle Sorted Tree (MST) in a depth-first manner
- * @param map a mapping of CID string -> actual bytes
- * @param pointer a CID link to the root of the MST
- * @returns a generator that yields the entries of the MST
- */
-export function* walkMstEntries(map: BlockMap, pointer: CID.CidLink): Generator<NodeEntry> {
-	const data = readBlock(map, pointer, isMstNode);
-	const entries = data.e;
-
-	let lastKey = '';
-
-	if (data.l !== null) {
-		yield* walkMstEntries(map, data.l);
-	}
-
-	for (let i = 0, il = entries.length; i < il; i++) {
-		const entry = entries[i];
-
-		const key_str = decodeUtf8From(CBOR.fromBytes(entry.k));
-		const key = lastKey.slice(0, entry.p) + key_str;
-
-		lastKey = key;
-
-		yield { key: key, cid: entry.v };
-
-		if (entry.t !== null) {
-			yield* walkMstEntries(map, entry.t);
-		}
-	}
-}

package/lib/v4/repo-reader/sync-repo-reader.ts

@@ -1,27 +0,0 @@
-import * as CarReader from '../car-reader/index.js';
-import { assert } from '../utils.js';
-
-import { isCommit } from './mst.js';
-import { collectBlock, readBlock, walkMstEntries } from './sync-blockmap.js';
-import { RepoEntry } from './types.js';
-
-export function* fromUint8Array(buf: Uint8Array): Generator<RepoEntry> {
-	const car = CarReader.fromUint8Array(buf);
-	const roots = car.roots;
-
-	assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
-
-	const blockmap = collectBlock(car);
-	assert(blockmap.size > 0, `expected at least 1 block in the archive; got=${blockmap.size}`);
-
-	const commit = readBlock(blockmap, roots[0], isCommit);
-
-	for (const { key, cid } of walkMstEntries(blockmap, commit.data)) {
-		const [collection, rkey] = key.split('/');
-
-		const carEntry = blockmap.get(cid.$link);
-		assert(carEntry != null, `cid not found in blockmap; cid=${cid}`);
-
-		yield new RepoEntry(collection, rkey, cid, carEntry);
-	}
-}

package/lib/v4/repo-reader/types.ts

@@ -1,32 +0,0 @@
-import * as CBOR from '@atcute/cbor';
-import * as CID from '@atcute/cid';
-
-import { type CarEntry } from '../car-reader/index.js';
-
-export class RepoEntry {
-	/** @internal */
-	constructor(
-		/** the collection this record belongs to */
-		public readonly collection: string,
-		/** record key */
-		public readonly rkey: string,
-		/** CID of this record */
-		public readonly cid: CID.CidLink,
-		/** the associated CarEntry for this record */
-		public readonly carEntry: CarEntry,
-	) {}
-
-	/**
-	 * raw contents of this record
-	 */
-	get bytes(): Uint8Array {
-		return this.carEntry.bytes;
-	}
-
-	/**
-	 * decoded contents of this record
-	 */
-	get record(): unknown {
-		return CBOR.decode(this.bytes);
-	}
-}