@wovin/core 0.1.36 → 0.2.2

package/src/applog/datom-types.ts

@@ -0,0 +1,167 @@
+// import type { AgentHash } from '../pubsub/pubsub-types.ts'
+// import type { CID } from '@oddjs/odd'
+import { FormatRegistry, Static, TSchema, Type } from '@sinclair/typebox'
+import { TypeCompiler } from '@sinclair/typebox/compiler'
+import { CID } from 'multiformats/cid'
+import type { PartialBy, Tagged } from '../types/typescript-utils.ts'
+
+export const Nullable = <T extends TSchema>(schema: T) => Type.Union([schema, Type.Null()])
+export const EntityID_LENGTH = 7
+// const bagu = 'baguqeerav3h4b46j2pyxikqhtm5si5vhzsyrba2duhrtltfutrlmj42anmvq'
+// const k51q = 'k51qzi5uqu5dhe1bxxjxj144bj2a225o1681yobevns26xlxtsfidjgnpwknfd'
+const isCID = /^(k51qz|baguq)[0-9a-z]{56,57}$/ // FIXME: k51 is not really a CID, is it?
+const isShortHash = /^[0-9A-Fa-f]{7,8}$/g // TODO awkward why are some 7 and some 8 long
+// engine level: min 6 (lenient within reason)
+// note3 TBD... either fixed for all entity types VS 7 for pub/sub, 8 for tags, 9 for blocks, 10 for relations etc...
+
+FormatRegistry.Set('EntityID', (value) => !!value.match(isShortHash) || !!value.match(isCID))
+export const EntityID = Type.String() // HACK how to configure ID format?
+/*{  format: 'EntityID'  }*/
+export type EntityID = Static<typeof EntityID>
+
+export type AgentHash = Tagged<string, 'AgentHash'>
+export type DatomPart = string // TODO refactor
+export type CidString = Tagged<string, CID>
+export type IpnsString = Tagged<CidString, 'IPNS'>
+export type AgentID = EntityID
+export type Attribute = string
+// JSON-serializable value space for a datom's `vl` field.
+export type JsonPrimitive = string | boolean | number | null
+export interface JsonObject {
+	[key: string]: JsonValue
+}
+export type JsonArray = JsonValue[]
+export type JsonValue = JsonPrimitive | JsonObject | JsonArray
+export type ApplogValue = JsonValue // TODO: use Tagged types
+
+export interface Atom {
+	en: EntityID
+	at: Attribute
+	vl: ApplogValue
+}
+
+export type Timestamp = string
+export interface Applog extends Atom {
+	cid: CidString
+	pv: CidString | null // ? | CID
+	ts: Timestamp
+	ag: AgentHash
+}
+export type ApplogNoCid = Omit<Applog, 'cid'>
+export type ApplogOptionalCid = PartialBy<Applog, 'cid'>
+export type ApplogForInsert = PartialBy<ApplogNoCid, 'ts' | 'pv'>
+export type ApplogForInsertOptionalAgent = PartialBy<ApplogForInsert, 'ag'>
+
+export interface ApplogEnc {
+	cid: CidString
+	enc: Uint8Array
+	iv?: Uint8Array // iv may be needed if we prefer a different strategy for transmitting iv (odd appends a random iv to the payload via keystoreAES.(en|de)cryptBytes)
+}
+export type ApplogEncNoCid = Omit<ApplogEnc, 'cid'>
+
+export type ApplogArrayMaybeEncrypted = (Applog | ApplogEnc)[]
+export type ApplogArrayMaybeEncryptedRO = readonly (Applog | ApplogEnc)[]
+export type ApplogArrayNoCIDMaybeEncryptedRO = readonly (ApplogNoCid | ApplogEncNoCid)[]
+
+export type ApplogOfSomeSort = Applog | ApplogEnc | ApplogNoCid | ApplogEncNoCid | ApplogForInsert
+
+export const isEncryptedApplog = (l: ApplogOfSomeSort): l is ApplogEnc => (l as ApplogEnc)?.enc instanceof Uint8Array
+
+export type AtomPattern = Atom | Applog
+
+export interface DatalogStateIdentifier {
+	lastTS: Timestamp
+}
+
+// New generic type for fields that can be a value, an array of that, or a function
+export type ValueOrMatcher<T> = T | readonly T[] | ReadonlySet<T> | ((value: T) => boolean)
+// Generic type that applies ValueOrMatcher to each field of T
+export type WithMatchers<T extends Record<string, any>> = {
+	[K in keyof T & string as `${K}` | `!${K}`]?: ValueOrMatcher<T[K]>
+}
+
+export type DatalogQueryPattern = Partial<WithMatchers<Applog>>
+export type DatalogQueryPatternArray = DatalogQueryPattern[]
+export interface DatalogQuery<SELECT extends string> {
+	find: readonly SELECT[] // see: https://github.com/microsoft/TypeScript/issues/20965#issuecomment-868981458
+	where: DatalogQueryPatternArray
+	onlyLatest?: boolean
+}
+export type DatalogQueryResultEntry<SELECT extends string> = Record<
+	// SELECT,
+	StripPrefix<'?', SELECT>,
+	DatomPart
+>
+export type DatalogQueryResultRows<SELECT extends string> = DatalogQueryResultEntry<SELECT>[]
+// export type StripTest = StripPrefix<'?', '?A' | '?B'>
+// export type DatalogQueryResultEntryTEST = DatalogQueryResultEntry<'?A' | '?B'>
+// export type DatalogQueryResultEntryTESTX = MapKeysStripPrefix<'?A' | '?B', '?'>
+
+export interface SearchContext {
+	[key: string]: ApplogValue
+}
+export interface SearchContextWithLog {
+	context: SearchContext
+	applog?: Applog
+}
+
+export type ResultContext = SearchContext | null
+
+/* https://stackoverflow.com/a/72497461 */
+type StripPrefix<
+	TPrefix extends string,
+	T extends string,
+> = T extends `${TPrefix}${infer R}` ? R : never
+
+type MapKeysStripPrefix<SELECT extends string, TPrefix extends string> = {
+	[K in SELECT as StripPrefix<TPrefix, K>]: DatomPart
+}
+
+FormatRegistry.Set('CID', (value) => !!value.match(isCID))
+export const CIDTB = Type.String({ format: 'CID' })
+export type CIDTB = Static<typeof EntityID>
+
+const isURL = /^http([s]?):\/\/.*\..*/
+FormatRegistry.Set('URL', (value) => !!value.match(isURL))
+export const URL = Type.String({ format: 'URL' })
+export type URL = Static<typeof URL>
+
+// Recursive JSON value: primitives plus nested arrays/objects (mirrors ApplogValue / JsonValue).
+export const JsonValueTB = Type.Recursive(This =>
+	Type.Union([
+		Type.String(),
+		Type.Number(),
+		Type.Boolean(),
+		Type.Null(),
+		Type.Array(This),
+		Type.Record(Type.String(), This),
+	])
+)
+export type JsonValueTB = Static<typeof JsonValueTB>
+
+export const AppLogNoCidTB = Type.Object({
+	en: EntityID, // EntityID
+	at: Type.String(), // Attribute
+	vl: JsonValueTB, // ApplogValue (JSON-serializable: primitives, arrays, objects)
+	ts: Type.String(), // Timestamp
+	ag: Type.String(), // AgentHash
+	pv: Nullable(CIDTB), // CidString
+})
+export type AppLogNoCidTB = Static<typeof AppLogNoCidTB> // type T = {
+
+export const AppLogNoCidTBC = TypeCompiler.Compile(AppLogNoCidTB)
+export const getApplogNoCidTypeErrors = (obj: any) => Array.from(AppLogNoCidTBC.Errors(obj))
+export const isValidApplogNoCid = AppLogNoCidTBC.Check.bind(AppLogNoCidTBC) // ? Include CID
+
+export const AppLogTB = Type.Composite([
+	Type.Object({
+		cid: CIDTB,
+	}),
+	AppLogNoCidTB,
+])
+export type AppLogTB = Static<typeof AppLogTB> // type T = {
+
+export const AppLogTBC = TypeCompiler.Compile(AppLogTB)
+export const getApplogTypeErrors = (obj: any) => Array.from(AppLogTBC.Errors(obj))
+export const isValidApplog = AppLogTBC.Check.bind(AppLogTBC) // ? Include CID
+// maybe useful for defaulting https://github.com/sinclairzx81/typebox#cast

package/src/applog/object-values.test.ts

@@ -0,0 +1,106 @@
+/**
+ * Tests for JSON object/array support in a datom's `vl` field.
+ *
+ * Covers: schema validation, deep-equality dedup, value-index keying,
+ * matcher semantics (deep-eq literal match, anyOf membership, predicate
+ * for nested access) and the fail-loud behaviour for bare-array patterns.
+ */
+import { describe, expect, it } from 'vitest'
+import { isValidApplog } from './datom-types.ts'
+import { matchPartStatic, valueEq, valueKey } from './applog-utils.ts'
+import { anyOf } from '../query/matchers.ts'
+import { finalizeApplogForInsert } from './applog-helpers.ts'
+import { applogsByAttrValue } from '../thread/indexes.ts'
+import { ThreadInMemory } from '../thread/writeable.ts'
+import type { Applog, ApplogForInsert } from './datom-types.ts'
+
+let tsCounter = 0
+function makeLog(spec: Pick<ApplogForInsert, 'en' | 'at' | 'vl'> & Partial<Applog>): Applog {
+	tsCounter++
+	return finalizeApplogForInsert({
+		ts: new Date(1700000000000 + tsCounter * 1000).toISOString(),
+		pv: null,
+		ag: 'testAgent',
+		...spec,
+	} as ApplogForInsert, {})
+}
+
+describe('object/array applog values', () => {
+	it('schema accepts nested objects and arrays as vl', () => {
+		const obj = makeLog({ en: 'e1', at: 'config', vl: { theme: 'dark', nested: { n: [1, 2, 3] } } })
+		const arr = makeLog({ en: 'e1', at: 'tags', vl: ['a', 'b', { x: 1 }] })
+		expect(isValidApplog(obj)).toBe(true)
+		expect(isValidApplog(arr)).toBe(true)
+		// primitives still valid
+		expect(isValidApplog(makeLog({ en: 'e1', at: 'name', vl: 'hi' }))).toBe(true)
+		expect(isValidApplog(makeLog({ en: 'e1', at: 'n', vl: null }))).toBe(true)
+	})
+
+	it('valueEq compares object/array values by structure', () => {
+		expect(valueEq({ a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true)
+		expect(valueEq([1, 2, 3], [1, 2, 3])).toBe(true)
+		expect(valueEq({ a: 1 }, { a: 2 })).toBe(false)
+		expect(valueEq('x', 'x')).toBe(true)
+	})
+
+	it('valueKey canonicalizes objects (key-order independent) without colliding with strings', () => {
+		expect(valueKey({ a: 1, b: 2 })).toBe(valueKey({ b: 2, a: 1 }))
+		// a literal string equal to an object's serialization must NOT collide
+		expect(valueKey('{"a":1}')).not.toBe(valueKey({ a: 1 }))
+		// primitives pass through as their own value
+		expect(valueKey('hi')).toBe('hi')
+		expect(valueKey(42)).toBe(42)
+		expect(valueKey(null)).toBe(null)
+	})
+
+	it('hasApplogWithDiffTs matches structurally-equal object values (deep, key-order independent)', () => {
+		const existing = makeLog({ en: 'e1', at: 'config', vl: { a: 1, b: 2 } })
+		const thread = ThreadInMemory.fromArray([existing], 'dedup')
+		// same en/at/ag, deep-equal vl with different key order → recognised as the same datom
+		expect(thread.hasApplogWithDiffTs({ en: 'e1', at: 'config', vl: { b: 2, a: 1 }, ag: 'testAgent' } as any)).toBeTruthy()
+		// a structurally different value is NOT matched
+		expect(thread.hasApplogWithDiffTs({ en: 'e1', at: 'config', vl: { a: 9 }, ag: 'testAgent' } as any)).toBeFalsy()
+	})
+
+	it('applogsByAttrValue groups structurally-equal object values into one bucket', () => {
+		const logs = [
+			makeLog({ en: 'e1', at: 'config', vl: { a: 1, b: 2 } }),
+			makeLog({ en: 'e2', at: 'config', vl: { b: 2, a: 1 } }),
+			makeLog({ en: 'e3', at: 'config', vl: { a: 9 } }),
+		]
+		const thread = ThreadInMemory.fromArray(logs, 'idx')
+		const index = applogsByAttrValue(thread, 'config').value
+		expect(index.get(valueKey({ a: 1, b: 2 }))).toHaveLength(2)
+		expect(index.get(valueKey({ a: 9 }))).toHaveLength(1)
+	})
+
+	it('matchPartStatic matches a literal object value by deep equality', () => {
+		expect(matchPartStatic('vl', { a: 1, b: 2 }, { b: 2, a: 1 })).toBe(true)
+		expect(matchPartStatic('vl', { a: 1 }, { a: 2 })).toBe(false)
+	})
+
+	it('matching a literal array value requires a predicate (a bare array throws)', () => {
+		// bare arrays are reserved/rejected, so an array value is matched via a predicate
+		expect(matchPartStatic('vl', (v: any) => valueEq(v, [1, 2, 3]), [1, 2, 3])).toBe(true)
+		expect(() => matchPartStatic('vl', [1, 2, 3] as any, [1, 2, 3])).toThrow(/anyOf/)
+	})
+
+	it('anyOf(...) provides set-membership and matches via matchPartStatic', () => {
+		expect(matchPartStatic('at', anyOf('a', 'b', 'c'), 'b')).toBe(true)
+		expect(matchPartStatic('at', anyOf('a', 'b'), 'z')).toBe(false)
+	})
+
+	it('a predicate matcher can reach into nested object values', () => {
+		const isDark = (v: any) => v?.theme === 'dark'
+		expect(matchPartStatic('vl', isDark, { theme: 'dark' })).toBe(true)
+		expect(matchPartStatic('vl', isDark, { theme: 'light' })).toBe(false)
+	})
+
+	it('a bare array pattern fails loudly rather than matching ambiguously', () => {
+		expect(() => matchPartStatic('at', ['a', 'b'] as any, 'a')).toThrow(/anyOf/)
+	})
+
+	it('anyOf rejects object/array members (would silently never match)', () => {
+		expect(() => anyOf({ a: 1 } as any)).toThrow(/object\/array/)
+	})
+})

package/src/applog.ts

@@ -0,0 +1,3 @@
+export * from './applog/applog-helpers.ts'
+export * from './applog/applog-utils.ts'
+export * from './applog/datom-types.ts'

package/src/blockstore/index.ts

@@ -0,0 +1,36 @@
+import type { CID } from 'multiformats/cid'
+
+/** Minimal async block store interface — get/put/has. */
+export interface BlockStore {
+	get(cid: CID): Promise<Uint8Array>
+	put(cid: CID, bytes: Uint8Array): Promise<void>
+	has(cid: CID): Promise<boolean>
+}
+
+/**
+ * A block store that reads locally first, with optional remote fallback.
+ * On get: local hit → return; local miss + remote → fetch, write-back, return.
+ * put/has always operate on local only.
+ */
+export class LocalFirstBlockStore implements BlockStore {
+	constructor(
+		private local: BlockStore,
+		private remote?: Pick<BlockStore, 'get'>,
+	) {}
+
+	async get(cid: CID): Promise<Uint8Array> {
+		if (await this.local.has(cid)) return this.local.get(cid)
+		if (!this.remote) throw new Error(`Block not found: ${cid}`)
+		const bytes = await this.remote.get(cid)
+		await this.local.put(cid, bytes)
+		return bytes
+	}
+
+	put(cid: CID, bytes: Uint8Array): Promise<void> {
+		return this.local.put(cid, bytes)
+	}
+
+	has(cid: CID): Promise<boolean> {
+		return this.local.has(cid)
+	}
+}

package/src/blockstore.ts

@@ -0,0 +1 @@
+export * from './blockstore/index.ts'

package/src/index.ts

@@ -0,0 +1,8 @@
+export * from './applog.ts'
+export * from './ipfs.ts'
+export * from './pubsub.ts'
+export * from './query.ts'
+export * from './retrieve.ts'
+export * from './thread.ts'
+export * from './types.ts'
+// export * from './utils.ts'

package/src/ipfs/car.ts

@@ -0,0 +1,291 @@
+import { CarReader, CarWriter } from '@ipld/car'
+import * as dagJson from '@ipld/dag-json'
+import { Logger } from 'besonders-logger'
+import { BlockView, CID } from 'multiformats'
+import { sortApplogsByTs } from '../applog/applog-utils.ts'
+import { Applog, ApplogArrayMaybeEncrypted, CidString } from '../applog/datom-types.ts'
+import { unchunkApplogsBlock } from '../pubsub/snap-push.ts'
+import { SnapBlockLogs, SnapBlockLogsOrChunks, SnapRootBlock } from '../pubsub/pubsub-types.ts'
+import { areCidsEqual, containsCid } from './ipfs-utils.ts'
+
+const { WARN, LOG, DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO) // eslint-disable-line no-unused-vars
+
+export type CIDForCar = CID // Exclude<Parameters<(typeof CarWriter)['create']>[0], void>
+export type BlockForCar = Parameters<CarWriter['put']>[0]
+
+export interface BlockStoreish {
+	get(cid: CID): PromiseLike<Uint8Array> // (i) not using decoded version to be similar to blockstore-idb
+}
+
+export interface DecodedCar {
+	rootCID: CID
+	// blocks: Map<CidString, any>
+	blockStore: BlockStoreish
+}
+
+/** Warning: unsorted & maybe encrypted */
+export async function decodePubFromCar(car: CarReader) {
+	const decoded = await getBlocksOfCar(car)
+	return await decodePubFromBlocks(decoded)
+}
+
+export async function decodePubFromBlocks(
+	{ rootCID, blockStore }: DecodedCar,
+	_recursionTrace: CID[] = [],  // DEPRECATED: kept for API compat, unused in iterative version
+	stopAtCID?: CID  // NEW: stop iteration when we hit this CID
+) {
+	if (!rootCID || !blockStore) {
+		throw ERROR('Empty roots/blocks', { rootCID, blockStore })
+	}
+
+	let allApplogs: ApplogArrayMaybeEncrypted = []
+	let firstInfo: { logs: CID[] } | null = null
+	let currentCID: CID | undefined = rootCID
+	const visited = new Set<string>()  // Loop detection (replaces recursionTrace)
+	let applogsCID: CID | null = null  // From first snapshot only
+
+	while (currentCID) {
+		const cidStr = currentCID.toString()
+
+		// Loop detection
+		if (visited.has(cidStr)) {
+			throw ERROR('[decodePubFromBlocks] pub chain has a loop', {
+				currentCID: cidStr,
+				visited: [...visited]
+			})
+		}
+		visited.add(cidStr)
+
+		// Decode current snapshot
+		const root = (await getDecodedBlock(blockStore, currentCID)) as SnapRootBlock
+		VERBOSE(`[decodePubFromBlocks] root:`, cidStr, root, { blockStore })
+		if (!root) {
+			throw ERROR('[decodePubFromBlocks] root not found in blockStore', { blockStore, currentCID: cidStr })
+		}
+
+		// Decode applogs for this snapshot
+		let pubLogsArray: CID[]
+		if (root?.info) {
+			// New(er) format
+			if (!applogsCID) applogsCID = root.applogs  // Save from first snapshot
+			const applogsBlock = (await getDecodedBlock(blockStore, root.applogs)) as SnapBlockLogsOrChunks
+			pubLogsArray = await unchunkApplogsBlock(applogsBlock, blockStore)
+			// Info only from first (most recent) snapshot
+			if (!firstInfo) {
+				firstInfo = (await getDecodedBlock(blockStore, root.info)) as SnapBlockLogs
+				DEBUG(`new format - infoLogs`, firstInfo.logs.map(l => ({ [l.toString()]: l })))
+			}
+			// TODO: verify signatures
+		} else {
+			// Old format
+			pubLogsArray = root.applogs as any as CID[]
+		}
+
+		const resolveLogFromCidLink = async (cidOrLink: CID) => {
+			const cid = cidOrLink
+			const applog = (await getDecodedBlock(blockStore, cid)) as Applog
+			if (!applog) {
+				ERROR(`Could not find applog CID in pub blocks:`, cid.toString(), { cid, root, blockStore })
+				throw new Error(`Could not find applog CID in pub blocks: ${cid.toString()}`)
+			}
+			if ((applog.pv as any) instanceof CID) applog.pv = (applog.pv as any as CID).toV1().toString()
+			return {
+				...applog,
+				cid: cid.toV1().toString(),
+			}
+		}
+
+		const snapshotApplogs = await Promise.all(pubLogsArray.map(resolveLogFromCidLink))
+		allApplogs = allApplogs.concat(snapshotApplogs)
+
+		// Check if we should stop
+		if (!root.prev) {
+			break  // End of chain
+		}
+		if (stopAtCID && areCidsEqual(root.prev, stopAtCID)) {
+			DEBUG('[decodePubFromBlocks] stopping at stopAtCID:', stopAtCID.toString())
+			break  // Reached already-pulled snapshot
+		}
+
+		// Verify prev exists before continuing
+		const prevBytes = await blockStore.get(root.prev)
+		if (!prevBytes) {
+			throw ERROR('[decodePubFromBlocks] prev snapshot missing from blockStore', {
+				currentCID: cidStr,
+				prev: root.prev.toString(),
+				stopAtCID: stopAtCID?.toString(),
+				visited: [...visited]
+			})
+		}
+
+		currentCID = root.prev  // Move to previous snapshot
+	}
+
+	const result = {
+		cid: rootCID,
+		info: firstInfo ? {
+			...firstInfo,
+			logs: await Promise.all(firstInfo.logs.map(async (cidOrLink: CID) => {
+				const cid = cidOrLink
+				const applog = (await getDecodedBlock(blockStore, cid)) as Applog
+				if (!applog) {
+					ERROR(`Could not find info log CID in pub blocks:`, cid.toString(), { cid, blockStore })
+					throw new Error(`Could not find info log CID in pub blocks: ${cid.toString()}`)
+				}
+				if ((applog.pv as any) instanceof CID) applog.pv = (applog.pv as any as CID).toV1().toString()
+				return {
+					...applog,
+					cid: cid.toV1().toString(),
+				}
+			})),
+		} : null,
+		applogsCID,
+		applogs: allApplogs,
+	}
+	DEBUG('[decodePubFromBlocks] result:', result, { rootCID: rootCID.toString(), blockStore, applogs: allApplogs })
+	return result
+}
+
+export async function getBlocksOfCar(car: CarReader) {
+	const rootsFromCar = await car.getRoots()
+	const roots = rootsFromCar.map(c => ((typeof c.toV1 === 'function') ? c : CID.decode(c.bytes)).toV1().toString() as CidString) // HACK
+	const blocks = new Map<CidString, any>()
+	for await (const { cid: cidFromCarblocks, bytes } of car.blocks()) {
+		const cid = (typeof cidFromCarblocks.toV1 === 'function') ? cidFromCarblocks : CID.decode(cidFromCarblocks.bytes)
+		VERBOSE({ cidFromCarblocks, cid })
+		// blocks.set(cid.toV1().toString(), dagJson.decode(bytes)) // HACK: tried using CID as map key, but because it's based on referential equality it's not working
+		blocks.set(cid.toV1().toString(), bytes) // HACK: tried using CID as map key, but because it's based on referential equality it's not working
+	}
+	if (roots.length !== 1) {
+		WARN('Unexpected roots count:', roots)
+	}
+	return {
+		rootCID: CID.parse(roots[0]),
+		blockStore: {
+			get: (cid) => blocks.get(cid.toV1().toString()),
+		},
+	} satisfies DecodedCar
+}
+export async function getDecodedBlock(blockStore: BlockStoreish, cid: CID) {
+	try {
+		var blob = await blockStore.get(cid)
+		if (!blob) {
+			WARN('returning null')
+			return null // I don't think this ever happens actually
+		}
+	} catch (err) {
+		if ((err as any).message === 'Not Found') return null
+		throw err
+	}
+	return dagJson.decode(blob)
+}
+
+// make out in the car... been a while but also sounds nice
+export async function makeCarOut(roots: CIDForCar, blocks: BlockForCar[]) {
+	const { writer, out } = CarWriter.create(Array.isArray(roots) ? roots : [roots])
+
+	// add the blocks to the CAR and close it
+	VERBOSE(`Writing ${blocks.length} blocks to CAR`, { roots, blocks })
+	blocks.forEach(b => writer.put(b))
+	writer.close()
+	// VERBOSE(`Wrote ${blocks.length} blocks to CAR`, writer)
+	return out
+} /** create a new CarWriter, with the encoded block as the root */
+
+// export async function makeCarReader(roots: CIDForCar, blocks: BlockForCar[]) {
+// 	const out = await makeCarOut(roots, blocks)
+
+// 	// create a new CarReader we can hand to web3.storage.putCar
+// 	const reader = await CarReader.fromIterable(out)
+// 	VERBOSE(`CAR reader`, reader)
+// 	return reader
+// } /** create a new CarWriter, with the encoded block as the root */
+
+export async function makeCarBlob(roots: CIDForCar, blocks: BlockForCar[]) {
+	const carOut = await makeCarOut(roots, blocks)
+	const chunks = []
+	for await (const chunk of carOut) {
+		chunks.push(chunk)
+	}
+	const blob = new Blob(chunks)
+	return blob
+}
+export async function carFromBlob(blob: Blob | File): Promise<CarReader> {
+	return CarReader.fromBytes(new Uint8Array(await blob.arrayBuffer()))
+}
+
+function extractCids(value: unknown): CID[] {
+	if (value instanceof CID) return [value]
+	if (Array.isArray(value)) return value.flatMap(extractCids)
+	if (value && typeof value === 'object') return Object.values(value).flatMap(extractCids)
+	return []
+}
+
+const MAX_COLLECT_BLOCKS = 1_000_000
+
+export async function collectDagBlocks(
+	startCID: CID,
+	blockStore: BlockStoreish,
+): Promise<BlockForCar[]> {
+	const visited = new Set<string>()
+	const blocks: BlockForCar[] = []
+	const queue: CID[] = [startCID]
+
+	while (queue.length > 0) {
+		if (blocks.length >= MAX_COLLECT_BLOCKS) {
+			WARN(`[collectDagBlocks] hit ${MAX_COLLECT_BLOCKS} block limit, returning partial result`)
+			break
+		}
+
+		const cid = queue.shift()!
+		const cidStr = cid.toString()
+		if (visited.has(cidStr)) continue
+		visited.add(cidStr)
+
+		let bytes: Uint8Array
+		try {
+			bytes = await blockStore.get(cid)
+		} catch {
+			WARN(`[collectDagBlocks] block not found: ${cidStr}, stopping this branch`)
+			continue
+		}
+		if (!bytes) {
+			WARN(`[collectDagBlocks] block not found: ${cidStr}, stopping this branch`)
+			continue
+		}
+
+		blocks.push({ cid, bytes })
+
+		if (blocks.length % 1000 === 0) {
+			LOG(`[collectDagBlocks] collected ${blocks.length} blocks...`)
+		}
+
+		try {
+			const decoded = dagJson.decode(bytes)
+			const childCids = extractCids(decoded)
+			for (const child of childCids) {
+				if (!visited.has(child.toString())) {
+					queue.push(child)
+				}
+			}
+		} catch {
+			// Not dag-json — leaf block, no children to walk
+		}
+	}
+
+	DEBUG(`[collectDagBlocks] collected ${blocks.length} blocks from ${startCID.toString()}`)
+	return blocks
+}
+
+export function streamReaderToIterable(bodyReader: ReadableStreamDefaultReader<Uint8Array>): AsyncIterable<Uint8Array> {
+	return (async function*() {
+		while (true) {
+			const { done, value } = await bodyReader.read()
+			VERBOSE(`[car] chunk`, { done, value })
+			if (done) {
+				break
+			}
+			yield value
+		}
+	})()
+}