@e280/mammoth 0.1.0-10 → 0.1.0-12

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@e280/mammoth",
-	"version": "0.1.0-10",
+	"version": "0.1.0-12",
 	"description": "big file storage",
 	"license": "MIT",
 	"author": "Chase Moskal <chasemoskal@gmail.com>",

package/s/core/mammoth.ts

@@ -5,9 +5,15 @@ import {lane, queue} from "@e280/stz"
 import {consts} from "./consts.js"
 import {save} from "./utils/save.js"
 import {Hash, Bucket} from "./types.js"
-import {Manifest} from "./utils/manifest.js"
+import {cleanup} from "./utils/cleanup.js"
 import {randomId} from "./utils/random-id.js"
 import {MemoryBucket} from "./memory-bucket.js"
+import {Manifest} from "./manifest/manifest.js"
+import {commit} from "./manifest/fns/commit.js"
+import {needInfo} from "./manifest/fns/need-info.js"
+import {getStats} from "./manifest/fns/get-stats.js"
+import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
+import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
 
 /** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
@@ -21,47 +27,47 @@ export class Mammoth {
 	}
 
 	async* hashes() {
-		yield* this.#manifest.hashes()
+		yield* this.#manifest.catalog.keys()
 	}
 
 	async has(hash: Hash) {
-		return this.#manifest.hasHash(hash)
+		return this.#manifest.catalog.has(hash)
 	}
 
 	async info(hash: Hash) {
-		return this.#manifest.needInfo(hash)
+		return needInfo(this.#manifest, hash)
 	}
 
 	async stats() {
-		return this.#manifest.getStats()
+		return getStats(this.#manifest)
 	}
 
 	async read(hash: Hash) {
-		const {id} = await this.#manifest.needInfo(hash)
+		const {id} = await this.info(hash)
 		return this.#bucket.read(id)
 	}
 
 	async delete(hash: Hash) {
 		await this.#wholesome(async() => {
-			const info = await this.#manifest.getInfo(hash)
+			const info = await this.#manifest.catalog.get(hash)
 			if (info)
-				await this.#manifest.scheduleDeletion(hash, info)
+				await scheduleDeletion(this.#manifest, hash, info)
 		})
 		await this.#cleanup()
 	}
 
 	async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
 		const id = randomId()
-		await this.#manifest.addWip(id)
+		await this.#manifest.writes.set(id, {created: Date.now()})
 		try {
 			const analysis = await save(this.#bucket, id, readable)
 			const {hash, size} = analysis
 			const info = {id, size, added: Date.now()}
-			await this.#wholesome(() => this.#manifest.commit(hash, info))
+			await this.#wholesome(() => commit(this.#manifest, hash, info))
 			return analysis.hash
 		}
 		catch (error) {
-			await this.#manifest.moveWipToTrash(id)
+			await moveWriteToTrash(this.#manifest, id)
 			throw error
 		}
 		finally {
@@ -69,14 +75,6 @@ export class Mammoth {
 		}
 	}
 
-	#cleanup = queue(async() => {
-		for await (const id of this.#manifest.getExpiredWipIds())
-			await this.#manifest.moveWipToTrash(id)
-
-		for await (const id of this.#manifest.listTrashIds()) {
-			await this.#bucket.delete(id)
-			await this.#manifest.dropTrashRecord(id)
-		}
-	}, consts.max_jobs)
+	#cleanup = queue(() => cleanup(this.#bucket, this.#manifest), consts.max_jobs)
 }
 

package/s/core/manifest/fns/commit.ts

@@ -0,0 +1,27 @@
+
+import {getStats} from "./get-stats.js"
+import {Manifest} from "../manifest.js"
+import {Hash, Info} from "../../types.js"
+
+export async function commit(manifest: Manifest, hash: Hash, info: Info) {
+	const {kv, stats, catalog, writes, trash} = manifest
+	const isNewFile = !await catalog.has(hash)
+
+	if (isNewFile) {
+		const s = await getStats(manifest)
+		s.count += 1
+		s.size += info.size
+		await kv.transaction(() => [
+			writes.write.del(info.id),
+			catalog.write.set(hash, info),
+			stats.write.set(s)
+		])
+	}
+	else {
+		await kv.transaction(() => [
+			writes.write.del(info.id),
+			trash.write.set(info.id, true),
+		])
+	}
+}
+

package/s/core/manifest/fns/get-expired-writes.ts

@@ -0,0 +1,11 @@
+
+import {Manifest} from "../manifest.js"
+import {isExpired} from "../../utils/is-expired.js"
+
+export async function* getExpiredWrites({writes}: Manifest) {
+	for await (const [id, w] of writes.entries()) {
+		if (isExpired(w))
+			yield id
+	}
+}
+

package/s/core/manifest/fns/get-stats.ts

@@ -0,0 +1,10 @@
+
+import {Manifest} from "../manifest.js"
+
+export async function getStats({stats}: Manifest) {
+	return structuredClone(
+		(await stats.get())
+			?? {count: 0, size: 0}
+	)
+}
+

package/s/core/manifest/fns/move-write-to-trash.ts

@@ -0,0 +1,11 @@
+
+import {Id} from "../../types.js"
+import {Manifest} from "../manifest.js"
+
+export async function moveWriteToTrash({kv, writes, trash}: Manifest, id: Id) {
+	await kv.transaction(() => [
+		writes.write.del(id),
+		trash.write.set(id, true),
+	])
+}
+

package/s/core/manifest/fns/need-info.ts

@@ -0,0 +1,10 @@
+
+import {got} from "@e280/stz"
+import {Hash} from "../../types.js"
+import {Manifest} from "../manifest.js"
+import {notFound} from "../../utils/not-found.js"
+
+export async function needInfo({catalog}: Manifest, hash: Hash) {
+	return got(await catalog.get(hash), notFound(hash))
+}
+

package/s/core/manifest/fns/schedule-deletion.ts

@@ -0,0 +1,19 @@
+
+import {getStats} from "./get-stats.js"
+import {Manifest} from "../manifest.js"
+import {Hash, Info} from "../../types.js"
+
+export async function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info) {
+	const {kv, stats, catalog, trash} = manifest
+
+	const s = await getStats(manifest)
+	s.count -= 1
+	s.size -= info.size
+
+	await kv.transaction(() => [
+		catalog.write.del(hash),
+		trash.write.set(info.id, true),
+		stats.write.set(s),
+	])
+}
+

package/s/core/manifest/manifest.ts

@@ -0,0 +1,26 @@
+
+import {Kv} from "@e280/kv"
+import {Info, Stats, WriteRecord} from "../types.js"
+
+export class Manifest {
+
+	/** statistics about the whole datalake */
+	stats
+
+	/** associates file hashes with bucket ids */
+	catalog
+
+	/** temporary record of writes in-progress */
+	writes
+
+	/** bucket ids that are pending deletion */
+	trash
+
+	constructor(public kv: Kv) {
+		this.stats = kv.store<Stats>("stats")
+		this.catalog = kv.scope<Info>("catalog")
+		this.writes = kv.scope<WriteRecord>("writes")
+		this.trash = kv.scope<true>("trash")
+	}
+}
+

package/s/core/types.ts

@@ -43,7 +43,7 @@ export type Stats = {
 	count: number
 }
 
-export type Wip = {
+export type WriteRecord = {
 	created: number
 }
 

package/s/core/utils/cleanup.ts

@@ -0,0 +1,16 @@
+
+import {Bucket} from "../types.js"
+import {Manifest} from "../manifest/manifest.js"
+import {getExpiredWrites} from "../manifest/fns/get-expired-writes.js"
+import {moveWriteToTrash} from "../manifest/fns/move-write-to-trash.js"
+
+export async function cleanup(bucket: Bucket, manifest: Manifest) {
+	for await (const id of getExpiredWrites(manifest))
+		await moveWriteToTrash(manifest, id)
+
+	for await (const id of manifest.trash.keys()) {
+		await bucket.delete(id)
+		await manifest.trash.del(id)
+	}
+}
+

package/s/core/utils/is-expired.ts

@@ -1,9 +1,9 @@
 
 import {time} from "@e280/stz"
-import {Wip} from "../types.js"
 import {consts} from "../consts.js"
+import {WriteRecord} from "../types.js"
 
-export function isExpired({created}: Wip) {
+export function isExpired({created}: WriteRecord) {
 	const since = Date.now() - created
 	return since > time.days(consts.expiry_time)
 }

package/s/core/utils/save.ts

@@ -10,19 +10,17 @@ export async function save(
 	): Promise<Analysis> {
 
 	let size = 0
-	const pipe = new TransformStream()
-	const done = bucket.write(id, pipe.readable)
-	const writer = pipe.writable.getWriter()
 	const hasher = blake3.create()
 
-	for await (const chunk of readable) {
-		await writer.write(chunk as Uint8Array<ArrayBuffer>)
-		hasher.update(chunk)
-		size += chunk.byteLength
-	}
-
-	await writer.close()
-	await done
+	await bucket.write(id, readable.pipeThrough(
+		new TransformStream({
+			transform(chunk, controller) {
+				hasher.update(chunk)
+				size += chunk.byteLength
+				controller.enqueue(chunk)
+			},
+		})
+	))
 
 	const hash = hex.fromBytes(hasher.digest())
 	return {hash, size}

package/x/core/mammoth.js

@@ -2,9 +2,15 @@ import { Kv } from "@e280/kv";
 import { lane, queue } from "@e280/stz";
 import { consts } from "./consts.js";
 import { save } from "./utils/save.js";
-import { Manifest } from "./utils/manifest.js";
+import { cleanup } from "./utils/cleanup.js";
 import { randomId } from "./utils/random-id.js";
 import { MemoryBucket } from "./memory-bucket.js";
+import { Manifest } from "./manifest/manifest.js";
+import { commit } from "./manifest/fns/commit.js";
+import { needInfo } from "./manifest/fns/need-info.js";
+import { getStats } from "./manifest/fns/get-stats.js";
+import { scheduleDeletion } from "./manifest/fns/schedule-deletion.js";
+import { moveWriteToTrash } from "./manifest/fns/move-write-to-trash.js";
 /** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
     #bucket;
@@ -15,54 +21,47 @@ export class Mammoth {
         this.#manifest = new Manifest(kv);
     }
     async *hashes() {
-        yield* this.#manifest.hashes();
+        yield* this.#manifest.catalog.keys();
     }
     async has(hash) {
-        return this.#manifest.hasHash(hash);
+        return this.#manifest.catalog.has(hash);
     }
     async info(hash) {
-        return this.#manifest.needInfo(hash);
+        return needInfo(this.#manifest, hash);
     }
     async stats() {
-        return this.#manifest.getStats();
+        return getStats(this.#manifest);
     }
     async read(hash) {
-        const { id } = await this.#manifest.needInfo(hash);
+        const { id } = await this.info(hash);
         return this.#bucket.read(id);
     }
     async delete(hash) {
         await this.#wholesome(async () => {
-            const info = await this.#manifest.getInfo(hash);
+            const info = await this.#manifest.catalog.get(hash);
             if (info)
-                await this.#manifest.scheduleDeletion(hash, info);
+                await scheduleDeletion(this.#manifest, hash, info);
         });
         await this.#cleanup();
     }
     async write(readable) {
         const id = randomId();
-        await this.#manifest.addWip(id);
+        await this.#manifest.writes.set(id, { created: Date.now() });
         try {
             const analysis = await save(this.#bucket, id, readable);
             const { hash, size } = analysis;
             const info = { id, size, added: Date.now() };
-            await this.#wholesome(() => this.#manifest.commit(hash, info));
+            await this.#wholesome(() => commit(this.#manifest, hash, info));
             return analysis.hash;
         }
         catch (error) {
-            await this.#manifest.moveWipToTrash(id);
+            await moveWriteToTrash(this.#manifest, id);
             throw error;
         }
         finally {
             void this.#cleanup().catch(e => console.error("cleanup failed", e));
         }
     }
-    #cleanup = queue(async () => {
-        for await (const id of this.#manifest.getExpiredWipIds())
-            await this.#manifest.moveWipToTrash(id);
-        for await (const id of this.#manifest.listTrashIds()) {
-            await this.#bucket.delete(id);
-            await this.#manifest.dropTrashRecord(id);
-        }
-    }, consts.max_jobs);
+    #cleanup = queue(() => cleanup(this.#bucket, this.#manifest), consts.max_jobs);
 }
 //# sourceMappingURL=mammoth.js.map

package/x/core/mammoth.js.map

@@ -1 +1 @@
-{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAE/C,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/C,IAAI,IAAI;gBACP,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC9D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;YACvC,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,KAAK,IAAG,EAAE;QAC1B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE;YACvD,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;QAExC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC;YACtD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;QACzC,CAAC;IACF,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CACnB"}
+{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AAEtE,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnD,IAAI,IAAI;gBACP,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;QAC1D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC/D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YAC1C,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CAC9E"}

package/x/core/manifest/fns/commit.d.ts

@@ -0,0 +1,3 @@
+import { Manifest } from "../manifest.js";
+import { Hash, Info } from "../../types.js";
+export declare function commit(manifest: Manifest, hash: Hash, info: Info): Promise<void>;

package/x/core/manifest/fns/commit.js

@@ -0,0 +1,22 @@
+import { getStats } from "./get-stats.js";
+export async function commit(manifest, hash, info) {
+    const { kv, stats, catalog, writes, trash } = manifest;
+    const isNewFile = !await catalog.has(hash);
+    if (isNewFile) {
+        const s = await getStats(manifest);
+        s.count += 1;
+        s.size += info.size;
+        await kv.transaction(() => [
+            writes.write.del(info.id),
+            catalog.write.set(hash, info),
+            stats.write.set(s)
+        ]);
+    }
+    else {
+        await kv.transaction(() => [
+            writes.write.del(info.id),
+            trash.write.set(info.id, true),
+        ]);
+    }
+}
+//# sourceMappingURL=commit.js.map

package/x/core/manifest/fns/commit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"commit.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/commit.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IACtE,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IACpD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAClB,CAAC,CAAA;IACH,CAAC;SACI,CAAC;QACL,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAA;IACH,CAAC;AACF,CAAC"}

package/x/core/manifest/fns/get-expired-writes.d.ts

@@ -0,0 +1,2 @@
+import { Manifest } from "../manifest.js";
+export declare function getExpiredWrites({ writes }: Manifest): AsyncGenerator<string, void, unknown>;

package/x/core/manifest/fns/get-expired-writes.js

@@ -0,0 +1,8 @@
+import { isExpired } from "../../utils/is-expired.js";
+export async function* getExpiredWrites({ writes }) {
+    for await (const [id, w] of writes.entries()) {
+        if (isExpired(w))
+            yield id;
+    }
+}
+//# sourceMappingURL=get-expired-writes.js.map

package/x/core/manifest/fns/get-expired-writes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-expired-writes.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-expired-writes.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAA;AAEnD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAW;IACzD,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}

package/x/core/manifest/fns/get-stats.d.ts

@@ -0,0 +1,2 @@
+import { Manifest } from "../manifest.js";
+export declare function getStats({ stats }: Manifest): Promise<import("../../types.js").Stats>;

package/x/core/manifest/fns/get-stats.js

@@ -0,0 +1,5 @@
+export async function getStats({ stats }) {
+    return structuredClone((await stats.get())
+        ?? { count: 0, size: 0 });
+}
+//# sourceMappingURL=get-stats.js.map

package/x/core/manifest/fns/get-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-stats.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,KAAK,EAAW;IAC/C,OAAO,eAAe,CACrB,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;WACf,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CACvB,CAAA;AACF,CAAC"}

package/x/core/manifest/fns/move-write-to-trash.d.ts

@@ -0,0 +1,3 @@
+import { Id } from "../../types.js";
+import { Manifest } from "../manifest.js";
+export declare function moveWriteToTrash({ kv, writes, trash }: Manifest, id: Id): Promise<void>;

package/x/core/manifest/fns/move-write-to-trash.js

@@ -0,0 +1,7 @@
+export async function moveWriteToTrash({ kv, writes, trash }, id) {
+    await kv.transaction(() => [
+        writes.write.del(id),
+        trash.write.set(id, true),
+    ]);
+}
+//# sourceMappingURL=move-write-to-trash.js.map

package/x/core/manifest/fns/move-write-to-trash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"move-write-to-trash.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/move-write-to-trash.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAW,EAAE,EAAM;IAC3E,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC,CAAA;AACH,CAAC"}

package/x/core/manifest/fns/need-info.d.ts

@@ -0,0 +1,3 @@
+import { Hash } from "../../types.js";
+import { Manifest } from "../manifest.js";
+export declare function needInfo({ catalog }: Manifest, hash: Hash): Promise<import("../../types.js").Info>;

package/x/core/manifest/fns/need-info.js

@@ -0,0 +1,6 @@
+import { got } from "@e280/stz";
+import { notFound } from "../../utils/not-found.js";
+export async function needInfo({ catalog }, hash) {
+    return got(await catalog.get(hash), notFound(hash));
+}
+//# sourceMappingURL=need-info.js.map

package/x/core/manifest/fns/need-info.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"need-info.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/need-info.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAG7B,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,OAAO,EAAW,EAAE,IAAU;IAC7D,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACpD,CAAC"}

package/x/core/manifest/fns/schedule-deletion.d.ts

@@ -0,0 +1,3 @@
+import { Manifest } from "../manifest.js";
+import { Hash, Info } from "../../types.js";
+export declare function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info): Promise<void>;

package/x/core/manifest/fns/schedule-deletion.js

@@ -0,0 +1,13 @@
+import { getStats } from "./get-stats.js";
+export async function scheduleDeletion(manifest, hash, info) {
+    const { kv, stats, catalog, trash } = manifest;
+    const s = await getStats(manifest);
+    s.count -= 1;
+    s.size -= info.size;
+    await kv.transaction(() => [
+        catalog.write.del(hash),
+        trash.write.set(info.id, true),
+        stats.write.set(s),
+    ]);
+}
+//# sourceMappingURL=schedule-deletion.js.map

package/x/core/manifest/fns/schedule-deletion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schedule-deletion.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/schedule-deletion.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IAChF,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IAE5C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;IAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;QAC9B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAClB,CAAC,CAAA;AACH,CAAC"}

package/x/core/manifest/manifest.d.ts

@@ -0,0 +1,14 @@
+import { Kv } from "@e280/kv";
+import { Info, Stats, WriteRecord } from "../types.js";
+export declare class Manifest {
+    kv: Kv;
+    /** statistics about the whole datalake */
+    stats: import("@e280/kv").Store<Stats>;
+    /** associates file hashes with bucket ids */
+    catalog: Kv<Info>;
+    /** temporary record of writes in-progress */
+    writes: Kv<WriteRecord>;
+    /** bucket ids that are pending deletion */
+    trash: Kv<true>;
+    constructor(kv: Kv);
+}

package/x/core/manifest/manifest.js

@@ -0,0 +1,19 @@
+export class Manifest {
+    kv;
+    /** statistics about the whole datalake */
+    stats;
+    /** associates file hashes with bucket ids */
+    catalog;
+    /** temporary record of writes in-progress */
+    writes;
+    /** bucket ids that are pending deletion */
+    trash;
+    constructor(kv) {
+        this.kv = kv;
+        this.stats = kv.store("stats");
+        this.catalog = kv.scope("catalog");
+        this.writes = kv.scope("writes");
+        this.trash = kv.scope("trash");
+    }
+}
+//# sourceMappingURL=manifest.js.map

package/x/core/manifest/manifest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAc,QAAQ,CAAC,CAAA;QAC7C,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}

package/x/core/types.d.ts

@@ -30,6 +30,6 @@ export type Stats = {
     /** total number of files in the datalake. */
     count: number;
 };
-export type Wip = {
+export type WriteRecord = {
     created: number;
 };

package/x/core/utils/cleanup.d.ts

@@ -0,0 +1,3 @@
+import { Bucket } from "../types.js";
+import { Manifest } from "../manifest/manifest.js";
+export declare function cleanup(bucket: Bucket, manifest: Manifest): Promise<void>;

package/x/core/utils/cleanup.js

@@ -0,0 +1,11 @@
+import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
+import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
+export async function cleanup(bucket, manifest) {
+    for await (const id of getExpiredWrites(manifest))
+        await moveWriteToTrash(manifest, id);
+    for await (const id of manifest.trash.keys()) {
+        await bucket.delete(id);
+        await manifest.trash.del(id);
+    }
+}
+//# sourceMappingURL=cleanup.js.map

package/x/core/utils/cleanup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}

package/x/core/utils/is-expired.d.ts

@@ -1,2 +1,2 @@
-import { Wip } from "../types.js";
-export declare function isExpired({ created }: Wip): boolean;
+import { WriteRecord } from "../types.js";
+export declare function isExpired({ created }: WriteRecord): boolean;

package/x/core/utils/is-expired.js.map

@@ -1 +1 @@
-{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAEnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAM;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
+{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAc;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}

package/x/core/utils/save.js

@@ -2,17 +2,14 @@ import { hex } from "@e280/stz";
 import { blake3 } from "@noble/hashes/blake3.js";
 export async function save(bucket, id, readable) {
     let size = 0;
-    const pipe = new TransformStream();
-    const done = bucket.write(id, pipe.readable);
-    const writer = pipe.writable.getWriter();
     const hasher = blake3.create();
-    for await (const chunk of readable) {
-        await writer.write(chunk);
-        hasher.update(chunk);
-        size += chunk.byteLength;
-    }
-    await writer.close();
-    await done;
+    await bucket.write(id, readable.pipeThrough(new TransformStream({
+        transform(chunk, controller) {
+            hasher.update(chunk);
+            size += chunk.byteLength;
+            controller.enqueue(chunk);
+        },
+    })));
     const hash = hex.fromBytes(hasher.digest());
     return { hash, size };
 }

package/x/core/utils/save.js.map

@@ -1 +1 @@
-{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;IACzB,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,IAAI,CAAA;IAEV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
+{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,WAAW,CAC1C,IAAI,eAAe,CAAC;QACnB,SAAS,CAAC,KAAK,EAAE,UAAU;YAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;YACxB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;KACD,CAAC,CACF,CAAC,CAAA;IAEF,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}

package/s/core/utils/manifest.ts

@@ -1,113 +0,0 @@
-
-import {Kv} from "@e280/kv"
-import {got} from "@e280/stz"
-import {notFound} from "./not-found.js"
-import {isExpired} from "./is-expired.js"
-import {Hash, Id, Info, Stats, Wip} from "../types.js"
-
-export class Manifest {
-	#kv
-
-	/** statistics about the whole datalake */
-	#stats
-
-	/** associates file hashes with bucket ids */
-	#info
-
-	/** temporary record of writes in-progress */
-	#wip
-
-	/** statistics about the whole datalake */
-	#trash
-
-	constructor(kv = new Kv()) {
-		this.#kv = kv
-		this.#stats = kv.store<Stats>("stats")
-		this.#info = kv.scope<Info>("info")
-		this.#wip = kv.scope<Wip>("wip")
-		this.#trash = kv.scope<true>("trash")
-	}
-
-	async getStats() {
-		return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
-	}
-
-	async* hashes() {
-		yield* this.#info.keys()
-	}
-
-	async hasHash(hash: Hash) {
-		return this.#info.has(hash)
-	}
-
-	async getInfo(hash: Hash) {
-		return await this.#info.get(hash)
-	}
-
-	async needInfo(hash: Hash) {
-		return got(await this.getInfo(hash), notFound(hash))
-	}
-
-	async addWip(id: Id) {
-		await this.#wip.set(id, {created: Date.now()})
-	}
-
-	async deleteWip(...ids: Id[]) {
-		await this.#wip.del(...ids)
-	}
-
-	async* getExpiredWipIds() {
-		for await (const [id, wip] of this.#wip.entries()) {
-			if (isExpired(wip))
-				yield id
-		}
-	}
-
-	async moveWipToTrash(id: Id) {
-		await this.#kv.transaction(() => [
-			this.#wip.write.del(id),
-			this.#trash.write.set(id, true),
-		])
-	}
-
-	async dropTrashRecord(id: Id) {
-		await this.#trash.del(id)
-	}
-
-	async* listTrashIds() {
-		yield* this.#trash.keys()
-	}
-
-	async scheduleDeletion(hash: Hash, info: Info) {
-		const stats = await this.getStats()
-		stats.count -= 1
-		stats.size -= info.size
-		await this.#kv.transaction(() => [
-			this.#info.write.del(hash),
-			this.#trash.write.set(info.id, true),
-			this.#stats.write.set(stats),
-		])
-	}
-
-	async commit(hash: Hash, info: Info) {
-		const isNewFile = !await this.hasHash(hash)
-
-		if (isNewFile) {
-			const stats = await this.getStats()
-			stats.count += 1
-			stats.size += info.size
-			await this.#kv.transaction(() => [
-				this.#wip.write.del(info.id),
-				this.#info.write.set(hash, info),
-				this.#stats.write.set(stats)
-			])
-		}
-		else {
-			await this.#kv.transaction(() => [
-				this.#wip.write.del(info.id),
-				this.#trash.write.set(info.id, true),
-			])
-		}
-	}
-}
-

package/x/core/utils/manifest.d.ts

@@ -1,19 +0,0 @@
-import { Kv } from "@e280/kv";
-import { Hash, Id, Info, Stats } from "../types.js";
-export declare class Manifest {
-    #private;
-    constructor(kv?: Kv<any>);
-    getStats(): Promise<Stats>;
-    hashes(): AsyncGenerator<string, void, unknown>;
-    hasHash(hash: Hash): Promise<boolean>;
-    getInfo(hash: Hash): Promise<Info | undefined>;
-    needInfo(hash: Hash): Promise<Info>;
-    addWip(id: Id): Promise<void>;
-    deleteWip(...ids: Id[]): Promise<void>;
-    getExpiredWipIds(): AsyncGenerator<string, void, unknown>;
-    moveWipToTrash(id: Id): Promise<void>;
-    dropTrashRecord(id: Id): Promise<void>;
-    listTrashIds(): AsyncGenerator<string, void, unknown>;
-    scheduleDeletion(hash: Hash, info: Info): Promise<void>;
-    commit(hash: Hash, info: Info): Promise<void>;
-}

package/x/core/utils/manifest.js

@@ -1,91 +0,0 @@
-import { Kv } from "@e280/kv";
-import { got } from "@e280/stz";
-import { notFound } from "./not-found.js";
-import { isExpired } from "./is-expired.js";
-export class Manifest {
-    #kv;
-    /** statistics about the whole datalake */
-    #stats;
-    /** associates file hashes with bucket ids */
-    #info;
-    /** temporary record of writes in-progress */
-    #wip;
-    /** statistics about the whole datalake */
-    #trash;
-    constructor(kv = new Kv()) {
-        this.#kv = kv;
-        this.#stats = kv.store("stats");
-        this.#info = kv.scope("info");
-        this.#wip = kv.scope("wip");
-        this.#trash = kv.scope("trash");
-    }
-    async getStats() {
-        return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
-    }
-    async *hashes() {
-        yield* this.#info.keys();
-    }
-    async hasHash(hash) {
-        return this.#info.has(hash);
-    }
-    async getInfo(hash) {
-        return await this.#info.get(hash);
-    }
-    async needInfo(hash) {
-        return got(await this.getInfo(hash), notFound(hash));
-    }
-    async addWip(id) {
-        await this.#wip.set(id, { created: Date.now() });
-    }
-    async deleteWip(...ids) {
-        await this.#wip.del(...ids);
-    }
-    async *getExpiredWipIds() {
-        for await (const [id, wip] of this.#wip.entries()) {
-            if (isExpired(wip))
-                yield id;
-        }
-    }
-    async moveWipToTrash(id) {
-        await this.#kv.transaction(() => [
-            this.#wip.write.del(id),
-            this.#trash.write.set(id, true),
-        ]);
-    }
-    async dropTrashRecord(id) {
-        await this.#trash.del(id);
-    }
-    async *listTrashIds() {
-        yield* this.#trash.keys();
-    }
-    async scheduleDeletion(hash, info) {
-        const stats = await this.getStats();
-        stats.count -= 1;
-        stats.size -= info.size;
-        await this.#kv.transaction(() => [
-            this.#info.write.del(hash),
-            this.#trash.write.set(info.id, true),
-            this.#stats.write.set(stats),
-        ]);
-    }
-    async commit(hash, info) {
-        const isNewFile = !await this.hasHash(hash);
-        if (isNewFile) {
-            const stats = await this.getStats();
-            stats.count += 1;
-            stats.size += info.size;
-            await this.#kv.transaction(() => [
-                this.#wip.write.del(info.id),
-                this.#info.write.set(hash, info),
-                this.#stats.write.set(stats)
-            ]);
-        }
-        else {
-            await this.#kv.transaction(() => [
-                this.#wip.write.del(info.id),
-                this.#trash.write.set(info.id, true),
-            ]);
-        }
-    }
-}
-//# sourceMappingURL=manifest.js.map

package/x/core/utils/manifest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IACpB,GAAG,CAAA;IAEH,0CAA0C;IAC1C,MAAM,CAAA;IAEN,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAA,CAAE,gBAAgB;QACtB,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,SAAS,CAAC,GAAG,CAAC;gBACjB,MAAM,EAAE,CAAA;QACV,CAAC;IACF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAM;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAM;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAA,CAAE,YAAY;QAClB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;QAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,IAAU;QAClC,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;QACH,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;aACpC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;CACD"}