@e280/mammoth 0.1.0-6 → 0.1.0-8

package/README.md

@@ -4,7 +4,7 @@
 # 🦣 mammoth
 > *big files. small api.*
 
-**mammoth is a content-addressed file store.** files are streamed into a bucket, identified by their blake3 hash, and automatically deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs.
+**mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs.
 
 ```bash
 npm install @e280/mammoth
@@ -17,7 +17,7 @@ const mammoth = new Mammoth()
 ```
 
 ### 🦣 mammoth stores files.
-- **write a file,** and you get back its blake3 hash.
+- **write a file,** and you get back its blake3 `hash` and `size` in bytes.
     ```ts
     const hash = await mammoth.write(blob.stream())
     ```
@@ -37,11 +37,15 @@ const mammoth = new Mammoth()
   import {Kv} from "@e280/kv"
 
   const mammoth = new Mammoth(
+
+    // bucket for blob storage
     new MemoryBucket(),
+
+    // kv for metadata bookkeeping
     new Kv(),
   )
   ```
-  defaults shown, so, this is equivalent:
+  defaults shown, this is equivalent:
   ```ts
   const mammoth = new Mammoth()
   ```
@@ -74,7 +78,7 @@ const mammoth = new Mammoth()
     ```ts
     const exists = await mammoth.has(hash)
     ```
-- **get file info,** including size in bytes, added timestamp, and bucket id.
+- **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
     ```ts
     const {size, added, id} = await mammoth.info(hash)
     ```

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@e280/mammoth",
-	"version": "0.1.0-6",
+	"version": "0.1.0-8",
 	"description": "big file storage",
 	"license": "MIT",
 	"author": "Chase Moskal <chasemoskal@gmail.com>",

package/s/browser/example.ts

@@ -20,6 +20,6 @@ export async function example() {
 
 	// write a file as an example
 	const hash = await mammoth.write(myFile.stream())
-	console.log(`mammoth stored file by hash "${hash}"`)
+	console.log(`mammoth stored file "${hash}"`)
 }
 

package/s/core/{hash-stream.ts → analyze.ts}

@@ -1,19 +1,23 @@
 
 import {hex, nap} from "@e280/stz"
 import {blake3} from "@noble/hashes/blake3.js"
+import {Analysis} from "./types.js"
 import {relaxer} from "./utils/relaxer.js"
 
-export async function hashStream(readable: ReadableStream<Uint8Array>) {
+export async function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis> {
+	let size = 0
 	const hasher = blake3.create()
 	const relax = relaxer()
 
 	for await (const chunk of readable) {
 		hasher.update(chunk)
+		size += chunk.byteLength
 
 		if (relax())
 			await nap()
 	}
 
-	return hex.fromBytes(hasher.digest())
+	const hash = hex.fromBytes(hasher.digest())
+	return {hash, size}
 }
 

package/s/core/consts.ts

@@ -3,7 +3,6 @@ import {time} from "@e280/stz"
 
 export const consts = {
 	max_jobs: 64,
-	self_clean_limit: 64,
 	expiry_time: time.days(7),
 }
 

package/s/core/index.ts

@@ -1,6 +1,6 @@
 
-export * from "./hash-stream.js"
-export * from "./memory-bucket.js"
+export * from "./analyze.js"
 export * from "./mammoth.js"
+export * from "./memory-bucket.js"
 export * from "./types.js"
 

package/s/core/mammoth.ts

@@ -3,11 +3,11 @@ import {Kv} from "@e280/kv"
 import {lane, queue} from "@e280/stz"
 
 import {consts} from "./consts.js"
-import {Hash, Bucket} from "./types.js"
+import {save} from "./utils/save.js"
 import {Manifest} from "./utils/manifest.js"
 import {randomId} from "./utils/random-id.js"
 import {MemoryBucket} from "./memory-bucket.js"
-import {saveAndHash} from "./utils/save-and-hash.js"
+import {Hash, Bucket, Analysis} from "./types.js"
 
 /** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
@@ -20,6 +20,10 @@ export class Mammoth {
 		this.#manifest = new Manifest(kv)
 	}
 
+	async* hashes() {
+		yield* this.#manifest.hashes()
+	}
+
 	async has(hash: Hash) {
 		return this.#manifest.hasHash(hash)
 	}
@@ -28,6 +32,10 @@ export class Mammoth {
 		return this.#manifest.needInfo(hash)
 	}
 
+	async stats() {
+		return this.#manifest.getStats()
+	}
+
 	async read(hash: Hash) {
 		const {id} = await this.#manifest.needInfo(hash)
 		return this.#bucket.read(id)
@@ -36,47 +44,39 @@ export class Mammoth {
 	async delete(hash: Hash) {
 		await this.#wholesome(async() => {
 			const info = await this.#manifest.getInfo(hash)
-			if (info) {
-				await this.#manifest.deleteInfo(hash)
-				await this.#bucket.delete(info.id)
-				await this.#manifest.statsRemoveFile(info.size)
-			}
+			if (info)
+				await this.#manifest.scheduleDeletion(hash, info)
 		})
+		await this.#cleanup()
 	}
 
 	async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
 		const id = randomId()
-
 		await this.#manifest.addWip(id)
-		const {hash, size} = await saveAndHash(this.#bucket, id, readable)
-
-		await this.#wholesome(async() => {
-			if (await this.#manifest.hasHash(hash)) {
-				await this.#bucket.delete(id) // forget this new file (we already have it)
-			}
-			else {
-				await this.#manifest.saveInfo(hash, {id, size, added: Date.now()})
-				await this.#manifest.statsAddFile(size)
-			}
-		})
-
-		await this.#manifest.deleteWip(id)
-		void this.#selfClean().catch(() => {})
-		return hash
-	}
-
-	async* hashes() {
-		yield* this.#manifest.hashes()
+		try {
+			const analysis = await save(this.#bucket, id, readable)
+			const {hash, size} = analysis
+			const info = {id, size, added: Date.now()}
+			await this.#wholesome(() => this.#manifest.commit(hash, info))
+			return analysis.hash
+		}
+		catch (error) {
+			await this.#manifest.moveWipToTrash(id)
+			throw error
+		}
+		finally {
+			void this.#cleanup().catch(e => console.error("cleanup failed", e))
+		}
 	}
 
-	async stats() {
-		return this.#manifest.getStats()
-	}
+	#cleanup = queue(async() => {
+		for await (const id of this.#manifest.getExpiredWipIds())
+			await this.#manifest.moveWipToTrash(id)
 
-	#selfClean = queue(async() => {
-		const expiredIds = await this.#manifest.getExpiredIds()
-		await Promise.all(expiredIds.map(id => this.#bucket.delete(id)))
-		await this.#manifest.deleteWip(...expiredIds)
-	})
+		for await (const id of this.#manifest.listTrashIds()) {
+			await this.#bucket.delete(id)
+			await this.#manifest.dropTrashRecord(id)
+		}
+	}, consts.max_jobs)
 }
 

package/s/core/types.ts

@@ -10,13 +10,23 @@ export type Bucket = {
 	write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
 }
 
+/** report about a file */
+export type Analysis = {
+
+	/** blake3 hash of the file's contents */
+	hash: Hash
+
+	/** filesize in bytes */
+	size: number
+}
+
 /** metadata for a single file. */
 export type Info = {
 
 	/** bucket id for this file's data. */
 	id: Id
 
-	/** file size in bytes. */
+	/** filesize in bytes. */
 	size: number
 
 	/** when this file was added to the datalake. */

package/s/core/utils/manifest.ts

@@ -1,12 +1,15 @@
 
 import {Kv} from "@e280/kv"
-import {collect, got} from "@e280/stz"
-import {consts} from "../consts.js"
+import {got} from "@e280/stz"
 import {notFound} from "./not-found.js"
 import {isExpired} from "./is-expired.js"
 import {Hash, Id, Info, Stats, Wip} from "../types.js"
 
 export class Manifest {
+	#kv
+
+	/** statistics about the whole datalake */
+	#stats
 
 	/** associates file hashes with bucket ids */
 	#info
@@ -15,12 +18,18 @@ export class Manifest {
 	#wip
 
 	/** statistics about the whole datalake */
-	#stats
+	#trash
 
 	constructor(kv = new Kv()) {
+		this.#kv = kv
+		this.#stats = kv.store<Stats>("stats")
 		this.#info = kv.scope<Info>("info")
 		this.#wip = kv.scope<Wip>("wip")
-		this.#stats = kv.store<Stats>("stats")
+		this.#trash = kv.scope<true>("trash")
+	}
+
+	async getStats() {
+		return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
 	}
 
 	async* hashes() {
@@ -31,10 +40,6 @@ export class Manifest {
 		return this.#info.has(hash)
 	}
 
-	async saveInfo(hash: Hash, info: Info) {
-		await this.#info.set(hash, info)
-	}
-
 	async getInfo(hash: Hash) {
 		return await this.#info.get(hash)
 	}
@@ -43,10 +48,6 @@ export class Manifest {
 		return got(await this.getInfo(hash), notFound(hash))
 	}
 
-	async deleteInfo(hash: Hash) {
-		await this.#info.del(hash)
-	}
-
 	async addWip(id: Id) {
 		await this.#wip.set(id, {created: Date.now()})
 	}
@@ -55,35 +56,58 @@ export class Manifest {
 		await this.#wip.del(...ids)
 	}
 
-	async statsAddFile(size: number) {
-		await this.#updateStats(stats => {
-			stats.count += 1
-			stats.size += size
-		})
+	async* getExpiredWipIds() {
+		for await (const [id, wip] of this.#wip.entries()) {
+			if (isExpired(wip))
+				yield id
+		}
 	}
 
-	async statsRemoveFile(size: number) {
-		await this.#updateStats(stats => {
-			stats.count -= 1
-			stats.size -= size
-		})
+	async moveWipToTrash(id: Id) {
+		await this.#kv.transaction(() => [
+			this.#wip.write.del(id),
+			this.#trash.write.set(id, true),
+		])
 	}
 
-	async getExpiredIds() {
-		const limit = consts.self_clean_limit
-		return (await collect(this.#wip.entries({limit})))
-			.filter(([,wip]) => isExpired(wip))
-			.map(([id]) => id)
+	async dropTrashRecord(id: Id) {
+		await this.#trash.del(id)
 	}
 
-	async getStats() {
-		return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
+	async* listTrashIds() {
+		yield* this.#trash.keys()
 	}
 
-	async #updateStats(fn: (stats: Stats) => void) {
+	async scheduleDeletion(hash: Hash, info: Info) {
 		const stats = await this.getStats()
-		fn(stats)
-		await this.#stats.set(stats)
+		stats.count -= 1
+		stats.size -= info.size
+		await this.#kv.transaction(() => [
+			this.#info.write.del(hash),
+			this.#trash.write.set(info.id, true),
+			this.#stats.write.set(stats),
+		])
+	}
+
+	async commit(hash: Hash, info: Info) {
+		const isNewFile = !await this.hasHash(hash)
+
+		if (isNewFile) {
+			const stats = await this.getStats()
+			stats.count += 1
+			stats.size += info.size
+			await this.#kv.transaction(() => [
+				this.#wip.write.del(info.id),
+				this.#info.write.set(hash, info),
+				this.#stats.write.set(stats)
+			])
+		}
+		else {
+			await this.#kv.transaction(() => [
+				this.#wip.write.del(info.id),
+				this.#trash.write.set(info.id, true),
+			])
+		}
 	}
 }
 

package/s/core/utils/{save-and-hash.ts → save.ts}

@@ -1,9 +1,14 @@
 
 import {hex} from "@e280/stz"
 import {blake3} from "@noble/hashes/blake3.js"
-import {Bucket, Id} from "../types.js"
+import {Analysis, Bucket, Id} from "../types.js"
+
+export async function save(
+		bucket: Bucket,
+		id: Id,
+		readable: ReadableStream<Uint8Array>,
+	): Promise<Analysis> {
 
-export async function saveAndHash(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>) {
 	let size = 0
 	const pipe = new TransformStream()
 	const done = bucket.write(id, pipe.readable)

package/x/browser/example.js

@@ -14,6 +14,6 @@ export async function example() {
     const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
     // write a file as an example
     const hash = await mammoth.write(myFile.stream());
-    console.log(`mammoth stored file by hash "${hash}"`);
+    console.log(`mammoth stored file "${hash}"`);
 }
 //# sourceMappingURL=example.js.map

package/x/browser/example.js.map

@@ -1 +1 @@
-{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
+{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,CAAA;AAC7C,CAAC"}

package/x/core/analyze.d.ts

@@ -0,0 +1,2 @@
+import { Analysis } from "./types.js";
+export declare function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis>;

package/x/core/{hash-stream.js → analyze.js}

@@ -1,14 +1,17 @@
 import { hex, nap } from "@e280/stz";
 import { blake3 } from "@noble/hashes/blake3.js";
 import { relaxer } from "./utils/relaxer.js";
-export async function hashStream(readable) {
+export async function analyze(readable) {
+    let size = 0;
     const hasher = blake3.create();
     const relax = relaxer();
     for await (const chunk of readable) {
         hasher.update(chunk);
+        size += chunk.byteLength;
         if (relax())
             await nap();
     }
-    return hex.fromBytes(hasher.digest());
+    const hash = hex.fromBytes(hasher.digest());
+    return { hash, size };
 }
-//# sourceMappingURL=hash-stream.js.map
+//# sourceMappingURL=analyze.js.map

package/x/core/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../s/core/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAoC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}

package/x/core/consts.d.ts

@@ -1,5 +1,4 @@
 export declare const consts: {
     max_jobs: number;
-    self_clean_limit: number;
     expiry_time: number;
 };

package/x/core/consts.js

@@ -1,7 +1,6 @@
 import { time } from "@e280/stz";
 export const consts = {
     max_jobs: 64,
-    self_clean_limit: 64,
     expiry_time: time.days(7),
 };
 //# sourceMappingURL=consts.js.map

package/x/core/consts.js.map

@@ -1 +1 @@
-{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,gBAAgB,EAAE,EAAE;IACpB,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;CACzB,CAAA"}
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;CACzB,CAAA"}

package/x/core/index.d.ts

@@ -1,4 +1,4 @@
-export * from "./hash-stream.js";
-export * from "./memory-bucket.js";
+export * from "./analyze.js";
 export * from "./mammoth.js";
+export * from "./memory-bucket.js";
 export * from "./types.js";

package/x/core/index.js

@@ -1,5 +1,5 @@
-export * from "./hash-stream.js";
-export * from "./memory-bucket.js";
+export * from "./analyze.js";
 export * from "./mammoth.js";
+export * from "./memory-bucket.js";
 export * from "./types.js";
 //# sourceMappingURL=index.js.map

package/x/core/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,oBAAoB,CAAA;AAClC,cAAc,cAAc,CAAA;AAC5B,cAAc,YAAY,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,cAAc,CAAA;AAC5B,cAAc,cAAc,CAAA;AAC5B,cAAc,oBAAoB,CAAA;AAClC,cAAc,YAAY,CAAA"}

package/x/core/mammoth.d.ts

@@ -4,11 +4,11 @@ import { Hash, Bucket } from "./types.js";
 export declare class Mammoth {
     #private;
     constructor(bucket?: Bucket, kv?: Kv<any>);
+    hashes(): AsyncGenerator<string, void, unknown>;
     has(hash: Hash): Promise<boolean>;
     info(hash: Hash): Promise<import("./types.js").Info>;
+    stats(): Promise<import("./types.js").Stats>;
     read(hash: Hash): Promise<Blob>;
     delete(hash: Hash): Promise<void>;
     write(readable: ReadableStream<Uint8Array>): Promise<Hash>;
-    hashes(): AsyncGenerator<string, void, unknown>;
-    stats(): Promise<import("./types.js").Stats>;
 }

package/x/core/mammoth.js

@@ -1,10 +1,10 @@
 import { Kv } from "@e280/kv";
 import { lane, queue } from "@e280/stz";
 import { consts } from "./consts.js";
+import { save } from "./utils/save.js";
 import { Manifest } from "./utils/manifest.js";
 import { randomId } from "./utils/random-id.js";
 import { MemoryBucket } from "./memory-bucket.js";
-import { saveAndHash } from "./utils/save-and-hash.js";
 /** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
     #bucket;
@@ -14,12 +14,18 @@ export class Mammoth {
         this.#bucket = bucket;
         this.#manifest = new Manifest(kv);
     }
+    async *hashes() {
+        yield* this.#manifest.hashes();
+    }
     async has(hash) {
         return this.#manifest.hasHash(hash);
     }
     async info(hash) {
         return this.#manifest.needInfo(hash);
     }
+    async stats() {
+        return this.#manifest.getStats();
+    }
     async read(hash) {
         const { id } = await this.#manifest.needInfo(hash);
         return this.#bucket.read(id);
@@ -27,40 +33,36 @@ export class Mammoth {
     async delete(hash) {
         await this.#wholesome(async () => {
             const info = await this.#manifest.getInfo(hash);
-            if (info) {
-                await this.#manifest.deleteInfo(hash);
-                await this.#bucket.delete(info.id);
-                await this.#manifest.statsRemoveFile(info.size);
-            }
+            if (info)
+                await this.#manifest.scheduleDeletion(hash, info);
         });
+        await this.#cleanup();
     }
     async write(readable) {
         const id = randomId();
         await this.#manifest.addWip(id);
-        const { hash, size } = await saveAndHash(this.#bucket, id, readable);
-        await this.#wholesome(async () => {
-            if (await this.#manifest.hasHash(hash)) {
-                await this.#bucket.delete(id); // forget this new file (we already have it)
-            }
-            else {
-                await this.#manifest.saveInfo(hash, { id, size, added: Date.now() });
-                await this.#manifest.statsAddFile(size);
-            }
-        });
-        await this.#manifest.deleteWip(id);
-        void this.#selfClean().catch(() => { });
-        return hash;
-    }
-    async *hashes() {
-        yield* this.#manifest.hashes();
-    }
-    async stats() {
-        return this.#manifest.getStats();
+        try {
+            const analysis = await save(this.#bucket, id, readable);
+            const { hash, size } = analysis;
+            const info = { id, size, added: Date.now() };
+            await this.#wholesome(() => this.#manifest.commit(hash, info));
+            return analysis.hash;
+        }
+        catch (error) {
+            await this.#manifest.moveWipToTrash(id);
+            throw error;
+        }
+        finally {
+            void this.#cleanup().catch(e => console.error("cleanup failed", e));
+        }
     }
-    #selfClean = queue(async () => {
-        const expiredIds = await this.#manifest.getExpiredIds();
-        await Promise.all(expiredIds.map(id => this.#bucket.delete(id)));
-        await this.#manifest.deleteWip(...expiredIds);
-    });
+    #cleanup = queue(async () => {
+        for await (const id of this.#manifest.getExpiredWipIds())
+            await this.#manifest.moveWipToTrash(id);
+        for await (const id of this.#manifest.listTrashIds()) {
+            await this.#bucket.delete(id);
+            await this.#manifest.dropTrashRecord(id);
+        }
+    }, consts.max_jobs);
 }
 //# sourceMappingURL=mammoth.js.map

package/x/core/mammoth.js.map

@@ -1 +1 @@
-{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAElC,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAA;AAEpD,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/C,IAAI,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;gBACrC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;gBAClC,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChD,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QAErB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;QAElE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA,CAAC,4CAA4C;YAC3E,CAAC;iBACI,CAAC;gBACL,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;gBAClE,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;YACxC,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;QAClC,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACtC,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,UAAU,GAAG,KAAK,CAAC,KAAK,IAAG,EAAE;QAC5B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,EAAE,CAAA;QACvD,MAAM,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAChE,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC,CAAA;IAC9C,CAAC,CAAC,CAAA;CACF"}
+{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AACpC,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAG/C,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/C,IAAI,IAAI;gBACP,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC9D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;YACvC,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,KAAK,IAAG,EAAE;QAC1B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE;YACvD,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;QAExC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC;YACtD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;QACzC,CAAC;IACF,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CACnB"}

package/x/core/types.d.ts

@@ -7,11 +7,18 @@ export type Bucket = {
     read(id: Id): Promise<Blob>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
 };
+/** report about a file */
+export type Analysis = {
+    /** blake3 hash of the file's contents */
+    hash: Hash;
+    /** filesize in bytes */
+    size: number;
+};
 /** metadata for a single file. */
 export type Info = {
     /** bucket id for this file's data. */
     id: Id;
-    /** file size in bytes. */
+    /** filesize in bytes. */
     size: number;
     /** when this file was added to the datalake. */
     added: number;

package/x/core/utils/manifest.d.ts

@@ -3,16 +3,17 @@ import { Hash, Id, Info, Stats } from "../types.js";
 export declare class Manifest {
     #private;
     constructor(kv?: Kv<any>);
+    getStats(): Promise<Stats>;
     hashes(): AsyncGenerator<string, void, unknown>;
     hasHash(hash: Hash): Promise<boolean>;
-    saveInfo(hash: Hash, info: Info): Promise<void>;
     getInfo(hash: Hash): Promise<Info | undefined>;
     needInfo(hash: Hash): Promise<Info>;
-    deleteInfo(hash: Hash): Promise<void>;
     addWip(id: Id): Promise<void>;
     deleteWip(...ids: Id[]): Promise<void>;
-    statsAddFile(size: number): Promise<void>;
-    statsRemoveFile(size: number): Promise<void>;
-    getExpiredIds(): Promise<string[]>;
-    getStats(): Promise<Stats>;
+    getExpiredWipIds(): AsyncGenerator<string, void, unknown>;
+    moveWipToTrash(id: Id): Promise<void>;
+    dropTrashRecord(id: Id): Promise<void>;
+    listTrashIds(): AsyncGenerator<string, void, unknown>;
+    scheduleDeletion(hash: Hash, info: Info): Promise<void>;
+    commit(hash: Hash, info: Info): Promise<void>;
 }

package/x/core/utils/manifest.js

@@ -1,19 +1,26 @@
 import { Kv } from "@e280/kv";
-import { collect, got } from "@e280/stz";
-import { consts } from "../consts.js";
+import { got } from "@e280/stz";
 import { notFound } from "./not-found.js";
 import { isExpired } from "./is-expired.js";
 export class Manifest {
+    #kv;
+    /** statistics about the whole datalake */
+    #stats;
     /** associates file hashes with bucket ids */
     #info;
     /** temporary record of writes in-progress */
     #wip;
     /** statistics about the whole datalake */
-    #stats;
+    #trash;
     constructor(kv = new Kv()) {
+        this.#kv = kv;
+        this.#stats = kv.store("stats");
         this.#info = kv.scope("info");
         this.#wip = kv.scope("wip");
-        this.#stats = kv.store("stats");
+        this.#trash = kv.scope("trash");
+    }
+    async getStats() {
+        return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
     }
     async *hashes() {
         yield* this.#info.keys();
@@ -21,49 +28,64 @@ export class Manifest {
     async hasHash(hash) {
         return this.#info.has(hash);
     }
-    async saveInfo(hash, info) {
-        await this.#info.set(hash, info);
-    }
     async getInfo(hash) {
         return await this.#info.get(hash);
     }
     async needInfo(hash) {
         return got(await this.getInfo(hash), notFound(hash));
     }
-    async deleteInfo(hash) {
-        await this.#info.del(hash);
-    }
     async addWip(id) {
         await this.#wip.set(id, { created: Date.now() });
     }
     async deleteWip(...ids) {
         await this.#wip.del(...ids);
     }
-    async statsAddFile(size) {
-        await this.#updateStats(stats => {
-            stats.count += 1;
-            stats.size += size;
-        });
+    async *getExpiredWipIds() {
+        for await (const [id, wip] of this.#wip.entries()) {
+            if (isExpired(wip))
+                yield id;
+        }
     }
-    async statsRemoveFile(size) {
-        await this.#updateStats(stats => {
-            stats.count -= 1;
-            stats.size -= size;
-        });
+    async moveWipToTrash(id) {
+        await this.#kv.transaction(() => [
+            this.#wip.write.del(id),
+            this.#trash.write.set(id, true),
+        ]);
     }
-    async getExpiredIds() {
-        const limit = consts.self_clean_limit;
-        return (await collect(this.#wip.entries({ limit })))
-            .filter(([, wip]) => isExpired(wip))
-            .map(([id]) => id);
+    async dropTrashRecord(id) {
+        await this.#trash.del(id);
     }
-    async getStats() {
-        return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
+    async *listTrashIds() {
+        yield* this.#trash.keys();
     }
-    async #updateStats(fn) {
+    async scheduleDeletion(hash, info) {
         const stats = await this.getStats();
-        fn(stats);
-        await this.#stats.set(stats);
+        stats.count -= 1;
+        stats.size -= info.size;
+        await this.#kv.transaction(() => [
+            this.#info.write.del(hash),
+            this.#trash.write.set(info.id, true),
+            this.#stats.write.set(stats),
+        ]);
+    }
+    async commit(hash, info) {
+        const isNewFile = !await this.hasHash(hash);
+        if (isNewFile) {
+            const stats = await this.getStats();
+            stats.count += 1;
+            stats.size += info.size;
+            await this.#kv.transaction(() => [
+                this.#wip.write.del(info.id),
+                this.#info.write.set(hash, info),
+                this.#stats.write.set(stats)
+            ]);
+        }
+        else {
+            await this.#kv.transaction(() => [
+                this.#wip.write.del(info.id),
+                this.#trash.write.set(info.id, true),
+            ]);
+        }
     }
 }
 //# sourceMappingURL=manifest.js.map

package/x/core/utils/manifest.js.map

@@ -1 +1 @@
-{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,OAAO,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AACtC,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AACnC,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IAEpB,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;IACvC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU,EAAE,IAAU;QACpC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAU;QAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,IAAY;QAC9B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE;YAC/B,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAA;QACnB,CAAC,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,IAAY;QACjC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE;YAC/B,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAA;QACnB,CAAC,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,aAAa;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,gBAAgB,CAAA;QACrC,OAAO,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAC,KAAK,EAAC,CAAC,CAAC,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAC,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAA;IACpB,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAA0B;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,EAAE,CAAC,KAAK,CAAC,CAAA;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC;CACD"}
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IACpB,GAAG,CAAA;IAEH,0CAA0C;IAC1C,MAAM,CAAA;IAEN,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAA,CAAE,gBAAgB;QACtB,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,SAAS,CAAC,GAAG,CAAC;gBACjB,MAAM,EAAE,CAAA;QACV,CAAC;IACF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAM;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAM;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAA,CAAE,YAAY;QAClB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;QAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,IAAU;QAClC,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;QACH,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;aACpC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;CACD"}

package/x/core/utils/save.d.ts

@@ -0,0 +1,2 @@
+import { Analysis, Bucket, Id } from "../types.js";
+export declare function save(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>): Promise<Analysis>;

package/x/core/utils/{save-and-hash.js → save.js}

@@ -1,6 +1,6 @@
 import { hex } from "@e280/stz";
 import { blake3 } from "@noble/hashes/blake3.js";
-export async function saveAndHash(bucket, id, readable) {
+export async function save(bucket, id, readable) {
     let size = 0;
     const pipe = new TransformStream();
     const done = bucket.write(id, pipe.readable);
@@ -16,4 +16,4 @@ export async function saveAndHash(bucket, id, readable) {
     const hash = hex.fromBytes(hasher.digest());
     return { hash, size };
 }
-//# sourceMappingURL=save-and-hash.js.map
+//# sourceMappingURL=save.js.map

package/x/core/utils/save.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;IACzB,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,IAAI,CAAA;IAEV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}

package/x/core/hash-stream.d.ts

@@ -1 +0,0 @@
-export declare function hashStream(readable: ReadableStream<Uint8Array>): Promise<string>;

package/x/core/hash-stream.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"hash-stream.js","sourceRoot":"","sources":["../../s/core/hash-stream.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAC9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAoC;IACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;AACtC,CAAC"}

package/x/core/utils/save-and-hash.d.ts

@@ -1,5 +0,0 @@
-import { Bucket, Id } from "../types.js";
-export declare function saveAndHash(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>): Promise<{
-    hash: string;
-    size: number;
-}>;

package/x/core/utils/save-and-hash.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"save-and-hash.js","sourceRoot":"","sources":["../../../s/core/utils/save-and-hash.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,EAAM,EAAE,QAAoC;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;IACzB,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,IAAI,CAAA;IAEV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}