@e280/mammoth 0.1.0-1 → 0.1.0-10

package/s/node/disk-bucket.ts

@@ -0,0 +1,55 @@
+
+import {join} from "node:path"
+import {Readable} from "node:stream"
+import {pipeline} from "node:stream/promises"
+import {mkdir, stat, unlink} from "node:fs/promises"
+import {createWriteStream, openAsBlob} from "node:fs"
+
+import {Bucket, Id} from "../core/types.js"
+import {isNotFound} from "./utils/is-not-found.js"
+
+export class DiskBucket implements Bucket {
+	#directory
+
+	constructor(directory: string) {
+		this.#directory = directory
+	}
+
+	async has(id: Id) {
+		try {
+			await stat(this.#path(id))
+			return true
+		}
+		catch (error) {
+			if (isNotFound(error)) return false
+			throw error
+		}
+	}
+
+	async delete(id: Id) {
+		try {
+			await unlink(this.#path(id))
+		}
+		catch (error) {
+			if (!isNotFound(error))
+				throw error
+		}
+	}
+
+	async read(id: Id) {
+		return openAsBlob(this.#path(id))
+	}
+
+	async write(id: Id, readable: ReadableStream<Uint8Array>) {
+		await mkdir(this.#directory, {recursive: true})
+		await pipeline(
+			Readable.fromWeb(readable as any),
+			createWriteStream(this.#path(id)),
+		)
+	}
+
+	#path(id: Id) {
+		return join(this.#directory, id)
+	}
+}
+

package/s/node/index.ts

@@ -0,0 +1,4 @@
+
+export * from "../core/index.js"
+export * from "./disk-bucket.js"
+

package/s/node/utils/is-not-found.ts

@@ -0,0 +1,9 @@
+
+export function isNotFound(error: unknown): error is NodeJS.ErrnoException {
+	return (
+		error instanceof Error
+		&& "code" in error
+		&& error.code === "ENOENT"
+	)
+}
+

package/s/test.ts

@@ -1,24 +1,35 @@
 
 import {Kv} from "@e280/kv"
+import {rm} from "fs/promises"
 import {bytes, collect} from "@e280/stz"
 import {science, test, expect} from "@e280/science"
 
 import {Mammoth} from "./core/mammoth.js"
+import {DiskBucket} from "./node/disk-bucket.js"
 import {randomId} from "./core/utils/random-id.js"
-import {IcebergMemory} from "./core/iceberg-memory.js"
+import {MemoryBucket} from "./core/memory-bucket.js"
 
 const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
 const quickstream = (b: number[]) => new Blob([new Uint8Array(b)]).stream()
 
 function setup() {
-	const index = new Kv<string>()
-	const iceberg = new IcebergMemory()
-	const mammoth = new Mammoth(index, iceberg)
+	const kv = new Kv()
+	const bucket = new MemoryBucket()
+	const mammoth = new Mammoth(bucket, kv)
 	return {mammoth}
 }
 
 await science.run({
-	"mammoth": {
+	"mammoth node": test(async() => {
+		const dataDir = "data"
+		await rm(dataDir, {recursive: true, force: true})
+		const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv())
+		const hash = await mammoth.write(blob().stream())
+		const second = await mammoth.read(hash)
+		expect(bytes.eq(await second.bytes(), await blob().bytes()))
+	}),
+
+	"mammoth memory": {
 		".write and .read": test(async() => {
 			const {mammoth} = setup()
 			const hash = await mammoth.write(blob().stream())
@@ -30,7 +41,7 @@ await science.run({
 			const {mammoth} = setup()
 			const hashA = await mammoth.write(blob().stream())
 			const hashB = await mammoth.write(blob().stream())
-			const keys = await collect(mammoth.keys())
+			const keys = await collect(mammoth.hashes())
 			expect(hashA).is(hashB)
 			expect(keys.length).is(1)
 			expect(keys[0]).is(hashA)
@@ -39,7 +50,7 @@ await science.run({
 		".write empty file": test(async() => {
 			const {mammoth} = setup()
 			const hash = await mammoth.write(new Blob().stream())
-			expect(await mammoth.size(hash)).is(0)
+			expect((await mammoth.info(hash)).size).is(0)
 			expect((await mammoth.read(hash)).size).is(0)
 		}),
 
@@ -48,7 +59,20 @@ await science.run({
 			const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
 			const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
 			expect(hashA).not.is(hashB)
-			expect((await collect(mammoth.keys())).length).is(2)
+			expect((await collect(mammoth.hashes())).length).is(2)
+		}),
+
+		".stats": test(async() => {
+			const {mammoth} = setup()
+			expect(await mammoth.stats()).deep({count: 0, size: 0})
+			const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
+			expect(await mammoth.stats()).deep({count: 1, size: 3})
+			const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
+			expect(await mammoth.stats()).deep({count: 2, size: 7})
+			await mammoth.delete(hashA)
+			expect(await mammoth.stats()).deep({count: 1, size: 4})
+			await mammoth.delete(hashB)
+			expect(await mammoth.stats()).deep({count: 0, size: 0})
 		}),
 
 		".has": test(async() => {
@@ -60,8 +84,9 @@ await science.run({
 
 		".size": test(async() => {
 			const {mammoth} = setup()
-			const hash = await mammoth.write(blob().stream())
-			expect(await mammoth.size(hash)).is(4)
+			const b = blob()
+			const hash = await mammoth.write(b.stream())
+			expect((await mammoth.info(hash)).size).is(b.size)
 		}),
 
 		".delete": test(async() => {
@@ -69,7 +94,7 @@ await science.run({
 			const hash = await mammoth.write(blob().stream())
 			await mammoth.delete(hash)
 			expect(await mammoth.has(hash)).is(false)
-			expect(await collect(mammoth.keys())).deep([])
+			expect(await collect(mammoth.hashes())).deep([])
 		}),
 
 		".delete idempotent": test(async() => {
@@ -80,7 +105,7 @@ await science.run({
 		".keys": test(async() => {
 			const {mammoth} = setup()
 			const hash = await mammoth.write(blob().stream())
-			const keys = await collect(mammoth.keys())
+			const keys = await collect(mammoth.hashes())
 			expect(keys.length).is(1)
 			expect(keys[0]).is(hash)
 		}),

package/x/browser/example.js

@@ -1,20 +1,19 @@
 import { Kv, StorageDriver } from "@e280/kv";
 import { Mammoth } from "../core/mammoth.js";
-import { IcebergOpfs } from "./iceberg-opfs.js";
+import { OpfsBucket } from "./opfs-bucket.js";
 export async function example() {
-    // setup the localstorage kv index
+    // setup the localstorage kv
     const driver = new StorageDriver(window.localStorage);
-    const kv = new Kv(driver);
-    const index = kv.scope("glacier_index");
-    // setup the opfs iceberg
+    const kv = new Kv(driver).scope("mammoth");
+    // setup the opfs file bucket
     const root = await navigator.storage.getDirectory();
-    const directory = await root.getDirectoryHandle("glacier", { create: true });
-    const iceberg = new IcebergOpfs(directory);
+    const directory = await root.getDirectoryHandle("mammoth", { create: true });
+    const bucket = new OpfsBucket(directory);
     // setup the mammoth
-    const mammoth = new Mammoth(index, iceberg);
-    const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
+    const mammoth = new Mammoth(bucket, kv);
+    const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
     // write a file as an example
-    const hash = await mammoth.write(bigFile.stream());
-    console.log(`mammoth stored file by hash "${hash}"`);
+    const hash = await mammoth.write(myFile.stream());
+    console.log(`mammoth stored file "${hash}"`);
 }
 //# sourceMappingURL=example.js.map

package/x/browser/example.js.map

@@ -1 +1 @@
-{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;IAEvC,yBAAyB;IACzB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAA;IAE1C,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEpE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAClD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
+{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,CAAA;AAC7C,CAAC"}

package/x/browser/index.d.ts

@@ -1,2 +1,2 @@
 export * from "../core/index.js";
-export * from "./iceberg-opfs.js";
+export * from "./opfs-bucket.js";

package/x/browser/index.js

@@ -1,3 +1,3 @@
 export * from "../core/index.js";
-export * from "./iceberg-opfs.js";
+export * from "./opfs-bucket.js";
 //# sourceMappingURL=index.js.map

package/x/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/browser/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,mBAAmB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/browser/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,kBAAkB,CAAA"}

package/x/browser/{iceberg-opfs.d.ts → opfs-bucket.d.ts}

@@ -1,9 +1,8 @@
-import { Iceberg, Id } from "../core/types.js";
-export declare class IcebergOpfs implements Iceberg {
+import { Bucket, Id } from "../core/types.js";
+export declare class OpfsBucket implements Bucket {
     #private;
     constructor(directory: FileSystemDirectoryHandle);
     has(id: Id): Promise<boolean>;
-    size(id: Id): Promise<number>;
     delete(id: Id): Promise<void>;
     read(id: Id): Promise<File>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;

package/x/browser/{iceberg-opfs.js → opfs-bucket.js}

@@ -1,5 +1,5 @@
 import { isNotFound } from "./utils/is-not-found.js";
-export class IcebergOpfs {
+export class OpfsBucket {
     #directory;
     constructor(directory) {
         this.#directory = directory;
@@ -15,11 +15,6 @@ export class IcebergOpfs {
             throw error;
         }
     }
-    async size(id) {
-        const handle = await this.#directory.getFileHandle(id);
-        const file = await handle.getFile();
-        return file.size;
-    }
     async delete(id) {
         try {
             await this.#directory.removeEntry(id);
@@ -39,4 +34,4 @@ export class IcebergOpfs {
         await readable.pipeTo(writable);
     }
 }
-//# sourceMappingURL=iceberg-opfs.js.map
+//# sourceMappingURL=opfs-bucket.js.map

package/x/browser/opfs-bucket.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"opfs-bucket.js","sourceRoot":"","sources":["../../s/browser/opfs-bucket.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}

package/x/core/analyze.d.ts

@@ -0,0 +1,2 @@
+import { Analysis } from "./types.js";
+export declare function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis>;

package/x/core/{hash-stream.js → analyze.js}

@@ -1,14 +1,17 @@
 import { hex, nap } from "@e280/stz";
 import { blake3 } from "@noble/hashes/blake3.js";
 import { relaxer } from "./utils/relaxer.js";
-export async function hashStream(readable) {
+export async function analyze(readable) {
+    let size = 0;
     const hasher = blake3.create();
     const relax = relaxer();
     for await (const chunk of readable) {
         hasher.update(chunk);
+        size += chunk.byteLength;
         if (relax())
             await nap();
     }
-    return hex.fromBytes(hasher.digest());
+    const hash = hex.fromBytes(hasher.digest());
+    return { hash, size };
 }
-//# sourceMappingURL=hash-stream.js.map
+//# sourceMappingURL=analyze.js.map

package/x/core/analyze.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../s/core/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAoC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}

package/x/core/consts.d.ts

@@ -0,0 +1,4 @@
+export declare const consts: {
+    max_jobs: number;
+    expiry_time: number;
+};

package/x/core/consts.js

@@ -0,0 +1,6 @@
+import { time } from "@e280/stz";
+export const consts = {
+    max_jobs: 64,
+    expiry_time: time.days(7),
+};
+//# sourceMappingURL=consts.js.map

package/x/core/consts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;CACzB,CAAA"}

package/x/core/index.d.ts

@@ -1,4 +1,5 @@
-export * from "./hash-stream.js";
-export * from "./iceberg-memory.js";
+export * from "./analyze.js";
 export * from "./mammoth.js";
+export * from "./memory-bucket.js";
+export * from "./streamify.js";
 export * from "./types.js";

package/x/core/index.js

@@ -1,5 +1,6 @@
-export * from "./hash-stream.js";
-export * from "./iceberg-memory.js";
+export * from "./analyze.js";
 export * from "./mammoth.js";
+export * from "./memory-bucket.js";
+export * from "./streamify.js";
 export * from "./types.js";
 //# sourceMappingURL=index.js.map

package/x/core/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,qBAAqB,CAAA;AACnC,cAAc,cAAc,CAAA;AAC5B,cAAc,YAAY,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,cAAc,CAAA;AAC5B,cAAc,cAAc,CAAA;AAC5B,cAAc,oBAAoB,CAAA;AAClC,cAAc,gBAAgB,CAAA;AAC9B,cAAc,YAAY,CAAA"}

package/x/core/mammoth.d.ts

@@ -1,12 +1,14 @@
 import { Kv } from "@e280/kv";
-import { Glacier, Hash, Iceberg } from "./types.js";
-export declare class Mammoth implements Glacier {
+import { Hash, Bucket } from "./types.js";
+/** file storage datalake, content-addressed with blake3 hashes. */
+export declare class Mammoth {
     #private;
-    constructor(index: Kv<string>, iceberg: Iceberg);
+    constructor(bucket?: Bucket, kv?: Kv<any>);
+    hashes(): AsyncGenerator<string, void, unknown>;
     has(hash: Hash): Promise<boolean>;
-    size(hash: Hash): Promise<number>;
+    info(hash: Hash): Promise<import("./types.js").Info>;
+    stats(): Promise<import("./types.js").Stats>;
     read(hash: Hash): Promise<Blob>;
     delete(hash: Hash): Promise<void>;
     write(readable: ReadableStream<Uint8Array>): Promise<Hash>;
-    keys(): AsyncGenerator<string, void, unknown>;
 }

package/x/core/mammoth.js

@@ -1,56 +1,68 @@
-import { got, hex } from "@e280/stz";
-import { blake3 } from "@noble/hashes/blake3.js";
+import { Kv } from "@e280/kv";
+import { lane, queue } from "@e280/stz";
+import { consts } from "./consts.js";
+import { save } from "./utils/save.js";
+import { Manifest } from "./utils/manifest.js";
 import { randomId } from "./utils/random-id.js";
+import { MemoryBucket } from "./memory-bucket.js";
+/** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
-    #index;
-    #iceberg;
-    constructor(index, iceberg) {
-        this.#index = index;
-        this.#iceberg = iceberg;
+    #bucket;
+    #manifest;
+    #wholesome = lane(consts.max_jobs);
+    constructor(bucket = new MemoryBucket(), kv = new Kv()) {
+        this.#bucket = bucket;
+        this.#manifest = new Manifest(kv);
+    }
+    async *hashes() {
+        yield* this.#manifest.hashes();
     }
     async has(hash) {
-        return this.#index.has(hash);
+        return this.#manifest.hasHash(hash);
+    }
+    async info(hash) {
+        return this.#manifest.needInfo(hash);
     }
-    async size(hash) {
-        const id = await this.#needFileId(hash);
-        return this.#iceberg.size(id);
+    async stats() {
+        return this.#manifest.getStats();
     }
     async read(hash) {
-        const id = await this.#needFileId(hash);
-        return this.#iceberg.read(id);
+        const { id } = await this.#manifest.needInfo(hash);
+        return this.#bucket.read(id);
     }
     async delete(hash) {
-        const id = await this.#getFileId(hash);
-        if (id) {
-            await this.#index.del(hash);
-            await this.#iceberg.delete(id);
-        }
+        await this.#wholesome(async () => {
+            const info = await this.#manifest.getInfo(hash);
+            if (info)
+                await this.#manifest.scheduleDeletion(hash, info);
+        });
+        await this.#cleanup();
     }
     async write(readable) {
         const id = randomId();
-        const pipe = new TransformStream();
-        const done = this.#iceberg.write(id, pipe.readable);
-        const writer = pipe.writable.getWriter();
-        const hasher = blake3.create();
-        for await (const chunk of readable) {
-            await writer.write(chunk);
-            hasher.update(chunk);
+        await this.#manifest.addWip(id);
+        try {
+            const analysis = await save(this.#bucket, id, readable);
+            const { hash, size } = analysis;
+            const info = { id, size, added: Date.now() };
+            await this.#wholesome(() => this.#manifest.commit(hash, info));
+            return analysis.hash;
+        }
+        catch (error) {
+            await this.#manifest.moveWipToTrash(id);
+            throw error;
+        }
+        finally {
+            void this.#cleanup().catch(e => console.error("cleanup failed", e));
         }
-        await writer.close();
-        await done;
-        const hash = hex.fromBytes(hasher.digest());
-        await this.delete(hash);
-        await this.#index.set(hash, id);
-        return hash;
-    }
-    async #getFileId(hash) {
-        return this.#index.get(hash);
-    }
-    async #needFileId(hash) {
-        return got(await this.#getFileId(hash), `file not found by hash "${hash}"`);
-    }
-    async *keys() {
-        yield* this.#index.keys();
     }
+    #cleanup = queue(async () => {
+        for await (const id of this.#manifest.getExpiredWipIds())
+            await this.#manifest.moveWipToTrash(id);
+        for await (const id of this.#manifest.listTrashIds()) {
+            await this.#bucket.delete(id);
+            await this.#manifest.dropTrashRecord(id);
+        }
+    }, consts.max_jobs);
 }
 //# sourceMappingURL=mammoth.js.map

package/x/core/mammoth.js.map

@@ -1 +1 @@
-{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAG7C,MAAM,OAAO,OAAO;IACnB,MAAM,CAAA;IACN,QAAQ,CAAA;IAER,YAAY,KAAiB,EAAE,OAAgB;QAC9C,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QACvC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QACvC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAC3B,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;QACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;YACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,IAAI,CAAA;QACV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;QAC3C,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC/B,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAU;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAU;QAC3B,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,2BAA2B,IAAI,GAAG,CAAC,CAAA;IAC5E,CAAC;IAED,KAAK,CAAA,CAAE,IAAI;QACV,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;CACD"}
+{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAE/C,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/C,IAAI,IAAI;gBACP,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC9D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;YACvC,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,KAAK,IAAG,EAAE;QAC1B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE;YACvD,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;QAExC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC;YACtD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;QACzC,CAAC;IACF,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CACnB"}

package/x/core/{iceberg-memory.d.ts → memory-bucket.d.ts}

@@ -1,8 +1,7 @@
-import { Iceberg, Id } from "./types.js";
-export declare class IcebergMemory implements Iceberg {
+import { Bucket, Id } from "./types.js";
+export declare class MemoryBucket implements Bucket {
     #private;
     has(id: Id): Promise<boolean>;
-    size(id: Id): Promise<number>;
     delete(id: Id): Promise<void>;
     read(id: Id): Promise<Blob>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;

package/x/core/{iceberg-memory.js → memory-bucket.js}

@@ -1,13 +1,10 @@
 import { bytes, got, nap } from "@e280/stz";
 import { relaxer } from "./utils/relaxer.js";
-export class IcebergMemory {
+export class MemoryBucket {
     #map = new Map();
     async has(id) {
         return this.#map.has(id);
     }
-    async size(id) {
-        return got(this.#map.get(id)).byteLength;
-    }
     async delete(id) {
         this.#map.delete(id);
     }
@@ -26,4 +23,4 @@ export class IcebergMemory {
         this.#map.set(id, payload);
     }
 }
-//# sourceMappingURL=iceberg-memory.js.map
+//# sourceMappingURL=memory-bucket.js.map

package/x/core/memory-bucket.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-bucket.js","sourceRoot":"","sources":["../../s/core/memory-bucket.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,OAAO,YAAY;IACxB,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAA;IAEpC,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,IAAI,IAAI,CACd,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAA4B,CAAC,EACnD,EAAC,IAAI,EAAE,0BAA0B,EAAC,CAClC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;QACvB,MAAM,KAAK,GAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACjB,IAAI,KAAK,EAAE;gBAAE,MAAM,GAAG,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;CACD"}

package/x/core/streamify.d.ts

@@ -0,0 +1 @@
+export declare function streamify(bytes: Uint8Array, chunkSize?: number): ReadableStream<Uint8Array<ArrayBufferLike>>;

package/x/core/streamify.js

@@ -0,0 +1,15 @@
+export function streamify(bytes, chunkSize = 64 * 1024) {
+    let offset = 0;
+    return new ReadableStream({
+        pull(controller) {
+            if (offset >= bytes.length) {
+                controller.close();
+                return;
+            }
+            const end = Math.min(offset + chunkSize, bytes.length);
+            controller.enqueue(bytes.subarray(offset, end));
+            offset = end;
+        },
+    });
+}
+//# sourceMappingURL=streamify.js.map

package/x/core/streamify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamify.js","sourceRoot":"","sources":["../../s/core/streamify.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,SAAS,CAAC,KAAiB,EAAE,SAAS,GAAG,EAAE,GAAG,IAAI;IACjE,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,OAAO,IAAI,cAAc,CAAa;QACrC,IAAI,CAAC,UAAU;YACd,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC5B,UAAU,CAAC,KAAK,EAAE,CAAA;gBAClB,OAAM;YACP,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;YACtD,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC"}

package/x/core/types.d.ts

@@ -1,17 +1,35 @@
 export type Hash = string;
 export type Id = string;
-export type Iceberg = {
+/** file blob store. */
+export type Bucket = {
     has(id: Id): Promise<boolean>;
-    size(id: Id): Promise<number>;
     delete(id: Id): Promise<void>;
     read(id: Id): Promise<Blob>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
 };
-export type Glacier = {
-    has(hash: Hash): Promise<boolean>;
-    size(hash: Hash): Promise<number>;
-    read(hash: Hash): Promise<Blob>;
-    delete(hash: Hash): Promise<void>;
-    write(readable: ReadableStream<Uint8Array>): Promise<Hash>;
-    keys(): AsyncIterable<Hash>;
+/** report about a file */
+export type Analysis = {
+    /** blake3 hash of the file's contents */
+    hash: Hash;
+    /** filesize in bytes */
+    size: number;
+};
+/** metadata for a single file. */
+export type Info = {
+    /** bucket id for this file's data. */
+    id: Id;
+    /** filesize in bytes. */
+    size: number;
+    /** when this file was added to the datalake. */
+    added: number;
+};
+/** statistics for the whole datalake. */
+export type Stats = {
+    /** total number of bytes in the whole datalake. */
+    size: number;
+    /** total number of files in the datalake. */
+    count: number;
+};
+export type Wip = {
+    created: number;
 };

package/x/core/utils/is-expired.d.ts

@@ -0,0 +1,2 @@
+import { Wip } from "../types.js";
+export declare function isExpired({ created }: Wip): boolean;

package/x/core/utils/is-expired.js

@@ -0,0 +1,7 @@
+import { time } from "@e280/stz";
+import { consts } from "../consts.js";
+export function isExpired({ created }) {
+    const since = Date.now() - created;
+    return since > time.days(consts.expiry_time);
+}
+//# sourceMappingURL=is-expired.js.map

package/x/core/utils/is-expired.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAEnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAM;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}