@e280/mammoth 0.1.0-13 → 0.1.0-2

package/x/browser/iceberg-opfs.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"iceberg-opfs.js","sourceRoot":"","sources":["../../s/browser/iceberg-opfs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,WAAW;IACvB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAA;QACnC,OAAO,IAAI,CAAC,IAAI,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}

package/x/browser/index.d.ts

@@ -1,2 +1,2 @@
 export * from "../core/index.js";
-export * from "./opfs-bucket.js";
+export * from "./iceberg-opfs.js";

package/x/browser/index.js

@@ -1,3 +1,3 @@
 export * from "../core/index.js";
-export * from "./opfs-bucket.js";
+export * from "./iceberg-opfs.js";
 //# sourceMappingURL=index.js.map

package/x/browser/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/browser/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,kBAAkB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/browser/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,mBAAmB,CAAA"}

package/x/core/hash-stream.d.ts

@@ -0,0 +1 @@
+export declare function hashStream(readable: ReadableStream<Uint8Array>): Promise<string>;

package/x/core/{analyze.js → hash-stream.js}

@@ -1,17 +1,14 @@
 import { hex, nap } from "@e280/stz";
 import { blake3 } from "@noble/hashes/blake3.js";
 import { relaxer } from "./utils/relaxer.js";
-export async function analyze(readable) {
-    let size = 0;
+export async function hashStream(readable) {
     const hasher = blake3.create();
     const relax = relaxer();
     for await (const chunk of readable) {
         hasher.update(chunk);
-        size += chunk.byteLength;
         if (relax())
             await nap();
     }
-    const hash = hex.fromBytes(hasher.digest());
-    return { hash, size };
+    return hex.fromBytes(hasher.digest());
 }
-//# sourceMappingURL=analyze.js.map
+//# sourceMappingURL=hash-stream.js.map

package/x/core/hash-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash-stream.js","sourceRoot":"","sources":["../../s/core/hash-stream.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAC9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAoC;IACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;AACtC,CAAC"}

package/x/core/{memory-bucket.d.ts → iceberg-memory.d.ts}

@@ -1,7 +1,8 @@
-import { Bucket, Id } from "./types.js";
-export declare class MemoryBucket implements Bucket {
+import { Iceberg, Id } from "./types.js";
+export declare class IcebergMemory implements Iceberg {
     #private;
     has(id: Id): Promise<boolean>;
+    size(id: Id): Promise<number>;
     delete(id: Id): Promise<void>;
     read(id: Id): Promise<Blob>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;

package/x/core/{memory-bucket.js → iceberg-memory.js}

@@ -1,10 +1,13 @@
 import { bytes, got, nap } from "@e280/stz";
 import { relaxer } from "./utils/relaxer.js";
-export class MemoryBucket {
+export class IcebergMemory {
     #map = new Map();
     async has(id) {
         return this.#map.has(id);
     }
+    async size(id) {
+        return got(this.#map.get(id)).byteLength;
+    }
     async delete(id) {
         this.#map.delete(id);
     }
@@ -23,4 +26,4 @@ export class MemoryBucket {
         this.#map.set(id, payload);
     }
 }
-//# sourceMappingURL=memory-bucket.js.map
+//# sourceMappingURL=iceberg-memory.js.map

package/x/core/iceberg-memory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"iceberg-memory.js","sourceRoot":"","sources":["../../s/core/iceberg-memory.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,OAAO,aAAa;IACzB,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAA;IAEpC,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,IAAI,IAAI,CACd,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAA4B,CAAC,EACnD,EAAC,IAAI,EAAE,0BAA0B,EAAC,CAClC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;QACvB,MAAM,KAAK,GAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACjB,IAAI,KAAK,EAAE;gBAAE,MAAM,GAAG,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;CACD"}

package/x/core/index.d.ts

@@ -1,5 +1,4 @@
-export * from "./analyze.js";
+export * from "./hash-stream.js";
+export * from "./iceberg-memory.js";
 export * from "./mammoth.js";
-export * from "./memory-bucket.js";
-export * from "./streamify.js";
 export * from "./types.js";

package/x/core/index.js

@@ -1,6 +1,5 @@
-export * from "./analyze.js";
+export * from "./hash-stream.js";
+export * from "./iceberg-memory.js";
 export * from "./mammoth.js";
-export * from "./memory-bucket.js";
-export * from "./streamify.js";
 export * from "./types.js";
 //# sourceMappingURL=index.js.map

package/x/core/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,cAAc,CAAA;AAC5B,cAAc,cAAc,CAAA;AAC5B,cAAc,oBAAoB,CAAA;AAClC,cAAc,gBAAgB,CAAA;AAC9B,cAAc,YAAY,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,qBAAqB,CAAA;AACnC,cAAc,cAAc,CAAA;AAC5B,cAAc,YAAY,CAAA"}

package/x/core/mammoth.d.ts

@@ -1,14 +1,12 @@
 import { Kv } from "@e280/kv";
-import { Hash, Bucket } from "./types.js";
-/** file storage datalake, content-addressed with blake3 hashes. */
-export declare class Mammoth {
+import { Glacier, Hash, Iceberg } from "./types.js";
+export declare class Mammoth implements Glacier {
     #private;
-    constructor(bucket?: Bucket, kv?: Kv<any>);
-    entries(): AsyncGenerator<[string, import("./types.js").Info], void, unknown>;
+    constructor(index: Kv<string>, iceberg: Iceberg);
     has(hash: Hash): Promise<boolean>;
-    info(hash: Hash): Promise<import("./types.js").Info>;
-    stats(): Promise<import("./types.js").Stats>;
+    size(hash: Hash): Promise<number>;
     read(hash: Hash): Promise<Blob>;
     delete(hash: Hash): Promise<void>;
     write(readable: ReadableStream<Uint8Array>): Promise<Hash>;
+    keys(): AsyncGenerator<string, void, unknown>;
 }

package/x/core/mammoth.js

@@ -1,67 +1,56 @@
-import { Kv } from "@e280/kv";
-import { lane, queue } from "@e280/stz";
-import { consts } from "./consts.js";
-import { save } from "./utils/save.js";
-import { cleanup } from "./utils/cleanup.js";
+import { got, hex } from "@e280/stz";
+import { blake3 } from "@noble/hashes/blake3.js";
 import { randomId } from "./utils/random-id.js";
-import { MemoryBucket } from "./memory-bucket.js";
-import { Manifest } from "./manifest/manifest.js";
-import { commit } from "./manifest/fns/commit.js";
-import { needInfo } from "./manifest/fns/need-info.js";
-import { getStats } from "./manifest/fns/get-stats.js";
-import { scheduleDeletion } from "./manifest/fns/schedule-deletion.js";
-import { moveWriteToTrash } from "./manifest/fns/move-write-to-trash.js";
-/** file storage datalake, content-addressed with blake3 hashes. */
 export class Mammoth {
-    #bucket;
-    #manifest;
-    #wholesome = lane(consts.max_jobs);
-    constructor(bucket = new MemoryBucket(), kv = new Kv()) {
-        this.#bucket = bucket;
-        this.#manifest = new Manifest(kv);
-    }
-    async *entries() {
-        yield* this.#manifest.catalog.entries();
+    #index;
+    #iceberg;
+    constructor(index, iceberg) {
+        this.#index = index;
+        this.#iceberg = iceberg;
     }
     async has(hash) {
-        return this.#manifest.catalog.has(hash);
-    }
-    async info(hash) {
-        return needInfo(this.#manifest, hash);
+        return this.#index.has(hash);
     }
-    async stats() {
-        return getStats(this.#manifest);
+    async size(hash) {
+        const id = await this.#needFileId(hash);
+        return this.#iceberg.size(id);
     }
     async read(hash) {
-        const { id } = await this.info(hash);
-        return this.#bucket.read(id);
+        const id = await this.#needFileId(hash);
+        return this.#iceberg.read(id);
     }
     async delete(hash) {
-        await this.#wholesome(async () => {
-            const info = await this.#manifest.catalog.get(hash);
-            if (info)
-                await scheduleDeletion(this.#manifest, hash, info);
-        });
-        await this.#cleanup();
+        const id = await this.#getFileId(hash);
+        if (id) {
+            await this.#index.del(hash);
+            await this.#iceberg.delete(id);
+        }
     }
     async write(readable) {
         const id = randomId();
-        await this.#manifest.writes.set(id, { created: Date.now() });
-        try {
-            const analysis = await save(this.#bucket, id, readable);
-            const { hash, size } = analysis;
-            const info = { id, size, added: Date.now() };
-            await this.#wholesome(() => commit(this.#manifest, hash, info));
-            return analysis.hash;
-        }
-        catch (error) {
-            await moveWriteToTrash(this.#manifest, id);
-            throw error;
-        }
-        finally {
-            void this.#cleanup().catch(e => console.error("cleanup failed", e));
+        const pipe = new TransformStream();
+        const done = this.#iceberg.write(id, pipe.readable);
+        const writer = pipe.writable.getWriter();
+        const hasher = blake3.create();
+        for await (const chunk of readable) {
+            await writer.write(chunk);
+            hasher.update(chunk);
         }
+        await writer.close();
+        await done;
+        const hash = hex.fromBytes(hasher.digest());
+        await this.delete(hash);
+        await this.#index.set(hash, id);
+        return hash;
+    }
+    async #getFileId(hash) {
+        return this.#index.get(hash);
+    }
+    async #needFileId(hash) {
+        return got(await this.#getFileId(hash), `file not found by hash "${hash}"`);
+    }
+    async *keys() {
+        yield* this.#index.keys();
     }
-    #cleanup = queue(() => cleanup(this.#bucket, this.#manifest), consts.max_jobs);
 }
 //# sourceMappingURL=mammoth.js.map

package/x/core/mammoth.js.map

@@ -1 +1 @@
-{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AAEtE,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,OAAO;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnD,IAAI,IAAI;gBACP,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;QAC1D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC/D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YAC1C,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CAC9E"}
+{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAG7C,MAAM,OAAO,OAAO;IACnB,MAAM,CAAA;IACN,QAAQ,CAAA;IAER,YAAY,KAAiB,EAAE,OAAgB;QAC9C,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;QACnB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QACvC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAA;QACvC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC9B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QACtC,IAAI,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAC3B,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;QACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;YACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACrB,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;QACpB,MAAM,IAAI,CAAA;QACV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;QAC3C,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QACvB,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;QAC/B,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAU;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAS,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAU;QAC3B,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,2BAA2B,IAAI,GAAG,CAAC,CAAA;IAC5E,CAAC;IAED,KAAK,CAAA,CAAE,IAAI;QACV,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;CACD"}

package/x/core/types.d.ts

@@ -1,35 +1,17 @@
 export type Hash = string;
 export type Id = string;
-/** file blob store. */
-export type Bucket = {
+export type Iceberg = {
     has(id: Id): Promise<boolean>;
+    size(id: Id): Promise<number>;
     delete(id: Id): Promise<void>;
     read(id: Id): Promise<Blob>;
     write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
 };
-/** report about a file */
-export type Analysis = {
-    /** blake3 hash of the file's contents */
-    hash: Hash;
-    /** filesize in bytes */
-    size: number;
-};
-/** metadata for a single file. */
-export type Info = {
-    /** bucket id for this file's data. */
-    id: Id;
-    /** filesize in bytes. */
-    size: number;
-    /** when this file was added to the datalake. */
-    added: number;
-};
-/** statistics for the whole datalake. */
-export type Stats = {
-    /** total number of bytes in the whole datalake. */
-    size: number;
-    /** total number of files in the datalake. */
-    count: number;
-};
-export type WriteRecord = {
-    created: number;
+export type Glacier = {
+    has(hash: Hash): Promise<boolean>;
+    size(hash: Hash): Promise<number>;
+    read(hash: Hash): Promise<Blob>;
+    delete(hash: Hash): Promise<void>;
+    write(readable: ReadableStream<Uint8Array>): Promise<Hash>;
+    keys(): AsyncIterable<Hash>;
 };

package/x/test.js

@@ -1,29 +1,19 @@
 import { Kv } from "@e280/kv";
-import { rm } from "fs/promises";
 import { bytes, collect } from "@e280/stz";
 import { science, test, expect } from "@e280/science";
 import { Mammoth } from "./core/mammoth.js";
-import { DiskBucket } from "./node/disk-bucket.js";
 import { randomId } from "./core/utils/random-id.js";
-import { MemoryBucket } from "./core/memory-bucket.js";
+import { IcebergMemory } from "./core/iceberg-memory.js";
 const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
 const quickstream = (b) => new Blob([new Uint8Array(b)]).stream();
 function setup() {
-    const kv = new Kv();
-    const bucket = new MemoryBucket();
-    const mammoth = new Mammoth(bucket, kv);
+    const index = new Kv();
+    const iceberg = new IcebergMemory();
+    const mammoth = new Mammoth(index, iceberg);
     return { mammoth };
 }
 await science.run({
-    "mammoth node": test(async () => {
-        const dataDir = "data";
-        await rm(dataDir, { recursive: true, force: true });
-        const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv());
-        const hash = await mammoth.write(blob().stream());
-        const second = await mammoth.read(hash);
-        expect(bytes.eq(await second.bytes(), await blob().bytes()));
-    }),
-    "mammoth memory": {
+    "mammoth": {
         ".write and .read": test(async () => {
             const { mammoth } = setup();
             const hash = await mammoth.write(blob().stream());
@@ -34,15 +24,15 @@ await science.run({
             const { mammoth } = setup();
             const hashA = await mammoth.write(blob().stream());
             const hashB = await mammoth.write(blob().stream());
-            const entries = await collect(mammoth.entries());
+            const keys = await collect(mammoth.keys());
             expect(hashA).is(hashB);
-            expect(entries.length).is(1);
-            expect(entries[0][0]).is(hashA);
+            expect(keys.length).is(1);
+            expect(keys[0]).is(hashA);
         }),
         ".write empty file": test(async () => {
             const { mammoth } = setup();
             const hash = await mammoth.write(new Blob().stream());
-            expect((await mammoth.info(hash)).size).is(0);
+            expect(await mammoth.size(hash)).is(0);
             expect((await mammoth.read(hash)).size).is(0);
         }),
         ".write distinct files": test(async () => {
@@ -50,19 +40,7 @@ await science.run({
             const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
             const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
             expect(hashA).not.is(hashB);
-            expect((await collect(mammoth.entries())).length).is(2);
-        }),
-        ".stats": test(async () => {
-            const { mammoth } = setup();
-            expect(await mammoth.stats()).deep({ count: 0, size: 0 });
-            const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
-            expect(await mammoth.stats()).deep({ count: 1, size: 3 });
-            const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
-            expect(await mammoth.stats()).deep({ count: 2, size: 7 });
-            await mammoth.delete(hashA);
-            expect(await mammoth.stats()).deep({ count: 1, size: 4 });
-            await mammoth.delete(hashB);
-            expect(await mammoth.stats()).deep({ count: 0, size: 0 });
+            expect((await collect(mammoth.keys())).length).is(2);
         }),
         ".has": test(async () => {
             const { mammoth } = setup();
@@ -72,16 +50,15 @@ await science.run({
         }),
         ".size": test(async () => {
             const { mammoth } = setup();
-            const b = blob();
-            const hash = await mammoth.write(b.stream());
-            expect((await mammoth.info(hash)).size).is(b.size);
+            const hash = await mammoth.write(blob().stream());
+            expect(await mammoth.size(hash)).is(4);
         }),
         ".delete": test(async () => {
             const { mammoth } = setup();
             const hash = await mammoth.write(blob().stream());
             await mammoth.delete(hash);
             expect(await mammoth.has(hash)).is(false);
-            expect((await collect(mammoth.entries())).length).is(0);
+            expect(await collect(mammoth.keys())).deep([]);
         }),
         ".delete idempotent": test(async () => {
             const { mammoth } = setup();
@@ -90,9 +67,9 @@ await science.run({
         ".keys": test(async () => {
             const { mammoth } = setup();
             const hash = await mammoth.write(blob().stream());
-            const entries = await collect(mammoth.entries());
-            expect(entries.length).is(1);
-            expect(entries[0][0]).is(hash);
+            const keys = await collect(mammoth.keys());
+            expect(keys.length).is(1);
+            expect(keys[0]).is(hash);
         }),
     },
 });

package/x/test.js.map

@@ -1 +1 @@
-{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC;IAEF,gBAAgB,EAAE;QACjB,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAChD,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC,CAAC;QAEF,mBAAmB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACnC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACrD,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC7C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC;QAEF,uBAAuB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACxB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,MAAM,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,GAAG,IAAI,EAAE,CAAA;YAChB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACzB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,oBAAoB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACpC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;QACjC,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAChD,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC,CAAC;KACF;CACD,CAAC,CAAA"}
+{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,aAAa,EAAC,MAAM,0BAA0B,CAAA;AAEtD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,KAAK,GAAG,IAAI,EAAE,EAAU,CAAA;IAC9B,MAAM,OAAO,GAAG,IAAI,aAAa,EAAE,CAAA;IACnC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3C,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,SAAS,EAAE;QACV,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACvB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC,CAAC;QAEF,mBAAmB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACnC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACrD,MAAM,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC;QAEF,uBAAuB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACrD,CAAC,CAAC;QAEF,MAAM,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACvC,CAAC,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACzB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QAC/C,CAAC,CAAC;QAEF,oBAAoB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACpC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;QACjC,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAA;YAC1C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC,CAAC;KACF;CACD,CAAC,CAAA"}

package/s/core/consts.ts

@@ -1,8 +0,0 @@
-
-import {time} from "@e280/stz"
-
-export const consts = {
-	max_jobs: 64,
-	expiry_time: time.days(7),
-}
-

package/s/core/manifest/fns/commit.ts

@@ -1,27 +0,0 @@
-
-import {getStats} from "./get-stats.js"
-import {Manifest} from "../manifest.js"
-import {Hash, Info} from "../../types.js"
-
-export async function commit(manifest: Manifest, hash: Hash, info: Info) {
-	const {kv, stats, catalog, writes, trash} = manifest
-	const isNewFile = !await catalog.has(hash)
-
-	if (isNewFile) {
-		const s = await getStats(manifest)
-		s.count += 1
-		s.size += info.size
-		await kv.transaction(() => [
-			writes.write.del(info.id),
-			catalog.write.set(hash, info),
-			stats.write.set(s)
-		])
-	}
-	else {
-		await kv.transaction(() => [
-			writes.write.del(info.id),
-			trash.write.set(info.id, true),
-		])
-	}
-}
-

package/s/core/manifest/fns/get-expired-writes.ts

@@ -1,11 +0,0 @@
-
-import {Manifest} from "../manifest.js"
-import {isExpired} from "../../utils/is-expired.js"
-
-export async function* getExpiredWrites({writes}: Manifest) {
-	for await (const [id, w] of writes.entries()) {
-		if (isExpired(w))
-			yield id
-	}
-}
-

package/s/core/manifest/fns/get-stats.ts

@@ -1,10 +0,0 @@
-
-import {Manifest} from "../manifest.js"
-
-export async function getStats({stats}: Manifest) {
-	return structuredClone(
-		(await stats.get())
-			?? {count: 0, size: 0}
-	)
-}
-

package/s/core/manifest/fns/move-write-to-trash.ts

@@ -1,11 +0,0 @@
-
-import {Id} from "../../types.js"
-import {Manifest} from "../manifest.js"
-
-export async function moveWriteToTrash({kv, writes, trash}: Manifest, id: Id) {
-	await kv.transaction(() => [
-		writes.write.del(id),
-		trash.write.set(id, true),
-	])
-}
-

package/s/core/manifest/fns/need-info.ts

@@ -1,10 +0,0 @@
-
-import {got} from "@e280/stz"
-import {Hash} from "../../types.js"
-import {Manifest} from "../manifest.js"
-import {notFound} from "../../utils/not-found.js"
-
-export async function needInfo({catalog}: Manifest, hash: Hash) {
-	return got(await catalog.get(hash), notFound(hash))
-}
-

package/s/core/manifest/fns/schedule-deletion.ts

@@ -1,19 +0,0 @@
-
-import {getStats} from "./get-stats.js"
-import {Manifest} from "../manifest.js"
-import {Hash, Info} from "../../types.js"
-
-export async function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info) {
-	const {kv, stats, catalog, trash} = manifest
-
-	const s = await getStats(manifest)
-	s.count -= 1
-	s.size -= info.size
-
-	await kv.transaction(() => [
-		catalog.write.del(hash),
-		trash.write.set(info.id, true),
-		stats.write.set(s),
-	])
-}
-

package/s/core/manifest/manifest.ts

@@ -1,26 +0,0 @@
-
-import {Kv} from "@e280/kv"
-import {Info, Stats, WriteRecord} from "../types.js"
-
-export class Manifest {
-
-	/** statistics about the whole datalake */
-	stats
-
-	/** associates file hashes with bucket ids */
-	catalog
-
-	/** temporary record of writes in-progress */
-	writes
-
-	/** bucket ids that are pending deletion */
-	trash
-
-	constructor(public kv: Kv) {
-		this.stats = kv.store<Stats>("stats")
-		this.catalog = kv.scope<Info>("catalog")
-		this.writes = kv.scope<WriteRecord>("writes")
-		this.trash = kv.scope<true>("trash")
-	}
-}
-

package/s/core/streamify.ts

@@ -1,18 +0,0 @@
-
-export function streamify(bytes: Uint8Array, chunkSize = 64 * 1024) {
-	let offset = 0
-
-	return new ReadableStream<Uint8Array>({
-		pull(controller) {
-			if (offset >= bytes.length) {
-				controller.close()
-				return
-			}
-
-			const end = Math.min(offset + chunkSize, bytes.length)
-			controller.enqueue(bytes.subarray(offset, end))
-			offset = end
-		},
-	})
-}
-

package/s/core/utils/cleanup.ts

@@ -1,16 +0,0 @@
-
-import {Bucket} from "../types.js"
-import {Manifest} from "../manifest/manifest.js"
-import {getExpiredWrites} from "../manifest/fns/get-expired-writes.js"
-import {moveWriteToTrash} from "../manifest/fns/move-write-to-trash.js"
-
-export async function cleanup(bucket: Bucket, manifest: Manifest) {
-	for await (const id of getExpiredWrites(manifest))
-		await moveWriteToTrash(manifest, id)
-
-	for await (const id of manifest.trash.keys()) {
-		await bucket.delete(id)
-		await manifest.trash.del(id)
-	}
-}
-

package/s/core/utils/is-expired.ts

@@ -1,10 +0,0 @@
-
-import {time} from "@e280/stz"
-import {consts} from "../consts.js"
-import {WriteRecord} from "../types.js"
-
-export function isExpired({created}: WriteRecord) {
-	const since = Date.now() - created
-	return since > time.days(consts.expiry_time)
-}
-

package/s/core/utils/not-found.ts

@@ -1,7 +0,0 @@
-
-import {Hash} from "../types.js"
-
-export function notFound(hash: Hash) {
-	return `file not found by hash "${hash}"`
-}
-