@e280/mammoth 0.1.0-2 → 0.1.0-20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -5
- package/package.json +7 -8
- package/s/core/{iceberg-memory.ts → buckets/memory-bucket.ts} +3 -7
- package/s/{browser/iceberg-opfs.ts → core/buckets/opfs-bucket.ts} +5 -11
- package/s/core/consts.ts +8 -0
- package/s/core/index.ts +6 -2
- package/s/core/mammoth.ts +54 -46
- package/s/core/manifest/fns/commit.ts +27 -0
- package/s/core/manifest/fns/get-expired-writes.ts +11 -0
- package/s/core/manifest/fns/get-stats.ts +10 -0
- package/s/core/manifest/fns/move-write-to-trash.ts +11 -0
- package/s/core/manifest/fns/need-info.ts +10 -0
- package/s/core/manifest/fns/schedule-deletion.ts +19 -0
- package/s/core/manifest/manifest.ts +27 -0
- package/s/core/manifest/types.ts +5 -0
- package/s/core/parts/analyze.ts +26 -0
- package/s/core/parts/streamify.ts +18 -0
- package/s/core/types.ts +33 -9
- package/s/core/utils/cleanup.ts +16 -0
- package/s/core/utils/is-expired.ts +9 -0
- package/s/{browser → core}/utils/is-not-found.ts +1 -1
- package/s/core/utils/not-found.ts +7 -0
- package/s/core/utils/save.ts +28 -0
- package/s/node/disk-bucket.ts +55 -0
- package/s/{browser → node}/index.ts +1 -1
- package/s/node/utils/is-not-found.ts +9 -0
- package/s/test.ts +41 -16
- package/x/core/{iceberg-memory.d.ts → buckets/memory-bucket.d.ts} +2 -3
- package/x/core/{iceberg-memory.js → buckets/memory-bucket.js} +3 -6
- package/x/core/buckets/memory-bucket.js.map +1 -0
- package/x/{browser/iceberg-opfs.d.ts → core/buckets/opfs-bucket.d.ts} +2 -3
- package/x/{browser/iceberg-opfs.js → core/buckets/opfs-bucket.js} +5 -10
- package/x/core/buckets/opfs-bucket.js.map +1 -0
- package/x/core/consts.d.ts +4 -0
- package/x/core/consts.js +6 -0
- package/x/core/consts.js.map +1 -0
- package/x/core/index.d.ts +4 -2
- package/x/core/index.js +4 -2
- package/x/core/index.js.map +1 -1
- package/x/core/mammoth.d.ts +7 -5
- package/x/core/mammoth.js +51 -40
- package/x/core/mammoth.js.map +1 -1
- package/x/core/manifest/fns/commit.d.ts +3 -0
- package/x/core/manifest/fns/commit.js +22 -0
- package/x/core/manifest/fns/commit.js.map +1 -0
- package/x/core/manifest/fns/get-expired-writes.d.ts +2 -0
- package/x/core/manifest/fns/get-expired-writes.js +8 -0
- package/x/core/manifest/fns/get-expired-writes.js.map +1 -0
- package/x/core/manifest/fns/get-stats.d.ts +2 -0
- package/x/core/manifest/fns/get-stats.js +5 -0
- package/x/core/manifest/fns/get-stats.js.map +1 -0
- package/x/core/manifest/fns/move-write-to-trash.d.ts +3 -0
- package/x/core/manifest/fns/move-write-to-trash.js +7 -0
- package/x/core/manifest/fns/move-write-to-trash.js.map +1 -0
- package/x/core/manifest/fns/need-info.d.ts +3 -0
- package/x/core/manifest/fns/need-info.js +6 -0
- package/x/core/manifest/fns/need-info.js.map +1 -0
- package/x/core/manifest/fns/schedule-deletion.d.ts +3 -0
- package/x/core/manifest/fns/schedule-deletion.js +13 -0
- package/x/core/manifest/fns/schedule-deletion.js.map +1 -0
- package/x/core/manifest/manifest.d.ts +15 -0
- package/x/core/manifest/manifest.js +19 -0
- package/x/core/manifest/manifest.js.map +1 -0
- package/x/core/manifest/types.d.ts +3 -0
- package/x/core/manifest/types.js +2 -0
- package/x/core/manifest/types.js.map +1 -0
- package/x/core/parts/analyze.d.ts +2 -0
- package/x/core/parts/analyze.js +17 -0
- package/x/core/parts/analyze.js.map +1 -0
- package/x/core/parts/streamify.d.ts +1 -0
- package/x/core/parts/streamify.js +15 -0
- package/x/core/parts/streamify.js.map +1 -0
- package/x/core/types.d.ts +24 -9
- package/x/core/utils/cleanup.d.ts +3 -0
- package/x/core/utils/cleanup.js +11 -0
- package/x/core/utils/cleanup.js.map +1 -0
- package/x/core/utils/is-expired.d.ts +1 -0
- package/x/core/utils/is-expired.js +7 -0
- package/x/core/utils/is-expired.js.map +1 -0
- package/x/core/utils/is-not-found.d.ts +1 -0
- package/x/{browser → core}/utils/is-not-found.js +1 -1
- package/x/core/utils/is-not-found.js.map +1 -0
- package/x/core/utils/not-found.d.ts +2 -0
- package/x/core/utils/not-found.js +4 -0
- package/x/core/utils/not-found.js.map +1 -0
- package/x/core/utils/save.d.ts +2 -0
- package/x/core/utils/save.js +16 -0
- package/x/core/utils/save.js.map +1 -0
- package/x/node/disk-bucket.d.ts +9 -0
- package/x/node/disk-bucket.js +43 -0
- package/x/node/disk-bucket.js.map +1 -0
- package/x/node/index.d.ts +2 -0
- package/x/node/index.js +3 -0
- package/x/node/index.js.map +1 -0
- package/x/node/utils/is-not-found.d.ts +1 -0
- package/x/node/utils/is-not-found.js +6 -0
- package/x/node/utils/is-not-found.js.map +1 -0
- package/x/test.js +39 -16
- package/x/test.js.map +1 -1
- package/s/browser/example.ts +0 -26
- package/s/core/hash-stream.ts +0 -19
- package/x/browser/example.d.ts +0 -1
- package/x/browser/example.js +0 -20
- package/x/browser/example.js.map +0 -1
- package/x/browser/iceberg-opfs.js.map +0 -1
- package/x/browser/index.d.ts +0 -2
- package/x/browser/index.js +0 -3
- package/x/browser/index.js.map +0 -1
- package/x/browser/utils/is-not-found.d.ts +0 -1
- package/x/browser/utils/is-not-found.js.map +0 -1
- package/x/core/hash-stream.d.ts +0 -1
- package/x/core/hash-stream.js +0 -14
- package/x/core/hash-stream.js.map +0 -1
- package/x/core/iceberg-memory.js.map +0 -1
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { Kv } from "@e280/kv";
|
|
2
|
+
import { Write } from "./types.js";
|
|
3
|
+
import { Info, Stats } from "../types.js";
|
|
4
|
+
export declare class Manifest {
|
|
5
|
+
kv: Kv;
|
|
6
|
+
/** statistics about the whole datalake */
|
|
7
|
+
stats: import("@e280/kv").Store<Stats>;
|
|
8
|
+
/** associates file hashes with bucket ids */
|
|
9
|
+
catalog: Kv<Info>;
|
|
10
|
+
/** temporary record of writes in-progress */
|
|
11
|
+
writes: Kv<Write>;
|
|
12
|
+
/** bucket ids that are pending deletion */
|
|
13
|
+
trash: Kv<true>;
|
|
14
|
+
constructor(kv: Kv);
|
|
15
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export class Manifest {
|
|
2
|
+
kv;
|
|
3
|
+
/** statistics about the whole datalake */
|
|
4
|
+
stats;
|
|
5
|
+
/** associates file hashes with bucket ids */
|
|
6
|
+
catalog;
|
|
7
|
+
/** temporary record of writes in-progress */
|
|
8
|
+
writes;
|
|
9
|
+
/** bucket ids that are pending deletion */
|
|
10
|
+
trash;
|
|
11
|
+
constructor(kv) {
|
|
12
|
+
this.kv = kv;
|
|
13
|
+
this.stats = kv.store("stats");
|
|
14
|
+
this.catalog = kv.scope("catalog");
|
|
15
|
+
this.writes = kv.scope("writes");
|
|
16
|
+
this.trash = kv.scope("trash");
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=manifest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,QAAQ,CAAC,CAAA;QACvC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../s/core/manifest/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { hex, nap } from "@e280/stz";
|
|
2
|
+
import { blake3 } from "@awasm/noble";
|
|
3
|
+
import { relaxer } from "../utils/relaxer.js";
|
|
4
|
+
export async function analyze(readable) {
|
|
5
|
+
let size = 0;
|
|
6
|
+
const hasher = blake3.create();
|
|
7
|
+
const relax = relaxer();
|
|
8
|
+
for await (const chunk of readable) {
|
|
9
|
+
hasher.update(chunk);
|
|
10
|
+
size += chunk.byteLength;
|
|
11
|
+
if (relax())
|
|
12
|
+
await nap();
|
|
13
|
+
}
|
|
14
|
+
const hash = hex.fromBytes(hasher.digest());
|
|
15
|
+
return { hash, size };
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=analyze.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../../s/core/parts/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAEnC,OAAO,EAAC,OAAO,EAAC,MAAM,qBAAqB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function streamify(bytes: Uint8Array, chunkSize?: number): ReadableStream<Uint8Array<ArrayBufferLike>>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export function streamify(bytes, chunkSize = 64 * 1024) {
|
|
2
|
+
let offset = 0;
|
|
3
|
+
return new ReadableStream({
|
|
4
|
+
pull(controller) {
|
|
5
|
+
if (offset >= bytes.length) {
|
|
6
|
+
controller.close();
|
|
7
|
+
return;
|
|
8
|
+
}
|
|
9
|
+
const end = Math.min(offset + chunkSize, bytes.length);
|
|
10
|
+
controller.enqueue(bytes.subarray(offset, end));
|
|
11
|
+
offset = end;
|
|
12
|
+
},
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=streamify.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streamify.js","sourceRoot":"","sources":["../../../s/core/parts/streamify.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,SAAS,CAAC,KAAiB,EAAE,SAAS,GAAG,EAAE,GAAG,IAAI;IACjE,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,OAAO,IAAI,cAAc,CAAa;QACrC,IAAI,CAAC,UAAU;YACd,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC5B,UAAU,CAAC,KAAK,EAAE,CAAA;gBAClB,OAAM;YACP,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;YACtD,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC"}
|
package/x/core/types.d.ts
CHANGED
|
@@ -1,17 +1,32 @@
|
|
|
1
1
|
export type Hash = string;
|
|
2
2
|
export type Id = string;
|
|
3
|
-
|
|
3
|
+
/** file blob store. */
|
|
4
|
+
export type Bucket = {
|
|
4
5
|
has(id: Id): Promise<boolean>;
|
|
5
|
-
size(id: Id): Promise<number>;
|
|
6
6
|
delete(id: Id): Promise<void>;
|
|
7
7
|
read(id: Id): Promise<Blob>;
|
|
8
8
|
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
|
|
9
9
|
};
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
10
|
+
/** report about a file */
|
|
11
|
+
export type Analysis = {
|
|
12
|
+
/** blake3 hash of the file's contents */
|
|
13
|
+
hash: Hash;
|
|
14
|
+
/** filesize in bytes */
|
|
15
|
+
size: number;
|
|
16
|
+
};
|
|
17
|
+
/** metadata for a single file. */
|
|
18
|
+
export type Info = {
|
|
19
|
+
/** bucket id for this file's data. */
|
|
20
|
+
id: Id;
|
|
21
|
+
/** filesize in bytes. */
|
|
22
|
+
size: number;
|
|
23
|
+
/** when this file was added to the datalake. */
|
|
24
|
+
added: number;
|
|
25
|
+
};
|
|
26
|
+
/** statistics for the whole datalake. */
|
|
27
|
+
export type Stats = {
|
|
28
|
+
/** total number of bytes in the whole datalake. */
|
|
29
|
+
size: number;
|
|
30
|
+
/** total number of files in the datalake. */
|
|
31
|
+
count: number;
|
|
17
32
|
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
|
|
2
|
+
import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
|
|
3
|
+
export async function cleanup(bucket, manifest) {
|
|
4
|
+
for await (const id of getExpiredWrites(manifest))
|
|
5
|
+
await moveWriteToTrash(manifest, id);
|
|
6
|
+
for await (const id of manifest.trash.keys()) {
|
|
7
|
+
await bucket.delete(id);
|
|
8
|
+
await manifest.trash.delete(id);
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=cleanup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAChC,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isExpired(created: number): boolean;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAEnC,MAAM,UAAU,SAAS,CAAC,OAAe;IACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isNotFoundErr(error: unknown): boolean;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"is-not-found.js","sourceRoot":"","sources":["../../../s/core/utils/is-not-found.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,aAAa,CAAC,KAAc;IAC3C,OAAO,KAAK,YAAY,YAAY;WAChC,KAAK,CAAC,IAAI,KAAK,eAAe,CAAA;AACnC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"not-found.js","sourceRoot":"","sources":["../../../s/core/utils/not-found.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,eAAe,CAAC,IAAU;IACzC,OAAO,2BAA2B,IAAI,GAAG,CAAA;AAC1C,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { hex } from "@e280/stz";
|
|
2
|
+
import { blake3 } from "@awasm/noble";
|
|
3
|
+
export async function save(bucket, id, readable) {
|
|
4
|
+
let size = 0;
|
|
5
|
+
const hasher = blake3.create();
|
|
6
|
+
await bucket.write(id, readable.pipeThrough(new TransformStream({
|
|
7
|
+
transform(chunk, controller) {
|
|
8
|
+
hasher.update(chunk);
|
|
9
|
+
size += chunk.byteLength;
|
|
10
|
+
controller.enqueue(chunk);
|
|
11
|
+
},
|
|
12
|
+
})));
|
|
13
|
+
const hash = hex.fromBytes(hasher.digest());
|
|
14
|
+
return { hash, size };
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=save.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,WAAW,CAC1C,IAAI,eAAe,CAAC;QACnB,SAAS,CAAC,KAAK,EAAE,UAAU;YAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;YACxB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;KACD,CAAC,CACF,CAAC,CAAA;IAEF,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Bucket, Id } from "../core/types.js";
|
|
2
|
+
export declare class DiskBucket implements Bucket {
|
|
3
|
+
#private;
|
|
4
|
+
constructor(directory: string);
|
|
5
|
+
has(id: Id): Promise<boolean>;
|
|
6
|
+
delete(id: Id): Promise<void>;
|
|
7
|
+
read(id: Id): Promise<Blob>;
|
|
8
|
+
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { Readable } from "node:stream";
|
|
3
|
+
import { pipeline } from "node:stream/promises";
|
|
4
|
+
import { mkdir, stat, unlink } from "node:fs/promises";
|
|
5
|
+
import { createWriteStream, openAsBlob } from "node:fs";
|
|
6
|
+
import { isNotFound } from "./utils/is-not-found.js";
|
|
7
|
+
export class DiskBucket {
|
|
8
|
+
#directory;
|
|
9
|
+
constructor(directory) {
|
|
10
|
+
this.#directory = directory;
|
|
11
|
+
}
|
|
12
|
+
async has(id) {
|
|
13
|
+
try {
|
|
14
|
+
await stat(this.#path(id));
|
|
15
|
+
return true;
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
if (isNotFound(error))
|
|
19
|
+
return false;
|
|
20
|
+
throw error;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
async delete(id) {
|
|
24
|
+
try {
|
|
25
|
+
await unlink(this.#path(id));
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
if (!isNotFound(error))
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async read(id) {
|
|
33
|
+
return openAsBlob(this.#path(id));
|
|
34
|
+
}
|
|
35
|
+
async write(id, readable) {
|
|
36
|
+
await mkdir(this.#directory, { recursive: true });
|
|
37
|
+
await pipeline(Readable.fromWeb(readable), createWriteStream(this.#path(id)));
|
|
38
|
+
}
|
|
39
|
+
#path(id) {
|
|
40
|
+
return join(this.#directory, id);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=disk-bucket.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"disk-bucket.js","sourceRoot":"","sources":["../../s/node/disk-bucket.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAA;AACpC,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAC,iBAAiB,EAAE,UAAU,EAAC,MAAM,SAAS,CAAA;AAGrD,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAiB;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1B,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;QAC7B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAA;QAC/C,MAAM,QAAQ,CACb,QAAQ,CAAC,OAAO,CAAC,QAAe,CAAC,EACjC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CACjC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,EAAM;QACX,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACjC,CAAC;CACD"}
|
package/x/node/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/node/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,kBAAkB,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isNotFound(error: unknown): error is NodeJS.ErrnoException;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"is-not-found.js","sourceRoot":"","sources":["../../../s/node/utils/is-not-found.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,OAAO,CACN,KAAK,YAAY,KAAK;WACnB,MAAM,IAAI,KAAK;WACf,KAAK,CAAC,IAAI,KAAK,QAAQ,CAC1B,CAAA;AACF,CAAC"}
|
package/x/test.js
CHANGED
|
@@ -1,19 +1,29 @@
|
|
|
1
1
|
import { Kv } from "@e280/kv";
|
|
2
|
+
import { rm } from "fs/promises";
|
|
2
3
|
import { bytes, collect } from "@e280/stz";
|
|
3
4
|
import { science, test, expect } from "@e280/science";
|
|
4
5
|
import { Mammoth } from "./core/mammoth.js";
|
|
6
|
+
import { DiskBucket } from "./node/disk-bucket.js";
|
|
5
7
|
import { randomId } from "./core/utils/random-id.js";
|
|
6
|
-
import {
|
|
8
|
+
import { MemoryBucket } from "./core/buckets/memory-bucket.js";
|
|
7
9
|
const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
|
|
8
10
|
const quickstream = (b) => new Blob([new Uint8Array(b)]).stream();
|
|
9
11
|
function setup() {
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const mammoth = new Mammoth(
|
|
12
|
+
const kv = new Kv();
|
|
13
|
+
const bucket = new MemoryBucket();
|
|
14
|
+
const mammoth = new Mammoth(bucket, kv);
|
|
13
15
|
return { mammoth };
|
|
14
16
|
}
|
|
15
17
|
await science.run({
|
|
16
|
-
"mammoth": {
|
|
18
|
+
"mammoth node": test(async () => {
|
|
19
|
+
const dataDir = "data";
|
|
20
|
+
await rm(dataDir, { recursive: true, force: true });
|
|
21
|
+
const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv());
|
|
22
|
+
const hash = await mammoth.write(blob().stream());
|
|
23
|
+
const second = await mammoth.read(hash);
|
|
24
|
+
expect(bytes.eq(await second.bytes(), await blob().bytes()));
|
|
25
|
+
}),
|
|
26
|
+
"mammoth memory": {
|
|
17
27
|
".write and .read": test(async () => {
|
|
18
28
|
const { mammoth } = setup();
|
|
19
29
|
const hash = await mammoth.write(blob().stream());
|
|
@@ -24,15 +34,15 @@ await science.run({
|
|
|
24
34
|
const { mammoth } = setup();
|
|
25
35
|
const hashA = await mammoth.write(blob().stream());
|
|
26
36
|
const hashB = await mammoth.write(blob().stream());
|
|
27
|
-
const
|
|
37
|
+
const entries = await collect(mammoth.entries());
|
|
28
38
|
expect(hashA).is(hashB);
|
|
29
|
-
expect(
|
|
30
|
-
expect(
|
|
39
|
+
expect(entries.length).is(1);
|
|
40
|
+
expect(entries[0][0]).is(hashA);
|
|
31
41
|
}),
|
|
32
42
|
".write empty file": test(async () => {
|
|
33
43
|
const { mammoth } = setup();
|
|
34
44
|
const hash = await mammoth.write(new Blob().stream());
|
|
35
|
-
expect(await mammoth.
|
|
45
|
+
expect((await mammoth.info(hash)).size).is(0);
|
|
36
46
|
expect((await mammoth.read(hash)).size).is(0);
|
|
37
47
|
}),
|
|
38
48
|
".write distinct files": test(async () => {
|
|
@@ -40,7 +50,19 @@ await science.run({
|
|
|
40
50
|
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
|
|
41
51
|
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
|
|
42
52
|
expect(hashA).not.is(hashB);
|
|
43
|
-
expect((await collect(mammoth.
|
|
53
|
+
expect((await collect(mammoth.entries())).length).is(2);
|
|
54
|
+
}),
|
|
55
|
+
".stats": test(async () => {
|
|
56
|
+
const { mammoth } = setup();
|
|
57
|
+
expect(await mammoth.stats()).deep({ count: 0, size: 0 });
|
|
58
|
+
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
|
|
59
|
+
expect(await mammoth.stats()).deep({ count: 1, size: 3 });
|
|
60
|
+
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
|
|
61
|
+
expect(await mammoth.stats()).deep({ count: 2, size: 7 });
|
|
62
|
+
await mammoth.delete(hashA);
|
|
63
|
+
expect(await mammoth.stats()).deep({ count: 1, size: 4 });
|
|
64
|
+
await mammoth.delete(hashB);
|
|
65
|
+
expect(await mammoth.stats()).deep({ count: 0, size: 0 });
|
|
44
66
|
}),
|
|
45
67
|
".has": test(async () => {
|
|
46
68
|
const { mammoth } = setup();
|
|
@@ -50,15 +72,16 @@ await science.run({
|
|
|
50
72
|
}),
|
|
51
73
|
".size": test(async () => {
|
|
52
74
|
const { mammoth } = setup();
|
|
53
|
-
const
|
|
54
|
-
|
|
75
|
+
const b = blob();
|
|
76
|
+
const hash = await mammoth.write(b.stream());
|
|
77
|
+
expect((await mammoth.info(hash)).size).is(b.size);
|
|
55
78
|
}),
|
|
56
79
|
".delete": test(async () => {
|
|
57
80
|
const { mammoth } = setup();
|
|
58
81
|
const hash = await mammoth.write(blob().stream());
|
|
59
82
|
await mammoth.delete(hash);
|
|
60
83
|
expect(await mammoth.has(hash)).is(false);
|
|
61
|
-
expect(await collect(mammoth.
|
|
84
|
+
expect((await collect(mammoth.entries())).length).is(0);
|
|
62
85
|
}),
|
|
63
86
|
".delete idempotent": test(async () => {
|
|
64
87
|
const { mammoth } = setup();
|
|
@@ -67,9 +90,9 @@ await science.run({
|
|
|
67
90
|
".keys": test(async () => {
|
|
68
91
|
const { mammoth } = setup();
|
|
69
92
|
const hash = await mammoth.write(blob().stream());
|
|
70
|
-
const
|
|
71
|
-
expect(
|
|
72
|
-
expect(
|
|
93
|
+
const entries = await collect(mammoth.entries());
|
|
94
|
+
expect(entries.length).is(1);
|
|
95
|
+
expect(entries[0][0]).is(hash);
|
|
73
96
|
}),
|
|
74
97
|
},
|
|
75
98
|
});
|
package/x/test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,iCAAiC,CAAA;AAE5D,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC;IAEF,gBAAgB,EAAE;QACjB,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAChD,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC,CAAC;QAEF,mBAAmB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACnC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACrD,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC7C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC;QAEF,uBAAuB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACxB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,MAAM,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,GAAG,IAAI,EAAE,CAAA;YAChB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACzB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,oBAAoB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACpC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;QACjC,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAChD,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC,CAAC;KACF;CACD,CAAC,CAAA"}
|
package/s/browser/example.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import {Kv, StorageDriver} from "@e280/kv"
|
|
3
|
-
import {Mammoth} from "../core/mammoth.js"
|
|
4
|
-
import {IcebergOpfs} from "./iceberg-opfs.js"
|
|
5
|
-
|
|
6
|
-
export async function example() {
|
|
7
|
-
|
|
8
|
-
// setup the localstorage kv index
|
|
9
|
-
const driver = new StorageDriver(window.localStorage)
|
|
10
|
-
const kv = new Kv(driver)
|
|
11
|
-
const index = kv.scope("glacier_index")
|
|
12
|
-
|
|
13
|
-
// setup the opfs iceberg
|
|
14
|
-
const root = await navigator.storage.getDirectory()
|
|
15
|
-
const directory = await root.getDirectoryHandle("glacier", {create: true})
|
|
16
|
-
const iceberg = new IcebergOpfs(directory)
|
|
17
|
-
|
|
18
|
-
// setup the mammoth
|
|
19
|
-
const mammoth = new Mammoth(index, iceberg)
|
|
20
|
-
const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
|
|
21
|
-
|
|
22
|
-
// write a file as an example
|
|
23
|
-
const hash = await mammoth.write(bigFile.stream())
|
|
24
|
-
console.log(`mammoth stored file by hash "${hash}"`)
|
|
25
|
-
}
|
|
26
|
-
|
package/s/core/hash-stream.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import {hex, nap} from "@e280/stz"
|
|
3
|
-
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
|
-
import {relaxer} from "./utils/relaxer.js"
|
|
5
|
-
|
|
6
|
-
export async function hashStream(readable: ReadableStream<Uint8Array>) {
|
|
7
|
-
const hasher = blake3.create()
|
|
8
|
-
const relax = relaxer()
|
|
9
|
-
|
|
10
|
-
for await (const chunk of readable) {
|
|
11
|
-
hasher.update(chunk)
|
|
12
|
-
|
|
13
|
-
if (relax())
|
|
14
|
-
await nap()
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
return hex.fromBytes(hasher.digest())
|
|
18
|
-
}
|
|
19
|
-
|
package/x/browser/example.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function example(): Promise<void>;
|
package/x/browser/example.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import { Kv, StorageDriver } from "@e280/kv";
|
|
2
|
-
import { Mammoth } from "../core/mammoth.js";
|
|
3
|
-
import { IcebergOpfs } from "./iceberg-opfs.js";
|
|
4
|
-
export async function example() {
|
|
5
|
-
// setup the localstorage kv index
|
|
6
|
-
const driver = new StorageDriver(window.localStorage);
|
|
7
|
-
const kv = new Kv(driver);
|
|
8
|
-
const index = kv.scope("glacier_index");
|
|
9
|
-
// setup the opfs iceberg
|
|
10
|
-
const root = await navigator.storage.getDirectory();
|
|
11
|
-
const directory = await root.getDirectoryHandle("glacier", { create: true });
|
|
12
|
-
const iceberg = new IcebergOpfs(directory);
|
|
13
|
-
// setup the mammoth
|
|
14
|
-
const mammoth = new Mammoth(index, iceberg);
|
|
15
|
-
const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
|
|
16
|
-
// write a file as an example
|
|
17
|
-
const hash = await mammoth.write(bigFile.stream());
|
|
18
|
-
console.log(`mammoth stored file by hash "${hash}"`);
|
|
19
|
-
}
|
|
20
|
-
//# sourceMappingURL=example.js.map
|
package/x/browser/example.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;IAEvC,yBAAyB;IACzB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAA;IAE1C,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEpE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAClD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"iceberg-opfs.js","sourceRoot":"","sources":["../../s/browser/iceberg-opfs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,WAAW;IACvB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAA;QACnC,OAAO,IAAI,CAAC,IAAI,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}
|
package/x/browser/index.d.ts
DELETED
package/x/browser/index.js
DELETED
package/x/browser/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/browser/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,mBAAmB,CAAA"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function isNotFound(error: unknown): boolean;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"is-not-found.js","sourceRoot":"","sources":["../../../s/browser/utils/is-not-found.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,OAAO,KAAK,YAAY,YAAY;WAChC,KAAK,CAAC,IAAI,KAAK,eAAe,CAAA;AACnC,CAAC"}
|
package/x/core/hash-stream.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function hashStream(readable: ReadableStream<Uint8Array>): Promise<string>;
|
package/x/core/hash-stream.js
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { hex, nap } from "@e280/stz";
|
|
2
|
-
import { blake3 } from "@noble/hashes/blake3.js";
|
|
3
|
-
import { relaxer } from "./utils/relaxer.js";
|
|
4
|
-
export async function hashStream(readable) {
|
|
5
|
-
const hasher = blake3.create();
|
|
6
|
-
const relax = relaxer();
|
|
7
|
-
for await (const chunk of readable) {
|
|
8
|
-
hasher.update(chunk);
|
|
9
|
-
if (relax())
|
|
10
|
-
await nap();
|
|
11
|
-
}
|
|
12
|
-
return hex.fromBytes(hasher.digest());
|
|
13
|
-
}
|
|
14
|
-
//# sourceMappingURL=hash-stream.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hash-stream.js","sourceRoot":"","sources":["../../s/core/hash-stream.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAC9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAoC;IACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;AACtC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"iceberg-memory.js","sourceRoot":"","sources":["../../s/core/iceberg-memory.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,OAAO,aAAa;IACzB,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAA;IAEpC,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAA;IACzC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,IAAI,IAAI,CACd,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAA4B,CAAC,EACnD,EAAC,IAAI,EAAE,0BAA0B,EAAC,CAClC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;QACvB,MAAM,KAAK,GAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACjB,IAAI,KAAK,EAAE;gBAAE,MAAM,GAAG,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;CACD"}
|