@e280/mammoth 0.1.0-13 → 0.1.0-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/README.md +5 -114
  2. package/package.json +4 -7
  3. package/s/browser/example.ts +11 -10
  4. package/s/browser/{opfs-bucket.ts → iceberg-opfs.ts} +8 -2
  5. package/s/browser/index.ts +1 -1
  6. package/s/core/{analyze.ts → hash-stream.ts} +2 -6
  7. package/s/core/{memory-bucket.ts → iceberg-memory.ts} +6 -2
  8. package/s/core/index.ts +2 -3
  9. package/s/core/mammoth.ts +46 -54
  10. package/s/core/types.ts +9 -37
  11. package/s/test.ts +16 -41
  12. package/x/browser/example.js +11 -10
  13. package/x/browser/example.js.map +1 -1
  14. package/x/browser/{opfs-bucket.d.ts → iceberg-opfs.d.ts} +3 -2
  15. package/x/browser/{opfs-bucket.js → iceberg-opfs.js} +7 -2
  16. package/x/browser/iceberg-opfs.js.map +1 -0
  17. package/x/browser/index.d.ts +1 -1
  18. package/x/browser/index.js +1 -1
  19. package/x/browser/index.js.map +1 -1
  20. package/x/core/hash-stream.d.ts +1 -0
  21. package/x/core/{analyze.js → hash-stream.js} +3 -6
  22. package/x/core/hash-stream.js.map +1 -0
  23. package/x/core/{memory-bucket.d.ts → iceberg-memory.d.ts} +3 -2
  24. package/x/core/{memory-bucket.js → iceberg-memory.js} +5 -2
  25. package/x/core/iceberg-memory.js.map +1 -0
  26. package/x/core/index.d.ts +2 -3
  27. package/x/core/index.js +2 -3
  28. package/x/core/index.js.map +1 -1
  29. package/x/core/mammoth.d.ts +5 -7
  30. package/x/core/mammoth.js +40 -51
  31. package/x/core/mammoth.js.map +1 -1
  32. package/x/core/types.d.ts +9 -27
  33. package/x/test.js +16 -39
  34. package/x/test.js.map +1 -1
  35. package/s/core/consts.ts +0 -8
  36. package/s/core/manifest/fns/commit.ts +0 -27
  37. package/s/core/manifest/fns/get-expired-writes.ts +0 -11
  38. package/s/core/manifest/fns/get-stats.ts +0 -10
  39. package/s/core/manifest/fns/move-write-to-trash.ts +0 -11
  40. package/s/core/manifest/fns/need-info.ts +0 -10
  41. package/s/core/manifest/fns/schedule-deletion.ts +0 -19
  42. package/s/core/manifest/manifest.ts +0 -26
  43. package/s/core/streamify.ts +0 -18
  44. package/s/core/utils/cleanup.ts +0 -16
  45. package/s/core/utils/is-expired.ts +0 -10
  46. package/s/core/utils/not-found.ts +0 -7
  47. package/s/core/utils/save.ts +0 -28
  48. package/s/node/disk-bucket.ts +0 -55
  49. package/s/node/index.ts +0 -4
  50. package/s/node/utils/is-not-found.ts +0 -9
  51. package/x/browser/opfs-bucket.js.map +0 -1
  52. package/x/core/analyze.d.ts +0 -2
  53. package/x/core/analyze.js.map +0 -1
  54. package/x/core/consts.d.ts +0 -4
  55. package/x/core/consts.js +0 -6
  56. package/x/core/consts.js.map +0 -1
  57. package/x/core/manifest/fns/commit.d.ts +0 -3
  58. package/x/core/manifest/fns/commit.js +0 -22
  59. package/x/core/manifest/fns/commit.js.map +0 -1
  60. package/x/core/manifest/fns/get-expired-writes.d.ts +0 -2
  61. package/x/core/manifest/fns/get-expired-writes.js +0 -8
  62. package/x/core/manifest/fns/get-expired-writes.js.map +0 -1
  63. package/x/core/manifest/fns/get-stats.d.ts +0 -2
  64. package/x/core/manifest/fns/get-stats.js +0 -5
  65. package/x/core/manifest/fns/get-stats.js.map +0 -1
  66. package/x/core/manifest/fns/move-write-to-trash.d.ts +0 -3
  67. package/x/core/manifest/fns/move-write-to-trash.js +0 -7
  68. package/x/core/manifest/fns/move-write-to-trash.js.map +0 -1
  69. package/x/core/manifest/fns/need-info.d.ts +0 -3
  70. package/x/core/manifest/fns/need-info.js +0 -6
  71. package/x/core/manifest/fns/need-info.js.map +0 -1
  72. package/x/core/manifest/fns/schedule-deletion.d.ts +0 -3
  73. package/x/core/manifest/fns/schedule-deletion.js +0 -13
  74. package/x/core/manifest/fns/schedule-deletion.js.map +0 -1
  75. package/x/core/manifest/manifest.d.ts +0 -14
  76. package/x/core/manifest/manifest.js +0 -19
  77. package/x/core/manifest/manifest.js.map +0 -1
  78. package/x/core/memory-bucket.js.map +0 -1
  79. package/x/core/streamify.d.ts +0 -1
  80. package/x/core/streamify.js +0 -15
  81. package/x/core/streamify.js.map +0 -1
  82. package/x/core/utils/cleanup.d.ts +0 -3
  83. package/x/core/utils/cleanup.js +0 -11
  84. package/x/core/utils/cleanup.js.map +0 -1
  85. package/x/core/utils/is-expired.d.ts +0 -2
  86. package/x/core/utils/is-expired.js +0 -7
  87. package/x/core/utils/is-expired.js.map +0 -1
  88. package/x/core/utils/not-found.d.ts +0 -2
  89. package/x/core/utils/not-found.js +0 -4
  90. package/x/core/utils/not-found.js.map +0 -1
  91. package/x/core/utils/save.d.ts +0 -2
  92. package/x/core/utils/save.js +0 -16
  93. package/x/core/utils/save.js.map +0 -1
  94. package/x/node/disk-bucket.d.ts +0 -9
  95. package/x/node/disk-bucket.js +0 -43
  96. package/x/node/disk-bucket.js.map +0 -1
  97. package/x/node/index.d.ts +0 -2
  98. package/x/node/index.js +0 -3
  99. package/x/node/index.js.map +0 -1
  100. package/x/node/utils/is-not-found.d.ts +0 -1
  101. package/x/node/utils/is-not-found.js +0 -6
  102. package/x/node/utils/is-not-found.js.map +0 -1
@@ -1,28 +0,0 @@
1
-
2
- import {hex} from "@e280/stz"
3
- import {blake3} from "@noble/hashes/blake3.js"
4
- import {Analysis, Bucket, Id} from "../types.js"
5
-
6
- export async function save(
7
- bucket: Bucket,
8
- id: Id,
9
- readable: ReadableStream<Uint8Array>,
10
- ): Promise<Analysis> {
11
-
12
- let size = 0
13
- const hasher = blake3.create()
14
-
15
- await bucket.write(id, readable.pipeThrough(
16
- new TransformStream({
17
- transform(chunk, controller) {
18
- hasher.update(chunk)
19
- size += chunk.byteLength
20
- controller.enqueue(chunk)
21
- },
22
- })
23
- ))
24
-
25
- const hash = hex.fromBytes(hasher.digest())
26
- return {hash, size}
27
- }
28
-
@@ -1,55 +0,0 @@
1
-
2
- import {join} from "node:path"
3
- import {Readable} from "node:stream"
4
- import {pipeline} from "node:stream/promises"
5
- import {mkdir, stat, unlink} from "node:fs/promises"
6
- import {createWriteStream, openAsBlob} from "node:fs"
7
-
8
- import {Bucket, Id} from "../core/types.js"
9
- import {isNotFound} from "./utils/is-not-found.js"
10
-
11
- export class DiskBucket implements Bucket {
12
- #directory
13
-
14
- constructor(directory: string) {
15
- this.#directory = directory
16
- }
17
-
18
- async has(id: Id) {
19
- try {
20
- await stat(this.#path(id))
21
- return true
22
- }
23
- catch (error) {
24
- if (isNotFound(error)) return false
25
- throw error
26
- }
27
- }
28
-
29
- async delete(id: Id) {
30
- try {
31
- await unlink(this.#path(id))
32
- }
33
- catch (error) {
34
- if (!isNotFound(error))
35
- throw error
36
- }
37
- }
38
-
39
- async read(id: Id) {
40
- return openAsBlob(this.#path(id))
41
- }
42
-
43
- async write(id: Id, readable: ReadableStream<Uint8Array>) {
44
- await mkdir(this.#directory, {recursive: true})
45
- await pipeline(
46
- Readable.fromWeb(readable as any),
47
- createWriteStream(this.#path(id)),
48
- )
49
- }
50
-
51
- #path(id: Id) {
52
- return join(this.#directory, id)
53
- }
54
- }
55
-
package/s/node/index.ts DELETED
@@ -1,4 +0,0 @@
1
-
2
- export * from "../core/index.js"
3
- export * from "./disk-bucket.js"
4
-
@@ -1,9 +0,0 @@
1
-
2
- export function isNotFound(error: unknown): error is NodeJS.ErrnoException {
3
- return (
4
- error instanceof Error
5
- && "code" in error
6
- && error.code === "ENOENT"
7
- )
8
- }
9
-
@@ -1 +0,0 @@
1
- {"version":3,"file":"opfs-bucket.js","sourceRoot":"","sources":["../../s/browser/opfs-bucket.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}
@@ -1,2 +0,0 @@
1
- import { Analysis } from "./types.js";
2
- export declare function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis>;
@@ -1 +0,0 @@
1
- {"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../s/core/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAoC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
@@ -1,4 +0,0 @@
1
- export declare const consts: {
2
- max_jobs: number;
3
- expiry_time: number;
4
- };
package/x/core/consts.js DELETED
@@ -1,6 +0,0 @@
1
- import { time } from "@e280/stz";
2
- export const consts = {
3
- max_jobs: 64,
4
- expiry_time: time.days(7),
5
- };
6
- //# sourceMappingURL=consts.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;CACzB,CAAA"}
@@ -1,3 +0,0 @@
1
- import { Manifest } from "../manifest.js";
2
- import { Hash, Info } from "../../types.js";
3
- export declare function commit(manifest: Manifest, hash: Hash, info: Info): Promise<void>;
@@ -1,22 +0,0 @@
1
- import { getStats } from "./get-stats.js";
2
- export async function commit(manifest, hash, info) {
3
- const { kv, stats, catalog, writes, trash } = manifest;
4
- const isNewFile = !await catalog.has(hash);
5
- if (isNewFile) {
6
- const s = await getStats(manifest);
7
- s.count += 1;
8
- s.size += info.size;
9
- await kv.transaction(() => [
10
- writes.write.del(info.id),
11
- catalog.write.set(hash, info),
12
- stats.write.set(s)
13
- ]);
14
- }
15
- else {
16
- await kv.transaction(() => [
17
- writes.write.del(info.id),
18
- trash.write.set(info.id, true),
19
- ]);
20
- }
21
- }
22
- //# sourceMappingURL=commit.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"commit.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/commit.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IACtE,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IACpD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAClB,CAAC,CAAA;IACH,CAAC;SACI,CAAC;QACL,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAA;IACH,CAAC;AACF,CAAC"}
@@ -1,2 +0,0 @@
1
- import { Manifest } from "../manifest.js";
2
- export declare function getExpiredWrites({ writes }: Manifest): AsyncGenerator<string, void, unknown>;
@@ -1,8 +0,0 @@
1
- import { isExpired } from "../../utils/is-expired.js";
2
- export async function* getExpiredWrites({ writes }) {
3
- for await (const [id, w] of writes.entries()) {
4
- if (isExpired(w))
5
- yield id;
6
- }
7
- }
8
- //# sourceMappingURL=get-expired-writes.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"get-expired-writes.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-expired-writes.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAA;AAEnD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAW;IACzD,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}
@@ -1,2 +0,0 @@
1
- import { Manifest } from "../manifest.js";
2
- export declare function getStats({ stats }: Manifest): Promise<import("../../types.js").Stats>;
@@ -1,5 +0,0 @@
1
- export async function getStats({ stats }) {
2
- return structuredClone((await stats.get())
3
- ?? { count: 0, size: 0 });
4
- }
5
- //# sourceMappingURL=get-stats.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"get-stats.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,KAAK,EAAW;IAC/C,OAAO,eAAe,CACrB,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;WACf,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CACvB,CAAA;AACF,CAAC"}
@@ -1,3 +0,0 @@
1
- import { Id } from "../../types.js";
2
- import { Manifest } from "../manifest.js";
3
- export declare function moveWriteToTrash({ kv, writes, trash }: Manifest, id: Id): Promise<void>;
@@ -1,7 +0,0 @@
1
- export async function moveWriteToTrash({ kv, writes, trash }, id) {
2
- await kv.transaction(() => [
3
- writes.write.del(id),
4
- trash.write.set(id, true),
5
- ]);
6
- }
7
- //# sourceMappingURL=move-write-to-trash.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"move-write-to-trash.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/move-write-to-trash.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAW,EAAE,EAAM;IAC3E,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC,CAAA;AACH,CAAC"}
@@ -1,3 +0,0 @@
1
- import { Hash } from "../../types.js";
2
- import { Manifest } from "../manifest.js";
3
- export declare function needInfo({ catalog }: Manifest, hash: Hash): Promise<import("../../types.js").Info>;
@@ -1,6 +0,0 @@
1
- import { got } from "@e280/stz";
2
- import { notFound } from "../../utils/not-found.js";
3
- export async function needInfo({ catalog }, hash) {
4
- return got(await catalog.get(hash), notFound(hash));
5
- }
6
- //# sourceMappingURL=need-info.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"need-info.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/need-info.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAG7B,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,OAAO,EAAW,EAAE,IAAU;IAC7D,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACpD,CAAC"}
@@ -1,3 +0,0 @@
1
- import { Manifest } from "../manifest.js";
2
- import { Hash, Info } from "../../types.js";
3
- export declare function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info): Promise<void>;
@@ -1,13 +0,0 @@
1
- import { getStats } from "./get-stats.js";
2
- export async function scheduleDeletion(manifest, hash, info) {
3
- const { kv, stats, catalog, trash } = manifest;
4
- const s = await getStats(manifest);
5
- s.count -= 1;
6
- s.size -= info.size;
7
- await kv.transaction(() => [
8
- catalog.write.del(hash),
9
- trash.write.set(info.id, true),
10
- stats.write.set(s),
11
- ]);
12
- }
13
- //# sourceMappingURL=schedule-deletion.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"schedule-deletion.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/schedule-deletion.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IAChF,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IAE5C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;IAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;QAC9B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAClB,CAAC,CAAA;AACH,CAAC"}
@@ -1,14 +0,0 @@
1
- import { Kv } from "@e280/kv";
2
- import { Info, Stats, WriteRecord } from "../types.js";
3
- export declare class Manifest {
4
- kv: Kv;
5
- /** statistics about the whole datalake */
6
- stats: import("@e280/kv").Store<Stats>;
7
- /** associates file hashes with bucket ids */
8
- catalog: Kv<Info>;
9
- /** temporary record of writes in-progress */
10
- writes: Kv<WriteRecord>;
11
- /** bucket ids that are pending deletion */
12
- trash: Kv<true>;
13
- constructor(kv: Kv);
14
- }
@@ -1,19 +0,0 @@
1
- export class Manifest {
2
- kv;
3
- /** statistics about the whole datalake */
4
- stats;
5
- /** associates file hashes with bucket ids */
6
- catalog;
7
- /** temporary record of writes in-progress */
8
- writes;
9
- /** bucket ids that are pending deletion */
10
- trash;
11
- constructor(kv) {
12
- this.kv = kv;
13
- this.stats = kv.store("stats");
14
- this.catalog = kv.scope("catalog");
15
- this.writes = kv.scope("writes");
16
- this.trash = kv.scope("trash");
17
- }
18
- }
19
- //# sourceMappingURL=manifest.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAc,QAAQ,CAAC,CAAA;QAC7C,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"memory-bucket.js","sourceRoot":"","sources":["../../s/core/memory-bucket.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,OAAO,YAAY;IACxB,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAA;IAEpC,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,IAAI,IAAI,CACd,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAA4B,CAAC,EACnD,EAAC,IAAI,EAAE,0BAA0B,EAAC,CAClC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;QACvB,MAAM,KAAK,GAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACjB,IAAI,KAAK,EAAE;gBAAE,MAAM,GAAG,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;CACD"}
@@ -1 +0,0 @@
1
- export declare function streamify(bytes: Uint8Array, chunkSize?: number): ReadableStream<Uint8Array<ArrayBufferLike>>;
@@ -1,15 +0,0 @@
1
- export function streamify(bytes, chunkSize = 64 * 1024) {
2
- let offset = 0;
3
- return new ReadableStream({
4
- pull(controller) {
5
- if (offset >= bytes.length) {
6
- controller.close();
7
- return;
8
- }
9
- const end = Math.min(offset + chunkSize, bytes.length);
10
- controller.enqueue(bytes.subarray(offset, end));
11
- offset = end;
12
- },
13
- });
14
- }
15
- //# sourceMappingURL=streamify.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"streamify.js","sourceRoot":"","sources":["../../s/core/streamify.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,SAAS,CAAC,KAAiB,EAAE,SAAS,GAAG,EAAE,GAAG,IAAI;IACjE,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,OAAO,IAAI,cAAc,CAAa;QACrC,IAAI,CAAC,UAAU;YACd,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC5B,UAAU,CAAC,KAAK,EAAE,CAAA;gBAClB,OAAM;YACP,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;YACtD,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC"}
@@ -1,3 +0,0 @@
1
- import { Bucket } from "../types.js";
2
- import { Manifest } from "../manifest/manifest.js";
3
- export declare function cleanup(bucket: Bucket, manifest: Manifest): Promise<void>;
@@ -1,11 +0,0 @@
1
- import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
2
- import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
3
- export async function cleanup(bucket, manifest) {
4
- for await (const id of getExpiredWrites(manifest))
5
- await moveWriteToTrash(manifest, id);
6
- for await (const id of manifest.trash.keys()) {
7
- await bucket.delete(id);
8
- await manifest.trash.del(id);
9
- }
10
- }
11
- //# sourceMappingURL=cleanup.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
@@ -1,2 +0,0 @@
1
- import { WriteRecord } from "../types.js";
2
- export declare function isExpired({ created }: WriteRecord): boolean;
@@ -1,7 +0,0 @@
1
- import { time } from "@e280/stz";
2
- import { consts } from "../consts.js";
3
- export function isExpired({ created }) {
4
- const since = Date.now() - created;
5
- return since > time.days(consts.expiry_time);
6
- }
7
- //# sourceMappingURL=is-expired.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAc;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
@@ -1,2 +0,0 @@
1
- import { Hash } from "../types.js";
2
- export declare function notFound(hash: Hash): string;
@@ -1,4 +0,0 @@
1
- export function notFound(hash) {
2
- return `file not found by hash "${hash}"`;
3
- }
4
- //# sourceMappingURL=not-found.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"not-found.js","sourceRoot":"","sources":["../../../s/core/utils/not-found.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,QAAQ,CAAC,IAAU;IAClC,OAAO,2BAA2B,IAAI,GAAG,CAAA;AAC1C,CAAC"}
@@ -1,2 +0,0 @@
1
- import { Analysis, Bucket, Id } from "../types.js";
2
- export declare function save(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>): Promise<Analysis>;
@@ -1,16 +0,0 @@
1
- import { hex } from "@e280/stz";
2
- import { blake3 } from "@noble/hashes/blake3.js";
3
- export async function save(bucket, id, readable) {
4
- let size = 0;
5
- const hasher = blake3.create();
6
- await bucket.write(id, readable.pipeThrough(new TransformStream({
7
- transform(chunk, controller) {
8
- hasher.update(chunk);
9
- size += chunk.byteLength;
10
- controller.enqueue(chunk);
11
- },
12
- })));
13
- const hash = hex.fromBytes(hasher.digest());
14
- return { hash, size };
15
- }
16
- //# sourceMappingURL=save.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,WAAW,CAC1C,IAAI,eAAe,CAAC;QACnB,SAAS,CAAC,KAAK,EAAE,UAAU;YAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;YACxB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;KACD,CAAC,CACF,CAAC,CAAA;IAEF,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
@@ -1,9 +0,0 @@
1
- import { Bucket, Id } from "../core/types.js";
2
- export declare class DiskBucket implements Bucket {
3
- #private;
4
- constructor(directory: string);
5
- has(id: Id): Promise<boolean>;
6
- delete(id: Id): Promise<void>;
7
- read(id: Id): Promise<Blob>;
8
- write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
9
- }
@@ -1,43 +0,0 @@
1
- import { join } from "node:path";
2
- import { Readable } from "node:stream";
3
- import { pipeline } from "node:stream/promises";
4
- import { mkdir, stat, unlink } from "node:fs/promises";
5
- import { createWriteStream, openAsBlob } from "node:fs";
6
- import { isNotFound } from "./utils/is-not-found.js";
7
- export class DiskBucket {
8
- #directory;
9
- constructor(directory) {
10
- this.#directory = directory;
11
- }
12
- async has(id) {
13
- try {
14
- await stat(this.#path(id));
15
- return true;
16
- }
17
- catch (error) {
18
- if (isNotFound(error))
19
- return false;
20
- throw error;
21
- }
22
- }
23
- async delete(id) {
24
- try {
25
- await unlink(this.#path(id));
26
- }
27
- catch (error) {
28
- if (!isNotFound(error))
29
- throw error;
30
- }
31
- }
32
- async read(id) {
33
- return openAsBlob(this.#path(id));
34
- }
35
- async write(id, readable) {
36
- await mkdir(this.#directory, { recursive: true });
37
- await pipeline(Readable.fromWeb(readable), createWriteStream(this.#path(id)));
38
- }
39
- #path(id) {
40
- return join(this.#directory, id);
41
- }
42
- }
43
- //# sourceMappingURL=disk-bucket.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"disk-bucket.js","sourceRoot":"","sources":["../../s/node/disk-bucket.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAA;AACpC,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAC,iBAAiB,EAAE,UAAU,EAAC,MAAM,SAAS,CAAA;AAGrD,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAiB;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1B,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;QAC7B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAA;QAC/C,MAAM,QAAQ,CACb,QAAQ,CAAC,OAAO,CAAC,QAAe,CAAC,EACjC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CACjC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,EAAM;QACX,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACjC,CAAC;CACD"}
package/x/node/index.d.ts DELETED
@@ -1,2 +0,0 @@
1
- export * from "../core/index.js";
2
- export * from "./disk-bucket.js";
package/x/node/index.js DELETED
@@ -1,3 +0,0 @@
1
- export * from "../core/index.js";
2
- export * from "./disk-bucket.js";
3
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/node/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,kBAAkB,CAAA"}
@@ -1 +0,0 @@
1
- export declare function isNotFound(error: unknown): error is NodeJS.ErrnoException;
@@ -1,6 +0,0 @@
1
- export function isNotFound(error) {
2
- return (error instanceof Error
3
- && "code" in error
4
- && error.code === "ENOENT");
5
- }
6
- //# sourceMappingURL=is-not-found.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"is-not-found.js","sourceRoot":"","sources":["../../../s/node/utils/is-not-found.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,OAAO,CACN,KAAK,YAAY,KAAK;WACnB,MAAM,IAAI,KAAK;WACf,KAAK,CAAC,IAAI,KAAK,QAAQ,CAC1B,CAAA;AACF,CAAC"}