@e280/mammoth 0.1.0-13 → 0.1.0-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -114
- package/package.json +4 -7
- package/s/browser/example.ts +11 -10
- package/s/browser/{opfs-bucket.ts → iceberg-opfs.ts} +8 -2
- package/s/browser/index.ts +1 -1
- package/s/core/{analyze.ts → hash-stream.ts} +2 -6
- package/s/core/{memory-bucket.ts → iceberg-memory.ts} +6 -2
- package/s/core/index.ts +2 -3
- package/s/core/mammoth.ts +46 -54
- package/s/core/types.ts +9 -37
- package/s/test.ts +16 -41
- package/x/browser/example.js +11 -10
- package/x/browser/example.js.map +1 -1
- package/x/browser/{opfs-bucket.d.ts → iceberg-opfs.d.ts} +3 -2
- package/x/browser/{opfs-bucket.js → iceberg-opfs.js} +7 -2
- package/x/browser/iceberg-opfs.js.map +1 -0
- package/x/browser/index.d.ts +1 -1
- package/x/browser/index.js +1 -1
- package/x/browser/index.js.map +1 -1
- package/x/core/hash-stream.d.ts +1 -0
- package/x/core/{analyze.js → hash-stream.js} +3 -6
- package/x/core/hash-stream.js.map +1 -0
- package/x/core/{memory-bucket.d.ts → iceberg-memory.d.ts} +3 -2
- package/x/core/{memory-bucket.js → iceberg-memory.js} +5 -2
- package/x/core/iceberg-memory.js.map +1 -0
- package/x/core/index.d.ts +2 -3
- package/x/core/index.js +2 -3
- package/x/core/index.js.map +1 -1
- package/x/core/mammoth.d.ts +5 -7
- package/x/core/mammoth.js +40 -51
- package/x/core/mammoth.js.map +1 -1
- package/x/core/types.d.ts +9 -27
- package/x/test.js +16 -39
- package/x/test.js.map +1 -1
- package/s/core/consts.ts +0 -8
- package/s/core/manifest/fns/commit.ts +0 -27
- package/s/core/manifest/fns/get-expired-writes.ts +0 -11
- package/s/core/manifest/fns/get-stats.ts +0 -10
- package/s/core/manifest/fns/move-write-to-trash.ts +0 -11
- package/s/core/manifest/fns/need-info.ts +0 -10
- package/s/core/manifest/fns/schedule-deletion.ts +0 -19
- package/s/core/manifest/manifest.ts +0 -26
- package/s/core/streamify.ts +0 -18
- package/s/core/utils/cleanup.ts +0 -16
- package/s/core/utils/is-expired.ts +0 -10
- package/s/core/utils/not-found.ts +0 -7
- package/s/core/utils/save.ts +0 -28
- package/s/node/disk-bucket.ts +0 -55
- package/s/node/index.ts +0 -4
- package/s/node/utils/is-not-found.ts +0 -9
- package/x/browser/opfs-bucket.js.map +0 -1
- package/x/core/analyze.d.ts +0 -2
- package/x/core/analyze.js.map +0 -1
- package/x/core/consts.d.ts +0 -4
- package/x/core/consts.js +0 -6
- package/x/core/consts.js.map +0 -1
- package/x/core/manifest/fns/commit.d.ts +0 -3
- package/x/core/manifest/fns/commit.js +0 -22
- package/x/core/manifest/fns/commit.js.map +0 -1
- package/x/core/manifest/fns/get-expired-writes.d.ts +0 -2
- package/x/core/manifest/fns/get-expired-writes.js +0 -8
- package/x/core/manifest/fns/get-expired-writes.js.map +0 -1
- package/x/core/manifest/fns/get-stats.d.ts +0 -2
- package/x/core/manifest/fns/get-stats.js +0 -5
- package/x/core/manifest/fns/get-stats.js.map +0 -1
- package/x/core/manifest/fns/move-write-to-trash.d.ts +0 -3
- package/x/core/manifest/fns/move-write-to-trash.js +0 -7
- package/x/core/manifest/fns/move-write-to-trash.js.map +0 -1
- package/x/core/manifest/fns/need-info.d.ts +0 -3
- package/x/core/manifest/fns/need-info.js +0 -6
- package/x/core/manifest/fns/need-info.js.map +0 -1
- package/x/core/manifest/fns/schedule-deletion.d.ts +0 -3
- package/x/core/manifest/fns/schedule-deletion.js +0 -13
- package/x/core/manifest/fns/schedule-deletion.js.map +0 -1
- package/x/core/manifest/manifest.d.ts +0 -14
- package/x/core/manifest/manifest.js +0 -19
- package/x/core/manifest/manifest.js.map +0 -1
- package/x/core/memory-bucket.js.map +0 -1
- package/x/core/streamify.d.ts +0 -1
- package/x/core/streamify.js +0 -15
- package/x/core/streamify.js.map +0 -1
- package/x/core/utils/cleanup.d.ts +0 -3
- package/x/core/utils/cleanup.js +0 -11
- package/x/core/utils/cleanup.js.map +0 -1
- package/x/core/utils/is-expired.d.ts +0 -2
- package/x/core/utils/is-expired.js +0 -7
- package/x/core/utils/is-expired.js.map +0 -1
- package/x/core/utils/not-found.d.ts +0 -2
- package/x/core/utils/not-found.js +0 -4
- package/x/core/utils/not-found.js.map +0 -1
- package/x/core/utils/save.d.ts +0 -2
- package/x/core/utils/save.js +0 -16
- package/x/core/utils/save.js.map +0 -1
- package/x/node/disk-bucket.d.ts +0 -9
- package/x/node/disk-bucket.js +0 -43
- package/x/node/disk-bucket.js.map +0 -1
- package/x/node/index.d.ts +0 -2
- package/x/node/index.js +0 -3
- package/x/node/index.js.map +0 -1
- package/x/node/utils/is-not-found.d.ts +0 -1
- package/x/node/utils/is-not-found.js +0 -6
- package/x/node/utils/is-not-found.js.map +0 -1
package/s/core/utils/save.ts
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import {hex} from "@e280/stz"
|
|
3
|
-
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
|
-
import {Analysis, Bucket, Id} from "../types.js"
|
|
5
|
-
|
|
6
|
-
export async function save(
|
|
7
|
-
bucket: Bucket,
|
|
8
|
-
id: Id,
|
|
9
|
-
readable: ReadableStream<Uint8Array>,
|
|
10
|
-
): Promise<Analysis> {
|
|
11
|
-
|
|
12
|
-
let size = 0
|
|
13
|
-
const hasher = blake3.create()
|
|
14
|
-
|
|
15
|
-
await bucket.write(id, readable.pipeThrough(
|
|
16
|
-
new TransformStream({
|
|
17
|
-
transform(chunk, controller) {
|
|
18
|
-
hasher.update(chunk)
|
|
19
|
-
size += chunk.byteLength
|
|
20
|
-
controller.enqueue(chunk)
|
|
21
|
-
},
|
|
22
|
-
})
|
|
23
|
-
))
|
|
24
|
-
|
|
25
|
-
const hash = hex.fromBytes(hasher.digest())
|
|
26
|
-
return {hash, size}
|
|
27
|
-
}
|
|
28
|
-
|
package/s/node/disk-bucket.ts
DELETED
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import {join} from "node:path"
|
|
3
|
-
import {Readable} from "node:stream"
|
|
4
|
-
import {pipeline} from "node:stream/promises"
|
|
5
|
-
import {mkdir, stat, unlink} from "node:fs/promises"
|
|
6
|
-
import {createWriteStream, openAsBlob} from "node:fs"
|
|
7
|
-
|
|
8
|
-
import {Bucket, Id} from "../core/types.js"
|
|
9
|
-
import {isNotFound} from "./utils/is-not-found.js"
|
|
10
|
-
|
|
11
|
-
export class DiskBucket implements Bucket {
|
|
12
|
-
#directory
|
|
13
|
-
|
|
14
|
-
constructor(directory: string) {
|
|
15
|
-
this.#directory = directory
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
async has(id: Id) {
|
|
19
|
-
try {
|
|
20
|
-
await stat(this.#path(id))
|
|
21
|
-
return true
|
|
22
|
-
}
|
|
23
|
-
catch (error) {
|
|
24
|
-
if (isNotFound(error)) return false
|
|
25
|
-
throw error
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
async delete(id: Id) {
|
|
30
|
-
try {
|
|
31
|
-
await unlink(this.#path(id))
|
|
32
|
-
}
|
|
33
|
-
catch (error) {
|
|
34
|
-
if (!isNotFound(error))
|
|
35
|
-
throw error
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
async read(id: Id) {
|
|
40
|
-
return openAsBlob(this.#path(id))
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async write(id: Id, readable: ReadableStream<Uint8Array>) {
|
|
44
|
-
await mkdir(this.#directory, {recursive: true})
|
|
45
|
-
await pipeline(
|
|
46
|
-
Readable.fromWeb(readable as any),
|
|
47
|
-
createWriteStream(this.#path(id)),
|
|
48
|
-
)
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
#path(id: Id) {
|
|
52
|
-
return join(this.#directory, id)
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
package/s/node/index.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"opfs-bucket.js","sourceRoot":"","sources":["../../s/browser/opfs-bucket.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}
|
package/x/core/analyze.d.ts
DELETED
package/x/core/analyze.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../s/core/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAoC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
package/x/core/consts.d.ts
DELETED
package/x/core/consts.js
DELETED
package/x/core/consts.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;CACzB,CAAA"}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { getStats } from "./get-stats.js";
|
|
2
|
-
export async function commit(manifest, hash, info) {
|
|
3
|
-
const { kv, stats, catalog, writes, trash } = manifest;
|
|
4
|
-
const isNewFile = !await catalog.has(hash);
|
|
5
|
-
if (isNewFile) {
|
|
6
|
-
const s = await getStats(manifest);
|
|
7
|
-
s.count += 1;
|
|
8
|
-
s.size += info.size;
|
|
9
|
-
await kv.transaction(() => [
|
|
10
|
-
writes.write.del(info.id),
|
|
11
|
-
catalog.write.set(hash, info),
|
|
12
|
-
stats.write.set(s)
|
|
13
|
-
]);
|
|
14
|
-
}
|
|
15
|
-
else {
|
|
16
|
-
await kv.transaction(() => [
|
|
17
|
-
writes.write.del(info.id),
|
|
18
|
-
trash.write.set(info.id, true),
|
|
19
|
-
]);
|
|
20
|
-
}
|
|
21
|
-
}
|
|
22
|
-
//# sourceMappingURL=commit.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"commit.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/commit.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IACtE,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IACpD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAClB,CAAC,CAAA;IACH,CAAC;SACI,CAAC;QACL,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAA;IACH,CAAC;AACF,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"get-expired-writes.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-expired-writes.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAA;AAEnD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAW;IACzD,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"get-stats.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,KAAK,EAAW;IAC/C,OAAO,eAAe,CACrB,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;WACf,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CACvB,CAAA;AACF,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"move-write-to-trash.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/move-write-to-trash.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAW,EAAE,EAAM;IAC3E,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"need-info.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/need-info.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAG7B,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,OAAO,EAAW,EAAE,IAAU;IAC7D,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACpD,CAAC"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { getStats } from "./get-stats.js";
|
|
2
|
-
export async function scheduleDeletion(manifest, hash, info) {
|
|
3
|
-
const { kv, stats, catalog, trash } = manifest;
|
|
4
|
-
const s = await getStats(manifest);
|
|
5
|
-
s.count -= 1;
|
|
6
|
-
s.size -= info.size;
|
|
7
|
-
await kv.transaction(() => [
|
|
8
|
-
catalog.write.del(hash),
|
|
9
|
-
trash.write.set(info.id, true),
|
|
10
|
-
stats.write.set(s),
|
|
11
|
-
]);
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=schedule-deletion.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"schedule-deletion.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/schedule-deletion.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IAChF,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IAE5C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;IAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;QAC9B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAClB,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { Kv } from "@e280/kv";
|
|
2
|
-
import { Info, Stats, WriteRecord } from "../types.js";
|
|
3
|
-
export declare class Manifest {
|
|
4
|
-
kv: Kv;
|
|
5
|
-
/** statistics about the whole datalake */
|
|
6
|
-
stats: import("@e280/kv").Store<Stats>;
|
|
7
|
-
/** associates file hashes with bucket ids */
|
|
8
|
-
catalog: Kv<Info>;
|
|
9
|
-
/** temporary record of writes in-progress */
|
|
10
|
-
writes: Kv<WriteRecord>;
|
|
11
|
-
/** bucket ids that are pending deletion */
|
|
12
|
-
trash: Kv<true>;
|
|
13
|
-
constructor(kv: Kv);
|
|
14
|
-
}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
export class Manifest {
|
|
2
|
-
kv;
|
|
3
|
-
/** statistics about the whole datalake */
|
|
4
|
-
stats;
|
|
5
|
-
/** associates file hashes with bucket ids */
|
|
6
|
-
catalog;
|
|
7
|
-
/** temporary record of writes in-progress */
|
|
8
|
-
writes;
|
|
9
|
-
/** bucket ids that are pending deletion */
|
|
10
|
-
trash;
|
|
11
|
-
constructor(kv) {
|
|
12
|
-
this.kv = kv;
|
|
13
|
-
this.stats = kv.store("stats");
|
|
14
|
-
this.catalog = kv.scope("catalog");
|
|
15
|
-
this.writes = kv.scope("writes");
|
|
16
|
-
this.trash = kv.scope("trash");
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
//# sourceMappingURL=manifest.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAc,QAAQ,CAAC,CAAA;QAC7C,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"memory-bucket.js","sourceRoot":"","sources":["../../s/core/memory-bucket.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,OAAO,YAAY;IACxB,IAAI,GAAG,IAAI,GAAG,EAAsB,CAAA;IAEpC,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,IAAI,IAAI,CACd,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAA4B,CAAC,EACnD,EAAC,IAAI,EAAE,0BAA0B,EAAC,CAClC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;QACvB,MAAM,KAAK,GAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACjB,IAAI,KAAK,EAAE;gBAAE,MAAM,GAAG,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;CACD"}
|
package/x/core/streamify.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function streamify(bytes: Uint8Array, chunkSize?: number): ReadableStream<Uint8Array<ArrayBufferLike>>;
|
package/x/core/streamify.js
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
export function streamify(bytes, chunkSize = 64 * 1024) {
|
|
2
|
-
let offset = 0;
|
|
3
|
-
return new ReadableStream({
|
|
4
|
-
pull(controller) {
|
|
5
|
-
if (offset >= bytes.length) {
|
|
6
|
-
controller.close();
|
|
7
|
-
return;
|
|
8
|
-
}
|
|
9
|
-
const end = Math.min(offset + chunkSize, bytes.length);
|
|
10
|
-
controller.enqueue(bytes.subarray(offset, end));
|
|
11
|
-
offset = end;
|
|
12
|
-
},
|
|
13
|
-
});
|
|
14
|
-
}
|
|
15
|
-
//# sourceMappingURL=streamify.js.map
|
package/x/core/streamify.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"streamify.js","sourceRoot":"","sources":["../../s/core/streamify.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,SAAS,CAAC,KAAiB,EAAE,SAAS,GAAG,EAAE,GAAG,IAAI;IACjE,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,OAAO,IAAI,cAAc,CAAa;QACrC,IAAI,CAAC,UAAU;YACd,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC5B,UAAU,CAAC,KAAK,EAAE,CAAA;gBAClB,OAAM;YACP,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAA;YACtD,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAA;YAC/C,MAAM,GAAG,GAAG,CAAA;QACb,CAAC;KACD,CAAC,CAAA;AACH,CAAC"}
|
package/x/core/utils/cleanup.js
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
|
|
2
|
-
import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
|
|
3
|
-
export async function cleanup(bucket, manifest) {
|
|
4
|
-
for await (const id of getExpiredWrites(manifest))
|
|
5
|
-
await moveWriteToTrash(manifest, id);
|
|
6
|
-
for await (const id of manifest.trash.keys()) {
|
|
7
|
-
await bucket.delete(id);
|
|
8
|
-
await manifest.trash.del(id);
|
|
9
|
-
}
|
|
10
|
-
}
|
|
11
|
-
//# sourceMappingURL=cleanup.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAc;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"not-found.js","sourceRoot":"","sources":["../../../s/core/utils/not-found.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,QAAQ,CAAC,IAAU;IAClC,OAAO,2BAA2B,IAAI,GAAG,CAAA;AAC1C,CAAC"}
|
package/x/core/utils/save.d.ts
DELETED
package/x/core/utils/save.js
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { hex } from "@e280/stz";
|
|
2
|
-
import { blake3 } from "@noble/hashes/blake3.js";
|
|
3
|
-
export async function save(bucket, id, readable) {
|
|
4
|
-
let size = 0;
|
|
5
|
-
const hasher = blake3.create();
|
|
6
|
-
await bucket.write(id, readable.pipeThrough(new TransformStream({
|
|
7
|
-
transform(chunk, controller) {
|
|
8
|
-
hasher.update(chunk);
|
|
9
|
-
size += chunk.byteLength;
|
|
10
|
-
controller.enqueue(chunk);
|
|
11
|
-
},
|
|
12
|
-
})));
|
|
13
|
-
const hash = hex.fromBytes(hasher.digest());
|
|
14
|
-
return { hash, size };
|
|
15
|
-
}
|
|
16
|
-
//# sourceMappingURL=save.js.map
|
package/x/core/utils/save.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,MAAM,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,CAAC,WAAW,CAC1C,IAAI,eAAe,CAAC;QACnB,SAAS,CAAC,KAAK,EAAE,UAAU;YAC1B,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;YACxB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC;KACD,CAAC,CACF,CAAC,CAAA;IAEF,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
package/x/node/disk-bucket.d.ts
DELETED
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { Bucket, Id } from "../core/types.js";
|
|
2
|
-
export declare class DiskBucket implements Bucket {
|
|
3
|
-
#private;
|
|
4
|
-
constructor(directory: string);
|
|
5
|
-
has(id: Id): Promise<boolean>;
|
|
6
|
-
delete(id: Id): Promise<void>;
|
|
7
|
-
read(id: Id): Promise<Blob>;
|
|
8
|
-
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
|
|
9
|
-
}
|
package/x/node/disk-bucket.js
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import { join } from "node:path";
|
|
2
|
-
import { Readable } from "node:stream";
|
|
3
|
-
import { pipeline } from "node:stream/promises";
|
|
4
|
-
import { mkdir, stat, unlink } from "node:fs/promises";
|
|
5
|
-
import { createWriteStream, openAsBlob } from "node:fs";
|
|
6
|
-
import { isNotFound } from "./utils/is-not-found.js";
|
|
7
|
-
export class DiskBucket {
|
|
8
|
-
#directory;
|
|
9
|
-
constructor(directory) {
|
|
10
|
-
this.#directory = directory;
|
|
11
|
-
}
|
|
12
|
-
async has(id) {
|
|
13
|
-
try {
|
|
14
|
-
await stat(this.#path(id));
|
|
15
|
-
return true;
|
|
16
|
-
}
|
|
17
|
-
catch (error) {
|
|
18
|
-
if (isNotFound(error))
|
|
19
|
-
return false;
|
|
20
|
-
throw error;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
async delete(id) {
|
|
24
|
-
try {
|
|
25
|
-
await unlink(this.#path(id));
|
|
26
|
-
}
|
|
27
|
-
catch (error) {
|
|
28
|
-
if (!isNotFound(error))
|
|
29
|
-
throw error;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
async read(id) {
|
|
33
|
-
return openAsBlob(this.#path(id));
|
|
34
|
-
}
|
|
35
|
-
async write(id, readable) {
|
|
36
|
-
await mkdir(this.#directory, { recursive: true });
|
|
37
|
-
await pipeline(Readable.fromWeb(readable), createWriteStream(this.#path(id)));
|
|
38
|
-
}
|
|
39
|
-
#path(id) {
|
|
40
|
-
return join(this.#directory, id);
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=disk-bucket.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"disk-bucket.js","sourceRoot":"","sources":["../../s/node/disk-bucket.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,QAAQ,EAAC,MAAM,aAAa,CAAA;AACpC,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAC,iBAAiB,EAAE,UAAU,EAAC,MAAM,SAAS,CAAA;AAGrD,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,UAAU;IACtB,UAAU,CAAA;IAEV,YAAY,SAAiB;QAC5B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;YAC1B,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;QAC7B,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,OAAO,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAA;QAC/C,MAAM,QAAQ,CACb,QAAQ,CAAC,OAAO,CAAC,QAAe,CAAC,EACjC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CACjC,CAAA;IACF,CAAC;IAED,KAAK,CAAC,EAAM;QACX,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;IACjC,CAAC;CACD"}
|
package/x/node/index.d.ts
DELETED
package/x/node/index.js
DELETED
package/x/node/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/node/index.ts"],"names":[],"mappings":"AACA,cAAc,kBAAkB,CAAA;AAChC,cAAc,kBAAkB,CAAA"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function isNotFound(error: unknown): error is NodeJS.ErrnoException;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"is-not-found.js","sourceRoot":"","sources":["../../../s/node/utils/is-not-found.ts"],"names":[],"mappings":"AACA,MAAM,UAAU,UAAU,CAAC,KAAc;IACxC,OAAO,CACN,KAAK,YAAY,KAAK;WACnB,MAAM,IAAI,KAAK;WACf,KAAK,CAAC,IAAI,KAAK,QAAQ,CAC1B,CAAA;AACF,CAAC"}
|