@e280/mammoth 0.1.0-14 → 0.1.0-3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -114
- package/package.json +5 -8
- package/s/browser/example.ts +11 -10
- package/s/browser/{opfs-bucket.ts → iceberg-opfs.ts} +8 -2
- package/s/browser/index.ts +1 -1
- package/s/core/hash-stream.ts +19 -0
- package/s/core/{memory-bucket.ts → iceberg-memory.ts} +6 -2
- package/s/core/index.ts +2 -3
- package/s/core/mammoth.ts +46 -54
- package/s/core/types.ts +9 -37
- package/s/test.ts +16 -41
- package/x/browser/example.js +11 -10
- package/x/browser/example.js.map +1 -1
- package/x/browser/{opfs-bucket.d.ts → iceberg-opfs.d.ts} +3 -2
- package/x/browser/{opfs-bucket.js → iceberg-opfs.js} +7 -2
- package/x/browser/iceberg-opfs.js.map +1 -0
- package/x/browser/index.d.ts +1 -1
- package/x/browser/index.js +1 -1
- package/x/browser/index.js.map +1 -1
- package/x/core/hash-stream.d.ts +1 -0
- package/x/core/{analyze.js → hash-stream.js} +4 -7
- package/x/core/hash-stream.js.map +1 -0
- package/x/core/{memory-bucket.d.ts → iceberg-memory.d.ts} +3 -2
- package/x/core/{memory-bucket.js → iceberg-memory.js} +5 -2
- package/x/core/iceberg-memory.js.map +1 -0
- package/x/core/index.d.ts +2 -3
- package/x/core/index.js +2 -3
- package/x/core/index.js.map +1 -1
- package/x/core/mammoth.d.ts +5 -7
- package/x/core/mammoth.js +40 -51
- package/x/core/mammoth.js.map +1 -1
- package/x/core/types.d.ts +9 -27
- package/x/test.js +16 -39
- package/x/test.js.map +1 -1
- package/s/core/analyze.ts +0 -23
- package/s/core/consts.ts +0 -8
- package/s/core/manifest/fns/commit.ts +0 -27
- package/s/core/manifest/fns/get-expired-writes.ts +0 -11
- package/s/core/manifest/fns/get-stats.ts +0 -10
- package/s/core/manifest/fns/move-write-to-trash.ts +0 -11
- package/s/core/manifest/fns/need-info.ts +0 -10
- package/s/core/manifest/fns/schedule-deletion.ts +0 -19
- package/s/core/manifest/manifest.ts +0 -26
- package/s/core/streamify.ts +0 -18
- package/s/core/utils/cleanup.ts +0 -16
- package/s/core/utils/is-expired.ts +0 -10
- package/s/core/utils/not-found.ts +0 -7
- package/s/core/utils/save.ts +0 -28
- package/s/node/disk-bucket.ts +0 -55
- package/s/node/index.ts +0 -4
- package/s/node/utils/is-not-found.ts +0 -9
- package/x/browser/opfs-bucket.js.map +0 -1
- package/x/core/analyze.d.ts +0 -2
- package/x/core/analyze.js.map +0 -1
- package/x/core/consts.d.ts +0 -4
- package/x/core/consts.js +0 -6
- package/x/core/consts.js.map +0 -1
- package/x/core/manifest/fns/commit.d.ts +0 -3
- package/x/core/manifest/fns/commit.js +0 -22
- package/x/core/manifest/fns/commit.js.map +0 -1
- package/x/core/manifest/fns/get-expired-writes.d.ts +0 -2
- package/x/core/manifest/fns/get-expired-writes.js +0 -8
- package/x/core/manifest/fns/get-expired-writes.js.map +0 -1
- package/x/core/manifest/fns/get-stats.d.ts +0 -2
- package/x/core/manifest/fns/get-stats.js +0 -5
- package/x/core/manifest/fns/get-stats.js.map +0 -1
- package/x/core/manifest/fns/move-write-to-trash.d.ts +0 -3
- package/x/core/manifest/fns/move-write-to-trash.js +0 -7
- package/x/core/manifest/fns/move-write-to-trash.js.map +0 -1
- package/x/core/manifest/fns/need-info.d.ts +0 -3
- package/x/core/manifest/fns/need-info.js +0 -6
- package/x/core/manifest/fns/need-info.js.map +0 -1
- package/x/core/manifest/fns/schedule-deletion.d.ts +0 -3
- package/x/core/manifest/fns/schedule-deletion.js +0 -13
- package/x/core/manifest/fns/schedule-deletion.js.map +0 -1
- package/x/core/manifest/manifest.d.ts +0 -14
- package/x/core/manifest/manifest.js +0 -19
- package/x/core/manifest/manifest.js.map +0 -1
- package/x/core/memory-bucket.js.map +0 -1
- package/x/core/streamify.d.ts +0 -1
- package/x/core/streamify.js +0 -15
- package/x/core/streamify.js.map +0 -1
- package/x/core/utils/cleanup.d.ts +0 -3
- package/x/core/utils/cleanup.js +0 -11
- package/x/core/utils/cleanup.js.map +0 -1
- package/x/core/utils/is-expired.d.ts +0 -2
- package/x/core/utils/is-expired.js +0 -7
- package/x/core/utils/is-expired.js.map +0 -1
- package/x/core/utils/not-found.d.ts +0 -2
- package/x/core/utils/not-found.js +0 -4
- package/x/core/utils/not-found.js.map +0 -1
- package/x/core/utils/save.d.ts +0 -2
- package/x/core/utils/save.js +0 -16
- package/x/core/utils/save.js.map +0 -1
- package/x/node/disk-bucket.d.ts +0 -9
- package/x/node/disk-bucket.js +0 -43
- package/x/node/disk-bucket.js.map +0 -1
- package/x/node/index.d.ts +0 -2
- package/x/node/index.js +0 -3
- package/x/node/index.js.map +0 -1
- package/x/node/utils/is-not-found.d.ts +0 -1
- package/x/node/utils/is-not-found.js +0 -6
- package/x/node/utils/is-not-found.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,122 +1,13 @@
|
|
|
1
1
|
|
|
2
|
-
|
|
2
|
+
# 🦣 @e280/mammoth
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
> *big files. small api.*
|
|
6
|
-
|
|
7
|
-
**mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs. mammoth uses [kv](https://github.com/e280/kv) for bookkeeping.
|
|
8
|
-
|
|
9
|
-
```bash
|
|
10
|
-
npm install @e280/mammoth @e280/kv
|
|
11
|
-
```
|
|
12
|
-
|
|
13
|
-
```ts
|
|
14
|
-
import {Mammoth} from "@e280/mammoth"
|
|
15
|
-
|
|
16
|
-
const mammoth = new Mammoth()
|
|
17
|
-
```
|
|
18
|
-
|
|
19
|
-
## 🦣 mammoth stores files.
|
|
20
|
-
- **write a file,** and you get back its blake3 `hash` and `size` in bytes.
|
|
21
|
-
```ts
|
|
22
|
-
const hash = await mammoth.write(blob.stream())
|
|
23
|
-
```
|
|
24
|
-
- **read a file,** identified by its hash.
|
|
25
|
-
```ts
|
|
26
|
-
const blob = await mammoth.read(hash)
|
|
27
|
-
```
|
|
28
|
-
- **delete a file.**
|
|
29
|
-
```ts
|
|
30
|
-
await mammoth.delete(hash)
|
|
31
|
-
```
|
|
32
|
-
|
|
33
|
-
## 🦣 mammoth is bucket-agnostic.
|
|
34
|
-
- **memory bucket,** for testing.
|
|
35
|
-
```ts
|
|
36
|
-
import {Mammoth, MemoryBucket} from "@e280/mammoth"
|
|
37
|
-
import {Kv} from "@e280/kv"
|
|
38
|
-
|
|
39
|
-
const mammoth = new Mammoth(
|
|
40
|
-
|
|
41
|
-
// bucket for blob storage
|
|
42
|
-
new MemoryBucket(),
|
|
43
|
-
|
|
44
|
-
// kv for metadata bookkeeping
|
|
45
|
-
new Kv(),
|
|
46
|
-
)
|
|
47
|
-
```
|
|
48
|
-
defaults shown, this is equivalent:
|
|
49
|
-
```ts
|
|
50
|
-
const mammoth = new Mammoth()
|
|
51
|
-
```
|
|
52
|
-
- **disk bucket,** for nodejs servers. *(note the import paths)*
|
|
53
|
-
```ts
|
|
54
|
-
import {Mammoth} from "@e280/mammoth"
|
|
55
|
-
import {DiskBucket} from "@e280/mammoth/node"
|
|
56
|
-
import {Kv} from "@e280/kv"
|
|
57
|
-
import {LevelDriver} from "@e280/kv/level"
|
|
58
|
-
|
|
59
|
-
const mammoth = new Mammoth(
|
|
60
|
-
new DiskBucket("./data/bucket"),
|
|
61
|
-
new Kv(new LevelDriver("./data/kv")),
|
|
62
|
-
)
|
|
63
|
-
```
|
|
64
|
-
- **opfs bucket,** for local storage in the browser. *(note the import paths)*
|
|
65
|
-
```ts
|
|
66
|
-
import {Mammoth} from "@e280/mammoth"
|
|
67
|
-
import {OpfsBucket} from "@e280/mammoth/browser"
|
|
68
|
-
import {Kv, StorageDriver} from "@e280/kv"
|
|
69
|
-
|
|
70
|
-
const mammoth = new Mammoth(
|
|
71
|
-
new OpfsBucket(await navigator.storage.getDirectory()),
|
|
72
|
-
new Kv(new StorageDriver(localStorage)),
|
|
73
|
-
)
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
## 🦣 more mammoth methods.
|
|
77
|
-
- **check if a file exists,** get back a boolean.
|
|
78
|
-
```ts
|
|
79
|
-
const exists = await mammoth.has(hash)
|
|
80
|
-
```
|
|
81
|
-
- **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
|
|
82
|
-
```ts
|
|
83
|
-
const {size, added, id} = await mammoth.info(hash)
|
|
84
|
-
```
|
|
85
|
-
- **get stats,** for the whole datalake.
|
|
86
|
-
```ts
|
|
87
|
-
await mammoth.stats()
|
|
88
|
-
// {count: 123, size: 123456789}
|
|
89
|
-
```
|
|
90
|
-
- **loop over every file,** with their hashes and infos.
|
|
91
|
-
```ts
|
|
92
|
-
for await (const [hash, info] of mammoth.entries())
|
|
93
|
-
console.log(hash, info)
|
|
94
|
-
```
|
|
95
|
-
|
|
96
|
-
## 🦣 mammoth-brained tips.
|
|
97
|
-
- **`analyze` is for hashing files.**
|
|
98
|
-
```ts
|
|
99
|
-
import {analyze} from "@e280/mammoth"
|
|
100
|
-
|
|
101
|
-
const {hash, size} = await analyze(blob.stream())
|
|
102
|
-
```
|
|
103
|
-
- **`analyze` can help you avoid unnecessary uploads.**
|
|
104
|
-
```ts
|
|
105
|
-
const {hash} = await analyze(blob.stream())
|
|
106
|
-
|
|
107
|
-
if (!await mammoth.has(hash))
|
|
108
|
-
await mammoth.write(blob.stream())
|
|
109
|
-
```
|
|
110
|
-
- **`streamify` makes a stream for a Uint8Array.**
|
|
111
|
-
```ts
|
|
112
|
-
import {streamify} from "@e280/mammoth"
|
|
113
|
-
|
|
114
|
-
const readable = await streamify(new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF]))
|
|
115
|
-
```
|
|
4
|
+
big files. small api.
|
|
116
5
|
|
|
117
6
|
|
|
118
7
|
|
|
119
8
|
<br/><br/>
|
|
120
9
|
|
|
121
|
-
|
|
10
|
+
## 💖 mammoth is by e280
|
|
11
|
+
reward us with github stars
|
|
12
|
+
build with us at https://e280.org/ but only if you're cool
|
|
122
13
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@e280/mammoth",
|
|
3
|
-
"version": "0.1.0-
|
|
3
|
+
"version": "0.1.0-3",
|
|
4
4
|
"description": "big file storage",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Chase Moskal <chasemoskal@gmail.com>",
|
|
@@ -9,8 +9,7 @@
|
|
|
9
9
|
"sideEffects": false,
|
|
10
10
|
"exports": {
|
|
11
11
|
".": "./x/core/index.js",
|
|
12
|
-
"./browser": "./x/browser/index.js"
|
|
13
|
-
"./node": "./x/node/index.js"
|
|
12
|
+
"./browser": "./x/browser/index.js"
|
|
14
13
|
},
|
|
15
14
|
"files": [
|
|
16
15
|
"x",
|
|
@@ -22,12 +21,10 @@
|
|
|
22
21
|
"test": "node x/test.js",
|
|
23
22
|
"count": "find s -path '*/_archive' -prune -o -name '*.ts' -exec wc -l {} +"
|
|
24
23
|
},
|
|
25
|
-
"peerDependencies": {
|
|
26
|
-
"@e280/kv": "^0.1.5"
|
|
27
|
-
},
|
|
28
24
|
"dependencies": {
|
|
29
|
-
"@
|
|
30
|
-
"@e280/stz": "^0.3.
|
|
25
|
+
"@e280/kv": "^0.1.5",
|
|
26
|
+
"@e280/stz": "^0.3.4",
|
|
27
|
+
"@noble/hashes": "^2.2.0"
|
|
31
28
|
},
|
|
32
29
|
"devDependencies": {
|
|
33
30
|
"@e280/octo": "^0.1.3",
|
package/s/browser/example.ts
CHANGED
|
@@ -1,25 +1,26 @@
|
|
|
1
1
|
|
|
2
2
|
import {Kv, StorageDriver} from "@e280/kv"
|
|
3
3
|
import {Mammoth} from "../core/mammoth.js"
|
|
4
|
-
import {
|
|
4
|
+
import {IcebergOpfs} from "./iceberg-opfs.js"
|
|
5
5
|
|
|
6
6
|
export async function example() {
|
|
7
7
|
|
|
8
|
-
// setup the localstorage kv
|
|
8
|
+
// setup the localstorage kv index
|
|
9
9
|
const driver = new StorageDriver(window.localStorage)
|
|
10
|
-
const kv = new Kv(driver)
|
|
10
|
+
const kv = new Kv(driver)
|
|
11
|
+
const index = kv.scope("glacier_index")
|
|
11
12
|
|
|
12
|
-
// setup the opfs
|
|
13
|
+
// setup the opfs iceberg
|
|
13
14
|
const root = await navigator.storage.getDirectory()
|
|
14
|
-
const directory = await root.getDirectoryHandle("
|
|
15
|
-
const
|
|
15
|
+
const directory = await root.getDirectoryHandle("glacier", {create: true})
|
|
16
|
+
const iceberg = new IcebergOpfs(directory)
|
|
16
17
|
|
|
17
18
|
// setup the mammoth
|
|
18
|
-
const mammoth = new Mammoth(
|
|
19
|
-
const
|
|
19
|
+
const mammoth = new Mammoth(index, iceberg)
|
|
20
|
+
const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
|
|
20
21
|
|
|
21
22
|
// write a file as an example
|
|
22
|
-
const hash = await mammoth.write(
|
|
23
|
-
console.log(`mammoth stored file "${hash}"`)
|
|
23
|
+
const hash = await mammoth.write(bigFile.stream())
|
|
24
|
+
console.log(`mammoth stored file by hash "${hash}"`)
|
|
24
25
|
}
|
|
25
26
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
|
|
2
|
-
import {
|
|
2
|
+
import {Iceberg, Id} from "../core/types.js"
|
|
3
3
|
import {isNotFound} from "./utils/is-not-found.js"
|
|
4
4
|
|
|
5
|
-
export class
|
|
5
|
+
export class IcebergOpfs implements Iceberg {
|
|
6
6
|
#directory
|
|
7
7
|
|
|
8
8
|
constructor(directory: FileSystemDirectoryHandle) {
|
|
@@ -20,6 +20,12 @@ export class OpfsBucket implements Bucket {
|
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
async size(id: Id) {
|
|
24
|
+
const handle = await this.#directory.getFileHandle(id)
|
|
25
|
+
const file = await handle.getFile()
|
|
26
|
+
return file.size
|
|
27
|
+
}
|
|
28
|
+
|
|
23
29
|
async delete(id: Id) {
|
|
24
30
|
try {
|
|
25
31
|
await this.#directory.removeEntry(id)
|
package/s/browser/index.ts
CHANGED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
|
|
2
|
+
import {hex, nap} from "@e280/stz"
|
|
3
|
+
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
|
+
import {relaxer} from "./utils/relaxer.js"
|
|
5
|
+
|
|
6
|
+
export async function hashStream(readable: ReadableStream<Uint8Array>) {
|
|
7
|
+
const hasher = blake3.create()
|
|
8
|
+
const relax = relaxer()
|
|
9
|
+
|
|
10
|
+
for await (const chunk of readable) {
|
|
11
|
+
hasher.update(chunk)
|
|
12
|
+
|
|
13
|
+
if (relax())
|
|
14
|
+
await nap()
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
return hex.fromBytes(hasher.digest())
|
|
18
|
+
}
|
|
19
|
+
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
|
|
2
2
|
import {bytes, got, nap} from "@e280/stz"
|
|
3
|
-
import {
|
|
3
|
+
import {Iceberg, Id} from "./types.js"
|
|
4
4
|
import {relaxer} from "./utils/relaxer.js"
|
|
5
5
|
|
|
6
|
-
export class
|
|
6
|
+
export class IcebergMemory implements Iceberg {
|
|
7
7
|
#map = new Map<string, Uint8Array>()
|
|
8
8
|
|
|
9
9
|
async has(id: Id) {
|
|
10
10
|
return this.#map.has(id)
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
async size(id: Id) {
|
|
14
|
+
return got(this.#map.get(id)).byteLength
|
|
15
|
+
}
|
|
16
|
+
|
|
13
17
|
async delete(id: Id) {
|
|
14
18
|
this.#map.delete(id)
|
|
15
19
|
}
|
package/s/core/index.ts
CHANGED
package/s/core/mammoth.ts
CHANGED
|
@@ -1,80 +1,72 @@
|
|
|
1
1
|
|
|
2
2
|
import {Kv} from "@e280/kv"
|
|
3
|
-
import {
|
|
3
|
+
import {got, hex} from "@e280/stz"
|
|
4
|
+
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
5
|
|
|
5
|
-
import {consts} from "./consts.js"
|
|
6
|
-
import {save} from "./utils/save.js"
|
|
7
|
-
import {Hash, Bucket} from "./types.js"
|
|
8
|
-
import {cleanup} from "./utils/cleanup.js"
|
|
9
6
|
import {randomId} from "./utils/random-id.js"
|
|
10
|
-
import {
|
|
11
|
-
import {Manifest} from "./manifest/manifest.js"
|
|
12
|
-
import {commit} from "./manifest/fns/commit.js"
|
|
13
|
-
import {needInfo} from "./manifest/fns/need-info.js"
|
|
14
|
-
import {getStats} from "./manifest/fns/get-stats.js"
|
|
15
|
-
import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
|
|
16
|
-
import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
|
|
7
|
+
import {Glacier, Hash, Iceberg, Id} from "./types.js"
|
|
17
8
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
#
|
|
21
|
-
#manifest
|
|
22
|
-
#wholesome = lane(consts.max_jobs)
|
|
9
|
+
export class Mammoth implements Glacier {
|
|
10
|
+
#index
|
|
11
|
+
#iceberg
|
|
23
12
|
|
|
24
|
-
constructor(
|
|
25
|
-
this.#
|
|
26
|
-
this.#
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
async* entries() {
|
|
30
|
-
yield* this.#manifest.catalog.entries()
|
|
13
|
+
constructor(index: Kv<string>, iceberg: Iceberg) {
|
|
14
|
+
this.#index = index
|
|
15
|
+
this.#iceberg = iceberg
|
|
31
16
|
}
|
|
32
17
|
|
|
33
18
|
async has(hash: Hash) {
|
|
34
|
-
return this.#
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
async info(hash: Hash) {
|
|
38
|
-
return needInfo(this.#manifest, hash)
|
|
19
|
+
return this.#index.has(hash)
|
|
39
20
|
}
|
|
40
21
|
|
|
41
|
-
async
|
|
42
|
-
|
|
22
|
+
async size(hash: Hash) {
|
|
23
|
+
const id = await this.#needFileId(hash)
|
|
24
|
+
return this.#iceberg.size(id)
|
|
43
25
|
}
|
|
44
26
|
|
|
45
27
|
async read(hash: Hash) {
|
|
46
|
-
const
|
|
47
|
-
return this.#
|
|
28
|
+
const id = await this.#needFileId(hash)
|
|
29
|
+
return this.#iceberg.read(id)
|
|
48
30
|
}
|
|
49
31
|
|
|
50
32
|
async delete(hash: Hash) {
|
|
51
|
-
await this.#
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
}
|
|
56
|
-
await this.#cleanup()
|
|
33
|
+
const id = await this.#getFileId(hash)
|
|
34
|
+
if (id) {
|
|
35
|
+
await this.#index.del(hash)
|
|
36
|
+
await this.#iceberg.delete(id)
|
|
37
|
+
}
|
|
57
38
|
}
|
|
58
39
|
|
|
59
40
|
async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
|
|
60
41
|
const id = randomId()
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
catch (error) {
|
|
70
|
-
await moveWriteToTrash(this.#manifest, id)
|
|
71
|
-
throw error
|
|
72
|
-
}
|
|
73
|
-
finally {
|
|
74
|
-
void this.#cleanup().catch(e => console.error("cleanup failed", e))
|
|
42
|
+
const pipe = new TransformStream()
|
|
43
|
+
const done = this.#iceberg.write(id, pipe.readable)
|
|
44
|
+
const writer = pipe.writable.getWriter()
|
|
45
|
+
const hasher = blake3.create()
|
|
46
|
+
|
|
47
|
+
for await (const chunk of readable) {
|
|
48
|
+
await writer.write(chunk as Uint8Array<ArrayBuffer>)
|
|
49
|
+
hasher.update(chunk)
|
|
75
50
|
}
|
|
51
|
+
|
|
52
|
+
await writer.close()
|
|
53
|
+
await done
|
|
54
|
+
const hash = hex.fromBytes(hasher.digest())
|
|
55
|
+
await this.delete(hash)
|
|
56
|
+
await this.#index.set(hash, id)
|
|
57
|
+
return hash
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async #getFileId(hash: Hash): Promise<Id | undefined> {
|
|
61
|
+
return this.#index.get<string>(hash)
|
|
76
62
|
}
|
|
77
63
|
|
|
78
|
-
#
|
|
64
|
+
async #needFileId(hash: Hash): Promise<Id> {
|
|
65
|
+
return got(await this.#getFileId(hash), `file not found by hash "${hash}"`)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async* keys() {
|
|
69
|
+
yield* this.#index.keys()
|
|
70
|
+
}
|
|
79
71
|
}
|
|
80
72
|
|
package/s/core/types.ts
CHANGED
|
@@ -2,48 +2,20 @@
|
|
|
2
2
|
export type Hash = string
|
|
3
3
|
export type Id = string
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
export type Bucket = {
|
|
5
|
+
export type Iceberg = {
|
|
7
6
|
has(id: Id): Promise<boolean>
|
|
7
|
+
size(id: Id): Promise<number>
|
|
8
8
|
delete(id: Id): Promise<void>
|
|
9
9
|
read(id: Id): Promise<Blob>
|
|
10
10
|
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
hash: Hash
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
size: number
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
/** metadata for a single file. */
|
|
24
|
-
export type Info = {
|
|
25
|
-
|
|
26
|
-
/** bucket id for this file's data. */
|
|
27
|
-
id: Id
|
|
28
|
-
|
|
29
|
-
/** filesize in bytes. */
|
|
30
|
-
size: number
|
|
31
|
-
|
|
32
|
-
/** when this file was added to the datalake. */
|
|
33
|
-
added: number
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/** statistics for the whole datalake. */
|
|
37
|
-
export type Stats = {
|
|
38
|
-
|
|
39
|
-
/** total number of bytes in the whole datalake. */
|
|
40
|
-
size: number
|
|
41
|
-
|
|
42
|
-
/** total number of files in the datalake. */
|
|
43
|
-
count: number
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
export type WriteRecord = {
|
|
47
|
-
created: number
|
|
13
|
+
export type Glacier = {
|
|
14
|
+
has(hash: Hash): Promise<boolean>
|
|
15
|
+
size(hash: Hash): Promise<number>
|
|
16
|
+
read(hash: Hash): Promise<Blob>
|
|
17
|
+
delete(hash: Hash): Promise<void>
|
|
18
|
+
write(readable: ReadableStream<Uint8Array>): Promise<Hash>
|
|
19
|
+
keys(): AsyncIterable<Hash>
|
|
48
20
|
}
|
|
49
21
|
|
package/s/test.ts
CHANGED
|
@@ -1,35 +1,24 @@
|
|
|
1
1
|
|
|
2
2
|
import {Kv} from "@e280/kv"
|
|
3
|
-
import {rm} from "fs/promises"
|
|
4
3
|
import {bytes, collect} from "@e280/stz"
|
|
5
4
|
import {science, test, expect} from "@e280/science"
|
|
6
5
|
|
|
7
6
|
import {Mammoth} from "./core/mammoth.js"
|
|
8
|
-
import {DiskBucket} from "./node/disk-bucket.js"
|
|
9
7
|
import {randomId} from "./core/utils/random-id.js"
|
|
10
|
-
import {
|
|
8
|
+
import {IcebergMemory} from "./core/iceberg-memory.js"
|
|
11
9
|
|
|
12
10
|
const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
|
|
13
11
|
const quickstream = (b: number[]) => new Blob([new Uint8Array(b)]).stream()
|
|
14
12
|
|
|
15
13
|
function setup() {
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
const mammoth = new Mammoth(
|
|
14
|
+
const index = new Kv<string>()
|
|
15
|
+
const iceberg = new IcebergMemory()
|
|
16
|
+
const mammoth = new Mammoth(index, iceberg)
|
|
19
17
|
return {mammoth}
|
|
20
18
|
}
|
|
21
19
|
|
|
22
20
|
await science.run({
|
|
23
|
-
"mammoth
|
|
24
|
-
const dataDir = "data"
|
|
25
|
-
await rm(dataDir, {recursive: true, force: true})
|
|
26
|
-
const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv())
|
|
27
|
-
const hash = await mammoth.write(blob().stream())
|
|
28
|
-
const second = await mammoth.read(hash)
|
|
29
|
-
expect(bytes.eq(await second.bytes(), await blob().bytes()))
|
|
30
|
-
}),
|
|
31
|
-
|
|
32
|
-
"mammoth memory": {
|
|
21
|
+
"mammoth": {
|
|
33
22
|
".write and .read": test(async() => {
|
|
34
23
|
const {mammoth} = setup()
|
|
35
24
|
const hash = await mammoth.write(blob().stream())
|
|
@@ -41,16 +30,16 @@ await science.run({
|
|
|
41
30
|
const {mammoth} = setup()
|
|
42
31
|
const hashA = await mammoth.write(blob().stream())
|
|
43
32
|
const hashB = await mammoth.write(blob().stream())
|
|
44
|
-
const
|
|
33
|
+
const keys = await collect(mammoth.keys())
|
|
45
34
|
expect(hashA).is(hashB)
|
|
46
|
-
expect(
|
|
47
|
-
expect(
|
|
35
|
+
expect(keys.length).is(1)
|
|
36
|
+
expect(keys[0]).is(hashA)
|
|
48
37
|
}),
|
|
49
38
|
|
|
50
39
|
".write empty file": test(async() => {
|
|
51
40
|
const {mammoth} = setup()
|
|
52
41
|
const hash = await mammoth.write(new Blob().stream())
|
|
53
|
-
expect(
|
|
42
|
+
expect(await mammoth.size(hash)).is(0)
|
|
54
43
|
expect((await mammoth.read(hash)).size).is(0)
|
|
55
44
|
}),
|
|
56
45
|
|
|
@@ -59,20 +48,7 @@ await science.run({
|
|
|
59
48
|
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
|
|
60
49
|
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
|
|
61
50
|
expect(hashA).not.is(hashB)
|
|
62
|
-
expect((await collect(mammoth.
|
|
63
|
-
}),
|
|
64
|
-
|
|
65
|
-
".stats": test(async() => {
|
|
66
|
-
const {mammoth} = setup()
|
|
67
|
-
expect(await mammoth.stats()).deep({count: 0, size: 0})
|
|
68
|
-
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
|
|
69
|
-
expect(await mammoth.stats()).deep({count: 1, size: 3})
|
|
70
|
-
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
|
|
71
|
-
expect(await mammoth.stats()).deep({count: 2, size: 7})
|
|
72
|
-
await mammoth.delete(hashA)
|
|
73
|
-
expect(await mammoth.stats()).deep({count: 1, size: 4})
|
|
74
|
-
await mammoth.delete(hashB)
|
|
75
|
-
expect(await mammoth.stats()).deep({count: 0, size: 0})
|
|
51
|
+
expect((await collect(mammoth.keys())).length).is(2)
|
|
76
52
|
}),
|
|
77
53
|
|
|
78
54
|
".has": test(async() => {
|
|
@@ -84,9 +60,8 @@ await science.run({
|
|
|
84
60
|
|
|
85
61
|
".size": test(async() => {
|
|
86
62
|
const {mammoth} = setup()
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
expect((await mammoth.info(hash)).size).is(b.size)
|
|
63
|
+
const hash = await mammoth.write(blob().stream())
|
|
64
|
+
expect(await mammoth.size(hash)).is(4)
|
|
90
65
|
}),
|
|
91
66
|
|
|
92
67
|
".delete": test(async() => {
|
|
@@ -94,7 +69,7 @@ await science.run({
|
|
|
94
69
|
const hash = await mammoth.write(blob().stream())
|
|
95
70
|
await mammoth.delete(hash)
|
|
96
71
|
expect(await mammoth.has(hash)).is(false)
|
|
97
|
-
expect(
|
|
72
|
+
expect(await collect(mammoth.keys())).deep([])
|
|
98
73
|
}),
|
|
99
74
|
|
|
100
75
|
".delete idempotent": test(async() => {
|
|
@@ -105,9 +80,9 @@ await science.run({
|
|
|
105
80
|
".keys": test(async() => {
|
|
106
81
|
const {mammoth} = setup()
|
|
107
82
|
const hash = await mammoth.write(blob().stream())
|
|
108
|
-
const
|
|
109
|
-
expect(
|
|
110
|
-
expect(
|
|
83
|
+
const keys = await collect(mammoth.keys())
|
|
84
|
+
expect(keys.length).is(1)
|
|
85
|
+
expect(keys[0]).is(hash)
|
|
111
86
|
}),
|
|
112
87
|
},
|
|
113
88
|
})
|
package/x/browser/example.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import { Kv, StorageDriver } from "@e280/kv";
|
|
2
2
|
import { Mammoth } from "../core/mammoth.js";
|
|
3
|
-
import {
|
|
3
|
+
import { IcebergOpfs } from "./iceberg-opfs.js";
|
|
4
4
|
export async function example() {
|
|
5
|
-
// setup the localstorage kv
|
|
5
|
+
// setup the localstorage kv index
|
|
6
6
|
const driver = new StorageDriver(window.localStorage);
|
|
7
|
-
const kv = new Kv(driver)
|
|
8
|
-
|
|
7
|
+
const kv = new Kv(driver);
|
|
8
|
+
const index = kv.scope("glacier_index");
|
|
9
|
+
// setup the opfs iceberg
|
|
9
10
|
const root = await navigator.storage.getDirectory();
|
|
10
|
-
const directory = await root.getDirectoryHandle("
|
|
11
|
-
const
|
|
11
|
+
const directory = await root.getDirectoryHandle("glacier", { create: true });
|
|
12
|
+
const iceberg = new IcebergOpfs(directory);
|
|
12
13
|
// setup the mammoth
|
|
13
|
-
const mammoth = new Mammoth(
|
|
14
|
-
const
|
|
14
|
+
const mammoth = new Mammoth(index, iceberg);
|
|
15
|
+
const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
|
|
15
16
|
// write a file as an example
|
|
16
|
-
const hash = await mammoth.write(
|
|
17
|
-
console.log(`mammoth stored file "${hash}"`);
|
|
17
|
+
const hash = await mammoth.write(bigFile.stream());
|
|
18
|
+
console.log(`mammoth stored file by hash "${hash}"`);
|
|
18
19
|
}
|
|
19
20
|
//# sourceMappingURL=example.js.map
|
package/x/browser/example.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;IAEvC,yBAAyB;IACzB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAA;IAE1C,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEpE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAClD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare class
|
|
1
|
+
import { Iceberg, Id } from "../core/types.js";
|
|
2
|
+
export declare class IcebergOpfs implements Iceberg {
|
|
3
3
|
#private;
|
|
4
4
|
constructor(directory: FileSystemDirectoryHandle);
|
|
5
5
|
has(id: Id): Promise<boolean>;
|
|
6
|
+
size(id: Id): Promise<number>;
|
|
6
7
|
delete(id: Id): Promise<void>;
|
|
7
8
|
read(id: Id): Promise<File>;
|
|
8
9
|
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isNotFound } from "./utils/is-not-found.js";
|
|
2
|
-
export class
|
|
2
|
+
export class IcebergOpfs {
|
|
3
3
|
#directory;
|
|
4
4
|
constructor(directory) {
|
|
5
5
|
this.#directory = directory;
|
|
@@ -15,6 +15,11 @@ export class OpfsBucket {
|
|
|
15
15
|
throw error;
|
|
16
16
|
}
|
|
17
17
|
}
|
|
18
|
+
async size(id) {
|
|
19
|
+
const handle = await this.#directory.getFileHandle(id);
|
|
20
|
+
const file = await handle.getFile();
|
|
21
|
+
return file.size;
|
|
22
|
+
}
|
|
18
23
|
async delete(id) {
|
|
19
24
|
try {
|
|
20
25
|
await this.#directory.removeEntry(id);
|
|
@@ -34,4 +39,4 @@ export class OpfsBucket {
|
|
|
34
39
|
await readable.pipeTo(writable);
|
|
35
40
|
}
|
|
36
41
|
}
|
|
37
|
-
//# sourceMappingURL=opfs
|
|
42
|
+
//# sourceMappingURL=iceberg-opfs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"iceberg-opfs.js","sourceRoot":"","sources":["../../s/browser/iceberg-opfs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,WAAW;IACvB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAA;QACnC,OAAO,IAAI,CAAC,IAAI,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}
|