@e280/mammoth 0.1.0-14 → 0.1.0-3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +5 -114
  2. package/package.json +5 -8
  3. package/s/browser/example.ts +11 -10
  4. package/s/browser/{opfs-bucket.ts → iceberg-opfs.ts} +8 -2
  5. package/s/browser/index.ts +1 -1
  6. package/s/core/hash-stream.ts +19 -0
  7. package/s/core/{memory-bucket.ts → iceberg-memory.ts} +6 -2
  8. package/s/core/index.ts +2 -3
  9. package/s/core/mammoth.ts +46 -54
  10. package/s/core/types.ts +9 -37
  11. package/s/test.ts +16 -41
  12. package/x/browser/example.js +11 -10
  13. package/x/browser/example.js.map +1 -1
  14. package/x/browser/{opfs-bucket.d.ts → iceberg-opfs.d.ts} +3 -2
  15. package/x/browser/{opfs-bucket.js → iceberg-opfs.js} +7 -2
  16. package/x/browser/iceberg-opfs.js.map +1 -0
  17. package/x/browser/index.d.ts +1 -1
  18. package/x/browser/index.js +1 -1
  19. package/x/browser/index.js.map +1 -1
  20. package/x/core/hash-stream.d.ts +1 -0
  21. package/x/core/{analyze.js → hash-stream.js} +4 -7
  22. package/x/core/hash-stream.js.map +1 -0
  23. package/x/core/{memory-bucket.d.ts → iceberg-memory.d.ts} +3 -2
  24. package/x/core/{memory-bucket.js → iceberg-memory.js} +5 -2
  25. package/x/core/iceberg-memory.js.map +1 -0
  26. package/x/core/index.d.ts +2 -3
  27. package/x/core/index.js +2 -3
  28. package/x/core/index.js.map +1 -1
  29. package/x/core/mammoth.d.ts +5 -7
  30. package/x/core/mammoth.js +40 -51
  31. package/x/core/mammoth.js.map +1 -1
  32. package/x/core/types.d.ts +9 -27
  33. package/x/test.js +16 -39
  34. package/x/test.js.map +1 -1
  35. package/s/core/analyze.ts +0 -23
  36. package/s/core/consts.ts +0 -8
  37. package/s/core/manifest/fns/commit.ts +0 -27
  38. package/s/core/manifest/fns/get-expired-writes.ts +0 -11
  39. package/s/core/manifest/fns/get-stats.ts +0 -10
  40. package/s/core/manifest/fns/move-write-to-trash.ts +0 -11
  41. package/s/core/manifest/fns/need-info.ts +0 -10
  42. package/s/core/manifest/fns/schedule-deletion.ts +0 -19
  43. package/s/core/manifest/manifest.ts +0 -26
  44. package/s/core/streamify.ts +0 -18
  45. package/s/core/utils/cleanup.ts +0 -16
  46. package/s/core/utils/is-expired.ts +0 -10
  47. package/s/core/utils/not-found.ts +0 -7
  48. package/s/core/utils/save.ts +0 -28
  49. package/s/node/disk-bucket.ts +0 -55
  50. package/s/node/index.ts +0 -4
  51. package/s/node/utils/is-not-found.ts +0 -9
  52. package/x/browser/opfs-bucket.js.map +0 -1
  53. package/x/core/analyze.d.ts +0 -2
  54. package/x/core/analyze.js.map +0 -1
  55. package/x/core/consts.d.ts +0 -4
  56. package/x/core/consts.js +0 -6
  57. package/x/core/consts.js.map +0 -1
  58. package/x/core/manifest/fns/commit.d.ts +0 -3
  59. package/x/core/manifest/fns/commit.js +0 -22
  60. package/x/core/manifest/fns/commit.js.map +0 -1
  61. package/x/core/manifest/fns/get-expired-writes.d.ts +0 -2
  62. package/x/core/manifest/fns/get-expired-writes.js +0 -8
  63. package/x/core/manifest/fns/get-expired-writes.js.map +0 -1
  64. package/x/core/manifest/fns/get-stats.d.ts +0 -2
  65. package/x/core/manifest/fns/get-stats.js +0 -5
  66. package/x/core/manifest/fns/get-stats.js.map +0 -1
  67. package/x/core/manifest/fns/move-write-to-trash.d.ts +0 -3
  68. package/x/core/manifest/fns/move-write-to-trash.js +0 -7
  69. package/x/core/manifest/fns/move-write-to-trash.js.map +0 -1
  70. package/x/core/manifest/fns/need-info.d.ts +0 -3
  71. package/x/core/manifest/fns/need-info.js +0 -6
  72. package/x/core/manifest/fns/need-info.js.map +0 -1
  73. package/x/core/manifest/fns/schedule-deletion.d.ts +0 -3
  74. package/x/core/manifest/fns/schedule-deletion.js +0 -13
  75. package/x/core/manifest/fns/schedule-deletion.js.map +0 -1
  76. package/x/core/manifest/manifest.d.ts +0 -14
  77. package/x/core/manifest/manifest.js +0 -19
  78. package/x/core/manifest/manifest.js.map +0 -1
  79. package/x/core/memory-bucket.js.map +0 -1
  80. package/x/core/streamify.d.ts +0 -1
  81. package/x/core/streamify.js +0 -15
  82. package/x/core/streamify.js.map +0 -1
  83. package/x/core/utils/cleanup.d.ts +0 -3
  84. package/x/core/utils/cleanup.js +0 -11
  85. package/x/core/utils/cleanup.js.map +0 -1
  86. package/x/core/utils/is-expired.d.ts +0 -2
  87. package/x/core/utils/is-expired.js +0 -7
  88. package/x/core/utils/is-expired.js.map +0 -1
  89. package/x/core/utils/not-found.d.ts +0 -2
  90. package/x/core/utils/not-found.js +0 -4
  91. package/x/core/utils/not-found.js.map +0 -1
  92. package/x/core/utils/save.d.ts +0 -2
  93. package/x/core/utils/save.js +0 -16
  94. package/x/core/utils/save.js.map +0 -1
  95. package/x/node/disk-bucket.d.ts +0 -9
  96. package/x/node/disk-bucket.js +0 -43
  97. package/x/node/disk-bucket.js.map +0 -1
  98. package/x/node/index.d.ts +0 -2
  99. package/x/node/index.js +0 -3
  100. package/x/node/index.js.map +0 -1
  101. package/x/node/utils/is-not-found.d.ts +0 -1
  102. package/x/node/utils/is-not-found.js +0 -6
  103. package/x/node/utils/is-not-found.js.map +0 -1
package/README.md CHANGED
@@ -1,122 +1,13 @@
1
1
 
2
- ![](https://i.imgur.com/1TQmFwW.png)
2
+ # 🦣 @e280/mammoth
3
3
 
4
- # 🦣 mammoth
5
- > *big files. small api.*
6
-
7
- **mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs. mammoth uses [kv](https://github.com/e280/kv) for bookkeeping.
8
-
9
- ```bash
10
- npm install @e280/mammoth @e280/kv
11
- ```
12
-
13
- ```ts
14
- import {Mammoth} from "@e280/mammoth"
15
-
16
- const mammoth = new Mammoth()
17
- ```
18
-
19
- ## 🦣 mammoth stores files.
20
- - **write a file,** and you get back its blake3 `hash` and `size` in bytes.
21
- ```ts
22
- const hash = await mammoth.write(blob.stream())
23
- ```
24
- - **read a file,** identified by its hash.
25
- ```ts
26
- const blob = await mammoth.read(hash)
27
- ```
28
- - **delete a file.**
29
- ```ts
30
- await mammoth.delete(hash)
31
- ```
32
-
33
- ## 🦣 mammoth is bucket-agnostic.
34
- - **memory bucket,** for testing.
35
- ```ts
36
- import {Mammoth, MemoryBucket} from "@e280/mammoth"
37
- import {Kv} from "@e280/kv"
38
-
39
- const mammoth = new Mammoth(
40
-
41
- // bucket for blob storage
42
- new MemoryBucket(),
43
-
44
- // kv for metadata bookkeeping
45
- new Kv(),
46
- )
47
- ```
48
- defaults shown, this is equivalent:
49
- ```ts
50
- const mammoth = new Mammoth()
51
- ```
52
- - **disk bucket,** for nodejs servers. *(note the import paths)*
53
- ```ts
54
- import {Mammoth} from "@e280/mammoth"
55
- import {DiskBucket} from "@e280/mammoth/node"
56
- import {Kv} from "@e280/kv"
57
- import {LevelDriver} from "@e280/kv/level"
58
-
59
- const mammoth = new Mammoth(
60
- new DiskBucket("./data/bucket"),
61
- new Kv(new LevelDriver("./data/kv")),
62
- )
63
- ```
64
- - **opfs bucket,** for local storage in the browser. *(note the import paths)*
65
- ```ts
66
- import {Mammoth} from "@e280/mammoth"
67
- import {OpfsBucket} from "@e280/mammoth/browser"
68
- import {Kv, StorageDriver} from "@e280/kv"
69
-
70
- const mammoth = new Mammoth(
71
- new OpfsBucket(await navigator.storage.getDirectory()),
72
- new Kv(new StorageDriver(localStorage)),
73
- )
74
- ```
75
-
76
- ## 🦣 more mammoth methods.
77
- - **check if a file exists,** get back a boolean.
78
- ```ts
79
- const exists = await mammoth.has(hash)
80
- ```
81
- - **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
82
- ```ts
83
- const {size, added, id} = await mammoth.info(hash)
84
- ```
85
- - **get stats,** for the whole datalake.
86
- ```ts
87
- await mammoth.stats()
88
- // {count: 123, size: 123456789}
89
- ```
90
- - **loop over every file,** with their hashes and infos.
91
- ```ts
92
- for await (const [hash, info] of mammoth.entries())
93
- console.log(hash, info)
94
- ```
95
-
96
- ## 🦣 mammoth-brained tips.
97
- - **`analyze` is for hashing files.**
98
- ```ts
99
- import {analyze} from "@e280/mammoth"
100
-
101
- const {hash, size} = await analyze(blob.stream())
102
- ```
103
- - **`analyze` can help you avoid unnecessary uploads.**
104
- ```ts
105
- const {hash} = await analyze(blob.stream())
106
-
107
- if (!await mammoth.has(hash))
108
- await mammoth.write(blob.stream())
109
- ```
110
- - **`streamify` makes a stream for a Uint8Array.**
111
- ```ts
112
- import {streamify} from "@e280/mammoth"
113
-
114
- const readable = await streamify(new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF]))
115
- ```
4
+ big files. small api.
116
5
 
117
6
 
118
7
 
119
8
  <br/><br/>
120
9
 
121
- *https://e280.org/*
10
+ ## 💖 mammoth is by e280
11
+ reward us with github stars
12
+ build with us at https://e280.org/ but only if you're cool
122
13
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@e280/mammoth",
3
- "version": "0.1.0-14",
3
+ "version": "0.1.0-3",
4
4
  "description": "big file storage",
5
5
  "license": "MIT",
6
6
  "author": "Chase Moskal <chasemoskal@gmail.com>",
@@ -9,8 +9,7 @@
9
9
  "sideEffects": false,
10
10
  "exports": {
11
11
  ".": "./x/core/index.js",
12
- "./browser": "./x/browser/index.js",
13
- "./node": "./x/node/index.js"
12
+ "./browser": "./x/browser/index.js"
14
13
  },
15
14
  "files": [
16
15
  "x",
@@ -22,12 +21,10 @@
22
21
  "test": "node x/test.js",
23
22
  "count": "find s -path '*/_archive' -prune -o -name '*.ts' -exec wc -l {} +"
24
23
  },
25
- "peerDependencies": {
26
- "@e280/kv": "^0.1.5"
27
- },
28
24
  "dependencies": {
29
- "@awasm/noble": "^0.1.4",
30
- "@e280/stz": "^0.3.5"
25
+ "@e280/kv": "^0.1.5",
26
+ "@e280/stz": "^0.3.4",
27
+ "@noble/hashes": "^2.2.0"
31
28
  },
32
29
  "devDependencies": {
33
30
  "@e280/octo": "^0.1.3",
@@ -1,25 +1,26 @@
1
1
 
2
2
  import {Kv, StorageDriver} from "@e280/kv"
3
3
  import {Mammoth} from "../core/mammoth.js"
4
- import {OpfsBucket} from "./opfs-bucket.js"
4
+ import {IcebergOpfs} from "./iceberg-opfs.js"
5
5
 
6
6
  export async function example() {
7
7
 
8
- // setup the localstorage kv
8
+ // setup the localstorage kv index
9
9
  const driver = new StorageDriver(window.localStorage)
10
- const kv = new Kv(driver).scope("mammoth")
10
+ const kv = new Kv(driver)
11
+ const index = kv.scope("glacier_index")
11
12
 
12
- // setup the opfs file bucket
13
+ // setup the opfs iceberg
13
14
  const root = await navigator.storage.getDirectory()
14
- const directory = await root.getDirectoryHandle("mammoth", {create: true})
15
- const bucket = new OpfsBucket(directory)
15
+ const directory = await root.getDirectoryHandle("glacier", {create: true})
16
+ const iceberg = new IcebergOpfs(directory)
16
17
 
17
18
  // setup the mammoth
18
- const mammoth = new Mammoth(bucket, kv)
19
- const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
19
+ const mammoth = new Mammoth(index, iceberg)
20
+ const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
20
21
 
21
22
  // write a file as an example
22
- const hash = await mammoth.write(myFile.stream())
23
- console.log(`mammoth stored file "${hash}"`)
23
+ const hash = await mammoth.write(bigFile.stream())
24
+ console.log(`mammoth stored file by hash "${hash}"`)
24
25
  }
25
26
 
@@ -1,8 +1,8 @@
1
1
 
2
- import {Bucket, Id} from "../core/types.js"
2
+ import {Iceberg, Id} from "../core/types.js"
3
3
  import {isNotFound} from "./utils/is-not-found.js"
4
4
 
5
- export class OpfsBucket implements Bucket {
5
+ export class IcebergOpfs implements Iceberg {
6
6
  #directory
7
7
 
8
8
  constructor(directory: FileSystemDirectoryHandle) {
@@ -20,6 +20,12 @@ export class OpfsBucket implements Bucket {
20
20
  }
21
21
  }
22
22
 
23
+ async size(id: Id) {
24
+ const handle = await this.#directory.getFileHandle(id)
25
+ const file = await handle.getFile()
26
+ return file.size
27
+ }
28
+
23
29
  async delete(id: Id) {
24
30
  try {
25
31
  await this.#directory.removeEntry(id)
@@ -1,4 +1,4 @@
1
1
 
2
2
  export * from "../core/index.js"
3
- export * from "./opfs-bucket.js"
3
+ export * from "./iceberg-opfs.js"
4
4
 
@@ -0,0 +1,19 @@
1
+
2
+ import {hex, nap} from "@e280/stz"
3
+ import {blake3} from "@noble/hashes/blake3.js"
4
+ import {relaxer} from "./utils/relaxer.js"
5
+
6
+ export async function hashStream(readable: ReadableStream<Uint8Array>) {
7
+ const hasher = blake3.create()
8
+ const relax = relaxer()
9
+
10
+ for await (const chunk of readable) {
11
+ hasher.update(chunk)
12
+
13
+ if (relax())
14
+ await nap()
15
+ }
16
+
17
+ return hex.fromBytes(hasher.digest())
18
+ }
19
+
@@ -1,15 +1,19 @@
1
1
 
2
2
  import {bytes, got, nap} from "@e280/stz"
3
- import {Bucket, Id} from "./types.js"
3
+ import {Iceberg, Id} from "./types.js"
4
4
  import {relaxer} from "./utils/relaxer.js"
5
5
 
6
- export class MemoryBucket implements Bucket {
6
+ export class IcebergMemory implements Iceberg {
7
7
  #map = new Map<string, Uint8Array>()
8
8
 
9
9
  async has(id: Id) {
10
10
  return this.#map.has(id)
11
11
  }
12
12
 
13
+ async size(id: Id) {
14
+ return got(this.#map.get(id)).byteLength
15
+ }
16
+
13
17
  async delete(id: Id) {
14
18
  this.#map.delete(id)
15
19
  }
package/s/core/index.ts CHANGED
@@ -1,7 +1,6 @@
1
1
 
2
- export * from "./analyze.js"
2
+ export * from "./hash-stream.js"
3
+ export * from "./iceberg-memory.js"
3
4
  export * from "./mammoth.js"
4
- export * from "./memory-bucket.js"
5
- export * from "./streamify.js"
6
5
  export * from "./types.js"
7
6
 
package/s/core/mammoth.ts CHANGED
@@ -1,80 +1,72 @@
1
1
 
2
2
  import {Kv} from "@e280/kv"
3
- import {lane, queue} from "@e280/stz"
3
+ import {got, hex} from "@e280/stz"
4
+ import {blake3} from "@noble/hashes/blake3.js"
4
5
 
5
- import {consts} from "./consts.js"
6
- import {save} from "./utils/save.js"
7
- import {Hash, Bucket} from "./types.js"
8
- import {cleanup} from "./utils/cleanup.js"
9
6
  import {randomId} from "./utils/random-id.js"
10
- import {MemoryBucket} from "./memory-bucket.js"
11
- import {Manifest} from "./manifest/manifest.js"
12
- import {commit} from "./manifest/fns/commit.js"
13
- import {needInfo} from "./manifest/fns/need-info.js"
14
- import {getStats} from "./manifest/fns/get-stats.js"
15
- import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
16
- import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
7
+ import {Glacier, Hash, Iceberg, Id} from "./types.js"
17
8
 
18
- /** file storage datalake, content-addressed with blake3 hashes. */
19
- export class Mammoth {
20
- #bucket
21
- #manifest
22
- #wholesome = lane(consts.max_jobs)
9
+ export class Mammoth implements Glacier {
10
+ #index
11
+ #iceberg
23
12
 
24
- constructor(bucket: Bucket = new MemoryBucket(), kv = new Kv()) {
25
- this.#bucket = bucket
26
- this.#manifest = new Manifest(kv)
27
- }
28
-
29
- async* entries() {
30
- yield* this.#manifest.catalog.entries()
13
+ constructor(index: Kv<string>, iceberg: Iceberg) {
14
+ this.#index = index
15
+ this.#iceberg = iceberg
31
16
  }
32
17
 
33
18
  async has(hash: Hash) {
34
- return this.#manifest.catalog.has(hash)
35
- }
36
-
37
- async info(hash: Hash) {
38
- return needInfo(this.#manifest, hash)
19
+ return this.#index.has(hash)
39
20
  }
40
21
 
41
- async stats() {
42
- return getStats(this.#manifest)
22
+ async size(hash: Hash) {
23
+ const id = await this.#needFileId(hash)
24
+ return this.#iceberg.size(id)
43
25
  }
44
26
 
45
27
  async read(hash: Hash) {
46
- const {id} = await this.info(hash)
47
- return this.#bucket.read(id)
28
+ const id = await this.#needFileId(hash)
29
+ return this.#iceberg.read(id)
48
30
  }
49
31
 
50
32
  async delete(hash: Hash) {
51
- await this.#wholesome(async() => {
52
- const info = await this.#manifest.catalog.get(hash)
53
- if (info)
54
- await scheduleDeletion(this.#manifest, hash, info)
55
- })
56
- await this.#cleanup()
33
+ const id = await this.#getFileId(hash)
34
+ if (id) {
35
+ await this.#index.del(hash)
36
+ await this.#iceberg.delete(id)
37
+ }
57
38
  }
58
39
 
59
40
  async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
60
41
  const id = randomId()
61
- await this.#manifest.writes.set(id, {created: Date.now()})
62
- try {
63
- const analysis = await save(this.#bucket, id, readable)
64
- const {hash, size} = analysis
65
- const info = {id, size, added: Date.now()}
66
- await this.#wholesome(() => commit(this.#manifest, hash, info))
67
- return analysis.hash
68
- }
69
- catch (error) {
70
- await moveWriteToTrash(this.#manifest, id)
71
- throw error
72
- }
73
- finally {
74
- void this.#cleanup().catch(e => console.error("cleanup failed", e))
42
+ const pipe = new TransformStream()
43
+ const done = this.#iceberg.write(id, pipe.readable)
44
+ const writer = pipe.writable.getWriter()
45
+ const hasher = blake3.create()
46
+
47
+ for await (const chunk of readable) {
48
+ await writer.write(chunk as Uint8Array<ArrayBuffer>)
49
+ hasher.update(chunk)
75
50
  }
51
+
52
+ await writer.close()
53
+ await done
54
+ const hash = hex.fromBytes(hasher.digest())
55
+ await this.delete(hash)
56
+ await this.#index.set(hash, id)
57
+ return hash
58
+ }
59
+
60
+ async #getFileId(hash: Hash): Promise<Id | undefined> {
61
+ return this.#index.get<string>(hash)
76
62
  }
77
63
 
78
- #cleanup = queue(() => cleanup(this.#bucket, this.#manifest), consts.max_jobs)
64
+ async #needFileId(hash: Hash): Promise<Id> {
65
+ return got(await this.#getFileId(hash), `file not found by hash "${hash}"`)
66
+ }
67
+
68
+ async* keys() {
69
+ yield* this.#index.keys()
70
+ }
79
71
  }
80
72
 
package/s/core/types.ts CHANGED
@@ -2,48 +2,20 @@
2
2
  export type Hash = string
3
3
  export type Id = string
4
4
 
5
- /** file blob store. */
6
- export type Bucket = {
5
+ export type Iceberg = {
7
6
  has(id: Id): Promise<boolean>
7
+ size(id: Id): Promise<number>
8
8
  delete(id: Id): Promise<void>
9
9
  read(id: Id): Promise<Blob>
10
10
  write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
11
11
  }
12
12
 
13
- /** report about a file */
14
- export type Analysis = {
15
-
16
- /** blake3 hash of the file's contents */
17
- hash: Hash
18
-
19
- /** filesize in bytes */
20
- size: number
21
- }
22
-
23
- /** metadata for a single file. */
24
- export type Info = {
25
-
26
- /** bucket id for this file's data. */
27
- id: Id
28
-
29
- /** filesize in bytes. */
30
- size: number
31
-
32
- /** when this file was added to the datalake. */
33
- added: number
34
- }
35
-
36
- /** statistics for the whole datalake. */
37
- export type Stats = {
38
-
39
- /** total number of bytes in the whole datalake. */
40
- size: number
41
-
42
- /** total number of files in the datalake. */
43
- count: number
44
- }
45
-
46
- export type WriteRecord = {
47
- created: number
13
+ export type Glacier = {
14
+ has(hash: Hash): Promise<boolean>
15
+ size(hash: Hash): Promise<number>
16
+ read(hash: Hash): Promise<Blob>
17
+ delete(hash: Hash): Promise<void>
18
+ write(readable: ReadableStream<Uint8Array>): Promise<Hash>
19
+ keys(): AsyncIterable<Hash>
48
20
  }
49
21
 
package/s/test.ts CHANGED
@@ -1,35 +1,24 @@
1
1
 
2
2
  import {Kv} from "@e280/kv"
3
- import {rm} from "fs/promises"
4
3
  import {bytes, collect} from "@e280/stz"
5
4
  import {science, test, expect} from "@e280/science"
6
5
 
7
6
  import {Mammoth} from "./core/mammoth.js"
8
- import {DiskBucket} from "./node/disk-bucket.js"
9
7
  import {randomId} from "./core/utils/random-id.js"
10
- import {MemoryBucket} from "./core/memory-bucket.js"
8
+ import {IcebergMemory} from "./core/iceberg-memory.js"
11
9
 
12
10
  const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
13
11
  const quickstream = (b: number[]) => new Blob([new Uint8Array(b)]).stream()
14
12
 
15
13
  function setup() {
16
- const kv = new Kv()
17
- const bucket = new MemoryBucket()
18
- const mammoth = new Mammoth(bucket, kv)
14
+ const index = new Kv<string>()
15
+ const iceberg = new IcebergMemory()
16
+ const mammoth = new Mammoth(index, iceberg)
19
17
  return {mammoth}
20
18
  }
21
19
 
22
20
  await science.run({
23
- "mammoth node": test(async() => {
24
- const dataDir = "data"
25
- await rm(dataDir, {recursive: true, force: true})
26
- const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv())
27
- const hash = await mammoth.write(blob().stream())
28
- const second = await mammoth.read(hash)
29
- expect(bytes.eq(await second.bytes(), await blob().bytes()))
30
- }),
31
-
32
- "mammoth memory": {
21
+ "mammoth": {
33
22
  ".write and .read": test(async() => {
34
23
  const {mammoth} = setup()
35
24
  const hash = await mammoth.write(blob().stream())
@@ -41,16 +30,16 @@ await science.run({
41
30
  const {mammoth} = setup()
42
31
  const hashA = await mammoth.write(blob().stream())
43
32
  const hashB = await mammoth.write(blob().stream())
44
- const entries = await collect(mammoth.entries())
33
+ const keys = await collect(mammoth.keys())
45
34
  expect(hashA).is(hashB)
46
- expect(entries.length).is(1)
47
- expect(entries[0][0]).is(hashA)
35
+ expect(keys.length).is(1)
36
+ expect(keys[0]).is(hashA)
48
37
  }),
49
38
 
50
39
  ".write empty file": test(async() => {
51
40
  const {mammoth} = setup()
52
41
  const hash = await mammoth.write(new Blob().stream())
53
- expect((await mammoth.info(hash)).size).is(0)
42
+ expect(await mammoth.size(hash)).is(0)
54
43
  expect((await mammoth.read(hash)).size).is(0)
55
44
  }),
56
45
 
@@ -59,20 +48,7 @@ await science.run({
59
48
  const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
60
49
  const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
61
50
  expect(hashA).not.is(hashB)
62
- expect((await collect(mammoth.entries())).length).is(2)
63
- }),
64
-
65
- ".stats": test(async() => {
66
- const {mammoth} = setup()
67
- expect(await mammoth.stats()).deep({count: 0, size: 0})
68
- const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
69
- expect(await mammoth.stats()).deep({count: 1, size: 3})
70
- const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
71
- expect(await mammoth.stats()).deep({count: 2, size: 7})
72
- await mammoth.delete(hashA)
73
- expect(await mammoth.stats()).deep({count: 1, size: 4})
74
- await mammoth.delete(hashB)
75
- expect(await mammoth.stats()).deep({count: 0, size: 0})
51
+ expect((await collect(mammoth.keys())).length).is(2)
76
52
  }),
77
53
 
78
54
  ".has": test(async() => {
@@ -84,9 +60,8 @@ await science.run({
84
60
 
85
61
  ".size": test(async() => {
86
62
  const {mammoth} = setup()
87
- const b = blob()
88
- const hash = await mammoth.write(b.stream())
89
- expect((await mammoth.info(hash)).size).is(b.size)
63
+ const hash = await mammoth.write(blob().stream())
64
+ expect(await mammoth.size(hash)).is(4)
90
65
  }),
91
66
 
92
67
  ".delete": test(async() => {
@@ -94,7 +69,7 @@ await science.run({
94
69
  const hash = await mammoth.write(blob().stream())
95
70
  await mammoth.delete(hash)
96
71
  expect(await mammoth.has(hash)).is(false)
97
- expect((await collect(mammoth.entries())).length).is(0)
72
+ expect(await collect(mammoth.keys())).deep([])
98
73
  }),
99
74
 
100
75
  ".delete idempotent": test(async() => {
@@ -105,9 +80,9 @@ await science.run({
105
80
  ".keys": test(async() => {
106
81
  const {mammoth} = setup()
107
82
  const hash = await mammoth.write(blob().stream())
108
- const entries = await collect(mammoth.entries())
109
- expect(entries.length).is(1)
110
- expect(entries[0][0]).is(hash)
83
+ const keys = await collect(mammoth.keys())
84
+ expect(keys.length).is(1)
85
+ expect(keys[0]).is(hash)
111
86
  }),
112
87
  },
113
88
  })
@@ -1,19 +1,20 @@
1
1
  import { Kv, StorageDriver } from "@e280/kv";
2
2
  import { Mammoth } from "../core/mammoth.js";
3
- import { OpfsBucket } from "./opfs-bucket.js";
3
+ import { IcebergOpfs } from "./iceberg-opfs.js";
4
4
  export async function example() {
5
- // setup the localstorage kv
5
+ // setup the localstorage kv index
6
6
  const driver = new StorageDriver(window.localStorage);
7
- const kv = new Kv(driver).scope("mammoth");
8
- // setup the opfs file bucket
7
+ const kv = new Kv(driver);
8
+ const index = kv.scope("glacier_index");
9
+ // setup the opfs iceberg
9
10
  const root = await navigator.storage.getDirectory();
10
- const directory = await root.getDirectoryHandle("mammoth", { create: true });
11
- const bucket = new OpfsBucket(directory);
11
+ const directory = await root.getDirectoryHandle("glacier", { create: true });
12
+ const iceberg = new IcebergOpfs(directory);
12
13
  // setup the mammoth
13
- const mammoth = new Mammoth(bucket, kv);
14
- const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
14
+ const mammoth = new Mammoth(index, iceberg);
15
+ const bigFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
15
16
  // write a file as an example
16
- const hash = await mammoth.write(myFile.stream());
17
- console.log(`mammoth stored file "${hash}"`);
17
+ const hash = await mammoth.write(bigFile.stream());
18
+ console.log(`mammoth stored file by hash "${hash}"`);
18
19
  }
19
20
  //# sourceMappingURL=example.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,CAAA;AAC7C,CAAC"}
1
+ {"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,WAAW,EAAC,MAAM,mBAAmB,CAAA;AAE7C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAA;IACzB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,CAAA;IAEvC,yBAAyB;IACzB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,SAAS,CAAC,CAAA;IAE1C,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3C,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEpE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAClD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
@@ -1,8 +1,9 @@
1
- import { Bucket, Id } from "../core/types.js";
2
- export declare class OpfsBucket implements Bucket {
1
+ import { Iceberg, Id } from "../core/types.js";
2
+ export declare class IcebergOpfs implements Iceberg {
3
3
  #private;
4
4
  constructor(directory: FileSystemDirectoryHandle);
5
5
  has(id: Id): Promise<boolean>;
6
+ size(id: Id): Promise<number>;
6
7
  delete(id: Id): Promise<void>;
7
8
  read(id: Id): Promise<File>;
8
9
  write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
@@ -1,5 +1,5 @@
1
1
  import { isNotFound } from "./utils/is-not-found.js";
2
- export class OpfsBucket {
2
+ export class IcebergOpfs {
3
3
  #directory;
4
4
  constructor(directory) {
5
5
  this.#directory = directory;
@@ -15,6 +15,11 @@ export class OpfsBucket {
15
15
  throw error;
16
16
  }
17
17
  }
18
+ async size(id) {
19
+ const handle = await this.#directory.getFileHandle(id);
20
+ const file = await handle.getFile();
21
+ return file.size;
22
+ }
18
23
  async delete(id) {
19
24
  try {
20
25
  await this.#directory.removeEntry(id);
@@ -34,4 +39,4 @@ export class OpfsBucket {
34
39
  await readable.pipeTo(writable);
35
40
  }
36
41
  }
37
- //# sourceMappingURL=opfs-bucket.js.map
42
+ //# sourceMappingURL=iceberg-opfs.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"iceberg-opfs.js","sourceRoot":"","sources":["../../s/browser/iceberg-opfs.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAA;AAElD,MAAM,OAAO,WAAW;IACvB,UAAU,CAAA;IAEV,YAAY,SAAoC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAAM;QACf,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;YACvC,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAA;QACnC,OAAO,IAAI,CAAC,IAAI,CAAA;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC,CAAA;QACtC,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAA;QACb,CAAC;IACF,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAM;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC,CAAA;QACtD,OAAO,MAAM,CAAC,OAAO,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAM,EAAE,QAAoC;QACvD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;QACtE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,EAAE,CAAA;QAC9C,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAChC,CAAC;CACD"}