@e280/mammoth 0.1.0-4 → 0.1.0-5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/package.json +1 -1
- package/s/browser/example.ts +1 -1
- package/s/core/mammoth.ts +37 -75
- package/s/core/types.ts +10 -0
- package/s/core/utils/is-expired.ts +9 -0
- package/s/core/utils/not-found.ts +7 -0
- package/s/core/utils/save-and-hash.ts +25 -0
- package/s/test.ts +9 -11
- package/x/browser/example.js +1 -1
- package/x/browser/example.js.map +1 -1
- package/x/core/mammoth.d.ts +3 -9
- package/x/core/mammoth.js +32 -55
- package/x/core/mammoth.js.map +1 -1
- package/x/core/types.d.ts +7 -0
- package/x/core/utils/is-expired.d.ts +2 -0
- package/x/core/utils/is-expired.js +6 -0
- package/x/core/utils/is-expired.js.map +1 -0
- package/x/core/utils/not-found.d.ts +2 -0
- package/x/core/utils/not-found.js +4 -0
- package/x/core/utils/not-found.js.map +1 -0
- package/x/core/utils/save-and-hash.d.ts +5 -0
- package/x/core/utils/save-and-hash.js +19 -0
- package/x/core/utils/save-and-hash.js.map +1 -0
- package/x/test.js +9 -11
- package/x/test.js.map +1 -1
package/README.md
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
|
|
2
|
-

|
|
3
3
|
|
|
4
|
-
# 🦣
|
|
5
|
-
|
|
4
|
+
# 🦣 mammoth
|
|
5
|
+
> *big files. small api.*
|
|
6
|
+
|
|
7
|
+
**mammoth is a content-addressed file store.** files are streamed into a bucket, identified by their blake3 hash, and automatically deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs.
|
|
6
8
|
|
|
7
9
|
```bash
|
|
8
10
|
npm install @e280/mammoth
|
|
@@ -35,8 +37,8 @@ const mammoth = new Mammoth()
|
|
|
35
37
|
import {Kv} from "@e280/kv"
|
|
36
38
|
|
|
37
39
|
const mammoth = new Mammoth(
|
|
38
|
-
new Kv(),
|
|
39
40
|
new MemoryBucket(),
|
|
41
|
+
new Kv(),
|
|
40
42
|
)
|
|
41
43
|
```
|
|
42
44
|
defaults shown, so, this is equivalent:
|
|
@@ -51,8 +53,8 @@ const mammoth = new Mammoth()
|
|
|
51
53
|
import {LevelDriver} from "@e280/kv/level"
|
|
52
54
|
|
|
53
55
|
const mammoth = new Mammoth(
|
|
54
|
-
new Kv(new LevelDriver("./data/kv")),
|
|
55
56
|
new DiskBucket("./data/bucket"),
|
|
57
|
+
new Kv(new LevelDriver("./data/kv")),
|
|
56
58
|
)
|
|
57
59
|
```
|
|
58
60
|
- **opfs bucket,** for local storage in the browser. *(note the import paths)*
|
|
@@ -62,12 +64,12 @@ const mammoth = new Mammoth()
|
|
|
62
64
|
import {Kv, StorageDriver} from "@e280/kv"
|
|
63
65
|
|
|
64
66
|
const mammoth = new Mammoth(
|
|
65
|
-
new Kv(new StorageDriver(localStorage)),
|
|
66
67
|
new OpfsBucket(await navigator.storage.getDirectory()),
|
|
68
|
+
new Kv(new StorageDriver(localStorage)),
|
|
67
69
|
)
|
|
68
70
|
```
|
|
69
71
|
|
|
70
|
-
### 🦣 mammoth
|
|
72
|
+
### 🦣 more mammoth methods.
|
|
71
73
|
- **check if a file exists,** get back a boolean.
|
|
72
74
|
```ts
|
|
73
75
|
const exists = await mammoth.has(hash)
|
|
@@ -79,7 +81,7 @@ const mammoth = new Mammoth()
|
|
|
79
81
|
- **get stats,** for the whole datalake.
|
|
80
82
|
```ts
|
|
81
83
|
await mammoth.stats()
|
|
82
|
-
// {size:
|
|
84
|
+
// {count: 123, size: 123456789}
|
|
83
85
|
```
|
|
84
86
|
- **loop over all hashes,** for all stored files.
|
|
85
87
|
```ts
|
|
@@ -91,5 +93,5 @@ const mammoth = new Mammoth()
|
|
|
91
93
|
|
|
92
94
|
<br/><br/>
|
|
93
95
|
|
|
94
|
-
|
|
96
|
+
*https://e280.org/*
|
|
95
97
|
|
package/package.json
CHANGED
package/s/browser/example.ts
CHANGED
|
@@ -15,7 +15,7 @@ export async function example() {
|
|
|
15
15
|
const bucket = new OpfsBucket(directory)
|
|
16
16
|
|
|
17
17
|
// setup the mammoth
|
|
18
|
-
const mammoth = new Mammoth(
|
|
18
|
+
const mammoth = new Mammoth(bucket, kv)
|
|
19
19
|
const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
|
|
20
20
|
|
|
21
21
|
// write a file as an example
|
package/s/core/mammoth.ts
CHANGED
|
@@ -1,31 +1,25 @@
|
|
|
1
1
|
|
|
2
2
|
import {Kv} from "@e280/kv"
|
|
3
|
-
import {
|
|
4
|
-
import {blake3} from "@noble/hashes/blake3.js"
|
|
3
|
+
import {collect, got, queue} from "@e280/stz"
|
|
5
4
|
|
|
6
|
-
import {Hash, Bucket, Id} from "./types.js"
|
|
7
5
|
import {randomId} from "./utils/random-id.js"
|
|
6
|
+
import {notFound} from "./utils/not-found.js"
|
|
8
7
|
import {MemoryBucket} from "./memory-bucket.js"
|
|
8
|
+
import {isExpired} from "./utils/is-expired.js"
|
|
9
|
+
import {saveAndHash} from "./utils/save-and-hash.js"
|
|
10
|
+
import {Hash, Bucket, Id, Stats, Wip} from "./types.js"
|
|
9
11
|
|
|
10
12
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
11
13
|
export class Mammoth {
|
|
12
|
-
#ids
|
|
13
|
-
#wip
|
|
14
|
-
#stats
|
|
15
|
-
#bucket
|
|
14
|
+
#ids // associates file hashes with bucket ids
|
|
15
|
+
#wip // temporary record of writes in-progress
|
|
16
|
+
#stats // statistics about the whole datalake
|
|
17
|
+
#bucket // file blobstore
|
|
16
18
|
|
|
17
|
-
constructor(
|
|
18
|
-
|
|
19
|
-
/** key-value metadata manifest. */
|
|
20
|
-
kv = new Kv(),
|
|
21
|
-
|
|
22
|
-
/** dumb raw file blob storage. */
|
|
23
|
-
bucket: Bucket = new MemoryBucket(),
|
|
24
|
-
|
|
25
|
-
) {
|
|
19
|
+
constructor(bucket: Bucket = new MemoryBucket(), kv = new Kv()) {
|
|
26
20
|
this.#ids = kv.scope<string>("ids")
|
|
27
|
-
this.#wip = kv.scope<
|
|
28
|
-
this.#stats = kv.store<
|
|
21
|
+
this.#wip = kv.scope<Wip>("wip")
|
|
22
|
+
this.#stats = kv.store<Stats>("stats")
|
|
29
23
|
this.#bucket = bucket
|
|
30
24
|
}
|
|
31
25
|
|
|
@@ -34,36 +28,37 @@ export class Mammoth {
|
|
|
34
28
|
}
|
|
35
29
|
|
|
36
30
|
async size(hash: Hash) {
|
|
37
|
-
const id = await this.#
|
|
31
|
+
const id = got(await this.#ids.get(hash), notFound(hash))
|
|
38
32
|
return this.#bucket.size(id)
|
|
39
33
|
}
|
|
40
34
|
|
|
41
35
|
async read(hash: Hash) {
|
|
42
|
-
const id = await this.#
|
|
36
|
+
const id = got(await this.#ids.get(hash), notFound(hash))
|
|
43
37
|
return this.#bucket.read(id)
|
|
44
38
|
}
|
|
45
39
|
|
|
46
40
|
async delete(hash: Hash) {
|
|
47
|
-
const id = await this.#
|
|
41
|
+
const id = await this.#ids.get(hash)
|
|
48
42
|
if (id) {
|
|
49
43
|
const size = await this.#bucket.size(id)
|
|
50
44
|
await this.#ids.del(hash)
|
|
51
45
|
await this.#bucket.delete(id)
|
|
52
|
-
await this.#
|
|
46
|
+
await this.#updateStats(stats => {
|
|
47
|
+
stats.count--
|
|
48
|
+
stats.size -= size
|
|
49
|
+
})
|
|
53
50
|
}
|
|
54
51
|
}
|
|
55
52
|
|
|
56
53
|
async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
|
|
57
54
|
const id = randomId()
|
|
58
|
-
await this.#wip.set(id, {created: Date.now()})
|
|
59
|
-
|
|
60
|
-
const {hash, size} = await this.#hashAndSave(id, readable)
|
|
61
55
|
|
|
56
|
+
await this.#wip.set(id, {created: Date.now()})
|
|
57
|
+
const {hash, size} = await saveAndHash(this.#bucket, id, readable)
|
|
62
58
|
await this.#wip.del(id)
|
|
63
|
-
await this.#finalizeWrite(hash, id, size)
|
|
64
59
|
|
|
60
|
+
await this.#finalizeWrite(hash, id, size)
|
|
65
61
|
void this.#selfClean().catch(() => {})
|
|
66
|
-
|
|
67
62
|
return hash
|
|
68
63
|
}
|
|
69
64
|
|
|
@@ -72,65 +67,32 @@ export class Mammoth {
|
|
|
72
67
|
}
|
|
73
68
|
|
|
74
69
|
async stats() {
|
|
75
|
-
return (await this.#stats.get()) ?? {size: 0}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
async #getId(hash: Hash): Promise<Id | undefined> {
|
|
79
|
-
return this.#ids.get<string>(hash)
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
async #needId(hash: Hash): Promise<Id> {
|
|
83
|
-
return got(await this.#getId(hash), `file not found by hash "${hash}"`)
|
|
70
|
+
return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
|
|
84
71
|
}
|
|
85
72
|
|
|
86
|
-
async
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
const hasher = blake3.create()
|
|
92
|
-
|
|
93
|
-
for await (const chunk of readable) {
|
|
94
|
-
await writer.write(chunk as Uint8Array<ArrayBuffer>)
|
|
95
|
-
hasher.update(chunk)
|
|
96
|
-
size += chunk.byteLength
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
await writer.close()
|
|
100
|
-
await done
|
|
101
|
-
|
|
102
|
-
const hash = hex.fromBytes(hasher.digest())
|
|
103
|
-
return {hash, size}
|
|
104
|
-
}
|
|
73
|
+
#updateStats = queue(async(fn: (stats: Stats) => void) => {
|
|
74
|
+
const stats = await this.stats()
|
|
75
|
+
fn(stats)
|
|
76
|
+
await this.#stats.set(stats)
|
|
77
|
+
})
|
|
105
78
|
|
|
106
79
|
#finalizeWrite = queue(async(hash: Hash, id: Id, size: number) => {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
if (existingId) {
|
|
110
|
-
await this.#bucket.delete(id)
|
|
80
|
+
if (await this.#ids.has(hash)) {
|
|
81
|
+
await this.#bucket.delete(id) // forget this new file (we already have it)
|
|
111
82
|
}
|
|
112
83
|
else {
|
|
113
84
|
await this.#ids.set(hash, id)
|
|
114
|
-
await this.#
|
|
85
|
+
await this.#updateStats(stats => {
|
|
86
|
+
stats.count++
|
|
87
|
+
stats.size += size
|
|
88
|
+
})
|
|
115
89
|
}
|
|
116
90
|
})
|
|
117
91
|
|
|
118
|
-
#addSize = queue(async(sizeChange: number) => {
|
|
119
|
-
const oldStats = await this.stats()
|
|
120
|
-
await this.#stats.set({...oldStats, size: oldStats.size + sizeChange})
|
|
121
|
-
})
|
|
122
|
-
|
|
123
92
|
#selfClean = queue(async() => {
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
for await (const [id, {created}] of this.#wip.entries({limit: 64})) {
|
|
128
|
-
const since = now - created
|
|
129
|
-
const isExpired = since > time.days(7)
|
|
130
|
-
if (isExpired)
|
|
131
|
-
expiredIds.push(id)
|
|
132
|
-
}
|
|
133
|
-
|
|
93
|
+
const expiredIds = (await collect(this.#wip.entries({limit: 64})))
|
|
94
|
+
.filter(([,wip]) => isExpired(wip))
|
|
95
|
+
.map(([id]) => id)
|
|
134
96
|
await Promise.all(expiredIds.map(id => this.#bucket.delete(id)))
|
|
135
97
|
await this.#wip.del(...expiredIds)
|
|
136
98
|
})
|
package/s/core/types.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
export type Hash = string
|
|
3
3
|
export type Id = string
|
|
4
4
|
|
|
5
|
+
/** file blob store. */
|
|
5
6
|
export type Bucket = {
|
|
6
7
|
has(id: Id): Promise<boolean>
|
|
7
8
|
size(id: Id): Promise<number>
|
|
@@ -11,6 +12,15 @@ export type Bucket = {
|
|
|
11
12
|
}
|
|
12
13
|
|
|
13
14
|
export type Stats = {
|
|
15
|
+
|
|
16
|
+
/** total number of bytes in the whole datalake. */
|
|
14
17
|
size: number
|
|
18
|
+
|
|
19
|
+
/** total number of known files in the datalake. */
|
|
20
|
+
count: number
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type Wip = {
|
|
24
|
+
created: number
|
|
15
25
|
}
|
|
16
26
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
|
|
2
|
+
import {hex} from "@e280/stz"
|
|
3
|
+
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
|
+
import {Bucket, Id} from "../types.js"
|
|
5
|
+
|
|
6
|
+
export async function saveAndHash(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>) {
|
|
7
|
+
let size = 0
|
|
8
|
+
const pipe = new TransformStream()
|
|
9
|
+
const done = bucket.write(id, pipe.readable)
|
|
10
|
+
const writer = pipe.writable.getWriter()
|
|
11
|
+
const hasher = blake3.create()
|
|
12
|
+
|
|
13
|
+
for await (const chunk of readable) {
|
|
14
|
+
await writer.write(chunk as Uint8Array<ArrayBuffer>)
|
|
15
|
+
hasher.update(chunk)
|
|
16
|
+
size += chunk.byteLength
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
await writer.close()
|
|
20
|
+
await done
|
|
21
|
+
|
|
22
|
+
const hash = hex.fromBytes(hasher.digest())
|
|
23
|
+
return {hash, size}
|
|
24
|
+
}
|
|
25
|
+
|
package/s/test.ts
CHANGED
|
@@ -13,9 +13,9 @@ const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
|
|
|
13
13
|
const quickstream = (b: number[]) => new Blob([new Uint8Array(b)]).stream()
|
|
14
14
|
|
|
15
15
|
function setup() {
|
|
16
|
-
const
|
|
16
|
+
const kv = new Kv()
|
|
17
17
|
const bucket = new MemoryBucket()
|
|
18
|
-
const mammoth = new Mammoth(
|
|
18
|
+
const mammoth = new Mammoth(bucket, kv)
|
|
19
19
|
return {mammoth}
|
|
20
20
|
}
|
|
21
21
|
|
|
@@ -23,15 +23,13 @@ await science.run({
|
|
|
23
23
|
"mammoth node": test(async() => {
|
|
24
24
|
const dataDir = "data"
|
|
25
25
|
await rm(dataDir, {recursive: true, force: true})
|
|
26
|
-
const
|
|
27
|
-
const bucket = new DiskBucket(dataDir)
|
|
28
|
-
const mammoth = new Mammoth(manifest, bucket)
|
|
26
|
+
const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv())
|
|
29
27
|
const hash = await mammoth.write(blob().stream())
|
|
30
28
|
const second = await mammoth.read(hash)
|
|
31
29
|
expect(bytes.eq(await second.bytes(), await blob().bytes()))
|
|
32
30
|
}),
|
|
33
31
|
|
|
34
|
-
"mammoth": {
|
|
32
|
+
"mammoth memory": {
|
|
35
33
|
".write and .read": test(async() => {
|
|
36
34
|
const {mammoth} = setup()
|
|
37
35
|
const hash = await mammoth.write(blob().stream())
|
|
@@ -66,15 +64,15 @@ await science.run({
|
|
|
66
64
|
|
|
67
65
|
".stats": test(async() => {
|
|
68
66
|
const {mammoth} = setup()
|
|
69
|
-
expect(await mammoth.stats()).deep({size: 0})
|
|
67
|
+
expect(await mammoth.stats()).deep({count: 0, size: 0})
|
|
70
68
|
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
|
|
71
|
-
expect(await mammoth.stats()).deep({size: 3})
|
|
69
|
+
expect(await mammoth.stats()).deep({count: 1, size: 3})
|
|
72
70
|
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
|
|
73
|
-
expect(await mammoth.stats()).deep({size: 7})
|
|
71
|
+
expect(await mammoth.stats()).deep({count: 2, size: 7})
|
|
74
72
|
await mammoth.delete(hashA)
|
|
75
|
-
expect(await mammoth.stats()).deep({size: 4})
|
|
73
|
+
expect(await mammoth.stats()).deep({count: 1, size: 4})
|
|
76
74
|
await mammoth.delete(hashB)
|
|
77
|
-
expect(await mammoth.stats()).deep({size: 0})
|
|
75
|
+
expect(await mammoth.stats()).deep({count: 0, size: 0})
|
|
78
76
|
}),
|
|
79
77
|
|
|
80
78
|
".has": test(async() => {
|
package/x/browser/example.js
CHANGED
|
@@ -10,7 +10,7 @@ export async function example() {
|
|
|
10
10
|
const directory = await root.getDirectoryHandle("mammoth", { create: true });
|
|
11
11
|
const bucket = new OpfsBucket(directory);
|
|
12
12
|
// setup the mammoth
|
|
13
|
-
const mammoth = new Mammoth(
|
|
13
|
+
const mammoth = new Mammoth(bucket, kv);
|
|
14
14
|
const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
|
|
15
15
|
// write a file as an example
|
|
16
16
|
const hash = await mammoth.write(myFile.stream());
|
package/x/browser/example.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACjD,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,GAAG,CAAC,CAAA;AACrD,CAAC"}
|
package/x/core/mammoth.d.ts
CHANGED
|
@@ -1,20 +1,14 @@
|
|
|
1
1
|
import { Kv } from "@e280/kv";
|
|
2
|
-
import { Hash, Bucket } from "./types.js";
|
|
2
|
+
import { Hash, Bucket, Stats } from "./types.js";
|
|
3
3
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
4
4
|
export declare class Mammoth {
|
|
5
5
|
#private;
|
|
6
|
-
constructor(
|
|
7
|
-
/** key-value metadata manifest. */
|
|
8
|
-
kv?: Kv<any>,
|
|
9
|
-
/** dumb raw file blob storage. */
|
|
10
|
-
bucket?: Bucket);
|
|
6
|
+
constructor(bucket?: Bucket, kv?: Kv<any>);
|
|
11
7
|
has(hash: Hash): Promise<boolean>;
|
|
12
8
|
size(hash: Hash): Promise<number>;
|
|
13
9
|
read(hash: Hash): Promise<Blob>;
|
|
14
10
|
delete(hash: Hash): Promise<void>;
|
|
15
11
|
write(readable: ReadableStream<Uint8Array>): Promise<Hash>;
|
|
16
12
|
hashes(): AsyncGenerator<string, void, unknown>;
|
|
17
|
-
stats(): Promise<
|
|
18
|
-
size: number;
|
|
19
|
-
}>;
|
|
13
|
+
stats(): Promise<Stats>;
|
|
20
14
|
}
|
package/x/core/mammoth.js
CHANGED
|
@@ -1,19 +1,17 @@
|
|
|
1
1
|
import { Kv } from "@e280/kv";
|
|
2
|
-
import {
|
|
3
|
-
import { blake3 } from "@noble/hashes/blake3.js";
|
|
2
|
+
import { collect, got, queue } from "@e280/stz";
|
|
4
3
|
import { randomId } from "./utils/random-id.js";
|
|
4
|
+
import { notFound } from "./utils/not-found.js";
|
|
5
5
|
import { MemoryBucket } from "./memory-bucket.js";
|
|
6
|
+
import { isExpired } from "./utils/is-expired.js";
|
|
7
|
+
import { saveAndHash } from "./utils/save-and-hash.js";
|
|
6
8
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
7
9
|
export class Mammoth {
|
|
8
|
-
#ids;
|
|
9
|
-
#wip;
|
|
10
|
-
#stats;
|
|
11
|
-
#bucket;
|
|
12
|
-
constructor(
|
|
13
|
-
/** key-value metadata manifest. */
|
|
14
|
-
kv = new Kv(),
|
|
15
|
-
/** dumb raw file blob storage. */
|
|
16
|
-
bucket = new MemoryBucket()) {
|
|
10
|
+
#ids; // associates file hashes with bucket ids
|
|
11
|
+
#wip; // temporary record of writes in-progress
|
|
12
|
+
#stats; // statistics about the whole datalake
|
|
13
|
+
#bucket; // file blobstore
|
|
14
|
+
constructor(bucket = new MemoryBucket(), kv = new Kv()) {
|
|
17
15
|
this.#ids = kv.scope("ids");
|
|
18
16
|
this.#wip = kv.scope("wip");
|
|
19
17
|
this.#stats = kv.store("stats");
|
|
@@ -23,26 +21,29 @@ export class Mammoth {
|
|
|
23
21
|
return this.#ids.has(hash);
|
|
24
22
|
}
|
|
25
23
|
async size(hash) {
|
|
26
|
-
const id = await this.#
|
|
24
|
+
const id = got(await this.#ids.get(hash), notFound(hash));
|
|
27
25
|
return this.#bucket.size(id);
|
|
28
26
|
}
|
|
29
27
|
async read(hash) {
|
|
30
|
-
const id = await this.#
|
|
28
|
+
const id = got(await this.#ids.get(hash), notFound(hash));
|
|
31
29
|
return this.#bucket.read(id);
|
|
32
30
|
}
|
|
33
31
|
async delete(hash) {
|
|
34
|
-
const id = await this.#
|
|
32
|
+
const id = await this.#ids.get(hash);
|
|
35
33
|
if (id) {
|
|
36
34
|
const size = await this.#bucket.size(id);
|
|
37
35
|
await this.#ids.del(hash);
|
|
38
36
|
await this.#bucket.delete(id);
|
|
39
|
-
await this.#
|
|
37
|
+
await this.#updateStats(stats => {
|
|
38
|
+
stats.count--;
|
|
39
|
+
stats.size -= size;
|
|
40
|
+
});
|
|
40
41
|
}
|
|
41
42
|
}
|
|
42
43
|
async write(readable) {
|
|
43
44
|
const id = randomId();
|
|
44
45
|
await this.#wip.set(id, { created: Date.now() });
|
|
45
|
-
const { hash, size } = await this.#
|
|
46
|
+
const { hash, size } = await saveAndHash(this.#bucket, id, readable);
|
|
46
47
|
await this.#wip.del(id);
|
|
47
48
|
await this.#finalizeWrite(hash, id, size);
|
|
48
49
|
void this.#selfClean().catch(() => { });
|
|
@@ -52,53 +53,29 @@ export class Mammoth {
|
|
|
52
53
|
yield* this.#ids.keys();
|
|
53
54
|
}
|
|
54
55
|
async stats() {
|
|
55
|
-
return (await this.#stats.get()) ?? { size: 0 };
|
|
56
|
-
}
|
|
57
|
-
async #getId(hash) {
|
|
58
|
-
return this.#ids.get(hash);
|
|
59
|
-
}
|
|
60
|
-
async #needId(hash) {
|
|
61
|
-
return got(await this.#getId(hash), `file not found by hash "${hash}"`);
|
|
62
|
-
}
|
|
63
|
-
async #hashAndSave(id, readable) {
|
|
64
|
-
let size = 0;
|
|
65
|
-
const pipe = new TransformStream();
|
|
66
|
-
const done = this.#bucket.write(id, pipe.readable);
|
|
67
|
-
const writer = pipe.writable.getWriter();
|
|
68
|
-
const hasher = blake3.create();
|
|
69
|
-
for await (const chunk of readable) {
|
|
70
|
-
await writer.write(chunk);
|
|
71
|
-
hasher.update(chunk);
|
|
72
|
-
size += chunk.byteLength;
|
|
73
|
-
}
|
|
74
|
-
await writer.close();
|
|
75
|
-
await done;
|
|
76
|
-
const hash = hex.fromBytes(hasher.digest());
|
|
77
|
-
return { hash, size };
|
|
56
|
+
return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
|
|
78
57
|
}
|
|
58
|
+
#updateStats = queue(async (fn) => {
|
|
59
|
+
const stats = await this.stats();
|
|
60
|
+
fn(stats);
|
|
61
|
+
await this.#stats.set(stats);
|
|
62
|
+
});
|
|
79
63
|
#finalizeWrite = queue(async (hash, id, size) => {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
await this.#bucket.delete(id);
|
|
64
|
+
if (await this.#ids.has(hash)) {
|
|
65
|
+
await this.#bucket.delete(id); // forget this new file (we already have it)
|
|
83
66
|
}
|
|
84
67
|
else {
|
|
85
68
|
await this.#ids.set(hash, id);
|
|
86
|
-
await this.#
|
|
69
|
+
await this.#updateStats(stats => {
|
|
70
|
+
stats.count++;
|
|
71
|
+
stats.size += size;
|
|
72
|
+
});
|
|
87
73
|
}
|
|
88
74
|
});
|
|
89
|
-
#addSize = queue(async (sizeChange) => {
|
|
90
|
-
const oldStats = await this.stats();
|
|
91
|
-
await this.#stats.set({ ...oldStats, size: oldStats.size + sizeChange });
|
|
92
|
-
});
|
|
93
75
|
#selfClean = queue(async () => {
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const since = now - created;
|
|
98
|
-
const isExpired = since > time.days(7);
|
|
99
|
-
if (isExpired)
|
|
100
|
-
expiredIds.push(id);
|
|
101
|
-
}
|
|
76
|
+
const expiredIds = (await collect(this.#wip.entries({ limit: 64 })))
|
|
77
|
+
.filter(([, wip]) => isExpired(wip))
|
|
78
|
+
.map(([id]) => id);
|
|
102
79
|
await Promise.all(expiredIds.map(id => this.#bucket.delete(id)));
|
|
103
80
|
await this.#wip.del(...expiredIds);
|
|
104
81
|
});
|
package/x/core/mammoth.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAE7C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,SAAS,EAAC,MAAM,uBAAuB,CAAA;AAC/C,OAAO,EAAC,WAAW,EAAC,MAAM,0BAA0B,CAAA;AAGpD,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,IAAI,CAAA,CAAC,yCAAyC;IAC9C,IAAI,CAAA,CAAC,yCAAyC;IAC9C,MAAM,CAAA,CAAC,sCAAsC;IAC7C,OAAO,CAAA,CAAC,iBAAiB;IAEzB,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAS,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC3B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACpC,IAAI,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACxC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACzB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE;gBAC/B,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,KAAK,CAAC,IAAI,IAAI,IAAI,CAAA;YACnB,CAAC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QAErB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;QAC9C,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;QAClE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEvB,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;QACzC,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACtC,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAA;IACxB,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,YAAY,GAAG,KAAK,CAAC,KAAK,EAAC,EAA0B,EAAE,EAAE;QACxD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;QAChC,EAAE,CAAC,KAAK,CAAC,CAAA;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC7B,CAAC,CAAC,CAAA;IAEF,cAAc,GAAG,KAAK,CAAC,KAAK,EAAC,IAAU,EAAE,EAAM,EAAE,IAAY,EAAE,EAAE;QAChE,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA,CAAC,4CAA4C;QAC3E,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAC7B,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE;gBAC/B,KAAK,CAAC,KAAK,EAAE,CAAA;gBACb,KAAK,CAAC,IAAI,IAAI,IAAI,CAAA;YACnB,CAAC,CAAC,CAAA;QACH,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,UAAU,GAAG,KAAK,CAAC,KAAK,IAAG,EAAE;QAC5B,MAAM,UAAU,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAC,KAAK,EAAE,EAAE,EAAC,CAAC,CAAC,CAAC;aAChE,MAAM,CAAC,CAAC,CAAC,EAAC,GAAG,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAA;QACnB,MAAM,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAChE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAA;IACnC,CAAC,CAAC,CAAA;CACF"}
|
package/x/core/types.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export type Hash = string;
|
|
2
2
|
export type Id = string;
|
|
3
|
+
/** file blob store. */
|
|
3
4
|
export type Bucket = {
|
|
4
5
|
has(id: Id): Promise<boolean>;
|
|
5
6
|
size(id: Id): Promise<number>;
|
|
@@ -8,5 +9,11 @@ export type Bucket = {
|
|
|
8
9
|
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
|
|
9
10
|
};
|
|
10
11
|
export type Stats = {
|
|
12
|
+
/** total number of bytes in the whole datalake. */
|
|
11
13
|
size: number;
|
|
14
|
+
/** total number of known files in the datalake. */
|
|
15
|
+
count: number;
|
|
16
|
+
};
|
|
17
|
+
export type Wip = {
|
|
18
|
+
created: number;
|
|
12
19
|
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAG9B,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAM;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAC5B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"not-found.js","sourceRoot":"","sources":["../../../s/core/utils/not-found.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,QAAQ,CAAC,IAAU;IAClC,OAAO,2BAA2B,IAAI,GAAG,CAAA;AAC1C,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { hex } from "@e280/stz";
|
|
2
|
+
import { blake3 } from "@noble/hashes/blake3.js";
|
|
3
|
+
export async function saveAndHash(bucket, id, readable) {
|
|
4
|
+
let size = 0;
|
|
5
|
+
const pipe = new TransformStream();
|
|
6
|
+
const done = bucket.write(id, pipe.readable);
|
|
7
|
+
const writer = pipe.writable.getWriter();
|
|
8
|
+
const hasher = blake3.create();
|
|
9
|
+
for await (const chunk of readable) {
|
|
10
|
+
await writer.write(chunk);
|
|
11
|
+
hasher.update(chunk);
|
|
12
|
+
size += chunk.byteLength;
|
|
13
|
+
}
|
|
14
|
+
await writer.close();
|
|
15
|
+
await done;
|
|
16
|
+
const hash = hex.fromBytes(hasher.digest());
|
|
17
|
+
return { hash, size };
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=save-and-hash.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"save-and-hash.js","sourceRoot":"","sources":["../../../s/core/utils/save-and-hash.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,EAAM,EAAE,QAAoC;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;IACzB,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,IAAI,CAAA;IAEV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
package/x/test.js
CHANGED
|
@@ -9,23 +9,21 @@ import { MemoryBucket } from "./core/memory-bucket.js";
|
|
|
9
9
|
const blob = () => new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
|
|
10
10
|
const quickstream = (b) => new Blob([new Uint8Array(b)]).stream();
|
|
11
11
|
function setup() {
|
|
12
|
-
const
|
|
12
|
+
const kv = new Kv();
|
|
13
13
|
const bucket = new MemoryBucket();
|
|
14
|
-
const mammoth = new Mammoth(
|
|
14
|
+
const mammoth = new Mammoth(bucket, kv);
|
|
15
15
|
return { mammoth };
|
|
16
16
|
}
|
|
17
17
|
await science.run({
|
|
18
18
|
"mammoth node": test(async () => {
|
|
19
19
|
const dataDir = "data";
|
|
20
20
|
await rm(dataDir, { recursive: true, force: true });
|
|
21
|
-
const
|
|
22
|
-
const bucket = new DiskBucket(dataDir);
|
|
23
|
-
const mammoth = new Mammoth(manifest, bucket);
|
|
21
|
+
const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv());
|
|
24
22
|
const hash = await mammoth.write(blob().stream());
|
|
25
23
|
const second = await mammoth.read(hash);
|
|
26
24
|
expect(bytes.eq(await second.bytes(), await blob().bytes()));
|
|
27
25
|
}),
|
|
28
|
-
"mammoth": {
|
|
26
|
+
"mammoth memory": {
|
|
29
27
|
".write and .read": test(async () => {
|
|
30
28
|
const { mammoth } = setup();
|
|
31
29
|
const hash = await mammoth.write(blob().stream());
|
|
@@ -56,15 +54,15 @@ await science.run({
|
|
|
56
54
|
}),
|
|
57
55
|
".stats": test(async () => {
|
|
58
56
|
const { mammoth } = setup();
|
|
59
|
-
expect(await mammoth.stats()).deep({ size: 0 });
|
|
57
|
+
expect(await mammoth.stats()).deep({ count: 0, size: 0 });
|
|
60
58
|
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
|
|
61
|
-
expect(await mammoth.stats()).deep({ size: 3 });
|
|
59
|
+
expect(await mammoth.stats()).deep({ count: 1, size: 3 });
|
|
62
60
|
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
|
|
63
|
-
expect(await mammoth.stats()).deep({ size: 7 });
|
|
61
|
+
expect(await mammoth.stats()).deep({ count: 2, size: 7 });
|
|
64
62
|
await mammoth.delete(hashA);
|
|
65
|
-
expect(await mammoth.stats()).deep({ size: 4 });
|
|
63
|
+
expect(await mammoth.stats()).deep({ count: 1, size: 4 });
|
|
66
64
|
await mammoth.delete(hashB);
|
|
67
|
-
expect(await mammoth.stats()).deep({ size: 0 });
|
|
65
|
+
expect(await mammoth.stats()).deep({ count: 0, size: 0 });
|
|
68
66
|
}),
|
|
69
67
|
".has": test(async () => {
|
|
70
68
|
const { mammoth } = setup();
|
package/x/test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,
|
|
1
|
+
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC;IAEF,gBAAgB,EAAE;QACjB,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACvB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC,CAAC;QAEF,mBAAmB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACnC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACrD,MAAM,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACtC,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC;QAEF,uBAAuB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACvD,CAAC,CAAC;QAEF,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACxB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,MAAM,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACvC,CAAC,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACzB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjD,CAAC,CAAC;QAEF,oBAAoB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACpC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;QACjC,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC,CAAC;KACF;CACD,CAAC,CAAA"}
|