@e280/mammoth 0.1.0-6 → 0.1.0-7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -5
- package/package.json +1 -1
- package/s/browser/example.ts +2 -2
- package/s/core/{hash-stream.ts → analyze.ts} +6 -2
- package/s/core/consts.ts +0 -1
- package/s/core/index.ts +2 -2
- package/s/core/mammoth.ts +36 -36
- package/s/core/types.ts +11 -1
- package/s/core/utils/manifest.ts +56 -32
- package/s/core/utils/{save-and-hash.ts → save.ts} +7 -2
- package/s/test.ts +11 -11
- package/x/browser/example.js +2 -2
- package/x/browser/example.js.map +1 -1
- package/x/core/analyze.d.ts +2 -0
- package/x/core/{hash-stream.js → analyze.js} +6 -3
- package/x/core/analyze.js.map +1 -0
- package/x/core/consts.d.ts +0 -1
- package/x/core/consts.js +0 -1
- package/x/core/consts.js.map +1 -1
- package/x/core/index.d.ts +2 -2
- package/x/core/index.js +2 -2
- package/x/core/index.js.map +1 -1
- package/x/core/mammoth.d.ts +4 -4
- package/x/core/mammoth.js +32 -30
- package/x/core/mammoth.js.map +1 -1
- package/x/core/types.d.ts +8 -1
- package/x/core/utils/manifest.d.ts +7 -6
- package/x/core/utils/manifest.js +52 -30
- package/x/core/utils/manifest.js.map +1 -1
- package/x/core/utils/save.d.ts +2 -0
- package/x/core/utils/{save-and-hash.js → save.js} +2 -2
- package/x/core/utils/save.js.map +1 -0
- package/x/test.js +11 -11
- package/x/test.js.map +1 -1
- package/x/core/hash-stream.d.ts +0 -1
- package/x/core/hash-stream.js.map +0 -1
- package/x/core/utils/save-and-hash.d.ts +0 -5
- package/x/core/utils/save-and-hash.js.map +0 -1
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# 🦣 mammoth
|
|
5
5
|
> *big files. small api.*
|
|
6
6
|
|
|
7
|
-
**mammoth is a
|
|
7
|
+
**mammoth is a file storage typescript library.** we use it in our web apps to store files like user uploads. it's content-addressed. files are streamed into a bucket, identified by their blake3 hash, and deduplicated. mammoth works the same whether it's backed by a bucket in memory, on disk, in the cloud, or in the browser's opfs.
|
|
8
8
|
|
|
9
9
|
```bash
|
|
10
10
|
npm install @e280/mammoth
|
|
@@ -17,9 +17,9 @@ const mammoth = new Mammoth()
|
|
|
17
17
|
```
|
|
18
18
|
|
|
19
19
|
### 🦣 mammoth stores files.
|
|
20
|
-
- **write a file,** and you get back its blake3 hash.
|
|
20
|
+
- **write a file,** and you get back its blake3 `hash` and `size` in bytes.
|
|
21
21
|
```ts
|
|
22
|
-
const hash = await mammoth.write(blob.stream())
|
|
22
|
+
const {hash, size} = await mammoth.write(blob.stream())
|
|
23
23
|
```
|
|
24
24
|
- **read a file,** identified by its hash.
|
|
25
25
|
```ts
|
|
@@ -37,11 +37,15 @@ const mammoth = new Mammoth()
|
|
|
37
37
|
import {Kv} from "@e280/kv"
|
|
38
38
|
|
|
39
39
|
const mammoth = new Mammoth(
|
|
40
|
+
|
|
41
|
+
// bucket for blob storage
|
|
40
42
|
new MemoryBucket(),
|
|
43
|
+
|
|
44
|
+
// kv for metadata bookkeeping
|
|
41
45
|
new Kv(),
|
|
42
46
|
)
|
|
43
47
|
```
|
|
44
|
-
defaults shown,
|
|
48
|
+
defaults shown, this is equivalent:
|
|
45
49
|
```ts
|
|
46
50
|
const mammoth = new Mammoth()
|
|
47
51
|
```
|
|
@@ -74,7 +78,7 @@ const mammoth = new Mammoth()
|
|
|
74
78
|
```ts
|
|
75
79
|
const exists = await mammoth.has(hash)
|
|
76
80
|
```
|
|
77
|
-
- **get file info,** including size in bytes, added timestamp, and bucket id
|
|
81
|
+
- **get file info,** including `size` in bytes, `added` timestamp, and bucket `id`.
|
|
78
82
|
```ts
|
|
79
83
|
const {size, added, id} = await mammoth.info(hash)
|
|
80
84
|
```
|
package/package.json
CHANGED
package/s/browser/example.ts
CHANGED
|
@@ -19,7 +19,7 @@ export async function example() {
|
|
|
19
19
|
const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])])
|
|
20
20
|
|
|
21
21
|
// write a file as an example
|
|
22
|
-
const hash = await mammoth.write(myFile.stream())
|
|
23
|
-
console.log(`mammoth stored file
|
|
22
|
+
const {hash} = await mammoth.write(myFile.stream())
|
|
23
|
+
console.log(`mammoth stored file "${hash}"`)
|
|
24
24
|
}
|
|
25
25
|
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
|
|
2
2
|
import {hex, nap} from "@e280/stz"
|
|
3
3
|
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
|
+
import {Analysis} from "./types.js"
|
|
4
5
|
import {relaxer} from "./utils/relaxer.js"
|
|
5
6
|
|
|
6
|
-
export async function
|
|
7
|
+
export async function analyze(readable: ReadableStream<Uint8Array>): Promise<Analysis> {
|
|
8
|
+
let size = 0
|
|
7
9
|
const hasher = blake3.create()
|
|
8
10
|
const relax = relaxer()
|
|
9
11
|
|
|
10
12
|
for await (const chunk of readable) {
|
|
11
13
|
hasher.update(chunk)
|
|
14
|
+
size += chunk.byteLength
|
|
12
15
|
|
|
13
16
|
if (relax())
|
|
14
17
|
await nap()
|
|
15
18
|
}
|
|
16
19
|
|
|
17
|
-
|
|
20
|
+
const hash = hex.fromBytes(hasher.digest())
|
|
21
|
+
return {hash, size}
|
|
18
22
|
}
|
|
19
23
|
|
package/s/core/consts.ts
CHANGED
package/s/core/index.ts
CHANGED
package/s/core/mammoth.ts
CHANGED
|
@@ -3,11 +3,11 @@ import {Kv} from "@e280/kv"
|
|
|
3
3
|
import {lane, queue} from "@e280/stz"
|
|
4
4
|
|
|
5
5
|
import {consts} from "./consts.js"
|
|
6
|
-
import {
|
|
6
|
+
import {save} from "./utils/save.js"
|
|
7
7
|
import {Manifest} from "./utils/manifest.js"
|
|
8
8
|
import {randomId} from "./utils/random-id.js"
|
|
9
9
|
import {MemoryBucket} from "./memory-bucket.js"
|
|
10
|
-
import {
|
|
10
|
+
import {Hash, Bucket, Analysis} from "./types.js"
|
|
11
11
|
|
|
12
12
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
13
13
|
export class Mammoth {
|
|
@@ -20,6 +20,10 @@ export class Mammoth {
|
|
|
20
20
|
this.#manifest = new Manifest(kv)
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
async* hashes() {
|
|
24
|
+
yield* this.#manifest.hashes()
|
|
25
|
+
}
|
|
26
|
+
|
|
23
27
|
async has(hash: Hash) {
|
|
24
28
|
return this.#manifest.hasHash(hash)
|
|
25
29
|
}
|
|
@@ -28,6 +32,10 @@ export class Mammoth {
|
|
|
28
32
|
return this.#manifest.needInfo(hash)
|
|
29
33
|
}
|
|
30
34
|
|
|
35
|
+
async stats() {
|
|
36
|
+
return this.#manifest.getStats()
|
|
37
|
+
}
|
|
38
|
+
|
|
31
39
|
async read(hash: Hash) {
|
|
32
40
|
const {id} = await this.#manifest.needInfo(hash)
|
|
33
41
|
return this.#bucket.read(id)
|
|
@@ -36,47 +44,39 @@ export class Mammoth {
|
|
|
36
44
|
async delete(hash: Hash) {
|
|
37
45
|
await this.#wholesome(async() => {
|
|
38
46
|
const info = await this.#manifest.getInfo(hash)
|
|
39
|
-
if (info)
|
|
40
|
-
await this.#manifest.
|
|
41
|
-
await this.#bucket.delete(info.id)
|
|
42
|
-
await this.#manifest.statsRemoveFile(info.size)
|
|
43
|
-
}
|
|
47
|
+
if (info)
|
|
48
|
+
await this.#manifest.scheduleDeletion(hash, info)
|
|
44
49
|
})
|
|
50
|
+
await this.#cleanup()
|
|
45
51
|
}
|
|
46
52
|
|
|
47
|
-
async write(readable: ReadableStream<Uint8Array>): Promise<
|
|
53
|
+
async write(readable: ReadableStream<Uint8Array>): Promise<Analysis> {
|
|
48
54
|
const id = randomId()
|
|
49
|
-
|
|
50
55
|
await this.#manifest.addWip(id)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
return hash
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
async* hashes() {
|
|
69
|
-
yield* this.#manifest.hashes()
|
|
56
|
+
try {
|
|
57
|
+
const analysis = await save(this.#bucket, id, readable)
|
|
58
|
+
const {hash, size} = analysis
|
|
59
|
+
const info = {id, size, added: Date.now()}
|
|
60
|
+
await this.#wholesome(() => this.#manifest.commit(hash, info))
|
|
61
|
+
return analysis
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
await this.#manifest.moveWipToTrash(id)
|
|
65
|
+
throw error
|
|
66
|
+
}
|
|
67
|
+
finally {
|
|
68
|
+
void this.#cleanup().catch(e => console.error("cleanup failed", e))
|
|
69
|
+
}
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
#cleanup = queue(async() => {
|
|
73
|
+
for await (const id of this.#manifest.getExpiredWipIds())
|
|
74
|
+
await this.#manifest.moveWipToTrash(id)
|
|
75
75
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
})
|
|
76
|
+
for await (const id of this.#manifest.listTrashIds()) {
|
|
77
|
+
await this.#bucket.delete(id)
|
|
78
|
+
await this.#manifest.dropTrashRecord(id)
|
|
79
|
+
}
|
|
80
|
+
}, consts.max_jobs)
|
|
81
81
|
}
|
|
82
82
|
|
package/s/core/types.ts
CHANGED
|
@@ -10,13 +10,23 @@ export type Bucket = {
|
|
|
10
10
|
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
/** report about a file */
|
|
14
|
+
export type Analysis = {
|
|
15
|
+
|
|
16
|
+
/** blake3 hash of the file's contents */
|
|
17
|
+
hash: Hash
|
|
18
|
+
|
|
19
|
+
/** filesize in bytes */
|
|
20
|
+
size: number
|
|
21
|
+
}
|
|
22
|
+
|
|
13
23
|
/** metadata for a single file. */
|
|
14
24
|
export type Info = {
|
|
15
25
|
|
|
16
26
|
/** bucket id for this file's data. */
|
|
17
27
|
id: Id
|
|
18
28
|
|
|
19
|
-
/**
|
|
29
|
+
/** filesize in bytes. */
|
|
20
30
|
size: number
|
|
21
31
|
|
|
22
32
|
/** when this file was added to the datalake. */
|
package/s/core/utils/manifest.ts
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
|
|
2
2
|
import {Kv} from "@e280/kv"
|
|
3
|
-
import {
|
|
4
|
-
import {consts} from "../consts.js"
|
|
3
|
+
import {got} from "@e280/stz"
|
|
5
4
|
import {notFound} from "./not-found.js"
|
|
6
5
|
import {isExpired} from "./is-expired.js"
|
|
7
6
|
import {Hash, Id, Info, Stats, Wip} from "../types.js"
|
|
8
7
|
|
|
9
8
|
export class Manifest {
|
|
9
|
+
#kv
|
|
10
|
+
|
|
11
|
+
/** statistics about the whole datalake */
|
|
12
|
+
#stats
|
|
10
13
|
|
|
11
14
|
/** associates file hashes with bucket ids */
|
|
12
15
|
#info
|
|
@@ -15,12 +18,18 @@ export class Manifest {
|
|
|
15
18
|
#wip
|
|
16
19
|
|
|
17
20
|
/** statistics about the whole datalake */
|
|
18
|
-
#
|
|
21
|
+
#trash
|
|
19
22
|
|
|
20
23
|
constructor(kv = new Kv()) {
|
|
24
|
+
this.#kv = kv
|
|
25
|
+
this.#stats = kv.store<Stats>("stats")
|
|
21
26
|
this.#info = kv.scope<Info>("info")
|
|
22
27
|
this.#wip = kv.scope<Wip>("wip")
|
|
23
|
-
this.#
|
|
28
|
+
this.#trash = kv.scope<true>("trash")
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async getStats() {
|
|
32
|
+
return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
|
|
24
33
|
}
|
|
25
34
|
|
|
26
35
|
async* hashes() {
|
|
@@ -31,10 +40,6 @@ export class Manifest {
|
|
|
31
40
|
return this.#info.has(hash)
|
|
32
41
|
}
|
|
33
42
|
|
|
34
|
-
async saveInfo(hash: Hash, info: Info) {
|
|
35
|
-
await this.#info.set(hash, info)
|
|
36
|
-
}
|
|
37
|
-
|
|
38
43
|
async getInfo(hash: Hash) {
|
|
39
44
|
return await this.#info.get(hash)
|
|
40
45
|
}
|
|
@@ -43,10 +48,6 @@ export class Manifest {
|
|
|
43
48
|
return got(await this.getInfo(hash), notFound(hash))
|
|
44
49
|
}
|
|
45
50
|
|
|
46
|
-
async deleteInfo(hash: Hash) {
|
|
47
|
-
await this.#info.del(hash)
|
|
48
|
-
}
|
|
49
|
-
|
|
50
51
|
async addWip(id: Id) {
|
|
51
52
|
await this.#wip.set(id, {created: Date.now()})
|
|
52
53
|
}
|
|
@@ -55,35 +56,58 @@ export class Manifest {
|
|
|
55
56
|
await this.#wip.del(...ids)
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
async
|
|
59
|
-
await this.#
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
}
|
|
59
|
+
async* getExpiredWipIds() {
|
|
60
|
+
for await (const [id, wip] of this.#wip.entries()) {
|
|
61
|
+
if (isExpired(wip))
|
|
62
|
+
yield id
|
|
63
|
+
}
|
|
63
64
|
}
|
|
64
65
|
|
|
65
|
-
async
|
|
66
|
-
await this.#
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
async moveWipToTrash(id: Id) {
|
|
67
|
+
await this.#kv.transaction(() => [
|
|
68
|
+
this.#wip.write.del(id),
|
|
69
|
+
this.#trash.write.set(id, true),
|
|
70
|
+
])
|
|
70
71
|
}
|
|
71
72
|
|
|
72
|
-
async
|
|
73
|
-
|
|
74
|
-
return (await collect(this.#wip.entries({limit})))
|
|
75
|
-
.filter(([,wip]) => isExpired(wip))
|
|
76
|
-
.map(([id]) => id)
|
|
73
|
+
async dropTrashRecord(id: Id) {
|
|
74
|
+
await this.#trash.del(id)
|
|
77
75
|
}
|
|
78
76
|
|
|
79
|
-
async
|
|
80
|
-
|
|
77
|
+
async* listTrashIds() {
|
|
78
|
+
yield* this.#trash.keys()
|
|
81
79
|
}
|
|
82
80
|
|
|
83
|
-
async
|
|
81
|
+
async scheduleDeletion(hash: Hash, info: Info) {
|
|
84
82
|
const stats = await this.getStats()
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
stats.count -= 1
|
|
84
|
+
stats.size -= info.size
|
|
85
|
+
await this.#kv.transaction(() => [
|
|
86
|
+
this.#info.write.del(hash),
|
|
87
|
+
this.#trash.write.set(info.id, true),
|
|
88
|
+
this.#stats.write.set(stats),
|
|
89
|
+
])
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
async commit(hash: Hash, info: Info) {
|
|
93
|
+
const isNewFile = !await this.hasHash(hash)
|
|
94
|
+
|
|
95
|
+
if (isNewFile) {
|
|
96
|
+
const stats = await this.getStats()
|
|
97
|
+
stats.count += 1
|
|
98
|
+
stats.size += info.size
|
|
99
|
+
await this.#kv.transaction(() => [
|
|
100
|
+
this.#wip.write.del(info.id),
|
|
101
|
+
this.#info.write.set(hash, info),
|
|
102
|
+
this.#stats.write.set(stats)
|
|
103
|
+
])
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
await this.#kv.transaction(() => [
|
|
107
|
+
this.#wip.write.del(info.id),
|
|
108
|
+
this.#trash.write.set(info.id, true),
|
|
109
|
+
])
|
|
110
|
+
}
|
|
87
111
|
}
|
|
88
112
|
}
|
|
89
113
|
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
|
|
2
2
|
import {hex} from "@e280/stz"
|
|
3
3
|
import {blake3} from "@noble/hashes/blake3.js"
|
|
4
|
-
import {Bucket, Id} from "../types.js"
|
|
4
|
+
import {Analysis, Bucket, Id} from "../types.js"
|
|
5
|
+
|
|
6
|
+
export async function save(
|
|
7
|
+
bucket: Bucket,
|
|
8
|
+
id: Id,
|
|
9
|
+
readable: ReadableStream<Uint8Array>,
|
|
10
|
+
): Promise<Analysis> {
|
|
5
11
|
|
|
6
|
-
export async function saveAndHash(bucket: Bucket, id: Id, readable: ReadableStream<Uint8Array>) {
|
|
7
12
|
let size = 0
|
|
8
13
|
const pipe = new TransformStream()
|
|
9
14
|
const done = bucket.write(id, pipe.readable)
|
package/s/test.ts
CHANGED
|
@@ -24,7 +24,7 @@ await science.run({
|
|
|
24
24
|
const dataDir = "data"
|
|
25
25
|
await rm(dataDir, {recursive: true, force: true})
|
|
26
26
|
const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv())
|
|
27
|
-
const hash = await mammoth.write(blob().stream())
|
|
27
|
+
const {hash} = await mammoth.write(blob().stream())
|
|
28
28
|
const second = await mammoth.read(hash)
|
|
29
29
|
expect(bytes.eq(await second.bytes(), await blob().bytes()))
|
|
30
30
|
}),
|
|
@@ -32,15 +32,15 @@ await science.run({
|
|
|
32
32
|
"mammoth memory": {
|
|
33
33
|
".write and .read": test(async() => {
|
|
34
34
|
const {mammoth} = setup()
|
|
35
|
-
const hash = await mammoth.write(blob().stream())
|
|
35
|
+
const {hash} = await mammoth.write(blob().stream())
|
|
36
36
|
const second = await mammoth.read(hash)
|
|
37
37
|
expect(bytes.eq(await second.bytes(), await blob().bytes()))
|
|
38
38
|
}),
|
|
39
39
|
|
|
40
40
|
".write deduplication": test(async() => {
|
|
41
41
|
const {mammoth} = setup()
|
|
42
|
-
const hashA = await mammoth.write(blob().stream())
|
|
43
|
-
const hashB = await mammoth.write(blob().stream())
|
|
42
|
+
const {hash: hashA} = await mammoth.write(blob().stream())
|
|
43
|
+
const {hash: hashB} = await mammoth.write(blob().stream())
|
|
44
44
|
const keys = await collect(mammoth.hashes())
|
|
45
45
|
expect(hashA).is(hashB)
|
|
46
46
|
expect(keys.length).is(1)
|
|
@@ -49,7 +49,7 @@ await science.run({
|
|
|
49
49
|
|
|
50
50
|
".write empty file": test(async() => {
|
|
51
51
|
const {mammoth} = setup()
|
|
52
|
-
const hash = await mammoth.write(new Blob().stream())
|
|
52
|
+
const {hash} = await mammoth.write(new Blob().stream())
|
|
53
53
|
expect((await mammoth.info(hash)).size).is(0)
|
|
54
54
|
expect((await mammoth.read(hash)).size).is(0)
|
|
55
55
|
}),
|
|
@@ -65,9 +65,9 @@ await science.run({
|
|
|
65
65
|
".stats": test(async() => {
|
|
66
66
|
const {mammoth} = setup()
|
|
67
67
|
expect(await mammoth.stats()).deep({count: 0, size: 0})
|
|
68
|
-
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
|
|
68
|
+
const {hash: hashA} = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
|
|
69
69
|
expect(await mammoth.stats()).deep({count: 1, size: 3})
|
|
70
|
-
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
|
|
70
|
+
const {hash: hashB} = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
|
|
71
71
|
expect(await mammoth.stats()).deep({count: 2, size: 7})
|
|
72
72
|
await mammoth.delete(hashA)
|
|
73
73
|
expect(await mammoth.stats()).deep({count: 1, size: 4})
|
|
@@ -77,7 +77,7 @@ await science.run({
|
|
|
77
77
|
|
|
78
78
|
".has": test(async() => {
|
|
79
79
|
const {mammoth} = setup()
|
|
80
|
-
const hash = await mammoth.write(blob().stream())
|
|
80
|
+
const {hash} = await mammoth.write(blob().stream())
|
|
81
81
|
expect(await mammoth.has(hash)).is(true)
|
|
82
82
|
expect(await mammoth.has(randomId())).is(false)
|
|
83
83
|
}),
|
|
@@ -85,13 +85,13 @@ await science.run({
|
|
|
85
85
|
".size": test(async() => {
|
|
86
86
|
const {mammoth} = setup()
|
|
87
87
|
const b = blob()
|
|
88
|
-
const hash = await mammoth.write(b.stream())
|
|
88
|
+
const {hash} = await mammoth.write(b.stream())
|
|
89
89
|
expect((await mammoth.info(hash)).size).is(b.size)
|
|
90
90
|
}),
|
|
91
91
|
|
|
92
92
|
".delete": test(async() => {
|
|
93
93
|
const {mammoth} = setup()
|
|
94
|
-
const hash = await mammoth.write(blob().stream())
|
|
94
|
+
const {hash} = await mammoth.write(blob().stream())
|
|
95
95
|
await mammoth.delete(hash)
|
|
96
96
|
expect(await mammoth.has(hash)).is(false)
|
|
97
97
|
expect(await collect(mammoth.hashes())).deep([])
|
|
@@ -104,7 +104,7 @@ await science.run({
|
|
|
104
104
|
|
|
105
105
|
".keys": test(async() => {
|
|
106
106
|
const {mammoth} = setup()
|
|
107
|
-
const hash = await mammoth.write(blob().stream())
|
|
107
|
+
const {hash} = await mammoth.write(blob().stream())
|
|
108
108
|
const keys = await collect(mammoth.hashes())
|
|
109
109
|
expect(keys.length).is(1)
|
|
110
110
|
expect(keys[0]).is(hash)
|
package/x/browser/example.js
CHANGED
|
@@ -13,7 +13,7 @@ export async function example() {
|
|
|
13
13
|
const mammoth = new Mammoth(bucket, kv);
|
|
14
14
|
const myFile = new Blob([new Uint8Array([0xDE, 0xAD, 0xBE, 0xEF])]);
|
|
15
15
|
// write a file as an example
|
|
16
|
-
const hash = await mammoth.write(myFile.stream());
|
|
17
|
-
console.log(`mammoth stored file
|
|
16
|
+
const { hash } = await mammoth.write(myFile.stream());
|
|
17
|
+
console.log(`mammoth stored file "${hash}"`);
|
|
18
18
|
}
|
|
19
19
|
//# sourceMappingURL=example.js.map
|
package/x/browser/example.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"example.js","sourceRoot":"","sources":["../../s/browser/example.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAE,aAAa,EAAC,MAAM,UAAU,CAAA;AAC1C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,UAAU,EAAC,MAAM,kBAAkB,CAAA;AAE3C,MAAM,CAAC,KAAK,UAAU,OAAO;IAE5B,4BAA4B;IAC5B,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,EAAE,GAAG,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAE1C,6BAA6B;IAC7B,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,EAAE,EAAC,MAAM,EAAE,IAAI,EAAC,CAAC,CAAA;IAC1E,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;IAExC,oBAAoB;IACpB,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnE,6BAA6B;IAC7B,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACnD,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,CAAC,CAAA;AAC7C,CAAC"}
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import { hex, nap } from "@e280/stz";
|
|
2
2
|
import { blake3 } from "@noble/hashes/blake3.js";
|
|
3
3
|
import { relaxer } from "./utils/relaxer.js";
|
|
4
|
-
export async function
|
|
4
|
+
export async function analyze(readable) {
|
|
5
|
+
let size = 0;
|
|
5
6
|
const hasher = blake3.create();
|
|
6
7
|
const relax = relaxer();
|
|
7
8
|
for await (const chunk of readable) {
|
|
8
9
|
hasher.update(chunk);
|
|
10
|
+
size += chunk.byteLength;
|
|
9
11
|
if (relax())
|
|
10
12
|
await nap();
|
|
11
13
|
}
|
|
12
|
-
|
|
14
|
+
const hash = hex.fromBytes(hasher.digest());
|
|
15
|
+
return { hash, size };
|
|
13
16
|
}
|
|
14
|
-
//# sourceMappingURL=
|
|
17
|
+
//# sourceMappingURL=analyze.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../../s/core/analyze.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,QAAoC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;QAExB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
package/x/core/consts.d.ts
CHANGED
package/x/core/consts.js
CHANGED
package/x/core/consts.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,
|
|
1
|
+
{"version":3,"file":"consts.js","sourceRoot":"","sources":["../../s/core/consts.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,MAAM,CAAC,MAAM,MAAM,GAAG;IACrB,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;CACzB,CAAA"}
|
package/x/core/index.d.ts
CHANGED
package/x/core/index.js
CHANGED
package/x/core/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../s/core/index.ts"],"names":[],"mappings":"AACA,cAAc,cAAc,CAAA;AAC5B,cAAc,cAAc,CAAA;AAC5B,cAAc,oBAAoB,CAAA;AAClC,cAAc,YAAY,CAAA"}
|
package/x/core/mammoth.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { Kv } from "@e280/kv";
|
|
2
|
-
import { Hash, Bucket } from "./types.js";
|
|
2
|
+
import { Hash, Bucket, Analysis } from "./types.js";
|
|
3
3
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
4
4
|
export declare class Mammoth {
|
|
5
5
|
#private;
|
|
6
6
|
constructor(bucket?: Bucket, kv?: Kv<any>);
|
|
7
|
+
hashes(): AsyncGenerator<string, void, unknown>;
|
|
7
8
|
has(hash: Hash): Promise<boolean>;
|
|
8
9
|
info(hash: Hash): Promise<import("./types.js").Info>;
|
|
10
|
+
stats(): Promise<import("./types.js").Stats>;
|
|
9
11
|
read(hash: Hash): Promise<Blob>;
|
|
10
12
|
delete(hash: Hash): Promise<void>;
|
|
11
|
-
write(readable: ReadableStream<Uint8Array>): Promise<
|
|
12
|
-
hashes(): AsyncGenerator<string, void, unknown>;
|
|
13
|
-
stats(): Promise<import("./types.js").Stats>;
|
|
13
|
+
write(readable: ReadableStream<Uint8Array>): Promise<Analysis>;
|
|
14
14
|
}
|
package/x/core/mammoth.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Kv } from "@e280/kv";
|
|
2
2
|
import { lane, queue } from "@e280/stz";
|
|
3
3
|
import { consts } from "./consts.js";
|
|
4
|
+
import { save } from "./utils/save.js";
|
|
4
5
|
import { Manifest } from "./utils/manifest.js";
|
|
5
6
|
import { randomId } from "./utils/random-id.js";
|
|
6
7
|
import { MemoryBucket } from "./memory-bucket.js";
|
|
7
|
-
import { saveAndHash } from "./utils/save-and-hash.js";
|
|
8
8
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
9
9
|
export class Mammoth {
|
|
10
10
|
#bucket;
|
|
@@ -14,12 +14,18 @@ export class Mammoth {
|
|
|
14
14
|
this.#bucket = bucket;
|
|
15
15
|
this.#manifest = new Manifest(kv);
|
|
16
16
|
}
|
|
17
|
+
async *hashes() {
|
|
18
|
+
yield* this.#manifest.hashes();
|
|
19
|
+
}
|
|
17
20
|
async has(hash) {
|
|
18
21
|
return this.#manifest.hasHash(hash);
|
|
19
22
|
}
|
|
20
23
|
async info(hash) {
|
|
21
24
|
return this.#manifest.needInfo(hash);
|
|
22
25
|
}
|
|
26
|
+
async stats() {
|
|
27
|
+
return this.#manifest.getStats();
|
|
28
|
+
}
|
|
23
29
|
async read(hash) {
|
|
24
30
|
const { id } = await this.#manifest.needInfo(hash);
|
|
25
31
|
return this.#bucket.read(id);
|
|
@@ -27,40 +33,36 @@ export class Mammoth {
|
|
|
27
33
|
async delete(hash) {
|
|
28
34
|
await this.#wholesome(async () => {
|
|
29
35
|
const info = await this.#manifest.getInfo(hash);
|
|
30
|
-
if (info)
|
|
31
|
-
await this.#manifest.
|
|
32
|
-
await this.#bucket.delete(info.id);
|
|
33
|
-
await this.#manifest.statsRemoveFile(info.size);
|
|
34
|
-
}
|
|
36
|
+
if (info)
|
|
37
|
+
await this.#manifest.scheduleDeletion(hash, info);
|
|
35
38
|
});
|
|
39
|
+
await this.#cleanup();
|
|
36
40
|
}
|
|
37
41
|
async write(readable) {
|
|
38
42
|
const id = randomId();
|
|
39
43
|
await this.#manifest.addWip(id);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
async *hashes() {
|
|
55
|
-
yield* this.#manifest.hashes();
|
|
56
|
-
}
|
|
57
|
-
async stats() {
|
|
58
|
-
return this.#manifest.getStats();
|
|
44
|
+
try {
|
|
45
|
+
const analysis = await save(this.#bucket, id, readable);
|
|
46
|
+
const { hash, size } = analysis;
|
|
47
|
+
const info = { id, size, added: Date.now() };
|
|
48
|
+
await this.#wholesome(() => this.#manifest.commit(hash, info));
|
|
49
|
+
return analysis;
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
await this.#manifest.moveWipToTrash(id);
|
|
53
|
+
throw error;
|
|
54
|
+
}
|
|
55
|
+
finally {
|
|
56
|
+
void this.#cleanup().catch(e => console.error("cleanup failed", e));
|
|
57
|
+
}
|
|
59
58
|
}
|
|
60
|
-
#
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
await this.#manifest.
|
|
64
|
-
|
|
59
|
+
#cleanup = queue(async () => {
|
|
60
|
+
for await (const id of this.#manifest.getExpiredWipIds())
|
|
61
|
+
await this.#manifest.moveWipToTrash(id);
|
|
62
|
+
for await (const id of this.#manifest.listTrashIds()) {
|
|
63
|
+
await this.#bucket.delete(id);
|
|
64
|
+
await this.#manifest.dropTrashRecord(id);
|
|
65
|
+
}
|
|
66
|
+
}, consts.max_jobs);
|
|
65
67
|
}
|
|
66
68
|
//# sourceMappingURL=mammoth.js.map
|
package/x/core/mammoth.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AACpC,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAG/C,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/C,IAAI,IAAI;gBACP,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC9D,OAAO,QAAQ,CAAA;QAChB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;YACvC,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,KAAK,IAAG,EAAE;QAC1B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE;YACvD,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;QAExC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,EAAE,CAAC;YACtD,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YAC7B,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;QACzC,CAAC;IACF,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CACnB"}
|
package/x/core/types.d.ts
CHANGED
|
@@ -7,11 +7,18 @@ export type Bucket = {
|
|
|
7
7
|
read(id: Id): Promise<Blob>;
|
|
8
8
|
write(id: Id, readable: ReadableStream<Uint8Array>): Promise<void>;
|
|
9
9
|
};
|
|
10
|
+
/** report about a file */
|
|
11
|
+
export type Analysis = {
|
|
12
|
+
/** blake3 hash of the file's contents */
|
|
13
|
+
hash: Hash;
|
|
14
|
+
/** filesize in bytes */
|
|
15
|
+
size: number;
|
|
16
|
+
};
|
|
10
17
|
/** metadata for a single file. */
|
|
11
18
|
export type Info = {
|
|
12
19
|
/** bucket id for this file's data. */
|
|
13
20
|
id: Id;
|
|
14
|
-
/**
|
|
21
|
+
/** filesize in bytes. */
|
|
15
22
|
size: number;
|
|
16
23
|
/** when this file was added to the datalake. */
|
|
17
24
|
added: number;
|
|
@@ -3,16 +3,17 @@ import { Hash, Id, Info, Stats } from "../types.js";
|
|
|
3
3
|
export declare class Manifest {
|
|
4
4
|
#private;
|
|
5
5
|
constructor(kv?: Kv<any>);
|
|
6
|
+
getStats(): Promise<Stats>;
|
|
6
7
|
hashes(): AsyncGenerator<string, void, unknown>;
|
|
7
8
|
hasHash(hash: Hash): Promise<boolean>;
|
|
8
|
-
saveInfo(hash: Hash, info: Info): Promise<void>;
|
|
9
9
|
getInfo(hash: Hash): Promise<Info | undefined>;
|
|
10
10
|
needInfo(hash: Hash): Promise<Info>;
|
|
11
|
-
deleteInfo(hash: Hash): Promise<void>;
|
|
12
11
|
addWip(id: Id): Promise<void>;
|
|
13
12
|
deleteWip(...ids: Id[]): Promise<void>;
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
13
|
+
getExpiredWipIds(): AsyncGenerator<string, void, unknown>;
|
|
14
|
+
moveWipToTrash(id: Id): Promise<void>;
|
|
15
|
+
dropTrashRecord(id: Id): Promise<void>;
|
|
16
|
+
listTrashIds(): AsyncGenerator<string, void, unknown>;
|
|
17
|
+
scheduleDeletion(hash: Hash, info: Info): Promise<void>;
|
|
18
|
+
commit(hash: Hash, info: Info): Promise<void>;
|
|
18
19
|
}
|
package/x/core/utils/manifest.js
CHANGED
|
@@ -1,19 +1,26 @@
|
|
|
1
1
|
import { Kv } from "@e280/kv";
|
|
2
|
-
import {
|
|
3
|
-
import { consts } from "../consts.js";
|
|
2
|
+
import { got } from "@e280/stz";
|
|
4
3
|
import { notFound } from "./not-found.js";
|
|
5
4
|
import { isExpired } from "./is-expired.js";
|
|
6
5
|
export class Manifest {
|
|
6
|
+
#kv;
|
|
7
|
+
/** statistics about the whole datalake */
|
|
8
|
+
#stats;
|
|
7
9
|
/** associates file hashes with bucket ids */
|
|
8
10
|
#info;
|
|
9
11
|
/** temporary record of writes in-progress */
|
|
10
12
|
#wip;
|
|
11
13
|
/** statistics about the whole datalake */
|
|
12
|
-
#
|
|
14
|
+
#trash;
|
|
13
15
|
constructor(kv = new Kv()) {
|
|
16
|
+
this.#kv = kv;
|
|
17
|
+
this.#stats = kv.store("stats");
|
|
14
18
|
this.#info = kv.scope("info");
|
|
15
19
|
this.#wip = kv.scope("wip");
|
|
16
|
-
this.#
|
|
20
|
+
this.#trash = kv.scope("trash");
|
|
21
|
+
}
|
|
22
|
+
async getStats() {
|
|
23
|
+
return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
|
|
17
24
|
}
|
|
18
25
|
async *hashes() {
|
|
19
26
|
yield* this.#info.keys();
|
|
@@ -21,49 +28,64 @@ export class Manifest {
|
|
|
21
28
|
async hasHash(hash) {
|
|
22
29
|
return this.#info.has(hash);
|
|
23
30
|
}
|
|
24
|
-
async saveInfo(hash, info) {
|
|
25
|
-
await this.#info.set(hash, info);
|
|
26
|
-
}
|
|
27
31
|
async getInfo(hash) {
|
|
28
32
|
return await this.#info.get(hash);
|
|
29
33
|
}
|
|
30
34
|
async needInfo(hash) {
|
|
31
35
|
return got(await this.getInfo(hash), notFound(hash));
|
|
32
36
|
}
|
|
33
|
-
async deleteInfo(hash) {
|
|
34
|
-
await this.#info.del(hash);
|
|
35
|
-
}
|
|
36
37
|
async addWip(id) {
|
|
37
38
|
await this.#wip.set(id, { created: Date.now() });
|
|
38
39
|
}
|
|
39
40
|
async deleteWip(...ids) {
|
|
40
41
|
await this.#wip.del(...ids);
|
|
41
42
|
}
|
|
42
|
-
async
|
|
43
|
-
await this.#
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
43
|
+
async *getExpiredWipIds() {
|
|
44
|
+
for await (const [id, wip] of this.#wip.entries()) {
|
|
45
|
+
if (isExpired(wip))
|
|
46
|
+
yield id;
|
|
47
|
+
}
|
|
47
48
|
}
|
|
48
|
-
async
|
|
49
|
-
await this.#
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
49
|
+
async moveWipToTrash(id) {
|
|
50
|
+
await this.#kv.transaction(() => [
|
|
51
|
+
this.#wip.write.del(id),
|
|
52
|
+
this.#trash.write.set(id, true),
|
|
53
|
+
]);
|
|
53
54
|
}
|
|
54
|
-
async
|
|
55
|
-
|
|
56
|
-
return (await collect(this.#wip.entries({ limit })))
|
|
57
|
-
.filter(([, wip]) => isExpired(wip))
|
|
58
|
-
.map(([id]) => id);
|
|
55
|
+
async dropTrashRecord(id) {
|
|
56
|
+
await this.#trash.del(id);
|
|
59
57
|
}
|
|
60
|
-
async
|
|
61
|
-
|
|
58
|
+
async *listTrashIds() {
|
|
59
|
+
yield* this.#trash.keys();
|
|
62
60
|
}
|
|
63
|
-
async
|
|
61
|
+
async scheduleDeletion(hash, info) {
|
|
64
62
|
const stats = await this.getStats();
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
stats.count -= 1;
|
|
64
|
+
stats.size -= info.size;
|
|
65
|
+
await this.#kv.transaction(() => [
|
|
66
|
+
this.#info.write.del(hash),
|
|
67
|
+
this.#trash.write.set(info.id, true),
|
|
68
|
+
this.#stats.write.set(stats),
|
|
69
|
+
]);
|
|
70
|
+
}
|
|
71
|
+
async commit(hash, info) {
|
|
72
|
+
const isNewFile = !await this.hasHash(hash);
|
|
73
|
+
if (isNewFile) {
|
|
74
|
+
const stats = await this.getStats();
|
|
75
|
+
stats.count += 1;
|
|
76
|
+
stats.size += info.size;
|
|
77
|
+
await this.#kv.transaction(() => [
|
|
78
|
+
this.#wip.write.del(info.id),
|
|
79
|
+
this.#info.write.set(hash, info),
|
|
80
|
+
this.#stats.write.set(stats)
|
|
81
|
+
]);
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
await this.#kv.transaction(() => [
|
|
85
|
+
this.#wip.write.del(info.id),
|
|
86
|
+
this.#trash.write.set(info.id, true),
|
|
87
|
+
]);
|
|
88
|
+
}
|
|
67
89
|
}
|
|
68
90
|
}
|
|
69
91
|
//# sourceMappingURL=manifest.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IACpB,GAAG,CAAA;IAEH,0CAA0C;IAC1C,MAAM,CAAA;IAEN,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAA,CAAE,gBAAgB;QACtB,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,SAAS,CAAC,GAAG,CAAC;gBACjB,MAAM,EAAE,CAAA;QACV,CAAC;IACF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAM;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAM;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAA,CAAE,YAAY;QAClB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;QAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,IAAU;QAClC,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;QACH,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;aACpC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;CACD"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { hex } from "@e280/stz";
|
|
2
2
|
import { blake3 } from "@noble/hashes/blake3.js";
|
|
3
|
-
export async function
|
|
3
|
+
export async function save(bucket, id, readable) {
|
|
4
4
|
let size = 0;
|
|
5
5
|
const pipe = new TransformStream();
|
|
6
6
|
const done = bucket.write(id, pipe.readable);
|
|
@@ -16,4 +16,4 @@ export async function saveAndHash(bucket, id, readable) {
|
|
|
16
16
|
const hash = hex.fromBytes(hasher.digest());
|
|
17
17
|
return { hash, size };
|
|
18
18
|
}
|
|
19
|
-
//# sourceMappingURL=save
|
|
19
|
+
//# sourceMappingURL=save.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"save.js","sourceRoot":"","sources":["../../../s/core/utils/save.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,IAAI,CACxB,MAAc,EACd,EAAM,EACN,QAAoC;IAGrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;IACzB,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,IAAI,CAAA;IAEV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|
package/x/test.js
CHANGED
|
@@ -19,21 +19,21 @@ await science.run({
|
|
|
19
19
|
const dataDir = "data";
|
|
20
20
|
await rm(dataDir, { recursive: true, force: true });
|
|
21
21
|
const mammoth = new Mammoth(new DiskBucket(dataDir), new Kv());
|
|
22
|
-
const hash = await mammoth.write(blob().stream());
|
|
22
|
+
const { hash } = await mammoth.write(blob().stream());
|
|
23
23
|
const second = await mammoth.read(hash);
|
|
24
24
|
expect(bytes.eq(await second.bytes(), await blob().bytes()));
|
|
25
25
|
}),
|
|
26
26
|
"mammoth memory": {
|
|
27
27
|
".write and .read": test(async () => {
|
|
28
28
|
const { mammoth } = setup();
|
|
29
|
-
const hash = await mammoth.write(blob().stream());
|
|
29
|
+
const { hash } = await mammoth.write(blob().stream());
|
|
30
30
|
const second = await mammoth.read(hash);
|
|
31
31
|
expect(bytes.eq(await second.bytes(), await blob().bytes()));
|
|
32
32
|
}),
|
|
33
33
|
".write deduplication": test(async () => {
|
|
34
34
|
const { mammoth } = setup();
|
|
35
|
-
const hashA = await mammoth.write(blob().stream());
|
|
36
|
-
const hashB = await mammoth.write(blob().stream());
|
|
35
|
+
const { hash: hashA } = await mammoth.write(blob().stream());
|
|
36
|
+
const { hash: hashB } = await mammoth.write(blob().stream());
|
|
37
37
|
const keys = await collect(mammoth.hashes());
|
|
38
38
|
expect(hashA).is(hashB);
|
|
39
39
|
expect(keys.length).is(1);
|
|
@@ -41,7 +41,7 @@ await science.run({
|
|
|
41
41
|
}),
|
|
42
42
|
".write empty file": test(async () => {
|
|
43
43
|
const { mammoth } = setup();
|
|
44
|
-
const hash = await mammoth.write(new Blob().stream());
|
|
44
|
+
const { hash } = await mammoth.write(new Blob().stream());
|
|
45
45
|
expect((await mammoth.info(hash)).size).is(0);
|
|
46
46
|
expect((await mammoth.read(hash)).size).is(0);
|
|
47
47
|
}),
|
|
@@ -55,9 +55,9 @@ await science.run({
|
|
|
55
55
|
".stats": test(async () => {
|
|
56
56
|
const { mammoth } = setup();
|
|
57
57
|
expect(await mammoth.stats()).deep({ count: 0, size: 0 });
|
|
58
|
-
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
|
|
58
|
+
const { hash: hashA } = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
|
|
59
59
|
expect(await mammoth.stats()).deep({ count: 1, size: 3 });
|
|
60
|
-
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
|
|
60
|
+
const { hash: hashB } = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
|
|
61
61
|
expect(await mammoth.stats()).deep({ count: 2, size: 7 });
|
|
62
62
|
await mammoth.delete(hashA);
|
|
63
63
|
expect(await mammoth.stats()).deep({ count: 1, size: 4 });
|
|
@@ -66,19 +66,19 @@ await science.run({
|
|
|
66
66
|
}),
|
|
67
67
|
".has": test(async () => {
|
|
68
68
|
const { mammoth } = setup();
|
|
69
|
-
const hash = await mammoth.write(blob().stream());
|
|
69
|
+
const { hash } = await mammoth.write(blob().stream());
|
|
70
70
|
expect(await mammoth.has(hash)).is(true);
|
|
71
71
|
expect(await mammoth.has(randomId())).is(false);
|
|
72
72
|
}),
|
|
73
73
|
".size": test(async () => {
|
|
74
74
|
const { mammoth } = setup();
|
|
75
75
|
const b = blob();
|
|
76
|
-
const hash = await mammoth.write(b.stream());
|
|
76
|
+
const { hash } = await mammoth.write(b.stream());
|
|
77
77
|
expect((await mammoth.info(hash)).size).is(b.size);
|
|
78
78
|
}),
|
|
79
79
|
".delete": test(async () => {
|
|
80
80
|
const { mammoth } = setup();
|
|
81
|
-
const hash = await mammoth.write(blob().stream());
|
|
81
|
+
const { hash } = await mammoth.write(blob().stream());
|
|
82
82
|
await mammoth.delete(hash);
|
|
83
83
|
expect(await mammoth.has(hash)).is(false);
|
|
84
84
|
expect(await collect(mammoth.hashes())).deep([]);
|
|
@@ -89,7 +89,7 @@ await science.run({
|
|
|
89
89
|
}),
|
|
90
90
|
".keys": test(async () => {
|
|
91
91
|
const { mammoth } = setup();
|
|
92
|
-
const hash = await mammoth.write(blob().stream());
|
|
92
|
+
const { hash } = await mammoth.write(blob().stream());
|
|
93
93
|
const keys = await collect(mammoth.hashes());
|
|
94
94
|
expect(keys.length).is(1);
|
|
95
95
|
expect(keys[0]).is(hash);
|
package/x/test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC;IAEF,gBAAgB,EAAE;QACjB,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACnD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAC1D,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACvB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAC1B,CAAC,CAAC;QAEF,mBAAmB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACnC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACvD,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC7C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC;QAEF,uBAAuB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACvD,CAAC,CAAC;QAEF,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACxB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAC1E,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YAC/E,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,MAAM,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACnD,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,GAAG,IAAI,EAAE,CAAA;YAChB,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YAC9C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACzB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACnD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACjD,CAAC,CAAC;QAEF,oBAAoB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACpC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;QACjC,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,EAAC,IAAI,EAAC,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACnD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YACzB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QACzB,CAAC,CAAC;KACF;CACD,CAAC,CAAA"}
|
package/x/core/hash-stream.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export declare function hashStream(readable: ReadableStream<Uint8Array>): Promise<string>;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"hash-stream.js","sourceRoot":"","sources":["../../s/core/hash-stream.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAE,GAAG,EAAC,MAAM,WAAW,CAAA;AAClC,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAC9C,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAE1C,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAAoC;IACpE,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAG,OAAO,EAAE,CAAA;IAEvB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAEpB,IAAI,KAAK,EAAE;YACV,MAAM,GAAG,EAAE,CAAA;IACb,CAAC;IAED,OAAO,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;AACtC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"save-and-hash.js","sourceRoot":"","sources":["../../../s/core/utils/save-and-hash.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,MAAM,EAAC,MAAM,yBAAyB,CAAA;AAG9C,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,EAAM,EAAE,QAAoC;IAC7F,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAA;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAA;IAE9B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAA;QACpD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACpB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAA;IACzB,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACpB,MAAM,IAAI,CAAA;IAEV,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAA;AACpB,CAAC"}
|