@e280/mammoth 0.1.0-11 → 0.1.0-13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/package.json +1 -1
- package/s/core/mammoth.ts +17 -12
- package/s/core/manifest/fns/commit.ts +27 -0
- package/s/core/manifest/fns/get-expired-writes.ts +11 -0
- package/s/core/manifest/fns/get-stats.ts +10 -0
- package/s/core/manifest/fns/move-write-to-trash.ts +11 -0
- package/s/core/manifest/fns/need-info.ts +10 -0
- package/s/core/manifest/fns/schedule-deletion.ts +19 -0
- package/s/core/manifest/manifest.ts +26 -0
- package/s/core/types.ts +1 -1
- package/s/core/utils/cleanup.ts +7 -5
- package/s/core/utils/is-expired.ts +2 -2
- package/s/test.ts +8 -8
- package/x/core/mammoth.d.ts +1 -1
- package/x/core/mammoth.js +17 -12
- package/x/core/mammoth.js.map +1 -1
- package/x/core/manifest/fns/commit.d.ts +3 -0
- package/x/core/manifest/fns/commit.js +22 -0
- package/x/core/manifest/fns/commit.js.map +1 -0
- package/x/core/manifest/fns/get-expired-writes.d.ts +2 -0
- package/x/core/manifest/fns/get-expired-writes.js +8 -0
- package/x/core/manifest/fns/get-expired-writes.js.map +1 -0
- package/x/core/manifest/fns/get-stats.d.ts +2 -0
- package/x/core/manifest/fns/get-stats.js +5 -0
- package/x/core/manifest/fns/get-stats.js.map +1 -0
- package/x/core/manifest/fns/move-write-to-trash.d.ts +3 -0
- package/x/core/manifest/fns/move-write-to-trash.js +7 -0
- package/x/core/manifest/fns/move-write-to-trash.js.map +1 -0
- package/x/core/manifest/fns/need-info.d.ts +3 -0
- package/x/core/manifest/fns/need-info.js +6 -0
- package/x/core/manifest/fns/need-info.js.map +1 -0
- package/x/core/manifest/fns/schedule-deletion.d.ts +3 -0
- package/x/core/manifest/fns/schedule-deletion.js +13 -0
- package/x/core/manifest/fns/schedule-deletion.js.map +1 -0
- package/x/core/manifest/manifest.d.ts +14 -0
- package/x/core/manifest/manifest.js +19 -0
- package/x/core/manifest/manifest.js.map +1 -0
- package/x/core/types.d.ts +1 -1
- package/x/core/utils/cleanup.d.ts +1 -1
- package/x/core/utils/cleanup.js +6 -4
- package/x/core/utils/cleanup.js.map +1 -1
- package/x/core/utils/is-expired.d.ts +2 -2
- package/x/core/utils/is-expired.js.map +1 -1
- package/x/test.js +8 -8
- package/x/test.js.map +1 -1
- package/s/core/utils/manifest.ts +0 -113
- package/x/core/utils/manifest.d.ts +0 -19
- package/x/core/utils/manifest.js +0 -91
- package/x/core/utils/manifest.js.map +0 -1
package/README.md
CHANGED
|
@@ -87,10 +87,10 @@ const mammoth = new Mammoth()
|
|
|
87
87
|
await mammoth.stats()
|
|
88
88
|
// {count: 123, size: 123456789}
|
|
89
89
|
```
|
|
90
|
-
- **loop over
|
|
90
|
+
- **loop over every file,** with their hashes and infos.
|
|
91
91
|
```ts
|
|
92
|
-
for await (const hash of mammoth.
|
|
93
|
-
console.log(hash)
|
|
92
|
+
for await (const [hash, info] of mammoth.entries())
|
|
93
|
+
console.log(hash, info)
|
|
94
94
|
```
|
|
95
95
|
|
|
96
96
|
## 🦣 mammoth-brained tips.
|
package/package.json
CHANGED
package/s/core/mammoth.ts
CHANGED
|
@@ -6,9 +6,14 @@ import {consts} from "./consts.js"
|
|
|
6
6
|
import {save} from "./utils/save.js"
|
|
7
7
|
import {Hash, Bucket} from "./types.js"
|
|
8
8
|
import {cleanup} from "./utils/cleanup.js"
|
|
9
|
-
import {Manifest} from "./utils/manifest.js"
|
|
10
9
|
import {randomId} from "./utils/random-id.js"
|
|
11
10
|
import {MemoryBucket} from "./memory-bucket.js"
|
|
11
|
+
import {Manifest} from "./manifest/manifest.js"
|
|
12
|
+
import {commit} from "./manifest/fns/commit.js"
|
|
13
|
+
import {needInfo} from "./manifest/fns/need-info.js"
|
|
14
|
+
import {getStats} from "./manifest/fns/get-stats.js"
|
|
15
|
+
import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
|
|
16
|
+
import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
|
|
12
17
|
|
|
13
18
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
14
19
|
export class Mammoth {
|
|
@@ -21,48 +26,48 @@ export class Mammoth {
|
|
|
21
26
|
this.#manifest = new Manifest(kv)
|
|
22
27
|
}
|
|
23
28
|
|
|
24
|
-
async*
|
|
25
|
-
yield* this.#manifest.
|
|
29
|
+
async* entries() {
|
|
30
|
+
yield* this.#manifest.catalog.entries()
|
|
26
31
|
}
|
|
27
32
|
|
|
28
33
|
async has(hash: Hash) {
|
|
29
|
-
return this.#manifest.
|
|
34
|
+
return this.#manifest.catalog.has(hash)
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
async info(hash: Hash) {
|
|
33
|
-
return this.#manifest
|
|
38
|
+
return needInfo(this.#manifest, hash)
|
|
34
39
|
}
|
|
35
40
|
|
|
36
41
|
async stats() {
|
|
37
|
-
return this.#manifest
|
|
42
|
+
return getStats(this.#manifest)
|
|
38
43
|
}
|
|
39
44
|
|
|
40
45
|
async read(hash: Hash) {
|
|
41
|
-
const {id} = await this
|
|
46
|
+
const {id} = await this.info(hash)
|
|
42
47
|
return this.#bucket.read(id)
|
|
43
48
|
}
|
|
44
49
|
|
|
45
50
|
async delete(hash: Hash) {
|
|
46
51
|
await this.#wholesome(async() => {
|
|
47
|
-
const info = await this.#manifest.
|
|
52
|
+
const info = await this.#manifest.catalog.get(hash)
|
|
48
53
|
if (info)
|
|
49
|
-
await this.#manifest
|
|
54
|
+
await scheduleDeletion(this.#manifest, hash, info)
|
|
50
55
|
})
|
|
51
56
|
await this.#cleanup()
|
|
52
57
|
}
|
|
53
58
|
|
|
54
59
|
async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
|
|
55
60
|
const id = randomId()
|
|
56
|
-
await this.#manifest.
|
|
61
|
+
await this.#manifest.writes.set(id, {created: Date.now()})
|
|
57
62
|
try {
|
|
58
63
|
const analysis = await save(this.#bucket, id, readable)
|
|
59
64
|
const {hash, size} = analysis
|
|
60
65
|
const info = {id, size, added: Date.now()}
|
|
61
|
-
await this.#wholesome(() => this.#manifest
|
|
66
|
+
await this.#wholesome(() => commit(this.#manifest, hash, info))
|
|
62
67
|
return analysis.hash
|
|
63
68
|
}
|
|
64
69
|
catch (error) {
|
|
65
|
-
await this.#manifest
|
|
70
|
+
await moveWriteToTrash(this.#manifest, id)
|
|
66
71
|
throw error
|
|
67
72
|
}
|
|
68
73
|
finally {
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
|
|
2
|
+
import {getStats} from "./get-stats.js"
|
|
3
|
+
import {Manifest} from "../manifest.js"
|
|
4
|
+
import {Hash, Info} from "../../types.js"
|
|
5
|
+
|
|
6
|
+
export async function commit(manifest: Manifest, hash: Hash, info: Info) {
|
|
7
|
+
const {kv, stats, catalog, writes, trash} = manifest
|
|
8
|
+
const isNewFile = !await catalog.has(hash)
|
|
9
|
+
|
|
10
|
+
if (isNewFile) {
|
|
11
|
+
const s = await getStats(manifest)
|
|
12
|
+
s.count += 1
|
|
13
|
+
s.size += info.size
|
|
14
|
+
await kv.transaction(() => [
|
|
15
|
+
writes.write.del(info.id),
|
|
16
|
+
catalog.write.set(hash, info),
|
|
17
|
+
stats.write.set(s)
|
|
18
|
+
])
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
await kv.transaction(() => [
|
|
22
|
+
writes.write.del(info.id),
|
|
23
|
+
trash.write.set(info.id, true),
|
|
24
|
+
])
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
|
|
2
|
+
import {got} from "@e280/stz"
|
|
3
|
+
import {Hash} from "../../types.js"
|
|
4
|
+
import {Manifest} from "../manifest.js"
|
|
5
|
+
import {notFound} from "../../utils/not-found.js"
|
|
6
|
+
|
|
7
|
+
export async function needInfo({catalog}: Manifest, hash: Hash) {
|
|
8
|
+
return got(await catalog.get(hash), notFound(hash))
|
|
9
|
+
}
|
|
10
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
|
|
2
|
+
import {getStats} from "./get-stats.js"
|
|
3
|
+
import {Manifest} from "../manifest.js"
|
|
4
|
+
import {Hash, Info} from "../../types.js"
|
|
5
|
+
|
|
6
|
+
export async function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info) {
|
|
7
|
+
const {kv, stats, catalog, trash} = manifest
|
|
8
|
+
|
|
9
|
+
const s = await getStats(manifest)
|
|
10
|
+
s.count -= 1
|
|
11
|
+
s.size -= info.size
|
|
12
|
+
|
|
13
|
+
await kv.transaction(() => [
|
|
14
|
+
catalog.write.del(hash),
|
|
15
|
+
trash.write.set(info.id, true),
|
|
16
|
+
stats.write.set(s),
|
|
17
|
+
])
|
|
18
|
+
}
|
|
19
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
|
|
2
|
+
import {Kv} from "@e280/kv"
|
|
3
|
+
import {Info, Stats, WriteRecord} from "../types.js"
|
|
4
|
+
|
|
5
|
+
export class Manifest {
|
|
6
|
+
|
|
7
|
+
/** statistics about the whole datalake */
|
|
8
|
+
stats
|
|
9
|
+
|
|
10
|
+
/** associates file hashes with bucket ids */
|
|
11
|
+
catalog
|
|
12
|
+
|
|
13
|
+
/** temporary record of writes in-progress */
|
|
14
|
+
writes
|
|
15
|
+
|
|
16
|
+
/** bucket ids that are pending deletion */
|
|
17
|
+
trash
|
|
18
|
+
|
|
19
|
+
constructor(public kv: Kv) {
|
|
20
|
+
this.stats = kv.store<Stats>("stats")
|
|
21
|
+
this.catalog = kv.scope<Info>("catalog")
|
|
22
|
+
this.writes = kv.scope<WriteRecord>("writes")
|
|
23
|
+
this.trash = kv.scope<true>("trash")
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
package/s/core/types.ts
CHANGED
package/s/core/utils/cleanup.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
|
|
2
2
|
import {Bucket} from "../types.js"
|
|
3
|
-
import {Manifest} from "
|
|
3
|
+
import {Manifest} from "../manifest/manifest.js"
|
|
4
|
+
import {getExpiredWrites} from "../manifest/fns/get-expired-writes.js"
|
|
5
|
+
import {moveWriteToTrash} from "../manifest/fns/move-write-to-trash.js"
|
|
4
6
|
|
|
5
7
|
export async function cleanup(bucket: Bucket, manifest: Manifest) {
|
|
6
|
-
for await (const id of manifest
|
|
7
|
-
await manifest
|
|
8
|
+
for await (const id of getExpiredWrites(manifest))
|
|
9
|
+
await moveWriteToTrash(manifest, id)
|
|
8
10
|
|
|
9
|
-
for await (const id of manifest.
|
|
11
|
+
for await (const id of manifest.trash.keys()) {
|
|
10
12
|
await bucket.delete(id)
|
|
11
|
-
await manifest.
|
|
13
|
+
await manifest.trash.del(id)
|
|
12
14
|
}
|
|
13
15
|
}
|
|
14
16
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
|
|
2
2
|
import {time} from "@e280/stz"
|
|
3
|
-
import {Wip} from "../types.js"
|
|
4
3
|
import {consts} from "../consts.js"
|
|
4
|
+
import {WriteRecord} from "../types.js"
|
|
5
5
|
|
|
6
|
-
export function isExpired({created}:
|
|
6
|
+
export function isExpired({created}: WriteRecord) {
|
|
7
7
|
const since = Date.now() - created
|
|
8
8
|
return since > time.days(consts.expiry_time)
|
|
9
9
|
}
|
package/s/test.ts
CHANGED
|
@@ -41,10 +41,10 @@ await science.run({
|
|
|
41
41
|
const {mammoth} = setup()
|
|
42
42
|
const hashA = await mammoth.write(blob().stream())
|
|
43
43
|
const hashB = await mammoth.write(blob().stream())
|
|
44
|
-
const
|
|
44
|
+
const entries = await collect(mammoth.entries())
|
|
45
45
|
expect(hashA).is(hashB)
|
|
46
|
-
expect(
|
|
47
|
-
expect(
|
|
46
|
+
expect(entries.length).is(1)
|
|
47
|
+
expect(entries[0][0]).is(hashA)
|
|
48
48
|
}),
|
|
49
49
|
|
|
50
50
|
".write empty file": test(async() => {
|
|
@@ -59,7 +59,7 @@ await science.run({
|
|
|
59
59
|
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]))
|
|
60
60
|
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]))
|
|
61
61
|
expect(hashA).not.is(hashB)
|
|
62
|
-
expect((await collect(mammoth.
|
|
62
|
+
expect((await collect(mammoth.entries())).length).is(2)
|
|
63
63
|
}),
|
|
64
64
|
|
|
65
65
|
".stats": test(async() => {
|
|
@@ -94,7 +94,7 @@ await science.run({
|
|
|
94
94
|
const hash = await mammoth.write(blob().stream())
|
|
95
95
|
await mammoth.delete(hash)
|
|
96
96
|
expect(await mammoth.has(hash)).is(false)
|
|
97
|
-
expect(await collect(mammoth.
|
|
97
|
+
expect((await collect(mammoth.entries())).length).is(0)
|
|
98
98
|
}),
|
|
99
99
|
|
|
100
100
|
".delete idempotent": test(async() => {
|
|
@@ -105,9 +105,9 @@ await science.run({
|
|
|
105
105
|
".keys": test(async() => {
|
|
106
106
|
const {mammoth} = setup()
|
|
107
107
|
const hash = await mammoth.write(blob().stream())
|
|
108
|
-
const
|
|
109
|
-
expect(
|
|
110
|
-
expect(
|
|
108
|
+
const entries = await collect(mammoth.entries())
|
|
109
|
+
expect(entries.length).is(1)
|
|
110
|
+
expect(entries[0][0]).is(hash)
|
|
111
111
|
}),
|
|
112
112
|
},
|
|
113
113
|
})
|
package/x/core/mammoth.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { Hash, Bucket } from "./types.js";
|
|
|
4
4
|
export declare class Mammoth {
|
|
5
5
|
#private;
|
|
6
6
|
constructor(bucket?: Bucket, kv?: Kv<any>);
|
|
7
|
-
|
|
7
|
+
entries(): AsyncGenerator<[string, import("./types.js").Info], void, unknown>;
|
|
8
8
|
has(hash: Hash): Promise<boolean>;
|
|
9
9
|
info(hash: Hash): Promise<import("./types.js").Info>;
|
|
10
10
|
stats(): Promise<import("./types.js").Stats>;
|
package/x/core/mammoth.js
CHANGED
|
@@ -3,9 +3,14 @@ import { lane, queue } from "@e280/stz";
|
|
|
3
3
|
import { consts } from "./consts.js";
|
|
4
4
|
import { save } from "./utils/save.js";
|
|
5
5
|
import { cleanup } from "./utils/cleanup.js";
|
|
6
|
-
import { Manifest } from "./utils/manifest.js";
|
|
7
6
|
import { randomId } from "./utils/random-id.js";
|
|
8
7
|
import { MemoryBucket } from "./memory-bucket.js";
|
|
8
|
+
import { Manifest } from "./manifest/manifest.js";
|
|
9
|
+
import { commit } from "./manifest/fns/commit.js";
|
|
10
|
+
import { needInfo } from "./manifest/fns/need-info.js";
|
|
11
|
+
import { getStats } from "./manifest/fns/get-stats.js";
|
|
12
|
+
import { scheduleDeletion } from "./manifest/fns/schedule-deletion.js";
|
|
13
|
+
import { moveWriteToTrash } from "./manifest/fns/move-write-to-trash.js";
|
|
9
14
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
10
15
|
export class Mammoth {
|
|
11
16
|
#bucket;
|
|
@@ -15,42 +20,42 @@ export class Mammoth {
|
|
|
15
20
|
this.#bucket = bucket;
|
|
16
21
|
this.#manifest = new Manifest(kv);
|
|
17
22
|
}
|
|
18
|
-
async *
|
|
19
|
-
yield* this.#manifest.
|
|
23
|
+
async *entries() {
|
|
24
|
+
yield* this.#manifest.catalog.entries();
|
|
20
25
|
}
|
|
21
26
|
async has(hash) {
|
|
22
|
-
return this.#manifest.
|
|
27
|
+
return this.#manifest.catalog.has(hash);
|
|
23
28
|
}
|
|
24
29
|
async info(hash) {
|
|
25
|
-
return this.#manifest
|
|
30
|
+
return needInfo(this.#manifest, hash);
|
|
26
31
|
}
|
|
27
32
|
async stats() {
|
|
28
|
-
return this.#manifest
|
|
33
|
+
return getStats(this.#manifest);
|
|
29
34
|
}
|
|
30
35
|
async read(hash) {
|
|
31
|
-
const { id } = await this
|
|
36
|
+
const { id } = await this.info(hash);
|
|
32
37
|
return this.#bucket.read(id);
|
|
33
38
|
}
|
|
34
39
|
async delete(hash) {
|
|
35
40
|
await this.#wholesome(async () => {
|
|
36
|
-
const info = await this.#manifest.
|
|
41
|
+
const info = await this.#manifest.catalog.get(hash);
|
|
37
42
|
if (info)
|
|
38
|
-
await this.#manifest
|
|
43
|
+
await scheduleDeletion(this.#manifest, hash, info);
|
|
39
44
|
});
|
|
40
45
|
await this.#cleanup();
|
|
41
46
|
}
|
|
42
47
|
async write(readable) {
|
|
43
48
|
const id = randomId();
|
|
44
|
-
await this.#manifest.
|
|
49
|
+
await this.#manifest.writes.set(id, { created: Date.now() });
|
|
45
50
|
try {
|
|
46
51
|
const analysis = await save(this.#bucket, id, readable);
|
|
47
52
|
const { hash, size } = analysis;
|
|
48
53
|
const info = { id, size, added: Date.now() };
|
|
49
|
-
await this.#wholesome(() => this.#manifest
|
|
54
|
+
await this.#wholesome(() => commit(this.#manifest, hash, info));
|
|
50
55
|
return analysis.hash;
|
|
51
56
|
}
|
|
52
57
|
catch (error) {
|
|
53
|
-
await this.#manifest
|
|
58
|
+
await moveWriteToTrash(this.#manifest, id);
|
|
54
59
|
throw error;
|
|
55
60
|
}
|
|
56
61
|
finally {
|
package/x/core/mammoth.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AAEtE,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,OAAO;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnD,IAAI,IAAI;gBACP,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;QAC1D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC/D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YAC1C,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CAC9E"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { getStats } from "./get-stats.js";
|
|
2
|
+
export async function commit(manifest, hash, info) {
|
|
3
|
+
const { kv, stats, catalog, writes, trash } = manifest;
|
|
4
|
+
const isNewFile = !await catalog.has(hash);
|
|
5
|
+
if (isNewFile) {
|
|
6
|
+
const s = await getStats(manifest);
|
|
7
|
+
s.count += 1;
|
|
8
|
+
s.size += info.size;
|
|
9
|
+
await kv.transaction(() => [
|
|
10
|
+
writes.write.del(info.id),
|
|
11
|
+
catalog.write.set(hash, info),
|
|
12
|
+
stats.write.set(s)
|
|
13
|
+
]);
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
await kv.transaction(() => [
|
|
17
|
+
writes.write.del(info.id),
|
|
18
|
+
trash.write.set(info.id, true),
|
|
19
|
+
]);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=commit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"commit.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/commit.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IACtE,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IACpD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAClB,CAAC,CAAA;IACH,CAAC;SACI,CAAC;QACL,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAA;IACH,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-expired-writes.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-expired-writes.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAA;AAEnD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAW;IACzD,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-stats.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,KAAK,EAAW;IAC/C,OAAO,eAAe,CACrB,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;WACf,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CACvB,CAAA;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"move-write-to-trash.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/move-write-to-trash.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAW,EAAE,EAAM;IAC3E,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"need-info.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/need-info.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAG7B,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,OAAO,EAAW,EAAE,IAAU;IAC7D,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACpD,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { getStats } from "./get-stats.js";
|
|
2
|
+
export async function scheduleDeletion(manifest, hash, info) {
|
|
3
|
+
const { kv, stats, catalog, trash } = manifest;
|
|
4
|
+
const s = await getStats(manifest);
|
|
5
|
+
s.count -= 1;
|
|
6
|
+
s.size -= info.size;
|
|
7
|
+
await kv.transaction(() => [
|
|
8
|
+
catalog.write.del(hash),
|
|
9
|
+
trash.write.set(info.id, true),
|
|
10
|
+
stats.write.set(s),
|
|
11
|
+
]);
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=schedule-deletion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schedule-deletion.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/schedule-deletion.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IAChF,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IAE5C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;IAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;QAC9B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAClB,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Kv } from "@e280/kv";
|
|
2
|
+
import { Info, Stats, WriteRecord } from "../types.js";
|
|
3
|
+
export declare class Manifest {
|
|
4
|
+
kv: Kv;
|
|
5
|
+
/** statistics about the whole datalake */
|
|
6
|
+
stats: import("@e280/kv").Store<Stats>;
|
|
7
|
+
/** associates file hashes with bucket ids */
|
|
8
|
+
catalog: Kv<Info>;
|
|
9
|
+
/** temporary record of writes in-progress */
|
|
10
|
+
writes: Kv<WriteRecord>;
|
|
11
|
+
/** bucket ids that are pending deletion */
|
|
12
|
+
trash: Kv<true>;
|
|
13
|
+
constructor(kv: Kv);
|
|
14
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export class Manifest {
|
|
2
|
+
kv;
|
|
3
|
+
/** statistics about the whole datalake */
|
|
4
|
+
stats;
|
|
5
|
+
/** associates file hashes with bucket ids */
|
|
6
|
+
catalog;
|
|
7
|
+
/** temporary record of writes in-progress */
|
|
8
|
+
writes;
|
|
9
|
+
/** bucket ids that are pending deletion */
|
|
10
|
+
trash;
|
|
11
|
+
constructor(kv) {
|
|
12
|
+
this.kv = kv;
|
|
13
|
+
this.stats = kv.store("stats");
|
|
14
|
+
this.catalog = kv.scope("catalog");
|
|
15
|
+
this.writes = kv.scope("writes");
|
|
16
|
+
this.trash = kv.scope("trash");
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=manifest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAc,QAAQ,CAAC,CAAA;QAC7C,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}
|
package/x/core/types.d.ts
CHANGED
package/x/core/utils/cleanup.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
|
|
2
|
+
import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
|
|
1
3
|
export async function cleanup(bucket, manifest) {
|
|
2
|
-
for await (const id of manifest
|
|
3
|
-
await manifest
|
|
4
|
-
for await (const id of manifest.
|
|
4
|
+
for await (const id of getExpiredWrites(manifest))
|
|
5
|
+
await moveWriteToTrash(manifest, id);
|
|
6
|
+
for await (const id of manifest.trash.keys()) {
|
|
5
7
|
await bucket.delete(id);
|
|
6
|
-
await manifest.
|
|
8
|
+
await manifest.trash.del(id);
|
|
7
9
|
}
|
|
8
10
|
}
|
|
9
11
|
//# sourceMappingURL=cleanup.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function isExpired({ created }:
|
|
1
|
+
import { WriteRecord } from "../types.js";
|
|
2
|
+
export declare function isExpired({ created }: WriteRecord): boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAc;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
|
package/x/test.js
CHANGED
|
@@ -34,10 +34,10 @@ await science.run({
|
|
|
34
34
|
const { mammoth } = setup();
|
|
35
35
|
const hashA = await mammoth.write(blob().stream());
|
|
36
36
|
const hashB = await mammoth.write(blob().stream());
|
|
37
|
-
const
|
|
37
|
+
const entries = await collect(mammoth.entries());
|
|
38
38
|
expect(hashA).is(hashB);
|
|
39
|
-
expect(
|
|
40
|
-
expect(
|
|
39
|
+
expect(entries.length).is(1);
|
|
40
|
+
expect(entries[0][0]).is(hashA);
|
|
41
41
|
}),
|
|
42
42
|
".write empty file": test(async () => {
|
|
43
43
|
const { mammoth } = setup();
|
|
@@ -50,7 +50,7 @@ await science.run({
|
|
|
50
50
|
const hashA = await mammoth.write(quickstream([0xC0, 0xFF, 0xEE]));
|
|
51
51
|
const hashB = await mammoth.write(quickstream([0xB0, 0x0B, 0x1E, 0x5]));
|
|
52
52
|
expect(hashA).not.is(hashB);
|
|
53
|
-
expect((await collect(mammoth.
|
|
53
|
+
expect((await collect(mammoth.entries())).length).is(2);
|
|
54
54
|
}),
|
|
55
55
|
".stats": test(async () => {
|
|
56
56
|
const { mammoth } = setup();
|
|
@@ -81,7 +81,7 @@ await science.run({
|
|
|
81
81
|
const hash = await mammoth.write(blob().stream());
|
|
82
82
|
await mammoth.delete(hash);
|
|
83
83
|
expect(await mammoth.has(hash)).is(false);
|
|
84
|
-
expect(await collect(mammoth.
|
|
84
|
+
expect((await collect(mammoth.entries())).length).is(0);
|
|
85
85
|
}),
|
|
86
86
|
".delete idempotent": test(async () => {
|
|
87
87
|
const { mammoth } = setup();
|
|
@@ -90,9 +90,9 @@ await science.run({
|
|
|
90
90
|
".keys": test(async () => {
|
|
91
91
|
const { mammoth } = setup();
|
|
92
92
|
const hash = await mammoth.write(blob().stream());
|
|
93
|
-
const
|
|
94
|
-
expect(
|
|
95
|
-
expect(
|
|
93
|
+
const entries = await collect(mammoth.entries());
|
|
94
|
+
expect(entries.length).is(1);
|
|
95
|
+
expect(entries[0][0]).is(hash);
|
|
96
96
|
}),
|
|
97
97
|
},
|
|
98
98
|
});
|
package/x/test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC;IAEF,gBAAgB,EAAE;QACjB,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,
|
|
1
|
+
{"version":3,"file":"test.js","sourceRoot":"","sources":["../s/test.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,EAAE,EAAC,MAAM,aAAa,CAAA;AAC9B,OAAO,EAAC,KAAK,EAAE,OAAO,EAAC,MAAM,WAAW,CAAA;AACxC,OAAO,EAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,eAAe,CAAA;AAEnD,OAAO,EAAC,OAAO,EAAC,MAAM,mBAAmB,CAAA;AACzC,OAAO,EAAC,UAAU,EAAC,MAAM,uBAAuB,CAAA;AAChD,OAAO,EAAC,QAAQ,EAAC,MAAM,2BAA2B,CAAA;AAClD,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAA;AAEpD,MAAM,IAAI,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvE,MAAM,WAAW,GAAG,CAAC,CAAW,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAA;AAE3E,SAAS,KAAK;IACb,MAAM,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;IACnB,MAAM,MAAM,GAAG,IAAI,YAAY,EAAE,CAAA;IACjC,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACvC,OAAO,EAAC,OAAO,EAAC,CAAA;AACjB,CAAC;AAED,MAAM,OAAO,CAAC,GAAG,CAAC;IACjB,cAAc,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAA;QACtB,MAAM,EAAE,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAA;QACjD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAA;QAC9D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;IAC7D,CAAC,CAAC;IAEF,gBAAgB,EAAE;QACjB,kBAAkB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YAClC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC7D,CAAC,CAAC;QAEF,sBAAsB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAChD,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChC,CAAC,CAAC;QAEF,mBAAmB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACnC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACrD,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC7C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC;QAEF,uBAAuB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,QAAQ,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACxB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAA;YAClE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;YACvD,MAAM,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YAC3B,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,MAAM,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACtB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;QAChD,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,CAAC,GAAG,IAAI,EAAE,CAAA;YAChB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;YAC5C,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC;QAEF,SAAS,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACzB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAC1B,MAAM,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAA;YACzC,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC;QAEF,oBAAoB,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACpC,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;QACjC,CAAC,CAAC;QAEF,OAAO,EAAE,IAAI,CAAC,KAAK,IAAG,EAAE;YACvB,MAAM,EAAC,OAAO,EAAC,GAAG,KAAK,EAAE,CAAA;YACzB,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;YAChD,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAA;QAC/B,CAAC,CAAC;KACF;CACD,CAAC,CAAA"}
|
package/s/core/utils/manifest.ts
DELETED
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import {Kv} from "@e280/kv"
|
|
3
|
-
import {got} from "@e280/stz"
|
|
4
|
-
import {notFound} from "./not-found.js"
|
|
5
|
-
import {isExpired} from "./is-expired.js"
|
|
6
|
-
import {Hash, Id, Info, Stats, Wip} from "../types.js"
|
|
7
|
-
|
|
8
|
-
export class Manifest {
|
|
9
|
-
#kv
|
|
10
|
-
|
|
11
|
-
/** statistics about the whole datalake */
|
|
12
|
-
#stats
|
|
13
|
-
|
|
14
|
-
/** associates file hashes with bucket ids */
|
|
15
|
-
#info
|
|
16
|
-
|
|
17
|
-
/** temporary record of writes in-progress */
|
|
18
|
-
#wip
|
|
19
|
-
|
|
20
|
-
/** statistics about the whole datalake */
|
|
21
|
-
#trash
|
|
22
|
-
|
|
23
|
-
constructor(kv = new Kv()) {
|
|
24
|
-
this.#kv = kv
|
|
25
|
-
this.#stats = kv.store<Stats>("stats")
|
|
26
|
-
this.#info = kv.scope<Info>("info")
|
|
27
|
-
this.#wip = kv.scope<Wip>("wip")
|
|
28
|
-
this.#trash = kv.scope<true>("trash")
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
async getStats() {
|
|
32
|
-
return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
async* hashes() {
|
|
36
|
-
yield* this.#info.keys()
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
async hasHash(hash: Hash) {
|
|
40
|
-
return this.#info.has(hash)
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async getInfo(hash: Hash) {
|
|
44
|
-
return await this.#info.get(hash)
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
async needInfo(hash: Hash) {
|
|
48
|
-
return got(await this.getInfo(hash), notFound(hash))
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
async addWip(id: Id) {
|
|
52
|
-
await this.#wip.set(id, {created: Date.now()})
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
async deleteWip(...ids: Id[]) {
|
|
56
|
-
await this.#wip.del(...ids)
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
async* getExpiredWipIds() {
|
|
60
|
-
for await (const [id, wip] of this.#wip.entries()) {
|
|
61
|
-
if (isExpired(wip))
|
|
62
|
-
yield id
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
async moveWipToTrash(id: Id) {
|
|
67
|
-
await this.#kv.transaction(() => [
|
|
68
|
-
this.#wip.write.del(id),
|
|
69
|
-
this.#trash.write.set(id, true),
|
|
70
|
-
])
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
async dropTrashRecord(id: Id) {
|
|
74
|
-
await this.#trash.del(id)
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
async* listTrashIds() {
|
|
78
|
-
yield* this.#trash.keys()
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async scheduleDeletion(hash: Hash, info: Info) {
|
|
82
|
-
const stats = await this.getStats()
|
|
83
|
-
stats.count -= 1
|
|
84
|
-
stats.size -= info.size
|
|
85
|
-
await this.#kv.transaction(() => [
|
|
86
|
-
this.#info.write.del(hash),
|
|
87
|
-
this.#trash.write.set(info.id, true),
|
|
88
|
-
this.#stats.write.set(stats),
|
|
89
|
-
])
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
async commit(hash: Hash, info: Info) {
|
|
93
|
-
const isNewFile = !await this.hasHash(hash)
|
|
94
|
-
|
|
95
|
-
if (isNewFile) {
|
|
96
|
-
const stats = await this.getStats()
|
|
97
|
-
stats.count += 1
|
|
98
|
-
stats.size += info.size
|
|
99
|
-
await this.#kv.transaction(() => [
|
|
100
|
-
this.#wip.write.del(info.id),
|
|
101
|
-
this.#info.write.set(hash, info),
|
|
102
|
-
this.#stats.write.set(stats)
|
|
103
|
-
])
|
|
104
|
-
}
|
|
105
|
-
else {
|
|
106
|
-
await this.#kv.transaction(() => [
|
|
107
|
-
this.#wip.write.del(info.id),
|
|
108
|
-
this.#trash.write.set(info.id, true),
|
|
109
|
-
])
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { Kv } from "@e280/kv";
|
|
2
|
-
import { Hash, Id, Info, Stats } from "../types.js";
|
|
3
|
-
export declare class Manifest {
|
|
4
|
-
#private;
|
|
5
|
-
constructor(kv?: Kv<any>);
|
|
6
|
-
getStats(): Promise<Stats>;
|
|
7
|
-
hashes(): AsyncGenerator<string, void, unknown>;
|
|
8
|
-
hasHash(hash: Hash): Promise<boolean>;
|
|
9
|
-
getInfo(hash: Hash): Promise<Info | undefined>;
|
|
10
|
-
needInfo(hash: Hash): Promise<Info>;
|
|
11
|
-
addWip(id: Id): Promise<void>;
|
|
12
|
-
deleteWip(...ids: Id[]): Promise<void>;
|
|
13
|
-
getExpiredWipIds(): AsyncGenerator<string, void, unknown>;
|
|
14
|
-
moveWipToTrash(id: Id): Promise<void>;
|
|
15
|
-
dropTrashRecord(id: Id): Promise<void>;
|
|
16
|
-
listTrashIds(): AsyncGenerator<string, void, unknown>;
|
|
17
|
-
scheduleDeletion(hash: Hash, info: Info): Promise<void>;
|
|
18
|
-
commit(hash: Hash, info: Info): Promise<void>;
|
|
19
|
-
}
|
package/x/core/utils/manifest.js
DELETED
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import { Kv } from "@e280/kv";
|
|
2
|
-
import { got } from "@e280/stz";
|
|
3
|
-
import { notFound } from "./not-found.js";
|
|
4
|
-
import { isExpired } from "./is-expired.js";
|
|
5
|
-
export class Manifest {
|
|
6
|
-
#kv;
|
|
7
|
-
/** statistics about the whole datalake */
|
|
8
|
-
#stats;
|
|
9
|
-
/** associates file hashes with bucket ids */
|
|
10
|
-
#info;
|
|
11
|
-
/** temporary record of writes in-progress */
|
|
12
|
-
#wip;
|
|
13
|
-
/** statistics about the whole datalake */
|
|
14
|
-
#trash;
|
|
15
|
-
constructor(kv = new Kv()) {
|
|
16
|
-
this.#kv = kv;
|
|
17
|
-
this.#stats = kv.store("stats");
|
|
18
|
-
this.#info = kv.scope("info");
|
|
19
|
-
this.#wip = kv.scope("wip");
|
|
20
|
-
this.#trash = kv.scope("trash");
|
|
21
|
-
}
|
|
22
|
-
async getStats() {
|
|
23
|
-
return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
|
|
24
|
-
}
|
|
25
|
-
async *hashes() {
|
|
26
|
-
yield* this.#info.keys();
|
|
27
|
-
}
|
|
28
|
-
async hasHash(hash) {
|
|
29
|
-
return this.#info.has(hash);
|
|
30
|
-
}
|
|
31
|
-
async getInfo(hash) {
|
|
32
|
-
return await this.#info.get(hash);
|
|
33
|
-
}
|
|
34
|
-
async needInfo(hash) {
|
|
35
|
-
return got(await this.getInfo(hash), notFound(hash));
|
|
36
|
-
}
|
|
37
|
-
async addWip(id) {
|
|
38
|
-
await this.#wip.set(id, { created: Date.now() });
|
|
39
|
-
}
|
|
40
|
-
async deleteWip(...ids) {
|
|
41
|
-
await this.#wip.del(...ids);
|
|
42
|
-
}
|
|
43
|
-
async *getExpiredWipIds() {
|
|
44
|
-
for await (const [id, wip] of this.#wip.entries()) {
|
|
45
|
-
if (isExpired(wip))
|
|
46
|
-
yield id;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
async moveWipToTrash(id) {
|
|
50
|
-
await this.#kv.transaction(() => [
|
|
51
|
-
this.#wip.write.del(id),
|
|
52
|
-
this.#trash.write.set(id, true),
|
|
53
|
-
]);
|
|
54
|
-
}
|
|
55
|
-
async dropTrashRecord(id) {
|
|
56
|
-
await this.#trash.del(id);
|
|
57
|
-
}
|
|
58
|
-
async *listTrashIds() {
|
|
59
|
-
yield* this.#trash.keys();
|
|
60
|
-
}
|
|
61
|
-
async scheduleDeletion(hash, info) {
|
|
62
|
-
const stats = await this.getStats();
|
|
63
|
-
stats.count -= 1;
|
|
64
|
-
stats.size -= info.size;
|
|
65
|
-
await this.#kv.transaction(() => [
|
|
66
|
-
this.#info.write.del(hash),
|
|
67
|
-
this.#trash.write.set(info.id, true),
|
|
68
|
-
this.#stats.write.set(stats),
|
|
69
|
-
]);
|
|
70
|
-
}
|
|
71
|
-
async commit(hash, info) {
|
|
72
|
-
const isNewFile = !await this.hasHash(hash);
|
|
73
|
-
if (isNewFile) {
|
|
74
|
-
const stats = await this.getStats();
|
|
75
|
-
stats.count += 1;
|
|
76
|
-
stats.size += info.size;
|
|
77
|
-
await this.#kv.transaction(() => [
|
|
78
|
-
this.#wip.write.del(info.id),
|
|
79
|
-
this.#info.write.set(hash, info),
|
|
80
|
-
this.#stats.write.set(stats)
|
|
81
|
-
]);
|
|
82
|
-
}
|
|
83
|
-
else {
|
|
84
|
-
await this.#kv.transaction(() => [
|
|
85
|
-
this.#wip.write.del(info.id),
|
|
86
|
-
this.#trash.write.set(info.id, true),
|
|
87
|
-
]);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
//# sourceMappingURL=manifest.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IACpB,GAAG,CAAA;IAEH,0CAA0C;IAC1C,MAAM,CAAA;IAEN,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAA,CAAE,gBAAgB;QACtB,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,SAAS,CAAC,GAAG,CAAC;gBACjB,MAAM,EAAE,CAAA;QACV,CAAC;IACF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAM;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAM;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAA,CAAE,YAAY;QAClB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;QAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,IAAU;QAClC,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;QACH,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;aACpC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;CACD"}
|