@e280/mammoth 0.1.0-11 → 0.1.0-12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/s/core/mammoth.ts +16 -11
- package/s/core/manifest/fns/commit.ts +27 -0
- package/s/core/manifest/fns/get-expired-writes.ts +11 -0
- package/s/core/manifest/fns/get-stats.ts +10 -0
- package/s/core/manifest/fns/move-write-to-trash.ts +11 -0
- package/s/core/manifest/fns/need-info.ts +10 -0
- package/s/core/manifest/fns/schedule-deletion.ts +19 -0
- package/s/core/manifest/manifest.ts +26 -0
- package/s/core/types.ts +1 -1
- package/s/core/utils/cleanup.ts +7 -5
- package/s/core/utils/is-expired.ts +2 -2
- package/x/core/mammoth.js +16 -11
- package/x/core/mammoth.js.map +1 -1
- package/x/core/manifest/fns/commit.d.ts +3 -0
- package/x/core/manifest/fns/commit.js +22 -0
- package/x/core/manifest/fns/commit.js.map +1 -0
- package/x/core/manifest/fns/get-expired-writes.d.ts +2 -0
- package/x/core/manifest/fns/get-expired-writes.js +8 -0
- package/x/core/manifest/fns/get-expired-writes.js.map +1 -0
- package/x/core/manifest/fns/get-stats.d.ts +2 -0
- package/x/core/manifest/fns/get-stats.js +5 -0
- package/x/core/manifest/fns/get-stats.js.map +1 -0
- package/x/core/manifest/fns/move-write-to-trash.d.ts +3 -0
- package/x/core/manifest/fns/move-write-to-trash.js +7 -0
- package/x/core/manifest/fns/move-write-to-trash.js.map +1 -0
- package/x/core/manifest/fns/need-info.d.ts +3 -0
- package/x/core/manifest/fns/need-info.js +6 -0
- package/x/core/manifest/fns/need-info.js.map +1 -0
- package/x/core/manifest/fns/schedule-deletion.d.ts +3 -0
- package/x/core/manifest/fns/schedule-deletion.js +13 -0
- package/x/core/manifest/fns/schedule-deletion.js.map +1 -0
- package/x/core/manifest/manifest.d.ts +14 -0
- package/x/core/manifest/manifest.js +19 -0
- package/x/core/manifest/manifest.js.map +1 -0
- package/x/core/types.d.ts +1 -1
- package/x/core/utils/cleanup.d.ts +1 -1
- package/x/core/utils/cleanup.js +6 -4
- package/x/core/utils/cleanup.js.map +1 -1
- package/x/core/utils/is-expired.d.ts +2 -2
- package/x/core/utils/is-expired.js.map +1 -1
- package/s/core/utils/manifest.ts +0 -113
- package/x/core/utils/manifest.d.ts +0 -19
- package/x/core/utils/manifest.js +0 -91
- package/x/core/utils/manifest.js.map +0 -1
package/package.json
CHANGED
package/s/core/mammoth.ts
CHANGED
|
@@ -6,9 +6,14 @@ import {consts} from "./consts.js"
|
|
|
6
6
|
import {save} from "./utils/save.js"
|
|
7
7
|
import {Hash, Bucket} from "./types.js"
|
|
8
8
|
import {cleanup} from "./utils/cleanup.js"
|
|
9
|
-
import {Manifest} from "./utils/manifest.js"
|
|
10
9
|
import {randomId} from "./utils/random-id.js"
|
|
11
10
|
import {MemoryBucket} from "./memory-bucket.js"
|
|
11
|
+
import {Manifest} from "./manifest/manifest.js"
|
|
12
|
+
import {commit} from "./manifest/fns/commit.js"
|
|
13
|
+
import {needInfo} from "./manifest/fns/need-info.js"
|
|
14
|
+
import {getStats} from "./manifest/fns/get-stats.js"
|
|
15
|
+
import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
|
|
16
|
+
import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
|
|
12
17
|
|
|
13
18
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
14
19
|
export class Mammoth {
|
|
@@ -22,47 +27,47 @@ export class Mammoth {
|
|
|
22
27
|
}
|
|
23
28
|
|
|
24
29
|
async* hashes() {
|
|
25
|
-
yield* this.#manifest.
|
|
30
|
+
yield* this.#manifest.catalog.keys()
|
|
26
31
|
}
|
|
27
32
|
|
|
28
33
|
async has(hash: Hash) {
|
|
29
|
-
return this.#manifest.
|
|
34
|
+
return this.#manifest.catalog.has(hash)
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
async info(hash: Hash) {
|
|
33
|
-
return this.#manifest
|
|
38
|
+
return needInfo(this.#manifest, hash)
|
|
34
39
|
}
|
|
35
40
|
|
|
36
41
|
async stats() {
|
|
37
|
-
return this.#manifest
|
|
42
|
+
return getStats(this.#manifest)
|
|
38
43
|
}
|
|
39
44
|
|
|
40
45
|
async read(hash: Hash) {
|
|
41
|
-
const {id} = await this
|
|
46
|
+
const {id} = await this.info(hash)
|
|
42
47
|
return this.#bucket.read(id)
|
|
43
48
|
}
|
|
44
49
|
|
|
45
50
|
async delete(hash: Hash) {
|
|
46
51
|
await this.#wholesome(async() => {
|
|
47
|
-
const info = await this.#manifest.
|
|
52
|
+
const info = await this.#manifest.catalog.get(hash)
|
|
48
53
|
if (info)
|
|
49
|
-
await this.#manifest
|
|
54
|
+
await scheduleDeletion(this.#manifest, hash, info)
|
|
50
55
|
})
|
|
51
56
|
await this.#cleanup()
|
|
52
57
|
}
|
|
53
58
|
|
|
54
59
|
async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
|
|
55
60
|
const id = randomId()
|
|
56
|
-
await this.#manifest.
|
|
61
|
+
await this.#manifest.writes.set(id, {created: Date.now()})
|
|
57
62
|
try {
|
|
58
63
|
const analysis = await save(this.#bucket, id, readable)
|
|
59
64
|
const {hash, size} = analysis
|
|
60
65
|
const info = {id, size, added: Date.now()}
|
|
61
|
-
await this.#wholesome(() => this.#manifest
|
|
66
|
+
await this.#wholesome(() => commit(this.#manifest, hash, info))
|
|
62
67
|
return analysis.hash
|
|
63
68
|
}
|
|
64
69
|
catch (error) {
|
|
65
|
-
await this.#manifest
|
|
70
|
+
await moveWriteToTrash(this.#manifest, id)
|
|
66
71
|
throw error
|
|
67
72
|
}
|
|
68
73
|
finally {
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
|
|
2
|
+
import {getStats} from "./get-stats.js"
|
|
3
|
+
import {Manifest} from "../manifest.js"
|
|
4
|
+
import {Hash, Info} from "../../types.js"
|
|
5
|
+
|
|
6
|
+
export async function commit(manifest: Manifest, hash: Hash, info: Info) {
|
|
7
|
+
const {kv, stats, catalog, writes, trash} = manifest
|
|
8
|
+
const isNewFile = !await catalog.has(hash)
|
|
9
|
+
|
|
10
|
+
if (isNewFile) {
|
|
11
|
+
const s = await getStats(manifest)
|
|
12
|
+
s.count += 1
|
|
13
|
+
s.size += info.size
|
|
14
|
+
await kv.transaction(() => [
|
|
15
|
+
writes.write.del(info.id),
|
|
16
|
+
catalog.write.set(hash, info),
|
|
17
|
+
stats.write.set(s)
|
|
18
|
+
])
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
await kv.transaction(() => [
|
|
22
|
+
writes.write.del(info.id),
|
|
23
|
+
trash.write.set(info.id, true),
|
|
24
|
+
])
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
|
|
2
|
+
import {got} from "@e280/stz"
|
|
3
|
+
import {Hash} from "../../types.js"
|
|
4
|
+
import {Manifest} from "../manifest.js"
|
|
5
|
+
import {notFound} from "../../utils/not-found.js"
|
|
6
|
+
|
|
7
|
+
export async function needInfo({catalog}: Manifest, hash: Hash) {
|
|
8
|
+
return got(await catalog.get(hash), notFound(hash))
|
|
9
|
+
}
|
|
10
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
|
|
2
|
+
import {getStats} from "./get-stats.js"
|
|
3
|
+
import {Manifest} from "../manifest.js"
|
|
4
|
+
import {Hash, Info} from "../../types.js"
|
|
5
|
+
|
|
6
|
+
export async function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info) {
|
|
7
|
+
const {kv, stats, catalog, trash} = manifest
|
|
8
|
+
|
|
9
|
+
const s = await getStats(manifest)
|
|
10
|
+
s.count -= 1
|
|
11
|
+
s.size -= info.size
|
|
12
|
+
|
|
13
|
+
await kv.transaction(() => [
|
|
14
|
+
catalog.write.del(hash),
|
|
15
|
+
trash.write.set(info.id, true),
|
|
16
|
+
stats.write.set(s),
|
|
17
|
+
])
|
|
18
|
+
}
|
|
19
|
+
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
|
|
2
|
+
import {Kv} from "@e280/kv"
|
|
3
|
+
import {Info, Stats, WriteRecord} from "../types.js"
|
|
4
|
+
|
|
5
|
+
export class Manifest {
|
|
6
|
+
|
|
7
|
+
/** statistics about the whole datalake */
|
|
8
|
+
stats
|
|
9
|
+
|
|
10
|
+
/** associates file hashes with bucket ids */
|
|
11
|
+
catalog
|
|
12
|
+
|
|
13
|
+
/** temporary record of writes in-progress */
|
|
14
|
+
writes
|
|
15
|
+
|
|
16
|
+
/** bucket ids that are pending deletion */
|
|
17
|
+
trash
|
|
18
|
+
|
|
19
|
+
constructor(public kv: Kv) {
|
|
20
|
+
this.stats = kv.store<Stats>("stats")
|
|
21
|
+
this.catalog = kv.scope<Info>("catalog")
|
|
22
|
+
this.writes = kv.scope<WriteRecord>("writes")
|
|
23
|
+
this.trash = kv.scope<true>("trash")
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
package/s/core/types.ts
CHANGED
package/s/core/utils/cleanup.ts
CHANGED
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
|
|
2
2
|
import {Bucket} from "../types.js"
|
|
3
|
-
import {Manifest} from "
|
|
3
|
+
import {Manifest} from "../manifest/manifest.js"
|
|
4
|
+
import {getExpiredWrites} from "../manifest/fns/get-expired-writes.js"
|
|
5
|
+
import {moveWriteToTrash} from "../manifest/fns/move-write-to-trash.js"
|
|
4
6
|
|
|
5
7
|
export async function cleanup(bucket: Bucket, manifest: Manifest) {
|
|
6
|
-
for await (const id of manifest
|
|
7
|
-
await manifest
|
|
8
|
+
for await (const id of getExpiredWrites(manifest))
|
|
9
|
+
await moveWriteToTrash(manifest, id)
|
|
8
10
|
|
|
9
|
-
for await (const id of manifest.
|
|
11
|
+
for await (const id of manifest.trash.keys()) {
|
|
10
12
|
await bucket.delete(id)
|
|
11
|
-
await manifest.
|
|
13
|
+
await manifest.trash.del(id)
|
|
12
14
|
}
|
|
13
15
|
}
|
|
14
16
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
|
|
2
2
|
import {time} from "@e280/stz"
|
|
3
|
-
import {Wip} from "../types.js"
|
|
4
3
|
import {consts} from "../consts.js"
|
|
4
|
+
import {WriteRecord} from "../types.js"
|
|
5
5
|
|
|
6
|
-
export function isExpired({created}:
|
|
6
|
+
export function isExpired({created}: WriteRecord) {
|
|
7
7
|
const since = Date.now() - created
|
|
8
8
|
return since > time.days(consts.expiry_time)
|
|
9
9
|
}
|
package/x/core/mammoth.js
CHANGED
|
@@ -3,9 +3,14 @@ import { lane, queue } from "@e280/stz";
|
|
|
3
3
|
import { consts } from "./consts.js";
|
|
4
4
|
import { save } from "./utils/save.js";
|
|
5
5
|
import { cleanup } from "./utils/cleanup.js";
|
|
6
|
-
import { Manifest } from "./utils/manifest.js";
|
|
7
6
|
import { randomId } from "./utils/random-id.js";
|
|
8
7
|
import { MemoryBucket } from "./memory-bucket.js";
|
|
8
|
+
import { Manifest } from "./manifest/manifest.js";
|
|
9
|
+
import { commit } from "./manifest/fns/commit.js";
|
|
10
|
+
import { needInfo } from "./manifest/fns/need-info.js";
|
|
11
|
+
import { getStats } from "./manifest/fns/get-stats.js";
|
|
12
|
+
import { scheduleDeletion } from "./manifest/fns/schedule-deletion.js";
|
|
13
|
+
import { moveWriteToTrash } from "./manifest/fns/move-write-to-trash.js";
|
|
9
14
|
/** file storage datalake, content-addressed with blake3 hashes. */
|
|
10
15
|
export class Mammoth {
|
|
11
16
|
#bucket;
|
|
@@ -16,41 +21,41 @@ export class Mammoth {
|
|
|
16
21
|
this.#manifest = new Manifest(kv);
|
|
17
22
|
}
|
|
18
23
|
async *hashes() {
|
|
19
|
-
yield* this.#manifest.
|
|
24
|
+
yield* this.#manifest.catalog.keys();
|
|
20
25
|
}
|
|
21
26
|
async has(hash) {
|
|
22
|
-
return this.#manifest.
|
|
27
|
+
return this.#manifest.catalog.has(hash);
|
|
23
28
|
}
|
|
24
29
|
async info(hash) {
|
|
25
|
-
return this.#manifest
|
|
30
|
+
return needInfo(this.#manifest, hash);
|
|
26
31
|
}
|
|
27
32
|
async stats() {
|
|
28
|
-
return this.#manifest
|
|
33
|
+
return getStats(this.#manifest);
|
|
29
34
|
}
|
|
30
35
|
async read(hash) {
|
|
31
|
-
const { id } = await this
|
|
36
|
+
const { id } = await this.info(hash);
|
|
32
37
|
return this.#bucket.read(id);
|
|
33
38
|
}
|
|
34
39
|
async delete(hash) {
|
|
35
40
|
await this.#wholesome(async () => {
|
|
36
|
-
const info = await this.#manifest.
|
|
41
|
+
const info = await this.#manifest.catalog.get(hash);
|
|
37
42
|
if (info)
|
|
38
|
-
await this.#manifest
|
|
43
|
+
await scheduleDeletion(this.#manifest, hash, info);
|
|
39
44
|
});
|
|
40
45
|
await this.#cleanup();
|
|
41
46
|
}
|
|
42
47
|
async write(readable) {
|
|
43
48
|
const id = randomId();
|
|
44
|
-
await this.#manifest.
|
|
49
|
+
await this.#manifest.writes.set(id, { created: Date.now() });
|
|
45
50
|
try {
|
|
46
51
|
const analysis = await save(this.#bucket, id, readable);
|
|
47
52
|
const { hash, size } = analysis;
|
|
48
53
|
const info = { id, size, added: Date.now() };
|
|
49
|
-
await this.#wholesome(() => this.#manifest
|
|
54
|
+
await this.#wholesome(() => commit(this.#manifest, hash, info));
|
|
50
55
|
return analysis.hash;
|
|
51
56
|
}
|
|
52
57
|
catch (error) {
|
|
53
|
-
await this.#manifest
|
|
58
|
+
await moveWriteToTrash(this.#manifest, id);
|
|
54
59
|
throw error;
|
|
55
60
|
}
|
|
56
61
|
finally {
|
package/x/core/mammoth.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,
|
|
1
|
+
{"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AAEtE,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnD,IAAI,IAAI;gBACP,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;QAC1D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC/D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YAC1C,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CAC9E"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { getStats } from "./get-stats.js";
|
|
2
|
+
export async function commit(manifest, hash, info) {
|
|
3
|
+
const { kv, stats, catalog, writes, trash } = manifest;
|
|
4
|
+
const isNewFile = !await catalog.has(hash);
|
|
5
|
+
if (isNewFile) {
|
|
6
|
+
const s = await getStats(manifest);
|
|
7
|
+
s.count += 1;
|
|
8
|
+
s.size += info.size;
|
|
9
|
+
await kv.transaction(() => [
|
|
10
|
+
writes.write.del(info.id),
|
|
11
|
+
catalog.write.set(hash, info),
|
|
12
|
+
stats.write.set(s)
|
|
13
|
+
]);
|
|
14
|
+
}
|
|
15
|
+
else {
|
|
16
|
+
await kv.transaction(() => [
|
|
17
|
+
writes.write.del(info.id),
|
|
18
|
+
trash.write.set(info.id, true),
|
|
19
|
+
]);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=commit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"commit.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/commit.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IACtE,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IACpD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAClB,CAAC,CAAA;IACH,CAAC;SACI,CAAC;QACL,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAA;IACH,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-expired-writes.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-expired-writes.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAA;AAEnD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAW;IACzD,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-stats.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,KAAK,EAAW;IAC/C,OAAO,eAAe,CACrB,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;WACf,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CACvB,CAAA;AACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"move-write-to-trash.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/move-write-to-trash.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAW,EAAE,EAAM;IAC3E,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"need-info.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/need-info.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAG7B,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,OAAO,EAAW,EAAE,IAAU;IAC7D,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACpD,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { getStats } from "./get-stats.js";
|
|
2
|
+
export async function scheduleDeletion(manifest, hash, info) {
|
|
3
|
+
const { kv, stats, catalog, trash } = manifest;
|
|
4
|
+
const s = await getStats(manifest);
|
|
5
|
+
s.count -= 1;
|
|
6
|
+
s.size -= info.size;
|
|
7
|
+
await kv.transaction(() => [
|
|
8
|
+
catalog.write.del(hash),
|
|
9
|
+
trash.write.set(info.id, true),
|
|
10
|
+
stats.write.set(s),
|
|
11
|
+
]);
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=schedule-deletion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schedule-deletion.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/schedule-deletion.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IAChF,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IAE5C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;IAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;QAC9B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAClB,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Kv } from "@e280/kv";
|
|
2
|
+
import { Info, Stats, WriteRecord } from "../types.js";
|
|
3
|
+
export declare class Manifest {
|
|
4
|
+
kv: Kv;
|
|
5
|
+
/** statistics about the whole datalake */
|
|
6
|
+
stats: import("@e280/kv").Store<Stats>;
|
|
7
|
+
/** associates file hashes with bucket ids */
|
|
8
|
+
catalog: Kv<Info>;
|
|
9
|
+
/** temporary record of writes in-progress */
|
|
10
|
+
writes: Kv<WriteRecord>;
|
|
11
|
+
/** bucket ids that are pending deletion */
|
|
12
|
+
trash: Kv<true>;
|
|
13
|
+
constructor(kv: Kv);
|
|
14
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export class Manifest {
|
|
2
|
+
kv;
|
|
3
|
+
/** statistics about the whole datalake */
|
|
4
|
+
stats;
|
|
5
|
+
/** associates file hashes with bucket ids */
|
|
6
|
+
catalog;
|
|
7
|
+
/** temporary record of writes in-progress */
|
|
8
|
+
writes;
|
|
9
|
+
/** bucket ids that are pending deletion */
|
|
10
|
+
trash;
|
|
11
|
+
constructor(kv) {
|
|
12
|
+
this.kv = kv;
|
|
13
|
+
this.stats = kv.store("stats");
|
|
14
|
+
this.catalog = kv.scope("catalog");
|
|
15
|
+
this.writes = kv.scope("writes");
|
|
16
|
+
this.trash = kv.scope("trash");
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=manifest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAc,QAAQ,CAAC,CAAA;QAC7C,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}
|
package/x/core/types.d.ts
CHANGED
package/x/core/utils/cleanup.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
|
|
2
|
+
import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
|
|
1
3
|
export async function cleanup(bucket, manifest) {
|
|
2
|
-
for await (const id of manifest
|
|
3
|
-
await manifest
|
|
4
|
-
for await (const id of manifest.
|
|
4
|
+
for await (const id of getExpiredWrites(manifest))
|
|
5
|
+
await moveWriteToTrash(manifest, id);
|
|
6
|
+
for await (const id of manifest.trash.keys()) {
|
|
5
7
|
await bucket.delete(id);
|
|
6
|
-
await manifest.
|
|
8
|
+
await manifest.trash.del(id);
|
|
7
9
|
}
|
|
8
10
|
}
|
|
9
11
|
//# sourceMappingURL=cleanup.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export declare function isExpired({ created }:
|
|
1
|
+
import { WriteRecord } from "../types.js";
|
|
2
|
+
export declare function isExpired({ created }: WriteRecord): boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAc;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
|
package/s/core/utils/manifest.ts
DELETED
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
import {Kv} from "@e280/kv"
|
|
3
|
-
import {got} from "@e280/stz"
|
|
4
|
-
import {notFound} from "./not-found.js"
|
|
5
|
-
import {isExpired} from "./is-expired.js"
|
|
6
|
-
import {Hash, Id, Info, Stats, Wip} from "../types.js"
|
|
7
|
-
|
|
8
|
-
export class Manifest {
|
|
9
|
-
#kv
|
|
10
|
-
|
|
11
|
-
/** statistics about the whole datalake */
|
|
12
|
-
#stats
|
|
13
|
-
|
|
14
|
-
/** associates file hashes with bucket ids */
|
|
15
|
-
#info
|
|
16
|
-
|
|
17
|
-
/** temporary record of writes in-progress */
|
|
18
|
-
#wip
|
|
19
|
-
|
|
20
|
-
/** statistics about the whole datalake */
|
|
21
|
-
#trash
|
|
22
|
-
|
|
23
|
-
constructor(kv = new Kv()) {
|
|
24
|
-
this.#kv = kv
|
|
25
|
-
this.#stats = kv.store<Stats>("stats")
|
|
26
|
-
this.#info = kv.scope<Info>("info")
|
|
27
|
-
this.#wip = kv.scope<Wip>("wip")
|
|
28
|
-
this.#trash = kv.scope<true>("trash")
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
async getStats() {
|
|
32
|
-
return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
async* hashes() {
|
|
36
|
-
yield* this.#info.keys()
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
async hasHash(hash: Hash) {
|
|
40
|
-
return this.#info.has(hash)
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async getInfo(hash: Hash) {
|
|
44
|
-
return await this.#info.get(hash)
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
async needInfo(hash: Hash) {
|
|
48
|
-
return got(await this.getInfo(hash), notFound(hash))
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
async addWip(id: Id) {
|
|
52
|
-
await this.#wip.set(id, {created: Date.now()})
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
async deleteWip(...ids: Id[]) {
|
|
56
|
-
await this.#wip.del(...ids)
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
async* getExpiredWipIds() {
|
|
60
|
-
for await (const [id, wip] of this.#wip.entries()) {
|
|
61
|
-
if (isExpired(wip))
|
|
62
|
-
yield id
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
async moveWipToTrash(id: Id) {
|
|
67
|
-
await this.#kv.transaction(() => [
|
|
68
|
-
this.#wip.write.del(id),
|
|
69
|
-
this.#trash.write.set(id, true),
|
|
70
|
-
])
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
async dropTrashRecord(id: Id) {
|
|
74
|
-
await this.#trash.del(id)
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
async* listTrashIds() {
|
|
78
|
-
yield* this.#trash.keys()
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
async scheduleDeletion(hash: Hash, info: Info) {
|
|
82
|
-
const stats = await this.getStats()
|
|
83
|
-
stats.count -= 1
|
|
84
|
-
stats.size -= info.size
|
|
85
|
-
await this.#kv.transaction(() => [
|
|
86
|
-
this.#info.write.del(hash),
|
|
87
|
-
this.#trash.write.set(info.id, true),
|
|
88
|
-
this.#stats.write.set(stats),
|
|
89
|
-
])
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
async commit(hash: Hash, info: Info) {
|
|
93
|
-
const isNewFile = !await this.hasHash(hash)
|
|
94
|
-
|
|
95
|
-
if (isNewFile) {
|
|
96
|
-
const stats = await this.getStats()
|
|
97
|
-
stats.count += 1
|
|
98
|
-
stats.size += info.size
|
|
99
|
-
await this.#kv.transaction(() => [
|
|
100
|
-
this.#wip.write.del(info.id),
|
|
101
|
-
this.#info.write.set(hash, info),
|
|
102
|
-
this.#stats.write.set(stats)
|
|
103
|
-
])
|
|
104
|
-
}
|
|
105
|
-
else {
|
|
106
|
-
await this.#kv.transaction(() => [
|
|
107
|
-
this.#wip.write.del(info.id),
|
|
108
|
-
this.#trash.write.set(info.id, true),
|
|
109
|
-
])
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
import { Kv } from "@e280/kv";
|
|
2
|
-
import { Hash, Id, Info, Stats } from "../types.js";
|
|
3
|
-
export declare class Manifest {
|
|
4
|
-
#private;
|
|
5
|
-
constructor(kv?: Kv<any>);
|
|
6
|
-
getStats(): Promise<Stats>;
|
|
7
|
-
hashes(): AsyncGenerator<string, void, unknown>;
|
|
8
|
-
hasHash(hash: Hash): Promise<boolean>;
|
|
9
|
-
getInfo(hash: Hash): Promise<Info | undefined>;
|
|
10
|
-
needInfo(hash: Hash): Promise<Info>;
|
|
11
|
-
addWip(id: Id): Promise<void>;
|
|
12
|
-
deleteWip(...ids: Id[]): Promise<void>;
|
|
13
|
-
getExpiredWipIds(): AsyncGenerator<string, void, unknown>;
|
|
14
|
-
moveWipToTrash(id: Id): Promise<void>;
|
|
15
|
-
dropTrashRecord(id: Id): Promise<void>;
|
|
16
|
-
listTrashIds(): AsyncGenerator<string, void, unknown>;
|
|
17
|
-
scheduleDeletion(hash: Hash, info: Info): Promise<void>;
|
|
18
|
-
commit(hash: Hash, info: Info): Promise<void>;
|
|
19
|
-
}
|
package/x/core/utils/manifest.js
DELETED
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
import { Kv } from "@e280/kv";
|
|
2
|
-
import { got } from "@e280/stz";
|
|
3
|
-
import { notFound } from "./not-found.js";
|
|
4
|
-
import { isExpired } from "./is-expired.js";
|
|
5
|
-
export class Manifest {
|
|
6
|
-
#kv;
|
|
7
|
-
/** statistics about the whole datalake */
|
|
8
|
-
#stats;
|
|
9
|
-
/** associates file hashes with bucket ids */
|
|
10
|
-
#info;
|
|
11
|
-
/** temporary record of writes in-progress */
|
|
12
|
-
#wip;
|
|
13
|
-
/** statistics about the whole datalake */
|
|
14
|
-
#trash;
|
|
15
|
-
constructor(kv = new Kv()) {
|
|
16
|
-
this.#kv = kv;
|
|
17
|
-
this.#stats = kv.store("stats");
|
|
18
|
-
this.#info = kv.scope("info");
|
|
19
|
-
this.#wip = kv.scope("wip");
|
|
20
|
-
this.#trash = kv.scope("trash");
|
|
21
|
-
}
|
|
22
|
-
async getStats() {
|
|
23
|
-
return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
|
|
24
|
-
}
|
|
25
|
-
async *hashes() {
|
|
26
|
-
yield* this.#info.keys();
|
|
27
|
-
}
|
|
28
|
-
async hasHash(hash) {
|
|
29
|
-
return this.#info.has(hash);
|
|
30
|
-
}
|
|
31
|
-
async getInfo(hash) {
|
|
32
|
-
return await this.#info.get(hash);
|
|
33
|
-
}
|
|
34
|
-
async needInfo(hash) {
|
|
35
|
-
return got(await this.getInfo(hash), notFound(hash));
|
|
36
|
-
}
|
|
37
|
-
async addWip(id) {
|
|
38
|
-
await this.#wip.set(id, { created: Date.now() });
|
|
39
|
-
}
|
|
40
|
-
async deleteWip(...ids) {
|
|
41
|
-
await this.#wip.del(...ids);
|
|
42
|
-
}
|
|
43
|
-
async *getExpiredWipIds() {
|
|
44
|
-
for await (const [id, wip] of this.#wip.entries()) {
|
|
45
|
-
if (isExpired(wip))
|
|
46
|
-
yield id;
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
async moveWipToTrash(id) {
|
|
50
|
-
await this.#kv.transaction(() => [
|
|
51
|
-
this.#wip.write.del(id),
|
|
52
|
-
this.#trash.write.set(id, true),
|
|
53
|
-
]);
|
|
54
|
-
}
|
|
55
|
-
async dropTrashRecord(id) {
|
|
56
|
-
await this.#trash.del(id);
|
|
57
|
-
}
|
|
58
|
-
async *listTrashIds() {
|
|
59
|
-
yield* this.#trash.keys();
|
|
60
|
-
}
|
|
61
|
-
async scheduleDeletion(hash, info) {
|
|
62
|
-
const stats = await this.getStats();
|
|
63
|
-
stats.count -= 1;
|
|
64
|
-
stats.size -= info.size;
|
|
65
|
-
await this.#kv.transaction(() => [
|
|
66
|
-
this.#info.write.del(hash),
|
|
67
|
-
this.#trash.write.set(info.id, true),
|
|
68
|
-
this.#stats.write.set(stats),
|
|
69
|
-
]);
|
|
70
|
-
}
|
|
71
|
-
async commit(hash, info) {
|
|
72
|
-
const isNewFile = !await this.hasHash(hash);
|
|
73
|
-
if (isNewFile) {
|
|
74
|
-
const stats = await this.getStats();
|
|
75
|
-
stats.count += 1;
|
|
76
|
-
stats.size += info.size;
|
|
77
|
-
await this.#kv.transaction(() => [
|
|
78
|
-
this.#wip.write.del(info.id),
|
|
79
|
-
this.#info.write.set(hash, info),
|
|
80
|
-
this.#stats.write.set(stats)
|
|
81
|
-
]);
|
|
82
|
-
}
|
|
83
|
-
else {
|
|
84
|
-
await this.#kv.transaction(() => [
|
|
85
|
-
this.#wip.write.del(info.id),
|
|
86
|
-
this.#trash.write.set(info.id, true),
|
|
87
|
-
]);
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
//# sourceMappingURL=manifest.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IACpB,GAAG,CAAA;IAEH,0CAA0C;IAC1C,MAAM,CAAA;IAEN,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAA,CAAE,gBAAgB;QACtB,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,SAAS,CAAC,GAAG,CAAC;gBACjB,MAAM,EAAE,CAAA;QACV,CAAC;IACF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAM;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAM;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAA,CAAE,YAAY;QAClB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;QAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,IAAU;QAClC,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;QACH,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;aACpC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;CACD"}
|