@e280/mammoth 0.1.0-11 → 0.1.0-12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/package.json +1 -1
  2. package/s/core/mammoth.ts +16 -11
  3. package/s/core/manifest/fns/commit.ts +27 -0
  4. package/s/core/manifest/fns/get-expired-writes.ts +11 -0
  5. package/s/core/manifest/fns/get-stats.ts +10 -0
  6. package/s/core/manifest/fns/move-write-to-trash.ts +11 -0
  7. package/s/core/manifest/fns/need-info.ts +10 -0
  8. package/s/core/manifest/fns/schedule-deletion.ts +19 -0
  9. package/s/core/manifest/manifest.ts +26 -0
  10. package/s/core/types.ts +1 -1
  11. package/s/core/utils/cleanup.ts +7 -5
  12. package/s/core/utils/is-expired.ts +2 -2
  13. package/x/core/mammoth.js +16 -11
  14. package/x/core/mammoth.js.map +1 -1
  15. package/x/core/manifest/fns/commit.d.ts +3 -0
  16. package/x/core/manifest/fns/commit.js +22 -0
  17. package/x/core/manifest/fns/commit.js.map +1 -0
  18. package/x/core/manifest/fns/get-expired-writes.d.ts +2 -0
  19. package/x/core/manifest/fns/get-expired-writes.js +8 -0
  20. package/x/core/manifest/fns/get-expired-writes.js.map +1 -0
  21. package/x/core/manifest/fns/get-stats.d.ts +2 -0
  22. package/x/core/manifest/fns/get-stats.js +5 -0
  23. package/x/core/manifest/fns/get-stats.js.map +1 -0
  24. package/x/core/manifest/fns/move-write-to-trash.d.ts +3 -0
  25. package/x/core/manifest/fns/move-write-to-trash.js +7 -0
  26. package/x/core/manifest/fns/move-write-to-trash.js.map +1 -0
  27. package/x/core/manifest/fns/need-info.d.ts +3 -0
  28. package/x/core/manifest/fns/need-info.js +6 -0
  29. package/x/core/manifest/fns/need-info.js.map +1 -0
  30. package/x/core/manifest/fns/schedule-deletion.d.ts +3 -0
  31. package/x/core/manifest/fns/schedule-deletion.js +13 -0
  32. package/x/core/manifest/fns/schedule-deletion.js.map +1 -0
  33. package/x/core/manifest/manifest.d.ts +14 -0
  34. package/x/core/manifest/manifest.js +19 -0
  35. package/x/core/manifest/manifest.js.map +1 -0
  36. package/x/core/types.d.ts +1 -1
  37. package/x/core/utils/cleanup.d.ts +1 -1
  38. package/x/core/utils/cleanup.js +6 -4
  39. package/x/core/utils/cleanup.js.map +1 -1
  40. package/x/core/utils/is-expired.d.ts +2 -2
  41. package/x/core/utils/is-expired.js.map +1 -1
  42. package/s/core/utils/manifest.ts +0 -113
  43. package/x/core/utils/manifest.d.ts +0 -19
  44. package/x/core/utils/manifest.js +0 -91
  45. package/x/core/utils/manifest.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@e280/mammoth",
3
- "version": "0.1.0-11",
3
+ "version": "0.1.0-12",
4
4
  "description": "big file storage",
5
5
  "license": "MIT",
6
6
  "author": "Chase Moskal <chasemoskal@gmail.com>",
package/s/core/mammoth.ts CHANGED
@@ -6,9 +6,14 @@ import {consts} from "./consts.js"
6
6
  import {save} from "./utils/save.js"
7
7
  import {Hash, Bucket} from "./types.js"
8
8
  import {cleanup} from "./utils/cleanup.js"
9
- import {Manifest} from "./utils/manifest.js"
10
9
  import {randomId} from "./utils/random-id.js"
11
10
  import {MemoryBucket} from "./memory-bucket.js"
11
+ import {Manifest} from "./manifest/manifest.js"
12
+ import {commit} from "./manifest/fns/commit.js"
13
+ import {needInfo} from "./manifest/fns/need-info.js"
14
+ import {getStats} from "./manifest/fns/get-stats.js"
15
+ import {scheduleDeletion} from "./manifest/fns/schedule-deletion.js"
16
+ import {moveWriteToTrash} from "./manifest/fns/move-write-to-trash.js"
12
17
 
13
18
  /** file storage datalake, content-addressed with blake3 hashes. */
14
19
  export class Mammoth {
@@ -22,47 +27,47 @@ export class Mammoth {
22
27
  }
23
28
 
24
29
  async* hashes() {
25
- yield* this.#manifest.hashes()
30
+ yield* this.#manifest.catalog.keys()
26
31
  }
27
32
 
28
33
  async has(hash: Hash) {
29
- return this.#manifest.hasHash(hash)
34
+ return this.#manifest.catalog.has(hash)
30
35
  }
31
36
 
32
37
  async info(hash: Hash) {
33
- return this.#manifest.needInfo(hash)
38
+ return needInfo(this.#manifest, hash)
34
39
  }
35
40
 
36
41
  async stats() {
37
- return this.#manifest.getStats()
42
+ return getStats(this.#manifest)
38
43
  }
39
44
 
40
45
  async read(hash: Hash) {
41
- const {id} = await this.#manifest.needInfo(hash)
46
+ const {id} = await this.info(hash)
42
47
  return this.#bucket.read(id)
43
48
  }
44
49
 
45
50
  async delete(hash: Hash) {
46
51
  await this.#wholesome(async() => {
47
- const info = await this.#manifest.getInfo(hash)
52
+ const info = await this.#manifest.catalog.get(hash)
48
53
  if (info)
49
- await this.#manifest.scheduleDeletion(hash, info)
54
+ await scheduleDeletion(this.#manifest, hash, info)
50
55
  })
51
56
  await this.#cleanup()
52
57
  }
53
58
 
54
59
  async write(readable: ReadableStream<Uint8Array>): Promise<Hash> {
55
60
  const id = randomId()
56
- await this.#manifest.addWip(id)
61
+ await this.#manifest.writes.set(id, {created: Date.now()})
57
62
  try {
58
63
  const analysis = await save(this.#bucket, id, readable)
59
64
  const {hash, size} = analysis
60
65
  const info = {id, size, added: Date.now()}
61
- await this.#wholesome(() => this.#manifest.commit(hash, info))
66
+ await this.#wholesome(() => commit(this.#manifest, hash, info))
62
67
  return analysis.hash
63
68
  }
64
69
  catch (error) {
65
- await this.#manifest.moveWipToTrash(id)
70
+ await moveWriteToTrash(this.#manifest, id)
66
71
  throw error
67
72
  }
68
73
  finally {
@@ -0,0 +1,27 @@
1
+
2
+ import {getStats} from "./get-stats.js"
3
+ import {Manifest} from "../manifest.js"
4
+ import {Hash, Info} from "../../types.js"
5
+
6
+ export async function commit(manifest: Manifest, hash: Hash, info: Info) {
7
+ const {kv, stats, catalog, writes, trash} = manifest
8
+ const isNewFile = !await catalog.has(hash)
9
+
10
+ if (isNewFile) {
11
+ const s = await getStats(manifest)
12
+ s.count += 1
13
+ s.size += info.size
14
+ await kv.transaction(() => [
15
+ writes.write.del(info.id),
16
+ catalog.write.set(hash, info),
17
+ stats.write.set(s)
18
+ ])
19
+ }
20
+ else {
21
+ await kv.transaction(() => [
22
+ writes.write.del(info.id),
23
+ trash.write.set(info.id, true),
24
+ ])
25
+ }
26
+ }
27
+
@@ -0,0 +1,11 @@
1
+
2
+ import {Manifest} from "../manifest.js"
3
+ import {isExpired} from "../../utils/is-expired.js"
4
+
5
+ export async function* getExpiredWrites({writes}: Manifest) {
6
+ for await (const [id, w] of writes.entries()) {
7
+ if (isExpired(w))
8
+ yield id
9
+ }
10
+ }
11
+
@@ -0,0 +1,10 @@
1
+
2
+ import {Manifest} from "../manifest.js"
3
+
4
+ export async function getStats({stats}: Manifest) {
5
+ return structuredClone(
6
+ (await stats.get())
7
+ ?? {count: 0, size: 0}
8
+ )
9
+ }
10
+
@@ -0,0 +1,11 @@
1
+
2
+ import {Id} from "../../types.js"
3
+ import {Manifest} from "../manifest.js"
4
+
5
+ export async function moveWriteToTrash({kv, writes, trash}: Manifest, id: Id) {
6
+ await kv.transaction(() => [
7
+ writes.write.del(id),
8
+ trash.write.set(id, true),
9
+ ])
10
+ }
11
+
@@ -0,0 +1,10 @@
1
+
2
+ import {got} from "@e280/stz"
3
+ import {Hash} from "../../types.js"
4
+ import {Manifest} from "../manifest.js"
5
+ import {notFound} from "../../utils/not-found.js"
6
+
7
+ export async function needInfo({catalog}: Manifest, hash: Hash) {
8
+ return got(await catalog.get(hash), notFound(hash))
9
+ }
10
+
@@ -0,0 +1,19 @@
1
+
2
+ import {getStats} from "./get-stats.js"
3
+ import {Manifest} from "../manifest.js"
4
+ import {Hash, Info} from "../../types.js"
5
+
6
+ export async function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info) {
7
+ const {kv, stats, catalog, trash} = manifest
8
+
9
+ const s = await getStats(manifest)
10
+ s.count -= 1
11
+ s.size -= info.size
12
+
13
+ await kv.transaction(() => [
14
+ catalog.write.del(hash),
15
+ trash.write.set(info.id, true),
16
+ stats.write.set(s),
17
+ ])
18
+ }
19
+
@@ -0,0 +1,26 @@
1
+
2
+ import {Kv} from "@e280/kv"
3
+ import {Info, Stats, WriteRecord} from "../types.js"
4
+
5
+ export class Manifest {
6
+
7
+ /** statistics about the whole datalake */
8
+ stats
9
+
10
+ /** associates file hashes with bucket ids */
11
+ catalog
12
+
13
+ /** temporary record of writes in-progress */
14
+ writes
15
+
16
+ /** bucket ids that are pending deletion */
17
+ trash
18
+
19
+ constructor(public kv: Kv) {
20
+ this.stats = kv.store<Stats>("stats")
21
+ this.catalog = kv.scope<Info>("catalog")
22
+ this.writes = kv.scope<WriteRecord>("writes")
23
+ this.trash = kv.scope<true>("trash")
24
+ }
25
+ }
26
+
package/s/core/types.ts CHANGED
@@ -43,7 +43,7 @@ export type Stats = {
43
43
  count: number
44
44
  }
45
45
 
46
- export type Wip = {
46
+ export type WriteRecord = {
47
47
  created: number
48
48
  }
49
49
 
@@ -1,14 +1,16 @@
1
1
 
2
2
  import {Bucket} from "../types.js"
3
- import {Manifest} from "./manifest.js"
3
+ import {Manifest} from "../manifest/manifest.js"
4
+ import {getExpiredWrites} from "../manifest/fns/get-expired-writes.js"
5
+ import {moveWriteToTrash} from "../manifest/fns/move-write-to-trash.js"
4
6
 
5
7
  export async function cleanup(bucket: Bucket, manifest: Manifest) {
6
- for await (const id of manifest.getExpiredWipIds())
7
- await manifest.moveWipToTrash(id)
8
+ for await (const id of getExpiredWrites(manifest))
9
+ await moveWriteToTrash(manifest, id)
8
10
 
9
- for await (const id of manifest.listTrashIds()) {
11
+ for await (const id of manifest.trash.keys()) {
10
12
  await bucket.delete(id)
11
- await manifest.dropTrashRecord(id)
13
+ await manifest.trash.del(id)
12
14
  }
13
15
  }
14
16
 
@@ -1,9 +1,9 @@
1
1
 
2
2
  import {time} from "@e280/stz"
3
- import {Wip} from "../types.js"
4
3
  import {consts} from "../consts.js"
4
+ import {WriteRecord} from "../types.js"
5
5
 
6
- export function isExpired({created}: Wip) {
6
+ export function isExpired({created}: WriteRecord) {
7
7
  const since = Date.now() - created
8
8
  return since > time.days(consts.expiry_time)
9
9
  }
package/x/core/mammoth.js CHANGED
@@ -3,9 +3,14 @@ import { lane, queue } from "@e280/stz";
3
3
  import { consts } from "./consts.js";
4
4
  import { save } from "./utils/save.js";
5
5
  import { cleanup } from "./utils/cleanup.js";
6
- import { Manifest } from "./utils/manifest.js";
7
6
  import { randomId } from "./utils/random-id.js";
8
7
  import { MemoryBucket } from "./memory-bucket.js";
8
+ import { Manifest } from "./manifest/manifest.js";
9
+ import { commit } from "./manifest/fns/commit.js";
10
+ import { needInfo } from "./manifest/fns/need-info.js";
11
+ import { getStats } from "./manifest/fns/get-stats.js";
12
+ import { scheduleDeletion } from "./manifest/fns/schedule-deletion.js";
13
+ import { moveWriteToTrash } from "./manifest/fns/move-write-to-trash.js";
9
14
  /** file storage datalake, content-addressed with blake3 hashes. */
10
15
  export class Mammoth {
11
16
  #bucket;
@@ -16,41 +21,41 @@ export class Mammoth {
16
21
  this.#manifest = new Manifest(kv);
17
22
  }
18
23
  async *hashes() {
19
- yield* this.#manifest.hashes();
24
+ yield* this.#manifest.catalog.keys();
20
25
  }
21
26
  async has(hash) {
22
- return this.#manifest.hasHash(hash);
27
+ return this.#manifest.catalog.has(hash);
23
28
  }
24
29
  async info(hash) {
25
- return this.#manifest.needInfo(hash);
30
+ return needInfo(this.#manifest, hash);
26
31
  }
27
32
  async stats() {
28
- return this.#manifest.getStats();
33
+ return getStats(this.#manifest);
29
34
  }
30
35
  async read(hash) {
31
- const { id } = await this.#manifest.needInfo(hash);
36
+ const { id } = await this.info(hash);
32
37
  return this.#bucket.read(id);
33
38
  }
34
39
  async delete(hash) {
35
40
  await this.#wholesome(async () => {
36
- const info = await this.#manifest.getInfo(hash);
41
+ const info = await this.#manifest.catalog.get(hash);
37
42
  if (info)
38
- await this.#manifest.scheduleDeletion(hash, info);
43
+ await scheduleDeletion(this.#manifest, hash, info);
39
44
  });
40
45
  await this.#cleanup();
41
46
  }
42
47
  async write(readable) {
43
48
  const id = randomId();
44
- await this.#manifest.addWip(id);
49
+ await this.#manifest.writes.set(id, { created: Date.now() });
45
50
  try {
46
51
  const analysis = await save(this.#bucket, id, readable);
47
52
  const { hash, size } = analysis;
48
53
  const info = { id, size, added: Date.now() };
49
- await this.#wholesome(() => this.#manifest.commit(hash, info));
54
+ await this.#wholesome(() => commit(this.#manifest, hash, info));
50
55
  return analysis.hash;
51
56
  }
52
57
  catch (error) {
53
- await this.#manifest.moveWipToTrash(id);
58
+ await moveWriteToTrash(this.#manifest, id);
54
59
  throw error;
55
60
  }
56
61
  finally {
@@ -1 +1 @@
1
- {"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,qBAAqB,CAAA;AAC5C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAE/C,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAA;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IACpC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAA;IACjC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;YAC/C,IAAI,IAAI;gBACP,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAC/B,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC9D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;YACvC,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CAC9E"}
1
+ {"version":3,"file":"mammoth.js","sourceRoot":"","sources":["../../s/core/mammoth.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,IAAI,EAAE,KAAK,EAAC,MAAM,WAAW,CAAA;AAErC,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAA;AAClC,OAAO,EAAC,IAAI,EAAC,MAAM,iBAAiB,CAAA;AAEpC,OAAO,EAAC,OAAO,EAAC,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAC,QAAQ,EAAC,MAAM,sBAAsB,CAAA;AAC7C,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAA;AAC/C,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAA;AAC/C,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,QAAQ,EAAC,MAAM,6BAA6B,CAAA;AACpD,OAAO,EAAC,gBAAgB,EAAC,MAAM,qCAAqC,CAAA;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AAEtE,mEAAmE;AACnE,MAAM,OAAO,OAAO;IACnB,OAAO,CAAA;IACP,SAAS,CAAA;IACT,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAElC,YAAY,MAAM,GAAW,IAAI,YAAY,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,EAAE;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,SAAS,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAU;QACnB,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,KAAK;QACV,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAU;QACpB,MAAM,EAAC,EAAE,EAAC,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU;QACtB,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,IAAG,EAAE;YAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YACnD,IAAI,IAAI;gBACP,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;IACtB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,QAAoC;QAC/C,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAA;QACrB,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;QAC1D,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,CAAA;YACvD,MAAM,EAAC,IAAI,EAAE,IAAI,EAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,IAAI,GAAG,EAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAA;YAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;YAC/D,OAAO,QAAQ,CAAC,IAAI,CAAA;QACrB,CAAC;QACD,OAAO,KAAK,EAAE,CAAC;YACd,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;YAC1C,MAAM,KAAK,CAAA;QACZ,CAAC;gBACO,CAAC;YACR,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,QAAQ,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;CAC9E"}
@@ -0,0 +1,3 @@
1
+ import { Manifest } from "../manifest.js";
2
+ import { Hash, Info } from "../../types.js";
3
+ export declare function commit(manifest: Manifest, hash: Hash, info: Info): Promise<void>;
@@ -0,0 +1,22 @@
1
+ import { getStats } from "./get-stats.js";
2
+ export async function commit(manifest, hash, info) {
3
+ const { kv, stats, catalog, writes, trash } = manifest;
4
+ const isNewFile = !await catalog.has(hash);
5
+ if (isNewFile) {
6
+ const s = await getStats(manifest);
7
+ s.count += 1;
8
+ s.size += info.size;
9
+ await kv.transaction(() => [
10
+ writes.write.del(info.id),
11
+ catalog.write.set(hash, info),
12
+ stats.write.set(s)
13
+ ]);
14
+ }
15
+ else {
16
+ await kv.transaction(() => [
17
+ writes.write.del(info.id),
18
+ trash.write.set(info.id, true),
19
+ ]);
20
+ }
21
+ }
22
+ //# sourceMappingURL=commit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"commit.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/commit.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IACtE,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IACpD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;QACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;YAC7B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;SAClB,CAAC,CAAA;IACH,CAAC;SACI,CAAC;QACL,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAA;IACH,CAAC;AACF,CAAC"}
@@ -0,0 +1,2 @@
1
+ import { Manifest } from "../manifest.js";
2
+ export declare function getExpiredWrites({ writes }: Manifest): AsyncGenerator<string, void, unknown>;
@@ -0,0 +1,8 @@
1
+ import { isExpired } from "../../utils/is-expired.js";
2
+ export async function* getExpiredWrites({ writes }) {
3
+ for await (const [id, w] of writes.entries()) {
4
+ if (isExpired(w))
5
+ yield id;
6
+ }
7
+ }
8
+ //# sourceMappingURL=get-expired-writes.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"get-expired-writes.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-expired-writes.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,2BAA2B,CAAA;AAEnD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,EAAC,MAAM,EAAW;IACzD,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,SAAS,CAAC,CAAC,CAAC;YACf,MAAM,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}
@@ -0,0 +1,2 @@
1
+ import { Manifest } from "../manifest.js";
2
+ export declare function getStats({ stats }: Manifest): Promise<import("../../types.js").Stats>;
@@ -0,0 +1,5 @@
1
+ export async function getStats({ stats }) {
2
+ return structuredClone((await stats.get())
3
+ ?? { count: 0, size: 0 });
4
+ }
5
+ //# sourceMappingURL=get-stats.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"get-stats.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/get-stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,KAAK,EAAW;IAC/C,OAAO,eAAe,CACrB,CAAC,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;WACf,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CACvB,CAAA;AACF,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Id } from "../../types.js";
2
+ import { Manifest } from "../manifest.js";
3
+ export declare function moveWriteToTrash({ kv, writes, trash }: Manifest, id: Id): Promise<void>;
@@ -0,0 +1,7 @@
1
+ export async function moveWriteToTrash({ kv, writes, trash }, id) {
2
+ await kv.transaction(() => [
3
+ writes.write.del(id),
4
+ trash.write.set(id, true),
5
+ ]);
6
+ }
7
+ //# sourceMappingURL=move-write-to-trash.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"move-write-to-trash.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/move-write-to-trash.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAW,EAAE,EAAM;IAC3E,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;KACzB,CAAC,CAAA;AACH,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Hash } from "../../types.js";
2
+ import { Manifest } from "../manifest.js";
3
+ export declare function needInfo({ catalog }: Manifest, hash: Hash): Promise<import("../../types.js").Info>;
@@ -0,0 +1,6 @@
1
+ import { got } from "@e280/stz";
2
+ import { notFound } from "../../utils/not-found.js";
3
+ export async function needInfo({ catalog }, hash) {
4
+ return got(await catalog.get(hash), notFound(hash));
5
+ }
6
+ //# sourceMappingURL=need-info.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"need-info.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/need-info.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAG7B,OAAO,EAAC,QAAQ,EAAC,MAAM,0BAA0B,CAAA;AAEjD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAC,OAAO,EAAW,EAAE,IAAU;IAC7D,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;AACpD,CAAC"}
@@ -0,0 +1,3 @@
1
+ import { Manifest } from "../manifest.js";
2
+ import { Hash, Info } from "../../types.js";
3
+ export declare function scheduleDeletion(manifest: Manifest, hash: Hash, info: Info): Promise<void>;
@@ -0,0 +1,13 @@
1
+ import { getStats } from "./get-stats.js";
2
+ export async function scheduleDeletion(manifest, hash, info) {
3
+ const { kv, stats, catalog, trash } = manifest;
4
+ const s = await getStats(manifest);
5
+ s.count -= 1;
6
+ s.size -= info.size;
7
+ await kv.transaction(() => [
8
+ catalog.write.del(hash),
9
+ trash.write.set(info.id, true),
10
+ stats.write.set(s),
11
+ ]);
12
+ }
13
+ //# sourceMappingURL=schedule-deletion.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"schedule-deletion.js","sourceRoot":"","sources":["../../../../s/core/manifest/fns/schedule-deletion.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AAIvC,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,IAAU,EAAE,IAAU;IAChF,MAAM,EAAC,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAC,GAAG,QAAQ,CAAA;IAE5C,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;IAClC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAA;IACZ,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;IAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;QAC9B,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;KAClB,CAAC,CAAA;AACH,CAAC"}
@@ -0,0 +1,14 @@
1
+ import { Kv } from "@e280/kv";
2
+ import { Info, Stats, WriteRecord } from "../types.js";
3
+ export declare class Manifest {
4
+ kv: Kv;
5
+ /** statistics about the whole datalake */
6
+ stats: import("@e280/kv").Store<Stats>;
7
+ /** associates file hashes with bucket ids */
8
+ catalog: Kv<Info>;
9
+ /** temporary record of writes in-progress */
10
+ writes: Kv<WriteRecord>;
11
+ /** bucket ids that are pending deletion */
12
+ trash: Kv<true>;
13
+ constructor(kv: Kv);
14
+ }
@@ -0,0 +1,19 @@
1
+ export class Manifest {
2
+ kv;
3
+ /** statistics about the whole datalake */
4
+ stats;
5
+ /** associates file hashes with bucket ids */
6
+ catalog;
7
+ /** temporary record of writes in-progress */
8
+ writes;
9
+ /** bucket ids that are pending deletion */
10
+ trash;
11
+ constructor(kv) {
12
+ this.kv = kv;
13
+ this.stats = kv.store("stats");
14
+ this.catalog = kv.scope("catalog");
15
+ this.writes = kv.scope("writes");
16
+ this.trash = kv.scope("trash");
17
+ }
18
+ }
19
+ //# sourceMappingURL=manifest.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/manifest/manifest.ts"],"names":[],"mappings":"AAIA,MAAM,OAAO,QAAQ;IAcD,EAAE;IAZrB,0CAA0C;IAC1C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,OAAO,CAAA;IAEP,6CAA6C;IAC7C,MAAM,CAAA;IAEN,2CAA2C;IAC3C,KAAK,CAAA;IAEL,YAAmB,EAAM;kBAAN,EAAE;QACpB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,KAAK,CAAO,SAAS,CAAC,CAAA;QACxC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAc,QAAQ,CAAC,CAAA;QAC7C,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACrC,CAAC;CACD"}
package/x/core/types.d.ts CHANGED
@@ -30,6 +30,6 @@ export type Stats = {
30
30
  /** total number of files in the datalake. */
31
31
  count: number;
32
32
  };
33
- export type Wip = {
33
+ export type WriteRecord = {
34
34
  created: number;
35
35
  };
@@ -1,3 +1,3 @@
1
1
  import { Bucket } from "../types.js";
2
- import { Manifest } from "./manifest.js";
2
+ import { Manifest } from "../manifest/manifest.js";
3
3
  export declare function cleanup(bucket: Bucket, manifest: Manifest): Promise<void>;
@@ -1,9 +1,11 @@
1
+ import { getExpiredWrites } from "../manifest/fns/get-expired-writes.js";
2
+ import { moveWriteToTrash } from "../manifest/fns/move-write-to-trash.js";
1
3
  export async function cleanup(bucket, manifest) {
2
- for await (const id of manifest.getExpiredWipIds())
3
- await manifest.moveWipToTrash(id);
4
- for await (const id of manifest.listTrashIds()) {
4
+ for await (const id of getExpiredWrites(manifest))
5
+ await moveWriteToTrash(manifest, id);
6
+ for await (const id of manifest.trash.keys()) {
5
7
  await bucket.delete(id);
6
- await manifest.dropTrashRecord(id);
8
+ await manifest.trash.del(id);
7
9
  }
8
10
  }
9
11
  //# sourceMappingURL=cleanup.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAIA,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,gBAAgB,EAAE;QACjD,MAAM,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC,CAAA;IAElC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC;QAChD,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;IACnC,CAAC;AACF,CAAC"}
1
+ {"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../s/core/utils/cleanup.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,gBAAgB,EAAC,MAAM,uCAAuC,CAAA;AACtE,OAAO,EAAC,gBAAgB,EAAC,MAAM,wCAAwC,CAAA;AAEvE,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,MAAc,EAAE,QAAkB;IAC/D,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC;QAChD,MAAM,gBAAgB,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QACvB,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC7B,CAAC;AACF,CAAC"}
@@ -1,2 +1,2 @@
1
- import { Wip } from "../types.js";
2
- export declare function isExpired({ created }: Wip): boolean;
1
+ import { WriteRecord } from "../types.js";
2
+ export declare function isExpired({ created }: WriteRecord): boolean;
@@ -1 +1 @@
1
- {"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAE9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAEnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAM;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
1
+ {"version":3,"file":"is-expired.js","sourceRoot":"","sources":["../../../s/core/utils/is-expired.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,IAAI,EAAC,MAAM,WAAW,CAAA;AAC9B,OAAO,EAAC,MAAM,EAAC,MAAM,cAAc,CAAA;AAGnC,MAAM,UAAU,SAAS,CAAC,EAAC,OAAO,EAAc;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAA;IAClC,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;AAC7C,CAAC"}
@@ -1,113 +0,0 @@
1
-
2
- import {Kv} from "@e280/kv"
3
- import {got} from "@e280/stz"
4
- import {notFound} from "./not-found.js"
5
- import {isExpired} from "./is-expired.js"
6
- import {Hash, Id, Info, Stats, Wip} from "../types.js"
7
-
8
- export class Manifest {
9
- #kv
10
-
11
- /** statistics about the whole datalake */
12
- #stats
13
-
14
- /** associates file hashes with bucket ids */
15
- #info
16
-
17
- /** temporary record of writes in-progress */
18
- #wip
19
-
20
- /** statistics about the whole datalake */
21
- #trash
22
-
23
- constructor(kv = new Kv()) {
24
- this.#kv = kv
25
- this.#stats = kv.store<Stats>("stats")
26
- this.#info = kv.scope<Info>("info")
27
- this.#wip = kv.scope<Wip>("wip")
28
- this.#trash = kv.scope<true>("trash")
29
- }
30
-
31
- async getStats() {
32
- return structuredClone((await this.#stats.get()) ?? {count: 0, size: 0})
33
- }
34
-
35
- async* hashes() {
36
- yield* this.#info.keys()
37
- }
38
-
39
- async hasHash(hash: Hash) {
40
- return this.#info.has(hash)
41
- }
42
-
43
- async getInfo(hash: Hash) {
44
- return await this.#info.get(hash)
45
- }
46
-
47
- async needInfo(hash: Hash) {
48
- return got(await this.getInfo(hash), notFound(hash))
49
- }
50
-
51
- async addWip(id: Id) {
52
- await this.#wip.set(id, {created: Date.now()})
53
- }
54
-
55
- async deleteWip(...ids: Id[]) {
56
- await this.#wip.del(...ids)
57
- }
58
-
59
- async* getExpiredWipIds() {
60
- for await (const [id, wip] of this.#wip.entries()) {
61
- if (isExpired(wip))
62
- yield id
63
- }
64
- }
65
-
66
- async moveWipToTrash(id: Id) {
67
- await this.#kv.transaction(() => [
68
- this.#wip.write.del(id),
69
- this.#trash.write.set(id, true),
70
- ])
71
- }
72
-
73
- async dropTrashRecord(id: Id) {
74
- await this.#trash.del(id)
75
- }
76
-
77
- async* listTrashIds() {
78
- yield* this.#trash.keys()
79
- }
80
-
81
- async scheduleDeletion(hash: Hash, info: Info) {
82
- const stats = await this.getStats()
83
- stats.count -= 1
84
- stats.size -= info.size
85
- await this.#kv.transaction(() => [
86
- this.#info.write.del(hash),
87
- this.#trash.write.set(info.id, true),
88
- this.#stats.write.set(stats),
89
- ])
90
- }
91
-
92
- async commit(hash: Hash, info: Info) {
93
- const isNewFile = !await this.hasHash(hash)
94
-
95
- if (isNewFile) {
96
- const stats = await this.getStats()
97
- stats.count += 1
98
- stats.size += info.size
99
- await this.#kv.transaction(() => [
100
- this.#wip.write.del(info.id),
101
- this.#info.write.set(hash, info),
102
- this.#stats.write.set(stats)
103
- ])
104
- }
105
- else {
106
- await this.#kv.transaction(() => [
107
- this.#wip.write.del(info.id),
108
- this.#trash.write.set(info.id, true),
109
- ])
110
- }
111
- }
112
- }
113
-
@@ -1,19 +0,0 @@
1
- import { Kv } from "@e280/kv";
2
- import { Hash, Id, Info, Stats } from "../types.js";
3
- export declare class Manifest {
4
- #private;
5
- constructor(kv?: Kv<any>);
6
- getStats(): Promise<Stats>;
7
- hashes(): AsyncGenerator<string, void, unknown>;
8
- hasHash(hash: Hash): Promise<boolean>;
9
- getInfo(hash: Hash): Promise<Info | undefined>;
10
- needInfo(hash: Hash): Promise<Info>;
11
- addWip(id: Id): Promise<void>;
12
- deleteWip(...ids: Id[]): Promise<void>;
13
- getExpiredWipIds(): AsyncGenerator<string, void, unknown>;
14
- moveWipToTrash(id: Id): Promise<void>;
15
- dropTrashRecord(id: Id): Promise<void>;
16
- listTrashIds(): AsyncGenerator<string, void, unknown>;
17
- scheduleDeletion(hash: Hash, info: Info): Promise<void>;
18
- commit(hash: Hash, info: Info): Promise<void>;
19
- }
@@ -1,91 +0,0 @@
1
- import { Kv } from "@e280/kv";
2
- import { got } from "@e280/stz";
3
- import { notFound } from "./not-found.js";
4
- import { isExpired } from "./is-expired.js";
5
- export class Manifest {
6
- #kv;
7
- /** statistics about the whole datalake */
8
- #stats;
9
- /** associates file hashes with bucket ids */
10
- #info;
11
- /** temporary record of writes in-progress */
12
- #wip;
13
- /** statistics about the whole datalake */
14
- #trash;
15
- constructor(kv = new Kv()) {
16
- this.#kv = kv;
17
- this.#stats = kv.store("stats");
18
- this.#info = kv.scope("info");
19
- this.#wip = kv.scope("wip");
20
- this.#trash = kv.scope("trash");
21
- }
22
- async getStats() {
23
- return structuredClone((await this.#stats.get()) ?? { count: 0, size: 0 });
24
- }
25
- async *hashes() {
26
- yield* this.#info.keys();
27
- }
28
- async hasHash(hash) {
29
- return this.#info.has(hash);
30
- }
31
- async getInfo(hash) {
32
- return await this.#info.get(hash);
33
- }
34
- async needInfo(hash) {
35
- return got(await this.getInfo(hash), notFound(hash));
36
- }
37
- async addWip(id) {
38
- await this.#wip.set(id, { created: Date.now() });
39
- }
40
- async deleteWip(...ids) {
41
- await this.#wip.del(...ids);
42
- }
43
- async *getExpiredWipIds() {
44
- for await (const [id, wip] of this.#wip.entries()) {
45
- if (isExpired(wip))
46
- yield id;
47
- }
48
- }
49
- async moveWipToTrash(id) {
50
- await this.#kv.transaction(() => [
51
- this.#wip.write.del(id),
52
- this.#trash.write.set(id, true),
53
- ]);
54
- }
55
- async dropTrashRecord(id) {
56
- await this.#trash.del(id);
57
- }
58
- async *listTrashIds() {
59
- yield* this.#trash.keys();
60
- }
61
- async scheduleDeletion(hash, info) {
62
- const stats = await this.getStats();
63
- stats.count -= 1;
64
- stats.size -= info.size;
65
- await this.#kv.transaction(() => [
66
- this.#info.write.del(hash),
67
- this.#trash.write.set(info.id, true),
68
- this.#stats.write.set(stats),
69
- ]);
70
- }
71
- async commit(hash, info) {
72
- const isNewFile = !await this.hasHash(hash);
73
- if (isNewFile) {
74
- const stats = await this.getStats();
75
- stats.count += 1;
76
- stats.size += info.size;
77
- await this.#kv.transaction(() => [
78
- this.#wip.write.del(info.id),
79
- this.#info.write.set(hash, info),
80
- this.#stats.write.set(stats)
81
- ]);
82
- }
83
- else {
84
- await this.#kv.transaction(() => [
85
- this.#wip.write.del(info.id),
86
- this.#trash.write.set(info.id, true),
87
- ]);
88
- }
89
- }
90
- }
91
- //# sourceMappingURL=manifest.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../../s/core/utils/manifest.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,EAAE,EAAC,MAAM,UAAU,CAAA;AAC3B,OAAO,EAAC,GAAG,EAAC,MAAM,WAAW,CAAA;AAC7B,OAAO,EAAC,QAAQ,EAAC,MAAM,gBAAgB,CAAA;AACvC,OAAO,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAA;AAGzC,MAAM,OAAO,QAAQ;IACpB,GAAG,CAAA;IAEH,0CAA0C;IAC1C,MAAM,CAAA;IAEN,6CAA6C;IAC7C,KAAK,CAAA;IAEL,6CAA6C;IAC7C,IAAI,CAAA;IAEJ,0CAA0C;IAC1C,MAAM,CAAA;IAEN,YAAY,EAAE,GAAG,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC,GAAG,GAAG,EAAE,CAAA;QACb,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAQ,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAO,MAAM,CAAC,CAAA;QACnC,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC,KAAK,CAAM,KAAK,CAAC,CAAA;QAChC,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC,KAAK,CAAO,OAAO,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ;QACb,OAAO,eAAe,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAC,CAAC,CAAA;IACzE,CAAC;IAED,KAAK,CAAA,CAAE,MAAM;QACZ,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;IACzB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAU;QACvB,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAU;QACxB,OAAO,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;IACrD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAM;QAClB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,EAAC,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAA;IAC/C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAG,GAAS;QAC3B,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,KAAK,CAAA,CAAE,gBAAgB;QACtB,IAAI,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACnD,IAAI,SAAS,CAAC,GAAG,CAAC;gBACjB,MAAM,EAAE,CAAA;QACV,CAAC;IACF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,EAAM;QAC1B,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC;SAC/B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,EAAM;QAC3B,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,CAAA,CAAE,YAAY;QAClB,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,IAAU,EAAE,IAAU;QAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;QACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;QAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;QACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;YACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;SAC5B,CAAC,CAAA;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAU,EAAE,IAAU;QAClC,MAAM,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAE3C,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAA;YACnC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAA;YAChB,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAA;YACvB,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC;gBAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;QACH,CAAC;aACI,CAAC;YACL,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC;aACpC,CAAC,CAAA;QACH,CAAC;IACF,CAAC;CACD"}