@pithy-sh/storage 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/capability.d.ts +43 -0
  2. package/dist/capability.d.ts.map +1 -0
  3. package/dist/capability.js +74 -0
  4. package/dist/config/config.d.ts +25 -0
  5. package/dist/config/config.d.ts.map +1 -0
  6. package/dist/config/config.js +47 -0
  7. package/dist/data/share.d.ts +25 -0
  8. package/dist/data/share.d.ts.map +1 -0
  9. package/dist/data/share.js +26 -0
  10. package/dist/data/storageObject.d.ts +53 -0
  11. package/dist/data/storageObject.d.ts.map +1 -0
  12. package/dist/data/storageObject.js +42 -0
  13. package/dist/data/tables.d.ts +21 -0
  14. package/dist/data/tables.d.ts.map +1 -0
  15. package/dist/data/tables.js +24 -0
  16. package/dist/error/errors.d.ts +58 -0
  17. package/dist/error/errors.d.ts.map +1 -0
  18. package/dist/error/errors.js +87 -0
  19. package/dist/http/guard.d.ts +18 -0
  20. package/dist/http/guard.d.ts.map +1 -0
  21. package/dist/http/guard.js +27 -0
  22. package/dist/http/handlers.d.ts +230 -0
  23. package/dist/http/handlers.d.ts.map +1 -0
  24. package/dist/http/handlers.js +505 -0
  25. package/dist/http/routes.d.ts +68 -0
  26. package/dist/http/routes.d.ts.map +1 -0
  27. package/dist/http/routes.js +133 -0
  28. package/dist/http/schemas.d.ts +72 -0
  29. package/dist/http/schemas.d.ts.map +1 -0
  30. package/dist/http/schemas.js +91 -0
  31. package/dist/http/serve.d.ts +119 -0
  32. package/dist/http/serve.d.ts.map +1 -0
  33. package/dist/http/serve.js +207 -0
  34. package/dist/index.d.ts +21 -0
  35. package/dist/index.d.ts.map +1 -0
  36. package/dist/index.js +12 -0
  37. package/dist/migrations/0001_objects.d.ts +22 -0
  38. package/dist/migrations/0001_objects.d.ts.map +1 -0
  39. package/dist/migrations/0001_objects.js +39 -0
  40. package/dist/object/cloudflare.d.ts +21 -0
  41. package/dist/object/cloudflare.d.ts.map +1 -0
  42. package/dist/object/cloudflare.js +46 -0
  43. package/dist/object/key.d.ts +27 -0
  44. package/dist/object/key.d.ts.map +1 -0
  45. package/dist/object/key.js +35 -0
  46. package/dist/object/multipart.d.ts +76 -0
  47. package/dist/object/multipart.d.ts.map +1 -0
  48. package/dist/object/multipart.js +129 -0
  49. package/dist/object/store.d.ts +201 -0
  50. package/dist/object/store.d.ts.map +1 -0
  51. package/dist/object/store.js +176 -0
  52. package/dist/provision/provisionStorage.d.ts +137 -0
  53. package/dist/provision/provisionStorage.d.ts.map +1 -0
  54. package/dist/provision/provisionStorage.js +110 -0
  55. package/dist/provision/resolveStorageConfig.d.ts +43 -0
  56. package/dist/provision/resolveStorageConfig.d.ts.map +1 -0
  57. package/dist/provision/resolveStorageConfig.js +31 -0
  58. package/dist/quota/quota.d.ts +125 -0
  59. package/dist/quota/quota.d.ts.map +1 -0
  60. package/dist/quota/quota.js +100 -0
  61. package/dist/secret/registry.d.ts +70 -0
  62. package/dist/secret/registry.d.ts.map +1 -0
  63. package/dist/secret/registry.js +82 -0
  64. package/dist/seeds/example.d.ts +3 -0
  65. package/dist/seeds/example.d.ts.map +1 -0
  66. package/dist/seeds/example.js +109 -0
  67. package/dist/test-utils/liveStorage.d.ts +50 -0
  68. package/dist/test-utils/liveStorage.d.ts.map +1 -0
  69. package/dist/test-utils/liveStorage.js +196 -0
  70. package/dist/version.generated.d.ts +5 -0
  71. package/dist/version.generated.d.ts.map +1 -0
  72. package/dist/version.generated.js +7 -0
  73. package/dist/workflows/retryPolicy.d.ts +37 -0
  74. package/dist/workflows/retryPolicy.d.ts.map +1 -0
  75. package/dist/workflows/retryPolicy.js +41 -0
  76. package/dist/workflows/specs.d.ts +55 -0
  77. package/dist/workflows/specs.d.ts.map +1 -0
  78. package/dist/workflows/specs.js +58 -0
  79. package/dist/workflows/sweep.d.ts +82 -0
  80. package/dist/workflows/sweep.d.ts.map +1 -0
  81. package/dist/workflows/sweep.js +143 -0
  82. package/dist/workflows/worker.d.ts +49 -0
  83. package/dist/workflows/worker.d.ts.map +1 -0
  84. package/dist/workflows/worker.js +50 -0
  85. package/package.json +22 -11
  86. package/src/version.generated.ts +1 -1
@@ -0,0 +1,37 @@
1
+ import type { WorkflowRetryPolicy } from "@pithy-sh/core/src/workflow/faults";
2
+ /**
3
+ * **What the orphan sweep retries, and what it refuses to.**
4
+ *
5
+ * The sweep reconciles D1 against R2, and it has already decided what a failure means: every delete and
6
+ * every multipart abort is `.catch(() => {})` on purpose, because an object this run could not remove is
7
+ * an object the next run finds again. That is a reconciler's contract — not error swallowing — and it is
8
+ * what leaves this record empty (pithy-sh/pithy#348).
9
+ *
10
+ * **An empty record is a statement, not an omission.** Core answers for D1's transient vocabulary
11
+ * through `withD1Retry`, so a database under contention is still re-driven by the step, and nothing
12
+ * about that is restated here. What the empty record adds is that storage retries none of its *own*
13
+ * codes.
14
+ *
15
+ * ## Terminal, and why
16
+ *
17
+ * - **`storage/*`** — every one of them is a *request's* refusal: a file that is not there, a file that
18
+ * is not yours, a quota, an upload that never finished, an expired share. The sweep raises none of
19
+ * them, and a sweep that somehow did has found a bug rather than an outage.
20
+ * - **`validation/invalid_input`** — a `StorageSweepParams` an operator dispatched by hand. A payload
21
+ * parses the same way on the fifth attempt as on the first.
22
+ * - **`secrets/*`** — the R2 credentials the multipart abort needs. A key that is not in the bound key
23
+ * set will not be in it a minute later; this wants `pithy secrets provision`, not a backoff.
24
+ *
25
+ * **The cron is the outer retry, and that is why terminal is cheap here.** The sweep is idempotent by
26
+ * construction — a second pass over a reconciled bucket finds nothing left to do — so a run that fails
27
+ * costs one day, and the next day's run does the whole job. Five platform attempts against a refusal
28
+ * that cannot change cost the same day *and* bury the reason under four repeats of it.
29
+ *
30
+ * **What this cannot say, and it is worth knowing.** A bucket listing that fails mid-sweep throws
31
+ * whatever the R2 binding threw, which is not a `PithyError` and carries no code — so it is
32
+ * `unclassified`, which is terminal. That is the right default and the correct outcome here (the next
33
+ * fire re-lists from the top), but it is a default rather than a decision, and no policy record can
34
+ * turn it into one.
35
+ */
36
+ export declare const storageWorkflowRetry: WorkflowRetryPolicy;
37
+ //# sourceMappingURL=retryPolicy.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retryPolicy.d.ts","sourceRoot":"","sources":["../../src/workflows/retryPolicy.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,oBAAoB,EAAE,mBAGlC,CAAC"}
@@ -0,0 +1,41 @@
1
+ //#region src/workflows/retryPolicy.ts
2
+ /**
3
+ * **What the orphan sweep retries, and what it refuses to.**
4
+ *
5
+ * The sweep reconciles D1 against R2, and it has already decided what a failure means: every delete and
6
+ * every multipart abort is `.catch(() => {})` on purpose, because an object this run could not remove is
7
+ * an object the next run finds again. That is a reconciler's contract — not error swallowing — and it is
8
+ * what leaves this record empty (pithy-sh/pithy#348).
9
+ *
10
+ * **An empty record is a statement, not an omission.** Core answers for D1's transient vocabulary
11
+ * through `withD1Retry`, so a database under contention is still re-driven by the step, and nothing
12
+ * about that is restated here. What the empty record adds is that storage retries none of its *own*
13
+ * codes.
14
+ *
15
+ * ## Terminal, and why
16
+ *
17
+ * - **`storage/*`** — every one of them is a *request's* refusal: a file that is not there, a file that
18
+ * is not yours, a quota, an upload that never finished, an expired share. The sweep raises none of
19
+ * them, and a sweep that somehow did has found a bug rather than an outage.
20
+ * - **`validation/invalid_input`** — a `StorageSweepParams` an operator dispatched by hand. A payload
21
+ * parses the same way on the fifth attempt as on the first.
22
+ * - **`secrets/*`** — the R2 credentials the multipart abort needs. A key that is not in the bound key
23
+ * set will not be in it a minute later; this wants `pithy secrets provision`, not a backoff.
24
+ *
25
+ * **The cron is the outer retry, and that is why terminal is cheap here.** The sweep is idempotent by
26
+ * construction — a second pass over a reconciled bucket finds nothing left to do — so a run that fails
27
+ * costs one day, and the next day's run does the whole job. Five platform attempts against a refusal
28
+ * that cannot change cost the same day *and* bury the reason under four repeats of it.
29
+ *
30
+ * **What this cannot say, and it is worth knowing.** A bucket listing that fails mid-sweep throws
31
+ * whatever the R2 binding threw, which is not a `PithyError` and carries no code — so it is
32
+ * `unclassified`, which is terminal. That is the right default and the correct outcome here (the next
33
+ * fire re-lists from the top), but it is a default rather than a decision, and no policy record can
34
+ * turn it into one.
35
+ */
36
+ const storageWorkflowRetry = {
37
+ capability: "storage",
38
+ retryable: {}
39
+ };
40
+ //#endregion
41
+ export { storageWorkflowRetry };
@@ -0,0 +1,55 @@
1
+ import type { WorkflowRegistry } from "@pithy-sh/core/src/workflow/spec";
2
+ import { z } from "zod";
3
+ /**
4
+ * The one durable job storage owns: the orphan sweep.
5
+ *
6
+ * Declared once, here. `capability.ts` derives its Workflow binding from this map, `provision/`
7
+ * derives the host worker's `workflows` array and its cron from it, and any caller triggers it by the
8
+ * `storage/sweep` key. There is no second place where the binding name, the class name, or the
9
+ * schedule is written down, so none of them can drift.
10
+ *
11
+ * `optional: true` because the Workflow lives in the prebuilt sweep worker, which exists only once
12
+ * `pithy storage provision` has run. Until then an app composing storage must still boot and serve
13
+ * every upload and download route; an absent binding degrades to a logged skip.
14
+ */
15
+ /** The capability name — the first segment of the dispatch key and of every deployed resource name. */
16
+ export declare const STORAGE_CAPABILITY = "storage";
17
+ /**
18
+ * The sweep's parameters. Every field is optional, which is a requirement rather than a convenience:
19
+ * a cron supplies no input, and `createEntrypoint` dispatches a scheduled job with `{}`. A schema
20
+ * that demanded a field could never run on its schedule.
21
+ */
22
+ export declare const StorageSweepParams: z.ZodObject<{
23
+ olderThanSeconds: z.ZodOptional<z.ZodNumber>;
24
+ dryRun: z.ZodOptional<z.ZodBoolean>;
25
+ maxPages: z.ZodOptional<z.ZodNumber>;
26
+ }, z.core.$strip>;
27
+ export type StorageSweepParams = z.infer<typeof StorageSweepParams>;
28
+ /**
29
+ * Storage's durable jobs, keyed by job name.
30
+ *
31
+ * The schedule is daily at 03:00 UTC. Divergence between the bucket and the table accumulates slowly
32
+ * — an abandoned upload here, an interrupted delete there — so an hourly pass would list the whole
33
+ * bucket twenty-four times to find nothing. It is a cron *and* a dispatch target: a sweep nobody can
34
+ * run on demand cannot be tested in staging, which is precisely when you want to know what it does.
35
+ */
36
+ export declare const storageWorkflows: {
37
+ readonly sweep: {
38
+ readonly binding: "STORAGE_SWEEP";
39
+ readonly className: "StorageSweepWorkflow";
40
+ readonly params: z.ZodObject<{
41
+ olderThanSeconds: z.ZodOptional<z.ZodNumber>;
42
+ dryRun: z.ZodOptional<z.ZodBoolean>;
43
+ maxPages: z.ZodOptional<z.ZodNumber>;
44
+ }, z.core.$strip>;
45
+ readonly schedule: "0 3 * * *";
46
+ readonly optional: true;
47
+ };
48
+ };
49
+ /**
50
+ * Storage's jobs as a dispatch registry, keyed `storage/<job>`. Built here rather than through
51
+ * `composeWorkflows` because the sweep worker dispatches its own job before any project-wide registry
52
+ * exists — and the key format comes from core's {@link workflowKey} either way, so the two cannot drift.
53
+ */
54
+ export declare const storageWorkflowRegistry: WorkflowRegistry;
55
+ //# sourceMappingURL=specs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"specs.d.ts","sourceRoot":"","sources":["../../src/workflows/specs.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAmB,MAAM,kCAAkC,CAAC;AAC1F,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;;;;;;GAWG;AAEH,uGAAuG;AACvG,eAAO,MAAM,kBAAkB,YAAY,CAAC;AAE5C;;;;GAIG;AACH,eAAO,MAAM,kBAAkB;;;;iBAyB8E,CAAC;AAC9G,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEpE;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB;;iBAEzB,OAAO,EAAE,eAAe;iBACxB,SAAS,EAAE,sBAAsB;iBACjC,MAAM;;;;;iBACN,QAAQ,EAAE,WAAW;iBACrB,QAAQ;;CAEwB,CAAC;AAErC;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,gBAKrC,CAAC"}
@@ -0,0 +1,58 @@
1
+ import { z } from "zod";
2
+ import { workflowKey } from "@pithy-sh/core/src/workflow/naming";
3
+ //#region src/workflows/specs.ts
4
+ /**
5
+ * The one durable job storage owns: the orphan sweep.
6
+ *
7
+ * Declared once, here. `capability.ts` derives its Workflow binding from this map, `provision/`
8
+ * derives the host worker's `workflows` array and its cron from it, and any caller triggers it by the
9
+ * `storage/sweep` key. There is no second place where the binding name, the class name, or the
10
+ * schedule is written down, so none of them can drift.
11
+ *
12
+ * `optional: true` because the Workflow lives in the prebuilt sweep worker, which exists only once
13
+ * `pithy storage provision` has run. Until then an app composing storage must still boot and serve
14
+ * every upload and download route; an absent binding degrades to a logged skip.
15
+ */
16
+ /** The capability name — the first segment of the dispatch key and of every deployed resource name. */
17
+ const STORAGE_CAPABILITY = "storage";
18
+ /**
19
+ * The sweep's parameters. Every field is optional, which is a requirement rather than a convenience:
20
+ * a cron supplies no input, and `createEntrypoint` dispatches a scheduled job with `{}`. A schema
21
+ * that demanded a field could never run on its schedule.
22
+ */
23
+ const StorageSweepParams = z.object({
24
+ olderThanSeconds: z.number().int().positive().optional().describe("Override how old a `pending` row must be before it is reclaimed. Omitted uses the config's `pendingTtlSeconds`. Lower it to reproduce a reclaim in staging without waiting a day."),
25
+ dryRun: z.boolean().optional().describe("Report what would be reclaimed and delete nothing. The safe way to answer 'what is this sweep about to do to prod' before letting it run."),
26
+ maxPages: z.number().int().positive().optional().describe("Cap how many 1,000-key bucket pages one run scans. A bound on the work, so a first sweep over a very large bucket finishes rather than running until it is killed.")
27
+ }).describe("What one orphan-sweep run should do. Every field optional, because a cron passes none of them.");
28
+ /**
29
+ * Storage's durable jobs, keyed by job name.
30
+ *
31
+ * The schedule is daily at 03:00 UTC. Divergence between the bucket and the table accumulates slowly
32
+ * — an abandoned upload here, an interrupted delete there — so an hourly pass would list the whole
33
+ * bucket twenty-four times to find nothing. It is a cron *and* a dispatch target: a sweep nobody can
34
+ * run on demand cannot be tested in staging, which is precisely when you want to know what it does.
35
+ */
36
+ const storageWorkflows = { sweep: {
37
+ binding: "STORAGE_SWEEP",
38
+ className: "StorageSweepWorkflow",
39
+ params: StorageSweepParams,
40
+ schedule: "0 3 * * *",
41
+ optional: true
42
+ } };
43
+ /**
44
+ * Storage's jobs as a dispatch registry, keyed `storage/<job>`. Built here rather than through
45
+ * `composeWorkflows` because the sweep worker dispatches its own job before any project-wide registry
46
+ * exists — and the key format comes from core's {@link workflowKey} either way, so the two cannot drift.
47
+ */
48
+ const storageWorkflowRegistry = Object.fromEntries(Object.entries(storageWorkflows).map(([job, spec]) => {
49
+ const key = workflowKey(STORAGE_CAPABILITY, job);
50
+ return [key, {
51
+ key,
52
+ capability: STORAGE_CAPABILITY,
53
+ job,
54
+ spec
55
+ }];
56
+ }));
57
+ //#endregion
58
+ export { STORAGE_CAPABILITY, StorageSweepParams, storageWorkflowRegistry, storageWorkflows };
@@ -0,0 +1,82 @@
1
+ import type { Logger } from "@pithy-sh/core/src/logger/logger";
2
+ import { type StorageDatabase } from "../data/tables";
3
+ import type { ObjectStore } from "../object/store";
4
+ /**
5
+ * The orphan sweep: reconciling two stores that can only ever be *eventually* consistent.
6
+ *
7
+ * A file is two writes — a row in D1 and an object in R2 — and no transaction spans them. Every
8
+ * interruption between the two leaves one behind, and the divergence goes **both** ways:
9
+ *
10
+ * - **A row with no object.** An upload started (the row reserves its quota) and never finished.
11
+ * The reservation is real: it counts against the owner's limit forever unless something returns
12
+ * it. This is the direction that silently locks a user out of their own quota.
13
+ * - **An object no row bills for.** A delete removed the row and then failed on R2, a client uploaded
14
+ * to a presigned URL after its row was deleted, or an upload was aborted and the URL used anyway —
15
+ * the row survives as `failed` and reserves nothing. The bytes are unreachable and unbilled to
16
+ * anyone — a pure leak, paid for monthly.
17
+ *
18
+ * A sweep that only walked one direction would leave the other to accumulate indefinitely, so this
19
+ * one does both, in that order: reclaiming rows first means an object whose row was just dropped is
20
+ * caught by the same run rather than waiting a day for the next.
21
+ *
22
+ * **Two things keep it from deleting live data.** Only keys carrying storage's own `obj/` prefix are
23
+ * ever considered, so a bucket shared with another tool is left alone. And an object is only orphaned
24
+ * if it is *older than the cutoff*: a row is written before its upload URL is minted, but R2's view
25
+ * and D1's view of "now" are independent, and an object uploaded seconds ago whose row this run has
26
+ * not yet read is not an orphan — it is a race. The age check turns that race into a wait.
27
+ *
28
+ * Pure over injected dependencies: no bindings, no `cloudflare:workers`, no config import. The
29
+ * Workflow in `worker.ts` is a durable shell around this function, which is what makes the reconciling
30
+ * logic testable against a deliberately divergent bucket and table.
31
+ */
32
+ /** What one sweep run needs. */
33
+ export interface SweepDeps {
34
+ /** The storage database. */
35
+ db: StorageDatabase;
36
+ /** The object plane — used for `abortMultipart`, `delete`, and the bucket listing. */
37
+ store: ObjectStore;
38
+ /** The current time. */
39
+ now: () => Date;
40
+ /** How old a `pending` row (and an unmatched object) must be before it is reclaimed. */
41
+ ttlSeconds: number;
42
+ /** Report and delete nothing. */
43
+ dryRun?: boolean;
44
+ /** Cap on bucket pages scanned in one run. */
45
+ maxPages?: number;
46
+ /**
47
+ * Keys per bucket list page. Defaults to R2's own cap of 1,000, which is what a real run should ask
48
+ * for. Exposed so a test can force pagination without writing a thousand objects — the paging and
49
+ * truncation logic is otherwise only reachable at a scale no test suite should pay for.
50
+ */
51
+ pageSize?: number;
52
+ }
53
+ /** What one sweep run found and did. */
54
+ export interface SweepResult {
55
+ /** `pending` rows past the TTL: their uploads aborted, their bytes dropped, their rows removed. */
56
+ reclaimedRows: number;
57
+ /** Bucket objects with no row, older than the cutoff: deleted. */
58
+ deletedObjects: number;
59
+ /** Bucket objects examined — the run's cost, and how a `maxPages` cap is noticed. */
60
+ scannedObjects: number;
61
+ /** Whether the run stopped at its page cap with more of the bucket unexamined. */
62
+ truncated: boolean;
63
+ /** Whether this run only reported. */
64
+ dryRun: boolean;
65
+ }
66
+ /**
67
+ * Run one sweep. Idempotent by construction — a second run over a reconciled bucket finds nothing and
68
+ * deletes nothing — so a retried Workflow step, a manual trigger after a cron, and a nervous operator
69
+ * running it twice all cost the same.
70
+ */
71
+ export declare function sweepStorage(deps: SweepDeps): Promise<SweepResult>;
72
+ /**
73
+ * Report a finished run. The tally is the sweep's only visible output — a sweep whose findings are
74
+ * invisible is a sweep nobody can tell has stopped working — so it goes out as fields, which is what
75
+ * makes "how much quota did we reclaim this month" a query rather than a grep.
76
+ *
77
+ * A truncated run is `warn`, not `info`. It succeeded and left part of the bucket unexamined, so the
78
+ * orphans it did not reach are paid for until a later pass gets to them. That is the one outcome an
79
+ * operator should be able to find without reading every run that went fine.
80
+ */
81
+ export declare function reportSweep(log: Logger, result: SweepResult): void;
82
+ //# sourceMappingURL=sweep.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../../src/workflows/sweep.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAE/D,OAAO,EAAyB,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAE7E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGnD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,gCAAgC;AAChC,MAAM,WAAW,SAAS;IACxB,4BAA4B;IAC5B,EAAE,EAAE,eAAe,CAAC;IACpB,sFAAsF;IACtF,KAAK,EAAE,WAAW,CAAC;IACnB,wBAAwB;IACxB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,wFAAwF;IACxF,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wCAAwC;AACxC,MAAM,WAAW,WAAW;IAC1B,mGAAmG;IACnG,aAAa,EAAE,MAAM,CAAC;IACtB,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,cAAc,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,SAAS,EAAE,OAAO,CAAC;IACnB,sCAAsC;IACtC,MAAM,EAAE,OAAO,CAAC;CACjB;AAoHD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAWxE;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAOlE"}
@@ -0,0 +1,143 @@
1
+ import { StorageObject } from "../data/storageObject.js";
2
+ import { STORAGE_OBJECTS_TABLE } from "../data/tables.js";
3
+ import { OBJECT_KEY_PREFIX, isDerivedObjectKey } from "../object/key.js";
4
+ import { QUOTA_COUNTED_STATUSES } from "../quota/quota.js";
5
+ import { SQLiteDate } from "@pithy-sh/core/src/data/codecs";
6
+ import { chunkByBoundParameters } from "@pithy-sh/core/src/data/boundParameters";
7
+ //#region src/workflows/sweep.ts
8
+ /** R2's own cap on a list page. Asking for more is silently clamped, so ask for exactly this. */
9
+ const LIST_PAGE_SIZE = 1e3;
10
+ /** Default cap on pages per run: a million keys, which is a long sweep and a sane bound. */
11
+ const DEFAULT_MAX_PAGES = 1e3;
12
+ /**
13
+ * The membership lookup binds its status filter besides the key list, so the chunk gets D1's budget
14
+ * less those two. Add another `where` to that query and this number moves with it.
15
+ */
16
+ const CLAIMED_KEYS_FIXED_PARAMETERS = QUOTA_COUNTED_STATUSES.length;
17
+ /**
18
+ * Reclaim `pending` rows older than the cutoff.
19
+ *
20
+ * Each row's in-flight multipart is aborted (R2 bills stored parts, so leaving them is a real cost),
21
+ * then any bytes that did land are deleted, then the row goes — object first, row second, so a
22
+ * failure part-way leaves an object with no row, which the second phase collects. The other order
23
+ * would leave a row pointing at nothing, which nothing collects.
24
+ *
25
+ * Both R2 calls are best-effort. An upload id R2 has already forgotten, or a key that was never
26
+ * written, must not stop the row from being reclaimed — the whole point of the run is to unwedge
27
+ * state, not to insist it was tidy.
28
+ */
29
+ async function reclaimPendingRows(deps, cutoff) {
30
+ const rows = await deps.db.selectFrom(STORAGE_OBJECTS_TABLE).selectAll().where("status", "=", "pending").where("createdAt", "<", SQLiteDate.encode(cutoff)).execute();
31
+ if (deps.dryRun) return rows.length;
32
+ let reclaimed = 0;
33
+ for (const row of rows) {
34
+ const object = StorageObject.parse(row);
35
+ if (object.uploadId) await deps.store.abortMultipart(object.key, object.uploadId).catch(() => {});
36
+ await deps.store.delete(object.key).catch(() => {});
37
+ await deps.db.deleteFrom(STORAGE_OBJECTS_TABLE).where("id", "=", object.id).execute();
38
+ reclaimed += 1;
39
+ }
40
+ return reclaimed;
41
+ }
42
+ /**
43
+ * Delete bucket objects that no row claims.
44
+ *
45
+ * **A row claims a key only while it bills for it.** The membership test filters on the same statuses
46
+ * the quota sums ({@link QUOTA_COUNTED_STATUSES}), so a `failed` row — the record an abort or a refused
47
+ * completion leaves behind — reserves nothing *and* shields nothing. Without that filter the two
48
+ * predicates disagree, and the disagreement is a leak with no floor: abort a single-PUT upload, PUT to
49
+ * the presigned URL you still hold, and the surviving `failed` row marks the key as claimed forever
50
+ * while `usedBytes` counts none of it. Phase one skips it (it only reclaims `pending` rows), so nothing
51
+ * automated would ever collect those bytes again.
52
+ *
53
+ * The row itself is left alone either way. It is the owner's record that an upload was attempted and
54
+ * abandoned; only its claim on the bytes is withdrawn.
55
+ *
56
+ * The membership test is a batched `WHERE key IN (…)` rather than one query per key: a page is a
57
+ * thousand keys, and a thousand round trips to D1 is the difference between a sweep that finishes and
58
+ * one that times out. It is batched rather than one query per page because D1 caps a statement at 100
59
+ * bound parameters and a page carries up to a thousand keys — so the page is chunked under that cap
60
+ * and the claimed sets unioned before anything is deleted. The two limits are unrelated knobs: the
61
+ * page size is R2's listing cost, the chunk size is D1's statement cost, and tying one to the other
62
+ * would be a coincidence rather than a reason.
63
+ *
64
+ * Only keys this capability derived are considered, and only ones older than the cutoff — see the
65
+ * module comment on why the age check is load-bearing rather than cautious.
66
+ */
67
+ async function deleteOrphanedObjects(deps, cutoff) {
68
+ const maxPages = deps.maxPages ?? DEFAULT_MAX_PAGES;
69
+ let cursor;
70
+ let deleted = 0;
71
+ let scanned = 0;
72
+ let pages = 0;
73
+ for (;;) {
74
+ const page = await deps.store.list({
75
+ prefix: OBJECT_KEY_PREFIX,
76
+ cursor,
77
+ limit: deps.pageSize ?? LIST_PAGE_SIZE
78
+ });
79
+ pages += 1;
80
+ scanned += page.objects.length;
81
+ const candidates = page.objects.filter((object) => isDerivedObjectKey(object.key) && object.uploaded.getTime() < cutoff.getTime());
82
+ if (candidates.length > 0) {
83
+ const keys = candidates.map((object) => object.key);
84
+ const claimed = /* @__PURE__ */ new Set();
85
+ for (const group of chunkByBoundParameters(keys, CLAIMED_KEYS_FIXED_PARAMETERS)) {
86
+ const known = await deps.db.selectFrom(STORAGE_OBJECTS_TABLE).select("key").where("key", "in", group).where("status", "in", [...QUOTA_COUNTED_STATUSES]).execute();
87
+ for (const row of known) claimed.add(row.key);
88
+ }
89
+ for (const object of candidates) {
90
+ if (claimed.has(object.key)) continue;
91
+ if (!deps.dryRun) await deps.store.delete(object.key).catch(() => {});
92
+ deleted += 1;
93
+ }
94
+ }
95
+ cursor = page.cursor;
96
+ if (!cursor) return {
97
+ deleted,
98
+ scanned,
99
+ truncated: false
100
+ };
101
+ if (pages >= maxPages) return {
102
+ deleted,
103
+ scanned,
104
+ truncated: true
105
+ };
106
+ }
107
+ }
108
+ /**
109
+ * Run one sweep. Idempotent by construction — a second run over a reconciled bucket finds nothing and
110
+ * deletes nothing — so a retried Workflow step, a manual trigger after a cron, and a nervous operator
111
+ * running it twice all cost the same.
112
+ */
113
+ async function sweepStorage(deps) {
114
+ const cutoff = /* @__PURE__ */ new Date(deps.now().getTime() - deps.ttlSeconds * 1e3);
115
+ const reclaimedRows = await reclaimPendingRows(deps, cutoff);
116
+ const objects = await deleteOrphanedObjects(deps, cutoff);
117
+ return {
118
+ reclaimedRows,
119
+ deletedObjects: objects.deleted,
120
+ scannedObjects: objects.scanned,
121
+ truncated: objects.truncated,
122
+ dryRun: deps.dryRun === true
123
+ };
124
+ }
125
+ /**
126
+ * Report a finished run. The tally is the sweep's only visible output — a sweep whose findings are
127
+ * invisible is a sweep nobody can tell has stopped working — so it goes out as fields, which is what
128
+ * makes "how much quota did we reclaim this month" a query rather than a grep.
129
+ *
130
+ * A truncated run is `warn`, not `info`. It succeeded and left part of the bucket unexamined, so the
131
+ * orphans it did not reach are paid for until a later pass gets to them. That is the one outcome an
132
+ * operator should be able to find without reading every run that went fine.
133
+ */
134
+ function reportSweep(log, result) {
135
+ const fields = { ...result };
136
+ if (result.truncated) {
137
+ log.warn("storage sweep stopped at its page cap", fields);
138
+ return;
139
+ }
140
+ log.info("storage sweep complete", fields);
141
+ }
142
+ //#endregion
143
+ export { reportSweep, sweepStorage };
@@ -0,0 +1,49 @@
1
+ import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudflare:workers";
2
+ import type { D1Database, R2Bucket } from "@cloudflare/workers-types";
3
+ import type { SecretsStoreEnv } from "@pithy-sh/secrets/src/env/bindings";
4
+ import { StorageSweepParams } from "./specs";
5
+ /**
6
+ * The prebuilt storage sweep worker. `pithy storage provision` deploys one per environment; the
7
+ * adopter authors none of it. It hosts a single Workflow — the orphan sweep — and fires it on a daily
8
+ * cron.
9
+ *
10
+ * **Why its own worker rather than the app worker's `scheduled()`.** The sweep lists an entire bucket
11
+ * and can delete from it. Keeping it out of the request-serving deployment means it cannot compete
12
+ * with a user's download for CPU, and — more to the point — the app worker never needs a binding that
13
+ * can delete arbitrary objects. It is also a Workflow rather than a plain scheduled pass because a
14
+ * Worker invocation is wall-clock bounded while a Workflow step is not: a bucket with a million keys
15
+ * is a thousand list pages, and that is a job you want checkpointed rather than restarted.
16
+ *
17
+ * This module imports `cloudflare:workers`, so it runs only in the Workers runtime and is excluded
18
+ * from the node `.describe()` meta-test.
19
+ */
20
+ /** The sweep worker's env: the app database, the bucket, the master key, and the resolved config. */
21
+ export interface StorageWorkerEnv extends SecretsStoreEnv {
22
+ /** The app database the `pithy_storage_*` tables live in. */
23
+ DB: D1Database;
24
+ /** The bucket the sweep lists and deletes from. */
25
+ STORAGE_BUCKET: R2Bucket;
26
+ /** The resolved storage config as a JSON string, filled at provision. Absent falls back to defaults. */
27
+ STORAGE_CONFIG?: string;
28
+ /** This worker's own Workflow binding — how `scheduled()` starts an instance. */
29
+ STORAGE_SWEEP: {
30
+ create(options?: {
31
+ id?: string;
32
+ params?: unknown;
33
+ }): Promise<unknown>;
34
+ };
35
+ }
36
+ /** The orphan sweep, as a cron-triggered Workflow. One step, because one run is one reconciliation. */
37
+ export declare class StorageSweepWorkflow extends WorkflowEntrypoint<StorageWorkerEnv, StorageSweepParams> {
38
+ run(event: WorkflowEvent<StorageSweepParams>, step: WorkflowStep): Promise<void>;
39
+ }
40
+ declare const _default: {
41
+ /**
42
+ * Cron entry: start one sweep instance per fire, with empty parameters — the defaults are the
43
+ * scheduled behavior. The same Workflow stays dispatchable with a payload (`dryRun`, a shorter
44
+ * TTL, a page cap), which is how the sweep is exercised in staging without waiting for 03:00.
45
+ */
46
+ scheduled(_controller: unknown, env: StorageWorkerEnv): Promise<void>;
47
+ };
48
+ export default _default;
49
+ //# sourceMappingURL=worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/workflows/worker.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,KAAK,aAAa,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE/F,OAAO,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAGtE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AAO1E,OAAO,EAAsB,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAGjE;;;;;;;;;;;;;;GAcG;AAEH,qGAAqG;AACrG,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD,6DAA6D;IAC7D,EAAE,EAAE,UAAU,CAAC;IACf,mDAAmD;IACnD,cAAc,EAAE,QAAQ,CAAC;IACzB,wGAAwG;IACxG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,aAAa,EAAE;QAAE,MAAM,CAAC,OAAO,CAAC,EAAE;YAAE,EAAE,CAAC,EAAE,MAAM,CAAC;YAAC,MAAM,CAAC,EAAE,OAAO,CAAA;SAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;KAAE,CAAC;CAC1F;AAMD,uGAAuG;AACvG,qBAAa,oBAAqB,SAAQ,kBAAkB,CAAC,gBAAgB,EAAE,kBAAkB,CAAC;IACjF,GAAG,CAAC,KAAK,EAAE,aAAa,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAsC9F;CACF;;IAGC;;;;OAIG;IACG,SAAS,cAAc,OAAO,OAAO,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC"}
@@ -0,0 +1,50 @@
1
+ import { StorageConfig } from "../config/config.js";
2
+ import { storageDatabase } from "../data/tables.js";
3
+ import { STORAGE_R2_SECRET, storageSecretsRegistry } from "../secret/registry.js";
4
+ import { objectStore } from "../object/store.js";
5
+ import { STORAGE_CAPABILITY, StorageSweepParams } from "./specs.js";
6
+ import { reportSweep, sweepStorage } from "./sweep.js";
7
+ import { storageWorkflowRetry } from "./retryPolicy.js";
8
+ import { configureSharedSecrets } from "@pithy-sh/secrets/src/sharedSecretsStore";
9
+ import { WorkflowEntrypoint } from "cloudflare:workers";
10
+ import { NonRetryableError } from "cloudflare:workflows";
11
+ import { bindWorkflowContext, createWorkerLogger } from "@pithy-sh/core/src/logger/worker";
12
+ import { classifiedSteps } from "@pithy-sh/core/src/workflow/faults";
13
+ //#region src/workflows/worker.ts
14
+ configureSharedSecrets({ registry: storageSecretsRegistry });
15
+ /** The orphan sweep, as a cron-triggered Workflow. One step, because one run is one reconciliation. */
16
+ var StorageSweepWorkflow = class extends WorkflowEntrypoint {
17
+ async run(event, step) {
18
+ const params = StorageSweepParams.parse(event.payload ?? {});
19
+ const config = StorageConfig.parse(this.env.STORAGE_CONFIG ? JSON.parse(this.env.STORAGE_CONFIG) : {});
20
+ const log = bindWorkflowContext(createWorkerLogger({ name: STORAGE_CAPABILITY }), {
21
+ workflow: event.workflowName,
22
+ instance: event.instanceId,
23
+ env: this.env.ENVIRONMENT ?? "unknown"
24
+ });
25
+ const result = await classifiedSteps(step, storageWorkflowRetry, NonRetryableError).do("sweep", async () => sweepStorage({
26
+ db: storageDatabase(this.env.DB),
27
+ store: objectStore({
28
+ bucket: this.env.STORAGE_BUCKET,
29
+ env: this.env,
30
+ secretName: STORAGE_R2_SECRET
31
+ }),
32
+ now: () => /* @__PURE__ */ new Date(),
33
+ ttlSeconds: params.olderThanSeconds ?? config.pendingTtlSeconds,
34
+ dryRun: params.dryRun,
35
+ maxPages: params.maxPages
36
+ }));
37
+ reportSweep(log, result);
38
+ }
39
+ };
40
+ var worker_default = {
41
+ /**
42
+ * Cron entry: start one sweep instance per fire, with empty parameters — the defaults are the
43
+ * scheduled behavior. The same Workflow stays dispatchable with a payload (`dryRun`, a shorter
44
+ * TTL, a page cap), which is how the sweep is exercised in staging without waiting for 03:00.
45
+ */
46
+ async scheduled(_controller, env) {
47
+ await env.STORAGE_SWEEP.create({ params: {} });
48
+ } };
49
+ //#endregion
50
+ export { StorageSweepWorkflow, worker_default as default };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pithy-sh/storage",
3
- "version": "0.1.2",
3
+ "version": "0.1.3",
4
4
  "license": "MIT",
5
5
  "repository": {
6
6
  "type": "git",
@@ -8,19 +8,24 @@
8
8
  "directory": "packages/storage"
9
9
  },
10
10
  "files": [
11
+ "dist",
11
12
  "src",
12
13
  "pithy.manifest.json",
13
- "!src/**/*.test.*"
14
+ "!src/**/*.test.*",
15
+ "!dist/**/*.test.*"
14
16
  ],
15
17
  "type": "module",
16
18
  "engines": {
17
19
  "node": ">=22"
18
20
  },
19
21
  "exports": {
20
- "./src/*": "./src/*.ts"
22
+ "./src/*": {
23
+ "types": "./dist/*.d.ts",
24
+ "default": "./dist/*.js"
25
+ }
21
26
  },
22
27
  "scripts": {
23
- "build": "tsc -p tsconfig.json --noEmit false --outDir dist",
28
+ "build": "tsdown && tsc -p tsconfig.build.json",
24
29
  "typecheck": "tsc -p tsconfig.json",
25
30
  "test": "vitest run",
26
31
  "test:node": "vitest run --project=node",
@@ -32,21 +37,27 @@
32
37
  "dependencies": {
33
38
  "@cloudflare/workers-types": "^5.20260729.1",
34
39
  "@hono/zod-validator": "^0.9.0",
35
- "@pithy-sh/cloudflare": "^0.1.2",
36
- "@pithy-sh/core": "^0.1.2",
37
- "@pithy-sh/secrets": "^0.1.2",
38
- "hono": "^4.13.2",
39
- "kysely": "^0.29.0",
40
- "zod": "^4.0.0"
40
+ "@pithy-sh/cloudflare": "^0.1.3",
41
+ "@pithy-sh/core": "^0.1.3",
42
+ "@pithy-sh/secrets": "^0.1.3"
41
43
  },
42
44
  "devDependencies": {
43
45
  "@cloudflare/vitest-plugin": "^1.0.0",
44
46
  "@pithy-sh/tsconfig": "workspace:*",
45
47
  "@types/node": "^22.15.0",
46
48
  "@vitest/coverage-v8": "^4.1.0",
49
+ "hono": "^4.13.2",
50
+ "kysely": "^0.29.0",
47
51
  "kysely-d1": "^0.4.0",
52
+ "tsdown": "^0.23.0",
48
53
  "typescript": "^7.0.2",
49
54
  "vitest": "^4.1.0",
50
- "wrangler": "^4.115.0"
55
+ "wrangler": "^4.115.0",
56
+ "zod": "^4.4.0"
57
+ },
58
+ "peerDependencies": {
59
+ "hono": "^4.13.2",
60
+ "kysely": "^0.29.0",
61
+ "zod": "^4.4.0"
51
62
  }
52
63
  }
@@ -13,4 +13,4 @@
13
13
  export const PACKAGE_NAME = "@pithy-sh/storage";
14
14
 
15
15
  /** This package's version, stamped from its own package.json at generation time. */
16
- export const PACKAGE_VERSION = "0.1.2";
16
+ export const PACKAGE_VERSION = "0.1.3";