@pithy-sh/storage 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/capability.d.ts +45 -0
  2. package/dist/capability.d.ts.map +1 -0
  3. package/dist/capability.js +76 -0
  4. package/dist/config/config.d.ts +27 -0
  5. package/dist/config/config.d.ts.map +1 -0
  6. package/dist/config/config.js +49 -0
  7. package/dist/data/share.d.ts +27 -0
  8. package/dist/data/share.d.ts.map +1 -0
  9. package/dist/data/share.js +28 -0
  10. package/dist/data/storageObject.d.ts +55 -0
  11. package/dist/data/storageObject.d.ts.map +1 -0
  12. package/dist/data/storageObject.js +44 -0
  13. package/dist/data/tables.d.ts +23 -0
  14. package/dist/data/tables.d.ts.map +1 -0
  15. package/dist/data/tables.js +26 -0
  16. package/dist/error/errors.d.ts +60 -0
  17. package/dist/error/errors.d.ts.map +1 -0
  18. package/dist/error/errors.js +89 -0
  19. package/dist/http/guard.d.ts +20 -0
  20. package/dist/http/guard.d.ts.map +1 -0
  21. package/dist/http/guard.js +29 -0
  22. package/dist/http/handlers.d.ts +232 -0
  23. package/dist/http/handlers.d.ts.map +1 -0
  24. package/dist/http/handlers.js +507 -0
  25. package/dist/http/routes.d.ts +70 -0
  26. package/dist/http/routes.d.ts.map +1 -0
  27. package/dist/http/routes.js +135 -0
  28. package/dist/http/schemas.d.ts +74 -0
  29. package/dist/http/schemas.d.ts.map +1 -0
  30. package/dist/http/schemas.js +93 -0
  31. package/dist/http/serve.d.ts +121 -0
  32. package/dist/http/serve.d.ts.map +1 -0
  33. package/dist/http/serve.js +209 -0
  34. package/dist/index.d.ts +23 -0
  35. package/dist/index.d.ts.map +1 -0
  36. package/dist/index.js +14 -0
  37. package/dist/migrations/0001_objects.d.ts +24 -0
  38. package/dist/migrations/0001_objects.d.ts.map +1 -0
  39. package/dist/migrations/0001_objects.js +41 -0
  40. package/dist/object/cloudflare.d.ts +23 -0
  41. package/dist/object/cloudflare.d.ts.map +1 -0
  42. package/dist/object/cloudflare.js +48 -0
  43. package/dist/object/key.d.ts +29 -0
  44. package/dist/object/key.d.ts.map +1 -0
  45. package/dist/object/key.js +37 -0
  46. package/dist/object/multipart.d.ts +78 -0
  47. package/dist/object/multipart.d.ts.map +1 -0
  48. package/dist/object/multipart.js +131 -0
  49. package/dist/object/store.d.ts +203 -0
  50. package/dist/object/store.d.ts.map +1 -0
  51. package/dist/object/store.js +178 -0
  52. package/dist/provision/provisionStorage.d.ts +139 -0
  53. package/dist/provision/provisionStorage.d.ts.map +1 -0
  54. package/dist/provision/provisionStorage.js +112 -0
  55. package/dist/provision/resolveStorageConfig.d.ts +45 -0
  56. package/dist/provision/resolveStorageConfig.d.ts.map +1 -0
  57. package/dist/provision/resolveStorageConfig.js +33 -0
  58. package/dist/quota/quota.d.ts +127 -0
  59. package/dist/quota/quota.d.ts.map +1 -0
  60. package/dist/quota/quota.js +102 -0
  61. package/dist/secret/registry.d.ts +72 -0
  62. package/dist/secret/registry.d.ts.map +1 -0
  63. package/dist/secret/registry.js +84 -0
  64. package/dist/seeds/example.d.ts +5 -0
  65. package/dist/seeds/example.d.ts.map +1 -0
  66. package/dist/seeds/example.js +111 -0
  67. package/dist/test-utils/liveStorage.d.ts +52 -0
  68. package/dist/test-utils/liveStorage.d.ts.map +1 -0
  69. package/dist/test-utils/liveStorage.js +198 -0
  70. package/dist/version.generated.d.ts +7 -0
  71. package/dist/version.generated.d.ts.map +1 -0
  72. package/dist/version.generated.js +9 -0
  73. package/dist/workflows/retryPolicy.d.ts +39 -0
  74. package/dist/workflows/retryPolicy.d.ts.map +1 -0
  75. package/dist/workflows/retryPolicy.js +43 -0
  76. package/dist/workflows/specs.d.ts +57 -0
  77. package/dist/workflows/specs.d.ts.map +1 -0
  78. package/dist/workflows/specs.js +60 -0
  79. package/dist/workflows/sweep.d.ts +84 -0
  80. package/dist/workflows/sweep.d.ts.map +1 -0
  81. package/dist/workflows/sweep.js +145 -0
  82. package/dist/workflows/worker.d.ts +51 -0
  83. package/dist/workflows/worker.d.ts.map +1 -0
  84. package/dist/workflows/worker.js +52 -0
  85. package/package.json +22 -11
  86. package/src/version.generated.ts +1 -1
@@ -0,0 +1,52 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import type { CloudflareR2Manager } from "@pithy-sh/cloudflare/src/r2/r2Manager";
4
+ import { type IntegrationCreds } from "@pithy-sh/cloudflare/src/test-utils/harness";
5
+ import { type StorageDatabase } from "../data/tables";
6
+ import { type ObjectStore } from "../object/store";
7
+ /**
8
+ * Scaffolding for this package's `*.integration.test.ts` suites — the live half of storage.
9
+ *
10
+ * `@pithy-sh/cloudflare`'s harness finds credentials and guarantees teardown. Storage needs two more
11
+ * things on top: the seam under test built the way a Worker builds it (an {@link objectStore} over a
12
+ * *resolved credential bundle*, not an injected fake), and a real database to reconcile against.
13
+ *
14
+ * **What is live here, and what cannot be.** The presigned half — presign, the whole multipart
15
+ * lifecycle, server-side copy — is real R2 over the S3 protocol, and it is the half Miniflare cannot
16
+ * emulate at all: it serves no S3 endpoint, so a presigned URL has nothing to address. The binding
17
+ * half is the mirror image. An `R2Bucket` exists only inside workerd, so outside a Worker there is no
18
+ * binding to call — {@link liveObjectStore} serves `head`, `list` and `delete` from the *same real
19
+ * bucket* over S3 instead, and leaves `get` throwing.
20
+ *
21
+ * That split is deliberate and it is the whole reason the two suites are complementary rather than
22
+ * redundant. `store.workers.test.ts` drives the binding against Miniflare with the presign half
23
+ * throwing; this drives the presign half against real R2 with the binding-only read throwing. Neither
24
+ * can quietly take the other's path and pass.
25
+ */
26
+ /** One live object plane: the store under test, the bucket it addresses, and the S3 manager behind it. */
27
+ export interface LiveObjectPlane {
28
+ /** The throwaway bucket's name. Every key in a test lands here and nowhere else. */
29
+ bucketName: string;
30
+ /** The seam under test — production `objectStore`, with the binding half served over S3. */
31
+ store: ObjectStore;
32
+ /** The S3 manager, for a test that needs an oracle the seam does not expose. */
33
+ manager: CloudflareR2Manager;
34
+ }
35
+ /**
36
+ * Run `exercise` against a fresh throwaway bucket, then empty and delete it.
37
+ *
38
+ * Everything but the bucket is built *before* the create, so the one thing teardown has to undo is the
39
+ * one thing `create` did. One bucket per test keeps a failure's debris out of the next test — and out
40
+ * of the account.
41
+ */
42
+ export declare function withLiveBucket<T>(creds: IntegrationCreds, exercise: (plane: LiveObjectPlane) => Promise<T>): Promise<T>;
43
+ /**
44
+ * Run `exercise` against a fresh throwaway D1, migrated to storage's schema, then delete it.
45
+ *
46
+ * A real database rather than Miniflare's, for one reason: this pool is Node, and Miniflare's D1 lives
47
+ * in workerd. Standing one up here would mean running the sweep against an emulator on one side and
48
+ * real R2 on the other, which is the mixed picture the live suite exists to replace. It also buys a
49
+ * second proof for free — the migration runs on real D1 over REST, exactly as `pithy migrate` runs it.
50
+ */
51
+ export declare function withLiveDatabase<T>(creds: IntegrationCreds, exercise: (db: StorageDatabase) => Promise<T>): Promise<T>;
52
+ //# sourceMappingURL=liveStorage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"liveStorage.d.ts","sourceRoot":"","sources":["../../src/test-utils/liveStorage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AACjF,OAAO,EAAE,KAAK,gBAAgB,EAAqC,MAAM,6CAA6C,CAAC;AAMvH,OAAO,EAAE,KAAK,eAAe,EAAmB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,EAAiC,KAAK,WAAW,EAAe,MAAM,iBAAiB,CAAC;AAG/F;;;;;;;;;;;;;;;;;;GAkBG;AAEH,0GAA0G;AAC1G,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,KAAK,EAAE,WAAW,CAAC;IACnB,gFAAgF;IAChF,OAAO,EAAE,mBAAmB,CAAC;CAC9B;AA2ID;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,CAAC,EACpC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,CAAC,CAAC,GAC/C,OAAO,CAAC,CAAC,CAAC,CAyBZ;AAiBD;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,CAAC,EAAE,EAAE,eAAe,KAAK,OAAO,CAAC,CAAC,CAAC,GAC5C,OAAO,CAAC,CAAC,CAAC,CAaZ"}
@@ -0,0 +1,198 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import { storageDatabase } from "../data/tables.js";
4
+ import { STORAGE_R2_SECRET, storageSecretsRegistry } from "../secret/registry.js";
5
+ import { ObjectListing, ObjectMetadata, objectStore } from "../object/store.js";
6
+ import { storage_0001_objects } from "../migrations/0001_objects.js";
7
+ import "../capability.js";
8
+ import { configureSharedSecrets } from "@pithy-sh/secrets/src/sharedSecretsStore";
9
+ import { CloudflareClients } from "@pithy-sh/cloudflare/src/client/clients";
10
+ import { uniqueName, withThrowawayResource } from "@pithy-sh/cloudflare/src/test-utils/harness";
11
+ import { createMigrationRegistry } from "@pithy-sh/core/src/migrations/registry";
12
+ import { runMigrations } from "@pithy-sh/core/src/migrations/runner";
13
+ //#region src/test-utils/liveStorage.ts
14
+ /**
15
+ * The R2 binding, deliberately absent.
16
+ *
17
+ * `objectStore` takes a bucket binding, and no such thing exists outside workerd. Rather than hand it
18
+ * something that pretends, every method throws: a test that reaches the binding by accident fails
19
+ * loudly instead of passing against a stand-in. The methods that matter are re-implemented over S3 in
20
+ * {@link liveObjectStore}, so this is only ever reached by a path that has no live equivalent.
21
+ */
22
+ function absentBinding() {
23
+ const refuse = (method) => () => {
24
+ throw new Error(`R2 binding '${method}' has no equivalent outside a Worker — use the S3-backed seam.`);
25
+ };
26
+ return {
27
+ get: refuse("get"),
28
+ head: refuse("head"),
29
+ put: refuse("put"),
30
+ delete: refuse("delete"),
31
+ list: refuse("list"),
32
+ createMultipartUpload: refuse("createMultipartUpload"),
33
+ resumeMultipartUpload: refuse("resumeMultipartUpload")
34
+ };
35
+ }
36
+ /**
37
+ * Build the object store a live test drives.
38
+ *
39
+ * The credential path is the production one end to end: the bundle is injected as a `.dev.vars`-shaped
40
+ * string under the registry name, `objectStore` resolves it through `sharedSecretsStore`, and the
41
+ * registry's Zod schema validates it before a URL is ever signed. So a drift in
42
+ * {@link R2StorageCredentials} fails here the same way it would fail in a deployed Worker.
43
+ *
44
+ * `head`, `list` and `delete` are then re-pointed at the same bucket over S3, because the binding they
45
+ * would otherwise use does not exist in Node. `get` is left throwing: a range or a conditional read
46
+ * through the binding has no S3 mapping this file could write without re-implementing workerd, and a
47
+ * re-implementation is exactly the mock these tests exist to avoid. The suites assert R2's range and
48
+ * conditional *semantics* over a presigned GET instead, where the answer comes from R2 itself.
49
+ *
50
+ * Multipart uploads are recorded as they open and forgotten as they close, so teardown can abort
51
+ * whatever a failure left in flight — stored parts hold a bucket against deletion and cost money.
52
+ */
53
+ function liveObjectStore(credentials, manager, open) {
54
+ const env = { [STORAGE_R2_SECRET]: JSON.stringify(credentials) };
55
+ configureSharedSecrets({ registry: storageSecretsRegistry });
56
+ const store = objectStore({
57
+ bucket: absentBinding(),
58
+ env
59
+ });
60
+ async function head(key) {
61
+ const object = await manager.headObject(key);
62
+ if (!object) return null;
63
+ if (!object.uploaded) throw new Error(`R2 reported no LastModified for '${key}'`);
64
+ return ObjectMetadata.parse({
65
+ key,
66
+ size: object.size,
67
+ etag: object.etag.replace(/^"|"$/g, ""),
68
+ contentType: object.contentType,
69
+ uploaded: object.uploaded
70
+ });
71
+ }
72
+ return {
73
+ ...store,
74
+ async initMultipart(key, contentType) {
75
+ const uploadId = await store.initMultipart(key, contentType);
76
+ open.set(`${key}\0${uploadId}`, {
77
+ key,
78
+ uploadId
79
+ });
80
+ return uploadId;
81
+ },
82
+ async completeMultipart(key, uploadId, parts) {
83
+ await store.completeMultipart(key, uploadId, parts);
84
+ open.delete(`${key}\0${uploadId}`);
85
+ },
86
+ async abortMultipart(key, uploadId) {
87
+ await store.abortMultipart(key, uploadId);
88
+ open.delete(`${key}\0${uploadId}`);
89
+ },
90
+ head,
91
+ async list(options) {
92
+ const page = await manager.listObjects({
93
+ prefix: options?.prefix,
94
+ cursor: options?.cursor,
95
+ maxKeys: options?.limit
96
+ });
97
+ const objects = [];
98
+ for (const key of page.keys) {
99
+ const metadata = await head(key);
100
+ if (metadata) objects.push(metadata);
101
+ }
102
+ return ObjectListing.parse({
103
+ objects,
104
+ cursor: page.cursor
105
+ });
106
+ },
107
+ async delete(key) {
108
+ await manager.deleteObject(key);
109
+ },
110
+ get() {
111
+ throw new Error("`get` reads through the R2 binding, which no Node context has. Presign a GET instead.");
112
+ }
113
+ };
114
+ }
115
+ /**
116
+ * Empty a bucket so R2 will delete it — the same drain `pithy storage deprovision --storage` runs, so a
117
+ * break in it fails a test here rather than an operator's teardown.
118
+ *
119
+ * Two things hold a bucket: stored objects, and the parts of a multipart upload that was never completed
120
+ * or aborted — precisely what a test that failed mid-upload leaves behind. `emptyBucket` handles both.
121
+ * The uploads this run opened are aborted first and best-effort, because a test may already have aborted
122
+ * one and this map is only local bookkeeping; the authoritative sweep is R2's own listing inside
123
+ * `emptyBucket`, which is also all a reaper cleaning up after a *previous* run has.
124
+ */
125
+ async function purgeBucket(manager, open) {
126
+ for (const upload of open.values()) await manager.abortMultipartUpload(upload.key, upload.uploadId).catch(() => void 0);
127
+ open.clear();
128
+ await manager.emptyBucket();
129
+ }
130
+ /**
131
+ * Run `exercise` against a fresh throwaway bucket, then empty and delete it.
132
+ *
133
+ * Everything but the bucket is built *before* the create, so the one thing teardown has to undo is the
134
+ * one thing `create` did. One bucket per test keeps a failure's debris out of the next test — and out
135
+ * of the account.
136
+ */
137
+ async function withLiveBucket(creds, exercise) {
138
+ const r2 = creds.r2;
139
+ if (!r2) throw new Error("withLiveBucket needs R2_CREDENTIALS — gate the suite on `creds.r2`.");
140
+ const clients = new CloudflareClients({
141
+ accountId: creds.accountId,
142
+ apiToken: creds.apiToken
143
+ });
144
+ const provisioner = clients.r2Provisioner();
145
+ const bucketName = uniqueName("storage");
146
+ const manager = clients.r2({
147
+ ...r2,
148
+ bucketName
149
+ });
150
+ const open = /* @__PURE__ */ new Map();
151
+ const store = liveObjectStore({
152
+ ...r2,
153
+ accountId: creds.accountId,
154
+ apiToken: creds.apiToken,
155
+ bucket: bucketName
156
+ }, manager, open);
157
+ return withThrowawayResource(() => provisioner.createBucket(bucketName), () => exercise({
158
+ bucketName,
159
+ store,
160
+ manager
161
+ }), async () => {
162
+ await purgeBucket(manager, open).catch(() => void 0);
163
+ await provisioner.deleteBucket(bucketName);
164
+ });
165
+ }
166
+ /** The storage migrations, composed the way `pithy migrate` composes them for the app database. */
167
+ function storageMigrations() {
168
+ const provider = createMigrationRegistry([{
169
+ database: "app",
170
+ namespace: "storage",
171
+ order: 800,
172
+ migrations: { "0001_objects": storage_0001_objects }
173
+ }]).app;
174
+ if (!provider) throw new Error("no app migration provider");
175
+ return provider;
176
+ }
177
+ /**
178
+ * Run `exercise` against a fresh throwaway D1, migrated to storage's schema, then delete it.
179
+ *
180
+ * A real database rather than Miniflare's, for one reason: this pool is Node, and Miniflare's D1 lives
181
+ * in workerd. Standing one up here would mean running the sweep against an emulator on one side and
182
+ * real R2 on the other, which is the mixed picture the live suite exists to replace. It also buys a
183
+ * second proof for free — the migration runs on real D1 over REST, exactly as `pithy migrate` runs it.
184
+ */
185
+ async function withLiveDatabase(creds, exercise) {
186
+ const clients = new CloudflareClients({
187
+ accountId: creds.accountId,
188
+ apiToken: creds.apiToken
189
+ });
190
+ const provisioner = clients.d1Provisioner();
191
+ return withThrowawayResource(() => provisioner.createDatabase(uniqueName("storage")), async (database) => {
192
+ const d1 = clients.d1(database.uuid);
193
+ await runMigrations(d1, storageMigrations());
194
+ return exercise(storageDatabase(d1));
195
+ }, (database) => provisioner.deleteDatabase(database.uuid));
196
+ }
197
+ //#endregion
198
+ export { withLiveBucket, withLiveDatabase };
@@ -0,0 +1,7 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ /** This package's npm name — the join key against a release feed. */
4
+ export declare const PACKAGE_NAME = "@pithy-sh/storage";
5
+ /** This package's version, stamped from its own package.json at generation time. */
6
+ export declare const PACKAGE_VERSION = "0.1.4";
7
+ //# sourceMappingURL=version.generated.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"version.generated.d.ts","sourceRoot":"","sources":["../src/version.generated.ts"],"names":[],"mappings":"AAWA,qEAAqE;AACrE,eAAO,MAAM,YAAY,sBAAsB,CAAC;AAEhD,oFAAoF;AACpF,eAAO,MAAM,eAAe,UAAU,CAAC"}
@@ -0,0 +1,9 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ //#region src/version.generated.ts
4
+ /** This package's npm name — the join key against a release feed. */
5
+ const PACKAGE_NAME = "@pithy-sh/storage";
6
+ /** This package's version, stamped from its own package.json at generation time. */
7
+ const PACKAGE_VERSION = "0.1.4";
8
+ //#endregion
9
+ export { PACKAGE_NAME, PACKAGE_VERSION };
@@ -0,0 +1,39 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import type { WorkflowRetryPolicy } from "@pithy-sh/core/src/workflow/faults";
4
+ /**
5
+ * **What the orphan sweep retries, and what it refuses to.**
6
+ *
7
+ * The sweep reconciles D1 against R2, and it has already decided what a failure means: every delete and
8
+ * every multipart abort is `.catch(() => {})` on purpose, because an object this run could not remove is
9
+ * an object the next run finds again. That is a reconciler's contract — not error swallowing — and it is
10
+ * what leaves this record empty (pithy-sh/pithy#348).
11
+ *
12
+ * **An empty record is a statement, not an omission.** Core answers for D1's transient vocabulary
13
+ * through `withD1Retry`, so a database under contention is still re-driven by the step, and nothing
14
+ * about that is restated here. What the empty record adds is that storage retries none of its *own*
15
+ * codes.
16
+ *
17
+ * ## Terminal, and why
18
+ *
19
+ * - **`storage/*`** — every one of them is a *request's* refusal: a file that is not there, a file that
20
+ * is not yours, a quota, an upload that never finished, an expired share. The sweep raises none of
21
+ * them, and a sweep that somehow did has found a bug rather than an outage.
22
+ * - **`validation/invalid_input`** — a `StorageSweepParams` an operator dispatched by hand. A payload
23
+ * parses the same way on the fifth attempt as on the first.
24
+ * - **`secrets/*`** — the R2 credentials the multipart abort needs. A key that is not in the bound key
25
+ * set will not be in it a minute later; this wants `pithy secrets provision`, not a backoff.
26
+ *
27
+ * **The cron is the outer retry, and that is why terminal is cheap here.** The sweep is idempotent by
28
+ * construction — a second pass over a reconciled bucket finds nothing left to do — so a run that fails
29
+ * costs one day, and the next day's run does the whole job. Five platform attempts against a refusal
30
+ * that cannot change cost the same day *and* bury the reason under four repeats of it.
31
+ *
32
+ * **What this cannot say, and it is worth knowing.** A bucket listing that fails mid-sweep throws
33
+ * whatever the R2 binding threw, which is not a `PithyError` and carries no code — so it is
34
+ * `unclassified`, which is terminal. That is the right default and the correct outcome here (the next
35
+ * fire re-lists from the top), but it is a default rather than a decision, and no policy record can
36
+ * turn it into one.
37
+ */
38
+ export declare const storageWorkflowRetry: WorkflowRetryPolicy;
39
+ //# sourceMappingURL=retryPolicy.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retryPolicy.d.ts","sourceRoot":"","sources":["../../src/workflows/retryPolicy.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,eAAO,MAAM,oBAAoB,EAAE,mBAGlC,CAAC"}
@@ -0,0 +1,43 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ //#region src/workflows/retryPolicy.ts
4
+ /**
5
+ * **What the orphan sweep retries, and what it refuses to.**
6
+ *
7
+ * The sweep reconciles D1 against R2, and it has already decided what a failure means: every delete and
8
+ * every multipart abort is `.catch(() => {})` on purpose, because an object this run could not remove is
9
+ * an object the next run finds again. That is a reconciler's contract — not error swallowing — and it is
10
+ * what leaves this record empty (pithy-sh/pithy#348).
11
+ *
12
+ * **An empty record is a statement, not an omission.** Core answers for D1's transient vocabulary
13
+ * through `withD1Retry`, so a database under contention is still re-driven by the step, and nothing
14
+ * about that is restated here. What the empty record adds is that storage retries none of its *own*
15
+ * codes.
16
+ *
17
+ * ## Terminal, and why
18
+ *
19
+ * - **`storage/*`** — every one of them is a *request's* refusal: a file that is not there, a file that
20
+ * is not yours, a quota, an upload that never finished, an expired share. The sweep raises none of
21
+ * them, and a sweep that somehow did has found a bug rather than an outage.
22
+ * - **`validation/invalid_input`** — a `StorageSweepParams` an operator dispatched by hand. A payload
23
+ * parses the same way on the fifth attempt as on the first.
24
+ * - **`secrets/*`** — the R2 credentials the multipart abort needs. A key that is not in the bound key
25
+ * set will not be in it a minute later; this wants `pithy secrets provision`, not a backoff.
26
+ *
27
+ * **The cron is the outer retry, and that is why terminal is cheap here.** The sweep is idempotent by
28
+ * construction — a second pass over a reconciled bucket finds nothing left to do — so a run that fails
29
+ * costs one day, and the next day's run does the whole job. Five platform attempts against a refusal
30
+ * that cannot change cost the same day *and* bury the reason under four repeats of it.
31
+ *
32
+ * **What this cannot say, and it is worth knowing.** A bucket listing that fails mid-sweep throws
33
+ * whatever the R2 binding threw, which is not a `PithyError` and carries no code — so it is
34
+ * `unclassified`, which is terminal. That is the right default and the correct outcome here (the next
35
+ * fire re-lists from the top), but it is a default rather than a decision, and no policy record can
36
+ * turn it into one.
37
+ */
38
+ const storageWorkflowRetry = {
39
+ capability: "storage",
40
+ retryable: {}
41
+ };
42
+ //#endregion
43
+ export { storageWorkflowRetry };
@@ -0,0 +1,57 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import type { WorkflowRegistry } from "@pithy-sh/core/src/workflow/spec";
4
+ import { z } from "zod";
5
+ /**
6
+ * The one durable job storage owns: the orphan sweep.
7
+ *
8
+ * Declared once, here. `capability.ts` derives its Workflow binding from this map, `provision/`
9
+ * derives the host worker's `workflows` array and its cron from it, and any caller triggers it by the
10
+ * `storage/sweep` key. There is no second place where the binding name, the class name, or the
11
+ * schedule is written down, so none of them can drift.
12
+ *
13
+ * `optional: true` because the Workflow lives in the prebuilt sweep worker, which exists only once
14
+ * `pithy storage provision` has run. Until then an app composing storage must still boot and serve
15
+ * every upload and download route; an absent binding degrades to a logged skip.
16
+ */
17
+ /** The capability name — the first segment of the dispatch key and of every deployed resource name. */
18
+ export declare const STORAGE_CAPABILITY = "storage";
19
+ /**
20
+ * The sweep's parameters. Every field is optional, which is a requirement rather than a convenience:
21
+ * a cron supplies no input, and `createEntrypoint` dispatches a scheduled job with `{}`. A schema
22
+ * that demanded a field could never run on its schedule.
23
+ */
24
+ export declare const StorageSweepParams: z.ZodObject<{
25
+ olderThanSeconds: z.ZodOptional<z.ZodNumber>;
26
+ dryRun: z.ZodOptional<z.ZodBoolean>;
27
+ maxPages: z.ZodOptional<z.ZodNumber>;
28
+ }, z.core.$strip>;
29
+ export type StorageSweepParams = z.infer<typeof StorageSweepParams>;
30
+ /**
31
+ * Storage's durable jobs, keyed by job name.
32
+ *
33
+ * The schedule is daily at 03:00 UTC. Divergence between the bucket and the table accumulates slowly
34
+ * — an abandoned upload here, an interrupted delete there — so an hourly pass would list the whole
35
+ * bucket twenty-four times to find nothing. It is a cron *and* a dispatch target: a sweep nobody can
36
+ * run on demand cannot be tested in staging, which is precisely when you want to know what it does.
37
+ */
38
+ export declare const storageWorkflows: {
39
+ readonly sweep: {
40
+ readonly binding: "STORAGE_SWEEP";
41
+ readonly className: "StorageSweepWorkflow";
42
+ readonly params: z.ZodObject<{
43
+ olderThanSeconds: z.ZodOptional<z.ZodNumber>;
44
+ dryRun: z.ZodOptional<z.ZodBoolean>;
45
+ maxPages: z.ZodOptional<z.ZodNumber>;
46
+ }, z.core.$strip>;
47
+ readonly schedule: "0 3 * * *";
48
+ readonly optional: true;
49
+ };
50
+ };
51
+ /**
52
+ * Storage's jobs as a dispatch registry, keyed `storage/<job>`. Built here rather than through
53
+ * `composeWorkflows` because the sweep worker dispatches its own job before any project-wide registry
54
+ * exists — and the key format comes from core's {@link workflowKey} either way, so the two cannot drift.
55
+ */
56
+ export declare const storageWorkflowRegistry: WorkflowRegistry;
57
+ //# sourceMappingURL=specs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"specs.d.ts","sourceRoot":"","sources":["../../src/workflows/specs.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAmB,MAAM,kCAAkC,CAAC;AAC1F,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;;;;;;GAWG;AAEH,uGAAuG;AACvG,eAAO,MAAM,kBAAkB,YAAY,CAAC;AAE5C;;;;GAIG;AACH,eAAO,MAAM,kBAAkB;;;;iBAyB8E,CAAC;AAC9G,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAEpE;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB;;iBAEzB,OAAO,EAAE,eAAe;iBACxB,SAAS,EAAE,sBAAsB;iBACjC,MAAM;;;;;iBACN,QAAQ,EAAE,WAAW;iBACrB,QAAQ;;CAEwB,CAAC;AAErC;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,EAAE,gBAKrC,CAAC"}
@@ -0,0 +1,60 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import { z } from "zod";
4
+ import { workflowKey } from "@pithy-sh/core/src/workflow/naming";
5
+ //#region src/workflows/specs.ts
6
+ /**
7
+ * The one durable job storage owns: the orphan sweep.
8
+ *
9
+ * Declared once, here. `capability.ts` derives its Workflow binding from this map, `provision/`
10
+ * derives the host worker's `workflows` array and its cron from it, and any caller triggers it by the
11
+ * `storage/sweep` key. There is no second place where the binding name, the class name, or the
12
+ * schedule is written down, so none of them can drift.
13
+ *
14
+ * `optional: true` because the Workflow lives in the prebuilt sweep worker, which exists only once
15
+ * `pithy storage provision` has run. Until then an app composing storage must still boot and serve
16
+ * every upload and download route; an absent binding degrades to a logged skip.
17
+ */
18
+ /** The capability name — the first segment of the dispatch key and of every deployed resource name. */
19
+ const STORAGE_CAPABILITY = "storage";
20
+ /**
21
+ * The sweep's parameters. Every field is optional, which is a requirement rather than a convenience:
22
+ * a cron supplies no input, and `createEntrypoint` dispatches a scheduled job with `{}`. A schema
23
+ * that demanded a field could never run on its schedule.
24
+ */
25
+ const StorageSweepParams = z.object({
26
+ olderThanSeconds: z.number().int().positive().optional().describe("Override how old a `pending` row must be before it is reclaimed. Omitted uses the config's `pendingTtlSeconds`. Lower it to reproduce a reclaim in staging without waiting a day."),
27
+ dryRun: z.boolean().optional().describe("Report what would be reclaimed and delete nothing. The safe way to answer 'what is this sweep about to do to prod' before letting it run."),
28
+ maxPages: z.number().int().positive().optional().describe("Cap how many 1,000-key bucket pages one run scans. A bound on the work, so a first sweep over a very large bucket finishes rather than running until it is killed.")
29
+ }).describe("What one orphan-sweep run should do. Every field optional, because a cron passes none of them.");
30
+ /**
31
+ * Storage's durable jobs, keyed by job name.
32
+ *
33
+ * The schedule is daily at 03:00 UTC. Divergence between the bucket and the table accumulates slowly
34
+ * — an abandoned upload here, an interrupted delete there — so an hourly pass would list the whole
35
+ * bucket twenty-four times to find nothing. It is a cron *and* a dispatch target: a sweep nobody can
36
+ * run on demand cannot be tested in staging, which is precisely when you want to know what it does.
37
+ */
38
+ const storageWorkflows = { sweep: {
39
+ binding: "STORAGE_SWEEP",
40
+ className: "StorageSweepWorkflow",
41
+ params: StorageSweepParams,
42
+ schedule: "0 3 * * *",
43
+ optional: true
44
+ } };
45
+ /**
46
+ * Storage's jobs as a dispatch registry, keyed `storage/<job>`. Built here rather than through
47
+ * `composeWorkflows` because the sweep worker dispatches its own job before any project-wide registry
48
+ * exists — and the key format comes from core's {@link workflowKey} either way, so the two cannot drift.
49
+ */
50
+ const storageWorkflowRegistry = Object.fromEntries(Object.entries(storageWorkflows).map(([job, spec]) => {
51
+ const key = workflowKey(STORAGE_CAPABILITY, job);
52
+ return [key, {
53
+ key,
54
+ capability: STORAGE_CAPABILITY,
55
+ job,
56
+ spec
57
+ }];
58
+ }));
59
+ //#endregion
60
+ export { STORAGE_CAPABILITY, StorageSweepParams, storageWorkflowRegistry, storageWorkflows };
@@ -0,0 +1,84 @@
1
+ // SPDX-FileCopyrightText: 2026 Pithy
2
+ // SPDX-License-Identifier: MIT
3
+ import type { Logger } from "@pithy-sh/core/src/logger/logger";
4
+ import { type StorageDatabase } from "../data/tables";
5
+ import type { ObjectStore } from "../object/store";
6
+ /**
7
+ * The orphan sweep: reconciling two stores that can only ever be *eventually* consistent.
8
+ *
9
+ * A file is two writes — a row in D1 and an object in R2 — and no transaction spans them. Every
10
+ * interruption between the two leaves one behind, and the divergence goes **both** ways:
11
+ *
12
+ * - **A row with no object.** An upload started (the row reserves its quota) and never finished.
13
+ * The reservation is real: it counts against the owner's limit forever unless something returns
14
+ * it. This is the direction that silently locks a user out of their own quota.
15
+ * - **An object no row bills for.** A delete removed the row and then failed on R2, a client uploaded
16
+ * to a presigned URL after its row was deleted, or an upload was aborted and the URL used anyway —
17
+ * the row survives as `failed` and reserves nothing. The bytes are unreachable and unbilled to
18
+ * anyone — a pure leak, paid for monthly.
19
+ *
20
+ * A sweep that only walked one direction would leave the other to accumulate indefinitely, so this
21
+ * one does both, in that order: reclaiming rows first means an object whose row was just dropped is
22
+ * caught by the same run rather than waiting a day for the next.
23
+ *
24
+ * **Two things keep it from deleting live data.** Only keys carrying storage's own `obj/` prefix are
25
+ * ever considered, so a bucket shared with another tool is left alone. And an object is only orphaned
26
+ * if it is *older than the cutoff*: a row is written before its upload URL is minted, but R2's view
27
+ * and D1's view of "now" are independent, and an object uploaded seconds ago whose row this run has
28
+ * not yet read is not an orphan — it is a race. The age check turns that race into a wait.
29
+ *
30
+ * Pure over injected dependencies: no bindings, no `cloudflare:workers`, no config import. The
31
+ * Workflow in `worker.ts` is a durable shell around this function, which is what makes the reconciling
32
+ * logic testable against a deliberately divergent bucket and table.
33
+ */
34
+ /** What one sweep run needs. */
35
+ export interface SweepDeps {
36
+ /** The storage database. */
37
+ db: StorageDatabase;
38
+ /** The object plane — used for `abortMultipart`, `delete`, and the bucket listing. */
39
+ store: ObjectStore;
40
+ /** The current time. */
41
+ now: () => Date;
42
+ /** How old a `pending` row (and an unmatched object) must be before it is reclaimed. */
43
+ ttlSeconds: number;
44
+ /** Report and delete nothing. */
45
+ dryRun?: boolean;
46
+ /** Cap on bucket pages scanned in one run. */
47
+ maxPages?: number;
48
+ /**
49
+ * Keys per bucket list page. Defaults to R2's own cap of 1,000, which is what a real run should ask
50
+ * for. Exposed so a test can force pagination without writing a thousand objects — the paging and
51
+ * truncation logic is otherwise only reachable at a scale no test suite should pay for.
52
+ */
53
+ pageSize?: number;
54
+ }
55
+ /** What one sweep run found and did. */
56
+ export interface SweepResult {
57
+ /** `pending` rows past the TTL: their uploads aborted, their bytes dropped, their rows removed. */
58
+ reclaimedRows: number;
59
+ /** Bucket objects with no row, older than the cutoff: deleted. */
60
+ deletedObjects: number;
61
+ /** Bucket objects examined — the run's cost, and how a `maxPages` cap is noticed. */
62
+ scannedObjects: number;
63
+ /** Whether the run stopped at its page cap with more of the bucket unexamined. */
64
+ truncated: boolean;
65
+ /** Whether this run only reported. */
66
+ dryRun: boolean;
67
+ }
68
+ /**
69
+ * Run one sweep. Idempotent by construction — a second run over a reconciled bucket finds nothing and
70
+ * deletes nothing — so a retried Workflow step, a manual trigger after a cron, and a nervous operator
71
+ * running it twice all cost the same.
72
+ */
73
+ export declare function sweepStorage(deps: SweepDeps): Promise<SweepResult>;
74
+ /**
75
+ * Report a finished run. The tally is the sweep's only visible output — a sweep whose findings are
76
+ * invisible is a sweep nobody can tell has stopped working — so it goes out as fields, which is what
77
+ * makes "how much quota did we reclaim this month" a query rather than a grep.
78
+ *
79
+ * A truncated run is `warn`, not `info`. It succeeded and left part of the bucket unexamined, so the
80
+ * orphans it did not reach are paid for until a later pass gets to them. That is the one outcome an
81
+ * operator should be able to find without reading every run that went fine.
82
+ */
83
+ export declare function reportSweep(log: Logger, result: SweepResult): void;
84
+ //# sourceMappingURL=sweep.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../../src/workflows/sweep.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAE/D,OAAO,EAAyB,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAE7E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGnD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,gCAAgC;AAChC,MAAM,WAAW,SAAS;IACxB,4BAA4B;IAC5B,EAAE,EAAE,eAAe,CAAC;IACpB,sFAAsF;IACtF,KAAK,EAAE,WAAW,CAAC;IACnB,wBAAwB;IACxB,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,wFAAwF;IACxF,UAAU,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wCAAwC;AACxC,MAAM,WAAW,WAAW;IAC1B,mGAAmG;IACnG,aAAa,EAAE,MAAM,CAAC;IACtB,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,qFAAqF;IACrF,cAAc,EAAE,MAAM,CAAC;IACvB,kFAAkF;IAClF,SAAS,EAAE,OAAO,CAAC;IACnB,sCAAsC;IACtC,MAAM,EAAE,OAAO,CAAC;CACjB;AAoHD;;;;GAIG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,CAWxE;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,IAAI,CAOlE"}