@pithy-sh/storage 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/package.json +52 -0
- package/pithy.manifest.json +70 -0
- package/src/capability.ts +104 -0
- package/src/cloudflare-test.d.ts +14 -0
- package/src/config/config.ts +105 -0
- package/src/data/share.ts +36 -0
- package/src/data/storageObject.ts +81 -0
- package/src/data/tables.ts +37 -0
- package/src/error/errors.ts +140 -0
- package/src/http/guard.ts +32 -0
- package/src/http/handlers.ts +684 -0
- package/src/http/routes.ts +313 -0
- package/src/http/schemas.ts +214 -0
- package/src/http/serve.ts +288 -0
- package/src/index.ts +42 -0
- package/src/migrations/0001_objects.ts +86 -0
- package/src/object/cloudflare.ts +55 -0
- package/src/object/key.ts +40 -0
- package/src/object/multipart.ts +166 -0
- package/src/object/store.ts +371 -0
- package/src/provision/provisionStorage.ts +192 -0
- package/src/provision/resolveStorageConfig.ts +80 -0
- package/src/quota/quota.ts +241 -0
- package/src/secret/registry.ts +118 -0
- package/src/seeds/example.ts +120 -0
- package/src/test-utils/liveStorage.ts +261 -0
- package/src/version.generated.ts +16 -0
- package/src/workflows/retryPolicy.ts +43 -0
- package/src/workflows/specs.ts +85 -0
- package/src/workflows/sweep.ts +226 -0
- package/src/workflows/worker.ts +103 -0
- package/src/workflows/wrangler.jsonc +54 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { InternalError } from "@pithy-sh/core/src/error/pithyError";
|
|
5
|
+
import type { DeclaredEnvironments } from "@pithy-sh/core/src/naming/environment";
|
|
6
|
+
import { resourceNames } from "@pithy-sh/core/src/naming/resourceNames";
|
|
7
|
+
import { type ManagedEnvironment, managedEnvironments } from "@pithy-sh/secrets/src/scope";
|
|
8
|
+
import { STORAGE_CAPABILITY } from "../workflows/specs";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The provisioning orchestration for the storage capability — the live counterpart to
|
|
12
|
+
* `pithy add storage`'s config wiring.
|
|
13
|
+
*
|
|
14
|
+
* The split is deliberate and worth stating, because the two commands look like one. `pithy add`
|
|
15
|
+
* writes *bindings*: it installs the package, adds `storage()` to `pithy.config.ts`, and puts a
|
|
16
|
+
* `STORAGE_BUCKET` entry in `wrangler.jsonc`. It touches no Cloudflare account and provisions
|
|
17
|
+
* nothing — an `add` that reached out to an account would make adding a capability an operation you
|
|
18
|
+
* could not do offline, or in CI, or without credentials. This command stands up what those bindings
|
|
19
|
+
* point at: the R2 bucket, the per-environment credential secret, and the sweep worker.
|
|
20
|
+
*
|
|
21
|
+
* The `STORAGE_SWEEP` Workflow binding is the exception, and it belongs to this side of the split.
|
|
22
|
+
* Wrangler requires a `name` and a `class_name` on every `workflows` entry, and the deployed name is
|
|
23
|
+
* per environment — so `add` writes none (a partial entry stops wrangler loading the config) and the
|
|
24
|
+
* CLI writes the complete entry here, once the host worker exists.
|
|
25
|
+
*
|
|
26
|
+
* The live Cloudflare/wrangler steps sit behind the {@link StorageProvisioner} seam, so the
|
|
27
|
+
* orchestration — order, idempotency, per-environment fan-out — is unit-tested with a fake that
|
|
28
|
+
* records its call order, and no account. **Every step is idempotent**: find-then-create for the
|
|
29
|
+
* bucket, create-then-update for the secret, and a deploy that overwrites. Re-running is a no-op.
|
|
30
|
+
*
|
|
31
|
+
* **What it does not do.** Cloudflare exposes no API for minting an R2 S3 access-key pair, so nothing
|
|
32
|
+
* here mints one. The pair is supplied by the operator — flags, or `R2_CREDENTIALS` in `.dev.vars` —
|
|
33
|
+
* and written into the secret as given. That is a real limitation, and the manifest says so rather
|
|
34
|
+
* than promising a mint that does not happen.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The bucket objects live in, **per project and per environment** — `acme-staging-storage`,
|
|
39
|
+
* `acme-prod-storage`.
|
|
40
|
+
*
|
|
41
|
+
* One bucket per account would mean staging writes objects into the bucket prod reads, and a staging
|
|
42
|
+
* teardown deletes prod's files. Buckets are free — the cost is bytes and operations — so a shared one
|
|
43
|
+
* buys nothing and risks everything. The same posture `@pithy-sh/media` takes, and the opposite of
|
|
44
|
+
* `@pithy-sh/email`'s deliberately project-shared suppression database.
|
|
45
|
+
*
|
|
46
|
+
* The project segment is what makes provisioning's find-then-create safe. R2's namespace is flat and
|
|
47
|
+
* account-wide, so an unprefixed `pithy-storage-prod` would be *found* by a second Pithy project in the
|
|
48
|
+
* same account and silently adopted — two apps writing objects into one bucket, and either teardown
|
|
49
|
+
* deleting both. The name is the only partition R2 offers, so the name carries the owner.
|
|
50
|
+
*
|
|
51
|
+
* Named as an **R2 bucket** through core's naming facade, which is the one namespace whose rule is
|
|
52
|
+
* genuinely 3–63 lowercase characters, starting and ending alphanumeric. Every other namespace Pithy
|
|
53
|
+
* writes into was once held to that same 63 for no reason; asking for a bucket by name is how this one
|
|
54
|
+
* keeps it on purpose.
|
|
55
|
+
*/
|
|
56
|
+
export function storageBucketName(project: string, env: ManagedEnvironment): string {
|
|
57
|
+
return resourceNames(project).env(env).r2(STORAGE_CAPABILITY);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The deployed sweep-worker name for a project's environment — also its resolved config's basename.
|
|
62
|
+
*
|
|
63
|
+
* A **Worker script**, so 63 rather than the Workflow's 64, and refused rather than truncated: a script
|
|
64
|
+
* cannot be renamed once a deploy or a `service` binding points at it.
|
|
65
|
+
*/
|
|
66
|
+
export function storageWorkerName(project: string, env: ManagedEnvironment): string {
|
|
67
|
+
return resourceNames(project).env(env).worker(STORAGE_CAPABILITY);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** The provisioned resources an environment's worker and secret are wired to. */
|
|
71
|
+
export interface StorageResources {
|
|
72
|
+
/** The R2 bucket objects live in. */
|
|
73
|
+
bucketName: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The live Cloudflare/wrangler seam. Each step must be idempotent. */
|
|
77
|
+
export interface StorageProvisioner {
|
|
78
|
+
/**
|
|
79
|
+
* Verify account prerequisites before any resource is created — most importantly a registered
|
|
80
|
+
* `workers.dev` subdomain, which Cloudflare requires to deploy a Workflow-hosting worker. Failing
|
|
81
|
+
* here means failing before a bucket exists, rather than half way through a fan-out.
|
|
82
|
+
*/
|
|
83
|
+
preflight(): Promise<void>;
|
|
84
|
+
/** Create (or reuse) this environment's R2 bucket; returns its name. Idempotent. */
|
|
85
|
+
ensureBucket(env: ManagedEnvironment): Promise<{ bucketName: string }>;
|
|
86
|
+
/**
|
|
87
|
+
* Write this environment's `storage-r2-credentials` secret — the account id, S3 key pair, scoped
|
|
88
|
+
* token, and bucket name the object store presigns with. Idempotent (create, else update). Runs
|
|
89
|
+
* before the worker is deployed: a worker that boots without its credentials fails on its first
|
|
90
|
+
* multipart abort.
|
|
91
|
+
*/
|
|
92
|
+
writeCredentials(env: ManagedEnvironment, resources: StorageResources): Promise<void>;
|
|
93
|
+
/** Deploy the prebuilt sweep worker for this environment, wired to the provisioned bucket. */
|
|
94
|
+
deployWorker(env: ManagedEnvironment, resources: StorageResources): Promise<void>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** What provisioning produced, per environment. */
|
|
98
|
+
export interface StorageProvisionResult {
|
|
99
|
+
/** Each environment provisioned, with the resources its worker and secret were wired to. */
|
|
100
|
+
environments: Array<{ env: ManagedEnvironment } & StorageResources>;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Provision the storage infrastructure: check the account, create every environment's bucket, write
|
|
105
|
+
* every environment's credentials, then deploy every environment's sweep worker.
|
|
106
|
+
*
|
|
107
|
+
* The phase order is the contract. A secret must not name a bucket that does not exist, and a worker
|
|
108
|
+
* must not boot before the secret it reads. Fanning each phase across all environments — rather than
|
|
109
|
+
* completing one environment end to end before starting the next — means a failure creating prod's
|
|
110
|
+
* bucket stops the run before staging's worker is deployed against a half-provisioned account.
|
|
111
|
+
*
|
|
112
|
+
* `environments` is the project's declaration from the root `pithy.config.ts` (#241). Every declared
|
|
113
|
+
* environment is provisioned; an environment this skipped would be one the project deploys to with no
|
|
114
|
+
* resources behind it — the silence the closed `ManagedEnvironment` enum used to produce.
|
|
115
|
+
*/
|
|
116
|
+
export async function provisionStorage(
|
|
117
|
+
provisioner: StorageProvisioner,
|
|
118
|
+
environments: DeclaredEnvironments | readonly string[],
|
|
119
|
+
): Promise<StorageProvisionResult> {
|
|
120
|
+
await provisioner.preflight();
|
|
121
|
+
|
|
122
|
+
const resources = new Map<ManagedEnvironment, StorageResources>();
|
|
123
|
+
for (const env of managedEnvironments(environments)) {
|
|
124
|
+
const { bucketName } = await provisioner.ensureBucket(env);
|
|
125
|
+
resources.set(env, { bucketName });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
for (const env of managedEnvironments(environments)) {
|
|
129
|
+
await provisioner.writeCredentials(env, resourcesFor(resources, env));
|
|
130
|
+
}
|
|
131
|
+
for (const env of managedEnvironments(environments)) {
|
|
132
|
+
await provisioner.deployWorker(env, resourcesFor(resources, env));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return { environments: managedEnvironments(environments).map((env) => ({ env, ...resourcesFor(resources, env) })) };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Read back an environment's resources. Absent is impossible by construction, so the throw is a bug check. */
|
|
139
|
+
function resourcesFor(resources: Map<ManagedEnvironment, StorageResources>, env: ManagedEnvironment): StorageResources {
|
|
140
|
+
const found = resources.get(env);
|
|
141
|
+
if (!found) throw new InternalError({ message: `No provisioned resources for the ${env} environment.` });
|
|
142
|
+
return found;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** The teardown seam — the inverse of {@link StorageProvisioner}. Every step idempotent. */
|
|
146
|
+
export interface StorageDeprovisioner {
|
|
147
|
+
/** Delete this environment's sweep worker. Idempotent (a missing worker is a no-op). */
|
|
148
|
+
deleteWorker(env: ManagedEnvironment): Promise<void>;
|
|
149
|
+
/**
|
|
150
|
+
* Delete this environment's R2 bucket **and every file in it**. **Destructive** — nothing here is
|
|
151
|
+
* recoverable. Idempotent (a missing bucket is a no-op).
|
|
152
|
+
*
|
|
153
|
+
* Emptying the bucket is part of the contract, not a nicety: R2 refuses to delete a bucket that still
|
|
154
|
+
* holds an object, or the parts of a multipart upload that was never completed. An implementation that
|
|
155
|
+
* only called the control-plane delete would work on an untouched bucket and fail on every bucket
|
|
156
|
+
* anyone had used.
|
|
157
|
+
*/
|
|
158
|
+
deleteBucket(env: ManagedEnvironment): Promise<void>;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Teardown options. By default the stored files are **kept** — only the sweep workers come down. */
|
|
162
|
+
export interface StorageDeprovisionOptions {
|
|
163
|
+
/**
|
|
164
|
+
* Also delete the R2 buckets, with every file in them. Off by default, because those files are the
|
|
165
|
+
* adopter's data and no deploy can restore them — the flag is the confirmation.
|
|
166
|
+
*/
|
|
167
|
+
deleteStorage?: boolean;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Tear down the storage infrastructure, reversing {@link provisionStorage}: delete every environment's
|
|
172
|
+
* sweep worker first (they bind the bucket), then — only when `deleteStorage` is set — the buckets and
|
|
173
|
+
* everything in them. Idempotent end to end.
|
|
174
|
+
*
|
|
175
|
+
* `environments` is the project's declaration from the root `pithy.config.ts` (#241). Every declared
|
|
176
|
+
* environment is provisioned; an environment this skipped would be one the project deploys to with no
|
|
177
|
+
* resources behind it — the silence the closed `ManagedEnvironment` enum used to produce.
|
|
178
|
+
*/
|
|
179
|
+
export async function deprovisionStorage(
|
|
180
|
+
deprovisioner: StorageDeprovisioner,
|
|
181
|
+
environments: DeclaredEnvironments | readonly string[],
|
|
182
|
+
options: StorageDeprovisionOptions = {},
|
|
183
|
+
): Promise<void> {
|
|
184
|
+
for (const env of managedEnvironments(environments)) {
|
|
185
|
+
await deprovisioner.deleteWorker(env);
|
|
186
|
+
}
|
|
187
|
+
if (options.deleteStorage) {
|
|
188
|
+
for (const env of managedEnvironments(environments)) {
|
|
189
|
+
await deprovisioner.deleteBucket(env);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { hostWorkflowsFor, resolveWorkflowHost, type WorkflowHostTemplate } from "@pithy-sh/core/src/workflow/host";
|
|
5
|
+
import { masterKeySecretName } from "@pithy-sh/secrets/src/provision/provisionSecrets";
|
|
6
|
+
import type { ManagedEnvironment } from "@pithy-sh/secrets/src/scope";
|
|
7
|
+
import type { StorageConfig } from "../config/config";
|
|
8
|
+
import { STORAGE_CAPABILITY, storageWorkflowRegistry } from "../workflows/specs";
|
|
9
|
+
import type { StorageResources } from "./provisionStorage";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Resolve the sweep worker's committed `wrangler.jsonc` template into one environment's standalone
|
|
13
|
+
* config. Every per-environment decision lives here; everything static — the compatibility date, the
|
|
14
|
+
* bindings' names — stays as the template committed it.
|
|
15
|
+
*
|
|
16
|
+
* Thin over core's {@link resolveWorkflowHost}, which owns the mechanics (clone, fill by binding name,
|
|
17
|
+
* stamp `ENVIRONMENT`). What this file adds is one thing the generic resolver deliberately does not
|
|
18
|
+
* do: it **rewrites `workflows` and `triggers.crons` from the capability's specs** rather than from
|
|
19
|
+
* the template's own block. The template carries both so it reads as a complete, deployable config,
|
|
20
|
+
* but `workflows/specs.ts` is the single source of the binding name, the class name, and the
|
|
21
|
+
* schedule — so changing the sweep's cron is a one-line spec edit, not a spec edit plus a JSONC edit
|
|
22
|
+
* that nothing checks agree.
|
|
23
|
+
*
|
|
24
|
+
* Pure: the caller parses the template and writes the result.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
/** The resolved ids and per-env values for one environment's sweep-worker deploy. */
|
|
28
|
+
export interface StorageConfigParams {
|
|
29
|
+
/**
|
|
30
|
+
* The project name — the `<project>` segment the deployed worker and Workflow names lead with. The
|
|
31
|
+
* root `pithy.config.ts` `name`, resolved by `requireProjectName` and never guessed: a Worker script
|
|
32
|
+
* name is account-scoped, so a wrong value here overwrites another project's running host.
|
|
33
|
+
*/
|
|
34
|
+
project: string;
|
|
35
|
+
/** The target environment. */
|
|
36
|
+
env: ManagedEnvironment;
|
|
37
|
+
/** The app database id for this environment — where the `pithy_storage_*` tables live. */
|
|
38
|
+
appDatabaseId: string;
|
|
39
|
+
/** This environment's secrets database id (`<project>-<env>-secrets`) — holds the R2 credentials. */
|
|
40
|
+
secretsDatabaseId: string;
|
|
41
|
+
/** The CF Secrets Store id holding the per-env master key. */
|
|
42
|
+
storeId: string;
|
|
43
|
+
/** The provisioned bucket the worker binds. */
|
|
44
|
+
resources: StorageResources;
|
|
45
|
+
/** The app's resolved storage config — serialized into the worker's `STORAGE_CONFIG` var. */
|
|
46
|
+
storageConfig: StorageConfig;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Fill the template for one environment. */
|
|
50
|
+
export function resolveStorageConfig(
|
|
51
|
+
template: WorkflowHostTemplate,
|
|
52
|
+
params: StorageConfigParams,
|
|
53
|
+
): WorkflowHostTemplate {
|
|
54
|
+
const { project, env, appDatabaseId, secretsDatabaseId, storeId, resources, storageConfig } = params;
|
|
55
|
+
|
|
56
|
+
// Derived before the resolve rather than assigned after it: `resolveWorkflowHost` refuses to fill a
|
|
57
|
+
// template that declares `workflows` without them, because the only unscoped name it could invent is
|
|
58
|
+
// one a second project in the same account would collide with.
|
|
59
|
+
const derived = hostWorkflowsFor(storageWorkflowRegistry, { project, capability: STORAGE_CAPABILITY, env });
|
|
60
|
+
|
|
61
|
+
const resolved = resolveWorkflowHost(template, {
|
|
62
|
+
project,
|
|
63
|
+
capability: STORAGE_CAPABILITY,
|
|
64
|
+
env,
|
|
65
|
+
databaseIds: { DB: appDatabaseId, SECRETS: secretsDatabaseId },
|
|
66
|
+
r2BucketNames: { STORAGE_BUCKET: resources.bucketName },
|
|
67
|
+
secretsStoreId: storeId,
|
|
68
|
+
// The master key entry is project- and env-scoped, matching what the secrets manager wrote.
|
|
69
|
+
masterKeySecretName: masterKeySecretName(project, env),
|
|
70
|
+
vars: { STORAGE_CONFIG: JSON.stringify(storageConfig) },
|
|
71
|
+
workflows: derived.workflows,
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// Only declare a cron block when a spec actually carries one. An empty `crons` array is a
|
|
75
|
+
// declaration wrangler honors, and a worker that advertises a schedule it does not have is a
|
|
76
|
+
// deployment nobody can reason about.
|
|
77
|
+
resolved.triggers = derived.crons.length > 0 ? { crons: derived.crons } : undefined;
|
|
78
|
+
|
|
79
|
+
return resolved;
|
|
80
|
+
}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { sql } from "kysely";
|
|
5
|
+
import type { StorageObjectRow, StorageObjectStatus } from "../data/storageObject";
|
|
6
|
+
import { STORAGE_OBJECTS_TABLE, type StorageDatabase } from "../data/tables";
|
|
7
|
+
import { StorageQuotaExceededError } from "../error/errors";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Per-owner byte quotas.
|
|
11
|
+
*
|
|
12
|
+
* **Quota is checked when an upload *starts*, and the sum counts `pending` rows alongside `stored`
|
|
13
|
+
* ones.** That is the whole design, and it is not an optimization.
|
|
14
|
+
*
|
|
15
|
+
* Counting only completed uploads would make the check meaningless under concurrency: ten clients
|
|
16
|
+
* that each declare 1 GiB against a 1 GiB quota all read a used total of zero, all pass, and all
|
|
17
|
+
* upload. Nothing in that sequence is a bug in any single request — the check is simply measuring the
|
|
18
|
+
* wrong thing. Making the client declare its size up front, and having the `pending` row *reserve*
|
|
19
|
+
* those bytes the moment it is written, turns the quota into something a concurrent burst cannot
|
|
20
|
+
* walk past: the second init reads the first's reservation.
|
|
21
|
+
*
|
|
22
|
+
* **The reservation only holds if the sum and the insert are one statement.** A `SELECT sum(size)`
|
|
23
|
+
* followed by a separate `INSERT` reproduces the very race it was meant to close, one round trip
|
|
24
|
+
* later: ten concurrent inits all read a used total that none of them has yet contributed to, all
|
|
25
|
+
* pass, and all insert. D1 has no interactive transactions, so a `BEGIN`/`COMMIT` wrapper is not
|
|
26
|
+
* available — {@link insertReservingQuota} is the mechanism instead. It writes the pending row with a
|
|
27
|
+
* conditional `INSERT … SELECT … WHERE`, so the sum is evaluated *inside* the write, and a row that
|
|
28
|
+
* would breach the limit simply does not appear.
|
|
29
|
+
*
|
|
30
|
+
* {@link assertWithinQuota} survives as the cheap pre-check a handler runs before it creates anything
|
|
31
|
+
* that would need cleaning up. It is a courtesy, not the enforcement.
|
|
32
|
+
*
|
|
33
|
+
* **Completion settles the same way it reserved.** A part URL carries no signed `Content-Length`, so
|
|
34
|
+
* an owner may PUT more than they declared, and the overshoot has to be re-asserted when the row turns
|
|
35
|
+
* `stored`. That re-assertion is {@link updateSettlingQuota} — a conditional `UPDATE`, the same shape
|
|
36
|
+
* as the conditional `INSERT` — because two completions racing on a plain check both read a total
|
|
37
|
+
* neither has yet contributed and both pass. The rule is one rule: every write that changes what an
|
|
38
|
+
* owner holds evaluates the sum inside itself.
|
|
39
|
+
*
|
|
40
|
+
* The reservation is not free — an abandoned upload holds bytes it never stored. That is what
|
|
41
|
+
* `pendingTtlSeconds` and the orphan sweep are for: a `pending` row past its TTL is aborted and
|
|
42
|
+
* dropped, returning its reservation. Over-counting briefly and reclaiming is the right way round;
|
|
43
|
+
* under-counting has no recovery.
|
|
44
|
+
*
|
|
45
|
+
* A `null` limit is unlimited and skips the query entirely. A `null` owner (a system object) has no
|
|
46
|
+
* quota — there is no principal to bill it to.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
/** What a quota check needs. */
|
|
50
|
+
export interface QuotaCheck {
|
|
51
|
+
/** The storage database. */
|
|
52
|
+
db: StorageDatabase;
|
|
53
|
+
/** The owner the bytes are billed to. Null is a system object, which no quota applies to. */
|
|
54
|
+
ownerId: string | null;
|
|
55
|
+
/** The configured per-owner ceiling in bytes. Null is unlimited. */
|
|
56
|
+
limitBytes: number | null;
|
|
57
|
+
/** The bytes this upload declares — what the pending row is about to reserve. */
|
|
58
|
+
additionalBytes: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** A quota check plus the row whose insertion *is* the reservation. */
|
|
62
|
+
export interface QuotaReservation extends QuotaCheck {
|
|
63
|
+
/** The row to write, already through `StorageObject.encode` — the SQLite shape, never the app one. */
|
|
64
|
+
record: StorageObjectRow;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A quota check plus the row whose update *is* the settlement. */
|
|
68
|
+
export interface QuotaSettlement extends QuotaCheck {
|
|
69
|
+
/**
|
|
70
|
+
* The whole row to write, already through `StorageObject.encode`. Its `id` is what the update
|
|
71
|
+
* targets — the settlement rewrites one known row rather than a set.
|
|
72
|
+
*/
|
|
73
|
+
record: StorageObjectRow;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The statuses that bill an owner: a `pending` row reserves its declared bytes and a `stored` one
|
|
78
|
+
* holds real ones. `failed` bills nothing.
|
|
79
|
+
*
|
|
80
|
+
* Exported because the orphan sweep asks the same question of the same rows — a key is *claimed* if
|
|
81
|
+
* and only if a row bills for it — and the two predicates drifting apart is what let a `failed` row
|
|
82
|
+
* shield its R2 object from collection while counting toward nobody's quota
|
|
83
|
+
* (`workflows/sweep.ts`).
|
|
84
|
+
*/
|
|
85
|
+
export const QUOTA_COUNTED_STATUSES: readonly StorageObjectStatus[] = ["pending", "stored"];
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Bytes an owner currently holds: every `pending` reservation plus every `stored` object. `failed`
|
|
89
|
+
* rows are excluded — they hold no bytes and reserve nothing.
|
|
90
|
+
*/
|
|
91
|
+
export async function usedBytes(db: StorageDatabase, ownerId: string): Promise<number> {
|
|
92
|
+
const row = await db
|
|
93
|
+
.selectFrom(STORAGE_OBJECTS_TABLE)
|
|
94
|
+
// `size` is nullable (a caller may not declare one), and SUM over no rows is NULL — coalesce both
|
|
95
|
+
// to zero here rather than letting a null leak into arithmetic.
|
|
96
|
+
.select(sql<number>`coalesce(sum(size), 0)`.as("used"))
|
|
97
|
+
.where("ownerId", "=", ownerId)
|
|
98
|
+
.where("status", "in", [...QUOTA_COUNTED_STATUSES])
|
|
99
|
+
.executeTakeFirst();
|
|
100
|
+
return Number(row?.used ?? 0);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** A size a caller declared has to be a real byte count. Zero is fine; negative is a client bug. */
|
|
104
|
+
function assertDeclaredSize(additionalBytes: number): void {
|
|
105
|
+
if (additionalBytes >= 0) return;
|
|
106
|
+
throw new StorageQuotaExceededError({
|
|
107
|
+
message: "An upload needs a size.",
|
|
108
|
+
action: "Declare the file's byte count when you start the upload.",
|
|
109
|
+
detail: `negative declared size ${additionalBytes}`,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Throw `storage/quota_exceeded` when this upload would put the owner over their limit. The
|
|
115
|
+
* comparison is `used + additional > limit`, so an upload landing exactly on the limit is allowed —
|
|
116
|
+
* a quota is a ceiling you may reach, not one you must stay under.
|
|
117
|
+
*
|
|
118
|
+
* **This is a pre-check, not the reservation.** It reads a total nothing holds still, so between it
|
|
119
|
+
* and any later write a concurrent init can have taken the room. Use it to refuse an obviously
|
|
120
|
+
* oversized upload *before* opening an R2 multipart upload there would then be nothing to clean up
|
|
121
|
+
* for, and let {@link insertReservingQuota} be the boundary that actually decides.
|
|
122
|
+
*/
|
|
123
|
+
export async function assertWithinQuota(check: QuotaCheck): Promise<void> {
|
|
124
|
+
if (check.limitBytes === null || check.ownerId === null) return;
|
|
125
|
+
assertDeclaredSize(check.additionalBytes);
|
|
126
|
+
|
|
127
|
+
const used = await usedBytes(check.db, check.ownerId);
|
|
128
|
+
if (used + check.additionalBytes > check.limitBytes) {
|
|
129
|
+
throw new StorageQuotaExceededError({
|
|
130
|
+
detail: `owner would hold ${used + check.additionalBytes} bytes against a limit of ${check.limitBytes}`,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Insert a storage row **only if** it fits inside the owner's quota, and throw
|
|
137
|
+
* `storage/quota_exceeded` when it does not.
|
|
138
|
+
*
|
|
139
|
+
* The sum and the write are one statement — `INSERT … SELECT <values> WHERE (SELECT sum(size) …) <=
|
|
140
|
+
* limit - additional` — because two statements cannot be made to hold a reservation without a
|
|
141
|
+
* transaction, and D1 has none to offer. SQLite evaluates the sub-select while it holds the write
|
|
142
|
+
* lock, so a burst of concurrent inits is serialized by the database itself: each one sees every
|
|
143
|
+
* reservation that landed before it, and the first that would breach the limit inserts nothing.
|
|
144
|
+
*
|
|
145
|
+
* `RETURNING id` is how "inserted nothing" is read back. Zero rows is not an error to SQLite — the
|
|
146
|
+
* `WHERE` simply matched nothing — so the absence of a returned row *is* the quota answer.
|
|
147
|
+
*/
|
|
148
|
+
export async function insertReservingQuota(reservation: QuotaReservation): Promise<void> {
|
|
149
|
+
const { db, ownerId, limitBytes, additionalBytes, record } = reservation;
|
|
150
|
+
if (limitBytes === null || ownerId === null) {
|
|
151
|
+
await db.insertInto(STORAGE_OBJECTS_TABLE).values(record).execute();
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
assertDeclaredSize(additionalBytes);
|
|
155
|
+
|
|
156
|
+
// The column list is read off the encoded record rather than spelled out, so a new column on
|
|
157
|
+
// `StorageObject` is carried here without an edit — and stays camelCase, which `CamelCasePlugin`
|
|
158
|
+
// snake-cases on the way out.
|
|
159
|
+
const columns = Object.keys(record) as (keyof StorageObjectRow)[];
|
|
160
|
+
const values = db
|
|
161
|
+
.selectNoFrom((eb) => columns.map((column) => eb.val(record[column]).as(column)))
|
|
162
|
+
// `used <= limit - additional` rather than `used + additional <= limit`: same integers, but the
|
|
163
|
+
// owner's total stays alone on the left where the sub-select can be compared against a constant.
|
|
164
|
+
.where((eb) =>
|
|
165
|
+
eb(
|
|
166
|
+
eb
|
|
167
|
+
.selectFrom(STORAGE_OBJECTS_TABLE)
|
|
168
|
+
.select(sql<number>`coalesce(sum(size), 0)`.as("used"))
|
|
169
|
+
.where("ownerId", "=", ownerId)
|
|
170
|
+
.where("status", "in", [...QUOTA_COUNTED_STATUSES]),
|
|
171
|
+
"<=",
|
|
172
|
+
limitBytes - additionalBytes,
|
|
173
|
+
),
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
const inserted = await db
|
|
177
|
+
.insertInto(STORAGE_OBJECTS_TABLE)
|
|
178
|
+
.columns(columns)
|
|
179
|
+
.expression(values)
|
|
180
|
+
.returning("id")
|
|
181
|
+
.executeTakeFirst();
|
|
182
|
+
|
|
183
|
+
if (!inserted) {
|
|
184
|
+
throw new StorageQuotaExceededError({
|
|
185
|
+
detail: `reservation of ${additionalBytes} bytes lost against a limit of ${limitBytes}`,
|
|
186
|
+
});
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Rewrite one row **only if** the bytes it newly claims still fit inside the owner's quota, and throw
|
|
192
|
+
* `storage/quota_exceeded` when they do not.
|
|
193
|
+
*
|
|
194
|
+
* This is {@link insertReservingQuota}'s other half, and it exists for the identical reason. A
|
|
195
|
+
* completion measures what R2 actually holds against what the pending row reserved, and the difference
|
|
196
|
+
* is bytes the owner has not been granted yet. Settling that difference with a separate
|
|
197
|
+
* {@link assertWithinQuota} would put the enforcement back where it never worked: two completions
|
|
198
|
+
* racing both read a total neither has yet contributed to, both pass, and the owner ends over the
|
|
199
|
+
* limit — the same defect the conditional insert was written to close, one lifecycle stage later.
|
|
200
|
+
*
|
|
201
|
+
* `additionalBytes` is the **overshoot**, not the object's size. The row being updated is still
|
|
202
|
+
* `pending`, so the sub-select in the `WHERE` already counts its reservation: SQLite evaluates the
|
|
203
|
+
* condition against the pre-update table. Passing the full size would bill the reservation twice.
|
|
204
|
+
*
|
|
205
|
+
* `RETURNING id` reads back whether the row moved. Zero rows updated is not an error to SQLite — the
|
|
206
|
+
* `WHERE` simply matched nothing — so the absence of a returned row *is* the quota answer.
|
|
207
|
+
*/
|
|
208
|
+
export async function updateSettlingQuota(settlement: QuotaSettlement): Promise<void> {
|
|
209
|
+
const { db, ownerId, limitBytes, additionalBytes, record } = settlement;
|
|
210
|
+
if (limitBytes === null || ownerId === null) {
|
|
211
|
+
await db.updateTable(STORAGE_OBJECTS_TABLE).set(record).where("id", "=", record.id).execute();
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
assertDeclaredSize(additionalBytes);
|
|
215
|
+
|
|
216
|
+
const updated = await db
|
|
217
|
+
.updateTable(STORAGE_OBJECTS_TABLE)
|
|
218
|
+
.set(record)
|
|
219
|
+
.where("id", "=", record.id)
|
|
220
|
+
// Same comparison as the reservation — `used <= limit - additional` — so both halves of the
|
|
221
|
+
// lifecycle answer the same arithmetic and one of them cannot drift.
|
|
222
|
+
.where((eb) =>
|
|
223
|
+
eb(
|
|
224
|
+
eb
|
|
225
|
+
.selectFrom(STORAGE_OBJECTS_TABLE)
|
|
226
|
+
.select(sql<number>`coalesce(sum(size), 0)`.as("used"))
|
|
227
|
+
.where("ownerId", "=", ownerId)
|
|
228
|
+
.where("status", "in", [...QUOTA_COUNTED_STATUSES]),
|
|
229
|
+
"<=",
|
|
230
|
+
limitBytes - additionalBytes,
|
|
231
|
+
),
|
|
232
|
+
)
|
|
233
|
+
.returning("id")
|
|
234
|
+
.executeTakeFirst();
|
|
235
|
+
|
|
236
|
+
if (!updated) {
|
|
237
|
+
throw new StorageQuotaExceededError({
|
|
238
|
+
detail: `settlement of ${additionalBytes} further bytes lost against a limit of ${limitBytes}`,
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { R2Credentials } from "@pithy-sh/cloudflare/src/r2/r2Credentials";
|
|
5
|
+
import type { SecretOrigin, SecretRotation } from "@pithy-sh/core/src/capability/secretOrigin";
|
|
6
|
+
import { defineSecretRegistry, type SecretRegistry } from "@pithy-sh/secrets/src/registry";
|
|
7
|
+
import { z } from "zod";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The R2 credential bundle the {@link ObjectStore} seam reads, and the **factory** that declares it
|
|
11
|
+
* under any name (CLAUDE.md §Secrets).
|
|
12
|
+
*
|
|
13
|
+
* Why a factory and not a bare name. `sharedSecretsStore(env, registry)` throws for any name absent
|
|
14
|
+
* from the *aggregated* registry, and that registry is built only from the composed capabilities'
|
|
15
|
+
* `secretRegistry` slices. So `objectStore({ bucket, secretName })` cannot resolve a string on its
|
|
16
|
+
* own — something must have declared that name, with a schema, on a capability. The factory is that
|
|
17
|
+
* something: storage declares `storage-r2-credentials` for `STORAGE_BUCKET`, `@pithy-sh/media`
|
|
18
|
+
* declares `media-r2-credentials` for `MEDIA_BUCKET`, and both point the same seam at their own
|
|
19
|
+
* bucket without either package knowing the other's name.
|
|
20
|
+
*
|
|
21
|
+
* One factory is also what makes the join key safe. `aggregateSecretRegistries` allows a name to be
|
|
22
|
+
* declared twice only when every axis agrees (`backend`, `scope`, `valueType`, `rotatable`); two
|
|
23
|
+
* hand-written declarations drift, one factory cannot.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The credentials one R2 bucket is addressed and presigned with. Composed from
|
|
28
|
+
* {@link R2Credentials} rather than redeclaring the key pair — that shape is `@pithy-sh/cloudflare`'s
|
|
29
|
+
* to define, and `CloudflareR2Manager` validates its config against it.
|
|
30
|
+
*/
|
|
31
|
+
export const R2StorageCredentials = R2Credentials.extend({
|
|
32
|
+
accountId: z.string().min(1).describe("The Cloudflare account id the bucket lives in — the S3 endpoint host."),
|
|
33
|
+
bucket: z.string().min(1).describe("The R2 bucket name every presigned URL and server-side object call targets."),
|
|
34
|
+
apiToken: z
|
|
35
|
+
.string()
|
|
36
|
+
.min(1)
|
|
37
|
+
.describe(
|
|
38
|
+
"The R2-scoped Cloudflare API token the key pair was derived from. Carried alongside because an R2 S3 access key IS a CF API token — the id is the key id, the SHA-256 of the value is the secret — so whatever provisions the pair already holds it, and `CloudflareR2Manager` needs it to prove bucket access.",
|
|
39
|
+
),
|
|
40
|
+
}).describe("The account, key pair, scoped token, and bucket name the object store presigns and addresses R2 with.");
|
|
41
|
+
export type R2StorageCredentials = z.output<typeof R2StorageCredentials>;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* One R2-credentials registry entry. Spelled out as a type so the factory can return
|
|
45
|
+
* `Record<N, R2CredentialsEntry>` — a computed key alone widens to a string index signature, which
|
|
46
|
+
* would lose the literal name `SecretsAccessor.get` narrows on.
|
|
47
|
+
*/
|
|
48
|
+
export interface R2CredentialsEntry {
|
|
49
|
+
/**
|
|
50
|
+
* An encrypted row in the per-environment secrets D1 — where these values actually live.
|
|
51
|
+
*
|
|
52
|
+
* No `wrangler.jsonc` binds an R2 credential bundle from the Cloudflare Secrets Store; `pithy storage
|
|
53
|
+
* provision` and `pithy media provision` write it through `dispatchSecretWrite` → the manager
|
|
54
|
+
* Workflow → `SystemSecretsStore`, which is the D1 path. The registry's `backend` is the *single*
|
|
55
|
+
* place a secret's storage location is decided and is what the read seam routes on, so declaring
|
|
56
|
+
* `cf-secrets-store` here would send every deployed read to a binding that does not exist.
|
|
57
|
+
*/
|
|
58
|
+
backend: "d1";
|
|
59
|
+
/** Each environment addresses its own bucket with its own key pair. */
|
|
60
|
+
scope: "environment";
|
|
61
|
+
/** R2 S3 key pairs are minted and replaced whole, not rotated with overlap windows. */
|
|
62
|
+
rotatable: false;
|
|
63
|
+
/** A JSON bundle, validated against {@link R2StorageCredentials} before it is exposed. */
|
|
64
|
+
valueType: "json";
|
|
65
|
+
/** The bundle's shape. */
|
|
66
|
+
schema: typeof R2StorageCredentials;
|
|
67
|
+
/** Where the pair comes from. `obtained`, always — see {@link R2_CREDENTIALS_PAGE}. */
|
|
68
|
+
origin: SecretOrigin;
|
|
69
|
+
/** How the pair is replaced. `manual`, always — the same page, by the same human. */
|
|
70
|
+
rotation: SecretRotation;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Where a human makes an R2 S3 access-key pair, and where the same human makes the next one.
|
|
75
|
+
*
|
|
76
|
+
* **Nothing here will ever mint one, and that is the fact the declaration carries.** Cloudflare has no
|
|
77
|
+
* API that returns an S3 access-key pair, so `pithy storage provision` and `pithy media provision` both
|
|
78
|
+
* take one as a flag and write it as given — a generated value would open no bucket and would replace a
|
|
79
|
+
* loud gap with a quiet one. Naming the page is the whole of the help the kit can offer, which is exactly
|
|
80
|
+
* what `obtained` is for.
|
|
81
|
+
*
|
|
82
|
+
* Origin and rotation name the same page because it is the same page: replacement is making another pair
|
|
83
|
+
* and deleting the old one. That is also why {@link R2CredentialsEntry.rotatable} is false — there is no
|
|
84
|
+
* overlap window to hold two live versions through.
|
|
85
|
+
*/
|
|
86
|
+
const R2_CREDENTIALS_PAGE = "https://developers.cloudflare.com/r2/api/tokens/";
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The one entry shape every name shares — declared once so no two declarations can disagree.
|
|
90
|
+
*
|
|
91
|
+
* The two declaration axes live here rather than at each name for the reason the rest of the entry does:
|
|
92
|
+
* `storage-r2-credentials` and `media-r2-credentials` are the same kind of credential from the same
|
|
93
|
+
* issuer, and `aggregateSecretRegistries` refuses a name two capabilities describe differently. One
|
|
94
|
+
* factory is how they cannot.
|
|
95
|
+
*/
|
|
96
|
+
const R2_CREDENTIALS_ENTRY: R2CredentialsEntry = {
|
|
97
|
+
backend: "d1",
|
|
98
|
+
scope: "environment",
|
|
99
|
+
rotatable: false,
|
|
100
|
+
valueType: "json",
|
|
101
|
+
schema: R2StorageCredentials,
|
|
102
|
+
origin: { kind: "obtained", issuer: "cloudflare", documentation: R2_CREDENTIALS_PAGE },
|
|
103
|
+
rotation: { kind: "manual", issuer: "cloudflare", documentation: R2_CREDENTIALS_PAGE },
|
|
104
|
+
};
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* A one-entry secret-registry slice declaring `name` as an R2 credential bundle. Hang it on the
|
|
108
|
+
* declaring capability's `secretRegistry`, then pass the same `name` to `objectStore`.
|
|
109
|
+
*/
|
|
110
|
+
export function r2CredentialsRegistry<const N extends string>(name: N): Record<N, R2CredentialsEntry> {
|
|
111
|
+
return defineSecretRegistry({ [name]: R2_CREDENTIALS_ENTRY } as Record<N, R2CredentialsEntry> & SecretRegistry);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** The name storage's own bucket credentials are stored and resolved under. */
|
|
115
|
+
export const STORAGE_R2_SECRET = "storage-r2-credentials";
|
|
116
|
+
|
|
117
|
+
/** The storage capability's secret-registry slice — aggregated into the shared accessor at startup. */
|
|
118
|
+
export const storageSecretsRegistry = r2CredentialsRegistry(STORAGE_R2_SECRET);
|