@pithy-sh/storage 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capability.d.ts +12 -3
- package/dist/capability.d.ts.map +1 -1
- package/dist/capability.js +35 -9
- package/dist/config/config.d.ts +20 -4
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +23 -6
- package/dist/data/storageObject.d.ts +12 -0
- package/dist/data/storageObject.d.ts.map +1 -1
- package/dist/data/storageObject.js +13 -2
- package/dist/data/tables.js +1 -1
- package/dist/error/errors.js +2 -2
- package/dist/http/handlers.d.ts +45 -14
- package/dist/http/handlers.d.ts.map +1 -1
- package/dist/http/handlers.js +169 -66
- package/dist/http/responses.d.ts +6 -0
- package/dist/http/responses.d.ts.map +1 -1
- package/dist/http/responses.js +2 -1
- package/dist/http/routes.d.ts +13 -2
- package/dist/http/routes.d.ts.map +1 -1
- package/dist/http/routes.js +37 -9
- package/dist/http/schemas.d.ts +6 -0
- package/dist/http/schemas.d.ts.map +1 -1
- package/dist/http/schemas.js +6 -5
- package/dist/http/scopes.d.ts +25 -0
- package/dist/http/scopes.d.ts.map +1 -0
- package/dist/http/scopes.js +32 -0
- package/dist/http/serve.d.ts +5 -2
- package/dist/http/serve.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/migrations/0002_ownership.d.ts +5 -0
- package/dist/migrations/0002_ownership.d.ts.map +1 -0
- package/dist/migrations/0002_ownership.js +114 -0
- package/dist/object/cloudflare.d.ts +2 -1
- package/dist/object/cloudflare.d.ts.map +1 -1
- package/dist/object/cloudflare.js +2 -2
- package/dist/provision/provisionStorage.d.ts +14 -2
- package/dist/provision/provisionStorage.d.ts.map +1 -1
- package/dist/provision/provisionStorage.js +19 -4
- package/dist/provision/resolveStorageConfig.js +1 -1
- package/dist/quota/ceiling.d.ts +36 -0
- package/dist/quota/ceiling.d.ts.map +1 -0
- package/dist/quota/ceiling.js +31 -0
- package/dist/quota/quota.d.ts +24 -7
- package/dist/quota/quota.d.ts.map +1 -1
- package/dist/quota/quota.js +40 -12
- package/dist/secret/bundleToken.d.ts +24 -0
- package/dist/secret/bundleToken.d.ts.map +1 -0
- package/dist/secret/bundleToken.js +74 -0
- package/dist/secret/registry.d.ts +5 -1
- package/dist/secret/registry.d.ts.map +1 -1
- package/dist/secret/registry.js +7 -3
- package/dist/seeds/example.d.ts.map +1 -1
- package/dist/seeds/example.js +3 -0
- package/dist/test-utils/liveStorage.d.ts.map +1 -1
- package/dist/test-utils/liveStorage.js +5 -2
- package/dist/version.generated.d.ts +1 -1
- package/dist/version.generated.js +1 -1
- package/package.json +5 -5
- package/pithy.manifest.json +20 -10
- package/src/capability.ts +52 -8
- package/src/config/config.ts +24 -6
- package/src/data/storageObject.ts +22 -6
- package/src/error/errors.ts +2 -2
- package/src/http/handlers.ts +212 -79
- package/src/http/responses.ts +13 -1
- package/src/http/routes.ts +64 -10
- package/src/http/schemas.ts +10 -4
- package/src/http/scopes.ts +39 -0
- package/src/http/serve.ts +5 -2
- package/src/index.ts +2 -0
- package/src/migrations/0002_ownership.ts +176 -0
- package/src/object/cloudflare.ts +2 -2
- package/src/provision/provisionStorage.ts +22 -3
- package/src/quota/ceiling.ts +44 -0
- package/src/quota/quota.ts +63 -20
- package/src/secret/bundleToken.ts +81 -0
- package/src/secret/registry.ts +6 -8
- package/src/seeds/example.ts +3 -0
- package/src/test-utils/liveStorage.ts +3 -6
- package/src/version.generated.ts +1 -1
package/src/http/routes.ts
CHANGED
|
@@ -2,16 +2,21 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
|
|
4
4
|
import type { D1Database, R2Bucket } from "@cloudflare/workers-types";
|
|
5
|
+
import { accessCaller, type PermissionHook } from "@pithy-sh/core/src/access/access";
|
|
6
|
+
import { UsageQuery, UsageResponse, usageRollup } from "@pithy-sh/core/src/access/usage";
|
|
5
7
|
import type { PithyHonoEnv } from "@pithy-sh/core/src/capability/capability";
|
|
8
|
+
import { requireControlPlane } from "@pithy-sh/core/src/controlPlane/http/guard";
|
|
6
9
|
import { InternalError } from "@pithy-sh/core/src/error/pithyError";
|
|
7
10
|
import { describeRoute, type RouteResponse } from "@pithy-sh/core/src/http/routeSpec";
|
|
8
11
|
import { validate } from "@pithy-sh/core/src/http/validation";
|
|
9
12
|
import type { VerificationStrategy } from "@pithy-sh/core/src/http/verification";
|
|
10
13
|
import type { SecretsStoreEnv } from "@pithy-sh/secrets/src/env/bindings";
|
|
11
14
|
import type { Context, Hono } from "hono";
|
|
15
|
+
import type { Kysely } from "kysely";
|
|
12
16
|
import type { StorageConfig } from "../config/config";
|
|
13
|
-
import { storageDatabase } from "../data/tables";
|
|
17
|
+
import { STORAGE_OBJECTS_TABLE, storageDatabase } from "../data/tables";
|
|
14
18
|
import { objectStore } from "../object/store";
|
|
19
|
+
import { storageCeiling } from "../quota/ceiling";
|
|
15
20
|
import { STORAGE_R2_SECRET } from "../secret/registry";
|
|
16
21
|
import { requireAuth } from "./guard";
|
|
17
22
|
import {
|
|
@@ -51,6 +56,7 @@ import {
|
|
|
51
56
|
ShareTokenParam,
|
|
52
57
|
UpdateObjectInput,
|
|
53
58
|
} from "./schemas";
|
|
59
|
+
import { STORAGE_USAGE_READ_SCOPE } from "./scopes";
|
|
54
60
|
import {
|
|
55
61
|
parseConditions,
|
|
56
62
|
parseRangeHeader,
|
|
@@ -65,8 +71,10 @@ import {
|
|
|
65
71
|
*
|
|
66
72
|
* POST /storage → start an upload (bearer | session)
|
|
67
73
|
* json: CreateUploadInput
|
|
68
|
-
* GET /storage → list
|
|
74
|
+
* GET /storage → list the files you may see (bearer | session)
|
|
69
75
|
* query: ListObjectsQuery
|
|
76
|
+
* GET /storage/admin/usage → files and bytes held (control-plane, storage:usage:read)
|
|
77
|
+
* query: UsageQuery
|
|
70
78
|
* DELETE /storage/shares/:token → revoke a share (bearer | session, owner-scoped)
|
|
71
79
|
* param: ShareTokenParam
|
|
72
80
|
* POST /storage/:id/complete → finalize a multipart (bearer | session, owner-scoped)
|
|
@@ -82,7 +90,7 @@ import {
|
|
|
82
90
|
* GET /storage/:id/url → presigned direct URL (bearer | session, owner-or-public)
|
|
83
91
|
* param: ObjectIdParam
|
|
84
92
|
* GET /storage/:id → stream the bytes (public when `visibility: 'public'`,
|
|
85
|
-
* otherwise bearer | session +
|
|
93
|
+
* otherwise bearer | session + readable)
|
|
86
94
|
* param: ObjectIdParam, query: ObjectReadQuery
|
|
87
95
|
* HEAD /storage/:id → metadata only (bearer | session, owner-scoped)
|
|
88
96
|
* param: ObjectIdParam
|
|
@@ -98,6 +106,12 @@ import {
|
|
|
98
106
|
* /storage/:id/abort` carries no json validator because it reads no body: adding one would 400 the
|
|
99
107
|
* bodyless POST every client sends today.
|
|
100
108
|
*
|
|
109
|
+
* **"Owner-scoped" above means the access rules, not the owner alone.** Every route reads a file only if its
|
|
110
|
+
* visibility admits the caller — its owner acting for its organization, any member for `organization`,
|
|
111
|
+
* anyone for `public` — and changes one only for its owner or a role holding `storage:manage` in its
|
|
112
|
+
* organization, with the adopter's permission hook asked about both. A file the caller may not read is
|
|
113
|
+
* not found, never forbidden (`handlers.ts`).
|
|
114
|
+
*
|
|
101
115
|
* `GET /storage/:id` is the one route with two strategies, because a public file must be readable by
|
|
102
116
|
* someone with no session at all. It is therefore **not** wrapped in {@link requireAuth}: the guard
|
|
103
117
|
* would deny before the handler could see that the object is public. Authorization happens inside the
|
|
@@ -118,6 +132,8 @@ export interface StorageRoutesOptions {
|
|
|
118
132
|
basePath?: string;
|
|
119
133
|
/** Where share fetches mount. Defaults to `/s` — short, because the whole URL gets pasted around. */
|
|
120
134
|
sharePath?: string;
|
|
135
|
+
/** The adopter's permission hook: allow, deny, or the default rules, asked on every file read and write. */
|
|
136
|
+
permission?: PermissionHook;
|
|
121
137
|
/** Test seam: resolve handler deps from the request. Defaults to the env-based resolver. */
|
|
122
138
|
resolveDeps?: (c: Context<PithyHonoEnv>) => Promise<HandlerDeps>;
|
|
123
139
|
}
|
|
@@ -147,7 +163,10 @@ function requireBinding<T>(env: unknown, name: string, kind: string): T {
|
|
|
147
163
|
}
|
|
148
164
|
|
|
149
165
|
/** Build the default per-request dependency resolver from the resolved config. */
|
|
150
|
-
function defaultResolveDeps(
|
|
166
|
+
function defaultResolveDeps(
|
|
167
|
+
config: StorageConfig,
|
|
168
|
+
permission: PermissionHook | undefined,
|
|
169
|
+
): (c: Context<PithyHonoEnv>) => Promise<HandlerDeps> {
|
|
151
170
|
return async (c) => {
|
|
152
171
|
const env = c.env as unknown as StorageEnv;
|
|
153
172
|
const bucket = requireBinding<R2Bucket>(env, "STORAGE_BUCKET", "an R2 bucket");
|
|
@@ -158,16 +177,31 @@ function defaultResolveDeps(config: StorageConfig): (c: Context<PithyHonoEnv>) =
|
|
|
158
177
|
// binding never touches the secrets store.
|
|
159
178
|
store: objectStore({ bucket, env, secretName: STORAGE_R2_SECRET }),
|
|
160
179
|
config,
|
|
161
|
-
|
|
180
|
+
caller: await accessCaller(c.var),
|
|
181
|
+
...(permission ? { permission } : {}),
|
|
182
|
+
env: c.env,
|
|
162
183
|
newId: () => crypto.randomUUID(),
|
|
163
184
|
// 32 bytes of CSPRNG output, base64url. The token *is* the credential for a share link, so its
|
|
164
185
|
// entropy is the only thing standing between a link and anyone guessing one.
|
|
165
186
|
newToken: () => shareToken(),
|
|
166
187
|
now: () => new Date(),
|
|
188
|
+
// The ceiling from the `storageBytes` meter where the entitlement provider enforces one — the
|
|
189
|
+
// caller's plan, or what was pinned for them — and the configured `bytesPerOwner` where it does not.
|
|
190
|
+
// Read once per request, and only by a write that needs it.
|
|
191
|
+
quotaLimit: memoized(() => storageCeiling(c, config)),
|
|
167
192
|
};
|
|
168
193
|
};
|
|
169
194
|
}
|
|
170
195
|
|
|
196
|
+
/** One computation per request, however many writes ask for it. */
|
|
197
|
+
function memoized<T>(compute: () => Promise<T>): () => Promise<T> {
|
|
198
|
+
let pending: Promise<T> | undefined;
|
|
199
|
+
return () => {
|
|
200
|
+
pending ??= compute();
|
|
201
|
+
return pending;
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
171
205
|
/** A 256-bit share token, base64url-encoded and unpadded so it is safe in a path segment. */
|
|
172
206
|
function shareToken(): string {
|
|
173
207
|
const bytes = crypto.getRandomValues(new Uint8Array(32));
|
|
@@ -193,7 +227,7 @@ const BYTE_RESPONSES: Readonly<Record<number, RouteResponse>> = {
|
|
|
193
227
|
export function registerStorageRoutes(options: StorageRoutesOptions): (app: Hono<PithyHonoEnv>) => void {
|
|
194
228
|
const base = options.basePath ?? "/storage";
|
|
195
229
|
const share = options.sharePath ?? "/s";
|
|
196
|
-
const resolve = options.resolveDeps ?? defaultResolveDeps(options.config);
|
|
230
|
+
const resolve = options.resolveDeps ?? defaultResolveDeps(options.config, options.permission);
|
|
197
231
|
|
|
198
232
|
/** Stream one object, honoring `Range` and `If-None-Match`. Shared by the id route and the share route. */
|
|
199
233
|
const stream = async (
|
|
@@ -244,7 +278,9 @@ export function registerStorageRoutes(options: StorageRoutesOptions): (app: Hono
|
|
|
244
278
|
app.get(
|
|
245
279
|
base,
|
|
246
280
|
describeRoute({
|
|
247
|
-
summary: "List the caller
|
|
281
|
+
summary: "List the files the caller may see.",
|
|
282
|
+
description:
|
|
283
|
+
"The caller's own files and their organization's, by path. `includePublic=true` adds every public file.",
|
|
248
284
|
verification: AUTHENTICATED,
|
|
249
285
|
responses: { 200: ObjectListResponse },
|
|
250
286
|
}),
|
|
@@ -256,6 +292,25 @@ export function registerStorageRoutes(options: StorageRoutesOptions): (app: Hono
|
|
|
256
292
|
);
|
|
257
293
|
|
|
258
294
|
// Static before `:id` — see the file doc comment.
|
|
295
|
+
app.get(
|
|
296
|
+
`${base}/admin/usage`,
|
|
297
|
+
describeRoute({
|
|
298
|
+
summary: "Count files held, per organization or per owner.",
|
|
299
|
+
description:
|
|
300
|
+
"Stored files and their bytes. A per-owner read names its organization, so no figure ever spans two.",
|
|
301
|
+
verification: "control-plane",
|
|
302
|
+
scopes: [STORAGE_USAGE_READ_SCOPE],
|
|
303
|
+
responses: { 200: UsageResponse },
|
|
304
|
+
}),
|
|
305
|
+
requireControlPlane(STORAGE_USAGE_READ_SCOPE),
|
|
306
|
+
validate("query", UsageQuery),
|
|
307
|
+
async (c) => {
|
|
308
|
+
const d1 = requireBinding<D1Database>(c.env, "DB", "a D1 database");
|
|
309
|
+
const db = storageDatabase(d1) as unknown as Kysely<unknown>;
|
|
310
|
+
return c.json((await usageRollup(db, STORAGE_OBJECTS_TABLE, c.req.valid("query"))) satisfies UsageResponse);
|
|
311
|
+
},
|
|
312
|
+
);
|
|
313
|
+
|
|
259
314
|
app.delete(
|
|
260
315
|
`${base}/shares/:token`,
|
|
261
316
|
describeRoute({
|
|
@@ -355,8 +410,7 @@ export function registerStorageRoutes(options: StorageRoutesOptions): (app: Hono
|
|
|
355
410
|
`${base}/:id/url`,
|
|
356
411
|
describeRoute({
|
|
357
412
|
summary: "Mint a direct download URL.",
|
|
358
|
-
description:
|
|
359
|
-
"For a public file or one the caller owns. The URL cannot be revoked, so it lapses in five minutes.",
|
|
413
|
+
description: "For a file the caller may read. The URL cannot be revoked, so it lapses in five minutes.",
|
|
360
414
|
verification: AUTHENTICATED,
|
|
361
415
|
responses: { 200: PresignedUrlResponse },
|
|
362
416
|
}),
|
|
@@ -373,7 +427,7 @@ export function registerStorageRoutes(options: StorageRoutesOptions): (app: Hono
|
|
|
373
427
|
describeRoute({
|
|
374
428
|
summary: "Read a file's bytes.",
|
|
375
429
|
description:
|
|
376
|
-
"Public files need no credential.
|
|
430
|
+
"Public files need no credential. Any other needs the bearer token or session of someone it is visible to, and reads as missing to anyone else. Honors `Range` and `If-None-Match`.",
|
|
377
431
|
verification: ["public", "bearer", "session"],
|
|
378
432
|
responses: BYTE_RESPONSES,
|
|
379
433
|
}),
|
package/src/http/schemas.ts
CHANGED
|
@@ -120,7 +120,7 @@ export const CreateUploadInput = z
|
|
|
120
120
|
"The exact byte count. Required: the presigned PUT signs Content-Length, and the pending row reserves these bytes against the owner's quota the moment it is written.",
|
|
121
121
|
),
|
|
122
122
|
visibility: StorageVisibility.optional().describe(
|
|
123
|
-
"Who may read the file once stored. Omitted takes the configured default
|
|
123
|
+
"Who may read the file once stored. Omitted takes the configured `visibility.default`, which is private; a level outside `visibility.allowed` is refused, never coerced.",
|
|
124
124
|
),
|
|
125
125
|
})
|
|
126
126
|
.describe("A request to start an upload — what the file is called, what it is, and how big it is.");
|
|
@@ -149,7 +149,7 @@ export const UpdateObjectInput = z
|
|
|
149
149
|
.object({
|
|
150
150
|
path: logicalPath.optional(),
|
|
151
151
|
visibility: StorageVisibility.optional().describe(
|
|
152
|
-
"Change who may read the file. Takes effect on the next request.",
|
|
152
|
+
"Change who may read the file. Takes effect on the next request. A level outside `visibility.allowed` is refused.",
|
|
153
153
|
),
|
|
154
154
|
})
|
|
155
155
|
.describe("A rename, a visibility change, or both. At least one field must be present.")
|
|
@@ -196,7 +196,13 @@ export const ListObjectsQuery = z
|
|
|
196
196
|
.string()
|
|
197
197
|
.max(MAX_OBJECT_KEY_BYTES)
|
|
198
198
|
.optional()
|
|
199
|
-
.describe("Only files whose logical path starts with this. Omitted lists everything you
|
|
199
|
+
.describe("Only files whose logical path starts with this. Omitted lists everything you may see."),
|
|
200
|
+
includePublic: z
|
|
201
|
+
.enum(["true", "false"])
|
|
202
|
+
.optional()
|
|
203
|
+
.describe(
|
|
204
|
+
"`true` adds every public file to your own and your organization's. Off by default, so nobody receives what they did not ask for.",
|
|
205
|
+
),
|
|
200
206
|
cursor: z
|
|
201
207
|
.string()
|
|
202
208
|
.min(1)
|
|
@@ -210,5 +216,5 @@ export const ListObjectsQuery = z
|
|
|
210
216
|
.optional()
|
|
211
217
|
.describe("Files per page. Defaults to 50, capped at 100 so one request cannot scan an entire owner's files."),
|
|
212
218
|
})
|
|
213
|
-
.describe("Query parameters for the
|
|
219
|
+
.describe("Query parameters for the file listing: your own files, your organization's, and public ones on request.");
|
|
214
220
|
export type ListObjectsQuery = z.output<typeof ListObjectsQuery>;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { AdminRoute } from "@pithy-sh/core/src/controlPlane/discovery/adminRoute";
|
|
5
|
+
import type { ControlPlaneScope } from "@pithy-sh/core/src/controlPlane/scope/scope";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Storage's control-plane scope, and the admin surface a manifest advertises.
|
|
9
|
+
*
|
|
10
|
+
* **This module imports types and nothing else**, because a management client in a browser reads the scope
|
|
11
|
+
* names to render what a connection may do, and `tooling/browser-scopes` compiles every scope the kit
|
|
12
|
+
* declares against the DOM lib alone.
|
|
13
|
+
*
|
|
14
|
+
* **One scope, a read, and it is numbers.** Files and bytes held, per owner and per organization — never a
|
|
15
|
+
* path, a key or a link. A per-owner figure is always one organization's, so a tool holding this scope
|
|
16
|
+
* still cannot learn that a person holds an account in two of them.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** Files and bytes held, per organization and per owner within one. */
|
|
20
|
+
export const STORAGE_USAGE_READ_SCOPE: ControlPlaneScope = "storage:usage:read";
|
|
21
|
+
|
|
22
|
+
/** Every control-plane scope this capability defines. */
|
|
23
|
+
export const STORAGE_CONTROL_PLANE_SCOPES: readonly ControlPlaneScope[] = [STORAGE_USAGE_READ_SCOPE];
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The admin routes this capability advertises on `GET /control-plane/manifest`, built from the **resolved**
|
|
27
|
+
* mount path, never the default.
|
|
28
|
+
*/
|
|
29
|
+
export function storageAdminRoutes(basePath: string): AdminRoute[] {
|
|
30
|
+
return [
|
|
31
|
+
{
|
|
32
|
+
method: "GET",
|
|
33
|
+
path: `${basePath}/admin/usage`,
|
|
34
|
+
scope: STORAGE_USAGE_READ_SCOPE,
|
|
35
|
+
summary:
|
|
36
|
+
"Files and bytes held, per organization or per owner within one organization. Never across organizations.",
|
|
37
|
+
},
|
|
38
|
+
];
|
|
39
|
+
}
|
package/src/http/serve.ts
CHANGED
|
@@ -128,8 +128,11 @@ export interface ServeOptions {
|
|
|
128
128
|
metadata: ObjectMetadata;
|
|
129
129
|
/** The logical path — its last segment becomes the download filename. */
|
|
130
130
|
path: string;
|
|
131
|
-
/**
|
|
132
|
-
|
|
131
|
+
/**
|
|
132
|
+
* Who may read the object, which decides whether a shared cache may hold it: only a public object may
|
|
133
|
+
* be cached publicly, so an organization's file is as uncacheable as a private one.
|
|
134
|
+
*/
|
|
135
|
+
visibility: "private" | "organization" | "public";
|
|
133
136
|
/** Serve as a download (`attachment`) rather than inline. */
|
|
134
137
|
download?: boolean;
|
|
135
138
|
}
|
package/src/index.ts
CHANGED
|
@@ -22,6 +22,8 @@ export { maxObjectBytes, StorageConfig, type StorageConfigInput, StorageQuota }
|
|
|
22
22
|
export { StorageShare } from "./data/share";
|
|
23
23
|
export { StorageObject, StorageObjectStatus, StorageVisibility } from "./data/storageObject";
|
|
24
24
|
export { STORAGE_OBJECTS_TABLE, STORAGE_SHARES_TABLE, type StorageDatabase, storageDatabase } from "./data/tables";
|
|
25
|
+
export { STORAGE_MANAGE_POWER } from "./http/handlers";
|
|
26
|
+
export { STORAGE_USAGE_READ_SCOPE } from "./http/scopes";
|
|
25
27
|
export { deriveObjectKey, isDerivedObjectKey, OBJECT_KEY_PREFIX } from "./object/key";
|
|
26
28
|
export { collectParts, MultipartPlan, needsMultipart, PartPlan, planMultipart, ReportedPart } from "./object/multipart";
|
|
27
29
|
export {
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { type Kysely, sql } from "kysely";
|
|
5
|
+
import type { Migration } from "kysely/migration";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The organization a stored file belongs to, and the middle visibility level.
|
|
9
|
+
*
|
|
10
|
+
* **`organizationId` is added beside the existing `ownerId`**, nullable and in the same terms: the
|
|
11
|
+
* organization the owner was acting for when the file was written, or null when none was. Existing rows
|
|
12
|
+
* get null, which is true of them — nothing recorded one.
|
|
13
|
+
*
|
|
14
|
+
* **`visibility` gains `organization`, and in SQLite a `CHECK` cannot be altered.** So the objects table is
|
|
15
|
+
* rebuilt: a new table with the new constraint, every row copied across, the old one dropped, the new one
|
|
16
|
+
* renamed into its place. The shares table is rebuilt beside it, in the order that keeps its foreign key
|
|
17
|
+
* honest — a `DROP TABLE` of a parent runs an implicit `DELETE`, and with `ON DELETE CASCADE` that would
|
|
18
|
+
* take every share link with it. So the new shares table points at the new objects table from the start,
|
|
19
|
+
* the old shares go before the old objects, nothing references the old objects when they are dropped, and
|
|
20
|
+
* the rename carries the foreign key across to the new name. Not one share is lost, and the tests count.
|
|
21
|
+
*
|
|
22
|
+
* **Indexed the way the reads ask.** The existing `(ownerId, path)` and `(status, createdAt)` indexes come
|
|
23
|
+
* back with the table, and two join them: `(ownerId, createdAt)` and `(organizationId, createdAt)`, so
|
|
24
|
+
* *this owner's files over a window* and *this organization's* are each one index range.
|
|
25
|
+
*
|
|
26
|
+
* `down` is the tested inverse: the same rebuild the other way. A row at `organization` goes back to
|
|
27
|
+
* `private` — the narrower level, so an inverse can never publish a file — and `organizationId` is
|
|
28
|
+
* dropped.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const OBJECTS = "pithyStorageObjects";
|
|
32
|
+
const OBJECTS_NEXT = "pithyStorageObjectsNext";
|
|
33
|
+
const SHARES = "pithyStorageShares";
|
|
34
|
+
const SHARES_NEXT = "pithyStorageSharesNext";
|
|
35
|
+
|
|
36
|
+
/** The columns both shapes of the objects table share, in the order they were declared. */
|
|
37
|
+
const SHARED_COLUMNS = [
|
|
38
|
+
"id",
|
|
39
|
+
"key",
|
|
40
|
+
"path",
|
|
41
|
+
"ownerId",
|
|
42
|
+
"contentType",
|
|
43
|
+
"size",
|
|
44
|
+
"visibility",
|
|
45
|
+
"checksum",
|
|
46
|
+
"status",
|
|
47
|
+
"uploadId",
|
|
48
|
+
"createdAt",
|
|
49
|
+
"updatedAt",
|
|
50
|
+
] as const;
|
|
51
|
+
|
|
52
|
+
/** Every column of the shares table. */
|
|
53
|
+
const SHARE_COLUMNS = ["token", "objectId", "expiresAt", "revokedAt", "createdAt"] as const;
|
|
54
|
+
|
|
55
|
+
/** A comma-separated list of column references, snake-cased by the `CamelCasePlugin`. */
|
|
56
|
+
function columnList(columns: readonly string[]) {
|
|
57
|
+
return sql.join(columns.map((column) => sql.ref(column)));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Create an objects table under `name`, with or without the organization column and level. */
|
|
61
|
+
async function createObjects(db: Kysely<unknown>, name: string, withOrganization: boolean): Promise<void> {
|
|
62
|
+
let table = db.schema
|
|
63
|
+
.createTable(name)
|
|
64
|
+
// Text, not autoincrement: the id is in every route path, and a sequential id is an enumeration oracle.
|
|
65
|
+
.addColumn("id", "text", (c) => c.primaryKey())
|
|
66
|
+
.addColumn("key", "text", (c) => c.notNull().unique())
|
|
67
|
+
.addColumn("path", "text", (c) => c.notNull())
|
|
68
|
+
.addColumn("ownerId", "text");
|
|
69
|
+
if (withOrganization) table = table.addColumn("organizationId", "text");
|
|
70
|
+
await table
|
|
71
|
+
.addColumn("contentType", "text", (c) => c.notNull())
|
|
72
|
+
.addColumn("size", "integer")
|
|
73
|
+
.addColumn("visibility", "text", (c) => c.notNull().defaultTo("private"))
|
|
74
|
+
.addColumn("checksum", "text")
|
|
75
|
+
.addColumn("status", "text", (c) => c.notNull().defaultTo("pending"))
|
|
76
|
+
.addColumn("uploadId", "text")
|
|
77
|
+
.addColumn("createdAt", "integer", (c) => c.notNull())
|
|
78
|
+
.addColumn("updatedAt", "integer", (c) => c.notNull())
|
|
79
|
+
.addCheckConstraint(
|
|
80
|
+
"pithyStorageObjectsVisibility",
|
|
81
|
+
withOrganization
|
|
82
|
+
? sql`visibility IN ('private', 'organization', 'public')`
|
|
83
|
+
: sql`visibility IN ('private', 'public')`,
|
|
84
|
+
)
|
|
85
|
+
.addCheckConstraint("pithyStorageObjectsStatus", sql`status IN ('pending', 'stored', 'failed')`)
|
|
86
|
+
.execute();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Create a shares table under `name`, pointing at the objects table `parent`. */
|
|
90
|
+
async function createShares(db: Kysely<unknown>, name: string, parent: string): Promise<void> {
|
|
91
|
+
await db.schema
|
|
92
|
+
.createTable(name)
|
|
93
|
+
// The token IS the credential, so it is the key — a lookup is a primary-key hit, not a scan.
|
|
94
|
+
.addColumn("token", "text", (c) => c.primaryKey())
|
|
95
|
+
.addColumn("objectId", "text", (c) => c.notNull())
|
|
96
|
+
.addColumn("expiresAt", "integer")
|
|
97
|
+
.addColumn("revokedAt", "integer")
|
|
98
|
+
.addColumn("createdAt", "integer", (c) => c.notNull())
|
|
99
|
+
.addForeignKeyConstraint("pithyStorageSharesObjectFk", ["objectId"], parent, ["id"], (fk) => fk.onDelete("cascade"))
|
|
100
|
+
.execute();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The indexes the objects table carries in both shapes. */
|
|
104
|
+
async function createObjectIndexes(db: Kysely<unknown>, withOrganization: boolean): Promise<void> {
|
|
105
|
+
await db.schema.createIndex("pithyStorageObjectsOwnerPathIdx").on(OBJECTS).columns(["ownerId", "path"]).execute();
|
|
106
|
+
await db.schema.createIndex("pithyStorageObjectsStatusIdx").on(OBJECTS).columns(["status", "createdAt"]).execute();
|
|
107
|
+
await db.schema.createIndex("pithyStorageSharesObjectIdx").on(SHARES).columns(["objectId"]).execute();
|
|
108
|
+
if (!withOrganization) return;
|
|
109
|
+
await db.schema
|
|
110
|
+
.createIndex("pithyStorageObjectsOwnerCreatedIdx")
|
|
111
|
+
.on(OBJECTS)
|
|
112
|
+
.columns(["ownerId", "createdAt"])
|
|
113
|
+
.execute();
|
|
114
|
+
await db.schema
|
|
115
|
+
.createIndex("pithyStorageObjectsOrganizationCreatedIdx")
|
|
116
|
+
.on(OBJECTS)
|
|
117
|
+
.columns(["organizationId", "createdAt"])
|
|
118
|
+
.execute();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Drop the indexes the objects table carries in one of its two shapes — the inverse of `createObjectIndexes`. */
|
|
122
|
+
async function dropObjectIndexes(db: Kysely<unknown>, withOrganization: boolean): Promise<void> {
|
|
123
|
+
if (withOrganization) {
|
|
124
|
+
await db.schema.dropIndex("pithyStorageObjectsOrganizationCreatedIdx").execute();
|
|
125
|
+
await db.schema.dropIndex("pithyStorageObjectsOwnerCreatedIdx").execute();
|
|
126
|
+
}
|
|
127
|
+
await db.schema.dropIndex("pithyStorageSharesObjectIdx").execute();
|
|
128
|
+
await db.schema.dropIndex("pithyStorageObjectsStatusIdx").execute();
|
|
129
|
+
await db.schema.dropIndex("pithyStorageObjectsOwnerPathIdx").execute();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Rebuild both tables into a new shape, copying every row. `visibility` is the expression the copied
|
|
134
|
+
* visibility is read through — the column itself going up, a narrowing going down.
|
|
135
|
+
*/
|
|
136
|
+
async function rebuild(
|
|
137
|
+
db: Kysely<unknown>,
|
|
138
|
+
withOrganization: boolean,
|
|
139
|
+
visibility: ReturnType<typeof sql>,
|
|
140
|
+
): Promise<void> {
|
|
141
|
+
await createObjects(db, OBJECTS_NEXT, withOrganization);
|
|
142
|
+
const copied = SHARED_COLUMNS.filter((column) => column !== "visibility");
|
|
143
|
+
await sql`insert into ${sql.table(OBJECTS_NEXT)} (${columnList(copied)}, ${sql.ref("visibility")}) select ${columnList(copied)}, ${visibility} from ${sql.table(OBJECTS)}`.execute(
|
|
144
|
+
db,
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
await createShares(db, SHARES_NEXT, OBJECTS_NEXT);
|
|
148
|
+
await sql`insert into ${sql.table(SHARES_NEXT)} (${columnList(SHARE_COLUMNS)}) select ${columnList(SHARE_COLUMNS)} from ${sql.table(SHARES)}`.execute(
|
|
149
|
+
db,
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
// The old tables' indexes first, by name, as every other migration's inverse drops them: their names are
|
|
153
|
+
// the ones the rebuilt tables take.
|
|
154
|
+
await dropObjectIndexes(db, !withOrganization);
|
|
155
|
+
// Children first: once the old shares are gone, nothing references the old objects, so dropping them
|
|
156
|
+
// cascades into nothing.
|
|
157
|
+
await db.schema.dropTable(SHARES).execute();
|
|
158
|
+
await db.schema.dropTable(OBJECTS).execute();
|
|
159
|
+
// Renaming the parent rewrites the new shares' foreign key to the name it now has.
|
|
160
|
+
await db.schema.alterTable(OBJECTS_NEXT).renameTo(OBJECTS).execute();
|
|
161
|
+
await db.schema.alterTable(SHARES_NEXT).renameTo(SHARES).execute();
|
|
162
|
+
await createObjectIndexes(db, withOrganization);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export const storage_0002_ownership: Migration = {
|
|
166
|
+
up: async (db: Kysely<unknown>): Promise<void> => {
|
|
167
|
+
await rebuild(db, true, sql.ref("visibility"));
|
|
168
|
+
},
|
|
169
|
+
down: async (db: Kysely<unknown>): Promise<void> => {
|
|
170
|
+
await rebuild(
|
|
171
|
+
db,
|
|
172
|
+
false,
|
|
173
|
+
sql`case when ${sql.ref("visibility")} = 'organization' then 'private' else ${sql.ref("visibility")} end`,
|
|
174
|
+
);
|
|
175
|
+
},
|
|
176
|
+
};
|
package/src/object/cloudflare.ts
CHANGED
|
@@ -41,12 +41,12 @@ export function r2Manager(manager: CloudflareR2Manager): PresignedObjects {
|
|
|
41
41
|
|
|
42
42
|
/**
|
|
43
43
|
* The default port factory: build an R2 manager from the resolved credential bundle and adapt it.
|
|
44
|
-
* This is what `objectStore` calls when no `presigned` seam is injected.
|
|
44
|
+
* This is what `objectStore` calls when no `presigned` seam is injected. S3 only: the bundle carries no
|
|
45
|
+
* Cloudflare API token, and nothing here calls the Cloudflare API (#728).
|
|
45
46
|
*/
|
|
46
47
|
export function r2Presigned(credentials: R2StorageCredentials): PresignedObjects {
|
|
47
48
|
return r2Manager(
|
|
48
49
|
new CloudflareR2Manager({
|
|
49
|
-
apiToken: credentials.apiToken,
|
|
50
50
|
accountId: credentials.accountId,
|
|
51
51
|
accessKeyId: credentials.accessKeyId,
|
|
52
52
|
secretAccessKey: credentials.secretAccessKey,
|
|
@@ -4,13 +4,14 @@
|
|
|
4
4
|
import { InternalError } from "@pithy-sh/core/src/error/pithyError";
|
|
5
5
|
import type { DeclaredEnvironments } from "@pithy-sh/core/src/naming/environment";
|
|
6
6
|
import { resourceNames } from "@pithy-sh/core/src/naming/resourceNames";
|
|
7
|
+
import { hostWorkflowsFor } from "@pithy-sh/core/src/workflow/host";
|
|
7
8
|
import {
|
|
8
9
|
type DeprovisionTarget,
|
|
9
10
|
deprovisionTarget,
|
|
10
11
|
type ManagedEnvironment,
|
|
11
12
|
managedEnvironments,
|
|
12
13
|
} from "@pithy-sh/secrets/src/scope";
|
|
13
|
-
import { STORAGE_CAPABILITY } from "../workflows/specs";
|
|
14
|
+
import { STORAGE_CAPABILITY, storageWorkflowRegistry } from "../workflows/specs";
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* The provisioning orchestration for the storage capability — the live counterpart to
|
|
@@ -34,7 +35,7 @@ import { STORAGE_CAPABILITY } from "../workflows/specs";
|
|
|
34
35
|
* bucket, create-then-update for the secret, and a deploy that overwrites. Re-running is a no-op.
|
|
35
36
|
*
|
|
36
37
|
* **What it does not do.** Cloudflare exposes no API for minting an R2 S3 access-key pair, so nothing
|
|
37
|
-
* here mints one. The pair is supplied by the operator — flags, or `R2_CREDENTIALS` in
|
|
38
|
+
* here mints one. The pair is supplied by the operator — flags, or `R2_CREDENTIALS` in the account config —
|
|
38
39
|
* and written into the secret as given. That is a real limitation, and the manifest says so rather
|
|
39
40
|
* than promising a mint that does not happen.
|
|
40
41
|
*/
|
|
@@ -80,6 +81,18 @@ export function storageWorkerName(project: string, env: ManagedEnvironment): str
|
|
|
80
81
|
return resourceNames(project).env(env).worker(STORAGE_CAPABILITY);
|
|
81
82
|
}
|
|
82
83
|
|
|
84
|
+
/**
|
|
85
|
+
* The Workflows the sweep worker is deployed with in one environment — `<project>-<env>-storage-sweep`.
|
|
86
|
+
* Derived from the specs by the call `resolveStorageConfig` deploys with, so teardown deletes the names
|
|
87
|
+
* provision created and no others (#789). Deleting a Worker does not delete the Workflows registered
|
|
88
|
+
* against it.
|
|
89
|
+
*/
|
|
90
|
+
export function storageWorkflowNames(project: string, env: ManagedEnvironment): string[] {
|
|
91
|
+
return hostWorkflowsFor(storageWorkflowRegistry, { project, capability: STORAGE_CAPABILITY, env }).workflows.map(
|
|
92
|
+
(workflow) => workflow.name,
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
83
96
|
/** The provisioned resources an environment's worker and secret are wired to. */
|
|
84
97
|
export interface StorageResources {
|
|
85
98
|
/** The R2 bucket objects live in. */
|
|
@@ -158,6 +171,11 @@ function resourcesFor(resources: Map<ManagedEnvironment, StorageResources>, env:
|
|
|
158
171
|
|
|
159
172
|
/** The teardown seam — the inverse of {@link StorageProvisioner}. Every step idempotent. */
|
|
160
173
|
export interface StorageDeprovisioner {
|
|
174
|
+
/**
|
|
175
|
+
* Delete this environment's sweep Workflows, by {@link storageWorkflowNames}. Idempotent (a Workflow
|
|
176
|
+
* already gone is a no-op). Deleting the worker leaves them registered, so they go first (#789).
|
|
177
|
+
*/
|
|
178
|
+
deleteWorkflows(env: ManagedEnvironment): Promise<void>;
|
|
161
179
|
/** Delete this environment's sweep worker. Idempotent (a missing worker is a no-op). */
|
|
162
180
|
deleteWorker(env: ManagedEnvironment): Promise<void>;
|
|
163
181
|
/**
|
|
@@ -183,7 +201,7 @@ export interface StorageDeprovisionOptions {
|
|
|
183
201
|
|
|
184
202
|
/**
|
|
185
203
|
* Tear down **one named environment's** storage infrastructure, reversing {@link provisionStorage} for it:
|
|
186
|
-
* its sweep worker
|
|
204
|
+
* its sweep Workflows, then its sweep worker (it binds the bucket), then — only when `deleteStorage` is set — its bucket and
|
|
187
205
|
* everything in it. Idempotent end to end.
|
|
188
206
|
*
|
|
189
207
|
* The environment is resolved by `@pithy-sh/secrets`' {@link deprovisionTarget}, before anything is deleted:
|
|
@@ -197,6 +215,7 @@ export async function deprovisionStorage(
|
|
|
197
215
|
options: StorageDeprovisionOptions = {},
|
|
198
216
|
): Promise<{ env: ManagedEnvironment }> {
|
|
199
217
|
const env = deprovisionTarget(target);
|
|
218
|
+
await deprovisioner.deleteWorkflows(env);
|
|
200
219
|
await deprovisioner.deleteWorker(env);
|
|
201
220
|
if (options.deleteStorage) await deprovisioner.deleteBucket(env);
|
|
202
221
|
return { env };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { PithyHonoEnv } from "@pithy-sh/core/src/capability/capability";
|
|
5
|
+
import { meterLimit } from "@pithy-sh/core/src/entitlement/allowance";
|
|
6
|
+
import type { Context } from "hono";
|
|
7
|
+
import type { StorageConfig } from "../config/config";
|
|
8
|
+
|
|
9
|
+
/** The byte ceiling one request's writes are held to, and whose bytes it is the whole of. */
|
|
10
|
+
export interface StorageCeiling {
|
|
11
|
+
/** The ceiling in bytes. Null is unlimited. */
|
|
12
|
+
limitBytes: number | null;
|
|
13
|
+
/**
|
|
14
|
+
* The organization the ceiling covers, or null when it is the caller's own. Set, every byte stored for
|
|
15
|
+
* that organization counts against it, whoever uploaded it — an organization's plan is the organization's.
|
|
16
|
+
*/
|
|
17
|
+
tenant: string | null;
|
|
18
|
+
/**
|
|
19
|
+
* Whether the number is an enforced `storageBytes` meter's. A refusal is then the plan's —
|
|
20
|
+
* `payments/quota_exceeded` at 402, the remedy a bigger tier — and otherwise the configured
|
|
21
|
+
* `bytesPerOwner`'s, `storage/quota_exceeded` at 413.
|
|
22
|
+
*/
|
|
23
|
+
metered: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The byte ceiling for one request: the `storageBytes` quota where an entitlement provider enforces that
|
|
28
|
+
* meter, the configured `quota.bytesPerOwner` where it does not.
|
|
29
|
+
*
|
|
30
|
+
* **Configured behavior is the fallback, never replaced by surprise.** With no provider composed, or one that
|
|
31
|
+
* leaves `storageBytes` advisory, this is `bytesPerOwner` exactly, counted per owner — which is what every
|
|
32
|
+
* project had before plans could say how much a customer may store. Enforced, it is the billing subject's
|
|
33
|
+
* effective quota: their plan's number, a number pinned for them, or `null` — unlimited — for an
|
|
34
|
+
* `unrestricted` subject. Under organization billing that subject is the organization, and so is the sum.
|
|
35
|
+
*
|
|
36
|
+
* A provider whose read fails refuses the write rather than serving it unbounded: `meterLimit` throws, and
|
|
37
|
+
* the request fails with it. Over-counting is recoverable; under-counting is not.
|
|
38
|
+
*/
|
|
39
|
+
export async function storageCeiling(c: Context<PithyHonoEnv>, config: StorageConfig): Promise<StorageCeiling> {
|
|
40
|
+
const ceiling = await meterLimit(c, "storageBytes");
|
|
41
|
+
return ceiling.enforced
|
|
42
|
+
? { limitBytes: ceiling.limit, tenant: ceiling.tenant, metered: true }
|
|
43
|
+
: { limitBytes: config.quota.bytesPerOwner, tenant: null, metered: false };
|
|
44
|
+
}
|