@pithy-sh/storage 0.2.10 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capability.d.ts +12 -3
- package/dist/capability.d.ts.map +1 -1
- package/dist/capability.js +35 -9
- package/dist/config/config.d.ts +20 -4
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +23 -6
- package/dist/data/storageObject.d.ts +12 -0
- package/dist/data/storageObject.d.ts.map +1 -1
- package/dist/data/storageObject.js +13 -2
- package/dist/data/tables.js +1 -1
- package/dist/error/errors.js +2 -2
- package/dist/http/handlers.d.ts +48 -16
- package/dist/http/handlers.d.ts.map +1 -1
- package/dist/http/handlers.js +169 -66
- package/dist/http/responses.d.ts +173 -0
- package/dist/http/responses.d.ts.map +1 -0
- package/dist/http/responses.js +92 -0
- package/dist/http/routes.d.ts +13 -2
- package/dist/http/routes.d.ts.map +1 -1
- package/dist/http/routes.js +134 -22
- package/dist/http/schemas.d.ts +6 -0
- package/dist/http/schemas.d.ts.map +1 -1
- package/dist/http/schemas.js +7 -6
- package/dist/http/scopes.d.ts +25 -0
- package/dist/http/scopes.d.ts.map +1 -0
- package/dist/http/scopes.js +32 -0
- package/dist/http/serve.d.ts +5 -2
- package/dist/http/serve.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -3
- package/dist/migrations/0002_ownership.d.ts +5 -0
- package/dist/migrations/0002_ownership.d.ts.map +1 -0
- package/dist/migrations/0002_ownership.js +114 -0
- package/dist/object/cloudflare.d.ts +3 -2
- package/dist/object/cloudflare.d.ts.map +1 -1
- package/dist/object/cloudflare.js +3 -3
- package/dist/object/store.d.ts +2 -7
- package/dist/object/store.d.ts.map +1 -1
- package/dist/object/store.js +1 -6
- package/dist/object/uploadedPart.d.ts +11 -0
- package/dist/object/uploadedPart.d.ts.map +1 -0
- package/dist/object/uploadedPart.js +12 -0
- package/dist/provision/provisionStorage.d.ts +14 -2
- package/dist/provision/provisionStorage.d.ts.map +1 -1
- package/dist/provision/provisionStorage.js +19 -4
- package/dist/provision/resolveStorageConfig.js +1 -1
- package/dist/quota/ceiling.d.ts +36 -0
- package/dist/quota/ceiling.d.ts.map +1 -0
- package/dist/quota/ceiling.js +31 -0
- package/dist/quota/quota.d.ts +24 -7
- package/dist/quota/quota.d.ts.map +1 -1
- package/dist/quota/quota.js +40 -12
- package/dist/secret/bundleToken.d.ts +24 -0
- package/dist/secret/bundleToken.d.ts.map +1 -0
- package/dist/secret/bundleToken.js +74 -0
- package/dist/secret/registry.d.ts +5 -1
- package/dist/secret/registry.d.ts.map +1 -1
- package/dist/secret/registry.js +7 -3
- package/dist/seeds/example.d.ts.map +1 -1
- package/dist/seeds/example.js +3 -0
- package/dist/test-utils/liveStorage.d.ts.map +1 -1
- package/dist/test-utils/liveStorage.js +5 -2
- package/dist/version.generated.d.ts +1 -1
- package/dist/version.generated.d.ts.map +1 -1
- package/dist/version.generated.js +1 -1
- package/package.json +7 -7
- package/pithy.manifest.json +20 -10
- package/src/capability.ts +52 -8
- package/src/config/config.ts +24 -6
- package/src/data/storageObject.ts +22 -6
- package/src/error/errors.ts +2 -2
- package/src/http/handlers.ts +215 -81
- package/src/http/responses.ts +168 -0
- package/src/http/routes.ts +245 -46
- package/src/http/schemas.ts +11 -5
- package/src/http/scopes.ts +39 -0
- package/src/http/serve.ts +5 -2
- package/src/index.ts +2 -0
- package/src/migrations/0002_ownership.ts +176 -0
- package/src/object/cloudflare.ts +4 -3
- package/src/object/store.ts +6 -10
- package/src/object/uploadedPart.ts +17 -0
- package/src/provision/provisionStorage.ts +22 -3
- package/src/quota/ceiling.ts +44 -0
- package/src/quota/quota.ts +63 -20
- package/src/secret/bundleToken.ts +81 -0
- package/src/secret/registry.ts +6 -8
- package/src/seeds/example.ts +3 -0
- package/src/test-utils/liveStorage.ts +3 -6
- package/src/version.generated.ts +1 -1
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { AdminRoute } from "@pithy-sh/core/src/controlPlane/discovery/adminRoute";
|
|
5
|
+
import type { ControlPlaneScope } from "@pithy-sh/core/src/controlPlane/scope/scope";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Storage's control-plane scope, and the admin surface a manifest advertises.
|
|
9
|
+
*
|
|
10
|
+
* **This module imports types and nothing else**, because a management client in a browser reads the scope
|
|
11
|
+
* names to render what a connection may do, and `tooling/browser-scopes` compiles every scope the kit
|
|
12
|
+
* declares against the DOM lib alone.
|
|
13
|
+
*
|
|
14
|
+
* **One scope, a read, and it is numbers.** Files and bytes held, per owner and per organization — never a
|
|
15
|
+
* path, a key or a link. A per-owner figure is always one organization's, so a tool holding this scope
|
|
16
|
+
* still cannot learn that a person holds an account in two of them.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** Files and bytes held, per organization and per owner within one. */
|
|
20
|
+
export const STORAGE_USAGE_READ_SCOPE: ControlPlaneScope = "storage:usage:read";
|
|
21
|
+
|
|
22
|
+
/** Every control-plane scope this capability defines. */
|
|
23
|
+
export const STORAGE_CONTROL_PLANE_SCOPES: readonly ControlPlaneScope[] = [STORAGE_USAGE_READ_SCOPE];
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The admin routes this capability advertises on `GET /control-plane/manifest`, built from the **resolved**
|
|
27
|
+
* mount path, never the default.
|
|
28
|
+
*/
|
|
29
|
+
export function storageAdminRoutes(basePath: string): AdminRoute[] {
|
|
30
|
+
return [
|
|
31
|
+
{
|
|
32
|
+
method: "GET",
|
|
33
|
+
path: `${basePath}/admin/usage`,
|
|
34
|
+
scope: STORAGE_USAGE_READ_SCOPE,
|
|
35
|
+
summary:
|
|
36
|
+
"Files and bytes held, per organization or per owner within one organization. Never across organizations.",
|
|
37
|
+
},
|
|
38
|
+
];
|
|
39
|
+
}
|
package/src/http/serve.ts
CHANGED
|
@@ -128,8 +128,11 @@ export interface ServeOptions {
|
|
|
128
128
|
metadata: ObjectMetadata;
|
|
129
129
|
/** The logical path — its last segment becomes the download filename. */
|
|
130
130
|
path: string;
|
|
131
|
-
/**
|
|
132
|
-
|
|
131
|
+
/**
|
|
132
|
+
* Who may read the object, which decides whether a shared cache may hold it: only a public object may
|
|
133
|
+
* be cached publicly, so an organization's file is as uncacheable as a private one.
|
|
134
|
+
*/
|
|
135
|
+
visibility: "private" | "organization" | "public";
|
|
133
136
|
/** Serve as a download (`attachment`) rather than inline. */
|
|
134
137
|
download?: boolean;
|
|
135
138
|
}
|
package/src/index.ts
CHANGED
|
@@ -22,6 +22,8 @@ export { maxObjectBytes, StorageConfig, type StorageConfigInput, StorageQuota }
|
|
|
22
22
|
export { StorageShare } from "./data/share";
|
|
23
23
|
export { StorageObject, StorageObjectStatus, StorageVisibility } from "./data/storageObject";
|
|
24
24
|
export { STORAGE_OBJECTS_TABLE, STORAGE_SHARES_TABLE, type StorageDatabase, storageDatabase } from "./data/tables";
|
|
25
|
+
export { STORAGE_MANAGE_POWER } from "./http/handlers";
|
|
26
|
+
export { STORAGE_USAGE_READ_SCOPE } from "./http/scopes";
|
|
25
27
|
export { deriveObjectKey, isDerivedObjectKey, OBJECT_KEY_PREFIX } from "./object/key";
|
|
26
28
|
export { collectParts, MultipartPlan, needsMultipart, PartPlan, planMultipart, ReportedPart } from "./object/multipart";
|
|
27
29
|
export {
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { type Kysely, sql } from "kysely";
|
|
5
|
+
import type { Migration } from "kysely/migration";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The organization a stored file belongs to, and the middle visibility level.
|
|
9
|
+
*
|
|
10
|
+
* **`organizationId` is added beside the existing `ownerId`**, nullable and in the same terms: the
|
|
11
|
+
* organization the owner was acting for when the file was written, or null when none was. Existing rows
|
|
12
|
+
* get null, which is true of them — nothing recorded one.
|
|
13
|
+
*
|
|
14
|
+
* **`visibility` gains `organization`, and in SQLite a `CHECK` cannot be altered.** So the objects table is
|
|
15
|
+
* rebuilt: a new table with the new constraint, every row copied across, the old one dropped, the new one
|
|
16
|
+
* renamed into its place. The shares table is rebuilt beside it, in the order that keeps its foreign key
|
|
17
|
+
* honest — a `DROP TABLE` of a parent runs an implicit `DELETE`, and with `ON DELETE CASCADE` that would
|
|
18
|
+
* take every share link with it. So the new shares table points at the new objects table from the start,
|
|
19
|
+
* the old shares go before the old objects, nothing references the old objects when they are dropped, and
|
|
20
|
+
* the rename carries the foreign key across to the new name. Not one share is lost, and the tests count.
|
|
21
|
+
*
|
|
22
|
+
* **Indexed the way the reads ask.** The existing `(ownerId, path)` and `(status, createdAt)` indexes come
|
|
23
|
+
* back with the table, and two join them: `(ownerId, createdAt)` and `(organizationId, createdAt)`, so
|
|
24
|
+
* *this owner's files over a window* and *this organization's* are each one index range.
|
|
25
|
+
*
|
|
26
|
+
* `down` is the tested inverse: the same rebuild the other way. A row at `organization` goes back to
|
|
27
|
+
* `private` — the narrower level, so an inverse can never publish a file — and `organizationId` is
|
|
28
|
+
* dropped.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const OBJECTS = "pithyStorageObjects";
|
|
32
|
+
const OBJECTS_NEXT = "pithyStorageObjectsNext";
|
|
33
|
+
const SHARES = "pithyStorageShares";
|
|
34
|
+
const SHARES_NEXT = "pithyStorageSharesNext";
|
|
35
|
+
|
|
36
|
+
/** The columns both shapes of the objects table share, in the order they were declared. */
|
|
37
|
+
const SHARED_COLUMNS = [
|
|
38
|
+
"id",
|
|
39
|
+
"key",
|
|
40
|
+
"path",
|
|
41
|
+
"ownerId",
|
|
42
|
+
"contentType",
|
|
43
|
+
"size",
|
|
44
|
+
"visibility",
|
|
45
|
+
"checksum",
|
|
46
|
+
"status",
|
|
47
|
+
"uploadId",
|
|
48
|
+
"createdAt",
|
|
49
|
+
"updatedAt",
|
|
50
|
+
] as const;
|
|
51
|
+
|
|
52
|
+
/** Every column of the shares table. */
|
|
53
|
+
const SHARE_COLUMNS = ["token", "objectId", "expiresAt", "revokedAt", "createdAt"] as const;
|
|
54
|
+
|
|
55
|
+
/** A comma-separated list of column references, snake-cased by the `CamelCasePlugin`. */
|
|
56
|
+
function columnList(columns: readonly string[]) {
|
|
57
|
+
return sql.join(columns.map((column) => sql.ref(column)));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Create an objects table under `name`, with or without the organization column and level. */
|
|
61
|
+
async function createObjects(db: Kysely<unknown>, name: string, withOrganization: boolean): Promise<void> {
|
|
62
|
+
let table = db.schema
|
|
63
|
+
.createTable(name)
|
|
64
|
+
// Text, not autoincrement: the id is in every route path, and a sequential id is an enumeration oracle.
|
|
65
|
+
.addColumn("id", "text", (c) => c.primaryKey())
|
|
66
|
+
.addColumn("key", "text", (c) => c.notNull().unique())
|
|
67
|
+
.addColumn("path", "text", (c) => c.notNull())
|
|
68
|
+
.addColumn("ownerId", "text");
|
|
69
|
+
if (withOrganization) table = table.addColumn("organizationId", "text");
|
|
70
|
+
await table
|
|
71
|
+
.addColumn("contentType", "text", (c) => c.notNull())
|
|
72
|
+
.addColumn("size", "integer")
|
|
73
|
+
.addColumn("visibility", "text", (c) => c.notNull().defaultTo("private"))
|
|
74
|
+
.addColumn("checksum", "text")
|
|
75
|
+
.addColumn("status", "text", (c) => c.notNull().defaultTo("pending"))
|
|
76
|
+
.addColumn("uploadId", "text")
|
|
77
|
+
.addColumn("createdAt", "integer", (c) => c.notNull())
|
|
78
|
+
.addColumn("updatedAt", "integer", (c) => c.notNull())
|
|
79
|
+
.addCheckConstraint(
|
|
80
|
+
"pithyStorageObjectsVisibility",
|
|
81
|
+
withOrganization
|
|
82
|
+
? sql`visibility IN ('private', 'organization', 'public')`
|
|
83
|
+
: sql`visibility IN ('private', 'public')`,
|
|
84
|
+
)
|
|
85
|
+
.addCheckConstraint("pithyStorageObjectsStatus", sql`status IN ('pending', 'stored', 'failed')`)
|
|
86
|
+
.execute();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Create a shares table under `name`, pointing at the objects table `parent`. */
|
|
90
|
+
async function createShares(db: Kysely<unknown>, name: string, parent: string): Promise<void> {
|
|
91
|
+
await db.schema
|
|
92
|
+
.createTable(name)
|
|
93
|
+
// The token IS the credential, so it is the key — a lookup is a primary-key hit, not a scan.
|
|
94
|
+
.addColumn("token", "text", (c) => c.primaryKey())
|
|
95
|
+
.addColumn("objectId", "text", (c) => c.notNull())
|
|
96
|
+
.addColumn("expiresAt", "integer")
|
|
97
|
+
.addColumn("revokedAt", "integer")
|
|
98
|
+
.addColumn("createdAt", "integer", (c) => c.notNull())
|
|
99
|
+
.addForeignKeyConstraint("pithyStorageSharesObjectFk", ["objectId"], parent, ["id"], (fk) => fk.onDelete("cascade"))
|
|
100
|
+
.execute();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The indexes the objects table carries in both shapes. */
|
|
104
|
+
async function createObjectIndexes(db: Kysely<unknown>, withOrganization: boolean): Promise<void> {
|
|
105
|
+
await db.schema.createIndex("pithyStorageObjectsOwnerPathIdx").on(OBJECTS).columns(["ownerId", "path"]).execute();
|
|
106
|
+
await db.schema.createIndex("pithyStorageObjectsStatusIdx").on(OBJECTS).columns(["status", "createdAt"]).execute();
|
|
107
|
+
await db.schema.createIndex("pithyStorageSharesObjectIdx").on(SHARES).columns(["objectId"]).execute();
|
|
108
|
+
if (!withOrganization) return;
|
|
109
|
+
await db.schema
|
|
110
|
+
.createIndex("pithyStorageObjectsOwnerCreatedIdx")
|
|
111
|
+
.on(OBJECTS)
|
|
112
|
+
.columns(["ownerId", "createdAt"])
|
|
113
|
+
.execute();
|
|
114
|
+
await db.schema
|
|
115
|
+
.createIndex("pithyStorageObjectsOrganizationCreatedIdx")
|
|
116
|
+
.on(OBJECTS)
|
|
117
|
+
.columns(["organizationId", "createdAt"])
|
|
118
|
+
.execute();
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Drop the indexes the objects table carries in one of its two shapes — the inverse of `createObjectIndexes`. */
|
|
122
|
+
async function dropObjectIndexes(db: Kysely<unknown>, withOrganization: boolean): Promise<void> {
|
|
123
|
+
if (withOrganization) {
|
|
124
|
+
await db.schema.dropIndex("pithyStorageObjectsOrganizationCreatedIdx").execute();
|
|
125
|
+
await db.schema.dropIndex("pithyStorageObjectsOwnerCreatedIdx").execute();
|
|
126
|
+
}
|
|
127
|
+
await db.schema.dropIndex("pithyStorageSharesObjectIdx").execute();
|
|
128
|
+
await db.schema.dropIndex("pithyStorageObjectsStatusIdx").execute();
|
|
129
|
+
await db.schema.dropIndex("pithyStorageObjectsOwnerPathIdx").execute();
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Rebuild both tables into a new shape, copying every row. `visibility` is the expression the copied
|
|
134
|
+
* visibility is read through — the column itself going up, a narrowing going down.
|
|
135
|
+
*/
|
|
136
|
+
async function rebuild(
|
|
137
|
+
db: Kysely<unknown>,
|
|
138
|
+
withOrganization: boolean,
|
|
139
|
+
visibility: ReturnType<typeof sql>,
|
|
140
|
+
): Promise<void> {
|
|
141
|
+
await createObjects(db, OBJECTS_NEXT, withOrganization);
|
|
142
|
+
const copied = SHARED_COLUMNS.filter((column) => column !== "visibility");
|
|
143
|
+
await sql`insert into ${sql.table(OBJECTS_NEXT)} (${columnList(copied)}, ${sql.ref("visibility")}) select ${columnList(copied)}, ${visibility} from ${sql.table(OBJECTS)}`.execute(
|
|
144
|
+
db,
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
await createShares(db, SHARES_NEXT, OBJECTS_NEXT);
|
|
148
|
+
await sql`insert into ${sql.table(SHARES_NEXT)} (${columnList(SHARE_COLUMNS)}) select ${columnList(SHARE_COLUMNS)} from ${sql.table(SHARES)}`.execute(
|
|
149
|
+
db,
|
|
150
|
+
);
|
|
151
|
+
|
|
152
|
+
// The old tables' indexes first, by name, as every other migration's inverse drops them: their names are
|
|
153
|
+
// the ones the rebuilt tables take.
|
|
154
|
+
await dropObjectIndexes(db, !withOrganization);
|
|
155
|
+
// Children first: once the old shares are gone, nothing references the old objects, so dropping them
|
|
156
|
+
// cascades into nothing.
|
|
157
|
+
await db.schema.dropTable(SHARES).execute();
|
|
158
|
+
await db.schema.dropTable(OBJECTS).execute();
|
|
159
|
+
// Renaming the parent rewrites the new shares' foreign key to the name it now has.
|
|
160
|
+
await db.schema.alterTable(OBJECTS_NEXT).renameTo(OBJECTS).execute();
|
|
161
|
+
await db.schema.alterTable(SHARES_NEXT).renameTo(SHARES).execute();
|
|
162
|
+
await createObjectIndexes(db, withOrganization);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export const storage_0002_ownership: Migration = {
|
|
166
|
+
up: async (db: Kysely<unknown>): Promise<void> => {
|
|
167
|
+
await rebuild(db, true, sql.ref("visibility"));
|
|
168
|
+
},
|
|
169
|
+
down: async (db: Kysely<unknown>): Promise<void> => {
|
|
170
|
+
await rebuild(
|
|
171
|
+
db,
|
|
172
|
+
false,
|
|
173
|
+
sql`case when ${sql.ref("visibility")} = 'organization' then 'private' else ${sql.ref("visibility")} end`,
|
|
174
|
+
);
|
|
175
|
+
},
|
|
176
|
+
};
|
package/src/object/cloudflare.ts
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
import { CloudflareR2Manager } from "@pithy-sh/cloudflare/src/r2/r2Manager";
|
|
5
5
|
import type { R2StorageCredentials } from "../secret/registry";
|
|
6
|
-
import {
|
|
6
|
+
import type { PresignedObjects } from "./store";
|
|
7
|
+
import { UploadedPart } from "./uploadedPart";
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* The SDK-friction boundary: the one file in this package that touches `@pithy-sh/cloudflare`.
|
|
@@ -40,12 +41,12 @@ export function r2Manager(manager: CloudflareR2Manager): PresignedObjects {
|
|
|
40
41
|
|
|
41
42
|
/**
|
|
42
43
|
* The default port factory: build an R2 manager from the resolved credential bundle and adapt it.
|
|
43
|
-
* This is what `objectStore` calls when no `presigned` seam is injected.
|
|
44
|
+
* This is what `objectStore` calls when no `presigned` seam is injected. S3 only: the bundle carries no
|
|
45
|
+
* Cloudflare API token, and nothing here calls the Cloudflare API (#728).
|
|
44
46
|
*/
|
|
45
47
|
export function r2Presigned(credentials: R2StorageCredentials): PresignedObjects {
|
|
46
48
|
return r2Manager(
|
|
47
49
|
new CloudflareR2Manager({
|
|
48
|
-
apiToken: credentials.apiToken,
|
|
49
50
|
accountId: credentials.accountId,
|
|
50
51
|
accessKeyId: credentials.accessKeyId,
|
|
51
52
|
secretAccessKey: credentials.secretAccessKey,
|
package/src/object/store.ts
CHANGED
|
@@ -8,6 +8,12 @@ import { z } from "zod";
|
|
|
8
8
|
import { type R2StorageCredentials, r2CredentialsRegistry, STORAGE_R2_SECRET } from "../secret/registry";
|
|
9
9
|
import { r2Presigned } from "./cloudflare";
|
|
10
10
|
import type { ReportedPart } from "./multipart";
|
|
11
|
+
import type { UploadedPart } from "./uploadedPart";
|
|
12
|
+
|
|
13
|
+
// Re-exported, which this kit otherwise never does, because this path is a public address: the folder path
|
|
14
|
+
// is the import path, and `UploadedPart` was published here before it moved to `./uploadedPart` so a browser
|
|
15
|
+
// could import it without the S3 client (#719). Removing the line breaks every import of the old address.
|
|
16
|
+
export { UploadedPart } from "./uploadedPart";
|
|
11
17
|
|
|
12
18
|
/**
|
|
13
19
|
* `ObjectStore` — the object-plane seam, and the one piece of this package that is reusable outside it.
|
|
@@ -108,16 +114,6 @@ export const ObjectListing = z
|
|
|
108
114
|
.describe("One page of a bucket listing. Pagination is caller-driven — pass `cursor` back to advance.");
|
|
109
115
|
export type ObjectListing = z.output<typeof ObjectListing>;
|
|
110
116
|
|
|
111
|
-
/** One part already stored against an in-flight multipart upload — what makes an upload resumable. */
|
|
112
|
-
export const UploadedPart = z
|
|
113
|
-
.object({
|
|
114
|
-
partNumber: z.number().int().min(1).describe("The part's 1-based index within the upload."),
|
|
115
|
-
etag: z.string().min(1).describe("The part's entity tag, verbatim. Pass it back unchanged to complete."),
|
|
116
|
-
size: z.number().int().nonnegative().describe("The part's size in bytes, as R2 stored it."),
|
|
117
|
-
})
|
|
118
|
-
.describe("One uploaded part — the unit a resumed upload skips re-sending.");
|
|
119
|
-
export type UploadedPart = z.output<typeof UploadedPart>;
|
|
120
|
-
|
|
121
117
|
/** How long a presigned URL stays valid. */
|
|
122
118
|
export interface PresignOptions {
|
|
123
119
|
/** Lifetime in seconds. Defaults to one hour — long enough for a large part, short enough to leak little. */
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
|
|
6
|
+
// Its own module, importing nothing but zod, because `http/responses.ts` hands it to a browser and
|
|
7
|
+
// `./store.ts` beside it reaches the S3 client and the Workers runtime.
|
|
8
|
+
|
|
9
|
+
/** One part already stored against an in-flight multipart upload — what makes an upload resumable. */
|
|
10
|
+
export const UploadedPart = z
|
|
11
|
+
.object({
|
|
12
|
+
partNumber: z.number().int().min(1).describe("The part's 1-based index within the upload."),
|
|
13
|
+
etag: z.string().min(1).describe("The part's entity tag, verbatim. Pass it back unchanged to complete."),
|
|
14
|
+
size: z.number().int().nonnegative().describe("The part's size in bytes, as R2 stored it."),
|
|
15
|
+
})
|
|
16
|
+
.describe("One uploaded part — the unit a resumed upload skips re-sending.");
|
|
17
|
+
export type UploadedPart = z.output<typeof UploadedPart>;
|
|
@@ -4,13 +4,14 @@
|
|
|
4
4
|
import { InternalError } from "@pithy-sh/core/src/error/pithyError";
|
|
5
5
|
import type { DeclaredEnvironments } from "@pithy-sh/core/src/naming/environment";
|
|
6
6
|
import { resourceNames } from "@pithy-sh/core/src/naming/resourceNames";
|
|
7
|
+
import { hostWorkflowsFor } from "@pithy-sh/core/src/workflow/host";
|
|
7
8
|
import {
|
|
8
9
|
type DeprovisionTarget,
|
|
9
10
|
deprovisionTarget,
|
|
10
11
|
type ManagedEnvironment,
|
|
11
12
|
managedEnvironments,
|
|
12
13
|
} from "@pithy-sh/secrets/src/scope";
|
|
13
|
-
import { STORAGE_CAPABILITY } from "../workflows/specs";
|
|
14
|
+
import { STORAGE_CAPABILITY, storageWorkflowRegistry } from "../workflows/specs";
|
|
14
15
|
|
|
15
16
|
/**
|
|
16
17
|
* The provisioning orchestration for the storage capability — the live counterpart to
|
|
@@ -34,7 +35,7 @@ import { STORAGE_CAPABILITY } from "../workflows/specs";
|
|
|
34
35
|
* bucket, create-then-update for the secret, and a deploy that overwrites. Re-running is a no-op.
|
|
35
36
|
*
|
|
36
37
|
* **What it does not do.** Cloudflare exposes no API for minting an R2 S3 access-key pair, so nothing
|
|
37
|
-
* here mints one. The pair is supplied by the operator — flags, or `R2_CREDENTIALS` in
|
|
38
|
+
* here mints one. The pair is supplied by the operator — flags, or `R2_CREDENTIALS` in the account config —
|
|
38
39
|
* and written into the secret as given. That is a real limitation, and the manifest says so rather
|
|
39
40
|
* than promising a mint that does not happen.
|
|
40
41
|
*/
|
|
@@ -80,6 +81,18 @@ export function storageWorkerName(project: string, env: ManagedEnvironment): str
|
|
|
80
81
|
return resourceNames(project).env(env).worker(STORAGE_CAPABILITY);
|
|
81
82
|
}
|
|
82
83
|
|
|
84
|
+
/**
|
|
85
|
+
* The Workflows the sweep worker is deployed with in one environment — `<project>-<env>-storage-sweep`.
|
|
86
|
+
* Derived from the specs by the call `resolveStorageConfig` deploys with, so teardown deletes the names
|
|
87
|
+
* provision created and no others (#789). Deleting a Worker does not delete the Workflows registered
|
|
88
|
+
* against it.
|
|
89
|
+
*/
|
|
90
|
+
export function storageWorkflowNames(project: string, env: ManagedEnvironment): string[] {
|
|
91
|
+
return hostWorkflowsFor(storageWorkflowRegistry, { project, capability: STORAGE_CAPABILITY, env }).workflows.map(
|
|
92
|
+
(workflow) => workflow.name,
|
|
93
|
+
);
|
|
94
|
+
}
|
|
95
|
+
|
|
83
96
|
/** The provisioned resources an environment's worker and secret are wired to. */
|
|
84
97
|
export interface StorageResources {
|
|
85
98
|
/** The R2 bucket objects live in. */
|
|
@@ -158,6 +171,11 @@ function resourcesFor(resources: Map<ManagedEnvironment, StorageResources>, env:
|
|
|
158
171
|
|
|
159
172
|
/** The teardown seam — the inverse of {@link StorageProvisioner}. Every step idempotent. */
|
|
160
173
|
export interface StorageDeprovisioner {
|
|
174
|
+
/**
|
|
175
|
+
* Delete this environment's sweep Workflows, by {@link storageWorkflowNames}. Idempotent (a Workflow
|
|
176
|
+
* already gone is a no-op). Deleting the worker leaves them registered, so they go first (#789).
|
|
177
|
+
*/
|
|
178
|
+
deleteWorkflows(env: ManagedEnvironment): Promise<void>;
|
|
161
179
|
/** Delete this environment's sweep worker. Idempotent (a missing worker is a no-op). */
|
|
162
180
|
deleteWorker(env: ManagedEnvironment): Promise<void>;
|
|
163
181
|
/**
|
|
@@ -183,7 +201,7 @@ export interface StorageDeprovisionOptions {
|
|
|
183
201
|
|
|
184
202
|
/**
|
|
185
203
|
* Tear down **one named environment's** storage infrastructure, reversing {@link provisionStorage} for it:
|
|
186
|
-
* its sweep worker
|
|
204
|
+
* its sweep Workflows, then its sweep worker (it binds the bucket), then — only when `deleteStorage` is set — its bucket and
|
|
187
205
|
* everything in it. Idempotent end to end.
|
|
188
206
|
*
|
|
189
207
|
* The environment is resolved by `@pithy-sh/secrets`' {@link deprovisionTarget}, before anything is deleted:
|
|
@@ -197,6 +215,7 @@ export async function deprovisionStorage(
|
|
|
197
215
|
options: StorageDeprovisionOptions = {},
|
|
198
216
|
): Promise<{ env: ManagedEnvironment }> {
|
|
199
217
|
const env = deprovisionTarget(target);
|
|
218
|
+
await deprovisioner.deleteWorkflows(env);
|
|
200
219
|
await deprovisioner.deleteWorker(env);
|
|
201
220
|
if (options.deleteStorage) await deprovisioner.deleteBucket(env);
|
|
202
221
|
return { env };
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { PithyHonoEnv } from "@pithy-sh/core/src/capability/capability";
|
|
5
|
+
import { meterLimit } from "@pithy-sh/core/src/entitlement/allowance";
|
|
6
|
+
import type { Context } from "hono";
|
|
7
|
+
import type { StorageConfig } from "../config/config";
|
|
8
|
+
|
|
9
|
+
/** The byte ceiling one request's writes are held to, and whose bytes it is the whole of. */
|
|
10
|
+
export interface StorageCeiling {
|
|
11
|
+
/** The ceiling in bytes. Null is unlimited. */
|
|
12
|
+
limitBytes: number | null;
|
|
13
|
+
/**
|
|
14
|
+
* The organization the ceiling covers, or null when it is the caller's own. Set, every byte stored for
|
|
15
|
+
* that organization counts against it, whoever uploaded it — an organization's plan is the organization's.
|
|
16
|
+
*/
|
|
17
|
+
tenant: string | null;
|
|
18
|
+
/**
|
|
19
|
+
* Whether the number is an enforced `storageBytes` meter's. A refusal is then the plan's —
|
|
20
|
+
* `payments/quota_exceeded` at 402, the remedy a bigger tier — and otherwise the configured
|
|
21
|
+
* `bytesPerOwner`'s, `storage/quota_exceeded` at 413.
|
|
22
|
+
*/
|
|
23
|
+
metered: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* The byte ceiling for one request: the `storageBytes` quota where an entitlement provider enforces that
|
|
28
|
+
* meter, the configured `quota.bytesPerOwner` where it does not.
|
|
29
|
+
*
|
|
30
|
+
* **Configured behavior is the fallback, never replaced by surprise.** With no provider composed, or one that
|
|
31
|
+
* leaves `storageBytes` advisory, this is `bytesPerOwner` exactly, counted per owner — which is what every
|
|
32
|
+
* project had before plans could say how much a customer may store. Enforced, it is the billing subject's
|
|
33
|
+
* effective quota: their plan's number, a number pinned for them, or `null` — unlimited — for an
|
|
34
|
+
* `unrestricted` subject. Under organization billing that subject is the organization, and so is the sum.
|
|
35
|
+
*
|
|
36
|
+
* A provider whose read fails refuses the write rather than serving it unbounded: `meterLimit` throws, and
|
|
37
|
+
* the request fails with it. Over-counting is recoverable; under-counting is not.
|
|
38
|
+
*/
|
|
39
|
+
export async function storageCeiling(c: Context<PithyHonoEnv>, config: StorageConfig): Promise<StorageCeiling> {
|
|
40
|
+
const ceiling = await meterLimit(c, "storageBytes");
|
|
41
|
+
return ceiling.enforced
|
|
42
|
+
? { limitBytes: ceiling.limit, tenant: ceiling.tenant, metered: true }
|
|
43
|
+
: { limitBytes: config.quota.bytesPerOwner, tenant: null, metered: false };
|
|
44
|
+
}
|
package/src/quota/quota.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
|
|
4
|
+
import { PithyError } from "@pithy-sh/core/src/error/pithyError";
|
|
4
5
|
import { sql } from "kysely";
|
|
5
6
|
import type { StorageObjectRow, StorageObjectStatus } from "../data/storageObject";
|
|
6
7
|
import { STORAGE_OBJECTS_TABLE, type StorageDatabase } from "../data/tables";
|
|
@@ -44,6 +45,10 @@ import { StorageQuotaExceededError } from "../error/errors";
|
|
|
44
45
|
*
|
|
45
46
|
* A `null` limit is unlimited and skips the query entirely. A `null` owner (a system object) has no
|
|
46
47
|
* quota — there is no principal to bill it to.
|
|
48
|
+
*
|
|
49
|
+
* **Whose bytes are summed is the ceiling's to say.** A configured `bytesPerOwner` is per owner. An
|
|
50
|
+
* organization's plan is the organization's, so a check carrying `organizationId` sums every row held for
|
|
51
|
+
* that organization, whoever uploaded it — counted per member, the plan's number would be per member.
|
|
47
52
|
*/
|
|
48
53
|
|
|
49
54
|
/** What a quota check needs. */
|
|
@@ -52,8 +57,19 @@ export interface QuotaCheck {
|
|
|
52
57
|
db: StorageDatabase;
|
|
53
58
|
/** The owner the bytes are billed to. Null is a system object, which no quota applies to. */
|
|
54
59
|
ownerId: string | null;
|
|
55
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* The organization the ceiling covers, when it is an organization's. Set, the sum is every row held for it
|
|
62
|
+
* rather than the owner's alone. The row being written must be held for it too, or it would never count.
|
|
63
|
+
*/
|
|
64
|
+
organizationId?: string;
|
|
65
|
+
/** The ceiling in bytes — the configured `bytesPerOwner`, or a plan's. Null is unlimited. */
|
|
56
66
|
limitBytes: number | null;
|
|
67
|
+
/**
|
|
68
|
+
* Whether `limitBytes` is an enforced `storageBytes` meter's. True, a refusal is the plan's own —
|
|
69
|
+
* `payments/quota_exceeded` at 402, as the `emails` and `mediaItems` meters refuse — because the remedy
|
|
70
|
+
* is a bigger tier or a pinned number, not a config edit. Absent or false, it is `storage/quota_exceeded`.
|
|
71
|
+
*/
|
|
72
|
+
metered?: boolean;
|
|
57
73
|
/** The bytes this upload declares — what the pending row is about to reserve. */
|
|
58
74
|
additionalBytes: number;
|
|
59
75
|
}
|
|
@@ -84,17 +100,24 @@ export interface QuotaSettlement extends QuotaCheck {
|
|
|
84
100
|
*/
|
|
85
101
|
export const QUOTA_COUNTED_STATUSES: readonly StorageObjectStatus[] = ["pending", "stored"];
|
|
86
102
|
|
|
103
|
+
/** The column and value a sum is keyed on: the organization's rows where the ceiling is one's, the owner's otherwise. */
|
|
104
|
+
function heldBy(ownerId: string, organizationId: string | undefined): ["ownerId" | "organizationId", string] {
|
|
105
|
+
return organizationId === undefined ? ["ownerId", ownerId] : ["organizationId", organizationId];
|
|
106
|
+
}
|
|
107
|
+
|
|
87
108
|
/**
|
|
88
|
-
* Bytes an owner currently holds
|
|
89
|
-
* rows are excluded — they hold no bytes
|
|
109
|
+
* Bytes an owner currently holds — or, with `organizationId`, every byte held for that organization:
|
|
110
|
+
* every `pending` reservation plus every `stored` object. `failed` rows are excluded — they hold no bytes
|
|
111
|
+
* and reserve nothing.
|
|
90
112
|
*/
|
|
91
|
-
export async function usedBytes(db: StorageDatabase, ownerId: string): Promise<number> {
|
|
113
|
+
export async function usedBytes(db: StorageDatabase, ownerId: string, organizationId?: string): Promise<number> {
|
|
114
|
+
const [column, holder] = heldBy(ownerId, organizationId);
|
|
92
115
|
const row = await db
|
|
93
116
|
.selectFrom(STORAGE_OBJECTS_TABLE)
|
|
94
117
|
// `size` is nullable (a caller may not declare one), and SUM over no rows is NULL — coalesce both
|
|
95
118
|
// to zero here rather than letting a null leak into arithmetic.
|
|
96
119
|
.select(sql<number>`coalesce(sum(size), 0)`.as("used"))
|
|
97
|
-
.where(
|
|
120
|
+
.where(column, "=", holder)
|
|
98
121
|
.where("status", "in", [...QUOTA_COUNTED_STATUSES])
|
|
99
122
|
.executeTakeFirst();
|
|
100
123
|
return Number(row?.used ?? 0);
|
|
@@ -111,7 +134,25 @@ function assertDeclaredSize(additionalBytes: number): void {
|
|
|
111
134
|
}
|
|
112
135
|
|
|
113
136
|
/**
|
|
114
|
-
*
|
|
137
|
+
* The refusal for a write past `check`'s ceiling: the plan's where the ceiling is a meter's, storage's own
|
|
138
|
+
* where it is the configured `bytesPerOwner`. `detail` is throw-site context and never crosses the wire.
|
|
139
|
+
*/
|
|
140
|
+
function quotaRefusal(check: QuotaCheck, detail: string): PithyError {
|
|
141
|
+
if (!check.metered) return new StorageQuotaExceededError({ detail });
|
|
142
|
+
return new PithyError({
|
|
143
|
+
code: "payments/quota_exceeded",
|
|
144
|
+
status: 402,
|
|
145
|
+
message: "This plan's storage allowance is used up.",
|
|
146
|
+
action:
|
|
147
|
+
"Delete something first, upgrade the plan, or pin a larger storageBytes allowance for this subject from the control plane.",
|
|
148
|
+
params: { meter: "storageBytes", limit: check.limitBytes ?? 0 },
|
|
149
|
+
detail,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Throw `storage/quota_exceeded` when this upload would put the owner over their limit — or
|
|
155
|
+
* `payments/quota_exceeded` when the limit is a metered plan's ({@link QuotaCheck.metered}). The
|
|
115
156
|
* comparison is `used + additional > limit`, so an upload landing exactly on the limit is allowed —
|
|
116
157
|
* a quota is a ceiling you may reach, not one you must stay under.
|
|
117
158
|
*
|
|
@@ -124,17 +165,18 @@ export async function assertWithinQuota(check: QuotaCheck): Promise<void> {
|
|
|
124
165
|
if (check.limitBytes === null || check.ownerId === null) return;
|
|
125
166
|
assertDeclaredSize(check.additionalBytes);
|
|
126
167
|
|
|
127
|
-
const used = await usedBytes(check.db, check.ownerId);
|
|
168
|
+
const used = await usedBytes(check.db, check.ownerId, check.organizationId);
|
|
128
169
|
if (used + check.additionalBytes > check.limitBytes) {
|
|
129
|
-
throw
|
|
130
|
-
|
|
131
|
-
|
|
170
|
+
throw quotaRefusal(
|
|
171
|
+
check,
|
|
172
|
+
`owner would hold ${used + check.additionalBytes} bytes against a limit of ${check.limitBytes}`,
|
|
173
|
+
);
|
|
132
174
|
}
|
|
133
175
|
}
|
|
134
176
|
|
|
135
177
|
/**
|
|
136
178
|
* Insert a storage row **only if** it fits inside the owner's quota, and throw
|
|
137
|
-
* `storage/quota_exceeded` when it does not.
|
|
179
|
+
* `storage/quota_exceeded` (or a metered plan's `payments/quota_exceeded`) when it does not.
|
|
138
180
|
*
|
|
139
181
|
* The sum and the write are one statement — `INSERT … SELECT <values> WHERE (SELECT sum(size) …) <=
|
|
140
182
|
* limit - additional` — because two statements cannot be made to hold a reservation without a
|
|
@@ -152,6 +194,7 @@ export async function insertReservingQuota(reservation: QuotaReservation): Promi
|
|
|
152
194
|
return;
|
|
153
195
|
}
|
|
154
196
|
assertDeclaredSize(additionalBytes);
|
|
197
|
+
const [column, holder] = heldBy(ownerId, reservation.organizationId);
|
|
155
198
|
|
|
156
199
|
// The column list is read off the encoded record rather than spelled out, so a new column on
|
|
157
200
|
// `StorageObject` is carried here without an edit — and stays camelCase, which `CamelCasePlugin`
|
|
@@ -166,7 +209,7 @@ export async function insertReservingQuota(reservation: QuotaReservation): Promi
|
|
|
166
209
|
eb
|
|
167
210
|
.selectFrom(STORAGE_OBJECTS_TABLE)
|
|
168
211
|
.select(sql<number>`coalesce(sum(size), 0)`.as("used"))
|
|
169
|
-
.where(
|
|
212
|
+
.where(column, "=", holder)
|
|
170
213
|
.where("status", "in", [...QUOTA_COUNTED_STATUSES]),
|
|
171
214
|
"<=",
|
|
172
215
|
limitBytes - additionalBytes,
|
|
@@ -181,15 +224,13 @@ export async function insertReservingQuota(reservation: QuotaReservation): Promi
|
|
|
181
224
|
.executeTakeFirst();
|
|
182
225
|
|
|
183
226
|
if (!inserted) {
|
|
184
|
-
throw
|
|
185
|
-
detail: `reservation of ${additionalBytes} bytes lost against a limit of ${limitBytes}`,
|
|
186
|
-
});
|
|
227
|
+
throw quotaRefusal(reservation, `reservation of ${additionalBytes} bytes lost against a limit of ${limitBytes}`);
|
|
187
228
|
}
|
|
188
229
|
}
|
|
189
230
|
|
|
190
231
|
/**
|
|
191
232
|
* Rewrite one row **only if** the bytes it newly claims still fit inside the owner's quota, and throw
|
|
192
|
-
* `storage/quota_exceeded` when they do not.
|
|
233
|
+
* `storage/quota_exceeded` (or a metered plan's `payments/quota_exceeded`) when they do not.
|
|
193
234
|
*
|
|
194
235
|
* This is {@link insertReservingQuota}'s other half, and it exists for the identical reason. A
|
|
195
236
|
* completion measures what R2 actually holds against what the pending row reserved, and the difference
|
|
@@ -212,6 +253,7 @@ export async function updateSettlingQuota(settlement: QuotaSettlement): Promise<
|
|
|
212
253
|
return;
|
|
213
254
|
}
|
|
214
255
|
assertDeclaredSize(additionalBytes);
|
|
256
|
+
const [column, holder] = heldBy(ownerId, settlement.organizationId);
|
|
215
257
|
|
|
216
258
|
const updated = await db
|
|
217
259
|
.updateTable(STORAGE_OBJECTS_TABLE)
|
|
@@ -224,7 +266,7 @@ export async function updateSettlingQuota(settlement: QuotaSettlement): Promise<
|
|
|
224
266
|
eb
|
|
225
267
|
.selectFrom(STORAGE_OBJECTS_TABLE)
|
|
226
268
|
.select(sql<number>`coalesce(sum(size), 0)`.as("used"))
|
|
227
|
-
.where(
|
|
269
|
+
.where(column, "=", holder)
|
|
228
270
|
.where("status", "in", [...QUOTA_COUNTED_STATUSES]),
|
|
229
271
|
"<=",
|
|
230
272
|
limitBytes - additionalBytes,
|
|
@@ -234,8 +276,9 @@ export async function updateSettlingQuota(settlement: QuotaSettlement): Promise<
|
|
|
234
276
|
.executeTakeFirst();
|
|
235
277
|
|
|
236
278
|
if (!updated) {
|
|
237
|
-
throw
|
|
238
|
-
|
|
239
|
-
|
|
279
|
+
throw quotaRefusal(
|
|
280
|
+
settlement,
|
|
281
|
+
`settlement of ${additionalBytes} further bytes lost against a limit of ${limitBytes}`,
|
|
282
|
+
);
|
|
240
283
|
}
|
|
241
284
|
}
|