@pithy-sh/storage 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capability.d.ts +43 -0
- package/dist/capability.d.ts.map +1 -0
- package/dist/capability.js +74 -0
- package/dist/config/config.d.ts +25 -0
- package/dist/config/config.d.ts.map +1 -0
- package/dist/config/config.js +47 -0
- package/dist/data/share.d.ts +25 -0
- package/dist/data/share.d.ts.map +1 -0
- package/dist/data/share.js +26 -0
- package/dist/data/storageObject.d.ts +53 -0
- package/dist/data/storageObject.d.ts.map +1 -0
- package/dist/data/storageObject.js +42 -0
- package/dist/data/tables.d.ts +21 -0
- package/dist/data/tables.d.ts.map +1 -0
- package/dist/data/tables.js +24 -0
- package/dist/error/errors.d.ts +58 -0
- package/dist/error/errors.d.ts.map +1 -0
- package/dist/error/errors.js +87 -0
- package/dist/http/guard.d.ts +18 -0
- package/dist/http/guard.d.ts.map +1 -0
- package/dist/http/guard.js +27 -0
- package/dist/http/handlers.d.ts +230 -0
- package/dist/http/handlers.d.ts.map +1 -0
- package/dist/http/handlers.js +505 -0
- package/dist/http/routes.d.ts +68 -0
- package/dist/http/routes.d.ts.map +1 -0
- package/dist/http/routes.js +133 -0
- package/dist/http/schemas.d.ts +72 -0
- package/dist/http/schemas.d.ts.map +1 -0
- package/dist/http/schemas.js +91 -0
- package/dist/http/serve.d.ts +119 -0
- package/dist/http/serve.d.ts.map +1 -0
- package/dist/http/serve.js +207 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/migrations/0001_objects.d.ts +22 -0
- package/dist/migrations/0001_objects.d.ts.map +1 -0
- package/dist/migrations/0001_objects.js +39 -0
- package/dist/object/cloudflare.d.ts +21 -0
- package/dist/object/cloudflare.d.ts.map +1 -0
- package/dist/object/cloudflare.js +46 -0
- package/dist/object/key.d.ts +27 -0
- package/dist/object/key.d.ts.map +1 -0
- package/dist/object/key.js +35 -0
- package/dist/object/multipart.d.ts +76 -0
- package/dist/object/multipart.d.ts.map +1 -0
- package/dist/object/multipart.js +129 -0
- package/dist/object/store.d.ts +201 -0
- package/dist/object/store.d.ts.map +1 -0
- package/dist/object/store.js +176 -0
- package/dist/provision/provisionStorage.d.ts +137 -0
- package/dist/provision/provisionStorage.d.ts.map +1 -0
- package/dist/provision/provisionStorage.js +110 -0
- package/dist/provision/resolveStorageConfig.d.ts +43 -0
- package/dist/provision/resolveStorageConfig.d.ts.map +1 -0
- package/dist/provision/resolveStorageConfig.js +31 -0
- package/dist/quota/quota.d.ts +125 -0
- package/dist/quota/quota.d.ts.map +1 -0
- package/dist/quota/quota.js +100 -0
- package/dist/secret/registry.d.ts +70 -0
- package/dist/secret/registry.d.ts.map +1 -0
- package/dist/secret/registry.js +82 -0
- package/dist/seeds/example.d.ts +3 -0
- package/dist/seeds/example.d.ts.map +1 -0
- package/dist/seeds/example.js +109 -0
- package/dist/test-utils/liveStorage.d.ts +50 -0
- package/dist/test-utils/liveStorage.d.ts.map +1 -0
- package/dist/test-utils/liveStorage.js +196 -0
- package/dist/version.generated.d.ts +5 -0
- package/dist/version.generated.d.ts.map +1 -0
- package/dist/version.generated.js +7 -0
- package/dist/workflows/retryPolicy.d.ts +37 -0
- package/dist/workflows/retryPolicy.d.ts.map +1 -0
- package/dist/workflows/retryPolicy.js +41 -0
- package/dist/workflows/specs.d.ts +55 -0
- package/dist/workflows/specs.d.ts.map +1 -0
- package/dist/workflows/specs.js +58 -0
- package/dist/workflows/sweep.d.ts +82 -0
- package/dist/workflows/sweep.d.ts.map +1 -0
- package/dist/workflows/sweep.js +143 -0
- package/dist/workflows/worker.d.ts +49 -0
- package/dist/workflows/worker.d.ts.map +1 -0
- package/dist/workflows/worker.js +50 -0
- package/package.json +22 -11
- package/src/version.generated.ts +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quota.d.ts","sourceRoot":"","sources":["../../src/quota/quota.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAyB,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAG7E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,gCAAgC;AAChC,MAAM,WAAW,UAAU;IACzB,4BAA4B;IAC5B,EAAE,EAAE,eAAe,CAAC;IACpB,6FAA6F;IAC7F,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,oEAAoE;IACpE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,iFAAiF;IACjF,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,uEAAuE;AACvE,MAAM,WAAW,gBAAiB,SAAQ,UAAU;IAClD,sGAAsG;IACtG,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,mEAAmE;AACnE,MAAM,WAAW,eAAgB,SAAQ,UAAU;IACjD;;;OAGG;IACH,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,SAAS,mBAAmB,EAA0B,CAAC;AAE5F;;;GAGG;AACH,wBAAsB,SAAS,CAAC,EAAE,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAUrF;AAYD;;;;;;;;;GASG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAUxE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,oBAAoB,CAAC,WAAW,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAwCvF;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAsB,mBAAmB,CAAC,UAAU,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAiCpF"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { StorageQuotaExceededError } from "../error/errors.js";
|
|
2
|
+
import { STORAGE_OBJECTS_TABLE } from "../data/tables.js";
|
|
3
|
+
import { sql } from "kysely";
|
|
4
|
+
//#region src/quota/quota.ts
|
|
5
|
+
/**
|
|
6
|
+
* The statuses that bill an owner: a `pending` row reserves its declared bytes and a `stored` one
|
|
7
|
+
* holds real ones. `failed` bills nothing.
|
|
8
|
+
*
|
|
9
|
+
* Exported because the orphan sweep asks the same question of the same rows — a key is *claimed* if
|
|
10
|
+
* and only if a row bills for it — and the two predicates drifting apart is what let a `failed` row
|
|
11
|
+
* shield its R2 object from collection while counting toward nobody's quota
|
|
12
|
+
* (`workflows/sweep.ts`).
|
|
13
|
+
*/
|
|
14
|
+
const QUOTA_COUNTED_STATUSES = ["pending", "stored"];
|
|
15
|
+
/**
|
|
16
|
+
* Bytes an owner currently holds: every `pending` reservation plus every `stored` object. `failed`
|
|
17
|
+
* rows are excluded — they hold no bytes and reserve nothing.
|
|
18
|
+
*/
|
|
19
|
+
async function usedBytes(db, ownerId) {
|
|
20
|
+
const row = await db.selectFrom(STORAGE_OBJECTS_TABLE).select(sql`coalesce(sum(size), 0)`.as("used")).where("ownerId", "=", ownerId).where("status", "in", [...QUOTA_COUNTED_STATUSES]).executeTakeFirst();
|
|
21
|
+
return Number(row?.used ?? 0);
|
|
22
|
+
}
|
|
23
|
+
/** A size a caller declared has to be a real byte count. Zero is fine; negative is a client bug. */
|
|
24
|
+
function assertDeclaredSize(additionalBytes) {
|
|
25
|
+
if (additionalBytes >= 0) return;
|
|
26
|
+
throw new StorageQuotaExceededError({
|
|
27
|
+
message: "An upload needs a size.",
|
|
28
|
+
action: "Declare the file's byte count when you start the upload.",
|
|
29
|
+
detail: `negative declared size ${additionalBytes}`
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Throw `storage/quota_exceeded` when this upload would put the owner over their limit. The
|
|
34
|
+
* comparison is `used + additional > limit`, so an upload landing exactly on the limit is allowed —
|
|
35
|
+
* a quota is a ceiling you may reach, not one you must stay under.
|
|
36
|
+
*
|
|
37
|
+
* **This is a pre-check, not the reservation.** It reads a total nothing holds still, so between it
|
|
38
|
+
* and any later write a concurrent init can have taken the room. Use it to refuse an obviously
|
|
39
|
+
* oversized upload *before* opening an R2 multipart upload there would then be nothing to clean up
|
|
40
|
+
* for, and let {@link insertReservingQuota} be the boundary that actually decides.
|
|
41
|
+
*/
|
|
42
|
+
async function assertWithinQuota(check) {
|
|
43
|
+
if (check.limitBytes === null || check.ownerId === null) return;
|
|
44
|
+
assertDeclaredSize(check.additionalBytes);
|
|
45
|
+
const used = await usedBytes(check.db, check.ownerId);
|
|
46
|
+
if (used + check.additionalBytes > check.limitBytes) throw new StorageQuotaExceededError({ detail: `owner would hold ${used + check.additionalBytes} bytes against a limit of ${check.limitBytes}` });
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Insert a storage row **only if** it fits inside the owner's quota, and throw
|
|
50
|
+
* `storage/quota_exceeded` when it does not.
|
|
51
|
+
*
|
|
52
|
+
* The sum and the write are one statement — `INSERT … SELECT <values> WHERE (SELECT sum(size) …) <=
|
|
53
|
+
* limit - additional` — because two statements cannot be made to hold a reservation without a
|
|
54
|
+
* transaction, and D1 has none to offer. SQLite evaluates the sub-select while it holds the write
|
|
55
|
+
* lock, so a burst of concurrent inits is serialized by the database itself: each one sees every
|
|
56
|
+
* reservation that landed before it, and the first that would breach the limit inserts nothing.
|
|
57
|
+
*
|
|
58
|
+
* `RETURNING id` is how "inserted nothing" is read back. Zero rows is not an error to SQLite — the
|
|
59
|
+
* `WHERE` simply matched nothing — so the absence of a returned row *is* the quota answer.
|
|
60
|
+
*/
|
|
61
|
+
async function insertReservingQuota(reservation) {
|
|
62
|
+
const { db, ownerId, limitBytes, additionalBytes, record } = reservation;
|
|
63
|
+
if (limitBytes === null || ownerId === null) {
|
|
64
|
+
await db.insertInto(STORAGE_OBJECTS_TABLE).values(record).execute();
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
assertDeclaredSize(additionalBytes);
|
|
68
|
+
const columns = Object.keys(record);
|
|
69
|
+
const values = db.selectNoFrom((eb) => columns.map((column) => eb.val(record[column]).as(column))).where((eb) => eb(eb.selectFrom(STORAGE_OBJECTS_TABLE).select(sql`coalesce(sum(size), 0)`.as("used")).where("ownerId", "=", ownerId).where("status", "in", [...QUOTA_COUNTED_STATUSES]), "<=", limitBytes - additionalBytes));
|
|
70
|
+
if (!await db.insertInto("pithyStorageObjects").columns(columns).expression(values).returning("id").executeTakeFirst()) throw new StorageQuotaExceededError({ detail: `reservation of ${additionalBytes} bytes lost against a limit of ${limitBytes}` });
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Rewrite one row **only if** the bytes it newly claims still fit inside the owner's quota, and throw
|
|
74
|
+
* `storage/quota_exceeded` when they do not.
|
|
75
|
+
*
|
|
76
|
+
* This is {@link insertReservingQuota}'s other half, and it exists for the identical reason. A
|
|
77
|
+
* completion measures what R2 actually holds against what the pending row reserved, and the difference
|
|
78
|
+
* is bytes the owner has not been granted yet. Settling that difference with a separate
|
|
79
|
+
* {@link assertWithinQuota} would put the enforcement back where it never worked: two completions
|
|
80
|
+
* racing both read a total neither has yet contributed to, both pass, and the owner ends over the
|
|
81
|
+
* limit — the same defect the conditional insert was written to close, one lifecycle stage later.
|
|
82
|
+
*
|
|
83
|
+
* `additionalBytes` is the **overshoot**, not the object's size. The row being updated is still
|
|
84
|
+
* `pending`, so the sub-select in the `WHERE` already counts its reservation: SQLite evaluates the
|
|
85
|
+
* condition against the pre-update table. Passing the full size would bill the reservation twice.
|
|
86
|
+
*
|
|
87
|
+
* `RETURNING id` reads back whether the row moved. Zero rows updated is not an error to SQLite — the
|
|
88
|
+
* `WHERE` simply matched nothing — so the absence of a returned row *is* the quota answer.
|
|
89
|
+
*/
|
|
90
|
+
async function updateSettlingQuota(settlement) {
|
|
91
|
+
const { db, ownerId, limitBytes, additionalBytes, record } = settlement;
|
|
92
|
+
if (limitBytes === null || ownerId === null) {
|
|
93
|
+
await db.updateTable(STORAGE_OBJECTS_TABLE).set(record).where("id", "=", record.id).execute();
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
assertDeclaredSize(additionalBytes);
|
|
97
|
+
if (!await db.updateTable("pithyStorageObjects").set(record).where("id", "=", record.id).where((eb) => eb(eb.selectFrom("pithyStorageObjects").select(sql`coalesce(sum(size), 0)`.as("used")).where("ownerId", "=", ownerId).where("status", "in", [...QUOTA_COUNTED_STATUSES]), "<=", limitBytes - additionalBytes)).returning("id").executeTakeFirst()) throw new StorageQuotaExceededError({ detail: `settlement of ${additionalBytes} further bytes lost against a limit of ${limitBytes}` });
|
|
98
|
+
}
|
|
99
|
+
//#endregion
|
|
100
|
+
export { QUOTA_COUNTED_STATUSES, assertWithinQuota, insertReservingQuota, updateSettlingQuota, usedBytes };
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { SecretOrigin, SecretRotation } from "@pithy-sh/core/src/capability/secretOrigin";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
/**
|
|
4
|
+
* The R2 credential bundle the {@link ObjectStore} seam reads, and the **factory** that declares it
|
|
5
|
+
* under any name (CLAUDE.md §Secrets).
|
|
6
|
+
*
|
|
7
|
+
* Why a factory and not a bare name. `sharedSecretsStore(env, registry)` throws for any name absent
|
|
8
|
+
* from the *aggregated* registry, and that registry is built only from the composed capabilities'
|
|
9
|
+
* `secretRegistry` slices. So `objectStore({ bucket, secretName })` cannot resolve a string on its
|
|
10
|
+
* own — something must have declared that name, with a schema, on a capability. The factory is that
|
|
11
|
+
* something: storage declares `storage-r2-credentials` for `STORAGE_BUCKET`, `@pithy-sh/media`
|
|
12
|
+
* declares `media-r2-credentials` for `MEDIA_BUCKET`, and both point the same seam at their own
|
|
13
|
+
* bucket without either package knowing the other's name.
|
|
14
|
+
*
|
|
15
|
+
* One factory is also what makes the join key safe. `aggregateSecretRegistries` allows a name to be
|
|
16
|
+
* declared twice only when every axis agrees (`backend`, `scope`, `valueType`, `rotatable`); two
|
|
17
|
+
* hand-written declarations drift, one factory cannot.
|
|
18
|
+
*/
|
|
19
|
+
/**
|
|
20
|
+
* The credentials one R2 bucket is addressed and presigned with. Composed from
|
|
21
|
+
* {@link R2Credentials} rather than redeclaring the key pair — that shape is `@pithy-sh/cloudflare`'s
|
|
22
|
+
* to define, and `CloudflareR2Manager` validates its config against it.
|
|
23
|
+
*/
|
|
24
|
+
export declare const R2StorageCredentials: z.ZodObject<{
|
|
25
|
+
accessKeyId: z.ZodString;
|
|
26
|
+
secretAccessKey: z.ZodString;
|
|
27
|
+
accountId: z.ZodString;
|
|
28
|
+
bucket: z.ZodString;
|
|
29
|
+
apiToken: z.ZodString;
|
|
30
|
+
}, z.core.$strip>;
|
|
31
|
+
export type R2StorageCredentials = z.output<typeof R2StorageCredentials>;
|
|
32
|
+
/**
|
|
33
|
+
* One R2-credentials registry entry. Spelled out as a type so the factory can return
|
|
34
|
+
* `Record<N, R2CredentialsEntry>` — a computed key alone widens to a string index signature, which
|
|
35
|
+
* would lose the literal name `SecretsAccessor.get` narrows on.
|
|
36
|
+
*/
|
|
37
|
+
export interface R2CredentialsEntry {
|
|
38
|
+
/**
|
|
39
|
+
* An encrypted row in the per-environment secrets D1 — where these values actually live.
|
|
40
|
+
*
|
|
41
|
+
* No `wrangler.jsonc` binds an R2 credential bundle from the Cloudflare Secrets Store; `pithy storage
|
|
42
|
+
* provision` and `pithy media provision` write it through `dispatchSecretWrite` → the manager
|
|
43
|
+
* Workflow → `SystemSecretsStore`, which is the D1 path. The registry's `backend` is the *single*
|
|
44
|
+
* place a secret's storage location is decided and is what the read seam routes on, so declaring
|
|
45
|
+
* `cf-secrets-store` here would send every deployed read to a binding that does not exist.
|
|
46
|
+
*/
|
|
47
|
+
backend: "d1";
|
|
48
|
+
/** Each environment addresses its own bucket with its own key pair. */
|
|
49
|
+
scope: "environment";
|
|
50
|
+
/** R2 S3 key pairs are minted and replaced whole, not rotated with overlap windows. */
|
|
51
|
+
rotatable: false;
|
|
52
|
+
/** A JSON bundle, validated against {@link R2StorageCredentials} before it is exposed. */
|
|
53
|
+
valueType: "json";
|
|
54
|
+
/** The bundle's shape. */
|
|
55
|
+
schema: typeof R2StorageCredentials;
|
|
56
|
+
/** Where the pair comes from. `obtained`, always — see {@link R2_CREDENTIALS_PAGE}. */
|
|
57
|
+
origin: SecretOrigin;
|
|
58
|
+
/** How the pair is replaced. `manual`, always — the same page, by the same human. */
|
|
59
|
+
rotation: SecretRotation;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* A one-entry secret-registry slice declaring `name` as an R2 credential bundle. Hang it on the
|
|
63
|
+
* declaring capability's `secretRegistry`, then pass the same `name` to `objectStore`.
|
|
64
|
+
*/
|
|
65
|
+
export declare function r2CredentialsRegistry<const N extends string>(name: N): Record<N, R2CredentialsEntry>;
|
|
66
|
+
/** The name storage's own bucket credentials are stored and resolved under. */
|
|
67
|
+
export declare const STORAGE_R2_SECRET = "storage-r2-credentials";
|
|
68
|
+
/** The storage capability's secret-registry slice — aggregated into the shared accessor at startup. */
|
|
69
|
+
export declare const storageSecretsRegistry: Record<"storage-r2-credentials", R2CredentialsEntry>;
|
|
70
|
+
//# sourceMappingURL=registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/secret/registry.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,4CAA4C,CAAC;AAE/F,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;;;;;;;;;;GAeG;AAEH;;;;GAIG;AACH,eAAO,MAAM,oBAAoB;;;;;;iBASmF,CAAC;AACrH,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAEzE;;;;GAIG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;;;;;OAQG;IACH,OAAO,EAAE,IAAI,CAAC;IACd,uEAAuE;IACvE,KAAK,EAAE,aAAa,CAAC;IACrB,uFAAuF;IACvF,SAAS,EAAE,KAAK,CAAC;IACjB,0FAA0F;IAC1F,SAAS,EAAE,MAAM,CAAC;IAClB,0BAA0B;IAC1B,MAAM,EAAE,OAAO,oBAAoB,CAAC;IACpC,uFAAuF;IACvF,MAAM,EAAE,YAAY,CAAC;IACrB,qFAAqF;IACrF,QAAQ,EAAE,cAAc,CAAC;CAC1B;AAmCD;;;GAGG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,CAAC,CAAC,SAAS,MAAM,EAAE,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAEpG;AAED,+EAA+E;AAC/E,eAAO,MAAM,iBAAiB,2BAA2B,CAAC;AAE1D,uGAAuG;AACvG,eAAO,MAAM,sBAAsB,sDAA2C,CAAC"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { R2Credentials } from "@pithy-sh/cloudflare/src/r2/r2Credentials";
|
|
3
|
+
import { defineSecretRegistry } from "@pithy-sh/secrets/src/registry";
|
|
4
|
+
//#region src/secret/registry.ts
|
|
5
|
+
/**
|
|
6
|
+
* The R2 credential bundle the {@link ObjectStore} seam reads, and the **factory** that declares it
|
|
7
|
+
* under any name (CLAUDE.md §Secrets).
|
|
8
|
+
*
|
|
9
|
+
* Why a factory and not a bare name. `sharedSecretsStore(env, registry)` throws for any name absent
|
|
10
|
+
* from the *aggregated* registry, and that registry is built only from the composed capabilities'
|
|
11
|
+
* `secretRegistry` slices. So `objectStore({ bucket, secretName })` cannot resolve a string on its
|
|
12
|
+
* own — something must have declared that name, with a schema, on a capability. The factory is that
|
|
13
|
+
* something: storage declares `storage-r2-credentials` for `STORAGE_BUCKET`, `@pithy-sh/media`
|
|
14
|
+
* declares `media-r2-credentials` for `MEDIA_BUCKET`, and both point the same seam at their own
|
|
15
|
+
* bucket without either package knowing the other's name.
|
|
16
|
+
*
|
|
17
|
+
* One factory is also what makes the join key safe. `aggregateSecretRegistries` allows a name to be
|
|
18
|
+
* declared twice only when every axis agrees (`backend`, `scope`, `valueType`, `rotatable`); two
|
|
19
|
+
* hand-written declarations drift, one factory cannot.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* The credentials one R2 bucket is addressed and presigned with. Composed from
|
|
23
|
+
* {@link R2Credentials} rather than redeclaring the key pair — that shape is `@pithy-sh/cloudflare`'s
|
|
24
|
+
* to define, and `CloudflareR2Manager` validates its config against it.
|
|
25
|
+
*/
|
|
26
|
+
const R2StorageCredentials = R2Credentials.extend({
|
|
27
|
+
accountId: z.string().min(1).describe("The Cloudflare account id the bucket lives in — the S3 endpoint host."),
|
|
28
|
+
bucket: z.string().min(1).describe("The R2 bucket name every presigned URL and server-side object call targets."),
|
|
29
|
+
apiToken: z.string().min(1).describe("The R2-scoped Cloudflare API token the key pair was derived from. Carried alongside because an R2 S3 access key IS a CF API token — the id is the key id, the SHA-256 of the value is the secret — so whatever provisions the pair already holds it, and `CloudflareR2Manager` needs it to prove bucket access.")
|
|
30
|
+
}).describe("The account, key pair, scoped token, and bucket name the object store presigns and addresses R2 with.");
|
|
31
|
+
/**
|
|
32
|
+
* Where a human makes an R2 S3 access-key pair, and where the same human makes the next one.
|
|
33
|
+
*
|
|
34
|
+
* **Nothing here will ever mint one, and that is the fact the declaration carries.** Cloudflare has no
|
|
35
|
+
* API that returns an S3 access-key pair, so `pithy storage provision` and `pithy media provision` both
|
|
36
|
+
* take one as a flag and write it as given — a generated value would open no bucket and would replace a
|
|
37
|
+
* loud gap with a quiet one. Naming the page is the whole of the help the kit can offer, which is exactly
|
|
38
|
+
* what `obtained` is for.
|
|
39
|
+
*
|
|
40
|
+
* Origin and rotation name the same page because it is the same page: replacement is making another pair
|
|
41
|
+
* and deleting the old one. That is also why {@link R2CredentialsEntry.rotatable} is false — there is no
|
|
42
|
+
* overlap window to hold two live versions through.
|
|
43
|
+
*/
|
|
44
|
+
const R2_CREDENTIALS_PAGE = "https://developers.cloudflare.com/r2/api/tokens/";
|
|
45
|
+
/**
|
|
46
|
+
* The one entry shape every name shares — declared once so no two declarations can disagree.
|
|
47
|
+
*
|
|
48
|
+
* The two declaration axes live here rather than at each name for the reason the rest of the entry does:
|
|
49
|
+
* `storage-r2-credentials` and `media-r2-credentials` are the same kind of credential from the same
|
|
50
|
+
* issuer, and `aggregateSecretRegistries` refuses a name two capabilities describe differently. One
|
|
51
|
+
* factory is how they cannot.
|
|
52
|
+
*/
|
|
53
|
+
const R2_CREDENTIALS_ENTRY = {
|
|
54
|
+
backend: "d1",
|
|
55
|
+
scope: "environment",
|
|
56
|
+
rotatable: false,
|
|
57
|
+
valueType: "json",
|
|
58
|
+
schema: R2StorageCredentials,
|
|
59
|
+
origin: {
|
|
60
|
+
kind: "obtained",
|
|
61
|
+
issuer: "cloudflare",
|
|
62
|
+
documentation: R2_CREDENTIALS_PAGE
|
|
63
|
+
},
|
|
64
|
+
rotation: {
|
|
65
|
+
kind: "manual",
|
|
66
|
+
issuer: "cloudflare",
|
|
67
|
+
documentation: R2_CREDENTIALS_PAGE
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* A one-entry secret-registry slice declaring `name` as an R2 credential bundle. Hang it on the
|
|
72
|
+
* declaring capability's `secretRegistry`, then pass the same `name` to `objectStore`.
|
|
73
|
+
*/
|
|
74
|
+
function r2CredentialsRegistry(name) {
|
|
75
|
+
return defineSecretRegistry({ [name]: R2_CREDENTIALS_ENTRY });
|
|
76
|
+
}
|
|
77
|
+
/** The name storage's own bucket credentials are stored and resolved under. */
|
|
78
|
+
const STORAGE_R2_SECRET = "storage-r2-credentials";
|
|
79
|
+
/** The storage capability's secret-registry slice — aggregated into the shared accessor at startup. */
|
|
80
|
+
const storageSecretsRegistry = r2CredentialsRegistry(STORAGE_R2_SECRET);
|
|
81
|
+
//#endregion
|
|
82
|
+
export { R2StorageCredentials, STORAGE_R2_SECRET, r2CredentialsRegistry, storageSecretsRegistry };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"example.d.ts","sourceRoot":"","sources":["../../src/seeds/example.ts"],"names":[],"mappings":"AAIA,OAAO,EAA4C,KAAK,OAAO,EAAE,MAAM,8BAA8B,CAAC;AA+DtG,eAAO,MAAM,kBAAkB,EAAE,OAoD/B,CAAC"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { StorageObject } from "../data/storageObject.js";
|
|
2
|
+
import { STORAGE_OBJECTS_TABLE } from "../data/tables.js";
|
|
3
|
+
import { OBJECT_KEY_PREFIX } from "../object/key.js";
|
|
4
|
+
import { EXAMPLE_ADA, EXAMPLE_GRACE } from "@pithy-sh/core/src/seed/exampleIdentities";
|
|
5
|
+
import { d1SeedGroup, defineSeed } from "@pithy-sh/core/src/seed/seed";
|
|
6
|
+
//#region src/seeds/example.ts
|
|
7
|
+
/**
|
|
8
|
+
* Where the example set sorts among the whole project's seed registry. It runs after `auth` (100),
|
|
9
|
+
* whose example seeds the users these files belong to, so the owning identities exist first — the
|
|
10
|
+
* order encodes that dependency, exactly like the migration registry. It need not line up with
|
|
11
|
+
* {@link STORAGE_MIGRATION_ORDER}: a different registry, composed separately by `pithy seed`.
|
|
12
|
+
*/
|
|
13
|
+
const STORAGE_EXAMPLE_SEED_ORDER = 230;
|
|
14
|
+
/**
|
|
15
|
+
* A handful of demo files for the canonical example cast ({@link EXAMPLE_ADA} et al.).
|
|
16
|
+
*
|
|
17
|
+
* **Both halves are seeded — the rows and the bytes.** A storage row with no object in the bucket is
|
|
18
|
+
* precisely the divergence the orphan sweep exists to reconcile, so a fixture that seeded only D1
|
|
19
|
+
* would ship a demo where every download 404s and the sweep has work to do on a fresh install. The
|
|
20
|
+
* `r2` items below write the same keys the rows name, so `GET /storage/:id` returns real content on
|
|
21
|
+
* a freshly seeded environment.
|
|
22
|
+
*
|
|
23
|
+
* The ids and keys are **fixed**, not generated, which is what makes the set idempotent: re-seeding
|
|
24
|
+
* writes the same rows over the same keys rather than accumulating a new copy of Ada's notes on every
|
|
25
|
+
* run. One file is `public` on purpose, so the unauthenticated read path is exercised by the demo
|
|
26
|
+
* data rather than only by the tests.
|
|
27
|
+
*
|
|
28
|
+
* Composed in only when the project turns on `seed.includeExamples` (`pithy.config.ts`), and only for
|
|
29
|
+
* `dev` and `staging` — an example fixture never targets `prod`, regardless of that setting.
|
|
30
|
+
*/
|
|
31
|
+
/** Fixed ids, so the fixture is idempotent and the D1 rows and R2 objects cannot drift apart. */
|
|
32
|
+
const NOTES_ID = "6f1c8f36-30a5-4d5f-9f4d-6a5a52b9f0a1";
|
|
33
|
+
const PLAN_ID = "8c2d9a47-41b6-4e60-a05e-7b6b63c0e1b2";
|
|
34
|
+
const README_ID = "9d3ea058-52c7-4f71-b16f-8c7c74d1f2c3";
|
|
35
|
+
const CREATED_AT = /* @__PURE__ */ new Date("2026-01-01T00:00:00.000Z");
|
|
36
|
+
/** The demo bodies. Declared once so a row's `size` is the byte count actually written to R2. */
|
|
37
|
+
const BODIES = {
|
|
38
|
+
[NOTES_ID]: "Lovelace, notes on the Analytical Engine.\n",
|
|
39
|
+
[PLAN_ID]: "Hopper, plan for the next compiler.\n",
|
|
40
|
+
[README_ID]: "A public file. Anyone with the id may read this one.\n"
|
|
41
|
+
};
|
|
42
|
+
/** Bytes of a demo body — `size` must be the encoded length, not the character count. */
|
|
43
|
+
function byteLength(id) {
|
|
44
|
+
return new TextEncoder().encode(BODIES[id]).length;
|
|
45
|
+
}
|
|
46
|
+
/** The R2 object key a demo file lives under — the same opaque `obj/<uuid>` shape the handlers derive. */
|
|
47
|
+
function demoKey(id) {
|
|
48
|
+
return `${OBJECT_KEY_PREFIX}${id}`;
|
|
49
|
+
}
|
|
50
|
+
/** The bucket objects the rows point at, so a seeded environment can actually serve the bytes. */
|
|
51
|
+
const objects = Object.keys(BODIES).map((id) => ({
|
|
52
|
+
binding: "STORAGE_BUCKET",
|
|
53
|
+
key: demoKey(id),
|
|
54
|
+
body: BODIES[id],
|
|
55
|
+
contentType: "text/plain"
|
|
56
|
+
}));
|
|
57
|
+
const storageExampleSeed = defineSeed({
|
|
58
|
+
name: "example",
|
|
59
|
+
order: STORAGE_EXAMPLE_SEED_ORDER,
|
|
60
|
+
environments: ["dev", "staging"],
|
|
61
|
+
example: true,
|
|
62
|
+
d1: [d1SeedGroup("app", STORAGE_OBJECTS_TABLE, StorageObject, [
|
|
63
|
+
{
|
|
64
|
+
id: NOTES_ID,
|
|
65
|
+
key: demoKey(NOTES_ID),
|
|
66
|
+
path: "notes/analytical-engine.txt",
|
|
67
|
+
ownerId: EXAMPLE_ADA.id,
|
|
68
|
+
contentType: "text/plain",
|
|
69
|
+
size: byteLength(NOTES_ID),
|
|
70
|
+
visibility: "private",
|
|
71
|
+
checksum: null,
|
|
72
|
+
status: "stored",
|
|
73
|
+
uploadId: null,
|
|
74
|
+
createdAt: CREATED_AT,
|
|
75
|
+
updatedAt: CREATED_AT
|
|
76
|
+
},
|
|
77
|
+
{
|
|
78
|
+
id: PLAN_ID,
|
|
79
|
+
key: demoKey(PLAN_ID),
|
|
80
|
+
path: "plans/compiler.txt",
|
|
81
|
+
ownerId: EXAMPLE_GRACE.id,
|
|
82
|
+
contentType: "text/plain",
|
|
83
|
+
size: byteLength(PLAN_ID),
|
|
84
|
+
visibility: "private",
|
|
85
|
+
checksum: null,
|
|
86
|
+
status: "stored",
|
|
87
|
+
uploadId: null,
|
|
88
|
+
createdAt: CREATED_AT,
|
|
89
|
+
updatedAt: CREATED_AT
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
id: README_ID,
|
|
93
|
+
key: demoKey(README_ID),
|
|
94
|
+
path: "public/readme.txt",
|
|
95
|
+
ownerId: EXAMPLE_ADA.id,
|
|
96
|
+
contentType: "text/plain",
|
|
97
|
+
size: byteLength(README_ID),
|
|
98
|
+
visibility: "public",
|
|
99
|
+
checksum: null,
|
|
100
|
+
status: "stored",
|
|
101
|
+
uploadId: null,
|
|
102
|
+
createdAt: CREATED_AT,
|
|
103
|
+
updatedAt: CREATED_AT
|
|
104
|
+
}
|
|
105
|
+
])],
|
|
106
|
+
r2: objects
|
|
107
|
+
});
|
|
108
|
+
//#endregion
|
|
109
|
+
export { storageExampleSeed };
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { CloudflareR2Manager } from "@pithy-sh/cloudflare/src/r2/r2Manager";
|
|
2
|
+
import { type IntegrationCreds } from "@pithy-sh/cloudflare/src/test-utils/harness";
|
|
3
|
+
import { type StorageDatabase } from "../data/tables";
|
|
4
|
+
import { type ObjectStore } from "../object/store";
|
|
5
|
+
/**
|
|
6
|
+
* Scaffolding for this package's `*.integration.test.ts` suites — the live half of storage.
|
|
7
|
+
*
|
|
8
|
+
* `@pithy-sh/cloudflare`'s harness finds credentials and guarantees teardown. Storage needs two more
|
|
9
|
+
* things on top: the seam under test built the way a Worker builds it (an {@link objectStore} over a
|
|
10
|
+
* *resolved credential bundle*, not an injected fake), and a real database to reconcile against.
|
|
11
|
+
*
|
|
12
|
+
* **What is live here, and what cannot be.** The presigned half — presign, the whole multipart
|
|
13
|
+
* lifecycle, server-side copy — is real R2 over the S3 protocol, and it is the half Miniflare cannot
|
|
14
|
+
* emulate at all: it serves no S3 endpoint, so a presigned URL has nothing to address. The binding
|
|
15
|
+
* half is the mirror image. An `R2Bucket` exists only inside workerd, so outside a Worker there is no
|
|
16
|
+
* binding to call — {@link liveObjectStore} serves `head`, `list` and `delete` from the *same real
|
|
17
|
+
* bucket* over S3 instead, and leaves `get` throwing.
|
|
18
|
+
*
|
|
19
|
+
* That split is deliberate and it is the whole reason the two suites are complementary rather than
|
|
20
|
+
* redundant. `store.workers.test.ts` drives the binding against Miniflare with the presign half
|
|
21
|
+
* throwing; this drives the presign half against real R2 with the binding-only read throwing. Neither
|
|
22
|
+
* can quietly take the other's path and pass.
|
|
23
|
+
*/
|
|
24
|
+
/** One live object plane: the store under test, the bucket it addresses, and the S3 manager behind it. */
|
|
25
|
+
export interface LiveObjectPlane {
|
|
26
|
+
/** The throwaway bucket's name. Every key in a test lands here and nowhere else. */
|
|
27
|
+
bucketName: string;
|
|
28
|
+
/** The seam under test — production `objectStore`, with the binding half served over S3. */
|
|
29
|
+
store: ObjectStore;
|
|
30
|
+
/** The S3 manager, for a test that needs an oracle the seam does not expose. */
|
|
31
|
+
manager: CloudflareR2Manager;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Run `exercise` against a fresh throwaway bucket, then empty and delete it.
|
|
35
|
+
*
|
|
36
|
+
* Everything but the bucket is built *before* the create, so the one thing teardown has to undo is the
|
|
37
|
+
* one thing `create` did. One bucket per test keeps a failure's debris out of the next test — and out
|
|
38
|
+
* of the account.
|
|
39
|
+
*/
|
|
40
|
+
export declare function withLiveBucket<T>(creds: IntegrationCreds, exercise: (plane: LiveObjectPlane) => Promise<T>): Promise<T>;
|
|
41
|
+
/**
|
|
42
|
+
* Run `exercise` against a fresh throwaway D1, migrated to storage's schema, then delete it.
|
|
43
|
+
*
|
|
44
|
+
* A real database rather than Miniflare's, for one reason: this pool is Node, and Miniflare's D1 lives
|
|
45
|
+
* in workerd. Standing one up here would mean running the sweep against an emulator on one side and
|
|
46
|
+
* real R2 on the other, which is the mixed picture the live suite exists to replace. It also buys a
|
|
47
|
+
* second proof for free — the migration runs on real D1 over REST, exactly as `pithy migrate` runs it.
|
|
48
|
+
*/
|
|
49
|
+
export declare function withLiveDatabase<T>(creds: IntegrationCreds, exercise: (db: StorageDatabase) => Promise<T>): Promise<T>;
|
|
50
|
+
//# sourceMappingURL=liveStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"liveStorage.d.ts","sourceRoot":"","sources":["../../src/test-utils/liveStorage.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uCAAuC,CAAC;AACjF,OAAO,EAAE,KAAK,gBAAgB,EAAqC,MAAM,6CAA6C,CAAC;AAMvH,OAAO,EAAE,KAAK,eAAe,EAAmB,MAAM,gBAAgB,CAAC;AAEvE,OAAO,EAAiC,KAAK,WAAW,EAAe,MAAM,iBAAiB,CAAC;AAG/F;;;;;;;;;;;;;;;;;;GAkBG;AAEH,0GAA0G;AAC1G,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,4FAA4F;IAC5F,KAAK,EAAE,WAAW,CAAC;IACnB,gFAAgF;IAChF,OAAO,EAAE,mBAAmB,CAAC;CAC9B;AA2ID;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,CAAC,EACpC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC,CAAC,CAAC,GAC/C,OAAO,CAAC,CAAC,CAAC,CAyBZ;AAiBD;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,KAAK,EAAE,gBAAgB,EACvB,QAAQ,EAAE,CAAC,EAAE,EAAE,eAAe,KAAK,OAAO,CAAC,CAAC,CAAC,GAC5C,OAAO,CAAC,CAAC,CAAC,CAaZ"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { storageDatabase } from "../data/tables.js";
|
|
2
|
+
import { STORAGE_R2_SECRET, storageSecretsRegistry } from "../secret/registry.js";
|
|
3
|
+
import { ObjectListing, ObjectMetadata, objectStore } from "../object/store.js";
|
|
4
|
+
import { storage_0001_objects } from "../migrations/0001_objects.js";
|
|
5
|
+
import "../capability.js";
|
|
6
|
+
import { configureSharedSecrets } from "@pithy-sh/secrets/src/sharedSecretsStore";
|
|
7
|
+
import { CloudflareClients } from "@pithy-sh/cloudflare/src/client/clients";
|
|
8
|
+
import { uniqueName, withThrowawayResource } from "@pithy-sh/cloudflare/src/test-utils/harness";
|
|
9
|
+
import { createMigrationRegistry } from "@pithy-sh/core/src/migrations/registry";
|
|
10
|
+
import { runMigrations } from "@pithy-sh/core/src/migrations/runner";
|
|
11
|
+
//#region src/test-utils/liveStorage.ts
|
|
12
|
+
/**
|
|
13
|
+
* The R2 binding, deliberately absent.
|
|
14
|
+
*
|
|
15
|
+
* `objectStore` takes a bucket binding, and no such thing exists outside workerd. Rather than hand it
|
|
16
|
+
* something that pretends, every method throws: a test that reaches the binding by accident fails
|
|
17
|
+
* loudly instead of passing against a stand-in. The methods that matter are re-implemented over S3 in
|
|
18
|
+
* {@link liveObjectStore}, so this is only ever reached by a path that has no live equivalent.
|
|
19
|
+
*/
|
|
20
|
+
function absentBinding() {
|
|
21
|
+
const refuse = (method) => () => {
|
|
22
|
+
throw new Error(`R2 binding '${method}' has no equivalent outside a Worker — use the S3-backed seam.`);
|
|
23
|
+
};
|
|
24
|
+
return {
|
|
25
|
+
get: refuse("get"),
|
|
26
|
+
head: refuse("head"),
|
|
27
|
+
put: refuse("put"),
|
|
28
|
+
delete: refuse("delete"),
|
|
29
|
+
list: refuse("list"),
|
|
30
|
+
createMultipartUpload: refuse("createMultipartUpload"),
|
|
31
|
+
resumeMultipartUpload: refuse("resumeMultipartUpload")
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Build the object store a live test drives.
|
|
36
|
+
*
|
|
37
|
+
* The credential path is the production one end to end: the bundle is injected as a `.dev.vars`-shaped
|
|
38
|
+
* string under the registry name, `objectStore` resolves it through `sharedSecretsStore`, and the
|
|
39
|
+
* registry's Zod schema validates it before a URL is ever signed. So a drift in
|
|
40
|
+
* {@link R2StorageCredentials} fails here the same way it would fail in a deployed Worker.
|
|
41
|
+
*
|
|
42
|
+
* `head`, `list` and `delete` are then re-pointed at the same bucket over S3, because the binding they
|
|
43
|
+
* would otherwise use does not exist in Node. `get` is left throwing: a range or a conditional read
|
|
44
|
+
* through the binding has no S3 mapping this file could write without re-implementing workerd, and a
|
|
45
|
+
* re-implementation is exactly the mock these tests exist to avoid. The suites assert R2's range and
|
|
46
|
+
* conditional *semantics* over a presigned GET instead, where the answer comes from R2 itself.
|
|
47
|
+
*
|
|
48
|
+
* Multipart uploads are recorded as they open and forgotten as they close, so teardown can abort
|
|
49
|
+
* whatever a failure left in flight — stored parts hold a bucket against deletion and cost money.
|
|
50
|
+
*/
|
|
51
|
+
function liveObjectStore(credentials, manager, open) {
|
|
52
|
+
const env = { [STORAGE_R2_SECRET]: JSON.stringify(credentials) };
|
|
53
|
+
configureSharedSecrets({ registry: storageSecretsRegistry });
|
|
54
|
+
const store = objectStore({
|
|
55
|
+
bucket: absentBinding(),
|
|
56
|
+
env
|
|
57
|
+
});
|
|
58
|
+
async function head(key) {
|
|
59
|
+
const object = await manager.headObject(key);
|
|
60
|
+
if (!object) return null;
|
|
61
|
+
if (!object.uploaded) throw new Error(`R2 reported no LastModified for '${key}'`);
|
|
62
|
+
return ObjectMetadata.parse({
|
|
63
|
+
key,
|
|
64
|
+
size: object.size,
|
|
65
|
+
etag: object.etag.replace(/^"|"$/g, ""),
|
|
66
|
+
contentType: object.contentType,
|
|
67
|
+
uploaded: object.uploaded
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
...store,
|
|
72
|
+
async initMultipart(key, contentType) {
|
|
73
|
+
const uploadId = await store.initMultipart(key, contentType);
|
|
74
|
+
open.set(`${key}\0${uploadId}`, {
|
|
75
|
+
key,
|
|
76
|
+
uploadId
|
|
77
|
+
});
|
|
78
|
+
return uploadId;
|
|
79
|
+
},
|
|
80
|
+
async completeMultipart(key, uploadId, parts) {
|
|
81
|
+
await store.completeMultipart(key, uploadId, parts);
|
|
82
|
+
open.delete(`${key}\0${uploadId}`);
|
|
83
|
+
},
|
|
84
|
+
async abortMultipart(key, uploadId) {
|
|
85
|
+
await store.abortMultipart(key, uploadId);
|
|
86
|
+
open.delete(`${key}\0${uploadId}`);
|
|
87
|
+
},
|
|
88
|
+
head,
|
|
89
|
+
async list(options) {
|
|
90
|
+
const page = await manager.listObjects({
|
|
91
|
+
prefix: options?.prefix,
|
|
92
|
+
cursor: options?.cursor,
|
|
93
|
+
maxKeys: options?.limit
|
|
94
|
+
});
|
|
95
|
+
const objects = [];
|
|
96
|
+
for (const key of page.keys) {
|
|
97
|
+
const metadata = await head(key);
|
|
98
|
+
if (metadata) objects.push(metadata);
|
|
99
|
+
}
|
|
100
|
+
return ObjectListing.parse({
|
|
101
|
+
objects,
|
|
102
|
+
cursor: page.cursor
|
|
103
|
+
});
|
|
104
|
+
},
|
|
105
|
+
async delete(key) {
|
|
106
|
+
await manager.deleteObject(key);
|
|
107
|
+
},
|
|
108
|
+
get() {
|
|
109
|
+
throw new Error("`get` reads through the R2 binding, which no Node context has. Presign a GET instead.");
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Empty a bucket so R2 will delete it — the same drain `pithy storage deprovision --storage` runs, so a
|
|
115
|
+
* break in it fails a test here rather than an operator's teardown.
|
|
116
|
+
*
|
|
117
|
+
* Two things hold a bucket: stored objects, and the parts of a multipart upload that was never completed
|
|
118
|
+
* or aborted — precisely what a test that failed mid-upload leaves behind. `emptyBucket` handles both.
|
|
119
|
+
* The uploads this run opened are aborted first and best-effort, because a test may already have aborted
|
|
120
|
+
* one and this map is only local bookkeeping; the authoritative sweep is R2's own listing inside
|
|
121
|
+
* `emptyBucket`, which is also all a reaper cleaning up after a *previous* run has.
|
|
122
|
+
*/
|
|
123
|
+
async function purgeBucket(manager, open) {
|
|
124
|
+
for (const upload of open.values()) await manager.abortMultipartUpload(upload.key, upload.uploadId).catch(() => void 0);
|
|
125
|
+
open.clear();
|
|
126
|
+
await manager.emptyBucket();
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Run `exercise` against a fresh throwaway bucket, then empty and delete it.
|
|
130
|
+
*
|
|
131
|
+
* Everything but the bucket is built *before* the create, so the one thing teardown has to undo is the
|
|
132
|
+
* one thing `create` did. One bucket per test keeps a failure's debris out of the next test — and out
|
|
133
|
+
* of the account.
|
|
134
|
+
*/
|
|
135
|
+
async function withLiveBucket(creds, exercise) {
|
|
136
|
+
const r2 = creds.r2;
|
|
137
|
+
if (!r2) throw new Error("withLiveBucket needs R2_CREDENTIALS — gate the suite on `creds.r2`.");
|
|
138
|
+
const clients = new CloudflareClients({
|
|
139
|
+
accountId: creds.accountId,
|
|
140
|
+
apiToken: creds.apiToken
|
|
141
|
+
});
|
|
142
|
+
const provisioner = clients.r2Provisioner();
|
|
143
|
+
const bucketName = uniqueName("storage");
|
|
144
|
+
const manager = clients.r2({
|
|
145
|
+
...r2,
|
|
146
|
+
bucketName
|
|
147
|
+
});
|
|
148
|
+
const open = /* @__PURE__ */ new Map();
|
|
149
|
+
const store = liveObjectStore({
|
|
150
|
+
...r2,
|
|
151
|
+
accountId: creds.accountId,
|
|
152
|
+
apiToken: creds.apiToken,
|
|
153
|
+
bucket: bucketName
|
|
154
|
+
}, manager, open);
|
|
155
|
+
return withThrowawayResource(() => provisioner.createBucket(bucketName), () => exercise({
|
|
156
|
+
bucketName,
|
|
157
|
+
store,
|
|
158
|
+
manager
|
|
159
|
+
}), async () => {
|
|
160
|
+
await purgeBucket(manager, open).catch(() => void 0);
|
|
161
|
+
await provisioner.deleteBucket(bucketName);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
/** The storage migrations, composed the way `pithy migrate` composes them for the app database. */
|
|
165
|
+
function storageMigrations() {
|
|
166
|
+
const provider = createMigrationRegistry([{
|
|
167
|
+
database: "app",
|
|
168
|
+
namespace: "storage",
|
|
169
|
+
order: 800,
|
|
170
|
+
migrations: { "0001_objects": storage_0001_objects }
|
|
171
|
+
}]).app;
|
|
172
|
+
if (!provider) throw new Error("no app migration provider");
|
|
173
|
+
return provider;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Run `exercise` against a fresh throwaway D1, migrated to storage's schema, then delete it.
|
|
177
|
+
*
|
|
178
|
+
* A real database rather than Miniflare's, for one reason: this pool is Node, and Miniflare's D1 lives
|
|
179
|
+
* in workerd. Standing one up here would mean running the sweep against an emulator on one side and
|
|
180
|
+
* real R2 on the other, which is the mixed picture the live suite exists to replace. It also buys a
|
|
181
|
+
* second proof for free — the migration runs on real D1 over REST, exactly as `pithy migrate` runs it.
|
|
182
|
+
*/
|
|
183
|
+
async function withLiveDatabase(creds, exercise) {
|
|
184
|
+
const clients = new CloudflareClients({
|
|
185
|
+
accountId: creds.accountId,
|
|
186
|
+
apiToken: creds.apiToken
|
|
187
|
+
});
|
|
188
|
+
const provisioner = clients.d1Provisioner();
|
|
189
|
+
return withThrowawayResource(() => provisioner.createDatabase(uniqueName("storage")), async (database) => {
|
|
190
|
+
const d1 = clients.d1(database.uuid);
|
|
191
|
+
await runMigrations(d1, storageMigrations());
|
|
192
|
+
return exercise(storageDatabase(d1));
|
|
193
|
+
}, (database) => provisioner.deleteDatabase(database.uuid));
|
|
194
|
+
}
|
|
195
|
+
//#endregion
|
|
196
|
+
export { withLiveBucket, withLiveDatabase };
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** This package's npm name — the join key against a release feed. */
|
|
2
|
+
export declare const PACKAGE_NAME = "@pithy-sh/storage";
|
|
3
|
+
/** This package's version, stamped from its own package.json at generation time. */
|
|
4
|
+
export declare const PACKAGE_VERSION = "0.1.3";
|
|
5
|
+
//# sourceMappingURL=version.generated.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"version.generated.d.ts","sourceRoot":"","sources":["../src/version.generated.ts"],"names":[],"mappings":"AAWA,qEAAqE;AACrE,eAAO,MAAM,YAAY,sBAAsB,CAAC;AAEhD,oFAAoF;AACpF,eAAO,MAAM,eAAe,UAAU,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
//#region src/version.generated.ts
|
|
2
|
+
/** This package's npm name — the join key against a release feed. */
|
|
3
|
+
const PACKAGE_NAME = "@pithy-sh/storage";
|
|
4
|
+
/** This package's version, stamped from its own package.json at generation time. */
|
|
5
|
+
const PACKAGE_VERSION = "0.1.3";
|
|
6
|
+
//#endregion
|
|
7
|
+
export { PACKAGE_NAME, PACKAGE_VERSION };
|