@pithy-sh/storage 0.1.0

package/src/seeds/example.ts

@@ -0,0 +1,120 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { EXAMPLE_ADA, EXAMPLE_GRACE } from "@pithy-sh/core/src/seed/exampleIdentities";
+import { d1SeedGroup, defineSeed, type R2SeedItem, type SeedSet } from "@pithy-sh/core/src/seed/seed";
+import { StorageObject } from "../data/storageObject";
+import { STORAGE_OBJECTS_TABLE } from "../data/tables";
+import { OBJECT_KEY_PREFIX } from "../object/key";
+
+/**
+ * Where the example set sorts among the whole project's seed registry. It runs after `auth` (100),
+ * whose example seeds the users these files belong to, so the owning identities exist first — the
+ * order encodes that dependency, exactly like the migration registry. It need not line up with
+ * {@link STORAGE_MIGRATION_ORDER}: a different registry, composed separately by `pithy seed`.
+ */
+const STORAGE_EXAMPLE_SEED_ORDER = 230;
+
+/**
+ * A handful of demo files for the canonical example cast ({@link EXAMPLE_ADA} et al.).
+ *
+ * **Both halves are seeded — the rows and the bytes.** A storage row with no object in the bucket is
+ * precisely the divergence the orphan sweep exists to reconcile, so a fixture that seeded only D1
+ * would ship a demo where every download 404s and the sweep has work to do on a fresh install. The
+ * `r2` items below write the same keys the rows name, so `GET /storage/:id` returns real content on
+ * a freshly seeded environment.
+ *
+ * The ids and keys are **fixed**, not generated, which is what makes the set idempotent: re-seeding
+ * writes the same rows over the same keys rather than accumulating a new copy of Ada's notes on every
+ * run. One file is `public` on purpose, so the unauthenticated read path is exercised by the demo
+ * data rather than only by the tests.
+ *
+ * Composed in only when the project turns on `seed.includeExamples` (`pithy.config.ts`), and only for
+ * `dev` and `staging` — an example fixture never targets `prod`, regardless of that setting.
+ */
+
+/** Fixed ids, so the fixture is idempotent and the D1 rows and R2 objects cannot drift apart. */
+const NOTES_ID = "6f1c8f36-30a5-4d5f-9f4d-6a5a52b9f0a1";
+const PLAN_ID = "8c2d9a47-41b6-4e60-a05e-7b6b63c0e1b2";
+const README_ID = "9d3ea058-52c7-4f71-b16f-8c7c74d1f2c3";
+
+const CREATED_AT = new Date("2026-01-01T00:00:00.000Z");
+
+/** The demo bodies. Declared once so a row's `size` is the byte count actually written to R2. */
+const BODIES = {
+  [NOTES_ID]: "Lovelace, notes on the Analytical Engine.\n",
+  [PLAN_ID]: "Hopper, plan for the next compiler.\n",
+  [README_ID]: "A public file. Anyone with the id may read this one.\n",
+} as const;
+
+/** Bytes of a demo body — `size` must be the encoded length, not the character count. */
+function byteLength(id: keyof typeof BODIES): number {
+  return new TextEncoder().encode(BODIES[id]).length;
+}
+
+/** The R2 object key a demo file lives under — the same opaque `obj/<uuid>` shape the handlers derive. */
+function demoKey(id: string): string {
+  return `${OBJECT_KEY_PREFIX}${id}`;
+}
+
+/** The bucket objects the rows point at, so a seeded environment can actually serve the bytes. */
+const objects: R2SeedItem[] = (Object.keys(BODIES) as Array<keyof typeof BODIES>).map((id) => ({
+  binding: "STORAGE_BUCKET",
+  key: demoKey(id),
+  body: BODIES[id],
+  contentType: "text/plain",
+}));
+
+export const storageExampleSeed: SeedSet = defineSeed({
+  name: "example",
+  order: STORAGE_EXAMPLE_SEED_ORDER,
+  environments: ["dev", "staging"],
+  example: true,
+  d1: [
+    d1SeedGroup("app", STORAGE_OBJECTS_TABLE, StorageObject, [
+      {
+        id: NOTES_ID,
+        key: demoKey(NOTES_ID),
+        path: "notes/analytical-engine.txt",
+        ownerId: EXAMPLE_ADA.id,
+        contentType: "text/plain",
+        size: byteLength(NOTES_ID),
+        visibility: "private",
+        checksum: null,
+        status: "stored",
+        uploadId: null,
+        createdAt: CREATED_AT,
+        updatedAt: CREATED_AT,
+      },
+      {
+        id: PLAN_ID,
+        key: demoKey(PLAN_ID),
+        path: "plans/compiler.txt",
+        ownerId: EXAMPLE_GRACE.id,
+        contentType: "text/plain",
+        size: byteLength(PLAN_ID),
+        visibility: "private",
+        checksum: null,
+        status: "stored",
+        uploadId: null,
+        createdAt: CREATED_AT,
+        updatedAt: CREATED_AT,
+      },
+      {
+        id: README_ID,
+        key: demoKey(README_ID),
+        path: "public/readme.txt",
+        ownerId: EXAMPLE_ADA.id,
+        contentType: "text/plain",
+        size: byteLength(README_ID),
+        visibility: "public",
+        checksum: null,
+        status: "stored",
+        uploadId: null,
+        createdAt: CREATED_AT,
+        updatedAt: CREATED_AT,
+      },
+    ]),
+  ],
+  r2: objects,
+});

package/src/test-utils/liveStorage.ts

@@ -0,0 +1,261 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import type { D1Database, R2Bucket } from "@cloudflare/workers-types";
+import { CloudflareClients } from "@pithy-sh/cloudflare/src/client/clients";
+import type { CloudflareR2Manager } from "@pithy-sh/cloudflare/src/r2/r2Manager";
+import { type IntegrationCreds, uniqueName, withThrowawayResource } from "@pithy-sh/cloudflare/src/test-utils/harness";
+import { createMigrationRegistry } from "@pithy-sh/core/src/migrations/registry";
+import { runMigrations } from "@pithy-sh/core/src/migrations/runner";
+import type { SecretsStoreEnv } from "@pithy-sh/secrets/src/env/bindings";
+import { configureSharedSecrets } from "@pithy-sh/secrets/src/sharedSecretsStore";
+import { STORAGE_MIGRATION_ORDER } from "../capability";
+import { type StorageDatabase, storageDatabase } from "../data/tables";
+import { storage_0001_objects } from "../migrations/0001_objects";
+import { ObjectListing, ObjectMetadata, type ObjectStore, objectStore } from "../object/store";
+import { type R2StorageCredentials, STORAGE_R2_SECRET, storageSecretsRegistry } from "../secret/registry";
+
+/**
+ * Scaffolding for this package's `*.integration.test.ts` suites — the live half of storage.
+ *
+ * `@pithy-sh/cloudflare`'s harness finds credentials and guarantees teardown. Storage needs two more
+ * things on top: the seam under test built the way a Worker builds it (an {@link objectStore} over a
+ * *resolved credential bundle*, not an injected fake), and a real database to reconcile against.
+ *
+ * **What is live here, and what cannot be.** The presigned half — presign, the whole multipart
+ * lifecycle, server-side copy — is real R2 over the S3 protocol, and it is the half Miniflare cannot
+ * emulate at all: it serves no S3 endpoint, so a presigned URL has nothing to address. The binding
+ * half is the mirror image. An `R2Bucket` exists only inside workerd, so outside a Worker there is no
+ * binding to call — {@link liveObjectStore} serves `head`, `list` and `delete` from the *same real
+ * bucket* over S3 instead, and leaves `get` throwing.
+ *
+ * That split is deliberate and it is the whole reason the two suites are complementary rather than
+ * redundant. `store.workers.test.ts` drives the binding against Miniflare with the presign half
+ * throwing; this drives the presign half against real R2 with the binding-only read throwing. Neither
+ * can quietly take the other's path and pass.
+ */
+
+/** One live object plane: the store under test, the bucket it addresses, and the S3 manager behind it. */
+export interface LiveObjectPlane {
+  /** The throwaway bucket's name. Every key in a test lands here and nowhere else. */
+  bucketName: string;
+  /** The seam under test — production `objectStore`, with the binding half served over S3. */
+  store: ObjectStore;
+  /** The S3 manager, for a test that needs an oracle the seam does not expose. */
+  manager: CloudflareR2Manager;
+}
+
+/** One multipart upload this run opened and has not yet closed — teardown's list of what to abort. */
+interface OpenUpload {
+  key: string;
+  uploadId: string;
+}
+
+/**
+ * The R2 binding, deliberately absent.
+ *
+ * `objectStore` takes a bucket binding, and no such thing exists outside workerd. Rather than hand it
+ * something that pretends, every method throws: a test that reaches the binding by accident fails
+ * loudly instead of passing against a stand-in. The methods that matter are re-implemented over S3 in
+ * {@link liveObjectStore}, so this is only ever reached by a path that has no live equivalent.
+ */
+function absentBinding(): R2Bucket {
+  const refuse = (method: string) => () => {
+    throw new Error(`R2 binding '${method}' has no equivalent outside a Worker — use the S3-backed seam.`);
+  };
+  return {
+    get: refuse("get"),
+    head: refuse("head"),
+    put: refuse("put"),
+    delete: refuse("delete"),
+    list: refuse("list"),
+    createMultipartUpload: refuse("createMultipartUpload"),
+    resumeMultipartUpload: refuse("resumeMultipartUpload"),
+  } as unknown as R2Bucket;
+}
+
+/**
+ * Build the object store a live test drives.
+ *
+ * The credential path is the production one end to end: the bundle is injected as a `.dev.vars`-shaped
+ * string under the registry name, `objectStore` resolves it through `sharedSecretsStore`, and the
+ * registry's Zod schema validates it before a URL is ever signed. So a drift in
+ * {@link R2StorageCredentials} fails here the same way it would fail in a deployed Worker.
+ *
+ * `head`, `list` and `delete` are then re-pointed at the same bucket over S3, because the binding they
+ * would otherwise use does not exist in Node. `get` is left throwing: a range or a conditional read
+ * through the binding has no S3 mapping this file could write without re-implementing workerd, and a
+ * re-implementation is exactly the mock these tests exist to avoid. The suites assert R2's range and
+ * conditional *semantics* over a presigned GET instead, where the answer comes from R2 itself.
+ *
+ * Multipart uploads are recorded as they open and forgotten as they close, so teardown can abort
+ * whatever a failure left in flight — stored parts hold a bucket against deletion and cost money.
+ */
+function liveObjectStore(
+  credentials: R2StorageCredentials,
+  manager: CloudflareR2Manager,
+  open: Map<string, OpenUpload>,
+): ObjectStore {
+  // Local dev's own resolution path: no `ENVIRONMENT` var, so every secret comes from its injected
+  // string in the shape it is stored. Configuring the shared accessor also clears its TTL cache, which
+  // is what keeps one test's bucket credentials from leaking into the next test's store.
+  const env = { [STORAGE_R2_SECRET]: JSON.stringify(credentials) } as unknown as SecretsStoreEnv;
+  configureSharedSecrets({ registry: storageSecretsRegistry });
+  const store = objectStore({ bucket: absentBinding(), env });
+
+  async function head(key: string): Promise<ObjectMetadata | null> {
+    const object = await manager.headObject(key);
+    if (!object) return null;
+    if (!object.uploaded) throw new Error(`R2 reported no LastModified for '${key}'`);
+    return ObjectMetadata.parse({
+      key,
+      size: object.size,
+      // R2 quotes an ETag over the S3 protocol and leaves it unquoted through the binding. Strip it so
+      // the seam's shape is identical either way — no caller should be able to tell which half served it.
+      etag: object.etag.replace(/^"|"$/g, ""),
+      contentType: object.contentType,
+      uploaded: object.uploaded,
+    });
+  }
+
+  return {
+    ...store,
+
+    async initMultipart(key, contentType) {
+      const uploadId = await store.initMultipart(key, contentType);
+      open.set(`${key}\0${uploadId}`, { key, uploadId });
+      return uploadId;
+    },
+
+    async completeMultipart(key, uploadId, parts) {
+      await store.completeMultipart(key, uploadId, parts);
+      open.delete(`${key}\0${uploadId}`);
+    },
+
+    async abortMultipart(key, uploadId) {
+      await store.abortMultipart(key, uploadId);
+      open.delete(`${key}\0${uploadId}`);
+    },
+
+    head,
+
+    async list(options) {
+      const page = await manager.listObjects({
+        prefix: options?.prefix,
+        cursor: options?.cursor,
+        maxKeys: options?.limit,
+      });
+      const objects: ObjectMetadata[] = [];
+      for (const key of page.keys) {
+        // A key listed and then deleted before its HEAD is not an error — a bucket is not a snapshot.
+        const metadata = await head(key);
+        if (metadata) objects.push(metadata);
+      }
+      return ObjectListing.parse({ objects, cursor: page.cursor });
+    },
+
+    async delete(key) {
+      await manager.deleteObject(key);
+    },
+
+    get() {
+      throw new Error("`get` reads through the R2 binding, which no Node context has. Presign a GET instead.");
+    },
+  };
+}
+
+/**
+ * Empty a bucket so R2 will delete it — the same drain `pithy storage deprovision --storage` runs, so a
+ * break in it fails a test here rather than an operator's teardown.
+ *
+ * Two things hold a bucket: stored objects, and the parts of a multipart upload that was never completed
+ * or aborted — precisely what a test that failed mid-upload leaves behind. `emptyBucket` handles both.
+ * The uploads this run opened are aborted first and best-effort, because a test may already have aborted
+ * one and this map is only local bookkeeping; the authoritative sweep is R2's own listing inside
+ * `emptyBucket`, which is also all a reaper cleaning up after a *previous* run has.
+ */
+async function purgeBucket(manager: CloudflareR2Manager, open: Map<string, OpenUpload>): Promise<void> {
+  for (const upload of open.values()) {
+    await manager.abortMultipartUpload(upload.key, upload.uploadId).catch(() => undefined);
+  }
+  open.clear();
+  await manager.emptyBucket();
+}
+
+/**
+ * Run `exercise` against a fresh throwaway bucket, then empty and delete it.
+ *
+ * Everything but the bucket is built *before* the create, so the one thing teardown has to undo is the
+ * one thing `create` did. One bucket per test keeps a failure's debris out of the next test — and out
+ * of the account.
+ */
+export async function withLiveBucket<T>(
+  creds: IntegrationCreds,
+  exercise: (plane: LiveObjectPlane) => Promise<T>,
+): Promise<T> {
+  const r2 = creds.r2;
+  if (!r2) throw new Error("withLiveBucket needs R2_CREDENTIALS — gate the suite on `creds.r2`.");
+
+  const clients = new CloudflareClients({ accountId: creds.accountId, apiToken: creds.apiToken });
+  const provisioner = clients.r2Provisioner();
+  const bucketName = uniqueName("storage");
+  const manager = clients.r2({ ...r2, bucketName });
+  const open = new Map<string, OpenUpload>();
+  const store = liveObjectStore(
+    { ...r2, accountId: creds.accountId, apiToken: creds.apiToken, bucket: bucketName },
+    manager,
+    open,
+  );
+
+  return withThrowawayResource(
+    () => provisioner.createBucket(bucketName),
+    () => exercise({ bucketName, store, manager }),
+    async () => {
+      // A purge failure would only resurface as a bucket-delete failure; let that be the loud one, so a
+      // genuine assertion error is never masked by teardown noise.
+      await purgeBucket(manager, open).catch(() => undefined);
+      await provisioner.deleteBucket(bucketName);
+    },
+  );
+}
+
+/** The storage migrations, composed the way `pithy migrate` composes them for the app database. */
+function storageMigrations() {
+  const registry = createMigrationRegistry([
+    {
+      database: "app",
+      namespace: "storage",
+      order: STORAGE_MIGRATION_ORDER,
+      migrations: { "0001_objects": storage_0001_objects },
+    },
+  ]);
+  const provider = registry.app;
+  if (!provider) throw new Error("no app migration provider");
+  return provider;
+}
+
+/**
+ * Run `exercise` against a fresh throwaway D1, migrated to storage's schema, then delete it.
+ *
+ * A real database rather than Miniflare's, for one reason: this pool is Node, and Miniflare's D1 lives
+ * in workerd. Standing one up here would mean running the sweep against an emulator on one side and
+ * real R2 on the other, which is the mixed picture the live suite exists to replace. It also buys a
+ * second proof for free — the migration runs on real D1 over REST, exactly as `pithy migrate` runs it.
+ */
+export async function withLiveDatabase<T>(
+  creds: IntegrationCreds,
+  exercise: (db: StorageDatabase) => Promise<T>,
+): Promise<T> {
+  const clients = new CloudflareClients({ accountId: creds.accountId, apiToken: creds.apiToken });
+  const provisioner = clients.d1Provisioner();
+
+  return withThrowawayResource(
+    () => provisioner.createDatabase(uniqueName("storage")),
+    async (database) => {
+      const d1 = clients.d1(database.uuid) as unknown as D1Database;
+      await runMigrations(d1, storageMigrations());
+      return exercise(storageDatabase(d1));
+    },
+    (database) => provisioner.deleteDatabase(database.uuid),
+  );
+}

package/src/version.generated.ts

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+// GENERATED by scripts/stampVersions.ts — do not edit by hand. Regenerate with `bun run stamp-versions`.
+//
+// A Worker cannot read its own package.json, so this is how @pithy-sh/storage knows its own version at
+// runtime. The capability attaches it, and `GET /control-plane/manifest` reports it per capability —
+// which is what answers "should this project upgrade" and "is this customer exposed to what we just
+// fixed". Those questions are only answerable per module, because a project composes some capabilities
+// and not others.
+
+/** This package's npm name — the join key against a release feed. */
+export const PACKAGE_NAME = "@pithy-sh/storage";
+
+/** This package's version, stamped from its own package.json at generation time. */
+export const PACKAGE_VERSION = "0.1.0";

package/src/workflows/retryPolicy.ts

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import type { WorkflowRetryPolicy } from "@pithy-sh/core/src/workflow/faults";
+
+/**
+ * **What the orphan sweep retries, and what it refuses to.**
+ *
+ * The sweep reconciles D1 against R2, and it has already decided what a failure means: every delete and
+ * every multipart abort is `.catch(() => {})` on purpose, because an object this run could not remove is
+ * an object the next run finds again. That is a reconciler's contract — not error swallowing — and it is
+ * what leaves this record empty (pithy-sh/pithy#348).
+ *
+ * **An empty record is a statement, not an omission.** Core answers for D1's transient vocabulary
+ * through `withD1Retry`, so a database under contention is still re-driven by the step, and nothing
+ * about that is restated here. What the empty record adds is that storage retries none of its *own*
+ * codes.
+ *
+ * ## Terminal, and why
+ *
+ * - **`storage/*`** — every one of them is a *request's* refusal: a file that is not there, a file that
+ *   is not yours, a quota, an upload that never finished, an expired share. The sweep raises none of
+ *   them, and a sweep that somehow did has found a bug rather than an outage.
+ * - **`validation/invalid_input`** — a `StorageSweepParams` an operator dispatched by hand. A payload
+ *   parses the same way on the fifth attempt as on the first.
+ * - **`secrets/*`** — the R2 credentials the multipart abort needs. A key that is not in the bound key
+ *   set will not be in it a minute later; this wants `pithy secrets provision`, not a backoff.
+ *
+ * **The cron is the outer retry, and that is why terminal is cheap here.** The sweep is idempotent by
+ * construction — a second pass over a reconciled bucket finds nothing left to do — so a run that fails
+ * costs one day, and the next day's run does the whole job. Five platform attempts against a refusal
+ * that cannot change cost the same day *and* bury the reason under four repeats of it.
+ *
+ * **What this cannot say, and it is worth knowing.** A bucket listing that fails mid-sweep throws
+ * whatever the R2 binding threw, which is not a `PithyError` and carries no code — so it is
+ * `unclassified`, which is terminal. That is the right default and the correct outcome here (the next
+ * fire re-lists from the top), but it is a default rather than a decision, and no policy record can
+ * turn it into one.
+ */
+export const storageWorkflowRetry: WorkflowRetryPolicy = {
+  capability: "storage",
+  retryable: {},
+};

package/src/workflows/specs.ts

@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: 2026 Pithy
+// SPDX-License-Identifier: MIT
+
+import { workflowKey } from "@pithy-sh/core/src/workflow/naming";
+import type { WorkflowRegistry, WorkflowSpecMap } from "@pithy-sh/core/src/workflow/spec";
+import { z } from "zod";
+
+/**
+ * The one durable job storage owns: the orphan sweep.
+ *
+ * Declared once, here. `capability.ts` derives its Workflow binding from this map, `provision/`
+ * derives the host worker's `workflows` array and its cron from it, and any caller triggers it by the
+ * `storage/sweep` key. There is no second place where the binding name, the class name, or the
+ * schedule is written down, so none of them can drift.
+ *
+ * `optional: true` because the Workflow lives in the prebuilt sweep worker, which exists only once
+ * `pithy storage provision` has run. Until then an app composing storage must still boot and serve
+ * every upload and download route; an absent binding degrades to a logged skip.
+ */
+
+/** The capability name — the first segment of the dispatch key and of every deployed resource name. */
+export const STORAGE_CAPABILITY = "storage";
+
+/**
+ * The sweep's parameters. Every field is optional, which is a requirement rather than a convenience:
+ * a cron supplies no input, and `createEntrypoint` dispatches a scheduled job with `{}`. A schema
+ * that demanded a field could never run on its schedule.
+ */
+export const StorageSweepParams = z
+  .object({
+    olderThanSeconds: z
+      .number()
+      .int()
+      .positive()
+      .optional()
+      .describe(
+        "Override how old a `pending` row must be before it is reclaimed. Omitted uses the config's `pendingTtlSeconds`. Lower it to reproduce a reclaim in staging without waiting a day.",
+      ),
+    dryRun: z
+      .boolean()
+      .optional()
+      .describe(
+        "Report what would be reclaimed and delete nothing. The safe way to answer 'what is this sweep about to do to prod' before letting it run.",
+      ),
+    maxPages: z
+      .number()
+      .int()
+      .positive()
+      .optional()
+      .describe(
+        "Cap how many 1,000-key bucket pages one run scans. A bound on the work, so a first sweep over a very large bucket finishes rather than running until it is killed.",
+      ),
+  })
+  .describe("What one orphan-sweep run should do. Every field optional, because a cron passes none of them.");
+export type StorageSweepParams = z.infer<typeof StorageSweepParams>;
+
+/**
+ * Storage's durable jobs, keyed by job name.
+ *
+ * The schedule is daily at 03:00 UTC. Divergence between the bucket and the table accumulates slowly
+ * — an abandoned upload here, an interrupted delete there — so an hourly pass would list the whole
+ * bucket twenty-four times to find nothing. It is a cron *and* a dispatch target: a sweep nobody can
+ * run on demand cannot be tested in staging, which is precisely when you want to know what it does.
+ */
+export const storageWorkflows = {
+  sweep: {
+    binding: "STORAGE_SWEEP",
+    className: "StorageSweepWorkflow",
+    params: StorageSweepParams,
+    schedule: "0 3 * * *",
+    optional: true,
+  },
+} as const satisfies WorkflowSpecMap;
+
+/**
+ * Storage's jobs as a dispatch registry, keyed `storage/<job>`. Built here rather than through
+ * `composeWorkflows` because the sweep worker dispatches its own job before any project-wide registry
+ * exists — and the key format comes from core's {@link workflowKey} either way, so the two cannot drift.
+ */
+export const storageWorkflowRegistry: WorkflowRegistry = Object.fromEntries(
+  Object.entries(storageWorkflows).map(([job, spec]) => {
+    const key = workflowKey(STORAGE_CAPABILITY, job);
+    return [key, { key, capability: STORAGE_CAPABILITY, job, spec }];
+  }),
+);