@everystack/server 0.4.19 → 0.4.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/server",
-  "version": "0.4.19",
+  "version": "0.4.21",
   "description": "Server runtime primitives for Lambda — event adapters, routing, SSR, image processing",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -163,8 +163,8 @@
     "sst": "4.13.1",
     "ts-jest": "29.4.9",
     "typescript": "5.9.3",
-    "@everystack/auth": "0.4.2",
-    "@everystack/cli": "0.4.49"
+    "@everystack/auth": "0.4.3",
+    "@everystack/cli": "0.4.60"
   },
   "scripts": {
     "test": "jest",

package/src/backup-run.ts

@@ -99,6 +99,42 @@ export interface BackupTarget {
  * on a successful upload alone. The id+key are minted by the caller (the dispatcher) so the CLI
  * knows the artifact at dispatch time; this IO never stamps its own clock into the id.
  */
+/**
+ * Build a backup's `.meta.json` — the manifest the destructive-apply gate reads.
+ *
+ * Extracted as a pure choke point so the `metaVersion` stamp is a testable property rather than a
+ * line buried inside a function that spawns pg_dump and talks to S3. Deleting the stamp inside
+ * `runBackup` left all 687 server tests green while silently making every future backup unusable
+ * to the gate; with one choke point, presence IS the property.
+ *
+ * `metaVersion: 2` means the fingerprint was computed by the OPERATOR and passed in, not computed
+ * here. The gate refuses anything older, because a task-computed fingerprint came from a copy of
+ * the CLI frozen into a digest-pinned image. It is stamped ONLY when a fingerprint actually rode in
+ * — a manifest with no fingerprint must not claim v2 provenance for a value it does not have.
+ */
+export function buildBackupMeta(input: {
+  id: string;
+  stage: string;
+  key: string;
+  createdAt: string;
+  bytes?: number;
+  fingerprint?: string;
+  /** Who computed it. Defaults to 'operator' — the only producer today. */
+  fingerprintSource?: string;
+  // Returns the CONSUMER's type, not Record<string, unknown>: that is what makes producer/consumer
+  // drift a compile error instead of a runtime surprise the gate discovers.
+}): BackupManifest {
+  const { id, stage, key, createdAt, bytes, fingerprint, fingerprintSource } = input;
+  return {
+    id, stage, createdAt, bytes, key,
+    // `fingerprintSource` records HOW the value was obtained, not just that it exists. The gate
+    // accepts an operator-computed fingerprint by design, so that decision should be auditable
+    // after the fact rather than reconstructed from version archaeology — and a future gate can
+    // tighten on it without guessing.
+    ...(fingerprint ? { fingerprint, metaVersion: 2, fingerprintSource: fingerprintSource ?? 'operator' } : {}),
+  };
+}
+
 export async function runBackup(deps: BackupRunDeps, target: BackupTarget): Promise<BackupSummary> {
   const { bucket, region, adminUrl } = deps;
   const { stage, id, key, fingerprint } = target;
@@ -153,7 +189,7 @@ export async function runBackup(deps: BackupRunDeps, target: BackupTarget): Prom
     bytes = head.ContentLength;
   } catch { /* size is best-effort */ }
 
-  const meta = { id, stage, createdAt: new Date().toISOString(), bytes, key, ...(fingerprint ? { fingerprint } : {}) };
+  const meta = buildBackupMeta({ id, stage, key, bytes, fingerprint, createdAt: new Date().toISOString() });
   await s3.send(new PutObjectCommand({
     Bucket: bucket, Key: mkey, Body: JSON.stringify(meta),
     ContentType: 'application/json', ServerSideEncryption: 'AES256',
@@ -262,11 +298,49 @@ export async function latestBackupManifest(deps: BackupRunDeps, stage: string):
  * PROVEN stable across the dump window (a concurrent DDL), so the safety gate never trusts a manifest
  * whose fp might not match the schema the dump actually holds.
  */
-export async function deleteBackup(deps: Pick<BackupRunDeps, 'bucket' | 'region'>, key: string): Promise<void> {
-  const s3 = await getS3(deps.region);
-  const { DeleteObjectCommand } = await import('@aws-sdk/client-s3');
-  await s3.send(new DeleteObjectCommand({ Bucket: deps.bucket, Key: key })).catch(() => {});
-  await s3.send(new DeleteObjectCommand({ Bucket: deps.bucket, Key: metaKey(key) })).catch(() => {});
+export async function deleteBackup(
+  deps: Pick<BackupRunDeps, 'bucket' | 'region'> & {
+    /** Injected S3 client — the seam that lets the retraction's failure paths be tested. */
+    s3?: { send: (command: unknown) => Promise<unknown> };
+  },
+  key: string,
+): Promise<void> {
+  const s3 = deps.s3 ?? (await getS3(deps.region));
+  const { DeleteObjectCommand, HeadObjectCommand } = await import('@aws-sdk/client-s3');
+
+  // S3 DELETE is idempotent and answers 204 for a key that was never there, so a successful call is
+  // not proof the object is gone. Confirm with a HEAD — and discriminate the error: only a NOT-FOUND
+  // means gone. Treating every failure as success (throttling, AccessDenied, an expired credential,
+  // a socket timeout) makes the confirmation vacuous exactly when it matters.
+  const gone = async (k: string): Promise<boolean> => {
+    try {
+      await s3.send(new HeadObjectCommand({ Bucket: deps.bucket, Key: k }));
+      return false;
+    } catch (err: any) {
+      const status = err?.$metadata?.httpStatusCode;
+      const name = err?.name ?? err?.Code;
+      if (status === 404 || name === 'NotFound' || name === 'NoSuchKey') return true;
+      throw new Error(`could not confirm ${k} was deleted (${name ?? 'unknown error'}): ${err?.message ?? String(err)}`);
+    }
+  };
+
+  // MANIFEST FIRST, and confirm it before touching the dump. The two orientations are not
+  // symmetric: a surviving manifest whose dump is gone reads as a valid backup to
+  // `fetchBackupManifest`, and nothing downstream checks the dump exists — so a destructive apply
+  // would proceed over a "snapshot" that cannot be restored. A surviving dump whose manifest is
+  // gone simply fails to resolve, which refuses. Delete toward the safe orientation.
+  //
+  // Do NOT swallow either call. This used to `.catch(() => {})` both, so a retraction that failed
+  // was reported as a successful one while the unproven artifact stayed in the bucket.
+  const mkey = metaKey(key);
+  await s3.send(new DeleteObjectCommand({ Bucket: deps.bucket, Key: mkey }));
+  if (!(await gone(mkey))) {
+    throw new Error(`the manifest ${mkey} still exists after DELETE — it may be under an object lock or a retention policy. Remove it by hand; the safety gate must never resolve it.`);
+  }
+  await s3.send(new DeleteObjectCommand({ Bucket: deps.bucket, Key: key }));
+  if (!(await gone(key))) {
+    throw new Error(`the manifest is gone but the dump ${key} still exists after DELETE — the backup can no longer be resolved by the gate, but remove the dump by hand to reclaim the storage.`);
+  }
 }
 
 /** List a stage's backups (newest first) from S3 object metadata — one list call, no per-item gets. */

package/src/backup.ts

@@ -146,6 +146,23 @@ export interface BackupManifest {
   createdAt: string;
   /** The base-schema fingerprint captured at dump time (absent on pre-B3 backups). */
   fingerprint?: string;
+  /**
+   * Who computed `fingerprint`.
+   *
+   * Absent or 1: the TASK IMAGE computed it, from a copy of the CLI frozen into a digest-pinned
+   * image at publish time. That copy could disagree with the CLI now doing the comparing, so such a
+   * fingerprint cannot be trusted as the safety gate's tie.
+   *
+   * 2: the OPERATOR computed it, with the same code `db:apply` uses, and the CLI verified the task
+   * stamped exactly what it sent.
+   */
+  metaVersion?: number;
+  /**
+   * HOW the fingerprint was obtained ('operator' today). Recorded so accepting an
+   * operator-computed value is an auditable decision rather than version archaeology, and so a
+   * future gate can tighten on provenance without a cast.
+   */
+  fingerprintSource?: string;
   bytes?: number;
   key: string;
 }
@@ -182,14 +199,28 @@ export function verifySnapshotRef(input: SnapshotVerifyInput): SnapshotVerdict {
   if (!manifest.fingerprint) {
     return { ok: false, reason: `snapshot ${manifest.id} predates fingerprint stamping — ${takeFresh}` };
   }
-  if (manifest.fingerprint !== planFrom) {
-    return { ok: false, reason: `snapshot ${manifest.id} captured schema ${manifest.fingerprint.slice(0, 12)}, but this plan starts from ${planFrom.slice(0, 12)} — it would not restore the state being changed. ${takeFresh}` };
-  }
+  // Freshness first among the remaining checks: it is independent of everything below and is by far
+  // the most common reason a snapshot is rejected, so it should be what the operator reads.
   const ageMs = now.getTime() - Date.parse(manifest.createdAt);
   if (!(ageMs >= 0) || ageMs > freshnessWindowMs) {
     const mins = Math.round(ageMs / 60_000);
     return { ok: false, reason: `snapshot ${manifest.id} is ${mins} min old (freshness window ${Math.round(freshnessWindowMs / 60_000)} min) — ${takeFresh}` };
   }
+  // Provenance BEFORE the tie: a pre-v2 manifest's fingerprint was computed INSIDE the task image,
+  // by a copy of the CLI frozen at publish time, so comparing it against this plan's fingerprint
+  // compares two things that were not necessarily produced the same way. Checking the tie first
+  // would report a mismatch as if the schema differed, sending the operator after the wrong thing.
+  // With an hour-long freshness window this only ever refuses a backup taken in the last hour by a
+  // pre-upgrade CLI — and the remedy is the backup they were going to take anyway.
+  if (!(manifest.metaVersion && manifest.metaVersion >= 2)) {
+    return {
+      ok: false,
+      reason: `snapshot ${manifest.id} carries a fingerprint computed inside the task image, which can disagree with this CLI — ${takeFresh}`,
+    };
+  }
+  if (manifest.fingerprint !== planFrom) {
+    return { ok: false, reason: `snapshot ${manifest.id} captured schema ${manifest.fingerprint.slice(0, 12)}, but this plan starts from ${planFrom.slice(0, 12)} — it would not restore the state being changed. ${takeFresh}` };
+  }
   return { ok: true, id: manifest.id };
 }
 

package/src/db.ts

@@ -273,12 +273,34 @@ let adminSqlClient: ReturnType<typeof postgres> | null = null;
 let adminFallbackWarned = false;
 
 /**
- * Operator database connection for privileged tasks (migrate, seed, console).
+ * Operator database connection — THE OPS CHANNEL ONLY.
  *
- * Connects with ADMIN_DATABASE_URL when configured. When it isn't, falls back
- * to the regular DATABASE_URL connection with a one-time warning — existing
- * single-credential apps keep working, but RLS is only fail-closed once the
- * credentials are split. See docs/plans/admin-database-url.md.
+ * migrate · seed · console · backup. Nothing else.
+ *
+ * **Never call this from request-serving code.** This credential owns the schema, so RLS
+ * does not apply to it. A request handler holding it has unrestricted database access and
+ * every RLS policy you wrote is decoration. It will work, which is exactly the problem —
+ * the failure is silent and total.
+ *
+ * Trusted app writes (issuing a credential at signup, recording a webhook, a background
+ * reconcile) do NOT belong here. The lawful pattern is a NOLOGIN service role granted to
+ * `authenticator`, entered via `withRole` after the handler's own authorization check:
+ *
+ *   if (!(await mayIssueCredential(user))) throw new HttpError(403);
+ *   await withRole(sql, 'service', claims, (tx) => tx`INSERT INTO credentials ...`);
+ *
+ * That keeps the trusted lane on the app plane, scoped to the privileges the lane actually
+ * needs. See the "Two Credential Channels" section of everystack://security.
+ *
+ * Connects with ADMIN_DATABASE_URL when configured. When it isn't, falls back to the
+ * Postgres master, then to the API connection. **That fallback is a migration aid, not a
+ * supported posture** — RLS is only fail-closed once the credentials are split.
+ * `everystack db:provision --stage <name>` splits a DEPLOYED stage; it requires `--stage`
+ * because the new credentials are never displayed and must land in the secret store. Local
+ * single-credential development has no equivalent yet, so a local run legitimately takes this
+ * fallback — which is why the guard that refuses a privileged app credential must not treat
+ * local dev as a violation. See
+ * plan:the-operator-credential-is-the-ops-channel-only-doctrine-doc.
  */
 export function createAdminDb<T extends Record<string, unknown>>(
   schema: T,
@@ -315,9 +337,14 @@ export function createAdminDb<T extends Record<string, unknown>>(
 }
 
 /**
- * Raw postgres.js client for the operator connection.
- * Must be called after createAdminDb(). Falls back to the API client when
- * the credentials aren't split (same fallback as createAdminDb).
+ * Raw postgres.js client for the operator connection — THE OPS CHANNEL ONLY.
+ *
+ * Same rule as `createAdminDb`: never reachable from a request handler. If you are holding
+ * this in code that serves a request, you have turned RLS off for that path. Use a NOLOGIN
+ * service role via `withRole` instead — see `createAdminDb` above.
+ *
+ * Must be called after createAdminDb(). Falls back to the API client when the credentials
+ * aren't split (same fallback, same caveat: a migration aid, not a supported posture).
  */
 export function getAdminSql(): ReturnType<typeof postgres> {
   if (adminSqlClient) return adminSqlClient;

package/src/image.ts

@@ -1,3 +1,12 @@
+/// <reference path="./heic-decode.d.ts" />
+// `heic-decode` ships no types, so this file needs the ambient shim NEXT TO IT — not merely
+// present in the package. ts-jest compiles per file, and an ambient .d.ts nothing imports
+// enters the program only via `include`/`files`, which this package's own tsconfig supplies
+// and a CONSUMER's does not. So the shim covered `packages/server`'s suite and no one else:
+// the example app type-checked this same source and failed on TS7016. The reference directive
+// binds the declaration to the source that needs it, so every program compiling image.ts gets
+// it. (Inert at runtime — erased with the types.)
+
 /**
  * @everystack/server/image — On-demand image processing Lambda.
  *