@pramen/server 0.0.58 → 0.0.60

package/src/sdk/iso-timestamps.ts

@@ -0,0 +1,146 @@
+// Rewriting timestamps written before `expr.now()` emitted ISO-8601.
+//
+// `expr.now()` used to emit `datetime('now')` — the `CURRENT_TIMESTAMP` space form,
+// `'2026-09-03 21:33:07'`. It now emits `'2026-09-03T21:33:07.222Z'`. Changing a column's
+// DEFAULT is a MODIFIER change, so `migrate()` rebuilds the table — and a rebuild copies
+// existing values through untouched. New rows would get the new shape and old rows would
+// keep the old one, in the same column.
+//
+// That is worse than either format alone, and the failure is narrower and nastier than
+// "the column is inconsistent". Both forms open with the same `YYYY-MM-DD`, so for values
+// on DIFFERENT dates the date prefix decides and ordering is still correct. It is values on
+// the SAME date that break: index 10 is ` ` (0x20) in the space form and `T` (0x54) in ISO,
+// so a same-day space-form value always sorts BEFORE a same-day ISO one whatever the
+// time-of-day — 23:00 in the old shape sorts below 01:00 in the new.
+//
+// Which makes the deploy window itself the blast radius: the rows written just before and
+// just after the change are exactly the same-day pairs that invert. And a `{ lte: $now() }`
+// policy is the sharp version, because `$now()` is ISO: a space-form value dated today
+// always compares as less than it, so a row scheduled for later TODAY reads as already
+// published until midnight. Nothing reports any of it.
+//
+// So the format change comes with a rewrite, and the rewrite is a DATA migration: it is
+// imperative, it must run exactly once, and it must fail closed. Declaring it is the app's
+// job — this builds it.
+
+import { DEFAULT_PARTITION, ISO_NOW_SQL, entitiesInPartition } from "./schema";
+import type { DataMigration, MigrationContext } from "./handlers";
+import type { FieldDef, SchemaDef } from "./schema";
+
+/** The old default's SQL, for recognizing a column that carried it. */
+export const LEGACY_NOW_SQL = "datetime('now')";
+
+/** Columns to rewrite beyond the ones the schema identifies, as `{ table: [column, …] }`.
+ *
+ * The schema can only find columns whose DEFAULT is `expr.now()`. A column written by
+ * HANDLER code in the same space form — `@pramen/cms` stamped `cms_pages.publishedAt` that
+ * way — has no marker on it at all, so the app has to name it. `@pramen/cms` exports its
+ * own set as `CMS_LEGACY_TIMESTAMP_COLUMNS`. */
+export type ExtraTimestampColumns = Readonly<Record<string, readonly string[]>>;
+
+export interface IsoTimestampBackfillOpts {
+  /** Ledger id. Defaults to `pramen:iso-timestamps`. Override only if you have already
+   * used that id for something else — it is the key that makes this run once. */
+  id?: string;
+  /** Partition to run in. A partition-DO only sees its own tables, so an app with several
+   * partitions declares one of these per partition, each with its own id. */
+  partition?: string;
+  /** Columns the schema cannot identify — see {@link ExtraTimestampColumns}. */
+  extraColumns?: ExtraTimestampColumns;
+}
+
+/**
+ * A `DataMigration` that rewrites space-form timestamps to ISO-8601 in place.
+ *
+ * ```ts
+ * import { isoTimestampBackfill } from "@pramen/server";
+ * import { CMS_LEGACY_TIMESTAMP_COLUMNS } from "@pramen/cms";
+ *
+ * export const app = {
+ *   migrations: [isoTimestampBackfill({ extraColumns: CMS_LEGACY_TIMESTAMP_COLUMNS })],
+ *   // …
+ * };
+ * ```
+ *
+ * A **new** deployment can declare it too and pay nothing: every `UPDATE` matches no rows.
+ * Declaring it unconditionally is the cheaper habit, because "was this store ever written
+ * by an older build?" is not a question the code can answer later.
+ *
+ * It is not idempotent-by-accident, it is idempotent-by-shape: the `WHERE` matches only the
+ * space form, and what it writes is not the space form. That matters because a data
+ * migration is not required to be idempotent (it is recorded once) but this one runs over
+ * columns an app may also be writing, and a second pass must not corrupt a converted value.
+ */
+export function isoTimestampBackfill(opts: IsoTimestampBackfillOpts = {}): DataMigration {
+  const partition = opts.partition ?? DEFAULT_PARTITION;
+  return {
+    id: opts.id ?? "pramen:iso-timestamps",
+    partition,
+    async up(ctx: MigrationContext): Promise<void> {
+      for (const [table, columns] of timestampColumns(ctx.schema, partition, opts.extraColumns)) {
+        for (const column of columns) {
+          // A bulk UPDATE, not a walk: this runs inside `blockConcurrencyWhile` on a
+          // tenant's first fetch, so a row-by-row rewrite of a large table is what stalls
+          // that request and risks the DO's wall-clock limit.
+          //
+          // The predicate is deliberately exact rather than "does not look like ISO":
+          // length 19 with a space at index 11 (SQLite's `substr` is 1-based) is the space
+          // form and nothing else. A column holding some third format an app wrote by hand
+          // is left alone, because guessing at it would be a silent rewrite of data this
+          // migration does not understand.
+          await ctx.driver.exec(
+            `UPDATE ${q(table)} SET ${q(column)} = replace(${q(column)}, ' ', 'T') || '.000Z' ` +
+              `WHERE ${q(column)} IS NOT NULL AND length(${q(column)}) = 19 AND substr(${q(column)}, 11, 1) = ' '`,
+            [],
+          );
+        }
+      }
+    },
+  };
+}
+
+/**
+ * Which `(table, column)` pairs this migration touches — the schema's `expr.now()` columns
+ * in `partition`, plus whatever the app named.
+ *
+ * Exported for the tests and for anyone who wants to see the list before running it. An
+ * extra column naming a table or column that is not in the schema is IGNORED rather than
+ * throwing: the list is a hand-written constant an app may keep across a schema change, and
+ * failing the whole boot over a stale entry in it would be a worse outcome than skipping it.
+ */
+export function timestampColumns(
+  schema: SchemaDef,
+  partition: string = DEFAULT_PARTITION,
+  extra: ExtraTimestampColumns = {},
+): Map<string, string[]> {
+  const out = new Map<string, string[]>();
+  const inPartition = new Set(entitiesInPartition(schema, partition));
+  for (const table of inPartition) {
+    const fields = schema[table]?.fields as Record<string, FieldDef> | undefined;
+    if (!fields) continue;
+    const cols = Object.entries(fields)
+      // Both spellings: a store written by an older build has the legacy default recorded
+      // in its own DDL, and `migrate()` may not have rebuilt the table yet when this runs
+      // (it does — migrations run after — but the check costs nothing and makes the set
+      // independent of that ordering).
+      .filter(([, f]) => f.defaultExpr === ISO_NOW_SQL || f.defaultExpr === LEGACY_NOW_SQL)
+      .map(([name]) => name);
+    if (cols.length > 0) out.set(table, cols);
+  }
+  for (const [table, columns] of Object.entries(extra)) {
+    if (!inPartition.has(table)) continue;
+    const fields = schema[table]?.fields as Record<string, FieldDef> | undefined;
+    if (!fields) continue;
+    const existing = out.get(table) ?? [];
+    for (const column of columns) {
+      if (fields[column] && !existing.includes(column)) existing.push(column);
+    }
+    if (existing.length > 0) out.set(table, existing);
+  }
+  return out;
+}
+
+/** Double-quote an identifier for SQLite. Table and column names here come from the app's
+ * own schema, never from a request — but the SQL is built by concatenation, so they are
+ * quoted rather than trusted to be quote-free. */
+const q = (ident: string): string => `"${ident.replace(/"/g, '""')}"`;

package/src/sdk/schema.ts

@@ -257,11 +257,48 @@ export class ExprDefault {
   constructor(readonly sql: string) {}
 }
 
-/** SQL-expression defaults for `defaultTo(field, expr.now())`. `now()` is the current
- * UTC timestamp as TEXT (`'YYYY-MM-DD HH:MM:SS'`, like `CURRENT_TIMESTAMP`) — pair it
- * with `t.text()`. `raw(sql)` is an escape hatch for any other SQLite default expression. */
+/**
+ * The SQL `expr.now()` emits: the current UTC instant as **ISO-8601 TEXT with
+ * milliseconds and a `Z`** — `'2026-09-03T21:33:07.222Z'`, byte-for-byte what
+ * `new Date().toISOString()` produces.
+ *
+ * Exported because three things have to agree on it and none of them can see the others:
+ * the DDL that writes the default, {@link isoTimestampBackfill} which rewrites rows
+ * written before this format, and the migration diff that decides a column's DEFAULT
+ * changed at all.
+ */
+export const ISO_NOW_SQL = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
+
+/**
+ * SQL-expression defaults for `defaultTo(field, expr.now())`.
+ *
+ * `now()` is the current UTC instant as ISO-8601 TEXT (`'2026-09-03T21:33:07.222Z'`) —
+ * pair it with `t.text()`.
+ *
+ * **It used to emit `datetime('now')`**, the `CURRENT_TIMESTAMP` space form
+ * (`'2026-09-03 21:33:07'`). That form is wrong for this codebase in two ways that both
+ * fail silently:
+ *
+ *   - It does not compare against `$now()`, which is an ISO string. Lexicographic TEXT
+ *     comparison is exact within one format and wrong across two: both open with
+ *     `YYYY-MM-DD`, so values on different dates still order correctly, but on the SAME
+ *     date index 10 decides — a space (0x20) always sorts below `T` (0x54), whatever the
+ *     time-of-day. So `{ publishedAt: { lte: $now() } }` over a space-form column matches
+ *     any row dated today, including one scheduled for later today. A time boundary that
+ *     silently does not hold, for a day at a time.
+ *   - It is second-resolution, so two writes in the same second are indistinguishable and
+ *     `ORDER BY createdAt` falls back to an arbitrary tiebreak. `%f` gives milliseconds.
+ *
+ * Changing it means a column's stored values change shape, so **existing rows must be
+ * rewritten** — a rebuild copies values through untouched and would leave the two formats
+ * mixed in one column, which is worse than either alone. {@link isoTimestampBackfill} is
+ * that rewrite; declare it in `app.migrations`.
+ *
+ * `raw(sql)` remains the escape hatch for any other SQLite default expression, including
+ * `expr.raw("datetime('now')")` if you deliberately want the old shape.
+ */
 export const expr = {
-  now: (): ExprDefault => new ExprDefault("datetime('now')"),
+  now: (): ExprDefault => new ExprDefault(ISO_NOW_SQL),
   raw: (sql: string): ExprDefault => new ExprDefault(sql),
 };
 

package/src/worker.ts

@@ -9,6 +9,7 @@ import { dispatch, tasksFacade, bindTasks } from "./runtime/dispatch";
 import type { EnvBag } from "./sdk/handlers";
 import type { JsonValue } from "./sdk/infer";
 import { ensureOutbox, drainOutbox, listTasks } from "./runtime/outbox";
+import { appliedMigrations, runDataMigrations } from "./runtime/data-migrations";
 import { createMail } from "./runtime/mail";
 import { createQueue, type QueueProducerBinding } from "./runtime/queue";
 import { dispatchQueueBatch, type QueueBatch, type QueueContext } from "./runtime/queue-consumer";
@@ -234,6 +235,43 @@ export function makeWorker(app: PramenApp) {
     }
   };
 
+  // The mirror of the DO's runDataMigrations() for the D1 store, run once per isolate after
+  // migration. It diverges in one deliberate way: it runs EVERY declared migration whatever
+  // partition it names, because D1 is ONE shared database with no partition split — every
+  // entity's table lives in it, so a migration declared for "audit" has real rows to touch
+  // here and would otherwise be permanently unrunnable on this store. Each is still recorded
+  // under its OWN declared partition key, so a ledger read is comparable across stores.
+  //
+  // Errors are NOT swallowed (unlike runBootstrapD1): the .catch in ensureD1Migrated clears
+  // `d1Ready`, so a failed migration fails this request and is retried on the next one —
+  // the same fail-closed contract as the DO path.
+  //
+  // ATOMICITY CAVEAT: D1's `transaction(fn)` is `fn()` (no interactive transactions), so
+  // here the claim and the work do NOT commit together. The runner claims the ledger row
+  // before running (which is what keeps two cold isolates from both applying the same
+  // backfill — `d1Ready` is per-isolate and there is no single writer) and releases it on a
+  // throw, so a failed migration leaves partial writes and re-runs. Write SQL that tolerates
+  // that (`WHERE col IS NULL`) when the D1 store is in play.
+  const runDataMigrationsD1 = async (driver: Driver): Promise<void> => {
+    const migrations = app.migrations;
+    if (!migrations?.length) return;
+    const db = new Db(driver, { acl: d1Acl, identity: { roles: ["admin"] }, system: true, schema: app.schema, suppressTriggers: true }, app.schema);
+    await runDataMigrations(driver, migrations, {
+      // No `partition` — see the "runs every declared migration" note above.
+      makeContext: (partition) => ({ db, driver, schema: app.schema, partition }),
+    });
+  };
+
+  /** Partitions an admin route may address: the schema's, the default (always addressable),
+   * and any a data migration declares. Computed once — the app is static. */
+  let knownPartitionsCache: Set<string> | undefined;
+  const knownPartitions = (): Set<string> => {
+    if (!knownPartitionsCache) {
+      knownPartitionsCache = new Set([DEFAULT_PARTITION, ...partitionsOf(app.schema), ...(app.migrations ?? []).map((m) => m.partition ?? DEFAULT_PARTITION)]);
+    }
+    return knownPartitionsCache;
+  };
+
   let d1Ready: Promise<void> | undefined;
   /** Run one handler against the D1 store, in the Worker. The request path and the
    * PRIVILEGED path (routes, which have no ctx.db) both come through here, so the two
@@ -293,6 +331,7 @@ export function makeWorker(app: PramenApp) {
   const ensureD1Migrated = (driver: Driver, allowDestructive: boolean): Promise<void> => {
     if (!d1Ready) {
       d1Ready = migrate(driver, app.schema, { allowDestructive })
+        .then(() => runDataMigrationsD1(driver)) // imperative, recorded backfills (fail closed)
         .then(() => ensureOutbox(driver)) // the deferred-tasks table also lives in D1
         .then(() => runBootstrapD1(driver)) // converge code-defined reference data
         .then(() => undefined)
@@ -354,6 +393,26 @@ export function makeWorker(app: PramenApp) {
     return listTasks(driver, { status, limit });
   };
 
+  /** Read the data-migration ledger straight from D1 in the Worker (there is no DO on this
+   * path), shaped exactly like the DO's /__migrations answer so the CLI can't tell them
+   * apart. The D1 ledger holds every partition's rows; filter to the one asked for.
+   *
+   * Strictly READ-ONLY: deliberately NOT ensureD1Migrated / ensureMigrationsTable. Booting
+   * the store here would apply every pending backfill as a side effect of asking about them,
+   * so `migrations status` could never report PENDING — which is the only question the
+   * command exists to answer. An absent ledger table is simply "none applied". */
+  const listD1Migrations = async (env: Env, tenant: string, partition: string): Promise<unknown> => {
+    if (!env.DB) throw new BadRequest("D1 store is not configured");
+    // The same COMMINGLING GUARD as every other D1 entry point: one shared database with no
+    // tenant column, so reporting it as some specific tenant's ledger requires the opt-in.
+    if (tenant !== "main" && env.PRAMEN_D1_ALLOW_MULTITENANT !== "true") {
+      throw new Forbidden(`D1 store for tenant '${tenant}' (shared D1 has no tenant isolation — set PRAMEN_D1_ALLOW_MULTITENANT=true to allow)`);
+    }
+    const driver = new D1Driver(env.DB, { start: "first-primary" });
+    const rows = await appliedMigrations(driver, partition).catch(() => []);
+    return { partition, applied: rows.map((r) => ({ id: r.id, appliedAt: r.appliedAt })) };
+  };
+
   return {
     async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
     const url = new URL(request.url);
@@ -482,6 +541,35 @@ export function makeWorker(app: PramenApp) {
       return withCors(res, cors);
     }
 
+    // --- admin: which data migrations a tenant has applied. The fleet-wide "is it safe to
+    // prune this id?" signal — a cold tenant is unmigrated until touched, so nothing else
+    // can answer it. `x-pramen-store: d1` reads the Worker's shared ledger instead. ---
+    if (url.pathname === "/admin/migrations") {
+      if (!isAdmin(identity)) return withCors(forbidden("migrations"), cors);
+      const partition = url.searchParams.get("partition") || DEFAULT_PARTITION;
+      const tenant = url.searchParams.get("tenant") ?? "main";
+      // `partition` is caller-supplied and reaches partitionStubFor, which INSTANTIATES a
+      // DO — the same reason /live validates it: without this an admin typo mints a junk DO
+      // and a permanent registry key for a partition nothing lives in. A migration may
+      // declare a partition of its own, so accept those too.
+      if (!knownPartitions().has(partition)) return withCors(badRequest(`unknown partition '${partition}'`), cors);
+      try {
+        if (request.headers.get("x-pramen-store") === "d1") {
+          return withCors(json({ ok: true, result: await listD1Migrations(env, tenant, partition) }), cors);
+        }
+        const stub = partitionStubFor(env, tenant, partition);
+        const res = await stub.fetch(
+          new Request("https://do/__migrations", { headers: { "x-pramen-tenant": tenant, "x-pramen-partition": partition } }),
+        );
+        return withCors(res, cors);
+      } catch (err) {
+        // The DO branch throws a plain Error on a D1-ONLY deployment (no PRAMEN binding).
+        // Uncaught that is an opaque 500; surfaced, it's the actionable "pin the D1 store".
+        const { status, body } = toResponse(err);
+        return withCors(json(body, status), cors);
+      }
+    }
+
     // --- admin: generic data ops over a tenant's tables (browse/edit any row).
     // Body: { tenant, table, op: list|get|create|update|delete|count, ... }. Runs
     // in the DO under SYSTEM scope (ACL bypassed) — gated to admins here. ---
@@ -564,7 +652,7 @@ export function makeWorker(app: PramenApp) {
           "Header X-Pramen-Tenant selects the store (default: main). " +
           "Admin (optional partition selects the partition DO, default: " + DEFAULT_PARTITION + "): " +
           "GET /tenants, POST /admin/recover {tenant,timestamp,partition?}, GET /admin/schema?tenant=&partition=, " +
-          "POST /admin/data {tenant,table,op,partition?}.\n",
+          "GET /admin/migrations?tenant=&partition=, POST /admin/data {tenant,table,op,partition?}.\n",
         { headers: { "content-type": "text/plain" } },
       );
     }