@pramen/server 0.0.59 → 0.0.61

package/dist/cli.js

@@ -10,6 +10,8 @@
 //   pramen schema snapshot            save the current schema to .pramen/schema.json
 //   pramen schema diff                compare the schema to the snapshot (safe vs unsafe changes)
 //   pramen schema status [--tenant t] [--url u] [--token jwt]   compare a deployed tenant to the schema
+//   pramen migrations list            declared data-migration ids, in order
+//   pramen migrations status          applied vs pending, per tenant + partition
 //   pramen token <sub> [roles...] [--tenant a,b]                mint a dev JWT
 //
 // The bin uses a `bun` shebang: the `schema *` commands import your app module (a .ts
@@ -21,7 +23,8 @@ import { dirname, resolve } from "node:path";
 import { createTableSql } from "./runtime/ddl";
 import { schemaHash } from "./runtime/migrate";
 import { diffSchemaFingerprint, schemaFingerprint } from "./runtime/schema-diff";
-import { entitiesInPartition, partitionsOf } from "./sdk/schema";
+import { DEFAULT_PARTITION, entitiesInPartition, partitionsOf } from "./sdk/schema";
+import { migrationsForPartition } from "./runtime/data-migrations";
 import { signDevToken } from "./runtime/dev-token";
 const sign = (payload) => signDevToken(payload);
 /** The sub-schema of a single partition (used to mirror the DO's per-partition hash,
@@ -57,7 +60,7 @@ async function loadApp() {
         if (existsSync(p)) {
             const mod = (await import(p));
             if (mod.app?.schema)
-                return mod.app;
+                return { schema: mod.app.schema, migrations: mod.app.migrations };
             fail(`${c} does not export { app }`);
         }
     }
@@ -75,6 +78,9 @@ Usage: pramen <command>
   schema diff               compare the schema to the snapshot (safe vs unsafe)
   schema status             compare a deployed tenant's schema to the local schema
                             [--tenant t] [--url u] [--token jwt]
+  migrations list           declared data migration ids, in order (id + partition)
+  migrations status         applied vs pending data migrations for a deployed tenant
+                            [--tenant t] [--all-tenants] [--store d1] [--url u] [--token jwt]
   token <sub> [roles...]    mint a dev JWT [--tenant a,b]
 
 Flags: --app <path> to point at your app module (default ./app.ts or ./example/app.ts).`;
@@ -179,6 +185,95 @@ async function schemaCmd(sub) {
     }
     console.log(HELP);
 }
+/** GET the applied-migration ledger for one (tenant, partition). Exits non-zero rather
+ * than reporting a partial fleet — "not answered" must never read as "not applied". */
+async function fetchApplied(url, token, target, store) {
+    const qs = target.partition === DEFAULT_PARTITION ? "" : `&partition=${encodeURIComponent(target.partition)}`;
+    const headers = new Headers({ authorization: `Bearer ${token}` });
+    // The ledger lives in the Worker's shared D1 on that store — there is no DO to ask, and
+    // without this header the request routes to a DO binding a D1-only deploy doesn't have.
+    if (store === "d1")
+        headers.set("x-pramen-store", "d1");
+    const res = await fetch(`${url}/admin/migrations?tenant=${encodeURIComponent(target.tenant)}${qs}`, {
+        headers,
+    }).catch((e) => fail(`migrations status: cannot reach ${url} (${e.message})`));
+    const body = (await res.json().catch(() => ({})));
+    if (!res.ok || !body.ok || !body.result) {
+        fail(`migrations status failed (tenant ${target.tenant}, partition ${target.partition}): ${body.error ?? res.status}`);
+    }
+    return body.result.applied;
+}
+async function migrationsCmd(sub) {
+    if (sub === "list") {
+        const { migrations } = await loadApp();
+        if (!migrations?.length) {
+            console.log("no migrations declared.");
+            return;
+        }
+        for (const m of migrations)
+            console.log(`${m.id}  (partition: ${m.partition ?? DEFAULT_PARTITION})`);
+        return;
+    }
+    if (sub === "status") {
+        const { schema, migrations } = await loadApp();
+        const declared = migrations ?? [];
+        const url = flag("url") ?? "http://localhost:8787";
+        const token = flag("token") ?? (await sign({ sub: "cli", roles: ["admin"] }));
+        const store = flag("store");
+        if (store !== undefined && store !== "d1" && store !== "do")
+            fail(`migrations status: --store must be "do" or "d1"`);
+        // --all-tenants asks the registry which (tenant, partition) DOs actually exist — the
+        // only way to answer "is it safe to prune this id?", since a tenant nobody has touched
+        // since the migration shipped is still unmigrated and no local artifact knows that.
+        let targets;
+        if (argv.includes("--all-tenants")) {
+            const res = await fetch(`${url}/tenants`, { headers: { authorization: `Bearer ${token}` } }).catch((e) => fail(`migrations status: cannot reach ${url} (${e.message})`));
+            const body = (await res.json().catch(() => ({})));
+            if (!res.ok || !body.ok || !body.result)
+                fail(`migrations status: /tenants failed: ${body.error ?? res.status}`);
+            targets = body.result;
+        }
+        else {
+            const tenant = flag("tenant") ?? "main";
+            // Union the schema's partitions with the ones the migrations themselves declare.
+            // partitionsOf() only returns partitions an ENTITY lives in, so an app whose every
+            // entity is partitioned would drop the default partition from the report — and a
+            // default-partition migration would silently never be listed, in the one command
+            // whose answer gates deleting it.
+            const declaredIn = declared.map((m) => m.partition ?? DEFAULT_PARTITION);
+            targets = [...new Set([...partitionsOf(schema), ...declaredIn])].map((partition) => ({ tenant, partition }));
+        }
+        for (const target of targets) {
+            console.log(`\ntenant: ${target.tenant}   partition: ${target.partition}`);
+            const applied = await fetchApplied(url, token, target, store);
+            const appliedAt = new Map(applied.map((a) => [a.id, a.appliedAt]));
+            const forHere = migrationsForPartition(declared, target.partition);
+            let pending = 0;
+            for (const m of forHere) {
+                const at = appliedAt.get(m.id);
+                if (at)
+                    console.log(`  ✓ ${m.id}  (applied ${at})`);
+                else {
+                    console.log(`  • ${m.id} PENDING`);
+                    pending++;
+                }
+            }
+            // A ledger row with no declaration left in the codebase — the pruning question in
+            // reverse. Harmless (it stays applied), but it means this deploy no longer describes
+            // what ran, so a fresh tenant and this one are NOT converging on the same history.
+            const declaredIds = new Set(forHere.map((m) => m.id));
+            for (const a of applied)
+                if (!declaredIds.has(a.id))
+                    console.log(`  ? ${a.id} applied but NOT DECLARED`);
+            console.log(`  ${forHere.length - pending} applied, ${pending} pending`);
+        }
+        console.log("\nA migration is only safe to DELETE once EVERY live tenant reports it applied — the same trap as\n" +
+            "`renamedFrom`: migration is lazy and per-DO, so a tenant nobody has touched is still unmigrated\n" +
+            "and would silently skip the id if it were gone. Check with --all-tenants.");
+        return;
+    }
+    console.log(HELP);
+}
 async function tokenCmd(args) {
     const pos = positionals(args);
     const sub = pos[0];
@@ -287,6 +382,8 @@ async function main() {
             return initCmd(argv.slice(1));
         case "schema":
             return schemaCmd(argv[1]);
+        case "migrations":
+            return migrationsCmd(argv[1]);
         case "token":
             return tokenCmd(argv.slice(1));
         default:

package/dist/durable-object.d.ts

@@ -46,6 +46,7 @@ export declare class PramenDOBase extends DurableObject<DoEnv> {
     private readonly subsBySocket;
     constructor(ctx: DurableObjectState, env: DoEnv, app: PramenApp);
     private ensureMigrated;
+    private runDataMigrations;
     private runBootstrap;
     fetch(request: Request): Promise<Response>;
     /** Arm the drain alarm soon after a mutation enqueued task(s). setAlarm replaces any
@@ -75,6 +76,7 @@ export declare class PramenDOBase extends DurableObject<DoEnv> {
     private ensureRegistered;
     private handleRecover;
     private handleSchema;
+    private handleMigrations;
     private handleAdminData;
     private ctxFor;
     private widenedEnv;

package/dist/durable-object.js

@@ -20,6 +20,7 @@ import { dispatch, tasksFacade, bindTasks } from "./runtime/dispatch";
 import { createMail } from "./runtime/mail";
 import { createQueue } from "./runtime/queue";
 import { ensureOutbox, drainOutbox, listTasks } from "./runtime/outbox";
+import { appliedMigrations, runDataMigrations as runPendingDataMigrations } from "./runtime/data-migrations";
 import { Db } from "./runtime/db";
 import { digest } from "./runtime/digest";
 import { compileAcl } from "./runtime/acl";
@@ -101,11 +102,33 @@ export class PramenDOBase extends DurableObject {
             if (this.migrated)
                 return; // a concurrent first request already migrated
             await this.driver.transaction(() => migrate(this.driver, this.app.schema, { allowDestructive, partition }).then(() => { }));
+            await this.runDataMigrations(); // imperative, recorded backfills for THIS partition
             await ensureOutbox(this.driver); // the deferred-tasks table (internal, all partitions)
             await this.runBootstrap(); // converge code-defined reference data (default partition only)
             this.migrated = true;
         });
     }
+    // Run this partition's pending app.migrations once, right after migrate() (so a backfill
+    // sees the column ADD COLUMN just created) and before bootstrap. Unlike runBootstrap this
+    // is NOT default-partition-only: a partition's rows live in ITS DO, so each partition-DO
+    // runs its own partition's migrations against its own tables.
+    //
+    // It deliberately does NOT swallow errors, which is the one place it diverges from
+    // runBootstrap. A bootstrap reconciler is idempotent and retried every boot, so logging a
+    // failure and carrying on is safe; a data migration is a ONE-SHOT transformation of user
+    // rows, and swallowing a half-finished backfill would leave the store silently corrupt
+    // AND (via the ledger row it never wrote) look unmigrated forever. Failing closed brings
+    // the tenant's first fetch down instead, and the migration retries on the next one.
+    async runDataMigrations() {
+        const migrations = this.app.migrations;
+        if (!migrations?.length)
+            return;
+        const db = new Db(this.driver, { acl: this.acl, identity: { roles: ["admin"] }, system: true, partition: this.partition, schema: this.app.schema, suppressTriggers: true }, this.app.schema);
+        await runPendingDataMigrations(this.driver, migrations, {
+            partition: this.partition,
+            makeContext: (partition) => ({ db, driver: this.driver, schema: this.app.schema, partition }),
+        });
+    }
     // Run app.bootstrap() once per DO lifetime, right after migration and inside the same
     // blockConcurrencyWhile block, so the first request sees a converged store and two
     // concurrent first fetches can't double-run it. Default partition ONLY: reference data
@@ -137,6 +160,14 @@ export class PramenDOBase extends DurableObject {
         const partitionHeader = request.headers.get("x-pramen-partition");
         if (partitionHeader)
             this.partition = partitionHeader;
+        // READ-ONLY PROBE, answered before the boot. Everything below this line migrates:
+        // ensureMigrated() runs migrate(), the data migrations and bootstrap. Answering the
+        // ledger after that would make `pramen migrations status` apply every pending backfill
+        // as a side effect of ASKING — it could never report PENDING, and `--all-tenants`
+        // (advertised as the read-only "is it safe to prune?" check) would silently migrate the
+        // whole fleet. Reads no table it might have to create; see handleMigrations.
+        if (new URL(request.url).pathname === "/__migrations")
+            return this.handleMigrations();
         await this.ensureMigrated();
         await this.ensureRegistered(request);
         // Persist (tenant, partition) once per instance so a cold alarm can rebuild the
@@ -508,6 +539,23 @@ export class PramenDOBase extends DurableObject {
         const tables = byKey.has(tablesKey) ? JSON.parse(byKey.get(tablesKey)) : {};
         return Response.json({ ok: true, result: { hash: byKey.get(hashKey) ?? null, tables } });
     }
+    // Introspection: which data migrations this partition has applied (admin-gated at the
+    // Worker). Powers the CLI's `migrations status` — the fleet-wide "is it safe to prune this
+    // id yet?" answer, which nothing else can give: a tenant nobody has touched since the
+    // migration shipped is still unmigrated, and the schema hash says nothing about backfills.
+    // Ensure the ledger exists first so a DO that has never applied one answers with [] rather
+    // than a missing-table error.
+    async handleMigrations() {
+        // Strictly read-only — NOT ensureMigrationsTable(). A DO that has never applied a
+        // migration has no ledger table, and CREATE-ing one here would make the probe a write
+        // (and would boot an otherwise-untouched tenant's store just to answer a question about
+        // it). An absent table is simply "none applied".
+        const rows = await appliedMigrations(this.driver, this.partition).catch(() => []);
+        return Response.json({
+            ok: true,
+            result: { partition: this.partition, applied: rows.map((r) => ({ id: r.id, appliedAt: r.appliedAt })) },
+        });
+    }
     // Generic admin data ops (admin-gated at the Worker). Runs through a SYSTEM-mode
     // Db, so ACL is bypassed — admin can browse/edit any row of any table — while the
     // json/fileRef codec, transactions, and live-query broadcast still apply.

package/dist/index.d.ts

@@ -1,10 +1,12 @@
-export { Entity, defineSchema, renamedFrom, notNull, unique, indexed, hidden, defaultTo, primaryKey, generated, expr, ExprDefault, trigger, partitionOf, DEFAULT_PARTITION } from "./sdk/schema";
+export { Entity, defineSchema, renamedFrom, notNull, unique, indexed, hidden, defaultTo, primaryKey, generated, expr, ExprDefault, ISO_NOW_SQL, trigger, partitionOf, DEFAULT_PARTITION } from "./sdk/schema";
+export { isoTimestampBackfill, timestampColumns, LEGACY_NOW_SQL } from "./sdk/iso-timestamps";
+export type { ExtraTimestampColumns, IsoTimestampBackfillOpts } from "./sdk/iso-timestamps";
 export type { TriggerDef, TriggerOp } from "./sdk/schema";
 export { isValidUuid } from "./sdk/uuid";
 export type { DefaultValue, FieldType, FieldDef, EntityFields, EntityDef, SchemaDef, RelationDef, RelationDefs, BelongsToDef, HasManyDef, ManyToManyDef, OneHasOneDef, OneHasOneInverseDef, OnDelete, } from "./sdk/schema";
 export { createApp } from "./sdk/app";
 export { query, mutation, authorizeHandler } from "./sdk/handlers";
-export type { EnvBag, Handler, HandlerContext, HandlerKind, HandlerMap, HandlerOpts, HandlerAuth, Tasks, TaskHandler, AppTaskMap, BootstrapContext, BootstrapFn } from "./sdk/handlers";
+export type { EnvBag, Handler, HandlerContext, HandlerKind, HandlerMap, HandlerOpts, HandlerAuth, Tasks, TaskHandler, AppTaskMap, BootstrapContext, BootstrapFn, MigrationContext, DataMigration } from "./sdk/handlers";
 export { $identity, $input, $now, allow, deny, policy, resolve, role, isAllow, isDeny, isResolver, isIdentityMarker, isInputMarker, isNowMarker } from "./sdk/acl";
 export type { Action, Identity, IdentityMarker, InputMarker, NowMarker, Policy, PolicyRule, PolicyRules, Role, Validator, WhereRule, ConditionalFields, FieldsFn, RelationAclRule, SetValue, ResolverFn, ResolverContext, ResolverDb, } from "./sdk/acl";
 export type { Cell, FieldsOf, InferInsert, InferRow, InferUpdate, JsonValue, JsonObject, SqlValue, CellValue, Row, ProjectedRow, RelationsOf, RelationsResult, WhereClause, WhereInput, WhereOps, } from "./sdk/infer";

package/dist/index.js

@@ -7,7 +7,9 @@
 // which only exists in the Workers runtime; keeping it separate lets the CLI, tests,
 // and codegen load an app.ts for its schema without dragging in the DO runtime.
 // --- schema authoring ---
-export { Entity, defineSchema, renamedFrom, notNull, unique, indexed, hidden, defaultTo, primaryKey, generated, expr, ExprDefault, trigger, partitionOf, DEFAULT_PARTITION } from "./sdk/schema";
+export { Entity, defineSchema, renamedFrom, notNull, unique, indexed, hidden, defaultTo, primaryKey, generated, expr, ExprDefault, ISO_NOW_SQL, trigger, partitionOf, DEFAULT_PARTITION } from "./sdk/schema";
+// The one-off rewrite that goes with `expr.now()` changing shape — see ./sdk/iso-timestamps.
+export { isoTimestampBackfill, timestampColumns, LEGACY_NOW_SQL } from "./sdk/iso-timestamps";
 export { isValidUuid } from "./sdk/uuid";
 // --- app + handlers ---
 export { createApp } from "./sdk/app";

package/dist/pramen.d.ts

@@ -1,7 +1,7 @@
 import { type Env } from "./worker";
 import { pramenDO, type DoEnv } from "./durable-object";
 import { type SchemaDef } from "./sdk/schema";
-import type { AppTaskMap, HandlerMap, BootstrapFn } from "./sdk/handlers";
+import type { AppTaskMap, HandlerMap, BootstrapFn, DataMigration } from "./sdk/handlers";
 import type { AppQueueMap, QueueBatch } from "./runtime/queue-consumer";
 import type { Role } from "./sdk/acl";
 import type { EnvBag } from "./sdk/handlers";
@@ -47,6 +47,12 @@ export interface PramenApp {
      * code-defined reference data into the store (see `BootstrapFn`). Each runs with a
      * privileged system Db; failures are logged, never fatal. */
     bootstrap?: readonly BootstrapFn[];
+    /** Imperative DATA migrations — backfills, splits, normalizations — run in declaration
+     * order after `migrate()` and before `bootstrap`, each recorded ONCE per (id, partition)
+     * in `_pramen_migrations` (see `DataMigration`). The declarative migrator diffs shapes
+     * and so can only enact structure; this is the transformation half. Unlike `bootstrap`,
+     * a failure is NOT swallowed — it fails the boot closed and retries next fetch. */
+    migrations?: readonly DataMigration[];
 }
 export type { Env, DoEnv };
 /** Build the deployable pair for an app. `scheduled` is a Cron Trigger entry that

package/dist/pramen.js

@@ -14,11 +14,13 @@
 import { makeWorker } from "./worker";
 import { pramenDO } from "./durable-object";
 import { validateTriggerTasks } from "./sdk/schema";
+import { validateMigrations } from "./runtime/data-migrations";
 /** Build the deployable pair for an app. `scheduled` is a Cron Trigger entry that
  * drains the D1 outbox (the DO path self-drains via an alarm) — wire it only if you
  * use the D1 store with deferred tasks. */
 export function createPramen(app) {
     validateTriggerTasks(app.schema, Object.keys(app.tasks ?? {})); // fail fast on a typo'd trigger task
+    validateMigrations(app.schema, app.migrations); // fail fast on a duplicate/empty id or an unknown partition
     const worker = makeWorker(app);
     return { fetch: worker.fetch, scheduled: worker.scheduled, queue: worker.queue, PramenDO: pramenDO(app) };
 }

package/dist/runtime/data-migrations.d.ts

@@ -0,0 +1,88 @@
+import { type SchemaDef } from "../sdk/schema";
+import type { DataMigration, MigrationContext } from "../sdk/handlers";
+import type { Driver } from "./driver";
+export declare const MIGRATIONS_TABLE = "_pramen_migrations";
+/** One applied-migration ledger row. */
+export interface AppliedMigration {
+    id: string;
+    partition: string;
+    /** ISO-8601 UTC instant the migration committed. */
+    appliedAt: string;
+}
+/** Create the ledger if absent. Idempotent — run on every boot before the runner and from
+ * the /__migrations endpoint (a DO that has never applied one still answers). Internal
+ * table (`_pramen_` prefix), so `isInternalTable()` keeps the migrator's hands off it.
+ *
+ * `partition` is a reserved-ish word in some engines, so every identifier goes through
+ * `dialect.id(...)` (the single `quoteIdent` source of truth) rather than being interpolated
+ * bare — here and in every query below. */
+export declare function ensureMigrationsTable(driver: Driver): Promise<void>;
+/** Ledger rows for one partition — or every partition when `partition` is omitted (the D1
+ * store, which holds one shared ledger for all of them). Oldest first. */
+export declare function appliedMigrations(driver: Driver, partition?: string): Promise<AppliedMigration[]>;
+/** The migrations that belong to `partition`: an entry with no declared partition belongs
+ * to the default one. Both boot paths select through this, so "which partition owns this
+ * migration" is answered in exactly one place. */
+export declare function migrationsForPartition(migrations: readonly DataMigration[], partition: string): readonly DataMigration[];
+export interface RunMigrationsResult {
+    /** Ids applied by THIS run, in the order they committed. */
+    applied: string[];
+    /** Ids already in the ledger (a previous boot applied them). */
+    skipped: string[];
+}
+/** Lease timings. Defaults are deliberate, not arbitrary:
+ *
+ *  - `ttlMs` must EXCEED the slowest migration, because a lease that expires under a holder
+ *    still running is exactly the double-apply this whole mechanism prevents. 60s is above
+ *    anything that can finish inside a Worker's limits at all, and the docs already require
+ *    backfills to be bounded.
+ *  - `waitMs` bounds how long a runner blocks on someone else's live lease before failing
+ *    closed. Long enough for an ordinary backfill to land (so the waiter proceeds against
+ *    fully-migrated data), short enough not to hold a request open.
+ *  - `pollMs` is the re-check interval while waiting. */
+export interface LeaseOpts {
+    ttlMs: number;
+    waitMs: number;
+    pollMs: number;
+}
+export declare const DEFAULT_LEASE: LeaseOpts;
+export interface RunMigrationsOpts {
+    /** The partition to run: only its migrations are selected, and each ledger row is keyed
+     * by it. OMIT it on the D1 store — see `runDataMigrations` for why that store runs every
+     * declared migration regardless of partition. */
+    partition?: string;
+    /** Build the privileged context for one migration. Called per migration with the
+     * partition its ledger row will be keyed under, so a caller can scope `db` accordingly. */
+    makeContext: (partition: string) => MigrationContext;
+    /** Override the lease timings (see `LeaseOpts`) — for tests, and for a deployment whose
+     * backfills legitimately run long. Absent ⇒ `DEFAULT_LEASE`. */
+    lease?: Partial<LeaseOpts>;
+}
+/** Run every pending migration, in DECLARATION order, each inside its own
+ * `driver.transaction()`, CLAIMING its ledger row before it runs the work — so "the work
+ * happened" and "the work is recorded" cannot come apart on a substrate with real
+ * transactions (the DO), and cannot double-run on one without (D1). A migration whose claim
+ * loses the race (another isolate holds it, or a previous boot applied it) is skipped.
+ *
+ * D1 CAVEAT: `D1Driver.transaction(fn)` is `fn()` (D1 has no interactive transactions), so
+ * there the claim and the work do NOT commit together. A throw therefore RELEASES the claim
+ * explicitly (a compensating DELETE, issued inside the transaction so the DO simply rolls it
+ * back with everything else) — otherwise a failed migration would stay marked as applied,
+ * which is the one outcome the fail-closed contract exists to prevent. A holder that dies
+ * without reaching that DELETE is covered by the lease expiring instead. Prefer SQL that
+ * tolerates a re-run (`WHERE col IS NULL`) when the D1 store is in play: a mid-flight failure
+ * leaves the partial writes behind.
+ *
+ * Fails CLOSED: the first throw aborts the run, so migrations declared after it do not run
+ * either (order is a contract — a later one may depend on an earlier one's output). */
+export declare function runDataMigrations(driver: Driver, migrations: readonly DataMigration[], opts: RunMigrationsOpts): Promise<RunMigrationsResult>;
+/** Static validation, called from `createPramen` next to `validateTriggerTasks` — these are
+ * declaration bugs, and the only honest time to surface them is before a single tenant has
+ * booted. Throws on the first violation:
+ *
+ *  - an empty id (the ledger key would be meaningless);
+ *  - a duplicate id — ids are GLOBALLY unique across the array, not per partition, so a
+ *    copy-pasted id can never quietly mark a different migration as already applied;
+ *  - a declared `partition` no entity lives in — the migration would be dead code on the
+ *    DO path (no DO serves that partition) while still running on D1. */
+export declare function validateMigrations(schema: SchemaDef, migrations: readonly DataMigration[] | undefined): void;