@pramen/server 0.0.58 → 0.0.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +99 -2
- package/dist/durable-object.d.ts +2 -0
- package/dist/durable-object.js +48 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.js +4 -2
- package/dist/pramen.d.ts +7 -1
- package/dist/pramen.js +2 -0
- package/dist/runtime/data-migrations.d.ts +88 -0
- package/dist/runtime/data-migrations.js +256 -0
- package/dist/runtime/mail.d.ts +34 -2
- package/dist/runtime/mail.js +82 -9
- package/dist/sdk/acl.d.ts +13 -5
- package/dist/sdk/acl.js +13 -5
- package/dist/sdk/handlers.d.ts +41 -0
- package/dist/sdk/iso-timestamps.d.ts +54 -0
- package/dist/sdk/iso-timestamps.js +119 -0
- package/dist/sdk/schema.d.ts +39 -3
- package/dist/sdk/schema.js +40 -4
- package/dist/worker.js +88 -1
- package/package.json +1 -1
- package/src/cli.ts +125 -4
- package/src/durable-object.ts +53 -0
- package/src/index.ts +6 -3
- package/src/pramen.ts +9 -1
- package/src/runtime/data-migrations.ts +343 -0
- package/src/runtime/mail.ts +80 -9
- package/src/sdk/acl.ts +13 -5
- package/src/sdk/handlers.ts +43 -0
- package/src/sdk/iso-timestamps.ts +146 -0
- package/src/sdk/schema.ts +41 -4
- package/src/worker.ts +89 -1
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// Rewriting timestamps written before `expr.now()` emitted ISO-8601.
|
|
2
|
+
//
|
|
3
|
+
// `expr.now()` used to emit `datetime('now')` — the `CURRENT_TIMESTAMP` space form,
|
|
4
|
+
// `'2026-09-03 21:33:07'`. It now emits `'2026-09-03T21:33:07.222Z'`. Changing a column's
|
|
5
|
+
// DEFAULT is a MODIFIER change, so `migrate()` rebuilds the table — and a rebuild copies
|
|
6
|
+
// existing values through untouched. New rows would get the new shape and old rows would
|
|
7
|
+
// keep the old one, in the same column.
|
|
8
|
+
//
|
|
9
|
+
// That is worse than either format alone, and the failure is narrower and nastier than
|
|
10
|
+
// "the column is inconsistent". Both forms open with the same `YYYY-MM-DD`, so for values
|
|
11
|
+
// on DIFFERENT dates the date prefix decides and ordering is still correct. It is values on
|
|
12
|
+
// the SAME date that break: index 10 is ` ` (0x20) in the space form and `T` (0x54) in ISO,
|
|
13
|
+
// so a same-day space-form value always sorts BEFORE a same-day ISO one whatever the
|
|
14
|
+
// time-of-day — 23:00 in the old shape sorts below 01:00 in the new.
|
|
15
|
+
//
|
|
16
|
+
// Which makes the deploy window itself the blast radius: the rows written just before and
|
|
17
|
+
// just after the change are exactly the same-day pairs that invert. And a `{ lte: $now() }`
|
|
18
|
+
// policy is the sharp version, because `$now()` is ISO: a space-form value dated today
|
|
19
|
+
// always compares as less than it, so a row scheduled for later TODAY reads as already
|
|
20
|
+
// published until midnight. Nothing reports any of it.
|
|
21
|
+
//
|
|
22
|
+
// So the format change comes with a rewrite, and the rewrite is a DATA migration: it is
|
|
23
|
+
// imperative, it must run exactly once, and it must fail closed. Declaring it is the app's
|
|
24
|
+
// job — this builds it.
|
|
25
|
+
import { DEFAULT_PARTITION, ISO_NOW_SQL, entitiesInPartition } from "./schema";
|
|
26
|
+
/** The old default's SQL, for recognizing a column that carried it. */
|
|
27
|
+
export const LEGACY_NOW_SQL = "datetime('now')";
|
|
28
|
+
/**
|
|
29
|
+
* A `DataMigration` that rewrites space-form timestamps to ISO-8601 in place.
|
|
30
|
+
*
|
|
31
|
+
* ```ts
|
|
32
|
+
* import { isoTimestampBackfill } from "@pramen/server";
|
|
33
|
+
* import { CMS_LEGACY_TIMESTAMP_COLUMNS } from "@pramen/cms";
|
|
34
|
+
*
|
|
35
|
+
* export const app = {
|
|
36
|
+
* migrations: [isoTimestampBackfill({ extraColumns: CMS_LEGACY_TIMESTAMP_COLUMNS })],
|
|
37
|
+
* // …
|
|
38
|
+
* };
|
|
39
|
+
* ```
|
|
40
|
+
*
|
|
41
|
+
* A **new** deployment can declare it too and pay nothing: every `UPDATE` matches no rows.
|
|
42
|
+
* Declaring it unconditionally is the cheaper habit, because "was this store ever written
|
|
43
|
+
* by an older build?" is not a question the code can answer later.
|
|
44
|
+
*
|
|
45
|
+
* It is not idempotent-by-accident, it is idempotent-by-shape: the `WHERE` matches only the
|
|
46
|
+
* space form, and what it writes is not the space form. That matters because a data
|
|
47
|
+
* migration is not required to be idempotent (it is recorded once) but this one runs over
|
|
48
|
+
* columns an app may also be writing, and a second pass must not corrupt a converted value.
|
|
49
|
+
*/
|
|
50
|
+
export function isoTimestampBackfill(opts = {}) {
|
|
51
|
+
const partition = opts.partition ?? DEFAULT_PARTITION;
|
|
52
|
+
return {
|
|
53
|
+
id: opts.id ?? "pramen:iso-timestamps",
|
|
54
|
+
partition,
|
|
55
|
+
async up(ctx) {
|
|
56
|
+
for (const [table, columns] of timestampColumns(ctx.schema, partition, opts.extraColumns)) {
|
|
57
|
+
for (const column of columns) {
|
|
58
|
+
// A bulk UPDATE, not a walk: this runs inside `blockConcurrencyWhile` on a
|
|
59
|
+
// tenant's first fetch, so a row-by-row rewrite of a large table is what stalls
|
|
60
|
+
// that request and risks the DO's wall-clock limit.
|
|
61
|
+
//
|
|
62
|
+
// The predicate is deliberately exact rather than "does not look like ISO":
|
|
63
|
+
// length 19 with a space at index 11 (SQLite's `substr` is 1-based) is the space
|
|
64
|
+
// form and nothing else. A column holding some third format an app wrote by hand
|
|
65
|
+
// is left alone, because guessing at it would be a silent rewrite of data this
|
|
66
|
+
// migration does not understand.
|
|
67
|
+
await ctx.driver.exec(`UPDATE ${q(table)} SET ${q(column)} = replace(${q(column)}, ' ', 'T') || '.000Z' ` +
|
|
68
|
+
`WHERE ${q(column)} IS NOT NULL AND length(${q(column)}) = 19 AND substr(${q(column)}, 11, 1) = ' '`, []);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Which `(table, column)` pairs this migration touches — the schema's `expr.now()` columns
|
|
76
|
+
* in `partition`, plus whatever the app named.
|
|
77
|
+
*
|
|
78
|
+
* Exported for the tests and for anyone who wants to see the list before running it. An
|
|
79
|
+
* extra column naming a table or column that is not in the schema is IGNORED rather than
|
|
80
|
+
* throwing: the list is a hand-written constant an app may keep across a schema change, and
|
|
81
|
+
* failing the whole boot over a stale entry in it would be a worse outcome than skipping it.
|
|
82
|
+
*/
|
|
83
|
+
export function timestampColumns(schema, partition = DEFAULT_PARTITION, extra = {}) {
|
|
84
|
+
const out = new Map();
|
|
85
|
+
const inPartition = new Set(entitiesInPartition(schema, partition));
|
|
86
|
+
for (const table of inPartition) {
|
|
87
|
+
const fields = schema[table]?.fields;
|
|
88
|
+
if (!fields)
|
|
89
|
+
continue;
|
|
90
|
+
const cols = Object.entries(fields)
|
|
91
|
+
// Both spellings: a store written by an older build has the legacy default recorded
|
|
92
|
+
// in its own DDL, and `migrate()` may not have rebuilt the table yet when this runs
|
|
93
|
+
// (it does — migrations run after — but the check costs nothing and makes the set
|
|
94
|
+
// independent of that ordering).
|
|
95
|
+
.filter(([, f]) => f.defaultExpr === ISO_NOW_SQL || f.defaultExpr === LEGACY_NOW_SQL)
|
|
96
|
+
.map(([name]) => name);
|
|
97
|
+
if (cols.length > 0)
|
|
98
|
+
out.set(table, cols);
|
|
99
|
+
}
|
|
100
|
+
for (const [table, columns] of Object.entries(extra)) {
|
|
101
|
+
if (!inPartition.has(table))
|
|
102
|
+
continue;
|
|
103
|
+
const fields = schema[table]?.fields;
|
|
104
|
+
if (!fields)
|
|
105
|
+
continue;
|
|
106
|
+
const existing = out.get(table) ?? [];
|
|
107
|
+
for (const column of columns) {
|
|
108
|
+
if (fields[column] && !existing.includes(column))
|
|
109
|
+
existing.push(column);
|
|
110
|
+
}
|
|
111
|
+
if (existing.length > 0)
|
|
112
|
+
out.set(table, existing);
|
|
113
|
+
}
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
/** Double-quote an identifier for SQLite. Table and column names here come from the app's
|
|
117
|
+
* own schema, never from a request — but the SQL is built by concatenation, so they are
|
|
118
|
+
* quoted rather than trusted to be quote-free. */
|
|
119
|
+
const q = (ident) => `"${ident.replace(/"/g, '""')}"`;
|
package/dist/sdk/schema.d.ts
CHANGED
|
@@ -242,9 +242,45 @@ export declare class ExprDefault {
|
|
|
242
242
|
readonly sql: string;
|
|
243
243
|
constructor(sql: string);
|
|
244
244
|
}
|
|
245
|
-
/**
|
|
246
|
-
*
|
|
247
|
-
*
|
|
245
|
+
/**
|
|
246
|
+
* The SQL `expr.now()` emits: the current UTC instant as **ISO-8601 TEXT with
|
|
247
|
+
* milliseconds and a `Z`** — `'2026-09-03T21:33:07.222Z'`, byte-for-byte what
|
|
248
|
+
* `new Date().toISOString()` produces.
|
|
249
|
+
*
|
|
250
|
+
* Exported because three things have to agree on it and none of them can see the others:
|
|
251
|
+
* the DDL that writes the default, {@link isoTimestampBackfill} which rewrites rows
|
|
252
|
+
* written before this format, and the migration diff that decides a column's DEFAULT
|
|
253
|
+
* changed at all.
|
|
254
|
+
*/
|
|
255
|
+
export declare const ISO_NOW_SQL = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
|
|
256
|
+
/**
|
|
257
|
+
* SQL-expression defaults for `defaultTo(field, expr.now())`.
|
|
258
|
+
*
|
|
259
|
+
* `now()` is the current UTC instant as ISO-8601 TEXT (`'2026-09-03T21:33:07.222Z'`) —
|
|
260
|
+
* pair it with `t.text()`.
|
|
261
|
+
*
|
|
262
|
+
* **It used to emit `datetime('now')`**, the `CURRENT_TIMESTAMP` space form
|
|
263
|
+
* (`'2026-09-03 21:33:07'`). That form is wrong for this codebase in two ways that both
|
|
264
|
+
* fail silently:
|
|
265
|
+
*
|
|
266
|
+
* - It does not compare against `$now()`, which is an ISO string. Lexicographic TEXT
|
|
267
|
+
* comparison is exact within one format and wrong across two: both open with
|
|
268
|
+
* `YYYY-MM-DD`, so values on different dates still order correctly, but on the SAME
|
|
269
|
+
* date index 10 decides — a space (0x20) always sorts below `T` (0x54), whatever the
|
|
270
|
+
* time-of-day. So `{ publishedAt: { lte: $now() } }` over a space-form column matches
|
|
271
|
+
* any row dated today, including one scheduled for later today. A time boundary that
|
|
272
|
+
* silently does not hold, for a day at a time.
|
|
273
|
+
* - It is second-resolution, so two writes in the same second are indistinguishable and
|
|
274
|
+
* `ORDER BY createdAt` falls back to an arbitrary tiebreak. `%f` gives milliseconds.
|
|
275
|
+
*
|
|
276
|
+
* Changing it means a column's stored values change shape, so **existing rows must be
|
|
277
|
+
* rewritten** — a rebuild copies values through untouched and would leave the two formats
|
|
278
|
+
* mixed in one column, which is worse than either alone. {@link isoTimestampBackfill} is
|
|
279
|
+
* that rewrite; declare it in `app.migrations`.
|
|
280
|
+
*
|
|
281
|
+
* `raw(sql)` remains the escape hatch for any other SQLite default expression, including
|
|
282
|
+
* `expr.raw("datetime('now')")` if you deliberately want the old shape.
|
|
283
|
+
*/
|
|
248
284
|
export declare const expr: {
|
|
249
285
|
now: () => ExprDefault;
|
|
250
286
|
raw: (sql: string) => ExprDefault;
|
package/dist/sdk/schema.js
CHANGED
|
@@ -112,11 +112,47 @@ export class ExprDefault {
|
|
|
112
112
|
this.sql = sql;
|
|
113
113
|
}
|
|
114
114
|
}
|
|
115
|
-
/**
|
|
116
|
-
*
|
|
117
|
-
*
|
|
115
|
+
/**
|
|
116
|
+
* The SQL `expr.now()` emits: the current UTC instant as **ISO-8601 TEXT with
|
|
117
|
+
* milliseconds and a `Z`** — `'2026-09-03T21:33:07.222Z'`, byte-for-byte what
|
|
118
|
+
* `new Date().toISOString()` produces.
|
|
119
|
+
*
|
|
120
|
+
* Exported because three things have to agree on it and none of them can see the others:
|
|
121
|
+
* the DDL that writes the default, {@link isoTimestampBackfill} which rewrites rows
|
|
122
|
+
* written before this format, and the migration diff that decides a column's DEFAULT
|
|
123
|
+
* changed at all.
|
|
124
|
+
*/
|
|
125
|
+
export const ISO_NOW_SQL = "strftime('%Y-%m-%dT%H:%M:%fZ','now')";
|
|
126
|
+
/**
|
|
127
|
+
* SQL-expression defaults for `defaultTo(field, expr.now())`.
|
|
128
|
+
*
|
|
129
|
+
* `now()` is the current UTC instant as ISO-8601 TEXT (`'2026-09-03T21:33:07.222Z'`) —
|
|
130
|
+
* pair it with `t.text()`.
|
|
131
|
+
*
|
|
132
|
+
* **It used to emit `datetime('now')`**, the `CURRENT_TIMESTAMP` space form
|
|
133
|
+
* (`'2026-09-03 21:33:07'`). That form is wrong for this codebase in two ways that both
|
|
134
|
+
* fail silently:
|
|
135
|
+
*
|
|
136
|
+
* - It does not compare against `$now()`, which is an ISO string. Lexicographic TEXT
|
|
137
|
+
* comparison is exact within one format and wrong across two: both open with
|
|
138
|
+
* `YYYY-MM-DD`, so values on different dates still order correctly, but on the SAME
|
|
139
|
+
* date index 10 decides — a space (0x20) always sorts below `T` (0x54), whatever the
|
|
140
|
+
* time-of-day. So `{ publishedAt: { lte: $now() } }` over a space-form column matches
|
|
141
|
+
* any row dated today, including one scheduled for later today. A time boundary that
|
|
142
|
+
* silently does not hold, for a day at a time.
|
|
143
|
+
* - It is second-resolution, so two writes in the same second are indistinguishable and
|
|
144
|
+
* `ORDER BY createdAt` falls back to an arbitrary tiebreak. `%f` gives milliseconds.
|
|
145
|
+
*
|
|
146
|
+
* Changing it means a column's stored values change shape, so **existing rows must be
|
|
147
|
+
* rewritten** — a rebuild copies values through untouched and would leave the two formats
|
|
148
|
+
* mixed in one column, which is worse than either alone. {@link isoTimestampBackfill} is
|
|
149
|
+
* that rewrite; declare it in `app.migrations`.
|
|
150
|
+
*
|
|
151
|
+
* `raw(sql)` remains the escape hatch for any other SQLite default expression, including
|
|
152
|
+
* `expr.raw("datetime('now')")` if you deliberately want the old shape.
|
|
153
|
+
*/
|
|
118
154
|
export const expr = {
|
|
119
|
-
now: () => new ExprDefault(
|
|
155
|
+
now: () => new ExprDefault(ISO_NOW_SQL),
|
|
120
156
|
raw: (sql) => new ExprDefault(sql),
|
|
121
157
|
};
|
|
122
158
|
export function defaultTo(field, value) {
|
package/dist/worker.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { authorizeTenant, HmacStrategy, isAdmin, JwksStrategy, resolveIdentity } from "./auth";
|
|
7
7
|
import { dispatch, tasksFacade, bindTasks } from "./runtime/dispatch";
|
|
8
8
|
import { ensureOutbox, drainOutbox, listTasks } from "./runtime/outbox";
|
|
9
|
+
import { appliedMigrations, runDataMigrations } from "./runtime/data-migrations";
|
|
9
10
|
import { createMail } from "./runtime/mail";
|
|
10
11
|
import { createQueue } from "./runtime/queue";
|
|
11
12
|
import { dispatchQueueBatch } from "./runtime/queue-consumer";
|
|
@@ -169,6 +170,42 @@ export function makeWorker(app) {
|
|
|
169
170
|
}
|
|
170
171
|
}
|
|
171
172
|
};
|
|
173
|
+
// The mirror of the DO's runDataMigrations() for the D1 store, run once per isolate after
|
|
174
|
+
// migration. It diverges in one deliberate way: it runs EVERY declared migration whatever
|
|
175
|
+
// partition it names, because D1 is ONE shared database with no partition split — every
|
|
176
|
+
// entity's table lives in it, so a migration declared for "audit" has real rows to touch
|
|
177
|
+
// here and would otherwise be permanently unrunnable on this store. Each is still recorded
|
|
178
|
+
// under its OWN declared partition key, so a ledger read is comparable across stores.
|
|
179
|
+
//
|
|
180
|
+
// Errors are NOT swallowed (unlike runBootstrapD1): the .catch in ensureD1Migrated clears
|
|
181
|
+
// `d1Ready`, so a failed migration fails this request and is retried on the next one —
|
|
182
|
+
// the same fail-closed contract as the DO path.
|
|
183
|
+
//
|
|
184
|
+
// ATOMICITY CAVEAT: D1's `transaction(fn)` is `fn()` (no interactive transactions), so
|
|
185
|
+
// here the claim and the work do NOT commit together. The runner claims the ledger row
|
|
186
|
+
// before running (which is what keeps two cold isolates from both applying the same
|
|
187
|
+
// backfill — `d1Ready` is per-isolate and there is no single writer) and releases it on a
|
|
188
|
+
// throw, so a failed migration leaves partial writes and re-runs. Write SQL that tolerates
|
|
189
|
+
// that (`WHERE col IS NULL`) when the D1 store is in play.
|
|
190
|
+
const runDataMigrationsD1 = async (driver) => {
|
|
191
|
+
const migrations = app.migrations;
|
|
192
|
+
if (!migrations?.length)
|
|
193
|
+
return;
|
|
194
|
+
const db = new Db(driver, { acl: d1Acl, identity: { roles: ["admin"] }, system: true, schema: app.schema, suppressTriggers: true }, app.schema);
|
|
195
|
+
await runDataMigrations(driver, migrations, {
|
|
196
|
+
// No `partition` — see the "runs every declared migration" note above.
|
|
197
|
+
makeContext: (partition) => ({ db, driver, schema: app.schema, partition }),
|
|
198
|
+
});
|
|
199
|
+
};
|
|
200
|
+
/** Partitions an admin route may address: the schema's, the default (always addressable),
|
|
201
|
+
* and any a data migration declares. Computed once — the app is static. */
|
|
202
|
+
let knownPartitionsCache;
|
|
203
|
+
const knownPartitions = () => {
|
|
204
|
+
if (!knownPartitionsCache) {
|
|
205
|
+
knownPartitionsCache = new Set([DEFAULT_PARTITION, ...partitionsOf(app.schema), ...(app.migrations ?? []).map((m) => m.partition ?? DEFAULT_PARTITION)]);
|
|
206
|
+
}
|
|
207
|
+
return knownPartitionsCache;
|
|
208
|
+
};
|
|
172
209
|
let d1Ready;
|
|
173
210
|
/** Run one handler against the D1 store, in the Worker. The request path and the
|
|
174
211
|
* PRIVILEGED path (routes, which have no ctx.db) both come through here, so the two
|
|
@@ -214,6 +251,7 @@ export function makeWorker(app) {
|
|
|
214
251
|
const ensureD1Migrated = (driver, allowDestructive) => {
|
|
215
252
|
if (!d1Ready) {
|
|
216
253
|
d1Ready = migrate(driver, app.schema, { allowDestructive })
|
|
254
|
+
.then(() => runDataMigrationsD1(driver)) // imperative, recorded backfills (fail closed)
|
|
217
255
|
.then(() => ensureOutbox(driver)) // the deferred-tasks table also lives in D1
|
|
218
256
|
.then(() => runBootstrapD1(driver)) // converge code-defined reference data
|
|
219
257
|
.then(() => undefined)
|
|
@@ -271,6 +309,26 @@ export function makeWorker(app) {
|
|
|
271
309
|
await ensureD1Migrated(driver, env.PRAMEN_ALLOW_DESTRUCTIVE === "true");
|
|
272
310
|
return listTasks(driver, { status, limit });
|
|
273
311
|
};
|
|
312
|
+
/** Read the data-migration ledger straight from D1 in the Worker (there is no DO on this
|
|
313
|
+
* path), shaped exactly like the DO's /__migrations answer so the CLI can't tell them
|
|
314
|
+
* apart. The D1 ledger holds every partition's rows; filter to the one asked for.
|
|
315
|
+
*
|
|
316
|
+
* Strictly READ-ONLY: deliberately NOT ensureD1Migrated / ensureMigrationsTable. Booting
|
|
317
|
+
* the store here would apply every pending backfill as a side effect of asking about them,
|
|
318
|
+
* so `migrations status` could never report PENDING — which is the only question the
|
|
319
|
+
* command exists to answer. An absent ledger table is simply "none applied". */
|
|
320
|
+
const listD1Migrations = async (env, tenant, partition) => {
|
|
321
|
+
if (!env.DB)
|
|
322
|
+
throw new BadRequest("D1 store is not configured");
|
|
323
|
+
// The same COMMINGLING GUARD as every other D1 entry point: one shared database with no
|
|
324
|
+
// tenant column, so reporting it as some specific tenant's ledger requires the opt-in.
|
|
325
|
+
if (tenant !== "main" && env.PRAMEN_D1_ALLOW_MULTITENANT !== "true") {
|
|
326
|
+
throw new Forbidden(`D1 store for tenant '${tenant}' (shared D1 has no tenant isolation — set PRAMEN_D1_ALLOW_MULTITENANT=true to allow)`);
|
|
327
|
+
}
|
|
328
|
+
const driver = new D1Driver(env.DB, { start: "first-primary" });
|
|
329
|
+
const rows = await appliedMigrations(driver, partition).catch(() => []);
|
|
330
|
+
return { partition, applied: rows.map((r) => ({ id: r.id, appliedAt: r.appliedAt })) };
|
|
331
|
+
};
|
|
274
332
|
return {
|
|
275
333
|
async fetch(request, env, ctx) {
|
|
276
334
|
const url = new URL(request.url);
|
|
@@ -397,6 +455,35 @@ export function makeWorker(app) {
|
|
|
397
455
|
const res = await stub.fetch(new Request("https://do/__schema", { headers: { "x-pramen-tenant": tenant, "x-pramen-partition": partition } }));
|
|
398
456
|
return withCors(res, cors);
|
|
399
457
|
}
|
|
458
|
+
// --- admin: which data migrations a tenant has applied. The fleet-wide "is it safe to
|
|
459
|
+
// prune this id?" signal — a cold tenant is unmigrated until touched, so nothing else
|
|
460
|
+
// can answer it. `x-pramen-store: d1` reads the Worker's shared ledger instead. ---
|
|
461
|
+
if (url.pathname === "/admin/migrations") {
|
|
462
|
+
if (!isAdmin(identity))
|
|
463
|
+
return withCors(forbidden("migrations"), cors);
|
|
464
|
+
const partition = url.searchParams.get("partition") || DEFAULT_PARTITION;
|
|
465
|
+
const tenant = url.searchParams.get("tenant") ?? "main";
|
|
466
|
+
// `partition` is caller-supplied and reaches partitionStubFor, which INSTANTIATES a
|
|
467
|
+
// DO — the same reason /live validates it: without this an admin typo mints a junk DO
|
|
468
|
+
// and a permanent registry key for a partition nothing lives in. A migration may
|
|
469
|
+
// declare a partition of its own, so accept those too.
|
|
470
|
+
if (!knownPartitions().has(partition))
|
|
471
|
+
return withCors(badRequest(`unknown partition '${partition}'`), cors);
|
|
472
|
+
try {
|
|
473
|
+
if (request.headers.get("x-pramen-store") === "d1") {
|
|
474
|
+
return withCors(json({ ok: true, result: await listD1Migrations(env, tenant, partition) }), cors);
|
|
475
|
+
}
|
|
476
|
+
const stub = partitionStubFor(env, tenant, partition);
|
|
477
|
+
const res = await stub.fetch(new Request("https://do/__migrations", { headers: { "x-pramen-tenant": tenant, "x-pramen-partition": partition } }));
|
|
478
|
+
return withCors(res, cors);
|
|
479
|
+
}
|
|
480
|
+
catch (err) {
|
|
481
|
+
// The DO branch throws a plain Error on a D1-ONLY deployment (no PRAMEN binding).
|
|
482
|
+
// Uncaught that is an opaque 500; surfaced, it's the actionable "pin the D1 store".
|
|
483
|
+
const { status, body } = toResponse(err);
|
|
484
|
+
return withCors(json(body, status), cors);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
400
487
|
// --- admin: generic data ops over a tenant's tables (browse/edit any row).
|
|
401
488
|
// Body: { tenant, table, op: list|get|create|update|delete|count, ... }. Runs
|
|
402
489
|
// in the DO under SYSTEM scope (ACL bypassed) — gated to admins here. ---
|
|
@@ -475,7 +562,7 @@ export function makeWorker(app) {
|
|
|
475
562
|
"Header X-Pramen-Tenant selects the store (default: main). " +
|
|
476
563
|
"Admin (optional partition selects the partition DO, default: " + DEFAULT_PARTITION + "): " +
|
|
477
564
|
"GET /tenants, POST /admin/recover {tenant,timestamp,partition?}, GET /admin/schema?tenant=&partition=, " +
|
|
478
|
-
"POST /admin/data {tenant,table,op,partition?}.\n", { headers: { "content-type": "text/plain" } });
|
|
565
|
+
"GET /admin/migrations?tenant=&partition=, POST /admin/data {tenant,table,op,partition?}.\n", { headers: { "content-type": "text/plain" } });
|
|
479
566
|
}
|
|
480
567
|
// Authorize the tenant against the identity before reaching the DO, so a
|
|
481
568
|
// caller can't address (or register) tenants they have no claim to.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pramen/server",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.60",
|
|
4
4
|
"description": "pramen server runtime \u2014 schema, ACL, ORM, live queries, files, and the createPramen(app) factory for Cloudflare Workers + Durable Objects.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
package/src/cli.ts
CHANGED
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
// pramen schema snapshot save the current schema to .pramen/schema.json
|
|
11
11
|
// pramen schema diff compare the schema to the snapshot (safe vs unsafe changes)
|
|
12
12
|
// pramen schema status [--tenant t] [--url u] [--token jwt] compare a deployed tenant to the schema
|
|
13
|
+
// pramen migrations list declared data-migration ids, in order
|
|
14
|
+
// pramen migrations status applied vs pending, per tenant + partition
|
|
13
15
|
// pramen token <sub> [roles...] [--tenant a,b] mint a dev JWT
|
|
14
16
|
//
|
|
15
17
|
// The bin uses a `bun` shebang: the `schema *` commands import your app module (a .ts
|
|
@@ -22,7 +24,9 @@ import { dirname, resolve } from "node:path";
|
|
|
22
24
|
import { createTableSql } from "./runtime/ddl";
|
|
23
25
|
import { schemaHash } from "./runtime/migrate";
|
|
24
26
|
import { diffSchemaFingerprint, schemaFingerprint, type SchemaFingerprint } from "./runtime/schema-diff";
|
|
25
|
-
import { entitiesInPartition, partitionsOf, type SchemaDef } from "./sdk/schema";
|
|
27
|
+
import { DEFAULT_PARTITION, entitiesInPartition, partitionsOf, type SchemaDef } from "./sdk/schema";
|
|
28
|
+
import { migrationsForPartition } from "./runtime/data-migrations";
|
|
29
|
+
import type { DataMigration } from "./sdk/handlers";
|
|
26
30
|
import { signDevToken } from "./runtime/dev-token";
|
|
27
31
|
|
|
28
32
|
/** The dev JWT claims `pramen token` mints. See `runtime/dev-token.ts` for the signer. */
|
|
@@ -57,14 +61,21 @@ function fail(msg: string): never {
|
|
|
57
61
|
process.exit(1);
|
|
58
62
|
}
|
|
59
63
|
|
|
60
|
-
|
|
64
|
+
/** The slice of an app module the CLI needs: the schema, plus the declared data
|
|
65
|
+
* migrations (`migrations list`/`status` read them; every other command ignores them). */
|
|
66
|
+
interface LoadedApp {
|
|
67
|
+
schema: SchemaDef;
|
|
68
|
+
migrations?: readonly DataMigration[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function loadApp(): Promise<LoadedApp> {
|
|
61
72
|
const explicit = flag("app");
|
|
62
73
|
const candidates = explicit ? [explicit] : ["./app.ts", "./example/app.ts"];
|
|
63
74
|
for (const c of candidates) {
|
|
64
75
|
const p = resolve(process.cwd(), c);
|
|
65
76
|
if (existsSync(p)) {
|
|
66
|
-
const mod = (await import(p)) as { app?:
|
|
67
|
-
if (mod.app?.schema) return mod.app
|
|
77
|
+
const mod = (await import(p)) as { app?: Partial<LoadedApp> };
|
|
78
|
+
if (mod.app?.schema) return { schema: mod.app.schema, migrations: mod.app.migrations };
|
|
68
79
|
fail(`${c} does not export { app }`);
|
|
69
80
|
}
|
|
70
81
|
}
|
|
@@ -83,6 +94,9 @@ Usage: pramen <command>
|
|
|
83
94
|
schema diff compare the schema to the snapshot (safe vs unsafe)
|
|
84
95
|
schema status compare a deployed tenant's schema to the local schema
|
|
85
96
|
[--tenant t] [--url u] [--token jwt]
|
|
97
|
+
migrations list declared data migration ids, in order (id + partition)
|
|
98
|
+
migrations status applied vs pending data migrations for a deployed tenant
|
|
99
|
+
[--tenant t] [--all-tenants] [--store d1] [--url u] [--token jwt]
|
|
86
100
|
token <sub> [roles...] mint a dev JWT [--tenant a,b]
|
|
87
101
|
|
|
88
102
|
Flags: --app <path> to point at your app module (default ./app.ts or ./example/app.ts).`;
|
|
@@ -200,6 +214,111 @@ async function schemaCmd(sub: string | undefined): Promise<void> {
|
|
|
200
214
|
console.log(HELP);
|
|
201
215
|
}
|
|
202
216
|
|
|
217
|
+
/** One (tenant, partition) the ledger is read for. `--all-tenants` fans out over the
|
|
218
|
+
* registry's real pairs; otherwise the schema's partitions for one tenant. */
|
|
219
|
+
interface MigrationTarget {
|
|
220
|
+
tenant: string;
|
|
221
|
+
partition: string;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** GET the applied-migration ledger for one (tenant, partition). Exits non-zero rather
|
|
225
|
+
* than reporting a partial fleet — "not answered" must never read as "not applied". */
|
|
226
|
+
async function fetchApplied(
|
|
227
|
+
url: string,
|
|
228
|
+
token: string,
|
|
229
|
+
target: MigrationTarget,
|
|
230
|
+
store: string | undefined,
|
|
231
|
+
): Promise<{ id: string; appliedAt: string }[]> {
|
|
232
|
+
const qs = target.partition === DEFAULT_PARTITION ? "" : `&partition=${encodeURIComponent(target.partition)}`;
|
|
233
|
+
const headers = new Headers({ authorization: `Bearer ${token}` });
|
|
234
|
+
// The ledger lives in the Worker's shared D1 on that store — there is no DO to ask, and
|
|
235
|
+
// without this header the request routes to a DO binding a D1-only deploy doesn't have.
|
|
236
|
+
if (store === "d1") headers.set("x-pramen-store", "d1");
|
|
237
|
+
const res = await fetch(`${url}/admin/migrations?tenant=${encodeURIComponent(target.tenant)}${qs}`, {
|
|
238
|
+
headers,
|
|
239
|
+
}).catch((e: Error) => fail(`migrations status: cannot reach ${url} (${e.message})`));
|
|
240
|
+
const body = (await res.json().catch(() => ({}))) as {
|
|
241
|
+
ok?: boolean;
|
|
242
|
+
result?: { partition: string; applied: { id: string; appliedAt: string }[] };
|
|
243
|
+
error?: string;
|
|
244
|
+
};
|
|
245
|
+
if (!res.ok || !body.ok || !body.result) {
|
|
246
|
+
fail(`migrations status failed (tenant ${target.tenant}, partition ${target.partition}): ${body.error ?? res.status}`);
|
|
247
|
+
}
|
|
248
|
+
return body.result!.applied;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async function migrationsCmd(sub: string | undefined): Promise<void> {
|
|
252
|
+
if (sub === "list") {
|
|
253
|
+
const { migrations } = await loadApp();
|
|
254
|
+
if (!migrations?.length) {
|
|
255
|
+
console.log("no migrations declared.");
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
for (const m of migrations) console.log(`${m.id} (partition: ${m.partition ?? DEFAULT_PARTITION})`);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
if (sub === "status") {
|
|
262
|
+
const { schema, migrations } = await loadApp();
|
|
263
|
+
const declared = migrations ?? [];
|
|
264
|
+
const url = flag("url") ?? "http://localhost:8787";
|
|
265
|
+
const token = flag("token") ?? (await sign({ sub: "cli", roles: ["admin"] }));
|
|
266
|
+
const store = flag("store");
|
|
267
|
+
if (store !== undefined && store !== "d1" && store !== "do") fail(`migrations status: --store must be "do" or "d1"`);
|
|
268
|
+
|
|
269
|
+
// --all-tenants asks the registry which (tenant, partition) DOs actually exist — the
|
|
270
|
+
// only way to answer "is it safe to prune this id?", since a tenant nobody has touched
|
|
271
|
+
// since the migration shipped is still unmigrated and no local artifact knows that.
|
|
272
|
+
let targets: MigrationTarget[];
|
|
273
|
+
if (argv.includes("--all-tenants")) {
|
|
274
|
+
const res = await fetch(`${url}/tenants`, { headers: { authorization: `Bearer ${token}` } }).catch(
|
|
275
|
+
(e: Error) => fail(`migrations status: cannot reach ${url} (${e.message})`),
|
|
276
|
+
);
|
|
277
|
+
const body = (await res.json().catch(() => ({}))) as { ok?: boolean; result?: MigrationTarget[]; error?: string };
|
|
278
|
+
if (!res.ok || !body.ok || !body.result) fail(`migrations status: /tenants failed: ${body.error ?? res.status}`);
|
|
279
|
+
targets = body.result!;
|
|
280
|
+
} else {
|
|
281
|
+
const tenant = flag("tenant") ?? "main";
|
|
282
|
+
// Union the schema's partitions with the ones the migrations themselves declare.
|
|
283
|
+
// partitionsOf() only returns partitions an ENTITY lives in, so an app whose every
|
|
284
|
+
// entity is partitioned would drop the default partition from the report — and a
|
|
285
|
+
// default-partition migration would silently never be listed, in the one command
|
|
286
|
+
// whose answer gates deleting it.
|
|
287
|
+
const declaredIn = declared.map((m) => m.partition ?? DEFAULT_PARTITION);
|
|
288
|
+
targets = [...new Set([...partitionsOf(schema), ...declaredIn])].map((partition) => ({ tenant, partition }));
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
for (const target of targets) {
|
|
292
|
+
console.log(`\ntenant: ${target.tenant} partition: ${target.partition}`);
|
|
293
|
+
const applied = await fetchApplied(url, token, target, store);
|
|
294
|
+
const appliedAt = new Map(applied.map((a) => [a.id, a.appliedAt]));
|
|
295
|
+
const forHere = migrationsForPartition(declared, target.partition);
|
|
296
|
+
let pending = 0;
|
|
297
|
+
for (const m of forHere) {
|
|
298
|
+
const at = appliedAt.get(m.id);
|
|
299
|
+
if (at) console.log(` ✓ ${m.id} (applied ${at})`);
|
|
300
|
+
else {
|
|
301
|
+
console.log(` • ${m.id} PENDING`);
|
|
302
|
+
pending++;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
// A ledger row with no declaration left in the codebase — the pruning question in
|
|
306
|
+
// reverse. Harmless (it stays applied), but it means this deploy no longer describes
|
|
307
|
+
// what ran, so a fresh tenant and this one are NOT converging on the same history.
|
|
308
|
+
const declaredIds = new Set(forHere.map((m) => m.id));
|
|
309
|
+
for (const a of applied) if (!declaredIds.has(a.id)) console.log(` ? ${a.id} applied but NOT DECLARED`);
|
|
310
|
+
console.log(` ${forHere.length - pending} applied, ${pending} pending`);
|
|
311
|
+
}
|
|
312
|
+
console.log(
|
|
313
|
+
"\nA migration is only safe to DELETE once EVERY live tenant reports it applied — the same trap as\n" +
|
|
314
|
+
"`renamedFrom`: migration is lazy and per-DO, so a tenant nobody has touched is still unmigrated\n" +
|
|
315
|
+
"and would silently skip the id if it were gone. Check with --all-tenants.",
|
|
316
|
+
);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
console.log(HELP);
|
|
320
|
+
}
|
|
321
|
+
|
|
203
322
|
async function tokenCmd(args: string[]): Promise<void> {
|
|
204
323
|
const pos = positionals(args);
|
|
205
324
|
const sub = pos[0];
|
|
@@ -310,6 +429,8 @@ async function main(): Promise<void> {
|
|
|
310
429
|
return initCmd(argv.slice(1));
|
|
311
430
|
case "schema":
|
|
312
431
|
return schemaCmd(argv[1]);
|
|
432
|
+
case "migrations":
|
|
433
|
+
return migrationsCmd(argv[1]);
|
|
313
434
|
case "token":
|
|
314
435
|
return tokenCmd(argv.slice(1));
|
|
315
436
|
default:
|
package/src/durable-object.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { dispatch, tasksFacade, bindTasks } from "./runtime/dispatch";
|
|
|
21
21
|
import { createMail } from "./runtime/mail";
|
|
22
22
|
import { createQueue } from "./runtime/queue";
|
|
23
23
|
import { ensureOutbox, drainOutbox, listTasks } from "./runtime/outbox";
|
|
24
|
+
import { appliedMigrations, runDataMigrations as runPendingDataMigrations } from "./runtime/data-migrations";
|
|
24
25
|
import { Db } from "./runtime/db";
|
|
25
26
|
import { digest } from "./runtime/digest";
|
|
26
27
|
import { compileAcl, type AclContext, type CompiledAcl } from "./runtime/acl";
|
|
@@ -148,12 +149,38 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
148
149
|
await this.driver.transaction(() =>
|
|
149
150
|
migrate(this.driver, this.app.schema, { allowDestructive, partition }).then(() => {}),
|
|
150
151
|
);
|
|
152
|
+
await this.runDataMigrations(); // imperative, recorded backfills for THIS partition
|
|
151
153
|
await ensureOutbox(this.driver); // the deferred-tasks table (internal, all partitions)
|
|
152
154
|
await this.runBootstrap(); // converge code-defined reference data (default partition only)
|
|
153
155
|
this.migrated = true;
|
|
154
156
|
});
|
|
155
157
|
}
|
|
156
158
|
|
|
159
|
+
// Run this partition's pending app.migrations once, right after migrate() (so a backfill
|
|
160
|
+
// sees the column ADD COLUMN just created) and before bootstrap. Unlike runBootstrap this
|
|
161
|
+
// is NOT default-partition-only: a partition's rows live in ITS DO, so each partition-DO
|
|
162
|
+
// runs its own partition's migrations against its own tables.
|
|
163
|
+
//
|
|
164
|
+
// It deliberately does NOT swallow errors, which is the one place it diverges from
|
|
165
|
+
// runBootstrap. A bootstrap reconciler is idempotent and retried every boot, so logging a
|
|
166
|
+
// failure and carrying on is safe; a data migration is a ONE-SHOT transformation of user
|
|
167
|
+
// rows, and swallowing a half-finished backfill would leave the store silently corrupt
|
|
168
|
+
// AND (via the ledger row it never wrote) look unmigrated forever. Failing closed brings
|
|
169
|
+
// the tenant's first fetch down instead, and the migration retries on the next one.
|
|
170
|
+
private async runDataMigrations(): Promise<void> {
|
|
171
|
+
const migrations = this.app.migrations;
|
|
172
|
+
if (!migrations?.length) return;
|
|
173
|
+
const db = new Db(
|
|
174
|
+
this.driver,
|
|
175
|
+
{ acl: this.acl, identity: { roles: ["admin"] }, system: true, partition: this.partition, schema: this.app.schema, suppressTriggers: true },
|
|
176
|
+
this.app.schema,
|
|
177
|
+
);
|
|
178
|
+
await runPendingDataMigrations(this.driver, migrations, {
|
|
179
|
+
partition: this.partition,
|
|
180
|
+
makeContext: (partition) => ({ db, driver: this.driver, schema: this.app.schema, partition }),
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
|
|
157
184
|
// Run app.bootstrap() once per DO lifetime, right after migration and inside the same
|
|
158
185
|
// blockConcurrencyWhile block, so the first request sees a converged store and two
|
|
159
186
|
// concurrent first fetches can't double-run it. Default partition ONLY: reference data
|
|
@@ -187,6 +214,14 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
187
214
|
const partitionHeader = request.headers.get("x-pramen-partition");
|
|
188
215
|
if (partitionHeader) this.partition = partitionHeader;
|
|
189
216
|
|
|
217
|
+
// READ-ONLY PROBE, answered before the boot. Everything below this line migrates:
|
|
218
|
+
// ensureMigrated() runs migrate(), the data migrations and bootstrap. Answering the
|
|
219
|
+
// ledger after that would make `pramen migrations status` apply every pending backfill
|
|
220
|
+
// as a side effect of ASKING — it could never report PENDING, and `--all-tenants`
|
|
221
|
+
// (advertised as the read-only "is it safe to prune?" check) would silently migrate the
|
|
222
|
+
// whole fleet. Reads no table it might have to create; see handleMigrations.
|
|
223
|
+
if (new URL(request.url).pathname === "/__migrations") return this.handleMigrations();
|
|
224
|
+
|
|
190
225
|
await this.ensureMigrated();
|
|
191
226
|
await this.ensureRegistered(request);
|
|
192
227
|
// Persist (tenant, partition) once per instance so a cold alarm can rebuild the
|
|
@@ -583,6 +618,24 @@ export class PramenDOBase extends DurableObject<DoEnv> {
|
|
|
583
618
|
return Response.json({ ok: true, result: { hash: byKey.get(hashKey) ?? null, tables } });
|
|
584
619
|
}
|
|
585
620
|
|
|
621
|
+
// Introspection: which data migrations this partition has applied (admin-gated at the
|
|
622
|
+
// Worker). Powers the CLI's `migrations status` — the fleet-wide "is it safe to prune this
|
|
623
|
+
// id yet?" answer, which nothing else can give: a tenant nobody has touched since the
|
|
624
|
+
// migration shipped is still unmigrated, and the schema hash says nothing about backfills.
|
|
625
|
+
// Ensure the ledger exists first so a DO that has never applied one answers with [] rather
|
|
626
|
+
// than a missing-table error.
|
|
627
|
+
private async handleMigrations(): Promise<Response> {
|
|
628
|
+
// Strictly read-only — NOT ensureMigrationsTable(). A DO that has never applied a
|
|
629
|
+
// migration has no ledger table, and CREATE-ing one here would make the probe a write
|
|
630
|
+
// (and would boot an otherwise-untouched tenant's store just to answer a question about
|
|
631
|
+
// it). An absent table is simply "none applied".
|
|
632
|
+
const rows = await appliedMigrations(this.driver, this.partition).catch(() => []);
|
|
633
|
+
return Response.json({
|
|
634
|
+
ok: true,
|
|
635
|
+
result: { partition: this.partition, applied: rows.map((r) => ({ id: r.id, appliedAt: r.appliedAt })) },
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
|
|
586
639
|
// Generic admin data ops (admin-gated at the Worker). Runs through a SYSTEM-mode
|
|
587
640
|
// Db, so ACL is bypassed — admin can browse/edit any row of any table — while the
|
|
588
641
|
// json/fileRef codec, transactions, and live-query broadcast still apply.
|