@pramen/server 0.0.13 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +17 -2
- package/dist/auth.js +26 -7
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +311 -0
- package/dist/durable-object.d.ts +22 -4
- package/dist/durable-object.js +121 -55
- package/dist/index.d.ts +4 -0
- package/dist/index.js +3 -0
- package/dist/pramen.d.ts +6 -0
- package/dist/pramen.js +1 -1
- package/dist/runtime/acl.js +28 -7
- package/dist/runtime/db.d.ts +6 -0
- package/dist/runtime/db.js +86 -10
- package/dist/runtime/ddl.d.ts +16 -3
- package/dist/runtime/ddl.js +28 -8
- package/dist/runtime/dispatch.js +2 -0
- package/dist/runtime/driver.d.ts +41 -7
- package/dist/runtime/driver.js +38 -11
- package/dist/runtime/migrate.d.ts +1 -1
- package/dist/runtime/migrate.js +222 -33
- package/dist/runtime/outbox.js +28 -6
- package/dist/runtime/queue-consumer.d.ts +71 -0
- package/dist/runtime/queue-consumer.js +63 -0
- package/dist/runtime/queue.d.ts +72 -0
- package/dist/runtime/queue.js +110 -0
- package/dist/runtime/read-engine.js +7 -2
- package/dist/runtime/schema-diff.d.ts +28 -5
- package/dist/runtime/schema-diff.js +111 -19
- package/dist/runtime/storage.d.ts +7 -0
- package/dist/runtime/storage.js +0 -0
- package/dist/sdk/handlers.d.ts +7 -0
- package/dist/worker.d.ts +36 -0
- package/dist/worker.js +128 -18
- package/package.json +6 -2
- package/src/auth.ts +64 -21
- package/src/cli.ts +336 -0
- package/src/durable-object.ts +118 -52
- package/src/index.ts +6 -0
- package/src/pramen.ts +7 -1
- package/src/runtime/acl.ts +25 -5
- package/src/runtime/db.ts +80 -9
- package/src/runtime/ddl.ts +26 -8
- package/src/runtime/dispatch.ts +2 -0
- package/src/runtime/driver.ts +52 -9
- package/src/runtime/migrate.ts +246 -34
- package/src/runtime/outbox.ts +30 -7
- package/src/runtime/queue-consumer.ts +116 -0
- package/src/runtime/queue.ts +155 -0
- package/src/runtime/read-engine.ts +7 -2
- package/src/runtime/schema-diff.ts +137 -23
- package/src/runtime/storage.ts +0 -0
- package/src/sdk/handlers.ts +7 -0
- package/src/worker.ts +162 -19
package/dist/runtime/ddl.d.ts
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
|
-
import type { EntityFields, FieldDef } from "../sdk/schema";
|
|
1
|
+
import type { DefaultValue, EntityFields, FieldDef } from "../sdk/schema";
|
|
2
2
|
export declare const sqlType: (f: FieldDef) => string;
|
|
3
|
+
/** Render a DEFAULT literal. Strings are single-quote-escaped; booleans → 0/1.
|
|
4
|
+
* Exported for the migrator, which reconstructs a column's expected DEFAULT text to
|
|
5
|
+
* compare against the live `PRAGMA table_info.dflt_value`. */
|
|
6
|
+
export declare function defaultLiteral(v: DefaultValue): string;
|
|
7
|
+
/** The SQL text of a column's DEFAULT value (the part after `DEFAULT `), or null when
|
|
8
|
+
* the column declares no default. A raw-SQL `defaultExpr` is returned unquoted (e.g.
|
|
9
|
+
* `datetime('now')`); a literal `default` is rendered via {@link defaultLiteral}. Used
|
|
10
|
+
* by the migrator both to detect a default add/change on an existing column and to
|
|
11
|
+
* COALESCE-backfill a NOT NULL column during a rebuild. */
|
|
12
|
+
export declare function defaultSqlValue(f: FieldDef): string | null;
|
|
3
13
|
export declare function createTableSql(table: string, def: {
|
|
4
14
|
fields: EntityFields;
|
|
5
15
|
}): string;
|
|
@@ -10,7 +20,10 @@ export declare function addColumnSql(name: string, f: FieldDef): string;
|
|
|
10
20
|
/** Index name for a column's unique/index constraint. */
|
|
11
21
|
export declare function indexName(table: string, col: string): string;
|
|
12
22
|
/** CREATE [UNIQUE] INDEX statements for a table's unique/index columns (idempotent
|
|
13
|
-
* via IF NOT EXISTS). Unique wins if a column declares both.
|
|
23
|
+
* via IF NOT EXISTS). Unique wins if a column declares both. `skipCols` omits specific
|
|
24
|
+
* columns — the migrator uses it to avoid emitting a UNIQUE index that would throw
|
|
25
|
+
* (duplicate values present on a column that just gained `unique()`); that delta is
|
|
26
|
+
* reported as skipped instead. */
|
|
14
27
|
export declare function indexStatements(table: string, def: {
|
|
15
28
|
fields: EntityFields;
|
|
16
|
-
}): string[];
|
|
29
|
+
}, skipCols?: ReadonlySet<string>): string[];
|
package/dist/runtime/ddl.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// DDL generation — CREATE TABLE for a new entity and the additive ALTER fragment
|
|
2
2
|
// for a new column. Runs in TS inside the isolate; see runtime/migrate.ts for how
|
|
3
3
|
// these are applied.
|
|
4
|
+
import { quoteIdent } from "./driver";
|
|
4
5
|
// SQLite has no boolean type; store as INTEGER 0/1. json + fileRef + uuid are
|
|
5
6
|
// stored as TEXT. Exported for the migrator, which compares declared column types
|
|
6
7
|
// (and CASTs on a type change).
|
|
@@ -9,8 +10,10 @@ export const sqlType = (f) => f.type === "boolean"
|
|
|
9
10
|
: f.type === "json" || f.type === "fileRef" || f.type === "uuid"
|
|
10
11
|
? "TEXT"
|
|
11
12
|
: f.type.toUpperCase();
|
|
12
|
-
/** Render a DEFAULT literal. Strings are single-quote-escaped; booleans → 0/1.
|
|
13
|
-
|
|
13
|
+
/** Render a DEFAULT literal. Strings are single-quote-escaped; booleans → 0/1.
|
|
14
|
+
* Exported for the migrator, which reconstructs a column's expected DEFAULT text to
|
|
15
|
+
* compare against the live `PRAGMA table_info.dflt_value`. */
|
|
16
|
+
export function defaultLiteral(v) {
|
|
14
17
|
if (v === null)
|
|
15
18
|
return "NULL";
|
|
16
19
|
if (typeof v === "boolean")
|
|
@@ -19,6 +22,18 @@ function defaultLiteral(v) {
|
|
|
19
22
|
return String(v);
|
|
20
23
|
return `'${v.replace(/'/g, "''")}'`;
|
|
21
24
|
}
|
|
25
|
+
/** The SQL text of a column's DEFAULT value (the part after `DEFAULT `), or null when
|
|
26
|
+
* the column declares no default. A raw-SQL `defaultExpr` is returned unquoted (e.g.
|
|
27
|
+
* `datetime('now')`); a literal `default` is rendered via {@link defaultLiteral}. Used
|
|
28
|
+
* by the migrator both to detect a default add/change on an existing column and to
|
|
29
|
+
* COALESCE-backfill a NOT NULL column during a rebuild. */
|
|
30
|
+
export function defaultSqlValue(f) {
|
|
31
|
+
if (f.defaultExpr !== undefined)
|
|
32
|
+
return f.defaultExpr;
|
|
33
|
+
if (f.default !== undefined)
|
|
34
|
+
return defaultLiteral(f.default);
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
22
37
|
/** The ` DEFAULT x` fragment for a column, or "" when it has no default. A raw-SQL
|
|
23
38
|
* `defaultExpr` (e.g. `datetime('now')`) is emitted UNQUOTED; a literal `default` is
|
|
24
39
|
* quote-escaped. UNIQUE/index are NOT inline — they're emitted as separate index
|
|
@@ -32,7 +47,7 @@ function defaultSql(f) {
|
|
|
32
47
|
return f.default !== undefined ? ` DEFAULT ${defaultLiteral(f.default)}` : "";
|
|
33
48
|
}
|
|
34
49
|
function columnSql(name, f) {
|
|
35
|
-
let s = `${name} ${sqlType(f)}`;
|
|
50
|
+
let s = `${quoteIdent(name)} ${sqlType(f)}`;
|
|
36
51
|
if (f.primaryKey)
|
|
37
52
|
s += " PRIMARY KEY";
|
|
38
53
|
if (f.autoIncrement)
|
|
@@ -44,13 +59,13 @@ function columnSql(name, f) {
|
|
|
44
59
|
}
|
|
45
60
|
export function createTableSql(table, def) {
|
|
46
61
|
const cols = Object.entries(def.fields).map(([n, f]) => columnSql(n, f));
|
|
47
|
-
return `CREATE TABLE IF NOT EXISTS ${table} (${cols.join(", ")})`;
|
|
62
|
+
return `CREATE TABLE IF NOT EXISTS ${quoteIdent(table)} (${cols.join(", ")})`;
|
|
48
63
|
}
|
|
49
64
|
/** Column definition for ALTER TABLE ADD COLUMN. No PRIMARY KEY / AUTOINCREMENT.
|
|
50
65
|
* NOT NULL is only emitted alongside a DEFAULT (SQLite can't add a bare NOT NULL to
|
|
51
66
|
* a populated table); a DEFAULT alone backfills existing rows. */
|
|
52
67
|
export function addColumnSql(name, f) {
|
|
53
|
-
let s = `${name} ${sqlType(f)}`;
|
|
68
|
+
let s = `${quoteIdent(name)} ${sqlType(f)}`;
|
|
54
69
|
if (f.notNull && f.default !== undefined)
|
|
55
70
|
s += " NOT NULL";
|
|
56
71
|
s += defaultSql(f);
|
|
@@ -61,14 +76,19 @@ export function indexName(table, col) {
|
|
|
61
76
|
return `pramen_idx_${table}_${col}`;
|
|
62
77
|
}
|
|
63
78
|
/** CREATE [UNIQUE] INDEX statements for a table's unique/index columns (idempotent
|
|
64
|
-
* via IF NOT EXISTS). Unique wins if a column declares both.
|
|
65
|
-
|
|
79
|
+
* via IF NOT EXISTS). Unique wins if a column declares both. `skipCols` omits specific
|
|
80
|
+
* columns — the migrator uses it to avoid emitting a UNIQUE index that would throw
|
|
81
|
+
* (duplicate values present on a column that just gained `unique()`); that delta is
|
|
82
|
+
* reported as skipped instead. */
|
|
83
|
+
export function indexStatements(table, def, skipCols) {
|
|
66
84
|
const out = [];
|
|
67
85
|
for (const [col, f] of Object.entries(def.fields)) {
|
|
68
86
|
if (!f.unique && !f.index)
|
|
69
87
|
continue;
|
|
88
|
+
if (skipCols?.has(col))
|
|
89
|
+
continue;
|
|
70
90
|
const kind = f.unique ? "UNIQUE INDEX" : "INDEX";
|
|
71
|
-
out.push(`CREATE ${kind} IF NOT EXISTS ${indexName(table, col)} ON ${table} (${col})`);
|
|
91
|
+
out.push(`CREATE ${kind} IF NOT EXISTS ${quoteIdent(indexName(table, col))} ON ${quoteIdent(table)} (${quoteIdent(col)})`);
|
|
72
92
|
}
|
|
73
93
|
return out;
|
|
74
94
|
}
|
package/dist/runtime/dispatch.js
CHANGED
|
@@ -12,6 +12,7 @@ import { warmup } from "./acl";
|
|
|
12
12
|
import { BadRequest, Forbidden } from "./errors";
|
|
13
13
|
import { enqueueTask } from "./outbox";
|
|
14
14
|
import { createMail } from "./mail";
|
|
15
|
+
import { createQueue } from "./queue";
|
|
15
16
|
import { authorizeHandler } from "../sdk/handlers";
|
|
16
17
|
/** The `ctx.tasks` facade over the outbox. `onEnqueue` lets the caller count enqueues
|
|
17
18
|
* so it can wake the drainer. */
|
|
@@ -63,6 +64,7 @@ export async function dispatch(handlers, schema, driver, kv, files, env, acl, na
|
|
|
63
64
|
identity: acl.identity,
|
|
64
65
|
tasks: tasksFacade(driver, () => enqueued++),
|
|
65
66
|
mail: createMail(env, kv),
|
|
67
|
+
queue: createQueue(env),
|
|
66
68
|
};
|
|
67
69
|
const result = handler.kind === "query"
|
|
68
70
|
? await handler.run(ctx, parsed)
|
package/dist/runtime/driver.d.ts
CHANGED
|
@@ -9,8 +9,15 @@ export interface Dialect {
|
|
|
9
9
|
/** Coerce a JS value for binding (e.g. boolean → 0/1 on SQLite). */
|
|
10
10
|
encode(v: unknown): unknown;
|
|
11
11
|
}
|
|
12
|
-
/**
|
|
13
|
-
*
|
|
12
|
+
/** Render an identifier as a standard double-quoted name (`"order"`), guarding its
|
|
13
|
+
* shape first. SQLite (DO SQLite + D1) and Postgres all accept double-quoted
|
|
14
|
+
* identifiers, so a column/table named after a reserved word (`order`, `group`, …)
|
|
15
|
+
* is safe and case is preserved. The single source of truth for both dialects and
|
|
16
|
+
* the DDL generator — keep every emitted identifier going through this. */
|
|
17
|
+
export declare function quoteIdent(name: string): string;
|
|
18
|
+
/** SQLite (DO SQLite and D1 both speak this). Double-quoted identifiers (so reserved
|
|
19
|
+
* words like `order` work), `?` placeholders, booleans stored as INTEGER 0/1,
|
|
20
|
+
* RETURNING supported. */
|
|
14
21
|
export declare const sqliteDialect: Dialect;
|
|
15
22
|
/** Postgres (e.g. over Hyperdrive). Double-quoted identifiers preserve case (so
|
|
16
23
|
* `ownerId` doesn't fold to `ownerid`), `$n` placeholders, native booleans,
|
|
@@ -33,13 +40,40 @@ export declare class DoSqliteDriver implements Driver {
|
|
|
33
40
|
exec(sql: string, params: unknown[]): Promise<Row[]>;
|
|
34
41
|
transaction<T>(fn: () => Promise<T>): Promise<T>;
|
|
35
42
|
}
|
|
36
|
-
/**
|
|
37
|
-
* `
|
|
38
|
-
*
|
|
43
|
+
/** How a D1Driver's session is anchored (passed to `db.withSession`):
|
|
44
|
+
* - `"first-primary"` — first query hits the primary (current data), the rest
|
|
45
|
+
* read replicas consistent with the session bookmark. Use
|
|
46
|
+
* for a MUTATION (reads must see current data; writes go
|
|
47
|
+
* to primary anyway).
|
|
48
|
+
* - `"first-unconstrained"` — first query may hit the nearest replica. Use for a QUERY.
|
|
49
|
+
* - a bookmark string — anchor at a prior write's bookmark for read-your-writes
|
|
50
|
+
* (the client carries it forward via a header).
|
|
51
|
+
* A bookmark always wins over a constraint when one is supplied. */
|
|
52
|
+
export type D1SessionStart = "first-primary" | "first-unconstrained" | (string & {});
|
|
53
|
+
/** D1 — SQLite over RPC. Async by nature.
|
|
54
|
+
*
|
|
55
|
+
* Read replicas (Sessions API): every D1Driver opens ONE `db.withSession(start)` and
|
|
56
|
+
* runs all `exec` through it. Writes in a session always land on the primary; the
|
|
57
|
+
* `start` only chooses where the FIRST read may begin. The session maintains a
|
|
58
|
+
* bookmark (`getBookmark()`) so later reads are sequentially consistent with earlier
|
|
59
|
+
* writes — read-your-writes when the bookmark is threaded across requests.
|
|
60
|
+
*
|
|
61
|
+
* ATOMICITY LIMIT (intentional): D1 has NO interactive transactions — a session can't
|
|
62
|
+
* read mid-`batch()`, and pramen mutations interleave reads + writes + RETURNING +
|
|
63
|
+
* trigger-into-outbox inside one `transaction()`. So `transaction(fn) = fn()`: each
|
|
64
|
+
* statement auto-commits on its own, and a multi-statement mutation does NOT roll back
|
|
65
|
+
* on throw the way it does on a DO. Single-statement mutations are atomic; anything
|
|
66
|
+
* multi-statement is not. Use the DO store when you need atomic mutations. */
|
|
39
67
|
export declare class D1Driver implements Driver {
|
|
40
|
-
private readonly db;
|
|
41
68
|
readonly dialect: Dialect;
|
|
42
|
-
|
|
69
|
+
private readonly session;
|
|
70
|
+
constructor(db: D1Database, opts?: {
|
|
71
|
+
start?: D1SessionStart;
|
|
72
|
+
});
|
|
43
73
|
exec(sql: string, params: unknown[]): Promise<Row[]>;
|
|
74
|
+
/** The session's latest bookmark (null before any query). Threaded back to the client
|
|
75
|
+
* via the `x-pramen-d1-bookmark` response header so a subsequent request can anchor a
|
|
76
|
+
* fresh session at it and read its own writes. */
|
|
77
|
+
getBookmark(): string | null;
|
|
44
78
|
transaction<T>(fn: () => Promise<T>): Promise<T>;
|
|
45
79
|
}
|
package/dist/runtime/driver.js
CHANGED
|
@@ -18,10 +18,19 @@ function checkIdent(name) {
|
|
|
18
18
|
throw new Error(`invalid identifier: ${name}`);
|
|
19
19
|
return name;
|
|
20
20
|
}
|
|
21
|
-
/**
|
|
22
|
-
*
|
|
21
|
+
/** Render an identifier as a standard double-quoted name (`"order"`), guarding its
|
|
22
|
+
* shape first. SQLite (DO SQLite + D1) and Postgres all accept double-quoted
|
|
23
|
+
* identifiers, so a column/table named after a reserved word (`order`, `group`, …)
|
|
24
|
+
* is safe and case is preserved. The single source of truth for both dialects and
|
|
25
|
+
* the DDL generator — keep every emitted identifier going through this. */
|
|
26
|
+
export function quoteIdent(name) {
|
|
27
|
+
return `"${checkIdent(name)}"`;
|
|
28
|
+
}
|
|
29
|
+
/** SQLite (DO SQLite and D1 both speak this). Double-quoted identifiers (so reserved
|
|
30
|
+
* words like `order` work), `?` placeholders, booleans stored as INTEGER 0/1,
|
|
31
|
+
* RETURNING supported. */
|
|
23
32
|
export const sqliteDialect = {
|
|
24
|
-
id:
|
|
33
|
+
id: quoteIdent,
|
|
25
34
|
placeholder: () => "?",
|
|
26
35
|
returning: true,
|
|
27
36
|
encode: (v) => (typeof v === "boolean" ? (v ? 1 : 0) : v),
|
|
@@ -30,7 +39,7 @@ export const sqliteDialect = {
|
|
|
30
39
|
* `ownerId` doesn't fold to `ownerid`), `$n` placeholders, native booleans,
|
|
31
40
|
* RETURNING supported. */
|
|
32
41
|
export const postgresDialect = {
|
|
33
|
-
id:
|
|
42
|
+
id: quoteIdent,
|
|
34
43
|
placeholder: (n) => `$${n}`,
|
|
35
44
|
returning: true,
|
|
36
45
|
encode: (v) => v, // the pg driver handles type encoding
|
|
@@ -50,20 +59,38 @@ export class DoSqliteDriver {
|
|
|
50
59
|
return this.storage.transaction(fn);
|
|
51
60
|
}
|
|
52
61
|
}
|
|
53
|
-
/** D1 — SQLite over RPC. Async by nature.
|
|
54
|
-
*
|
|
55
|
-
*
|
|
62
|
+
/** D1 — SQLite over RPC. Async by nature.
|
|
63
|
+
*
|
|
64
|
+
* Read replicas (Sessions API): every D1Driver opens ONE `db.withSession(start)` and
|
|
65
|
+
* runs all `exec` through it. Writes in a session always land on the primary; the
|
|
66
|
+
* `start` only chooses where the FIRST read may begin. The session maintains a
|
|
67
|
+
* bookmark (`getBookmark()`) so later reads are sequentially consistent with earlier
|
|
68
|
+
* writes — read-your-writes when the bookmark is threaded across requests.
|
|
69
|
+
*
|
|
70
|
+
* ATOMICITY LIMIT (intentional): D1 has NO interactive transactions — a session can't
|
|
71
|
+
* read mid-`batch()`, and pramen mutations interleave reads + writes + RETURNING +
|
|
72
|
+
* trigger-into-outbox inside one `transaction()`. So `transaction(fn) = fn()`: each
|
|
73
|
+
* statement auto-commits on its own, and a multi-statement mutation does NOT roll back
|
|
74
|
+
* on throw the way it does on a DO. Single-statement mutations are atomic; anything
|
|
75
|
+
* multi-statement is not. Use the DO store when you need atomic mutations. */
|
|
56
76
|
export class D1Driver {
|
|
57
|
-
db;
|
|
58
77
|
dialect = sqliteDialect;
|
|
59
|
-
|
|
60
|
-
|
|
78
|
+
session;
|
|
79
|
+
constructor(db, opts) {
|
|
80
|
+
this.session = db.withSession(opts?.start ?? "first-unconstrained");
|
|
61
81
|
}
|
|
62
82
|
async exec(sql, params) {
|
|
63
|
-
const stmt = params.length ? this.
|
|
83
|
+
const stmt = params.length ? this.session.prepare(sql).bind(...params) : this.session.prepare(sql);
|
|
64
84
|
const { results } = await stmt.all();
|
|
65
85
|
return results ?? [];
|
|
66
86
|
}
|
|
87
|
+
/** The session's latest bookmark (null before any query). Threaded back to the client
|
|
88
|
+
* via the `x-pramen-d1-bookmark` response header so a subsequent request can anchor a
|
|
89
|
+
* fresh session at it and read its own writes. */
|
|
90
|
+
getBookmark() {
|
|
91
|
+
return this.session.getBookmark();
|
|
92
|
+
}
|
|
93
|
+
// D1 has no interactive/atomic transactions — see the class doc. Run `fn` as-is.
|
|
67
94
|
transaction(fn) {
|
|
68
95
|
return fn();
|
|
69
96
|
}
|
package/dist/runtime/migrate.js
CHANGED
|
@@ -5,10 +5,30 @@
|
|
|
5
5
|
// Reconciles the live store with the declared schema in two passes:
|
|
6
6
|
// 1. additive (no data loss): missing table -> CREATE TABLE; missing column ->
|
|
7
7
|
// ALTER TABLE ADD COLUMN (nullable).
|
|
8
|
-
// 2.
|
|
9
|
-
// change is applied,
|
|
10
|
-
//
|
|
11
|
-
//
|
|
8
|
+
// 2. reconcile existing columns + drop obsolete ones. A live column the schema no
|
|
9
|
+
// longer declares is DROPPED, a type change is applied, a `renamedFrom` column is
|
|
10
|
+
// renamed, and a MODIFIER change on an existing column (NOT NULL / DEFAULT /
|
|
11
|
+
// PRIMARY KEY) is enacted — all via the standard SQLite table-rebuild (create new,
|
|
12
|
+
// copy, drop old, rename); UNIQUE is reconciled with a CREATE/DROP INDEX. Each
|
|
13
|
+
// change is classified SAFE (loses no data — applied always: a DEFAULT add/change,
|
|
14
|
+
// dropping a constraint, adding NOT NULL when a backfill/default covers it, adding
|
|
15
|
+
// UNIQUE with no duplicates) or DESTRUCTIVE (GATED behind PRAMEN_ALLOW_DESTRUCTIVE,
|
|
16
|
+
// off by default: a drop, a type change, a rename, a PRIMARY KEY change, adding
|
|
17
|
+
// NOT NULL over NULL rows with no default). Adding UNIQUE over duplicate values is
|
|
18
|
+
// always SKIPPED (the index can't build). `hidden()`/`generated()` are ORM-only —
|
|
19
|
+
// no physical column change, so they don't appear here.
|
|
20
|
+
//
|
|
21
|
+
// The guiding invariant: the schema hash is recorded ONLY when the store fully matches
|
|
22
|
+
// the schema. Any detected change that is SKIPPED (destructive-gated, a UNIQUE-over-
|
|
23
|
+
// duplicates, or a partition MOVE — see below) leaves the hash UNWRITTEN, so `schema
|
|
24
|
+
// status` keeps reporting drift and a later opt-in / data-fixed deploy retries. This is
|
|
25
|
+
// what stops a modifier change (e.g. an unenforced NOT NULL) from silently diverging the
|
|
26
|
+
// store from the schema while the hash claims "in sync". Local dev sets the flag on; a
|
|
27
|
+
// bad deploy CAN then lose data (WIP, no backward-compat).
|
|
28
|
+
//
|
|
29
|
+
// A partition MOVE (an entity reassigned to a different Durable Object) is never auto-
|
|
30
|
+
// applied — the data can't cross DOs — so it's detected and reported as a skipped manual
|
|
31
|
+
// migration (hash withheld), leaving the source DO's data intact.
|
|
12
32
|
//
|
|
13
33
|
// A schema hash in the internal `_pramen_meta` table lets an unchanged schema skip
|
|
14
34
|
// introspection entirely on warm boots. The live table (PRAGMA) is the ground
|
|
@@ -17,9 +37,10 @@
|
|
|
17
37
|
// ADD COLUMN is always nullable (SQLite can't add NOT NULL to a populated table).
|
|
18
38
|
// A rename can't be inferred from a diff (a removed + added column is ambiguous),
|
|
19
39
|
// so it must be declared with `renamedFrom`; otherwise it is applied as drop+add.
|
|
20
|
-
import { addColumnSql, createTableSql, indexStatements, sqlType } from "./ddl";
|
|
40
|
+
import { addColumnSql, createTableSql, defaultSqlValue, indexName, indexStatements, sqlType } from "./ddl";
|
|
21
41
|
import { digest } from "./digest";
|
|
22
|
-
import {
|
|
42
|
+
import { quoteIdent } from "./driver";
|
|
43
|
+
import { entitiesInPartition, partitionOf, validateSchema } from "../sdk/schema";
|
|
23
44
|
/** Internal bookkeeping tables the migrator must never touch — pramen's own, SQLite's,
|
|
24
45
|
* and the substrate's (D1 keeps `_cf_*` / `d1_*` tables in sqlite_master and forbids
|
|
25
46
|
* dropping them). Matched case-insensitively. */
|
|
@@ -31,11 +52,6 @@ function isInternalTable(name) {
|
|
|
31
52
|
n.startsWith("_cf_") ||
|
|
32
53
|
n.startsWith("d1_"));
|
|
33
54
|
}
|
|
34
|
-
function ident(name) {
|
|
35
|
-
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name))
|
|
36
|
-
throw new Error(`invalid identifier: ${name}`);
|
|
37
|
-
return name;
|
|
38
|
-
}
|
|
39
55
|
export function schemaHash(schema) {
|
|
40
56
|
const canon = {};
|
|
41
57
|
for (const [table, def] of Object.entries(schema))
|
|
@@ -45,9 +61,78 @@ export function schemaHash(schema) {
|
|
|
45
61
|
/** Live columns of a table -> their declared SQL type (uppercased). Empty if the
|
|
46
62
|
* table doesn't exist. */
|
|
47
63
|
async function tableColumns(driver, table) {
|
|
48
|
-
const rows = (await driver.exec(`PRAGMA table_info(${
|
|
64
|
+
const rows = (await driver.exec(`PRAGMA table_info(${quoteIdent(table)})`, []));
|
|
49
65
|
return new Map(rows.map((r) => [r.name, (r.type || "").toUpperCase()]));
|
|
50
66
|
}
|
|
67
|
+
/** Full live column info (type + modifiers) via PRAGMA table_info. Empty if absent. */
|
|
68
|
+
async function liveColumnInfo(driver, table) {
|
|
69
|
+
const rows = (await driver.exec(`PRAGMA table_info(${quoteIdent(table)})`, []));
|
|
70
|
+
const out = new Map();
|
|
71
|
+
for (const r of rows) {
|
|
72
|
+
out.set(r.name, {
|
|
73
|
+
type: (r.type || "").toUpperCase(),
|
|
74
|
+
notNull: r.notnull === 1,
|
|
75
|
+
pk: r.pk > 0,
|
|
76
|
+
default: r.dflt_value ?? null,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
/** The columns backed by a single-column UNIQUE index (a `unique()` constraint). Reads
|
|
82
|
+
* PRAGMA index_list + index_info; multi-column indexes are ignored (pramen only emits
|
|
83
|
+
* single-column ones). */
|
|
84
|
+
async function liveUniqueColumns(driver, table) {
|
|
85
|
+
const idx = (await driver.exec(`PRAGMA index_list(${quoteIdent(table)})`, []));
|
|
86
|
+
const out = new Set();
|
|
87
|
+
for (const i of idx) {
|
|
88
|
+
if (i.unique !== 1)
|
|
89
|
+
continue;
|
|
90
|
+
const cols = (await driver.exec(`PRAGMA index_info(${quoteIdent(i.name)})`, []));
|
|
91
|
+
if (cols.length === 1 && cols[0]?.name)
|
|
92
|
+
out.add(cols[0].name);
|
|
93
|
+
}
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
/** Does the column currently hold any NULL? (Adding NOT NULL to such a column is
|
|
97
|
+
* unsafe without a backfill default.) */
|
|
98
|
+
async function columnHasNulls(driver, table, col) {
|
|
99
|
+
const rows = await driver.exec(`SELECT 1 FROM ${quoteIdent(table)} WHERE ${quoteIdent(col)} IS NULL LIMIT 1`, []);
|
|
100
|
+
return rows.length > 0;
|
|
101
|
+
}
|
|
102
|
+
/** Does the column hold a duplicate non-NULL value? (Adding UNIQUE to such a column
|
|
103
|
+
* can't build the index.) NULLs are never "equal" in SQLite, so they're excluded. */
|
|
104
|
+
async function columnHasDuplicates(driver, table, col) {
|
|
105
|
+
const rows = await driver.exec(`SELECT ${quoteIdent(col)} FROM ${quoteIdent(table)} WHERE ${quoteIdent(col)} IS NOT NULL GROUP BY ${quoteIdent(col)} HAVING COUNT(*) > 1 LIMIT 1`, []);
|
|
106
|
+
return rows.length > 0;
|
|
107
|
+
}
|
|
108
|
+
/** Normalize a DEFAULT's SQL text for comparison: trim, and strip balanced outer
|
|
109
|
+
* parens (SQLite reports an expr default with or without the wrapping parens the DDL
|
|
110
|
+
* emitted — `(datetime('now'))` vs `datetime('now')` — depending on the engine, so the
|
|
111
|
+
* comparison must not depend on them). */
|
|
112
|
+
function normalizeDefault(s) {
|
|
113
|
+
if (s == null)
|
|
114
|
+
return null;
|
|
115
|
+
let t = s.trim();
|
|
116
|
+
while (t.length >= 2 && t[0] === "(" && t[t.length - 1] === ")" && outerParensBalanced(t)) {
|
|
117
|
+
t = t.slice(1, -1).trim();
|
|
118
|
+
}
|
|
119
|
+
return t;
|
|
120
|
+
}
|
|
121
|
+
/** Does the leading `(` in `t` match the trailing `)` (i.e. is the whole string wrapped
|
|
122
|
+
* in one paren group)? Prevents stripping `(a) + (b)`. */
|
|
123
|
+
function outerParensBalanced(t) {
|
|
124
|
+
let depth = 0;
|
|
125
|
+
for (let i = 0; i < t.length; i++) {
|
|
126
|
+
if (t[i] === "(")
|
|
127
|
+
depth++;
|
|
128
|
+
else if (t[i] === ")") {
|
|
129
|
+
depth--;
|
|
130
|
+
if (depth === 0 && i !== t.length - 1)
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return depth === 0;
|
|
135
|
+
}
|
|
51
136
|
async function readMeta(driver, key) {
|
|
52
137
|
const rows = (await driver.exec(`SELECT value FROM _pramen_meta WHERE key = ?`, [key]));
|
|
53
138
|
return rows[0]?.value;
|
|
@@ -60,7 +145,7 @@ async function writeMeta(driver, key, value) {
|
|
|
60
145
|
* type change; brand-new columns left NULL), drop the old table, rename the temp. */
|
|
61
146
|
async function rebuildTable(driver, table, def, live) {
|
|
62
147
|
const tmp = `__pramen_rebuild_${table}`;
|
|
63
|
-
await driver.exec(`DROP TABLE IF EXISTS ${
|
|
148
|
+
await driver.exec(`DROP TABLE IF EXISTS ${quoteIdent(tmp)}`, []);
|
|
64
149
|
await driver.exec(createTableSql(tmp, def), []);
|
|
65
150
|
const destCols = [];
|
|
66
151
|
const srcExprs = [];
|
|
@@ -70,14 +155,22 @@ async function rebuildTable(driver, table, def, live) {
|
|
|
70
155
|
if (!src)
|
|
71
156
|
continue; // brand-new column with no source -> leave NULL
|
|
72
157
|
const target = sqlType(f);
|
|
73
|
-
|
|
74
|
-
|
|
158
|
+
let expr = live.get(src) === target ? quoteIdent(src) : `CAST(${quoteIdent(src)} AS ${target})`;
|
|
159
|
+
// Backfill NULLs when the target is NOT NULL and carries a default — makes adding
|
|
160
|
+
// NOT NULL to a column with NULL rows safe (the copy fills them from the default),
|
|
161
|
+
// instead of the INSERT failing the new NOT NULL constraint.
|
|
162
|
+
const notNull = !!f.notNull || !!f.primaryKey;
|
|
163
|
+
const dflt = defaultSqlValue(f);
|
|
164
|
+
if (notNull && dflt !== null)
|
|
165
|
+
expr = `COALESCE(${expr}, ${dflt})`;
|
|
166
|
+
destCols.push(quoteIdent(name));
|
|
167
|
+
srcExprs.push(expr);
|
|
75
168
|
}
|
|
76
169
|
if (destCols.length > 0) {
|
|
77
|
-
await driver.exec(`INSERT INTO ${
|
|
170
|
+
await driver.exec(`INSERT INTO ${quoteIdent(tmp)} (${destCols.join(", ")}) SELECT ${srcExprs.join(", ")} FROM ${quoteIdent(table)}`, []);
|
|
78
171
|
}
|
|
79
|
-
await driver.exec(`DROP TABLE ${
|
|
80
|
-
await driver.exec(`ALTER TABLE ${
|
|
172
|
+
await driver.exec(`DROP TABLE ${quoteIdent(table)}`, []);
|
|
173
|
+
await driver.exec(`ALTER TABLE ${quoteIdent(tmp)} RENAME TO ${quoteIdent(table)}`, []);
|
|
81
174
|
}
|
|
82
175
|
export async function migrate(driver, schema, opts = {}) {
|
|
83
176
|
// Static schema invariants (relation targets exist, no cross-partition relations) —
|
|
@@ -113,6 +206,9 @@ export async function migrate(driver, schema, opts = {}) {
|
|
|
113
206
|
const rebuilt = [];
|
|
114
207
|
const droppedTables = [];
|
|
115
208
|
const skipped = [];
|
|
209
|
+
// Per-table: columns whose new `unique()` can't be indexed (duplicate values) — the
|
|
210
|
+
// index pass must skip them so it doesn't throw. They're already reported in `skipped`.
|
|
211
|
+
const uniqueIndexSkip = new Map();
|
|
116
212
|
for (const [table, def] of entries) {
|
|
117
213
|
const existing = await tableColumns(driver, table);
|
|
118
214
|
if (existing.size === 0) {
|
|
@@ -133,12 +229,16 @@ export async function migrate(driver, schema, opts = {}) {
|
|
|
133
229
|
needsAdditiveRebuild = true;
|
|
134
230
|
continue;
|
|
135
231
|
}
|
|
136
|
-
await driver.exec(`ALTER TABLE ${
|
|
232
|
+
await driver.exec(`ALTER TABLE ${quoteIdent(table)} ADD COLUMN ${addColumnSql(name, field)}`, []);
|
|
137
233
|
added.push(`${table}.${name}`);
|
|
138
234
|
}
|
|
139
|
-
// Pass 2 —
|
|
140
|
-
// column changed type,
|
|
141
|
-
|
|
235
|
+
// Pass 2 — reconcile existing columns against the schema. Rebuild the table when a
|
|
236
|
+
// live column must be dropped, a declared column changed type, a rename hint points
|
|
237
|
+
// at an existing live column, or a MODIFIER changed on an existing column (NOT NULL,
|
|
238
|
+
// DEFAULT, PRIMARY KEY). UNIQUE is reconciled with an index (create/drop), no rebuild.
|
|
239
|
+
const liveInfo = await liveColumnInfo(driver, table); // re-read (includes additively-added columns)
|
|
240
|
+
const liveUnique = await liveUniqueColumns(driver, table);
|
|
241
|
+
const live = new Map([...liveInfo].map(([name, info]) => [name, info.type])); // name -> type
|
|
142
242
|
const desired = new Set(Object.keys(def.fields));
|
|
143
243
|
const renamedSources = new Set();
|
|
144
244
|
for (const f of Object.values(def.fields)) {
|
|
@@ -148,24 +248,112 @@ export async function migrate(driver, schema, opts = {}) {
|
|
|
148
248
|
}
|
|
149
249
|
const needsDrop = [...live.keys()].some((c) => !desired.has(c) && !renamedSources.has(c));
|
|
150
250
|
const needsTypeChange = Object.entries(def.fields).some(([n, f]) => live.has(n) && live.get(n) !== sqlType(f));
|
|
151
|
-
|
|
251
|
+
// Modifier reconciliation on existing (same-named) columns. A change to NOT NULL /
|
|
252
|
+
// DEFAULT / PRIMARY KEY is enacted by a table rebuild (which reconstructs the column
|
|
253
|
+
// to its exact declared shape); UNIQUE is an index op. Classify each as either a
|
|
254
|
+
// SAFE change (loses no data — applied always) or a DESTRUCTIVE one (gated behind
|
|
255
|
+
// allowDestructive). `hidden()`/`generated()` are ORM-only (no physical column
|
|
256
|
+
// change), so they never appear here.
|
|
257
|
+
let modifierRebuildSafe = false; // default add/change/remove, notNull widen, safe notNull add
|
|
258
|
+
let modifierRebuildDestructive = false; // notNull add over NULL rows w/o default, PK change
|
|
259
|
+
const destructiveReasons = [];
|
|
260
|
+
const dropUniqueCols = []; // `unique()` removed -> drop the managed index
|
|
261
|
+
for (const [name, field] of Object.entries(def.fields)) {
|
|
262
|
+
const info = liveInfo.get(name);
|
|
263
|
+
if (!info)
|
|
264
|
+
continue; // new column (Pass 1) or a rename target — not an existing column
|
|
265
|
+
const f = field;
|
|
266
|
+
const fieldPk = !!f.primaryKey;
|
|
267
|
+
const fieldNotNull = !!f.notNull || fieldPk;
|
|
268
|
+
// DEFAULT add / change / remove — a rebuild backfills existing rows and applies the
|
|
269
|
+
// new default going forward; no data loss.
|
|
270
|
+
if (normalizeDefault(defaultSqlValue(f)) !== normalizeDefault(info.default))
|
|
271
|
+
modifierRebuildSafe = true;
|
|
272
|
+
// NOT NULL — PRAGMA reports notnull=0 for a PRIMARY KEY column, so only compare on
|
|
273
|
+
// non-PK columns (PK-ness is compared separately below).
|
|
274
|
+
if (!fieldPk && !info.pk) {
|
|
275
|
+
if (fieldNotNull && !info.notNull) {
|
|
276
|
+
const backfillable = defaultSqlValue(f) !== null;
|
|
277
|
+
if (!backfillable && (await columnHasNulls(driver, table, name))) {
|
|
278
|
+
modifierRebuildDestructive = true;
|
|
279
|
+
destructiveReasons.push(`NOT NULL ${name} (NULL rows, no default)`);
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
modifierRebuildSafe = true; // no NULLs, or backfilled from the default
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
else if (!fieldNotNull && info.notNull) {
|
|
286
|
+
modifierRebuildSafe = true; // dropping NOT NULL only widens — safe
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
// PRIMARY KEY change — reshapes the table's key; treat as destructive.
|
|
290
|
+
if (fieldPk !== info.pk) {
|
|
291
|
+
modifierRebuildDestructive = true;
|
|
292
|
+
destructiveReasons.push(`PRIMARY KEY ${name}`);
|
|
293
|
+
}
|
|
294
|
+
// UNIQUE — reconciled with an index (create/drop), not a rebuild.
|
|
295
|
+
const fieldUnique = !!f.unique;
|
|
296
|
+
if (fieldUnique && !liveUnique.has(name)) {
|
|
297
|
+
// Added: safe only if no duplicate values exist; otherwise the index can't build.
|
|
298
|
+
if (await columnHasDuplicates(driver, table, name)) {
|
|
299
|
+
(uniqueIndexSkip.get(table) ?? uniqueIndexSkip.set(table, new Set()).get(table)).add(name);
|
|
300
|
+
skipped.push(`add UNIQUE ${table}.${name} (duplicate values present)`);
|
|
301
|
+
}
|
|
302
|
+
// else: the index pass creates it below (no rebuild needed).
|
|
303
|
+
}
|
|
304
|
+
else if (!fieldUnique && liveUnique.has(name)) {
|
|
305
|
+
dropUniqueCols.push(name); // drop the managed unique index (safe — no data loss)
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
const destructive = needsDrop || needsTypeChange || renamedSources.size > 0 || modifierRebuildDestructive;
|
|
152
309
|
if (destructive && !allowDestructive) {
|
|
153
|
-
// The destructive part is gated off — skip the whole rebuild (any pending
|
|
154
|
-
//
|
|
155
|
-
|
|
310
|
+
// The destructive part is gated off — skip the whole rebuild (any pending safe
|
|
311
|
+
// rebuild for this table waits until destructive migrations are allowed).
|
|
312
|
+
const reasons = [
|
|
313
|
+
...(needsDrop ? ["drop"] : []),
|
|
314
|
+
...(needsTypeChange ? ["type-change"] : []),
|
|
315
|
+
...(renamedSources.size > 0 ? ["rename"] : []),
|
|
316
|
+
...destructiveReasons,
|
|
317
|
+
];
|
|
318
|
+
skipped.push(`rebuild ${table} (${reasons.join(", ")})`);
|
|
156
319
|
}
|
|
157
|
-
else if (destructive || needsAdditiveRebuild) {
|
|
158
|
-
//
|
|
159
|
-
// it loses no data.
|
|
320
|
+
else if (destructive || needsAdditiveRebuild || modifierRebuildSafe) {
|
|
321
|
+
// A safe rebuild (expr-default column, default/notNull modifier change) needs no
|
|
322
|
+
// permission — it loses no data.
|
|
160
323
|
await rebuildTable(driver, table, def, live);
|
|
161
324
|
rebuilt.push(table);
|
|
162
325
|
}
|
|
326
|
+
// Drop the managed unique index for a column that no longer declares `unique()`. A
|
|
327
|
+
// rebuild already dropped every index (and the index pass won't recreate this one),
|
|
328
|
+
// so this only matters when no rebuild ran — DROP INDEX IF EXISTS is a safe no-op
|
|
329
|
+
// otherwise. No data loss either way.
|
|
330
|
+
for (const col of dropUniqueCols) {
|
|
331
|
+
await driver.exec(`DROP INDEX IF EXISTS ${quoteIdent(indexName(table, col))}`, []);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
// A partition MOVE — an entity that was applied in THIS partition before but the
|
|
335
|
+
// current schema assigns to a DIFFERENT partition — is NOT auto-migratable: the data
|
|
336
|
+
// lives in this DO's SQLite and boot migration can't move it across DOs. Detect it
|
|
337
|
+
// (scoped path only; the unscoped/single-store path never strands data), report it as
|
|
338
|
+
// a skipped manual migration, and leave the table in place so its data is preserved
|
|
339
|
+
// for a hand-run migration. Leaving it in `skipped` also withholds the hash.
|
|
340
|
+
if (opts.partition !== undefined) {
|
|
341
|
+
const prevRaw = await readMeta(driver, tablesKey);
|
|
342
|
+
if (prevRaw) {
|
|
343
|
+
const prevApplied = JSON.parse(prevRaw);
|
|
344
|
+
for (const t of Object.keys(prevApplied)) {
|
|
345
|
+
if (!inScope.has(t) && t in schema && partitionOf(schema, t) !== opts.partition) {
|
|
346
|
+
skipped.push(`move partition ${t} (${opts.partition} → ${partitionOf(schema, t)}) — data stays in this DO; manual cross-DO migration required`);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
163
350
|
}
|
|
164
351
|
// Ensure unique/index declarations (idempotent, via IF NOT EXISTS). Indexes can be
|
|
165
352
|
// added to an existing table without a rebuild; a stale index from a removed
|
|
166
|
-
// declaration is
|
|
353
|
+
// declaration is dropped above. A column whose new `unique()` has duplicate values is
|
|
354
|
+
// skipped (reported above) so this doesn't throw.
|
|
167
355
|
for (const [table, def] of entries) {
|
|
168
|
-
for (const stmt of indexStatements(table, def))
|
|
356
|
+
for (const stmt of indexStatements(table, def, uniqueIndexSkip.get(table)))
|
|
169
357
|
await driver.exec(stmt, []);
|
|
170
358
|
}
|
|
171
359
|
// Drop tables the schema no longer declares (internal bookkeeping tables skipped).
|
|
@@ -181,7 +369,7 @@ export async function migrate(driver, schema, opts = {}) {
|
|
|
181
369
|
if (isInternalTable(name) || inScope.has(name) || otherPartitionTables.has(name))
|
|
182
370
|
continue;
|
|
183
371
|
if (allowDestructive) {
|
|
184
|
-
await driver.exec(`DROP TABLE ${
|
|
372
|
+
await driver.exec(`DROP TABLE ${quoteIdent(name)}`, []);
|
|
185
373
|
droppedTables.push(name);
|
|
186
374
|
}
|
|
187
375
|
else {
|
|
@@ -199,7 +387,8 @@ export async function migrate(driver, schema, opts = {}) {
|
|
|
199
387
|
await writeMeta(driver, tablesKey, tablesValue());
|
|
200
388
|
}
|
|
201
389
|
else {
|
|
202
|
-
console.warn(`pramen: ${skipped.length}
|
|
390
|
+
console.warn(`pramen: ${skipped.length} migration(s) skipped, schema hash left unwritten (a gated change needs ` +
|
|
391
|
+
`PRAMEN_ALLOW_DESTRUCTIVE=true; a UNIQUE-over-duplicates or partition move needs a manual fix): ${skipped.join("; ")}`);
|
|
203
392
|
}
|
|
204
393
|
return { changed: true, created, added, rebuilt, droppedTables, skipped };
|
|
205
394
|
}
|