@mason-data/cli 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +268 -2
- package/dist/src/api/error.js +18 -0
- package/dist/src/api/postgres/catalog-worker.js +92 -0
- package/dist/src/api/postgres/catalog.js +477 -0
- package/dist/src/api/postgres/encode.js +541 -0
- package/dist/src/api/postgres/explain.js +143 -0
- package/dist/src/api/postgres/listener.js +60 -0
- package/dist/src/api/postgres/parser-worker.js +58 -0
- package/dist/src/api/postgres/parser.js +121 -0
- package/dist/src/api/postgres/plain.js +33 -0
- package/dist/src/api/postgres/protocol.js +404 -0
- package/dist/src/api/postgres/schema.js +54 -0
- package/dist/src/api/postgres/server.js +1771 -0
- package/dist/src/api/postgres/translate.js +248 -0
- package/dist/src/api/request-log.js +50 -0
- package/dist/src/cdc/pg-wire.js +588 -0
- package/dist/src/cdc/pgoutput.js +107 -0
- package/dist/src/cdc/poll.js +67 -0
- package/dist/src/cdc/source.js +65 -0
- package/dist/src/cli/background.js +198 -0
- package/dist/src/cli/command.js +173 -0
- package/dist/src/cli/commands.js +1277 -0
- package/dist/src/cli/control.js +36 -0
- package/dist/src/cli/demo.js +149 -0
- package/dist/src/cli/dev.js +669 -0
- package/dist/src/cli/doctor.js +260 -0
- package/dist/src/cli/guides.js +158 -0
- package/dist/src/cli/init.js +366 -0
- package/dist/src/cli/machine.js +162 -0
- package/dist/src/cli/main.js +83 -0
- package/dist/src/cli/mcp.js +726 -0
- package/dist/src/cli/pg-tls.js +100 -0
- package/dist/src/cli/pg-url.js +52 -0
- package/dist/src/cli/pin.js +36 -0
- package/dist/src/cli/ports.js +118 -0
- package/dist/src/cli/preflight.js +347 -0
- package/dist/src/cli/project.js +129 -0
- package/dist/src/cli/readiness.js +76 -0
- package/dist/src/cli/registry.js +220 -0
- package/dist/src/cli/replication-checks.js +120 -0
- package/dist/src/cli/replication-mode.js +148 -0
- package/dist/src/cli/run-server.js +311 -0
- package/dist/src/cli/runs.js +120 -0
- package/dist/src/cli/security.js +86 -0
- package/dist/src/cli/self-signed-cert.js +64 -0
- package/dist/src/cli/serve.js +301 -0
- package/dist/src/cli/source-edit.js +116 -0
- package/dist/src/cli/sources.js +564 -0
- package/dist/src/cli/state.js +58 -0
- package/dist/src/cli/style.js +187 -0
- package/dist/src/cli/supervise.js +248 -0
- package/dist/src/cli/sync.js +94 -0
- package/dist/src/cli/ui.js +107 -0
- package/dist/src/cli/user.js +229 -0
- package/dist/src/cli/version.js +23 -0
- package/dist/src/demo/seed.js +237 -0
- package/dist/src/demo/source.js +655 -0
- package/dist/src/duck/analyze.js +113 -0
- package/dist/src/duck/cancel.js +49 -0
- package/dist/src/duck/distinct-key.js +228 -0
- package/dist/src/duck/executor-worker.js +268 -0
- package/dist/src/duck/executor.js +567 -0
- package/dist/src/duck/explain.js +238 -0
- package/dist/src/duck/functions.js +819 -0
- package/dist/src/duck/gaps.js +94 -0
- package/dist/src/duck/jsonb-order.js +175 -0
- package/dist/src/duck/jsonb.js +188 -0
- package/dist/src/duck/kept-route.js +299 -0
- package/dist/src/duck/kept.js +964 -0
- package/dist/src/duck/late.js +221 -0
- package/dist/src/duck/libc-order.js +190 -0
- package/dist/src/duck/pg-eval-worker.js +34 -0
- package/dist/src/duck/pg-eval.js +59 -0
- package/dist/src/duck/pgtree.js +378 -0
- package/dist/src/duck/plan.js +259 -0
- package/dist/src/duck/publication.js +207 -0
- package/dist/src/duck/requests.js +209 -0
- package/dist/src/duck/resolved.js +248 -0
- package/dist/src/duck/rewrite.js +5101 -0
- package/dist/src/duck/store.js +907 -0
- package/dist/src/duck/stream.js +518 -0
- package/dist/src/duck/sync.js +2929 -0
- package/dist/src/duck/to-char.js +482 -0
- package/dist/src/duck/token.js +31 -0
- package/dist/src/duck/types.js +427 -0
- package/dist/src/duck/users.js +231 -0
- package/dist/src/duck/verified.js +1001 -0
- package/dist/src/duck/views.js +150 -0
- package/dist/src/lib/database-url.js +74 -0
- package/dist/src/lib/did-you-mean.js +43 -0
- package/dist/src/lib/dotenv.js +72 -0
- package/dist/src/lib/exit-when-written.js +20 -0
- package/dist/src/lib/float-equal.js +69 -0
- package/dist/src/lib/http-json.js +22 -0
- package/dist/src/lib/log.js +161 -0
- package/dist/src/lib/pg-numeric.js +489 -0
- package/dist/src/lib/port.js +22 -0
- package/dist/src/lib/project-file.js +49 -0
- package/dist/src/lib/project-files.js +43 -0
- package/dist/src/lib/project-version.js +76 -0
- package/dist/src/lib/runnable.js +14 -0
- package/dist/src/lib/shell-word.js +4 -0
- package/dist/src/lib/timezone.js +237 -0
- package/dist/src/lib/zone-names.js +114 -0
- package/dist/src/web/app.css +76 -0
- package/dist/src/web/app.js +491 -0
- package/dist/src/web/index.html +44 -0
- package/package.json +22 -4
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE CATALOG SIDECAR: a real Postgres inside the process answers a BI tool's introspection. A tool's startup is
|
|
3
|
+
* hundreds of lines over `pg_catalog` and `information_schema` — `format_type`, `pg_get_expr`, `obj_description`,
|
|
4
|
+
* regclass casts, CTEs — and emulating that
|
|
5
|
+
* is a second query engine. So nothing is emulated: `@electric-sql/pglite` (Postgres 18.6 as WebAssembly) holds the
|
|
6
|
+
* copied tables' `PgSchema` (src/api/postgres/schema.ts) as EMPTY tables with their Postgres types, primary keys, a
|
|
7
|
+
* foreign key per foreign key between copied tables and a COMMENT per description, and the server hands it a tool's
|
|
8
|
+
* raw protocol message and forwards the raw reply, Postgres's own bytes. The DuckDB plan is made here too (src/duck/plan.ts:
|
|
9
|
+
* Postgres analyses the statement over these tables).
|
|
10
|
+
*
|
|
11
|
+
* Two halves. `catalogDdl` is pure: the statements that create the schema, pinned by tests/postgres-api/catalog.test.ts
|
|
12
|
+
* so a change to what a tool sees is read in a diff. `PgCatalog` owns the instance, under five rules:
|
|
13
|
+
*
|
|
14
|
+
* - OFF THE MAIN THREAD, WITH A DEADLINE. PGlite runs synchronously inside its WebAssembly, so an instance on the
|
|
15
|
+
* listener's own thread stopped the event loop for the length of any statement — `SELECT pg_sleep(3600)` from any
|
|
16
|
+
* session froze every other, and PGlite ignores `statement_timeout` (review of #480). The instance lives on a worker
|
|
17
|
+
* thread (catalog-worker.ts); every call has `statementMs` to answer, and past it the worker is TERMINATED and
|
|
18
|
+
* discarded (the call rejects saying so; the next call starts a fresh instance). Its ~820 MB is the worker's, not
|
|
19
|
+
* the API's heap.
|
|
20
|
+
* - LAZY AND IDLE-STOPPED. An instance costs ~0.8 s to start and ~820 MB resident on Linux (1.5 GB on macOS), too much
|
|
21
|
+
* for a deployment nobody connects a BI tool to, so it starts on the first `run` and stops after `idleMs` without
|
|
22
|
+
* one. The idle timer is `unref`'d: it never keeps a process alive.
|
|
23
|
+
* - SERIALISED. PGlite is a single connection; every `run`, `query`, `stop` and `replace` is queued on one chain so
|
|
24
|
+
* two callers never interleave their protocol messages and the DDL is applied exactly once per start.
|
|
25
|
+
* - THE SCHEMA IS THE CATALOG. When the copied tables change the server calls `replace` with the new schema: the running instance is
|
|
26
|
+
* stopped and the next `run` starts with the new DDL. There is no ALTER path — a fresh instance is cheaper than
|
|
27
|
+
* diffing two schemas correctly.
|
|
28
|
+
* - A START FAILURE IS THE CALLER'S. When the WASM cannot load or the DDL is refused, `run` rejects with an Error that
|
|
29
|
+
* says PGlite could not start and why, and the object is left able to try again (nothing is cached half-started).
|
|
30
|
+
*
|
|
31
|
+
* The instance's TimeZone is UTC (the API treats every time as UTC, §4.8); its search_path is Postgres's default, so
|
|
32
|
+
* `public."orders"` and `orders` both resolve, as a tool expects.
|
|
33
|
+
*/
|
|
34
|
+
import { Worker } from "node:worker_threads";
|
|
35
|
+
import { fileURLToPath } from "node:url";
|
|
36
|
+
import { runnable } from "../../lib/runnable.js";
|
|
37
|
+
import { PG_API_TYPE_SPELLING, typeNameParts } from "./schema.js";
|
|
38
|
+
/** Ten minutes: long enough to outlive a tool's startup burst and a person's pause between two queries. */
|
|
39
|
+
const DEFAULT_IDLE_MS = 10 * 60_000;
|
|
40
|
+
/** A catalog statement's deadline (`MASON_PG_CATALOG_TIMEOUT_MS`): a tool's introspection answers in milliseconds; a
|
|
41
|
+
* statement still running after this is runaway, and the instance is discarded. The start (WASM ~0.8 s, the DDL) has
|
|
42
|
+
* the same budget. */
|
|
43
|
+
export const DEFAULT_STATEMENT_MS = 10_000;
|
|
44
|
+
/** the start (PGlite's own boot and the whole DDL, a second or more on a loaded host) gets at least this long, whatever
|
|
45
|
+
* a statement's deadline is: a short statement deadline is for a statement, not for discarding every start */
|
|
46
|
+
export const MIN_START_MS = 30_000;
|
|
47
|
+
/** A SQL identifier, double-quoted with an inner quote doubled. */
|
|
48
|
+
const ident = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
49
|
+
/** A SQL string literal, single-quoted with an inner quote doubled. */
|
|
50
|
+
const literal = (text) => `'${text.replaceAll("'", "''")}'`;
|
|
51
|
+
/** a table's name in the catalog: its schema (Mason's source schema, else public) and name */
|
|
52
|
+
const qualified = (table) => `${table.schema === undefined ? "public" : ident(table.schema)}.${ident(table.name)}`;
|
|
53
|
+
/** the table a join names: `<schema>.<table>`, or a bare name in public */
|
|
54
|
+
const joinTarget = (schema, table) => schema.tables.find((t) => `${t.schema ?? "public"}.${t.name}` === table) ?? schema.tables.find((t) => t.schema === undefined && t.name === table);
|
|
55
|
+
/**
|
|
56
|
+
* Whether a join can be a FOREIGN KEY in Postgres: a `many_to_one` or `one_to_one` edge (the side that holds the key),
|
|
57
|
+
* to a table in the schema, whose target column is that table's WHOLE primary key or the whole of one of its unique
|
|
58
|
+
* indexes (Postgres refuses a foreign key to a column no unique index covers exactly), from a column this table lists.
|
|
59
|
+
* Any other join is a comment on its column.
|
|
60
|
+
*/
|
|
61
|
+
function isForeignKey(schema, table, join) {
|
|
62
|
+
if (join.relationship !== "many_to_one" && join.relationship !== "one_to_one")
|
|
63
|
+
return false;
|
|
64
|
+
if (!table.columns.some((c) => c.name === join.on[0]))
|
|
65
|
+
return false;
|
|
66
|
+
const target = joinTarget(schema, join.table);
|
|
67
|
+
if (target === undefined)
|
|
68
|
+
return false;
|
|
69
|
+
return [target.primaryKey, ...(target.uniqueKeys ?? []).map((u) => u.columns)].some((k) => k.length === 1 && k[0] === join.on[1]);
|
|
70
|
+
}
|
|
71
|
+
const joinNote = (join) => `joins ${join.table} on ${join.on[1]} (${join.relationship})`;
|
|
72
|
+
/**
|
|
73
|
+
* THE CATALOG'S DATABASE IS THE ONE A TOOL CONNECTED TO (the deployment): current_database(), pg_database — pgjdbc's
|
|
74
|
+
* getCatalogs and every TABLE_CAT, DBeaver's navigator — once said the sidecar's own `postgres`. The current
|
|
75
|
+
* database's row is renamed (ALTER DATABASE refuses the current one; the catalog row does not), after moving aside any
|
|
76
|
+
* OTHER database already holding the name — PGlite's `template0` / `template1` — since pg_database's names are unique
|
|
77
|
+
* (#484's review: a deployment named `template1` failed on pg_database_datname_index and the catalog never started).
|
|
78
|
+
*/
|
|
79
|
+
export function databaseNameSql(name) {
|
|
80
|
+
const q = `'${name.replaceAll("'", "''")}'`;
|
|
81
|
+
return [
|
|
82
|
+
`UPDATE pg_catalog.pg_database SET datname = datname || '_pglite' WHERE datname = ${q} AND datname <> current_database()`,
|
|
83
|
+
`UPDATE pg_catalog.pg_database SET datname = ${q} WHERE datname = current_database()`,
|
|
84
|
+
];
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* The statements that create the schema in Postgres, in the schema's own order: every table (its columns and its
|
|
88
|
+
* primary key), the comments from its descriptions, and — after every table exists, since a constraint may point
|
|
89
|
+
* forward — one foreign key per join that can be one. A join that cannot (a `one_to_many`, a target that is not a whole
|
|
90
|
+
* primary key, a target outside the schema) is a note appended to its column's comment; a join from a column the table
|
|
91
|
+
* does not list has no column to carry it and is noted on the table instead.
|
|
92
|
+
*/
|
|
93
|
+
export function catalogDdl(schema) {
|
|
94
|
+
const statements = [];
|
|
95
|
+
const constraints = [];
|
|
96
|
+
// a constraint's name as Postgres chooses one (ChooseConstraintName): `<table>_<column>_fkey`, and a number appended
|
|
97
|
+
// from 1 when a constraint of the table already has it — two joins from one column (the shop's `customers.id` to
|
|
98
|
+
// `latest_orders` and to `latest_status`) are two constraints, and one name for both was refused
|
|
99
|
+
const named = new Set();
|
|
100
|
+
const constraintName = (base) => {
|
|
101
|
+
let name = base;
|
|
102
|
+
for (let n = 1; named.has(name); n++)
|
|
103
|
+
name = `${base}${n}`;
|
|
104
|
+
named.add(name);
|
|
105
|
+
return name;
|
|
106
|
+
};
|
|
107
|
+
// an extension's type a column has (Mason copies a citext as itself): its extension, which the worker
|
|
108
|
+
// bundles (catalog-worker.ts), before the tables
|
|
109
|
+
if (schema.tables.some((t) => t.columns.some((c) => c.spelled === "citext")))
|
|
110
|
+
statements.push("CREATE EXTENSION IF NOT EXISTS citext;");
|
|
111
|
+
// a source schema the prototype copies from (decision C): created before its tables
|
|
112
|
+
for (const name of [...new Set(schema.tables.map((t) => t.schema).filter((x) => x !== undefined && x !== "public"))]) {
|
|
113
|
+
statements.push(`CREATE SCHEMA IF NOT EXISTS ${ident(name)};`);
|
|
114
|
+
}
|
|
115
|
+
// a source enum a column has: the type, its labels in their declared order, before the tables —
|
|
116
|
+
// spelled as the source's format_type spells it (`mood`, `other.level`, `"Odd"`), its schema made first
|
|
117
|
+
const enums = new Map();
|
|
118
|
+
for (const t of schema.tables)
|
|
119
|
+
for (const c of t.columns)
|
|
120
|
+
if (c.enumLabels !== undefined && c.spelled !== undefined)
|
|
121
|
+
enums.set(c.spelled.replace(/\[\]$/, ""), c.enumLabels);
|
|
122
|
+
for (const [type, labels] of enums) {
|
|
123
|
+
const qualifier = typeNameParts(type).schema;
|
|
124
|
+
if (qualifier !== null && qualifier !== '"public"' && qualifier !== "public")
|
|
125
|
+
statements.push(`CREATE SCHEMA IF NOT EXISTS ${qualifier};`);
|
|
126
|
+
statements.push(`CREATE TYPE ${type} AS ENUM (${labels.map(literal).join(", ")});`);
|
|
127
|
+
}
|
|
128
|
+
for (const table of schema.tables) {
|
|
129
|
+
const lines = [];
|
|
130
|
+
for (const c of table.columns)
|
|
131
|
+
lines.push(`${ident(c.name)} ${c.spelled ?? PG_API_TYPE_SPELLING[c.type]}${c.notNull === true ? " NOT NULL" : ""}`);
|
|
132
|
+
if (table.primaryKey.length > 0)
|
|
133
|
+
lines.push(`PRIMARY KEY (${table.primaryKey.map(ident).join(", ")})`);
|
|
134
|
+
statements.push(`CREATE TABLE ${qualified(table)} (\n ${lines.join(",\n ")}\n);`);
|
|
135
|
+
// the source's unique indexes, under its names: before the foreign keys that may reference them
|
|
136
|
+
for (const u of table.uniqueKeys ?? []) {
|
|
137
|
+
const cols = `(${u.columns.map(ident).join(", ")})`;
|
|
138
|
+
const nulls = u.nullsNotDistinct ? "NULLS NOT DISTINCT" : "";
|
|
139
|
+
// the two spellings put NULLS NOT DISTINCT on either side of the columns
|
|
140
|
+
statements.push(u.constraint ? `ALTER TABLE ${qualified(table)} ADD CONSTRAINT ${ident(u.name)} UNIQUE ${nulls} ${cols};`
|
|
141
|
+
: `CREATE UNIQUE INDEX ${ident(u.name)} ON ${qualified(table)} ${cols} ${nulls};`);
|
|
142
|
+
}
|
|
143
|
+
const notesOn = new Map();
|
|
144
|
+
const tableNotes = [];
|
|
145
|
+
for (const join of table.joins) {
|
|
146
|
+
if (isForeignKey(schema, table, join)) {
|
|
147
|
+
constraints.push(`ALTER TABLE ${qualified(table)} ADD CONSTRAINT ${ident(constraintName(`${table.name}_${join.on[0]}_fkey`))}` +
|
|
148
|
+
` FOREIGN KEY (${ident(join.on[0])}) REFERENCES ${qualified(joinTarget(schema, join.table))} (${ident(join.on[1])});`);
|
|
149
|
+
}
|
|
150
|
+
else if (table.columns.some((c) => c.name === join.on[0])) {
|
|
151
|
+
notesOn.set(join.on[0], [...(notesOn.get(join.on[0]) ?? []), joinNote(join)]);
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
tableNotes.push(`${join.on[0]} ${joinNote(join)}`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const tableComment = [table.description, ...tableNotes].filter((s) => s !== undefined && s !== "").join("; ");
|
|
158
|
+
if (tableComment !== "")
|
|
159
|
+
statements.push(`COMMENT ON TABLE ${qualified(table)} IS ${literal(tableComment)};`);
|
|
160
|
+
for (const c of table.columns) {
|
|
161
|
+
const parts = [c.description, ...(notesOn.get(c.name) ?? [])].filter((s) => s !== undefined && s !== "");
|
|
162
|
+
if (parts.length > 0)
|
|
163
|
+
statements.push(`COMMENT ON COLUMN ${qualified(table)}.${ident(c.name)} IS ${literal(parts.join("; "))};`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return [...statements, ...constraints, ...sourceTypeNames(schema)];
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* THE NAME OF A TYPE BY THE OID THE WIRE SENDS FOR IT. A column of a type the source made (an enum, a citext) is
|
|
170
|
+
* sent with the SOURCE's OID (src/duck/plan.ts `userTypes`: what the source's own clients cached), which in this
|
|
171
|
+
* catalog means another type or none (both count from 16384): `format_type(<that oid>, …)` answered `???`, or another
|
|
172
|
+
* table's row type. Two readers ask by two OID spaces, and one OID can be in both:
|
|
173
|
+
* - `public.mason_wire_type(oid, integer)` names an OID the WIRE sent: a source OID as its type here, every other as
|
|
174
|
+
* Postgres does. `mason mcp` names a RowDescription's columns by it; this map is the one place a wire OID is named;
|
|
175
|
+
* - the catalog's `format_type(oid, integer)` — the `version()` override's way, search_path listing pg_catalog after
|
|
176
|
+
* public — keeps the catalog's meaning for an OID the catalog has (a reader of pg_attribute.atttypid asks by those)
|
|
177
|
+
* and names any other by the wire's map. `pg_catalog.format_type`, written out, is Postgres's own.
|
|
178
|
+
*/
|
|
179
|
+
export function sourceTypeNames(schema) {
|
|
180
|
+
const names = new Map();
|
|
181
|
+
for (const t of schema.tables) {
|
|
182
|
+
if (t.rowTypeOid !== undefined)
|
|
183
|
+
names.set(t.rowTypeOid, qualified(t));
|
|
184
|
+
for (const c of t.columns) {
|
|
185
|
+
if (c.typeOid === undefined || c.spelled === undefined)
|
|
186
|
+
continue;
|
|
187
|
+
// an array column's own OID is its array type's; an element's array type is the element's spelling with []
|
|
188
|
+
names.set(c.typeOid, c.spelled);
|
|
189
|
+
if (c.typeArrayOid !== undefined && !c.spelled.endsWith("[]"))
|
|
190
|
+
names.set(c.typeArrayOid, `${c.spelled}[]`);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
// with no type the source made, every OID the wire sends is Postgres's: the wire's name is format_type's
|
|
194
|
+
if (names.size === 0)
|
|
195
|
+
return [`CREATE OR REPLACE FUNCTION public.mason_wire_type(oid, integer) RETURNS text LANGUAGE sql STABLE AS $$ SELECT pg_catalog.format_type($1, $2) $$;`];
|
|
196
|
+
const values = [...names].map(([oid, spelled]) => `(${oid}::oid, ${literal(spelled)})`).join(", ");
|
|
197
|
+
return [`CREATE OR REPLACE FUNCTION public.mason_wire_type(oid, integer) RETURNS text LANGUAGE sql STABLE AS $$
|
|
198
|
+
SELECT coalesce((SELECT pg_catalog.format_type(pg_catalog.to_regtype(v.spelled), $2) FROM (VALUES ${values}) AS v(source, spelled) WHERE v.source = $1),
|
|
199
|
+
pg_catalog.format_type($1, $2)) $$;`,
|
|
200
|
+
`CREATE OR REPLACE FUNCTION public.format_type(oid, integer) RETURNS text LANGUAGE sql STABLE AS $$
|
|
201
|
+
SELECT CASE WHEN EXISTS (SELECT FROM pg_catalog.pg_type t WHERE t.oid = $1) THEN pg_catalog.format_type($1, $2) ELSE public.mason_wire_type($1, $2) END $$;`];
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* The source's column defaults, one statement each, run after the schema's DDL by
|
|
205
|
+
* `catalogDdl`'s caller one at a time: a default the catalog cannot create (a function or a sequence it does not have,
|
|
206
|
+
* `uuid_generate_v4()` without its extension) is said in the log and leaves the column without one, never the start
|
|
207
|
+
* refused. A serial's sequence is created and OWNED BY its column and an identity column is made one under its own
|
|
208
|
+
* sequence's name, so `pg_get_serial_sequence` and `information_schema.columns` answer as the source's. A generated
|
|
209
|
+
* column is declared as a plain one (its expression is no default, src/cli/preflight.ts).
|
|
210
|
+
*/
|
|
211
|
+
export function catalogDefaults(schema) {
|
|
212
|
+
const out = [];
|
|
213
|
+
for (const table of schema.tables)
|
|
214
|
+
for (const c of table.columns) {
|
|
215
|
+
const column = `${qualified(table)} ALTER COLUMN ${ident(c.name)}`;
|
|
216
|
+
if (c.identity !== undefined) {
|
|
217
|
+
const named = c.sequence !== undefined ? ` (SEQUENCE NAME ${c.sequence})` : "";
|
|
218
|
+
out.push(`ALTER TABLE ${column} ADD GENERATED ${c.identity === "a" ? "ALWAYS" : "BY DEFAULT"} AS IDENTITY${named};`);
|
|
219
|
+
}
|
|
220
|
+
else if (c.default !== undefined && c.sequence !== undefined) {
|
|
221
|
+
// a serial's sequence is its column's integer type (CREATE SEQUENCE's default is bigint)
|
|
222
|
+
const declared = c.spelled ?? PG_API_TYPE_SPELLING[c.type];
|
|
223
|
+
const as = ["smallint", "integer", "bigint"].includes(declared) ? ` AS ${declared}` : "";
|
|
224
|
+
// one statement: the sequence, the default reading it and its owner stand or fall together
|
|
225
|
+
out.push(`CREATE SEQUENCE ${c.sequence}${as}; ALTER TABLE ${column} SET DEFAULT ${c.default}; ALTER SEQUENCE ${c.sequence} OWNED BY ${qualified(table)}.${ident(c.name)};`);
|
|
226
|
+
}
|
|
227
|
+
else if (c.default !== undefined) {
|
|
228
|
+
out.push(`ALTER TABLE ${column} SET DEFAULT ${c.default};`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return out;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* The source's views, one statement each beside `catalogDefaults`'s: a view reading a table
|
|
235
|
+
* Mason has not copied (or a function the catalog lacks) is said in the log and left out. Each definition qualifies
|
|
236
|
+
* every name outside pg_catalog (src/cli/preflight.ts VIEWS_SQL), so it reads the same tables here; a view over a view
|
|
237
|
+
* is created once the other is (the worker runs these in passes).
|
|
238
|
+
*/
|
|
239
|
+
export function catalogViews(schema) {
|
|
240
|
+
return (schema.views ?? []).map((v) => {
|
|
241
|
+
const name = `${ident(v.schema)}.${ident(v.name)}`;
|
|
242
|
+
const comment = v.description ? ` COMMENT ON VIEW ${name} IS ${literal(v.description)};` : "";
|
|
243
|
+
return `CREATE SCHEMA IF NOT EXISTS ${ident(v.schema)}; CREATE VIEW ${name} AS ${v.definition.replace(/;\s*$/, "")};${comment}`;
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* WHAT A LEFT-OUT DEFAULT OR VIEW IS SAID AS, in one line (the test server's log printed every left-out source
|
|
248
|
+
* view's whole definition at each start of the instance — 7 multi-line entries per start, most of a 190 KB log, their
|
|
249
|
+
* continuation lines without a timestamp, and the views' own columns matched a grep for "error"): the object named,
|
|
250
|
+
* and the reason in the reader's words — a view reading a relation Mason has not copied says so. The statement itself
|
|
251
|
+
* is the trace's (`PgCatalogOptions.trace`).
|
|
252
|
+
*/
|
|
253
|
+
export function leftOutLine(r) {
|
|
254
|
+
const view = /CREATE VIEW ("(?:[^"]|"")+")\.("(?:[^"]|"")+") AS /.exec(r.statement);
|
|
255
|
+
const unquote = (s) => s.slice(1, -1).replaceAll('""', '"');
|
|
256
|
+
if (view === null)
|
|
257
|
+
return `catalog: a source default left out (${r.message}): ${r.statement}`;
|
|
258
|
+
const missing = /^relation "([^"]+)" does not exist$/.exec(r.message)?.[1];
|
|
259
|
+
const name = `${unquote(view[1])}.${unquote(view[2])}`;
|
|
260
|
+
return missing !== undefined ? `catalog: view ${name} left out: it reads ${missing}, which is not copied` : `catalog: view ${name} left out (${r.message})`;
|
|
261
|
+
}
|
|
262
|
+
const errorMessage = (e) => (e instanceof Error ? e.message : String(e));
|
|
263
|
+
/** One worker thread holding one PGlite: requests one at a time (PgCatalog's chain), each with a deadline. */
|
|
264
|
+
class CatalogWorker {
|
|
265
|
+
#worker;
|
|
266
|
+
#pending = null;
|
|
267
|
+
#dead = null;
|
|
268
|
+
#next = 1;
|
|
269
|
+
constructor() {
|
|
270
|
+
const file = new URL("./catalog-worker.ts", import.meta.url);
|
|
271
|
+
// `execArgv: []`: the worker must not inherit this process's Node flags (src/api/postgres/parser.ts says why)
|
|
272
|
+
this.#worker = new Worker(runnable(fileURLToPath(file)), { execArgv: [], argv: [] });
|
|
273
|
+
this.#worker.unref();
|
|
274
|
+
this.#worker.on("message", (reply) => {
|
|
275
|
+
const p = this.#pending;
|
|
276
|
+
if (p === null || p.id !== reply.id)
|
|
277
|
+
return; // a reply past its deadline: its worker is being terminated
|
|
278
|
+
this.#pending = null;
|
|
279
|
+
clearTimeout(p.timer);
|
|
280
|
+
p.resolve(reply);
|
|
281
|
+
});
|
|
282
|
+
this.#worker.on("error", (e) => this.#die(new Error(`the catalog's worker failed: ${e.message}`)));
|
|
283
|
+
this.#worker.on("exit", (code) => this.#die(new Error(`the catalog's worker exited with code ${code}`)));
|
|
284
|
+
}
|
|
285
|
+
get alive() { return this.#dead === null; }
|
|
286
|
+
#die(e) {
|
|
287
|
+
if (this.#dead !== null)
|
|
288
|
+
return;
|
|
289
|
+
this.#dead = e;
|
|
290
|
+
const p = this.#pending;
|
|
291
|
+
this.#pending = null;
|
|
292
|
+
if (p !== null) {
|
|
293
|
+
clearTimeout(p.timer);
|
|
294
|
+
p.reject(e);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
/** one request; past `ms` the worker is terminated and the request rejects naming the deadline */
|
|
298
|
+
request(message, ms, what, transfer = []) {
|
|
299
|
+
if (this.#dead !== null)
|
|
300
|
+
return Promise.reject(this.#dead);
|
|
301
|
+
if (this.#pending !== null)
|
|
302
|
+
return Promise.reject(new Error("the catalog's worker is busy: PgCatalog serialises its calls"));
|
|
303
|
+
return new Promise((resolve, reject) => {
|
|
304
|
+
const id = this.#next++;
|
|
305
|
+
const timer = setTimeout(() => {
|
|
306
|
+
this.#die(new Error(`the catalog did not answer ${what} within ${ms / 1000} s: PGlite was discarded (the next statement starts a fresh one; MASON_PG_CATALOG_TIMEOUT_MS raises the deadline)`));
|
|
307
|
+
void this.#worker.terminate();
|
|
308
|
+
}, ms);
|
|
309
|
+
timer.unref();
|
|
310
|
+
this.#pending = { id, resolve, reject, timer };
|
|
311
|
+
this.#worker.postMessage({ ...message, id }, transfer);
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
async terminate() {
|
|
315
|
+
this.#die(new Error("the catalog was stopped"));
|
|
316
|
+
await this.#worker.terminate();
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
export class PgCatalog {
|
|
320
|
+
#schema;
|
|
321
|
+
#idleMs;
|
|
322
|
+
#statementMs;
|
|
323
|
+
#log;
|
|
324
|
+
#setup;
|
|
325
|
+
#trace;
|
|
326
|
+
/** the left-out lines said for this schema (`leftOutLine`): each once, not at every start of the instance; cleared
|
|
327
|
+
* with the schema (`replace`), so a change is said again */
|
|
328
|
+
#said = new Set();
|
|
329
|
+
#db = null;
|
|
330
|
+
/** the one queue every call runs on; always resolved, so a failed call never blocks the next (its rejection is the caller's) */
|
|
331
|
+
#chain = Promise.resolve();
|
|
332
|
+
#idleTimer = null;
|
|
333
|
+
#lastUsed = 0;
|
|
334
|
+
constructor(schema, opts = {}) {
|
|
335
|
+
this.#schema = schema;
|
|
336
|
+
this.#idleMs = opts.idleMs ?? DEFAULT_IDLE_MS;
|
|
337
|
+
this.#statementMs = opts.statementMs ?? DEFAULT_STATEMENT_MS;
|
|
338
|
+
this.#log = opts.log ?? ((line) => console.error(line));
|
|
339
|
+
this.#setup = opts.setup ?? [];
|
|
340
|
+
this.#trace = opts.trace === true;
|
|
341
|
+
}
|
|
342
|
+
/** whether an instance is running (started and its DDL applied) */
|
|
343
|
+
get started() {
|
|
344
|
+
return this.#db !== null && this.#db.alive;
|
|
345
|
+
}
|
|
346
|
+
/** Runs raw frontend protocol bytes — one message or several (Parse, Bind, Describe, Execute, Sync in one buffer
|
|
347
|
+
* works) — and returns the backend's raw reply, starting the instance when needed. */
|
|
348
|
+
run(message) {
|
|
349
|
+
return this.#enqueue(async () => {
|
|
350
|
+
const db = await this.#ensureStarted();
|
|
351
|
+
const copy = new Uint8Array(message);
|
|
352
|
+
const reply = await this.#ask(db, { op: "raw", bytes: copy }, "a statement", [copy.buffer]);
|
|
353
|
+
this.#touch();
|
|
354
|
+
return reply.bytes ?? new Uint8Array(0);
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
/** The server's own catalog lookups and the tests: a statement through PGlite's API, its rows as objects. */
|
|
358
|
+
query(sql) {
|
|
359
|
+
return this.#enqueue(async () => {
|
|
360
|
+
const db = await this.#ensureStarted();
|
|
361
|
+
const reply = await this.#ask(db, { op: "query", sql }, "a query");
|
|
362
|
+
this.#touch();
|
|
363
|
+
return { rows: reply.rows ?? [] };
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
/** Stops the instance if one is running; the idle timer calls this. Resolves once it is gone. */
|
|
367
|
+
stop() {
|
|
368
|
+
return this.#enqueue(() => this.#close("stopped"));
|
|
369
|
+
}
|
|
370
|
+
/** Swaps in a new schema (after an apply): the running instance is stopped and the next `run` starts with the new
|
|
371
|
+
* DDL. Resolves once the old instance is gone; a caller need not wait for it. */
|
|
372
|
+
replace(schema) {
|
|
373
|
+
this.#schema = schema;
|
|
374
|
+
this.#said.clear();
|
|
375
|
+
return this.#enqueue(() => this.#close("replaced"));
|
|
376
|
+
}
|
|
377
|
+
#enqueue(fn) {
|
|
378
|
+
const result = this.#chain.then(fn);
|
|
379
|
+
// the chain only ORDERS calls: a rejection reaches the caller through `result`, and the next call must still run
|
|
380
|
+
this.#chain = result.then(() => undefined, () => undefined);
|
|
381
|
+
return result;
|
|
382
|
+
}
|
|
383
|
+
/** one request of the worker; a worker that died (the deadline, a crash) is dropped so the next call starts afresh */
|
|
384
|
+
async #ask(db, message, what, transfer = []) {
|
|
385
|
+
let reply;
|
|
386
|
+
try {
|
|
387
|
+
reply = await db.request(message, message["op"] === "init" ? Math.max(this.#statementMs, MIN_START_MS) : this.#statementMs, what, transfer);
|
|
388
|
+
}
|
|
389
|
+
catch (e) {
|
|
390
|
+
if (this.#db === db) {
|
|
391
|
+
this.#db = null;
|
|
392
|
+
this.#log(`catalog: ${errorMessage(e)}`);
|
|
393
|
+
}
|
|
394
|
+
throw e;
|
|
395
|
+
}
|
|
396
|
+
if (!reply.ok)
|
|
397
|
+
throw new Error(reply.message);
|
|
398
|
+
return reply;
|
|
399
|
+
}
|
|
400
|
+
async #ensureStarted() {
|
|
401
|
+
if (this.#db !== null && this.#db.alive)
|
|
402
|
+
return this.#db;
|
|
403
|
+
this.#db = null;
|
|
404
|
+
const schema = this.#schema;
|
|
405
|
+
const t0 = Date.now();
|
|
406
|
+
const db = new CatalogWorker();
|
|
407
|
+
try {
|
|
408
|
+
const started = await this.#ask(db, { op: "init", statements: ["SET TimeZone = 'UTC'", ...catalogDdl(schema), ...this.#setup], optional: [...catalogDefaults(schema), ...catalogViews(schema)] }, "its start");
|
|
409
|
+
// each left-out default or view: one line, once per schema (the instance starts again after every idle stop)
|
|
410
|
+
for (const r of started.refused ?? []) {
|
|
411
|
+
const line = leftOutLine(r);
|
|
412
|
+
if (this.#said.has(line))
|
|
413
|
+
continue;
|
|
414
|
+
this.#said.add(line);
|
|
415
|
+
this.#log(line);
|
|
416
|
+
if (this.#trace)
|
|
417
|
+
this.#log(`trace catalog: the statement left out: ${r.statement}`);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
catch (e) {
|
|
421
|
+
// the half-made worker is discarded so the next `run` tries afresh
|
|
422
|
+
await db.terminate();
|
|
423
|
+
throw new Error(`PGlite could not start: ${errorMessage(e)}`, { cause: e });
|
|
424
|
+
}
|
|
425
|
+
this.#db = db;
|
|
426
|
+
this.#log(`catalog: PGlite started in ${Date.now() - t0} ms with ${schema.tables.length} table(s)`);
|
|
427
|
+
this.#touch();
|
|
428
|
+
return db;
|
|
429
|
+
}
|
|
430
|
+
/** Every call resets the idle timer; the timer stops the instance only when nothing has used it since. */
|
|
431
|
+
#touch() {
|
|
432
|
+
this.#lastUsed = Date.now();
|
|
433
|
+
if (this.#idleTimer)
|
|
434
|
+
clearTimeout(this.#idleTimer);
|
|
435
|
+
this.#arm(this.#idleMs);
|
|
436
|
+
}
|
|
437
|
+
#arm(ms) {
|
|
438
|
+
this.#idleTimer = setTimeout(() => { this.#idleTimer = null; void this.#idleStop(); }, ms);
|
|
439
|
+
this.#idleTimer.unref();
|
|
440
|
+
}
|
|
441
|
+
#idleStop() {
|
|
442
|
+
return this.#enqueue(async () => {
|
|
443
|
+
const idle = Date.now() - this.#lastUsed;
|
|
444
|
+
if (idle < this.#idleMs) {
|
|
445
|
+
// a call queued ahead of this one has touched the timer again (it armed a new one; it decides) — OR the firing
|
|
446
|
+
// was a millisecond early by the wall clock (timers run on the monotonic clock, `lastUsed` on Date.now(), and
|
|
447
|
+
// the two round differently): then nothing is armed, and an instance that returned here stayed up for good
|
|
448
|
+
// (CI: ~820 MB until the next statement). Arm again for what is left, never leave it running.
|
|
449
|
+
if (this.#idleTimer === null && this.#db !== null)
|
|
450
|
+
this.#arm(Math.max(1, this.#idleMs - idle));
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
await this.#close(`stopped after ${Math.round(this.#idleMs / 1000)} s idle`);
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
async #close(why) {
|
|
457
|
+
if (this.#idleTimer) {
|
|
458
|
+
clearTimeout(this.#idleTimer);
|
|
459
|
+
this.#idleTimer = null;
|
|
460
|
+
}
|
|
461
|
+
const db = this.#db;
|
|
462
|
+
if (!db)
|
|
463
|
+
return;
|
|
464
|
+
this.#db = null;
|
|
465
|
+
if (db.alive) {
|
|
466
|
+
try {
|
|
467
|
+
await this.#ask(db, { op: "close" }, "its close");
|
|
468
|
+
}
|
|
469
|
+
catch (e) {
|
|
470
|
+
// the instance is discarded either way (terminated below); the failure is reported, not hidden
|
|
471
|
+
this.#log(`catalog: PGlite did not close cleanly: ${errorMessage(e)}`);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
await db.terminate();
|
|
475
|
+
this.#log(`catalog: PGlite ${why}`);
|
|
476
|
+
}
|
|
477
|
+
}
|