@mason-data/cli 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +90 -71
- package/dist/src/api/error.js +3 -0
- package/dist/src/api/postgres/catalog.js +222 -13
- package/dist/src/api/postgres/encode.js +3 -1
- package/dist/src/api/postgres/explain.js +3 -2
- package/dist/src/api/postgres/protocol.js +39 -23
- package/dist/src/api/postgres/server.js +1173 -207
- package/dist/src/api/postgres/translate.js +81 -9
- package/dist/src/api/request-log.js +18 -5
- package/dist/src/cdc/poll.js +5 -5
- package/dist/src/cli/background.js +29 -6
- package/dist/src/cli/cancel.js +75 -0
- package/dist/src/cli/command.js +1 -0
- package/dist/src/cli/commands.js +622 -183
- package/dist/src/cli/control.js +59 -0
- package/dist/src/cli/demo.js +19 -10
- package/dist/src/cli/dev.js +351 -111
- package/dist/src/cli/doctor.js +20 -10
- package/dist/src/cli/guides.js +79 -31
- package/dist/src/cli/init.js +119 -58
- package/dist/src/cli/machine.js +12 -11
- package/dist/src/cli/main.js +3 -3
- package/dist/src/cli/mcp.js +29 -20
- package/dist/src/cli/metrics.js +79 -0
- package/dist/src/cli/pg-tls.js +13 -3
- package/dist/src/cli/pg-url.js +4 -0
- package/dist/src/cli/pin.js +13 -12
- package/dist/src/cli/preflight.js +8 -7
- package/dist/src/cli/project.js +43 -4
- package/dist/src/cli/readiness.js +38 -4
- package/dist/src/cli/registry.js +53 -34
- package/dist/src/cli/replication-checks.js +7 -5
- package/dist/src/cli/replication-mode.js +105 -26
- package/dist/src/cli/request-filter.js +134 -0
- package/dist/src/cli/roles.js +584 -0
- package/dist/src/cli/run-server.js +483 -41
- package/dist/src/cli/runs.js +1 -1
- package/dist/src/cli/security.js +10 -8
- package/dist/src/cli/serve.js +144 -42
- package/dist/src/cli/share-run.js +286 -0
- package/dist/src/cli/share.js +286 -0
- package/dist/src/cli/source-edit.js +65 -13
- package/dist/src/cli/sources.js +287 -38
- package/dist/src/cli/supervise.js +12 -1
- package/dist/src/cli/sync.js +55 -25
- package/dist/src/cli/tunnel.js +200 -0
- package/dist/src/cli/ui.js +9 -6
- package/dist/src/cli/usage.js +128 -0
- package/dist/src/cli/user.js +496 -40
- package/dist/src/cli/version.js +9 -7
- package/dist/src/demo/seed.js +3 -0
- package/dist/src/duck/cancel.js +10 -1
- package/dist/src/duck/events.js +134 -0
- package/dist/src/duck/executor-worker.js +18 -9
- package/dist/src/duck/executor.js +43 -30
- package/dist/src/duck/explain.js +56 -46
- package/dist/src/duck/extensions.js +160 -0
- package/dist/src/duck/functions.js +13 -14
- package/dist/src/duck/gaps.js +17 -8
- package/dist/src/duck/grants.js +176 -0
- package/dist/src/duck/history.js +123 -0
- package/dist/src/duck/kept.js +6 -6
- package/dist/src/duck/plan.js +102 -25
- package/dist/src/duck/publication.js +6 -6
- package/dist/src/duck/requests.js +265 -93
- package/dist/src/duck/rewrite.js +25 -5
- package/dist/src/duck/row-filter.js +649 -0
- package/dist/src/duck/store.js +156 -120
- package/dist/src/duck/stream.js +10 -5
- package/dist/src/duck/sync.js +835 -268
- package/dist/src/duck/types.js +6 -5
- package/dist/src/duck/usage.js +324 -0
- package/dist/src/duck/users.js +64 -15
- package/dist/src/duck/verified.js +3 -0
- package/dist/src/duck/views.js +2 -3
- package/dist/src/lib/database-name.js +62 -0
- package/dist/src/lib/dotenv.js +17 -4
- package/dist/src/lib/log-buffer.js +68 -0
- package/dist/src/lib/pg-numeric.js +5 -2
- package/dist/src/lib/project-version.js +23 -23
- package/dist/src/lib/table-sync.js +267 -0
- package/dist/src/web/app.css +706 -66
- package/dist/src/web/app.js +327 -410
- package/dist/src/web/charts.js +229 -0
- package/dist/src/web/index.html +82 -18
- package/dist/src/web/pages/queries.js +192 -0
- package/dist/src/web/pages/status.js +446 -0
- package/dist/src/web/pages/syncs.js +155 -0
- package/dist/src/web/pages/tables.js +434 -0
- package/dist/src/web/pages/usage.js +397 -0
- package/dist/src/web/pages/users.js +218 -0
- package/dist/src/web/shared.js +568 -0
- package/dist/src/web/vendor/README.md +25 -0
- package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
- package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
- package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
- package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
- package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
- package/package.json +10 -3
package/dist/src/duck/explain.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { agoText, syncText } from "../lib/table-sync.js";
|
|
1
2
|
/** a node of DuckDB's JSON, either form: `name` (EXPLAIN) or `type` with `intermediate_rows` and `timing` (the profile).
|
|
2
3
|
* The profile's RESULT_COLLECTOR and EXECUTE are the result's plumbing, not the plan's: their child stands for them */
|
|
3
4
|
export function planNodes(json) {
|
|
@@ -27,7 +28,7 @@ export function profiled(doc) {
|
|
|
27
28
|
return { sql: typeof d.query?.sql === "string" ? d.query.sql : null, plan: planNodes(d.operator ?? []) };
|
|
28
29
|
}
|
|
29
30
|
/** what EXPLAIN ANALYZE says of a statement DuckDB's profiler timed no operator of */
|
|
30
|
-
export const UNPROFILED = "
|
|
31
|
+
export const UNPROFILED = "the engine's profiler recorded no operator of this statement: the engine answered it while planning it (as it answers a count(*) of a whole table from the table's row count), so the plan is its EXPLAIN, with the rows it answered";
|
|
31
32
|
/** DuckDB's operator as DuckDB's own plan prints it (`HASH_GROUP_BY` is `Hash Group By`), a scan `Table Scan` as its
|
|
32
33
|
* profiler names it: DuckDB's EXPLAIN says SEQ_SCAN for the same operator, and Postgres's `Seq Scan` says otherwise */
|
|
33
34
|
export const operatorName = (type) => type.toLowerCase().split("_").map((w) => w === "n" ? "N" : w.charAt(0).toUpperCase() + w.slice(1)).join(" ");
|
|
@@ -35,7 +36,7 @@ export const operatorName = (type) => type.toLowerCase().split("_").map((w) => w
|
|
|
35
36
|
const quotedName = (s) => /^[a-z_][a-z0-9_$]*$/.test(s) ? s : `"${s.replaceAll('"', '""')}"`;
|
|
36
37
|
/**
|
|
37
38
|
* A TABLE AS POSTGRES WRITES IT, each part quoted where it must be (`public.orders`, `public."Clients"`,
|
|
38
|
-
* `public."x; y"`): the ONE way EXPLAIN names a copy (its line, FORMAT JSON's `Mason.
|
|
39
|
+
* `public."x; y"`): the ONE way EXPLAIN names a copy (its line, FORMAT JSON's `Mason.Tables[].Table`), which a statement
|
|
39
40
|
* can read back as it is (`mason mcp`'s tables and describe name tables by it too).
|
|
40
41
|
*/
|
|
41
42
|
export const qualifiedName = (t) => `${quotedName(t.schema)}.${quotedName(t.name)}`;
|
|
@@ -93,35 +94,39 @@ const utc = (iso) => `${iso.slice(0, 19).replace("T", " ")} UTC`;
|
|
|
93
94
|
const UNKNOWN = "how it is kept current is unknown: the sync has no record of it";
|
|
94
95
|
/**
|
|
95
96
|
* HOW A COPY IS KEPT CURRENT, in EXPLAIN's words, from the sync's one derivation (src/duck/sync.ts `KeptCurrent`, which
|
|
96
|
-
* `mason status` renders too): a copy not yet copied or whose copy failed says so first; then
|
|
97
|
-
*
|
|
98
|
-
* copied
|
|
99
|
-
*
|
|
97
|
+
* `mason status` renders too): a copy not yet copied or whose copy failed says so first; then its sync and its age —
|
|
98
|
+
* `off: last synced 3 days ago (…)` with the command that syncs it, `polled every 30s, last 4 s ago (…)` — or served as
|
|
99
|
+
* copied and skipped by the passes (to be copied again, caught up first), copied again whole, or, in replication mode,
|
|
100
|
+
* its part in the stream, how far the stream has applied and why it is not running. A copy's age is never left out:
|
|
101
|
+
* a BI tool or an agent reading a stale count cannot tell it is one. "kept current" is said only of a copy that is
|
|
102
|
+
* (`keptCurrentBy`). `rows` false says no count of its rows: a filtered table's are every row's, not the role's.
|
|
100
103
|
*/
|
|
101
|
-
export function keptText(f) {
|
|
104
|
+
export function keptText(f, now = Date.now(), rows = true) {
|
|
102
105
|
if (f === null)
|
|
103
106
|
return UNKNOWN;
|
|
104
107
|
if (f.phase === "waiting")
|
|
105
|
-
return "waiting for its
|
|
108
|
+
return "waiting for its first sync to start";
|
|
106
109
|
if (f.phase === "copying")
|
|
107
|
-
return `being
|
|
110
|
+
return `being read whole${rows ? ` (${count(f.rowsSoFar ?? 0)} rows so far)` : ""}: kept current once synced`;
|
|
108
111
|
if (f.phase === "failed")
|
|
109
|
-
return `its
|
|
112
|
+
return `its first sync failed: ${(f.error ?? "mason logs says why").split("\n")[0]}`;
|
|
110
113
|
const k = f.kept;
|
|
111
114
|
if (k === undefined)
|
|
112
115
|
return UNKNOWN;
|
|
113
116
|
if (k.by === "poll") {
|
|
114
|
-
const
|
|
117
|
+
const synced = k.syncedAt === null ? "never synced" : `last synced ${agoText(k.syncedAt, now)} (${utc(k.syncedAt)})`;
|
|
115
118
|
if (k.pending !== undefined)
|
|
116
|
-
return `NOT kept current (${
|
|
117
|
-
if (k.
|
|
118
|
-
return `
|
|
119
|
-
return `
|
|
119
|
+
return `NOT kept current (${synced}): ${k.pending}`;
|
|
120
|
+
if (k.sync.mode === "off")
|
|
121
|
+
return `off: ${synced}; mason sync ${f.source} syncs it now`;
|
|
122
|
+
return `polled every ${k.sync.every}, ${k.syncedAt === null ? "not polled yet" : `last ${agoText(k.syncedAt, now)} (${utc(k.syncedAt)})`}`;
|
|
120
123
|
}
|
|
121
124
|
if (k.by === "recopy")
|
|
122
|
-
return `kept current by
|
|
125
|
+
return `kept current by reading it again whole, ${k.copiedAt === null ? "no sync recorded yet" : `last synced ${utc(k.copiedAt)}`}`;
|
|
123
126
|
const stream = `logical replication (slot ${k.slot})`;
|
|
124
|
-
|
|
127
|
+
// the copy's age as status says it (replicated, at <lsn>, <age>), then the newest change it applied
|
|
128
|
+
const synced = k.syncedAt === null ? "not synced yet" : `last synced ${agoText(k.syncedAt, now)} (${utc(k.syncedAt)})`;
|
|
129
|
+
const where = `replicated, at ${k.position}, ${synced}, ${k.lastCommit === null ? "no change applied this run" : `last change committed ${utc(k.lastCommit)}`}`;
|
|
125
130
|
if (k.state === "streamed")
|
|
126
131
|
return k.notRunning === null ? `kept current by ${stream}, ${where}`
|
|
127
132
|
: `streamed by ${stream}, ${where}; NOT kept current now, the stream is not running: ${k.notRunning}`;
|
|
@@ -129,54 +134,59 @@ export function keptText(f) {
|
|
|
129
134
|
: `${stream}, not streamed yet: ${k.why ?? ""}`;
|
|
130
135
|
}
|
|
131
136
|
/** what keeps a copy current NOW, as FORMAT JSON's `Kept Current By` says it: null for a copy nothing does (not copied
|
|
132
|
-
* yet, skipped by the passes,
|
|
137
|
+
* yet, skipped by the passes, its sync off, not streamed, the stream down), so a reader keying on it is
|
|
133
138
|
* never told of a mode that is not at work on it */
|
|
134
139
|
export function keptCurrentBy(f) {
|
|
135
140
|
const k = f?.phase === "copied" ? f.kept : undefined;
|
|
136
141
|
if (k === undefined)
|
|
137
142
|
return null;
|
|
138
143
|
if (k.by === "poll")
|
|
139
|
-
return k.pending === undefined && k.
|
|
144
|
+
return k.pending === undefined && k.sync.mode === "poll" ? "polling" : null;
|
|
140
145
|
if (k.by === "recopy")
|
|
141
146
|
return "copying again";
|
|
142
147
|
return k.state === "streamed" && k.notRunning === null ? "logical replication" : null;
|
|
143
148
|
}
|
|
144
149
|
/** how a copy is kept current, as FORMAT JSON's keys say it (`keptText`'s facts) */
|
|
145
|
-
function keptJson(f) {
|
|
150
|
+
function keptJson(f, rows) {
|
|
146
151
|
const k = f?.phase === "copied" ? f.kept : undefined;
|
|
147
152
|
return {
|
|
148
|
-
"Kept Current By": keptCurrentBy(f), "Kept Current": keptText(f),
|
|
149
|
-
...(k?.by === "poll" ? { "Last Poll": k.
|
|
150
|
-
...(k?.by === "replication" ? { Slot: k.slot, "Stream State": k.state, "Applied Through": k.position, "Last Change Committed": k.lastCommit, "Stream Not Running": k.notRunning } : {}),
|
|
153
|
+
"Kept Current By": keptCurrentBy(f), "Kept Current": keptText(f, undefined, rows),
|
|
154
|
+
...(k?.by === "poll" ? { Sync: syncText(k.sync), "Last Synced": k.syncedAt, "Next Poll": k.nextAt } : {}),
|
|
155
|
+
...(k?.by === "replication" ? { Sync: "replication", Slot: k.slot, "Stream State": k.state, "Applied Through": k.position, "Last Synced": k.syncedAt, "Last Change Committed": k.lastCommit, "Stream Not Running": k.notRunning } : {}),
|
|
151
156
|
};
|
|
152
157
|
}
|
|
158
|
+
/** what EXPLAIN says of a statement that reads a filtered table, in the place of the engine's plan */
|
|
159
|
+
const FILTERED = "the engine's plan is not shown for a statement that reads a table the role filters: it would say what the engine knows of every row; a user with no row filter reads the full plan";
|
|
153
160
|
const ms = (x) => x.toFixed(3);
|
|
154
161
|
const count = (n) => n.toLocaleString("en-US");
|
|
155
162
|
const detail = (v) => Array.isArray(v) ? v.join(", ") : v;
|
|
156
163
|
/** the details a plan line carries without VERBOSE: everything but the projections and filters, which VERBOSE adds */
|
|
157
164
|
const VERBOSE_ONLY = new Set(["Projections", "Filters", "Table", "Type"]);
|
|
165
|
+
/** a filtered table says no row count: the estimate is every row's, which the role does not read */
|
|
158
166
|
function copyLine(c) {
|
|
159
|
-
const rows = c.rows === null ? "rows unknown" : `about ${count(c.rows)} rows`;
|
|
160
|
-
return `${c.table} (${rows}
|
|
167
|
+
const rows = c.filter !== undefined ? "" : c.rows === null ? "rows unknown; " : `about ${count(c.rows)} rows; `;
|
|
168
|
+
return `${c.table} (${rows}${keptText(c.facts, undefined, c.filter === undefined)}${c.filter !== undefined ? `; ${c.filter}` : ""})`;
|
|
161
169
|
}
|
|
162
170
|
/** EXPLAIN as Postgres's text format: one line a row */
|
|
163
171
|
export function explainText(e, o) {
|
|
164
172
|
const lines = [];
|
|
165
|
-
lines.push(e.copies.length === 0 ? "Mason: answered
|
|
166
|
-
: e.copies.length === 1 ? `Mason: answered
|
|
173
|
+
lines.push(e.copies.length === 0 ? "Mason: answered reading no synced table"
|
|
174
|
+
: e.copies.length === 1 ? `Mason: answered from the synced table ${copyLine(e.copies[0])}` : "Mason: answered from the synced tables");
|
|
167
175
|
if (e.copies.length > 1)
|
|
168
176
|
for (const c of e.copies)
|
|
169
177
|
lines.push(` ${copyLine(c)}`);
|
|
170
|
-
lines.push(` Sent to
|
|
171
|
-
lines.push(` Computed by Mason after
|
|
178
|
+
lines.push(` Sent to the engine: ${e.sql}`);
|
|
179
|
+
lines.push(` Computed by Mason after the engine: ${e.computed.length === 0 ? "nothing" : e.computed.join("; ")}`);
|
|
172
180
|
if (e.functions.length > 0)
|
|
173
|
-
lines.push(` Mason's functions
|
|
181
|
+
lines.push(` Mason's functions the engine calls: ${e.functions.join(", ")}`);
|
|
174
182
|
for (const name of o.ignored)
|
|
175
|
-
lines.push(` Note: ${name} has no
|
|
183
|
+
lines.push(` Note: ${name} has no meaning to the engine: ignored`);
|
|
184
|
+
if (e.filtered === true)
|
|
185
|
+
lines.push(` Note: ${FILTERED}`);
|
|
176
186
|
if (e.analyzed?.profiled === false)
|
|
177
187
|
lines.push(` Note: ${UNPROFILED}`);
|
|
178
188
|
else if (e.analyzed !== undefined && o.timing)
|
|
179
|
-
lines.push(" Note: an operator's thread time is
|
|
189
|
+
lines.push(" Note: an operator's thread time is the engine's, summed over the threads that ran it and the operators under it: a parallel one's exceeds the Execution Time");
|
|
180
190
|
const walk = (n, depth) => {
|
|
181
191
|
const lead = depth === 0 ? "" : `${" ".repeat(6 * depth - 4)}-> `;
|
|
182
192
|
const on = scannedTable(n);
|
|
@@ -193,8 +203,8 @@ export function explainText(e, o) {
|
|
|
193
203
|
e.plan.forEach((n) => walk(n, 0));
|
|
194
204
|
if (e.analyzed !== undefined && o.summary) {
|
|
195
205
|
const p = e.analyzed.planning, x = e.analyzed.execution;
|
|
196
|
-
lines.push(`Planning Time: ${ms(p.analysisMs + p.rewriteMs + p.duckdbMs)} ms (Postgres's analysis ${ms(p.analysisMs)}, Mason's rewrite ${ms(p.rewriteMs)},
|
|
197
|
-
lines.push(`Execution Time: ${ms(x.duckdbMs + x.masonMs)} ms (
|
|
206
|
+
lines.push(`Planning Time: ${ms(p.analysisMs + p.rewriteMs + p.duckdbMs)} ms (Postgres's analysis ${ms(p.analysisMs)}, Mason's rewrite ${ms(p.rewriteMs)}, the engine's planning ${ms(p.duckdbMs)})`);
|
|
207
|
+
lines.push(`Execution Time: ${ms(x.duckdbMs + x.masonMs)} ms (the engine ${ms(x.duckdbMs)}, Mason's conversion and exact numerics ${ms(x.masonMs)})`);
|
|
198
208
|
}
|
|
199
209
|
return lines;
|
|
200
210
|
}
|
|
@@ -208,31 +218,31 @@ export function explainJson(e, o) {
|
|
|
208
218
|
"Node Type": operatorName(n.type),
|
|
209
219
|
...(on === null ? {} : { "Relation Name": on.name, Schema: on.schema }),
|
|
210
220
|
...(o.costs && n.estimate !== null ? { "Plan Rows": n.estimate } : {}),
|
|
211
|
-
...(n.actualRows === undefined ? {} : { "Actual Rows": n.actualRows, "Actual Loops": 1, ...(o.timing && n.threadMs !== undefined ? { "
|
|
212
|
-
...Object.fromEntries(Object.entries(n.extra).filter(([k]) => o.verbose ? k !== "Table" : !VERBOSE_ONLY.has(k)).map(([k, v]) => [`
|
|
221
|
+
...(n.actualRows === undefined ? {} : { "Actual Rows": n.actualRows, "Actual Loops": 1, ...(o.timing && n.threadMs !== undefined ? { "Engine Thread Time": Number(ms(n.threadMs)) } : {}) }),
|
|
222
|
+
...Object.fromEntries(Object.entries(n.extra).filter(([k]) => o.verbose ? k !== "Table" : !VERBOSE_ONLY.has(k)).map(([k, v]) => [`Engine ${k}`, v])),
|
|
213
223
|
...(n.children.length === 0 ? {} : { Plans: n.children.map(node) }),
|
|
214
224
|
};
|
|
215
225
|
};
|
|
216
226
|
// Postgres's plan has one root; DuckDB's has one too (a statement's), kept as a list should it ever have more
|
|
217
227
|
const roots = e.plan.map(node);
|
|
218
228
|
const top = {
|
|
219
|
-
Plan: roots.length === 1 ? roots[0] : { "Node Type": "
|
|
229
|
+
Plan: e.filtered === true ? { "Node Type": "Plan Not Shown" } : roots.length === 1 ? roots[0] : { "Node Type": "Engine Plans", Plans: roots },
|
|
220
230
|
Mason: {
|
|
221
|
-
"Answered By": "
|
|
222
|
-
|
|
223
|
-
"Sent To
|
|
224
|
-
"Computed After
|
|
225
|
-
"Mason Functions
|
|
226
|
-
...(o.ignored.length === 0 ? {} : { Ignored: o.ignored.map((n) => `${n} has no
|
|
227
|
-
...(e.analyzed?.profiled === false ? { Note: UNPROFILED } : {}),
|
|
231
|
+
"Answered By": "the engine",
|
|
232
|
+
Tables: e.copies.map((c) => ({ Table: c.table, ...(c.filter === undefined ? { "Estimated Rows": c.rows } : {}), ...keptJson(c.facts, c.filter === undefined), ...(c.filter !== undefined ? { "Row Filter": c.filter } : {}) })),
|
|
233
|
+
"Sent To Engine": e.sql,
|
|
234
|
+
"Computed After Engine": e.computed,
|
|
235
|
+
"Mason Functions Engine Calls": e.functions,
|
|
236
|
+
...(o.ignored.length === 0 ? {} : { Ignored: o.ignored.map((n) => `${n} has no meaning to the engine`) }),
|
|
237
|
+
...(e.filtered === true ? { Note: FILTERED } : e.analyzed?.profiled === false ? { Note: UNPROFILED } : {}),
|
|
228
238
|
},
|
|
229
239
|
};
|
|
230
240
|
if (e.analyzed !== undefined && o.summary) {
|
|
231
241
|
const p = e.analyzed.planning, x = e.analyzed.execution;
|
|
232
242
|
top["Planning Time"] = Number(ms(p.analysisMs + p.rewriteMs + p.duckdbMs));
|
|
233
243
|
top["Execution Time"] = Number(ms(x.duckdbMs + x.masonMs));
|
|
234
|
-
top.Mason["Planning"] = { "Postgres Analysis": Number(ms(p.analysisMs)), "Mason Rewrite": Number(ms(p.rewriteMs)), "
|
|
235
|
-
top.Mason["Execution"] = {
|
|
244
|
+
top.Mason["Planning"] = { "Postgres Analysis": Number(ms(p.analysisMs)), "Mason Rewrite": Number(ms(p.rewriteMs)), "Engine Planning": Number(ms(p.duckdbMs)) };
|
|
245
|
+
top.Mason["Execution"] = { Engine: Number(ms(x.duckdbMs)), "Mason Conversion": Number(ms(x.masonMs)) };
|
|
236
246
|
}
|
|
237
247
|
return JSON.stringify([top], null, 2);
|
|
238
248
|
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE COPY'S EXTENSIONS SHIP WITH MASON. The copy reads the source through DuckDB's `postgres` extension
|
|
3
|
+
* (src/duck/sync.ts attach), and a statement that casts a value to inet needs DuckDB's `inet`; DuckDB bundles
|
|
4
|
+
* neither, and by default downloads one from its extension repository at the first statement that needs it. Mason
|
|
5
|
+
* ships both instead, ONE npm PACKAGE PER PLATFORM (`@mason-data/extensions-<platform>`), as DuckDB's node bindings ship
|
|
6
|
+
* their binary: an optional dependency of @mason-data/cli whose `os` / `cpu` / `libc` make npm install the one this
|
|
7
|
+
* machine runs, built by cli/extensions.ts from the files' checksums pinned in cli/extensions.json. The image installs
|
|
8
|
+
* the same package (docker/mason-data.Dockerfile).
|
|
9
|
+
*
|
|
10
|
+
* - A package found (`findExtensions`): the instance's `extension_directory` IS the package's folder and nothing is
|
|
11
|
+
* ever installed (`extensionSettings`): `LOAD postgres` reads it, an inet cast autoloads it from there, and
|
|
12
|
+
* autoinstall is off, so no statement downloads anything. DuckDB checks each file's signature at LOAD
|
|
13
|
+
* (`allow_unsigned_extensions` stays off: the official extensions are signed), and LOAD writes nothing, so the
|
|
14
|
+
* folder may be read-only (a global npm install, the image's /app).
|
|
15
|
+
* - None (`npm install --omit=optional`, a platform no package is built for — musl, Windows on arm64 — or a checkout):
|
|
16
|
+
* DuckDB's own way, said with the reason: the first copy installs the extension from the internet into
|
|
17
|
+
* ~/.duckdb/extensions (src/duck/sync.ts attach). `mason doctor` says which (`probeExtensions`).
|
|
18
|
+
*
|
|
19
|
+
* tests/duck-extensions.test.ts holds the resolver per platform over a fake package; the integration test
|
|
20
|
+
* (tests/integration/extensions.itest.ts) holds a copy and an inet cast to the package with nothing downloaded.
|
|
21
|
+
*/
|
|
22
|
+
import * as fs from "node:fs";
|
|
23
|
+
import * as path from "node:path";
|
|
24
|
+
import { createRequire } from "node:module";
|
|
25
|
+
import { DuckDBInstance, version } from "@duckdb/node-api";
|
|
26
|
+
/** Each platform a package is built for: Node's `<platform>-<arch>` (the package's name) and DuckDB's name for it (the
|
|
27
|
+
* folder of the extension repository, and of `extension_directory`). Linux is glibc's: musl has no package. */
|
|
28
|
+
export const EXTENSION_PLATFORMS = {
|
|
29
|
+
"darwin-arm64": "osx_arm64",
|
|
30
|
+
"darwin-x64": "osx_amd64",
|
|
31
|
+
"linux-arm64": "linux_arm64",
|
|
32
|
+
"linux-x64": "linux_amd64",
|
|
33
|
+
"win32-x64": "windows_amd64",
|
|
34
|
+
};
|
|
35
|
+
/** The extensions each package holds, by the name DuckDB installs them under (`INSTALL postgres` is `postgres_scanner`). */
|
|
36
|
+
export const EXTENSION_NAMES = ["postgres_scanner", "inet"];
|
|
37
|
+
export const packageName = (p) => `@mason-data/extensions-${p}`;
|
|
38
|
+
/** Where a package keeps the files: `<package>/extensions/<engine>/<DuckDB's platform>/<name>.duckdb_extension`, the
|
|
39
|
+
* layout DuckDB reads `extension_directory` in. */
|
|
40
|
+
export const extensionFile = (root, engine, platform, name) => path.join(root, "extensions", engine, EXTENSION_PLATFORMS[platform], `${name}.duckdb_extension`);
|
|
41
|
+
export function thisHost() {
|
|
42
|
+
// Node's own report names glibc's version where glibc runs; musl (Alpine) has none (detect-libc's first check). Its
|
|
43
|
+
// network part is left out, as detect-libc leaves it: enumerating the interfaces has taken seconds in a container
|
|
44
|
+
let header;
|
|
45
|
+
if (process.platform === "linux") {
|
|
46
|
+
// `excludeNetwork` is Node's (20+), not yet in @types/node
|
|
47
|
+
const report = process.report;
|
|
48
|
+
const network = report.excludeNetwork;
|
|
49
|
+
report.excludeNetwork = true;
|
|
50
|
+
try {
|
|
51
|
+
header = report.getReport().header;
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
report.excludeNetwork = network;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { platform: process.platform, arch: process.arch, musl: process.platform === "linux" && header?.glibcVersionRuntime === undefined };
|
|
58
|
+
}
|
|
59
|
+
/** The platform's key as npm and DuckDB's bindings name it: `linux-x64`, `linux-x64-musl`, `darwin-arm64`. */
|
|
60
|
+
export const hostKey = (h) => `${h.platform}-${h.arch}${h.musl ? "-musl" : ""}`;
|
|
61
|
+
const requireHere = createRequire(import.meta.url);
|
|
62
|
+
/**
|
|
63
|
+
* THE PACKAGE FOR THIS MACHINE, found where Node would find it from this module (the CLI's node_modules, nested or
|
|
64
|
+
* hoisted; the image's /app/node_modules), holding both files for THIS engine. Reads two files' existence: no engine
|
|
65
|
+
* is opened. `resolve` and `host` are a test's.
|
|
66
|
+
*/
|
|
67
|
+
export function findExtensions(o = {}) {
|
|
68
|
+
const host = o.host ?? thisHost();
|
|
69
|
+
const engine = o.engine ?? `v${version().replace(/^v/, "")}`;
|
|
70
|
+
const key = hostKey(host);
|
|
71
|
+
if (!(key in EXTENSION_PLATFORMS)) {
|
|
72
|
+
return { shipped: false, package: null, why: `no package of Mason's extensions is built for ${key} (only ${Object.keys(EXTENSION_PLATFORMS).join(", ")})` };
|
|
73
|
+
}
|
|
74
|
+
const platform = key;
|
|
75
|
+
const name = packageName(platform);
|
|
76
|
+
let manifest;
|
|
77
|
+
try {
|
|
78
|
+
manifest = (o.resolve ?? requireHere.resolve)(`${name}/package.json`);
|
|
79
|
+
}
|
|
80
|
+
catch (e) {
|
|
81
|
+
// not installed is the one answer expected here; anything else is said as it is
|
|
82
|
+
if (e.code !== "MODULE_NOT_FOUND")
|
|
83
|
+
throw e;
|
|
84
|
+
return { shipped: false, package: name, why: `${name} is not installed beside Mason (npm installs it with @mason-data/cli unless optional dependencies are left out: --omit=optional)` };
|
|
85
|
+
}
|
|
86
|
+
const root = path.dirname(manifest);
|
|
87
|
+
let pkgVersion;
|
|
88
|
+
try {
|
|
89
|
+
pkgVersion = String(JSON.parse(fs.readFileSync(manifest, "utf8")).version ?? "");
|
|
90
|
+
}
|
|
91
|
+
catch (e) {
|
|
92
|
+
// a broken install: refused in Mason's words with the way out, never the parser's error
|
|
93
|
+
return { shipped: false, package: name, why: `${name}'s package.json at ${manifest} cannot be read (${e.message}): install @mason-data/cli again` };
|
|
94
|
+
}
|
|
95
|
+
const missing = EXTENSION_NAMES.filter((n) => !fs.existsSync(extensionFile(root, engine, platform, n)));
|
|
96
|
+
if (missing.length > 0) {
|
|
97
|
+
const has = fs.existsSync(path.join(root, "extensions")) ? fs.readdirSync(path.join(root, "extensions")) : [];
|
|
98
|
+
return { shipped: false, package: name, why: `${name} ${pkgVersion} holds no ${missing.join(" or ")} for the engine Mason runs (${engine}; it holds ${has.length ? has.join(", ") : "none"}): ` +
|
|
99
|
+
"install @mason-data/cli again, whose packages are released together" };
|
|
100
|
+
}
|
|
101
|
+
return { shipped: true, package: name, version: pkgVersion, directory: path.join(root, "extensions"), platform };
|
|
102
|
+
}
|
|
103
|
+
/** The instance's settings for what was found: with the package, its folder and no download, ever; without, DuckDB's
|
|
104
|
+
* defaults (the first copy installs into ~/.duckdb/extensions). */
|
|
105
|
+
export function extensionSettings(x) {
|
|
106
|
+
return x.shipped ? { extension_directory: x.directory, autoinstall_known_extensions: "false", autoload_known_extensions: "true" } : {};
|
|
107
|
+
}
|
|
108
|
+
/** The statement the copy loads its postgres extension with: from the package, a LOAD; without one, DuckDB's INSTALL
|
|
109
|
+
* first (a download, once a machine). */
|
|
110
|
+
export const loadPostgresSql = (x) => x.shipped ? "LOAD postgres" : "INSTALL postgres; LOAD postgres;";
|
|
111
|
+
/** What a copy that could not load the postgres extension says (src/duck/sync.ts attach), with the way out. */
|
|
112
|
+
export function extensionFailure(x, error) {
|
|
113
|
+
return x.shipped
|
|
114
|
+
? `Mason could not load its postgres extension from ${x.package} ${x.version} (${x.directory}): ${error}. Install @mason-data/cli again; mason doctor checks it`
|
|
115
|
+
: `Mason could not load its postgres extension: ${error}. ${x.why}, so the first sync on a machine downloads it ` +
|
|
116
|
+
"(then ~/.duckdb/extensions): connect to the internet once, and the sync is tried again at the next pass";
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* THE PROBE `mason doctor` SAYS (the contract answers a probe, never a file's existence): with the package, both
|
|
120
|
+
* extensions LOADED by an instance of their own with the run's settings — the signature and the platform checked as
|
|
121
|
+
* the run's will be; without one, whether DuckDB has them installed in ~/.duckdb/extensions already, and `ok` null when
|
|
122
|
+
* it has not (the first copy will download them: fine online, which no probe can tell). Downloads nothing.
|
|
123
|
+
*/
|
|
124
|
+
export async function probeExtensions(x = findExtensions()) {
|
|
125
|
+
let instance;
|
|
126
|
+
try {
|
|
127
|
+
instance = await DuckDBInstance.create(":memory:", { autoinstall_known_extensions: "false", ...extensionSettings(x) });
|
|
128
|
+
}
|
|
129
|
+
catch (e) {
|
|
130
|
+
return { ok: null, detail: `could not open an engine to check them: ${e.message}` };
|
|
131
|
+
}
|
|
132
|
+
try {
|
|
133
|
+
const c = await instance.connect();
|
|
134
|
+
try {
|
|
135
|
+
if (x.shipped) {
|
|
136
|
+
for (const n of EXTENSION_NAMES) {
|
|
137
|
+
try {
|
|
138
|
+
await c.run(`LOAD ${n}`);
|
|
139
|
+
}
|
|
140
|
+
catch (e) {
|
|
141
|
+
return { ok: false, detail: `${x.package} ${x.version} is installed, and its ${n.replace("_scanner", "")} extension does not load: ${e.message.split("\n")[0]}. Install @mason-data/cli again` };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return { ok: true, detail: `postgres and inet ship with Mason (${x.package} ${x.version}): nothing is downloaded` };
|
|
145
|
+
}
|
|
146
|
+
const rows = (await c.runAndReadAll(`SELECT extension_name, installed FROM duckdb_extensions() WHERE extension_name IN (${EXTENSION_NAMES.map((n) => `'${n}'`).join(", ")})`)).getRowObjects();
|
|
147
|
+
const absent = EXTENSION_NAMES.filter((n) => !rows.some((r) => r.extension_name === n && r.installed === true)).map((n) => n.replace("_scanner", ""));
|
|
148
|
+
if (absent.length === 0)
|
|
149
|
+
return { ok: true, detail: `postgres and inet were downloaded on this machine before (~/.duckdb/extensions), not shipped: ${x.why}` };
|
|
150
|
+
// not a failure: online, the first copy downloads them; whether this machine will be online then, no probe can tell
|
|
151
|
+
return { ok: null, detail: `${absent.join(" and ")} not on this machine yet, so the first sync downloads them and needs the internet once: ${x.why}` };
|
|
152
|
+
}
|
|
153
|
+
finally {
|
|
154
|
+
c.closeSync();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
finally {
|
|
158
|
+
instance.closeSync();
|
|
159
|
+
}
|
|
160
|
+
}
|
|
@@ -23,7 +23,7 @@ import { compare, div, evaluateText, NumericError, numericText, parseNumeric } f
|
|
|
23
23
|
import { toCharNumber, toCharTime } from "./to-char.js";
|
|
24
24
|
import { atTimeZoneOf } from "../lib/timezone.js";
|
|
25
25
|
import { gapMark } from "./gaps.js";
|
|
26
|
-
import {
|
|
26
|
+
import { beat, cancelTable, isCancelled, slotOf } from "./cancel.js";
|
|
27
27
|
// ------------------------------------------------------------------------------------------------- case mapping
|
|
28
28
|
/** one character's simple mapping, as towupper / towlower give it: a character whose full mapping is several (`ß`,
|
|
29
29
|
* `ʼn`) keeps itself */
|
|
@@ -496,6 +496,7 @@ function scalar(conn, name, params, returns, body, opts = {}) {
|
|
|
496
496
|
const vectors = all.slice(1);
|
|
497
497
|
try {
|
|
498
498
|
for (let r = 0; r < input.rowCount; r++) {
|
|
499
|
+
beat(cancels); // one vector of wide documents is seconds of this loop: the watch must see it run
|
|
499
500
|
const args = vectors.map((v) => v.getItem(r));
|
|
500
501
|
out.setItem(r, opts.nulls !== "pass" && args.some((a) => a === null) ? null : body(args));
|
|
501
502
|
}
|
|
@@ -638,17 +639,20 @@ function icuKeysOf(v) {
|
|
|
638
639
|
* EVERY CALL CARRIES ITS SESSION, FIRST (the test server: a client leaving mid-statement hung the process
|
|
639
640
|
* for good). A scalar function runs on the thread that registered it — the executor worker (src/duck/executor.ts) —
|
|
640
641
|
* while an executor thread waits for it, so that thread can never wait for the executor — and closing a result or a
|
|
641
|
-
* connection does, to cancel the executor's tasks.
|
|
642
|
-
*
|
|
643
|
-
*
|
|
644
|
-
*
|
|
645
|
-
* stopped (src/duck/store.ts `cleanUp`; a deadlock under load). The table is shared memory: the listener's thread writes it
|
|
642
|
+
* connection does, to cancel the executor's tasks. So such a statement is cancelled through its session as well as
|
|
643
|
+
* interrupted: its session's cell of the CANCEL TABLE (src/duck/cancel.ts) says it is cancelled, the statement's next call
|
|
644
|
+
* raises, and its result is closed only once DuckDB ended the query off this thread (src/duck/store.ts `cleanUp`; a
|
|
645
|
+
* deadlock under load). The table is shared memory: the listener's thread writes it
|
|
646
646
|
* while this one is busy with the statement's calls. The rewrite (src/duck/rewrite.ts `fn`) passes
|
|
647
647
|
* `getvariable('__mason_session')` as every `__mason_*` function's first argument, which DuckDB folds to the session's
|
|
648
648
|
* literal at bind time; the session runner sets the variable on its connection (its slot, `s<n>`). A connection without
|
|
649
649
|
* it (the writer, a copy's) passes NULL: never cancelled this way.
|
|
650
650
|
*/
|
|
651
651
|
export const SESSION_VARIABLE = "__mason_session";
|
|
652
|
+
/** the session's settings, name to value, as `current_setting(name)` answers them (src/api/postgres/server.ts
|
|
653
|
+
* `duckSession`): set on a session's connection before each statement that finds them changed (src/duck/store.ts
|
|
654
|
+
* `settle`), read by the rewrite's `settingRead` (src/duck/rewrite.ts) */
|
|
655
|
+
export const SETTINGS_VARIABLE = "__mason_settings";
|
|
652
656
|
export const CANCELLED = "mason[57014]: canceling statement due to user request";
|
|
653
657
|
/** how many vectors each function was called with (a call is a hop to the main thread): what a test or a measurement
|
|
654
658
|
* reads to know whether a statement reached a function at all (the rewrite answers a `->>` natively where it can) */
|
|
@@ -660,16 +664,11 @@ export const FUNCTION_ROWS = new Map();
|
|
|
660
664
|
let cancels = cancelTable();
|
|
661
665
|
export function useCancelTable(t) { cancels = t; }
|
|
662
666
|
export function cancelsTable() { return cancels; }
|
|
663
|
-
/**
|
|
664
|
-
*
|
|
665
|
-
* `cleanUp`) */
|
|
666
|
-
const lastCallAt = new Float64Array(CANCEL_SLOTS);
|
|
667
|
-
export function msSinceCall(slot) { return performance.now() - (lastCallAt[slot] ?? 0); }
|
|
668
|
-
/** the session's vector (the first argument, a constant): raises when the session's statement is cancelled */
|
|
667
|
+
/** the session's vector (the first argument, a constant): raises when the session's statement is cancelled. Every call
|
|
668
|
+
* starts here, so every call beats (src/duck/cancel.ts) */
|
|
669
669
|
function cancelled(info, vectors, rows) {
|
|
670
|
+
beat(cancels);
|
|
670
671
|
const slot = rows > 0 ? slotOf(text(vectors[0]?.getItem(0))) : null;
|
|
671
|
-
if (slot !== null)
|
|
672
|
-
lastCallAt[slot] = performance.now();
|
|
673
672
|
if (slot === null || !isCancelled(cancels, slot))
|
|
674
673
|
return false;
|
|
675
674
|
info.setError(CANCELLED);
|
package/dist/src/duck/gaps.js
CHANGED
|
@@ -17,11 +17,12 @@
|
|
|
17
17
|
*/
|
|
18
18
|
export const GAPS = {
|
|
19
19
|
// ---------------------------------------------------------------------------------------------- refused
|
|
20
|
-
unverified: { kind: "refused", what: "a function, operator, cast, type or construct
|
|
20
|
+
unverified: { kind: "refused", what: "a function, operator, cast, type or construct Mason has not yet verified against Postgres", postgres: "answers", mason: "refused, naming each piece", wayOut: "write the statement without it; report it to have it verified", heldBy: "tests/duck-wire.test.ts" },
|
|
21
21
|
"explain-option": { kind: "refused", what: "EXPLAIN (FORMAT XML), (FORMAT YAML), (GENERIC_PLAN)", postgres: "the plan in XML or YAML; a plan for unbound parameters", mason: "refused by name: EXPLAIN is DuckDB's plan, in Postgres's text or JSON (docs/DESIGN.md)", wayOut: "FORMAT TEXT or FORMAT JSON; bind the parameters' values and EXPLAIN that" },
|
|
22
|
+
"set-config-forms": { kind: "refused", what: "set_config() anywhere but alone in a SELECT of no table with literal arguments: beside other columns or a FROM, in a subquery, with an expression, a parameter or NULL, under EXPLAIN", postgres: "sets the parameter (for the transaction, under is_local) and answers its value", mason: "refused (SELECT set_config('name', 'value', false) alone is the session's SET, and with true its SET LOCAL, answered with its value as SHOW answers it)", wayOut: "SET name = 'value', or SELECT set_config('name', 'value', false) on its own" },
|
|
22
23
|
"setop-ordinal-star": { kind: "refused", what: "ORDER BY a column number of a UNION / INTERSECT / EXCEPT whose first SELECT lists *", postgres: "sorts by that column", mason: "refused", wayOut: "ORDER BY the column's name" },
|
|
23
24
|
"with-ties-distinct": { kind: "refused", what: "FETCH FIRST … WITH TIES over SELECT DISTINCT", postgres: "answers", mason: "refused (DuckDB has no WITH TIES; Mason's rank over DISTINCT is not Postgres's)", wayOut: "select the DISTINCT rows in a subquery and FETCH … WITH TIES from it" },
|
|
24
|
-
"collation-unknown": { kind: "refused", what: "COLLATE with a collation other than C, POSIX, default or an ICU locale
|
|
25
|
+
"collation-unknown": { kind: "refused", what: "COLLATE with a collation other than C, POSIX, default or an ICU locale the engine has", postgres: "sorts and compares under it", mason: "refused", wayOut: "COLLATE \"C\" for byte order, or no COLLATE for the database's" },
|
|
25
26
|
"like-any-array": { kind: "refused", what: "LIKE / ILIKE ANY (or ALL) over an array that is not written out", postgres: "answers", mason: "refused", wayOut: "write the patterns as ARRAY['…', '…'], or OR the LIKEs" },
|
|
26
27
|
"similar-to-forms": { kind: "refused", what: "SIMILAR TO with a pattern or ESCAPE that is not a literal, its substring markers, or substring(… SIMILAR … ESCAPE …)", postgres: "answers", mason: "refused", wayOut: "a literal pattern, or ~ with a POSIX regular expression" },
|
|
27
28
|
"array-remove-type": { kind: "refused", what: "array_remove of an array of a type other than text, integers, floats, booleans, dates or timestamps", postgres: "answers", mason: "refused", wayOut: "array_remove an array of one of those types" },
|
|
@@ -55,7 +56,7 @@ export const GAPS = {
|
|
|
55
56
|
"inet-order": { kind: "refused", what: "ORDER BY, <, >, min or max over an inet", postgres: "orders addresses by network, then host", mason: "refused (equality, IS NULL and the value itself are answered)", wayOut: "sort or compare another column" },
|
|
56
57
|
"citext-forms": { kind: "refused", what: "GROUP BY, DISTINCT, min, max, a set operation, or = ANY (an array) over a citext", postgres: "groups and sorts without regard to case, answering one spelling", mason: "refused (=, <>, <, >, IN (…), LIKE and ORDER BY are answered, case-insensitively)", wayOut: "GROUP BY lower(x), or x::text where case matters" },
|
|
57
58
|
"bytea-forms": { kind: "refused", what: "encode / decode to a format other than hex, base64 or escape, or one not written as a literal; convert_to / convert_from of an encoding other than UTF8", postgres: "answers", mason: "refused", wayOut: "encode(x, 'hex'), 'base64' or 'escape'; the UTF8 encoding" },
|
|
58
|
-
"row-value": { kind: "refused", what: "a row as a value outside the SELECT list, a cast to text and the json functions (compared, tested with IS NULL, grouped, sorted); a row of a row; a row of a table Mason
|
|
59
|
+
"row-value": { kind: "refused", what: "a row as a value outside the SELECT list, a cast to text and the json functions (compared, tested with IS NULL, grouped, sorted); a row of a row; a row of a table Mason synced in part", postgres: "compares, tests and groups a row by its columns", mason: "refused", wayOut: "compare, test or sort its columns" },
|
|
59
60
|
"zone-abbreviation": { kind: "refused", what: "a time zone written as an abbreviation or a POSIX zone (`CST`, `IST`, `UTC+3`, `+05`) in AT TIME ZONE or date_trunc", postgres: "a fixed offset (`CST` is -06 all year; `UTC+3` is three hours WEST)", mason: "refused, written or held by a row (DuckDB reads several as another zone: `CST` as Chicago's, `IST` as India's, `UTC+3` as east)", wayOut: "an IANA zone name (America/Chicago, Asia/Kolkata), in any case; UT, Z and UTC are UTC", heldBy: "tests/duck-wire.test.ts" },
|
|
60
61
|
"volatile-window": { kind: "refused", what: "a window ordered or partitioned by random() or another volatile function (a random row per group)", postgres: "a new value per row", mason: "refused (DuckDB refuses a volatile expression there, and computes a deterministic one once for every row)", wayOut: "SELECT …, row_number() OVER (ORDER BY r) FROM (SELECT *, random() AS r FROM t) AS s" },
|
|
61
62
|
"numeric-special": { kind: "refused", what: "arithmetic on a numeric NaN or Infinity read from a text", postgres: "NaN and ±Infinity by its own rules (NaN + 1 is NaN)", mason: "refused when a row holds one", wayOut: "filter the special values out first (x <> 'NaN')" },
|
|
@@ -67,7 +68,7 @@ export const GAPS = {
|
|
|
67
68
|
"relative-time-literal": { kind: "refused", what: "a relative time combined with anything else ('today 10:00', 'now'::date + … written as one literal), or read from a row as a date or a time", postgres: "the transaction's start (a row's value: when the row is read)", mason: "refused (a bare 'now', 'today', 'tomorrow', 'yesterday' literal is answered: the transaction's start at the Parse)", wayOut: "current_date + time '10:00', now(), current_date + 1" },
|
|
68
69
|
// ---------------------------------------------------------------------------------------------- differs
|
|
69
70
|
"limit-errors": { kind: "differs", what: "a value that raises (`'x'::int`, a bad JSON number) in a row a LIMIT leaves out: in the select list, where no index gives the ORDER BY; or in the argument of a LEFT JOIN of a list's rows ON TRUE (unnest(ARRAY[…, 'x'::int]), jsonb_each of a non-object), under a LIMIT with no ORDER BY", postgres: "the select list's raises: its plan sorts every row, its select list computed for each (through an index it stops at the limit and answers); the lateral's answers: it stops at the limit", mason: "the select list's answers: a sorted, limited select list is computed for the rows the limit keeps (src/duck/late.ts), as Postgres's index scan does; the lateral's raises: DuckDB computes its argument for a whole vector of rows (2,048) before the limit stops it", wayOut: "none needed for the select list's answer; filter the bad rows out to have neither raise" },
|
|
70
|
-
"limit-lateral-filter": { kind: "differs", what: "a LEFT JOIN of a json document's rows ON TRUE (json_each, jsonb_array_elements, …) under a LIMIT, or a row count an Execute sets on the wire, and no ORDER BY, with a filter above the join that
|
|
71
|
+
"limit-lateral-filter": { kind: "differs", what: "a LEFT JOIN of a json document's rows ON TRUE (json_each, jsonb_array_elements, …) under a LIMIT, or a row count an Execute sets on the wire, and no ORDER BY, with a filter above the join that is not pushed down to the base: a WHERE on the lateral's own column (d.key <> 'n'), one holding a bound parameter ($1) in a SELECT around the join, or one in a SELECT that reads the join through a CTE", postgres: "the first rows of the base, in its order, each row's members in theirs", mason: "the engine plans the lateral under such a filter as a delimiter join and hands the rows back in an order of its own, so the LIMIT keeps other rows (a filter on the base in the join's own SELECT, literal or parameter, or a literal one around it through a derived table, keeps Postgres's)", wayOut: "ORDER BY the base's key before the LIMIT, or filter the base in the SELECT that holds the join", heldBy: "tests/duck-wire.test.ts" },
|
|
71
72
|
"limit-lateral-argument": { kind: "differs", what: "a LEFT JOIN of a list's rows ON TRUE (unnest, json_each, jsonb_array_elements, …) under a LIMIT, or a row count an Execute sets on the wire, and no ORDER BY, whose argument holds a subquery (unnest(ARRAY(SELECT …))), or reads two or more columns of the row and no array (CASE WHEN t.kind = 'doc' THEN t.body END) in a SELECT whose list is a bare * or inside a join with a name of its own ((a LEFT JOIN … ON TRUE) AS j)", postgres: "the first rows of the base, in its order, each row's members in theirs", mason: "DuckDB plans such a lateral as a delimiter join and hands the rows back in an order of its own, so the LIMIT keeps other rows (elsewhere an argument of two columns is computed in a step of its own before the lateral, which a * would read as a column; a subquery is a delimiter join in every form)", wayOut: "name the columns instead of *, or ORDER BY the base's key before the LIMIT" },
|
|
72
73
|
"window-partition-order": { kind: "differs", what: "a window function that reads its frame in order (row_number, lag, lead, first_value, last_value, nth_value, ntile) over a window with PARTITION BY and no ORDER BY", postgres: "a partition's rows in the order its sort hands them back: in practice the order they came in", mason: "a partition's rows in the order DuckDB's partitioning hands them back, another one", wayOut: "ORDER BY in the window (OVER (PARTITION BY a ORDER BY b))" },
|
|
73
74
|
"mixed-scale-numeric": { kind: "differs", what: "a numeric column whose values have several scales: an integer UNION ALL a numeric, lead(x, 1, 0) beside numeric(2,1) values, an unconstrained column holding 1 and 1.50", postgres: "each value its own scale: 1, 1.5", mason: "one scale per column: 1.0, 1.5", wayOut: "cast to numeric(p, s), or round(x, s)" },
|
|
@@ -80,11 +81,12 @@ export const GAPS = {
|
|
|
80
81
|
"jsonb-equal-text": { kind: "differs", what: "GROUP BY or DISTINCT of jsonb values that are equal but written apart ('1.0' and '1.00', '{\"a\": 1}' and '{\"a\": 1.0}')", postgres: "one group, written as the value its plan reads first", mason: "one group, written as the value DuckDB reads first: either may be any of them", wayOut: "GROUP BY j::text where the text must decide", heldBy: "tests/duck-jsonb-order.test.ts" },
|
|
81
82
|
"char-n-minmax": { kind: "differs", what: "greatest / least of a char(n) beside a text", postgres: "compares without the padding, answers the first of equal values", mason: "compares the padded text", wayOut: "cast the char(n) to text first" },
|
|
82
83
|
"nested-exact-numeric": { kind: "differs", what: "an expression over a numeric a subquery computed exactly (a division, an avg, a round), outside that subquery: SELECT x * 2 FROM (SELECT a / 3 AS x …)", postgres: "exact digits at every step", mason: "the subquery's value as DuckDB computed it (a double's 15 digits) where the outer query computes with it; passed through as it is (SELECT *, s.x), it is exact", wayOut: "compute it in the outermost query, or round(…, n) inside" },
|
|
84
|
+
"show-all": { kind: "differs", what: "SHOW ALL", postgres: "every server setting with the source database's values", mason: "every setting the listener's catalog Postgres has, with this session's values laid over (search_path, TimeZone, ssl, …): Mason's settings, never your source database's", wayOut: "ask the source database for its settings (SHOW ALL or SHOW <name> there): Mason answers its own" },
|
|
83
85
|
// ---------------------------------------------------------------------------------------------- limits
|
|
84
|
-
|
|
85
|
-
"poll-full-scan": { kind: "limit", what: "keeping current by the poll (
|
|
86
|
-
"writes": { kind: "limit", what: "INSERT, UPDATE, DELETE, DDL, a read-write transaction", postgres: "answers", mason: "refused by name: the
|
|
87
|
-
"numeric-over-38-digits": { kind: "limit", what: "a numeric of more than 38 digits, or holding NaN or Infinity: a column, or a constant ('1e400'::numeric) inside
|
|
86
|
+
roles: { kind: "limit", what: "access", postgres: "roles, grants on tables and columns, row-level security", mason: "a role (mason/roles.yml) grants a user tables and columns, held on the analysed tree and refused as Postgres refuses them (42501 permission denied for table …); information_schema and mason mcp list a role what it may read, pg_catalog still lists every synced column; a role's where: or through: filters a table's rows by the user's attributes (mason user add --set), every read of it filtered on the analysed tree and held by Postgres row security in the catalog behind a security barrier as a Postgres policy is (a statement's own expressions run only on the rows the filter keeps; timing is not guaranteed), a whole row or TABLESAMPLE of a filtered table refused, so is a grouping set relying on its key for a column it does not group, EXPLAIN of a statement reading one said in Mason's lines alone with no engine plan and EXPLAIN ANALYZE of it refused (either would say what the engine knows of every row), a user missing a value refused (42501) at every statement; a user with no role reads as mason/roles.yml's default: role, nothing when the file names none, every synced table when there is no file; a token reads every synced table", wayOut: "mason user add <name> --role <role> --set <attribute>=<value>; name the filtered table's columns in place of its whole row; read the full plan as a user with no row filter", heldBy: "tests/duck-roles-wire.test.ts" },
|
|
87
|
+
"poll-full-scan": { kind: "limit", what: "keeping current by the poll (a table's sync: poll <interval>)", postgres: "-", mason: "the xmin poll scans its table whole, every poll (no index covers xmin): ~13–17 s for 13 GiB", wayOut: "logical replication for that table: sync: replication on its entry (mason add <table> --sync replication)" },
|
|
88
|
+
"writes": { kind: "limit", what: "INSERT, UPDATE, DELETE, DDL, a read-write transaction", postgres: "answers", mason: "refused by name: the synced data is read-only (a read-write transaction: 25006)", wayOut: "write to the source database; the synced data follows" },
|
|
89
|
+
"numeric-over-38-digits": { kind: "limit", what: "a numeric of more than 38 digits, or holding NaN or Infinity: a column, or a constant ('1e400'::numeric) inside Mason's arithmetic or a WHERE", postgres: "exact", mason: "a double (15 digits), said in the log for a column; a constant the outermost SELECT answers is exact", wayOut: "numeric(p, s) with p ≤ 38" },
|
|
88
90
|
"numeric-array": { kind: "limit", what: "an unconstrained numeric[] column", postgres: "exact elements", mason: "double precision elements (15 digits), said in the log", wayOut: "numeric(p, s)[]" },
|
|
89
91
|
"numeric-parameter": { kind: "limit", what: "a numeric parameter ($1) compared in a WHERE", postgres: "exact", mason: "a double (exact to 15 digits)", wayOut: "a literal" },
|
|
90
92
|
"log-literals": { kind: "limit", what: ".mason/mason.log's statement lines", postgres: "-", mason: "each statement DuckDB ran, whole, literals included", wayOut: "keep the log private" },
|
|
@@ -92,3 +94,10 @@ export const GAPS = {
|
|
|
92
94
|
/** the words a runtime refusal carries for the oracle and a reader: `Mason gap <id>` */
|
|
93
95
|
export const gapMark = (id) => `Mason gap ${id}`;
|
|
94
96
|
export const GAP_MARK = /\bMason gap ([a-z0-9-]+)/;
|
|
97
|
+
/** The entry a runtime refusal's TEXT names (`gapMark`), or null: only for a refusal raised inside DuckDB (a Mason
|
|
98
|
+
* function's error, src/duck/functions.ts), which crosses DuckDB as its message alone (src/duck/store.ts pgErrorOf);
|
|
99
|
+
* a refusal built outside it carries its id on the error (src/api/postgres/server.ts gapRefusal). */
|
|
100
|
+
export function gapIn(text) {
|
|
101
|
+
const id = GAP_MARK.exec(text)?.[1];
|
|
102
|
+
return id !== undefined && Object.hasOwn(GAPS, id) ? id : null;
|
|
103
|
+
}
|