@mason-data/cli 0.0.0-stage → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +209 -2
- package/dist/src/api/error.js +18 -0
- package/dist/src/api/postgres/catalog-worker.js +92 -0
- package/dist/src/api/postgres/catalog.js +442 -0
- package/dist/src/api/postgres/encode.js +541 -0
- package/dist/src/api/postgres/explain.js +143 -0
- package/dist/src/api/postgres/listener.js +60 -0
- package/dist/src/api/postgres/parser-worker.js +58 -0
- package/dist/src/api/postgres/parser.js +121 -0
- package/dist/src/api/postgres/plain.js +33 -0
- package/dist/src/api/postgres/protocol.js +404 -0
- package/dist/src/api/postgres/schema.js +54 -0
- package/dist/src/api/postgres/server.js +1670 -0
- package/dist/src/api/postgres/translate.js +248 -0
- package/dist/src/api/request-log.js +50 -0
- package/dist/src/cdc/pg-wire.js +482 -0
- package/dist/src/cdc/pgoutput.js +107 -0
- package/dist/src/cdc/poll.js +67 -0
- package/dist/src/cdc/source.js +65 -0
- package/dist/src/cli/background.js +174 -0
- package/dist/src/cli/command.js +172 -0
- package/dist/src/cli/commands.js +1145 -0
- package/dist/src/cli/control.js +36 -0
- package/dist/src/cli/dev.js +576 -0
- package/dist/src/cli/doctor.js +257 -0
- package/dist/src/cli/guides.js +134 -0
- package/dist/src/cli/init.js +326 -0
- package/dist/src/cli/machine.js +157 -0
- package/dist/src/cli/main.js +83 -0
- package/dist/src/cli/pg-tls.js +100 -0
- package/dist/src/cli/pg-url.js +52 -0
- package/dist/src/cli/pin.js +36 -0
- package/dist/src/cli/ports.js +118 -0
- package/dist/src/cli/preflight.js +338 -0
- package/dist/src/cli/project.js +127 -0
- package/dist/src/cli/readiness.js +76 -0
- package/dist/src/cli/registry.js +217 -0
- package/dist/src/cli/replication-checks.js +120 -0
- package/dist/src/cli/replication-mode.js +148 -0
- package/dist/src/cli/run-server.js +388 -0
- package/dist/src/cli/runs.js +120 -0
- package/dist/src/cli/security.js +86 -0
- package/dist/src/cli/self-signed-cert.js +64 -0
- package/dist/src/cli/serve.js +296 -0
- package/dist/src/cli/sources.js +562 -0
- package/dist/src/cli/state.js +58 -0
- package/dist/src/cli/style.js +187 -0
- package/dist/src/cli/supervise.js +248 -0
- package/dist/src/cli/sync.js +94 -0
- package/dist/src/cli/ui.js +107 -0
- package/dist/src/cli/user.js +229 -0
- package/dist/src/cli/version.js +23 -0
- package/dist/src/duck/analyze.js +113 -0
- package/dist/src/duck/cancel.js +49 -0
- package/dist/src/duck/executor-worker.js +263 -0
- package/dist/src/duck/executor.js +526 -0
- package/dist/src/duck/explain.js +119 -0
- package/dist/src/duck/functions.js +764 -0
- package/dist/src/duck/gaps.js +89 -0
- package/dist/src/duck/jsonb.js +186 -0
- package/dist/src/duck/kept-route.js +219 -0
- package/dist/src/duck/kept.js +797 -0
- package/dist/src/duck/late.js +221 -0
- package/dist/src/duck/libc-order.js +190 -0
- package/dist/src/duck/pg-eval-worker.js +34 -0
- package/dist/src/duck/pg-eval.js +59 -0
- package/dist/src/duck/pgtree.js +338 -0
- package/dist/src/duck/plan.js +246 -0
- package/dist/src/duck/publication.js +207 -0
- package/dist/src/duck/requests.js +209 -0
- package/dist/src/duck/resolved.js +248 -0
- package/dist/src/duck/rewrite.js +4001 -0
- package/dist/src/duck/store.js +831 -0
- package/dist/src/duck/stream.js +518 -0
- package/dist/src/duck/sync.js +2864 -0
- package/dist/src/duck/to-char.js +482 -0
- package/dist/src/duck/token.js +31 -0
- package/dist/src/duck/types.js +427 -0
- package/dist/src/duck/users.js +231 -0
- package/dist/src/duck/verified.js +995 -0
- package/dist/src/duck/views.js +150 -0
- package/dist/src/lib/database-url.js +74 -0
- package/dist/src/lib/did-you-mean.js +43 -0
- package/dist/src/lib/dotenv.js +72 -0
- package/dist/src/lib/exit-when-written.js +20 -0
- package/dist/src/lib/float-equal.js +69 -0
- package/dist/src/lib/http-json.js +22 -0
- package/dist/src/lib/log.js +161 -0
- package/dist/src/lib/pg-numeric.js +489 -0
- package/dist/src/lib/port.js +22 -0
- package/dist/src/lib/project-file.js +49 -0
- package/dist/src/lib/project-files.js +43 -0
- package/dist/src/lib/project-version.js +76 -0
- package/dist/src/lib/runnable.js +14 -0
- package/dist/src/lib/shell-word.js +4 -0
- package/dist/src/lib/timezone.js +236 -0
- package/dist/src/lib/zone-names.js +114 -0
- package/dist/src/web/app.css +65 -0
- package/dist/src/web/app.js +336 -0
- package/dist/src/web/index.html +39 -0
- package/package.json +22 -4
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
2
|
+
const kindOf = (n) => isObject(n) ? Object.keys(n)[0] ?? null : null;
|
|
3
|
+
const bodyOf = (n) => n[kindOf(n)];
|
|
4
|
+
const S = (sval) => ({ String: { sval } });
|
|
5
|
+
const ref = (...names) => ({ ColumnRef: { fields: names.map(S) } });
|
|
6
|
+
const LATE = "__mason_late";
|
|
7
|
+
/** what may stand in a select list that is computed after the limit: one value per row, reading its row alone */
|
|
8
|
+
const ROW_WISE = new Set(["ColumnRef", "A_Const", "ParamRef", "FuncCall", "TypeCast", "A_Expr", "BoolExpr", "CaseExpr", "CaseWhen",
|
|
9
|
+
"CoalesceExpr", "NullTest", "BooleanTest", "MinMaxExpr", "A_ArrayExpr", "A_Indirection", "A_Indices", "CollateClause", "NamedArgExpr",
|
|
10
|
+
"RowExpr", "SQLValueFunction", "TypeName", "String", "Integer", "Float", "Boolean", "List"]);
|
|
11
|
+
/** DuckDB's set-returning functions: a row each would be many */
|
|
12
|
+
const SETS = new Set(["unnest", "generate_series", "range", "generate_subscripts", "json_each", "json_tree"]);
|
|
13
|
+
/** Postgres's analysis: no query of the statement aggregates, windows, returns sets from its list or holds a subquery
|
|
14
|
+
* expression — so every row of every SELECT is one row of its FROM, and a select list reads its own row alone */
|
|
15
|
+
function rowWiseStatement(tree) {
|
|
16
|
+
let queries = 0, ok = true;
|
|
17
|
+
const visit = (v) => {
|
|
18
|
+
if (!ok)
|
|
19
|
+
return;
|
|
20
|
+
if (Array.isArray(v)) {
|
|
21
|
+
for (const x of v)
|
|
22
|
+
visit(x);
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
if (v === null || typeof v !== "object")
|
|
26
|
+
return;
|
|
27
|
+
if (v._node === "QUERY") {
|
|
28
|
+
queries++;
|
|
29
|
+
for (const f of ["hasAggs", "hasWindowFuncs", "hasTargetSRFs", "hasSubLinks"])
|
|
30
|
+
if (v[f] !== "false") {
|
|
31
|
+
ok = false;
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
for (const [k, x] of Object.entries(v))
|
|
36
|
+
if (k !== "_node")
|
|
37
|
+
visit(x);
|
|
38
|
+
};
|
|
39
|
+
visit(tree);
|
|
40
|
+
return ok && queries > 0;
|
|
41
|
+
}
|
|
42
|
+
/** every node of a select-list expression is row-wise, and it computes something (a plain column needs no deferring) */
|
|
43
|
+
function rowWise(n) {
|
|
44
|
+
let ok = true, computes = false;
|
|
45
|
+
const visit = (v) => {
|
|
46
|
+
if (!ok)
|
|
47
|
+
return;
|
|
48
|
+
if (Array.isArray(v)) {
|
|
49
|
+
v.forEach(visit);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (!isObject(v))
|
|
53
|
+
return;
|
|
54
|
+
const k = kindOf(v);
|
|
55
|
+
if (/^[A-Z]/.test(k) && isObject(v[k])) {
|
|
56
|
+
if (!ROW_WISE.has(k)) {
|
|
57
|
+
ok = false;
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const b = bodyOf(v);
|
|
61
|
+
if (k === "FuncCall") {
|
|
62
|
+
const name = b.funcname.map((x) => bodyOf(x).sval).join(".").toLowerCase();
|
|
63
|
+
if (b.over !== undefined || b.agg_star === true || b.agg_distinct === true || b.agg_order !== undefined || b.agg_filter !== undefined || SETS.has(name)) {
|
|
64
|
+
ok = false;
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (k === "ColumnRef" && b.fields.some((f) => kindOf(f) === "A_Star")) {
|
|
69
|
+
ok = false;
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (k !== "ColumnRef" && k !== "A_Const" && k !== "ParamRef" && k !== "String" && k !== "Integer" && k !== "Float" && k !== "Boolean")
|
|
73
|
+
computes = true;
|
|
74
|
+
for (const x of Object.values(b))
|
|
75
|
+
visit(x);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
for (const x of Object.values(v))
|
|
79
|
+
visit(x);
|
|
80
|
+
};
|
|
81
|
+
visit(n);
|
|
82
|
+
return { ok, computes };
|
|
83
|
+
}
|
|
84
|
+
/** the same tree with every ColumnRef replaced by `__mason_late.<the column it is read as>` */
|
|
85
|
+
function overLate(n, columns) {
|
|
86
|
+
if (Array.isArray(n))
|
|
87
|
+
return n.map((x) => overLate(x, columns));
|
|
88
|
+
if (!isObject(n))
|
|
89
|
+
return n;
|
|
90
|
+
if (kindOf(n) === "ColumnRef" && isObject(n.ColumnRef)) {
|
|
91
|
+
const key = JSON.stringify(n.ColumnRef.fields);
|
|
92
|
+
let name = columns.get(key);
|
|
93
|
+
if (name === undefined) {
|
|
94
|
+
name = `__mason_c${columns.size + 1}`;
|
|
95
|
+
columns.set(key, name);
|
|
96
|
+
}
|
|
97
|
+
return ref(LATE, name);
|
|
98
|
+
}
|
|
99
|
+
return Object.fromEntries(Object.entries(n).map(([k, v]) => [k, overLate(v, columns)]));
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* The statement with every sorted, limited SELECT written with its select list computed after its limit (see above),
|
|
103
|
+
* or null where none is. `tree` is the rewrite's DuckDB tree, `analysis` Postgres's of the original.
|
|
104
|
+
*/
|
|
105
|
+
export function selectListAfterLimit(tree, analysis) {
|
|
106
|
+
if (!rowWiseStatement(analysis))
|
|
107
|
+
return null;
|
|
108
|
+
let changed = false;
|
|
109
|
+
// bottom up: a SELECT's derived tables and WITH queries first, then the SELECT (never what it was written as)
|
|
110
|
+
const visit = (n) => {
|
|
111
|
+
if (Array.isArray(n))
|
|
112
|
+
return n.map(visit);
|
|
113
|
+
if (!isObject(n))
|
|
114
|
+
return n;
|
|
115
|
+
const out = Object.fromEntries(Object.entries(n).map(([k, v]) => [k, visit(v)]));
|
|
116
|
+
if (kindOf(out) !== "SelectStmt")
|
|
117
|
+
return out;
|
|
118
|
+
// a set operation's arms are SELECT bodies of their own (libpg-query writes them unwrapped)
|
|
119
|
+
const body = bodyOf(out);
|
|
120
|
+
for (const arm of ["larg", "rarg"]) {
|
|
121
|
+
if (!isObject(body[arm]))
|
|
122
|
+
continue;
|
|
123
|
+
const late = afterLimit({ SelectStmt: body[arm] });
|
|
124
|
+
if (late !== null) {
|
|
125
|
+
body[arm] = bodyOf(late);
|
|
126
|
+
changed = true;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const late = afterLimit(out);
|
|
130
|
+
if (late === null)
|
|
131
|
+
return out;
|
|
132
|
+
changed = true;
|
|
133
|
+
return late;
|
|
134
|
+
};
|
|
135
|
+
const written = visit(tree);
|
|
136
|
+
return changed ? written : null;
|
|
137
|
+
}
|
|
138
|
+
/** one SELECT written so, or null where it is not so shaped */
|
|
139
|
+
function afterLimit(select) {
|
|
140
|
+
const s = bodyOf(select);
|
|
141
|
+
if (s.op !== undefined && s.op !== "SETOP_NONE")
|
|
142
|
+
return null;
|
|
143
|
+
for (const f of ["withClause", "distinctClause", "groupClause", "havingClause", "windowClause", "lockingClause", "intoClause", "valuesLists"]) {
|
|
144
|
+
if (s[f] !== undefined && !(Array.isArray(s[f]) && s[f].length === 0))
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
if (s.limitCount === undefined || (s.limitOption !== undefined && s.limitOption !== "LIMIT_OPTION_COUNT" && s.limitOption !== "LIMIT_OPTION_DEFAULT"))
|
|
148
|
+
return null;
|
|
149
|
+
const targets = s.targetList;
|
|
150
|
+
const sorts = s.sortClause;
|
|
151
|
+
if (targets === undefined || sorts === undefined || sorts.length === 0)
|
|
152
|
+
return null;
|
|
153
|
+
let computes = false;
|
|
154
|
+
for (const t of targets) {
|
|
155
|
+
if (kindOf(t) !== "ResTarget")
|
|
156
|
+
return null;
|
|
157
|
+
const r = rowWise(bodyOf(t).val);
|
|
158
|
+
if (!r.ok)
|
|
159
|
+
return null;
|
|
160
|
+
computes ||= r.computes;
|
|
161
|
+
}
|
|
162
|
+
if (!computes)
|
|
163
|
+
return null;
|
|
164
|
+
// each sort key as an expression over the FROM clause: an ordinal or an output column's name is that output's own
|
|
165
|
+
// expression, as Postgres reads it (an output name before an input column's)
|
|
166
|
+
const outputName = (t) => {
|
|
167
|
+
const b = bodyOf(t);
|
|
168
|
+
if (typeof b.name === "string")
|
|
169
|
+
return b.name;
|
|
170
|
+
const v = b.val;
|
|
171
|
+
if (kindOf(v) === "ColumnRef") {
|
|
172
|
+
const last = bodyOf(v).fields.at(-1);
|
|
173
|
+
return kindOf(last) === "String" ? bodyOf(last).sval : null;
|
|
174
|
+
}
|
|
175
|
+
return null;
|
|
176
|
+
};
|
|
177
|
+
const keys = [];
|
|
178
|
+
for (const sort of sorts) {
|
|
179
|
+
if (kindOf(sort) !== "SortBy")
|
|
180
|
+
return null;
|
|
181
|
+
const b = bodyOf(sort);
|
|
182
|
+
if (b.useOp !== undefined)
|
|
183
|
+
return null;
|
|
184
|
+
const node = b.node;
|
|
185
|
+
let target;
|
|
186
|
+
if (kindOf(node) === "A_Const" && isObject(bodyOf(node).ival)) {
|
|
187
|
+
target = targets[Number(bodyOf(node).ival.ival ?? 0) - 1];
|
|
188
|
+
if (target === undefined)
|
|
189
|
+
return null;
|
|
190
|
+
}
|
|
191
|
+
else if (kindOf(node) === "ColumnRef" && bodyOf(node).fields.length === 1) {
|
|
192
|
+
const name = bodyOf(bodyOf(node).fields[0]).sval;
|
|
193
|
+
const named = targets.filter((t) => outputName(t) === name);
|
|
194
|
+
if (named.length > 1)
|
|
195
|
+
return null;
|
|
196
|
+
target = named[0];
|
|
197
|
+
}
|
|
198
|
+
if (target !== undefined) {
|
|
199
|
+
// a sort by a computed output computes it for every row anyway: nothing to defer
|
|
200
|
+
if (rowWise(bodyOf(target).val).computes)
|
|
201
|
+
return null;
|
|
202
|
+
keys.push(structuredClone(bodyOf(target).val));
|
|
203
|
+
}
|
|
204
|
+
else if (kindOf(node) === "A_Const")
|
|
205
|
+
return null;
|
|
206
|
+
else
|
|
207
|
+
keys.push(structuredClone(node));
|
|
208
|
+
}
|
|
209
|
+
const columns = new Map();
|
|
210
|
+
const list = targets.map((t) => ({ ResTarget: { ...bodyOf(t), val: overLate(bodyOf(t).val, columns) } }));
|
|
211
|
+
const inner = { ...s, targetList: [
|
|
212
|
+
...[...columns].map(([fields, name]) => ({ ResTarget: { name, val: { ColumnRef: { fields: JSON.parse(fields) } } } })),
|
|
213
|
+
...keys.map((k, i) => ({ ResTarget: { name: `__mason_o${i + 1}`, val: k } })),
|
|
214
|
+
], sortClause: sorts.map((sort, i) => ({ SortBy: { ...bodyOf(sort), node: keys[i] } })) };
|
|
215
|
+
return { SelectStmt: {
|
|
216
|
+
targetList: list,
|
|
217
|
+
fromClause: [{ RangeSubselect: { subquery: { SelectStmt: inner }, alias: { aliasname: LATE } } }],
|
|
218
|
+
sortClause: sorts.map((sort, i) => ({ SortBy: { ...bodyOf(sort), node: ref(LATE, `__mason_o${i + 1}`) } })),
|
|
219
|
+
limitOption: "LIMIT_OPTION_DEFAULT", op: "SETOP_NONE",
|
|
220
|
+
} };
|
|
221
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TEXT IN A GLIBC LOCALE'S ORDER, MEASURED FROM THE SOURCE (the BI-tools test of #517: Metabase's field lists
|
|
3
|
+
* under en_US.utf8 listed Hangul, `$` / `€` and U+FFFD elsewhere than Postgres, and `s < 'b'` counted other rows). The
|
|
4
|
+
* order was emulated from assumptions until then — ICU's order of the letters and digits, the rest ignored, then the
|
|
5
|
+
* punctuation by its bytes — and glibc differs from each: Hangul has no letter weight in it, `$` and `€` have one where
|
|
6
|
+
* `@` and `~` have none, U+FFFD sorts after the letters. So the order is no longer assumed: it is READ from the source
|
|
7
|
+
* database, once (read-only SELECTs, a few seconds), kept in the copy (`mason.libc_order`, src/duck/store.ts), read again
|
|
8
|
+
* when the database's collation version moves (`pg_database.datcollversion`: glibc's), and every sort key is built from it.
|
|
9
|
+
*
|
|
10
|
+
* What glibc compares, as measured (Postgres 18 / glibc 2.41 en_US.utf8, the disposable server's):
|
|
11
|
+
* 1. the FIRST level: a weight per character that has one (letters, digits, `$`, U+FFFD; not punctuation, Hangul or
|
|
12
|
+
* spaces), compared in order. Read as CLASSES: the characters in the database's order, two neighbours x < y in one
|
|
13
|
+
* class when x||'b' > y||'a' (only the second character told them apart). A character equal there to a string of
|
|
14
|
+
* others (ß = ss, æ = ae, fi = fi) is that string: proposed by its compatibility decomposition, kept only when the
|
|
15
|
+
* source says it is equal;
|
|
16
|
+
* 2. accents, then 3. case: ICU's second and third levels over the characters of level 1 (the source's own agree
|
|
17
|
+
* with them over everything the sample below holds; nothing here reads them from the source);
|
|
18
|
+
* 4. the LAST level: for each character that has a weight there (punctuation, spaces, Hangul's one shared weight,
|
|
19
|
+
* Latin letters; not CJK), the pair (its position in the string, its weight), position first — `三 < :三` (`三` has
|
|
20
|
+
* none) and `가三 < 三가` both follow. The punctuation's weights are their order (they differ in nothing else); a
|
|
21
|
+
* character of level 1 is placed among them by binary search (p||x < x exactly when p's weight is below x's), and
|
|
22
|
+
* one placed below all of them has none when x||chr(1) is not below chr(1)||x;
|
|
23
|
+
* 5. a tie at every level is Postgres's byte order (`varstr_cmp`'s strcmp), which the caller adds.
|
|
24
|
+
* Held against the source's ORDER BY over 30,000 random strings of 17 scripts and symbol sets: 2 out of order, both
|
|
25
|
+
* full-width digits beside Hangul. Not covered, so REFUSED by name when a value holding one is ordered
|
|
26
|
+
* (`libc-uncovered`): a combining mark (glibc weighs `e` + U+0301 otherwise than ICU), a Thai or Lao vowel written
|
|
27
|
+
* before its consonant (glibc reorders them), a code point this Node does not know as assigned, a private use one. A
|
|
28
|
+
* known difference (`libc-symbol-punctuation`): punctuation right after a digit or symbol outranks a case difference
|
|
29
|
+
* after it in glibc (`1-X < 1x`), and not here.
|
|
30
|
+
*/
|
|
31
|
+
import { gapMark } from "./gaps.js";
|
|
32
|
+
import { PgEvalError } from "./pg-eval.js";
|
|
33
|
+
const MAX = 0x110000;
|
|
34
|
+
/** The code points read from the source: assigned, not private use, not a surrogate (the rest are uncovered). */
|
|
35
|
+
export function assignedCodePoints() {
|
|
36
|
+
const out = [];
|
|
37
|
+
for (let g = 1; g < MAX; g++) {
|
|
38
|
+
if (g >= 0xd800 && g <= 0xdfff)
|
|
39
|
+
continue;
|
|
40
|
+
if (/[\p{Cn}\p{Co}]/u.test(String.fromCodePoint(g)))
|
|
41
|
+
continue;
|
|
42
|
+
out.push(g);
|
|
43
|
+
}
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
/** what glibc orders otherwise than this ordering: combining marks, Thai and Lao vowels written before their consonant */
|
|
47
|
+
const UNCOVERED = /[\p{M}เ-ไເ-ໄ]/u;
|
|
48
|
+
const ints = (xs) => `'{${xs.join(",")}}'::int[]`;
|
|
49
|
+
const texts = (xs) => `ARRAY[${xs.map((x) => `'${x.replaceAll("'", "''")}'`).join(",")}]::text[]`;
|
|
50
|
+
/** Reads the source database's order of text (its default collation), as the module's header says. */
|
|
51
|
+
export async function readLibcOrder(query, collate, version) {
|
|
52
|
+
const points = assignedCodePoints();
|
|
53
|
+
const l1 = new Uint32Array(MAX), l4 = new Uint32Array(MAX), uncovered = new Uint8Array(MAX).fill(1);
|
|
54
|
+
// each character: whether it has a first-level weight ('b'||c < 'ba' and c||'b' > 'aa' only for one that has none);
|
|
55
|
+
// those that have one in the database's order, each marked when its neighbour below is of its class; the rest in
|
|
56
|
+
// the database's order, each marked when its neighbour below has the same last-level weight (pp > c only then)
|
|
57
|
+
const rows = await query(`WITH c AS (SELECT g, chr(g) AS ch FROM unnest(${ints(points)}) AS u(g)),
|
|
58
|
+
k AS (SELECT g, ch, ('b' || ch < 'ba') AND (ch || 'b' > 'aa') AS ignored FROM c),
|
|
59
|
+
n AS (SELECT g, ch, lag(ch) OVER (ORDER BY ch) AS prev, row_number() OVER (ORDER BY ch) AS pos FROM k WHERE NOT ignored),
|
|
60
|
+
i AS (SELECT g, ch, lag(ch) OVER (ORDER BY ch) AS prev, row_number() OVER (ORDER BY ch) AS pos FROM k WHERE ignored)
|
|
61
|
+
SELECT 'n', g::text, coalesce(prev || 'b' > ch || 'a', false)::text, pos::text FROM n
|
|
62
|
+
UNION ALL SELECT 'i', g::text, coalesce(prev || prev > ch, false)::text, pos::text FROM i`);
|
|
63
|
+
// each kind in the database's order
|
|
64
|
+
rows.sort((a, b) => a[0] === b[0] ? Number(a[3]) - Number(b[3]) : a[0] < b[0] ? -1 : 1);
|
|
65
|
+
const weighted = [];
|
|
66
|
+
const ignoredClasses = []; // one character per last-level weight of the ignored ones, in order
|
|
67
|
+
let k = 0;
|
|
68
|
+
for (const [kind, g0, same] of rows) {
|
|
69
|
+
const g = Number(g0);
|
|
70
|
+
uncovered[g] = UNCOVERED.test(String.fromCodePoint(g)) ? 1 : 0;
|
|
71
|
+
if (kind === "n") {
|
|
72
|
+
if (same !== "true")
|
|
73
|
+
k++;
|
|
74
|
+
l1[g] = k;
|
|
75
|
+
weighted.push(g);
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
if (same !== "true" || ignoredClasses.length === 0)
|
|
79
|
+
ignoredClasses.push(g);
|
|
80
|
+
l4[g] = 2 * ignoredClasses.length;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
if (weighted.length === 0 || ignoredClasses.length === 0)
|
|
84
|
+
throw new Error(`reading how the database orders text (${collate}) found no ${weighted.length === 0 ? "letters" : "punctuation"}: Mason cannot sort text as it does`);
|
|
85
|
+
// expansions: proposed by each character's compatibility decomposition (its marks dropped) and the letters NFKD keeps
|
|
86
|
+
// whole, kept where the source says the two are equal at the first level (each sorts after the other given a later b)
|
|
87
|
+
const proposed = [];
|
|
88
|
+
for (const g of weighted) {
|
|
89
|
+
const d = String.fromCodePoint(g).normalize("NFKD").replace(/\p{M}/gu, "");
|
|
90
|
+
if ([...d].length > 1 && !d.includes("\0"))
|
|
91
|
+
proposed.push([g, d]);
|
|
92
|
+
}
|
|
93
|
+
for (const [ch, e] of [["ß", "ss"], ["ẞ", "SS"], ["æ", "ae"], ["Æ", "AE"], ["œ", "oe"], ["Œ", "OE"], ["þ", "th"], ["Þ", "TH"]])
|
|
94
|
+
proposed.push([ch.codePointAt(0), e]);
|
|
95
|
+
const expansions = new Map();
|
|
96
|
+
for (let at = 0; at < proposed.length; at += 5000) {
|
|
97
|
+
const part = proposed.slice(at, at + 5000);
|
|
98
|
+
const eq = await query(`SELECT g::text, ((chr(g) || 'b' > e || 'a') AND (e || 'b' > chr(g) || 'a'))::text FROM unnest(${ints(part.map((p) => p[0]))}, ${texts(part.map((p) => p[1]))}) AS u(g, e)`);
|
|
99
|
+
eq.forEach(([g, yes], j) => { if (yes === "true")
|
|
100
|
+
expansions.set(Number(g), [...part[j][1]].map((c) => c.codePointAt(0))); });
|
|
101
|
+
}
|
|
102
|
+
// each first-level character's last-level weight, among the ignored ones' by binary search
|
|
103
|
+
const lo = new Int32Array(weighted.length), hi = new Int32Array(weighted.length).fill(ignoredClasses.length);
|
|
104
|
+
for (;;) {
|
|
105
|
+
const idx = [], mid = [];
|
|
106
|
+
for (let i = 0; i < weighted.length; i++)
|
|
107
|
+
if (lo[i] < hi[i]) {
|
|
108
|
+
idx.push(i);
|
|
109
|
+
mid.push((lo[i] + hi[i]) >> 1);
|
|
110
|
+
}
|
|
111
|
+
if (idx.length === 0)
|
|
112
|
+
break;
|
|
113
|
+
const below = await query(`SELECT (chr(p) || chr(x) < chr(x))::text FROM unnest(${ints(mid.map((m) => ignoredClasses[m]))}, ${ints(idx.map((i) => weighted[i]))}) WITH ORDINALITY AS u(p, x, n) ORDER BY n`);
|
|
114
|
+
below.forEach(([b], j) => { const i = idx[j]; if (b === "true")
|
|
115
|
+
lo[i] = mid[j] + 1;
|
|
116
|
+
else
|
|
117
|
+
hi[i] = mid[j]; });
|
|
118
|
+
}
|
|
119
|
+
const bottom = [];
|
|
120
|
+
weighted.forEach((g, i) => { l4[g] = 2 * lo[i] + 1; if (lo[i] === 0)
|
|
121
|
+
bottom.push(g); });
|
|
122
|
+
if (bottom.length) {
|
|
123
|
+
const has = await query(`SELECT x::text, (chr(x) || chr(1) < chr(1) || chr(x))::text FROM unnest(${ints(bottom)}) AS u(x)`);
|
|
124
|
+
for (const [x, yes] of has)
|
|
125
|
+
if (yes !== "true")
|
|
126
|
+
l4[Number(x)] = 0;
|
|
127
|
+
}
|
|
128
|
+
return { collate, version, l1, l4, uncovered, expansions };
|
|
129
|
+
}
|
|
130
|
+
/** The characters ICU's accent and case levels read: those with a first-level weight, in order. */
|
|
131
|
+
export function libcKept(o, s) {
|
|
132
|
+
let kept = "";
|
|
133
|
+
for (const ch of s)
|
|
134
|
+
if (o.l1[ch.codePointAt(0)] > 0)
|
|
135
|
+
kept += ch;
|
|
136
|
+
return kept;
|
|
137
|
+
}
|
|
138
|
+
const hex6 = (n) => n.toString(16).padStart(6, "0");
|
|
139
|
+
/** ICU's sort key from its accent level on: what follows the first level's terminator (byte 01, at an even offset) */
|
|
140
|
+
function afterPrimary(icuKey) {
|
|
141
|
+
for (let i = 0; i < icuKey.length; i += 2)
|
|
142
|
+
if (icuKey.charCodeAt(i) === 48 && icuKey.charCodeAt(i + 1) === 49)
|
|
143
|
+
return icuKey.slice(i + 2);
|
|
144
|
+
return "";
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* The sort key of `s` as one text that orders as glibc orders (bytewise; a space parts the levels, below every hex digit,
|
|
148
|
+
* so a shorter level sorts first): the first-level classes, ICU's accent and case levels of `libcKept(s)` (`icuKey`),
|
|
149
|
+
* the last level's (position, weight) pairs. A value holding a character not covered is refused, naming it.
|
|
150
|
+
*/
|
|
151
|
+
export function libcKey(o, s, icuKey) {
|
|
152
|
+
let first = "", last = "", at = 0;
|
|
153
|
+
for (const ch of s) {
|
|
154
|
+
at++;
|
|
155
|
+
const g = ch.codePointAt(0);
|
|
156
|
+
if (o.uncovered[g]) {
|
|
157
|
+
throw new PgEvalError("0A000", `the text ${JSON.stringify(s.length > 40 ? `${s.slice(0, 40)}…` : s)} holds U+${g.toString(16).toUpperCase().padStart(4, "0")}, which Mason cannot order as the database's collation ${o.collate} does (${gapMark("libc-uncovered")})`);
|
|
158
|
+
}
|
|
159
|
+
if (o.l1[g] > 0)
|
|
160
|
+
for (const x of o.expansions.get(g) ?? [g])
|
|
161
|
+
if (o.l1[x] > 0)
|
|
162
|
+
first += hex6(o.l1[x]);
|
|
163
|
+
if (o.l4[g] > 0)
|
|
164
|
+
last += hex6(at) + hex6(o.l4[g]);
|
|
165
|
+
}
|
|
166
|
+
return `${first} ${afterPrimary(icuKey)} ${last}`;
|
|
167
|
+
}
|
|
168
|
+
/** The order as bytes, to keep in the copy: the three arrays and the expansions, as JSON. */
|
|
169
|
+
export function encodeLibcOrder(o) {
|
|
170
|
+
// only the code points that are not 0 / covered: the arrays are mostly empty
|
|
171
|
+
const sparse = (a, skip) => {
|
|
172
|
+
const out = [];
|
|
173
|
+
for (let g = 0; g < MAX; g++)
|
|
174
|
+
if (a[g] !== skip)
|
|
175
|
+
out.push([g, a[g]]);
|
|
176
|
+
return out;
|
|
177
|
+
};
|
|
178
|
+
return JSON.stringify({ collate: o.collate, version: o.version, l1: sparse(o.l1, 0), l4: sparse(o.l4, 0), covered: sparse(o.uncovered, 1).map(([g]) => g), expansions: [...o.expansions] });
|
|
179
|
+
}
|
|
180
|
+
export function decodeLibcOrder(text) {
|
|
181
|
+
const v = JSON.parse(text);
|
|
182
|
+
const l1 = new Uint32Array(MAX), l4 = new Uint32Array(MAX), uncovered = new Uint8Array(MAX).fill(1);
|
|
183
|
+
for (const [g, x] of v.l1)
|
|
184
|
+
l1[g] = x;
|
|
185
|
+
for (const [g, x] of v.l4)
|
|
186
|
+
l4[g] = x;
|
|
187
|
+
for (const g of v.covered)
|
|
188
|
+
uncovered[g] = 0;
|
|
189
|
+
return { collate: v.collate, version: v.version, l1, l4, uncovered, expansions: new Map(v.expansions) };
|
|
190
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* POSTGRES'S OWN FUNCTIONS, ON A WORKER (src/duck/pg-eval.ts): a PGlite with no tables, asked by DuckDB's scalar
|
|
3
|
+
* functions what Postgres makes of a chunk of values — a text read as a date, an interval, a boolean; a jsonpath
|
|
4
|
+
* evaluated. The main thread blocks on `signal` while this thread answers, so nothing here may wait on the main thread.
|
|
5
|
+
*
|
|
6
|
+
* In (on `port`): `{ sql, params, timeZone }`. Out: `{ ok: true, rows }` (each value its text, an array the JSON of its
|
|
7
|
+
* elements' text, or null) or
|
|
8
|
+
* `{ ok: false, code, message }` (Postgres's SQLSTATE and words); then `signal[0]` is set to 1 and notified.
|
|
9
|
+
*/
|
|
10
|
+
import { workerData } from "node:worker_threads";
|
|
11
|
+
import { PGlite } from "@electric-sql/pglite";
|
|
12
|
+
const { port, signal } = workerData;
|
|
13
|
+
const db = new PGlite();
|
|
14
|
+
let zone = null;
|
|
15
|
+
const reply = (m) => { port.postMessage(m); Atomics.store(signal, 0, 1); Atomics.notify(signal, 0); };
|
|
16
|
+
port.on("message", (m) => {
|
|
17
|
+
void (async () => {
|
|
18
|
+
try {
|
|
19
|
+
await db.waitReady;
|
|
20
|
+
if (zone !== m.timeZone) {
|
|
21
|
+
await db.exec(`SET TimeZone = '${m.timeZone.replace(/'/g, "''")}'`);
|
|
22
|
+
zone = m.timeZone;
|
|
23
|
+
}
|
|
24
|
+
const result = await db.query(m.sql, m.params, { rowMode: "array" });
|
|
25
|
+
// an array as the JSON of its elements' text (the `list` kind, src/duck/functions.ts)
|
|
26
|
+
const textOf = (v) => v === null ? null : Array.isArray(v) ? JSON.stringify(v.map(textOf)) : String(v);
|
|
27
|
+
reply({ ok: true, rows: result.rows.map((r) => r.map(textOf)) });
|
|
28
|
+
}
|
|
29
|
+
catch (e) {
|
|
30
|
+
const err = e;
|
|
31
|
+
reply({ ok: false, code: typeof err.code === "string" ? err.code : "XX000", message: err.message ?? String(e) });
|
|
32
|
+
}
|
|
33
|
+
})();
|
|
34
|
+
});
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* POSTGRES'S OWN FUNCTIONS, A CHUNK AT A TIME (the corpus run's unverified casts). What a text MEANS as a
|
|
3
|
+
* date, a timestamp, an interval, a boolean, an integer or a numeric is Postgres's input functions' to say — they read
|
|
4
|
+
* `01/02/2026`, `January 2, 2026`, `J2461043`, `2026-01-02 America/New_York`, `1 day ago`, `P1DT2H`, ` yes `, `0x1F`,
|
|
5
|
+
* `1_000` — and DuckDB reads ISO alone. So nothing reimplements them: a DuckDB scalar function (src/duck/functions.ts
|
|
6
|
+
* `__mason_pg_*`) hands its chunk's distinct values to a PGlite on a worker thread (pg-eval-worker.ts), which applies
|
|
7
|
+
* Postgres's own function under the session's TimeZone and answers each value, or Postgres's error. The same route
|
|
8
|
+
* answers the few functions DuckDB has no equivalent of (jsonpath's).
|
|
9
|
+
*
|
|
10
|
+
* DuckDB calls a scalar function synchronously, so the call BLOCKS the main thread until the worker answers
|
|
11
|
+
* (`Atomics.wait`, then `receiveMessageOnPort`): about 8 ms for 2,048 values. A worker that does not answer within
|
|
12
|
+
* `TIMEOUT_MS` is terminated and the statement fails; the next call starts another.
|
|
13
|
+
*/
|
|
14
|
+
import { MessageChannel, receiveMessageOnPort, Worker } from "node:worker_threads";
|
|
15
|
+
import { fileURLToPath } from "node:url";
|
|
16
|
+
import { runnable } from "../lib/runnable.js";
|
|
17
|
+
const TIMEOUT_MS = 30_000;
|
|
18
|
+
export class PgEvalError extends Error {
|
|
19
|
+
code;
|
|
20
|
+
constructor(code, message) { super(message); this.code = code; }
|
|
21
|
+
}
|
|
22
|
+
let current = null;
|
|
23
|
+
function start() {
|
|
24
|
+
const signal = new Int32Array(new SharedArrayBuffer(4));
|
|
25
|
+
const { port1, port2 } = new MessageChannel();
|
|
26
|
+
const worker = // `runnable`: the package ships this worker as .js (cli/build.ts), and a URL to the .ts would wait out TIMEOUT_MS and fail
|
|
27
|
+
new Worker(runnable(fileURLToPath(new URL("./pg-eval-worker.ts", import.meta.url))), { workerData: { port: port2, signal }, transferList: [port2] });
|
|
28
|
+
// the process may end while it lives: it holds nothing that must be closed
|
|
29
|
+
worker.unref();
|
|
30
|
+
port1.unref();
|
|
31
|
+
worker.on("error", (e) => { console.error(`mason: the Postgres function worker failed: ${e.message}`); if (current?.worker === worker)
|
|
32
|
+
current = null; });
|
|
33
|
+
return { worker, port: port1, signal };
|
|
34
|
+
}
|
|
35
|
+
/** Runs one statement on the worker's Postgres and returns its rows as text; Postgres's error is a PgEvalError. */
|
|
36
|
+
export function pgEvalSync(sql, params, timeZone) {
|
|
37
|
+
current ??= start();
|
|
38
|
+
const w = current;
|
|
39
|
+
Atomics.store(w.signal, 0, 0);
|
|
40
|
+
w.port.postMessage({ sql, params, timeZone });
|
|
41
|
+
const waited = Atomics.wait(w.signal, 0, 0, TIMEOUT_MS);
|
|
42
|
+
const got = receiveMessageOnPort(w.port);
|
|
43
|
+
if (waited === "timed-out" || got === undefined) {
|
|
44
|
+
current = null;
|
|
45
|
+
void w.worker.terminate();
|
|
46
|
+
throw new PgEvalError("57014", `Postgres's function did not answer within ${TIMEOUT_MS / 1000} s (Mason computes it on its own Postgres)`);
|
|
47
|
+
}
|
|
48
|
+
const r = got.message;
|
|
49
|
+
if (!r.ok)
|
|
50
|
+
throw new PgEvalError(r.code, r.message);
|
|
51
|
+
return r.rows;
|
|
52
|
+
}
|
|
53
|
+
/** Stops the worker (a store that closes). */
|
|
54
|
+
export async function stopPgEval() {
|
|
55
|
+
const w = current;
|
|
56
|
+
current = null;
|
|
57
|
+
if (w !== null)
|
|
58
|
+
await w.worker.terminate();
|
|
59
|
+
}
|