@mason-data/cli 0.0.0-stage → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +209 -2
  3. package/dist/src/api/error.js +18 -0
  4. package/dist/src/api/postgres/catalog-worker.js +92 -0
  5. package/dist/src/api/postgres/catalog.js +442 -0
  6. package/dist/src/api/postgres/encode.js +541 -0
  7. package/dist/src/api/postgres/explain.js +143 -0
  8. package/dist/src/api/postgres/listener.js +60 -0
  9. package/dist/src/api/postgres/parser-worker.js +58 -0
  10. package/dist/src/api/postgres/parser.js +121 -0
  11. package/dist/src/api/postgres/plain.js +33 -0
  12. package/dist/src/api/postgres/protocol.js +404 -0
  13. package/dist/src/api/postgres/schema.js +54 -0
  14. package/dist/src/api/postgres/server.js +1670 -0
  15. package/dist/src/api/postgres/translate.js +248 -0
  16. package/dist/src/api/request-log.js +50 -0
  17. package/dist/src/cdc/pg-wire.js +482 -0
  18. package/dist/src/cdc/pgoutput.js +107 -0
  19. package/dist/src/cdc/poll.js +67 -0
  20. package/dist/src/cdc/source.js +65 -0
  21. package/dist/src/cli/background.js +174 -0
  22. package/dist/src/cli/command.js +172 -0
  23. package/dist/src/cli/commands.js +1145 -0
  24. package/dist/src/cli/control.js +36 -0
  25. package/dist/src/cli/dev.js +576 -0
  26. package/dist/src/cli/doctor.js +257 -0
  27. package/dist/src/cli/guides.js +134 -0
  28. package/dist/src/cli/init.js +326 -0
  29. package/dist/src/cli/machine.js +157 -0
  30. package/dist/src/cli/main.js +83 -0
  31. package/dist/src/cli/pg-tls.js +100 -0
  32. package/dist/src/cli/pg-url.js +52 -0
  33. package/dist/src/cli/pin.js +36 -0
  34. package/dist/src/cli/ports.js +118 -0
  35. package/dist/src/cli/preflight.js +338 -0
  36. package/dist/src/cli/project.js +127 -0
  37. package/dist/src/cli/readiness.js +76 -0
  38. package/dist/src/cli/registry.js +217 -0
  39. package/dist/src/cli/replication-checks.js +120 -0
  40. package/dist/src/cli/replication-mode.js +148 -0
  41. package/dist/src/cli/run-server.js +388 -0
  42. package/dist/src/cli/runs.js +120 -0
  43. package/dist/src/cli/security.js +86 -0
  44. package/dist/src/cli/self-signed-cert.js +64 -0
  45. package/dist/src/cli/serve.js +296 -0
  46. package/dist/src/cli/sources.js +562 -0
  47. package/dist/src/cli/state.js +58 -0
  48. package/dist/src/cli/style.js +187 -0
  49. package/dist/src/cli/supervise.js +248 -0
  50. package/dist/src/cli/sync.js +94 -0
  51. package/dist/src/cli/ui.js +107 -0
  52. package/dist/src/cli/user.js +229 -0
  53. package/dist/src/cli/version.js +23 -0
  54. package/dist/src/duck/analyze.js +113 -0
  55. package/dist/src/duck/cancel.js +49 -0
  56. package/dist/src/duck/executor-worker.js +263 -0
  57. package/dist/src/duck/executor.js +526 -0
  58. package/dist/src/duck/explain.js +119 -0
  59. package/dist/src/duck/functions.js +764 -0
  60. package/dist/src/duck/gaps.js +89 -0
  61. package/dist/src/duck/jsonb.js +186 -0
  62. package/dist/src/duck/kept-route.js +219 -0
  63. package/dist/src/duck/kept.js +797 -0
  64. package/dist/src/duck/late.js +221 -0
  65. package/dist/src/duck/libc-order.js +190 -0
  66. package/dist/src/duck/pg-eval-worker.js +34 -0
  67. package/dist/src/duck/pg-eval.js +59 -0
  68. package/dist/src/duck/pgtree.js +338 -0
  69. package/dist/src/duck/plan.js +246 -0
  70. package/dist/src/duck/publication.js +207 -0
  71. package/dist/src/duck/requests.js +209 -0
  72. package/dist/src/duck/resolved.js +248 -0
  73. package/dist/src/duck/rewrite.js +4001 -0
  74. package/dist/src/duck/store.js +831 -0
  75. package/dist/src/duck/stream.js +518 -0
  76. package/dist/src/duck/sync.js +2864 -0
  77. package/dist/src/duck/to-char.js +482 -0
  78. package/dist/src/duck/token.js +31 -0
  79. package/dist/src/duck/types.js +427 -0
  80. package/dist/src/duck/users.js +231 -0
  81. package/dist/src/duck/verified.js +995 -0
  82. package/dist/src/duck/views.js +150 -0
  83. package/dist/src/lib/database-url.js +74 -0
  84. package/dist/src/lib/did-you-mean.js +43 -0
  85. package/dist/src/lib/dotenv.js +72 -0
  86. package/dist/src/lib/exit-when-written.js +20 -0
  87. package/dist/src/lib/float-equal.js +69 -0
  88. package/dist/src/lib/http-json.js +22 -0
  89. package/dist/src/lib/log.js +161 -0
  90. package/dist/src/lib/pg-numeric.js +489 -0
  91. package/dist/src/lib/port.js +22 -0
  92. package/dist/src/lib/project-file.js +49 -0
  93. package/dist/src/lib/project-files.js +43 -0
  94. package/dist/src/lib/project-version.js +76 -0
  95. package/dist/src/lib/runnable.js +14 -0
  96. package/dist/src/lib/shell-word.js +4 -0
  97. package/dist/src/lib/timezone.js +236 -0
  98. package/dist/src/lib/zone-names.js +114 -0
  99. package/dist/src/web/app.css +65 -0
  100. package/dist/src/web/app.js +336 -0
  101. package/dist/src/web/index.html +39 -0
  102. package/package.json +22 -4
@@ -0,0 +1,221 @@
1
+ const isObject = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
2
+ const kindOf = (n) => isObject(n) ? Object.keys(n)[0] ?? null : null;
3
+ const bodyOf = (n) => n[kindOf(n)];
4
+ const S = (sval) => ({ String: { sval } });
5
+ const ref = (...names) => ({ ColumnRef: { fields: names.map(S) } });
6
+ const LATE = "__mason_late";
7
+ /** what may stand in a select list that is computed after the limit: one value per row, reading its row alone */
8
+ const ROW_WISE = new Set(["ColumnRef", "A_Const", "ParamRef", "FuncCall", "TypeCast", "A_Expr", "BoolExpr", "CaseExpr", "CaseWhen",
9
+ "CoalesceExpr", "NullTest", "BooleanTest", "MinMaxExpr", "A_ArrayExpr", "A_Indirection", "A_Indices", "CollateClause", "NamedArgExpr",
10
+ "RowExpr", "SQLValueFunction", "TypeName", "String", "Integer", "Float", "Boolean", "List"]);
11
+ /** DuckDB's set-returning functions: a row each would be many */
12
+ const SETS = new Set(["unnest", "generate_series", "range", "generate_subscripts", "json_each", "json_tree"]);
13
+ /** Postgres's analysis: no query of the statement aggregates, windows, returns sets from its list or holds a subquery
14
+ * expression — so every row of every SELECT is one row of its FROM, and a select list reads its own row alone */
15
+ function rowWiseStatement(tree) {
16
+ let queries = 0, ok = true;
17
+ const visit = (v) => {
18
+ if (!ok)
19
+ return;
20
+ if (Array.isArray(v)) {
21
+ for (const x of v)
22
+ visit(x);
23
+ return;
24
+ }
25
+ if (v === null || typeof v !== "object")
26
+ return;
27
+ if (v._node === "QUERY") {
28
+ queries++;
29
+ for (const f of ["hasAggs", "hasWindowFuncs", "hasTargetSRFs", "hasSubLinks"])
30
+ if (v[f] !== "false") {
31
+ ok = false;
32
+ return;
33
+ }
34
+ }
35
+ for (const [k, x] of Object.entries(v))
36
+ if (k !== "_node")
37
+ visit(x);
38
+ };
39
+ visit(tree);
40
+ return ok && queries > 0;
41
+ }
42
+ /** every node of a select-list expression is row-wise, and it computes something (a plain column needs no deferring) */
43
+ function rowWise(n) {
44
+ let ok = true, computes = false;
45
+ const visit = (v) => {
46
+ if (!ok)
47
+ return;
48
+ if (Array.isArray(v)) {
49
+ v.forEach(visit);
50
+ return;
51
+ }
52
+ if (!isObject(v))
53
+ return;
54
+ const k = kindOf(v);
55
+ if (/^[A-Z]/.test(k) && isObject(v[k])) {
56
+ if (!ROW_WISE.has(k)) {
57
+ ok = false;
58
+ return;
59
+ }
60
+ const b = bodyOf(v);
61
+ if (k === "FuncCall") {
62
+ const name = b.funcname.map((x) => bodyOf(x).sval).join(".").toLowerCase();
63
+ if (b.over !== undefined || b.agg_star === true || b.agg_distinct === true || b.agg_order !== undefined || b.agg_filter !== undefined || SETS.has(name)) {
64
+ ok = false;
65
+ return;
66
+ }
67
+ }
68
+ if (k === "ColumnRef" && b.fields.some((f) => kindOf(f) === "A_Star")) {
69
+ ok = false;
70
+ return;
71
+ }
72
+ if (k !== "ColumnRef" && k !== "A_Const" && k !== "ParamRef" && k !== "String" && k !== "Integer" && k !== "Float" && k !== "Boolean")
73
+ computes = true;
74
+ for (const x of Object.values(b))
75
+ visit(x);
76
+ return;
77
+ }
78
+ for (const x of Object.values(v))
79
+ visit(x);
80
+ };
81
+ visit(n);
82
+ return { ok, computes };
83
+ }
84
+ /** the same tree with every ColumnRef replaced by `__mason_late.<the column it is read as>` */
85
+ function overLate(n, columns) {
86
+ if (Array.isArray(n))
87
+ return n.map((x) => overLate(x, columns));
88
+ if (!isObject(n))
89
+ return n;
90
+ if (kindOf(n) === "ColumnRef" && isObject(n.ColumnRef)) {
91
+ const key = JSON.stringify(n.ColumnRef.fields);
92
+ let name = columns.get(key);
93
+ if (name === undefined) {
94
+ name = `__mason_c${columns.size + 1}`;
95
+ columns.set(key, name);
96
+ }
97
+ return ref(LATE, name);
98
+ }
99
+ return Object.fromEntries(Object.entries(n).map(([k, v]) => [k, overLate(v, columns)]));
100
+ }
101
+ /**
102
+ * The statement with every sorted, limited SELECT written with its select list computed after its limit (see above),
103
+ * or null where none is. `tree` is the rewrite's DuckDB tree, `analysis` Postgres's of the original.
104
+ */
105
+ export function selectListAfterLimit(tree, analysis) {
106
+ if (!rowWiseStatement(analysis))
107
+ return null;
108
+ let changed = false;
109
+ // bottom up: a SELECT's derived tables and WITH queries first, then the SELECT (never what it was written as)
110
+ const visit = (n) => {
111
+ if (Array.isArray(n))
112
+ return n.map(visit);
113
+ if (!isObject(n))
114
+ return n;
115
+ const out = Object.fromEntries(Object.entries(n).map(([k, v]) => [k, visit(v)]));
116
+ if (kindOf(out) !== "SelectStmt")
117
+ return out;
118
+ // a set operation's arms are SELECT bodies of their own (libpg-query writes them unwrapped)
119
+ const body = bodyOf(out);
120
+ for (const arm of ["larg", "rarg"]) {
121
+ if (!isObject(body[arm]))
122
+ continue;
123
+ const late = afterLimit({ SelectStmt: body[arm] });
124
+ if (late !== null) {
125
+ body[arm] = bodyOf(late);
126
+ changed = true;
127
+ }
128
+ }
129
+ const late = afterLimit(out);
130
+ if (late === null)
131
+ return out;
132
+ changed = true;
133
+ return late;
134
+ };
135
+ const written = visit(tree);
136
+ return changed ? written : null;
137
+ }
138
+ /** one SELECT written so, or null where it is not so shaped */
139
+ function afterLimit(select) {
140
+ const s = bodyOf(select);
141
+ if (s.op !== undefined && s.op !== "SETOP_NONE")
142
+ return null;
143
+ for (const f of ["withClause", "distinctClause", "groupClause", "havingClause", "windowClause", "lockingClause", "intoClause", "valuesLists"]) {
144
+ if (s[f] !== undefined && !(Array.isArray(s[f]) && s[f].length === 0))
145
+ return null;
146
+ }
147
+ if (s.limitCount === undefined || (s.limitOption !== undefined && s.limitOption !== "LIMIT_OPTION_COUNT" && s.limitOption !== "LIMIT_OPTION_DEFAULT"))
148
+ return null;
149
+ const targets = s.targetList;
150
+ const sorts = s.sortClause;
151
+ if (targets === undefined || sorts === undefined || sorts.length === 0)
152
+ return null;
153
+ let computes = false;
154
+ for (const t of targets) {
155
+ if (kindOf(t) !== "ResTarget")
156
+ return null;
157
+ const r = rowWise(bodyOf(t).val);
158
+ if (!r.ok)
159
+ return null;
160
+ computes ||= r.computes;
161
+ }
162
+ if (!computes)
163
+ return null;
164
+ // each sort key as an expression over the FROM clause: an ordinal or an output column's name is that output's own
165
+ // expression, as Postgres reads it (an output name before an input column's)
166
+ const outputName = (t) => {
167
+ const b = bodyOf(t);
168
+ if (typeof b.name === "string")
169
+ return b.name;
170
+ const v = b.val;
171
+ if (kindOf(v) === "ColumnRef") {
172
+ const last = bodyOf(v).fields.at(-1);
173
+ return kindOf(last) === "String" ? bodyOf(last).sval : null;
174
+ }
175
+ return null;
176
+ };
177
+ const keys = [];
178
+ for (const sort of sorts) {
179
+ if (kindOf(sort) !== "SortBy")
180
+ return null;
181
+ const b = bodyOf(sort);
182
+ if (b.useOp !== undefined)
183
+ return null;
184
+ const node = b.node;
185
+ let target;
186
+ if (kindOf(node) === "A_Const" && isObject(bodyOf(node).ival)) {
187
+ target = targets[Number(bodyOf(node).ival.ival ?? 0) - 1];
188
+ if (target === undefined)
189
+ return null;
190
+ }
191
+ else if (kindOf(node) === "ColumnRef" && bodyOf(node).fields.length === 1) {
192
+ const name = bodyOf(bodyOf(node).fields[0]).sval;
193
+ const named = targets.filter((t) => outputName(t) === name);
194
+ if (named.length > 1)
195
+ return null;
196
+ target = named[0];
197
+ }
198
+ if (target !== undefined) {
199
+ // a sort by a computed output computes it for every row anyway: nothing to defer
200
+ if (rowWise(bodyOf(target).val).computes)
201
+ return null;
202
+ keys.push(structuredClone(bodyOf(target).val));
203
+ }
204
+ else if (kindOf(node) === "A_Const")
205
+ return null;
206
+ else
207
+ keys.push(structuredClone(node));
208
+ }
209
+ const columns = new Map();
210
+ const list = targets.map((t) => ({ ResTarget: { ...bodyOf(t), val: overLate(bodyOf(t).val, columns) } }));
211
+ const inner = { ...s, targetList: [
212
+ ...[...columns].map(([fields, name]) => ({ ResTarget: { name, val: { ColumnRef: { fields: JSON.parse(fields) } } } })),
213
+ ...keys.map((k, i) => ({ ResTarget: { name: `__mason_o${i + 1}`, val: k } })),
214
+ ], sortClause: sorts.map((sort, i) => ({ SortBy: { ...bodyOf(sort), node: keys[i] } })) };
215
+ return { SelectStmt: {
216
+ targetList: list,
217
+ fromClause: [{ RangeSubselect: { subquery: { SelectStmt: inner }, alias: { aliasname: LATE } } }],
218
+ sortClause: sorts.map((sort, i) => ({ SortBy: { ...bodyOf(sort), node: ref(LATE, `__mason_o${i + 1}`) } })),
219
+ limitOption: "LIMIT_OPTION_DEFAULT", op: "SETOP_NONE",
220
+ } };
221
+ }
@@ -0,0 +1,190 @@
1
+ /**
2
+ * TEXT IN A GLIBC LOCALE'S ORDER, MEASURED FROM THE SOURCE (the BI-tools test of #517: Metabase's field lists
3
+ * under en_US.utf8 listed Hangul, `$` / `€` and U+FFFD elsewhere than Postgres, and `s < 'b'` counted other rows). The
4
+ * order was emulated from assumptions until then — ICU's order of the letters and digits, the rest ignored, then the
5
+ * punctuation by its bytes — and glibc differs from each: Hangul has no letter weight in it, `$` and `€` have one where
6
+ * `@` and `~` have none, U+FFFD sorts after the letters. So the order is no longer assumed: it is READ from the source
7
+ * database, once (read-only SELECTs, a few seconds), kept in the copy (`mason.libc_order`, src/duck/store.ts), read again
8
+ * when the database's collation version moves (`pg_database.datcollversion`: glibc's), and every sort key is built from it.
9
+ *
10
+ * What glibc compares, as measured (Postgres 18 / glibc 2.41 en_US.utf8, the disposable server's):
11
+ * 1. the FIRST level: a weight per character that has one (letters, digits, `$`, U+FFFD; not punctuation, Hangul or
12
+ * spaces), compared in order. Read as CLASSES: the characters in the database's order, two neighbours x < y in one
13
+ * class when x||'b' > y||'a' (only the second character told them apart). A character equal there to a string of
14
+ * others (ß = ss, æ = ae, fi = fi) is that string: proposed by its compatibility decomposition, kept only when the
15
+ * source says it is equal;
16
+ * 2. accents, then 3. case: ICU's second and third levels over the characters of level 1 (the source's own agree
17
+ * with them over everything the sample below holds; nothing here reads them from the source);
18
+ * 4. the LAST level: for each character that has a weight there (punctuation, spaces, Hangul's one shared weight,
19
+ * Latin letters; not CJK), the pair (its position in the string, its weight), position first — `三 < :三` (`三` has
20
+ * none) and `가三 < 三가` both follow. The punctuation's weights are their order (they differ in nothing else); a
21
+ * character of level 1 is placed among them by binary search (p||x < x exactly when p's weight is below x's), and
22
+ * one placed below all of them has none when x||chr(1) is not below chr(1)||x;
23
+ * 5. a tie at every level is Postgres's byte order (`varstr_cmp`'s strcmp), which the caller adds.
24
+ * Held against the source's ORDER BY over 30,000 random strings of 17 scripts and symbol sets: 2 out of order, both
25
+ * full-width digits beside Hangul. Not covered, so REFUSED by name when a value holding one is ordered
26
+ * (`libc-uncovered`): a combining mark (glibc weighs `e` + U+0301 otherwise than ICU), a Thai or Lao vowel written
27
+ * before its consonant (glibc reorders them), a code point this Node does not know as assigned, a private use one. A
28
+ * known difference (`libc-symbol-punctuation`): punctuation right after a digit or symbol outranks a case difference
29
+ * after it in glibc (`1-X < 1x`), and not here.
30
+ */
31
+ import { gapMark } from "./gaps.js";
32
+ import { PgEvalError } from "./pg-eval.js";
33
+ const MAX = 0x110000;
34
+ /** The code points read from the source: assigned, not private use, not a surrogate (the rest are uncovered). */
35
+ export function assignedCodePoints() {
36
+ const out = [];
37
+ for (let g = 1; g < MAX; g++) {
38
+ if (g >= 0xd800 && g <= 0xdfff)
39
+ continue;
40
+ if (/[\p{Cn}\p{Co}]/u.test(String.fromCodePoint(g)))
41
+ continue;
42
+ out.push(g);
43
+ }
44
+ return out;
45
+ }
46
+ /** what glibc orders otherwise than this ordering: combining marks, Thai and Lao vowels written before their consonant */
47
+ const UNCOVERED = /[\p{M}เ-ไເ-ໄ]/u;
48
+ const ints = (xs) => `'{${xs.join(",")}}'::int[]`;
49
+ const texts = (xs) => `ARRAY[${xs.map((x) => `'${x.replaceAll("'", "''")}'`).join(",")}]::text[]`;
50
+ /** Reads the source database's order of text (its default collation), as the module's header says. */
51
+ export async function readLibcOrder(query, collate, version) {
52
+ const points = assignedCodePoints();
53
+ const l1 = new Uint32Array(MAX), l4 = new Uint32Array(MAX), uncovered = new Uint8Array(MAX).fill(1);
54
+ // each character: whether it has a first-level weight ('b'||c < 'ba' and c||'b' > 'aa' only for one that has none);
55
+ // those that have one in the database's order, each marked when its neighbour below is of its class; the rest in
56
+ // the database's order, each marked when its neighbour below has the same last-level weight (pp > c only then)
57
+ const rows = await query(`WITH c AS (SELECT g, chr(g) AS ch FROM unnest(${ints(points)}) AS u(g)),
58
+ k AS (SELECT g, ch, ('b' || ch < 'ba') AND (ch || 'b' > 'aa') AS ignored FROM c),
59
+ n AS (SELECT g, ch, lag(ch) OVER (ORDER BY ch) AS prev, row_number() OVER (ORDER BY ch) AS pos FROM k WHERE NOT ignored),
60
+ i AS (SELECT g, ch, lag(ch) OVER (ORDER BY ch) AS prev, row_number() OVER (ORDER BY ch) AS pos FROM k WHERE ignored)
61
+ SELECT 'n', g::text, coalesce(prev || 'b' > ch || 'a', false)::text, pos::text FROM n
62
+ UNION ALL SELECT 'i', g::text, coalesce(prev || prev > ch, false)::text, pos::text FROM i`);
63
+ // each kind in the database's order
64
+ rows.sort((a, b) => a[0] === b[0] ? Number(a[3]) - Number(b[3]) : a[0] < b[0] ? -1 : 1);
65
+ const weighted = [];
66
+ const ignoredClasses = []; // one character per last-level weight of the ignored ones, in order
67
+ let k = 0;
68
+ for (const [kind, g0, same] of rows) {
69
+ const g = Number(g0);
70
+ uncovered[g] = UNCOVERED.test(String.fromCodePoint(g)) ? 1 : 0;
71
+ if (kind === "n") {
72
+ if (same !== "true")
73
+ k++;
74
+ l1[g] = k;
75
+ weighted.push(g);
76
+ }
77
+ else {
78
+ if (same !== "true" || ignoredClasses.length === 0)
79
+ ignoredClasses.push(g);
80
+ l4[g] = 2 * ignoredClasses.length;
81
+ }
82
+ }
83
+ if (weighted.length === 0 || ignoredClasses.length === 0)
84
+ throw new Error(`reading how the database orders text (${collate}) found no ${weighted.length === 0 ? "letters" : "punctuation"}: Mason cannot sort text as it does`);
85
+ // expansions: proposed by each character's compatibility decomposition (its marks dropped) and the letters NFKD keeps
86
+ // whole, kept where the source says the two are equal at the first level (each sorts after the other given a later b)
87
+ const proposed = [];
88
+ for (const g of weighted) {
89
+ const d = String.fromCodePoint(g).normalize("NFKD").replace(/\p{M}/gu, "");
90
+ if ([...d].length > 1 && !d.includes("\0"))
91
+ proposed.push([g, d]);
92
+ }
93
+ for (const [ch, e] of [["ß", "ss"], ["ẞ", "SS"], ["æ", "ae"], ["Æ", "AE"], ["œ", "oe"], ["Œ", "OE"], ["þ", "th"], ["Þ", "TH"]])
94
+ proposed.push([ch.codePointAt(0), e]);
95
+ const expansions = new Map();
96
+ for (let at = 0; at < proposed.length; at += 5000) {
97
+ const part = proposed.slice(at, at + 5000);
98
+ const eq = await query(`SELECT g::text, ((chr(g) || 'b' > e || 'a') AND (e || 'b' > chr(g) || 'a'))::text FROM unnest(${ints(part.map((p) => p[0]))}, ${texts(part.map((p) => p[1]))}) AS u(g, e)`);
99
+ eq.forEach(([g, yes], j) => { if (yes === "true")
100
+ expansions.set(Number(g), [...part[j][1]].map((c) => c.codePointAt(0))); });
101
+ }
102
+ // each first-level character's last-level weight, among the ignored ones' by binary search
103
+ const lo = new Int32Array(weighted.length), hi = new Int32Array(weighted.length).fill(ignoredClasses.length);
104
+ for (;;) {
105
+ const idx = [], mid = [];
106
+ for (let i = 0; i < weighted.length; i++)
107
+ if (lo[i] < hi[i]) {
108
+ idx.push(i);
109
+ mid.push((lo[i] + hi[i]) >> 1);
110
+ }
111
+ if (idx.length === 0)
112
+ break;
113
+ const below = await query(`SELECT (chr(p) || chr(x) < chr(x))::text FROM unnest(${ints(mid.map((m) => ignoredClasses[m]))}, ${ints(idx.map((i) => weighted[i]))}) WITH ORDINALITY AS u(p, x, n) ORDER BY n`);
114
+ below.forEach(([b], j) => { const i = idx[j]; if (b === "true")
115
+ lo[i] = mid[j] + 1;
116
+ else
117
+ hi[i] = mid[j]; });
118
+ }
119
+ const bottom = [];
120
+ weighted.forEach((g, i) => { l4[g] = 2 * lo[i] + 1; if (lo[i] === 0)
121
+ bottom.push(g); });
122
+ if (bottom.length) {
123
+ const has = await query(`SELECT x::text, (chr(x) || chr(1) < chr(1) || chr(x))::text FROM unnest(${ints(bottom)}) AS u(x)`);
124
+ for (const [x, yes] of has)
125
+ if (yes !== "true")
126
+ l4[Number(x)] = 0;
127
+ }
128
+ return { collate, version, l1, l4, uncovered, expansions };
129
+ }
130
+ /** The characters ICU's accent and case levels read: those with a first-level weight, in order. */
131
+ export function libcKept(o, s) {
132
+ let kept = "";
133
+ for (const ch of s)
134
+ if (o.l1[ch.codePointAt(0)] > 0)
135
+ kept += ch;
136
+ return kept;
137
+ }
138
+ const hex6 = (n) => n.toString(16).padStart(6, "0");
139
+ /** ICU's sort key from its accent level on: what follows the first level's terminator (byte 01, at an even offset) */
140
+ function afterPrimary(icuKey) {
141
+ for (let i = 0; i < icuKey.length; i += 2)
142
+ if (icuKey.charCodeAt(i) === 48 && icuKey.charCodeAt(i + 1) === 49)
143
+ return icuKey.slice(i + 2);
144
+ return "";
145
+ }
146
+ /**
147
+ * The sort key of `s` as one text that orders as glibc orders (bytewise; a space parts the levels, below every hex digit,
148
+ * so a shorter level sorts first): the first-level classes, ICU's accent and case levels of `libcKept(s)` (`icuKey`),
149
+ * the last level's (position, weight) pairs. A value holding a character not covered is refused, naming it.
150
+ */
151
+ export function libcKey(o, s, icuKey) {
152
+ let first = "", last = "", at = 0;
153
+ for (const ch of s) {
154
+ at++;
155
+ const g = ch.codePointAt(0);
156
+ if (o.uncovered[g]) {
157
+ throw new PgEvalError("0A000", `the text ${JSON.stringify(s.length > 40 ? `${s.slice(0, 40)}…` : s)} holds U+${g.toString(16).toUpperCase().padStart(4, "0")}, which Mason cannot order as the database's collation ${o.collate} does (${gapMark("libc-uncovered")})`);
158
+ }
159
+ if (o.l1[g] > 0)
160
+ for (const x of o.expansions.get(g) ?? [g])
161
+ if (o.l1[x] > 0)
162
+ first += hex6(o.l1[x]);
163
+ if (o.l4[g] > 0)
164
+ last += hex6(at) + hex6(o.l4[g]);
165
+ }
166
+ return `${first} ${afterPrimary(icuKey)} ${last}`;
167
+ }
168
+ /** The order as bytes, to keep in the copy: the three arrays and the expansions, as JSON. */
169
+ export function encodeLibcOrder(o) {
170
+ // only the code points that are not 0 / covered: the arrays are mostly empty
171
+ const sparse = (a, skip) => {
172
+ const out = [];
173
+ for (let g = 0; g < MAX; g++)
174
+ if (a[g] !== skip)
175
+ out.push([g, a[g]]);
176
+ return out;
177
+ };
178
+ return JSON.stringify({ collate: o.collate, version: o.version, l1: sparse(o.l1, 0), l4: sparse(o.l4, 0), covered: sparse(o.uncovered, 1).map(([g]) => g), expansions: [...o.expansions] });
179
+ }
180
+ export function decodeLibcOrder(text) {
181
+ const v = JSON.parse(text);
182
+ const l1 = new Uint32Array(MAX), l4 = new Uint32Array(MAX), uncovered = new Uint8Array(MAX).fill(1);
183
+ for (const [g, x] of v.l1)
184
+ l1[g] = x;
185
+ for (const [g, x] of v.l4)
186
+ l4[g] = x;
187
+ for (const g of v.covered)
188
+ uncovered[g] = 0;
189
+ return { collate: v.collate, version: v.version, l1, l4, uncovered, expansions: new Map(v.expansions) };
190
+ }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * POSTGRES'S OWN FUNCTIONS, ON A WORKER (src/duck/pg-eval.ts): a PGlite with no tables, asked by DuckDB's scalar
3
+ * functions what Postgres makes of a chunk of values — a text read as a date, an interval, a boolean; a jsonpath
4
+ * evaluated. The main thread blocks on `signal` while this thread answers, so nothing here may wait on the main thread.
5
+ *
6
+ * In (on `port`): `{ sql, params, timeZone }`. Out: `{ ok: true, rows }` (each value its text, an array the JSON of its
7
+ * elements' text, or null) or
8
+ * `{ ok: false, code, message }` (Postgres's SQLSTATE and words); then `signal[0]` is set to 1 and notified.
9
+ */
10
+ import { workerData } from "node:worker_threads";
11
+ import { PGlite } from "@electric-sql/pglite";
12
+ const { port, signal } = workerData;
13
+ const db = new PGlite();
14
+ let zone = null;
15
+ const reply = (m) => { port.postMessage(m); Atomics.store(signal, 0, 1); Atomics.notify(signal, 0); };
16
+ port.on("message", (m) => {
17
+ void (async () => {
18
+ try {
19
+ await db.waitReady;
20
+ if (zone !== m.timeZone) {
21
+ await db.exec(`SET TimeZone = '${m.timeZone.replace(/'/g, "''")}'`);
22
+ zone = m.timeZone;
23
+ }
24
+ const result = await db.query(m.sql, m.params, { rowMode: "array" });
25
+ // an array as the JSON of its elements' text (the `list` kind, src/duck/functions.ts)
26
+ const textOf = (v) => v === null ? null : Array.isArray(v) ? JSON.stringify(v.map(textOf)) : String(v);
27
+ reply({ ok: true, rows: result.rows.map((r) => r.map(textOf)) });
28
+ }
29
+ catch (e) {
30
+ const err = e;
31
+ reply({ ok: false, code: typeof err.code === "string" ? err.code : "XX000", message: err.message ?? String(e) });
32
+ }
33
+ })();
34
+ });
@@ -0,0 +1,59 @@
1
+ /**
2
+ * POSTGRES'S OWN FUNCTIONS, A CHUNK AT A TIME (the corpus run's unverified casts). What a text MEANS as a
3
+ * date, a timestamp, an interval, a boolean, an integer or a numeric is Postgres's input functions' to say — they read
4
+ * `01/02/2026`, `January 2, 2026`, `J2461043`, `2026-01-02 America/New_York`, `1 day ago`, `P1DT2H`, ` yes `, `0x1F`,
5
+ * `1_000` — and DuckDB reads ISO alone. So nothing reimplements them: a DuckDB scalar function (src/duck/functions.ts
6
+ * `__mason_pg_*`) hands its chunk's distinct values to a PGlite on a worker thread (pg-eval-worker.ts), which applies
7
+ * Postgres's own function under the session's TimeZone and answers each value, or Postgres's error. The same route
8
+ * answers the few functions DuckDB has no equivalent of (jsonpath's).
9
+ *
10
+ * DuckDB calls a scalar function synchronously, so the call BLOCKS the main thread until the worker answers
11
+ * (`Atomics.wait`, then `receiveMessageOnPort`): about 8 ms for 2,048 values. A worker that does not answer within
12
+ * `TIMEOUT_MS` is terminated and the statement fails; the next call starts another.
13
+ */
14
+ import { MessageChannel, receiveMessageOnPort, Worker } from "node:worker_threads";
15
+ import { fileURLToPath } from "node:url";
16
+ import { runnable } from "../lib/runnable.js";
17
+ const TIMEOUT_MS = 30_000;
18
+ export class PgEvalError extends Error {
19
+ code;
20
+ constructor(code, message) { super(message); this.code = code; }
21
+ }
22
+ let current = null;
23
+ function start() {
24
+ const signal = new Int32Array(new SharedArrayBuffer(4));
25
+ const { port1, port2 } = new MessageChannel();
26
+ const worker = // `runnable`: the package ships this worker as .js (cli/build.ts), and a URL to the .ts would wait out TIMEOUT_MS and fail
27
+ new Worker(runnable(fileURLToPath(new URL("./pg-eval-worker.ts", import.meta.url))), { workerData: { port: port2, signal }, transferList: [port2] });
28
+ // the process may end while it lives: it holds nothing that must be closed
29
+ worker.unref();
30
+ port1.unref();
31
+ worker.on("error", (e) => { console.error(`mason: the Postgres function worker failed: ${e.message}`); if (current?.worker === worker)
32
+ current = null; });
33
+ return { worker, port: port1, signal };
34
+ }
35
+ /** Runs one statement on the worker's Postgres and returns its rows as text; Postgres's error is a PgEvalError. */
36
+ export function pgEvalSync(sql, params, timeZone) {
37
+ current ??= start();
38
+ const w = current;
39
+ Atomics.store(w.signal, 0, 0);
40
+ w.port.postMessage({ sql, params, timeZone });
41
+ const waited = Atomics.wait(w.signal, 0, 0, TIMEOUT_MS);
42
+ const got = receiveMessageOnPort(w.port);
43
+ if (waited === "timed-out" || got === undefined) {
44
+ current = null;
45
+ void w.worker.terminate();
46
+ throw new PgEvalError("57014", `Postgres's function did not answer within ${TIMEOUT_MS / 1000} s (Mason computes it on its own Postgres)`);
47
+ }
48
+ const r = got.message;
49
+ if (!r.ok)
50
+ throw new PgEvalError(r.code, r.message);
51
+ return r.rows;
52
+ }
53
+ /** Stops the worker (a store that closes). */
54
+ export async function stopPgEval() {
55
+ const w = current;
56
+ current = null;
57
+ if (w !== null)
58
+ await w.worker.terminate();
59
+ }