@sqlanvil/cli 1.27.2 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +1578 -32
- package/package.json +1 -1
package/bundle.js
CHANGED
|
@@ -50646,7 +50646,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
50646
50646
|
.filter(validationQuery => !!validationQuery.query);
|
|
50647
50647
|
}
|
|
50648
50648
|
|
|
50649
|
-
const version = "1.
|
|
50649
|
+
const version = "1.28.0";
|
|
50650
50650
|
const dataformVersion = "3.0.61";
|
|
50651
50651
|
|
|
50652
50652
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -51496,13 +51496,17 @@ function renderDeclarationSqlx(opts) {
|
|
|
51496
51496
|
lines.push(` schema: ${quote(opts.schema)},`);
|
|
51497
51497
|
}
|
|
51498
51498
|
lines.push(` name: ${quote(opts.name)},`);
|
|
51499
|
-
const typeLines = opts.columns.map(function (c) {
|
|
51499
|
+
const typeLines = opts.columns.map(function (c) {
|
|
51500
|
+
return ` ${keyToken(c.name.toLowerCase())}: ${quote(c.type)}`;
|
|
51501
|
+
});
|
|
51500
51502
|
const described = opts.columns.filter(function (c) { return !!c.description; });
|
|
51501
51503
|
lines.push(` columnTypes: {`);
|
|
51502
51504
|
lines.push(typeLines.join(",\n"));
|
|
51503
51505
|
lines.push(described.length > 0 ? ` },` : ` }`);
|
|
51504
51506
|
if (described.length > 0) {
|
|
51505
|
-
const descLines = described.map(function (c) {
|
|
51507
|
+
const descLines = described.map(function (c) {
|
|
51508
|
+
return ` ${keyToken(c.name.toLowerCase())}: ${quote(c.description)}`;
|
|
51509
|
+
});
|
|
51506
51510
|
lines.push(` columns: {`);
|
|
51507
51511
|
lines.push(descLines.join(",\n"));
|
|
51508
51512
|
lines.push(` }`);
|
|
@@ -52586,18 +52590,34 @@ class PostgresDbAdapter {
|
|
|
52586
52590
|
|
|
52587
52591
|
const MAX_PARAMS_PER_INSERT = 60000;
|
|
52588
52592
|
const quoteIdent = (id) => `"${String(id).replace(/"/g, '""')}"`;
|
|
52593
|
+
function foldColumns(columnTypes, target) {
|
|
52594
|
+
var _a;
|
|
52595
|
+
const cols = Object.keys(columnTypes).map(source => ({ source, target: source.toLowerCase() }));
|
|
52596
|
+
const byFolded = new Map();
|
|
52597
|
+
for (const c of cols) {
|
|
52598
|
+
byFolded.set(c.target, [...((_a = byFolded.get(c.target)) !== null && _a !== void 0 ? _a : []), c.source]);
|
|
52599
|
+
}
|
|
52600
|
+
for (const [folded, sources] of byFolded) {
|
|
52601
|
+
if (sources.length > 1) {
|
|
52602
|
+
throw new Error(`Source ${target.schema}.${target.name} has columns that differ only in case ` +
|
|
52603
|
+
`(${sources.join(", ")}); PostgreSQL folds them to the same identifier "${folded}". ` +
|
|
52604
|
+
`Rename one at the source, or select them with distinct aliases.`);
|
|
52605
|
+
}
|
|
52606
|
+
}
|
|
52607
|
+
return cols;
|
|
52608
|
+
}
|
|
52589
52609
|
async function createPgLoader(args) {
|
|
52590
52610
|
var _a;
|
|
52591
52611
|
const { target, columnTypes } = args;
|
|
52592
|
-
const cols = Object.keys(columnTypes);
|
|
52593
52612
|
const coerce = args.coerce || ((v) => (v === undefined ? null : v));
|
|
52613
|
+
const cols = foldColumns(columnTypes, target);
|
|
52594
52614
|
const pg = await PostgresDbAdapter.create(args.pg, {
|
|
52595
52615
|
concurrencyLimit: 1,
|
|
52596
52616
|
disableSslForTestsOnly: args.disableSslForTestsOnly
|
|
52597
52617
|
});
|
|
52598
52618
|
try {
|
|
52599
52619
|
const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
|
|
52600
|
-
const colDefs = cols.map(c => `${quoteIdent(c)} ${columnTypes[c]}`).join(", ");
|
|
52620
|
+
const colDefs = cols.map(c => `${quoteIdent(c.target)} ${columnTypes[c.source]}`).join(", ");
|
|
52601
52621
|
await pg.execute(`create schema if not exists ${quoteIdent(target.schema)}`);
|
|
52602
52622
|
const { rows: existing } = await pg.execute(`select c.relkind from pg_class c join pg_namespace n on n.oid = c.relnamespace ` +
|
|
52603
52623
|
`where n.nspname = $1 and c.relname = $2`, { params: [target.schema, target.name] });
|
|
@@ -52617,7 +52637,7 @@ async function createPgLoader(args) {
|
|
|
52617
52637
|
await pg.close();
|
|
52618
52638
|
throw e;
|
|
52619
52639
|
}
|
|
52620
|
-
const colIdents = cols.map(quoteIdent).join(", ");
|
|
52640
|
+
const colIdents = cols.map(c => quoteIdent(c.target)).join(", ");
|
|
52621
52641
|
const qualified = `${quoteIdent(target.schema)}.${quoteIdent(target.name)}`;
|
|
52622
52642
|
const batchRows = Math.max(1, Math.min(1000, Math.floor(MAX_PARAMS_PER_INSERT / cols.length)));
|
|
52623
52643
|
return {
|
|
@@ -52628,7 +52648,7 @@ async function createPgLoader(args) {
|
|
|
52628
52648
|
const params = [];
|
|
52629
52649
|
const tuples = batch.map((row, r) => {
|
|
52630
52650
|
const placeholders = cols.map((c, ci) => `$${r * cols.length + ci + 1}`);
|
|
52631
|
-
cols.forEach(c => params.push(coerce(row[c])));
|
|
52651
|
+
cols.forEach(c => params.push(coerce(row[c.source])));
|
|
52632
52652
|
return `(${placeholders.join(", ")})`;
|
|
52633
52653
|
});
|
|
52634
52654
|
await pg.execute(`insert into ${qualified} (${colIdents}) values ${tuples.join(", ")}`, {
|
|
@@ -54493,6 +54513,399 @@ ${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></t
|
|
|
54493
54513
|
`;
|
|
54494
54514
|
}
|
|
54495
54515
|
|
|
54516
|
+
function tokenize(sql) {
|
|
54517
|
+
const tokens = [];
|
|
54518
|
+
let i = 0;
|
|
54519
|
+
const push = (kind, start, end, value) => tokens.push({ kind, start, end, text: sql.slice(start, end), value });
|
|
54520
|
+
while (i < sql.length) {
|
|
54521
|
+
const ch = sql[i];
|
|
54522
|
+
if (/\s/.test(ch)) {
|
|
54523
|
+
const start = i;
|
|
54524
|
+
while (i < sql.length && /\s/.test(sql[i]))
|
|
54525
|
+
i++;
|
|
54526
|
+
push("whitespace", start, i);
|
|
54527
|
+
continue;
|
|
54528
|
+
}
|
|
54529
|
+
if (ch === "$" && sql[i + 1] === "{") {
|
|
54530
|
+
const start = i;
|
|
54531
|
+
let depth = 0;
|
|
54532
|
+
for (; i < sql.length; i++) {
|
|
54533
|
+
if (sql[i] === "{")
|
|
54534
|
+
depth++;
|
|
54535
|
+
else if (sql[i] === "}" && --depth === 0) {
|
|
54536
|
+
i++;
|
|
54537
|
+
break;
|
|
54538
|
+
}
|
|
54539
|
+
}
|
|
54540
|
+
push("template", start, i);
|
|
54541
|
+
continue;
|
|
54542
|
+
}
|
|
54543
|
+
if (ch === "-" && sql[i + 1] === "-") {
|
|
54544
|
+
const start = i;
|
|
54545
|
+
while (i < sql.length && sql[i] !== "\n")
|
|
54546
|
+
i++;
|
|
54547
|
+
push("comment", start, i);
|
|
54548
|
+
continue;
|
|
54549
|
+
}
|
|
54550
|
+
if (ch === "#") {
|
|
54551
|
+
const start = i;
|
|
54552
|
+
while (i < sql.length && sql[i] !== "\n")
|
|
54553
|
+
i++;
|
|
54554
|
+
push("comment", start, i);
|
|
54555
|
+
continue;
|
|
54556
|
+
}
|
|
54557
|
+
if (ch === "/" && sql[i + 1] === "*") {
|
|
54558
|
+
const start = i;
|
|
54559
|
+
i += 2;
|
|
54560
|
+
while (i < sql.length && !(sql[i] === "*" && sql[i + 1] === "/"))
|
|
54561
|
+
i++;
|
|
54562
|
+
i = Math.min(sql.length, i + 2);
|
|
54563
|
+
push("comment", start, i);
|
|
54564
|
+
continue;
|
|
54565
|
+
}
|
|
54566
|
+
const rawPrefix = (ch === "r" || ch === "R") && (sql[i + 1] === "'" || sql[i + 1] === '"');
|
|
54567
|
+
if (ch === "'" || ch === '"' || rawPrefix) {
|
|
54568
|
+
const start = i;
|
|
54569
|
+
if (rawPrefix)
|
|
54570
|
+
i++;
|
|
54571
|
+
const quote = sql[i];
|
|
54572
|
+
i++;
|
|
54573
|
+
for (; i < sql.length; i++) {
|
|
54574
|
+
if (sql[i] !== quote)
|
|
54575
|
+
continue;
|
|
54576
|
+
if (quote === "'" && sql[i + 1] === "'") {
|
|
54577
|
+
i++;
|
|
54578
|
+
continue;
|
|
54579
|
+
}
|
|
54580
|
+
i++;
|
|
54581
|
+
break;
|
|
54582
|
+
}
|
|
54583
|
+
const raw = sql.slice(start, i);
|
|
54584
|
+
const inner = raw.slice(rawPrefix ? 2 : 1, raw.length - (raw.endsWith(quote) ? 1 : 0));
|
|
54585
|
+
push(quote === '"' ? "quoted-ident" : "string", start, i, inner);
|
|
54586
|
+
continue;
|
|
54587
|
+
}
|
|
54588
|
+
if (ch === "`") {
|
|
54589
|
+
const start = i;
|
|
54590
|
+
i++;
|
|
54591
|
+
while (i < sql.length && sql[i] !== "`")
|
|
54592
|
+
i++;
|
|
54593
|
+
i = Math.min(sql.length, i + 1);
|
|
54594
|
+
push("backtick-ident", start, i, sql.slice(start + 1, i - 1));
|
|
54595
|
+
continue;
|
|
54596
|
+
}
|
|
54597
|
+
if (/[0-9]/.test(ch)) {
|
|
54598
|
+
const start = i;
|
|
54599
|
+
while (i < sql.length && /[0-9._eE]/.test(sql[i]))
|
|
54600
|
+
i++;
|
|
54601
|
+
push("number", start, i);
|
|
54602
|
+
continue;
|
|
54603
|
+
}
|
|
54604
|
+
if (/[A-Za-z_]/.test(ch)) {
|
|
54605
|
+
const start = i;
|
|
54606
|
+
while (i < sql.length && /[A-Za-z0-9_$]/.test(sql[i]))
|
|
54607
|
+
i++;
|
|
54608
|
+
push("word", start, i);
|
|
54609
|
+
continue;
|
|
54610
|
+
}
|
|
54611
|
+
push("punct", i, i + 1);
|
|
54612
|
+
i++;
|
|
54613
|
+
}
|
|
54614
|
+
return tokens;
|
|
54615
|
+
}
|
|
54616
|
+
function significant(tokens) {
|
|
54617
|
+
return tokens.filter(t => t.kind !== "whitespace" && t.kind !== "comment");
|
|
54618
|
+
}
|
|
54619
|
+
function isWord(t, ...words) {
|
|
54620
|
+
return (!!t && t.kind === "word" && words.some(w => t.text.toLowerCase() === w.toLowerCase()));
|
|
54621
|
+
}
|
|
54622
|
+
function matchBracket(toks, open) {
|
|
54623
|
+
var _a;
|
|
54624
|
+
const openCh = (_a = toks[open]) === null || _a === void 0 ? void 0 : _a.text;
|
|
54625
|
+
const closeCh = openCh === "(" ? ")" : openCh === "[" ? "]" : null;
|
|
54626
|
+
if (!closeCh)
|
|
54627
|
+
return -1;
|
|
54628
|
+
let depth = 0;
|
|
54629
|
+
for (let i = open; i < toks.length; i++) {
|
|
54630
|
+
if (toks[i].text === openCh)
|
|
54631
|
+
depth++;
|
|
54632
|
+
else if (toks[i].text === closeCh && --depth === 0)
|
|
54633
|
+
return i;
|
|
54634
|
+
}
|
|
54635
|
+
return -1;
|
|
54636
|
+
}
|
|
54637
|
+
const CLAUSE_ENDS = ["where", "group", "order", "having", "window", "limit", "qualify", "union", "intersect", "except"];
|
|
54638
|
+
function selectScopes(toks) {
|
|
54639
|
+
const scopes = [];
|
|
54640
|
+
for (let i = 0; i < toks.length; i++) {
|
|
54641
|
+
if (!isWord(toks[i], "select"))
|
|
54642
|
+
continue;
|
|
54643
|
+
let depth = 0;
|
|
54644
|
+
let from = -1;
|
|
54645
|
+
let listEnd = -1;
|
|
54646
|
+
let end = toks.length;
|
|
54647
|
+
for (let j = i + 1; j < toks.length; j++) {
|
|
54648
|
+
const t = toks[j];
|
|
54649
|
+
if (t.text === "(" || t.text === "[")
|
|
54650
|
+
depth++;
|
|
54651
|
+
else if (t.text === ")" || t.text === "]") {
|
|
54652
|
+
if (depth === 0) {
|
|
54653
|
+
end = j;
|
|
54654
|
+
break;
|
|
54655
|
+
}
|
|
54656
|
+
depth--;
|
|
54657
|
+
}
|
|
54658
|
+
else if (depth === 0 && t.kind === "word") {
|
|
54659
|
+
const w = t.text.toLowerCase();
|
|
54660
|
+
if (w === "from" && from === -1) {
|
|
54661
|
+
from = j;
|
|
54662
|
+
listEnd = j;
|
|
54663
|
+
}
|
|
54664
|
+
else if (CLAUSE_ENDS.includes(w)) {
|
|
54665
|
+
if (w === "except" && from === -1)
|
|
54666
|
+
continue;
|
|
54667
|
+
end = j;
|
|
54668
|
+
break;
|
|
54669
|
+
}
|
|
54670
|
+
}
|
|
54671
|
+
else if (depth === 0 && t.text === ";") {
|
|
54672
|
+
end = j;
|
|
54673
|
+
break;
|
|
54674
|
+
}
|
|
54675
|
+
}
|
|
54676
|
+
if (listEnd === -1)
|
|
54677
|
+
listEnd = end;
|
|
54678
|
+
scopes.push({
|
|
54679
|
+
select: i,
|
|
54680
|
+
listStart: i + 1,
|
|
54681
|
+
listEnd,
|
|
54682
|
+
from,
|
|
54683
|
+
fromStart: from === -1 ? -1 : from + 1,
|
|
54684
|
+
fromEnd: from === -1 ? -1 : end,
|
|
54685
|
+
});
|
|
54686
|
+
}
|
|
54687
|
+
return scopes;
|
|
54688
|
+
}
|
|
54689
|
+
function splitOnCommas(toks, start, end) {
|
|
54690
|
+
const out = [];
|
|
54691
|
+
let depth = 0;
|
|
54692
|
+
const stop = Math.min(end, toks.length);
|
|
54693
|
+
let from = Math.max(0, start);
|
|
54694
|
+
for (let i = from; i < stop; i++) {
|
|
54695
|
+
const t = toks[i].text;
|
|
54696
|
+
if (t === "(" || t === "[")
|
|
54697
|
+
depth++;
|
|
54698
|
+
else if (t === ")" || t === "]")
|
|
54699
|
+
depth--;
|
|
54700
|
+
else if (t === "," && depth === 0) {
|
|
54701
|
+
out.push([from, i]);
|
|
54702
|
+
from = i + 1;
|
|
54703
|
+
}
|
|
54704
|
+
}
|
|
54705
|
+
if (from < stop)
|
|
54706
|
+
out.push([from, stop]);
|
|
54707
|
+
return out;
|
|
54708
|
+
}
|
|
54709
|
+
const JOIN_NOISE = new Set([
|
|
54710
|
+
"on", "using", "left", "right", "inner", "full", "cross", "outer", "join", "lateral", "natural",
|
|
54711
|
+
]);
|
|
54712
|
+
function relationsIn(toks, start, end) {
|
|
54713
|
+
var _a, _b, _c;
|
|
54714
|
+
const rels = [];
|
|
54715
|
+
let i = start;
|
|
54716
|
+
while (i < end) {
|
|
54717
|
+
if (i > start && !isWord(toks[i - 1], "from", "join")) {
|
|
54718
|
+
i++;
|
|
54719
|
+
continue;
|
|
54720
|
+
}
|
|
54721
|
+
let j = i;
|
|
54722
|
+
let subquery = false;
|
|
54723
|
+
const parts = [];
|
|
54724
|
+
if (((_a = toks[j]) === null || _a === void 0 ? void 0 : _a.text) === "(") {
|
|
54725
|
+
const close = matchBracket(toks, j);
|
|
54726
|
+
if (close < 0)
|
|
54727
|
+
break;
|
|
54728
|
+
subquery = true;
|
|
54729
|
+
parts.push(...toks.slice(j, close + 1));
|
|
54730
|
+
j = close + 1;
|
|
54731
|
+
}
|
|
54732
|
+
else {
|
|
54733
|
+
while (j < end &&
|
|
54734
|
+
(toks[j].kind === "template" ||
|
|
54735
|
+
toks[j].kind === "word" ||
|
|
54736
|
+
toks[j].kind === "quoted-ident" ||
|
|
54737
|
+
toks[j].kind === "backtick-ident" ||
|
|
54738
|
+
toks[j].text === ".")) {
|
|
54739
|
+
if (isWord(toks[j], "as"))
|
|
54740
|
+
break;
|
|
54741
|
+
if (toks[j].kind === "word" && JOIN_NOISE.has(toks[j].text.toLowerCase()))
|
|
54742
|
+
break;
|
|
54743
|
+
parts.push(toks[j]);
|
|
54744
|
+
j++;
|
|
54745
|
+
}
|
|
54746
|
+
}
|
|
54747
|
+
if (!parts.length) {
|
|
54748
|
+
i++;
|
|
54749
|
+
continue;
|
|
54750
|
+
}
|
|
54751
|
+
let alias;
|
|
54752
|
+
let k = j;
|
|
54753
|
+
if (isWord(toks[k], "as"))
|
|
54754
|
+
k++;
|
|
54755
|
+
if (((_b = toks[k]) === null || _b === void 0 ? void 0 : _b.kind) === "word" && !JOIN_NOISE.has(toks[k].text.toLowerCase())) {
|
|
54756
|
+
alias = toks[k].text.toLowerCase();
|
|
54757
|
+
k++;
|
|
54758
|
+
}
|
|
54759
|
+
else {
|
|
54760
|
+
const last = [...parts].reverse().find(p => p.kind === "word" || p.kind === "quoted-ident");
|
|
54761
|
+
if (last && !subquery)
|
|
54762
|
+
alias = ((_c = last.value) !== null && _c !== void 0 ? _c : last.text).toLowerCase();
|
|
54763
|
+
}
|
|
54764
|
+
rels.push({ tokens: parts, alias, subquery });
|
|
54765
|
+
i = Math.max(k, j);
|
|
54766
|
+
}
|
|
54767
|
+
return rels;
|
|
54768
|
+
}
|
|
54769
|
+
|
|
54770
|
+
function findStructArrays(file, source) {
|
|
54771
|
+
var _a, _b, _c, _d, _e, _f;
|
|
54772
|
+
const toks = significant(tokenize(source));
|
|
54773
|
+
const found = [];
|
|
54774
|
+
for (let i = 0; i < toks.length; i++) {
|
|
54775
|
+
if (!isWord(toks[i], "array") || ((_a = toks[i + 1]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
54776
|
+
continue;
|
|
54777
|
+
const close = matchBracket(toks, i + 1);
|
|
54778
|
+
if (close < 0)
|
|
54779
|
+
continue;
|
|
54780
|
+
if (!isWord(toks[i + 2], "select") || !isWord(toks[i + 3], "as") || !isWord(toks[i + 4], "struct")) {
|
|
54781
|
+
continue;
|
|
54782
|
+
}
|
|
54783
|
+
const inner = toks.slice(i + 5, close);
|
|
54784
|
+
const innerSql = inner.map(t => t.text).join(" ");
|
|
54785
|
+
const innerToks = significant(tokenize(`select ${innerSql}`));
|
|
54786
|
+
const [scope] = selectScopes(innerToks);
|
|
54787
|
+
const fields = [];
|
|
54788
|
+
if (scope) {
|
|
54789
|
+
for (const [s, e] of splitOnCommas(innerToks, scope.listStart, scope.listEnd)) {
|
|
54790
|
+
const item = innerToks.slice(s, e);
|
|
54791
|
+
const last = item[item.length - 1];
|
|
54792
|
+
fields.push(((_c = (_b = last === null || last === void 0 ? void 0 : last.value) !== null && _b !== void 0 ? _b : last === null || last === void 0 ? void 0 : last.text) !== null && _c !== void 0 ? _c : "").replace(/^"|"$/g, ""));
|
|
54793
|
+
}
|
|
54794
|
+
}
|
|
54795
|
+
const rels = scope && scope.from !== -1 ? relationsIn(innerToks, scope.fromStart, scope.fromEnd) : [];
|
|
54796
|
+
const source_ = (_e = (_d = rels[0]) === null || _d === void 0 ? void 0 : _d.tokens.map(t => t.text).join("")) !== null && _e !== void 0 ? _e : "";
|
|
54797
|
+
let correlation;
|
|
54798
|
+
const whereIdx = innerToks.findIndex(t => isWord(t, "where"));
|
|
54799
|
+
if (whereIdx !== -1) {
|
|
54800
|
+
const eq = innerToks.findIndex((t, k) => k > whereIdx && t.text === "=");
|
|
54801
|
+
const left = innerToks.slice(Math.max(whereIdx + 1, eq - 3), eq);
|
|
54802
|
+
const right = innerToks.slice(eq + 1, eq + 4);
|
|
54803
|
+
const tail = (parts) => {
|
|
54804
|
+
var _a;
|
|
54805
|
+
const words = parts.filter(t => t.kind === "word" || t.kind === "quoted-ident");
|
|
54806
|
+
return words.length ? ((_a = words[words.length - 1].value) !== null && _a !== void 0 ? _a : words[words.length - 1].text) : "";
|
|
54807
|
+
};
|
|
54808
|
+
if (eq !== -1)
|
|
54809
|
+
correlation = { sourceColumn: tail(left), parentColumn: tail(right) };
|
|
54810
|
+
}
|
|
54811
|
+
let end = toks[close].end;
|
|
54812
|
+
let column = "";
|
|
54813
|
+
if (isWord(toks[close + 1], "as") && toks[close + 2]) {
|
|
54814
|
+
column = ((_f = toks[close + 2].value) !== null && _f !== void 0 ? _f : toks[close + 2].text).replace(/^"|"$/g, "");
|
|
54815
|
+
end = toks[close + 2].end;
|
|
54816
|
+
}
|
|
54817
|
+
found.push({
|
|
54818
|
+
file,
|
|
54819
|
+
line: source.slice(0, toks[i].start).split("\n").length,
|
|
54820
|
+
column,
|
|
54821
|
+
fields,
|
|
54822
|
+
source: source_,
|
|
54823
|
+
correlation,
|
|
54824
|
+
start: toks[i].start,
|
|
54825
|
+
end,
|
|
54826
|
+
});
|
|
54827
|
+
}
|
|
54828
|
+
return found;
|
|
54829
|
+
}
|
|
54830
|
+
function suggestedSql(site) {
|
|
54831
|
+
var _a, _b, _c, _d, _e, _f;
|
|
54832
|
+
const fields = site.fields.length ? site.fields : ["…"];
|
|
54833
|
+
const key = (_b = (_a = site.correlation) === null || _a === void 0 ? void 0 : _a.sourceColumn) !== null && _b !== void 0 ? _b : "<parent key>";
|
|
54834
|
+
const parentKey = (_d = (_c = site.correlation) === null || _c === void 0 ? void 0 : _c.parentColumn) !== null && _d !== void 0 ? _d : "<parent key>";
|
|
54835
|
+
const src = site.source || "<source relation>";
|
|
54836
|
+
if (site.strategy === "collapse" && site.consumers.length) {
|
|
54837
|
+
const [k, v] = [(_e = fields[0]) !== null && _e !== void 0 ? _e : "key", (_f = fields[1]) !== null && _f !== void 0 ? _f : "value"];
|
|
54838
|
+
return [
|
|
54839
|
+
`-- Drop the \`${site.column}\` column from this action entirely, and have each reader`,
|
|
54840
|
+
`-- pivot ${src} directly. A consumer that today reads:`,
|
|
54841
|
+
`--`,
|
|
54842
|
+
`-- (select any_value(${v}) from unnest(${site.column}) where ${k} = '<name>')`,
|
|
54843
|
+
`--`,
|
|
54844
|
+
`-- becomes, selecting from ${src} grouped by ${parentKey}:`,
|
|
54845
|
+
``,
|
|
54846
|
+
`max(${v}) filter (where ${k} = '<name>') as <name>`,
|
|
54847
|
+
].join("\n");
|
|
54848
|
+
}
|
|
54849
|
+
if (site.strategy === "collapse") {
|
|
54850
|
+
return [
|
|
54851
|
+
`-- Nothing reads \`${site.column}\`. Confirm that, then delete the column — porting an`,
|
|
54852
|
+
`-- unused array costs a model change for no consumer.`,
|
|
54853
|
+
].join("\n");
|
|
54854
|
+
}
|
|
54855
|
+
if (site.strategy === "child-table") {
|
|
54856
|
+
const cols = fields.map(f => ` ${f}`).join(",\n");
|
|
54857
|
+
return [
|
|
54858
|
+
`-- A new action holding the rows, keyed by the parent — this IS what the array was:`,
|
|
54859
|
+
``,
|
|
54860
|
+
`config { type: "view" }`,
|
|
54861
|
+
``,
|
|
54862
|
+
`select`,
|
|
54863
|
+
` ${key} as ${parentKey},`,
|
|
54864
|
+
cols,
|
|
54865
|
+
`from \${ref("${src.replace(/^\$\{ref\(["']|["']\)\}$/g, "")}")}`,
|
|
54866
|
+
``,
|
|
54867
|
+
`-- Then drop \`${site.column}\` from the parent and have readers join instead:`,
|
|
54868
|
+
`-- join \${ref("<this new action>")} c on c.${parentKey} = p.${parentKey}`,
|
|
54869
|
+
].join("\n");
|
|
54870
|
+
}
|
|
54871
|
+
const pairs = fields.map(f => `'${f}', ${f}`).join(", ");
|
|
54872
|
+
return [
|
|
54873
|
+
`-- Fallback. Keeps the shape but pushes a document model into a relational warehouse:`,
|
|
54874
|
+
``,
|
|
54875
|
+
`(select jsonb_agg(jsonb_build_object(${pairs})) from ${src} …) as ${site.column}`,
|
|
54876
|
+
``,
|
|
54877
|
+
`-- Readers then need jsonb_array_elements(...) and ->> instead of UNNEST.`,
|
|
54878
|
+
].join("\n");
|
|
54879
|
+
}
|
|
54880
|
+
function chooseStrategies(sites, files) {
|
|
54881
|
+
return sites.map(site => {
|
|
54882
|
+
const consumers = [];
|
|
54883
|
+
for (const f of files) {
|
|
54884
|
+
if (f.file === site.file || !site.column)
|
|
54885
|
+
continue;
|
|
54886
|
+
const toks = significant(tokenize(f.source));
|
|
54887
|
+
const uses = toks
|
|
54888
|
+
.map((t, i) => ({ t, i }))
|
|
54889
|
+
.filter(({ t }) => t.kind === "word" && t.text.toLowerCase() === site.column.toLowerCase());
|
|
54890
|
+
if (!uses.length)
|
|
54891
|
+
continue;
|
|
54892
|
+
const unnestOnly = uses.every(({ i }) => { var _a; return isWord(toks[i - 2], "unnest") && ((_a = toks[i - 1]) === null || _a === void 0 ? void 0 : _a.text) === "("; });
|
|
54893
|
+
consumers.push({ file: f.file, unnestOnly });
|
|
54894
|
+
}
|
|
54895
|
+
const readers = consumers.length;
|
|
54896
|
+
if (readers && consumers.every(c => c.unnestOnly)) {
|
|
54897
|
+
return Object.assign(Object.assign({}, site), { consumers, strategy: "collapse", rationale: `every read of \`${site.column}\` is an UNNEST (${readers} file(s)), so the array is a ` +
|
|
54898
|
+
`round trip — read ${site.source || "the source"} directly and drop the array` });
|
|
54899
|
+
}
|
|
54900
|
+
if (!readers) {
|
|
54901
|
+
return Object.assign(Object.assign({}, site), { consumers, strategy: "collapse", rationale: `nothing in the project reads \`${site.column}\` — confirm it is unused, then drop it ` +
|
|
54902
|
+
`rather than porting it` });
|
|
54903
|
+
}
|
|
54904
|
+
return Object.assign(Object.assign({}, site), { consumers, strategy: "child-table", rationale: `\`${site.column}\` is read in ${readers} file(s) and not only via UNNEST, so it needs to ` +
|
|
54905
|
+
`exist as a relation: a child table keyed by the parent, joined back` });
|
|
54906
|
+
});
|
|
54907
|
+
}
|
|
54908
|
+
|
|
54496
54909
|
function containsPath(parent, child) {
|
|
54497
54910
|
const rel = path__namespace.relative(parent, child);
|
|
54498
54911
|
return rel === "" || (!rel.startsWith("..") && !path__namespace.isAbsolute(rel));
|
|
@@ -54623,6 +55036,192 @@ function parseSqlxConfig(body) {
|
|
|
54623
55036
|
description: str("description", body),
|
|
54624
55037
|
};
|
|
54625
55038
|
}
|
|
55039
|
+
const CLASS_META = {
|
|
55040
|
+
qualify: {
|
|
55041
|
+
title: "QUALIFY clause",
|
|
55042
|
+
severity: "blocks-compile",
|
|
55043
|
+
owner: "mechanical",
|
|
55044
|
+
postgres: "wrap the query and filter on the window function in an outer WHERE",
|
|
55045
|
+
why: "PostgreSQL has no QUALIFY; the window result must be computed in a subquery first.",
|
|
55046
|
+
},
|
|
55047
|
+
struct: {
|
|
55048
|
+
title: "STRUCT / ARRAY of STRUCT",
|
|
55049
|
+
severity: "blocks-compile",
|
|
55050
|
+
owner: "needs-decision",
|
|
55051
|
+
postgres: "a child table joined back, jsonb, or eliminate the array entirely",
|
|
55052
|
+
why: "An array of structs IS a one-to-many relationship. A child table is usually the better " +
|
|
55053
|
+
"model, and where the array is only ever UNNESTed straight back into a pivot it can be " +
|
|
55054
|
+
"dropped altogether — but which of the three is right depends on what reads it.",
|
|
55055
|
+
},
|
|
55056
|
+
unnest: {
|
|
55057
|
+
title: "UNNEST",
|
|
55058
|
+
severity: "blocks-compile",
|
|
55059
|
+
owner: "needs-decision",
|
|
55060
|
+
postgres: "cross join unnest(array) — but only for arrays of scalars",
|
|
55061
|
+
why: "PostgreSQL's unnest takes arrays only, so an UNNEST over structs depends on how the STRUCT above is resolved.",
|
|
55062
|
+
},
|
|
55063
|
+
"array-type": {
|
|
55064
|
+
title: "ARRAY<T> type syntax",
|
|
55065
|
+
severity: "blocks-compile",
|
|
55066
|
+
owner: "mechanical",
|
|
55067
|
+
postgres: "T[]",
|
|
55068
|
+
},
|
|
55069
|
+
"parse-date": {
|
|
55070
|
+
title: "PARSE_DATE / PARSE_TIMESTAMP",
|
|
55071
|
+
severity: "fails-at-run",
|
|
55072
|
+
owner: "mechanical",
|
|
55073
|
+
postgres: "to_date / to_timestamp, with the format tokens translated",
|
|
55074
|
+
why: "The format tokens differ (%B %Y → 'Month YYYY'). A wrong token does not fail the " +
|
|
55075
|
+
"statement — it fails on the rows that do not match, so verify against real data.",
|
|
55076
|
+
},
|
|
55077
|
+
"format-date": {
|
|
55078
|
+
title: "FORMAT_DATE / FORMAT_TIMESTAMP",
|
|
55079
|
+
severity: "fails-at-run",
|
|
55080
|
+
owner: "mechanical",
|
|
55081
|
+
postgres: "to_char, with the format tokens translated",
|
|
55082
|
+
},
|
|
55083
|
+
"date-diff-unit": {
|
|
55084
|
+
title: "DATE_DIFF with an unhandled unit",
|
|
55085
|
+
severity: "blocks-compile",
|
|
55086
|
+
owner: "mechanical",
|
|
55087
|
+
postgres: "date subtraction, or EXTRACT(EPOCH FROM …) scaled to the unit",
|
|
55088
|
+
why: "DAY and HOUR are converted automatically; other units need the right arithmetic per unit.",
|
|
55089
|
+
},
|
|
55090
|
+
"pseudo-column": {
|
|
55091
|
+
title: "BigQuery pseudo-column",
|
|
55092
|
+
severity: "blocks-compile",
|
|
55093
|
+
owner: "needs-decision",
|
|
55094
|
+
why: "_TABLE_SUFFIX / _PARTITIONTIME have no PostgreSQL analogue; the model usually needs restructuring.",
|
|
55095
|
+
},
|
|
55096
|
+
"export-data": {
|
|
55097
|
+
title: "EXPORT DATA statement",
|
|
55098
|
+
severity: "blocks-compile",
|
|
55099
|
+
owner: "mechanical",
|
|
55100
|
+
postgres: 'a sqlanvil `type: "export"` action',
|
|
55101
|
+
},
|
|
55102
|
+
scripting: {
|
|
55103
|
+
title: "BigQuery scripting (EXECUTE IMMEDIATE)",
|
|
55104
|
+
severity: "blocks-compile",
|
|
55105
|
+
owner: "needs-decision",
|
|
55106
|
+
postgres: "a PL/pgSQL DO block, or an operations action",
|
|
55107
|
+
why: "Dynamic SQL rarely ports mechanically — the shape of the rewrite depends on what it is generating.",
|
|
55108
|
+
},
|
|
55109
|
+
"generate-uuid": {
|
|
55110
|
+
title: "GENERATE_UUID()",
|
|
55111
|
+
severity: "blocks-compile",
|
|
55112
|
+
owner: "mechanical",
|
|
55113
|
+
postgres: "gen_random_uuid()",
|
|
55114
|
+
},
|
|
55115
|
+
"group-by-all": {
|
|
55116
|
+
title: "GROUP BY ALL",
|
|
55117
|
+
severity: "blocks-compile",
|
|
55118
|
+
owner: "mechanical",
|
|
55119
|
+
postgres: "`sqlanvil migrate-fix` rewrites it as ordinals (group by 1, 2, 5)",
|
|
55120
|
+
why: "Window functions must be excluded as well as aggregates — PostgreSQL rejects them in " +
|
|
55121
|
+
"GROUP BY, and an aggregate followed by OVER is a window function, not an aggregate.",
|
|
55122
|
+
},
|
|
55123
|
+
"star-except": {
|
|
55124
|
+
title: "SELECT * EXCEPT (…)",
|
|
55125
|
+
severity: "blocks-compile",
|
|
55126
|
+
owner: "mechanical",
|
|
55127
|
+
postgres: "`sqlanvil migrate-fix` expands it, once introspect has scaffolded the columns",
|
|
55128
|
+
why: "The column list comes from the source relation — a declaration's columnTypes, or another " +
|
|
55129
|
+
"action's select list resolved recursively. Enumerating also means a column added upstream " +
|
|
55130
|
+
"will no longer appear automatically.",
|
|
55131
|
+
},
|
|
55132
|
+
"not-enforced": {
|
|
55133
|
+
title: "NOT ENFORCED constraint (translated, left commented)",
|
|
55134
|
+
severity: "changes-meaning",
|
|
55135
|
+
owner: "needs-decision",
|
|
55136
|
+
postgres: "the constraint is written out above, commented",
|
|
55137
|
+
why: "BigQuery never checked these. Enforcing them may fail on data that has never been " +
|
|
55138
|
+
"validated; dropping them loses the declared model. Enable per table, deliberately.",
|
|
55139
|
+
},
|
|
55140
|
+
};
|
|
55141
|
+
function analyseStructArrays(report, outDir) {
|
|
55142
|
+
const files = report.files
|
|
55143
|
+
.filter(f => f.file.endsWith(".sqlx"))
|
|
55144
|
+
.map(f => {
|
|
55145
|
+
try {
|
|
55146
|
+
return { file: f.file, source: fs__namespace$1.readFileSync(path__namespace.join(outDir, f.file), "utf8") };
|
|
55147
|
+
}
|
|
55148
|
+
catch (_a) {
|
|
55149
|
+
return null;
|
|
55150
|
+
}
|
|
55151
|
+
})
|
|
55152
|
+
.filter((f) => f !== null);
|
|
55153
|
+
const raw = files.flatMap(f => findStructArrays(f.file, f.source));
|
|
55154
|
+
if (!raw.length)
|
|
55155
|
+
return;
|
|
55156
|
+
const sites = chooseStrategies(raw, files);
|
|
55157
|
+
report.todo = report.todo.filter(t => t.id !== "struct");
|
|
55158
|
+
const meta = CLASS_META.struct;
|
|
55159
|
+
report.todo.push({
|
|
55160
|
+
id: "struct",
|
|
55161
|
+
title: meta.title,
|
|
55162
|
+
count: sites.length,
|
|
55163
|
+
severity: meta.severity,
|
|
55164
|
+
owner: meta.owner,
|
|
55165
|
+
postgres: "per site below — collapse, a child table, or jsonb",
|
|
55166
|
+
why: meta.why,
|
|
55167
|
+
sql: sites.map(s => ({
|
|
55168
|
+
file: s.file,
|
|
55169
|
+
line: s.line,
|
|
55170
|
+
note: `${s.strategy} — ${s.rationale}`,
|
|
55171
|
+
sql: suggestedSql(s),
|
|
55172
|
+
})),
|
|
55173
|
+
locations: sites.map(s => ({ file: s.file, lines: [s.line] })),
|
|
55174
|
+
});
|
|
55175
|
+
}
|
|
55176
|
+
function buildTriage(report) {
|
|
55177
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
55178
|
+
const todo = new Map();
|
|
55179
|
+
const applied = new Map();
|
|
55180
|
+
for (const file of report.files) {
|
|
55181
|
+
for (const f of (_a = file.findings) !== null && _a !== void 0 ? _a : []) {
|
|
55182
|
+
const id = (_b = f.class) !== null && _b !== void 0 ? _b : classIdFor(f.note);
|
|
55183
|
+
const meta = CLASS_META[id];
|
|
55184
|
+
if (f.kind === "rewrite") {
|
|
55185
|
+
const seen = (_c = applied.get(id)) !== null && _c !== void 0 ? _c : { id, title: (_d = meta === null || meta === void 0 ? void 0 : meta.title) !== null && _d !== void 0 ? _d : f.note, count: 0 };
|
|
55186
|
+
seen.count++;
|
|
55187
|
+
applied.set(id, seen);
|
|
55188
|
+
if (!meta || meta.owner !== "needs-decision")
|
|
55189
|
+
continue;
|
|
55190
|
+
}
|
|
55191
|
+
const entry = (_e = todo.get(id)) !== null && _e !== void 0 ? _e : {
|
|
55192
|
+
id,
|
|
55193
|
+
title: (_f = meta === null || meta === void 0 ? void 0 : meta.title) !== null && _f !== void 0 ? _f : f.note,
|
|
55194
|
+
count: 0,
|
|
55195
|
+
severity: (_g = meta === null || meta === void 0 ? void 0 : meta.severity) !== null && _g !== void 0 ? _g : "blocks-compile",
|
|
55196
|
+
owner: (_h = meta === null || meta === void 0 ? void 0 : meta.owner) !== null && _h !== void 0 ? _h : "mechanical",
|
|
55197
|
+
postgres: meta === null || meta === void 0 ? void 0 : meta.postgres,
|
|
55198
|
+
why: meta === null || meta === void 0 ? void 0 : meta.why,
|
|
55199
|
+
locations: [],
|
|
55200
|
+
};
|
|
55201
|
+
entry.count++;
|
|
55202
|
+
const loc = entry.locations.find(l => l.file === file.file);
|
|
55203
|
+
if (loc)
|
|
55204
|
+
loc.lines.push(f.line);
|
|
55205
|
+
else
|
|
55206
|
+
entry.locations.push({ file: file.file, lines: [f.line] });
|
|
55207
|
+
todo.set(id, entry);
|
|
55208
|
+
}
|
|
55209
|
+
}
|
|
55210
|
+
const rank = {
|
|
55211
|
+
"changes-meaning": 0,
|
|
55212
|
+
"fails-at-run": 1,
|
|
55213
|
+
"blocks-compile": 2,
|
|
55214
|
+
};
|
|
55215
|
+
report.todo = [...todo.values()].sort((a, b) => rank[a.severity] - rank[b.severity] || b.count - a.count);
|
|
55216
|
+
report.applied = [...applied.values()].sort((a, b) => b.count - a.count);
|
|
55217
|
+
}
|
|
55218
|
+
function classIdFor(note) {
|
|
55219
|
+
return note
|
|
55220
|
+
.slice(0, 40)
|
|
55221
|
+
.toLowerCase()
|
|
55222
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
55223
|
+
.replace(/^-|-$/g, "");
|
|
55224
|
+
}
|
|
54626
55225
|
const GLOBAL_RULES = [
|
|
54627
55226
|
{
|
|
54628
55227
|
pattern: /\bdataform\.projectConfig\b/g,
|
|
@@ -54632,12 +55231,6 @@ const GLOBAL_RULES = [
|
|
|
54632
55231
|
},
|
|
54633
55232
|
];
|
|
54634
55233
|
const SQL_RULES = [
|
|
54635
|
-
{
|
|
54636
|
-
pattern: /\bSAFE_CAST\s*\(/g,
|
|
54637
|
-
replacement: "CAST(",
|
|
54638
|
-
kind: "rewrite",
|
|
54639
|
-
note: "SAFE_CAST → CAST — Postgres CAST raises on bad input; verify the data or guard it",
|
|
54640
|
-
},
|
|
54641
55234
|
{
|
|
54642
55235
|
pattern: /\bCURRENT_DATE\s*\(\s*\)/g,
|
|
54643
55236
|
replacement: "CURRENT_DATE",
|
|
@@ -54650,32 +55243,50 @@ const SQL_RULES = [
|
|
|
54650
55243
|
note: "CURRENT_DATE('<tz>') — Postgres: (CURRENT_TIMESTAMP AT TIME ZONE '<tz>')::date",
|
|
54651
55244
|
},
|
|
54652
55245
|
{
|
|
54653
|
-
pattern:
|
|
55246
|
+
pattern: /\*\s*except\s*\(/gi,
|
|
54654
55247
|
kind: "flag",
|
|
54655
|
-
note: "
|
|
55248
|
+
note: "SELECT * EXCEPT — run `sqlanvil migrate-fix` after introspect to expand it",
|
|
55249
|
+
id: "star-except",
|
|
54656
55250
|
},
|
|
54657
55251
|
{
|
|
54658
|
-
pattern: /\
|
|
55252
|
+
pattern: /\bGROUP\s+BY\s+ALL\b/gi,
|
|
54659
55253
|
kind: "flag",
|
|
54660
|
-
note: "
|
|
55254
|
+
note: "GROUP BY ALL — run `sqlanvil migrate-fix` after introspect to rewrite it",
|
|
55255
|
+
id: "group-by-all",
|
|
54661
55256
|
},
|
|
54662
|
-
{ pattern: /\bQUALIFY\b/gi, kind: "flag", note: "QUALIFY — rewrite as a subquery/CTE filter on the window function"
|
|
54663
|
-
|
|
54664
|
-
{ pattern: /\
|
|
54665
|
-
|
|
54666
|
-
{ pattern: /\
|
|
54667
|
-
|
|
54668
|
-
{ pattern: /\
|
|
55257
|
+
{ pattern: /\bQUALIFY\b/gi, kind: "flag", note: "QUALIFY — rewrite as a subquery/CTE filter on the window function",
|
|
55258
|
+
id: "qualify", },
|
|
55259
|
+
{ pattern: /\bUNNEST\s*\(/g, kind: "flag", note: "UNNEST — Postgres unnest() works on arrays only (no STRUCT arrays); verify semantics",
|
|
55260
|
+
id: "unnest", },
|
|
55261
|
+
{ pattern: /\bSTRUCT\s*[(<]/g, kind: "flag", note: "STRUCT — no Postgres equivalent; consider jsonb or a composite type",
|
|
55262
|
+
id: "struct", },
|
|
55263
|
+
{ pattern: /\bARRAY\s*</g, kind: "flag", note: "ARRAY<T> type syntax — Postgres uses T[]",
|
|
55264
|
+
id: "array-type", },
|
|
55265
|
+
{ pattern: /\bFORMAT_DATE\s*\(/g, kind: "flag", note: "FORMAT_DATE → TO_CHAR (different format tokens)",
|
|
55266
|
+
id: "format-date", },
|
|
55267
|
+
{ pattern: /\bPARSE_DATE\s*\(/g, kind: "flag", note: "PARSE_DATE → TO_DATE (different format tokens)",
|
|
55268
|
+
id: "parse-date", },
|
|
55269
|
+
{ pattern: /\bFORMAT_TIMESTAMP\s*\(/g, kind: "flag", note: "FORMAT_TIMESTAMP → TO_CHAR (different format tokens)",
|
|
55270
|
+
id: "format-date", },
|
|
54669
55271
|
{
|
|
54670
55272
|
pattern: /\b(DATE_ADD|DATE_SUB|TIMESTAMP_ADD|TIMESTAMP_SUB|DATETIME_ADD|DATETIME_SUB)\s*\(/g,
|
|
54671
55273
|
kind: "flag",
|
|
54672
55274
|
note: "BigQuery date arithmetic — Postgres uses interval arithmetic (x + INTERVAL '1 day')",
|
|
54673
55275
|
},
|
|
54674
|
-
{ pattern: /\bDATE_DIFF\s*\(/g, kind: "flag", note: "DATE_DIFF — Postgres: subtraction / EXTRACT(EPOCH FROM ...) depending on the part"
|
|
54675
|
-
|
|
54676
|
-
{ pattern:
|
|
54677
|
-
|
|
54678
|
-
{
|
|
55276
|
+
{ pattern: /\bDATE_DIFF\s*\(/g, kind: "flag", note: "DATE_DIFF — Postgres: subtraction / EXTRACT(EPOCH FROM ...) depending on the part",
|
|
55277
|
+
id: "date-diff-unit", },
|
|
55278
|
+
{ pattern: /_PARTITIONTIME|_PARTITIONDATE|_TABLE_SUFFIX/g, kind: "flag", note: "BigQuery pseudo-column has no Postgres equivalent",
|
|
55279
|
+
id: "pseudo-column", },
|
|
55280
|
+
{
|
|
55281
|
+
pattern: /\bEXPORT\s+DATA\b/gi,
|
|
55282
|
+
kind: "flag",
|
|
55283
|
+
note: 'EXPORT DATA — use a sqlanvil `type: "export"` action instead',
|
|
55284
|
+
id: "export-data",
|
|
55285
|
+
},
|
|
55286
|
+
{ pattern: /\bEXECUTE\s+IMMEDIATE\b/gi, kind: "flag", note: "BigQuery scripting (EXECUTE IMMEDIATE) — rewrite as a PL/pgSQL DO block or operations",
|
|
55287
|
+
id: "scripting", },
|
|
55288
|
+
{ pattern: /\bGENERATE_UUID\s*\(\s*\)/g, kind: "flag", note: "GENERATE_UUID() → gen_random_uuid()",
|
|
55289
|
+
id: "generate-uuid", },
|
|
54679
55290
|
{
|
|
54680
55291
|
pattern: /`[A-Za-z0-9_-]+\.[A-Za-z0-9_$]+\.[A-Za-z0-9_$]+`/g,
|
|
54681
55292
|
kind: "flag",
|
|
@@ -54712,10 +55323,394 @@ ${description} columnTypes: {
|
|
|
54712
55323
|
],
|
|
54713
55324
|
};
|
|
54714
55325
|
}
|
|
55326
|
+
function maskSql(text, keepStrings = false) {
|
|
55327
|
+
let out = text.replace(/--[^\n]*/g, m => " ".repeat(m.length));
|
|
55328
|
+
out = out.replace(/\/\*[\s\S]*?\*\//g, m => " ".repeat(m.length));
|
|
55329
|
+
if (!keepStrings) {
|
|
55330
|
+
out = out.replace(/'(?:[^']|'')*'/g, m => `'${" ".repeat(Math.max(0, m.length - 2))}'`);
|
|
55331
|
+
}
|
|
55332
|
+
return out;
|
|
55333
|
+
}
|
|
55334
|
+
function matchParen(text, open) {
|
|
55335
|
+
let depth = 0;
|
|
55336
|
+
for (let i = open; i < text.length; i++) {
|
|
55337
|
+
if (text[i] === "(")
|
|
55338
|
+
depth++;
|
|
55339
|
+
else if (text[i] === ")" && --depth === 0)
|
|
55340
|
+
return i;
|
|
55341
|
+
}
|
|
55342
|
+
return -1;
|
|
55343
|
+
}
|
|
55344
|
+
function splitArgs(src, mask, from, to) {
|
|
55345
|
+
const parts = [];
|
|
55346
|
+
let depth = 0;
|
|
55347
|
+
let start = from;
|
|
55348
|
+
for (let i = from; i < to; i++) {
|
|
55349
|
+
const ch = mask[i];
|
|
55350
|
+
if (ch === "(")
|
|
55351
|
+
depth++;
|
|
55352
|
+
else if (ch === ")")
|
|
55353
|
+
depth--;
|
|
55354
|
+
else if (ch === "," && depth === 0) {
|
|
55355
|
+
parts.push(src.slice(start, i).trim());
|
|
55356
|
+
start = i + 1;
|
|
55357
|
+
}
|
|
55358
|
+
}
|
|
55359
|
+
parts.push(src.slice(start, to).trim());
|
|
55360
|
+
return parts;
|
|
55361
|
+
}
|
|
55362
|
+
const INTERVAL_ARG = /^interval\s+(\d+)\s+([a-z]+)$/i;
|
|
55363
|
+
const TYPE_NAMES = {
|
|
55364
|
+
string: "text",
|
|
55365
|
+
int64: "bigint",
|
|
55366
|
+
float64: "double precision",
|
|
55367
|
+
numeric: "numeric",
|
|
55368
|
+
bignumeric: "numeric",
|
|
55369
|
+
bool: "boolean",
|
|
55370
|
+
bytes: "bytea",
|
|
55371
|
+
datetime: "timestamp",
|
|
55372
|
+
timestamp: "timestamptz",
|
|
55373
|
+
};
|
|
55374
|
+
function pgType(bqType) {
|
|
55375
|
+
var _a;
|
|
55376
|
+
const t = bqType.trim().toLowerCase();
|
|
55377
|
+
return (_a = TYPE_NAMES[t]) !== null && _a !== void 0 ? _a : t;
|
|
55378
|
+
}
|
|
55379
|
+
const CALL_RULES = [
|
|
55380
|
+
{
|
|
55381
|
+
name: "cast",
|
|
55382
|
+
note: "CAST(x AS <BigQuery type>) → the PostgreSQL type name",
|
|
55383
|
+
rewrite(args) {
|
|
55384
|
+
if (args.length !== 1)
|
|
55385
|
+
return null;
|
|
55386
|
+
const m = /^([\s\S]*?)\s+as\s+([A-Za-z][\w ]*)$/i.exec(args[0]);
|
|
55387
|
+
if (!m)
|
|
55388
|
+
return null;
|
|
55389
|
+
const t = pgType(m[2]);
|
|
55390
|
+
return t === m[2].trim().toLowerCase() ? null : `cast(${m[1]} as ${t})`;
|
|
55391
|
+
},
|
|
55392
|
+
},
|
|
55393
|
+
{
|
|
55394
|
+
name: "safe_cast",
|
|
55395
|
+
note: "SAFE_CAST → guarded cast via pg_input_is_valid (NULL on bad input, as in BigQuery)",
|
|
55396
|
+
rewrite(args) {
|
|
55397
|
+
if (args.length !== 1)
|
|
55398
|
+
return null;
|
|
55399
|
+
const m = /^([\s\S]*?)\s+as\s+([A-Za-z][\w ]*)$/i.exec(args[0]);
|
|
55400
|
+
if (!m)
|
|
55401
|
+
return null;
|
|
55402
|
+
const [, expr, type] = m;
|
|
55403
|
+
const t = pgType(type);
|
|
55404
|
+
if (t === "text")
|
|
55405
|
+
return `(${expr})::text`;
|
|
55406
|
+
return `(case when pg_input_is_valid((${expr})::text, '${t}') then (${expr})::${t} end)`;
|
|
55407
|
+
},
|
|
55408
|
+
},
|
|
55409
|
+
{
|
|
55410
|
+
name: "safe_divide",
|
|
55411
|
+
note: "SAFE_DIVIDE(a, b) → a / nullif(b, 0)",
|
|
55412
|
+
rewrite: args => (args.length === 2 ? `(${args[0]} / nullif(${args[1]}, 0))` : null),
|
|
55413
|
+
},
|
|
55414
|
+
{
|
|
55415
|
+
name: "collate",
|
|
55416
|
+
note: "COLLATE(x, '') → x (collation stripping is a no-op in PostgreSQL)",
|
|
55417
|
+
rewrite: args => args.length === 2 && (args[1] === "''" || args[1] === '""') ? args[0] : null,
|
|
55418
|
+
},
|
|
55419
|
+
{
|
|
55420
|
+
name: "regexp_extract",
|
|
55421
|
+
note: "REGEXP_EXTRACT(x, pattern) → substring(x from pattern)",
|
|
55422
|
+
rewrite: args => (args.length === 2 ? `substring(${args[0]} from ${args[1]})` : null),
|
|
55423
|
+
},
|
|
55424
|
+
{
|
|
55425
|
+
name: "split",
|
|
55426
|
+
note: "SPLIT(x, d) → string_to_array(x, d); an empty delimiter becomes NULL (per character)",
|
|
55427
|
+
rewrite(args) {
|
|
55428
|
+
if (args.length !== 2)
|
|
55429
|
+
return null;
|
|
55430
|
+
const d = args[1] === "''" || args[1] === '""' ? "NULL" : args[1];
|
|
55431
|
+
return `string_to_array(${args[0]}, ${d})`;
|
|
55432
|
+
},
|
|
55433
|
+
},
|
|
55434
|
+
{
|
|
55435
|
+
name: "date_sub",
|
|
55436
|
+
note: "DATE_SUB(d, INTERVAL n unit) → (d - interval 'n unit')",
|
|
55437
|
+
rewrite(args) {
|
|
55438
|
+
if (args.length !== 2)
|
|
55439
|
+
return null;
|
|
55440
|
+
const m = INTERVAL_ARG.exec(args[1]);
|
|
55441
|
+
return m ? `(${args[0]} - interval '${m[1]} ${m[2].toLowerCase()}')` : null;
|
|
55442
|
+
},
|
|
55443
|
+
},
|
|
55444
|
+
{
|
|
55445
|
+
name: "date_add",
|
|
55446
|
+
note: "DATE_ADD(d, INTERVAL n unit) → (d + interval 'n unit')",
|
|
55447
|
+
rewrite(args) {
|
|
55448
|
+
if (args.length !== 2)
|
|
55449
|
+
return null;
|
|
55450
|
+
const m = INTERVAL_ARG.exec(args[1]);
|
|
55451
|
+
return m ? `(${args[0]} + interval '${m[1]} ${m[2].toLowerCase()}')` : null;
|
|
55452
|
+
},
|
|
55453
|
+
},
|
|
55454
|
+
{
|
|
55455
|
+
name: "date_diff",
|
|
55456
|
+
note: "DATE_DIFF(a, b, DAY|HOUR) → PostgreSQL date/epoch arithmetic",
|
|
55457
|
+
rewrite(args) {
|
|
55458
|
+
if (args.length !== 3)
|
|
55459
|
+
return null;
|
|
55460
|
+
const unit = args[2].trim().toLowerCase();
|
|
55461
|
+
if (unit === "day")
|
|
55462
|
+
return `(${args[0]}::date - ${args[1]}::date)`;
|
|
55463
|
+
if (unit === "hour") {
|
|
55464
|
+
return (`(extract(epoch from date_trunc('hour', ${args[0]})` +
|
|
55465
|
+
` - date_trunc('hour', ${args[1]})) / 3600)::bigint`);
|
|
55466
|
+
}
|
|
55467
|
+
return null;
|
|
55468
|
+
},
|
|
55469
|
+
},
|
|
55470
|
+
{
|
|
55471
|
+
name: "unix_seconds",
|
|
55472
|
+
note: "UNIX_SECONDS(ts) → extract(epoch from ts)::bigint",
|
|
55473
|
+
rewrite: args => (args.length === 1 ? `extract(epoch from ${args[0]})::bigint` : null),
|
|
55474
|
+
},
|
|
55475
|
+
];
|
|
55476
|
+
function applyCallRules(text, isRewritableOffset) {
|
|
55477
|
+
const applied = [];
|
|
55478
|
+
let result = text;
|
|
55479
|
+
for (let pass = 0; pass < 10; pass++) {
|
|
55480
|
+
const mask = maskSql(result);
|
|
55481
|
+
const sites = [];
|
|
55482
|
+
for (const rule of CALL_RULES) {
|
|
55483
|
+
const finder = new RegExp(`\\b${rule.name}\\s*\\(`, "gi");
|
|
55484
|
+
let m;
|
|
55485
|
+
while ((m = finder.exec(mask)) !== null) {
|
|
55486
|
+
const open = m.index + m[0].length - 1;
|
|
55487
|
+
const close = matchParen(mask, open);
|
|
55488
|
+
if (close < 0 || !isRewritableOffset(m.index))
|
|
55489
|
+
continue;
|
|
55490
|
+
const out = rule.rewrite(splitArgs(result, mask, open + 1, close));
|
|
55491
|
+
if (out === null)
|
|
55492
|
+
continue;
|
|
55493
|
+
sites.push({ start: m.index, end: close + 1, out, note: rule.note });
|
|
55494
|
+
}
|
|
55495
|
+
}
|
|
55496
|
+
if (!sites.length)
|
|
55497
|
+
break;
|
|
55498
|
+
sites.sort((a, b) => b.start - a.start);
|
|
55499
|
+
const seen = [];
|
|
55500
|
+
for (const s of sites) {
|
|
55501
|
+
if (seen.some(p => s.start < p.end && p.start < s.end))
|
|
55502
|
+
continue;
|
|
55503
|
+
seen.push(s);
|
|
55504
|
+
applied.push({ line: result.slice(0, s.start).split("\n").length, note: s.note });
|
|
55505
|
+
result = result.slice(0, s.start) + s.out + result.slice(s.end);
|
|
55506
|
+
}
|
|
55507
|
+
}
|
|
55508
|
+
const before = result;
|
|
55509
|
+
result = result.replace(/\[\s*(?:SAFE_)?OFFSET\s*\(\s*([^()]*?)\s*\)\s*\]/gi, (_all, idx) => (/^\d+$/.test(idx) ? `[${Number(idx) + 1}]` : `[(${idx}) + 1]`));
|
|
55510
|
+
if (result !== before) {
|
|
55511
|
+
applied.push({ line: 1, note: "array[OFFSET(n)] → array[n + 1] (PostgreSQL is 1-based)" });
|
|
55512
|
+
}
|
|
55513
|
+
result = parenthesiseSubscriptedCalls(result);
|
|
55514
|
+
return { text: result, applied };
|
|
55515
|
+
}
|
|
55516
|
+
function jsBlockSpans(source) {
|
|
55517
|
+
const spans = [];
|
|
55518
|
+
const re = /(^|\n)\s*js\s*\{/g;
|
|
55519
|
+
let m;
|
|
55520
|
+
while ((m = re.exec(source))) {
|
|
55521
|
+
const open = source.indexOf("{", m.index + m[0].length - 1);
|
|
55522
|
+
const close = matchParenLike(source, open, "{", "}");
|
|
55523
|
+
spans.push({ start: m.index, end: close < 0 ? source.length : close + 1 });
|
|
55524
|
+
}
|
|
55525
|
+
return spans;
|
|
55526
|
+
}
|
|
55527
|
+
function matchParenLike(text, open, o, c) {
|
|
55528
|
+
let depth = 0;
|
|
55529
|
+
for (let i = open; i < text.length; i++) {
|
|
55530
|
+
if (text[i] === o)
|
|
55531
|
+
depth++;
|
|
55532
|
+
else if (text[i] === c && --depth === 0)
|
|
55533
|
+
return i;
|
|
55534
|
+
}
|
|
55535
|
+
return -1;
|
|
55536
|
+
}
|
|
55537
|
+
function applyTextRules(text, isSqlOffset, applied) {
|
|
55538
|
+
const note = (offset, n) => applied.push({ line: text.slice(0, offset).split("\n").length, note: n });
|
|
55539
|
+
const rewrite = (src, mask, pattern, replace, ruleNote) => {
|
|
55540
|
+
const edits = [];
|
|
55541
|
+
let m;
|
|
55542
|
+
pattern.lastIndex = 0;
|
|
55543
|
+
while ((m = pattern.exec(mask)) !== null) {
|
|
55544
|
+
if (!isSqlOffset(m.index))
|
|
55545
|
+
continue;
|
|
55546
|
+
const sub = replace(m);
|
|
55547
|
+
if (sub !== null)
|
|
55548
|
+
edits.push({ start: m.index, end: m.index + m[0].length, out: sub });
|
|
55549
|
+
}
|
|
55550
|
+
let out = src;
|
|
55551
|
+
for (const e of edits.reverse()) {
|
|
55552
|
+
note(e.start, ruleNote);
|
|
55553
|
+
out = out.slice(0, e.start) + e.out + out.slice(e.end);
|
|
55554
|
+
}
|
|
55555
|
+
return out;
|
|
55556
|
+
};
|
|
55557
|
+
let result = text;
|
|
55558
|
+
result = rewrite(result, maskSql(result, true), /(?<![\w"'])#[^\n]*/g, m => `--${m[0].slice(1)}`, "# line comment → -- (PostgreSQL has no # comment)");
|
|
55559
|
+
result = rewrite(result, maskSql(result, true), /(?<![A-Za-z0-9_])[rR](?=['"])/g, () => "", "r'…' raw string → '…' (no raw-string prefix in PostgreSQL)");
|
|
55560
|
+
result = rewrite(result, maskSql(result, true), /(\bas\s+)?"([^"\n]*)"/gi, m => {
|
|
55561
|
+
if (m[1])
|
|
55562
|
+
return null;
|
|
55563
|
+
return m[2].includes("${") || m[2].includes("'") ? null : `'${m[2]}'`;
|
|
55564
|
+
}, 'BigQuery "string" → \'string\' (double quotes are identifiers in PostgreSQL)');
|
|
55565
|
+
result = rewrite(result, maskSql(result), /`([^`\n]+)`/g, m => (m[1].includes(".") || m[1].includes("${") ? null : `"${m[1]}"`), "`identifier` → \"identifier\"");
|
|
55566
|
+
result = rewrite(result, maskSql(result, true), /\s+collate\s+(''|"")/gi, () => "", "x COLLATE '' → x (collation stripping is a no-op in PostgreSQL)");
|
|
55567
|
+
for (;;) {
|
|
55568
|
+
const mask = maskSql(result);
|
|
55569
|
+
const m = /\bextract\s*\(\s*dayofweek\s+from\s+/i.exec(mask);
|
|
55570
|
+
if (!m || !isSqlOffset(m.index))
|
|
55571
|
+
break;
|
|
55572
|
+
const close = matchParen(mask, mask.indexOf("(", m.index));
|
|
55573
|
+
if (close < 0)
|
|
55574
|
+
break;
|
|
55575
|
+
const inner = result.slice(m.index + m[0].length, close).trim();
|
|
55576
|
+
note(m.index, "EXTRACT(DAYOFWEEK FROM x) → extract(dow from x) + 1 (BigQuery is 1-based)");
|
|
55577
|
+
result =
|
|
55578
|
+
result.slice(0, m.index) + `(extract(dow from ${inner}) + 1)` + result.slice(close + 1);
|
|
55579
|
+
}
|
|
55580
|
+
return rewriteUnenforcedConstraints(result, isSqlOffset, applied);
|
|
55581
|
+
}
|
|
55582
|
+
const UNENFORCED_HEADER = [
|
|
55583
|
+
"BigQuery declared the constraint below NOT ENFORCED — advisory metadata for BI tools,",
|
|
55584
|
+
"never checked by the warehouse. PostgreSQL has no unenforced constraint, so the real",
|
|
55585
|
+
"equivalent is written out here and left commented: uncommenting enforces it, and it will",
|
|
55586
|
+
"fail if the data violates it. A foreign key also needs the referenced table's own key to",
|
|
55587
|
+
"exist, so enable those upstream first.",
|
|
55588
|
+
];
|
|
55589
|
+
function rewriteUnenforcedConstraints(text, isSqlOffset, applied) {
|
|
55590
|
+
const mask = maskSql(text);
|
|
55591
|
+
const sites = [];
|
|
55592
|
+
const finder = /\bALTER\s+TABLE\b/gi;
|
|
55593
|
+
let m;
|
|
55594
|
+
while ((m = finder.exec(mask)) !== null) {
|
|
55595
|
+
let depth = 0;
|
|
55596
|
+
let end = mask.length;
|
|
55597
|
+
for (let i = m.index; i < mask.length; i++) {
|
|
55598
|
+
if (mask[i] === "(")
|
|
55599
|
+
depth++;
|
|
55600
|
+
else if (mask[i] === ")")
|
|
55601
|
+
depth--;
|
|
55602
|
+
else if (mask[i] === ";" && depth === 0) {
|
|
55603
|
+
end = i + 1;
|
|
55604
|
+
break;
|
|
55605
|
+
}
|
|
55606
|
+
}
|
|
55607
|
+
finder.lastIndex = end;
|
|
55608
|
+
const stmt = text.slice(m.index, end);
|
|
55609
|
+
if (!/NOT\s+ENFORCED/i.test(stmt) || !isSqlOffset(m.index))
|
|
55610
|
+
continue;
|
|
55611
|
+
const body = stmt.replace(/;\s*$/, "").replace(/^ALTER\s+TABLE\s*/i, "");
|
|
55612
|
+
const firstAction = /\b(ADD|DROP|ALTER)\b/i.exec(body);
|
|
55613
|
+
if (!firstAction)
|
|
55614
|
+
continue;
|
|
55615
|
+
const target = body.slice(0, firstAction.index).trim();
|
|
55616
|
+
const actions = splitTopLevel(body.slice(firstAction.index))
|
|
55617
|
+
.map(a => a.trim())
|
|
55618
|
+
.filter(a => /^ADD\b/i.test(a))
|
|
55619
|
+
.map(a => a.replace(/\s*\bNOT\s+ENFORCED\b/gi, "").trim());
|
|
55620
|
+
if (!actions.length)
|
|
55621
|
+
continue;
|
|
55622
|
+
const lineStart = text.lastIndexOf("\n", m.index) + 1;
|
|
55623
|
+
const indent = /^[ \t]*/.exec(text.slice(lineStart))[0];
|
|
55624
|
+
const commented = [
|
|
55625
|
+
...UNENFORCED_HEADER.map(l => `-- ${l}`),
|
|
55626
|
+
...(actions.length === 1
|
|
55627
|
+
? [`-- ALTER TABLE ${target} ${actions[0]};`]
|
|
55628
|
+
: [
|
|
55629
|
+
`-- ALTER TABLE ${target}`,
|
|
55630
|
+
...actions.map((a, i) => `-- ${a}${i === actions.length - 1 ? ";" : ","}`),
|
|
55631
|
+
]),
|
|
55632
|
+
].join(`\n${indent}`);
|
|
55633
|
+
sites.push({ start: m.index, end, out: commented });
|
|
55634
|
+
}
|
|
55635
|
+
let result = text;
|
|
55636
|
+
for (const s of sites.reverse()) {
|
|
55637
|
+
applied.push({
|
|
55638
|
+
line: text.slice(0, s.start).split("\n").length,
|
|
55639
|
+
note: "NOT ENFORCED constraint → PostgreSQL equivalent, commented out " +
|
|
55640
|
+
"(enforcing it may fail on data that has never been checked)",
|
|
55641
|
+
});
|
|
55642
|
+
result = result.slice(0, s.start) + s.out + result.slice(s.end);
|
|
55643
|
+
}
|
|
55644
|
+
return result;
|
|
55645
|
+
}
|
|
55646
|
+
function splitTopLevel(s) {
|
|
55647
|
+
const parts = [];
|
|
55648
|
+
let depth = 0;
|
|
55649
|
+
let cur = "";
|
|
55650
|
+
for (const ch of s) {
|
|
55651
|
+
if (ch === "(")
|
|
55652
|
+
depth++;
|
|
55653
|
+
else if (ch === ")")
|
|
55654
|
+
depth--;
|
|
55655
|
+
if (ch === "," && depth === 0) {
|
|
55656
|
+
parts.push(cur);
|
|
55657
|
+
cur = "";
|
|
55658
|
+
}
|
|
55659
|
+
else
|
|
55660
|
+
cur += ch;
|
|
55661
|
+
}
|
|
55662
|
+
parts.push(cur);
|
|
55663
|
+
return parts;
|
|
55664
|
+
}
|
|
55665
|
+
function parenthesiseSubscriptedCalls(text) {
|
|
55666
|
+
const finder = /\b[A-Za-z_]\w*\s*\(/g;
|
|
55667
|
+
const spans = [];
|
|
55668
|
+
let m;
|
|
55669
|
+
while ((m = finder.exec(text)) !== null) {
|
|
55670
|
+
const close = matchParen(text, m.index + m[0].length - 1);
|
|
55671
|
+
if (close < 0)
|
|
55672
|
+
continue;
|
|
55673
|
+
if (/^\s*\[/.test(text.slice(close + 1)))
|
|
55674
|
+
spans.push({ start: m.index, end: close + 1 });
|
|
55675
|
+
}
|
|
55676
|
+
let out = text;
|
|
55677
|
+
for (const s of spans.reverse()) {
|
|
55678
|
+
out = `${out.slice(0, s.start)}(${out.slice(s.start, s.end)})${out.slice(s.end)}`;
|
|
55679
|
+
}
|
|
55680
|
+
return out;
|
|
55681
|
+
}
|
|
54715
55682
|
function convertTarget(source, jsMode, dialect = "postgres") {
|
|
55683
|
+
var _a;
|
|
54716
55684
|
const dialectPass = dialect === "postgres";
|
|
54717
|
-
|
|
55685
|
+
let span = jsMode ? null : findConfigBlock(source);
|
|
54718
55686
|
const findings = [];
|
|
55687
|
+
if (dialectPass) {
|
|
55688
|
+
const outsideConfig = (off) => {
|
|
55689
|
+
const s = jsMode ? null : findConfigBlock(source);
|
|
55690
|
+
return !s || off < s.start || off >= s.end;
|
|
55691
|
+
};
|
|
55692
|
+
const called = applyCallRules(source, outsideConfig);
|
|
55693
|
+
if (called.text !== source)
|
|
55694
|
+
source = called.text;
|
|
55695
|
+
for (const a of called.applied) {
|
|
55696
|
+
findings.push({ line: a.line, kind: "rewrite", note: a.note, class: classIdFor(a.note) });
|
|
55697
|
+
}
|
|
55698
|
+
const textApplied = [];
|
|
55699
|
+
const jsSpans = jsMode ? [{ start: 0, end: source.length }] : jsBlockSpans(source);
|
|
55700
|
+
const templates = tokenize(source).filter(t => t.kind === "template");
|
|
55701
|
+
source = applyTextRules(source, off => outsideConfig(off) &&
|
|
55702
|
+
!jsSpans.some(s => off >= s.start && off < s.end) &&
|
|
55703
|
+
!templates.some(t => off >= t.start && off < t.end), textApplied);
|
|
55704
|
+
for (const a of textApplied) {
|
|
55705
|
+
findings.push({
|
|
55706
|
+
line: a.line,
|
|
55707
|
+
kind: "rewrite",
|
|
55708
|
+
note: a.note,
|
|
55709
|
+
class: a.note.startsWith("NOT ENFORCED") ? "not-enforced" : classIdFor(a.note),
|
|
55710
|
+
});
|
|
55711
|
+
}
|
|
55712
|
+
span = jsMode ? null : findConfigBlock(source);
|
|
55713
|
+
}
|
|
54719
55714
|
const lines = source.split("\n");
|
|
54720
55715
|
const jsLines = jsMode ? new Set() : jsBlockLines(source);
|
|
54721
55716
|
let configStartLine = -1;
|
|
@@ -54780,15 +55775,16 @@ function convertTarget(source, jsMode, dialect = "postgres") {
|
|
|
54780
55775
|
rule.pattern.lastIndex = 0;
|
|
54781
55776
|
if (!rule.pattern.test(line))
|
|
54782
55777
|
continue;
|
|
55778
|
+
const cls = (_a = rule.id) !== null && _a !== void 0 ? _a : classIdFor(rule.note);
|
|
54783
55779
|
if (rule.replacement !== undefined) {
|
|
54784
55780
|
rule.pattern.lastIndex = 0;
|
|
54785
55781
|
line = line.replace(rule.pattern, rule.replacement);
|
|
54786
|
-
findings.push({ line: lineNo, kind: "rewrite", note: rule.note });
|
|
55782
|
+
findings.push({ line: lineNo, kind: "rewrite", note: rule.note, class: cls });
|
|
54787
55783
|
if (markersSafe)
|
|
54788
55784
|
notesForLine.push(rule.note);
|
|
54789
55785
|
}
|
|
54790
55786
|
else {
|
|
54791
|
-
findings.push({ line: lineNo, kind: "flag", note: rule.note });
|
|
55787
|
+
findings.push({ line: lineNo, kind: "flag", note: rule.note, class: cls });
|
|
54792
55788
|
if (markersSafe)
|
|
54793
55789
|
notesForLine.push(rule.note);
|
|
54794
55790
|
}
|
|
@@ -54864,6 +55860,8 @@ async function migrateDataform(opts) {
|
|
|
54864
55860
|
files: [],
|
|
54865
55861
|
skippedForSafety: [],
|
|
54866
55862
|
warnings: [],
|
|
55863
|
+
todo: [],
|
|
55864
|
+
applied: [],
|
|
54867
55865
|
};
|
|
54868
55866
|
const sqlxFiles = [];
|
|
54869
55867
|
for (const rel of files) {
|
|
@@ -55095,6 +56093,8 @@ async function migrateDataform(opts) {
|
|
|
55095
56093
|
if (skippedForSafety.size > 0) {
|
|
55096
56094
|
report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
|
|
55097
56095
|
}
|
|
56096
|
+
buildTriage(report);
|
|
56097
|
+
analyseStructArrays(report, outDir);
|
|
55098
56098
|
writer.write("migration-report.json", JSON.stringify(report, null, 2));
|
|
55099
56099
|
writer.write("migration-report.md", renderReportMd(report));
|
|
55100
56100
|
return report;
|
|
@@ -55156,6 +56156,71 @@ function renderReportMd(r) {
|
|
|
55156
56156
|
`of BigQuery connections.`);
|
|
55157
56157
|
}
|
|
55158
56158
|
}
|
|
56159
|
+
if (r.todo.length) {
|
|
56160
|
+
const needing = r.todo.filter(t => t.owner === "needs-decision");
|
|
56161
|
+
const mechanical = r.todo.filter(t => t.owner === "mechanical");
|
|
56162
|
+
lines.push("");
|
|
56163
|
+
lines.push(`## What is left to do`);
|
|
56164
|
+
lines.push("");
|
|
56165
|
+
lines.push(`${r.todo.reduce((n, t) => n + t.count, 0)} site(s) across ${r.todo.length} construct(s). ` +
|
|
56166
|
+
`Work them largest-first within each section — the ordering below already does that.`);
|
|
56167
|
+
lines.push("");
|
|
56168
|
+
lines.push(`**${mechanical.reduce((n, t) => n + t.count, 0)} site(s) are mechanical** — one correct ` +
|
|
56169
|
+
`answer, no judgement needed. **${needing.reduce((n, t) => n + t.count, 0)} site(s) need a ` +
|
|
56170
|
+
`decision** that cannot be read off the SQL: whether a feature is still wanted, what a ` +
|
|
56171
|
+
`structure should become, whether a behaviour was deliberate.`);
|
|
56172
|
+
const severityLabel = {
|
|
56173
|
+
"changes-meaning": "⚠ runs, but may return DIFFERENT RESULTS",
|
|
56174
|
+
"fails-at-run": "fails when the action runs",
|
|
56175
|
+
"blocks-compile": "fails to compile",
|
|
56176
|
+
};
|
|
56177
|
+
const section = (title, items, preamble) => {
|
|
56178
|
+
var _a;
|
|
56179
|
+
if (!items.length)
|
|
56180
|
+
return;
|
|
56181
|
+
lines.push("");
|
|
56182
|
+
lines.push(`### ${title}`);
|
|
56183
|
+
lines.push("");
|
|
56184
|
+
lines.push(preamble);
|
|
56185
|
+
for (const t of items) {
|
|
56186
|
+
lines.push("");
|
|
56187
|
+
lines.push(`- [ ] **${t.title}** — ${t.count} site(s) · ${severityLabel[t.severity]}`);
|
|
56188
|
+
if (t.postgres)
|
|
56189
|
+
lines.push(` - PostgreSQL: ${t.postgres}`);
|
|
56190
|
+
if (t.why)
|
|
56191
|
+
lines.push(` - ${t.why}`);
|
|
56192
|
+
const shown = t.locations.slice(0, 8);
|
|
56193
|
+
for (const l of shown) {
|
|
56194
|
+
lines.push(` - \`${l.file}\` L${l.lines.slice(0, 6).join(", L")}`);
|
|
56195
|
+
}
|
|
56196
|
+
if (t.locations.length > shown.length) {
|
|
56197
|
+
lines.push(` - …and ${t.locations.length - shown.length} more file(s)`);
|
|
56198
|
+
}
|
|
56199
|
+
for (const s of (_a = t.sql) !== null && _a !== void 0 ? _a : []) {
|
|
56200
|
+
lines.push("");
|
|
56201
|
+
lines.push(` \`${s.file}\` L${s.line} — ${s.note}`);
|
|
56202
|
+
lines.push("");
|
|
56203
|
+
lines.push(" ```sql");
|
|
56204
|
+
for (const l of s.sql.split("\n"))
|
|
56205
|
+
lines.push(` ${l}`);
|
|
56206
|
+
lines.push(" ```");
|
|
56207
|
+
}
|
|
56208
|
+
}
|
|
56209
|
+
};
|
|
56210
|
+
section("Needs a decision", needing, "Start here — these gate the rest, and each is a question for whoever owns the data, not " +
|
|
56211
|
+
"something to work out from the code.");
|
|
56212
|
+
section("Mechanical", mechanical, "These have one correct answer and can be worked through without further input.");
|
|
56213
|
+
}
|
|
56214
|
+
if (r.applied.length) {
|
|
56215
|
+
lines.push("");
|
|
56216
|
+
lines.push(`## Rewrites applied automatically`);
|
|
56217
|
+
lines.push("");
|
|
56218
|
+
lines.push(`Listed so they can be reviewed rather than trusted — a converted project is worth reading ` +
|
|
56219
|
+
`before it is run.`);
|
|
56220
|
+
lines.push("");
|
|
56221
|
+
for (const a of r.applied)
|
|
56222
|
+
lines.push(`- ${a.title} — ${a.count}`);
|
|
56223
|
+
}
|
|
55159
56224
|
lines.push("");
|
|
55160
56225
|
lines.push(`## Target files (${targets.length})`);
|
|
55161
56226
|
lines.push("");
|
|
@@ -55217,6 +56282,454 @@ function renderReportMd(r) {
|
|
|
55217
56282
|
return lines.join("\n");
|
|
55218
56283
|
}
|
|
55219
56284
|
|
|
56285
|
+
function parseColumnTypes(config) {
|
|
56286
|
+
const m = /\bcolumnTypes\s*:\s*\{/.exec(config);
|
|
56287
|
+
if (!m)
|
|
56288
|
+
return [];
|
|
56289
|
+
let depth = 0;
|
|
56290
|
+
let end = config.length;
|
|
56291
|
+
for (let i = m.index + m[0].length - 1; i < config.length; i++) {
|
|
56292
|
+
if (config[i] === "{")
|
|
56293
|
+
depth++;
|
|
56294
|
+
else if (config[i] === "}" && --depth === 0) {
|
|
56295
|
+
end = i;
|
|
56296
|
+
break;
|
|
56297
|
+
}
|
|
56298
|
+
}
|
|
56299
|
+
const body = config.slice(m.index + m[0].length, end);
|
|
56300
|
+
const withoutComments = body.replace(/\/\/[^\n]*/g, "");
|
|
56301
|
+
return [...withoutComments.matchAll(/(^|,)\s*([A-Za-z_]\w*)\s*:/g)].map(x => x[2]);
|
|
56302
|
+
}
|
|
56303
|
+
function loadActions(projectDir) {
|
|
56304
|
+
const actions = [];
|
|
56305
|
+
const walk = (dir) => {
|
|
56306
|
+
var _a;
|
|
56307
|
+
if (!fs__namespace.existsSync(dir))
|
|
56308
|
+
return;
|
|
56309
|
+
for (const entry of fs__namespace.readdirSync(dir, { withFileTypes: true })) {
|
|
56310
|
+
const full = path__namespace.join(dir, entry.name);
|
|
56311
|
+
if (entry.isDirectory())
|
|
56312
|
+
walk(full);
|
|
56313
|
+
else if (entry.name.endsWith(".sqlx")) {
|
|
56314
|
+
const source = fs__namespace.readFileSync(full, "utf8");
|
|
56315
|
+
const span = findConfigBlock(source);
|
|
56316
|
+
const config = span ? source.slice(span.start, span.end) : "";
|
|
56317
|
+
const parsed = parseSqlxConfig(config);
|
|
56318
|
+
actions.push({
|
|
56319
|
+
file: path__namespace.relative(projectDir, full),
|
|
56320
|
+
type: parsed.type,
|
|
56321
|
+
name: (_a = parsed.name) !== null && _a !== void 0 ? _a : path__namespace.basename(entry.name, ".sqlx"),
|
|
56322
|
+
schema: parsed.schema,
|
|
56323
|
+
columns: parseColumnTypes(config),
|
|
56324
|
+
body: span ? source.slice(span.end) : source,
|
|
56325
|
+
bodyOffset: span ? span.end : 0,
|
|
56326
|
+
source,
|
|
56327
|
+
});
|
|
56328
|
+
}
|
|
56329
|
+
}
|
|
56330
|
+
};
|
|
56331
|
+
walk(path__namespace.join(projectDir, "definitions"));
|
|
56332
|
+
return actions;
|
|
56333
|
+
}
|
|
56334
|
+
function outputName(toks, start, end) {
|
|
56335
|
+
var _a, _b;
|
|
56336
|
+
const items = toks.slice(start, end);
|
|
56337
|
+
if (!items.length)
|
|
56338
|
+
return null;
|
|
56339
|
+
if (items.some(t => t.text === "*"))
|
|
56340
|
+
return null;
|
|
56341
|
+
const last = items[items.length - 1];
|
|
56342
|
+
if (isWord(items[items.length - 2], "as")) {
|
|
56343
|
+
return ((_a = last.value) !== null && _a !== void 0 ? _a : last.text).replace(/^"|"$/g, "");
|
|
56344
|
+
}
|
|
56345
|
+
const bare = items.filter(t => t.text !== ".");
|
|
56346
|
+
if (bare.every(t => t.kind === "word" || t.kind === "quoted-ident")) {
|
|
56347
|
+
return ((_b = last.value) !== null && _b !== void 0 ? _b : last.text).replace(/^"|"$/g, "");
|
|
56348
|
+
}
|
|
56349
|
+
return null;
|
|
56350
|
+
}
|
|
56351
|
+
class Resolver {
|
|
56352
|
+
constructor(actions) {
|
|
56353
|
+
var _a;
|
|
56354
|
+
this.actions = actions;
|
|
56355
|
+
this.byName = new Map();
|
|
56356
|
+
this.byPair = new Map();
|
|
56357
|
+
this.memo = new Map();
|
|
56358
|
+
for (const a of actions) {
|
|
56359
|
+
this.byName.set(a.name, [...((_a = this.byName.get(a.name)) !== null && _a !== void 0 ? _a : []), a]);
|
|
56360
|
+
if (a.schema)
|
|
56361
|
+
this.byPair.set(`${a.schema}.${a.name}`, a);
|
|
56362
|
+
}
|
|
56363
|
+
}
|
|
56364
|
+
resolveRef(tok) {
|
|
56365
|
+
var _a;
|
|
56366
|
+
const args = [...tok.text.matchAll(/["']([^"']+)["']/g)].map(m => m[1]);
|
|
56367
|
+
if (!args.length)
|
|
56368
|
+
return { reason: "ref with no literal arguments" };
|
|
56369
|
+
const name = args[args.length - 1];
|
|
56370
|
+
if (args.length >= 2) {
|
|
56371
|
+
const hit = this.byPair.get(`${args[args.length - 2]}.${name}`);
|
|
56372
|
+
return hit ? { action: hit } : { reason: `no action ${args[args.length - 2]}.${name}` };
|
|
56373
|
+
}
|
|
56374
|
+
const candidates = (_a = this.byName.get(name)) !== null && _a !== void 0 ? _a : [];
|
|
56375
|
+
if (!candidates.length)
|
|
56376
|
+
return { reason: `unknown action ${name}` };
|
|
56377
|
+
if (candidates.length > 1)
|
|
56378
|
+
return { reason: `ambiguous action ${name}` };
|
|
56379
|
+
return { action: candidates[0] };
|
|
56380
|
+
}
|
|
56381
|
+
columnsOf(action, seen = new Set()) {
|
|
56382
|
+
if (action.columns.length)
|
|
56383
|
+
return action.columns;
|
|
56384
|
+
if (action.type === "declaration")
|
|
56385
|
+
return null;
|
|
56386
|
+
if (seen.has(action.file))
|
|
56387
|
+
return null;
|
|
56388
|
+
const memo = this.memo.get(action.file);
|
|
56389
|
+
if (memo !== undefined && !seen.size)
|
|
56390
|
+
return memo;
|
|
56391
|
+
seen.add(action.file);
|
|
56392
|
+
const result = this.selectColumns(action.body, new Set(seen));
|
|
56393
|
+
if (!seen.size)
|
|
56394
|
+
this.memo.set(action.file, result);
|
|
56395
|
+
return result;
|
|
56396
|
+
}
|
|
56397
|
+
selectColumns(sql, seen) {
|
|
56398
|
+
var _a;
|
|
56399
|
+
const toks = significant(tokenize(sql));
|
|
56400
|
+
const scopes = selectScopes(toks);
|
|
56401
|
+
if (!scopes.length)
|
|
56402
|
+
return null;
|
|
56403
|
+
const scope = scopes.reduce((best, s) => (s.select > best.select ? s : best), scopes[0]);
|
|
56404
|
+
const rels = scope.from === -1 ? [] : relationsIn(toks, scope.fromStart, scope.fromEnd);
|
|
56405
|
+
const names = [];
|
|
56406
|
+
for (const [s, e] of splitOnCommas(toks, scope.listStart, scope.listEnd)) {
|
|
56407
|
+
const item = toks.slice(s, e);
|
|
56408
|
+
const star = item.findIndex(t => t.text === "*");
|
|
56409
|
+
if (star === -1) {
|
|
56410
|
+
const n = outputName(toks, s, e);
|
|
56411
|
+
if (n === null)
|
|
56412
|
+
return null;
|
|
56413
|
+
names.push(n);
|
|
56414
|
+
continue;
|
|
56415
|
+
}
|
|
56416
|
+
const qualifier = star >= 2 && ((_a = item[star - 1]) === null || _a === void 0 ? void 0 : _a.text) === "." ? item[star - 2].text.toLowerCase() : null;
|
|
56417
|
+
const rel = qualifier ? rels.find(r => r.alias === qualifier) : rels[0];
|
|
56418
|
+
if (!rel)
|
|
56419
|
+
return null;
|
|
56420
|
+
const cols = this.relationColumns(rel.tokens, rel.subquery, seen);
|
|
56421
|
+
if (!cols)
|
|
56422
|
+
return null;
|
|
56423
|
+
const excluded = starExclusions(item, star);
|
|
56424
|
+
names.push(...cols.filter(c => !excluded.has(c.toLowerCase())));
|
|
56425
|
+
}
|
|
56426
|
+
return names.length ? names : null;
|
|
56427
|
+
}
|
|
56428
|
+
relationColumns(tokens, subquery, seen) {
|
|
56429
|
+
if (subquery) {
|
|
56430
|
+
const inner = tokens
|
|
56431
|
+
.slice(1, -1)
|
|
56432
|
+
.map(t => t.text)
|
|
56433
|
+
.join(" ");
|
|
56434
|
+
return this.selectColumns(inner, seen);
|
|
56435
|
+
}
|
|
56436
|
+
const tmpl = tokens.find(t => t.kind === "template");
|
|
56437
|
+
if (tmpl) {
|
|
56438
|
+
const { action } = this.resolveRef(tmpl);
|
|
56439
|
+
return action ? this.columnsOf(action, seen) : null;
|
|
56440
|
+
}
|
|
56441
|
+
return null;
|
|
56442
|
+
}
|
|
56443
|
+
}
|
|
56444
|
+
function starExclusions(item, star) {
|
|
56445
|
+
var _a, _b;
|
|
56446
|
+
const out = new Set();
|
|
56447
|
+
if (!isWord(item[star + 1], "except"))
|
|
56448
|
+
return out;
|
|
56449
|
+
const open = star + 2;
|
|
56450
|
+
if (((_a = item[open]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
56451
|
+
return out;
|
|
56452
|
+
const close = matchBracket(item, open);
|
|
56453
|
+
if (close < 0)
|
|
56454
|
+
return out;
|
|
56455
|
+
for (const [s, e] of splitOnCommas(item, open + 1, close)) {
|
|
56456
|
+
const t = item[s];
|
|
56457
|
+
if (t)
|
|
56458
|
+
out.add(((_b = t.value) !== null && _b !== void 0 ? _b : t.text).replace(/^"|"$/g, "").toLowerCase());
|
|
56459
|
+
}
|
|
56460
|
+
return out;
|
|
56461
|
+
}
|
|
56462
|
+
function quoteIfNeeded(col) {
|
|
56463
|
+
return /^[a-z_][a-z0-9_]*$/.test(col) ? col : `"${col}"`;
|
|
56464
|
+
}
|
|
56465
|
+
const MARKER = "-- sqlanvil:star-except";
|
|
56466
|
+
async function migrateFix(opts) {
|
|
56467
|
+
var _a;
|
|
56468
|
+
const result = { expanded: 0, groupByAll: 0, files: [], unresolved: [] };
|
|
56469
|
+
const touched = new Set();
|
|
56470
|
+
for (let pass = 0; pass < 12; pass++) {
|
|
56471
|
+
const actions = loadActions(opts.projectDir);
|
|
56472
|
+
const resolver = new Resolver(actions);
|
|
56473
|
+
let changedThisPass = false;
|
|
56474
|
+
result.unresolved = [];
|
|
56475
|
+
for (const action of actions) {
|
|
56476
|
+
const source = action.source;
|
|
56477
|
+
const toks = significant(tokenize(source));
|
|
56478
|
+
const scopes = selectScopes(toks);
|
|
56479
|
+
const edits = [];
|
|
56480
|
+
for (const scope of scopes) {
|
|
56481
|
+
const rels = scope.from === -1 ? [] : relationsIn(toks, scope.fromStart, scope.fromEnd);
|
|
56482
|
+
for (const [s, e] of splitOnCommas(toks, scope.listStart, scope.listEnd)) {
|
|
56483
|
+
const item = toks.slice(s, e);
|
|
56484
|
+
const star = item.findIndex(t => t.text === "*");
|
|
56485
|
+
if (star === -1 || !isWord(item[star + 1], "except"))
|
|
56486
|
+
continue;
|
|
56487
|
+
const line = source.slice(0, item[star].start).split("\n").length;
|
|
56488
|
+
const qualifier = star >= 2 && ((_a = item[star - 1]) === null || _a === void 0 ? void 0 : _a.text) === "." ? item[star - 2].text.toLowerCase() : null;
|
|
56489
|
+
const rel = qualifier ? rels.find(r => r.alias === qualifier) : rels[0];
|
|
56490
|
+
if (!rel) {
|
|
56491
|
+
result.unresolved.push({
|
|
56492
|
+
file: action.file,
|
|
56493
|
+
line,
|
|
56494
|
+
reason: qualifier
|
|
56495
|
+
? `no relation aliased "${qualifier}" in the FROM/JOIN chain`
|
|
56496
|
+
: "star has no FROM relation",
|
|
56497
|
+
});
|
|
56498
|
+
continue;
|
|
56499
|
+
}
|
|
56500
|
+
let cols = resolver.relationColumns(rel.tokens, rel.subquery, new Set());
|
|
56501
|
+
if (!cols) {
|
|
56502
|
+
const bare = rel.tokens.map(t => t.text).join("");
|
|
56503
|
+
const cte = cteColumns(source, bare, resolver);
|
|
56504
|
+
cols = cte;
|
|
56505
|
+
}
|
|
56506
|
+
if (!cols) {
|
|
56507
|
+
const shown = rel.tokens.map(t => t.text).join("");
|
|
56508
|
+
result.unresolved.push({
|
|
56509
|
+
file: action.file,
|
|
56510
|
+
line,
|
|
56511
|
+
reason: rel.tokens.some(t => t.kind === "template")
|
|
56512
|
+
? `columns of ${shown} are unknown — has introspect run for it?`
|
|
56513
|
+
: `cannot determine the columns of ${shown}`,
|
|
56514
|
+
});
|
|
56515
|
+
continue;
|
|
56516
|
+
}
|
|
56517
|
+
const excluded = starExclusions(item, star);
|
|
56518
|
+
const missing = [...excluded].filter(x => !cols.some(c => c.toLowerCase() === x));
|
|
56519
|
+
if (missing.length) {
|
|
56520
|
+
result.unresolved.push({
|
|
56521
|
+
file: action.file,
|
|
56522
|
+
line,
|
|
56523
|
+
reason: `EXCEPT names ${missing.join(", ")}, which the source does not have`,
|
|
56524
|
+
});
|
|
56525
|
+
continue;
|
|
56526
|
+
}
|
|
56527
|
+
const keep = cols.filter(c => !excluded.has(c.toLowerCase()));
|
|
56528
|
+
if (!keep.length) {
|
|
56529
|
+
result.unresolved.push({
|
|
56530
|
+
file: action.file,
|
|
56531
|
+
line,
|
|
56532
|
+
reason: "the EXCEPT list removes every column",
|
|
56533
|
+
});
|
|
56534
|
+
continue;
|
|
56535
|
+
}
|
|
56536
|
+
const close = matchBracket(item, star + 2);
|
|
56537
|
+
const lineStart = source.lastIndexOf("\n", item[star].start) + 1;
|
|
56538
|
+
const indent = /^[ \t]*/.exec(source.slice(lineStart))[0] + " ";
|
|
56539
|
+
const prefix = qualifier ? `${qualifier}.` : "";
|
|
56540
|
+
const body = keep.map(c => `${indent}${prefix}${quoteIfNeeded(c)}`).join(",\n");
|
|
56541
|
+
const marker = `${indent}${MARKER} ${prefix}* from ` +
|
|
56542
|
+
`${rel.tokens.map(t => t.text).join("")} minus (${[...excluded].join(", ")})`;
|
|
56543
|
+
edits.push({
|
|
56544
|
+
start: qualifier ? item[star - 2].start : item[star].start,
|
|
56545
|
+
end: item[close].end,
|
|
56546
|
+
out: `\n${marker}\n${body}`,
|
|
56547
|
+
});
|
|
56548
|
+
}
|
|
56549
|
+
}
|
|
56550
|
+
if (!edits.length)
|
|
56551
|
+
continue;
|
|
56552
|
+
let out = source;
|
|
56553
|
+
for (const edit of edits.sort((a, b) => b.start - a.start)) {
|
|
56554
|
+
const head = out.slice(0, edit.start).replace(/[ \t]+$/, "");
|
|
56555
|
+
out = head + edit.out + out.slice(edit.end);
|
|
56556
|
+
result.expanded++;
|
|
56557
|
+
}
|
|
56558
|
+
changedThisPass = true;
|
|
56559
|
+
touched.add(action.file);
|
|
56560
|
+
if (opts.write) {
|
|
56561
|
+
fs__namespace.writeFileSync(path__namespace.join(opts.projectDir, action.file), out);
|
|
56562
|
+
}
|
|
56563
|
+
else {
|
|
56564
|
+
action.source = out;
|
|
56565
|
+
}
|
|
56566
|
+
}
|
|
56567
|
+
if (!changedThisPass || !opts.write)
|
|
56568
|
+
break;
|
|
56569
|
+
}
|
|
56570
|
+
for (const action of loadActions(opts.projectDir)) {
|
|
56571
|
+
const { text, expanded, unresolved } = expandGroupByAll(action.source);
|
|
56572
|
+
for (const u of unresolved) {
|
|
56573
|
+
result.unresolved.push({ file: action.file, line: u.line, reason: u.reason });
|
|
56574
|
+
}
|
|
56575
|
+
if (!expanded)
|
|
56576
|
+
continue;
|
|
56577
|
+
result.groupByAll += expanded;
|
|
56578
|
+
touched.add(action.file);
|
|
56579
|
+
if (opts.write)
|
|
56580
|
+
fs__namespace.writeFileSync(path__namespace.join(opts.projectDir, action.file), text);
|
|
56581
|
+
}
|
|
56582
|
+
result.files = [...touched].sort();
|
|
56583
|
+
if (opts.write)
|
|
56584
|
+
updateMigrationReport(opts.projectDir, result);
|
|
56585
|
+
return result;
|
|
56586
|
+
}
|
|
56587
|
+
function updateMigrationReport(projectDir, result) {
|
|
56588
|
+
var _a, _b, _c;
|
|
56589
|
+
const jsonPath = path__namespace.join(projectDir, "migration-report.json");
|
|
56590
|
+
if (!fs__namespace.existsSync(jsonPath))
|
|
56591
|
+
return;
|
|
56592
|
+
let report;
|
|
56593
|
+
try {
|
|
56594
|
+
report = JSON.parse(fs__namespace.readFileSync(jsonPath, "utf8"));
|
|
56595
|
+
}
|
|
56596
|
+
catch (_d) {
|
|
56597
|
+
return;
|
|
56598
|
+
}
|
|
56599
|
+
if (!Array.isArray(report.todo))
|
|
56600
|
+
return;
|
|
56601
|
+
const OWNED = new Set(["star-except", "group-by-all"]);
|
|
56602
|
+
const remaining = new Map();
|
|
56603
|
+
for (const site of result.unresolved) {
|
|
56604
|
+
const id = /GROUP BY ALL/i.test(site.reason) || /star expands/i.test(site.reason)
|
|
56605
|
+
? "group-by-all"
|
|
56606
|
+
: "star-except";
|
|
56607
|
+
const prior = report.todo.find(t => t.id === id);
|
|
56608
|
+
const entry = (_a = remaining.get(id)) !== null && _a !== void 0 ? _a : {
|
|
56609
|
+
id,
|
|
56610
|
+
title: (_b = prior === null || prior === void 0 ? void 0 : prior.title) !== null && _b !== void 0 ? _b : id,
|
|
56611
|
+
count: 0,
|
|
56612
|
+
severity: (_c = prior === null || prior === void 0 ? void 0 : prior.severity) !== null && _c !== void 0 ? _c : "blocks-compile",
|
|
56613
|
+
owner: "needs-decision",
|
|
56614
|
+
postgres: prior === null || prior === void 0 ? void 0 : prior.postgres,
|
|
56615
|
+
why: site.reason,
|
|
56616
|
+
locations: [],
|
|
56617
|
+
};
|
|
56618
|
+
entry.count++;
|
|
56619
|
+
const loc = entry.locations.find(l => l.file === site.file);
|
|
56620
|
+
if (loc)
|
|
56621
|
+
loc.lines.push(site.line);
|
|
56622
|
+
else
|
|
56623
|
+
entry.locations.push({ file: site.file, lines: [site.line] });
|
|
56624
|
+
remaining.set(id, entry);
|
|
56625
|
+
}
|
|
56626
|
+
report.todo = [...report.todo.filter(t => !OWNED.has(t.id)), ...remaining.values()];
|
|
56627
|
+
const applied = (n, id, title) => {
|
|
56628
|
+
if (!n)
|
|
56629
|
+
return;
|
|
56630
|
+
const seen = report.applied.find(a => a.id === id);
|
|
56631
|
+
if (seen)
|
|
56632
|
+
seen.count = n;
|
|
56633
|
+
else
|
|
56634
|
+
report.applied.push({ id, title, count: n });
|
|
56635
|
+
};
|
|
56636
|
+
applied(result.expanded, "star-except", "SELECT * EXCEPT expanded (migrate-fix)");
|
|
56637
|
+
applied(result.groupByAll, "group-by-all", "GROUP BY ALL rewritten as ordinals (migrate-fix)");
|
|
56638
|
+
fs__namespace.writeFileSync(jsonPath, JSON.stringify(report, null, 2));
|
|
56639
|
+
fs__namespace.writeFileSync(path__namespace.join(projectDir, "migration-report.md"), renderReportMd(report));
|
|
56640
|
+
}
|
|
56641
|
+
const AGGREGATES = new Set([
|
|
56642
|
+
"count", "sum", "avg", "min", "max", "array_agg", "string_agg", "bool_and", "bool_or", "every",
|
|
56643
|
+
"stddev", "stddev_pop", "stddev_samp", "variance", "var_pop", "var_samp", "jsonb_agg",
|
|
56644
|
+
"json_agg", "jsonb_object_agg", "json_object_agg", "corr", "covar_pop", "covar_samp",
|
|
56645
|
+
"percentile_cont", "percentile_disc", "bit_and", "bit_or", "logical_and", "logical_or",
|
|
56646
|
+
"any_value", "countif",
|
|
56647
|
+
]);
|
|
56648
|
+
function isGroupable(item) {
|
|
56649
|
+
var _a, _b;
|
|
56650
|
+
for (let i = 0; i < item.length; i++) {
|
|
56651
|
+
if (item[i].kind !== "word" || !AGGREGATES.has(item[i].text.toLowerCase()))
|
|
56652
|
+
continue;
|
|
56653
|
+
if (((_a = item[i + 1]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
56654
|
+
continue;
|
|
56655
|
+
const close = matchBracket(item, i + 1);
|
|
56656
|
+
if (close < 0)
|
|
56657
|
+
return false;
|
|
56658
|
+
let after = close + 1;
|
|
56659
|
+
if (isWord(item[after], "filter") && ((_b = item[after + 1]) === null || _b === void 0 ? void 0 : _b.text) === "(") {
|
|
56660
|
+
const filterClose = matchBracket(item, after + 1);
|
|
56661
|
+
if (filterClose > 0)
|
|
56662
|
+
after = filterClose + 1;
|
|
56663
|
+
}
|
|
56664
|
+
if (!isWord(item[after], "over"))
|
|
56665
|
+
return false;
|
|
56666
|
+
}
|
|
56667
|
+
return !item.some((t, i) => isWord(t, "over") && i > 0);
|
|
56668
|
+
}
|
|
56669
|
+
function isStarItem(item) {
|
|
56670
|
+
if (!item.length)
|
|
56671
|
+
return false;
|
|
56672
|
+
if (item[0].text === "*")
|
|
56673
|
+
return true;
|
|
56674
|
+
return item.length >= 3 && item[1].text === "." && item[2].text === "*";
|
|
56675
|
+
}
|
|
56676
|
+
function expandGroupByAll(source) {
|
|
56677
|
+
const toks = significant(tokenize(source));
|
|
56678
|
+
const scopes = selectScopes(toks);
|
|
56679
|
+
const edits = [];
|
|
56680
|
+
const unresolved = [];
|
|
56681
|
+
for (let i = 0; i + 2 < toks.length; i++) {
|
|
56682
|
+
if (!isWord(toks[i], "group") || !isWord(toks[i + 1], "by") || !isWord(toks[i + 2], "all")) {
|
|
56683
|
+
continue;
|
|
56684
|
+
}
|
|
56685
|
+
const line = source.slice(0, toks[i].start).split("\n").length;
|
|
56686
|
+
const scope = scopes.filter(s => s.select < i).pop();
|
|
56687
|
+
if (!scope)
|
|
56688
|
+
continue;
|
|
56689
|
+
const items = splitOnCommas(toks, scope.listStart, scope.listEnd);
|
|
56690
|
+
if (items.some(([s, e]) => isStarItem(toks.slice(s, e)))) {
|
|
56691
|
+
unresolved.push({
|
|
56692
|
+
line,
|
|
56693
|
+
reason: "GROUP BY ALL over a select list containing `*` — the ordinals depend on how many " +
|
|
56694
|
+
"columns the star expands to",
|
|
56695
|
+
});
|
|
56696
|
+
continue;
|
|
56697
|
+
}
|
|
56698
|
+
const ordinals = items
|
|
56699
|
+
.map(([s, e], idx) => (isGroupable(toks.slice(s, e)) ? idx + 1 : null))
|
|
56700
|
+
.filter((n) => n !== null);
|
|
56701
|
+
if (!ordinals.length) {
|
|
56702
|
+
unresolved.push({ line, reason: "every select-list item is an aggregate" });
|
|
56703
|
+
continue;
|
|
56704
|
+
}
|
|
56705
|
+
edits.push({ start: toks[i + 2].start, end: toks[i + 2].end, out: ordinals.join(", ") });
|
|
56706
|
+
}
|
|
56707
|
+
let text = source;
|
|
56708
|
+
for (const e of edits.reverse()) {
|
|
56709
|
+
text = text.slice(0, e.start) + e.out + text.slice(e.end);
|
|
56710
|
+
}
|
|
56711
|
+
return { text, expanded: edits.length, unresolved };
|
|
56712
|
+
}
|
|
56713
|
+
function cteColumns(source, name, resolver) {
|
|
56714
|
+
var _a;
|
|
56715
|
+
const toks = significant(tokenize(source));
|
|
56716
|
+
for (let i = 0; i < toks.length; i++) {
|
|
56717
|
+
if (toks[i].kind !== "word" || toks[i].text.toLowerCase() !== name.toLowerCase())
|
|
56718
|
+
continue;
|
|
56719
|
+
if (!isWord(toks[i + 1], "as") || ((_a = toks[i + 2]) === null || _a === void 0 ? void 0 : _a.text) !== "(")
|
|
56720
|
+
continue;
|
|
56721
|
+
const close = matchBracket(toks, i + 2);
|
|
56722
|
+
if (close < 0)
|
|
56723
|
+
return null;
|
|
56724
|
+
const inner = toks
|
|
56725
|
+
.slice(i + 3, close)
|
|
56726
|
+
.map(t => t.text)
|
|
56727
|
+
.join(" ");
|
|
56728
|
+
return resolver.selectColumns(inner, new Set());
|
|
56729
|
+
}
|
|
56730
|
+
return null;
|
|
56731
|
+
}
|
|
56732
|
+
|
|
55220
56733
|
function actuallyResolve(...filePaths) {
|
|
55221
56734
|
return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
|
|
55222
56735
|
}
|
|
@@ -58354,6 +59867,39 @@ function runCli() {
|
|
|
58354
59867
|
print(`Next: sqlanvil compile ${argv["out-dir"]}`);
|
|
58355
59868
|
return 0;
|
|
58356
59869
|
}
|
|
59870
|
+
},
|
|
59871
|
+
{
|
|
59872
|
+
format: "migrate-fix [project-dir]",
|
|
59873
|
+
description: "Finish a converted project's dialect work, AFTER `scripts/introspect_all.sh` has run. " +
|
|
59874
|
+
"Expands `SELECT * EXCEPT (...)` into explicit column lists and rewrites `GROUP BY ALL` " +
|
|
59875
|
+
"as positional ordinals — both need the source columns that only exist once " +
|
|
59876
|
+
"declarations have been introspected, so this is the " +
|
|
59877
|
+
"third phase of a migration (convert, introspect, fix), not part of the conversion. " +
|
|
59878
|
+
"Re-runnable; reports anything it could not resolve instead of guessing.",
|
|
59879
|
+
positionalOptions: [projectDirMustExistOption],
|
|
59880
|
+
options: [
|
|
59881
|
+
option("dry-run", {
|
|
59882
|
+
describe: "Report what would change without writing.",
|
|
59883
|
+
type: "boolean",
|
|
59884
|
+
default: false
|
|
59885
|
+
})
|
|
59886
|
+
],
|
|
59887
|
+
processFn: async (argv) => {
|
|
59888
|
+
const result = await migrateFix({
|
|
59889
|
+
projectDir: argv["project-dir"],
|
|
59890
|
+
write: !argv["dry-run"]
|
|
59891
|
+
});
|
|
59892
|
+
const verb = argv["dry-run"] ? "would rewrite" : "rewrote";
|
|
59893
|
+
print(`${verb} ${result.expanded} star-except site(s) and ${result.groupByAll} ` +
|
|
59894
|
+
`GROUP BY ALL clause(s) across ${result.files.length} file(s).`);
|
|
59895
|
+
if (result.unresolved.length) {
|
|
59896
|
+
printError(`${result.unresolved.length} site(s) need a look:`);
|
|
59897
|
+
for (const u of result.unresolved) {
|
|
59898
|
+
printError(` ${u.file}:${u.line} — ${u.reason}`);
|
|
59899
|
+
}
|
|
59900
|
+
}
|
|
59901
|
+
return result.unresolved.length ? 1 : 0;
|
|
59902
|
+
}
|
|
58357
59903
|
}
|
|
58358
59904
|
]
|
|
58359
59905
|
})
|