@sqlanvil/cli 1.21.0 → 1.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +1428 -554
- package/package.json +1 -1
package/bundle.js
CHANGED
|
@@ -50606,7 +50606,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
50606
50606
|
.filter(validationQuery => !!validationQuery.query);
|
|
50607
50607
|
}
|
|
50608
50608
|
|
|
50609
|
-
const version = "1.
|
|
50609
|
+
const version = "1.23.0";
|
|
50610
50610
|
const dataformVersion = "3.0.61";
|
|
50611
50611
|
|
|
50612
50612
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -51016,7 +51016,8 @@ function mysqlCredentialsTemplate() {
|
|
|
51016
51016
|
};
|
|
51017
51017
|
return `${JSON.stringify(template, null, 2)}\n`;
|
|
51018
51018
|
}
|
|
51019
|
-
async function init(projectDir, projectConfig) {
|
|
51019
|
+
async function init(projectDir, projectConfig, options = {}) {
|
|
51020
|
+
var _a;
|
|
51020
51021
|
const workflowSettingsYamlPath = path__namespace.join(projectDir, "workflow_settings.yaml");
|
|
51021
51022
|
const packageJsonPath = path__namespace.join(projectDir, "package.json");
|
|
51022
51023
|
const gitignorePath = path__namespace.join(projectDir, ".gitignore");
|
|
@@ -51064,7 +51065,11 @@ async function init(projectDir, projectConfig) {
|
|
|
51064
51065
|
if (projectConfig.builtinAssertionNamePrefix) {
|
|
51065
51066
|
workflowSettings.builtinAssertionNamePrefix = projectConfig.builtinAssertionNamePrefix;
|
|
51066
51067
|
}
|
|
51067
|
-
|
|
51068
|
+
const includeSample = options.includeSample !== false;
|
|
51069
|
+
const withBigQuerySource = includeSample &&
|
|
51070
|
+
warehouse !== "mysql" &&
|
|
51071
|
+
(!isPostgresLike || options.includeBigQuerySource !== false);
|
|
51072
|
+
if (isPostgresLike && withBigQuerySource) {
|
|
51068
51073
|
workflowSettings.connections = {
|
|
51069
51074
|
bigquery_public: sqlanvil.ConnectionConfig.create({
|
|
51070
51075
|
platform: "bigquery",
|
|
@@ -51080,7 +51085,7 @@ async function init(projectDir, projectConfig) {
|
|
|
51080
51085
|
fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
|
|
51081
51086
|
filesWritten.push(gitignorePath);
|
|
51082
51087
|
if (!isBigQuery) {
|
|
51083
|
-
fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse));
|
|
51088
|
+
fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
|
|
51084
51089
|
filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
|
|
51085
51090
|
}
|
|
51086
51091
|
const mkdir = (...segments) => {
|
|
@@ -51101,7 +51106,10 @@ async function init(projectDir, projectConfig) {
|
|
|
51101
51106
|
const reportingDir = mkdir("definitions", "outputs", "reporting");
|
|
51102
51107
|
const testDir = mkdir("definitions", "test");
|
|
51103
51108
|
keep(mkdir("includes"));
|
|
51104
|
-
|
|
51109
|
+
if (!includeSample) {
|
|
51110
|
+
[sourcesDir, intermediateDir, salesDir, reportingDir, testDir].forEach(keep);
|
|
51111
|
+
return { filesWritten, dirsCreated };
|
|
51112
|
+
}
|
|
51105
51113
|
write(path__namespace.join(sourcesDir, "app_orders.sqlx"), `-- A declaration: a table that already exists in your warehouse (created by your
|
|
51106
51114
|
-- application, an import, or another pipeline). Point name/dataset at a real table —
|
|
51107
51115
|
-- downstream models read it with \${ref("app_orders")}.
|
|
@@ -53541,7 +53549,7 @@ class Runner {
|
|
|
53541
53549
|
return actionResult;
|
|
53542
53550
|
}
|
|
53543
53551
|
async executeTask(client, task, parentAction, options, action) {
|
|
53544
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
53552
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
|
|
53545
53553
|
const timer = Timer.start();
|
|
53546
53554
|
const taskResult = {
|
|
53547
53555
|
status: sqlanvil.TaskResult.ExecutionStatus.RUNNING,
|
|
@@ -53613,7 +53621,14 @@ class Runner {
|
|
|
53613
53621
|
}
|
|
53614
53622
|
else if (task.type === "extract") {
|
|
53615
53623
|
try {
|
|
53616
|
-
|
|
53624
|
+
if (Object.keys((_e = (_d = action === null || action === void 0 ? void 0 : action.extract) === null || _d === void 0 ? void 0 : _d.columnTypes) !== null && _e !== void 0 ? _e : {}).length === 0) {
|
|
53625
|
+
const ds = ((_f = action === null || action === void 0 ? void 0 : action.extract) === null || _f === void 0 ? void 0 : _f.platform) === "mysql"
|
|
53626
|
+
? (_g = action === null || action === void 0 ? void 0 : action.extract) === null || _g === void 0 ? void 0 : _g.database : (_h = action === null || action === void 0 ? void 0 : action.extract) === null || _h === void 0 ? void 0 : _h.dataset;
|
|
53627
|
+
throw new Error(`declaration "${(_j = action === null || action === void 0 ? void 0 : action.target) === null || _j === void 0 ? void 0 : _j.name}" has no columnTypes — scaffold them with: ` +
|
|
53628
|
+
`sqlanvil introspect ${(_k = action === null || action === void 0 ? void 0 : action.extract) === null || _k === void 0 ? void 0 : _k.connectionName} ${ds}.${(_l = action === null || action === void 0 ? void 0 : action.extract) === null || _l === void 0 ? void 0 : _l.sourceName} ` +
|
|
53629
|
+
`--output <its .sqlx file>`);
|
|
53630
|
+
}
|
|
53631
|
+
const extractor = ((_m = action === null || action === void 0 ? void 0 : action.extract) === null || _m === void 0 ? void 0 : _m.platform) === "mysql"
|
|
53617
53632
|
? this.executionOptions.mysqlExtract || runMysqlExtract
|
|
53618
53633
|
: this.executionOptions.bigQueryExtract || runBigQueryExtract;
|
|
53619
53634
|
await extractor({
|
|
@@ -53643,7 +53658,7 @@ class Runner {
|
|
|
53643
53658
|
}), task.type === "operation" ? 1 : options.bigquery.actionRetryLimit + 1 || 1);
|
|
53644
53659
|
taskResult.metadata = metadata;
|
|
53645
53660
|
if (task.type === "assertion") {
|
|
53646
|
-
const rowCount = (
|
|
53661
|
+
const rowCount = (_o = rows[0]) === null || _o === void 0 ? void 0 : _o[Object.keys(rows[0])[0]];
|
|
53647
53662
|
if (rowCount > 0) {
|
|
53648
53663
|
throw new Error(`Assertion failed: query returned ${rowCount} row(s).`);
|
|
53649
53664
|
}
|
|
@@ -53655,7 +53670,7 @@ class Runner {
|
|
|
53655
53670
|
? sqlanvil.TaskResult.ExecutionStatus.CANCELLED
|
|
53656
53671
|
: sqlanvil.TaskResult.ExecutionStatus.FAILED;
|
|
53657
53672
|
taskResult.errorMessage = `${this.graph.projectConfig.warehouse} error: ${e.message}`;
|
|
53658
|
-
if ((
|
|
53673
|
+
if ((_q = (_p = e.metadata) === null || _p === void 0 ? void 0 : _p.bigquery) === null || _q === void 0 ? void 0 : _q.jobId) {
|
|
53659
53674
|
taskResult.metadata = {
|
|
53660
53675
|
bigquery: {
|
|
53661
53676
|
jobId: e.metadata.bigquery.jobId
|
|
@@ -54175,443 +54190,594 @@ ${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></t
|
|
|
54175
54190
|
`;
|
|
54176
54191
|
}
|
|
54177
54192
|
|
|
54178
|
-
|
|
54179
|
-
|
|
54180
|
-
|
|
54181
|
-
|
|
54182
|
-
|
|
54183
|
-
|
|
54184
|
-
|
|
54185
|
-
py_version = platform.python_version()
|
|
54186
|
-
|
|
54187
|
-
def load_packaging():
|
|
54188
|
-
try:
|
|
54189
|
-
from packaging.specifiers import SpecifierSet
|
|
54190
|
-
from packaging.requirements import Requirement
|
|
54191
|
-
return SpecifierSet, Requirement
|
|
54192
|
-
except ImportError:
|
|
54193
|
-
pass
|
|
54194
|
-
try:
|
|
54195
|
-
from pip._vendor.packaging.specifiers import SpecifierSet
|
|
54196
|
-
from pip._vendor.packaging.requirements import Requirement
|
|
54197
|
-
return SpecifierSet, Requirement
|
|
54198
|
-
except ImportError:
|
|
54199
|
-
return None, None
|
|
54200
|
-
|
|
54201
|
-
SpecifierSet, Requirement = load_packaging()
|
|
54202
|
-
|
|
54203
|
-
def mini_parse(v):
|
|
54204
|
-
import re
|
|
54205
|
-
return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
|
|
54206
|
-
|
|
54207
|
-
def mini_check(version, spec):
|
|
54208
|
-
import re
|
|
54209
|
-
ok = True
|
|
54210
|
-
for clause in spec.split(","):
|
|
54211
|
-
clause = clause.strip()
|
|
54212
|
-
m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
|
|
54213
|
-
if not m:
|
|
54214
|
-
return None
|
|
54215
|
-
op, want = m.group(1), m.group(2).strip()
|
|
54216
|
-
if want.endswith(".*"):
|
|
54217
|
-
prefix = mini_parse(want[:-2])
|
|
54218
|
-
have = mini_parse(version)[:len(prefix)]
|
|
54219
|
-
eq = have == prefix
|
|
54220
|
-
if (op in ("==", "===") and not eq) or (op == "!=" and eq):
|
|
54221
|
-
ok = False
|
|
54222
|
-
continue
|
|
54223
|
-
a, b = mini_parse(version), mini_parse(want)
|
|
54224
|
-
n = max(len(a), len(b))
|
|
54225
|
-
a = a + [0] * (n - len(a))
|
|
54226
|
-
b = b + [0] * (n - len(b))
|
|
54227
|
-
if op in ("==", "==="):
|
|
54228
|
-
res = a == b
|
|
54229
|
-
elif op == "!=":
|
|
54230
|
-
res = a != b
|
|
54231
|
-
elif op == "<=":
|
|
54232
|
-
res = a <= b
|
|
54233
|
-
elif op == ">=":
|
|
54234
|
-
res = a >= b
|
|
54235
|
-
elif op == "<":
|
|
54236
|
-
res = a < b
|
|
54237
|
-
elif op == ">":
|
|
54238
|
-
res = a > b
|
|
54239
|
-
else: # ~= : at least b, within b's series
|
|
54240
|
-
wb = mini_parse(want)
|
|
54241
|
-
res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
|
|
54242
|
-
if not res:
|
|
54243
|
-
ok = False
|
|
54244
|
-
return ok
|
|
54245
|
-
|
|
54246
|
-
def check_version(version, spec, what):
|
|
54247
|
-
if SpecifierSet is not None:
|
|
54248
|
-
try:
|
|
54249
|
-
if version not in SpecifierSet(spec, prereleases=True):
|
|
54250
|
-
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
54251
|
-
return
|
|
54252
|
-
except Exception as e:
|
|
54253
|
-
errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
|
|
54254
|
-
return
|
|
54255
|
-
if mini_check(version, spec) is False:
|
|
54256
|
-
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
54257
|
-
|
|
54258
|
-
if version_spec:
|
|
54259
|
-
check_version(py_version, version_spec, "python interpreter")
|
|
54260
|
-
|
|
54261
|
-
if requirements_path:
|
|
54262
|
-
from importlib import metadata
|
|
54263
|
-
try:
|
|
54264
|
-
lines = open(requirements_path).read().splitlines()
|
|
54265
|
-
except OSError as e:
|
|
54266
|
-
errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
|
|
54267
|
-
lines = []
|
|
54268
|
-
for raw in lines:
|
|
54269
|
-
line = raw.split("#")[0].strip()
|
|
54270
|
-
if not line or line.startswith("-"):
|
|
54271
|
-
continue
|
|
54272
|
-
if Requirement is not None:
|
|
54273
|
-
try:
|
|
54274
|
-
req = Requirement(line)
|
|
54275
|
-
except Exception:
|
|
54276
|
-
errors.append('requirements: cannot parse "%s"' % line)
|
|
54277
|
-
continue
|
|
54278
|
-
if req.marker is not None and not req.marker.evaluate():
|
|
54279
|
-
continue
|
|
54280
|
-
name, spec = req.name, str(req.specifier)
|
|
54281
|
-
else:
|
|
54282
|
-
import re
|
|
54283
|
-
if ";" in line:
|
|
54284
|
-
line = line.split(";")[0].strip()
|
|
54285
|
-
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
|
|
54286
|
-
if not m:
|
|
54287
|
-
errors.append('requirements: cannot parse "%s"' % line)
|
|
54288
|
-
continue
|
|
54289
|
-
name, spec = m.group(1), (m.group(3) or "").strip()
|
|
54290
|
-
try:
|
|
54291
|
-
installed = metadata.version(name)
|
|
54292
|
-
except metadata.PackageNotFoundError:
|
|
54293
|
-
errors.append(
|
|
54294
|
-
"requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
|
|
54295
|
-
)
|
|
54296
|
-
continue
|
|
54297
|
-
if spec:
|
|
54298
|
-
check_version(installed, spec, "requirements: %s" % name)
|
|
54299
|
-
|
|
54300
|
-
try:
|
|
54301
|
-
with open(script_path, "rb") as f:
|
|
54302
|
-
compile(f.read(), script_path, "exec")
|
|
54303
|
-
except SyntaxError as e:
|
|
54304
|
-
errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
|
|
54305
|
-
except (OSError, ValueError) as e:
|
|
54306
|
-
errors.append("syntax: cannot compile %s: %s" % (script_path, e))
|
|
54307
|
-
|
|
54308
|
-
print(json.dumps({"interpreter_version": py_version, "errors": errors}))
|
|
54309
|
-
`;
|
|
54310
|
-
const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
|
|
54311
|
-
childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
|
|
54312
|
-
if (err) {
|
|
54313
|
-
reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
|
|
54314
|
-
}
|
|
54315
|
-
else {
|
|
54316
|
-
resolve(stdout);
|
|
54317
|
-
}
|
|
54318
|
-
});
|
|
54319
|
-
});
|
|
54320
|
-
const failure = (message) => ({
|
|
54321
|
-
status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
|
|
54322
|
-
error: { message }
|
|
54323
|
-
});
|
|
54324
|
-
async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
|
|
54325
|
-
if (script.language !== "python") {
|
|
54326
|
-
return [failure(`no validator for script language "${script.language}"`)];
|
|
54327
|
-
}
|
|
54328
|
-
let interpreter;
|
|
54329
|
-
try {
|
|
54330
|
-
interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
|
|
54331
|
-
}
|
|
54332
|
-
catch (e) {
|
|
54333
|
-
return [failure(e.message)];
|
|
54334
|
-
}
|
|
54335
|
-
let stdout;
|
|
54336
|
-
try {
|
|
54337
|
-
stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
|
|
54338
|
-
}
|
|
54339
|
-
catch (e) {
|
|
54340
|
-
return [failure(e.message)];
|
|
54341
|
-
}
|
|
54342
|
-
let parsed;
|
|
54343
|
-
try {
|
|
54344
|
-
parsed = JSON.parse(stdout);
|
|
54345
|
-
}
|
|
54346
|
-
catch (e) {
|
|
54347
|
-
return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
|
|
54193
|
+
function containsPath(parent, child) {
|
|
54194
|
+
const rel = path__namespace.relative(parent, child);
|
|
54195
|
+
return rel === "" || (!rel.startsWith("..") && !path__namespace.isAbsolute(rel));
|
|
54196
|
+
}
|
|
54197
|
+
class OutWriter {
|
|
54198
|
+
constructor(outDir) {
|
|
54199
|
+
this.outDir = outDir;
|
|
54348
54200
|
}
|
|
54349
|
-
|
|
54350
|
-
|
|
54201
|
+
write(relPath, contents) {
|
|
54202
|
+
const full = path__namespace.resolve(this.outDir, relPath);
|
|
54203
|
+
if (!containsPath(path__namespace.resolve(this.outDir), full)) {
|
|
54204
|
+
throw new Error(`Refusing to write outside the output directory: ${relPath}`);
|
|
54205
|
+
}
|
|
54206
|
+
fs__namespace$1.mkdirSync(path__namespace.dirname(full), { recursive: true });
|
|
54207
|
+
fs__namespace$1.writeFileSync(full, contents);
|
|
54351
54208
|
}
|
|
54352
|
-
return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
|
|
54353
54209
|
}
|
|
54354
|
-
|
|
54355
|
-
|
|
54356
|
-
|
|
54210
|
+
const SAFETY_SKIP = [
|
|
54211
|
+
/^\.git(\/|$)/,
|
|
54212
|
+
/(^|\/)node_modules(\/|$)/,
|
|
54213
|
+
/(^|\/)\.df-credentials[^/]*\.json$/,
|
|
54214
|
+
/service_account[^/]*\.json$/i,
|
|
54215
|
+
/(^|\/)\.env([^/]*)$/,
|
|
54216
|
+
/\.pem$/,
|
|
54217
|
+
/(^|\/)compile_output\.json$/,
|
|
54218
|
+
/(^|\/)\.DS_Store$/,
|
|
54219
|
+
/(^|\/)target(\/|$)/,
|
|
54220
|
+
];
|
|
54221
|
+
function walkFiles(root) {
|
|
54222
|
+
const out = [];
|
|
54223
|
+
const walk = (rel) => {
|
|
54224
|
+
for (const entry of fs__namespace$1.readdirSync(path__namespace.join(root, rel), { withFileTypes: true })) {
|
|
54225
|
+
const relPath = rel ? `${rel}/${entry.name}` : entry.name;
|
|
54226
|
+
if (SAFETY_SKIP.some(re => re.test(relPath) || re.test(relPath + "/"))) {
|
|
54227
|
+
skippedForSafety.add(relPath);
|
|
54228
|
+
continue;
|
|
54229
|
+
}
|
|
54230
|
+
if (entry.isDirectory()) {
|
|
54231
|
+
walk(relPath);
|
|
54232
|
+
}
|
|
54233
|
+
else if (entry.isFile()) {
|
|
54234
|
+
out.push(relPath);
|
|
54235
|
+
}
|
|
54236
|
+
}
|
|
54237
|
+
};
|
|
54238
|
+
walk("");
|
|
54239
|
+
return out;
|
|
54357
54240
|
}
|
|
54358
|
-
|
|
54359
|
-
|
|
54360
|
-
const
|
|
54361
|
-
|
|
54362
|
-
|
|
54363
|
-
|
|
54364
|
-
|
|
54365
|
-
|
|
54366
|
-
|
|
54241
|
+
let skippedForSafety;
|
|
54242
|
+
function findConfigBlock(source) {
|
|
54243
|
+
const m = /(^|\n)\s*config\s*\{/.exec(source);
|
|
54244
|
+
if (!m)
|
|
54245
|
+
return null;
|
|
54246
|
+
const open = source.indexOf("{", m.index + m[0].indexOf("config"));
|
|
54247
|
+
let depth = 0;
|
|
54248
|
+
let inString = null;
|
|
54249
|
+
for (let i = open; i < source.length; i++) {
|
|
54250
|
+
const ch = source[i];
|
|
54251
|
+
const prev = source[i - 1];
|
|
54252
|
+
if (inString) {
|
|
54253
|
+
if (ch === inString && prev !== "\\")
|
|
54254
|
+
inString = null;
|
|
54255
|
+
continue;
|
|
54367
54256
|
}
|
|
54368
|
-
|
|
54369
|
-
|
|
54370
|
-
.filter(n => (indegree.get(n.key) || 0) === 0)
|
|
54371
|
-
.map(n => n.key)
|
|
54372
|
-
.sort();
|
|
54373
|
-
const ordered = [];
|
|
54374
|
-
const emitted = new Set();
|
|
54375
|
-
while (ready.length) {
|
|
54376
|
-
const key = ready.shift();
|
|
54377
|
-
if (emitted.has(key)) {
|
|
54257
|
+
if (ch === '"' || ch === "'" || ch === "`") {
|
|
54258
|
+
inString = ch;
|
|
54378
54259
|
continue;
|
|
54379
54260
|
}
|
|
54380
|
-
|
|
54381
|
-
|
|
54382
|
-
|
|
54383
|
-
|
|
54384
|
-
if (
|
|
54385
|
-
|
|
54386
|
-
|
|
54261
|
+
if (ch === "{")
|
|
54262
|
+
depth++;
|
|
54263
|
+
if (ch === "}") {
|
|
54264
|
+
depth--;
|
|
54265
|
+
if (depth === 0) {
|
|
54266
|
+
return {
|
|
54267
|
+
start: m.index + (m[1] ? m[1].length : 0),
|
|
54268
|
+
open,
|
|
54269
|
+
end: i + 1,
|
|
54270
|
+
body: source.slice(open + 1, i),
|
|
54271
|
+
};
|
|
54387
54272
|
}
|
|
54388
54273
|
}
|
|
54389
54274
|
}
|
|
54390
|
-
|
|
54391
|
-
|
|
54392
|
-
|
|
54393
|
-
|
|
54394
|
-
|
|
54275
|
+
return null;
|
|
54276
|
+
}
|
|
54277
|
+
function jsBlockLines(source) {
|
|
54278
|
+
const lines = new Set();
|
|
54279
|
+
const re = /(^|\n)\s*js\s*\{/g;
|
|
54280
|
+
let m;
|
|
54281
|
+
while ((m = re.exec(source))) {
|
|
54282
|
+
const open = source.indexOf("{", m.index + m[0].length - 1);
|
|
54283
|
+
let depth = 0;
|
|
54284
|
+
let end = -1;
|
|
54285
|
+
for (let i = open; i < source.length; i++) {
|
|
54286
|
+
if (source[i] === "{")
|
|
54287
|
+
depth++;
|
|
54288
|
+
if (source[i] === "}") {
|
|
54289
|
+
depth--;
|
|
54290
|
+
if (depth === 0) {
|
|
54291
|
+
end = i;
|
|
54292
|
+
break;
|
|
54293
|
+
}
|
|
54395
54294
|
}
|
|
54396
54295
|
}
|
|
54296
|
+
if (end < 0)
|
|
54297
|
+
end = source.length - 1;
|
|
54298
|
+
const startLine = source.slice(0, m.index + (m[1] ? m[1].length : 0)).split("\n").length - 1;
|
|
54299
|
+
const endLine = source.slice(0, end).split("\n").length - 1;
|
|
54300
|
+
for (let l = startLine; l <= endLine; l++)
|
|
54301
|
+
lines.add(l);
|
|
54397
54302
|
}
|
|
54398
|
-
return
|
|
54399
|
-
}
|
|
54400
|
-
function dependencyBlocked(dependencyKeys, statusByKey) {
|
|
54401
|
-
return dependencyKeys.some(dep => {
|
|
54402
|
-
const status = statusByKey.get(dep);
|
|
54403
|
-
return status !== undefined && status !== "PASS";
|
|
54404
|
-
});
|
|
54303
|
+
return lines;
|
|
54405
54304
|
}
|
|
54406
|
-
const
|
|
54407
|
-
|
|
54408
|
-
|
|
54305
|
+
const str = (field, body) => {
|
|
54306
|
+
const re = new RegExp(`${field}\\s*:\\s*["'\`]([^"'\`]+)["'\`]`);
|
|
54307
|
+
const m = re.exec(body);
|
|
54308
|
+
return m ? m[1] : null;
|
|
54309
|
+
};
|
|
54310
|
+
function parseSqlxConfig(body) {
|
|
54311
|
+
let database = str("database", body);
|
|
54312
|
+
if (!database && /database\s*:\s*(dataform|sqlanvil)\.projectConfig\.defaultDatabase/.test(body)) {
|
|
54313
|
+
database = "<defaultDatabase>";
|
|
54314
|
+
}
|
|
54315
|
+
return {
|
|
54316
|
+
type: str("type", body),
|
|
54317
|
+
schema: str("schema", body),
|
|
54318
|
+
database,
|
|
54319
|
+
name: str("name", body),
|
|
54320
|
+
description: str("description", body),
|
|
54321
|
+
};
|
|
54409
54322
|
}
|
|
54410
|
-
|
|
54411
|
-
|
|
54412
|
-
|
|
54323
|
+
const GLOBAL_RULES = [
|
|
54324
|
+
{
|
|
54325
|
+
pattern: /\bdataform\.projectConfig\b/g,
|
|
54326
|
+
replacement: "sqlanvil.projectConfig",
|
|
54327
|
+
kind: "rewrite",
|
|
54328
|
+
note: "dataform.projectConfig → sqlanvil.projectConfig (compile global renamed)",
|
|
54329
|
+
},
|
|
54330
|
+
];
|
|
54331
|
+
const SQL_RULES = [
|
|
54332
|
+
{
|
|
54333
|
+
pattern: /\bSAFE_CAST\s*\(/g,
|
|
54334
|
+
replacement: "CAST(",
|
|
54335
|
+
kind: "rewrite",
|
|
54336
|
+
note: "SAFE_CAST → CAST — Postgres CAST raises on bad input; verify the data or guard it",
|
|
54337
|
+
},
|
|
54338
|
+
{
|
|
54339
|
+
pattern: /\bCURRENT_DATE\s*\(\s*\)/g,
|
|
54340
|
+
replacement: "CURRENT_DATE",
|
|
54341
|
+
kind: "rewrite",
|
|
54342
|
+
note: "CURRENT_DATE() → CURRENT_DATE",
|
|
54343
|
+
},
|
|
54344
|
+
{
|
|
54345
|
+
pattern: /\bCURRENT_DATE\s*\(\s*['"]/g,
|
|
54346
|
+
kind: "flag",
|
|
54347
|
+
note: "CURRENT_DATE('<tz>') — Postgres: (CURRENT_TIMESTAMP AT TIME ZONE '<tz>')::date",
|
|
54348
|
+
},
|
|
54349
|
+
{
|
|
54350
|
+
pattern: /\bSAFE_DIVIDE\s*\(/g,
|
|
54351
|
+
kind: "flag",
|
|
54352
|
+
note: "SAFE_DIVIDE(a, b) — Postgres: a / NULLIF(b, 0)",
|
|
54353
|
+
},
|
|
54354
|
+
{
|
|
54355
|
+
pattern: /\bSAFE_OFFSET\s*\(/g,
|
|
54356
|
+
kind: "flag",
|
|
54357
|
+
note: "array[SAFE_OFFSET(n)] — Postgres arrays are 1-based; use (split_part / arr[n+1])",
|
|
54358
|
+
},
|
|
54359
|
+
{ pattern: /\bQUALIFY\b/gi, kind: "flag", note: "QUALIFY — rewrite as a subquery/CTE filter on the window function" },
|
|
54360
|
+
{ pattern: /\bUNNEST\s*\(/g, kind: "flag", note: "UNNEST — Postgres unnest() works on arrays only (no STRUCT arrays); verify semantics" },
|
|
54361
|
+
{ pattern: /\bSTRUCT\s*[(<]/g, kind: "flag", note: "STRUCT — no Postgres equivalent; consider jsonb or a composite type" },
|
|
54362
|
+
{ pattern: /\bARRAY\s*</g, kind: "flag", note: "ARRAY<T> type syntax — Postgres uses T[]" },
|
|
54363
|
+
{ pattern: /\bFORMAT_DATE\s*\(/g, kind: "flag", note: "FORMAT_DATE → TO_CHAR (different format tokens)" },
|
|
54364
|
+
{ pattern: /\bPARSE_DATE\s*\(/g, kind: "flag", note: "PARSE_DATE → TO_DATE (different format tokens)" },
|
|
54365
|
+
{ pattern: /\bFORMAT_TIMESTAMP\s*\(/g, kind: "flag", note: "FORMAT_TIMESTAMP → TO_CHAR (different format tokens)" },
|
|
54366
|
+
{
|
|
54367
|
+
pattern: /\b(DATE_ADD|DATE_SUB|TIMESTAMP_ADD|TIMESTAMP_SUB|DATETIME_ADD|DATETIME_SUB)\s*\(/g,
|
|
54368
|
+
kind: "flag",
|
|
54369
|
+
note: "BigQuery date arithmetic — Postgres uses interval arithmetic (x + INTERVAL '1 day')",
|
|
54370
|
+
},
|
|
54371
|
+
{ pattern: /\bDATE_DIFF\s*\(/g, kind: "flag", note: "DATE_DIFF — Postgres: subtraction / EXTRACT(EPOCH FROM ...) depending on the part" },
|
|
54372
|
+
{ pattern: /_PARTITIONTIME|_PARTITIONDATE|_TABLE_SUFFIX/g, kind: "flag", note: "BigQuery pseudo-column has no Postgres equivalent" },
|
|
54373
|
+
{ pattern: /\bEXPORT\s+DATA\b/gi, kind: "flag", note: "EXPORT DATA — use a sqlanvil `type: \"export\"` action instead" },
|
|
54374
|
+
{ pattern: /\bEXECUTE\s+IMMEDIATE\b/gi, kind: "flag", note: "BigQuery scripting (EXECUTE IMMEDIATE) — rewrite as a PL/pgSQL DO block or operations" },
|
|
54375
|
+
{ pattern: /\bGENERATE_UUID\s*\(\s*\)/g, kind: "flag", note: "GENERATE_UUID() → gen_random_uuid()" },
|
|
54376
|
+
{
|
|
54377
|
+
pattern: /`[A-Za-z0-9_-]+\.[A-Za-z0-9_$]+\.[A-Za-z0-9_$]+`/g,
|
|
54378
|
+
kind: "flag",
|
|
54379
|
+
note: "backticked BigQuery FQN — use ${ref(...)} (declare it as a source) or a quoted Postgres name",
|
|
54380
|
+
},
|
|
54381
|
+
];
|
|
54382
|
+
const CONFIG_BQ_KEYS = /^(\s*)(bigquery\s*:|partitionBy\s*:|clusterBy\s*:|requirePartitionFilter\s*:|partitionExpirationDays\s*:|database\s*:)/;
|
|
54383
|
+
const sanitize = (s) => s.toLowerCase().replace(/[^a-z0-9_]/g, "_");
|
|
54384
|
+
const connectionNameFor = (project) => `bq_${sanitize(project)}`;
|
|
54385
|
+
function convertDeclaration(source, parsed, connection, sourceDataset, filePath) {
|
|
54386
|
+
const span = findConfigBlock(source);
|
|
54387
|
+
const description = parsed.description
|
|
54388
|
+
? ` description: ${JSON.stringify(parsed.description)},\n`
|
|
54389
|
+
: "";
|
|
54390
|
+
const newConfig = `config {
|
|
54391
|
+
type: "declaration",
|
|
54392
|
+
connection: ${JSON.stringify(connection)},
|
|
54393
|
+
schema: ${JSON.stringify(sourceDataset)},
|
|
54394
|
+
name: ${JSON.stringify(parsed.name)},
|
|
54395
|
+
${description} columnTypes: {
|
|
54396
|
+
// SQLANVIL-MIGRATE(TODO): scaffold the columns from the live source:
|
|
54397
|
+
// sqlanvil introspect ${connection} ${sourceDataset}.${parsed.name} --output ${filePath}
|
|
54398
|
+
},
|
|
54399
|
+
}`;
|
|
54400
|
+
const content = source.slice(0, span.start) + newConfig + source.slice(span.end);
|
|
54401
|
+
return {
|
|
54402
|
+
content,
|
|
54403
|
+
findings: [
|
|
54404
|
+
{
|
|
54405
|
+
line: 1,
|
|
54406
|
+
kind: "rewrite",
|
|
54407
|
+
note: `declaration now reads BigQuery through connection "${connection}" (runner-extract); columnTypes need introspect`,
|
|
54408
|
+
},
|
|
54409
|
+
],
|
|
54410
|
+
};
|
|
54413
54411
|
}
|
|
54414
|
-
function
|
|
54415
|
-
|
|
54416
|
-
|
|
54417
|
-
|
|
54418
|
-
|
|
54419
|
-
|
|
54420
|
-
|
|
54421
|
-
|
|
54422
|
-
|
|
54423
|
-
|
|
54424
|
-
|
|
54425
|
-
|
|
54426
|
-
|
|
54427
|
-
|
|
54412
|
+
function convertTarget(source, jsMode) {
|
|
54413
|
+
const span = jsMode ? null : findConfigBlock(source);
|
|
54414
|
+
const findings = [];
|
|
54415
|
+
const lines = source.split("\n");
|
|
54416
|
+
const jsLines = jsMode ? new Set() : jsBlockLines(source);
|
|
54417
|
+
let configStartLine = -1;
|
|
54418
|
+
let configEndLine = -1;
|
|
54419
|
+
if (span) {
|
|
54420
|
+
configStartLine = source.slice(0, span.start).split("\n").length - 1;
|
|
54421
|
+
configEndLine = source.slice(0, span.end).split("\n").length - 1;
|
|
54422
|
+
}
|
|
54423
|
+
const out = [];
|
|
54424
|
+
let commentingConfigBlockDepth = 0;
|
|
54425
|
+
let swallowLeadingComma = false;
|
|
54426
|
+
for (let i = 0; i < lines.length; i++) {
|
|
54427
|
+
let line = lines[i];
|
|
54428
|
+
const inConfig = i >= configStartLine && i <= configEndLine && configStartLine >= 0;
|
|
54429
|
+
const markersSafe = !jsMode && !jsLines.has(i);
|
|
54430
|
+
const marker = (note) => inConfig ? `// SQLANVIL-MIGRATE: ${note}` : `-- SQLANVIL-MIGRATE: ${note}`;
|
|
54431
|
+
const lineNo = i + 1;
|
|
54432
|
+
const notesForLine = [];
|
|
54433
|
+
for (const rule of GLOBAL_RULES) {
|
|
54434
|
+
rule.pattern.lastIndex = 0;
|
|
54435
|
+
if (rule.pattern.test(line)) {
|
|
54436
|
+
line = line.replace(rule.pattern, rule.replacement);
|
|
54437
|
+
findings.push({ line: lineNo, kind: "rewrite", note: rule.note });
|
|
54438
|
+
}
|
|
54439
|
+
}
|
|
54440
|
+
if (swallowLeadingComma) {
|
|
54441
|
+
swallowLeadingComma = false;
|
|
54442
|
+
if (/^\s*,\s*$/.test(line)) {
|
|
54443
|
+
out.push(`// ${line}`);
|
|
54444
|
+
continue;
|
|
54428
54445
|
}
|
|
54429
|
-
|
|
54446
|
+
if (/^\s*,/.test(line)) {
|
|
54447
|
+
line = line.replace(/,/, " ");
|
|
54448
|
+
}
|
|
54449
|
+
}
|
|
54450
|
+
if (inConfig && commentingConfigBlockDepth === 0) {
|
|
54451
|
+
const m = CONFIG_BQ_KEYS.exec(line);
|
|
54452
|
+
if (m) {
|
|
54453
|
+
const note = `BigQuery-only setting — not applicable on Postgres (partitioning/clustering → postgres: {}; project qualifiers drop)`;
|
|
54454
|
+
findings.push({ line: lineNo, kind: "flag", note });
|
|
54455
|
+
out.push(marker(note));
|
|
54456
|
+
const opens = (line.match(/\{/g) || []).length;
|
|
54457
|
+
const closes = (line.match(/\}/g) || []).length;
|
|
54458
|
+
commentingConfigBlockDepth = Math.max(0, opens - closes);
|
|
54459
|
+
if (commentingConfigBlockDepth === 0)
|
|
54460
|
+
swallowLeadingComma = !/,\s*$/.test(line) ? true : false;
|
|
54461
|
+
out.push(`// ${line}`);
|
|
54462
|
+
continue;
|
|
54463
|
+
}
|
|
54464
|
+
}
|
|
54465
|
+
else if (commentingConfigBlockDepth > 0) {
|
|
54466
|
+
const opens = (line.match(/\{/g) || []).length;
|
|
54467
|
+
const closes = (line.match(/\}/g) || []).length;
|
|
54468
|
+
commentingConfigBlockDepth = Math.max(0, commentingConfigBlockDepth + opens - closes);
|
|
54469
|
+
if (commentingConfigBlockDepth === 0 && !/,\s*$/.test(line))
|
|
54470
|
+
swallowLeadingComma = true;
|
|
54471
|
+
out.push(`// ${line}`);
|
|
54472
|
+
continue;
|
|
54473
|
+
}
|
|
54474
|
+
if (!inConfig || jsMode) {
|
|
54475
|
+
for (const rule of SQL_RULES) {
|
|
54476
|
+
rule.pattern.lastIndex = 0;
|
|
54477
|
+
if (!rule.pattern.test(line))
|
|
54478
|
+
continue;
|
|
54479
|
+
if (rule.replacement !== undefined) {
|
|
54480
|
+
rule.pattern.lastIndex = 0;
|
|
54481
|
+
line = line.replace(rule.pattern, rule.replacement);
|
|
54482
|
+
findings.push({ line: lineNo, kind: "rewrite", note: rule.note });
|
|
54483
|
+
if (markersSafe)
|
|
54484
|
+
notesForLine.push(rule.note);
|
|
54485
|
+
}
|
|
54486
|
+
else {
|
|
54487
|
+
findings.push({ line: lineNo, kind: "flag", note: rule.note });
|
|
54488
|
+
if (markersSafe)
|
|
54489
|
+
notesForLine.push(rule.note);
|
|
54490
|
+
}
|
|
54430
54491
|
}
|
|
54431
54492
|
}
|
|
54493
|
+
for (const note of notesForLine) {
|
|
54494
|
+
out.push(marker(note));
|
|
54495
|
+
}
|
|
54496
|
+
out.push(line);
|
|
54432
54497
|
}
|
|
54433
|
-
|
|
54498
|
+
let content = out.join("\n");
|
|
54499
|
+
if (jsMode && findings.length > 0) {
|
|
54500
|
+
const noted = findings.slice(0, 12).map(f => ` * L${f.line}: ${f.note}`);
|
|
54501
|
+
const more = findings.length > 12 ? [` * …and ${findings.length - 12} more (see migration-report.json)`] : [];
|
|
54502
|
+
content =
|
|
54503
|
+
["/* SQLANVIL-MIGRATE — findings in this file (inline markers are unsafe in JS templates):", ...noted, ...more, " */"].join("\n") +
|
|
54504
|
+
"\n" +
|
|
54505
|
+
content;
|
|
54434
54506
|
}
|
|
54507
|
+
return { content, findings };
|
|
54435
54508
|
}
|
|
54436
|
-
function
|
|
54437
|
-
|
|
54438
|
-
|
|
54439
|
-
|
|
54440
|
-
|
|
54441
|
-
|
|
54442
|
-
|
|
54443
|
-
return "table";
|
|
54509
|
+
function readSourceConfig(srcDir) {
|
|
54510
|
+
var _a, _b, _c, _d, _e;
|
|
54511
|
+
const wsPath = path__namespace.join(srcDir, "workflow_settings.yaml");
|
|
54512
|
+
const djPath = path__namespace.join(srcDir, "dataform.json");
|
|
54513
|
+
let raw;
|
|
54514
|
+
if (fs__namespace$1.existsSync(wsPath)) {
|
|
54515
|
+
raw = jsYaml.load(fs__namespace$1.readFileSync(wsPath, "utf8")) || {};
|
|
54444
54516
|
}
|
|
54445
|
-
|
|
54446
|
-
|
|
54447
|
-
return (deps || []).map(targetKey);
|
|
54448
|
-
}
|
|
54449
|
-
async function validate(compiledGraph, deps, options = {}) {
|
|
54450
|
-
const nodes = [];
|
|
54451
|
-
for (const table of compiledGraph.tables || []) {
|
|
54452
|
-
nodes.push({
|
|
54453
|
-
key: targetKey(table.target),
|
|
54454
|
-
dependencyKeys: depKeys(table.dependencyTargets),
|
|
54455
|
-
kind: "table",
|
|
54456
|
-
type: tableType(table.enumType),
|
|
54457
|
-
target: table.target,
|
|
54458
|
-
table,
|
|
54459
|
-
action: table
|
|
54460
|
-
});
|
|
54517
|
+
else if (fs__namespace$1.existsSync(djPath)) {
|
|
54518
|
+
raw = JSON.parse(fs__namespace$1.readFileSync(djPath, "utf8"));
|
|
54461
54519
|
}
|
|
54462
|
-
|
|
54463
|
-
|
|
54464
|
-
key: targetKey(assertion.target),
|
|
54465
|
-
dependencyKeys: depKeys(assertion.dependencyTargets),
|
|
54466
|
-
kind: "assertion",
|
|
54467
|
-
type: "assertion",
|
|
54468
|
-
target: assertion.target,
|
|
54469
|
-
action: assertion
|
|
54470
|
-
});
|
|
54520
|
+
else {
|
|
54521
|
+
throw new Error(`${srcDir} does not look like a Dataform project (no workflow_settings.yaml or dataform.json).`);
|
|
54471
54522
|
}
|
|
54472
|
-
|
|
54473
|
-
|
|
54523
|
+
const s = (k) => (typeof raw[k] === "string" ? raw[k] : null);
|
|
54524
|
+
return {
|
|
54525
|
+
defaultProject: (_a = s("defaultProject")) !== null && _a !== void 0 ? _a : s("defaultDatabase"),
|
|
54526
|
+
defaultDataset: (_c = (_b = s("defaultDataset")) !== null && _b !== void 0 ? _b : s("defaultSchema")) !== null && _c !== void 0 ? _c : "dataform",
|
|
54527
|
+
defaultAssertionDataset: (_d = s("defaultAssertionDataset")) !== null && _d !== void 0 ? _d : s("assertionSchema"),
|
|
54528
|
+
vars: (_e = raw.vars) !== null && _e !== void 0 ? _e : null,
|
|
54529
|
+
raw,
|
|
54530
|
+
};
|
|
54531
|
+
}
|
|
54532
|
+
async function migrateDataform(opts) {
|
|
54533
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
54534
|
+
const srcDir = path__namespace.resolve(opts.srcDir);
|
|
54535
|
+
const outDir = path__namespace.resolve(opts.outDir);
|
|
54536
|
+
if (containsPath(srcDir, outDir)) {
|
|
54537
|
+
throw new Error("The output directory must not be inside the source project.");
|
|
54538
|
+
}
|
|
54539
|
+
if (containsPath(outDir, srcDir)) {
|
|
54540
|
+
throw new Error("The source project must not be inside the output directory.");
|
|
54541
|
+
}
|
|
54542
|
+
if (fs__namespace$1.existsSync(outDir) && fs__namespace$1.readdirSync(outDir).length > 0) {
|
|
54543
|
+
throw new Error(`Output directory ${outDir} is not empty.`);
|
|
54544
|
+
}
|
|
54545
|
+
const sourceConfig = readSourceConfig(srcDir);
|
|
54546
|
+
const writer = new OutWriter(outDir);
|
|
54547
|
+
skippedForSafety = new Set();
|
|
54548
|
+
const files = walkFiles(srcDir);
|
|
54549
|
+
const report = {
|
|
54550
|
+
generator: `sqlanvil migrate-dataform ${version}`,
|
|
54551
|
+
sourceDir: srcDir,
|
|
54552
|
+
sourceConfig: sourceConfig.raw,
|
|
54553
|
+
inventory: { sqlxFiles: 0, jsFiles: 0, byType: {} },
|
|
54554
|
+
connections: [],
|
|
54555
|
+
overlappingSchemas: [],
|
|
54556
|
+
files: [],
|
|
54557
|
+
skippedForSafety: [],
|
|
54558
|
+
warnings: [],
|
|
54559
|
+
};
|
|
54560
|
+
const sqlxFiles = [];
|
|
54561
|
+
for (const rel of files) {
|
|
54562
|
+
if (!rel.endsWith(".sqlx"))
|
|
54474
54563
|
continue;
|
|
54564
|
+
const source = fs__namespace$1.readFileSync(path__namespace.join(srcDir, rel), "utf8");
|
|
54565
|
+
const span = findConfigBlock(source);
|
|
54566
|
+
const parsed = span
|
|
54567
|
+
? parseSqlxConfig(span.body)
|
|
54568
|
+
: { type: null, schema: null, database: null, name: null, description: null };
|
|
54569
|
+
sqlxFiles.push({ rel, source, parsed });
|
|
54570
|
+
report.inventory.sqlxFiles++;
|
|
54571
|
+
const t = (_a = parsed.type) !== null && _a !== void 0 ? _a : "(none)";
|
|
54572
|
+
report.inventory.byType[t] = ((_b = report.inventory.byType[t]) !== null && _b !== void 0 ? _b : 0) + 1;
|
|
54573
|
+
}
|
|
54574
|
+
const resolveDatabase = (db) => { var _a; return !db || db === "<defaultDatabase>" ? (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "UNKNOWN_PROJECT" : db; };
|
|
54575
|
+
const declarationSchemas = new Set();
|
|
54576
|
+
const targetSchemas = new Set();
|
|
54577
|
+
const connectionMap = new Map();
|
|
54578
|
+
for (const f of sqlxFiles) {
|
|
54579
|
+
if (f.parsed.type === "declaration") {
|
|
54580
|
+
const dataset = (_c = f.parsed.schema) !== null && _c !== void 0 ? _c : sourceConfig.defaultDataset;
|
|
54581
|
+
const project = resolveDatabase(f.parsed.database);
|
|
54582
|
+
declarationSchemas.add(dataset);
|
|
54583
|
+
const name = connectionNameFor(project);
|
|
54584
|
+
const existing = connectionMap.get(name);
|
|
54585
|
+
if (existing) {
|
|
54586
|
+
existing.declarationCount++;
|
|
54587
|
+
if (!existing.datasets.includes(dataset))
|
|
54588
|
+
existing.datasets.push(dataset);
|
|
54589
|
+
}
|
|
54590
|
+
else {
|
|
54591
|
+
connectionMap.set(name, {
|
|
54592
|
+
name,
|
|
54593
|
+
project,
|
|
54594
|
+
datasets: [dataset],
|
|
54595
|
+
declarationCount: 1,
|
|
54596
|
+
introspectExample: `sqlanvil introspect ${name} <dataset>.<table> --output <declaration.sqlx>`,
|
|
54597
|
+
});
|
|
54598
|
+
}
|
|
54475
54599
|
}
|
|
54476
|
-
|
|
54477
|
-
|
|
54478
|
-
dependencyKeys: depKeys(operation.dependencyTargets),
|
|
54479
|
-
kind: "operation",
|
|
54480
|
-
type: "operation",
|
|
54481
|
-
target: operation.target,
|
|
54482
|
-
action: undefined
|
|
54483
|
-
});
|
|
54484
|
-
}
|
|
54485
|
-
for (const imp of compiledGraph.imports || []) {
|
|
54486
|
-
if (!imp.target) {
|
|
54487
|
-
continue;
|
|
54600
|
+
else if (f.parsed.type) {
|
|
54601
|
+
targetSchemas.add((_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset);
|
|
54488
54602
|
}
|
|
54489
|
-
nodes.push({
|
|
54490
|
-
key: targetKey(imp.target),
|
|
54491
|
-
dependencyKeys: depKeys(imp.dependencyTargets),
|
|
54492
|
-
kind: "import",
|
|
54493
|
-
type: "import",
|
|
54494
|
-
target: imp.target,
|
|
54495
|
-
action: undefined
|
|
54496
|
-
});
|
|
54497
54603
|
}
|
|
54498
|
-
|
|
54499
|
-
|
|
54604
|
+
report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
54605
|
+
report.overlappingSchemas = [...declarationSchemas].filter(s => targetSchemas.has(s)).sort();
|
|
54606
|
+
for (const rel of files) {
|
|
54607
|
+
const abs = path__namespace.join(srcDir, rel);
|
|
54608
|
+
if (rel === "workflow_settings.yaml" || rel === "dataform.json") {
|
|
54500
54609
|
continue;
|
|
54501
54610
|
}
|
|
54502
|
-
|
|
54503
|
-
|
|
54504
|
-
|
|
54505
|
-
|
|
54506
|
-
|
|
54507
|
-
|
|
54508
|
-
|
|
54509
|
-
|
|
54510
|
-
|
|
54511
|
-
|
|
54512
|
-
|
|
54513
|
-
|
|
54514
|
-
|
|
54515
|
-
|
|
54516
|
-
|
|
54517
|
-
|
|
54518
|
-
await deps.execute(deps.createSchemaSql(schema));
|
|
54519
|
-
}
|
|
54520
|
-
for (const node of ordered) {
|
|
54521
|
-
if (node.kind === "operation" ||
|
|
54522
|
-
node.kind === "import" ||
|
|
54523
|
-
(node.kind === "script" && !deps.checkScript)) {
|
|
54524
|
-
statusByKey.set(node.key, "SKIPPED");
|
|
54525
|
-
results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
|
|
54526
|
-
continue;
|
|
54527
|
-
}
|
|
54528
|
-
if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
|
|
54529
|
-
statusByKey.set(node.key, "BLOCKED");
|
|
54530
|
-
results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
|
|
54531
|
-
continue;
|
|
54532
|
-
}
|
|
54533
|
-
const evaluations = node.kind === "script"
|
|
54534
|
-
? await deps.checkScript(node.script)
|
|
54535
|
-
: await deps.evaluate(node.action);
|
|
54536
|
-
const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
|
|
54537
|
-
const status = failed ? "FAILURE" : "PASS";
|
|
54538
|
-
statusByKey.set(node.key, status);
|
|
54539
|
-
results.push({ target: node.target, type: node.type, status, errors: evaluations });
|
|
54540
|
-
if (status === "PASS" && node.kind === "table") {
|
|
54541
|
-
try {
|
|
54542
|
-
await deps.execute(deps.validationStubSql(node.table));
|
|
54543
|
-
}
|
|
54544
|
-
catch (e) {
|
|
54545
|
-
}
|
|
54611
|
+
if (rel.endsWith(".sqlx")) {
|
|
54612
|
+
const info = sqlxFiles.find(f => f.rel === rel);
|
|
54613
|
+
if (info.parsed.type === "declaration") {
|
|
54614
|
+
const dataset = (_e = info.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset;
|
|
54615
|
+
const project = resolveDatabase(info.parsed.database);
|
|
54616
|
+
const connection = connectionNameFor(project);
|
|
54617
|
+
const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
|
|
54618
|
+
writer.write(rel, content);
|
|
54619
|
+
report.files.push({
|
|
54620
|
+
file: rel,
|
|
54621
|
+
action: "declaration",
|
|
54622
|
+
type: "declaration",
|
|
54623
|
+
status: "rewritten",
|
|
54624
|
+
findings,
|
|
54625
|
+
connection,
|
|
54626
|
+
});
|
|
54546
54627
|
}
|
|
54547
|
-
|
|
54548
|
-
|
|
54549
|
-
|
|
54550
|
-
|
|
54551
|
-
|
|
54552
|
-
|
|
54553
|
-
|
|
54554
|
-
|
|
54555
|
-
|
|
54556
|
-
|
|
54628
|
+
else {
|
|
54629
|
+
const { content, findings } = convertTarget(info.source, false);
|
|
54630
|
+
writer.write(rel, content);
|
|
54631
|
+
report.files.push({
|
|
54632
|
+
file: rel,
|
|
54633
|
+
action: "target",
|
|
54634
|
+
type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
|
|
54635
|
+
status: findings.length === 0
|
|
54636
|
+
? "clean"
|
|
54637
|
+
: findings.some(f => f.kind === "flag")
|
|
54638
|
+
? "flagged"
|
|
54639
|
+
: "rewritten",
|
|
54640
|
+
findings,
|
|
54641
|
+
});
|
|
54557
54642
|
}
|
|
54643
|
+
continue;
|
|
54558
54644
|
}
|
|
54559
|
-
|
|
54560
|
-
|
|
54561
|
-
|
|
54562
|
-
|
|
54563
|
-
|
|
54564
|
-
|
|
54565
|
-
|
|
54566
|
-
|
|
54567
|
-
|
|
54568
|
-
|
|
54569
|
-
|
|
54570
|
-
|
|
54571
|
-
|
|
54572
|
-
|
|
54573
|
-
|
|
54574
|
-
|
|
54575
|
-
|
|
54576
|
-
|
|
54577
|
-
|
|
54578
|
-
|
|
54579
|
-
|
|
54580
|
-
try {
|
|
54581
|
-
await queryExecutor.verifyConnection();
|
|
54582
|
-
}
|
|
54583
|
-
catch (e) {
|
|
54584
|
-
await queryExecutor.close().catch(() => undefined);
|
|
54585
|
-
throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
|
|
54586
|
-
`as "${credentials.user}": ${e.message}`, e);
|
|
54645
|
+
if (rel.endsWith(".js") && (rel.startsWith("definitions/") || rel.startsWith("includes/"))) {
|
|
54646
|
+
report.inventory.jsFiles++;
|
|
54647
|
+
const source = fs__namespace$1.readFileSync(abs, "utf8");
|
|
54648
|
+
const { content, findings } = convertTarget(source, true);
|
|
54649
|
+
writer.write(rel, content);
|
|
54650
|
+
const isIncludes = rel.startsWith("includes/");
|
|
54651
|
+
if (isIncludes && findings.length > 0) {
|
|
54652
|
+
report.warnings.push(`includes helper ${rel} emits BigQuery-flavored SQL — review its ${findings.length} finding(s)`);
|
|
54653
|
+
}
|
|
54654
|
+
report.files.push({
|
|
54655
|
+
file: rel,
|
|
54656
|
+
action: isIncludes ? "includes" : "target",
|
|
54657
|
+
type: "js",
|
|
54658
|
+
status: findings.length === 0
|
|
54659
|
+
? "clean"
|
|
54660
|
+
: findings.some(f => f.kind === "flag")
|
|
54661
|
+
? "flagged"
|
|
54662
|
+
: "rewritten",
|
|
54663
|
+
findings,
|
|
54664
|
+
});
|
|
54665
|
+
continue;
|
|
54587
54666
|
}
|
|
54588
|
-
|
|
54667
|
+
writer.write(rel, fs__namespace$1.readFileSync(abs));
|
|
54668
|
+
report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
|
|
54589
54669
|
}
|
|
54590
|
-
|
|
54591
|
-
|
|
54592
|
-
|
|
54593
|
-
|
|
54594
|
-
|
|
54595
|
-
|
|
54596
|
-
|
|
54597
|
-
|
|
54598
|
-
|
|
54599
|
-
|
|
54600
|
-
|
|
54601
|
-
return
|
|
54602
|
-
|
|
54670
|
+
const settings = {
|
|
54671
|
+
warehouse: "supabase",
|
|
54672
|
+
defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
|
|
54673
|
+
defaultAssertionDataset: ((_g = sourceConfig.defaultAssertionDataset) !== null && _g !== void 0 ? _g : "sqlanvil_assertions").toLowerCase(),
|
|
54674
|
+
sqlanvilCoreVersion: (_h = opts.coreVersion) !== null && _h !== void 0 ? _h : version,
|
|
54675
|
+
};
|
|
54676
|
+
if (sourceConfig.vars)
|
|
54677
|
+
settings.vars = sourceConfig.vars;
|
|
54678
|
+
if (report.connections.length > 0) {
|
|
54679
|
+
settings.connections = Object.fromEntries(report.connections.map(c => {
|
|
54680
|
+
var _a;
|
|
54681
|
+
return [
|
|
54682
|
+
c.name,
|
|
54683
|
+
{
|
|
54684
|
+
platform: "bigquery",
|
|
54685
|
+
project: c.project,
|
|
54686
|
+
billingProject: (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "REPLACE_WITH_YOUR_GCP_PROJECT",
|
|
54687
|
+
mode: "runner-extract",
|
|
54688
|
+
},
|
|
54689
|
+
];
|
|
54690
|
+
}));
|
|
54603
54691
|
}
|
|
54604
|
-
|
|
54605
|
-
|
|
54606
|
-
|
|
54607
|
-
|
|
54608
|
-
});
|
|
54609
|
-
return result.rows.length > 0;
|
|
54610
|
-
}
|
|
54611
|
-
catch (_a) {
|
|
54612
|
-
return false;
|
|
54613
|
-
}
|
|
54692
|
+
writer.write("workflow_settings.yaml", jsYaml.dump(settings));
|
|
54693
|
+
report.skippedForSafety = [...skippedForSafety].sort();
|
|
54694
|
+
if (skippedForSafety.size > 0) {
|
|
54695
|
+
report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
|
|
54614
54696
|
}
|
|
54697
|
+
writer.write("migration-report.json", JSON.stringify(report, null, 2));
|
|
54698
|
+
writer.write("migration-report.md", renderReportMd(report));
|
|
54699
|
+
return report;
|
|
54700
|
+
}
|
|
54701
|
+
function renderReportMd(r) {
|
|
54702
|
+
const targets = r.files.filter(f => f.action === "target");
|
|
54703
|
+
const flagged = targets.filter(f => f.status === "flagged");
|
|
54704
|
+
const rewritten = targets.filter(f => f.status === "rewritten");
|
|
54705
|
+
const clean = targets.filter(f => f.status === "clean");
|
|
54706
|
+
const declarations = r.files.filter(f => f.action === "declaration");
|
|
54707
|
+
const lines = [];
|
|
54708
|
+
lines.push(`# Dataform → SQLAnvil migration report`);
|
|
54709
|
+
lines.push("");
|
|
54710
|
+
lines.push(`Generated by ${r.generator} from \`${r.sourceDir}\` (source untouched).`);
|
|
54711
|
+
lines.push("");
|
|
54712
|
+
lines.push(`## Inventory`);
|
|
54713
|
+
lines.push("");
|
|
54714
|
+
lines.push(`- ${r.inventory.sqlxFiles} .sqlx, ${r.inventory.jsFiles} .js`);
|
|
54715
|
+
for (const [t, n] of Object.entries(r.inventory.byType).sort((a, b) => b[1] - a[1])) {
|
|
54716
|
+
lines.push(` - ${t}: ${n}`);
|
|
54717
|
+
}
|
|
54718
|
+
lines.push("");
|
|
54719
|
+
lines.push(`## Source connections (${r.connections.length})`);
|
|
54720
|
+
lines.push("");
|
|
54721
|
+
lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
|
|
54722
|
+
`(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
|
|
54723
|
+
`before it can extract; scaffold them with \`sqlanvil introspect\`:`);
|
|
54724
|
+
lines.push("");
|
|
54725
|
+
for (const c of r.connections) {
|
|
54726
|
+
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54727
|
+
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54728
|
+
(c.datasets.length > 8 ? ", …" : ""));
|
|
54729
|
+
lines.push(` - \`${c.introspectExample}\``);
|
|
54730
|
+
}
|
|
54731
|
+
if (r.overlappingSchemas.length > 0) {
|
|
54732
|
+
lines.push("");
|
|
54733
|
+
lines.push(`## ⚠ Schemas that are BOTH source and target`);
|
|
54734
|
+
lines.push("");
|
|
54735
|
+
lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
|
|
54736
|
+
.map(s => `\`${s}\``)
|
|
54737
|
+
.join(", ")}. ` +
|
|
54738
|
+
`Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
|
|
54739
|
+
`producers move to Postgres, convert the declarations to plain local declarations instead ` +
|
|
54740
|
+
`of BigQuery connections.`);
|
|
54741
|
+
}
|
|
54742
|
+
lines.push("");
|
|
54743
|
+
lines.push(`## Target files (${targets.length})`);
|
|
54744
|
+
lines.push("");
|
|
54745
|
+
lines.push(`- clean: ${clean.length}`);
|
|
54746
|
+
lines.push(`- rewritten (safe lexical changes only): ${rewritten.length}`);
|
|
54747
|
+
lines.push(`- **flagged (need dialect review): ${flagged.length}** — every location is marked ` +
|
|
54748
|
+
`inline with \`SQLANVIL-MIGRATE:\``);
|
|
54749
|
+
lines.push("");
|
|
54750
|
+
for (const f of flagged) {
|
|
54751
|
+
lines.push(`- \`${f.file}\` (${f.type})`);
|
|
54752
|
+
for (const finding of f.findings.filter(x => x.kind === "flag").slice(0, 6)) {
|
|
54753
|
+
lines.push(` - L${finding.line}: ${finding.note}`);
|
|
54754
|
+
}
|
|
54755
|
+
const more = f.findings.filter(x => x.kind === "flag").length - 6;
|
|
54756
|
+
if (more > 0)
|
|
54757
|
+
lines.push(` - …and ${more} more (see migration-report.json)`);
|
|
54758
|
+
}
|
|
54759
|
+
lines.push("");
|
|
54760
|
+
lines.push(`## Declarations (${declarations.length})`);
|
|
54761
|
+
lines.push("");
|
|
54762
|
+
lines.push(`Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
|
|
54763
|
+
if (r.warnings.length > 0) {
|
|
54764
|
+
lines.push("");
|
|
54765
|
+
lines.push(`## Warnings`);
|
|
54766
|
+
lines.push("");
|
|
54767
|
+
for (const w of r.warnings)
|
|
54768
|
+
lines.push(`- ${w}`);
|
|
54769
|
+
}
|
|
54770
|
+
lines.push("");
|
|
54771
|
+
lines.push(`## Finish the migration`);
|
|
54772
|
+
lines.push("");
|
|
54773
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54774
|
+
lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
|
|
54775
|
+
lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
|
|
54776
|
+
`model is the migration to-do list. Work the flagged files until validate is green.`);
|
|
54777
|
+
lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
|
|
54778
|
+
`the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
|
|
54779
|
+
lines.push("");
|
|
54780
|
+
return lines.join("\n");
|
|
54615
54781
|
}
|
|
54616
54782
|
|
|
54617
54783
|
function actuallyResolve(...filePaths) {
|
|
@@ -54819,6 +54985,20 @@ function getTestInput(questionText) {
|
|
|
54819
54985
|
throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
|
|
54820
54986
|
}
|
|
54821
54987
|
}
|
|
54988
|
+
function passwordQuestion(questionText) {
|
|
54989
|
+
return prompt(questionText, {
|
|
54990
|
+
hideEchoBack: true,
|
|
54991
|
+
mask: ""
|
|
54992
|
+
});
|
|
54993
|
+
}
|
|
54994
|
+
function interactivePasswordQuestion(questionText) {
|
|
54995
|
+
if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
|
|
54996
|
+
const testInput = getTestInput(questionText);
|
|
54997
|
+
print(`${questionText} ********\n`);
|
|
54998
|
+
return testInput;
|
|
54999
|
+
}
|
|
55000
|
+
return passwordQuestion(questionText);
|
|
55001
|
+
}
|
|
54822
55002
|
function selectionQuestion(questionText, options) {
|
|
54823
55003
|
return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
|
|
54824
55004
|
cancel: false
|
|
@@ -54961,162 +55141,809 @@ function printExecutionGraph(executionGraph, asJson) {
|
|
|
54961
55141
|
operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
|
|
54962
55142
|
}
|
|
54963
55143
|
}
|
|
54964
|
-
}
|
|
54965
|
-
function printExecutedAction(executedAction, executionAction, dryRun) {
|
|
54966
|
-
const jobIds = executedAction.tasks
|
|
54967
|
-
.filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
|
|
54968
|
-
.map(task => task.metadata.bigquery.jobId);
|
|
54969
|
-
const bytesBilled = executedAction.tasks
|
|
54970
|
-
.filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
|
|
54971
|
-
.map(task => {
|
|
54972
|
-
var _a, _b, _c, _d;
|
|
54973
|
-
const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
|
|
54974
|
-
return formatBytesInHumanReadableFormat(bytes);
|
|
54975
|
-
});
|
|
54976
|
-
const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
|
|
54977
|
-
switch (executedAction.status) {
|
|
54978
|
-
case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
|
|
54979
|
-
switch (executionAction.type) {
|
|
54980
|
-
case "table": {
|
|
54981
|
-
writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
54982
|
-
return;
|
|
54983
|
-
}
|
|
54984
|
-
case "assertion": {
|
|
54985
|
-
writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
54986
|
-
return;
|
|
54987
|
-
}
|
|
54988
|
-
case "operation": {
|
|
54989
|
-
writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
54990
|
-
return;
|
|
54991
|
-
}
|
|
54992
|
-
}
|
|
55144
|
+
}
|
|
55145
|
+
function printExecutedAction(executedAction, executionAction, dryRun) {
|
|
55146
|
+
const jobIds = executedAction.tasks
|
|
55147
|
+
.filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
|
|
55148
|
+
.map(task => task.metadata.bigquery.jobId);
|
|
55149
|
+
const bytesBilled = executedAction.tasks
|
|
55150
|
+
.filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
|
|
55151
|
+
.map(task => {
|
|
55152
|
+
var _a, _b, _c, _d;
|
|
55153
|
+
const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
|
|
55154
|
+
return formatBytesInHumanReadableFormat(bytes);
|
|
55155
|
+
});
|
|
55156
|
+
const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
|
|
55157
|
+
switch (executedAction.status) {
|
|
55158
|
+
case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
|
|
55159
|
+
switch (executionAction.type) {
|
|
55160
|
+
case "table": {
|
|
55161
|
+
writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55162
|
+
return;
|
|
55163
|
+
}
|
|
55164
|
+
case "assertion": {
|
|
55165
|
+
writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55166
|
+
return;
|
|
55167
|
+
}
|
|
55168
|
+
case "operation": {
|
|
55169
|
+
writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55170
|
+
return;
|
|
55171
|
+
}
|
|
55172
|
+
}
|
|
55173
|
+
}
|
|
55174
|
+
case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
|
|
55175
|
+
switch (executionAction.type) {
|
|
55176
|
+
case "table": {
|
|
55177
|
+
writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55178
|
+
break;
|
|
55179
|
+
}
|
|
55180
|
+
case "assertion": {
|
|
55181
|
+
writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55182
|
+
break;
|
|
55183
|
+
}
|
|
55184
|
+
case "operation": {
|
|
55185
|
+
writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55186
|
+
break;
|
|
55187
|
+
}
|
|
55188
|
+
}
|
|
55189
|
+
printExecutedActionErrors(executedAction, executionAction);
|
|
55190
|
+
return;
|
|
55191
|
+
}
|
|
55192
|
+
case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
|
|
55193
|
+
switch (executionAction.type) {
|
|
55194
|
+
case "table": {
|
|
55195
|
+
writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
|
|
55196
|
+
return;
|
|
55197
|
+
}
|
|
55198
|
+
case "assertion": {
|
|
55199
|
+
writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55200
|
+
return;
|
|
55201
|
+
}
|
|
55202
|
+
case "operation": {
|
|
55203
|
+
writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55204
|
+
return;
|
|
55205
|
+
}
|
|
55206
|
+
}
|
|
55207
|
+
return;
|
|
55208
|
+
}
|
|
55209
|
+
case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
|
|
55210
|
+
switch (executionAction.type) {
|
|
55211
|
+
case "table": {
|
|
55212
|
+
writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
|
|
55213
|
+
return;
|
|
55214
|
+
}
|
|
55215
|
+
case "assertion": {
|
|
55216
|
+
writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55217
|
+
return;
|
|
55218
|
+
}
|
|
55219
|
+
case "operation": {
|
|
55220
|
+
writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55221
|
+
return;
|
|
55222
|
+
}
|
|
55223
|
+
}
|
|
55224
|
+
return;
|
|
55225
|
+
}
|
|
55226
|
+
}
|
|
55227
|
+
}
|
|
55228
|
+
function printFormatFilesResult(formatResults) {
|
|
55229
|
+
const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
|
|
55230
|
+
const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
|
|
55231
|
+
const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
|
|
55232
|
+
const failedFormatResults = sorted.filter(result => !!result.err);
|
|
55233
|
+
if (successfulFormatResults.length > 0) {
|
|
55234
|
+
printSuccess("Successfully formatted:");
|
|
55235
|
+
successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
|
|
55236
|
+
}
|
|
55237
|
+
if (needsFormattingResults.length > 0) {
|
|
55238
|
+
printError("Files that need formatting:");
|
|
55239
|
+
needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
|
|
55240
|
+
}
|
|
55241
|
+
if (failedFormatResults.length > 0) {
|
|
55242
|
+
printError("Errors encountered during formatting:");
|
|
55243
|
+
failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
|
|
55244
|
+
}
|
|
55245
|
+
}
|
|
55246
|
+
function datasetString(target, datasetType, disabled) {
|
|
55247
|
+
return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
|
|
55248
|
+
}
|
|
55249
|
+
function assertionString(target, disabled) {
|
|
55250
|
+
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55251
|
+
}
|
|
55252
|
+
function operationString(target, disabled) {
|
|
55253
|
+
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55254
|
+
}
|
|
55255
|
+
function plainTargetString(target) {
|
|
55256
|
+
return `${target.schema}.${target.name}`;
|
|
55257
|
+
}
|
|
55258
|
+
function targetString(target) {
|
|
55259
|
+
return calloutOutput(`${target.schema}.${target.name}`);
|
|
55260
|
+
}
|
|
55261
|
+
function dotRepresentation(graph, interactive) {
|
|
55262
|
+
var _a, _b, _c;
|
|
55263
|
+
const nodes = [];
|
|
55264
|
+
const edges = [];
|
|
55265
|
+
const formatTarget = interactive ? targetString : plainTargetString;
|
|
55266
|
+
(_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
|
|
55267
|
+
var _a;
|
|
55268
|
+
const nodeName = `${formatTarget(table.target)}`;
|
|
55269
|
+
nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
|
|
55270
|
+
(_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55271
|
+
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55272
|
+
});
|
|
55273
|
+
});
|
|
55274
|
+
(_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
|
|
55275
|
+
var _a;
|
|
55276
|
+
const nodeName = `${formatTarget(assertion.target)}`;
|
|
55277
|
+
nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
|
|
55278
|
+
(_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55279
|
+
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55280
|
+
});
|
|
55281
|
+
});
|
|
55282
|
+
(_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
|
|
55283
|
+
var _a;
|
|
55284
|
+
const nodeName = `${formatTarget(operation.target)}`;
|
|
55285
|
+
nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
|
|
55286
|
+
(_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55287
|
+
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55288
|
+
});
|
|
55289
|
+
});
|
|
55290
|
+
return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
|
|
55291
|
+
}
|
|
55292
|
+
function printExecutedActionErrors(executedAction, executionAction) {
|
|
55293
|
+
const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
|
|
55294
|
+
failingTasks.forEach((task, i) => {
|
|
55295
|
+
executionAction.tasks[i].statement.split("\n").forEach(line => {
|
|
55296
|
+
writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
|
|
55297
|
+
});
|
|
55298
|
+
printError(task.errorMessage, 1);
|
|
55299
|
+
});
|
|
55300
|
+
}
|
|
55301
|
+
|
|
55302
|
+
const INIT_MODE_QUESTION = "Start a fresh project, or convert an existing Dataform project? (fresh/convert)";
|
|
55303
|
+
const INIT_WAREHOUSE_QUESTION = "Which warehouse? (supabase/postgres/bigquery/mysql)";
|
|
55304
|
+
const INIT_PROJECT_DIR_QUESTION = "Project directory?";
|
|
55305
|
+
const INIT_BQ_PROJECT_QUESTION = "Google Cloud project ID (the default project models build in)?";
|
|
55306
|
+
const INIT_BQ_LOCATION_QUESTION = "Default BigQuery location (e.g. US, us-central1)?";
|
|
55307
|
+
const INIT_DEFAULT_SCHEMA_QUESTION = "Default schema for models?";
|
|
55308
|
+
const INIT_INCLUDE_SAMPLE_QUESTION = "Include the sample project (sources -> staging views -> outputs -> assertion)?";
|
|
55309
|
+
const INIT_INCLUDE_BQ_SOURCE_QUESTION = "Include the cross-warehouse BigQuery sample source (Google public ZIP data)?";
|
|
55310
|
+
const INIT_CONFIGURE_CREDS_QUESTION = `Configure warehouse credentials now (written to the gitignored ${CREDENTIALS_FILENAME})?`;
|
|
55311
|
+
const CREDS_HOST_QUESTION = "Host?";
|
|
55312
|
+
const CREDS_PORT_QUESTION = "Port?";
|
|
55313
|
+
const CREDS_DATABASE_QUESTION = "Database?";
|
|
55314
|
+
const CREDS_USER_QUESTION = "User?";
|
|
55315
|
+
const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
|
|
55316
|
+
const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
|
|
55317
|
+
const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
|
|
55318
|
+
const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
|
|
55319
|
+
const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
|
|
55320
|
+
"aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
|
|
55321
|
+
"db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
|
|
55322
|
+
const MAX_ATTEMPTS = 20;
|
|
55323
|
+
function withDefault(question, defaultValue) {
|
|
55324
|
+
return defaultValue ? `${question} [${defaultValue}]` : question;
|
|
55325
|
+
}
|
|
55326
|
+
function ask(question, defaultValue) {
|
|
55327
|
+
const answer = interactiveQuestion(withDefault(question, defaultValue)).trim();
|
|
55328
|
+
return answer === "" ? defaultValue !== null && defaultValue !== void 0 ? defaultValue : "" : answer;
|
|
55329
|
+
}
|
|
55330
|
+
function askRequired(question) {
|
|
55331
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55332
|
+
const answer = ask(question);
|
|
55333
|
+
if (answer !== "") {
|
|
55334
|
+
return answer;
|
|
55335
|
+
}
|
|
55336
|
+
printError("A value is required.");
|
|
55337
|
+
}
|
|
55338
|
+
throw new Error(`No answer provided for: ${question}`);
|
|
55339
|
+
}
|
|
55340
|
+
function askChoice(question, choices, defaultValue) {
|
|
55341
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55342
|
+
const answer = ask(question, defaultValue).toLowerCase();
|
|
55343
|
+
if (choices.includes(answer)) {
|
|
55344
|
+
return answer;
|
|
55345
|
+
}
|
|
55346
|
+
printError(`Please answer one of: ${choices.join(", ")}.`);
|
|
55347
|
+
}
|
|
55348
|
+
throw new Error(`No valid answer provided for: ${question}`);
|
|
55349
|
+
}
|
|
55350
|
+
function askYesNo(question, defaultValue) {
|
|
55351
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55352
|
+
const answer = ask(question, defaultValue ? "y" : "n").toLowerCase();
|
|
55353
|
+
if (answer === "y" || answer === "yes") {
|
|
55354
|
+
return true;
|
|
55355
|
+
}
|
|
55356
|
+
if (answer === "n" || answer === "no") {
|
|
55357
|
+
return false;
|
|
55358
|
+
}
|
|
55359
|
+
printError("Please answer y or n.");
|
|
55360
|
+
}
|
|
55361
|
+
throw new Error(`No valid answer provided for: ${question}`);
|
|
55362
|
+
}
|
|
55363
|
+
function askInt(question, defaultValue) {
|
|
55364
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55365
|
+
const value = parseInt(ask(question, String(defaultValue)), 10);
|
|
55366
|
+
if (!isNaN(value)) {
|
|
55367
|
+
return value;
|
|
55368
|
+
}
|
|
55369
|
+
printError("Entered value must be an integer.");
|
|
55370
|
+
}
|
|
55371
|
+
throw new Error(`No valid answer provided for: ${question}`);
|
|
55372
|
+
}
|
|
55373
|
+
function collectCredentials(warehouse) {
|
|
55374
|
+
if (warehouse === "mysql") {
|
|
55375
|
+
const creds = {
|
|
55376
|
+
host: ask(CREDS_HOST_QUESTION, "localhost"),
|
|
55377
|
+
port: askInt(CREDS_PORT_QUESTION, 3306),
|
|
55378
|
+
database: ask(CREDS_DATABASE_QUESTION, "sqlanvil"),
|
|
55379
|
+
user: ask(CREDS_USER_QUESTION, "root"),
|
|
55380
|
+
password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
|
|
55381
|
+
sslMode: ask(CREDS_SSLMODE_QUESTION, "disable")
|
|
55382
|
+
};
|
|
55383
|
+
return `${JSON.stringify(creds, null, 2)}\n`;
|
|
55384
|
+
}
|
|
55385
|
+
const isSupabase = warehouse === "supabase";
|
|
55386
|
+
if (isSupabase) {
|
|
55387
|
+
print(SUPABASE_POOLER_HINT);
|
|
55388
|
+
}
|
|
55389
|
+
const host = ask(CREDS_HOST_QUESTION, isSupabase ? "aws-1-<region>.pooler.supabase.com" : "localhost");
|
|
55390
|
+
if (/^db\..+\.supabase\.co$/.test(host)) {
|
|
55391
|
+
printError("Warning: that is the DIRECT Supabase host, which is IPv6-only — connections fail with " +
|
|
55392
|
+
"ENOTFOUND on most networks. Prefer the Session pooler host (Dashboard -> Connect).");
|
|
55393
|
+
}
|
|
55394
|
+
const creds = {
|
|
55395
|
+
host,
|
|
55396
|
+
port: askInt(CREDS_PORT_QUESTION, 5432),
|
|
55397
|
+
database: ask(CREDS_DATABASE_QUESTION, "postgres"),
|
|
55398
|
+
user: ask(CREDS_USER_QUESTION, isSupabase ? "postgres.<your-project-ref>" : "postgres"),
|
|
55399
|
+
password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
|
|
55400
|
+
sslMode: ask(CREDS_SSLMODE_QUESTION, isSupabase ? "require" : "disable"),
|
|
55401
|
+
defaultSchema: "public"
|
|
55402
|
+
};
|
|
55403
|
+
return `${JSON.stringify(creds, null, 2)}\n`;
|
|
55404
|
+
}
|
|
55405
|
+
function printMigrationSummary(report, outDir) {
|
|
55406
|
+
const targets = report.files.filter(f => f.action === "target");
|
|
55407
|
+
const flagged = targets.filter(f => f.status === "flagged").length;
|
|
55408
|
+
printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
|
|
55409
|
+
`${report.connections.length} source connection(s) over ` +
|
|
55410
|
+
`${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
|
|
55411
|
+
`${targets.length} target file(s), ${flagged} flagged for dialect review.`);
|
|
55412
|
+
print(`Report: ${path__namespace.join(outDir, "migration-report.md")}`);
|
|
55413
|
+
}
|
|
55414
|
+
function ensureGitignoreCoversCredentials(dir) {
|
|
55415
|
+
const gitignorePath = path__namespace.join(dir, ".gitignore");
|
|
55416
|
+
const existing = fs__namespace$1.existsSync(gitignorePath) ? fs__namespace$1.readFileSync(gitignorePath, "utf8") : "";
|
|
55417
|
+
if (existing.includes(".df-credentials")) {
|
|
55418
|
+
return;
|
|
55419
|
+
}
|
|
55420
|
+
const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
|
|
55421
|
+
fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
|
|
55422
|
+
}
|
|
55423
|
+
async function runConvertFlow() {
|
|
55424
|
+
let srcDir = "";
|
|
55425
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55426
|
+
const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
|
|
55427
|
+
if (fs__namespace$1.existsSync(candidate)) {
|
|
55428
|
+
srcDir = candidate;
|
|
55429
|
+
break;
|
|
55430
|
+
}
|
|
55431
|
+
printError(`${candidate} does not exist.`);
|
|
55432
|
+
}
|
|
55433
|
+
if (!srcDir) {
|
|
55434
|
+
throw new Error("No usable source directory provided.");
|
|
55435
|
+
}
|
|
55436
|
+
const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
|
|
55437
|
+
print("\nConverting...\n");
|
|
55438
|
+
const report = await migrateDataform({ srcDir, outDir });
|
|
55439
|
+
printMigrationSummary(report, outDir);
|
|
55440
|
+
if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
|
|
55441
|
+
const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
|
|
55442
|
+
ensureGitignoreCoversCredentials(outDir);
|
|
55443
|
+
fs__namespace$1.writeFileSync(credentialsPath, collectCredentials("supabase"));
|
|
55444
|
+
printSuccess(`Credentials written to ${credentialsPath}`);
|
|
55445
|
+
}
|
|
55446
|
+
print("\nNext steps:");
|
|
55447
|
+
print(` 1. sqlanvil compile ${outDir}`);
|
|
55448
|
+
print(` 2. Review ${path__namespace.join(outDir, "migration-report.md")} — BigQuery source connections ` +
|
|
55449
|
+
`need columnTypes (the introspect commands are in the report).`);
|
|
55450
|
+
print(` 3. sqlanvil validate ${outDir} (PASS/FAILURE/BLOCKED per model = the migration to-do list)`);
|
|
55451
|
+
return 0;
|
|
55452
|
+
}
|
|
55453
|
+
async function runFreshFlow(defaultProjectDir) {
|
|
55454
|
+
const warehouse = askChoice(INIT_WAREHOUSE_QUESTION, ["supabase", "postgres", "bigquery", "mysql"], "supabase");
|
|
55455
|
+
const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
|
|
55456
|
+
const projectDir = actuallyResolve(ask(INIT_PROJECT_DIR_QUESTION, defaultProjectDir));
|
|
55457
|
+
const projectConfig = { warehouse };
|
|
55458
|
+
if (warehouse === "bigquery") {
|
|
55459
|
+
projectConfig.defaultDatabase = askRequired(INIT_BQ_PROJECT_QUESTION);
|
|
55460
|
+
projectConfig.defaultLocation = askRequired(INIT_BQ_LOCATION_QUESTION);
|
|
55461
|
+
}
|
|
55462
|
+
projectConfig.defaultSchema = ask(INIT_DEFAULT_SCHEMA_QUESTION, isPostgresLike ? "public" : "sqlanvil");
|
|
55463
|
+
const includeSample = askYesNo(INIT_INCLUDE_SAMPLE_QUESTION, true);
|
|
55464
|
+
const includeBigQuerySource = includeSample && isPostgresLike ? askYesNo(INIT_INCLUDE_BQ_SOURCE_QUESTION, true) : undefined;
|
|
55465
|
+
let credentialsJson;
|
|
55466
|
+
if (warehouse !== "bigquery" && askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
|
|
55467
|
+
credentialsJson = collectCredentials(warehouse);
|
|
55468
|
+
}
|
|
55469
|
+
print("\nWriting project files...\n");
|
|
55470
|
+
const result = await init(projectDir, projectConfig, {
|
|
55471
|
+
includeSample,
|
|
55472
|
+
includeBigQuerySource,
|
|
55473
|
+
credentialsJson
|
|
55474
|
+
});
|
|
55475
|
+
printInitResult(result);
|
|
55476
|
+
const steps = [];
|
|
55477
|
+
if (warehouse === "bigquery") {
|
|
55478
|
+
steps.push(`sqlanvil init-creds ${projectDir} (BigQuery credentials)`);
|
|
55479
|
+
}
|
|
55480
|
+
else if (!credentialsJson) {
|
|
55481
|
+
steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
|
|
55482
|
+
}
|
|
55483
|
+
if (includeSample) {
|
|
55484
|
+
steps.push("Point definitions/sources/app_orders.sqlx at a real table in your warehouse.");
|
|
55485
|
+
}
|
|
55486
|
+
if (isPostgresLike && includeBigQuerySource) {
|
|
55487
|
+
steps.push("Replace REPLACE_WITH_YOUR_GCP_PROJECT in workflow_settings.yaml (billingProject for " +
|
|
55488
|
+
"the BigQuery sample source).");
|
|
55489
|
+
}
|
|
55490
|
+
steps.push(`sqlanvil compile ${projectDir}`);
|
|
55491
|
+
steps.push(`sqlanvil validate ${projectDir} (checks every model against the warehouse without executing)`);
|
|
55492
|
+
print("\nNext steps:");
|
|
55493
|
+
steps.forEach((step, index) => print(` ${index + 1}. ${step}`));
|
|
55494
|
+
return 0;
|
|
55495
|
+
}
|
|
55496
|
+
async function runInteractiveInit(defaultProjectDir) {
|
|
55497
|
+
if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS === undefined) {
|
|
55498
|
+
printError("init --interactive needs an interactive terminal (stdin is not a TTY). " +
|
|
55499
|
+
"Use the non-interactive form instead: sqlanvil init [project-dir] --warehouse=... " +
|
|
55500
|
+
'(see "sqlanvil help init").');
|
|
55501
|
+
return 1;
|
|
55502
|
+
}
|
|
55503
|
+
const mode = askChoice(INIT_MODE_QUESTION, ["fresh", "convert"], "fresh");
|
|
55504
|
+
if (mode === "convert") {
|
|
55505
|
+
return runConvertFlow();
|
|
55506
|
+
}
|
|
55507
|
+
return runFreshFlow(defaultProjectDir);
|
|
55508
|
+
}
|
|
55509
|
+
|
|
55510
|
+
const CHECKER_SOURCE = `
|
|
55511
|
+
import json, os, sys
|
|
55512
|
+
|
|
55513
|
+
script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
55514
|
+
errors = []
|
|
55515
|
+
|
|
55516
|
+
import platform
|
|
55517
|
+
py_version = platform.python_version()
|
|
55518
|
+
|
|
55519
|
+
def load_packaging():
|
|
55520
|
+
try:
|
|
55521
|
+
from packaging.specifiers import SpecifierSet
|
|
55522
|
+
from packaging.requirements import Requirement
|
|
55523
|
+
return SpecifierSet, Requirement
|
|
55524
|
+
except ImportError:
|
|
55525
|
+
pass
|
|
55526
|
+
try:
|
|
55527
|
+
from pip._vendor.packaging.specifiers import SpecifierSet
|
|
55528
|
+
from pip._vendor.packaging.requirements import Requirement
|
|
55529
|
+
return SpecifierSet, Requirement
|
|
55530
|
+
except ImportError:
|
|
55531
|
+
return None, None
|
|
55532
|
+
|
|
55533
|
+
SpecifierSet, Requirement = load_packaging()
|
|
55534
|
+
|
|
55535
|
+
def mini_parse(v):
|
|
55536
|
+
import re
|
|
55537
|
+
return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
|
|
55538
|
+
|
|
55539
|
+
def mini_check(version, spec):
|
|
55540
|
+
import re
|
|
55541
|
+
ok = True
|
|
55542
|
+
for clause in spec.split(","):
|
|
55543
|
+
clause = clause.strip()
|
|
55544
|
+
m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
|
|
55545
|
+
if not m:
|
|
55546
|
+
return None
|
|
55547
|
+
op, want = m.group(1), m.group(2).strip()
|
|
55548
|
+
if want.endswith(".*"):
|
|
55549
|
+
prefix = mini_parse(want[:-2])
|
|
55550
|
+
have = mini_parse(version)[:len(prefix)]
|
|
55551
|
+
eq = have == prefix
|
|
55552
|
+
if (op in ("==", "===") and not eq) or (op == "!=" and eq):
|
|
55553
|
+
ok = False
|
|
55554
|
+
continue
|
|
55555
|
+
a, b = mini_parse(version), mini_parse(want)
|
|
55556
|
+
n = max(len(a), len(b))
|
|
55557
|
+
a = a + [0] * (n - len(a))
|
|
55558
|
+
b = b + [0] * (n - len(b))
|
|
55559
|
+
if op in ("==", "==="):
|
|
55560
|
+
res = a == b
|
|
55561
|
+
elif op == "!=":
|
|
55562
|
+
res = a != b
|
|
55563
|
+
elif op == "<=":
|
|
55564
|
+
res = a <= b
|
|
55565
|
+
elif op == ">=":
|
|
55566
|
+
res = a >= b
|
|
55567
|
+
elif op == "<":
|
|
55568
|
+
res = a < b
|
|
55569
|
+
elif op == ">":
|
|
55570
|
+
res = a > b
|
|
55571
|
+
else: # ~= : at least b, within b's series
|
|
55572
|
+
wb = mini_parse(want)
|
|
55573
|
+
res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
|
|
55574
|
+
if not res:
|
|
55575
|
+
ok = False
|
|
55576
|
+
return ok
|
|
55577
|
+
|
|
55578
|
+
def check_version(version, spec, what):
|
|
55579
|
+
if SpecifierSet is not None:
|
|
55580
|
+
try:
|
|
55581
|
+
if version not in SpecifierSet(spec, prereleases=True):
|
|
55582
|
+
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
55583
|
+
return
|
|
55584
|
+
except Exception as e:
|
|
55585
|
+
errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
|
|
55586
|
+
return
|
|
55587
|
+
if mini_check(version, spec) is False:
|
|
55588
|
+
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
55589
|
+
|
|
55590
|
+
if version_spec:
|
|
55591
|
+
check_version(py_version, version_spec, "python interpreter")
|
|
55592
|
+
|
|
55593
|
+
if requirements_path:
|
|
55594
|
+
from importlib import metadata
|
|
55595
|
+
try:
|
|
55596
|
+
lines = open(requirements_path).read().splitlines()
|
|
55597
|
+
except OSError as e:
|
|
55598
|
+
errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
|
|
55599
|
+
lines = []
|
|
55600
|
+
for raw in lines:
|
|
55601
|
+
line = raw.split("#")[0].strip()
|
|
55602
|
+
if not line or line.startswith("-"):
|
|
55603
|
+
continue
|
|
55604
|
+
if Requirement is not None:
|
|
55605
|
+
try:
|
|
55606
|
+
req = Requirement(line)
|
|
55607
|
+
except Exception:
|
|
55608
|
+
errors.append('requirements: cannot parse "%s"' % line)
|
|
55609
|
+
continue
|
|
55610
|
+
if req.marker is not None and not req.marker.evaluate():
|
|
55611
|
+
continue
|
|
55612
|
+
name, spec = req.name, str(req.specifier)
|
|
55613
|
+
else:
|
|
55614
|
+
import re
|
|
55615
|
+
if ";" in line:
|
|
55616
|
+
line = line.split(";")[0].strip()
|
|
55617
|
+
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
|
|
55618
|
+
if not m:
|
|
55619
|
+
errors.append('requirements: cannot parse "%s"' % line)
|
|
55620
|
+
continue
|
|
55621
|
+
name, spec = m.group(1), (m.group(3) or "").strip()
|
|
55622
|
+
try:
|
|
55623
|
+
installed = metadata.version(name)
|
|
55624
|
+
except metadata.PackageNotFoundError:
|
|
55625
|
+
errors.append(
|
|
55626
|
+
"requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
|
|
55627
|
+
)
|
|
55628
|
+
continue
|
|
55629
|
+
if spec:
|
|
55630
|
+
check_version(installed, spec, "requirements: %s" % name)
|
|
55631
|
+
|
|
55632
|
+
try:
|
|
55633
|
+
with open(script_path, "rb") as f:
|
|
55634
|
+
compile(f.read(), script_path, "exec")
|
|
55635
|
+
except SyntaxError as e:
|
|
55636
|
+
errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
|
|
55637
|
+
except (OSError, ValueError) as e:
|
|
55638
|
+
errors.append("syntax: cannot compile %s: %s" % (script_path, e))
|
|
55639
|
+
|
|
55640
|
+
print(json.dumps({"interpreter_version": py_version, "errors": errors}))
|
|
55641
|
+
`;
|
|
55642
|
+
const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
|
|
55643
|
+
childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
|
|
55644
|
+
if (err) {
|
|
55645
|
+
reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
|
|
55646
|
+
}
|
|
55647
|
+
else {
|
|
55648
|
+
resolve(stdout);
|
|
55649
|
+
}
|
|
55650
|
+
});
|
|
55651
|
+
});
|
|
55652
|
+
const failure = (message) => ({
|
|
55653
|
+
status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
|
|
55654
|
+
error: { message }
|
|
55655
|
+
});
|
|
55656
|
+
async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
|
|
55657
|
+
if (script.language !== "python") {
|
|
55658
|
+
return [failure(`no validator for script language "${script.language}"`)];
|
|
55659
|
+
}
|
|
55660
|
+
let interpreter;
|
|
55661
|
+
try {
|
|
55662
|
+
interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
|
|
55663
|
+
}
|
|
55664
|
+
catch (e) {
|
|
55665
|
+
return [failure(e.message)];
|
|
55666
|
+
}
|
|
55667
|
+
let stdout;
|
|
55668
|
+
try {
|
|
55669
|
+
stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
|
|
55670
|
+
}
|
|
55671
|
+
catch (e) {
|
|
55672
|
+
return [failure(e.message)];
|
|
55673
|
+
}
|
|
55674
|
+
let parsed;
|
|
55675
|
+
try {
|
|
55676
|
+
parsed = JSON.parse(stdout);
|
|
55677
|
+
}
|
|
55678
|
+
catch (e) {
|
|
55679
|
+
return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
|
|
55680
|
+
}
|
|
55681
|
+
if (parsed.errors && parsed.errors.length > 0) {
|
|
55682
|
+
return parsed.errors.map(failure);
|
|
55683
|
+
}
|
|
55684
|
+
return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
|
|
55685
|
+
}
|
|
55686
|
+
|
|
55687
|
+
function targetKey(target) {
|
|
55688
|
+
return [target.database, target.schema, target.name].filter(Boolean).join(".");
|
|
55689
|
+
}
|
|
55690
|
+
function topoOrder(nodes) {
|
|
55691
|
+
const byKey = new Map(nodes.map(n => [n.key, n]));
|
|
55692
|
+
const indegree = new Map();
|
|
55693
|
+
const dependents = new Map();
|
|
55694
|
+
for (const n of nodes) {
|
|
55695
|
+
const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
|
|
55696
|
+
indegree.set(n.key, inGraphDeps.length);
|
|
55697
|
+
for (const dep of inGraphDeps) {
|
|
55698
|
+
dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
|
|
55699
|
+
}
|
|
55700
|
+
}
|
|
55701
|
+
const ready = nodes
|
|
55702
|
+
.filter(n => (indegree.get(n.key) || 0) === 0)
|
|
55703
|
+
.map(n => n.key)
|
|
55704
|
+
.sort();
|
|
55705
|
+
const ordered = [];
|
|
55706
|
+
const emitted = new Set();
|
|
55707
|
+
while (ready.length) {
|
|
55708
|
+
const key = ready.shift();
|
|
55709
|
+
if (emitted.has(key)) {
|
|
55710
|
+
continue;
|
|
55711
|
+
}
|
|
55712
|
+
emitted.add(key);
|
|
55713
|
+
ordered.push(byKey.get(key));
|
|
55714
|
+
for (const dep of (dependents.get(key) || []).slice().sort()) {
|
|
55715
|
+
indegree.set(dep, (indegree.get(dep) || 0) - 1);
|
|
55716
|
+
if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
|
|
55717
|
+
ready.push(dep);
|
|
55718
|
+
ready.sort();
|
|
55719
|
+
}
|
|
55720
|
+
}
|
|
55721
|
+
}
|
|
55722
|
+
if (ordered.length < nodes.length) {
|
|
55723
|
+
for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
|
|
55724
|
+
if (!emitted.has(n.key)) {
|
|
55725
|
+
emitted.add(n.key);
|
|
55726
|
+
ordered.push(n);
|
|
55727
|
+
}
|
|
55728
|
+
}
|
|
55729
|
+
}
|
|
55730
|
+
return ordered;
|
|
55731
|
+
}
|
|
55732
|
+
function dependencyBlocked(dependencyKeys, statusByKey) {
|
|
55733
|
+
return dependencyKeys.some(dep => {
|
|
55734
|
+
const status = statusByKey.get(dep);
|
|
55735
|
+
return status !== undefined && status !== "PASS";
|
|
55736
|
+
});
|
|
55737
|
+
}
|
|
55738
|
+
const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
|
|
55739
|
+
function validateShadowSuffix(nowMs) {
|
|
55740
|
+
return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
|
|
55741
|
+
}
|
|
55742
|
+
function parseShadowTimestamp(schemaName) {
|
|
55743
|
+
const match = schemaName.match(/sqlanvil_validate_(\d+)/);
|
|
55744
|
+
return match ? Number(match[1]) : null;
|
|
55745
|
+
}
|
|
55746
|
+
function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
|
|
55747
|
+
return schemaNames.filter(name => {
|
|
55748
|
+
const ts = parseShadowTimestamp(name);
|
|
55749
|
+
return ts !== null && nowMs - ts > maxAgeMs;
|
|
55750
|
+
});
|
|
55751
|
+
}
|
|
55752
|
+
|
|
55753
|
+
const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
|
|
55754
|
+
async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
|
|
55755
|
+
try {
|
|
55756
|
+
const schemas = await deps.listSchemas();
|
|
55757
|
+
for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
|
|
55758
|
+
try {
|
|
55759
|
+
await deps.execute(deps.dropSchemaCascadeSql(schema));
|
|
55760
|
+
}
|
|
55761
|
+
catch (e) {
|
|
55762
|
+
}
|
|
55763
|
+
}
|
|
55764
|
+
}
|
|
55765
|
+
catch (e) {
|
|
55766
|
+
}
|
|
55767
|
+
}
|
|
55768
|
+
function tableType(enumType) {
|
|
55769
|
+
switch (enumType) {
|
|
55770
|
+
case sqlanvil.TableType.VIEW:
|
|
55771
|
+
return "view";
|
|
55772
|
+
case sqlanvil.TableType.INCREMENTAL:
|
|
55773
|
+
return "incremental";
|
|
55774
|
+
default:
|
|
55775
|
+
return "table";
|
|
55776
|
+
}
|
|
55777
|
+
}
|
|
55778
|
+
function depKeys(deps) {
|
|
55779
|
+
return (deps || []).map(targetKey);
|
|
55780
|
+
}
|
|
55781
|
+
async function validate(compiledGraph, deps, options = {}) {
|
|
55782
|
+
const nodes = [];
|
|
55783
|
+
for (const table of compiledGraph.tables || []) {
|
|
55784
|
+
nodes.push({
|
|
55785
|
+
key: targetKey(table.target),
|
|
55786
|
+
dependencyKeys: depKeys(table.dependencyTargets),
|
|
55787
|
+
kind: "table",
|
|
55788
|
+
type: tableType(table.enumType),
|
|
55789
|
+
target: table.target,
|
|
55790
|
+
table,
|
|
55791
|
+
action: table
|
|
55792
|
+
});
|
|
55793
|
+
}
|
|
55794
|
+
for (const assertion of compiledGraph.assertions || []) {
|
|
55795
|
+
nodes.push({
|
|
55796
|
+
key: targetKey(assertion.target),
|
|
55797
|
+
dependencyKeys: depKeys(assertion.dependencyTargets),
|
|
55798
|
+
kind: "assertion",
|
|
55799
|
+
type: "assertion",
|
|
55800
|
+
target: assertion.target,
|
|
55801
|
+
action: assertion
|
|
55802
|
+
});
|
|
55803
|
+
}
|
|
55804
|
+
for (const operation of compiledGraph.operations || []) {
|
|
55805
|
+
if (!operation.target) {
|
|
55806
|
+
continue;
|
|
55807
|
+
}
|
|
55808
|
+
nodes.push({
|
|
55809
|
+
key: targetKey(operation.target),
|
|
55810
|
+
dependencyKeys: depKeys(operation.dependencyTargets),
|
|
55811
|
+
kind: "operation",
|
|
55812
|
+
type: "operation",
|
|
55813
|
+
target: operation.target,
|
|
55814
|
+
action: undefined
|
|
55815
|
+
});
|
|
55816
|
+
}
|
|
55817
|
+
for (const imp of compiledGraph.imports || []) {
|
|
55818
|
+
if (!imp.target) {
|
|
55819
|
+
continue;
|
|
55820
|
+
}
|
|
55821
|
+
nodes.push({
|
|
55822
|
+
key: targetKey(imp.target),
|
|
55823
|
+
dependencyKeys: depKeys(imp.dependencyTargets),
|
|
55824
|
+
kind: "import",
|
|
55825
|
+
type: "import",
|
|
55826
|
+
target: imp.target,
|
|
55827
|
+
action: undefined
|
|
55828
|
+
});
|
|
55829
|
+
}
|
|
55830
|
+
for (const script of compiledGraph.scripts || []) {
|
|
55831
|
+
if (!script.target) {
|
|
55832
|
+
continue;
|
|
55833
|
+
}
|
|
55834
|
+
nodes.push({
|
|
55835
|
+
key: targetKey(script.target),
|
|
55836
|
+
dependencyKeys: depKeys(script.dependencyTargets),
|
|
55837
|
+
kind: "script",
|
|
55838
|
+
type: "script",
|
|
55839
|
+
target: script.target,
|
|
55840
|
+
script,
|
|
55841
|
+
action: undefined
|
|
55842
|
+
});
|
|
55843
|
+
}
|
|
55844
|
+
const ordered = topoOrder(nodes);
|
|
55845
|
+
const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
|
|
55846
|
+
const statusByKey = new Map();
|
|
55847
|
+
const results = [];
|
|
55848
|
+
try {
|
|
55849
|
+
for (const schema of shadowSchemas) {
|
|
55850
|
+
await deps.execute(deps.createSchemaSql(schema));
|
|
54993
55851
|
}
|
|
54994
|
-
|
|
54995
|
-
|
|
54996
|
-
|
|
54997
|
-
|
|
54998
|
-
|
|
54999
|
-
}
|
|
55000
|
-
|
|
55001
|
-
writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55002
|
-
break;
|
|
55003
|
-
}
|
|
55004
|
-
case "operation": {
|
|
55005
|
-
writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55006
|
-
break;
|
|
55007
|
-
}
|
|
55852
|
+
for (const node of ordered) {
|
|
55853
|
+
if (node.kind === "operation" ||
|
|
55854
|
+
node.kind === "import" ||
|
|
55855
|
+
(node.kind === "script" && !deps.checkScript)) {
|
|
55856
|
+
statusByKey.set(node.key, "SKIPPED");
|
|
55857
|
+
results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
|
|
55858
|
+
continue;
|
|
55008
55859
|
}
|
|
55009
|
-
|
|
55010
|
-
|
|
55011
|
-
|
|
55012
|
-
|
|
55013
|
-
|
|
55014
|
-
|
|
55015
|
-
|
|
55016
|
-
|
|
55017
|
-
|
|
55018
|
-
|
|
55019
|
-
|
|
55020
|
-
|
|
55860
|
+
if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
|
|
55861
|
+
statusByKey.set(node.key, "BLOCKED");
|
|
55862
|
+
results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
|
|
55863
|
+
continue;
|
|
55864
|
+
}
|
|
55865
|
+
const evaluations = node.kind === "script"
|
|
55866
|
+
? await deps.checkScript(node.script)
|
|
55867
|
+
: await deps.evaluate(node.action);
|
|
55868
|
+
const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
|
|
55869
|
+
const status = failed ? "FAILURE" : "PASS";
|
|
55870
|
+
statusByKey.set(node.key, status);
|
|
55871
|
+
results.push({ target: node.target, type: node.type, status, errors: evaluations });
|
|
55872
|
+
if (status === "PASS" && node.kind === "table") {
|
|
55873
|
+
try {
|
|
55874
|
+
await deps.execute(deps.validationStubSql(node.table));
|
|
55021
55875
|
}
|
|
55022
|
-
|
|
55023
|
-
writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55024
|
-
return;
|
|
55876
|
+
catch (e) {
|
|
55025
55877
|
}
|
|
55026
55878
|
}
|
|
55027
|
-
return;
|
|
55028
55879
|
}
|
|
55029
|
-
|
|
55030
|
-
|
|
55031
|
-
|
|
55032
|
-
|
|
55033
|
-
|
|
55034
|
-
|
|
55035
|
-
case "assertion": {
|
|
55036
|
-
writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55037
|
-
return;
|
|
55880
|
+
}
|
|
55881
|
+
finally {
|
|
55882
|
+
if (!options.keepShadow) {
|
|
55883
|
+
for (const schema of shadowSchemas.slice().reverse()) {
|
|
55884
|
+
try {
|
|
55885
|
+
await deps.execute(deps.dropSchemaCascadeSql(schema));
|
|
55038
55886
|
}
|
|
55039
|
-
|
|
55040
|
-
writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55041
|
-
return;
|
|
55887
|
+
catch (e) {
|
|
55042
55888
|
}
|
|
55043
55889
|
}
|
|
55044
|
-
return;
|
|
55045
55890
|
}
|
|
55046
55891
|
}
|
|
55892
|
+
return results;
|
|
55047
55893
|
}
|
|
55048
|
-
|
|
55049
|
-
|
|
55050
|
-
|
|
55051
|
-
|
|
55052
|
-
|
|
55053
|
-
|
|
55054
|
-
|
|
55055
|
-
|
|
55056
|
-
|
|
55057
|
-
|
|
55058
|
-
|
|
55059
|
-
|
|
55894
|
+
|
|
55895
|
+
class SupabaseDbAdapter extends PostgresDbAdapter {
|
|
55896
|
+
static async create(credentials, options) {
|
|
55897
|
+
const sslMode = (credentials.sslMode || "").toLowerCase();
|
|
55898
|
+
const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
|
|
55899
|
+
const clientConfig = {
|
|
55900
|
+
host: credentials.host,
|
|
55901
|
+
port: credentials.port,
|
|
55902
|
+
database: credentials.database,
|
|
55903
|
+
user: credentials.user,
|
|
55904
|
+
password: credentials.password,
|
|
55905
|
+
ssl: sslEnabled
|
|
55906
|
+
? {
|
|
55907
|
+
rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
|
|
55908
|
+
}
|
|
55909
|
+
: false
|
|
55910
|
+
};
|
|
55911
|
+
const queryExecutor = new PgPoolExecutor(clientConfig, options);
|
|
55912
|
+
try {
|
|
55913
|
+
await queryExecutor.verifyConnection();
|
|
55914
|
+
}
|
|
55915
|
+
catch (e) {
|
|
55916
|
+
await queryExecutor.close().catch(() => undefined);
|
|
55917
|
+
throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
|
|
55918
|
+
`as "${credentials.user}": ${e.message}`, e);
|
|
55919
|
+
}
|
|
55920
|
+
return new SupabaseDbAdapter(queryExecutor);
|
|
55060
55921
|
}
|
|
55061
|
-
|
|
55062
|
-
|
|
55063
|
-
failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
|
|
55922
|
+
constructor(queryExecutor) {
|
|
55923
|
+
super(queryExecutor);
|
|
55064
55924
|
}
|
|
55065
|
-
|
|
55066
|
-
function datasetString(target, datasetType, disabled) {
|
|
55067
|
-
return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
|
|
55068
|
-
}
|
|
55069
|
-
function assertionString(target, disabled) {
|
|
55070
|
-
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55071
|
-
}
|
|
55072
|
-
function operationString(target, disabled) {
|
|
55073
|
-
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55074
|
-
}
|
|
55075
|
-
function plainTargetString(target) {
|
|
55076
|
-
return `${target.schema}.${target.name}`;
|
|
55077
|
-
}
|
|
55078
|
-
function targetString(target) {
|
|
55079
|
-
return calloutOutput(`${target.schema}.${target.name}`);
|
|
55080
|
-
}
|
|
55081
|
-
function dotRepresentation(graph, interactive) {
|
|
55082
|
-
var _a, _b, _c;
|
|
55083
|
-
const nodes = [];
|
|
55084
|
-
const edges = [];
|
|
55085
|
-
const formatTarget = interactive ? targetString : plainTargetString;
|
|
55086
|
-
(_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
|
|
55087
|
-
var _a;
|
|
55088
|
-
const nodeName = `${formatTarget(table.target)}`;
|
|
55089
|
-
nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
|
|
55090
|
-
(_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55091
|
-
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55092
|
-
});
|
|
55093
|
-
});
|
|
55094
|
-
(_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
|
|
55095
|
-
var _a;
|
|
55096
|
-
const nodeName = `${formatTarget(assertion.target)}`;
|
|
55097
|
-
nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
|
|
55098
|
-
(_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55099
|
-
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55100
|
-
});
|
|
55101
|
-
});
|
|
55102
|
-
(_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
|
|
55925
|
+
async verifyServiceRolePermissions() {
|
|
55103
55926
|
var _a;
|
|
55104
|
-
|
|
55105
|
-
|
|
55106
|
-
|
|
55107
|
-
|
|
55108
|
-
}
|
|
55109
|
-
|
|
55110
|
-
|
|
55111
|
-
}
|
|
55112
|
-
|
|
55113
|
-
|
|
55114
|
-
|
|
55115
|
-
|
|
55116
|
-
|
|
55117
|
-
|
|
55118
|
-
|
|
55119
|
-
|
|
55927
|
+
try {
|
|
55928
|
+
const result = await this.execute("select current_user");
|
|
55929
|
+
const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
|
|
55930
|
+
return !!currentUser;
|
|
55931
|
+
}
|
|
55932
|
+
catch (_b) {
|
|
55933
|
+
return false;
|
|
55934
|
+
}
|
|
55935
|
+
}
|
|
55936
|
+
async validatePublicationState(pubName = "supabase_realtime") {
|
|
55937
|
+
try {
|
|
55938
|
+
const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
|
|
55939
|
+
params: [pubName]
|
|
55940
|
+
});
|
|
55941
|
+
return result.rows.length > 0;
|
|
55942
|
+
}
|
|
55943
|
+
catch (_a) {
|
|
55944
|
+
return false;
|
|
55945
|
+
}
|
|
55946
|
+
}
|
|
55120
55947
|
}
|
|
55121
55948
|
|
|
55122
55949
|
function getBigQueryCredentials() {
|
|
@@ -55990,6 +56817,18 @@ const icebergOption = option("iceberg", {
|
|
|
55990
56817
|
type: "boolean",
|
|
55991
56818
|
default: false
|
|
55992
56819
|
});
|
|
56820
|
+
const bareOption = option("bare", {
|
|
56821
|
+
describe: "Skip the sample project files — scaffold bare (gitkept) directories only.",
|
|
56822
|
+
type: "boolean",
|
|
56823
|
+
default: false
|
|
56824
|
+
});
|
|
56825
|
+
const interactiveOption = option("interactive", {
|
|
56826
|
+
describe: "Guided Q&A setup: start a fresh project (warehouse, sample project, credentials) or " +
|
|
56827
|
+
"convert an existing Dataform project. Other init arguments are ignored except " +
|
|
56828
|
+
"[project-dir], which seeds the directory prompt.",
|
|
56829
|
+
type: "boolean",
|
|
56830
|
+
default: false
|
|
56831
|
+
});
|
|
55993
56832
|
const warehouseOption = option("warehouse", {
|
|
55994
56833
|
describe: "Target warehouse for the new project.",
|
|
55995
56834
|
type: "string",
|
|
@@ -56239,14 +57078,17 @@ function runCli() {
|
|
|
56239
57078
|
{
|
|
56240
57079
|
format: `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` +
|
|
56241
57080
|
` [${ProjectConfigOptions.defaultLocation.name}]`,
|
|
56242
|
-
description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB)."
|
|
57081
|
+
description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB). " +
|
|
57082
|
+
"Use --interactive for a guided setup, including converting a Dataform project.",
|
|
56243
57083
|
positionalOptions: [
|
|
56244
57084
|
projectDirOption,
|
|
56245
57085
|
positionalOption(ProjectConfigOptions.defaultDatabase.name, {
|
|
56246
57086
|
describe: "The default database to use, equivalent to Google Cloud Project ID."
|
|
56247
57087
|
}, (argv) => {
|
|
56248
57088
|
const warehouse = argv[warehouseOption.name] || "bigquery";
|
|
56249
|
-
if (warehouse === "bigquery" &&
|
|
57089
|
+
if (warehouse === "bigquery" &&
|
|
57090
|
+
!argv[interactiveOption.name] &&
|
|
57091
|
+
!argv[ProjectConfigOptions.defaultDatabase.name]) {
|
|
56250
57092
|
throw new Error(`The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` +
|
|
56251
57093
|
`required for BigQuery projects. Use "sqlanvil help init" for more info.`);
|
|
56252
57094
|
}
|
|
@@ -56256,15 +57098,20 @@ function runCli() {
|
|
|
56256
57098
|
"https://cloud.google.com/bigquery/docs/locations for supported values."
|
|
56257
57099
|
}, (argv) => {
|
|
56258
57100
|
const warehouse = argv[warehouseOption.name] || "bigquery";
|
|
56259
|
-
if (warehouse === "bigquery" &&
|
|
57101
|
+
if (warehouse === "bigquery" &&
|
|
57102
|
+
!argv[interactiveOption.name] &&
|
|
57103
|
+
!argv[ProjectConfigOptions.defaultLocation.name]) {
|
|
56260
57104
|
throw new Error(`The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` +
|
|
56261
57105
|
`required for BigQuery projects. Use "sqlanvil help init" for more info.`);
|
|
56262
57106
|
}
|
|
56263
57107
|
})
|
|
56264
57108
|
],
|
|
56265
|
-
options: [warehouseOption, icebergOption],
|
|
57109
|
+
options: [warehouseOption, icebergOption, bareOption, interactiveOption],
|
|
56266
57110
|
processFn: async (argv) => {
|
|
56267
57111
|
const projectDir = argv[projectDirOption.name];
|
|
57112
|
+
if (argv[interactiveOption.name]) {
|
|
57113
|
+
return runInteractiveInit(projectDir);
|
|
57114
|
+
}
|
|
56268
57115
|
const warehouse = argv[warehouseOption.name] || "bigquery";
|
|
56269
57116
|
const projectConfig = { warehouse };
|
|
56270
57117
|
if (warehouse === "bigquery") {
|
|
@@ -56278,7 +57125,9 @@ function runCli() {
|
|
|
56278
57125
|
}
|
|
56279
57126
|
}
|
|
56280
57127
|
print("Writing project files...\n");
|
|
56281
|
-
const initResult = await init(projectDir, projectConfig
|
|
57128
|
+
const initResult = await init(projectDir, projectConfig, {
|
|
57129
|
+
includeSample: !argv[bareOption.name]
|
|
57130
|
+
});
|
|
56282
57131
|
printInitResult(initResult);
|
|
56283
57132
|
return 0;
|
|
56284
57133
|
}
|
|
@@ -56843,6 +57692,31 @@ function runCli() {
|
|
|
56843
57692
|
}
|
|
56844
57693
|
return 0;
|
|
56845
57694
|
}
|
|
57695
|
+
},
|
|
57696
|
+
{
|
|
57697
|
+
format: "migrate-dataform <source-dir> <out-dir>",
|
|
57698
|
+
description: "Convert a Dataform/BigQuery project into a sqlanvil-on-Postgres project. The source " +
|
|
57699
|
+
"directory is READ-ONLY; the converted project + migration-report.{md,json} land in " +
|
|
57700
|
+
"<out-dir> (must be empty). Sources stay in BigQuery as named connections; target SQL " +
|
|
57701
|
+
"gets safe rewrites + inline SQLANVIL-MIGRATE markers for dialect review.",
|
|
57702
|
+
positionalOptions: [
|
|
57703
|
+
positionalOption("source-dir", {
|
|
57704
|
+
describe: "The Dataform project to convert (never modified)."
|
|
57705
|
+
}),
|
|
57706
|
+
positionalOption("out-dir", {
|
|
57707
|
+
describe: "Where the converted sqlanvil project is written (created; must be empty)."
|
|
57708
|
+
})
|
|
57709
|
+
],
|
|
57710
|
+
options: [],
|
|
57711
|
+
processFn: async (argv) => {
|
|
57712
|
+
const report = await migrateDataform({
|
|
57713
|
+
srcDir: argv["source-dir"],
|
|
57714
|
+
outDir: argv["out-dir"]
|
|
57715
|
+
});
|
|
57716
|
+
printMigrationSummary(report, argv["out-dir"]);
|
|
57717
|
+
print(`Next: sqlanvil compile ${argv["out-dir"]}`);
|
|
57718
|
+
return 0;
|
|
57719
|
+
}
|
|
56846
57720
|
}
|
|
56847
57721
|
]
|
|
56848
57722
|
})
|