@sqlanvil/cli 1.21.0 → 1.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/bundle.js +1428 -554
  2. package/package.json +1 -1
package/bundle.js CHANGED
@@ -50606,7 +50606,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
50606
50606
  .filter(validationQuery => !!validationQuery.query);
50607
50607
  }
50608
50608
 
50609
- const version = "1.21.0";
50609
+ const version = "1.23.0";
50610
50610
  const dataformVersion = "3.0.61";
50611
50611
 
50612
50612
  async function build(compiledGraph, runConfig, dbadapter) {
@@ -51016,7 +51016,8 @@ function mysqlCredentialsTemplate() {
51016
51016
  };
51017
51017
  return `${JSON.stringify(template, null, 2)}\n`;
51018
51018
  }
51019
- async function init(projectDir, projectConfig) {
51019
+ async function init(projectDir, projectConfig, options = {}) {
51020
+ var _a;
51020
51021
  const workflowSettingsYamlPath = path__namespace.join(projectDir, "workflow_settings.yaml");
51021
51022
  const packageJsonPath = path__namespace.join(projectDir, "package.json");
51022
51023
  const gitignorePath = path__namespace.join(projectDir, ".gitignore");
@@ -51064,7 +51065,11 @@ async function init(projectDir, projectConfig) {
51064
51065
  if (projectConfig.builtinAssertionNamePrefix) {
51065
51066
  workflowSettings.builtinAssertionNamePrefix = projectConfig.builtinAssertionNamePrefix;
51066
51067
  }
51067
- if (isPostgresLike) {
51068
+ const includeSample = options.includeSample !== false;
51069
+ const withBigQuerySource = includeSample &&
51070
+ warehouse !== "mysql" &&
51071
+ (!isPostgresLike || options.includeBigQuerySource !== false);
51072
+ if (isPostgresLike && withBigQuerySource) {
51068
51073
  workflowSettings.connections = {
51069
51074
  bigquery_public: sqlanvil.ConnectionConfig.create({
51070
51075
  platform: "bigquery",
@@ -51080,7 +51085,7 @@ async function init(projectDir, projectConfig) {
51080
51085
  fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
51081
51086
  filesWritten.push(gitignorePath);
51082
51087
  if (!isBigQuery) {
51083
- fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse));
51088
+ fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
51084
51089
  filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
51085
51090
  }
51086
51091
  const mkdir = (...segments) => {
@@ -51101,7 +51106,10 @@ async function init(projectDir, projectConfig) {
51101
51106
  const reportingDir = mkdir("definitions", "outputs", "reporting");
51102
51107
  const testDir = mkdir("definitions", "test");
51103
51108
  keep(mkdir("includes"));
51104
- const withBigQuerySource = warehouse !== "mysql";
51109
+ if (!includeSample) {
51110
+ [sourcesDir, intermediateDir, salesDir, reportingDir, testDir].forEach(keep);
51111
+ return { filesWritten, dirsCreated };
51112
+ }
51105
51113
  write(path__namespace.join(sourcesDir, "app_orders.sqlx"), `-- A declaration: a table that already exists in your warehouse (created by your
51106
51114
  -- application, an import, or another pipeline). Point name/dataset at a real table —
51107
51115
  -- downstream models read it with \${ref("app_orders")}.
@@ -53541,7 +53549,7 @@ class Runner {
53541
53549
  return actionResult;
53542
53550
  }
53543
53551
  async executeTask(client, task, parentAction, options, action) {
53544
- var _a, _b, _c, _d, _e, _f, _g;
53552
+ var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
53545
53553
  const timer = Timer.start();
53546
53554
  const taskResult = {
53547
53555
  status: sqlanvil.TaskResult.ExecutionStatus.RUNNING,
@@ -53613,7 +53621,14 @@ class Runner {
53613
53621
  }
53614
53622
  else if (task.type === "extract") {
53615
53623
  try {
53616
- const extractor = ((_d = action === null || action === void 0 ? void 0 : action.extract) === null || _d === void 0 ? void 0 : _d.platform) === "mysql"
53624
+ if (Object.keys((_e = (_d = action === null || action === void 0 ? void 0 : action.extract) === null || _d === void 0 ? void 0 : _d.columnTypes) !== null && _e !== void 0 ? _e : {}).length === 0) {
53625
+ const ds = ((_f = action === null || action === void 0 ? void 0 : action.extract) === null || _f === void 0 ? void 0 : _f.platform) === "mysql"
53626
+ ? (_g = action === null || action === void 0 ? void 0 : action.extract) === null || _g === void 0 ? void 0 : _g.database : (_h = action === null || action === void 0 ? void 0 : action.extract) === null || _h === void 0 ? void 0 : _h.dataset;
53627
+ throw new Error(`declaration "${(_j = action === null || action === void 0 ? void 0 : action.target) === null || _j === void 0 ? void 0 : _j.name}" has no columnTypes — scaffold them with: ` +
53628
+ `sqlanvil introspect ${(_k = action === null || action === void 0 ? void 0 : action.extract) === null || _k === void 0 ? void 0 : _k.connectionName} ${ds}.${(_l = action === null || action === void 0 ? void 0 : action.extract) === null || _l === void 0 ? void 0 : _l.sourceName} ` +
53629
+ `--output <its .sqlx file>`);
53630
+ }
53631
+ const extractor = ((_m = action === null || action === void 0 ? void 0 : action.extract) === null || _m === void 0 ? void 0 : _m.platform) === "mysql"
53617
53632
  ? this.executionOptions.mysqlExtract || runMysqlExtract
53618
53633
  : this.executionOptions.bigQueryExtract || runBigQueryExtract;
53619
53634
  await extractor({
@@ -53643,7 +53658,7 @@ class Runner {
53643
53658
  }), task.type === "operation" ? 1 : options.bigquery.actionRetryLimit + 1 || 1);
53644
53659
  taskResult.metadata = metadata;
53645
53660
  if (task.type === "assertion") {
53646
- const rowCount = (_e = rows[0]) === null || _e === void 0 ? void 0 : _e[Object.keys(rows[0])[0]];
53661
+ const rowCount = (_o = rows[0]) === null || _o === void 0 ? void 0 : _o[Object.keys(rows[0])[0]];
53647
53662
  if (rowCount > 0) {
53648
53663
  throw new Error(`Assertion failed: query returned ${rowCount} row(s).`);
53649
53664
  }
@@ -53655,7 +53670,7 @@ class Runner {
53655
53670
  ? sqlanvil.TaskResult.ExecutionStatus.CANCELLED
53656
53671
  : sqlanvil.TaskResult.ExecutionStatus.FAILED;
53657
53672
  taskResult.errorMessage = `${this.graph.projectConfig.warehouse} error: ${e.message}`;
53658
- if ((_g = (_f = e.metadata) === null || _f === void 0 ? void 0 : _f.bigquery) === null || _g === void 0 ? void 0 : _g.jobId) {
53673
+ if ((_q = (_p = e.metadata) === null || _p === void 0 ? void 0 : _p.bigquery) === null || _q === void 0 ? void 0 : _q.jobId) {
53659
53674
  taskResult.metadata = {
53660
53675
  bigquery: {
53661
53676
  jobId: e.metadata.bigquery.jobId
@@ -54175,443 +54190,594 @@ ${columnRows || '<tr><td colspan="3" class="meta">No documented columns.</td></t
54175
54190
  `;
54176
54191
  }
54177
54192
 
54178
- const CHECKER_SOURCE = `
54179
- import json, os, sys
54180
-
54181
- script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
54182
- errors = []
54183
-
54184
- import platform
54185
- py_version = platform.python_version()
54186
-
54187
- def load_packaging():
54188
- try:
54189
- from packaging.specifiers import SpecifierSet
54190
- from packaging.requirements import Requirement
54191
- return SpecifierSet, Requirement
54192
- except ImportError:
54193
- pass
54194
- try:
54195
- from pip._vendor.packaging.specifiers import SpecifierSet
54196
- from pip._vendor.packaging.requirements import Requirement
54197
- return SpecifierSet, Requirement
54198
- except ImportError:
54199
- return None, None
54200
-
54201
- SpecifierSet, Requirement = load_packaging()
54202
-
54203
- def mini_parse(v):
54204
- import re
54205
- return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
54206
-
54207
- def mini_check(version, spec):
54208
- import re
54209
- ok = True
54210
- for clause in spec.split(","):
54211
- clause = clause.strip()
54212
- m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
54213
- if not m:
54214
- return None
54215
- op, want = m.group(1), m.group(2).strip()
54216
- if want.endswith(".*"):
54217
- prefix = mini_parse(want[:-2])
54218
- have = mini_parse(version)[:len(prefix)]
54219
- eq = have == prefix
54220
- if (op in ("==", "===") and not eq) or (op == "!=" and eq):
54221
- ok = False
54222
- continue
54223
- a, b = mini_parse(version), mini_parse(want)
54224
- n = max(len(a), len(b))
54225
- a = a + [0] * (n - len(a))
54226
- b = b + [0] * (n - len(b))
54227
- if op in ("==", "==="):
54228
- res = a == b
54229
- elif op == "!=":
54230
- res = a != b
54231
- elif op == "<=":
54232
- res = a <= b
54233
- elif op == ">=":
54234
- res = a >= b
54235
- elif op == "<":
54236
- res = a < b
54237
- elif op == ">":
54238
- res = a > b
54239
- else: # ~= : at least b, within b's series
54240
- wb = mini_parse(want)
54241
- res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
54242
- if not res:
54243
- ok = False
54244
- return ok
54245
-
54246
- def check_version(version, spec, what):
54247
- if SpecifierSet is not None:
54248
- try:
54249
- if version not in SpecifierSet(spec, prereleases=True):
54250
- errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
54251
- return
54252
- except Exception as e:
54253
- errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
54254
- return
54255
- if mini_check(version, spec) is False:
54256
- errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
54257
-
54258
- if version_spec:
54259
- check_version(py_version, version_spec, "python interpreter")
54260
-
54261
- if requirements_path:
54262
- from importlib import metadata
54263
- try:
54264
- lines = open(requirements_path).read().splitlines()
54265
- except OSError as e:
54266
- errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
54267
- lines = []
54268
- for raw in lines:
54269
- line = raw.split("#")[0].strip()
54270
- if not line or line.startswith("-"):
54271
- continue
54272
- if Requirement is not None:
54273
- try:
54274
- req = Requirement(line)
54275
- except Exception:
54276
- errors.append('requirements: cannot parse "%s"' % line)
54277
- continue
54278
- if req.marker is not None and not req.marker.evaluate():
54279
- continue
54280
- name, spec = req.name, str(req.specifier)
54281
- else:
54282
- import re
54283
- if ";" in line:
54284
- line = line.split(";")[0].strip()
54285
- m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
54286
- if not m:
54287
- errors.append('requirements: cannot parse "%s"' % line)
54288
- continue
54289
- name, spec = m.group(1), (m.group(3) or "").strip()
54290
- try:
54291
- installed = metadata.version(name)
54292
- except metadata.PackageNotFoundError:
54293
- errors.append(
54294
- "requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
54295
- )
54296
- continue
54297
- if spec:
54298
- check_version(installed, spec, "requirements: %s" % name)
54299
-
54300
- try:
54301
- with open(script_path, "rb") as f:
54302
- compile(f.read(), script_path, "exec")
54303
- except SyntaxError as e:
54304
- errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
54305
- except (OSError, ValueError) as e:
54306
- errors.append("syntax: cannot compile %s: %s" % (script_path, e))
54307
-
54308
- print(json.dumps({"interpreter_version": py_version, "errors": errors}))
54309
- `;
54310
- const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
54311
- childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
54312
- if (err) {
54313
- reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
54314
- }
54315
- else {
54316
- resolve(stdout);
54317
- }
54318
- });
54319
- });
54320
- const failure = (message) => ({
54321
- status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
54322
- error: { message }
54323
- });
54324
- async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
54325
- if (script.language !== "python") {
54326
- return [failure(`no validator for script language "${script.language}"`)];
54327
- }
54328
- let interpreter;
54329
- try {
54330
- interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
54331
- }
54332
- catch (e) {
54333
- return [failure(e.message)];
54334
- }
54335
- let stdout;
54336
- try {
54337
- stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
54338
- }
54339
- catch (e) {
54340
- return [failure(e.message)];
54341
- }
54342
- let parsed;
54343
- try {
54344
- parsed = JSON.parse(stdout);
54345
- }
54346
- catch (e) {
54347
- return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
54193
+ function containsPath(parent, child) {
54194
+ const rel = path__namespace.relative(parent, child);
54195
+ return rel === "" || (!rel.startsWith("..") && !path__namespace.isAbsolute(rel));
54196
+ }
54197
+ class OutWriter {
54198
+ constructor(outDir) {
54199
+ this.outDir = outDir;
54348
54200
  }
54349
- if (parsed.errors && parsed.errors.length > 0) {
54350
- return parsed.errors.map(failure);
54201
+ write(relPath, contents) {
54202
+ const full = path__namespace.resolve(this.outDir, relPath);
54203
+ if (!containsPath(path__namespace.resolve(this.outDir), full)) {
54204
+ throw new Error(`Refusing to write outside the output directory: ${relPath}`);
54205
+ }
54206
+ fs__namespace$1.mkdirSync(path__namespace.dirname(full), { recursive: true });
54207
+ fs__namespace$1.writeFileSync(full, contents);
54351
54208
  }
54352
- return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
54353
54209
  }
54354
-
54355
- function targetKey(target) {
54356
- return [target.database, target.schema, target.name].filter(Boolean).join(".");
54210
+ const SAFETY_SKIP = [
54211
+ /^\.git(\/|$)/,
54212
+ /(^|\/)node_modules(\/|$)/,
54213
+ /(^|\/)\.df-credentials[^/]*\.json$/,
54214
+ /service_account[^/]*\.json$/i,
54215
+ /(^|\/)\.env([^/]*)$/,
54216
+ /\.pem$/,
54217
+ /(^|\/)compile_output\.json$/,
54218
+ /(^|\/)\.DS_Store$/,
54219
+ /(^|\/)target(\/|$)/,
54220
+ ];
54221
+ function walkFiles(root) {
54222
+ const out = [];
54223
+ const walk = (rel) => {
54224
+ for (const entry of fs__namespace$1.readdirSync(path__namespace.join(root, rel), { withFileTypes: true })) {
54225
+ const relPath = rel ? `${rel}/${entry.name}` : entry.name;
54226
+ if (SAFETY_SKIP.some(re => re.test(relPath) || re.test(relPath + "/"))) {
54227
+ skippedForSafety.add(relPath);
54228
+ continue;
54229
+ }
54230
+ if (entry.isDirectory()) {
54231
+ walk(relPath);
54232
+ }
54233
+ else if (entry.isFile()) {
54234
+ out.push(relPath);
54235
+ }
54236
+ }
54237
+ };
54238
+ walk("");
54239
+ return out;
54357
54240
  }
54358
- function topoOrder(nodes) {
54359
- const byKey = new Map(nodes.map(n => [n.key, n]));
54360
- const indegree = new Map();
54361
- const dependents = new Map();
54362
- for (const n of nodes) {
54363
- const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
54364
- indegree.set(n.key, inGraphDeps.length);
54365
- for (const dep of inGraphDeps) {
54366
- dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
54241
+ let skippedForSafety;
54242
+ function findConfigBlock(source) {
54243
+ const m = /(^|\n)\s*config\s*\{/.exec(source);
54244
+ if (!m)
54245
+ return null;
54246
+ const open = source.indexOf("{", m.index + m[0].indexOf("config"));
54247
+ let depth = 0;
54248
+ let inString = null;
54249
+ for (let i = open; i < source.length; i++) {
54250
+ const ch = source[i];
54251
+ const prev = source[i - 1];
54252
+ if (inString) {
54253
+ if (ch === inString && prev !== "\\")
54254
+ inString = null;
54255
+ continue;
54367
54256
  }
54368
- }
54369
- const ready = nodes
54370
- .filter(n => (indegree.get(n.key) || 0) === 0)
54371
- .map(n => n.key)
54372
- .sort();
54373
- const ordered = [];
54374
- const emitted = new Set();
54375
- while (ready.length) {
54376
- const key = ready.shift();
54377
- if (emitted.has(key)) {
54257
+ if (ch === '"' || ch === "'" || ch === "`") {
54258
+ inString = ch;
54378
54259
  continue;
54379
54260
  }
54380
- emitted.add(key);
54381
- ordered.push(byKey.get(key));
54382
- for (const dep of (dependents.get(key) || []).slice().sort()) {
54383
- indegree.set(dep, (indegree.get(dep) || 0) - 1);
54384
- if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
54385
- ready.push(dep);
54386
- ready.sort();
54261
+ if (ch === "{")
54262
+ depth++;
54263
+ if (ch === "}") {
54264
+ depth--;
54265
+ if (depth === 0) {
54266
+ return {
54267
+ start: m.index + (m[1] ? m[1].length : 0),
54268
+ open,
54269
+ end: i + 1,
54270
+ body: source.slice(open + 1, i),
54271
+ };
54387
54272
  }
54388
54273
  }
54389
54274
  }
54390
- if (ordered.length < nodes.length) {
54391
- for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
54392
- if (!emitted.has(n.key)) {
54393
- emitted.add(n.key);
54394
- ordered.push(n);
54275
+ return null;
54276
+ }
54277
+ function jsBlockLines(source) {
54278
+ const lines = new Set();
54279
+ const re = /(^|\n)\s*js\s*\{/g;
54280
+ let m;
54281
+ while ((m = re.exec(source))) {
54282
+ const open = source.indexOf("{", m.index + m[0].length - 1);
54283
+ let depth = 0;
54284
+ let end = -1;
54285
+ for (let i = open; i < source.length; i++) {
54286
+ if (source[i] === "{")
54287
+ depth++;
54288
+ if (source[i] === "}") {
54289
+ depth--;
54290
+ if (depth === 0) {
54291
+ end = i;
54292
+ break;
54293
+ }
54395
54294
  }
54396
54295
  }
54296
+ if (end < 0)
54297
+ end = source.length - 1;
54298
+ const startLine = source.slice(0, m.index + (m[1] ? m[1].length : 0)).split("\n").length - 1;
54299
+ const endLine = source.slice(0, end).split("\n").length - 1;
54300
+ for (let l = startLine; l <= endLine; l++)
54301
+ lines.add(l);
54397
54302
  }
54398
- return ordered;
54399
- }
54400
- function dependencyBlocked(dependencyKeys, statusByKey) {
54401
- return dependencyKeys.some(dep => {
54402
- const status = statusByKey.get(dep);
54403
- return status !== undefined && status !== "PASS";
54404
- });
54303
+ return lines;
54405
54304
  }
54406
- const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
54407
- function validateShadowSuffix(nowMs) {
54408
- return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
54305
+ const str = (field, body) => {
54306
+ const re = new RegExp(`${field}\\s*:\\s*["'\`]([^"'\`]+)["'\`]`);
54307
+ const m = re.exec(body);
54308
+ return m ? m[1] : null;
54309
+ };
54310
+ function parseSqlxConfig(body) {
54311
+ let database = str("database", body);
54312
+ if (!database && /database\s*:\s*(dataform|sqlanvil)\.projectConfig\.defaultDatabase/.test(body)) {
54313
+ database = "<defaultDatabase>";
54314
+ }
54315
+ return {
54316
+ type: str("type", body),
54317
+ schema: str("schema", body),
54318
+ database,
54319
+ name: str("name", body),
54320
+ description: str("description", body),
54321
+ };
54409
54322
  }
54410
- function parseShadowTimestamp(schemaName) {
54411
- const match = schemaName.match(/sqlanvil_validate_(\d+)/);
54412
- return match ? Number(match[1]) : null;
54323
+ const GLOBAL_RULES = [
54324
+ {
54325
+ pattern: /\bdataform\.projectConfig\b/g,
54326
+ replacement: "sqlanvil.projectConfig",
54327
+ kind: "rewrite",
54328
+ note: "dataform.projectConfig → sqlanvil.projectConfig (compile global renamed)",
54329
+ },
54330
+ ];
54331
+ const SQL_RULES = [
54332
+ {
54333
+ pattern: /\bSAFE_CAST\s*\(/g,
54334
+ replacement: "CAST(",
54335
+ kind: "rewrite",
54336
+ note: "SAFE_CAST → CAST — Postgres CAST raises on bad input; verify the data or guard it",
54337
+ },
54338
+ {
54339
+ pattern: /\bCURRENT_DATE\s*\(\s*\)/g,
54340
+ replacement: "CURRENT_DATE",
54341
+ kind: "rewrite",
54342
+ note: "CURRENT_DATE() → CURRENT_DATE",
54343
+ },
54344
+ {
54345
+ pattern: /\bCURRENT_DATE\s*\(\s*['"]/g,
54346
+ kind: "flag",
54347
+ note: "CURRENT_DATE('<tz>') — Postgres: (CURRENT_TIMESTAMP AT TIME ZONE '<tz>')::date",
54348
+ },
54349
+ {
54350
+ pattern: /\bSAFE_DIVIDE\s*\(/g,
54351
+ kind: "flag",
54352
+ note: "SAFE_DIVIDE(a, b) — Postgres: a / NULLIF(b, 0)",
54353
+ },
54354
+ {
54355
+ pattern: /\bSAFE_OFFSET\s*\(/g,
54356
+ kind: "flag",
54357
+ note: "array[SAFE_OFFSET(n)] — Postgres arrays are 1-based; use (split_part / arr[n+1])",
54358
+ },
54359
+ { pattern: /\bQUALIFY\b/gi, kind: "flag", note: "QUALIFY — rewrite as a subquery/CTE filter on the window function" },
54360
+ { pattern: /\bUNNEST\s*\(/g, kind: "flag", note: "UNNEST — Postgres unnest() works on arrays only (no STRUCT arrays); verify semantics" },
54361
+ { pattern: /\bSTRUCT\s*[(<]/g, kind: "flag", note: "STRUCT — no Postgres equivalent; consider jsonb or a composite type" },
54362
+ { pattern: /\bARRAY\s*</g, kind: "flag", note: "ARRAY<T> type syntax — Postgres uses T[]" },
54363
+ { pattern: /\bFORMAT_DATE\s*\(/g, kind: "flag", note: "FORMAT_DATE → TO_CHAR (different format tokens)" },
54364
+ { pattern: /\bPARSE_DATE\s*\(/g, kind: "flag", note: "PARSE_DATE → TO_DATE (different format tokens)" },
54365
+ { pattern: /\bFORMAT_TIMESTAMP\s*\(/g, kind: "flag", note: "FORMAT_TIMESTAMP → TO_CHAR (different format tokens)" },
54366
+ {
54367
+ pattern: /\b(DATE_ADD|DATE_SUB|TIMESTAMP_ADD|TIMESTAMP_SUB|DATETIME_ADD|DATETIME_SUB)\s*\(/g,
54368
+ kind: "flag",
54369
+ note: "BigQuery date arithmetic — Postgres uses interval arithmetic (x + INTERVAL '1 day')",
54370
+ },
54371
+ { pattern: /\bDATE_DIFF\s*\(/g, kind: "flag", note: "DATE_DIFF — Postgres: subtraction / EXTRACT(EPOCH FROM ...) depending on the part" },
54372
+ { pattern: /_PARTITIONTIME|_PARTITIONDATE|_TABLE_SUFFIX/g, kind: "flag", note: "BigQuery pseudo-column has no Postgres equivalent" },
54373
+ { pattern: /\bEXPORT\s+DATA\b/gi, kind: "flag", note: "EXPORT DATA — use a sqlanvil `type: \"export\"` action instead" },
54374
+ { pattern: /\bEXECUTE\s+IMMEDIATE\b/gi, kind: "flag", note: "BigQuery scripting (EXECUTE IMMEDIATE) — rewrite as a PL/pgSQL DO block or operations" },
54375
+ { pattern: /\bGENERATE_UUID\s*\(\s*\)/g, kind: "flag", note: "GENERATE_UUID() → gen_random_uuid()" },
54376
+ {
54377
+ pattern: /`[A-Za-z0-9_-]+\.[A-Za-z0-9_$]+\.[A-Za-z0-9_$]+`/g,
54378
+ kind: "flag",
54379
+ note: "backticked BigQuery FQN — use ${ref(...)} (declare it as a source) or a quoted Postgres name",
54380
+ },
54381
+ ];
54382
+ const CONFIG_BQ_KEYS = /^(\s*)(bigquery\s*:|partitionBy\s*:|clusterBy\s*:|requirePartitionFilter\s*:|partitionExpirationDays\s*:|database\s*:)/;
54383
+ const sanitize = (s) => s.toLowerCase().replace(/[^a-z0-9_]/g, "_");
54384
+ const connectionNameFor = (project) => `bq_${sanitize(project)}`;
54385
+ function convertDeclaration(source, parsed, connection, sourceDataset, filePath) {
54386
+ const span = findConfigBlock(source);
54387
+ const description = parsed.description
54388
+ ? ` description: ${JSON.stringify(parsed.description)},\n`
54389
+ : "";
54390
+ const newConfig = `config {
54391
+ type: "declaration",
54392
+ connection: ${JSON.stringify(connection)},
54393
+ schema: ${JSON.stringify(sourceDataset)},
54394
+ name: ${JSON.stringify(parsed.name)},
54395
+ ${description} columnTypes: {
54396
+ // SQLANVIL-MIGRATE(TODO): scaffold the columns from the live source:
54397
+ // sqlanvil introspect ${connection} ${sourceDataset}.${parsed.name} --output ${filePath}
54398
+ },
54399
+ }`;
54400
+ const content = source.slice(0, span.start) + newConfig + source.slice(span.end);
54401
+ return {
54402
+ content,
54403
+ findings: [
54404
+ {
54405
+ line: 1,
54406
+ kind: "rewrite",
54407
+ note: `declaration now reads BigQuery through connection "${connection}" (runner-extract); columnTypes need introspect`,
54408
+ },
54409
+ ],
54410
+ };
54413
54411
  }
54414
- function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
54415
- return schemaNames.filter(name => {
54416
- const ts = parseShadowTimestamp(name);
54417
- return ts !== null && nowMs - ts > maxAgeMs;
54418
- });
54419
- }
54420
-
54421
- const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
54422
- async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
54423
- try {
54424
- const schemas = await deps.listSchemas();
54425
- for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
54426
- try {
54427
- await deps.execute(deps.dropSchemaCascadeSql(schema));
54412
+ function convertTarget(source, jsMode) {
54413
+ const span = jsMode ? null : findConfigBlock(source);
54414
+ const findings = [];
54415
+ const lines = source.split("\n");
54416
+ const jsLines = jsMode ? new Set() : jsBlockLines(source);
54417
+ let configStartLine = -1;
54418
+ let configEndLine = -1;
54419
+ if (span) {
54420
+ configStartLine = source.slice(0, span.start).split("\n").length - 1;
54421
+ configEndLine = source.slice(0, span.end).split("\n").length - 1;
54422
+ }
54423
+ const out = [];
54424
+ let commentingConfigBlockDepth = 0;
54425
+ let swallowLeadingComma = false;
54426
+ for (let i = 0; i < lines.length; i++) {
54427
+ let line = lines[i];
54428
+ const inConfig = i >= configStartLine && i <= configEndLine && configStartLine >= 0;
54429
+ const markersSafe = !jsMode && !jsLines.has(i);
54430
+ const marker = (note) => inConfig ? `// SQLANVIL-MIGRATE: ${note}` : `-- SQLANVIL-MIGRATE: ${note}`;
54431
+ const lineNo = i + 1;
54432
+ const notesForLine = [];
54433
+ for (const rule of GLOBAL_RULES) {
54434
+ rule.pattern.lastIndex = 0;
54435
+ if (rule.pattern.test(line)) {
54436
+ line = line.replace(rule.pattern, rule.replacement);
54437
+ findings.push({ line: lineNo, kind: "rewrite", note: rule.note });
54438
+ }
54439
+ }
54440
+ if (swallowLeadingComma) {
54441
+ swallowLeadingComma = false;
54442
+ if (/^\s*,\s*$/.test(line)) {
54443
+ out.push(`// ${line}`);
54444
+ continue;
54428
54445
  }
54429
- catch (e) {
54446
+ if (/^\s*,/.test(line)) {
54447
+ line = line.replace(/,/, " ");
54448
+ }
54449
+ }
54450
+ if (inConfig && commentingConfigBlockDepth === 0) {
54451
+ const m = CONFIG_BQ_KEYS.exec(line);
54452
+ if (m) {
54453
+ const note = `BigQuery-only setting — not applicable on Postgres (partitioning/clustering → postgres: {}; project qualifiers drop)`;
54454
+ findings.push({ line: lineNo, kind: "flag", note });
54455
+ out.push(marker(note));
54456
+ const opens = (line.match(/\{/g) || []).length;
54457
+ const closes = (line.match(/\}/g) || []).length;
54458
+ commentingConfigBlockDepth = Math.max(0, opens - closes);
54459
+ if (commentingConfigBlockDepth === 0)
54460
+ swallowLeadingComma = !/,\s*$/.test(line) ? true : false;
54461
+ out.push(`// ${line}`);
54462
+ continue;
54463
+ }
54464
+ }
54465
+ else if (commentingConfigBlockDepth > 0) {
54466
+ const opens = (line.match(/\{/g) || []).length;
54467
+ const closes = (line.match(/\}/g) || []).length;
54468
+ commentingConfigBlockDepth = Math.max(0, commentingConfigBlockDepth + opens - closes);
54469
+ if (commentingConfigBlockDepth === 0 && !/,\s*$/.test(line))
54470
+ swallowLeadingComma = true;
54471
+ out.push(`// ${line}`);
54472
+ continue;
54473
+ }
54474
+ if (!inConfig || jsMode) {
54475
+ for (const rule of SQL_RULES) {
54476
+ rule.pattern.lastIndex = 0;
54477
+ if (!rule.pattern.test(line))
54478
+ continue;
54479
+ if (rule.replacement !== undefined) {
54480
+ rule.pattern.lastIndex = 0;
54481
+ line = line.replace(rule.pattern, rule.replacement);
54482
+ findings.push({ line: lineNo, kind: "rewrite", note: rule.note });
54483
+ if (markersSafe)
54484
+ notesForLine.push(rule.note);
54485
+ }
54486
+ else {
54487
+ findings.push({ line: lineNo, kind: "flag", note: rule.note });
54488
+ if (markersSafe)
54489
+ notesForLine.push(rule.note);
54490
+ }
54430
54491
  }
54431
54492
  }
54493
+ for (const note of notesForLine) {
54494
+ out.push(marker(note));
54495
+ }
54496
+ out.push(line);
54432
54497
  }
54433
- catch (e) {
54498
+ let content = out.join("\n");
54499
+ if (jsMode && findings.length > 0) {
54500
+ const noted = findings.slice(0, 12).map(f => ` * L${f.line}: ${f.note}`);
54501
+ const more = findings.length > 12 ? [` * …and ${findings.length - 12} more (see migration-report.json)`] : [];
54502
+ content =
54503
+ ["/* SQLANVIL-MIGRATE — findings in this file (inline markers are unsafe in JS templates):", ...noted, ...more, " */"].join("\n") +
54504
+ "\n" +
54505
+ content;
54434
54506
  }
54507
+ return { content, findings };
54435
54508
  }
54436
- function tableType(enumType) {
54437
- switch (enumType) {
54438
- case sqlanvil.TableType.VIEW:
54439
- return "view";
54440
- case sqlanvil.TableType.INCREMENTAL:
54441
- return "incremental";
54442
- default:
54443
- return "table";
54509
+ function readSourceConfig(srcDir) {
54510
+ var _a, _b, _c, _d, _e;
54511
+ const wsPath = path__namespace.join(srcDir, "workflow_settings.yaml");
54512
+ const djPath = path__namespace.join(srcDir, "dataform.json");
54513
+ let raw;
54514
+ if (fs__namespace$1.existsSync(wsPath)) {
54515
+ raw = jsYaml.load(fs__namespace$1.readFileSync(wsPath, "utf8")) || {};
54444
54516
  }
54445
- }
54446
- function depKeys(deps) {
54447
- return (deps || []).map(targetKey);
54448
- }
54449
- async function validate(compiledGraph, deps, options = {}) {
54450
- const nodes = [];
54451
- for (const table of compiledGraph.tables || []) {
54452
- nodes.push({
54453
- key: targetKey(table.target),
54454
- dependencyKeys: depKeys(table.dependencyTargets),
54455
- kind: "table",
54456
- type: tableType(table.enumType),
54457
- target: table.target,
54458
- table,
54459
- action: table
54460
- });
54517
+ else if (fs__namespace$1.existsSync(djPath)) {
54518
+ raw = JSON.parse(fs__namespace$1.readFileSync(djPath, "utf8"));
54461
54519
  }
54462
- for (const assertion of compiledGraph.assertions || []) {
54463
- nodes.push({
54464
- key: targetKey(assertion.target),
54465
- dependencyKeys: depKeys(assertion.dependencyTargets),
54466
- kind: "assertion",
54467
- type: "assertion",
54468
- target: assertion.target,
54469
- action: assertion
54470
- });
54520
+ else {
54521
+ throw new Error(`${srcDir} does not look like a Dataform project (no workflow_settings.yaml or dataform.json).`);
54471
54522
  }
54472
- for (const operation of compiledGraph.operations || []) {
54473
- if (!operation.target) {
54523
+ const s = (k) => (typeof raw[k] === "string" ? raw[k] : null);
54524
+ return {
54525
+ defaultProject: (_a = s("defaultProject")) !== null && _a !== void 0 ? _a : s("defaultDatabase"),
54526
+ defaultDataset: (_c = (_b = s("defaultDataset")) !== null && _b !== void 0 ? _b : s("defaultSchema")) !== null && _c !== void 0 ? _c : "dataform",
54527
+ defaultAssertionDataset: (_d = s("defaultAssertionDataset")) !== null && _d !== void 0 ? _d : s("assertionSchema"),
54528
+ vars: (_e = raw.vars) !== null && _e !== void 0 ? _e : null,
54529
+ raw,
54530
+ };
54531
+ }
54532
+ async function migrateDataform(opts) {
54533
+ var _a, _b, _c, _d, _e, _f, _g, _h;
54534
+ const srcDir = path__namespace.resolve(opts.srcDir);
54535
+ const outDir = path__namespace.resolve(opts.outDir);
54536
+ if (containsPath(srcDir, outDir)) {
54537
+ throw new Error("The output directory must not be inside the source project.");
54538
+ }
54539
+ if (containsPath(outDir, srcDir)) {
54540
+ throw new Error("The source project must not be inside the output directory.");
54541
+ }
54542
+ if (fs__namespace$1.existsSync(outDir) && fs__namespace$1.readdirSync(outDir).length > 0) {
54543
+ throw new Error(`Output directory ${outDir} is not empty.`);
54544
+ }
54545
+ const sourceConfig = readSourceConfig(srcDir);
54546
+ const writer = new OutWriter(outDir);
54547
+ skippedForSafety = new Set();
54548
+ const files = walkFiles(srcDir);
54549
+ const report = {
54550
+ generator: `sqlanvil migrate-dataform ${version}`,
54551
+ sourceDir: srcDir,
54552
+ sourceConfig: sourceConfig.raw,
54553
+ inventory: { sqlxFiles: 0, jsFiles: 0, byType: {} },
54554
+ connections: [],
54555
+ overlappingSchemas: [],
54556
+ files: [],
54557
+ skippedForSafety: [],
54558
+ warnings: [],
54559
+ };
54560
+ const sqlxFiles = [];
54561
+ for (const rel of files) {
54562
+ if (!rel.endsWith(".sqlx"))
54474
54563
  continue;
54564
+ const source = fs__namespace$1.readFileSync(path__namespace.join(srcDir, rel), "utf8");
54565
+ const span = findConfigBlock(source);
54566
+ const parsed = span
54567
+ ? parseSqlxConfig(span.body)
54568
+ : { type: null, schema: null, database: null, name: null, description: null };
54569
+ sqlxFiles.push({ rel, source, parsed });
54570
+ report.inventory.sqlxFiles++;
54571
+ const t = (_a = parsed.type) !== null && _a !== void 0 ? _a : "(none)";
54572
+ report.inventory.byType[t] = ((_b = report.inventory.byType[t]) !== null && _b !== void 0 ? _b : 0) + 1;
54573
+ }
54574
+ const resolveDatabase = (db) => { var _a; return !db || db === "<defaultDatabase>" ? (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "UNKNOWN_PROJECT" : db; };
54575
+ const declarationSchemas = new Set();
54576
+ const targetSchemas = new Set();
54577
+ const connectionMap = new Map();
54578
+ for (const f of sqlxFiles) {
54579
+ if (f.parsed.type === "declaration") {
54580
+ const dataset = (_c = f.parsed.schema) !== null && _c !== void 0 ? _c : sourceConfig.defaultDataset;
54581
+ const project = resolveDatabase(f.parsed.database);
54582
+ declarationSchemas.add(dataset);
54583
+ const name = connectionNameFor(project);
54584
+ const existing = connectionMap.get(name);
54585
+ if (existing) {
54586
+ existing.declarationCount++;
54587
+ if (!existing.datasets.includes(dataset))
54588
+ existing.datasets.push(dataset);
54589
+ }
54590
+ else {
54591
+ connectionMap.set(name, {
54592
+ name,
54593
+ project,
54594
+ datasets: [dataset],
54595
+ declarationCount: 1,
54596
+ introspectExample: `sqlanvil introspect ${name} <dataset>.<table> --output <declaration.sqlx>`,
54597
+ });
54598
+ }
54475
54599
  }
54476
- nodes.push({
54477
- key: targetKey(operation.target),
54478
- dependencyKeys: depKeys(operation.dependencyTargets),
54479
- kind: "operation",
54480
- type: "operation",
54481
- target: operation.target,
54482
- action: undefined
54483
- });
54484
- }
54485
- for (const imp of compiledGraph.imports || []) {
54486
- if (!imp.target) {
54487
- continue;
54600
+ else if (f.parsed.type) {
54601
+ targetSchemas.add((_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset);
54488
54602
  }
54489
- nodes.push({
54490
- key: targetKey(imp.target),
54491
- dependencyKeys: depKeys(imp.dependencyTargets),
54492
- kind: "import",
54493
- type: "import",
54494
- target: imp.target,
54495
- action: undefined
54496
- });
54497
54603
  }
54498
- for (const script of compiledGraph.scripts || []) {
54499
- if (!script.target) {
54604
+ report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
54605
+ report.overlappingSchemas = [...declarationSchemas].filter(s => targetSchemas.has(s)).sort();
54606
+ for (const rel of files) {
54607
+ const abs = path__namespace.join(srcDir, rel);
54608
+ if (rel === "workflow_settings.yaml" || rel === "dataform.json") {
54500
54609
  continue;
54501
54610
  }
54502
- nodes.push({
54503
- key: targetKey(script.target),
54504
- dependencyKeys: depKeys(script.dependencyTargets),
54505
- kind: "script",
54506
- type: "script",
54507
- target: script.target,
54508
- script,
54509
- action: undefined
54510
- });
54511
- }
54512
- const ordered = topoOrder(nodes);
54513
- const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
54514
- const statusByKey = new Map();
54515
- const results = [];
54516
- try {
54517
- for (const schema of shadowSchemas) {
54518
- await deps.execute(deps.createSchemaSql(schema));
54519
- }
54520
- for (const node of ordered) {
54521
- if (node.kind === "operation" ||
54522
- node.kind === "import" ||
54523
- (node.kind === "script" && !deps.checkScript)) {
54524
- statusByKey.set(node.key, "SKIPPED");
54525
- results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
54526
- continue;
54527
- }
54528
- if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
54529
- statusByKey.set(node.key, "BLOCKED");
54530
- results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
54531
- continue;
54532
- }
54533
- const evaluations = node.kind === "script"
54534
- ? await deps.checkScript(node.script)
54535
- : await deps.evaluate(node.action);
54536
- const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
54537
- const status = failed ? "FAILURE" : "PASS";
54538
- statusByKey.set(node.key, status);
54539
- results.push({ target: node.target, type: node.type, status, errors: evaluations });
54540
- if (status === "PASS" && node.kind === "table") {
54541
- try {
54542
- await deps.execute(deps.validationStubSql(node.table));
54543
- }
54544
- catch (e) {
54545
- }
54611
+ if (rel.endsWith(".sqlx")) {
54612
+ const info = sqlxFiles.find(f => f.rel === rel);
54613
+ if (info.parsed.type === "declaration") {
54614
+ const dataset = (_e = info.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset;
54615
+ const project = resolveDatabase(info.parsed.database);
54616
+ const connection = connectionNameFor(project);
54617
+ const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
54618
+ writer.write(rel, content);
54619
+ report.files.push({
54620
+ file: rel,
54621
+ action: "declaration",
54622
+ type: "declaration",
54623
+ status: "rewritten",
54624
+ findings,
54625
+ connection,
54626
+ });
54546
54627
  }
54547
- }
54548
- }
54549
- finally {
54550
- if (!options.keepShadow) {
54551
- for (const schema of shadowSchemas.slice().reverse()) {
54552
- try {
54553
- await deps.execute(deps.dropSchemaCascadeSql(schema));
54554
- }
54555
- catch (e) {
54556
- }
54628
+ else {
54629
+ const { content, findings } = convertTarget(info.source, false);
54630
+ writer.write(rel, content);
54631
+ report.files.push({
54632
+ file: rel,
54633
+ action: "target",
54634
+ type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
54635
+ status: findings.length === 0
54636
+ ? "clean"
54637
+ : findings.some(f => f.kind === "flag")
54638
+ ? "flagged"
54639
+ : "rewritten",
54640
+ findings,
54641
+ });
54557
54642
  }
54643
+ continue;
54558
54644
  }
54559
- }
54560
- return results;
54561
- }
54562
-
54563
- class SupabaseDbAdapter extends PostgresDbAdapter {
54564
- static async create(credentials, options) {
54565
- const sslMode = (credentials.sslMode || "").toLowerCase();
54566
- const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
54567
- const clientConfig = {
54568
- host: credentials.host,
54569
- port: credentials.port,
54570
- database: credentials.database,
54571
- user: credentials.user,
54572
- password: credentials.password,
54573
- ssl: sslEnabled
54574
- ? {
54575
- rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
54576
- }
54577
- : false
54578
- };
54579
- const queryExecutor = new PgPoolExecutor(clientConfig, options);
54580
- try {
54581
- await queryExecutor.verifyConnection();
54582
- }
54583
- catch (e) {
54584
- await queryExecutor.close().catch(() => undefined);
54585
- throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
54586
- `as "${credentials.user}": ${e.message}`, e);
54645
+ if (rel.endsWith(".js") && (rel.startsWith("definitions/") || rel.startsWith("includes/"))) {
54646
+ report.inventory.jsFiles++;
54647
+ const source = fs__namespace$1.readFileSync(abs, "utf8");
54648
+ const { content, findings } = convertTarget(source, true);
54649
+ writer.write(rel, content);
54650
+ const isIncludes = rel.startsWith("includes/");
54651
+ if (isIncludes && findings.length > 0) {
54652
+ report.warnings.push(`includes helper ${rel} emits BigQuery-flavored SQL — review its ${findings.length} finding(s)`);
54653
+ }
54654
+ report.files.push({
54655
+ file: rel,
54656
+ action: isIncludes ? "includes" : "target",
54657
+ type: "js",
54658
+ status: findings.length === 0
54659
+ ? "clean"
54660
+ : findings.some(f => f.kind === "flag")
54661
+ ? "flagged"
54662
+ : "rewritten",
54663
+ findings,
54664
+ });
54665
+ continue;
54587
54666
  }
54588
- return new SupabaseDbAdapter(queryExecutor);
54667
+ writer.write(rel, fs__namespace$1.readFileSync(abs));
54668
+ report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
54589
54669
  }
54590
- constructor(queryExecutor) {
54591
- super(queryExecutor);
54592
- }
54593
- async verifyServiceRolePermissions() {
54594
- var _a;
54595
- try {
54596
- const result = await this.execute("select current_user");
54597
- const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
54598
- return !!currentUser;
54599
- }
54600
- catch (_b) {
54601
- return false;
54602
- }
54670
+ const settings = {
54671
+ warehouse: "supabase",
54672
+ defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
54673
+ defaultAssertionDataset: ((_g = sourceConfig.defaultAssertionDataset) !== null && _g !== void 0 ? _g : "sqlanvil_assertions").toLowerCase(),
54674
+ sqlanvilCoreVersion: (_h = opts.coreVersion) !== null && _h !== void 0 ? _h : version,
54675
+ };
54676
+ if (sourceConfig.vars)
54677
+ settings.vars = sourceConfig.vars;
54678
+ if (report.connections.length > 0) {
54679
+ settings.connections = Object.fromEntries(report.connections.map(c => {
54680
+ var _a;
54681
+ return [
54682
+ c.name,
54683
+ {
54684
+ platform: "bigquery",
54685
+ project: c.project,
54686
+ billingProject: (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "REPLACE_WITH_YOUR_GCP_PROJECT",
54687
+ mode: "runner-extract",
54688
+ },
54689
+ ];
54690
+ }));
54603
54691
  }
54604
- async validatePublicationState(pubName = "supabase_realtime") {
54605
- try {
54606
- const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
54607
- params: [pubName]
54608
- });
54609
- return result.rows.length > 0;
54610
- }
54611
- catch (_a) {
54612
- return false;
54613
- }
54692
+ writer.write("workflow_settings.yaml", jsYaml.dump(settings));
54693
+ report.skippedForSafety = [...skippedForSafety].sort();
54694
+ if (skippedForSafety.size > 0) {
54695
+ report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
54614
54696
  }
54697
+ writer.write("migration-report.json", JSON.stringify(report, null, 2));
54698
+ writer.write("migration-report.md", renderReportMd(report));
54699
+ return report;
54700
+ }
54701
+ function renderReportMd(r) {
54702
+ const targets = r.files.filter(f => f.action === "target");
54703
+ const flagged = targets.filter(f => f.status === "flagged");
54704
+ const rewritten = targets.filter(f => f.status === "rewritten");
54705
+ const clean = targets.filter(f => f.status === "clean");
54706
+ const declarations = r.files.filter(f => f.action === "declaration");
54707
+ const lines = [];
54708
+ lines.push(`# Dataform → SQLAnvil migration report`);
54709
+ lines.push("");
54710
+ lines.push(`Generated by ${r.generator} from \`${r.sourceDir}\` (source untouched).`);
54711
+ lines.push("");
54712
+ lines.push(`## Inventory`);
54713
+ lines.push("");
54714
+ lines.push(`- ${r.inventory.sqlxFiles} .sqlx, ${r.inventory.jsFiles} .js`);
54715
+ for (const [t, n] of Object.entries(r.inventory.byType).sort((a, b) => b[1] - a[1])) {
54716
+ lines.push(` - ${t}: ${n}`);
54717
+ }
54718
+ lines.push("");
54719
+ lines.push(`## Source connections (${r.connections.length})`);
54720
+ lines.push("");
54721
+ lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
54722
+ `(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
54723
+ `before it can extract; scaffold them with \`sqlanvil introspect\`:`);
54724
+ lines.push("");
54725
+ for (const c of r.connections) {
54726
+ lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
54727
+ `${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
54728
+ (c.datasets.length > 8 ? ", …" : ""));
54729
+ lines.push(` - \`${c.introspectExample}\``);
54730
+ }
54731
+ if (r.overlappingSchemas.length > 0) {
54732
+ lines.push("");
54733
+ lines.push(`## ⚠ Schemas that are BOTH source and target`);
54734
+ lines.push("");
54735
+ lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
54736
+ .map(s => `\`${s}\``)
54737
+ .join(", ")}. ` +
54738
+ `Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
54739
+ `producers move to Postgres, convert the declarations to plain local declarations instead ` +
54740
+ `of BigQuery connections.`);
54741
+ }
54742
+ lines.push("");
54743
+ lines.push(`## Target files (${targets.length})`);
54744
+ lines.push("");
54745
+ lines.push(`- clean: ${clean.length}`);
54746
+ lines.push(`- rewritten (safe lexical changes only): ${rewritten.length}`);
54747
+ lines.push(`- **flagged (need dialect review): ${flagged.length}** — every location is marked ` +
54748
+ `inline with \`SQLANVIL-MIGRATE:\``);
54749
+ lines.push("");
54750
+ for (const f of flagged) {
54751
+ lines.push(`- \`${f.file}\` (${f.type})`);
54752
+ for (const finding of f.findings.filter(x => x.kind === "flag").slice(0, 6)) {
54753
+ lines.push(` - L${finding.line}: ${finding.note}`);
54754
+ }
54755
+ const more = f.findings.filter(x => x.kind === "flag").length - 6;
54756
+ if (more > 0)
54757
+ lines.push(` - …and ${more} more (see migration-report.json)`);
54758
+ }
54759
+ lines.push("");
54760
+ lines.push(`## Declarations (${declarations.length})`);
54761
+ lines.push("");
54762
+ lines.push(`Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
54763
+ if (r.warnings.length > 0) {
54764
+ lines.push("");
54765
+ lines.push(`## Warnings`);
54766
+ lines.push("");
54767
+ for (const w of r.warnings)
54768
+ lines.push(`- ${w}`);
54769
+ }
54770
+ lines.push("");
54771
+ lines.push(`## Finish the migration`);
54772
+ lines.push("");
54773
+ lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
54774
+ lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
54775
+ lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
54776
+ `model is the migration to-do list. Work the flagged files until validate is green.`);
54777
+ lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
54778
+ `the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
54779
+ lines.push("");
54780
+ return lines.join("\n");
54615
54781
  }
54616
54782
 
54617
54783
  function actuallyResolve(...filePaths) {
@@ -54819,6 +54985,20 @@ function getTestInput(questionText) {
54819
54985
  throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
54820
54986
  }
54821
54987
  }
54988
+ function passwordQuestion(questionText) {
54989
+ return prompt(questionText, {
54990
+ hideEchoBack: true,
54991
+ mask: ""
54992
+ });
54993
+ }
54994
+ function interactivePasswordQuestion(questionText) {
54995
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
54996
+ const testInput = getTestInput(questionText);
54997
+ print(`${questionText} ********\n`);
54998
+ return testInput;
54999
+ }
55000
+ return passwordQuestion(questionText);
55001
+ }
54822
55002
  function selectionQuestion(questionText, options) {
54823
55003
  return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
54824
55004
  cancel: false
@@ -54961,162 +55141,809 @@ function printExecutionGraph(executionGraph, asJson) {
54961
55141
  operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
54962
55142
  }
54963
55143
  }
54964
- }
54965
- function printExecutedAction(executedAction, executionAction, dryRun) {
54966
- const jobIds = executedAction.tasks
54967
- .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
54968
- .map(task => task.metadata.bigquery.jobId);
54969
- const bytesBilled = executedAction.tasks
54970
- .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
54971
- .map(task => {
54972
- var _a, _b, _c, _d;
54973
- const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
54974
- return formatBytesInHumanReadableFormat(bytes);
54975
- });
54976
- const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
54977
- switch (executedAction.status) {
54978
- case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
54979
- switch (executionAction.type) {
54980
- case "table": {
54981
- writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
54982
- return;
54983
- }
54984
- case "assertion": {
54985
- writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
54986
- return;
54987
- }
54988
- case "operation": {
54989
- writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
54990
- return;
54991
- }
54992
- }
55144
+ }
55145
+ function printExecutedAction(executedAction, executionAction, dryRun) {
55146
+ const jobIds = executedAction.tasks
55147
+ .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55148
+ .map(task => task.metadata.bigquery.jobId);
55149
+ const bytesBilled = executedAction.tasks
55150
+ .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55151
+ .map(task => {
55152
+ var _a, _b, _c, _d;
55153
+ const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
55154
+ return formatBytesInHumanReadableFormat(bytes);
55155
+ });
55156
+ const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
55157
+ switch (executedAction.status) {
55158
+ case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
55159
+ switch (executionAction.type) {
55160
+ case "table": {
55161
+ writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55162
+ return;
55163
+ }
55164
+ case "assertion": {
55165
+ writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55166
+ return;
55167
+ }
55168
+ case "operation": {
55169
+ writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55170
+ return;
55171
+ }
55172
+ }
55173
+ }
55174
+ case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
55175
+ switch (executionAction.type) {
55176
+ case "table": {
55177
+ writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55178
+ break;
55179
+ }
55180
+ case "assertion": {
55181
+ writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55182
+ break;
55183
+ }
55184
+ case "operation": {
55185
+ writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55186
+ break;
55187
+ }
55188
+ }
55189
+ printExecutedActionErrors(executedAction, executionAction);
55190
+ return;
55191
+ }
55192
+ case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
55193
+ switch (executionAction.type) {
55194
+ case "table": {
55195
+ writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55196
+ return;
55197
+ }
55198
+ case "assertion": {
55199
+ writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55200
+ return;
55201
+ }
55202
+ case "operation": {
55203
+ writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55204
+ return;
55205
+ }
55206
+ }
55207
+ return;
55208
+ }
55209
+ case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
55210
+ switch (executionAction.type) {
55211
+ case "table": {
55212
+ writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55213
+ return;
55214
+ }
55215
+ case "assertion": {
55216
+ writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55217
+ return;
55218
+ }
55219
+ case "operation": {
55220
+ writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55221
+ return;
55222
+ }
55223
+ }
55224
+ return;
55225
+ }
55226
+ }
55227
+ }
55228
+ function printFormatFilesResult(formatResults) {
55229
+ const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
55230
+ const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
55231
+ const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
55232
+ const failedFormatResults = sorted.filter(result => !!result.err);
55233
+ if (successfulFormatResults.length > 0) {
55234
+ printSuccess("Successfully formatted:");
55235
+ successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
55236
+ }
55237
+ if (needsFormattingResults.length > 0) {
55238
+ printError("Files that need formatting:");
55239
+ needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
55240
+ }
55241
+ if (failedFormatResults.length > 0) {
55242
+ printError("Errors encountered during formatting:");
55243
+ failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
55244
+ }
55245
+ }
55246
+ function datasetString(target, datasetType, disabled) {
55247
+ return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
55248
+ }
55249
+ function assertionString(target, disabled) {
55250
+ return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55251
+ }
55252
+ function operationString(target, disabled) {
55253
+ return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55254
+ }
55255
+ function plainTargetString(target) {
55256
+ return `${target.schema}.${target.name}`;
55257
+ }
55258
+ function targetString(target) {
55259
+ return calloutOutput(`${target.schema}.${target.name}`);
55260
+ }
55261
+ function dotRepresentation(graph, interactive) {
55262
+ var _a, _b, _c;
55263
+ const nodes = [];
55264
+ const edges = [];
55265
+ const formatTarget = interactive ? targetString : plainTargetString;
55266
+ (_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
55267
+ var _a;
55268
+ const nodeName = `${formatTarget(table.target)}`;
55269
+ nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
55270
+ (_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55271
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55272
+ });
55273
+ });
55274
+ (_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
55275
+ var _a;
55276
+ const nodeName = `${formatTarget(assertion.target)}`;
55277
+ nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
55278
+ (_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55279
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55280
+ });
55281
+ });
55282
+ (_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
55283
+ var _a;
55284
+ const nodeName = `${formatTarget(operation.target)}`;
55285
+ nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
55286
+ (_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55287
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55288
+ });
55289
+ });
55290
+ return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
55291
+ }
55292
+ function printExecutedActionErrors(executedAction, executionAction) {
55293
+ const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
55294
+ failingTasks.forEach((task, i) => {
55295
+ executionAction.tasks[i].statement.split("\n").forEach(line => {
55296
+ writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
55297
+ });
55298
+ printError(task.errorMessage, 1);
55299
+ });
55300
+ }
55301
+
55302
+ const INIT_MODE_QUESTION = "Start a fresh project, or convert an existing Dataform project? (fresh/convert)";
55303
+ const INIT_WAREHOUSE_QUESTION = "Which warehouse? (supabase/postgres/bigquery/mysql)";
55304
+ const INIT_PROJECT_DIR_QUESTION = "Project directory?";
55305
+ const INIT_BQ_PROJECT_QUESTION = "Google Cloud project ID (the default project models build in)?";
55306
+ const INIT_BQ_LOCATION_QUESTION = "Default BigQuery location (e.g. US, us-central1)?";
55307
+ const INIT_DEFAULT_SCHEMA_QUESTION = "Default schema for models?";
55308
+ const INIT_INCLUDE_SAMPLE_QUESTION = "Include the sample project (sources -> staging views -> outputs -> assertion)?";
55309
+ const INIT_INCLUDE_BQ_SOURCE_QUESTION = "Include the cross-warehouse BigQuery sample source (Google public ZIP data)?";
55310
+ const INIT_CONFIGURE_CREDS_QUESTION = `Configure warehouse credentials now (written to the gitignored ${CREDENTIALS_FILENAME})?`;
55311
+ const CREDS_HOST_QUESTION = "Host?";
55312
+ const CREDS_PORT_QUESTION = "Port?";
55313
+ const CREDS_DATABASE_QUESTION = "Database?";
55314
+ const CREDS_USER_QUESTION = "User?";
55315
+ const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
55316
+ const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
55317
+ const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
55318
+ const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
55319
+ const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
55320
+ "aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
55321
+ "db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
55322
+ const MAX_ATTEMPTS = 20;
55323
+ function withDefault(question, defaultValue) {
55324
+ return defaultValue ? `${question} [${defaultValue}]` : question;
55325
+ }
55326
+ function ask(question, defaultValue) {
55327
+ const answer = interactiveQuestion(withDefault(question, defaultValue)).trim();
55328
+ return answer === "" ? defaultValue !== null && defaultValue !== void 0 ? defaultValue : "" : answer;
55329
+ }
55330
+ function askRequired(question) {
55331
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55332
+ const answer = ask(question);
55333
+ if (answer !== "") {
55334
+ return answer;
55335
+ }
55336
+ printError("A value is required.");
55337
+ }
55338
+ throw new Error(`No answer provided for: ${question}`);
55339
+ }
55340
+ function askChoice(question, choices, defaultValue) {
55341
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55342
+ const answer = ask(question, defaultValue).toLowerCase();
55343
+ if (choices.includes(answer)) {
55344
+ return answer;
55345
+ }
55346
+ printError(`Please answer one of: ${choices.join(", ")}.`);
55347
+ }
55348
+ throw new Error(`No valid answer provided for: ${question}`);
55349
+ }
55350
+ function askYesNo(question, defaultValue) {
55351
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55352
+ const answer = ask(question, defaultValue ? "y" : "n").toLowerCase();
55353
+ if (answer === "y" || answer === "yes") {
55354
+ return true;
55355
+ }
55356
+ if (answer === "n" || answer === "no") {
55357
+ return false;
55358
+ }
55359
+ printError("Please answer y or n.");
55360
+ }
55361
+ throw new Error(`No valid answer provided for: ${question}`);
55362
+ }
55363
+ function askInt(question, defaultValue) {
55364
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55365
+ const value = parseInt(ask(question, String(defaultValue)), 10);
55366
+ if (!isNaN(value)) {
55367
+ return value;
55368
+ }
55369
+ printError("Entered value must be an integer.");
55370
+ }
55371
+ throw new Error(`No valid answer provided for: ${question}`);
55372
+ }
55373
+ function collectCredentials(warehouse) {
55374
+ if (warehouse === "mysql") {
55375
+ const creds = {
55376
+ host: ask(CREDS_HOST_QUESTION, "localhost"),
55377
+ port: askInt(CREDS_PORT_QUESTION, 3306),
55378
+ database: ask(CREDS_DATABASE_QUESTION, "sqlanvil"),
55379
+ user: ask(CREDS_USER_QUESTION, "root"),
55380
+ password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
55381
+ sslMode: ask(CREDS_SSLMODE_QUESTION, "disable")
55382
+ };
55383
+ return `${JSON.stringify(creds, null, 2)}\n`;
55384
+ }
55385
+ const isSupabase = warehouse === "supabase";
55386
+ if (isSupabase) {
55387
+ print(SUPABASE_POOLER_HINT);
55388
+ }
55389
+ const host = ask(CREDS_HOST_QUESTION, isSupabase ? "aws-1-<region>.pooler.supabase.com" : "localhost");
55390
+ if (/^db\..+\.supabase\.co$/.test(host)) {
55391
+ printError("Warning: that is the DIRECT Supabase host, which is IPv6-only — connections fail with " +
55392
+ "ENOTFOUND on most networks. Prefer the Session pooler host (Dashboard -> Connect).");
55393
+ }
55394
+ const creds = {
55395
+ host,
55396
+ port: askInt(CREDS_PORT_QUESTION, 5432),
55397
+ database: ask(CREDS_DATABASE_QUESTION, "postgres"),
55398
+ user: ask(CREDS_USER_QUESTION, isSupabase ? "postgres.<your-project-ref>" : "postgres"),
55399
+ password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
55400
+ sslMode: ask(CREDS_SSLMODE_QUESTION, isSupabase ? "require" : "disable"),
55401
+ defaultSchema: "public"
55402
+ };
55403
+ return `${JSON.stringify(creds, null, 2)}\n`;
55404
+ }
55405
+ function printMigrationSummary(report, outDir) {
55406
+ const targets = report.files.filter(f => f.action === "target");
55407
+ const flagged = targets.filter(f => f.status === "flagged").length;
55408
+ printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
55409
+ `${report.connections.length} source connection(s) over ` +
55410
+ `${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
55411
+ `${targets.length} target file(s), ${flagged} flagged for dialect review.`);
55412
+ print(`Report: ${path__namespace.join(outDir, "migration-report.md")}`);
55413
+ }
55414
+ function ensureGitignoreCoversCredentials(dir) {
55415
+ const gitignorePath = path__namespace.join(dir, ".gitignore");
55416
+ const existing = fs__namespace$1.existsSync(gitignorePath) ? fs__namespace$1.readFileSync(gitignorePath, "utf8") : "";
55417
+ if (existing.includes(".df-credentials")) {
55418
+ return;
55419
+ }
55420
+ const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
55421
+ fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
55422
+ }
55423
+ async function runConvertFlow() {
55424
+ let srcDir = "";
55425
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55426
+ const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
55427
+ if (fs__namespace$1.existsSync(candidate)) {
55428
+ srcDir = candidate;
55429
+ break;
55430
+ }
55431
+ printError(`${candidate} does not exist.`);
55432
+ }
55433
+ if (!srcDir) {
55434
+ throw new Error("No usable source directory provided.");
55435
+ }
55436
+ const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
55437
+ print("\nConverting...\n");
55438
+ const report = await migrateDataform({ srcDir, outDir });
55439
+ printMigrationSummary(report, outDir);
55440
+ if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
55441
+ const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
55442
+ ensureGitignoreCoversCredentials(outDir);
55443
+ fs__namespace$1.writeFileSync(credentialsPath, collectCredentials("supabase"));
55444
+ printSuccess(`Credentials written to ${credentialsPath}`);
55445
+ }
55446
+ print("\nNext steps:");
55447
+ print(` 1. sqlanvil compile ${outDir}`);
55448
+ print(` 2. Review ${path__namespace.join(outDir, "migration-report.md")} — BigQuery source connections ` +
55449
+ `need columnTypes (the introspect commands are in the report).`);
55450
+ print(` 3. sqlanvil validate ${outDir} (PASS/FAILURE/BLOCKED per model = the migration to-do list)`);
55451
+ return 0;
55452
+ }
55453
+ async function runFreshFlow(defaultProjectDir) {
55454
+ const warehouse = askChoice(INIT_WAREHOUSE_QUESTION, ["supabase", "postgres", "bigquery", "mysql"], "supabase");
55455
+ const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
55456
+ const projectDir = actuallyResolve(ask(INIT_PROJECT_DIR_QUESTION, defaultProjectDir));
55457
+ const projectConfig = { warehouse };
55458
+ if (warehouse === "bigquery") {
55459
+ projectConfig.defaultDatabase = askRequired(INIT_BQ_PROJECT_QUESTION);
55460
+ projectConfig.defaultLocation = askRequired(INIT_BQ_LOCATION_QUESTION);
55461
+ }
55462
+ projectConfig.defaultSchema = ask(INIT_DEFAULT_SCHEMA_QUESTION, isPostgresLike ? "public" : "sqlanvil");
55463
+ const includeSample = askYesNo(INIT_INCLUDE_SAMPLE_QUESTION, true);
55464
+ const includeBigQuerySource = includeSample && isPostgresLike ? askYesNo(INIT_INCLUDE_BQ_SOURCE_QUESTION, true) : undefined;
55465
+ let credentialsJson;
55466
+ if (warehouse !== "bigquery" && askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
55467
+ credentialsJson = collectCredentials(warehouse);
55468
+ }
55469
+ print("\nWriting project files...\n");
55470
+ const result = await init(projectDir, projectConfig, {
55471
+ includeSample,
55472
+ includeBigQuerySource,
55473
+ credentialsJson
55474
+ });
55475
+ printInitResult(result);
55476
+ const steps = [];
55477
+ if (warehouse === "bigquery") {
55478
+ steps.push(`sqlanvil init-creds ${projectDir} (BigQuery credentials)`);
55479
+ }
55480
+ else if (!credentialsJson) {
55481
+ steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
55482
+ }
55483
+ if (includeSample) {
55484
+ steps.push("Point definitions/sources/app_orders.sqlx at a real table in your warehouse.");
55485
+ }
55486
+ if (isPostgresLike && includeBigQuerySource) {
55487
+ steps.push("Replace REPLACE_WITH_YOUR_GCP_PROJECT in workflow_settings.yaml (billingProject for " +
55488
+ "the BigQuery sample source).");
55489
+ }
55490
+ steps.push(`sqlanvil compile ${projectDir}`);
55491
+ steps.push(`sqlanvil validate ${projectDir} (checks every model against the warehouse without executing)`);
55492
+ print("\nNext steps:");
55493
+ steps.forEach((step, index) => print(` ${index + 1}. ${step}`));
55494
+ return 0;
55495
+ }
55496
+ async function runInteractiveInit(defaultProjectDir) {
55497
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS === undefined) {
55498
+ printError("init --interactive needs an interactive terminal (stdin is not a TTY). " +
55499
+ "Use the non-interactive form instead: sqlanvil init [project-dir] --warehouse=... " +
55500
+ '(see "sqlanvil help init").');
55501
+ return 1;
55502
+ }
55503
+ const mode = askChoice(INIT_MODE_QUESTION, ["fresh", "convert"], "fresh");
55504
+ if (mode === "convert") {
55505
+ return runConvertFlow();
55506
+ }
55507
+ return runFreshFlow(defaultProjectDir);
55508
+ }
55509
+
55510
+ const CHECKER_SOURCE = `
55511
+ import json, os, sys
55512
+
55513
+ script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
55514
+ errors = []
55515
+
55516
+ import platform
55517
+ py_version = platform.python_version()
55518
+
55519
+ def load_packaging():
55520
+ try:
55521
+ from packaging.specifiers import SpecifierSet
55522
+ from packaging.requirements import Requirement
55523
+ return SpecifierSet, Requirement
55524
+ except ImportError:
55525
+ pass
55526
+ try:
55527
+ from pip._vendor.packaging.specifiers import SpecifierSet
55528
+ from pip._vendor.packaging.requirements import Requirement
55529
+ return SpecifierSet, Requirement
55530
+ except ImportError:
55531
+ return None, None
55532
+
55533
+ SpecifierSet, Requirement = load_packaging()
55534
+
55535
+ def mini_parse(v):
55536
+ import re
55537
+ return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
55538
+
55539
+ def mini_check(version, spec):
55540
+ import re
55541
+ ok = True
55542
+ for clause in spec.split(","):
55543
+ clause = clause.strip()
55544
+ m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
55545
+ if not m:
55546
+ return None
55547
+ op, want = m.group(1), m.group(2).strip()
55548
+ if want.endswith(".*"):
55549
+ prefix = mini_parse(want[:-2])
55550
+ have = mini_parse(version)[:len(prefix)]
55551
+ eq = have == prefix
55552
+ if (op in ("==", "===") and not eq) or (op == "!=" and eq):
55553
+ ok = False
55554
+ continue
55555
+ a, b = mini_parse(version), mini_parse(want)
55556
+ n = max(len(a), len(b))
55557
+ a = a + [0] * (n - len(a))
55558
+ b = b + [0] * (n - len(b))
55559
+ if op in ("==", "==="):
55560
+ res = a == b
55561
+ elif op == "!=":
55562
+ res = a != b
55563
+ elif op == "<=":
55564
+ res = a <= b
55565
+ elif op == ">=":
55566
+ res = a >= b
55567
+ elif op == "<":
55568
+ res = a < b
55569
+ elif op == ">":
55570
+ res = a > b
55571
+ else: # ~= : at least b, within b's series
55572
+ wb = mini_parse(want)
55573
+ res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
55574
+ if not res:
55575
+ ok = False
55576
+ return ok
55577
+
55578
+ def check_version(version, spec, what):
55579
+ if SpecifierSet is not None:
55580
+ try:
55581
+ if version not in SpecifierSet(spec, prereleases=True):
55582
+ errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
55583
+ return
55584
+ except Exception as e:
55585
+ errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
55586
+ return
55587
+ if mini_check(version, spec) is False:
55588
+ errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
55589
+
55590
+ if version_spec:
55591
+ check_version(py_version, version_spec, "python interpreter")
55592
+
55593
+ if requirements_path:
55594
+ from importlib import metadata
55595
+ try:
55596
+ lines = open(requirements_path).read().splitlines()
55597
+ except OSError as e:
55598
+ errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
55599
+ lines = []
55600
+ for raw in lines:
55601
+ line = raw.split("#")[0].strip()
55602
+ if not line or line.startswith("-"):
55603
+ continue
55604
+ if Requirement is not None:
55605
+ try:
55606
+ req = Requirement(line)
55607
+ except Exception:
55608
+ errors.append('requirements: cannot parse "%s"' % line)
55609
+ continue
55610
+ if req.marker is not None and not req.marker.evaluate():
55611
+ continue
55612
+ name, spec = req.name, str(req.specifier)
55613
+ else:
55614
+ import re
55615
+ if ";" in line:
55616
+ line = line.split(";")[0].strip()
55617
+ m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
55618
+ if not m:
55619
+ errors.append('requirements: cannot parse "%s"' % line)
55620
+ continue
55621
+ name, spec = m.group(1), (m.group(3) or "").strip()
55622
+ try:
55623
+ installed = metadata.version(name)
55624
+ except metadata.PackageNotFoundError:
55625
+ errors.append(
55626
+ "requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
55627
+ )
55628
+ continue
55629
+ if spec:
55630
+ check_version(installed, spec, "requirements: %s" % name)
55631
+
55632
+ try:
55633
+ with open(script_path, "rb") as f:
55634
+ compile(f.read(), script_path, "exec")
55635
+ except SyntaxError as e:
55636
+ errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
55637
+ except (OSError, ValueError) as e:
55638
+ errors.append("syntax: cannot compile %s: %s" % (script_path, e))
55639
+
55640
+ print(json.dumps({"interpreter_version": py_version, "errors": errors}))
55641
+ `;
55642
+ const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
55643
+ childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
55644
+ if (err) {
55645
+ reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
55646
+ }
55647
+ else {
55648
+ resolve(stdout);
55649
+ }
55650
+ });
55651
+ });
55652
+ const failure = (message) => ({
55653
+ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
55654
+ error: { message }
55655
+ });
55656
+ async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
55657
+ if (script.language !== "python") {
55658
+ return [failure(`no validator for script language "${script.language}"`)];
55659
+ }
55660
+ let interpreter;
55661
+ try {
55662
+ interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
55663
+ }
55664
+ catch (e) {
55665
+ return [failure(e.message)];
55666
+ }
55667
+ let stdout;
55668
+ try {
55669
+ stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
55670
+ }
55671
+ catch (e) {
55672
+ return [failure(e.message)];
55673
+ }
55674
+ let parsed;
55675
+ try {
55676
+ parsed = JSON.parse(stdout);
55677
+ }
55678
+ catch (e) {
55679
+ return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
55680
+ }
55681
+ if (parsed.errors && parsed.errors.length > 0) {
55682
+ return parsed.errors.map(failure);
55683
+ }
55684
+ return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
55685
+ }
55686
+
55687
+ function targetKey(target) {
55688
+ return [target.database, target.schema, target.name].filter(Boolean).join(".");
55689
+ }
55690
+ function topoOrder(nodes) {
55691
+ const byKey = new Map(nodes.map(n => [n.key, n]));
55692
+ const indegree = new Map();
55693
+ const dependents = new Map();
55694
+ for (const n of nodes) {
55695
+ const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
55696
+ indegree.set(n.key, inGraphDeps.length);
55697
+ for (const dep of inGraphDeps) {
55698
+ dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
55699
+ }
55700
+ }
55701
+ const ready = nodes
55702
+ .filter(n => (indegree.get(n.key) || 0) === 0)
55703
+ .map(n => n.key)
55704
+ .sort();
55705
+ const ordered = [];
55706
+ const emitted = new Set();
55707
+ while (ready.length) {
55708
+ const key = ready.shift();
55709
+ if (emitted.has(key)) {
55710
+ continue;
55711
+ }
55712
+ emitted.add(key);
55713
+ ordered.push(byKey.get(key));
55714
+ for (const dep of (dependents.get(key) || []).slice().sort()) {
55715
+ indegree.set(dep, (indegree.get(dep) || 0) - 1);
55716
+ if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
55717
+ ready.push(dep);
55718
+ ready.sort();
55719
+ }
55720
+ }
55721
+ }
55722
+ if (ordered.length < nodes.length) {
55723
+ for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
55724
+ if (!emitted.has(n.key)) {
55725
+ emitted.add(n.key);
55726
+ ordered.push(n);
55727
+ }
55728
+ }
55729
+ }
55730
+ return ordered;
55731
+ }
55732
+ function dependencyBlocked(dependencyKeys, statusByKey) {
55733
+ return dependencyKeys.some(dep => {
55734
+ const status = statusByKey.get(dep);
55735
+ return status !== undefined && status !== "PASS";
55736
+ });
55737
+ }
55738
+ const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
55739
+ function validateShadowSuffix(nowMs) {
55740
+ return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
55741
+ }
55742
+ function parseShadowTimestamp(schemaName) {
55743
+ const match = schemaName.match(/sqlanvil_validate_(\d+)/);
55744
+ return match ? Number(match[1]) : null;
55745
+ }
55746
+ function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
55747
+ return schemaNames.filter(name => {
55748
+ const ts = parseShadowTimestamp(name);
55749
+ return ts !== null && nowMs - ts > maxAgeMs;
55750
+ });
55751
+ }
55752
+
55753
+ const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
55754
+ async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
55755
+ try {
55756
+ const schemas = await deps.listSchemas();
55757
+ for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
55758
+ try {
55759
+ await deps.execute(deps.dropSchemaCascadeSql(schema));
55760
+ }
55761
+ catch (e) {
55762
+ }
55763
+ }
55764
+ }
55765
+ catch (e) {
55766
+ }
55767
+ }
55768
+ function tableType(enumType) {
55769
+ switch (enumType) {
55770
+ case sqlanvil.TableType.VIEW:
55771
+ return "view";
55772
+ case sqlanvil.TableType.INCREMENTAL:
55773
+ return "incremental";
55774
+ default:
55775
+ return "table";
55776
+ }
55777
+ }
55778
+ function depKeys(deps) {
55779
+ return (deps || []).map(targetKey);
55780
+ }
55781
+ async function validate(compiledGraph, deps, options = {}) {
55782
+ const nodes = [];
55783
+ for (const table of compiledGraph.tables || []) {
55784
+ nodes.push({
55785
+ key: targetKey(table.target),
55786
+ dependencyKeys: depKeys(table.dependencyTargets),
55787
+ kind: "table",
55788
+ type: tableType(table.enumType),
55789
+ target: table.target,
55790
+ table,
55791
+ action: table
55792
+ });
55793
+ }
55794
+ for (const assertion of compiledGraph.assertions || []) {
55795
+ nodes.push({
55796
+ key: targetKey(assertion.target),
55797
+ dependencyKeys: depKeys(assertion.dependencyTargets),
55798
+ kind: "assertion",
55799
+ type: "assertion",
55800
+ target: assertion.target,
55801
+ action: assertion
55802
+ });
55803
+ }
55804
+ for (const operation of compiledGraph.operations || []) {
55805
+ if (!operation.target) {
55806
+ continue;
55807
+ }
55808
+ nodes.push({
55809
+ key: targetKey(operation.target),
55810
+ dependencyKeys: depKeys(operation.dependencyTargets),
55811
+ kind: "operation",
55812
+ type: "operation",
55813
+ target: operation.target,
55814
+ action: undefined
55815
+ });
55816
+ }
55817
+ for (const imp of compiledGraph.imports || []) {
55818
+ if (!imp.target) {
55819
+ continue;
55820
+ }
55821
+ nodes.push({
55822
+ key: targetKey(imp.target),
55823
+ dependencyKeys: depKeys(imp.dependencyTargets),
55824
+ kind: "import",
55825
+ type: "import",
55826
+ target: imp.target,
55827
+ action: undefined
55828
+ });
55829
+ }
55830
+ for (const script of compiledGraph.scripts || []) {
55831
+ if (!script.target) {
55832
+ continue;
55833
+ }
55834
+ nodes.push({
55835
+ key: targetKey(script.target),
55836
+ dependencyKeys: depKeys(script.dependencyTargets),
55837
+ kind: "script",
55838
+ type: "script",
55839
+ target: script.target,
55840
+ script,
55841
+ action: undefined
55842
+ });
55843
+ }
55844
+ const ordered = topoOrder(nodes);
55845
+ const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
55846
+ const statusByKey = new Map();
55847
+ const results = [];
55848
+ try {
55849
+ for (const schema of shadowSchemas) {
55850
+ await deps.execute(deps.createSchemaSql(schema));
54993
55851
  }
54994
- case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
54995
- switch (executionAction.type) {
54996
- case "table": {
54997
- writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
54998
- break;
54999
- }
55000
- case "assertion": {
55001
- writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55002
- break;
55003
- }
55004
- case "operation": {
55005
- writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55006
- break;
55007
- }
55852
+ for (const node of ordered) {
55853
+ if (node.kind === "operation" ||
55854
+ node.kind === "import" ||
55855
+ (node.kind === "script" && !deps.checkScript)) {
55856
+ statusByKey.set(node.key, "SKIPPED");
55857
+ results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
55858
+ continue;
55008
55859
  }
55009
- printExecutedActionErrors(executedAction, executionAction);
55010
- return;
55011
- }
55012
- case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
55013
- switch (executionAction.type) {
55014
- case "table": {
55015
- writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55016
- return;
55017
- }
55018
- case "assertion": {
55019
- writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55020
- return;
55860
+ if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
55861
+ statusByKey.set(node.key, "BLOCKED");
55862
+ results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
55863
+ continue;
55864
+ }
55865
+ const evaluations = node.kind === "script"
55866
+ ? await deps.checkScript(node.script)
55867
+ : await deps.evaluate(node.action);
55868
+ const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
55869
+ const status = failed ? "FAILURE" : "PASS";
55870
+ statusByKey.set(node.key, status);
55871
+ results.push({ target: node.target, type: node.type, status, errors: evaluations });
55872
+ if (status === "PASS" && node.kind === "table") {
55873
+ try {
55874
+ await deps.execute(deps.validationStubSql(node.table));
55021
55875
  }
55022
- case "operation": {
55023
- writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55024
- return;
55876
+ catch (e) {
55025
55877
  }
55026
55878
  }
55027
- return;
55028
55879
  }
55029
- case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
55030
- switch (executionAction.type) {
55031
- case "table": {
55032
- writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55033
- return;
55034
- }
55035
- case "assertion": {
55036
- writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55037
- return;
55880
+ }
55881
+ finally {
55882
+ if (!options.keepShadow) {
55883
+ for (const schema of shadowSchemas.slice().reverse()) {
55884
+ try {
55885
+ await deps.execute(deps.dropSchemaCascadeSql(schema));
55038
55886
  }
55039
- case "operation": {
55040
- writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55041
- return;
55887
+ catch (e) {
55042
55888
  }
55043
55889
  }
55044
- return;
55045
55890
  }
55046
55891
  }
55892
+ return results;
55047
55893
  }
55048
- function printFormatFilesResult(formatResults) {
55049
- const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
55050
- const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
55051
- const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
55052
- const failedFormatResults = sorted.filter(result => !!result.err);
55053
- if (successfulFormatResults.length > 0) {
55054
- printSuccess("Successfully formatted:");
55055
- successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
55056
- }
55057
- if (needsFormattingResults.length > 0) {
55058
- printError("Files that need formatting:");
55059
- needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
55894
+
55895
+ class SupabaseDbAdapter extends PostgresDbAdapter {
55896
+ static async create(credentials, options) {
55897
+ const sslMode = (credentials.sslMode || "").toLowerCase();
55898
+ const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
55899
+ const clientConfig = {
55900
+ host: credentials.host,
55901
+ port: credentials.port,
55902
+ database: credentials.database,
55903
+ user: credentials.user,
55904
+ password: credentials.password,
55905
+ ssl: sslEnabled
55906
+ ? {
55907
+ rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
55908
+ }
55909
+ : false
55910
+ };
55911
+ const queryExecutor = new PgPoolExecutor(clientConfig, options);
55912
+ try {
55913
+ await queryExecutor.verifyConnection();
55914
+ }
55915
+ catch (e) {
55916
+ await queryExecutor.close().catch(() => undefined);
55917
+ throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
55918
+ `as "${credentials.user}": ${e.message}`, e);
55919
+ }
55920
+ return new SupabaseDbAdapter(queryExecutor);
55060
55921
  }
55061
- if (failedFormatResults.length > 0) {
55062
- printError("Errors encountered during formatting:");
55063
- failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
55922
+ constructor(queryExecutor) {
55923
+ super(queryExecutor);
55064
55924
  }
55065
- }
55066
- function datasetString(target, datasetType, disabled) {
55067
- return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
55068
- }
55069
- function assertionString(target, disabled) {
55070
- return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55071
- }
55072
- function operationString(target, disabled) {
55073
- return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55074
- }
55075
- function plainTargetString(target) {
55076
- return `${target.schema}.${target.name}`;
55077
- }
55078
- function targetString(target) {
55079
- return calloutOutput(`${target.schema}.${target.name}`);
55080
- }
55081
- function dotRepresentation(graph, interactive) {
55082
- var _a, _b, _c;
55083
- const nodes = [];
55084
- const edges = [];
55085
- const formatTarget = interactive ? targetString : plainTargetString;
55086
- (_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
55087
- var _a;
55088
- const nodeName = `${formatTarget(table.target)}`;
55089
- nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
55090
- (_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55091
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55092
- });
55093
- });
55094
- (_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
55095
- var _a;
55096
- const nodeName = `${formatTarget(assertion.target)}`;
55097
- nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
55098
- (_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55099
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55100
- });
55101
- });
55102
- (_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
55925
+ async verifyServiceRolePermissions() {
55103
55926
  var _a;
55104
- const nodeName = `${formatTarget(operation.target)}`;
55105
- nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
55106
- (_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55107
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55108
- });
55109
- });
55110
- return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
55111
- }
55112
- function printExecutedActionErrors(executedAction, executionAction) {
55113
- const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
55114
- failingTasks.forEach((task, i) => {
55115
- executionAction.tasks[i].statement.split("\n").forEach(line => {
55116
- writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
55117
- });
55118
- printError(task.errorMessage, 1);
55119
- });
55927
+ try {
55928
+ const result = await this.execute("select current_user");
55929
+ const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
55930
+ return !!currentUser;
55931
+ }
55932
+ catch (_b) {
55933
+ return false;
55934
+ }
55935
+ }
55936
+ async validatePublicationState(pubName = "supabase_realtime") {
55937
+ try {
55938
+ const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
55939
+ params: [pubName]
55940
+ });
55941
+ return result.rows.length > 0;
55942
+ }
55943
+ catch (_a) {
55944
+ return false;
55945
+ }
55946
+ }
55120
55947
  }
55121
55948
 
55122
55949
  function getBigQueryCredentials() {
@@ -55990,6 +56817,18 @@ const icebergOption = option("iceberg", {
55990
56817
  type: "boolean",
55991
56818
  default: false
55992
56819
  });
56820
+ const bareOption = option("bare", {
56821
+ describe: "Skip the sample project files — scaffold bare (gitkept) directories only.",
56822
+ type: "boolean",
56823
+ default: false
56824
+ });
56825
+ const interactiveOption = option("interactive", {
56826
+ describe: "Guided Q&A setup: start a fresh project (warehouse, sample project, credentials) or " +
56827
+ "convert an existing Dataform project. Other init arguments are ignored except " +
56828
+ "[project-dir], which seeds the directory prompt.",
56829
+ type: "boolean",
56830
+ default: false
56831
+ });
55993
56832
  const warehouseOption = option("warehouse", {
55994
56833
  describe: "Target warehouse for the new project.",
55995
56834
  type: "string",
@@ -56239,14 +57078,17 @@ function runCli() {
56239
57078
  {
56240
57079
  format: `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` +
56241
57080
  ` [${ProjectConfigOptions.defaultLocation.name}]`,
56242
- description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB).",
57081
+ description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB). " +
57082
+ "Use --interactive for a guided setup, including converting a Dataform project.",
56243
57083
  positionalOptions: [
56244
57084
  projectDirOption,
56245
57085
  positionalOption(ProjectConfigOptions.defaultDatabase.name, {
56246
57086
  describe: "The default database to use, equivalent to Google Cloud Project ID."
56247
57087
  }, (argv) => {
56248
57088
  const warehouse = argv[warehouseOption.name] || "bigquery";
56249
- if (warehouse === "bigquery" && !argv[ProjectConfigOptions.defaultDatabase.name]) {
57089
+ if (warehouse === "bigquery" &&
57090
+ !argv[interactiveOption.name] &&
57091
+ !argv[ProjectConfigOptions.defaultDatabase.name]) {
56250
57092
  throw new Error(`The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` +
56251
57093
  `required for BigQuery projects. Use "sqlanvil help init" for more info.`);
56252
57094
  }
@@ -56256,15 +57098,20 @@ function runCli() {
56256
57098
  "https://cloud.google.com/bigquery/docs/locations for supported values."
56257
57099
  }, (argv) => {
56258
57100
  const warehouse = argv[warehouseOption.name] || "bigquery";
56259
- if (warehouse === "bigquery" && !argv[ProjectConfigOptions.defaultLocation.name]) {
57101
+ if (warehouse === "bigquery" &&
57102
+ !argv[interactiveOption.name] &&
57103
+ !argv[ProjectConfigOptions.defaultLocation.name]) {
56260
57104
  throw new Error(`The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` +
56261
57105
  `required for BigQuery projects. Use "sqlanvil help init" for more info.`);
56262
57106
  }
56263
57107
  })
56264
57108
  ],
56265
- options: [warehouseOption, icebergOption],
57109
+ options: [warehouseOption, icebergOption, bareOption, interactiveOption],
56266
57110
  processFn: async (argv) => {
56267
57111
  const projectDir = argv[projectDirOption.name];
57112
+ if (argv[interactiveOption.name]) {
57113
+ return runInteractiveInit(projectDir);
57114
+ }
56268
57115
  const warehouse = argv[warehouseOption.name] || "bigquery";
56269
57116
  const projectConfig = { warehouse };
56270
57117
  if (warehouse === "bigquery") {
@@ -56278,7 +57125,9 @@ function runCli() {
56278
57125
  }
56279
57126
  }
56280
57127
  print("Writing project files...\n");
56281
- const initResult = await init(projectDir, projectConfig);
57128
+ const initResult = await init(projectDir, projectConfig, {
57129
+ includeSample: !argv[bareOption.name]
57130
+ });
56282
57131
  printInitResult(initResult);
56283
57132
  return 0;
56284
57133
  }
@@ -56843,6 +57692,31 @@ function runCli() {
56843
57692
  }
56844
57693
  return 0;
56845
57694
  }
57695
+ },
57696
+ {
57697
+ format: "migrate-dataform <source-dir> <out-dir>",
57698
+ description: "Convert a Dataform/BigQuery project into a sqlanvil-on-Postgres project. The source " +
57699
+ "directory is READ-ONLY; the converted project + migration-report.{md,json} land in " +
57700
+ "<out-dir> (must be empty). Sources stay in BigQuery as named connections; target SQL " +
57701
+ "gets safe rewrites + inline SQLANVIL-MIGRATE markers for dialect review.",
57702
+ positionalOptions: [
57703
+ positionalOption("source-dir", {
57704
+ describe: "The Dataform project to convert (never modified)."
57705
+ }),
57706
+ positionalOption("out-dir", {
57707
+ describe: "Where the converted sqlanvil project is written (created; must be empty)."
57708
+ })
57709
+ ],
57710
+ options: [],
57711
+ processFn: async (argv) => {
57712
+ const report = await migrateDataform({
57713
+ srcDir: argv["source-dir"],
57714
+ outDir: argv["out-dir"]
57715
+ });
57716
+ printMigrationSummary(report, argv["out-dir"]);
57717
+ print(`Next: sqlanvil compile ${argv["out-dir"]}`);
57718
+ return 0;
57719
+ }
56846
57720
  }
56847
57721
  ]
56848
57722
  })