@sqlanvil/cli 1.22.0 → 1.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/bundle.js +1089 -843
  2. package/package.json +1 -1
package/bundle.js CHANGED
@@ -50606,7 +50606,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
50606
50606
  .filter(validationQuery => !!validationQuery.query);
50607
50607
  }
50608
50608
 
50609
- const version = "1.22.0";
50609
+ const version = "1.23.0";
50610
50610
  const dataformVersion = "3.0.61";
50611
50611
 
50612
50612
  async function build(compiledGraph, runConfig, dbadapter) {
@@ -51016,7 +51016,8 @@ function mysqlCredentialsTemplate() {
51016
51016
  };
51017
51017
  return `${JSON.stringify(template, null, 2)}\n`;
51018
51018
  }
51019
- async function init(projectDir, projectConfig) {
51019
+ async function init(projectDir, projectConfig, options = {}) {
51020
+ var _a;
51020
51021
  const workflowSettingsYamlPath = path__namespace.join(projectDir, "workflow_settings.yaml");
51021
51022
  const packageJsonPath = path__namespace.join(projectDir, "package.json");
51022
51023
  const gitignorePath = path__namespace.join(projectDir, ".gitignore");
@@ -51064,7 +51065,11 @@ async function init(projectDir, projectConfig) {
51064
51065
  if (projectConfig.builtinAssertionNamePrefix) {
51065
51066
  workflowSettings.builtinAssertionNamePrefix = projectConfig.builtinAssertionNamePrefix;
51066
51067
  }
51067
- if (isPostgresLike) {
51068
+ const includeSample = options.includeSample !== false;
51069
+ const withBigQuerySource = includeSample &&
51070
+ warehouse !== "mysql" &&
51071
+ (!isPostgresLike || options.includeBigQuerySource !== false);
51072
+ if (isPostgresLike && withBigQuerySource) {
51068
51073
  workflowSettings.connections = {
51069
51074
  bigquery_public: sqlanvil.ConnectionConfig.create({
51070
51075
  platform: "bigquery",
@@ -51080,7 +51085,7 @@ async function init(projectDir, projectConfig) {
51080
51085
  fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
51081
51086
  filesWritten.push(gitignorePath);
51082
51087
  if (!isBigQuery) {
51083
- fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse));
51088
+ fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
51084
51089
  filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
51085
51090
  }
51086
51091
  const mkdir = (...segments) => {
@@ -51101,7 +51106,10 @@ async function init(projectDir, projectConfig) {
51101
51106
  const reportingDir = mkdir("definitions", "outputs", "reporting");
51102
51107
  const testDir = mkdir("definitions", "test");
51103
51108
  keep(mkdir("includes"));
51104
- const withBigQuerySource = warehouse !== "mysql";
51109
+ if (!includeSample) {
51110
+ [sourcesDir, intermediateDir, salesDir, reportingDir, testDir].forEach(keep);
51111
+ return { filesWritten, dirsCreated };
51112
+ }
51105
51113
  write(path__namespace.join(sourcesDir, "app_orders.sqlx"), `-- A declaration: a table that already exists in your warehouse (created by your
51106
51114
  -- application, an import, or another pipeline). Point name/dataset at a real table —
51107
51115
  -- downstream models read it with \${ref("app_orders")}.
@@ -54772,948 +54780,1170 @@ function renderReportMd(r) {
54772
54780
  return lines.join("\n");
54773
54781
  }
54774
54782
 
54775
- const CHECKER_SOURCE = `
54776
- import json, os, sys
54777
-
54778
- script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
54779
- errors = []
54780
-
54781
- import platform
54782
- py_version = platform.python_version()
54783
-
54784
- def load_packaging():
54785
- try:
54786
- from packaging.specifiers import SpecifierSet
54787
- from packaging.requirements import Requirement
54788
- return SpecifierSet, Requirement
54789
- except ImportError:
54790
- pass
54791
- try:
54792
- from pip._vendor.packaging.specifiers import SpecifierSet
54793
- from pip._vendor.packaging.requirements import Requirement
54794
- return SpecifierSet, Requirement
54795
- except ImportError:
54796
- return None, None
54797
-
54798
- SpecifierSet, Requirement = load_packaging()
54799
-
54800
- def mini_parse(v):
54801
- import re
54802
- return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
54803
-
54804
- def mini_check(version, spec):
54805
- import re
54806
- ok = True
54807
- for clause in spec.split(","):
54808
- clause = clause.strip()
54809
- m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
54810
- if not m:
54811
- return None
54812
- op, want = m.group(1), m.group(2).strip()
54813
- if want.endswith(".*"):
54814
- prefix = mini_parse(want[:-2])
54815
- have = mini_parse(version)[:len(prefix)]
54816
- eq = have == prefix
54817
- if (op in ("==", "===") and not eq) or (op == "!=" and eq):
54818
- ok = False
54819
- continue
54820
- a, b = mini_parse(version), mini_parse(want)
54821
- n = max(len(a), len(b))
54822
- a = a + [0] * (n - len(a))
54823
- b = b + [0] * (n - len(b))
54824
- if op in ("==", "==="):
54825
- res = a == b
54826
- elif op == "!=":
54827
- res = a != b
54828
- elif op == "<=":
54829
- res = a <= b
54830
- elif op == ">=":
54831
- res = a >= b
54832
- elif op == "<":
54833
- res = a < b
54834
- elif op == ">":
54835
- res = a > b
54836
- else: # ~= : at least b, within b's series
54837
- wb = mini_parse(want)
54838
- res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
54839
- if not res:
54840
- ok = False
54841
- return ok
54842
-
54843
- def check_version(version, spec, what):
54844
- if SpecifierSet is not None:
54845
- try:
54846
- if version not in SpecifierSet(spec, prereleases=True):
54847
- errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
54848
- return
54849
- except Exception as e:
54850
- errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
54851
- return
54852
- if mini_check(version, spec) is False:
54853
- errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
54854
-
54855
- if version_spec:
54856
- check_version(py_version, version_spec, "python interpreter")
54857
-
54858
- if requirements_path:
54859
- from importlib import metadata
54860
- try:
54861
- lines = open(requirements_path).read().splitlines()
54862
- except OSError as e:
54863
- errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
54864
- lines = []
54865
- for raw in lines:
54866
- line = raw.split("#")[0].strip()
54867
- if not line or line.startswith("-"):
54868
- continue
54869
- if Requirement is not None:
54870
- try:
54871
- req = Requirement(line)
54872
- except Exception:
54873
- errors.append('requirements: cannot parse "%s"' % line)
54874
- continue
54875
- if req.marker is not None and not req.marker.evaluate():
54876
- continue
54877
- name, spec = req.name, str(req.specifier)
54878
- else:
54879
- import re
54880
- if ";" in line:
54881
- line = line.split(";")[0].strip()
54882
- m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
54883
- if not m:
54884
- errors.append('requirements: cannot parse "%s"' % line)
54885
- continue
54886
- name, spec = m.group(1), (m.group(3) or "").strip()
54887
- try:
54888
- installed = metadata.version(name)
54889
- except metadata.PackageNotFoundError:
54890
- errors.append(
54891
- "requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
54892
- )
54893
- continue
54894
- if spec:
54895
- check_version(installed, spec, "requirements: %s" % name)
54896
-
54897
- try:
54898
- with open(script_path, "rb") as f:
54899
- compile(f.read(), script_path, "exec")
54900
- except SyntaxError as e:
54901
- errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
54902
- except (OSError, ValueError) as e:
54903
- errors.append("syntax: cannot compile %s: %s" % (script_path, e))
54904
-
54905
- print(json.dumps({"interpreter_version": py_version, "errors": errors}))
54906
- `;
54907
- const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
54908
- childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
54909
- if (err) {
54910
- reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
54911
- }
54912
- else {
54913
- resolve(stdout);
54914
- }
54915
- });
54916
- });
54917
- const failure = (message) => ({
54918
- status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
54919
- error: { message }
54920
- });
54921
- async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
54922
- if (script.language !== "python") {
54923
- return [failure(`no validator for script language "${script.language}"`)];
54924
- }
54925
- let interpreter;
54926
- try {
54927
- interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
54928
- }
54929
- catch (e) {
54930
- return [failure(e.message)];
54931
- }
54932
- let stdout;
54933
- try {
54934
- stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
54935
- }
54936
- catch (e) {
54937
- return [failure(e.message)];
54938
- }
54939
- let parsed;
54940
- try {
54941
- parsed = JSON.parse(stdout);
54783
+ function actuallyResolve(...filePaths) {
54784
+ return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
54785
+ }
54786
+ function assertPathExists(checkPath) {
54787
+ if (!fs__namespace$1.existsSync(checkPath)) {
54788
+ throw new Error(`${checkPath} does not exist!`);
54942
54789
  }
54943
- catch (e) {
54944
- return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
54790
+ }
54791
+ function compiledGraphHasErrors(graph) {
54792
+ var _a, _b;
54793
+ return ((_b = (_a = graph.graphErrors) === null || _a === void 0 ? void 0 : _a.compilationErrors) === null || _b === void 0 ? void 0 : _b.length) > 0;
54794
+ }
54795
+ function formatExecutionSuffix(jobIds, bytesBilled) {
54796
+ const jobMetadataParts = [];
54797
+ if (jobIds.length > 0) {
54798
+ jobMetadataParts.push(`\n \t jobId: ${jobIds.join(", ")}`);
54945
54799
  }
54946
- if (parsed.errors && parsed.errors.length > 0) {
54947
- return parsed.errors.map(failure);
54800
+ if (bytesBilled.length > 0) {
54801
+ jobMetadataParts.push(`\n \t Bytes billed: ${bytesBilled.join(", ")}`);
54948
54802
  }
54949
- return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
54950
- }
54951
-
54952
- function targetKey(target) {
54953
- return [target.database, target.schema, target.name].filter(Boolean).join(".");
54803
+ return jobMetadataParts.length > 0 ? ` ${jobMetadataParts}` : "";
54954
54804
  }
54955
- function topoOrder(nodes) {
54956
- const byKey = new Map(nodes.map(n => [n.key, n]));
54957
- const indegree = new Map();
54958
- const dependents = new Map();
54959
- for (const n of nodes) {
54960
- const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
54961
- indegree.set(n.key, inGraphDeps.length);
54962
- for (const dep of inGraphDeps) {
54963
- dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
54964
- }
54805
+ function formatBytesInHumanReadableFormat(bytes) {
54806
+ if (bytes <= 0) {
54807
+ return '0 B';
54965
54808
  }
54966
- const ready = nodes
54967
- .filter(n => (indegree.get(n.key) || 0) === 0)
54968
- .map(n => n.key)
54969
- .sort();
54970
- const ordered = [];
54971
- const emitted = new Set();
54972
- while (ready.length) {
54973
- const key = ready.shift();
54974
- if (emitted.has(key)) {
54975
- continue;
54976
- }
54977
- emitted.add(key);
54978
- ordered.push(byKey.get(key));
54979
- for (const dep of (dependents.get(key) || []).slice().sort()) {
54980
- indegree.set(dep, (indegree.get(dep) || 0) - 1);
54981
- if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
54982
- ready.push(dep);
54983
- ready.sort();
54809
+ const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
54810
+ const k = 1024;
54811
+ const i = Math.floor(Math.log(bytes) / Math.log(k));
54812
+ const value = (bytes / Math.pow(k, i)).toFixed(2);
54813
+ return `${value} ${units[i]}`;
54814
+ }
54815
+ function promptForIcebergConfig() {
54816
+ print(ICEBERG_CONFIG_PROMPT_TEXT);
54817
+ print(ICEBERG_CONFIG_PROMPT_HINT);
54818
+ const tempIcebergConfig = {};
54819
+ let bucketName;
54820
+ while (true) {
54821
+ print(ICEBERG_BUCKET_NAME_HINT);
54822
+ bucketName = interactiveQuestion(ICEBERG_BUCKET_NAME_PROMPT_QUESTION);
54823
+ try {
54824
+ if (bucketName) {
54825
+ validateIcebergConfigBucketName(bucketName);
54826
+ tempIcebergConfig.bucketName = bucketName;
54984
54827
  }
54828
+ break;
54985
54829
  }
54986
- }
54987
- if (ordered.length < nodes.length) {
54988
- for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
54989
- if (!emitted.has(n.key)) {
54990
- emitted.add(n.key);
54991
- ordered.push(n);
54992
- }
54830
+ catch (e) {
54831
+ printError(`Validation Error: ${e.message}`);
54832
+ print("Please try again.");
54993
54833
  }
54994
54834
  }
54995
- return ordered;
54996
- }
54997
- function dependencyBlocked(dependencyKeys, statusByKey) {
54998
- return dependencyKeys.some(dep => {
54999
- const status = statusByKey.get(dep);
55000
- return status !== undefined && status !== "PASS";
55001
- });
54835
+ let tableFolderRoot;
54836
+ while (true) {
54837
+ print(ICEBERG_TABLE_FOLDER_ROOT_HINT);
54838
+ tableFolderRoot = interactiveQuestion(ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION);
54839
+ try {
54840
+ if (tableFolderRoot) {
54841
+ validateIcebergConfigTableFolderRoot(tableFolderRoot);
54842
+ tempIcebergConfig.tableFolderRoot = tableFolderRoot;
54843
+ }
54844
+ break;
54845
+ }
54846
+ catch (e) {
54847
+ printError(`Validation Error: ${e.message}`);
54848
+ print("Please try again.");
54849
+ }
54850
+ }
54851
+ let tableFolderSubpath;
54852
+ while (true) {
54853
+ print(ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT);
54854
+ tableFolderSubpath = interactiveQuestion(ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION);
54855
+ try {
54856
+ if (tableFolderSubpath) {
54857
+ validateIcebergConfigTableFolderSubpath(tableFolderSubpath);
54858
+ tempIcebergConfig.tableFolderSubpath = tableFolderSubpath;
54859
+ }
54860
+ break;
54861
+ }
54862
+ catch (e) {
54863
+ printError(`Validation Error: ${e.message}`);
54864
+ print("Please try again.");
54865
+ }
54866
+ }
54867
+ let connection;
54868
+ while (true) {
54869
+ print(ICEBERG_CONNECTION_HINT);
54870
+ connection = interactiveQuestion(ICEBERG_CONNECTION_QUESTION);
54871
+ try {
54872
+ if (connection) {
54873
+ validateConnectionFormat(connection);
54874
+ tempIcebergConfig.connection = connection;
54875
+ }
54876
+ break;
54877
+ }
54878
+ catch (e) {
54879
+ printError(`Validation Error: ${e.message}`);
54880
+ print("Please try again.");
54881
+ }
54882
+ }
54883
+ printSuccess(ICEBERG_CONFIG_COLLECTED_TEXT);
54884
+ if (Object.keys(tempIcebergConfig).length > 0) {
54885
+ return tempIcebergConfig;
54886
+ }
54887
+ return undefined;
55002
54888
  }
55003
- const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
55004
- function validateShadowSuffix(nowMs) {
55005
- return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
54889
+ function validateIcebergConfigBucketName(bucketName) {
54890
+ if (bucketName.length < 3 || bucketName.length > 63) {
54891
+ throw new Error("Bucket name must be between 3 and 63 characters long.");
54892
+ }
54893
+ if (!/^[a-z0-9][a-z0-9-._]{1,220}[a-z0-9]$/.test(bucketName) || bucketName.includes("..")) {
54894
+ throw new Error("Invalid bucket name. Must start and end with a letter or number, contain only lowercase letters, numbers, hyphens (-), underscores (_), and periods (.).");
54895
+ }
54896
+ if (bucketName.startsWith("goog") || bucketName.includes("--")) {
54897
+ throw new Error("Bucket name cannot start with 'goog' or contain '--'.");
54898
+ }
54899
+ if (bucketName.includes("google") || bucketName.includes("g00gle")) {
54900
+ throw new Error("Bucket name cannot contain 'google' or close misspellings such as 'g00gle'.");
54901
+ }
55006
54902
  }
55007
- function parseShadowTimestamp(schemaName) {
55008
- const match = schemaName.match(/sqlanvil_validate_(\d+)/);
55009
- return match ? Number(match[1]) : null;
54903
+ function validateIcebergConfigTableFolderRoot(tableFolderRoot) {
54904
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9-._]{1,220}[a-zA-Z0-9]$/.test(tableFolderRoot) || tableFolderRoot.includes("..")) {
54905
+ throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), and periods (.).");
54906
+ }
55010
54907
  }
55011
- function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
55012
- return schemaNames.filter(name => {
55013
- const ts = parseShadowTimestamp(name);
55014
- return ts !== null && nowMs - ts > maxAgeMs;
55015
- });
54908
+ function validateIcebergConfigTableFolderSubpath(tableFolderSubpath) {
54909
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9-._/]{1,220}[a-zA-Z0-9]$/.test(tableFolderSubpath) || tableFolderSubpath.includes("..")) {
54910
+ throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), periods (.), and forward slashes (/). The sequence '..' is also disallowed.");
54911
+ }
54912
+ if (tableFolderSubpath.includes("./") || tableFolderSubpath.includes("../")) {
54913
+ throw new Error("Input cannot contain './' or '../'.");
54914
+ }
55016
54915
  }
54916
+ const ICEBERG_BUCKET_NAME_HINT = "The bucket name must comply with https://cloud.google.com/storage/docs/buckets#naming. If you do not want to provide a workflow-level default bucket name, leave this input empty.\n";
54917
+ const ICEBERG_TABLE_FOLDER_ROOT_HINT = "Table folder root must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores and periods. If you do not want to provide a workflow_level default table folder root, leave this input empty.\n";
54918
+ const ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT = "Table folder subpath must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores, periods and forward slashes. If you do not want to provide a workflow_level default table folder subpath, leave this input empty.\n";
54919
+ const ICEBERG_CONFIG_PROMPT_HINT = "Set repository-level configuration for Iceberg bucket name, table folder root, table folder subpath and connection. If you do not want to set a field, enter an empty string in response to the prompt.\n";
54920
+ const ICEBERG_CONFIG_PROMPT_TEXT = "\n--- Iceberg Configuration ---\n";
54921
+ const ICEBERG_CONFIG_COLLECTED_TEXT = "Iceberg configuration collected.\n";
54922
+ const ICEBERG_BUCKET_NAME_PROMPT_QUESTION = "Enter the default Iceberg bucket name:";
54923
+ const ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION = "Enter the default Iceberg table folder root:";
54924
+ const ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION = "Enter the default Iceberg table folder subpath:";
54925
+ const ICEBERG_CONNECTION_HINT = "The connection can have the form `{project}.{location}.{connection_id}` or `projects/{project}/locations/{location}/connections/{connection_id} or be set to DEFAULT. If you do not want to provide a workflow-level default connection, leave this input empty.\n";
54926
+ const ICEBERG_CONNECTION_QUESTION = "Enter the default connection:";
55017
54927
 
55018
- const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
55019
- async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
54928
+ const noColor = process.env.NO_COLOR && ["1", "true", "yes"].includes(process.env.NO_COLOR.toLowerCase());
54929
+ const ansiColorCodes = Object.freeze({
54930
+ red: 91,
54931
+ green: 32,
54932
+ yellow: 93,
54933
+ cyan: 36
54934
+ });
54935
+ function output(text, ansiColorCode) {
54936
+ if (noColor) {
54937
+ return text;
54938
+ }
54939
+ return `\x1b[${ansiColorCode}m${text}\x1b[0m`;
54940
+ }
54941
+ const successOutput = (text) => output(text, ansiColorCodes.green);
54942
+ const warningOutput = (text) => output(text, ansiColorCodes.yellow);
54943
+ const errorOutput = (text) => output(text, ansiColorCodes.red);
54944
+ const calloutOutput = (text) => output(text, ansiColorCodes.cyan);
54945
+ const write = (stream, text, indentCount) => stream.write(`${" ".repeat(indentCount)}${text}\n`);
54946
+ const writeStdOut = (text, indentCount = 0) => write(process.stdout, text, indentCount);
54947
+ const writeStdErr = (text, indentCount = 0) => write(process.stderr, text, indentCount);
54948
+ const DEFAULT_PROMPT = "> ";
54949
+ function question(questionText) {
54950
+ return prompt(questionText);
54951
+ }
54952
+ function interactiveQuestion(questionText) {
54953
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
54954
+ const testInput = getTestInput(questionText);
54955
+ print(`${questionText} ${testInput}\n`);
54956
+ return testInput;
54957
+ }
54958
+ print(`${questionText} `);
54959
+ return readlineSync__namespace.question("");
54960
+ }
54961
+ function getTestInput(questionText) {
54962
+ const envVar = process.env.DATAFORM_CLI_TEST_INPUTS;
54963
+ if (!envVar) {
54964
+ throw new Error("Environment variable DATAFORM_CLI_TEST_INPUTS not set.");
54965
+ }
55020
54966
  try {
55021
- const schemas = await deps.listSchemas();
55022
- for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
55023
- try {
55024
- await deps.execute(deps.dropSchemaCascadeSql(schema));
54967
+ const parsed = JSON.parse(envVar);
54968
+ if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
54969
+ const inputs = new Map(Object.entries(parsed).map(([key, value]) => [key, String(value)]));
54970
+ const trimmedQuestion = questionText.trim();
54971
+ const answer = inputs.get(trimmedQuestion);
54972
+ if (answer !== undefined) {
54973
+ print(`[TEST_INPUT for "${trimmedQuestion}"]: "${answer}"`);
54974
+ return answer;
55025
54975
  }
55026
- catch (e) {
54976
+ else {
54977
+ throw new Error(`[MISSING TEST_INPUT for "${trimmedQuestion}"]`);
55027
54978
  }
55028
54979
  }
54980
+ else {
54981
+ throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: Expected a JSON object, but got ${typeof parsed}.`);
54982
+ }
55029
54983
  }
55030
54984
  catch (e) {
54985
+ throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
55031
54986
  }
55032
54987
  }
55033
- function tableType(enumType) {
55034
- switch (enumType) {
55035
- case sqlanvil.TableType.VIEW:
55036
- return "view";
55037
- case sqlanvil.TableType.INCREMENTAL:
55038
- return "incremental";
55039
- default:
55040
- return "table";
54988
+ function passwordQuestion(questionText) {
54989
+ return prompt(questionText, {
54990
+ hideEchoBack: true,
54991
+ mask: ""
54992
+ });
54993
+ }
54994
+ function interactivePasswordQuestion(questionText) {
54995
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
54996
+ const testInput = getTestInput(questionText);
54997
+ print(`${questionText} ********\n`);
54998
+ return testInput;
55041
54999
  }
55000
+ return passwordQuestion(questionText);
55042
55001
  }
55043
- function depKeys(deps) {
55044
- return (deps || []).map(targetKey);
55002
+ function selectionQuestion(questionText, options) {
55003
+ return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
55004
+ cancel: false
55005
+ });
55045
55006
  }
55046
- async function validate(compiledGraph, deps, options = {}) {
55047
- const nodes = [];
55048
- for (const table of compiledGraph.tables || []) {
55049
- nodes.push({
55050
- key: targetKey(table.target),
55051
- dependencyKeys: depKeys(table.dependencyTargets),
55052
- kind: "table",
55053
- type: tableType(table.enumType),
55054
- target: table.target,
55055
- table,
55056
- action: table
55057
- });
55007
+ function prompt(questionText, options) {
55008
+ writeStdOut(questionText);
55009
+ return readlineSync__namespace.prompt(Object.assign(Object.assign({}, options), { prompt: (options && options.prompt && options.prompt + DEFAULT_PROMPT) || DEFAULT_PROMPT }));
55010
+ }
55011
+ function print(text) {
55012
+ writeStdOut(text);
55013
+ }
55014
+ function printSuccess(text) {
55015
+ writeStdOut(successOutput(text));
55016
+ }
55017
+ function printError(errorText, indentCount = 0) {
55018
+ writeStdErr(errorOutput(errorText), indentCount);
55019
+ }
55020
+ function printInitResult(result) {
55021
+ if (result.dirsCreated && result.dirsCreated.length) {
55022
+ writeStdOut(successOutput("Directories successfully created:"));
55023
+ result.dirsCreated.forEach(dir => writeStdOut(dir, 1));
55058
55024
  }
55059
- for (const assertion of compiledGraph.assertions || []) {
55060
- nodes.push({
55061
- key: targetKey(assertion.target),
55062
- dependencyKeys: depKeys(assertion.dependencyTargets),
55063
- kind: "assertion",
55064
- type: "assertion",
55065
- target: assertion.target,
55066
- action: assertion
55067
- });
55025
+ if (result.filesWritten && result.filesWritten.length) {
55026
+ writeStdOut(successOutput("Files successfully written:"));
55027
+ result.filesWritten.forEach(file => writeStdOut(file, 1));
55068
55028
  }
55069
- for (const operation of compiledGraph.operations || []) {
55070
- if (!operation.target) {
55071
- continue;
55072
- }
55073
- nodes.push({
55074
- key: targetKey(operation.target),
55075
- dependencyKeys: depKeys(operation.dependencyTargets),
55076
- kind: "operation",
55077
- type: "operation",
55078
- target: operation.target,
55079
- action: undefined
55080
- });
55029
+ }
55030
+ function printInitCredsResult(writtenFilePath) {
55031
+ writeStdOut(successOutput("Credentials file successfully written:"));
55032
+ writeStdOut(writtenFilePath, 1);
55033
+ writeStdOut("To change connection settings, edit this file directly.");
55034
+ }
55035
+ function isInteractive({ stream = process.stdout } = {}) {
55036
+ return Boolean(stream && stream.isTTY &&
55037
+ process.env.TERM !== 'dumb' &&
55038
+ !('CI' in process.env));
55039
+ }
55040
+ var compiledGraphOutputType;
55041
+ (function (compiledGraphOutputType) {
55042
+ compiledGraphOutputType["Json"] = "json";
55043
+ compiledGraphOutputType["Dot"] = "dot";
55044
+ compiledGraphOutputType["Summary"] = "summary";
55045
+ })(compiledGraphOutputType || (compiledGraphOutputType = {}));
55046
+ function printCompiledGraph(graph, outputType, quietCompilation) {
55047
+ const interactive = isInteractive();
55048
+ if (outputType === compiledGraphOutputType.Json) {
55049
+ writeStdOut(prettyJsonStringify(graph));
55081
55050
  }
55082
- for (const imp of compiledGraph.imports || []) {
55083
- if (!imp.target) {
55084
- continue;
55051
+ else if (outputType === compiledGraphOutputType.Dot) {
55052
+ writeStdOut(dotRepresentation(graph, interactive));
55053
+ }
55054
+ else {
55055
+ const actionCount = 0 +
55056
+ (graph.tables ? graph.tables.length : 0) +
55057
+ (graph.assertions ? graph.assertions.length : 0) +
55058
+ (graph.operations ? graph.operations.length : 0);
55059
+ writeStdOut(successOutput(`Compiled ${actionCount} action(s).`));
55060
+ if (graph.tables && graph.tables.length) {
55061
+ graph.tables.forEach(setOrValidateTableEnumType);
55062
+ writeStdOut(`${graph.tables.length} dataset(s)${quietCompilation ? "." : ":"}`);
55063
+ if (!quietCompilation) {
55064
+ graph.tables.forEach(compiledTable => {
55065
+ writeStdOut(`${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`, 1);
55066
+ });
55067
+ }
55068
+ }
55069
+ if (graph.assertions && graph.assertions.length) {
55070
+ writeStdOut(`${graph.assertions.length} assertion(s)${quietCompilation ? "." : ":"}`);
55071
+ if (!quietCompilation) {
55072
+ graph.assertions.forEach(assertion => {
55073
+ writeStdOut(assertionString(assertion.target, assertion.disabled), 1);
55074
+ });
55075
+ }
55076
+ }
55077
+ if (graph.operations && graph.operations.length) {
55078
+ writeStdOut(`${graph.operations.length} operation(s)${quietCompilation ? "." : ":"}`);
55079
+ if (!quietCompilation) {
55080
+ graph.operations.forEach(operation => {
55081
+ writeStdOut(operationString(operation.target, operation.disabled), 1);
55082
+ });
55083
+ }
55085
55084
  }
55086
- nodes.push({
55087
- key: targetKey(imp.target),
55088
- dependencyKeys: depKeys(imp.dependencyTargets),
55089
- kind: "import",
55090
- type: "import",
55091
- target: imp.target,
55092
- action: undefined
55093
- });
55094
55085
  }
55095
- for (const script of compiledGraph.scripts || []) {
55096
- if (!script.target) {
55097
- continue;
55086
+ }
55087
+ function formatStackTraceForQuietCompilation(compileError) {
55088
+ var _a, _b;
55089
+ if ((_a = compileError === null || compileError === void 0 ? void 0 : compileError.message) === null || _a === void 0 ? void 0 : _a.includes("Unexpected identifier")) {
55090
+ if (!compileError.stack) {
55091
+ return "";
55098
55092
  }
55099
- nodes.push({
55100
- key: targetKey(script.target),
55101
- dependencyKeys: depKeys(script.dependencyTargets),
55102
- kind: "script",
55103
- type: "script",
55104
- target: script.target,
55105
- script,
55106
- action: undefined
55093
+ const stackLines = ((_b = compileError.stack) === null || _b === void 0 ? void 0 : _b.split("\n")) || [];
55094
+ return stackLines.slice(0, Math.min(3, stackLines.length)).join("\n");
55095
+ }
55096
+ return "";
55097
+ }
55098
+ function printCompiledGraphErrors(graphErrors, quietCompilation) {
55099
+ if (graphErrors.compilationErrors && graphErrors.compilationErrors.length > 0) {
55100
+ printError("Compilation errors:", 1);
55101
+ graphErrors.compilationErrors.forEach(compileError => {
55102
+ writeStdErr(`${calloutOutput(compileError.fileName)}: ${errorOutput(quietCompilation ? (compileError.message + " " + formatStackTraceForQuietCompilation(compileError) || compileError.stack) : (compileError.stack || compileError.message))}`, 1);
55107
55103
  });
55108
55104
  }
55109
- const ordered = topoOrder(nodes);
55110
- const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
55111
- const statusByKey = new Map();
55112
- const results = [];
55113
- try {
55114
- for (const schema of shadowSchemas) {
55115
- await deps.execute(deps.createSchemaSql(schema));
55116
- }
55117
- for (const node of ordered) {
55118
- if (node.kind === "operation" ||
55119
- node.kind === "import" ||
55120
- (node.kind === "script" && !deps.checkScript)) {
55121
- statusByKey.set(node.key, "SKIPPED");
55122
- results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
55123
- continue;
55105
+ }
55106
+ function printTestResult(testResult) {
55107
+ writeStdOut(`${testResult.name}: ${testResult.successful ? successOutput("passed") : errorOutput("failed")}`);
55108
+ if (!testResult.successful) {
55109
+ testResult.messages.forEach(message => writeStdErr(message, 1));
55110
+ }
55111
+ }
55112
+ function printExecutionGraph(executionGraph, asJson) {
55113
+ if (asJson) {
55114
+ writeStdOut(prettyJsonStringify(executionGraph.toJSON()));
55115
+ }
55116
+ else {
55117
+ const actionsByType = {
55118
+ table: [],
55119
+ assertion: [],
55120
+ operation: []
55121
+ };
55122
+ executionGraph.actions.forEach(action => {
55123
+ if (!(action.type === "table" || action.type === "assertion" || action.type === "operation")) {
55124
+ throw new Error(`Unrecognized action type: ${action.type}`);
55124
55125
  }
55125
- if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
55126
- statusByKey.set(node.key, "BLOCKED");
55127
- results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
55128
- continue;
55126
+ actionsByType[action.type].push(action);
55127
+ });
55128
+ const datasetActions = actionsByType.table;
55129
+ if (datasetActions && datasetActions.length) {
55130
+ writeStdOut(`${datasetActions.length} dataset(s):`);
55131
+ datasetActions.forEach(datasetAction => writeStdOut(datasetString(datasetAction.target, datasetAction.type, datasetAction.tasks.length === 0), 1));
55132
+ }
55133
+ const assertionActions = actionsByType.assertion;
55134
+ if (assertionActions && assertionActions.length) {
55135
+ writeStdOut(`${assertionActions.length} assertion(s):`);
55136
+ assertionActions.forEach(assertionAction => writeStdOut(assertionString(assertionAction.target, assertionAction.tasks.length === 0), 1));
55137
+ }
55138
+ const operationActions = actionsByType.operation;
55139
+ if (operationActions && operationActions.length) {
55140
+ writeStdOut(`${operationActions.length} operation(s):`);
55141
+ operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
55142
+ }
55143
+ }
55144
+ }
55145
+ function printExecutedAction(executedAction, executionAction, dryRun) {
55146
+ const jobIds = executedAction.tasks
55147
+ .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55148
+ .map(task => task.metadata.bigquery.jobId);
55149
+ const bytesBilled = executedAction.tasks
55150
+ .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55151
+ .map(task => {
55152
+ var _a, _b, _c, _d;
55153
+ const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
55154
+ return formatBytesInHumanReadableFormat(bytes);
55155
+ });
55156
+ const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
55157
+ switch (executedAction.status) {
55158
+ case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
55159
+ switch (executionAction.type) {
55160
+ case "table": {
55161
+ writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55162
+ return;
55163
+ }
55164
+ case "assertion": {
55165
+ writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55166
+ return;
55167
+ }
55168
+ case "operation": {
55169
+ writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55170
+ return;
55171
+ }
55129
55172
  }
55130
- const evaluations = node.kind === "script"
55131
- ? await deps.checkScript(node.script)
55132
- : await deps.evaluate(node.action);
55133
- const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
55134
- const status = failed ? "FAILURE" : "PASS";
55135
- statusByKey.set(node.key, status);
55136
- results.push({ target: node.target, type: node.type, status, errors: evaluations });
55137
- if (status === "PASS" && node.kind === "table") {
55138
- try {
55139
- await deps.execute(deps.validationStubSql(node.table));
55173
+ }
55174
+ case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
55175
+ switch (executionAction.type) {
55176
+ case "table": {
55177
+ writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55178
+ break;
55179
+ }
55180
+ case "assertion": {
55181
+ writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55182
+ break;
55140
55183
  }
55141
- catch (e) {
55184
+ case "operation": {
55185
+ writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55186
+ break;
55142
55187
  }
55143
55188
  }
55189
+ printExecutedActionErrors(executedAction, executionAction);
55190
+ return;
55144
55191
  }
55145
- }
55146
- finally {
55147
- if (!options.keepShadow) {
55148
- for (const schema of shadowSchemas.slice().reverse()) {
55149
- try {
55150
- await deps.execute(deps.dropSchemaCascadeSql(schema));
55192
+ case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
55193
+ switch (executionAction.type) {
55194
+ case "table": {
55195
+ writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55196
+ return;
55151
55197
  }
55152
- catch (e) {
55198
+ case "assertion": {
55199
+ writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55200
+ return;
55201
+ }
55202
+ case "operation": {
55203
+ writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55204
+ return;
55153
55205
  }
55154
55206
  }
55207
+ return;
55155
55208
  }
55156
- }
55157
- return results;
55158
- }
55159
-
55160
- class SupabaseDbAdapter extends PostgresDbAdapter {
55161
- static async create(credentials, options) {
55162
- const sslMode = (credentials.sslMode || "").toLowerCase();
55163
- const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
55164
- const clientConfig = {
55165
- host: credentials.host,
55166
- port: credentials.port,
55167
- database: credentials.database,
55168
- user: credentials.user,
55169
- password: credentials.password,
55170
- ssl: sslEnabled
55171
- ? {
55172
- rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
55209
+ case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
55210
+ switch (executionAction.type) {
55211
+ case "table": {
55212
+ writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55213
+ return;
55173
55214
  }
55174
- : false
55175
- };
55176
- const queryExecutor = new PgPoolExecutor(clientConfig, options);
55177
- try {
55178
- await queryExecutor.verifyConnection();
55179
- }
55180
- catch (e) {
55181
- await queryExecutor.close().catch(() => undefined);
55182
- throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
55183
- `as "${credentials.user}": ${e.message}`, e);
55215
+ case "assertion": {
55216
+ writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55217
+ return;
55218
+ }
55219
+ case "operation": {
55220
+ writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55221
+ return;
55222
+ }
55223
+ }
55224
+ return;
55184
55225
  }
55185
- return new SupabaseDbAdapter(queryExecutor);
55186
55226
  }
55187
- constructor(queryExecutor) {
55188
- super(queryExecutor);
55227
+ }
55228
+ function printFormatFilesResult(formatResults) {
55229
+ const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
55230
+ const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
55231
+ const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
55232
+ const failedFormatResults = sorted.filter(result => !!result.err);
55233
+ if (successfulFormatResults.length > 0) {
55234
+ printSuccess("Successfully formatted:");
55235
+ successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
55189
55236
  }
55190
- async verifyServiceRolePermissions() {
55191
- var _a;
55192
- try {
55193
- const result = await this.execute("select current_user");
55194
- const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
55195
- return !!currentUser;
55196
- }
55197
- catch (_b) {
55198
- return false;
55199
- }
55237
+ if (needsFormattingResults.length > 0) {
55238
+ printError("Files that need formatting:");
55239
+ needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
55200
55240
  }
55201
- async validatePublicationState(pubName = "supabase_realtime") {
55202
- try {
55203
- const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
55204
- params: [pubName]
55205
- });
55206
- return result.rows.length > 0;
55207
- }
55208
- catch (_a) {
55209
- return false;
55210
- }
55241
+ if (failedFormatResults.length > 0) {
55242
+ printError("Errors encountered during formatting:");
55243
+ failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
55211
55244
  }
55212
55245
  }
55213
-
55214
- function actuallyResolve(...filePaths) {
55215
- return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
55246
+ function datasetString(target, datasetType, disabled) {
55247
+ return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
55216
55248
  }
55217
- function assertPathExists(checkPath) {
55218
- if (!fs__namespace$1.existsSync(checkPath)) {
55219
- throw new Error(`${checkPath} does not exist!`);
55220
- }
55249
+ function assertionString(target, disabled) {
55250
+ return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55221
55251
  }
55222
- function compiledGraphHasErrors(graph) {
55223
- var _a, _b;
55224
- return ((_b = (_a = graph.graphErrors) === null || _a === void 0 ? void 0 : _a.compilationErrors) === null || _b === void 0 ? void 0 : _b.length) > 0;
55252
+ function operationString(target, disabled) {
55253
+ return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55225
55254
  }
55226
- function formatExecutionSuffix(jobIds, bytesBilled) {
55227
- const jobMetadataParts = [];
55228
- if (jobIds.length > 0) {
55229
- jobMetadataParts.push(`\n \t jobId: ${jobIds.join(", ")}`);
55230
- }
55231
- if (bytesBilled.length > 0) {
55232
- jobMetadataParts.push(`\n \t Bytes billed: ${bytesBilled.join(", ")}`);
55233
- }
55234
- return jobMetadataParts.length > 0 ? ` ${jobMetadataParts}` : "";
55255
+ function plainTargetString(target) {
55256
+ return `${target.schema}.${target.name}`;
55235
55257
  }
55236
- function formatBytesInHumanReadableFormat(bytes) {
55237
- if (bytes <= 0) {
55238
- return '0 B';
55239
- }
55240
- const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
55241
- const k = 1024;
55242
- const i = Math.floor(Math.log(bytes) / Math.log(k));
55243
- const value = (bytes / Math.pow(k, i)).toFixed(2);
55244
- return `${value} ${units[i]}`;
55258
+ function targetString(target) {
55259
+ return calloutOutput(`${target.schema}.${target.name}`);
55245
55260
  }
55246
- function promptForIcebergConfig() {
55247
- print(ICEBERG_CONFIG_PROMPT_TEXT);
55248
- print(ICEBERG_CONFIG_PROMPT_HINT);
55249
- const tempIcebergConfig = {};
55250
- let bucketName;
55251
- while (true) {
55252
- print(ICEBERG_BUCKET_NAME_HINT);
55253
- bucketName = interactiveQuestion(ICEBERG_BUCKET_NAME_PROMPT_QUESTION);
55254
- try {
55255
- if (bucketName) {
55256
- validateIcebergConfigBucketName(bucketName);
55257
- tempIcebergConfig.bucketName = bucketName;
55258
- }
55259
- break;
55260
- }
55261
- catch (e) {
55262
- printError(`Validation Error: ${e.message}`);
55263
- print("Please try again.");
55264
- }
55265
- }
55266
- let tableFolderRoot;
55267
- while (true) {
55268
- print(ICEBERG_TABLE_FOLDER_ROOT_HINT);
55269
- tableFolderRoot = interactiveQuestion(ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION);
55270
- try {
55271
- if (tableFolderRoot) {
55272
- validateIcebergConfigTableFolderRoot(tableFolderRoot);
55273
- tempIcebergConfig.tableFolderRoot = tableFolderRoot;
55274
- }
55275
- break;
55276
- }
55277
- catch (e) {
55278
- printError(`Validation Error: ${e.message}`);
55279
- print("Please try again.");
55280
- }
55281
- }
55282
- let tableFolderSubpath;
55283
- while (true) {
55284
- print(ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT);
55285
- tableFolderSubpath = interactiveQuestion(ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION);
55286
- try {
55287
- if (tableFolderSubpath) {
55288
- validateIcebergConfigTableFolderSubpath(tableFolderSubpath);
55289
- tempIcebergConfig.tableFolderSubpath = tableFolderSubpath;
55290
- }
55291
- break;
55292
- }
55293
- catch (e) {
55294
- printError(`Validation Error: ${e.message}`);
55295
- print("Please try again.");
55296
- }
55297
- }
55298
- let connection;
55299
- while (true) {
55300
- print(ICEBERG_CONNECTION_HINT);
55301
- connection = interactiveQuestion(ICEBERG_CONNECTION_QUESTION);
55302
- try {
55303
- if (connection) {
55304
- validateConnectionFormat(connection);
55305
- tempIcebergConfig.connection = connection;
55306
- }
55307
- break;
55308
- }
55309
- catch (e) {
55310
- printError(`Validation Error: ${e.message}`);
55311
- print("Please try again.");
55261
+ function dotRepresentation(graph, interactive) {
55262
+ var _a, _b, _c;
55263
+ const nodes = [];
55264
+ const edges = [];
55265
+ const formatTarget = interactive ? targetString : plainTargetString;
55266
+ (_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
55267
+ var _a;
55268
+ const nodeName = `${formatTarget(table.target)}`;
55269
+ nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
55270
+ (_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55271
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55272
+ });
55273
+ });
55274
+ (_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
55275
+ var _a;
55276
+ const nodeName = `${formatTarget(assertion.target)}`;
55277
+ nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
55278
+ (_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55279
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55280
+ });
55281
+ });
55282
+ (_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
55283
+ var _a;
55284
+ const nodeName = `${formatTarget(operation.target)}`;
55285
+ nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
55286
+ (_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55287
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55288
+ });
55289
+ });
55290
+ return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
55291
+ }
55292
+ function printExecutedActionErrors(executedAction, executionAction) {
55293
+ const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
55294
+ failingTasks.forEach((task, i) => {
55295
+ executionAction.tasks[i].statement.split("\n").forEach(line => {
55296
+ writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
55297
+ });
55298
+ printError(task.errorMessage, 1);
55299
+ });
55300
+ }
55301
+
55302
+ const INIT_MODE_QUESTION = "Start a fresh project, or convert an existing Dataform project? (fresh/convert)";
55303
+ const INIT_WAREHOUSE_QUESTION = "Which warehouse? (supabase/postgres/bigquery/mysql)";
55304
+ const INIT_PROJECT_DIR_QUESTION = "Project directory?";
55305
+ const INIT_BQ_PROJECT_QUESTION = "Google Cloud project ID (the default project models build in)?";
55306
+ const INIT_BQ_LOCATION_QUESTION = "Default BigQuery location (e.g. US, us-central1)?";
55307
+ const INIT_DEFAULT_SCHEMA_QUESTION = "Default schema for models?";
55308
+ const INIT_INCLUDE_SAMPLE_QUESTION = "Include the sample project (sources -> staging views -> outputs -> assertion)?";
55309
+ const INIT_INCLUDE_BQ_SOURCE_QUESTION = "Include the cross-warehouse BigQuery sample source (Google public ZIP data)?";
55310
+ const INIT_CONFIGURE_CREDS_QUESTION = `Configure warehouse credentials now (written to the gitignored ${CREDENTIALS_FILENAME})?`;
55311
+ const CREDS_HOST_QUESTION = "Host?";
55312
+ const CREDS_PORT_QUESTION = "Port?";
55313
+ const CREDS_DATABASE_QUESTION = "Database?";
55314
+ const CREDS_USER_QUESTION = "User?";
55315
+ const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
55316
+ const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
55317
+ const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
55318
+ const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
55319
+ const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
55320
+ "aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
55321
+ "db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
55322
+ const MAX_ATTEMPTS = 20;
55323
+ function withDefault(question, defaultValue) {
55324
+ return defaultValue ? `${question} [${defaultValue}]` : question;
55325
+ }
55326
+ function ask(question, defaultValue) {
55327
+ const answer = interactiveQuestion(withDefault(question, defaultValue)).trim();
55328
+ return answer === "" ? defaultValue !== null && defaultValue !== void 0 ? defaultValue : "" : answer;
55329
+ }
55330
+ function askRequired(question) {
55331
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55332
+ const answer = ask(question);
55333
+ if (answer !== "") {
55334
+ return answer;
55312
55335
  }
55336
+ printError("A value is required.");
55313
55337
  }
55314
- printSuccess(ICEBERG_CONFIG_COLLECTED_TEXT);
55315
- if (Object.keys(tempIcebergConfig).length > 0) {
55316
- return tempIcebergConfig;
55317
- }
55318
- return undefined;
55338
+ throw new Error(`No answer provided for: ${question}`);
55319
55339
  }
55320
- function validateIcebergConfigBucketName(bucketName) {
55321
- if (bucketName.length < 3 || bucketName.length > 63) {
55322
- throw new Error("Bucket name must be between 3 and 63 characters long.");
55323
- }
55324
- if (!/^[a-z0-9][a-z0-9-._]{1,220}[a-z0-9]$/.test(bucketName) || bucketName.includes("..")) {
55325
- throw new Error("Invalid bucket name. Must start and end with a letter or number, contain only lowercase letters, numbers, hyphens (-), underscores (_), and periods (.).");
55326
- }
55327
- if (bucketName.startsWith("goog") || bucketName.includes("--")) {
55328
- throw new Error("Bucket name cannot start with 'goog' or contain '--'.");
55329
- }
55330
- if (bucketName.includes("google") || bucketName.includes("g00gle")) {
55331
- throw new Error("Bucket name cannot contain 'google' or close misspellings such as 'g00gle'.");
55340
+ function askChoice(question, choices, defaultValue) {
55341
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55342
+ const answer = ask(question, defaultValue).toLowerCase();
55343
+ if (choices.includes(answer)) {
55344
+ return answer;
55345
+ }
55346
+ printError(`Please answer one of: ${choices.join(", ")}.`);
55332
55347
  }
55348
+ throw new Error(`No valid answer provided for: ${question}`);
55333
55349
  }
55334
- function validateIcebergConfigTableFolderRoot(tableFolderRoot) {
55335
- if (!/^[a-zA-Z0-9][a-zA-Z0-9-._]{1,220}[a-zA-Z0-9]$/.test(tableFolderRoot) || tableFolderRoot.includes("..")) {
55336
- throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), and periods (.).");
55350
+ function askYesNo(question, defaultValue) {
55351
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55352
+ const answer = ask(question, defaultValue ? "y" : "n").toLowerCase();
55353
+ if (answer === "y" || answer === "yes") {
55354
+ return true;
55355
+ }
55356
+ if (answer === "n" || answer === "no") {
55357
+ return false;
55358
+ }
55359
+ printError("Please answer y or n.");
55337
55360
  }
55361
+ throw new Error(`No valid answer provided for: ${question}`);
55338
55362
  }
55339
- function validateIcebergConfigTableFolderSubpath(tableFolderSubpath) {
55340
- if (!/^[a-zA-Z0-9][a-zA-Z0-9-._/]{1,220}[a-zA-Z0-9]$/.test(tableFolderSubpath) || tableFolderSubpath.includes("..")) {
55341
- throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), periods (.), and forward slashes (/). The sequence '..' is also disallowed.");
55363
+ function askInt(question, defaultValue) {
55364
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55365
+ const value = parseInt(ask(question, String(defaultValue)), 10);
55366
+ if (!isNaN(value)) {
55367
+ return value;
55368
+ }
55369
+ printError("Entered value must be an integer.");
55342
55370
  }
55343
- if (tableFolderSubpath.includes("./") || tableFolderSubpath.includes("../")) {
55344
- throw new Error("Input cannot contain './' or '../'.");
55371
+ throw new Error(`No valid answer provided for: ${question}`);
55372
+ }
55373
+ function collectCredentials(warehouse) {
55374
+ if (warehouse === "mysql") {
55375
+ const creds = {
55376
+ host: ask(CREDS_HOST_QUESTION, "localhost"),
55377
+ port: askInt(CREDS_PORT_QUESTION, 3306),
55378
+ database: ask(CREDS_DATABASE_QUESTION, "sqlanvil"),
55379
+ user: ask(CREDS_USER_QUESTION, "root"),
55380
+ password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
55381
+ sslMode: ask(CREDS_SSLMODE_QUESTION, "disable")
55382
+ };
55383
+ return `${JSON.stringify(creds, null, 2)}\n`;
55345
55384
  }
55385
+ const isSupabase = warehouse === "supabase";
55386
+ if (isSupabase) {
55387
+ print(SUPABASE_POOLER_HINT);
55388
+ }
55389
+ const host = ask(CREDS_HOST_QUESTION, isSupabase ? "aws-1-<region>.pooler.supabase.com" : "localhost");
55390
+ if (/^db\..+\.supabase\.co$/.test(host)) {
55391
+ printError("Warning: that is the DIRECT Supabase host, which is IPv6-only — connections fail with " +
55392
+ "ENOTFOUND on most networks. Prefer the Session pooler host (Dashboard -> Connect).");
55393
+ }
55394
+ const creds = {
55395
+ host,
55396
+ port: askInt(CREDS_PORT_QUESTION, 5432),
55397
+ database: ask(CREDS_DATABASE_QUESTION, "postgres"),
55398
+ user: ask(CREDS_USER_QUESTION, isSupabase ? "postgres.<your-project-ref>" : "postgres"),
55399
+ password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
55400
+ sslMode: ask(CREDS_SSLMODE_QUESTION, isSupabase ? "require" : "disable"),
55401
+ defaultSchema: "public"
55402
+ };
55403
+ return `${JSON.stringify(creds, null, 2)}\n`;
55346
55404
  }
55347
- const ICEBERG_BUCKET_NAME_HINT = "The bucket name must comply with https://cloud.google.com/storage/docs/buckets#naming. If you do not want to provide a workflow-level default bucket name, leave this input empty.\n";
55348
- const ICEBERG_TABLE_FOLDER_ROOT_HINT = "Table folder root must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores and periods. If you do not want to provide a workflow_level default table folder root, leave this input empty.\n";
55349
- const ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT = "Table folder subpath must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores, periods and forward slashes. If you do not want to provide a workflow_level default table folder subpath, leave this input empty.\n";
55350
- const ICEBERG_CONFIG_PROMPT_HINT = "Set repository-level configuration for Iceberg bucket name, table folder root, table folder subpath and connection. If you do not want to set a field, enter an empty string in response to the prompt.\n";
55351
- const ICEBERG_CONFIG_PROMPT_TEXT = "\n--- Iceberg Configuration ---\n";
55352
- const ICEBERG_CONFIG_COLLECTED_TEXT = "Iceberg configuration collected.\n";
55353
- const ICEBERG_BUCKET_NAME_PROMPT_QUESTION = "Enter the default Iceberg bucket name:";
55354
- const ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION = "Enter the default Iceberg table folder root:";
55355
- const ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION = "Enter the default Iceberg table folder subpath:";
55356
- const ICEBERG_CONNECTION_HINT = "The connection can have the form `{project}.{location}.{connection_id}` or `projects/{project}/locations/{location}/connections/{connection_id} or be set to DEFAULT. If you do not want to provide a workflow-level default connection, leave this input empty.\n";
55357
- const ICEBERG_CONNECTION_QUESTION = "Enter the default connection:";
55358
-
55359
- const noColor = process.env.NO_COLOR && ["1", "true", "yes"].includes(process.env.NO_COLOR.toLowerCase());
55360
- const ansiColorCodes = Object.freeze({
55361
- red: 91,
55362
- green: 32,
55363
- yellow: 93,
55364
- cyan: 36
55365
- });
55366
- function output(text, ansiColorCode) {
55367
- if (noColor) {
55368
- return text;
55405
+ function printMigrationSummary(report, outDir) {
55406
+ const targets = report.files.filter(f => f.action === "target");
55407
+ const flagged = targets.filter(f => f.status === "flagged").length;
55408
+ printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
55409
+ `${report.connections.length} source connection(s) over ` +
55410
+ `${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
55411
+ `${targets.length} target file(s), ${flagged} flagged for dialect review.`);
55412
+ print(`Report: ${path__namespace.join(outDir, "migration-report.md")}`);
55413
+ }
55414
+ function ensureGitignoreCoversCredentials(dir) {
55415
+ const gitignorePath = path__namespace.join(dir, ".gitignore");
55416
+ const existing = fs__namespace$1.existsSync(gitignorePath) ? fs__namespace$1.readFileSync(gitignorePath, "utf8") : "";
55417
+ if (existing.includes(".df-credentials")) {
55418
+ return;
55369
55419
  }
55370
- return `\x1b[${ansiColorCode}m${text}\x1b[0m`;
55420
+ const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
55421
+ fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
55371
55422
  }
55372
- const successOutput = (text) => output(text, ansiColorCodes.green);
55373
- const warningOutput = (text) => output(text, ansiColorCodes.yellow);
55374
- const errorOutput = (text) => output(text, ansiColorCodes.red);
55375
- const calloutOutput = (text) => output(text, ansiColorCodes.cyan);
55376
- const write = (stream, text, indentCount) => stream.write(`${" ".repeat(indentCount)}${text}\n`);
55377
- const writeStdOut = (text, indentCount = 0) => write(process.stdout, text, indentCount);
55378
- const writeStdErr = (text, indentCount = 0) => write(process.stderr, text, indentCount);
55379
- const DEFAULT_PROMPT = "> ";
55380
- function question(questionText) {
55381
- return prompt(questionText);
55423
+ async function runConvertFlow() {
55424
+ let srcDir = "";
55425
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55426
+ const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
55427
+ if (fs__namespace$1.existsSync(candidate)) {
55428
+ srcDir = candidate;
55429
+ break;
55430
+ }
55431
+ printError(`${candidate} does not exist.`);
55432
+ }
55433
+ if (!srcDir) {
55434
+ throw new Error("No usable source directory provided.");
55435
+ }
55436
+ const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
55437
+ print("\nConverting...\n");
55438
+ const report = await migrateDataform({ srcDir, outDir });
55439
+ printMigrationSummary(report, outDir);
55440
+ if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
55441
+ const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
55442
+ ensureGitignoreCoversCredentials(outDir);
55443
+ fs__namespace$1.writeFileSync(credentialsPath, collectCredentials("supabase"));
55444
+ printSuccess(`Credentials written to ${credentialsPath}`);
55445
+ }
55446
+ print("\nNext steps:");
55447
+ print(` 1. sqlanvil compile ${outDir}`);
55448
+ print(` 2. Review ${path__namespace.join(outDir, "migration-report.md")} — BigQuery source connections ` +
55449
+ `need columnTypes (the introspect commands are in the report).`);
55450
+ print(` 3. sqlanvil validate ${outDir} (PASS/FAILURE/BLOCKED per model = the migration to-do list)`);
55451
+ return 0;
55382
55452
  }
55383
- function interactiveQuestion(questionText) {
55384
- if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
55385
- const testInput = getTestInput(questionText);
55386
- print(`${questionText} ${testInput}\n`);
55387
- return testInput;
55453
+ async function runFreshFlow(defaultProjectDir) {
55454
+ const warehouse = askChoice(INIT_WAREHOUSE_QUESTION, ["supabase", "postgres", "bigquery", "mysql"], "supabase");
55455
+ const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
55456
+ const projectDir = actuallyResolve(ask(INIT_PROJECT_DIR_QUESTION, defaultProjectDir));
55457
+ const projectConfig = { warehouse };
55458
+ if (warehouse === "bigquery") {
55459
+ projectConfig.defaultDatabase = askRequired(INIT_BQ_PROJECT_QUESTION);
55460
+ projectConfig.defaultLocation = askRequired(INIT_BQ_LOCATION_QUESTION);
55461
+ }
55462
+ projectConfig.defaultSchema = ask(INIT_DEFAULT_SCHEMA_QUESTION, isPostgresLike ? "public" : "sqlanvil");
55463
+ const includeSample = askYesNo(INIT_INCLUDE_SAMPLE_QUESTION, true);
55464
+ const includeBigQuerySource = includeSample && isPostgresLike ? askYesNo(INIT_INCLUDE_BQ_SOURCE_QUESTION, true) : undefined;
55465
+ let credentialsJson;
55466
+ if (warehouse !== "bigquery" && askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
55467
+ credentialsJson = collectCredentials(warehouse);
55468
+ }
55469
+ print("\nWriting project files...\n");
55470
+ const result = await init(projectDir, projectConfig, {
55471
+ includeSample,
55472
+ includeBigQuerySource,
55473
+ credentialsJson
55474
+ });
55475
+ printInitResult(result);
55476
+ const steps = [];
55477
+ if (warehouse === "bigquery") {
55478
+ steps.push(`sqlanvil init-creds ${projectDir} (BigQuery credentials)`);
55388
55479
  }
55389
- print(`${questionText} `);
55390
- return readlineSync__namespace.question("");
55480
+ else if (!credentialsJson) {
55481
+ steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
55482
+ }
55483
+ if (includeSample) {
55484
+ steps.push("Point definitions/sources/app_orders.sqlx at a real table in your warehouse.");
55485
+ }
55486
+ if (isPostgresLike && includeBigQuerySource) {
55487
+ steps.push("Replace REPLACE_WITH_YOUR_GCP_PROJECT in workflow_settings.yaml (billingProject for " +
55488
+ "the BigQuery sample source).");
55489
+ }
55490
+ steps.push(`sqlanvil compile ${projectDir}`);
55491
+ steps.push(`sqlanvil validate ${projectDir} (checks every model against the warehouse without executing)`);
55492
+ print("\nNext steps:");
55493
+ steps.forEach((step, index) => print(` ${index + 1}. ${step}`));
55494
+ return 0;
55391
55495
  }
55392
- function getTestInput(questionText) {
55393
- const envVar = process.env.DATAFORM_CLI_TEST_INPUTS;
55394
- if (!envVar) {
55395
- throw new Error("Environment variable DATAFORM_CLI_TEST_INPUTS not set.");
55496
+ async function runInteractiveInit(defaultProjectDir) {
55497
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS === undefined) {
55498
+ printError("init --interactive needs an interactive terminal (stdin is not a TTY). " +
55499
+ "Use the non-interactive form instead: sqlanvil init [project-dir] --warehouse=... " +
55500
+ '(see "sqlanvil help init").');
55501
+ return 1;
55396
55502
  }
55397
- try {
55398
- const parsed = JSON.parse(envVar);
55399
- if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
55400
- const inputs = new Map(Object.entries(parsed).map(([key, value]) => [key, String(value)]));
55401
- const trimmedQuestion = questionText.trim();
55402
- const answer = inputs.get(trimmedQuestion);
55403
- if (answer !== undefined) {
55404
- print(`[TEST_INPUT for "${trimmedQuestion}"]: "${answer}"`);
55405
- return answer;
55406
- }
55407
- else {
55408
- throw new Error(`[MISSING TEST_INPUT for "${trimmedQuestion}"]`);
55409
- }
55503
+ const mode = askChoice(INIT_MODE_QUESTION, ["fresh", "convert"], "fresh");
55504
+ if (mode === "convert") {
55505
+ return runConvertFlow();
55506
+ }
55507
+ return runFreshFlow(defaultProjectDir);
55508
+ }
55509
+
55510
+ const CHECKER_SOURCE = `
55511
+ import json, os, sys
55512
+
55513
+ script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
55514
+ errors = []
55515
+
55516
+ import platform
55517
+ py_version = platform.python_version()
55518
+
55519
+ def load_packaging():
55520
+ try:
55521
+ from packaging.specifiers import SpecifierSet
55522
+ from packaging.requirements import Requirement
55523
+ return SpecifierSet, Requirement
55524
+ except ImportError:
55525
+ pass
55526
+ try:
55527
+ from pip._vendor.packaging.specifiers import SpecifierSet
55528
+ from pip._vendor.packaging.requirements import Requirement
55529
+ return SpecifierSet, Requirement
55530
+ except ImportError:
55531
+ return None, None
55532
+
55533
+ SpecifierSet, Requirement = load_packaging()
55534
+
55535
+ def mini_parse(v):
55536
+ import re
55537
+ return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
55538
+
55539
+ def mini_check(version, spec):
55540
+ import re
55541
+ ok = True
55542
+ for clause in spec.split(","):
55543
+ clause = clause.strip()
55544
+ m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
55545
+ if not m:
55546
+ return None
55547
+ op, want = m.group(1), m.group(2).strip()
55548
+ if want.endswith(".*"):
55549
+ prefix = mini_parse(want[:-2])
55550
+ have = mini_parse(version)[:len(prefix)]
55551
+ eq = have == prefix
55552
+ if (op in ("==", "===") and not eq) or (op == "!=" and eq):
55553
+ ok = False
55554
+ continue
55555
+ a, b = mini_parse(version), mini_parse(want)
55556
+ n = max(len(a), len(b))
55557
+ a = a + [0] * (n - len(a))
55558
+ b = b + [0] * (n - len(b))
55559
+ if op in ("==", "==="):
55560
+ res = a == b
55561
+ elif op == "!=":
55562
+ res = a != b
55563
+ elif op == "<=":
55564
+ res = a <= b
55565
+ elif op == ">=":
55566
+ res = a >= b
55567
+ elif op == "<":
55568
+ res = a < b
55569
+ elif op == ">":
55570
+ res = a > b
55571
+ else: # ~= : at least b, within b's series
55572
+ wb = mini_parse(want)
55573
+ res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
55574
+ if not res:
55575
+ ok = False
55576
+ return ok
55577
+
55578
+ def check_version(version, spec, what):
55579
+ if SpecifierSet is not None:
55580
+ try:
55581
+ if version not in SpecifierSet(spec, prereleases=True):
55582
+ errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
55583
+ return
55584
+ except Exception as e:
55585
+ errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
55586
+ return
55587
+ if mini_check(version, spec) is False:
55588
+ errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
55589
+
55590
+ if version_spec:
55591
+ check_version(py_version, version_spec, "python interpreter")
55592
+
55593
+ if requirements_path:
55594
+ from importlib import metadata
55595
+ try:
55596
+ lines = open(requirements_path).read().splitlines()
55597
+ except OSError as e:
55598
+ errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
55599
+ lines = []
55600
+ for raw in lines:
55601
+ line = raw.split("#")[0].strip()
55602
+ if not line or line.startswith("-"):
55603
+ continue
55604
+ if Requirement is not None:
55605
+ try:
55606
+ req = Requirement(line)
55607
+ except Exception:
55608
+ errors.append('requirements: cannot parse "%s"' % line)
55609
+ continue
55610
+ if req.marker is not None and not req.marker.evaluate():
55611
+ continue
55612
+ name, spec = req.name, str(req.specifier)
55613
+ else:
55614
+ import re
55615
+ if ";" in line:
55616
+ line = line.split(";")[0].strip()
55617
+ m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
55618
+ if not m:
55619
+ errors.append('requirements: cannot parse "%s"' % line)
55620
+ continue
55621
+ name, spec = m.group(1), (m.group(3) or "").strip()
55622
+ try:
55623
+ installed = metadata.version(name)
55624
+ except metadata.PackageNotFoundError:
55625
+ errors.append(
55626
+ "requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
55627
+ )
55628
+ continue
55629
+ if spec:
55630
+ check_version(installed, spec, "requirements: %s" % name)
55631
+
55632
+ try:
55633
+ with open(script_path, "rb") as f:
55634
+ compile(f.read(), script_path, "exec")
55635
+ except SyntaxError as e:
55636
+ errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
55637
+ except (OSError, ValueError) as e:
55638
+ errors.append("syntax: cannot compile %s: %s" % (script_path, e))
55639
+
55640
+ print(json.dumps({"interpreter_version": py_version, "errors": errors}))
55641
+ `;
55642
+ const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
55643
+ childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
55644
+ if (err) {
55645
+ reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
55410
55646
  }
55411
55647
  else {
55412
- throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: Expected a JSON object, but got ${typeof parsed}.`);
55648
+ resolve(stdout);
55413
55649
  }
55650
+ });
55651
+ });
55652
+ const failure = (message) => ({
55653
+ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
55654
+ error: { message }
55655
+ });
55656
+ async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
55657
+ if (script.language !== "python") {
55658
+ return [failure(`no validator for script language "${script.language}"`)];
55659
+ }
55660
+ let interpreter;
55661
+ try {
55662
+ interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
55414
55663
  }
55415
55664
  catch (e) {
55416
- throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
55665
+ return [failure(e.message)];
55417
55666
  }
55418
- }
55419
- function selectionQuestion(questionText, options) {
55420
- return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
55421
- cancel: false
55422
- });
55423
- }
55424
- function prompt(questionText, options) {
55425
- writeStdOut(questionText);
55426
- return readlineSync__namespace.prompt(Object.assign(Object.assign({}, options), { prompt: (options && options.prompt && options.prompt + DEFAULT_PROMPT) || DEFAULT_PROMPT }));
55427
- }
55428
- function print(text) {
55429
- writeStdOut(text);
55430
- }
55431
- function printSuccess(text) {
55432
- writeStdOut(successOutput(text));
55433
- }
55434
- function printError(errorText, indentCount = 0) {
55435
- writeStdErr(errorOutput(errorText), indentCount);
55436
- }
55437
- function printInitResult(result) {
55438
- if (result.dirsCreated && result.dirsCreated.length) {
55439
- writeStdOut(successOutput("Directories successfully created:"));
55440
- result.dirsCreated.forEach(dir => writeStdOut(dir, 1));
55667
+ let stdout;
55668
+ try {
55669
+ stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
55441
55670
  }
55442
- if (result.filesWritten && result.filesWritten.length) {
55443
- writeStdOut(successOutput("Files successfully written:"));
55444
- result.filesWritten.forEach(file => writeStdOut(file, 1));
55671
+ catch (e) {
55672
+ return [failure(e.message)];
55445
55673
  }
55674
+ let parsed;
55675
+ try {
55676
+ parsed = JSON.parse(stdout);
55677
+ }
55678
+ catch (e) {
55679
+ return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
55680
+ }
55681
+ if (parsed.errors && parsed.errors.length > 0) {
55682
+ return parsed.errors.map(failure);
55683
+ }
55684
+ return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
55446
55685
  }
55447
- function printInitCredsResult(writtenFilePath) {
55448
- writeStdOut(successOutput("Credentials file successfully written:"));
55449
- writeStdOut(writtenFilePath, 1);
55450
- writeStdOut("To change connection settings, edit this file directly.");
55451
- }
55452
- function isInteractive({ stream = process.stdout } = {}) {
55453
- return Boolean(stream && stream.isTTY &&
55454
- process.env.TERM !== 'dumb' &&
55455
- !('CI' in process.env));
55686
+
55687
+ function targetKey(target) {
55688
+ return [target.database, target.schema, target.name].filter(Boolean).join(".");
55456
55689
  }
55457
- var compiledGraphOutputType;
55458
- (function (compiledGraphOutputType) {
55459
- compiledGraphOutputType["Json"] = "json";
55460
- compiledGraphOutputType["Dot"] = "dot";
55461
- compiledGraphOutputType["Summary"] = "summary";
55462
- })(compiledGraphOutputType || (compiledGraphOutputType = {}));
55463
- function printCompiledGraph(graph, outputType, quietCompilation) {
55464
- const interactive = isInteractive();
55465
- if (outputType === compiledGraphOutputType.Json) {
55466
- writeStdOut(prettyJsonStringify(graph));
55467
- }
55468
- else if (outputType === compiledGraphOutputType.Dot) {
55469
- writeStdOut(dotRepresentation(graph, interactive));
55690
+ function topoOrder(nodes) {
55691
+ const byKey = new Map(nodes.map(n => [n.key, n]));
55692
+ const indegree = new Map();
55693
+ const dependents = new Map();
55694
+ for (const n of nodes) {
55695
+ const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
55696
+ indegree.set(n.key, inGraphDeps.length);
55697
+ for (const dep of inGraphDeps) {
55698
+ dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
55699
+ }
55470
55700
  }
55471
- else {
55472
- const actionCount = 0 +
55473
- (graph.tables ? graph.tables.length : 0) +
55474
- (graph.assertions ? graph.assertions.length : 0) +
55475
- (graph.operations ? graph.operations.length : 0);
55476
- writeStdOut(successOutput(`Compiled ${actionCount} action(s).`));
55477
- if (graph.tables && graph.tables.length) {
55478
- graph.tables.forEach(setOrValidateTableEnumType);
55479
- writeStdOut(`${graph.tables.length} dataset(s)${quietCompilation ? "." : ":"}`);
55480
- if (!quietCompilation) {
55481
- graph.tables.forEach(compiledTable => {
55482
- writeStdOut(`${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`, 1);
55483
- });
55484
- }
55701
+ const ready = nodes
55702
+ .filter(n => (indegree.get(n.key) || 0) === 0)
55703
+ .map(n => n.key)
55704
+ .sort();
55705
+ const ordered = [];
55706
+ const emitted = new Set();
55707
+ while (ready.length) {
55708
+ const key = ready.shift();
55709
+ if (emitted.has(key)) {
55710
+ continue;
55485
55711
  }
55486
- if (graph.assertions && graph.assertions.length) {
55487
- writeStdOut(`${graph.assertions.length} assertion(s)${quietCompilation ? "." : ":"}`);
55488
- if (!quietCompilation) {
55489
- graph.assertions.forEach(assertion => {
55490
- writeStdOut(assertionString(assertion.target, assertion.disabled), 1);
55491
- });
55712
+ emitted.add(key);
55713
+ ordered.push(byKey.get(key));
55714
+ for (const dep of (dependents.get(key) || []).slice().sort()) {
55715
+ indegree.set(dep, (indegree.get(dep) || 0) - 1);
55716
+ if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
55717
+ ready.push(dep);
55718
+ ready.sort();
55492
55719
  }
55493
55720
  }
55494
- if (graph.operations && graph.operations.length) {
55495
- writeStdOut(`${graph.operations.length} operation(s)${quietCompilation ? "." : ":"}`);
55496
- if (!quietCompilation) {
55497
- graph.operations.forEach(operation => {
55498
- writeStdOut(operationString(operation.target, operation.disabled), 1);
55499
- });
55721
+ }
55722
+ if (ordered.length < nodes.length) {
55723
+ for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
55724
+ if (!emitted.has(n.key)) {
55725
+ emitted.add(n.key);
55726
+ ordered.push(n);
55500
55727
  }
55501
55728
  }
55502
55729
  }
55730
+ return ordered;
55503
55731
  }
55504
- function formatStackTraceForQuietCompilation(compileError) {
55505
- var _a, _b;
55506
- if ((_a = compileError === null || compileError === void 0 ? void 0 : compileError.message) === null || _a === void 0 ? void 0 : _a.includes("Unexpected identifier")) {
55507
- if (!compileError.stack) {
55508
- return "";
55732
+ function dependencyBlocked(dependencyKeys, statusByKey) {
55733
+ return dependencyKeys.some(dep => {
55734
+ const status = statusByKey.get(dep);
55735
+ return status !== undefined && status !== "PASS";
55736
+ });
55737
+ }
55738
+ const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
55739
+ function validateShadowSuffix(nowMs) {
55740
+ return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
55741
+ }
55742
+ function parseShadowTimestamp(schemaName) {
55743
+ const match = schemaName.match(/sqlanvil_validate_(\d+)/);
55744
+ return match ? Number(match[1]) : null;
55745
+ }
55746
+ function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
55747
+ return schemaNames.filter(name => {
55748
+ const ts = parseShadowTimestamp(name);
55749
+ return ts !== null && nowMs - ts > maxAgeMs;
55750
+ });
55751
+ }
55752
+
55753
+ const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
55754
+ async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
55755
+ try {
55756
+ const schemas = await deps.listSchemas();
55757
+ for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
55758
+ try {
55759
+ await deps.execute(deps.dropSchemaCascadeSql(schema));
55760
+ }
55761
+ catch (e) {
55762
+ }
55509
55763
  }
55510
- const stackLines = ((_b = compileError.stack) === null || _b === void 0 ? void 0 : _b.split("\n")) || [];
55511
- return stackLines.slice(0, Math.min(3, stackLines.length)).join("\n");
55512
55764
  }
55513
- return "";
55514
- }
55515
- function printCompiledGraphErrors(graphErrors, quietCompilation) {
55516
- if (graphErrors.compilationErrors && graphErrors.compilationErrors.length > 0) {
55517
- printError("Compilation errors:", 1);
55518
- graphErrors.compilationErrors.forEach(compileError => {
55519
- writeStdErr(`${calloutOutput(compileError.fileName)}: ${errorOutput(quietCompilation ? (compileError.message + " " + formatStackTraceForQuietCompilation(compileError) || compileError.stack) : (compileError.stack || compileError.message))}`, 1);
55520
- });
55765
+ catch (e) {
55521
55766
  }
55522
55767
  }
55523
- function printTestResult(testResult) {
55524
- writeStdOut(`${testResult.name}: ${testResult.successful ? successOutput("passed") : errorOutput("failed")}`);
55525
- if (!testResult.successful) {
55526
- testResult.messages.forEach(message => writeStdErr(message, 1));
55768
+ function tableType(enumType) {
55769
+ switch (enumType) {
55770
+ case sqlanvil.TableType.VIEW:
55771
+ return "view";
55772
+ case sqlanvil.TableType.INCREMENTAL:
55773
+ return "incremental";
55774
+ default:
55775
+ return "table";
55527
55776
  }
55528
55777
  }
55529
- function printExecutionGraph(executionGraph, asJson) {
55530
- if (asJson) {
55531
- writeStdOut(prettyJsonStringify(executionGraph.toJSON()));
55778
+ function depKeys(deps) {
55779
+ return (deps || []).map(targetKey);
55780
+ }
55781
+ async function validate(compiledGraph, deps, options = {}) {
55782
+ const nodes = [];
55783
+ for (const table of compiledGraph.tables || []) {
55784
+ nodes.push({
55785
+ key: targetKey(table.target),
55786
+ dependencyKeys: depKeys(table.dependencyTargets),
55787
+ kind: "table",
55788
+ type: tableType(table.enumType),
55789
+ target: table.target,
55790
+ table,
55791
+ action: table
55792
+ });
55532
55793
  }
55533
- else {
55534
- const actionsByType = {
55535
- table: [],
55536
- assertion: [],
55537
- operation: []
55538
- };
55539
- executionGraph.actions.forEach(action => {
55540
- if (!(action.type === "table" || action.type === "assertion" || action.type === "operation")) {
55541
- throw new Error(`Unrecognized action type: ${action.type}`);
55542
- }
55543
- actionsByType[action.type].push(action);
55794
+ for (const assertion of compiledGraph.assertions || []) {
55795
+ nodes.push({
55796
+ key: targetKey(assertion.target),
55797
+ dependencyKeys: depKeys(assertion.dependencyTargets),
55798
+ kind: "assertion",
55799
+ type: "assertion",
55800
+ target: assertion.target,
55801
+ action: assertion
55544
55802
  });
55545
- const datasetActions = actionsByType.table;
55546
- if (datasetActions && datasetActions.length) {
55547
- writeStdOut(`${datasetActions.length} dataset(s):`);
55548
- datasetActions.forEach(datasetAction => writeStdOut(datasetString(datasetAction.target, datasetAction.type, datasetAction.tasks.length === 0), 1));
55803
+ }
55804
+ for (const operation of compiledGraph.operations || []) {
55805
+ if (!operation.target) {
55806
+ continue;
55549
55807
  }
55550
- const assertionActions = actionsByType.assertion;
55551
- if (assertionActions && assertionActions.length) {
55552
- writeStdOut(`${assertionActions.length} assertion(s):`);
55553
- assertionActions.forEach(assertionAction => writeStdOut(assertionString(assertionAction.target, assertionAction.tasks.length === 0), 1));
55808
+ nodes.push({
55809
+ key: targetKey(operation.target),
55810
+ dependencyKeys: depKeys(operation.dependencyTargets),
55811
+ kind: "operation",
55812
+ type: "operation",
55813
+ target: operation.target,
55814
+ action: undefined
55815
+ });
55816
+ }
55817
+ for (const imp of compiledGraph.imports || []) {
55818
+ if (!imp.target) {
55819
+ continue;
55554
55820
  }
55555
- const operationActions = actionsByType.operation;
55556
- if (operationActions && operationActions.length) {
55557
- writeStdOut(`${operationActions.length} operation(s):`);
55558
- operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
55821
+ nodes.push({
55822
+ key: targetKey(imp.target),
55823
+ dependencyKeys: depKeys(imp.dependencyTargets),
55824
+ kind: "import",
55825
+ type: "import",
55826
+ target: imp.target,
55827
+ action: undefined
55828
+ });
55829
+ }
55830
+ for (const script of compiledGraph.scripts || []) {
55831
+ if (!script.target) {
55832
+ continue;
55559
55833
  }
55834
+ nodes.push({
55835
+ key: targetKey(script.target),
55836
+ dependencyKeys: depKeys(script.dependencyTargets),
55837
+ kind: "script",
55838
+ type: "script",
55839
+ target: script.target,
55840
+ script,
55841
+ action: undefined
55842
+ });
55560
55843
  }
55561
- }
55562
- function printExecutedAction(executedAction, executionAction, dryRun) {
55563
- const jobIds = executedAction.tasks
55564
- .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55565
- .map(task => task.metadata.bigquery.jobId);
55566
- const bytesBilled = executedAction.tasks
55567
- .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55568
- .map(task => {
55569
- var _a, _b, _c, _d;
55570
- const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
55571
- return formatBytesInHumanReadableFormat(bytes);
55572
- });
55573
- const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
55574
- switch (executedAction.status) {
55575
- case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
55576
- switch (executionAction.type) {
55577
- case "table": {
55578
- writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55579
- return;
55580
- }
55581
- case "assertion": {
55582
- writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55583
- return;
55584
- }
55585
- case "operation": {
55586
- writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55587
- return;
55588
- }
55589
- }
55844
+ const ordered = topoOrder(nodes);
55845
+ const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
55846
+ const statusByKey = new Map();
55847
+ const results = [];
55848
+ try {
55849
+ for (const schema of shadowSchemas) {
55850
+ await deps.execute(deps.createSchemaSql(schema));
55590
55851
  }
55591
- case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
55592
- switch (executionAction.type) {
55593
- case "table": {
55594
- writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55595
- break;
55596
- }
55597
- case "assertion": {
55598
- writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55599
- break;
55600
- }
55601
- case "operation": {
55602
- writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55603
- break;
55604
- }
55852
+ for (const node of ordered) {
55853
+ if (node.kind === "operation" ||
55854
+ node.kind === "import" ||
55855
+ (node.kind === "script" && !deps.checkScript)) {
55856
+ statusByKey.set(node.key, "SKIPPED");
55857
+ results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
55858
+ continue;
55605
55859
  }
55606
- printExecutedActionErrors(executedAction, executionAction);
55607
- return;
55608
- }
55609
- case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
55610
- switch (executionAction.type) {
55611
- case "table": {
55612
- writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55613
- return;
55614
- }
55615
- case "assertion": {
55616
- writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55617
- return;
55860
+ if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
55861
+ statusByKey.set(node.key, "BLOCKED");
55862
+ results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
55863
+ continue;
55864
+ }
55865
+ const evaluations = node.kind === "script"
55866
+ ? await deps.checkScript(node.script)
55867
+ : await deps.evaluate(node.action);
55868
+ const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
55869
+ const status = failed ? "FAILURE" : "PASS";
55870
+ statusByKey.set(node.key, status);
55871
+ results.push({ target: node.target, type: node.type, status, errors: evaluations });
55872
+ if (status === "PASS" && node.kind === "table") {
55873
+ try {
55874
+ await deps.execute(deps.validationStubSql(node.table));
55618
55875
  }
55619
- case "operation": {
55620
- writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55621
- return;
55876
+ catch (e) {
55622
55877
  }
55623
55878
  }
55624
- return;
55625
55879
  }
55626
- case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
55627
- switch (executionAction.type) {
55628
- case "table": {
55629
- writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55630
- return;
55631
- }
55632
- case "assertion": {
55633
- writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55634
- return;
55880
+ }
55881
+ finally {
55882
+ if (!options.keepShadow) {
55883
+ for (const schema of shadowSchemas.slice().reverse()) {
55884
+ try {
55885
+ await deps.execute(deps.dropSchemaCascadeSql(schema));
55635
55886
  }
55636
- case "operation": {
55637
- writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55638
- return;
55887
+ catch (e) {
55639
55888
  }
55640
55889
  }
55641
- return;
55642
55890
  }
55643
55891
  }
55892
+ return results;
55644
55893
  }
55645
- function printFormatFilesResult(formatResults) {
55646
- const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
55647
- const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
55648
- const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
55649
- const failedFormatResults = sorted.filter(result => !!result.err);
55650
- if (successfulFormatResults.length > 0) {
55651
- printSuccess("Successfully formatted:");
55652
- successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
55653
- }
55654
- if (needsFormattingResults.length > 0) {
55655
- printError("Files that need formatting:");
55656
- needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
55894
+
55895
+ class SupabaseDbAdapter extends PostgresDbAdapter {
55896
+ static async create(credentials, options) {
55897
+ const sslMode = (credentials.sslMode || "").toLowerCase();
55898
+ const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
55899
+ const clientConfig = {
55900
+ host: credentials.host,
55901
+ port: credentials.port,
55902
+ database: credentials.database,
55903
+ user: credentials.user,
55904
+ password: credentials.password,
55905
+ ssl: sslEnabled
55906
+ ? {
55907
+ rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
55908
+ }
55909
+ : false
55910
+ };
55911
+ const queryExecutor = new PgPoolExecutor(clientConfig, options);
55912
+ try {
55913
+ await queryExecutor.verifyConnection();
55914
+ }
55915
+ catch (e) {
55916
+ await queryExecutor.close().catch(() => undefined);
55917
+ throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
55918
+ `as "${credentials.user}": ${e.message}`, e);
55919
+ }
55920
+ return new SupabaseDbAdapter(queryExecutor);
55657
55921
  }
55658
- if (failedFormatResults.length > 0) {
55659
- printError("Errors encountered during formatting:");
55660
- failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
55922
+ constructor(queryExecutor) {
55923
+ super(queryExecutor);
55661
55924
  }
55662
- }
55663
- function datasetString(target, datasetType, disabled) {
55664
- return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
55665
- }
55666
- function assertionString(target, disabled) {
55667
- return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55668
- }
55669
- function operationString(target, disabled) {
55670
- return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55671
- }
55672
- function plainTargetString(target) {
55673
- return `${target.schema}.${target.name}`;
55674
- }
55675
- function targetString(target) {
55676
- return calloutOutput(`${target.schema}.${target.name}`);
55677
- }
55678
- function dotRepresentation(graph, interactive) {
55679
- var _a, _b, _c;
55680
- const nodes = [];
55681
- const edges = [];
55682
- const formatTarget = interactive ? targetString : plainTargetString;
55683
- (_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
55684
- var _a;
55685
- const nodeName = `${formatTarget(table.target)}`;
55686
- nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
55687
- (_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55688
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55689
- });
55690
- });
55691
- (_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
55692
- var _a;
55693
- const nodeName = `${formatTarget(assertion.target)}`;
55694
- nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
55695
- (_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55696
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55697
- });
55698
- });
55699
- (_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
55925
+ async verifyServiceRolePermissions() {
55700
55926
  var _a;
55701
- const nodeName = `${formatTarget(operation.target)}`;
55702
- nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
55703
- (_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55704
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55705
- });
55706
- });
55707
- return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
55708
- }
55709
- function printExecutedActionErrors(executedAction, executionAction) {
55710
- const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
55711
- failingTasks.forEach((task, i) => {
55712
- executionAction.tasks[i].statement.split("\n").forEach(line => {
55713
- writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
55714
- });
55715
- printError(task.errorMessage, 1);
55716
- });
55927
+ try {
55928
+ const result = await this.execute("select current_user");
55929
+ const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
55930
+ return !!currentUser;
55931
+ }
55932
+ catch (_b) {
55933
+ return false;
55934
+ }
55935
+ }
55936
+ async validatePublicationState(pubName = "supabase_realtime") {
55937
+ try {
55938
+ const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
55939
+ params: [pubName]
55940
+ });
55941
+ return result.rows.length > 0;
55942
+ }
55943
+ catch (_a) {
55944
+ return false;
55945
+ }
55946
+ }
55717
55947
  }
55718
55948
 
55719
55949
  function getBigQueryCredentials() {
@@ -56587,6 +56817,18 @@ const icebergOption = option("iceberg", {
56587
56817
  type: "boolean",
56588
56818
  default: false
56589
56819
  });
56820
+ const bareOption = option("bare", {
56821
+ describe: "Skip the sample project files — scaffold bare (gitkept) directories only.",
56822
+ type: "boolean",
56823
+ default: false
56824
+ });
56825
+ const interactiveOption = option("interactive", {
56826
+ describe: "Guided Q&A setup: start a fresh project (warehouse, sample project, credentials) or " +
56827
+ "convert an existing Dataform project. Other init arguments are ignored except " +
56828
+ "[project-dir], which seeds the directory prompt.",
56829
+ type: "boolean",
56830
+ default: false
56831
+ });
56590
56832
  const warehouseOption = option("warehouse", {
56591
56833
  describe: "Target warehouse for the new project.",
56592
56834
  type: "string",
@@ -56836,14 +57078,17 @@ function runCli() {
56836
57078
  {
56837
57079
  format: `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` +
56838
57080
  ` [${ProjectConfigOptions.defaultLocation.name}]`,
56839
- description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB).",
57081
+ description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB). " +
57082
+ "Use --interactive for a guided setup, including converting a Dataform project.",
56840
57083
  positionalOptions: [
56841
57084
  projectDirOption,
56842
57085
  positionalOption(ProjectConfigOptions.defaultDatabase.name, {
56843
57086
  describe: "The default database to use, equivalent to Google Cloud Project ID."
56844
57087
  }, (argv) => {
56845
57088
  const warehouse = argv[warehouseOption.name] || "bigquery";
56846
- if (warehouse === "bigquery" && !argv[ProjectConfigOptions.defaultDatabase.name]) {
57089
+ if (warehouse === "bigquery" &&
57090
+ !argv[interactiveOption.name] &&
57091
+ !argv[ProjectConfigOptions.defaultDatabase.name]) {
56847
57092
  throw new Error(`The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` +
56848
57093
  `required for BigQuery projects. Use "sqlanvil help init" for more info.`);
56849
57094
  }
@@ -56853,15 +57098,20 @@ function runCli() {
56853
57098
  "https://cloud.google.com/bigquery/docs/locations for supported values."
56854
57099
  }, (argv) => {
56855
57100
  const warehouse = argv[warehouseOption.name] || "bigquery";
56856
- if (warehouse === "bigquery" && !argv[ProjectConfigOptions.defaultLocation.name]) {
57101
+ if (warehouse === "bigquery" &&
57102
+ !argv[interactiveOption.name] &&
57103
+ !argv[ProjectConfigOptions.defaultLocation.name]) {
56857
57104
  throw new Error(`The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` +
56858
57105
  `required for BigQuery projects. Use "sqlanvil help init" for more info.`);
56859
57106
  }
56860
57107
  })
56861
57108
  ],
56862
- options: [warehouseOption, icebergOption],
57109
+ options: [warehouseOption, icebergOption, bareOption, interactiveOption],
56863
57110
  processFn: async (argv) => {
56864
57111
  const projectDir = argv[projectDirOption.name];
57112
+ if (argv[interactiveOption.name]) {
57113
+ return runInteractiveInit(projectDir);
57114
+ }
56865
57115
  const warehouse = argv[warehouseOption.name] || "bigquery";
56866
57116
  const projectConfig = { warehouse };
56867
57117
  if (warehouse === "bigquery") {
@@ -56875,7 +57125,9 @@ function runCli() {
56875
57125
  }
56876
57126
  }
56877
57127
  print("Writing project files...\n");
56878
- const initResult = await init(projectDir, projectConfig);
57128
+ const initResult = await init(projectDir, projectConfig, {
57129
+ includeSample: !argv[bareOption.name]
57130
+ });
56879
57131
  printInitResult(initResult);
56880
57132
  return 0;
56881
57133
  }
@@ -57461,13 +57713,7 @@ function runCli() {
57461
57713
  srcDir: argv["source-dir"],
57462
57714
  outDir: argv["out-dir"]
57463
57715
  });
57464
- const targets = report.files.filter(f => f.action === "target");
57465
- const flagged = targets.filter(f => f.status === "flagged").length;
57466
- printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
57467
- `${report.connections.length} source connection(s) over ` +
57468
- `${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
57469
- `${targets.length} target file(s), ${flagged} flagged for dialect review.`);
57470
- print(`Report: ${path__namespace.join(argv["out-dir"], "migration-report.md")}`);
57716
+ printMigrationSummary(report, argv["out-dir"]);
57471
57717
  print(`Next: sqlanvil compile ${argv["out-dir"]}`);
57472
57718
  return 0;
57473
57719
  }