@sqlanvil/cli 1.22.0 → 1.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/bundle.js +1358 -888
  2. package/package.json +1 -1
package/bundle.js CHANGED
@@ -49315,10 +49315,25 @@ function unknownToValueShallow(raw) {
49315
49315
  })();
49316
49316
 
49317
49317
  const nativeRequire = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require;
49318
+ function globToRegExp(pattern) {
49319
+ const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\*/g, ".*");
49320
+ return new RegExp(`^${escaped}$`);
49321
+ }
49318
49322
  function matchPatterns(patterns, values) {
49319
49323
  const fullyQualifiedActions = [];
49320
49324
  patterns.forEach(pattern => {
49321
- if (pattern.includes(".")) {
49325
+ if (pattern.includes("*")) {
49326
+ const regExp = globToRegExp(pattern);
49327
+ const scope = pattern.includes(".")
49328
+ ? values
49329
+ : values.map(value => value.split(".").slice(-1)[0]);
49330
+ values.forEach((value, i) => {
49331
+ if (regExp.test(scope[i])) {
49332
+ fullyQualifiedActions.push(value);
49333
+ }
49334
+ });
49335
+ }
49336
+ else if (pattern.includes(".")) {
49322
49337
  if (values.includes(pattern)) {
49323
49338
  fullyQualifiedActions.push(pattern);
49324
49339
  }
@@ -50606,7 +50621,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
50606
50621
  .filter(validationQuery => !!validationQuery.query);
50607
50622
  }
50608
50623
 
50609
- const version = "1.22.0";
50624
+ const version = "1.24.0";
50610
50625
  const dataformVersion = "3.0.61";
50611
50626
 
50612
50627
  async function build(compiledGraph, runConfig, dbadapter) {
@@ -50987,6 +51002,123 @@ async function test$1(dbadapter, timeoutMs = 10000) {
50987
51002
  }
50988
51003
  }
50989
51004
 
51005
+ function claudeMdBridgeContents() {
51006
+ return "@AGENTS.md\n";
51007
+ }
51008
+ function agentsMdContents(options) {
51009
+ const { warehouse, defaultDataset, version, converted } = options;
51010
+ const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
51011
+ const isSupabase = warehouse === "supabase";
51012
+ const isMysql = warehouse === "mysql";
51013
+ const sections = [];
51014
+ sections.push(`<!-- generated by sqlanvil ${version} — this file is yours; edit freely -->
51015
+
51016
+ # SQLAnvil project — guide for coding agents
51017
+
51018
+ This is a **SQLAnvil** data project (warehouse: **${warehouse}**, default dataset/schema:
51019
+ \`${defaultDataset}\`). SQLAnvil is a fork of Dataform, so Dataform instincts are mostly
51020
+ right: \`.sqlx\` files, \`config {}\` blocks, \`\${ref()}\`, \`\${self()}\`, declarations,
51021
+ assertions, tags, incremental tables, and \`pre_operations\`/\`post_operations\` all work the
51022
+ same. The rules below are where the differences bite.
51023
+
51024
+ ## The essentials
51025
+
51026
+ - The CLI is \`sqlanvil\` (\`npm i -g @sqlanvil/cli\`) — there is no \`dataform\` CLI and no
51027
+ npm scripts in this project.
51028
+ - \`workflow_settings.yaml\` pins \`sqlanvilCoreVersion:\` — sqlanvil's OWN SemVer line, NOT
51029
+ \`dataformCoreVersion:\`.
51030
+ - Credentials live in the gitignored \`.df-credentials.json\`. Never commit it; never put
51031
+ secrets in \`workflow_settings.yaml\`.
51032
+ - The statement separator in \`operations\` / \`pre_operations\` / \`post_operations\` is
51033
+ \`---\` on its own line — NEVER \`;\`. Dollar-quoted bodies (\`$$ … ; … $$\`) survive
51034
+ intact. A \`table\`/\`view\` model body is exactly ONE \`SELECT\` — no \`---\`.
51035
+ - Procedures, functions, triggers, grants, extensions → \`type: "operations"\` (there is no
51036
+ dedicated action type).
51037
+ - Two different \`uniqueKey\`s: top-level \`uniqueKey\` on an incremental is the merge/upsert
51038
+ key; \`assertions: { uniqueKey }\` generates a data-quality assertion. They are independent.
51039
+ - Always reference tables with \`\${ref("name")}\` — never hardcoded paths. Declare external
51040
+ tables (\`type: "declaration"\`) before ref'ing them.
51041
+ - Document every table: \`description:\` + \`columns: {}\` compile to real comments.
51042
+
51043
+ ## Commands
51044
+
51045
+ \`\`\`bash
51046
+ sqlanvil compile .
51047
+ sqlanvil validate . --credentials .df-credentials.json # planner-checks the whole DAG without executing — run this before every run
51048
+ sqlanvil run . --credentials .df-credentials.json
51049
+ sqlanvil run . --credentials .df-credentials.json --actions <name> --include-deps
51050
+ sqlanvil test . --credentials .df-credentials.json
51051
+ \`\`\`
51052
+
51053
+ \`target/\` holds compile/run artifacts (queryable via \`sqlanvil query\`/\`inspect\`) — keep
51054
+ it gitignored.`);
51055
+ if (isPostgresLike) {
51056
+ sections.push(`## ${isSupabase ? "Supabase/PostgreSQL" : "PostgreSQL"} specifics
51057
+
51058
+ - \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
51059
+ sslMode defaultSchema\` — not nested under a key, not \`username\`/\`databaseName\`/\`ssl\`.${isSupabase
51060
+ ? `
51061
+ Use the Session pooler host (\`aws-…pooler.supabase.com\`, user \`postgres.<project-ref>\`,
51062
+ port 5432): the direct \`db.<ref>.supabase.co\` host is IPv6-only and fails on most networks.`
51063
+ : ""}
51064
+ - Storage, indexes, and partitioning are first-class in the \`postgres: {}\` config block —
51065
+ never hand-roll \`CREATE INDEX\` or \`SET (fillfactor …)\` in \`post_operations\`.
51066
+ - Index \`method\` is a NUMERIC enum (\`BTREE=0, HASH=1, GIN=2, GIST=3, BRIN=4\`) — omit it
51067
+ for btree; the string \`method: "btree"\` fails the config check. \`opclass\` is a single
51068
+ string, not an array.
51069
+ - No BigQuery-isms: \`bigquery: {}\`, \`partitionBy\`, \`clusterBy\`, \`OPTIONS(...)\` — use
51070
+ \`postgres: { partition: {...}, indexes: [...] }\` (partition \`kind\`: RANGE=0, LIST=1,
51071
+ HASH=2).
51072
+ - Materialized views: \`type: "view", materialized: true\` (+ optional \`postgres: {
51073
+ refreshPolicy: "on_dependency_change", noData, indexes }\`). Default is drop+recreate each
51074
+ run; in-place refresh does not pick up definition changes.
51075
+ - One-time DDL on incrementals (e.g. \`ADD PRIMARY KEY\`) must be wrapped in
51076
+ \`\${when(!incremental(), \\\`…\\\`)}\` — an unwrapped \`post_operations\` block re-runs on
51077
+ EVERY append and errors the second time.
51078
+ - Read tables in OTHER warehouses via named \`connections:\` in \`workflow_settings.yaml\`
51079
+ plus a \`connection:\`-tagged declaration with \`columnTypes\` in POSTGRES types — scaffold
51080
+ it with \`sqlanvil introspect <connection> <schema.table>\`. Never hand-write FDW/foreign
51081
+ tables.${isSupabase
51082
+ ? `
51083
+ - Supabase extras: the \`supabase: {}\` block (\`enableRls\`, \`vectors\`) and dedicated
51084
+ action types \`rlsPolicy\`, \`realtimePublication\`, \`vectorIndex\`. \`enableRls\` only
51085
+ flips RLS on — declare policies via \`rlsPolicy\` actions.`
51086
+ : ""}`);
51087
+ }
51088
+ if (isMysql) {
51089
+ sections.push(`## MySQL/MariaDB specifics (several Postgres rules INVERT here)
51090
+
51091
+ - \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
51092
+ sslMode\` — no \`defaultSchema\` (MySQL's "schema" IS the database).
51093
+ - Use the \`mysql: {}\` config block for indexes, table options (\`engine\`, \`charset\`,
51094
+ \`rowFormat\`), and partitioning — never \`postgres: {}\` (wrong dialect, silently ignored).
51095
+ - An incremental's \`uniqueKey\` compiles to \`INSERT … ON DUPLICATE KEY UPDATE\` and the
51096
+ adapter AUTO-CREATES the matching unique index — do not add your own PK/unique index for
51097
+ the merge.
51098
+ - \`materialized: true\` is an emulated snapshot (drop + CTAS each run), not a native
51099
+ matview; no \`refreshPolicy\`/\`noData\`.
51100
+ - Backtick-quote identifiers in raw DDL; never \`DELIMITER\` (client-only directive — the
51101
+ \`---\` separator already keeps \`;\` bodies intact).
51102
+ - A mysql warehouse cannot read cross-warehouse \`connections:\` (single write warehouse;
51103
+ MySQL works as a SOURCE for a Postgres/Supabase project, not the other way).`);
51104
+ }
51105
+ if (converted) {
51106
+ sections.push(`## This project was converted from Dataform
51107
+
51108
+ - Work the to-do list in \`migration-report.md\` (machine-readable twin:
51109
+ \`migration-report.json\`). Inline \`SQLANVIL-MIGRATE:\` markers flag every rewritten or
51110
+ questionable spot.
51111
+ - \`sqlanvil validate\` is the completion loop — converge on all-PASS.
51112
+ - The original Dataform repo was read-only during conversion and remains the rollback path.`);
51113
+ }
51114
+ sections.push(`---
51115
+
51116
+ Docs: https://sqlanvil.com/docs (agent bundle: https://sqlanvil.com/llms.txt) · Full
51117
+ authoring skill: \`npx skills add SQLAnvil/agent-skills\` · Hosted CI/scheduling:
51118
+ https://app.sqlanvil.com (remote MCP server at \`/api/mcp\`).`);
51119
+ return sections.join("\n\n") + "\n";
51120
+ }
51121
+
50990
51122
  const gitIgnoreContents = `
50991
51123
  ${CREDENTIALS_FILENAME}
50992
51124
  .df-credentials*.json
@@ -51016,7 +51148,8 @@ function mysqlCredentialsTemplate() {
51016
51148
  };
51017
51149
  return `${JSON.stringify(template, null, 2)}\n`;
51018
51150
  }
51019
- async function init(projectDir, projectConfig) {
51151
+ async function init(projectDir, projectConfig, options = {}) {
51152
+ var _a;
51020
51153
  const workflowSettingsYamlPath = path__namespace.join(projectDir, "workflow_settings.yaml");
51021
51154
  const packageJsonPath = path__namespace.join(projectDir, "package.json");
51022
51155
  const gitignorePath = path__namespace.join(projectDir, ".gitignore");
@@ -51064,7 +51197,11 @@ async function init(projectDir, projectConfig) {
51064
51197
  if (projectConfig.builtinAssertionNamePrefix) {
51065
51198
  workflowSettings.builtinAssertionNamePrefix = projectConfig.builtinAssertionNamePrefix;
51066
51199
  }
51067
- if (isPostgresLike) {
51200
+ const includeSample = options.includeSample !== false;
51201
+ const withBigQuerySource = includeSample &&
51202
+ warehouse !== "mysql" &&
51203
+ (!isPostgresLike || options.includeBigQuerySource !== false);
51204
+ if (isPostgresLike && withBigQuerySource) {
51068
51205
  workflowSettings.connections = {
51069
51206
  bigquery_public: sqlanvil.ConnectionConfig.create({
51070
51207
  platform: "bigquery",
@@ -51079,8 +51216,22 @@ async function init(projectDir, projectConfig) {
51079
51216
  filesWritten.push(workflowSettingsYamlPath);
51080
51217
  fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
51081
51218
  filesWritten.push(gitignorePath);
51219
+ const agentsMdPath = path__namespace.join(projectDir, "AGENTS.md");
51220
+ if (!fs__namespace$1.existsSync(agentsMdPath)) {
51221
+ fs__namespace$1.writeFileSync(agentsMdPath, agentsMdContents({
51222
+ warehouse,
51223
+ defaultDataset: workflowSettings.defaultDataset || "public",
51224
+ version
51225
+ }));
51226
+ filesWritten.push(agentsMdPath);
51227
+ }
51228
+ const claudeMdPath = path__namespace.join(projectDir, "CLAUDE.md");
51229
+ if (!fs__namespace$1.existsSync(claudeMdPath)) {
51230
+ fs__namespace$1.writeFileSync(claudeMdPath, claudeMdBridgeContents());
51231
+ filesWritten.push(claudeMdPath);
51232
+ }
51082
51233
  if (!isBigQuery) {
51083
- fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse));
51234
+ fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
51084
51235
  filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
51085
51236
  }
51086
51237
  const mkdir = (...segments) => {
@@ -51101,7 +51252,10 @@ async function init(projectDir, projectConfig) {
51101
51252
  const reportingDir = mkdir("definitions", "outputs", "reporting");
51102
51253
  const testDir = mkdir("definitions", "test");
51103
51254
  keep(mkdir("includes"));
51104
- const withBigQuerySource = warehouse !== "mysql";
51255
+ if (!includeSample) {
51256
+ [sourcesDir, intermediateDir, salesDir, reportingDir, testDir].forEach(keep);
51257
+ return { filesWritten, dirsCreated };
51258
+ }
51105
51259
  write(path__namespace.join(sourcesDir, "app_orders.sqlx"), `-- A declaration: a table that already exists in your warehouse (created by your
51106
51260
  -- application, an import, or another pipeline). Point name/dataset at a real table —
51107
51261
  -- downstream models read it with \${ref("app_orders")}.
@@ -54401,7 +54555,8 @@ ${description} columnTypes: {
54401
54555
  ],
54402
54556
  };
54403
54557
  }
54404
- function convertTarget(source, jsMode) {
54558
+ function convertTarget(source, jsMode, dialect = "postgres") {
54559
+ const dialectPass = dialect === "postgres";
54405
54560
  const span = jsMode ? null : findConfigBlock(source);
54406
54561
  const findings = [];
54407
54562
  const lines = source.split("\n");
@@ -54439,7 +54594,7 @@ function convertTarget(source, jsMode) {
54439
54594
  line = line.replace(/,/, " ");
54440
54595
  }
54441
54596
  }
54442
- if (inConfig && commentingConfigBlockDepth === 0) {
54597
+ if (dialectPass && inConfig && commentingConfigBlockDepth === 0) {
54443
54598
  const m = CONFIG_BQ_KEYS.exec(line);
54444
54599
  if (m) {
54445
54600
  const note = `BigQuery-only setting — not applicable on Postgres (partitioning/clustering → postgres: {}; project qualifiers drop)`;
@@ -54463,7 +54618,7 @@ function convertTarget(source, jsMode) {
54463
54618
  out.push(`// ${line}`);
54464
54619
  continue;
54465
54620
  }
54466
- if (!inConfig || jsMode) {
54621
+ if (dialectPass && (!inConfig || jsMode)) {
54467
54622
  for (const rule of SQL_RULES) {
54468
54623
  rule.pattern.lastIndex = 0;
54469
54624
  if (!rule.pattern.test(line))
@@ -54515,6 +54670,7 @@ function readSourceConfig(srcDir) {
54515
54670
  const s = (k) => (typeof raw[k] === "string" ? raw[k] : null);
54516
54671
  return {
54517
54672
  defaultProject: (_a = s("defaultProject")) !== null && _a !== void 0 ? _a : s("defaultDatabase"),
54673
+ defaultLocation: s("defaultLocation"),
54518
54674
  defaultDataset: (_c = (_b = s("defaultDataset")) !== null && _b !== void 0 ? _b : s("defaultSchema")) !== null && _c !== void 0 ? _c : "dataform",
54519
54675
  defaultAssertionDataset: (_d = s("defaultAssertionDataset")) !== null && _d !== void 0 ? _d : s("assertionSchema"),
54520
54676
  vars: (_e = raw.vars) !== null && _e !== void 0 ? _e : null,
@@ -54522,7 +54678,7 @@ function readSourceConfig(srcDir) {
54522
54678
  };
54523
54679
  }
54524
54680
  async function migrateDataform(opts) {
54525
- var _a, _b, _c, _d, _e, _f, _g, _h;
54681
+ var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
54526
54682
  const srcDir = path__namespace.resolve(opts.srcDir);
54527
54683
  const outDir = path__namespace.resolve(opts.outDir);
54528
54684
  if (containsPath(srcDir, outDir)) {
@@ -54534,11 +54690,14 @@ async function migrateDataform(opts) {
54534
54690
  if (fs__namespace$1.existsSync(outDir) && fs__namespace$1.readdirSync(outDir).length > 0) {
54535
54691
  throw new Error(`Output directory ${outDir} is not empty.`);
54536
54692
  }
54693
+ const targetWarehouse = (_a = opts.targetWarehouse) !== null && _a !== void 0 ? _a : "supabase";
54694
+ const isBigQueryTarget = targetWarehouse === "bigquery";
54537
54695
  const sourceConfig = readSourceConfig(srcDir);
54538
54696
  const writer = new OutWriter(outDir);
54539
54697
  skippedForSafety = new Set();
54540
54698
  const files = walkFiles(srcDir);
54541
54699
  const report = {
54700
+ targetWarehouse,
54542
54701
  generator: `sqlanvil migrate-dataform ${version}`,
54543
54702
  sourceDir: srcDir,
54544
54703
  sourceConfig: sourceConfig.raw,
@@ -54560,16 +54719,16 @@ async function migrateDataform(opts) {
54560
54719
  : { type: null, schema: null, database: null, name: null, description: null };
54561
54720
  sqlxFiles.push({ rel, source, parsed });
54562
54721
  report.inventory.sqlxFiles++;
54563
- const t = (_a = parsed.type) !== null && _a !== void 0 ? _a : "(none)";
54564
- report.inventory.byType[t] = ((_b = report.inventory.byType[t]) !== null && _b !== void 0 ? _b : 0) + 1;
54722
+ const t = (_b = parsed.type) !== null && _b !== void 0 ? _b : "(none)";
54723
+ report.inventory.byType[t] = ((_c = report.inventory.byType[t]) !== null && _c !== void 0 ? _c : 0) + 1;
54565
54724
  }
54566
54725
  const resolveDatabase = (db) => { var _a; return !db || db === "<defaultDatabase>" ? (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "UNKNOWN_PROJECT" : db; };
54567
54726
  const declarationSchemas = new Set();
54568
54727
  const targetSchemas = new Set();
54569
54728
  const connectionMap = new Map();
54570
- for (const f of sqlxFiles) {
54729
+ for (const f of isBigQueryTarget ? [] : sqlxFiles) {
54571
54730
  if (f.parsed.type === "declaration") {
54572
- const dataset = (_c = f.parsed.schema) !== null && _c !== void 0 ? _c : sourceConfig.defaultDataset;
54731
+ const dataset = (_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset;
54573
54732
  const project = resolveDatabase(f.parsed.database);
54574
54733
  declarationSchemas.add(dataset);
54575
54734
  const name = connectionNameFor(project);
@@ -54590,7 +54749,7 @@ async function migrateDataform(opts) {
54590
54749
  }
54591
54750
  }
54592
54751
  else if (f.parsed.type) {
54593
- targetSchemas.add((_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset);
54752
+ targetSchemas.add((_e = f.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset);
54594
54753
  }
54595
54754
  }
54596
54755
  report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
@@ -54602,8 +54761,20 @@ async function migrateDataform(opts) {
54602
54761
  }
54603
54762
  if (rel.endsWith(".sqlx")) {
54604
54763
  const info = sqlxFiles.find(f => f.rel === rel);
54764
+ if (isBigQueryTarget) {
54765
+ const { content, findings } = convertTarget(info.source, false, "none");
54766
+ writer.write(rel, content);
54767
+ report.files.push({
54768
+ file: rel,
54769
+ action: info.parsed.type === "declaration" ? "declaration" : "target",
54770
+ type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
54771
+ status: findings.length === 0 ? "clean" : "rewritten",
54772
+ findings,
54773
+ });
54774
+ continue;
54775
+ }
54605
54776
  if (info.parsed.type === "declaration") {
54606
- const dataset = (_e = info.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset;
54777
+ const dataset = (_g = info.parsed.schema) !== null && _g !== void 0 ? _g : sourceConfig.defaultDataset;
54607
54778
  const project = resolveDatabase(info.parsed.database);
54608
54779
  const connection = connectionNameFor(project);
54609
54780
  const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
@@ -54623,7 +54794,7 @@ async function migrateDataform(opts) {
54623
54794
  report.files.push({
54624
54795
  file: rel,
54625
54796
  action: "target",
54626
- type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
54797
+ type: (_h = info.parsed.type) !== null && _h !== void 0 ? _h : "(none)",
54627
54798
  status: findings.length === 0
54628
54799
  ? "clean"
54629
54800
  : findings.some(f => f.kind === "flag")
@@ -54637,10 +54808,10 @@ async function migrateDataform(opts) {
54637
54808
  if (rel.endsWith(".js") && (rel.startsWith("definitions/") || rel.startsWith("includes/"))) {
54638
54809
  report.inventory.jsFiles++;
54639
54810
  const source = fs__namespace$1.readFileSync(abs, "utf8");
54640
- const { content, findings } = convertTarget(source, true);
54811
+ const { content, findings } = convertTarget(source, true, isBigQueryTarget ? "none" : "postgres");
54641
54812
  writer.write(rel, content);
54642
54813
  const isIncludes = rel.startsWith("includes/");
54643
- if (isIncludes && findings.length > 0) {
54814
+ if (isIncludes && findings.length > 0 && !isBigQueryTarget) {
54644
54815
  report.warnings.push(`includes helper ${rel} emits BigQuery-flavored SQL — review its ${findings.length} finding(s)`);
54645
54816
  }
54646
54817
  report.files.push({
@@ -54659,11 +54830,12 @@ async function migrateDataform(opts) {
54659
54830
  writer.write(rel, fs__namespace$1.readFileSync(abs));
54660
54831
  report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
54661
54832
  }
54662
- const settings = {
54663
- warehouse: "supabase",
54833
+ const settings = isBigQueryTarget
54834
+ ? Object.assign(Object.assign(Object.assign({}, (sourceConfig.defaultProject ? { defaultProject: sourceConfig.defaultProject } : {})), (sourceConfig.defaultLocation ? { defaultLocation: sourceConfig.defaultLocation } : {})), { defaultDataset: sourceConfig.defaultDataset, defaultAssertionDataset: (_j = sourceConfig.defaultAssertionDataset) !== null && _j !== void 0 ? _j : "sqlanvil_assertions", sqlanvilCoreVersion: (_k = opts.coreVersion) !== null && _k !== void 0 ? _k : version }) : {
54835
+ warehouse: targetWarehouse,
54664
54836
  defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
54665
- defaultAssertionDataset: ((_g = sourceConfig.defaultAssertionDataset) !== null && _g !== void 0 ? _g : "sqlanvil_assertions").toLowerCase(),
54666
- sqlanvilCoreVersion: (_h = opts.coreVersion) !== null && _h !== void 0 ? _h : version,
54837
+ defaultAssertionDataset: ((_l = sourceConfig.defaultAssertionDataset) !== null && _l !== void 0 ? _l : "sqlanvil_assertions").toLowerCase(),
54838
+ sqlanvilCoreVersion: (_m = opts.coreVersion) !== null && _m !== void 0 ? _m : version,
54667
54839
  };
54668
54840
  if (sourceConfig.vars)
54669
54841
  settings.vars = sourceConfig.vars;
@@ -54682,6 +54854,22 @@ async function migrateDataform(opts) {
54682
54854
  }));
54683
54855
  }
54684
54856
  writer.write("workflow_settings.yaml", jsYaml.dump(settings));
54857
+ if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "AGENTS.md"))) {
54858
+ writer.write("AGENTS.md", agentsMdContents({
54859
+ warehouse: targetWarehouse,
54860
+ defaultDataset: settings.defaultDataset,
54861
+ version: (_o = opts.coreVersion) !== null && _o !== void 0 ? _o : version,
54862
+ converted: true
54863
+ }));
54864
+ report.files.push({ file: "AGENTS.md", action: "generated", status: "clean", findings: [] });
54865
+ }
54866
+ else {
54867
+ report.warnings.push("Source AGENTS.md copied as-is — its guidance targets Dataform; consider replacing it with the sqlanvil version (`sqlanvil init --bare` in an empty dir generates one).");
54868
+ }
54869
+ if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "CLAUDE.md"))) {
54870
+ writer.write("CLAUDE.md", claudeMdBridgeContents());
54871
+ report.files.push({ file: "CLAUDE.md", action: "generated", status: "clean", findings: [] });
54872
+ }
54685
54873
  report.skippedForSafety = [...skippedForSafety].sort();
54686
54874
  if (skippedForSafety.size > 0) {
54687
54875
  report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
@@ -54708,28 +54896,38 @@ function renderReportMd(r) {
54708
54896
  lines.push(` - ${t}: ${n}`);
54709
54897
  }
54710
54898
  lines.push("");
54711
- lines.push(`## Source connections (${r.connections.length})`);
54712
- lines.push("");
54713
- lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
54714
- `(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
54715
- `before it can extract; scaffold them with \`sqlanvil introspect\`:`);
54716
- lines.push("");
54717
- for (const c of r.connections) {
54718
- lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
54719
- `${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
54720
- (c.datasets.length > 8 ? ", …" : ""));
54721
- lines.push(` - \`${c.introspectExample}\``);
54899
+ if (r.targetWarehouse === "bigquery") {
54900
+ lines.push(`## Same-warehouse conversion (BigQuery → BigQuery)`);
54901
+ lines.push("");
54902
+ lines.push(`This is a tooling swap: the project keeps running on the SAME BigQuery warehouse. SQL ` +
54903
+ `bodies, \`bigquery: {}\` config blocks, and declarations passed through untouched; ` +
54904
+ `only the compile global was renamed (\`dataform.projectConfig\` → ` +
54905
+ `\`sqlanvil.projectConfig\`) and the settings file converted.`);
54722
54906
  }
54723
- if (r.overlappingSchemas.length > 0) {
54907
+ else {
54908
+ lines.push(`## Source connections (${r.connections.length})`);
54724
54909
  lines.push("");
54725
- lines.push(`## ⚠ Schemas that are BOTH source and target`);
54910
+ lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
54911
+ `(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
54912
+ `before it can extract; scaffold them with \`sqlanvil introspect\`:`);
54726
54913
  lines.push("");
54727
- lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
54728
- .map(s => `\`${s}\``)
54729
- .join(", ")}. ` +
54730
- `Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
54731
- `producers move to Postgres, convert the declarations to plain local declarations instead ` +
54732
- `of BigQuery connections.`);
54914
+ for (const c of r.connections) {
54915
+ lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
54916
+ `${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
54917
+ (c.datasets.length > 8 ? ", …" : ""));
54918
+ lines.push(` - \`${c.introspectExample}\``);
54919
+ }
54920
+ if (r.overlappingSchemas.length > 0) {
54921
+ lines.push("");
54922
+ lines.push(`## ⚠ Schemas that are BOTH source and target`);
54923
+ lines.push("");
54924
+ lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
54925
+ .map(s => `\`${s}\``)
54926
+ .join(", ")}. ` +
54927
+ `Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
54928
+ `producers move to Postgres, convert the declarations to plain local declarations instead ` +
54929
+ `of BigQuery connections.`);
54930
+ }
54733
54931
  }
54734
54932
  lines.push("");
54735
54933
  lines.push(`## Target files (${targets.length})`);
@@ -54751,7 +54949,9 @@ function renderReportMd(r) {
54751
54949
  lines.push("");
54752
54950
  lines.push(`## Declarations (${declarations.length})`);
54753
54951
  lines.push("");
54754
- lines.push(`Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
54952
+ lines.push(r.targetWarehouse === "bigquery"
54953
+ ? `Declarations are unchanged — the sources are native to the warehouse.`
54954
+ : `Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
54755
54955
  if (r.warnings.length > 0) {
54756
54956
  lines.push("");
54757
54957
  lines.push(`## Warnings`);
@@ -54762,958 +54962,1201 @@ function renderReportMd(r) {
54762
54962
  lines.push("");
54763
54963
  lines.push(`## Finish the migration`);
54764
54964
  lines.push("");
54765
- lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
54766
- lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
54767
- lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
54768
- `model is the migration to-do list. Work the flagged files until validate is green.`);
54769
- lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
54770
- `the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
54965
+ if (r.targetWarehouse === "bigquery") {
54966
+ lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
54967
+ lines.push(`2. Credentials: \`gcloud auth application-default login\` (ADC), or a service-account ` +
54968
+ `key in a gitignored \`.df-credentials.json\`.`);
54969
+ lines.push(`3. \`sqlanvil validate\` — dry-runs every model against BigQuery without executing; ` +
54970
+ `all-PASS means the swap is complete.`);
54971
+ lines.push(`4. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
54972
+ `sample of actions — they should match apart from the compile-global rename.`);
54973
+ }
54974
+ else {
54975
+ lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
54976
+ lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
54977
+ lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
54978
+ `model is the migration to-do list. Work the flagged files until validate is green.`);
54979
+ lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
54980
+ `the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
54981
+ }
54771
54982
  lines.push("");
54772
54983
  return lines.join("\n");
54773
54984
  }
54774
54985
 
54775
- const CHECKER_SOURCE = `
54776
- import json, os, sys
54777
-
54778
- script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
54779
- errors = []
54780
-
54781
- import platform
54782
- py_version = platform.python_version()
54783
-
54784
- def load_packaging():
54785
- try:
54786
- from packaging.specifiers import SpecifierSet
54787
- from packaging.requirements import Requirement
54788
- return SpecifierSet, Requirement
54789
- except ImportError:
54790
- pass
54791
- try:
54792
- from pip._vendor.packaging.specifiers import SpecifierSet
54793
- from pip._vendor.packaging.requirements import Requirement
54794
- return SpecifierSet, Requirement
54795
- except ImportError:
54796
- return None, None
54797
-
54798
- SpecifierSet, Requirement = load_packaging()
54799
-
54800
- def mini_parse(v):
54801
- import re
54802
- return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
54803
-
54804
- def mini_check(version, spec):
54805
- import re
54806
- ok = True
54807
- for clause in spec.split(","):
54808
- clause = clause.strip()
54809
- m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
54810
- if not m:
54811
- return None
54812
- op, want = m.group(1), m.group(2).strip()
54813
- if want.endswith(".*"):
54814
- prefix = mini_parse(want[:-2])
54815
- have = mini_parse(version)[:len(prefix)]
54816
- eq = have == prefix
54817
- if (op in ("==", "===") and not eq) or (op == "!=" and eq):
54818
- ok = False
54819
- continue
54820
- a, b = mini_parse(version), mini_parse(want)
54821
- n = max(len(a), len(b))
54822
- a = a + [0] * (n - len(a))
54823
- b = b + [0] * (n - len(b))
54824
- if op in ("==", "==="):
54825
- res = a == b
54826
- elif op == "!=":
54827
- res = a != b
54828
- elif op == "<=":
54829
- res = a <= b
54830
- elif op == ">=":
54831
- res = a >= b
54832
- elif op == "<":
54833
- res = a < b
54834
- elif op == ">":
54835
- res = a > b
54836
- else: # ~= : at least b, within b's series
54837
- wb = mini_parse(want)
54838
- res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
54839
- if not res:
54840
- ok = False
54841
- return ok
54842
-
54843
- def check_version(version, spec, what):
54844
- if SpecifierSet is not None:
54845
- try:
54846
- if version not in SpecifierSet(spec, prereleases=True):
54847
- errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
54848
- return
54849
- except Exception as e:
54850
- errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
54851
- return
54852
- if mini_check(version, spec) is False:
54853
- errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
54854
-
54855
- if version_spec:
54856
- check_version(py_version, version_spec, "python interpreter")
54857
-
54858
- if requirements_path:
54859
- from importlib import metadata
54860
- try:
54861
- lines = open(requirements_path).read().splitlines()
54862
- except OSError as e:
54863
- errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
54864
- lines = []
54865
- for raw in lines:
54866
- line = raw.split("#")[0].strip()
54867
- if not line or line.startswith("-"):
54868
- continue
54869
- if Requirement is not None:
54870
- try:
54871
- req = Requirement(line)
54872
- except Exception:
54873
- errors.append('requirements: cannot parse "%s"' % line)
54874
- continue
54875
- if req.marker is not None and not req.marker.evaluate():
54876
- continue
54877
- name, spec = req.name, str(req.specifier)
54878
- else:
54879
- import re
54880
- if ";" in line:
54881
- line = line.split(";")[0].strip()
54882
- m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
54883
- if not m:
54884
- errors.append('requirements: cannot parse "%s"' % line)
54885
- continue
54886
- name, spec = m.group(1), (m.group(3) or "").strip()
54887
- try:
54888
- installed = metadata.version(name)
54889
- except metadata.PackageNotFoundError:
54890
- errors.append(
54891
- "requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
54892
- )
54893
- continue
54894
- if spec:
54895
- check_version(installed, spec, "requirements: %s" % name)
54896
-
54897
- try:
54898
- with open(script_path, "rb") as f:
54899
- compile(f.read(), script_path, "exec")
54900
- except SyntaxError as e:
54901
- errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
54902
- except (OSError, ValueError) as e:
54903
- errors.append("syntax: cannot compile %s: %s" % (script_path, e))
54904
-
54905
- print(json.dumps({"interpreter_version": py_version, "errors": errors}))
54906
- `;
54907
- const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
54908
- childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
54909
- if (err) {
54910
- reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
54911
- }
54912
- else {
54913
- resolve(stdout);
54914
- }
54915
- });
54916
- });
54917
- const failure = (message) => ({
54918
- status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
54919
- error: { message }
54920
- });
54921
- async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
54922
- if (script.language !== "python") {
54923
- return [failure(`no validator for script language "${script.language}"`)];
54924
- }
54925
- let interpreter;
54926
- try {
54927
- interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
54928
- }
54929
- catch (e) {
54930
- return [failure(e.message)];
54931
- }
54932
- let stdout;
54933
- try {
54934
- stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
54935
- }
54936
- catch (e) {
54937
- return [failure(e.message)];
54938
- }
54939
- let parsed;
54940
- try {
54941
- parsed = JSON.parse(stdout);
54986
+ function actuallyResolve(...filePaths) {
54987
+ return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
54988
+ }
54989
+ function assertPathExists(checkPath) {
54990
+ if (!fs__namespace$1.existsSync(checkPath)) {
54991
+ throw new Error(`${checkPath} does not exist!`);
54942
54992
  }
54943
- catch (e) {
54944
- return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
54993
+ }
54994
+ function compiledGraphHasErrors(graph) {
54995
+ var _a, _b;
54996
+ return ((_b = (_a = graph.graphErrors) === null || _a === void 0 ? void 0 : _a.compilationErrors) === null || _b === void 0 ? void 0 : _b.length) > 0;
54997
+ }
54998
+ function formatExecutionSuffix(jobIds, bytesBilled) {
54999
+ const jobMetadataParts = [];
55000
+ if (jobIds.length > 0) {
55001
+ jobMetadataParts.push(`\n \t jobId: ${jobIds.join(", ")}`);
54945
55002
  }
54946
- if (parsed.errors && parsed.errors.length > 0) {
54947
- return parsed.errors.map(failure);
55003
+ if (bytesBilled.length > 0) {
55004
+ jobMetadataParts.push(`\n \t Bytes billed: ${bytesBilled.join(", ")}`);
54948
55005
  }
54949
- return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
54950
- }
54951
-
54952
- function targetKey(target) {
54953
- return [target.database, target.schema, target.name].filter(Boolean).join(".");
55006
+ return jobMetadataParts.length > 0 ? ` ${jobMetadataParts}` : "";
54954
55007
  }
54955
- function topoOrder(nodes) {
54956
- const byKey = new Map(nodes.map(n => [n.key, n]));
54957
- const indegree = new Map();
54958
- const dependents = new Map();
54959
- for (const n of nodes) {
54960
- const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
54961
- indegree.set(n.key, inGraphDeps.length);
54962
- for (const dep of inGraphDeps) {
54963
- dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
54964
- }
55008
+ function formatBytesInHumanReadableFormat(bytes) {
55009
+ if (bytes <= 0) {
55010
+ return '0 B';
54965
55011
  }
54966
- const ready = nodes
54967
- .filter(n => (indegree.get(n.key) || 0) === 0)
54968
- .map(n => n.key)
54969
- .sort();
54970
- const ordered = [];
54971
- const emitted = new Set();
54972
- while (ready.length) {
54973
- const key = ready.shift();
54974
- if (emitted.has(key)) {
54975
- continue;
55012
+ const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
55013
+ const k = 1024;
55014
+ const i = Math.floor(Math.log(bytes) / Math.log(k));
55015
+ const value = (bytes / Math.pow(k, i)).toFixed(2);
55016
+ return `${value} ${units[i]}`;
55017
+ }
55018
+ function promptForIcebergConfig() {
55019
+ print(ICEBERG_CONFIG_PROMPT_TEXT);
55020
+ print(ICEBERG_CONFIG_PROMPT_HINT);
55021
+ const tempIcebergConfig = {};
55022
+ let bucketName;
55023
+ while (true) {
55024
+ print(ICEBERG_BUCKET_NAME_HINT);
55025
+ bucketName = interactiveQuestion(ICEBERG_BUCKET_NAME_PROMPT_QUESTION);
55026
+ try {
55027
+ if (bucketName) {
55028
+ validateIcebergConfigBucketName(bucketName);
55029
+ tempIcebergConfig.bucketName = bucketName;
55030
+ }
55031
+ break;
54976
55032
  }
54977
- emitted.add(key);
54978
- ordered.push(byKey.get(key));
54979
- for (const dep of (dependents.get(key) || []).slice().sort()) {
54980
- indegree.set(dep, (indegree.get(dep) || 0) - 1);
54981
- if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
54982
- ready.push(dep);
54983
- ready.sort();
55033
+ catch (e) {
55034
+ printError(`Validation Error: ${e.message}`);
55035
+ print("Please try again.");
55036
+ }
55037
+ }
55038
+ let tableFolderRoot;
55039
+ while (true) {
55040
+ print(ICEBERG_TABLE_FOLDER_ROOT_HINT);
55041
+ tableFolderRoot = interactiveQuestion(ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION);
55042
+ try {
55043
+ if (tableFolderRoot) {
55044
+ validateIcebergConfigTableFolderRoot(tableFolderRoot);
55045
+ tempIcebergConfig.tableFolderRoot = tableFolderRoot;
54984
55046
  }
55047
+ break;
55048
+ }
55049
+ catch (e) {
55050
+ printError(`Validation Error: ${e.message}`);
55051
+ print("Please try again.");
54985
55052
  }
54986
55053
  }
54987
- if (ordered.length < nodes.length) {
54988
- for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
54989
- if (!emitted.has(n.key)) {
54990
- emitted.add(n.key);
54991
- ordered.push(n);
55054
+ let tableFolderSubpath;
55055
+ while (true) {
55056
+ print(ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT);
55057
+ tableFolderSubpath = interactiveQuestion(ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION);
55058
+ try {
55059
+ if (tableFolderSubpath) {
55060
+ validateIcebergConfigTableFolderSubpath(tableFolderSubpath);
55061
+ tempIcebergConfig.tableFolderSubpath = tableFolderSubpath;
55062
+ }
55063
+ break;
55064
+ }
55065
+ catch (e) {
55066
+ printError(`Validation Error: ${e.message}`);
55067
+ print("Please try again.");
55068
+ }
55069
+ }
55070
+ let connection;
55071
+ while (true) {
55072
+ print(ICEBERG_CONNECTION_HINT);
55073
+ connection = interactiveQuestion(ICEBERG_CONNECTION_QUESTION);
55074
+ try {
55075
+ if (connection) {
55076
+ validateConnectionFormat(connection);
55077
+ tempIcebergConfig.connection = connection;
54992
55078
  }
55079
+ break;
55080
+ }
55081
+ catch (e) {
55082
+ printError(`Validation Error: ${e.message}`);
55083
+ print("Please try again.");
54993
55084
  }
54994
55085
  }
54995
- return ordered;
54996
- }
54997
- function dependencyBlocked(dependencyKeys, statusByKey) {
54998
- return dependencyKeys.some(dep => {
54999
- const status = statusByKey.get(dep);
55000
- return status !== undefined && status !== "PASS";
55001
- });
55086
+ printSuccess(ICEBERG_CONFIG_COLLECTED_TEXT);
55087
+ if (Object.keys(tempIcebergConfig).length > 0) {
55088
+ return tempIcebergConfig;
55089
+ }
55090
+ return undefined;
55002
55091
  }
55003
- const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
55004
- function validateShadowSuffix(nowMs) {
55005
- return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
55092
+ function validateIcebergConfigBucketName(bucketName) {
55093
+ if (bucketName.length < 3 || bucketName.length > 63) {
55094
+ throw new Error("Bucket name must be between 3 and 63 characters long.");
55095
+ }
55096
+ if (!/^[a-z0-9][a-z0-9-._]{1,220}[a-z0-9]$/.test(bucketName) || bucketName.includes("..")) {
55097
+ throw new Error("Invalid bucket name. Must start and end with a letter or number, contain only lowercase letters, numbers, hyphens (-), underscores (_), and periods (.).");
55098
+ }
55099
+ if (bucketName.startsWith("goog") || bucketName.includes("--")) {
55100
+ throw new Error("Bucket name cannot start with 'goog' or contain '--'.");
55101
+ }
55102
+ if (bucketName.includes("google") || bucketName.includes("g00gle")) {
55103
+ throw new Error("Bucket name cannot contain 'google' or close misspellings such as 'g00gle'.");
55104
+ }
55006
55105
  }
55007
- function parseShadowTimestamp(schemaName) {
55008
- const match = schemaName.match(/sqlanvil_validate_(\d+)/);
55009
- return match ? Number(match[1]) : null;
55106
+ function validateIcebergConfigTableFolderRoot(tableFolderRoot) {
55107
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9-._]{1,220}[a-zA-Z0-9]$/.test(tableFolderRoot) || tableFolderRoot.includes("..")) {
55108
+ throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), and periods (.).");
55109
+ }
55010
55110
  }
55011
- function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
55012
- return schemaNames.filter(name => {
55013
- const ts = parseShadowTimestamp(name);
55014
- return ts !== null && nowMs - ts > maxAgeMs;
55015
- });
55111
+ function validateIcebergConfigTableFolderSubpath(tableFolderSubpath) {
55112
+ if (!/^[a-zA-Z0-9][a-zA-Z0-9-._/]{1,220}[a-zA-Z0-9]$/.test(tableFolderSubpath) || tableFolderSubpath.includes("..")) {
55113
+ throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), periods (.), and forward slashes (/). The sequence '..' is also disallowed.");
55114
+ }
55115
+ if (tableFolderSubpath.includes("./") || tableFolderSubpath.includes("../")) {
55116
+ throw new Error("Input cannot contain './' or '../'.");
55117
+ }
55016
55118
  }
55119
+ const ICEBERG_BUCKET_NAME_HINT = "The bucket name must comply with https://cloud.google.com/storage/docs/buckets#naming. If you do not want to provide a workflow-level default bucket name, leave this input empty.\n";
55120
+ const ICEBERG_TABLE_FOLDER_ROOT_HINT = "Table folder root must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores and periods. If you do not want to provide a workflow_level default table folder root, leave this input empty.\n";
55121
+ const ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT = "Table folder subpath must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores, periods and forward slashes. If you do not want to provide a workflow_level default table folder subpath, leave this input empty.\n";
55122
+ const ICEBERG_CONFIG_PROMPT_HINT = "Set repository-level configuration for Iceberg bucket name, table folder root, table folder subpath and connection. If you do not want to set a field, enter an empty string in response to the prompt.\n";
55123
+ const ICEBERG_CONFIG_PROMPT_TEXT = "\n--- Iceberg Configuration ---\n";
55124
+ const ICEBERG_CONFIG_COLLECTED_TEXT = "Iceberg configuration collected.\n";
55125
+ const ICEBERG_BUCKET_NAME_PROMPT_QUESTION = "Enter the default Iceberg bucket name:";
55126
+ const ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION = "Enter the default Iceberg table folder root:";
55127
+ const ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION = "Enter the default Iceberg table folder subpath:";
55128
+ const ICEBERG_CONNECTION_HINT = "The connection can have the form `{project}.{location}.{connection_id}` or `projects/{project}/locations/{location}/connections/{connection_id} or be set to DEFAULT. If you do not want to provide a workflow-level default connection, leave this input empty.\n";
55129
+ const ICEBERG_CONNECTION_QUESTION = "Enter the default connection:";
55017
55130
 
55018
- const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
55019
- async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
55131
+ const noColor = process.env.NO_COLOR && ["1", "true", "yes"].includes(process.env.NO_COLOR.toLowerCase());
55132
+ const ansiColorCodes = Object.freeze({
55133
+ red: 91,
55134
+ green: 32,
55135
+ yellow: 93,
55136
+ cyan: 36
55137
+ });
55138
+ function output(text, ansiColorCode) {
55139
+ if (noColor) {
55140
+ return text;
55141
+ }
55142
+ return `\x1b[${ansiColorCode}m${text}\x1b[0m`;
55143
+ }
55144
+ const successOutput = (text) => output(text, ansiColorCodes.green);
55145
+ const warningOutput = (text) => output(text, ansiColorCodes.yellow);
55146
+ const errorOutput = (text) => output(text, ansiColorCodes.red);
55147
+ const calloutOutput = (text) => output(text, ansiColorCodes.cyan);
55148
+ const write = (stream, text, indentCount) => stream.write(`${" ".repeat(indentCount)}${text}\n`);
55149
+ const writeStdOut = (text, indentCount = 0) => write(process.stdout, text, indentCount);
55150
+ const writeStdErr = (text, indentCount = 0) => write(process.stderr, text, indentCount);
55151
+ const DEFAULT_PROMPT = "> ";
55152
+ function question(questionText) {
55153
+ return prompt(questionText);
55154
+ }
55155
+ function interactiveQuestion(questionText) {
55156
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
55157
+ const testInput = getTestInput(questionText);
55158
+ print(`${questionText} ${testInput}\n`);
55159
+ return testInput;
55160
+ }
55161
+ print(`${questionText} `);
55162
+ return readlineSync__namespace.question("");
55163
+ }
55164
+ function getTestInput(questionText) {
55165
+ const envVar = process.env.DATAFORM_CLI_TEST_INPUTS;
55166
+ if (!envVar) {
55167
+ throw new Error("Environment variable DATAFORM_CLI_TEST_INPUTS not set.");
55168
+ }
55020
55169
  try {
55021
- const schemas = await deps.listSchemas();
55022
- for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
55023
- try {
55024
- await deps.execute(deps.dropSchemaCascadeSql(schema));
55170
+ const parsed = JSON.parse(envVar);
55171
+ if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
55172
+ const inputs = new Map(Object.entries(parsed).map(([key, value]) => [key, String(value)]));
55173
+ const trimmedQuestion = questionText.trim();
55174
+ const answer = inputs.get(trimmedQuestion);
55175
+ if (answer !== undefined) {
55176
+ print(`[TEST_INPUT for "${trimmedQuestion}"]: "${answer}"`);
55177
+ return answer;
55025
55178
  }
55026
- catch (e) {
55179
+ else {
55180
+ throw new Error(`[MISSING TEST_INPUT for "${trimmedQuestion}"]`);
55027
55181
  }
55028
55182
  }
55183
+ else {
55184
+ throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: Expected a JSON object, but got ${typeof parsed}.`);
55185
+ }
55029
55186
  }
55030
55187
  catch (e) {
55188
+ throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
55031
55189
  }
55032
55190
  }
55033
- function tableType(enumType) {
55034
- switch (enumType) {
55035
- case sqlanvil.TableType.VIEW:
55036
- return "view";
55037
- case sqlanvil.TableType.INCREMENTAL:
55038
- return "incremental";
55039
- default:
55040
- return "table";
55191
+ function passwordQuestion(questionText) {
55192
+ return prompt(questionText, {
55193
+ hideEchoBack: true,
55194
+ mask: ""
55195
+ });
55196
+ }
55197
+ function interactivePasswordQuestion(questionText) {
55198
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
55199
+ const testInput = getTestInput(questionText);
55200
+ print(`${questionText} ********\n`);
55201
+ return testInput;
55041
55202
  }
55203
+ return passwordQuestion(questionText);
55042
55204
  }
55043
- function depKeys(deps) {
55044
- return (deps || []).map(targetKey);
55205
+ function selectionQuestion(questionText, options) {
55206
+ return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
55207
+ cancel: false
55208
+ });
55045
55209
  }
55046
- async function validate(compiledGraph, deps, options = {}) {
55047
- const nodes = [];
55048
- for (const table of compiledGraph.tables || []) {
55049
- nodes.push({
55050
- key: targetKey(table.target),
55051
- dependencyKeys: depKeys(table.dependencyTargets),
55052
- kind: "table",
55053
- type: tableType(table.enumType),
55054
- target: table.target,
55055
- table,
55056
- action: table
55057
- });
55058
- }
55059
- for (const assertion of compiledGraph.assertions || []) {
55060
- nodes.push({
55061
- key: targetKey(assertion.target),
55062
- dependencyKeys: depKeys(assertion.dependencyTargets),
55063
- kind: "assertion",
55064
- type: "assertion",
55065
- target: assertion.target,
55066
- action: assertion
55067
- });
55210
+ function prompt(questionText, options) {
55211
+ writeStdOut(questionText);
55212
+ return readlineSync__namespace.prompt(Object.assign(Object.assign({}, options), { prompt: (options && options.prompt && options.prompt + DEFAULT_PROMPT) || DEFAULT_PROMPT }));
55213
+ }
55214
+ function print(text) {
55215
+ writeStdOut(text);
55216
+ }
55217
+ function printSuccess(text) {
55218
+ writeStdOut(successOutput(text));
55219
+ }
55220
+ function printError(errorText, indentCount = 0) {
55221
+ writeStdErr(errorOutput(errorText), indentCount);
55222
+ }
55223
+ function printInitResult(result) {
55224
+ if (result.dirsCreated && result.dirsCreated.length) {
55225
+ writeStdOut(successOutput("Directories successfully created:"));
55226
+ result.dirsCreated.forEach(dir => writeStdOut(dir, 1));
55068
55227
  }
55069
- for (const operation of compiledGraph.operations || []) {
55070
- if (!operation.target) {
55071
- continue;
55072
- }
55073
- nodes.push({
55074
- key: targetKey(operation.target),
55075
- dependencyKeys: depKeys(operation.dependencyTargets),
55076
- kind: "operation",
55077
- type: "operation",
55078
- target: operation.target,
55079
- action: undefined
55080
- });
55228
+ if (result.filesWritten && result.filesWritten.length) {
55229
+ writeStdOut(successOutput("Files successfully written:"));
55230
+ result.filesWritten.forEach(file => writeStdOut(file, 1));
55081
55231
  }
55082
- for (const imp of compiledGraph.imports || []) {
55083
- if (!imp.target) {
55084
- continue;
55085
- }
55086
- nodes.push({
55087
- key: targetKey(imp.target),
55088
- dependencyKeys: depKeys(imp.dependencyTargets),
55089
- kind: "import",
55090
- type: "import",
55091
- target: imp.target,
55092
- action: undefined
55093
- });
55232
+ }
55233
+ function printInitCredsResult(writtenFilePath) {
55234
+ writeStdOut(successOutput("Credentials file successfully written:"));
55235
+ writeStdOut(writtenFilePath, 1);
55236
+ writeStdOut("To change connection settings, edit this file directly.");
55237
+ }
55238
+ function isInteractive({ stream = process.stdout } = {}) {
55239
+ return Boolean(stream && stream.isTTY &&
55240
+ process.env.TERM !== 'dumb' &&
55241
+ !('CI' in process.env));
55242
+ }
55243
+ var compiledGraphOutputType;
55244
+ (function (compiledGraphOutputType) {
55245
+ compiledGraphOutputType["Json"] = "json";
55246
+ compiledGraphOutputType["Dot"] = "dot";
55247
+ compiledGraphOutputType["Summary"] = "summary";
55248
+ })(compiledGraphOutputType || (compiledGraphOutputType = {}));
55249
+ function printCompiledGraph(graph, outputType, quietCompilation) {
55250
+ const interactive = isInteractive();
55251
+ if (outputType === compiledGraphOutputType.Json) {
55252
+ writeStdOut(prettyJsonStringify(graph));
55094
55253
  }
55095
- for (const script of compiledGraph.scripts || []) {
55096
- if (!script.target) {
55097
- continue;
55098
- }
55099
- nodes.push({
55100
- key: targetKey(script.target),
55101
- dependencyKeys: depKeys(script.dependencyTargets),
55102
- kind: "script",
55103
- type: "script",
55104
- target: script.target,
55105
- script,
55106
- action: undefined
55107
- });
55254
+ else if (outputType === compiledGraphOutputType.Dot) {
55255
+ writeStdOut(dotRepresentation(graph, interactive));
55108
55256
  }
55109
- const ordered = topoOrder(nodes);
55110
- const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
55111
- const statusByKey = new Map();
55112
- const results = [];
55113
- try {
55114
- for (const schema of shadowSchemas) {
55115
- await deps.execute(deps.createSchemaSql(schema));
55116
- }
55117
- for (const node of ordered) {
55118
- if (node.kind === "operation" ||
55119
- node.kind === "import" ||
55120
- (node.kind === "script" && !deps.checkScript)) {
55121
- statusByKey.set(node.key, "SKIPPED");
55122
- results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
55123
- continue;
55124
- }
55125
- if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
55126
- statusByKey.set(node.key, "BLOCKED");
55127
- results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
55128
- continue;
55257
+ else {
55258
+ const actionCount = 0 +
55259
+ (graph.tables ? graph.tables.length : 0) +
55260
+ (graph.assertions ? graph.assertions.length : 0) +
55261
+ (graph.operations ? graph.operations.length : 0);
55262
+ writeStdOut(successOutput(`Compiled ${actionCount} action(s).`));
55263
+ if (graph.tables && graph.tables.length) {
55264
+ graph.tables.forEach(setOrValidateTableEnumType);
55265
+ writeStdOut(`${graph.tables.length} dataset(s)${quietCompilation ? "." : ":"}`);
55266
+ if (!quietCompilation) {
55267
+ graph.tables.forEach(compiledTable => {
55268
+ writeStdOut(`${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`, 1);
55269
+ });
55129
55270
  }
55130
- const evaluations = node.kind === "script"
55131
- ? await deps.checkScript(node.script)
55132
- : await deps.evaluate(node.action);
55133
- const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
55134
- const status = failed ? "FAILURE" : "PASS";
55135
- statusByKey.set(node.key, status);
55136
- results.push({ target: node.target, type: node.type, status, errors: evaluations });
55137
- if (status === "PASS" && node.kind === "table") {
55138
- try {
55139
- await deps.execute(deps.validationStubSql(node.table));
55140
- }
55141
- catch (e) {
55142
- }
55271
+ }
55272
+ if (graph.assertions && graph.assertions.length) {
55273
+ writeStdOut(`${graph.assertions.length} assertion(s)${quietCompilation ? "." : ":"}`);
55274
+ if (!quietCompilation) {
55275
+ graph.assertions.forEach(assertion => {
55276
+ writeStdOut(assertionString(assertion.target, assertion.disabled), 1);
55277
+ });
55143
55278
  }
55144
55279
  }
55145
- }
55146
- finally {
55147
- if (!options.keepShadow) {
55148
- for (const schema of shadowSchemas.slice().reverse()) {
55149
- try {
55150
- await deps.execute(deps.dropSchemaCascadeSql(schema));
55151
- }
55152
- catch (e) {
55153
- }
55280
+ if (graph.operations && graph.operations.length) {
55281
+ writeStdOut(`${graph.operations.length} operation(s)${quietCompilation ? "." : ":"}`);
55282
+ if (!quietCompilation) {
55283
+ graph.operations.forEach(operation => {
55284
+ writeStdOut(operationString(operation.target, operation.disabled), 1);
55285
+ });
55154
55286
  }
55155
55287
  }
55156
55288
  }
55157
- return results;
55158
55289
  }
55159
-
55160
- class SupabaseDbAdapter extends PostgresDbAdapter {
55161
- static async create(credentials, options) {
55162
- const sslMode = (credentials.sslMode || "").toLowerCase();
55163
- const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
55164
- const clientConfig = {
55165
- host: credentials.host,
55166
- port: credentials.port,
55167
- database: credentials.database,
55168
- user: credentials.user,
55169
- password: credentials.password,
55170
- ssl: sslEnabled
55171
- ? {
55172
- rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
55173
- }
55174
- : false
55175
- };
55176
- const queryExecutor = new PgPoolExecutor(clientConfig, options);
55177
- try {
55178
- await queryExecutor.verifyConnection();
55179
- }
55180
- catch (e) {
55181
- await queryExecutor.close().catch(() => undefined);
55182
- throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
55183
- `as "${credentials.user}": ${e.message}`, e);
55184
- }
55185
- return new SupabaseDbAdapter(queryExecutor);
55186
- }
55187
- constructor(queryExecutor) {
55188
- super(queryExecutor);
55189
- }
55190
- async verifyServiceRolePermissions() {
55191
- var _a;
55192
- try {
55193
- const result = await this.execute("select current_user");
55194
- const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
55195
- return !!currentUser;
55196
- }
55197
- catch (_b) {
55198
- return false;
55199
- }
55200
- }
55201
- async validatePublicationState(pubName = "supabase_realtime") {
55202
- try {
55203
- const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
55204
- params: [pubName]
55205
- });
55206
- return result.rows.length > 0;
55207
- }
55208
- catch (_a) {
55209
- return false;
55290
+ function formatStackTraceForQuietCompilation(compileError) {
55291
+ var _a, _b;
55292
+ if ((_a = compileError === null || compileError === void 0 ? void 0 : compileError.message) === null || _a === void 0 ? void 0 : _a.includes("Unexpected identifier")) {
55293
+ if (!compileError.stack) {
55294
+ return "";
55210
55295
  }
55296
+ const stackLines = ((_b = compileError.stack) === null || _b === void 0 ? void 0 : _b.split("\n")) || [];
55297
+ return stackLines.slice(0, Math.min(3, stackLines.length)).join("\n");
55211
55298
  }
55299
+ return "";
55212
55300
  }
55213
-
55214
- function actuallyResolve(...filePaths) {
55215
- return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
55216
- }
55217
- function assertPathExists(checkPath) {
55218
- if (!fs__namespace$1.existsSync(checkPath)) {
55219
- throw new Error(`${checkPath} does not exist!`);
55301
+ function printCompiledGraphErrors(graphErrors, quietCompilation) {
55302
+ if (graphErrors.compilationErrors && graphErrors.compilationErrors.length > 0) {
55303
+ printError("Compilation errors:", 1);
55304
+ graphErrors.compilationErrors.forEach(compileError => {
55305
+ writeStdErr(`${calloutOutput(compileError.fileName)}: ${errorOutput(quietCompilation ? (compileError.message + " " + formatStackTraceForQuietCompilation(compileError) || compileError.stack) : (compileError.stack || compileError.message))}`, 1);
55306
+ });
55220
55307
  }
55221
55308
  }
55222
- function compiledGraphHasErrors(graph) {
55223
- var _a, _b;
55224
- return ((_b = (_a = graph.graphErrors) === null || _a === void 0 ? void 0 : _a.compilationErrors) === null || _b === void 0 ? void 0 : _b.length) > 0;
55225
- }
55226
- function formatExecutionSuffix(jobIds, bytesBilled) {
55227
- const jobMetadataParts = [];
55228
- if (jobIds.length > 0) {
55229
- jobMetadataParts.push(`\n \t jobId: ${jobIds.join(", ")}`);
55230
- }
55231
- if (bytesBilled.length > 0) {
55232
- jobMetadataParts.push(`\n \t Bytes billed: ${bytesBilled.join(", ")}`);
55309
+ function printTestResult(testResult) {
55310
+ writeStdOut(`${testResult.name}: ${testResult.successful ? successOutput("passed") : errorOutput("failed")}`);
55311
+ if (!testResult.successful) {
55312
+ testResult.messages.forEach(message => writeStdErr(message, 1));
55233
55313
  }
55234
- return jobMetadataParts.length > 0 ? ` ${jobMetadataParts}` : "";
55235
55314
  }
55236
- function formatBytesInHumanReadableFormat(bytes) {
55237
- if (bytes <= 0) {
55238
- return '0 B';
55315
+ function printExecutionGraph(executionGraph, asJson) {
55316
+ if (asJson) {
55317
+ writeStdOut(prettyJsonStringify(executionGraph.toJSON()));
55239
55318
  }
55240
- const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
55241
- const k = 1024;
55242
- const i = Math.floor(Math.log(bytes) / Math.log(k));
55243
- const value = (bytes / Math.pow(k, i)).toFixed(2);
55244
- return `${value} ${units[i]}`;
55245
- }
55246
- function promptForIcebergConfig() {
55247
- print(ICEBERG_CONFIG_PROMPT_TEXT);
55248
- print(ICEBERG_CONFIG_PROMPT_HINT);
55249
- const tempIcebergConfig = {};
55250
- let bucketName;
55251
- while (true) {
55252
- print(ICEBERG_BUCKET_NAME_HINT);
55253
- bucketName = interactiveQuestion(ICEBERG_BUCKET_NAME_PROMPT_QUESTION);
55254
- try {
55255
- if (bucketName) {
55256
- validateIcebergConfigBucketName(bucketName);
55257
- tempIcebergConfig.bucketName = bucketName;
55319
+ else {
55320
+ const actionsByType = {
55321
+ table: [],
55322
+ assertion: [],
55323
+ operation: []
55324
+ };
55325
+ executionGraph.actions.forEach(action => {
55326
+ if (!(action.type === "table" || action.type === "assertion" || action.type === "operation")) {
55327
+ throw new Error(`Unrecognized action type: ${action.type}`);
55258
55328
  }
55259
- break;
55260
- }
55261
- catch (e) {
55262
- printError(`Validation Error: ${e.message}`);
55263
- print("Please try again.");
55329
+ actionsByType[action.type].push(action);
55330
+ });
55331
+ const datasetActions = actionsByType.table;
55332
+ if (datasetActions && datasetActions.length) {
55333
+ writeStdOut(`${datasetActions.length} dataset(s):`);
55334
+ datasetActions.forEach(datasetAction => writeStdOut(datasetString(datasetAction.target, datasetAction.type, datasetAction.tasks.length === 0), 1));
55264
55335
  }
55265
- }
55266
- let tableFolderRoot;
55267
- while (true) {
55268
- print(ICEBERG_TABLE_FOLDER_ROOT_HINT);
55269
- tableFolderRoot = interactiveQuestion(ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION);
55270
- try {
55271
- if (tableFolderRoot) {
55272
- validateIcebergConfigTableFolderRoot(tableFolderRoot);
55273
- tempIcebergConfig.tableFolderRoot = tableFolderRoot;
55274
- }
55275
- break;
55336
+ const assertionActions = actionsByType.assertion;
55337
+ if (assertionActions && assertionActions.length) {
55338
+ writeStdOut(`${assertionActions.length} assertion(s):`);
55339
+ assertionActions.forEach(assertionAction => writeStdOut(assertionString(assertionAction.target, assertionAction.tasks.length === 0), 1));
55276
55340
  }
55277
- catch (e) {
55278
- printError(`Validation Error: ${e.message}`);
55279
- print("Please try again.");
55341
+ const operationActions = actionsByType.operation;
55342
+ if (operationActions && operationActions.length) {
55343
+ writeStdOut(`${operationActions.length} operation(s):`);
55344
+ operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
55280
55345
  }
55281
55346
  }
55282
- let tableFolderSubpath;
55283
- while (true) {
55284
- print(ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT);
55285
- tableFolderSubpath = interactiveQuestion(ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION);
55286
- try {
55287
- if (tableFolderSubpath) {
55288
- validateIcebergConfigTableFolderSubpath(tableFolderSubpath);
55289
- tempIcebergConfig.tableFolderSubpath = tableFolderSubpath;
55347
+ }
55348
+ function printExecutedAction(executedAction, executionAction, dryRun) {
55349
+ const jobIds = executedAction.tasks
55350
+ .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55351
+ .map(task => task.metadata.bigquery.jobId);
55352
+ const bytesBilled = executedAction.tasks
55353
+ .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55354
+ .map(task => {
55355
+ var _a, _b, _c, _d;
55356
+ const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
55357
+ return formatBytesInHumanReadableFormat(bytes);
55358
+ });
55359
+ const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
55360
+ switch (executedAction.status) {
55361
+ case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
55362
+ switch (executionAction.type) {
55363
+ case "table": {
55364
+ writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55365
+ return;
55366
+ }
55367
+ case "assertion": {
55368
+ writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55369
+ return;
55370
+ }
55371
+ case "operation": {
55372
+ writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55373
+ return;
55374
+ }
55290
55375
  }
55291
- break;
55292
55376
  }
55293
- catch (e) {
55294
- printError(`Validation Error: ${e.message}`);
55295
- print("Please try again.");
55377
+ case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
55378
+ switch (executionAction.type) {
55379
+ case "table": {
55380
+ writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55381
+ break;
55382
+ }
55383
+ case "assertion": {
55384
+ writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55385
+ break;
55386
+ }
55387
+ case "operation": {
55388
+ writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55389
+ break;
55390
+ }
55391
+ }
55392
+ printExecutedActionErrors(executedAction, executionAction);
55393
+ return;
55296
55394
  }
55297
- }
55298
- let connection;
55299
- while (true) {
55300
- print(ICEBERG_CONNECTION_HINT);
55301
- connection = interactiveQuestion(ICEBERG_CONNECTION_QUESTION);
55302
- try {
55303
- if (connection) {
55304
- validateConnectionFormat(connection);
55305
- tempIcebergConfig.connection = connection;
55395
+ case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
55396
+ switch (executionAction.type) {
55397
+ case "table": {
55398
+ writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55399
+ return;
55400
+ }
55401
+ case "assertion": {
55402
+ writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55403
+ return;
55404
+ }
55405
+ case "operation": {
55406
+ writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55407
+ return;
55408
+ }
55306
55409
  }
55307
- break;
55410
+ return;
55308
55411
  }
55309
- catch (e) {
55310
- printError(`Validation Error: ${e.message}`);
55311
- print("Please try again.");
55412
+ case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
55413
+ switch (executionAction.type) {
55414
+ case "table": {
55415
+ writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55416
+ return;
55417
+ }
55418
+ case "assertion": {
55419
+ writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55420
+ return;
55421
+ }
55422
+ case "operation": {
55423
+ writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55424
+ return;
55425
+ }
55426
+ }
55427
+ return;
55312
55428
  }
55313
55429
  }
55314
- printSuccess(ICEBERG_CONFIG_COLLECTED_TEXT);
55315
- if (Object.keys(tempIcebergConfig).length > 0) {
55316
- return tempIcebergConfig;
55317
- }
55318
- return undefined;
55319
55430
  }
55320
- function validateIcebergConfigBucketName(bucketName) {
55321
- if (bucketName.length < 3 || bucketName.length > 63) {
55322
- throw new Error("Bucket name must be between 3 and 63 characters long.");
55431
+ function printFormatFilesResult(formatResults) {
55432
+ const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
55433
+ const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
55434
+ const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
55435
+ const failedFormatResults = sorted.filter(result => !!result.err);
55436
+ if (successfulFormatResults.length > 0) {
55437
+ printSuccess("Successfully formatted:");
55438
+ successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
55323
55439
  }
55324
- if (!/^[a-z0-9][a-z0-9-._]{1,220}[a-z0-9]$/.test(bucketName) || bucketName.includes("..")) {
55325
- throw new Error("Invalid bucket name. Must start and end with a letter or number, contain only lowercase letters, numbers, hyphens (-), underscores (_), and periods (.).");
55440
+ if (needsFormattingResults.length > 0) {
55441
+ printError("Files that need formatting:");
55442
+ needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
55326
55443
  }
55327
- if (bucketName.startsWith("goog") || bucketName.includes("--")) {
55328
- throw new Error("Bucket name cannot start with 'goog' or contain '--'.");
55444
+ if (failedFormatResults.length > 0) {
55445
+ printError("Errors encountered during formatting:");
55446
+ failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
55329
55447
  }
55330
- if (bucketName.includes("google") || bucketName.includes("g00gle")) {
55331
- throw new Error("Bucket name cannot contain 'google' or close misspellings such as 'g00gle'.");
55448
+ }
55449
+ function datasetString(target, datasetType, disabled) {
55450
+ return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
55451
+ }
55452
+ function assertionString(target, disabled) {
55453
+ return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55454
+ }
55455
+ function operationString(target, disabled) {
55456
+ return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55457
+ }
55458
+ function plainTargetString(target) {
55459
+ return `${target.schema}.${target.name}`;
55460
+ }
55461
+ function targetString(target) {
55462
+ return calloutOutput(`${target.schema}.${target.name}`);
55463
+ }
55464
+ function dotRepresentation(graph, interactive) {
55465
+ var _a, _b, _c;
55466
+ const nodes = [];
55467
+ const edges = [];
55468
+ const formatTarget = interactive ? targetString : plainTargetString;
55469
+ (_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
55470
+ var _a;
55471
+ const nodeName = `${formatTarget(table.target)}`;
55472
+ nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
55473
+ (_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55474
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55475
+ });
55476
+ });
55477
+ (_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
55478
+ var _a;
55479
+ const nodeName = `${formatTarget(assertion.target)}`;
55480
+ nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
55481
+ (_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55482
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55483
+ });
55484
+ });
55485
+ (_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
55486
+ var _a;
55487
+ const nodeName = `${formatTarget(operation.target)}`;
55488
+ nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
55489
+ (_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55490
+ edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55491
+ });
55492
+ });
55493
+ return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
55494
+ }
55495
+ function printExecutedActionErrors(executedAction, executionAction) {
55496
+ const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
55497
+ failingTasks.forEach((task, i) => {
55498
+ executionAction.tasks[i].statement.split("\n").forEach(line => {
55499
+ writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
55500
+ });
55501
+ printError(task.errorMessage, 1);
55502
+ });
55503
+ }
55504
+
55505
+ const INIT_MODE_QUESTION = "Start a fresh project, or convert an existing Dataform project? (fresh/convert)";
55506
+ const INIT_WAREHOUSE_QUESTION = "Which warehouse? (supabase/postgres/bigquery/mysql)";
55507
+ const INIT_PROJECT_DIR_QUESTION = "Project directory?";
55508
+ const INIT_BQ_PROJECT_QUESTION = "Google Cloud project ID (the default project models build in)?";
55509
+ const INIT_BQ_LOCATION_QUESTION = "Default BigQuery location (e.g. US, us-central1)?";
55510
+ const INIT_DEFAULT_SCHEMA_QUESTION = "Default schema for models?";
55511
+ const INIT_INCLUDE_SAMPLE_QUESTION = "Include the sample project (sources -> staging views -> outputs -> assertion)?";
55512
+ const INIT_INCLUDE_BQ_SOURCE_QUESTION = "Include the cross-warehouse BigQuery sample source (Google public ZIP data)?";
55513
+ const INIT_CONFIGURE_CREDS_QUESTION = `Configure warehouse credentials now (written to the gitignored ${CREDENTIALS_FILENAME})?`;
55514
+ const CREDS_HOST_QUESTION = "Host?";
55515
+ const CREDS_PORT_QUESTION = "Port?";
55516
+ const CREDS_DATABASE_QUESTION = "Database?";
55517
+ const CREDS_USER_QUESTION = "User?";
55518
+ const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
55519
+ const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
55520
+ const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
55521
+ const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
55522
+ const CONVERT_TARGET_QUESTION = "Target warehouse? (bigquery = keep running on BigQuery, tooling swap only; supabase/postgres = move off BigQuery)";
55523
+ const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
55524
+ "aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
55525
+ "db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
55526
+ const MAX_ATTEMPTS = 20;
55527
+ function withDefault(question, defaultValue) {
55528
+ return defaultValue ? `${question} [${defaultValue}]` : question;
55529
+ }
55530
+ function ask(question, defaultValue) {
55531
+ const answer = interactiveQuestion(withDefault(question, defaultValue)).trim();
55532
+ return answer === "" ? defaultValue !== null && defaultValue !== void 0 ? defaultValue : "" : answer;
55533
+ }
55534
+ function askRequired(question) {
55535
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55536
+ const answer = ask(question);
55537
+ if (answer !== "") {
55538
+ return answer;
55539
+ }
55540
+ printError("A value is required.");
55332
55541
  }
55542
+ throw new Error(`No answer provided for: ${question}`);
55333
55543
  }
55334
- function validateIcebergConfigTableFolderRoot(tableFolderRoot) {
55335
- if (!/^[a-zA-Z0-9][a-zA-Z0-9-._]{1,220}[a-zA-Z0-9]$/.test(tableFolderRoot) || tableFolderRoot.includes("..")) {
55336
- throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), and periods (.).");
55544
+ function askChoice(question, choices, defaultValue) {
55545
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55546
+ const answer = ask(question, defaultValue).toLowerCase();
55547
+ if (choices.includes(answer)) {
55548
+ return answer;
55549
+ }
55550
+ printError(`Please answer one of: ${choices.join(", ")}.`);
55337
55551
  }
55552
+ throw new Error(`No valid answer provided for: ${question}`);
55338
55553
  }
55339
- function validateIcebergConfigTableFolderSubpath(tableFolderSubpath) {
55340
- if (!/^[a-zA-Z0-9][a-zA-Z0-9-._/]{1,220}[a-zA-Z0-9]$/.test(tableFolderSubpath) || tableFolderSubpath.includes("..")) {
55341
- throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), periods (.), and forward slashes (/). The sequence '..' is also disallowed.");
55554
+ function askYesNo(question, defaultValue) {
55555
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55556
+ const answer = ask(question, defaultValue ? "y" : "n").toLowerCase();
55557
+ if (answer === "y" || answer === "yes") {
55558
+ return true;
55559
+ }
55560
+ if (answer === "n" || answer === "no") {
55561
+ return false;
55562
+ }
55563
+ printError("Please answer y or n.");
55342
55564
  }
55343
- if (tableFolderSubpath.includes("./") || tableFolderSubpath.includes("../")) {
55344
- throw new Error("Input cannot contain './' or '../'.");
55565
+ throw new Error(`No valid answer provided for: ${question}`);
55566
+ }
55567
+ function askInt(question, defaultValue) {
55568
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55569
+ const value = parseInt(ask(question, String(defaultValue)), 10);
55570
+ if (!isNaN(value)) {
55571
+ return value;
55572
+ }
55573
+ printError("Entered value must be an integer.");
55345
55574
  }
55575
+ throw new Error(`No valid answer provided for: ${question}`);
55346
55576
  }
55347
- const ICEBERG_BUCKET_NAME_HINT = "The bucket name must comply with https://cloud.google.com/storage/docs/buckets#naming. If you do not want to provide a workflow-level default bucket name, leave this input empty.\n";
55348
- const ICEBERG_TABLE_FOLDER_ROOT_HINT = "Table folder root must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores and periods. If you do not want to provide a workflow_level default table folder root, leave this input empty.\n";
55349
- const ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT = "Table folder subpath must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores, periods and forward slashes. If you do not want to provide a workflow_level default table folder subpath, leave this input empty.\n";
55350
- const ICEBERG_CONFIG_PROMPT_HINT = "Set repository-level configuration for Iceberg bucket name, table folder root, table folder subpath and connection. If you do not want to set a field, enter an empty string in response to the prompt.\n";
55351
- const ICEBERG_CONFIG_PROMPT_TEXT = "\n--- Iceberg Configuration ---\n";
55352
- const ICEBERG_CONFIG_COLLECTED_TEXT = "Iceberg configuration collected.\n";
55353
- const ICEBERG_BUCKET_NAME_PROMPT_QUESTION = "Enter the default Iceberg bucket name:";
55354
- const ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION = "Enter the default Iceberg table folder root:";
55355
- const ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION = "Enter the default Iceberg table folder subpath:";
55356
- const ICEBERG_CONNECTION_HINT = "The connection can have the form `{project}.{location}.{connection_id}` or `projects/{project}/locations/{location}/connections/{connection_id} or be set to DEFAULT. If you do not want to provide a workflow-level default connection, leave this input empty.\n";
55357
- const ICEBERG_CONNECTION_QUESTION = "Enter the default connection:";
55358
-
55359
- const noColor = process.env.NO_COLOR && ["1", "true", "yes"].includes(process.env.NO_COLOR.toLowerCase());
55360
- const ansiColorCodes = Object.freeze({
55361
- red: 91,
55362
- green: 32,
55363
- yellow: 93,
55364
- cyan: 36
55365
- });
55366
- function output(text, ansiColorCode) {
55367
- if (noColor) {
55368
- return text;
55577
+ function collectCredentials(warehouse) {
55578
+ if (warehouse === "mysql") {
55579
+ const creds = {
55580
+ host: ask(CREDS_HOST_QUESTION, "localhost"),
55581
+ port: askInt(CREDS_PORT_QUESTION, 3306),
55582
+ database: ask(CREDS_DATABASE_QUESTION, "sqlanvil"),
55583
+ user: ask(CREDS_USER_QUESTION, "root"),
55584
+ password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
55585
+ sslMode: ask(CREDS_SSLMODE_QUESTION, "disable")
55586
+ };
55587
+ return `${JSON.stringify(creds, null, 2)}\n`;
55369
55588
  }
55370
- return `\x1b[${ansiColorCode}m${text}\x1b[0m`;
55589
+ const isSupabase = warehouse === "supabase";
55590
+ if (isSupabase) {
55591
+ print(SUPABASE_POOLER_HINT);
55592
+ }
55593
+ const host = ask(CREDS_HOST_QUESTION, isSupabase ? "aws-1-<region>.pooler.supabase.com" : "localhost");
55594
+ if (/^db\..+\.supabase\.co$/.test(host)) {
55595
+ printError("Warning: that is the DIRECT Supabase host, which is IPv6-only — connections fail with " +
55596
+ "ENOTFOUND on most networks. Prefer the Session pooler host (Dashboard -> Connect).");
55597
+ }
55598
+ const creds = {
55599
+ host,
55600
+ port: askInt(CREDS_PORT_QUESTION, 5432),
55601
+ database: ask(CREDS_DATABASE_QUESTION, "postgres"),
55602
+ user: ask(CREDS_USER_QUESTION, isSupabase ? "postgres.<your-project-ref>" : "postgres"),
55603
+ password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
55604
+ sslMode: ask(CREDS_SSLMODE_QUESTION, isSupabase ? "require" : "disable"),
55605
+ defaultSchema: "public"
55606
+ };
55607
+ return `${JSON.stringify(creds, null, 2)}\n`;
55371
55608
  }
55372
- const successOutput = (text) => output(text, ansiColorCodes.green);
55373
- const warningOutput = (text) => output(text, ansiColorCodes.yellow);
55374
- const errorOutput = (text) => output(text, ansiColorCodes.red);
55375
- const calloutOutput = (text) => output(text, ansiColorCodes.cyan);
55376
- const write = (stream, text, indentCount) => stream.write(`${" ".repeat(indentCount)}${text}\n`);
55377
- const writeStdOut = (text, indentCount = 0) => write(process.stdout, text, indentCount);
55378
- const writeStdErr = (text, indentCount = 0) => write(process.stderr, text, indentCount);
55379
- const DEFAULT_PROMPT = "> ";
55380
- function question(questionText) {
55381
- return prompt(questionText);
55609
+ function printMigrationSummary(report, outDir) {
55610
+ const targets = report.files.filter(f => f.action === "target");
55611
+ const flagged = targets.filter(f => f.status === "flagged").length;
55612
+ printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
55613
+ `${report.connections.length} source connection(s) over ` +
55614
+ `${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
55615
+ `${targets.length} target file(s), ${flagged} flagged for dialect review.`);
55616
+ print(`Report: ${path__namespace.join(outDir, "migration-report.md")}`);
55382
55617
  }
55383
- function interactiveQuestion(questionText) {
55384
- if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
55385
- const testInput = getTestInput(questionText);
55386
- print(`${questionText} ${testInput}\n`);
55387
- return testInput;
55618
+ function ensureGitignoreCoversCredentials(dir) {
55619
+ const gitignorePath = path__namespace.join(dir, ".gitignore");
55620
+ const existing = fs__namespace$1.existsSync(gitignorePath) ? fs__namespace$1.readFileSync(gitignorePath, "utf8") : "";
55621
+ if (existing.includes(".df-credentials")) {
55622
+ return;
55388
55623
  }
55389
- print(`${questionText} `);
55390
- return readlineSync__namespace.question("");
55624
+ const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
55625
+ fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
55391
55626
  }
55392
- function getTestInput(questionText) {
55393
- const envVar = process.env.DATAFORM_CLI_TEST_INPUTS;
55394
- if (!envVar) {
55395
- throw new Error("Environment variable DATAFORM_CLI_TEST_INPUTS not set.");
55627
+ async function runConvertFlow() {
55628
+ let srcDir = "";
55629
+ for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
55630
+ const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
55631
+ if (fs__namespace$1.existsSync(candidate)) {
55632
+ srcDir = candidate;
55633
+ break;
55634
+ }
55635
+ printError(`${candidate} does not exist.`);
55636
+ }
55637
+ if (!srcDir) {
55638
+ throw new Error("No usable source directory provided.");
55639
+ }
55640
+ const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
55641
+ const targetWarehouse = askChoice(CONVERT_TARGET_QUESTION, ["bigquery", "supabase", "postgres"], "bigquery");
55642
+ print("\nConverting...\n");
55643
+ const report = await migrateDataform({ srcDir, outDir, targetWarehouse });
55644
+ printMigrationSummary(report, outDir);
55645
+ if (targetWarehouse === "bigquery") {
55646
+ print("\nNext steps:");
55647
+ print(` 1. sqlanvil compile ${outDir}`);
55648
+ print(" 2. Credentials: gcloud auth application-default login (ADC), or a service-account " +
55649
+ `key in a gitignored ${CREDENTIALS_FILENAME}.`);
55650
+ print(` 3. sqlanvil validate ${outDir} (dry-runs every model against BigQuery; all-PASS = swap complete)`);
55651
+ return 0;
55396
55652
  }
55397
- try {
55398
- const parsed = JSON.parse(envVar);
55399
- if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
55400
- const inputs = new Map(Object.entries(parsed).map(([key, value]) => [key, String(value)]));
55401
- const trimmedQuestion = questionText.trim();
55402
- const answer = inputs.get(trimmedQuestion);
55403
- if (answer !== undefined) {
55404
- print(`[TEST_INPUT for "${trimmedQuestion}"]: "${answer}"`);
55405
- return answer;
55406
- }
55407
- else {
55408
- throw new Error(`[MISSING TEST_INPUT for "${trimmedQuestion}"]`);
55409
- }
55653
+ if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
55654
+ const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
55655
+ ensureGitignoreCoversCredentials(outDir);
55656
+ fs__namespace$1.writeFileSync(credentialsPath, collectCredentials(targetWarehouse));
55657
+ printSuccess(`Credentials written to ${credentialsPath}`);
55658
+ }
55659
+ print("\nNext steps:");
55660
+ print(` 1. sqlanvil compile ${outDir}`);
55661
+ print(` 2. Review ${path__namespace.join(outDir, "migration-report.md")} — BigQuery source connections ` +
55662
+ `need columnTypes (the introspect commands are in the report).`);
55663
+ print(` 3. sqlanvil validate ${outDir} (PASS/FAILURE/BLOCKED per model = the migration to-do list)`);
55664
+ return 0;
55665
+ }
55666
+ async function runFreshFlow(defaultProjectDir) {
55667
+ const warehouse = askChoice(INIT_WAREHOUSE_QUESTION, ["supabase", "postgres", "bigquery", "mysql"], "supabase");
55668
+ const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
55669
+ const projectDir = actuallyResolve(ask(INIT_PROJECT_DIR_QUESTION, defaultProjectDir));
55670
+ const projectConfig = { warehouse };
55671
+ if (warehouse === "bigquery") {
55672
+ projectConfig.defaultDatabase = askRequired(INIT_BQ_PROJECT_QUESTION);
55673
+ projectConfig.defaultLocation = askRequired(INIT_BQ_LOCATION_QUESTION);
55674
+ }
55675
+ projectConfig.defaultSchema = ask(INIT_DEFAULT_SCHEMA_QUESTION, isPostgresLike ? "public" : "sqlanvil");
55676
+ const includeSample = askYesNo(INIT_INCLUDE_SAMPLE_QUESTION, true);
55677
+ const includeBigQuerySource = includeSample && isPostgresLike ? askYesNo(INIT_INCLUDE_BQ_SOURCE_QUESTION, true) : undefined;
55678
+ let credentialsJson;
55679
+ if (warehouse !== "bigquery" && askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
55680
+ credentialsJson = collectCredentials(warehouse);
55681
+ }
55682
+ print("\nWriting project files...\n");
55683
+ const result = await init(projectDir, projectConfig, {
55684
+ includeSample,
55685
+ includeBigQuerySource,
55686
+ credentialsJson
55687
+ });
55688
+ printInitResult(result);
55689
+ const steps = [];
55690
+ if (warehouse === "bigquery") {
55691
+ steps.push(`sqlanvil init-creds ${projectDir} (BigQuery credentials)`);
55692
+ }
55693
+ else if (!credentialsJson) {
55694
+ steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
55695
+ }
55696
+ if (includeSample) {
55697
+ steps.push("Point definitions/sources/app_orders.sqlx at a real table in your warehouse.");
55698
+ }
55699
+ if (isPostgresLike && includeBigQuerySource) {
55700
+ steps.push("Replace REPLACE_WITH_YOUR_GCP_PROJECT in workflow_settings.yaml (billingProject for " +
55701
+ "the BigQuery sample source).");
55702
+ }
55703
+ steps.push(`sqlanvil compile ${projectDir}`);
55704
+ steps.push(`sqlanvil validate ${projectDir} (checks every model against the warehouse without executing)`);
55705
+ print("\nNext steps:");
55706
+ steps.forEach((step, index) => print(` ${index + 1}. ${step}`));
55707
+ return 0;
55708
+ }
55709
+ async function runInteractiveInit(defaultProjectDir) {
55710
+ if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS === undefined) {
55711
+ printError("init --interactive needs an interactive terminal (stdin is not a TTY). " +
55712
+ "Use the non-interactive form instead: sqlanvil init [project-dir] --warehouse=... " +
55713
+ '(see "sqlanvil help init").');
55714
+ return 1;
55715
+ }
55716
+ const mode = askChoice(INIT_MODE_QUESTION, ["fresh", "convert"], "fresh");
55717
+ if (mode === "convert") {
55718
+ return runConvertFlow();
55719
+ }
55720
+ return runFreshFlow(defaultProjectDir);
55721
+ }
55722
+
55723
+ const CHECKER_SOURCE = `
55724
+ import json, os, sys
55725
+
55726
+ script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
55727
+ errors = []
55728
+
55729
+ import platform
55730
+ py_version = platform.python_version()
55731
+
55732
+ def load_packaging():
55733
+ try:
55734
+ from packaging.specifiers import SpecifierSet
55735
+ from packaging.requirements import Requirement
55736
+ return SpecifierSet, Requirement
55737
+ except ImportError:
55738
+ pass
55739
+ try:
55740
+ from pip._vendor.packaging.specifiers import SpecifierSet
55741
+ from pip._vendor.packaging.requirements import Requirement
55742
+ return SpecifierSet, Requirement
55743
+ except ImportError:
55744
+ return None, None
55745
+
55746
+ SpecifierSet, Requirement = load_packaging()
55747
+
55748
+ def mini_parse(v):
55749
+ import re
55750
+ return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
55751
+
55752
+ def mini_check(version, spec):
55753
+ import re
55754
+ ok = True
55755
+ for clause in spec.split(","):
55756
+ clause = clause.strip()
55757
+ m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
55758
+ if not m:
55759
+ return None
55760
+ op, want = m.group(1), m.group(2).strip()
55761
+ if want.endswith(".*"):
55762
+ prefix = mini_parse(want[:-2])
55763
+ have = mini_parse(version)[:len(prefix)]
55764
+ eq = have == prefix
55765
+ if (op in ("==", "===") and not eq) or (op == "!=" and eq):
55766
+ ok = False
55767
+ continue
55768
+ a, b = mini_parse(version), mini_parse(want)
55769
+ n = max(len(a), len(b))
55770
+ a = a + [0] * (n - len(a))
55771
+ b = b + [0] * (n - len(b))
55772
+ if op in ("==", "==="):
55773
+ res = a == b
55774
+ elif op == "!=":
55775
+ res = a != b
55776
+ elif op == "<=":
55777
+ res = a <= b
55778
+ elif op == ">=":
55779
+ res = a >= b
55780
+ elif op == "<":
55781
+ res = a < b
55782
+ elif op == ">":
55783
+ res = a > b
55784
+ else: # ~= : at least b, within b's series
55785
+ wb = mini_parse(want)
55786
+ res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
55787
+ if not res:
55788
+ ok = False
55789
+ return ok
55790
+
55791
+ def check_version(version, spec, what):
55792
+ if SpecifierSet is not None:
55793
+ try:
55794
+ if version not in SpecifierSet(spec, prereleases=True):
55795
+ errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
55796
+ return
55797
+ except Exception as e:
55798
+ errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
55799
+ return
55800
+ if mini_check(version, spec) is False:
55801
+ errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
55802
+
55803
+ if version_spec:
55804
+ check_version(py_version, version_spec, "python interpreter")
55805
+
55806
+ if requirements_path:
55807
+ from importlib import metadata
55808
+ try:
55809
+ lines = open(requirements_path).read().splitlines()
55810
+ except OSError as e:
55811
+ errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
55812
+ lines = []
55813
+ for raw in lines:
55814
+ line = raw.split("#")[0].strip()
55815
+ if not line or line.startswith("-"):
55816
+ continue
55817
+ if Requirement is not None:
55818
+ try:
55819
+ req = Requirement(line)
55820
+ except Exception:
55821
+ errors.append('requirements: cannot parse "%s"' % line)
55822
+ continue
55823
+ if req.marker is not None and not req.marker.evaluate():
55824
+ continue
55825
+ name, spec = req.name, str(req.specifier)
55826
+ else:
55827
+ import re
55828
+ if ";" in line:
55829
+ line = line.split(";")[0].strip()
55830
+ m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
55831
+ if not m:
55832
+ errors.append('requirements: cannot parse "%s"' % line)
55833
+ continue
55834
+ name, spec = m.group(1), (m.group(3) or "").strip()
55835
+ try:
55836
+ installed = metadata.version(name)
55837
+ except metadata.PackageNotFoundError:
55838
+ errors.append(
55839
+ "requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
55840
+ )
55841
+ continue
55842
+ if spec:
55843
+ check_version(installed, spec, "requirements: %s" % name)
55844
+
55845
+ try:
55846
+ with open(script_path, "rb") as f:
55847
+ compile(f.read(), script_path, "exec")
55848
+ except SyntaxError as e:
55849
+ errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
55850
+ except (OSError, ValueError) as e:
55851
+ errors.append("syntax: cannot compile %s: %s" % (script_path, e))
55852
+
55853
+ print(json.dumps({"interpreter_version": py_version, "errors": errors}))
55854
+ `;
55855
+ const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
55856
+ childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
55857
+ if (err) {
55858
+ reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
55410
55859
  }
55411
55860
  else {
55412
- throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: Expected a JSON object, but got ${typeof parsed}.`);
55861
+ resolve(stdout);
55413
55862
  }
55863
+ });
55864
+ });
55865
+ const failure = (message) => ({
55866
+ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
55867
+ error: { message }
55868
+ });
55869
+ async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
55870
+ if (script.language !== "python") {
55871
+ return [failure(`no validator for script language "${script.language}"`)];
55872
+ }
55873
+ let interpreter;
55874
+ try {
55875
+ interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
55876
+ }
55877
+ catch (e) {
55878
+ return [failure(e.message)];
55879
+ }
55880
+ let stdout;
55881
+ try {
55882
+ stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
55414
55883
  }
55415
55884
  catch (e) {
55416
- throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
55885
+ return [failure(e.message)];
55417
55886
  }
55418
- }
55419
- function selectionQuestion(questionText, options) {
55420
- return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
55421
- cancel: false
55422
- });
55423
- }
55424
- function prompt(questionText, options) {
55425
- writeStdOut(questionText);
55426
- return readlineSync__namespace.prompt(Object.assign(Object.assign({}, options), { prompt: (options && options.prompt && options.prompt + DEFAULT_PROMPT) || DEFAULT_PROMPT }));
55427
- }
55428
- function print(text) {
55429
- writeStdOut(text);
55430
- }
55431
- function printSuccess(text) {
55432
- writeStdOut(successOutput(text));
55433
- }
55434
- function printError(errorText, indentCount = 0) {
55435
- writeStdErr(errorOutput(errorText), indentCount);
55436
- }
55437
- function printInitResult(result) {
55438
- if (result.dirsCreated && result.dirsCreated.length) {
55439
- writeStdOut(successOutput("Directories successfully created:"));
55440
- result.dirsCreated.forEach(dir => writeStdOut(dir, 1));
55887
+ let parsed;
55888
+ try {
55889
+ parsed = JSON.parse(stdout);
55441
55890
  }
55442
- if (result.filesWritten && result.filesWritten.length) {
55443
- writeStdOut(successOutput("Files successfully written:"));
55444
- result.filesWritten.forEach(file => writeStdOut(file, 1));
55891
+ catch (e) {
55892
+ return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
55445
55893
  }
55894
+ if (parsed.errors && parsed.errors.length > 0) {
55895
+ return parsed.errors.map(failure);
55896
+ }
55897
+ return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
55446
55898
  }
55447
- function printInitCredsResult(writtenFilePath) {
55448
- writeStdOut(successOutput("Credentials file successfully written:"));
55449
- writeStdOut(writtenFilePath, 1);
55450
- writeStdOut("To change connection settings, edit this file directly.");
55451
- }
55452
- function isInteractive({ stream = process.stdout } = {}) {
55453
- return Boolean(stream && stream.isTTY &&
55454
- process.env.TERM !== 'dumb' &&
55455
- !('CI' in process.env));
55899
+
55900
+ function targetKey(target) {
55901
+ return [target.database, target.schema, target.name].filter(Boolean).join(".");
55456
55902
  }
55457
- var compiledGraphOutputType;
55458
- (function (compiledGraphOutputType) {
55459
- compiledGraphOutputType["Json"] = "json";
55460
- compiledGraphOutputType["Dot"] = "dot";
55461
- compiledGraphOutputType["Summary"] = "summary";
55462
- })(compiledGraphOutputType || (compiledGraphOutputType = {}));
55463
- function printCompiledGraph(graph, outputType, quietCompilation) {
55464
- const interactive = isInteractive();
55465
- if (outputType === compiledGraphOutputType.Json) {
55466
- writeStdOut(prettyJsonStringify(graph));
55467
- }
55468
- else if (outputType === compiledGraphOutputType.Dot) {
55469
- writeStdOut(dotRepresentation(graph, interactive));
55903
+ function topoOrder(nodes) {
55904
+ const byKey = new Map(nodes.map(n => [n.key, n]));
55905
+ const indegree = new Map();
55906
+ const dependents = new Map();
55907
+ for (const n of nodes) {
55908
+ const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
55909
+ indegree.set(n.key, inGraphDeps.length);
55910
+ for (const dep of inGraphDeps) {
55911
+ dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
55912
+ }
55470
55913
  }
55471
- else {
55472
- const actionCount = 0 +
55473
- (graph.tables ? graph.tables.length : 0) +
55474
- (graph.assertions ? graph.assertions.length : 0) +
55475
- (graph.operations ? graph.operations.length : 0);
55476
- writeStdOut(successOutput(`Compiled ${actionCount} action(s).`));
55477
- if (graph.tables && graph.tables.length) {
55478
- graph.tables.forEach(setOrValidateTableEnumType);
55479
- writeStdOut(`${graph.tables.length} dataset(s)${quietCompilation ? "." : ":"}`);
55480
- if (!quietCompilation) {
55481
- graph.tables.forEach(compiledTable => {
55482
- writeStdOut(`${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`, 1);
55483
- });
55484
- }
55914
+ const ready = nodes
55915
+ .filter(n => (indegree.get(n.key) || 0) === 0)
55916
+ .map(n => n.key)
55917
+ .sort();
55918
+ const ordered = [];
55919
+ const emitted = new Set();
55920
+ while (ready.length) {
55921
+ const key = ready.shift();
55922
+ if (emitted.has(key)) {
55923
+ continue;
55485
55924
  }
55486
- if (graph.assertions && graph.assertions.length) {
55487
- writeStdOut(`${graph.assertions.length} assertion(s)${quietCompilation ? "." : ":"}`);
55488
- if (!quietCompilation) {
55489
- graph.assertions.forEach(assertion => {
55490
- writeStdOut(assertionString(assertion.target, assertion.disabled), 1);
55491
- });
55925
+ emitted.add(key);
55926
+ ordered.push(byKey.get(key));
55927
+ for (const dep of (dependents.get(key) || []).slice().sort()) {
55928
+ indegree.set(dep, (indegree.get(dep) || 0) - 1);
55929
+ if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
55930
+ ready.push(dep);
55931
+ ready.sort();
55492
55932
  }
55493
55933
  }
55494
- if (graph.operations && graph.operations.length) {
55495
- writeStdOut(`${graph.operations.length} operation(s)${quietCompilation ? "." : ":"}`);
55496
- if (!quietCompilation) {
55497
- graph.operations.forEach(operation => {
55498
- writeStdOut(operationString(operation.target, operation.disabled), 1);
55499
- });
55934
+ }
55935
+ if (ordered.length < nodes.length) {
55936
+ for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
55937
+ if (!emitted.has(n.key)) {
55938
+ emitted.add(n.key);
55939
+ ordered.push(n);
55500
55940
  }
55501
55941
  }
55502
55942
  }
55943
+ return ordered;
55503
55944
  }
55504
- function formatStackTraceForQuietCompilation(compileError) {
55505
- var _a, _b;
55506
- if ((_a = compileError === null || compileError === void 0 ? void 0 : compileError.message) === null || _a === void 0 ? void 0 : _a.includes("Unexpected identifier")) {
55507
- if (!compileError.stack) {
55508
- return "";
55945
+ function dependencyBlocked(dependencyKeys, statusByKey) {
55946
+ return dependencyKeys.some(dep => {
55947
+ const status = statusByKey.get(dep);
55948
+ return status !== undefined && status !== "PASS";
55949
+ });
55950
+ }
55951
+ const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
55952
+ function validateShadowSuffix(nowMs) {
55953
+ return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
55954
+ }
55955
+ function parseShadowTimestamp(schemaName) {
55956
+ const match = schemaName.match(/sqlanvil_validate_(\d+)/);
55957
+ return match ? Number(match[1]) : null;
55958
+ }
55959
+ function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
55960
+ return schemaNames.filter(name => {
55961
+ const ts = parseShadowTimestamp(name);
55962
+ return ts !== null && nowMs - ts > maxAgeMs;
55963
+ });
55964
+ }
55965
+
55966
+ const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
55967
+ async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
55968
+ try {
55969
+ const schemas = await deps.listSchemas();
55970
+ for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
55971
+ try {
55972
+ await deps.execute(deps.dropSchemaCascadeSql(schema));
55973
+ }
55974
+ catch (e) {
55975
+ }
55509
55976
  }
55510
- const stackLines = ((_b = compileError.stack) === null || _b === void 0 ? void 0 : _b.split("\n")) || [];
55511
- return stackLines.slice(0, Math.min(3, stackLines.length)).join("\n");
55512
55977
  }
55513
- return "";
55514
- }
55515
- function printCompiledGraphErrors(graphErrors, quietCompilation) {
55516
- if (graphErrors.compilationErrors && graphErrors.compilationErrors.length > 0) {
55517
- printError("Compilation errors:", 1);
55518
- graphErrors.compilationErrors.forEach(compileError => {
55519
- writeStdErr(`${calloutOutput(compileError.fileName)}: ${errorOutput(quietCompilation ? (compileError.message + " " + formatStackTraceForQuietCompilation(compileError) || compileError.stack) : (compileError.stack || compileError.message))}`, 1);
55520
- });
55978
+ catch (e) {
55521
55979
  }
55522
55980
  }
55523
- function printTestResult(testResult) {
55524
- writeStdOut(`${testResult.name}: ${testResult.successful ? successOutput("passed") : errorOutput("failed")}`);
55525
- if (!testResult.successful) {
55526
- testResult.messages.forEach(message => writeStdErr(message, 1));
55981
+ function tableType(enumType) {
55982
+ switch (enumType) {
55983
+ case sqlanvil.TableType.VIEW:
55984
+ return "view";
55985
+ case sqlanvil.TableType.INCREMENTAL:
55986
+ return "incremental";
55987
+ default:
55988
+ return "table";
55527
55989
  }
55528
55990
  }
55529
- function printExecutionGraph(executionGraph, asJson) {
55530
- if (asJson) {
55531
- writeStdOut(prettyJsonStringify(executionGraph.toJSON()));
55991
+ function depKeys(deps) {
55992
+ return (deps || []).map(targetKey);
55993
+ }
55994
+ async function validate(compiledGraph, deps, options = {}) {
55995
+ const nodes = [];
55996
+ for (const table of compiledGraph.tables || []) {
55997
+ nodes.push({
55998
+ key: targetKey(table.target),
55999
+ dependencyKeys: depKeys(table.dependencyTargets),
56000
+ kind: "table",
56001
+ type: tableType(table.enumType),
56002
+ target: table.target,
56003
+ table,
56004
+ action: table
56005
+ });
55532
56006
  }
55533
- else {
55534
- const actionsByType = {
55535
- table: [],
55536
- assertion: [],
55537
- operation: []
55538
- };
55539
- executionGraph.actions.forEach(action => {
55540
- if (!(action.type === "table" || action.type === "assertion" || action.type === "operation")) {
55541
- throw new Error(`Unrecognized action type: ${action.type}`);
55542
- }
55543
- actionsByType[action.type].push(action);
56007
+ for (const assertion of compiledGraph.assertions || []) {
56008
+ nodes.push({
56009
+ key: targetKey(assertion.target),
56010
+ dependencyKeys: depKeys(assertion.dependencyTargets),
56011
+ kind: "assertion",
56012
+ type: "assertion",
56013
+ target: assertion.target,
56014
+ action: assertion
55544
56015
  });
55545
- const datasetActions = actionsByType.table;
55546
- if (datasetActions && datasetActions.length) {
55547
- writeStdOut(`${datasetActions.length} dataset(s):`);
55548
- datasetActions.forEach(datasetAction => writeStdOut(datasetString(datasetAction.target, datasetAction.type, datasetAction.tasks.length === 0), 1));
56016
+ }
56017
+ for (const operation of compiledGraph.operations || []) {
56018
+ if (!operation.target) {
56019
+ continue;
55549
56020
  }
55550
- const assertionActions = actionsByType.assertion;
55551
- if (assertionActions && assertionActions.length) {
55552
- writeStdOut(`${assertionActions.length} assertion(s):`);
55553
- assertionActions.forEach(assertionAction => writeStdOut(assertionString(assertionAction.target, assertionAction.tasks.length === 0), 1));
56021
+ nodes.push({
56022
+ key: targetKey(operation.target),
56023
+ dependencyKeys: depKeys(operation.dependencyTargets),
56024
+ kind: "operation",
56025
+ type: "operation",
56026
+ target: operation.target,
56027
+ action: undefined
56028
+ });
56029
+ }
56030
+ for (const imp of compiledGraph.imports || []) {
56031
+ if (!imp.target) {
56032
+ continue;
55554
56033
  }
55555
- const operationActions = actionsByType.operation;
55556
- if (operationActions && operationActions.length) {
55557
- writeStdOut(`${operationActions.length} operation(s):`);
55558
- operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
56034
+ nodes.push({
56035
+ key: targetKey(imp.target),
56036
+ dependencyKeys: depKeys(imp.dependencyTargets),
56037
+ kind: "import",
56038
+ type: "import",
56039
+ target: imp.target,
56040
+ action: undefined
56041
+ });
56042
+ }
56043
+ for (const script of compiledGraph.scripts || []) {
56044
+ if (!script.target) {
56045
+ continue;
55559
56046
  }
56047
+ nodes.push({
56048
+ key: targetKey(script.target),
56049
+ dependencyKeys: depKeys(script.dependencyTargets),
56050
+ kind: "script",
56051
+ type: "script",
56052
+ target: script.target,
56053
+ script,
56054
+ action: undefined
56055
+ });
55560
56056
  }
55561
- }
55562
- function printExecutedAction(executedAction, executionAction, dryRun) {
55563
- const jobIds = executedAction.tasks
55564
- .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55565
- .map(task => task.metadata.bigquery.jobId);
55566
- const bytesBilled = executedAction.tasks
55567
- .filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
55568
- .map(task => {
55569
- var _a, _b, _c, _d;
55570
- const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
55571
- return formatBytesInHumanReadableFormat(bytes);
55572
- });
55573
- const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
55574
- switch (executedAction.status) {
55575
- case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
55576
- switch (executionAction.type) {
55577
- case "table": {
55578
- writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55579
- return;
55580
- }
55581
- case "assertion": {
55582
- writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55583
- return;
55584
- }
55585
- case "operation": {
55586
- writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55587
- return;
55588
- }
55589
- }
56057
+ const ordered = topoOrder(nodes);
56058
+ const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
56059
+ const statusByKey = new Map();
56060
+ const results = [];
56061
+ try {
56062
+ for (const schema of shadowSchemas) {
56063
+ await deps.execute(deps.createSchemaSql(schema));
55590
56064
  }
55591
- case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
55592
- switch (executionAction.type) {
55593
- case "table": {
55594
- writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
55595
- break;
55596
- }
55597
- case "assertion": {
55598
- writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55599
- break;
55600
- }
55601
- case "operation": {
55602
- writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
55603
- break;
55604
- }
56065
+ for (const node of ordered) {
56066
+ if (node.kind === "operation" ||
56067
+ node.kind === "import" ||
56068
+ (node.kind === "script" && !deps.checkScript)) {
56069
+ statusByKey.set(node.key, "SKIPPED");
56070
+ results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
56071
+ continue;
55605
56072
  }
55606
- printExecutedActionErrors(executedAction, executionAction);
55607
- return;
55608
- }
55609
- case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
55610
- switch (executionAction.type) {
55611
- case "table": {
55612
- writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55613
- return;
55614
- }
55615
- case "assertion": {
55616
- writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55617
- return;
56073
+ if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
56074
+ statusByKey.set(node.key, "BLOCKED");
56075
+ results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
56076
+ continue;
56077
+ }
56078
+ const evaluations = node.kind === "script"
56079
+ ? await deps.checkScript(node.script)
56080
+ : await deps.evaluate(node.action);
56081
+ const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
56082
+ const status = failed ? "FAILURE" : "PASS";
56083
+ statusByKey.set(node.key, status);
56084
+ results.push({ target: node.target, type: node.type, status, errors: evaluations });
56085
+ if (status === "PASS" && node.kind === "table") {
56086
+ try {
56087
+ await deps.execute(deps.validationStubSql(node.table));
55618
56088
  }
55619
- case "operation": {
55620
- writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55621
- return;
56089
+ catch (e) {
55622
56090
  }
55623
56091
  }
55624
- return;
55625
56092
  }
55626
- case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
55627
- switch (executionAction.type) {
55628
- case "table": {
55629
- writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
55630
- return;
55631
- }
55632
- case "assertion": {
55633
- writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
55634
- return;
56093
+ }
56094
+ finally {
56095
+ if (!options.keepShadow) {
56096
+ for (const schema of shadowSchemas.slice().reverse()) {
56097
+ try {
56098
+ await deps.execute(deps.dropSchemaCascadeSql(schema));
55635
56099
  }
55636
- case "operation": {
55637
- writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
55638
- return;
56100
+ catch (e) {
55639
56101
  }
55640
56102
  }
55641
- return;
55642
56103
  }
55643
56104
  }
56105
+ return results;
55644
56106
  }
55645
- function printFormatFilesResult(formatResults) {
55646
- const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
55647
- const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
55648
- const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
55649
- const failedFormatResults = sorted.filter(result => !!result.err);
55650
- if (successfulFormatResults.length > 0) {
55651
- printSuccess("Successfully formatted:");
55652
- successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
55653
- }
55654
- if (needsFormattingResults.length > 0) {
55655
- printError("Files that need formatting:");
55656
- needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
56107
+
56108
+ class SupabaseDbAdapter extends PostgresDbAdapter {
56109
+ static async create(credentials, options) {
56110
+ const sslMode = (credentials.sslMode || "").toLowerCase();
56111
+ const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
56112
+ const clientConfig = {
56113
+ host: credentials.host,
56114
+ port: credentials.port,
56115
+ database: credentials.database,
56116
+ user: credentials.user,
56117
+ password: credentials.password,
56118
+ ssl: sslEnabled
56119
+ ? {
56120
+ rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
56121
+ }
56122
+ : false
56123
+ };
56124
+ const queryExecutor = new PgPoolExecutor(clientConfig, options);
56125
+ try {
56126
+ await queryExecutor.verifyConnection();
56127
+ }
56128
+ catch (e) {
56129
+ await queryExecutor.close().catch(() => undefined);
56130
+ throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
56131
+ `as "${credentials.user}": ${e.message}`, e);
56132
+ }
56133
+ return new SupabaseDbAdapter(queryExecutor);
55657
56134
  }
55658
- if (failedFormatResults.length > 0) {
55659
- printError("Errors encountered during formatting:");
55660
- failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
56135
+ constructor(queryExecutor) {
56136
+ super(queryExecutor);
55661
56137
  }
55662
- }
55663
- function datasetString(target, datasetType, disabled) {
55664
- return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
55665
- }
55666
- function assertionString(target, disabled) {
55667
- return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55668
- }
55669
- function operationString(target, disabled) {
55670
- return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
55671
- }
55672
- function plainTargetString(target) {
55673
- return `${target.schema}.${target.name}`;
55674
- }
55675
- function targetString(target) {
55676
- return calloutOutput(`${target.schema}.${target.name}`);
55677
- }
55678
- function dotRepresentation(graph, interactive) {
55679
- var _a, _b, _c;
55680
- const nodes = [];
55681
- const edges = [];
55682
- const formatTarget = interactive ? targetString : plainTargetString;
55683
- (_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
55684
- var _a;
55685
- const nodeName = `${formatTarget(table.target)}`;
55686
- nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
55687
- (_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55688
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55689
- });
55690
- });
55691
- (_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
55692
- var _a;
55693
- const nodeName = `${formatTarget(assertion.target)}`;
55694
- nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
55695
- (_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55696
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55697
- });
55698
- });
55699
- (_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
56138
+ async verifyServiceRolePermissions() {
55700
56139
  var _a;
55701
- const nodeName = `${formatTarget(operation.target)}`;
55702
- nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
55703
- (_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
55704
- edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
55705
- });
55706
- });
55707
- return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
55708
- }
55709
- function printExecutedActionErrors(executedAction, executionAction) {
55710
- const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
55711
- failingTasks.forEach((task, i) => {
55712
- executionAction.tasks[i].statement.split("\n").forEach(line => {
55713
- writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
55714
- });
55715
- printError(task.errorMessage, 1);
55716
- });
56140
+ try {
56141
+ const result = await this.execute("select current_user");
56142
+ const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
56143
+ return !!currentUser;
56144
+ }
56145
+ catch (_b) {
56146
+ return false;
56147
+ }
56148
+ }
56149
+ async validatePublicationState(pubName = "supabase_realtime") {
56150
+ try {
56151
+ const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
56152
+ params: [pubName]
56153
+ });
56154
+ return result.rows.length > 0;
56155
+ }
56156
+ catch (_a) {
56157
+ return false;
56158
+ }
56159
+ }
55717
56160
  }
55718
56161
 
55719
56162
  function getBigQueryCredentials() {
@@ -56587,6 +57030,26 @@ const icebergOption = option("iceberg", {
56587
57030
  type: "boolean",
56588
57031
  default: false
56589
57032
  });
57033
+ const bareOption = option("bare", {
57034
+ describe: "Skip the sample project files — scaffold bare (gitkept) directories only.",
57035
+ type: "boolean",
57036
+ default: false
57037
+ });
57038
+ const interactiveOption = option("interactive", {
57039
+ describe: "Guided Q&A setup: start a fresh project (warehouse, sample project, credentials) or " +
57040
+ "convert an existing Dataform project. Other init arguments are ignored except " +
57041
+ "[project-dir], which seeds the directory prompt.",
57042
+ type: "boolean",
57043
+ default: false
57044
+ });
57045
+ const targetWarehouseOption = option("target-warehouse", {
57046
+ describe: "Where the CONVERTED project runs: supabase/postgres = move off BigQuery (sources become " +
57047
+ "named connections, SQL gets the dialect pass); bigquery = keep the SAME warehouse — a " +
57048
+ "tooling swap with SQL and bigquery:{} blocks untouched.",
57049
+ type: "string",
57050
+ choices: ["supabase", "postgres", "bigquery"],
57051
+ default: "supabase"
57052
+ });
56590
57053
  const warehouseOption = option("warehouse", {
56591
57054
  describe: "Target warehouse for the new project.",
56592
57055
  type: "string",
@@ -56836,14 +57299,17 @@ function runCli() {
56836
57299
  {
56837
57300
  format: `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` +
56838
57301
  ` [${ProjectConfigOptions.defaultLocation.name}]`,
56839
- description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB).",
57302
+ description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB). " +
57303
+ "Use --interactive for a guided setup, including converting a Dataform project.",
56840
57304
  positionalOptions: [
56841
57305
  projectDirOption,
56842
57306
  positionalOption(ProjectConfigOptions.defaultDatabase.name, {
56843
57307
  describe: "The default database to use, equivalent to Google Cloud Project ID."
56844
57308
  }, (argv) => {
56845
57309
  const warehouse = argv[warehouseOption.name] || "bigquery";
56846
- if (warehouse === "bigquery" && !argv[ProjectConfigOptions.defaultDatabase.name]) {
57310
+ if (warehouse === "bigquery" &&
57311
+ !argv[interactiveOption.name] &&
57312
+ !argv[ProjectConfigOptions.defaultDatabase.name]) {
56847
57313
  throw new Error(`The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` +
56848
57314
  `required for BigQuery projects. Use "sqlanvil help init" for more info.`);
56849
57315
  }
@@ -56853,15 +57319,20 @@ function runCli() {
56853
57319
  "https://cloud.google.com/bigquery/docs/locations for supported values."
56854
57320
  }, (argv) => {
56855
57321
  const warehouse = argv[warehouseOption.name] || "bigquery";
56856
- if (warehouse === "bigquery" && !argv[ProjectConfigOptions.defaultLocation.name]) {
57322
+ if (warehouse === "bigquery" &&
57323
+ !argv[interactiveOption.name] &&
57324
+ !argv[ProjectConfigOptions.defaultLocation.name]) {
56857
57325
  throw new Error(`The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` +
56858
57326
  `required for BigQuery projects. Use "sqlanvil help init" for more info.`);
56859
57327
  }
56860
57328
  })
56861
57329
  ],
56862
- options: [warehouseOption, icebergOption],
57330
+ options: [warehouseOption, icebergOption, bareOption, interactiveOption],
56863
57331
  processFn: async (argv) => {
56864
57332
  const projectDir = argv[projectDirOption.name];
57333
+ if (argv[interactiveOption.name]) {
57334
+ return runInteractiveInit(projectDir);
57335
+ }
56865
57336
  const warehouse = argv[warehouseOption.name] || "bigquery";
56866
57337
  const projectConfig = { warehouse };
56867
57338
  if (warehouse === "bigquery") {
@@ -56875,7 +57346,9 @@ function runCli() {
56875
57346
  }
56876
57347
  }
56877
57348
  print("Writing project files...\n");
56878
- const initResult = await init(projectDir, projectConfig);
57349
+ const initResult = await init(projectDir, projectConfig, {
57350
+ includeSample: !argv[bareOption.name]
57351
+ });
56879
57352
  printInitResult(initResult);
56880
57353
  return 0;
56881
57354
  }
@@ -57443,10 +57916,12 @@ function runCli() {
57443
57916
  },
57444
57917
  {
57445
57918
  format: "migrate-dataform <source-dir> <out-dir>",
57446
- description: "Convert a Dataform/BigQuery project into a sqlanvil-on-Postgres project. The source " +
57447
- "directory is READ-ONLY; the converted project + migration-report.{md,json} land in " +
57448
- "<out-dir> (must be empty). Sources stay in BigQuery as named connections; target SQL " +
57449
- "gets safe rewrites + inline SQLANVIL-MIGRATE markers for dialect review.",
57919
+ description: "Convert a Dataform/BigQuery project to sqlanvil. The source directory is READ-ONLY; " +
57920
+ "the converted project + migration-report.{md,json} land in <out-dir> (must be empty). " +
57921
+ "Default target (supabase/postgres) MOVES the warehouse: sources stay in BigQuery as " +
57922
+ "named connections and target SQL gets safe rewrites + inline SQLANVIL-MIGRATE markers. " +
57923
+ "--target-warehouse bigquery is a TOOLING SWAP: the project keeps running on the same " +
57924
+ "BigQuery warehouse, SQL and bigquery:{} blocks untouched.",
57450
57925
  positionalOptions: [
57451
57926
  positionalOption("source-dir", {
57452
57927
  describe: "The Dataform project to convert (never modified)."
@@ -57455,19 +57930,14 @@ function runCli() {
57455
57930
  describe: "Where the converted sqlanvil project is written (created; must be empty)."
57456
57931
  })
57457
57932
  ],
57458
- options: [],
57933
+ options: [targetWarehouseOption],
57459
57934
  processFn: async (argv) => {
57460
57935
  const report = await migrateDataform({
57461
57936
  srcDir: argv["source-dir"],
57462
- outDir: argv["out-dir"]
57937
+ outDir: argv["out-dir"],
57938
+ targetWarehouse: argv[targetWarehouseOption.name]
57463
57939
  });
57464
- const targets = report.files.filter(f => f.action === "target");
57465
- const flagged = targets.filter(f => f.status === "flagged").length;
57466
- printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
57467
- `${report.connections.length} source connection(s) over ` +
57468
- `${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
57469
- `${targets.length} target file(s), ${flagged} flagged for dialect review.`);
57470
- print(`Report: ${path__namespace.join(argv["out-dir"], "migration-report.md")}`);
57940
+ printMigrationSummary(report, argv["out-dir"]);
57471
57941
  print(`Next: sqlanvil compile ${argv["out-dir"]}`);
57472
57942
  return 0;
57473
57943
  }