@sqlanvil/cli 1.25.2 → 1.26.1

package/bundle.js

@@ -50646,7 +50646,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
         .filter(validationQuery => !!validationQuery.query);
 }
 
-const version = "1.25.2";
+const version = "1.26.1";
 const dataformVersion = "3.0.61";
 
 async function build(compiledGraph, runConfig, dbadapter) {
@@ -54722,7 +54722,7 @@ function readSourceConfig(srcDir) {
     };
 }
 async function migrateDataform(opts) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
     const srcDir = path__namespace.resolve(opts.srcDir);
     const outDir = path__namespace.resolve(opts.outDir);
     if (containsPath(srcDir, outDir)) {
@@ -54770,12 +54770,15 @@ async function migrateDataform(opts) {
     const declarationSchemas = new Set();
     const targetSchemas = new Set();
     const connectionMap = new Map();
+    const introspectCommands = [];
     for (const f of isBigQueryTarget ? [] : sqlxFiles) {
         if (f.parsed.type === "declaration") {
             const dataset = (_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset;
             const project = resolveDatabase(f.parsed.database);
             declarationSchemas.add(dataset);
             const name = connectionNameFor(project);
+            const tableName = (_e = f.parsed.name) !== null && _e !== void 0 ? _e : path__namespace.basename(f.rel, ".sqlx");
+            introspectCommands.push(`sqlanvil introspect ${name} ${JSON.stringify(`${dataset}.${tableName}`)} --output ${JSON.stringify(f.rel)}`);
             const existing = connectionMap.get(name);
             if (existing) {
                 existing.declarationCount++;
@@ -54793,7 +54796,7 @@ async function migrateDataform(opts) {
             }
         }
         else if (f.parsed.type) {
-            targetSchemas.add((_e = f.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset);
+            targetSchemas.add((_f = f.parsed.schema) !== null && _f !== void 0 ? _f : sourceConfig.defaultDataset);
         }
     }
     report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
@@ -54811,14 +54814,14 @@ async function migrateDataform(opts) {
                 report.files.push({
                     file: rel,
                     action: info.parsed.type === "declaration" ? "declaration" : "target",
-                    type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
+                    type: (_g = info.parsed.type) !== null && _g !== void 0 ? _g : "(none)",
                     status: findings.length === 0 ? "clean" : "rewritten",
                     findings,
                 });
                 continue;
             }
             if (info.parsed.type === "declaration") {
-                const dataset = (_g = info.parsed.schema) !== null && _g !== void 0 ? _g : sourceConfig.defaultDataset;
+                const dataset = (_h = info.parsed.schema) !== null && _h !== void 0 ? _h : sourceConfig.defaultDataset;
                 const project = resolveDatabase(info.parsed.database);
                 const connection = connectionNameFor(project);
                 const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
@@ -54838,7 +54841,7 @@ async function migrateDataform(opts) {
                 report.files.push({
                     file: rel,
                     action: "target",
-                    type: (_h = info.parsed.type) !== null && _h !== void 0 ? _h : "(none)",
+                    type: (_j = info.parsed.type) !== null && _j !== void 0 ? _j : "(none)",
                     status: findings.length === 0
                         ? "clean"
                         : findings.some(f => f.kind === "flag")
@@ -54875,14 +54878,17 @@ async function migrateDataform(opts) {
         report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
     }
     const settings = isBigQueryTarget
-        ? Object.assign(Object.assign(Object.assign({}, (sourceConfig.defaultProject ? { defaultProject: sourceConfig.defaultProject } : {})), (sourceConfig.defaultLocation ? { defaultLocation: sourceConfig.defaultLocation } : {})), { defaultDataset: sourceConfig.defaultDataset, defaultAssertionDataset: (_j = sourceConfig.defaultAssertionDataset) !== null && _j !== void 0 ? _j : "sqlanvil_assertions", sqlanvilCoreVersion: (_k = opts.coreVersion) !== null && _k !== void 0 ? _k : version }) : {
+        ? Object.assign(Object.assign(Object.assign({}, (sourceConfig.defaultProject ? { defaultProject: sourceConfig.defaultProject } : {})), (sourceConfig.defaultLocation ? { defaultLocation: sourceConfig.defaultLocation } : {})), { defaultDataset: sourceConfig.defaultDataset, defaultAssertionDataset: (_k = sourceConfig.defaultAssertionDataset) !== null && _k !== void 0 ? _k : "sqlanvil_assertions", sqlanvilCoreVersion: (_l = opts.coreVersion) !== null && _l !== void 0 ? _l : version }) : {
         warehouse: targetWarehouse,
         defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
-        defaultAssertionDataset: ((_l = sourceConfig.defaultAssertionDataset) !== null && _l !== void 0 ? _l : "sqlanvil_assertions").toLowerCase(),
-        sqlanvilCoreVersion: (_m = opts.coreVersion) !== null && _m !== void 0 ? _m : version,
+        defaultAssertionDataset: ((_m = sourceConfig.defaultAssertionDataset) !== null && _m !== void 0 ? _m : "sqlanvil_assertions").toLowerCase(),
+        sqlanvilCoreVersion: (_o = opts.coreVersion) !== null && _o !== void 0 ? _o : version,
     };
     if (sourceConfig.vars)
         settings.vars = sourceConfig.vars;
+    if (isBigQueryTarget) {
+        settings.environments = { test: { schemaSuffix: "test" } };
+    }
     if (report.connections.length > 0) {
         settings.connections = Object.fromEntries(report.connections.map(c => {
             var _a;
@@ -54898,11 +54904,53 @@ async function migrateDataform(opts) {
         }));
     }
     writer.write("workflow_settings.yaml", jsYaml.dump(settings));
+    if (!isBigQueryTarget && introspectCommands.length > 0) {
+        writer.write("scripts/introspect_all.sh", `#!/bin/sh\n` +
+            `# Generated by sqlanvil migrate-dataform: scaffold columnTypes for every converted\n` +
+            `# declaration (${introspectCommands.length} commands). Requires BigQuery read access\n` +
+            `# (gcloud auth application-default login). Re-runnable; then commit the changes.\n` +
+            `set -e\n` +
+            introspectCommands.join("\n") +
+            `\n`);
+        report.files.push({
+            file: "scripts/introspect_all.sh",
+            action: "generated",
+            status: "clean",
+            findings: []
+        });
+    }
+    if (isBigQueryTarget && sourceConfig.defaultProject) {
+        const adcCredentials = { projectId: sourceConfig.defaultProject };
+        if (sourceConfig.defaultLocation) {
+            adcCredentials.location = sourceConfig.defaultLocation;
+        }
+        writer.write(".df-credentials.json", `${JSON.stringify(adcCredentials, null, 2)}\n`);
+        report.files.push({
+            file: ".df-credentials.json",
+            action: "generated",
+            status: "clean",
+            findings: []
+        });
+        const gitignorePath = path__namespace.join(outDir, ".gitignore");
+        const existingGitignore = fs__namespace$1.existsSync(gitignorePath)
+            ? fs__namespace$1.readFileSync(gitignorePath, "utf8")
+            : "";
+        if (!existingGitignore.includes(".df-credentials")) {
+            const separator = existingGitignore === "" || existingGitignore.endsWith("\n") ? "" : "\n";
+            fs__namespace$1.writeFileSync(gitignorePath, `${existingGitignore}${separator}.df-credentials*.json\n`);
+            report.files.push({
+                file: ".gitignore",
+                action: existingGitignore === "" ? "generated" : "updated",
+                status: "clean",
+                findings: []
+            });
+        }
+    }
     if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "AGENTS.md"))) {
         writer.write("AGENTS.md", agentsMdContents({
             warehouse: targetWarehouse,
             defaultDataset: settings.defaultDataset,
-            version: (_o = opts.coreVersion) !== null && _o !== void 0 ? _o : version,
+            version: (_p = opts.coreVersion) !== null && _p !== void 0 ? _p : version,
             converted: true
         }));
         report.files.push({ file: "AGENTS.md", action: "generated", status: "clean", findings: [] });
@@ -54953,7 +55001,13 @@ function renderReportMd(r) {
         lines.push("");
         lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
             `(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
-            `before it can extract; scaffold them with \`sqlanvil introspect\`:`);
+            `before it can extract. **Every concrete command is in the generated ` +
+            `\`scripts/introspect_all.sh\`** — run it on a machine with BigQuery read access ` +
+            `(\`gcloud auth application-default login\`), then commit what it writes:`);
+        lines.push("");
+        lines.push("```sh");
+        lines.push("sh scripts/introspect_all.sh && git add -A && git commit -m 'columnTypes' && git push");
+        lines.push("```");
         lines.push("");
         for (const c of r.connections) {
             lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
@@ -55008,16 +55062,23 @@ function renderReportMd(r) {
     lines.push("");
     if (r.targetWarehouse === "bigquery") {
         lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
-        lines.push(`2. Credentials: \`gcloud auth application-default login\` (ADC), or a service-account ` +
-            `key in a gitignored \`.df-credentials.json\`.`);
-        lines.push(`3. \`sqlanvil validate\` — dry-runs every model against BigQuery without executing; ` +
-            `all-PASS means the swap is complete.`);
-        lines.push(`4. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
+        lines.push(`2. Authenticate the same way Dataform runs locally: \`gcloud auth application-default ` +
+            `login\` (ADC). A secretless \`.df-credentials.json\` (project + location only) was ` +
+            `generated for you — nothing else is needed. Service-account keys and keyless ` +
+            `\`accessToken\` auth are supported for CI (see the BigQuery guide).`);
+        lines.push(`3. \`sqlanvil validate\` — read-only: dry-runs every model against BigQuery without ` +
+            `executing; all-PASS means the swap is complete.`);
+        lines.push(`4. First real run in the scaffolded TEST environment — \`sqlanvil run . --environment ` +
+            `test\` writes to \`<dataset>_test\` datasets, leaving production untouched. To land ` +
+            `tests in a separate GCP project instead, set \`environments.test.defaultDatabase\` in ` +
+            `workflow_settings.yaml. Once verified: \`sqlanvil run .\`.`);
+        lines.push(`5. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
             `sample of actions — they should match apart from the compile-global rename.`);
     }
     else {
         lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
-        lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
+        lines.push(`2. Scaffold \`columnTypes\`: \`sh scripts/introspect_all.sh\` (every declaration's ` +
+            `concrete command, generated), then commit what it writes.`);
         lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
             `model is the migration to-do list. Work the flagged files until validate is green.`);
         lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
@@ -55564,6 +55625,14 @@ const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
 const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
 const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
 const CONVERT_TARGET_QUESTION = "Target warehouse? (bigquery = keep running on BigQuery, tooling swap only; supabase/postgres = move off BigQuery)";
+const BQ_AUTH_QUESTION = "How should local runs authenticate to BigQuery? (adc = gcloud Application Default " +
+    "Credentials, like Dataform local runs; key = service-account key file; later)";
+const BQ_AUTH_LOGIN_QUESTION = "No Application Default Credentials found. Run `gcloud auth application-default login` now?";
+const BQ_KEY_PATH_QUESTION = "Path to the service-account key JSON file?";
+const BQ_TEST_TARGET_QUESTION = "Where should test runs land before you touch production? (suffix = same project, " +
+    "`<dataset>_test` datasets; project = a separate GCP project; none = no test environment)";
+const BQ_TEST_SUFFIX_QUESTION = "Dataset suffix for the test environment?";
+const BQ_TEST_PROJECT_QUESTION = "GCP project ID for the test environment?";
 const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
     "aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
     "db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
@@ -55668,7 +55737,104 @@ function ensureGitignoreCoversCredentials(dir) {
     const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
     fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
 }
+function detectAdc() {
+    if (process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
+        return false;
+    }
+    try {
+        const result = childProcess.spawnSync("gcloud", ["auth", "application-default", "print-access-token"], {
+            stdio: "ignore",
+            timeout: 15000
+        });
+        return result.status === 0;
+    }
+    catch (e) {
+        return false;
+    }
+}
+function runBigQueryAuthStep(projectDir) {
+    const mode = askChoice(BQ_AUTH_QUESTION, ["adc", "key", "later"], "adc");
+    if (mode === "adc") {
+        if (detectAdc()) {
+            printSuccess("Application Default Credentials detected — runs authenticate as your gcloud account.");
+            return;
+        }
+        if (askYesNo(BQ_AUTH_LOGIN_QUESTION, true)) {
+            childProcess.spawnSync("gcloud", ["auth", "application-default", "login"], { stdio: "inherit" });
+            if (detectAdc()) {
+                printSuccess("ADC configured — runs authenticate as your gcloud account.");
+                return;
+            }
+        }
+        print("Run `gcloud auth application-default login` before `sqlanvil validate` / `run` — no " +
+            "other credential setup is needed.");
+        return;
+    }
+    if (mode === "key") {
+        for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+            const keyPath = actuallyResolve(askRequired(BQ_KEY_PATH_QUESTION));
+            if (!fs__namespace$1.existsSync(keyPath)) {
+                printError(`${keyPath} does not exist.`);
+                continue;
+            }
+            try {
+                const keyContents = fs__namespace$1.readFileSync(keyPath, "utf8");
+                const parsed = JSON.parse(keyContents);
+                if (!parsed.client_email) {
+                    printError(`${keyPath} does not look like a service-account key (no client_email).`);
+                    continue;
+                }
+                const credentialsPath = path__namespace.join(projectDir, CREDENTIALS_FILENAME);
+                const existing = fs__namespace$1.existsSync(credentialsPath)
+                    ? JSON.parse(fs__namespace$1.readFileSync(credentialsPath, "utf8"))
+                    : {};
+                ensureGitignoreCoversCredentials(projectDir);
+                fs__namespace$1.writeFileSync(credentialsPath, `${JSON.stringify(Object.assign(Object.assign({}, existing), { credentials: keyContents }), null, 2)}\n`);
+                printSuccess(`Runs authenticate as ${parsed.client_email} (key embedded in the gitignored ` +
+                    `${CREDENTIALS_FILENAME}).`);
+                return;
+            }
+            catch (e) {
+                printError(`Could not read ${keyPath}: ${e instanceof Error ? e.message : e}`);
+            }
+        }
+        return;
+    }
+    print("Skipped. Before `sqlanvil validate` / `run`: `gcloud auth application-default login`, or " +
+        `add a service-account key as the \`credentials\` field of ${CREDENTIALS_FILENAME}.`);
+}
+function runBigQueryTestTargetStep(projectDir) {
+    var _a;
+    const choice = askChoice(BQ_TEST_TARGET_QUESTION, ["suffix", "project", "none"], "suffix");
+    const settingsPath = path__namespace.join(projectDir, "workflow_settings.yaml");
+    const settings = (jsYaml.load(fs__namespace$1.readFileSync(settingsPath, "utf8")) || {});
+    if (choice === "none") {
+        if ((_a = settings.environments) === null || _a === void 0 ? void 0 : _a.test) {
+            delete settings.environments.test;
+            if (Object.keys(settings.environments).length === 0) {
+                delete settings.environments;
+            }
+            fs__namespace$1.writeFileSync(settingsPath, jsYaml.dump(settings));
+        }
+        print("No test environment — runs go straight to the configured datasets.");
+        return;
+    }
+    const testEnv = {};
+    if (choice === "suffix") {
+        testEnv.schemaSuffix = ask(BQ_TEST_SUFFIX_QUESTION, "test");
+    }
+    else {
+        testEnv.defaultDatabase = askRequired(BQ_TEST_PROJECT_QUESTION);
+    }
+    settings.environments = Object.assign(Object.assign({}, (settings.environments || {})), { test: testEnv });
+    fs__namespace$1.writeFileSync(settingsPath, jsYaml.dump(settings));
+    printSuccess(choice === "suffix"
+        ? `Test runs write to \`<dataset>_${testEnv.schemaSuffix}\` datasets ` +
+            `(sqlanvil run . --environment test).`
+        : `Test runs land in project ${testEnv.defaultDatabase} (sqlanvil run . --environment test).`);
+}
 async function runConvertFlow() {
+    var _a;
     let srcDir = "";
     for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
         const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
@@ -55687,11 +55853,23 @@ async function runConvertFlow() {
     const report = await migrateDataform({ srcDir, outDir, targetWarehouse });
     printMigrationSummary(report, outDir);
     if (targetWarehouse === "bigquery") {
+        print("");
+        runBigQueryAuthStep(outDir);
+        print("");
+        runBigQueryTestTargetStep(outDir);
+        const hasTestEnv = fs__namespace$1.existsSync(path__namespace.join(outDir, "workflow_settings.yaml"))
+            ? !!((_a = (jsYaml.load(fs__namespace$1.readFileSync(path__namespace.join(outDir, "workflow_settings.yaml"), "utf8")) || {}).environments) === null || _a === void 0 ? void 0 : _a.test)
+            : false;
         print("\nNext steps:");
         print(`  1. sqlanvil compile ${outDir}`);
-        print("  2. Credentials: gcloud auth application-default login (ADC), or a service-account " +
-            `key in a gitignored ${CREDENTIALS_FILENAME}.`);
-        print(`  3. sqlanvil validate ${outDir}  (dry-runs every model against BigQuery; all-PASS = swap complete)`);
+        print(`  2. sqlanvil validate ${outDir}  (read-only dry-run of every model; all-PASS = swap complete)`);
+        if (hasTestEnv) {
+            print(`  3. sqlanvil run ${outDir} --environment test  (first real run — production untouched)`);
+            print(`  4. sqlanvil run ${outDir}  (once the test run is verified)`);
+        }
+        else {
+            print(`  3. sqlanvil run ${outDir}`);
+        }
         return 0;
     }
     if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
@@ -55730,10 +55908,17 @@ async function runFreshFlow(defaultProjectDir) {
         credentialsJson
     });
     printInitResult(result);
-    const steps = [];
     if (warehouse === "bigquery") {
-        steps.push(`sqlanvil init-creds ${projectDir}  (BigQuery credentials)`);
+        const credentialsPath = path__namespace.join(projectDir, CREDENTIALS_FILENAME);
+        if (!fs__namespace$1.existsSync(credentialsPath)) {
+            ensureGitignoreCoversCredentials(projectDir);
+            fs__namespace$1.writeFileSync(credentialsPath, `${JSON.stringify(Object.assign({ projectId: projectConfig.defaultDatabase }, (projectConfig.defaultLocation ? { location: projectConfig.defaultLocation } : {})), null, 2)}\n`);
+        }
+        print("");
+        runBigQueryAuthStep(projectDir);
     }
+    const steps = [];
+    if (warehouse === "bigquery") ;
     else if (!credentialsJson) {
         steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
     }

package/package.json

@@ -71,7 +71,7 @@
     "bin": {
         "sqlanvil": "bundle.js"
     },
-    "version": "1.25.2",
+    "version": "1.26.1",
     "name": "@sqlanvil/cli",
     "description": "sqlanvil command line interface.",
     "main": "bundle.js"