@sqlanvil/cli 1.25.1 → 1.26.0

package/bundle.js

@@ -50646,7 +50646,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
         .filter(validationQuery => !!validationQuery.query);
 }
 
-const version = "1.25.1";
+const version = "1.26.0";
 const dataformVersion = "3.0.61";
 
 async function build(compiledGraph, runConfig, dbadapter) {
@@ -54883,6 +54883,9 @@ async function migrateDataform(opts) {
     };
     if (sourceConfig.vars)
         settings.vars = sourceConfig.vars;
+    if (isBigQueryTarget) {
+        settings.environments = { test: { schemaSuffix: "test" } };
+    }
     if (report.connections.length > 0) {
         settings.connections = Object.fromEntries(report.connections.map(c => {
             var _a;
@@ -54898,6 +54901,33 @@ async function migrateDataform(opts) {
         }));
     }
     writer.write("workflow_settings.yaml", jsYaml.dump(settings));
+    if (isBigQueryTarget && sourceConfig.defaultProject) {
+        const adcCredentials = { projectId: sourceConfig.defaultProject };
+        if (sourceConfig.defaultLocation) {
+            adcCredentials.location = sourceConfig.defaultLocation;
+        }
+        writer.write(".df-credentials.json", `${JSON.stringify(adcCredentials, null, 2)}\n`);
+        report.files.push({
+            file: ".df-credentials.json",
+            action: "generated",
+            status: "clean",
+            findings: []
+        });
+        const gitignorePath = path__namespace.join(outDir, ".gitignore");
+        const existingGitignore = fs__namespace$1.existsSync(gitignorePath)
+            ? fs__namespace$1.readFileSync(gitignorePath, "utf8")
+            : "";
+        if (!existingGitignore.includes(".df-credentials")) {
+            const separator = existingGitignore === "" || existingGitignore.endsWith("\n") ? "" : "\n";
+            fs__namespace$1.writeFileSync(gitignorePath, `${existingGitignore}${separator}.df-credentials*.json\n`);
+            report.files.push({
+                file: ".gitignore",
+                action: existingGitignore === "" ? "generated" : "updated",
+                status: "clean",
+                findings: []
+            });
+        }
+    }
     if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "AGENTS.md"))) {
         writer.write("AGENTS.md", agentsMdContents({
             warehouse: targetWarehouse,
@@ -55008,11 +55038,17 @@ function renderReportMd(r) {
     lines.push("");
     if (r.targetWarehouse === "bigquery") {
         lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
-        lines.push(`2. Credentials: \`gcloud auth application-default login\` (ADC), or a service-account ` +
-            `key in a gitignored \`.df-credentials.json\`.`);
-        lines.push(`3. \`sqlanvil validate\` — dry-runs every model against BigQuery without executing; ` +
-            `all-PASS means the swap is complete.`);
-        lines.push(`4. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
+        lines.push(`2. Authenticate the same way Dataform runs locally: \`gcloud auth application-default ` +
+            `login\` (ADC). A secretless \`.df-credentials.json\` (project + location only) was ` +
+            `generated for you — nothing else is needed. Service-account keys and keyless ` +
+            `\`accessToken\` auth are supported for CI (see the BigQuery guide).`);
+        lines.push(`3. \`sqlanvil validate\` — read-only: dry-runs every model against BigQuery without ` +
+            `executing; all-PASS means the swap is complete.`);
+        lines.push(`4. First real run in the scaffolded TEST environment — \`sqlanvil run . --environment ` +
+            `test\` writes to \`<dataset>_test\` datasets, leaving production untouched. To land ` +
+            `tests in a separate GCP project instead, set \`environments.test.defaultDatabase\` in ` +
+            `workflow_settings.yaml. Once verified: \`sqlanvil run .\`.`);
+        lines.push(`5. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
             `sample of actions — they should match apart from the compile-global rename.`);
     }
     else {
@@ -55564,6 +55600,14 @@ const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
 const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
 const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
 const CONVERT_TARGET_QUESTION = "Target warehouse? (bigquery = keep running on BigQuery, tooling swap only; supabase/postgres = move off BigQuery)";
+const BQ_AUTH_QUESTION = "How should local runs authenticate to BigQuery? (adc = gcloud Application Default " +
+    "Credentials, like Dataform local runs; key = service-account key file; later)";
+const BQ_AUTH_LOGIN_QUESTION = "No Application Default Credentials found. Run `gcloud auth application-default login` now?";
+const BQ_KEY_PATH_QUESTION = "Path to the service-account key JSON file?";
+const BQ_TEST_TARGET_QUESTION = "Where should test runs land before you touch production? (suffix = same project, " +
+    "`<dataset>_test` datasets; project = a separate GCP project; none = no test environment)";
+const BQ_TEST_SUFFIX_QUESTION = "Dataset suffix for the test environment?";
+const BQ_TEST_PROJECT_QUESTION = "GCP project ID for the test environment?";
 const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
     "aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
     "db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
@@ -55668,7 +55712,104 @@ function ensureGitignoreCoversCredentials(dir) {
     const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
     fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
 }
+function detectAdc() {
+    if (process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
+        return false;
+    }
+    try {
+        const result = childProcess.spawnSync("gcloud", ["auth", "application-default", "print-access-token"], {
+            stdio: "ignore",
+            timeout: 15000
+        });
+        return result.status === 0;
+    }
+    catch (e) {
+        return false;
+    }
+}
+function runBigQueryAuthStep(projectDir) {
+    const mode = askChoice(BQ_AUTH_QUESTION, ["adc", "key", "later"], "adc");
+    if (mode === "adc") {
+        if (detectAdc()) {
+            printSuccess("Application Default Credentials detected — runs authenticate as your gcloud account.");
+            return;
+        }
+        if (askYesNo(BQ_AUTH_LOGIN_QUESTION, true)) {
+            childProcess.spawnSync("gcloud", ["auth", "application-default", "login"], { stdio: "inherit" });
+            if (detectAdc()) {
+                printSuccess("ADC configured — runs authenticate as your gcloud account.");
+                return;
+            }
+        }
+        print("Run `gcloud auth application-default login` before `sqlanvil validate` / `run` — no " +
+            "other credential setup is needed.");
+        return;
+    }
+    if (mode === "key") {
+        for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+            const keyPath = actuallyResolve(askRequired(BQ_KEY_PATH_QUESTION));
+            if (!fs__namespace$1.existsSync(keyPath)) {
+                printError(`${keyPath} does not exist.`);
+                continue;
+            }
+            try {
+                const keyContents = fs__namespace$1.readFileSync(keyPath, "utf8");
+                const parsed = JSON.parse(keyContents);
+                if (!parsed.client_email) {
+                    printError(`${keyPath} does not look like a service-account key (no client_email).`);
+                    continue;
+                }
+                const credentialsPath = path__namespace.join(projectDir, CREDENTIALS_FILENAME);
+                const existing = fs__namespace$1.existsSync(credentialsPath)
+                    ? JSON.parse(fs__namespace$1.readFileSync(credentialsPath, "utf8"))
+                    : {};
+                ensureGitignoreCoversCredentials(projectDir);
+                fs__namespace$1.writeFileSync(credentialsPath, `${JSON.stringify(Object.assign(Object.assign({}, existing), { credentials: keyContents }), null, 2)}\n`);
+                printSuccess(`Runs authenticate as ${parsed.client_email} (key embedded in the gitignored ` +
+                    `${CREDENTIALS_FILENAME}).`);
+                return;
+            }
+            catch (e) {
+                printError(`Could not read ${keyPath}: ${e instanceof Error ? e.message : e}`);
+            }
+        }
+        return;
+    }
+    print("Skipped. Before `sqlanvil validate` / `run`: `gcloud auth application-default login`, or " +
+        `add a service-account key as the \`credentials\` field of ${CREDENTIALS_FILENAME}.`);
+}
+function runBigQueryTestTargetStep(projectDir) {
+    var _a;
+    const choice = askChoice(BQ_TEST_TARGET_QUESTION, ["suffix", "project", "none"], "suffix");
+    const settingsPath = path__namespace.join(projectDir, "workflow_settings.yaml");
+    const settings = (jsYaml.load(fs__namespace$1.readFileSync(settingsPath, "utf8")) || {});
+    if (choice === "none") {
+        if ((_a = settings.environments) === null || _a === void 0 ? void 0 : _a.test) {
+            delete settings.environments.test;
+            if (Object.keys(settings.environments).length === 0) {
+                delete settings.environments;
+            }
+            fs__namespace$1.writeFileSync(settingsPath, jsYaml.dump(settings));
+        }
+        print("No test environment — runs go straight to the configured datasets.");
+        return;
+    }
+    const testEnv = {};
+    if (choice === "suffix") {
+        testEnv.schemaSuffix = ask(BQ_TEST_SUFFIX_QUESTION, "test");
+    }
+    else {
+        testEnv.defaultDatabase = askRequired(BQ_TEST_PROJECT_QUESTION);
+    }
+    settings.environments = Object.assign(Object.assign({}, (settings.environments || {})), { test: testEnv });
+    fs__namespace$1.writeFileSync(settingsPath, jsYaml.dump(settings));
+    printSuccess(choice === "suffix"
+        ? `Test runs write to \`<dataset>_${testEnv.schemaSuffix}\` datasets ` +
+            `(sqlanvil run . --environment test).`
+        : `Test runs land in project ${testEnv.defaultDatabase} (sqlanvil run . --environment test).`);
+}
 async function runConvertFlow() {
+    var _a;
     let srcDir = "";
     for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
         const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
@@ -55687,11 +55828,23 @@ async function runConvertFlow() {
     const report = await migrateDataform({ srcDir, outDir, targetWarehouse });
     printMigrationSummary(report, outDir);
     if (targetWarehouse === "bigquery") {
+        print("");
+        runBigQueryAuthStep(outDir);
+        print("");
+        runBigQueryTestTargetStep(outDir);
+        const hasTestEnv = fs__namespace$1.existsSync(path__namespace.join(outDir, "workflow_settings.yaml"))
+            ? !!((_a = (jsYaml.load(fs__namespace$1.readFileSync(path__namespace.join(outDir, "workflow_settings.yaml"), "utf8")) || {}).environments) === null || _a === void 0 ? void 0 : _a.test)
+            : false;
         print("\nNext steps:");
         print(`  1. sqlanvil compile ${outDir}`);
-        print("  2. Credentials: gcloud auth application-default login (ADC), or a service-account " +
-            `key in a gitignored ${CREDENTIALS_FILENAME}.`);
-        print(`  3. sqlanvil validate ${outDir}  (dry-runs every model against BigQuery; all-PASS = swap complete)`);
+        print(`  2. sqlanvil validate ${outDir}  (read-only dry-run of every model; all-PASS = swap complete)`);
+        if (hasTestEnv) {
+            print(`  3. sqlanvil run ${outDir} --environment test  (first real run — production untouched)`);
+            print(`  4. sqlanvil run ${outDir}  (once the test run is verified)`);
+        }
+        else {
+            print(`  3. sqlanvil run ${outDir}`);
+        }
         return 0;
     }
     if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
@@ -55730,10 +55883,17 @@ async function runFreshFlow(defaultProjectDir) {
         credentialsJson
     });
     printInitResult(result);
-    const steps = [];
     if (warehouse === "bigquery") {
-        steps.push(`sqlanvil init-creds ${projectDir}  (BigQuery credentials)`);
+        const credentialsPath = path__namespace.join(projectDir, CREDENTIALS_FILENAME);
+        if (!fs__namespace$1.existsSync(credentialsPath)) {
+            ensureGitignoreCoversCredentials(projectDir);
+            fs__namespace$1.writeFileSync(credentialsPath, `${JSON.stringify(Object.assign({ projectId: projectConfig.defaultDatabase }, (projectConfig.defaultLocation ? { location: projectConfig.defaultLocation } : {})), null, 2)}\n`);
+        }
+        print("");
+        runBigQueryAuthStep(projectDir);
     }
+    const steps = [];
+    if (warehouse === "bigquery") ;
     else if (!credentialsJson) {
         steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
     }
@@ -55992,6 +56152,28 @@ function dependencyBlocked(dependencyKeys, statusByKey) {
         return status !== undefined && status !== "PASS";
     });
 }
+function rewriteSelfReferences(graph, shadowSuffix, resolveTarget) {
+    var _a;
+    const suffix = `_${shadowSuffix}`;
+    for (const action of [...(graph.tables || []), ...(graph.assertions || [])]) {
+        const target = action.target;
+        if (!((_a = target === null || target === void 0 ? void 0 : target.schema) === null || _a === void 0 ? void 0 : _a.endsWith(suffix))) {
+            continue;
+        }
+        const prodTarget = sqlanvil.Target.create(Object.assign(Object.assign({}, target), { schema: target.schema.slice(0, -suffix.length) }));
+        const shadowName = resolveTarget(target);
+        const prodName = resolveTarget(prodTarget);
+        const rewrite = (sql) => sql.split(shadowName).join(prodName);
+        if (action.query) {
+            action.query = rewrite(action.query);
+        }
+        const table = action;
+        if (table.incrementalQuery) {
+            table.incrementalQuery = rewrite(table.incrementalQuery);
+        }
+    }
+    return graph;
+}
 const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
 function validateShadowSuffix(nowMs) {
     return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
@@ -56035,6 +56217,22 @@ function tableType(enumType) {
 function depKeys(deps) {
     return (deps || []).map(targetKey);
 }
+function stripOps(table) {
+    const bare = sqlanvil.Table.create(table);
+    bare.preOps = [];
+    bare.postOps = [];
+    bare.incrementalPreOps = [];
+    bare.incrementalPostOps = [];
+    return bare;
+}
+function collectOps(table) {
+    return [
+        ...(table.preOps || []),
+        ...(table.postOps || []),
+        ...(table.incrementalPreOps || []),
+        ...(table.incrementalPostOps || [])
+    ].filter(op => !!op && !!op.trim());
+}
 async function validate(compiledGraph, deps, options = {}) {
     const nodes = [];
     for (const table of compiledGraph.tables || []) {
@@ -56119,20 +56317,33 @@ async function validate(compiledGraph, deps, options = {}) {
                 results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
                 continue;
             }
+            const evaluationAction = node.kind === "table" ? stripOps(node.table) : node.action;
             const evaluations = node.kind === "script"
                 ? await deps.checkScript(node.script)
-                : await deps.evaluate(node.action);
+                : await deps.evaluate(evaluationAction);
             const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
-            const status = failed ? "FAILURE" : "PASS";
-            statusByKey.set(node.key, status);
-            results.push({ target: node.target, type: node.type, status, errors: evaluations });
+            let status = failed ? "FAILURE" : "PASS";
+            let stubCreated = false;
             if (status === "PASS" && node.kind === "table") {
                 try {
                     await deps.execute(deps.validationStubSql(node.table));
+                    stubCreated = true;
                 }
                 catch (e) {
                 }
             }
+            if (status === "PASS" && node.kind === "table" && stubCreated && deps.evaluateOp) {
+                for (const op of collectOps(node.table)) {
+                    const opEvaluations = await deps.evaluateOp(op);
+                    evaluations.push(...opEvaluations);
+                    if (opEvaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE)) {
+                        status = "FAILURE";
+                        break;
+                    }
+                }
+            }
+            statusByKey.set(node.key, status);
+            results.push({ target: node.target, type: node.type, status, errors: evaluations });
         }
     }
     finally {
@@ -57203,6 +57414,7 @@ async function runValidate(argv) {
         tags: argv[tagsOption.name]
     });
     const executionSql = new ExecutionSql(compiledGraph.projectConfig, dataformVersion);
+    rewriteSelfReferences(prunedGraph, shadowSuffix, target => executionSql.resolveTarget(target));
     const shadowSchemas = Array.from(new Set((prunedGraph.tables || []).map(table => table.target.schema)));
     process.on("SIGINT", () => {
         Promise.all(shadowSchemas.map(schema => dbadapter.execute(executionSql.dropSchemaCascadeSql(schema)).catch(() => undefined))).then(() => process.exit(1));
@@ -57211,6 +57423,7 @@ async function runValidate(argv) {
         evaluate: action => dbadapter.evaluate(action.enumType !== undefined
             ? sqlanvil.Table.create(action)
             : sqlanvil.Assertion.create(action)),
+        evaluateOp: warehouse === "bigquery" ? (sql) => dbadapter.evaluate(sql) : undefined,
         execute: sql => dbadapter.execute(sql).then(() => undefined),
         validationStubSql: table => executionSql.validationStubSql(table),
         createSchemaSql: schema => executionSql.createSchemaSql(schema),

package/package.json

@@ -71,7 +71,7 @@
     "bin": {
         "sqlanvil": "bundle.js"
     },
-    "version": "1.25.1",
+    "version": "1.26.0",
     "name": "@sqlanvil/cli",
     "description": "sqlanvil command line interface.",
     "main": "bundle.js"