@sqlanvil/cli 1.8.3 → 1.9.0

package/bundle.js

@@ -42996,6 +42996,24 @@ from (${query}) as insertions`;
             `) AS\n${exp.query}`;
         return [sqlanvil.ExecutionTask.create({ type: "statement", statement })];
     }
+    validationStubSql(table) {
+        const target = this.resolveTarget(table.target);
+        if (table.enumType === sqlanvil.TableType.VIEW) {
+            return `create view ${target} as ${table.query}`;
+        }
+        return `create table ${target} as select * from (${table.query}) limit 0`;
+    }
+    createSchemaSql(schema) {
+        return `create schema if not exists ${this.qualifiedSchema(schema)}`;
+    }
+    dropSchemaCascadeSql(schema) {
+        return `drop schema if exists ${this.qualifiedSchema(schema)} cascade`;
+    }
+    qualifiedSchema(schema) {
+        return this.project.defaultDatabase
+            ? `\`${this.project.defaultDatabase}.${schema}\``
+            : `\`${schema}\``;
+    }
     buildIncrementalSchemaChangeTasks(tasks, table) {
         const uniqueId = this.uniqueIdGenerator();
         const emptyTempTableTarget = Object.assign(Object.assign({}, table.target), { name: `${table.target.name}_sa_temp_${uniqueId}_empty` });
@@ -43206,6 +43224,19 @@ class MysqlExecutionSql {
     createExportTasks(exp) {
         throw new Error("type: \"export\" is not supported on MySQL/MariaDB yet.");
     }
+    validationStubSql(table) {
+        const target = this.resolveTarget(table.target);
+        if (table.enumType === sqlanvil.TableType.VIEW) {
+            return `create or replace view ${target} as ${table.query}`;
+        }
+        return `create table ${target} as select * from (${table.query}) as _sa_stub limit 0`;
+    }
+    createSchemaSql(schema) {
+        return `create database if not exists \`${schema}\``;
+    }
+    dropSchemaCascadeSql(schema) {
+        return `drop database if exists \`${schema}\``;
+    }
     publishTasks(table, runConfig, tableMetadata) {
         const tasks = new Tasks();
         const target = this.resolveTarget(table.target);
@@ -43370,6 +43401,19 @@ class PostgresExecutionSql {
         const kind = type === sqlanvil.TableMetadata.Type.VIEW ? "view" : "table";
         return `drop ${kind} if exists ${this.resolveTarget(target)} cascade`;
     }
+    validationStubSql(table) {
+        const target = this.resolveTarget(table.target);
+        if (table.enumType === sqlanvil.TableType.VIEW) {
+            return `create view ${target} as ${table.query}`;
+        }
+        return `create table ${target} as ${table.query} with no data`;
+    }
+    createSchemaSql(schema) {
+        return `create schema if not exists "${schema}"`;
+    }
+    dropSchemaCascadeSql(schema) {
+        return `drop schema if exists "${schema}" cascade`;
+    }
     createExportTasks(exp) {
         return [sqlanvil.ExecutionTask.create({ type: "export", statement: exp.query })];
     }
@@ -43682,6 +43726,15 @@ class ExecutionSql {
     dropIfExists(target, type) {
         return this.delegate.dropIfExists(target, type);
     }
+    validationStubSql(table) {
+        return this.delegate.validationStubSql(table);
+    }
+    createSchemaSql(schema) {
+        return this.delegate.createSchemaSql(schema);
+    }
+    dropSchemaCascadeSql(schema) {
+        return this.delegate.dropSchemaCascadeSql(schema);
+    }
 }
 function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q) => q) {
     const validationQueries = new Array();
@@ -43739,7 +43792,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
         .filter(validationQuery => !!validationQuery.query);
 }
 
-const version = "1.8.3";
+const version = "1.9.0";
 const dataformVersion = "3.0.60";
 
 async function build(compiledGraph, runConfig, dbadapter) {
@@ -45151,6 +45204,183 @@ function resolveCredentials(envCredentials, cliCredentials, defaultFilename) {
     return envCredentials || defaultFilename;
 }
 
+function targetKey(target) {
+    return [target.database, target.schema, target.name].filter(Boolean).join(".");
+}
+function topoOrder(nodes) {
+    const byKey = new Map(nodes.map(n => [n.key, n]));
+    const indegree = new Map();
+    const dependents = new Map();
+    for (const n of nodes) {
+        const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
+        indegree.set(n.key, inGraphDeps.length);
+        for (const dep of inGraphDeps) {
+            dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
+        }
+    }
+    const ready = nodes
+        .filter(n => (indegree.get(n.key) || 0) === 0)
+        .map(n => n.key)
+        .sort();
+    const ordered = [];
+    const emitted = new Set();
+    while (ready.length) {
+        const key = ready.shift();
+        if (emitted.has(key)) {
+            continue;
+        }
+        emitted.add(key);
+        ordered.push(byKey.get(key));
+        for (const dep of (dependents.get(key) || []).slice().sort()) {
+            indegree.set(dep, (indegree.get(dep) || 0) - 1);
+            if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
+                ready.push(dep);
+                ready.sort();
+            }
+        }
+    }
+    if (ordered.length < nodes.length) {
+        for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
+            if (!emitted.has(n.key)) {
+                emitted.add(n.key);
+                ordered.push(n);
+            }
+        }
+    }
+    return ordered;
+}
+function dependencyBlocked(dependencyKeys, statusByKey) {
+    return dependencyKeys.some(dep => {
+        const status = statusByKey.get(dep);
+        return status !== undefined && status !== "PASS";
+    });
+}
+const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
+function validateShadowSuffix(nowMs) {
+    return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
+}
+function parseShadowTimestamp(schemaName) {
+    const match = schemaName.match(/sqlanvil_validate_(\d+)/);
+    return match ? Number(match[1]) : null;
+}
+function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
+    return schemaNames.filter(name => {
+        const ts = parseShadowTimestamp(name);
+        return ts !== null && nowMs - ts > maxAgeMs;
+    });
+}
+
+const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
+async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
+    try {
+        const schemas = await deps.listSchemas();
+        for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
+            try {
+                await deps.execute(deps.dropSchemaCascadeSql(schema));
+            }
+            catch (e) {
+            }
+        }
+    }
+    catch (e) {
+    }
+}
+function tableType(enumType) {
+    switch (enumType) {
+        case sqlanvil.TableType.VIEW:
+            return "view";
+        case sqlanvil.TableType.INCREMENTAL:
+            return "incremental";
+        default:
+            return "table";
+    }
+}
+function depKeys(deps) {
+    return (deps || []).map(targetKey);
+}
+async function validate(compiledGraph, deps, options = {}) {
+    const nodes = [];
+    for (const table of compiledGraph.tables || []) {
+        nodes.push({
+            key: targetKey(table.target),
+            dependencyKeys: depKeys(table.dependencyTargets),
+            kind: "table",
+            type: tableType(table.enumType),
+            target: table.target,
+            table,
+            action: table
+        });
+    }
+    for (const assertion of compiledGraph.assertions || []) {
+        nodes.push({
+            key: targetKey(assertion.target),
+            dependencyKeys: depKeys(assertion.dependencyTargets),
+            kind: "assertion",
+            type: "assertion",
+            target: assertion.target,
+            action: assertion
+        });
+    }
+    for (const operation of compiledGraph.operations || []) {
+        if (!operation.target) {
+            continue;
+        }
+        nodes.push({
+            key: targetKey(operation.target),
+            dependencyKeys: depKeys(operation.dependencyTargets),
+            kind: "operation",
+            type: "operation",
+            target: operation.target,
+            action: undefined
+        });
+    }
+    const ordered = topoOrder(nodes);
+    const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
+    const statusByKey = new Map();
+    const results = [];
+    try {
+        for (const schema of shadowSchemas) {
+            await deps.execute(deps.createSchemaSql(schema));
+        }
+        for (const node of ordered) {
+            if (node.kind === "operation") {
+                statusByKey.set(node.key, "SKIPPED");
+                results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
+                continue;
+            }
+            if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
+                statusByKey.set(node.key, "BLOCKED");
+                results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
+                continue;
+            }
+            const evaluations = await deps.evaluate(node.action);
+            const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
+            const status = failed ? "FAILURE" : "PASS";
+            statusByKey.set(node.key, status);
+            results.push({ target: node.target, type: node.type, status, errors: evaluations });
+            if (status === "PASS" && node.kind === "table") {
+                try {
+                    await deps.execute(deps.validationStubSql(node.table));
+                }
+                catch (e) {
+                }
+            }
+        }
+    }
+    finally {
+        if (!options.keepShadow) {
+            for (const schema of shadowSchemas.slice().reverse()) {
+                try {
+                    await deps.execute(deps.dropSchemaCascadeSql(schema));
+                }
+                catch (e) {
+                }
+            }
+        }
+    }
+    return results;
+}
+
 function parsePostgresEvalError(_query, error) {
     return sqlanvil.QueryEvaluationError.create({
         message: (error === null || error === void 0 ? void 0 : error.message) ? String(error.message) : String(error)
@@ -47734,6 +47964,11 @@ const timeoutOption = option("timeout", {
     default: null,
     coerce: (rawTimeoutString) => rawTimeoutString ? parseDuration__default["default"](rawTimeoutString) : null
 });
+const keepShadowOption = option("keep-shadow", {
+    describe: "If set, `validate` leaves its temporary shadow schema(s) in place instead of dropping them " +
+        "(debugging aid).",
+    type: "boolean"
+});
 const jobPrefixOption = option("job-prefix", {
     describe: "Adds an additional prefix in the form of `sqlanvil-${jobPrefix}-`.",
     type: "string",
@@ -47795,6 +48030,108 @@ function credentialsPathWithEnvironment(projectDir, argv) {
     const chosen = resolveCredentials(envCredentials, argv[credentialsOption.name], CREDENTIALS_FILENAME);
     return getCredentialsPath(projectDir, chosen);
 }
+function printValidationResults(results, json) {
+    const failures = results.filter(r => r.status === "FAILURE");
+    const blocked = results.filter(r => r.status === "BLOCKED");
+    const passed = results.filter(r => r.status === "PASS");
+    const skipped = results.filter(r => r.status === "SKIPPED");
+    if (json) {
+        print(prettyJsonStringify(results));
+    }
+    else {
+        for (const result of results) {
+            const label = targetAsReadableString(result.target);
+            if (result.status === "PASS") {
+                printSuccess(`  PASS   ${label}`);
+            }
+            else if (result.status === "SKIPPED") {
+                print(`  SKIP   ${label} (${result.type} — not validated)`);
+            }
+            else if (result.status === "BLOCKED") {
+                printError(`  BLOCK  ${label} — blocked by an upstream failure`);
+            }
+            else {
+                printError(`  FAIL   ${label}`);
+                result.errors
+                    .filter(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE)
+                    .forEach(e => {
+                    var _a;
+                    const loc = ((_a = e.error) === null || _a === void 0 ? void 0 : _a.errorLocation) ? ` (line ${e.error.errorLocation.line}, col ${e.error.errorLocation.column})`
+                        : "";
+                    printError(`           ${(e.error && e.error.message) || "validation failed"}${loc}`);
+                });
+            }
+        }
+        print(`\n${passed.length} passed, ${failures.length} failed, ${blocked.length} blocked` +
+            (skipped.length ? `, ${skipped.length} skipped` : ""));
+    }
+    return failures.length > 0 || blocked.length > 0 ? 1 : 0;
+}
+async function runValidate(argv) {
+    const projectDir = argv[projectDirOption.name];
+    if (!argv[jsonOutputOption.name]) {
+        print("Compiling...\n");
+    }
+    const baseOverride = projectConfigOverrideWithEnvironment(projectDir, argv);
+    const shadowSuffix = validateShadowSuffix(Date.now());
+    const compiledGraph = await compile({
+        projectDir,
+        projectConfigOverride: Object.assign(Object.assign({}, baseOverride), { schemaSuffix: [baseOverride.schemaSuffix, shadowSuffix].filter(Boolean).join("_") }),
+        timeoutMillis: argv[timeoutOption.name] || undefined
+    });
+    if (compiledGraphHasErrors(compiledGraph)) {
+        printCompiledGraphErrors(compiledGraph.graphErrors, argv[quietCompileOption.name]);
+        return 1;
+    }
+    if (!argv[jsonOutputOption.name]) {
+        printSuccess("Compiled successfully.\n");
+    }
+    const warehouse = (compiledGraph.projectConfig.warehouse || "bigquery").toLowerCase();
+    const readCredentials = read(credentialsPathWithEnvironment(projectDir, argv), warehouse);
+    let dbadapter;
+    if (warehouse === "supabase") {
+        dbadapter = await SupabaseDbAdapter.create(readCredentials);
+    }
+    else if (warehouse === "mysql") {
+        dbadapter = await MySqlDbAdapter.create(readCredentials);
+    }
+    else if (warehouse === "bigquery") {
+        dbadapter = new BigQueryDbAdapter(readCredentials);
+    }
+    else {
+        dbadapter = await PostgresDbAdapter.create(readCredentials);
+    }
+    const prunedGraph = prune(compiledGraph, {
+        actions: argv[actionsOption.name],
+        includeDependencies: argv[includeDepsOption.name],
+        includeDependents: argv[includeDependentsOption.name],
+        tags: argv[tagsOption.name]
+    });
+    const executionSql = new ExecutionSql(compiledGraph.projectConfig, dataformVersion);
+    const shadowSchemas = Array.from(new Set((prunedGraph.tables || []).map(table => table.target.schema)));
+    process.on("SIGINT", () => {
+        Promise.all(shadowSchemas.map(schema => dbadapter.execute(executionSql.dropSchemaCascadeSql(schema)).catch(() => undefined))).then(() => process.exit(1));
+    });
+    const deps = {
+        evaluate: action => dbadapter.evaluate(action.enumType !== undefined
+            ? sqlanvil.Table.create(action)
+            : sqlanvil.Assertion.create(action)),
+        execute: sql => dbadapter.execute(sql).then(() => undefined),
+        validationStubSql: table => executionSql.validationStubSql(table),
+        createSchemaSql: schema => executionSql.createSchemaSql(schema),
+        dropSchemaCascadeSql: schema => executionSql.dropSchemaCascadeSql(schema),
+        listSchemas: async () => {
+            const result = await dbadapter.execute("select schema_name as name from information_schema.schemata");
+            return ((result && result.rows) || []).map((row) => row.name);
+        }
+    };
+    await sweepOrphanShadows(deps, Date.now());
+    if (!argv[jsonOutputOption.name]) {
+        print("Validating...\n");
+    }
+    const results = await validate(prunedGraph, deps, { keepShadow: argv[keepShadowOption.name] });
+    return printValidationResults(results, argv[jsonOutputOption.name]);
+}
 function runCli() {
     const builtYargs = createYargsCli({
         commands: [
@@ -48075,6 +48412,24 @@ function runCli() {
                     return testResults.every(testResult => testResult.successful) ? 0 : 1;
                 }
             },
+            {
+                format: `validate [${projectDirMustExistOption.name}]`,
+                description: "Validate the project's SQL against the warehouse planner (EXPLAIN/dry-run) without " +
+                    "executing. Postgres/Supabase/MySQL only.",
+                positionalOptions: [projectDirMustExistOption],
+                options: [
+                    actionsOption,
+                    tagsOption,
+                    includeDepsOption,
+                    includeDependentsOption,
+                    credentialsOption,
+                    jsonOutputOption,
+                    timeoutOption,
+                    keepShadowOption,
+                    ...ProjectConfigOptions.allYargsOptions
+                ],
+                processFn: async (argv) => runValidate(argv)
+            },
             {
                 format: `run [${projectDirMustExistOption.name}]`,
                 description: "Run the sqlanvil project.",
@@ -48127,6 +48482,9 @@ function runCli() {
                         printSuccess("Compiled successfully.\n");
                     }
                     const warehouse = compiledGraph.projectConfig.warehouse || "bigquery";
+                    if (argv[dryRunOptionName] && warehouse.toLowerCase() !== "bigquery") {
+                        return runValidate(argv);
+                    }
                     const readCredentials = read(credentialsPathWithEnvironment(argv[projectDirOption.name], argv), warehouse);
                     let dbadapter;
                     if (warehouse.toLowerCase() === "supabase") {

package/package.json

@@ -63,7 +63,7 @@
     "bin": {
         "sqlanvil": "bundle.js"
     },
-    "version": "1.8.3",
+    "version": "1.9.0",
     "name": "@sqlanvil/cli",
     "description": "sqlanvil command line interface.",
     "main": "bundle.js"