@lightdash/cli 1.147.0 → 1.148.0

package/dist/handlers/preAggregateCheckExternal.d.ts

@@ -0,0 +1,13 @@
+import { type CompileHandlerOptions } from './compile';
+type PreAggregateCheckExternalOptions = CompileHandlerOptions & {
+    model: string | undefined;
+    name: string | undefined;
+    all: boolean;
+    json: boolean;
+    sql: boolean;
+    failOnMismatch: boolean;
+    clearCache: boolean;
+};
+export declare const preAggregateCheckExternalHandler: (options: PreAggregateCheckExternalOptions) => Promise<void>;
+export {};
+//# sourceMappingURL=preAggregateCheckExternal.d.ts.map

package/dist/handlers/preAggregateCheckExternal.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preAggregateCheckExternal.d.ts","sourceRoot":"","sources":["../../src/handlers/preAggregateCheckExternal.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAW,KAAK,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAIhE,KAAK,gCAAgC,GAAG,qBAAqB,GAAG;IAC5D,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,GAAG,EAAE,OAAO,CAAC;IACb,cAAc,EAAE,OAAO,CAAC;IACxB,UAAU,EAAE,OAAO,CAAC;CACvB,CAAC;AAsZF,eAAO,MAAM,gCAAgC,YAChC,gCAAgC,kBAiJ5C,CAAC"}

package/dist/handlers/preAggregateCheckExternal.js

@@ -0,0 +1,372 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preAggregateCheckExternalHandler = void 0;
+const tslib_1 = require("tslib");
+const common_1 = require("@lightdash/common");
+const inquirer_1 = tslib_1.__importDefault(require("inquirer"));
+const path = tslib_1.__importStar(require("path"));
+const context_1 = require("../dbt/context");
+const globalState_1 = tslib_1.__importDefault(require("../globalState"));
+const styles = tslib_1.__importStar(require("../styles"));
+const compile_1 = require("./compile");
+const getDbtVersion_1 = require("./dbt/getDbtVersion");
+const getWarehouseClient_1 = tslib_1.__importDefault(require("./dbt/getWarehouseClient"));
+const CLI_QUERY_TAGS = { query_context: common_1.QueryExecutionContext.CLI };
+const probeShape = async (warehouseClient, sql, clearCache) => {
+    let fields = {};
+    let hasRows = false;
+    // Nonce defeats result caches keyed on query text (BigQuery/Snowflake
+    // serve the stale schema for a repeated probe after a table is replaced)
+    const probeSql = clearCache ? `${sql} -- ${Date.now().toString(36)}` : sql;
+    await warehouseClient.streamQuery(probeSql, ({ fields: streamedFields, rows }) => {
+        if (Object.keys(streamedFields).length > 0) {
+            fields = streamedFields;
+        }
+        if (rows.length > 0) {
+            hasRows = true;
+        }
+    }, { tags: CLI_QUERY_TAGS });
+    return { fields, hasRows };
+};
+const confirmWarehouseValidation = async () => {
+    const interactive = !globalState_1.default.isNonInteractive() &&
+        process.stdin.isTTY === true &&
+        process.stdout.isTTY === true;
+    if (!interactive)
+        return true;
+    const { confirmed } = await inquirer_1.default.prompt([
+        {
+            type: 'confirm',
+            name: 'confirmed',
+            message: 'Run warehouse validation queries? This executes the generated materialization SQL and external table checks with LIMIT 1.',
+            default: false,
+        },
+    ]);
+    return confirmed;
+};
+const editDistance = (a, b) => {
+    const dp = Array.from({ length: a.length + 1 }, (_, i) => {
+        const row = Array.from({ length: b.length + 1 }, () => 0);
+        row[0] = i;
+        return row;
+    });
+    for (let j = 0; j <= b.length; j += 1)
+        dp[0][j] = j;
+    for (let i = 1; i <= a.length; i += 1) {
+        for (let j = 1; j <= b.length; j += 1) {
+            dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
+        }
+    }
+    return dp[a.length][b.length];
+};
+const describeRole = (column) => {
+    switch (column.role) {
+        case common_1.preAggregateMaterialization.MaterializationColumnRole
+            .TIME_DIMENSION:
+            return `time dimension (${column.granularity?.toLowerCase()})`;
+        case common_1.preAggregateMaterialization.MaterializationColumnRole.METRIC:
+            return `metric (${column.aggregation})`;
+        case common_1.preAggregateMaterialization.MaterializationColumnRole
+            .METRIC_COMPONENT:
+            return 'metric component';
+        case common_1.preAggregateMaterialization.MaterializationColumnRole.DIMENSION:
+            return 'dimension';
+        default:
+            return (0, common_1.assertUnreachable)(column.role, `Unknown column role: ${column.role}`);
+    }
+};
+const checkPreAggregate = async ({ explore, preAggregateDef, warehouseClient, clearCache, executeValidation, }) => {
+    const base = {
+        model: explore.name,
+        name: preAggregateDef.name,
+        table: preAggregateDef.table ?? null,
+        sql: null,
+        columns: [],
+        extraColumns: [],
+        tableError: null,
+        error: null,
+        validationSkippedReason: null,
+    };
+    let rendered;
+    try {
+        rendered = common_1.preAggregateMaterialization.renderMaterializationSql({
+            sourceExplore: explore,
+            preAggregateDef,
+            warehouseSqlBuilder: warehouseClient,
+        });
+    }
+    catch (e) {
+        return { ...base, error: (0, common_1.getErrorMessage)(e) };
+    }
+    base.sql = rendered.sql;
+    base.columns = rendered.columns.map((column) => ({
+        name: column.name,
+        role: describeRole(column),
+        expectedType: null,
+        actualType: null,
+        status: null,
+    }));
+    if (!executeValidation) {
+        base.validationSkippedReason = 'permission_declined';
+        return base;
+    }
+    // Match SQL Runner column discovery: execute one row and read its fields.
+    let expectedFields;
+    try {
+        const probe = await probeShape(warehouseClient, `SELECT * FROM (\n${rendered.sql}\n) AS ld_shape_probe LIMIT 1`, clearCache);
+        if (!probe.hasRows) {
+            base.validationSkippedReason = 'generated_query_no_rows';
+            return base;
+        }
+        expectedFields = probe.fields;
+    }
+    catch (e) {
+        return {
+            ...base,
+            error: `Generated materialization SQL failed against the warehouse: ${(0, common_1.getErrorMessage)(e)}`,
+        };
+    }
+    let actualFields = null;
+    if (preAggregateDef.table) {
+        try {
+            const probe = await probeShape(warehouseClient, `SELECT * FROM ${preAggregateDef.table} AS ld_shape_probe LIMIT 1`, clearCache);
+            if (!probe.hasRows) {
+                base.validationSkippedReason = 'external_table_no_rows';
+            }
+            else {
+                actualFields = probe.fields;
+            }
+        }
+        catch (e) {
+            base.tableError = (0, common_1.getErrorMessage)(warehouseClient.parseError(e instanceof Error ? e : new Error((0, common_1.getErrorMessage)(e))));
+        }
+    }
+    base.columns = rendered.columns.map((column) => {
+        const expectedType = expectedFields[column.name]?.type ?? null;
+        const actualType = actualFields?.[column.name]?.type ?? null;
+        let status = null;
+        if (actualFields) {
+            if (actualType === null) {
+                status = 'missing';
+            }
+            else if (expectedType !== null && actualType !== expectedType) {
+                status = 'type_mismatch';
+            }
+            else {
+                status = 'match';
+            }
+        }
+        return {
+            name: column.name,
+            role: describeRole(column),
+            expectedType,
+            actualType,
+            status,
+        };
+    });
+    if (actualFields) {
+        const expectedNames = new Set(base.columns.map((c) => c.name));
+        const missingNames = base.columns
+            .filter((c) => c.status === 'missing')
+            .map((c) => c.name);
+        base.extraColumns = Object.entries(actualFields)
+            .filter(([name]) => !expectedNames.has(name))
+            .map(([name, { type }]) => {
+            const closest = missingNames
+                .map((candidate) => ({
+                candidate,
+                distance: editDistance(name, candidate),
+            }))
+                .sort((a, b) => a.distance - b.distance)[0];
+            return {
+                name,
+                type,
+                didYouMean: closest &&
+                    closest.distance <=
+                        Math.max(3, Math.floor(closest.candidate.length / 4))
+                    ? closest.candidate
+                    : null,
+            };
+        });
+    }
+    return base;
+};
+const hasMismatch = (result) => result.error !== null ||
+    result.tableError !== null ||
+    result.columns.some((column) => column.status === 'missing');
+const renderHuman = (result, log) => {
+    const tableLabel = result.table
+        ? ` → ${result.table}`
+        : ` ${styles.secondary('(no table: declared — expected shape only)')}`;
+    log('');
+    log(`${styles.bold(`${result.model} / ${result.name}`)}${tableLabel}`);
+    if (result.error) {
+        log(`  ${styles.error('✗')} ${result.error}`);
+        return;
+    }
+    if (result.tableError) {
+        log(`  ${styles.warning('⚠')} Could not introspect declared table: ${result.tableError}`);
+    }
+    const nameWidth = Math.max(...result.columns.map((c) => c.name.length), 'EXPECTED'.length);
+    const typeWidth = Math.max(...result.columns.map((c) => (c.expectedType ?? '').length), 'TYPE'.length);
+    const roleWidth = Math.max(...result.columns.map((c) => c.role.length), 'ROLE'.length);
+    log('');
+    log(`  ${'EXPECTED'.padEnd(nameWidth)}  ${'TYPE'.padEnd(typeWidth)}  ${'ROLE'.padEnd(roleWidth)}${result.columns.some((c) => c.status !== null) ? '  ACTUAL' : ''}`);
+    result.columns.forEach((column) => {
+        let actual = '';
+        switch (column.status) {
+            case 'match':
+                actual = `  ${styles.success('✓')} ${column.actualType}`;
+                break;
+            case 'type_mismatch':
+                actual = `  ${styles.warning('⚠')} ${column.actualType} ${styles.secondary(`(expected ${column.expectedType})`)}`;
+                break;
+            case 'missing':
+                actual = `  ${styles.error('✗')} missing`;
+                break;
+            case null:
+                break;
+            default:
+                (0, common_1.assertUnreachable)(column.status, `Unknown column status: ${column.status}`);
+        }
+        log(`  ${column.name.padEnd(nameWidth)}  ${(column.expectedType ?? '?').padEnd(typeWidth)}  ${column.role.padEnd(roleWidth)}${actual}`);
+    });
+    result.extraColumns.forEach((extra) => {
+        const hint = extra.didYouMean
+            ? styles.secondary(` — did you mean "${extra.didYouMean}"?`)
+            : '';
+        log(`  ${extra.name} ${styles.secondary(`(${extra.type})`)} ${styles.warning('unmatched actual column')}${hint}`);
+    });
+    log('');
+    if (result.validationSkippedReason) {
+        switch (result.validationSkippedReason) {
+            case 'permission_declined':
+                log(`  ${styles.secondary('Warehouse validation skipped — showing generated SQL only.')}`);
+                break;
+            case 'generated_query_no_rows':
+                log(`  ${styles.warning('Generated query returned no rows — unable to validate.')}`);
+                break;
+            case 'external_table_no_rows':
+                log(`  ${styles.warning(`${result.table} returned no rows — unable to validate.`)}`);
+                break;
+            default:
+                (0, common_1.assertUnreachable)(result.validationSkippedReason, `Unknown validation skipped reason: ${result.validationSkippedReason}`);
+        }
+    }
+    else if (result.columns.some((c) => c.status !== null)) {
+        const missingCount = result.columns.filter((c) => c.status === 'missing').length;
+        const matchCount = result.columns.length - missingCount;
+        if (missingCount > 0) {
+            log(`  ${styles.error('✗')} ${missingCount} missing column${missingCount === 1 ? '' : 's'}. Rebuild ${result.table} with the SQL below, then re-run.`);
+        }
+        else {
+            log(`  ${styles.success('✓')} Table matches — ${matchCount}/${result.columns.length} columns.`);
+        }
+    }
+    else if (!result.tableError) {
+        log(`  ${styles.secondary('No live table checked — showing expected shape and SQL only.')}`);
+    }
+};
+const preAggregateCheckExternalHandler = async (options) => {
+    globalState_1.default.setVerbose(options.verbose);
+    if (!options.all && (!options.model || !options.name)) {
+        console.error(styles.error('Specify --model <model> and --name <pre-aggregate-name>, or --all for every external pre-aggregate.'));
+        process.exit(1);
+    }
+    const dbtVersion = await (0, getDbtVersion_1.getDbtVersion)();
+    if (dbtVersion.isDbtCloudCLI) {
+        console.error(styles.error('pre-aggregate-check-external needs direct warehouse access and does not support the dbt Cloud CLI.'));
+        process.exit(1);
+    }
+    const executeValidation = await confirmWarehouseValidation();
+    const explores = await (0, compile_1.compile)(executeValidation
+        ? options
+        : { ...options, skipWarehouseCatalog: true });
+    const validExplores = explores.filter((explore) => !(0, common_1.isExploreError)(explore));
+    const targets = [];
+    if (options.all) {
+        validExplores.forEach((explore) => {
+            (explore.preAggregates ?? []).forEach((preAggregateDef) => {
+                if (preAggregateDef.table) {
+                    targets.push({ explore, preAggregateDef });
+                }
+            });
+        });
+        if (targets.length === 0) {
+            console.error(styles.warning('No external pre-aggregates (with a `table:` key) found in the project.'));
+            return;
+        }
+    }
+    else {
+        const explore = validExplores.find((candidate) => candidate.name === options.model);
+        if (!explore) {
+            console.error(styles.error(`Model "${options.model}" not found in project.`));
+            process.exit(1);
+        }
+        const preAggregateDef = (explore.preAggregates ?? []).find((candidate) => candidate.name === options.name);
+        if (!preAggregateDef) {
+            console.error(styles.error(`Pre-aggregate "${options.name}" not found on model "${options.model}". Available: ${(explore.preAggregates ?? [])
+                .map((candidate) => candidate.name)
+                .join(', ')}`));
+            process.exit(1);
+        }
+        targets.push({ explore, preAggregateDef });
+    }
+    const dbtContext = await (0, context_1.getDbtContext)({
+        projectDir: path.resolve(options.projectDir),
+        targetPath: options.targetPath,
+    });
+    const { warehouseClient } = await (0, getWarehouseClient_1.default)({
+        isDbtCloudCLI: false,
+        profilesDir: options.profilesDir,
+        profile: options.profile || dbtContext.profileName,
+        target: options.target,
+        startOfWeek: options.startOfWeek,
+    });
+    const results = [];
+    // Sequential on purpose: one warehouse connection, readable progress
+    for await (const target of targets) {
+        results.push(await checkPreAggregate({
+            explore: target.explore,
+            preAggregateDef: target.preAggregateDef,
+            warehouseClient,
+            clearCache: options.clearCache,
+            executeValidation,
+        }));
+    }
+    // --sql keeps stdout machine-readable; humans read stderr
+    const log = options.sql
+        ? (line) => console.error(line)
+        : (line) => console.log(line);
+    if (options.json) {
+        console.log(JSON.stringify(options.all ? results : results[0], null, 2));
+    }
+    else {
+        results.forEach((result) => {
+            renderHuman(result, log);
+            if (result.sql && !options.sql) {
+                log('');
+                log(`  Materialization SQL (${warehouseClient.getAdapterType()}):`);
+                log('');
+                log(result.sql
+                    .split('\n')
+                    .map((line) => `  ${line}`)
+                    .join('\n'));
+            }
+        });
+        if (options.sql) {
+            results.forEach((result) => {
+                if (result.sql) {
+                    if (results.length > 1) {
+                        console.log(`-- ${result.model} / ${result.name}`);
+                    }
+                    console.log(result.sql);
+                }
+            });
+        }
+    }
+    if (options.failOnMismatch && results.some(hasMismatch)) {
+        process.exit(1);
+    }
+};
+exports.preAggregateCheckExternalHandler = preAggregateCheckExternalHandler;

package/dist/index.js

@@ -27,6 +27,7 @@ const lint_1 = require("./handlers/lint");
 const listProjects_1 = require("./handlers/listProjects");
 const login_1 = require("./handlers/login");
 const preAggregateAudit_1 = require("./handlers/preAggregateAudit");
+const preAggregateCheckExternal_1 = require("./handlers/preAggregateCheckExternal");
 const preview_2 = require("./handlers/preview");
 const renameHandler_1 = require("./handlers/renameHandler");
 const renameProject_1 = require("./handlers/renameProject");
@@ -652,6 +653,26 @@ commander_1.program
     .option('--fail-on-miss', 'Exit 1 if any eligible tile is a pre-aggregate miss (CI-friendly)', false)
     .option('--verbose', 'Expand the ineligible section in human output', false)
     .action(preAggregateAudit_1.preAggregateAuditHandler);
+commander_1.program
+    .command('pre-aggregate-check-external')
+    .description('Generate the expected table shape and materialization SQL for an external pre-aggregate, and check the declared table against it. Runs fully locally against the dbt project and warehouse credentials from profiles.yml.')
+    .option('--model <model>', 'Model the pre-aggregate is defined on')
+    .option('--name <name>', 'Pre-aggregate name')
+    .option('--all', 'Check every external pre-aggregate (with a `table:` key) in the project', false)
+    .option('--project-dir <path>', 'The directory of the dbt project', env_1.DEFAULT_DBT_PROJECT_DIR)
+    .option('--profiles-dir <path>', 'The directory of the dbt profiles', env_1.DEFAULT_DBT_PROFILES_DIR)
+    .option('--profile <name>', 'The name of the profile to use (defaults to profile name in dbt_project.yml)', undefined)
+    .option('--target <name>', 'target to use in profiles.yml file', undefined)
+    .option('--target-path <path>', 'The target directory for dbt (overrides DBT_TARGET_PATH and dbt_project.yml)', undefined)
+    .option('--vars <vars>')
+    .option('--skip-dbt-compile', 'Skip `dbt compile` and deploy from the existing ./target/manifest.json', false)
+    .option('--skip-warehouse-catalog', 'Skip fetch warehouse catalog and use types defined in the YAML.', false)
+    .option('--json', 'Emit machine-readable JSON instead of human output', false)
+    .option('--sql', 'Print bare materialization SQL on stdout (human output goes to stderr)', false)
+    .option('--fail-on-mismatch', 'Exit 1 if any checked pre-aggregate has missing columns, an unreachable table, or an SQL error (CI-friendly)', false)
+    .option('--clear-cache', 'Bypass warehouse result caches when probing table shapes (use after rebuilding a table, or the warehouse may report its old schema)', false)
+    .option('--verbose', undefined, false)
+    .action(preAggregateCheckExternal_1.preAggregateCheckExternalHandler);
 commander_1.program
     .command('sql')
     .description('Run raw SQL query against the warehouse using project credentials')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/cli",
-  "version": "1.147.0",
+  "version": "1.148.0",
   "description": "Lightdash CLI tool",
   "license": "MIT",
   "repository": {
@@ -42,8 +42,8 @@
     "unique-names-generator": "4.7.1",
     "uuid": "11.1.1",
     "yaml": "2.8.3",
-    "@lightdash/warehouses": "1.147.0",
-    "@lightdash/common": "1.147.0"
+    "@lightdash/common": "1.148.0",
+    "@lightdash/warehouses": "1.148.0"
   },
   "devDependencies": {
     "@types/inquirer": "8.2.4",