@duckcodeailabs/dql-core 1.6.33 → 1.7.0

package/dist/semantic/semantic-layer.js

@@ -285,9 +285,9 @@ export class SemanticLayer {
             return null;
         // Resolve metric definitions
         const resolvedMetrics = metrics
-            .map((m) => this.metrics.get(m))
+            .map((m) => this.resolveComposableMetric(m))
             .filter(Boolean);
-        if (resolvedMetrics.length === 0)
+        if (resolvedMetrics.length !== metrics.length)
             return null;
         // Resolve dimension definitions
         const resolvedDimensions = dimensions
@@ -298,6 +298,18 @@ export class SemanticLayer {
         if (timeDimension) {
             timeDimDef = this.dimensions.get(timeDimension.name);
         }
+        const resolvedFilterDimensions = (filters ?? [])
+            .map((filter) => this.dimensions.get(filter.dimension))
+            .filter(Boolean);
+        // Multiple metrics from different fact tables must never be aggregated after
+        // one raw fact-to-fact join: that multiplies rows and silently inflates sums.
+        // Compile one aggregate island per metric at the requested conformed grain,
+        // then join the small aggregate results. Each island still uses the semantic
+        // graph and refuses independently when a dimension/filter is unjoinable.
+        const metricTables = new Set(resolvedMetrics.map((metric) => metric.table));
+        if (metricTables.size > 1) {
+            return this.composeAggregateIslands(options, resolvedMetrics, resolvedDimensions, timeDimDef, dialect);
+        }
         // Find the primary table (from first metric)
         const primaryTable = resolvedMetrics[0].table;
         // Collect all tables referenced
@@ -306,6 +318,8 @@ export class SemanticLayer {
             allTables.add(m.table);
         for (const d of resolvedDimensions)
             allTables.add(d.table);
+        for (const d of resolvedFilterDimensions)
+            allTables.add(d.table);
         if (timeDimDef)
             allTables.add(timeDimDef.table);
         // Find which cube corresponds to a table
@@ -345,6 +359,27 @@ export class SemanticLayer {
         }
         if (unjoinedTables.size > 0)
             return null;
+        // Metric-scoped filters (G1 correctness). A governed metric may declare a
+        // `filter` that scopes its aggregate to a subset of rows (e.g.
+        // completed_revenue = SUM(amount) WHERE status='completed'). Dropping it
+        // silently returns a WRONG number at the highest-trust tier, so we either
+        // apply it or refuse to compose (null) — the caller then falls through to
+        // generated SQL rather than surfacing a governed-but-wrong answer.
+        const resolveFilterColumn = (ref) => {
+            const dim = this.dimensions.get(ref);
+            if (dim?.sql)
+                return dim.sql;
+            return ref.includes('__') ? ref.split('__').pop() ?? ref : ref;
+        };
+        const metricFilters = resolvedMetrics.map((m) => computeMetricFilterPredicate(m, resolveFilterColumn));
+        // A metric that declares a filter we cannot render fails the whole compose.
+        if (metricFilters.some((r) => r.kind === 'fail'))
+            return null;
+        const okFilters = metricFilters.filter((r) => r.kind === 'ok');
+        // When EVERY metric carries the SAME filter, hoist it to a single WHERE
+        // predicate (cleaner and equivalent); otherwise apply per-metric via CASE WHEN.
+        const distinctPredicates = new Set(okFilters.map((r) => r.sql));
+        const hoistFilterToWhere = okFilters.length === resolvedMetrics.length && resolvedMetrics.length > 0 && distinctPredicates.size === 1;
         // Build SELECT
         const selectParts = [];
         // Add dimensions
@@ -360,9 +395,20 @@ export class SemanticLayer {
             const truncated = dialect.dateTrunc(grain, tdSql);
             selectParts.push(`${truncated} AS ${timeDimDef.name}_${grain}`);
         }
-        // Add metrics
-        for (const m of resolvedMetrics) {
-            selectParts.push(`${renderMetricExpression(m)} AS ${m.name}`);
+        // Add metrics (applying per-metric scoped filters via CASE WHEN unless hoisted)
+        for (let i = 0; i < resolvedMetrics.length; i++) {
+            const m = resolvedMetrics[i];
+            const fr = metricFilters[i];
+            let expr = renderMetricExpression(m);
+            if (fr.kind === 'ok' && !hoistFilterToWhere) {
+                const wrapped = wrapAggregateWithFilter(expr, fr.sql);
+                // Unparseable aggregate + a per-metric filter → refuse rather than emit
+                // a partially-applied filter.
+                if (wrapped === null)
+                    return null;
+                expr = wrapped;
+            }
+            selectParts.push(`${expr} AS ${m.name}`);
         }
         // Build FROM + JOINs (apply tableMapping for actual DB table names)
         let fromClause = `FROM ${resolveTable(primaryTable)}`;
@@ -393,7 +439,9 @@ export class SemanticLayer {
             if (!f.dimension)
                 continue;
             const dimDef = this.dimensions.get(f.dimension);
-            const dimSql = dimDef?.sql ?? f.dimension;
+            const dimSql = dimDef
+                ? (dimDef.sql.includes('.') || dimDef.table === primaryTable ? dimDef.sql : `${dimDef.table}.${dimDef.sql}`)
+                : f.dimension;
             const v0 = f.values?.[0] ?? '';
             const v1 = f.values?.[1] ?? '';
             // Detect if value looks numeric (no quoting needed)
@@ -455,6 +503,10 @@ export class SemanticLayer {
                     break;
             }
         }
+        // Governed metric filter shared by every metric: applied once as WHERE.
+        if (hoistFilterToWhere && okFilters.length > 0) {
+            whereParts.push(okFilters[0].sql);
+        }
         // Build ORDER BY
         const orderByParts = [];
         for (const o of orderBy ?? []) {
@@ -476,6 +528,121 @@ export class SemanticLayer {
             sql,
             joins: joinClauses,
             tables: Array.from(allTables),
+            strategy: 'direct_join',
+            grainKeys: [
+                ...resolvedDimensions.map((dimension) => dimension.name),
+                ...(timeDimDef && timeDimension ? [`${timeDimDef.name}_${timeDimension.granularity}`] : []),
+            ],
+        };
+    }
+    /**
+     * dbt simple metrics may store physical ownership on their input measure
+     * instead of duplicating it on the metric node. Materialize that contract for
+     * native composition; derived/ratio/cumulative metrics remain MetricFlow-only.
+     */
+    resolveComposableMetric(name) {
+        const metric = this.metrics.get(name);
+        if (!metric)
+            return undefined;
+        if (metric.table)
+            return metric;
+        if (metric.metricType && metric.metricType !== 'simple')
+            return undefined;
+        const measureNames = [];
+        const direct = metric.typeParams?.measure;
+        if (typeof direct === 'string')
+            measureNames.push(direct);
+        else if (direct && typeof direct === 'object' && !Array.isArray(direct)) {
+            const directName = direct.name;
+            if (typeof directName === 'string')
+                measureNames.push(directName);
+        }
+        const inputs = metric.typeParams?.input_measures;
+        if (Array.isArray(inputs)) {
+            for (const input of inputs) {
+                if (typeof input === 'string')
+                    measureNames.push(input);
+                else if (input && typeof input === 'object' && !Array.isArray(input)) {
+                    const inputName = input.name;
+                    if (typeof inputName === 'string')
+                        measureNames.push(inputName);
+                }
+            }
+        }
+        measureNames.push(metric.name);
+        const measure = measureNames
+            .map((measureName) => this.measures.get(measureName))
+            .find((candidate) => Boolean(candidate?.table));
+        if (!measure || measure.nonAdditiveDimension)
+            return undefined;
+        const type = normalizeMeasureMetricType(measure.agg);
+        if (!type)
+            return undefined;
+        return {
+            ...metric,
+            table: measure.table,
+            cube: metric.cube ?? measure.cube,
+            sql: measure.expr?.trim() || (type === 'count' ? '*' : measure.name),
+            type,
+        };
+    }
+    composeAggregateIslands(options, metrics, dimensions, timeDimension, dialect) {
+        const grainKeys = [
+            ...dimensions.map((dimension) => dimension.name),
+            ...(timeDimension && options.timeDimension
+                ? [`${timeDimension.name}_${options.timeDimension.granularity}`]
+                : []),
+        ];
+        const islands = metrics.map((metric) => this.composeQuery({
+            ...options,
+            metrics: [metric.name],
+            // Ranking/limits apply to the consolidated result, never inside an island.
+            orderBy: undefined,
+            limit: undefined,
+        }));
+        if (islands.some((island) => !island))
+            return null;
+        const compiled = islands;
+        const cteNames = metrics.map((metric, index) => `metric_${index + 1}_${sanitizeSqlAlias(metric.name)}`);
+        const ctes = compiled.map((island, index) => `${cteNames[index]} AS (\n${indentSql(island.sql)}\n)`);
+        let body;
+        const joinClauses = [];
+        if (grainKeys.length === 0) {
+            const select = metrics.map((metric, index) => `${cteNames[index]}.${metric.name} AS ${metric.name}`);
+            body = `SELECT\n  ${select.join(',\n  ')}\nFROM ${cteNames[0]}`;
+            for (const cte of cteNames.slice(1)) {
+                const clause = `CROSS JOIN ${cte}`;
+                joinClauses.push(clause);
+                body += `\n${clause}`;
+            }
+        }
+        else {
+            const keysCte = `grain_keys AS (\n${cteNames.map((cte) => `  SELECT ${grainKeys.join(', ')} FROM ${cte}`).join('\n  UNION\n')}\n)`;
+            ctes.push(keysCte);
+            const select = [
+                ...grainKeys.map((key) => `grain_keys.${key} AS ${key}`),
+                ...metrics.map((metric, index) => `${cteNames[index]}.${metric.name} AS ${metric.name}`),
+            ];
+            body = `SELECT\n  ${select.join(',\n  ')}\nFROM grain_keys`;
+            for (let index = 0; index < cteNames.length; index += 1) {
+                const cte = cteNames[index];
+                const predicate = grainKeys.map((key) => `grain_keys.${key} = ${cte}.${key}`).join(' AND ');
+                const clause = `LEFT JOIN ${cte} ON ${predicate}`;
+                joinClauses.push(clause);
+                body += `\n${clause}`;
+            }
+        }
+        if ((options.orderBy ?? []).length > 0) {
+            body += `\nORDER BY ${options.orderBy.map((order) => `${order.name} ${order.direction.toUpperCase()}`).join(', ')}`;
+        }
+        if (options.limit)
+            body += `\n${dialect.limitClause(options.limit)}`;
+        return {
+            sql: `WITH ${ctes.join(',\n')}\n${body}`,
+            joins: joinClauses,
+            tables: Array.from(new Set(compiled.flatMap((island) => island.tables))),
+            strategy: 'aggregate_islands',
+            grainKeys,
         };
     }
     addHierarchy(hierarchy) {
@@ -715,7 +882,6 @@ export class SemanticLayer {
             .filter((metric) => Boolean(metric));
         if (resolvedMetrics.length === 0)
             return [];
-        const reachableTables = new Set();
         const resolveCubeNameForTable = (table) => {
             for (const cube of this.cubes.values()) {
                 if (cube.table === table || cube.name === table)
@@ -723,28 +889,61 @@ export class SemanticLayer {
             }
             return undefined;
         };
-        for (const metric of resolvedMetrics) {
-            reachableTables.add(metric.table);
-            const cubeName = resolveCubeNameForTable(metric.table);
-            if (!cubeName)
-                continue;
-            const queue = [cubeName];
-            const visited = new Set(queue);
-            while (queue.length > 0) {
-                const current = queue.shift();
-                const cube = this.cubes.get(current);
-                if (cube?.table)
-                    reachableTables.add(cube.table);
-                const joins = this.joinGraph.get(current) ?? [];
-                for (const join of joins) {
-                    const next = join.right;
-                    if (visited.has(next))
-                        continue;
-                    visited.add(next);
-                    queue.push(next);
+        const metricTables = resolvedMetrics.map((metric) => {
+            const reachableTables = new Set();
+            const measureNames = new Set();
+            const measure = metric.typeParams?.measure;
+            if (measure && typeof measure === 'object' && !Array.isArray(measure) && typeof measure.name === 'string') {
+                measureNames.add(String(measure.name));
+            }
+            const inputs = metric.typeParams?.input_measures;
+            if (Array.isArray(inputs)) {
+                for (const input of inputs) {
+                    if (input && typeof input === 'object' && typeof input.name === 'string') {
+                        measureNames.add(String(input.name));
+                    }
                 }
             }
-        }
+            if (metric.table)
+                reachableTables.add(metric.table);
+            for (const measureName of measureNames) {
+                const definition = this.measures.get(measureName);
+                if (definition?.table)
+                    reachableTables.add(definition.table);
+                for (const model of this.semanticModels.values()) {
+                    if (model.measures.includes(measureName) && model.table)
+                        reachableTables.add(model.table);
+                }
+            }
+            for (const table of Array.from(reachableTables)) {
+                const cubeName = resolveCubeNameForTable(table);
+                if (!cubeName)
+                    continue;
+                const queue = [cubeName];
+                const visited = new Set(queue);
+                while (queue.length > 0) {
+                    const current = queue.shift();
+                    const cube = this.cubes.get(current);
+                    if (cube?.table)
+                        reachableTables.add(cube.table);
+                    const joins = this.joinGraph.get(current) ?? [];
+                    for (const join of joins) {
+                        const next = join.right;
+                        if (visited.has(next))
+                            continue;
+                        visited.add(next);
+                        queue.push(next);
+                    }
+                }
+            }
+            return reachableTables;
+        });
+        const reachableTables = metricTables.slice(1).reduce((common, tables) => {
+            for (const table of common)
+                if (!tables.has(table))
+                    common.delete(table);
+            return common;
+        }, new Set(metricTables[0]));
         return Array.from(this.dimensions.values())
             .filter((dimension) => reachableTables.has(dimension.table))
             .sort((a, b) => a.label.localeCompare(b.label));
@@ -825,6 +1024,151 @@ function renderMetricExpression(metric) {
             return sql;
     }
 }
+function normalizeMeasureMetricType(agg) {
+    switch (agg.toLowerCase()) {
+        case 'sum':
+        case 'count':
+        case 'count_distinct':
+        case 'avg':
+        case 'min':
+        case 'max':
+            return agg.toLowerCase();
+        case 'average':
+            return 'avg';
+        default:
+            return null;
+    }
+}
+/** Quote a scalar value for SQL, matching composeQuery's WHERE conventions. */
+function quoteSqlValue(v) {
+    return /^-?\d+(\.\d+)?$/.test(v.trim()) ? v : `'${v.replace(/'/g, "''")}'`;
+}
+/**
+ * Substitute MetricFlow-style `{{ Dimension('entity__dim') }}` /
+ * `{{ TimeDimension('entity__dim', 'day') }}` references in a raw SQL filter
+ * string. Plain SQL predicates (no Jinja) are trusted and returned as-is.
+ * Returns null when any non-Dimension Jinja tag remains, so the caller fails safe.
+ */
+function resolveFilterJinja(raw, resolveColumn) {
+    if (!raw.includes('{{'))
+        return raw;
+    const out = raw.replace(/\{\{\s*(?:Time)?Dimension\(\s*['"]([^'"]+)['"](?:\s*,\s*['"][^'"]+['"])?\s*\)\s*\}\}/g, (_full, ref) => resolveColumn(String(ref)));
+    return out.includes('{{') ? null : out;
+}
+/** Render a single object-shaped metric filter (`{ field, operator, value(s) }`). */
+function renderObjectFilter(raw, resolveColumn) {
+    const field = raw.field ?? raw.dimension ?? raw.column ?? raw.name;
+    if (field == null || String(field).trim() === '') {
+        if (typeof raw.sql === 'string' && raw.sql.trim())
+            return resolveFilterJinja(raw.sql.trim(), resolveColumn);
+        return null;
+    }
+    const col = resolveColumn(String(field));
+    const rawValues = raw.values ?? (raw.value !== undefined ? [raw.value] : []);
+    const values = (Array.isArray(rawValues) ? rawValues : [rawValues]).map((v) => String(v));
+    const op = String(raw.operator ?? raw.op ?? (values.length > 1 ? 'in' : 'equals')).toLowerCase();
+    const q = quoteSqlValue;
+    switch (op) {
+        case 'equals':
+        case '=':
+        case 'eq':
+            return values.length > 1 ? `${col} IN (${values.map(q).join(', ')})` : `${col} = ${q(values[0] ?? '')}`;
+        case 'not_equals':
+        case '!=':
+        case 'neq':
+            return `${col} != ${q(values[0] ?? '')}`;
+        case 'in':
+            return values.length ? `${col} IN (${values.map(q).join(', ')})` : null;
+        case 'not_in':
+            return values.length ? `${col} NOT IN (${values.map(q).join(', ')})` : null;
+        case 'gt':
+        case '>': return `${col} > ${q(values[0] ?? '')}`;
+        case 'gte':
+        case '>=': return `${col} >= ${q(values[0] ?? '')}`;
+        case 'lt':
+        case '<': return `${col} < ${q(values[0] ?? '')}`;
+        case 'lte':
+        case '<=': return `${col} <= ${q(values[0] ?? '')}`;
+        case 'contains': return `${col} LIKE '%${String(values[0] ?? '').replace(/'/g, "''")}%'`;
+        case 'is_null': return `${col} IS NULL`;
+        case 'is_not_null': return `${col} IS NOT NULL`;
+        default: return null;
+    }
+}
+/** Render a metric `filter` spec (string | object | array-of-objects) to SQL, or null if unrenderable. */
+function renderFilterSpec(spec, resolveColumn) {
+    if (typeof spec === 'string') {
+        const trimmed = spec.trim();
+        return trimmed ? resolveFilterJinja(trimmed, resolveColumn) : '';
+    }
+    if (Array.isArray(spec)) {
+        const parts = [];
+        for (const item of spec) {
+            if (!item || typeof item !== 'object')
+                return null;
+            const r = renderObjectFilter(item, resolveColumn);
+            if (r === null)
+                return null;
+            if (r)
+                parts.push(`(${r})`);
+        }
+        return parts.length ? parts.join(' AND ') : '';
+    }
+    if (spec && typeof spec === 'object')
+        return renderObjectFilter(spec, resolveColumn);
+    return null;
+}
+/** Compute a metric's scoped-filter predicate, distinguishing none / ok / unrenderable. */
+function computeMetricFilterPredicate(metric, resolveColumn) {
+    const f = metric.filter;
+    if (f === undefined || f === null)
+        return { kind: 'none' };
+    const rendered = renderFilterSpec(f, resolveColumn);
+    if (rendered === null)
+        return { kind: 'fail' };
+    if (!rendered)
+        return { kind: 'none' };
+    return { kind: 'ok', sql: rendered };
+}
+/**
+ * Inject a metric's scoped filter INSIDE its aggregate via CASE WHEN so that a
+ * multi-metric query with differing per-metric filters stays correct, e.g.
+ * `SUM(amount)` + `status='completed'` → `SUM(CASE WHEN status='completed' THEN amount END)`.
+ * Returns null when the expression is not a single recognizable aggregate call
+ * (e.g. a ratio `SUM(a)/SUM(b)`), so the caller fails safe rather than emit
+ * a filter that only partially applies.
+ */
+function wrapAggregateWithFilter(expr, predicate) {
+    const head = expr.match(/^\s*(sum|avg|min|max|count)\s*\(/i);
+    if (!head)
+        return null;
+    const fn = head[1].toUpperCase();
+    const openIdx = expr.indexOf('(', head.index ?? 0);
+    // Find the close paren that matches the aggregate's opening paren. If it is
+    // not the end of the expression, this is NOT a single aggregate call (e.g. a
+    // ratio `SUM(a) / SUM(b)`) — refuse rather than inject a half-applied filter.
+    let depth = 0;
+    let closeIdx = -1;
+    for (let i = openIdx; i < expr.length; i++) {
+        if (expr[i] === '(')
+            depth++;
+        else if (expr[i] === ')' && --depth === 0) {
+            closeIdx = i;
+            break;
+        }
+    }
+    if (closeIdx === -1 || expr.slice(closeIdx + 1).trim() !== '')
+        return null;
+    let inner = expr.slice(openIdx + 1, closeIdx).trim();
+    const distinctMatch = inner.match(/^distinct\s+/i);
+    const distinct = Boolean(distinctMatch);
+    if (distinct)
+        inner = inner.slice(distinctMatch[0].length).trim();
+    if (fn === 'COUNT' && !distinct && inner === '*')
+        return `COUNT(CASE WHEN ${predicate} THEN 1 END)`;
+    const distinctKw = distinct ? 'DISTINCT ' : '';
+    return `${fn}(${distinctKw}CASE WHEN ${predicate} THEN ${inner} END)`;
+}
 export function parseCubeDefinition(raw) {
     const measuresRaw = Array.isArray(raw.measures) ? raw.measures : [];
     const dimensionsRaw = Array.isArray(raw.dimensions) ? raw.dimensions : [];
@@ -887,6 +1231,13 @@ export function parseCubeDefinition(raw) {
         source: cubeSource,
     };
 }
+function sanitizeSqlAlias(value) {
+    const normalized = value.replace(/[^a-zA-Z0-9_]+/g, '_').replace(/^_+|_+$/g, '');
+    return normalized || 'metric';
+}
+function indentSql(sql) {
+    return sql.split(/\r?\n/).map((line) => `  ${line}`).join('\n');
+}
 function validateMetricType(t) {
     const valid = ['sum', 'count', 'count_distinct', 'avg', 'min', 'max', 'custom'];
     return valid.includes(t) ? t : 'sum';