@duckcodeailabs/dql-agent 1.6.31 → 1.6.33

package/dist/metadata/analysis-planner.js

@@ -26,7 +26,10 @@ export function buildAnalysisQuestionPlan(question, followUp) {
     const normalizedQuestion = normalizeSearchText(cleanQuestion);
     const lower = cleanQuestion.toLowerCase();
     const entities = extractEntities(cleanQuestion);
-    const metricTerms = extractMetricTerms(cleanQuestion);
+    const metricTerms = uniqueStrings([
+        ...extractMetricTerms(cleanQuestion),
+        ...extractFollowUpMetricTerms(followUp, lower),
+    ]);
     const dimensionTerms = extractDimensionTerms(cleanQuestion);
     const filterTerms = extractFilterTerms(cleanQuestion, entities);
     const timeTerms = extractTimeTerms(cleanQuestion);
@@ -36,6 +39,15 @@ export function buildAnalysisQuestionPlan(question, followUp) {
     const shouldConsiderCertifiedExact = certifiedExactIsPlausible(mode, entities);
     const needsGeneratedSql = generatedSqlIsLikely(mode, shouldConsiderCertifiedExact);
     const needsResearchWorkspace = researchWorkspaceIsLikely(mode, lower);
+    const requestedShape = buildRequestedAnswerShape(cleanQuestion, {
+        lower,
+        mode,
+        metricTerms,
+        dimensionTerms,
+        filterTerms,
+        timeTerms,
+        followUp,
+    });
     const searchTerms = uniqueStrings([
         ...tokenize(cleanQuestion),
         ...entities.flatMap((entity) => tokenize(entity.text)),
@@ -72,6 +84,7 @@ export function buildAnalysisQuestionPlan(question, followUp) {
         needsResearchWorkspace,
         searchQueries,
         searchTerms,
+        requestedShape,
         confidence: planConfidence(mode, entities, metricTerms, dimensionTerms),
         reasons,
     };
@@ -209,6 +222,12 @@ export function scoreAllowedSqlRelationWithAnalysisPlan(relation, plan) {
                 : 0;
     const usabilityBonus = generatedSqlUsabilityScore(relation, plan);
     const columnShapeBonus = relationColumnShapeScore(relation, plan);
+    const semanticColumnBonus = semanticColumnMapScore(relation, plan);
+    const incompleteSourceShapePenalty = relation.source.includes('certified source SQL') &&
+        sourceShapeMustCoverRequestedOutputs(plan) &&
+        semanticColumnBonus.missingRequired.length > 0
+        ? Math.min(42, semanticColumnBonus.missingRequired.length * 34)
+        : 0;
     const score = Number((search.score * 3 +
         metrics.score * 8 +
         dimensions.score * 6 +
@@ -217,6 +236,8 @@ export function scoreAllowedSqlRelationWithAnalysisPlan(relation, plan) {
         sourceBonus +
         usabilityBonus +
         columnShapeBonus +
+        semanticColumnBonus.score +
+        -incompleteSourceShapePenalty +
         Math.min(relation.columns.length, 24) * 0.2).toFixed(3));
     const reasons = [
         search.matched.length ? `analysis terms matched: ${search.matched.slice(0, 5).join(', ')}` : '',
@@ -228,6 +249,8 @@ export function scoreAllowedSqlRelationWithAnalysisPlan(relation, plan) {
         usabilityBonus > 0 ? 'relation has usable columns for generated SQL' : '',
         usabilityBonus < 0 ? 'relation has no inspected/projected columns for generated SQL' : '',
         columnShapeBonus > 0 ? 'columns match requested analytical shape' : '',
+        semanticColumnBonus.matched.length ? `semantic column map matched: ${semanticColumnBonus.matched.slice(0, 6).join(', ')}` : '',
+        incompleteSourceShapePenalty > 0 ? `certified source SQL shape is missing requested output(s): ${semanticColumnBonus.missingRequired.join(', ')}` : '',
         relation.columns.length > 0 ? `${relation.columns.length} inspected/projected columns` : '',
     ].filter(Boolean);
     return {
@@ -266,6 +289,152 @@ function relationColumnShapeScore(relation, plan) {
     }
     return score;
 }
+function semanticColumnMapScore(relation, plan) {
+    if (relation.columns.length === 0)
+        return { score: 0, matched: [], missingRequired: [] };
+    const requiredConcepts = uniqueStrings(plan.requestedShape.requiredOutputs.map(canonicalColumnConcept).filter(Boolean));
+    const requestedOutputs = uniqueStrings([
+        ...requiredConcepts,
+        ...plan.requestedShape.dimensions,
+        ...plan.requestedShape.measures,
+        ...plan.dimensionTerms,
+        ...plan.metricTerms,
+    ].map(canonicalColumnConcept).filter(Boolean));
+    const matched = [];
+    const missingRequired = [];
+    let score = 0;
+    for (const concept of requestedOutputs) {
+        const best = bestColumnConceptScore(concept, relation.columns);
+        if (best.score <= 0) {
+            if (requiredConcepts.includes(concept)) {
+                missingRequired.push(concept);
+                score -= 12;
+            }
+            continue;
+        }
+        matched.push(`${concept}->${best.column}`);
+        score += best.score * (requiredConcepts.includes(concept) ? 1.35 : 1);
+    }
+    const capped = Math.max(-36, Math.min(score, plan.needsGeneratedSql ? 44 : 28));
+    return { score: capped, matched: uniqueStrings(matched), missingRequired: uniqueStrings(missingRequired) };
+}
+function sourceShapeMustCoverRequestedOutputs(plan) {
+    return plan.mode === 'ranking' ||
+        plan.mode === 'driver_breakdown' ||
+        plan.mode === 'comparison' ||
+        plan.mode === 'trend' ||
+        plan.mode === 'general_analysis';
+}
+function canonicalColumnConcept(value) {
+    return normalizeTerm(value)
+        .replace(/\bname\b$/i, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+function bestColumnConceptScore(concept, columns) {
+    let best = { score: 0, column: '' };
+    for (const column of columns) {
+        const score = columnConceptScore(concept, column);
+        if (score > best.score)
+            best = { score, column: column.name };
+    }
+    return best;
+}
+function columnConceptScore(concept, column) {
+    const normalizedConcept = normalizeTerm(concept.replace(/_name$/, ''));
+    if (!normalizedConcept)
+        return 0;
+    const columnName = normalizeSearchText(column.name);
+    const columnTokens = tokenize(column.name);
+    const columnText = normalizeSearchText([
+        column.name,
+        column.description ?? '',
+    ].join(' '));
+    const aliases = semanticColumnAliases(normalizedConcept);
+    let score = 0;
+    if (columnName === normalizedConcept || columnName === `${normalizedConcept} name`)
+        score += 12;
+    if (columnTokens.includes(normalizedConcept))
+        score += 9;
+    for (const alias of aliases) {
+        const aliasText = normalizeSearchText(alias);
+        const aliasTokens = tokenize(alias).filter((token) => !GENERIC_COLUMN_ALIAS_TOKENS.has(token));
+        if (!aliasText)
+            continue;
+        if (columnName === aliasText)
+            score += 11;
+        else if (columnText.includes(aliasText))
+            score += 7;
+        else if (aliasTokens.some((token) => columnTokens.includes(token)))
+            score += 5;
+    }
+    if (lineageAliasesIncludeConcept(column.description, normalizedConcept, aliases))
+        score += 12;
+    if (isCertifiedSourceShapeDescription(column.description))
+        score -= 16;
+    if (isMetricConcept(normalizedConcept) && isIdentifierLikeColumn(column.name))
+        score -= 8;
+    if (isDimensionConcept(normalizedConcept) && isMetricLikeColumn(column.name))
+        score -= 6;
+    return Math.max(0, score);
+}
+const GENERIC_COLUMN_ALIAS_TOKENS = new Set(['name', 'title', 'type', 'value', 'total']);
+function lineageAliasesIncludeConcept(description, concept, aliases) {
+    const match = /governed aliases from lineage:\s*([^.]*)/i.exec(description ?? '');
+    if (!match?.[1])
+        return false;
+    const aliasText = normalizeSearchText(match[1]);
+    const candidates = [concept, ...aliases].map(normalizeSearchText).filter(Boolean);
+    return candidates.some((candidate) => aliasText === candidate ||
+        aliasText.split(/\s*,\s*/).some((alias) => normalizeSearchText(alias) === candidate));
+}
+function isCertifiedSourceShapeDescription(description) {
+    return /projected by certified source sql shape/i.test(description ?? '');
+}
+function semanticColumnAliases(concept) {
+    switch (concept) {
+        case 'revenue':
+        case 'sale':
+        case 'sales':
+            return ['revenue', 'sales', 'amount', 'gross amount', 'net amount', 'price', 'product price', 'order amount', 'total'];
+        case 'usage':
+            return ['usage', 'use', 'consumption', 'activity', 'events', 'sessions', 'volume'];
+        case 'order':
+        case 'orders':
+            return ['order count', 'orders', 'order total', 'count lifetime orders'];
+        case 'product':
+            return ['product', 'product name', 'product title', 'sku', 'item', 'item name'];
+        case 'category':
+            return ['category', 'category name', 'product category', 'product type', 'type', 'class'];
+        case 'customer':
+            return ['customer', 'customer name', 'buyer', 'client', 'account name', 'full name'];
+        case 'segment':
+            return ['segment', 'customer segment', 'market segment'];
+        case 'region':
+        case 'market':
+            return ['region', 'market', 'geo', 'location', 'country', 'state'];
+        case 'player':
+            return ['player', 'player name', 'athlete', 'athlete name'];
+        case 'month':
+        case 'week':
+        case 'year':
+            return [concept, `${concept} date`, `${concept} start`, 'date', 'period'];
+        default:
+            return [concept.replace(/_/g, ' ')];
+    }
+}
+function isMetricConcept(concept) {
+    return METRIC_WORDS.includes(concept) || ['sale', 'sales', 'order', 'orders'].includes(concept);
+}
+function isDimensionConcept(concept) {
+    return DIMENSION_WORDS.includes(concept);
+}
+function isIdentifierLikeColumn(column) {
+    return /\b(id|key|uuid|code|hash)\b/i.test(column.replace(/_/g, ' '));
+}
+function isMetricLikeColumn(column) {
+    return /\b(revenue|sales|amount|price|spend|cost|total|count|score|points?|quantity|value|rate|volume)\b/i.test(column.replace(/_/g, ' '));
+}
 function inferQuestionMode(input) {
     const { lower } = input;
     if (followUpKind(input.followUp) === 'drilldown')
@@ -375,6 +544,12 @@ function extractMetricTerms(question) {
         terms.add('score');
         terms.add('scoring');
     }
+    if (/\b(how many|number of)\b/i.test(lower)) {
+        terms.add('count');
+    }
+    if (/\bhow much\b/i.test(lower)) {
+        terms.add('amount');
+    }
     for (const word of METRIC_WORDS) {
         if (new RegExp(`\\b${escapeRegExp(word)}s?\\b`, 'i').test(lower))
             terms.add(normalizeTerm(word));
@@ -388,8 +563,10 @@ function extractMetricTerms(question) {
 function extractDimensionTerms(question) {
     const lower = question.toLowerCase();
     const terms = new Set();
+    if (/\b(?:cusomers?|custmers?|costomers?|clients?|buyers?)\b/i.test(lower))
+        terms.add('customer');
     for (const word of DIMENSION_WORDS) {
-        if (new RegExp(`\\b${escapeRegExp(word)}s?\\b`, 'i').test(lower))
+        if (new RegExp(`\\b(?:${escapeRegExp(word)}|${escapeRegExp(pluralizeDimensionWord(word))})\\b`, 'i').test(lower))
             terms.add(normalizeTerm(word));
     }
     for (const match of lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g)) {
@@ -399,7 +576,15 @@ function extractDimensionTerms(question) {
     }
     return uniqueStrings([...terms]).slice(0, 16);
 }
+function pluralizeDimensionWord(word) {
+    if (word.endsWith('y'))
+        return `${word.slice(0, -1)}ies`;
+    return `${word}s`;
+}
 function extractFilterTerms(question, entities) {
+    // Filter VALUES come from the question's own entities and time phrases only.
+    // (Real value binding happens later against the runtime value index / sampled
+    // column values — never from a hard-coded project-specific vocabulary.)
     return uniqueStrings([
         ...entities.flatMap((entity) => tokenize(entity.text)),
         ...Array.from(question.matchAll(/\b(?:last|this|next|previous|prior|current)\s+(day|week|month|quarter|year|season)\b/gi)).map((match) => match[0].toLowerCase()),
@@ -416,6 +601,249 @@ function extractTimeTerms(question) {
     }
     return uniqueStrings(terms).slice(0, 12);
 }
+function extractFollowUpMetricTerms(followUp, lowerQuestion) {
+    if (!followUp || typeof followUp !== 'object' || Array.isArray(followUp))
+        return [];
+    const record = followUp;
+    // Advisory 'contextual' carry must not inject prior measures into a genuinely-new
+    // question's plan (it would bias retrieval and the block-fit gate toward the old
+    // topic). Carry prior measures only when the question textually refers back.
+    if (record.kind === 'contextual' && lowerQuestion !== undefined
+        && !/\b(these|those|that|them|this|same|prior|previous|above)\b/.test(lowerQuestion)) {
+        return [];
+    }
+    return uniqueStrings([
+        ...cleanStringArray(record.priorMeasures),
+        ...cleanStringArray(record.priorResultColumns).filter((column) => METRIC_WORDS.some((word) => new RegExp(`\\b${word}\\b`, 'i').test(column.replace(/_/g, ' ')))),
+    ]).slice(0, 8);
+}
+function cleanStringArray(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value
+        .map((item) => typeof item === 'string' ? item.trim() : '')
+        .filter(Boolean);
+}
+function buildRequestedAnswerShape(question, input) {
+    const topN = parseTopN(question);
+    const followUpReferences = extractFollowUpReferences(question, input.followUp);
+    const rawDimensions = uniqueStrings([
+        ...input.dimensionTerms.map(canonicalShapeTerm),
+        ...followUpDimensionsForRequestedShape(input.followUp, followUpReferences),
+    ].filter(Boolean));
+    const dimensions = scalarValueQuestionUsesEntitiesAsMeasures(input.mode, input.lower)
+        ? []
+        : rawDimensions;
+    const measures = uniqueStrings(input.metricTerms.map(canonicalShapeTerm).filter(Boolean));
+    const requiredOutputs = extractRequiredOutputs(question, dimensions, measures);
+    const filters = uniqueStrings([
+        ...input.filterTerms,
+        ...followUpFiltersForRequestedShape(input.followUp, followUpReferences),
+    ]);
+    const ambiguities = /\bimpact(?:ed|s|ing)?\b/i.test(question)
+        ? [{
+                term: 'impacted',
+                defaultInterpretation: 'highest contribution to the requested metric',
+                requiresClarification: false,
+            }]
+        : [];
+    return {
+        grain: inferRequestedShapeGrain(dimensions, input.timeTerms),
+        dimensions,
+        measures,
+        requiredOutputs,
+        filters,
+        ...(topN ? { topN } : {}),
+        rankingDirection: rankingDirection(question),
+        followUpReferences,
+        ambiguities,
+    };
+}
+function scalarValueQuestionUsesEntitiesAsMeasures(mode, lower) {
+    if (mode !== 'exact_lookup' && mode !== 'general_analysis')
+        return false;
+    return /\b(how many|how much|what is|what was|total|count|number of)\b/.test(lower)
+        && !/\b(by|per|each|every|top|bottom|rank|ranking|list|which|who|break\s*down|breakdown|split|segment|trend|over time)\b/.test(lower);
+}
+function parseTopN(question) {
+    const lower = question.toLowerCase();
+    const numeric = lower.match(/\b(?:top|bottom|first|last)\s+(\d{1,3})\b/);
+    const n = numeric ? Number(numeric[1]) : wordNumberFromTopN(lower);
+    if (!n || n <= 0)
+        return undefined;
+    const scope = /\b(?:per|for each|within each)\s+\w+/i.test(lower)
+        || /\bby\s+(?:category|segment|region|channel|product|customer|account|user|month|week|year)\b/i.test(lower)
+        ? 'per_group'
+        : 'overall';
+    return { n, scope };
+}
+function wordNumberFromTopN(lower) {
+    const words = {
+        one: 1,
+        two: 2,
+        three: 3,
+        four: 4,
+        five: 5,
+        six: 6,
+        seven: 7,
+        eight: 8,
+        nine: 9,
+        ten: 10,
+    };
+    const match = lower.match(/\b(?:top|bottom|first|last)\s+(one|two|three|four|five|six|seven|eight|nine|ten)\b/);
+    return match ? words[match[1]] : undefined;
+}
+function extractRequiredOutputs(question, dimensions, measures) {
+    const lower = question.toLowerCase();
+    const outputs = new Set();
+    for (const dim of dimensions)
+        outputs.add(dim);
+    for (const measure of measures)
+        outputs.add(measure);
+    if (/\bproduct\s+name\b/i.test(lower))
+        outputs.add('product_name');
+    if (/\bcustomer\s+name\b/i.test(lower))
+        outputs.add('customer_name');
+    if (/\bcategory\s+name\b/i.test(lower))
+        outputs.add('category_name');
+    if (/\b(?:name|names)\b/i.test(lower)) {
+        if (dimensions.includes('product'))
+            outputs.add('product_name');
+        if (dimensions.includes('customer'))
+            outputs.add('customer_name');
+        if (dimensions.includes('category'))
+            outputs.add('category_name');
+    }
+    return uniqueStrings([...outputs]).slice(0, 24);
+}
+function extractFollowUpReferences(question, followUp) {
+    const refs = [];
+    const lower = question.toLowerCase();
+    const record = followUpRecord(followUp);
+    const hasFollowUp = Boolean(record);
+    for (const match of lower.matchAll(/\b(these|those|that|same|prior|previous)\s+([a-z][a-z0-9_ -]{1,30})\b/g)) {
+        const phrase = match[0];
+        const noun = canonicalShapeTerm(match[2]);
+        let kind = 'ambiguous';
+        if (/period|date|day|week|month|quarter|year|time/.test(noun))
+            kind = 'prior_timeframe';
+        else if (hasFollowUp && /\b(category|product|customer|account|user|region|segment|channel)\b/.test(noun))
+            kind = 'prior_dimension_values';
+        else if (hasFollowUp)
+            kind = 'prior_entities';
+        refs.push({
+            phrase,
+            kind,
+            ...resolvedFollowUpValues(record, noun, kind),
+        });
+    }
+    if (refs.length === 0 && record) {
+        const bare = lower.match(/\b(these|those|that|them|same)\b/);
+        const dimension = bare ? singleFollowUpDimension(record) : undefined;
+        if (bare && dimension) {
+            refs.push({
+                phrase: bare[0],
+                kind: 'prior_dimension_values',
+                ...resolvedFollowUpValues(record, dimension, 'prior_dimension_values'),
+            });
+        }
+    }
+    return refs.slice(0, 8);
+}
+function followUpDimensionsForRequestedShape(followUp, refs) {
+    const record = followUpRecord(followUp);
+    if (!record || refs.every((ref) => ref.kind !== 'prior_dimension_values'))
+        return [];
+    const explicit = cleanStringArray(record.dimensions).map(contextDimensionTerm).filter(Boolean);
+    if (explicit.length > 0)
+        return uniqueStrings(explicit);
+    const single = singleFollowUpDimension(record);
+    return single ? [single] : [];
+}
+function followUpFiltersForRequestedShape(followUp, refs) {
+    const record = followUpRecord(followUp);
+    if (!record || refs.every((ref) => ref.kind !== 'prior_dimension_values'))
+        return [];
+    return uniqueStrings([
+        ...cleanStringArray(record.filters),
+        ...refs.flatMap((ref) => ref.resolvedValues ?? []),
+    ]);
+}
+function resolvedFollowUpValues(record, dimension, kind) {
+    if (!record || kind !== 'prior_dimension_values')
+        return {};
+    const values = followUpValuesForDimension(record, dimension);
+    return values.length > 0 ? { resolvedValues: values } : {};
+}
+function followUpValuesForDimension(record, dimension) {
+    const valuesByDimension = cleanStringRecord(record.priorResultValues);
+    const canonicalDimension = dimension ? contextDimensionTerm(dimension) : singleFollowUpDimension(record);
+    const values = canonicalDimension
+        ? valuesByDimension[canonicalDimension] ?? valuesByDimension[`${canonicalDimension}_name`] ?? []
+        : [];
+    return uniqueStrings([
+        ...values,
+        ...cleanStringArray(record.filters),
+    ]).slice(0, 24);
+}
+function singleFollowUpDimension(record) {
+    const explicit = uniqueStrings(cleanStringArray(record.dimensions).map(contextDimensionTerm).filter(Boolean));
+    if (explicit.length === 1)
+        return explicit[0];
+    if (explicit.length > 1)
+        return undefined;
+    const valuesByDimension = cleanStringRecord(record.priorResultValues);
+    const inferred = uniqueStrings(Object.keys(valuesByDimension)
+        .map(contextDimensionTerm)
+        .filter((term) => term && !METRIC_WORDS.includes(term)));
+    return inferred.length === 1 ? inferred[0] : undefined;
+}
+function contextDimensionTerm(value) {
+    const term = canonicalShapeTerm(value);
+    if (term.includes('category'))
+        return 'category';
+    if (term.includes('product'))
+        return 'product';
+    if (term.includes('customer'))
+        return 'customer';
+    if (term.includes('account'))
+        return 'account';
+    if (term.includes('user'))
+        return 'user';
+    if (term.includes('region'))
+        return 'region';
+    if (term.includes('segment'))
+        return 'segment';
+    if (term.includes('channel'))
+        return 'channel';
+    return term;
+}
+function cleanStringRecord(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        return {};
+    const out = {};
+    for (const [key, raw] of Object.entries(value)) {
+        const cleanKey = contextDimensionTerm(key);
+        const values = cleanStringArray(raw).slice(0, 24);
+        if (cleanKey && values.length > 0)
+            out[cleanKey] = values;
+    }
+    return out;
+}
+function followUpRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+function inferRequestedShapeGrain(dimensions, timeTerms) {
+    if (dimensions.length > 0)
+        return dimensions[0];
+    const time = timeTerms.find((term) => /\b(hour|day|week|month|quarter|year|season|period|date)\b/i.test(term));
+    return time ? canonicalShapeTerm(time) : undefined;
+}
+function canonicalShapeTerm(value) {
+    return normalizeTerm(value).replace(/\s+/g, '_');
+}
 function buildSearchQueries(input) {
     const entityText = input.entities.map((entity) => entity.text).join(' ');
     const profileTerms = input.mode === 'entity_profile' ? PROFILE_WORDS.join(' ') : '';
@@ -599,6 +1027,8 @@ function normalizeSearchText(value) {
 }
 function normalizeTerm(value) {
     const clean = normalizeSearchText(value);
+    if (/^(cusomer|custmer|costomer|client|buyer)s?$/.test(clean))
+        return 'customer';
     if (clean.endsWith('ies') && clean.length > 4)
         return `${clean.slice(0, -3)}y`;
     if (clean.endsWith('s') && clean.length > 4)