@duckcodeailabs/dql-agent 1.14.0 → 1.14.1

package/dist/answer-loop.js

@@ -1,3 +1,4 @@
+import { packContext } from './cascade/packer.js';
 /**
  * Dynamic-first governed answer loop.
  *
@@ -26,7 +27,7 @@ import { domainContextSearchDomains } from './domain-context.js';
 import { deriveGeneratedDraftSlug, renderGeneratedSqlDqlArtifact } from './metadata/drafts.js';
 import { buildAnalysisQuestionPlan } from './metadata/analysis-planner.js';
 import { certifiedFitAllowsTier1, certifiedTerminationVerdict, evaluateCertifiedBlockFit } from './metadata/block-fit.js';
-import { buildGovernedMetricFirstSql, matchSemanticMetric, resolveGovernedMetricDefinition, resolveGovernedMetricSql } from './metadata/metric-match.js';
+import { buildGovernedMetricFirstSql, matchSemanticMetric, resolveGovernedMetricDefinition, resolveGovernedMetricSql, semanticMetricEmbeddingOptions } from './metadata/metric-match.js';
 import { decideAgentAction } from './intent-controller.js';
 import { createContextLedger } from './grounding/context-ledger.js';
 import { validateAnswerResultShape } from './answer-shape.js';
@@ -421,7 +422,35 @@ const MANIFEST_KINDS = ['dbt_model', 'dbt_source'];
  * surface shows THIS; the validator's machine message (relation ids, repair
  * tool guidance) belongs in refusalDetails/validationWarnings only.
  */
-function renderContextValidationRefusalForUser(code, machineError, memberBindings) {
+/**
+ * Name the ONE aggregation check that actually fired.
+ *
+ * The refusal used to list all four possible causes — "rounding too early,
+ * losing decimal precision, summing a non-additive value, or multiplying rows
+ * across a join" — because the specific one was never plumbed into the message.
+ * The system knows: `aggregationIntegrityIssuesForSql` classifies each issue and
+ * the proof carries the codes. A menu of four causes reads as "something is
+ * wrong somewhere", which is exactly the report a user cannot act on.
+ */
+function aggregationRefusalReason(issueCodes = []) {
+    const codes = new Set(issueCodes.map((code) => code.toUpperCase()));
+    // Ordered by how badly each distorts the number, so a query tripping several
+    // leads with the one that matters most.
+    if (codes.has('FANOUT')) {
+        return 'joining those tables multiplies rows, so the total would be inflated. Aggregate at the row-level grain first, then join';
+    }
+    if (codes.has('NON_ADDITIVE_MEASURE')) {
+        return 'it sums a measure that is not additive (an average or a ratio cannot be added up). Ask for the governed metric, which carries the right rule';
+    }
+    if (codes.has('PREMATURE_ROUNDING')) {
+        return 'it rounds each value before adding them, which loses money at scale. Rounding belongs on the final total';
+    }
+    if (codes.has('LOSSY_NUMERIC_CAST')) {
+        return 'it converts amounts to floating point before adding them, which drifts on decimal currency';
+    }
+    return undefined;
+}
+export function renderContextValidationRefusalForUser(code, machineError, memberBindings, aggregationIssueCodes) {
     switch (code) {
         case 'unknown_relation':
             return 'I drafted a query, but it uses a table that was not part of the metadata retrieved for this question, so I did not run it. Ask again (retrieval usually finds it on a follow-up), name the table or metric explicitly, or re-sync the dbt metadata.';
@@ -438,8 +467,15 @@ function renderContextValidationRefusalForUser(code, machineError, memberBinding
             return 'This comparison needs a baseline period or value that the drafted query did not include. Say what to compare against (for example, the prior month) and I will run it.';
         case 'unsafe_sql':
             return 'The drafted query used a statement type that is not allowed in this governed preview, so I did not run it.';
-        case 'unsafe_aggregation':
-            return 'I drafted a query, but it could change the metric meaning by rounding too early, losing decimal precision, summing a non-additive value, or multiplying rows across a join. I did not run it. Use the governed semantic metric, or review the native grain and join keys before retrying.';
+        case 'unsafe_aggregation': {
+            const reason = aggregationRefusalReason(aggregationIssueCodes);
+            if (reason) {
+                return `I drafted a query but did not run it, because ${reason}. Ask for the governed metric and I will use the safe version.`;
+            }
+            // No classified code reached us — say that plainly rather than listing
+            // every cause as if one of them were known to apply.
+            return 'I drafted a query, but it would change how the metric is calculated, so I did not run it. Ask for the governed semantic metric and I will use the safe version.';
+        }
         case 'insufficient_context':
             if (/could not be parsed|parse error|syntax/i.test(machineError)) {
                 return 'I drafted a query, but its SQL syntax did not match the connected warehouse, so I did not run it. The failed draft is available in Inspect; retrying will generate a warehouse-specific query.';
@@ -601,7 +637,10 @@ function tryCrossResultAnswer(input) {
     const prior = input.followUp?.priorResult;
     if (!prior || !Array.isArray(prior.rows) || prior.rows.length === 0)
         return null;
-    const op = detectResultSetOperation(input.question);
+    // Pass the prior result's shape so a follow-up that names one of its columns
+    // ("what's the average lifetime_spend?") is recognised without requiring a
+    // demonstrative pronoun.
+    const op = detectResultSetOperation(input.question, prior);
     if (!op)
         return null;
     const computed = computeResultSetOperation(op, prior);
@@ -696,6 +735,10 @@ async function freezeLegacySemanticSelection(input) {
     const match = await matchSemanticMetric(input.question, metricNodes, {
         measureTerms: [...questionPlan.requestedShape.measures, ...questionPlan.metricTerms],
         ...(semanticLayer ? { canExecute: (name) => semanticLayer.canComposeMetric(name) } : {}),
+        ...(snapshotVectorMetricShortlist(input.contextPack).length
+            ? { vectorMetricShortlist: snapshotVectorMetricShortlist(input.contextPack) }
+            : {}),
+        ...semanticMetricEmbeddingOptions(input.embeddingProvider),
     }).catch(() => null);
     if (!match)
         return undefined;
@@ -813,6 +856,26 @@ async function freezeLegacySemanticSelection(input) {
         mode: 'authoritative',
     });
 }
+/**
+ * Reuse the immutable context pack's independent vector lane.  Metric matching
+ * never opens the catalog or runs a second metadata retrieval pass; it only
+ * narrows its vector-only candidate admission to IDs already retrieved for this
+ * request snapshot.
+ */
+function snapshotVectorMetricShortlist(contextPack) {
+    const lane = contextPack?.retrievalDiagnostics.lanes?.find((item) => item.lane === 'vector');
+    if (!lane?.candidates.length || !contextPack)
+        return [];
+    const objects = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
+    const result = [];
+    for (const candidate of lane.candidates) {
+        const object = objects.get(candidate.objectKey);
+        if (!object || (object.objectType !== 'semantic_metric' && object.objectType !== 'semantic_measure'))
+            continue;
+        result.push(candidate.objectKey, object.name);
+    }
+    return [...new Set(result)];
+}
 export async function answer(input) {
     const inheritedPlan = !input.resolvedAnalyticalPlan
         && input.followUp?.priorResolvedAnalyticalPlan
@@ -1543,6 +1606,10 @@ async function runAnswerLoop(input) {
             : await matchSemanticMetric(semanticQuestion, semanticMetricNodes, {
                 measureTerms: [...questionPlan.requestedShape.measures, ...questionPlan.metricTerms],
                 ...(canExecuteSemanticMetricForMatch ? { canExecute: canExecuteSemanticMetricForMatch } : {}),
+                ...(snapshotVectorMetricShortlist(input.contextPack).length
+                    ? { vectorMetricShortlist: snapshotVectorMetricShortlist(input.contextPack) }
+                    : {}),
+                ...semanticMetricEmbeddingOptions(input.embeddingProvider),
             }).catch(() => null);
     // Stage 1: certified artifact match. Blocks can be executed; dashboards,
     // Apps, and notebooks are returned as governed citations/navigation targets.
@@ -2317,7 +2384,18 @@ async function runAnswerLoop(input) {
             onCompiled: (record) => compiledSemanticRecords.push(record),
         }),
     ];
-    if (semanticBridgeAnswer) {
+    if (input.forcedGeneratedProposal) {
+        // The analyst loop already did its provider/tool work. Re-entering ordinary
+        // generation here would create a second proposal whose SQL is not covered
+        // by the request-scoped execution capability.
+        parsed = {
+            sql: input.forcedGeneratedProposal.sql,
+            text: input.forcedGeneratedProposal.summary
+                ?? 'Prepared from identifiers observed during this bounded analyst run. Review-required until an analyst promotes it.',
+            ...(input.forcedGeneratedProposal.suggestedViz ? { viz: input.forcedGeneratedProposal.suggestedViz } : {}),
+        };
+    }
+    else if (semanticBridgeAnswer) {
         governedMetricAnswer = true;
         parsed = {
             sql: semanticBridgeAnswer.sql,
@@ -2503,7 +2581,7 @@ async function runAnswerLoop(input) {
     let deepCandidateResult;
     let deepCandidateExecutionError;
     let deepCandidateNotes = [];
-    if (!governedMetricAnswer && input.analysisDepth === 'deep' && parsed.sql) {
+    if (!input.forcedGeneratedProposal && !governedMetricAnswer && input.analysisDepth === 'deep' && parsed.sql) {
         const selection = await selectDeepGeneratedProposalCandidate({
             provider,
             messages,
@@ -2705,7 +2783,12 @@ async function runAnswerLoop(input) {
                 };
             }
         }
-        const tightenedFlow = tightenSourceTargetFlowProjection(parsed.sql, question, questionPlan);
+        // The analyst capability binds the exact composed bytes. Any projection
+        // tightening is a new SQL proposal and therefore must wait for a fresh
+        // analyst pass/capability rather than silently rewriting this one.
+        const tightenedFlow = input.forcedGeneratedProposal
+            ? undefined
+            : tightenSourceTargetFlowProjection(parsed.sql, question, questionPlan);
         if (tightenedFlow && tightenedFlow.sql !== parsed.sql) {
             parsed.sql = tightenedFlow.sql;
             parsed.outputs = tightenedFlow.outputs;
@@ -2723,7 +2806,7 @@ async function runAnswerLoop(input) {
     // model emitted to its real warehouse relation from the runtime schema BEFORE
     // governance validation. Same resolver the build path uses — one grounding,
     // no weak path. `allowedSqlContext` relations are already qualified.
-    if (parsed.sql) {
+    if (parsed.sql && !input.forcedGeneratedProposal) {
         parsed.sql = contextLedger.qualifySql(parsed.sql).sql;
     }
     // Validation gate. Governed metric SQL synthesized from the semantic layer is
@@ -2840,7 +2923,8 @@ async function runAnswerLoop(input) {
     // that predicate entirely, inject it deterministically before spending the
     // bounded AI validation repair. Misbound or ambiguous predicates are never
     // rewritten here; they continue through the guarded correction/refusal path.
-    if (!contextValidation.ok
+    if (!input.forcedGeneratedProposal
+        && !contextValidation.ok
         && contextValidation.code === 'misbound_filter'
         && !contextValidation.offending?.relation
         && !contextValidation.offending?.column
@@ -2892,7 +2976,8 @@ async function runAnswerLoop(input) {
     //      stands instead of guessing a measure.
     //   2) Otherwise fall back to a CLEAN governed-metric definition (direct or exact
     //      leaf-measure; no fuzzy family guess that could answer the wrong measure).
-    if (!contextValidation.ok && contextValidation.code !== 'unsafe_aggregation' && semanticMetricRoute && semanticMetricMatch && !rankedGrainGap) {
+    if (!input.forcedGeneratedProposal
+        && !contextValidation.ok && contextValidation.code !== 'unsafe_aggregation' && semanticMetricRoute && semanticMetricMatch && !rankedGrainGap) {
         const grounded = contextLedger.validateRuntimeGrounding(parsed.sql);
         // Runtime grounding proves that relations and columns exist; it cannot prove
         // that a resolved business member was bound to the right dimension. Never
@@ -2922,7 +3007,8 @@ async function runAnswerLoop(input) {
     // nothing, or found the table and something else rejected the SQL — which are
     // three different fixes.
     let regroundDiagnostic;
-    if (!contextValidation.ok && contextValidation.code !== 'unsafe_aggregation' && !governedMetricAnswer && input.expandGroundingContext && canUseLaneRepair(repairBudgetState, 'reground')) {
+    if (!input.forcedGeneratedProposal
+        && !contextValidation.ok && contextValidation.code !== 'unsafe_aggregation' && !governedMetricAnswer && input.expandGroundingContext && canUseLaneRepair(repairBudgetState, 'reground')) {
         recordLaneRepair(repairBudgetState, 'reground');
         try {
             const expansion = await input.expandGroundingContext({
@@ -3019,6 +3105,7 @@ async function runAnswerLoop(input) {
         && input.resolvedAnalyticalPlan?.mode !== 'authoritative'
         && !semanticAggregationAuthorityBlocked
         && !governedMetricAnswer
+        && !input.forcedGeneratedProposal
         && canUseLaneRepair(repairBudgetState, 'validation')) {
         recordLaneRepair(repairBudgetState, 'validation');
         try {
@@ -3094,7 +3181,7 @@ async function runAnswerLoop(input) {
         // Business-language chat text; the validator's machine message (with
         // relation ids and tool guidance for the repair prompt) stays in
         // refusalDetails + validationWarnings for the Inspect surface.
-        const text = renderContextValidationRefusalForUser(contextValidation.code, contextValidation.error, input.followUp?.memberBindings);
+        const text = renderContextValidationRefusalForUser(contextValidation.code, contextValidation.error, input.followUp?.memberBindings, contextValidation.aggregationSafetyProof?.issueCodes);
         const analysisPlan = buildAnalysisPlan({
             question,
             intent,
@@ -3283,8 +3370,6 @@ async function runAnswerLoop(input) {
             && semanticBridgeAnswer?.sql.trim() === parsed.sql?.trim()) {
             return input.executeDqlArtifact({ ...semanticArtifact, limit: requestedLimit });
         }
-        if (!input.executeGeneratedSql)
-            throw new Error('No generated SQL executor is configured.');
         const artifact = buildGeneratedSqlDqlArtifact({
             question,
             sql: parsed.sql,
@@ -3300,9 +3385,20 @@ async function runAnswerLoop(input) {
             validationWarnings: [...contextValidation.warnings, ...deepCandidateNotes],
             outputs: parsed.outputs,
         });
-        return input.executeGeneratedSql(parsed.sql, artifact ? { ...artifact, limit: requestedLimit } : undefined);
+        const boundedArtifact = artifact ? { ...artifact, limit: requestedLimit } : undefined;
+        if (input.forcedGeneratedProposal) {
+            if (!input.agenticSqlExecutionCapability || !input.executeAgenticGeneratedSql) {
+                throw Object.assign(new Error('The analyst-approved SQL has no live execution capability, so DQL did not execute it.'), {
+                    code: 'AGENTIC_EXECUTION_CAPABILITY_REQUIRED',
+                });
+            }
+            return input.executeAgenticGeneratedSql(input.agenticSqlExecutionCapability, parsed.sql, boundedArtifact);
+        }
+        if (!input.executeGeneratedSql)
+            throw new Error('No generated SQL executor is configured.');
+        return input.executeGeneratedSql(parsed.sql, boundedArtifact);
     };
-    if (input.executeGeneratedSql && !result) {
+    if ((input.executeGeneratedSql || (input.forcedGeneratedProposal && input.executeAgenticGeneratedSql)) && !result) {
         // Fanout gate for native semantic direct-joins: a duplicate join-key row on
         // the joined side multiplies fact rows BEFORE aggregation, so every summed
         // value inflates silently (seen in the field as trillions-scale "governed"
@@ -3311,6 +3407,8 @@ async function runAnswerLoop(input) {
         // repair, which would regenerate the same multiplying join.
         let fanoutContradiction = false;
         if (!executionError
+            && !input.forcedGeneratedProposal
+            && input.executeGeneratedSql
             && semanticBridgeAnswer?.composeResult?.fanoutProbeSql
             && semanticBridgeAnswer.sql.trim() === parsed.sql?.trim()) {
             const inflationError = await probeSemanticJoinFanout(semanticBridgeAnswer.composeResult.fanoutProbeSql, semanticBridgeAnswer.composeResult.tables, input.executeGeneratedSql);
@@ -3327,7 +3425,7 @@ async function runAnswerLoop(input) {
             warehouseFailure = normalizeWarehouseSqlFailure(err, input.semanticDriver);
             executionError = warehouseFailure.redactedMessage;
         }
-        if (executionError && !fanoutContradiction && !authoritativeSemanticBinding) {
+        if (executionError && !input.forcedGeneratedProposal && !fanoutContradiction && !authoritativeSemanticBinding) {
             warehouseFailure ??= normalizeWarehouseSqlFailure(executionError, input.semanticDriver);
             if (isRetryableGeneratedSqlError(warehouseFailure)) {
                 const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError, schemaContext);
@@ -5251,15 +5349,26 @@ function renderContextPackForPrompt(contextPack, budget) {
     const warnings = contextPack.warnings.length
         ? `Warnings:\n${contextPack.warnings.slice(0, budget.warningLimit).map((warning) => `- ${warning}`).join('\n')}\n`
         : '';
-    const objects = contextPack.objects.slice(0, budget.contextObjectLimit).map((object) => {
-        const detail = [
-            object.objectType,
-            object.domain ? `domain: ${object.domain}` : '',
-            object.status ? `status: ${object.status}` : '',
-            object.description ? `description: ${object.description}` : '',
-        ].filter(Boolean).join('; ');
-        return `- ${object.objectKey} (${detail})`;
-    }).join('\n');
+    // Pack by value per token instead of truncating at a fixed position. The
+    // objects arrive already ranked, so position IS the value — but cost is not
+    // uniform: one object with a long description can crowd out several cheap
+    // ones that rank just below it, and the old `slice` cut by rank alone. This
+    // keeps the same ordering pressure while letting an inexpensive object that
+    // ranked 19th still fit. The budget is derived from the existing item limit
+    // so the window stays the size the caller already tuned for.
+    const objects = packContext(contextPack.objects.map((object, index) => ({
+        id: object.objectKey,
+        score: contextPack.objects.length - index,
+        render: () => {
+            const detail = [
+                object.objectType,
+                object.domain ? `domain: ${object.domain}` : '',
+                object.status ? `status: ${object.status}` : '',
+                object.description ? `description: ${object.description}` : '',
+            ].filter(Boolean).join('; ');
+            return `- ${object.objectKey} (${detail})`;
+        },
+    })), Math.max(1, budget.contextObjectLimit) * 45).text;
     const conflicts = contextPack.retrievalDiagnostics.candidateConflicts.length
         ? `\nCandidate conflicts:\n${contextPack.retrievalDiagnostics.candidateConflicts.slice(0, 4).map((conflict) => `- ${conflict.reason} ${conflict.prompt}`).join('\n')}`
         : '';
@@ -5785,6 +5894,28 @@ function flowFieldCovers(field, term) {
 function normalizeFlowField(value) {
     return value.toLowerCase().replace(/[_-]+/g, ' ').replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
 }
+/**
+ * Remove the model's QUERY PLAN scaffolding from user-facing prose.
+ *
+ * Rule 4 of the SQL prompt tells the model to state its grain, measures, join
+ * path, and join keys BEFORE writing SQL. That instruction is load-bearing —
+ * it is what prevents wrong-grain answers and fan-out joins — but it is
+ * reasoning, not an answer, and it was reaching the reader verbatim:
+ *
+ *   "QUERY PLAN: grain = one row per customer, filtered to the named customer
+ *    (Matthew Meyer, honorific stripped) ... FROM dev.customers c, filtered on
+ *    c.customer_name ILIKE '%Matthew Meyer%'."
+ *
+ * Someone who asked why a customer tops a list should not be handed a join
+ * plan. The plan stays in the artifact and the trace; it leaves the prose.
+ */
+export function stripQueryPlanScaffolding(text) {
+    return text
+        // A QUERY PLAN section runs to the next blank line or the end of the prose.
+        .replace(/(?:^|\n)\s*(?:\*\*)?QUERY PLAN(?:\*\*)?\s*[::][\s\S]*?(?=\n\s*\n|$)/gi, '')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
 /**
  * Public for tests. Prefer the structured W2.7 JSON proposal contract, then
  * fall back to the legacy prose + ```sql block + Viz line format.
@@ -5798,11 +5929,11 @@ export function parseProposal(raw) {
     const vizMatch = raw.match(/^Viz:\s*([a-z_]+)/im);
     const viz = vizMatch ? vizMatch[1].trim().toLowerCase() : undefined;
     // Strip the SQL block + Viz line from the prose to keep the summary clean.
-    const text = raw
+    const text = stripQueryPlanScaffolding(raw
         .replace(/```json[\s\S]*?```/gi, '')
         .replace(/```sql[\s\S]*?```/gi, '')
         .replace(/^Viz:.*$/gim, '')
-        .trim();
+        .trim());
     return { text, sql, viz };
 }
 function parseStructuredProposal(raw) {
@@ -5830,7 +5961,7 @@ function parsedProposalFromJson(value) {
         return undefined;
     const record = value;
     const sql = firstJsonString(record.sql, record.query)?.trim();
-    const text = firstJsonString(record.summary, record.text, record.answer, record.description)?.trim() ?? '';
+    const text = stripQueryPlanScaffolding(firstJsonString(record.summary, record.text, record.answer, record.description)?.trim() ?? '');
     const viz = firstJsonString(record.viz, record.visualization, record.chartType)
         ?.trim()
         .toLowerCase()