@duckcodeailabs/dql-cli 1.6.31 → 1.6.32

package/dist/llm/providers/dql-agent-provider.js

@@ -1,5 +1,8 @@
-import { ClaudeProvider, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, GeminiProvider, OllamaProvider, OpenAIProvider, answer, buildLocalContextPack, loadSkills, reindexProject, } from '@duckcodeailabs/dql-agent';
+import { ClaudeProvider, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, GeminiProvider, loadAgentSemanticLayer, OllamaProvider, OpenAIProvider, answer, buildAnalysisQuestionPlan, buildLocalContextPack, contextRetrievalBudgetForQuestion, loadSkills, reindexProject, } from '@duckcodeailabs/dql-agent';
+import { normalizeDqlArtifactReference } from '@duckcodeailabs/dql-core';
 import { existsSync } from 'node:fs';
+import { buildAnswerLoopTools, createGroundingContextExpander } from '../answer-loop-tools.js';
+import { blockProposalDqlMetadata } from '../proposal-metadata.js';
 import { getEffectiveProviderConfig } from '../../settings/provider-settings.js';
 import { ClaudeCodeCliProvider, CodexCliProvider } from '../../providers/subscription-cli.js';
 import { ClaudeOAuthProvider, claudeOAuthConnected } from '../../providers/oauth/claude-oauth.js';
@@ -67,6 +70,104 @@ const SPECS = {
         },
     },
 };
+function createCertifiedFitConfirmation(provider, signal) {
+    return async ({ question, questionPlan, block, fit }) => {
+        const response = await provider.generate([
+            {
+                role: 'system',
+                content: [
+                    'You are a strict governed analytics routing judge.',
+                    'Decide whether the certified block can directly answer the user question.',
+                    'Allow only when the block covers the requested metric, grain, dimensions, filters, required columns, ranking, and top-N.',
+                    'If the block is merely useful context, close but wrong grain, missing requested columns, or ambiguous, reject it.',
+                    'Return JSON only: {"allow":boolean,"confidence":"high|medium|low","reason":"short reason"}.',
+                ].join(' '),
+            },
+            {
+                role: 'user',
+                content: JSON.stringify({
+                    question,
+                    requestedShape: questionPlan.requestedShape,
+                    candidateBlock: summarizeBlockForFitConfirmation(block),
+                    deterministicFit: {
+                        kind: fit.kind,
+                        confidence: fit.confidence,
+                        reasons: fit.reasons,
+                        missingOutputs: fit.missingOutputs,
+                        missingDimensions: fit.missingDimensions,
+                        unsupportedFilters: fit.unsupportedFilters,
+                        grainMismatch: fit.grainMismatch,
+                        topNAction: fit.topNAction,
+                        inferredContract: fit.inferredContract,
+                    },
+                }, null, 2),
+            },
+        ], { maxTokens: 220, temperature: 0, signal });
+        return parseCertifiedFitConfirmation(response);
+    };
+}
+function summarizeBlockForFitConfirmation(block) {
+    const payload = block.payload ?? {};
+    return {
+        objectKey: block.objectKey,
+        objectType: block.objectType,
+        name: block.name,
+        status: block.status,
+        description: block.description ?? stringValue(payload.description),
+        grain: stringValue(payload.grain),
+        dimensions: stringArray(payload.dimensions),
+        entities: stringArray(payload.entities),
+        declaredOutputs: stringArray(payload.declaredOutputs),
+        outputContract: payload.outputContract,
+        allowedFilters: stringArray(payload.allowedFilters),
+        sql: truncateForFitPrompt(stringValue(payload.sql), 1200),
+        llmContext: truncateForFitPrompt(stringValue(payload.llmContext), 800),
+    };
+}
+function parseCertifiedFitConfirmation(text) {
+    const jsonText = extractJsonObject(text);
+    if (!jsonText) {
+        return { allow: false, confidence: 'low', reason: 'fit confirmation did not return JSON' };
+    }
+    try {
+        const parsed = JSON.parse(jsonText);
+        const confidence = parsed.confidence === 'high' || parsed.confidence === 'medium' || parsed.confidence === 'low'
+            ? parsed.confidence
+            : undefined;
+        return {
+            allow: parsed.allow === true,
+            ...(confidence ? { confidence } : {}),
+            reason: typeof parsed.reason === 'string' ? parsed.reason : undefined,
+        };
+    }
+    catch {
+        return { allow: false, confidence: 'low', reason: 'fit confirmation returned malformed JSON' };
+    }
+}
+function extractJsonObject(text) {
+    const trimmed = text.trim();
+    if (trimmed.startsWith('{') && trimmed.endsWith('}'))
+        return trimmed;
+    const fenced = trimmed.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/i);
+    if (fenced?.[1])
+        return fenced[1];
+    const start = trimmed.indexOf('{');
+    const end = trimmed.lastIndexOf('}');
+    return start >= 0 && end > start ? trimmed.slice(start, end + 1) : undefined;
+}
+function stringArray(value) {
+    return Array.isArray(value)
+        ? value.filter((item) => typeof item === 'string' && item.trim().length > 0)
+        : [];
+}
+function stringValue(value) {
+    return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
+}
+function truncateForFitPrompt(value, max) {
+    if (!value)
+        return undefined;
+    return value.length > max ? `${value.slice(0, max - 1)}…` : value;
+}
 function emitProposalFromText(text, emit) {
     const match = text.match(/DQL_BLOCK_PROPOSAL\s*[:=]?\s*(\{[\s\S]*\})\s*$/);
     if (!match)
@@ -84,6 +185,7 @@ function emitProposalFromText(text, emit) {
                 owner: String(raw.owner ?? ''),
                 description: String(raw.description ?? ''),
                 sql: String(raw.sql),
+                ...blockProposalDqlMetadata(raw),
                 tags: Array.isArray(raw.tags) ? raw.tags.map(String) : undefined,
                 chartType: typeof raw.chartType === 'string' ? raw.chartType : undefined,
             },
@@ -111,14 +213,18 @@ export function createDqlAgentProviderRunner(id) {
                     emit({ kind: 'thinking', text: 'Building the local agent knowledge graph from terms, business views, blocks, apps, dashboards, dbt, and semantic metadata.' });
                 }
                 await reindexProject(req.projectRoot, { kgPath });
-                const question = resolveEffectiveQuestion(req);
-                if (!question) {
+                const rawQuestion = resolveEffectiveQuestion(req);
+                if (!rawQuestion) {
                     emit({ kind: 'error', message: 'No user question found.' });
                     return;
                 }
                 const memory = new MemoryStore(defaultMemoryPath(req.projectRoot));
                 const kg = new KGStore(kgPath);
                 try {
+                    const conversationSnapshot = conversationSnapshotFromContext(req.conversationContext);
+                    const rawFollowUp = followUpFromConversationContext(req, rawQuestion) ?? inferFollowUpContext(req, rawQuestion);
+                    const followUp = applyTopicShiftGuard(rawFollowUp, conversationSnapshot);
+                    const question = rewriteFollowUpQuestion(rawQuestion, followUp);
                     // Retrieve durable learnings only — notebook/project/user/artifact scope.
                     // `thread` (per-conversation) memory is intentionally excluded: it is
                     // raw-chat residue, not a governed learning, and bloats the prompt.
@@ -132,13 +238,26 @@ export function createDqlAgentProviderRunner(id) {
                         ? await req.getSchemaContext(question).catch(() => [])
                         : [];
                     const skills = loadSkills(req.projectRoot).skills;
-                    const followUp = followUpFromConversationContext(req, question) ?? inferFollowUpContext(req, question);
+                    const semanticLayer = loadAgentSemanticLayer(req.projectRoot);
+                    const questionPlan = buildAnalysisQuestionPlan(question, followUp);
+                    const contextBudget = contextRetrievalBudgetForQuestion({
+                        questionPlan,
+                        requestedDepth: req.analysisDepth,
+                        reasoningEffort: req.reasoningEffort,
+                    });
                     const selectedContext = selectedContextForMetadata(req, question);
                     const contextPack = await buildLocalContextPack(req.projectRoot, {
                         question,
                         surface: 'notebook',
                         followUp,
                         selectedContext,
+                        strictness: contextBudget.strictness,
+                        limit: contextBudget.limit,
+                        confirmCertifiedFit: createCertifiedFitConfirmation(provider, signal),
+                        // Conversation-aware reuse: same-topic follow-ups seed (or, for
+                        // filter-only refinements, re-stamp) the prior turn's context pack.
+                        priorContextPackId: priorContextPackIdFromSnapshot(conversationSnapshot),
+                        conversationTopicRelation: conversationSnapshot?.topicRelation,
                         runtimeSchemaSnapshot: schemaContext.length > 0
                             ? {
                                 source: 'agent provider schema context',
@@ -169,13 +288,20 @@ export function createDqlAgentProviderRunner(id) {
                         skills,
                         blockHints,
                         followUp,
+                        conversationSnapshot,
                         memoryContext,
                         schemaContext,
+                        semanticLayer,
                         contextPack,
                         signal,
                         reasoningEffort: req.reasoningEffort,
+                        analysisDepth: contextBudget.analysisDepth,
+                        ...(req.semanticDriver ? { semanticDriver: req.semanticDriver } : {}),
+                        ...(req.semanticTableMapping ? { semanticTableMapping: req.semanticTableMapping } : {}),
                         executeCertifiedBlock: req.executeCertifiedBlock,
                         executeGeneratedSql: req.executeGeneratedSql,
+                        expandGroundingContext: createGroundingContextExpander(req.projectRoot),
+                        answerLoopTools: buildAnswerLoopTools(req.projectRoot),
                         // NOTE: no captureGeneratedDraft here — a plain answer/research question must NOT
                         // auto-write a draft into the blocks space. A draft is created only when the user
                         // explicitly acts (the "Create DQL draft" action → the dql_block_draft route).
@@ -258,6 +384,61 @@ export function resolveEffectiveQuestion(req) {
         return current;
     return `${original} — clarification: ${current}`;
 }
+export function rewriteFollowUpQuestion(question, followUp) {
+    if (!followUp || followUp.kind === 'contextual')
+        return question;
+    const pieces = [
+        `Follow-up request: ${question}`,
+        followUp.sourceQuestion ? `Prior question: ${followUp.sourceQuestion}` : '',
+        followUp.sourceBlockName ? `Prior certified block: ${followUp.sourceBlockName}` : '',
+        followUp.priorResultRef ? formatPriorResultRefForQuestion(followUp.priorResultRef) : '',
+        followUp.priorDqlArtifact ? formatPriorDqlArtifactForQuestion(followUp.priorDqlArtifact) : '',
+        followUp.priorResultValues ? `Resolved prior result values: ${formatResultDimensionValues(followUp.priorResultValues)}` : '',
+        followUp.filters?.length ? `Apply referenced filters: ${formatFollowUpFilters(followUp)}` : '',
+        followUp.dimensions?.length ? `Referenced dimensions: ${followUp.dimensions.join(', ')}` : '',
+        followUp.priorMeasures?.length ? `Prior measures: ${followUp.priorMeasures.join(', ')}` : '',
+    ].filter(Boolean);
+    return pieces.length > 1 ? pieces.join('\n') : question;
+}
+function formatPriorResultRefForQuestion(ref) {
+    const parts = [
+        `Prior result ref: result:${ref.id}`,
+        ref.columns.length ? `schema=[${ref.columns.slice(0, 24).join(', ')}]` : '',
+        typeof ref.rowCount === 'number' ? `row_count=${ref.rowCount}` : '',
+        ref.sourceSql ? `source_sql=${compactInline(ref.sourceSql, 500)}` : '',
+    ].filter(Boolean);
+    return parts.join('; ');
+}
+function formatPriorDqlArtifactForQuestion(artifact) {
+    const parts = [
+        `Prior DQL artifact: kind=${artifact.kind}`,
+        artifact.name ? `name=${artifact.name}` : '',
+        artifact.sourcePath ? `path=${artifact.sourcePath}` : '',
+        artifact.metrics?.length ? `metrics=[${artifact.metrics.slice(0, 12).join(', ')}]` : '',
+        artifact.dimensions?.length ? `dimensions=[${artifact.dimensions.slice(0, 12).join(', ')}]` : '',
+        artifact.filters?.length ? `filters=[${artifact.filters.slice(0, 8).map(formatDqlArtifactFilterInline).join('; ')}]` : '',
+        artifact.timeDimension ? `time=${artifact.timeDimension.name}/${artifact.timeDimension.granularity}` : '',
+        artifact.orderBy?.length ? `order_by=[${artifact.orderBy.slice(0, 8).map((order) => `${order.name} ${order.direction}`).join(', ')}]` : '',
+        typeof artifact.limit === 'number' ? `limit=${artifact.limit}` : '',
+        artifact.source ? `source=${compactInline(artifact.source, 900)}` : '',
+    ].filter(Boolean);
+    return parts.join('; ');
+}
+function formatDqlArtifactFilterInline(filter) {
+    return `${filter.dimension} ${filter.operator} ${filter.values.slice(0, 8).join(', ')}`;
+}
+function formatFollowUpFilters(followUp) {
+    if (!followUp.filters?.length)
+        return '';
+    if (followUp.dimensions?.length === 1) {
+        return `${followUp.dimensions[0]} in [${followUp.filters.join(', ')}]`;
+    }
+    return followUp.filters.join(', ');
+}
+function compactInline(value, max) {
+    const compact = value.replace(/\s+/g, ' ').trim();
+    return compact.length > max ? `${compact.slice(0, max - 3).trimEnd()}...` : compact;
+}
 function renderExtraContext(req, followUp) {
     const parts = [];
     const upstream = req.upstream?.sql?.trim();
@@ -268,7 +449,10 @@ function renderExtraContext(req, followUp) {
         parts.push(`${label}:\n${upstream}`);
     }
     const context = req.conversationContext;
-    if (context) {
+    // Thread-scoped runs render the structured conversation snapshot in its own
+    // prompt section (answer-loop) — skip this text recap to avoid double-carrying.
+    const hasServerSnapshot = Boolean(context?.serverSnapshot);
+    if (context && !hasServerSnapshot) {
         // Bound the carried-forward "conversation memory" defensively: only the most
         // recent turn's signals, with hard caps on the summary length and list sizes
         // so prompts don't grow across a long multi-turn chat.
@@ -282,9 +466,12 @@ function renderExtraContext(req, followUp) {
             context.trustLabel ? `trust label: ${context.trustLabel}` : '',
             context.reviewStatus ? `review status: ${context.reviewStatus}` : '',
             context.draftBlockPath ? `draft block: ${context.draftBlockPath}` : '',
+            context.dqlArtifact ? `prior DQL artifact:\n${formatPriorDqlArtifactForQuestion(context.dqlArtifact)}` : '',
             context.requestedFilters?.length ? `remembered filters: ${clampList(context.requestedFilters)}` : '',
             context.requestedDimensions?.length ? `remembered dimensions: ${clampList(context.requestedDimensions)}` : '',
             context.outputColumns?.length ? `prior output columns: ${clampList(context.outputColumns)}` : '',
+            context.resultDimensionValues ? `prior result values: ${formatResultDimensionValues(context.resultDimensionValues)}` : '',
+            context.turns?.length ? `recent analytical turns:\n${formatConversationTurnsForPrompt(context.turns)}` : '',
         ].filter(Boolean);
         if (contextLines.length > 0) {
             parts.push(`Conversation memory:\n${contextLines.join('\n')}`);
@@ -293,8 +480,13 @@ function renderExtraContext(req, followUp) {
     if (followUp?.sourceBlockName) {
         const suffix = followUp.kind === 'drilldown'
             ? 'Use it as source context, but prefer a distinct certified drilldown block or a review-required draft.'
-            : 'Reuse it for this generic follow-up.';
-        parts.push(`Follow-up context: the user is referring to certified block "${followUp.sourceBlockName}". ${suffix}`);
+            : followUp.kind === 'contextual'
+                ? 'This is advisory prior-turn context — use it only if the question refers to it; on a new topic, ignore it.'
+                : 'Reuse it for this generic follow-up.';
+        const lead = followUp.kind === 'contextual'
+            ? `Prior turn used certified block "${followUp.sourceBlockName}".`
+            : `Follow-up context: the user is referring to certified block "${followUp.sourceBlockName}".`;
+        parts.push(`${lead} ${suffix}`);
     }
     if (followUp?.filters?.length) {
         parts.push(`Requested follow-up filters: ${followUp.filters.join(', ')}`);
@@ -302,6 +494,12 @@ function renderExtraContext(req, followUp) {
     if (followUp?.dimensions?.length) {
         parts.push(`Requested follow-up dimensions: ${followUp.dimensions.join(', ')}`);
     }
+    if (followUp?.priorResultRef) {
+        parts.push(`Prior result reference:\n${formatPriorResultRefForQuestion(followUp.priorResultRef)}`);
+    }
+    if (followUp?.priorDqlArtifact) {
+        parts.push(`Prior DQL artifact reference:\n${formatPriorDqlArtifactForQuestion(followUp.priorDqlArtifact)}`);
+    }
     return parts.length > 0 ? parts.join('\n\n') : undefined;
 }
 function selectedContextForMetadata(req, question) {
@@ -365,9 +563,9 @@ function extractSelectedBlockHints(req) {
     }
 }
 function inferFollowUpContext(req, question) {
-    const kind = isGenericFollowUp(question) ? 'generic' : isDrilldownFollowUp(question) ? 'drilldown' : null;
-    if (!kind)
-        return undefined;
+    // Messages-only fallback (no structured conversationContext). Regexes classify the
+    // kind; a non-matching question still carries the prior turn as advisory 'contextual'.
+    const kind = isGenericFollowUp(question) ? 'generic' : isDrilldownFollowUp(question) ? 'drilldown' : 'contextual';
     for (let i = req.messages.length - 1; i >= 0; i--) {
         const msg = req.messages[i];
         if (msg.role !== 'assistant')
@@ -385,38 +583,335 @@ function inferFollowUpContext(req, question) {
     }
     return undefined;
 }
+function priorContextPackIdFromSnapshot(snapshot) {
+    const fromState = snapshot?.workingState?.lastContextPackId;
+    if (typeof fromState === 'string' && fromState.trim())
+        return fromState;
+    const fromTurns = snapshot?.recentTurns?.length
+        ? snapshot.recentTurns[snapshot.recentTurns.length - 1]?.contextPackId
+        : undefined;
+    return typeof fromTurns === 'string' && fromTurns.trim() ? fromTurns : undefined;
+}
+/** Parse the server-attached conversation snapshot (thread-scoped runs only). */
+function conversationSnapshotFromContext(context) {
+    const raw = context?.serverSnapshot;
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw))
+        return undefined;
+    const snapshot = raw;
+    return typeof snapshot.threadId === 'string' && Array.isArray(snapshot.recentTurns)
+        ? snapshot
+        : undefined;
+}
+/**
+ * Deterministic stale-context protection: when the persisted working state says
+ * the new question is a topic SHIFT, prior-turn filters must not be forced into
+ * the follow-up (a "by X" phrasing can regex-classify as drilldown even on a
+ * genuinely new topic). Question-derived filters are kept; carried ones drop.
+ */
+function applyTopicShiftGuard(followUp, snapshot) {
+    if (!followUp || snapshot?.topicRelation !== 'shift')
+        return followUp;
+    if (followUp.kind !== 'drilldown')
+        return followUp;
+    return {
+        ...followUp,
+        kind: 'contextual',
+        filters: undefined,
+        dimensions: undefined,
+        priorResultValues: undefined,
+        priorMeasures: undefined,
+        priorLimit: undefined,
+    };
+}
 function followUpFromConversationContext(req, question) {
     const context = req.conversationContext;
     if (!context)
         return undefined;
-    const sourceBlockName = cleanOptionalString(context.sourceCertifiedBlock);
-    if (!sourceBlockName)
+    const turns = conversationTurnsFromContext(context);
+    const activeTurn = activeConversationTurn(context, turns, question);
+    const activeResult = activeTurn?.result && typeof activeTurn.result === 'object' && !Array.isArray(activeTurn.result)
+        ? activeTurn.result
+        : undefined;
+    const sourceBlockName = cleanOptionalString(activeTurn?.sourceCertifiedBlock) ?? cleanOptionalString(context.sourceCertifiedBlock);
+    const priorResultValues = cleanStringRecordArray(activeResult?.dimensionValues) ?? cleanStringRecordArray(context.resultDimensionValues);
+    const priorResultColumns = mergeStrings(arrayValue(activeResult?.columns), context.resultColumns, context.outputColumns);
+    const priorResultRef = priorResultRefFromTurn(activeTurn, activeResult, priorResultColumns);
+    const priorDqlArtifact = cleanDqlArtifactReference(activeTurn?.dqlArtifact) ?? cleanDqlArtifactReference(context.dqlArtifact);
+    const resolvedReferences = resolveConversationReferences(question, turns, priorResultValues);
+    const hasUsefulContext = Boolean(sourceBlockName || priorResultColumns?.length || priorResultValues || priorDqlArtifact);
+    if (!hasUsefulContext)
         return undefined;
     const inferredKind = isGenericFollowUp(question)
         ? 'generic'
         : isDrilldownFollowUp(question)
             ? 'drilldown'
             : null;
-    const kind = inferredKind ?? (context.followupKind === 'generic' || context.followupKind === 'drilldown' ? context.followupKind : null);
-    if (!kind)
-        return undefined;
+    // Always-on carry: the regexes only CLASSIFY the follow-up kind — they no longer
+    // gate whether conversation context exists at all. A question that matches neither
+    // pattern still carries the prior turn as advisory 'contextual' state; the model
+    // (not a regex) decides whether it's relevant to the new question.
+    const kind = inferredKind
+        ?? (context.followupKind === 'generic' || context.followupKind === 'drilldown' ? context.followupKind : null)
+        ?? 'contextual';
     return {
         kind,
+        sourceTurnId: cleanOptionalString(activeTurn?.id) ?? cleanOptionalString(context.sourceAnswerId),
         sourceBlockName,
-        sourceQuestion: cleanOptionalString(context.sourceQuestion),
-        sourceAnswer: cleanOptionalString(context.sourceAnswerSummary),
+        sourceQuestion: cleanOptionalString(activeTurn?.question) ?? cleanOptionalString(context.sourceQuestion),
+        sourceAnswer: cleanOptionalString(activeTurn?.answerSummary) ?? cleanOptionalString(context.sourceAnswerSummary),
         filters: kind === 'drilldown'
-            ? mergeStrings(context.requestedFilters, extractDrilldownFilters(question))
+            ? mergeStrings(activeTurnStringArray(activeTurn, 'requestedFilters'), context.requestedFilters, extractDrilldownFilters(question), resolvedReferences.filters)
             : undefined,
         dimensions: kind === 'drilldown'
-            ? mergeStrings(context.requestedDimensions, extractDrilldownDimensions(question))
+            ? mergeStrings(activeTurnStringArray(activeTurn, 'requestedDimensions'), context.requestedDimensions, extractDrilldownDimensions(question), resolvedReferences.dimensions)
             : undefined,
+        priorResultColumns,
+        priorResultValues,
+        priorResultRef,
+        priorDqlArtifact,
+        priorLimit: activeTurnNumber(activeTurn, 'topN') ?? (typeof context.priorLimit === 'number' ? context.priorLimit : undefined),
+        priorMeasures: mergeStrings(activeTurnStringArray(activeTurn, 'requestedMeasures'), arrayValue(activeResult?.measureColumns), context.priorMeasures, inferredMeasuresFromAnswerContract(context.answerContract), inferredMeasureColumns(priorResultColumns)),
+        resolvedReferences: resolvedReferences.labels,
+        unresolvedReferences: resolvedReferences.unresolved,
+    };
+}
+function conversationTurnsFromContext(context) {
+    const explicit = Array.isArray(context?.turns)
+        ? context.turns.map(cleanRecord).filter((turn) => Boolean(turn))
+        : [];
+    const snapshot = conversationSnapshotFromContext(context);
+    const snapshotTurns = [
+        ...(snapshot?.recalledTurns ?? []).map((turn) => snapshotTurnToConversationRecord(turn, 'recalled')),
+        ...(snapshot?.recentTurns ?? []).map((turn) => snapshotTurnToConversationRecord(turn, 'recent')),
+    ].filter((turn) => Boolean(turn));
+    const merged = mergeConversationTurnRecords([...snapshotTurns, ...explicit]);
+    if (merged.length > 0)
+        return merged.slice(-12);
+    const legacy = cleanRecord({
+        id: context?.sourceAnswerId,
+        question: context?.sourceQuestion,
+        answerSummary: context?.sourceAnswerSummary,
+        sourceCertifiedBlock: context?.sourceCertifiedBlock,
+        requestedFilters: context?.requestedFilters,
+        requestedDimensions: context?.requestedDimensions,
+        requestedMeasures: context?.priorMeasures,
+        topN: context?.priorLimit,
+        result: {
+            columns: context?.resultColumns ?? context?.outputColumns,
+            rowsSample: context?.resultRowsSample,
+            dimensionValues: context?.resultDimensionValues,
+            measureColumns: context?.priorMeasures,
+        },
+        route: context?.route,
+        trustLabel: context?.trustLabel,
+        reviewStatus: context?.reviewStatus,
+        certification: context?.certification,
+        contextPackId: context?.contextPackId,
+        sourceSql: context?.sourceSql,
+        dqlArtifact: context?.dqlArtifact,
+        cascade: context?.cascade,
+    });
+    return legacy ? [legacy] : [];
+}
+function snapshotTurnToConversationRecord(turn, snapshotSource) {
+    return compactConversationRecord({
+        id: turn.id,
+        question: turn.question,
+        answerSummary: turn.answerSummary,
+        sourceCertifiedBlock: turn.sourceCertifiedBlock,
+        route: turn.route,
+        contextPackId: turn.contextPackId,
+        sourceSql: turn.sourceSql,
+        dqlArtifact: turn.dqlArtifact,
+        cascade: turn.cascade,
+        snapshotSource,
+        result: compactConversationRecord({
+            columns: turn.resultColumns,
+            dimensionValues: turn.resultDimensionValues,
+            rowCount: turn.resultRowCount,
+        }),
+    });
+}
+function mergeConversationTurnRecords(turns) {
+    const byId = new Map();
+    const anonymous = [];
+    for (const turn of turns) {
+        const id = cleanOptionalString(turn.id);
+        if (!id) {
+            anonymous.push(turn);
+            continue;
+        }
+        const current = byId.get(id);
+        byId.set(id, current ? { ...current, ...turn } : turn);
+    }
+    return [...anonymous, ...byId.values()];
+}
+function activeConversationTurn(context, turns, question) {
+    if (turns.length === 0)
+        return undefined;
+    const activeId = cleanOptionalString(context?.activeTurnId);
+    const activeMatch = activeId
+        ? turns.find((turn) => cleanOptionalString(turn.id) === activeId)
+        : undefined;
+    const recalled = turns.filter((turn) => cleanOptionalString(turn.snapshotSource) === 'recalled' && turnHasUsefulResult(turn));
+    if (recalled.length > 0) {
+        const bestRecalled = recalled
+            .map((turn) => ({ turn, score: scoreTurnForQuestion(turn, question) }))
+            .sort((a, b) => b.score - a.score)[0];
+        const activeScore = activeMatch ? scoreTurnForQuestion(activeMatch, question) : 0;
+        if (bestRecalled && shouldPreferRecalledPriorResult(question, bestRecalled.score, activeScore)) {
+            return bestRecalled.turn;
+        }
+    }
+    if (activeId) {
+        if (activeMatch)
+            return activeMatch;
+    }
+    return [...turns].reverse().find((turn) => {
+        return turnHasUsefulResult(turn);
+    }) ?? turns[turns.length - 1];
+}
+function turnHasUsefulResult(turn) {
+    const result = cleanRecord(turn.result);
+    const columns = arrayValue(result?.columns);
+    const rows = arrayValue(result?.rowsSample);
+    const values = cleanStringRecordArray(result?.dimensionValues);
+    return Boolean(columns?.length || rows?.length || values);
+}
+function scoreTurnForQuestion(turn, question) {
+    const queryTokens = tokenSet(question);
+    if (queryTokens.size === 0)
+        return 0;
+    const result = cleanRecord(turn.result);
+    const values = cleanStringRecordArray(result?.dimensionValues);
+    const text = [
+        cleanOptionalString(turn.question),
+        cleanOptionalString(turn.answerSummary),
+        ...(arrayValue(result?.columns) ?? []).map((value) => cleanOptionalString(value)).filter((value) => Boolean(value)),
+        ...Object.entries(values ?? {}).flatMap(([key, list]) => [key, ...list.slice(0, 8)]),
+    ].filter(Boolean).join(' ');
+    let score = 0;
+    const turnTokens = tokenSet(text);
+    for (const token of queryTokens) {
+        if (turnTokens.has(token))
+            score += 1;
+    }
+    return score;
+}
+function shouldPreferRecalledPriorResult(question, recalledScore, activeScore) {
+    if (recalledScore >= Math.max(activeScore + 2, 2))
+        return true;
+    if (!wantsPriorResultReference(question))
+        return false;
+    return recalledScore > 0 && recalledScore > activeScore;
+}
+function wantsPriorResultReference(question) {
+    return /\b(?:previous|prior|earlier|above)\s+(?:results?|outputs?|rows?|turns?)\b/i.test(question)
+        || /\b(?:with|from|using|include)\s+(?:the\s+)?(?:previous|prior|earlier|above)\b/i.test(question);
+}
+function tokenSet(value) {
+    const tokens = value
+        .toLowerCase()
+        .match(/[a-z0-9_]+/g) ?? [];
+    return new Set(tokens
+        .map((token) => token.replace(/s$/, ''))
+        .filter((token) => token.length > 2 && !GENERIC_FOLLOW_UP_WORDS.has(token)));
+}
+function priorResultRefFromTurn(activeTurn, activeResult, priorResultColumns) {
+    const columns = priorResultColumns?.slice(0, 32) ?? [];
+    if (columns.length === 0)
+        return undefined;
+    const id = cleanOptionalString(activeTurn?.id) ?? 'previous';
+    const rowCount = typeof activeResult?.rowCount === 'number' && Number.isFinite(activeResult.rowCount)
+        ? activeResult.rowCount
+        : Array.isArray(activeResult?.rowsSample)
+            ? activeResult.rowsSample.length
+            : undefined;
+    const sourceSql = cleanOptionalString(activeTurn?.sourceSql);
+    return {
+        id,
+        question: cleanOptionalString(activeTurn?.question),
+        columns,
+        rowCount,
+        sourceSql: sourceSql ? sourceSql.slice(0, 1200) : undefined,
     };
 }
+function resolveConversationReferences(question, turns, activeValues) {
+    const dimensions = resolveDeicticDimensions(question, activeValues) ?? [];
+    let filters = resolveDeicticFilters(question, activeValues) ?? [];
+    const labels = [];
+    if (dimensions.length > 0 && filters.length === 0) {
+        for (const turn of [...turns].reverse()) {
+            const values = cleanStringRecordArray(cleanRecord(turn.result)?.dimensionValues);
+            if (!values)
+                continue;
+            filters = dimensions.flatMap((dimension) => valuesForPriorDimension(values, dimension));
+            if (filters.length > 0)
+                break;
+        }
+    }
+    for (const dimension of dimensions) {
+        const values = (activeValues ? valuesForPriorDimension(activeValues, dimension) : []).slice(0, 5);
+        labels.push(values.length ? `${dimension}: ${values.join(', ')}` : `${dimension}: unresolved`);
+    }
+    const unresolved = referencesNeedValues(question) && filters.length === 0
+        ? ['Could not resolve the referenced prior result values from conversation state.']
+        : undefined;
+    return {
+        filters: filters.length > 0 ? Array.from(new Set(filters)).slice(0, 24) : undefined,
+        dimensions: dimensions.length > 0 ? dimensions : undefined,
+        labels: labels.length > 0 ? labels : undefined,
+        unresolved,
+    };
+}
+function cleanRecord(value) {
+    return value && typeof value === 'object' && !Array.isArray(value)
+        ? value
+        : undefined;
+}
+function compactConversationRecord(record) {
+    const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
+    return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+}
+function cleanDqlArtifactReference(value) {
+    const artifact = normalizeDqlArtifactReference(value);
+    if (!artifact)
+        return undefined;
+    return {
+        ...artifact,
+        source: artifact.source.slice(0, 3000),
+        metrics: uniqueStringList(artifact.metrics),
+        dimensions: uniqueStringList(artifact.dimensions),
+        filters: artifact.filters
+            ?.filter((filter) => filter.values.length > 0)
+            .slice(0, 12)
+            .map((filter) => ({ ...filter, values: uniqueStringList(filter.values)?.slice(0, 12) ?? [] })),
+        orderBy: artifact.orderBy?.slice(0, 12),
+    };
+}
+function uniqueStringList(value) {
+    if (!value?.length)
+        return undefined;
+    const unique = Array.from(new Set(value)).slice(0, 24);
+    return unique.length > 0 ? unique : undefined;
+}
+function activeTurnStringArray(turn, key) {
+    return arrayValue(turn?.[key]);
+}
+function activeTurnNumber(turn, key) {
+    const value = turn?.[key];
+    return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+}
+function referencesNeedValues(question) {
+    return /\b(?:this|that|these|those|same|above|previous|prior)\b/i.test(question);
+}
 function extractCertifiedBlockName(content) {
     const fromAnswer = content.match(/Answered by certified block \*\*([^*]+)\*\*/i)?.[1];
+    const fromRoute = content.match(/Answered from certified block\s+([A-Za-z0-9_.-]+)/i)?.[1];
     const fromCitation = content.match(/^- block:\s*([A-Za-z0-9_.-]+)/im)?.[1];
-    return (fromAnswer ?? fromCitation)?.trim();
+    // The name pattern admits dots, so a sentence period lands in the match
+    // ("... block food_vs_drink_revenue.") — strip trailing punctuation.
+    return (fromAnswer ?? fromRoute ?? fromCitation)?.trim().replace(/[.,;:]+$/, '') || undefined;
 }
 function cleanOptionalString(value) {
     return typeof value === 'string' && value.trim() ? value.trim() : undefined;
@@ -429,6 +924,168 @@ function mergeStrings(...groups) {
     const unique = Array.from(new Set(values)).slice(0, 24);
     return unique.length > 0 ? unique : undefined;
 }
+function inferredMeasuresFromAnswerContract(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        return undefined;
+    const record = value;
+    const requestedShape = record.requestedShape && typeof record.requestedShape === 'object' && !Array.isArray(record.requestedShape)
+        ? record.requestedShape
+        : undefined;
+    return mergeStrings(arrayValue(record.measures), arrayValue(requestedShape?.measures));
+}
+function inferredMeasureColumns(columns) {
+    if (!columns?.length)
+        return undefined;
+    const measures = columns.filter((column) => /\b(revenue|sales|amount|total|count|average|avg|sum|spend|cost|margin|profit|value|points?|score|quantity|rate|volume)\b/i.test(column.replace(/_/g, ' ')));
+    return measures.length > 0 ? measures : undefined;
+}
+function arrayValue(value) {
+    return Array.isArray(value) ? value : undefined;
+}
+function cleanStringRecordArray(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        return undefined;
+    const out = {};
+    for (const [key, raw] of Object.entries(value)) {
+        const values = Array.isArray(raw)
+            ? raw.map(cleanOptionalString).filter((item) => Boolean(item)).slice(0, 24)
+            : [];
+        if (key.trim() && values.length > 0)
+            out[key.trim()] = values;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+}
+function resolveDeicticFilters(question, priorValues) {
+    if (!priorValues)
+        return undefined;
+    const hasExplicitPluralReference = /\b(?:these|those|same)\s+(?:customers?|products?|cat(?:egor|agor|ogor)(?:y|ies)|segments?|regions?)\b/i.test(question);
+    if (!hasExplicitPluralReference) {
+        const singular = resolveSingularDeicticDimension(question, priorValues);
+        if (singular) {
+            const values = valuesForPriorDimension(priorValues, singular).slice(0, 1);
+            return values.length > 0 ? values : undefined;
+        }
+    }
+    const dims = resolveDeicticDimensions(question, priorValues) ?? [];
+    const values = dims.flatMap((dim) => valuesForPriorDimension(priorValues, dim));
+    return values.length > 0 ? Array.from(new Set(values)).slice(0, 24) : undefined;
+}
+function resolveDeicticDimensions(question, priorValues) {
+    if (!priorValues)
+        return undefined;
+    const lower = question.toLowerCase();
+    const dims = [];
+    const candidates = [
+        [/\b(?:these|those|the)\s+cat(?:egor|agor|ogor)(?:y|ies)\b/, 'category'],
+        [/\b(?:this|that|these|those|the)\s+products?\b/, 'product'],
+        [/\b(?:this|that|these|those|the)\s+customers?\b/, 'customer'],
+        [/\b(?:above|previous|prior)\s+(?:orders?|results?|rows?)\b/, 'customer'],
+        [/\b(?:this|that|these|those|the)\s+segments?\b/, 'segment'],
+        [/\b(?:this|that|these|those|the)\s+regions?\b/, 'region'],
+    ];
+    for (const [pattern, dim] of candidates) {
+        if (!pattern.test(lower))
+            continue;
+        if (dim === 'product' && /\bthe\s+product\s+cat(?:egor|agor|ogor)(?:y|ies)\b/.test(lower))
+            continue;
+        if (valuesForPriorDimension(priorValues, dim).length)
+            dims.push(dim);
+    }
+    if (dims.length === 0 && /\b(?:these|those|that|them|same|above|previous|prior)\b/.test(lower)) {
+        const single = singlePriorValueDimension(priorValues);
+        if (single)
+            dims.push(single);
+    }
+    return dims.length > 0 ? Array.from(new Set(dims)) : undefined;
+}
+function resolveSingularDeicticDimension(question, priorValues) {
+    const lower = question.toLowerCase();
+    const candidates = [
+        [/\b(?:this|that|the)\s+product\b/, 'product'],
+        [/\b(?:this|that|the)\s+customer\b/, 'customer'],
+        [/\b(?:this|that|the)\s+category\b/, 'category'],
+        [/\b(?:this|that|the)\s+segment\b/, 'segment'],
+        [/\b(?:this|that|the)\s+region\b/, 'region'],
+    ];
+    for (const [pattern, dim] of candidates) {
+        if (pattern.test(lower) && valuesForPriorDimension(priorValues, dim).length)
+            return dim;
+    }
+    return undefined;
+}
+function valuesForPriorDimension(values, dim) {
+    const aliases = [dim, `${dim}_name`];
+    if (dim === 'product')
+        aliases.push('sku');
+    if (dim === 'category')
+        aliases.push('product_type', 'category_name');
+    return Array.from(new Set(aliases.flatMap((alias) => values[alias] ?? []))).filter(Boolean);
+}
+function formatResultDimensionValues(value) {
+    return Object.entries(value)
+        .slice(0, 8)
+        .map(([key, values]) => `${key}=[${values.slice(0, 8).join(', ')}]`)
+        .join('; ');
+}
+function formatConversationTurnsForPrompt(turns) {
+    return turns
+        .map(cleanRecord)
+        .filter((turn) => Boolean(turn))
+        .slice(-4)
+        .map((turn, index) => {
+        const result = cleanRecord(turn.result);
+        const columns = arrayValue(result?.columns)?.map(cleanOptionalString).filter(Boolean).slice(0, 6) ?? [];
+        const values = cleanStringRecordArray(result?.dimensionValues);
+        const valueText = values ? formatResultDimensionValues(values) : '';
+        const cascade = cleanRecord(turn.cascade);
+        const cascadeText = [
+            cleanOptionalString(cascade?.terminalLane),
+            cleanOptionalString(cascade?.routeTier),
+        ].filter(Boolean).join('/');
+        return [
+            `${index + 1}. ${cleanOptionalString(turn.question) ?? 'prior turn'}`,
+            cascadeText ? `cascade=${cascadeText}` : '',
+            cleanOptionalString(turn.answerSummary) ? `summary=${cleanOptionalString(turn.answerSummary)}` : '',
+            columns.length ? `columns=${columns.join(', ')}` : '',
+            valueText ? `values=${valueText}` : '',
+        ].filter(Boolean).join(' | ');
+    })
+        .join('\n');
+}
+function singlePriorValueDimension(priorValues) {
+    const dims = Array.from(new Set(Object.keys(priorValues).map(normalizePriorValueDimension).filter(Boolean)));
+    return dims.length === 1 ? dims[0] : undefined;
+}
+function normalizePriorValueDimension(value) {
+    const lower = value.toLowerCase();
+    if (lower.includes('category'))
+        return 'category';
+    if (lower.includes('product'))
+        return 'product';
+    if (lower.includes('customer'))
+        return 'customer';
+    if (lower.includes('account'))
+        return 'account';
+    if (lower.includes('user'))
+        return 'user';
+    if (lower.includes('region'))
+        return 'region';
+    if (lower.includes('segment'))
+        return 'segment';
+    if (lower.includes('channel'))
+        return 'channel';
+    return lower.replace(/[_\s-]+name$/, '').replace(/[^a-z0-9_ -]+/g, '').trim();
+}
+export const __test__ = {
+    applyTopicShiftGuard,
+    buildAnswerLoopTools,
+    createCertifiedFitConfirmation,
+    followUpFromConversationContext,
+    inferFollowUpContext,
+    formatCascadeOutcome,
+    parseCertifiedFitConfirmation,
+    rewriteFollowUpQuestion,
+};
 const GENERIC_FOLLOW_UP_WORDS = new Set([
     'a', 'about', 'again', 'all', 'also', 'and', 'as', 'be', 'block', 'can', 'could', 'data', 'do',
     'execute', 'for', 'from', 'get', 'give', 'import', 'it', 'its', 'let', 'lets', 'me', 'metrics',
@@ -437,6 +1094,9 @@ const GENERIC_FOLLOW_UP_WORDS = new Set([
 ]);
 function isGenericFollowUp(question) {
     const lower = question.toLowerCase();
+    if (/\b(?:combine|merge|join|final|summari[sz]e)\b/.test(lower) && /\b(?:previous|prior|above|these|those|results?|outputs?|turns?)\b/.test(lower)) {
+        return true;
+    }
     if (!/\b(block|data|execute|import|it|result|results|run|solution|that|this)\b/.test(lower))
         return false;
     const tokens = lower.match(/[a-z0-9_]+/g) ?? [];
@@ -445,8 +1105,9 @@ function isGenericFollowUp(question) {
 }
 function isDrilldownFollowUp(question) {
     const lower = question.toLowerCase();
-    return /\b(drill|break\s*down|slice|segment|filter|compare|split|why|changed?|change|driver|root cause|increase|decrease|drop|spike|variance|by|for|only|where|last week|this week|last month|this month|enterprise|region|customer|channel|product)\b/.test(lower)
-        && !/\b(what is|what are|define|definition|meaning of)\b/.test(lower);
+    const deicticDrilldown = /\b(?:this|that|these|those|same|above|previous|prior)\s+(?:orders?|results?|rows?|customers?|products?|cat(?:egor|agor|ogor)(?:y|ies)|segments?|regions?)\b/.test(lower);
+    return /\b(drill|break\s*down|slice|segment|filter|compare|split|why|changed?|change|driver|root cause|increase|decrease|drop|spike|variance|by|for|only|where|last week|this week|last month|this month|enterprise|region|customer|channel|product|category|categories|catagor(?:y|ies)|catogor(?:y|ies))\b/.test(lower)
+        && (deicticDrilldown || !/\b(what is|what are|define|definition|meaning of)\b/.test(lower));
 }
 function extractDrilldownFilters(question) {
     const filters = [];
@@ -476,7 +1137,7 @@ function extractDrilldownDimensions(question) {
         if (value)
             dims.push(value);
     }
-    for (const dim of ['segment', 'region', 'customer', 'channel', 'product', 'week', 'month']) {
+    for (const dim of ['segment', 'region', 'customer', 'channel', 'product', 'category', 'week', 'month']) {
         if (new RegExp(`\\b${dim}\\b`, 'i').test(question))
             dims.push(dim);
     }
@@ -493,9 +1154,58 @@ function formatAgentAnswer(result) {
     const citations = result.citations.length > 0
         ? '\n\nCitations:\n' + result.citations.map((c) => `- ${c.kind}: ${c.name}${c.provenance ? ` (${c.provenance})` : ''}`).join('\n')
         : '';
+    const cascade = formatCascadeOutcome(result.cascade);
+    const cascadeLine = cascade ? `\nCascade: ${cascade}` : '';
     const resultPreview = formatResultPreview(result.result);
-    const sql = result.proposedSql ? `\n\nProposed SQL:\n\`\`\`sql\n${result.proposedSql}\n\`\`\`` : '';
-    return `[${badge}]\n\n${result.text}${resultPreview}${citations}${sql}`;
+    const dql = result.dqlArtifact?.source?.trim()
+        ? `\n\nDQL Artifact (${result.dqlArtifact.kind}):\n\`\`\`dql\n${result.dqlArtifact.source.trim()}\n\`\`\``
+        : '';
+    const sql = result.proposedSql ? `\n\nCompiled SQL preview:\n\`\`\`sql\n${result.proposedSql}\n\`\`\`` : '';
+    return `[${badge}]${cascadeLine}\n\n${result.text}${resultPreview}${citations}${dql}${sql}`;
+}
+function formatCascadeOutcome(cascade) {
+    if (!cascade?.terminalLane && !cascade?.routeTier)
+        return undefined;
+    const lane = formatCascadeLane(cascade.terminalLane);
+    const tier = formatCascadeTier(cascade.routeTier);
+    return [lane, tier].filter(Boolean).join(' · ') || undefined;
+}
+function formatCascadeLane(value) {
+    switch (value) {
+        case 'certified':
+            return 'Lane 1 certified';
+        case 'semantic':
+            return 'Lane 2 semantic';
+        case 'generated':
+            return 'Lane 3 generated';
+        case 'refusal':
+            return 'Lane 4 refusal';
+        default:
+            return value ? formatLabel(value) : undefined;
+    }
+}
+function formatCascadeTier(value) {
+    switch (value) {
+        case 'certified_block':
+            return 'Certified block';
+        case 'semantic_metric':
+            return 'Semantic metric';
+        case 'generated_sql':
+            return 'Generated SQL';
+        case 'business_context':
+            return 'Business context';
+        case 'no_answer':
+            return 'No answer';
+        default:
+            return value ? formatLabel(value) : undefined;
+    }
+}
+function formatLabel(value) {
+    return value
+        .replace(/[_-]+/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .replace(/\b\w/g, (char) => char.toUpperCase());
 }
 function formatResultPreview(result) {
     if (!result)
@@ -542,13 +1252,34 @@ function escapeMarkdownTable(value) {
 }
 function emitDraftProposal(result, question, emit) {
     const isDrilldown = result.evidence?.route.some((step) => step.tool === 'propose_drilldown' && step.status === 'checked') ?? false;
+    const dqlArtifact = result.dqlArtifact;
+    const semanticArtifact = dqlArtifact?.kind === 'semantic_block' ? dqlArtifact : undefined;
     const proposal = {
         name: slugify(question).slice(0, 56) || 'ai_generated_analysis',
         domain: inferProposalDomain(result) ?? '',
         owner: `${process.env.USER ?? 'analyst'}@local`,
         description: result.text.slice(0, 240),
         sql: result.proposedSql,
-        tags: ['ai-generated', 'needs-review', result.sourceTier ?? 'dbt_manifest', ...(isDrilldown ? ['drilldown'] : [])],
+        blockType: semanticArtifact ? 'semantic' : 'custom',
+        ...(dqlArtifact
+            ? {
+                dqlSource: dqlArtifact.source,
+            }
+            : {}),
+        ...(semanticArtifact
+            ? {
+                metrics: semanticArtifact.metrics,
+                dimensions: semanticArtifact.dimensions,
+                ...(semanticArtifact.filters ? { filters: semanticArtifact.filters } : {}),
+                ...(semanticArtifact.timeDimension ? { timeDimension: semanticArtifact.timeDimension } : {}),
+            }
+            : {}),
+        tags: [
+            'ai-generated',
+            'needs-review',
+            semanticArtifact ? 'semantic' : result.sourceTier ?? 'dbt_manifest',
+            ...(isDrilldown ? ['drilldown'] : []),
+        ],
         chartType: result.suggestedViz,
     };
     emit({