@duckcodeailabs/dql-cli 1.14.0 → 1.14.2

package/dist/commands/agent.js

@@ -21,6 +21,8 @@
  *   dql agent feedback <up|down> --block <id> --question "..."
  *     Records feedback into the KG. Used by clients without MCP access.
  */
+import { answerFromRuntimeRun, driveViaRuntime, projectRuntimeRun, } from './agent-eval-runtime.js';
+import { CassetteStore, cassetteEvidenceSummary, cassetteDirFor, evalCassetteCanonicalizationV2, withCassette, } from './agent-eval-cassette.js';
 import { existsSync, readFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { load as loadYaml } from 'js-yaml';
@@ -551,7 +553,14 @@ async function runEval(rest, flags) {
     if (!existsSync(kgPath))
         await reindexProject(projectRoot, { kgPath });
     const providerName = flags.provider;
-    const provider = await pickProvider(providerName);
+    const rawProvider = await pickProvider(providerName);
+    // Cassettes make a provider-backed suite repeatable and free to re-run. They
+    // apply to the in-process driver here; `--via runtime` needs the SERVER
+    // started with DQL_EVAL_CASSETTE_DIR, since it owns its own provider.
+    const cassetteMode = flags.cassette;
+    const provider = cassetteMode === 'record' || cassetteMode === 'replay'
+        ? withCassette(rawProvider, new CassetteStore(cassetteDirFor(projectRoot, rest[0] ?? 'agent-evals')), cassetteMode, evalCassetteCanonicalizationV2(projectRoot))
+        : rawProvider;
     const reasoningEffort = cliReasoningEffort(flags);
     const requestedDepth = cliAnalysisDepth(flags);
     const kg = new KGStore(kgPath);
@@ -562,10 +571,22 @@ async function runEval(rest, flags) {
     // gracefully when no provider is available so offline eval stays deterministic.
     const judge = Boolean(flags.judge);
     const judgeComplete = async ({ system, user }) => provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], {});
+    // Which half of the stack is under test. `loop` preserves today's behaviour;
+    // `runtime` is the one that exercises routing and gates end to end.
+    const via = flags.via === 'runtime' ? 'runtime' : 'loop';
     const runtimeBase = flags.runtimeUrl
         ?? flags.runtime
         ?? process.env.DQL_RUNTIME_URL
         ?? 'http://127.0.0.1:3474';
+    if (via === 'runtime') {
+        // Fail fast and loudly. Without this, an unreachable server turns every case
+        // into a transport error and the report reads as a false-refusal spike that
+        // no code change caused.
+        const probe = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/health`).catch(() => null);
+        if (!probe?.ok) {
+            throw new Error(`--via runtime needs a running server at ${runtimeBase}. Start one with \`dql serve\`, or use --via loop to score the answer loop in-process.`);
+        }
+    }
     const semanticLayer = loadAgentSemanticLayer(projectRoot);
     const expandGroundingContext = createGroundingContextExpander(projectRoot);
     const answerLoopTools = buildAnswerLoopTools(projectRoot);
@@ -605,35 +626,71 @@ async function runEval(rest, flags) {
                     }
                     : undefined,
             }).catch(() => undefined);
-            const result = await answer({
-                question: testCase.question,
-                domain: testCase.domain,
-                domainContext: testCase.domain && manifest
-                    ? resolveDomainContextEnvelope({ manifest, activeDomain: testCase.domain, source: 'explicit_api' })
-                    : undefined,
-                provider,
-                kg,
-                manifest: manifest ?? undefined,
-                skills,
-                memoryContext,
-                followUp: testCase.followUp,
-                semanticLayer,
-                schemaContext,
-                contextPack,
-                reasoningEffort,
-                analysisDepth: contextBudget.analysisDepth,
-                expandGroundingContext,
-                answerLoopTools,
-                executeCertifiedBlock: execute && manifest
-                    ? createCertifiedBlockExecutor(projectRoot, manifest, runtimeBase)
-                    : undefined,
-                executeGeneratedSql: execute
-                    ? createGeneratedSqlExecutor(runtimeBase)
-                    : undefined,
-                captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack: draftContextPack, sourceBlock, sourceDqlArtifact, dqlArtifact, proposedEntity, requestedFilters, requestedDimensions, validationWarnings, outputs }) => {
-                    const slug = deriveGeneratedDraftSlug(draftQuestion);
-                    const proposedDomain = sourceBlock?.domain ?? draftContextPack?.objects.find((object) => object.domain)?.domain ?? testCase.domain ?? 'misc';
-                    if (dqlArtifact?.kind === 'semantic_block') {
+            // `--via runtime` posts to a running `dql serve` so the case exercises the
+            // router, engine, plan boundary, and gates. The in-process driver below
+            // calls the answer loop directly and cannot observe any of them, which is
+            // why a refusal metric taken from it reads cleaner than users experience.
+            const runtimeRun = via === 'runtime'
+                ? await driveViaRuntime({ runtimeBase, question: testCase.question })
+                : undefined;
+            // Runtime mode is scored from the persisted AgentRun. The transport
+            // adapter intentionally has no AgentAnswer.contextPack, so borrowing the
+            // local preflight pack here would fabricate retrieval/route evidence for a
+            // different execution path.
+            const runtimeProjection = runtimeRun ? projectRuntimeRun(runtimeRun) : undefined;
+            const result = runtimeRun
+                ? answerFromRuntimeRun(runtimeRun)
+                : await answer({
+                    question: testCase.question,
+                    domain: testCase.domain,
+                    domainContext: testCase.domain && manifest
+                        ? resolveDomainContextEnvelope({ manifest, activeDomain: testCase.domain, source: 'explicit_api' })
+                        : undefined,
+                    provider,
+                    kg,
+                    manifest: manifest ?? undefined,
+                    skills,
+                    memoryContext,
+                    followUp: testCase.followUp,
+                    semanticLayer,
+                    schemaContext,
+                    contextPack,
+                    reasoningEffort,
+                    analysisDepth: contextBudget.analysisDepth,
+                    expandGroundingContext,
+                    answerLoopTools,
+                    executeCertifiedBlock: execute && manifest
+                        ? createCertifiedBlockExecutor(projectRoot, manifest, runtimeBase)
+                        : undefined,
+                    executeGeneratedSql: execute
+                        ? createGeneratedSqlExecutor(runtimeBase)
+                        : undefined,
+                    captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack: draftContextPack, sourceBlock, sourceDqlArtifact, dqlArtifact, proposedEntity, requestedFilters, requestedDimensions, validationWarnings, outputs }) => {
+                        const slug = deriveGeneratedDraftSlug(draftQuestion);
+                        const proposedDomain = sourceBlock?.domain ?? draftContextPack?.objects.find((object) => object.domain)?.domain ?? testCase.domain ?? 'misc';
+                        if (dqlArtifact?.kind === 'semantic_block') {
+                            if (!flags.save) {
+                                return {
+                                    path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
+                                    askedTimes: 0,
+                                    proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                                };
+                            }
+                            return upsertGeneratedDqlArtifactDraft(projectRoot, {
+                                slug,
+                                question: draftQuestion,
+                                proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                                proposedDomain,
+                                dqlArtifact,
+                                sourceQuestion: followUp?.sourceQuestion,
+                                sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
+                                followupKind: followUp?.kind,
+                                outputs,
+                                contextPackId: draftContextPack?.id,
+                                routeIntent: String(intent),
+                                validationWarnings,
+                            });
+                        }
                         if (!flags.save) {
                             return {
                                 path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
@@ -641,51 +698,30 @@ async function runEval(rest, flags) {
                                 proposedContractId: `${proposedDomain}.Unknown.${slug}`,
                             };
                         }
-                        return upsertGeneratedDqlArtifactDraft(projectRoot, {
+                        return upsertGeneratedDraft(projectRoot, {
                             slug,
                             question: draftQuestion,
+                            proposedSql: sql,
                             proposedContractId: `${proposedDomain}.Unknown.${slug}`,
                             proposedDomain,
-                            dqlArtifact,
+                            proposedEntity,
+                            sourceDqlArtifact,
                             sourceQuestion: followUp?.sourceQuestion,
                             sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
                             followupKind: followUp?.kind,
+                            requestedFilters,
+                            requestedDimensions,
                             outputs,
                             contextPackId: draftContextPack?.id,
                             routeIntent: String(intent),
                             validationWarnings,
                         });
-                    }
-                    if (!flags.save) {
-                        return {
-                            path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
-                            askedTimes: 0,
-                            proposedContractId: `${proposedDomain}.Unknown.${slug}`,
-                        };
-                    }
-                    return upsertGeneratedDraft(projectRoot, {
-                        slug,
-                        question: draftQuestion,
-                        proposedSql: sql,
-                        proposedContractId: `${proposedDomain}.Unknown.${slug}`,
-                        proposedDomain,
-                        proposedEntity,
-                        sourceDqlArtifact,
-                        sourceQuestion: followUp?.sourceQuestion,
-                        sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
-                        followupKind: followUp?.kind,
-                        requestedFilters,
-                        requestedDimensions,
-                        outputs,
-                        contextPackId: draftContextPack?.id,
-                        routeIntent: String(intent),
-                        validationWarnings,
-                    });
-                },
-            });
-            const evaluation = evaluateCase(testCase, result);
+                    },
+                });
+            const evaluation = evaluateCase(testCase, result, runtimeProjection);
             const durationMs = Date.now() - startedAt;
             const draftSaved = Boolean(result.draftBlock?.path ?? result.draftBlockId);
+            const narration = narrationOutcomeForEval(runtimeRun?.narrationIntegrityReceipt);
             const judgeVerdict = judge
                 ? await judgeAnswer({
                     question: testCase.question,
@@ -700,17 +736,33 @@ async function runEval(rest, flags) {
                 passed: evaluation.failures.length === 0,
                 failures: evaluation.failures,
                 durationMs,
+                // The persisted receipt is the only evidence for this metric. Rows and
+                // reader prose are intentionally ignored: a row-bearing answer can be
+                // skipped, while a deterministic fallback can render different wording.
+                narrationAttempted: narration.narrationAttempted,
+                narrationFallback: narration.narrationFallback,
                 executionMs: result.result?.executionTime,
                 executionMatched: evaluation.executionMatched,
                 ...(judgeVerdict ? { judgeScore: judgeVerdict.score, judgePass: judgeVerdict.pass } : {}),
                 kind: result.kind,
-                route: result.contextPack?.routeDecision.route,
-                intent: result.contextPack?.routeDecision.intent,
+                route: runtimeProjection?.route ?? result.contextPack?.routeDecision.route,
+                // Only the runtime driver can see the router's clarification options.
+                // In-process runs leave this undefined, so a clarify there scores as a
+                // dead end — the conservative reading, and another reason `--via runtime`
+                // is the truthful one.
+                ...(runtimeRun ? {
+                    clarificationOptionCount: runtimeRun.clarificationOptions?.length ?? 0,
+                    conversational: runtimeRun.route === 'conversation' || runtimeRun.answerKind === 'conversational',
+                    conversationalAnswer: (runtimeRun.route === 'conversation' || runtimeRun.answerKind === 'conversational')
+                        && Boolean(runtimeRun.answer?.trim()),
+                    meaningResolved: Boolean(runtimeRun.routeDecision?.meaningResolution),
+                } : {}),
+                intent: runtimeRun?.routeDecision?.category ?? result.contextPack?.routeDecision.intent,
                 reviewStatus: result.reviewStatus,
-                contextObjects: result.contextPack?.objects.length ?? 0,
+                contextObjects: runtimeProjection?.retrievalCandidateCount ?? result.contextPack?.objects.length,
                 followUp: Boolean(testCase.followUp),
                 draftSaved,
-                toolCalls: result.evidence?.toolCalls?.length ?? 0,
+                toolCalls: runtimeProjection?.toolCallCount ?? result.evidence?.toolCalls?.length ?? 0,
                 expected: testCase.expected,
                 validationCode: evaluation.validationCode,
                 trace: buildEvalTrace({
@@ -719,6 +771,7 @@ async function runEval(rest, flags) {
                     evaluation,
                     durationMs,
                     draftSaved,
+                    runtime: runtimeProjection,
                 }),
             });
         }
@@ -729,16 +782,39 @@ async function runEval(rest, flags) {
     }
     const passed = results.filter((r) => r.passed).length;
     const metrics = computeEvalMetrics(results);
+    // Runtime evals execute in a separate host, so its cassette directory is
+    // supplied explicitly by the eval workflow for reporting. The summary does
+    // not inspect prompts or provider credentials; it only classifies the
+    // checked-in response provenance.
+    const cassetteDirectory = via === 'runtime'
+        ? process.env.DQL_EVAL_CASSETTE_DIR
+        : cassetteMode === 'record' || cassetteMode === 'replay'
+            ? cassetteDirFor(projectRoot, rest[0] ?? 'agent-evals')
+            : undefined;
+    const cassetteEvidence = cassetteDirectory
+        ? cassetteEvidenceSummary(new CassetteStore(cassetteDirectory))
+        : undefined;
     const thresholds = {
         minToolRequirement: flags.minToolRequirement ?? null,
         minExecutionMatch: flags.minExecutionMatch ?? null,
         minJudgePass: flags.minJudgePass ?? null,
         maxWrongCertified: flags.maxWrongCertified ?? null,
+        maxFalseRefusal: flags.maxFalseRefusal ?? null,
+        minRefusalRecall: flags.minRefusalRecall ?? null,
+        minGroundedNarration: flags.minGroundedNarration ?? null,
     };
     const thresholdsPassed = agentEvalThresholdsPass(metrics, thresholds);
     const ok = passed === results.length && thresholdsPassed;
     if (flags.format === 'json') {
-        console.log(JSON.stringify({ ok, passed, total: results.length, thresholds, metrics, results }, null, 2));
+        console.log(JSON.stringify({
+            ok,
+            passed,
+            total: results.length,
+            thresholds,
+            metrics,
+            ...(cassetteEvidence ? { cassetteEvidence } : {}),
+            results,
+        }, null, 2));
         if (!ok)
             process.exitCode = 1;
         return;
@@ -752,6 +828,22 @@ async function runEval(rest, flags) {
     console.log(`Certified hit rate: ${formatRate(metrics.certified_hit_rate)}`);
     console.log(`Generated follow-up pass rate: ${formatRate(metrics.generated_followup_pass_rate)}`);
     console.log(`Safe refusal rate: ${formatRate(metrics.safe_refusal_rate)}`);
+    if (cassetteEvidence) {
+        console.log(`Cassette replay entries: ${cassetteEvidence.totalEntries} `
+            + `(${cassetteEvidence.migratedLegacyDeterministicFixtureEntries} migrated legacy deterministic fixture, `
+            + `${cassetteEvidence.syntheticDeterministicOrchestrationFixtureEntries} synthetic deterministic orchestration fixture).`);
+        console.log(cassetteEvidence.realProviderQualityEligible
+            ? 'Real-provider quality evidence: eligible.'
+            : `Real-provider quality evidence: excluded (${cassetteEvidence.realProviderQualityExclusionReasons.join(', ')}).`);
+    }
+    console.log(`False refusal rate: ${formatRate(metrics.false_refusal_rate)} (${metrics.false_refusal_count}/${metrics.answerable_case_count} answerable cases refused)`);
+    console.log(`Clarification rate: ${formatRate(metrics.clarification_rate)} (answerable cases asked instead of answered)`);
+    if (metrics.meaning_resolved_rate !== null && metrics.meaning_resolved_rate < 1) {
+        console.log(`  ! Semantic judgment ran for only ${formatRate(metrics.meaning_resolved_rate)} of cases. `
+            + 'Without a reachable provider DQL will not settle a reading by lexical rank (AGT-017), so ambiguous '
+            + 'questions clarify by design — treat the clarification rate above as an artifact, not a product signal.');
+    }
+    console.log(`Refusal recall: ${formatRate(metrics.refusal_recall)} (${metrics.refusal_required_case_count} case(s) that must refuse)`);
     console.log(`Execution match rate: ${formatRate(metrics.execution_match_rate)}`);
     console.log(`Tool requirement pass rate: ${formatRate(metrics.tool_requirement_pass_rate)}`);
     console.log(`Tool-observed case count: ${metrics.tool_observed_case_count}`);
@@ -767,6 +859,15 @@ async function runEval(rest, flags) {
     if (thresholds.minJudgePass !== null) {
         console.log(`Judge-pass threshold: ${thresholds.minJudgePass} (actual ${formatRate(metrics.judge_pass_rate)})`);
     }
+    if (thresholds.maxFalseRefusal !== null) {
+        console.log(`False-refusal ceiling: ${thresholds.maxFalseRefusal} (actual ${formatRate(metrics.false_refusal_rate)})`);
+    }
+    if (thresholds.minRefusalRecall !== null) {
+        console.log(`Refusal-recall threshold: ${thresholds.minRefusalRecall} (actual ${formatRate(metrics.refusal_recall)})`);
+    }
+    if (thresholds.minGroundedNarration !== null && thresholds.minGroundedNarration !== undefined) {
+        console.log(`Grounded-narration threshold: ${thresholds.minGroundedNarration} (actual ${formatRate(metrics.grounded_narration_rate)} over ${metrics.grounded_narration_attempted} attempted)`);
+    }
     if (thresholds.maxWrongCertified !== null) {
         console.log(`Wrong-certified ceiling: ${thresholds.maxWrongCertified} (actual ${metrics.wrong_certified_count})`);
     }
@@ -784,13 +885,39 @@ function previewGeneratedDraftPath(projectRoot, domain, slug) {
     }
     return `blocks/_drafts/${slug}.dql`;
 }
-function evaluateCase(testCase, result) {
+function evaluateCase(testCase, result, runtime) {
     const expected = testCase.expected;
     if (!expected)
         return { failures: [] };
     const failures = [];
     let validationCode;
     let executionMatched;
+    // Answerability is asserted per case as well as aggregated into
+    // false_refusal_rate, so a single dead-end fails its own case instead of only
+    // nudging a rate someone has to notice.
+    const answerable = evalCaseIsAnswerable(expected);
+    // Three distinct outcomes, not two. An option-bearing clarification neither
+    // answers nor dead-ends: it must not fail an answerable case (its cost is
+    // tracked by `clarification_rate`), and it must not fail a must-refuse case
+    // either, because it did not assert anything about the data.
+    const clarifiedWithOptions = (result.clarificationOptions?.length ?? 0) > 0;
+    // A conversational reply ("I'm here to help you explore your data…") asserts
+    // nothing about the warehouse. For an out-of-scope question that is the CORRECT
+    // outcome — declining politely — so it must not be scored as an answer.
+    const conversational = result.answerKind === 'conversational';
+    const answerText = typeof result.text === 'string' ? result.text.trim() : '';
+    // Conversational replies split two ways, and the case's own expectation says
+    // which is right: for an answerable question a substantive reply IS the answer
+    // (a definition), and for an out-of-scope one it is the correct decline.
+    const conversationalAnswer = conversational && answerText.length > 0;
+    const producedDataAnswer = result.kind !== 'no_answer' && !conversational;
+    const deadEnded = !producedDataAnswer && !clarifiedWithOptions && !conversationalAnswer;
+    if (answerable === true && deadEnded) {
+        failures.push(`FALSE REFUSAL: this question is answerable, but the run dead-ended with no answer and no options${result.refusalCode ? ` (${result.refusalCode})` : ''}`);
+    }
+    if (answerable === false && producedDataAnswer) {
+        failures.push(`expected a refusal (question is out of scope / unanswerable), but the run answered with kind ${result.kind}`);
+    }
     if (expected.kind && result.kind !== expected.kind)
         failures.push(`kind expected ${expected.kind}, got ${result.kind}`);
     if (expected.sourceTier && result.sourceTier !== expected.sourceTier)
@@ -799,12 +926,27 @@ function evaluateCase(testCase, result) {
         failures.push(`certification expected ${expected.certification}, got ${result.certification}`);
     if (expected.reviewStatus && result.reviewStatus !== expected.reviewStatus)
         failures.push(`reviewStatus expected ${expected.reviewStatus}, got ${result.reviewStatus}`);
-    if (expected.route && result.contextPack?.routeDecision.route !== expected.route)
-        failures.push(`route expected ${expected.route}, got ${result.contextPack?.routeDecision.route ?? 'none'}`);
+    const observedRoute = runtime?.route ?? result.contextPack?.routeDecision.route;
+    if (expected.route && observedRoute !== expected.route)
+        failures.push(`route expected ${expected.route}, got ${observedRoute ?? 'none'}`);
     if (expected.intent && result.contextPack?.routeDecision.intent !== expected.intent)
         failures.push(`intent expected ${expected.intent}, got ${result.contextPack?.routeDecision.intent ?? 'none'}`);
-    if (expected.missingContextKind && !result.contextPack?.missingContext.some((item) => item.kind === expected.missingContextKind))
+    // `modeling_gap` is a broad terminal kind. A relationship expectation is
+    // satisfied only by the router's persisted relationship-specific witness;
+    // otherwise a missing metric/dimension tuple would be misreported as a
+    // relationship repair opportunity.
+    const runtimeReportsMissingContext = expected.missingContextKind === 'relationship'
+        ? runtime?.terminalOutcome?.gap?.code === 'MISSING_RELATIONSHIP'
+        : runtime?.terminalOutcome?.kind === 'modeling_gap'
+            && expected.missingContextKind === 'modeling_gap';
+    if (expected.missingContextKind
+        && !runtimeReportsMissingContext
+        && !result.contextPack?.missingContext.some((item) => item.kind === expected.missingContextKind)) {
         failures.push(`missing context kind ${expected.missingContextKind} was not reported`);
+    }
+    if (expected.terminalOutcomeKind && runtime?.terminalOutcome?.kind !== expected.terminalOutcomeKind) {
+        failures.push(`terminal outcome expected ${expected.terminalOutcomeKind}, got ${runtime?.terminalOutcome?.kind ?? 'none'}`);
+    }
     for (const token of stringList(expected.sqlContains)) {
         if (!result.proposedSql?.toLowerCase().includes(token.toLowerCase()))
             failures.push(`SQL did not contain "${token}"`);
@@ -825,7 +967,7 @@ function evaluateCase(testCase, result) {
             failures.push(`draftSaved expected ${expected.draftSaved}, got ${saved}`);
     }
     if (typeof expected.minToolCalls === 'number') {
-        const actualToolCalls = result.evidence?.toolCalls?.length ?? 0;
+        const actualToolCalls = runtime?.toolCallCount ?? result.evidence?.toolCalls?.length ?? 0;
         if (actualToolCalls < expected.minToolCalls) {
             failures.push(`toolCalls expected at least ${expected.minToolCalls}, got ${actualToolCalls}`);
         }
@@ -849,7 +991,75 @@ function evaluateCase(testCase, result) {
     }
     return { failures, validationCode, executionMatched };
 }
+/**
+ * Is the case answerable? Explicit `expected.answerable` wins; otherwise infer
+ * from the expectations already present, so the metric covers legacy case files.
+ * A case with no expectations at all is excluded — it asserts nothing, so it can
+ * neither prove nor disprove a false refusal.
+ */
+export function evalCaseIsAnswerable(expected) {
+    if (!expected)
+        return undefined;
+    if (expected.answerable !== undefined)
+        return expected.answerable;
+    if (expected.kind === 'no_answer')
+        return false;
+    if (expected.sourceTier === 'no_answer')
+        return false;
+    if (expected.route === 'clarify' || expected.route === 'blocked')
+        return false;
+    if (Object.keys(expected).length === 0)
+        return undefined;
+    return true;
+}
+/**
+ * Did the run leave the user with NO way forward?
+ *
+ * Deliberately narrower than "did not answer". A clarification that offers
+ * selectable options is answerable on the next turn — worth minimising, tracked
+ * separately as `clarification_rate`, but not the defect. A clarification with
+ * ZERO options is a true dead end: the reported production loop was exactly
+ * this, and a free-text reply to it reproduced the same question forever.
+ */
+export function evalResultRefused(result) {
+    // A substantive conversational reply is an ANSWER, not a dead end. A governed
+    // definition ("**top_customers** — Top 10 customers by lifetime spend…") is
+    // exactly what a "what does X mean?" turn should return, and scoring it as a
+    // refusal would report the feature working as the feature failing.
+    if (result.conversationalAnswer)
+        return false;
+    // Order matters: the drivers collapse every clarification to `no_answer`
+    // (it is not an answer), so the option check has to run FIRST or an
+    // option-bearing clarification is miscounted as a dead end.
+    if (result.route === 'clarify' && (result.clarificationOptionCount ?? 0) > 0)
+        return false;
+    return result.kind === 'no_answer' || result.route === 'clarify';
+}
+/** Did the run ask an answerable clarification rather than answering outright? */
+export function evalResultClarified(result) {
+    return result.route === 'clarify' && (result.clarificationOptionCount ?? 0) > 0;
+}
+/**
+ * Translate only the durable narration receipt into evaluation fields.
+ *
+ * Reader prose, row count, and result shape are deliberately absent: a skipped
+ * narration can have rows, and a deterministic fallback can use any wording.
+ */
+export function narrationOutcomeForEval(receipt) {
+    if (receipt?.mode !== 'verified_facts' || !receipt.attempted)
+        return {};
+    return {
+        // A durable receipt is the only source of this metric.  An infrastructure
+        // error started a verified narration but did not produce a grounded answer,
+        // so it belongs in the denominator just like the deterministic floor.  The
+        // old mapping treated it as a success because only fallback was negative.
+        narrationAttempted: true,
+        narrationFallback: receipt.outcome !== 'success',
+    };
+}
 function computeEvalMetrics(results) {
+    const answerableCases = results.filter((result) => evalCaseIsAnswerable(result.expected) === true);
+    const refusalRequiredCases = results.filter((result) => evalCaseIsAnswerable(result.expected) === false);
     const certifiedCases = results.filter((result) => result.expected?.kind === 'certified' ||
         result.expected?.certification === 'certified' ||
         result.expected?.route === 'certified');
@@ -874,10 +1084,61 @@ function computeEvalMetrics(results) {
         wrong_certified_count: results.filter((result) => result.kind === 'certified' &&
             (result.expected?.kind ? result.expected.kind !== 'certified' : result.followUp)).length,
         outside_context_rejection_count: results.filter((result) => result.validationCode === 'unknown_relation' || result.validationCode === 'unknown_column').length,
+        /**
+         * THE headline number: how often an answerable question was refused.
+         * Bounds every other quality metric — a run that refuses cannot be wrong,
+         * so a falling false-refusal rate must be read together with
+         * `execution_match_rate` to be sure refusals were replaced by CORRECT answers.
+         */
+        false_refusal_rate: ratio(answerableCases.filter(evalResultRefused).length, answerableCases.length),
+        false_refusal_count: answerableCases.filter(evalResultRefused).length,
+        answerable_case_count: answerableCases.length,
+        /**
+         * Answerable cases that asked an option-bearing clarification instead of
+         * answering. Not a defect, but a direct cost in turns — read it next to
+         * false_refusal_rate so a fall in refusals is not just a rise in questions.
+         */
+        clarification_rate: ratio(answerableCases.filter(evalResultClarified).length, answerableCases.length),
+        /**
+         * Cases where semantic judgment ran. Without a provider `mayAssumeInterpretation`
+         * is false (AGT-017), so every ambiguous question clarifies by design and
+         * `clarification_rate` says nothing about product quality.
+         */
+        meaning_resolved_rate: ratio(results.filter((result) => result.meaningResolved === true).length, results.length),
+        /**
+         * Latency, which the acceptance matrix asked for and nothing measured. A
+         * quality gain paid for entirely in wall clock is not a gain: the plan's
+         * two-tier target is certified/semantic under 5s while research takes
+         * minutes, and only a per-class p95 can tell those apart from a regression.
+         */
+        latency_p50_ms: percentileMs(results, 0.5),
+        latency_p95_ms: percentileMs(results, 0.95),
+        latency_p95_answerable_ms: percentileMs(answerableCases, 0.95),
+        /**
+         * How often verified narration survived. When the drafted narration fails
+         * its fact check the reader gets the deterministic record under a
+         * disclaimer — correct, but visibly worse. A silent fall here is exactly
+         * the truncation defect that shipped unnoticed, so it is measured.
+         */
+        grounded_narration_rate: ratio(results.filter((result) => result.narrationAttempted && result.narrationFallback === false).length, results.filter((result) => result.narrationAttempted).length),
+        grounded_narration_attempted: results.filter((result) => result.narrationAttempted).length,
+        /**
+         * The guard on the above: cases that must NOT produce a data answer.
+         * Scored on "did not answer" rather than "dead-ended", because declining via
+         * a clarification is still declining — what would be wrong is asserting
+         * something about data the project does not have.
+         */
+        refusal_recall: ratio(refusalRequiredCases.filter((result) => result.kind === 'no_answer' || result.conversational === true).length, refusalRequiredCases.length),
+        refusal_required_case_count: refusalRequiredCases.length,
         draft_saved_count: results.filter((result) => result.draftSaved).length,
         tool_observed_case_count: results.filter((result) => result.toolCalls > 0).length,
         avg_tool_calls: average(toolCallCounts),
-        avg_context_objects: average(results.map((result) => result.contextObjects)),
+        // Runtime runs only report this when the persisted router recorded an
+        // explicit retrieval count. Treat absent evidence as unknown, not as an
+        // invented empty context pack.
+        avg_context_objects: average(results
+            .map((result) => result.contextObjects)
+            .filter((count) => typeof count === 'number')),
         avg_execution_ms: executionTimes.length ? average(executionTimes) : null,
     };
 }
@@ -885,20 +1146,27 @@ function agentEvalThresholdsPass(metrics, thresholds) {
     // A rate threshold with no applicable cases (metric === null) is vacuously
     // satisfied — you only fail when the metric exists and falls below the bar.
     const rateOk = (metric, min) => min === null || min === undefined || metric === null || metric >= min;
-    return rateOk(metrics.tool_requirement_pass_rate, thresholds.minToolRequirement)
+    // A ceiling is only meaningful when the metric has data; `null` means no
+    // answerable case was scored, which is "unknown", not "perfect".
+    const ceilingOk = (metric, max) => max === null || max === undefined || metric === null || metric <= max;
+    return rateOk(metrics.grounded_narration_rate, thresholds.minGroundedNarration)
+        && rateOk(metrics.tool_requirement_pass_rate, thresholds.minToolRequirement)
         && rateOk(metrics.execution_match_rate, thresholds.minExecutionMatch)
         && rateOk(metrics.judge_pass_rate, thresholds.minJudgePass)
+        && ceilingOk(metrics.false_refusal_rate, thresholds.maxFalseRefusal)
+        && rateOk(metrics.refusal_recall, thresholds.minRefusalRecall)
         && (thresholds.maxWrongCertified === null
             || thresholds.maxWrongCertified === undefined
             || metrics.wrong_certified_count <= thresholds.maxWrongCertified);
 }
 function buildEvalTrace(input) {
-    const { testCase, result, evaluation, durationMs, draftSaved } = input;
+    const { testCase, result, evaluation, durationMs, draftSaved, runtime } = input;
     const routeDecision = result.contextPack?.routeDecision;
     const selectedRelations = result.contextPack?.retrievalDiagnostics.selectedRelations ?? [];
     const allowedRelations = result.contextPack?.allowedSqlContext?.relations ?? [];
     const followUp = testCase.followUp;
     const toolCalls = result.evidence?.toolCalls ?? [];
+    const observedToolCallCount = runtime?.toolCallCount ?? toolCalls.length;
     const routeEvidence = result.evidence?.route ?? [];
     const executionStatus = result.executionError
         ? 'failed'
@@ -914,33 +1182,44 @@ function buildEvalTrace(input) {
     const rowsExpected = testCase.expected?.rows !== undefined;
     const expectedMinToolCalls = testCase.expected?.minToolCalls;
     const toolStatus = typeof expectedMinToolCalls === 'number'
-        ? toolCalls.length >= expectedMinToolCalls ? 'passed' : 'failed'
-        : toolCalls.length > 0 ? 'passed' : routeEvidence.length > 0 ? 'info' : 'not_run';
+        ? observedToolCallCount >= expectedMinToolCalls ? 'passed' : 'failed'
+        : observedToolCallCount > 0 ? 'passed' : routeEvidence.length > 0 ? 'info' : 'not_run';
     const toolMessage = typeof expectedMinToolCalls === 'number'
-        ? toolCalls.length >= expectedMinToolCalls
-            ? `Observed ${toolCalls.length} provider tool call(s), meeting the minimum of ${expectedMinToolCalls}.`
-            : `Observed ${toolCalls.length} provider tool call(s), below the minimum of ${expectedMinToolCalls}.`
-        : toolCalls.length > 0
-            ? `Observed ${toolCalls.length} provider tool call(s).`
+        ? observedToolCallCount >= expectedMinToolCalls
+            ? `Observed ${observedToolCallCount} provider tool call(s), meeting the minimum of ${expectedMinToolCalls}.`
+            : `Observed ${observedToolCallCount} provider tool call(s), below the minimum of ${expectedMinToolCalls}.`
+        : observedToolCallCount > 0
+            ? `Observed ${observedToolCallCount} provider tool call(s).`
             : routeEvidence.length > 0
                 ? `Captured ${routeEvidence.length} deterministic route evidence step(s).`
                 : 'No provider tool calls were observed for this answer.';
     return [
         {
             stage: 'context',
-            status: result.contextPack ? 'passed' : 'not_run',
-            message: result.contextPack
-                ? `Context pack ${result.contextPack.id} selected ${result.contextPack.objects.length} object(s).`
-                : 'No context pack was attached to the answer.',
-            payload: result.contextPack
+            status: runtime && (runtime.retrievalCandidateCount !== undefined || runtime.sourceCoverage?.length || runtime.terminalOutcome)
+                ? 'passed'
+                : result.contextPack ? 'passed' : 'not_run',
+            message: runtime && (runtime.retrievalCandidateCount !== undefined || runtime.sourceCoverage?.length || runtime.terminalOutcome)
+                ? `Persisted route evidence recorded ${runtime.retrievalCandidateCount ?? 'an unspecified number of'} retrieved candidate(s).`
+                : result.contextPack
+                    ? `Context pack ${result.contextPack.id} selected ${result.contextPack.objects.length} object(s).`
+                    : 'No context pack was attached to the answer.',
+            payload: runtime && (runtime.retrievalCandidateCount !== undefined || runtime.sourceCoverage?.length || runtime.terminalOutcome)
                 ? {
-                    contextPackId: result.contextPack.id,
-                    selectedObjectCount: result.contextPack.objects.length,
-                    allowedRelationCount: allowedRelations.length,
-                    selectedRelations: selectedRelations.slice(0, 12).map((relation) => relation.relation),
-                    missingContext: result.contextPack.missingContext,
+                    evidenceSource: 'persisted_agent_run',
+                    retrievalCandidateCount: runtime.retrievalCandidateCount,
+                    sourceCoverage: runtime.sourceCoverage,
+                    terminalOutcome: runtime.terminalOutcome,
                 }
-                : undefined,
+                : result.contextPack
+                    ? {
+                        contextPackId: result.contextPack.id,
+                        selectedObjectCount: result.contextPack.objects.length,
+                        allowedRelationCount: allowedRelations.length,
+                        selectedRelations: selectedRelations.slice(0, 12).map((relation) => relation.relation),
+                        missingContext: result.contextPack.missingContext,
+                    }
+                    : undefined,
         },
         {
             stage: 'rewrite',
@@ -952,29 +1231,40 @@ function buildEvalTrace(input) {
         },
         {
             stage: 'lane',
-            status: routeDecision ? 'passed' : 'not_run',
-            message: routeDecision
-                ? `Lane ${routeDecision.route} / ${routeDecision.intent}.`
-                : 'No lane decision was attached to the answer.',
-            payload: routeDecision
+            status: runtime ? 'passed' : routeDecision ? 'passed' : 'not_run',
+            message: runtime
+                ? `Persisted engine route ${runtime.runRoute}${runtime.route ? ` evaluated as ${runtime.route}` : ''}.`
+                : routeDecision
+                    ? `Lane ${routeDecision.route} / ${routeDecision.intent}.`
+                    : 'No lane decision was attached to the answer.',
+            payload: runtime
                 ? {
-                    route: routeDecision.route,
-                    intent: routeDecision.intent,
-                    reason: routeDecision.reason,
-                    trustLabel: routeDecision.trustLabel,
-                    reviewStatus: routeDecision.reviewStatus,
-                    exactObjectKey: routeDecision.exactObjectKey,
+                    engineRoute: runtime.runRoute,
+                    evalRoute: runtime.route,
+                    status: runtime.status,
+                    trustState: runtime.trustState,
+                    terminalOutcome: runtime.terminalOutcome,
                 }
-                : undefined,
+                : routeDecision
+                    ? {
+                        route: routeDecision.route,
+                        intent: routeDecision.intent,
+                        reason: routeDecision.reason,
+                        trustLabel: routeDecision.trustLabel,
+                        reviewStatus: routeDecision.reviewStatus,
+                        exactObjectKey: routeDecision.exactObjectKey,
+                    }
+                    : undefined,
         },
         {
             stage: 'tools',
             status: toolStatus,
             message: toolMessage,
             payload: {
-                observedToolCalls: toolCalls.length,
+                observedToolCalls: observedToolCallCount,
                 expectedMinToolCalls,
-                providerToolCalls: toolCalls.slice(0, 12).map((call) => ({
+                ...(runtime ? { evidenceSource: 'persisted_agent_run.telemetry' } : {}),
+                providerToolCalls: runtime ? [] : toolCalls.slice(0, 12).map((call) => ({
                     order: call.order,
                     name: call.name,
                     status: call.status,
@@ -1207,6 +1497,20 @@ export const __test__ = {
     cliAnalysisDepth,
     cliReasoningEffort,
     computeEvalMetrics,
+    narrationOutcomeForEval,
     evaluateCase,
 };
+/** Percentile over observed case durations. Returns null when nothing timed. */
+function percentileMs(results, q) {
+    const observed = results
+        .map((result) => result.durationMs)
+        .filter((value) => typeof value === 'number' && Number.isFinite(value) && value >= 0)
+        .sort((left, right) => left - right);
+    if (observed.length === 0)
+        return null;
+    // Nearest-rank: with a handful of cases an interpolated percentile invents a
+    // duration nothing actually took.
+    const rank = Math.min(observed.length - 1, Math.max(0, Math.ceil(q * observed.length) - 1));
+    return observed[rank] ?? null;
+}
 //# sourceMappingURL=agent.js.map