@duckcodeailabs/dql-cli 1.14.0 → 1.14.1

package/dist/commands/agent.js

@@ -21,6 +21,8 @@
  *   dql agent feedback <up|down> --block <id> --question "..."
  *     Records feedback into the KG. Used by clients without MCP access.
  */
+import { answerFromRuntimeRun, driveViaRuntime, evalRouteForRun } from './agent-eval-runtime.js';
+import { CassetteStore, cassetteDirFor, withCassette } from './agent-eval-cassette.js';
 import { existsSync, readFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { load as loadYaml } from 'js-yaml';
@@ -551,7 +553,14 @@ async function runEval(rest, flags) {
     if (!existsSync(kgPath))
         await reindexProject(projectRoot, { kgPath });
     const providerName = flags.provider;
-    const provider = await pickProvider(providerName);
+    const rawProvider = await pickProvider(providerName);
+    // Cassettes make a provider-backed suite repeatable and free to re-run. They
+    // apply to the in-process driver here; `--via runtime` needs the SERVER
+    // started with DQL_EVAL_CASSETTE_DIR, since it owns its own provider.
+    const cassetteMode = flags.cassette;
+    const provider = cassetteMode === 'record' || cassetteMode === 'replay'
+        ? withCassette(rawProvider, new CassetteStore(cassetteDirFor(projectRoot, rest[0] ?? 'agent-evals')), cassetteMode)
+        : rawProvider;
     const reasoningEffort = cliReasoningEffort(flags);
     const requestedDepth = cliAnalysisDepth(flags);
     const kg = new KGStore(kgPath);
@@ -562,10 +571,22 @@ async function runEval(rest, flags) {
     // gracefully when no provider is available so offline eval stays deterministic.
     const judge = Boolean(flags.judge);
     const judgeComplete = async ({ system, user }) => provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], {});
+    // Which half of the stack is under test. `loop` preserves today's behaviour;
+    // `runtime` is the one that exercises routing and gates end to end.
+    const via = flags.via === 'runtime' ? 'runtime' : 'loop';
     const runtimeBase = flags.runtimeUrl
         ?? flags.runtime
         ?? process.env.DQL_RUNTIME_URL
         ?? 'http://127.0.0.1:3474';
+    if (via === 'runtime') {
+        // Fail fast and loudly. Without this, an unreachable server turns every case
+        // into a transport error and the report reads as a false-refusal spike that
+        // no code change caused.
+        const probe = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/health`).catch(() => null);
+        if (!probe?.ok) {
+            throw new Error(`--via runtime needs a running server at ${runtimeBase}. Start one with \`dql serve\`, or use --via loop to score the answer loop in-process.`);
+        }
+    }
     const semanticLayer = loadAgentSemanticLayer(projectRoot);
     const expandGroundingContext = createGroundingContextExpander(projectRoot);
     const answerLoopTools = buildAnswerLoopTools(projectRoot);
@@ -605,35 +626,66 @@ async function runEval(rest, flags) {
                     }
                     : undefined,
             }).catch(() => undefined);
-            const result = await answer({
-                question: testCase.question,
-                domain: testCase.domain,
-                domainContext: testCase.domain && manifest
-                    ? resolveDomainContextEnvelope({ manifest, activeDomain: testCase.domain, source: 'explicit_api' })
-                    : undefined,
-                provider,
-                kg,
-                manifest: manifest ?? undefined,
-                skills,
-                memoryContext,
-                followUp: testCase.followUp,
-                semanticLayer,
-                schemaContext,
-                contextPack,
-                reasoningEffort,
-                analysisDepth: contextBudget.analysisDepth,
-                expandGroundingContext,
-                answerLoopTools,
-                executeCertifiedBlock: execute && manifest
-                    ? createCertifiedBlockExecutor(projectRoot, manifest, runtimeBase)
-                    : undefined,
-                executeGeneratedSql: execute
-                    ? createGeneratedSqlExecutor(runtimeBase)
-                    : undefined,
-                captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack: draftContextPack, sourceBlock, sourceDqlArtifact, dqlArtifact, proposedEntity, requestedFilters, requestedDimensions, validationWarnings, outputs }) => {
-                    const slug = deriveGeneratedDraftSlug(draftQuestion);
-                    const proposedDomain = sourceBlock?.domain ?? draftContextPack?.objects.find((object) => object.domain)?.domain ?? testCase.domain ?? 'misc';
-                    if (dqlArtifact?.kind === 'semantic_block') {
+            // `--via runtime` posts to a running `dql serve` so the case exercises the
+            // router, engine, plan boundary, and gates. The in-process driver below
+            // calls the answer loop directly and cannot observe any of them, which is
+            // why a refusal metric taken from it reads cleaner than users experience.
+            const runtimeRun = via === 'runtime'
+                ? await driveViaRuntime({ runtimeBase, question: testCase.question })
+                : undefined;
+            const result = runtimeRun
+                ? answerFromRuntimeRun(runtimeRun)
+                : await answer({
+                    question: testCase.question,
+                    domain: testCase.domain,
+                    domainContext: testCase.domain && manifest
+                        ? resolveDomainContextEnvelope({ manifest, activeDomain: testCase.domain, source: 'explicit_api' })
+                        : undefined,
+                    provider,
+                    kg,
+                    manifest: manifest ?? undefined,
+                    skills,
+                    memoryContext,
+                    followUp: testCase.followUp,
+                    semanticLayer,
+                    schemaContext,
+                    contextPack,
+                    reasoningEffort,
+                    analysisDepth: contextBudget.analysisDepth,
+                    expandGroundingContext,
+                    answerLoopTools,
+                    executeCertifiedBlock: execute && manifest
+                        ? createCertifiedBlockExecutor(projectRoot, manifest, runtimeBase)
+                        : undefined,
+                    executeGeneratedSql: execute
+                        ? createGeneratedSqlExecutor(runtimeBase)
+                        : undefined,
+                    captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack: draftContextPack, sourceBlock, sourceDqlArtifact, dqlArtifact, proposedEntity, requestedFilters, requestedDimensions, validationWarnings, outputs }) => {
+                        const slug = deriveGeneratedDraftSlug(draftQuestion);
+                        const proposedDomain = sourceBlock?.domain ?? draftContextPack?.objects.find((object) => object.domain)?.domain ?? testCase.domain ?? 'misc';
+                        if (dqlArtifact?.kind === 'semantic_block') {
+                            if (!flags.save) {
+                                return {
+                                    path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
+                                    askedTimes: 0,
+                                    proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                                };
+                            }
+                            return upsertGeneratedDqlArtifactDraft(projectRoot, {
+                                slug,
+                                question: draftQuestion,
+                                proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                                proposedDomain,
+                                dqlArtifact,
+                                sourceQuestion: followUp?.sourceQuestion,
+                                sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
+                                followupKind: followUp?.kind,
+                                outputs,
+                                contextPackId: draftContextPack?.id,
+                                routeIntent: String(intent),
+                                validationWarnings,
+                            });
+                        }
                         if (!flags.save) {
                             return {
                                 path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
@@ -641,51 +693,30 @@ async function runEval(rest, flags) {
                                 proposedContractId: `${proposedDomain}.Unknown.${slug}`,
                             };
                         }
-                        return upsertGeneratedDqlArtifactDraft(projectRoot, {
+                        return upsertGeneratedDraft(projectRoot, {
                             slug,
                             question: draftQuestion,
+                            proposedSql: sql,
                             proposedContractId: `${proposedDomain}.Unknown.${slug}`,
                             proposedDomain,
-                            dqlArtifact,
+                            proposedEntity,
+                            sourceDqlArtifact,
                             sourceQuestion: followUp?.sourceQuestion,
                             sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
                             followupKind: followUp?.kind,
+                            requestedFilters,
+                            requestedDimensions,
                             outputs,
                             contextPackId: draftContextPack?.id,
                             routeIntent: String(intent),
                             validationWarnings,
                         });
-                    }
-                    if (!flags.save) {
-                        return {
-                            path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
-                            askedTimes: 0,
-                            proposedContractId: `${proposedDomain}.Unknown.${slug}`,
-                        };
-                    }
-                    return upsertGeneratedDraft(projectRoot, {
-                        slug,
-                        question: draftQuestion,
-                        proposedSql: sql,
-                        proposedContractId: `${proposedDomain}.Unknown.${slug}`,
-                        proposedDomain,
-                        proposedEntity,
-                        sourceDqlArtifact,
-                        sourceQuestion: followUp?.sourceQuestion,
-                        sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
-                        followupKind: followUp?.kind,
-                        requestedFilters,
-                        requestedDimensions,
-                        outputs,
-                        contextPackId: draftContextPack?.id,
-                        routeIntent: String(intent),
-                        validationWarnings,
-                    });
-                },
-            });
+                    },
+                });
             const evaluation = evaluateCase(testCase, result);
             const durationMs = Date.now() - startedAt;
             const draftSaved = Boolean(result.draftBlock?.path ?? result.draftBlockId);
+            const narration = narrationOutcomeForEval(runtimeRun?.narrationIntegrityReceipt);
             const judgeVerdict = judge
                 ? await judgeAnswer({
                     question: testCase.question,
@@ -700,11 +731,27 @@ async function runEval(rest, flags) {
                 passed: evaluation.failures.length === 0,
                 failures: evaluation.failures,
                 durationMs,
+                // The persisted receipt is the only evidence for this metric. Rows and
+                // reader prose are intentionally ignored: a row-bearing answer can be
+                // skipped, while a deterministic fallback can render different wording.
+                narrationAttempted: narration.narrationAttempted,
+                narrationFallback: narration.narrationFallback,
                 executionMs: result.result?.executionTime,
                 executionMatched: evaluation.executionMatched,
                 ...(judgeVerdict ? { judgeScore: judgeVerdict.score, judgePass: judgeVerdict.pass } : {}),
                 kind: result.kind,
-                route: result.contextPack?.routeDecision.route,
+                route: runtimeRun ? evalRouteForRun(runtimeRun.route) : result.contextPack?.routeDecision.route,
+                // Only the runtime driver can see the router's clarification options.
+                // In-process runs leave this undefined, so a clarify there scores as a
+                // dead end — the conservative reading, and another reason `--via runtime`
+                // is the truthful one.
+                ...(runtimeRun ? {
+                    clarificationOptionCount: runtimeRun.clarificationOptions?.length ?? 0,
+                    conversational: runtimeRun.route === 'conversation' || runtimeRun.answerKind === 'conversational',
+                    conversationalAnswer: (runtimeRun.route === 'conversation' || runtimeRun.answerKind === 'conversational')
+                        && Boolean(runtimeRun.answer?.trim()),
+                    meaningResolved: Boolean(runtimeRun.routeDecision?.meaningResolution),
+                } : {}),
                 intent: result.contextPack?.routeDecision.intent,
                 reviewStatus: result.reviewStatus,
                 contextObjects: result.contextPack?.objects.length ?? 0,
@@ -734,6 +781,9 @@ async function runEval(rest, flags) {
         minExecutionMatch: flags.minExecutionMatch ?? null,
         minJudgePass: flags.minJudgePass ?? null,
         maxWrongCertified: flags.maxWrongCertified ?? null,
+        maxFalseRefusal: flags.maxFalseRefusal ?? null,
+        minRefusalRecall: flags.minRefusalRecall ?? null,
+        minGroundedNarration: flags.minGroundedNarration ?? null,
     };
     const thresholdsPassed = agentEvalThresholdsPass(metrics, thresholds);
     const ok = passed === results.length && thresholdsPassed;
@@ -752,6 +802,14 @@ async function runEval(rest, flags) {
     console.log(`Certified hit rate: ${formatRate(metrics.certified_hit_rate)}`);
     console.log(`Generated follow-up pass rate: ${formatRate(metrics.generated_followup_pass_rate)}`);
     console.log(`Safe refusal rate: ${formatRate(metrics.safe_refusal_rate)}`);
+    console.log(`False refusal rate: ${formatRate(metrics.false_refusal_rate)} (${metrics.false_refusal_count}/${metrics.answerable_case_count} answerable cases refused)`);
+    console.log(`Clarification rate: ${formatRate(metrics.clarification_rate)} (answerable cases asked instead of answered)`);
+    if (metrics.meaning_resolved_rate !== null && metrics.meaning_resolved_rate < 1) {
+        console.log(`  ! Semantic judgment ran for only ${formatRate(metrics.meaning_resolved_rate)} of cases. `
+            + 'Without a reachable provider DQL will not settle a reading by lexical rank (AGT-017), so ambiguous '
+            + 'questions clarify by design — treat the clarification rate above as an artifact, not a product signal.');
+    }
+    console.log(`Refusal recall: ${formatRate(metrics.refusal_recall)} (${metrics.refusal_required_case_count} case(s) that must refuse)`);
     console.log(`Execution match rate: ${formatRate(metrics.execution_match_rate)}`);
     console.log(`Tool requirement pass rate: ${formatRate(metrics.tool_requirement_pass_rate)}`);
     console.log(`Tool-observed case count: ${metrics.tool_observed_case_count}`);
@@ -767,6 +825,15 @@ async function runEval(rest, flags) {
     if (thresholds.minJudgePass !== null) {
         console.log(`Judge-pass threshold: ${thresholds.minJudgePass} (actual ${formatRate(metrics.judge_pass_rate)})`);
     }
+    if (thresholds.maxFalseRefusal !== null) {
+        console.log(`False-refusal ceiling: ${thresholds.maxFalseRefusal} (actual ${formatRate(metrics.false_refusal_rate)})`);
+    }
+    if (thresholds.minRefusalRecall !== null) {
+        console.log(`Refusal-recall threshold: ${thresholds.minRefusalRecall} (actual ${formatRate(metrics.refusal_recall)})`);
+    }
+    if (thresholds.minGroundedNarration !== null && thresholds.minGroundedNarration !== undefined) {
+        console.log(`Grounded-narration threshold: ${thresholds.minGroundedNarration} (actual ${formatRate(metrics.grounded_narration_rate)} over ${metrics.grounded_narration_attempted} attempted)`);
+    }
     if (thresholds.maxWrongCertified !== null) {
         console.log(`Wrong-certified ceiling: ${thresholds.maxWrongCertified} (actual ${metrics.wrong_certified_count})`);
     }
@@ -791,6 +858,32 @@ function evaluateCase(testCase, result) {
     const failures = [];
     let validationCode;
     let executionMatched;
+    // Answerability is asserted per case as well as aggregated into
+    // false_refusal_rate, so a single dead-end fails its own case instead of only
+    // nudging a rate someone has to notice.
+    const answerable = evalCaseIsAnswerable(expected);
+    // Three distinct outcomes, not two. An option-bearing clarification neither
+    // answers nor dead-ends: it must not fail an answerable case (its cost is
+    // tracked by `clarification_rate`), and it must not fail a must-refuse case
+    // either, because it did not assert anything about the data.
+    const clarifiedWithOptions = (result.clarificationOptions?.length ?? 0) > 0;
+    // A conversational reply ("I'm here to help you explore your data…") asserts
+    // nothing about the warehouse. For an out-of-scope question that is the CORRECT
+    // outcome — declining politely — so it must not be scored as an answer.
+    const conversational = result.answerKind === 'conversational';
+    const answerText = typeof result.text === 'string' ? result.text.trim() : '';
+    // Conversational replies split two ways, and the case's own expectation says
+    // which is right: for an answerable question a substantive reply IS the answer
+    // (a definition), and for an out-of-scope one it is the correct decline.
+    const conversationalAnswer = conversational && answerText.length > 0;
+    const producedDataAnswer = result.kind !== 'no_answer' && !conversational;
+    const deadEnded = !producedDataAnswer && !clarifiedWithOptions && !conversationalAnswer;
+    if (answerable === true && deadEnded) {
+        failures.push(`FALSE REFUSAL: this question is answerable, but the run dead-ended with no answer and no options${result.refusalCode ? ` (${result.refusalCode})` : ''}`);
+    }
+    if (answerable === false && producedDataAnswer) {
+        failures.push(`expected a refusal (question is out of scope / unanswerable), but the run answered with kind ${result.kind}`);
+    }
     if (expected.kind && result.kind !== expected.kind)
         failures.push(`kind expected ${expected.kind}, got ${result.kind}`);
     if (expected.sourceTier && result.sourceTier !== expected.sourceTier)
@@ -849,7 +942,75 @@ function evaluateCase(testCase, result) {
     }
     return { failures, validationCode, executionMatched };
 }
+/**
+ * Is the case answerable? Explicit `expected.answerable` wins; otherwise infer
+ * from the expectations already present, so the metric covers legacy case files.
+ * A case with no expectations at all is excluded — it asserts nothing, so it can
+ * neither prove nor disprove a false refusal.
+ */
+export function evalCaseIsAnswerable(expected) {
+    if (!expected)
+        return undefined;
+    if (expected.answerable !== undefined)
+        return expected.answerable;
+    if (expected.kind === 'no_answer')
+        return false;
+    if (expected.sourceTier === 'no_answer')
+        return false;
+    if (expected.route === 'clarify')
+        return false;
+    if (Object.keys(expected).length === 0)
+        return undefined;
+    return true;
+}
+/**
+ * Did the run leave the user with NO way forward?
+ *
+ * Deliberately narrower than "did not answer". A clarification that offers
+ * selectable options is answerable on the next turn — worth minimising, tracked
+ * separately as `clarification_rate`, but not the defect. A clarification with
+ * ZERO options is a true dead end: the reported production loop was exactly
+ * this, and a free-text reply to it reproduced the same question forever.
+ */
+export function evalResultRefused(result) {
+    // A substantive conversational reply is an ANSWER, not a dead end. A governed
+    // definition ("**top_customers** — Top 10 customers by lifetime spend…") is
+    // exactly what a "what does X mean?" turn should return, and scoring it as a
+    // refusal would report the feature working as the feature failing.
+    if (result.conversationalAnswer)
+        return false;
+    // Order matters: the drivers collapse every clarification to `no_answer`
+    // (it is not an answer), so the option check has to run FIRST or an
+    // option-bearing clarification is miscounted as a dead end.
+    if (result.route === 'clarify' && (result.clarificationOptionCount ?? 0) > 0)
+        return false;
+    return result.kind === 'no_answer' || result.route === 'clarify';
+}
+/** Did the run ask an answerable clarification rather than answering outright? */
+export function evalResultClarified(result) {
+    return result.route === 'clarify' && (result.clarificationOptionCount ?? 0) > 0;
+}
+/**
+ * Translate only the durable narration receipt into evaluation fields.
+ *
+ * Reader prose, row count, and result shape are deliberately absent: a skipped
+ * narration can have rows, and a deterministic fallback can use any wording.
+ */
+export function narrationOutcomeForEval(receipt) {
+    if (receipt?.mode !== 'verified_facts' || !receipt.attempted)
+        return {};
+    return {
+        // A durable receipt is the only source of this metric.  An infrastructure
+        // error started a verified narration but did not produce a grounded answer,
+        // so it belongs in the denominator just like the deterministic floor.  The
+        // old mapping treated it as a success because only fallback was negative.
+        narrationAttempted: true,
+        narrationFallback: receipt.outcome !== 'success',
+    };
+}
 function computeEvalMetrics(results) {
+    const answerableCases = results.filter((result) => evalCaseIsAnswerable(result.expected) === true);
+    const refusalRequiredCases = results.filter((result) => evalCaseIsAnswerable(result.expected) === false);
     const certifiedCases = results.filter((result) => result.expected?.kind === 'certified' ||
         result.expected?.certification === 'certified' ||
         result.expected?.route === 'certified');
@@ -874,6 +1035,52 @@ function computeEvalMetrics(results) {
         wrong_certified_count: results.filter((result) => result.kind === 'certified' &&
             (result.expected?.kind ? result.expected.kind !== 'certified' : result.followUp)).length,
         outside_context_rejection_count: results.filter((result) => result.validationCode === 'unknown_relation' || result.validationCode === 'unknown_column').length,
+        /**
+         * THE headline number: how often an answerable question was refused.
+         * Bounds every other quality metric — a run that refuses cannot be wrong,
+         * so a falling false-refusal rate must be read together with
+         * `execution_match_rate` to be sure refusals were replaced by CORRECT answers.
+         */
+        false_refusal_rate: ratio(answerableCases.filter(evalResultRefused).length, answerableCases.length),
+        false_refusal_count: answerableCases.filter(evalResultRefused).length,
+        answerable_case_count: answerableCases.length,
+        /**
+         * Answerable cases that asked an option-bearing clarification instead of
+         * answering. Not a defect, but a direct cost in turns — read it next to
+         * false_refusal_rate so a fall in refusals is not just a rise in questions.
+         */
+        clarification_rate: ratio(answerableCases.filter(evalResultClarified).length, answerableCases.length),
+        /**
+         * Cases where semantic judgment ran. Without a provider `mayAssumeInterpretation`
+         * is false (AGT-017), so every ambiguous question clarifies by design and
+         * `clarification_rate` says nothing about product quality.
+         */
+        meaning_resolved_rate: ratio(results.filter((result) => result.meaningResolved === true).length, results.length),
+        /**
+         * Latency, which the acceptance matrix asked for and nothing measured. A
+         * quality gain paid for entirely in wall clock is not a gain: the plan's
+         * two-tier target is certified/semantic under 5s while research takes
+         * minutes, and only a per-class p95 can tell those apart from a regression.
+         */
+        latency_p50_ms: percentileMs(results, 0.5),
+        latency_p95_ms: percentileMs(results, 0.95),
+        latency_p95_answerable_ms: percentileMs(answerableCases, 0.95),
+        /**
+         * How often verified narration survived. When the drafted narration fails
+         * its fact check the reader gets the deterministic record under a
+         * disclaimer — correct, but visibly worse. A silent fall here is exactly
+         * the truncation defect that shipped unnoticed, so it is measured.
+         */
+        grounded_narration_rate: ratio(results.filter((result) => result.narrationAttempted && result.narrationFallback === false).length, results.filter((result) => result.narrationAttempted).length),
+        grounded_narration_attempted: results.filter((result) => result.narrationAttempted).length,
+        /**
+         * The guard on the above: cases that must NOT produce a data answer.
+         * Scored on "did not answer" rather than "dead-ended", because declining via
+         * a clarification is still declining — what would be wrong is asserting
+         * something about data the project does not have.
+         */
+        refusal_recall: ratio(refusalRequiredCases.filter((result) => result.kind === 'no_answer' || result.conversational === true).length, refusalRequiredCases.length),
+        refusal_required_case_count: refusalRequiredCases.length,
         draft_saved_count: results.filter((result) => result.draftSaved).length,
         tool_observed_case_count: results.filter((result) => result.toolCalls > 0).length,
         avg_tool_calls: average(toolCallCounts),
@@ -885,9 +1092,15 @@ function agentEvalThresholdsPass(metrics, thresholds) {
     // A rate threshold with no applicable cases (metric === null) is vacuously
     // satisfied — you only fail when the metric exists and falls below the bar.
     const rateOk = (metric, min) => min === null || min === undefined || metric === null || metric >= min;
-    return rateOk(metrics.tool_requirement_pass_rate, thresholds.minToolRequirement)
+    // A ceiling is only meaningful when the metric has data; `null` means no
+    // answerable case was scored, which is "unknown", not "perfect".
+    const ceilingOk = (metric, max) => max === null || max === undefined || metric === null || metric <= max;
+    return rateOk(metrics.grounded_narration_rate, thresholds.minGroundedNarration)
+        && rateOk(metrics.tool_requirement_pass_rate, thresholds.minToolRequirement)
         && rateOk(metrics.execution_match_rate, thresholds.minExecutionMatch)
         && rateOk(metrics.judge_pass_rate, thresholds.minJudgePass)
+        && ceilingOk(metrics.false_refusal_rate, thresholds.maxFalseRefusal)
+        && rateOk(metrics.refusal_recall, thresholds.minRefusalRecall)
         && (thresholds.maxWrongCertified === null
             || thresholds.maxWrongCertified === undefined
             || metrics.wrong_certified_count <= thresholds.maxWrongCertified);
@@ -1207,6 +1420,20 @@ export const __test__ = {
     cliAnalysisDepth,
     cliReasoningEffort,
     computeEvalMetrics,
+    narrationOutcomeForEval,
     evaluateCase,
 };
+/** Percentile over observed case durations. Returns null when nothing timed. */
+function percentileMs(results, q) {
+    const observed = results
+        .map((result) => result.durationMs)
+        .filter((value) => typeof value === 'number' && Number.isFinite(value) && value >= 0)
+        .sort((left, right) => left - right);
+    if (observed.length === 0)
+        return null;
+    // Nearest-rank: with a handful of cases an interpolated percentile invents a
+    // duration nothing actually took.
+    const rank = Math.min(observed.length - 1, Math.max(0, Math.ceil(q * observed.length) - 1));
+    return observed[rank] ?? null;
+}
 //# sourceMappingURL=agent.js.map