@duckcodeailabs/dql-cli 1.14.0 → 1.14.1

package/dist/commands/agent-eval-cassette.js

@@ -0,0 +1,170 @@
+/**
+ * Provider cassettes — record a suite once against a real model, replay it
+ * offline forever.
+ *
+ * A provider-backed eval is the only kind that measures the product users get:
+ * without one, `mayAssumeInterpretation` is false and every ambiguous question
+ * clarifies by design (AGT-017), which makes a clarification rate an artifact.
+ * But calling a real model on every PR is slow, costly, and non-deterministic —
+ * so the suite could never gate anything.
+ *
+ * Recording once and replaying fixes all three. Model drift then shows up as a
+ * reviewable diff when cassettes are re-recorded, instead of as a mystery
+ * regression on an unrelated pull request.
+ */
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+/**
+ * Stable key for one provider dispatch.
+ *
+ * Deliberately includes the full message list: the prompt IS the input, and two
+ * turns that differ only in retrieved context are genuinely different calls.
+ * Tool NAMES are included but not their schemas — a description edit should not
+ * invalidate every cassette in the suite.
+ */
+export function cassetteKey(input) {
+    const canonical = JSON.stringify({
+        p: input.providerName,
+        o: input.operation,
+        m: input.messages.map((message) => ({ r: message.role, c: message.content })),
+        t: [...(input.toolNames ?? [])].sort(),
+        e: input.options?.reasoningEffort ?? null,
+        x: input.options?.maxTokens ?? null,
+        z: input.options?.temperature ?? null,
+    });
+    return createHash('sha256').update(canonical).digest('hex').slice(0, 32);
+}
+export class CassetteStore {
+    dir;
+    loaded = new Map();
+    constructor(dir) {
+        this.dir = dir;
+        if (!existsSync(dir))
+            return;
+        for (const file of readdirSync(dir)) {
+            if (!file.endsWith('.json'))
+                continue;
+            try {
+                const entry = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
+                if (entry?.key)
+                    this.loaded.set(entry.key, entry);
+            }
+            catch {
+                // A corrupt cassette is skipped, not fatal: it re-records on the next
+                // --record run, and a replay miss reports the key that is missing.
+            }
+        }
+    }
+    get(key) {
+        return this.loaded.get(key);
+    }
+    put(entry) {
+        this.loaded.set(entry.key, entry);
+        mkdirSync(this.dir, { recursive: true });
+        writeFileSync(join(this.dir, `${entry.key}.json`), `${JSON.stringify(entry, null, 2)}\n`, 'utf-8');
+    }
+    size() {
+        return this.loaded.size;
+    }
+}
+export class CassetteMissError extends Error {
+    key;
+    constructor(key, operation) {
+        super(`No cassette for ${operation} (${key}). CI replays recorded provider responses and never calls a live model; `
+            + 're-record with --record against a real provider and commit the new cassettes.');
+        this.name = 'CassetteMissError';
+        this.key = key;
+    }
+}
+/**
+ * Wrap a provider so its dispatches are recorded or replayed.
+ *
+ * On replay of a tool loop the recorded tool calls are RE-INVOKED against the
+ * live tools rather than stubbed. The model's choices stay frozen (that is the
+ * point), but compilation, validation, and execution still really happen — so
+ * the suite keeps catching a broken compiler or a wrong number, which a fully
+ * stubbed replay would sail straight past.
+ */
+export function withCassette(provider, store, mode) {
+    if (mode === 'live')
+        return provider;
+    const wrapped = {
+        name: provider.name,
+        available: () => (mode === 'replay' ? Promise.resolve(true) : provider.available()),
+        async generate(messages, options) {
+            const key = cassetteKey({ providerName: provider.name, operation: 'generate', messages, options });
+            const hit = store.get(key);
+            if (hit)
+                return hit.text;
+            if (mode === 'replay')
+                throw new CassetteMissError(key, 'generate');
+            const text = await provider.generate(messages, options);
+            store.put({
+                key, operation: 'generate', text,
+                providerName: provider.name, recordedAt: new Date().toISOString(),
+            });
+            return text;
+        },
+    };
+    if (provider.generateWithTools) {
+        wrapped.generateWithTools = async (messages, tools, options) => {
+            const key = cassetteKey({
+                providerName: provider.name,
+                operation: 'generate_with_tools',
+                messages,
+                toolNames: tools.map((tool) => tool.name),
+                options: options,
+            });
+            const hit = store.get(key);
+            if (hit) {
+                const byName = new Map(tools.map((tool) => [tool.name, tool]));
+                for (const call of hit.toolCalls ?? []) {
+                    const tool = byName.get(call.name);
+                    // A cassette recorded against a tool set that no longer has this tool
+                    // is stale in a way the run should surface, not paper over.
+                    if (!tool)
+                        continue;
+                    await tool.run(call.args).catch(() => undefined);
+                }
+                return hit.text;
+            }
+            if (mode === 'replay')
+                throw new CassetteMissError(key, 'generate_with_tools');
+            const observed = [];
+            const recordingTools = tools.map((tool) => ({
+                ...tool,
+                run: async (args) => {
+                    observed.push({ name: tool.name, args });
+                    return tool.run(args);
+                },
+            }));
+            const text = await provider.generateWithTools(messages, recordingTools, options);
+            store.put({
+                key, operation: 'generate_with_tools', text, toolCalls: observed,
+                providerName: provider.name, recordedAt: new Date().toISOString(),
+            });
+            return text;
+        };
+    }
+    return wrapped;
+}
+/**
+ * Pick the cassette mode from the environment.
+ *
+ * Defaults to `replay` for ANY value that is not explicitly `record` or `live`,
+ * including a missing or misspelled one. If someone sets the cassette directory
+ * in CI and forgets the mode, the safe outcome is "never call a live model",
+ * not a surprise bill and a non-deterministic suite.
+ */
+export function resolveCassetteModeFromEnv(env) {
+    const raw = env.DQL_EVAL_CASSETTE_MODE;
+    return raw === 'record' || raw === 'live' ? raw : 'replay';
+}
+/** Where a project's eval cassettes live. */
+export function cassetteDirFor(projectRoot, suiteName) {
+    const safe = suiteName.replace(/[^a-z0-9._-]+/gi, '-').replace(/\.(ya?ml)$/i, '');
+    return join(projectRoot, 'test-cassettes', safe);
+}
+export { dirname as _dirname };
+//# sourceMappingURL=agent-eval-cassette.js.map

package/dist/commands/agent-eval-cassette.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-cassette.js","sourceRoot":"","sources":["../../src/commands/agent-eval-cassette.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AA6B1C;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAAC,KAM3B;IACC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAC/B,CAAC,EAAE,KAAK,CAAC,YAAY;QACrB,CAAC,EAAE,KAAK,CAAC,SAAS;QAClB,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7E,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QACtC,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,eAAe,IAAI,IAAI;QACzC,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI;QACnC,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,WAAW,IAAI,IAAI;KACtC,CAAC,CAAC;IACH,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,aAAa;IACP,GAAG,CAAS;IACZ,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAE3D,YAAY,GAAW;QACrB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO;QAC7B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAkB,CAAC;gBAClF,IAAI,KAAK,EAAE,GAAG;oBAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,sEAAsE;gBACtE,mEAAmE;YACrE,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,GAAG,CAAC,KAAoB;QACtB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACjC,GAAG,CAAS;IACrB,YAAY,GAAW,EAAE,SAAiB;QACxC,KAAK,CACH,mBAAmB,SAAS,KAAK,GAAG,0EAA0E;cAC5G,+EAA+E,CAClF,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC1B,QAAuB,EACvB,KAAoB,EACpB,IAAkB;IAElB,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,QAAQ,CAAC;IAErC,MAAM,OAAO,GAAkB;QAC7B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAEnF,KAAK,CAAC,QAAQ,CAAC,QAAwB,EAAE,OAA4B;YACnE,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YACnG,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC,IAAI,CAAC;YACzB,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;YACpE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACxD,KAAK,CAAC,GAAG,CAAC;gBACR,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI;gBAChC,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aAClE,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;IAEF,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,CAAC,iBAAiB,GAAG,KAAK,EAC/B,QAAwB,EACxB,KAA4B,EAC5B,OAAiC,EAChB,EAAE;YACnB,MAAM,GAAG,GAAG,WAAW,CAAC;gBACtB,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,SAAS,EAAE,qBAAqB;gBAChC,QAAQ;gBACR,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;gBACzC,OAAO,EAAE,OAAoD;aAC9D,CAAC,CAAC;YACH,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC/D,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;oBACvC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACnC,sEAAsE;oBACtE,4DAA4D;oBAC5D,IAAI,CAAC,IAAI;wBAAE,SAAS;oBACpB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACnD,CAAC;gBACD,OAAO,GAAG,CAAC,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,IAAI,iBAAiB,CAAC,GAAG,EAAE,qBAAqB,CAAC,CAAC;YAC/E,MAAM,QAAQ,GAAuB,EAAE,CAAC;YACxC,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC1C,GAAG,IAAI;gBACP,GAAG,EAAE,KAAK,EAAE,IAAa,EAAE,EAAE;oBAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;aACF,CAAC,CAAC,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,iBAAkB,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;YAClF,KAAK,CAAC,GAAG,CAAC;gBACR,GAAG,EAAE,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ;gBAChE,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aAClE,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CAAC,GAAuC;IAChF,MAAM,GAAG,GAAG,GAAG,CAAC,sBAAsB,CAAC;IACvC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC7D,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,cAAc,CAAC,WAAmB,EAAE,SAAiB;IACnE,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,WAAW,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AACnD,CAAC;AAED,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,CAAC"}

package/dist/commands/agent-eval-runtime.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Runtime driver for `dql agent eval`.
+ *
+ * The in-process driver calls `answer()` directly, which skips
+ * `createHybridRouter`, `AgentRunEngine`, `enforceOrdinaryAnalyticalPlanBoundary`
+ * and `defaultAgentRunGates`. Every dead-end this harness exists to measure lives
+ * in that skipped half — the router can block before the cascade runs, the engine
+ * synthesizes its own modeling-gap block, and the gates declare refusal codes
+ * terminal. Measuring the answer loop alone reports a cleaner picture than users
+ * experience.
+ *
+ * This driver posts the same body the Ask panel posts and scores the persisted
+ * `AgentRun`, so the suite exercises routing, gates, and the transport projection
+ * as a real end-to-end contract.
+ */
+import type { AgentAnswer, AgentRun, AgentRunRoute, AgentRunStatus, AgentRunTrustState } from '@duckcodeailabs/dql-agent';
+/** Eval-facing view of a run, shaped like the fields the scorer already reads. */
+export interface RuntimeDrivenRun {
+    kind: 'certified' | 'uncertified' | 'no_answer';
+    route?: 'certified' | 'generated_sql' | 'research' | 'clarify';
+    runRoute: AgentRunRoute;
+    status: AgentRunStatus;
+    trustState: AgentRunTrustState;
+    answer?: string;
+    proposedSql?: string;
+    rows?: unknown[];
+    clarificationOptionCount: number;
+    refusalCode?: string;
+    runId: string;
+    /**
+     * A conversational reply (greeting, capability, polite redirect). It asserts
+     * nothing about the data, so it is neither an answer nor a refusal — scoring it
+     * as either misreads the correct outcome for an out-of-scope question.
+     */
+    conversational: boolean;
+    /**
+     * Did the meaning resolver actually run for this turn?
+     *
+     * With no provider configured it cannot, and `mayAssumeInterpretation` goes
+     * false (AGT-017) — DQL deliberately refuses to settle `booked_revenue` vs
+     * `billed_revenue` by lexical rank with semantic judgment switched off. Every
+     * ambiguous question then clarifies. That is correct behaviour, but it makes a
+     * clarification rate measured without a provider meaningless, so the harness
+     * has to be able to say so.
+     */
+    meaningResolved: boolean;
+}
+/**
+ * Did this run decline to produce a usable answer?
+ *
+ * `needs_review` is deliberately NOT a refusal: a review-required generated
+ * answer is the intended outcome when no governed plan froze, and counting it as
+ * a refusal would make the false-refusal metric punish the very behaviour the
+ * cascade is supposed to produce.
+ */
+export declare function runtimeRunRefused(run: Pick<AgentRun, 'route' | 'status'>): boolean;
+/** Map an engine route onto the vocabulary the eval cases already use. */
+export declare function evalRouteForRun(route: AgentRunRoute): RuntimeDrivenRun['route'];
+/**
+ * Collapse a run into the coarse `kind` the existing expectations assert.
+ * Certification follows TRUST, not route: a certified route that degraded to a
+ * review-required answer must not be scored as certified.
+ */
+export declare function evalKindForRun(run: Pick<AgentRun, 'route' | 'status' | 'trustState'>): RuntimeDrivenRun['kind'];
+/** Pull SQL and result rows out of the run's artifacts, when it produced any. */
+export declare function runtimeRunOutputs(run: AgentRun): {
+    proposedSql?: string;
+    rows?: unknown[];
+};
+export declare function projectRuntimeRun(run: AgentRun): RuntimeDrivenRun;
+export interface RuntimeDriverOptions {
+    runtimeBase: string;
+    question: string;
+    requestedMode?: 'ask' | 'auto' | 'research';
+    threadId?: string;
+    timeoutMs?: number;
+    fetchImpl?: typeof fetch;
+}
+/**
+ * Post one question to a running `dql serve` and return the persisted run.
+ *
+ * Failures are surfaced rather than swallowed: a harness that silently scores a
+ * transport error as a refusal would report a false-refusal spike that no code
+ * change caused.
+ */
+export declare function driveViaRuntime(options: RuntimeDriverOptions): Promise<AgentRun>;
+/**
+ * Adapt a persisted run into the `AgentAnswer` shape the existing scorer reads,
+ * so both drivers share one set of assertions.
+ *
+ * Fields the run genuinely does not carry are left UNDEFINED rather than
+ * defaulted. An expectation that references one then fails loudly instead of
+ * passing against a fabricated value — a harness that quietly invents
+ * `sourceTier` would report agreement it never observed.
+ */
+export declare function answerFromRuntimeRun(run: AgentRun): AgentAnswer;
+//# sourceMappingURL=agent-eval-runtime.d.ts.map

package/dist/commands/agent-eval-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-runtime.d.ts","sourceRoot":"","sources":["../../src/commands/agent-eval-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE1H,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,WAAW,GAAG,aAAa,GAAG,WAAW,CAAC;IAChD,KAAK,CAAC,EAAE,WAAW,GAAG,eAAe,GAAG,UAAU,GAAG,SAAS,CAAC;IAC/D,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;IACjB,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,cAAc,EAAE,OAAO,CAAC;IACxB;;;;;;;;;OASG;IACH,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,QAAQ,CAAC,GAAG,OAAO,CAKlF;AAED,0EAA0E;AAC1E,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAU/E;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,YAAY,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAI/G;AAYD,iFAAiF;AACjF,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAA;CAAE,CAe3F;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG,gBAAgB,CAgBjE;AAED,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,UAAU,CAAC;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,CA8BtF;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,QAAQ,GAAG,WAAW,CAuB/D"}

package/dist/commands/agent-eval-runtime.js

@@ -0,0 +1,155 @@
+/**
+ * Did this run decline to produce a usable answer?
+ *
+ * `needs_review` is deliberately NOT a refusal: a review-required generated
+ * answer is the intended outcome when no governed plan froze, and counting it as
+ * a refusal would make the false-refusal metric punish the very behaviour the
+ * cascade is supposed to produce.
+ */
+export function runtimeRunRefused(run) {
+    return run.route === 'blocked'
+        || run.route === 'clarify'
+        || run.status === 'blocked'
+        || run.status === 'needs_clarification';
+}
+/** Map an engine route onto the vocabulary the eval cases already use. */
+export function evalRouteForRun(route) {
+    switch (route) {
+        case 'certified_answer': return 'certified';
+        case 'semantic_answer': return 'certified';
+        case 'generated_answer': return 'generated_sql';
+        case 'research': return 'research';
+        case 'clarify':
+        case 'blocked': return 'clarify';
+        default: return undefined;
+    }
+}
+/**
+ * Collapse a run into the coarse `kind` the existing expectations assert.
+ * Certification follows TRUST, not route: a certified route that degraded to a
+ * review-required answer must not be scored as certified.
+ */
+export function evalKindForRun(run) {
+    if (runtimeRunRefused(run))
+        return 'no_answer';
+    if (run.trustState === 'certified')
+        return 'certified';
+    return 'uncertified';
+}
+function firstArtifactPayload(run, kinds) {
+    for (const artifact of run.artifacts ?? []) {
+        if (!kinds.includes(String(artifact.kind)))
+            continue;
+        if (artifact.payload && typeof artifact.payload === 'object') {
+            return artifact.payload;
+        }
+    }
+    return undefined;
+}
+/** Pull SQL and result rows out of the run's artifacts, when it produced any. */
+export function runtimeRunOutputs(run) {
+    const payload = firstArtifactPayload(run, ['sql_cell', 'dql_block', 'generated_answer', 'answer', 'result']);
+    const sql = typeof payload?.sql === 'string'
+        ? payload.sql
+        : typeof payload?.compiledSql === 'string' ? payload.compiledSql : undefined;
+    const resultRecord = payload?.result && typeof payload.result === 'object'
+        ? payload.result
+        : undefined;
+    const rows = Array.isArray(resultRecord?.rows)
+        ? resultRecord.rows
+        : Array.isArray(payload?.rows) ? payload.rows : undefined;
+    return {
+        ...(sql ? { proposedSql: sql } : {}),
+        ...(rows ? { rows } : {}),
+    };
+}
+export function projectRuntimeRun(run) {
+    const refusalEvaluation = (run.evaluations ?? []).find((evaluation) => !evaluation.passed);
+    return {
+        kind: evalKindForRun(run),
+        route: evalRouteForRun(run.route),
+        runRoute: run.route,
+        status: run.status,
+        trustState: run.trustState,
+        clarificationOptionCount: run.clarificationOptions?.length ?? 0,
+        runId: run.id,
+        conversational: run.route === 'conversation' || run.answerKind === 'conversational',
+        meaningResolved: Boolean(run.routeDecision?.meaningResolution),
+        ...(run.answer ? { answer: run.answer } : {}),
+        ...(refusalEvaluation?.id ? { refusalCode: refusalEvaluation.id } : {}),
+        ...runtimeRunOutputs(run),
+    };
+}
+/**
+ * Post one question to a running `dql serve` and return the persisted run.
+ *
+ * Failures are surfaced rather than swallowed: a harness that silently scores a
+ * transport error as a refusal would report a false-refusal spike that no code
+ * change caused.
+ */
+export async function driveViaRuntime(options) {
+    const base = options.runtimeBase.replace(/\/$/, '');
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 120_000);
+    try {
+        const response = await fetchImpl(`${base}/api/agent-runs`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                question: options.question,
+                requestedMode: options.requestedMode ?? 'ask',
+                ...(options.threadId ? { threadId: options.threadId } : {}),
+            }),
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Runtime returned ${response.status}: ${(await response.text()).slice(0, 400)}`);
+        }
+        // The endpoint wraps the record (`{ run }`); older/streamed paths return it
+        // bare. Accept both rather than guessing one and failing the whole suite.
+        const body = await response.json();
+        const run = body.run ?? body;
+        if (!run || typeof run.id !== 'string') {
+            throw new Error(`Runtime response was not an agent run: ${JSON.stringify(body).slice(0, 200)}`);
+        }
+        return run;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Adapt a persisted run into the `AgentAnswer` shape the existing scorer reads,
+ * so both drivers share one set of assertions.
+ *
+ * Fields the run genuinely does not carry are left UNDEFINED rather than
+ * defaulted. An expectation that references one then fails loudly instead of
+ * passing against a fabricated value — a harness that quietly invents
+ * `sourceTier` would report agreement it never observed.
+ */
+export function answerFromRuntimeRun(run) {
+    const projected = projectRuntimeRun(run);
+    const certification = projected.kind === 'no_answer'
+        ? 'analyst_review_required'
+        : run.trustState === 'certified'
+            ? 'certified'
+            : run.trustState === 'review_required'
+                ? 'analyst_review_required'
+                : 'ai_generated';
+    return {
+        kind: projected.kind,
+        certification,
+        reviewStatus: run.trustState === 'review_required' ? 'analyst_review_required' : 'none',
+        text: run.answer ?? run.summary,
+        citations: [],
+        considered: [],
+        ...(projected.proposedSql ? { proposedSql: projected.proposedSql } : {}),
+        ...(projected.rows ? { result: { columns: [], rows: projected.rows, rowCount: projected.rows.length } } : {}),
+        ...(projected.refusalCode ? { refusalCode: projected.refusalCode } : {}),
+        // Carried so the scorer can tell an answerable clarification from a dead end.
+        ...(run.clarificationOptions?.length ? { clarificationOptions: run.clarificationOptions } : {}),
+        ...(projected.conversational ? { answerKind: 'conversational' } : {}),
+    };
+}
+//# sourceMappingURL=agent-eval-runtime.js.map

package/dist/commands/agent-eval-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-runtime.js","sourceRoot":"","sources":["../../src/commands/agent-eval-runtime.ts"],"names":[],"mappings":"AAiDA;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAuC;IACvE,OAAO,GAAG,CAAC,KAAK,KAAK,SAAS;WACzB,GAAG,CAAC,KAAK,KAAK,SAAS;WACvB,GAAG,CAAC,MAAM,KAAK,SAAS;WACxB,GAAG,CAAC,MAAM,KAAK,qBAAqB,CAAC;AAC5C,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,kBAAkB,CAAC,CAAC,OAAO,WAAW,CAAC;QAC5C,KAAK,iBAAiB,CAAC,CAAC,OAAO,WAAW,CAAC;QAC3C,KAAK,kBAAkB,CAAC,CAAC,OAAO,eAAe,CAAC;QAChD,KAAK,UAAU,CAAC,CAAC,OAAO,UAAU,CAAC;QACnC,KAAK,SAAS,CAAC;QACf,KAAK,SAAS,CAAC,CAAC,OAAO,SAAS,CAAC;QACjC,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC;IAC5B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAsD;IACnF,IAAI,iBAAiB,CAAC,GAAG,CAAC;QAAE,OAAO,WAAW,CAAC;IAC/C,IAAI,GAAG,CAAC,UAAU,KAAK,WAAW;QAAE,OAAO,WAAW,CAAC;IACvD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CAAC,GAAa,EAAE,KAAe;IAC1D,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAAE,SAAS;QACrD,IAAI,QAAQ,CAAC,OAAO,IAAI,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC7D,OAAO,QAAQ,CAAC,OAAkC,CAAC;QACrD,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,iBAAiB,CAAC,GAAa;IAC7C,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7G,MAAM,GAAG,GAAG,OAAO,OAAO,EAAE,GAAG,KAAK,QAAQ;QAC1C,CAAC,CAAC,OAAO,CAAC,GAAG;QACb,CAAC,CAAC,OAAO,OAAO,EAAE,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/E,MAAM,YAAY,GAAG,OAAO,EAAE,MAAM,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ;QACxE,CAAC,CAAC,OAAO,CAAC,MAAiC;QAC3C,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;QAC5C,CAAC,CAAC,YAAY,CAAC,IAAiB;QAChC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,OAAO;QACL,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAa;IAC7C,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3F,OAAO;QACL,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC;QACzB,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC;QACjC,QAAQ,EAAE,GAAG,CAAC,KAAK;QACnB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,wBAAwB,EAAE,GAAG,CAAC,oBAAoB,EAAE,MAAM,IAAI,CAAC;QAC/D,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,cAAc,EAAE,GAAG,CAAC,KAAK,KAAK,cAAc,IAAI,GAAG,CAAC,UAAU,KAAK,gBAAgB;QACnF,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAC9D,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,iBAAiB,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,iBAAiB,CAAC,GAAG,CAAC;KAC1B,CAAC;AACJ,CAAC;AAWD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA6B;IACjE,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC;IACjF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,iBAAiB,EAAE;YACzD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,KAAK;gBAC7C,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5D,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC;QACD,4EAA4E;QAC5E,0EAA0E;QAC1E,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAmC,CAAC;QACpE,MAAM,GAAG,GAAI,IAA2B,CAAC,GAAG,IAAI,IAAgB,CAAC;QACjE,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAa;IAChD,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW;QAClD,CAAC,CAAC,yBAAyB;QAC3B,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,WAAW;YAC9B,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,iBAAiB;gBACpC,CAAC,CAAC,yBAAyB;gBAC3B,CAAC,CAAC,cAAc,CAAC;IACvB,OAAO;QACL,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,aAAa;QACb,YAAY,EAAE,GAAG,CAAC,UAAU,KAAK,iBAAiB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,MAAM;QACvF,IAAI,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO;QAC/B,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;QACd,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7G,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,8EAA8E;QAC9E,GAAG,CAAC,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5C,CAAC;AAC9B,CAAC"}

package/dist/commands/agent.d.ts

@@ -21,7 +21,7 @@
  *   dql agent feedback <up|down> --block <id> --question "..."
  *     Records feedback into the KG. Used by clients without MCP access.
  */
-import { answer, type AgentAnswer, type AgentFollowUpContext, type AnalysisDepth, type ReasoningEffort } from '@duckcodeailabs/dql-agent';
+import { answer, type AgentAnswer, type AgentFollowUpContext, type AnalysisDepth, type NarrationIntegrityReceiptV1, type ReasoningEffort } from '@duckcodeailabs/dql-agent';
 import type { CLIFlags } from '../args.js';
 declare function cliReasoningEffort(flags: CLIFlags): ReasoningEffort | undefined;
 declare function cliAnalysisDepth(flags: CLIFlags): AnalysisDepth | undefined;
@@ -49,6 +49,18 @@ interface AgentEvalCase {
         draftSaved?: boolean;
         minToolCalls?: number;
         rows?: unknown[];
+        /**
+         * Is this question answerable at all?
+         *
+         * When true, ANY refusal (`no_answer` / `clarify`) is a FALSE REFUSAL — the
+         * single number that makes "Ask AI refuses too much" measurable instead of
+         * anecdotal. When false, the case belongs to the genuine-refusal class and a
+         * refusal is the correct outcome; answering it would be a hallucination.
+         *
+         * Omitted, it is inferred from the other expectations, so existing case files
+         * contribute to the metric without being rewritten.
+         */
+        answerable?: boolean;
     };
 }
 interface AgentEvalResult {
@@ -56,6 +68,20 @@ interface AgentEvalResult {
     passed: boolean;
     failures: string[];
     durationMs: number;
+    /**
+     * True when verified narration failed its fact check and the deterministic
+     * record was shown instead. A silent rise here is exactly the truncation
+     * defect that shipped unnoticed, so it is measured.
+     */
+    narrationFallback?: boolean;
+    /**
+     * Was verified-fact narration ATTEMPTED at all? Undefined on cases that never
+     * reach the narrator (refusals, conversational replies). Without this the
+     * grounded-narration denominator counted every case, diluting real failures
+     * with runs that were never at risk — a metric that masks the defect it was
+     * added to catch.
+     */
+    narrationAttempted?: boolean;
     executionMs?: number;
     executionMatched?: boolean;
     kind: AgentAnswer['kind'];
@@ -71,6 +97,18 @@ interface AgentEvalResult {
     toolCalls: number;
     judgeScore?: number;
     judgePass?: boolean;
+    /**
+     * Selectable options offered with a clarification. A clarification that offers
+     * real choices is answerable in one more turn; one that offers none is the
+     * dead end this suite exists to catch.
+     */
+    clarificationOptionCount?: number;
+    /** True when the run replied conversationally instead of asserting data. */
+    conversational?: boolean;
+    /** True when that conversational reply actually carried content. */
+    conversationalAnswer?: boolean;
+    /** True when the meaning resolver ran (a provider was reachable). */
+    meaningResolved?: boolean;
 }
 type AgentEvalTraceStageName = 'context' | 'rewrite' | 'lane' | 'tools' | 'answer' | 'validation' | 'execution' | 'draft' | 'scoring';
 interface AgentEvalTraceStage {
@@ -84,6 +122,32 @@ declare function evaluateCase(testCase: AgentEvalCase, result: Awaited<ReturnTyp
     validationCode?: string;
     executionMatched?: boolean;
 };
+/**
+ * Is the case answerable? Explicit `expected.answerable` wins; otherwise infer
+ * from the expectations already present, so the metric covers legacy case files.
+ * A case with no expectations at all is excluded — it asserts nothing, so it can
+ * neither prove nor disprove a false refusal.
+ */
+export declare function evalCaseIsAnswerable(expected: AgentEvalCase['expected']): boolean | undefined;
+/**
+ * Did the run leave the user with NO way forward?
+ *
+ * Deliberately narrower than "did not answer". A clarification that offers
+ * selectable options is answerable on the next turn — worth minimising, tracked
+ * separately as `clarification_rate`, but not the defect. A clarification with
+ * ZERO options is a true dead end: the reported production loop was exactly
+ * this, and a free-text reply to it reproduced the same question forever.
+ */
+export declare function evalResultRefused(result: Pick<AgentEvalResult, 'kind' | 'route' | 'clarificationOptionCount' | 'conversationalAnswer'>): boolean;
+/** Did the run ask an answerable clarification rather than answering outright? */
+export declare function evalResultClarified(result: Pick<AgentEvalResult, 'route' | 'clarificationOptionCount'>): boolean;
+/**
+ * Translate only the durable narration receipt into evaluation fields.
+ *
+ * Reader prose, row count, and result shape are deliberately absent: a skipped
+ * narration can have rows, and a deterministic fallback can use any wording.
+ */
+export declare function narrationOutcomeForEval(receipt: NarrationIntegrityReceiptV1 | undefined): Pick<AgentEvalResult, 'narrationAttempted' | 'narrationFallback'>;
 declare function computeEvalMetrics(results: AgentEvalResult[]): {
     certified_hit_rate: number | null;
     judge_mean_score: number | null;
@@ -94,6 +158,52 @@ declare function computeEvalMetrics(results: AgentEvalResult[]): {
     tool_requirement_pass_rate: number | null;
     wrong_certified_count: number;
     outside_context_rejection_count: number;
+    /**
+     * THE headline number: how often an answerable question was refused.
+     * Bounds every other quality metric — a run that refuses cannot be wrong,
+     * so a falling false-refusal rate must be read together with
+     * `execution_match_rate` to be sure refusals were replaced by CORRECT answers.
+     */
+    false_refusal_rate: number | null;
+    false_refusal_count: number;
+    answerable_case_count: number;
+    /**
+     * Answerable cases that asked an option-bearing clarification instead of
+     * answering. Not a defect, but a direct cost in turns — read it next to
+     * false_refusal_rate so a fall in refusals is not just a rise in questions.
+     */
+    clarification_rate: number | null;
+    /**
+     * Cases where semantic judgment ran. Without a provider `mayAssumeInterpretation`
+     * is false (AGT-017), so every ambiguous question clarifies by design and
+     * `clarification_rate` says nothing about product quality.
+     */
+    meaning_resolved_rate: number | null;
+    /**
+     * Latency, which the acceptance matrix asked for and nothing measured. A
+     * quality gain paid for entirely in wall clock is not a gain: the plan's
+     * two-tier target is certified/semantic under 5s while research takes
+     * minutes, and only a per-class p95 can tell those apart from a regression.
+     */
+    latency_p50_ms: number | null;
+    latency_p95_ms: number | null;
+    latency_p95_answerable_ms: number | null;
+    /**
+     * How often verified narration survived. When the drafted narration fails
+     * its fact check the reader gets the deterministic record under a
+     * disclaimer — correct, but visibly worse. A silent fall here is exactly
+     * the truncation defect that shipped unnoticed, so it is measured.
+     */
+    grounded_narration_rate: number | null;
+    grounded_narration_attempted: number;
+    /**
+     * The guard on the above: cases that must NOT produce a data answer.
+     * Scored on "did not answer" rather than "dead-ended", because declining via
+     * a clarification is still declining — what would be wrong is asserting
+     * something about data the project does not have.
+     */
+    refusal_recall: number | null;
+    refusal_required_case_count: number;
     draft_saved_count: number;
     tool_observed_case_count: number;
     avg_tool_calls: number;
@@ -105,6 +215,9 @@ declare function agentEvalThresholdsPass(metrics: ReturnType<typeof computeEvalM
     minExecutionMatch?: number | null;
     minJudgePass?: number | null;
     maxWrongCertified?: number | null;
+    maxFalseRefusal?: number | null;
+    minRefusalRecall?: number | null;
+    minGroundedNarration?: number | null;
 }): boolean;
 declare function buildEvalTrace(input: {
     testCase: AgentEvalCase;
@@ -119,6 +232,7 @@ export declare const __test__: {
     cliAnalysisDepth: typeof cliAnalysisDepth;
     cliReasoningEffort: typeof cliReasoningEffort;
     computeEvalMetrics: typeof computeEvalMetrics;
+    narrationOutcomeForEval: typeof narrationOutcomeForEval;
     evaluateCase: typeof evaluateCase;
 };
 export {};

package/dist/commands/agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/commands/agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAKH,OAAO,EAQL,MAAM,EAcN,KAAK,WAAW,EAChB,KAAK,oBAAoB,EAGzB,KAAK,aAAa,EAClB,KAAK,eAAe,EACrB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA+G3C,iBAAS,kBAAkB,CAAC,KAAK,EAAE,QAAQ,GAAG,eAAe,GAAG,SAAS,CAExE;AAED,iBAAS,gBAAgB,CAAC,KAAK,EAAE,QAAQ,GAAG,aAAa,GAAG,SAAS,CAGpE;AAcD,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,GAAG,IAAI,EAClB,IAAI,EAAE,MAAM,EAAE,EACd,KAAK,EAAE,QAAQ,GACd,OAAO,CAAC,IAAI,CAAC,CAsBf;AAkZD,UAAU,aAAa;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QACT,UAAU,CAAC,EAAE,oBAAoB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,cAAc,GAAG,WAAW,CAAC;QACzG,aAAa,CAAC,EAAE,WAAW,GAAG,cAAc,GAAG,yBAAyB,CAAC;QACzE,IAAI,CAAC,EAAE,WAAW,GAAG,aAAa,GAAG,WAAW,CAAC;QACjD,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAChC,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QACnC,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;QACjC,KAAK,CAAC,EAAE,WAAW,GAAG,eAAe,GAAG,UAAU,GAAG,SAAS,CAAC;QAC/D,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,yBAAyB,GAAG,WAAW,CAAC;QAChF,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,oBAAoB,CAAC,EAAE,OAAO,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;KAClB,CAAC;CACH;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,KAAK,uBAAuB,GACxB,SAAS,GACT,SAAS,GACT,MAAM,GACN,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,WAAW,GACX,OAAO,GACP,SAAS,CAAC;AAEd,UAAU,mBAAmB;IAC3B,KAAK,EAAE,uBAAuB,CAAC;IAC/B,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAyPD,iBAAS,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC,GAAG;IAC1F,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAkDA;AAED,iBAAS,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE;;;;;;;;;;;;;;;EA4CrD;AAED,iBAAS,uBAAuB,CAC9B,OAAO,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,EAC9C,UAAU,EAAE;IACV,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC,GACA,OAAO,CAWT;AAED,iBAAS,cAAc,CAAC,KAAK,EAAE;IAC7B,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC;IAC3C,UAAU,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB,GAAG,mBAAmB,EAAE,CAyKxB;AAuKD,eAAO,MAAM,QAAQ;;;;;;;CAOpB,CAAC"}
+{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../src/commands/agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAOH,OAAO,EAQL,MAAM,EAcN,KAAK,WAAW,EAChB,KAAK,oBAAoB,EAGzB,KAAK,aAAa,EAClB,KAAK,2BAA2B,EAChC,KAAK,eAAe,EACrB,MAAM,2BAA2B,CAAC;AAEnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA+G3C,iBAAS,kBAAkB,CAAC,KAAK,EAAE,QAAQ,GAAG,eAAe,GAAG,SAAS,CAExE;AAED,iBAAS,gBAAgB,CAAC,KAAK,EAAE,QAAQ,GAAG,aAAa,GAAG,SAAS,CAGpE;AAcD,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,GAAG,IAAI,EAClB,IAAI,EAAE,MAAM,EAAE,EACd,KAAK,EAAE,QAAQ,GACd,OAAO,CAAC,IAAI,CAAC,CAsBf;AAkZD,UAAU,aAAa;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE;QACT,UAAU,CAAC,EAAE,oBAAoB,GAAG,kBAAkB,GAAG,gBAAgB,GAAG,cAAc,GAAG,WAAW,CAAC;QACzG,aAAa,CAAC,EAAE,WAAW,GAAG,cAAc,GAAG,yBAAyB,CAAC;QACzE,IAAI,CAAC,EAAE,WAAW,GAAG,aAAa,GAAG,WAAW,CAAC;QACjD,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QAChC,cAAc,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;QACnC,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;QACjC,KAAK,CAAC,EAAE,WAAW,GAAG,eAAe,GAAG,UAAU,GAAG,SAAS,CAAC;QAC/D,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,YAAY,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,yBAAyB,GAAG,WAAW,CAAC;QAChF,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,oBAAoB,CAAC,EAAE,OAAO,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;QACjB;;;;;;;;;;WAUG;QACH,UAAU,CAAC,EAAE,OAAO,CAAC;KACtB,CAAC;CACH;AAED,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IACrC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,mBAAmB,EAAE,CAAC;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;;;;OAIG;IACH,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,4EAA4E;IAC5E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,oEAAoE;IACpE,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,qEAAqE;IACrE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,KAAK,uBAAuB,GACxB,SAAS,GACT,SAAS,GACT,MAAM,GACN,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,WAAW,GACX,OAAO,GACP,SAAS,CAAC;AAEd,UAAU,mBAAmB;IAC3B,KAAK,EAAE,uBAAuB,CAAC;IAC/B,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AA8TD,iBAAS,YAAY,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC,GAAG;IAC1F,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CA6EA;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,SAAS,CAQ7F;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,GAAG,OAAO,GAAG,0BAA0B,GAAG,sBAAsB,CAAC,GACpG,OAAO,CAWT;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,OAAO,GAAG,0BAA0B,CAAC,GAClE,OAAO,CAET;AAED;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,2BAA2B,GAAG,SAAS,GAC/C,IAAI,CAAC,eAAe,EAAE,oBAAoB,GAAG,mBAAmB,CAAC,CAUnE;AAED,iBAAS,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE;;;;;;;;;;IAwClD;;;;;OAKG;;;;IAIH;;;;OAIG;;IAEH;;;;OAIG;;IAEH;;;;;OAKG;;;;IAIH;;;;;OAKG;;;IAMH;;;;;OAKG;;;;;;;;EAYN;AAED,iBAAS,uBAAuB,CAC9B,OAAO,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,EAC9C,UAAU,EAAE;IACV,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oBAAoB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACtC,GACA,OAAO,CAkBT;AAED,iBAAS,cAAc,CAAC,KAAK,EAAE;IAC7B,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC;IAC3C,UAAU,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,OAAO,CAAC;CACrB,GAAG,mBAAmB,EAAE,CAyKxB;AAuKD,eAAO,MAAM,QAAQ;;;;;;;;CAQpB,CAAC"}