@duckcodeailabs/dql-cli 1.14.0 → 1.14.2

package/dist/commands/agent-eval-cassette.js

@@ -0,0 +1,313 @@
+/**
+ * Provider cassettes — record a suite once against a real model, replay it
+ * offline forever. A deliberately-labelled deterministic migration can keep a
+ * legacy response usable for orchestration replay, but it is never evidence of
+ * current provider quality.
+ *
+ * A provider-backed eval is the only kind that measures the product users get:
+ * without one, `mayAssumeInterpretation` is false and every ambiguous question
+ * clarifies by design (AGT-017), which makes a clarification rate an artifact.
+ * But calling a real model on every PR is slow, costly, and non-deterministic —
+ * so the suite could never gate anything.
+ *
+ * Recording once and replaying fixes all three. Model drift then shows up as a
+ * reviewable diff when cassettes are re-recorded, instead of as a mystery
+ * regression on an unrelated pull request.
+ */
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+const EVAL_ORCHESTRATION_TRANSIENT_LABELS = [
+    'context_pack_id',
+    'run_id',
+    'snapshot_id',
+    'plan_id',
+    'reference_instant',
+    'generated_at',
+    'retrieved_at',
+];
+const EVAL_ORCHESTRATION_TRANSIENT_LABEL_PATTERN = EVAL_ORCHESTRATION_TRANSIENT_LABELS.join('|');
+/**
+ * This is intentionally not a general UUID/date/path scrubber. The provider
+ * still sees and the fingerprint still binds every business date, user term,
+ * relation, column, qualified ID, join, lineage edge, SQL fragment, role, and
+ * option. Only producer-labelled orchestration values and the explicitly
+ * supplied eval project root are non-semantic enough to normalize.
+ */
+function canonicalizeEvalMessageContentV2(content, projectRoot) {
+    const applied = new Set();
+    const replacePlainLabel = (_whole, prefix, label) => {
+        applied.add(`producer_label:${label.toLowerCase()}`);
+        return `${prefix}<ORCHESTRATION_TRANSIENT>`;
+    };
+    const replaceJsonLabel = (_whole, prefix, label, suffix) => {
+        applied.add(`producer_label:${label.toLowerCase()}`);
+        return `${prefix}"<ORCHESTRATION_TRANSIENT>"${suffix}`;
+    };
+    let normalized = content
+        .replace(new RegExp(`^(\\s*)((?:${EVAL_ORCHESTRATION_TRANSIENT_LABEL_PATTERN}))\\s*:\\s*[^\\r\\n]*$`, 'gim'), (_whole, indent, label) => replacePlainLabel(_whole, `${indent}${label}: `, label))
+        .replace(new RegExp(`^(\\s*"((?:${EVAL_ORCHESTRATION_TRANSIENT_LABEL_PATTERN}))"\\s*:\\s*)"[^"\\r\\n]*"(\\s*,?\\s*)$`, 'gim'), (_whole, prefix, label, suffix) => replaceJsonLabel(_whole, prefix, label, suffix));
+    if (projectRoot && normalized.includes(projectRoot)) {
+        normalized = normalized.split(projectRoot).join('<PROJECT_ROOT>');
+        applied.add('project_root');
+    }
+    return { content: normalized, appliedRuleClasses: Array.from(applied).sort() };
+}
+export function evalCassetteCanonicalizationV2(projectRoot) {
+    return { version: 2, projectRoot };
+}
+function hashDispatch(value) {
+    return createHash('sha256').update(JSON.stringify(value)).digest('hex');
+}
+function canonicalDispatch(input, messages, version) {
+    return {
+        ...(version ? { v: version } : {}),
+        p: input.providerName,
+        o: input.operation,
+        m: messages.map((message) => ({ r: message.role, c: message.content })),
+        // V1 intentionally treated a tool-set reorder as equivalent. V2 is
+        // narrower: canonicalization must not rewrite provider/tool dispatch
+        // semantics, including tool order.
+        t: version ? [...(input.toolNames ?? [])] : [...(input.toolNames ?? [])].sort(),
+        e: input.options?.reasoningEffort ?? null,
+        x: input.options?.maxTokens ?? null,
+        z: input.options?.temperature ?? null,
+    };
+}
+/**
+ * Stable key for one provider dispatch.
+ *
+ * Deliberately includes the full message list: the prompt IS the input, and two
+ * turns that differ only in retrieved context are genuinely different calls.
+ * Tool NAMES are included but not their schemas — a description edit should not
+ * invalidate every cassette in the suite.
+ */
+export function cassetteFingerprint(input) {
+    const legacyHash = hashDispatch(canonicalDispatch(input, input.messages));
+    if (!input.canonicalization)
+        return { key: legacyHash.slice(0, 32) };
+    const applied = new Set();
+    const messages = input.messages.map((message) => {
+        // A user may literally ask about an ID, date, or local path. Only
+        // producer-owned messages can contain the labelled orchestration fields
+        // that this eval-only canonicalizer is permitted to normalize.
+        if (message.role === 'user')
+            return message;
+        const normalized = canonicalizeEvalMessageContentV2(message.content, input.canonicalization.projectRoot);
+        for (const rule of normalized.appliedRuleClasses)
+            applied.add(rule);
+        return { ...message, content: normalized.content };
+    });
+    const postHash = hashDispatch(canonicalDispatch(input, messages, 2));
+    return {
+        key: postHash.slice(0, 32),
+        legacyKey: legacyHash.slice(0, 32),
+        diagnostics: {
+            version: 2,
+            messageCount: input.messages.length,
+            messageRoles: input.messages.map((message) => message.role),
+            preCanonicalHash: hashDispatch(canonicalDispatch(input, input.messages, 2)),
+            postCanonicalHash: postHash,
+            appliedRuleClasses: Array.from(applied).sort(),
+        },
+    };
+}
+export function cassetteKey(input) {
+    return cassetteFingerprint(input).key;
+}
+export class CassetteStore {
+    dir;
+    loaded = new Map();
+    constructor(dir) {
+        this.dir = dir;
+        if (!existsSync(dir))
+            return;
+        for (const file of readdirSync(dir)) {
+            if (!file.endsWith('.json'))
+                continue;
+            try {
+                const entry = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
+                if (entry?.key)
+                    this.loaded.set(entry.key, entry);
+            }
+            catch {
+                // A corrupt cassette is skipped, not fatal: it re-records on the next
+                // --record run, and a replay miss reports the key that is missing.
+            }
+        }
+    }
+    get(key) {
+        return this.loaded.get(key);
+    }
+    put(entry) {
+        this.loaded.set(entry.key, entry);
+        mkdirSync(this.dir, { recursive: true });
+        writeFileSync(join(this.dir, `${entry.key}.json`), `${JSON.stringify(entry, null, 2)}\n`, 'utf-8');
+    }
+    size() {
+        return this.loaded.size;
+    }
+    entries() {
+        return Array.from(this.loaded.values()).sort((left, right) => left.key.localeCompare(right.key));
+    }
+    /**
+     * A replay-only runtime with no configured provider still needs the identity
+     * that was used to derive cassette keys. Keep that identity in the cassette
+     * store rather than guessing a provider from user settings.
+     */
+    providerNames() {
+        return Array.from(new Set(Array.from(this.loaded.values())
+            .map((entry) => entry.providerName.trim())
+            .filter(Boolean))).sort();
+    }
+}
+export function cassetteEvidenceSummary(store) {
+    const entries = store.entries();
+    const recordedProviderEntries = entries.filter((entry) => entry.provenance?.kind === 'recorded_provider').length;
+    const migratedLegacyDeterministicFixtureEntries = entries.filter((entry) => entry.provenance?.kind === 'migrated_legacy_deterministic_fixture').length;
+    const syntheticDeterministicOrchestrationFixtureEntries = entries.filter((entry) => entry.provenance?.kind === 'synthetic_deterministic_orchestration_fixture').length;
+    const unknownProvenanceEntries = entries.length
+        - recordedProviderEntries
+        - migratedLegacyDeterministicFixtureEntries
+        - syntheticDeterministicOrchestrationFixtureEntries;
+    const realProviderQualityExclusionReasons = [
+        ...(migratedLegacyDeterministicFixtureEntries > 0 ? ['migrated_legacy_deterministic_fixture'] : []),
+        ...(syntheticDeterministicOrchestrationFixtureEntries > 0 ? ['synthetic_deterministic_orchestration_fixture'] : []),
+        ...(unknownProvenanceEntries > 0 ? ['unknown_cassette_provenance'] : []),
+        ...(entries.length === 0 ? ['no_cassette_entries'] : []),
+    ];
+    return {
+        totalEntries: entries.length,
+        recordedProviderEntries,
+        migratedLegacyDeterministicFixtureEntries,
+        syntheticDeterministicOrchestrationFixtureEntries,
+        unknownProvenanceEntries,
+        orchestrationReplayEligible: entries.length > 0,
+        realProviderQualityEligible: realProviderQualityExclusionReasons.length === 0,
+        realProviderQualityExclusionReasons,
+    };
+}
+export class CassetteMissError extends Error {
+    key;
+    fingerprintDiagnostics;
+    constructor(key, operation, fingerprintDiagnostics) {
+        const safeDiagnostics = fingerprintDiagnostics
+            ? ` Safe V2 dispatch facts: messages=${fingerprintDiagnostics.messageCount}; roles=${fingerprintDiagnostics.messageRoles.join(',')}; pre=${fingerprintDiagnostics.preCanonicalHash}; post=${fingerprintDiagnostics.postCanonicalHash}; rules=${fingerprintDiagnostics.appliedRuleClasses.join(',') || 'none'}.`
+            : '';
+        super(`No cassette for ${operation} (${key}). CI replays recorded provider responses and never calls a live model; `
+            + 're-record with --record against a real provider and commit the new cassettes.'
+            + safeDiagnostics);
+        this.name = 'CassetteMissError';
+        this.key = key;
+        this.fingerprintDiagnostics = fingerprintDiagnostics;
+    }
+}
+/**
+ * Wrap a provider so its dispatches are recorded or replayed.
+ *
+ * On replay of a tool loop the recorded tool calls are RE-INVOKED against the
+ * live tools rather than stubbed. The model's choices stay frozen (that is the
+ * point), but compilation, validation, and execution still really happen — so
+ * the suite keeps catching a broken compiler or a wrong number, which a fully
+ * stubbed replay would sail straight past.
+ */
+export function withCassette(provider, store, mode, canonicalization) {
+    if (mode === 'live')
+        return provider;
+    const wrapped = {
+        name: provider.name,
+        available: () => (mode === 'replay' ? Promise.resolve(true) : provider.available()),
+        async generate(messages, options) {
+            const fingerprint = cassetteFingerprint({ providerName: provider.name, operation: 'generate', messages, options, canonicalization });
+            // V2 dispatch identity is deliberately exact. A V1 key can describe a
+            // different full runtime envelope (for example, direct runner versus
+            // runtime runner); silently falling back would replay a response for the
+            // wrong prompt. Migrations must create an explicitly labelled V2 entry.
+            const hit = store.get(fingerprint.key);
+            if (hit)
+                return hit.text;
+            if (mode === 'replay')
+                throw new CassetteMissError(fingerprint.key, 'generate', fingerprint.diagnostics);
+            const text = await provider.generate(messages, options);
+            store.put({
+                key: fingerprint.key, operation: 'generate', text,
+                providerName: provider.name, recordedAt: new Date().toISOString(),
+                provenance: {
+                    kind: 'recorded_provider',
+                    replayClassification: 'recorded_provider',
+                    providerQuality: 'eligible',
+                },
+                ...(fingerprint.diagnostics ? { fingerprintDiagnostics: fingerprint.diagnostics } : {}),
+            });
+            return text;
+        },
+    };
+    if (provider.generateWithTools) {
+        wrapped.generateWithTools = async (messages, tools, options) => {
+            const fingerprint = cassetteFingerprint({
+                providerName: provider.name,
+                operation: 'generate_with_tools',
+                messages,
+                toolNames: tools.map((tool) => tool.name),
+                options: options,
+                canonicalization,
+            });
+            // See `generate`: never alias a V2 tool-loop dispatch to a legacy key.
+            const hit = store.get(fingerprint.key);
+            if (hit) {
+                const byName = new Map(tools.map((tool) => [tool.name, tool]));
+                for (const call of hit.toolCalls ?? []) {
+                    const tool = byName.get(call.name);
+                    // A cassette recorded against a tool set that no longer has this tool
+                    // is stale in a way the run should surface, not paper over.
+                    if (!tool)
+                        continue;
+                    await tool.run(call.args).catch(() => undefined);
+                }
+                return hit.text;
+            }
+            if (mode === 'replay')
+                throw new CassetteMissError(fingerprint.key, 'generate_with_tools', fingerprint.diagnostics);
+            const observed = [];
+            const recordingTools = tools.map((tool) => ({
+                ...tool,
+                run: async (args) => {
+                    observed.push({ name: tool.name, args });
+                    return tool.run(args);
+                },
+            }));
+            const text = await provider.generateWithTools(messages, recordingTools, options);
+            store.put({
+                key: fingerprint.key, operation: 'generate_with_tools', text, toolCalls: observed,
+                providerName: provider.name, recordedAt: new Date().toISOString(),
+                provenance: {
+                    kind: 'recorded_provider',
+                    replayClassification: 'recorded_provider',
+                    providerQuality: 'eligible',
+                },
+                ...(fingerprint.diagnostics ? { fingerprintDiagnostics: fingerprint.diagnostics } : {}),
+            });
+            return text;
+        };
+    }
+    return wrapped;
+}
+/**
+ * Pick the cassette mode from the environment.
+ *
+ * Defaults to `replay` for ANY value that is not explicitly `record` or `live`,
+ * including a missing or misspelled one. If someone sets the cassette directory
+ * in CI and forgets the mode, the safe outcome is "never call a live model",
+ * not a surprise bill and a non-deterministic suite.
+ */
+export function resolveCassetteModeFromEnv(env) {
+    const raw = env.DQL_EVAL_CASSETTE_MODE;
+    return raw === 'record' || raw === 'live' ? raw : 'replay';
+}
+/** Where a project's eval cassettes live. */
+export function cassetteDirFor(projectRoot, suiteName) {
+    const safe = suiteName.replace(/[^a-z0-9._-]+/gi, '-').replace(/\.(ya?ml)$/i, '');
+    return join(projectRoot, 'test-cassettes', safe);
+}
+export { dirname as _dirname };
+//# sourceMappingURL=agent-eval-cassette.js.map

package/dist/commands/agent-eval-cassette.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-cassette.js","sourceRoot":"","sources":["../../src/commands/agent-eval-cassette.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAoH1C,MAAM,mCAAmC,GAAG;IAC1C,iBAAiB;IACjB,QAAQ;IACR,aAAa;IACb,SAAS;IACT,mBAAmB;IACnB,cAAc;IACd,cAAc;CACN,CAAC;AAEX,MAAM,0CAA0C,GAAG,mCAAmC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEjG;;;;;;GAMG;AACH,SAAS,gCAAgC,CAAC,OAAe,EAAE,WAAmB;IAI5E,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,iBAAiB,GAAG,CAAC,MAAc,EAAE,MAAc,EAAE,KAAa,EAAU,EAAE;QAClF,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACrD,OAAO,GAAG,MAAM,2BAA2B,CAAC;IAC9C,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,CAAC,MAAc,EAAE,MAAc,EAAE,KAAa,EAAE,MAAc,EAAU,EAAE;QACjG,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACrD,OAAO,GAAG,MAAM,8BAA8B,MAAM,EAAE,CAAC;IACzD,CAAC,CAAC;IACF,IAAI,UAAU,GAAG,OAAO;SACrB,OAAO,CACN,IAAI,MAAM,CAAC,cAAc,0CAA0C,wBAAwB,EAAE,KAAK,CAAC,EACnG,CAAC,MAAM,EAAE,MAAc,EAAE,KAAa,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,EAAE,GAAG,MAAM,GAAG,KAAK,IAAI,EAAE,KAAK,CAAC,CACnG;SACA,OAAO,CACN,IAAI,MAAM,CAAC,cAAc,0CAA0C,yCAAyC,EAAE,KAAK,CAAC,EACpH,CAAC,MAAM,EAAE,MAAc,EAAE,KAAa,EAAE,MAAc,EAAE,EAAE,CAAC,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAC3G,CAAC;IACJ,IAAI,WAAW,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACpD,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AACjF,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,WAAmB;IAChE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,iBAAiB,CAAC,KAM1B,EAAE,QAAwB,EAAE,OAAW;IACtC,OAAO;QACL,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClC,CAAC,EAAE,KAAK,CAAC,YAAY;QACrB,CAAC,EAAE,KAAK,CAAC,SAAS;QAClB,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,mEAAmE;QACnE,qEAAqE;QACrE,mCAAmC;QACnC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;QAC/E,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,eAAe,IAAI,IAAI;QACzC,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI;QACnC,CAAC,EAAE,KAAK,CAAC,OAAO,EAAE,WAAW,IAAI,IAAI;KACtC,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAOnC;IACC,MAAM,UAAU,GAAG,YAAY,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1E,IAAI,CAAC,KAAK,CAAC,gBAAgB;QAAE,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAErE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9C,kEAAkE;QAClE,wEAAwE;QACxE,+DAA+D;QAC/D,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,OAAO,CAAC;QAC5C,MAAM,UAAU,GAAG,gCAAgC,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,gBAAiB,CAAC,WAAW,CAAC,CAAC;QAC1G,KAAK,MAAM,IAAI,IAAI,UAAU,CAAC,kBAAkB;YAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpE,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;IACrD,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,YAAY,CAAC,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IACrE,OAAO;QACL,GAAG,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAC1B,SAAS,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QAClC,WAAW,EAAE;YACX,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM;YACnC,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;YAC3D,gBAAgB,EAAE,YAAY,CAAC,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAC3E,iBAAiB,EAAE,QAAQ;YAC3B,kBAAkB,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC/C;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAgD;IAC1E,OAAO,mBAAmB,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC;AACxC,CAAC;AAED,MAAM,OAAO,aAAa;IACP,GAAG,CAAS;IACZ,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAE3D,YAAY,GAAW;QACrB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO;QAC7B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAkB,CAAC;gBAClF,IAAI,KAAK,EAAE,GAAG;oBAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,sEAAsE;gBACtE,mEAAmE;YACrE,CAAC;QACH,CAAC;IACH,CAAC;IAED,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IAED,GAAG,CAAC,KAAoB;QACtB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAClC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,OAAO,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrG,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACnG,CAAC;IAED;;;;OAIG;IACH,aAAa;QACX,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CACvB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;aAC7B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;aACzC,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;CACF;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAoB;IAC1D,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC;IAChC,MAAM,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,mBAAmB,CAAC,CAAC,MAAM,CAAC;IACjH,MAAM,yCAAyC,GAAG,OAAO,CAAC,MAAM,CAC9D,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,uCAAuC,CAC9E,CAAC,MAAM,CAAC;IACT,MAAM,iDAAiD,GAAG,OAAO,CAAC,MAAM,CACtE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,KAAK,+CAA+C,CACtF,CAAC,MAAM,CAAC;IACT,MAAM,wBAAwB,GAAG,OAAO,CAAC,MAAM;UAC3C,uBAAuB;UACvB,yCAAyC;UACzC,iDAAiD,CAAC;IACtD,MAAM,mCAAmC,GAAG;QAC1C,GAAG,CAAC,yCAAyC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,uCAAuC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnG,GAAG,CAAC,iDAAiD,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,+CAA+C,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnH,GAAG,CAAC,wBAAwB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KACzD,CAAC;IACF,OAAO;QACL,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,uBAAuB;QACvB,yCAAyC;QACzC,iDAAiD;QACjD,wBAAwB;QACxB,2BAA2B,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;QAC/C,2BAA2B,EAAE,mCAAmC,CAAC,MAAM,KAAK,CAAC;QAC7E,mCAAmC;KACpC,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IACjC,GAAG,CAAS;IACZ,sBAAsB,CAAoC;IACnE,YAAY,GAAW,EAAE,SAAiB,EAAE,sBAAyD;QACnG,MAAM,eAAe,GAAG,sBAAsB;YAC5C,CAAC,CAAC,qCAAqC,sBAAsB,CAAC,YAAY,WAAW,sBAAsB,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,sBAAsB,CAAC,gBAAgB,UAAU,sBAAsB,CAAC,iBAAiB,WAAW,sBAAsB,CAAC,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,MAAM,GAAG;YAC/S,CAAC,CAAC,EAAE,CAAC;QACP,KAAK,CACH,mBAAmB,SAAS,KAAK,GAAG,0EAA0E;cAC5G,+EAA+E;cAC/E,eAAe,CAClB,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAC;IACvD,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC1B,QAAuB,EACvB,KAAoB,EACpB,IAAkB,EAClB,gBAAiD;IAEjD,IAAI,IAAI,KAAK,MAAM;QAAE,OAAO,QAAQ,CAAC;IAErC,MAAM,OAAO,GAAkB;QAC7B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAEnF,KAAK,CAAC,QAAQ,CAAC,QAAwB,EAAE,OAA4B;YACnE,MAAM,WAAW,GAAG,mBAAmB,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;YACrI,sEAAsE;YACtE,qEAAqE;YACrE,yEAAyE;YACzE,wEAAwE;YACxE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,GAAG;gBAAE,OAAO,GAAG,CAAC,IAAI,CAAC;YACzB,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,IAAI,iBAAiB,CAAC,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;YACzG,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACxD,KAAK,CAAC,GAAG,CAAC;gBACR,GAAG,EAAE,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI;gBACjD,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACjE,UAAU,EAAE;oBACV,IAAI,EAAE,mBAAmB;oBACzB,oBAAoB,EAAE,mBAAmB;oBACzC,eAAe,EAAE,UAAU;iBAC5B;gBACD,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxF,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;IAEF,IAAI,QAAQ,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,CAAC,iBAAiB,GAAG,KAAK,EAC/B,QAAwB,EACxB,KAA4B,EAC5B,OAAiC,EAChB,EAAE;YACnB,MAAM,WAAW,GAAG,mBAAmB,CAAC;gBACtC,YAAY,EAAE,QAAQ,CAAC,IAAI;gBAC3B,SAAS,EAAE,qBAAqB;gBAChC,QAAQ;gBACR,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;gBACzC,OAAO,EAAE,OAAoD;gBAC7D,gBAAgB;aACjB,CAAC,CAAC;YACH,uEAAuE;YACvE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;gBAC/D,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;oBACvC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACnC,sEAAsE;oBACtE,4DAA4D;oBAC5D,IAAI,CAAC,IAAI;wBAAE,SAAS;oBACpB,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACnD,CAAC;gBACD,OAAO,GAAG,CAAC,IAAI,CAAC;YAClB,CAAC;YACD,IAAI,IAAI,KAAK,QAAQ;gBAAE,MAAM,IAAI,iBAAiB,CAAC,WAAW,CAAC,GAAG,EAAE,qBAAqB,EAAE,WAAW,CAAC,WAAW,CAAC,CAAC;YACpH,MAAM,QAAQ,GAAuB,EAAE,CAAC;YACxC,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC1C,GAAG,IAAI;gBACP,GAAG,EAAE,KAAK,EAAE,IAAa,EAAE,EAAE;oBAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;aACF,CAAC,CAAC,CAAC;YACJ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,iBAAkB,CAAC,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC;YAClF,KAAK,CAAC,GAAG,CAAC;gBACR,GAAG,EAAE,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ;gBACjF,YAAY,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACjE,UAAU,EAAE;oBACV,IAAI,EAAE,mBAAmB;oBACzB,oBAAoB,EAAE,mBAAmB;oBACzC,eAAe,EAAE,UAAU;iBAC5B;gBACD,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,sBAAsB,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxF,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,0BAA0B,CAAC,GAAuC;IAChF,MAAM,GAAG,GAAG,GAAG,CAAC,sBAAsB,CAAC;IACvC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;AAC7D,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,cAAc,CAAC,WAAmB,EAAE,SAAiB;IACnE,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IAClF,OAAO,IAAI,CAAC,WAAW,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AACnD,CAAC;AAED,OAAO,EAAE,OAAO,IAAI,QAAQ,EAAE,CAAC"}

package/dist/commands/agent-eval-runtime.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Runtime driver for `dql agent eval`.
+ *
+ * The in-process driver calls `answer()` directly, which skips
+ * `createHybridRouter`, `AgentRunEngine`, `enforceOrdinaryAnalyticalPlanBoundary`
+ * and `defaultAgentRunGates`. Every dead-end this harness exists to measure lives
+ * in that skipped half — the router can block before the cascade runs, the engine
+ * synthesizes its own modeling-gap block, and the gates declare refusal codes
+ * terminal. Measuring the answer loop alone reports a cleaner picture than users
+ * experience.
+ *
+ * This driver posts the same body the Ask panel posts and scores the persisted
+ * `AgentRun`, so the suite exercises routing, gates, and the transport projection
+ * as a real end-to-end contract.
+ */
+import type { AgentAnswer, AgentRun, AgentRunRoute, AgentRunStatus, AgentRunTrustState } from '@duckcodeailabs/dql-agent';
+/** Eval-facing view of a run, shaped like the fields the scorer already reads. */
+export interface RuntimeDrivenRun {
+    kind: 'certified' | 'uncertified' | 'no_answer';
+    route?: 'certified' | 'generated_sql' | 'research' | 'clarify' | 'blocked';
+    runRoute: AgentRunRoute;
+    status: AgentRunStatus;
+    trustState: AgentRunTrustState;
+    answer?: string;
+    proposedSql?: string;
+    rows?: unknown[];
+    clarificationOptionCount: number;
+    refusalCode?: string;
+    runId: string;
+    /**
+     * A conversational reply (greeting, capability, polite redirect). It asserts
+     * nothing about the data, so it is neither an answer nor a refusal — scoring it
+     * as either misreads the correct outcome for an out-of-scope question.
+     */
+    conversational: boolean;
+    /**
+     * Did the meaning resolver actually run for this turn?
+     *
+     * With no provider configured it cannot, and `mayAssumeInterpretation` goes
+     * false (AGT-017) — DQL deliberately refuses to settle `booked_revenue` vs
+     * `billed_revenue` by lexical rank with semantic judgment switched off. Every
+     * ambiguous question then clarifies. That is correct behaviour, but it makes a
+     * clarification rate measured without a provider meaningless, so the harness
+     * has to be able to say so.
+    */
+    meaningResolved: boolean;
+    /**
+     * Count captured by the persisted router retrieval receipt. This is not a
+     * synthetic context-pack size: runtime runs do not return an AgentAnswer
+     * context pack through the transport projection.
+     */
+    retrievalCandidateCount?: number;
+    /** Source-lane coverage retained by the router-owned cascade receipt. */
+    sourceCoverage?: NonNullable<AgentRun['diagnosticReceiptV3']>['sourceCoverage'];
+    /** Typed terminal authority, distinct from a user-facing clarification. */
+    terminalOutcome?: NonNullable<AgentRun['routeDecision']>['terminalOutcome'];
+    /** Provider/tool count recorded by persisted runtime telemetry. */
+    toolCallCount: number;
+}
+/**
+ * Did this run decline to produce a usable answer?
+ *
+ * `needs_review` is deliberately NOT a refusal: a review-required generated
+ * answer is the intended outcome when no governed plan froze, and counting it as
+ * a refusal would make the false-refusal metric punish the very behaviour the
+ * cascade is supposed to produce.
+ */
+export declare function runtimeRunRefused(run: Pick<AgentRun, 'route' | 'status'>): boolean;
+/** Map an engine route onto the vocabulary the eval cases already use. */
+export declare function evalRouteForRun(route: AgentRunRoute): RuntimeDrivenRun['route'];
+/**
+ * Collapse a run into the coarse `kind` the existing expectations assert.
+ * Certification follows TRUST, not route: a certified route that degraded to a
+ * review-required answer must not be scored as certified.
+ */
+export declare function evalKindForRun(run: Pick<AgentRun, 'route' | 'status' | 'trustState'>): RuntimeDrivenRun['kind'];
+/** Pull SQL and result rows out of the run's artifacts, when it produced any. */
+export declare function runtimeRunOutputs(run: AgentRun): {
+    proposedSql?: string;
+    rows?: unknown[];
+};
+export declare function projectRuntimeRun(run: AgentRun): RuntimeDrivenRun;
+export interface RuntimeDriverOptions {
+    runtimeBase: string;
+    question: string;
+    requestedMode?: 'ask' | 'auto' | 'research';
+    threadId?: string;
+    timeoutMs?: number;
+    fetchImpl?: typeof fetch;
+}
+/**
+ * Post one question to a running `dql serve` and return the persisted run.
+ *
+ * Failures are surfaced rather than swallowed: a harness that silently scores a
+ * transport error as a refusal would report a false-refusal spike that no code
+ * change caused.
+ */
+export declare function driveViaRuntime(options: RuntimeDriverOptions): Promise<AgentRun>;
+/**
+ * Adapt a persisted run into the `AgentAnswer` shape the existing scorer reads,
+ * so both drivers share one set of assertions.
+ *
+ * Fields the run genuinely does not carry are left UNDEFINED rather than
+ * defaulted. An expectation that references one then fails loudly instead of
+ * passing against a fabricated value — a harness that quietly invents
+ * `sourceTier` would report agreement it never observed.
+ */
+export declare function answerFromRuntimeRun(run: AgentRun): AgentAnswer;
+//# sourceMappingURL=agent-eval-runtime.d.ts.map

package/dist/commands/agent-eval-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-runtime.d.ts","sourceRoot":"","sources":["../../src/commands/agent-eval-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE1H,kFAAkF;AAClF,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,WAAW,GAAG,aAAa,GAAG,WAAW,CAAC;IAChD,KAAK,CAAC,EAAE,WAAW,GAAG,eAAe,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;IAC3E,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;IACjB,wBAAwB,EAAE,MAAM,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,cAAc,EAAE,OAAO,CAAC;IACxB;;;;;;;;;MASE;IACF,eAAe,EAAE,OAAO,CAAC;IACzB;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,yEAAyE;IACzE,cAAc,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC;IAChF,2EAA2E;IAC3E,eAAe,CAAC,EAAE,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAC5E,mEAAmE;IACnE,aAAa,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,QAAQ,CAAC,GAAG,OAAO,CAKlF;AAED,0EAA0E;AAC1E,wBAAgB,eAAe,CAAC,KAAK,EAAE,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAa/E;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,YAAY,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAI/G;AAYD,iFAAiF;AACjF,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAA;CAAE,CAe3F;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,QAAQ,GAAG,gBAAgB,CAuBjE;AAED,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,UAAU,CAAC;IAC5C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,CA8BtF;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,QAAQ,GAAG,WAAW,CAuB/D"}

package/dist/commands/agent-eval-runtime.js

@@ -0,0 +1,165 @@
+/**
+ * Did this run decline to produce a usable answer?
+ *
+ * `needs_review` is deliberately NOT a refusal: a review-required generated
+ * answer is the intended outcome when no governed plan froze, and counting it as
+ * a refusal would make the false-refusal metric punish the very behaviour the
+ * cascade is supposed to produce.
+ */
+export function runtimeRunRefused(run) {
+    return run.route === 'blocked'
+        || run.route === 'clarify'
+        || run.status === 'blocked'
+        || run.status === 'needs_clarification';
+}
+/** Map an engine route onto the vocabulary the eval cases already use. */
+export function evalRouteForRun(route) {
+    switch (route) {
+        case 'certified_answer': return 'certified';
+        case 'semantic_answer': return 'certified';
+        case 'generated_answer': return 'generated_sql';
+        case 'research': return 'research';
+        case 'clarify': return 'clarify';
+        // A typed terminal modeling/policy gap is not a clarification. Collapsing
+        // it to `clarify` made eval reports claim the user had a choice when the
+        // persisted run explicitly recorded a blocked no-answer.
+        case 'blocked': return 'blocked';
+        default: return undefined;
+    }
+}
+/**
+ * Collapse a run into the coarse `kind` the existing expectations assert.
+ * Certification follows TRUST, not route: a certified route that degraded to a
+ * review-required answer must not be scored as certified.
+ */
+export function evalKindForRun(run) {
+    if (runtimeRunRefused(run))
+        return 'no_answer';
+    if (run.trustState === 'certified')
+        return 'certified';
+    return 'uncertified';
+}
+function firstArtifactPayload(run, kinds) {
+    for (const artifact of run.artifacts ?? []) {
+        if (!kinds.includes(String(artifact.kind)))
+            continue;
+        if (artifact.payload && typeof artifact.payload === 'object') {
+            return artifact.payload;
+        }
+    }
+    return undefined;
+}
+/** Pull SQL and result rows out of the run's artifacts, when it produced any. */
+export function runtimeRunOutputs(run) {
+    const payload = firstArtifactPayload(run, ['sql_cell', 'dql_block', 'generated_answer', 'answer', 'result']);
+    const sql = typeof payload?.sql === 'string'
+        ? payload.sql
+        : typeof payload?.compiledSql === 'string' ? payload.compiledSql : undefined;
+    const resultRecord = payload?.result && typeof payload.result === 'object'
+        ? payload.result
+        : undefined;
+    const rows = Array.isArray(resultRecord?.rows)
+        ? resultRecord.rows
+        : Array.isArray(payload?.rows) ? payload.rows : undefined;
+    return {
+        ...(sql ? { proposedSql: sql } : {}),
+        ...(rows ? { rows } : {}),
+    };
+}
+export function projectRuntimeRun(run) {
+    const refusalEvaluation = (run.evaluations ?? []).find((evaluation) => !evaluation.passed);
+    const sourceCoverage = run.diagnosticReceiptV3?.sourceCoverage
+        ?? run.routeDecision?.analyticalCascadeDecision?.sourceCoverage;
+    const retrievalCandidateCount = run.routeDecision?.retrievalEvidence?.candidateCount;
+    return {
+        kind: evalKindForRun(run),
+        route: evalRouteForRun(run.route),
+        runRoute: run.route,
+        status: run.status,
+        trustState: run.trustState,
+        clarificationOptionCount: run.clarificationOptions?.length ?? 0,
+        runId: run.id,
+        conversational: run.route === 'conversation' || run.answerKind === 'conversational',
+        meaningResolved: Boolean(run.routeDecision?.meaningResolution),
+        toolCallCount: run.telemetry?.toolCalls ?? 0,
+        ...(typeof retrievalCandidateCount === 'number' ? { retrievalCandidateCount } : {}),
+        ...(sourceCoverage ? { sourceCoverage } : {}),
+        ...(run.routeDecision?.terminalOutcome ? { terminalOutcome: run.routeDecision.terminalOutcome } : {}),
+        ...(run.answer ? { answer: run.answer } : {}),
+        ...(refusalEvaluation?.id ? { refusalCode: refusalEvaluation.id } : {}),
+        ...runtimeRunOutputs(run),
+    };
+}
+/**
+ * Post one question to a running `dql serve` and return the persisted run.
+ *
+ * Failures are surfaced rather than swallowed: a harness that silently scores a
+ * transport error as a refusal would report a false-refusal spike that no code
+ * change caused.
+ */
+export async function driveViaRuntime(options) {
+    const base = options.runtimeBase.replace(/\/$/, '');
+    const fetchImpl = options.fetchImpl ?? fetch;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? 120_000);
+    try {
+        const response = await fetchImpl(`${base}/api/agent-runs`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                question: options.question,
+                requestedMode: options.requestedMode ?? 'ask',
+                ...(options.threadId ? { threadId: options.threadId } : {}),
+            }),
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Runtime returned ${response.status}: ${(await response.text()).slice(0, 400)}`);
+        }
+        // The endpoint wraps the record (`{ run }`); older/streamed paths return it
+        // bare. Accept both rather than guessing one and failing the whole suite.
+        const body = await response.json();
+        const run = body.run ?? body;
+        if (!run || typeof run.id !== 'string') {
+            throw new Error(`Runtime response was not an agent run: ${JSON.stringify(body).slice(0, 200)}`);
+        }
+        return run;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/**
+ * Adapt a persisted run into the `AgentAnswer` shape the existing scorer reads,
+ * so both drivers share one set of assertions.
+ *
+ * Fields the run genuinely does not carry are left UNDEFINED rather than
+ * defaulted. An expectation that references one then fails loudly instead of
+ * passing against a fabricated value — a harness that quietly invents
+ * `sourceTier` would report agreement it never observed.
+ */
+export function answerFromRuntimeRun(run) {
+    const projected = projectRuntimeRun(run);
+    const certification = projected.kind === 'no_answer'
+        ? 'analyst_review_required'
+        : run.trustState === 'certified'
+            ? 'certified'
+            : run.trustState === 'review_required'
+                ? 'analyst_review_required'
+                : 'ai_generated';
+    return {
+        kind: projected.kind,
+        certification,
+        reviewStatus: run.trustState === 'review_required' ? 'analyst_review_required' : 'none',
+        text: run.answer ?? run.summary,
+        citations: [],
+        considered: [],
+        ...(projected.proposedSql ? { proposedSql: projected.proposedSql } : {}),
+        ...(projected.rows ? { result: { columns: [], rows: projected.rows, rowCount: projected.rows.length } } : {}),
+        ...(projected.refusalCode ? { refusalCode: projected.refusalCode } : {}),
+        // Carried so the scorer can tell an answerable clarification from a dead end.
+        ...(run.clarificationOptions?.length ? { clarificationOptions: run.clarificationOptions } : {}),
+        ...(projected.conversational ? { answerKind: 'conversational' } : {}),
+    };
+}
+//# sourceMappingURL=agent-eval-runtime.js.map

package/dist/commands/agent-eval-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-eval-runtime.js","sourceRoot":"","sources":["../../src/commands/agent-eval-runtime.ts"],"names":[],"mappings":"AA6DA;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAuC;IACvE,OAAO,GAAG,CAAC,KAAK,KAAK,SAAS;WACzB,GAAG,CAAC,KAAK,KAAK,SAAS;WACvB,GAAG,CAAC,MAAM,KAAK,SAAS;WACxB,GAAG,CAAC,MAAM,KAAK,qBAAqB,CAAC;AAC5C,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,eAAe,CAAC,KAAoB;IAClD,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,kBAAkB,CAAC,CAAC,OAAO,WAAW,CAAC;QAC5C,KAAK,iBAAiB,CAAC,CAAC,OAAO,WAAW,CAAC;QAC3C,KAAK,kBAAkB,CAAC,CAAC,OAAO,eAAe,CAAC;QAChD,KAAK,UAAU,CAAC,CAAC,OAAO,UAAU,CAAC;QACnC,KAAK,SAAS,CAAC,CAAC,OAAO,SAAS,CAAC;QACjC,0EAA0E;QAC1E,yEAAyE;QACzE,yDAAyD;QACzD,KAAK,SAAS,CAAC,CAAC,OAAO,SAAS,CAAC;QACjC,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC;IAC5B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAsD;IACnF,IAAI,iBAAiB,CAAC,GAAG,CAAC;QAAE,OAAO,WAAW,CAAC;IAC/C,IAAI,GAAG,CAAC,UAAU,KAAK,WAAW;QAAE,OAAO,WAAW,CAAC;IACvD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,oBAAoB,CAAC,GAAa,EAAE,KAAe;IAC1D,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAAE,SAAS;QACrD,IAAI,QAAQ,CAAC,OAAO,IAAI,OAAO,QAAQ,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC7D,OAAO,QAAQ,CAAC,OAAkC,CAAC;QACrD,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,iBAAiB,CAAC,GAAa;IAC7C,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IAC7G,MAAM,GAAG,GAAG,OAAO,OAAO,EAAE,GAAG,KAAK,QAAQ;QAC1C,CAAC,CAAC,OAAO,CAAC,GAAG;QACb,CAAC,CAAC,OAAO,OAAO,EAAE,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/E,MAAM,YAAY,GAAG,OAAO,EAAE,MAAM,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ;QACxE,CAAC,CAAC,OAAO,CAAC,MAAiC;QAC3C,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;QAC5C,CAAC,CAAC,YAAY,CAAC,IAAiB;QAChC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAiB,CAAC,CAAC,CAAC,SAAS,CAAC;IACzE,OAAO;QACL,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAa;IAC7C,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3F,MAAM,cAAc,GAAG,GAAG,CAAC,mBAAmB,EAAE,cAAc;WACzD,GAAG,CAAC,aAAa,EAAE,yBAAyB,EAAE,cAAc,CAAC;IAClE,MAAM,uBAAuB,GAAG,GAAG,CAAC,aAAa,EAAE,iBAAiB,EAAE,cAAc,CAAC;IACrF,OAAO;QACL,IAAI,EAAE,cAAc,CAAC,GAAG,CAAC;QACzB,KAAK,EAAE,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC;QACjC,QAAQ,EAAE,GAAG,CAAC,KAAK;QACnB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,wBAAwB,EAAE,GAAG,CAAC,oBAAoB,EAAE,MAAM,IAAI,CAAC;QAC/D,KAAK,EAAE,GAAG,CAAC,EAAE;QACb,cAAc,EAAE,GAAG,CAAC,KAAK,KAAK,cAAc,IAAI,GAAG,CAAC,UAAU,KAAK,gBAAgB;QACnF,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,aAAa,EAAE,iBAAiB,CAAC;QAC9D,aAAa,EAAE,GAAG,CAAC,SAAS,EAAE,SAAS,IAAI,CAAC;QAC5C,GAAG,CAAC,OAAO,uBAAuB,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,GAAG,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,GAAG,CAAC,aAAa,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,GAAG,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,iBAAiB,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,iBAAiB,CAAC,GAAG,CAAC;KAC1B,CAAC;AACJ,CAAC;AAWD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA6B;IACjE,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC7C,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC;IACjF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,IAAI,iBAAiB,EAAE;YACzD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,KAAK;gBAC7C,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5D,CAAC;YACF,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACnG,CAAC;QACD,4EAA4E;QAC5E,0EAA0E;QAC1E,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAmC,CAAC;QACpE,MAAM,GAAG,GAAI,IAA2B,CAAC,GAAG,IAAI,IAAgB,CAAC;QACjE,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CAAC,0CAA0C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,GAAa;IAChD,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,aAAa,GAAG,SAAS,CAAC,IAAI,KAAK,WAAW;QAClD,CAAC,CAAC,yBAAyB;QAC3B,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,WAAW;YAC9B,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,GAAG,CAAC,UAAU,KAAK,iBAAiB;gBACpC,CAAC,CAAC,yBAAyB;gBAC3B,CAAC,CAAC,cAAc,CAAC;IACvB,OAAO;QACL,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,aAAa;QACb,YAAY,EAAE,GAAG,CAAC,UAAU,KAAK,iBAAiB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,MAAM;QACvF,IAAI,EAAE,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,OAAO;QAC/B,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,EAAE;QACd,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7G,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,8EAA8E;QAC9E,GAAG,CAAC,GAAG,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/F,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC5C,CAAC;AAC9B,CAAC"}