@duckcodeailabs/dql-cli 1.14.1 → 1.14.3-rc.1

package/dist/commands/agent.js

@@ -21,17 +21,19 @@
  *   dql agent feedback <up|down> --block <id> --question "..."
  *     Records feedback into the KG. Used by clients without MCP access.
  */
-import { answerFromRuntimeRun, driveViaRuntime, evalRouteForRun } from './agent-eval-runtime.js';
-import { CassetteStore, cassetteDirFor, withCassette } from './agent-eval-cassette.js';
+import { answerFromRuntimeRun, driveViaRuntime, projectRuntimeRun, } from './agent-eval-runtime.js';
+import { CassetteStore, cassetteEvidenceSummary, cassetteDirFor, evalCassetteCanonicalizationV2, withCassette, } from './agent-eval-cassette.js';
 import { existsSync, readFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { load as loadYaml } from 'js-yaml';
-import { KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, reindexProject, loadSkills, pickProvider, answer, resolveDomainContextEnvelope, buildAnalysisQuestionPlan, buildLocalContextPack, coerceReasoningEffort, contextRetrievalBudgetForQuestion, deriveGeneratedDraftSlug, loadAgentSemanticLayer, recordQueryRun, recordRuntimeSchemaSnapshot, upsertGeneratedDqlArtifactDraft, upsertGeneratedDraft, validateSqlAgainstLocalContext, } from '@duckcodeailabs/dql-agent';
+import { AskTraceSqliteStoreV1, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, reindexProject, loadSkills, pickProvider, answer, resolveDomainContextEnvelope, buildAnalysisQuestionPlan, buildLocalContextPack, classifyProviderFailure, coerceReasoningEffort, contextRetrievalBudgetForQuestion, deriveGeneratedDraftSlug, loadAgentSemanticLayer, recordQueryRun, recordRuntimeSchemaSnapshot, upsertGeneratedDqlArtifactDraft, upsertGeneratedDraft, validateSqlAgainstLocalContext, createAskTraceObserverV1, defaultAskTraceSqlitePath, } from '@duckcodeailabs/dql-agent';
+import { createHash, randomUUID } from 'node:crypto';
 import { buildManifest, resolveDbtManifestPath } from '@duckcodeailabs/dql-core';
 import { findProjectRoot } from '../local-runtime.js';
 import { buildAnswerLoopTools, createGroundingContextExpander } from '../llm/answer-loop-tools.js';
 import { judgeAnswer } from './eval-judge.js';
 import { startProjectRuntime } from './notebook.js';
+import { runAgentTrace } from './agent-trace.js';
 /**
  * Resolve the runtime the agent posts certified blocks / generated SQL to.
  *
@@ -55,7 +57,56 @@ async function resolveAgentRuntime(projectRoot, flags) {
         return { runtimeBase: base, close: async () => { } };
     }
     const handle = await startProjectRuntime(projectRoot, { preferredPort: 0 });
-    return { runtimeBase: handle.url, close: handle.close };
+    return { runtimeBase: handle.url, close: handle.close, askTraceCapability: handle.askTraceCapability };
+}
+function isLoopbackRuntimeUrl(runtimeBase) {
+    try {
+        const hostname = new URL(runtimeBase).hostname;
+        return hostname === '127.0.0.1' || hostname === 'localhost' || hostname === '::1' || hostname === '[::1]';
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * An already-running local Notebook runtime mints a one-shot, loopback-only
+ * capability for this CLI request. Remote runtimes deliberately receive no
+ * capability and therefore cannot be relabelled as CLI by arbitrary text.
+ */
+export async function requestLoopbackCliAskTraceCapability(runtimeBase) {
+    if (!isLoopbackRuntimeUrl(runtimeBase))
+        return undefined;
+    try {
+        const response = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/ask-traces/cli-capability`, {
+            headers: { Accept: 'application/json' },
+        });
+        if (!response.ok)
+            return undefined;
+        const payload = await response.json();
+        return typeof payload.capability === 'string'
+            && typeof payload.expiresAt === 'string'
+            && payload.scope === 'agent-runs'
+            ? payload.capability
+            : undefined;
+    }
+    catch {
+        // A pre-observability runtime remains usable. Its request stays browser
+        // attributed rather than fabricating a client-controlled CLI surface.
+        return undefined;
+    }
+}
+function runtimeProviderForCliFlag(value) {
+    if (typeof value !== 'string' || !value.trim())
+        return undefined;
+    switch (value.trim().toLowerCase()) {
+        case 'claude': return 'anthropic';
+        case 'openai': return 'openai';
+        case 'gemini': return 'gemini';
+        case 'ollama': return 'ollama';
+        // Preserve an invalid explicit value through the host request so the
+        // canonical preflight can return a typed `model_not_found` diagnostic.
+        default: return value.trim().toLowerCase();
+    }
 }
 async function fetchRuntimeSchemaContext(runtimeBase) {
     try {
@@ -142,6 +193,137 @@ function cliAnalysisDepth(flags) {
     const value = flags.analysisDepth?.trim().toLowerCase();
     return value === 'quick' || value === 'deep' ? value : undefined;
 }
+/**
+ * Compatibility test adapter for a standalone provider boundary. Production
+ * `dql agent ask` no longer calls this: it uses the runtime AgentRun engine so
+ * the router, cascade, freeze, tools, SQL, and provider spans share one
+ * canonical trace. Keep this adapter truthful for lower-level provider tests
+ * without making it an alternate orchestration authority.
+ */
+export function createDirectCliAskTraceProvider(provider, trace) {
+    let attemptIndex = 0;
+    let lastFailedSpanId;
+    const pending = new Map();
+    const keyFor = (event) => `${event.provider}:${event.operation}:${event.attemptIndex}`;
+    const attempt = (event, retryOfSpanId) => ({
+        version: 1,
+        phase: 'generation',
+        physicalAttemptIndex: ++attemptIndex,
+        providerFingerprint: `sha256:${createHash('sha256').update(event.provider).digest('hex')}`,
+        ...(event.model ? { modelFingerprint: `sha256:${createHash('sha256').update(event.model).digest('hex')}` } : {}),
+        ...(retryOfSpanId ? { retryOfSpanId } : {}),
+        admission: 'admitted',
+        provenance: 'live',
+    });
+    const finish = (entry, outcome, error) => {
+        if (!entry.spanId)
+            return;
+        const diagnostic = outcome === 'ok' ? undefined : classifyProviderFailure({
+            phase: 'generation',
+            code: error && typeof error === 'object' ? String(error.code ?? '') : undefined,
+            message: error instanceof Error ? error.message : String(error ?? ''),
+            providerFingerprint: entry.attempt.providerFingerprint,
+            modelFingerprint: entry.attempt.modelFingerprint,
+        });
+        const finalAttempt = outcome === 'ok'
+            ? entry.attempt
+            : {
+                ...entry.attempt,
+                ...(diagnostic?.httpStatusClass ? { httpStatusClass: diagnostic.httpStatusClass } : {}),
+                ...(diagnostic ? { retryable: diagnostic.retryable, safeAction: diagnostic.safeAction } : {}),
+                cause: outcome === 'cancelled' ? 'cancelled' : diagnostic?.cause ?? 'unknown',
+            };
+        trace.finishSpan(entry.spanId, {
+            outcome: outcome === 'ok' ? 'ok' : outcome === 'cancelled' ? 'cancelled' : 'error',
+            reasonCode: outcome === 'ok' ? 'completed' : outcome === 'cancelled' ? 'cancelled' : 'provider_failure',
+            payload: { kind: 'provider', attempt: finalAttempt },
+        });
+        if (outcome !== 'ok')
+            lastFailedSpanId = entry.spanId;
+    };
+    const observedOptions = (options = {}) => ({
+        ...options,
+        onProviderDispatch: (event) => {
+            const tracedAttempt = attempt(event, lastFailedSpanId);
+            const spanId = trace.startSpan({
+                name: 'provider.attempt',
+                stage: 'provider',
+                reasonCode: 'started',
+                payload: { kind: 'provider', attempt: tracedAttempt },
+            });
+            const key = keyFor(event);
+            pending.set(key, [...(pending.get(key) ?? []), { spanId, attempt: tracedAttempt }]);
+            return options.onProviderDispatch?.(event) ?? event.envelope;
+        },
+        onProviderDispatchComplete: (event) => {
+            const key = keyFor(event);
+            const entries = pending.get(key) ?? [];
+            const entry = entries[0];
+            if (entry && event.outcome === 'ok' && (event.settlement === 'transport' || event.settlement === 'process')) {
+                entry.attempt = {
+                    ...entry.attempt,
+                    ...(event.settlement === 'transport' ? { transportOutcome: 'ok' } : { processOutcome: 'ok' }),
+                };
+            }
+            else {
+                const closed = entries.shift();
+                if (entries.length > 0)
+                    pending.set(key, entries);
+                else
+                    pending.delete(key);
+                if (closed)
+                    finish(closed, event.outcome, event.error ?? (typeof event.httpStatus === 'number' ? Object.assign(new Error(`HTTP ${event.httpStatus}`), { code: `HTTP_${event.httpStatus}` }) : undefined));
+            }
+            options.onProviderDispatchComplete?.(event);
+        },
+        onProviderDispatchRejected: (event) => {
+            const denied = trace.startSpan({
+                name: 'provider.attempt',
+                stage: 'provider',
+                reasonCode: 'provider_failure',
+                payload: {
+                    kind: 'provider',
+                    attempt: {
+                        ...attempt(event, lastFailedSpanId),
+                        admission: 'denied',
+                    },
+                },
+            });
+            trace.finishSpan(denied, { outcome: 'denied', reasonCode: 'provider_failure' });
+            lastFailedSpanId = denied;
+            options.onProviderDispatchRejected?.(event);
+        },
+    });
+    const invoke = async (options, call) => {
+        try {
+            const result = await call(observedOptions(options));
+            for (const entries of pending.values())
+                for (const entry of entries)
+                    finish(entry, 'ok');
+            pending.clear();
+            return result;
+        }
+        catch (error) {
+            const outcome = options?.signal?.aborted ? 'cancelled' : 'error';
+            for (const entries of pending.values())
+                for (const entry of entries)
+                    finish(entry, outcome, error);
+            pending.clear();
+            throw error;
+        }
+    };
+    return {
+        name: provider.name,
+        available: () => provider.available(),
+        generate: (messages, options) => invoke(options, (observed) => provider.generate(messages, observed)),
+        ...(provider.generateWithTools ? {
+            generateWithTools: (messages, tools, options) => invoke(options, (observed) => provider.generateWithTools(messages, tools, observed)),
+        } : {}),
+        ...(provider.generateStream ? {
+            generateStream: (messages, options, onDelta) => invoke(options, (observed) => provider.generateStream(messages, observed, onDelta)),
+        } : {}),
+    };
+}
 /** A DQL runtime answers `/api/connections` with a connector/connection payload. */
 async function isDqlRuntime(base) {
     try {
@@ -161,6 +343,8 @@ export async function runAgent(sub, rest, flags) {
             return runAsk(rest, flags);
         case 'threads':
             return runThreads(flags);
+        case 'trace':
+            return runAgentTrace(rest, flags);
         case 'reindex':
             return runReindex(rest, flags);
         case 'feedback':
@@ -168,18 +352,41 @@ export async function runAgent(sub, rest, flags) {
         case 'eval':
             return runEval(rest, flags);
         default:
-            throw new Error('Usage: dql agent <ask|threads|reindex|feedback|eval> [args]\n' +
+            throw new Error('Usage: dql agent <ask|threads|trace|reindex|feedback|eval> [args]\n' +
                 '  dql agent ask "<question>" [--provider claude|openai|gemini|ollama] [--user <id>] [--domain <d>] [--purpose <approved-purpose>] [--thread <id>]\n' +
                 '  dql agent threads [--runtime-url <url>]\n' +
+                '  dql agent trace list|show|export|validate|replay|compare\n' +
                 '  dql agent reindex [path]\n' +
                 '  dql agent feedback up|down --block <id> --question "..."\n' +
                 '  dql agent eval agent-evals.yml [--provider claude|openai|gemini|ollama] [--execute] [--save]');
     }
 }
+/**
+ * Canonical CLI Ask entrypoint. Direct and threaded CLI questions use the
+ * exact local AgentRun engine as the browser, so trace evidence follows the
+ * router's candidate/cascade/freeze/execution authority rather than a legacy
+ * answer-loop approximation.
+ */
 async function runAsk(rest, flags) {
     const question = rest.join(' ').trim();
     if (!question)
         throw new Error('Usage: dql agent ask "<question>"');
+    const threadId = flags.thread;
+    return threadId
+        ? runThreadAsk(question, threadId, flags)
+        : runCanonicalCliAsk(question, undefined, flags);
+}
+/**
+ * Compatibility entrypoint retained for downstream imports during the CLI
+ * transition. It deliberately delegates before allocating any legacy state,
+ * so even an old internal caller gets the canonical AgentRun trace rather than
+ * an incomplete synthetic receipt.
+ */
+async function runLegacyDirectAsk(rest, flags) {
+    const question = rest.join(' ').trim();
+    if (!question)
+        throw new Error('Usage: dql agent ask "<question>"');
+    return runCanonicalCliAsk(question, flags.thread, flags);
     // Thread-scoped ask: hand the question to the runtime's agent-run engine with
     // the thread id, so the SERVER injects prior turns and persists this run as a
     // new turn (the same conversation store the notebook UI uses).
@@ -187,8 +394,31 @@ async function runAsk(rest, flags) {
     if (threadId)
         return runThreadAsk(question, threadId, flags);
     const projectRoot = findProjectRoot(process.cwd());
+    const traceStore = new AskTraceSqliteStoreV1({ path: defaultAskTraceSqlitePath(projectRoot) });
+    const trace = createAskTraceObserverV1({
+        store: traceStore,
+        runId: `cli-${randomUUID()}`,
+        surface: 'cli',
+        mode: 'ask',
+        questionFingerprint: `sha256:${createHash('sha256').update(question).digest('hex')}`,
+    });
+    const classifySpan = trace.startSpan({
+        name: 'request.classify',
+        stage: 'request',
+        reasonCode: 'started',
+        payload: { kind: 'stage', route: 'direct_cli_legacy' },
+    });
     const kgPath = defaultKgPath(projectRoot);
-    await reindexProject(projectRoot, { kgPath });
+    try {
+        await reindexProject(projectRoot, { kgPath });
+        trace.finishSpan(classifySpan, { outcome: 'ok', reasonCode: 'completed' });
+    }
+    catch (error) {
+        trace.finishSpan(classifySpan, { outcome: 'error', reasonCode: 'unknown' });
+        trace.finalize({ status: 'failed' });
+        traceStore.close();
+        throw error;
+    }
     const providerName = flags.provider;
     const userId = flags.user;
     const domain = flags.domain;
@@ -196,12 +426,68 @@ async function runAsk(rest, flags) {
     const format = flags.format;
     const reasoningEffort = cliReasoningEffort(flags);
     const requestedDepth = cliAnalysisDepth(flags);
-    const provider = await pickProvider(providerName);
+    let provider;
+    try {
+        provider = await pickProvider(providerName);
+    }
+    catch (error) {
+        trace.finalize({ status: 'failed' });
+        traceStore.close();
+        throw error;
+    }
     const kg = new KGStore(kgPath);
     const memory = new MemoryStore(defaultMemoryPath(projectRoot));
     const { skills } = loadSkills(projectRoot);
     let closeRuntime;
     try {
+        const preflight = trace.startSpan({
+            name: 'provider.preflight',
+            stage: 'provider',
+            reasonCode: 'started',
+            payload: {
+                kind: 'provider',
+                attempt: {
+                    version: 1,
+                    phase: 'preflight',
+                    physicalAttemptIndex: 0,
+                    providerFingerprint: `sha256:${createHash('sha256').update(provider.name).digest('hex')}`,
+                    readiness: 'unknown',
+                    admission: 'unknown',
+                    provenance: 'live',
+                },
+            },
+        });
+        let providerReady = false;
+        try {
+            providerReady = await provider.available();
+        }
+        catch {
+            providerReady = false;
+        }
+        trace.finishSpan(preflight, {
+            outcome: providerReady ? 'ok' : 'unavailable',
+            reasonCode: providerReady ? 'completed' : 'provider_preflight',
+            payload: {
+                kind: 'provider',
+                attempt: {
+                    version: 1,
+                    phase: 'preflight',
+                    physicalAttemptIndex: 0,
+                    providerFingerprint: `sha256:${createHash('sha256').update(provider.name).digest('hex')}`,
+                    readiness: providerReady ? 'ready' : 'unavailable',
+                    admission: 'unknown',
+                    cause: providerReady ? undefined : 'authentication',
+                    safeAction: providerReady ? undefined : 'fix_provider_configuration',
+                    provenance: 'live',
+                },
+            },
+        });
+        if (!providerReady) {
+            throw Object.assign(new Error('The selected AI provider is not ready. Configure or sign in to the provider and retry.'), {
+                code: 'AUTHENTICATION_FAILED',
+            });
+        }
+        const tracedProvider = createDirectCliAskTraceProvider(provider, trace);
         const memoryContext = memory.search({
             query: question,
             scopes: ['project', 'user', 'artifact'],
@@ -220,6 +506,12 @@ async function runAsk(rest, flags) {
         closeRuntime = close;
         const schemaContext = await fetchRuntimeSchemaContext(runtimeBase);
         recordCliRuntimeSchemaSnapshot(projectRoot, schemaContext, 'direct CLI runtime schema');
+        const retrievalSpan = trace.startSpan({
+            name: 'retrieval',
+            stage: 'retrieval',
+            reasonCode: 'started',
+            payload: { kind: 'retrieval', candidateCount: 0 },
+        });
         const contextPack = await buildLocalContextPack(projectRoot, {
             question,
             surface: 'cli',
@@ -233,10 +525,15 @@ async function runAsk(rest, flags) {
                 }
                 : undefined,
         }).catch(() => undefined);
+        trace.finishSpan(retrievalSpan, {
+            outcome: 'ok',
+            reasonCode: 'completed',
+            payload: { kind: 'retrieval', candidateCount: contextPack?.objects.length ?? 0 },
+        });
         const answerLoopTools = buildAnswerLoopTools(projectRoot);
         const result = await answer({
             question,
-            provider,
+            provider: tracedProvider,
             kg,
             manifest,
             skills,
@@ -365,6 +662,7 @@ async function runAsk(rest, flags) {
                 });
             },
         });
+        trace.finalize({ status: 'completed' });
         if (format === 'json') {
             console.log(JSON.stringify(result, null, 2));
             return;
@@ -379,17 +677,25 @@ async function runAsk(rest, flags) {
             : '';
         const footer = result.provenanceFooter ? `\n\n— ${result.provenanceFooter}` : '';
         console.log(`${badge}\n\n${result.text}${footer}${cite}`);
-        if (result.result) {
-            console.log(`\nRows: ${result.result.rowCount}`);
-            console.log(JSON.stringify(result.result.rows.slice(0, 5), null, 2));
+        const resultPayload = result.result;
+        if (resultPayload) {
+            console.log(`\nRows: ${resultPayload.rowCount}`);
+            console.log(JSON.stringify(resultPayload.rows.slice(0, 5), null, 2));
         }
         printDqlArtifactPreview(result);
     }
+    catch (error) {
+        const cancelled = error instanceof Error && error.name === 'AbortError';
+        trace.finalize({ status: cancelled ? 'cancelled' : 'failed' });
+        throw error;
+    }
     finally {
         kg.close();
         memory.close();
-        if (closeRuntime)
-            await closeRuntime();
+        const close = closeRuntime;
+        if (close)
+            await close();
+        traceStore.close();
     }
 }
 function printDqlArtifactPreview(result) {
@@ -410,56 +716,103 @@ function printDqlArtifactPreview(result) {
             console.log(`Promote: ${result.promoteCommand}`);
     }
 }
+function printCanonicalCliAskRun(run, threadId) {
+    const badge = run.trustState === 'certified'
+        ? '✓ Certified'
+        : run.trustState === 'grounded'
+            ? '✓ Verified (grounded)'
+            : run.trustState === 'review_required'
+                ? '! AI-generated · review required'
+                : run.trustState === 'blocked'
+                    ? '✕ Blocked'
+                    : '· Reply';
+    console.log(`${badge}\n\n${(run.answer ?? run.summary ?? '').trim()}`);
+    if (threadId)
+        console.log(`\nThread: ${threadId}`);
+    const trace = run.traceReference;
+    if (trace?.traceId) {
+        console.log(`\nTrace: ${trace.traceId} (${trace.recordingStatus ?? 'recording'})`);
+    }
+    else {
+        console.log('\nTrace: unavailable');
+    }
+}
 /**
- * `dql agent ask --thread <id>` — POST the question to the runtime's
- * `/api/agent-runs` with the threadId in the body. The server injects the
- * thread's prior turns into the conversation context and records the completed
- * run as the next turn, so follow-ups resolve "those"/"that product" correctly
- * across CLI invocations (and across the notebook UI, which shares the store).
+ * Submit every user-visible CLI Ask through the runtime AgentRun endpoint.
+ * The runtime owns run IDs, conversation hydration, candidate provenance,
+ * cascade/freeze authority, physical tool/SQL execution, and final trace
+ * reference. The CLI only supplies user intent and an optional host-minted
+ * surface capability.
  */
-async function runThreadAsk(question, threadId, flags) {
+export async function runCanonicalCliAsk(question, threadId, flags) {
     const projectRoot = findProjectRoot(process.cwd());
     const format = flags.format;
-    const { runtimeBase, close } = await resolveAgentRuntime(projectRoot, flags);
+    const { runtimeBase, close, askTraceCapability: embeddedCapability } = await resolveAgentRuntime(projectRoot, flags);
     try {
+        const askTraceCapability = embeddedCapability ?? await requestLoopbackCliAskTraceCapability(runtimeBase);
         const reasoningEffort = cliReasoningEffort(flags);
         const analysisDepth = cliAnalysisDepth(flags);
+        const domain = flags.domain;
+        const purpose = flags.purpose || undefined;
+        const userId = flags.user;
+        const provider = runtimeProviderForCliFlag(flags.provider);
+        // `surface` is deliberately absent from public JSON. The runtime assigns
+        // trace surface only after consuming its own loopback capability; this
+        // context carries user intent, never attribution authority.
+        const workspaceContext = {
+            ...(domain ? { domain } : {}),
+            ...(purpose ? { purpose } : {}),
+            ...(userId ? { userId } : {}),
+            ...(provider ? { provider } : {}),
+        };
+        const hasWorkspaceContext = Object.keys(workspaceContext).length > 0;
         const response = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/agent-runs`, {
             method: 'POST',
-            headers: { 'Content-Type': 'application/json' },
+            headers: {
+                'Content-Type': 'application/json',
+                ...(askTraceCapability ? { 'X-DQL-Ask-Trace-Capability': askTraceCapability } : {}),
+            },
             body: JSON.stringify({
                 question,
-                threadId,
+                ...(threadId ? { threadId } : {}),
                 ...(reasoningEffort ? { reasoningEffort } : {}),
                 ...(analysisDepth ? { analysisDepth } : {}),
+                ...(hasWorkspaceContext ? { workspaceContext } : {}),
             }),
         });
         if (!response.ok)
             throw new Error(`Runtime returned ${response.status}: ${await response.text()}`);
         const payload = (await response.json());
-        if (!payload.run)
-            throw new Error('Runtime did not return an agent run.');
+        if (!payload.run?.id)
+            throw new Error('Runtime did not return a canonical agent run.');
+        const run = payload.run;
         if (format === 'json') {
-            console.log(JSON.stringify(payload.run, null, 2));
+            // Preserve the established top-level run shape while adding compact,
+            // content-free trace discoverability for scripts and support bundles.
+            console.log(JSON.stringify({
+                ...run,
+                runId: run.id,
+                ...(run.traceReference?.traceId ? { traceId: run.traceReference.traceId } : {}),
+                traceRecordingStatus: run.traceReference?.recordingStatus ?? 'unavailable',
+            }, null, 2));
             return;
         }
-        const run = payload.run;
-        const badge = run.trustState === 'certified'
-            ? '✓ Certified'
-            : run.trustState === 'grounded'
-                ? '✓ Verified (grounded)'
-                : run.trustState === 'review_required'
-                    ? '! AI-generated · review required'
-                    : run.trustState === 'blocked'
-                        ? '✕ Blocked'
-                        : '· Reply';
-        console.log(`${badge}\n\n${(run.answer ?? run.summary ?? '').trim()}`);
-        console.log(`\nThread: ${threadId}`);
+        printCanonicalCliAskRun(run, threadId);
     }
     finally {
         await close();
     }
 }
+/**
+ * `dql agent ask --thread <id>` — POST the question to the runtime's
+ * `/api/agent-runs` with the threadId in the body. The server injects the
+ * thread's prior turns into the conversation context and records the completed
+ * run as the next turn, so follow-ups resolve "those"/"that product" correctly
+ * across CLI invocations (and across the notebook UI, which shares the store).
+ */
+async function runThreadAsk(question, threadId, flags) {
+    return runCanonicalCliAsk(question, threadId, flags);
+}
 /** `dql agent threads` — list server-persisted conversation threads. */
 async function runThreads(flags) {
     const projectRoot = findProjectRoot(process.cwd());
@@ -559,7 +912,7 @@ async function runEval(rest, flags) {
     // started with DQL_EVAL_CASSETTE_DIR, since it owns its own provider.
     const cassetteMode = flags.cassette;
     const provider = cassetteMode === 'record' || cassetteMode === 'replay'
-        ? withCassette(rawProvider, new CassetteStore(cassetteDirFor(projectRoot, rest[0] ?? 'agent-evals')), cassetteMode)
+        ? withCassette(rawProvider, new CassetteStore(cassetteDirFor(projectRoot, rest[0] ?? 'agent-evals')), cassetteMode, evalCassetteCanonicalizationV2(projectRoot))
         : rawProvider;
     const reasoningEffort = cliReasoningEffort(flags);
     const requestedDepth = cliAnalysisDepth(flags);
@@ -633,6 +986,11 @@ async function runEval(rest, flags) {
             const runtimeRun = via === 'runtime'
                 ? await driveViaRuntime({ runtimeBase, question: testCase.question })
                 : undefined;
+            // Runtime mode is scored from the persisted AgentRun. The transport
+            // adapter intentionally has no AgentAnswer.contextPack, so borrowing the
+            // local preflight pack here would fabricate retrieval/route evidence for a
+            // different execution path.
+            const runtimeProjection = runtimeRun ? projectRuntimeRun(runtimeRun) : undefined;
             const result = runtimeRun
                 ? answerFromRuntimeRun(runtimeRun)
                 : await answer({
@@ -713,7 +1071,7 @@ async function runEval(rest, flags) {
                         });
                     },
                 });
-            const evaluation = evaluateCase(testCase, result);
+            const evaluation = evaluateCase(testCase, result, runtimeProjection);
             const durationMs = Date.now() - startedAt;
             const draftSaved = Boolean(result.draftBlock?.path ?? result.draftBlockId);
             const narration = narrationOutcomeForEval(runtimeRun?.narrationIntegrityReceipt);
@@ -740,7 +1098,7 @@ async function runEval(rest, flags) {
                 executionMatched: evaluation.executionMatched,
                 ...(judgeVerdict ? { judgeScore: judgeVerdict.score, judgePass: judgeVerdict.pass } : {}),
                 kind: result.kind,
-                route: runtimeRun ? evalRouteForRun(runtimeRun.route) : result.contextPack?.routeDecision.route,
+                route: runtimeProjection?.route ?? result.contextPack?.routeDecision.route,
                 // Only the runtime driver can see the router's clarification options.
                 // In-process runs leave this undefined, so a clarify there scores as a
                 // dead end — the conservative reading, and another reason `--via runtime`
@@ -752,12 +1110,13 @@ async function runEval(rest, flags) {
                         && Boolean(runtimeRun.answer?.trim()),
                     meaningResolved: Boolean(runtimeRun.routeDecision?.meaningResolution),
                 } : {}),
-                intent: result.contextPack?.routeDecision.intent,
+                ...(runtimeProjection?.observability ? { observability: runtimeProjection.observability } : {}),
+                intent: runtimeRun?.routeDecision?.category ?? result.contextPack?.routeDecision.intent,
                 reviewStatus: result.reviewStatus,
-                contextObjects: result.contextPack?.objects.length ?? 0,
+                contextObjects: runtimeProjection?.retrievalCandidateCount ?? result.contextPack?.objects.length,
                 followUp: Boolean(testCase.followUp),
                 draftSaved,
-                toolCalls: result.evidence?.toolCalls?.length ?? 0,
+                toolCalls: runtimeProjection?.toolCallCount ?? result.evidence?.toolCalls?.length ?? 0,
                 expected: testCase.expected,
                 validationCode: evaluation.validationCode,
                 trace: buildEvalTrace({
@@ -766,6 +1125,7 @@ async function runEval(rest, flags) {
                     evaluation,
                     durationMs,
                     draftSaved,
+                    runtime: runtimeProjection,
                 }),
             });
         }
@@ -776,6 +1136,18 @@ async function runEval(rest, flags) {
     }
     const passed = results.filter((r) => r.passed).length;
     const metrics = computeEvalMetrics(results);
+    // Runtime evals execute in a separate host, so its cassette directory is
+    // supplied explicitly by the eval workflow for reporting. The summary does
+    // not inspect prompts or provider credentials; it only classifies the
+    // checked-in response provenance.
+    const cassetteDirectory = via === 'runtime'
+        ? process.env.DQL_EVAL_CASSETTE_DIR
+        : cassetteMode === 'record' || cassetteMode === 'replay'
+            ? cassetteDirFor(projectRoot, rest[0] ?? 'agent-evals')
+            : undefined;
+    const cassetteEvidence = cassetteDirectory
+        ? cassetteEvidenceSummary(new CassetteStore(cassetteDirectory))
+        : undefined;
     const thresholds = {
         minToolRequirement: flags.minToolRequirement ?? null,
         minExecutionMatch: flags.minExecutionMatch ?? null,
@@ -788,7 +1160,15 @@ async function runEval(rest, flags) {
     const thresholdsPassed = agentEvalThresholdsPass(metrics, thresholds);
     const ok = passed === results.length && thresholdsPassed;
     if (flags.format === 'json') {
-        console.log(JSON.stringify({ ok, passed, total: results.length, thresholds, metrics, results }, null, 2));
+        console.log(JSON.stringify({
+            ok,
+            passed,
+            total: results.length,
+            thresholds,
+            metrics,
+            ...(cassetteEvidence ? { cassetteEvidence } : {}),
+            results,
+        }, null, 2));
         if (!ok)
             process.exitCode = 1;
         return;
@@ -802,6 +1182,14 @@ async function runEval(rest, flags) {
     console.log(`Certified hit rate: ${formatRate(metrics.certified_hit_rate)}`);
     console.log(`Generated follow-up pass rate: ${formatRate(metrics.generated_followup_pass_rate)}`);
     console.log(`Safe refusal rate: ${formatRate(metrics.safe_refusal_rate)}`);
+    if (cassetteEvidence) {
+        console.log(`Cassette replay entries: ${cassetteEvidence.totalEntries} `
+            + `(${cassetteEvidence.migratedLegacyDeterministicFixtureEntries} migrated legacy deterministic fixture, `
+            + `${cassetteEvidence.syntheticDeterministicOrchestrationFixtureEntries} synthetic deterministic orchestration fixture).`);
+        console.log(cassetteEvidence.realProviderQualityEligible
+            ? 'Real-provider quality evidence: eligible.'
+            : `Real-provider quality evidence: excluded (${cassetteEvidence.realProviderQualityExclusionReasons.join(', ')}).`);
+    }
     console.log(`False refusal rate: ${formatRate(metrics.false_refusal_rate)} (${metrics.false_refusal_count}/${metrics.answerable_case_count} answerable cases refused)`);
     console.log(`Clarification rate: ${formatRate(metrics.clarification_rate)} (answerable cases asked instead of answered)`);
     if (metrics.meaning_resolved_rate !== null && metrics.meaning_resolved_rate < 1) {
@@ -851,7 +1239,7 @@ function previewGeneratedDraftPath(projectRoot, domain, slug) {
     }
     return `blocks/_drafts/${slug}.dql`;
 }
-function evaluateCase(testCase, result) {
+function evaluateCase(testCase, result, runtime) {
     const expected = testCase.expected;
     if (!expected)
         return { failures: [] };
@@ -892,12 +1280,30 @@ function evaluateCase(testCase, result) {
         failures.push(`certification expected ${expected.certification}, got ${result.certification}`);
     if (expected.reviewStatus && result.reviewStatus !== expected.reviewStatus)
         failures.push(`reviewStatus expected ${expected.reviewStatus}, got ${result.reviewStatus}`);
-    if (expected.route && result.contextPack?.routeDecision.route !== expected.route)
-        failures.push(`route expected ${expected.route}, got ${result.contextPack?.routeDecision.route ?? 'none'}`);
+    const observedRoute = runtime?.route ?? result.contextPack?.routeDecision.route;
+    if (expected.route && observedRoute !== expected.route)
+        failures.push(`route expected ${expected.route}, got ${observedRoute ?? 'none'}`);
     if (expected.intent && result.contextPack?.routeDecision.intent !== expected.intent)
         failures.push(`intent expected ${expected.intent}, got ${result.contextPack?.routeDecision.intent ?? 'none'}`);
-    if (expected.missingContextKind && !result.contextPack?.missingContext.some((item) => item.kind === expected.missingContextKind))
+    // `modeling_gap` is a broad terminal kind. A relationship expectation is
+    // satisfied only by the router's persisted relationship-specific witness;
+    // otherwise a missing metric/dimension tuple would be misreported as a
+    // relationship repair opportunity.
+    const runtimeReportsMissingContext = expected.missingContextKind === 'relationship'
+        ? runtime?.terminalOutcome?.gap?.code === 'MISSING_RELATIONSHIP'
+        : runtime?.terminalOutcome?.kind === 'modeling_gap'
+            && expected.missingContextKind === 'modeling_gap';
+    if (expected.missingContextKind
+        && !runtimeReportsMissingContext
+        && !result.contextPack?.missingContext.some((item) => item.kind === expected.missingContextKind)) {
         failures.push(`missing context kind ${expected.missingContextKind} was not reported`);
+    }
+    if (expected.terminalOutcomeKind && runtime?.terminalOutcome?.kind !== expected.terminalOutcomeKind) {
+        failures.push(`terminal outcome expected ${expected.terminalOutcomeKind}, got ${runtime?.terminalOutcome?.kind ?? 'none'}`);
+    }
+    if (expected.traceRecordingStatus && runtime?.observability?.recordingStatus !== expected.traceRecordingStatus) {
+        failures.push(`trace recording status expected ${expected.traceRecordingStatus}, got ${runtime?.observability?.recordingStatus ?? 'unavailable'}`);
+    }
     for (const token of stringList(expected.sqlContains)) {
         if (!result.proposedSql?.toLowerCase().includes(token.toLowerCase()))
             failures.push(`SQL did not contain "${token}"`);
@@ -918,7 +1324,7 @@ function evaluateCase(testCase, result) {
             failures.push(`draftSaved expected ${expected.draftSaved}, got ${saved}`);
     }
     if (typeof expected.minToolCalls === 'number') {
-        const actualToolCalls = result.evidence?.toolCalls?.length ?? 0;
+        const actualToolCalls = runtime?.toolCallCount ?? result.evidence?.toolCalls?.length ?? 0;
         if (actualToolCalls < expected.minToolCalls) {
             failures.push(`toolCalls expected at least ${expected.minToolCalls}, got ${actualToolCalls}`);
         }
@@ -957,7 +1363,7 @@ export function evalCaseIsAnswerable(expected) {
         return false;
     if (expected.sourceTier === 'no_answer')
         return false;
-    if (expected.route === 'clarify')
+    if (expected.route === 'clarify' || expected.route === 'blocked')
         return false;
     if (Object.keys(expected).length === 0)
         return undefined;
@@ -1084,7 +1490,12 @@ function computeEvalMetrics(results) {
         draft_saved_count: results.filter((result) => result.draftSaved).length,
         tool_observed_case_count: results.filter((result) => result.toolCalls > 0).length,
         avg_tool_calls: average(toolCallCounts),
-        avg_context_objects: average(results.map((result) => result.contextObjects)),
+        // Runtime runs only report this when the persisted router recorded an
+        // explicit retrieval count. Treat absent evidence as unknown, not as an
+        // invented empty context pack.
+        avg_context_objects: average(results
+            .map((result) => result.contextObjects)
+            .filter((count) => typeof count === 'number')),
         avg_execution_ms: executionTimes.length ? average(executionTimes) : null,
     };
 }
@@ -1106,12 +1517,13 @@ function agentEvalThresholdsPass(metrics, thresholds) {
             || metrics.wrong_certified_count <= thresholds.maxWrongCertified);
 }
 function buildEvalTrace(input) {
-    const { testCase, result, evaluation, durationMs, draftSaved } = input;
+    const { testCase, result, evaluation, durationMs, draftSaved, runtime } = input;
     const routeDecision = result.contextPack?.routeDecision;
     const selectedRelations = result.contextPack?.retrievalDiagnostics.selectedRelations ?? [];
     const allowedRelations = result.contextPack?.allowedSqlContext?.relations ?? [];
     const followUp = testCase.followUp;
     const toolCalls = result.evidence?.toolCalls ?? [];
+    const observedToolCallCount = runtime?.toolCallCount ?? toolCalls.length;
     const routeEvidence = result.evidence?.route ?? [];
     const executionStatus = result.executionError
         ? 'failed'
@@ -1127,33 +1539,44 @@ function buildEvalTrace(input) {
     const rowsExpected = testCase.expected?.rows !== undefined;
     const expectedMinToolCalls = testCase.expected?.minToolCalls;
     const toolStatus = typeof expectedMinToolCalls === 'number'
-        ? toolCalls.length >= expectedMinToolCalls ? 'passed' : 'failed'
-        : toolCalls.length > 0 ? 'passed' : routeEvidence.length > 0 ? 'info' : 'not_run';
+        ? observedToolCallCount >= expectedMinToolCalls ? 'passed' : 'failed'
+        : observedToolCallCount > 0 ? 'passed' : routeEvidence.length > 0 ? 'info' : 'not_run';
     const toolMessage = typeof expectedMinToolCalls === 'number'
-        ? toolCalls.length >= expectedMinToolCalls
-            ? `Observed ${toolCalls.length} provider tool call(s), meeting the minimum of ${expectedMinToolCalls}.`
-            : `Observed ${toolCalls.length} provider tool call(s), below the minimum of ${expectedMinToolCalls}.`
-        : toolCalls.length > 0
-            ? `Observed ${toolCalls.length} provider tool call(s).`
+        ? observedToolCallCount >= expectedMinToolCalls
+            ? `Observed ${observedToolCallCount} provider tool call(s), meeting the minimum of ${expectedMinToolCalls}.`
+            : `Observed ${observedToolCallCount} provider tool call(s), below the minimum of ${expectedMinToolCalls}.`
+        : observedToolCallCount > 0
+            ? `Observed ${observedToolCallCount} provider tool call(s).`
             : routeEvidence.length > 0
                 ? `Captured ${routeEvidence.length} deterministic route evidence step(s).`
                 : 'No provider tool calls were observed for this answer.';
     return [
         {
             stage: 'context',
-            status: result.contextPack ? 'passed' : 'not_run',
-            message: result.contextPack
-                ? `Context pack ${result.contextPack.id} selected ${result.contextPack.objects.length} object(s).`
-                : 'No context pack was attached to the answer.',
-            payload: result.contextPack
+            status: runtime && (runtime.retrievalCandidateCount !== undefined || runtime.sourceCoverage?.length || runtime.terminalOutcome)
+                ? 'passed'
+                : result.contextPack ? 'passed' : 'not_run',
+            message: runtime && (runtime.retrievalCandidateCount !== undefined || runtime.sourceCoverage?.length || runtime.terminalOutcome)
+                ? `Persisted route evidence recorded ${runtime.retrievalCandidateCount ?? 'an unspecified number of'} retrieved candidate(s).`
+                : result.contextPack
+                    ? `Context pack ${result.contextPack.id} selected ${result.contextPack.objects.length} object(s).`
+                    : 'No context pack was attached to the answer.',
+            payload: runtime && (runtime.retrievalCandidateCount !== undefined || runtime.sourceCoverage?.length || runtime.terminalOutcome)
                 ? {
-                    contextPackId: result.contextPack.id,
-                    selectedObjectCount: result.contextPack.objects.length,
-                    allowedRelationCount: allowedRelations.length,
-                    selectedRelations: selectedRelations.slice(0, 12).map((relation) => relation.relation),
-                    missingContext: result.contextPack.missingContext,
+                    evidenceSource: 'persisted_agent_run',
+                    retrievalCandidateCount: runtime.retrievalCandidateCount,
+                    sourceCoverage: runtime.sourceCoverage,
+                    terminalOutcome: runtime.terminalOutcome,
                 }
-                : undefined,
+                : result.contextPack
+                    ? {
+                        contextPackId: result.contextPack.id,
+                        selectedObjectCount: result.contextPack.objects.length,
+                        allowedRelationCount: allowedRelations.length,
+                        selectedRelations: selectedRelations.slice(0, 12).map((relation) => relation.relation),
+                        missingContext: result.contextPack.missingContext,
+                    }
+                    : undefined,
         },
         {
             stage: 'rewrite',
@@ -1165,29 +1588,40 @@ function buildEvalTrace(input) {
         },
         {
             stage: 'lane',
-            status: routeDecision ? 'passed' : 'not_run',
-            message: routeDecision
-                ? `Lane ${routeDecision.route} / ${routeDecision.intent}.`
-                : 'No lane decision was attached to the answer.',
-            payload: routeDecision
+            status: runtime ? 'passed' : routeDecision ? 'passed' : 'not_run',
+            message: runtime
+                ? `Persisted engine route ${runtime.runRoute}${runtime.route ? ` evaluated as ${runtime.route}` : ''}.`
+                : routeDecision
+                    ? `Lane ${routeDecision.route} / ${routeDecision.intent}.`
+                    : 'No lane decision was attached to the answer.',
+            payload: runtime
                 ? {
-                    route: routeDecision.route,
-                    intent: routeDecision.intent,
-                    reason: routeDecision.reason,
-                    trustLabel: routeDecision.trustLabel,
-                    reviewStatus: routeDecision.reviewStatus,
-                    exactObjectKey: routeDecision.exactObjectKey,
+                    engineRoute: runtime.runRoute,
+                    evalRoute: runtime.route,
+                    status: runtime.status,
+                    trustState: runtime.trustState,
+                    terminalOutcome: runtime.terminalOutcome,
                 }
-                : undefined,
+                : routeDecision
+                    ? {
+                        route: routeDecision.route,
+                        intent: routeDecision.intent,
+                        reason: routeDecision.reason,
+                        trustLabel: routeDecision.trustLabel,
+                        reviewStatus: routeDecision.reviewStatus,
+                        exactObjectKey: routeDecision.exactObjectKey,
+                    }
+                    : undefined,
         },
         {
             stage: 'tools',
             status: toolStatus,
             message: toolMessage,
             payload: {
-                observedToolCalls: toolCalls.length,
+                observedToolCalls: observedToolCallCount,
                 expectedMinToolCalls,
-                providerToolCalls: toolCalls.slice(0, 12).map((call) => ({
+                ...(runtime ? { evidenceSource: 'persisted_agent_run.telemetry' } : {}),
+                providerToolCalls: runtime ? [] : toolCalls.slice(0, 12).map((call) => ({
                     order: call.order,
                     name: call.name,
                     status: call.status,
@@ -1261,6 +1695,22 @@ function buildEvalTrace(input) {
                 promoteCommand: result.promoteCommand,
             },
         },
+        {
+            stage: 'observability',
+            status: runtime?.observability?.recordingStatus === 'complete'
+                ? 'passed'
+                : runtime?.observability ? 'info' : 'not_run',
+            message: runtime?.observability
+                ? `Local Ask trace recording ${runtime.observability.recordingStatus}.`
+                : 'No runtime trace receipt was attached to this evaluation run.',
+            payload: runtime?.observability
+                ? {
+                    recordingStatus: runtime.observability.recordingStatus,
+                    storeSchemaVersion: runtime.observability.storeSchemaVersion,
+                    ...(runtime.observability.traceFingerprint ? { traceFingerprint: runtime.observability.traceFingerprint } : {}),
+                }
+                : undefined,
+        },
         {
             stage: 'scoring',
             status: evaluation.failures.length === 0 ? 'passed' : 'failed',