@duckcodeailabs/dql-cli 1.6.31 → 1.6.33

package/dist/commands/agent.js

@@ -6,6 +6,13 @@
  *     [--user alice@acme.com]   (filters Skills + records feedback as this user)
  *     [--domain growth]         (scopes KG search)
  *     [--format json]           (emits structured JSON instead of prose)
+ *     [--thread <id>]           (continue a persisted conversation thread: the
+ *                                question runs through the runtime's agent-run
+ *                                engine, which injects prior turns and records
+ *                                this one server-side)
+ *
+ *   dql agent threads
+ *     Lists persisted conversation threads (id, updated, title) from the runtime.
  *
  *   dql agent reindex [path]
  *     Rebuilds .dql/cache/agent-kg.sqlite and metadata.sqlite from the
@@ -17,9 +24,11 @@
 import { existsSync, readFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { load as loadYaml } from 'js-yaml';
-import { KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, reindexProject, loadSkills, pickProvider, answer, buildLocalContextPack, deriveGeneratedDraftSlug, recordQueryRun, upsertGeneratedDraft, validateSqlAgainstLocalContext, } from '@duckcodeailabs/dql-agent';
+import { KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, reindexProject, loadSkills, pickProvider, answer, buildAnalysisQuestionPlan, buildLocalContextPack, coerceReasoningEffort, contextRetrievalBudgetForQuestion, deriveGeneratedDraftSlug, loadAgentSemanticLayer, recordQueryRun, recordRuntimeSchemaSnapshot, upsertGeneratedDqlArtifactDraft, upsertGeneratedDraft, validateSqlAgainstLocalContext, } from '@duckcodeailabs/dql-agent';
 import { buildManifest, resolveDbtManifestPath } from '@duckcodeailabs/dql-core';
 import { findProjectRoot } from '../local-runtime.js';
+import { buildAnswerLoopTools, createGroundingContextExpander } from '../llm/answer-loop-tools.js';
+import { judgeAnswer } from './eval-judge.js';
 import { startProjectRuntime } from './notebook.js';
 /**
  * Resolve the runtime the agent posts certified blocks / generated SQL to.
@@ -46,6 +55,91 @@ async function resolveAgentRuntime(projectRoot, flags) {
     const handle = await startProjectRuntime(projectRoot, { preferredPort: 0 });
     return { runtimeBase: handle.url, close: handle.close };
 }
+async function fetchRuntimeSchemaContext(runtimeBase) {
+    try {
+        const response = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/schema`);
+        if (!response.ok)
+            return [];
+        const raw = await response.json();
+        if (!Array.isArray(raw))
+            return [];
+        return raw
+            .map(normalizeRuntimeSchemaTable)
+            .filter((table) => Boolean(table))
+            .slice(0, 500);
+    }
+    catch {
+        return [];
+    }
+}
+function normalizeRuntimeSchemaTable(raw) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw))
+        return undefined;
+    const table = raw;
+    const relation = cleanRuntimeSchemaString(table.path) ?? cleanRuntimeSchemaString(table.name);
+    if (!relation)
+        return undefined;
+    const columns = Array.isArray(table.columns)
+        ? table.columns
+            .map(normalizeRuntimeSchemaColumn)
+            .filter((column) => Boolean(column))
+            .slice(0, 120)
+        : [];
+    return {
+        relation,
+        name: cleanRuntimeSchemaString(table.name) ?? relation.split('.').pop() ?? relation,
+        source: cleanRuntimeSchemaString(table.source) ?? 'runtime schema',
+        columns,
+    };
+}
+function normalizeRuntimeSchemaColumn(raw) {
+    if (!raw || typeof raw !== 'object' || Array.isArray(raw))
+        return undefined;
+    const column = raw;
+    const name = cleanRuntimeSchemaString(column.name);
+    if (!name)
+        return undefined;
+    return {
+        name,
+        type: cleanRuntimeSchemaString(column.type),
+        description: cleanRuntimeSchemaString(column.description),
+    };
+}
+function recordCliRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
+    if (schemaContext.length === 0)
+        return;
+    try {
+        recordRuntimeSchemaSnapshot(projectRoot, {
+            source,
+            tables: schemaContext.map((table) => ({
+                relation: table.relation,
+                schema: table.schema,
+                name: table.name,
+                description: table.description,
+                source: table.source,
+                columns: table.columns.map((column) => ({
+                    name: column.name,
+                    type: column.type,
+                    description: column.description,
+                    sampleValues: column.sampleValues?.slice(0, 8),
+                })),
+            })),
+        });
+    }
+    catch {
+        // Runtime schema snapshots are advisory local metadata and must not block answers.
+    }
+}
+function cleanRuntimeSchemaString(value) {
+    return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
+}
+function cliReasoningEffort(flags) {
+    return coerceReasoningEffort(flags.reasoningEffort);
+}
+function cliAnalysisDepth(flags) {
+    const value = flags.analysisDepth?.trim().toLowerCase();
+    return value === 'quick' || value === 'deep' ? value : undefined;
+}
 /** A DQL runtime answers `/api/connections` with a connector/connection payload. */
 async function isDqlRuntime(base) {
     try {
@@ -63,6 +157,8 @@ export async function runAgent(sub, rest, flags) {
     switch (sub) {
         case 'ask':
             return runAsk(rest, flags);
+        case 'threads':
+            return runThreads(flags);
         case 'reindex':
             return runReindex(rest, flags);
         case 'feedback':
@@ -70,8 +166,9 @@ export async function runAgent(sub, rest, flags) {
         case 'eval':
             return runEval(rest, flags);
         default:
-            throw new Error('Usage: dql agent <ask|reindex|feedback|eval> [args]\n' +
-                '  dql agent ask "<question>" [--provider claude|openai|gemini|ollama] [--user <id>] [--domain <d>]\n' +
+            throw new Error('Usage: dql agent <ask|threads|reindex|feedback|eval> [args]\n' +
+                '  dql agent ask "<question>" [--provider claude|openai|gemini|ollama] [--user <id>] [--domain <d>] [--thread <id>]\n' +
+                '  dql agent threads [--runtime-url <url>]\n' +
                 '  dql agent reindex [path]\n' +
                 '  dql agent feedback up|down --block <id> --question "..."\n' +
                 '  dql agent eval agent-evals.yml [--provider claude|openai|gemini|ollama] [--execute] [--save]');
@@ -81,6 +178,12 @@ async function runAsk(rest, flags) {
     const question = rest.join(' ').trim();
     if (!question)
         throw new Error('Usage: dql agent ask "<question>"');
+    // Thread-scoped ask: hand the question to the runtime's agent-run engine with
+    // the thread id, so the SERVER injects prior turns and persists this run as a
+    // new turn (the same conversation store the notebook UI uses).
+    const threadId = flags.thread;
+    if (threadId)
+        return runThreadAsk(question, threadId, flags);
     const projectRoot = findProjectRoot(process.cwd());
     const kgPath = defaultKgPath(projectRoot);
     await reindexProject(projectRoot, { kgPath });
@@ -88,6 +191,8 @@ async function runAsk(rest, flags) {
     const userId = flags.user;
     const domain = flags.domain;
     const format = flags.format;
+    const reasoningEffort = cliReasoningEffort(flags);
+    const requestedDepth = cliAnalysisDepth(flags);
     const provider = await pickProvider(providerName);
     const kg = new KGStore(kgPath);
     const memory = new MemoryStore(defaultMemoryPath(projectRoot));
@@ -99,10 +204,31 @@ async function runAsk(rest, flags) {
             scopes: ['project', 'user', 'artifact'],
             limit: 6,
         });
-        const contextPack = await buildLocalContextPack(projectRoot, { question, limit: 80 }).catch(() => undefined);
+        const semanticLayer = loadAgentSemanticLayer(projectRoot);
+        const questionPlan = buildAnalysisQuestionPlan(question);
+        const contextBudget = contextRetrievalBudgetForQuestion({
+            questionPlan,
+            requestedDepth,
+            reasoningEffort,
+        });
         const manifest = buildManifest({ projectRoot, dbtManifestPath: resolveDbtManifestPath(projectRoot) ?? undefined });
         const { runtimeBase, close } = await resolveAgentRuntime(projectRoot, flags);
         closeRuntime = close;
+        const schemaContext = await fetchRuntimeSchemaContext(runtimeBase);
+        recordCliRuntimeSchemaSnapshot(projectRoot, schemaContext, 'direct CLI runtime schema');
+        const contextPack = await buildLocalContextPack(projectRoot, {
+            question,
+            surface: 'cli',
+            strictness: contextBudget.strictness,
+            limit: contextBudget.limit,
+            runtimeSchemaSnapshot: schemaContext.length > 0
+                ? {
+                    source: 'direct CLI runtime schema',
+                    tables: schemaContext,
+                }
+                : undefined,
+        }).catch(() => undefined);
+        const answerLoopTools = buildAnswerLoopTools(projectRoot, { serverUrl: runtimeBase });
         const result = await answer({
             question,
             provider,
@@ -111,7 +237,13 @@ async function runAsk(rest, flags) {
             userId,
             domain,
             memoryContext,
+            semanticLayer,
+            schemaContext,
             contextPack,
+            reasoningEffort,
+            analysisDepth: contextBudget.analysisDepth,
+            expandGroundingContext: createGroundingContextExpander(projectRoot),
+            answerLoopTools,
             executeCertifiedBlock: async (node) => {
                 const block = manifest.blocks[node.name] ?? manifest.blocks[node.nodeId.replace(/^block:/, '')];
                 if (!block)
@@ -187,20 +319,39 @@ async function runAsk(rest, flags) {
                 });
                 return result;
             },
-            captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack, sourceBlock, validationWarnings }) => {
+            captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack, sourceBlock, sourceDqlArtifact, dqlArtifact, proposedEntity, requestedFilters, requestedDimensions, validationWarnings, outputs }) => {
                 const slug = deriveGeneratedDraftSlug(draftQuestion);
                 const proposedDomain = sourceBlock?.domain ?? contextPack?.objects.find((object) => object.domain)?.domain ?? domain ?? 'misc';
+                if (dqlArtifact?.kind === 'semantic_block') {
+                    return upsertGeneratedDqlArtifactDraft(projectRoot, {
+                        slug,
+                        question: draftQuestion,
+                        proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                        proposedDomain,
+                        dqlArtifact,
+                        sourceQuestion: followUp?.sourceQuestion,
+                        sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
+                        followupKind: followUp?.kind,
+                        outputs,
+                        contextPackId: contextPack?.id,
+                        routeIntent: String(intent),
+                        validationWarnings,
+                    });
+                }
                 return upsertGeneratedDraft(projectRoot, {
                     slug,
                     question: draftQuestion,
                     proposedSql: sql,
                     proposedContractId: `${proposedDomain}.Unknown.${slug}`,
                     proposedDomain,
+                    proposedEntity,
+                    sourceDqlArtifact,
                     sourceQuestion: followUp?.sourceQuestion,
                     sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
                     followupKind: followUp?.kind,
-                    requestedFilters: followUp?.filters,
-                    requestedDimensions: followUp?.dimensions,
+                    requestedFilters,
+                    requestedDimensions,
+                    outputs,
                     contextPackId: contextPack?.id,
                     routeIntent: String(intent),
                     validationWarnings,
@@ -224,15 +375,7 @@ async function runAsk(rest, flags) {
             console.log(`\nRows: ${result.result.rowCount}`);
             console.log(JSON.stringify(result.result.rows.slice(0, 5), null, 2));
         }
-        if (result.proposedSql) {
-            console.log(`\n--- Proposed SQL (review before saving as a block) ---\n${result.proposedSql}`);
-            if (result.suggestedViz)
-                console.log(`Viz: ${result.suggestedViz}`);
-            if (result.draftBlock?.path)
-                console.log(`Draft: ${result.draftBlock.path}`);
-            if (result.promoteCommand)
-                console.log(`Promote: ${result.promoteCommand}`);
-        }
+        printDqlArtifactPreview(result);
     }
     finally {
         kg.close();
@@ -241,6 +384,103 @@ async function runAsk(rest, flags) {
             await closeRuntime();
     }
 }
+function printDqlArtifactPreview(result) {
+    const dqlSource = result.dqlArtifact?.source?.trim();
+    if (dqlSource) {
+        console.log(`\n--- DQL artifact (${result.dqlArtifact?.kind ?? 'draft'}) ---\n${dqlSource}`);
+    }
+    if (result.proposedSql) {
+        const label = dqlSource
+            ? 'Compiled SQL preview'
+            : 'Proposed SQL (review before saving as a block)';
+        console.log(`\n--- ${label} ---\n${result.proposedSql}`);
+        if (result.suggestedViz)
+            console.log(`Viz: ${result.suggestedViz}`);
+        if (result.draftBlock?.path)
+            console.log(`Draft: ${result.draftBlock.path}`);
+        if (result.promoteCommand)
+            console.log(`Promote: ${result.promoteCommand}`);
+    }
+}
+/**
+ * `dql agent ask --thread <id>` — POST the question to the runtime's
+ * `/api/agent-runs` with the threadId in the body. The server injects the
+ * thread's prior turns into the conversation context and records the completed
+ * run as the next turn, so follow-ups resolve "those"/"that product" correctly
+ * across CLI invocations (and across the notebook UI, which shares the store).
+ */
+async function runThreadAsk(question, threadId, flags) {
+    const projectRoot = findProjectRoot(process.cwd());
+    const format = flags.format;
+    const { runtimeBase, close } = await resolveAgentRuntime(projectRoot, flags);
+    try {
+        const reasoningEffort = cliReasoningEffort(flags);
+        const analysisDepth = cliAnalysisDepth(flags);
+        const response = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/agent-runs`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                question,
+                threadId,
+                ...(reasoningEffort ? { reasoningEffort } : {}),
+                ...(analysisDepth ? { analysisDepth } : {}),
+            }),
+        });
+        if (!response.ok)
+            throw new Error(`Runtime returned ${response.status}: ${await response.text()}`);
+        const payload = (await response.json());
+        if (!payload.run)
+            throw new Error('Runtime did not return an agent run.');
+        if (format === 'json') {
+            console.log(JSON.stringify(payload.run, null, 2));
+            return;
+        }
+        const run = payload.run;
+        const badge = run.trustState === 'certified'
+            ? '✓ Certified'
+            : run.trustState === 'grounded'
+                ? '✓ Verified (grounded)'
+                : run.trustState === 'review_required'
+                    ? '! AI-generated · review required'
+                    : run.trustState === 'blocked'
+                        ? '✕ Blocked'
+                        : '· Reply';
+        console.log(`${badge}\n\n${(run.answer ?? run.summary ?? '').trim()}`);
+        console.log(`\nThread: ${threadId}`);
+    }
+    finally {
+        await close();
+    }
+}
+/** `dql agent threads` — list server-persisted conversation threads. */
+async function runThreads(flags) {
+    const projectRoot = findProjectRoot(process.cwd());
+    const format = flags.format;
+    const { runtimeBase, close } = await resolveAgentRuntime(projectRoot, flags);
+    try {
+        const response = await fetch(`${runtimeBase.replace(/\/$/, '')}/api/agent/threads?limit=50`);
+        if (!response.ok)
+            throw new Error(`Runtime returned ${response.status}: ${await response.text()}`);
+        const payload = (await response.json());
+        const threads = Array.isArray(payload.threads) ? payload.threads : [];
+        if (format === 'json') {
+            console.log(JSON.stringify({ threads }, null, 2));
+            return;
+        }
+        if (threads.length === 0) {
+            console.log('No conversation threads yet. Ask from the notebook UI, or continue one here with `dql agent ask "<question>" --thread <id>`.');
+            return;
+        }
+        for (const thread of threads) {
+            const updated = thread.updatedAt ? new Date(thread.updatedAt).toISOString().replace('T', ' ').slice(0, 16) : 'unknown';
+            console.log(`  ${thread.id}  ${updated}  [${thread.surface ?? 'notebook'}]  ${thread.title ?? '(untitled)'}`);
+        }
+        console.log(`\n${threads.length} thread(s). Continue one: dql agent ask "<question>" --thread <id>`);
+    }
+    finally {
+        await close();
+    }
+}
 function recordCliQueryRun(projectRoot, run) {
     try {
         recordQueryRun(projectRoot, run);
@@ -256,7 +496,9 @@ async function runReindex(rest, flags) {
         console.log(JSON.stringify({ ok: true, ...stats }, null, 2));
         return;
     }
-    console.log(`  ✓ KG and metadata catalog rebuilt — ${stats.nodes} nodes, ${stats.edges} edges, ${stats.skills} skill(s).`);
+    const kgStatus = stats.kgRebuilt ? 'KG rebuilt' : 'KG fresh';
+    const catalogStatus = stats.metadataRefreshed ? 'metadata refreshed' : 'metadata fresh';
+    console.log(`  ✓ ${kgStatus}; ${catalogStatus} — ${stats.nodes} nodes, ${stats.edges} edges, ${stats.skills} skill(s).`);
 }
 async function runFeedback(rest, flags) {
     const rating = rest[0];
@@ -304,14 +546,27 @@ async function runEval(rest, flags) {
         await reindexProject(projectRoot, { kgPath });
     const providerName = flags.provider;
     const provider = await pickProvider(providerName);
+    const reasoningEffort = cliReasoningEffort(flags);
+    const requestedDepth = cliAnalysisDepth(flags);
     const kg = new KGStore(kgPath);
     const memory = new MemoryStore(defaultMemoryPath(projectRoot));
     const { skills } = loadSkills(projectRoot);
     const execute = Boolean(flags.execute);
+    // R3.2: optional LLM-as-judge. Uses the same provider's completion; skipped
+    // gracefully when no provider is available so offline eval stays deterministic.
+    const judge = Boolean(flags.judge);
+    const judgeComplete = async ({ system, user }) => provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], {});
     const runtimeBase = flags.runtimeUrl
         ?? flags.runtime
         ?? process.env.DQL_RUNTIME_URL
         ?? 'http://127.0.0.1:3474';
+    const semanticLayer = loadAgentSemanticLayer(projectRoot);
+    const expandGroundingContext = createGroundingContextExpander(projectRoot);
+    const answerLoopTools = buildAnswerLoopTools(projectRoot, { serverUrl: runtimeBase });
+    const schemaContext = execute
+        ? await fetchRuntimeSchemaContext(runtimeBase)
+        : [];
+    recordCliRuntimeSchemaSnapshot(projectRoot, schemaContext, 'CLI eval runtime schema');
     const manifest = execute
         ? buildManifest({ projectRoot, dbtManifestPath: resolveDbtManifestPath(projectRoot) ?? undefined })
         : null;
@@ -324,12 +579,25 @@ async function runEval(rest, flags) {
                 scopes: ['project', 'user', 'artifact'],
                 limit: 6,
             });
+            const questionPlan = buildAnalysisQuestionPlan(testCase.question, testCase.followUp);
+            const contextBudget = contextRetrievalBudgetForQuestion({
+                questionPlan,
+                requestedDepth,
+                reasoningEffort,
+            });
             const contextPack = await buildLocalContextPack(projectRoot, {
                 question: testCase.question,
                 surface: 'cli-eval',
                 followUp: testCase.followUp,
                 selectedContext: testCase.selectedContext,
-                limit: 80,
+                strictness: contextBudget.strictness,
+                limit: contextBudget.limit,
+                runtimeSchemaSnapshot: schemaContext.length > 0
+                    ? {
+                        source: 'CLI eval runtime schema',
+                        tables: schemaContext,
+                    }
+                    : undefined,
             }).catch(() => undefined);
             const result = await answer({
                 question: testCase.question,
@@ -339,16 +607,45 @@ async function runEval(rest, flags) {
                 skills,
                 memoryContext,
                 followUp: testCase.followUp,
+                semanticLayer,
+                schemaContext,
                 contextPack,
+                reasoningEffort,
+                analysisDepth: contextBudget.analysisDepth,
+                expandGroundingContext,
+                answerLoopTools,
                 executeCertifiedBlock: execute && manifest
                     ? createCertifiedBlockExecutor(projectRoot, manifest, runtimeBase)
                     : undefined,
                 executeGeneratedSql: execute
                     ? createGeneratedSqlExecutor(runtimeBase)
                     : undefined,
-                captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack: draftContextPack, sourceBlock, validationWarnings }) => {
+                captureGeneratedDraft: ({ question: draftQuestion, sql, intent, followUp, contextPack: draftContextPack, sourceBlock, sourceDqlArtifact, dqlArtifact, proposedEntity, requestedFilters, requestedDimensions, validationWarnings, outputs }) => {
                     const slug = deriveGeneratedDraftSlug(draftQuestion);
                     const proposedDomain = sourceBlock?.domain ?? draftContextPack?.objects.find((object) => object.domain)?.domain ?? testCase.domain ?? 'misc';
+                    if (dqlArtifact?.kind === 'semantic_block') {
+                        if (!flags.save) {
+                            return {
+                                path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
+                                askedTimes: 0,
+                                proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                            };
+                        }
+                        return upsertGeneratedDqlArtifactDraft(projectRoot, {
+                            slug,
+                            question: draftQuestion,
+                            proposedContractId: `${proposedDomain}.Unknown.${slug}`,
+                            proposedDomain,
+                            dqlArtifact,
+                            sourceQuestion: followUp?.sourceQuestion,
+                            sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
+                            followupKind: followUp?.kind,
+                            outputs,
+                            contextPackId: draftContextPack?.id,
+                            routeIntent: String(intent),
+                            validationWarnings,
+                        });
+                    }
                     if (!flags.save) {
                         return {
                             path: previewGeneratedDraftPath(projectRoot, proposedDomain, slug),
@@ -362,11 +659,14 @@ async function runEval(rest, flags) {
                         proposedSql: sql,
                         proposedContractId: `${proposedDomain}.Unknown.${slug}`,
                         proposedDomain,
+                        proposedEntity,
+                        sourceDqlArtifact,
                         sourceQuestion: followUp?.sourceQuestion,
                         sourceBlock: followUp?.sourceBlockName ?? sourceBlock?.name,
                         followupKind: followUp?.kind,
-                        requestedFilters: followUp?.filters,
-                        requestedDimensions: followUp?.dimensions,
+                        requestedFilters,
+                        requestedDimensions,
+                        outputs,
                         contextPackId: draftContextPack?.id,
                         routeIntent: String(intent),
                         validationWarnings,
@@ -374,21 +674,42 @@ async function runEval(rest, flags) {
                 },
             });
             const evaluation = evaluateCase(testCase, result);
+            const durationMs = Date.now() - startedAt;
+            const draftSaved = Boolean(result.draftBlock?.path ?? result.draftBlockId);
+            const judgeVerdict = judge
+                ? await judgeAnswer({
+                    question: testCase.question,
+                    sql: result.proposedSql ?? result.sql,
+                    answerText: result.text,
+                    trustLabel: result.trustLabelInfo?.display ?? result.certification,
+                    resultSample: result.result?.rows,
+                }, judgeComplete)
+                : undefined;
             results.push({
                 name: testCase.name ?? testCase.question,
                 passed: evaluation.failures.length === 0,
                 failures: evaluation.failures,
-                durationMs: Date.now() - startedAt,
+                durationMs,
                 executionMs: result.result?.executionTime,
+                executionMatched: evaluation.executionMatched,
+                ...(judgeVerdict ? { judgeScore: judgeVerdict.score, judgePass: judgeVerdict.pass } : {}),
                 kind: result.kind,
                 route: result.contextPack?.routeDecision.route,
                 intent: result.contextPack?.routeDecision.intent,
                 reviewStatus: result.reviewStatus,
                 contextObjects: result.contextPack?.objects.length ?? 0,
                 followUp: Boolean(testCase.followUp),
-                draftSaved: Boolean(result.draftBlock?.path ?? result.draftBlockId),
+                draftSaved,
+                toolCalls: result.evidence?.toolCalls?.length ?? 0,
                 expected: testCase.expected,
                 validationCode: evaluation.validationCode,
+                trace: buildEvalTrace({
+                    testCase,
+                    result,
+                    evaluation,
+                    durationMs,
+                    draftSaved,
+                }),
             });
         }
     }
@@ -398,8 +719,18 @@ async function runEval(rest, flags) {
     }
     const passed = results.filter((r) => r.passed).length;
     const metrics = computeEvalMetrics(results);
+    const thresholds = {
+        minToolRequirement: flags.minToolRequirement ?? null,
+        minExecutionMatch: flags.minExecutionMatch ?? null,
+        minJudgePass: flags.minJudgePass ?? null,
+        maxWrongCertified: flags.maxWrongCertified ?? null,
+    };
+    const thresholdsPassed = agentEvalThresholdsPass(metrics, thresholds);
+    const ok = passed === results.length && thresholdsPassed;
     if (flags.format === 'json') {
-        console.log(JSON.stringify({ ok: passed === results.length, passed, total: results.length, metrics, results }, null, 2));
+        console.log(JSON.stringify({ ok, passed, total: results.length, thresholds, metrics, results }, null, 2));
+        if (!ok)
+            process.exitCode = 1;
         return;
     }
     for (const result of results) {
@@ -411,9 +742,25 @@ async function runEval(rest, flags) {
     console.log(`Certified hit rate: ${formatRate(metrics.certified_hit_rate)}`);
     console.log(`Generated follow-up pass rate: ${formatRate(metrics.generated_followup_pass_rate)}`);
     console.log(`Safe refusal rate: ${formatRate(metrics.safe_refusal_rate)}`);
+    console.log(`Execution match rate: ${formatRate(metrics.execution_match_rate)}`);
+    console.log(`Tool requirement pass rate: ${formatRate(metrics.tool_requirement_pass_rate)}`);
+    console.log(`Tool-observed case count: ${metrics.tool_observed_case_count}`);
+    console.log(`Average tool calls: ${metrics.avg_tool_calls}`);
     console.log(`Wrong certified count: ${metrics.wrong_certified_count}`);
     console.log(`Draft saved count: ${metrics.draft_saved_count}`);
-    if (passed !== results.length)
+    if (thresholds.minToolRequirement !== null) {
+        console.log(`Tool requirement threshold: ${thresholds.minToolRequirement} (actual ${formatRate(metrics.tool_requirement_pass_rate)})`);
+    }
+    if (thresholds.minExecutionMatch !== null) {
+        console.log(`Execution-match threshold: ${thresholds.minExecutionMatch} (actual ${formatRate(metrics.execution_match_rate)})`);
+    }
+    if (thresholds.minJudgePass !== null) {
+        console.log(`Judge-pass threshold: ${thresholds.minJudgePass} (actual ${formatRate(metrics.judge_pass_rate)})`);
+    }
+    if (thresholds.maxWrongCertified !== null) {
+        console.log(`Wrong-certified ceiling: ${thresholds.maxWrongCertified} (actual ${metrics.wrong_certified_count})`);
+    }
+    if (!ok)
         process.exitCode = 1;
 }
 function previewGeneratedDraftPath(projectRoot, domain, slug) {
@@ -433,6 +780,7 @@ function evaluateCase(testCase, result) {
         return { failures: [] };
     const failures = [];
     let validationCode;
+    let executionMatched;
     if (expected.kind && result.kind !== expected.kind)
         failures.push(`kind expected ${expected.kind}, got ${result.kind}`);
     if (expected.sourceTier && result.sourceTier !== expected.sourceTier)
@@ -466,6 +814,12 @@ function evaluateCase(testCase, result) {
         if (saved !== expected.draftSaved)
             failures.push(`draftSaved expected ${expected.draftSaved}, got ${saved}`);
     }
+    if (typeof expected.minToolCalls === 'number') {
+        const actualToolCalls = result.evidence?.toolCalls?.length ?? 0;
+        if (actualToolCalls < expected.minToolCalls) {
+            failures.push(`toolCalls expected at least ${expected.minToolCalls}, got ${actualToolCalls}`);
+        }
+    }
     if ((expected.allowedRelationsOnly || expected.allowedColumnsOnly) && result.proposedSql) {
         const validation = validateSqlAgainstLocalContext(result.proposedSql, result.contextPack, {
             question: testCase.question,
@@ -479,10 +833,11 @@ function evaluateCase(testCase, result) {
     }
     if (expected.rows) {
         const actualRows = result.result?.rows ?? [];
-        if (!rowsEqual(actualRows, expected.rows))
+        executionMatched = rowsEqual(actualRows, expected.rows);
+        if (!executionMatched)
             failures.push('executed rows did not match expected rows');
     }
-    return { failures, validationCode };
+    return { failures, validationCode, executionMatched };
 }
 function computeEvalMetrics(results) {
     const certifiedCases = results.filter((result) => result.expected?.kind === 'certified' ||
@@ -494,18 +849,256 @@ function computeEvalMetrics(results) {
     const executionTimes = results
         .map((result) => result.executionMs)
         .filter((value) => typeof value === 'number' && Number.isFinite(value));
+    const executionMatchCases = results.filter((result) => result.executionMatched !== undefined);
+    const toolRequiredCases = results.filter((result) => typeof result.expected?.minToolCalls === 'number');
+    const toolCallCounts = results.map((result) => result.toolCalls);
+    const judged = results.filter((result) => typeof result.judgeScore === 'number');
     return {
         certified_hit_rate: ratio(certifiedCases.filter((result) => result.passed && result.kind === 'certified').length, certifiedCases.length),
+        judge_mean_score: judged.length ? average(judged.map((result) => result.judgeScore ?? 0)) : null,
+        judge_pass_rate: judged.length ? ratio(judged.filter((result) => result.judgePass).length, judged.length) : null,
         generated_followup_pass_rate: ratio(generatedFollowUpCases.filter((result) => result.passed).length, generatedFollowUpCases.length),
         safe_refusal_rate: ratio(refusalCases.filter((result) => result.passed && result.kind === 'no_answer').length, refusalCases.length),
+        execution_match_rate: ratio(executionMatchCases.filter((result) => result.executionMatched).length, executionMatchCases.length),
+        tool_requirement_pass_rate: ratio(toolRequiredCases.filter((result) => result.toolCalls >= (result.expected?.minToolCalls ?? 0)).length, toolRequiredCases.length),
         wrong_certified_count: results.filter((result) => result.kind === 'certified' &&
             (result.expected?.kind ? result.expected.kind !== 'certified' : result.followUp)).length,
         outside_context_rejection_count: results.filter((result) => result.validationCode === 'unknown_relation' || result.validationCode === 'unknown_column').length,
         draft_saved_count: results.filter((result) => result.draftSaved).length,
+        tool_observed_case_count: results.filter((result) => result.toolCalls > 0).length,
+        avg_tool_calls: average(toolCallCounts),
         avg_context_objects: average(results.map((result) => result.contextObjects)),
         avg_execution_ms: executionTimes.length ? average(executionTimes) : null,
     };
 }
+function agentEvalThresholdsPass(metrics, thresholds) {
+    // A rate threshold with no applicable cases (metric === null) is vacuously
+    // satisfied — you only fail when the metric exists and falls below the bar.
+    const rateOk = (metric, min) => min === null || min === undefined || metric === null || metric >= min;
+    return rateOk(metrics.tool_requirement_pass_rate, thresholds.minToolRequirement)
+        && rateOk(metrics.execution_match_rate, thresholds.minExecutionMatch)
+        && rateOk(metrics.judge_pass_rate, thresholds.minJudgePass)
+        && (thresholds.maxWrongCertified === null
+            || thresholds.maxWrongCertified === undefined
+            || metrics.wrong_certified_count <= thresholds.maxWrongCertified);
+}
+function buildEvalTrace(input) {
+    const { testCase, result, evaluation, durationMs, draftSaved } = input;
+    const routeDecision = result.contextPack?.routeDecision;
+    const selectedRelations = result.contextPack?.retrievalDiagnostics.selectedRelations ?? [];
+    const allowedRelations = result.contextPack?.allowedSqlContext?.relations ?? [];
+    const followUp = testCase.followUp;
+    const toolCalls = result.evidence?.toolCalls ?? [];
+    const routeEvidence = result.evidence?.route ?? [];
+    const executionStatus = result.executionError
+        ? 'failed'
+        : result.result
+            ? 'passed'
+            : 'not_run';
+    const validationExpected = Boolean(testCase.expected?.allowedRelationsOnly || testCase.expected?.allowedColumnsOnly);
+    const validationStatus = evaluation.validationCode
+        ? 'failed'
+        : validationExpected
+            ? 'passed'
+            : 'not_run';
+    const rowsExpected = testCase.expected?.rows !== undefined;
+    const expectedMinToolCalls = testCase.expected?.minToolCalls;
+    const toolStatus = typeof expectedMinToolCalls === 'number'
+        ? toolCalls.length >= expectedMinToolCalls ? 'passed' : 'failed'
+        : toolCalls.length > 0 ? 'passed' : routeEvidence.length > 0 ? 'info' : 'not_run';
+    const toolMessage = typeof expectedMinToolCalls === 'number'
+        ? toolCalls.length >= expectedMinToolCalls
+            ? `Observed ${toolCalls.length} provider tool call(s), meeting the minimum of ${expectedMinToolCalls}.`
+            : `Observed ${toolCalls.length} provider tool call(s), below the minimum of ${expectedMinToolCalls}.`
+        : toolCalls.length > 0
+            ? `Observed ${toolCalls.length} provider tool call(s).`
+            : routeEvidence.length > 0
+                ? `Captured ${routeEvidence.length} deterministic route evidence step(s).`
+                : 'No provider tool calls were observed for this answer.';
+    return [
+        {
+            stage: 'context',
+            status: result.contextPack ? 'passed' : 'not_run',
+            message: result.contextPack
+                ? `Context pack ${result.contextPack.id} selected ${result.contextPack.objects.length} object(s).`
+                : 'No context pack was attached to the answer.',
+            payload: result.contextPack
+                ? {
+                    contextPackId: result.contextPack.id,
+                    selectedObjectCount: result.contextPack.objects.length,
+                    allowedRelationCount: allowedRelations.length,
+                    selectedRelations: selectedRelations.slice(0, 12).map((relation) => relation.relation),
+                    missingContext: result.contextPack.missingContext,
+                }
+                : undefined,
+        },
+        {
+            stage: 'rewrite',
+            status: followUp ? 'passed' : 'not_run',
+            message: followUp
+                ? `Follow-up context attached (${followUp.kind}).`
+                : 'No follow-up rewrite/context was supplied for this case.',
+            payload: summarizeFollowUpForTrace(followUp),
+        },
+        {
+            stage: 'lane',
+            status: routeDecision ? 'passed' : 'not_run',
+            message: routeDecision
+                ? `Lane ${routeDecision.route} / ${routeDecision.intent}.`
+                : 'No lane decision was attached to the answer.',
+            payload: routeDecision
+                ? {
+                    route: routeDecision.route,
+                    intent: routeDecision.intent,
+                    reason: routeDecision.reason,
+                    trustLabel: routeDecision.trustLabel,
+                    reviewStatus: routeDecision.reviewStatus,
+                    exactObjectKey: routeDecision.exactObjectKey,
+                }
+                : undefined,
+        },
+        {
+            stage: 'tools',
+            status: toolStatus,
+            message: toolMessage,
+            payload: {
+                observedToolCalls: toolCalls.length,
+                expectedMinToolCalls,
+                providerToolCalls: toolCalls.slice(0, 12).map((call) => ({
+                    order: call.order,
+                    name: call.name,
+                    status: call.status,
+                    inputSummary: call.inputSummary,
+                    outputSummary: call.outputSummary,
+                })),
+                routeEvidence: routeEvidence.slice(0, 12).map((step) => ({
+                    tool: step.tool,
+                    status: step.status,
+                    label: step.label,
+                    detail: step.detail,
+                })),
+            },
+        },
+        {
+            stage: 'answer',
+            status: result.kind === 'no_answer' ? 'failed' : 'passed',
+            message: `Answer kind ${result.kind}${result.sourceTier ? ` from ${result.sourceTier}` : ''}.`,
+            payload: {
+                kind: result.kind,
+                sourceTier: result.sourceTier,
+                certification: result.certification,
+                reviewStatus: result.reviewStatus,
+                route: result.route?.tier,
+                refusalCode: result.refusalCode,
+                sourceCertifiedBlock: result.sourceCertifiedBlock,
+                dqlArtifactKind: result.dqlArtifact?.kind,
+                providerUsed: result.providerUsed,
+            },
+        },
+        {
+            stage: 'validation',
+            status: validationStatus,
+            message: evaluation.validationCode
+                ? `SQL context validation failed with ${evaluation.validationCode}.`
+                : validationExpected
+                    ? 'SQL context validation passed.'
+                    : 'SQL context validation was not required by this case.',
+            payload: {
+                validationCode: evaluation.validationCode,
+                failures: evaluation.failures,
+                expectedAllowedRelationsOnly: testCase.expected?.allowedRelationsOnly,
+                expectedAllowedColumnsOnly: testCase.expected?.allowedColumnsOnly,
+            },
+        },
+        {
+            stage: 'execution',
+            status: executionStatus,
+            message: result.executionError
+                ? result.executionError
+                : result.result
+                    ? `Executed and returned ${result.result.rowCount} row(s).`
+                    : 'No SQL/block execution result was captured.',
+            payload: {
+                rowCount: result.result?.rowCount,
+                executionTime: result.result?.executionTime,
+                executionMatched: rowsExpected ? evaluation.executionMatched : undefined,
+                expectedRows: rowsExpected ? testCase.expected?.rows?.length : undefined,
+                columns: summarizeResultColumns(result.result?.columns),
+            },
+        },
+        {
+            stage: 'draft',
+            status: draftSaved ? 'passed' : 'not_run',
+            message: draftSaved
+                ? `Draft captured at ${result.draftBlock?.path ?? result.draftBlockId}.`
+                : 'No generated draft was captured.',
+            payload: {
+                draftBlockId: result.draftBlockId,
+                draftPath: result.draftBlock?.path,
+                promoteCommand: result.promoteCommand,
+            },
+        },
+        {
+            stage: 'scoring',
+            status: evaluation.failures.length === 0 ? 'passed' : 'failed',
+            message: evaluation.failures.length === 0
+                ? `Case passed in ${durationMs}ms.`
+                : `Case failed ${evaluation.failures.length} check(s) in ${durationMs}ms.`,
+            payload: {
+                durationMs,
+                expected: testCase.expected,
+            },
+        },
+    ];
+}
+function summarizeFollowUpForTrace(followUp) {
+    if (!followUp)
+        return undefined;
+    const priorResultRef = followUp.priorResultRef
+        ? {
+            id: followUp.priorResultRef.id,
+            question: followUp.priorResultRef.question,
+            columns: followUp.priorResultRef.columns,
+            rowCount: followUp.priorResultRef.rowCount,
+            sourceSql: truncateTraceText(followUp.priorResultRef.sourceSql, 4000),
+        }
+        : undefined;
+    const priorDqlArtifact = followUp.priorDqlArtifact
+        ? {
+            kind: followUp.priorDqlArtifact.kind,
+            name: followUp.priorDqlArtifact.name,
+            sourcePath: followUp.priorDqlArtifact.sourcePath,
+            source: truncateTraceText(followUp.priorDqlArtifact.source, 4000),
+            metrics: followUp.priorDqlArtifact.metrics,
+            dimensions: followUp.priorDqlArtifact.dimensions,
+            filters: followUp.priorDqlArtifact.filters,
+            timeDimension: followUp.priorDqlArtifact.timeDimension,
+            orderBy: followUp.priorDqlArtifact.orderBy,
+            limit: followUp.priorDqlArtifact.limit,
+        }
+        : undefined;
+    return {
+        kind: followUp.kind,
+        sourceTurnId: followUp.sourceTurnId,
+        sourceBlockName: followUp.sourceBlockName,
+        sourceQuestion: followUp.sourceQuestion,
+        filters: followUp.filters,
+        dimensions: followUp.dimensions,
+        priorResultColumns: followUp.priorResultColumns,
+        priorResultValues: followUp.priorResultValues,
+        priorResultRef,
+        priorDqlArtifact,
+        priorLimit: followUp.priorLimit,
+        priorMeasures: followUp.priorMeasures,
+        resolvedReferences: followUp.resolvedReferences,
+        unresolvedReferences: followUp.unresolvedReferences,
+    };
+}
+function truncateTraceText(value, maxLength) {
+    if (!value || value.length <= maxLength)
+        return value;
+    return `${value.slice(0, maxLength)}...`;
+}
 function ratio(numerator, denominator) {
     if (denominator === 0)
         return null;
@@ -516,6 +1109,17 @@ function average(values) {
         return 0;
     return Number((values.reduce((sum, value) => sum + value, 0) / values.length).toFixed(2));
 }
+function summarizeResultColumns(columns) {
+    return (columns ?? []).map((column) => {
+        if (typeof column === 'string')
+            return column;
+        if (column && typeof column === 'object' && 'name' in column) {
+            const name = column.name;
+            return typeof name === 'string' ? name : JSON.stringify(column);
+        }
+        return String(column);
+    });
+}
 function formatRate(value) {
     return value === null ? 'n/a' : `${Math.round(value * 1000) / 10}%`;
 }
@@ -588,6 +1192,10 @@ async function executeRuntimeCell(runtimeBase, cell) {
     return payload;
 }
 export const __test__ = {
+    agentEvalThresholdsPass,
+    buildEvalTrace,
+    cliAnalysisDepth,
+    cliReasoningEffort,
     computeEvalMetrics,
     evaluateCase,
 };