@duckcodeailabs/dql-cli 1.14.0 → 1.14.1

package/dist/llm/providers/dql-agent-provider.js

@@ -1,6 +1,10 @@
-import { ClaudeProvider, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, GeminiProvider, loadAgentSemanticLayer, OllamaProvider, OpenAIProvider, answer, buildAnalysisQuestionPlan, buildLocalContextPack, contextRetrievalBudgetForQuestion, ensureAgentProjectReady, isLikelyClarificationReply, isTrustedConversationTurn, createProviderDispatchEgressReceipt, createProviderEgressReceipt, assertProviderPayloadAllowed, prepareProviderContextForDispatch, prepareProviderWireEnvelopeForDispatch, prepareServerOwnedProviderSchemaContext, } from '@duckcodeailabs/dql-agent';
+import { deadlineScale, planAnalystTurn, rerankCandidates } from '@duckcodeailabs/dql-agent';
+import { buildAnalystLoopTools } from '../analyst-loop-tools.js';
+import { ClaudeProvider, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, GeminiProvider, loadAgentSemanticLayer, OllamaProvider, OpenAIProvider, answer, buildAnalysisQuestionPlan, buildLocalContextPack, contextRetrievalBudgetForQuestion, ensureAgentProjectReady, isLikelyClarificationReply, isTrustedConversationTurn, createProviderDispatchEgressReceipt, createProviderEgressReceipt, assertProviderPayloadAllowed, prepareProviderContextForDispatch, prepareProviderWireEnvelopeForDispatch, prepareServerOwnedProviderSchemaContext, projectEmbeddingProvider, answerAgentic, qualifyAuthorizationReferences, createAnalystLaneHandler, parseProposal, renderContextValidationRefusalForUser, validateSqlAgainstLocalContext, resolveOrchestratorPolicy, } from '@duckcodeailabs/dql-agent';
+import { CassetteStore, resolveCassetteModeFromEnv, withCassette } from '../../commands/agent-eval-cassette.js';
 import { buildManifest, normalizeDqlArtifactReference, resolveDbtManifestPath } from '@duckcodeailabs/dql-core';
-import { existsSync } from 'node:fs';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { buildAnswerLoopTools, createGroundingContextExpander } from '../answer-loop-tools.js';
 import { getSemanticRuntimeStatus } from '../../semantic-runtime.js';
 import { blockProposalDqlMetadata } from '../proposal-metadata.js';
@@ -257,12 +261,131 @@ function emitProposalFromText(text, emit) {
         // Ignore malformed proposal text. The visible assistant response still streams as text.
     }
 }
+/**
+ * Route the runtime's own provider through an eval cassette when the host asks.
+ *
+ * The client-side cassette in `dql agent eval` only covers `--via loop`: with
+ * `--via runtime` the SERVER owns the provider, so without this hook the one
+ * driver that actually exercises routing and gates could never be made
+ * deterministic — and a suite that cannot be deterministic cannot gate a PR.
+ *
+ * Opt-in through the environment, never through a request field: a caller must
+ * not be able to redirect a production run onto recorded responses.
+ */
+/**
+ * Read `agent.orchestrator` straight from dql.config.json.
+ *
+ * Deliberately NOT `loadProjectConfig` from local-runtime: that module imports
+ * this one (local-runtime.ts:164), so reaching back would close an import cycle
+ * through a 34k-line file. A few lines of JSON reading is the cheaper trade.
+ *
+ * Cached per project root because this runs on every turn and the answer is a
+ * migration setting, not live state. A malformed file resolves to `null`, which
+ * `resolveOrchestratorPolicy` turns into `legacy` — a broken config must not
+ * route real questions onto an unproven path.
+ */
+const agentConfigCache = new Map();
+function readAgentConfig(projectRoot) {
+    const cached = agentConfigCache.get(projectRoot);
+    if (cached !== undefined)
+        return cached;
+    let resolved = null;
+    try {
+        const raw = readFileSync(join(projectRoot, 'dql.config.json'), 'utf-8');
+        const parsed = JSON.parse(raw);
+        resolved = parsed?.agent && typeof parsed.agent === 'object'
+            ? parsed.agent
+            : null;
+    }
+    catch {
+        resolved = null;
+    }
+    agentConfigCache.set(projectRoot, resolved);
+    return resolved;
+}
+function readOrchestratorConfig(projectRoot) {
+    const orchestrator = readAgentConfig(projectRoot)?.orchestrator;
+    return orchestrator && typeof orchestrator === 'object' ? orchestrator : null;
+}
+/**
+ * Is on-demand value lookup permitted for this project?
+ *
+ * Mirrors `resolveAgentRuntimeValueGrounding`, read locally rather than imported
+ * — local-runtime imports this module, so reaching back would close a cycle
+ * through a 34k-line file. Anything other than an explicit `safe_automatic`
+ * resolves to disabled: value lookup touches warehouse cell values, so a
+ * malformed setting must fail closed.
+ */
+/**
+ * Structured turn planning is OFF by default.
+ *
+ * It costs one provider dispatch before the loop makes any tool call, and on a
+ * slow provider that dispatch can consume enough of the discovery window that
+ * the loop is then refused admission and falls back to the legacy path —
+ * measured on ollama, where all three agentic dispatches planned successfully
+ * and then died at "soft target elapsed before this provider dispatch could
+ * start". Its payoff is a legible trace, and `onStep` is not wired to SSE yet,
+ * so today it buys nothing a user can see. Opt in with
+ * `agent.orchestrator.turnPlanning: true` once streaming lands.
+ */
+function turnPlanningEnabled(projectRoot) {
+    const orchestrator = readAgentConfig(projectRoot)?.orchestrator;
+    if (!orchestrator || typeof orchestrator !== 'object')
+        return false;
+    return orchestrator.turnPlanning === true;
+}
+function valueLookupEnabled(projectRoot) {
+    const grounding = readAgentConfig(projectRoot)?.runtimeValueGrounding;
+    if (!grounding || typeof grounding !== 'object')
+        return false;
+    return grounding.mode === 'safe_automatic';
+}
+/**
+ * Which migration lane this turn belongs to.
+ *
+ * Research is identified by the server-resolved `orchestrationMode`, not by
+ * `analysisDepth`. A reader can ask Ask AI to think deeply without opting into
+ * the Research workflow, its wider dispatch budget, or any row-bearing tools.
+ *
+ * Deliberately coarse: the seam only needs to know which bucket a turn falls in
+ * so a lane can be enabled independently. The fine-grained triage between
+ * certified, semantic, and generated stays inside the answer path, where the
+ * retrieval evidence lives.
+ */
+function agenticLaneForRequest(req) {
+    return req.orchestrationMode === 'research' ? 'research' : 'generated';
+}
+export function applyEvalCassette(provider) {
+    const dir = process.env.DQL_EVAL_CASSETTE_DIR;
+    if (!dir)
+        return provider;
+    return withCassette(provider, new CassetteStore(dir), resolveCassetteModeFromEnv(process.env));
+}
+/**
+ * A raw text provider for planning calls that are not the answer itself.
+ *
+ * Research hypothesis planning needs `generate`, not the full agent runner —
+ * and the runner cannot be reused for it, because the runner IS the governed
+ * answer path. Cassettes apply, so a recorded run stays hermetic.
+ */
+export function createGovernedTextProvider(id, projectRoot) {
+    const spec = SPECS[id];
+    if (!spec)
+        return undefined;
+    try {
+        return applyEvalCassette(spec.create(projectRoot));
+    }
+    catch {
+        return undefined;
+    }
+}
 export function createDqlAgentProviderRunner(id, providerOverride) {
     return {
         async run(req, emit, signal) {
             const spec = SPECS[id];
-            const rawProvider = providerOverride ?? spec.create(req.projectRoot);
-            const isResearch = req.analysisDepth === 'deep';
+            const rawProvider = applyEvalCassette(providerOverride ?? spec.create(req.projectRoot));
+            const isResearch = req.orchestrationMode === 'research';
+            const maxProviderDispatches = isResearch ? 8 : 4;
             const researchRowsOptIn = isResearch && req.researchResultRowsOptIn === true;
             const sharedDispatchEvidence = req.providerDispatchEvidenceSink;
             let providerRoundTrips = 0;
@@ -296,7 +419,7 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                 // A tool-calling loop capped at two physical sends can make exactly one
                 // tool call before it must answer, which is not enough to look something
                 // up and then use it. The run-scoped ledger is the real guardrail.
-                maxProviderDispatches: isResearch ? 8 : 4,
+                maxProviderDispatches,
                 ...(sharedDispatchEvidence?.mayStartToolCall
                     ? { mayStartToolCall: () => sharedDispatchEvidence.mayStartToolCall() }
                     : {}),
@@ -439,6 +562,14 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                         strictness: contextBudget.strictness,
                         limit: contextBudget.limit,
                         confirmCertifiedFit: createCertifiedFitConfirmation(provider, signal),
+                        // A cross-encoder pass over the fused candidates. Advisory: it may
+                        // only reorder what retrieval returned, and a failure or timeout
+                        // leaves retrieval's ordering untouched — so it can improve the
+                        // pack and cannot break it.
+                        rerankCandidates: (rerankQuestion, candidates) => rerankCandidates(provider, rerankQuestion, candidates, {
+                            ...(signal ? { signal } : {}),
+                            timeoutMs: Math.round(2_500 * deadlineScale()),
+                        }),
                         // Conversation-aware reuse: same-topic follow-ups seed (or, for
                         // filter-only refinements, re-stamp) the prior turn's context pack.
                         priorContextPackId: priorContextPackIdFromSnapshot(conversationSnapshot),
@@ -533,11 +664,17 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                         maxResultRows: 0,
                         purpose: 'answer_generation',
                     });
-                    const result = await answer({
+                    // The strangler seam. `answerAgentic` and `answer` are interchangeable
+                    // here; which runs is a per-lane config decision that defaults to
+                    // legacy, so this is a no-op until a lane is explicitly enabled.
+                    const answerLoopInput = {
                         question,
                         ...(req.resolvedAnalyticalPlan
                             ? { resolvedAnalyticalPlan: req.resolvedAnalyticalPlan }
                             : {}),
+                        ...(req.generatedProposalTargetFingerprint
+                            ? { generatedProposalTargetFingerprint: req.generatedProposalTargetFingerprint }
+                            : {}),
                         ...(req.analyticalReferenceInstant
                             ? { analyticalReferenceInstant: req.analyticalReferenceInstant }
                             : {}),
@@ -568,6 +705,12 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                             }
                             : {}),
                         contextPack,
+                        // The project's configured embedder (dql.config.json ai.embeddings).
+                        // Without it, matchSemanticMetric falls back to the offline hashed
+                        // provider, whose vectors can never ground a match on similarity
+                        // alone — so a metric named only by synonym or acronym is invisible
+                        // and the router dead-ends on a bare-ranking clarification.
+                        embeddingProvider: projectEmbeddingProvider(req.projectRoot),
                         signal,
                         reasoningEffort: req.reasoningEffort,
                         analysisDepth: contextBudget.analysisDepth,
@@ -583,6 +726,19 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                         executeGeneratedSql: req.executeGeneratedSql
                             ? async (...args) => { guardSnapshot(); sqlExecutions += 1; return req.executeGeneratedSql(...args); }
                             : undefined,
+                        executeAgenticGeneratedSql: req.executeAgenticGeneratedSql
+                            ? async (capability, sql, artifact) => {
+                                guardSnapshot();
+                                sqlExecutions += 1;
+                                return req.executeAgenticGeneratedSql(capability, sql, artifact);
+                            }
+                            : undefined,
+                        agenticExecutionScope: {
+                            runId: req.agentRunId,
+                            snapshotId: req.projectSnapshot?.snapshotId,
+                            planId: req.resolvedAnalyticalPlan?.planId,
+                            targetFingerprint: req.generatedProposalTargetFingerprint,
+                        },
                         executeDqlArtifact: req.executeDqlArtifact
                             ? async (...args) => { guardSnapshot(); sqlExecutions += 1; return req.executeDqlArtifact(...args); }
                             : undefined,
@@ -623,6 +779,111 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                         // NOTE: no captureGeneratedDraft here — a plain answer/research question must NOT
                         // auto-write a draft into the blocks space. A draft is created only when the user
                         // explicitly acts (the "Create DQL draft" action → the dql_block_draft route).
+                    };
+                    const result = await answerAgentic(answerLoopInput, {
+                        policy: resolveOrchestratorPolicy({
+                            config: readOrchestratorConfig(req.projectRoot),
+                        }),
+                        lane: agenticLaneForRequest(req),
+                        legacy: answer,
+                        // The generated lane runs the analyst loop: it verifies every
+                        // identifier against a tool observation before the SQL is executed.
+                        // Certified and semantic lanes are deliberately NOT registered —
+                        // their answers already come from a governed contract, so a
+                        // verification pass would add latency and no safety.
+                        handlers: {
+                            generated: createAnalystLaneHandler({
+                                legacy: answer,
+                                buildDeps: (loopInput) => {
+                                    const execute = loopInput.executeGeneratedSql;
+                                    const valuesEnabled = valueLookupEnabled(req.projectRoot);
+                                    const tools = buildAnalystLoopTools(loopInput, { valuesEnabled });
+                                    if (process.env.DQL_ORCHESTRATOR_TRACE) {
+                                        console.warn(`[dql] analyst loop deps: tools=${tools.length} preview=${Boolean(execute)} values=${valuesEnabled}`);
+                                    }
+                                    if (tools.length === 0)
+                                        return undefined;
+                                    return {
+                                        tools,
+                                        maxIterations: resolveOrchestratorPolicy({
+                                            config: readOrchestratorConfig(req.projectRoot),
+                                        }).maxIterations,
+                                        // Match the physical wrapper cap and reserve the final
+                                        // composition dispatch: an ordinary cap of four permits
+                                        // at most three text-protocol tool observations.
+                                        maxProviderDispatches,
+                                        // Scaled by the same knob as every other agent deadline, so
+                                        // a local model that needs seconds per call is not planned
+                                        // out of existence by a budget calibrated for a hosted one.
+                                        ...(turnPlanningEnabled(req.projectRoot) ? {
+                                            planTurn: async (question, toolNames) => {
+                                                const plan = await planAnalystTurn(loopInput.provider, question, toolNames, {
+                                                    timeoutMs: Math.round(2_500 * deadlineScale()),
+                                                    ...(loopInput.signal ? { signal: loopInput.signal } : {}),
+                                                });
+                                                if (process.env.DQL_ORCHESTRATOR_TRACE) {
+                                                    console.warn(plan
+                                                        ? `[dql] analyst turn plan: "${plan.restatement}" establish=${plan.mustEstablish.length} opening=${plan.openingTool ?? 'unset'}`
+                                                        : '[dql] analyst turn plan: unavailable (loop proceeds unplanned)');
+                                                }
+                                                return plan;
+                                            },
+                                        } : {}),
+                                        // The loop's trace, on the channel the provider already uses
+                                        // for progress. `thinking` turns become `onProgress`, which
+                                        // the run engine emits as `executor.started` over SSE — so
+                                        // this needs no new event type, no contract change, and no
+                                        // UI work. Without it the loop does real work (tool calls,
+                                        // identifier checks, a bounded repair) behind a silent
+                                        // spinner, which reads as a hang rather than as an analyst
+                                        // establishing facts.
+                                        onStep: (step) => {
+                                            emit({
+                                                kind: 'thinking',
+                                                text: step.detail ? `${step.label} — ${step.detail}` : step.label,
+                                            });
+                                        },
+                                        // Reuse the legacy parser and validator rather than forking
+                                        // a second SQL front end that would drift from the first.
+                                        parseSql: (raw) => parseProposal(raw).sql,
+                                        extractReferences: (sql) => {
+                                            const validation = validateSqlAgainstLocalContext(sql, loopInput.contextPack);
+                                            const qualified = qualifyAuthorizationReferences(sql, {
+                                                relations: validation.referencedRelations ?? [],
+                                                columns: validation.referencedColumns ?? [],
+                                            });
+                                            return {
+                                                relations: validation.referencedRelations ?? [],
+                                                columns: qualified.filter((reference) => !validation.referencedRelations?.includes(reference)),
+                                            };
+                                        },
+                                        // The safety verifiers keep their logic; only what happens on
+                                        // failure changes. Instead of ending the turn, the specific
+                                        // check that fired comes back as something the model can act
+                                        // on — "joining those tables multiplies rows" rather than
+                                        // "nothing was executed".
+                                        verifySql: (sql) => {
+                                            const validation = validateSqlAgainstLocalContext(sql, loopInput.contextPack);
+                                            if (validation.ok)
+                                                return undefined;
+                                            return renderContextValidationRefusalForUser(validation.code, validation.error, loopInput.followUp?.memberBindings, validation.aggregationSafetyProof?.issueCodes);
+                                        },
+                                    };
+                                },
+                            }),
+                        },
+                        onDiagnostic: (event) => {
+                            if (event.kind === 'fallback') {
+                                console.warn(`[dql] agentic orchestrator fell back on ${event.lane}: ${event.reason}`);
+                            }
+                            else if (process.env.DQL_ORCHESTRATOR_TRACE) {
+                                // Opt-in dispatch trace. A migration where the new path silently
+                                // never runs looks identical to one where it runs and agrees,
+                                // and that ambiguity costs more to debug than the log costs to
+                                // carry.
+                                console.warn(`[dql] agentic orchestrator dispatched lane=${event.lane} mode=${event.mode}`);
+                            }
+                        },
                     });
                     const answerDurationMs = Date.now() - answerStartedAt;
                     // CTX-002: an answer built from one snapshot must never be published
@@ -652,7 +913,7 @@ export function createDqlAgentProviderRunner(id, providerOverride) {
                         sqlExecutions,
                         repairs: result.analysisPlan?.repairAttempts ?? 0,
                     };
-                    if (req.analysisDepth === 'deep' && !providerEgressReceipts.some((receipt) => receipt.purpose === 'research_tool')) {
+                    if (isResearch && !providerEgressReceipts.some((receipt) => receipt.purpose === 'research_tool')) {
                         providerEgressReceipts.push(createProviderEgressReceipt({
                             purpose: 'research_tool',
                             provider: provider.name,
@@ -1097,6 +1358,28 @@ function resolveAgentFollowUpContextRaw(rawContext, question) {
     const context = rawContext;
     if (!context)
         return undefined;
+    const taskDependency = analyticalTaskDependencyBindingFromContext(context);
+    // A compound child receives exactly one server-computed parent value plus
+    // result/row proofs. Do not replay parent prose, SQL, or result rows into the
+    // child; its normal governed retrieval and immutable-plan checks still run.
+    if (taskDependency) {
+        return {
+            kind: 'drilldown',
+            sourceTurnId: `task:${taskDependency.sourceTaskId}`,
+            filters: [taskDependency.value],
+            dimensions: [taskDependency.canonicalColumn],
+            priorResultColumns: [taskDependency.canonicalColumn],
+            priorResultValues: { [taskDependency.canonicalColumn]: [taskDependency.value] },
+            memberBindings: [{
+                    dimension: taskDependency.canonicalColumn,
+                    values: [taskDependency.value],
+                    source: 'prior_result',
+                    confidence: 'exact',
+                    sourceTurnId: `task:${taskDependency.sourceTaskId}`,
+                }],
+            resolvedReferences: [`${taskDependency.canonicalColumn}: ${taskDependency.value}`],
+        };
+    }
     const turns = conversationTurnsFromContext(context);
     const activeTurn = activeConversationTurn(context, turns, question);
     const activeResult = activeTurn?.result && typeof activeTurn.result === 'object' && !Array.isArray(activeTurn.result)
@@ -1193,6 +1476,21 @@ function resolveAgentFollowUpContextRaw(rawContext, question) {
         priorResult: relativeComparison ? undefined : priorResultDataFromTurn(activeResult, priorResultColumns),
     };
 }
+function analyticalTaskDependencyBindingFromContext(context) {
+    const raw = cleanRecord(context.analyticalTaskDependencyBinding);
+    if (!raw || raw.version !== 1)
+        return undefined;
+    const sourceTaskId = cleanOptionalString(raw.sourceTaskId);
+    const sourceResultFingerprint = cleanOptionalString(raw.sourceResultFingerprint)?.toLowerCase();
+    const canonicalColumn = cleanOptionalString(raw.canonicalColumn);
+    const value = cleanOptionalString(raw.value);
+    const rowFingerprint = cleanOptionalString(raw.rowFingerprint)?.toLowerCase();
+    if (!sourceTaskId || !canonicalColumn || !value
+        || !/^[a-f0-9]{64}$/.test(sourceResultFingerprint ?? '')
+        || !/^[a-f0-9]{64}$/.test(rowFingerprint ?? ''))
+        return undefined;
+    return { sourceTaskId, sourceResultFingerprint: sourceResultFingerprint, canonicalColumn, value, rowFingerprint: rowFingerprint };
+}
 /** Build the bounded prior-result rows (aligned to columns) for cross-result
  *  follow-up computation, from a turn's persisted result sample. */
 function priorResultDataFromTurn(activeResult, columns) {
@@ -1776,6 +2074,7 @@ function normalizePriorValueDimension(value) {
     return lower.replace(/[_\s-]+name$/, '').replace(/[^a-z0-9_ -]+/g, '').trim();
 }
 export const __test__ = {
+    agenticLaneForRequest,
     applyTopicShiftGuard,
     isDrilldownFollowUp,
     buildAnswerLoopTools,