@duckcodeailabs/dql-cli 1.10.0 → 1.10.1

package/dist/local-runtime.js

@@ -10,7 +10,7 @@ import { Readable } from "node:stream";
 import { buildMixedSourceWarehouseFallbackSql, findMentionedNotebookDataset, planMixedSourceNotebookSql, planMixedSourceSql, } from "./mixed-source-sql.js";
 import { resolveNpmInvocation } from './npm-runtime.js';
 import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, hasStandaloneSemanticRef, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
-import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, resolveRepoSource, getDialect, Parser, NodeKind, blockParameterDefinitions, buildLineageGraph, buildManifest, collectInputFiles, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, domainFolderSlug, previewModelingChange, applyModelingChange, previewDbtSourcePatch, applyDbtSourcePatch, loadDomainPackageRegistry, loadDbtNodeAuthoringDetail, relationshipValidationProofFingerprint, discoverDbtDomains, renderDomainDeclaration, ProjectSnapshotService, analyzeSqlReferences, } from '@duckcodeailabs/dql-core';
+import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, resolveRepoSource, getDialect, Parser, NodeKind, blockParameterDefinitions, buildLineageGraph, buildManifest, collectInputFiles, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, domainFolderSlug, previewModelingChange, applyModelingChange, previewDbtSourcePatch, applyDbtSourcePatch, loadDomainPackageRegistry, loadDbtNodeAuthoringDetail, normalizeAnalyticalFailureV1, relationshipValidationProofFingerprint, discoverDbtDomains, renderDomainDeclaration, ProjectSnapshotService, analyzeSqlReferences, } from '@duckcodeailabs/dql-core';
 import { load as loadYaml } from 'js-yaml';
 import { listBlockTemplates } from './block-templates.js';
 import { getRunner as getLLMRunner } from './llm/index.js';
@@ -19,7 +19,7 @@ import { fetchLatestPublishedDqlVersion, resolveDqlRuntimeVersionStatus } from '
 import { resolveRetrievalHealthStatus } from './retrieval-health.js';
 import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
 import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
-import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, } from '@duckcodeailabs/dql-agent';
+import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, } from '@duckcodeailabs/dql-agent';
 import { gatherProposeEnrichment } from './propose-enrich.js';
 import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
 import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
@@ -279,6 +279,53 @@ export function agentAnswerHasExecutionFailure(governedAnswer) {
     return typeof governedAnswer.executionError === 'string'
         && governedAnswer.executionError.trim().length > 0;
 }
+/** Rebuild the immutable failed-run input from the artifact retained by API-007. */
+export function analyticalFailedRunFromAgentRun(run) {
+    for (const artifact of run.artifacts) {
+        const payload = agentRunRecord(artifact.payload);
+        if (!payload)
+            continue;
+        const failure = normalizeAnalyticalFailureV1(payload.analyticalFailure);
+        const plan = agentRunRecord(payload.resolvedAnalyticalPlan);
+        const planFingerprint = agentRunString(plan?.fingerprint) ?? failure?.planFingerprint;
+        if (!failure || !planFingerprint)
+            continue;
+        const dqlArtifact = normalizeDqlArtifactReference(payload.dqlArtifact);
+        const dqlSource = dqlArtifact?.source;
+        const compiledSql = agentRunString(payload.sql)
+            ?? agentRunString(payload.proposedSql)
+            ?? dqlArtifact?.compiledSql;
+        const graph = agentRunRecord(payload.analyticalExecutionGraph);
+        const rawRoute = agentRunString(graph?.route) ?? agentRunString(plan?.recommendedRoute);
+        const route = rawRoute === 'certified'
+            ? 'certified'
+            : rawRoute === 'semantic'
+                ? 'semantic'
+                : rawRoute === 'governed_sql'
+                    ? 'governed_sql'
+                    : 'generated_sql';
+        const rawTrust = dqlArtifact?.trustState ?? agentRunString(payload.reviewStatus) ?? artifact.trustState;
+        const trustState = rawTrust === 'certified'
+            ? 'certified'
+            : rawTrust === 'governed'
+                ? 'governed'
+                : 'review_required';
+        return {
+            version: 1,
+            runId: failure.runId,
+            snapshotId: failure.snapshotId,
+            route,
+            trustState,
+            planFingerprint,
+            ...(dqlSource ? { dqlSource } : {}),
+            ...(failure.dqlFingerprint ? { dqlFingerprint: failure.dqlFingerprint } : {}),
+            ...(compiledSql ? { compiledSql } : {}),
+            ...(failure.sqlFingerprint ? { sqlFingerprint: failure.sqlFingerprint } : {}),
+            failure,
+        };
+    }
+    return undefined;
+}
 function businessNarrativeGaps(warnings) {
     const businessRelevant = (warnings ?? []).filter((warning) => !(/missing requested output column/i.test(warning)
         || /\bquery plan\b/i.test(warning)
@@ -1220,6 +1267,57 @@ export async function startLocalServer(opts) {
             },
             ...(semanticDriver ? { semanticDriver } : {}),
             ...(semanticTableMapping ? { semanticTableMapping } : {}),
+            ...(semanticLayer && semanticConnection ? {
+                resolveAnalyticalFreshness: async (freshnessRequest) => {
+                    const binding = freshnessRequest.authorizedAdapterRequest;
+                    if (!binding || binding.route !== 'semantic') {
+                        throw Object.assign(new Error('The analytical freshness request is not bound to an authorized semantic route.'), {
+                            code: 'FRESHNESS_BINDING_REQUIRED',
+                        });
+                    }
+                    if (freshnessRequest.snapshotId !== routeDecision?.resolvedAnalyticalPlan?.snapshotId) {
+                        throw Object.assign(new Error('The analytical freshness request does not match the routed metadata snapshot.'), {
+                            code: 'SNAPSHOT_DRIFT',
+                        });
+                    }
+                    projectSnapshot();
+                    projectSnapshots.assertCurrent(runProjectSnapshot.snapshotId);
+                    const compiled = await compileSemanticRuntimeQuery({
+                        metrics: [binding.metric],
+                        dimensions: [],
+                        timeDimension: {
+                            name: binding.timeDimension,
+                            granularity: binding.granularity,
+                        },
+                        orderBy: [{ name: binding.outputField, direction: 'desc' }],
+                        limit: 1,
+                    }, {
+                        projectRoot,
+                        projectConfig,
+                        detectedProvider: semanticDetectedProvider,
+                        semanticLayer: semanticLayer,
+                        driver: semanticDriver,
+                        tableMapping: semanticTableMapping,
+                    });
+                    if (!compiled) {
+                        throw Object.assign(new Error('The authorized semantic freshness query could not be compiled.'), {
+                            code: 'COMPILATION_FAILED',
+                        });
+                    }
+                    const prepared = prepareLocalExecution(compiled.sql, semanticConnection, projectRoot, projectConfig);
+                    const result = await executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
+                    projectSnapshot();
+                    projectSnapshots.assertCurrent(runProjectSnapshot.snapshotId);
+                    return {
+                        version: 1,
+                        snapshotId: freshnessRequest.snapshotId,
+                        metricId: freshnessRequest.metricId,
+                        timeDimensionId: freshnessRequest.timeDimensionId,
+                        observedThrough: analyticalFreshnessObservedThrough(result, freshnessRequest),
+                        observedAt: new Date().toISOString(),
+                    };
+                },
+            } : {}),
             ...(semanticLayer ? {
                 semanticQueryCompiler: async (selection) => {
                     const compiled = await compileSemanticRuntimeQuery({
@@ -1262,6 +1360,7 @@ export async function startLocalServer(opts) {
             ...(routeDecision?.resolvedAnalyticalPlan
                 ? { resolvedAnalyticalPlan: routeDecision.resolvedAnalyticalPlan }
                 : {}),
+            analyticalReferenceInstant: new Date().toISOString(),
             executeCertifiedBlock: executeCertifiedBlockForAgent,
             executeGeneratedSql: (sql, artifact) => executeGeneratedArtifactForAgent(request.question, sql, artifact),
             executeDqlArtifact: (artifact) => executeArtifactReferenceForAgent(artifact, request.question),
@@ -1622,6 +1721,8 @@ export async function startLocalServer(opts) {
             && (governedAnswer.refusalCode === 'grounding_gap' || governedAnswer.refusalCode === 'modeling_gap')
             && !isExploratory;
         const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
+        const isAnalyticalFailure = Boolean(governedAnswer.analyticalFailure);
+        const isTerminalFailure = isProviderError || isExecutionFailure || isAnalyticalFailure;
         // AGT-004: a rejected attribution/export/proof policy is a deliberate
         // governance boundary, not an ambiguous user question and not a repairable
         // retrieval miss. Keep the precise policy detail visible, but never burn two
@@ -1681,14 +1782,14 @@ export async function startLocalServer(opts) {
                 synthesizedAnswer = undefined;
             }
         }
-        const status = isProviderError || isExecutionFailure
+        const status = isTerminalFailure
             ? 'blocked'
             : needsClarification
                 ? 'needs_clarification'
                 : isCertified || isSemantic
                     ? 'completed'
                     : 'needs_review';
-        const trustState = isProviderError || isExecutionFailure
+        const trustState = isTerminalFailure
             ? 'blocked'
             : needsClarification
                 ? 'not_applicable'
@@ -1697,7 +1798,7 @@ export async function startLocalServer(opts) {
                     : isSemantic
                         ? 'governed'
                         : 'review_required';
-        const stopReason = isProviderError || isExecutionFailure
+        const stopReason = isTerminalFailure
             ? 'blocked'
             : needsClarification
                 ? 'needs_clarification'
@@ -1706,10 +1807,23 @@ export async function startLocalServer(opts) {
                     : isSemantic
                         ? 'governed_semantic_answer'
                         : 'human_review_required';
+        const terminalFailureActions = governedAnswer.analyticalFailure
+            ? governedAnswer.analyticalFailure.code === 'PERMISSION_DENIED'
+                ? [{ id: 'repair-access', label: 'Change connection or request access', route: 'blocked' }]
+                : governedAnswer.analyticalFailure.code === 'TIMEOUT'
+                    ? [{ id: 'retry-same-plan', label: 'Retry the same bounded plan', route: 'generated_answer' }]
+                    : governedAnswer.analyticalFailure.code === 'SNAPSHOT_DRIFT'
+                        || governedAnswer.analyticalFailure.code === 'COLUMN_NOT_FOUND'
+                        || governedAnswer.analyticalFailure.code === 'RELATION_NOT_FOUND'
+                        ? [{ id: 'refresh-snapshot', label: 'Refresh the governed snapshot before retrying', route: 'blocked' }]
+                        : [{ id: 'review-analytical-failure', label: 'Review the plan, DQL, SQL, and safe repair actions', route: 'blocked' }]
+            : isProviderError
+                ? [{ id: 'retry-after-provider', label: 'Retry after fixing the AI provider', route: 'generated_answer' }]
+                : [{ id: 'retry-after-connection', label: 'Retry after fixing the database connection', route: 'generated_answer' }];
         const nextActions = needsClarification
             ? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
-            : isExecutionFailure
-                ? [{ id: 'retry-after-connection', label: 'Retry after fixing the database connection', route: 'generated_answer' }]
+            : isTerminalFailure
+                ? terminalFailureActions
                 : isGroundingGap
                     ? [{ id: 'research-gap', label: 'Research missing metadata coverage', route: 'research', artifactKind: 'research_run' }]
                     : [
@@ -1726,15 +1840,15 @@ export async function startLocalServer(opts) {
             resolvedRoute,
             answerRefusalCode: governedAnswer.kind === 'no_answer' ? governedAnswer.refusalCode : undefined,
             answerTier: governedAnswer.route?.tier,
-            summary: isExecutionFailure
+            summary: isTerminalFailure
                 ? 'The governed query could not be executed.'
                 : governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : isExploratory ? 'Exploratory DBT-grounded analysis requires review.' : 'Answered with review-required generated analysis.'),
             answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
             status,
             trustState,
             stopReason,
-            artifacts: isExecutionFailure
-                ? []
+            artifacts: isTerminalFailure
+                ? [agentRunArtifact('answer', 'Failed governed analytical run', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, 'blocked')]
                 : governedAnswer.kind === 'no_answer'
                     // A refusal still keeps the DQL draft the answer loop produced (when any),
                     // so the "Review DQL draft" next-action isn't a dead link and the user can
@@ -2415,6 +2529,8 @@ export async function startLocalServer(opts) {
             snapshotId: pack.knowledgeLens.snapshotId,
             sourceFingerprint: pack.freshness.fingerprint ?? undefined,
             knowledgeLens: pack.knowledgeLens,
+            analyticalPolicies: pack.skills.flatMap((skill) => skill.analyticalPolicy ? [skill.analyticalPolicy] : []),
+            contextObjects: pack.objects,
             durationMs: Date.now() - startedAt,
             truncated: pack.retrievalDiagnostics.topRejected.length > 0,
         });
@@ -4808,6 +4924,50 @@ export async function startLocalServer(opts) {
             res.end(serializeJSON({ runs, total: agentRunStore.list().length, limit }));
             return;
         }
+        if (req.method === 'POST' && /^\/api\/agent-runs\/[^/]+\/analytical-repair$/.test(path)) {
+            const match = path.match(/^\/api\/agent-runs\/([^/]+)\/analytical-repair$/);
+            const id = decodeURIComponent(match?.[1] ?? '');
+            const run = await agentRunStore.get(id);
+            if (!run) {
+                res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: 'Agent run not found.' }));
+                return;
+            }
+            const source = analyticalFailedRunFromAgentRun(run);
+            if (!source) {
+                res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: 'This run has no retained analytical failure artifact.' }));
+                return;
+            }
+            const body = await readJSON(req).catch(() => null);
+            const record = agentRunRecord(body);
+            const repair = (agentRunRecord(record?.repair) ?? record);
+            if (!repair) {
+                res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: 'A v1 analytical repair request is required.' }));
+                return;
+            }
+            // API-007: snapshot refresh is resolved by the runtime, not trusted from
+            // browser input. projectSnapshot() recompiles when governed sources have
+            // changed and gives the immutable identity used by the derivation.
+            const repairWithRuntimeContext = repair.action === 'refresh_snapshot'
+                ? { ...repair, refreshedSnapshotId: projectSnapshot().snapshotId }
+                : repair;
+            const result = deriveAnalyticalRepair(source, repairWithRuntimeContext);
+            if (result.status === 'blocked') {
+                const status = result.code === 'SOURCE_FINGERPRINT_MISMATCH'
+                    ? 409
+                    : result.code === 'PERMISSION_FAILURE_TERMINAL' || result.code === 'ACTION_NOT_ALLOWED'
+                        ? 403
+                        : 400;
+                res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON(result));
+                return;
+            }
+            res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
+            res.end(serializeJSON(result));
+            return;
+        }
         if (req.method === 'POST' && /^\/api\/agent-runs\/[^/]+\/cancel$/.test(path)) {
             const match = path.match(/^\/api\/agent-runs\/([^/]+)\/cancel$/);
             const id = decodeURIComponent(match?.[1] ?? '');
@@ -11462,6 +11622,48 @@ export function serializeJSON(value) {
         return current;
     });
 }
+/**
+ * Extract the single time value returned by an authorized semantic freshness
+ * lookup. Adapter aliases may differ by warehouse, so typed date metadata is
+ * the only fallback; arbitrary string columns are never treated as freshness.
+ * Acceptance: AGT-019, SEC-004.
+ */
+export function analyticalFreshnessObservedThrough(result, request) {
+    const binding = request.authorizedAdapterRequest;
+    if (!binding || binding.route !== 'semantic') {
+        throw Object.assign(new Error('The freshness lookup has no authorized semantic adapter binding.'), {
+            code: 'FRESHNESS_BINDING_REQUIRED',
+        });
+    }
+    const row = result.rows[0];
+    if (!row) {
+        throw Object.assign(new Error('The authorized freshness lookup returned no rows.'), {
+            code: 'FRESHNESS_OBSERVATION_MISSING',
+        });
+    }
+    const normalize = (value) => value.replace(/[^a-zA-Z0-9]+/g, '_').toLowerCase();
+    const target = normalize(binding.outputField);
+    const exact = result.columns.find((column) => normalize(column.name) === target);
+    const typedDates = result.columns.filter((column) => column.type === 'date' || column.type === 'datetime');
+    const column = exact ?? (typedDates.length === 1 ? typedDates[0] : undefined);
+    if (!column) {
+        throw Object.assign(new Error('The authorized freshness lookup did not return one identifiable time field.'), {
+            code: 'FRESHNESS_OBSERVATION_INVALID',
+        });
+    }
+    const value = row[column.name];
+    const instant = value instanceof Date
+        ? value
+        : typeof value === 'string' && value.trim()
+            ? new Date(value)
+            : undefined;
+    if (!instant || Number.isNaN(instant.getTime())) {
+        throw Object.assign(new Error('The authorized freshness value is not a valid instant.'), {
+            code: 'FRESHNESS_OBSERVATION_INVALID',
+        });
+    }
+    return instant.toISOString();
+}
 /** Serialize a Skill to the shared API contract shape (spec 16). */
 function serializeSkill(skill) {
     return {