@duckcodeailabs/dql-cli 1.7.2 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +69 -4
- package/dist/apps-api.js.map +1 -1
- package/dist/assets/dql-notebook/assets/{index-BgD9b9Om.js → index-siDRLswd.js} +380 -369
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/llm/answer-loop-tools.js +17 -4
- package/dist/llm/answer-loop-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts +1 -0
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +143 -26
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/types.d.ts +13 -0
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +26 -0
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +679 -139
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/providers/subscription-cli.d.ts +1 -0
- package/dist/providers/subscription-cli.d.ts.map +1 -1
- package/dist/providers/subscription-cli.js +40 -8
- package/dist/providers/subscription-cli.js.map +1 -1
- package/package.json +11 -11
package/dist/local-runtime.js
CHANGED
|
@@ -14,9 +14,9 @@ import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetri
|
|
|
14
14
|
import { load as loadYaml } from 'js-yaml';
|
|
15
15
|
import { listBlockTemplates } from './block-templates.js';
|
|
16
16
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
17
|
-
import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
|
|
17
|
+
import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
|
|
18
18
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
19
|
-
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode,
|
|
19
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, buildLocalContextPack, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
20
20
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
21
21
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
22
22
|
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
@@ -153,6 +153,20 @@ export function parseAgentRunRequestBody(body) {
|
|
|
153
153
|
},
|
|
154
154
|
};
|
|
155
155
|
}
|
|
156
|
+
const AGENT_LOOKUP_DEADLINE_MS = 45_000;
|
|
157
|
+
const AGENT_RESEARCH_DEADLINE_MS = 120_000;
|
|
158
|
+
/**
|
|
159
|
+
* PERF-002: one wall-clock budget follows the request through routing, provider
|
|
160
|
+
* calls, repair, and execution. Ordinary Ask never inherits Research's budget
|
|
161
|
+
* merely because it spans two tables; explicit/deep investigation does.
|
|
162
|
+
*/
|
|
163
|
+
export function agentRunDeadlineMs(request) {
|
|
164
|
+
if (request.requestedMode === 'research' || request.analysisDepth === 'deep') {
|
|
165
|
+
return AGENT_RESEARCH_DEADLINE_MS;
|
|
166
|
+
}
|
|
167
|
+
const plan = buildAnalysisQuestionPlan(request.question);
|
|
168
|
+
return plan.needsResearchWorkspace ? AGENT_RESEARCH_DEADLINE_MS : AGENT_LOOKUP_DEADLINE_MS;
|
|
169
|
+
}
|
|
156
170
|
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
157
171
|
if (governedAnswer.kind === 'no_answer')
|
|
158
172
|
return false;
|
|
@@ -283,6 +297,7 @@ function conversationTurnInputFromRun(run) {
|
|
|
283
297
|
sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
|
|
284
298
|
?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
|
|
285
299
|
contextPackId: agentRunString(payload?.contextPackId) ?? agentRunString(contextPack?.id),
|
|
300
|
+
knowledgeLens: agentRunRecord(contextPack?.knowledgeLens),
|
|
286
301
|
sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
|
|
287
302
|
dqlArtifact: agentRunRecord(payload?.dqlArtifact),
|
|
288
303
|
cascade: agentRunRecord(payload?.cascade),
|
|
@@ -442,6 +457,102 @@ export async function startLocalServer(opts) {
|
|
|
442
457
|
error: snapshot.error,
|
|
443
458
|
};
|
|
444
459
|
};
|
|
460
|
+
const latestDbtPreparationJob = () => Array.from(onboardingJobs.values())
|
|
461
|
+
.reverse()
|
|
462
|
+
.find((job) => job.kind === 'dbt_prepare' || job.kind === 'dbt_refresh');
|
|
463
|
+
const startDbtPreparationJob = (input) => {
|
|
464
|
+
const now = new Date().toISOString();
|
|
465
|
+
const id = apiRequestId(input.kind === 'dbt_refresh' ? 'dbt-refresh' : 'dbt-prepare');
|
|
466
|
+
const job = {
|
|
467
|
+
id,
|
|
468
|
+
kind: input.kind,
|
|
469
|
+
status: 'running',
|
|
470
|
+
stage: 'indexing',
|
|
471
|
+
progress: 65,
|
|
472
|
+
message: 'Indexing dbt models, columns, semantic metrics, certified blocks, and governed relationships.',
|
|
473
|
+
createdAt: now,
|
|
474
|
+
updatedAt: now,
|
|
475
|
+
snapshotId: input.snapshotId,
|
|
476
|
+
phases: [
|
|
477
|
+
{ id: 'artifact_validation', label: 'Validated dbt project and artifacts', status: 'completed', durationMs: input.validationDurationMs },
|
|
478
|
+
{ id: 'snapshot_compile', label: 'Compiled immutable project snapshot', status: 'completed', durationMs: input.compileDurationMs },
|
|
479
|
+
{ id: 'search_index', label: 'Build governed search indexes', status: 'running' },
|
|
480
|
+
],
|
|
481
|
+
};
|
|
482
|
+
if (onboardingJobs.size >= 24) {
|
|
483
|
+
for (const [existingId, existing] of onboardingJobs) {
|
|
484
|
+
if (existing.status === 'running' || existing.status === 'queued')
|
|
485
|
+
continue;
|
|
486
|
+
onboardingJobs.delete(existingId);
|
|
487
|
+
if (onboardingJobs.size < 16)
|
|
488
|
+
break;
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
onboardingJobs.set(id, job);
|
|
492
|
+
// Start after the Apply response can be returned. Governed Ask calls the
|
|
493
|
+
// same versioned preparation service, so an early first question awaits
|
|
494
|
+
// this in-flight promise instead of starting a duplicate cold rebuild.
|
|
495
|
+
setTimeout(() => {
|
|
496
|
+
void (async () => {
|
|
497
|
+
const indexStartedAt = Date.now();
|
|
498
|
+
try {
|
|
499
|
+
const prepared = await ensureAgentProjectReady(projectRoot, {
|
|
500
|
+
kgPath: defaultKgPath(projectRoot),
|
|
501
|
+
manifest: input.manifest,
|
|
502
|
+
});
|
|
503
|
+
const current = onboardingJobs.get(id);
|
|
504
|
+
if (!current || current.status === 'cancelled')
|
|
505
|
+
return;
|
|
506
|
+
const indexDurationMs = Date.now() - indexStartedAt;
|
|
507
|
+
const completedAt = new Date().toISOString();
|
|
508
|
+
onboardingJobs.set(id, {
|
|
509
|
+
...current,
|
|
510
|
+
status: 'completed',
|
|
511
|
+
stage: 'ready',
|
|
512
|
+
progress: 100,
|
|
513
|
+
message: `Ready. Indexed ${prepared.nodes.toLocaleString()} governed objects for fast search.`,
|
|
514
|
+
updatedAt: completedAt,
|
|
515
|
+
phases: current.phases.map((phase) => phase.id === 'search_index'
|
|
516
|
+
? { ...phase, status: 'completed', durationMs: indexDurationMs }
|
|
517
|
+
: phase),
|
|
518
|
+
result: {
|
|
519
|
+
snapshotId: input.snapshotId,
|
|
520
|
+
cacheHit: prepared.cacheHit,
|
|
521
|
+
sourceVersion: prepared.sourceVersion,
|
|
522
|
+
objectCount: prepared.nodes,
|
|
523
|
+
edgeCount: prepared.edges,
|
|
524
|
+
metadataFingerprint: prepared.metadataFingerprint,
|
|
525
|
+
kgFingerprint: prepared.kgFingerprint,
|
|
526
|
+
phaseDurationsMs: {
|
|
527
|
+
artifactValidation: input.validationDurationMs,
|
|
528
|
+
snapshotCompile: input.compileDurationMs,
|
|
529
|
+
searchIndex: indexDurationMs,
|
|
530
|
+
total: input.validationDurationMs + input.compileDurationMs + indexDurationMs,
|
|
531
|
+
},
|
|
532
|
+
completedAt,
|
|
533
|
+
},
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
catch (error) {
|
|
537
|
+
const current = onboardingJobs.get(id);
|
|
538
|
+
if (!current || current.status === 'cancelled')
|
|
539
|
+
return;
|
|
540
|
+
onboardingJobs.set(id, {
|
|
541
|
+
...current,
|
|
542
|
+
status: 'failed',
|
|
543
|
+
progress: 65,
|
|
544
|
+
message: 'The dbt project is connected, but its governed search indexes need attention.',
|
|
545
|
+
updatedAt: new Date().toISOString(),
|
|
546
|
+
phases: current.phases.map((phase) => phase.id === 'search_index'
|
|
547
|
+
? { ...phase, status: 'failed', durationMs: Date.now() - indexStartedAt }
|
|
548
|
+
: phase),
|
|
549
|
+
error: error instanceof Error ? error.message : String(error),
|
|
550
|
+
});
|
|
551
|
+
}
|
|
552
|
+
})();
|
|
553
|
+
}, 0);
|
|
554
|
+
return job;
|
|
555
|
+
};
|
|
445
556
|
const onboardingDbtPaths = (body = {}) => {
|
|
446
557
|
const repoUrl = typeof body.repoUrl === 'string' && body.repoUrl.trim() ? body.repoUrl.trim() : undefined;
|
|
447
558
|
const branch = typeof body.branch === 'string' && body.branch.trim() ? body.branch.trim() : undefined;
|
|
@@ -538,28 +649,6 @@ export async function startLocalServer(opts) {
|
|
|
538
649
|
}
|
|
539
650
|
return candidate;
|
|
540
651
|
};
|
|
541
|
-
// Zero-config semantic search: if the user is running Ollama with an embedding model
|
|
542
|
-
// but hasn't set an embed env var, auto-detect it so retrieval + metric/block matching
|
|
543
|
-
// get real semantic recall out of the box (match by meaning, not just keywords) — the
|
|
544
|
-
// biggest lever for "find the right metric/block instead of jumping to raw SQL".
|
|
545
|
-
// Explicit config always wins; when nothing is found we stay on the deterministic
|
|
546
|
-
// keyword matcher. Scoped to the app server (not eval/CI) and fully best-effort.
|
|
547
|
-
if (!process.env.DQL_OLLAMA_EMBED_URL && !process.env.DQL_OPENAI_API_KEY && !process.env.OPENAI_API_KEY) {
|
|
548
|
-
try {
|
|
549
|
-
const detected = await probeLocalOllamaEmbeddings();
|
|
550
|
-
if (detected) {
|
|
551
|
-
process.env.DQL_OLLAMA_EMBED_URL = detected.endpoint;
|
|
552
|
-
process.env.DQL_OLLAMA_EMBED_MODEL = detected.model;
|
|
553
|
-
console.log(`[dql] Semantic search on: local Ollama embeddings (${detected.model}) — matching by meaning, not just keywords.`);
|
|
554
|
-
}
|
|
555
|
-
else {
|
|
556
|
-
console.log('[dql] Semantic search: keyword-only. For higher matching accuracy, install Ollama and run `ollama pull nomic-embed-text` — DQL detects and uses it automatically, fully local & free.');
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
catch {
|
|
560
|
-
// Probe failure never blocks startup — fall back to the keyword matcher.
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
652
|
// Auto-ensure the active connection's driver so a configured connection is never
|
|
564
653
|
// left "broken" after a fresh clone, a CLI upgrade, or a Node version change (the
|
|
565
654
|
// driver lives in gitignored, per-project .dql/connectors). Best-effort + non-fatal.
|
|
@@ -695,6 +784,15 @@ export async function startLocalServer(opts) {
|
|
|
695
784
|
const nested = agentRunRecord(workspace.context);
|
|
696
785
|
return agentRunString(workspace[key]) ?? (nested ? agentRunString(nested[key]) : undefined);
|
|
697
786
|
};
|
|
787
|
+
const agentRunWorkspaceValues = (request, key) => {
|
|
788
|
+
const workspace = request.workspaceContext ?? {};
|
|
789
|
+
const nested = agentRunRecord(workspace.context);
|
|
790
|
+
const raw = workspace[key] ?? nested?.[key];
|
|
791
|
+
if (!Array.isArray(raw))
|
|
792
|
+
return undefined;
|
|
793
|
+
const values = raw.filter((item) => typeof item === 'string' && Boolean(item.trim())).map((item) => item.trim());
|
|
794
|
+
return values.length > 0 ? [...new Set(values)] : undefined;
|
|
795
|
+
};
|
|
698
796
|
const agentRunNotebookPath = (request, runId) => (agentRunWorkspaceValue(request, 'notebookPath')
|
|
699
797
|
?? (request.selectedObject?.kind === 'notebook' || request.selectedObject?.kind === 'cell' ? request.selectedObject.path : undefined)
|
|
700
798
|
?? `notebooks/agent-research/${runId}.dqlnb`);
|
|
@@ -876,7 +974,7 @@ export async function startLocalServer(opts) {
|
|
|
876
974
|
return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
|
|
877
975
|
},
|
|
878
976
|
});
|
|
879
|
-
async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer', onProgress) {
|
|
977
|
+
async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer', onProgress, routeDecision) {
|
|
880
978
|
const governed = resolveGovernedAnswerRunner(projectRoot);
|
|
881
979
|
const resolvedProvider = governed?.provider ?? null;
|
|
882
980
|
const runner = governed?.runner ?? null;
|
|
@@ -902,6 +1000,9 @@ export async function startLocalServer(opts) {
|
|
|
902
1000
|
workspaceContext: request.workspaceContext,
|
|
903
1001
|
instruction: [
|
|
904
1002
|
'Route through the governed DQL answer loop.',
|
|
1003
|
+
...(routeDecision?.meaningResolution?.selectedConceptIds.length
|
|
1004
|
+
? [`Meaning resolution selected these retrieved qualified IDs: ${routeDecision.meaningResolution.selectedConceptIds.join(', ')}. Treat them as strong evidence, but still validate grain, dimensions, filters, authorization, and runtime compatibility before execution.`]
|
|
1005
|
+
: []),
|
|
905
1006
|
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
906
1007
|
'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
|
|
907
1008
|
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
@@ -914,6 +1015,9 @@ export async function startLocalServer(opts) {
|
|
|
914
1015
|
const controller = request.runId
|
|
915
1016
|
? activeAgentRunControllers.get(request.runId) ?? new AbortController()
|
|
916
1017
|
: new AbortController();
|
|
1018
|
+
const runSignal = request.signal
|
|
1019
|
+
? AbortSignal.any([request.signal, controller.signal])
|
|
1020
|
+
: controller.signal;
|
|
917
1021
|
// Best-effort active warehouse dialect so Lane-2 semantic compiles emit
|
|
918
1022
|
// dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
|
|
919
1023
|
// connection is configured — the compiler then uses its default dialect.
|
|
@@ -934,6 +1038,7 @@ export async function startLocalServer(opts) {
|
|
|
934
1038
|
activeDomain: requestedDomain,
|
|
935
1039
|
purpose: requestedPurpose,
|
|
936
1040
|
modelAreaId: requestedModelAreaId,
|
|
1041
|
+
skillRefs: agentRunWorkspaceValues(request, 'skillRefs'),
|
|
937
1042
|
source: 'explicit_ui',
|
|
938
1043
|
snapshotId: runProjectSnapshot.snapshotId,
|
|
939
1044
|
})
|
|
@@ -955,6 +1060,7 @@ export async function startLocalServer(opts) {
|
|
|
955
1060
|
reasoningEffort,
|
|
956
1061
|
...(analysisDepth ? { analysisDepth } : {}),
|
|
957
1062
|
projectRoot,
|
|
1063
|
+
preparedContextPack: preparedAgentContextPacks.get(request),
|
|
958
1064
|
domainContext,
|
|
959
1065
|
projectSnapshot: { snapshotId: runProjectSnapshot.snapshotId, manifest: runProjectSnapshot.manifest },
|
|
960
1066
|
assertProjectSnapshot: (snapshotId) => {
|
|
@@ -963,6 +1069,12 @@ export async function startLocalServer(opts) {
|
|
|
963
1069
|
projectSnapshots.assertCurrent(snapshotId);
|
|
964
1070
|
},
|
|
965
1071
|
...(semanticDriver ? { semanticDriver } : {}),
|
|
1072
|
+
...(routeDecision?.meaningResolution?.selectedConceptIds.length
|
|
1073
|
+
? { preferredEvidenceIds: routeDecision.meaningResolution.selectedConceptIds }
|
|
1074
|
+
: {}),
|
|
1075
|
+
...(routeDecision?.meaningResolution?.recommendedExecutionId
|
|
1076
|
+
? { preferredExecutionId: routeDecision.meaningResolution.recommendedExecutionId }
|
|
1077
|
+
: {}),
|
|
966
1078
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
967
1079
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
968
1080
|
getSchemaContext: getSchemaContextForAgent,
|
|
@@ -975,7 +1087,7 @@ export async function startLocalServer(opts) {
|
|
|
975
1087
|
if (turn.kind === 'error') {
|
|
976
1088
|
providerError = turn.message;
|
|
977
1089
|
}
|
|
978
|
-
},
|
|
1090
|
+
}, runSignal);
|
|
979
1091
|
if (!governedAnswer) {
|
|
980
1092
|
throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
|
|
981
1093
|
}
|
|
@@ -1017,9 +1129,11 @@ export async function startLocalServer(opts) {
|
|
|
1017
1129
|
return;
|
|
1018
1130
|
const existing = agentRunRecord(result.chartConfig) ?? {};
|
|
1019
1131
|
// A declared chart on the execution result came from authored DQL and is a
|
|
1020
|
-
// governed display contract.
|
|
1021
|
-
|
|
1022
|
-
|
|
1132
|
+
// governed display contract. Preserve its type/bindings, but still enrich a
|
|
1133
|
+
// missing display format from the result semantics; otherwise an authored
|
|
1134
|
+
// KPI for `lifetime_spend` renders as a generic `671.4K` instead of `$671.4K`.
|
|
1135
|
+
// Agent `suggestedViz` remains a soft preference when no chart was authored.
|
|
1136
|
+
const hasAuthoredChart = typeof existing.chart === 'string';
|
|
1023
1137
|
const recommendation = recommendVisualization(projectRoot, {
|
|
1024
1138
|
blockRef: governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name,
|
|
1025
1139
|
prompt: question,
|
|
@@ -1030,17 +1144,26 @@ export async function startLocalServer(opts) {
|
|
|
1030
1144
|
if (!recommendation.ok)
|
|
1031
1145
|
return;
|
|
1032
1146
|
const fieldHints = recommendation.display.fieldHints ?? {};
|
|
1033
|
-
const chart =
|
|
1147
|
+
const chart = hasAuthoredChart
|
|
1148
|
+
? String(existing.chart).replace(/_/g, '-')
|
|
1149
|
+
: recommendation.display.defaultVisualization.replace(/_/g, '-');
|
|
1034
1150
|
const agentChoice = typeof governedAnswer.suggestedViz === 'string'
|
|
1035
1151
|
? governedAnswer.suggestedViz.toLowerCase().replace(/_/g, '-')
|
|
1036
1152
|
: undefined;
|
|
1037
1153
|
result.chartConfig = {
|
|
1038
1154
|
...existing,
|
|
1039
1155
|
chart,
|
|
1040
|
-
decisionSource: agentChoice === chart ? 'agent' : 'data',
|
|
1041
|
-
rationale:
|
|
1042
|
-
|
|
1043
|
-
|
|
1156
|
+
decisionSource: hasAuthoredChart ? 'authored' : agentChoice === chart ? 'agent' : 'data',
|
|
1157
|
+
rationale: hasAuthoredChart
|
|
1158
|
+
? agentRunString(existing.rationale) ?? 'Authored DQL visualization enriched with result-aware display semantics.'
|
|
1159
|
+
: recommendation.display.rationale,
|
|
1160
|
+
...(typeof existing.x !== 'string' && typeof fieldHints.x === 'string' ? { x: fieldHints.x } : {}),
|
|
1161
|
+
...(typeof existing.y !== 'string' && typeof fieldHints.y === 'string' ? { y: fieldHints.y } : {}),
|
|
1162
|
+
...(typeof existing.color !== 'string' && typeof fieldHints.color === 'string' ? { color: fieldHints.color } : {}),
|
|
1163
|
+
...(typeof existing.format !== 'string'
|
|
1164
|
+
&& (fieldHints.format === 'currency' || fieldHints.format === 'percent' || fieldHints.format === 'number')
|
|
1165
|
+
? { format: fieldHints.format }
|
|
1166
|
+
: {}),
|
|
1044
1167
|
};
|
|
1045
1168
|
governedAnswer.suggestedViz = chart;
|
|
1046
1169
|
const evidence = governedAnswer.evidence ?? {
|
|
@@ -1130,7 +1253,7 @@ export async function startLocalServer(opts) {
|
|
|
1130
1253
|
const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emit }) => {
|
|
1131
1254
|
let governedAnswer;
|
|
1132
1255
|
try {
|
|
1133
|
-
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }));
|
|
1256
|
+
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }), routeDecision);
|
|
1134
1257
|
// Surface the approved Hint-Graph corrections that shaped this answer so the
|
|
1135
1258
|
// UI can show an "applied learnings" chip (memoryContext is already on the answer).
|
|
1136
1259
|
if (!governedAnswer.appliedHints) {
|
|
@@ -1294,49 +1417,49 @@ export async function startLocalServer(opts) {
|
|
|
1294
1417
|
// more provider calls retrying the same incompatible candidate.
|
|
1295
1418
|
const isPolicyBlocked = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'policy_blocked';
|
|
1296
1419
|
// The model tried to compose a governed query and declined despite having usable
|
|
1297
|
-
// context (e.g. it wasn't confident about a multi-table join).
|
|
1298
|
-
//
|
|
1299
|
-
//
|
|
1300
|
-
// engine's bounded loop, exactly as a grounding gap is retried today.
|
|
1420
|
+
// context (e.g. it wasn't confident about a multi-table join). The answer loop
|
|
1421
|
+
// has already spent its one evidence-aware repair. Keep this terminal and
|
|
1422
|
+
// inspectable; an ordinary Ask must never silently become a second Research run.
|
|
1301
1423
|
const isModelDeclined = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'model_declined';
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
: undefined;
|
|
1306
|
-
// Only a genuinely AMBIGUOUS question is surfaced as "needs clarification". A
|
|
1307
|
-
// grounding gap or a model decline is retried/escalated by the engine, and a
|
|
1308
|
-
// provider outage is surfaced as blocked so the UI offers a retry.
|
|
1424
|
+
// Only a genuinely AMBIGUOUS question is surfaced as "needs clarification".
|
|
1425
|
+
// Grounding/compiler gaps are terminal review states with their evidence trace;
|
|
1426
|
+
// provider outages are blocked so the UI can offer an explicit retry.
|
|
1309
1427
|
const needsClarification = governedAnswer.kind === 'no_answer'
|
|
1310
1428
|
&& !isGroundingGap && !isProviderError && !isModelDeclined && !isPolicyBlocked;
|
|
1311
1429
|
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
1312
1430
|
const runnableSql = governedAnswer.kind === 'no_answer' || (isExploratory && !governedAnswer.result)
|
|
1313
1431
|
? undefined
|
|
1314
1432
|
: sql;
|
|
1315
|
-
//
|
|
1316
|
-
//
|
|
1433
|
+
// Render executed rows deterministically for ordinary lookups. A second LLM
|
|
1434
|
+
// call is reserved for an explicit research route; certified, semantic, and
|
|
1435
|
+
// generated lookup answers must not pay another provider round-trip merely
|
|
1436
|
+
// to restate values the host already has.
|
|
1317
1437
|
let synthesizedAnswer;
|
|
1318
1438
|
if (shouldSynthesizeAgentRunAnswer(governedAnswer)) {
|
|
1319
1439
|
try {
|
|
1320
|
-
const provider =
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1440
|
+
const provider = route === 'research'
|
|
1441
|
+
? await createBlockStudioAssistProvider(projectRoot)
|
|
1442
|
+
: null;
|
|
1443
|
+
const preview = agentResultToSynthesisPreview(governedAnswer.result);
|
|
1444
|
+
const draft = governedAnswer.answer ?? governedAnswer.text;
|
|
1445
|
+
const result = await synthesizeAnswer({
|
|
1446
|
+
question: request.question,
|
|
1447
|
+
category: routeDecision?.category,
|
|
1448
|
+
// The primary Ask reply is always business-facing. Analysts keep
|
|
1449
|
+
// the full DQL, SQL, lineage, gates, and grain in the inspector.
|
|
1450
|
+
audience: 'stakeholder',
|
|
1451
|
+
resultPreview: preview,
|
|
1452
|
+
sql: sql,
|
|
1453
|
+
draftText: draft,
|
|
1454
|
+
gaps: businessNarrativeGaps(governedAnswer.validationWarnings),
|
|
1455
|
+
rankingDirection: governedAnswer.contextPack?.questionPlan.requestedShape.rankingDirection,
|
|
1456
|
+
}, provider
|
|
1457
|
+
? {
|
|
1335
1458
|
complete: ({ system, user, signal, onDelta }) => streamOrGenerate(provider, [{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 350, temperature: 0.3, signal }, onDelta ?? (() => { })),
|
|
1336
|
-
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1459
|
+
}
|
|
1460
|
+
: {});
|
|
1461
|
+
if (result.text)
|
|
1462
|
+
synthesizedAnswer = result.text;
|
|
1340
1463
|
}
|
|
1341
1464
|
catch {
|
|
1342
1465
|
// Keep the governed draft on any synthesis failure.
|
|
@@ -1427,26 +1550,22 @@ export async function startLocalServer(opts) {
|
|
|
1427
1550
|
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
1428
1551
|
...(isGroundingGap ? [
|
|
1429
1552
|
{
|
|
1430
|
-
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The
|
|
1553
|
+
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The bounded lookup could not prove the required metadata grounding. No automatic retry or Research escalation was started.', {
|
|
1431
1554
|
refusalCode: governedAnswer.refusalCode,
|
|
1432
1555
|
refusalDetails: governedAnswer.refusalDetails,
|
|
1433
1556
|
validationWarnings: governedAnswer.validationWarnings,
|
|
1434
1557
|
route: governedAnswer.route,
|
|
1435
1558
|
}),
|
|
1436
|
-
suggestedRepair: groundingRepairHint,
|
|
1437
|
-
repairAction: { kind: 'retry', hint: groundingRepairHint },
|
|
1438
1559
|
},
|
|
1439
1560
|
] : []),
|
|
1440
1561
|
...(isModelDeclined ? [
|
|
1441
1562
|
{
|
|
1442
|
-
...agentRunEvaluation('declined-despite-context', 'Answer grounding', false, 'blocking', 'The
|
|
1563
|
+
...agentRunEvaluation('declined-despite-context', 'Answer grounding', false, 'blocking', 'The bounded lookup could not compose a governed query after its in-lane repair. Start Research explicitly to investigate beyond this lookup budget.', {
|
|
1443
1564
|
refusalCode: governedAnswer.refusalCode,
|
|
1444
1565
|
refusalDetails: governedAnswer.refusalDetails,
|
|
1445
1566
|
validationWarnings: governedAnswer.validationWarnings,
|
|
1446
1567
|
route: governedAnswer.route,
|
|
1447
1568
|
}),
|
|
1448
|
-
suggestedRepair: declinedRepairHint,
|
|
1449
|
-
repairAction: { kind: 'escalate', route: 'research', hint: declinedRepairHint },
|
|
1450
1569
|
},
|
|
1451
1570
|
] : []),
|
|
1452
1571
|
...(isPolicyBlocked ? [
|
|
@@ -1984,7 +2103,138 @@ export async function startLocalServer(opts) {
|
|
|
1984
2103
|
};
|
|
1985
2104
|
},
|
|
1986
2105
|
};
|
|
1987
|
-
//
|
|
2106
|
+
// One immutable, question-ranked pack is shared by routing, planning, schema
|
|
2107
|
+
// lookup, and governed execution for the lifetime of a request. This removes
|
|
2108
|
+
// both positional catalog truncation and the previous duplicate retrieval pass.
|
|
2109
|
+
const preparedAgentContextPacks = new WeakMap();
|
|
2110
|
+
const pendingAgentContextPacks = new WeakMap();
|
|
2111
|
+
const buildAgentRunContextPack = async (request) => {
|
|
2112
|
+
const prepared = preparedAgentContextPacks.get(request);
|
|
2113
|
+
if (prepared)
|
|
2114
|
+
return prepared;
|
|
2115
|
+
const pending = pendingAgentContextPacks.get(request);
|
|
2116
|
+
if (pending)
|
|
2117
|
+
return pending;
|
|
2118
|
+
const snapshot = projectSnapshot();
|
|
2119
|
+
const requestedDomain = agentRunWorkspaceValue(request, 'domain');
|
|
2120
|
+
// CTX-003: resolve result entities/values before evidence retrieval. The
|
|
2121
|
+
// router, planner, and answer loop must rank the same typed follow-up; doing
|
|
2122
|
+
// this only inside the provider adapter allowed stale catalog matches to win
|
|
2123
|
+
// before "they" / "this amount" became customer-scoped context.
|
|
2124
|
+
const followUp = resolveAgentFollowUpContext(request.conversationContext, request.question);
|
|
2125
|
+
const serverSnapshot = agentRunRecord(request.conversationContext?.serverSnapshot);
|
|
2126
|
+
const topicRelation = agentRunString(serverSnapshot?.topicRelation);
|
|
2127
|
+
// The readiness marker is source-versioned. When it matches, pass the
|
|
2128
|
+
// already-built metadata identity into retrieval so buildLocalContextPack
|
|
2129
|
+
// opens the immutable snapshot directly instead of rebuilding all metadata
|
|
2130
|
+
// merely to rediscover the same fingerprint on every follow-up.
|
|
2131
|
+
const preparedMetadataFingerprint = isAgentProjectIndexReady(projectRoot)
|
|
2132
|
+
? currentMetadataFingerprint(projectRoot)
|
|
2133
|
+
: undefined;
|
|
2134
|
+
const task = buildLocalContextPack(projectRoot, {
|
|
2135
|
+
question: request.question,
|
|
2136
|
+
followUp,
|
|
2137
|
+
priorContextPackId: agentRunString(request.conversationContext?.contextPackId),
|
|
2138
|
+
conversationTopicRelation: topicRelation === 'continuation'
|
|
2139
|
+
|| topicRelation === 'refinement'
|
|
2140
|
+
|| topicRelation === 'return'
|
|
2141
|
+
|| topicRelation === 'shift'
|
|
2142
|
+
? topicRelation
|
|
2143
|
+
: undefined,
|
|
2144
|
+
preparedMetadataFingerprint,
|
|
2145
|
+
surface: 'notebook',
|
|
2146
|
+
selectedContext: {
|
|
2147
|
+
selectedObject: request.selectedObject,
|
|
2148
|
+
workspaceContext: request.workspaceContext,
|
|
2149
|
+
},
|
|
2150
|
+
strictness: request.analysisDepth === 'deep' ? 'exploratory' : 'balanced',
|
|
2151
|
+
limit: request.analysisDepth === 'deep' ? 120 : 80,
|
|
2152
|
+
domainContext: requestedDomain
|
|
2153
|
+
? resolveDomainContextEnvelope({
|
|
2154
|
+
manifest: snapshot.manifest,
|
|
2155
|
+
activeDomain: requestedDomain,
|
|
2156
|
+
purpose: agentRunWorkspaceValue(request, 'purpose'),
|
|
2157
|
+
modelAreaId: agentRunWorkspaceValue(request, 'modelAreaId'),
|
|
2158
|
+
skillRefs: agentRunWorkspaceValues(request, 'skillRefs'),
|
|
2159
|
+
source: 'explicit_ui',
|
|
2160
|
+
snapshotId: snapshot.snapshotId,
|
|
2161
|
+
})
|
|
2162
|
+
: undefined,
|
|
2163
|
+
}).then((pack) => {
|
|
2164
|
+
preparedAgentContextPacks.set(request, pack);
|
|
2165
|
+
pendingAgentContextPacks.delete(request);
|
|
2166
|
+
return pack;
|
|
2167
|
+
}).catch((error) => {
|
|
2168
|
+
pendingAgentContextPacks.delete(request);
|
|
2169
|
+
throw error;
|
|
2170
|
+
});
|
|
2171
|
+
pendingAgentContextPacks.set(request, task);
|
|
2172
|
+
return task;
|
|
2173
|
+
};
|
|
2174
|
+
const buildAgentRunEvidence = async (request) => {
|
|
2175
|
+
const startedAt = Date.now();
|
|
2176
|
+
const pack = await buildAgentRunContextPack(request);
|
|
2177
|
+
const meaningEvidence = pack.retrievalDiagnostics.meaningEvidence;
|
|
2178
|
+
if (!meaningEvidence) {
|
|
2179
|
+
return {
|
|
2180
|
+
snapshotId: pack.id,
|
|
2181
|
+
sourceFingerprint: pack.freshness.fingerprint ?? undefined,
|
|
2182
|
+
candidates: [],
|
|
2183
|
+
diagnostics: { durationMs: Date.now() - startedAt },
|
|
2184
|
+
};
|
|
2185
|
+
}
|
|
2186
|
+
const evidence = toAgentRetrievalEvidence(meaningEvidence, pack.questionPlan, {
|
|
2187
|
+
snapshotId: pack.id,
|
|
2188
|
+
sourceFingerprint: pack.freshness.fingerprint ?? undefined,
|
|
2189
|
+
durationMs: Date.now() - startedAt,
|
|
2190
|
+
truncated: pack.retrievalDiagnostics.topRejected.length > 0,
|
|
2191
|
+
});
|
|
2192
|
+
const certifiedFits = new Map(pack.retrievalDiagnostics.certifiedCandidateFits.map((fit) => [fit.objectKey, fit]));
|
|
2193
|
+
const semanticEvidence = new Set(pack.routeDecision.selectedEvidence
|
|
2194
|
+
.filter((item) => item.role === 'semantic_metric')
|
|
2195
|
+
.map((item) => item.objectKey));
|
|
2196
|
+
return {
|
|
2197
|
+
...evidence,
|
|
2198
|
+
candidates: evidence.candidates.map((candidate) => {
|
|
2199
|
+
if (candidate.kind === 'certified_block') {
|
|
2200
|
+
const fit = certifiedFits.get(candidate.id);
|
|
2201
|
+
return {
|
|
2202
|
+
...candidate,
|
|
2203
|
+
compatibility: fit?.action === 'certified_answer'
|
|
2204
|
+
? 'compatible'
|
|
2205
|
+
: fit?.action === 'rejected_for_fit'
|
|
2206
|
+
? 'incompatible'
|
|
2207
|
+
: 'partial',
|
|
2208
|
+
};
|
|
2209
|
+
}
|
|
2210
|
+
if ((candidate.kind === 'semantic_metric' || candidate.kind === 'semantic_member')
|
|
2211
|
+
&& semanticEvidence.has(candidate.id)
|
|
2212
|
+
&& pack.routeDecision.route !== 'clarify'
|
|
2213
|
+
&& pack.routeDecision.route !== 'conflict') {
|
|
2214
|
+
const requestedDimensions = pack.questionPlan.requestedShape.dimensions.map((dimension) => dimension.toLowerCase());
|
|
2215
|
+
const availableDimensions = (candidate.dimensions ?? []).map((dimension) => dimension.toLowerCase());
|
|
2216
|
+
const dimensionsFit = requestedDimensions.length === 0 || requestedDimensions.every((requested) => availableDimensions.some((available) => available === requested || available.endsWith(`.${requested}`)));
|
|
2217
|
+
const requestedTimeGrain = pack.questionPlan.timeTerms[0]?.toLowerCase();
|
|
2218
|
+
const availableTimeGrains = (candidate.timeGrains ?? []).map((grain) => grain.toLowerCase());
|
|
2219
|
+
const timeGrainFits = !requestedTimeGrain
|
|
2220
|
+
|| availableTimeGrains.includes(requestedTimeGrain);
|
|
2221
|
+
return { ...candidate, compatibility: dimensionsFit && timeGrainFits ? 'compatible' : 'partial' };
|
|
2222
|
+
}
|
|
2223
|
+
if (candidate.trustTier === 'governed_sql')
|
|
2224
|
+
return { ...candidate, compatibility: 'partial' };
|
|
2225
|
+
return candidate;
|
|
2226
|
+
}),
|
|
2227
|
+
};
|
|
2228
|
+
};
|
|
2229
|
+
const buildRankedAgentRunCatalogContext = async (request) => {
|
|
2230
|
+
const evidence = await buildAgentRunEvidence(request);
|
|
2231
|
+
return evidence.candidates.map((candidate) => {
|
|
2232
|
+
const detail = candidate.definition ? `: ${candidate.definition}` : '';
|
|
2233
|
+
return `- ${candidate.id} [${candidate.trustTier}; ${candidate.compatibility}]${detail}`;
|
|
2234
|
+
}).join('\n');
|
|
2235
|
+
};
|
|
2236
|
+
// Compact fallback used only for plain conversational replies. Analytical
|
|
2237
|
+
// turns use the structured, question-ranked evidence path above.
|
|
1988
2238
|
const buildAgentRunCatalogContext = () => {
|
|
1989
2239
|
try {
|
|
1990
2240
|
const blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
@@ -2020,7 +2270,7 @@ export async function startLocalServer(opts) {
|
|
|
2020
2270
|
{ role: 'user', content: user },
|
|
2021
2271
|
], { maxTokens: 700, temperature: 0.1, signal });
|
|
2022
2272
|
},
|
|
2023
|
-
getCatalogContext:
|
|
2273
|
+
getCatalogContext: buildRankedAgentRunCatalogContext,
|
|
2024
2274
|
});
|
|
2025
2275
|
// Hybrid router: keep the deterministic decision when it is confident (certified
|
|
2026
2276
|
// fast paths + greetings stay 0-LLM); spend one cheap classification call only for
|
|
@@ -2030,13 +2280,22 @@ export async function startLocalServer(opts) {
|
|
|
2030
2280
|
complete: async ({ system, user, signal }) => {
|
|
2031
2281
|
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
2032
2282
|
if (!provider)
|
|
2033
|
-
throw new Error('No AI provider configured for
|
|
2283
|
+
throw new Error('No AI provider configured for meaning resolution.');
|
|
2034
2284
|
return provider.generate([
|
|
2035
2285
|
{ role: 'system', content: system },
|
|
2036
2286
|
{ role: 'user', content: user },
|
|
2037
|
-
], {
|
|
2287
|
+
], {
|
|
2288
|
+
maxTokens: 600,
|
|
2289
|
+
temperature: 0,
|
|
2290
|
+
// AGT-009/PERF-002: ambiguity gets one bounded resolver call. If the
|
|
2291
|
+
// provider stalls, the router falls back to its evidence-only decision
|
|
2292
|
+
// instead of leaving the UI in "Generating and validating SQL" for a
|
|
2293
|
+
// minute or more.
|
|
2294
|
+
signal: boundedAgentMeaningSignal(signal),
|
|
2295
|
+
});
|
|
2038
2296
|
},
|
|
2039
|
-
|
|
2297
|
+
getEvidence: buildAgentRunEvidence,
|
|
2298
|
+
getCatalogContext: buildRankedAgentRunCatalogContext,
|
|
2040
2299
|
});
|
|
2041
2300
|
const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
|
|
2042
2301
|
// A run may outlive its streaming browser connection, so cancellation is
|
|
@@ -2319,30 +2578,24 @@ export async function startLocalServer(opts) {
|
|
|
2319
2578
|
};
|
|
2320
2579
|
};
|
|
2321
2580
|
const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question, preparedContextPack).catch(() => []);
|
|
2581
|
+
const valueGrounding = resolveAgentRuntimeValueGrounding(projectConfig);
|
|
2322
2582
|
if (catalogContext.length > 0) {
|
|
2323
|
-
if (!connection)
|
|
2583
|
+
if (!connection || valueGrounding.mode !== 'safe_automatic')
|
|
2324
2584
|
return catalogContext;
|
|
2325
|
-
//
|
|
2326
|
-
//
|
|
2327
|
-
//
|
|
2328
|
-
const
|
|
2329
|
-
|
|
2330
|
-
: undefined;
|
|
2331
|
-
const runtimeContext = runtimeScan?.ranked ?? [];
|
|
2332
|
-
const merged = mergeAgentSchemaContexts(catalogContext, runtimeContext);
|
|
2333
|
-
const enriched = await enrichAgentSchemaContextWithValueMatches(question, merged, executor, connection);
|
|
2334
|
-
recordAgentRuntimeSchemaSnapshot(projectRoot, !runtimeScan?.snapshot.length
|
|
2335
|
-
? enriched
|
|
2336
|
-
: mergeAgentSchemaSampleValues(runtimeScan.snapshot, enriched), runtimeContext.length > 0
|
|
2337
|
-
? 'full information_schema runtime scan for composite Ask AI question'
|
|
2338
|
-
: 'catalog enriched runtime schema');
|
|
2585
|
+
// The immutable dbt/DQL snapshot already owns schema discovery. A named
|
|
2586
|
+
// row value must not turn a warm Ask into an information_schema scan over
|
|
2587
|
+
// thousands of enterprise tables; only field-scoped value probes are live.
|
|
2588
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, catalogContext, executor, connection, valueGrounding.searchSafeColumns);
|
|
2589
|
+
recordAgentRuntimeSchemaSnapshot(projectRoot, catalogContext, 'catalog runtime schema');
|
|
2339
2590
|
return enriched;
|
|
2340
2591
|
}
|
|
2341
2592
|
if (!connection)
|
|
2342
2593
|
return [];
|
|
2343
2594
|
try {
|
|
2344
2595
|
const schemaContext = await scanRuntimeSchema();
|
|
2345
|
-
const enriched =
|
|
2596
|
+
const enriched = valueGrounding.mode === 'safe_automatic'
|
|
2597
|
+
? await enrichAgentSchemaContextWithValueMatches(question, schemaContext.ranked, executor, connection, valueGrounding.searchSafeColumns)
|
|
2598
|
+
: schemaContext.ranked;
|
|
2346
2599
|
recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext.snapshot, 'full information_schema runtime scan');
|
|
2347
2600
|
return enriched;
|
|
2348
2601
|
}
|
|
@@ -3475,6 +3728,18 @@ export async function startLocalServer(opts) {
|
|
|
3475
3728
|
snapshotError = error instanceof Error ? error.message : String(error);
|
|
3476
3729
|
}
|
|
3477
3730
|
}
|
|
3731
|
+
const preparation = latestDbtPreparationJob();
|
|
3732
|
+
const preparationForSnapshot = preparation?.snapshotId === snapshotId ? preparation : undefined;
|
|
3733
|
+
const searchIndexesPresent = isAgentProjectIndexReady(projectRoot);
|
|
3734
|
+
const snapshotState = preparationForSnapshot?.status === 'running' || preparationForSnapshot?.status === 'queued'
|
|
3735
|
+
? 'building'
|
|
3736
|
+
: preparationForSnapshot?.status === 'failed'
|
|
3737
|
+
? 'failed'
|
|
3738
|
+
: snapshotId && searchIndexesPresent
|
|
3739
|
+
? 'ready'
|
|
3740
|
+
: snapshotId
|
|
3741
|
+
? 'stale'
|
|
3742
|
+
: 'missing';
|
|
3478
3743
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3479
3744
|
res.end(serializeJSON({
|
|
3480
3745
|
requestId,
|
|
@@ -3491,14 +3756,27 @@ export async function startLocalServer(opts) {
|
|
|
3491
3756
|
subPath: projectConfig.dbt?.subPath,
|
|
3492
3757
|
projectFound: existsSync(join(dbtProjectDir, 'dbt_project.yml')),
|
|
3493
3758
|
manifestFound,
|
|
3759
|
+
artifactState: manifestFound ? 'ready' : 'missing',
|
|
3494
3760
|
},
|
|
3495
3761
|
modeling: {
|
|
3496
3762
|
enabled: projectConfig.manifestVersion === 3 && projectConfig.modeling?.mode === 'dbt-first',
|
|
3497
3763
|
manifestVersion: projectConfig.manifestVersion ?? 2,
|
|
3498
3764
|
mode: projectConfig.modeling?.mode,
|
|
3765
|
+
snapshotState,
|
|
3499
3766
|
},
|
|
3500
3767
|
domains: { count: registry.values().length, diagnostics: registry.diagnostics },
|
|
3501
3768
|
snapshot: { id: snapshotId, error: snapshotError },
|
|
3769
|
+
preparation,
|
|
3770
|
+
readiness: {
|
|
3771
|
+
project: {
|
|
3772
|
+
state: snapshotState === 'ready' ? 'ready' : snapshotState === 'building' ? 'preparing' : snapshotState === 'failed' ? 'failed' : dbtConfigured ? 'configured' : 'missing',
|
|
3773
|
+
message: preparationForSnapshot?.message ?? (snapshotState === 'ready'
|
|
3774
|
+
? 'dbt metadata and governed search indexes are ready.'
|
|
3775
|
+
: snapshotState === 'stale'
|
|
3776
|
+
? 'dbt metadata is configured; search indexes will be refreshed before governed Ask.'
|
|
3777
|
+
: undefined),
|
|
3778
|
+
},
|
|
3779
|
+
},
|
|
3502
3780
|
capabilities: {
|
|
3503
3781
|
warehouse: Boolean(connection),
|
|
3504
3782
|
ai: Boolean(process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.GEMINI_API_KEY || process.env.OLLAMA_BASE_URL),
|
|
@@ -3523,6 +3801,7 @@ export async function startLocalServer(opts) {
|
|
|
3523
3801
|
if (req.method === 'POST' && path === '/api/onboarding/dbt/apply') {
|
|
3524
3802
|
const requestId = apiRequestId('onboarding-dbt-apply');
|
|
3525
3803
|
try {
|
|
3804
|
+
const applyStartedAt = Date.now();
|
|
3526
3805
|
const body = await readJSON(req);
|
|
3527
3806
|
let preview;
|
|
3528
3807
|
try {
|
|
@@ -3558,6 +3837,7 @@ export async function startLocalServer(opts) {
|
|
|
3558
3837
|
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed after preview. Review the refreshed preview before applying.', nextActions: ['Refresh the preview and review the new diff.'] })));
|
|
3559
3838
|
return;
|
|
3560
3839
|
}
|
|
3840
|
+
const validationDurationMs = Date.now() - applyStartedAt;
|
|
3561
3841
|
const nextConfig = {
|
|
3562
3842
|
...projectConfig,
|
|
3563
3843
|
manifestVersion: 3,
|
|
@@ -3583,8 +3863,12 @@ export async function startLocalServer(opts) {
|
|
|
3583
3863
|
projectSnapshots.invalidate();
|
|
3584
3864
|
invalidateAgentProjectState(projectRoot);
|
|
3585
3865
|
let snapshotId;
|
|
3866
|
+
let snapshotManifest;
|
|
3867
|
+
const compileStartedAt = Date.now();
|
|
3586
3868
|
try {
|
|
3587
|
-
|
|
3869
|
+
const snapshot = projectSnapshot();
|
|
3870
|
+
snapshotId = snapshot.snapshotId;
|
|
3871
|
+
snapshotManifest = snapshot.manifest;
|
|
3588
3872
|
}
|
|
3589
3873
|
catch (error) {
|
|
3590
3874
|
const rollbackPath = `${configPath}.rollback-${process.pid}`;
|
|
@@ -3596,8 +3880,22 @@ export async function startLocalServer(opts) {
|
|
|
3596
3880
|
invalidateAgentProjectState(projectRoot);
|
|
3597
3881
|
throw Object.assign(new Error(`dbt-first configuration did not compile and was rolled back: ${error instanceof Error ? error.message : String(error)}`), { code: 'SNAPSHOT_BUILD_FAILED' });
|
|
3598
3882
|
}
|
|
3883
|
+
const preparation = startDbtPreparationJob({
|
|
3884
|
+
kind: 'dbt_prepare',
|
|
3885
|
+
snapshotId,
|
|
3886
|
+
manifest: snapshotManifest,
|
|
3887
|
+
validationDurationMs,
|
|
3888
|
+
compileDurationMs: Date.now() - compileStartedAt,
|
|
3889
|
+
});
|
|
3599
3890
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3600
|
-
res.end(serializeJSON({
|
|
3891
|
+
res.end(serializeJSON({
|
|
3892
|
+
requestId,
|
|
3893
|
+
applied: true,
|
|
3894
|
+
config: { manifestVersion: 3, modeling: nextConfig.modeling, dbt: nextConfig.dbt },
|
|
3895
|
+
fingerprint: preview.fingerprint,
|
|
3896
|
+
jobId: preparation.id,
|
|
3897
|
+
...preparation,
|
|
3898
|
+
}));
|
|
3601
3899
|
}
|
|
3602
3900
|
catch (error) {
|
|
3603
3901
|
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
@@ -3608,7 +3906,7 @@ export async function startLocalServer(opts) {
|
|
|
3608
3906
|
}
|
|
3609
3907
|
if (req.method === 'POST' && path === '/api/onboarding/refresh') {
|
|
3610
3908
|
const requestId = apiRequestId('onboarding-refresh');
|
|
3611
|
-
const
|
|
3909
|
+
const refreshStartedAt = Date.now();
|
|
3612
3910
|
try {
|
|
3613
3911
|
const body = await readJSON(req);
|
|
3614
3912
|
const currentArtifact = previewDbtOnboarding({});
|
|
@@ -3617,20 +3915,24 @@ export async function startLocalServer(opts) {
|
|
|
3617
3915
|
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed before refresh. Review the current artifact preview.', nextActions: ['Return to the dbt preview step and review the refreshed fingerprint.'] })));
|
|
3618
3916
|
return;
|
|
3619
3917
|
}
|
|
3918
|
+
const validationDurationMs = Date.now() - refreshStartedAt;
|
|
3919
|
+
const compileStartedAt = Date.now();
|
|
3620
3920
|
projectSnapshots.invalidate();
|
|
3621
3921
|
const snapshot = projectSnapshot();
|
|
3622
3922
|
invalidateAgentProjectState(projectRoot);
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3923
|
+
const job = startDbtPreparationJob({
|
|
3924
|
+
kind: 'dbt_refresh',
|
|
3925
|
+
snapshotId: snapshot.snapshotId,
|
|
3926
|
+
manifest: snapshot.manifest,
|
|
3927
|
+
validationDurationMs,
|
|
3928
|
+
compileDurationMs: Date.now() - compileStartedAt,
|
|
3929
|
+
});
|
|
3626
3930
|
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3627
|
-
res.end(serializeJSON({ requestId,
|
|
3931
|
+
res.end(serializeJSON({ requestId, jobId: job.id, ...job }));
|
|
3628
3932
|
}
|
|
3629
3933
|
catch (error) {
|
|
3630
|
-
const job = { id, kind: 'dbt_refresh', status: 'failed', createdAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) };
|
|
3631
|
-
onboardingJobs.set(id, job);
|
|
3632
3934
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3633
|
-
res.end(serializeJSON(
|
|
3935
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SNAPSHOT_BUILD_FAILED', message: error instanceof Error ? error.message : String(error), nextActions: ['Keep using the previous snapshot while fixing compile diagnostics.'] })));
|
|
3634
3936
|
}
|
|
3635
3937
|
return;
|
|
3636
3938
|
}
|
|
@@ -3649,10 +3951,17 @@ export async function startLocalServer(opts) {
|
|
|
3649
3951
|
const requestId = apiRequestId('onboarding-job-cancel');
|
|
3650
3952
|
const id = decodeURIComponent(path.slice('/api/onboarding/jobs/'.length));
|
|
3651
3953
|
const job = onboardingJobs.get(id);
|
|
3652
|
-
|
|
3653
|
-
|
|
3954
|
+
const cancelled = job ? {
|
|
3955
|
+
...job,
|
|
3956
|
+
status: 'cancelled',
|
|
3957
|
+
message: 'Stopped waiting for dbt preparation. The last valid snapshot remains available.',
|
|
3958
|
+
updatedAt: new Date().toISOString(),
|
|
3959
|
+
phases: job.phases.map((phase) => phase.status === 'running' ? { ...phase, status: 'cancelled' } : phase),
|
|
3960
|
+
} : undefined;
|
|
3961
|
+
if (cancelled)
|
|
3962
|
+
onboardingJobs.set(id, cancelled);
|
|
3654
3963
|
res.writeHead(job ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3655
|
-
res.end(serializeJSON(
|
|
3964
|
+
res.end(serializeJSON(cancelled ? { requestId, ...cancelled } : apiErrorEnvelope({ requestId, code: 'ONBOARDING_JOB_NOT_FOUND', message: `onboarding job not found: ${id}`, recoverable: false })));
|
|
3656
3965
|
return;
|
|
3657
3966
|
}
|
|
3658
3967
|
if (req.method === 'POST' && path === '/api/onboarding/domains/discover') {
|
|
@@ -3786,7 +4095,12 @@ export async function startLocalServer(opts) {
|
|
|
3786
4095
|
const requestId = apiRequestId('domain-workspace');
|
|
3787
4096
|
const suffix = decodeURIComponent(path.slice('/api/domain-workspaces/'.length));
|
|
3788
4097
|
const relatedSuffix = '/related-products';
|
|
3789
|
-
const
|
|
4098
|
+
const knowledgeSuffix = '/knowledge';
|
|
4099
|
+
const domainId = suffix.endsWith(relatedSuffix)
|
|
4100
|
+
? suffix.slice(0, -relatedSuffix.length)
|
|
4101
|
+
: suffix.endsWith(knowledgeSuffix)
|
|
4102
|
+
? suffix.slice(0, -knowledgeSuffix.length)
|
|
4103
|
+
: suffix;
|
|
3790
4104
|
const snapshot = projectSnapshot();
|
|
3791
4105
|
const manifest = snapshot.manifest;
|
|
3792
4106
|
if (!manifest.modeling?.packages[domainId]) {
|
|
@@ -3794,6 +4108,19 @@ export async function startLocalServer(opts) {
|
|
|
3794
4108
|
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DOMAIN_NOT_FOUND', message: `domain workspace not found: ${domainId}`, recoverable: false })));
|
|
3795
4109
|
return;
|
|
3796
4110
|
}
|
|
4111
|
+
if (suffix.endsWith(knowledgeSuffix)) {
|
|
4112
|
+
await ensureMetadataCatalogFresh(projectRoot, { manifest, semanticLayer });
|
|
4113
|
+
const knowledge = readIndexedDomainKnowledge(projectRoot, domainId)
|
|
4114
|
+
?? canonicalDomainKnowledge(manifest, domainId, snapshot.snapshotId);
|
|
4115
|
+
if (!knowledge) {
|
|
4116
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4117
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DOMAIN_KNOWLEDGE_NOT_FOUND', message: `domain knowledge capsule not found: ${domainId}`, recoverable: false })));
|
|
4118
|
+
return;
|
|
4119
|
+
}
|
|
4120
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4121
|
+
res.end(serializeJSON({ requestId, ...knowledge }));
|
|
4122
|
+
return;
|
|
4123
|
+
}
|
|
3797
4124
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3798
4125
|
res.end(serializeJSON(suffix.endsWith(relatedSuffix)
|
|
3799
4126
|
? { requestId, domain: domainId, ...relatedProductsForDomain(manifest, domainId), snapshotId: snapshot.snapshotId }
|
|
@@ -3910,12 +4237,35 @@ export async function startLocalServer(opts) {
|
|
|
3910
4237
|
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 50));
|
|
3911
4238
|
const cursor = Math.max(0, Number(url.searchParams.get('cursor')) || 0);
|
|
3912
4239
|
const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
|
|
4240
|
+
const queryTokens = query.split(/[^a-z0-9]+/).filter(Boolean);
|
|
3913
4241
|
const domain = (url.searchParams.get('domain') ?? '').trim();
|
|
3914
4242
|
const boundByDbtId = new Map(Object.values(manifest.modeling?.entities ?? {}).map((entity) => [entity.dbtUniqueId, entity]));
|
|
3915
4243
|
const nodes = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
3916
|
-
.filter((node) =>
|
|
4244
|
+
.filter((node) => {
|
|
4245
|
+
if (!queryTokens.length)
|
|
4246
|
+
return true;
|
|
4247
|
+
const haystack = `${node.name} ${node.uniqueId} ${node.relation ?? ''} ${node.sourcePath ?? ''}`.toLowerCase();
|
|
4248
|
+
return queryTokens.every((token) => haystack.includes(token));
|
|
4249
|
+
})
|
|
3917
4250
|
.filter((node) => !domain || boundByDbtId.get(node.uniqueId)?.domain === domain)
|
|
3918
|
-
.sort((a, b) =>
|
|
4251
|
+
.sort((a, b) => {
|
|
4252
|
+
if (!query)
|
|
4253
|
+
return a.uniqueId.localeCompare(b.uniqueId);
|
|
4254
|
+
const score = (node) => {
|
|
4255
|
+
const name = node.name.toLowerCase();
|
|
4256
|
+
const uniqueId = node.uniqueId.toLowerCase();
|
|
4257
|
+
let value = name === query || uniqueId === query ? 1_000 : 0;
|
|
4258
|
+
if (name.startsWith(query))
|
|
4259
|
+
value += 500;
|
|
4260
|
+
if (uniqueId.startsWith(query))
|
|
4261
|
+
value += 300;
|
|
4262
|
+
if (name.includes(query))
|
|
4263
|
+
value += 200;
|
|
4264
|
+
value += queryTokens.reduce((total, token) => total + (name.startsWith(token) ? 80 : name.includes(token) ? 40 : 10), 0);
|
|
4265
|
+
return value;
|
|
4266
|
+
};
|
|
4267
|
+
return score(b) - score(a) || a.uniqueId.localeCompare(b.uniqueId);
|
|
4268
|
+
});
|
|
3919
4269
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3920
4270
|
res.end(serializeJSON({
|
|
3921
4271
|
requestId,
|
|
@@ -4134,8 +4484,13 @@ export async function startLocalServer(opts) {
|
|
|
4134
4484
|
}
|
|
4135
4485
|
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
4136
4486
|
const runId = parsed.request.runId;
|
|
4487
|
+
const runController = new AbortController();
|
|
4488
|
+
parsed.request.signal = AbortSignal.any([
|
|
4489
|
+
runController.signal,
|
|
4490
|
+
AbortSignal.timeout(agentRunDeadlineMs(parsed.request)),
|
|
4491
|
+
]);
|
|
4137
4492
|
if (runId)
|
|
4138
|
-
activeAgentRunControllers.set(runId,
|
|
4493
|
+
activeAgentRunControllers.set(runId, runController);
|
|
4139
4494
|
try {
|
|
4140
4495
|
if (wantsStream) {
|
|
4141
4496
|
res.writeHead(200, {
|
|
@@ -9646,13 +10001,17 @@ export async function startLocalServer(opts) {
|
|
|
9646
10001
|
try {
|
|
9647
10002
|
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
9648
10003
|
const result = queryBusiness360(graph, rawNodeId);
|
|
9649
|
-
|
|
10004
|
+
const snapshot = projectSnapshot();
|
|
10005
|
+
await ensureMetadataCatalogFresh(projectRoot, { manifest: snapshot.manifest, semanticLayer });
|
|
10006
|
+
const knowledge = readIndexedKnowledge360(projectRoot, rawNodeId)
|
|
10007
|
+
?? canonicalKnowledge360(snapshot.manifest, rawNodeId, snapshot.snapshotId);
|
|
10008
|
+
if (!result && !knowledge) {
|
|
9650
10009
|
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9651
10010
|
res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
|
|
9652
10011
|
return;
|
|
9653
10012
|
}
|
|
9654
10013
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9655
|
-
res.end(serializeJSON(result));
|
|
10014
|
+
res.end(serializeJSON(result ? { ...result, knowledge } : { version: 3, knowledge }));
|
|
9656
10015
|
}
|
|
9657
10016
|
catch (error) {
|
|
9658
10017
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -16494,17 +16853,127 @@ function buildNotebookTemplate(title, template) {
|
|
|
16494
16853
|
return JSON.stringify({ dqlnbVersion: 2, version: 1, title, cells }, null, 2);
|
|
16495
16854
|
}
|
|
16496
16855
|
/** Build a lineage graph from the project's blocks and semantic layer. */
|
|
16497
|
-
//
|
|
16498
|
-
|
|
16499
|
-
const
|
|
16856
|
+
// Cache per project + source fingerprint. A process can serve different roots,
|
|
16857
|
+
// and a time-only singleton previously returned the wrong graph after a change.
|
|
16858
|
+
const _lineageCache = new Map();
|
|
16500
16859
|
function buildProjectLineageGraph(projectRoot, semanticLayer) {
|
|
16501
|
-
|
|
16502
|
-
|
|
16503
|
-
|
|
16860
|
+
const signature = lineageSourceSignature(projectRoot);
|
|
16861
|
+
const cached = _lineageCache.get(projectRoot);
|
|
16862
|
+
if (cached?.signature === signature)
|
|
16863
|
+
return cached.graph;
|
|
16504
16864
|
const graph = buildProjectLineageGraphUncached(projectRoot, semanticLayer);
|
|
16505
|
-
_lineageCache
|
|
16865
|
+
_lineageCache.set(projectRoot, { signature, graph });
|
|
16506
16866
|
return graph;
|
|
16507
16867
|
}
|
|
16868
|
+
function lineageSourceSignature(projectRoot) {
|
|
16869
|
+
const hash = createHash('sha256');
|
|
16870
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, {}) ?? undefined;
|
|
16871
|
+
const inputs = new Set(collectInputFiles({ projectRoot, dbtManifestPath }));
|
|
16872
|
+
const emittedManifest = join(projectRoot, 'dql-manifest.json');
|
|
16873
|
+
if (existsSync(emittedManifest))
|
|
16874
|
+
inputs.add(emittedManifest);
|
|
16875
|
+
for (const input of [...inputs].sort()) {
|
|
16876
|
+
try {
|
|
16877
|
+
const stats = statSync(input);
|
|
16878
|
+
hash.update(`${input}\0${stats.size}\0${stats.mtimeMs}\n`);
|
|
16879
|
+
}
|
|
16880
|
+
catch {
|
|
16881
|
+
hash.update(`${input}\0missing\n`);
|
|
16882
|
+
}
|
|
16883
|
+
}
|
|
16884
|
+
return hash.digest('hex');
|
|
16885
|
+
}
|
|
16886
|
+
/** UI-008: bounded compiler-owned Domain Knowledge Capsule response. */
|
|
16887
|
+
function canonicalDomainKnowledge(manifest, domainId, snapshotId) {
|
|
16888
|
+
const graph = manifest.knowledgeGraph;
|
|
16889
|
+
if (!graph)
|
|
16890
|
+
return null;
|
|
16891
|
+
const capsule = graph.domainCapsules[domainId]
|
|
16892
|
+
?? Object.values(graph.domainCapsules).find((item) => item.domainId === domainId && !item.modelAreaId)
|
|
16893
|
+
?? Object.values(graph.domainCapsules).find((item) => item.id === domainId || item.name === domainId);
|
|
16894
|
+
const canonicalDomainId = capsule?.domainId
|
|
16895
|
+
?? Object.values(graph.objects ?? {}).find((item) => item.kind === 'domain' && (item.id === domainId || item.localId === domainId || item.aliases?.includes(domainId)))?.localId;
|
|
16896
|
+
if (!canonicalDomainId)
|
|
16897
|
+
return null;
|
|
16898
|
+
const objects = Object.values(graph.objects ?? {})
|
|
16899
|
+
.filter((item) => item.domainId === canonicalDomainId || item.id === `domain::${canonicalDomainId}`)
|
|
16900
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
16901
|
+
const objectIds = new Set(objects.map((item) => item.id));
|
|
16902
|
+
const edges = (graph.edges ?? [])
|
|
16903
|
+
.filter((edge) => objectIds.has(edge.from) || objectIds.has(edge.to))
|
|
16904
|
+
.slice(0, 1_500);
|
|
16905
|
+
const routes = graph.crossDomainRoutes.filter((route) => route.providerDomainId === canonicalDomainId || route.consumerDomainId === canonicalDomainId);
|
|
16906
|
+
const routeSummary = routes.reduce((counts, route) => {
|
|
16907
|
+
counts[route.state] = (counts[route.state] ?? 0) + 1;
|
|
16908
|
+
return counts;
|
|
16909
|
+
}, {});
|
|
16910
|
+
return {
|
|
16911
|
+
schemaVersion: graph.schemaVersion,
|
|
16912
|
+
snapshotId,
|
|
16913
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
16914
|
+
domainId: canonicalDomainId,
|
|
16915
|
+
capsule: capsule ?? graph.domainCapsules[canonicalDomainId],
|
|
16916
|
+
counts: {
|
|
16917
|
+
objects: objects.length,
|
|
16918
|
+
edges: edges.length,
|
|
16919
|
+
routes: routes.length,
|
|
16920
|
+
routeStates: routeSummary,
|
|
16921
|
+
},
|
|
16922
|
+
objects: objects.slice(0, 750),
|
|
16923
|
+
edges,
|
|
16924
|
+
routes,
|
|
16925
|
+
truncated: objects.length > 750 || edges.length >= 1_500,
|
|
16926
|
+
};
|
|
16927
|
+
}
|
|
16928
|
+
/** REL-003: qualified-object neighborhood with route policy and provenance. */
|
|
16929
|
+
function canonicalKnowledge360(manifest, rawId, snapshotId) {
|
|
16930
|
+
const graph = manifest.knowledgeGraph;
|
|
16931
|
+
if (!graph)
|
|
16932
|
+
return null;
|
|
16933
|
+
const graphObjects = graph.objects ?? {};
|
|
16934
|
+
const graphEdges = graph.edges ?? [];
|
|
16935
|
+
const exact = graphObjects[rawId];
|
|
16936
|
+
const matches = exact ? [exact] : Object.values(graphObjects).filter((item) => item.localId === rawId || item.aliases?.includes(rawId) || item.id.endsWith(`::${rawId}`));
|
|
16937
|
+
if (matches.length !== 1) {
|
|
16938
|
+
return matches.length > 1 ? {
|
|
16939
|
+
snapshotId,
|
|
16940
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
16941
|
+
ambiguous: true,
|
|
16942
|
+
candidates: matches.slice(0, 20).map((item) => ({ id: item.id, kind: item.kind, domainId: item.domainId })),
|
|
16943
|
+
} : null;
|
|
16944
|
+
}
|
|
16945
|
+
const focus = matches[0];
|
|
16946
|
+
const ids = new Set([focus.id]);
|
|
16947
|
+
let frontier = new Set([focus.id]);
|
|
16948
|
+
for (let depth = 0; depth < 2 && frontier.size > 0 && ids.size < 160; depth += 1) {
|
|
16949
|
+
const next = new Set();
|
|
16950
|
+
for (const edge of graphEdges) {
|
|
16951
|
+
if (frontier.has(edge.from) && !ids.has(edge.to))
|
|
16952
|
+
next.add(edge.to);
|
|
16953
|
+
if (frontier.has(edge.to) && !ids.has(edge.from))
|
|
16954
|
+
next.add(edge.from);
|
|
16955
|
+
}
|
|
16956
|
+
for (const id of next) {
|
|
16957
|
+
if (ids.size >= 160)
|
|
16958
|
+
break;
|
|
16959
|
+
ids.add(id);
|
|
16960
|
+
}
|
|
16961
|
+
frontier = next;
|
|
16962
|
+
}
|
|
16963
|
+
const objects = [...ids].flatMap((id) => graphObjects[id] ? [graphObjects[id]] : []);
|
|
16964
|
+
const edges = graphEdges.filter((edge) => ids.has(edge.from) && ids.has(edge.to)).slice(0, 500);
|
|
16965
|
+
const domains = new Set(objects.flatMap((item) => item.domainId ? [item.domainId] : []));
|
|
16966
|
+
const routes = graph.crossDomainRoutes.filter((route) => domains.has(route.providerDomainId) && domains.has(route.consumerDomainId));
|
|
16967
|
+
return {
|
|
16968
|
+
snapshotId,
|
|
16969
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
16970
|
+
focus,
|
|
16971
|
+
objects,
|
|
16972
|
+
edges,
|
|
16973
|
+
routes,
|
|
16974
|
+
truncated: ids.size >= 160 || edges.length >= 500,
|
|
16975
|
+
};
|
|
16976
|
+
}
|
|
16508
16977
|
function buildProjectLineageGraphUncached(projectRoot, semanticLayer) {
|
|
16509
16978
|
const manifestPath = join(projectRoot, 'dql-manifest.json');
|
|
16510
16979
|
if (existsSync(manifestPath)) {
|
|
@@ -18167,6 +18636,13 @@ async function buildAgentSchemaContextFromCatalog(projectRoot, question, prepare
|
|
|
18167
18636
|
}
|
|
18168
18637
|
/** How long a stored live-warehouse schema snapshot is trusted before a rescan (P6). */
|
|
18169
18638
|
const RUNTIME_SNAPSHOT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
18639
|
+
// A resolver compares at most 12 compact cards and never performs tool calls;
|
|
18640
|
+
// ten seconds is the full allowance, not the start of another planning loop.
|
|
18641
|
+
const AGENT_MEANING_TIMEOUT_MS = 10_000;
|
|
18642
|
+
export function boundedAgentMeaningSignal(signal, timeoutMs = AGENT_MEANING_TIMEOUT_MS) {
|
|
18643
|
+
const timeout = AbortSignal.timeout(Math.max(1, timeoutMs));
|
|
18644
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
18645
|
+
}
|
|
18170
18646
|
/**
|
|
18171
18647
|
* Whether the project's stored live-schema snapshot is missing or older than the
|
|
18172
18648
|
* freshness window (P6). Used to force a fresh information_schema scan even when the
|
|
@@ -18202,7 +18678,6 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
|
|
|
18202
18678
|
name: column.name,
|
|
18203
18679
|
type: column.type,
|
|
18204
18680
|
description: column.description,
|
|
18205
|
-
sampleValues: column.sampleValues?.slice(0, 8),
|
|
18206
18681
|
})),
|
|
18207
18682
|
})),
|
|
18208
18683
|
});
|
|
@@ -19135,25 +19610,29 @@ export function shouldAugmentAgentRuntimeSchema(question, questionPlan) {
|
|
|
19135
19610
|
const plannedCompositeMetric = (questionPlan?.metricTerms?.length ?? 0) > 0 && plannedConcepts.size >= 2;
|
|
19136
19611
|
return explicitJoin || referencesPriorRows || plannedCompositeMetric || (multiEntity && (wantsMetric || wantsDetail));
|
|
19137
19612
|
}
|
|
19138
|
-
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
|
|
19613
|
+
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection, searchSafeColumns) {
|
|
19139
19614
|
const searchTerms = extractAgentValueSearchTerms(question);
|
|
19140
19615
|
if (schemaContext.length === 0 || searchTerms.length === 0)
|
|
19141
19616
|
return schemaContext;
|
|
19142
19617
|
const matches = new Map();
|
|
19143
|
-
|
|
19618
|
+
const probes = rankAgentValueProbeColumns(schemaContext, searchSafeColumns).slice(0, 3).map(async (candidate) => {
|
|
19144
19619
|
try {
|
|
19145
|
-
const result = await executor.executeQuery(buildAgentValueProbeSql(candidate.table, candidate.column.name, searchTerms, connection), [], runtimeVariables({}), connection);
|
|
19146
|
-
const values = uniqueStrings(result.rows.flatMap(valueProbeRowValues)).slice(0,
|
|
19147
|
-
|
|
19148
|
-
continue;
|
|
19149
|
-
const tableMatches = matches.get(candidate.table.relation) ?? new Map();
|
|
19150
|
-
tableMatches.set(candidate.column.name, values);
|
|
19151
|
-
matches.set(candidate.table.relation, tableMatches);
|
|
19620
|
+
const result = await withAgentValueProbeTimeout(executor.executeQuery(buildAgentValueProbeSql(candidate.table, candidate.column.name, searchTerms, connection), [], runtimeVariables({}), connection), 2_000);
|
|
19621
|
+
const values = uniqueStrings(result.rows.flatMap(valueProbeRowValues)).slice(0, 25);
|
|
19622
|
+
return values.length > 0 ? { candidate, values } : undefined;
|
|
19152
19623
|
}
|
|
19153
19624
|
catch {
|
|
19154
19625
|
// Value probes are advisory. Unsupported casts, privileges, and large-table
|
|
19155
19626
|
// failures should not block the metadata-backed answer path.
|
|
19627
|
+
return undefined;
|
|
19156
19628
|
}
|
|
19629
|
+
});
|
|
19630
|
+
for (const match of await Promise.all(probes)) {
|
|
19631
|
+
if (!match)
|
|
19632
|
+
continue;
|
|
19633
|
+
const tableMatches = matches.get(match.candidate.table.relation) ?? new Map();
|
|
19634
|
+
tableMatches.set(match.candidate.column.name, match.values);
|
|
19635
|
+
matches.set(match.candidate.table.relation, tableMatches);
|
|
19157
19636
|
}
|
|
19158
19637
|
if (matches.size === 0)
|
|
19159
19638
|
return schemaContext;
|
|
@@ -19166,12 +19645,27 @@ async function enrichAgentSchemaContextWithValueMatches(question, schemaContext,
|
|
|
19166
19645
|
columns: table.columns.map((column) => {
|
|
19167
19646
|
const sampleValues = tableMatches.get(column.name);
|
|
19168
19647
|
return sampleValues?.length
|
|
19169
|
-
? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0,
|
|
19648
|
+
? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0, 25) }
|
|
19170
19649
|
: column;
|
|
19171
19650
|
}),
|
|
19172
19651
|
};
|
|
19173
19652
|
});
|
|
19174
19653
|
}
|
|
19654
|
+
async function withAgentValueProbeTimeout(promise, timeoutMs) {
|
|
19655
|
+
let timer;
|
|
19656
|
+
try {
|
|
19657
|
+
return await Promise.race([
|
|
19658
|
+
promise,
|
|
19659
|
+
new Promise((_resolve, reject) => {
|
|
19660
|
+
timer = setTimeout(() => reject(new Error('VALUE_LOOKUP_TIMEOUT')), timeoutMs);
|
|
19661
|
+
}),
|
|
19662
|
+
]);
|
|
19663
|
+
}
|
|
19664
|
+
finally {
|
|
19665
|
+
if (timer)
|
|
19666
|
+
clearTimeout(timer);
|
|
19667
|
+
}
|
|
19668
|
+
}
|
|
19175
19669
|
function scoreAgentSchemaTable(table, tokens) {
|
|
19176
19670
|
let score = 0;
|
|
19177
19671
|
const relationTokens = agentSchemaTokens(`${table.schema ?? ''} ${table.name} ${table.relation}`);
|
|
@@ -19252,11 +19746,37 @@ function scoreAgentValueProbeTable(table) {
|
|
|
19252
19746
|
}
|
|
19253
19747
|
return Math.min(score, 18);
|
|
19254
19748
|
}
|
|
19255
|
-
|
|
19749
|
+
/**
|
|
19750
|
+
* Resolve the project-admin boundary for live value lookup. An absent/malformed
|
|
19751
|
+
* policy is deliberately disabled; a broad table or wildcard cannot make an
|
|
19752
|
+
* unknown column search-safe.
|
|
19753
|
+
*/
|
|
19754
|
+
export function resolveAgentRuntimeValueGrounding(config) {
|
|
19755
|
+
const configured = config.agent?.runtimeValueGrounding;
|
|
19756
|
+
if (configured?.mode !== 'safe_automatic') {
|
|
19757
|
+
return { mode: 'disabled', searchSafeColumns: new Set() };
|
|
19758
|
+
}
|
|
19759
|
+
const searchSafeColumns = new Set((configured.searchSafeColumns ?? [])
|
|
19760
|
+
.map(normalizeAgentSafeColumnReference)
|
|
19761
|
+
.filter((value) => value.split('.').length >= 2 && !value.includes('*')));
|
|
19762
|
+
return searchSafeColumns.size > 0
|
|
19763
|
+
? { mode: 'safe_automatic', searchSafeColumns }
|
|
19764
|
+
: { mode: 'disabled', searchSafeColumns };
|
|
19765
|
+
}
|
|
19766
|
+
function normalizeAgentSafeColumnReference(value) {
|
|
19767
|
+
return value.trim().replace(/[`"\[\]]/g, '').toLowerCase();
|
|
19768
|
+
}
|
|
19769
|
+
function isExplicitlySearchSafeAgentColumn(table, column, searchSafeColumns) {
|
|
19770
|
+
const qualified = normalizeAgentSafeColumnReference(`${table.relation}.${column.name}`);
|
|
19771
|
+
const relationParts = table.relation.split('.').filter(Boolean);
|
|
19772
|
+
const shortQualified = normalizeAgentSafeColumnReference(`${relationParts.slice(-1)[0] ?? table.relation}.${column.name}`);
|
|
19773
|
+
return searchSafeColumns.has(qualified) || searchSafeColumns.has(shortQualified);
|
|
19774
|
+
}
|
|
19775
|
+
function rankAgentValueProbeColumns(schemaContext, searchSafeColumns) {
|
|
19256
19776
|
const ranked = [];
|
|
19257
19777
|
for (const table of schemaContext) {
|
|
19258
19778
|
for (const column of table.columns) {
|
|
19259
|
-
if (!isAgentValueProbeColumn(column))
|
|
19779
|
+
if (!isAgentValueProbeColumn(column) || !isExplicitlySearchSafeAgentColumn(table, column, searchSafeColumns))
|
|
19260
19780
|
continue;
|
|
19261
19781
|
ranked.push({
|
|
19262
19782
|
table,
|
|
@@ -19277,9 +19797,18 @@ function scoreAgentValueProbeColumn(table, column) {
|
|
|
19277
19797
|
score += 3;
|
|
19278
19798
|
return score;
|
|
19279
19799
|
}
|
|
19280
|
-
function isAgentValueProbeColumn(column) {
|
|
19800
|
+
export function isAgentValueProbeColumn(column) {
|
|
19281
19801
|
const name = column.name.toLowerCase();
|
|
19282
|
-
|
|
19802
|
+
// Tokenize underscore/camel names before applying the hard deny-list. This is
|
|
19803
|
+
// intentionally independent of an allowlist: secrets and free-text payloads
|
|
19804
|
+
// can never be probed through automatic grounding.
|
|
19805
|
+
const normalizedName = column.name
|
|
19806
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
19807
|
+
.replace(/[_-]+/g, ' ')
|
|
19808
|
+
.toLowerCase();
|
|
19809
|
+
if (/\b(password|secret|token|credential|hash|salt|notes?|comments?|description|message|body|payload|content)\b/.test(normalizedName))
|
|
19810
|
+
return false;
|
|
19811
|
+
if (/\bemail\b/.test(normalizedName))
|
|
19283
19812
|
return false;
|
|
19284
19813
|
if (!hasAgentSchemaToken(name, [
|
|
19285
19814
|
'account',
|
|
@@ -19317,15 +19846,22 @@ export function buildAgentValueProbeSql(table, column, searchTerms, connection)
|
|
|
19317
19846
|
const relation = quoteAgentRelation(table.relation, connection);
|
|
19318
19847
|
const identifier = quoteAgentIdentifier(column, connection);
|
|
19319
19848
|
const castValue = `LOWER(CAST(${identifier} AS ${agentTextCastType(connection.driver)}))`;
|
|
19320
|
-
const predicates = searchTerms
|
|
19321
|
-
.
|
|
19322
|
-
.
|
|
19849
|
+
const predicates = uniqueStrings(searchTerms.flatMap((term) => {
|
|
19850
|
+
const normalized = term.toLowerCase().replace(/\s+/g, ' ').trim();
|
|
19851
|
+
const tokens = normalized.split(' ').filter((token) => token.length >= 4);
|
|
19852
|
+
return [
|
|
19853
|
+
`${castValue} = ${sqlStringLiteral(normalized)}`,
|
|
19854
|
+
...tokens.slice(0, 2).map((token) => `${castValue} LIKE ${sqlStringLiteral(`${escapeSqlLike(token)}%`)} ESCAPE '\\'`),
|
|
19855
|
+
];
|
|
19856
|
+
}))
|
|
19857
|
+
.slice(0, 8)
|
|
19858
|
+
.map((predicate) => predicate)
|
|
19323
19859
|
.join(' OR ');
|
|
19324
19860
|
return [
|
|
19325
19861
|
`SELECT DISTINCT CAST(${identifier} AS ${agentTextCastType(connection.driver)}) AS value`,
|
|
19326
19862
|
`FROM ${relation}`,
|
|
19327
19863
|
`WHERE ${identifier} IS NOT NULL AND (${predicates})`,
|
|
19328
|
-
'LIMIT
|
|
19864
|
+
'LIMIT 25',
|
|
19329
19865
|
].join('\n');
|
|
19330
19866
|
}
|
|
19331
19867
|
function agentTextCastType(driver) {
|
|
@@ -19409,6 +19945,9 @@ export function extractAgentValueSearchTerms(question) {
|
|
|
19409
19945
|
for (const match of question.matchAll(/\b(?:for|named|called|only|where|customer|user|account|product)\s+([A-Za-z0-9@._-]+(?:\s+[A-Za-z0-9@._-]+){0,3})/gi)) {
|
|
19410
19946
|
terms.push(match[1]);
|
|
19411
19947
|
}
|
|
19948
|
+
for (const match of question.matchAll(/\b(?:than|versus|vs\.?)\s+([A-Za-z0-9@._-]+(?:\s+[A-Za-z0-9@._-]+){0,3})/gi)) {
|
|
19949
|
+
terms.push(match[1]);
|
|
19950
|
+
}
|
|
19412
19951
|
return uniqueStrings(terms
|
|
19413
19952
|
.map(cleanAgentValueSearchTerm)
|
|
19414
19953
|
.filter((term) => term.length >= 3 && !AGENT_VALUE_SEARCH_STOP_PHRASES.has(term.toLowerCase()))).slice(0, 6);
|
|
@@ -19419,6 +19958,7 @@ function cleanAgentValueSearchTerm(term) {
|
|
|
19419
19958
|
.replace(/\s+/g, ' ')
|
|
19420
19959
|
.trim()
|
|
19421
19960
|
.replace(/^(?:account|customer|member|named|called|product|sku|subscriber|user)\s+/i, '')
|
|
19961
|
+
.replace(/\s+\b(?:got|get|gets|bought|buy|buys|purchased|purchase|purchases|spent|spend|spends|has|have|with)\b.*$/i, '')
|
|
19422
19962
|
.replace(/\s+\b(?:last|next|this)\b.*$/i, '')
|
|
19423
19963
|
.replace(/\s+\b(?:last|this)\s+(?:day|week|month|quarter|year)\b.*$/i, '')
|
|
19424
19964
|
.replace(/\s+\b(?:daily|weekly|monthly|quarterly|yearly)\b.*$/i, '')
|