@duckcodeailabs/dql-cli 1.7.2 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +69 -4
- package/dist/apps-api.js.map +1 -1
- package/dist/assets/dql-notebook/assets/{index-BgD9b9Om.js → index-siDRLswd.js} +380 -369
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/llm/answer-loop-tools.js +17 -4
- package/dist/llm/answer-loop-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts +1 -0
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +143 -26
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/types.d.ts +13 -0
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +26 -0
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +748 -175
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/providers/subscription-cli.d.ts +1 -0
- package/dist/providers/subscription-cli.d.ts.map +1 -1
- package/dist/providers/subscription-cli.js +40 -8
- package/dist/providers/subscription-cli.js.map +1 -1
- package/package.json +11 -11
package/dist/local-runtime.js
CHANGED
|
@@ -14,9 +14,9 @@ import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetri
|
|
|
14
14
|
import { load as loadYaml } from 'js-yaml';
|
|
15
15
|
import { listBlockTemplates } from './block-templates.js';
|
|
16
16
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
17
|
-
import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
|
|
17
|
+
import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
|
|
18
18
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
19
|
-
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode,
|
|
19
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, buildLocalContextPack, toAgentRetrievalEvidence, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
20
20
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
21
21
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
22
22
|
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
@@ -36,18 +36,26 @@ const NOTEBOOK_FAVICON_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0
|
|
|
36
36
|
export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
|
|
37
37
|
const configured = projectConfig.semanticLayer;
|
|
38
38
|
const dbtProjectDir = projectConfig.dbt?.projectDir;
|
|
39
|
+
const dbtManifestPath = projectConfig.dbt?.manifestPath;
|
|
40
|
+
if (configured?.provider === 'dbt') {
|
|
41
|
+
return {
|
|
42
|
+
...configured,
|
|
43
|
+
projectPath: configured.projectPath ?? dbtProjectDir,
|
|
44
|
+
manifestPath: configured.manifestPath ?? dbtManifestPath,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
39
47
|
if (dbtProjectDir
|
|
40
48
|
&& (!configured || configured.provider === 'dql')
|
|
41
|
-
&& hasDbtSemanticArtifacts(projectRoot, dbtProjectDir)) {
|
|
42
|
-
return { provider: 'dbt', projectPath: dbtProjectDir };
|
|
49
|
+
&& hasDbtSemanticArtifacts(projectRoot, dbtProjectDir, dbtManifestPath)) {
|
|
50
|
+
return { provider: 'dbt', projectPath: dbtProjectDir, manifestPath: dbtManifestPath };
|
|
43
51
|
}
|
|
44
52
|
return configured;
|
|
45
53
|
}
|
|
46
|
-
function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
|
|
54
|
+
function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir, configuredManifestPath) {
|
|
47
55
|
const dbtRoot = resolve(projectRoot, dbtProjectDir);
|
|
48
|
-
|
|
56
|
+
const manifestPath = resolve(dbtRoot, configuredManifestPath ?? 'target/manifest.json');
|
|
57
|
+
if (existsSync(join(dirname(manifestPath), 'semantic_manifest.json')))
|
|
49
58
|
return true;
|
|
50
|
-
const manifestPath = join(dbtRoot, 'target', 'manifest.json');
|
|
51
59
|
if (!existsSync(manifestPath))
|
|
52
60
|
return false;
|
|
53
61
|
try {
|
|
@@ -153,6 +161,20 @@ export function parseAgentRunRequestBody(body) {
|
|
|
153
161
|
},
|
|
154
162
|
};
|
|
155
163
|
}
|
|
164
|
+
const AGENT_LOOKUP_DEADLINE_MS = 45_000;
|
|
165
|
+
const AGENT_RESEARCH_DEADLINE_MS = 120_000;
|
|
166
|
+
/**
|
|
167
|
+
* PERF-002: one wall-clock budget follows the request through routing, provider
|
|
168
|
+
* calls, repair, and execution. Ordinary Ask never inherits Research's budget
|
|
169
|
+
* merely because it spans two tables; explicit/deep investigation does.
|
|
170
|
+
*/
|
|
171
|
+
export function agentRunDeadlineMs(request) {
|
|
172
|
+
if (request.requestedMode === 'research' || request.analysisDepth === 'deep') {
|
|
173
|
+
return AGENT_RESEARCH_DEADLINE_MS;
|
|
174
|
+
}
|
|
175
|
+
const plan = buildAnalysisQuestionPlan(request.question);
|
|
176
|
+
return plan.needsResearchWorkspace ? AGENT_RESEARCH_DEADLINE_MS : AGENT_LOOKUP_DEADLINE_MS;
|
|
177
|
+
}
|
|
156
178
|
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
157
179
|
if (governedAnswer.kind === 'no_answer')
|
|
158
180
|
return false;
|
|
@@ -283,6 +305,7 @@ function conversationTurnInputFromRun(run) {
|
|
|
283
305
|
sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
|
|
284
306
|
?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
|
|
285
307
|
contextPackId: agentRunString(payload?.contextPackId) ?? agentRunString(contextPack?.id),
|
|
308
|
+
knowledgeLens: agentRunRecord(contextPack?.knowledgeLens),
|
|
286
309
|
sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
|
|
287
310
|
dqlArtifact: agentRunRecord(payload?.dqlArtifact),
|
|
288
311
|
cascade: agentRunRecord(payload?.cascade),
|
|
@@ -406,6 +429,7 @@ export async function startLocalServer(opts) {
|
|
|
406
429
|
if (gitRoot)
|
|
407
430
|
ensureLocalRuntimeGitignore(projectRoot);
|
|
408
431
|
let projectConfig = loadProjectConfig(projectRoot);
|
|
432
|
+
recordAgentRuntimeVersion(projectRoot, runtimeVersion);
|
|
409
433
|
const configuredConnection = rawConnection
|
|
410
434
|
? normalizeProjectConnection(rawConnection, projectRoot)
|
|
411
435
|
: projectConfig.defaultConnection
|
|
@@ -442,6 +466,104 @@ export async function startLocalServer(opts) {
|
|
|
442
466
|
error: snapshot.error,
|
|
443
467
|
};
|
|
444
468
|
};
|
|
469
|
+
const latestDbtPreparationJob = () => Array.from(onboardingJobs.values())
|
|
470
|
+
.reverse()
|
|
471
|
+
.find((job) => job.kind === 'dbt_prepare' || job.kind === 'dbt_refresh');
|
|
472
|
+
const startDbtPreparationJob = (input) => {
|
|
473
|
+
const now = new Date().toISOString();
|
|
474
|
+
const id = apiRequestId(input.kind === 'dbt_refresh' ? 'dbt-refresh' : 'dbt-prepare');
|
|
475
|
+
const job = {
|
|
476
|
+
id,
|
|
477
|
+
kind: input.kind,
|
|
478
|
+
status: 'running',
|
|
479
|
+
stage: 'indexing',
|
|
480
|
+
progress: 65,
|
|
481
|
+
message: 'Indexing dbt models, columns, semantic metrics, certified blocks, and governed relationships.',
|
|
482
|
+
createdAt: now,
|
|
483
|
+
updatedAt: now,
|
|
484
|
+
snapshotId: input.snapshotId,
|
|
485
|
+
phases: [
|
|
486
|
+
{ id: 'artifact_validation', label: 'Validated dbt project and artifacts', status: 'completed', durationMs: input.validationDurationMs },
|
|
487
|
+
{ id: 'snapshot_compile', label: 'Compiled immutable project snapshot', status: 'completed', durationMs: input.compileDurationMs },
|
|
488
|
+
{ id: 'search_index', label: 'Build governed search indexes', status: 'running' },
|
|
489
|
+
],
|
|
490
|
+
};
|
|
491
|
+
if (onboardingJobs.size >= 24) {
|
|
492
|
+
for (const [existingId, existing] of onboardingJobs) {
|
|
493
|
+
if (existing.status === 'running' || existing.status === 'queued')
|
|
494
|
+
continue;
|
|
495
|
+
onboardingJobs.delete(existingId);
|
|
496
|
+
if (onboardingJobs.size < 16)
|
|
497
|
+
break;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
onboardingJobs.set(id, job);
|
|
501
|
+
// Start after the Apply response can be returned. Governed Ask calls the
|
|
502
|
+
// same versioned preparation service, so an early first question awaits
|
|
503
|
+
// this in-flight promise instead of starting a duplicate cold rebuild.
|
|
504
|
+
setTimeout(() => {
|
|
505
|
+
void (async () => {
|
|
506
|
+
const indexStartedAt = Date.now();
|
|
507
|
+
try {
|
|
508
|
+
const prepared = await ensureAgentProjectReady(projectRoot, {
|
|
509
|
+
kgPath: defaultKgPath(projectRoot),
|
|
510
|
+
manifest: input.manifest,
|
|
511
|
+
forceKgIndex: input.forceRebuild,
|
|
512
|
+
forceMetadataCatalog: input.forceRebuild,
|
|
513
|
+
});
|
|
514
|
+
const current = onboardingJobs.get(id);
|
|
515
|
+
if (!current || current.status === 'cancelled')
|
|
516
|
+
return;
|
|
517
|
+
const indexDurationMs = Date.now() - indexStartedAt;
|
|
518
|
+
const completedAt = new Date().toISOString();
|
|
519
|
+
onboardingJobs.set(id, {
|
|
520
|
+
...current,
|
|
521
|
+
status: 'completed',
|
|
522
|
+
stage: 'ready',
|
|
523
|
+
progress: 100,
|
|
524
|
+
message: `Ready. Indexed ${prepared.nodes.toLocaleString()} governed objects for fast search.`,
|
|
525
|
+
updatedAt: completedAt,
|
|
526
|
+
phases: current.phases.map((phase) => phase.id === 'search_index'
|
|
527
|
+
? { ...phase, status: 'completed', durationMs: indexDurationMs }
|
|
528
|
+
: phase),
|
|
529
|
+
result: {
|
|
530
|
+
snapshotId: input.snapshotId,
|
|
531
|
+
cacheHit: prepared.cacheHit,
|
|
532
|
+
sourceVersion: prepared.sourceVersion,
|
|
533
|
+
objectCount: prepared.nodes,
|
|
534
|
+
edgeCount: prepared.edges,
|
|
535
|
+
metadataFingerprint: prepared.metadataFingerprint,
|
|
536
|
+
kgFingerprint: prepared.kgFingerprint,
|
|
537
|
+
phaseDurationsMs: {
|
|
538
|
+
artifactValidation: input.validationDurationMs,
|
|
539
|
+
snapshotCompile: input.compileDurationMs,
|
|
540
|
+
searchIndex: indexDurationMs,
|
|
541
|
+
total: input.validationDurationMs + input.compileDurationMs + indexDurationMs,
|
|
542
|
+
},
|
|
543
|
+
completedAt,
|
|
544
|
+
},
|
|
545
|
+
});
|
|
546
|
+
}
|
|
547
|
+
catch (error) {
|
|
548
|
+
const current = onboardingJobs.get(id);
|
|
549
|
+
if (!current || current.status === 'cancelled')
|
|
550
|
+
return;
|
|
551
|
+
onboardingJobs.set(id, {
|
|
552
|
+
...current,
|
|
553
|
+
status: 'failed',
|
|
554
|
+
progress: 65,
|
|
555
|
+
message: 'The dbt project is connected, but its governed search indexes need attention.',
|
|
556
|
+
updatedAt: new Date().toISOString(),
|
|
557
|
+
phases: current.phases.map((phase) => phase.id === 'search_index'
|
|
558
|
+
? { ...phase, status: 'failed', durationMs: Date.now() - indexStartedAt }
|
|
559
|
+
: phase),
|
|
560
|
+
error: error instanceof Error ? error.message : String(error),
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
})();
|
|
564
|
+
}, 0);
|
|
565
|
+
return job;
|
|
566
|
+
};
|
|
445
567
|
const onboardingDbtPaths = (body = {}) => {
|
|
446
568
|
const repoUrl = typeof body.repoUrl === 'string' && body.repoUrl.trim() ? body.repoUrl.trim() : undefined;
|
|
447
569
|
const branch = typeof body.branch === 'string' && body.branch.trim() ? body.branch.trim() : undefined;
|
|
@@ -538,28 +660,6 @@ export async function startLocalServer(opts) {
|
|
|
538
660
|
}
|
|
539
661
|
return candidate;
|
|
540
662
|
};
|
|
541
|
-
// Zero-config semantic search: if the user is running Ollama with an embedding model
|
|
542
|
-
// but hasn't set an embed env var, auto-detect it so retrieval + metric/block matching
|
|
543
|
-
// get real semantic recall out of the box (match by meaning, not just keywords) — the
|
|
544
|
-
// biggest lever for "find the right metric/block instead of jumping to raw SQL".
|
|
545
|
-
// Explicit config always wins; when nothing is found we stay on the deterministic
|
|
546
|
-
// keyword matcher. Scoped to the app server (not eval/CI) and fully best-effort.
|
|
547
|
-
if (!process.env.DQL_OLLAMA_EMBED_URL && !process.env.DQL_OPENAI_API_KEY && !process.env.OPENAI_API_KEY) {
|
|
548
|
-
try {
|
|
549
|
-
const detected = await probeLocalOllamaEmbeddings();
|
|
550
|
-
if (detected) {
|
|
551
|
-
process.env.DQL_OLLAMA_EMBED_URL = detected.endpoint;
|
|
552
|
-
process.env.DQL_OLLAMA_EMBED_MODEL = detected.model;
|
|
553
|
-
console.log(`[dql] Semantic search on: local Ollama embeddings (${detected.model}) — matching by meaning, not just keywords.`);
|
|
554
|
-
}
|
|
555
|
-
else {
|
|
556
|
-
console.log('[dql] Semantic search: keyword-only. For higher matching accuracy, install Ollama and run `ollama pull nomic-embed-text` — DQL detects and uses it automatically, fully local & free.');
|
|
557
|
-
}
|
|
558
|
-
}
|
|
559
|
-
catch {
|
|
560
|
-
// Probe failure never blocks startup — fall back to the keyword matcher.
|
|
561
|
-
}
|
|
562
|
-
}
|
|
563
663
|
// Auto-ensure the active connection's driver so a configured connection is never
|
|
564
664
|
// left "broken" after a fresh clone, a CLI upgrade, or a Node version change (the
|
|
565
665
|
// driver lives in gitignored, per-project .dql/connectors). Best-effort + non-fatal.
|
|
@@ -571,18 +671,25 @@ export async function startLocalServer(opts) {
|
|
|
571
671
|
const semanticLayerDir = join(projectRoot, 'semantic-layer');
|
|
572
672
|
let semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
573
673
|
const userPrefsPath = join(projectRoot, '.dql-user-prefs.json');
|
|
574
|
-
|
|
674
|
+
let semanticConfig = resolveProjectSemanticConfig(projectConfig, projectRoot);
|
|
575
675
|
let semanticLastSyncTime = null;
|
|
576
|
-
{
|
|
676
|
+
const reloadSemanticLayer = async () => {
|
|
677
|
+
semanticConfig = resolveProjectSemanticConfig(projectConfig, projectRoot);
|
|
577
678
|
const semanticConnection = connection;
|
|
578
679
|
const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
|
|
579
680
|
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
|
|
580
681
|
: undefined;
|
|
581
682
|
const result = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
|
|
582
|
-
semanticLayer = result.layer;
|
|
583
683
|
semanticLayerErrors = result.errors;
|
|
584
684
|
semanticDetectedProvider = result.detectedProvider;
|
|
585
|
-
|
|
685
|
+
if (result.layer) {
|
|
686
|
+
semanticLayer = result.layer;
|
|
687
|
+
semanticLastSyncTime = new Date().toISOString();
|
|
688
|
+
}
|
|
689
|
+
else if (result.errors.length === 0) {
|
|
690
|
+
semanticLayer = undefined;
|
|
691
|
+
semanticLastSyncTime = null;
|
|
692
|
+
}
|
|
586
693
|
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
587
694
|
// Legacy fallback if provider system returned nothing and no errors
|
|
588
695
|
if (!semanticLayer && semanticLayerErrors.length === 0 && existsSync(semanticLayerDir)) {
|
|
@@ -592,8 +699,28 @@ export async function startLocalServer(opts) {
|
|
|
592
699
|
}
|
|
593
700
|
catch { /* continue without */ }
|
|
594
701
|
}
|
|
595
|
-
|
|
702
|
+
return result;
|
|
703
|
+
};
|
|
704
|
+
await reloadSemanticLayer();
|
|
596
705
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
706
|
+
const startupDbtManifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
707
|
+
if (startupDbtManifestPath && !isAgentProjectIndexReady(projectRoot)) {
|
|
708
|
+
try {
|
|
709
|
+
const startupSnapshot = projectSnapshot();
|
|
710
|
+
startDbtPreparationJob({
|
|
711
|
+
kind: 'dbt_prepare',
|
|
712
|
+
snapshotId: startupSnapshot.snapshotId,
|
|
713
|
+
manifest: startupSnapshot.manifest,
|
|
714
|
+
validationDurationMs: 0,
|
|
715
|
+
compileDurationMs: 0,
|
|
716
|
+
forceRebuild: true,
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
catch {
|
|
720
|
+
// Startup remains available with the last valid cache; Setup exposes the
|
|
721
|
+
// compile error and the next explicit refresh retries preparation.
|
|
722
|
+
}
|
|
723
|
+
}
|
|
597
724
|
const recordDatasetMetadataSnapshot = (datasets = datasetWorkspace.list()) => {
|
|
598
725
|
if (datasets.length === 0)
|
|
599
726
|
return;
|
|
@@ -695,6 +822,15 @@ export async function startLocalServer(opts) {
|
|
|
695
822
|
const nested = agentRunRecord(workspace.context);
|
|
696
823
|
return agentRunString(workspace[key]) ?? (nested ? agentRunString(nested[key]) : undefined);
|
|
697
824
|
};
|
|
825
|
+
const agentRunWorkspaceValues = (request, key) => {
|
|
826
|
+
const workspace = request.workspaceContext ?? {};
|
|
827
|
+
const nested = agentRunRecord(workspace.context);
|
|
828
|
+
const raw = workspace[key] ?? nested?.[key];
|
|
829
|
+
if (!Array.isArray(raw))
|
|
830
|
+
return undefined;
|
|
831
|
+
const values = raw.filter((item) => typeof item === 'string' && Boolean(item.trim())).map((item) => item.trim());
|
|
832
|
+
return values.length > 0 ? [...new Set(values)] : undefined;
|
|
833
|
+
};
|
|
698
834
|
const agentRunNotebookPath = (request, runId) => (agentRunWorkspaceValue(request, 'notebookPath')
|
|
699
835
|
?? (request.selectedObject?.kind === 'notebook' || request.selectedObject?.kind === 'cell' ? request.selectedObject.path : undefined)
|
|
700
836
|
?? `notebooks/agent-research/${runId}.dqlnb`);
|
|
@@ -876,7 +1012,7 @@ export async function startLocalServer(opts) {
|
|
|
876
1012
|
return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
|
|
877
1013
|
},
|
|
878
1014
|
});
|
|
879
|
-
async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer', onProgress) {
|
|
1015
|
+
async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer', onProgress, routeDecision) {
|
|
880
1016
|
const governed = resolveGovernedAnswerRunner(projectRoot);
|
|
881
1017
|
const resolvedProvider = governed?.provider ?? null;
|
|
882
1018
|
const runner = governed?.runner ?? null;
|
|
@@ -902,6 +1038,9 @@ export async function startLocalServer(opts) {
|
|
|
902
1038
|
workspaceContext: request.workspaceContext,
|
|
903
1039
|
instruction: [
|
|
904
1040
|
'Route through the governed DQL answer loop.',
|
|
1041
|
+
...(routeDecision?.meaningResolution?.selectedConceptIds.length
|
|
1042
|
+
? [`Meaning resolution selected these retrieved qualified IDs: ${routeDecision.meaningResolution.selectedConceptIds.join(', ')}. Treat them as strong evidence, but still validate grain, dimensions, filters, authorization, and runtime compatibility before execution.`]
|
|
1043
|
+
: []),
|
|
905
1044
|
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
906
1045
|
'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
|
|
907
1046
|
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
@@ -914,6 +1053,9 @@ export async function startLocalServer(opts) {
|
|
|
914
1053
|
const controller = request.runId
|
|
915
1054
|
? activeAgentRunControllers.get(request.runId) ?? new AbortController()
|
|
916
1055
|
: new AbortController();
|
|
1056
|
+
const runSignal = request.signal
|
|
1057
|
+
? AbortSignal.any([request.signal, controller.signal])
|
|
1058
|
+
: controller.signal;
|
|
917
1059
|
// Best-effort active warehouse dialect so Lane-2 semantic compiles emit
|
|
918
1060
|
// dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
|
|
919
1061
|
// connection is configured — the compiler then uses its default dialect.
|
|
@@ -934,6 +1076,7 @@ export async function startLocalServer(opts) {
|
|
|
934
1076
|
activeDomain: requestedDomain,
|
|
935
1077
|
purpose: requestedPurpose,
|
|
936
1078
|
modelAreaId: requestedModelAreaId,
|
|
1079
|
+
skillRefs: agentRunWorkspaceValues(request, 'skillRefs'),
|
|
937
1080
|
source: 'explicit_ui',
|
|
938
1081
|
snapshotId: runProjectSnapshot.snapshotId,
|
|
939
1082
|
})
|
|
@@ -955,6 +1098,7 @@ export async function startLocalServer(opts) {
|
|
|
955
1098
|
reasoningEffort,
|
|
956
1099
|
...(analysisDepth ? { analysisDepth } : {}),
|
|
957
1100
|
projectRoot,
|
|
1101
|
+
preparedContextPack: preparedAgentContextPacks.get(request),
|
|
958
1102
|
domainContext,
|
|
959
1103
|
projectSnapshot: { snapshotId: runProjectSnapshot.snapshotId, manifest: runProjectSnapshot.manifest },
|
|
960
1104
|
assertProjectSnapshot: (snapshotId) => {
|
|
@@ -963,6 +1107,12 @@ export async function startLocalServer(opts) {
|
|
|
963
1107
|
projectSnapshots.assertCurrent(snapshotId);
|
|
964
1108
|
},
|
|
965
1109
|
...(semanticDriver ? { semanticDriver } : {}),
|
|
1110
|
+
...(routeDecision?.meaningResolution?.selectedConceptIds.length
|
|
1111
|
+
? { preferredEvidenceIds: routeDecision.meaningResolution.selectedConceptIds }
|
|
1112
|
+
: {}),
|
|
1113
|
+
...(routeDecision?.meaningResolution?.recommendedExecutionId
|
|
1114
|
+
? { preferredExecutionId: routeDecision.meaningResolution.recommendedExecutionId }
|
|
1115
|
+
: {}),
|
|
966
1116
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
967
1117
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
968
1118
|
getSchemaContext: getSchemaContextForAgent,
|
|
@@ -975,7 +1125,7 @@ export async function startLocalServer(opts) {
|
|
|
975
1125
|
if (turn.kind === 'error') {
|
|
976
1126
|
providerError = turn.message;
|
|
977
1127
|
}
|
|
978
|
-
},
|
|
1128
|
+
}, runSignal);
|
|
979
1129
|
if (!governedAnswer) {
|
|
980
1130
|
throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
|
|
981
1131
|
}
|
|
@@ -1017,9 +1167,11 @@ export async function startLocalServer(opts) {
|
|
|
1017
1167
|
return;
|
|
1018
1168
|
const existing = agentRunRecord(result.chartConfig) ?? {};
|
|
1019
1169
|
// A declared chart on the execution result came from authored DQL and is a
|
|
1020
|
-
// governed display contract.
|
|
1021
|
-
|
|
1022
|
-
|
|
1170
|
+
// governed display contract. Preserve its type/bindings, but still enrich a
|
|
1171
|
+
// missing display format from the result semantics; otherwise an authored
|
|
1172
|
+
// KPI for `lifetime_spend` renders as a generic `671.4K` instead of `$671.4K`.
|
|
1173
|
+
// Agent `suggestedViz` remains a soft preference when no chart was authored.
|
|
1174
|
+
const hasAuthoredChart = typeof existing.chart === 'string';
|
|
1023
1175
|
const recommendation = recommendVisualization(projectRoot, {
|
|
1024
1176
|
blockRef: governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name,
|
|
1025
1177
|
prompt: question,
|
|
@@ -1030,17 +1182,26 @@ export async function startLocalServer(opts) {
|
|
|
1030
1182
|
if (!recommendation.ok)
|
|
1031
1183
|
return;
|
|
1032
1184
|
const fieldHints = recommendation.display.fieldHints ?? {};
|
|
1033
|
-
const chart =
|
|
1185
|
+
const chart = hasAuthoredChart
|
|
1186
|
+
? String(existing.chart).replace(/_/g, '-')
|
|
1187
|
+
: recommendation.display.defaultVisualization.replace(/_/g, '-');
|
|
1034
1188
|
const agentChoice = typeof governedAnswer.suggestedViz === 'string'
|
|
1035
1189
|
? governedAnswer.suggestedViz.toLowerCase().replace(/_/g, '-')
|
|
1036
1190
|
: undefined;
|
|
1037
1191
|
result.chartConfig = {
|
|
1038
1192
|
...existing,
|
|
1039
1193
|
chart,
|
|
1040
|
-
decisionSource: agentChoice === chart ? 'agent' : 'data',
|
|
1041
|
-
rationale:
|
|
1042
|
-
|
|
1043
|
-
|
|
1194
|
+
decisionSource: hasAuthoredChart ? 'authored' : agentChoice === chart ? 'agent' : 'data',
|
|
1195
|
+
rationale: hasAuthoredChart
|
|
1196
|
+
? agentRunString(existing.rationale) ?? 'Authored DQL visualization enriched with result-aware display semantics.'
|
|
1197
|
+
: recommendation.display.rationale,
|
|
1198
|
+
...(typeof existing.x !== 'string' && typeof fieldHints.x === 'string' ? { x: fieldHints.x } : {}),
|
|
1199
|
+
...(typeof existing.y !== 'string' && typeof fieldHints.y === 'string' ? { y: fieldHints.y } : {}),
|
|
1200
|
+
...(typeof existing.color !== 'string' && typeof fieldHints.color === 'string' ? { color: fieldHints.color } : {}),
|
|
1201
|
+
...(typeof existing.format !== 'string'
|
|
1202
|
+
&& (fieldHints.format === 'currency' || fieldHints.format === 'percent' || fieldHints.format === 'number')
|
|
1203
|
+
? { format: fieldHints.format }
|
|
1204
|
+
: {}),
|
|
1044
1205
|
};
|
|
1045
1206
|
governedAnswer.suggestedViz = chart;
|
|
1046
1207
|
const evidence = governedAnswer.evidence ?? {
|
|
@@ -1130,7 +1291,7 @@ export async function startLocalServer(opts) {
|
|
|
1130
1291
|
const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emit }) => {
|
|
1131
1292
|
let governedAnswer;
|
|
1132
1293
|
try {
|
|
1133
|
-
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }));
|
|
1294
|
+
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }), routeDecision);
|
|
1134
1295
|
// Surface the approved Hint-Graph corrections that shaped this answer so the
|
|
1135
1296
|
// UI can show an "applied learnings" chip (memoryContext is already on the answer).
|
|
1136
1297
|
if (!governedAnswer.appliedHints) {
|
|
@@ -1294,49 +1455,49 @@ export async function startLocalServer(opts) {
|
|
|
1294
1455
|
// more provider calls retrying the same incompatible candidate.
|
|
1295
1456
|
const isPolicyBlocked = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'policy_blocked';
|
|
1296
1457
|
// The model tried to compose a governed query and declined despite having usable
|
|
1297
|
-
// context (e.g. it wasn't confident about a multi-table join).
|
|
1298
|
-
//
|
|
1299
|
-
//
|
|
1300
|
-
// engine's bounded loop, exactly as a grounding gap is retried today.
|
|
1458
|
+
// context (e.g. it wasn't confident about a multi-table join). The answer loop
|
|
1459
|
+
// has already spent its one evidence-aware repair. Keep this terminal and
|
|
1460
|
+
// inspectable; an ordinary Ask must never silently become a second Research run.
|
|
1301
1461
|
const isModelDeclined = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'model_declined';
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
: undefined;
|
|
1306
|
-
// Only a genuinely AMBIGUOUS question is surfaced as "needs clarification". A
|
|
1307
|
-
// grounding gap or a model decline is retried/escalated by the engine, and a
|
|
1308
|
-
// provider outage is surfaced as blocked so the UI offers a retry.
|
|
1462
|
+
// Only a genuinely AMBIGUOUS question is surfaced as "needs clarification".
|
|
1463
|
+
// Grounding/compiler gaps are terminal review states with their evidence trace;
|
|
1464
|
+
// provider outages are blocked so the UI can offer an explicit retry.
|
|
1309
1465
|
const needsClarification = governedAnswer.kind === 'no_answer'
|
|
1310
1466
|
&& !isGroundingGap && !isProviderError && !isModelDeclined && !isPolicyBlocked;
|
|
1311
1467
|
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
1312
1468
|
const runnableSql = governedAnswer.kind === 'no_answer' || (isExploratory && !governedAnswer.result)
|
|
1313
1469
|
? undefined
|
|
1314
1470
|
: sql;
|
|
1315
|
-
//
|
|
1316
|
-
//
|
|
1471
|
+
// Render executed rows deterministically for ordinary lookups. A second LLM
|
|
1472
|
+
// call is reserved for an explicit research route; certified, semantic, and
|
|
1473
|
+
// generated lookup answers must not pay another provider round-trip merely
|
|
1474
|
+
// to restate values the host already has.
|
|
1317
1475
|
let synthesizedAnswer;
|
|
1318
1476
|
if (shouldSynthesizeAgentRunAnswer(governedAnswer)) {
|
|
1319
1477
|
try {
|
|
1320
|
-
const provider =
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1478
|
+
const provider = route === 'research'
|
|
1479
|
+
? await createBlockStudioAssistProvider(projectRoot)
|
|
1480
|
+
: null;
|
|
1481
|
+
const preview = agentResultToSynthesisPreview(governedAnswer.result);
|
|
1482
|
+
const draft = governedAnswer.answer ?? governedAnswer.text;
|
|
1483
|
+
const result = await synthesizeAnswer({
|
|
1484
|
+
question: request.question,
|
|
1485
|
+
category: routeDecision?.category,
|
|
1486
|
+
// The primary Ask reply is always business-facing. Analysts keep
|
|
1487
|
+
// the full DQL, SQL, lineage, gates, and grain in the inspector.
|
|
1488
|
+
audience: 'stakeholder',
|
|
1489
|
+
resultPreview: preview,
|
|
1490
|
+
sql: sql,
|
|
1491
|
+
draftText: draft,
|
|
1492
|
+
gaps: businessNarrativeGaps(governedAnswer.validationWarnings),
|
|
1493
|
+
rankingDirection: governedAnswer.contextPack?.questionPlan.requestedShape.rankingDirection,
|
|
1494
|
+
}, provider
|
|
1495
|
+
? {
|
|
1335
1496
|
complete: ({ system, user, signal, onDelta }) => streamOrGenerate(provider, [{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 350, temperature: 0.3, signal }, onDelta ?? (() => { })),
|
|
1336
|
-
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1497
|
+
}
|
|
1498
|
+
: {});
|
|
1499
|
+
if (result.text)
|
|
1500
|
+
synthesizedAnswer = result.text;
|
|
1340
1501
|
}
|
|
1341
1502
|
catch {
|
|
1342
1503
|
// Keep the governed draft on any synthesis failure.
|
|
@@ -1427,26 +1588,22 @@ export async function startLocalServer(opts) {
|
|
|
1427
1588
|
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
1428
1589
|
...(isGroundingGap ? [
|
|
1429
1590
|
{
|
|
1430
|
-
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The
|
|
1591
|
+
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The bounded lookup could not prove the required metadata grounding. No automatic retry or Research escalation was started.', {
|
|
1431
1592
|
refusalCode: governedAnswer.refusalCode,
|
|
1432
1593
|
refusalDetails: governedAnswer.refusalDetails,
|
|
1433
1594
|
validationWarnings: governedAnswer.validationWarnings,
|
|
1434
1595
|
route: governedAnswer.route,
|
|
1435
1596
|
}),
|
|
1436
|
-
suggestedRepair: groundingRepairHint,
|
|
1437
|
-
repairAction: { kind: 'retry', hint: groundingRepairHint },
|
|
1438
1597
|
},
|
|
1439
1598
|
] : []),
|
|
1440
1599
|
...(isModelDeclined ? [
|
|
1441
1600
|
{
|
|
1442
|
-
...agentRunEvaluation('declined-despite-context', 'Answer grounding', false, 'blocking', 'The
|
|
1601
|
+
...agentRunEvaluation('declined-despite-context', 'Answer grounding', false, 'blocking', 'The bounded lookup could not compose a governed query after its in-lane repair. Start Research explicitly to investigate beyond this lookup budget.', {
|
|
1443
1602
|
refusalCode: governedAnswer.refusalCode,
|
|
1444
1603
|
refusalDetails: governedAnswer.refusalDetails,
|
|
1445
1604
|
validationWarnings: governedAnswer.validationWarnings,
|
|
1446
1605
|
route: governedAnswer.route,
|
|
1447
1606
|
}),
|
|
1448
|
-
suggestedRepair: declinedRepairHint,
|
|
1449
|
-
repairAction: { kind: 'escalate', route: 'research', hint: declinedRepairHint },
|
|
1450
1607
|
},
|
|
1451
1608
|
] : []),
|
|
1452
1609
|
...(isPolicyBlocked ? [
|
|
@@ -1984,7 +2141,138 @@ export async function startLocalServer(opts) {
|
|
|
1984
2141
|
};
|
|
1985
2142
|
},
|
|
1986
2143
|
};
|
|
1987
|
-
//
|
|
2144
|
+
// One immutable, question-ranked pack is shared by routing, planning, schema
|
|
2145
|
+
// lookup, and governed execution for the lifetime of a request. This removes
|
|
2146
|
+
// both positional catalog truncation and the previous duplicate retrieval pass.
|
|
2147
|
+
const preparedAgentContextPacks = new WeakMap();
|
|
2148
|
+
const pendingAgentContextPacks = new WeakMap();
|
|
2149
|
+
const buildAgentRunContextPack = async (request) => {
|
|
2150
|
+
const prepared = preparedAgentContextPacks.get(request);
|
|
2151
|
+
if (prepared)
|
|
2152
|
+
return prepared;
|
|
2153
|
+
const pending = pendingAgentContextPacks.get(request);
|
|
2154
|
+
if (pending)
|
|
2155
|
+
return pending;
|
|
2156
|
+
const snapshot = projectSnapshot();
|
|
2157
|
+
const requestedDomain = agentRunWorkspaceValue(request, 'domain');
|
|
2158
|
+
// CTX-003: resolve result entities/values before evidence retrieval. The
|
|
2159
|
+
// router, planner, and answer loop must rank the same typed follow-up; doing
|
|
2160
|
+
// this only inside the provider adapter allowed stale catalog matches to win
|
|
2161
|
+
// before "they" / "this amount" became customer-scoped context.
|
|
2162
|
+
const followUp = resolveAgentFollowUpContext(request.conversationContext, request.question);
|
|
2163
|
+
const serverSnapshot = agentRunRecord(request.conversationContext?.serverSnapshot);
|
|
2164
|
+
const topicRelation = agentRunString(serverSnapshot?.topicRelation);
|
|
2165
|
+
// The readiness marker is source-versioned. When it matches, pass the
|
|
2166
|
+
// already-built metadata identity into retrieval so buildLocalContextPack
|
|
2167
|
+
// opens the immutable snapshot directly instead of rebuilding all metadata
|
|
2168
|
+
// merely to rediscover the same fingerprint on every follow-up.
|
|
2169
|
+
const preparedMetadataFingerprint = isAgentProjectIndexReady(projectRoot)
|
|
2170
|
+
? currentMetadataFingerprint(projectRoot)
|
|
2171
|
+
: undefined;
|
|
2172
|
+
const task = buildLocalContextPack(projectRoot, {
|
|
2173
|
+
question: request.question,
|
|
2174
|
+
followUp,
|
|
2175
|
+
priorContextPackId: agentRunString(request.conversationContext?.contextPackId),
|
|
2176
|
+
conversationTopicRelation: topicRelation === 'continuation'
|
|
2177
|
+
|| topicRelation === 'refinement'
|
|
2178
|
+
|| topicRelation === 'return'
|
|
2179
|
+
|| topicRelation === 'shift'
|
|
2180
|
+
? topicRelation
|
|
2181
|
+
: undefined,
|
|
2182
|
+
preparedMetadataFingerprint,
|
|
2183
|
+
surface: 'notebook',
|
|
2184
|
+
selectedContext: {
|
|
2185
|
+
selectedObject: request.selectedObject,
|
|
2186
|
+
workspaceContext: request.workspaceContext,
|
|
2187
|
+
},
|
|
2188
|
+
strictness: request.analysisDepth === 'deep' ? 'exploratory' : 'balanced',
|
|
2189
|
+
limit: request.analysisDepth === 'deep' ? 120 : 80,
|
|
2190
|
+
domainContext: requestedDomain
|
|
2191
|
+
? resolveDomainContextEnvelope({
|
|
2192
|
+
manifest: snapshot.manifest,
|
|
2193
|
+
activeDomain: requestedDomain,
|
|
2194
|
+
purpose: agentRunWorkspaceValue(request, 'purpose'),
|
|
2195
|
+
modelAreaId: agentRunWorkspaceValue(request, 'modelAreaId'),
|
|
2196
|
+
skillRefs: agentRunWorkspaceValues(request, 'skillRefs'),
|
|
2197
|
+
source: 'explicit_ui',
|
|
2198
|
+
snapshotId: snapshot.snapshotId,
|
|
2199
|
+
})
|
|
2200
|
+
: undefined,
|
|
2201
|
+
}).then((pack) => {
|
|
2202
|
+
preparedAgentContextPacks.set(request, pack);
|
|
2203
|
+
pendingAgentContextPacks.delete(request);
|
|
2204
|
+
return pack;
|
|
2205
|
+
}).catch((error) => {
|
|
2206
|
+
pendingAgentContextPacks.delete(request);
|
|
2207
|
+
throw error;
|
|
2208
|
+
});
|
|
2209
|
+
pendingAgentContextPacks.set(request, task);
|
|
2210
|
+
return task;
|
|
2211
|
+
};
|
|
2212
|
+
const buildAgentRunEvidence = async (request) => {
|
|
2213
|
+
const startedAt = Date.now();
|
|
2214
|
+
const pack = await buildAgentRunContextPack(request);
|
|
2215
|
+
const meaningEvidence = pack.retrievalDiagnostics.meaningEvidence;
|
|
2216
|
+
if (!meaningEvidence) {
|
|
2217
|
+
return {
|
|
2218
|
+
snapshotId: pack.id,
|
|
2219
|
+
sourceFingerprint: pack.freshness.fingerprint ?? undefined,
|
|
2220
|
+
candidates: [],
|
|
2221
|
+
diagnostics: { durationMs: Date.now() - startedAt },
|
|
2222
|
+
};
|
|
2223
|
+
}
|
|
2224
|
+
const evidence = toAgentRetrievalEvidence(meaningEvidence, pack.questionPlan, {
|
|
2225
|
+
snapshotId: pack.id,
|
|
2226
|
+
sourceFingerprint: pack.freshness.fingerprint ?? undefined,
|
|
2227
|
+
durationMs: Date.now() - startedAt,
|
|
2228
|
+
truncated: pack.retrievalDiagnostics.topRejected.length > 0,
|
|
2229
|
+
});
|
|
2230
|
+
const certifiedFits = new Map(pack.retrievalDiagnostics.certifiedCandidateFits.map((fit) => [fit.objectKey, fit]));
|
|
2231
|
+
const semanticEvidence = new Set(pack.routeDecision.selectedEvidence
|
|
2232
|
+
.filter((item) => item.role === 'semantic_metric')
|
|
2233
|
+
.map((item) => item.objectKey));
|
|
2234
|
+
return {
|
|
2235
|
+
...evidence,
|
|
2236
|
+
candidates: evidence.candidates.map((candidate) => {
|
|
2237
|
+
if (candidate.kind === 'certified_block') {
|
|
2238
|
+
const fit = certifiedFits.get(candidate.id);
|
|
2239
|
+
return {
|
|
2240
|
+
...candidate,
|
|
2241
|
+
compatibility: fit?.action === 'certified_answer'
|
|
2242
|
+
? 'compatible'
|
|
2243
|
+
: fit?.action === 'rejected_for_fit'
|
|
2244
|
+
? 'incompatible'
|
|
2245
|
+
: 'partial',
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
if ((candidate.kind === 'semantic_metric' || candidate.kind === 'semantic_member')
|
|
2249
|
+
&& semanticEvidence.has(candidate.id)
|
|
2250
|
+
&& pack.routeDecision.route !== 'clarify'
|
|
2251
|
+
&& pack.routeDecision.route !== 'conflict') {
|
|
2252
|
+
const requestedDimensions = pack.questionPlan.requestedShape.dimensions.map((dimension) => dimension.toLowerCase());
|
|
2253
|
+
const availableDimensions = (candidate.dimensions ?? []).map((dimension) => dimension.toLowerCase());
|
|
2254
|
+
const dimensionsFit = requestedDimensions.length === 0 || requestedDimensions.every((requested) => availableDimensions.some((available) => available === requested || available.endsWith(`.${requested}`)));
|
|
2255
|
+
const requestedTimeGrain = pack.questionPlan.timeTerms[0]?.toLowerCase();
|
|
2256
|
+
const availableTimeGrains = (candidate.timeGrains ?? []).map((grain) => grain.toLowerCase());
|
|
2257
|
+
const timeGrainFits = !requestedTimeGrain
|
|
2258
|
+
|| availableTimeGrains.includes(requestedTimeGrain);
|
|
2259
|
+
return { ...candidate, compatibility: dimensionsFit && timeGrainFits ? 'compatible' : 'partial' };
|
|
2260
|
+
}
|
|
2261
|
+
if (candidate.trustTier === 'governed_sql')
|
|
2262
|
+
return { ...candidate, compatibility: 'partial' };
|
|
2263
|
+
return candidate;
|
|
2264
|
+
}),
|
|
2265
|
+
};
|
|
2266
|
+
};
|
|
2267
|
+
const buildRankedAgentRunCatalogContext = async (request) => {
|
|
2268
|
+
const evidence = await buildAgentRunEvidence(request);
|
|
2269
|
+
return evidence.candidates.map((candidate) => {
|
|
2270
|
+
const detail = candidate.definition ? `: ${candidate.definition}` : '';
|
|
2271
|
+
return `- ${candidate.id} [${candidate.trustTier}; ${candidate.compatibility}]${detail}`;
|
|
2272
|
+
}).join('\n');
|
|
2273
|
+
};
|
|
2274
|
+
// Compact fallback used only for plain conversational replies. Analytical
|
|
2275
|
+
// turns use the structured, question-ranked evidence path above.
|
|
1988
2276
|
const buildAgentRunCatalogContext = () => {
|
|
1989
2277
|
try {
|
|
1990
2278
|
const blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
@@ -2020,7 +2308,7 @@ export async function startLocalServer(opts) {
|
|
|
2020
2308
|
{ role: 'user', content: user },
|
|
2021
2309
|
], { maxTokens: 700, temperature: 0.1, signal });
|
|
2022
2310
|
},
|
|
2023
|
-
getCatalogContext:
|
|
2311
|
+
getCatalogContext: buildRankedAgentRunCatalogContext,
|
|
2024
2312
|
});
|
|
2025
2313
|
// Hybrid router: keep the deterministic decision when it is confident (certified
|
|
2026
2314
|
// fast paths + greetings stay 0-LLM); spend one cheap classification call only for
|
|
@@ -2030,13 +2318,22 @@ export async function startLocalServer(opts) {
|
|
|
2030
2318
|
complete: async ({ system, user, signal }) => {
|
|
2031
2319
|
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
2032
2320
|
if (!provider)
|
|
2033
|
-
throw new Error('No AI provider configured for
|
|
2321
|
+
throw new Error('No AI provider configured for meaning resolution.');
|
|
2034
2322
|
return provider.generate([
|
|
2035
2323
|
{ role: 'system', content: system },
|
|
2036
2324
|
{ role: 'user', content: user },
|
|
2037
|
-
], {
|
|
2325
|
+
], {
|
|
2326
|
+
maxTokens: 600,
|
|
2327
|
+
temperature: 0,
|
|
2328
|
+
// AGT-009/PERF-002: ambiguity gets one bounded resolver call. If the
|
|
2329
|
+
// provider stalls, the router falls back to its evidence-only decision
|
|
2330
|
+
// instead of leaving the UI in "Generating and validating SQL" for a
|
|
2331
|
+
// minute or more.
|
|
2332
|
+
signal: boundedAgentMeaningSignal(signal),
|
|
2333
|
+
});
|
|
2038
2334
|
},
|
|
2039
|
-
|
|
2335
|
+
getEvidence: buildAgentRunEvidence,
|
|
2336
|
+
getCatalogContext: buildRankedAgentRunCatalogContext,
|
|
2040
2337
|
});
|
|
2041
2338
|
const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
|
|
2042
2339
|
// A run may outlive its streaming browser connection, so cancellation is
|
|
@@ -2319,30 +2616,24 @@ export async function startLocalServer(opts) {
|
|
|
2319
2616
|
};
|
|
2320
2617
|
};
|
|
2321
2618
|
const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question, preparedContextPack).catch(() => []);
|
|
2619
|
+
const valueGrounding = resolveAgentRuntimeValueGrounding(projectConfig);
|
|
2322
2620
|
if (catalogContext.length > 0) {
|
|
2323
|
-
if (!connection)
|
|
2621
|
+
if (!connection || valueGrounding.mode !== 'safe_automatic')
|
|
2324
2622
|
return catalogContext;
|
|
2325
|
-
//
|
|
2326
|
-
//
|
|
2327
|
-
//
|
|
2328
|
-
const
|
|
2329
|
-
|
|
2330
|
-
: undefined;
|
|
2331
|
-
const runtimeContext = runtimeScan?.ranked ?? [];
|
|
2332
|
-
const merged = mergeAgentSchemaContexts(catalogContext, runtimeContext);
|
|
2333
|
-
const enriched = await enrichAgentSchemaContextWithValueMatches(question, merged, executor, connection);
|
|
2334
|
-
recordAgentRuntimeSchemaSnapshot(projectRoot, !runtimeScan?.snapshot.length
|
|
2335
|
-
? enriched
|
|
2336
|
-
: mergeAgentSchemaSampleValues(runtimeScan.snapshot, enriched), runtimeContext.length > 0
|
|
2337
|
-
? 'full information_schema runtime scan for composite Ask AI question'
|
|
2338
|
-
: 'catalog enriched runtime schema');
|
|
2623
|
+
// The immutable dbt/DQL snapshot already owns schema discovery. A named
|
|
2624
|
+
// row value must not turn a warm Ask into an information_schema scan over
|
|
2625
|
+
// thousands of enterprise tables; only field-scoped value probes are live.
|
|
2626
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, catalogContext, executor, connection, valueGrounding.searchSafeColumns);
|
|
2627
|
+
recordAgentRuntimeSchemaSnapshot(projectRoot, catalogContext, 'catalog runtime schema');
|
|
2339
2628
|
return enriched;
|
|
2340
2629
|
}
|
|
2341
2630
|
if (!connection)
|
|
2342
2631
|
return [];
|
|
2343
2632
|
try {
|
|
2344
2633
|
const schemaContext = await scanRuntimeSchema();
|
|
2345
|
-
const enriched =
|
|
2634
|
+
const enriched = valueGrounding.mode === 'safe_automatic'
|
|
2635
|
+
? await enrichAgentSchemaContextWithValueMatches(question, schemaContext.ranked, executor, connection, valueGrounding.searchSafeColumns)
|
|
2636
|
+
: schemaContext.ranked;
|
|
2346
2637
|
recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext.snapshot, 'full information_schema runtime scan');
|
|
2347
2638
|
return enriched;
|
|
2348
2639
|
}
|
|
@@ -3040,20 +3331,7 @@ export async function startLocalServer(opts) {
|
|
|
3040
3331
|
}
|
|
3041
3332
|
// Hot-reload semantic layer on change and notify frontend
|
|
3042
3333
|
if (dir === 'semantic-layer') {
|
|
3043
|
-
|
|
3044
|
-
const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
|
|
3045
|
-
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
|
|
3046
|
-
: undefined;
|
|
3047
|
-
resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery).then((refreshed) => {
|
|
3048
|
-
if (refreshed.layer) {
|
|
3049
|
-
semanticLayer = refreshed.layer;
|
|
3050
|
-
semanticLayerErrors = refreshed.errors;
|
|
3051
|
-
semanticLastSyncTime = new Date().toISOString();
|
|
3052
|
-
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
3053
|
-
}
|
|
3054
|
-
else if (refreshed.errors.length > 0) {
|
|
3055
|
-
semanticLayerErrors = refreshed.errors;
|
|
3056
|
-
}
|
|
3334
|
+
reloadSemanticLayer().then(() => {
|
|
3057
3335
|
// Notify all connected notebook clients to re-fetch the semantic layer
|
|
3058
3336
|
const reloadPayload = JSON.stringify({ type: 'semantic-reload' });
|
|
3059
3337
|
for (const client of sseClients) {
|
|
@@ -3070,7 +3348,7 @@ export async function startLocalServer(opts) {
|
|
|
3070
3348
|
}
|
|
3071
3349
|
catch { /* dir not watchable */ }
|
|
3072
3350
|
}
|
|
3073
|
-
const configuredDbtManifest = resolveDbtManifestPath(projectRoot);
|
|
3351
|
+
const configuredDbtManifest = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
3074
3352
|
if (configuredDbtManifest && existsSync(dirname(configuredDbtManifest))) {
|
|
3075
3353
|
try {
|
|
3076
3354
|
watch(dirname(configuredDbtManifest), { persistent: false }, (_eventType, filename) => {
|
|
@@ -3091,6 +3369,17 @@ export async function startLocalServer(opts) {
|
|
|
3091
3369
|
sseClients.delete(client);
|
|
3092
3370
|
}
|
|
3093
3371
|
}
|
|
3372
|
+
void reloadSemanticLayer().then(() => {
|
|
3373
|
+
const reloadPayload = JSON.stringify({ type: 'semantic-reload' });
|
|
3374
|
+
for (const client of sseClients) {
|
|
3375
|
+
try {
|
|
3376
|
+
client.write(`event: change\ndata: ${reloadPayload}\n\n`);
|
|
3377
|
+
}
|
|
3378
|
+
catch {
|
|
3379
|
+
sseClients.delete(client);
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
}).catch(() => { });
|
|
3094
3383
|
});
|
|
3095
3384
|
}
|
|
3096
3385
|
catch { /* dbt artifact directory not watchable */ }
|
|
@@ -3475,6 +3764,18 @@ export async function startLocalServer(opts) {
|
|
|
3475
3764
|
snapshotError = error instanceof Error ? error.message : String(error);
|
|
3476
3765
|
}
|
|
3477
3766
|
}
|
|
3767
|
+
const preparation = latestDbtPreparationJob();
|
|
3768
|
+
const preparationForSnapshot = preparation?.snapshotId === snapshotId ? preparation : undefined;
|
|
3769
|
+
const searchIndexesPresent = isAgentProjectIndexReady(projectRoot);
|
|
3770
|
+
const snapshotState = preparationForSnapshot?.status === 'running' || preparationForSnapshot?.status === 'queued'
|
|
3771
|
+
? 'building'
|
|
3772
|
+
: preparationForSnapshot?.status === 'failed'
|
|
3773
|
+
? 'failed'
|
|
3774
|
+
: snapshotId && searchIndexesPresent
|
|
3775
|
+
? 'ready'
|
|
3776
|
+
: snapshotId
|
|
3777
|
+
? 'stale'
|
|
3778
|
+
: 'missing';
|
|
3478
3779
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3479
3780
|
res.end(serializeJSON({
|
|
3480
3781
|
requestId,
|
|
@@ -3491,14 +3792,27 @@ export async function startLocalServer(opts) {
|
|
|
3491
3792
|
subPath: projectConfig.dbt?.subPath,
|
|
3492
3793
|
projectFound: existsSync(join(dbtProjectDir, 'dbt_project.yml')),
|
|
3493
3794
|
manifestFound,
|
|
3795
|
+
artifactState: manifestFound ? 'ready' : 'missing',
|
|
3494
3796
|
},
|
|
3495
3797
|
modeling: {
|
|
3496
3798
|
enabled: projectConfig.manifestVersion === 3 && projectConfig.modeling?.mode === 'dbt-first',
|
|
3497
3799
|
manifestVersion: projectConfig.manifestVersion ?? 2,
|
|
3498
3800
|
mode: projectConfig.modeling?.mode,
|
|
3801
|
+
snapshotState,
|
|
3499
3802
|
},
|
|
3500
3803
|
domains: { count: registry.values().length, diagnostics: registry.diagnostics },
|
|
3501
3804
|
snapshot: { id: snapshotId, error: snapshotError },
|
|
3805
|
+
preparation,
|
|
3806
|
+
readiness: {
|
|
3807
|
+
project: {
|
|
3808
|
+
state: snapshotState === 'ready' ? 'ready' : snapshotState === 'building' ? 'preparing' : snapshotState === 'failed' ? 'failed' : dbtConfigured ? 'configured' : 'missing',
|
|
3809
|
+
message: preparationForSnapshot?.message ?? (snapshotState === 'ready'
|
|
3810
|
+
? 'dbt metadata and governed search indexes are ready.'
|
|
3811
|
+
: snapshotState === 'stale'
|
|
3812
|
+
? 'dbt metadata is configured; search indexes will be refreshed before governed Ask.'
|
|
3813
|
+
: undefined),
|
|
3814
|
+
},
|
|
3815
|
+
},
|
|
3502
3816
|
capabilities: {
|
|
3503
3817
|
warehouse: Boolean(connection),
|
|
3504
3818
|
ai: Boolean(process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.GEMINI_API_KEY || process.env.OLLAMA_BASE_URL),
|
|
@@ -3523,6 +3837,7 @@ export async function startLocalServer(opts) {
|
|
|
3523
3837
|
if (req.method === 'POST' && path === '/api/onboarding/dbt/apply') {
|
|
3524
3838
|
const requestId = apiRequestId('onboarding-dbt-apply');
|
|
3525
3839
|
try {
|
|
3840
|
+
const applyStartedAt = Date.now();
|
|
3526
3841
|
const body = await readJSON(req);
|
|
3527
3842
|
let preview;
|
|
3528
3843
|
try {
|
|
@@ -3558,6 +3873,7 @@ export async function startLocalServer(opts) {
|
|
|
3558
3873
|
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed after preview. Review the refreshed preview before applying.', nextActions: ['Refresh the preview and review the new diff.'] })));
|
|
3559
3874
|
return;
|
|
3560
3875
|
}
|
|
3876
|
+
const validationDurationMs = Date.now() - applyStartedAt;
|
|
3561
3877
|
const nextConfig = {
|
|
3562
3878
|
...projectConfig,
|
|
3563
3879
|
manifestVersion: 3,
|
|
@@ -3583,8 +3899,16 @@ export async function startLocalServer(opts) {
|
|
|
3583
3899
|
projectSnapshots.invalidate();
|
|
3584
3900
|
invalidateAgentProjectState(projectRoot);
|
|
3585
3901
|
let snapshotId;
|
|
3902
|
+
let snapshotManifest;
|
|
3903
|
+
const compileStartedAt = Date.now();
|
|
3586
3904
|
try {
|
|
3587
|
-
|
|
3905
|
+
const semanticReload = await reloadSemanticLayer();
|
|
3906
|
+
if (semanticReload.errors.length > 0) {
|
|
3907
|
+
throw new Error(`dbt semantic artifacts could not be loaded: ${semanticReload.errors.join('; ')}`);
|
|
3908
|
+
}
|
|
3909
|
+
const snapshot = projectSnapshot();
|
|
3910
|
+
snapshotId = snapshot.snapshotId;
|
|
3911
|
+
snapshotManifest = snapshot.manifest;
|
|
3588
3912
|
}
|
|
3589
3913
|
catch (error) {
|
|
3590
3914
|
const rollbackPath = `${configPath}.rollback-${process.pid}`;
|
|
@@ -3594,10 +3918,25 @@ export async function startLocalServer(opts) {
|
|
|
3594
3918
|
connection = previousConnection;
|
|
3595
3919
|
projectSnapshots.invalidate();
|
|
3596
3920
|
invalidateAgentProjectState(projectRoot);
|
|
3921
|
+
await reloadSemanticLayer();
|
|
3597
3922
|
throw Object.assign(new Error(`dbt-first configuration did not compile and was rolled back: ${error instanceof Error ? error.message : String(error)}`), { code: 'SNAPSHOT_BUILD_FAILED' });
|
|
3598
3923
|
}
|
|
3924
|
+
const preparation = startDbtPreparationJob({
|
|
3925
|
+
kind: 'dbt_prepare',
|
|
3926
|
+
snapshotId,
|
|
3927
|
+
manifest: snapshotManifest,
|
|
3928
|
+
validationDurationMs,
|
|
3929
|
+
compileDurationMs: Date.now() - compileStartedAt,
|
|
3930
|
+
});
|
|
3599
3931
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3600
|
-
res.end(serializeJSON({
|
|
3932
|
+
res.end(serializeJSON({
|
|
3933
|
+
requestId,
|
|
3934
|
+
applied: true,
|
|
3935
|
+
config: { manifestVersion: 3, modeling: nextConfig.modeling, dbt: nextConfig.dbt },
|
|
3936
|
+
fingerprint: preview.fingerprint,
|
|
3937
|
+
jobId: preparation.id,
|
|
3938
|
+
...preparation,
|
|
3939
|
+
}));
|
|
3601
3940
|
}
|
|
3602
3941
|
catch (error) {
|
|
3603
3942
|
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
@@ -3608,7 +3947,7 @@ export async function startLocalServer(opts) {
|
|
|
3608
3947
|
}
|
|
3609
3948
|
if (req.method === 'POST' && path === '/api/onboarding/refresh') {
|
|
3610
3949
|
const requestId = apiRequestId('onboarding-refresh');
|
|
3611
|
-
const
|
|
3950
|
+
const refreshStartedAt = Date.now();
|
|
3612
3951
|
try {
|
|
3613
3952
|
const body = await readJSON(req);
|
|
3614
3953
|
const currentArtifact = previewDbtOnboarding({});
|
|
@@ -3617,20 +3956,25 @@ export async function startLocalServer(opts) {
|
|
|
3617
3956
|
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed before refresh. Review the current artifact preview.', nextActions: ['Return to the dbt preview step and review the refreshed fingerprint.'] })));
|
|
3618
3957
|
return;
|
|
3619
3958
|
}
|
|
3959
|
+
const validationDurationMs = Date.now() - refreshStartedAt;
|
|
3960
|
+
const compileStartedAt = Date.now();
|
|
3620
3961
|
projectSnapshots.invalidate();
|
|
3962
|
+
await reloadSemanticLayer();
|
|
3621
3963
|
const snapshot = projectSnapshot();
|
|
3622
3964
|
invalidateAgentProjectState(projectRoot);
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3965
|
+
const job = startDbtPreparationJob({
|
|
3966
|
+
kind: 'dbt_refresh',
|
|
3967
|
+
snapshotId: snapshot.snapshotId,
|
|
3968
|
+
manifest: snapshot.manifest,
|
|
3969
|
+
validationDurationMs,
|
|
3970
|
+
compileDurationMs: Date.now() - compileStartedAt,
|
|
3971
|
+
});
|
|
3626
3972
|
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3627
|
-
res.end(serializeJSON({ requestId,
|
|
3973
|
+
res.end(serializeJSON({ requestId, jobId: job.id, ...job }));
|
|
3628
3974
|
}
|
|
3629
3975
|
catch (error) {
|
|
3630
|
-
const job = { id, kind: 'dbt_refresh', status: 'failed', createdAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) };
|
|
3631
|
-
onboardingJobs.set(id, job);
|
|
3632
3976
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3633
|
-
res.end(serializeJSON(
|
|
3977
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SNAPSHOT_BUILD_FAILED', message: error instanceof Error ? error.message : String(error), nextActions: ['Keep using the previous snapshot while fixing compile diagnostics.'] })));
|
|
3634
3978
|
}
|
|
3635
3979
|
return;
|
|
3636
3980
|
}
|
|
@@ -3649,10 +3993,17 @@ export async function startLocalServer(opts) {
|
|
|
3649
3993
|
const requestId = apiRequestId('onboarding-job-cancel');
|
|
3650
3994
|
const id = decodeURIComponent(path.slice('/api/onboarding/jobs/'.length));
|
|
3651
3995
|
const job = onboardingJobs.get(id);
|
|
3652
|
-
|
|
3653
|
-
|
|
3996
|
+
const cancelled = job ? {
|
|
3997
|
+
...job,
|
|
3998
|
+
status: 'cancelled',
|
|
3999
|
+
message: 'Stopped waiting for dbt preparation. The last valid snapshot remains available.',
|
|
4000
|
+
updatedAt: new Date().toISOString(),
|
|
4001
|
+
phases: job.phases.map((phase) => phase.status === 'running' ? { ...phase, status: 'cancelled' } : phase),
|
|
4002
|
+
} : undefined;
|
|
4003
|
+
if (cancelled)
|
|
4004
|
+
onboardingJobs.set(id, cancelled);
|
|
3654
4005
|
res.writeHead(job ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3655
|
-
res.end(serializeJSON(
|
|
4006
|
+
res.end(serializeJSON(cancelled ? { requestId, ...cancelled } : apiErrorEnvelope({ requestId, code: 'ONBOARDING_JOB_NOT_FOUND', message: `onboarding job not found: ${id}`, recoverable: false })));
|
|
3656
4007
|
return;
|
|
3657
4008
|
}
|
|
3658
4009
|
if (req.method === 'POST' && path === '/api/onboarding/domains/discover') {
|
|
@@ -3786,7 +4137,12 @@ export async function startLocalServer(opts) {
|
|
|
3786
4137
|
const requestId = apiRequestId('domain-workspace');
|
|
3787
4138
|
const suffix = decodeURIComponent(path.slice('/api/domain-workspaces/'.length));
|
|
3788
4139
|
const relatedSuffix = '/related-products';
|
|
3789
|
-
const
|
|
4140
|
+
const knowledgeSuffix = '/knowledge';
|
|
4141
|
+
const domainId = suffix.endsWith(relatedSuffix)
|
|
4142
|
+
? suffix.slice(0, -relatedSuffix.length)
|
|
4143
|
+
: suffix.endsWith(knowledgeSuffix)
|
|
4144
|
+
? suffix.slice(0, -knowledgeSuffix.length)
|
|
4145
|
+
: suffix;
|
|
3790
4146
|
const snapshot = projectSnapshot();
|
|
3791
4147
|
const manifest = snapshot.manifest;
|
|
3792
4148
|
if (!manifest.modeling?.packages[domainId]) {
|
|
@@ -3794,6 +4150,19 @@ export async function startLocalServer(opts) {
|
|
|
3794
4150
|
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DOMAIN_NOT_FOUND', message: `domain workspace not found: ${domainId}`, recoverable: false })));
|
|
3795
4151
|
return;
|
|
3796
4152
|
}
|
|
4153
|
+
if (suffix.endsWith(knowledgeSuffix)) {
|
|
4154
|
+
await ensureMetadataCatalogFresh(projectRoot, { manifest, semanticLayer });
|
|
4155
|
+
const knowledge = readIndexedDomainKnowledge(projectRoot, domainId)
|
|
4156
|
+
?? canonicalDomainKnowledge(manifest, domainId, snapshot.snapshotId);
|
|
4157
|
+
if (!knowledge) {
|
|
4158
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4159
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DOMAIN_KNOWLEDGE_NOT_FOUND', message: `domain knowledge capsule not found: ${domainId}`, recoverable: false })));
|
|
4160
|
+
return;
|
|
4161
|
+
}
|
|
4162
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4163
|
+
res.end(serializeJSON({ requestId, ...knowledge }));
|
|
4164
|
+
return;
|
|
4165
|
+
}
|
|
3797
4166
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3798
4167
|
res.end(serializeJSON(suffix.endsWith(relatedSuffix)
|
|
3799
4168
|
? { requestId, domain: domainId, ...relatedProductsForDomain(manifest, domainId), snapshotId: snapshot.snapshotId }
|
|
@@ -3910,12 +4279,35 @@ export async function startLocalServer(opts) {
|
|
|
3910
4279
|
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 50));
|
|
3911
4280
|
const cursor = Math.max(0, Number(url.searchParams.get('cursor')) || 0);
|
|
3912
4281
|
const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
|
|
4282
|
+
const queryTokens = query.split(/[^a-z0-9]+/).filter(Boolean);
|
|
3913
4283
|
const domain = (url.searchParams.get('domain') ?? '').trim();
|
|
3914
4284
|
const boundByDbtId = new Map(Object.values(manifest.modeling?.entities ?? {}).map((entity) => [entity.dbtUniqueId, entity]));
|
|
3915
4285
|
const nodes = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
3916
|
-
.filter((node) =>
|
|
4286
|
+
.filter((node) => {
|
|
4287
|
+
if (!queryTokens.length)
|
|
4288
|
+
return true;
|
|
4289
|
+
const haystack = `${node.name} ${node.uniqueId} ${node.relation ?? ''} ${node.sourcePath ?? ''}`.toLowerCase();
|
|
4290
|
+
return queryTokens.every((token) => haystack.includes(token));
|
|
4291
|
+
})
|
|
3917
4292
|
.filter((node) => !domain || boundByDbtId.get(node.uniqueId)?.domain === domain)
|
|
3918
|
-
.sort((a, b) =>
|
|
4293
|
+
.sort((a, b) => {
|
|
4294
|
+
if (!query)
|
|
4295
|
+
return a.uniqueId.localeCompare(b.uniqueId);
|
|
4296
|
+
const score = (node) => {
|
|
4297
|
+
const name = node.name.toLowerCase();
|
|
4298
|
+
const uniqueId = node.uniqueId.toLowerCase();
|
|
4299
|
+
let value = name === query || uniqueId === query ? 1_000 : 0;
|
|
4300
|
+
if (name.startsWith(query))
|
|
4301
|
+
value += 500;
|
|
4302
|
+
if (uniqueId.startsWith(query))
|
|
4303
|
+
value += 300;
|
|
4304
|
+
if (name.includes(query))
|
|
4305
|
+
value += 200;
|
|
4306
|
+
value += queryTokens.reduce((total, token) => total + (name.startsWith(token) ? 80 : name.includes(token) ? 40 : 10), 0);
|
|
4307
|
+
return value;
|
|
4308
|
+
};
|
|
4309
|
+
return score(b) - score(a) || a.uniqueId.localeCompare(b.uniqueId);
|
|
4310
|
+
});
|
|
3919
4311
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3920
4312
|
res.end(serializeJSON({
|
|
3921
4313
|
requestId,
|
|
@@ -4134,8 +4526,13 @@ export async function startLocalServer(opts) {
|
|
|
4134
4526
|
}
|
|
4135
4527
|
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
4136
4528
|
const runId = parsed.request.runId;
|
|
4529
|
+
const runController = new AbortController();
|
|
4530
|
+
parsed.request.signal = AbortSignal.any([
|
|
4531
|
+
runController.signal,
|
|
4532
|
+
AbortSignal.timeout(agentRunDeadlineMs(parsed.request)),
|
|
4533
|
+
]);
|
|
4137
4534
|
if (runId)
|
|
4138
|
-
activeAgentRunControllers.set(runId,
|
|
4535
|
+
activeAgentRunControllers.set(runId, runController);
|
|
4139
4536
|
try {
|
|
4140
4537
|
if (wantsStream) {
|
|
4141
4538
|
res.writeHead(200, {
|
|
@@ -7788,16 +8185,7 @@ export async function startLocalServer(opts) {
|
|
|
7788
8185
|
}
|
|
7789
8186
|
if (req.method === 'POST' && path === '/api/semantic-layer/reload') {
|
|
7790
8187
|
try {
|
|
7791
|
-
|
|
7792
|
-
const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
|
|
7793
|
-
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
|
|
7794
|
-
: undefined;
|
|
7795
|
-
const refreshed = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
|
|
7796
|
-
semanticLayer = refreshed.layer;
|
|
7797
|
-
semanticLayerErrors = refreshed.errors;
|
|
7798
|
-
semanticDetectedProvider = refreshed.detectedProvider;
|
|
7799
|
-
semanticLastSyncTime = refreshed.layer ? new Date().toISOString() : null;
|
|
7800
|
-
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
8188
|
+
await reloadSemanticLayer();
|
|
7801
8189
|
const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
|
|
7802
8190
|
semanticLayer,
|
|
7803
8191
|
semanticErrors: semanticLayerErrors,
|
|
@@ -7807,7 +8195,7 @@ export async function startLocalServer(opts) {
|
|
|
7807
8195
|
});
|
|
7808
8196
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7809
8197
|
res.end(serializeJSON({
|
|
7810
|
-
ok: Boolean(
|
|
8198
|
+
ok: Boolean(semanticLayer),
|
|
7811
8199
|
...diagnostics,
|
|
7812
8200
|
}));
|
|
7813
8201
|
}
|
|
@@ -9646,13 +10034,17 @@ export async function startLocalServer(opts) {
|
|
|
9646
10034
|
try {
|
|
9647
10035
|
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
9648
10036
|
const result = queryBusiness360(graph, rawNodeId);
|
|
9649
|
-
|
|
10037
|
+
const snapshot = projectSnapshot();
|
|
10038
|
+
await ensureMetadataCatalogFresh(projectRoot, { manifest: snapshot.manifest, semanticLayer });
|
|
10039
|
+
const knowledge = readIndexedKnowledge360(projectRoot, rawNodeId)
|
|
10040
|
+
?? canonicalKnowledge360(snapshot.manifest, rawNodeId, snapshot.snapshotId);
|
|
10041
|
+
if (!result && !knowledge) {
|
|
9650
10042
|
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9651
10043
|
res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
|
|
9652
10044
|
return;
|
|
9653
10045
|
}
|
|
9654
10046
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9655
|
-
res.end(serializeJSON(result));
|
|
10047
|
+
res.end(serializeJSON(result ? { ...result, knowledge } : { version: 3, knowledge }));
|
|
9656
10048
|
}
|
|
9657
10049
|
catch (error) {
|
|
9658
10050
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -16494,17 +16886,127 @@ function buildNotebookTemplate(title, template) {
|
|
|
16494
16886
|
return JSON.stringify({ dqlnbVersion: 2, version: 1, title, cells }, null, 2);
|
|
16495
16887
|
}
|
|
16496
16888
|
/** Build a lineage graph from the project's blocks and semantic layer. */
|
|
16497
|
-
//
|
|
16498
|
-
|
|
16499
|
-
const
|
|
16889
|
+
// Cache per project + source fingerprint. A process can serve different roots,
|
|
16890
|
+
// and a time-only singleton previously returned the wrong graph after a change.
|
|
16891
|
+
const _lineageCache = new Map();
|
|
16500
16892
|
function buildProjectLineageGraph(projectRoot, semanticLayer) {
|
|
16501
|
-
|
|
16502
|
-
|
|
16503
|
-
|
|
16893
|
+
const signature = lineageSourceSignature(projectRoot);
|
|
16894
|
+
const cached = _lineageCache.get(projectRoot);
|
|
16895
|
+
if (cached?.signature === signature)
|
|
16896
|
+
return cached.graph;
|
|
16504
16897
|
const graph = buildProjectLineageGraphUncached(projectRoot, semanticLayer);
|
|
16505
|
-
_lineageCache
|
|
16898
|
+
_lineageCache.set(projectRoot, { signature, graph });
|
|
16506
16899
|
return graph;
|
|
16507
16900
|
}
|
|
16901
|
+
function lineageSourceSignature(projectRoot) {
|
|
16902
|
+
const hash = createHash('sha256');
|
|
16903
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, {}) ?? undefined;
|
|
16904
|
+
const inputs = new Set(collectInputFiles({ projectRoot, dbtManifestPath }));
|
|
16905
|
+
const emittedManifest = join(projectRoot, 'dql-manifest.json');
|
|
16906
|
+
if (existsSync(emittedManifest))
|
|
16907
|
+
inputs.add(emittedManifest);
|
|
16908
|
+
for (const input of [...inputs].sort()) {
|
|
16909
|
+
try {
|
|
16910
|
+
const stats = statSync(input);
|
|
16911
|
+
hash.update(`${input}\0${stats.size}\0${stats.mtimeMs}\n`);
|
|
16912
|
+
}
|
|
16913
|
+
catch {
|
|
16914
|
+
hash.update(`${input}\0missing\n`);
|
|
16915
|
+
}
|
|
16916
|
+
}
|
|
16917
|
+
return hash.digest('hex');
|
|
16918
|
+
}
|
|
16919
|
+
/** UI-008: bounded compiler-owned Domain Knowledge Capsule response. */
|
|
16920
|
+
function canonicalDomainKnowledge(manifest, domainId, snapshotId) {
|
|
16921
|
+
const graph = manifest.knowledgeGraph;
|
|
16922
|
+
if (!graph)
|
|
16923
|
+
return null;
|
|
16924
|
+
const capsule = graph.domainCapsules[domainId]
|
|
16925
|
+
?? Object.values(graph.domainCapsules).find((item) => item.domainId === domainId && !item.modelAreaId)
|
|
16926
|
+
?? Object.values(graph.domainCapsules).find((item) => item.id === domainId || item.name === domainId);
|
|
16927
|
+
const canonicalDomainId = capsule?.domainId
|
|
16928
|
+
?? Object.values(graph.objects ?? {}).find((item) => item.kind === 'domain' && (item.id === domainId || item.localId === domainId || item.aliases?.includes(domainId)))?.localId;
|
|
16929
|
+
if (!canonicalDomainId)
|
|
16930
|
+
return null;
|
|
16931
|
+
const objects = Object.values(graph.objects ?? {})
|
|
16932
|
+
.filter((item) => item.domainId === canonicalDomainId || item.id === `domain::${canonicalDomainId}`)
|
|
16933
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
16934
|
+
const objectIds = new Set(objects.map((item) => item.id));
|
|
16935
|
+
const edges = (graph.edges ?? [])
|
|
16936
|
+
.filter((edge) => objectIds.has(edge.from) || objectIds.has(edge.to))
|
|
16937
|
+
.slice(0, 1_500);
|
|
16938
|
+
const routes = graph.crossDomainRoutes.filter((route) => route.providerDomainId === canonicalDomainId || route.consumerDomainId === canonicalDomainId);
|
|
16939
|
+
const routeSummary = routes.reduce((counts, route) => {
|
|
16940
|
+
counts[route.state] = (counts[route.state] ?? 0) + 1;
|
|
16941
|
+
return counts;
|
|
16942
|
+
}, {});
|
|
16943
|
+
return {
|
|
16944
|
+
schemaVersion: graph.schemaVersion,
|
|
16945
|
+
snapshotId,
|
|
16946
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
16947
|
+
domainId: canonicalDomainId,
|
|
16948
|
+
capsule: capsule ?? graph.domainCapsules[canonicalDomainId],
|
|
16949
|
+
counts: {
|
|
16950
|
+
objects: objects.length,
|
|
16951
|
+
edges: edges.length,
|
|
16952
|
+
routes: routes.length,
|
|
16953
|
+
routeStates: routeSummary,
|
|
16954
|
+
},
|
|
16955
|
+
objects: objects.slice(0, 750),
|
|
16956
|
+
edges,
|
|
16957
|
+
routes,
|
|
16958
|
+
truncated: objects.length > 750 || edges.length >= 1_500,
|
|
16959
|
+
};
|
|
16960
|
+
}
|
|
16961
|
+
/** REL-003: qualified-object neighborhood with route policy and provenance. */
|
|
16962
|
+
function canonicalKnowledge360(manifest, rawId, snapshotId) {
|
|
16963
|
+
const graph = manifest.knowledgeGraph;
|
|
16964
|
+
if (!graph)
|
|
16965
|
+
return null;
|
|
16966
|
+
const graphObjects = graph.objects ?? {};
|
|
16967
|
+
const graphEdges = graph.edges ?? [];
|
|
16968
|
+
const exact = graphObjects[rawId];
|
|
16969
|
+
const matches = exact ? [exact] : Object.values(graphObjects).filter((item) => item.localId === rawId || item.aliases?.includes(rawId) || item.id.endsWith(`::${rawId}`));
|
|
16970
|
+
if (matches.length !== 1) {
|
|
16971
|
+
return matches.length > 1 ? {
|
|
16972
|
+
snapshotId,
|
|
16973
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
16974
|
+
ambiguous: true,
|
|
16975
|
+
candidates: matches.slice(0, 20).map((item) => ({ id: item.id, kind: item.kind, domainId: item.domainId })),
|
|
16976
|
+
} : null;
|
|
16977
|
+
}
|
|
16978
|
+
const focus = matches[0];
|
|
16979
|
+
const ids = new Set([focus.id]);
|
|
16980
|
+
let frontier = new Set([focus.id]);
|
|
16981
|
+
for (let depth = 0; depth < 2 && frontier.size > 0 && ids.size < 160; depth += 1) {
|
|
16982
|
+
const next = new Set();
|
|
16983
|
+
for (const edge of graphEdges) {
|
|
16984
|
+
if (frontier.has(edge.from) && !ids.has(edge.to))
|
|
16985
|
+
next.add(edge.to);
|
|
16986
|
+
if (frontier.has(edge.to) && !ids.has(edge.from))
|
|
16987
|
+
next.add(edge.from);
|
|
16988
|
+
}
|
|
16989
|
+
for (const id of next) {
|
|
16990
|
+
if (ids.size >= 160)
|
|
16991
|
+
break;
|
|
16992
|
+
ids.add(id);
|
|
16993
|
+
}
|
|
16994
|
+
frontier = next;
|
|
16995
|
+
}
|
|
16996
|
+
const objects = [...ids].flatMap((id) => graphObjects[id] ? [graphObjects[id]] : []);
|
|
16997
|
+
const edges = graphEdges.filter((edge) => ids.has(edge.from) && ids.has(edge.to)).slice(0, 500);
|
|
16998
|
+
const domains = new Set(objects.flatMap((item) => item.domainId ? [item.domainId] : []));
|
|
16999
|
+
const routes = graph.crossDomainRoutes.filter((route) => domains.has(route.providerDomainId) && domains.has(route.consumerDomainId));
|
|
17000
|
+
return {
|
|
17001
|
+
snapshotId,
|
|
17002
|
+
sourceFingerprint: graph.sourceFingerprint,
|
|
17003
|
+
focus,
|
|
17004
|
+
objects,
|
|
17005
|
+
edges,
|
|
17006
|
+
routes,
|
|
17007
|
+
truncated: ids.size >= 160 || edges.length >= 500,
|
|
17008
|
+
};
|
|
17009
|
+
}
|
|
16508
17010
|
function buildProjectLineageGraphUncached(projectRoot, semanticLayer) {
|
|
16509
17011
|
const manifestPath = join(projectRoot, 'dql-manifest.json');
|
|
16510
17012
|
if (existsSync(manifestPath)) {
|
|
@@ -18167,6 +18669,13 @@ async function buildAgentSchemaContextFromCatalog(projectRoot, question, prepare
|
|
|
18167
18669
|
}
|
|
18168
18670
|
/** How long a stored live-warehouse schema snapshot is trusted before a rescan (P6). */
|
|
18169
18671
|
const RUNTIME_SNAPSHOT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
18672
|
+
// A resolver compares at most 12 compact cards and never performs tool calls;
|
|
18673
|
+
// ten seconds is the full allowance, not the start of another planning loop.
|
|
18674
|
+
const AGENT_MEANING_TIMEOUT_MS = 10_000;
|
|
18675
|
+
export function boundedAgentMeaningSignal(signal, timeoutMs = AGENT_MEANING_TIMEOUT_MS) {
|
|
18676
|
+
const timeout = AbortSignal.timeout(Math.max(1, timeoutMs));
|
|
18677
|
+
return signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
18678
|
+
}
|
|
18170
18679
|
/**
|
|
18171
18680
|
* Whether the project's stored live-schema snapshot is missing or older than the
|
|
18172
18681
|
* freshness window (P6). Used to force a fresh information_schema scan even when the
|
|
@@ -18202,7 +18711,6 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
|
|
|
18202
18711
|
name: column.name,
|
|
18203
18712
|
type: column.type,
|
|
18204
18713
|
description: column.description,
|
|
18205
|
-
sampleValues: column.sampleValues?.slice(0, 8),
|
|
18206
18714
|
})),
|
|
18207
18715
|
})),
|
|
18208
18716
|
});
|
|
@@ -19135,25 +19643,29 @@ export function shouldAugmentAgentRuntimeSchema(question, questionPlan) {
|
|
|
19135
19643
|
const plannedCompositeMetric = (questionPlan?.metricTerms?.length ?? 0) > 0 && plannedConcepts.size >= 2;
|
|
19136
19644
|
return explicitJoin || referencesPriorRows || plannedCompositeMetric || (multiEntity && (wantsMetric || wantsDetail));
|
|
19137
19645
|
}
|
|
19138
|
-
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
|
|
19646
|
+
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection, searchSafeColumns) {
|
|
19139
19647
|
const searchTerms = extractAgentValueSearchTerms(question);
|
|
19140
19648
|
if (schemaContext.length === 0 || searchTerms.length === 0)
|
|
19141
19649
|
return schemaContext;
|
|
19142
19650
|
const matches = new Map();
|
|
19143
|
-
|
|
19651
|
+
const probes = rankAgentValueProbeColumns(schemaContext, searchSafeColumns).slice(0, 3).map(async (candidate) => {
|
|
19144
19652
|
try {
|
|
19145
|
-
const result = await executor.executeQuery(buildAgentValueProbeSql(candidate.table, candidate.column.name, searchTerms, connection), [], runtimeVariables({}), connection);
|
|
19146
|
-
const values = uniqueStrings(result.rows.flatMap(valueProbeRowValues)).slice(0,
|
|
19147
|
-
|
|
19148
|
-
continue;
|
|
19149
|
-
const tableMatches = matches.get(candidate.table.relation) ?? new Map();
|
|
19150
|
-
tableMatches.set(candidate.column.name, values);
|
|
19151
|
-
matches.set(candidate.table.relation, tableMatches);
|
|
19653
|
+
const result = await withAgentValueProbeTimeout(executor.executeQuery(buildAgentValueProbeSql(candidate.table, candidate.column.name, searchTerms, connection), [], runtimeVariables({}), connection), 2_000);
|
|
19654
|
+
const values = uniqueStrings(result.rows.flatMap(valueProbeRowValues)).slice(0, 25);
|
|
19655
|
+
return values.length > 0 ? { candidate, values } : undefined;
|
|
19152
19656
|
}
|
|
19153
19657
|
catch {
|
|
19154
19658
|
// Value probes are advisory. Unsupported casts, privileges, and large-table
|
|
19155
19659
|
// failures should not block the metadata-backed answer path.
|
|
19660
|
+
return undefined;
|
|
19156
19661
|
}
|
|
19662
|
+
});
|
|
19663
|
+
for (const match of await Promise.all(probes)) {
|
|
19664
|
+
if (!match)
|
|
19665
|
+
continue;
|
|
19666
|
+
const tableMatches = matches.get(match.candidate.table.relation) ?? new Map();
|
|
19667
|
+
tableMatches.set(match.candidate.column.name, match.values);
|
|
19668
|
+
matches.set(match.candidate.table.relation, tableMatches);
|
|
19157
19669
|
}
|
|
19158
19670
|
if (matches.size === 0)
|
|
19159
19671
|
return schemaContext;
|
|
@@ -19166,12 +19678,27 @@ async function enrichAgentSchemaContextWithValueMatches(question, schemaContext,
|
|
|
19166
19678
|
columns: table.columns.map((column) => {
|
|
19167
19679
|
const sampleValues = tableMatches.get(column.name);
|
|
19168
19680
|
return sampleValues?.length
|
|
19169
|
-
? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0,
|
|
19681
|
+
? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0, 25) }
|
|
19170
19682
|
: column;
|
|
19171
19683
|
}),
|
|
19172
19684
|
};
|
|
19173
19685
|
});
|
|
19174
19686
|
}
|
|
19687
|
+
async function withAgentValueProbeTimeout(promise, timeoutMs) {
|
|
19688
|
+
let timer;
|
|
19689
|
+
try {
|
|
19690
|
+
return await Promise.race([
|
|
19691
|
+
promise,
|
|
19692
|
+
new Promise((_resolve, reject) => {
|
|
19693
|
+
timer = setTimeout(() => reject(new Error('VALUE_LOOKUP_TIMEOUT')), timeoutMs);
|
|
19694
|
+
}),
|
|
19695
|
+
]);
|
|
19696
|
+
}
|
|
19697
|
+
finally {
|
|
19698
|
+
if (timer)
|
|
19699
|
+
clearTimeout(timer);
|
|
19700
|
+
}
|
|
19701
|
+
}
|
|
19175
19702
|
function scoreAgentSchemaTable(table, tokens) {
|
|
19176
19703
|
let score = 0;
|
|
19177
19704
|
const relationTokens = agentSchemaTokens(`${table.schema ?? ''} ${table.name} ${table.relation}`);
|
|
@@ -19252,11 +19779,37 @@ function scoreAgentValueProbeTable(table) {
|
|
|
19252
19779
|
}
|
|
19253
19780
|
return Math.min(score, 18);
|
|
19254
19781
|
}
|
|
19255
|
-
|
|
19782
|
+
/**
|
|
19783
|
+
* Resolve the project-admin boundary for live value lookup. An absent/malformed
|
|
19784
|
+
* policy is deliberately disabled; a broad table or wildcard cannot make an
|
|
19785
|
+
* unknown column search-safe.
|
|
19786
|
+
*/
|
|
19787
|
+
export function resolveAgentRuntimeValueGrounding(config) {
|
|
19788
|
+
const configured = config.agent?.runtimeValueGrounding;
|
|
19789
|
+
if (configured?.mode !== 'safe_automatic') {
|
|
19790
|
+
return { mode: 'disabled', searchSafeColumns: new Set() };
|
|
19791
|
+
}
|
|
19792
|
+
const searchSafeColumns = new Set((configured.searchSafeColumns ?? [])
|
|
19793
|
+
.map(normalizeAgentSafeColumnReference)
|
|
19794
|
+
.filter((value) => value.split('.').length >= 2 && !value.includes('*')));
|
|
19795
|
+
return searchSafeColumns.size > 0
|
|
19796
|
+
? { mode: 'safe_automatic', searchSafeColumns }
|
|
19797
|
+
: { mode: 'disabled', searchSafeColumns };
|
|
19798
|
+
}
|
|
19799
|
+
function normalizeAgentSafeColumnReference(value) {
|
|
19800
|
+
return value.trim().replace(/[`"\[\]]/g, '').toLowerCase();
|
|
19801
|
+
}
|
|
19802
|
+
function isExplicitlySearchSafeAgentColumn(table, column, searchSafeColumns) {
|
|
19803
|
+
const qualified = normalizeAgentSafeColumnReference(`${table.relation}.${column.name}`);
|
|
19804
|
+
const relationParts = table.relation.split('.').filter(Boolean);
|
|
19805
|
+
const shortQualified = normalizeAgentSafeColumnReference(`${relationParts.slice(-1)[0] ?? table.relation}.${column.name}`);
|
|
19806
|
+
return searchSafeColumns.has(qualified) || searchSafeColumns.has(shortQualified);
|
|
19807
|
+
}
|
|
19808
|
+
function rankAgentValueProbeColumns(schemaContext, searchSafeColumns) {
|
|
19256
19809
|
const ranked = [];
|
|
19257
19810
|
for (const table of schemaContext) {
|
|
19258
19811
|
for (const column of table.columns) {
|
|
19259
|
-
if (!isAgentValueProbeColumn(column))
|
|
19812
|
+
if (!isAgentValueProbeColumn(column) || !isExplicitlySearchSafeAgentColumn(table, column, searchSafeColumns))
|
|
19260
19813
|
continue;
|
|
19261
19814
|
ranked.push({
|
|
19262
19815
|
table,
|
|
@@ -19277,9 +19830,18 @@ function scoreAgentValueProbeColumn(table, column) {
|
|
|
19277
19830
|
score += 3;
|
|
19278
19831
|
return score;
|
|
19279
19832
|
}
|
|
19280
|
-
function isAgentValueProbeColumn(column) {
|
|
19833
|
+
export function isAgentValueProbeColumn(column) {
|
|
19281
19834
|
const name = column.name.toLowerCase();
|
|
19282
|
-
|
|
19835
|
+
// Tokenize underscore/camel names before applying the hard deny-list. This is
|
|
19836
|
+
// intentionally independent of an allowlist: secrets and free-text payloads
|
|
19837
|
+
// can never be probed through automatic grounding.
|
|
19838
|
+
const normalizedName = column.name
|
|
19839
|
+
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
|
|
19840
|
+
.replace(/[_-]+/g, ' ')
|
|
19841
|
+
.toLowerCase();
|
|
19842
|
+
if (/\b(password|secret|token|credential|hash|salt|notes?|comments?|description|message|body|payload|content)\b/.test(normalizedName))
|
|
19843
|
+
return false;
|
|
19844
|
+
if (/\bemail\b/.test(normalizedName))
|
|
19283
19845
|
return false;
|
|
19284
19846
|
if (!hasAgentSchemaToken(name, [
|
|
19285
19847
|
'account',
|
|
@@ -19317,15 +19879,22 @@ export function buildAgentValueProbeSql(table, column, searchTerms, connection)
|
|
|
19317
19879
|
const relation = quoteAgentRelation(table.relation, connection);
|
|
19318
19880
|
const identifier = quoteAgentIdentifier(column, connection);
|
|
19319
19881
|
const castValue = `LOWER(CAST(${identifier} AS ${agentTextCastType(connection.driver)}))`;
|
|
19320
|
-
const predicates = searchTerms
|
|
19321
|
-
.
|
|
19322
|
-
.
|
|
19882
|
+
const predicates = uniqueStrings(searchTerms.flatMap((term) => {
|
|
19883
|
+
const normalized = term.toLowerCase().replace(/\s+/g, ' ').trim();
|
|
19884
|
+
const tokens = normalized.split(' ').filter((token) => token.length >= 4);
|
|
19885
|
+
return [
|
|
19886
|
+
`${castValue} = ${sqlStringLiteral(normalized)}`,
|
|
19887
|
+
...tokens.slice(0, 2).map((token) => `${castValue} LIKE ${sqlStringLiteral(`${escapeSqlLike(token)}%`)} ESCAPE '\\'`),
|
|
19888
|
+
];
|
|
19889
|
+
}))
|
|
19890
|
+
.slice(0, 8)
|
|
19891
|
+
.map((predicate) => predicate)
|
|
19323
19892
|
.join(' OR ');
|
|
19324
19893
|
return [
|
|
19325
19894
|
`SELECT DISTINCT CAST(${identifier} AS ${agentTextCastType(connection.driver)}) AS value`,
|
|
19326
19895
|
`FROM ${relation}`,
|
|
19327
19896
|
`WHERE ${identifier} IS NOT NULL AND (${predicates})`,
|
|
19328
|
-
'LIMIT
|
|
19897
|
+
'LIMIT 25',
|
|
19329
19898
|
].join('\n');
|
|
19330
19899
|
}
|
|
19331
19900
|
function agentTextCastType(driver) {
|
|
@@ -19409,6 +19978,9 @@ export function extractAgentValueSearchTerms(question) {
|
|
|
19409
19978
|
for (const match of question.matchAll(/\b(?:for|named|called|only|where|customer|user|account|product)\s+([A-Za-z0-9@._-]+(?:\s+[A-Za-z0-9@._-]+){0,3})/gi)) {
|
|
19410
19979
|
terms.push(match[1]);
|
|
19411
19980
|
}
|
|
19981
|
+
for (const match of question.matchAll(/\b(?:than|versus|vs\.?)\s+([A-Za-z0-9@._-]+(?:\s+[A-Za-z0-9@._-]+){0,3})/gi)) {
|
|
19982
|
+
terms.push(match[1]);
|
|
19983
|
+
}
|
|
19412
19984
|
return uniqueStrings(terms
|
|
19413
19985
|
.map(cleanAgentValueSearchTerm)
|
|
19414
19986
|
.filter((term) => term.length >= 3 && !AGENT_VALUE_SEARCH_STOP_PHRASES.has(term.toLowerCase()))).slice(0, 6);
|
|
@@ -19419,6 +19991,7 @@ function cleanAgentValueSearchTerm(term) {
|
|
|
19419
19991
|
.replace(/\s+/g, ' ')
|
|
19420
19992
|
.trim()
|
|
19421
19993
|
.replace(/^(?:account|customer|member|named|called|product|sku|subscriber|user)\s+/i, '')
|
|
19994
|
+
.replace(/\s+\b(?:got|get|gets|bought|buy|buys|purchased|purchase|purchases|spent|spend|spends|has|have|with)\b.*$/i, '')
|
|
19422
19995
|
.replace(/\s+\b(?:last|next|this)\b.*$/i, '')
|
|
19423
19996
|
.replace(/\s+\b(?:last|this)\s+(?:day|week|month|quarter|year)\b.*$/i, '')
|
|
19424
19997
|
.replace(/\s+\b(?:daily|weekly|monthly|quarterly|yearly)\b.*$/i, '')
|