@duckcodeailabs/dql-agent 1.6.31 → 1.6.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/dist/agent-run-engine.d.ts +42 -6
  2. package/dist/agent-run-engine.d.ts.map +1 -1
  3. package/dist/agent-run-engine.js +133 -79
  4. package/dist/agent-run-engine.js.map +1 -1
  5. package/dist/agent-run-gates.d.ts.map +1 -1
  6. package/dist/agent-run-gates.js +72 -5
  7. package/dist/agent-run-gates.js.map +1 -1
  8. package/dist/agent-run-planner.d.ts.map +1 -1
  9. package/dist/agent-run-planner.js +13 -0
  10. package/dist/agent-run-planner.js.map +1 -1
  11. package/dist/answer-loop.d.ts +110 -8
  12. package/dist/answer-loop.d.ts.map +1 -1
  13. package/dist/answer-loop.js +1679 -1299
  14. package/dist/answer-loop.js.map +1 -1
  15. package/dist/answer-shape.d.ts +16 -0
  16. package/dist/answer-shape.d.ts.map +1 -0
  17. package/dist/answer-shape.js +85 -0
  18. package/dist/answer-shape.js.map +1 -0
  19. package/dist/cascade/budgets.d.ts +101 -0
  20. package/dist/cascade/budgets.d.ts.map +1 -0
  21. package/dist/cascade/budgets.js +184 -0
  22. package/dist/cascade/budgets.js.map +1 -0
  23. package/dist/cascade/cascade.d.ts +86 -0
  24. package/dist/cascade/cascade.d.ts.map +1 -0
  25. package/dist/cascade/cascade.js +115 -0
  26. package/dist/cascade/cascade.js.map +1 -0
  27. package/dist/cascade/route-policy.d.ts +14 -0
  28. package/dist/cascade/route-policy.d.ts.map +1 -0
  29. package/dist/cascade/route-policy.js +48 -0
  30. package/dist/cascade/route-policy.js.map +1 -0
  31. package/dist/cascade/triage.d.ts +20 -0
  32. package/dist/cascade/triage.d.ts.map +1 -0
  33. package/dist/cascade/triage.js +25 -0
  34. package/dist/cascade/triage.js.map +1 -0
  35. package/dist/conversation/rolling-summary.d.ts +13 -0
  36. package/dist/conversation/rolling-summary.d.ts.map +1 -0
  37. package/dist/conversation/rolling-summary.js +43 -0
  38. package/dist/conversation/rolling-summary.js.map +1 -0
  39. package/dist/conversation/session-store.d.ts +110 -0
  40. package/dist/conversation/session-store.d.ts.map +1 -0
  41. package/dist/conversation/session-store.js +387 -0
  42. package/dist/conversation/session-store.js.map +1 -0
  43. package/dist/conversation/snapshot.d.ts +59 -0
  44. package/dist/conversation/snapshot.d.ts.map +1 -0
  45. package/dist/conversation/snapshot.js +153 -0
  46. package/dist/conversation/snapshot.js.map +1 -0
  47. package/dist/conversation/working-state.d.ts +49 -0
  48. package/dist/conversation/working-state.d.ts.map +1 -0
  49. package/dist/conversation/working-state.js +249 -0
  50. package/dist/conversation/working-state.js.map +1 -0
  51. package/dist/embeddings/provider.d.ts +62 -0
  52. package/dist/embeddings/provider.d.ts.map +1 -1
  53. package/dist/embeddings/provider.js +98 -0
  54. package/dist/embeddings/provider.js.map +1 -1
  55. package/dist/grounding/context-ledger.d.ts +30 -0
  56. package/dist/grounding/context-ledger.d.ts.map +1 -0
  57. package/dist/grounding/context-ledger.js +54 -0
  58. package/dist/grounding/context-ledger.js.map +1 -0
  59. package/dist/grounding/regrounding.d.ts +24 -0
  60. package/dist/grounding/regrounding.d.ts.map +1 -0
  61. package/dist/grounding/regrounding.js +219 -0
  62. package/dist/grounding/regrounding.js.map +1 -0
  63. package/dist/grounding/value-index.d.ts +32 -0
  64. package/dist/grounding/value-index.d.ts.map +1 -0
  65. package/dist/grounding/value-index.js +67 -0
  66. package/dist/grounding/value-index.js.map +1 -0
  67. package/dist/index.d.ts +45 -14
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +102 -11
  70. package/dist/index.js.map +1 -1
  71. package/dist/intent-controller.d.ts +7 -0
  72. package/dist/intent-controller.d.ts.map +1 -1
  73. package/dist/intent-controller.js +22 -1
  74. package/dist/intent-controller.js.map +1 -1
  75. package/dist/kg/build.d.ts.map +1 -1
  76. package/dist/kg/build.js +44 -17
  77. package/dist/kg/build.js.map +1 -1
  78. package/dist/kg/sqlite-fts.d.ts +3 -1
  79. package/dist/kg/sqlite-fts.d.ts.map +1 -1
  80. package/dist/kg/sqlite-fts.js +16 -2
  81. package/dist/kg/sqlite-fts.js.map +1 -1
  82. package/dist/kg/types.d.ts +23 -1
  83. package/dist/kg/types.d.ts.map +1 -1
  84. package/dist/memory/fts-query.d.ts +7 -0
  85. package/dist/memory/fts-query.d.ts.map +1 -0
  86. package/dist/memory/fts-query.js +36 -0
  87. package/dist/memory/fts-query.js.map +1 -0
  88. package/dist/memory/sqlite-memory.d.ts.map +1 -1
  89. package/dist/memory/sqlite-memory.js +1 -30
  90. package/dist/memory/sqlite-memory.js.map +1 -1
  91. package/dist/metadata/analysis-planner.d.ts +23 -0
  92. package/dist/metadata/analysis-planner.d.ts.map +1 -1
  93. package/dist/metadata/analysis-planner.js +432 -2
  94. package/dist/metadata/analysis-planner.js.map +1 -1
  95. package/dist/metadata/block-fit.d.ts +24 -0
  96. package/dist/metadata/block-fit.d.ts.map +1 -0
  97. package/dist/metadata/block-fit.js +373 -0
  98. package/dist/metadata/block-fit.js.map +1 -0
  99. package/dist/metadata/catalog.d.ts +72 -3
  100. package/dist/metadata/catalog.d.ts.map +1 -1
  101. package/dist/metadata/catalog.js +1452 -72
  102. package/dist/metadata/catalog.js.map +1 -1
  103. package/dist/metadata/drafts.d.ts +25 -0
  104. package/dist/metadata/drafts.d.ts.map +1 -1
  105. package/dist/metadata/drafts.js +112 -2
  106. package/dist/metadata/drafts.js.map +1 -1
  107. package/dist/metadata/metric-match.d.ts +55 -6
  108. package/dist/metadata/metric-match.d.ts.map +1 -1
  109. package/dist/metadata/metric-match.js +158 -13
  110. package/dist/metadata/metric-match.js.map +1 -1
  111. package/dist/metadata/sql-context-validation.d.ts +13 -1
  112. package/dist/metadata/sql-context-validation.d.ts.map +1 -1
  113. package/dist/metadata/sql-context-validation.js +172 -19
  114. package/dist/metadata/sql-context-validation.js.map +1 -1
  115. package/dist/metadata/sql-retrieval.d.ts +3 -2
  116. package/dist/metadata/sql-retrieval.d.ts.map +1 -1
  117. package/dist/metadata/sql-retrieval.js +6 -4
  118. package/dist/metadata/sql-retrieval.js.map +1 -1
  119. package/dist/providers/claude.d.ts +2 -1
  120. package/dist/providers/claude.d.ts.map +1 -1
  121. package/dist/providers/claude.js +95 -0
  122. package/dist/providers/claude.js.map +1 -1
  123. package/dist/providers/index.d.ts +1 -1
  124. package/dist/providers/index.d.ts.map +1 -1
  125. package/dist/providers/index.js.map +1 -1
  126. package/dist/providers/openai.d.ts +2 -1
  127. package/dist/providers/openai.d.ts.map +1 -1
  128. package/dist/providers/openai.js +143 -0
  129. package/dist/providers/openai.js.map +1 -1
  130. package/dist/providers/tool-output.d.ts +2 -0
  131. package/dist/providers/tool-output.d.ts.map +1 -0
  132. package/dist/providers/tool-output.js +22 -0
  133. package/dist/providers/tool-output.js.map +1 -0
  134. package/dist/providers/types.d.ts +23 -0
  135. package/dist/providers/types.d.ts.map +1 -1
  136. package/dist/providers/types.js.map +1 -1
  137. package/dist/semantic-bridge/compose.d.ts +89 -0
  138. package/dist/semantic-bridge/compose.d.ts.map +1 -0
  139. package/dist/semantic-bridge/compose.js +528 -0
  140. package/dist/semantic-bridge/compose.js.map +1 -0
  141. package/dist/semantic-bridge/member-select.d.ts +23 -0
  142. package/dist/semantic-bridge/member-select.d.ts.map +1 -0
  143. package/dist/semantic-bridge/member-select.js +106 -0
  144. package/dist/semantic-bridge/member-select.js.map +1 -0
  145. package/dist/skills/defaults.d.ts.map +1 -1
  146. package/dist/skills/defaults.js +5 -0
  147. package/dist/skills/defaults.js.map +1 -1
  148. package/dist/synthesize.d.ts +26 -0
  149. package/dist/synthesize.d.ts.map +1 -1
  150. package/dist/synthesize.js +72 -3
  151. package/dist/synthesize.js.map +1 -1
  152. package/dist/tools/registry.d.ts +15 -0
  153. package/dist/tools/registry.d.ts.map +1 -0
  154. package/dist/tools/registry.js +692 -0
  155. package/dist/tools/registry.js.map +1 -0
  156. package/dist/trust/stamp.d.ts +26 -0
  157. package/dist/trust/stamp.d.ts.map +1 -0
  158. package/dist/trust/stamp.js +33 -0
  159. package/dist/trust/stamp.js.map +1 -0
  160. package/package.json +4 -4
@@ -10,12 +10,14 @@ import { createHash } from 'node:crypto';
10
10
  import { mkdirSync, readFileSync } from 'node:fs';
11
11
  import { createRequire } from 'node:module';
12
12
  import { dirname, join } from 'node:path';
13
- import { buildManifest, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, } from '@duckcodeailabs/dql-core';
13
+ import { buildManifest, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, extractColumnLineage, normalizeDqlArtifactReference, composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
14
14
  import { buildKGFromManifest, buildKGFromSemanticLayer } from '../kg/build.js';
15
15
  import { buildBlockBusinessFingerprint, buildBlockSqlFingerprints } from './block-fingerprints.js';
16
16
  import { buildAnalysisQuestionPlan, certifiedApplicabilityForObject, scoreAllowedSqlRelationWithAnalysisPlan, scoreMetadataObjectWithAnalysisPlan, sortAllowedSqlContextForAnalysisPlan, } from './analysis-planner.js';
17
17
  import { extractSimpleSelectShape, sourceSqlShapeColumns } from './sql-shape.js';
18
+ import { buildRuntimeValueIndex, normalizeValueIndexText, } from '../grounding/value-index.js';
18
19
  import { grainMatches, requestedGrainFromPlan, } from './grain-gate.js';
20
+ import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit, } from './block-fit.js';
19
21
  import { retrieveScopedHints } from '../hints/retrieval.js';
20
22
  const require = createRequire(import.meta.url);
21
23
  let databaseCtor = null;
@@ -23,8 +25,11 @@ function loadDatabase() {
23
25
  databaseCtor ??= require('better-sqlite3');
24
26
  return databaseCtor;
25
27
  }
28
+ const SCHEMA_SHAPE_CACHE_LIMIT = 64;
29
+ const schemaShapeCandidateCache = new Map();
26
30
  const OBJECT_PRIORITY = {
27
31
  dql_block: 1,
32
+ dql_block_output: 1.5,
28
33
  semantic_metric: 2,
29
34
  dql_term: 3,
30
35
  business_view: 4,
@@ -36,11 +41,14 @@ const OBJECT_PRIORITY = {
36
41
  dbt_source: 10,
37
42
  dbt_column: 11,
38
43
  warehouse_table: 12,
44
+ warehouse_column: 12.5,
45
+ runtime_value: 12.2,
39
46
  notebook: 13,
40
47
  dashboard: 14,
41
48
  app: 15,
42
49
  domain: 16,
43
50
  };
51
+ const COLUMN_OBJECT_TYPES = new Set(['dbt_column', 'warehouse_column', 'runtime_column', 'runtime_value']);
44
52
  export function defaultMetadataPath(projectRoot) {
45
53
  return join(projectRoot, '.dql', 'cache', 'metadata.sqlite');
46
54
  }
@@ -104,8 +112,54 @@ export async function buildLocalContextPack(projectRoot, request) {
104
112
  const mode = request.mode ?? 'question';
105
113
  const followUp = normalizeFollowUpContext(request.followUp);
106
114
  const questionPlan = buildAnalysisQuestionPlan(request.question, followUp ?? undefined);
115
+ // ── Conversation-aware context reuse ──────────────────────────────────
116
+ // The prior turn's pack is fuel: a filter/limit-only REFINEMENT re-stamps it
117
+ // (skipping FTS fan-out, both ranking passes, route planning, and the
118
+ // fit-confirm LLM); CONTINUATION/RETURN seed its objects into ranking as
119
+ // candidates only; a SHIFT ignores it. A metadata fingerprint mismatch
120
+ // always disqualifies reuse.
121
+ const reusePolicy = request.reusePolicy ?? 'seed';
122
+ const priorPack = reusePolicy !== 'off'
123
+ && request.priorContextPackId
124
+ && request.conversationTopicRelation
125
+ && request.conversationTopicRelation !== 'shift'
126
+ ? catalog.getContextPack(request.priorContextPackId)
127
+ : null;
128
+ const priorPackFresh = Boolean(priorPack && priorPack.freshness.fingerprint === catalog.state('fingerprint'));
129
+ if (priorPack
130
+ && priorPackFresh
131
+ && request.conversationTopicRelation === 'refinement'
132
+ && isFilterOnlyRefinement(priorPack.questionPlan, questionPlan)) {
133
+ // Route commitment: the prior route decision was already fit-validated for
134
+ // this grain/measures/dimensions; only filters/limit/timeframe changed.
135
+ const reusedPayload = {
136
+ ...priorPack,
137
+ question: request.question,
138
+ questionPlan,
139
+ followUp: followUp ?? undefined,
140
+ retrievalDiagnostics: {
141
+ ...priorPack.retrievalDiagnostics,
142
+ strategy: 'reused_pack_refinement',
143
+ },
144
+ freshness: {
145
+ catalogPath: defaultMetadataPath(projectRoot),
146
+ builtAt: catalog.state('built_at'),
147
+ fingerprint: catalog.state('fingerprint'),
148
+ },
149
+ };
150
+ delete reusedPayload.id;
151
+ const reusedId = catalog.insertContextPack(reusedPayload);
152
+ return { ...reusedPayload, id: reusedId };
153
+ }
154
+ const priorSeedObjects = priorPack && priorPackFresh
155
+ && (request.conversationTopicRelation === 'continuation'
156
+ || request.conversationTopicRelation === 'refinement'
157
+ || request.conversationTopicRelation === 'return')
158
+ ? priorPack.objects.slice(0, 24)
159
+ : [];
107
160
  const runtimeSnapshot = request.runtimeSchemaSnapshot ?? catalog.latestRuntimeSchemaSnapshot();
108
161
  const runtimeObjects = runtimeSnapshot ? runtimeSchemaObjects(runtimeSnapshot) : [];
162
+ const runtimeValueObjects = runtimeValueMatchObjects(catalog.searchRuntimeValues(metadataValueSearchTerms(request.question, questionPlan, followUp), 32));
109
163
  const selectedObjects = selectedContextObjects(request.selectedContext);
110
164
  const followUpObjects = followUpContextObjects(followUp);
111
165
  const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
@@ -118,38 +172,56 @@ export async function buildLocalContextPack(projectRoot, request) {
118
172
  objectTypes: request.objectTypes,
119
173
  limit: Math.max(request.limit ?? 80, 20),
120
174
  })));
121
- const schemaShapeCandidates = schemaShapeCandidateObjects(catalog, questionPlan, request, runtimeObjects);
175
+ const schemaShapeCandidates = schemaShapeCandidateObjects(catalog, questionPlan, request, mergeObjects([...runtimeObjects, ...runtimeValueObjects]));
122
176
  const schemaShapeObjects = schemaShapeCandidates.map((candidate) => candidate.object);
123
177
  const exact = request.focusObjectKey ? catalog.getObject(request.focusObjectKey) : null;
124
178
  const ranked = rankMetadataObjects({
125
179
  rows: mergeObjects(exact
126
- ? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...selectedObjects]
127
- : [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...selectedObjects]),
180
+ ? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]
181
+ : [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]),
128
182
  question: searchQueries.join(' '),
129
183
  questionPlan,
130
184
  limit: request.limit ?? 80,
131
185
  });
132
- const selected = mergeObjects([...followUpSourceObjects, ...followUpObjects, ...ranked.selected]);
186
+ // Advisory 'contextual' carry: the prior turn's block competes on rank (it is
187
+ // already in the candidate rows above) but is never FORCED into selection or
188
+ // allowed to become the derived focus of a possibly-new-topic question.
189
+ const selected = followUp?.kind === 'contextual'
190
+ ? ranked.selected
191
+ : mergeObjects([...followUpSourceObjects, ...followUpObjects, ...ranked.selected]);
133
192
  const focusObjectKey = request.focusObjectKey ?? selected[0]?.objectKey ?? null;
134
193
  const edgeWalk = catalog.edgesForKeys(selected.map((row) => row.objectKey), 3);
135
194
  const edgeObjectKeys = Array.from(new Set(edgeWalk.flatMap((edge) => [edge.fromKey, edge.toKey])));
136
195
  const graphObjects = catalog.getObjectsByKeys(edgeObjectKeys);
137
196
  const rankedObjects = rankMetadataObjects({
138
- rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...selectedObjects]),
197
+ rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects]),
139
198
  question: searchQueries.join(' '),
140
199
  questionPlan,
141
200
  limit: request.limit ?? 120,
142
201
  }).selected;
143
202
  const sqlParentObjects = sqlParentObjectsForSelectedColumns(rankedObjects, mergeObjects([...graphObjects, ...runtimeObjects]), questionPlan);
144
- const objects = mergeObjects([...rankedObjects, ...sqlParentObjects]);
203
+ // Deep-research over a SMALL catalog: skip top-k pruning and hand the model the
204
+ // entire relation set ("send everything, let the agent decide"). Only when the
205
+ // whole catalog fits comfortably in context — otherwise we keep ranked selection.
206
+ const fullCatalogObjects = request.strictness === 'exploratory'
207
+ ? collectFullCatalogObjects(catalog)
208
+ : undefined;
209
+ const usedFullCatalog = Boolean(fullCatalogObjects);
210
+ const objects = fullCatalogObjects
211
+ ? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects])
212
+ : mergeObjects([...rankedObjects, ...sqlParentObjects]);
145
213
  const objectKeys = objects.map((row) => row.objectKey);
214
+ const contextEdges = mergeMetadataEdges([
215
+ ...edgeWalk,
216
+ ...catalog.edgesForKeys(objectKeys, 2),
217
+ ]);
146
218
  const queryRuns = catalog.queryRunsForObjectKeys(objectKeys, 20);
147
219
  const diagnostics = catalog.diagnostics();
148
220
  const warnings = buildWarnings(diagnostics, objects);
149
221
  const trustLabel = deriveTrust(objects);
150
- const citations = buildCitations(objects, edgeWalk);
151
- const evidenceSummaries = buildEvidenceSummaries(objects, edgeWalk, queryRuns, diagnostics);
152
- const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, edgeWalk), questionPlan);
222
+ const citations = buildCitations(objects, contextEdges);
223
+ const evidenceSummaries = buildEvidenceSummaries(objects, contextEdges, queryRuns, diagnostics);
224
+ const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, contextEdges), questionPlan);
153
225
  const selectedRelations = allowedSqlContext.relations.slice(0, 24).map((relation, index) => {
154
226
  const scored = scoreAllowedSqlRelationWithAnalysisPlan(relation, questionPlan);
155
227
  return {
@@ -162,7 +234,7 @@ export async function buildLocalContextPack(projectRoot, request) {
162
234
  rank: index + 1,
163
235
  };
164
236
  });
165
- const selectedJoinPaths = buildSelectedJoinPaths(allowedSqlContext);
237
+ const selectedJoinPaths = buildSelectedJoinPaths(allowedSqlContext, contextEdges);
166
238
  const evidenceRoles = buildEvidenceRoles(objects, queryRuns);
167
239
  const reranked = rankMetadataObjects({
168
240
  rows: mergeObjects([...searchRows, ...schemaShapeObjects, ...objects]),
@@ -172,7 +244,7 @@ export async function buildLocalContextPack(projectRoot, request) {
172
244
  });
173
245
  const conflicts = buildCandidateConflicts(reranked.ranked);
174
246
  const compileConflicts = catalog.compileConflicts();
175
- const routeDecision = planContextPackRoute({
247
+ const routeDecision = withMetadataTrustLabelInfo(await planContextPackRoute({
176
248
  request,
177
249
  objects,
178
250
  allowedSqlContext,
@@ -182,6 +254,12 @@ export async function buildLocalContextPack(projectRoot, request) {
182
254
  questionPlan,
183
255
  compileConflicts,
184
256
  rankedObjects: reranked.ranked,
257
+ }));
258
+ const certifiedCandidateFits = buildCertifiedCandidateFitDiagnostics({
259
+ request,
260
+ objects,
261
+ questionPlan,
262
+ routeDecision,
185
263
  });
186
264
  // Approved scoped correction hints — folded in AFTER certified routing so
187
265
  // they never override a certified answer. On the `certified` route we still
@@ -199,8 +277,9 @@ export async function buildLocalContextPack(projectRoot, request) {
199
277
  mode,
200
278
  questionPlan,
201
279
  trustLabel,
280
+ trustLabelInfo: metadataTrustLabelInfo(trustLabel),
202
281
  objects,
203
- edges: edgeWalk,
282
+ edges: contextEdges,
204
283
  queryRuns,
205
284
  citations,
206
285
  evidenceSummaries,
@@ -213,7 +292,7 @@ export async function buildLocalContextPack(projectRoot, request) {
213
292
  appliedHints: hintResult.applied,
214
293
  hintConflicts: hintResult.conflicts,
215
294
  retrievalDiagnostics: {
216
- strategy: 'sqlite_fts',
295
+ strategy: usedFullCatalog ? 'full_catalog' : 'sqlite_fts',
217
296
  selectedObjects: objects.length,
218
297
  selectedEvidence: reranked.ranked.slice(0, 20).map((item) => ({
219
298
  objectKey: item.row.objectKey,
@@ -234,6 +313,7 @@ export async function buildLocalContextPack(projectRoot, request) {
234
313
  columns: candidate.relation.columns.slice(0, 16).map((column) => column.name),
235
314
  })),
236
315
  topRejected: reranked.rejected,
316
+ certifiedCandidateFits,
237
317
  candidateConflicts: conflicts,
238
318
  },
239
319
  freshness: {
@@ -310,6 +390,7 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer) {
310
390
  addDbtDagObjects(manifest, objects, edges, diagnostics);
311
391
  addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges, diagnostics);
312
392
  addBlockDependencyEdges(manifest, edges);
393
+ addBlockOutputLineageObjects(manifest, objects, edges);
313
394
  addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagnostics);
314
395
  const nodeKeyMap = new Map();
315
396
  for (const node of [...manifestGraph.nodes, ...semanticGraph.nodes]) {
@@ -429,6 +510,30 @@ export class MetadataCatalog {
429
510
  );
430
511
  CREATE INDEX IF NOT EXISTS idx_runtime_schema_snapshots_captured ON runtime_schema_snapshots(captured_at DESC);
431
512
 
513
+ CREATE TABLE IF NOT EXISTS runtime_value_index (
514
+ value_key TEXT PRIMARY KEY,
515
+ relation TEXT NOT NULL,
516
+ schema_name TEXT,
517
+ table_name TEXT,
518
+ column_name TEXT NOT NULL,
519
+ column_type TEXT,
520
+ value TEXT NOT NULL,
521
+ normalized_value TEXT NOT NULL,
522
+ source TEXT,
523
+ captured_at TEXT NOT NULL
524
+ );
525
+ CREATE INDEX IF NOT EXISTS idx_runtime_value_index_relation ON runtime_value_index(relation, column_name);
526
+ CREATE INDEX IF NOT EXISTS idx_runtime_value_index_captured ON runtime_value_index(captured_at DESC);
527
+
528
+ CREATE VIRTUAL TABLE IF NOT EXISTS runtime_value_fts USING fts5(
529
+ value_key UNINDEXED,
530
+ relation,
531
+ column_name,
532
+ value,
533
+ normalized_value,
534
+ tokenize = 'porter unicode61'
535
+ );
536
+
432
537
  CREATE TABLE IF NOT EXISTS metadata_state (
433
538
  key TEXT PRIMARY KEY,
434
539
  value TEXT NOT NULL
@@ -713,13 +818,56 @@ export class MetadataCatalog {
713
818
  tables: normalizeRuntimeSchemaTables(snapshot.tables).slice(0, 500),
714
819
  };
715
820
  const id = `schema_${Date.parse(capturedAt) || Date.now()}`;
716
- this.db.prepare(`
821
+ const valueEntries = buildRuntimeValueIndex(cleanSnapshot);
822
+ const insertSnapshot = this.db.prepare(`
717
823
  INSERT OR REPLACE INTO runtime_schema_snapshots (
718
824
  id, source, payload_json, captured_at
719
825
  ) VALUES (?, ?, ?, ?)
720
- `).run(id, cleanSnapshot.source ?? null, JSON.stringify(cleanSnapshot), capturedAt);
826
+ `);
827
+ const insertValue = this.db.prepare(`
828
+ INSERT OR REPLACE INTO runtime_value_index (
829
+ value_key, relation, schema_name, table_name, column_name, column_type,
830
+ value, normalized_value, source, captured_at
831
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
832
+ `);
833
+ const insertValueFts = this.db.prepare(`
834
+ INSERT INTO runtime_value_fts (value_key, relation, column_name, value, normalized_value)
835
+ VALUES (?, ?, ?, ?, ?)
836
+ `);
837
+ const setState = this.db.prepare(`
838
+ INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)
839
+ `);
840
+ const txn = this.db.transaction(() => {
841
+ insertSnapshot.run(id, cleanSnapshot.source ?? null, JSON.stringify(cleanSnapshot), capturedAt);
842
+ this.db.prepare('DELETE FROM runtime_value_fts').run();
843
+ this.db.prepare('DELETE FROM runtime_value_index').run();
844
+ for (const entry of valueEntries) {
845
+ insertValue.run(entry.valueKey, entry.relation, entry.schema ?? null, entry.tableName ?? null, entry.columnName, entry.columnType ?? null, entry.value, entry.normalizedValue, entry.source ?? null, entry.capturedAt);
846
+ insertValueFts.run(entry.valueKey, entry.relation, entry.columnName, entry.value, entry.normalizedValue);
847
+ }
848
+ setState.run('runtime_value_index_count', String(valueEntries.length));
849
+ setState.run('runtime_value_index_captured_at', capturedAt);
850
+ });
851
+ txn();
721
852
  return cleanSnapshot;
722
853
  }
854
+ searchRuntimeValues(terms, limit = 40) {
855
+ const cleanTerms = uniqueStrings(terms.map(normalizeValueIndexText).filter(Boolean)).slice(0, 8);
856
+ const query = sanitizeFtsQuery(cleanTerms.join(' '));
857
+ if (!query)
858
+ return [];
859
+ const rows = this.db.prepare(`
860
+ SELECT v.*,
861
+ bm25(runtime_value_fts) AS rank
862
+ FROM runtime_value_fts
863
+ JOIN runtime_value_index AS v ON v.value_key = runtime_value_fts.value_key
864
+ WHERE runtime_value_fts MATCH ?
865
+ ORDER BY rank
866
+ LIMIT ?
867
+ `).all(query, Math.max(1, limit));
868
+ return rows.map((row) => rowToRuntimeValueMatch(row, cleanTerms))
869
+ .sort((a, b) => b.score - a.score || a.relation.localeCompare(b.relation) || a.columnName.localeCompare(b.columnName));
870
+ }
723
871
  latestRuntimeSchemaSnapshot() {
724
872
  const row = this.db.prepare(`
725
873
  SELECT payload_json
@@ -821,20 +969,35 @@ function loadAgentSemanticLayer(projectRoot) {
821
969
  ? { provider: 'dql', path: config.semanticLayer.path }
822
970
  : undefined;
823
971
  const configured = resolveSemanticLayerWithDiagnostics(semanticConfig, projectRoot).layer;
824
- if (configured)
972
+ // An EMPTY configured layer (e.g. `provider: 'dql'` pointing at a folder
973
+ // with no definitions — the scaffold default) must not shadow a real dbt
974
+ // MetricFlow semantic layer: that starves the catalog of every
975
+ // semantic_metric object and the governed-metric answer tier can never
976
+ // fire. Mirror the runtime's resolveProjectSemanticConfig preference:
977
+ // substance wins over configuration.
978
+ if (configured && semanticLayerHasContent(configured))
825
979
  return configured;
826
980
  if (config.dbt?.projectDir) {
827
- return resolveSemanticLayerWithDiagnostics({
981
+ const dbtLayer = resolveSemanticLayerWithDiagnostics({
828
982
  provider: 'dbt',
829
983
  projectPath: config.dbt.projectDir,
830
- }, projectRoot).layer ?? undefined;
984
+ }, projectRoot).layer;
985
+ if (dbtLayer && semanticLayerHasContent(dbtLayer))
986
+ return dbtLayer;
831
987
  }
988
+ return configured ?? undefined;
832
989
  }
833
990
  catch {
834
991
  return undefined;
835
992
  }
836
993
  return undefined;
837
994
  }
995
+ /** True when the layer defines ANY semantics — metrics, dimensions, or measures. */
996
+ function semanticLayerHasContent(layer) {
997
+ return layer.listMetrics().length > 0
998
+ || layer.listDimensions().length > 0
999
+ || layer.listMeasures().length > 0;
1000
+ }
838
1001
  function manifestDiagnosticToMetadataDiagnostic(diagnostic) {
839
1002
  return {
840
1003
  kind: diagnostic.kind,
@@ -860,6 +1023,8 @@ function objectFromKGNode(node) {
860
1023
  grain: node.grain,
861
1024
  entities: node.entities ?? [],
862
1025
  declaredOutputs: node.declaredOutputs ?? [],
1026
+ outputs: node.outputs ?? [],
1027
+ outputContract: node.outputContract ?? [],
863
1028
  dimensions: node.dimensions ?? [],
864
1029
  allowedFilters: node.allowedFilters ?? [],
865
1030
  parameterPolicy: node.parameterPolicy ?? [],
@@ -964,6 +1129,7 @@ function addDbtDagObjects(manifest, objects, edges, diagnostics) {
964
1129
  materialized: model.materialized,
965
1130
  dependsOn: model.dependsOn,
966
1131
  columns: model.columns ?? [],
1132
+ columnCompleteness: 'partial',
967
1133
  }),
968
1134
  };
969
1135
  objects.set(objectKey, mergeObject(objects.get(objectKey), object));
@@ -982,6 +1148,7 @@ function addDbtDagObjects(manifest, objects, edges, diagnostics) {
982
1148
  uniqueId: model.uniqueId,
983
1149
  type: column.type,
984
1150
  relation,
1151
+ columnCompleteness: 'partial',
985
1152
  }),
986
1153
  }));
987
1154
  const edge = {
@@ -1029,18 +1196,30 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
1029
1196
  });
1030
1197
  return;
1031
1198
  }
1199
+ const catalogColumns = loadRawDbtCatalogColumns(join(dirname(manifestPath), 'catalog.json'));
1200
+ const entries = [];
1032
1201
  for (const [uniqueId, node] of Object.entries(raw.nodes ?? {})) {
1033
1202
  if (node.resource_type !== 'model')
1034
1203
  continue;
1035
1204
  const name = rawDbtName(node);
1036
1205
  if (!name)
1037
1206
  continue;
1038
- addRawDbtCatalogObject({
1207
+ const database = stringValue(node.database);
1208
+ const schema = stringValue(node.schema);
1209
+ const entry = {
1039
1210
  uniqueId,
1040
1211
  name,
1041
1212
  node,
1042
1213
  objectKey: `dbt:model:${name}`,
1043
1214
  objectType: 'dbt_model',
1215
+ relation: [database, schema, name].filter(Boolean).join('.'),
1216
+ database,
1217
+ schema,
1218
+ catalogColumns: catalogColumns.get(uniqueId),
1219
+ };
1220
+ entries.push(entry);
1221
+ addRawDbtCatalogObject({
1222
+ ...entry,
1044
1223
  objects,
1045
1224
  edges,
1046
1225
  });
@@ -1049,23 +1228,42 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
1049
1228
  const name = rawDbtName(source);
1050
1229
  if (!name)
1051
1230
  continue;
1052
- addRawDbtCatalogObject({
1231
+ const database = stringValue(source.database);
1232
+ const schema = stringValue(source.schema);
1233
+ const entry = {
1053
1234
  uniqueId,
1054
1235
  name,
1055
1236
  node: source,
1056
1237
  objectKey: `dbt:source:${name}`,
1057
1238
  objectType: 'dbt_source',
1239
+ relation: [database, schema, name].filter(Boolean).join('.'),
1240
+ database,
1241
+ schema,
1242
+ catalogColumns: catalogColumns.get(uniqueId),
1243
+ };
1244
+ entries.push(entry);
1245
+ addRawDbtCatalogObject({
1246
+ ...entry,
1058
1247
  objects,
1059
1248
  edges,
1060
1249
  });
1061
1250
  }
1251
+ const relationLookup = buildRawDbtRelationLookup(entries);
1252
+ for (const entry of entries) {
1253
+ if (entry.objectType !== 'dbt_model')
1254
+ continue;
1255
+ addRawDbtCompiledColumnLineage(entry, relationLookup, objects, edges);
1256
+ }
1062
1257
  }
1063
1258
  function addRawDbtCatalogObject(input) {
1064
1259
  const existing = input.objects.get(input.objectKey);
1065
- const database = stringValue(input.node.database);
1066
- const schema = stringValue(input.node.schema);
1067
- const relation = [database, schema, input.name].filter(Boolean).join('.');
1068
- const columns = rawDbtColumns(input.node.columns);
1260
+ const database = input.database ?? stringValue(input.node.database);
1261
+ const schema = input.schema ?? stringValue(input.node.schema);
1262
+ const relation = input.relation || [database, schema, input.name].filter(Boolean).join('.');
1263
+ const columns = input.catalogColumns && input.catalogColumns.length > 0
1264
+ ? input.catalogColumns
1265
+ : rawDbtColumns(input.node.columns);
1266
+ const columnCompleteness = input.catalogColumns && input.catalogColumns.length > 0 ? 'complete' : 'partial';
1069
1267
  input.objects.set(input.objectKey, mergeObject(existing, {
1070
1268
  objectKey: input.objectKey,
1071
1269
  objectType: input.objectType,
@@ -1085,6 +1283,7 @@ function addRawDbtCatalogObject(input) {
1085
1283
  dependsOn: rawDbtDependsOn(input.node),
1086
1284
  tags: metadataStringArray(input.node.tags),
1087
1285
  catalogOnly: existing ? undefined : true,
1286
+ columnCompleteness,
1088
1287
  columns,
1089
1288
  }),
1090
1289
  }));
@@ -1106,6 +1305,7 @@ function addRawDbtCatalogObject(input) {
1106
1305
  uniqueId: input.uniqueId,
1107
1306
  type: column.type,
1108
1307
  relation,
1308
+ columnCompleteness: 'partial',
1109
1309
  catalogOnly: existingColumn ? undefined : true,
1110
1310
  }),
1111
1311
  }));
@@ -1127,6 +1327,124 @@ function addRawDbtCatalogObject(input) {
1127
1327
  });
1128
1328
  }
1129
1329
  }
1330
+ function addRawDbtCompiledColumnLineage(entry, relationLookup, objects, edges) {
1331
+ const sql = rawDbtCompiledSql(entry.node);
1332
+ if (!sql)
1333
+ return;
1334
+ const lineage = extractColumnLineage(sql);
1335
+ if (!lineage.parsed || lineage.columns.length === 0)
1336
+ return;
1337
+ for (const column of lineage.columns) {
1338
+ if (!column.name || column.name === '*' || column.name.endsWith('.*'))
1339
+ continue;
1340
+ const outputKey = ensureDbtLineageOutputColumnObject(entry, column, objects);
1341
+ for (const source of column.sources) {
1342
+ if (!source.column || source.column === '*')
1343
+ continue;
1344
+ const targetKey = ensureDbtLineageSourceColumnObject(source, relationLookup, objects);
1345
+ putEdge(edges, {
1346
+ edgeType: 'derives_from',
1347
+ fromKey: outputKey,
1348
+ toKey: targetKey,
1349
+ confidence: column.unresolved ? 0.55 : 0.88,
1350
+ payload: compactObject({
1351
+ source: 'dbt compiled SQL column lineage',
1352
+ model: entry.name,
1353
+ output: column.name,
1354
+ table: source.table,
1355
+ column: source.column,
1356
+ isAggregate: column.isAggregate,
1357
+ aggregateFn: column.aggregateFn,
1358
+ }),
1359
+ });
1360
+ }
1361
+ }
1362
+ }
1363
+ function ensureDbtLineageOutputColumnObject(entry, column, objects) {
1364
+ const columnKey = `dbt:column:${entry.name}.${column.name}`;
1365
+ const existing = objects.get(columnKey);
1366
+ objects.set(columnKey, mergeObject(existing, {
1367
+ objectKey: columnKey,
1368
+ objectType: 'dbt_column',
1369
+ name: column.name,
1370
+ fullName: `${entry.name}.${column.name}`,
1371
+ status: existing?.status ?? 'dbt_compiled_lineage',
1372
+ sourcePath: existing?.sourcePath ?? stringValue(entry.node.original_file_path) ?? stringValue(entry.node.path),
1373
+ sourceSystem: existing?.sourceSystem ?? 'dbt compiled SQL lineage',
1374
+ payload: compactObject({
1375
+ ...(existing?.payload ?? {}),
1376
+ model: entry.name,
1377
+ uniqueId: entry.uniqueId,
1378
+ relation: entry.relation,
1379
+ compiledSqlLineage: true,
1380
+ isAggregate: column.isAggregate,
1381
+ aggregateFn: column.aggregateFn,
1382
+ lineageSources: column.sources,
1383
+ }),
1384
+ }));
1385
+ return columnKey;
1386
+ }
1387
+ function ensureDbtLineageSourceColumnObject(source, relationLookup, objects) {
1388
+ const dbtSource = resolveRawDbtRelationForTable(source.table, relationLookup);
1389
+ if (dbtSource) {
1390
+ const columnKey = `dbt:column:${dbtSource.name}.${source.column}`;
1391
+ const existing = objects.get(columnKey);
1392
+ if (existing)
1393
+ return columnKey;
1394
+ objects.set(columnKey, {
1395
+ objectKey: columnKey,
1396
+ objectType: 'dbt_column',
1397
+ name: source.column,
1398
+ fullName: `${dbtSource.name}.${source.column}`,
1399
+ status: 'dbt_compiled_lineage',
1400
+ sourceSystem: 'dbt compiled SQL lineage',
1401
+ payload: compactObject({
1402
+ model: dbtSource.name,
1403
+ uniqueId: dbtSource.uniqueId,
1404
+ relation: dbtSource.relation,
1405
+ inferredFromCompiledSql: true,
1406
+ }),
1407
+ });
1408
+ return columnKey;
1409
+ }
1410
+ return ensureWarehouseLineageSourceColumnObject(source, objects, 'dbt compiled SQL lineage');
1411
+ }
1412
+ function buildRawDbtRelationLookup(entries) {
1413
+ const lookup = new Map();
1414
+ const add = (key, entry) => {
1415
+ if (!key)
1416
+ return;
1417
+ const normalized = normalizeRelationKey(key);
1418
+ if (!normalized)
1419
+ return;
1420
+ const existing = lookup.get(normalized);
1421
+ if (existing)
1422
+ existing.add(entry);
1423
+ else
1424
+ lookup.set(normalized, new Set([entry]));
1425
+ };
1426
+ for (const entry of entries) {
1427
+ add(entry.uniqueId, entry);
1428
+ add(entry.name, entry);
1429
+ add(entry.relation, entry);
1430
+ for (const key of dbtModelLookupKeys(entry.name, entry.schema, entry.database)) {
1431
+ add(key, entry);
1432
+ }
1433
+ }
1434
+ return lookup;
1435
+ }
1436
+ function resolveRawDbtRelationForTable(tableRef, relationLookup) {
1437
+ for (const key of tableReferenceLookupKeys(tableRef)) {
1438
+ const matches = relationLookup.get(key);
1439
+ if (matches?.size === 1)
1440
+ return [...matches][0];
1441
+ }
1442
+ return undefined;
1443
+ }
1444
+ function rawDbtCompiledSql(node) {
1445
+ return stringValue(node.compiled_code)
1446
+ ?? stringValue(node.compiled_sql);
1447
+ }
1130
1448
  function rawDbtName(node) {
1131
1449
  return stringValue(node.alias) ?? stringValue(node.identifier) ?? stringValue(node.name);
1132
1450
  }
@@ -1140,6 +1458,41 @@ function rawDbtDependsOn(node) {
1140
1458
  : null;
1141
1459
  return metadataStringArray(dependsOn?.nodes);
1142
1460
  }
1461
+ function loadRawDbtCatalogColumns(catalogPath) {
1462
+ const out = new Map();
1463
+ let raw;
1464
+ try {
1465
+ raw = JSON.parse(readFileSync(catalogPath, 'utf-8'));
1466
+ }
1467
+ catch {
1468
+ return out;
1469
+ }
1470
+ for (const entries of [raw.nodes ?? {}, raw.sources ?? {}]) {
1471
+ for (const [uniqueId, node] of Object.entries(entries)) {
1472
+ const columns = rawDbtCatalogColumns(node.columns);
1473
+ if (columns.length > 0)
1474
+ out.set(uniqueId, columns);
1475
+ }
1476
+ }
1477
+ return out;
1478
+ }
1479
+ function rawDbtCatalogColumns(value) {
1480
+ if (!value || typeof value !== 'object' || Array.isArray(value))
1481
+ return [];
1482
+ return Object.entries(value).flatMap(([key, entry]) => {
1483
+ if (!entry || typeof entry !== 'object')
1484
+ return [];
1485
+ const record = entry;
1486
+ const name = stringValue(record.name) ?? stringValue(record.column_name) ?? key;
1487
+ if (!name)
1488
+ return [];
1489
+ return [{
1490
+ name,
1491
+ type: stringValue(record.type) ?? stringValue(record.data_type),
1492
+ description: stringValue(record.comment) ?? stringValue(record.description),
1493
+ }];
1494
+ });
1495
+ }
1143
1496
  function rawDbtColumns(value) {
1144
1497
  if (!value || typeof value !== 'object' || Array.isArray(value))
1145
1498
  return [];
@@ -1163,6 +1516,9 @@ function addManifestBlockDetails(manifest, objects) {
1163
1516
  const existing = objects.get(objectKey);
1164
1517
  if (!existing)
1165
1518
  continue;
1519
+ const contractOutputNames = block.declaredOutputs
1520
+ ?? block.outputContract?.map((output) => output.name).filter(Boolean)
1521
+ ?? block.outputs?.map((output) => output.name).filter(Boolean);
1166
1522
  objects.set(objectKey, mergeObject(existing, {
1167
1523
  ...existing,
1168
1524
  payload: compactObject({
@@ -1173,6 +1529,9 @@ function addManifestBlockDetails(manifest, objects) {
1173
1529
  refDependencies: block.refDependencies,
1174
1530
  metricRefs: block.metricRefs,
1175
1531
  dimensionRefs: block.dimensionRefs,
1532
+ declaredOutputs: block.declaredOutputs,
1533
+ outputs: block.outputs,
1534
+ outputContract: block.outputContract,
1176
1535
  dimensions: block.dimensions,
1177
1536
  chartType: block.chartType,
1178
1537
  blockType: block.blockType,
@@ -1187,7 +1546,7 @@ function addManifestBlockDetails(manifest, objects) {
1187
1546
  grain: block.grain,
1188
1547
  entities: block.entities,
1189
1548
  terms: block.termRefs,
1190
- outputs: block.declaredOutputs,
1549
+ outputs: contractOutputNames,
1191
1550
  dimensions: block.dimensions,
1192
1551
  filters: block.allowedFilters,
1193
1552
  sources: [...(block.tableDependencies ?? []), ...(block.rawTableRefs ?? [])],
@@ -1226,6 +1585,9 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
1226
1585
  }
1227
1586
  const latestContractKeyById = new Map();
1228
1587
  const contractKeyByVersionedRef = new Map();
1588
+ // entity (by name and domain.entity) -> physical binding ref, so relationships
1589
+ // can be resolved to concrete relations for grain-safe join hints.
1590
+ const entityBindingRef = new Map();
1229
1591
  for (const domain of raw.domains) {
1230
1592
  if (!domain?.name)
1231
1593
  continue;
@@ -1279,6 +1641,11 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
1279
1641
  if (!entity?.name)
1280
1642
  continue;
1281
1643
  const entityKey = `datalex:entity:${domain.name}.${entity.name}`;
1644
+ if (entity.binding?.ref) {
1645
+ const boundKey = objectKeyForDataLexBinding(entity.binding);
1646
+ entityBindingRef.set(entity.name.toLowerCase(), boundKey);
1647
+ entityBindingRef.set(`${domain.name}.${entity.name}`.toLowerCase(), boundKey);
1648
+ }
1282
1649
  objects.set(entityKey, mergeObject(objects.get(entityKey), {
1283
1650
  objectKey: entityKey,
1284
1651
  objectType: 'datalex_entity',
@@ -1350,6 +1717,33 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
1350
1717
  }
1351
1718
  }
1352
1719
  }
1720
+ // Typed cross-entity relationships → grain-safe join edges between the entities'
1721
+ // physical relations. buildSelectedJoinPaths turns these into `datalex`-sourced
1722
+ // join paths that outrank name-heuristic guesses.
1723
+ const resolveEntityRef = (endpoint) => entityBindingRef.get(`${endpoint.domain ?? ''}.${endpoint.entity}`.toLowerCase())
1724
+ ?? entityBindingRef.get(endpoint.entity.toLowerCase());
1725
+ for (const relationship of raw.relationships ?? []) {
1726
+ if (!relationship?.from?.entity || !relationship?.to?.entity)
1727
+ continue;
1728
+ const fromRef = resolveEntityRef(relationship.from);
1729
+ const toRef = resolveEntityRef(relationship.to);
1730
+ if (!fromRef || !toRef || fromRef.toLowerCase() === toRef.toLowerCase())
1731
+ continue;
1732
+ putEdge(edges, {
1733
+ edgeType: 'datalex_join',
1734
+ fromKey: fromRef,
1735
+ toKey: toRef,
1736
+ confidence: relationship.identifying ? 0.97 : 0.9,
1737
+ payload: compactObject({
1738
+ source: 'datalex relationship',
1739
+ name: relationship.name,
1740
+ fromColumn: relationship.from.column,
1741
+ toColumn: relationship.to.column,
1742
+ cardinality: relationship.cardinality,
1743
+ verb: relationship.verb,
1744
+ }),
1745
+ });
1746
+ }
1353
1747
  for (const block of Object.values(manifest.blocks ?? {})) {
1354
1748
  if (!block.datalexContract)
1355
1749
  continue;
@@ -1406,6 +1800,112 @@ function addBlockDependencyEdges(manifest, edges) {
1406
1800
  }
1407
1801
  }
1408
1802
  }
1803
+ function addBlockOutputLineageObjects(manifest, objects, edges) {
1804
+ const dbtLookup = buildDbtModelLookup(manifest);
1805
+ for (const block of Object.values(manifest.blocks ?? {})) {
1806
+ if (!isCertifiedManifestBlock(block))
1807
+ continue;
1808
+ if (!Array.isArray(block.outputs) || block.outputs.length === 0)
1809
+ continue;
1810
+ const blockKey = `dql:block:${block.name}`;
1811
+ for (const output of block.outputs) {
1812
+ const outputName = typeof output.name === 'string' ? output.name.trim() : '';
1813
+ if (!outputName)
1814
+ continue;
1815
+ const outputKey = blockOutputObjectKey(block.name, outputName);
1816
+ objects.set(outputKey, mergeObject(objects.get(outputKey), {
1817
+ objectKey: outputKey,
1818
+ objectType: 'dql_block_output',
1819
+ name: outputName,
1820
+ fullName: `${block.name}.${outputName}`,
1821
+ domain: block.domain,
1822
+ owner: block.owner,
1823
+ status: block.status,
1824
+ description: `${outputName} output column from DQL block ${block.name}.`,
1825
+ sourcePath: block.filePath,
1826
+ sourceSystem: 'DQL block column lineage',
1827
+ payload: compactObject({
1828
+ block: block.name,
1829
+ output: outputName,
1830
+ isAggregate: output.isAggregate,
1831
+ aggregateFn: output.aggregateFn,
1832
+ unresolved: output.unresolved,
1833
+ sources: output.sources ?? [],
1834
+ }),
1835
+ }));
1836
+ putEdge(edges, {
1837
+ edgeType: 'contains',
1838
+ fromKey: blockKey,
1839
+ toKey: outputKey,
1840
+ confidence: 1,
1841
+ payload: { source: 'dql block output lineage', output: outputName },
1842
+ });
1843
+ for (const source of output.sources ?? []) {
1844
+ const targetKey = ensureLineageSourceColumnObject(source, dbtLookup, objects);
1845
+ putEdge(edges, {
1846
+ edgeType: 'derives_from',
1847
+ fromKey: outputKey,
1848
+ toKey: targetKey,
1849
+ confidence: output.unresolved ? 0.55 : 0.92,
1850
+ payload: {
1851
+ source: 'dql block output column lineage',
1852
+ block: block.name,
1853
+ output: outputName,
1854
+ table: source.table,
1855
+ column: source.column,
1856
+ aggregateFn: output.aggregateFn,
1857
+ },
1858
+ });
1859
+ }
1860
+ }
1861
+ }
1862
+ }
1863
+ function isCertifiedManifestBlock(block) {
1864
+ return block.status === 'certified'
1865
+ || block.status === 'approved';
1866
+ }
1867
+ function ensureLineageSourceColumnObject(source, dbtLookup, objects) {
1868
+ const dbtModelKey = resolveDbtModelKeyForTable(source.table, dbtLookup);
1869
+ if (dbtModelKey) {
1870
+ const modelName = dbtModelKey.split(':').at(-1) ?? source.table;
1871
+ const columnKey = `dbt:column:${modelName}.${source.column}`;
1872
+ if (objects.has(columnKey))
1873
+ return columnKey;
1874
+ }
1875
+ return ensureWarehouseLineageSourceColumnObject(source, objects, 'DQL block column lineage');
1876
+ }
1877
+ function ensureWarehouseLineageSourceColumnObject(source, objects, sourceSystem) {
1878
+ const tableKey = `warehouse:table:${source.table}`;
1879
+ if (!objects.has(tableKey)) {
1880
+ objects.set(tableKey, {
1881
+ objectKey: tableKey,
1882
+ objectType: 'warehouse_table',
1883
+ name: source.table.split('.').at(-1) ?? source.table,
1884
+ fullName: source.table,
1885
+ status: 'referenced',
1886
+ sourceSystem,
1887
+ payload: compactObject({ relation: source.table }),
1888
+ });
1889
+ }
1890
+ const columnKey = `warehouse:column:${source.table}.${source.column}`;
1891
+ objects.set(columnKey, mergeObject(objects.get(columnKey), {
1892
+ objectKey: columnKey,
1893
+ objectType: 'warehouse_column',
1894
+ name: source.column,
1895
+ fullName: `${source.table}.${source.column}`,
1896
+ status: 'referenced',
1897
+ sourceSystem,
1898
+ payload: compactObject({
1899
+ relation: source.table,
1900
+ table: source.table,
1901
+ column: source.column,
1902
+ }),
1903
+ }));
1904
+ return columnKey;
1905
+ }
1906
+ function blockOutputObjectKey(blockName, outputName) {
1907
+ return `dql:block_output:${blockName}.${outputName}`;
1908
+ }
1409
1909
  function putEdge(edges, edge) {
1410
1910
  edges.set(`${edge.edgeType}\u0000${edge.fromKey}\u0000${edge.toKey}`, edge);
1411
1911
  }
@@ -1525,7 +2025,7 @@ function firstNonEmpty(...values) {
1525
2025
  function lowerOrUndef(value) {
1526
2026
  return value ? value.trim().toLowerCase() : undefined;
1527
2027
  }
1528
- function planContextPackRoute(input) {
2028
+ async function planContextPackRoute(input) {
1529
2029
  const intent = input.request.intent ?? input.questionPlan.routeIntent ?? classifyMetadataIntent(input.request.question, input.request.followUp);
1530
2030
  // Conflict route (additive, evaluated first): if the question's top governed
1531
2031
  // candidates include BOTH sides of a compile-time trust conflict, refuse to
@@ -1533,7 +2033,7 @@ function planContextPackRoute(input) {
1533
2033
  // prompt. Only fires when both conflicting sides are actually among the
1534
2034
  // selected candidates for THIS question, so non-conflicting asks are
1535
2035
  // unaffected even when a conflict exists elsewhere in the project.
1536
- const routeConflict = pickRouteConflict(input.compileConflicts ?? [], input.rankedObjects ?? [], input.objects);
2036
+ const routeConflict = pickRouteConflict(input.compileConflicts ?? [], input.rankedObjects ?? [], input.objects, input.request.question);
1537
2037
  if (routeConflict) {
1538
2038
  const selectedEvidenceForConflict = input.evidenceRoles.slice(0, 16);
1539
2039
  return {
@@ -1566,6 +2066,30 @@ function planContextPackRoute(input) {
1566
2066
  .filter((item) => item.kind === 'context_only')
1567
2067
  .sort((a, b) => b.score - a.score)[0];
1568
2068
  const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
2069
+ const directCertifiedBypass = Boolean(exact && (exactExampleMatch
2070
+ || intent === 'definition_lookup'
2071
+ || intent === 'exact_certified_lookup'
2072
+ || objectNameInQuestion(input.request.question, exact)));
2073
+ const blockFitObject = exact
2074
+ ?? (contextApplicability ? input.objects.find((object) => object.objectKey === contextApplicability.objectKey) : undefined);
2075
+ const rawBlockFit = blockFitObject
2076
+ ? evaluateCertifiedBlockFit({
2077
+ question: input.request.question,
2078
+ plan: input.questionPlan,
2079
+ block: blockFitObject,
2080
+ exactExampleMatch: exact ? exactExampleMatch : false,
2081
+ definitionLookup: Boolean(exact && intent === 'definition_lookup'),
2082
+ })
2083
+ : undefined;
2084
+ const blockFit = rawBlockFit && blockFitObject
2085
+ ? await confirmMediumCertifiedFit({
2086
+ request: input.request,
2087
+ questionPlan: input.questionPlan,
2088
+ block: blockFitObject,
2089
+ fit: rawBlockFit,
2090
+ directCertifiedBypass,
2091
+ })
2092
+ : rawBlockFit;
1569
2093
  const missingContext = buildMissingContext(input.request, intent, input.objects, input.allowedSqlContext);
1570
2094
  const selectedEvidence = input.evidenceRoles.slice(0, 16);
1571
2095
  // Grain / contract gate (refinement of the existing certified→generated
@@ -1596,7 +2120,9 @@ function planContextPackRoute(input) {
1596
2120
  input.allowedSqlContext.sourceBlockSql.length > 0 ||
1597
2121
  input.objects.some((object) => object.objectType.startsWith('semantic_'));
1598
2122
  const grainGateDemotes = Boolean(grainGate && !grainGate.allow && canGenerateFromContext);
1599
- if (!grainGateDemotes && exact && (certifiedApplicability?.kind === 'exact_answer'
2123
+ const blockFitDemotes = Boolean(blockFit && !certifiedFitAllowsTier1(blockFit) && !directCertifiedBypass && canGenerateFromContext);
2124
+ const certifiedFitPassed = !blockFit || certifiedFitAllowsTier1(blockFit) || directCertifiedBypass;
2125
+ if (!grainGateDemotes && !blockFitDemotes && certifiedFitPassed && exact && (certifiedApplicability?.kind === 'exact_answer'
1600
2126
  || certifiedApplicability?.kind === 'safe_parameterized'
1601
2127
  || intent === 'exact_certified_lookup'
1602
2128
  || intent === 'definition_lookup'
@@ -1608,6 +2134,7 @@ function planContextPackRoute(input) {
1608
2134
  reason: `Certified ${exact.objectType.replace(/_/g, ' ')} "${exact.name}" exactly matches the requested artifact, definition, or direct KPI grain.`,
1609
2135
  routeReason: grainGateInfo?.reason,
1610
2136
  grainGate: grainGateInfo,
2137
+ blockFit,
1611
2138
  trustLabel: 'certified',
1612
2139
  reviewStatus: 'certified',
1613
2140
  exactObjectKey: exact.objectKey,
@@ -1627,6 +2154,25 @@ function planContextPackRoute(input) {
1627
2154
  reason: `Certified block "${exact.name}" is close but answers a different grain than the question, so it is context only and SQL is generated for the requested grain.`,
1628
2155
  routeReason: grainGate.reason,
1629
2156
  grainGate: grainGateInfo,
2157
+ blockFit,
2158
+ trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
2159
+ reviewStatus: 'draft_ready',
2160
+ certifiedApplicability: certifiedApplicability
2161
+ ? { ...certifiedApplicability, kind: 'context_only' }
2162
+ : contextApplicability,
2163
+ selectedEvidence,
2164
+ missingContext,
2165
+ followUps: buildMetadataFollowUps(isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking', input.allowedSqlContext),
2166
+ };
2167
+ }
2168
+ if (blockFitDemotes && exact && blockFit) {
2169
+ return {
2170
+ route: 'generated_sql',
2171
+ intent: isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking',
2172
+ reason: `Certified block "${exact.name}" is relevant but does not satisfy the requested answer shape, so it is context only and SQL is generated for the requested result.`,
2173
+ routeReason: blockFit.reasons.join('; '),
2174
+ grainGate: grainGateInfo,
2175
+ blockFit,
1630
2176
  trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
1631
2177
  reviewStatus: 'draft_ready',
1632
2178
  certifiedApplicability: certifiedApplicability
@@ -1668,15 +2214,17 @@ function planContextPackRoute(input) {
1668
2214
  route: 'generated_sql',
1669
2215
  intent,
1670
2216
  reason: 'The question asks for a different grain, ranking, breakdown, comparison, entity drilldown, or diagnostic analysis, so certified artifacts are context only.',
2217
+ routeReason: blockFit && !certifiedFitAllowsTier1(blockFit) ? blockFit.reasons.join('; ') : undefined,
1671
2218
  trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
1672
2219
  reviewStatus: 'draft_ready',
1673
2220
  certifiedApplicability: contextApplicability,
2221
+ blockFit,
1674
2222
  selectedEvidence,
1675
2223
  missingContext,
1676
2224
  followUps: buildMetadataFollowUps(intent, input.allowedSqlContext),
1677
2225
  };
1678
2226
  }
1679
- if (exact) {
2227
+ if (exact && certifiedFitPassed) {
1680
2228
  return {
1681
2229
  route: 'certified',
1682
2230
  intent: 'exact_certified_lookup',
@@ -1685,17 +2233,93 @@ function planContextPackRoute(input) {
1685
2233
  reviewStatus: 'certified',
1686
2234
  exactObjectKey: exact.objectKey,
1687
2235
  certifiedApplicability,
2236
+ blockFit,
1688
2237
  selectedEvidence,
1689
2238
  missingContext: [],
1690
2239
  followUps: buildMetadataFollowUps('exact_certified_lookup', input.allowedSqlContext),
1691
2240
  };
1692
2241
  }
2242
+ if (exact && blockFit && !certifiedFitAllowsTier1(blockFit)) {
2243
+ return clarifyDecision(intent, input.trustLabel === 'certified' ? 'mixed' : input.trustLabel, selectedEvidence, [{
2244
+ kind: 'metadata',
2245
+ severity: 'blocking',
2246
+ message: `The closest certified block "${exact.name}" does not exactly answer this question (${blockFit.reasons.join('; ')}), and there is not enough SQL context to safely generate the requested answer.`,
2247
+ }]);
2248
+ }
1693
2249
  return clarifyDecision(intent, input.trustLabel, selectedEvidence, missingContext.length > 0 ? missingContext : [{
1694
2250
  kind: 'metadata',
1695
2251
  severity: 'blocking',
1696
2252
  message: 'The local metadata matched some context, but not enough to choose a safe metric, table, or grain.',
1697
2253
  }]);
1698
2254
  }
2255
+ const CERTIFIED_FIT_CONFIRM_TIMEOUT_MS = 2500;
2256
+ async function confirmMediumCertifiedFit(input) {
2257
+ const confirmer = input.request.confirmCertifiedFit;
2258
+ if (!confirmer || input.directCertifiedBypass)
2259
+ return input.fit;
2260
+ if (input.fit.confidence !== 'medium')
2261
+ return input.fit;
2262
+ if (input.fit.kind !== 'exact' && input.fit.kind !== 'trim_safe')
2263
+ return input.fit;
2264
+ try {
2265
+ const result = await withTimeout(confirmer({
2266
+ question: input.request.question,
2267
+ questionPlan: input.questionPlan,
2268
+ block: input.block,
2269
+ fit: input.fit,
2270
+ }), CERTIFIED_FIT_CONFIRM_TIMEOUT_MS, 'certified fit confirmation timed out');
2271
+ const reason = cleanConfirmationReason(result.reason);
2272
+ if (result.allow && result.confidence !== 'low') {
2273
+ return {
2274
+ ...input.fit,
2275
+ confidence: 'high',
2276
+ reasons: [
2277
+ ...input.fit.reasons,
2278
+ `fit confirmation accepted this certified block${reason ? `: ${reason}` : ''}`,
2279
+ ],
2280
+ };
2281
+ }
2282
+ return {
2283
+ ...input.fit,
2284
+ kind: 'context_only',
2285
+ confidence: 'high',
2286
+ reasons: [
2287
+ ...input.fit.reasons,
2288
+ `fit confirmation rejected this certified block as a direct answer${reason ? `: ${reason}` : ''}`,
2289
+ ],
2290
+ };
2291
+ }
2292
+ catch (error) {
2293
+ const message = error instanceof Error ? error.message : String(error);
2294
+ return {
2295
+ ...input.fit,
2296
+ reasons: [
2297
+ ...input.fit.reasons,
2298
+ `fit confirmation unavailable; kept review-required (${message})`,
2299
+ ],
2300
+ };
2301
+ }
2302
+ }
2303
+ async function withTimeout(promise, ms, message) {
2304
+ let timeout;
2305
+ try {
2306
+ return await Promise.race([
2307
+ promise,
2308
+ new Promise((_, reject) => {
2309
+ timeout = setTimeout(() => reject(new Error(message)), ms);
2310
+ }),
2311
+ ]);
2312
+ }
2313
+ finally {
2314
+ if (timeout)
2315
+ clearTimeout(timeout);
2316
+ }
2317
+ }
2318
+ function cleanConfirmationReason(value) {
2319
+ return typeof value === 'string'
2320
+ ? value.replace(/\s+/g, ' ').trim().slice(0, 240)
2321
+ : '';
2322
+ }
1699
2323
  function certifiedApplicabilities(objects, questionPlan) {
1700
2324
  const items = objects
1701
2325
  .filter((object) => object.objectType === 'dql_block' && isCertifiedMetadataObject(object))
@@ -1704,6 +2328,56 @@ function certifiedApplicabilities(objects, questionPlan) {
1704
2328
  .sort((a, b) => b.score - a.score);
1705
2329
  return new Map(items.map((item) => [item.objectKey, item]));
1706
2330
  }
2331
+ function buildCertifiedCandidateFitDiagnostics(input) {
2332
+ return input.objects
2333
+ .filter((object) => object.objectType === 'dql_block' && isCertifiedMetadataObject(object))
2334
+ .map((object) => {
2335
+ const applicability = certifiedApplicabilityForObject(object, input.questionPlan);
2336
+ const fit = evaluateCertifiedBlockFit({
2337
+ question: input.request.question,
2338
+ plan: input.questionPlan,
2339
+ block: object,
2340
+ exactExampleMatch: hasExactExampleQuestion(input.request.question, object),
2341
+ definitionLookup: input.routeDecision.intent === 'definition_lookup',
2342
+ });
2343
+ return {
2344
+ objectKey: object.objectKey,
2345
+ name: object.name,
2346
+ applicabilityKind: applicability.kind,
2347
+ applicabilityScore: applicability.score,
2348
+ action: certifiedCandidateFitAction(object.objectKey, applicability, fit, input.routeDecision),
2349
+ fit,
2350
+ };
2351
+ })
2352
+ .filter((item) => item.applicabilityKind !== 'not_applicable' ||
2353
+ item.fit.kind !== 'not_applicable' ||
2354
+ item.action === 'certified_answer')
2355
+ .sort((a, b) => certifiedCandidateActionRank(a.action) - certifiedCandidateActionRank(b.action) ||
2356
+ b.applicabilityScore - a.applicabilityScore ||
2357
+ a.name.localeCompare(b.name))
2358
+ .slice(0, 12);
2359
+ }
2360
+ function certifiedCandidateFitAction(objectKey, applicability, fit, routeDecision) {
2361
+ if (routeDecision.route === 'certified' && routeDecision.exactObjectKey === objectKey)
2362
+ return 'certified_answer';
2363
+ if (routeDecision.certifiedApplicability?.objectKey === objectKey && routeDecision.route !== 'certified')
2364
+ return 'context_only';
2365
+ if (!certifiedFitAllowsTier1(fit) || applicability.kind === 'context_only')
2366
+ return 'rejected_for_fit';
2367
+ return 'eligible_not_selected';
2368
+ }
2369
+ function certifiedCandidateActionRank(action) {
2370
+ switch (action) {
2371
+ case 'certified_answer':
2372
+ return 0;
2373
+ case 'context_only':
2374
+ return 1;
2375
+ case 'rejected_for_fit':
2376
+ return 2;
2377
+ case 'eligible_not_selected':
2378
+ return 3;
2379
+ }
2380
+ }
1707
2381
  function clarifyDecision(intent, trustLabel, selectedEvidence, missingContext) {
1708
2382
  return {
1709
2383
  route: 'clarify',
@@ -1994,14 +2668,18 @@ function buildEvidenceRoles(objects, queryRuns) {
1994
2668
  function evidenceRoleForObject(object) {
1995
2669
  if (object.objectType === 'dql_block' && isCertifiedMetadataObject(object))
1996
2670
  return 'certified_context';
2671
+ if (object.objectType === 'dql_block_output')
2672
+ return 'certified_context';
1997
2673
  if (object.objectType === 'semantic_metric')
1998
2674
  return 'semantic_metric';
1999
2675
  if (object.objectType === 'dql_term' || object.objectType === 'business_view')
2000
2676
  return 'business_context';
2001
2677
  if (object.objectType === 'dbt_model' || object.objectType === 'dbt_source' || object.objectType === 'dbt_column')
2002
2678
  return 'dbt_model';
2003
- if (object.objectType === 'warehouse_table')
2679
+ if (object.objectType === 'warehouse_table' || object.objectType === 'warehouse_column')
2004
2680
  return 'warehouse_schema';
2681
+ if (object.objectType === 'runtime_value')
2682
+ return 'value_match';
2005
2683
  if (object.objectType === 'runtime_table' || object.objectType === 'runtime_column')
2006
2684
  return 'runtime_schema';
2007
2685
  if (object.objectType === 'selected_context')
@@ -2017,6 +2695,8 @@ function evidenceReasonForObject(object) {
2017
2695
  return 'Semantic definition can ground metric and dimension meaning.';
2018
2696
  if (object.objectType.startsWith('dbt_'))
2019
2697
  return 'dbt metadata supplies physical model and column context.';
2698
+ if (object.objectType === 'runtime_value')
2699
+ return 'Observed runtime value matched a literal in the question.';
2020
2700
  if (object.objectType === 'runtime_table' || object.objectType === 'runtime_column')
2021
2701
  return 'Runtime schema supplies executable table and column context.';
2022
2702
  return reasonForObject(object);
@@ -2027,7 +2707,7 @@ function sqlParentObjectsForSelectedColumns(selectedObjects, candidateObjects, q
2027
2707
  for (const object of candidateObjects) {
2028
2708
  if (!isSqlParentObject(object) || selectedKeys.has(object.objectKey))
2029
2709
  continue;
2030
- const relation = metadataRelationFromObject(object);
2710
+ const relation = metadataObjectToAllowedSqlRelation(object);
2031
2711
  const key = relation ? normalizeRelationKey(relation.relation) : '';
2032
2712
  if (!relation || !key)
2033
2713
  continue;
@@ -2039,7 +2719,7 @@ function sqlParentObjectsForSelectedColumns(selectedObjects, candidateObjects, q
2039
2719
  for (const object of selectedObjects) {
2040
2720
  if (!isSqlColumnObject(object))
2041
2721
  continue;
2042
- const relation = metadataRelationFromObject(object);
2722
+ const relation = metadataObjectToAllowedSqlRelation(object);
2043
2723
  const key = relation ? normalizeRelationKey(relation.relation) : '';
2044
2724
  if (!key)
2045
2725
  continue;
@@ -2062,8 +2742,52 @@ function schemaShapeCandidateObjects(catalog, questionPlan, request, runtimeObje
2062
2742
  if (!questionPlan.needsGeneratedSql)
2063
2743
  return [];
2064
2744
  const candidates = new Map();
2745
+ for (const candidate of cachedCatalogSchemaShapeCandidates(catalog, questionPlan, request)) {
2746
+ candidates.set(candidate.object.objectKey, candidate);
2747
+ }
2748
+ const considerObject = (object) => {
2749
+ const relation = metadataObjectToAllowedSqlRelation(object);
2750
+ if (!relation || relation.columns.length === 0)
2751
+ return;
2752
+ const shape = schemaShapeMatchForQuestion(relation, questionPlan);
2753
+ if (shape.score <= 0)
2754
+ return;
2755
+ const relationScore = scoreAllowedSqlRelationWithAnalysisPlan(relation, questionPlan);
2756
+ const objectScore = scoreMetadataObjectWithAnalysisPlan(object, questionPlan);
2757
+ const score = Number((shape.score + relationScore.score + objectScore.score).toFixed(3));
2758
+ if (score < schemaShapeMinimumScore(questionPlan))
2759
+ return;
2760
+ candidates.set(object.objectKey, {
2761
+ object,
2762
+ relation,
2763
+ score,
2764
+ reasons: [
2765
+ ...shape.reasons,
2766
+ ...relationScore.reasons.filter((reason) => /analysis terms|metric terms|dimension terms|columns match|inspected\/projected columns/i.test(reason)).slice(0, 3),
2767
+ ],
2768
+ });
2769
+ trimSchemaShapeCandidateMap(candidates, 64);
2770
+ };
2771
+ for (const object of mergeObjects(runtimeObjects.filter(isSqlParentObject))) {
2772
+ considerObject(object);
2773
+ }
2774
+ return Array.from(candidates.values())
2775
+ .sort((a, b) => b.score - a.score || a.relation.relation.localeCompare(b.relation.relation))
2776
+ .slice(0, 24);
2777
+ }
2778
+ function cachedCatalogSchemaShapeCandidates(catalog, questionPlan, request) {
2779
+ const cacheKey = schemaShapeCacheKey(catalog, questionPlan);
2780
+ if (cacheKey) {
2781
+ const cached = schemaShapeCandidateCache.get(cacheKey);
2782
+ if (cached) {
2783
+ schemaShapeCandidateCache.delete(cacheKey);
2784
+ schemaShapeCandidateCache.set(cacheKey, cached);
2785
+ return cached;
2786
+ }
2787
+ }
2788
+ const candidates = new Map();
2065
2789
  const considerObject = (object) => {
2066
- const relation = metadataRelationFromObject(object);
2790
+ const relation = metadataObjectToAllowedSqlRelation(object);
2067
2791
  if (!relation || relation.columns.length === 0)
2068
2792
  return;
2069
2793
  const shape = schemaShapeMatchForQuestion(relation, questionPlan);
@@ -2092,12 +2816,25 @@ function schemaShapeCandidateObjects(catalog, questionPlan, request, runtimeObje
2092
2816
  for (const object of objects)
2093
2817
  considerObject(object);
2094
2818
  });
2095
- for (const object of mergeObjects(runtimeObjects.filter(isSqlParentObject))) {
2096
- considerObject(object);
2097
- }
2098
- return Array.from(candidates.values())
2819
+ const result = Array.from(candidates.values())
2099
2820
  .sort((a, b) => b.score - a.score || a.relation.relation.localeCompare(b.relation.relation))
2100
2821
  .slice(0, 24);
2822
+ if (cacheKey) {
2823
+ schemaShapeCandidateCache.set(cacheKey, result);
2824
+ while (schemaShapeCandidateCache.size > SCHEMA_SHAPE_CACHE_LIMIT) {
2825
+ const oldest = schemaShapeCandidateCache.keys().next().value;
2826
+ if (!oldest)
2827
+ break;
2828
+ schemaShapeCandidateCache.delete(oldest);
2829
+ }
2830
+ }
2831
+ return result;
2832
+ }
2833
+ function schemaShapeCacheKey(catalog, questionPlan) {
2834
+ const fingerprint = catalog.state('fingerprint');
2835
+ if (!fingerprint)
2836
+ return null;
2837
+ return `${fingerprint}:${sha256(stableStringify(questionPlan))}`;
2101
2838
  }
2102
2839
  function schemaShapeScanBatchSize(request) {
2103
2840
  if (request.strictness === 'exploratory')
@@ -2269,6 +3006,32 @@ function isSqlParentObject(object) {
2269
3006
  object.objectType === 'warehouse_table' ||
2270
3007
  object.objectType === 'runtime_table';
2271
3008
  }
3009
+ const FULL_CATALOG_RELATION_TYPES = ['dbt_model', 'dbt_source', 'warehouse_table', 'runtime_table'];
3010
+ const FULL_CATALOG_COLUMN_TYPES = ['dbt_column', 'warehouse_column', 'runtime_column'];
3011
+ /** Deep mode over a catalog this small hands the model everything instead of top-k. */
3012
+ const FULL_CATALOG_MAX_RELATIONS = 40;
3013
+ const FULL_CATALOG_MAX_COLUMNS = 2000;
3014
+ /**
3015
+ * When the whole catalog is small enough to fit in context, return every relation
3016
+ * and column object so deep-research mode can reason over the complete schema.
3017
+ * Returns undefined when the catalog exceeds the small-catalog budget, so ranked
3018
+ * top-k selection stays in force for large warehouses.
3019
+ */
3020
+ function collectFullCatalogObjects(catalog) {
3021
+ const relations = catalog.listObjects({
3022
+ objectTypes: FULL_CATALOG_RELATION_TYPES,
3023
+ limit: FULL_CATALOG_MAX_RELATIONS + 1,
3024
+ });
3025
+ if (relations.length === 0 || relations.length > FULL_CATALOG_MAX_RELATIONS)
3026
+ return undefined;
3027
+ const columns = catalog.listObjects({
3028
+ objectTypes: FULL_CATALOG_COLUMN_TYPES,
3029
+ limit: FULL_CATALOG_MAX_COLUMNS + 1,
3030
+ });
3031
+ if (columns.length > FULL_CATALOG_MAX_COLUMNS)
3032
+ return undefined;
3033
+ return mergeObjects([...relations, ...columns]);
3034
+ }
2272
3035
  function buildAllowedSqlContext(objects, edges) {
2273
3036
  const byRelation = new Map();
2274
3037
  const objectsByKey = new Map(objects.map((object) => [object.objectKey, object]));
@@ -2281,18 +3044,20 @@ function buildAllowedSqlContext(objects, edges) {
2281
3044
  byRelation.set(key, { ...relation, columns: dedupeRuntimeColumns(relation.columns).slice(0, 120) });
2282
3045
  return;
2283
3046
  }
3047
+ const columnCompleteness = mergeRelationCompletenessForCatalog(existing, relation);
2284
3048
  byRelation.set(key, {
2285
3049
  ...existing,
2286
3050
  objectKey: existing.objectKey ?? relation.objectKey,
2287
3051
  source: mergeRelationSources(existing.source, relation.source),
2288
- columns: dedupeRuntimeColumns([...existing.columns, ...relation.columns]).slice(0, 120),
3052
+ columnCompleteness,
3053
+ columns: mergeRelationColumns(existing, relation).slice(0, 120),
2289
3054
  });
2290
3055
  };
2291
3056
  for (const object of objects) {
2292
3057
  if (object.objectType === 'warehouse_table' && !warehouseTableHasTrustedReference(object, objectsByKey)) {
2293
3058
  continue;
2294
3059
  }
2295
- const relation = metadataRelationFromObject(object);
3060
+ const relation = metadataObjectToAllowedSqlRelation(object);
2296
3061
  if (relation)
2297
3062
  addRelation(relation);
2298
3063
  if (object.objectType === 'dql_block' && !isCertifiedMetadataObject(object)) {
@@ -2304,6 +3069,7 @@ function buildAllowedSqlContext(objects, edges) {
2304
3069
  name: table.split('.').at(-1) ?? table,
2305
3070
  objectKey: object.objectKey,
2306
3071
  source: 'certified block dependency',
3072
+ columnCompleteness: 'partial',
2307
3073
  columns: [],
2308
3074
  });
2309
3075
  }
@@ -2313,6 +3079,7 @@ function buildAllowedSqlContext(objects, edges) {
2313
3079
  name: table.split('.').at(-1) ?? table,
2314
3080
  objectKey: object.objectKey,
2315
3081
  source: 'certified block SQL reference',
3082
+ columnCompleteness: 'partial',
2316
3083
  columns: [],
2317
3084
  });
2318
3085
  }
@@ -2324,6 +3091,7 @@ function buildAllowedSqlContext(objects, edges) {
2324
3091
  name: shape.relation.split('.').at(-1) ?? shape.relation,
2325
3092
  objectKey: object.objectKey,
2326
3093
  source: 'certified source SQL shape',
3094
+ columnCompleteness: 'partial',
2327
3095
  columns: sourceSqlShapeColumns(sourceSql),
2328
3096
  });
2329
3097
  }
@@ -2333,11 +3101,12 @@ function buildAllowedSqlContext(objects, edges) {
2333
3101
  continue;
2334
3102
  const from = objectsByKey.get(edge.fromKey);
2335
3103
  const to = objectsByKey.get(edge.toKey);
2336
- const fromRelation = from ? metadataRelationFromObject(from) : null;
2337
- const toRelation = to ? metadataRelationFromObject(to) : null;
3104
+ const fromRelation = from ? metadataObjectToAllowedSqlRelation(from) : null;
3105
+ const toRelation = to ? metadataObjectToAllowedSqlRelation(to) : null;
2338
3106
  if (fromRelation && toRelation)
2339
- addRelation({ ...fromRelation, columns: toRelation.columns, source: 'dbt mapped warehouse table' });
3107
+ addRelation({ ...fromRelation, columns: toRelation.columns, columnCompleteness: toRelation.columnCompleteness, source: 'dbt mapped warehouse table' });
2340
3108
  }
3109
+ applyLineageColumnAliases(byRelation, objectsByKey, edges);
2341
3110
  return {
2342
3111
  relations: coalesceRelationAliases(Array.from(byRelation.values()))
2343
3112
  .sort((a, b) => a.relation.localeCompare(b.relation)),
@@ -2358,6 +3127,79 @@ function buildAllowedSqlContext(objects, edges) {
2358
3127
  })),
2359
3128
  };
2360
3129
  }
3130
+ function applyLineageColumnAliases(byRelation, objectsByKey, edges) {
3131
+ const aliasesByRelationAndColumn = new Map();
3132
+ for (const edge of edges) {
3133
+ if (edge.edgeType !== 'derives_from')
3134
+ continue;
3135
+ const from = objectsByKey.get(edge.fromKey);
3136
+ const to = objectsByKey.get(edge.toKey);
3137
+ if (!from || !to)
3138
+ continue;
3139
+ const targetRelation = metadataObjectToAllowedSqlRelation(to);
3140
+ if (!targetRelation)
3141
+ continue;
3142
+ const alias = semanticAliasFromLineageOutput(from);
3143
+ if (!alias || namesEqualForCatalog(alias, to.name))
3144
+ continue;
3145
+ const relationKey = normalizeRelationKey(targetRelation.relation);
3146
+ const columnKey = normalizeColumnKeyForCatalog(to.name);
3147
+ if (!relationKey || !columnKey)
3148
+ continue;
3149
+ const mapKey = `${relationKey}\u0000${columnKey}`;
3150
+ const existing = aliasesByRelationAndColumn.get(mapKey) ?? new Set();
3151
+ existing.add(alias);
3152
+ aliasesByRelationAndColumn.set(mapKey, existing);
3153
+ }
3154
+ if (aliasesByRelationAndColumn.size === 0)
3155
+ return;
3156
+ for (const [relationKey, relation] of byRelation) {
3157
+ const columns = relation.columns.map((column) => {
3158
+ const aliases = aliasesByRelationAndColumn.get(`${relationKey}\u0000${normalizeColumnKeyForCatalog(column.name)}`);
3159
+ if (!aliases?.size)
3160
+ return column;
3161
+ const aliasText = Array.from(aliases).sort().slice(0, 8).join(', ');
3162
+ const suffix = `Governed aliases from lineage: ${aliasText}.`;
3163
+ const description = column.description
3164
+ ? column.description.includes(suffix)
3165
+ ? column.description
3166
+ : `${column.description} ${suffix}`
3167
+ : suffix;
3168
+ return { ...column, description };
3169
+ });
3170
+ byRelation.set(relationKey, { ...relation, columns });
3171
+ }
3172
+ }
3173
+ function semanticAliasFromLineageOutput(object) {
3174
+ const candidates = [
3175
+ typeof object.payload?.output === 'string' ? object.payload.output : undefined,
3176
+ object.name,
3177
+ object.fullName?.split('.').at(-1),
3178
+ ];
3179
+ for (const candidate of candidates) {
3180
+ const alias = normalizeLineageAlias(candidate);
3181
+ if (alias)
3182
+ return alias;
3183
+ }
3184
+ return undefined;
3185
+ }
3186
+ function normalizeLineageAlias(value) {
3187
+ if (!value)
3188
+ return undefined;
3189
+ const normalized = value.replace(/["`]/g, '').replace(/\s+/g, ' ').trim();
3190
+ if (!normalized || normalized === '*' || normalized === '?')
3191
+ return undefined;
3192
+ if (normalized.length > 64)
3193
+ return undefined;
3194
+ return normalized;
3195
+ }
3196
+ function normalizeColumnKeyForCatalog(value) {
3197
+ return value.replace(/["`]/g, '').trim().toLowerCase();
3198
+ }
3199
+ function namesEqualForCatalog(left, right) {
3200
+ return normalizeColumnKeyForCatalog(left).replace(/[_\s]+/g, '') ===
3201
+ normalizeColumnKeyForCatalog(right).replace(/[_\s]+/g, '');
3202
+ }
2361
3203
  // Pull the first natural-language example question off a block payload so the
2362
3204
  // certified block can serve as a question→SQL few-shot exemplar.
2363
3205
  function firstExampleQuestion(examples) {
@@ -2374,12 +3216,66 @@ function firstExampleQuestion(examples) {
2374
3216
  }
2375
3217
  return undefined;
2376
3218
  }
2377
- function buildSelectedJoinPaths(allowedSqlContext) {
3219
+ function buildSelectedJoinPaths(allowedSqlContext, edges) {
2378
3220
  const relations = allowedSqlContext.relations
2379
3221
  .filter((relation) => relation.columns.some((column) => isJoinKeyColumnForCatalog(column.name)))
2380
3222
  .slice(0, 16);
2381
3223
  const joins = [];
2382
3224
  const seen = new Set();
3225
+ const lookup = buildSelectedRelationLookup(relations);
3226
+ const pushJoin = (join) => {
3227
+ const key = catalogJoinDedupeKey(join.leftRelation, join.leftColumn, join.rightRelation, join.rightColumn);
3228
+ if (seen.has(key))
3229
+ return;
3230
+ seen.add(key);
3231
+ joins.push(join);
3232
+ };
3233
+ for (const edge of edges) {
3234
+ if (edge.edgeType !== 'depends_on')
3235
+ continue;
3236
+ const left = lookupRelationForCatalogEdge(edge.fromKey, lookup);
3237
+ const right = lookupRelationForCatalogEdge(edge.toKey, lookup);
3238
+ if (!left || !right || normalizeRelationKey(left.relation) === normalizeRelationKey(right.relation))
3239
+ continue;
3240
+ const join = pickCatalogJoinColumns(left, right);
3241
+ if (!join)
3242
+ continue;
3243
+ pushJoin({
3244
+ leftRelation: left.relation,
3245
+ leftColumn: join.leftColumn,
3246
+ rightRelation: right.relation,
3247
+ rightColumn: join.rightColumn,
3248
+ reason: `dbt lineage: ${left.name} depends_on ${right.name}; ${catalogJoinReason(join.leftColumn, join.rightColumn, join.confidence)}`,
3249
+ confidence: Math.max(join.confidence, 0.98),
3250
+ source: 'dbt_lineage',
3251
+ });
3252
+ }
3253
+ // DataLex modeled relationships: grain-safe joins on the declared canonical
3254
+ // columns (verified to exist on both relations), ranked above name heuristics.
3255
+ const datalexLookup = buildSelectedRelationLookup(allowedSqlContext.relations);
3256
+ for (const edge of edges) {
3257
+ if (edge.edgeType !== 'datalex_join')
3258
+ continue;
3259
+ const left = lookupRelationForCatalogEdge(edge.fromKey, datalexLookup);
3260
+ const right = lookupRelationForCatalogEdge(edge.toKey, datalexLookup);
3261
+ if (!left || !right || normalizeRelationKey(left.relation) === normalizeRelationKey(right.relation))
3262
+ continue;
3263
+ const payload = (edge.payload ?? {});
3264
+ const guessed = pickCatalogJoinColumns(left, right);
3265
+ const leftColumn = relationHasColumn(left, payload.fromColumn) ? payload.fromColumn : guessed?.leftColumn;
3266
+ const rightColumn = relationHasColumn(right, payload.toColumn) ? payload.toColumn : guessed?.rightColumn;
3267
+ if (!leftColumn || !rightColumn)
3268
+ continue;
3269
+ pushJoin({
3270
+ leftRelation: left.relation,
3271
+ leftColumn,
3272
+ rightRelation: right.relation,
3273
+ rightColumn,
3274
+ reason: `DataLex relationship${payload.name ? ` ${payload.name}` : ''}${payload.cardinality ? ` (${payload.cardinality})` : ''}`,
3275
+ confidence: edge.confidence ?? 0.9,
3276
+ source: 'datalex',
3277
+ });
3278
+ }
2383
3279
  for (let leftIndex = 0; leftIndex < relations.length; leftIndex += 1) {
2384
3280
  for (let rightIndex = leftIndex + 1; rightIndex < relations.length; rightIndex += 1) {
2385
3281
  const left = relations[leftIndex];
@@ -2387,29 +3283,122 @@ function buildSelectedJoinPaths(allowedSqlContext) {
2387
3283
  const join = pickCatalogJoinColumns(left, right);
2388
3284
  if (!join)
2389
3285
  continue;
2390
- const key = [
2391
- normalizeRelationKey(left.relation),
2392
- join.leftColumn.toLowerCase(),
2393
- normalizeRelationKey(right.relation),
2394
- join.rightColumn.toLowerCase(),
2395
- ].join('|');
2396
- if (seen.has(key))
2397
- continue;
2398
- seen.add(key);
2399
- joins.push({
2400
- leftRelation: left.relation,
2401
- leftColumn: join.leftColumn,
2402
- rightRelation: right.relation,
2403
- rightColumn: join.rightColumn,
3286
+ const oriented = orientCatalogJoinPath(left, right, join);
3287
+ pushJoin({
3288
+ leftRelation: oriented.leftRelation.relation,
3289
+ leftColumn: oriented.leftColumn,
3290
+ rightRelation: oriented.rightRelation.relation,
3291
+ rightColumn: oriented.rightColumn,
2404
3292
  reason: catalogJoinReason(join.leftColumn, join.rightColumn, join.confidence),
2405
3293
  confidence: join.confidence,
3294
+ source: 'metadata_guess',
2406
3295
  });
2407
3296
  }
2408
3297
  }
2409
3298
  return joins
2410
- .sort((a, b) => b.confidence - a.confidence || a.leftRelation.localeCompare(b.leftRelation))
3299
+ .sort((a, b) => joinSourceRank(a.source) - joinSourceRank(b.source) || b.confidence - a.confidence || a.leftRelation.localeCompare(b.leftRelation))
2411
3300
  .slice(0, 12);
2412
3301
  }
3302
+ function buildSelectedRelationLookup(relations) {
3303
+ const byObjectKey = new Map();
3304
+ const byLookupKey = new Map();
3305
+ const addLookup = (key, relation) => {
3306
+ const normalized = normalizeRelationKey(key ?? '');
3307
+ if (!normalized)
3308
+ return;
3309
+ const existing = byLookupKey.get(normalized) ?? [];
3310
+ if (!existing.some((candidate) => normalizeRelationKey(candidate.relation) === normalizeRelationKey(relation.relation))) {
3311
+ existing.push(relation);
3312
+ }
3313
+ byLookupKey.set(normalized, existing);
3314
+ };
3315
+ for (const relation of relations) {
3316
+ if (relation.objectKey)
3317
+ byObjectKey.set(relation.objectKey, relation);
3318
+ addLookup(relation.relation, relation);
3319
+ addLookup(relation.name, relation);
3320
+ addLookup(relationTailKey(relation.relation), relation);
3321
+ }
3322
+ return { byObjectKey, byLookupKey };
3323
+ }
3324
+ function lookupRelationForCatalogEdge(edgeKey, lookup) {
3325
+ const exact = lookup.byObjectKey.get(edgeKey);
3326
+ if (exact)
3327
+ return exact;
3328
+ const tail = edgeKey.split(':').at(-1);
3329
+ const candidates = [
3330
+ edgeKey,
3331
+ tail,
3332
+ tail ? relationTailKey(tail) : undefined,
3333
+ ];
3334
+ for (const candidate of candidates) {
3335
+ const normalized = normalizeRelationKey(candidate ?? '');
3336
+ if (!normalized)
3337
+ continue;
3338
+ const matches = lookup.byLookupKey.get(normalized);
3339
+ if (matches?.length === 1)
3340
+ return matches[0];
3341
+ }
3342
+ return undefined;
3343
+ }
3344
+ function catalogJoinDedupeKey(leftRelation, leftColumn, rightRelation, rightColumn) {
3345
+ return [
3346
+ `${normalizeRelationKey(leftRelation)}.${normalizeColumnKeyForCatalog(leftColumn)}`,
3347
+ `${normalizeRelationKey(rightRelation)}.${normalizeColumnKeyForCatalog(rightColumn)}`,
3348
+ ].sort().join('|');
3349
+ }
3350
+ /** Map a DataLex physical binding to the metadata object key of its relation. */
3351
+ function objectKeyForDataLexBinding(binding) {
3352
+ switch (binding.kind) {
3353
+ case 'dbt_model': return `dbt:model:${binding.ref}`;
3354
+ case 'dbt_source': return `dbt:source:${binding.ref}`;
3355
+ default: return `warehouse:table:${binding.ref}`;
3356
+ }
3357
+ }
3358
+ function relationHasColumn(relation, column) {
3359
+ if (!column)
3360
+ return false;
3361
+ const target = normalizeColumnKeyForCatalog(column);
3362
+ return relation.columns.some((entry) => normalizeColumnKeyForCatalog(entry.name) === target);
3363
+ }
3364
+ function joinSourceRank(source) {
3365
+ switch (source) {
3366
+ case 'dbt_lineage': return 0;
3367
+ case 'datalex': return 1;
3368
+ case 'kg_path': return 2;
3369
+ case 'metadata_guess': return 3;
3370
+ default: return 4;
3371
+ }
3372
+ }
3373
+ function orientCatalogJoinPath(left, right, join) {
3374
+ if (catalogRelationFactScore(right) > catalogRelationFactScore(left)) {
3375
+ return {
3376
+ leftRelation: right,
3377
+ leftColumn: join.rightColumn,
3378
+ rightRelation: left,
3379
+ rightColumn: join.leftColumn,
3380
+ };
3381
+ }
3382
+ return {
3383
+ leftRelation: left,
3384
+ leftColumn: join.leftColumn,
3385
+ rightRelation: right,
3386
+ rightColumn: join.rightColumn,
3387
+ };
3388
+ }
3389
+ function catalogRelationFactScore(relation) {
3390
+ const text = normalizeRelationKey([relation.name, relation.relation].join(' ')).replace(/[._-]+/g, ' ');
3391
+ let score = 0;
3392
+ if (/\b(fct|fact)\b/i.test(text))
3393
+ score += 20;
3394
+ if (/\b(order items?|orders?|events?|transactions?|performance|activity|usage|revenue|sales|payments?|line items?|stats?)\b/i.test(text))
3395
+ score += 8;
3396
+ if (/\b(dim|dimension)\b/i.test(text))
3397
+ score -= 18;
3398
+ if (/\b(customers?|products?|players?|accounts?|users?|segments?|regions?)\b/i.test(text))
3399
+ score -= 4;
3400
+ return score;
3401
+ }
2413
3402
  function pickCatalogJoinColumns(left, right) {
2414
3403
  const candidates = [];
2415
3404
  const leftColumns = left.columns.filter((column) => isJoinKeyColumnForCatalog(column.name)).slice(0, 32);
@@ -2515,7 +3504,8 @@ function coalesceRelationAliases(relations) {
2515
3504
  ...target,
2516
3505
  objectKey: target.objectKey ?? relation.objectKey,
2517
3506
  source: mergeRelationSources(target.source, relation.source),
2518
- columns: dedupeRuntimeColumns([...target.columns, ...relation.columns]).slice(0, 120),
3507
+ columnCompleteness: mergeRelationCompletenessForCatalog(target, relation),
3508
+ columns: mergeRelationColumns(target, relation).slice(0, 120),
2519
3509
  });
2520
3510
  skipped.add(relationKey);
2521
3511
  }
@@ -2523,6 +3513,31 @@ function coalesceRelationAliases(relations) {
2523
3513
  .filter(([key]) => !skipped.has(key))
2524
3514
  .map(([, relation]) => relation);
2525
3515
  }
3516
+ function mergeRelationColumns(existing, incoming) {
3517
+ const existingIsSourceShape = relationSourceIncludes(existing.source, 'certified source SQL shape');
3518
+ const incomingIsSourceShape = relationSourceIncludes(incoming.source, 'certified source SQL shape');
3519
+ if (existingIsSourceShape && !incomingIsSourceShape) {
3520
+ return dedupeRuntimeColumns([...incoming.columns, ...existing.columns]);
3521
+ }
3522
+ if (!existingIsSourceShape && incomingIsSourceShape) {
3523
+ return dedupeRuntimeColumns([...existing.columns, ...incoming.columns]);
3524
+ }
3525
+ return dedupeRuntimeColumns([...existing.columns, ...incoming.columns]);
3526
+ }
3527
+ function mergeRelationCompletenessForCatalog(existing, incoming) {
3528
+ if (relationCompletenessValue(existing) === 'complete' || relationCompletenessValue(incoming) === 'complete') {
3529
+ return 'complete';
3530
+ }
3531
+ return 'partial';
3532
+ }
3533
+ function relationCompletenessValue(relation) {
3534
+ if (relation.columnCompleteness)
3535
+ return relation.columnCompleteness;
3536
+ return relation.columns.length === 0 ? 'partial' : 'complete';
3537
+ }
3538
+ function relationSourceIncludes(source, value) {
3539
+ return Boolean(source && source.split(/\s+\+\s+/).some((part) => part.trim() === value));
3540
+ }
2526
3541
  function relationTailKey(relation) {
2527
3542
  const parts = normalizeRelationKey(relation).split('.').filter(Boolean);
2528
3543
  return parts.at(-1) ?? '';
@@ -2547,8 +3562,26 @@ function warehouseTableHasTrustedReference(object, objectsByKey) {
2547
3562
  return block ? isCertifiedMetadataObject(block) : false;
2548
3563
  });
2549
3564
  }
2550
- function metadataRelationFromObject(object) {
2551
- if (object.objectType === 'dbt_column' || object.objectType === 'runtime_column') {
3565
+ export function metadataObjectToAllowedSqlRelation(object) {
3566
+ if (object.objectType === 'runtime_value') {
3567
+ const relation = metadataPayloadString(object, 'relation');
3568
+ const columnName = metadataPayloadString(object, 'column');
3569
+ if (!relation || !columnName)
3570
+ return null;
3571
+ return {
3572
+ relation,
3573
+ name: relation.split('.').at(-1) ?? relation,
3574
+ objectKey: object.objectKey,
3575
+ source: 'runtime value index',
3576
+ columnCompleteness: 'partial',
3577
+ columns: [{
3578
+ name: columnName,
3579
+ type: metadataPayloadString(object, 'type'),
3580
+ sampleValues: metadataStringArray(object.payload?.sampleValues),
3581
+ }],
3582
+ };
3583
+ }
3584
+ if (object.objectType === 'dbt_column' || object.objectType === 'runtime_column' || object.objectType === 'warehouse_column') {
2552
3585
  const relation = metadataPayloadString(object, 'relation');
2553
3586
  if (!relation)
2554
3587
  return null;
@@ -2556,7 +3589,12 @@ function metadataRelationFromObject(object) {
2556
3589
  relation,
2557
3590
  name: relation.split('.').at(-1) ?? relation,
2558
3591
  objectKey: object.objectKey,
2559
- source: object.objectType === 'runtime_column' ? 'runtime schema snapshot' : 'dbt manifest',
3592
+ source: object.objectType === 'runtime_column'
3593
+ ? 'runtime schema snapshot'
3594
+ : object.objectType === 'warehouse_column'
3595
+ ? 'DQL block column lineage'
3596
+ : 'dbt manifest',
3597
+ columnCompleteness: 'partial',
2560
3598
  columns: [{
2561
3599
  name: object.name,
2562
3600
  type: metadataPayloadString(object, 'type') ?? metadataPayloadString(object, 'data_type'),
@@ -2573,9 +3611,21 @@ function metadataRelationFromObject(object) {
2573
3611
  name: relation.split('.').at(-1) ?? object.name,
2574
3612
  objectKey: object.objectKey,
2575
3613
  source: object.objectType === 'runtime_table' ? 'runtime schema snapshot' : object.sourceSystem ?? 'local metadata catalog',
3614
+ columnCompleteness: relationColumnCompletenessFromObject(object),
2576
3615
  columns: metadataRuntimeColumns(object.payload?.columns),
2577
3616
  };
2578
3617
  }
3618
+ function relationColumnCompletenessFromObject(object) {
3619
+ if (object.objectType === 'runtime_table')
3620
+ return 'complete';
3621
+ if (object.payload?.columnCompleteness === 'complete' || object.payload?.catalogColumnCompleteness === 'complete')
3622
+ return 'complete';
3623
+ if (object.payload?.columnCompleteness === 'partial' || object.payload?.catalogColumnCompleteness === 'partial')
3624
+ return 'partial';
3625
+ if (object.objectType === 'warehouse_table')
3626
+ return 'partial';
3627
+ return 'partial';
3628
+ }
2579
3629
  function runtimeSchemaObjects(snapshot) {
2580
3630
  const capturedAt = snapshot.capturedAt ?? new Date().toISOString();
2581
3631
  return normalizeRuntimeSchemaTables(snapshot.tables).flatMap((table) => {
@@ -2592,6 +3642,7 @@ function runtimeSchemaObjects(snapshot) {
2592
3642
  payload: compactObject({
2593
3643
  relation,
2594
3644
  schema: table.schema,
3645
+ columnCompleteness: 'complete',
2595
3646
  columns: table.columns,
2596
3647
  }),
2597
3648
  updatedAt: capturedAt,
@@ -2607,6 +3658,7 @@ function runtimeSchemaObjects(snapshot) {
2607
3658
  payload: compactObject({
2608
3659
  relation,
2609
3660
  type: column.type,
3661
+ columnCompleteness: 'partial',
2610
3662
  sampleValues: column.sampleValues,
2611
3663
  }),
2612
3664
  updatedAt: capturedAt,
@@ -2614,6 +3666,30 @@ function runtimeSchemaObjects(snapshot) {
2614
3666
  return [tableObject, ...columns];
2615
3667
  });
2616
3668
  }
3669
+ function runtimeValueMatchObjects(matches) {
3670
+ return matches.map((match) => ({
3671
+ objectKey: match.valueKey,
3672
+ objectType: 'runtime_value',
3673
+ name: `${match.columnName} = ${match.value}`,
3674
+ fullName: `${match.relation}.${match.columnName}:${match.value}`,
3675
+ description: `Observed value "${match.value}" in ${match.relation}.${match.columnName}.`,
3676
+ status: 'runtime_observed',
3677
+ sourceSystem: match.source ?? 'runtime value index',
3678
+ score: match.score,
3679
+ payload: compactObject({
3680
+ relation: match.relation,
3681
+ schema: match.schema,
3682
+ table: match.tableName,
3683
+ column: match.columnName,
3684
+ type: match.columnType,
3685
+ value: match.value,
3686
+ normalizedValue: match.normalizedValue,
3687
+ sampleValues: [match.value],
3688
+ capturedAt: match.capturedAt,
3689
+ }),
3690
+ updatedAt: match.capturedAt,
3691
+ }));
3692
+ }
2617
3693
  function selectedContextObjects(value) {
2618
3694
  const root = value && typeof value === 'object' ? value : null;
2619
3695
  if (!root)
@@ -2643,7 +3719,13 @@ function normalizeFollowUpContext(value) {
2643
3719
  if (!value || typeof value !== 'object')
2644
3720
  return null;
2645
3721
  const record = value;
2646
- const kind = record.kind === 'drilldown' ? 'drilldown' : record.kind === 'generic' ? 'generic' : null;
3722
+ const kind = record.kind === 'drilldown'
3723
+ ? 'drilldown'
3724
+ : record.kind === 'generic'
3725
+ ? 'generic'
3726
+ : record.kind === 'contextual'
3727
+ ? 'contextual'
3728
+ : null;
2647
3729
  if (!kind)
2648
3730
  return null;
2649
3731
  return {
@@ -2653,8 +3735,77 @@ function normalizeFollowUpContext(value) {
2653
3735
  sourceAnswer: stringValue(record.sourceAnswer),
2654
3736
  filters: metadataStringArray(record.filters),
2655
3737
  dimensions: metadataStringArray(record.dimensions),
3738
+ priorResultColumns: metadataStringArray(record.priorResultColumns),
3739
+ priorResultValues: metadataStringRecordArray(record.priorResultValues),
3740
+ priorResultRef: normalizePriorResultRef(record.priorResultRef),
3741
+ priorDqlArtifact: normalizePriorDqlArtifact(record.priorDqlArtifact),
3742
+ priorLimit: typeof record.priorLimit === 'number' ? record.priorLimit : undefined,
3743
+ priorMeasures: metadataStringArray(record.priorMeasures),
3744
+ };
3745
+ }
3746
+ function normalizePriorResultRef(value) {
3747
+ if (!value || typeof value !== 'object' || Array.isArray(value))
3748
+ return undefined;
3749
+ const record = value;
3750
+ const id = stringValue(record.id);
3751
+ const columns = metadataStringArray(record.columns) ?? [];
3752
+ if (!id || columns.length === 0)
3753
+ return undefined;
3754
+ const rowCount = typeof record.rowCount === 'number' && Number.isFinite(record.rowCount)
3755
+ ? record.rowCount
3756
+ : undefined;
3757
+ const sourceSql = stringValue(record.sourceSql);
3758
+ return {
3759
+ id,
3760
+ question: stringValue(record.question),
3761
+ columns: columns.slice(0, 32),
3762
+ rowCount,
3763
+ sourceSql: sourceSql ? sourceSql.slice(0, 1200) : undefined,
2656
3764
  };
2657
3765
  }
3766
+ function normalizePriorDqlArtifact(value) {
3767
+ const artifact = normalizeDqlArtifactReference(value);
3768
+ if (!artifact)
3769
+ return undefined;
3770
+ return {
3771
+ ...artifact,
3772
+ source: artifact.source.slice(0, 3000),
3773
+ metrics: artifact.metrics?.slice(0, 32),
3774
+ dimensions: artifact.dimensions?.slice(0, 32),
3775
+ filters: artifact.filters
3776
+ ?.filter((filter) => filter.values.length > 0)
3777
+ .slice(0, 12)
3778
+ .map((filter) => ({ ...filter, values: filter.values.slice(0, 12) })),
3779
+ orderBy: artifact.orderBy?.slice(0, 12),
3780
+ };
3781
+ }
3782
+ /**
3783
+ * Deterministic route-commitment gate: true only when the new plan differs from
3784
+ * the prior one in FILTERS / TOP-N / TIMEFRAME alone. Any change to measures,
3785
+ * dimensions, grain, required outputs, or mode disqualifies reuse — those change
3786
+ * which artifacts/relations can answer, so the full pipeline must re-run.
3787
+ */
3788
+ export function isFilterOnlyRefinement(prior, next) {
3789
+ if (prior.mode !== next.mode)
3790
+ return false;
3791
+ const priorShape = prior.requestedShape;
3792
+ const nextShape = next.requestedShape;
3793
+ return setEqual(priorShape.measures, nextShape.measures)
3794
+ && setEqual(priorShape.dimensions, nextShape.dimensions)
3795
+ && setEqual(priorShape.requiredOutputs, nextShape.requiredOutputs)
3796
+ && (priorShape.grain ?? '') === (nextShape.grain ?? '')
3797
+ && setEqual(prior.entities.map((entity) => entity.text.toLowerCase()), next.entities.map((entity) => entity.text.toLowerCase()));
3798
+ }
3799
+ function setEqual(a, b) {
3800
+ const setA = new Set(a.map((value) => value.toLowerCase()));
3801
+ const setB = new Set(b.map((value) => value.toLowerCase()));
3802
+ if (setA.size !== setB.size)
3803
+ return false;
3804
+ for (const value of setA)
3805
+ if (!setB.has(value))
3806
+ return false;
3807
+ return true;
3808
+ }
2658
3809
  function followUpSourceObjectKeys(followUp) {
2659
3810
  if (!followUp?.sourceBlockName)
2660
3811
  return [];
@@ -2670,11 +3821,22 @@ function followUpContextObjects(followUp) {
2670
3821
  followUp.sourceAnswer ?? '',
2671
3822
  ...(followUp.filters ?? []),
2672
3823
  ...(followUp.dimensions ?? []),
3824
+ ...(followUp.priorResultColumns ?? []),
3825
+ followUp.priorResultRef?.id ? `result:${followUp.priorResultRef.id}` : '',
3826
+ followUp.priorResultRef?.question ?? '',
3827
+ ...(followUp.priorResultRef?.columns ?? []),
3828
+ followUp.priorResultRef?.sourceSql ? followUp.priorResultRef.sourceSql.slice(0, 600) : '',
3829
+ ...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: true }),
3830
+ ...Object.entries(followUp.priorResultValues ?? {}).flatMap(([key, values]) => [key, ...values]),
2673
3831
  ].join(' ');
2674
3832
  return [{
2675
3833
  objectKey: `selected:followup:${sha256(stableStringify(followUp)).slice(0, 16)}`,
2676
3834
  objectType: 'selected_context',
2677
- name: followUp.kind === 'drilldown' ? 'Follow-up drilldown request' : 'Follow-up request',
3835
+ name: followUp.kind === 'drilldown'
3836
+ ? 'Follow-up drilldown request'
3837
+ : followUp.kind === 'contextual'
3838
+ ? 'Conversation context (advisory)'
3839
+ : 'Follow-up request',
2678
3840
  description: text.trim() || undefined,
2679
3841
  status: 'transient_context',
2680
3842
  sourceSystem: 'agent follow-up context',
@@ -2685,20 +3847,112 @@ function followUpContextObjects(followUp) {
2685
3847
  sourceAnswer: followUp.sourceAnswer,
2686
3848
  filters: followUp.filters,
2687
3849
  dimensions: followUp.dimensions,
3850
+ priorResultColumns: followUp.priorResultColumns,
3851
+ priorResultValues: followUp.priorResultValues,
3852
+ priorResultRef: followUp.priorResultRef,
3853
+ priorDqlArtifact: followUp.priorDqlArtifact,
3854
+ priorLimit: followUp.priorLimit,
3855
+ priorMeasures: followUp.priorMeasures,
2688
3856
  }),
2689
3857
  }];
2690
3858
  }
3859
+ function metadataValueSearchTerms(question, questionPlan, followUp) {
3860
+ const terms = [
3861
+ ...questionPlan.entities.map((entity) => entity.text),
3862
+ ...questionPlan.filterTerms,
3863
+ ...(followUp?.filters ?? []),
3864
+ ...Object.values(followUp?.priorResultValues ?? {}).flat(),
3865
+ ];
3866
+ for (const match of question.matchAll(/["']([^"']{2,120})["']/g))
3867
+ terms.push(match[1]);
3868
+ for (const match of question.matchAll(/\b[\w.%+-]+@[\w.-]+\.[A-Za-z]{2,}\b/g))
3869
+ terms.push(match[0]);
3870
+ for (const match of question.matchAll(/\b[A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,4}\b/g))
3871
+ terms.push(match[0]);
3872
+ for (const match of question.matchAll(/\b(?:for|named|called|only|where|customer|user|account|product|region|segment|category|status)\s+([A-Za-z0-9@._-]+(?:\s+[A-Za-z0-9@._-]+){0,4})/gi)) {
3873
+ terms.push(match[1]);
3874
+ }
3875
+ return uniqueStrings(terms
3876
+ .map(cleanMetadataValueSearchTerm)
3877
+ .filter((term) => term.length >= 2 && !METADATA_VALUE_STOP_TERMS.has(term.toLowerCase()))).slice(0, 10);
3878
+ }
3879
+ function cleanMetadataValueSearchTerm(term) {
3880
+ return term
3881
+ .replace(/[?.,;:]+$/g, '')
3882
+ .replace(/\s+/g, ' ')
3883
+ .trim()
3884
+ .replace(/^(?:account|category|customer|member|named|called|only|product|region|segment|sku|status|subscriber|user|where)\s+/i, '')
3885
+ .replace(/\s+\b(?:last|next|this)\b.*$/i, '')
3886
+ .replace(/\s+\b(?:daily|weekly|monthly|quarterly|yearly)\b.*$/i, '')
3887
+ .trim();
3888
+ }
3889
+ const METADATA_VALUE_STOP_TERMS = new Set([
3890
+ 'account',
3891
+ 'category',
3892
+ 'customer',
3893
+ 'last month',
3894
+ 'last quarter',
3895
+ 'last week',
3896
+ 'last year',
3897
+ 'member',
3898
+ 'product',
3899
+ 'region',
3900
+ 'segment',
3901
+ 'sku',
3902
+ 'status',
3903
+ 'this month',
3904
+ 'this quarter',
3905
+ 'this week',
3906
+ 'this year',
3907
+ 'user',
3908
+ ]);
2691
3909
  function buildFollowUpSearchQuery(question, followUp) {
2692
3910
  if (!followUp)
2693
3911
  return question;
3912
+ // Contextual carry is advisory: enrich retrieval softly (prior question, columns,
3913
+ // measures) but never with concrete dimension VALUES or the block name — those pull
3914
+ // ranking hard toward the old topic when the user may be switching subjects. The
3915
+ // question-plan's own raw queries still run alongside this one either way.
3916
+ if (followUp.kind === 'contextual') {
3917
+ return [
3918
+ question,
3919
+ followUp.sourceQuestion ?? '',
3920
+ ...(followUp.priorResultColumns ?? []),
3921
+ followUp.priorResultRef?.question ?? '',
3922
+ ...(followUp.priorResultRef?.columns ?? []),
3923
+ ...(followUp.priorMeasures ?? []),
3924
+ ...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: false }),
3925
+ ].filter(Boolean).join(' ');
3926
+ }
2694
3927
  return [
2695
3928
  question,
2696
3929
  followUp.sourceBlockName ?? '',
2697
3930
  followUp.sourceQuestion ?? '',
2698
3931
  ...(followUp.filters ?? []),
2699
3932
  ...(followUp.dimensions ?? []),
3933
+ ...(followUp.priorResultColumns ?? []),
3934
+ followUp.priorResultRef?.question ?? '',
3935
+ ...(followUp.priorResultRef?.columns ?? []),
3936
+ followUp.priorResultRef?.sourceSql ? followUp.priorResultRef.sourceSql.slice(0, 600) : '',
3937
+ ...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: true }),
3938
+ ...Object.entries(followUp.priorResultValues ?? {}).flatMap(([key, values]) => [key, ...values]),
2700
3939
  ].filter(Boolean).join(' ');
2701
3940
  }
3941
+ function dqlArtifactSearchTerms(artifact, options) {
3942
+ if (!artifact)
3943
+ return [];
3944
+ return [
3945
+ artifact.name ?? '',
3946
+ artifact.sourcePath ?? '',
3947
+ ...(artifact.metrics ?? []),
3948
+ ...(artifact.dimensions ?? []),
3949
+ ...(artifact.filters ?? []).flatMap((filter) => [filter.dimension, filter.operator, ...filter.values]),
3950
+ artifact.timeDimension ? `${artifact.timeDimension.name} ${artifact.timeDimension.granularity}` : '',
3951
+ ...(artifact.orderBy ?? []).map((order) => `${order.name} ${order.direction}`),
3952
+ typeof artifact.limit === 'number' ? `limit ${artifact.limit}` : '',
3953
+ options.includeSource ? artifact.source.slice(0, 900) : '',
3954
+ ].filter(Boolean);
3955
+ }
2702
3956
  function uniqueMetadataSearchQueries(values) {
2703
3957
  const seen = new Set();
2704
3958
  const out = [];
@@ -2797,6 +4051,29 @@ function metadataStringArray(value) {
2797
4051
  return [];
2798
4052
  return value.filter((item) => typeof item === 'string' && item.trim().length > 0).map((item) => item.trim());
2799
4053
  }
4054
+ function uniqueStrings(values) {
4055
+ const seen = new Set();
4056
+ const out = [];
4057
+ for (const value of values) {
4058
+ const normalized = value.toLowerCase();
4059
+ if (!normalized || seen.has(normalized))
4060
+ continue;
4061
+ seen.add(normalized);
4062
+ out.push(value);
4063
+ }
4064
+ return out;
4065
+ }
4066
+ function metadataStringRecordArray(value) {
4067
+ if (!value || typeof value !== 'object' || Array.isArray(value))
4068
+ return undefined;
4069
+ const out = {};
4070
+ for (const [key, raw] of Object.entries(value)) {
4071
+ const values = metadataStringArray(raw).slice(0, 24);
4072
+ if (key.trim() && values.length > 0)
4073
+ out[key.trim()] = values;
4074
+ }
4075
+ return Object.keys(out).length > 0 ? out : undefined;
4076
+ }
2800
4077
  function metadataPayloadString(object, key) {
2801
4078
  return stringValue(object.payload?.[key]);
2802
4079
  }
@@ -2967,6 +4244,25 @@ function rowToQueryRun(row) {
2967
4244
  createdAt: row.created_at,
2968
4245
  };
2969
4246
  }
4247
+ function rowToRuntimeValueMatch(row, terms) {
4248
+ const normalized = row.normalized_value;
4249
+ const exactBoost = terms.some((term) => term === normalized) ? 1 : 0;
4250
+ const containsBoost = terms.some((term) => normalized.includes(term) || term.includes(normalized)) ? 0.5 : 0;
4251
+ const rankScore = row.rank ? 1 / (1 + Math.max(0, row.rank)) : 1;
4252
+ return {
4253
+ valueKey: row.value_key,
4254
+ relation: row.relation,
4255
+ schema: row.schema_name ?? undefined,
4256
+ tableName: row.table_name ?? undefined,
4257
+ columnName: row.column_name,
4258
+ columnType: row.column_type ?? undefined,
4259
+ value: row.value,
4260
+ normalizedValue: row.normalized_value,
4261
+ source: row.source ?? undefined,
4262
+ capturedAt: row.captured_at,
4263
+ score: Number((rankScore + exactBoost + containsBoost).toFixed(3)),
4264
+ };
4265
+ }
2970
4266
  function safeJson(raw, fallback) {
2971
4267
  if (!raw)
2972
4268
  return fallback;
@@ -2994,22 +4290,61 @@ function rankMetadataObjects(args) {
2994
4290
  })
2995
4291
  .sort((a, b) => b.score - a.score || objectPriority(a.row) - objectPriority(b.row) || a.row.name.localeCompare(b.row.name))
2996
4292
  .map((item, index) => ({ ...item, rank: index + 1 }));
2997
- const cutoff = ranked[Math.max(0, args.limit - 1)];
4293
+ const selectedRanked = selectRankedMetadataObjects(ranked, args.limit);
4294
+ const selectedKeys = new Set(selectedRanked.map((item) => item.row.objectKey));
4295
+ const rejectedRanked = ranked.filter((item) => !selectedKeys.has(item.row.objectKey));
4296
+ const cutoff = selectedRanked.at(-1);
2998
4297
  return {
2999
- selected: ranked.slice(0, args.limit).map((item) => item.row),
4298
+ selected: selectedRanked.map((item) => item.row),
3000
4299
  ranked,
3001
- rejected: ranked.slice(args.limit).slice(0, 24).map((item) => ({
4300
+ rejected: rejectedRanked.slice(0, 24).map((item) => ({
3002
4301
  objectKey: item.row.objectKey,
3003
4302
  objectType: item.row.objectType,
3004
4303
  name: item.row.name,
3005
4304
  reason: cutoff
3006
- ? `Lower retrieval score than selected context window (cutoff ${cutoff.score.toFixed(1)}); ${item.reason}`
4305
+ ? `Outside balanced context window (cutoff ${cutoff.score.toFixed(1)}); ${item.reason}`
3007
4306
  : item.reason,
3008
4307
  score: item.score,
3009
4308
  rejectedRank: item.rank,
3010
4309
  })),
3011
4310
  };
3012
4311
  }
4312
+ function selectRankedMetadataObjects(ranked, limit) {
4313
+ if (ranked.length <= limit)
4314
+ return ranked;
4315
+ const columnCount = ranked.filter((item) => COLUMN_OBJECT_TYPES.has(item.row.objectType)).length;
4316
+ const nonColumnCount = ranked.length - columnCount;
4317
+ if (columnCount === 0 || nonColumnCount === 0)
4318
+ return ranked.slice(0, limit);
4319
+ const columnCap = Math.max(4, Math.floor(limit * 0.35));
4320
+ const selected = [];
4321
+ const deferredColumns = [];
4322
+ let selectedColumns = 0;
4323
+ for (const item of ranked) {
4324
+ if (selected.length >= limit)
4325
+ break;
4326
+ if (COLUMN_OBJECT_TYPES.has(item.row.objectType)) {
4327
+ if (selectedColumns >= columnCap) {
4328
+ deferredColumns.push(item);
4329
+ continue;
4330
+ }
4331
+ selectedColumns += 1;
4332
+ }
4333
+ selected.push(item);
4334
+ }
4335
+ if (selected.length < limit) {
4336
+ const selectedKeys = new Set(selected.map((item) => item.row.objectKey));
4337
+ for (const item of [...deferredColumns, ...ranked]) {
4338
+ if (selected.length >= limit)
4339
+ break;
4340
+ if (selectedKeys.has(item.row.objectKey))
4341
+ continue;
4342
+ selected.push(item);
4343
+ selectedKeys.add(item.row.objectKey);
4344
+ }
4345
+ }
4346
+ return selected.sort((a, b) => a.rank - b.rank);
4347
+ }
3013
4348
  function scoreMetadataObject(row, terms) {
3014
4349
  let score = row.score ? row.score * 10 : 0;
3015
4350
  score += Math.max(0, 44 - objectPriority(row) * 2);
@@ -3023,6 +4358,8 @@ function scoreMetadataObject(row, terms) {
3023
4358
  score -= 16;
3024
4359
  if (row.objectType === 'semantic_metric')
3025
4360
  score += 10;
4361
+ if (row.objectType === 'runtime_value')
4362
+ score += 14;
3026
4363
  if (row.objectType === 'dbt_model' || row.objectType === 'dbt_column')
3027
4364
  score += 4;
3028
4365
  score += scoreText([
@@ -3045,6 +4382,8 @@ function priorityTier(row) {
3045
4382
  return 'certified_block';
3046
4383
  if (row.objectType === 'semantic_metric')
3047
4384
  return 'semantic_metric';
4385
+ if (row.objectType === 'runtime_value')
4386
+ return 'value_match';
3048
4387
  if (row.objectType === 'dql_term' || row.objectType === 'business_view')
3049
4388
  return 'business_context';
3050
4389
  if (row.objectType.startsWith('dbt_') || row.objectType === 'warehouse_table')
@@ -3066,11 +4405,15 @@ function selectionReason(row, score, plannerReasons = []) {
3066
4405
  function reasonForObject(row) {
3067
4406
  if (row.objectType === 'dql_block' && row.status === 'certified')
3068
4407
  return 'Certified reusable answer candidate';
4408
+ if (row.objectType === 'dql_block_output')
4409
+ return 'Certified block output column lineage';
3069
4410
  if (row.objectType === 'semantic_metric')
3070
4411
  return 'Semantic metric matched the question';
4412
+ if (row.objectType === 'runtime_value')
4413
+ return 'Observed runtime value matched the question literal';
3071
4414
  if (row.objectType === 'dql_term' || row.objectType === 'business_view')
3072
4415
  return 'DQL business context';
3073
- if (row.objectType.startsWith('dbt_') || row.objectType === 'warehouse_table')
4416
+ if (row.objectType.startsWith('dbt_') || row.objectType === 'warehouse_table' || row.objectType === 'warehouse_column')
3074
4417
  return 'dbt or warehouse metadata supplies physical context';
3075
4418
  if (row.objectType === 'app' || row.objectType === 'dashboard')
3076
4419
  return 'Published consumption context';
@@ -3084,6 +4427,13 @@ function mergeObjects(rows) {
3084
4427
  }
3085
4428
  return Array.from(byKey.values());
3086
4429
  }
4430
+ function mergeMetadataEdges(rows) {
4431
+ const byKey = new Map();
4432
+ for (const row of rows) {
4433
+ putEdge(byKey, row);
4434
+ }
4435
+ return Array.from(byKey.values());
4436
+ }
3087
4437
  function deriveTrust(objects) {
3088
4438
  if (objects.length === 0)
3089
4439
  return 'unknown';
@@ -3096,6 +4446,26 @@ function deriveTrust(objects) {
3096
4446
  return 'draft';
3097
4447
  return statuses.some((status) => status === 'certified') ? 'mixed' : 'unknown';
3098
4448
  }
4449
+ function withMetadataTrustLabelInfo(decision) {
4450
+ return {
4451
+ ...decision,
4452
+ trustLabelInfo: metadataTrustLabelInfo(decision.trustLabel),
4453
+ };
4454
+ }
4455
+ function metadataTrustLabelInfo(label) {
4456
+ switch (label) {
4457
+ case 'certified':
4458
+ return composeEffectiveTrust({ id: 'certified' });
4459
+ case 'conflict':
4460
+ return composeEffectiveTrust({ id: 'conflict' });
4461
+ case 'draft':
4462
+ return composeEffectiveTrust({ id: 'ai_generated' });
4463
+ case 'mixed':
4464
+ return composeEffectiveTrust({ id: 'ai_generated', existingQualifier: 'mixed context' });
4465
+ case 'unknown':
4466
+ return composeEffectiveTrust({ id: 'insufficient_context' });
4467
+ }
4468
+ }
3099
4469
  function buildCitations(objects, edges) {
3100
4470
  const citations = objects.slice(0, 24).map((row) => ({
3101
4471
  objectKey: row.objectKey,
@@ -3205,7 +4575,7 @@ function isGovernedCandidate(row) {
3205
4575
  * route scoped to questions the conflicting pair actually answers — a conflict
3206
4576
  * elsewhere in the project never hijacks an unrelated question.
3207
4577
  */
3208
- function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects) {
4578
+ function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects, question) {
3209
4579
  if (compileConflicts.length === 0)
3210
4580
  return undefined;
3211
4581
  // Names of the top candidate objects for this question (bounded — only the
@@ -3220,6 +4590,8 @@ function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects) {
3220
4590
  topNames.add(norm);
3221
4591
  }
3222
4592
  for (const conflict of compileConflicts) {
4593
+ if (!conflictMatchesQuestion(conflict, question))
4594
+ continue;
3223
4595
  const sideNames = conflict.sides.map((side) => normalizeConflictName(side.name)).filter(Boolean);
3224
4596
  if (sideNames.length < 2)
3225
4597
  continue;
@@ -3243,6 +4615,14 @@ function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects) {
3243
4615
  }
3244
4616
  return undefined;
3245
4617
  }
4618
+ function conflictMatchesQuestion(conflict, question) {
4619
+ const questionTokens = new Set(tokenize(question.replace(/_/g, ' ')));
4620
+ const conceptTokens = tokenize(conflict.concept.replace(/_/g, ' '));
4621
+ if (conceptTokens.length > 0 && conceptTokens.every((token) => questionTokens.has(token)))
4622
+ return true;
4623
+ const sideHits = conflict.sides.filter((side) => tokenize(side.name).some((token) => questionTokens.has(token))).length;
4624
+ return sideHits >= 2;
4625
+ }
3246
4626
  function normalizeConflictName(value) {
3247
4627
  return (value ?? '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
3248
4628
  }
@@ -3267,7 +4647,7 @@ function sanitizeFtsQuery(raw) {
3267
4647
  .map((term) => term.replace(/[^\p{L}\p{N}_]/gu, ''))
3268
4648
  .filter((term) => term.length > 1 && !STOP_WORDS.has(term.toLowerCase()))
3269
4649
  .slice(0, 48)
3270
- .map((term) => `"${term}"`)
4650
+ .map((term) => `"${term}"*`)
3271
4651
  .join(' OR ');
3272
4652
  }
3273
4653
  //# sourceMappingURL=catalog.js.map