@duckcodeailabs/dql-agent 1.6.33 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/dist/agent-run-engine.d.ts +39 -8
  2. package/dist/agent-run-engine.d.ts.map +1 -1
  3. package/dist/agent-run-engine.js +169 -10
  4. package/dist/agent-run-engine.js.map +1 -1
  5. package/dist/agent-run-gates.d.ts.map +1 -1
  6. package/dist/agent-run-gates.js +21 -3
  7. package/dist/agent-run-gates.js.map +1 -1
  8. package/dist/agent-run-planner.d.ts.map +1 -1
  9. package/dist/agent-run-planner.js +14 -3
  10. package/dist/agent-run-planner.js.map +1 -1
  11. package/dist/agentic/answer-contract.d.ts +37 -0
  12. package/dist/agentic/answer-contract.d.ts.map +1 -0
  13. package/dist/agentic/answer-contract.js +47 -0
  14. package/dist/agentic/answer-contract.js.map +1 -0
  15. package/dist/agentic/tool-loop.d.ts +48 -0
  16. package/dist/agentic/tool-loop.d.ts.map +1 -0
  17. package/dist/agentic/tool-loop.js +181 -0
  18. package/dist/agentic/tool-loop.js.map +1 -0
  19. package/dist/agentic/toolset.d.ts +40 -0
  20. package/dist/agentic/toolset.d.ts.map +1 -0
  21. package/dist/agentic/toolset.js +265 -0
  22. package/dist/agentic/toolset.js.map +1 -0
  23. package/dist/answer-loop.d.ts +82 -6
  24. package/dist/answer-loop.d.ts.map +1 -1
  25. package/dist/answer-loop.js +799 -112
  26. package/dist/answer-loop.js.map +1 -1
  27. package/dist/app-builder.d.ts.map +1 -1
  28. package/dist/app-builder.js +18 -10
  29. package/dist/app-builder.js.map +1 -1
  30. package/dist/cascade/budgets.d.ts +20 -2
  31. package/dist/cascade/budgets.d.ts.map +1 -1
  32. package/dist/cascade/budgets.js +83 -45
  33. package/dist/cascade/budgets.js.map +1 -1
  34. package/dist/cascade/route-policy.d.ts +1 -1
  35. package/dist/cascade/route-policy.d.ts.map +1 -1
  36. package/dist/cascade/route-policy.js +6 -0
  37. package/dist/cascade/route-policy.js.map +1 -1
  38. package/dist/domain-context.d.ts +30 -0
  39. package/dist/domain-context.d.ts.map +1 -0
  40. package/dist/domain-context.js +68 -0
  41. package/dist/domain-context.js.map +1 -0
  42. package/dist/embeddings/provider.d.ts +67 -4
  43. package/dist/embeddings/provider.d.ts.map +1 -1
  44. package/dist/embeddings/provider.js +167 -6
  45. package/dist/embeddings/provider.js.map +1 -1
  46. package/dist/hints/correction-eval.d.ts +30 -0
  47. package/dist/hints/correction-eval.d.ts.map +1 -0
  48. package/dist/hints/correction-eval.js +63 -0
  49. package/dist/hints/correction-eval.js.map +1 -0
  50. package/dist/hints/git-store.d.ts +39 -3
  51. package/dist/hints/git-store.d.ts.map +1 -1
  52. package/dist/hints/git-store.js +172 -2
  53. package/dist/hints/git-store.js.map +1 -1
  54. package/dist/hints/staleness.d.ts +30 -0
  55. package/dist/hints/staleness.d.ts.map +1 -0
  56. package/dist/hints/staleness.js +38 -0
  57. package/dist/hints/staleness.js.map +1 -0
  58. package/dist/hints/store.d.ts.map +1 -1
  59. package/dist/hints/store.js +3 -32
  60. package/dist/hints/store.js.map +1 -1
  61. package/dist/hints/types.d.ts +41 -0
  62. package/dist/hints/types.d.ts.map +1 -1
  63. package/dist/hints/types.js +1 -0
  64. package/dist/hints/types.js.map +1 -1
  65. package/dist/index.d.ts +46 -10
  66. package/dist/index.d.ts.map +1 -1
  67. package/dist/index.js +180 -9
  68. package/dist/index.js.map +1 -1
  69. package/dist/intent-controller.d.ts.map +1 -1
  70. package/dist/intent-controller.js +15 -3
  71. package/dist/intent-controller.js.map +1 -1
  72. package/dist/kg/build.d.ts.map +1 -1
  73. package/dist/kg/build.js +196 -4
  74. package/dist/kg/build.js.map +1 -1
  75. package/dist/kg/sqlite-fts.d.ts +13 -0
  76. package/dist/kg/sqlite-fts.d.ts.map +1 -1
  77. package/dist/kg/sqlite-fts.js +82 -49
  78. package/dist/kg/sqlite-fts.js.map +1 -1
  79. package/dist/kg/types.d.ts +9 -2
  80. package/dist/kg/types.d.ts.map +1 -1
  81. package/dist/memory/fts-query.d.ts +37 -4
  82. package/dist/memory/fts-query.d.ts.map +1 -1
  83. package/dist/memory/fts-query.js +53 -12
  84. package/dist/memory/fts-query.js.map +1 -1
  85. package/dist/metadata/analysis-planner.d.ts.map +1 -1
  86. package/dist/metadata/analysis-planner.js +179 -28
  87. package/dist/metadata/analysis-planner.js.map +1 -1
  88. package/dist/metadata/analytical-policy.d.ts +51 -0
  89. package/dist/metadata/analytical-policy.d.ts.map +1 -0
  90. package/dist/metadata/analytical-policy.js +309 -0
  91. package/dist/metadata/analytical-policy.js.map +1 -0
  92. package/dist/metadata/block-adapt.d.ts +37 -0
  93. package/dist/metadata/block-adapt.d.ts.map +1 -0
  94. package/dist/metadata/block-adapt.js +89 -0
  95. package/dist/metadata/block-adapt.js.map +1 -0
  96. package/dist/metadata/block-fit.js +37 -4
  97. package/dist/metadata/block-fit.js.map +1 -1
  98. package/dist/metadata/catalog.d.ts +85 -1
  99. package/dist/metadata/catalog.d.ts.map +1 -1
  100. package/dist/metadata/catalog.js +674 -92
  101. package/dist/metadata/catalog.js.map +1 -1
  102. package/dist/metadata/dbt-first-safety.d.ts +19 -0
  103. package/dist/metadata/dbt-first-safety.d.ts.map +1 -0
  104. package/dist/metadata/dbt-first-safety.js +17 -0
  105. package/dist/metadata/dbt-first-safety.js.map +1 -0
  106. package/dist/metadata/example-match.d.ts +30 -0
  107. package/dist/metadata/example-match.d.ts.map +1 -0
  108. package/dist/metadata/example-match.js +69 -0
  109. package/dist/metadata/example-match.js.map +1 -0
  110. package/dist/metadata/grain-ledger.d.ts +51 -0
  111. package/dist/metadata/grain-ledger.d.ts.map +1 -0
  112. package/dist/metadata/grain-ledger.js +174 -0
  113. package/dist/metadata/grain-ledger.js.map +1 -0
  114. package/dist/metadata/metric-match.d.ts +21 -1
  115. package/dist/metadata/metric-match.d.ts.map +1 -1
  116. package/dist/metadata/metric-match.js +157 -21
  117. package/dist/metadata/metric-match.js.map +1 -1
  118. package/dist/metadata/sql-context-validation.d.ts.map +1 -1
  119. package/dist/metadata/sql-context-validation.js +42 -2
  120. package/dist/metadata/sql-context-validation.js.map +1 -1
  121. package/dist/metadata/sql-grounding.d.ts +4 -2
  122. package/dist/metadata/sql-grounding.d.ts.map +1 -1
  123. package/dist/metadata/sql-grounding.js +29 -4
  124. package/dist/metadata/sql-grounding.js.map +1 -1
  125. package/dist/metadata/sql-retrieval.d.ts +5 -2
  126. package/dist/metadata/sql-retrieval.d.ts.map +1 -1
  127. package/dist/metadata/sql-retrieval.js +28 -6
  128. package/dist/metadata/sql-retrieval.js.map +1 -1
  129. package/dist/propose/build-from-prompt.d.ts +34 -0
  130. package/dist/propose/build-from-prompt.d.ts.map +1 -1
  131. package/dist/propose/build-from-prompt.js +205 -19
  132. package/dist/propose/build-from-prompt.js.map +1 -1
  133. package/dist/propose/failure-analysis.d.ts +48 -0
  134. package/dist/propose/failure-analysis.d.ts.map +1 -0
  135. package/dist/propose/failure-analysis.js +71 -0
  136. package/dist/propose/failure-analysis.js.map +1 -0
  137. package/dist/propose/join-mining.d.ts +18 -0
  138. package/dist/propose/join-mining.d.ts.map +1 -0
  139. package/dist/propose/join-mining.js +74 -0
  140. package/dist/propose/join-mining.js.map +1 -0
  141. package/dist/providers/claude.d.ts.map +1 -1
  142. package/dist/providers/claude.js +36 -1
  143. package/dist/providers/claude.js.map +1 -1
  144. package/dist/providers/index.d.ts +2 -2
  145. package/dist/providers/index.d.ts.map +1 -1
  146. package/dist/providers/index.js +1 -1
  147. package/dist/providers/index.js.map +1 -1
  148. package/dist/providers/openai.d.ts.map +1 -1
  149. package/dist/providers/openai.js +34 -1
  150. package/dist/providers/openai.js.map +1 -1
  151. package/dist/providers/reasoning-effort.d.ts +27 -0
  152. package/dist/providers/reasoning-effort.d.ts.map +1 -1
  153. package/dist/providers/reasoning-effort.js +32 -0
  154. package/dist/providers/reasoning-effort.js.map +1 -1
  155. package/dist/providers/types.d.ts +2 -1
  156. package/dist/providers/types.d.ts.map +1 -1
  157. package/dist/research-loop.d.ts.map +1 -1
  158. package/dist/research-loop.js +9 -2
  159. package/dist/research-loop.js.map +1 -1
  160. package/dist/semantic-bridge/compose.d.ts.map +1 -1
  161. package/dist/semantic-bridge/compose.js +316 -25
  162. package/dist/semantic-bridge/compose.js.map +1 -1
  163. package/dist/skills/bootstrap.d.ts +57 -0
  164. package/dist/skills/bootstrap.d.ts.map +1 -0
  165. package/dist/skills/bootstrap.js +260 -0
  166. package/dist/skills/bootstrap.js.map +1 -0
  167. package/dist/skills/defaults.d.ts +2 -2
  168. package/dist/skills/defaults.js +2 -2
  169. package/dist/skills/domain-skills.d.ts +24 -0
  170. package/dist/skills/domain-skills.d.ts.map +1 -0
  171. package/dist/skills/domain-skills.js +76 -0
  172. package/dist/skills/domain-skills.js.map +1 -0
  173. package/dist/skills/loader.d.ts +56 -8
  174. package/dist/skills/loader.d.ts.map +1 -1
  175. package/dist/skills/loader.js +257 -41
  176. package/dist/skills/loader.js.map +1 -1
  177. package/dist/synthesize.d.ts.map +1 -1
  178. package/dist/synthesize.js +152 -16
  179. package/dist/synthesize.js.map +1 -1
  180. package/dist/tools/registry.d.ts +1 -1
  181. package/dist/tools/registry.d.ts.map +1 -1
  182. package/dist/tools/registry.js +54 -12
  183. package/dist/tools/registry.js.map +1 -1
  184. package/package.json +4 -4
@@ -19,6 +19,10 @@ import { buildRuntimeValueIndex, normalizeValueIndexText, } from '../grounding/v
19
19
  import { grainMatches, requestedGrainFromPlan, } from './grain-gate.js';
20
20
  import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit, } from './block-fit.js';
21
21
  import { retrieveScopedHints } from '../hints/retrieval.js';
22
+ import { buildFtsMatch, sanitizeFtsQuery } from '../memory/fts-query.js';
23
+ import { envEmbeddingProvider } from '../embeddings/provider.js';
24
+ import { matchExampleParaphrase } from './example-match.js';
25
+ import { loadSkills, selectRelevantSkills } from '../skills/loader.js';
22
26
  const require = createRequire(import.meta.url);
23
27
  let databaseCtor = null;
24
28
  function loadDatabase() {
@@ -27,6 +31,39 @@ function loadDatabase() {
27
31
  }
28
32
  const SCHEMA_SHAPE_CACHE_LIMIT = 64;
29
33
  const schemaShapeCandidateCache = new Map();
34
+ /**
35
+ * Extract single-column `unique` dbt tests, keyed by model name (W5.3). A column
36
+ * a `unique` test guards is a grain key, which feeds the grain ledger's fan-out
37
+ * detection — the common way real dbt repos declare keys. Handles both the
38
+ * `attached_node` (newer dbt) and `depends_on` shapes, and top-level or kwargs
39
+ * column names.
40
+ */
41
+ export function extractDbtUniqueColumns(nodes) {
42
+ const byModel = new Map();
43
+ for (const node of Object.values(nodes)) {
44
+ if (node.resource_type !== 'test')
45
+ continue;
46
+ const testMetadata = node.test_metadata;
47
+ if (stringValue(testMetadata?.name) !== 'unique')
48
+ continue;
49
+ const column = stringValue(node.column_name) ?? stringValue(testMetadata?.kwargs?.column_name);
50
+ if (!column)
51
+ continue;
52
+ const dependsOn = node.depends_on?.nodes;
53
+ const attached = stringValue(node.attached_node)
54
+ ?? (Array.isArray(dependsOn) ? dependsOn.map(String).find((ref) => ref.startsWith('model.')) : undefined);
55
+ if (!attached)
56
+ continue;
57
+ const modelName = attached.split('.').at(-1);
58
+ if (!modelName)
59
+ continue;
60
+ const existing = byModel.get(modelName) ?? [];
61
+ if (!existing.includes(column))
62
+ existing.push(column);
63
+ byModel.set(modelName, existing);
64
+ }
65
+ return byModel;
66
+ }
30
67
  const OBJECT_PRIORITY = {
31
68
  dql_block: 1,
32
69
  dql_block_output: 1.5,
@@ -60,7 +97,8 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
60
97
  ? options.semanticLayer ?? undefined
61
98
  : loadAgentSemanticLayer(projectRoot);
62
99
  const manifest = options.manifest ?? loadAgentManifest(projectRoot);
63
- const snapshot = buildMetadataSnapshot(projectRoot, manifest, semanticLayer);
100
+ const skills = options.skills ?? loadSkills(projectRoot).skills;
101
+ const snapshot = buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skills);
64
102
  const catalog = openMetadataCatalog(projectRoot);
65
103
  try {
66
104
  const existing = catalog.state('fingerprint');
@@ -74,7 +112,15 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
74
112
  fingerprint: snapshot.fingerprint,
75
113
  };
76
114
  }
77
- catalog.rebuild(snapshot);
115
+ // W3.4 — incremental reindex when prior state exists (only re-tokenizes changed
116
+ // sources' FTS); `force` does a clean full rebuild. Proven equal to a full
117
+ // rebuild by incremental-reindex.test.ts.
118
+ if (options.force) {
119
+ catalog.rebuild(snapshot);
120
+ }
121
+ else {
122
+ catalog.rebuildIncremental(snapshot);
123
+ }
78
124
  return {
79
125
  path: defaultMetadataPath(projectRoot),
80
126
  refreshed: true,
@@ -106,7 +152,18 @@ export function upsertMetadataSnapshot(projectRoot, snapshot) {
106
152
  }
107
153
  }
108
154
  export async function buildLocalContextPack(projectRoot, request) {
109
- await ensureMetadataCatalogFresh(projectRoot);
155
+ let prepared = false;
156
+ if (request.preparedMetadataFingerprint) {
157
+ const catalog = openMetadataCatalog(projectRoot);
158
+ try {
159
+ prepared = catalog.state('fingerprint') === request.preparedMetadataFingerprint;
160
+ }
161
+ finally {
162
+ catalog.close();
163
+ }
164
+ }
165
+ if (!prepared)
166
+ await ensureMetadataCatalogFresh(projectRoot);
110
167
  const catalog = openMetadataCatalog(projectRoot);
111
168
  try {
112
169
  const mode = request.mode ?? 'question';
@@ -119,7 +176,7 @@ export async function buildLocalContextPack(projectRoot, request) {
119
176
  // candidates only; a SHIFT ignores it. A metadata fingerprint mismatch
120
177
  // always disqualifies reuse.
121
178
  const reusePolicy = request.reusePolicy ?? 'seed';
122
- const priorPack = reusePolicy !== 'off'
179
+ const priorPack = !request.domainContext && reusePolicy !== 'off'
123
180
  && request.priorContextPackId
124
181
  && request.conversationTopicRelation
125
182
  && request.conversationTopicRelation !== 'shift'
@@ -167,18 +224,24 @@ export async function buildLocalContextPack(projectRoot, request) {
167
224
  buildFollowUpSearchQuery(request.question, followUp),
168
225
  ...questionPlan.searchQueries,
169
226
  ]);
170
- const searchRows = mergeObjects(searchQueries.flatMap((query) => catalog.searchObjects({
227
+ const scopeObjects = (rows) => filterMetadataObjectsByDomainContext(rows, request.domainContext);
228
+ // Skill guidance is injected only through selectContextPackSkills below.
229
+ // Leaving it in generic FTS results would bypass status/domain/area/exclusion
230
+ // eligibility simply because a word in its body matched the question.
231
+ const retrievalObjects = (rows) => scopeObjects(rows).filter((row) => row.objectType !== 'skill');
232
+ const searchRows = retrievalObjects(mergeObjects(searchQueries.flatMap((query) => catalog.searchObjects({
171
233
  query,
172
234
  objectTypes: request.objectTypes,
173
235
  limit: Math.max(request.limit ?? 80, 20),
174
- })));
236
+ }))));
175
237
  const schemaShapeCandidates = schemaShapeCandidateObjects(catalog, questionPlan, request, mergeObjects([...runtimeObjects, ...runtimeValueObjects]));
176
238
  const schemaShapeObjects = schemaShapeCandidates.map((candidate) => candidate.object);
177
- const exact = request.focusObjectKey ? catalog.getObject(request.focusObjectKey) : null;
239
+ const exactCandidate = request.focusObjectKey ? catalog.getObject(request.focusObjectKey) : null;
240
+ const exact = exactCandidate && retrievalObjects([exactCandidate]).length > 0 ? exactCandidate : null;
178
241
  const ranked = rankMetadataObjects({
179
- rows: mergeObjects(exact
242
+ rows: retrievalObjects(mergeObjects(exact
180
243
  ? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]
181
- : [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]),
244
+ : [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects])),
182
245
  question: searchQueries.join(' '),
183
246
  questionPlan,
184
247
  limit: request.limit ?? 80,
@@ -192,9 +255,9 @@ export async function buildLocalContextPack(projectRoot, request) {
192
255
  const focusObjectKey = request.focusObjectKey ?? selected[0]?.objectKey ?? null;
193
256
  const edgeWalk = catalog.edgesForKeys(selected.map((row) => row.objectKey), 3);
194
257
  const edgeObjectKeys = Array.from(new Set(edgeWalk.flatMap((edge) => [edge.fromKey, edge.toKey])));
195
- const graphObjects = catalog.getObjectsByKeys(edgeObjectKeys);
258
+ const graphObjects = retrievalObjects(catalog.getObjectsByKeys(edgeObjectKeys));
196
259
  const rankedObjects = rankMetadataObjects({
197
- rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects]),
260
+ rows: retrievalObjects(mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects])),
198
261
  question: searchQueries.join(' '),
199
262
  questionPlan,
200
263
  limit: request.limit ?? 120,
@@ -204,24 +267,30 @@ export async function buildLocalContextPack(projectRoot, request) {
204
267
  // entire relation set ("send everything, let the agent decide"). Only when the
205
268
  // whole catalog fits comfortably in context — otherwise we keep ranked selection.
206
269
  const fullCatalogObjects = request.strictness === 'exploratory'
207
- ? collectFullCatalogObjects(catalog)
270
+ ? retrievalObjects(collectFullCatalogObjects(catalog) ?? [])
208
271
  : undefined;
209
272
  const usedFullCatalog = Boolean(fullCatalogObjects);
273
+ // Skills are selected from the catalog snapshot with the same hard domain,
274
+ // area, status, and exclusion gates as the loader. They are not left to FTS
275
+ // chance or re-read from disk after the snapshot has been fingerprinted.
276
+ const selectedSkills = selectContextPackSkills(catalog.listObjects({ objectTypes: ['skill'], limit: 500 }), request.question, request.domainContext);
277
+ const selectedSkillObjects = selectedSkills.map((item) => item.object);
210
278
  const objects = fullCatalogObjects
211
- ? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects])
212
- : mergeObjects([...rankedObjects, ...sqlParentObjects]);
279
+ ? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects, ...selectedSkillObjects])
280
+ : mergeObjects([...rankedObjects, ...sqlParentObjects, ...selectedSkillObjects]);
213
281
  const objectKeys = objects.map((row) => row.objectKey);
282
+ const allowedObjectKeys = new Set(objectKeys);
214
283
  const contextEdges = mergeMetadataEdges([
215
284
  ...edgeWalk,
216
285
  ...catalog.edgesForKeys(objectKeys, 2),
217
- ]);
286
+ ]).filter((edge) => allowedObjectKeys.has(edge.fromKey) && allowedObjectKeys.has(edge.toKey));
218
287
  const queryRuns = catalog.queryRunsForObjectKeys(objectKeys, 20);
219
288
  const diagnostics = catalog.diagnostics();
220
289
  const warnings = buildWarnings(diagnostics, objects);
221
290
  const trustLabel = deriveTrust(objects);
222
291
  const citations = buildCitations(objects, contextEdges);
223
292
  const evidenceSummaries = buildEvidenceSummaries(objects, contextEdges, queryRuns, diagnostics);
224
- const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, contextEdges), questionPlan);
293
+ const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, contextEdges, tokenizeQuestionForColumns(request.question)), questionPlan);
225
294
  const selectedRelations = allowedSqlContext.relations.slice(0, 24).map((relation, index) => {
226
295
  const scored = scoreAllowedSqlRelationWithAnalysisPlan(relation, questionPlan);
227
296
  return {
@@ -237,7 +306,7 @@ export async function buildLocalContextPack(projectRoot, request) {
237
306
  const selectedJoinPaths = buildSelectedJoinPaths(allowedSqlContext, contextEdges);
238
307
  const evidenceRoles = buildEvidenceRoles(objects, queryRuns);
239
308
  const reranked = rankMetadataObjects({
240
- rows: mergeObjects([...searchRows, ...schemaShapeObjects, ...objects]),
309
+ rows: retrievalObjects(mergeObjects([...searchRows, ...schemaShapeObjects, ...objects])),
241
310
  question: searchQueries.join(' '),
242
311
  questionPlan,
243
312
  limit: request.limit ?? 120,
@@ -279,6 +348,7 @@ export async function buildLocalContextPack(projectRoot, request) {
279
348
  trustLabel,
280
349
  trustLabelInfo: metadataTrustLabelInfo(trustLabel),
281
350
  objects,
351
+ skills: selectedSkills.map((item) => item.skill),
282
352
  edges: contextEdges,
283
353
  queryRuns,
284
354
  citations,
@@ -362,7 +432,17 @@ export function recordRuntimeSchemaSnapshot(projectRoot, snapshot) {
362
432
  catalog.close();
363
433
  }
364
434
  }
365
- export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer) {
435
+ /** The most-recent stored live-warehouse schema snapshot for a project, or null. */
436
+ export function latestRuntimeSchemaSnapshotForProject(projectRoot) {
437
+ const catalog = openMetadataCatalog(projectRoot);
438
+ try {
439
+ return catalog.latestRuntimeSchemaSnapshot();
440
+ }
441
+ finally {
442
+ catalog.close();
443
+ }
444
+ }
445
+ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skills = loadSkills(projectRoot).skills) {
366
446
  const manifestGraph = buildKGFromManifest(manifest);
367
447
  const semanticGraph = buildKGFromSemanticLayer(semanticLayer);
368
448
  const objects = new Map();
@@ -387,11 +467,16 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer) {
387
467
  }
388
468
  }
389
469
  addManifestBlockDetails(manifest, objects);
470
+ addSkillObjects(skills, objects, edges);
390
471
  addDbtDagObjects(manifest, objects, edges, diagnostics);
391
472
  addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges, diagnostics);
392
473
  addBlockDependencyEdges(manifest, edges);
393
474
  addBlockOutputLineageObjects(manifest, objects, edges);
394
- addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagnostics);
475
+ // Manifest v3 is the unified DQL runtime. DataLex is migration input only;
476
+ // retaining it here would reintroduce a second analytical source of truth.
477
+ if (manifest.manifestVersion !== 3) {
478
+ addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagnostics);
479
+ }
395
480
  const nodeKeyMap = new Map();
396
481
  for (const node of [...manifestGraph.nodes, ...semanticGraph.nodes]) {
397
482
  nodeKeyMap.set(node.nodeId, objectKeyFromKGNode(node));
@@ -643,23 +728,130 @@ export class MetadataCatalog {
643
728
  });
644
729
  txn();
645
730
  }
731
+ /**
732
+ * Incremental reindex (W3.4). Only re-inserts objects + FTS rows for sources
733
+ * whose per-source fingerprint changed (or vanished); unchanged sources' rows are
734
+ * left in place (skipping the expensive FTS re-tokenization at scale). Edges,
735
+ * diagnostics, domain shards, and sourceless objects are always rebuilt because
736
+ * they cross source boundaries. Provably equal to a full rebuild for the semantic
737
+ * content (object keys/types/names/domains/payloads + FTS + edges); only the
738
+ * `updated_at`/`built_at` timestamps of untouched rows differ. Falls back to a
739
+ * full rebuild when there is no prior fingerprint state.
740
+ */
741
+ rebuildIncremental(snapshot) {
742
+ const stored = new Map();
743
+ for (const row of this.db.prepare('SELECT source_path, fingerprint FROM metadata_source_fingerprints').all()) {
744
+ stored.set(row.source_path, row.fingerprint);
745
+ }
746
+ if (stored.size === 0) {
747
+ this.rebuild(snapshot);
748
+ return { mode: 'full', changedSources: 0 };
749
+ }
750
+ const now = new Date().toISOString();
751
+ const incoming = buildSourceFingerprints(snapshot.objects, now);
752
+ const incomingSources = new Set(incoming.map((f) => f.sourcePath));
753
+ const changedSources = new Set();
754
+ for (const fingerprint of incoming) {
755
+ if (stored.get(fingerprint.sourcePath) !== fingerprint.fingerprint)
756
+ changedSources.add(fingerprint.sourcePath);
757
+ }
758
+ const removedSources = [...stored.keys()].filter((source) => !incomingSources.has(source));
759
+ const sourceOf = (object) => object.sourcePath ?? object.sourceSystem ?? null;
760
+ const domainShards = buildDomainShards(snapshot.objects, now);
761
+ const insertObject = this.db.prepare(`
762
+ INSERT INTO metadata_objects (
763
+ object_key, object_type, name, full_name, domain, owner, status,
764
+ description, source_path, source_system, payload_json, updated_at
765
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
766
+ const insertFts = this.db.prepare(`
767
+ INSERT INTO metadata_fts (object_key, name, full_name, description, domain, owner, payload)
768
+ VALUES (?, ?, ?, ?, ?, ?, ?)`);
769
+ const insertEdge = this.db.prepare(`
770
+ INSERT OR IGNORE INTO metadata_edges (edge_type, from_key, to_key, confidence, payload_json, created_at)
771
+ VALUES (?, ?, ?, ?, ?, ?)`);
772
+ const insertDiagnostic = this.db.prepare(`
773
+ INSERT INTO metadata_diagnostics (id, kind, severity, message, object_key, file_path, created_at)
774
+ VALUES (?, ?, ?, ?, ?, ?, ?)`);
775
+ const insertSourceFingerprint = this.db.prepare(`
776
+ INSERT OR REPLACE INTO metadata_source_fingerprints (source_path, fingerprint, object_count, updated_at)
777
+ VALUES (?, ?, ?, ?)`);
778
+ const insertDomainShard = this.db.prepare(`
779
+ INSERT OR REPLACE INTO metadata_domain_shards (
780
+ domain, object_count, block_count, certified_block_count,
781
+ semantic_metric_count, dbt_object_count, updated_at
782
+ ) VALUES (?, ?, ?, ?, ?, ?, ?)`);
783
+ const setState = this.db.prepare('INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)');
784
+ const deleteFtsForSource = this.db.prepare('DELETE FROM metadata_fts WHERE object_key IN (SELECT object_key FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?)');
785
+ const deleteObjectsForSource = this.db.prepare('DELETE FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?');
786
+ const txn = this.db.transaction(() => {
787
+ // Drop changed + removed sources' rows, and ALL sourceless rows (never
788
+ // fingerprinted, so always refreshed).
789
+ for (const source of [...changedSources, ...removedSources]) {
790
+ deleteFtsForSource.run(source);
791
+ deleteObjectsForSource.run(source);
792
+ }
793
+ this.db.prepare('DELETE FROM metadata_fts WHERE object_key IN (SELECT object_key FROM metadata_objects WHERE source_path IS NULL AND source_system IS NULL)').run();
794
+ this.db.prepare('DELETE FROM metadata_objects WHERE source_path IS NULL AND source_system IS NULL').run();
795
+ // Re-insert only objects from changed sources or sourceless; unchanged
796
+ // sources' rows are left untouched.
797
+ for (const object of snapshot.objects) {
798
+ const source = sourceOf(object);
799
+ if (source !== null && !changedSources.has(source))
800
+ continue;
801
+ const payload = object.payload ?? {};
802
+ insertObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, object.domain ?? null, object.owner ?? null, object.status ?? null, object.description ?? null, object.sourcePath ?? null, object.sourceSystem ?? null, JSON.stringify(payload), object.updatedAt ?? now);
803
+ insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', JSON.stringify(payload));
804
+ }
805
+ // Cross-source tables: rebuild fully.
806
+ this.db.prepare('DELETE FROM metadata_edges').run();
807
+ for (const edge of snapshot.edges) {
808
+ insertEdge.run(edge.edgeType, edge.fromKey, edge.toKey, edge.confidence ?? 1, JSON.stringify(edge.payload ?? {}), now);
809
+ }
810
+ this.db.prepare('DELETE FROM metadata_diagnostics').run();
811
+ for (const diagnostic of snapshot.diagnostics) {
812
+ insertDiagnostic.run(diagnosticId(diagnostic), diagnostic.kind, diagnostic.severity, diagnostic.message, diagnostic.objectKey ?? null, diagnostic.filePath ?? null, now);
813
+ }
814
+ this.db.prepare('DELETE FROM metadata_domain_shards').run();
815
+ for (const shard of domainShards) {
816
+ insertDomainShard.run(shard.domain, shard.objectCount, shard.blockCount, shard.certifiedBlockCount, shard.semanticMetricCount, shard.dbtObjectCount, shard.updatedAt);
817
+ }
818
+ for (const source of removedSources) {
819
+ this.db.prepare('DELETE FROM metadata_source_fingerprints WHERE source_path = ?').run(source);
820
+ }
821
+ for (const fingerprint of incoming) {
822
+ insertSourceFingerprint.run(fingerprint.sourcePath, fingerprint.fingerprint, fingerprint.objectCount, fingerprint.updatedAt);
823
+ }
824
+ setState.run('built_at', now);
825
+ setState.run('fingerprint', snapshot.fingerprint);
826
+ setState.run('project_root', snapshot.projectRoot);
827
+ setState.run('object_count', String(snapshot.objects.length));
828
+ setState.run('edge_count', String(snapshot.edges.length));
829
+ setState.run('source_fingerprint_count', String(incoming.length));
830
+ setState.run('domain_shard_count', String(domainShards.length));
831
+ setState.run('diagnostics_json', JSON.stringify(snapshot.diagnostics));
832
+ setState.run('compile_conflicts_json', JSON.stringify(snapshot.compileConflicts ?? []));
833
+ setState.run('manifest_generated_at', snapshot.manifest.generatedAt);
834
+ });
835
+ txn();
836
+ return { mode: 'incremental', changedSources: changedSources.size };
837
+ }
646
838
  searchObjects(options) {
647
839
  const { query, objectTypes, domain, limit = 40 } = options;
648
- const sanitized = sanitizeFtsQuery(query);
649
- if (!sanitized)
840
+ const match = buildFtsMatch(query, { prefix: true });
841
+ if (!match.or)
650
842
  return this.listObjects({ objectTypes, domain, limit });
651
843
  const filters = [];
652
- const params = [sanitized];
844
+ const extraParams = [];
653
845
  if (objectTypes && objectTypes.length > 0) {
654
846
  filters.push(`o.object_type IN (${objectTypes.map(() => '?').join(', ')})`);
655
- params.push(...objectTypes);
847
+ extraParams.push(...objectTypes);
656
848
  }
657
849
  if (domain) {
658
850
  filters.push('o.domain = ?');
659
- params.push(domain);
851
+ extraParams.push(domain);
660
852
  }
661
853
  const whereExtra = filters.length > 0 ? ` AND ${filters.join(' AND ')}` : '';
662
- const rows = this.db.prepare(`
854
+ const runMatch = (matchExpr) => this.db.prepare(`
663
855
  SELECT o.*,
664
856
  bm25(metadata_fts) AS rank,
665
857
  snippet(metadata_fts, -1, '<mark>', '</mark>', '...', 12) AS snip
@@ -668,7 +860,15 @@ export class MetadataCatalog {
668
860
  WHERE metadata_fts MATCH ?${whereExtra}
669
861
  ORDER BY rank
670
862
  LIMIT ?
671
- `).all(...params, limit);
863
+ `).all(matchExpr, ...extraParams, limit);
864
+ // Precision-first, recall-preserving UNION: all-terms-co-occur (AND) matches
865
+ // lead, then OR-of-terms matches fill the remaining budget. A doc mentioning
866
+ // only one term (a relevant context column) still surfaces, while a doc
867
+ // containing every term ranks ahead of it. Dedup by object_key.
868
+ const andRows = match.and ? runMatch(match.and) : [];
869
+ const seen = new Set(andRows.map((row) => row.object_key));
870
+ const orRows = runMatch(match.or).filter((row) => !seen.has(row.object_key));
871
+ const rows = [...andRows, ...orRows].slice(0, limit);
672
872
  return rows.map((row) => ({
673
873
  ...rowToObject(row),
674
874
  score: row.rank ? 1 / (1 + Math.max(0, row.rank)) : 1,
@@ -839,6 +1039,14 @@ export class MetadataCatalog {
839
1039
  `);
840
1040
  const txn = this.db.transaction(() => {
841
1041
  insertSnapshot.run(id, cleanSnapshot.source ?? null, JSON.stringify(cleanSnapshot), capturedAt);
1042
+ // Only the most-recent snapshot is ever read; rows otherwise accumulate one per
1043
+ // question forever. Keep a small margin and prune the rest (P7).
1044
+ this.db.prepare(`
1045
+ DELETE FROM runtime_schema_snapshots
1046
+ WHERE id NOT IN (
1047
+ SELECT id FROM runtime_schema_snapshots ORDER BY captured_at DESC LIMIT 5
1048
+ )
1049
+ `).run();
842
1050
  this.db.prepare('DELETE FROM runtime_value_fts').run();
843
1051
  this.db.prepare('DELETE FROM runtime_value_index').run();
844
1052
  for (const entry of valueEntries) {
@@ -853,7 +1061,9 @@ export class MetadataCatalog {
853
1061
  }
854
1062
  searchRuntimeValues(terms, limit = 40) {
855
1063
  const cleanTerms = uniqueStrings(terms.map(normalizeValueIndexText).filter(Boolean)).slice(0, 8);
856
- const query = sanitizeFtsQuery(cleanTerms.join(' '));
1064
+ // Filter VALUES may legitimately be stop words (a category named "current");
1065
+ // fall back to raw tokens so value binding still fires rather than dropping to empty.
1066
+ const query = sanitizeFtsQuery(cleanTerms.join(' '), { prefix: true, fallbackToRawTokens: true });
857
1067
  if (!query)
858
1068
  return [];
859
1069
  const rows = this.db.prepare(`
@@ -1028,6 +1238,7 @@ function objectFromKGNode(node) {
1028
1238
  dimensions: node.dimensions ?? [],
1029
1239
  allowedFilters: node.allowedFilters ?? [],
1030
1240
  parameterPolicy: node.parameterPolicy ?? [],
1241
+ parameters: node.parameters ?? [],
1031
1242
  filterBindings: node.filterBindings ?? [],
1032
1243
  sourceSystems: node.sourceSystems ?? [],
1033
1244
  replacementFor: node.replacementFor ?? [],
@@ -1107,6 +1318,71 @@ function normalizeEdge(edge, fromKey, toKey) {
1107
1318
  payload: { kgSource: edge.src, kgTarget: edge.dst },
1108
1319
  };
1109
1320
  }
1321
+ /**
1322
+ * Skills are source-owned context, not a prompt-only side channel. Persist their
1323
+ * parsed fields in the catalog snapshot so selection, provenance, and
1324
+ * fingerprinting all observe the same immutable project state.
1325
+ */
1326
+ function addSkillObjects(skills, objects, edges) {
1327
+ for (const skill of skills) {
1328
+ const identity = skill.qualifiedId ?? skill.id;
1329
+ const objectKey = `skill:${identity}`;
1330
+ const domains = uniqueNonBlank([skill.domain, ...(skill.domains ?? [])]);
1331
+ const object = {
1332
+ objectKey,
1333
+ objectType: 'skill',
1334
+ name: skill.id,
1335
+ fullName: skill.qualifiedId ?? skill.id,
1336
+ domain: skill.domain,
1337
+ owner: skill.owner,
1338
+ status: skill.status ?? 'active',
1339
+ description: skill.description ?? compactSkillDescription(skill.body),
1340
+ sourcePath: skill.sourcePath,
1341
+ sourceSystem: 'DQL domain skill',
1342
+ payload: compactObject({
1343
+ skillId: skill.id,
1344
+ localId: skill.localId ?? skill.id,
1345
+ qualifiedId: skill.qualifiedId,
1346
+ scope: skill.scope,
1347
+ user: skill.user,
1348
+ domains,
1349
+ modelAreaRefs: skill.modelAreaRefs ?? [],
1350
+ kind: skill.kind,
1351
+ triggers: skill.triggers ?? [],
1352
+ exclusions: skill.exclusions ?? [],
1353
+ preferredMetrics: skill.preferredMetrics,
1354
+ preferredBlocks: skill.preferredBlocks,
1355
+ preferredDimensions: skill.preferredDimensions ?? [],
1356
+ requiredFilters: skill.requiredFilters ?? [],
1357
+ clarifyWhen: skill.clarifyWhen ?? [],
1358
+ examples: skill.examples ?? [],
1359
+ sourceRefs: skill.sourceRefs ?? [],
1360
+ vocabulary: skill.vocabulary,
1361
+ body: skill.body,
1362
+ isStarter: skill.isStarter,
1363
+ provenance: 'DQL domain skill',
1364
+ }),
1365
+ };
1366
+ objects.set(objectKey, object);
1367
+ for (const domain of domains) {
1368
+ const edge = {
1369
+ edgeType: 'contains',
1370
+ fromKey: `domain:${domain}`,
1371
+ toKey: objectKey,
1372
+ confidence: 1,
1373
+ payload: { provenance: 'DQL domain skill' },
1374
+ };
1375
+ edges.set(`${edge.edgeType}\u0000${edge.fromKey}\u0000${edge.toKey}`, edge);
1376
+ }
1377
+ }
1378
+ }
1379
+ function compactSkillDescription(body) {
1380
+ const normalized = body.replace(/\s+/g, ' ').trim();
1381
+ return normalized ? normalized.slice(0, 480) : undefined;
1382
+ }
1383
+ function uniqueNonBlank(values) {
1384
+ return Array.from(new Set(values.map((value) => value?.trim()).filter((value) => Boolean(value))));
1385
+ }
1110
1386
  function addDbtDagObjects(manifest, objects, edges, diagnostics) {
1111
1387
  const models = manifest.dbtImport?.dbtDag?.models ?? [];
1112
1388
  for (const model of models) {
@@ -1197,7 +1473,20 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
1197
1473
  return;
1198
1474
  }
1199
1475
  const catalogColumns = loadRawDbtCatalogColumns(join(dirname(manifestPath), 'catalog.json'));
1476
+ const uniqueColumnsByModel = extractDbtUniqueColumns(raw.nodes ?? {});
1200
1477
  const entries = [];
1478
+ // PERF-001: at enterprise scale, keep columns in their model payload and
1479
+ // load full node detail lazily from the immutable dbt artifact cache. A
1480
+ // separate FTS row + graph edge for every column makes a 300k-column project
1481
+ // consume gigabytes without improving the bounded model-first route.
1482
+ const rawEntries = [...Object.values(raw.nodes ?? {}), ...Object.values(raw.sources ?? {})];
1483
+ const totalColumnCount = rawEntries.reduce((sum, node) => {
1484
+ const columns = node.columns;
1485
+ return sum + (columns && typeof columns === 'object' && !Array.isArray(columns)
1486
+ ? Object.keys(columns).length
1487
+ : 0);
1488
+ }, 0);
1489
+ const includeColumnObjects = totalColumnCount <= 50_000;
1201
1490
  for (const [uniqueId, node] of Object.entries(raw.nodes ?? {})) {
1202
1491
  if (node.resource_type !== 'model')
1203
1492
  continue;
@@ -1216,12 +1505,14 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
1216
1505
  database,
1217
1506
  schema,
1218
1507
  catalogColumns: catalogColumns.get(uniqueId),
1508
+ uniqueColumns: uniqueColumnsByModel.get(name),
1219
1509
  };
1220
1510
  entries.push(entry);
1221
1511
  addRawDbtCatalogObject({
1222
1512
  ...entry,
1223
1513
  objects,
1224
1514
  edges,
1515
+ includeColumnObjects,
1225
1516
  });
1226
1517
  }
1227
1518
  for (const [uniqueId, source] of Object.entries(raw.sources ?? {})) {
@@ -1246,6 +1537,7 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
1246
1537
  ...entry,
1247
1538
  objects,
1248
1539
  edges,
1540
+ includeColumnObjects,
1249
1541
  });
1250
1542
  }
1251
1543
  const relationLookup = buildRawDbtRelationLookup(entries);
@@ -1285,38 +1577,41 @@ function addRawDbtCatalogObject(input) {
1285
1577
  catalogOnly: existing ? undefined : true,
1286
1578
  columnCompleteness,
1287
1579
  columns,
1580
+ // W5.3 — single-column `unique` dbt tests become grain-ledger keys.
1581
+ uniqueColumns: input.uniqueColumns && input.uniqueColumns.length > 0 ? input.uniqueColumns : undefined,
1288
1582
  }),
1289
1583
  }));
1290
- for (const column of columns) {
1291
- const columnKey = `dbt:column:${input.name}.${column.name}`;
1292
- const existingColumn = input.objects.get(columnKey);
1293
- input.objects.set(columnKey, mergeObject(existingColumn, {
1294
- objectKey: columnKey,
1295
- objectType: 'dbt_column',
1296
- name: column.name,
1297
- fullName: `${input.name}.${column.name}`,
1298
- description: column.description,
1299
- status: existingColumn?.status ?? 'dbt_catalog',
1300
- sourcePath: stringValue(input.node.original_file_path) ?? stringValue(input.node.path),
1301
- sourceSystem: existingColumn?.sourceSystem ?? 'dbt manifest.json catalog',
1302
- payload: compactObject({
1303
- ...(existingColumn?.payload ?? {}),
1304
- model: input.name,
1305
- uniqueId: input.uniqueId,
1306
- type: column.type,
1307
- relation,
1308
- columnCompleteness: 'partial',
1309
- catalogOnly: existingColumn ? undefined : true,
1310
- }),
1311
- }));
1312
- putEdge(input.edges, {
1313
- edgeType: 'contains',
1314
- fromKey: input.objectKey,
1315
- toKey: columnKey,
1316
- confidence: 1,
1317
- payload: { source: 'raw dbt manifest column catalog' },
1318
- });
1319
- }
1584
+ if (input.includeColumnObjects)
1585
+ for (const column of columns) {
1586
+ const columnKey = `dbt:column:${input.name}.${column.name}`;
1587
+ const existingColumn = input.objects.get(columnKey);
1588
+ input.objects.set(columnKey, mergeObject(existingColumn, {
1589
+ objectKey: columnKey,
1590
+ objectType: 'dbt_column',
1591
+ name: column.name,
1592
+ fullName: `${input.name}.${column.name}`,
1593
+ description: column.description,
1594
+ status: existingColumn?.status ?? 'dbt_catalog',
1595
+ sourcePath: stringValue(input.node.original_file_path) ?? stringValue(input.node.path),
1596
+ sourceSystem: existingColumn?.sourceSystem ?? 'dbt manifest.json catalog',
1597
+ payload: compactObject({
1598
+ ...(existingColumn?.payload ?? {}),
1599
+ model: input.name,
1600
+ uniqueId: input.uniqueId,
1601
+ type: column.type,
1602
+ relation,
1603
+ columnCompleteness: 'partial',
1604
+ catalogOnly: existingColumn ? undefined : true,
1605
+ }),
1606
+ }));
1607
+ putEdge(input.edges, {
1608
+ edgeType: 'contains',
1609
+ fromKey: input.objectKey,
1610
+ toKey: columnKey,
1611
+ confidence: 1,
1612
+ payload: { source: 'raw dbt manifest column catalog' },
1613
+ });
1614
+ }
1320
1615
  for (const dep of rawDbtDependsOn(input.node)) {
1321
1616
  putEdge(input.edges, {
1322
1617
  edgeType: 'depends_on',
@@ -1537,6 +1832,7 @@ function addManifestBlockDetails(manifest, objects) {
1537
1832
  blockType: block.blockType,
1538
1833
  tests: block.tests,
1539
1834
  parameterPolicy: block.parameterPolicy,
1835
+ parameters: block.parameters,
1540
1836
  filterBindings: block.filterBindings,
1541
1837
  sqlFingerprints: buildBlockSqlFingerprints(block.sql),
1542
1838
  businessFingerprint: buildBlockBusinessFingerprint({
@@ -1659,6 +1955,10 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
1659
1955
  payload: compactObject({
1660
1956
  tags: entity.tags ?? [],
1661
1957
  binding: entity.binding,
1958
+ // Entity-level grain + keys (W5.2) feed the grain ledger for fan-out safety.
1959
+ grain: entity.grain,
1960
+ candidateKeys: entity.candidate_keys,
1961
+ businessKeys: entity.business_keys,
1662
1962
  fields: (entity.fields ?? []).slice(0, 100).map((field) => compactObject({
1663
1963
  name: field.name,
1664
1964
  type: field.type,
@@ -1744,6 +2044,58 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
1744
2044
  }),
1745
2045
  });
1746
2046
  }
2047
+ // Concept-to-physical conformance (W5.1) → searchable `datalex_concept` nodes
2048
+ // carrying the canonical key + the physical relations that realize the concept,
2049
+ // plus `datalex_conforms` edges so retrieval can collapse several physical tables
2050
+ // into one business concept ("Customer" → dim_customer, stg_customers) and reason
2051
+ // at the concept grain. Modeling metadata only — no certification coupling.
2052
+ for (const conformance of raw.conformance ?? []) {
2053
+ if (!conformance?.concept)
2054
+ continue;
2055
+ const conceptDomain = conformance.domain ?? '';
2056
+ const conceptKey = `datalex:concept:${conceptDomain}.${conformance.concept}`;
2057
+ const physicalRefs = (conformance.physical ?? [])
2058
+ .map((physical) => (physical.binding ? objectKeyForDataLexBinding(physical.binding) : undefined))
2059
+ .filter((key) => Boolean(key));
2060
+ const canonicalKey = conformance.canonical_key ?? [];
2061
+ objects.set(conceptKey, mergeObject(objects.get(conceptKey), {
2062
+ objectKey: conceptKey,
2063
+ objectType: 'datalex_concept',
2064
+ name: conformance.concept,
2065
+ fullName: `${conceptDomain}.${conformance.concept}`,
2066
+ domain: conceptDomain || undefined,
2067
+ status: 'contract_evidence',
2068
+ description: `Business concept ${conformance.concept}`
2069
+ + (canonicalKey.length ? ` (canonical key: ${canonicalKey.join(', ')})` : '')
2070
+ + (physicalRefs.length ? `, realized by ${physicalRefs.length} physical relation(s).` : '.'),
2071
+ sourcePath: manifestPath,
2072
+ sourceSystem: 'DataLex manifest',
2073
+ payload: compactObject({
2074
+ canonicalKey,
2075
+ businessKey: conformance.business_key ?? [],
2076
+ implements: conformance.implements ?? [],
2077
+ physicalRefs,
2078
+ }),
2079
+ }));
2080
+ if (conceptDomain) {
2081
+ putEdge(edges, {
2082
+ edgeType: 'contains',
2083
+ fromKey: `datalex:domain:${conceptDomain}`,
2084
+ toKey: conceptKey,
2085
+ confidence: 1,
2086
+ payload: { source: 'datalex conformance' },
2087
+ });
2088
+ }
2089
+ for (const ref of physicalRefs) {
2090
+ putEdge(edges, {
2091
+ edgeType: 'datalex_conforms',
2092
+ fromKey: conceptKey,
2093
+ toKey: ref,
2094
+ confidence: 0.95,
2095
+ payload: compactObject({ source: 'datalex conformance', concept: conformance.concept, canonicalKey }),
2096
+ });
2097
+ }
2098
+ }
1747
2099
  for (const block of Object.values(manifest.blocks ?? {})) {
1748
2100
  if (!block.datalexContract)
1749
2101
  continue;
@@ -2056,15 +2408,62 @@ async function planContextPackRoute(input) {
2056
2408
  const exactByApplicability = [...applicabilityByKey.values()]
2057
2409
  .filter((item) => item.kind === 'exact_answer' || item.kind === 'safe_parameterized')
2058
2410
  .sort((a, b) => b.score - a.score)[0];
2059
- const exact = exactByApplicability
2411
+ // Analytical questions intentionally keep certified applicability conservative:
2412
+ // a lexical/content match alone must not terminate the cascade. A certified
2413
+ // block whose declared/inferred output contract fully covers the requested
2414
+ // answer shape is different, however — that is a proven Tier 1 answer even
2415
+ // when the planner also knows how to generate SQL. Promote only exact/trim-safe
2416
+ // contract fits here; near matches remain context for Tier 2.
2417
+ const exactByContract = input.objects
2418
+ .filter((object) => object.objectType === 'dql_block'
2419
+ && isCertifiedMetadataObject(object)
2420
+ && hasCompatibleMetadataRankingDirection(input.request.question, object))
2421
+ .map((object) => ({
2422
+ object,
2423
+ fit: evaluateCertifiedBlockFit({
2424
+ question: input.request.question,
2425
+ plan: input.questionPlan,
2426
+ block: object,
2427
+ exactExampleMatch: hasExactExampleQuestion(input.request.question, object),
2428
+ definitionLookup: intent === 'definition_lookup',
2429
+ }),
2430
+ applicabilityScore: applicabilityByKey.get(object.objectKey)?.score ?? 0,
2431
+ }))
2432
+ .filter(({ fit }) => (fit.kind === 'exact' || fit.kind === 'trim_safe')
2433
+ && (fit.confidence === 'high' || fit.confidence === 'medium'))
2434
+ .sort((a, b) => (b.fit.confidence === 'high' ? 1 : 0) - (a.fit.confidence === 'high' ? 1 : 0)
2435
+ || b.applicabilityScore - a.applicabilityScore)[0]?.object;
2436
+ let exact = exactByApplicability
2060
2437
  ? input.objects.find((object) => object.objectKey === exactByApplicability.objectKey)
2061
- : findExactCertifiedObject(input.request.question, intent, input.objects);
2062
- const certifiedApplicability = exact
2063
- ? applicabilityByKey.get(exact.objectKey) ?? certifiedApplicabilityForObject(exact, input.questionPlan)
2064
- : undefined;
2438
+ : findExactCertifiedObject(input.request.question, intent, input.objects) ?? exactByContract;
2065
2439
  const contextApplicability = [...applicabilityByKey.values()]
2066
2440
  .filter((item) => item.kind === 'context_only')
2067
2441
  .sort((a, b) => b.score - a.score)[0];
2442
+ // W2.1 — paraphrase promotion. When no block string/lexically matched, promote a
2443
+ // top context-only certified candidate whose example QUESTION the user paraphrased
2444
+ // (semantic cosine + direction) to the certified candidate. It still runs the full
2445
+ // shape/grain fit below (paraphraseExampleMatch is kept OUT of directCertifiedBypass,
2446
+ // the fit's exactExampleMatch, and the grain-gate skip) — so paraphrase never bypasses
2447
+ // grain the way a user naming the block directly does.
2448
+ let paraphraseExampleMatch = false;
2449
+ if (!exact && contextApplicability) {
2450
+ const candidate = input.objects.find((object) => object.objectKey === contextApplicability.objectKey);
2451
+ if (candidate && isCertifiedMetadataObject(candidate)
2452
+ && await matchExampleParaphrase(input.request.question, candidate, envEmbeddingProvider())) {
2453
+ exact = candidate;
2454
+ paraphraseExampleMatch = true;
2455
+ }
2456
+ }
2457
+ // Keep the strongest context-only certified candidate attached to the route
2458
+ // so the contract and grain gates can explain exactly why it is demoted.
2459
+ // This does not promote a loose match: both gates still have to pass before
2460
+ // Tier 1 is possible.
2461
+ if (!exact && contextApplicability) {
2462
+ exact = input.objects.find((object) => object.objectKey === contextApplicability.objectKey);
2463
+ }
2464
+ const certifiedApplicability = exact
2465
+ ? applicabilityByKey.get(exact.objectKey) ?? certifiedApplicabilityForObject(exact, input.questionPlan)
2466
+ : undefined;
2068
2467
  const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
2069
2468
  const directCertifiedBypass = Boolean(exact && (exactExampleMatch
2070
2469
  || intent === 'definition_lookup'
@@ -2127,6 +2526,8 @@ async function planContextPackRoute(input) {
2127
2526
  || intent === 'exact_certified_lookup'
2128
2527
  || intent === 'definition_lookup'
2129
2528
  || exactExampleMatch
2529
+ || paraphraseExampleMatch
2530
+ || (blockFit && certifiedFitAllowsTier1(blockFit) && hasCompatibleMetadataRankingDirection(input.request.question, exact))
2130
2531
  || (intent === 'ad_hoc_ranking' && objectNameInQuestion(input.request.question, exact)))) {
2131
2532
  return {
2132
2533
  route: 'certified',
@@ -2152,7 +2553,7 @@ async function planContextPackRoute(input) {
2152
2553
  route: 'generated_sql',
2153
2554
  intent: isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking',
2154
2555
  reason: `Certified block "${exact.name}" is close but answers a different grain than the question, so it is context only and SQL is generated for the requested grain.`,
2155
- routeReason: grainGate.reason,
2556
+ routeReason: `${grainGate.reason}; routed to Tier 2 generated SQL`,
2156
2557
  grainGate: grainGateInfo,
2157
2558
  blockFit,
2158
2559
  trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
@@ -2170,7 +2571,7 @@ async function planContextPackRoute(input) {
2170
2571
  route: 'generated_sql',
2171
2572
  intent: isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking',
2172
2573
  reason: `Certified block "${exact.name}" is relevant but does not satisfy the requested answer shape, so it is context only and SQL is generated for the requested result.`,
2173
- routeReason: blockFit.reasons.join('; '),
2574
+ routeReason: `${blockFit.reasons.join('; ')}; routed to Tier 2 generated SQL`,
2174
2575
  grainGate: grainGateInfo,
2175
2576
  blockFit,
2176
2577
  trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
@@ -2523,7 +2924,8 @@ function hasCompatibleMetadataRankingDirection(question, object) {
2523
2924
  const objectDirection = rankingDirection([
2524
2925
  object.name,
2525
2926
  object.description ?? '',
2526
- JSON.stringify(object.payload ?? {}),
2927
+ Array.isArray(object.payload?.tags) ? object.payload.tags.join(' ') : '',
2928
+ typeof object.payload?.sql === 'string' ? object.payload.sql : '',
2527
2929
  ].join(' '));
2528
2930
  if (!objectDirection)
2529
2931
  return true;
@@ -3006,6 +3408,55 @@ function isSqlParentObject(object) {
3006
3408
  object.objectType === 'warehouse_table' ||
3007
3409
  object.objectType === 'runtime_table';
3008
3410
  }
3411
+ /** Column budget for a single relation in the prompt's allowed-SQL context. */
3412
+ const MAX_ALLOWED_SQL_COLUMNS = 120;
3413
+ /**
3414
+ * Relevance score for keeping a column when a wide relation must be truncated (W3.3).
3415
+ * Structural columns (join keys, time/grain) always score so joins and grain survive;
3416
+ * a column whose name overlaps the question scores highest so the needed column on a
3417
+ * 300-column table is not dropped just because it sits past the budget cut.
3418
+ */
3419
+ function columnBudgetScore(name, questionTokens) {
3420
+ const lower = name.toLowerCase();
3421
+ let score = 0;
3422
+ if (/(^|_)(id|key)$/.test(lower) || lower === 'id')
3423
+ score += 3;
3424
+ if (/(date|time|_at$|_ts$|day|month|year|week|quarter)/.test(lower))
3425
+ score += 2;
3426
+ if (questionTokens) {
3427
+ for (const token of questionTokens) {
3428
+ if (token.length > 2 && lower.includes(token)) {
3429
+ score += 5;
3430
+ break;
3431
+ }
3432
+ }
3433
+ }
3434
+ return score;
3435
+ }
3436
+ /**
3437
+ * Cap a relation's column list to the prompt budget and downgrade completeness to
3438
+ * 'partial' when columns are dropped. Keeping a truncated relation 'complete' would
3439
+ * let column validation false-positive a valid column past the cut as unknown_column
3440
+ * (the >120-column latent bug, W1.4). When truncating, keep the columns most relevant
3441
+ * to the question + structural keys (W3.3) rather than the arbitrary first-N, then
3442
+ * restore original order among the survivors so prompt ordering is unchanged.
3443
+ */
3444
+ function capAllowedSqlColumns(columns, completeness, questionTokens) {
3445
+ if (columns.length <= MAX_ALLOWED_SQL_COLUMNS) {
3446
+ return { columns, columnCompleteness: completeness };
3447
+ }
3448
+ const kept = columns
3449
+ .map((column, index) => ({ column, index, score: columnBudgetScore(column.name, questionTokens) }))
3450
+ .sort((a, b) => (b.score - a.score) || (a.index - b.index))
3451
+ .slice(0, MAX_ALLOWED_SQL_COLUMNS)
3452
+ .sort((a, b) => a.index - b.index)
3453
+ .map((entry) => entry.column);
3454
+ return { columns: kept, columnCompleteness: 'partial' };
3455
+ }
3456
+ /** Lowercased word tokens from the question, for column-relevance ranking. */
3457
+ function tokenizeQuestionForColumns(question) {
3458
+ return (question.toLowerCase().match(/[a-z0-9_]+/g) ?? []).filter((token) => token.length > 2);
3459
+ }
3009
3460
  const FULL_CATALOG_RELATION_TYPES = ['dbt_model', 'dbt_source', 'warehouse_table', 'runtime_table'];
3010
3461
  const FULL_CATALOG_COLUMN_TYPES = ['dbt_column', 'warehouse_column', 'runtime_column'];
3011
3462
  /** Deep mode over a catalog this small hands the model everything instead of top-k. */
@@ -3032,7 +3483,7 @@ function collectFullCatalogObjects(catalog) {
3032
3483
  return undefined;
3033
3484
  return mergeObjects([...relations, ...columns]);
3034
3485
  }
3035
- function buildAllowedSqlContext(objects, edges) {
3486
+ export function buildAllowedSqlContext(objects, edges, questionTokens) {
3036
3487
  const byRelation = new Map();
3037
3488
  const objectsByKey = new Map(objects.map((object) => [object.objectKey, object]));
3038
3489
  const addRelation = (relation) => {
@@ -3041,16 +3492,17 @@ function buildAllowedSqlContext(objects, edges) {
3041
3492
  return;
3042
3493
  const existing = byRelation.get(key);
3043
3494
  if (!existing) {
3044
- byRelation.set(key, { ...relation, columns: dedupeRuntimeColumns(relation.columns).slice(0, 120) });
3495
+ const capped = capAllowedSqlColumns(dedupeRuntimeColumns(relation.columns), relation.columnCompleteness, questionTokens);
3496
+ byRelation.set(key, { ...relation, columns: capped.columns, columnCompleteness: capped.columnCompleteness });
3045
3497
  return;
3046
3498
  }
3047
- const columnCompleteness = mergeRelationCompletenessForCatalog(existing, relation);
3499
+ const capped = capAllowedSqlColumns(mergeRelationColumns(existing, relation), mergeRelationCompletenessForCatalog(existing, relation), questionTokens);
3048
3500
  byRelation.set(key, {
3049
3501
  ...existing,
3050
3502
  objectKey: existing.objectKey ?? relation.objectKey,
3051
3503
  source: mergeRelationSources(existing.source, relation.source),
3052
- columnCompleteness,
3053
- columns: mergeRelationColumns(existing, relation).slice(0, 120),
3504
+ columnCompleteness: capped.columnCompleteness,
3505
+ columns: capped.columns,
3054
3506
  });
3055
3507
  };
3056
3508
  for (const object of objects) {
@@ -3500,12 +3952,13 @@ function coalesceRelationAliases(relations) {
3500
3952
  continue;
3501
3953
  const target = targets[0];
3502
3954
  const targetKey = normalizeRelationKey(target.relation);
3955
+ const capped = capAllowedSqlColumns(mergeRelationColumns(target, relation), mergeRelationCompletenessForCatalog(target, relation));
3503
3956
  mergedByRelation.set(targetKey, {
3504
3957
  ...target,
3505
3958
  objectKey: target.objectKey ?? relation.objectKey,
3506
3959
  source: mergeRelationSources(target.source, relation.source),
3507
- columnCompleteness: mergeRelationCompletenessForCatalog(target, relation),
3508
- columns: mergeRelationColumns(target, relation).slice(0, 120),
3960
+ columnCompleteness: capped.columnCompleteness,
3961
+ columns: capped.columns,
3509
3962
  });
3510
3963
  skipped.add(relationKey);
3511
3964
  }
@@ -4273,10 +4726,39 @@ function safeJson(raw, fallback) {
4273
4726
  return fallback;
4274
4727
  }
4275
4728
  }
4729
+ /**
4730
+ * W3.2 — domain-affinity ranking boost for MULTI-DOMAIN catalogs. At scale
4731
+ * (thousands of models across domains), a question about one domain can be crowded
4732
+ * out of the top-k by lexically-similar objects from other domains. This infers the
4733
+ * dominant domain from the highest-scoring candidates and gives same-domain objects
4734
+ * a small additive bonus. It is RECALL-PRESERVING — it only reorders; every
4735
+ * candidate stays in the ranking, and the bonus (10% of the top score) is small
4736
+ * enough that a much stronger cross-domain signal still wins. No-op for single-domain
4737
+ * catalogs (fewer than 2 domains carry positive score).
4738
+ */
4739
+ export function applyDomainAffinityBoost(scored) {
4740
+ const domainScore = new Map();
4741
+ for (const item of scored) {
4742
+ const domain = item.row.domain;
4743
+ if (domain && item.score > 0)
4744
+ domainScore.set(domain, (domainScore.get(domain) ?? 0) + item.score);
4745
+ }
4746
+ if (domainScore.size < 2)
4747
+ return undefined;
4748
+ const [dominant] = [...domainScore.entries()].sort((a, b) => b[1] - a[1])[0];
4749
+ const maxScore = Math.max(0, ...scored.map((item) => item.score));
4750
+ const bonus = Number((0.1 * maxScore).toFixed(3));
4751
+ if (bonus <= 0)
4752
+ return undefined;
4753
+ for (const item of scored) {
4754
+ if (item.row.domain === dominant)
4755
+ item.score = Number((item.score + bonus).toFixed(3));
4756
+ }
4757
+ return dominant;
4758
+ }
4276
4759
  function rankMetadataObjects(args) {
4277
4760
  const terms = tokenize(args.question).slice(0, 12);
4278
- const ranked = mergeObjects(args.rows)
4279
- .map((row) => {
4761
+ const scored = mergeObjects(args.rows).map((row) => {
4280
4762
  const baseScore = scoreMetadataObject(row, terms);
4281
4763
  const planScore = args.questionPlan ? scoreMetadataObjectWithAnalysisPlan(row, args.questionPlan) : { score: 0, reasons: [] };
4282
4764
  const score = Number((baseScore + planScore.score).toFixed(3));
@@ -4287,7 +4769,9 @@ function rankMetadataObjects(args) {
4287
4769
  reason: selectionReason(row, score, planScore.reasons),
4288
4770
  priorityTier: priorityTier(row),
4289
4771
  };
4290
- })
4772
+ });
4773
+ applyDomainAffinityBoost(scored);
4774
+ const ranked = scored
4291
4775
  .sort((a, b) => b.score - a.score || objectPriority(a.row) - objectPriority(b.row) || a.row.name.localeCompare(b.row.name))
4292
4776
  .map((item, index) => ({ ...item, rank: index + 1 }));
4293
4777
  const selectedRanked = selectRankedMetadataObjects(ranked, args.limit);
@@ -4419,6 +4903,121 @@ function reasonForObject(row) {
4419
4903
  return 'Published consumption context';
4420
4904
  return 'Relevant project metadata';
4421
4905
  }
4906
+ function filterMetadataObjectsByDomainContext(rows, context) {
4907
+ if (!context?.activeDomain)
4908
+ return rows;
4909
+ const domains = new Set([
4910
+ context.activeDomain,
4911
+ ...context.ancestors,
4912
+ ...context.allowedImports.map((item) => item.providerDomain),
4913
+ ]);
4914
+ return rows.filter((row) => !row.domain || domains.has(row.domain));
4915
+ }
4916
+ function selectContextPackSkills(objects, question, context) {
4917
+ const byIdentity = new Map();
4918
+ const parsed = objects.flatMap((object) => {
4919
+ const skill = skillFromMetadataObject(object);
4920
+ if (!skill)
4921
+ return [];
4922
+ const identity = skill.qualifiedId ?? skill.id;
4923
+ byIdentity.set(identity, object);
4924
+ return [skill];
4925
+ });
4926
+ const domains = context?.activeDomain
4927
+ ? [context.activeDomain, ...context.ancestors, ...context.allowedImports.map((item) => item.providerDomain)]
4928
+ : [];
4929
+ const selected = selectRelevantSkills(parsed, question, {
4930
+ domains,
4931
+ modelAreaIds: context?.modelAreaId ? [context.modelAreaId] : [],
4932
+ });
4933
+ return selected.flatMap((skill) => {
4934
+ const object = byIdentity.get(skill.qualifiedId ?? skill.id);
4935
+ return object ? [{ object, skill: localContextSkillFromParsed(skill, object) }] : [];
4936
+ });
4937
+ }
4938
+ function skillFromMetadataObject(object) {
4939
+ if (object.objectType !== 'skill')
4940
+ return null;
4941
+ const payload = object.payload ?? {};
4942
+ const scope = payload.scope === 'personal' ? 'personal' : 'project';
4943
+ const status = payload.status === 'draft' || payload.status === 'deprecated' || payload.status === 'active'
4944
+ ? payload.status
4945
+ : object.status === 'draft' || object.status === 'deprecated' || object.status === 'active'
4946
+ ? object.status
4947
+ : 'active';
4948
+ return {
4949
+ id: stringPayload(payload.skillId) ?? object.name,
4950
+ localId: stringPayload(payload.localId) ?? object.name,
4951
+ qualifiedId: stringPayload(payload.qualifiedId) ?? object.fullName,
4952
+ scope,
4953
+ user: stringPayload(payload.user),
4954
+ domain: object.domain,
4955
+ domains: stringArrayPayload(payload.domains),
4956
+ modelAreaRefs: stringArrayPayload(payload.modelAreaRefs),
4957
+ kind: skillKindPayload(payload.kind),
4958
+ status,
4959
+ owner: object.owner,
4960
+ triggers: stringArrayPayload(payload.triggers),
4961
+ exclusions: stringArrayPayload(payload.exclusions),
4962
+ description: object.description,
4963
+ preferredMetrics: stringArrayPayload(payload.preferredMetrics),
4964
+ preferredBlocks: stringArrayPayload(payload.preferredBlocks),
4965
+ preferredDimensions: stringArrayPayload(payload.preferredDimensions),
4966
+ requiredFilters: stringArrayPayload(payload.requiredFilters),
4967
+ clarifyWhen: stringArrayPayload(payload.clarifyWhen),
4968
+ examples: stringArrayPayload(payload.examples),
4969
+ sourceRefs: stringArrayPayload(payload.sourceRefs),
4970
+ vocabulary: stringRecordPayload(payload.vocabulary),
4971
+ body: stringPayload(payload.body) ?? '',
4972
+ sourcePath: object.sourcePath ?? '',
4973
+ isStarter: payload.isStarter === true,
4974
+ };
4975
+ }
4976
+ function localContextSkillFromParsed(skill, object) {
4977
+ const guidance = skill.body.slice(0, 4_000);
4978
+ return {
4979
+ objectKey: object.objectKey,
4980
+ id: skill.id,
4981
+ qualifiedId: skill.qualifiedId,
4982
+ domain: skill.domain,
4983
+ domains: [...(skill.domains ?? [])],
4984
+ modelAreaRefs: [...(skill.modelAreaRefs ?? [])],
4985
+ kind: skill.kind,
4986
+ status: skill.status,
4987
+ owner: skill.owner,
4988
+ description: skill.description,
4989
+ triggers: [...(skill.triggers ?? [])],
4990
+ exclusions: [...(skill.exclusions ?? [])],
4991
+ preferredMetrics: [...skill.preferredMetrics],
4992
+ preferredBlocks: [...skill.preferredBlocks],
4993
+ preferredDimensions: [...(skill.preferredDimensions ?? [])],
4994
+ requiredFilters: [...(skill.requiredFilters ?? [])],
4995
+ clarifyWhen: [...(skill.clarifyWhen ?? [])],
4996
+ vocabulary: { ...skill.vocabulary },
4997
+ guidance,
4998
+ guidanceTruncated: guidance.length < skill.body.length,
4999
+ sourceRefs: [...(skill.sourceRefs ?? [])],
5000
+ provenance: stringPayload(object.payload?.provenance) ?? object.sourceSystem ?? 'DQL domain skill',
5001
+ sourcePath: object.sourcePath,
5002
+ };
5003
+ }
5004
+ function stringPayload(value) {
5005
+ return typeof value === 'string' && value.trim() ? value : undefined;
5006
+ }
5007
+ function stringArrayPayload(value) {
5008
+ return Array.isArray(value) ? value.filter((item) => typeof item === 'string' && Boolean(item.trim())) : [];
5009
+ }
5010
+ function stringRecordPayload(value) {
5011
+ if (!value || typeof value !== 'object' || Array.isArray(value))
5012
+ return {};
5013
+ return Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === 'string'));
5014
+ }
5015
+ function skillKindPayload(value) {
5016
+ return value === 'domain_reference' || value === 'metric_policy' || value === 'glossary'
5017
+ || value === 'analysis_pattern' || value === 'sql_policy' || value === 'custom'
5018
+ ? value
5019
+ : undefined;
5020
+ }
4422
5021
  function mergeObjects(rows) {
4423
5022
  const byKey = new Map();
4424
5023
  for (const row of rows) {
@@ -4633,21 +5232,4 @@ function scoreText(value, terms) {
4633
5232
  const lower = value.toLowerCase();
4634
5233
  return terms.reduce((score, term) => score + (lower.includes(term) ? 1 : 0), 0);
4635
5234
  }
4636
- const STOP_WORDS = new Set([
4637
- 'a', 'about', 'after', 'all', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'by',
4638
- 'can', 'could', 'did', 'do', 'does', 'down', 'for', 'from', 'give', 'had', 'has',
4639
- 'have', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'me', 'most', 'my', 'of',
4640
- 'on', 'or', 'please', 'query', 'show', 'sql', 'than', 'that', 'the', 'their',
4641
- 'them', 'this', 'to', 'up', 'using', 'was', 'we', 'were', 'what', 'when',
4642
- 'where', 'which', 'who', 'why', 'with', 'would', 'you', 'your',
4643
- ]);
4644
- function sanitizeFtsQuery(raw) {
4645
- return raw
4646
- .split(/\s+/)
4647
- .map((term) => term.replace(/[^\p{L}\p{N}_]/gu, ''))
4648
- .filter((term) => term.length > 1 && !STOP_WORDS.has(term.toLowerCase()))
4649
- .slice(0, 48)
4650
- .map((term) => `"${term}"*`)
4651
- .join(' OR ');
4652
- }
4653
5235
  //# sourceMappingURL=catalog.js.map