@duckcodeailabs/dql-agent 1.7.1 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/agent-run-engine.d.ts +2 -0
  2. package/dist/agent-run-engine.d.ts.map +1 -1
  3. package/dist/agent-run-engine.js +67 -19
  4. package/dist/agent-run-engine.js.map +1 -1
  5. package/dist/agent-run-gates.d.ts.map +1 -1
  6. package/dist/agent-run-gates.js +14 -50
  7. package/dist/agent-run-gates.js.map +1 -1
  8. package/dist/agent-run-planner.d.ts.map +1 -1
  9. package/dist/agent-run-planner.js +14 -4
  10. package/dist/agent-run-planner.js.map +1 -1
  11. package/dist/agentic/toolset.js +32 -2
  12. package/dist/agentic/toolset.js.map +1 -1
  13. package/dist/answer-loop.d.ts +24 -0
  14. package/dist/answer-loop.d.ts.map +1 -1
  15. package/dist/answer-loop.js +387 -55
  16. package/dist/answer-loop.js.map +1 -1
  17. package/dist/answer-shape.js +14 -1
  18. package/dist/answer-shape.js.map +1 -1
  19. package/dist/cascade/budgets.d.ts.map +1 -1
  20. package/dist/cascade/budgets.js +5 -2
  21. package/dist/cascade/budgets.js.map +1 -1
  22. package/dist/conversation/session-store.d.ts +3 -0
  23. package/dist/conversation/session-store.d.ts.map +1 -1
  24. package/dist/conversation/session-store.js +8 -3
  25. package/dist/conversation/session-store.js.map +1 -1
  26. package/dist/conversation/snapshot.d.ts +2 -0
  27. package/dist/conversation/snapshot.d.ts.map +1 -1
  28. package/dist/conversation/snapshot.js +1 -0
  29. package/dist/conversation/snapshot.js.map +1 -1
  30. package/dist/conversation/working-state.d.ts +3 -0
  31. package/dist/conversation/working-state.d.ts.map +1 -1
  32. package/dist/conversation/working-state.js +26 -0
  33. package/dist/conversation/working-state.js.map +1 -1
  34. package/dist/domain-context.d.ts +14 -0
  35. package/dist/domain-context.d.ts.map +1 -1
  36. package/dist/domain-context.js +10 -0
  37. package/dist/domain-context.js.map +1 -1
  38. package/dist/embeddings/provider.d.ts +3 -1
  39. package/dist/embeddings/provider.d.ts.map +1 -1
  40. package/dist/embeddings/provider.js +19 -8
  41. package/dist/embeddings/provider.js.map +1 -1
  42. package/dist/index.d.ts +6 -3
  43. package/dist/index.d.ts.map +1 -1
  44. package/dist/index.js +33 -4
  45. package/dist/index.js.map +1 -1
  46. package/dist/intent-controller.d.ts +15 -0
  47. package/dist/intent-controller.d.ts.map +1 -1
  48. package/dist/intent-controller.js +4 -1
  49. package/dist/intent-controller.js.map +1 -1
  50. package/dist/meaning-resolution.d.ts +110 -0
  51. package/dist/meaning-resolution.d.ts.map +1 -0
  52. package/dist/meaning-resolution.js +155 -0
  53. package/dist/meaning-resolution.js.map +1 -0
  54. package/dist/metadata/analysis-planner.d.ts +15 -1
  55. package/dist/metadata/analysis-planner.d.ts.map +1 -1
  56. package/dist/metadata/analysis-planner.js +169 -50
  57. package/dist/metadata/analysis-planner.js.map +1 -1
  58. package/dist/metadata/block-fit.d.ts.map +1 -1
  59. package/dist/metadata/block-fit.js +51 -5
  60. package/dist/metadata/block-fit.js.map +1 -1
  61. package/dist/metadata/catalog.d.ts +62 -3
  62. package/dist/metadata/catalog.d.ts.map +1 -1
  63. package/dist/metadata/catalog.js +911 -98
  64. package/dist/metadata/catalog.js.map +1 -1
  65. package/dist/metadata/grain-gate.js +27 -9
  66. package/dist/metadata/grain-gate.js.map +1 -1
  67. package/dist/metadata/meaning-evidence.d.ts +70 -0
  68. package/dist/metadata/meaning-evidence.d.ts.map +1 -0
  69. package/dist/metadata/meaning-evidence.js +333 -0
  70. package/dist/metadata/meaning-evidence.js.map +1 -0
  71. package/dist/metadata/metric-match.d.ts.map +1 -1
  72. package/dist/metadata/metric-match.js +5 -2
  73. package/dist/metadata/metric-match.js.map +1 -1
  74. package/dist/router.d.ts +16 -4
  75. package/dist/router.d.ts.map +1 -1
  76. package/dist/router.js +409 -21
  77. package/dist/router.js.map +1 -1
  78. package/dist/semantic-bridge/compose.d.ts +12 -0
  79. package/dist/semantic-bridge/compose.d.ts.map +1 -1
  80. package/dist/semantic-bridge/compose.js +177 -14
  81. package/dist/semantic-bridge/compose.js.map +1 -1
  82. package/dist/semantic-bridge/member-select.d.ts.map +1 -1
  83. package/dist/semantic-bridge/member-select.js +59 -10
  84. package/dist/semantic-bridge/member-select.js.map +1 -1
  85. package/dist/skills/loader.d.ts.map +1 -1
  86. package/dist/skills/loader.js +35 -33
  87. package/dist/skills/loader.js.map +1 -1
  88. package/dist/synthesize.d.ts +2 -0
  89. package/dist/synthesize.d.ts.map +1 -1
  90. package/dist/synthesize.js +84 -13
  91. package/dist/synthesize.js.map +1 -1
  92. package/package.json +4 -4
@@ -7,22 +7,24 @@
7
7
  * context pack before answering.
8
8
  */
9
9
  import { createHash } from 'node:crypto';
10
- import { mkdirSync, readFileSync } from 'node:fs';
10
+ import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
11
11
  import { createRequire } from 'node:module';
12
12
  import { dirname, join } from 'node:path';
13
- import { buildManifest, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, extractColumnLineage, normalizeDqlArtifactReference, composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
13
+ import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
14
+ import { buildManifest, buildManifestKnowledgeGraph, loadDomainPackageRegistry, loadManifestKnowledgeSkills, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, extractColumnLineage, normalizeDqlArtifactReference, composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
14
15
  import { buildKGFromManifest, buildKGFromSemanticLayer } from '../kg/build.js';
15
16
  import { buildBlockBusinessFingerprint, buildBlockSqlFingerprints } from './block-fingerprints.js';
16
17
  import { buildAnalysisQuestionPlan, certifiedApplicabilityForObject, scoreAllowedSqlRelationWithAnalysisPlan, scoreMetadataObjectWithAnalysisPlan, sortAllowedSqlContextForAnalysisPlan, } from './analysis-planner.js';
17
18
  import { extractSimpleSelectShape, sourceSqlShapeColumns } from './sql-shape.js';
18
- import { buildRuntimeValueIndex, normalizeValueIndexText, } from '../grounding/value-index.js';
19
19
  import { grainMatches, requestedGrainFromPlan, } from './grain-gate.js';
20
20
  import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit, } from './block-fit.js';
21
21
  import { retrieveScopedHints } from '../hints/retrieval.js';
22
- import { buildFtsMatch, sanitizeFtsQuery } from '../memory/fts-query.js';
22
+ import { buildFtsMatch } from '../memory/fts-query.js';
23
23
  import { envEmbeddingProvider } from '../embeddings/provider.js';
24
24
  import { matchExampleParaphrase } from './example-match.js';
25
25
  import { loadSkills, selectRelevantSkills } from '../skills/loader.js';
26
+ import { buildMeaningEvidencePackage, } from './meaning-evidence.js';
27
+ export { toAgentRetrievalEvidence } from './meaning-evidence.js';
26
28
  const require = createRequire(import.meta.url);
27
29
  let databaseCtor = null;
28
30
  function loadDatabase() {
@@ -73,6 +75,10 @@ const OBJECT_PRIORITY = {
73
75
  semantic_dimension: 5,
74
76
  semantic_measure: 6,
75
77
  semantic_entity: 7,
78
+ dql_entity: 7,
79
+ relationship: 7.2,
80
+ contract: 7.4,
81
+ model_area: 7.6,
76
82
  semantic_model: 8,
77
83
  dbt_model: 9,
78
84
  dbt_source: 10,
@@ -86,12 +92,143 @@ const OBJECT_PRIORITY = {
86
92
  domain: 16,
87
93
  };
88
94
  const COLUMN_OBJECT_TYPES = new Set(['dbt_column', 'warehouse_column', 'runtime_column', 'runtime_value']);
95
+ // v3 removes the legacy persisted plaintext runtime-value cache (SEC-003).
96
+ const METADATA_INDEX_VERSION = 'metadata-index-v4-content-addressed-skills';
89
97
  export function defaultMetadataPath(projectRoot) {
90
98
  return join(projectRoot, '.dql', 'cache', 'metadata.sqlite');
91
99
  }
100
+ export function metadataSnapshotPath(projectRoot, fingerprint) {
101
+ return join(projectRoot, '.dql', 'cache', 'snapshots', `${fingerprint}.sqlite`);
102
+ }
103
+ export function activeMetadataSnapshotPath(projectRoot) {
104
+ const pointerPath = join(projectRoot, '.dql', 'cache', 'active-snapshot.json');
105
+ if (!existsSync(pointerPath))
106
+ return null;
107
+ try {
108
+ const parsed = JSON.parse(readFileSync(pointerPath, 'utf8'));
109
+ const path = typeof parsed.snapshotPath === 'string' ? parsed.snapshotPath : '';
110
+ return path && existsSync(path) ? path : null;
111
+ }
112
+ catch {
113
+ return null;
114
+ }
115
+ }
116
+ /** Read the prepared catalog identity without rebuilding the metadata snapshot. */
117
+ export function currentMetadataFingerprint(projectRoot) {
118
+ if (!existsSync(defaultMetadataPath(projectRoot)))
119
+ return undefined;
120
+ const catalog = openMetadataCatalog(projectRoot);
121
+ try {
122
+ return catalog.state('fingerprint') ?? undefined;
123
+ }
124
+ finally {
125
+ catalog.close();
126
+ }
127
+ }
128
+ function activateMetadataSnapshot(projectRoot, fingerprint, catalog) {
129
+ const snapshotPath = metadataSnapshotPath(projectRoot, fingerprint);
130
+ catalog.exportSnapshot(snapshotPath);
131
+ const cacheDir = join(projectRoot, '.dql', 'cache');
132
+ const pointerPath = join(cacheDir, 'active-snapshot.json');
133
+ const candidate = `${pointerPath}.candidate`;
134
+ mkdirSync(cacheDir, { recursive: true });
135
+ writeFileSync(candidate, JSON.stringify({
136
+ snapshotId: fingerprint,
137
+ indexSchemaVersion: METADATA_INDEX_VERSION,
138
+ snapshotPath,
139
+ }, null, 2), 'utf8');
140
+ renameSync(candidate, pointerPath);
141
+ // Do not delete older immutable files here: a request that acquired the old
142
+ // pointer may still hold it open. Cache cleanup is an explicit maintenance
143
+ // concern until the snapshot service owns reference-counted leases.
144
+ return snapshotPath;
145
+ }
92
146
  export function openMetadataCatalog(projectRoot, dbPath = defaultMetadataPath(projectRoot)) {
93
147
  return new MetadataCatalog(dbPath);
94
148
  }
149
+ export function openActiveKnowledgeSnapshot(projectRoot) {
150
+ const path = activeMetadataSnapshotPath(projectRoot) ?? defaultMetadataPath(projectRoot);
151
+ return new MetadataCatalog(path, { readOnly: path !== defaultMetadataPath(projectRoot) });
152
+ }
153
+ export function readIndexedDomainKnowledge(projectRoot, domainId) {
154
+ const catalog = openActiveKnowledgeSnapshot(projectRoot);
155
+ try {
156
+ const snapshotId = catalog.state('fingerprint') ?? 'metadata-unavailable';
157
+ const capsuleObject = catalog.listAllObjects({ objectTypes: ['domain_capsule'], domain: domainId })
158
+ .find((item) => !stringValue(item.payload?.modelAreaId));
159
+ const capsule = capsuleObject?.payload;
160
+ if (!capsule)
161
+ return null;
162
+ const rows = catalog.listAllObjects({ domain: domainId })
163
+ .filter((item) => item.objectType !== 'domain_capsule' && item.objectType !== 'cross_domain_route');
164
+ const selectedRows = rows.slice(0, 750);
165
+ const byKey = new Map(selectedRows.map((row) => [row.objectKey, metadataObjectToKnowledge(row)]));
166
+ const edgeRows = catalog.edgesForKeys(selectedRows.map((row) => row.objectKey), 1).slice(0, 1_500);
167
+ for (const edge of edgeRows) {
168
+ if (!byKey.has(edge.fromKey)) {
169
+ const item = catalog.getObject(edge.fromKey);
170
+ if (item)
171
+ byKey.set(edge.fromKey, metadataObjectToKnowledge(item));
172
+ }
173
+ if (!byKey.has(edge.toKey)) {
174
+ const item = catalog.getObject(edge.toKey);
175
+ if (item)
176
+ byKey.set(edge.toKey, metadataObjectToKnowledge(item));
177
+ }
178
+ }
179
+ const routes = catalog.listAllObjects({ objectTypes: ['cross_domain_route'] })
180
+ .flatMap((item) => metadataRoute(item) ? [metadataRoute(item)] : [])
181
+ .filter((route) => route.providerDomainId === domainId || route.consumerDomainId === domainId);
182
+ const routeStates = routes.reduce((counts, route) => {
183
+ counts[route.state] = (counts[route.state] ?? 0) + 1;
184
+ return counts;
185
+ }, {});
186
+ return {
187
+ schemaVersion: 2,
188
+ snapshotId,
189
+ sourceFingerprint: stringValue(capsuleObject?.payload?.sourceFingerprint) ?? snapshotId,
190
+ domainId,
191
+ capsule,
192
+ counts: { objects: rows.length, edges: edgeRows.length, routes: routes.length, routeStates },
193
+ objects: selectedRows.map(metadataObjectToKnowledge),
194
+ edges: edgeRows.map((edge) => metadataEdgeToKnowledge(edge, byKey)),
195
+ routes,
196
+ truncated: rows.length > selectedRows.length || edgeRows.length >= 1_500,
197
+ };
198
+ }
199
+ finally {
200
+ catalog.close();
201
+ }
202
+ }
203
+ export function readIndexedKnowledge360(projectRoot, identity) {
204
+ const catalog = openActiveKnowledgeSnapshot(projectRoot);
205
+ try {
206
+ const focusRow = catalog.findObjectByIdentity(identity);
207
+ if (!focusRow)
208
+ return null;
209
+ const edgeRows = catalog.edgesForKeys([focusRow.objectKey], 2).slice(0, 500);
210
+ const keys = [...new Set([focusRow.objectKey, ...edgeRows.flatMap((edge) => [edge.fromKey, edge.toKey])])].slice(0, 160);
211
+ const rows = catalog.getObjectsByKeys(keys);
212
+ const byKey = new Map(rows.map((row) => [row.objectKey, metadataObjectToKnowledge(row)]));
213
+ const objects = rows.map(metadataObjectToKnowledge);
214
+ const domains = new Set(objects.flatMap((item) => item.domainId ? [item.domainId] : []));
215
+ const routes = catalog.listAllObjects({ objectTypes: ['cross_domain_route'] })
216
+ .flatMap((item) => metadataRoute(item) ? [metadataRoute(item)] : [])
217
+ .filter((route) => domains.has(route.providerDomainId) && domains.has(route.consumerDomainId));
218
+ return {
219
+ snapshotId: catalog.state('fingerprint') ?? 'metadata-unavailable',
220
+ sourceFingerprint: catalog.state('fingerprint') ?? 'metadata-unavailable',
221
+ focus: metadataObjectToKnowledge(focusRow),
222
+ objects,
223
+ edges: edgeRows.filter((edge) => byKey.has(edge.fromKey) && byKey.has(edge.toKey)).map((edge) => metadataEdgeToKnowledge(edge, byKey)),
224
+ routes,
225
+ truncated: keys.length >= 160 || edgeRows.length >= 500,
226
+ };
227
+ }
228
+ finally {
229
+ catalog.close();
230
+ }
231
+ }
95
232
  export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
96
233
  const semanticLayer = options.semanticLayer !== undefined
97
234
  ? options.semanticLayer ?? undefined
@@ -102,9 +239,12 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
102
239
  const catalog = openMetadataCatalog(projectRoot);
103
240
  try {
104
241
  const existing = catalog.state('fingerprint');
105
- if (!options.force && existing === snapshot.fingerprint) {
242
+ const compatibleIndex = catalog.state('index_version') === METADATA_INDEX_VERSION;
243
+ if (!options.force && compatibleIndex && existing === snapshot.fingerprint) {
244
+ const snapshotPath = activateMetadataSnapshot(projectRoot, snapshot.fingerprint, catalog);
106
245
  return {
107
246
  path: defaultMetadataPath(projectRoot),
247
+ snapshotPath,
108
248
  refreshed: false,
109
249
  objectCount: catalog.objectCount(),
110
250
  edgeCount: catalog.edgeCount(),
@@ -115,14 +255,16 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
115
255
  // W3.4 — incremental reindex when prior state exists (only re-tokenizes changed
116
256
  // sources' FTS); `force` does a clean full rebuild. Proven equal to a full
117
257
  // rebuild by incremental-reindex.test.ts.
118
- if (options.force) {
258
+ if (options.force || !compatibleIndex) {
119
259
  catalog.rebuild(snapshot);
120
260
  }
121
261
  else {
122
262
  catalog.rebuildIncremental(snapshot);
123
263
  }
264
+ const snapshotPath = activateMetadataSnapshot(projectRoot, snapshot.fingerprint, catalog);
124
265
  return {
125
266
  path: defaultMetadataPath(projectRoot),
267
+ snapshotPath,
126
268
  refreshed: true,
127
269
  objectCount: snapshot.objects.length,
128
270
  edgeCount: snapshot.edges.length,
@@ -138,8 +280,10 @@ export function upsertMetadataSnapshot(projectRoot, snapshot) {
138
280
  const catalog = openMetadataCatalog(projectRoot);
139
281
  try {
140
282
  catalog.rebuild(snapshot);
283
+ const snapshotPath = activateMetadataSnapshot(projectRoot, snapshot.fingerprint, catalog);
141
284
  return {
142
285
  path: defaultMetadataPath(projectRoot),
286
+ snapshotPath,
143
287
  refreshed: true,
144
288
  objectCount: snapshot.objects.length,
145
289
  edgeCount: snapshot.edges.length,
@@ -164,7 +308,12 @@ export async function buildLocalContextPack(projectRoot, request) {
164
308
  }
165
309
  if (!prepared)
166
310
  await ensureMetadataCatalogFresh(projectRoot);
167
- const catalog = openMetadataCatalog(projectRoot);
311
+ // Static governed knowledge is read from the immutable content-addressed
312
+ // snapshot selected at request start. Mutable run history, runtime schema,
313
+ // and context packs stay in the working catalog and never alter that view.
314
+ const snapshotPath = activeMetadataSnapshotPath(projectRoot) ?? defaultMetadataPath(projectRoot);
315
+ const catalog = openActiveKnowledgeSnapshot(projectRoot);
316
+ const runtimeCatalog = openMetadataCatalog(projectRoot);
168
317
  try {
169
318
  const mode = request.mode ?? 'question';
170
319
  const followUp = normalizeFollowUpContext(request.followUp);
@@ -180,7 +329,7 @@ export async function buildLocalContextPack(projectRoot, request) {
180
329
  && request.priorContextPackId
181
330
  && request.conversationTopicRelation
182
331
  && request.conversationTopicRelation !== 'shift'
183
- ? catalog.getContextPack(request.priorContextPackId)
332
+ ? runtimeCatalog.getContextPack(request.priorContextPackId)
184
333
  : null;
185
334
  const priorPackFresh = Boolean(priorPack && priorPack.freshness.fingerprint === catalog.state('fingerprint'));
186
335
  if (priorPack
@@ -199,13 +348,13 @@ export async function buildLocalContextPack(projectRoot, request) {
199
348
  strategy: 'reused_pack_refinement',
200
349
  },
201
350
  freshness: {
202
- catalogPath: defaultMetadataPath(projectRoot),
351
+ catalogPath: snapshotPath,
203
352
  builtAt: catalog.state('built_at'),
204
353
  fingerprint: catalog.state('fingerprint'),
205
354
  },
206
355
  };
207
356
  delete reusedPayload.id;
208
- const reusedId = catalog.insertContextPack(reusedPayload);
357
+ const reusedId = runtimeCatalog.insertContextPack(reusedPayload);
209
358
  return { ...reusedPayload, id: reusedId };
210
359
  }
211
360
  const priorSeedObjects = priorPack && priorPackFresh
@@ -214,17 +363,23 @@ export async function buildLocalContextPack(projectRoot, request) {
214
363
  || request.conversationTopicRelation === 'return')
215
364
  ? priorPack.objects.slice(0, 24)
216
365
  : [];
217
- const runtimeSnapshot = request.runtimeSchemaSnapshot ?? catalog.latestRuntimeSchemaSnapshot();
218
- const runtimeObjects = runtimeSnapshot ? runtimeSchemaObjects(runtimeSnapshot) : [];
219
- const runtimeValueObjects = runtimeValueMatchObjects(catalog.searchRuntimeValues(metadataValueSearchTerms(request.question, questionPlan, followUp), 32));
220
- const selectedObjects = selectedContextObjects(request.selectedContext);
221
- const followUpObjects = followUpContextObjects(followUp);
222
- const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
223
366
  const searchQueries = uniqueMetadataSearchQueries([
224
367
  buildFollowUpSearchQuery(request.question, followUp),
225
368
  ...questionPlan.searchQueries,
226
369
  ]);
227
- const areaObjects = filterMetadataObjectsByDomainContext(catalog.listObjects({ objectTypes: ['model_area'], limit: 500 }), request.domainContext);
370
+ // Runtime database schemas live in a separate persisted FTS lane. Normal Ask
371
+ // requests hydrate only relevant tables, so thousands of tables and hundreds
372
+ // of thousands of columns never become a request-path full-catalog scan.
373
+ // Explicit snapshots remain supported for callers that already hold a small,
374
+ // question-scoped schema payload.
375
+ const runtimeObjects = request.runtimeSchemaSnapshot
376
+ ? runtimeSchemaObjects(request.runtimeSchemaSnapshot)
377
+ : runtimeCatalog.searchRuntimeSchemaObjects(searchQueries.join(' '), Math.max(request.limit ?? 80, 20));
378
+ const runtimeValueObjects = runtimeValueMatchObjects(runtimeCatalog.searchRuntimeValues(metadataValueSearchTerms(request.question, questionPlan, followUp), 32));
379
+ const selectedObjects = selectedContextObjects(request.selectedContext);
380
+ const followUpObjects = followUpContextObjects(followUp);
381
+ const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
382
+ const areaObjects = filterMetadataObjectsByDomainContext(catalog.listAllObjects({ objectTypes: ['model_area'] }), request.domainContext);
228
383
  const focusedArea = resolveFocusedModelArea(request.domainContext, request.question, areaObjects);
229
384
  const effectiveDomainContext = focusedArea && request.domainContext
230
385
  ? { ...request.domainContext, modelAreaId: focusedArea.id }
@@ -280,18 +435,28 @@ export async function buildLocalContextPack(projectRoot, request) {
280
435
  // Skills are selected from the catalog snapshot with the same hard domain,
281
436
  // area, status, and exclusion gates as the loader. They are not left to FTS
282
437
  // chance or re-read from disk after the snapshot has been fingerprinted.
283
- const selectedSkills = selectContextPackSkills(catalog.listObjects({ objectTypes: ['skill'], limit: 500 }), request.question, effectiveDomainContext);
438
+ const selectedSkills = selectContextPackSkills(catalog, catalog.listAllObjects({ objectTypes: ['skill'] }), request.question, effectiveDomainContext);
284
439
  const selectedSkillObjects = selectedSkills.map((item) => item.object);
440
+ const routeCandidates = effectiveDomainContext?.activeDomain
441
+ ? catalog.crossDomainRouteObjects(effectiveDomainContext.activeDomain, 100)
442
+ : [];
443
+ const routeObjects = rankMetadataObjects({
444
+ rows: retrievalObjects(routeCandidates),
445
+ question: `${request.question} ${effectiveDomainContext?.purpose ?? ''}`,
446
+ questionPlan,
447
+ modelAreaId: focusedArea?.id,
448
+ limit: 8,
449
+ }).selected;
285
450
  const objects = fullCatalogObjects
286
- ? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects, ...selectedSkillObjects])
287
- : mergeObjects([...rankedObjects, ...sqlParentObjects, ...selectedSkillObjects]);
451
+ ? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects, ...selectedSkillObjects, ...routeObjects])
452
+ : mergeObjects([...rankedObjects, ...sqlParentObjects, ...selectedSkillObjects, ...routeObjects]);
288
453
  const objectKeys = objects.map((row) => row.objectKey);
289
454
  const allowedObjectKeys = new Set(objectKeys);
290
455
  const contextEdges = mergeMetadataEdges([
291
456
  ...edgeWalk,
292
457
  ...catalog.edgesForKeys(objectKeys, 2),
293
458
  ]).filter((edge) => allowedObjectKeys.has(edge.fromKey) && allowedObjectKeys.has(edge.toKey));
294
- const queryRuns = catalog.queryRunsForObjectKeys(objectKeys, 20);
459
+ const queryRuns = runtimeCatalog.queryRunsForObjectKeys(objectKeys, 20);
295
460
  const diagnostics = catalog.diagnostics();
296
461
  const warnings = buildWarnings(diagnostics, objects);
297
462
  const trustLabel = deriveTrust(objects);
@@ -320,6 +485,7 @@ export async function buildLocalContextPack(projectRoot, request) {
320
485
  limit: request.limit ?? 120,
321
486
  });
322
487
  const conflicts = buildCandidateConflicts(reranked.ranked);
488
+ const meaningEvidence = buildMeaningEvidencePackage(request.question, questionPlan, reranked.ranked);
323
489
  const compileConflicts = catalog.compileConflicts();
324
490
  const routeDecision = withMetadataTrustLabelInfo(await planContextPackRoute({
325
491
  request,
@@ -346,6 +512,7 @@ export async function buildLocalContextPack(projectRoot, request) {
346
512
  questionScope,
347
513
  limit: 6,
348
514
  }).catch(() => ({ applied: [], conflicts: [] }));
515
+ const knowledgeLens = buildKnowledgeLens(catalog, effectiveDomainContext, selectedSkills);
349
516
  const payload = {
350
517
  id: '',
351
518
  question: request.question,
@@ -357,6 +524,7 @@ export async function buildLocalContextPack(projectRoot, request) {
357
524
  trustLabelInfo: metadataTrustLabelInfo(trustLabel),
358
525
  objects,
359
526
  skills: selectedSkills.map((item) => item.skill),
527
+ knowledgeLens,
360
528
  edges: contextEdges,
361
529
  queryRuns,
362
530
  citations,
@@ -395,22 +563,49 @@ export async function buildLocalContextPack(projectRoot, request) {
395
563
  topRejected: reranked.rejected,
396
564
  certifiedCandidateFits,
397
565
  candidateConflicts: conflicts,
566
+ meaningEvidence,
398
567
  },
399
568
  freshness: {
400
- catalogPath: defaultMetadataPath(projectRoot),
569
+ catalogPath: snapshotPath,
401
570
  builtAt: catalog.state('built_at'),
402
571
  fingerprint: catalog.state('fingerprint'),
403
572
  },
404
573
  };
405
574
  const packPayload = { ...payload };
406
575
  delete packPayload.id;
407
- const id = catalog.insertContextPack(packPayload);
576
+ const id = runtimeCatalog.insertContextPack(packPayload);
408
577
  return { ...payload, id };
409
578
  }
410
579
  finally {
411
580
  catalog.close();
581
+ runtimeCatalog.close();
412
582
  }
413
583
  }
584
+ function buildKnowledgeLens(catalog, context, selectedSkills) {
585
+ const activeDomainId = context?.activeDomain ?? undefined;
586
+ const capsule = activeDomainId
587
+ ? catalog.listAllObjects({ objectTypes: ['domain_capsule'] })
588
+ .find((object) => object.domain === activeDomainId && !stringValue(object.payload?.modelAreaId))
589
+ : undefined;
590
+ const skillRefs = selectedSkills
591
+ .map(({ skill }) => skill.qualifiedId ?? skill.id)
592
+ .sort();
593
+ const skillFingerprints = Object.fromEntries(selectedSkills.flatMap(({ object, skill }) => {
594
+ const fingerprint = stringValue(object.payload?.sourceFingerprint);
595
+ return fingerprint ? [[skill.qualifiedId ?? skill.id, fingerprint]] : [];
596
+ }));
597
+ return {
598
+ mode: context?.source === 'explicit_api' || context?.source === 'explicit_ui' ? 'pinned' : 'auto',
599
+ activeDomainId,
600
+ modelAreaId: context?.modelAreaId,
601
+ purpose: context?.purpose,
602
+ skillRefs,
603
+ snapshotId: context?.snapshotId ?? catalog.state('fingerprint') ?? 'metadata-unavailable',
604
+ capsuleFingerprint: stringValue(capsule?.payload?.fingerprint)
605
+ ?? stringValue(capsule?.payload?.sourceFingerprint),
606
+ skillFingerprints: Object.keys(skillFingerprints).length > 0 ? skillFingerprints : undefined,
607
+ };
608
+ }
414
609
  export async function planAgentAnswer(projectRoot, request) {
415
610
  const contextPack = await buildLocalContextPack(projectRoot, request);
416
611
  return {
@@ -477,6 +672,7 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skil
477
672
  }
478
673
  }
479
674
  addManifestBlockDetails(manifest, objects);
675
+ addManifestKnowledgeGraph(materializeIndexedKnowledgeGraph(projectRoot, manifest), objects, edges);
480
676
  addSkillObjects(skills, objects, edges);
481
677
  addDbtDagObjects(manifest, objects, edges, diagnostics);
482
678
  addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges, diagnostics);
@@ -510,20 +706,285 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skil
510
706
  edges: Array.from(edges.values()).sort((a, b) => `${a.edgeType}|${a.fromKey}|${a.toKey}`.localeCompare(`${b.edgeType}|${b.fromKey}|${b.toKey}`)),
511
707
  diagnostics,
512
708
  compileConflicts,
709
+ skillBodies: [...new Map(skills.map((skill) => {
710
+ const bodyHash = sha256(skill.body);
711
+ return [bodyHash, { bodyHash, body: skill.body }];
712
+ })).values()].sort((a, b) => a.bodyHash.localeCompare(b.bodyHash)),
513
713
  generatedAt: new Date().toISOString(),
514
714
  fingerprint: '',
515
715
  };
516
716
  snapshot.fingerprint = fingerprintSnapshot(snapshot);
517
717
  return snapshot;
518
718
  }
719
+ /**
720
+ * Manifest graph v2 intentionally omits verbose objects and edges. Snapshot
721
+ * construction rematerializes that compiler graph once, then persists the
722
+ * detail in immutable SQLite. Normal Ask/API reads never repeat this work.
723
+ */
724
+ function materializeIndexedKnowledgeGraph(projectRoot, manifest) {
725
+ const compact = manifest.knowledgeGraph;
726
+ if (!compact || compact.storageMode !== 'indexed')
727
+ return manifest;
728
+ const { knowledgeGraph: _compactGraph, ...baseManifest } = manifest;
729
+ const registry = loadDomainPackageRegistry(projectRoot);
730
+ const skillCatalog = loadManifestKnowledgeSkills(projectRoot, registry);
731
+ const inline = buildManifestKnowledgeGraph({ manifest: baseManifest, skills: skillCatalog.skills });
732
+ const objects = { ...(inline.objects ?? {}) };
733
+ for (const ref of compact.objectRefs ?? []) {
734
+ objects[ref.id] ??= { ...ref };
735
+ }
736
+ return {
737
+ ...manifest,
738
+ knowledgeGraph: {
739
+ ...inline,
740
+ sourceFingerprint: compact.sourceFingerprint,
741
+ objects,
742
+ domainCapsules: compact.domainCapsules,
743
+ crossDomainRoutes: compact.crossDomainRoutes,
744
+ diagnostics: compact.diagnostics,
745
+ },
746
+ };
747
+ }
748
+ /**
749
+ * CTX-006: project search is a projection of the compiler-owned qualified
750
+ * policy graph. Legacy KG records are enriched in place so existing route keys
751
+ * remain compatible while identity, capsules, and governed route state come
752
+ * from one source.
753
+ */
754
+ function addManifestKnowledgeGraph(manifest, objects, edges) {
755
+ const graph = manifest.knowledgeGraph;
756
+ if (!graph)
757
+ return;
758
+ const keyByGraphId = new Map();
759
+ const graphObjects = Object.values(graph.objects ?? {});
760
+ const indexedObjects = graphObjects.length > 0 ? graphObjects : (graph.objectRefs ?? []);
761
+ for (const item of indexedObjects) {
762
+ const itemPayload = ('payload' in item ? item.payload : undefined);
763
+ const itemAliases = ('aliases' in item ? item.aliases : undefined);
764
+ const displayName = knowledgeDisplayName(item.kind, item.localId, itemPayload);
765
+ const objectKey = knowledgeMetadataKey(item.kind, displayName, item.id, item.source.system);
766
+ keyByGraphId.set(item.id, objectKey);
767
+ const description = stringValue(itemPayload?.description)
768
+ ?? stringValue(itemPayload?.businessContext)
769
+ ?? itemAliases?.find((alias) => alias !== item.localId);
770
+ const compiled = {
771
+ objectKey,
772
+ objectType: knowledgeMetadataType(item.kind),
773
+ name: displayName,
774
+ fullName: item.id,
775
+ domain: item.domainId,
776
+ owner: item.owner,
777
+ status: item.status,
778
+ description,
779
+ sourcePath: item.source.path,
780
+ sourceSystem: `DQL canonical knowledge graph (${item.source.system})`,
781
+ payload: compactObject({
782
+ ...(itemPayload ?? {}),
783
+ qualifiedId: item.id,
784
+ aliases: itemAliases ?? [],
785
+ modelAreaIds: item.modelAreaIds ?? [],
786
+ sourceFingerprint: item.source.fingerprint,
787
+ sourceNativeId: item.source.nativeId,
788
+ knowledgeGraphSchemaVersion: graph.schemaVersion,
789
+ }),
790
+ };
791
+ const existing = objects.get(objectKey);
792
+ objects.set(objectKey, existing ? mergeObject(existing, compiled) : compiled);
793
+ }
794
+ for (const capsule of Object.values(graph.domainCapsules)) {
795
+ const objectKey = `dql:domain_capsule:${capsule.id}`;
796
+ objects.set(objectKey, {
797
+ objectKey,
798
+ objectType: 'domain_capsule',
799
+ name: capsule.name,
800
+ fullName: capsule.id,
801
+ domain: capsule.domainId,
802
+ description: capsule.description,
803
+ sourceSystem: 'DQL compiled Domain Knowledge Capsule',
804
+ payload: compactObject({ ...capsule, sourceFingerprint: graph.sourceFingerprint }),
805
+ });
806
+ addMetadataEdge(edges, 'contains', keyByGraphId.get(`domain::${capsule.domainId}`) ?? `domain:${capsule.domainId}`, objectKey, { source: 'knowledge_graph' });
807
+ for (const skill of capsule.skillRefs)
808
+ addMetadataEdge(edges, 'guided_by', objectKey, `skill:${skill}`, { source: 'knowledge_graph' });
809
+ }
810
+ for (const route of graph.crossDomainRoutes) {
811
+ const objectKey = `dql:cross_domain_route:${route.id}`;
812
+ objects.set(objectKey, {
813
+ objectKey,
814
+ objectType: 'cross_domain_route',
815
+ name: `${route.providerDomainId} → ${route.consumerDomainId}`,
816
+ fullName: route.id,
817
+ domain: route.consumerDomainId,
818
+ status: route.state,
819
+ description: route.purpose ? `Approved purpose: ${route.purpose}` : 'Observed cross-domain dependency',
820
+ sourceSystem: 'DQL compiled cross-domain policy',
821
+ payload: compactObject(route),
822
+ });
823
+ addMetadataEdge(edges, 'cross_domain_route', keyByGraphId.get(`domain::${route.providerDomainId}`) ?? `domain:${route.providerDomainId}`, objectKey, { state: route.state, reasonCodes: route.reasonCodes });
824
+ addMetadataEdge(edges, 'cross_domain_route', objectKey, keyByGraphId.get(`domain::${route.consumerDomainId}`) ?? `domain:${route.consumerDomainId}`, { state: route.state, reasonCodes: route.reasonCodes });
825
+ }
826
+ for (const edge of graph.edges ?? []) {
827
+ const fromKey = keyByGraphId.get(edge.from) ?? edge.from;
828
+ const toKey = keyByGraphId.get(edge.to) ?? edge.to;
829
+ addMetadataEdge(edges, edge.kind, fromKey, toKey, {
830
+ state: edge.state,
831
+ domainPair: edge.domainPair,
832
+ evidenceRefs: edge.evidenceRefs ?? [],
833
+ reasonCodes: edge.reasonCodes ?? [],
834
+ fingerprint: edge.fingerprint,
835
+ source: 'knowledge_graph',
836
+ });
837
+ }
838
+ }
839
+ function addMetadataEdge(edges, edgeType, fromKey, toKey, payload) {
840
+ const key = `${edgeType}\u0000${fromKey}\u0000${toKey}`;
841
+ if (!edges.has(key))
842
+ edges.set(key, { edgeType, fromKey, toKey, confidence: 1, payload: compactObject(payload) });
843
+ }
844
+ function knowledgeMetadataKey(kind, localId, qualifiedId, sourceSystem) {
845
+ switch (kind) {
846
+ case 'block': return `dql:block:${localId}`;
847
+ case 'term': return `dql:term:${localId}`;
848
+ case 'business_view': return `dql:business_view:${localId}`;
849
+ case 'metric': return `semantic:metric:${localId}`;
850
+ case 'dimension': return `semantic:dimension:${localId}`;
851
+ case 'entity': return `dql:entity:${qualifiedId}`;
852
+ case 'model_area': return `dql:model_area:${qualifiedId}`;
853
+ case 'relationship': return `dql:relationship:${qualifiedId}`;
854
+ case 'contract': return `dql:contract:${qualifiedId}`;
855
+ case 'domain_export': return `dql:domain_export:${qualifiedId}`;
856
+ case 'domain_import': return `dql:domain_import:${qualifiedId}`;
857
+ case 'conformance': return `dql:conformance:${qualifiedId}`;
858
+ case 'policy': return `dql:policy:${qualifiedId}`;
859
+ case 'evaluation': return `dql:evaluation:${qualifiedId}`;
860
+ case 'skill': return `skill:${qualifiedId}`;
861
+ case 'domain': return `domain:${localId}`;
862
+ case 'dbt_model': return `dbt:model:${localId}`;
863
+ case 'dbt_source': return `dbt:source:${localId}`;
864
+ case 'source_table': return `warehouse:table:${localId}`;
865
+ case 'notebook': return `notebook:${localId}`;
866
+ case 'dashboard': return `dashboard:${localId}`;
867
+ case 'app': return `app:${localId}`;
868
+ default: return `${sourceSystem}:${kind}:${qualifiedId}`;
869
+ }
870
+ }
871
+ function knowledgeDisplayName(kind, localId, payload) {
872
+ if (kind === 'domain' || kind === 'app')
873
+ return stringValue(payload?.name) ?? localId;
874
+ if (kind === 'dashboard' || kind === 'notebook')
875
+ return stringValue(payload?.title) ?? localId;
876
+ return localId;
877
+ }
878
+ function knowledgeMetadataType(kind) {
879
+ switch (kind) {
880
+ case 'block': return 'dql_block';
881
+ case 'term': return 'dql_term';
882
+ case 'metric': return 'semantic_metric';
883
+ case 'dimension': return 'semantic_dimension';
884
+ case 'entity': return 'dql_entity';
885
+ case 'model_area': return 'model_area';
886
+ case 'dbt_model': return 'dbt_model';
887
+ case 'dbt_source': return 'dbt_source';
888
+ case 'source_table': return 'warehouse_table';
889
+ default: return kind;
890
+ }
891
+ }
892
+ function metadataKnowledgeKind(objectType) {
893
+ switch (objectType) {
894
+ case 'dql_block': return 'block';
895
+ case 'dql_term': return 'term';
896
+ case 'business_view': return 'business_view';
897
+ case 'semantic_metric': return 'metric';
898
+ case 'semantic_dimension': return 'dimension';
899
+ case 'semantic_model': return 'semantic_model';
900
+ case 'dql_entity': return 'entity';
901
+ case 'model_area': return 'model_area';
902
+ case 'dbt_model': return 'dbt_model';
903
+ case 'dbt_source': return 'dbt_source';
904
+ case 'warehouse_table': return 'source_table';
905
+ case 'domain_capsule': return 'domain';
906
+ default: return objectType;
907
+ }
908
+ }
909
+ function metadataObjectToKnowledge(item) {
910
+ const payload = item.payload ?? {};
911
+ const id = stringValue(payload.qualifiedId) ?? item.fullName ?? item.objectKey;
912
+ const system = item.sourceSystem?.toLowerCase().includes('dbt')
913
+ ? 'dbt'
914
+ : item.sourceSystem?.toLowerCase().includes('semantic')
915
+ ? 'semantic'
916
+ : 'dql';
917
+ return {
918
+ id,
919
+ kind: metadataKnowledgeKind(item.objectType),
920
+ localId: item.name,
921
+ domainId: item.domain,
922
+ modelAreaIds: metadataStringArray(payload.modelAreaIds),
923
+ aliases: metadataStringArray(payload.aliases),
924
+ status: item.status,
925
+ owner: item.owner,
926
+ source: {
927
+ system,
928
+ path: item.sourcePath,
929
+ nativeId: stringValue(payload.sourceNativeId),
930
+ fingerprint: stringValue(payload.sourceFingerprint) ?? sha256(stableStringify({ id, sourcePath: item.sourcePath, payload })),
931
+ },
932
+ payload,
933
+ };
934
+ }
935
+ function metadataEdgeToKnowledge(edge, byKey) {
936
+ const payload = edge.payload ?? {};
937
+ const from = byKey.get(edge.fromKey)?.id ?? edge.fromKey;
938
+ const to = byKey.get(edge.toKey)?.id ?? edge.toKey;
939
+ const fingerprint = stringValue(payload.fingerprint) ?? sha256(stableStringify({ kind: edge.edgeType, from, to, payload }));
940
+ return {
941
+ id: `edge::${fingerprint.slice(0, 20)}`,
942
+ kind: edge.edgeType,
943
+ from,
944
+ to,
945
+ state: payload.state,
946
+ domainPair: payload.domainPair,
947
+ evidenceRefs: metadataStringArray(payload.evidenceRefs),
948
+ reasonCodes: metadataStringArray(payload.reasonCodes),
949
+ fingerprint,
950
+ };
951
+ }
952
+ function metadataRoute(item) {
953
+ const payload = item.payload ?? {};
954
+ const providerDomainId = stringValue(payload.providerDomainId);
955
+ const consumerDomainId = stringValue(payload.consumerDomainId);
956
+ const relationshipId = stringValue(payload.relationshipId);
957
+ const state = stringValue(payload.state);
958
+ if (!providerDomainId || !consumerDomainId || !relationshipId || !state || !['observed', 'authorized', 'blocked', 'stale'].includes(state))
959
+ return null;
960
+ return {
961
+ id: stringValue(payload.id) ?? item.fullName ?? item.objectKey,
962
+ providerDomainId,
963
+ consumerDomainId,
964
+ purpose: stringValue(payload.purpose) ?? '',
965
+ relationshipId,
966
+ exportId: stringValue(payload.exportId),
967
+ importId: stringValue(payload.importId),
968
+ contractId: stringValue(payload.contractId),
969
+ state: state,
970
+ reasonCodes: metadataStringArray(payload.reasonCodes),
971
+ path: metadataStringArray(payload.path),
972
+ fingerprint: stringValue(payload.fingerprint) ?? sha256(stableStringify(payload)),
973
+ };
974
+ }
519
975
  export class MetadataCatalog {
520
976
  dbPath;
521
977
  db;
522
- constructor(dbPath) {
978
+ constructor(dbPath, options = {}) {
523
979
  this.dbPath = dbPath;
524
- mkdirSync(dirname(dbPath), { recursive: true });
980
+ if (!options.readOnly)
981
+ mkdirSync(dirname(dbPath), { recursive: true });
525
982
  const Database = loadDatabase();
526
- this.db = new Database(dbPath);
983
+ this.db = new Database(dbPath, options.readOnly ? { readonly: true, fileMustExist: true } : undefined);
984
+ if (options.readOnly) {
985
+ this.db.pragma('query_only = ON');
986
+ return;
987
+ }
527
988
  this.db.pragma('journal_mode = WAL');
528
989
  this.db.pragma('foreign_keys = ON');
529
990
  this.initSchema();
@@ -605,6 +1066,31 @@ export class MetadataCatalog {
605
1066
  );
606
1067
  CREATE INDEX IF NOT EXISTS idx_runtime_schema_snapshots_captured ON runtime_schema_snapshots(captured_at DESC);
607
1068
 
1069
+ CREATE TABLE IF NOT EXISTS runtime_schema_objects (
1070
+ object_key TEXT PRIMARY KEY,
1071
+ object_type TEXT NOT NULL,
1072
+ name TEXT NOT NULL,
1073
+ full_name TEXT,
1074
+ domain TEXT,
1075
+ owner TEXT,
1076
+ status TEXT,
1077
+ description TEXT,
1078
+ source_path TEXT,
1079
+ source_system TEXT,
1080
+ payload_json TEXT NOT NULL DEFAULT '{}',
1081
+ updated_at TEXT NOT NULL
1082
+ );
1083
+ CREATE INDEX IF NOT EXISTS idx_runtime_schema_objects_type ON runtime_schema_objects(object_type);
1084
+
1085
+ CREATE VIRTUAL TABLE IF NOT EXISTS runtime_schema_fts USING fts5(
1086
+ object_key UNINDEXED,
1087
+ name,
1088
+ full_name,
1089
+ description,
1090
+ payload,
1091
+ tokenize = 'porter unicode61'
1092
+ );
1093
+
608
1094
  CREATE TABLE IF NOT EXISTS runtime_value_index (
609
1095
  value_key TEXT PRIMARY KEY,
610
1096
  relation TEXT NOT NULL,
@@ -661,7 +1147,23 @@ export class MetadataCatalog {
661
1147
  dbt_object_count INTEGER NOT NULL,
662
1148
  updated_at TEXT NOT NULL
663
1149
  );
1150
+
1151
+ CREATE TABLE IF NOT EXISTS skill_bodies (
1152
+ body_hash TEXT PRIMARY KEY,
1153
+ encoding TEXT NOT NULL,
1154
+ compressed_body TEXT NOT NULL,
1155
+ original_length INTEGER NOT NULL
1156
+ );
664
1157
  `);
1158
+ // Runtime values may be used transiently inside one governed Ask, but they
1159
+ // must never survive in the rebuildable metadata database. Clear legacy v2
1160
+ // rows on open so an upgrade removes old plaintext without a migration.
1161
+ this.db.prepare('DELETE FROM runtime_value_fts').run();
1162
+ this.db.prepare('DELETE FROM runtime_value_index').run();
1163
+ this.db.prepare(`
1164
+ INSERT OR REPLACE INTO metadata_state (key, value)
1165
+ VALUES ('runtime_value_index_count', '0')
1166
+ `).run();
665
1167
  }
666
1168
  rebuild(snapshot) {
667
1169
  const now = new Date().toISOString();
@@ -698,6 +1200,10 @@ export class MetadataCatalog {
698
1200
  domain, object_count, block_count, certified_block_count,
699
1201
  semantic_metric_count, dbt_object_count, updated_at
700
1202
  ) VALUES (?, ?, ?, ?, ?, ?, ?)
1203
+ `);
1204
+ const insertSkillBody = this.db.prepare(`
1205
+ INSERT OR REPLACE INTO skill_bodies (body_hash, encoding, compressed_body, original_length)
1206
+ VALUES (?, 'br', ?, ?)
701
1207
  `);
702
1208
  const sourceFingerprints = buildSourceFingerprints(snapshot.objects, now);
703
1209
  const domainShards = buildDomainShards(snapshot.objects, now);
@@ -708,10 +1214,11 @@ export class MetadataCatalog {
708
1214
  this.db.prepare('DELETE FROM metadata_diagnostics').run();
709
1215
  this.db.prepare('DELETE FROM metadata_source_fingerprints').run();
710
1216
  this.db.prepare('DELETE FROM metadata_domain_shards').run();
1217
+ this.db.prepare('DELETE FROM skill_bodies').run();
711
1218
  for (const object of snapshot.objects) {
712
1219
  const payload = object.payload ?? {};
713
1220
  insertObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, object.domain ?? null, object.owner ?? null, object.status ?? null, object.description ?? null, object.sourcePath ?? null, object.sourceSystem ?? null, JSON.stringify(payload), object.updatedAt ?? now);
714
- insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', JSON.stringify(payload));
1221
+ insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', searchableMetadataPayload(payload));
715
1222
  }
716
1223
  for (const edge of snapshot.edges) {
717
1224
  insertEdge.run(edge.edgeType, edge.fromKey, edge.toKey, edge.confidence ?? 1, JSON.stringify(edge.payload ?? {}), now);
@@ -725,6 +1232,9 @@ export class MetadataCatalog {
725
1232
  for (const item of domainShards) {
726
1233
  insertDomainShard.run(item.domain, item.objectCount, item.blockCount, item.certifiedBlockCount, item.semanticMetricCount, item.dbtObjectCount, item.updatedAt);
727
1234
  }
1235
+ for (const item of snapshot.skillBodies ?? []) {
1236
+ insertSkillBody.run(item.bodyHash, brotliCompressSync(item.body, { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 } }).toString('base64'), Buffer.byteLength(item.body, 'utf8'));
1237
+ }
728
1238
  setState.run('built_at', now);
729
1239
  setState.run('fingerprint', snapshot.fingerprint);
730
1240
  setState.run('project_root', snapshot.projectRoot);
@@ -735,6 +1245,7 @@ export class MetadataCatalog {
735
1245
  setState.run('diagnostics_json', JSON.stringify(snapshot.diagnostics));
736
1246
  setState.run('compile_conflicts_json', JSON.stringify(snapshot.compileConflicts ?? []));
737
1247
  setState.run('manifest_generated_at', snapshot.manifest.generatedAt);
1248
+ setState.run('index_version', METADATA_INDEX_VERSION);
738
1249
  });
739
1250
  txn();
740
1251
  }
@@ -791,6 +1302,9 @@ export class MetadataCatalog {
791
1302
  semantic_metric_count, dbt_object_count, updated_at
792
1303
  ) VALUES (?, ?, ?, ?, ?, ?, ?)`);
793
1304
  const setState = this.db.prepare('INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)');
1305
+ const insertSkillBody = this.db.prepare(`
1306
+ INSERT OR REPLACE INTO skill_bodies (body_hash, encoding, compressed_body, original_length)
1307
+ VALUES (?, 'br', ?, ?)`);
794
1308
  const deleteFtsForSource = this.db.prepare('DELETE FROM metadata_fts WHERE object_key IN (SELECT object_key FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?)');
795
1309
  const deleteObjectsForSource = this.db.prepare('DELETE FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?');
796
1310
  const txn = this.db.transaction(() => {
@@ -810,7 +1324,7 @@ export class MetadataCatalog {
810
1324
  continue;
811
1325
  const payload = object.payload ?? {};
812
1326
  insertObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, object.domain ?? null, object.owner ?? null, object.status ?? null, object.description ?? null, object.sourcePath ?? null, object.sourceSystem ?? null, JSON.stringify(payload), object.updatedAt ?? now);
813
- insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', JSON.stringify(payload));
1327
+ insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', searchableMetadataPayload(payload));
814
1328
  }
815
1329
  // Cross-source tables: rebuild fully.
816
1330
  this.db.prepare('DELETE FROM metadata_edges').run();
@@ -825,6 +1339,10 @@ export class MetadataCatalog {
825
1339
  for (const shard of domainShards) {
826
1340
  insertDomainShard.run(shard.domain, shard.objectCount, shard.blockCount, shard.certifiedBlockCount, shard.semanticMetricCount, shard.dbtObjectCount, shard.updatedAt);
827
1341
  }
1342
+ this.db.prepare('DELETE FROM skill_bodies').run();
1343
+ for (const item of snapshot.skillBodies ?? []) {
1344
+ insertSkillBody.run(item.bodyHash, brotliCompressSync(item.body, { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 } }).toString('base64'), Buffer.byteLength(item.body, 'utf8'));
1345
+ }
828
1346
  for (const source of removedSources) {
829
1347
  this.db.prepare('DELETE FROM metadata_source_fingerprints WHERE source_path = ?').run(source);
830
1348
  }
@@ -841,6 +1359,7 @@ export class MetadataCatalog {
841
1359
  setState.run('diagnostics_json', JSON.stringify(snapshot.diagnostics));
842
1360
  setState.run('compile_conflicts_json', JSON.stringify(snapshot.compileConflicts ?? []));
843
1361
  setState.run('manifest_generated_at', snapshot.manifest.generatedAt);
1362
+ setState.run('index_version', METADATA_INDEX_VERSION);
844
1363
  });
845
1364
  txn();
846
1365
  return { mode: 'incremental', changedSources: changedSources.size };
@@ -879,9 +1398,16 @@ export class MetadataCatalog {
879
1398
  const seen = new Set(andRows.map((row) => row.object_key));
880
1399
  const orRows = runMatch(match.or).filter((row) => !seen.has(row.object_key));
881
1400
  const rows = [...andRows, ...orRows].slice(0, limit);
1401
+ const andKeys = new Set(andRows.map((row) => row.object_key));
1402
+ const maxAndMagnitude = strongestBm25Magnitude(andRows);
1403
+ const maxOrMagnitude = strongestBm25Magnitude(orRows);
882
1404
  return rows.map((row) => ({
883
1405
  ...rowToObject(row),
884
- score: row.rank ? 1 / (1 + Math.max(0, row.rank)) : 1,
1406
+ // FTS5 bm25() is negative and more-negative means stronger. The old
1407
+ // max(0, rank) conversion flattened virtually every hit to 1. Preserve
1408
+ // within-tier separation and keep all-terms (AND) evidence ahead of a
1409
+ // partial OR-only match without pretending BM25 is an absolute confidence.
1410
+ score: normalizedBm25Score(row.rank, andKeys.has(row.object_key) ? 'and' : 'or', andKeys.has(row.object_key) ? maxAndMagnitude : maxOrMagnitude),
885
1411
  snippet: row.snip ?? undefined,
886
1412
  }));
887
1413
  }
@@ -905,6 +1431,36 @@ export class MetadataCatalog {
905
1431
  `).all(...params, options.limit ?? 100);
906
1432
  return rows.map(rowToObject);
907
1433
  }
1434
+ /** Complete typed inventory; callers must apply their own bounded ranking. */
1435
+ listAllObjects(options = {}) {
1436
+ const filters = [];
1437
+ const params = [];
1438
+ if (options.objectTypes && options.objectTypes.length > 0) {
1439
+ filters.push(`object_type IN (${options.objectTypes.map(() => '?').join(', ')})`);
1440
+ params.push(...options.objectTypes);
1441
+ }
1442
+ if (options.domain) {
1443
+ filters.push('domain = ?');
1444
+ params.push(options.domain);
1445
+ }
1446
+ const where = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
1447
+ const rows = this.db.prepare(`
1448
+ SELECT * FROM metadata_objects
1449
+ ${where}
1450
+ ORDER BY object_key
1451
+ `).all(...params);
1452
+ return rows.map(rowToObject);
1453
+ }
1454
+ crossDomainRouteObjects(domainId, limit = 100) {
1455
+ const rows = this.db.prepare(`
1456
+ SELECT * FROM metadata_objects
1457
+ WHERE object_type = 'cross_domain_route'
1458
+ AND (domain = ? OR json_extract(payload_json, '$.providerDomainId') = ?)
1459
+ ORDER BY status, name, object_key
1460
+ LIMIT ?
1461
+ `).all(domainId, domainId, Math.max(1, Math.min(limit, 500)));
1462
+ return rows.map(rowToObject);
1463
+ }
908
1464
  scanObjects(options, visit) {
909
1465
  const filters = [];
910
1466
  const params = [];
@@ -942,7 +1498,20 @@ export class MetadataCatalog {
942
1498
  }
943
1499
  getObject(objectKey) {
944
1500
  const row = this.db.prepare('SELECT * FROM metadata_objects WHERE object_key = ?').get(objectKey);
945
- return row ? rowToObject(row) : null;
1501
+ if (row)
1502
+ return rowToObject(row);
1503
+ const legacy = legacyQualifiedAlias(objectKey);
1504
+ if (!legacy)
1505
+ return null;
1506
+ const matches = this.db.prepare(`
1507
+ SELECT * FROM metadata_objects
1508
+ WHERE object_type = ? AND name = ?
1509
+ ORDER BY object_key
1510
+ LIMIT 2
1511
+ `).all(legacy.objectType, legacy.name);
1512
+ // A legacy local-name lookup is safe only while it resolves uniquely. When
1513
+ // two domains define the same local ID, callers must use the qualified key.
1514
+ return matches.length === 1 ? rowToObject(matches[0]) : null;
946
1515
  }
947
1516
  getObjectsByKeys(keys) {
948
1517
  const unique = Array.from(new Set(keys.filter(Boolean)));
@@ -958,21 +1527,45 @@ export class MetadataCatalog {
958
1527
  `).all(...chunk);
959
1528
  rows.push(...fetched.map(rowToObject));
960
1529
  }
1530
+ const found = new Set(rows.map((row) => row.objectKey));
1531
+ for (const key of unique) {
1532
+ if (found.has(key))
1533
+ continue;
1534
+ const resolved = this.getObject(key);
1535
+ if (resolved && !rows.some((row) => row.objectKey === resolved.objectKey))
1536
+ rows.push(resolved);
1537
+ }
961
1538
  return rows;
962
1539
  }
1540
+ findObjectByIdentity(identity) {
1541
+ const rows = this.db.prepare(`
1542
+ SELECT * FROM metadata_objects
1543
+ WHERE object_key = ? OR full_name = ?
1544
+ ORDER BY CASE WHEN object_key = ? THEN 0 ELSE 1 END, object_key
1545
+ LIMIT 2
1546
+ `).all(identity, identity, identity);
1547
+ return rows.length === 1 ? rowToObject(rows[0]) : rows[0] ? rowToObject(rows[0]) : null;
1548
+ }
963
1549
  edgesForKeys(keys, hops = 1) {
964
1550
  let frontier = new Set(keys.filter(Boolean));
965
1551
  const seenKeys = new Set(frontier);
966
1552
  const edges = new Map();
967
1553
  for (let hop = 0; hop < hops && frontier.size > 0; hop += 1) {
968
- const current = Array.from(frontier).slice(0, 100);
1554
+ const current = Array.from(frontier).sort();
969
1555
  frontier = new Set();
970
- const rows = this.db.prepare(`
971
- SELECT * FROM metadata_edges
972
- WHERE from_key IN (${current.map(() => '?').join(', ')})
973
- OR to_key IN (${current.map(() => '?').join(', ')})
974
- LIMIT 500
975
- `).all(...current, ...current);
1556
+ const rows = [];
1557
+ for (let offset = 0; offset < current.length; offset += 100) {
1558
+ const chunk = current.slice(offset, offset + 100);
1559
+ rows.push(...this.db.prepare(`
1560
+ SELECT * FROM metadata_edges
1561
+ WHERE from_key IN (${chunk.map(() => '?').join(', ')})
1562
+ OR to_key IN (${chunk.map(() => '?').join(', ')})
1563
+ ORDER BY confidence DESC, edge_type, from_key, to_key
1564
+ LIMIT 500
1565
+ `).all(...chunk, ...chunk));
1566
+ }
1567
+ rows.sort((left, right) => right.confidence - left.confidence
1568
+ || `${left.edge_type}|${left.from_key}|${left.to_key}`.localeCompare(`${right.edge_type}|${right.from_key}|${right.to_key}`));
976
1569
  for (const row of rows) {
977
1570
  const edge = rowToEdge(row);
978
1571
  const edgeKey = `${edge.edgeType}\u0000${edge.fromKey}\u0000${edge.toKey}`;
@@ -1022,26 +1615,26 @@ export class MetadataCatalog {
1022
1615
  }
1023
1616
  recordRuntimeSchemaSnapshot(snapshot) {
1024
1617
  const capturedAt = snapshot.capturedAt ?? new Date().toISOString();
1618
+ const normalizedTables = normalizeRuntimeSchemaTables(snapshot.tables).slice(0, 10_000);
1025
1619
  const cleanSnapshot = {
1026
1620
  source: snapshot.source,
1027
1621
  capturedAt,
1028
- tables: normalizeRuntimeSchemaTables(snapshot.tables).slice(0, 500),
1622
+ tables: normalizedTables,
1029
1623
  };
1030
1624
  const id = `schema_${Date.parse(capturedAt) || Date.now()}`;
1031
- const valueEntries = buildRuntimeValueIndex(cleanSnapshot);
1032
1625
  const insertSnapshot = this.db.prepare(`
1033
1626
  INSERT OR REPLACE INTO runtime_schema_snapshots (
1034
1627
  id, source, payload_json, captured_at
1035
1628
  ) VALUES (?, ?, ?, ?)
1036
1629
  `);
1037
- const insertValue = this.db.prepare(`
1038
- INSERT OR REPLACE INTO runtime_value_index (
1039
- value_key, relation, schema_name, table_name, column_name, column_type,
1040
- value, normalized_value, source, captured_at
1041
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1630
+ const insertSchemaObject = this.db.prepare(`
1631
+ INSERT INTO runtime_schema_objects (
1632
+ object_key, object_type, name, full_name, domain, owner, status,
1633
+ description, source_path, source_system, payload_json, updated_at
1634
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1042
1635
  `);
1043
- const insertValueFts = this.db.prepare(`
1044
- INSERT INTO runtime_value_fts (value_key, relation, column_name, value, normalized_value)
1636
+ const insertSchemaFts = this.db.prepare(`
1637
+ INSERT INTO runtime_schema_fts (object_key, name, full_name, description, payload)
1045
1638
  VALUES (?, ?, ?, ?, ?)
1046
1639
  `);
1047
1640
  const setState = this.db.prepare(`
@@ -1059,34 +1652,47 @@ export class MetadataCatalog {
1059
1652
  `).run();
1060
1653
  this.db.prepare('DELETE FROM runtime_value_fts').run();
1061
1654
  this.db.prepare('DELETE FROM runtime_value_index').run();
1062
- for (const entry of valueEntries) {
1063
- insertValue.run(entry.valueKey, entry.relation, entry.schema ?? null, entry.tableName ?? null, entry.columnName, entry.columnType ?? null, entry.value, entry.normalizedValue, entry.source ?? null, entry.capturedAt);
1064
- insertValueFts.run(entry.valueKey, entry.relation, entry.columnName, entry.value, entry.normalizedValue);
1655
+ this.db.prepare('DELETE FROM runtime_schema_fts').run();
1656
+ this.db.prepare('DELETE FROM runtime_schema_objects').run();
1657
+ for (const table of normalizedTables) {
1658
+ const object = runtimeSchemaTableObject(table, cleanSnapshot.source, capturedAt);
1659
+ const payload = object.payload ?? {};
1660
+ insertSchemaObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, null, null, object.status ?? null, object.description ?? null, null, object.sourceSystem ?? null, JSON.stringify(payload), capturedAt);
1661
+ insertSchemaFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', searchableMetadataPayload(payload));
1065
1662
  }
1066
- setState.run('runtime_value_index_count', String(valueEntries.length));
1663
+ setState.run('runtime_value_index_count', '0');
1067
1664
  setState.run('runtime_value_index_captured_at', capturedAt);
1665
+ setState.run('runtime_schema_table_count', String(normalizedTables.length));
1666
+ setState.run('runtime_schema_index_captured_at', capturedAt);
1068
1667
  });
1069
1668
  txn();
1070
1669
  return cleanSnapshot;
1071
1670
  }
1072
1671
  searchRuntimeValues(terms, limit = 40) {
1073
- const cleanTerms = uniqueStrings(terms.map(normalizeValueIndexText).filter(Boolean)).slice(0, 8);
1074
- // Filter VALUES may legitimately be stop words (a category named "current");
1075
- // fall back to raw tokens so value binding still fires rather than dropping to empty.
1076
- const query = sanitizeFtsQuery(cleanTerms.join(' '), { prefix: true, fallbackToRawTokens: true });
1077
- if (!query)
1672
+ // Deliberately non-persistent. Approved live probes are attached to the
1673
+ // current in-memory schema context by the CLI and discarded after the Ask.
1674
+ void terms;
1675
+ void limit;
1676
+ return [];
1677
+ }
1678
+ searchRuntimeSchemaObjects(query, limit = 40) {
1679
+ const match = buildFtsMatch(query, { prefix: true });
1680
+ if (!match.or)
1078
1681
  return [];
1079
- const rows = this.db.prepare(`
1080
- SELECT v.*,
1081
- bm25(runtime_value_fts) AS rank
1082
- FROM runtime_value_fts
1083
- JOIN runtime_value_index AS v ON v.value_key = runtime_value_fts.value_key
1084
- WHERE runtime_value_fts MATCH ?
1682
+ const runMatch = (matchExpr) => this.db.prepare(`
1683
+ SELECT o.*,
1684
+ bm25(runtime_schema_fts) AS rank,
1685
+ snippet(runtime_schema_fts, -1, '<mark>', '</mark>', '...', 12) AS snip
1686
+ FROM runtime_schema_fts
1687
+ JOIN runtime_schema_objects AS o ON o.object_key = runtime_schema_fts.object_key
1688
+ WHERE runtime_schema_fts MATCH ?
1085
1689
  ORDER BY rank
1086
1690
  LIMIT ?
1087
- `).all(query, Math.max(1, limit));
1088
- return rows.map((row) => rowToRuntimeValueMatch(row, cleanTerms))
1089
- .sort((a, b) => b.score - a.score || a.relation.localeCompare(b.relation) || a.columnName.localeCompare(b.columnName));
1691
+ `).all(matchExpr, Math.max(1, limit));
1692
+ const andRows = match.and ? runMatch(match.and) : [];
1693
+ const seen = new Set(andRows.map((row) => row.object_key));
1694
+ const rows = [...andRows, ...runMatch(match.or).filter((row) => !seen.has(row.object_key))].slice(0, limit);
1695
+ return rows.map(rowToObject);
1090
1696
  }
1091
1697
  latestRuntimeSchemaSnapshot() {
1092
1698
  const row = this.db.prepare(`
@@ -1166,14 +1772,91 @@ export class MetadataCatalog {
1166
1772
  updatedAt: row.updated_at,
1167
1773
  }));
1168
1774
  }
1775
+ skillBody(bodyHash) {
1776
+ const row = this.db.prepare(`
1777
+ SELECT encoding, compressed_body
1778
+ FROM skill_bodies
1779
+ WHERE body_hash = ?
1780
+ `).get(bodyHash);
1781
+ if (!row)
1782
+ return null;
1783
+ if (row.encoding !== 'br')
1784
+ return row.compressed_body;
1785
+ const compressed = Uint8Array.from(Buffer.from(row.compressed_body, 'base64'));
1786
+ return brotliDecompressSync(compressed).toString('utf8');
1787
+ }
1788
+ exportSnapshot(destination) {
1789
+ mkdirSync(dirname(destination), { recursive: true });
1790
+ if (existsSync(destination))
1791
+ return;
1792
+ const candidate = `${destination}.candidate`;
1793
+ rmSync(candidate, { force: true });
1794
+ this.db.pragma('wal_checkpoint(TRUNCATE)');
1795
+ this.db.prepare('VACUUM INTO ?').run(candidate);
1796
+ const Database = loadDatabase();
1797
+ const sealed = new Database(candidate);
1798
+ try {
1799
+ sealed.exec(`
1800
+ DELETE FROM context_packs;
1801
+ DELETE FROM query_runs;
1802
+ DELETE FROM runtime_schema_snapshots;
1803
+ DELETE FROM runtime_schema_fts;
1804
+ DELETE FROM runtime_schema_objects;
1805
+ DELETE FROM runtime_value_fts;
1806
+ DELETE FROM runtime_value_index;
1807
+ VACUUM;
1808
+ `);
1809
+ sealed.pragma('journal_mode = DELETE');
1810
+ }
1811
+ finally {
1812
+ sealed.close();
1813
+ }
1814
+ renameSync(candidate, destination);
1815
+ }
1169
1816
  state(key) {
1170
1817
  const row = this.db.prepare('SELECT value FROM metadata_state WHERE key = ?').get(key);
1171
1818
  return row?.value ?? null;
1172
1819
  }
1820
+ setState(key, value) {
1821
+ this.db.prepare('INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)').run(key, value);
1822
+ }
1173
1823
  close() {
1174
1824
  this.db.close();
1175
1825
  }
1176
1826
  }
1827
+ function strongestBm25Magnitude(rows) {
1828
+ return rows.reduce((strongest, row) => Math.max(strongest, bm25Magnitude(row.rank)), 0);
1829
+ }
1830
+ function bm25Magnitude(rank) {
1831
+ return typeof rank === 'number' && Number.isFinite(rank) ? Math.max(0, -rank) : 0;
1832
+ }
1833
+ function normalizedBm25Score(rank, tier, strongest) {
1834
+ const relative = strongest > 0 ? bm25Magnitude(rank) / strongest : 0;
1835
+ const score = tier === 'and'
1836
+ ? 0.55 + relative * 0.45
1837
+ : 0.05 + relative * 0.45;
1838
+ return Number(score.toFixed(6));
1839
+ }
1840
+ function legacyQualifiedAlias(objectKey) {
1841
+ const mappings = [
1842
+ ['semantic:entity:', 'dql_entity'],
1843
+ ['model_area:', 'model_area'],
1844
+ ['relationship:', 'relationship'],
1845
+ ['contract:', 'contract'],
1846
+ ['domain_export:', 'domain_export'],
1847
+ ['domain_import:', 'domain_import'],
1848
+ ['conformance:', 'conformance'],
1849
+ ['policy:', 'policy'],
1850
+ ['evaluation:', 'evaluation'],
1851
+ ];
1852
+ for (const [prefix, objectType] of mappings) {
1853
+ if (!objectKey.startsWith(prefix))
1854
+ continue;
1855
+ const name = objectKey.slice(prefix.length);
1856
+ return name ? { objectType, name } : undefined;
1857
+ }
1858
+ return undefined;
1859
+ }
1177
1860
  function loadAgentManifest(projectRoot) {
1178
1861
  return buildManifest({
1179
1862
  projectRoot,
@@ -1227,6 +1910,7 @@ function manifestDiagnosticToMetadataDiagnostic(diagnostic) {
1227
1910
  };
1228
1911
  }
1229
1912
  function objectFromKGNode(node) {
1913
+ const qualifiedIdentity = qualifiedIdentityFromKGNode(node);
1230
1914
  const payload = {
1231
1915
  ...(node.payload ?? {}),
1232
1916
  kgNodeId: node.nodeId,
@@ -1261,13 +1945,21 @@ function objectFromKGNode(node) {
1261
1945
  businessRules: node.businessRules ?? [],
1262
1946
  caveats: node.caveats ?? [],
1263
1947
  llmContext: node.llmContext,
1948
+ label: kgContextField(node.llmContext, 'label'),
1949
+ metricType: kgContextField(node.llmContext, 'metric type'),
1950
+ aggregation: kgContextField(node.llmContext, 'aggregation'),
1951
+ table: kgContextField(node.llmContext, 'table'),
1952
+ formula: node.kind === 'metric' ? kgContextField(node.llmContext, 'sql') : undefined,
1953
+ semanticModel: ['metric', 'dimension', 'measure', 'entity'].includes(node.kind) && node.name.includes('.')
1954
+ ? node.name.split('.')[0]
1955
+ : undefined,
1264
1956
  referencedBy: node.referencedBy ?? [],
1265
1957
  };
1266
1958
  return {
1267
1959
  objectKey: objectKeyFromKGNode(node),
1268
1960
  objectType: objectTypeFromKGNode(node),
1269
1961
  name: node.name,
1270
- fullName: node.name,
1962
+ fullName: qualifiedIdentity ?? node.name,
1271
1963
  domain: node.domain,
1272
1964
  owner: node.owner,
1273
1965
  status: node.status ?? node.certification,
@@ -1277,7 +1969,18 @@ function objectFromKGNode(node) {
1277
1969
  payload: compactObject(payload),
1278
1970
  };
1279
1971
  }
1972
+ function kgContextField(context, field) {
1973
+ if (!context)
1974
+ return undefined;
1975
+ const prefix = `${field.toLowerCase()}:`;
1976
+ const line = context.split(/\r?\n/).find((candidate) => candidate.trim().toLowerCase().startsWith(prefix));
1977
+ return line?.trim().slice(prefix.length).trim() || undefined;
1978
+ }
1280
1979
  function objectKeyFromKGNode(node) {
1980
+ const qualifiedIdentity = qualifiedIdentityFromKGNode(node);
1981
+ if (qualifiedIdentity && isQualifiedDqlModelingNode(node)) {
1982
+ return `dql:${node.kind}:${qualifiedIdentity}`;
1983
+ }
1281
1984
  switch (node.kind) {
1282
1985
  case 'block': return `dql:block:${node.name}`;
1283
1986
  case 'term': return `dql:term:${node.name}`;
@@ -1300,6 +2003,29 @@ function objectKeyFromKGNode(node) {
1300
2003
  default: return `${node.kind}:${node.name}`;
1301
2004
  }
1302
2005
  }
2006
+ function isQualifiedDqlModelingNode(node) {
2007
+ return node.sourceTier === 'business_context' && [
2008
+ 'model_area',
2009
+ 'entity',
2010
+ 'relationship',
2011
+ 'contract',
2012
+ 'domain_export',
2013
+ 'domain_import',
2014
+ 'conformance',
2015
+ 'policy',
2016
+ 'evaluation',
2017
+ ].includes(node.kind);
2018
+ }
2019
+ function qualifiedIdentityFromKGNode(node) {
2020
+ const payload = node.payload ?? {};
2021
+ const explicit = stringValue(payload.qualifiedId);
2022
+ if (explicit)
2023
+ return explicit;
2024
+ if (!isQualifiedDqlModelingNode(node))
2025
+ return undefined;
2026
+ const prefix = `${node.kind}:`;
2027
+ return node.nodeId.startsWith(prefix) ? node.nodeId.slice(prefix.length) : undefined;
2028
+ }
1303
2029
  function objectTypeFromKGNode(node) {
1304
2030
  switch (node.kind) {
1305
2031
  case 'block': return 'dql_block';
@@ -1308,7 +2034,7 @@ function objectTypeFromKGNode(node) {
1308
2034
  case 'metric': return 'semantic_metric';
1309
2035
  case 'dimension': return 'semantic_dimension';
1310
2036
  case 'measure': return 'semantic_measure';
1311
- case 'entity': return 'semantic_entity';
2037
+ case 'entity': return isQualifiedDqlModelingNode(node) ? 'dql_entity' : 'semantic_entity';
1312
2038
  case 'semantic_model': return 'semantic_model';
1313
2039
  case 'saved_query': return 'semantic_saved_query';
1314
2040
  case 'dbt_model': return 'dbt_model';
@@ -1337,6 +2063,15 @@ function normalizeEdge(edge, fromKey, toKey) {
1337
2063
  function addSkillObjects(skills, objects, edges) {
1338
2064
  for (const skill of skills) {
1339
2065
  const identity = skill.qualifiedId ?? skill.id;
2066
+ const bodyHash = sha256(skill.body);
2067
+ const sourceFingerprint = sha256(stableStringify({
2068
+ identity,
2069
+ domain: skill.domain,
2070
+ domains: skill.domains ?? [],
2071
+ modelAreaRefs: skill.modelAreaRefs ?? [],
2072
+ status: skill.status ?? 'active',
2073
+ bodyHash,
2074
+ }));
1340
2075
  const objectKey = `skill:${identity}`;
1341
2076
  const domains = uniqueNonBlank([skill.domain, ...(skill.domains ?? [])]);
1342
2077
  const object = {
@@ -1369,12 +2104,13 @@ function addSkillObjects(skills, objects, edges) {
1369
2104
  examples: skill.examples ?? [],
1370
2105
  sourceRefs: skill.sourceRefs ?? [],
1371
2106
  vocabulary: skill.vocabulary,
1372
- body: skill.body,
2107
+ bodyHash,
2108
+ sourceFingerprint,
1373
2109
  isStarter: skill.isStarter,
1374
2110
  provenance: 'DQL domain skill',
1375
2111
  }),
1376
2112
  };
1377
- objects.set(objectKey, object);
2113
+ objects.set(objectKey, mergeObject(objects.get(objectKey), object));
1378
2114
  for (const domain of domains) {
1379
2115
  const edge = {
1380
2116
  edgeType: 'contains',
@@ -2444,9 +3180,16 @@ async function planContextPackRoute(input) {
2444
3180
  && (fit.confidence === 'high' || fit.confidence === 'medium'))
2445
3181
  .sort((a, b) => (b.fit.confidence === 'high' ? 1 : 0) - (a.fit.confidence === 'high' ? 1 : 0)
2446
3182
  || b.applicabilityScore - a.applicabilityScore)[0]?.object;
2447
- let exact = exactByApplicability
3183
+ const directlyNamedCertified = findExactCertifiedObject(input.request.question, intent, input.objects);
3184
+ const applicabilityCertified = exactByApplicability
2448
3185
  ? input.objects.find((object) => object.objectKey === exactByApplicability.objectKey)
2449
- : findExactCertifiedObject(input.request.question, intent, input.objects) ?? exactByContract;
3186
+ : undefined;
3187
+ // Selection order is governance-significant (AGT-009/AGT-010): an explicit
3188
+ // block/example reference wins, then a complete answer-shape fit, and only then
3189
+ // the older lexical applicability heuristic. This prevents a broad customer
3190
+ // block containing the word "spend" from displacing an exact beverage-scoped
3191
+ // customer ranking whose measure is named "beverage_revenue".
3192
+ let exact = directlyNamedCertified ?? exactByContract ?? applicabilityCertified;
2450
3193
  const contextApplicability = [...applicabilityByKey.values()]
2451
3194
  .filter((item) => item.kind === 'context_only')
2452
3195
  .sort((a, b) => b.score - a.score)[0];
@@ -2478,7 +3221,6 @@ async function planContextPackRoute(input) {
2478
3221
  const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
2479
3222
  const directCertifiedBypass = Boolean(exact && (exactExampleMatch
2480
3223
  || intent === 'definition_lookup'
2481
- || intent === 'exact_certified_lookup'
2482
3224
  || objectNameInQuestion(input.request.question, exact)));
2483
3225
  const blockFitObject = exact
2484
3226
  ?? (contextApplicability ? input.objects.find((object) => object.objectKey === contextApplicability.objectKey) : undefined);
@@ -2542,7 +3284,9 @@ async function planContextPackRoute(input) {
2542
3284
  || (intent === 'ad_hoc_ranking' && objectNameInQuestion(input.request.question, exact)))) {
2543
3285
  return {
2544
3286
  route: 'certified',
2545
- intent,
3287
+ intent: input.questionPlan.outputShape === 'value' && input.questionPlan.requestedShape.dimensions.length === 0
3288
+ ? 'exact_certified_lookup'
3289
+ : intent,
2546
3290
  reason: `Certified ${exact.objectType.replace(/_/g, ' ')} "${exact.name}" exactly matches the requested artifact, definition, or direct KPI grain.`,
2547
3291
  routeReason: grainGateInfo?.reason,
2548
3292
  grainGate: grainGateInfo,
@@ -4094,23 +4838,7 @@ function runtimeSchemaObjects(snapshot) {
4094
4838
  const capturedAt = snapshot.capturedAt ?? new Date().toISOString();
4095
4839
  return normalizeRuntimeSchemaTables(snapshot.tables).flatMap((table) => {
4096
4840
  const relation = table.relation;
4097
- const tableKey = `runtime:table:${relation}`;
4098
- const tableObject = {
4099
- objectKey: tableKey,
4100
- objectType: 'runtime_table',
4101
- name: table.name ?? relation.split('.').at(-1) ?? relation,
4102
- fullName: relation,
4103
- description: table.description,
4104
- status: 'runtime_observed',
4105
- sourceSystem: snapshot.source ?? table.source ?? 'runtime schema snapshot',
4106
- payload: compactObject({
4107
- relation,
4108
- schema: table.schema,
4109
- columnCompleteness: 'complete',
4110
- columns: table.columns,
4111
- }),
4112
- updatedAt: capturedAt,
4113
- };
4841
+ const tableObject = runtimeSchemaTableObject(table, snapshot.source, capturedAt);
4114
4842
  const columns = table.columns.map((column) => ({
4115
4843
  objectKey: `runtime:column:${relation}.${column.name}`,
4116
4844
  objectType: 'runtime_column',
@@ -4130,6 +4858,25 @@ function runtimeSchemaObjects(snapshot) {
4130
4858
  return [tableObject, ...columns];
4131
4859
  });
4132
4860
  }
4861
+ function runtimeSchemaTableObject(table, snapshotSource, capturedAt) {
4862
+ const relation = table.relation;
4863
+ return {
4864
+ objectKey: `runtime:table:${relation}`,
4865
+ objectType: 'runtime_table',
4866
+ name: table.name ?? relation.split('.').at(-1) ?? relation,
4867
+ fullName: relation,
4868
+ description: table.description,
4869
+ status: 'runtime_observed',
4870
+ sourceSystem: snapshotSource ?? table.source ?? 'runtime schema snapshot',
4871
+ payload: compactObject({
4872
+ relation,
4873
+ schema: table.schema,
4874
+ columnCompleteness: 'complete',
4875
+ columns: table.columns,
4876
+ }),
4877
+ updatedAt: capturedAt,
4878
+ };
4879
+ }
4133
4880
  function runtimeValueMatchObjects(matches) {
4134
4881
  return matches.map((match) => ({
4135
4882
  objectKey: match.valueKey,
@@ -4370,7 +5117,7 @@ const METADATA_VALUE_STOP_TERMS = new Set([
4370
5117
  'this year',
4371
5118
  'user',
4372
5119
  ]);
4373
- function buildFollowUpSearchQuery(question, followUp) {
5120
+ export function buildFollowUpSearchQuery(question, followUp) {
4374
5121
  if (!followUp)
4375
5122
  return question;
4376
5123
  // Contextual carry is advisory: enrich retrieval softly (prior question, columns,
@@ -4392,14 +5139,19 @@ function buildFollowUpSearchQuery(question, followUp) {
4392
5139
  question,
4393
5140
  followUp.sourceBlockName ?? '',
4394
5141
  followUp.sourceQuestion ?? '',
4395
- ...(followUp.filters ?? []),
4396
5142
  ...(followUp.dimensions ?? []),
4397
5143
  ...(followUp.priorResultColumns ?? []),
4398
5144
  followUp.priorResultRef?.question ?? '',
4399
5145
  ...(followUp.priorResultRef?.columns ?? []),
4400
- followUp.priorResultRef?.sourceSql ? followUp.priorResultRef.sourceSql.slice(0, 600) : '',
4401
- ...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: true }),
4402
- ...Object.entries(followUp.priorResultValues ?? {}).flatMap(([key, values]) => [key, ...values]),
5146
+ ...(followUp.priorMeasures ?? []),
5147
+ // Retrieval searches object meaning, not executable payloads or warehouse
5148
+ // members. The source block is loaded directly by key and the typed follow-up
5149
+ // object retains SQL + values for execution. Injecting 600-1500 characters of
5150
+ // SQL and every prior row value into FTS made an eight-customer drilldown spend
5151
+ // ~11 seconds tokenizing metadata before the provider even started.
5152
+ ...(followUp.priorDqlArtifact?.metrics ?? []),
5153
+ ...(followUp.priorDqlArtifact?.dimensions ?? []),
5154
+ ...(followUp.priorDqlArtifact?.filters ?? []).map((filter) => filter.dimension),
4403
5155
  ].filter(Boolean).join(' ');
4404
5156
  }
4405
5157
  function dqlArtifactSearchTerms(artifact, options) {
@@ -4448,7 +5200,15 @@ function normalizeRuntimeSchemaTables(tables) {
4448
5200
  name: table.name ?? relation.split('.').at(-1) ?? relation,
4449
5201
  description: table.description,
4450
5202
  source: table.source,
4451
- columns: dedupeRuntimeColumns(table.columns ?? []).slice(0, 160),
5203
+ // Persist structural schema only. Live sample values can contain PII and
5204
+ // are scoped to a single request by the CLI (SEC-003).
5205
+ columns: dedupeRuntimeColumns(table.columns ?? [])
5206
+ .slice(0, 160)
5207
+ .map((column) => ({
5208
+ name: column.name,
5209
+ type: column.type,
5210
+ description: column.description,
5211
+ })),
4452
5212
  };
4453
5213
  if (!current)
4454
5214
  byRelation.set(key, normalized);
@@ -4670,6 +5430,54 @@ function compactObject(value) {
4670
5430
  }
4671
5431
  return out;
4672
5432
  }
5433
+ const SEARCHABLE_METADATA_KEYS = new Set([
5434
+ 'aliases', 'synonyms', 'tags', 'label', 'metricType', 'aggregation', 'semanticModel',
5435
+ 'localId', 'qualifiedId', 'uniqueId',
5436
+ 'description', 'businessOutcome', 'decisionUse', 'grain', 'entities',
5437
+ 'dimensions', 'allowedFilters', 'declaredOutputs', 'outputContract', 'outputs',
5438
+ 'parameters', 'parameterPolicy', 'tableDependencies', 'sourceSystems', 'relation',
5439
+ 'table', 'database', 'schema', 'materialized', 'columns', 'intentExamples',
5440
+ 'examples', 'primaryTerms', 'businessRules', 'caveats', 'triggers', 'vocabulary',
5441
+ ]);
5442
+ const SEARCHABLE_NESTED_METADATA_KEYS = new Set([
5443
+ 'name', 'id', 'label', 'description', 'type', 'role', 'filter', 'binding',
5444
+ 'question', 'entity', 'column', 'relation', 'table', 'domain', 'term', 'alias',
5445
+ 'synonym', 'localId', 'qualifiedId', 'grain',
5446
+ ]);
5447
+ /**
5448
+ * FTS indexes business/schema evidence, not the complete payload. In particular,
5449
+ * raw SQL and provider/runtime fields add large amounts of noise and can contain
5450
+ * literals that should never become general retrieval context.
5451
+ */
5452
+ function searchableMetadataPayload(payload) {
5453
+ const tokens = [];
5454
+ const visit = (value, depth) => {
5455
+ if (tokens.length >= 2_000 || depth > 3 || value === null || value === undefined)
5456
+ return;
5457
+ if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
5458
+ const text = String(value).replace(/\s+/g, ' ').trim();
5459
+ if (text)
5460
+ tokens.push(text.slice(0, 500));
5461
+ return;
5462
+ }
5463
+ if (Array.isArray(value)) {
5464
+ for (const item of value.slice(0, 500))
5465
+ visit(item, depth + 1);
5466
+ return;
5467
+ }
5468
+ if (typeof value !== 'object')
5469
+ return;
5470
+ for (const [key, item] of Object.entries(value)) {
5471
+ if (depth === 0 ? !SEARCHABLE_METADATA_KEYS.has(key) : !SEARCHABLE_NESTED_METADATA_KEYS.has(key))
5472
+ continue;
5473
+ if (/sql|secret|password|token|credential|api.?key/i.test(key))
5474
+ continue;
5475
+ visit(item, depth + 1);
5476
+ }
5477
+ };
5478
+ visit(payload, 0);
5479
+ return tokens.join(' ');
5480
+ }
4673
5481
  function rowToObject(row) {
4674
5482
  return {
4675
5483
  objectKey: row.object_key,
@@ -4962,7 +5770,7 @@ function filterMetadataObjectsByDomainContext(rows, context) {
4962
5770
  ]);
4963
5771
  return rows.filter((row) => !row.domain || domains.has(row.domain));
4964
5772
  }
4965
- function selectContextPackSkills(objects, question, context) {
5773
+ function selectContextPackSkills(catalog, objects, question, context) {
4966
5774
  const byIdentity = new Map();
4967
5775
  const parsed = objects.flatMap((object) => {
4968
5776
  const skill = skillFromMetadataObject(object);
@@ -4978,10 +5786,15 @@ function selectContextPackSkills(objects, question, context) {
4978
5786
  const selected = selectRelevantSkills(parsed, question, {
4979
5787
  domains,
4980
5788
  modelAreaIds: context?.modelAreaId ? [context.modelAreaId] : [],
5789
+ pinnedIds: context?.skillRefs ? ['sql-conventions', ...context.skillRefs] : undefined,
4981
5790
  });
4982
5791
  return selected.flatMap((skill) => {
4983
5792
  const object = byIdentity.get(skill.qualifiedId ?? skill.id);
4984
- return object ? [{ object, skill: localContextSkillFromParsed(skill, object) }] : [];
5793
+ if (!object)
5794
+ return [];
5795
+ const bodyHash = stringPayload(object.payload?.bodyHash);
5796
+ const hydrated = bodyHash ? { ...skill, body: catalog.skillBody(bodyHash) ?? '' } : skill;
5797
+ return [{ object, skill: localContextSkillFromParsed(hydrated, object) }];
4985
5798
  });
4986
5799
  }
4987
5800
  function skillFromMetadataObject(object) {
@@ -5017,7 +5830,7 @@ function skillFromMetadataObject(object) {
5017
5830
  examples: stringArrayPayload(payload.examples),
5018
5831
  sourceRefs: stringArrayPayload(payload.sourceRefs),
5019
5832
  vocabulary: stringRecordPayload(payload.vocabulary),
5020
- body: stringPayload(payload.body) ?? '',
5833
+ body: '',
5021
5834
  sourcePath: object.sourcePath ?? '',
5022
5835
  isStarter: payload.isStarter === true,
5023
5836
  };