@duckcodeailabs/dql-agent 1.7.2 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-run-engine.d.ts +2 -0
- package/dist/agent-run-engine.d.ts.map +1 -1
- package/dist/agent-run-engine.js +67 -19
- package/dist/agent-run-engine.js.map +1 -1
- package/dist/agent-run-gates.d.ts.map +1 -1
- package/dist/agent-run-gates.js +14 -50
- package/dist/agent-run-gates.js.map +1 -1
- package/dist/agent-run-planner.d.ts.map +1 -1
- package/dist/agent-run-planner.js +14 -4
- package/dist/agent-run-planner.js.map +1 -1
- package/dist/agentic/toolset.js +32 -2
- package/dist/agentic/toolset.js.map +1 -1
- package/dist/answer-loop.d.ts +24 -0
- package/dist/answer-loop.d.ts.map +1 -1
- package/dist/answer-loop.js +387 -55
- package/dist/answer-loop.js.map +1 -1
- package/dist/answer-shape.js +14 -1
- package/dist/answer-shape.js.map +1 -1
- package/dist/cascade/budgets.d.ts.map +1 -1
- package/dist/cascade/budgets.js +5 -2
- package/dist/cascade/budgets.js.map +1 -1
- package/dist/conversation/session-store.d.ts +3 -0
- package/dist/conversation/session-store.d.ts.map +1 -1
- package/dist/conversation/session-store.js +8 -3
- package/dist/conversation/session-store.js.map +1 -1
- package/dist/conversation/snapshot.d.ts +2 -0
- package/dist/conversation/snapshot.d.ts.map +1 -1
- package/dist/conversation/snapshot.js +1 -0
- package/dist/conversation/snapshot.js.map +1 -1
- package/dist/conversation/working-state.d.ts +3 -0
- package/dist/conversation/working-state.d.ts.map +1 -1
- package/dist/conversation/working-state.js +26 -0
- package/dist/conversation/working-state.js.map +1 -1
- package/dist/domain-context.d.ts +14 -0
- package/dist/domain-context.d.ts.map +1 -1
- package/dist/domain-context.js +10 -0
- package/dist/domain-context.js.map +1 -1
- package/dist/embeddings/provider.d.ts +3 -1
- package/dist/embeddings/provider.d.ts.map +1 -1
- package/dist/embeddings/provider.js +19 -8
- package/dist/embeddings/provider.js.map +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +33 -4
- package/dist/index.js.map +1 -1
- package/dist/intent-controller.d.ts +15 -0
- package/dist/intent-controller.d.ts.map +1 -1
- package/dist/intent-controller.js +4 -1
- package/dist/intent-controller.js.map +1 -1
- package/dist/meaning-resolution.d.ts +110 -0
- package/dist/meaning-resolution.d.ts.map +1 -0
- package/dist/meaning-resolution.js +155 -0
- package/dist/meaning-resolution.js.map +1 -0
- package/dist/metadata/analysis-planner.d.ts +15 -1
- package/dist/metadata/analysis-planner.d.ts.map +1 -1
- package/dist/metadata/analysis-planner.js +169 -50
- package/dist/metadata/analysis-planner.js.map +1 -1
- package/dist/metadata/block-fit.d.ts.map +1 -1
- package/dist/metadata/block-fit.js +51 -5
- package/dist/metadata/block-fit.js.map +1 -1
- package/dist/metadata/catalog.d.ts +62 -3
- package/dist/metadata/catalog.d.ts.map +1 -1
- package/dist/metadata/catalog.js +911 -98
- package/dist/metadata/catalog.js.map +1 -1
- package/dist/metadata/grain-gate.js +27 -9
- package/dist/metadata/grain-gate.js.map +1 -1
- package/dist/metadata/meaning-evidence.d.ts +70 -0
- package/dist/metadata/meaning-evidence.d.ts.map +1 -0
- package/dist/metadata/meaning-evidence.js +333 -0
- package/dist/metadata/meaning-evidence.js.map +1 -0
- package/dist/metadata/metric-match.d.ts.map +1 -1
- package/dist/metadata/metric-match.js +5 -2
- package/dist/metadata/metric-match.js.map +1 -1
- package/dist/router.d.ts +16 -4
- package/dist/router.d.ts.map +1 -1
- package/dist/router.js +409 -21
- package/dist/router.js.map +1 -1
- package/dist/semantic-bridge/compose.d.ts +12 -0
- package/dist/semantic-bridge/compose.d.ts.map +1 -1
- package/dist/semantic-bridge/compose.js +177 -14
- package/dist/semantic-bridge/compose.js.map +1 -1
- package/dist/semantic-bridge/member-select.d.ts.map +1 -1
- package/dist/semantic-bridge/member-select.js +59 -10
- package/dist/semantic-bridge/member-select.js.map +1 -1
- package/dist/skills/loader.d.ts.map +1 -1
- package/dist/skills/loader.js +35 -33
- package/dist/skills/loader.js.map +1 -1
- package/dist/synthesize.d.ts +2 -0
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +84 -13
- package/dist/synthesize.js.map +1 -1
- package/package.json +4 -4
package/dist/metadata/catalog.js
CHANGED
|
@@ -7,22 +7,24 @@
|
|
|
7
7
|
* context pack before answering.
|
|
8
8
|
*/
|
|
9
9
|
import { createHash } from 'node:crypto';
|
|
10
|
-
import { mkdirSync, readFileSync } from 'node:fs';
|
|
10
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
11
11
|
import { createRequire } from 'node:module';
|
|
12
12
|
import { dirname, join } from 'node:path';
|
|
13
|
-
import {
|
|
13
|
+
import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib';
|
|
14
|
+
import { buildManifest, buildManifestKnowledgeGraph, loadDomainPackageRegistry, loadManifestKnowledgeSkills, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, extractColumnLineage, normalizeDqlArtifactReference, composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
|
|
14
15
|
import { buildKGFromManifest, buildKGFromSemanticLayer } from '../kg/build.js';
|
|
15
16
|
import { buildBlockBusinessFingerprint, buildBlockSqlFingerprints } from './block-fingerprints.js';
|
|
16
17
|
import { buildAnalysisQuestionPlan, certifiedApplicabilityForObject, scoreAllowedSqlRelationWithAnalysisPlan, scoreMetadataObjectWithAnalysisPlan, sortAllowedSqlContextForAnalysisPlan, } from './analysis-planner.js';
|
|
17
18
|
import { extractSimpleSelectShape, sourceSqlShapeColumns } from './sql-shape.js';
|
|
18
|
-
import { buildRuntimeValueIndex, normalizeValueIndexText, } from '../grounding/value-index.js';
|
|
19
19
|
import { grainMatches, requestedGrainFromPlan, } from './grain-gate.js';
|
|
20
20
|
import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit, } from './block-fit.js';
|
|
21
21
|
import { retrieveScopedHints } from '../hints/retrieval.js';
|
|
22
|
-
import { buildFtsMatch
|
|
22
|
+
import { buildFtsMatch } from '../memory/fts-query.js';
|
|
23
23
|
import { envEmbeddingProvider } from '../embeddings/provider.js';
|
|
24
24
|
import { matchExampleParaphrase } from './example-match.js';
|
|
25
25
|
import { loadSkills, selectRelevantSkills } from '../skills/loader.js';
|
|
26
|
+
import { buildMeaningEvidencePackage, } from './meaning-evidence.js';
|
|
27
|
+
export { toAgentRetrievalEvidence } from './meaning-evidence.js';
|
|
26
28
|
const require = createRequire(import.meta.url);
|
|
27
29
|
let databaseCtor = null;
|
|
28
30
|
function loadDatabase() {
|
|
@@ -73,6 +75,10 @@ const OBJECT_PRIORITY = {
|
|
|
73
75
|
semantic_dimension: 5,
|
|
74
76
|
semantic_measure: 6,
|
|
75
77
|
semantic_entity: 7,
|
|
78
|
+
dql_entity: 7,
|
|
79
|
+
relationship: 7.2,
|
|
80
|
+
contract: 7.4,
|
|
81
|
+
model_area: 7.6,
|
|
76
82
|
semantic_model: 8,
|
|
77
83
|
dbt_model: 9,
|
|
78
84
|
dbt_source: 10,
|
|
@@ -86,12 +92,143 @@ const OBJECT_PRIORITY = {
|
|
|
86
92
|
domain: 16,
|
|
87
93
|
};
|
|
88
94
|
const COLUMN_OBJECT_TYPES = new Set(['dbt_column', 'warehouse_column', 'runtime_column', 'runtime_value']);
|
|
95
|
+
// v3 removes the legacy persisted plaintext runtime-value cache (SEC-003).
|
|
96
|
+
const METADATA_INDEX_VERSION = 'metadata-index-v4-content-addressed-skills';
|
|
89
97
|
export function defaultMetadataPath(projectRoot) {
|
|
90
98
|
return join(projectRoot, '.dql', 'cache', 'metadata.sqlite');
|
|
91
99
|
}
|
|
100
|
+
export function metadataSnapshotPath(projectRoot, fingerprint) {
|
|
101
|
+
return join(projectRoot, '.dql', 'cache', 'snapshots', `${fingerprint}.sqlite`);
|
|
102
|
+
}
|
|
103
|
+
export function activeMetadataSnapshotPath(projectRoot) {
|
|
104
|
+
const pointerPath = join(projectRoot, '.dql', 'cache', 'active-snapshot.json');
|
|
105
|
+
if (!existsSync(pointerPath))
|
|
106
|
+
return null;
|
|
107
|
+
try {
|
|
108
|
+
const parsed = JSON.parse(readFileSync(pointerPath, 'utf8'));
|
|
109
|
+
const path = typeof parsed.snapshotPath === 'string' ? parsed.snapshotPath : '';
|
|
110
|
+
return path && existsSync(path) ? path : null;
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
/** Read the prepared catalog identity without rebuilding the metadata snapshot. */
|
|
117
|
+
export function currentMetadataFingerprint(projectRoot) {
|
|
118
|
+
if (!existsSync(defaultMetadataPath(projectRoot)))
|
|
119
|
+
return undefined;
|
|
120
|
+
const catalog = openMetadataCatalog(projectRoot);
|
|
121
|
+
try {
|
|
122
|
+
return catalog.state('fingerprint') ?? undefined;
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
catalog.close();
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function activateMetadataSnapshot(projectRoot, fingerprint, catalog) {
|
|
129
|
+
const snapshotPath = metadataSnapshotPath(projectRoot, fingerprint);
|
|
130
|
+
catalog.exportSnapshot(snapshotPath);
|
|
131
|
+
const cacheDir = join(projectRoot, '.dql', 'cache');
|
|
132
|
+
const pointerPath = join(cacheDir, 'active-snapshot.json');
|
|
133
|
+
const candidate = `${pointerPath}.candidate`;
|
|
134
|
+
mkdirSync(cacheDir, { recursive: true });
|
|
135
|
+
writeFileSync(candidate, JSON.stringify({
|
|
136
|
+
snapshotId: fingerprint,
|
|
137
|
+
indexSchemaVersion: METADATA_INDEX_VERSION,
|
|
138
|
+
snapshotPath,
|
|
139
|
+
}, null, 2), 'utf8');
|
|
140
|
+
renameSync(candidate, pointerPath);
|
|
141
|
+
// Do not delete older immutable files here: a request that acquired the old
|
|
142
|
+
// pointer may still hold it open. Cache cleanup is an explicit maintenance
|
|
143
|
+
// concern until the snapshot service owns reference-counted leases.
|
|
144
|
+
return snapshotPath;
|
|
145
|
+
}
|
|
92
146
|
export function openMetadataCatalog(projectRoot, dbPath = defaultMetadataPath(projectRoot)) {
|
|
93
147
|
return new MetadataCatalog(dbPath);
|
|
94
148
|
}
|
|
149
|
+
export function openActiveKnowledgeSnapshot(projectRoot) {
|
|
150
|
+
const path = activeMetadataSnapshotPath(projectRoot) ?? defaultMetadataPath(projectRoot);
|
|
151
|
+
return new MetadataCatalog(path, { readOnly: path !== defaultMetadataPath(projectRoot) });
|
|
152
|
+
}
|
|
153
|
+
export function readIndexedDomainKnowledge(projectRoot, domainId) {
|
|
154
|
+
const catalog = openActiveKnowledgeSnapshot(projectRoot);
|
|
155
|
+
try {
|
|
156
|
+
const snapshotId = catalog.state('fingerprint') ?? 'metadata-unavailable';
|
|
157
|
+
const capsuleObject = catalog.listAllObjects({ objectTypes: ['domain_capsule'], domain: domainId })
|
|
158
|
+
.find((item) => !stringValue(item.payload?.modelAreaId));
|
|
159
|
+
const capsule = capsuleObject?.payload;
|
|
160
|
+
if (!capsule)
|
|
161
|
+
return null;
|
|
162
|
+
const rows = catalog.listAllObjects({ domain: domainId })
|
|
163
|
+
.filter((item) => item.objectType !== 'domain_capsule' && item.objectType !== 'cross_domain_route');
|
|
164
|
+
const selectedRows = rows.slice(0, 750);
|
|
165
|
+
const byKey = new Map(selectedRows.map((row) => [row.objectKey, metadataObjectToKnowledge(row)]));
|
|
166
|
+
const edgeRows = catalog.edgesForKeys(selectedRows.map((row) => row.objectKey), 1).slice(0, 1_500);
|
|
167
|
+
for (const edge of edgeRows) {
|
|
168
|
+
if (!byKey.has(edge.fromKey)) {
|
|
169
|
+
const item = catalog.getObject(edge.fromKey);
|
|
170
|
+
if (item)
|
|
171
|
+
byKey.set(edge.fromKey, metadataObjectToKnowledge(item));
|
|
172
|
+
}
|
|
173
|
+
if (!byKey.has(edge.toKey)) {
|
|
174
|
+
const item = catalog.getObject(edge.toKey);
|
|
175
|
+
if (item)
|
|
176
|
+
byKey.set(edge.toKey, metadataObjectToKnowledge(item));
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const routes = catalog.listAllObjects({ objectTypes: ['cross_domain_route'] })
|
|
180
|
+
.flatMap((item) => metadataRoute(item) ? [metadataRoute(item)] : [])
|
|
181
|
+
.filter((route) => route.providerDomainId === domainId || route.consumerDomainId === domainId);
|
|
182
|
+
const routeStates = routes.reduce((counts, route) => {
|
|
183
|
+
counts[route.state] = (counts[route.state] ?? 0) + 1;
|
|
184
|
+
return counts;
|
|
185
|
+
}, {});
|
|
186
|
+
return {
|
|
187
|
+
schemaVersion: 2,
|
|
188
|
+
snapshotId,
|
|
189
|
+
sourceFingerprint: stringValue(capsuleObject?.payload?.sourceFingerprint) ?? snapshotId,
|
|
190
|
+
domainId,
|
|
191
|
+
capsule,
|
|
192
|
+
counts: { objects: rows.length, edges: edgeRows.length, routes: routes.length, routeStates },
|
|
193
|
+
objects: selectedRows.map(metadataObjectToKnowledge),
|
|
194
|
+
edges: edgeRows.map((edge) => metadataEdgeToKnowledge(edge, byKey)),
|
|
195
|
+
routes,
|
|
196
|
+
truncated: rows.length > selectedRows.length || edgeRows.length >= 1_500,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
finally {
|
|
200
|
+
catalog.close();
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
export function readIndexedKnowledge360(projectRoot, identity) {
|
|
204
|
+
const catalog = openActiveKnowledgeSnapshot(projectRoot);
|
|
205
|
+
try {
|
|
206
|
+
const focusRow = catalog.findObjectByIdentity(identity);
|
|
207
|
+
if (!focusRow)
|
|
208
|
+
return null;
|
|
209
|
+
const edgeRows = catalog.edgesForKeys([focusRow.objectKey], 2).slice(0, 500);
|
|
210
|
+
const keys = [...new Set([focusRow.objectKey, ...edgeRows.flatMap((edge) => [edge.fromKey, edge.toKey])])].slice(0, 160);
|
|
211
|
+
const rows = catalog.getObjectsByKeys(keys);
|
|
212
|
+
const byKey = new Map(rows.map((row) => [row.objectKey, metadataObjectToKnowledge(row)]));
|
|
213
|
+
const objects = rows.map(metadataObjectToKnowledge);
|
|
214
|
+
const domains = new Set(objects.flatMap((item) => item.domainId ? [item.domainId] : []));
|
|
215
|
+
const routes = catalog.listAllObjects({ objectTypes: ['cross_domain_route'] })
|
|
216
|
+
.flatMap((item) => metadataRoute(item) ? [metadataRoute(item)] : [])
|
|
217
|
+
.filter((route) => domains.has(route.providerDomainId) && domains.has(route.consumerDomainId));
|
|
218
|
+
return {
|
|
219
|
+
snapshotId: catalog.state('fingerprint') ?? 'metadata-unavailable',
|
|
220
|
+
sourceFingerprint: catalog.state('fingerprint') ?? 'metadata-unavailable',
|
|
221
|
+
focus: metadataObjectToKnowledge(focusRow),
|
|
222
|
+
objects,
|
|
223
|
+
edges: edgeRows.filter((edge) => byKey.has(edge.fromKey) && byKey.has(edge.toKey)).map((edge) => metadataEdgeToKnowledge(edge, byKey)),
|
|
224
|
+
routes,
|
|
225
|
+
truncated: keys.length >= 160 || edgeRows.length >= 500,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
catalog.close();
|
|
230
|
+
}
|
|
231
|
+
}
|
|
95
232
|
export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
|
|
96
233
|
const semanticLayer = options.semanticLayer !== undefined
|
|
97
234
|
? options.semanticLayer ?? undefined
|
|
@@ -102,9 +239,12 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
|
|
|
102
239
|
const catalog = openMetadataCatalog(projectRoot);
|
|
103
240
|
try {
|
|
104
241
|
const existing = catalog.state('fingerprint');
|
|
105
|
-
|
|
242
|
+
const compatibleIndex = catalog.state('index_version') === METADATA_INDEX_VERSION;
|
|
243
|
+
if (!options.force && compatibleIndex && existing === snapshot.fingerprint) {
|
|
244
|
+
const snapshotPath = activateMetadataSnapshot(projectRoot, snapshot.fingerprint, catalog);
|
|
106
245
|
return {
|
|
107
246
|
path: defaultMetadataPath(projectRoot),
|
|
247
|
+
snapshotPath,
|
|
108
248
|
refreshed: false,
|
|
109
249
|
objectCount: catalog.objectCount(),
|
|
110
250
|
edgeCount: catalog.edgeCount(),
|
|
@@ -115,14 +255,16 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
|
|
|
115
255
|
// W3.4 — incremental reindex when prior state exists (only re-tokenizes changed
|
|
116
256
|
// sources' FTS); `force` does a clean full rebuild. Proven equal to a full
|
|
117
257
|
// rebuild by incremental-reindex.test.ts.
|
|
118
|
-
if (options.force) {
|
|
258
|
+
if (options.force || !compatibleIndex) {
|
|
119
259
|
catalog.rebuild(snapshot);
|
|
120
260
|
}
|
|
121
261
|
else {
|
|
122
262
|
catalog.rebuildIncremental(snapshot);
|
|
123
263
|
}
|
|
264
|
+
const snapshotPath = activateMetadataSnapshot(projectRoot, snapshot.fingerprint, catalog);
|
|
124
265
|
return {
|
|
125
266
|
path: defaultMetadataPath(projectRoot),
|
|
267
|
+
snapshotPath,
|
|
126
268
|
refreshed: true,
|
|
127
269
|
objectCount: snapshot.objects.length,
|
|
128
270
|
edgeCount: snapshot.edges.length,
|
|
@@ -138,8 +280,10 @@ export function upsertMetadataSnapshot(projectRoot, snapshot) {
|
|
|
138
280
|
const catalog = openMetadataCatalog(projectRoot);
|
|
139
281
|
try {
|
|
140
282
|
catalog.rebuild(snapshot);
|
|
283
|
+
const snapshotPath = activateMetadataSnapshot(projectRoot, snapshot.fingerprint, catalog);
|
|
141
284
|
return {
|
|
142
285
|
path: defaultMetadataPath(projectRoot),
|
|
286
|
+
snapshotPath,
|
|
143
287
|
refreshed: true,
|
|
144
288
|
objectCount: snapshot.objects.length,
|
|
145
289
|
edgeCount: snapshot.edges.length,
|
|
@@ -164,7 +308,12 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
164
308
|
}
|
|
165
309
|
if (!prepared)
|
|
166
310
|
await ensureMetadataCatalogFresh(projectRoot);
|
|
167
|
-
|
|
311
|
+
// Static governed knowledge is read from the immutable content-addressed
|
|
312
|
+
// snapshot selected at request start. Mutable run history, runtime schema,
|
|
313
|
+
// and context packs stay in the working catalog and never alter that view.
|
|
314
|
+
const snapshotPath = activeMetadataSnapshotPath(projectRoot) ?? defaultMetadataPath(projectRoot);
|
|
315
|
+
const catalog = openActiveKnowledgeSnapshot(projectRoot);
|
|
316
|
+
const runtimeCatalog = openMetadataCatalog(projectRoot);
|
|
168
317
|
try {
|
|
169
318
|
const mode = request.mode ?? 'question';
|
|
170
319
|
const followUp = normalizeFollowUpContext(request.followUp);
|
|
@@ -180,7 +329,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
180
329
|
&& request.priorContextPackId
|
|
181
330
|
&& request.conversationTopicRelation
|
|
182
331
|
&& request.conversationTopicRelation !== 'shift'
|
|
183
|
-
?
|
|
332
|
+
? runtimeCatalog.getContextPack(request.priorContextPackId)
|
|
184
333
|
: null;
|
|
185
334
|
const priorPackFresh = Boolean(priorPack && priorPack.freshness.fingerprint === catalog.state('fingerprint'));
|
|
186
335
|
if (priorPack
|
|
@@ -199,13 +348,13 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
199
348
|
strategy: 'reused_pack_refinement',
|
|
200
349
|
},
|
|
201
350
|
freshness: {
|
|
202
|
-
catalogPath:
|
|
351
|
+
catalogPath: snapshotPath,
|
|
203
352
|
builtAt: catalog.state('built_at'),
|
|
204
353
|
fingerprint: catalog.state('fingerprint'),
|
|
205
354
|
},
|
|
206
355
|
};
|
|
207
356
|
delete reusedPayload.id;
|
|
208
|
-
const reusedId =
|
|
357
|
+
const reusedId = runtimeCatalog.insertContextPack(reusedPayload);
|
|
209
358
|
return { ...reusedPayload, id: reusedId };
|
|
210
359
|
}
|
|
211
360
|
const priorSeedObjects = priorPack && priorPackFresh
|
|
@@ -214,17 +363,23 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
214
363
|
|| request.conversationTopicRelation === 'return')
|
|
215
364
|
? priorPack.objects.slice(0, 24)
|
|
216
365
|
: [];
|
|
217
|
-
const runtimeSnapshot = request.runtimeSchemaSnapshot ?? catalog.latestRuntimeSchemaSnapshot();
|
|
218
|
-
const runtimeObjects = runtimeSnapshot ? runtimeSchemaObjects(runtimeSnapshot) : [];
|
|
219
|
-
const runtimeValueObjects = runtimeValueMatchObjects(catalog.searchRuntimeValues(metadataValueSearchTerms(request.question, questionPlan, followUp), 32));
|
|
220
|
-
const selectedObjects = selectedContextObjects(request.selectedContext);
|
|
221
|
-
const followUpObjects = followUpContextObjects(followUp);
|
|
222
|
-
const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
|
|
223
366
|
const searchQueries = uniqueMetadataSearchQueries([
|
|
224
367
|
buildFollowUpSearchQuery(request.question, followUp),
|
|
225
368
|
...questionPlan.searchQueries,
|
|
226
369
|
]);
|
|
227
|
-
|
|
370
|
+
// Runtime database schemas live in a separate persisted FTS lane. Normal Ask
|
|
371
|
+
// requests hydrate only relevant tables, so thousands of tables and hundreds
|
|
372
|
+
// of thousands of columns never become a request-path full-catalog scan.
|
|
373
|
+
// Explicit snapshots remain supported for callers that already hold a small,
|
|
374
|
+
// question-scoped schema payload.
|
|
375
|
+
const runtimeObjects = request.runtimeSchemaSnapshot
|
|
376
|
+
? runtimeSchemaObjects(request.runtimeSchemaSnapshot)
|
|
377
|
+
: runtimeCatalog.searchRuntimeSchemaObjects(searchQueries.join(' '), Math.max(request.limit ?? 80, 20));
|
|
378
|
+
const runtimeValueObjects = runtimeValueMatchObjects(runtimeCatalog.searchRuntimeValues(metadataValueSearchTerms(request.question, questionPlan, followUp), 32));
|
|
379
|
+
const selectedObjects = selectedContextObjects(request.selectedContext);
|
|
380
|
+
const followUpObjects = followUpContextObjects(followUp);
|
|
381
|
+
const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
|
|
382
|
+
const areaObjects = filterMetadataObjectsByDomainContext(catalog.listAllObjects({ objectTypes: ['model_area'] }), request.domainContext);
|
|
228
383
|
const focusedArea = resolveFocusedModelArea(request.domainContext, request.question, areaObjects);
|
|
229
384
|
const effectiveDomainContext = focusedArea && request.domainContext
|
|
230
385
|
? { ...request.domainContext, modelAreaId: focusedArea.id }
|
|
@@ -280,18 +435,28 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
280
435
|
// Skills are selected from the catalog snapshot with the same hard domain,
|
|
281
436
|
// area, status, and exclusion gates as the loader. They are not left to FTS
|
|
282
437
|
// chance or re-read from disk after the snapshot has been fingerprinted.
|
|
283
|
-
const selectedSkills = selectContextPackSkills(catalog.
|
|
438
|
+
const selectedSkills = selectContextPackSkills(catalog, catalog.listAllObjects({ objectTypes: ['skill'] }), request.question, effectiveDomainContext);
|
|
284
439
|
const selectedSkillObjects = selectedSkills.map((item) => item.object);
|
|
440
|
+
const routeCandidates = effectiveDomainContext?.activeDomain
|
|
441
|
+
? catalog.crossDomainRouteObjects(effectiveDomainContext.activeDomain, 100)
|
|
442
|
+
: [];
|
|
443
|
+
const routeObjects = rankMetadataObjects({
|
|
444
|
+
rows: retrievalObjects(routeCandidates),
|
|
445
|
+
question: `${request.question} ${effectiveDomainContext?.purpose ?? ''}`,
|
|
446
|
+
questionPlan,
|
|
447
|
+
modelAreaId: focusedArea?.id,
|
|
448
|
+
limit: 8,
|
|
449
|
+
}).selected;
|
|
285
450
|
const objects = fullCatalogObjects
|
|
286
|
-
? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects, ...selectedSkillObjects])
|
|
287
|
-
: mergeObjects([...rankedObjects, ...sqlParentObjects, ...selectedSkillObjects]);
|
|
451
|
+
? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects, ...selectedSkillObjects, ...routeObjects])
|
|
452
|
+
: mergeObjects([...rankedObjects, ...sqlParentObjects, ...selectedSkillObjects, ...routeObjects]);
|
|
288
453
|
const objectKeys = objects.map((row) => row.objectKey);
|
|
289
454
|
const allowedObjectKeys = new Set(objectKeys);
|
|
290
455
|
const contextEdges = mergeMetadataEdges([
|
|
291
456
|
...edgeWalk,
|
|
292
457
|
...catalog.edgesForKeys(objectKeys, 2),
|
|
293
458
|
]).filter((edge) => allowedObjectKeys.has(edge.fromKey) && allowedObjectKeys.has(edge.toKey));
|
|
294
|
-
const queryRuns =
|
|
459
|
+
const queryRuns = runtimeCatalog.queryRunsForObjectKeys(objectKeys, 20);
|
|
295
460
|
const diagnostics = catalog.diagnostics();
|
|
296
461
|
const warnings = buildWarnings(diagnostics, objects);
|
|
297
462
|
const trustLabel = deriveTrust(objects);
|
|
@@ -320,6 +485,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
320
485
|
limit: request.limit ?? 120,
|
|
321
486
|
});
|
|
322
487
|
const conflicts = buildCandidateConflicts(reranked.ranked);
|
|
488
|
+
const meaningEvidence = buildMeaningEvidencePackage(request.question, questionPlan, reranked.ranked);
|
|
323
489
|
const compileConflicts = catalog.compileConflicts();
|
|
324
490
|
const routeDecision = withMetadataTrustLabelInfo(await planContextPackRoute({
|
|
325
491
|
request,
|
|
@@ -346,6 +512,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
346
512
|
questionScope,
|
|
347
513
|
limit: 6,
|
|
348
514
|
}).catch(() => ({ applied: [], conflicts: [] }));
|
|
515
|
+
const knowledgeLens = buildKnowledgeLens(catalog, effectiveDomainContext, selectedSkills);
|
|
349
516
|
const payload = {
|
|
350
517
|
id: '',
|
|
351
518
|
question: request.question,
|
|
@@ -357,6 +524,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
357
524
|
trustLabelInfo: metadataTrustLabelInfo(trustLabel),
|
|
358
525
|
objects,
|
|
359
526
|
skills: selectedSkills.map((item) => item.skill),
|
|
527
|
+
knowledgeLens,
|
|
360
528
|
edges: contextEdges,
|
|
361
529
|
queryRuns,
|
|
362
530
|
citations,
|
|
@@ -395,22 +563,49 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
395
563
|
topRejected: reranked.rejected,
|
|
396
564
|
certifiedCandidateFits,
|
|
397
565
|
candidateConflicts: conflicts,
|
|
566
|
+
meaningEvidence,
|
|
398
567
|
},
|
|
399
568
|
freshness: {
|
|
400
|
-
catalogPath:
|
|
569
|
+
catalogPath: snapshotPath,
|
|
401
570
|
builtAt: catalog.state('built_at'),
|
|
402
571
|
fingerprint: catalog.state('fingerprint'),
|
|
403
572
|
},
|
|
404
573
|
};
|
|
405
574
|
const packPayload = { ...payload };
|
|
406
575
|
delete packPayload.id;
|
|
407
|
-
const id =
|
|
576
|
+
const id = runtimeCatalog.insertContextPack(packPayload);
|
|
408
577
|
return { ...payload, id };
|
|
409
578
|
}
|
|
410
579
|
finally {
|
|
411
580
|
catalog.close();
|
|
581
|
+
runtimeCatalog.close();
|
|
412
582
|
}
|
|
413
583
|
}
|
|
584
|
+
function buildKnowledgeLens(catalog, context, selectedSkills) {
|
|
585
|
+
const activeDomainId = context?.activeDomain ?? undefined;
|
|
586
|
+
const capsule = activeDomainId
|
|
587
|
+
? catalog.listAllObjects({ objectTypes: ['domain_capsule'] })
|
|
588
|
+
.find((object) => object.domain === activeDomainId && !stringValue(object.payload?.modelAreaId))
|
|
589
|
+
: undefined;
|
|
590
|
+
const skillRefs = selectedSkills
|
|
591
|
+
.map(({ skill }) => skill.qualifiedId ?? skill.id)
|
|
592
|
+
.sort();
|
|
593
|
+
const skillFingerprints = Object.fromEntries(selectedSkills.flatMap(({ object, skill }) => {
|
|
594
|
+
const fingerprint = stringValue(object.payload?.sourceFingerprint);
|
|
595
|
+
return fingerprint ? [[skill.qualifiedId ?? skill.id, fingerprint]] : [];
|
|
596
|
+
}));
|
|
597
|
+
return {
|
|
598
|
+
mode: context?.source === 'explicit_api' || context?.source === 'explicit_ui' ? 'pinned' : 'auto',
|
|
599
|
+
activeDomainId,
|
|
600
|
+
modelAreaId: context?.modelAreaId,
|
|
601
|
+
purpose: context?.purpose,
|
|
602
|
+
skillRefs,
|
|
603
|
+
snapshotId: context?.snapshotId ?? catalog.state('fingerprint') ?? 'metadata-unavailable',
|
|
604
|
+
capsuleFingerprint: stringValue(capsule?.payload?.fingerprint)
|
|
605
|
+
?? stringValue(capsule?.payload?.sourceFingerprint),
|
|
606
|
+
skillFingerprints: Object.keys(skillFingerprints).length > 0 ? skillFingerprints : undefined,
|
|
607
|
+
};
|
|
608
|
+
}
|
|
414
609
|
export async function planAgentAnswer(projectRoot, request) {
|
|
415
610
|
const contextPack = await buildLocalContextPack(projectRoot, request);
|
|
416
611
|
return {
|
|
@@ -477,6 +672,7 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skil
|
|
|
477
672
|
}
|
|
478
673
|
}
|
|
479
674
|
addManifestBlockDetails(manifest, objects);
|
|
675
|
+
addManifestKnowledgeGraph(materializeIndexedKnowledgeGraph(projectRoot, manifest), objects, edges);
|
|
480
676
|
addSkillObjects(skills, objects, edges);
|
|
481
677
|
addDbtDagObjects(manifest, objects, edges, diagnostics);
|
|
482
678
|
addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges, diagnostics);
|
|
@@ -510,20 +706,285 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skil
|
|
|
510
706
|
edges: Array.from(edges.values()).sort((a, b) => `${a.edgeType}|${a.fromKey}|${a.toKey}`.localeCompare(`${b.edgeType}|${b.fromKey}|${b.toKey}`)),
|
|
511
707
|
diagnostics,
|
|
512
708
|
compileConflicts,
|
|
709
|
+
skillBodies: [...new Map(skills.map((skill) => {
|
|
710
|
+
const bodyHash = sha256(skill.body);
|
|
711
|
+
return [bodyHash, { bodyHash, body: skill.body }];
|
|
712
|
+
})).values()].sort((a, b) => a.bodyHash.localeCompare(b.bodyHash)),
|
|
513
713
|
generatedAt: new Date().toISOString(),
|
|
514
714
|
fingerprint: '',
|
|
515
715
|
};
|
|
516
716
|
snapshot.fingerprint = fingerprintSnapshot(snapshot);
|
|
517
717
|
return snapshot;
|
|
518
718
|
}
|
|
719
|
+
/**
|
|
720
|
+
* Manifest graph v2 intentionally omits verbose objects and edges. Snapshot
|
|
721
|
+
* construction rematerializes that compiler graph once, then persists the
|
|
722
|
+
* detail in immutable SQLite. Normal Ask/API reads never repeat this work.
|
|
723
|
+
*/
|
|
724
|
+
function materializeIndexedKnowledgeGraph(projectRoot, manifest) {
|
|
725
|
+
const compact = manifest.knowledgeGraph;
|
|
726
|
+
if (!compact || compact.storageMode !== 'indexed')
|
|
727
|
+
return manifest;
|
|
728
|
+
const { knowledgeGraph: _compactGraph, ...baseManifest } = manifest;
|
|
729
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
730
|
+
const skillCatalog = loadManifestKnowledgeSkills(projectRoot, registry);
|
|
731
|
+
const inline = buildManifestKnowledgeGraph({ manifest: baseManifest, skills: skillCatalog.skills });
|
|
732
|
+
const objects = { ...(inline.objects ?? {}) };
|
|
733
|
+
for (const ref of compact.objectRefs ?? []) {
|
|
734
|
+
objects[ref.id] ??= { ...ref };
|
|
735
|
+
}
|
|
736
|
+
return {
|
|
737
|
+
...manifest,
|
|
738
|
+
knowledgeGraph: {
|
|
739
|
+
...inline,
|
|
740
|
+
sourceFingerprint: compact.sourceFingerprint,
|
|
741
|
+
objects,
|
|
742
|
+
domainCapsules: compact.domainCapsules,
|
|
743
|
+
crossDomainRoutes: compact.crossDomainRoutes,
|
|
744
|
+
diagnostics: compact.diagnostics,
|
|
745
|
+
},
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
/**
|
|
749
|
+
* CTX-006: project search is a projection of the compiler-owned qualified
|
|
750
|
+
* policy graph. Legacy KG records are enriched in place so existing route keys
|
|
751
|
+
* remain compatible while identity, capsules, and governed route state come
|
|
752
|
+
* from one source.
|
|
753
|
+
*/
|
|
754
|
+
function addManifestKnowledgeGraph(manifest, objects, edges) {
|
|
755
|
+
const graph = manifest.knowledgeGraph;
|
|
756
|
+
if (!graph)
|
|
757
|
+
return;
|
|
758
|
+
const keyByGraphId = new Map();
|
|
759
|
+
const graphObjects = Object.values(graph.objects ?? {});
|
|
760
|
+
const indexedObjects = graphObjects.length > 0 ? graphObjects : (graph.objectRefs ?? []);
|
|
761
|
+
for (const item of indexedObjects) {
|
|
762
|
+
const itemPayload = ('payload' in item ? item.payload : undefined);
|
|
763
|
+
const itemAliases = ('aliases' in item ? item.aliases : undefined);
|
|
764
|
+
const displayName = knowledgeDisplayName(item.kind, item.localId, itemPayload);
|
|
765
|
+
const objectKey = knowledgeMetadataKey(item.kind, displayName, item.id, item.source.system);
|
|
766
|
+
keyByGraphId.set(item.id, objectKey);
|
|
767
|
+
const description = stringValue(itemPayload?.description)
|
|
768
|
+
?? stringValue(itemPayload?.businessContext)
|
|
769
|
+
?? itemAliases?.find((alias) => alias !== item.localId);
|
|
770
|
+
const compiled = {
|
|
771
|
+
objectKey,
|
|
772
|
+
objectType: knowledgeMetadataType(item.kind),
|
|
773
|
+
name: displayName,
|
|
774
|
+
fullName: item.id,
|
|
775
|
+
domain: item.domainId,
|
|
776
|
+
owner: item.owner,
|
|
777
|
+
status: item.status,
|
|
778
|
+
description,
|
|
779
|
+
sourcePath: item.source.path,
|
|
780
|
+
sourceSystem: `DQL canonical knowledge graph (${item.source.system})`,
|
|
781
|
+
payload: compactObject({
|
|
782
|
+
...(itemPayload ?? {}),
|
|
783
|
+
qualifiedId: item.id,
|
|
784
|
+
aliases: itemAliases ?? [],
|
|
785
|
+
modelAreaIds: item.modelAreaIds ?? [],
|
|
786
|
+
sourceFingerprint: item.source.fingerprint,
|
|
787
|
+
sourceNativeId: item.source.nativeId,
|
|
788
|
+
knowledgeGraphSchemaVersion: graph.schemaVersion,
|
|
789
|
+
}),
|
|
790
|
+
};
|
|
791
|
+
const existing = objects.get(objectKey);
|
|
792
|
+
objects.set(objectKey, existing ? mergeObject(existing, compiled) : compiled);
|
|
793
|
+
}
|
|
794
|
+
for (const capsule of Object.values(graph.domainCapsules)) {
|
|
795
|
+
const objectKey = `dql:domain_capsule:${capsule.id}`;
|
|
796
|
+
objects.set(objectKey, {
|
|
797
|
+
objectKey,
|
|
798
|
+
objectType: 'domain_capsule',
|
|
799
|
+
name: capsule.name,
|
|
800
|
+
fullName: capsule.id,
|
|
801
|
+
domain: capsule.domainId,
|
|
802
|
+
description: capsule.description,
|
|
803
|
+
sourceSystem: 'DQL compiled Domain Knowledge Capsule',
|
|
804
|
+
payload: compactObject({ ...capsule, sourceFingerprint: graph.sourceFingerprint }),
|
|
805
|
+
});
|
|
806
|
+
addMetadataEdge(edges, 'contains', keyByGraphId.get(`domain::${capsule.domainId}`) ?? `domain:${capsule.domainId}`, objectKey, { source: 'knowledge_graph' });
|
|
807
|
+
for (const skill of capsule.skillRefs)
|
|
808
|
+
addMetadataEdge(edges, 'guided_by', objectKey, `skill:${skill}`, { source: 'knowledge_graph' });
|
|
809
|
+
}
|
|
810
|
+
for (const route of graph.crossDomainRoutes) {
|
|
811
|
+
const objectKey = `dql:cross_domain_route:${route.id}`;
|
|
812
|
+
objects.set(objectKey, {
|
|
813
|
+
objectKey,
|
|
814
|
+
objectType: 'cross_domain_route',
|
|
815
|
+
name: `${route.providerDomainId} → ${route.consumerDomainId}`,
|
|
816
|
+
fullName: route.id,
|
|
817
|
+
domain: route.consumerDomainId,
|
|
818
|
+
status: route.state,
|
|
819
|
+
description: route.purpose ? `Approved purpose: ${route.purpose}` : 'Observed cross-domain dependency',
|
|
820
|
+
sourceSystem: 'DQL compiled cross-domain policy',
|
|
821
|
+
payload: compactObject(route),
|
|
822
|
+
});
|
|
823
|
+
addMetadataEdge(edges, 'cross_domain_route', keyByGraphId.get(`domain::${route.providerDomainId}`) ?? `domain:${route.providerDomainId}`, objectKey, { state: route.state, reasonCodes: route.reasonCodes });
|
|
824
|
+
addMetadataEdge(edges, 'cross_domain_route', objectKey, keyByGraphId.get(`domain::${route.consumerDomainId}`) ?? `domain:${route.consumerDomainId}`, { state: route.state, reasonCodes: route.reasonCodes });
|
|
825
|
+
}
|
|
826
|
+
for (const edge of graph.edges ?? []) {
|
|
827
|
+
const fromKey = keyByGraphId.get(edge.from) ?? edge.from;
|
|
828
|
+
const toKey = keyByGraphId.get(edge.to) ?? edge.to;
|
|
829
|
+
addMetadataEdge(edges, edge.kind, fromKey, toKey, {
|
|
830
|
+
state: edge.state,
|
|
831
|
+
domainPair: edge.domainPair,
|
|
832
|
+
evidenceRefs: edge.evidenceRefs ?? [],
|
|
833
|
+
reasonCodes: edge.reasonCodes ?? [],
|
|
834
|
+
fingerprint: edge.fingerprint,
|
|
835
|
+
source: 'knowledge_graph',
|
|
836
|
+
});
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
function addMetadataEdge(edges, edgeType, fromKey, toKey, payload) {
|
|
840
|
+
const key = `${edgeType}\u0000${fromKey}\u0000${toKey}`;
|
|
841
|
+
if (!edges.has(key))
|
|
842
|
+
edges.set(key, { edgeType, fromKey, toKey, confidence: 1, payload: compactObject(payload) });
|
|
843
|
+
}
|
|
844
|
+
function knowledgeMetadataKey(kind, localId, qualifiedId, sourceSystem) {
|
|
845
|
+
switch (kind) {
|
|
846
|
+
case 'block': return `dql:block:${localId}`;
|
|
847
|
+
case 'term': return `dql:term:${localId}`;
|
|
848
|
+
case 'business_view': return `dql:business_view:${localId}`;
|
|
849
|
+
case 'metric': return `semantic:metric:${localId}`;
|
|
850
|
+
case 'dimension': return `semantic:dimension:${localId}`;
|
|
851
|
+
case 'entity': return `dql:entity:${qualifiedId}`;
|
|
852
|
+
case 'model_area': return `dql:model_area:${qualifiedId}`;
|
|
853
|
+
case 'relationship': return `dql:relationship:${qualifiedId}`;
|
|
854
|
+
case 'contract': return `dql:contract:${qualifiedId}`;
|
|
855
|
+
case 'domain_export': return `dql:domain_export:${qualifiedId}`;
|
|
856
|
+
case 'domain_import': return `dql:domain_import:${qualifiedId}`;
|
|
857
|
+
case 'conformance': return `dql:conformance:${qualifiedId}`;
|
|
858
|
+
case 'policy': return `dql:policy:${qualifiedId}`;
|
|
859
|
+
case 'evaluation': return `dql:evaluation:${qualifiedId}`;
|
|
860
|
+
case 'skill': return `skill:${qualifiedId}`;
|
|
861
|
+
case 'domain': return `domain:${localId}`;
|
|
862
|
+
case 'dbt_model': return `dbt:model:${localId}`;
|
|
863
|
+
case 'dbt_source': return `dbt:source:${localId}`;
|
|
864
|
+
case 'source_table': return `warehouse:table:${localId}`;
|
|
865
|
+
case 'notebook': return `notebook:${localId}`;
|
|
866
|
+
case 'dashboard': return `dashboard:${localId}`;
|
|
867
|
+
case 'app': return `app:${localId}`;
|
|
868
|
+
default: return `${sourceSystem}:${kind}:${qualifiedId}`;
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
function knowledgeDisplayName(kind, localId, payload) {
|
|
872
|
+
if (kind === 'domain' || kind === 'app')
|
|
873
|
+
return stringValue(payload?.name) ?? localId;
|
|
874
|
+
if (kind === 'dashboard' || kind === 'notebook')
|
|
875
|
+
return stringValue(payload?.title) ?? localId;
|
|
876
|
+
return localId;
|
|
877
|
+
}
|
|
878
|
+
function knowledgeMetadataType(kind) {
|
|
879
|
+
switch (kind) {
|
|
880
|
+
case 'block': return 'dql_block';
|
|
881
|
+
case 'term': return 'dql_term';
|
|
882
|
+
case 'metric': return 'semantic_metric';
|
|
883
|
+
case 'dimension': return 'semantic_dimension';
|
|
884
|
+
case 'entity': return 'dql_entity';
|
|
885
|
+
case 'model_area': return 'model_area';
|
|
886
|
+
case 'dbt_model': return 'dbt_model';
|
|
887
|
+
case 'dbt_source': return 'dbt_source';
|
|
888
|
+
case 'source_table': return 'warehouse_table';
|
|
889
|
+
default: return kind;
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
function metadataKnowledgeKind(objectType) {
|
|
893
|
+
switch (objectType) {
|
|
894
|
+
case 'dql_block': return 'block';
|
|
895
|
+
case 'dql_term': return 'term';
|
|
896
|
+
case 'business_view': return 'business_view';
|
|
897
|
+
case 'semantic_metric': return 'metric';
|
|
898
|
+
case 'semantic_dimension': return 'dimension';
|
|
899
|
+
case 'semantic_model': return 'semantic_model';
|
|
900
|
+
case 'dql_entity': return 'entity';
|
|
901
|
+
case 'model_area': return 'model_area';
|
|
902
|
+
case 'dbt_model': return 'dbt_model';
|
|
903
|
+
case 'dbt_source': return 'dbt_source';
|
|
904
|
+
case 'warehouse_table': return 'source_table';
|
|
905
|
+
case 'domain_capsule': return 'domain';
|
|
906
|
+
default: return objectType;
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
function metadataObjectToKnowledge(item) {
|
|
910
|
+
const payload = item.payload ?? {};
|
|
911
|
+
const id = stringValue(payload.qualifiedId) ?? item.fullName ?? item.objectKey;
|
|
912
|
+
const system = item.sourceSystem?.toLowerCase().includes('dbt')
|
|
913
|
+
? 'dbt'
|
|
914
|
+
: item.sourceSystem?.toLowerCase().includes('semantic')
|
|
915
|
+
? 'semantic'
|
|
916
|
+
: 'dql';
|
|
917
|
+
return {
|
|
918
|
+
id,
|
|
919
|
+
kind: metadataKnowledgeKind(item.objectType),
|
|
920
|
+
localId: item.name,
|
|
921
|
+
domainId: item.domain,
|
|
922
|
+
modelAreaIds: metadataStringArray(payload.modelAreaIds),
|
|
923
|
+
aliases: metadataStringArray(payload.aliases),
|
|
924
|
+
status: item.status,
|
|
925
|
+
owner: item.owner,
|
|
926
|
+
source: {
|
|
927
|
+
system,
|
|
928
|
+
path: item.sourcePath,
|
|
929
|
+
nativeId: stringValue(payload.sourceNativeId),
|
|
930
|
+
fingerprint: stringValue(payload.sourceFingerprint) ?? sha256(stableStringify({ id, sourcePath: item.sourcePath, payload })),
|
|
931
|
+
},
|
|
932
|
+
payload,
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
function metadataEdgeToKnowledge(edge, byKey) {
|
|
936
|
+
const payload = edge.payload ?? {};
|
|
937
|
+
const from = byKey.get(edge.fromKey)?.id ?? edge.fromKey;
|
|
938
|
+
const to = byKey.get(edge.toKey)?.id ?? edge.toKey;
|
|
939
|
+
const fingerprint = stringValue(payload.fingerprint) ?? sha256(stableStringify({ kind: edge.edgeType, from, to, payload }));
|
|
940
|
+
return {
|
|
941
|
+
id: `edge::${fingerprint.slice(0, 20)}`,
|
|
942
|
+
kind: edge.edgeType,
|
|
943
|
+
from,
|
|
944
|
+
to,
|
|
945
|
+
state: payload.state,
|
|
946
|
+
domainPair: payload.domainPair,
|
|
947
|
+
evidenceRefs: metadataStringArray(payload.evidenceRefs),
|
|
948
|
+
reasonCodes: metadataStringArray(payload.reasonCodes),
|
|
949
|
+
fingerprint,
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
function metadataRoute(item) {
|
|
953
|
+
const payload = item.payload ?? {};
|
|
954
|
+
const providerDomainId = stringValue(payload.providerDomainId);
|
|
955
|
+
const consumerDomainId = stringValue(payload.consumerDomainId);
|
|
956
|
+
const relationshipId = stringValue(payload.relationshipId);
|
|
957
|
+
const state = stringValue(payload.state);
|
|
958
|
+
if (!providerDomainId || !consumerDomainId || !relationshipId || !state || !['observed', 'authorized', 'blocked', 'stale'].includes(state))
|
|
959
|
+
return null;
|
|
960
|
+
return {
|
|
961
|
+
id: stringValue(payload.id) ?? item.fullName ?? item.objectKey,
|
|
962
|
+
providerDomainId,
|
|
963
|
+
consumerDomainId,
|
|
964
|
+
purpose: stringValue(payload.purpose) ?? '',
|
|
965
|
+
relationshipId,
|
|
966
|
+
exportId: stringValue(payload.exportId),
|
|
967
|
+
importId: stringValue(payload.importId),
|
|
968
|
+
contractId: stringValue(payload.contractId),
|
|
969
|
+
state: state,
|
|
970
|
+
reasonCodes: metadataStringArray(payload.reasonCodes),
|
|
971
|
+
path: metadataStringArray(payload.path),
|
|
972
|
+
fingerprint: stringValue(payload.fingerprint) ?? sha256(stableStringify(payload)),
|
|
973
|
+
};
|
|
974
|
+
}
|
|
519
975
|
export class MetadataCatalog {
|
|
520
976
|
dbPath;
|
|
521
977
|
db;
|
|
522
|
-
constructor(dbPath) {
|
|
978
|
+
constructor(dbPath, options = {}) {
|
|
523
979
|
this.dbPath = dbPath;
|
|
524
|
-
|
|
980
|
+
if (!options.readOnly)
|
|
981
|
+
mkdirSync(dirname(dbPath), { recursive: true });
|
|
525
982
|
const Database = loadDatabase();
|
|
526
|
-
this.db = new Database(dbPath);
|
|
983
|
+
this.db = new Database(dbPath, options.readOnly ? { readonly: true, fileMustExist: true } : undefined);
|
|
984
|
+
if (options.readOnly) {
|
|
985
|
+
this.db.pragma('query_only = ON');
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
527
988
|
this.db.pragma('journal_mode = WAL');
|
|
528
989
|
this.db.pragma('foreign_keys = ON');
|
|
529
990
|
this.initSchema();
|
|
@@ -605,6 +1066,31 @@ export class MetadataCatalog {
|
|
|
605
1066
|
);
|
|
606
1067
|
CREATE INDEX IF NOT EXISTS idx_runtime_schema_snapshots_captured ON runtime_schema_snapshots(captured_at DESC);
|
|
607
1068
|
|
|
1069
|
+
CREATE TABLE IF NOT EXISTS runtime_schema_objects (
|
|
1070
|
+
object_key TEXT PRIMARY KEY,
|
|
1071
|
+
object_type TEXT NOT NULL,
|
|
1072
|
+
name TEXT NOT NULL,
|
|
1073
|
+
full_name TEXT,
|
|
1074
|
+
domain TEXT,
|
|
1075
|
+
owner TEXT,
|
|
1076
|
+
status TEXT,
|
|
1077
|
+
description TEXT,
|
|
1078
|
+
source_path TEXT,
|
|
1079
|
+
source_system TEXT,
|
|
1080
|
+
payload_json TEXT NOT NULL DEFAULT '{}',
|
|
1081
|
+
updated_at TEXT NOT NULL
|
|
1082
|
+
);
|
|
1083
|
+
CREATE INDEX IF NOT EXISTS idx_runtime_schema_objects_type ON runtime_schema_objects(object_type);
|
|
1084
|
+
|
|
1085
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS runtime_schema_fts USING fts5(
|
|
1086
|
+
object_key UNINDEXED,
|
|
1087
|
+
name,
|
|
1088
|
+
full_name,
|
|
1089
|
+
description,
|
|
1090
|
+
payload,
|
|
1091
|
+
tokenize = 'porter unicode61'
|
|
1092
|
+
);
|
|
1093
|
+
|
|
608
1094
|
CREATE TABLE IF NOT EXISTS runtime_value_index (
|
|
609
1095
|
value_key TEXT PRIMARY KEY,
|
|
610
1096
|
relation TEXT NOT NULL,
|
|
@@ -661,7 +1147,23 @@ export class MetadataCatalog {
|
|
|
661
1147
|
dbt_object_count INTEGER NOT NULL,
|
|
662
1148
|
updated_at TEXT NOT NULL
|
|
663
1149
|
);
|
|
1150
|
+
|
|
1151
|
+
CREATE TABLE IF NOT EXISTS skill_bodies (
|
|
1152
|
+
body_hash TEXT PRIMARY KEY,
|
|
1153
|
+
encoding TEXT NOT NULL,
|
|
1154
|
+
compressed_body TEXT NOT NULL,
|
|
1155
|
+
original_length INTEGER NOT NULL
|
|
1156
|
+
);
|
|
664
1157
|
`);
|
|
1158
|
+
// Runtime values may be used transiently inside one governed Ask, but they
|
|
1159
|
+
// must never survive in the rebuildable metadata database. Clear legacy v2
|
|
1160
|
+
// rows on open so an upgrade removes old plaintext without a migration.
|
|
1161
|
+
this.db.prepare('DELETE FROM runtime_value_fts').run();
|
|
1162
|
+
this.db.prepare('DELETE FROM runtime_value_index').run();
|
|
1163
|
+
this.db.prepare(`
|
|
1164
|
+
INSERT OR REPLACE INTO metadata_state (key, value)
|
|
1165
|
+
VALUES ('runtime_value_index_count', '0')
|
|
1166
|
+
`).run();
|
|
665
1167
|
}
|
|
666
1168
|
rebuild(snapshot) {
|
|
667
1169
|
const now = new Date().toISOString();
|
|
@@ -698,6 +1200,10 @@ export class MetadataCatalog {
|
|
|
698
1200
|
domain, object_count, block_count, certified_block_count,
|
|
699
1201
|
semantic_metric_count, dbt_object_count, updated_at
|
|
700
1202
|
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1203
|
+
`);
|
|
1204
|
+
const insertSkillBody = this.db.prepare(`
|
|
1205
|
+
INSERT OR REPLACE INTO skill_bodies (body_hash, encoding, compressed_body, original_length)
|
|
1206
|
+
VALUES (?, 'br', ?, ?)
|
|
701
1207
|
`);
|
|
702
1208
|
const sourceFingerprints = buildSourceFingerprints(snapshot.objects, now);
|
|
703
1209
|
const domainShards = buildDomainShards(snapshot.objects, now);
|
|
@@ -708,10 +1214,11 @@ export class MetadataCatalog {
|
|
|
708
1214
|
this.db.prepare('DELETE FROM metadata_diagnostics').run();
|
|
709
1215
|
this.db.prepare('DELETE FROM metadata_source_fingerprints').run();
|
|
710
1216
|
this.db.prepare('DELETE FROM metadata_domain_shards').run();
|
|
1217
|
+
this.db.prepare('DELETE FROM skill_bodies').run();
|
|
711
1218
|
for (const object of snapshot.objects) {
|
|
712
1219
|
const payload = object.payload ?? {};
|
|
713
1220
|
insertObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, object.domain ?? null, object.owner ?? null, object.status ?? null, object.description ?? null, object.sourcePath ?? null, object.sourceSystem ?? null, JSON.stringify(payload), object.updatedAt ?? now);
|
|
714
|
-
insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '',
|
|
1221
|
+
insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', searchableMetadataPayload(payload));
|
|
715
1222
|
}
|
|
716
1223
|
for (const edge of snapshot.edges) {
|
|
717
1224
|
insertEdge.run(edge.edgeType, edge.fromKey, edge.toKey, edge.confidence ?? 1, JSON.stringify(edge.payload ?? {}), now);
|
|
@@ -725,6 +1232,9 @@ export class MetadataCatalog {
|
|
|
725
1232
|
for (const item of domainShards) {
|
|
726
1233
|
insertDomainShard.run(item.domain, item.objectCount, item.blockCount, item.certifiedBlockCount, item.semanticMetricCount, item.dbtObjectCount, item.updatedAt);
|
|
727
1234
|
}
|
|
1235
|
+
for (const item of snapshot.skillBodies ?? []) {
|
|
1236
|
+
insertSkillBody.run(item.bodyHash, brotliCompressSync(item.body, { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 } }).toString('base64'), Buffer.byteLength(item.body, 'utf8'));
|
|
1237
|
+
}
|
|
728
1238
|
setState.run('built_at', now);
|
|
729
1239
|
setState.run('fingerprint', snapshot.fingerprint);
|
|
730
1240
|
setState.run('project_root', snapshot.projectRoot);
|
|
@@ -735,6 +1245,7 @@ export class MetadataCatalog {
|
|
|
735
1245
|
setState.run('diagnostics_json', JSON.stringify(snapshot.diagnostics));
|
|
736
1246
|
setState.run('compile_conflicts_json', JSON.stringify(snapshot.compileConflicts ?? []));
|
|
737
1247
|
setState.run('manifest_generated_at', snapshot.manifest.generatedAt);
|
|
1248
|
+
setState.run('index_version', METADATA_INDEX_VERSION);
|
|
738
1249
|
});
|
|
739
1250
|
txn();
|
|
740
1251
|
}
|
|
@@ -791,6 +1302,9 @@ export class MetadataCatalog {
|
|
|
791
1302
|
semantic_metric_count, dbt_object_count, updated_at
|
|
792
1303
|
) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
793
1304
|
const setState = this.db.prepare('INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)');
|
|
1305
|
+
const insertSkillBody = this.db.prepare(`
|
|
1306
|
+
INSERT OR REPLACE INTO skill_bodies (body_hash, encoding, compressed_body, original_length)
|
|
1307
|
+
VALUES (?, 'br', ?, ?)`);
|
|
794
1308
|
const deleteFtsForSource = this.db.prepare('DELETE FROM metadata_fts WHERE object_key IN (SELECT object_key FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?)');
|
|
795
1309
|
const deleteObjectsForSource = this.db.prepare('DELETE FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?');
|
|
796
1310
|
const txn = this.db.transaction(() => {
|
|
@@ -810,7 +1324,7 @@ export class MetadataCatalog {
|
|
|
810
1324
|
continue;
|
|
811
1325
|
const payload = object.payload ?? {};
|
|
812
1326
|
insertObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, object.domain ?? null, object.owner ?? null, object.status ?? null, object.description ?? null, object.sourcePath ?? null, object.sourceSystem ?? null, JSON.stringify(payload), object.updatedAt ?? now);
|
|
813
|
-
insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '',
|
|
1327
|
+
insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', searchableMetadataPayload(payload));
|
|
814
1328
|
}
|
|
815
1329
|
// Cross-source tables: rebuild fully.
|
|
816
1330
|
this.db.prepare('DELETE FROM metadata_edges').run();
|
|
@@ -825,6 +1339,10 @@ export class MetadataCatalog {
|
|
|
825
1339
|
for (const shard of domainShards) {
|
|
826
1340
|
insertDomainShard.run(shard.domain, shard.objectCount, shard.blockCount, shard.certifiedBlockCount, shard.semanticMetricCount, shard.dbtObjectCount, shard.updatedAt);
|
|
827
1341
|
}
|
|
1342
|
+
this.db.prepare('DELETE FROM skill_bodies').run();
|
|
1343
|
+
for (const item of snapshot.skillBodies ?? []) {
|
|
1344
|
+
insertSkillBody.run(item.bodyHash, brotliCompressSync(item.body, { params: { [zlibConstants.BROTLI_PARAM_QUALITY]: 4 } }).toString('base64'), Buffer.byteLength(item.body, 'utf8'));
|
|
1345
|
+
}
|
|
828
1346
|
for (const source of removedSources) {
|
|
829
1347
|
this.db.prepare('DELETE FROM metadata_source_fingerprints WHERE source_path = ?').run(source);
|
|
830
1348
|
}
|
|
@@ -841,6 +1359,7 @@ export class MetadataCatalog {
|
|
|
841
1359
|
setState.run('diagnostics_json', JSON.stringify(snapshot.diagnostics));
|
|
842
1360
|
setState.run('compile_conflicts_json', JSON.stringify(snapshot.compileConflicts ?? []));
|
|
843
1361
|
setState.run('manifest_generated_at', snapshot.manifest.generatedAt);
|
|
1362
|
+
setState.run('index_version', METADATA_INDEX_VERSION);
|
|
844
1363
|
});
|
|
845
1364
|
txn();
|
|
846
1365
|
return { mode: 'incremental', changedSources: changedSources.size };
|
|
@@ -879,9 +1398,16 @@ export class MetadataCatalog {
|
|
|
879
1398
|
const seen = new Set(andRows.map((row) => row.object_key));
|
|
880
1399
|
const orRows = runMatch(match.or).filter((row) => !seen.has(row.object_key));
|
|
881
1400
|
const rows = [...andRows, ...orRows].slice(0, limit);
|
|
1401
|
+
const andKeys = new Set(andRows.map((row) => row.object_key));
|
|
1402
|
+
const maxAndMagnitude = strongestBm25Magnitude(andRows);
|
|
1403
|
+
const maxOrMagnitude = strongestBm25Magnitude(orRows);
|
|
882
1404
|
return rows.map((row) => ({
|
|
883
1405
|
...rowToObject(row),
|
|
884
|
-
|
|
1406
|
+
// FTS5 bm25() is negative and more-negative means stronger. The old
|
|
1407
|
+
// max(0, rank) conversion flattened virtually every hit to 1. Preserve
|
|
1408
|
+
// within-tier separation and keep all-terms (AND) evidence ahead of a
|
|
1409
|
+
// partial OR-only match without pretending BM25 is an absolute confidence.
|
|
1410
|
+
score: normalizedBm25Score(row.rank, andKeys.has(row.object_key) ? 'and' : 'or', andKeys.has(row.object_key) ? maxAndMagnitude : maxOrMagnitude),
|
|
885
1411
|
snippet: row.snip ?? undefined,
|
|
886
1412
|
}));
|
|
887
1413
|
}
|
|
@@ -905,6 +1431,36 @@ export class MetadataCatalog {
|
|
|
905
1431
|
`).all(...params, options.limit ?? 100);
|
|
906
1432
|
return rows.map(rowToObject);
|
|
907
1433
|
}
|
|
1434
|
+
/** Complete typed inventory; callers must apply their own bounded ranking. */
|
|
1435
|
+
listAllObjects(options = {}) {
|
|
1436
|
+
const filters = [];
|
|
1437
|
+
const params = [];
|
|
1438
|
+
if (options.objectTypes && options.objectTypes.length > 0) {
|
|
1439
|
+
filters.push(`object_type IN (${options.objectTypes.map(() => '?').join(', ')})`);
|
|
1440
|
+
params.push(...options.objectTypes);
|
|
1441
|
+
}
|
|
1442
|
+
if (options.domain) {
|
|
1443
|
+
filters.push('domain = ?');
|
|
1444
|
+
params.push(options.domain);
|
|
1445
|
+
}
|
|
1446
|
+
const where = filters.length > 0 ? `WHERE ${filters.join(' AND ')}` : '';
|
|
1447
|
+
const rows = this.db.prepare(`
|
|
1448
|
+
SELECT * FROM metadata_objects
|
|
1449
|
+
${where}
|
|
1450
|
+
ORDER BY object_key
|
|
1451
|
+
`).all(...params);
|
|
1452
|
+
return rows.map(rowToObject);
|
|
1453
|
+
}
|
|
1454
|
+
crossDomainRouteObjects(domainId, limit = 100) {
|
|
1455
|
+
const rows = this.db.prepare(`
|
|
1456
|
+
SELECT * FROM metadata_objects
|
|
1457
|
+
WHERE object_type = 'cross_domain_route'
|
|
1458
|
+
AND (domain = ? OR json_extract(payload_json, '$.providerDomainId') = ?)
|
|
1459
|
+
ORDER BY status, name, object_key
|
|
1460
|
+
LIMIT ?
|
|
1461
|
+
`).all(domainId, domainId, Math.max(1, Math.min(limit, 500)));
|
|
1462
|
+
return rows.map(rowToObject);
|
|
1463
|
+
}
|
|
908
1464
|
scanObjects(options, visit) {
|
|
909
1465
|
const filters = [];
|
|
910
1466
|
const params = [];
|
|
@@ -942,7 +1498,20 @@ export class MetadataCatalog {
|
|
|
942
1498
|
}
|
|
943
1499
|
getObject(objectKey) {
|
|
944
1500
|
const row = this.db.prepare('SELECT * FROM metadata_objects WHERE object_key = ?').get(objectKey);
|
|
945
|
-
|
|
1501
|
+
if (row)
|
|
1502
|
+
return rowToObject(row);
|
|
1503
|
+
const legacy = legacyQualifiedAlias(objectKey);
|
|
1504
|
+
if (!legacy)
|
|
1505
|
+
return null;
|
|
1506
|
+
const matches = this.db.prepare(`
|
|
1507
|
+
SELECT * FROM metadata_objects
|
|
1508
|
+
WHERE object_type = ? AND name = ?
|
|
1509
|
+
ORDER BY object_key
|
|
1510
|
+
LIMIT 2
|
|
1511
|
+
`).all(legacy.objectType, legacy.name);
|
|
1512
|
+
// A legacy local-name lookup is safe only while it resolves uniquely. When
|
|
1513
|
+
// two domains define the same local ID, callers must use the qualified key.
|
|
1514
|
+
return matches.length === 1 ? rowToObject(matches[0]) : null;
|
|
946
1515
|
}
|
|
947
1516
|
getObjectsByKeys(keys) {
|
|
948
1517
|
const unique = Array.from(new Set(keys.filter(Boolean)));
|
|
@@ -958,21 +1527,45 @@ export class MetadataCatalog {
|
|
|
958
1527
|
`).all(...chunk);
|
|
959
1528
|
rows.push(...fetched.map(rowToObject));
|
|
960
1529
|
}
|
|
1530
|
+
const found = new Set(rows.map((row) => row.objectKey));
|
|
1531
|
+
for (const key of unique) {
|
|
1532
|
+
if (found.has(key))
|
|
1533
|
+
continue;
|
|
1534
|
+
const resolved = this.getObject(key);
|
|
1535
|
+
if (resolved && !rows.some((row) => row.objectKey === resolved.objectKey))
|
|
1536
|
+
rows.push(resolved);
|
|
1537
|
+
}
|
|
961
1538
|
return rows;
|
|
962
1539
|
}
|
|
1540
|
+
findObjectByIdentity(identity) {
|
|
1541
|
+
const rows = this.db.prepare(`
|
|
1542
|
+
SELECT * FROM metadata_objects
|
|
1543
|
+
WHERE object_key = ? OR full_name = ?
|
|
1544
|
+
ORDER BY CASE WHEN object_key = ? THEN 0 ELSE 1 END, object_key
|
|
1545
|
+
LIMIT 2
|
|
1546
|
+
`).all(identity, identity, identity);
|
|
1547
|
+
return rows.length === 1 ? rowToObject(rows[0]) : rows[0] ? rowToObject(rows[0]) : null;
|
|
1548
|
+
}
|
|
963
1549
|
edgesForKeys(keys, hops = 1) {
|
|
964
1550
|
let frontier = new Set(keys.filter(Boolean));
|
|
965
1551
|
const seenKeys = new Set(frontier);
|
|
966
1552
|
const edges = new Map();
|
|
967
1553
|
for (let hop = 0; hop < hops && frontier.size > 0; hop += 1) {
|
|
968
|
-
const current = Array.from(frontier).
|
|
1554
|
+
const current = Array.from(frontier).sort();
|
|
969
1555
|
frontier = new Set();
|
|
970
|
-
const rows =
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
1556
|
+
const rows = [];
|
|
1557
|
+
for (let offset = 0; offset < current.length; offset += 100) {
|
|
1558
|
+
const chunk = current.slice(offset, offset + 100);
|
|
1559
|
+
rows.push(...this.db.prepare(`
|
|
1560
|
+
SELECT * FROM metadata_edges
|
|
1561
|
+
WHERE from_key IN (${chunk.map(() => '?').join(', ')})
|
|
1562
|
+
OR to_key IN (${chunk.map(() => '?').join(', ')})
|
|
1563
|
+
ORDER BY confidence DESC, edge_type, from_key, to_key
|
|
1564
|
+
LIMIT 500
|
|
1565
|
+
`).all(...chunk, ...chunk));
|
|
1566
|
+
}
|
|
1567
|
+
rows.sort((left, right) => right.confidence - left.confidence
|
|
1568
|
+
|| `${left.edge_type}|${left.from_key}|${left.to_key}`.localeCompare(`${right.edge_type}|${right.from_key}|${right.to_key}`));
|
|
976
1569
|
for (const row of rows) {
|
|
977
1570
|
const edge = rowToEdge(row);
|
|
978
1571
|
const edgeKey = `${edge.edgeType}\u0000${edge.fromKey}\u0000${edge.toKey}`;
|
|
@@ -1022,26 +1615,26 @@ export class MetadataCatalog {
|
|
|
1022
1615
|
}
|
|
1023
1616
|
recordRuntimeSchemaSnapshot(snapshot) {
|
|
1024
1617
|
const capturedAt = snapshot.capturedAt ?? new Date().toISOString();
|
|
1618
|
+
const normalizedTables = normalizeRuntimeSchemaTables(snapshot.tables).slice(0, 10_000);
|
|
1025
1619
|
const cleanSnapshot = {
|
|
1026
1620
|
source: snapshot.source,
|
|
1027
1621
|
capturedAt,
|
|
1028
|
-
tables:
|
|
1622
|
+
tables: normalizedTables,
|
|
1029
1623
|
};
|
|
1030
1624
|
const id = `schema_${Date.parse(capturedAt) || Date.now()}`;
|
|
1031
|
-
const valueEntries = buildRuntimeValueIndex(cleanSnapshot);
|
|
1032
1625
|
const insertSnapshot = this.db.prepare(`
|
|
1033
1626
|
INSERT OR REPLACE INTO runtime_schema_snapshots (
|
|
1034
1627
|
id, source, payload_json, captured_at
|
|
1035
1628
|
) VALUES (?, ?, ?, ?)
|
|
1036
1629
|
`);
|
|
1037
|
-
const
|
|
1038
|
-
INSERT
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1630
|
+
const insertSchemaObject = this.db.prepare(`
|
|
1631
|
+
INSERT INTO runtime_schema_objects (
|
|
1632
|
+
object_key, object_type, name, full_name, domain, owner, status,
|
|
1633
|
+
description, source_path, source_system, payload_json, updated_at
|
|
1634
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1042
1635
|
`);
|
|
1043
|
-
const
|
|
1044
|
-
INSERT INTO
|
|
1636
|
+
const insertSchemaFts = this.db.prepare(`
|
|
1637
|
+
INSERT INTO runtime_schema_fts (object_key, name, full_name, description, payload)
|
|
1045
1638
|
VALUES (?, ?, ?, ?, ?)
|
|
1046
1639
|
`);
|
|
1047
1640
|
const setState = this.db.prepare(`
|
|
@@ -1059,34 +1652,47 @@ export class MetadataCatalog {
|
|
|
1059
1652
|
`).run();
|
|
1060
1653
|
this.db.prepare('DELETE FROM runtime_value_fts').run();
|
|
1061
1654
|
this.db.prepare('DELETE FROM runtime_value_index').run();
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1655
|
+
this.db.prepare('DELETE FROM runtime_schema_fts').run();
|
|
1656
|
+
this.db.prepare('DELETE FROM runtime_schema_objects').run();
|
|
1657
|
+
for (const table of normalizedTables) {
|
|
1658
|
+
const object = runtimeSchemaTableObject(table, cleanSnapshot.source, capturedAt);
|
|
1659
|
+
const payload = object.payload ?? {};
|
|
1660
|
+
insertSchemaObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, null, null, object.status ?? null, object.description ?? null, null, object.sourceSystem ?? null, JSON.stringify(payload), capturedAt);
|
|
1661
|
+
insertSchemaFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', searchableMetadataPayload(payload));
|
|
1065
1662
|
}
|
|
1066
|
-
setState.run('runtime_value_index_count',
|
|
1663
|
+
setState.run('runtime_value_index_count', '0');
|
|
1067
1664
|
setState.run('runtime_value_index_captured_at', capturedAt);
|
|
1665
|
+
setState.run('runtime_schema_table_count', String(normalizedTables.length));
|
|
1666
|
+
setState.run('runtime_schema_index_captured_at', capturedAt);
|
|
1068
1667
|
});
|
|
1069
1668
|
txn();
|
|
1070
1669
|
return cleanSnapshot;
|
|
1071
1670
|
}
|
|
1072
1671
|
searchRuntimeValues(terms, limit = 40) {
|
|
1073
|
-
|
|
1074
|
-
//
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1672
|
+
// Deliberately non-persistent. Approved live probes are attached to the
|
|
1673
|
+
// current in-memory schema context by the CLI and discarded after the Ask.
|
|
1674
|
+
void terms;
|
|
1675
|
+
void limit;
|
|
1676
|
+
return [];
|
|
1677
|
+
}
|
|
1678
|
+
searchRuntimeSchemaObjects(query, limit = 40) {
|
|
1679
|
+
const match = buildFtsMatch(query, { prefix: true });
|
|
1680
|
+
if (!match.or)
|
|
1078
1681
|
return [];
|
|
1079
|
-
const
|
|
1080
|
-
SELECT
|
|
1081
|
-
bm25(
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1682
|
+
const runMatch = (matchExpr) => this.db.prepare(`
|
|
1683
|
+
SELECT o.*,
|
|
1684
|
+
bm25(runtime_schema_fts) AS rank,
|
|
1685
|
+
snippet(runtime_schema_fts, -1, '<mark>', '</mark>', '...', 12) AS snip
|
|
1686
|
+
FROM runtime_schema_fts
|
|
1687
|
+
JOIN runtime_schema_objects AS o ON o.object_key = runtime_schema_fts.object_key
|
|
1688
|
+
WHERE runtime_schema_fts MATCH ?
|
|
1085
1689
|
ORDER BY rank
|
|
1086
1690
|
LIMIT ?
|
|
1087
|
-
`).all(
|
|
1088
|
-
|
|
1089
|
-
|
|
1691
|
+
`).all(matchExpr, Math.max(1, limit));
|
|
1692
|
+
const andRows = match.and ? runMatch(match.and) : [];
|
|
1693
|
+
const seen = new Set(andRows.map((row) => row.object_key));
|
|
1694
|
+
const rows = [...andRows, ...runMatch(match.or).filter((row) => !seen.has(row.object_key))].slice(0, limit);
|
|
1695
|
+
return rows.map(rowToObject);
|
|
1090
1696
|
}
|
|
1091
1697
|
latestRuntimeSchemaSnapshot() {
|
|
1092
1698
|
const row = this.db.prepare(`
|
|
@@ -1166,14 +1772,91 @@ export class MetadataCatalog {
|
|
|
1166
1772
|
updatedAt: row.updated_at,
|
|
1167
1773
|
}));
|
|
1168
1774
|
}
|
|
1775
|
+
skillBody(bodyHash) {
|
|
1776
|
+
const row = this.db.prepare(`
|
|
1777
|
+
SELECT encoding, compressed_body
|
|
1778
|
+
FROM skill_bodies
|
|
1779
|
+
WHERE body_hash = ?
|
|
1780
|
+
`).get(bodyHash);
|
|
1781
|
+
if (!row)
|
|
1782
|
+
return null;
|
|
1783
|
+
if (row.encoding !== 'br')
|
|
1784
|
+
return row.compressed_body;
|
|
1785
|
+
const compressed = Uint8Array.from(Buffer.from(row.compressed_body, 'base64'));
|
|
1786
|
+
return brotliDecompressSync(compressed).toString('utf8');
|
|
1787
|
+
}
|
|
1788
|
+
exportSnapshot(destination) {
|
|
1789
|
+
mkdirSync(dirname(destination), { recursive: true });
|
|
1790
|
+
if (existsSync(destination))
|
|
1791
|
+
return;
|
|
1792
|
+
const candidate = `${destination}.candidate`;
|
|
1793
|
+
rmSync(candidate, { force: true });
|
|
1794
|
+
this.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
1795
|
+
this.db.prepare('VACUUM INTO ?').run(candidate);
|
|
1796
|
+
const Database = loadDatabase();
|
|
1797
|
+
const sealed = new Database(candidate);
|
|
1798
|
+
try {
|
|
1799
|
+
sealed.exec(`
|
|
1800
|
+
DELETE FROM context_packs;
|
|
1801
|
+
DELETE FROM query_runs;
|
|
1802
|
+
DELETE FROM runtime_schema_snapshots;
|
|
1803
|
+
DELETE FROM runtime_schema_fts;
|
|
1804
|
+
DELETE FROM runtime_schema_objects;
|
|
1805
|
+
DELETE FROM runtime_value_fts;
|
|
1806
|
+
DELETE FROM runtime_value_index;
|
|
1807
|
+
VACUUM;
|
|
1808
|
+
`);
|
|
1809
|
+
sealed.pragma('journal_mode = DELETE');
|
|
1810
|
+
}
|
|
1811
|
+
finally {
|
|
1812
|
+
sealed.close();
|
|
1813
|
+
}
|
|
1814
|
+
renameSync(candidate, destination);
|
|
1815
|
+
}
|
|
1169
1816
|
state(key) {
|
|
1170
1817
|
const row = this.db.prepare('SELECT value FROM metadata_state WHERE key = ?').get(key);
|
|
1171
1818
|
return row?.value ?? null;
|
|
1172
1819
|
}
|
|
1820
|
+
setState(key, value) {
|
|
1821
|
+
this.db.prepare('INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)').run(key, value);
|
|
1822
|
+
}
|
|
1173
1823
|
close() {
|
|
1174
1824
|
this.db.close();
|
|
1175
1825
|
}
|
|
1176
1826
|
}
|
|
1827
|
+
function strongestBm25Magnitude(rows) {
|
|
1828
|
+
return rows.reduce((strongest, row) => Math.max(strongest, bm25Magnitude(row.rank)), 0);
|
|
1829
|
+
}
|
|
1830
|
+
function bm25Magnitude(rank) {
|
|
1831
|
+
return typeof rank === 'number' && Number.isFinite(rank) ? Math.max(0, -rank) : 0;
|
|
1832
|
+
}
|
|
1833
|
+
function normalizedBm25Score(rank, tier, strongest) {
|
|
1834
|
+
const relative = strongest > 0 ? bm25Magnitude(rank) / strongest : 0;
|
|
1835
|
+
const score = tier === 'and'
|
|
1836
|
+
? 0.55 + relative * 0.45
|
|
1837
|
+
: 0.05 + relative * 0.45;
|
|
1838
|
+
return Number(score.toFixed(6));
|
|
1839
|
+
}
|
|
1840
|
+
function legacyQualifiedAlias(objectKey) {
|
|
1841
|
+
const mappings = [
|
|
1842
|
+
['semantic:entity:', 'dql_entity'],
|
|
1843
|
+
['model_area:', 'model_area'],
|
|
1844
|
+
['relationship:', 'relationship'],
|
|
1845
|
+
['contract:', 'contract'],
|
|
1846
|
+
['domain_export:', 'domain_export'],
|
|
1847
|
+
['domain_import:', 'domain_import'],
|
|
1848
|
+
['conformance:', 'conformance'],
|
|
1849
|
+
['policy:', 'policy'],
|
|
1850
|
+
['evaluation:', 'evaluation'],
|
|
1851
|
+
];
|
|
1852
|
+
for (const [prefix, objectType] of mappings) {
|
|
1853
|
+
if (!objectKey.startsWith(prefix))
|
|
1854
|
+
continue;
|
|
1855
|
+
const name = objectKey.slice(prefix.length);
|
|
1856
|
+
return name ? { objectType, name } : undefined;
|
|
1857
|
+
}
|
|
1858
|
+
return undefined;
|
|
1859
|
+
}
|
|
1177
1860
|
function loadAgentManifest(projectRoot) {
|
|
1178
1861
|
return buildManifest({
|
|
1179
1862
|
projectRoot,
|
|
@@ -1227,6 +1910,7 @@ function manifestDiagnosticToMetadataDiagnostic(diagnostic) {
|
|
|
1227
1910
|
};
|
|
1228
1911
|
}
|
|
1229
1912
|
function objectFromKGNode(node) {
|
|
1913
|
+
const qualifiedIdentity = qualifiedIdentityFromKGNode(node);
|
|
1230
1914
|
const payload = {
|
|
1231
1915
|
...(node.payload ?? {}),
|
|
1232
1916
|
kgNodeId: node.nodeId,
|
|
@@ -1261,13 +1945,21 @@ function objectFromKGNode(node) {
|
|
|
1261
1945
|
businessRules: node.businessRules ?? [],
|
|
1262
1946
|
caveats: node.caveats ?? [],
|
|
1263
1947
|
llmContext: node.llmContext,
|
|
1948
|
+
label: kgContextField(node.llmContext, 'label'),
|
|
1949
|
+
metricType: kgContextField(node.llmContext, 'metric type'),
|
|
1950
|
+
aggregation: kgContextField(node.llmContext, 'aggregation'),
|
|
1951
|
+
table: kgContextField(node.llmContext, 'table'),
|
|
1952
|
+
formula: node.kind === 'metric' ? kgContextField(node.llmContext, 'sql') : undefined,
|
|
1953
|
+
semanticModel: ['metric', 'dimension', 'measure', 'entity'].includes(node.kind) && node.name.includes('.')
|
|
1954
|
+
? node.name.split('.')[0]
|
|
1955
|
+
: undefined,
|
|
1264
1956
|
referencedBy: node.referencedBy ?? [],
|
|
1265
1957
|
};
|
|
1266
1958
|
return {
|
|
1267
1959
|
objectKey: objectKeyFromKGNode(node),
|
|
1268
1960
|
objectType: objectTypeFromKGNode(node),
|
|
1269
1961
|
name: node.name,
|
|
1270
|
-
fullName: node.name,
|
|
1962
|
+
fullName: qualifiedIdentity ?? node.name,
|
|
1271
1963
|
domain: node.domain,
|
|
1272
1964
|
owner: node.owner,
|
|
1273
1965
|
status: node.status ?? node.certification,
|
|
@@ -1277,7 +1969,18 @@ function objectFromKGNode(node) {
|
|
|
1277
1969
|
payload: compactObject(payload),
|
|
1278
1970
|
};
|
|
1279
1971
|
}
|
|
1972
|
+
function kgContextField(context, field) {
|
|
1973
|
+
if (!context)
|
|
1974
|
+
return undefined;
|
|
1975
|
+
const prefix = `${field.toLowerCase()}:`;
|
|
1976
|
+
const line = context.split(/\r?\n/).find((candidate) => candidate.trim().toLowerCase().startsWith(prefix));
|
|
1977
|
+
return line?.trim().slice(prefix.length).trim() || undefined;
|
|
1978
|
+
}
|
|
1280
1979
|
function objectKeyFromKGNode(node) {
|
|
1980
|
+
const qualifiedIdentity = qualifiedIdentityFromKGNode(node);
|
|
1981
|
+
if (qualifiedIdentity && isQualifiedDqlModelingNode(node)) {
|
|
1982
|
+
return `dql:${node.kind}:${qualifiedIdentity}`;
|
|
1983
|
+
}
|
|
1281
1984
|
switch (node.kind) {
|
|
1282
1985
|
case 'block': return `dql:block:${node.name}`;
|
|
1283
1986
|
case 'term': return `dql:term:${node.name}`;
|
|
@@ -1300,6 +2003,29 @@ function objectKeyFromKGNode(node) {
|
|
|
1300
2003
|
default: return `${node.kind}:${node.name}`;
|
|
1301
2004
|
}
|
|
1302
2005
|
}
|
|
2006
|
+
function isQualifiedDqlModelingNode(node) {
|
|
2007
|
+
return node.sourceTier === 'business_context' && [
|
|
2008
|
+
'model_area',
|
|
2009
|
+
'entity',
|
|
2010
|
+
'relationship',
|
|
2011
|
+
'contract',
|
|
2012
|
+
'domain_export',
|
|
2013
|
+
'domain_import',
|
|
2014
|
+
'conformance',
|
|
2015
|
+
'policy',
|
|
2016
|
+
'evaluation',
|
|
2017
|
+
].includes(node.kind);
|
|
2018
|
+
}
|
|
2019
|
+
function qualifiedIdentityFromKGNode(node) {
|
|
2020
|
+
const payload = node.payload ?? {};
|
|
2021
|
+
const explicit = stringValue(payload.qualifiedId);
|
|
2022
|
+
if (explicit)
|
|
2023
|
+
return explicit;
|
|
2024
|
+
if (!isQualifiedDqlModelingNode(node))
|
|
2025
|
+
return undefined;
|
|
2026
|
+
const prefix = `${node.kind}:`;
|
|
2027
|
+
return node.nodeId.startsWith(prefix) ? node.nodeId.slice(prefix.length) : undefined;
|
|
2028
|
+
}
|
|
1303
2029
|
function objectTypeFromKGNode(node) {
|
|
1304
2030
|
switch (node.kind) {
|
|
1305
2031
|
case 'block': return 'dql_block';
|
|
@@ -1308,7 +2034,7 @@ function objectTypeFromKGNode(node) {
|
|
|
1308
2034
|
case 'metric': return 'semantic_metric';
|
|
1309
2035
|
case 'dimension': return 'semantic_dimension';
|
|
1310
2036
|
case 'measure': return 'semantic_measure';
|
|
1311
|
-
case 'entity': return 'semantic_entity';
|
|
2037
|
+
case 'entity': return isQualifiedDqlModelingNode(node) ? 'dql_entity' : 'semantic_entity';
|
|
1312
2038
|
case 'semantic_model': return 'semantic_model';
|
|
1313
2039
|
case 'saved_query': return 'semantic_saved_query';
|
|
1314
2040
|
case 'dbt_model': return 'dbt_model';
|
|
@@ -1337,6 +2063,15 @@ function normalizeEdge(edge, fromKey, toKey) {
|
|
|
1337
2063
|
function addSkillObjects(skills, objects, edges) {
|
|
1338
2064
|
for (const skill of skills) {
|
|
1339
2065
|
const identity = skill.qualifiedId ?? skill.id;
|
|
2066
|
+
const bodyHash = sha256(skill.body);
|
|
2067
|
+
const sourceFingerprint = sha256(stableStringify({
|
|
2068
|
+
identity,
|
|
2069
|
+
domain: skill.domain,
|
|
2070
|
+
domains: skill.domains ?? [],
|
|
2071
|
+
modelAreaRefs: skill.modelAreaRefs ?? [],
|
|
2072
|
+
status: skill.status ?? 'active',
|
|
2073
|
+
bodyHash,
|
|
2074
|
+
}));
|
|
1340
2075
|
const objectKey = `skill:${identity}`;
|
|
1341
2076
|
const domains = uniqueNonBlank([skill.domain, ...(skill.domains ?? [])]);
|
|
1342
2077
|
const object = {
|
|
@@ -1369,12 +2104,13 @@ function addSkillObjects(skills, objects, edges) {
|
|
|
1369
2104
|
examples: skill.examples ?? [],
|
|
1370
2105
|
sourceRefs: skill.sourceRefs ?? [],
|
|
1371
2106
|
vocabulary: skill.vocabulary,
|
|
1372
|
-
|
|
2107
|
+
bodyHash,
|
|
2108
|
+
sourceFingerprint,
|
|
1373
2109
|
isStarter: skill.isStarter,
|
|
1374
2110
|
provenance: 'DQL domain skill',
|
|
1375
2111
|
}),
|
|
1376
2112
|
};
|
|
1377
|
-
objects.set(objectKey, object);
|
|
2113
|
+
objects.set(objectKey, mergeObject(objects.get(objectKey), object));
|
|
1378
2114
|
for (const domain of domains) {
|
|
1379
2115
|
const edge = {
|
|
1380
2116
|
edgeType: 'contains',
|
|
@@ -2444,9 +3180,16 @@ async function planContextPackRoute(input) {
|
|
|
2444
3180
|
&& (fit.confidence === 'high' || fit.confidence === 'medium'))
|
|
2445
3181
|
.sort((a, b) => (b.fit.confidence === 'high' ? 1 : 0) - (a.fit.confidence === 'high' ? 1 : 0)
|
|
2446
3182
|
|| b.applicabilityScore - a.applicabilityScore)[0]?.object;
|
|
2447
|
-
|
|
3183
|
+
const directlyNamedCertified = findExactCertifiedObject(input.request.question, intent, input.objects);
|
|
3184
|
+
const applicabilityCertified = exactByApplicability
|
|
2448
3185
|
? input.objects.find((object) => object.objectKey === exactByApplicability.objectKey)
|
|
2449
|
-
:
|
|
3186
|
+
: undefined;
|
|
3187
|
+
// Selection order is governance-significant (AGT-009/AGT-010): an explicit
|
|
3188
|
+
// block/example reference wins, then a complete answer-shape fit, and only then
|
|
3189
|
+
// the older lexical applicability heuristic. This prevents a broad customer
|
|
3190
|
+
// block containing the word "spend" from displacing an exact beverage-scoped
|
|
3191
|
+
// customer ranking whose measure is named "beverage_revenue".
|
|
3192
|
+
let exact = directlyNamedCertified ?? exactByContract ?? applicabilityCertified;
|
|
2450
3193
|
const contextApplicability = [...applicabilityByKey.values()]
|
|
2451
3194
|
.filter((item) => item.kind === 'context_only')
|
|
2452
3195
|
.sort((a, b) => b.score - a.score)[0];
|
|
@@ -2478,7 +3221,6 @@ async function planContextPackRoute(input) {
|
|
|
2478
3221
|
const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
|
|
2479
3222
|
const directCertifiedBypass = Boolean(exact && (exactExampleMatch
|
|
2480
3223
|
|| intent === 'definition_lookup'
|
|
2481
|
-
|| intent === 'exact_certified_lookup'
|
|
2482
3224
|
|| objectNameInQuestion(input.request.question, exact)));
|
|
2483
3225
|
const blockFitObject = exact
|
|
2484
3226
|
?? (contextApplicability ? input.objects.find((object) => object.objectKey === contextApplicability.objectKey) : undefined);
|
|
@@ -2542,7 +3284,9 @@ async function planContextPackRoute(input) {
|
|
|
2542
3284
|
|| (intent === 'ad_hoc_ranking' && objectNameInQuestion(input.request.question, exact)))) {
|
|
2543
3285
|
return {
|
|
2544
3286
|
route: 'certified',
|
|
2545
|
-
intent
|
|
3287
|
+
intent: input.questionPlan.outputShape === 'value' && input.questionPlan.requestedShape.dimensions.length === 0
|
|
3288
|
+
? 'exact_certified_lookup'
|
|
3289
|
+
: intent,
|
|
2546
3290
|
reason: `Certified ${exact.objectType.replace(/_/g, ' ')} "${exact.name}" exactly matches the requested artifact, definition, or direct KPI grain.`,
|
|
2547
3291
|
routeReason: grainGateInfo?.reason,
|
|
2548
3292
|
grainGate: grainGateInfo,
|
|
@@ -4094,23 +4838,7 @@ function runtimeSchemaObjects(snapshot) {
|
|
|
4094
4838
|
const capturedAt = snapshot.capturedAt ?? new Date().toISOString();
|
|
4095
4839
|
return normalizeRuntimeSchemaTables(snapshot.tables).flatMap((table) => {
|
|
4096
4840
|
const relation = table.relation;
|
|
4097
|
-
const
|
|
4098
|
-
const tableObject = {
|
|
4099
|
-
objectKey: tableKey,
|
|
4100
|
-
objectType: 'runtime_table',
|
|
4101
|
-
name: table.name ?? relation.split('.').at(-1) ?? relation,
|
|
4102
|
-
fullName: relation,
|
|
4103
|
-
description: table.description,
|
|
4104
|
-
status: 'runtime_observed',
|
|
4105
|
-
sourceSystem: snapshot.source ?? table.source ?? 'runtime schema snapshot',
|
|
4106
|
-
payload: compactObject({
|
|
4107
|
-
relation,
|
|
4108
|
-
schema: table.schema,
|
|
4109
|
-
columnCompleteness: 'complete',
|
|
4110
|
-
columns: table.columns,
|
|
4111
|
-
}),
|
|
4112
|
-
updatedAt: capturedAt,
|
|
4113
|
-
};
|
|
4841
|
+
const tableObject = runtimeSchemaTableObject(table, snapshot.source, capturedAt);
|
|
4114
4842
|
const columns = table.columns.map((column) => ({
|
|
4115
4843
|
objectKey: `runtime:column:${relation}.${column.name}`,
|
|
4116
4844
|
objectType: 'runtime_column',
|
|
@@ -4130,6 +4858,25 @@ function runtimeSchemaObjects(snapshot) {
|
|
|
4130
4858
|
return [tableObject, ...columns];
|
|
4131
4859
|
});
|
|
4132
4860
|
}
|
|
4861
|
+
function runtimeSchemaTableObject(table, snapshotSource, capturedAt) {
|
|
4862
|
+
const relation = table.relation;
|
|
4863
|
+
return {
|
|
4864
|
+
objectKey: `runtime:table:${relation}`,
|
|
4865
|
+
objectType: 'runtime_table',
|
|
4866
|
+
name: table.name ?? relation.split('.').at(-1) ?? relation,
|
|
4867
|
+
fullName: relation,
|
|
4868
|
+
description: table.description,
|
|
4869
|
+
status: 'runtime_observed',
|
|
4870
|
+
sourceSystem: snapshotSource ?? table.source ?? 'runtime schema snapshot',
|
|
4871
|
+
payload: compactObject({
|
|
4872
|
+
relation,
|
|
4873
|
+
schema: table.schema,
|
|
4874
|
+
columnCompleteness: 'complete',
|
|
4875
|
+
columns: table.columns,
|
|
4876
|
+
}),
|
|
4877
|
+
updatedAt: capturedAt,
|
|
4878
|
+
};
|
|
4879
|
+
}
|
|
4133
4880
|
function runtimeValueMatchObjects(matches) {
|
|
4134
4881
|
return matches.map((match) => ({
|
|
4135
4882
|
objectKey: match.valueKey,
|
|
@@ -4370,7 +5117,7 @@ const METADATA_VALUE_STOP_TERMS = new Set([
|
|
|
4370
5117
|
'this year',
|
|
4371
5118
|
'user',
|
|
4372
5119
|
]);
|
|
4373
|
-
function buildFollowUpSearchQuery(question, followUp) {
|
|
5120
|
+
export function buildFollowUpSearchQuery(question, followUp) {
|
|
4374
5121
|
if (!followUp)
|
|
4375
5122
|
return question;
|
|
4376
5123
|
// Contextual carry is advisory: enrich retrieval softly (prior question, columns,
|
|
@@ -4392,14 +5139,19 @@ function buildFollowUpSearchQuery(question, followUp) {
|
|
|
4392
5139
|
question,
|
|
4393
5140
|
followUp.sourceBlockName ?? '',
|
|
4394
5141
|
followUp.sourceQuestion ?? '',
|
|
4395
|
-
...(followUp.filters ?? []),
|
|
4396
5142
|
...(followUp.dimensions ?? []),
|
|
4397
5143
|
...(followUp.priorResultColumns ?? []),
|
|
4398
5144
|
followUp.priorResultRef?.question ?? '',
|
|
4399
5145
|
...(followUp.priorResultRef?.columns ?? []),
|
|
4400
|
-
followUp.
|
|
4401
|
-
|
|
4402
|
-
|
|
5146
|
+
...(followUp.priorMeasures ?? []),
|
|
5147
|
+
// Retrieval searches object meaning, not executable payloads or warehouse
|
|
5148
|
+
// members. The source block is loaded directly by key and the typed follow-up
|
|
5149
|
+
// object retains SQL + values for execution. Injecting 600-1500 characters of
|
|
5150
|
+
// SQL and every prior row value into FTS made an eight-customer drilldown spend
|
|
5151
|
+
// ~11 seconds tokenizing metadata before the provider even started.
|
|
5152
|
+
...(followUp.priorDqlArtifact?.metrics ?? []),
|
|
5153
|
+
...(followUp.priorDqlArtifact?.dimensions ?? []),
|
|
5154
|
+
...(followUp.priorDqlArtifact?.filters ?? []).map((filter) => filter.dimension),
|
|
4403
5155
|
].filter(Boolean).join(' ');
|
|
4404
5156
|
}
|
|
4405
5157
|
function dqlArtifactSearchTerms(artifact, options) {
|
|
@@ -4448,7 +5200,15 @@ function normalizeRuntimeSchemaTables(tables) {
|
|
|
4448
5200
|
name: table.name ?? relation.split('.').at(-1) ?? relation,
|
|
4449
5201
|
description: table.description,
|
|
4450
5202
|
source: table.source,
|
|
4451
|
-
|
|
5203
|
+
// Persist structural schema only. Live sample values can contain PII and
|
|
5204
|
+
// are scoped to a single request by the CLI (SEC-003).
|
|
5205
|
+
columns: dedupeRuntimeColumns(table.columns ?? [])
|
|
5206
|
+
.slice(0, 160)
|
|
5207
|
+
.map((column) => ({
|
|
5208
|
+
name: column.name,
|
|
5209
|
+
type: column.type,
|
|
5210
|
+
description: column.description,
|
|
5211
|
+
})),
|
|
4452
5212
|
};
|
|
4453
5213
|
if (!current)
|
|
4454
5214
|
byRelation.set(key, normalized);
|
|
@@ -4670,6 +5430,54 @@ function compactObject(value) {
|
|
|
4670
5430
|
}
|
|
4671
5431
|
return out;
|
|
4672
5432
|
}
|
|
5433
|
+
const SEARCHABLE_METADATA_KEYS = new Set([
|
|
5434
|
+
'aliases', 'synonyms', 'tags', 'label', 'metricType', 'aggregation', 'semanticModel',
|
|
5435
|
+
'localId', 'qualifiedId', 'uniqueId',
|
|
5436
|
+
'description', 'businessOutcome', 'decisionUse', 'grain', 'entities',
|
|
5437
|
+
'dimensions', 'allowedFilters', 'declaredOutputs', 'outputContract', 'outputs',
|
|
5438
|
+
'parameters', 'parameterPolicy', 'tableDependencies', 'sourceSystems', 'relation',
|
|
5439
|
+
'table', 'database', 'schema', 'materialized', 'columns', 'intentExamples',
|
|
5440
|
+
'examples', 'primaryTerms', 'businessRules', 'caveats', 'triggers', 'vocabulary',
|
|
5441
|
+
]);
|
|
5442
|
+
const SEARCHABLE_NESTED_METADATA_KEYS = new Set([
|
|
5443
|
+
'name', 'id', 'label', 'description', 'type', 'role', 'filter', 'binding',
|
|
5444
|
+
'question', 'entity', 'column', 'relation', 'table', 'domain', 'term', 'alias',
|
|
5445
|
+
'synonym', 'localId', 'qualifiedId', 'grain',
|
|
5446
|
+
]);
|
|
5447
|
+
/**
|
|
5448
|
+
* FTS indexes business/schema evidence, not the complete payload. In particular,
|
|
5449
|
+
* raw SQL and provider/runtime fields add large amounts of noise and can contain
|
|
5450
|
+
* literals that should never become general retrieval context.
|
|
5451
|
+
*/
|
|
5452
|
+
function searchableMetadataPayload(payload) {
|
|
5453
|
+
const tokens = [];
|
|
5454
|
+
const visit = (value, depth) => {
|
|
5455
|
+
if (tokens.length >= 2_000 || depth > 3 || value === null || value === undefined)
|
|
5456
|
+
return;
|
|
5457
|
+
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
|
5458
|
+
const text = String(value).replace(/\s+/g, ' ').trim();
|
|
5459
|
+
if (text)
|
|
5460
|
+
tokens.push(text.slice(0, 500));
|
|
5461
|
+
return;
|
|
5462
|
+
}
|
|
5463
|
+
if (Array.isArray(value)) {
|
|
5464
|
+
for (const item of value.slice(0, 500))
|
|
5465
|
+
visit(item, depth + 1);
|
|
5466
|
+
return;
|
|
5467
|
+
}
|
|
5468
|
+
if (typeof value !== 'object')
|
|
5469
|
+
return;
|
|
5470
|
+
for (const [key, item] of Object.entries(value)) {
|
|
5471
|
+
if (depth === 0 ? !SEARCHABLE_METADATA_KEYS.has(key) : !SEARCHABLE_NESTED_METADATA_KEYS.has(key))
|
|
5472
|
+
continue;
|
|
5473
|
+
if (/sql|secret|password|token|credential|api.?key/i.test(key))
|
|
5474
|
+
continue;
|
|
5475
|
+
visit(item, depth + 1);
|
|
5476
|
+
}
|
|
5477
|
+
};
|
|
5478
|
+
visit(payload, 0);
|
|
5479
|
+
return tokens.join(' ');
|
|
5480
|
+
}
|
|
4673
5481
|
function rowToObject(row) {
|
|
4674
5482
|
return {
|
|
4675
5483
|
objectKey: row.object_key,
|
|
@@ -4962,7 +5770,7 @@ function filterMetadataObjectsByDomainContext(rows, context) {
|
|
|
4962
5770
|
]);
|
|
4963
5771
|
return rows.filter((row) => !row.domain || domains.has(row.domain));
|
|
4964
5772
|
}
|
|
4965
|
-
function selectContextPackSkills(objects, question, context) {
|
|
5773
|
+
function selectContextPackSkills(catalog, objects, question, context) {
|
|
4966
5774
|
const byIdentity = new Map();
|
|
4967
5775
|
const parsed = objects.flatMap((object) => {
|
|
4968
5776
|
const skill = skillFromMetadataObject(object);
|
|
@@ -4978,10 +5786,15 @@ function selectContextPackSkills(objects, question, context) {
|
|
|
4978
5786
|
const selected = selectRelevantSkills(parsed, question, {
|
|
4979
5787
|
domains,
|
|
4980
5788
|
modelAreaIds: context?.modelAreaId ? [context.modelAreaId] : [],
|
|
5789
|
+
pinnedIds: context?.skillRefs ? ['sql-conventions', ...context.skillRefs] : undefined,
|
|
4981
5790
|
});
|
|
4982
5791
|
return selected.flatMap((skill) => {
|
|
4983
5792
|
const object = byIdentity.get(skill.qualifiedId ?? skill.id);
|
|
4984
|
-
|
|
5793
|
+
if (!object)
|
|
5794
|
+
return [];
|
|
5795
|
+
const bodyHash = stringPayload(object.payload?.bodyHash);
|
|
5796
|
+
const hydrated = bodyHash ? { ...skill, body: catalog.skillBody(bodyHash) ?? '' } : skill;
|
|
5797
|
+
return [{ object, skill: localContextSkillFromParsed(hydrated, object) }];
|
|
4985
5798
|
});
|
|
4986
5799
|
}
|
|
4987
5800
|
function skillFromMetadataObject(object) {
|
|
@@ -5017,7 +5830,7 @@ function skillFromMetadataObject(object) {
|
|
|
5017
5830
|
examples: stringArrayPayload(payload.examples),
|
|
5018
5831
|
sourceRefs: stringArrayPayload(payload.sourceRefs),
|
|
5019
5832
|
vocabulary: stringRecordPayload(payload.vocabulary),
|
|
5020
|
-
body:
|
|
5833
|
+
body: '',
|
|
5021
5834
|
sourcePath: object.sourcePath ?? '',
|
|
5022
5835
|
isStarter: payload.isStarter === true,
|
|
5023
5836
|
};
|