@duckcodeailabs/dql-agent 1.6.33 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-run-engine.d.ts +39 -8
- package/dist/agent-run-engine.d.ts.map +1 -1
- package/dist/agent-run-engine.js +169 -10
- package/dist/agent-run-engine.js.map +1 -1
- package/dist/agent-run-gates.d.ts.map +1 -1
- package/dist/agent-run-gates.js +21 -3
- package/dist/agent-run-gates.js.map +1 -1
- package/dist/agent-run-planner.d.ts.map +1 -1
- package/dist/agent-run-planner.js +14 -3
- package/dist/agent-run-planner.js.map +1 -1
- package/dist/agentic/answer-contract.d.ts +37 -0
- package/dist/agentic/answer-contract.d.ts.map +1 -0
- package/dist/agentic/answer-contract.js +47 -0
- package/dist/agentic/answer-contract.js.map +1 -0
- package/dist/agentic/tool-loop.d.ts +48 -0
- package/dist/agentic/tool-loop.d.ts.map +1 -0
- package/dist/agentic/tool-loop.js +181 -0
- package/dist/agentic/tool-loop.js.map +1 -0
- package/dist/agentic/toolset.d.ts +40 -0
- package/dist/agentic/toolset.d.ts.map +1 -0
- package/dist/agentic/toolset.js +265 -0
- package/dist/agentic/toolset.js.map +1 -0
- package/dist/answer-loop.d.ts +82 -6
- package/dist/answer-loop.d.ts.map +1 -1
- package/dist/answer-loop.js +799 -112
- package/dist/answer-loop.js.map +1 -1
- package/dist/app-builder.d.ts.map +1 -1
- package/dist/app-builder.js +18 -10
- package/dist/app-builder.js.map +1 -1
- package/dist/cascade/budgets.d.ts +20 -2
- package/dist/cascade/budgets.d.ts.map +1 -1
- package/dist/cascade/budgets.js +83 -45
- package/dist/cascade/budgets.js.map +1 -1
- package/dist/cascade/route-policy.d.ts +1 -1
- package/dist/cascade/route-policy.d.ts.map +1 -1
- package/dist/cascade/route-policy.js +6 -0
- package/dist/cascade/route-policy.js.map +1 -1
- package/dist/domain-context.d.ts +30 -0
- package/dist/domain-context.d.ts.map +1 -0
- package/dist/domain-context.js +68 -0
- package/dist/domain-context.js.map +1 -0
- package/dist/embeddings/provider.d.ts +67 -4
- package/dist/embeddings/provider.d.ts.map +1 -1
- package/dist/embeddings/provider.js +167 -6
- package/dist/embeddings/provider.js.map +1 -1
- package/dist/hints/correction-eval.d.ts +30 -0
- package/dist/hints/correction-eval.d.ts.map +1 -0
- package/dist/hints/correction-eval.js +63 -0
- package/dist/hints/correction-eval.js.map +1 -0
- package/dist/hints/git-store.d.ts +39 -3
- package/dist/hints/git-store.d.ts.map +1 -1
- package/dist/hints/git-store.js +172 -2
- package/dist/hints/git-store.js.map +1 -1
- package/dist/hints/staleness.d.ts +30 -0
- package/dist/hints/staleness.d.ts.map +1 -0
- package/dist/hints/staleness.js +38 -0
- package/dist/hints/staleness.js.map +1 -0
- package/dist/hints/store.d.ts.map +1 -1
- package/dist/hints/store.js +3 -32
- package/dist/hints/store.js.map +1 -1
- package/dist/hints/types.d.ts +41 -0
- package/dist/hints/types.d.ts.map +1 -1
- package/dist/hints/types.js +1 -0
- package/dist/hints/types.js.map +1 -1
- package/dist/index.d.ts +46 -10
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +180 -9
- package/dist/index.js.map +1 -1
- package/dist/intent-controller.d.ts.map +1 -1
- package/dist/intent-controller.js +15 -3
- package/dist/intent-controller.js.map +1 -1
- package/dist/kg/build.d.ts.map +1 -1
- package/dist/kg/build.js +196 -4
- package/dist/kg/build.js.map +1 -1
- package/dist/kg/sqlite-fts.d.ts +13 -0
- package/dist/kg/sqlite-fts.d.ts.map +1 -1
- package/dist/kg/sqlite-fts.js +82 -49
- package/dist/kg/sqlite-fts.js.map +1 -1
- package/dist/kg/types.d.ts +9 -2
- package/dist/kg/types.d.ts.map +1 -1
- package/dist/memory/fts-query.d.ts +37 -4
- package/dist/memory/fts-query.d.ts.map +1 -1
- package/dist/memory/fts-query.js +53 -12
- package/dist/memory/fts-query.js.map +1 -1
- package/dist/metadata/analysis-planner.d.ts.map +1 -1
- package/dist/metadata/analysis-planner.js +179 -28
- package/dist/metadata/analysis-planner.js.map +1 -1
- package/dist/metadata/analytical-policy.d.ts +51 -0
- package/dist/metadata/analytical-policy.d.ts.map +1 -0
- package/dist/metadata/analytical-policy.js +309 -0
- package/dist/metadata/analytical-policy.js.map +1 -0
- package/dist/metadata/block-adapt.d.ts +37 -0
- package/dist/metadata/block-adapt.d.ts.map +1 -0
- package/dist/metadata/block-adapt.js +89 -0
- package/dist/metadata/block-adapt.js.map +1 -0
- package/dist/metadata/block-fit.js +37 -4
- package/dist/metadata/block-fit.js.map +1 -1
- package/dist/metadata/catalog.d.ts +85 -1
- package/dist/metadata/catalog.d.ts.map +1 -1
- package/dist/metadata/catalog.js +674 -92
- package/dist/metadata/catalog.js.map +1 -1
- package/dist/metadata/dbt-first-safety.d.ts +19 -0
- package/dist/metadata/dbt-first-safety.d.ts.map +1 -0
- package/dist/metadata/dbt-first-safety.js +17 -0
- package/dist/metadata/dbt-first-safety.js.map +1 -0
- package/dist/metadata/example-match.d.ts +30 -0
- package/dist/metadata/example-match.d.ts.map +1 -0
- package/dist/metadata/example-match.js +69 -0
- package/dist/metadata/example-match.js.map +1 -0
- package/dist/metadata/grain-ledger.d.ts +51 -0
- package/dist/metadata/grain-ledger.d.ts.map +1 -0
- package/dist/metadata/grain-ledger.js +174 -0
- package/dist/metadata/grain-ledger.js.map +1 -0
- package/dist/metadata/metric-match.d.ts +21 -1
- package/dist/metadata/metric-match.d.ts.map +1 -1
- package/dist/metadata/metric-match.js +157 -21
- package/dist/metadata/metric-match.js.map +1 -1
- package/dist/metadata/sql-context-validation.d.ts.map +1 -1
- package/dist/metadata/sql-context-validation.js +42 -2
- package/dist/metadata/sql-context-validation.js.map +1 -1
- package/dist/metadata/sql-grounding.d.ts +4 -2
- package/dist/metadata/sql-grounding.d.ts.map +1 -1
- package/dist/metadata/sql-grounding.js +29 -4
- package/dist/metadata/sql-grounding.js.map +1 -1
- package/dist/metadata/sql-retrieval.d.ts +5 -2
- package/dist/metadata/sql-retrieval.d.ts.map +1 -1
- package/dist/metadata/sql-retrieval.js +28 -6
- package/dist/metadata/sql-retrieval.js.map +1 -1
- package/dist/propose/build-from-prompt.d.ts +34 -0
- package/dist/propose/build-from-prompt.d.ts.map +1 -1
- package/dist/propose/build-from-prompt.js +205 -19
- package/dist/propose/build-from-prompt.js.map +1 -1
- package/dist/propose/failure-analysis.d.ts +48 -0
- package/dist/propose/failure-analysis.d.ts.map +1 -0
- package/dist/propose/failure-analysis.js +71 -0
- package/dist/propose/failure-analysis.js.map +1 -0
- package/dist/propose/join-mining.d.ts +18 -0
- package/dist/propose/join-mining.d.ts.map +1 -0
- package/dist/propose/join-mining.js +74 -0
- package/dist/propose/join-mining.js.map +1 -0
- package/dist/providers/claude.d.ts.map +1 -1
- package/dist/providers/claude.js +36 -1
- package/dist/providers/claude.js.map +1 -1
- package/dist/providers/index.d.ts +2 -2
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +34 -1
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/reasoning-effort.d.ts +27 -0
- package/dist/providers/reasoning-effort.d.ts.map +1 -1
- package/dist/providers/reasoning-effort.js +32 -0
- package/dist/providers/reasoning-effort.js.map +1 -1
- package/dist/providers/types.d.ts +2 -1
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/research-loop.d.ts.map +1 -1
- package/dist/research-loop.js +9 -2
- package/dist/research-loop.js.map +1 -1
- package/dist/semantic-bridge/compose.d.ts.map +1 -1
- package/dist/semantic-bridge/compose.js +316 -25
- package/dist/semantic-bridge/compose.js.map +1 -1
- package/dist/skills/bootstrap.d.ts +57 -0
- package/dist/skills/bootstrap.d.ts.map +1 -0
- package/dist/skills/bootstrap.js +260 -0
- package/dist/skills/bootstrap.js.map +1 -0
- package/dist/skills/defaults.d.ts +2 -2
- package/dist/skills/defaults.js +2 -2
- package/dist/skills/domain-skills.d.ts +24 -0
- package/dist/skills/domain-skills.d.ts.map +1 -0
- package/dist/skills/domain-skills.js +76 -0
- package/dist/skills/domain-skills.js.map +1 -0
- package/dist/skills/loader.d.ts +56 -8
- package/dist/skills/loader.d.ts.map +1 -1
- package/dist/skills/loader.js +257 -41
- package/dist/skills/loader.js.map +1 -1
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +152 -16
- package/dist/synthesize.js.map +1 -1
- package/dist/tools/registry.d.ts +1 -1
- package/dist/tools/registry.d.ts.map +1 -1
- package/dist/tools/registry.js +54 -12
- package/dist/tools/registry.js.map +1 -1
- package/package.json +4 -4
package/dist/metadata/catalog.js
CHANGED
|
@@ -19,6 +19,10 @@ import { buildRuntimeValueIndex, normalizeValueIndexText, } from '../grounding/v
|
|
|
19
19
|
import { grainMatches, requestedGrainFromPlan, } from './grain-gate.js';
|
|
20
20
|
import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit, } from './block-fit.js';
|
|
21
21
|
import { retrieveScopedHints } from '../hints/retrieval.js';
|
|
22
|
+
import { buildFtsMatch, sanitizeFtsQuery } from '../memory/fts-query.js';
|
|
23
|
+
import { envEmbeddingProvider } from '../embeddings/provider.js';
|
|
24
|
+
import { matchExampleParaphrase } from './example-match.js';
|
|
25
|
+
import { loadSkills, selectRelevantSkills } from '../skills/loader.js';
|
|
22
26
|
const require = createRequire(import.meta.url);
|
|
23
27
|
let databaseCtor = null;
|
|
24
28
|
function loadDatabase() {
|
|
@@ -27,6 +31,39 @@ function loadDatabase() {
|
|
|
27
31
|
}
|
|
28
32
|
const SCHEMA_SHAPE_CACHE_LIMIT = 64;
|
|
29
33
|
const schemaShapeCandidateCache = new Map();
|
|
34
|
+
/**
|
|
35
|
+
* Extract single-column `unique` dbt tests, keyed by model name (W5.3). A column
|
|
36
|
+
* a `unique` test guards is a grain key, which feeds the grain ledger's fan-out
|
|
37
|
+
* detection — the common way real dbt repos declare keys. Handles both the
|
|
38
|
+
* `attached_node` (newer dbt) and `depends_on` shapes, and top-level or kwargs
|
|
39
|
+
* column names.
|
|
40
|
+
*/
|
|
41
|
+
export function extractDbtUniqueColumns(nodes) {
|
|
42
|
+
const byModel = new Map();
|
|
43
|
+
for (const node of Object.values(nodes)) {
|
|
44
|
+
if (node.resource_type !== 'test')
|
|
45
|
+
continue;
|
|
46
|
+
const testMetadata = node.test_metadata;
|
|
47
|
+
if (stringValue(testMetadata?.name) !== 'unique')
|
|
48
|
+
continue;
|
|
49
|
+
const column = stringValue(node.column_name) ?? stringValue(testMetadata?.kwargs?.column_name);
|
|
50
|
+
if (!column)
|
|
51
|
+
continue;
|
|
52
|
+
const dependsOn = node.depends_on?.nodes;
|
|
53
|
+
const attached = stringValue(node.attached_node)
|
|
54
|
+
?? (Array.isArray(dependsOn) ? dependsOn.map(String).find((ref) => ref.startsWith('model.')) : undefined);
|
|
55
|
+
if (!attached)
|
|
56
|
+
continue;
|
|
57
|
+
const modelName = attached.split('.').at(-1);
|
|
58
|
+
if (!modelName)
|
|
59
|
+
continue;
|
|
60
|
+
const existing = byModel.get(modelName) ?? [];
|
|
61
|
+
if (!existing.includes(column))
|
|
62
|
+
existing.push(column);
|
|
63
|
+
byModel.set(modelName, existing);
|
|
64
|
+
}
|
|
65
|
+
return byModel;
|
|
66
|
+
}
|
|
30
67
|
const OBJECT_PRIORITY = {
|
|
31
68
|
dql_block: 1,
|
|
32
69
|
dql_block_output: 1.5,
|
|
@@ -60,7 +97,8 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
|
|
|
60
97
|
? options.semanticLayer ?? undefined
|
|
61
98
|
: loadAgentSemanticLayer(projectRoot);
|
|
62
99
|
const manifest = options.manifest ?? loadAgentManifest(projectRoot);
|
|
63
|
-
const
|
|
100
|
+
const skills = options.skills ?? loadSkills(projectRoot).skills;
|
|
101
|
+
const snapshot = buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skills);
|
|
64
102
|
const catalog = openMetadataCatalog(projectRoot);
|
|
65
103
|
try {
|
|
66
104
|
const existing = catalog.state('fingerprint');
|
|
@@ -74,7 +112,15 @@ export async function ensureMetadataCatalogFresh(projectRoot, options = {}) {
|
|
|
74
112
|
fingerprint: snapshot.fingerprint,
|
|
75
113
|
};
|
|
76
114
|
}
|
|
77
|
-
|
|
115
|
+
// W3.4 — incremental reindex when prior state exists (only re-tokenizes changed
|
|
116
|
+
// sources' FTS); `force` does a clean full rebuild. Proven equal to a full
|
|
117
|
+
// rebuild by incremental-reindex.test.ts.
|
|
118
|
+
if (options.force) {
|
|
119
|
+
catalog.rebuild(snapshot);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
catalog.rebuildIncremental(snapshot);
|
|
123
|
+
}
|
|
78
124
|
return {
|
|
79
125
|
path: defaultMetadataPath(projectRoot),
|
|
80
126
|
refreshed: true,
|
|
@@ -106,7 +152,18 @@ export function upsertMetadataSnapshot(projectRoot, snapshot) {
|
|
|
106
152
|
}
|
|
107
153
|
}
|
|
108
154
|
export async function buildLocalContextPack(projectRoot, request) {
|
|
109
|
-
|
|
155
|
+
let prepared = false;
|
|
156
|
+
if (request.preparedMetadataFingerprint) {
|
|
157
|
+
const catalog = openMetadataCatalog(projectRoot);
|
|
158
|
+
try {
|
|
159
|
+
prepared = catalog.state('fingerprint') === request.preparedMetadataFingerprint;
|
|
160
|
+
}
|
|
161
|
+
finally {
|
|
162
|
+
catalog.close();
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (!prepared)
|
|
166
|
+
await ensureMetadataCatalogFresh(projectRoot);
|
|
110
167
|
const catalog = openMetadataCatalog(projectRoot);
|
|
111
168
|
try {
|
|
112
169
|
const mode = request.mode ?? 'question';
|
|
@@ -119,7 +176,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
119
176
|
// candidates only; a SHIFT ignores it. A metadata fingerprint mismatch
|
|
120
177
|
// always disqualifies reuse.
|
|
121
178
|
const reusePolicy = request.reusePolicy ?? 'seed';
|
|
122
|
-
const priorPack = reusePolicy !== 'off'
|
|
179
|
+
const priorPack = !request.domainContext && reusePolicy !== 'off'
|
|
123
180
|
&& request.priorContextPackId
|
|
124
181
|
&& request.conversationTopicRelation
|
|
125
182
|
&& request.conversationTopicRelation !== 'shift'
|
|
@@ -167,18 +224,24 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
167
224
|
buildFollowUpSearchQuery(request.question, followUp),
|
|
168
225
|
...questionPlan.searchQueries,
|
|
169
226
|
]);
|
|
170
|
-
const
|
|
227
|
+
const scopeObjects = (rows) => filterMetadataObjectsByDomainContext(rows, request.domainContext);
|
|
228
|
+
// Skill guidance is injected only through selectContextPackSkills below.
|
|
229
|
+
// Leaving it in generic FTS results would bypass status/domain/area/exclusion
|
|
230
|
+
// eligibility simply because a word in its body matched the question.
|
|
231
|
+
const retrievalObjects = (rows) => scopeObjects(rows).filter((row) => row.objectType !== 'skill');
|
|
232
|
+
const searchRows = retrievalObjects(mergeObjects(searchQueries.flatMap((query) => catalog.searchObjects({
|
|
171
233
|
query,
|
|
172
234
|
objectTypes: request.objectTypes,
|
|
173
235
|
limit: Math.max(request.limit ?? 80, 20),
|
|
174
|
-
})));
|
|
236
|
+
}))));
|
|
175
237
|
const schemaShapeCandidates = schemaShapeCandidateObjects(catalog, questionPlan, request, mergeObjects([...runtimeObjects, ...runtimeValueObjects]));
|
|
176
238
|
const schemaShapeObjects = schemaShapeCandidates.map((candidate) => candidate.object);
|
|
177
|
-
const
|
|
239
|
+
const exactCandidate = request.focusObjectKey ? catalog.getObject(request.focusObjectKey) : null;
|
|
240
|
+
const exact = exactCandidate && retrievalObjects([exactCandidate]).length > 0 ? exactCandidate : null;
|
|
178
241
|
const ranked = rankMetadataObjects({
|
|
179
|
-
rows: mergeObjects(exact
|
|
242
|
+
rows: retrievalObjects(mergeObjects(exact
|
|
180
243
|
? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]
|
|
181
|
-
: [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]),
|
|
244
|
+
: [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects])),
|
|
182
245
|
question: searchQueries.join(' '),
|
|
183
246
|
questionPlan,
|
|
184
247
|
limit: request.limit ?? 80,
|
|
@@ -192,9 +255,9 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
192
255
|
const focusObjectKey = request.focusObjectKey ?? selected[0]?.objectKey ?? null;
|
|
193
256
|
const edgeWalk = catalog.edgesForKeys(selected.map((row) => row.objectKey), 3);
|
|
194
257
|
const edgeObjectKeys = Array.from(new Set(edgeWalk.flatMap((edge) => [edge.fromKey, edge.toKey])));
|
|
195
|
-
const graphObjects = catalog.getObjectsByKeys(edgeObjectKeys);
|
|
258
|
+
const graphObjects = retrievalObjects(catalog.getObjectsByKeys(edgeObjectKeys));
|
|
196
259
|
const rankedObjects = rankMetadataObjects({
|
|
197
|
-
rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects]),
|
|
260
|
+
rows: retrievalObjects(mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects])),
|
|
198
261
|
question: searchQueries.join(' '),
|
|
199
262
|
questionPlan,
|
|
200
263
|
limit: request.limit ?? 120,
|
|
@@ -204,24 +267,30 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
204
267
|
// entire relation set ("send everything, let the agent decide"). Only when the
|
|
205
268
|
// whole catalog fits comfortably in context — otherwise we keep ranked selection.
|
|
206
269
|
const fullCatalogObjects = request.strictness === 'exploratory'
|
|
207
|
-
? collectFullCatalogObjects(catalog)
|
|
270
|
+
? retrievalObjects(collectFullCatalogObjects(catalog) ?? [])
|
|
208
271
|
: undefined;
|
|
209
272
|
const usedFullCatalog = Boolean(fullCatalogObjects);
|
|
273
|
+
// Skills are selected from the catalog snapshot with the same hard domain,
|
|
274
|
+
// area, status, and exclusion gates as the loader. They are not left to FTS
|
|
275
|
+
// chance or re-read from disk after the snapshot has been fingerprinted.
|
|
276
|
+
const selectedSkills = selectContextPackSkills(catalog.listObjects({ objectTypes: ['skill'], limit: 500 }), request.question, request.domainContext);
|
|
277
|
+
const selectedSkillObjects = selectedSkills.map((item) => item.object);
|
|
210
278
|
const objects = fullCatalogObjects
|
|
211
|
-
? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects])
|
|
212
|
-
: mergeObjects([...rankedObjects, ...sqlParentObjects]);
|
|
279
|
+
? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects, ...selectedSkillObjects])
|
|
280
|
+
: mergeObjects([...rankedObjects, ...sqlParentObjects, ...selectedSkillObjects]);
|
|
213
281
|
const objectKeys = objects.map((row) => row.objectKey);
|
|
282
|
+
const allowedObjectKeys = new Set(objectKeys);
|
|
214
283
|
const contextEdges = mergeMetadataEdges([
|
|
215
284
|
...edgeWalk,
|
|
216
285
|
...catalog.edgesForKeys(objectKeys, 2),
|
|
217
|
-
]);
|
|
286
|
+
]).filter((edge) => allowedObjectKeys.has(edge.fromKey) && allowedObjectKeys.has(edge.toKey));
|
|
218
287
|
const queryRuns = catalog.queryRunsForObjectKeys(objectKeys, 20);
|
|
219
288
|
const diagnostics = catalog.diagnostics();
|
|
220
289
|
const warnings = buildWarnings(diagnostics, objects);
|
|
221
290
|
const trustLabel = deriveTrust(objects);
|
|
222
291
|
const citations = buildCitations(objects, contextEdges);
|
|
223
292
|
const evidenceSummaries = buildEvidenceSummaries(objects, contextEdges, queryRuns, diagnostics);
|
|
224
|
-
const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, contextEdges), questionPlan);
|
|
293
|
+
const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, contextEdges, tokenizeQuestionForColumns(request.question)), questionPlan);
|
|
225
294
|
const selectedRelations = allowedSqlContext.relations.slice(0, 24).map((relation, index) => {
|
|
226
295
|
const scored = scoreAllowedSqlRelationWithAnalysisPlan(relation, questionPlan);
|
|
227
296
|
return {
|
|
@@ -237,7 +306,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
237
306
|
const selectedJoinPaths = buildSelectedJoinPaths(allowedSqlContext, contextEdges);
|
|
238
307
|
const evidenceRoles = buildEvidenceRoles(objects, queryRuns);
|
|
239
308
|
const reranked = rankMetadataObjects({
|
|
240
|
-
rows: mergeObjects([...searchRows, ...schemaShapeObjects, ...objects]),
|
|
309
|
+
rows: retrievalObjects(mergeObjects([...searchRows, ...schemaShapeObjects, ...objects])),
|
|
241
310
|
question: searchQueries.join(' '),
|
|
242
311
|
questionPlan,
|
|
243
312
|
limit: request.limit ?? 120,
|
|
@@ -279,6 +348,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
279
348
|
trustLabel,
|
|
280
349
|
trustLabelInfo: metadataTrustLabelInfo(trustLabel),
|
|
281
350
|
objects,
|
|
351
|
+
skills: selectedSkills.map((item) => item.skill),
|
|
282
352
|
edges: contextEdges,
|
|
283
353
|
queryRuns,
|
|
284
354
|
citations,
|
|
@@ -362,7 +432,17 @@ export function recordRuntimeSchemaSnapshot(projectRoot, snapshot) {
|
|
|
362
432
|
catalog.close();
|
|
363
433
|
}
|
|
364
434
|
}
|
|
365
|
-
|
|
435
|
+
/** The most-recent stored live-warehouse schema snapshot for a project, or null. */
|
|
436
|
+
export function latestRuntimeSchemaSnapshotForProject(projectRoot) {
|
|
437
|
+
const catalog = openMetadataCatalog(projectRoot);
|
|
438
|
+
try {
|
|
439
|
+
return catalog.latestRuntimeSchemaSnapshot();
|
|
440
|
+
}
|
|
441
|
+
finally {
|
|
442
|
+
catalog.close();
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer, skills = loadSkills(projectRoot).skills) {
|
|
366
446
|
const manifestGraph = buildKGFromManifest(manifest);
|
|
367
447
|
const semanticGraph = buildKGFromSemanticLayer(semanticLayer);
|
|
368
448
|
const objects = new Map();
|
|
@@ -387,11 +467,16 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer) {
|
|
|
387
467
|
}
|
|
388
468
|
}
|
|
389
469
|
addManifestBlockDetails(manifest, objects);
|
|
470
|
+
addSkillObjects(skills, objects, edges);
|
|
390
471
|
addDbtDagObjects(manifest, objects, edges, diagnostics);
|
|
391
472
|
addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges, diagnostics);
|
|
392
473
|
addBlockDependencyEdges(manifest, edges);
|
|
393
474
|
addBlockOutputLineageObjects(manifest, objects, edges);
|
|
394
|
-
|
|
475
|
+
// Manifest v3 is the unified DQL runtime. DataLex is migration input only;
|
|
476
|
+
// retaining it here would reintroduce a second analytical source of truth.
|
|
477
|
+
if (manifest.manifestVersion !== 3) {
|
|
478
|
+
addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagnostics);
|
|
479
|
+
}
|
|
395
480
|
const nodeKeyMap = new Map();
|
|
396
481
|
for (const node of [...manifestGraph.nodes, ...semanticGraph.nodes]) {
|
|
397
482
|
nodeKeyMap.set(node.nodeId, objectKeyFromKGNode(node));
|
|
@@ -643,23 +728,130 @@ export class MetadataCatalog {
|
|
|
643
728
|
});
|
|
644
729
|
txn();
|
|
645
730
|
}
|
|
731
|
+
/**
|
|
732
|
+
* Incremental reindex (W3.4). Only re-inserts objects + FTS rows for sources
|
|
733
|
+
* whose per-source fingerprint changed (or vanished); unchanged sources' rows are
|
|
734
|
+
* left in place (skipping the expensive FTS re-tokenization at scale). Edges,
|
|
735
|
+
* diagnostics, domain shards, and sourceless objects are always rebuilt because
|
|
736
|
+
* they cross source boundaries. Provably equal to a full rebuild for the semantic
|
|
737
|
+
* content (object keys/types/names/domains/payloads + FTS + edges); only the
|
|
738
|
+
* `updated_at`/`built_at` timestamps of untouched rows differ. Falls back to a
|
|
739
|
+
* full rebuild when there is no prior fingerprint state.
|
|
740
|
+
*/
|
|
741
|
+
rebuildIncremental(snapshot) {
|
|
742
|
+
const stored = new Map();
|
|
743
|
+
for (const row of this.db.prepare('SELECT source_path, fingerprint FROM metadata_source_fingerprints').all()) {
|
|
744
|
+
stored.set(row.source_path, row.fingerprint);
|
|
745
|
+
}
|
|
746
|
+
if (stored.size === 0) {
|
|
747
|
+
this.rebuild(snapshot);
|
|
748
|
+
return { mode: 'full', changedSources: 0 };
|
|
749
|
+
}
|
|
750
|
+
const now = new Date().toISOString();
|
|
751
|
+
const incoming = buildSourceFingerprints(snapshot.objects, now);
|
|
752
|
+
const incomingSources = new Set(incoming.map((f) => f.sourcePath));
|
|
753
|
+
const changedSources = new Set();
|
|
754
|
+
for (const fingerprint of incoming) {
|
|
755
|
+
if (stored.get(fingerprint.sourcePath) !== fingerprint.fingerprint)
|
|
756
|
+
changedSources.add(fingerprint.sourcePath);
|
|
757
|
+
}
|
|
758
|
+
const removedSources = [...stored.keys()].filter((source) => !incomingSources.has(source));
|
|
759
|
+
const sourceOf = (object) => object.sourcePath ?? object.sourceSystem ?? null;
|
|
760
|
+
const domainShards = buildDomainShards(snapshot.objects, now);
|
|
761
|
+
const insertObject = this.db.prepare(`
|
|
762
|
+
INSERT INTO metadata_objects (
|
|
763
|
+
object_key, object_type, name, full_name, domain, owner, status,
|
|
764
|
+
description, source_path, source_system, payload_json, updated_at
|
|
765
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
766
|
+
const insertFts = this.db.prepare(`
|
|
767
|
+
INSERT INTO metadata_fts (object_key, name, full_name, description, domain, owner, payload)
|
|
768
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
769
|
+
const insertEdge = this.db.prepare(`
|
|
770
|
+
INSERT OR IGNORE INTO metadata_edges (edge_type, from_key, to_key, confidence, payload_json, created_at)
|
|
771
|
+
VALUES (?, ?, ?, ?, ?, ?)`);
|
|
772
|
+
const insertDiagnostic = this.db.prepare(`
|
|
773
|
+
INSERT INTO metadata_diagnostics (id, kind, severity, message, object_key, file_path, created_at)
|
|
774
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
775
|
+
const insertSourceFingerprint = this.db.prepare(`
|
|
776
|
+
INSERT OR REPLACE INTO metadata_source_fingerprints (source_path, fingerprint, object_count, updated_at)
|
|
777
|
+
VALUES (?, ?, ?, ?)`);
|
|
778
|
+
const insertDomainShard = this.db.prepare(`
|
|
779
|
+
INSERT OR REPLACE INTO metadata_domain_shards (
|
|
780
|
+
domain, object_count, block_count, certified_block_count,
|
|
781
|
+
semantic_metric_count, dbt_object_count, updated_at
|
|
782
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
783
|
+
const setState = this.db.prepare('INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)');
|
|
784
|
+
const deleteFtsForSource = this.db.prepare('DELETE FROM metadata_fts WHERE object_key IN (SELECT object_key FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?)');
|
|
785
|
+
const deleteObjectsForSource = this.db.prepare('DELETE FROM metadata_objects WHERE COALESCE(source_path, source_system) = ?');
|
|
786
|
+
const txn = this.db.transaction(() => {
|
|
787
|
+
// Drop changed + removed sources' rows, and ALL sourceless rows (never
|
|
788
|
+
// fingerprinted, so always refreshed).
|
|
789
|
+
for (const source of [...changedSources, ...removedSources]) {
|
|
790
|
+
deleteFtsForSource.run(source);
|
|
791
|
+
deleteObjectsForSource.run(source);
|
|
792
|
+
}
|
|
793
|
+
this.db.prepare('DELETE FROM metadata_fts WHERE object_key IN (SELECT object_key FROM metadata_objects WHERE source_path IS NULL AND source_system IS NULL)').run();
|
|
794
|
+
this.db.prepare('DELETE FROM metadata_objects WHERE source_path IS NULL AND source_system IS NULL').run();
|
|
795
|
+
// Re-insert only objects from changed sources or sourceless; unchanged
|
|
796
|
+
// sources' rows are left untouched.
|
|
797
|
+
for (const object of snapshot.objects) {
|
|
798
|
+
const source = sourceOf(object);
|
|
799
|
+
if (source !== null && !changedSources.has(source))
|
|
800
|
+
continue;
|
|
801
|
+
const payload = object.payload ?? {};
|
|
802
|
+
insertObject.run(object.objectKey, object.objectType, object.name, object.fullName ?? null, object.domain ?? null, object.owner ?? null, object.status ?? null, object.description ?? null, object.sourcePath ?? null, object.sourceSystem ?? null, JSON.stringify(payload), object.updatedAt ?? now);
|
|
803
|
+
insertFts.run(object.objectKey, object.name, object.fullName ?? '', object.description ?? '', object.domain ?? '', object.owner ?? '', JSON.stringify(payload));
|
|
804
|
+
}
|
|
805
|
+
// Cross-source tables: rebuild fully.
|
|
806
|
+
this.db.prepare('DELETE FROM metadata_edges').run();
|
|
807
|
+
for (const edge of snapshot.edges) {
|
|
808
|
+
insertEdge.run(edge.edgeType, edge.fromKey, edge.toKey, edge.confidence ?? 1, JSON.stringify(edge.payload ?? {}), now);
|
|
809
|
+
}
|
|
810
|
+
this.db.prepare('DELETE FROM metadata_diagnostics').run();
|
|
811
|
+
for (const diagnostic of snapshot.diagnostics) {
|
|
812
|
+
insertDiagnostic.run(diagnosticId(diagnostic), diagnostic.kind, diagnostic.severity, diagnostic.message, diagnostic.objectKey ?? null, diagnostic.filePath ?? null, now);
|
|
813
|
+
}
|
|
814
|
+
this.db.prepare('DELETE FROM metadata_domain_shards').run();
|
|
815
|
+
for (const shard of domainShards) {
|
|
816
|
+
insertDomainShard.run(shard.domain, shard.objectCount, shard.blockCount, shard.certifiedBlockCount, shard.semanticMetricCount, shard.dbtObjectCount, shard.updatedAt);
|
|
817
|
+
}
|
|
818
|
+
for (const source of removedSources) {
|
|
819
|
+
this.db.prepare('DELETE FROM metadata_source_fingerprints WHERE source_path = ?').run(source);
|
|
820
|
+
}
|
|
821
|
+
for (const fingerprint of incoming) {
|
|
822
|
+
insertSourceFingerprint.run(fingerprint.sourcePath, fingerprint.fingerprint, fingerprint.objectCount, fingerprint.updatedAt);
|
|
823
|
+
}
|
|
824
|
+
setState.run('built_at', now);
|
|
825
|
+
setState.run('fingerprint', snapshot.fingerprint);
|
|
826
|
+
setState.run('project_root', snapshot.projectRoot);
|
|
827
|
+
setState.run('object_count', String(snapshot.objects.length));
|
|
828
|
+
setState.run('edge_count', String(snapshot.edges.length));
|
|
829
|
+
setState.run('source_fingerprint_count', String(incoming.length));
|
|
830
|
+
setState.run('domain_shard_count', String(domainShards.length));
|
|
831
|
+
setState.run('diagnostics_json', JSON.stringify(snapshot.diagnostics));
|
|
832
|
+
setState.run('compile_conflicts_json', JSON.stringify(snapshot.compileConflicts ?? []));
|
|
833
|
+
setState.run('manifest_generated_at', snapshot.manifest.generatedAt);
|
|
834
|
+
});
|
|
835
|
+
txn();
|
|
836
|
+
return { mode: 'incremental', changedSources: changedSources.size };
|
|
837
|
+
}
|
|
646
838
|
searchObjects(options) {
|
|
647
839
|
const { query, objectTypes, domain, limit = 40 } = options;
|
|
648
|
-
const
|
|
649
|
-
if (!
|
|
840
|
+
const match = buildFtsMatch(query, { prefix: true });
|
|
841
|
+
if (!match.or)
|
|
650
842
|
return this.listObjects({ objectTypes, domain, limit });
|
|
651
843
|
const filters = [];
|
|
652
|
-
const
|
|
844
|
+
const extraParams = [];
|
|
653
845
|
if (objectTypes && objectTypes.length > 0) {
|
|
654
846
|
filters.push(`o.object_type IN (${objectTypes.map(() => '?').join(', ')})`);
|
|
655
|
-
|
|
847
|
+
extraParams.push(...objectTypes);
|
|
656
848
|
}
|
|
657
849
|
if (domain) {
|
|
658
850
|
filters.push('o.domain = ?');
|
|
659
|
-
|
|
851
|
+
extraParams.push(domain);
|
|
660
852
|
}
|
|
661
853
|
const whereExtra = filters.length > 0 ? ` AND ${filters.join(' AND ')}` : '';
|
|
662
|
-
const
|
|
854
|
+
const runMatch = (matchExpr) => this.db.prepare(`
|
|
663
855
|
SELECT o.*,
|
|
664
856
|
bm25(metadata_fts) AS rank,
|
|
665
857
|
snippet(metadata_fts, -1, '<mark>', '</mark>', '...', 12) AS snip
|
|
@@ -668,7 +860,15 @@ export class MetadataCatalog {
|
|
|
668
860
|
WHERE metadata_fts MATCH ?${whereExtra}
|
|
669
861
|
ORDER BY rank
|
|
670
862
|
LIMIT ?
|
|
671
|
-
`).all(...
|
|
863
|
+
`).all(matchExpr, ...extraParams, limit);
|
|
864
|
+
// Precision-first, recall-preserving UNION: all-terms-co-occur (AND) matches
|
|
865
|
+
// lead, then OR-of-terms matches fill the remaining budget. A doc mentioning
|
|
866
|
+
// only one term (a relevant context column) still surfaces, while a doc
|
|
867
|
+
// containing every term ranks ahead of it. Dedup by object_key.
|
|
868
|
+
const andRows = match.and ? runMatch(match.and) : [];
|
|
869
|
+
const seen = new Set(andRows.map((row) => row.object_key));
|
|
870
|
+
const orRows = runMatch(match.or).filter((row) => !seen.has(row.object_key));
|
|
871
|
+
const rows = [...andRows, ...orRows].slice(0, limit);
|
|
672
872
|
return rows.map((row) => ({
|
|
673
873
|
...rowToObject(row),
|
|
674
874
|
score: row.rank ? 1 / (1 + Math.max(0, row.rank)) : 1,
|
|
@@ -839,6 +1039,14 @@ export class MetadataCatalog {
|
|
|
839
1039
|
`);
|
|
840
1040
|
const txn = this.db.transaction(() => {
|
|
841
1041
|
insertSnapshot.run(id, cleanSnapshot.source ?? null, JSON.stringify(cleanSnapshot), capturedAt);
|
|
1042
|
+
// Only the most-recent snapshot is ever read; rows otherwise accumulate one per
|
|
1043
|
+
// question forever. Keep a small margin and prune the rest (P7).
|
|
1044
|
+
this.db.prepare(`
|
|
1045
|
+
DELETE FROM runtime_schema_snapshots
|
|
1046
|
+
WHERE id NOT IN (
|
|
1047
|
+
SELECT id FROM runtime_schema_snapshots ORDER BY captured_at DESC LIMIT 5
|
|
1048
|
+
)
|
|
1049
|
+
`).run();
|
|
842
1050
|
this.db.prepare('DELETE FROM runtime_value_fts').run();
|
|
843
1051
|
this.db.prepare('DELETE FROM runtime_value_index').run();
|
|
844
1052
|
for (const entry of valueEntries) {
|
|
@@ -853,7 +1061,9 @@ export class MetadataCatalog {
|
|
|
853
1061
|
}
|
|
854
1062
|
searchRuntimeValues(terms, limit = 40) {
|
|
855
1063
|
const cleanTerms = uniqueStrings(terms.map(normalizeValueIndexText).filter(Boolean)).slice(0, 8);
|
|
856
|
-
|
|
1064
|
+
// Filter VALUES may legitimately be stop words (a category named "current");
|
|
1065
|
+
// fall back to raw tokens so value binding still fires rather than dropping to empty.
|
|
1066
|
+
const query = sanitizeFtsQuery(cleanTerms.join(' '), { prefix: true, fallbackToRawTokens: true });
|
|
857
1067
|
if (!query)
|
|
858
1068
|
return [];
|
|
859
1069
|
const rows = this.db.prepare(`
|
|
@@ -1028,6 +1238,7 @@ function objectFromKGNode(node) {
|
|
|
1028
1238
|
dimensions: node.dimensions ?? [],
|
|
1029
1239
|
allowedFilters: node.allowedFilters ?? [],
|
|
1030
1240
|
parameterPolicy: node.parameterPolicy ?? [],
|
|
1241
|
+
parameters: node.parameters ?? [],
|
|
1031
1242
|
filterBindings: node.filterBindings ?? [],
|
|
1032
1243
|
sourceSystems: node.sourceSystems ?? [],
|
|
1033
1244
|
replacementFor: node.replacementFor ?? [],
|
|
@@ -1107,6 +1318,71 @@ function normalizeEdge(edge, fromKey, toKey) {
|
|
|
1107
1318
|
payload: { kgSource: edge.src, kgTarget: edge.dst },
|
|
1108
1319
|
};
|
|
1109
1320
|
}
|
|
1321
|
+
/**
|
|
1322
|
+
* Skills are source-owned context, not a prompt-only side channel. Persist their
|
|
1323
|
+
* parsed fields in the catalog snapshot so selection, provenance, and
|
|
1324
|
+
* fingerprinting all observe the same immutable project state.
|
|
1325
|
+
*/
|
|
1326
|
+
function addSkillObjects(skills, objects, edges) {
|
|
1327
|
+
for (const skill of skills) {
|
|
1328
|
+
const identity = skill.qualifiedId ?? skill.id;
|
|
1329
|
+
const objectKey = `skill:${identity}`;
|
|
1330
|
+
const domains = uniqueNonBlank([skill.domain, ...(skill.domains ?? [])]);
|
|
1331
|
+
const object = {
|
|
1332
|
+
objectKey,
|
|
1333
|
+
objectType: 'skill',
|
|
1334
|
+
name: skill.id,
|
|
1335
|
+
fullName: skill.qualifiedId ?? skill.id,
|
|
1336
|
+
domain: skill.domain,
|
|
1337
|
+
owner: skill.owner,
|
|
1338
|
+
status: skill.status ?? 'active',
|
|
1339
|
+
description: skill.description ?? compactSkillDescription(skill.body),
|
|
1340
|
+
sourcePath: skill.sourcePath,
|
|
1341
|
+
sourceSystem: 'DQL domain skill',
|
|
1342
|
+
payload: compactObject({
|
|
1343
|
+
skillId: skill.id,
|
|
1344
|
+
localId: skill.localId ?? skill.id,
|
|
1345
|
+
qualifiedId: skill.qualifiedId,
|
|
1346
|
+
scope: skill.scope,
|
|
1347
|
+
user: skill.user,
|
|
1348
|
+
domains,
|
|
1349
|
+
modelAreaRefs: skill.modelAreaRefs ?? [],
|
|
1350
|
+
kind: skill.kind,
|
|
1351
|
+
triggers: skill.triggers ?? [],
|
|
1352
|
+
exclusions: skill.exclusions ?? [],
|
|
1353
|
+
preferredMetrics: skill.preferredMetrics,
|
|
1354
|
+
preferredBlocks: skill.preferredBlocks,
|
|
1355
|
+
preferredDimensions: skill.preferredDimensions ?? [],
|
|
1356
|
+
requiredFilters: skill.requiredFilters ?? [],
|
|
1357
|
+
clarifyWhen: skill.clarifyWhen ?? [],
|
|
1358
|
+
examples: skill.examples ?? [],
|
|
1359
|
+
sourceRefs: skill.sourceRefs ?? [],
|
|
1360
|
+
vocabulary: skill.vocabulary,
|
|
1361
|
+
body: skill.body,
|
|
1362
|
+
isStarter: skill.isStarter,
|
|
1363
|
+
provenance: 'DQL domain skill',
|
|
1364
|
+
}),
|
|
1365
|
+
};
|
|
1366
|
+
objects.set(objectKey, object);
|
|
1367
|
+
for (const domain of domains) {
|
|
1368
|
+
const edge = {
|
|
1369
|
+
edgeType: 'contains',
|
|
1370
|
+
fromKey: `domain:${domain}`,
|
|
1371
|
+
toKey: objectKey,
|
|
1372
|
+
confidence: 1,
|
|
1373
|
+
payload: { provenance: 'DQL domain skill' },
|
|
1374
|
+
};
|
|
1375
|
+
edges.set(`${edge.edgeType}\u0000${edge.fromKey}\u0000${edge.toKey}`, edge);
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
1379
|
+
function compactSkillDescription(body) {
|
|
1380
|
+
const normalized = body.replace(/\s+/g, ' ').trim();
|
|
1381
|
+
return normalized ? normalized.slice(0, 480) : undefined;
|
|
1382
|
+
}
|
|
1383
|
+
function uniqueNonBlank(values) {
|
|
1384
|
+
return Array.from(new Set(values.map((value) => value?.trim()).filter((value) => Boolean(value))));
|
|
1385
|
+
}
|
|
1110
1386
|
function addDbtDagObjects(manifest, objects, edges, diagnostics) {
|
|
1111
1387
|
const models = manifest.dbtImport?.dbtDag?.models ?? [];
|
|
1112
1388
|
for (const model of models) {
|
|
@@ -1197,7 +1473,20 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
|
|
|
1197
1473
|
return;
|
|
1198
1474
|
}
|
|
1199
1475
|
const catalogColumns = loadRawDbtCatalogColumns(join(dirname(manifestPath), 'catalog.json'));
|
|
1476
|
+
const uniqueColumnsByModel = extractDbtUniqueColumns(raw.nodes ?? {});
|
|
1200
1477
|
const entries = [];
|
|
1478
|
+
// PERF-001: at enterprise scale, keep columns in their model payload and
|
|
1479
|
+
// load full node detail lazily from the immutable dbt artifact cache. A
|
|
1480
|
+
// separate FTS row + graph edge for every column makes a 300k-column project
|
|
1481
|
+
// consume gigabytes without improving the bounded model-first route.
|
|
1482
|
+
const rawEntries = [...Object.values(raw.nodes ?? {}), ...Object.values(raw.sources ?? {})];
|
|
1483
|
+
const totalColumnCount = rawEntries.reduce((sum, node) => {
|
|
1484
|
+
const columns = node.columns;
|
|
1485
|
+
return sum + (columns && typeof columns === 'object' && !Array.isArray(columns)
|
|
1486
|
+
? Object.keys(columns).length
|
|
1487
|
+
: 0);
|
|
1488
|
+
}, 0);
|
|
1489
|
+
const includeColumnObjects = totalColumnCount <= 50_000;
|
|
1201
1490
|
for (const [uniqueId, node] of Object.entries(raw.nodes ?? {})) {
|
|
1202
1491
|
if (node.resource_type !== 'model')
|
|
1203
1492
|
continue;
|
|
@@ -1216,12 +1505,14 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
|
|
|
1216
1505
|
database,
|
|
1217
1506
|
schema,
|
|
1218
1507
|
catalogColumns: catalogColumns.get(uniqueId),
|
|
1508
|
+
uniqueColumns: uniqueColumnsByModel.get(name),
|
|
1219
1509
|
};
|
|
1220
1510
|
entries.push(entry);
|
|
1221
1511
|
addRawDbtCatalogObject({
|
|
1222
1512
|
...entry,
|
|
1223
1513
|
objects,
|
|
1224
1514
|
edges,
|
|
1515
|
+
includeColumnObjects,
|
|
1225
1516
|
});
|
|
1226
1517
|
}
|
|
1227
1518
|
for (const [uniqueId, source] of Object.entries(raw.sources ?? {})) {
|
|
@@ -1246,6 +1537,7 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
|
|
|
1246
1537
|
...entry,
|
|
1247
1538
|
objects,
|
|
1248
1539
|
edges,
|
|
1540
|
+
includeColumnObjects,
|
|
1249
1541
|
});
|
|
1250
1542
|
}
|
|
1251
1543
|
const relationLookup = buildRawDbtRelationLookup(entries);
|
|
@@ -1285,38 +1577,41 @@ function addRawDbtCatalogObject(input) {
|
|
|
1285
1577
|
catalogOnly: existing ? undefined : true,
|
|
1286
1578
|
columnCompleteness,
|
|
1287
1579
|
columns,
|
|
1580
|
+
// W5.3 — single-column `unique` dbt tests become grain-ledger keys.
|
|
1581
|
+
uniqueColumns: input.uniqueColumns && input.uniqueColumns.length > 0 ? input.uniqueColumns : undefined,
|
|
1288
1582
|
}),
|
|
1289
1583
|
}));
|
|
1290
|
-
|
|
1291
|
-
const
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1584
|
+
if (input.includeColumnObjects)
|
|
1585
|
+
for (const column of columns) {
|
|
1586
|
+
const columnKey = `dbt:column:${input.name}.${column.name}`;
|
|
1587
|
+
const existingColumn = input.objects.get(columnKey);
|
|
1588
|
+
input.objects.set(columnKey, mergeObject(existingColumn, {
|
|
1589
|
+
objectKey: columnKey,
|
|
1590
|
+
objectType: 'dbt_column',
|
|
1591
|
+
name: column.name,
|
|
1592
|
+
fullName: `${input.name}.${column.name}`,
|
|
1593
|
+
description: column.description,
|
|
1594
|
+
status: existingColumn?.status ?? 'dbt_catalog',
|
|
1595
|
+
sourcePath: stringValue(input.node.original_file_path) ?? stringValue(input.node.path),
|
|
1596
|
+
sourceSystem: existingColumn?.sourceSystem ?? 'dbt manifest.json catalog',
|
|
1597
|
+
payload: compactObject({
|
|
1598
|
+
...(existingColumn?.payload ?? {}),
|
|
1599
|
+
model: input.name,
|
|
1600
|
+
uniqueId: input.uniqueId,
|
|
1601
|
+
type: column.type,
|
|
1602
|
+
relation,
|
|
1603
|
+
columnCompleteness: 'partial',
|
|
1604
|
+
catalogOnly: existingColumn ? undefined : true,
|
|
1605
|
+
}),
|
|
1606
|
+
}));
|
|
1607
|
+
putEdge(input.edges, {
|
|
1608
|
+
edgeType: 'contains',
|
|
1609
|
+
fromKey: input.objectKey,
|
|
1610
|
+
toKey: columnKey,
|
|
1611
|
+
confidence: 1,
|
|
1612
|
+
payload: { source: 'raw dbt manifest column catalog' },
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1320
1615
|
for (const dep of rawDbtDependsOn(input.node)) {
|
|
1321
1616
|
putEdge(input.edges, {
|
|
1322
1617
|
edgeType: 'depends_on',
|
|
@@ -1537,6 +1832,7 @@ function addManifestBlockDetails(manifest, objects) {
|
|
|
1537
1832
|
blockType: block.blockType,
|
|
1538
1833
|
tests: block.tests,
|
|
1539
1834
|
parameterPolicy: block.parameterPolicy,
|
|
1835
|
+
parameters: block.parameters,
|
|
1540
1836
|
filterBindings: block.filterBindings,
|
|
1541
1837
|
sqlFingerprints: buildBlockSqlFingerprints(block.sql),
|
|
1542
1838
|
businessFingerprint: buildBlockBusinessFingerprint({
|
|
@@ -1659,6 +1955,10 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
|
|
|
1659
1955
|
payload: compactObject({
|
|
1660
1956
|
tags: entity.tags ?? [],
|
|
1661
1957
|
binding: entity.binding,
|
|
1958
|
+
// Entity-level grain + keys (W5.2) feed the grain ledger for fan-out safety.
|
|
1959
|
+
grain: entity.grain,
|
|
1960
|
+
candidateKeys: entity.candidate_keys,
|
|
1961
|
+
businessKeys: entity.business_keys,
|
|
1662
1962
|
fields: (entity.fields ?? []).slice(0, 100).map((field) => compactObject({
|
|
1663
1963
|
name: field.name,
|
|
1664
1964
|
type: field.type,
|
|
@@ -1744,6 +2044,58 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
|
|
|
1744
2044
|
}),
|
|
1745
2045
|
});
|
|
1746
2046
|
}
|
|
2047
|
+
// Concept-to-physical conformance (W5.1) → searchable `datalex_concept` nodes
|
|
2048
|
+
// carrying the canonical key + the physical relations that realize the concept,
|
|
2049
|
+
// plus `datalex_conforms` edges so retrieval can collapse several physical tables
|
|
2050
|
+
// into one business concept ("Customer" → dim_customer, stg_customers) and reason
|
|
2051
|
+
// at the concept grain. Modeling metadata only — no certification coupling.
|
|
2052
|
+
for (const conformance of raw.conformance ?? []) {
|
|
2053
|
+
if (!conformance?.concept)
|
|
2054
|
+
continue;
|
|
2055
|
+
const conceptDomain = conformance.domain ?? '';
|
|
2056
|
+
const conceptKey = `datalex:concept:${conceptDomain}.${conformance.concept}`;
|
|
2057
|
+
const physicalRefs = (conformance.physical ?? [])
|
|
2058
|
+
.map((physical) => (physical.binding ? objectKeyForDataLexBinding(physical.binding) : undefined))
|
|
2059
|
+
.filter((key) => Boolean(key));
|
|
2060
|
+
const canonicalKey = conformance.canonical_key ?? [];
|
|
2061
|
+
objects.set(conceptKey, mergeObject(objects.get(conceptKey), {
|
|
2062
|
+
objectKey: conceptKey,
|
|
2063
|
+
objectType: 'datalex_concept',
|
|
2064
|
+
name: conformance.concept,
|
|
2065
|
+
fullName: `${conceptDomain}.${conformance.concept}`,
|
|
2066
|
+
domain: conceptDomain || undefined,
|
|
2067
|
+
status: 'contract_evidence',
|
|
2068
|
+
description: `Business concept ${conformance.concept}`
|
|
2069
|
+
+ (canonicalKey.length ? ` (canonical key: ${canonicalKey.join(', ')})` : '')
|
|
2070
|
+
+ (physicalRefs.length ? `, realized by ${physicalRefs.length} physical relation(s).` : '.'),
|
|
2071
|
+
sourcePath: manifestPath,
|
|
2072
|
+
sourceSystem: 'DataLex manifest',
|
|
2073
|
+
payload: compactObject({
|
|
2074
|
+
canonicalKey,
|
|
2075
|
+
businessKey: conformance.business_key ?? [],
|
|
2076
|
+
implements: conformance.implements ?? [],
|
|
2077
|
+
physicalRefs,
|
|
2078
|
+
}),
|
|
2079
|
+
}));
|
|
2080
|
+
if (conceptDomain) {
|
|
2081
|
+
putEdge(edges, {
|
|
2082
|
+
edgeType: 'contains',
|
|
2083
|
+
fromKey: `datalex:domain:${conceptDomain}`,
|
|
2084
|
+
toKey: conceptKey,
|
|
2085
|
+
confidence: 1,
|
|
2086
|
+
payload: { source: 'datalex conformance' },
|
|
2087
|
+
});
|
|
2088
|
+
}
|
|
2089
|
+
for (const ref of physicalRefs) {
|
|
2090
|
+
putEdge(edges, {
|
|
2091
|
+
edgeType: 'datalex_conforms',
|
|
2092
|
+
fromKey: conceptKey,
|
|
2093
|
+
toKey: ref,
|
|
2094
|
+
confidence: 0.95,
|
|
2095
|
+
payload: compactObject({ source: 'datalex conformance', concept: conformance.concept, canonicalKey }),
|
|
2096
|
+
});
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
1747
2099
|
for (const block of Object.values(manifest.blocks ?? {})) {
|
|
1748
2100
|
if (!block.datalexContract)
|
|
1749
2101
|
continue;
|
|
@@ -2056,15 +2408,62 @@ async function planContextPackRoute(input) {
|
|
|
2056
2408
|
const exactByApplicability = [...applicabilityByKey.values()]
|
|
2057
2409
|
.filter((item) => item.kind === 'exact_answer' || item.kind === 'safe_parameterized')
|
|
2058
2410
|
.sort((a, b) => b.score - a.score)[0];
|
|
2059
|
-
|
|
2411
|
+
// Analytical questions intentionally keep certified applicability conservative:
|
|
2412
|
+
// a lexical/content match alone must not terminate the cascade. A certified
|
|
2413
|
+
// block whose declared/inferred output contract fully covers the requested
|
|
2414
|
+
// answer shape is different, however — that is a proven Tier 1 answer even
|
|
2415
|
+
// when the planner also knows how to generate SQL. Promote only exact/trim-safe
|
|
2416
|
+
// contract fits here; near matches remain context for Tier 2.
|
|
2417
|
+
const exactByContract = input.objects
|
|
2418
|
+
.filter((object) => object.objectType === 'dql_block'
|
|
2419
|
+
&& isCertifiedMetadataObject(object)
|
|
2420
|
+
&& hasCompatibleMetadataRankingDirection(input.request.question, object))
|
|
2421
|
+
.map((object) => ({
|
|
2422
|
+
object,
|
|
2423
|
+
fit: evaluateCertifiedBlockFit({
|
|
2424
|
+
question: input.request.question,
|
|
2425
|
+
plan: input.questionPlan,
|
|
2426
|
+
block: object,
|
|
2427
|
+
exactExampleMatch: hasExactExampleQuestion(input.request.question, object),
|
|
2428
|
+
definitionLookup: intent === 'definition_lookup',
|
|
2429
|
+
}),
|
|
2430
|
+
applicabilityScore: applicabilityByKey.get(object.objectKey)?.score ?? 0,
|
|
2431
|
+
}))
|
|
2432
|
+
.filter(({ fit }) => (fit.kind === 'exact' || fit.kind === 'trim_safe')
|
|
2433
|
+
&& (fit.confidence === 'high' || fit.confidence === 'medium'))
|
|
2434
|
+
.sort((a, b) => (b.fit.confidence === 'high' ? 1 : 0) - (a.fit.confidence === 'high' ? 1 : 0)
|
|
2435
|
+
|| b.applicabilityScore - a.applicabilityScore)[0]?.object;
|
|
2436
|
+
let exact = exactByApplicability
|
|
2060
2437
|
? input.objects.find((object) => object.objectKey === exactByApplicability.objectKey)
|
|
2061
|
-
: findExactCertifiedObject(input.request.question, intent, input.objects);
|
|
2062
|
-
const certifiedApplicability = exact
|
|
2063
|
-
? applicabilityByKey.get(exact.objectKey) ?? certifiedApplicabilityForObject(exact, input.questionPlan)
|
|
2064
|
-
: undefined;
|
|
2438
|
+
: findExactCertifiedObject(input.request.question, intent, input.objects) ?? exactByContract;
|
|
2065
2439
|
const contextApplicability = [...applicabilityByKey.values()]
|
|
2066
2440
|
.filter((item) => item.kind === 'context_only')
|
|
2067
2441
|
.sort((a, b) => b.score - a.score)[0];
|
|
2442
|
+
// W2.1 — paraphrase promotion. When no block string/lexically matched, promote a
|
|
2443
|
+
// top context-only certified candidate whose example QUESTION the user paraphrased
|
|
2444
|
+
// (semantic cosine + direction) to the certified candidate. It still runs the full
|
|
2445
|
+
// shape/grain fit below (paraphraseExampleMatch is kept OUT of directCertifiedBypass,
|
|
2446
|
+
// the fit's exactExampleMatch, and the grain-gate skip) — so paraphrase never bypasses
|
|
2447
|
+
// grain the way a user naming the block directly does.
|
|
2448
|
+
let paraphraseExampleMatch = false;
|
|
2449
|
+
if (!exact && contextApplicability) {
|
|
2450
|
+
const candidate = input.objects.find((object) => object.objectKey === contextApplicability.objectKey);
|
|
2451
|
+
if (candidate && isCertifiedMetadataObject(candidate)
|
|
2452
|
+
&& await matchExampleParaphrase(input.request.question, candidate, envEmbeddingProvider())) {
|
|
2453
|
+
exact = candidate;
|
|
2454
|
+
paraphraseExampleMatch = true;
|
|
2455
|
+
}
|
|
2456
|
+
}
|
|
2457
|
+
// Keep the strongest context-only certified candidate attached to the route
|
|
2458
|
+
// so the contract and grain gates can explain exactly why it is demoted.
|
|
2459
|
+
// This does not promote a loose match: both gates still have to pass before
|
|
2460
|
+
// Tier 1 is possible.
|
|
2461
|
+
if (!exact && contextApplicability) {
|
|
2462
|
+
exact = input.objects.find((object) => object.objectKey === contextApplicability.objectKey);
|
|
2463
|
+
}
|
|
2464
|
+
const certifiedApplicability = exact
|
|
2465
|
+
? applicabilityByKey.get(exact.objectKey) ?? certifiedApplicabilityForObject(exact, input.questionPlan)
|
|
2466
|
+
: undefined;
|
|
2068
2467
|
const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
|
|
2069
2468
|
const directCertifiedBypass = Boolean(exact && (exactExampleMatch
|
|
2070
2469
|
|| intent === 'definition_lookup'
|
|
@@ -2127,6 +2526,8 @@ async function planContextPackRoute(input) {
|
|
|
2127
2526
|
|| intent === 'exact_certified_lookup'
|
|
2128
2527
|
|| intent === 'definition_lookup'
|
|
2129
2528
|
|| exactExampleMatch
|
|
2529
|
+
|| paraphraseExampleMatch
|
|
2530
|
+
|| (blockFit && certifiedFitAllowsTier1(blockFit) && hasCompatibleMetadataRankingDirection(input.request.question, exact))
|
|
2130
2531
|
|| (intent === 'ad_hoc_ranking' && objectNameInQuestion(input.request.question, exact)))) {
|
|
2131
2532
|
return {
|
|
2132
2533
|
route: 'certified',
|
|
@@ -2152,7 +2553,7 @@ async function planContextPackRoute(input) {
|
|
|
2152
2553
|
route: 'generated_sql',
|
|
2153
2554
|
intent: isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking',
|
|
2154
2555
|
reason: `Certified block "${exact.name}" is close but answers a different grain than the question, so it is context only and SQL is generated for the requested grain.`,
|
|
2155
|
-
routeReason: grainGate.reason
|
|
2556
|
+
routeReason: `${grainGate.reason}; routed to Tier 2 generated SQL`,
|
|
2156
2557
|
grainGate: grainGateInfo,
|
|
2157
2558
|
blockFit,
|
|
2158
2559
|
trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
|
|
@@ -2170,7 +2571,7 @@ async function planContextPackRoute(input) {
|
|
|
2170
2571
|
route: 'generated_sql',
|
|
2171
2572
|
intent: isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking',
|
|
2172
2573
|
reason: `Certified block "${exact.name}" is relevant but does not satisfy the requested answer shape, so it is context only and SQL is generated for the requested result.`,
|
|
2173
|
-
routeReason: blockFit.reasons.join('; ')
|
|
2574
|
+
routeReason: `${blockFit.reasons.join('; ')}; routed to Tier 2 generated SQL`,
|
|
2174
2575
|
grainGate: grainGateInfo,
|
|
2175
2576
|
blockFit,
|
|
2176
2577
|
trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
|
|
@@ -2523,7 +2924,8 @@ function hasCompatibleMetadataRankingDirection(question, object) {
|
|
|
2523
2924
|
const objectDirection = rankingDirection([
|
|
2524
2925
|
object.name,
|
|
2525
2926
|
object.description ?? '',
|
|
2526
|
-
|
|
2927
|
+
Array.isArray(object.payload?.tags) ? object.payload.tags.join(' ') : '',
|
|
2928
|
+
typeof object.payload?.sql === 'string' ? object.payload.sql : '',
|
|
2527
2929
|
].join(' '));
|
|
2528
2930
|
if (!objectDirection)
|
|
2529
2931
|
return true;
|
|
@@ -3006,6 +3408,55 @@ function isSqlParentObject(object) {
|
|
|
3006
3408
|
object.objectType === 'warehouse_table' ||
|
|
3007
3409
|
object.objectType === 'runtime_table';
|
|
3008
3410
|
}
|
|
3411
|
+
/** Column budget for a single relation in the prompt's allowed-SQL context. */
|
|
3412
|
+
const MAX_ALLOWED_SQL_COLUMNS = 120;
|
|
3413
|
+
/**
|
|
3414
|
+
* Relevance score for keeping a column when a wide relation must be truncated (W3.3).
|
|
3415
|
+
* Structural columns (join keys, time/grain) always score so joins and grain survive;
|
|
3416
|
+
* a column whose name overlaps the question scores highest so the needed column on a
|
|
3417
|
+
* 300-column table is not dropped just because it sits past the budget cut.
|
|
3418
|
+
*/
|
|
3419
|
+
function columnBudgetScore(name, questionTokens) {
|
|
3420
|
+
const lower = name.toLowerCase();
|
|
3421
|
+
let score = 0;
|
|
3422
|
+
if (/(^|_)(id|key)$/.test(lower) || lower === 'id')
|
|
3423
|
+
score += 3;
|
|
3424
|
+
if (/(date|time|_at$|_ts$|day|month|year|week|quarter)/.test(lower))
|
|
3425
|
+
score += 2;
|
|
3426
|
+
if (questionTokens) {
|
|
3427
|
+
for (const token of questionTokens) {
|
|
3428
|
+
if (token.length > 2 && lower.includes(token)) {
|
|
3429
|
+
score += 5;
|
|
3430
|
+
break;
|
|
3431
|
+
}
|
|
3432
|
+
}
|
|
3433
|
+
}
|
|
3434
|
+
return score;
|
|
3435
|
+
}
|
|
3436
|
+
/**
|
|
3437
|
+
* Cap a relation's column list to the prompt budget and downgrade completeness to
|
|
3438
|
+
* 'partial' when columns are dropped. Keeping a truncated relation 'complete' would
|
|
3439
|
+
* let column validation false-positive a valid column past the cut as unknown_column
|
|
3440
|
+
* (the >120-column latent bug, W1.4). When truncating, keep the columns most relevant
|
|
3441
|
+
* to the question + structural keys (W3.3) rather than the arbitrary first-N, then
|
|
3442
|
+
* restore original order among the survivors so prompt ordering is unchanged.
|
|
3443
|
+
*/
|
|
3444
|
+
function capAllowedSqlColumns(columns, completeness, questionTokens) {
|
|
3445
|
+
if (columns.length <= MAX_ALLOWED_SQL_COLUMNS) {
|
|
3446
|
+
return { columns, columnCompleteness: completeness };
|
|
3447
|
+
}
|
|
3448
|
+
const kept = columns
|
|
3449
|
+
.map((column, index) => ({ column, index, score: columnBudgetScore(column.name, questionTokens) }))
|
|
3450
|
+
.sort((a, b) => (b.score - a.score) || (a.index - b.index))
|
|
3451
|
+
.slice(0, MAX_ALLOWED_SQL_COLUMNS)
|
|
3452
|
+
.sort((a, b) => a.index - b.index)
|
|
3453
|
+
.map((entry) => entry.column);
|
|
3454
|
+
return { columns: kept, columnCompleteness: 'partial' };
|
|
3455
|
+
}
|
|
3456
|
+
/** Lowercased word tokens from the question, for column-relevance ranking. */
|
|
3457
|
+
function tokenizeQuestionForColumns(question) {
|
|
3458
|
+
return (question.toLowerCase().match(/[a-z0-9_]+/g) ?? []).filter((token) => token.length > 2);
|
|
3459
|
+
}
|
|
3009
3460
|
const FULL_CATALOG_RELATION_TYPES = ['dbt_model', 'dbt_source', 'warehouse_table', 'runtime_table'];
|
|
3010
3461
|
const FULL_CATALOG_COLUMN_TYPES = ['dbt_column', 'warehouse_column', 'runtime_column'];
|
|
3011
3462
|
/** Deep mode over a catalog this small hands the model everything instead of top-k. */
|
|
@@ -3032,7 +3483,7 @@ function collectFullCatalogObjects(catalog) {
|
|
|
3032
3483
|
return undefined;
|
|
3033
3484
|
return mergeObjects([...relations, ...columns]);
|
|
3034
3485
|
}
|
|
3035
|
-
function buildAllowedSqlContext(objects, edges) {
|
|
3486
|
+
export function buildAllowedSqlContext(objects, edges, questionTokens) {
|
|
3036
3487
|
const byRelation = new Map();
|
|
3037
3488
|
const objectsByKey = new Map(objects.map((object) => [object.objectKey, object]));
|
|
3038
3489
|
const addRelation = (relation) => {
|
|
@@ -3041,16 +3492,17 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
3041
3492
|
return;
|
|
3042
3493
|
const existing = byRelation.get(key);
|
|
3043
3494
|
if (!existing) {
|
|
3044
|
-
|
|
3495
|
+
const capped = capAllowedSqlColumns(dedupeRuntimeColumns(relation.columns), relation.columnCompleteness, questionTokens);
|
|
3496
|
+
byRelation.set(key, { ...relation, columns: capped.columns, columnCompleteness: capped.columnCompleteness });
|
|
3045
3497
|
return;
|
|
3046
3498
|
}
|
|
3047
|
-
const
|
|
3499
|
+
const capped = capAllowedSqlColumns(mergeRelationColumns(existing, relation), mergeRelationCompletenessForCatalog(existing, relation), questionTokens);
|
|
3048
3500
|
byRelation.set(key, {
|
|
3049
3501
|
...existing,
|
|
3050
3502
|
objectKey: existing.objectKey ?? relation.objectKey,
|
|
3051
3503
|
source: mergeRelationSources(existing.source, relation.source),
|
|
3052
|
-
columnCompleteness,
|
|
3053
|
-
columns:
|
|
3504
|
+
columnCompleteness: capped.columnCompleteness,
|
|
3505
|
+
columns: capped.columns,
|
|
3054
3506
|
});
|
|
3055
3507
|
};
|
|
3056
3508
|
for (const object of objects) {
|
|
@@ -3500,12 +3952,13 @@ function coalesceRelationAliases(relations) {
|
|
|
3500
3952
|
continue;
|
|
3501
3953
|
const target = targets[0];
|
|
3502
3954
|
const targetKey = normalizeRelationKey(target.relation);
|
|
3955
|
+
const capped = capAllowedSqlColumns(mergeRelationColumns(target, relation), mergeRelationCompletenessForCatalog(target, relation));
|
|
3503
3956
|
mergedByRelation.set(targetKey, {
|
|
3504
3957
|
...target,
|
|
3505
3958
|
objectKey: target.objectKey ?? relation.objectKey,
|
|
3506
3959
|
source: mergeRelationSources(target.source, relation.source),
|
|
3507
|
-
columnCompleteness:
|
|
3508
|
-
columns:
|
|
3960
|
+
columnCompleteness: capped.columnCompleteness,
|
|
3961
|
+
columns: capped.columns,
|
|
3509
3962
|
});
|
|
3510
3963
|
skipped.add(relationKey);
|
|
3511
3964
|
}
|
|
@@ -4273,10 +4726,39 @@ function safeJson(raw, fallback) {
|
|
|
4273
4726
|
return fallback;
|
|
4274
4727
|
}
|
|
4275
4728
|
}
|
|
4729
|
+
/**
|
|
4730
|
+
* W3.2 — domain-affinity ranking boost for MULTI-DOMAIN catalogs. At scale
|
|
4731
|
+
* (thousands of models across domains), a question about one domain can be crowded
|
|
4732
|
+
* out of the top-k by lexically-similar objects from other domains. This infers the
|
|
4733
|
+
* dominant domain from the highest-scoring candidates and gives same-domain objects
|
|
4734
|
+
* a small additive bonus. It is RECALL-PRESERVING — it only reorders; every
|
|
4735
|
+
* candidate stays in the ranking, and the bonus (10% of the top score) is small
|
|
4736
|
+
* enough that a much stronger cross-domain signal still wins. No-op for single-domain
|
|
4737
|
+
* catalogs (fewer than 2 domains carry positive score).
|
|
4738
|
+
*/
|
|
4739
|
+
export function applyDomainAffinityBoost(scored) {
|
|
4740
|
+
const domainScore = new Map();
|
|
4741
|
+
for (const item of scored) {
|
|
4742
|
+
const domain = item.row.domain;
|
|
4743
|
+
if (domain && item.score > 0)
|
|
4744
|
+
domainScore.set(domain, (domainScore.get(domain) ?? 0) + item.score);
|
|
4745
|
+
}
|
|
4746
|
+
if (domainScore.size < 2)
|
|
4747
|
+
return undefined;
|
|
4748
|
+
const [dominant] = [...domainScore.entries()].sort((a, b) => b[1] - a[1])[0];
|
|
4749
|
+
const maxScore = Math.max(0, ...scored.map((item) => item.score));
|
|
4750
|
+
const bonus = Number((0.1 * maxScore).toFixed(3));
|
|
4751
|
+
if (bonus <= 0)
|
|
4752
|
+
return undefined;
|
|
4753
|
+
for (const item of scored) {
|
|
4754
|
+
if (item.row.domain === dominant)
|
|
4755
|
+
item.score = Number((item.score + bonus).toFixed(3));
|
|
4756
|
+
}
|
|
4757
|
+
return dominant;
|
|
4758
|
+
}
|
|
4276
4759
|
function rankMetadataObjects(args) {
|
|
4277
4760
|
const terms = tokenize(args.question).slice(0, 12);
|
|
4278
|
-
const
|
|
4279
|
-
.map((row) => {
|
|
4761
|
+
const scored = mergeObjects(args.rows).map((row) => {
|
|
4280
4762
|
const baseScore = scoreMetadataObject(row, terms);
|
|
4281
4763
|
const planScore = args.questionPlan ? scoreMetadataObjectWithAnalysisPlan(row, args.questionPlan) : { score: 0, reasons: [] };
|
|
4282
4764
|
const score = Number((baseScore + planScore.score).toFixed(3));
|
|
@@ -4287,7 +4769,9 @@ function rankMetadataObjects(args) {
|
|
|
4287
4769
|
reason: selectionReason(row, score, planScore.reasons),
|
|
4288
4770
|
priorityTier: priorityTier(row),
|
|
4289
4771
|
};
|
|
4290
|
-
})
|
|
4772
|
+
});
|
|
4773
|
+
applyDomainAffinityBoost(scored);
|
|
4774
|
+
const ranked = scored
|
|
4291
4775
|
.sort((a, b) => b.score - a.score || objectPriority(a.row) - objectPriority(b.row) || a.row.name.localeCompare(b.row.name))
|
|
4292
4776
|
.map((item, index) => ({ ...item, rank: index + 1 }));
|
|
4293
4777
|
const selectedRanked = selectRankedMetadataObjects(ranked, args.limit);
|
|
@@ -4419,6 +4903,121 @@ function reasonForObject(row) {
|
|
|
4419
4903
|
return 'Published consumption context';
|
|
4420
4904
|
return 'Relevant project metadata';
|
|
4421
4905
|
}
|
|
4906
|
+
function filterMetadataObjectsByDomainContext(rows, context) {
|
|
4907
|
+
if (!context?.activeDomain)
|
|
4908
|
+
return rows;
|
|
4909
|
+
const domains = new Set([
|
|
4910
|
+
context.activeDomain,
|
|
4911
|
+
...context.ancestors,
|
|
4912
|
+
...context.allowedImports.map((item) => item.providerDomain),
|
|
4913
|
+
]);
|
|
4914
|
+
return rows.filter((row) => !row.domain || domains.has(row.domain));
|
|
4915
|
+
}
|
|
4916
|
+
function selectContextPackSkills(objects, question, context) {
|
|
4917
|
+
const byIdentity = new Map();
|
|
4918
|
+
const parsed = objects.flatMap((object) => {
|
|
4919
|
+
const skill = skillFromMetadataObject(object);
|
|
4920
|
+
if (!skill)
|
|
4921
|
+
return [];
|
|
4922
|
+
const identity = skill.qualifiedId ?? skill.id;
|
|
4923
|
+
byIdentity.set(identity, object);
|
|
4924
|
+
return [skill];
|
|
4925
|
+
});
|
|
4926
|
+
const domains = context?.activeDomain
|
|
4927
|
+
? [context.activeDomain, ...context.ancestors, ...context.allowedImports.map((item) => item.providerDomain)]
|
|
4928
|
+
: [];
|
|
4929
|
+
const selected = selectRelevantSkills(parsed, question, {
|
|
4930
|
+
domains,
|
|
4931
|
+
modelAreaIds: context?.modelAreaId ? [context.modelAreaId] : [],
|
|
4932
|
+
});
|
|
4933
|
+
return selected.flatMap((skill) => {
|
|
4934
|
+
const object = byIdentity.get(skill.qualifiedId ?? skill.id);
|
|
4935
|
+
return object ? [{ object, skill: localContextSkillFromParsed(skill, object) }] : [];
|
|
4936
|
+
});
|
|
4937
|
+
}
|
|
4938
|
+
function skillFromMetadataObject(object) {
|
|
4939
|
+
if (object.objectType !== 'skill')
|
|
4940
|
+
return null;
|
|
4941
|
+
const payload = object.payload ?? {};
|
|
4942
|
+
const scope = payload.scope === 'personal' ? 'personal' : 'project';
|
|
4943
|
+
const status = payload.status === 'draft' || payload.status === 'deprecated' || payload.status === 'active'
|
|
4944
|
+
? payload.status
|
|
4945
|
+
: object.status === 'draft' || object.status === 'deprecated' || object.status === 'active'
|
|
4946
|
+
? object.status
|
|
4947
|
+
: 'active';
|
|
4948
|
+
return {
|
|
4949
|
+
id: stringPayload(payload.skillId) ?? object.name,
|
|
4950
|
+
localId: stringPayload(payload.localId) ?? object.name,
|
|
4951
|
+
qualifiedId: stringPayload(payload.qualifiedId) ?? object.fullName,
|
|
4952
|
+
scope,
|
|
4953
|
+
user: stringPayload(payload.user),
|
|
4954
|
+
domain: object.domain,
|
|
4955
|
+
domains: stringArrayPayload(payload.domains),
|
|
4956
|
+
modelAreaRefs: stringArrayPayload(payload.modelAreaRefs),
|
|
4957
|
+
kind: skillKindPayload(payload.kind),
|
|
4958
|
+
status,
|
|
4959
|
+
owner: object.owner,
|
|
4960
|
+
triggers: stringArrayPayload(payload.triggers),
|
|
4961
|
+
exclusions: stringArrayPayload(payload.exclusions),
|
|
4962
|
+
description: object.description,
|
|
4963
|
+
preferredMetrics: stringArrayPayload(payload.preferredMetrics),
|
|
4964
|
+
preferredBlocks: stringArrayPayload(payload.preferredBlocks),
|
|
4965
|
+
preferredDimensions: stringArrayPayload(payload.preferredDimensions),
|
|
4966
|
+
requiredFilters: stringArrayPayload(payload.requiredFilters),
|
|
4967
|
+
clarifyWhen: stringArrayPayload(payload.clarifyWhen),
|
|
4968
|
+
examples: stringArrayPayload(payload.examples),
|
|
4969
|
+
sourceRefs: stringArrayPayload(payload.sourceRefs),
|
|
4970
|
+
vocabulary: stringRecordPayload(payload.vocabulary),
|
|
4971
|
+
body: stringPayload(payload.body) ?? '',
|
|
4972
|
+
sourcePath: object.sourcePath ?? '',
|
|
4973
|
+
isStarter: payload.isStarter === true,
|
|
4974
|
+
};
|
|
4975
|
+
}
|
|
4976
|
+
function localContextSkillFromParsed(skill, object) {
|
|
4977
|
+
const guidance = skill.body.slice(0, 4_000);
|
|
4978
|
+
return {
|
|
4979
|
+
objectKey: object.objectKey,
|
|
4980
|
+
id: skill.id,
|
|
4981
|
+
qualifiedId: skill.qualifiedId,
|
|
4982
|
+
domain: skill.domain,
|
|
4983
|
+
domains: [...(skill.domains ?? [])],
|
|
4984
|
+
modelAreaRefs: [...(skill.modelAreaRefs ?? [])],
|
|
4985
|
+
kind: skill.kind,
|
|
4986
|
+
status: skill.status,
|
|
4987
|
+
owner: skill.owner,
|
|
4988
|
+
description: skill.description,
|
|
4989
|
+
triggers: [...(skill.triggers ?? [])],
|
|
4990
|
+
exclusions: [...(skill.exclusions ?? [])],
|
|
4991
|
+
preferredMetrics: [...skill.preferredMetrics],
|
|
4992
|
+
preferredBlocks: [...skill.preferredBlocks],
|
|
4993
|
+
preferredDimensions: [...(skill.preferredDimensions ?? [])],
|
|
4994
|
+
requiredFilters: [...(skill.requiredFilters ?? [])],
|
|
4995
|
+
clarifyWhen: [...(skill.clarifyWhen ?? [])],
|
|
4996
|
+
vocabulary: { ...skill.vocabulary },
|
|
4997
|
+
guidance,
|
|
4998
|
+
guidanceTruncated: guidance.length < skill.body.length,
|
|
4999
|
+
sourceRefs: [...(skill.sourceRefs ?? [])],
|
|
5000
|
+
provenance: stringPayload(object.payload?.provenance) ?? object.sourceSystem ?? 'DQL domain skill',
|
|
5001
|
+
sourcePath: object.sourcePath,
|
|
5002
|
+
};
|
|
5003
|
+
}
|
|
5004
|
+
function stringPayload(value) {
|
|
5005
|
+
return typeof value === 'string' && value.trim() ? value : undefined;
|
|
5006
|
+
}
|
|
5007
|
+
function stringArrayPayload(value) {
|
|
5008
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === 'string' && Boolean(item.trim())) : [];
|
|
5009
|
+
}
|
|
5010
|
+
function stringRecordPayload(value) {
|
|
5011
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
5012
|
+
return {};
|
|
5013
|
+
return Object.fromEntries(Object.entries(value).filter((entry) => typeof entry[1] === 'string'));
|
|
5014
|
+
}
|
|
5015
|
+
function skillKindPayload(value) {
|
|
5016
|
+
return value === 'domain_reference' || value === 'metric_policy' || value === 'glossary'
|
|
5017
|
+
|| value === 'analysis_pattern' || value === 'sql_policy' || value === 'custom'
|
|
5018
|
+
? value
|
|
5019
|
+
: undefined;
|
|
5020
|
+
}
|
|
4422
5021
|
function mergeObjects(rows) {
|
|
4423
5022
|
const byKey = new Map();
|
|
4424
5023
|
for (const row of rows) {
|
|
@@ -4633,21 +5232,4 @@ function scoreText(value, terms) {
|
|
|
4633
5232
|
const lower = value.toLowerCase();
|
|
4634
5233
|
return terms.reduce((score, term) => score + (lower.includes(term) ? 1 : 0), 0);
|
|
4635
5234
|
}
|
|
4636
|
-
const STOP_WORDS = new Set([
|
|
4637
|
-
'a', 'about', 'after', 'all', 'an', 'and', 'any', 'are', 'as', 'at', 'be', 'by',
|
|
4638
|
-
'can', 'could', 'did', 'do', 'does', 'down', 'for', 'from', 'give', 'had', 'has',
|
|
4639
|
-
'have', 'how', 'i', 'if', 'in', 'into', 'is', 'it', 'me', 'most', 'my', 'of',
|
|
4640
|
-
'on', 'or', 'please', 'query', 'show', 'sql', 'than', 'that', 'the', 'their',
|
|
4641
|
-
'them', 'this', 'to', 'up', 'using', 'was', 'we', 'were', 'what', 'when',
|
|
4642
|
-
'where', 'which', 'who', 'why', 'with', 'would', 'you', 'your',
|
|
4643
|
-
]);
|
|
4644
|
-
function sanitizeFtsQuery(raw) {
|
|
4645
|
-
return raw
|
|
4646
|
-
.split(/\s+/)
|
|
4647
|
-
.map((term) => term.replace(/[^\p{L}\p{N}_]/gu, ''))
|
|
4648
|
-
.filter((term) => term.length > 1 && !STOP_WORDS.has(term.toLowerCase()))
|
|
4649
|
-
.slice(0, 48)
|
|
4650
|
-
.map((term) => `"${term}"*`)
|
|
4651
|
-
.join(' OR ');
|
|
4652
|
-
}
|
|
4653
5235
|
//# sourceMappingURL=catalog.js.map
|