@duckcodeailabs/dql-agent 1.6.31 → 1.6.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-run-engine.d.ts +42 -6
- package/dist/agent-run-engine.d.ts.map +1 -1
- package/dist/agent-run-engine.js +133 -79
- package/dist/agent-run-engine.js.map +1 -1
- package/dist/agent-run-gates.d.ts.map +1 -1
- package/dist/agent-run-gates.js +72 -5
- package/dist/agent-run-gates.js.map +1 -1
- package/dist/agent-run-planner.d.ts.map +1 -1
- package/dist/agent-run-planner.js +13 -0
- package/dist/agent-run-planner.js.map +1 -1
- package/dist/answer-loop.d.ts +110 -8
- package/dist/answer-loop.d.ts.map +1 -1
- package/dist/answer-loop.js +1679 -1299
- package/dist/answer-loop.js.map +1 -1
- package/dist/answer-shape.d.ts +16 -0
- package/dist/answer-shape.d.ts.map +1 -0
- package/dist/answer-shape.js +85 -0
- package/dist/answer-shape.js.map +1 -0
- package/dist/cascade/budgets.d.ts +101 -0
- package/dist/cascade/budgets.d.ts.map +1 -0
- package/dist/cascade/budgets.js +184 -0
- package/dist/cascade/budgets.js.map +1 -0
- package/dist/cascade/cascade.d.ts +86 -0
- package/dist/cascade/cascade.d.ts.map +1 -0
- package/dist/cascade/cascade.js +115 -0
- package/dist/cascade/cascade.js.map +1 -0
- package/dist/cascade/route-policy.d.ts +14 -0
- package/dist/cascade/route-policy.d.ts.map +1 -0
- package/dist/cascade/route-policy.js +48 -0
- package/dist/cascade/route-policy.js.map +1 -0
- package/dist/cascade/triage.d.ts +20 -0
- package/dist/cascade/triage.d.ts.map +1 -0
- package/dist/cascade/triage.js +25 -0
- package/dist/cascade/triage.js.map +1 -0
- package/dist/conversation/rolling-summary.d.ts +13 -0
- package/dist/conversation/rolling-summary.d.ts.map +1 -0
- package/dist/conversation/rolling-summary.js +43 -0
- package/dist/conversation/rolling-summary.js.map +1 -0
- package/dist/conversation/session-store.d.ts +110 -0
- package/dist/conversation/session-store.d.ts.map +1 -0
- package/dist/conversation/session-store.js +387 -0
- package/dist/conversation/session-store.js.map +1 -0
- package/dist/conversation/snapshot.d.ts +59 -0
- package/dist/conversation/snapshot.d.ts.map +1 -0
- package/dist/conversation/snapshot.js +153 -0
- package/dist/conversation/snapshot.js.map +1 -0
- package/dist/conversation/working-state.d.ts +49 -0
- package/dist/conversation/working-state.d.ts.map +1 -0
- package/dist/conversation/working-state.js +249 -0
- package/dist/conversation/working-state.js.map +1 -0
- package/dist/embeddings/provider.d.ts +62 -0
- package/dist/embeddings/provider.d.ts.map +1 -1
- package/dist/embeddings/provider.js +98 -0
- package/dist/embeddings/provider.js.map +1 -1
- package/dist/grounding/context-ledger.d.ts +30 -0
- package/dist/grounding/context-ledger.d.ts.map +1 -0
- package/dist/grounding/context-ledger.js +54 -0
- package/dist/grounding/context-ledger.js.map +1 -0
- package/dist/grounding/regrounding.d.ts +24 -0
- package/dist/grounding/regrounding.d.ts.map +1 -0
- package/dist/grounding/regrounding.js +219 -0
- package/dist/grounding/regrounding.js.map +1 -0
- package/dist/grounding/value-index.d.ts +32 -0
- package/dist/grounding/value-index.d.ts.map +1 -0
- package/dist/grounding/value-index.js +67 -0
- package/dist/grounding/value-index.js.map +1 -0
- package/dist/index.d.ts +45 -14
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +102 -11
- package/dist/index.js.map +1 -1
- package/dist/intent-controller.d.ts +7 -0
- package/dist/intent-controller.d.ts.map +1 -1
- package/dist/intent-controller.js +22 -1
- package/dist/intent-controller.js.map +1 -1
- package/dist/kg/build.d.ts.map +1 -1
- package/dist/kg/build.js +44 -17
- package/dist/kg/build.js.map +1 -1
- package/dist/kg/sqlite-fts.d.ts +3 -1
- package/dist/kg/sqlite-fts.d.ts.map +1 -1
- package/dist/kg/sqlite-fts.js +16 -2
- package/dist/kg/sqlite-fts.js.map +1 -1
- package/dist/kg/types.d.ts +23 -1
- package/dist/kg/types.d.ts.map +1 -1
- package/dist/memory/fts-query.d.ts +7 -0
- package/dist/memory/fts-query.d.ts.map +1 -0
- package/dist/memory/fts-query.js +36 -0
- package/dist/memory/fts-query.js.map +1 -0
- package/dist/memory/sqlite-memory.d.ts.map +1 -1
- package/dist/memory/sqlite-memory.js +1 -30
- package/dist/memory/sqlite-memory.js.map +1 -1
- package/dist/metadata/analysis-planner.d.ts +23 -0
- package/dist/metadata/analysis-planner.d.ts.map +1 -1
- package/dist/metadata/analysis-planner.js +432 -2
- package/dist/metadata/analysis-planner.js.map +1 -1
- package/dist/metadata/block-fit.d.ts +24 -0
- package/dist/metadata/block-fit.d.ts.map +1 -0
- package/dist/metadata/block-fit.js +373 -0
- package/dist/metadata/block-fit.js.map +1 -0
- package/dist/metadata/catalog.d.ts +72 -3
- package/dist/metadata/catalog.d.ts.map +1 -1
- package/dist/metadata/catalog.js +1452 -72
- package/dist/metadata/catalog.js.map +1 -1
- package/dist/metadata/drafts.d.ts +25 -0
- package/dist/metadata/drafts.d.ts.map +1 -1
- package/dist/metadata/drafts.js +112 -2
- package/dist/metadata/drafts.js.map +1 -1
- package/dist/metadata/metric-match.d.ts +55 -6
- package/dist/metadata/metric-match.d.ts.map +1 -1
- package/dist/metadata/metric-match.js +158 -13
- package/dist/metadata/metric-match.js.map +1 -1
- package/dist/metadata/sql-context-validation.d.ts +13 -1
- package/dist/metadata/sql-context-validation.d.ts.map +1 -1
- package/dist/metadata/sql-context-validation.js +172 -19
- package/dist/metadata/sql-context-validation.js.map +1 -1
- package/dist/metadata/sql-retrieval.d.ts +3 -2
- package/dist/metadata/sql-retrieval.d.ts.map +1 -1
- package/dist/metadata/sql-retrieval.js +6 -4
- package/dist/metadata/sql-retrieval.js.map +1 -1
- package/dist/providers/claude.d.ts +2 -1
- package/dist/providers/claude.d.ts.map +1 -1
- package/dist/providers/claude.js +95 -0
- package/dist/providers/claude.js.map +1 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/openai.d.ts +2 -1
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +143 -0
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/tool-output.d.ts +2 -0
- package/dist/providers/tool-output.d.ts.map +1 -0
- package/dist/providers/tool-output.js +22 -0
- package/dist/providers/tool-output.js.map +1 -0
- package/dist/providers/types.d.ts +23 -0
- package/dist/providers/types.d.ts.map +1 -1
- package/dist/providers/types.js.map +1 -1
- package/dist/semantic-bridge/compose.d.ts +89 -0
- package/dist/semantic-bridge/compose.d.ts.map +1 -0
- package/dist/semantic-bridge/compose.js +528 -0
- package/dist/semantic-bridge/compose.js.map +1 -0
- package/dist/semantic-bridge/member-select.d.ts +23 -0
- package/dist/semantic-bridge/member-select.d.ts.map +1 -0
- package/dist/semantic-bridge/member-select.js +106 -0
- package/dist/semantic-bridge/member-select.js.map +1 -0
- package/dist/skills/defaults.d.ts.map +1 -1
- package/dist/skills/defaults.js +5 -0
- package/dist/skills/defaults.js.map +1 -1
- package/dist/synthesize.d.ts +26 -0
- package/dist/synthesize.d.ts.map +1 -1
- package/dist/synthesize.js +72 -3
- package/dist/synthesize.js.map +1 -1
- package/dist/tools/registry.d.ts +15 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/registry.js +692 -0
- package/dist/tools/registry.js.map +1 -0
- package/dist/trust/stamp.d.ts +26 -0
- package/dist/trust/stamp.d.ts.map +1 -0
- package/dist/trust/stamp.js +33 -0
- package/dist/trust/stamp.js.map +1 -0
- package/package.json +4 -4
package/dist/metadata/catalog.js
CHANGED
|
@@ -10,12 +10,14 @@ import { createHash } from 'node:crypto';
|
|
|
10
10
|
import { mkdirSync, readFileSync } from 'node:fs';
|
|
11
11
|
import { createRequire } from 'node:module';
|
|
12
12
|
import { dirname, join } from 'node:path';
|
|
13
|
-
import { buildManifest, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, } from '@duckcodeailabs/dql-core';
|
|
13
|
+
import { buildManifest, loadProjectConfig, parseContractRef, resolveDataLexManifestPath, resolveDbtManifestPath, resolveSemanticLayerWithDiagnostics, extractColumnLineage, normalizeDqlArtifactReference, composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
|
|
14
14
|
import { buildKGFromManifest, buildKGFromSemanticLayer } from '../kg/build.js';
|
|
15
15
|
import { buildBlockBusinessFingerprint, buildBlockSqlFingerprints } from './block-fingerprints.js';
|
|
16
16
|
import { buildAnalysisQuestionPlan, certifiedApplicabilityForObject, scoreAllowedSqlRelationWithAnalysisPlan, scoreMetadataObjectWithAnalysisPlan, sortAllowedSqlContextForAnalysisPlan, } from './analysis-planner.js';
|
|
17
17
|
import { extractSimpleSelectShape, sourceSqlShapeColumns } from './sql-shape.js';
|
|
18
|
+
import { buildRuntimeValueIndex, normalizeValueIndexText, } from '../grounding/value-index.js';
|
|
18
19
|
import { grainMatches, requestedGrainFromPlan, } from './grain-gate.js';
|
|
20
|
+
import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit, } from './block-fit.js';
|
|
19
21
|
import { retrieveScopedHints } from '../hints/retrieval.js';
|
|
20
22
|
const require = createRequire(import.meta.url);
|
|
21
23
|
let databaseCtor = null;
|
|
@@ -23,8 +25,11 @@ function loadDatabase() {
|
|
|
23
25
|
databaseCtor ??= require('better-sqlite3');
|
|
24
26
|
return databaseCtor;
|
|
25
27
|
}
|
|
28
|
+
const SCHEMA_SHAPE_CACHE_LIMIT = 64;
|
|
29
|
+
const schemaShapeCandidateCache = new Map();
|
|
26
30
|
const OBJECT_PRIORITY = {
|
|
27
31
|
dql_block: 1,
|
|
32
|
+
dql_block_output: 1.5,
|
|
28
33
|
semantic_metric: 2,
|
|
29
34
|
dql_term: 3,
|
|
30
35
|
business_view: 4,
|
|
@@ -36,11 +41,14 @@ const OBJECT_PRIORITY = {
|
|
|
36
41
|
dbt_source: 10,
|
|
37
42
|
dbt_column: 11,
|
|
38
43
|
warehouse_table: 12,
|
|
44
|
+
warehouse_column: 12.5,
|
|
45
|
+
runtime_value: 12.2,
|
|
39
46
|
notebook: 13,
|
|
40
47
|
dashboard: 14,
|
|
41
48
|
app: 15,
|
|
42
49
|
domain: 16,
|
|
43
50
|
};
|
|
51
|
+
const COLUMN_OBJECT_TYPES = new Set(['dbt_column', 'warehouse_column', 'runtime_column', 'runtime_value']);
|
|
44
52
|
export function defaultMetadataPath(projectRoot) {
|
|
45
53
|
return join(projectRoot, '.dql', 'cache', 'metadata.sqlite');
|
|
46
54
|
}
|
|
@@ -104,8 +112,54 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
104
112
|
const mode = request.mode ?? 'question';
|
|
105
113
|
const followUp = normalizeFollowUpContext(request.followUp);
|
|
106
114
|
const questionPlan = buildAnalysisQuestionPlan(request.question, followUp ?? undefined);
|
|
115
|
+
// ── Conversation-aware context reuse ──────────────────────────────────
|
|
116
|
+
// The prior turn's pack is fuel: a filter/limit-only REFINEMENT re-stamps it
|
|
117
|
+
// (skipping FTS fan-out, both ranking passes, route planning, and the
|
|
118
|
+
// fit-confirm LLM); CONTINUATION/RETURN seed its objects into ranking as
|
|
119
|
+
// candidates only; a SHIFT ignores it. A metadata fingerprint mismatch
|
|
120
|
+
// always disqualifies reuse.
|
|
121
|
+
const reusePolicy = request.reusePolicy ?? 'seed';
|
|
122
|
+
const priorPack = reusePolicy !== 'off'
|
|
123
|
+
&& request.priorContextPackId
|
|
124
|
+
&& request.conversationTopicRelation
|
|
125
|
+
&& request.conversationTopicRelation !== 'shift'
|
|
126
|
+
? catalog.getContextPack(request.priorContextPackId)
|
|
127
|
+
: null;
|
|
128
|
+
const priorPackFresh = Boolean(priorPack && priorPack.freshness.fingerprint === catalog.state('fingerprint'));
|
|
129
|
+
if (priorPack
|
|
130
|
+
&& priorPackFresh
|
|
131
|
+
&& request.conversationTopicRelation === 'refinement'
|
|
132
|
+
&& isFilterOnlyRefinement(priorPack.questionPlan, questionPlan)) {
|
|
133
|
+
// Route commitment: the prior route decision was already fit-validated for
|
|
134
|
+
// this grain/measures/dimensions; only filters/limit/timeframe changed.
|
|
135
|
+
const reusedPayload = {
|
|
136
|
+
...priorPack,
|
|
137
|
+
question: request.question,
|
|
138
|
+
questionPlan,
|
|
139
|
+
followUp: followUp ?? undefined,
|
|
140
|
+
retrievalDiagnostics: {
|
|
141
|
+
...priorPack.retrievalDiagnostics,
|
|
142
|
+
strategy: 'reused_pack_refinement',
|
|
143
|
+
},
|
|
144
|
+
freshness: {
|
|
145
|
+
catalogPath: defaultMetadataPath(projectRoot),
|
|
146
|
+
builtAt: catalog.state('built_at'),
|
|
147
|
+
fingerprint: catalog.state('fingerprint'),
|
|
148
|
+
},
|
|
149
|
+
};
|
|
150
|
+
delete reusedPayload.id;
|
|
151
|
+
const reusedId = catalog.insertContextPack(reusedPayload);
|
|
152
|
+
return { ...reusedPayload, id: reusedId };
|
|
153
|
+
}
|
|
154
|
+
const priorSeedObjects = priorPack && priorPackFresh
|
|
155
|
+
&& (request.conversationTopicRelation === 'continuation'
|
|
156
|
+
|| request.conversationTopicRelation === 'refinement'
|
|
157
|
+
|| request.conversationTopicRelation === 'return')
|
|
158
|
+
? priorPack.objects.slice(0, 24)
|
|
159
|
+
: [];
|
|
107
160
|
const runtimeSnapshot = request.runtimeSchemaSnapshot ?? catalog.latestRuntimeSchemaSnapshot();
|
|
108
161
|
const runtimeObjects = runtimeSnapshot ? runtimeSchemaObjects(runtimeSnapshot) : [];
|
|
162
|
+
const runtimeValueObjects = runtimeValueMatchObjects(catalog.searchRuntimeValues(metadataValueSearchTerms(request.question, questionPlan, followUp), 32));
|
|
109
163
|
const selectedObjects = selectedContextObjects(request.selectedContext);
|
|
110
164
|
const followUpObjects = followUpContextObjects(followUp);
|
|
111
165
|
const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
|
|
@@ -118,38 +172,56 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
118
172
|
objectTypes: request.objectTypes,
|
|
119
173
|
limit: Math.max(request.limit ?? 80, 20),
|
|
120
174
|
})));
|
|
121
|
-
const schemaShapeCandidates = schemaShapeCandidateObjects(catalog, questionPlan, request, runtimeObjects);
|
|
175
|
+
const schemaShapeCandidates = schemaShapeCandidateObjects(catalog, questionPlan, request, mergeObjects([...runtimeObjects, ...runtimeValueObjects]));
|
|
122
176
|
const schemaShapeObjects = schemaShapeCandidates.map((candidate) => candidate.object);
|
|
123
177
|
const exact = request.focusObjectKey ? catalog.getObject(request.focusObjectKey) : null;
|
|
124
178
|
const ranked = rankMetadataObjects({
|
|
125
179
|
rows: mergeObjects(exact
|
|
126
|
-
? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...selectedObjects]
|
|
127
|
-
: [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...selectedObjects]),
|
|
180
|
+
? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]
|
|
181
|
+
: [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects, ...priorSeedObjects]),
|
|
128
182
|
question: searchQueries.join(' '),
|
|
129
183
|
questionPlan,
|
|
130
184
|
limit: request.limit ?? 80,
|
|
131
185
|
});
|
|
132
|
-
|
|
186
|
+
// Advisory 'contextual' carry: the prior turn's block competes on rank (it is
|
|
187
|
+
// already in the candidate rows above) but is never FORCED into selection or
|
|
188
|
+
// allowed to become the derived focus of a possibly-new-topic question.
|
|
189
|
+
const selected = followUp?.kind === 'contextual'
|
|
190
|
+
? ranked.selected
|
|
191
|
+
: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...ranked.selected]);
|
|
133
192
|
const focusObjectKey = request.focusObjectKey ?? selected[0]?.objectKey ?? null;
|
|
134
193
|
const edgeWalk = catalog.edgesForKeys(selected.map((row) => row.objectKey), 3);
|
|
135
194
|
const edgeObjectKeys = Array.from(new Set(edgeWalk.flatMap((edge) => [edge.fromKey, edge.toKey])));
|
|
136
195
|
const graphObjects = catalog.getObjectsByKeys(edgeObjectKeys);
|
|
137
196
|
const rankedObjects = rankMetadataObjects({
|
|
138
|
-
rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...selectedObjects]),
|
|
197
|
+
rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...schemaShapeObjects, ...runtimeObjects, ...runtimeValueObjects, ...selectedObjects]),
|
|
139
198
|
question: searchQueries.join(' '),
|
|
140
199
|
questionPlan,
|
|
141
200
|
limit: request.limit ?? 120,
|
|
142
201
|
}).selected;
|
|
143
202
|
const sqlParentObjects = sqlParentObjectsForSelectedColumns(rankedObjects, mergeObjects([...graphObjects, ...runtimeObjects]), questionPlan);
|
|
144
|
-
|
|
203
|
+
// Deep-research over a SMALL catalog: skip top-k pruning and hand the model the
|
|
204
|
+
// entire relation set ("send everything, let the agent decide"). Only when the
|
|
205
|
+
// whole catalog fits comfortably in context — otherwise we keep ranked selection.
|
|
206
|
+
const fullCatalogObjects = request.strictness === 'exploratory'
|
|
207
|
+
? collectFullCatalogObjects(catalog)
|
|
208
|
+
: undefined;
|
|
209
|
+
const usedFullCatalog = Boolean(fullCatalogObjects);
|
|
210
|
+
const objects = fullCatalogObjects
|
|
211
|
+
? mergeObjects([...rankedObjects, ...sqlParentObjects, ...fullCatalogObjects])
|
|
212
|
+
: mergeObjects([...rankedObjects, ...sqlParentObjects]);
|
|
145
213
|
const objectKeys = objects.map((row) => row.objectKey);
|
|
214
|
+
const contextEdges = mergeMetadataEdges([
|
|
215
|
+
...edgeWalk,
|
|
216
|
+
...catalog.edgesForKeys(objectKeys, 2),
|
|
217
|
+
]);
|
|
146
218
|
const queryRuns = catalog.queryRunsForObjectKeys(objectKeys, 20);
|
|
147
219
|
const diagnostics = catalog.diagnostics();
|
|
148
220
|
const warnings = buildWarnings(diagnostics, objects);
|
|
149
221
|
const trustLabel = deriveTrust(objects);
|
|
150
|
-
const citations = buildCitations(objects,
|
|
151
|
-
const evidenceSummaries = buildEvidenceSummaries(objects,
|
|
152
|
-
const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects,
|
|
222
|
+
const citations = buildCitations(objects, contextEdges);
|
|
223
|
+
const evidenceSummaries = buildEvidenceSummaries(objects, contextEdges, queryRuns, diagnostics);
|
|
224
|
+
const allowedSqlContext = sortAllowedSqlContextForAnalysisPlan(buildAllowedSqlContext(objects, contextEdges), questionPlan);
|
|
153
225
|
const selectedRelations = allowedSqlContext.relations.slice(0, 24).map((relation, index) => {
|
|
154
226
|
const scored = scoreAllowedSqlRelationWithAnalysisPlan(relation, questionPlan);
|
|
155
227
|
return {
|
|
@@ -162,7 +234,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
162
234
|
rank: index + 1,
|
|
163
235
|
};
|
|
164
236
|
});
|
|
165
|
-
const selectedJoinPaths = buildSelectedJoinPaths(allowedSqlContext);
|
|
237
|
+
const selectedJoinPaths = buildSelectedJoinPaths(allowedSqlContext, contextEdges);
|
|
166
238
|
const evidenceRoles = buildEvidenceRoles(objects, queryRuns);
|
|
167
239
|
const reranked = rankMetadataObjects({
|
|
168
240
|
rows: mergeObjects([...searchRows, ...schemaShapeObjects, ...objects]),
|
|
@@ -172,7 +244,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
172
244
|
});
|
|
173
245
|
const conflicts = buildCandidateConflicts(reranked.ranked);
|
|
174
246
|
const compileConflicts = catalog.compileConflicts();
|
|
175
|
-
const routeDecision = planContextPackRoute({
|
|
247
|
+
const routeDecision = withMetadataTrustLabelInfo(await planContextPackRoute({
|
|
176
248
|
request,
|
|
177
249
|
objects,
|
|
178
250
|
allowedSqlContext,
|
|
@@ -182,6 +254,12 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
182
254
|
questionPlan,
|
|
183
255
|
compileConflicts,
|
|
184
256
|
rankedObjects: reranked.ranked,
|
|
257
|
+
}));
|
|
258
|
+
const certifiedCandidateFits = buildCertifiedCandidateFitDiagnostics({
|
|
259
|
+
request,
|
|
260
|
+
objects,
|
|
261
|
+
questionPlan,
|
|
262
|
+
routeDecision,
|
|
185
263
|
});
|
|
186
264
|
// Approved scoped correction hints — folded in AFTER certified routing so
|
|
187
265
|
// they never override a certified answer. On the `certified` route we still
|
|
@@ -199,8 +277,9 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
199
277
|
mode,
|
|
200
278
|
questionPlan,
|
|
201
279
|
trustLabel,
|
|
280
|
+
trustLabelInfo: metadataTrustLabelInfo(trustLabel),
|
|
202
281
|
objects,
|
|
203
|
-
edges:
|
|
282
|
+
edges: contextEdges,
|
|
204
283
|
queryRuns,
|
|
205
284
|
citations,
|
|
206
285
|
evidenceSummaries,
|
|
@@ -213,7 +292,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
213
292
|
appliedHints: hintResult.applied,
|
|
214
293
|
hintConflicts: hintResult.conflicts,
|
|
215
294
|
retrievalDiagnostics: {
|
|
216
|
-
strategy: 'sqlite_fts',
|
|
295
|
+
strategy: usedFullCatalog ? 'full_catalog' : 'sqlite_fts',
|
|
217
296
|
selectedObjects: objects.length,
|
|
218
297
|
selectedEvidence: reranked.ranked.slice(0, 20).map((item) => ({
|
|
219
298
|
objectKey: item.row.objectKey,
|
|
@@ -234,6 +313,7 @@ export async function buildLocalContextPack(projectRoot, request) {
|
|
|
234
313
|
columns: candidate.relation.columns.slice(0, 16).map((column) => column.name),
|
|
235
314
|
})),
|
|
236
315
|
topRejected: reranked.rejected,
|
|
316
|
+
certifiedCandidateFits,
|
|
237
317
|
candidateConflicts: conflicts,
|
|
238
318
|
},
|
|
239
319
|
freshness: {
|
|
@@ -310,6 +390,7 @@ export function buildMetadataSnapshot(projectRoot, manifest, semanticLayer) {
|
|
|
310
390
|
addDbtDagObjects(manifest, objects, edges, diagnostics);
|
|
311
391
|
addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges, diagnostics);
|
|
312
392
|
addBlockDependencyEdges(manifest, edges);
|
|
393
|
+
addBlockOutputLineageObjects(manifest, objects, edges);
|
|
313
394
|
addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagnostics);
|
|
314
395
|
const nodeKeyMap = new Map();
|
|
315
396
|
for (const node of [...manifestGraph.nodes, ...semanticGraph.nodes]) {
|
|
@@ -429,6 +510,30 @@ export class MetadataCatalog {
|
|
|
429
510
|
);
|
|
430
511
|
CREATE INDEX IF NOT EXISTS idx_runtime_schema_snapshots_captured ON runtime_schema_snapshots(captured_at DESC);
|
|
431
512
|
|
|
513
|
+
CREATE TABLE IF NOT EXISTS runtime_value_index (
|
|
514
|
+
value_key TEXT PRIMARY KEY,
|
|
515
|
+
relation TEXT NOT NULL,
|
|
516
|
+
schema_name TEXT,
|
|
517
|
+
table_name TEXT,
|
|
518
|
+
column_name TEXT NOT NULL,
|
|
519
|
+
column_type TEXT,
|
|
520
|
+
value TEXT NOT NULL,
|
|
521
|
+
normalized_value TEXT NOT NULL,
|
|
522
|
+
source TEXT,
|
|
523
|
+
captured_at TEXT NOT NULL
|
|
524
|
+
);
|
|
525
|
+
CREATE INDEX IF NOT EXISTS idx_runtime_value_index_relation ON runtime_value_index(relation, column_name);
|
|
526
|
+
CREATE INDEX IF NOT EXISTS idx_runtime_value_index_captured ON runtime_value_index(captured_at DESC);
|
|
527
|
+
|
|
528
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS runtime_value_fts USING fts5(
|
|
529
|
+
value_key UNINDEXED,
|
|
530
|
+
relation,
|
|
531
|
+
column_name,
|
|
532
|
+
value,
|
|
533
|
+
normalized_value,
|
|
534
|
+
tokenize = 'porter unicode61'
|
|
535
|
+
);
|
|
536
|
+
|
|
432
537
|
CREATE TABLE IF NOT EXISTS metadata_state (
|
|
433
538
|
key TEXT PRIMARY KEY,
|
|
434
539
|
value TEXT NOT NULL
|
|
@@ -713,13 +818,56 @@ export class MetadataCatalog {
|
|
|
713
818
|
tables: normalizeRuntimeSchemaTables(snapshot.tables).slice(0, 500),
|
|
714
819
|
};
|
|
715
820
|
const id = `schema_${Date.parse(capturedAt) || Date.now()}`;
|
|
716
|
-
|
|
821
|
+
const valueEntries = buildRuntimeValueIndex(cleanSnapshot);
|
|
822
|
+
const insertSnapshot = this.db.prepare(`
|
|
717
823
|
INSERT OR REPLACE INTO runtime_schema_snapshots (
|
|
718
824
|
id, source, payload_json, captured_at
|
|
719
825
|
) VALUES (?, ?, ?, ?)
|
|
720
|
-
`)
|
|
826
|
+
`);
|
|
827
|
+
const insertValue = this.db.prepare(`
|
|
828
|
+
INSERT OR REPLACE INTO runtime_value_index (
|
|
829
|
+
value_key, relation, schema_name, table_name, column_name, column_type,
|
|
830
|
+
value, normalized_value, source, captured_at
|
|
831
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
832
|
+
`);
|
|
833
|
+
const insertValueFts = this.db.prepare(`
|
|
834
|
+
INSERT INTO runtime_value_fts (value_key, relation, column_name, value, normalized_value)
|
|
835
|
+
VALUES (?, ?, ?, ?, ?)
|
|
836
|
+
`);
|
|
837
|
+
const setState = this.db.prepare(`
|
|
838
|
+
INSERT OR REPLACE INTO metadata_state (key, value) VALUES (?, ?)
|
|
839
|
+
`);
|
|
840
|
+
const txn = this.db.transaction(() => {
|
|
841
|
+
insertSnapshot.run(id, cleanSnapshot.source ?? null, JSON.stringify(cleanSnapshot), capturedAt);
|
|
842
|
+
this.db.prepare('DELETE FROM runtime_value_fts').run();
|
|
843
|
+
this.db.prepare('DELETE FROM runtime_value_index').run();
|
|
844
|
+
for (const entry of valueEntries) {
|
|
845
|
+
insertValue.run(entry.valueKey, entry.relation, entry.schema ?? null, entry.tableName ?? null, entry.columnName, entry.columnType ?? null, entry.value, entry.normalizedValue, entry.source ?? null, entry.capturedAt);
|
|
846
|
+
insertValueFts.run(entry.valueKey, entry.relation, entry.columnName, entry.value, entry.normalizedValue);
|
|
847
|
+
}
|
|
848
|
+
setState.run('runtime_value_index_count', String(valueEntries.length));
|
|
849
|
+
setState.run('runtime_value_index_captured_at', capturedAt);
|
|
850
|
+
});
|
|
851
|
+
txn();
|
|
721
852
|
return cleanSnapshot;
|
|
722
853
|
}
|
|
854
|
+
searchRuntimeValues(terms, limit = 40) {
|
|
855
|
+
const cleanTerms = uniqueStrings(terms.map(normalizeValueIndexText).filter(Boolean)).slice(0, 8);
|
|
856
|
+
const query = sanitizeFtsQuery(cleanTerms.join(' '));
|
|
857
|
+
if (!query)
|
|
858
|
+
return [];
|
|
859
|
+
const rows = this.db.prepare(`
|
|
860
|
+
SELECT v.*,
|
|
861
|
+
bm25(runtime_value_fts) AS rank
|
|
862
|
+
FROM runtime_value_fts
|
|
863
|
+
JOIN runtime_value_index AS v ON v.value_key = runtime_value_fts.value_key
|
|
864
|
+
WHERE runtime_value_fts MATCH ?
|
|
865
|
+
ORDER BY rank
|
|
866
|
+
LIMIT ?
|
|
867
|
+
`).all(query, Math.max(1, limit));
|
|
868
|
+
return rows.map((row) => rowToRuntimeValueMatch(row, cleanTerms))
|
|
869
|
+
.sort((a, b) => b.score - a.score || a.relation.localeCompare(b.relation) || a.columnName.localeCompare(b.columnName));
|
|
870
|
+
}
|
|
723
871
|
latestRuntimeSchemaSnapshot() {
|
|
724
872
|
const row = this.db.prepare(`
|
|
725
873
|
SELECT payload_json
|
|
@@ -821,20 +969,35 @@ function loadAgentSemanticLayer(projectRoot) {
|
|
|
821
969
|
? { provider: 'dql', path: config.semanticLayer.path }
|
|
822
970
|
: undefined;
|
|
823
971
|
const configured = resolveSemanticLayerWithDiagnostics(semanticConfig, projectRoot).layer;
|
|
824
|
-
|
|
972
|
+
// An EMPTY configured layer (e.g. `provider: 'dql'` pointing at a folder
|
|
973
|
+
// with no definitions — the scaffold default) must not shadow a real dbt
|
|
974
|
+
// MetricFlow semantic layer: that starves the catalog of every
|
|
975
|
+
// semantic_metric object and the governed-metric answer tier can never
|
|
976
|
+
// fire. Mirror the runtime's resolveProjectSemanticConfig preference:
|
|
977
|
+
// substance wins over configuration.
|
|
978
|
+
if (configured && semanticLayerHasContent(configured))
|
|
825
979
|
return configured;
|
|
826
980
|
if (config.dbt?.projectDir) {
|
|
827
|
-
|
|
981
|
+
const dbtLayer = resolveSemanticLayerWithDiagnostics({
|
|
828
982
|
provider: 'dbt',
|
|
829
983
|
projectPath: config.dbt.projectDir,
|
|
830
|
-
}, projectRoot).layer
|
|
984
|
+
}, projectRoot).layer;
|
|
985
|
+
if (dbtLayer && semanticLayerHasContent(dbtLayer))
|
|
986
|
+
return dbtLayer;
|
|
831
987
|
}
|
|
988
|
+
return configured ?? undefined;
|
|
832
989
|
}
|
|
833
990
|
catch {
|
|
834
991
|
return undefined;
|
|
835
992
|
}
|
|
836
993
|
return undefined;
|
|
837
994
|
}
|
|
995
|
+
/** True when the layer defines ANY semantics — metrics, dimensions, or measures. */
|
|
996
|
+
function semanticLayerHasContent(layer) {
|
|
997
|
+
return layer.listMetrics().length > 0
|
|
998
|
+
|| layer.listDimensions().length > 0
|
|
999
|
+
|| layer.listMeasures().length > 0;
|
|
1000
|
+
}
|
|
838
1001
|
function manifestDiagnosticToMetadataDiagnostic(diagnostic) {
|
|
839
1002
|
return {
|
|
840
1003
|
kind: diagnostic.kind,
|
|
@@ -860,6 +1023,8 @@ function objectFromKGNode(node) {
|
|
|
860
1023
|
grain: node.grain,
|
|
861
1024
|
entities: node.entities ?? [],
|
|
862
1025
|
declaredOutputs: node.declaredOutputs ?? [],
|
|
1026
|
+
outputs: node.outputs ?? [],
|
|
1027
|
+
outputContract: node.outputContract ?? [],
|
|
863
1028
|
dimensions: node.dimensions ?? [],
|
|
864
1029
|
allowedFilters: node.allowedFilters ?? [],
|
|
865
1030
|
parameterPolicy: node.parameterPolicy ?? [],
|
|
@@ -964,6 +1129,7 @@ function addDbtDagObjects(manifest, objects, edges, diagnostics) {
|
|
|
964
1129
|
materialized: model.materialized,
|
|
965
1130
|
dependsOn: model.dependsOn,
|
|
966
1131
|
columns: model.columns ?? [],
|
|
1132
|
+
columnCompleteness: 'partial',
|
|
967
1133
|
}),
|
|
968
1134
|
};
|
|
969
1135
|
objects.set(objectKey, mergeObject(objects.get(objectKey), object));
|
|
@@ -982,6 +1148,7 @@ function addDbtDagObjects(manifest, objects, edges, diagnostics) {
|
|
|
982
1148
|
uniqueId: model.uniqueId,
|
|
983
1149
|
type: column.type,
|
|
984
1150
|
relation,
|
|
1151
|
+
columnCompleteness: 'partial',
|
|
985
1152
|
}),
|
|
986
1153
|
}));
|
|
987
1154
|
const edge = {
|
|
@@ -1029,18 +1196,30 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
|
|
|
1029
1196
|
});
|
|
1030
1197
|
return;
|
|
1031
1198
|
}
|
|
1199
|
+
const catalogColumns = loadRawDbtCatalogColumns(join(dirname(manifestPath), 'catalog.json'));
|
|
1200
|
+
const entries = [];
|
|
1032
1201
|
for (const [uniqueId, node] of Object.entries(raw.nodes ?? {})) {
|
|
1033
1202
|
if (node.resource_type !== 'model')
|
|
1034
1203
|
continue;
|
|
1035
1204
|
const name = rawDbtName(node);
|
|
1036
1205
|
if (!name)
|
|
1037
1206
|
continue;
|
|
1038
|
-
|
|
1207
|
+
const database = stringValue(node.database);
|
|
1208
|
+
const schema = stringValue(node.schema);
|
|
1209
|
+
const entry = {
|
|
1039
1210
|
uniqueId,
|
|
1040
1211
|
name,
|
|
1041
1212
|
node,
|
|
1042
1213
|
objectKey: `dbt:model:${name}`,
|
|
1043
1214
|
objectType: 'dbt_model',
|
|
1215
|
+
relation: [database, schema, name].filter(Boolean).join('.'),
|
|
1216
|
+
database,
|
|
1217
|
+
schema,
|
|
1218
|
+
catalogColumns: catalogColumns.get(uniqueId),
|
|
1219
|
+
};
|
|
1220
|
+
entries.push(entry);
|
|
1221
|
+
addRawDbtCatalogObject({
|
|
1222
|
+
...entry,
|
|
1044
1223
|
objects,
|
|
1045
1224
|
edges,
|
|
1046
1225
|
});
|
|
@@ -1049,23 +1228,42 @@ function addRawDbtManifestCatalogObjects(projectRoot, manifest, objects, edges,
|
|
|
1049
1228
|
const name = rawDbtName(source);
|
|
1050
1229
|
if (!name)
|
|
1051
1230
|
continue;
|
|
1052
|
-
|
|
1231
|
+
const database = stringValue(source.database);
|
|
1232
|
+
const schema = stringValue(source.schema);
|
|
1233
|
+
const entry = {
|
|
1053
1234
|
uniqueId,
|
|
1054
1235
|
name,
|
|
1055
1236
|
node: source,
|
|
1056
1237
|
objectKey: `dbt:source:${name}`,
|
|
1057
1238
|
objectType: 'dbt_source',
|
|
1239
|
+
relation: [database, schema, name].filter(Boolean).join('.'),
|
|
1240
|
+
database,
|
|
1241
|
+
schema,
|
|
1242
|
+
catalogColumns: catalogColumns.get(uniqueId),
|
|
1243
|
+
};
|
|
1244
|
+
entries.push(entry);
|
|
1245
|
+
addRawDbtCatalogObject({
|
|
1246
|
+
...entry,
|
|
1058
1247
|
objects,
|
|
1059
1248
|
edges,
|
|
1060
1249
|
});
|
|
1061
1250
|
}
|
|
1251
|
+
const relationLookup = buildRawDbtRelationLookup(entries);
|
|
1252
|
+
for (const entry of entries) {
|
|
1253
|
+
if (entry.objectType !== 'dbt_model')
|
|
1254
|
+
continue;
|
|
1255
|
+
addRawDbtCompiledColumnLineage(entry, relationLookup, objects, edges);
|
|
1256
|
+
}
|
|
1062
1257
|
}
|
|
1063
1258
|
function addRawDbtCatalogObject(input) {
|
|
1064
1259
|
const existing = input.objects.get(input.objectKey);
|
|
1065
|
-
const database = stringValue(input.node.database);
|
|
1066
|
-
const schema = stringValue(input.node.schema);
|
|
1067
|
-
const relation = [database, schema, input.name].filter(Boolean).join('.');
|
|
1068
|
-
const columns =
|
|
1260
|
+
const database = input.database ?? stringValue(input.node.database);
|
|
1261
|
+
const schema = input.schema ?? stringValue(input.node.schema);
|
|
1262
|
+
const relation = input.relation || [database, schema, input.name].filter(Boolean).join('.');
|
|
1263
|
+
const columns = input.catalogColumns && input.catalogColumns.length > 0
|
|
1264
|
+
? input.catalogColumns
|
|
1265
|
+
: rawDbtColumns(input.node.columns);
|
|
1266
|
+
const columnCompleteness = input.catalogColumns && input.catalogColumns.length > 0 ? 'complete' : 'partial';
|
|
1069
1267
|
input.objects.set(input.objectKey, mergeObject(existing, {
|
|
1070
1268
|
objectKey: input.objectKey,
|
|
1071
1269
|
objectType: input.objectType,
|
|
@@ -1085,6 +1283,7 @@ function addRawDbtCatalogObject(input) {
|
|
|
1085
1283
|
dependsOn: rawDbtDependsOn(input.node),
|
|
1086
1284
|
tags: metadataStringArray(input.node.tags),
|
|
1087
1285
|
catalogOnly: existing ? undefined : true,
|
|
1286
|
+
columnCompleteness,
|
|
1088
1287
|
columns,
|
|
1089
1288
|
}),
|
|
1090
1289
|
}));
|
|
@@ -1106,6 +1305,7 @@ function addRawDbtCatalogObject(input) {
|
|
|
1106
1305
|
uniqueId: input.uniqueId,
|
|
1107
1306
|
type: column.type,
|
|
1108
1307
|
relation,
|
|
1308
|
+
columnCompleteness: 'partial',
|
|
1109
1309
|
catalogOnly: existingColumn ? undefined : true,
|
|
1110
1310
|
}),
|
|
1111
1311
|
}));
|
|
@@ -1127,6 +1327,124 @@ function addRawDbtCatalogObject(input) {
|
|
|
1127
1327
|
});
|
|
1128
1328
|
}
|
|
1129
1329
|
}
|
|
1330
|
+
function addRawDbtCompiledColumnLineage(entry, relationLookup, objects, edges) {
|
|
1331
|
+
const sql = rawDbtCompiledSql(entry.node);
|
|
1332
|
+
if (!sql)
|
|
1333
|
+
return;
|
|
1334
|
+
const lineage = extractColumnLineage(sql);
|
|
1335
|
+
if (!lineage.parsed || lineage.columns.length === 0)
|
|
1336
|
+
return;
|
|
1337
|
+
for (const column of lineage.columns) {
|
|
1338
|
+
if (!column.name || column.name === '*' || column.name.endsWith('.*'))
|
|
1339
|
+
continue;
|
|
1340
|
+
const outputKey = ensureDbtLineageOutputColumnObject(entry, column, objects);
|
|
1341
|
+
for (const source of column.sources) {
|
|
1342
|
+
if (!source.column || source.column === '*')
|
|
1343
|
+
continue;
|
|
1344
|
+
const targetKey = ensureDbtLineageSourceColumnObject(source, relationLookup, objects);
|
|
1345
|
+
putEdge(edges, {
|
|
1346
|
+
edgeType: 'derives_from',
|
|
1347
|
+
fromKey: outputKey,
|
|
1348
|
+
toKey: targetKey,
|
|
1349
|
+
confidence: column.unresolved ? 0.55 : 0.88,
|
|
1350
|
+
payload: compactObject({
|
|
1351
|
+
source: 'dbt compiled SQL column lineage',
|
|
1352
|
+
model: entry.name,
|
|
1353
|
+
output: column.name,
|
|
1354
|
+
table: source.table,
|
|
1355
|
+
column: source.column,
|
|
1356
|
+
isAggregate: column.isAggregate,
|
|
1357
|
+
aggregateFn: column.aggregateFn,
|
|
1358
|
+
}),
|
|
1359
|
+
});
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
function ensureDbtLineageOutputColumnObject(entry, column, objects) {
|
|
1364
|
+
const columnKey = `dbt:column:${entry.name}.${column.name}`;
|
|
1365
|
+
const existing = objects.get(columnKey);
|
|
1366
|
+
objects.set(columnKey, mergeObject(existing, {
|
|
1367
|
+
objectKey: columnKey,
|
|
1368
|
+
objectType: 'dbt_column',
|
|
1369
|
+
name: column.name,
|
|
1370
|
+
fullName: `${entry.name}.${column.name}`,
|
|
1371
|
+
status: existing?.status ?? 'dbt_compiled_lineage',
|
|
1372
|
+
sourcePath: existing?.sourcePath ?? stringValue(entry.node.original_file_path) ?? stringValue(entry.node.path),
|
|
1373
|
+
sourceSystem: existing?.sourceSystem ?? 'dbt compiled SQL lineage',
|
|
1374
|
+
payload: compactObject({
|
|
1375
|
+
...(existing?.payload ?? {}),
|
|
1376
|
+
model: entry.name,
|
|
1377
|
+
uniqueId: entry.uniqueId,
|
|
1378
|
+
relation: entry.relation,
|
|
1379
|
+
compiledSqlLineage: true,
|
|
1380
|
+
isAggregate: column.isAggregate,
|
|
1381
|
+
aggregateFn: column.aggregateFn,
|
|
1382
|
+
lineageSources: column.sources,
|
|
1383
|
+
}),
|
|
1384
|
+
}));
|
|
1385
|
+
return columnKey;
|
|
1386
|
+
}
|
|
1387
|
+
function ensureDbtLineageSourceColumnObject(source, relationLookup, objects) {
|
|
1388
|
+
const dbtSource = resolveRawDbtRelationForTable(source.table, relationLookup);
|
|
1389
|
+
if (dbtSource) {
|
|
1390
|
+
const columnKey = `dbt:column:${dbtSource.name}.${source.column}`;
|
|
1391
|
+
const existing = objects.get(columnKey);
|
|
1392
|
+
if (existing)
|
|
1393
|
+
return columnKey;
|
|
1394
|
+
objects.set(columnKey, {
|
|
1395
|
+
objectKey: columnKey,
|
|
1396
|
+
objectType: 'dbt_column',
|
|
1397
|
+
name: source.column,
|
|
1398
|
+
fullName: `${dbtSource.name}.${source.column}`,
|
|
1399
|
+
status: 'dbt_compiled_lineage',
|
|
1400
|
+
sourceSystem: 'dbt compiled SQL lineage',
|
|
1401
|
+
payload: compactObject({
|
|
1402
|
+
model: dbtSource.name,
|
|
1403
|
+
uniqueId: dbtSource.uniqueId,
|
|
1404
|
+
relation: dbtSource.relation,
|
|
1405
|
+
inferredFromCompiledSql: true,
|
|
1406
|
+
}),
|
|
1407
|
+
});
|
|
1408
|
+
return columnKey;
|
|
1409
|
+
}
|
|
1410
|
+
return ensureWarehouseLineageSourceColumnObject(source, objects, 'dbt compiled SQL lineage');
|
|
1411
|
+
}
|
|
1412
|
+
function buildRawDbtRelationLookup(entries) {
|
|
1413
|
+
const lookup = new Map();
|
|
1414
|
+
const add = (key, entry) => {
|
|
1415
|
+
if (!key)
|
|
1416
|
+
return;
|
|
1417
|
+
const normalized = normalizeRelationKey(key);
|
|
1418
|
+
if (!normalized)
|
|
1419
|
+
return;
|
|
1420
|
+
const existing = lookup.get(normalized);
|
|
1421
|
+
if (existing)
|
|
1422
|
+
existing.add(entry);
|
|
1423
|
+
else
|
|
1424
|
+
lookup.set(normalized, new Set([entry]));
|
|
1425
|
+
};
|
|
1426
|
+
for (const entry of entries) {
|
|
1427
|
+
add(entry.uniqueId, entry);
|
|
1428
|
+
add(entry.name, entry);
|
|
1429
|
+
add(entry.relation, entry);
|
|
1430
|
+
for (const key of dbtModelLookupKeys(entry.name, entry.schema, entry.database)) {
|
|
1431
|
+
add(key, entry);
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
return lookup;
|
|
1435
|
+
}
|
|
1436
|
+
function resolveRawDbtRelationForTable(tableRef, relationLookup) {
|
|
1437
|
+
for (const key of tableReferenceLookupKeys(tableRef)) {
|
|
1438
|
+
const matches = relationLookup.get(key);
|
|
1439
|
+
if (matches?.size === 1)
|
|
1440
|
+
return [...matches][0];
|
|
1441
|
+
}
|
|
1442
|
+
return undefined;
|
|
1443
|
+
}
|
|
1444
|
+
function rawDbtCompiledSql(node) {
|
|
1445
|
+
return stringValue(node.compiled_code)
|
|
1446
|
+
?? stringValue(node.compiled_sql);
|
|
1447
|
+
}
|
|
1130
1448
|
function rawDbtName(node) {
|
|
1131
1449
|
return stringValue(node.alias) ?? stringValue(node.identifier) ?? stringValue(node.name);
|
|
1132
1450
|
}
|
|
@@ -1140,6 +1458,41 @@ function rawDbtDependsOn(node) {
|
|
|
1140
1458
|
: null;
|
|
1141
1459
|
return metadataStringArray(dependsOn?.nodes);
|
|
1142
1460
|
}
|
|
1461
|
+
function loadRawDbtCatalogColumns(catalogPath) {
|
|
1462
|
+
const out = new Map();
|
|
1463
|
+
let raw;
|
|
1464
|
+
try {
|
|
1465
|
+
raw = JSON.parse(readFileSync(catalogPath, 'utf-8'));
|
|
1466
|
+
}
|
|
1467
|
+
catch {
|
|
1468
|
+
return out;
|
|
1469
|
+
}
|
|
1470
|
+
for (const entries of [raw.nodes ?? {}, raw.sources ?? {}]) {
|
|
1471
|
+
for (const [uniqueId, node] of Object.entries(entries)) {
|
|
1472
|
+
const columns = rawDbtCatalogColumns(node.columns);
|
|
1473
|
+
if (columns.length > 0)
|
|
1474
|
+
out.set(uniqueId, columns);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
return out;
|
|
1478
|
+
}
|
|
1479
|
+
function rawDbtCatalogColumns(value) {
|
|
1480
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
1481
|
+
return [];
|
|
1482
|
+
return Object.entries(value).flatMap(([key, entry]) => {
|
|
1483
|
+
if (!entry || typeof entry !== 'object')
|
|
1484
|
+
return [];
|
|
1485
|
+
const record = entry;
|
|
1486
|
+
const name = stringValue(record.name) ?? stringValue(record.column_name) ?? key;
|
|
1487
|
+
if (!name)
|
|
1488
|
+
return [];
|
|
1489
|
+
return [{
|
|
1490
|
+
name,
|
|
1491
|
+
type: stringValue(record.type) ?? stringValue(record.data_type),
|
|
1492
|
+
description: stringValue(record.comment) ?? stringValue(record.description),
|
|
1493
|
+
}];
|
|
1494
|
+
});
|
|
1495
|
+
}
|
|
1143
1496
|
function rawDbtColumns(value) {
|
|
1144
1497
|
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
1145
1498
|
return [];
|
|
@@ -1163,6 +1516,9 @@ function addManifestBlockDetails(manifest, objects) {
|
|
|
1163
1516
|
const existing = objects.get(objectKey);
|
|
1164
1517
|
if (!existing)
|
|
1165
1518
|
continue;
|
|
1519
|
+
const contractOutputNames = block.declaredOutputs
|
|
1520
|
+
?? block.outputContract?.map((output) => output.name).filter(Boolean)
|
|
1521
|
+
?? block.outputs?.map((output) => output.name).filter(Boolean);
|
|
1166
1522
|
objects.set(objectKey, mergeObject(existing, {
|
|
1167
1523
|
...existing,
|
|
1168
1524
|
payload: compactObject({
|
|
@@ -1173,6 +1529,9 @@ function addManifestBlockDetails(manifest, objects) {
|
|
|
1173
1529
|
refDependencies: block.refDependencies,
|
|
1174
1530
|
metricRefs: block.metricRefs,
|
|
1175
1531
|
dimensionRefs: block.dimensionRefs,
|
|
1532
|
+
declaredOutputs: block.declaredOutputs,
|
|
1533
|
+
outputs: block.outputs,
|
|
1534
|
+
outputContract: block.outputContract,
|
|
1176
1535
|
dimensions: block.dimensions,
|
|
1177
1536
|
chartType: block.chartType,
|
|
1178
1537
|
blockType: block.blockType,
|
|
@@ -1187,7 +1546,7 @@ function addManifestBlockDetails(manifest, objects) {
|
|
|
1187
1546
|
grain: block.grain,
|
|
1188
1547
|
entities: block.entities,
|
|
1189
1548
|
terms: block.termRefs,
|
|
1190
|
-
outputs:
|
|
1549
|
+
outputs: contractOutputNames,
|
|
1191
1550
|
dimensions: block.dimensions,
|
|
1192
1551
|
filters: block.allowedFilters,
|
|
1193
1552
|
sources: [...(block.tableDependencies ?? []), ...(block.rawTableRefs ?? [])],
|
|
@@ -1226,6 +1585,9 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
|
|
|
1226
1585
|
}
|
|
1227
1586
|
const latestContractKeyById = new Map();
|
|
1228
1587
|
const contractKeyByVersionedRef = new Map();
|
|
1588
|
+
// entity (by name and domain.entity) -> physical binding ref, so relationships
|
|
1589
|
+
// can be resolved to concrete relations for grain-safe join hints.
|
|
1590
|
+
const entityBindingRef = new Map();
|
|
1229
1591
|
for (const domain of raw.domains) {
|
|
1230
1592
|
if (!domain?.name)
|
|
1231
1593
|
continue;
|
|
@@ -1279,6 +1641,11 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
|
|
|
1279
1641
|
if (!entity?.name)
|
|
1280
1642
|
continue;
|
|
1281
1643
|
const entityKey = `datalex:entity:${domain.name}.${entity.name}`;
|
|
1644
|
+
if (entity.binding?.ref) {
|
|
1645
|
+
const boundKey = objectKeyForDataLexBinding(entity.binding);
|
|
1646
|
+
entityBindingRef.set(entity.name.toLowerCase(), boundKey);
|
|
1647
|
+
entityBindingRef.set(`${domain.name}.${entity.name}`.toLowerCase(), boundKey);
|
|
1648
|
+
}
|
|
1282
1649
|
objects.set(entityKey, mergeObject(objects.get(entityKey), {
|
|
1283
1650
|
objectKey: entityKey,
|
|
1284
1651
|
objectType: 'datalex_entity',
|
|
@@ -1350,6 +1717,33 @@ function addDataLexManifestObjects(projectRoot, manifest, objects, edges, diagno
|
|
|
1350
1717
|
}
|
|
1351
1718
|
}
|
|
1352
1719
|
}
|
|
1720
|
+
// Typed cross-entity relationships → grain-safe join edges between the entities'
|
|
1721
|
+
// physical relations. buildSelectedJoinPaths turns these into `datalex`-sourced
|
|
1722
|
+
// join paths that outrank name-heuristic guesses.
|
|
1723
|
+
const resolveEntityRef = (endpoint) => entityBindingRef.get(`${endpoint.domain ?? ''}.${endpoint.entity}`.toLowerCase())
|
|
1724
|
+
?? entityBindingRef.get(endpoint.entity.toLowerCase());
|
|
1725
|
+
for (const relationship of raw.relationships ?? []) {
|
|
1726
|
+
if (!relationship?.from?.entity || !relationship?.to?.entity)
|
|
1727
|
+
continue;
|
|
1728
|
+
const fromRef = resolveEntityRef(relationship.from);
|
|
1729
|
+
const toRef = resolveEntityRef(relationship.to);
|
|
1730
|
+
if (!fromRef || !toRef || fromRef.toLowerCase() === toRef.toLowerCase())
|
|
1731
|
+
continue;
|
|
1732
|
+
putEdge(edges, {
|
|
1733
|
+
edgeType: 'datalex_join',
|
|
1734
|
+
fromKey: fromRef,
|
|
1735
|
+
toKey: toRef,
|
|
1736
|
+
confidence: relationship.identifying ? 0.97 : 0.9,
|
|
1737
|
+
payload: compactObject({
|
|
1738
|
+
source: 'datalex relationship',
|
|
1739
|
+
name: relationship.name,
|
|
1740
|
+
fromColumn: relationship.from.column,
|
|
1741
|
+
toColumn: relationship.to.column,
|
|
1742
|
+
cardinality: relationship.cardinality,
|
|
1743
|
+
verb: relationship.verb,
|
|
1744
|
+
}),
|
|
1745
|
+
});
|
|
1746
|
+
}
|
|
1353
1747
|
for (const block of Object.values(manifest.blocks ?? {})) {
|
|
1354
1748
|
if (!block.datalexContract)
|
|
1355
1749
|
continue;
|
|
@@ -1406,6 +1800,112 @@ function addBlockDependencyEdges(manifest, edges) {
|
|
|
1406
1800
|
}
|
|
1407
1801
|
}
|
|
1408
1802
|
}
|
|
1803
|
+
function addBlockOutputLineageObjects(manifest, objects, edges) {
|
|
1804
|
+
const dbtLookup = buildDbtModelLookup(manifest);
|
|
1805
|
+
for (const block of Object.values(manifest.blocks ?? {})) {
|
|
1806
|
+
if (!isCertifiedManifestBlock(block))
|
|
1807
|
+
continue;
|
|
1808
|
+
if (!Array.isArray(block.outputs) || block.outputs.length === 0)
|
|
1809
|
+
continue;
|
|
1810
|
+
const blockKey = `dql:block:${block.name}`;
|
|
1811
|
+
for (const output of block.outputs) {
|
|
1812
|
+
const outputName = typeof output.name === 'string' ? output.name.trim() : '';
|
|
1813
|
+
if (!outputName)
|
|
1814
|
+
continue;
|
|
1815
|
+
const outputKey = blockOutputObjectKey(block.name, outputName);
|
|
1816
|
+
objects.set(outputKey, mergeObject(objects.get(outputKey), {
|
|
1817
|
+
objectKey: outputKey,
|
|
1818
|
+
objectType: 'dql_block_output',
|
|
1819
|
+
name: outputName,
|
|
1820
|
+
fullName: `${block.name}.${outputName}`,
|
|
1821
|
+
domain: block.domain,
|
|
1822
|
+
owner: block.owner,
|
|
1823
|
+
status: block.status,
|
|
1824
|
+
description: `${outputName} output column from DQL block ${block.name}.`,
|
|
1825
|
+
sourcePath: block.filePath,
|
|
1826
|
+
sourceSystem: 'DQL block column lineage',
|
|
1827
|
+
payload: compactObject({
|
|
1828
|
+
block: block.name,
|
|
1829
|
+
output: outputName,
|
|
1830
|
+
isAggregate: output.isAggregate,
|
|
1831
|
+
aggregateFn: output.aggregateFn,
|
|
1832
|
+
unresolved: output.unresolved,
|
|
1833
|
+
sources: output.sources ?? [],
|
|
1834
|
+
}),
|
|
1835
|
+
}));
|
|
1836
|
+
putEdge(edges, {
|
|
1837
|
+
edgeType: 'contains',
|
|
1838
|
+
fromKey: blockKey,
|
|
1839
|
+
toKey: outputKey,
|
|
1840
|
+
confidence: 1,
|
|
1841
|
+
payload: { source: 'dql block output lineage', output: outputName },
|
|
1842
|
+
});
|
|
1843
|
+
for (const source of output.sources ?? []) {
|
|
1844
|
+
const targetKey = ensureLineageSourceColumnObject(source, dbtLookup, objects);
|
|
1845
|
+
putEdge(edges, {
|
|
1846
|
+
edgeType: 'derives_from',
|
|
1847
|
+
fromKey: outputKey,
|
|
1848
|
+
toKey: targetKey,
|
|
1849
|
+
confidence: output.unresolved ? 0.55 : 0.92,
|
|
1850
|
+
payload: {
|
|
1851
|
+
source: 'dql block output column lineage',
|
|
1852
|
+
block: block.name,
|
|
1853
|
+
output: outputName,
|
|
1854
|
+
table: source.table,
|
|
1855
|
+
column: source.column,
|
|
1856
|
+
aggregateFn: output.aggregateFn,
|
|
1857
|
+
},
|
|
1858
|
+
});
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
function isCertifiedManifestBlock(block) {
|
|
1864
|
+
return block.status === 'certified'
|
|
1865
|
+
|| block.status === 'approved';
|
|
1866
|
+
}
|
|
1867
|
+
function ensureLineageSourceColumnObject(source, dbtLookup, objects) {
|
|
1868
|
+
const dbtModelKey = resolveDbtModelKeyForTable(source.table, dbtLookup);
|
|
1869
|
+
if (dbtModelKey) {
|
|
1870
|
+
const modelName = dbtModelKey.split(':').at(-1) ?? source.table;
|
|
1871
|
+
const columnKey = `dbt:column:${modelName}.${source.column}`;
|
|
1872
|
+
if (objects.has(columnKey))
|
|
1873
|
+
return columnKey;
|
|
1874
|
+
}
|
|
1875
|
+
return ensureWarehouseLineageSourceColumnObject(source, objects, 'DQL block column lineage');
|
|
1876
|
+
}
|
|
1877
|
+
function ensureWarehouseLineageSourceColumnObject(source, objects, sourceSystem) {
|
|
1878
|
+
const tableKey = `warehouse:table:${source.table}`;
|
|
1879
|
+
if (!objects.has(tableKey)) {
|
|
1880
|
+
objects.set(tableKey, {
|
|
1881
|
+
objectKey: tableKey,
|
|
1882
|
+
objectType: 'warehouse_table',
|
|
1883
|
+
name: source.table.split('.').at(-1) ?? source.table,
|
|
1884
|
+
fullName: source.table,
|
|
1885
|
+
status: 'referenced',
|
|
1886
|
+
sourceSystem,
|
|
1887
|
+
payload: compactObject({ relation: source.table }),
|
|
1888
|
+
});
|
|
1889
|
+
}
|
|
1890
|
+
const columnKey = `warehouse:column:${source.table}.${source.column}`;
|
|
1891
|
+
objects.set(columnKey, mergeObject(objects.get(columnKey), {
|
|
1892
|
+
objectKey: columnKey,
|
|
1893
|
+
objectType: 'warehouse_column',
|
|
1894
|
+
name: source.column,
|
|
1895
|
+
fullName: `${source.table}.${source.column}`,
|
|
1896
|
+
status: 'referenced',
|
|
1897
|
+
sourceSystem,
|
|
1898
|
+
payload: compactObject({
|
|
1899
|
+
relation: source.table,
|
|
1900
|
+
table: source.table,
|
|
1901
|
+
column: source.column,
|
|
1902
|
+
}),
|
|
1903
|
+
}));
|
|
1904
|
+
return columnKey;
|
|
1905
|
+
}
|
|
1906
|
+
function blockOutputObjectKey(blockName, outputName) {
|
|
1907
|
+
return `dql:block_output:${blockName}.${outputName}`;
|
|
1908
|
+
}
|
|
1409
1909
|
function putEdge(edges, edge) {
|
|
1410
1910
|
edges.set(`${edge.edgeType}\u0000${edge.fromKey}\u0000${edge.toKey}`, edge);
|
|
1411
1911
|
}
|
|
@@ -1525,7 +2025,7 @@ function firstNonEmpty(...values) {
|
|
|
1525
2025
|
function lowerOrUndef(value) {
|
|
1526
2026
|
return value ? value.trim().toLowerCase() : undefined;
|
|
1527
2027
|
}
|
|
1528
|
-
function planContextPackRoute(input) {
|
|
2028
|
+
async function planContextPackRoute(input) {
|
|
1529
2029
|
const intent = input.request.intent ?? input.questionPlan.routeIntent ?? classifyMetadataIntent(input.request.question, input.request.followUp);
|
|
1530
2030
|
// Conflict route (additive, evaluated first): if the question's top governed
|
|
1531
2031
|
// candidates include BOTH sides of a compile-time trust conflict, refuse to
|
|
@@ -1533,7 +2033,7 @@ function planContextPackRoute(input) {
|
|
|
1533
2033
|
// prompt. Only fires when both conflicting sides are actually among the
|
|
1534
2034
|
// selected candidates for THIS question, so non-conflicting asks are
|
|
1535
2035
|
// unaffected even when a conflict exists elsewhere in the project.
|
|
1536
|
-
const routeConflict = pickRouteConflict(input.compileConflicts ?? [], input.rankedObjects ?? [], input.objects);
|
|
2036
|
+
const routeConflict = pickRouteConflict(input.compileConflicts ?? [], input.rankedObjects ?? [], input.objects, input.request.question);
|
|
1537
2037
|
if (routeConflict) {
|
|
1538
2038
|
const selectedEvidenceForConflict = input.evidenceRoles.slice(0, 16);
|
|
1539
2039
|
return {
|
|
@@ -1566,6 +2066,30 @@ function planContextPackRoute(input) {
|
|
|
1566
2066
|
.filter((item) => item.kind === 'context_only')
|
|
1567
2067
|
.sort((a, b) => b.score - a.score)[0];
|
|
1568
2068
|
const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
|
|
2069
|
+
const directCertifiedBypass = Boolean(exact && (exactExampleMatch
|
|
2070
|
+
|| intent === 'definition_lookup'
|
|
2071
|
+
|| intent === 'exact_certified_lookup'
|
|
2072
|
+
|| objectNameInQuestion(input.request.question, exact)));
|
|
2073
|
+
const blockFitObject = exact
|
|
2074
|
+
?? (contextApplicability ? input.objects.find((object) => object.objectKey === contextApplicability.objectKey) : undefined);
|
|
2075
|
+
const rawBlockFit = blockFitObject
|
|
2076
|
+
? evaluateCertifiedBlockFit({
|
|
2077
|
+
question: input.request.question,
|
|
2078
|
+
plan: input.questionPlan,
|
|
2079
|
+
block: blockFitObject,
|
|
2080
|
+
exactExampleMatch: exact ? exactExampleMatch : false,
|
|
2081
|
+
definitionLookup: Boolean(exact && intent === 'definition_lookup'),
|
|
2082
|
+
})
|
|
2083
|
+
: undefined;
|
|
2084
|
+
const blockFit = rawBlockFit && blockFitObject
|
|
2085
|
+
? await confirmMediumCertifiedFit({
|
|
2086
|
+
request: input.request,
|
|
2087
|
+
questionPlan: input.questionPlan,
|
|
2088
|
+
block: blockFitObject,
|
|
2089
|
+
fit: rawBlockFit,
|
|
2090
|
+
directCertifiedBypass,
|
|
2091
|
+
})
|
|
2092
|
+
: rawBlockFit;
|
|
1569
2093
|
const missingContext = buildMissingContext(input.request, intent, input.objects, input.allowedSqlContext);
|
|
1570
2094
|
const selectedEvidence = input.evidenceRoles.slice(0, 16);
|
|
1571
2095
|
// Grain / contract gate (refinement of the existing certified→generated
|
|
@@ -1596,7 +2120,9 @@ function planContextPackRoute(input) {
|
|
|
1596
2120
|
input.allowedSqlContext.sourceBlockSql.length > 0 ||
|
|
1597
2121
|
input.objects.some((object) => object.objectType.startsWith('semantic_'));
|
|
1598
2122
|
const grainGateDemotes = Boolean(grainGate && !grainGate.allow && canGenerateFromContext);
|
|
1599
|
-
|
|
2123
|
+
const blockFitDemotes = Boolean(blockFit && !certifiedFitAllowsTier1(blockFit) && !directCertifiedBypass && canGenerateFromContext);
|
|
2124
|
+
const certifiedFitPassed = !blockFit || certifiedFitAllowsTier1(blockFit) || directCertifiedBypass;
|
|
2125
|
+
if (!grainGateDemotes && !blockFitDemotes && certifiedFitPassed && exact && (certifiedApplicability?.kind === 'exact_answer'
|
|
1600
2126
|
|| certifiedApplicability?.kind === 'safe_parameterized'
|
|
1601
2127
|
|| intent === 'exact_certified_lookup'
|
|
1602
2128
|
|| intent === 'definition_lookup'
|
|
@@ -1608,6 +2134,7 @@ function planContextPackRoute(input) {
|
|
|
1608
2134
|
reason: `Certified ${exact.objectType.replace(/_/g, ' ')} "${exact.name}" exactly matches the requested artifact, definition, or direct KPI grain.`,
|
|
1609
2135
|
routeReason: grainGateInfo?.reason,
|
|
1610
2136
|
grainGate: grainGateInfo,
|
|
2137
|
+
blockFit,
|
|
1611
2138
|
trustLabel: 'certified',
|
|
1612
2139
|
reviewStatus: 'certified',
|
|
1613
2140
|
exactObjectKey: exact.objectKey,
|
|
@@ -1627,6 +2154,25 @@ function planContextPackRoute(input) {
|
|
|
1627
2154
|
reason: `Certified block "${exact.name}" is close but answers a different grain than the question, so it is context only and SQL is generated for the requested grain.`,
|
|
1628
2155
|
routeReason: grainGate.reason,
|
|
1629
2156
|
grainGate: grainGateInfo,
|
|
2157
|
+
blockFit,
|
|
2158
|
+
trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
|
|
2159
|
+
reviewStatus: 'draft_ready',
|
|
2160
|
+
certifiedApplicability: certifiedApplicability
|
|
2161
|
+
? { ...certifiedApplicability, kind: 'context_only' }
|
|
2162
|
+
: contextApplicability,
|
|
2163
|
+
selectedEvidence,
|
|
2164
|
+
missingContext,
|
|
2165
|
+
followUps: buildMetadataFollowUps(isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking', input.allowedSqlContext),
|
|
2166
|
+
};
|
|
2167
|
+
}
|
|
2168
|
+
if (blockFitDemotes && exact && blockFit) {
|
|
2169
|
+
return {
|
|
2170
|
+
route: 'generated_sql',
|
|
2171
|
+
intent: isGeneratedMetadataIntent(intent) ? intent : 'ad_hoc_ranking',
|
|
2172
|
+
reason: `Certified block "${exact.name}" is relevant but does not satisfy the requested answer shape, so it is context only and SQL is generated for the requested result.`,
|
|
2173
|
+
routeReason: blockFit.reasons.join('; '),
|
|
2174
|
+
grainGate: grainGateInfo,
|
|
2175
|
+
blockFit,
|
|
1630
2176
|
trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
|
|
1631
2177
|
reviewStatus: 'draft_ready',
|
|
1632
2178
|
certifiedApplicability: certifiedApplicability
|
|
@@ -1668,15 +2214,17 @@ function planContextPackRoute(input) {
|
|
|
1668
2214
|
route: 'generated_sql',
|
|
1669
2215
|
intent,
|
|
1670
2216
|
reason: 'The question asks for a different grain, ranking, breakdown, comparison, entity drilldown, or diagnostic analysis, so certified artifacts are context only.',
|
|
2217
|
+
routeReason: blockFit && !certifiedFitAllowsTier1(blockFit) ? blockFit.reasons.join('; ') : undefined,
|
|
1671
2218
|
trustLabel: input.trustLabel === 'certified' ? 'mixed' : input.trustLabel,
|
|
1672
2219
|
reviewStatus: 'draft_ready',
|
|
1673
2220
|
certifiedApplicability: contextApplicability,
|
|
2221
|
+
blockFit,
|
|
1674
2222
|
selectedEvidence,
|
|
1675
2223
|
missingContext,
|
|
1676
2224
|
followUps: buildMetadataFollowUps(intent, input.allowedSqlContext),
|
|
1677
2225
|
};
|
|
1678
2226
|
}
|
|
1679
|
-
if (exact) {
|
|
2227
|
+
if (exact && certifiedFitPassed) {
|
|
1680
2228
|
return {
|
|
1681
2229
|
route: 'certified',
|
|
1682
2230
|
intent: 'exact_certified_lookup',
|
|
@@ -1685,17 +2233,93 @@ function planContextPackRoute(input) {
|
|
|
1685
2233
|
reviewStatus: 'certified',
|
|
1686
2234
|
exactObjectKey: exact.objectKey,
|
|
1687
2235
|
certifiedApplicability,
|
|
2236
|
+
blockFit,
|
|
1688
2237
|
selectedEvidence,
|
|
1689
2238
|
missingContext: [],
|
|
1690
2239
|
followUps: buildMetadataFollowUps('exact_certified_lookup', input.allowedSqlContext),
|
|
1691
2240
|
};
|
|
1692
2241
|
}
|
|
2242
|
+
if (exact && blockFit && !certifiedFitAllowsTier1(blockFit)) {
|
|
2243
|
+
return clarifyDecision(intent, input.trustLabel === 'certified' ? 'mixed' : input.trustLabel, selectedEvidence, [{
|
|
2244
|
+
kind: 'metadata',
|
|
2245
|
+
severity: 'blocking',
|
|
2246
|
+
message: `The closest certified block "${exact.name}" does not exactly answer this question (${blockFit.reasons.join('; ')}), and there is not enough SQL context to safely generate the requested answer.`,
|
|
2247
|
+
}]);
|
|
2248
|
+
}
|
|
1693
2249
|
return clarifyDecision(intent, input.trustLabel, selectedEvidence, missingContext.length > 0 ? missingContext : [{
|
|
1694
2250
|
kind: 'metadata',
|
|
1695
2251
|
severity: 'blocking',
|
|
1696
2252
|
message: 'The local metadata matched some context, but not enough to choose a safe metric, table, or grain.',
|
|
1697
2253
|
}]);
|
|
1698
2254
|
}
|
|
2255
|
+
const CERTIFIED_FIT_CONFIRM_TIMEOUT_MS = 2500;
|
|
2256
|
+
async function confirmMediumCertifiedFit(input) {
|
|
2257
|
+
const confirmer = input.request.confirmCertifiedFit;
|
|
2258
|
+
if (!confirmer || input.directCertifiedBypass)
|
|
2259
|
+
return input.fit;
|
|
2260
|
+
if (input.fit.confidence !== 'medium')
|
|
2261
|
+
return input.fit;
|
|
2262
|
+
if (input.fit.kind !== 'exact' && input.fit.kind !== 'trim_safe')
|
|
2263
|
+
return input.fit;
|
|
2264
|
+
try {
|
|
2265
|
+
const result = await withTimeout(confirmer({
|
|
2266
|
+
question: input.request.question,
|
|
2267
|
+
questionPlan: input.questionPlan,
|
|
2268
|
+
block: input.block,
|
|
2269
|
+
fit: input.fit,
|
|
2270
|
+
}), CERTIFIED_FIT_CONFIRM_TIMEOUT_MS, 'certified fit confirmation timed out');
|
|
2271
|
+
const reason = cleanConfirmationReason(result.reason);
|
|
2272
|
+
if (result.allow && result.confidence !== 'low') {
|
|
2273
|
+
return {
|
|
2274
|
+
...input.fit,
|
|
2275
|
+
confidence: 'high',
|
|
2276
|
+
reasons: [
|
|
2277
|
+
...input.fit.reasons,
|
|
2278
|
+
`fit confirmation accepted this certified block${reason ? `: ${reason}` : ''}`,
|
|
2279
|
+
],
|
|
2280
|
+
};
|
|
2281
|
+
}
|
|
2282
|
+
return {
|
|
2283
|
+
...input.fit,
|
|
2284
|
+
kind: 'context_only',
|
|
2285
|
+
confidence: 'high',
|
|
2286
|
+
reasons: [
|
|
2287
|
+
...input.fit.reasons,
|
|
2288
|
+
`fit confirmation rejected this certified block as a direct answer${reason ? `: ${reason}` : ''}`,
|
|
2289
|
+
],
|
|
2290
|
+
};
|
|
2291
|
+
}
|
|
2292
|
+
catch (error) {
|
|
2293
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2294
|
+
return {
|
|
2295
|
+
...input.fit,
|
|
2296
|
+
reasons: [
|
|
2297
|
+
...input.fit.reasons,
|
|
2298
|
+
`fit confirmation unavailable; kept review-required (${message})`,
|
|
2299
|
+
],
|
|
2300
|
+
};
|
|
2301
|
+
}
|
|
2302
|
+
}
|
|
2303
|
+
async function withTimeout(promise, ms, message) {
|
|
2304
|
+
let timeout;
|
|
2305
|
+
try {
|
|
2306
|
+
return await Promise.race([
|
|
2307
|
+
promise,
|
|
2308
|
+
new Promise((_, reject) => {
|
|
2309
|
+
timeout = setTimeout(() => reject(new Error(message)), ms);
|
|
2310
|
+
}),
|
|
2311
|
+
]);
|
|
2312
|
+
}
|
|
2313
|
+
finally {
|
|
2314
|
+
if (timeout)
|
|
2315
|
+
clearTimeout(timeout);
|
|
2316
|
+
}
|
|
2317
|
+
}
|
|
2318
|
+
function cleanConfirmationReason(value) {
|
|
2319
|
+
return typeof value === 'string'
|
|
2320
|
+
? value.replace(/\s+/g, ' ').trim().slice(0, 240)
|
|
2321
|
+
: '';
|
|
2322
|
+
}
|
|
1699
2323
|
function certifiedApplicabilities(objects, questionPlan) {
|
|
1700
2324
|
const items = objects
|
|
1701
2325
|
.filter((object) => object.objectType === 'dql_block' && isCertifiedMetadataObject(object))
|
|
@@ -1704,6 +2328,56 @@ function certifiedApplicabilities(objects, questionPlan) {
|
|
|
1704
2328
|
.sort((a, b) => b.score - a.score);
|
|
1705
2329
|
return new Map(items.map((item) => [item.objectKey, item]));
|
|
1706
2330
|
}
|
|
2331
|
+
function buildCertifiedCandidateFitDiagnostics(input) {
|
|
2332
|
+
return input.objects
|
|
2333
|
+
.filter((object) => object.objectType === 'dql_block' && isCertifiedMetadataObject(object))
|
|
2334
|
+
.map((object) => {
|
|
2335
|
+
const applicability = certifiedApplicabilityForObject(object, input.questionPlan);
|
|
2336
|
+
const fit = evaluateCertifiedBlockFit({
|
|
2337
|
+
question: input.request.question,
|
|
2338
|
+
plan: input.questionPlan,
|
|
2339
|
+
block: object,
|
|
2340
|
+
exactExampleMatch: hasExactExampleQuestion(input.request.question, object),
|
|
2341
|
+
definitionLookup: input.routeDecision.intent === 'definition_lookup',
|
|
2342
|
+
});
|
|
2343
|
+
return {
|
|
2344
|
+
objectKey: object.objectKey,
|
|
2345
|
+
name: object.name,
|
|
2346
|
+
applicabilityKind: applicability.kind,
|
|
2347
|
+
applicabilityScore: applicability.score,
|
|
2348
|
+
action: certifiedCandidateFitAction(object.objectKey, applicability, fit, input.routeDecision),
|
|
2349
|
+
fit,
|
|
2350
|
+
};
|
|
2351
|
+
})
|
|
2352
|
+
.filter((item) => item.applicabilityKind !== 'not_applicable' ||
|
|
2353
|
+
item.fit.kind !== 'not_applicable' ||
|
|
2354
|
+
item.action === 'certified_answer')
|
|
2355
|
+
.sort((a, b) => certifiedCandidateActionRank(a.action) - certifiedCandidateActionRank(b.action) ||
|
|
2356
|
+
b.applicabilityScore - a.applicabilityScore ||
|
|
2357
|
+
a.name.localeCompare(b.name))
|
|
2358
|
+
.slice(0, 12);
|
|
2359
|
+
}
|
|
2360
|
+
function certifiedCandidateFitAction(objectKey, applicability, fit, routeDecision) {
|
|
2361
|
+
if (routeDecision.route === 'certified' && routeDecision.exactObjectKey === objectKey)
|
|
2362
|
+
return 'certified_answer';
|
|
2363
|
+
if (routeDecision.certifiedApplicability?.objectKey === objectKey && routeDecision.route !== 'certified')
|
|
2364
|
+
return 'context_only';
|
|
2365
|
+
if (!certifiedFitAllowsTier1(fit) || applicability.kind === 'context_only')
|
|
2366
|
+
return 'rejected_for_fit';
|
|
2367
|
+
return 'eligible_not_selected';
|
|
2368
|
+
}
|
|
2369
|
+
function certifiedCandidateActionRank(action) {
|
|
2370
|
+
switch (action) {
|
|
2371
|
+
case 'certified_answer':
|
|
2372
|
+
return 0;
|
|
2373
|
+
case 'context_only':
|
|
2374
|
+
return 1;
|
|
2375
|
+
case 'rejected_for_fit':
|
|
2376
|
+
return 2;
|
|
2377
|
+
case 'eligible_not_selected':
|
|
2378
|
+
return 3;
|
|
2379
|
+
}
|
|
2380
|
+
}
|
|
1707
2381
|
function clarifyDecision(intent, trustLabel, selectedEvidence, missingContext) {
|
|
1708
2382
|
return {
|
|
1709
2383
|
route: 'clarify',
|
|
@@ -1994,14 +2668,18 @@ function buildEvidenceRoles(objects, queryRuns) {
|
|
|
1994
2668
|
function evidenceRoleForObject(object) {
|
|
1995
2669
|
if (object.objectType === 'dql_block' && isCertifiedMetadataObject(object))
|
|
1996
2670
|
return 'certified_context';
|
|
2671
|
+
if (object.objectType === 'dql_block_output')
|
|
2672
|
+
return 'certified_context';
|
|
1997
2673
|
if (object.objectType === 'semantic_metric')
|
|
1998
2674
|
return 'semantic_metric';
|
|
1999
2675
|
if (object.objectType === 'dql_term' || object.objectType === 'business_view')
|
|
2000
2676
|
return 'business_context';
|
|
2001
2677
|
if (object.objectType === 'dbt_model' || object.objectType === 'dbt_source' || object.objectType === 'dbt_column')
|
|
2002
2678
|
return 'dbt_model';
|
|
2003
|
-
if (object.objectType === 'warehouse_table')
|
|
2679
|
+
if (object.objectType === 'warehouse_table' || object.objectType === 'warehouse_column')
|
|
2004
2680
|
return 'warehouse_schema';
|
|
2681
|
+
if (object.objectType === 'runtime_value')
|
|
2682
|
+
return 'value_match';
|
|
2005
2683
|
if (object.objectType === 'runtime_table' || object.objectType === 'runtime_column')
|
|
2006
2684
|
return 'runtime_schema';
|
|
2007
2685
|
if (object.objectType === 'selected_context')
|
|
@@ -2017,6 +2695,8 @@ function evidenceReasonForObject(object) {
|
|
|
2017
2695
|
return 'Semantic definition can ground metric and dimension meaning.';
|
|
2018
2696
|
if (object.objectType.startsWith('dbt_'))
|
|
2019
2697
|
return 'dbt metadata supplies physical model and column context.';
|
|
2698
|
+
if (object.objectType === 'runtime_value')
|
|
2699
|
+
return 'Observed runtime value matched a literal in the question.';
|
|
2020
2700
|
if (object.objectType === 'runtime_table' || object.objectType === 'runtime_column')
|
|
2021
2701
|
return 'Runtime schema supplies executable table and column context.';
|
|
2022
2702
|
return reasonForObject(object);
|
|
@@ -2027,7 +2707,7 @@ function sqlParentObjectsForSelectedColumns(selectedObjects, candidateObjects, q
|
|
|
2027
2707
|
for (const object of candidateObjects) {
|
|
2028
2708
|
if (!isSqlParentObject(object) || selectedKeys.has(object.objectKey))
|
|
2029
2709
|
continue;
|
|
2030
|
-
const relation =
|
|
2710
|
+
const relation = metadataObjectToAllowedSqlRelation(object);
|
|
2031
2711
|
const key = relation ? normalizeRelationKey(relation.relation) : '';
|
|
2032
2712
|
if (!relation || !key)
|
|
2033
2713
|
continue;
|
|
@@ -2039,7 +2719,7 @@ function sqlParentObjectsForSelectedColumns(selectedObjects, candidateObjects, q
|
|
|
2039
2719
|
for (const object of selectedObjects) {
|
|
2040
2720
|
if (!isSqlColumnObject(object))
|
|
2041
2721
|
continue;
|
|
2042
|
-
const relation =
|
|
2722
|
+
const relation = metadataObjectToAllowedSqlRelation(object);
|
|
2043
2723
|
const key = relation ? normalizeRelationKey(relation.relation) : '';
|
|
2044
2724
|
if (!key)
|
|
2045
2725
|
continue;
|
|
@@ -2062,8 +2742,52 @@ function schemaShapeCandidateObjects(catalog, questionPlan, request, runtimeObje
|
|
|
2062
2742
|
if (!questionPlan.needsGeneratedSql)
|
|
2063
2743
|
return [];
|
|
2064
2744
|
const candidates = new Map();
|
|
2745
|
+
for (const candidate of cachedCatalogSchemaShapeCandidates(catalog, questionPlan, request)) {
|
|
2746
|
+
candidates.set(candidate.object.objectKey, candidate);
|
|
2747
|
+
}
|
|
2748
|
+
const considerObject = (object) => {
|
|
2749
|
+
const relation = metadataObjectToAllowedSqlRelation(object);
|
|
2750
|
+
if (!relation || relation.columns.length === 0)
|
|
2751
|
+
return;
|
|
2752
|
+
const shape = schemaShapeMatchForQuestion(relation, questionPlan);
|
|
2753
|
+
if (shape.score <= 0)
|
|
2754
|
+
return;
|
|
2755
|
+
const relationScore = scoreAllowedSqlRelationWithAnalysisPlan(relation, questionPlan);
|
|
2756
|
+
const objectScore = scoreMetadataObjectWithAnalysisPlan(object, questionPlan);
|
|
2757
|
+
const score = Number((shape.score + relationScore.score + objectScore.score).toFixed(3));
|
|
2758
|
+
if (score < schemaShapeMinimumScore(questionPlan))
|
|
2759
|
+
return;
|
|
2760
|
+
candidates.set(object.objectKey, {
|
|
2761
|
+
object,
|
|
2762
|
+
relation,
|
|
2763
|
+
score,
|
|
2764
|
+
reasons: [
|
|
2765
|
+
...shape.reasons,
|
|
2766
|
+
...relationScore.reasons.filter((reason) => /analysis terms|metric terms|dimension terms|columns match|inspected\/projected columns/i.test(reason)).slice(0, 3),
|
|
2767
|
+
],
|
|
2768
|
+
});
|
|
2769
|
+
trimSchemaShapeCandidateMap(candidates, 64);
|
|
2770
|
+
};
|
|
2771
|
+
for (const object of mergeObjects(runtimeObjects.filter(isSqlParentObject))) {
|
|
2772
|
+
considerObject(object);
|
|
2773
|
+
}
|
|
2774
|
+
return Array.from(candidates.values())
|
|
2775
|
+
.sort((a, b) => b.score - a.score || a.relation.relation.localeCompare(b.relation.relation))
|
|
2776
|
+
.slice(0, 24);
|
|
2777
|
+
}
|
|
2778
|
+
function cachedCatalogSchemaShapeCandidates(catalog, questionPlan, request) {
|
|
2779
|
+
const cacheKey = schemaShapeCacheKey(catalog, questionPlan);
|
|
2780
|
+
if (cacheKey) {
|
|
2781
|
+
const cached = schemaShapeCandidateCache.get(cacheKey);
|
|
2782
|
+
if (cached) {
|
|
2783
|
+
schemaShapeCandidateCache.delete(cacheKey);
|
|
2784
|
+
schemaShapeCandidateCache.set(cacheKey, cached);
|
|
2785
|
+
return cached;
|
|
2786
|
+
}
|
|
2787
|
+
}
|
|
2788
|
+
const candidates = new Map();
|
|
2065
2789
|
const considerObject = (object) => {
|
|
2066
|
-
const relation =
|
|
2790
|
+
const relation = metadataObjectToAllowedSqlRelation(object);
|
|
2067
2791
|
if (!relation || relation.columns.length === 0)
|
|
2068
2792
|
return;
|
|
2069
2793
|
const shape = schemaShapeMatchForQuestion(relation, questionPlan);
|
|
@@ -2092,12 +2816,25 @@ function schemaShapeCandidateObjects(catalog, questionPlan, request, runtimeObje
|
|
|
2092
2816
|
for (const object of objects)
|
|
2093
2817
|
considerObject(object);
|
|
2094
2818
|
});
|
|
2095
|
-
|
|
2096
|
-
considerObject(object);
|
|
2097
|
-
}
|
|
2098
|
-
return Array.from(candidates.values())
|
|
2819
|
+
const result = Array.from(candidates.values())
|
|
2099
2820
|
.sort((a, b) => b.score - a.score || a.relation.relation.localeCompare(b.relation.relation))
|
|
2100
2821
|
.slice(0, 24);
|
|
2822
|
+
if (cacheKey) {
|
|
2823
|
+
schemaShapeCandidateCache.set(cacheKey, result);
|
|
2824
|
+
while (schemaShapeCandidateCache.size > SCHEMA_SHAPE_CACHE_LIMIT) {
|
|
2825
|
+
const oldest = schemaShapeCandidateCache.keys().next().value;
|
|
2826
|
+
if (!oldest)
|
|
2827
|
+
break;
|
|
2828
|
+
schemaShapeCandidateCache.delete(oldest);
|
|
2829
|
+
}
|
|
2830
|
+
}
|
|
2831
|
+
return result;
|
|
2832
|
+
}
|
|
2833
|
+
function schemaShapeCacheKey(catalog, questionPlan) {
|
|
2834
|
+
const fingerprint = catalog.state('fingerprint');
|
|
2835
|
+
if (!fingerprint)
|
|
2836
|
+
return null;
|
|
2837
|
+
return `${fingerprint}:${sha256(stableStringify(questionPlan))}`;
|
|
2101
2838
|
}
|
|
2102
2839
|
function schemaShapeScanBatchSize(request) {
|
|
2103
2840
|
if (request.strictness === 'exploratory')
|
|
@@ -2269,6 +3006,32 @@ function isSqlParentObject(object) {
|
|
|
2269
3006
|
object.objectType === 'warehouse_table' ||
|
|
2270
3007
|
object.objectType === 'runtime_table';
|
|
2271
3008
|
}
|
|
3009
|
+
const FULL_CATALOG_RELATION_TYPES = ['dbt_model', 'dbt_source', 'warehouse_table', 'runtime_table'];
|
|
3010
|
+
const FULL_CATALOG_COLUMN_TYPES = ['dbt_column', 'warehouse_column', 'runtime_column'];
|
|
3011
|
+
/** Deep mode over a catalog this small hands the model everything instead of top-k. */
|
|
3012
|
+
const FULL_CATALOG_MAX_RELATIONS = 40;
|
|
3013
|
+
const FULL_CATALOG_MAX_COLUMNS = 2000;
|
|
3014
|
+
/**
|
|
3015
|
+
* When the whole catalog is small enough to fit in context, return every relation
|
|
3016
|
+
* and column object so deep-research mode can reason over the complete schema.
|
|
3017
|
+
* Returns undefined when the catalog exceeds the small-catalog budget, so ranked
|
|
3018
|
+
* top-k selection stays in force for large warehouses.
|
|
3019
|
+
*/
|
|
3020
|
+
function collectFullCatalogObjects(catalog) {
|
|
3021
|
+
const relations = catalog.listObjects({
|
|
3022
|
+
objectTypes: FULL_CATALOG_RELATION_TYPES,
|
|
3023
|
+
limit: FULL_CATALOG_MAX_RELATIONS + 1,
|
|
3024
|
+
});
|
|
3025
|
+
if (relations.length === 0 || relations.length > FULL_CATALOG_MAX_RELATIONS)
|
|
3026
|
+
return undefined;
|
|
3027
|
+
const columns = catalog.listObjects({
|
|
3028
|
+
objectTypes: FULL_CATALOG_COLUMN_TYPES,
|
|
3029
|
+
limit: FULL_CATALOG_MAX_COLUMNS + 1,
|
|
3030
|
+
});
|
|
3031
|
+
if (columns.length > FULL_CATALOG_MAX_COLUMNS)
|
|
3032
|
+
return undefined;
|
|
3033
|
+
return mergeObjects([...relations, ...columns]);
|
|
3034
|
+
}
|
|
2272
3035
|
function buildAllowedSqlContext(objects, edges) {
|
|
2273
3036
|
const byRelation = new Map();
|
|
2274
3037
|
const objectsByKey = new Map(objects.map((object) => [object.objectKey, object]));
|
|
@@ -2281,18 +3044,20 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
2281
3044
|
byRelation.set(key, { ...relation, columns: dedupeRuntimeColumns(relation.columns).slice(0, 120) });
|
|
2282
3045
|
return;
|
|
2283
3046
|
}
|
|
3047
|
+
const columnCompleteness = mergeRelationCompletenessForCatalog(existing, relation);
|
|
2284
3048
|
byRelation.set(key, {
|
|
2285
3049
|
...existing,
|
|
2286
3050
|
objectKey: existing.objectKey ?? relation.objectKey,
|
|
2287
3051
|
source: mergeRelationSources(existing.source, relation.source),
|
|
2288
|
-
|
|
3052
|
+
columnCompleteness,
|
|
3053
|
+
columns: mergeRelationColumns(existing, relation).slice(0, 120),
|
|
2289
3054
|
});
|
|
2290
3055
|
};
|
|
2291
3056
|
for (const object of objects) {
|
|
2292
3057
|
if (object.objectType === 'warehouse_table' && !warehouseTableHasTrustedReference(object, objectsByKey)) {
|
|
2293
3058
|
continue;
|
|
2294
3059
|
}
|
|
2295
|
-
const relation =
|
|
3060
|
+
const relation = metadataObjectToAllowedSqlRelation(object);
|
|
2296
3061
|
if (relation)
|
|
2297
3062
|
addRelation(relation);
|
|
2298
3063
|
if (object.objectType === 'dql_block' && !isCertifiedMetadataObject(object)) {
|
|
@@ -2304,6 +3069,7 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
2304
3069
|
name: table.split('.').at(-1) ?? table,
|
|
2305
3070
|
objectKey: object.objectKey,
|
|
2306
3071
|
source: 'certified block dependency',
|
|
3072
|
+
columnCompleteness: 'partial',
|
|
2307
3073
|
columns: [],
|
|
2308
3074
|
});
|
|
2309
3075
|
}
|
|
@@ -2313,6 +3079,7 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
2313
3079
|
name: table.split('.').at(-1) ?? table,
|
|
2314
3080
|
objectKey: object.objectKey,
|
|
2315
3081
|
source: 'certified block SQL reference',
|
|
3082
|
+
columnCompleteness: 'partial',
|
|
2316
3083
|
columns: [],
|
|
2317
3084
|
});
|
|
2318
3085
|
}
|
|
@@ -2324,6 +3091,7 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
2324
3091
|
name: shape.relation.split('.').at(-1) ?? shape.relation,
|
|
2325
3092
|
objectKey: object.objectKey,
|
|
2326
3093
|
source: 'certified source SQL shape',
|
|
3094
|
+
columnCompleteness: 'partial',
|
|
2327
3095
|
columns: sourceSqlShapeColumns(sourceSql),
|
|
2328
3096
|
});
|
|
2329
3097
|
}
|
|
@@ -2333,11 +3101,12 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
2333
3101
|
continue;
|
|
2334
3102
|
const from = objectsByKey.get(edge.fromKey);
|
|
2335
3103
|
const to = objectsByKey.get(edge.toKey);
|
|
2336
|
-
const fromRelation = from ?
|
|
2337
|
-
const toRelation = to ?
|
|
3104
|
+
const fromRelation = from ? metadataObjectToAllowedSqlRelation(from) : null;
|
|
3105
|
+
const toRelation = to ? metadataObjectToAllowedSqlRelation(to) : null;
|
|
2338
3106
|
if (fromRelation && toRelation)
|
|
2339
|
-
addRelation({ ...fromRelation, columns: toRelation.columns, source: 'dbt mapped warehouse table' });
|
|
3107
|
+
addRelation({ ...fromRelation, columns: toRelation.columns, columnCompleteness: toRelation.columnCompleteness, source: 'dbt mapped warehouse table' });
|
|
2340
3108
|
}
|
|
3109
|
+
applyLineageColumnAliases(byRelation, objectsByKey, edges);
|
|
2341
3110
|
return {
|
|
2342
3111
|
relations: coalesceRelationAliases(Array.from(byRelation.values()))
|
|
2343
3112
|
.sort((a, b) => a.relation.localeCompare(b.relation)),
|
|
@@ -2358,6 +3127,79 @@ function buildAllowedSqlContext(objects, edges) {
|
|
|
2358
3127
|
})),
|
|
2359
3128
|
};
|
|
2360
3129
|
}
|
|
3130
|
+
function applyLineageColumnAliases(byRelation, objectsByKey, edges) {
|
|
3131
|
+
const aliasesByRelationAndColumn = new Map();
|
|
3132
|
+
for (const edge of edges) {
|
|
3133
|
+
if (edge.edgeType !== 'derives_from')
|
|
3134
|
+
continue;
|
|
3135
|
+
const from = objectsByKey.get(edge.fromKey);
|
|
3136
|
+
const to = objectsByKey.get(edge.toKey);
|
|
3137
|
+
if (!from || !to)
|
|
3138
|
+
continue;
|
|
3139
|
+
const targetRelation = metadataObjectToAllowedSqlRelation(to);
|
|
3140
|
+
if (!targetRelation)
|
|
3141
|
+
continue;
|
|
3142
|
+
const alias = semanticAliasFromLineageOutput(from);
|
|
3143
|
+
if (!alias || namesEqualForCatalog(alias, to.name))
|
|
3144
|
+
continue;
|
|
3145
|
+
const relationKey = normalizeRelationKey(targetRelation.relation);
|
|
3146
|
+
const columnKey = normalizeColumnKeyForCatalog(to.name);
|
|
3147
|
+
if (!relationKey || !columnKey)
|
|
3148
|
+
continue;
|
|
3149
|
+
const mapKey = `${relationKey}\u0000${columnKey}`;
|
|
3150
|
+
const existing = aliasesByRelationAndColumn.get(mapKey) ?? new Set();
|
|
3151
|
+
existing.add(alias);
|
|
3152
|
+
aliasesByRelationAndColumn.set(mapKey, existing);
|
|
3153
|
+
}
|
|
3154
|
+
if (aliasesByRelationAndColumn.size === 0)
|
|
3155
|
+
return;
|
|
3156
|
+
for (const [relationKey, relation] of byRelation) {
|
|
3157
|
+
const columns = relation.columns.map((column) => {
|
|
3158
|
+
const aliases = aliasesByRelationAndColumn.get(`${relationKey}\u0000${normalizeColumnKeyForCatalog(column.name)}`);
|
|
3159
|
+
if (!aliases?.size)
|
|
3160
|
+
return column;
|
|
3161
|
+
const aliasText = Array.from(aliases).sort().slice(0, 8).join(', ');
|
|
3162
|
+
const suffix = `Governed aliases from lineage: ${aliasText}.`;
|
|
3163
|
+
const description = column.description
|
|
3164
|
+
? column.description.includes(suffix)
|
|
3165
|
+
? column.description
|
|
3166
|
+
: `${column.description} ${suffix}`
|
|
3167
|
+
: suffix;
|
|
3168
|
+
return { ...column, description };
|
|
3169
|
+
});
|
|
3170
|
+
byRelation.set(relationKey, { ...relation, columns });
|
|
3171
|
+
}
|
|
3172
|
+
}
|
|
3173
|
+
function semanticAliasFromLineageOutput(object) {
|
|
3174
|
+
const candidates = [
|
|
3175
|
+
typeof object.payload?.output === 'string' ? object.payload.output : undefined,
|
|
3176
|
+
object.name,
|
|
3177
|
+
object.fullName?.split('.').at(-1),
|
|
3178
|
+
];
|
|
3179
|
+
for (const candidate of candidates) {
|
|
3180
|
+
const alias = normalizeLineageAlias(candidate);
|
|
3181
|
+
if (alias)
|
|
3182
|
+
return alias;
|
|
3183
|
+
}
|
|
3184
|
+
return undefined;
|
|
3185
|
+
}
|
|
3186
|
+
function normalizeLineageAlias(value) {
|
|
3187
|
+
if (!value)
|
|
3188
|
+
return undefined;
|
|
3189
|
+
const normalized = value.replace(/["`]/g, '').replace(/\s+/g, ' ').trim();
|
|
3190
|
+
if (!normalized || normalized === '*' || normalized === '?')
|
|
3191
|
+
return undefined;
|
|
3192
|
+
if (normalized.length > 64)
|
|
3193
|
+
return undefined;
|
|
3194
|
+
return normalized;
|
|
3195
|
+
}
|
|
3196
|
+
function normalizeColumnKeyForCatalog(value) {
|
|
3197
|
+
return value.replace(/["`]/g, '').trim().toLowerCase();
|
|
3198
|
+
}
|
|
3199
|
+
function namesEqualForCatalog(left, right) {
|
|
3200
|
+
return normalizeColumnKeyForCatalog(left).replace(/[_\s]+/g, '') ===
|
|
3201
|
+
normalizeColumnKeyForCatalog(right).replace(/[_\s]+/g, '');
|
|
3202
|
+
}
|
|
2361
3203
|
// Pull the first natural-language example question off a block payload so the
|
|
2362
3204
|
// certified block can serve as a question→SQL few-shot exemplar.
|
|
2363
3205
|
function firstExampleQuestion(examples) {
|
|
@@ -2374,12 +3216,66 @@ function firstExampleQuestion(examples) {
|
|
|
2374
3216
|
}
|
|
2375
3217
|
return undefined;
|
|
2376
3218
|
}
|
|
2377
|
-
function buildSelectedJoinPaths(allowedSqlContext) {
|
|
3219
|
+
function buildSelectedJoinPaths(allowedSqlContext, edges) {
|
|
2378
3220
|
const relations = allowedSqlContext.relations
|
|
2379
3221
|
.filter((relation) => relation.columns.some((column) => isJoinKeyColumnForCatalog(column.name)))
|
|
2380
3222
|
.slice(0, 16);
|
|
2381
3223
|
const joins = [];
|
|
2382
3224
|
const seen = new Set();
|
|
3225
|
+
const lookup = buildSelectedRelationLookup(relations);
|
|
3226
|
+
const pushJoin = (join) => {
|
|
3227
|
+
const key = catalogJoinDedupeKey(join.leftRelation, join.leftColumn, join.rightRelation, join.rightColumn);
|
|
3228
|
+
if (seen.has(key))
|
|
3229
|
+
return;
|
|
3230
|
+
seen.add(key);
|
|
3231
|
+
joins.push(join);
|
|
3232
|
+
};
|
|
3233
|
+
for (const edge of edges) {
|
|
3234
|
+
if (edge.edgeType !== 'depends_on')
|
|
3235
|
+
continue;
|
|
3236
|
+
const left = lookupRelationForCatalogEdge(edge.fromKey, lookup);
|
|
3237
|
+
const right = lookupRelationForCatalogEdge(edge.toKey, lookup);
|
|
3238
|
+
if (!left || !right || normalizeRelationKey(left.relation) === normalizeRelationKey(right.relation))
|
|
3239
|
+
continue;
|
|
3240
|
+
const join = pickCatalogJoinColumns(left, right);
|
|
3241
|
+
if (!join)
|
|
3242
|
+
continue;
|
|
3243
|
+
pushJoin({
|
|
3244
|
+
leftRelation: left.relation,
|
|
3245
|
+
leftColumn: join.leftColumn,
|
|
3246
|
+
rightRelation: right.relation,
|
|
3247
|
+
rightColumn: join.rightColumn,
|
|
3248
|
+
reason: `dbt lineage: ${left.name} depends_on ${right.name}; ${catalogJoinReason(join.leftColumn, join.rightColumn, join.confidence)}`,
|
|
3249
|
+
confidence: Math.max(join.confidence, 0.98),
|
|
3250
|
+
source: 'dbt_lineage',
|
|
3251
|
+
});
|
|
3252
|
+
}
|
|
3253
|
+
// DataLex modeled relationships: grain-safe joins on the declared canonical
|
|
3254
|
+
// columns (verified to exist on both relations), ranked above name heuristics.
|
|
3255
|
+
const datalexLookup = buildSelectedRelationLookup(allowedSqlContext.relations);
|
|
3256
|
+
for (const edge of edges) {
|
|
3257
|
+
if (edge.edgeType !== 'datalex_join')
|
|
3258
|
+
continue;
|
|
3259
|
+
const left = lookupRelationForCatalogEdge(edge.fromKey, datalexLookup);
|
|
3260
|
+
const right = lookupRelationForCatalogEdge(edge.toKey, datalexLookup);
|
|
3261
|
+
if (!left || !right || normalizeRelationKey(left.relation) === normalizeRelationKey(right.relation))
|
|
3262
|
+
continue;
|
|
3263
|
+
const payload = (edge.payload ?? {});
|
|
3264
|
+
const guessed = pickCatalogJoinColumns(left, right);
|
|
3265
|
+
const leftColumn = relationHasColumn(left, payload.fromColumn) ? payload.fromColumn : guessed?.leftColumn;
|
|
3266
|
+
const rightColumn = relationHasColumn(right, payload.toColumn) ? payload.toColumn : guessed?.rightColumn;
|
|
3267
|
+
if (!leftColumn || !rightColumn)
|
|
3268
|
+
continue;
|
|
3269
|
+
pushJoin({
|
|
3270
|
+
leftRelation: left.relation,
|
|
3271
|
+
leftColumn,
|
|
3272
|
+
rightRelation: right.relation,
|
|
3273
|
+
rightColumn,
|
|
3274
|
+
reason: `DataLex relationship${payload.name ? ` ${payload.name}` : ''}${payload.cardinality ? ` (${payload.cardinality})` : ''}`,
|
|
3275
|
+
confidence: edge.confidence ?? 0.9,
|
|
3276
|
+
source: 'datalex',
|
|
3277
|
+
});
|
|
3278
|
+
}
|
|
2383
3279
|
for (let leftIndex = 0; leftIndex < relations.length; leftIndex += 1) {
|
|
2384
3280
|
for (let rightIndex = leftIndex + 1; rightIndex < relations.length; rightIndex += 1) {
|
|
2385
3281
|
const left = relations[leftIndex];
|
|
@@ -2387,29 +3283,122 @@ function buildSelectedJoinPaths(allowedSqlContext) {
|
|
|
2387
3283
|
const join = pickCatalogJoinColumns(left, right);
|
|
2388
3284
|
if (!join)
|
|
2389
3285
|
continue;
|
|
2390
|
-
const
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
if (seen.has(key))
|
|
2397
|
-
continue;
|
|
2398
|
-
seen.add(key);
|
|
2399
|
-
joins.push({
|
|
2400
|
-
leftRelation: left.relation,
|
|
2401
|
-
leftColumn: join.leftColumn,
|
|
2402
|
-
rightRelation: right.relation,
|
|
2403
|
-
rightColumn: join.rightColumn,
|
|
3286
|
+
const oriented = orientCatalogJoinPath(left, right, join);
|
|
3287
|
+
pushJoin({
|
|
3288
|
+
leftRelation: oriented.leftRelation.relation,
|
|
3289
|
+
leftColumn: oriented.leftColumn,
|
|
3290
|
+
rightRelation: oriented.rightRelation.relation,
|
|
3291
|
+
rightColumn: oriented.rightColumn,
|
|
2404
3292
|
reason: catalogJoinReason(join.leftColumn, join.rightColumn, join.confidence),
|
|
2405
3293
|
confidence: join.confidence,
|
|
3294
|
+
source: 'metadata_guess',
|
|
2406
3295
|
});
|
|
2407
3296
|
}
|
|
2408
3297
|
}
|
|
2409
3298
|
return joins
|
|
2410
|
-
.sort((a, b) => b.confidence - a.confidence || a.leftRelation.localeCompare(b.leftRelation))
|
|
3299
|
+
.sort((a, b) => joinSourceRank(a.source) - joinSourceRank(b.source) || b.confidence - a.confidence || a.leftRelation.localeCompare(b.leftRelation))
|
|
2411
3300
|
.slice(0, 12);
|
|
2412
3301
|
}
|
|
3302
|
+
function buildSelectedRelationLookup(relations) {
|
|
3303
|
+
const byObjectKey = new Map();
|
|
3304
|
+
const byLookupKey = new Map();
|
|
3305
|
+
const addLookup = (key, relation) => {
|
|
3306
|
+
const normalized = normalizeRelationKey(key ?? '');
|
|
3307
|
+
if (!normalized)
|
|
3308
|
+
return;
|
|
3309
|
+
const existing = byLookupKey.get(normalized) ?? [];
|
|
3310
|
+
if (!existing.some((candidate) => normalizeRelationKey(candidate.relation) === normalizeRelationKey(relation.relation))) {
|
|
3311
|
+
existing.push(relation);
|
|
3312
|
+
}
|
|
3313
|
+
byLookupKey.set(normalized, existing);
|
|
3314
|
+
};
|
|
3315
|
+
for (const relation of relations) {
|
|
3316
|
+
if (relation.objectKey)
|
|
3317
|
+
byObjectKey.set(relation.objectKey, relation);
|
|
3318
|
+
addLookup(relation.relation, relation);
|
|
3319
|
+
addLookup(relation.name, relation);
|
|
3320
|
+
addLookup(relationTailKey(relation.relation), relation);
|
|
3321
|
+
}
|
|
3322
|
+
return { byObjectKey, byLookupKey };
|
|
3323
|
+
}
|
|
3324
|
+
function lookupRelationForCatalogEdge(edgeKey, lookup) {
|
|
3325
|
+
const exact = lookup.byObjectKey.get(edgeKey);
|
|
3326
|
+
if (exact)
|
|
3327
|
+
return exact;
|
|
3328
|
+
const tail = edgeKey.split(':').at(-1);
|
|
3329
|
+
const candidates = [
|
|
3330
|
+
edgeKey,
|
|
3331
|
+
tail,
|
|
3332
|
+
tail ? relationTailKey(tail) : undefined,
|
|
3333
|
+
];
|
|
3334
|
+
for (const candidate of candidates) {
|
|
3335
|
+
const normalized = normalizeRelationKey(candidate ?? '');
|
|
3336
|
+
if (!normalized)
|
|
3337
|
+
continue;
|
|
3338
|
+
const matches = lookup.byLookupKey.get(normalized);
|
|
3339
|
+
if (matches?.length === 1)
|
|
3340
|
+
return matches[0];
|
|
3341
|
+
}
|
|
3342
|
+
return undefined;
|
|
3343
|
+
}
|
|
3344
|
+
function catalogJoinDedupeKey(leftRelation, leftColumn, rightRelation, rightColumn) {
|
|
3345
|
+
return [
|
|
3346
|
+
`${normalizeRelationKey(leftRelation)}.${normalizeColumnKeyForCatalog(leftColumn)}`,
|
|
3347
|
+
`${normalizeRelationKey(rightRelation)}.${normalizeColumnKeyForCatalog(rightColumn)}`,
|
|
3348
|
+
].sort().join('|');
|
|
3349
|
+
}
|
|
3350
|
+
/** Map a DataLex physical binding to the metadata object key of its relation. */
|
|
3351
|
+
function objectKeyForDataLexBinding(binding) {
|
|
3352
|
+
switch (binding.kind) {
|
|
3353
|
+
case 'dbt_model': return `dbt:model:${binding.ref}`;
|
|
3354
|
+
case 'dbt_source': return `dbt:source:${binding.ref}`;
|
|
3355
|
+
default: return `warehouse:table:${binding.ref}`;
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
function relationHasColumn(relation, column) {
|
|
3359
|
+
if (!column)
|
|
3360
|
+
return false;
|
|
3361
|
+
const target = normalizeColumnKeyForCatalog(column);
|
|
3362
|
+
return relation.columns.some((entry) => normalizeColumnKeyForCatalog(entry.name) === target);
|
|
3363
|
+
}
|
|
3364
|
+
function joinSourceRank(source) {
|
|
3365
|
+
switch (source) {
|
|
3366
|
+
case 'dbt_lineage': return 0;
|
|
3367
|
+
case 'datalex': return 1;
|
|
3368
|
+
case 'kg_path': return 2;
|
|
3369
|
+
case 'metadata_guess': return 3;
|
|
3370
|
+
default: return 4;
|
|
3371
|
+
}
|
|
3372
|
+
}
|
|
3373
|
+
function orientCatalogJoinPath(left, right, join) {
|
|
3374
|
+
if (catalogRelationFactScore(right) > catalogRelationFactScore(left)) {
|
|
3375
|
+
return {
|
|
3376
|
+
leftRelation: right,
|
|
3377
|
+
leftColumn: join.rightColumn,
|
|
3378
|
+
rightRelation: left,
|
|
3379
|
+
rightColumn: join.leftColumn,
|
|
3380
|
+
};
|
|
3381
|
+
}
|
|
3382
|
+
return {
|
|
3383
|
+
leftRelation: left,
|
|
3384
|
+
leftColumn: join.leftColumn,
|
|
3385
|
+
rightRelation: right,
|
|
3386
|
+
rightColumn: join.rightColumn,
|
|
3387
|
+
};
|
|
3388
|
+
}
|
|
3389
|
+
function catalogRelationFactScore(relation) {
|
|
3390
|
+
const text = normalizeRelationKey([relation.name, relation.relation].join(' ')).replace(/[._-]+/g, ' ');
|
|
3391
|
+
let score = 0;
|
|
3392
|
+
if (/\b(fct|fact)\b/i.test(text))
|
|
3393
|
+
score += 20;
|
|
3394
|
+
if (/\b(order items?|orders?|events?|transactions?|performance|activity|usage|revenue|sales|payments?|line items?|stats?)\b/i.test(text))
|
|
3395
|
+
score += 8;
|
|
3396
|
+
if (/\b(dim|dimension)\b/i.test(text))
|
|
3397
|
+
score -= 18;
|
|
3398
|
+
if (/\b(customers?|products?|players?|accounts?|users?|segments?|regions?)\b/i.test(text))
|
|
3399
|
+
score -= 4;
|
|
3400
|
+
return score;
|
|
3401
|
+
}
|
|
2413
3402
|
function pickCatalogJoinColumns(left, right) {
|
|
2414
3403
|
const candidates = [];
|
|
2415
3404
|
const leftColumns = left.columns.filter((column) => isJoinKeyColumnForCatalog(column.name)).slice(0, 32);
|
|
@@ -2515,7 +3504,8 @@ function coalesceRelationAliases(relations) {
|
|
|
2515
3504
|
...target,
|
|
2516
3505
|
objectKey: target.objectKey ?? relation.objectKey,
|
|
2517
3506
|
source: mergeRelationSources(target.source, relation.source),
|
|
2518
|
-
|
|
3507
|
+
columnCompleteness: mergeRelationCompletenessForCatalog(target, relation),
|
|
3508
|
+
columns: mergeRelationColumns(target, relation).slice(0, 120),
|
|
2519
3509
|
});
|
|
2520
3510
|
skipped.add(relationKey);
|
|
2521
3511
|
}
|
|
@@ -2523,6 +3513,31 @@ function coalesceRelationAliases(relations) {
|
|
|
2523
3513
|
.filter(([key]) => !skipped.has(key))
|
|
2524
3514
|
.map(([, relation]) => relation);
|
|
2525
3515
|
}
|
|
3516
|
+
function mergeRelationColumns(existing, incoming) {
|
|
3517
|
+
const existingIsSourceShape = relationSourceIncludes(existing.source, 'certified source SQL shape');
|
|
3518
|
+
const incomingIsSourceShape = relationSourceIncludes(incoming.source, 'certified source SQL shape');
|
|
3519
|
+
if (existingIsSourceShape && !incomingIsSourceShape) {
|
|
3520
|
+
return dedupeRuntimeColumns([...incoming.columns, ...existing.columns]);
|
|
3521
|
+
}
|
|
3522
|
+
if (!existingIsSourceShape && incomingIsSourceShape) {
|
|
3523
|
+
return dedupeRuntimeColumns([...existing.columns, ...incoming.columns]);
|
|
3524
|
+
}
|
|
3525
|
+
return dedupeRuntimeColumns([...existing.columns, ...incoming.columns]);
|
|
3526
|
+
}
|
|
3527
|
+
function mergeRelationCompletenessForCatalog(existing, incoming) {
|
|
3528
|
+
if (relationCompletenessValue(existing) === 'complete' || relationCompletenessValue(incoming) === 'complete') {
|
|
3529
|
+
return 'complete';
|
|
3530
|
+
}
|
|
3531
|
+
return 'partial';
|
|
3532
|
+
}
|
|
3533
|
+
function relationCompletenessValue(relation) {
|
|
3534
|
+
if (relation.columnCompleteness)
|
|
3535
|
+
return relation.columnCompleteness;
|
|
3536
|
+
return relation.columns.length === 0 ? 'partial' : 'complete';
|
|
3537
|
+
}
|
|
3538
|
+
function relationSourceIncludes(source, value) {
|
|
3539
|
+
return Boolean(source && source.split(/\s+\+\s+/).some((part) => part.trim() === value));
|
|
3540
|
+
}
|
|
2526
3541
|
function relationTailKey(relation) {
|
|
2527
3542
|
const parts = normalizeRelationKey(relation).split('.').filter(Boolean);
|
|
2528
3543
|
return parts.at(-1) ?? '';
|
|
@@ -2547,8 +3562,26 @@ function warehouseTableHasTrustedReference(object, objectsByKey) {
|
|
|
2547
3562
|
return block ? isCertifiedMetadataObject(block) : false;
|
|
2548
3563
|
});
|
|
2549
3564
|
}
|
|
2550
|
-
function
|
|
2551
|
-
if (object.objectType === '
|
|
3565
|
+
export function metadataObjectToAllowedSqlRelation(object) {
|
|
3566
|
+
if (object.objectType === 'runtime_value') {
|
|
3567
|
+
const relation = metadataPayloadString(object, 'relation');
|
|
3568
|
+
const columnName = metadataPayloadString(object, 'column');
|
|
3569
|
+
if (!relation || !columnName)
|
|
3570
|
+
return null;
|
|
3571
|
+
return {
|
|
3572
|
+
relation,
|
|
3573
|
+
name: relation.split('.').at(-1) ?? relation,
|
|
3574
|
+
objectKey: object.objectKey,
|
|
3575
|
+
source: 'runtime value index',
|
|
3576
|
+
columnCompleteness: 'partial',
|
|
3577
|
+
columns: [{
|
|
3578
|
+
name: columnName,
|
|
3579
|
+
type: metadataPayloadString(object, 'type'),
|
|
3580
|
+
sampleValues: metadataStringArray(object.payload?.sampleValues),
|
|
3581
|
+
}],
|
|
3582
|
+
};
|
|
3583
|
+
}
|
|
3584
|
+
if (object.objectType === 'dbt_column' || object.objectType === 'runtime_column' || object.objectType === 'warehouse_column') {
|
|
2552
3585
|
const relation = metadataPayloadString(object, 'relation');
|
|
2553
3586
|
if (!relation)
|
|
2554
3587
|
return null;
|
|
@@ -2556,7 +3589,12 @@ function metadataRelationFromObject(object) {
|
|
|
2556
3589
|
relation,
|
|
2557
3590
|
name: relation.split('.').at(-1) ?? relation,
|
|
2558
3591
|
objectKey: object.objectKey,
|
|
2559
|
-
source: object.objectType === 'runtime_column'
|
|
3592
|
+
source: object.objectType === 'runtime_column'
|
|
3593
|
+
? 'runtime schema snapshot'
|
|
3594
|
+
: object.objectType === 'warehouse_column'
|
|
3595
|
+
? 'DQL block column lineage'
|
|
3596
|
+
: 'dbt manifest',
|
|
3597
|
+
columnCompleteness: 'partial',
|
|
2560
3598
|
columns: [{
|
|
2561
3599
|
name: object.name,
|
|
2562
3600
|
type: metadataPayloadString(object, 'type') ?? metadataPayloadString(object, 'data_type'),
|
|
@@ -2573,9 +3611,21 @@ function metadataRelationFromObject(object) {
|
|
|
2573
3611
|
name: relation.split('.').at(-1) ?? object.name,
|
|
2574
3612
|
objectKey: object.objectKey,
|
|
2575
3613
|
source: object.objectType === 'runtime_table' ? 'runtime schema snapshot' : object.sourceSystem ?? 'local metadata catalog',
|
|
3614
|
+
columnCompleteness: relationColumnCompletenessFromObject(object),
|
|
2576
3615
|
columns: metadataRuntimeColumns(object.payload?.columns),
|
|
2577
3616
|
};
|
|
2578
3617
|
}
|
|
3618
|
+
function relationColumnCompletenessFromObject(object) {
|
|
3619
|
+
if (object.objectType === 'runtime_table')
|
|
3620
|
+
return 'complete';
|
|
3621
|
+
if (object.payload?.columnCompleteness === 'complete' || object.payload?.catalogColumnCompleteness === 'complete')
|
|
3622
|
+
return 'complete';
|
|
3623
|
+
if (object.payload?.columnCompleteness === 'partial' || object.payload?.catalogColumnCompleteness === 'partial')
|
|
3624
|
+
return 'partial';
|
|
3625
|
+
if (object.objectType === 'warehouse_table')
|
|
3626
|
+
return 'partial';
|
|
3627
|
+
return 'partial';
|
|
3628
|
+
}
|
|
2579
3629
|
function runtimeSchemaObjects(snapshot) {
|
|
2580
3630
|
const capturedAt = snapshot.capturedAt ?? new Date().toISOString();
|
|
2581
3631
|
return normalizeRuntimeSchemaTables(snapshot.tables).flatMap((table) => {
|
|
@@ -2592,6 +3642,7 @@ function runtimeSchemaObjects(snapshot) {
|
|
|
2592
3642
|
payload: compactObject({
|
|
2593
3643
|
relation,
|
|
2594
3644
|
schema: table.schema,
|
|
3645
|
+
columnCompleteness: 'complete',
|
|
2595
3646
|
columns: table.columns,
|
|
2596
3647
|
}),
|
|
2597
3648
|
updatedAt: capturedAt,
|
|
@@ -2607,6 +3658,7 @@ function runtimeSchemaObjects(snapshot) {
|
|
|
2607
3658
|
payload: compactObject({
|
|
2608
3659
|
relation,
|
|
2609
3660
|
type: column.type,
|
|
3661
|
+
columnCompleteness: 'partial',
|
|
2610
3662
|
sampleValues: column.sampleValues,
|
|
2611
3663
|
}),
|
|
2612
3664
|
updatedAt: capturedAt,
|
|
@@ -2614,6 +3666,30 @@ function runtimeSchemaObjects(snapshot) {
|
|
|
2614
3666
|
return [tableObject, ...columns];
|
|
2615
3667
|
});
|
|
2616
3668
|
}
|
|
3669
|
+
function runtimeValueMatchObjects(matches) {
|
|
3670
|
+
return matches.map((match) => ({
|
|
3671
|
+
objectKey: match.valueKey,
|
|
3672
|
+
objectType: 'runtime_value',
|
|
3673
|
+
name: `${match.columnName} = ${match.value}`,
|
|
3674
|
+
fullName: `${match.relation}.${match.columnName}:${match.value}`,
|
|
3675
|
+
description: `Observed value "${match.value}" in ${match.relation}.${match.columnName}.`,
|
|
3676
|
+
status: 'runtime_observed',
|
|
3677
|
+
sourceSystem: match.source ?? 'runtime value index',
|
|
3678
|
+
score: match.score,
|
|
3679
|
+
payload: compactObject({
|
|
3680
|
+
relation: match.relation,
|
|
3681
|
+
schema: match.schema,
|
|
3682
|
+
table: match.tableName,
|
|
3683
|
+
column: match.columnName,
|
|
3684
|
+
type: match.columnType,
|
|
3685
|
+
value: match.value,
|
|
3686
|
+
normalizedValue: match.normalizedValue,
|
|
3687
|
+
sampleValues: [match.value],
|
|
3688
|
+
capturedAt: match.capturedAt,
|
|
3689
|
+
}),
|
|
3690
|
+
updatedAt: match.capturedAt,
|
|
3691
|
+
}));
|
|
3692
|
+
}
|
|
2617
3693
|
function selectedContextObjects(value) {
|
|
2618
3694
|
const root = value && typeof value === 'object' ? value : null;
|
|
2619
3695
|
if (!root)
|
|
@@ -2643,7 +3719,13 @@ function normalizeFollowUpContext(value) {
|
|
|
2643
3719
|
if (!value || typeof value !== 'object')
|
|
2644
3720
|
return null;
|
|
2645
3721
|
const record = value;
|
|
2646
|
-
const kind = record.kind === 'drilldown'
|
|
3722
|
+
const kind = record.kind === 'drilldown'
|
|
3723
|
+
? 'drilldown'
|
|
3724
|
+
: record.kind === 'generic'
|
|
3725
|
+
? 'generic'
|
|
3726
|
+
: record.kind === 'contextual'
|
|
3727
|
+
? 'contextual'
|
|
3728
|
+
: null;
|
|
2647
3729
|
if (!kind)
|
|
2648
3730
|
return null;
|
|
2649
3731
|
return {
|
|
@@ -2653,8 +3735,77 @@ function normalizeFollowUpContext(value) {
|
|
|
2653
3735
|
sourceAnswer: stringValue(record.sourceAnswer),
|
|
2654
3736
|
filters: metadataStringArray(record.filters),
|
|
2655
3737
|
dimensions: metadataStringArray(record.dimensions),
|
|
3738
|
+
priorResultColumns: metadataStringArray(record.priorResultColumns),
|
|
3739
|
+
priorResultValues: metadataStringRecordArray(record.priorResultValues),
|
|
3740
|
+
priorResultRef: normalizePriorResultRef(record.priorResultRef),
|
|
3741
|
+
priorDqlArtifact: normalizePriorDqlArtifact(record.priorDqlArtifact),
|
|
3742
|
+
priorLimit: typeof record.priorLimit === 'number' ? record.priorLimit : undefined,
|
|
3743
|
+
priorMeasures: metadataStringArray(record.priorMeasures),
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
function normalizePriorResultRef(value) {
|
|
3747
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
3748
|
+
return undefined;
|
|
3749
|
+
const record = value;
|
|
3750
|
+
const id = stringValue(record.id);
|
|
3751
|
+
const columns = metadataStringArray(record.columns) ?? [];
|
|
3752
|
+
if (!id || columns.length === 0)
|
|
3753
|
+
return undefined;
|
|
3754
|
+
const rowCount = typeof record.rowCount === 'number' && Number.isFinite(record.rowCount)
|
|
3755
|
+
? record.rowCount
|
|
3756
|
+
: undefined;
|
|
3757
|
+
const sourceSql = stringValue(record.sourceSql);
|
|
3758
|
+
return {
|
|
3759
|
+
id,
|
|
3760
|
+
question: stringValue(record.question),
|
|
3761
|
+
columns: columns.slice(0, 32),
|
|
3762
|
+
rowCount,
|
|
3763
|
+
sourceSql: sourceSql ? sourceSql.slice(0, 1200) : undefined,
|
|
2656
3764
|
};
|
|
2657
3765
|
}
|
|
3766
|
+
function normalizePriorDqlArtifact(value) {
|
|
3767
|
+
const artifact = normalizeDqlArtifactReference(value);
|
|
3768
|
+
if (!artifact)
|
|
3769
|
+
return undefined;
|
|
3770
|
+
return {
|
|
3771
|
+
...artifact,
|
|
3772
|
+
source: artifact.source.slice(0, 3000),
|
|
3773
|
+
metrics: artifact.metrics?.slice(0, 32),
|
|
3774
|
+
dimensions: artifact.dimensions?.slice(0, 32),
|
|
3775
|
+
filters: artifact.filters
|
|
3776
|
+
?.filter((filter) => filter.values.length > 0)
|
|
3777
|
+
.slice(0, 12)
|
|
3778
|
+
.map((filter) => ({ ...filter, values: filter.values.slice(0, 12) })),
|
|
3779
|
+
orderBy: artifact.orderBy?.slice(0, 12),
|
|
3780
|
+
};
|
|
3781
|
+
}
|
|
3782
|
+
/**
|
|
3783
|
+
* Deterministic route-commitment gate: true only when the new plan differs from
|
|
3784
|
+
* the prior one in FILTERS / TOP-N / TIMEFRAME alone. Any change to measures,
|
|
3785
|
+
* dimensions, grain, required outputs, or mode disqualifies reuse — those change
|
|
3786
|
+
* which artifacts/relations can answer, so the full pipeline must re-run.
|
|
3787
|
+
*/
|
|
3788
|
+
export function isFilterOnlyRefinement(prior, next) {
|
|
3789
|
+
if (prior.mode !== next.mode)
|
|
3790
|
+
return false;
|
|
3791
|
+
const priorShape = prior.requestedShape;
|
|
3792
|
+
const nextShape = next.requestedShape;
|
|
3793
|
+
return setEqual(priorShape.measures, nextShape.measures)
|
|
3794
|
+
&& setEqual(priorShape.dimensions, nextShape.dimensions)
|
|
3795
|
+
&& setEqual(priorShape.requiredOutputs, nextShape.requiredOutputs)
|
|
3796
|
+
&& (priorShape.grain ?? '') === (nextShape.grain ?? '')
|
|
3797
|
+
&& setEqual(prior.entities.map((entity) => entity.text.toLowerCase()), next.entities.map((entity) => entity.text.toLowerCase()));
|
|
3798
|
+
}
|
|
3799
|
+
function setEqual(a, b) {
|
|
3800
|
+
const setA = new Set(a.map((value) => value.toLowerCase()));
|
|
3801
|
+
const setB = new Set(b.map((value) => value.toLowerCase()));
|
|
3802
|
+
if (setA.size !== setB.size)
|
|
3803
|
+
return false;
|
|
3804
|
+
for (const value of setA)
|
|
3805
|
+
if (!setB.has(value))
|
|
3806
|
+
return false;
|
|
3807
|
+
return true;
|
|
3808
|
+
}
|
|
2658
3809
|
function followUpSourceObjectKeys(followUp) {
|
|
2659
3810
|
if (!followUp?.sourceBlockName)
|
|
2660
3811
|
return [];
|
|
@@ -2670,11 +3821,22 @@ function followUpContextObjects(followUp) {
|
|
|
2670
3821
|
followUp.sourceAnswer ?? '',
|
|
2671
3822
|
...(followUp.filters ?? []),
|
|
2672
3823
|
...(followUp.dimensions ?? []),
|
|
3824
|
+
...(followUp.priorResultColumns ?? []),
|
|
3825
|
+
followUp.priorResultRef?.id ? `result:${followUp.priorResultRef.id}` : '',
|
|
3826
|
+
followUp.priorResultRef?.question ?? '',
|
|
3827
|
+
...(followUp.priorResultRef?.columns ?? []),
|
|
3828
|
+
followUp.priorResultRef?.sourceSql ? followUp.priorResultRef.sourceSql.slice(0, 600) : '',
|
|
3829
|
+
...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: true }),
|
|
3830
|
+
...Object.entries(followUp.priorResultValues ?? {}).flatMap(([key, values]) => [key, ...values]),
|
|
2673
3831
|
].join(' ');
|
|
2674
3832
|
return [{
|
|
2675
3833
|
objectKey: `selected:followup:${sha256(stableStringify(followUp)).slice(0, 16)}`,
|
|
2676
3834
|
objectType: 'selected_context',
|
|
2677
|
-
name: followUp.kind === 'drilldown'
|
|
3835
|
+
name: followUp.kind === 'drilldown'
|
|
3836
|
+
? 'Follow-up drilldown request'
|
|
3837
|
+
: followUp.kind === 'contextual'
|
|
3838
|
+
? 'Conversation context (advisory)'
|
|
3839
|
+
: 'Follow-up request',
|
|
2678
3840
|
description: text.trim() || undefined,
|
|
2679
3841
|
status: 'transient_context',
|
|
2680
3842
|
sourceSystem: 'agent follow-up context',
|
|
@@ -2685,20 +3847,112 @@ function followUpContextObjects(followUp) {
|
|
|
2685
3847
|
sourceAnswer: followUp.sourceAnswer,
|
|
2686
3848
|
filters: followUp.filters,
|
|
2687
3849
|
dimensions: followUp.dimensions,
|
|
3850
|
+
priorResultColumns: followUp.priorResultColumns,
|
|
3851
|
+
priorResultValues: followUp.priorResultValues,
|
|
3852
|
+
priorResultRef: followUp.priorResultRef,
|
|
3853
|
+
priorDqlArtifact: followUp.priorDqlArtifact,
|
|
3854
|
+
priorLimit: followUp.priorLimit,
|
|
3855
|
+
priorMeasures: followUp.priorMeasures,
|
|
2688
3856
|
}),
|
|
2689
3857
|
}];
|
|
2690
3858
|
}
|
|
3859
|
+
function metadataValueSearchTerms(question, questionPlan, followUp) {
|
|
3860
|
+
const terms = [
|
|
3861
|
+
...questionPlan.entities.map((entity) => entity.text),
|
|
3862
|
+
...questionPlan.filterTerms,
|
|
3863
|
+
...(followUp?.filters ?? []),
|
|
3864
|
+
...Object.values(followUp?.priorResultValues ?? {}).flat(),
|
|
3865
|
+
];
|
|
3866
|
+
for (const match of question.matchAll(/["']([^"']{2,120})["']/g))
|
|
3867
|
+
terms.push(match[1]);
|
|
3868
|
+
for (const match of question.matchAll(/\b[\w.%+-]+@[\w.-]+\.[A-Za-z]{2,}\b/g))
|
|
3869
|
+
terms.push(match[0]);
|
|
3870
|
+
for (const match of question.matchAll(/\b[A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,4}\b/g))
|
|
3871
|
+
terms.push(match[0]);
|
|
3872
|
+
for (const match of question.matchAll(/\b(?:for|named|called|only|where|customer|user|account|product|region|segment|category|status)\s+([A-Za-z0-9@._-]+(?:\s+[A-Za-z0-9@._-]+){0,4})/gi)) {
|
|
3873
|
+
terms.push(match[1]);
|
|
3874
|
+
}
|
|
3875
|
+
return uniqueStrings(terms
|
|
3876
|
+
.map(cleanMetadataValueSearchTerm)
|
|
3877
|
+
.filter((term) => term.length >= 2 && !METADATA_VALUE_STOP_TERMS.has(term.toLowerCase()))).slice(0, 10);
|
|
3878
|
+
}
|
|
3879
|
+
function cleanMetadataValueSearchTerm(term) {
|
|
3880
|
+
return term
|
|
3881
|
+
.replace(/[?.,;:]+$/g, '')
|
|
3882
|
+
.replace(/\s+/g, ' ')
|
|
3883
|
+
.trim()
|
|
3884
|
+
.replace(/^(?:account|category|customer|member|named|called|only|product|region|segment|sku|status|subscriber|user|where)\s+/i, '')
|
|
3885
|
+
.replace(/\s+\b(?:last|next|this)\b.*$/i, '')
|
|
3886
|
+
.replace(/\s+\b(?:daily|weekly|monthly|quarterly|yearly)\b.*$/i, '')
|
|
3887
|
+
.trim();
|
|
3888
|
+
}
|
|
3889
|
+
const METADATA_VALUE_STOP_TERMS = new Set([
|
|
3890
|
+
'account',
|
|
3891
|
+
'category',
|
|
3892
|
+
'customer',
|
|
3893
|
+
'last month',
|
|
3894
|
+
'last quarter',
|
|
3895
|
+
'last week',
|
|
3896
|
+
'last year',
|
|
3897
|
+
'member',
|
|
3898
|
+
'product',
|
|
3899
|
+
'region',
|
|
3900
|
+
'segment',
|
|
3901
|
+
'sku',
|
|
3902
|
+
'status',
|
|
3903
|
+
'this month',
|
|
3904
|
+
'this quarter',
|
|
3905
|
+
'this week',
|
|
3906
|
+
'this year',
|
|
3907
|
+
'user',
|
|
3908
|
+
]);
|
|
2691
3909
|
function buildFollowUpSearchQuery(question, followUp) {
|
|
2692
3910
|
if (!followUp)
|
|
2693
3911
|
return question;
|
|
3912
|
+
// Contextual carry is advisory: enrich retrieval softly (prior question, columns,
|
|
3913
|
+
// measures) but never with concrete dimension VALUES or the block name — those pull
|
|
3914
|
+
// ranking hard toward the old topic when the user may be switching subjects. The
|
|
3915
|
+
// question-plan's own raw queries still run alongside this one either way.
|
|
3916
|
+
if (followUp.kind === 'contextual') {
|
|
3917
|
+
return [
|
|
3918
|
+
question,
|
|
3919
|
+
followUp.sourceQuestion ?? '',
|
|
3920
|
+
...(followUp.priorResultColumns ?? []),
|
|
3921
|
+
followUp.priorResultRef?.question ?? '',
|
|
3922
|
+
...(followUp.priorResultRef?.columns ?? []),
|
|
3923
|
+
...(followUp.priorMeasures ?? []),
|
|
3924
|
+
...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: false }),
|
|
3925
|
+
].filter(Boolean).join(' ');
|
|
3926
|
+
}
|
|
2694
3927
|
return [
|
|
2695
3928
|
question,
|
|
2696
3929
|
followUp.sourceBlockName ?? '',
|
|
2697
3930
|
followUp.sourceQuestion ?? '',
|
|
2698
3931
|
...(followUp.filters ?? []),
|
|
2699
3932
|
...(followUp.dimensions ?? []),
|
|
3933
|
+
...(followUp.priorResultColumns ?? []),
|
|
3934
|
+
followUp.priorResultRef?.question ?? '',
|
|
3935
|
+
...(followUp.priorResultRef?.columns ?? []),
|
|
3936
|
+
followUp.priorResultRef?.sourceSql ? followUp.priorResultRef.sourceSql.slice(0, 600) : '',
|
|
3937
|
+
...dqlArtifactSearchTerms(followUp.priorDqlArtifact, { includeSource: true }),
|
|
3938
|
+
...Object.entries(followUp.priorResultValues ?? {}).flatMap(([key, values]) => [key, ...values]),
|
|
2700
3939
|
].filter(Boolean).join(' ');
|
|
2701
3940
|
}
|
|
3941
|
+
function dqlArtifactSearchTerms(artifact, options) {
|
|
3942
|
+
if (!artifact)
|
|
3943
|
+
return [];
|
|
3944
|
+
return [
|
|
3945
|
+
artifact.name ?? '',
|
|
3946
|
+
artifact.sourcePath ?? '',
|
|
3947
|
+
...(artifact.metrics ?? []),
|
|
3948
|
+
...(artifact.dimensions ?? []),
|
|
3949
|
+
...(artifact.filters ?? []).flatMap((filter) => [filter.dimension, filter.operator, ...filter.values]),
|
|
3950
|
+
artifact.timeDimension ? `${artifact.timeDimension.name} ${artifact.timeDimension.granularity}` : '',
|
|
3951
|
+
...(artifact.orderBy ?? []).map((order) => `${order.name} ${order.direction}`),
|
|
3952
|
+
typeof artifact.limit === 'number' ? `limit ${artifact.limit}` : '',
|
|
3953
|
+
options.includeSource ? artifact.source.slice(0, 900) : '',
|
|
3954
|
+
].filter(Boolean);
|
|
3955
|
+
}
|
|
2702
3956
|
function uniqueMetadataSearchQueries(values) {
|
|
2703
3957
|
const seen = new Set();
|
|
2704
3958
|
const out = [];
|
|
@@ -2797,6 +4051,29 @@ function metadataStringArray(value) {
|
|
|
2797
4051
|
return [];
|
|
2798
4052
|
return value.filter((item) => typeof item === 'string' && item.trim().length > 0).map((item) => item.trim());
|
|
2799
4053
|
}
|
|
4054
|
+
function uniqueStrings(values) {
|
|
4055
|
+
const seen = new Set();
|
|
4056
|
+
const out = [];
|
|
4057
|
+
for (const value of values) {
|
|
4058
|
+
const normalized = value.toLowerCase();
|
|
4059
|
+
if (!normalized || seen.has(normalized))
|
|
4060
|
+
continue;
|
|
4061
|
+
seen.add(normalized);
|
|
4062
|
+
out.push(value);
|
|
4063
|
+
}
|
|
4064
|
+
return out;
|
|
4065
|
+
}
|
|
4066
|
+
function metadataStringRecordArray(value) {
|
|
4067
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
4068
|
+
return undefined;
|
|
4069
|
+
const out = {};
|
|
4070
|
+
for (const [key, raw] of Object.entries(value)) {
|
|
4071
|
+
const values = metadataStringArray(raw).slice(0, 24);
|
|
4072
|
+
if (key.trim() && values.length > 0)
|
|
4073
|
+
out[key.trim()] = values;
|
|
4074
|
+
}
|
|
4075
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
4076
|
+
}
|
|
2800
4077
|
function metadataPayloadString(object, key) {
|
|
2801
4078
|
return stringValue(object.payload?.[key]);
|
|
2802
4079
|
}
|
|
@@ -2967,6 +4244,25 @@ function rowToQueryRun(row) {
|
|
|
2967
4244
|
createdAt: row.created_at,
|
|
2968
4245
|
};
|
|
2969
4246
|
}
|
|
4247
|
+
function rowToRuntimeValueMatch(row, terms) {
|
|
4248
|
+
const normalized = row.normalized_value;
|
|
4249
|
+
const exactBoost = terms.some((term) => term === normalized) ? 1 : 0;
|
|
4250
|
+
const containsBoost = terms.some((term) => normalized.includes(term) || term.includes(normalized)) ? 0.5 : 0;
|
|
4251
|
+
const rankScore = row.rank ? 1 / (1 + Math.max(0, row.rank)) : 1;
|
|
4252
|
+
return {
|
|
4253
|
+
valueKey: row.value_key,
|
|
4254
|
+
relation: row.relation,
|
|
4255
|
+
schema: row.schema_name ?? undefined,
|
|
4256
|
+
tableName: row.table_name ?? undefined,
|
|
4257
|
+
columnName: row.column_name,
|
|
4258
|
+
columnType: row.column_type ?? undefined,
|
|
4259
|
+
value: row.value,
|
|
4260
|
+
normalizedValue: row.normalized_value,
|
|
4261
|
+
source: row.source ?? undefined,
|
|
4262
|
+
capturedAt: row.captured_at,
|
|
4263
|
+
score: Number((rankScore + exactBoost + containsBoost).toFixed(3)),
|
|
4264
|
+
};
|
|
4265
|
+
}
|
|
2970
4266
|
function safeJson(raw, fallback) {
|
|
2971
4267
|
if (!raw)
|
|
2972
4268
|
return fallback;
|
|
@@ -2994,22 +4290,61 @@ function rankMetadataObjects(args) {
|
|
|
2994
4290
|
})
|
|
2995
4291
|
.sort((a, b) => b.score - a.score || objectPriority(a.row) - objectPriority(b.row) || a.row.name.localeCompare(b.row.name))
|
|
2996
4292
|
.map((item, index) => ({ ...item, rank: index + 1 }));
|
|
2997
|
-
const
|
|
4293
|
+
const selectedRanked = selectRankedMetadataObjects(ranked, args.limit);
|
|
4294
|
+
const selectedKeys = new Set(selectedRanked.map((item) => item.row.objectKey));
|
|
4295
|
+
const rejectedRanked = ranked.filter((item) => !selectedKeys.has(item.row.objectKey));
|
|
4296
|
+
const cutoff = selectedRanked.at(-1);
|
|
2998
4297
|
return {
|
|
2999
|
-
selected:
|
|
4298
|
+
selected: selectedRanked.map((item) => item.row),
|
|
3000
4299
|
ranked,
|
|
3001
|
-
rejected:
|
|
4300
|
+
rejected: rejectedRanked.slice(0, 24).map((item) => ({
|
|
3002
4301
|
objectKey: item.row.objectKey,
|
|
3003
4302
|
objectType: item.row.objectType,
|
|
3004
4303
|
name: item.row.name,
|
|
3005
4304
|
reason: cutoff
|
|
3006
|
-
? `
|
|
4305
|
+
? `Outside balanced context window (cutoff ${cutoff.score.toFixed(1)}); ${item.reason}`
|
|
3007
4306
|
: item.reason,
|
|
3008
4307
|
score: item.score,
|
|
3009
4308
|
rejectedRank: item.rank,
|
|
3010
4309
|
})),
|
|
3011
4310
|
};
|
|
3012
4311
|
}
|
|
4312
|
+
function selectRankedMetadataObjects(ranked, limit) {
|
|
4313
|
+
if (ranked.length <= limit)
|
|
4314
|
+
return ranked;
|
|
4315
|
+
const columnCount = ranked.filter((item) => COLUMN_OBJECT_TYPES.has(item.row.objectType)).length;
|
|
4316
|
+
const nonColumnCount = ranked.length - columnCount;
|
|
4317
|
+
if (columnCount === 0 || nonColumnCount === 0)
|
|
4318
|
+
return ranked.slice(0, limit);
|
|
4319
|
+
const columnCap = Math.max(4, Math.floor(limit * 0.35));
|
|
4320
|
+
const selected = [];
|
|
4321
|
+
const deferredColumns = [];
|
|
4322
|
+
let selectedColumns = 0;
|
|
4323
|
+
for (const item of ranked) {
|
|
4324
|
+
if (selected.length >= limit)
|
|
4325
|
+
break;
|
|
4326
|
+
if (COLUMN_OBJECT_TYPES.has(item.row.objectType)) {
|
|
4327
|
+
if (selectedColumns >= columnCap) {
|
|
4328
|
+
deferredColumns.push(item);
|
|
4329
|
+
continue;
|
|
4330
|
+
}
|
|
4331
|
+
selectedColumns += 1;
|
|
4332
|
+
}
|
|
4333
|
+
selected.push(item);
|
|
4334
|
+
}
|
|
4335
|
+
if (selected.length < limit) {
|
|
4336
|
+
const selectedKeys = new Set(selected.map((item) => item.row.objectKey));
|
|
4337
|
+
for (const item of [...deferredColumns, ...ranked]) {
|
|
4338
|
+
if (selected.length >= limit)
|
|
4339
|
+
break;
|
|
4340
|
+
if (selectedKeys.has(item.row.objectKey))
|
|
4341
|
+
continue;
|
|
4342
|
+
selected.push(item);
|
|
4343
|
+
selectedKeys.add(item.row.objectKey);
|
|
4344
|
+
}
|
|
4345
|
+
}
|
|
4346
|
+
return selected.sort((a, b) => a.rank - b.rank);
|
|
4347
|
+
}
|
|
3013
4348
|
function scoreMetadataObject(row, terms) {
|
|
3014
4349
|
let score = row.score ? row.score * 10 : 0;
|
|
3015
4350
|
score += Math.max(0, 44 - objectPriority(row) * 2);
|
|
@@ -3023,6 +4358,8 @@ function scoreMetadataObject(row, terms) {
|
|
|
3023
4358
|
score -= 16;
|
|
3024
4359
|
if (row.objectType === 'semantic_metric')
|
|
3025
4360
|
score += 10;
|
|
4361
|
+
if (row.objectType === 'runtime_value')
|
|
4362
|
+
score += 14;
|
|
3026
4363
|
if (row.objectType === 'dbt_model' || row.objectType === 'dbt_column')
|
|
3027
4364
|
score += 4;
|
|
3028
4365
|
score += scoreText([
|
|
@@ -3045,6 +4382,8 @@ function priorityTier(row) {
|
|
|
3045
4382
|
return 'certified_block';
|
|
3046
4383
|
if (row.objectType === 'semantic_metric')
|
|
3047
4384
|
return 'semantic_metric';
|
|
4385
|
+
if (row.objectType === 'runtime_value')
|
|
4386
|
+
return 'value_match';
|
|
3048
4387
|
if (row.objectType === 'dql_term' || row.objectType === 'business_view')
|
|
3049
4388
|
return 'business_context';
|
|
3050
4389
|
if (row.objectType.startsWith('dbt_') || row.objectType === 'warehouse_table')
|
|
@@ -3066,11 +4405,15 @@ function selectionReason(row, score, plannerReasons = []) {
|
|
|
3066
4405
|
function reasonForObject(row) {
|
|
3067
4406
|
if (row.objectType === 'dql_block' && row.status === 'certified')
|
|
3068
4407
|
return 'Certified reusable answer candidate';
|
|
4408
|
+
if (row.objectType === 'dql_block_output')
|
|
4409
|
+
return 'Certified block output column lineage';
|
|
3069
4410
|
if (row.objectType === 'semantic_metric')
|
|
3070
4411
|
return 'Semantic metric matched the question';
|
|
4412
|
+
if (row.objectType === 'runtime_value')
|
|
4413
|
+
return 'Observed runtime value matched the question literal';
|
|
3071
4414
|
if (row.objectType === 'dql_term' || row.objectType === 'business_view')
|
|
3072
4415
|
return 'DQL business context';
|
|
3073
|
-
if (row.objectType.startsWith('dbt_') || row.objectType === 'warehouse_table')
|
|
4416
|
+
if (row.objectType.startsWith('dbt_') || row.objectType === 'warehouse_table' || row.objectType === 'warehouse_column')
|
|
3074
4417
|
return 'dbt or warehouse metadata supplies physical context';
|
|
3075
4418
|
if (row.objectType === 'app' || row.objectType === 'dashboard')
|
|
3076
4419
|
return 'Published consumption context';
|
|
@@ -3084,6 +4427,13 @@ function mergeObjects(rows) {
|
|
|
3084
4427
|
}
|
|
3085
4428
|
return Array.from(byKey.values());
|
|
3086
4429
|
}
|
|
4430
|
+
function mergeMetadataEdges(rows) {
|
|
4431
|
+
const byKey = new Map();
|
|
4432
|
+
for (const row of rows) {
|
|
4433
|
+
putEdge(byKey, row);
|
|
4434
|
+
}
|
|
4435
|
+
return Array.from(byKey.values());
|
|
4436
|
+
}
|
|
3087
4437
|
function deriveTrust(objects) {
|
|
3088
4438
|
if (objects.length === 0)
|
|
3089
4439
|
return 'unknown';
|
|
@@ -3096,6 +4446,26 @@ function deriveTrust(objects) {
|
|
|
3096
4446
|
return 'draft';
|
|
3097
4447
|
return statuses.some((status) => status === 'certified') ? 'mixed' : 'unknown';
|
|
3098
4448
|
}
|
|
4449
|
+
function withMetadataTrustLabelInfo(decision) {
|
|
4450
|
+
return {
|
|
4451
|
+
...decision,
|
|
4452
|
+
trustLabelInfo: metadataTrustLabelInfo(decision.trustLabel),
|
|
4453
|
+
};
|
|
4454
|
+
}
|
|
4455
|
+
function metadataTrustLabelInfo(label) {
|
|
4456
|
+
switch (label) {
|
|
4457
|
+
case 'certified':
|
|
4458
|
+
return composeEffectiveTrust({ id: 'certified' });
|
|
4459
|
+
case 'conflict':
|
|
4460
|
+
return composeEffectiveTrust({ id: 'conflict' });
|
|
4461
|
+
case 'draft':
|
|
4462
|
+
return composeEffectiveTrust({ id: 'ai_generated' });
|
|
4463
|
+
case 'mixed':
|
|
4464
|
+
return composeEffectiveTrust({ id: 'ai_generated', existingQualifier: 'mixed context' });
|
|
4465
|
+
case 'unknown':
|
|
4466
|
+
return composeEffectiveTrust({ id: 'insufficient_context' });
|
|
4467
|
+
}
|
|
4468
|
+
}
|
|
3099
4469
|
function buildCitations(objects, edges) {
|
|
3100
4470
|
const citations = objects.slice(0, 24).map((row) => ({
|
|
3101
4471
|
objectKey: row.objectKey,
|
|
@@ -3205,7 +4575,7 @@ function isGovernedCandidate(row) {
|
|
|
3205
4575
|
* route scoped to questions the conflicting pair actually answers — a conflict
|
|
3206
4576
|
* elsewhere in the project never hijacks an unrelated question.
|
|
3207
4577
|
*/
|
|
3208
|
-
function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects) {
|
|
4578
|
+
function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects, question) {
|
|
3209
4579
|
if (compileConflicts.length === 0)
|
|
3210
4580
|
return undefined;
|
|
3211
4581
|
// Names of the top candidate objects for this question (bounded — only the
|
|
@@ -3220,6 +4590,8 @@ function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects) {
|
|
|
3220
4590
|
topNames.add(norm);
|
|
3221
4591
|
}
|
|
3222
4592
|
for (const conflict of compileConflicts) {
|
|
4593
|
+
if (!conflictMatchesQuestion(conflict, question))
|
|
4594
|
+
continue;
|
|
3223
4595
|
const sideNames = conflict.sides.map((side) => normalizeConflictName(side.name)).filter(Boolean);
|
|
3224
4596
|
if (sideNames.length < 2)
|
|
3225
4597
|
continue;
|
|
@@ -3243,6 +4615,14 @@ function pickRouteConflict(compileConflicts, rankedObjects, fallbackObjects) {
|
|
|
3243
4615
|
}
|
|
3244
4616
|
return undefined;
|
|
3245
4617
|
}
|
|
4618
|
+
function conflictMatchesQuestion(conflict, question) {
|
|
4619
|
+
const questionTokens = new Set(tokenize(question.replace(/_/g, ' ')));
|
|
4620
|
+
const conceptTokens = tokenize(conflict.concept.replace(/_/g, ' '));
|
|
4621
|
+
if (conceptTokens.length > 0 && conceptTokens.every((token) => questionTokens.has(token)))
|
|
4622
|
+
return true;
|
|
4623
|
+
const sideHits = conflict.sides.filter((side) => tokenize(side.name).some((token) => questionTokens.has(token))).length;
|
|
4624
|
+
return sideHits >= 2;
|
|
4625
|
+
}
|
|
3246
4626
|
function normalizeConflictName(value) {
|
|
3247
4627
|
return (value ?? '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
3248
4628
|
}
|
|
@@ -3267,7 +4647,7 @@ function sanitizeFtsQuery(raw) {
|
|
|
3267
4647
|
.map((term) => term.replace(/[^\p{L}\p{N}_]/gu, ''))
|
|
3268
4648
|
.filter((term) => term.length > 1 && !STOP_WORDS.has(term.toLowerCase()))
|
|
3269
4649
|
.slice(0, 48)
|
|
3270
|
-
.map((term) => `"${term}"
|
|
4650
|
+
.map((term) => `"${term}"*`)
|
|
3271
4651
|
.join(' OR ');
|
|
3272
4652
|
}
|
|
3273
4653
|
//# sourceMappingURL=catalog.js.map
|