@duckcodeailabs/dql-agent 1.6.31 → 1.6.33

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/dist/agent-run-engine.d.ts +42 -6
  2. package/dist/agent-run-engine.d.ts.map +1 -1
  3. package/dist/agent-run-engine.js +133 -79
  4. package/dist/agent-run-engine.js.map +1 -1
  5. package/dist/agent-run-gates.d.ts.map +1 -1
  6. package/dist/agent-run-gates.js +72 -5
  7. package/dist/agent-run-gates.js.map +1 -1
  8. package/dist/agent-run-planner.d.ts.map +1 -1
  9. package/dist/agent-run-planner.js +13 -0
  10. package/dist/agent-run-planner.js.map +1 -1
  11. package/dist/answer-loop.d.ts +110 -8
  12. package/dist/answer-loop.d.ts.map +1 -1
  13. package/dist/answer-loop.js +1679 -1299
  14. package/dist/answer-loop.js.map +1 -1
  15. package/dist/answer-shape.d.ts +16 -0
  16. package/dist/answer-shape.d.ts.map +1 -0
  17. package/dist/answer-shape.js +85 -0
  18. package/dist/answer-shape.js.map +1 -0
  19. package/dist/cascade/budgets.d.ts +101 -0
  20. package/dist/cascade/budgets.d.ts.map +1 -0
  21. package/dist/cascade/budgets.js +184 -0
  22. package/dist/cascade/budgets.js.map +1 -0
  23. package/dist/cascade/cascade.d.ts +86 -0
  24. package/dist/cascade/cascade.d.ts.map +1 -0
  25. package/dist/cascade/cascade.js +115 -0
  26. package/dist/cascade/cascade.js.map +1 -0
  27. package/dist/cascade/route-policy.d.ts +14 -0
  28. package/dist/cascade/route-policy.d.ts.map +1 -0
  29. package/dist/cascade/route-policy.js +48 -0
  30. package/dist/cascade/route-policy.js.map +1 -0
  31. package/dist/cascade/triage.d.ts +20 -0
  32. package/dist/cascade/triage.d.ts.map +1 -0
  33. package/dist/cascade/triage.js +25 -0
  34. package/dist/cascade/triage.js.map +1 -0
  35. package/dist/conversation/rolling-summary.d.ts +13 -0
  36. package/dist/conversation/rolling-summary.d.ts.map +1 -0
  37. package/dist/conversation/rolling-summary.js +43 -0
  38. package/dist/conversation/rolling-summary.js.map +1 -0
  39. package/dist/conversation/session-store.d.ts +110 -0
  40. package/dist/conversation/session-store.d.ts.map +1 -0
  41. package/dist/conversation/session-store.js +387 -0
  42. package/dist/conversation/session-store.js.map +1 -0
  43. package/dist/conversation/snapshot.d.ts +59 -0
  44. package/dist/conversation/snapshot.d.ts.map +1 -0
  45. package/dist/conversation/snapshot.js +153 -0
  46. package/dist/conversation/snapshot.js.map +1 -0
  47. package/dist/conversation/working-state.d.ts +49 -0
  48. package/dist/conversation/working-state.d.ts.map +1 -0
  49. package/dist/conversation/working-state.js +249 -0
  50. package/dist/conversation/working-state.js.map +1 -0
  51. package/dist/embeddings/provider.d.ts +62 -0
  52. package/dist/embeddings/provider.d.ts.map +1 -1
  53. package/dist/embeddings/provider.js +98 -0
  54. package/dist/embeddings/provider.js.map +1 -1
  55. package/dist/grounding/context-ledger.d.ts +30 -0
  56. package/dist/grounding/context-ledger.d.ts.map +1 -0
  57. package/dist/grounding/context-ledger.js +54 -0
  58. package/dist/grounding/context-ledger.js.map +1 -0
  59. package/dist/grounding/regrounding.d.ts +24 -0
  60. package/dist/grounding/regrounding.d.ts.map +1 -0
  61. package/dist/grounding/regrounding.js +219 -0
  62. package/dist/grounding/regrounding.js.map +1 -0
  63. package/dist/grounding/value-index.d.ts +32 -0
  64. package/dist/grounding/value-index.d.ts.map +1 -0
  65. package/dist/grounding/value-index.js +67 -0
  66. package/dist/grounding/value-index.js.map +1 -0
  67. package/dist/index.d.ts +45 -14
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +102 -11
  70. package/dist/index.js.map +1 -1
  71. package/dist/intent-controller.d.ts +7 -0
  72. package/dist/intent-controller.d.ts.map +1 -1
  73. package/dist/intent-controller.js +22 -1
  74. package/dist/intent-controller.js.map +1 -1
  75. package/dist/kg/build.d.ts.map +1 -1
  76. package/dist/kg/build.js +44 -17
  77. package/dist/kg/build.js.map +1 -1
  78. package/dist/kg/sqlite-fts.d.ts +3 -1
  79. package/dist/kg/sqlite-fts.d.ts.map +1 -1
  80. package/dist/kg/sqlite-fts.js +16 -2
  81. package/dist/kg/sqlite-fts.js.map +1 -1
  82. package/dist/kg/types.d.ts +23 -1
  83. package/dist/kg/types.d.ts.map +1 -1
  84. package/dist/memory/fts-query.d.ts +7 -0
  85. package/dist/memory/fts-query.d.ts.map +1 -0
  86. package/dist/memory/fts-query.js +36 -0
  87. package/dist/memory/fts-query.js.map +1 -0
  88. package/dist/memory/sqlite-memory.d.ts.map +1 -1
  89. package/dist/memory/sqlite-memory.js +1 -30
  90. package/dist/memory/sqlite-memory.js.map +1 -1
  91. package/dist/metadata/analysis-planner.d.ts +23 -0
  92. package/dist/metadata/analysis-planner.d.ts.map +1 -1
  93. package/dist/metadata/analysis-planner.js +432 -2
  94. package/dist/metadata/analysis-planner.js.map +1 -1
  95. package/dist/metadata/block-fit.d.ts +24 -0
  96. package/dist/metadata/block-fit.d.ts.map +1 -0
  97. package/dist/metadata/block-fit.js +373 -0
  98. package/dist/metadata/block-fit.js.map +1 -0
  99. package/dist/metadata/catalog.d.ts +72 -3
  100. package/dist/metadata/catalog.d.ts.map +1 -1
  101. package/dist/metadata/catalog.js +1452 -72
  102. package/dist/metadata/catalog.js.map +1 -1
  103. package/dist/metadata/drafts.d.ts +25 -0
  104. package/dist/metadata/drafts.d.ts.map +1 -1
  105. package/dist/metadata/drafts.js +112 -2
  106. package/dist/metadata/drafts.js.map +1 -1
  107. package/dist/metadata/metric-match.d.ts +55 -6
  108. package/dist/metadata/metric-match.d.ts.map +1 -1
  109. package/dist/metadata/metric-match.js +158 -13
  110. package/dist/metadata/metric-match.js.map +1 -1
  111. package/dist/metadata/sql-context-validation.d.ts +13 -1
  112. package/dist/metadata/sql-context-validation.d.ts.map +1 -1
  113. package/dist/metadata/sql-context-validation.js +172 -19
  114. package/dist/metadata/sql-context-validation.js.map +1 -1
  115. package/dist/metadata/sql-retrieval.d.ts +3 -2
  116. package/dist/metadata/sql-retrieval.d.ts.map +1 -1
  117. package/dist/metadata/sql-retrieval.js +6 -4
  118. package/dist/metadata/sql-retrieval.js.map +1 -1
  119. package/dist/providers/claude.d.ts +2 -1
  120. package/dist/providers/claude.d.ts.map +1 -1
  121. package/dist/providers/claude.js +95 -0
  122. package/dist/providers/claude.js.map +1 -1
  123. package/dist/providers/index.d.ts +1 -1
  124. package/dist/providers/index.d.ts.map +1 -1
  125. package/dist/providers/index.js.map +1 -1
  126. package/dist/providers/openai.d.ts +2 -1
  127. package/dist/providers/openai.d.ts.map +1 -1
  128. package/dist/providers/openai.js +143 -0
  129. package/dist/providers/openai.js.map +1 -1
  130. package/dist/providers/tool-output.d.ts +2 -0
  131. package/dist/providers/tool-output.d.ts.map +1 -0
  132. package/dist/providers/tool-output.js +22 -0
  133. package/dist/providers/tool-output.js.map +1 -0
  134. package/dist/providers/types.d.ts +23 -0
  135. package/dist/providers/types.d.ts.map +1 -1
  136. package/dist/providers/types.js.map +1 -1
  137. package/dist/semantic-bridge/compose.d.ts +89 -0
  138. package/dist/semantic-bridge/compose.d.ts.map +1 -0
  139. package/dist/semantic-bridge/compose.js +528 -0
  140. package/dist/semantic-bridge/compose.js.map +1 -0
  141. package/dist/semantic-bridge/member-select.d.ts +23 -0
  142. package/dist/semantic-bridge/member-select.d.ts.map +1 -0
  143. package/dist/semantic-bridge/member-select.js +106 -0
  144. package/dist/semantic-bridge/member-select.js.map +1 -0
  145. package/dist/skills/defaults.d.ts.map +1 -1
  146. package/dist/skills/defaults.js +5 -0
  147. package/dist/skills/defaults.js.map +1 -1
  148. package/dist/synthesize.d.ts +26 -0
  149. package/dist/synthesize.d.ts.map +1 -1
  150. package/dist/synthesize.js +72 -3
  151. package/dist/synthesize.js.map +1 -1
  152. package/dist/tools/registry.d.ts +15 -0
  153. package/dist/tools/registry.d.ts.map +1 -0
  154. package/dist/tools/registry.js +692 -0
  155. package/dist/tools/registry.js.map +1 -0
  156. package/dist/trust/stamp.d.ts +26 -0
  157. package/dist/trust/stamp.d.ts.map +1 -0
  158. package/dist/trust/stamp.js +33 -0
  159. package/dist/trust/stamp.js.map +1 -0
  160. package/package.json +4 -4
@@ -14,52 +14,132 @@
14
14
  * step. Tests can mock the provider with a canned response and exercise the
15
15
  * full pipeline.
16
16
  */
17
- import { composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
18
17
  import { buildSkillBlockHints, buildSkillsPrompt, selectRelevantSkills } from './skills/loader.js';
19
- import { matchSemanticMetric, resolveGovernedMetricSql } from './metadata/metric-match.js';
18
+ import { deriveGeneratedDraftSlug, renderGeneratedSqlDqlArtifact } from './metadata/drafts.js';
19
+ import { buildAnalysisQuestionPlan } from './metadata/analysis-planner.js';
20
+ import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit } from './metadata/block-fit.js';
21
+ import { buildGovernedMetricFirstSql, matchSemanticMetric, resolveGovernedMetricSql } from './metadata/metric-match.js';
20
22
  import { decideAgentAction } from './intent-controller.js';
21
- import { validateSqlAgainstLocalContext } from './metadata/sql-context-validation.js';
22
- import { buildGroundingFromRuntimeRelations, resolveRelationsInSql, validateSqlAgainstGrounding } from './metadata/sql-grounding.js';
23
+ import { createContextLedger } from './grounding/context-ledger.js';
24
+ import { validateAnswerResultShape } from './answer-shape.js';
23
25
  import { compactSqlSnippet, extractSimpleSelectShape, selectExpressionOutputName, } from './metadata/sql-shape.js';
26
+ import { composeSemanticQueryForQuestion, composeSemanticQueryFromMembers } from './semantic-bridge/compose.js';
27
+ import { selectSemanticMembersViaLlm } from './semantic-bridge/member-select.js';
28
+ import { normalizeValueIndexText } from './grounding/value-index.js';
29
+ import { cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, } from './cascade/cascade.js';
30
+ import { shouldClarifyBeforeGeneration } from './cascade/triage.js';
31
+ import { stampTrustLabel } from './trust/stamp.js';
32
+ import { QUICK_PROMPT_CONTEXT_BUDGET, canUseLaneRepair, cascadeBudgetTrace, createCascadeBudgetState, promptContextBudgetForQuestion, proposalToolBudgetForQuestion, recordLaneRepair, } from './cascade/budgets.js';
33
+ /**
34
+ * Physical column names whose sampled runtime values include `value`. Used by the
35
+ * semantic bridge to bind a filter literal to the dimension that actually carries
36
+ * it — a generic, project-agnostic replacement for hard-coded value→dimension maps.
37
+ */
38
+ function resolveFilterValueColumns(value, schemaContext) {
39
+ const needle = normalizeValueIndexText(value);
40
+ if (!needle)
41
+ return [];
42
+ const columns = [];
43
+ for (const table of schemaContext) {
44
+ for (const column of table.columns) {
45
+ if (column.sampleValues?.some((sample) => normalizeValueIndexText(sample) === needle)) {
46
+ columns.push(column.name);
47
+ }
48
+ }
49
+ }
50
+ return columns;
51
+ }
24
52
  const CERTIFIED_HIT_THRESHOLD = 0.18;
25
53
  const HARD_NEGATIVE_RATIO = 0.5;
26
54
  const EXECUTABLE_ARTIFACT_KINDS = ['block', 'dashboard', 'app', 'notebook'];
55
+ // Dashboards, apps, and notebooks are governed NAVIGATION targets — collections
56
+ // of tiles / a standing surface, not an executable data query. They can ground or
57
+ // be cited, but they never produce the row-level answer to an analytical question,
58
+ // so they must not terminate a data ask as a "certified answer" with no data.
59
+ const NAVIGATION_ARTIFACT_KINDS = ['dashboard', 'app', 'notebook'];
27
60
  const BUSINESS_CONTEXT_KINDS = ['term', 'business_view'];
28
61
  const ARTIFACT_KINDS = [...EXECUTABLE_ARTIFACT_KINDS, ...BUSINESS_CONTEXT_KINDS];
29
62
  const SEMANTIC_KINDS = ['metric', 'dimension', 'measure', 'entity', 'semantic_model', 'saved_query'];
30
63
  const MANIFEST_KINDS = ['dbt_model', 'dbt_source'];
64
+ function refusalCodeForValidation(code) {
65
+ if (code === 'unknown_relation' || code === 'unknown_column' || code === 'insufficient_context' || code === 'missing_baseline') {
66
+ return 'grounding_gap';
67
+ }
68
+ if (code === 'ambiguous_filter')
69
+ return 'ambiguous';
70
+ return 'model_declined';
71
+ }
72
+ function formatOffendingValidationToken(offending) {
73
+ if (!offending?.relation && !offending?.column)
74
+ return '';
75
+ const parts = [
76
+ offending.relation ? `relation=${offending.relation}` : undefined,
77
+ offending.column ? `column=${offending.column}` : undefined,
78
+ ].filter((part) => Boolean(part));
79
+ return `Offending metadata token: ${parts.join(', ')}`;
80
+ }
81
+ function formatValidationWarningsForPrompt(warnings) {
82
+ const useful = warnings.filter((warning) => warning.trim().length > 0).slice(0, 6);
83
+ return useful.length > 0 ? `Additional inspected-context notes:\n${useful.map((warning) => `- ${warning}`).join('\n')}` : '';
84
+ }
31
85
  /**
32
- * Map an answer-loop result's source tier + certification + review state to a
33
- * canonical trust-label id from the one shared vocabulary in dql-core.
34
- * Additive and lenient keeps the legacy `trustLabel` string untouched.
86
+ * Central honesty gate applied once at the single answer() exit so EVERY return
87
+ * site (certified, semantic-metric, generated) is covered by construction. A DATA
88
+ * answer must have PRODUCED ROWS: when execution was attempted (a result or an
89
+ * execution error came back) but no rows resulted, it may never read as a confident
90
+ * answer.
91
+ * - A generated / governed-metric (uncertified) answer downgrades to a low-
92
+ * confidence, review-required no-data state with an honest message — the SQL is
93
+ * still surfaced so the user can inspect and fix it.
94
+ * - A CERTIFIED block keeps its badge (an empty certified result can be a correct
95
+ * "none matched" answer) but gains a non-blocking note to verify data currency.
96
+ * An UN-executed answer (offline SQL preview / certified citation with no executor)
97
+ * is left untouched — that is a legitimate preview, not a hollow answer.
35
98
  */
36
- function canonicalTrustLabelId(result) {
99
+ function applyHollowAnswerGate(result) {
37
100
  if (result.kind === 'no_answer')
38
- return 'insufficient_context';
39
- if (result.certification === 'certified' || result.kind === 'certified')
40
- return 'certified';
41
- if (result.sourceTier === 'business_context' && result.reviewStatus === 'certified')
42
- return 'reviewed';
43
- if (result.certification === 'ai_generated' ||
44
- result.certification === 'analyst_review_required' ||
45
- result.reviewStatus === 'analyst_review_required' ||
46
- result.reviewStatus === 'draft_ready') {
47
- return 'ai_generated';
101
+ return result;
102
+ const attempted = result.result !== undefined || result.executionError !== undefined;
103
+ const producedRows = Boolean(result.result && typeof result.result.rowCount === 'number' && result.result.rowCount > 0);
104
+ if (!attempted || producedRows)
105
+ return result;
106
+ if (result.kind === 'certified') {
107
+ const note = result.executionError
108
+ ? 'This certified block failed to execute — review the source data before relying on it.'
109
+ : 'This certified block returned 0 rows — verify the source data is current before relying on it.';
110
+ return {
111
+ ...result,
112
+ text: [result.text, note].filter(Boolean).join('\n\n'),
113
+ answer: [result.answer ?? result.text, note].filter(Boolean).join('\n\n'),
114
+ validationWarnings: [...(result.validationWarnings ?? []), note],
115
+ };
48
116
  }
49
- return 'insufficient_context';
117
+ const honestText = [
118
+ result.executionError
119
+ ? `The governed query could not be executed (${result.executionError}).`
120
+ : 'The governed query executed but returned no rows.',
121
+ 'This usually means a filter, grain, or join is off — review the SQL preview and refine before reuse.',
122
+ ].join('\n\n');
123
+ const warning = result.executionError
124
+ ? 'The governed query failed to execute and returned no data — review before reuse.'
125
+ : 'The governed query executed but returned no rows — review the SQL, filters, and joins before reuse.';
126
+ return {
127
+ ...result,
128
+ reviewStatus: 'analyst_review_required',
129
+ confidence: Math.min(result.confidence ?? 0.2, 0.2),
130
+ text: honestText,
131
+ answer: honestText,
132
+ validationWarnings: [...(result.validationWarnings ?? []), warning],
133
+ };
50
134
  }
51
135
  export async function answer(input) {
52
- const result = await runAnswerLoop(input);
136
+ const result = applyHollowAnswerGate(await runAnswerLoop(input));
53
137
  // Attach the canonical trust label once, at the single exit point, so every
54
138
  // return site inside runAnswerLoop stays untouched and backward compatible.
55
139
  // Freshness-aware trust: for a certified answer, fold the source block's data
56
140
  // health (stale/failed upstream) into the label so it reads "Certified ·
57
141
  // stale data" / "Certified · upstream failed". Non-certified or fresh answers
58
142
  // are unaffected.
59
- const id = canonicalTrustLabelId(result);
60
- const dataState = id === 'certified'
61
- ? (result.block?.dataState)
62
- : undefined;
63
143
  const { _semanticMetricMatch, ...publicResult } = result;
64
144
  const chosenRoute = result.route ?? deriveAiRoute(result, _semanticMetricMatch);
65
145
  // P0 — record the high-level action this turn warranted, so callers can route
@@ -80,7 +160,21 @@ export async function answer(input) {
80
160
  return {
81
161
  ...publicResult,
82
162
  intentDecision,
83
- trustLabelInfo: composeEffectiveTrust({ id, dataState }),
163
+ trustLabelInfo: stampTrustLabel(result),
164
+ cascade: publicResult.cascade ?? createCascadeAnswerResult({
165
+ routeTier: chosenRoute.tier,
166
+ label: chosenRoute.label,
167
+ ref: chosenRoute.ref,
168
+ artifactKind: result.dqlArtifact?.kind,
169
+ refusalCode: result.refusalCode,
170
+ reason: result.refusalDetails?.message ?? (result.kind === 'no_answer' ? result.text : undefined),
171
+ rowCount: result.result?.rowCount,
172
+ executionStatus: cascadeExecutionStatus(result),
173
+ draftBlockId: result.draftBlockId ?? result.draftBlock?.path,
174
+ metrics: result.dqlArtifact?.metrics,
175
+ dimensions: result.dqlArtifact?.dimensions,
176
+ hasSqlPreview: Boolean(result.proposedSql ?? result.sql),
177
+ }),
84
178
  // Stamp the SELECTED skills that shaped the answer (transparency). Computed
85
179
  // here so every return site inside runAnswerLoop stays untouched.
86
180
  appliedSkills: result.appliedSkills ??
@@ -93,6 +187,17 @@ export async function answer(input) {
93
187
  route: chosenRoute,
94
188
  };
95
189
  }
190
+ function cascadeExecutionStatus(result) {
191
+ if (result.kind === 'no_answer')
192
+ return 'not_applicable';
193
+ if (result.executionError)
194
+ return 'failed';
195
+ if (result.result)
196
+ return 'executed';
197
+ if (result.block || result.dqlArtifact || result.proposedSql || result.sql)
198
+ return 'not_requested';
199
+ return 'not_applicable';
200
+ }
96
201
  /**
97
202
  * Derive the UI-facing route from a finished answer (spec 17, part C). The
98
203
  * semantic-metric tier is named explicitly when the loop matched a governed
@@ -118,7 +223,7 @@ function deriveAiRoute(result, metricMatch) {
118
223
  ref,
119
224
  };
120
225
  }
121
- // Uncertified: a governed metric matched → semantic_metric; else generated SQL.
226
+ // Uncertified: a governed metric matched → semantic_metric; else generated preview.
122
227
  if (result.sourceTier === 'semantic_layer' && metricMatch) {
123
228
  return {
124
229
  tier: 'semantic_metric',
@@ -126,7 +231,9 @@ function deriveAiRoute(result, metricMatch) {
126
231
  ref: metricMatch.metric.name,
127
232
  };
128
233
  }
129
- return { tier: 'generated_sql', label: 'Answered with generated SQL (review required).' };
234
+ return result.dqlArtifact
235
+ ? { tier: 'generated_sql', label: 'Prepared review-required DQL artifact with SQL preview.' }
236
+ : { tier: 'generated_sql', label: 'Prepared review-required SQL preview.' };
130
237
  }
131
238
  async function runAnswerLoop(input) {
132
239
  const { question, userId, domain, provider, kg, skills = [], blockHints = [] } = input;
@@ -152,6 +259,10 @@ async function runAnswerLoop(input) {
152
259
  const considered = mergeHits(artifactHits, semanticHits, manifestHits, kg.search({ query: question, domain, limit: 10 })).slice(0, 30);
153
260
  const schemaContext = schemaContextWithAllowedSqlContext(input.schemaContext ?? [], input.contextPack);
154
261
  const catalogRoute = input.contextPack?.routeDecision;
262
+ const questionPlan = input.contextPack?.questionPlan?.requestedShape
263
+ ? input.contextPack.questionPlan
264
+ : buildAnalysisQuestionPlan(question, input.followUp);
265
+ const repairBudgetState = createCascadeBudgetState(input.cascadeBudgetModel);
155
266
  const fallbackIntent = classifyAgentIntent({
156
267
  question,
157
268
  followUp: input.followUp,
@@ -160,13 +271,14 @@ async function runAnswerLoop(input) {
160
271
  manifestHits,
161
272
  schemaContext,
162
273
  });
163
- const intent = catalogRoute ? agentIntentFromCatalogRoute(catalogRoute) : fallbackIntent;
274
+ let intent = catalogRoute ? agentIntentFromCatalogRoute(catalogRoute) : fallbackIntent;
164
275
  // Stage 1: certified artifact match. Blocks can be executed; dashboards,
165
276
  // Apps, and notebooks are returned as governed citations/navigation targets.
166
277
  const drilldownCertifiedHit = input.followUp?.kind === 'drilldown'
167
278
  ? pickCertifiedDrilldownArtifact({
168
279
  executableArtifactHits,
169
280
  question,
281
+ questionPlan,
170
282
  followUp: input.followUp,
171
283
  excludedArtifactIds,
172
284
  kg,
@@ -179,23 +291,74 @@ async function runAnswerLoop(input) {
179
291
  executableArtifactHits,
180
292
  businessHits,
181
293
  question,
294
+ questionPlan,
182
295
  blockHints: input.followUp?.kind === 'drilldown' ? [] : effectiveBlockHints,
183
296
  excludedArtifactIds,
184
297
  kg,
185
298
  })
186
299
  : null);
187
- if (artifactHit) {
300
+ // A certified TERM / BUSINESS VIEW is documentation, not data: it can be the
301
+ // terminal answer only for a definition-style question with no data ask. A
302
+ // question that requests a data shape (dimensions/measures/outputs/top-N) or
303
+ // continues an ongoing data conversation must be ANSWERED WITH DATA — the
304
+ // matched business context falls through as grounding for the executable
305
+ // tiers (certified block → governed metric → generated SQL), never instead
306
+ // of them. "Certified" always means "a governed, executable definition
307
+ // produced this result", not "a certified document sounded related".
308
+ const businessContextTerminal = (() => {
309
+ if (!artifactHit || artifactHit.node.kind === 'block')
310
+ return true;
311
+ // An ongoing data conversation (prior turn produced result columns/values)
312
+ // must be answered with data even when the phrasing looks definitional
313
+ // ("so Matthew is the top — what is his 360 profile view?").
314
+ if (input.followUp?.priorResultValues || input.followUp?.priorResultColumns?.length)
315
+ return false;
316
+ // A dashboard / app / notebook is a NAVIGATION target, not executable data.
317
+ // It can only terminate a question that explicitly asks to open/see THAT
318
+ // artifact by name ("open the Jaffle Growth Command Center"); any analytical
319
+ // ask ("top customers who bought the top products with revenue") must fall
320
+ // through to the executable + generated tiers and use the artifact only as
321
+ // grounding — never returned as a no-data "certified answer".
322
+ if (NAVIGATION_ARTIFACT_KINDS.includes(artifactHit.node.kind)) {
323
+ return objectNameInQuestion(question, artifactHit.node);
324
+ }
325
+ // Cold questions fall through only on STRONG data-shape signals — bare
326
+ // measure/dimension words are normal in definition asks ("what is revenue
327
+ // health?") and must keep answering from the certified business context.
328
+ // "Strong" = outputs requested BEYOND the measure/dimension words themselves
329
+ // (e.g. product_name), explicit filters, or a top-N.
330
+ const requested = questionPlan.requestedShape;
331
+ const nonTrivialOutputs = requested.requiredOutputs.filter((output) => !requested.measures.includes(output) && !requested.dimensions.includes(output));
332
+ const strongShape = nonTrivialOutputs.length > 0
333
+ || requested.filters.length > 0
334
+ || Boolean(requested.topN);
335
+ return !strongShape;
336
+ })();
337
+ if (artifactHit && businessContextTerminal) {
188
338
  let result;
189
339
  let executionError;
190
340
  if (artifactHit.node.kind === 'block' && input.executeCertifiedBlock) {
191
341
  try {
192
342
  result = await input.executeCertifiedBlock(artifactHit.node);
343
+ result = trimResultToRequestedTopN(result, questionPlan);
193
344
  }
194
345
  catch (err) {
195
346
  executionError = err instanceof Error ? err.message : String(err);
196
347
  }
197
348
  }
198
- const text = composeCertifiedAnswer(artifactHit.node, question, result, executionError);
349
+ const resultShapeWarnings = result ? validateAnswerResultShape(questionPlan, result).warnings : [];
350
+ // When a certified block's execution was ATTEMPTED and FAILED, the answer
351
+ // cannot wear the certified badge — a failed run has no data to stand behind.
352
+ // Downgrade to analyst_review_required (the error is surfaced in the text).
353
+ // Note: a matched-but-unexecuted block (no executor / dry-run) legitimately
354
+ // stays a certified *citation* — non-execution is a separate axis surfaced via
355
+ // cascade.executionStatus ('not_requested') and freshness dataState, not a
356
+ // reason to strip source trust.
357
+ const certifiedShapePassed = executionError === undefined && resultShapeWarnings.length === 0;
358
+ const certifiedText = composeCertifiedAnswer(artifactHit.node, question, result, executionError);
359
+ const text = resultShapeWarnings.length > 0
360
+ ? `${certifiedText}\n\nReview required: ${resultShapeWarnings.join(' ')}`
361
+ : certifiedText;
199
362
  const sourceTier = artifactHit.node.sourceTier === 'business_context'
200
363
  ? 'business_context'
201
364
  : 'certified_artifact';
@@ -218,21 +381,24 @@ async function runAnswerLoop(input) {
218
381
  sql: result?.sql,
219
382
  suggestedViz: result?.chartConfig ? chartNameFromConfig(result.chartConfig) : undefined,
220
383
  });
384
+ const dqlArtifact = buildCertifiedBlockDqlArtifact(artifactHit.node, result);
221
385
  return {
222
- kind: 'certified',
386
+ kind: certifiedShapePassed ? 'certified' : 'uncertified',
223
387
  sourceTier,
224
- certification: 'certified',
225
- reviewStatus: 'certified',
226
- confidence: 0.95,
388
+ certification: certifiedShapePassed ? 'certified' : 'analyst_review_required',
389
+ reviewStatus: certifiedShapePassed ? 'certified' : 'analyst_review_required',
390
+ confidence: certifiedShapePassed ? 0.95 : 0.45,
227
391
  text,
228
392
  answer: text,
229
393
  block: artifactHit.node.kind === 'block' ? artifactHit.node : undefined,
230
394
  result,
231
395
  executionError,
232
396
  sql: result?.sql,
233
- trustLabel: input.contextPack?.trustLabel ?? 'certified',
397
+ dqlArtifact,
398
+ trustLabel: certifiedShapePassed ? input.contextPack?.trustLabel ?? 'certified' : 'mixed',
234
399
  sourceCertifiedBlock: artifactHit.node.kind === 'block' ? artifactHit.node.name : undefined,
235
400
  contextPackId: input.contextPack?.id,
401
+ validationWarnings: resultShapeWarnings,
236
402
  selectedEvidence: input.contextPack?.evidenceRoles?.slice(0, 12),
237
403
  citations,
238
404
  memoryContext: input.memoryContext,
@@ -246,6 +412,7 @@ async function runAnswerLoop(input) {
246
412
  considered,
247
413
  result,
248
414
  executionError,
415
+ resultShapeWarnings,
249
416
  executorWasAvailable: Boolean(input.executeCertifiedBlock),
250
417
  citations,
251
418
  memoryContext: input.memoryContext ?? [],
@@ -264,8 +431,19 @@ async function runAnswerLoop(input) {
264
431
  // cumulative_revenue). Certified-first is still preserved (checked above).
265
432
  const semanticMetricNodes = collectMetricCandidates(semanticHits, considered, kg);
266
433
  let semanticMetricMatch = await matchSemanticMetric(question, semanticMetricNodes).catch(() => null);
267
- // Clarify only when there is ALSO no confident governed-metric match.
268
- if ((intent === 'clarify' || catalogRoute?.route === 'clarify') && !semanticMetricMatch) {
434
+ const clarifyBeforeGeneration = shouldClarifyBeforeGeneration({
435
+ intent,
436
+ routeDecision: catalogRoute,
437
+ hasSemanticMetricMatch: Boolean(semanticMetricMatch),
438
+ schemaContextCount: schemaContext.length,
439
+ allowedRelationCount: input.contextPack?.allowedSqlContext?.relations.length ?? 0,
440
+ sourceBlockSqlCount: input.contextPack?.allowedSqlContext?.sourceBlockSql.length ?? 0,
441
+ metadataObjectCount: considered.length,
442
+ });
443
+ // Clarify only when there is ALSO no confident governed-metric match and no
444
+ // usable schema/catalog context. A conservative catalog "clarify" decision is
445
+ // not a terminal lane if runtime or SQL context can still answer with review.
446
+ if (clarifyBeforeGeneration) {
269
447
  const text = composeCatalogClarificationText(question, catalogRoute) ?? composeClarificationText(question, considered, schemaContext);
270
448
  const analysisPlan = buildAnalysisPlan({
271
449
  question,
@@ -298,12 +476,15 @@ async function runAnswerLoop(input) {
298
476
  considered,
299
477
  memoryContext: input.memoryContext ?? [],
300
478
  analysisPlan,
479
+ budgetTrace: cascadeBudgetTrace(repairBudgetState),
301
480
  }),
302
481
  contextPack: input.contextPack,
303
482
  considered,
304
483
  providerUsed: provider.name,
305
484
  };
306
485
  }
486
+ if (intent === 'clarify')
487
+ intent = questionPlan.routeIntent;
307
488
  // Stage 2/3: generate only after certified artifacts miss. Semantic context
308
489
  // wins over raw dbt manifest context; memory is appended last as advisory.
309
490
  // A confident metric match forces the semantic tier even when FTS returned no
@@ -329,9 +510,15 @@ async function runAnswerLoop(input) {
329
510
  ? [...matchedMetricHit, ...trustedArtifactContext, ...reviewRequiredArtifactHits, ...businessHits.slice(0, 4), ...semanticHits, ...manifestHits].slice(0, 14)
330
511
  : [...trustedArtifactContext, ...reviewRequiredArtifactHits, ...businessHits.slice(0, 4), ...manifestHits].slice(0, 14);
331
512
  const contextNodes = mergeNodes(followUpSourceBlock && input.followUp?.kind === 'drilldown' ? [followUpSourceBlock] : [], (contextHits.length > 0 ? contextHits : considered.slice(0, 6)).map((h) => h.node));
513
+ const kgJoinPathHints = buildKgJoinPathHints(kg, contextNodes, questionPlan);
332
514
  const contextBlocks = contextNodes.filter((n) => n.kind === 'block');
333
515
  const contextBusiness = contextNodes.filter((n) => BUSINESS_CONTEXT_KINDS.includes(n.kind));
334
516
  const contextOther = contextNodes.filter((n) => n.kind !== 'block' && !BUSINESS_CONTEXT_KINDS.includes(n.kind));
517
+ const promptBudget = promptContextBudgetForQuestion({
518
+ questionPlan,
519
+ requestedDepth: input.analysisDepth,
520
+ reasoningEffort: input.reasoningEffort,
521
+ });
335
522
  const messages = [
336
523
  { role: 'system', content: SYSTEM_PROMPT },
337
524
  ];
@@ -340,28 +527,123 @@ async function runAnswerLoop(input) {
340
527
  messages.push({ role: 'system', content: skillsPrompt });
341
528
  messages.push({
342
529
  role: 'system',
343
- content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp, schemaContext, intent, input.contextPack),
530
+ content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp, schemaContext, intent, input.contextPack, input.conversationSnapshot, kgJoinPathHints, promptBudget),
344
531
  });
345
532
  messages.push({ role: 'user', content: question });
346
- const localProposal = buildSchemaAwareProposal({
347
- question,
348
- intent,
349
- schemaContext,
350
- followUp: input.followUp,
351
- contextPack: input.contextPack,
352
- }) ?? buildContextPackAwareProposal({
353
- question,
354
- intent,
533
+ // ── Tier 2: semantic-layer metrics + dimensions (governed hierarchy) ──────
534
+ // Certified blocks already missed (Stage 1). Before LLM generation, a
535
+ // confidently matched governed metric whose definition can
536
+ // express the question's full requested shape (scalar KPI, or a group-by whose
537
+ // dimensions all resolve to columns on the metric's own table) answers
538
+ // deterministically. Only when no metric fits does the question fall through
539
+ // to SQL generation, where the metric still grounds the prompt as context.
540
+ let governedMetricAnswer = false;
541
+ const semanticBridgeToolCalls = [];
542
+ let semanticBridgeAnswer;
543
+ if (input.semanticLayer && semanticMetricMatch) {
544
+ semanticBridgeAnswer = composeSemanticQueryForQuestion({
545
+ semanticLayer: input.semanticLayer,
546
+ question,
547
+ questionPlan,
548
+ matchedMetric: semanticMetricMatch.metric,
549
+ filterValueColumns: (value) => resolveFilterValueColumns(value, schemaContext),
550
+ ...(input.semanticDriver ? { driver: input.semanticDriver } : {}),
551
+ ...(input.semanticTableMapping ? { tableMapping: input.semanticTableMapping } : {}),
552
+ });
553
+ // Lane-2 LLM fallback: the deterministic token matcher missed, but a metric
554
+ // matched so the semantic layer IS relevant. Spend ONE call to have the model
555
+ // pick members (the query_semantic_model contract); the compiler still owns the
556
+ // SQL, so a paraphrased metric question stays governed instead of dropping to
557
+ // Lane-3 generation.
558
+ if (!semanticBridgeAnswer) {
559
+ const selection = await selectSemanticMembersViaLlm({
560
+ provider,
561
+ semanticLayer: input.semanticLayer,
562
+ question,
563
+ signal: input.signal,
564
+ reasoningEffort: input.reasoningEffort,
565
+ });
566
+ if (selection) {
567
+ const composed = composeSemanticQueryFromMembers({
568
+ semanticLayer: input.semanticLayer,
569
+ question,
570
+ selection,
571
+ ...(input.semanticDriver ? { driver: input.semanticDriver } : {}),
572
+ ...(input.semanticTableMapping ? { tableMapping: input.semanticTableMapping } : {}),
573
+ });
574
+ // Coverage guard: if the question asked for a breakdown but the LLM
575
+ // selection produced none, the governed answer would silently DROP the
576
+ // requested grouping (governed-but-wrong). Fall through to Lane-3
577
+ // generation, which can express the breakdown.
578
+ const wantedBreakdown = questionPlan.requestedShape.dimensions.length > 0
579
+ || questionPlan.dimensionTerms.length > 0;
580
+ const dropsBreakdown = Boolean(composed)
581
+ && wantedBreakdown
582
+ && composed.dimensions.length === 0
583
+ && !composed.timeDimension;
584
+ if (composed && !dropsBreakdown) {
585
+ semanticBridgeAnswer = composed;
586
+ semanticBridgeToolCalls.push({
587
+ name: 'query_semantic_model',
588
+ status: 'checked',
589
+ inputSummary: `metrics: ${selection.metrics.join(', ')}${selection.dimensions?.length ? `; by ${selection.dimensions.join(', ')}` : ''}`,
590
+ outputSummary: 'LLM-selected semantic members compiled via composeQuery',
591
+ order: semanticBridgeToolCalls.length + 1,
592
+ });
593
+ }
594
+ }
595
+ }
596
+ }
597
+ const metricFirst = semanticMetricMatch
598
+ ? buildGovernedMetricFirstSql({
599
+ metric: semanticMetricMatch.metric,
600
+ pool: semanticMetricNodes,
601
+ requestedShape: questionPlan.requestedShape,
602
+ schemaTables: schemaContext.map((table) => ({
603
+ relation: table.relation,
604
+ name: table.name,
605
+ columns: table.columns.map((column) => ({ name: column.name })),
606
+ })),
607
+ semanticLayer: input.semanticLayer,
608
+ })
609
+ : undefined;
610
+ let contextLedger = createContextLedger({
355
611
  contextPack: input.contextPack,
612
+ schemaContext,
356
613
  });
357
614
  let proposed = '';
358
615
  let parsed;
359
- if (localProposal) {
360
- parsed = localProposal;
616
+ const proposalToolCalls = [...semanticBridgeToolCalls];
617
+ if (semanticBridgeAnswer) {
618
+ governedMetricAnswer = true;
619
+ parsed = {
620
+ sql: semanticBridgeAnswer.sql,
621
+ text: `Answered from governed semantic metric${semanticBridgeAnswer.metrics.length === 1 ? '' : 's'} ${semanticBridgeAnswer.metrics.join(', ')}${semanticBridgeAnswer.dimensions.length > 0 ? ` by ${semanticBridgeAnswer.dimensions.join(', ')}` : ''}. This result is uncertified until reviewed and promoted.`,
622
+ viz: semanticBridgeAnswer.dimensions.length === 0 && !semanticBridgeAnswer.timeDimension ? 'single_value' : undefined,
623
+ };
624
+ }
625
+ else if (metricFirst) {
626
+ semanticMetricMatch = { ...semanticMetricMatch, metric: metricFirst.metric };
627
+ governedMetricAnswer = true;
628
+ parsed = {
629
+ sql: metricFirst.sql,
630
+ text: `Answered from the governed metric ${metricFirst.metric.name}${metricFirst.dimensions.length > 0 ? ` by ${metricFirst.dimensions.join(', ')}` : ''}. This result is uncertified until reviewed and promoted.`,
631
+ viz: metricFirst.dimensions.length === 0 ? 'single_value' : undefined,
632
+ };
361
633
  }
362
634
  else {
363
635
  try {
364
- proposed = await provider.generate(messages, { signal: input.signal, reasoningEffort: input.reasoningEffort });
636
+ proposed = await generateProposalWithOptionalTools({
637
+ provider,
638
+ messages,
639
+ tools: input.answerLoopTools,
640
+ questionPlan,
641
+ intent,
642
+ signal: input.signal,
643
+ reasoningEffort: input.reasoningEffort,
644
+ analysisDepth: input.analysisDepth,
645
+ toolCalls: proposalToolCalls,
646
+ });
365
647
  }
366
648
  catch (err) {
367
649
  const text = `Provider error: ${err.message}`;
@@ -373,6 +655,8 @@ async function runAnswerLoop(input) {
373
655
  confidence: 0,
374
656
  text,
375
657
  answer: text,
658
+ refusalCode: 'provider_error',
659
+ refusalDetails: { code: 'provider_error', message: text },
376
660
  citations: [],
377
661
  memoryContext: input.memoryContext,
378
662
  evidence: buildNoAnswerEvidence({
@@ -384,6 +668,7 @@ async function runAnswerLoop(input) {
384
668
  manifestHits,
385
669
  considered,
386
670
  memoryContext: input.memoryContext ?? [],
671
+ toolCalls: proposalToolCalls,
387
672
  }),
388
673
  contextPack: input.contextPack,
389
674
  considered,
@@ -392,11 +677,35 @@ async function runAnswerLoop(input) {
392
677
  }
393
678
  parsed = parseProposal(proposed);
394
679
  }
395
- // True when `parsed.sql` was synthesized deterministically from a governed
396
- // semantic-layer metric (not the LLM). Such SQL is trusted and grounded against
397
- // the runtime schema, so it skips the hallucination-guard context validation
398
- // that exists to catch model-invented relations/columns.
399
- let governedMetricAnswer = false;
680
+ let deepCandidateResult;
681
+ let deepCandidateExecutionError;
682
+ let deepCandidateNotes = [];
683
+ if (!governedMetricAnswer && input.analysisDepth === 'deep' && parsed.sql) {
684
+ const selection = await selectDeepGeneratedProposalCandidate({
685
+ provider,
686
+ messages,
687
+ question,
688
+ questionPlan,
689
+ intent,
690
+ initial: { raw: proposed, parsed },
691
+ contextLedger,
692
+ executeGeneratedSql: input.executeGeneratedSql,
693
+ signal: input.signal,
694
+ reasoningEffort: input.reasoningEffort,
695
+ });
696
+ if (selection.selected) {
697
+ proposed = selection.selected.raw;
698
+ parsed = selection.selected.parsed;
699
+ deepCandidateResult = selection.selected.result;
700
+ deepCandidateExecutionError = selection.selected.executionError;
701
+ }
702
+ deepCandidateNotes = selection.notes;
703
+ }
704
+ // `governedMetricAnswer` (declared above): true when `parsed.sql` was
705
+ // synthesized deterministically from a governed semantic-layer metric (not the
706
+ // LLM). Such SQL is trusted and grounded against the runtime schema, so it
707
+ // skips the hallucination-guard context validation that exists to catch
708
+ // model-invented relations/columns.
400
709
  if (!parsed.sql) {
401
710
  // Spec 17, part C — if a governed metric matched confidently but the model
402
711
  // declined SQL, answer from the metric definition (deterministic, offline)
@@ -406,7 +715,7 @@ async function runAnswerLoop(input) {
406
715
  // (`order_item.revenue`). resolveGovernedMetricSql resolves a thin metric to
407
716
  // that synthesizable sibling so the route lands on a real number, not refusal.
408
717
  const resolved = activeTier === 'semantic_layer' && semanticMetricMatch
409
- ? resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes)
718
+ ? resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes, input.semanticLayer)
410
719
  : undefined;
411
720
  if (resolved) {
412
721
  // Point the match at the metric we actually answered from so the route
@@ -421,8 +730,54 @@ async function runAnswerLoop(input) {
421
730
  };
422
731
  }
423
732
  }
733
+ // Forced-join retry. The model declined SQL (rule 8 escape) — but a composite
734
+ // "top X who did top Y" that spans several concepts often trips the model into
735
+ // "there's no combined dataset — show them separately", which is a FALSE refusal
736
+ // when the grounded tables and KG join routes to connect them are right there in
737
+ // context. If the question genuinely wants generated data and we DO have usable
738
+ // relations/schema to build the join, re-issue ONCE with an explicit instruction
739
+ // to compose the join rather than refuse. A truly absent table/column/join key
740
+ // still falls through to the honest no_answer below.
741
+ const wantsGeneratedData = questionPlan.requestedShape.measures.length > 0
742
+ || questionPlan.requestedShape.dimensions.length > 0
743
+ || questionPlan.requestedShape.requiredOutputs.length > 0
744
+ || Boolean(questionPlan.requestedShape.topN);
745
+ const hasGeneratableContext = schemaContext.length > 0
746
+ || (input.contextPack?.allowedSqlContext?.relations.length ?? 0) > 0
747
+ || (input.contextPack?.allowedSqlContext?.sourceBlockSql.length ?? 0) > 0
748
+ || contextBlocks.length > 0;
749
+ if (!parsed.sql && !governedMetricAnswer && wantsGeneratedData && hasGeneratableContext) {
750
+ try {
751
+ proposed = await generateProposalWithOptionalTools({
752
+ provider,
753
+ messages: [...messages, { role: 'system', content: FORCE_JOIN_INSTRUCTION }],
754
+ tools: input.answerLoopTools,
755
+ questionPlan,
756
+ intent,
757
+ signal: input.signal,
758
+ reasoningEffort: input.reasoningEffort,
759
+ analysisDepth: input.analysisDepth,
760
+ toolCalls: proposalToolCalls,
761
+ });
762
+ parsed = parseProposal(proposed);
763
+ }
764
+ catch {
765
+ // keep the original decline; fall through to the honest no_answer.
766
+ }
767
+ }
424
768
  if (!parsed.sql) {
425
- const text = parsed.text || 'No answer (the model declined to propose SQL).';
769
+ // Deterministic honest refusal. The model's own decline prose is STOCHASTIC
770
+ // ("there's no combined dataset — show them separately" one run, a different
771
+ // phrasing the next), which reads as flaky and inconsistent across surfaces.
772
+ // When the question wanted data AND usable context existed (a groundable ask
773
+ // the model still declined even after the forced-join retry), surface ONE
774
+ // consistent, actionable message so the same question yields the same outcome
775
+ // every run and every surface — instead of passing through the model's varying
776
+ // text. A genuinely context-less ask keeps the plain honest message.
777
+ const declinedDespiteContext = wantsGeneratedData && hasGeneratableContext;
778
+ const text = declinedDespiteContext
779
+ ? 'I could not compose a governed query for this from the available tables and metrics. This usually needs a clearer join path or an explicit metric and grouping — name the specific measure and how to break it down, and I can generate a review-required draft.'
780
+ : parsed.text || 'No answer (the model declined to propose SQL).';
426
781
  return {
427
782
  kind: 'no_answer',
428
783
  sourceTier: 'no_answer',
@@ -430,6 +785,11 @@ async function runAnswerLoop(input) {
430
785
  reviewStatus: 'none',
431
786
  confidence: 0.1,
432
787
  text,
788
+ refusalCode: 'model_declined',
789
+ refusalDetails: {
790
+ code: 'model_declined',
791
+ message: text,
792
+ },
433
793
  answer: text,
434
794
  citations: [],
435
795
  memoryContext: input.memoryContext,
@@ -452,32 +812,30 @@ async function runAnswerLoop(input) {
452
812
  // model emitted to its real warehouse relation from the runtime schema BEFORE
453
813
  // governance validation. Same resolver the build path uses — one grounding,
454
814
  // no weak path. `allowedSqlContext` relations are already qualified.
455
- let grounding;
456
- if (parsed.sql && schemaContext.length > 0) {
457
- grounding = buildGroundingFromRuntimeRelations(schemaContext.map((table) => ({
458
- relation: table.relation,
459
- name: table.name,
460
- columns: table.columns.map((column) => ({ name: column.name, type: column.type, description: column.description })),
461
- })));
462
- parsed.sql = resolveRelationsInSql(parsed.sql, grounding, { prefer: 'qualified' }).sql;
815
+ if (parsed.sql) {
816
+ parsed.sql = contextLedger.qualifySql(parsed.sql).sql;
463
817
  }
464
818
  // Validation gate. Governed metric SQL synthesized from the semantic layer is
465
819
  // already trusted (deterministic + grounded); model SQL is validated against the
466
820
  // inspected context to catch hallucinated relations/columns.
467
821
  const semanticMetricRoute = activeTier === 'semantic_layer' && Boolean(semanticMetricMatch);
822
+ const scalarGovernedMetricRecoveryAllowed = questionPlan.requestedShape.dimensions.length === 0
823
+ && questionPlan.requestedShape.filters.length === 0
824
+ && !questionPlan.requestedShape.topN;
468
825
  let contextValidation;
469
826
  if (governedMetricAnswer) {
470
827
  contextValidation = { ok: true, warnings: [] };
471
828
  }
472
829
  else {
473
- const c = validateSqlAgainstLocalContext(parsed.sql, input.contextPack, {
830
+ const c = contextLedger.validateSql(parsed.sql, {
474
831
  question,
475
832
  intent,
476
833
  filterValues: input.followUp?.filters,
834
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
477
835
  });
478
836
  contextValidation = c.ok
479
837
  ? { ok: true, warnings: c.warnings }
480
- : { ok: false, error: c.error, warnings: c.warnings };
838
+ : { ok: false, code: c.code, error: c.error, warnings: c.warnings, offending: c.offending };
481
839
  }
482
840
  // Spec 17, part C — semantic-metric route recovery. A metric question is often
483
841
  // catalog-routed to clarify, leaving a thin contextPack that rejects otherwise-valid
@@ -489,41 +847,132 @@ async function runAnswerLoop(input) {
489
847
  // 2) Otherwise fall back to a CLEAN governed-metric definition (direct or exact
490
848
  // leaf-measure; no fuzzy family guess that could answer the wrong measure).
491
849
  if (!contextValidation.ok && semanticMetricRoute && semanticMetricMatch) {
492
- const grounded = grounding ? validateSqlAgainstGrounding(parsed.sql, grounding) : undefined;
850
+ const grounded = contextLedger.validateRuntimeGrounding(parsed.sql);
493
851
  if (grounded?.ok) {
494
852
  contextValidation = { ok: true, warnings: grounded.warnings };
495
853
  }
496
- else {
497
- const recovered = resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes);
854
+ else if (scalarGovernedMetricRecoveryAllowed) {
855
+ const recovered = resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes, input.semanticLayer);
498
856
  if (recovered) {
499
857
  semanticMetricMatch = { ...semanticMetricMatch, metric: recovered.metric };
500
- parsed.sql = grounding
501
- ? resolveRelationsInSql(recovered.sql, grounding, { prefer: 'qualified' }).sql
502
- : recovered.sql;
858
+ parsed.sql = contextLedger.qualifySql(recovered.sql).sql;
503
859
  parsed.viz = parsed.viz ?? 'single_value';
504
860
  parsed.text = parsed.text
505
861
  || `Answered from the governed metric ${recovered.metric.name}. This result is uncertified until reviewed and promoted.`;
506
862
  governedMetricAnswer = true;
507
863
  contextValidation = {
508
864
  ok: true,
509
- warnings: ['Generated SQL failed context validation; answered from the governed metric definition instead.'],
865
+ warnings: ['SQL preview failed context validation; answered from the governed metric definition instead.'],
510
866
  };
511
867
  }
512
868
  }
513
869
  }
870
+ if (!contextValidation.ok && !governedMetricAnswer && input.expandGroundingContext && canUseLaneRepair(repairBudgetState, 'reground')) {
871
+ recordLaneRepair(repairBudgetState, 'reground');
872
+ try {
873
+ const expansion = await input.expandGroundingContext({
874
+ question,
875
+ sql: parsed.sql,
876
+ code: contextValidation.code ?? 'insufficient_context',
877
+ offending: contextValidation.offending,
878
+ contextPack: contextLedger.contextPack,
879
+ schemaContext: contextLedger.schemaContext,
880
+ });
881
+ const merged = contextLedger.withExpansion(expansion);
882
+ if (merged.notes.length > 0) {
883
+ contextLedger = merged.ledger;
884
+ parsed.sql = contextLedger.qualifySql(parsed.sql).sql;
885
+ const revalidated = contextLedger.validateSql(parsed.sql, {
886
+ question,
887
+ intent,
888
+ filterValues: input.followUp?.filters,
889
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
890
+ });
891
+ contextValidation = revalidated.ok
892
+ ? {
893
+ ok: true,
894
+ warnings: [
895
+ ...revalidated.warnings,
896
+ ...merged.notes.map((note) => `Re-grounded metadata context before repair: ${note}`),
897
+ ],
898
+ }
899
+ : {
900
+ ok: false,
901
+ code: revalidated.code,
902
+ error: revalidated.error,
903
+ warnings: [
904
+ ...revalidated.warnings,
905
+ ...merged.notes.map((note) => `Re-grounded metadata context before repair: ${note}`),
906
+ ],
907
+ offending: revalidated.offending,
908
+ };
909
+ }
910
+ }
911
+ catch {
912
+ // Re-grounding is best-effort; the bounded self-repair below still runs.
913
+ }
914
+ }
915
+ // One bounded self-repair before refusing: hand the model the EXACT guard
916
+ // error (e.g. `column "product_name" outside the inspected columns for
917
+ // order_items`) so it can correct itself — usually by joining the relation
918
+ // that actually carries the column. Deterministic paths that never called the
919
+ // provider get the same single chance; any provider failure keeps the honest
920
+ // refusal below. This mirrors the engine-level repair loop, applied to the
921
+ // context-validation gate that previously refused on first failure.
922
+ if (!contextValidation.ok && !governedMetricAnswer && canUseLaneRepair(repairBudgetState, 'reground')) {
923
+ recordLaneRepair(repairBudgetState, 'reground');
924
+ try {
925
+ const failedSql = parsed.sql ?? '';
926
+ const repairPrompt = [
927
+ `Your SQL was rejected before execution: ${contextValidation.error}`,
928
+ formatOffendingValidationToken(contextValidation.offending),
929
+ formatValidationWarningsForPrompt(contextValidation.warnings),
930
+ 'Correct it using ONLY the relations and columns from the inspected context above.',
931
+ 'If a needed column lives on a different relation, JOIN that relation using the suggested join paths.',
932
+ 'If the requested column does not exist anywhere in the inspected context, return the closest answerable SQL and say what is missing.',
933
+ 'Return a single ```json fenced object: {"summary":"...","sql":"SELECT ...","viz":"table","outputs":["column"],"dql":{"entity":"...","dimensions":["..."],"filters":["..."]}}.',
934
+ ].join('\n');
935
+ const repairRaw = await provider.generate([...messages, { role: 'assistant', content: failedSql }, { role: 'user', content: repairPrompt }], { signal: input.signal, reasoningEffort: input.reasoningEffort });
936
+ const reparsed = parseProposal(repairRaw);
937
+ if (reparsed.sql) {
938
+ reparsed.sql = contextLedger.qualifySql(reparsed.sql).sql;
939
+ const revalidated = contextLedger.validateSql(reparsed.sql, {
940
+ question,
941
+ intent,
942
+ filterValues: input.followUp?.filters,
943
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
944
+ });
945
+ if (revalidated.ok) {
946
+ const repairedSql = reparsed.sql;
947
+ parsed.sql = repairedSql;
948
+ parsed.text = reparsed.text || parsed.text;
949
+ parsed.viz = reparsed.viz ?? parsed.viz;
950
+ applyParsedProposalMetadata(parsed, reparsed);
951
+ contextValidation = {
952
+ ok: true,
953
+ warnings: [...revalidated.warnings, 'Repaired after context-validation failure (1 repair).'],
954
+ };
955
+ }
956
+ }
957
+ }
958
+ catch {
959
+ // Repair is best-effort — the refusal below stays the honest fallback.
960
+ }
961
+ }
514
962
  if (!contextValidation.ok) {
515
- const text = `I could not safely prepare this generated SQL from the inspected context. ${contextValidation.error}`;
963
+ const text = `I could not safely prepare this review-required DQL artifact from the inspected context. The SQL preview failed validation: ${contextValidation.error}`;
516
964
  const analysisPlan = buildAnalysisPlan({
517
965
  question,
518
966
  intent,
519
- routeReason: catalogRoute?.reason ?? 'Generated SQL failed metadata context validation before preview execution or draft capture.',
967
+ routeReason: catalogRoute?.reason ?? 'SQL preview for the review-required DQL artifact failed metadata context validation before preview execution or draft capture.',
520
968
  selectedNodes: contextNodes,
521
969
  schemaContext,
522
970
  sql: parsed.sql,
523
971
  suggestedViz: parsed.viz ?? 'table',
524
972
  assumptions: [
525
- 'Generated SQL was rejected before execution because it did not match inspected metadata context.',
973
+ 'SQL preview was rejected before execution because it did not match inspected metadata context.',
526
974
  ...contextValidation.warnings,
975
+ ...deepCandidateNotes,
527
976
  ],
528
977
  });
529
978
  return {
@@ -533,13 +982,19 @@ async function runAnswerLoop(input) {
533
982
  reviewStatus: 'none',
534
983
  confidence: 0.15,
535
984
  text,
985
+ refusalCode: refusalCodeForValidation(contextValidation.code),
986
+ refusalDetails: {
987
+ code: contextValidation.code ?? 'insufficient_context',
988
+ message: contextValidation.error,
989
+ offending: contextValidation.offending,
990
+ },
536
991
  answer: text,
537
992
  proposedSql: parsed.sql,
538
993
  sql: parsed.sql,
539
994
  trustLabel: input.contextPack?.trustLabel,
540
995
  sourceCertifiedBlock: followUpSourceBlock?.name ?? input.followUp?.sourceBlockName,
541
996
  contextPackId: input.contextPack?.id,
542
- validationWarnings: contextValidation.warnings,
997
+ validationWarnings: [...contextValidation.warnings, ...deepCandidateNotes],
543
998
  selectedEvidence: input.contextPack?.evidenceRoles?.slice(0, 12),
544
999
  citations: [],
545
1000
  memoryContext: input.memoryContext,
@@ -557,7 +1012,7 @@ async function runAnswerLoop(input) {
557
1012
  }),
558
1013
  contextPack: input.contextPack,
559
1014
  considered,
560
- providerUsed: localProposal ? 'schema_planner' : provider.name,
1015
+ providerUsed: provider.name,
561
1016
  };
562
1017
  }
563
1018
  const generatedCitations = [
@@ -585,26 +1040,56 @@ async function runAnswerLoop(input) {
585
1040
  // The repair turn returns error-recovery prose ("the column X was not recognized…"),
586
1041
  // NOT a user-facing answer. Keep it for the trace only — never as the answer text.
587
1042
  let repairNarrative;
588
- if (input.executeGeneratedSql) {
1043
+ if (deepCandidateResult) {
1044
+ result = deepCandidateResult;
1045
+ }
1046
+ if (deepCandidateExecutionError) {
1047
+ executionError = deepCandidateExecutionError;
1048
+ }
1049
+ if (input.executeGeneratedSql && !result) {
589
1050
  try {
590
- result = await input.executeGeneratedSql(parsed.sql);
1051
+ if (!executionError)
1052
+ result = await input.executeGeneratedSql(parsed.sql);
591
1053
  }
592
1054
  catch (err) {
593
1055
  executionError = err instanceof Error ? err.message : String(err);
1056
+ }
1057
+ if (executionError) {
594
1058
  if (isRetryableGeneratedSqlError(executionError)) {
595
1059
  const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError, schemaContext);
596
- if (localRepairSql) {
597
- repairAttempts = 1;
598
- parsed.sql = localRepairSql;
599
- try {
600
- result = await input.executeGeneratedSql(parsed.sql);
601
- executionError = undefined;
1060
+ if (localRepairSql && canUseLaneRepair(repairBudgetState, 'execution')) {
1061
+ recordLaneRepair(repairBudgetState, 'execution');
1062
+ const qualifiedLocalRepairSql = contextLedger.qualifySql(localRepairSql).sql;
1063
+ const localRepairValidation = contextLedger.validateSql(qualifiedLocalRepairSql, {
1064
+ question,
1065
+ intent,
1066
+ filterValues: input.followUp?.filters,
1067
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
1068
+ });
1069
+ if (localRepairValidation.ok) {
1070
+ repairAttempts = 1;
1071
+ parsed.sql = qualifiedLocalRepairSql;
1072
+ contextValidation = {
1073
+ ok: true,
1074
+ warnings: [
1075
+ ...localRepairValidation.warnings,
1076
+ 'Execution-repaired SQL passed context validation.',
1077
+ ],
1078
+ };
1079
+ try {
1080
+ result = await input.executeGeneratedSql(parsed.sql);
1081
+ executionError = undefined;
1082
+ }
1083
+ catch (retryErr) {
1084
+ executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1085
+ }
602
1086
  }
603
- catch (retryErr) {
604
- executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1087
+ else {
1088
+ executionError = localRepairValidation.error;
605
1089
  }
606
1090
  }
607
- if (executionError) {
1091
+ if (executionError && canUseLaneRepair(repairBudgetState, 'execution')) {
1092
+ recordLaneRepair(repairBudgetState, 'execution');
608
1093
  const repairedRaw = await requestSqlRepair({
609
1094
  provider,
610
1095
  baseMessages: messages,
@@ -617,17 +1102,37 @@ async function runAnswerLoop(input) {
617
1102
  });
618
1103
  const repaired = parseProposal(repairedRaw);
619
1104
  if (repaired.sql) {
620
- repairAttempts += 1;
621
- // Adopt the corrected SQL, but do NOT let the repair prose become the answer.
622
- repairNarrative = repaired.text?.trim() || undefined;
623
- parsed.sql = repaired.sql;
624
- parsed.viz = repaired.viz ?? parsed.viz;
625
- try {
626
- result = await input.executeGeneratedSql(parsed.sql);
627
- executionError = undefined;
1105
+ repaired.sql = contextLedger.qualifySql(repaired.sql).sql;
1106
+ const repairedValidation = contextLedger.validateSql(repaired.sql, {
1107
+ question,
1108
+ intent,
1109
+ filterValues: input.followUp?.filters,
1110
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
1111
+ });
1112
+ if (repairedValidation.ok) {
1113
+ repairAttempts += 1;
1114
+ // Adopt the corrected SQL, but do NOT let the repair prose become the answer.
1115
+ repairNarrative = repaired.text?.trim() || undefined;
1116
+ parsed.sql = repaired.sql;
1117
+ parsed.viz = repaired.viz ?? parsed.viz;
1118
+ applyParsedProposalMetadata(parsed, repaired);
1119
+ contextValidation = {
1120
+ ok: true,
1121
+ warnings: [
1122
+ ...repairedValidation.warnings,
1123
+ 'Execution-repaired SQL passed context validation.',
1124
+ ],
1125
+ };
1126
+ try {
1127
+ result = await input.executeGeneratedSql(parsed.sql);
1128
+ executionError = undefined;
1129
+ }
1130
+ catch (retryErr) {
1131
+ executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1132
+ }
628
1133
  }
629
- catch (retryErr) {
630
- executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1134
+ else {
1135
+ executionError = repairedValidation.error;
631
1136
  }
632
1137
  }
633
1138
  }
@@ -638,26 +1143,73 @@ async function runAnswerLoop(input) {
638
1143
  question,
639
1144
  intent,
640
1145
  routeReason: catalogRoute?.reason ?? (intent === 'drillthrough'
641
- ? 'The user asked for a drill-through or follow-up, so DQL generated review-required SQL from the prior context and current metadata.'
1146
+ ? 'The user asked for a drill-through or follow-up, so DQL prepared a review-required artifact with a SQL preview from the prior context and current metadata.'
642
1147
  : 'The question asks for a custom analysis, ranking, breakdown, comparison, or grain that should not be answered by a loose certified block match.'),
643
1148
  selectedNodes: contextNodes,
644
1149
  schemaContext,
645
1150
  sql: parsed.sql,
646
1151
  suggestedViz: parsed.viz ?? 'table',
647
1152
  assumptions: [
648
- 'Generated SQL is an uncertified preview until an analyst reviews and promotes it.',
649
- ...(localProposal ? ['A local metadata planner selected a review-required SQL grain before provider generation.'] : []),
1153
+ 'The SQL preview is uncertified until an analyst reviews and promotes the DQL artifact.',
650
1154
  ...(repairNarrative ? [`Auto-corrected the query after an execution error: ${repairNarrative}`] : []),
651
1155
  ...contextValidation.warnings,
652
1156
  ...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
653
1157
  ],
654
1158
  repairAttempts,
655
1159
  });
1160
+ // Global top-N asks must return exactly N rows even when the generated SQL
1161
+ // returned more (missing/oversized LIMIT) — mirrors the certified path so a
1162
+ // "top 10" question never shows 200 rows. per_group scope is left intact by
1163
+ // trimResultToRequestedTopN. Domain-agnostic.
1164
+ let topNTrimNote;
1165
+ if (result) {
1166
+ const beforeRows = Array.isArray(result.rows) ? result.rows.length : result.rowCount;
1167
+ result = trimResultToRequestedTopN(result, questionPlan);
1168
+ const afterRows = Array.isArray(result.rows) ? result.rows.length : result.rowCount;
1169
+ if (afterRows < beforeRows) {
1170
+ topNTrimNote = `Showed the top ${questionPlan.requestedShape.topN?.n ?? afterRows} of ${beforeRows} rows the query returned.`;
1171
+ }
1172
+ }
1173
+ const resultShape = result ? validateAnswerResultShape(questionPlan, result) : undefined;
1174
+ // ANY question whose SQL executed but dropped multiple requested columns used to
1175
+ // REFUSE outright ("no governed answer"), throwing away a result that actually
1176
+ // ran. Instead, SURFACE the partial result (review-required) with a warning that
1177
+ // names the missing columns — a partial table the user can see and refine beats a
1178
+ // blank clarify. Domain-agnostic: works for every project, not just jaffle. The
1179
+ // shape gate downgrades trust rather than blocking the answer.
1180
+ const partialShapeMismatch = Boolean(resultShape && result && result.rowCount > 0
1181
+ && generatedResultShapeIsPartial(resultShape));
1182
+ const partialShapeWarning = partialShapeMismatch && resultShape
1183
+ ? `Partial answer: the result is missing ${resultShape.missingOutputs.join(', ')}`
1184
+ + `. Showing the closest table that executed — review before reuse, or narrow the question to the columns shown.`
1185
+ : undefined;
656
1186
  const validationWarnings = [
657
1187
  ...(input.contextPack?.warnings ?? []),
658
1188
  ...contextValidation.warnings,
1189
+ ...deepCandidateNotes,
1190
+ ...(resultShape?.warnings ?? []),
1191
+ ...(partialShapeWarning ? [partialShapeWarning] : []),
1192
+ ...(topNTrimNote ? [topNTrimNote] : []),
659
1193
  ...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
660
1194
  ];
1195
+ const generatedOutputs = parsed.outputs?.length ? parsed.outputs : resultColumnNames(result);
1196
+ const generatedRequestedFilters = mergeProposalStringLists(input.followUp?.filters, parsed.requestedFilters);
1197
+ const generatedRequestedDimensions = mergeProposalStringLists(input.followUp?.dimensions, parsed.requestedDimensions);
1198
+ const dqlArtifact = semanticBridgeAnswer?.dqlArtifact ?? buildGeneratedSqlDqlArtifact({
1199
+ question,
1200
+ sql: parsed.sql,
1201
+ intent,
1202
+ domain,
1203
+ followUp: input.followUp,
1204
+ contextPack: input.contextPack,
1205
+ sourceBlock: followUpSourceBlock ?? undefined,
1206
+ sourceDqlArtifact: input.followUp?.priorDqlArtifact,
1207
+ proposedEntity: parsed.proposedEntity,
1208
+ requestedFilters: generatedRequestedFilters,
1209
+ requestedDimensions: generatedRequestedDimensions,
1210
+ validationWarnings,
1211
+ outputs: generatedOutputs,
1212
+ });
661
1213
  let draftBlock;
662
1214
  let draftCaptureError;
663
1215
  if (input.captureGeneratedDraft && parsed.sql) {
@@ -669,7 +1221,13 @@ async function runAnswerLoop(input) {
669
1221
  followUp: input.followUp,
670
1222
  contextPack: input.contextPack,
671
1223
  sourceBlock: followUpSourceBlock ?? undefined,
1224
+ sourceDqlArtifact: input.followUp?.priorDqlArtifact,
1225
+ dqlArtifact,
1226
+ proposedEntity: parsed.proposedEntity,
1227
+ requestedFilters: generatedRequestedFilters,
1228
+ requestedDimensions: generatedRequestedDimensions,
672
1229
  validationWarnings,
1230
+ outputs: generatedOutputs,
673
1231
  });
674
1232
  }
675
1233
  catch (err) {
@@ -677,6 +1235,9 @@ async function runAnswerLoop(input) {
677
1235
  validationWarnings.push(`Draft capture failed: ${draftCaptureError}`);
678
1236
  }
679
1237
  }
1238
+ const answerDqlArtifact = dqlArtifact && draftBlock?.path
1239
+ ? { ...dqlArtifact, sourcePath: dqlArtifact.sourcePath ?? draftBlock.path }
1240
+ : dqlArtifact;
680
1241
  const sourceCertifiedBlock = followUpSourceBlock?.name ?? input.followUp?.sourceBlockName;
681
1242
  const trustExplanation = generatedTrustExplanation({
682
1243
  followUp: input.followUp,
@@ -684,15 +1245,20 @@ async function runAnswerLoop(input) {
684
1245
  draftBlock,
685
1246
  });
686
1247
  const cleanedSummary = cleanGeneratedSummary(parsed.text);
687
- const generatedText = trustExplanation
688
- ? [trustExplanation, cleanedSummary].filter(Boolean).join('\n\n')
689
- : cleanedSummary;
1248
+ const generatedText = [partialShapeWarning, trustExplanation, cleanedSummary]
1249
+ .filter(Boolean)
1250
+ .join('\n\n');
1251
+ const semanticMetricCertification = governedMetricAnswer && activeTier === 'semantic_layer'
1252
+ ? semanticMetricMatch?.metric.certification
1253
+ : undefined;
1254
+ const certifiedMetricAnswer = semanticMetricCertification === 'certified' || semanticMetricCertification === 'reviewed';
690
1255
  return {
691
1256
  kind: 'uncertified',
692
1257
  sourceTier: activeTier,
693
1258
  certification: 'ai_generated',
694
1259
  reviewStatus: 'draft_ready',
695
- confidence: activeTier === 'semantic_layer' ? 0.72 : 0.55,
1260
+ semanticMetricCertification,
1261
+ confidence: certifiedMetricAnswer ? 0.8 : governedMetricAnswer && activeTier === 'semantic_layer' ? 0.72 : 0.55,
696
1262
  text: generatedText,
697
1263
  answer: generatedText,
698
1264
  proposedSql: parsed.sql,
@@ -700,6 +1266,7 @@ async function runAnswerLoop(input) {
700
1266
  result,
701
1267
  executionError,
702
1268
  suggestedViz: parsed.viz ?? 'table',
1269
+ dqlArtifact: answerDqlArtifact,
703
1270
  draftBlock,
704
1271
  draftBlockId: draftBlock?.path,
705
1272
  promoteCommand: draftBlock ? `dql certify --from-draft ${draftBlock.path}` : undefined,
@@ -714,10 +1281,15 @@ async function runAnswerLoop(input) {
714
1281
  evidence: buildGeneratedEvidence({
715
1282
  question,
716
1283
  activeTier,
1284
+ terminalLane: governedMetricAnswer ? 'semantic' : 'generated',
1285
+ terminalDetail: governedMetricAnswer
1286
+ ? semanticBridgeAnswer?.metrics.join(', ') ?? semanticMetricMatch?.metric.name
1287
+ : answerDqlArtifact?.name ?? draftBlock?.path,
717
1288
  intent,
718
1289
  contextNodes,
719
1290
  schemaContext,
720
1291
  followUp: input.followUp,
1292
+ contextPack: input.contextPack,
721
1293
  businessHits,
722
1294
  semanticHits,
723
1295
  manifestHits,
@@ -728,15 +1300,42 @@ async function runAnswerLoop(input) {
728
1300
  executionError,
729
1301
  executorWasAvailable: Boolean(input.executeGeneratedSql),
730
1302
  analysisPlan,
1303
+ toolCalls: proposalToolCalls,
1304
+ budgetTrace: cascadeBudgetTrace(repairBudgetState),
731
1305
  }),
732
1306
  contextPack: input.contextPack,
733
1307
  considered,
734
- providerUsed: localProposal ? 'schema_planner' : provider.name,
1308
+ providerUsed: provider.name,
735
1309
  // Carry the governed metric match so the exit point can name a
736
1310
  // `semantic_metric` route (spec 17, part C).
737
- _semanticMetricMatch: activeTier === 'semantic_layer' ? semanticMetricMatch ?? undefined : undefined,
1311
+ _semanticMetricMatch: governedMetricAnswer && activeTier === 'semantic_layer' ? semanticMetricMatch ?? undefined : undefined,
1312
+ };
1313
+ }
1314
+ function trimResultToRequestedTopN(result, plan) {
1315
+ const topN = plan.requestedShape.topN;
1316
+ if (!topN || topN.scope === 'per_group' || !Array.isArray(result.rows) || result.rows.length <= topN.n)
1317
+ return result;
1318
+ return {
1319
+ ...result,
1320
+ rows: result.rows.slice(0, topN.n),
1321
+ rowCount: Math.min(result.rowCount, topN.n),
738
1322
  };
739
1323
  }
1324
+ function resultColumnNames(result) {
1325
+ if (!result?.columns)
1326
+ return undefined;
1327
+ const columns = result.columns
1328
+ .map((column) => {
1329
+ if (typeof column === 'string')
1330
+ return column.trim();
1331
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
1332
+ return column.name.trim();
1333
+ }
1334
+ return '';
1335
+ })
1336
+ .filter(Boolean);
1337
+ return columns?.length ? columns : undefined;
1338
+ }
740
1339
  const SYSTEM_PROMPT = `You are the DQL Analytics Agent.
741
1340
 
742
1341
  Rules:
@@ -746,16 +1345,21 @@ Rules:
746
1345
  2. Use certified DQL blocks only when the user's question exactly asks for that
747
1346
  saved block, direct KPI, or definition. For single-user/customer/account,
748
1347
  custom filters, rankings, breakdowns, comparisons, drill-throughs, or custom
749
- grains, generate review-required SQL from supplied metadata and cite
750
- certified context as evidence.
751
- 3. If you must generate SQL, return it inside a single \`\`\`sql code block.
752
- 4. Provide a one-paragraph natural-language summary BEFORE the SQL block. In it,
753
- state your QUERY PLAN first: the grain (one row per WHAT), the measures and how
754
- they aggregate, the dimensions/filters, and the exact join path + join keys
755
- between the grounded tables. Then write SQL that matches that plan — an explicit
756
- grain and join path prevents wrong-grain answers and fan-out (row-multiplying) joins.
757
- 5. Suggest a visualization type from this list, on a line starting with "Viz:":
758
- line, bar, area, pie, single_value, table, pivot, kpi.
1348
+ grains, prepare a review-required DQL artifact with a SQL preview from
1349
+ supplied metadata and cite certified context as evidence.
1350
+ 3. If you must create a SQL preview, return a single \`\`\`json fenced object with:
1351
+ {"summary": "...", "sql": "SELECT ...", "viz": "table", "outputs": ["column_a"], "dql": {"entity": "orders", "dimensions": ["region"], "filters": ["last 30 days"]}}.
1352
+ The SQL string must be one read-only SELECT/WITH statement. Do not include a
1353
+ separate SQL code fence when using JSON. The optional "dql" object is
1354
+ metadata for the deterministic draft-DQL renderer; do not hand-write a full
1355
+ DQL block.
1356
+ 4. In summary, state your QUERY PLAN first: the grain (one row per WHAT), the
1357
+ measures and how they aggregate, the dimensions/filters, and the exact join
1358
+ path + join keys between the grounded tables. Then make the SQL match that
1359
+ plan — an explicit grain and join path prevents wrong-grain answers and
1360
+ fan-out (row-multiplying) joins.
1361
+ 5. Use a visualization type from this list in the "viz" JSON field: line, bar,
1362
+ area, pie, single_value, table, pivot, kpi.
759
1363
  6. NEVER fabricate column names that are not present in the supplied schema,
760
1364
  dbt metadata, or certified source SQL shape context. If a requested filter
761
1365
  value is supplied as a matched value, prefer the table and column that
@@ -765,18 +1369,32 @@ Rules:
765
1369
  proposed SQL. Do not emit INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, COPY,
766
1370
  PRAGMA, SET, or multiple statements.
767
1371
  8. If the schema is insufficient to answer, say so explicitly and ask a
768
- clarifying question instead of guessing.
1372
+ clarifying question instead of guessing. But a MULTI-ENTITY question ("top
1373
+ customers who bought the top products", "accounts with the most overdue
1374
+ invoices") is NOT insufficient context when the grounded tables and a join
1375
+ route between them are supplied (see any "Knowledge graph join routes"
1376
+ section). In that case you MUST compose the joined SELECT that answers it
1377
+ directly — never refuse it or offer to show the parts as separate datasets.
1378
+ Reserve the clarifying question for a genuinely absent table, column, or join
1379
+ key.
769
1380
  9. Write directly to the analyst. Do not say "the user is asking", "the user
770
1381
  requested", "I will generate", or describe internal routing. State the
771
- answer, the certified context used, and the review requirement.
1382
+ answer, the certified context used, the DQL artifact expectation, and the
1383
+ review requirement.
772
1384
  10. For notebook research, SQL build or repair, DQL import/build, or DQL reuse
773
1385
  checks, start with one line in this form: "Outcome: <decision>". Use one of
774
- these decisions: Reuse certified block, Use existing draft, Generate SQL
775
- cell, Fix SQL, Create DQL draft, Needs review, Cannot answer yet.`;
776
- function renderContextPrompt(blocks, businessContext, others, activeTier, memoryContext, extraContext, followUp, schemaContext = [], intent = 'ad_hoc_analysis', contextPack) {
1386
+ these decisions: Reuse certified block, Use existing draft, Review SQL preview,
1387
+ Fix SQL, Create DQL draft, Needs review, Cannot answer yet.`;
1388
+ // Appended on a single retry when the model declined SQL for a question that the
1389
+ // grounded schema + KG join routes can actually answer. Turns a false "no
1390
+ // combined dataset — show them separately" refusal into the join the user asked
1391
+ // for, while still allowing an honest refusal if context is truly missing.
1392
+ const FORCE_JOIN_INSTRUCTION = `Your previous attempt declined to produce SQL. Re-read the supplied schema, metadata, and any "Knowledge graph join routes": this question CAN be answered by joining the grounded tables along their documented keys. Do NOT refuse, and do NOT suggest showing the datasets separately — compose ONE read-only SELECT/WITH that joins the relevant tables to answer it directly, following the JSON contract from rule 3. State the grain and the exact join path in the summary. Only if a required table, column, or join key is truly absent from the supplied context may you still ask a clarifying question.`;
1393
+ function renderContextPrompt(blocks, businessContext, others, activeTier, memoryContext, extraContext, followUp, schemaContext = [], intent = 'ad_hoc_analysis', contextPack, conversationSnapshot, kgJoinPathHints = [], budget = QUICK_PROMPT_CONTEXT_BUDGET) {
777
1394
  const intentSection = `## Routing intent\n\nintent: ${intent}\n${intent === 'exact_certified_lookup'
778
1395
  ? 'Use a certified artifact only if it exactly answers the question.'
779
- : 'Generate review-required SQL for this question. Certified blocks are trusted context, not a reason to answer the wrong grain.'}`;
1396
+ : 'Prepare a review-required DQL artifact with a SQL preview for this question. Certified blocks are trusted context, not a reason to answer the wrong grain.'}`;
1397
+ const budgetSection = `\n\n## Context budget\n\nContext budget: ${budget.label} (relations=${budget.relationCardLimit}, relation_columns=${budget.relationColumnLimit}, objects=${budget.contextObjectLimit}, edges=${budget.edgeLimit}).`;
780
1398
  const blockSection = blocks.length > 0
781
1399
  ? `## Relevant DQL blocks\n\n${blocks
782
1400
  .map((b) => `- \`${b.nodeId}\` (${b.domain ?? 'unscoped'}, ${b.status ?? b.certification ?? 'review_required'}): ${b.description ?? b.llmContext ?? '(no description)'}`)
@@ -792,12 +1410,15 @@ function renderContextPrompt(blocks, businessContext, others, activeTier, memory
792
1410
  .map((n) => `- ${n.kind} \`${n.name}\`${n.domain ? ` (domain: ${n.domain})` : ''}${n.description ? ` — ${n.description}` : ''}${n.llmContext ? `\n ${n.llmContext.replace(/\n/g, '\n ')}` : ''}`)
793
1411
  .join('\n')}`
794
1412
  : '';
1413
+ const kgJoinSection = kgJoinPathHints.length > 0
1414
+ ? `\n\n## Knowledge graph join routes\n\nUse these as relationship/navigation evidence for multi-entity analysis. SQL must still use inspected relation columns from the schema or metadata context.\n${kgJoinPathHints.map((hint) => `- ${hint}`).join('\n')}`
1415
+ : '';
795
1416
  const schemaSection = schemaContext.length > 0
796
1417
  ? `\n\n## Runtime schema context\n\nUse only these runtime relations and columns when generating SQL unless the dbt manifest context gives an equivalent relation.\n${schemaContext
797
- .slice(0, 12)
1418
+ .slice(0, budget.schemaTableLimit)
798
1419
  .map((table) => {
799
1420
  const cols = table.columns
800
- .slice(0, 50)
1421
+ .slice(0, budget.schemaColumnLimit)
801
1422
  .map((col) => {
802
1423
  const sampleValues = col.sampleValues?.length
803
1424
  ? `; matched values: ${col.sampleValues.slice(0, 4).map(formatPromptValue).join(', ')}`
@@ -817,27 +1438,118 @@ function renderContextPrompt(blocks, businessContext, others, activeTier, memory
817
1438
  : '';
818
1439
  // Approved, scope-matched correction hints from the Hint Graph. These are
819
1440
  // human-reviewed lessons (often from a prior wrong→right SQL correction), so
820
- // they carry more weight than advisory memory when generating SQL — but they
1441
+ // they carry more weight than advisory memory when preparing a SQL preview — but they
821
1442
  // still must NOT override a certified artifact that already answers exactly.
822
1443
  const appliedHints = contextPack?.appliedHints ?? [];
823
1444
  const hintsSection = appliedHints.length > 0
824
- ? `\n\n## Applied governed hints (human-approved corrections)\n\nReviewed, scope-matched corrections from your team. Apply them when generating SQL to avoid known mistakes; they refine generated SQL but MUST NOT override a certified artifact that already answers the question.\n${appliedHints
1445
+ ? `\n\n## Applied governed hints (human-approved corrections)\n\nReviewed, scope-matched corrections from your team. Apply them when preparing a SQL preview to avoid known mistakes; they refine the DQL artifact's SQL preview but MUST NOT override a certified artifact that already answers the question.\n${appliedHints
825
1446
  .slice(0, 6)
826
1447
  .map((h) => `- \`${h.title}\`: ${h.guidance}${h.correctedSql ? `\n corrected SQL pattern: ${h.correctedSql.replace(/\s+/g, ' ').trim().slice(0, 240)}` : ''}`)
827
1448
  .join('\n')}`
828
1449
  : '';
829
1450
  const extraSection = extraContext?.trim()
830
- ? `\n\n## Current notebook/app context\n\nThis context may help interpret the user's request, but it MUST NOT override certified artifacts, semantic metrics, dbt metadata, or generated SQL validation.\n\n${extraContext.trim()}`
1451
+ ? `\n\n## Current notebook/app context\n\nThis context may help interpret the user's request, but it MUST NOT override certified artifacts, semantic metrics, dbt metadata, or SQL preview validation.\n\n${extraContext.trim()}`
831
1452
  : '';
832
1453
  const followUpSection = followUp
833
1454
  ? `\n\n## Follow-up routing context\n\n${renderFollowUpContext(followUp)}`
834
1455
  : '';
1456
+ const conversationSection = conversationSnapshot
1457
+ ? `\n\n## Conversation state (session)\n\n${renderConversationSnapshot(conversationSnapshot)}`
1458
+ : '';
835
1459
  const contextPackSection = contextPack
836
- ? `\n\n## Local metadata context pack\n\n${renderContextPackForPrompt(contextPack)}`
1460
+ ? `\n\n## Local metadata context pack\n\n${renderContextPackForPrompt(contextPack, budget)}`
837
1461
  : '';
838
- return `${intentSection}\n\n${blockSection}${businessSection}${otherSection}${schemaSection}${contextPackSection}${memorySection}${hintsSection}${extraSection}${followUpSection}`;
1462
+ return `${intentSection}${budgetSection}\n\n${blockSection}${businessSection}${otherSection}${kgJoinSection}${schemaSection}${contextPackSection}${memorySection}${hintsSection}${conversationSection}${extraSection}${followUpSection}`;
839
1463
  }
840
- function renderContextPackForPrompt(contextPack) {
1464
+ function buildKgJoinPathHints(kg, nodes, questionPlan) {
1465
+ const candidates = nodes
1466
+ .filter((node) => ['block', 'business_view', 'metric', 'entity', 'semantic_model', 'dbt_model', 'dbt_source'].includes(node.kind))
1467
+ .slice(0, 8);
1468
+ if (candidates.length < 2)
1469
+ return [];
1470
+ const domains = new Set(candidates.map((node) => node.domain).filter((domain) => Boolean(domain)));
1471
+ const shouldSearch = domains.size > 1 || questionPlan.entities.length > 1 || questionPlan.dimensionTerms.length > 1;
1472
+ if (!shouldSearch)
1473
+ return [];
1474
+ const hints = [];
1475
+ const seen = new Set();
1476
+ for (let leftIndex = 0; leftIndex < candidates.length; leftIndex += 1) {
1477
+ for (let rightIndex = leftIndex + 1; rightIndex < candidates.length; rightIndex += 1) {
1478
+ if (hints.length >= 6)
1479
+ return hints;
1480
+ const left = candidates[leftIndex];
1481
+ const right = candidates[rightIndex];
1482
+ const path = kg.findJoinPath(left.nodeId, right.nodeId, 4);
1483
+ if (!path || path.length < 2)
1484
+ continue;
1485
+ const key = path.join('>');
1486
+ if (seen.has(key))
1487
+ continue;
1488
+ seen.add(key);
1489
+ hints.push(path.map((nodeId) => formatKgPathNode(kg, nodeId)).join(' -> '));
1490
+ }
1491
+ }
1492
+ return hints;
1493
+ }
1494
+ function formatKgPathNode(kg, nodeId) {
1495
+ const node = kg.getNode(nodeId);
1496
+ if (!node)
1497
+ return nodeId;
1498
+ return `${node.kind}:${node.name}`;
1499
+ }
1500
+ /**
1501
+ * Bounded rendering of the persisted conversation snapshot: working state one-liner
1502
+ * (≤300 chars), compacted rolling summary (≤600), and up to 4 recent turns (≤280 each).
1503
+ */
1504
+ function renderConversationSnapshot(snapshot) {
1505
+ const parts = [];
1506
+ const state = snapshot.workingState;
1507
+ if (state) {
1508
+ const stateLine = [
1509
+ state.entities.length ? `entities=${state.entities.join(',')}` : '',
1510
+ state.measures.length ? `measures=${state.measures.join(',')}` : '',
1511
+ state.dimensions.length ? `dimensions=${state.dimensions.join(',')}` : '',
1512
+ state.filters.length ? `filters=${state.filters.map((f) => f.value).join(',')}` : '',
1513
+ state.limit ? `limit=${state.limit}` : '',
1514
+ state.timeframe ? `timeframe=${state.timeframe}` : '',
1515
+ snapshot.topicRelation ? `topic=${snapshot.topicRelation}` : '',
1516
+ ].filter(Boolean).join('; ');
1517
+ if (stateLine)
1518
+ parts.push(`Working state: ${stateLine.slice(0, 300)}`);
1519
+ }
1520
+ else if (snapshot.topicRelation) {
1521
+ parts.push(`Working state: topic=${snapshot.topicRelation}`);
1522
+ }
1523
+ if (snapshot.rollingSummary) {
1524
+ parts.push(`Earlier in this conversation (compacted):\n${snapshot.rollingSummary.slice(0, 600)}`);
1525
+ }
1526
+ if (snapshot.recentTurns.length > 0) {
1527
+ const turns = snapshot.recentTurns.slice(-4).map((turn, index) => {
1528
+ return `${index + 1}. ${renderConversationSnapshotTurnLine(turn, 520)}`;
1529
+ });
1530
+ parts.push(`Recent turns:\n${turns.join('\n')}`);
1531
+ }
1532
+ if (snapshot.recalledTurns?.length) {
1533
+ parts.push(`Recalled earlier turns (semantic match):\n${snapshot.recalledTurns.slice(0, 3)
1534
+ .map((turn) => `- ${renderConversationSnapshotTurnLine(turn, 520)}`)
1535
+ .join('\n')}`);
1536
+ }
1537
+ parts.push('Use this only where the question refers to it. On a new topic, answer fresh.');
1538
+ return parts.join('\n\n');
1539
+ }
1540
+ function renderConversationSnapshotTurnLine(turn, max) {
1541
+ const line = [
1542
+ `Q: ${turn.question}`,
1543
+ turn.answerSummary ? `A: ${turn.answerSummary}` : '',
1544
+ turn.resultColumns?.length ? `cols: ${turn.resultColumns.slice(0, 6).join(', ')}` : '',
1545
+ typeof turn.resultRowCount === 'number' ? `rows: ${turn.resultRowCount}` : '',
1546
+ turn.sourceSql ? `sql: ${compactSqlSnippet(turn.sourceSql, 180)}` : '',
1547
+ turn.dqlArtifact?.name ? `dql: ${turn.dqlArtifact.name}` : '',
1548
+ turn.sourceCertifiedBlock ? `block: ${turn.sourceCertifiedBlock}` : '',
1549
+ ].filter(Boolean).join(' | ');
1550
+ return line.slice(0, max);
1551
+ }
1552
+ function renderContextPackForPrompt(contextPack, budget) {
841
1553
  const questionPlan = contextPack.questionPlan
842
1554
  ? [
843
1555
  `Question plan: ${contextPack.questionPlan.mode} -> ${contextPack.questionPlan.routeIntent}`,
@@ -852,10 +1564,13 @@ function renderContextPackForPrompt(contextPack) {
852
1564
  const certifiedApplicability = contextPack.routeDecision?.certifiedApplicability
853
1565
  ? `\nCertified applicability: ${contextPack.routeDecision.certifiedApplicability.name} is ${contextPack.routeDecision.certifiedApplicability.kind} (${contextPack.routeDecision.certifiedApplicability.reasons.join('; ')})`
854
1566
  : '';
1567
+ const blockFit = contextPack.routeDecision?.blockFit
1568
+ ? `\nCertified block fit: ${contextPack.routeDecision.blockFit.kind} / ${contextPack.routeDecision.blockFit.confidence} (${contextPack.routeDecision.blockFit.reasons.join('; ')})`
1569
+ : '';
855
1570
  const warnings = contextPack.warnings.length
856
- ? `Warnings:\n${contextPack.warnings.slice(0, 8).map((warning) => `- ${warning}`).join('\n')}\n`
1571
+ ? `Warnings:\n${contextPack.warnings.slice(0, budget.warningLimit).map((warning) => `- ${warning}`).join('\n')}\n`
857
1572
  : '';
858
- const objects = contextPack.objects.slice(0, 18).map((object) => {
1573
+ const objects = contextPack.objects.slice(0, budget.contextObjectLimit).map((object) => {
859
1574
  const detail = [
860
1575
  object.objectType,
861
1576
  object.domain ? `domain: ${object.domain}` : '',
@@ -868,38 +1583,43 @@ function renderContextPackForPrompt(contextPack) {
868
1583
  ? `\nCandidate conflicts:\n${contextPack.retrievalDiagnostics.candidateConflicts.slice(0, 4).map((conflict) => `- ${conflict.reason} ${conflict.prompt}`).join('\n')}`
869
1584
  : '';
870
1585
  const route = contextPack.routeDecision
871
- ? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}${certifiedApplicability}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
1586
+ ? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}${certifiedApplicability}${blockFit}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
872
1587
  : '';
873
- const allowed = renderAllowedSqlRelationsForPrompt(contextPack);
874
- const joins = renderCandidateJoinsForPrompt(contextPack);
1588
+ const allowed = renderAllowedSqlRelationsForPrompt(contextPack, budget);
1589
+ const joins = renderCandidateJoinsForPrompt(contextPack, budget);
1590
+ const lineage = renderColumnLineageForPrompt(contextPack, budget);
1591
+ const edges = renderContextEdgesForPrompt(contextPack, budget);
875
1592
  const relationDiagnostics = contextPack.retrievalDiagnostics?.selectedRelations?.length
876
- ? `\nSelected relation reasoning:\n${contextPack.retrievalDiagnostics.selectedRelations.slice(0, 8).map((relation) => `- ${relation.relation} (score ${relation.score.toFixed(1)}): ${relation.reason}`).join('\n')}`
1593
+ ? `\nSelected relation reasoning:\n${contextPack.retrievalDiagnostics.selectedRelations.slice(0, budget.selectedRelationReasonLimit).map((relation) => `- ${relation.relation} (score ${relation.score.toFixed(1)}): ${relation.reason}`).join('\n')}`
877
1594
  : '';
878
- const sourceSql = renderSourceBlockSqlContext(contextPack);
1595
+ const sourceSql = renderSourceBlockSqlContext(contextPack, budget);
879
1596
  return [
880
1597
  `context_pack_id: ${contextPack.id}`,
881
1598
  `trust_label: ${contextPack.trustLabel}`,
1599
+ contextPack.trustLabelInfo ? `trust_label_canonical: ${contextPack.trustLabelInfo.display}` : '',
882
1600
  questionPlan.trim(),
883
1601
  route.trim(),
884
1602
  warnings.trim(),
885
1603
  `Selected evidence:\n${objects || '- none'}`,
886
1604
  allowed.trim(),
887
1605
  joins.trim(),
1606
+ edges.trim(),
1607
+ lineage.trim(),
888
1608
  relationDiagnostics.trim(),
889
1609
  sourceSql.trim(),
890
1610
  conflicts.trim(),
891
1611
  ].filter(Boolean).join('\n');
892
1612
  }
893
- function renderCandidateJoinsForPrompt(contextPack) {
1613
+ function renderCandidateJoinsForPrompt(contextPack, budget) {
894
1614
  const joins = contextPack.retrievalDiagnostics.selectedJoinPaths?.length
895
- ? contextPack.retrievalDiagnostics.selectedJoinPaths.slice(0, 8).map((join) => ({
1615
+ ? contextPack.retrievalDiagnostics.selectedJoinPaths.slice(0, budget.joinPathLimit).map((join) => ({
896
1616
  leftRelation: join.leftRelation,
897
1617
  leftColumn: join.leftColumn,
898
1618
  rightRelation: join.rightRelation,
899
1619
  rightColumn: join.rightColumn,
900
1620
  reason: join.reason,
901
1621
  }))
902
- : buildCandidateJoinPaths(schemaContextWithAllowedSqlContext([], contextPack)).slice(0, 8);
1622
+ : buildCandidateJoinPaths(schemaContextWithAllowedSqlContext([], contextPack)).slice(0, budget.joinPathLimit);
903
1623
  if (joins.length === 0)
904
1624
  return '';
905
1625
  return [
@@ -907,18 +1627,59 @@ function renderCandidateJoinsForPrompt(contextPack) {
907
1627
  ...joins.map((join) => `- ${join.leftRelation}.${join.leftColumn} -> ${join.rightRelation}.${join.rightColumn}${join.reason ? ` (${join.reason})` : ''}`),
908
1628
  ].join('\n');
909
1629
  }
910
- function renderAllowedSqlRelationsForPrompt(contextPack) {
1630
+ function renderContextEdgesForPrompt(contextPack, budget) {
1631
+ if (budget.edgeLimit <= 0)
1632
+ return '';
1633
+ const edges = contextPack.edges.slice(0, budget.edgeLimit);
1634
+ if (edges.length === 0)
1635
+ return '';
1636
+ return [
1637
+ 'Context graph edges:',
1638
+ 'Use these object relationships as metadata evidence; SQL still must use inspected relation columns and join keys.',
1639
+ ...edges.map((edge) => {
1640
+ const confidence = typeof edge.confidence === 'number' ? ` (confidence ${edge.confidence.toFixed(2)})` : '';
1641
+ return `- ${edge.edgeType}: ${edge.fromKey} -> ${edge.toKey}${confidence}`;
1642
+ }),
1643
+ ].join('\n');
1644
+ }
1645
+ function renderColumnLineageForPrompt(contextPack, budget) {
1646
+ const lineageEdges = contextPack.edges
1647
+ .filter((edge) => edge.edgeType === 'derives_from')
1648
+ .slice(0, budget.lineageEdgeLimit);
1649
+ if (lineageEdges.length === 0)
1650
+ return '';
1651
+ const objectsByKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
1652
+ return [
1653
+ 'Column lineage from governed metadata:',
1654
+ 'Use this when adapting certified blocks or dbt models into a DQL artifact SQL preview so derived output columns map back to their physical source columns.',
1655
+ ...lineageEdges.map((edge) => {
1656
+ const from = objectsByKey.get(edge.fromKey);
1657
+ const to = objectsByKey.get(edge.toKey);
1658
+ const aggregateFn = typeof edge.payload?.aggregateFn === 'string' && edge.payload.aggregateFn.trim()
1659
+ ? ` via ${edge.payload.aggregateFn.trim().toUpperCase()}`
1660
+ : '';
1661
+ const confidence = typeof edge.confidence === 'number' ? `, confidence ${edge.confidence.toFixed(2)}` : '';
1662
+ return `- ${formatLineageObjectForPrompt(from, edge.fromKey)} derives from ${formatLineageObjectForPrompt(to, edge.toKey)}${aggregateFn}${confidence}`;
1663
+ }),
1664
+ ].join('\n');
1665
+ }
1666
+ function formatLineageObjectForPrompt(object, fallbackKey) {
1667
+ if (!object)
1668
+ return fallbackKey;
1669
+ return object.fullName ?? object.name ?? object.objectKey;
1670
+ }
1671
+ function renderAllowedSqlRelationsForPrompt(contextPack, budget) {
911
1672
  const relations = contextPack.allowedSqlContext?.relations ?? [];
912
1673
  if (relations.length === 0)
913
1674
  return '';
914
1675
  const selectedLookup = selectedRelationLookup(contextPack);
915
- const cards = relations.slice(0, 12).map((relation, index) => {
1676
+ const cards = relations.slice(0, budget.relationCardLimit).map((relation, index) => {
916
1677
  const selection = relationSelectionFor(relation.relation, selectedLookup);
917
1678
  const rank = selection?.rank ?? index + 1;
918
1679
  const score = typeof selection?.score === 'number' ? `, score ${selection.score.toFixed(1)}` : '';
919
1680
  const reason = selection?.reason ? `\n why selected: ${selection.reason}` : '';
920
1681
  const columns = relation.columns.length
921
- ? relation.columns.slice(0, 32).map(formatRelationColumnForPrompt).join(', ')
1682
+ ? relation.columns.slice(0, budget.relationColumnLimit).map(formatRelationColumnForPrompt).join(', ')
922
1683
  : '(columns unavailable; use certified source SQL shape or inspect metadata before inventing columns)';
923
1684
  return [
924
1685
  `- [rank ${rank}${score}] ${relation.relation} (${relation.source})`,
@@ -926,11 +1687,37 @@ function renderAllowedSqlRelationsForPrompt(contextPack) {
926
1687
  `\n columns: ${columns}`,
927
1688
  ].join('');
928
1689
  });
1690
+ const otherRelations = renderOtherAvailableRelationsForPrompt(contextPack, budget);
929
1691
  return [
930
1692
  'Selected SQL relation context:',
931
1693
  'Use these ranked relations and columns as the primary SQL-generation boundary. Prefer lower rank when multiple relations look plausible.',
932
1694
  ...cards,
933
- ].join('\n');
1695
+ otherRelations,
1696
+ ].filter(Boolean).join('\n');
1697
+ }
1698
+ function renderOtherAvailableRelationsForPrompt(contextPack, budget) {
1699
+ const names = new Map();
1700
+ for (const relation of (contextPack.allowedSqlContext?.relations ?? []).slice(budget.otherRelationStart, budget.otherRelationEnd)) {
1701
+ names.set(normalizePromptRelationName(relation.relation), relation.relation);
1702
+ }
1703
+ for (const item of contextPack.retrievalDiagnostics.topRejected ?? []) {
1704
+ if (!isRelationLikeRejectedObject(item.objectType))
1705
+ continue;
1706
+ names.set(normalizePromptRelationName(item.name), item.name);
1707
+ }
1708
+ const values = [...names.values()].slice(0, budget.otherRelationLimit);
1709
+ if (values.length === 0)
1710
+ return '';
1711
+ return `Other available relations (names only - expand context before using columns): ${values.join(', ')}`;
1712
+ }
1713
+ function isRelationLikeRejectedObject(objectType) {
1714
+ return objectType === 'dbt_model'
1715
+ || objectType === 'dbt_source'
1716
+ || objectType === 'warehouse_table'
1717
+ || objectType === 'runtime_table';
1718
+ }
1719
+ function normalizePromptRelationName(name) {
1720
+ return name.toLowerCase().replace(/["`]/g, '').trim();
934
1721
  }
935
1722
  function formatRelationColumnForPrompt(column) {
936
1723
  const type = column.type ? ` ${column.type}` : '';
@@ -940,17 +1727,17 @@ function formatRelationColumnForPrompt(column) {
940
1727
  : '';
941
1728
  return `${column.name}${type}${description}${samples}`;
942
1729
  }
943
- function renderSourceBlockSqlContext(contextPack) {
1730
+ function renderSourceBlockSqlContext(contextPack, budget) {
944
1731
  const sources = contextPack.allowedSqlContext?.sourceBlockSql ?? [];
945
1732
  if (sources.length === 0)
946
1733
  return '';
947
- const lines = sources.slice(0, 5).map((source) => {
1734
+ const lines = sources.slice(0, budget.sourceSqlLimit).map((source) => {
948
1735
  const shape = extractSimpleSelectShape(source.sql);
949
1736
  const projectedColumns = shape
950
1737
  ? shape.selectExpressions
951
1738
  .map(selectExpressionOutputName)
952
1739
  .filter((value) => Boolean(value))
953
- .slice(0, 24)
1740
+ .slice(0, budget.sourceSqlColumnLimit)
954
1741
  .join(', ')
955
1742
  : '';
956
1743
  const snippet = compactSqlSnippet(source.sql, 280);
@@ -966,7 +1753,7 @@ function renderSourceBlockSqlContext(contextPack) {
966
1753
  snippet ? ` sql: ${snippet}` : '',
967
1754
  ].filter(Boolean).join('\n');
968
1755
  });
969
- return `## Worked examples from certified blocks (few-shot patterns)\n\nThese certified blocks already answer similar questions. Learn their join paths, grain, and filters and ADAPT them to the question — do not copy blindly, and do not relabel generated SQL as certified.\n${lines.join('\n')}`;
1756
+ return `## Worked examples from certified blocks (few-shot patterns)\n\nThese certified blocks already answer similar questions. Learn their join paths, grain, and filters and ADAPT them to the question — do not copy blindly, and do not relabel a review-required DQL artifact or SQL preview as certified.\n${lines.join('\n')}`;
970
1757
  }
971
1758
  function contextPackCitations(contextPack, limit) {
972
1759
  if (!contextPack)
@@ -992,6 +1779,8 @@ function shouldUseCertifiedRoute(route, intent) {
992
1779
  return intent === 'exact_certified_lookup' || intent === 'definition_lookup';
993
1780
  }
994
1781
  function certifiedHitFromContextPack(contextPack, kg) {
1782
+ if (contextPack?.routeDecision.blockFit && !certifiedFitAllowsTier1(contextPack.routeDecision.blockFit))
1783
+ return null;
995
1784
  const key = contextPack?.routeDecision.exactObjectKey;
996
1785
  if (!key)
997
1786
  return null;
@@ -1040,16 +1829,6 @@ function sourceTierFromContextPack(contextPack) {
1040
1829
  return 'certified_artifact';
1041
1830
  return undefined;
1042
1831
  }
1043
- function isGeneratedAgentIntent(intent) {
1044
- return intent === 'ad_hoc_analysis'
1045
- || intent === 'drillthrough'
1046
- || intent === 'ad_hoc_ranking'
1047
- || intent === 'driver_breakdown'
1048
- || intent === 'diagnose_change'
1049
- || intent === 'segment_compare'
1050
- || intent === 'entity_drilldown'
1051
- || intent === 'anomaly_investigation';
1052
- }
1053
1832
  function metadataObjectKindForCitation(objectType) {
1054
1833
  if (objectType === 'dql_block')
1055
1834
  return 'block';
@@ -1092,12 +1871,51 @@ function renderFollowUpContext(followUp) {
1092
1871
  followUp.sourceAnswer ? `source answer: ${followUp.sourceAnswer.slice(0, 700)}` : '',
1093
1872
  followUp.filters?.length ? `requested filters: ${followUp.filters.join(', ')}` : '',
1094
1873
  followUp.dimensions?.length ? `requested dimensions: ${followUp.dimensions.join(', ')}` : '',
1874
+ followUp.priorResultColumns?.length ? `prior result columns: ${followUp.priorResultColumns.join(', ')}` : '',
1875
+ followUp.priorResultValues ? `prior result values: ${formatPriorResultValues(followUp.priorResultValues)}` : '',
1876
+ followUp.priorResultRef ? renderPriorResultReference(followUp.priorResultRef) : '',
1877
+ followUp.priorDqlArtifact ? renderPriorDqlArtifactReference(followUp.priorDqlArtifact) : '',
1878
+ followUp.priorLimit ? `prior result limit: ${followUp.priorLimit}` : '',
1095
1879
  ].filter(Boolean);
1096
1880
  const rule = followUp.kind === 'drilldown'
1097
- ? 'routing rule: find a distinct certified drilldown block first; if none exists, generate review-required SQL as a draft drilldown. Do not silently re-run the source block unless it explicitly supports the requested filter or dimension.'
1098
- : 'routing rule: reuse the prior certified block when the user asks a generic follow-up.';
1881
+ ? 'routing rule: find a distinct certified drilldown block first; if none exists, prepare a review-required DQL artifact with a SQL preview as a draft drilldown. Do not silently re-run the source block unless it explicitly supports the requested filter or dimension.'
1882
+ : followUp.kind === 'contextual'
1883
+ ? 'routing rule: this is prior-turn context (advisory). The user may be continuing from it — resolve references like "these"/"those"/"same" against it when the question refers to it. If this question starts a genuinely new topic, ignore the prior-turn context entirely and answer fresh.'
1884
+ : 'routing rule: reuse the prior certified block when the user asks a generic follow-up.';
1099
1885
  return [...parts, rule].join('\n');
1100
1886
  }
1887
+ function renderPriorResultReference(ref) {
1888
+ const lines = [
1889
+ `prior result ref: result:${ref.id}`,
1890
+ ref.question ? `prior result question: ${ref.question.slice(0, 220)}` : '',
1891
+ ref.columns.length ? `prior result schema: ${ref.columns.slice(0, 24).join(', ')}` : '',
1892
+ typeof ref.rowCount === 'number' ? `prior result row count: ${ref.rowCount}` : '',
1893
+ ref.sourceSql ? `prior result source SQL: ${compactSqlSnippet(ref.sourceSql, 500)}` : '',
1894
+ ].filter(Boolean);
1895
+ return lines.join('\n');
1896
+ }
1897
+ function renderPriorDqlArtifactReference(artifact) {
1898
+ const lines = [
1899
+ `prior DQL artifact kind: ${artifact.kind}`,
1900
+ artifact.name ? `prior DQL artifact name: ${artifact.name}` : '',
1901
+ artifact.sourcePath ? `prior DQL artifact path: ${artifact.sourcePath}` : '',
1902
+ artifact.metrics?.length ? `prior DQL metrics: ${artifact.metrics.slice(0, 12).join(', ')}` : '',
1903
+ artifact.dimensions?.length ? `prior DQL dimensions: ${artifact.dimensions.slice(0, 12).join(', ')}` : '',
1904
+ artifact.filters?.length ? `prior DQL filters: ${artifact.filters.slice(0, 8).map(formatDqlArtifactFilter).join('; ')}` : '',
1905
+ artifact.timeDimension ? `prior DQL time: ${artifact.timeDimension.name} / ${artifact.timeDimension.granularity}` : '',
1906
+ artifact.source.trim() ? `prior DQL artifact source:\n${compactSqlSnippet(artifact.source, 1400)}` : '',
1907
+ ].filter(Boolean);
1908
+ return lines.join('\n');
1909
+ }
1910
+ function formatDqlArtifactFilter(filter) {
1911
+ return `${filter.dimension} ${filter.operator} ${filter.values.slice(0, 8).join(', ')}`;
1912
+ }
1913
+ function formatPriorResultValues(values) {
1914
+ return Object.entries(values)
1915
+ .slice(0, 8)
1916
+ .map(([key, vals]) => `${key}=[${vals.slice(0, 12).join(', ')}]`)
1917
+ .join('; ');
1918
+ }
1101
1919
  function generatedTrustExplanation(input) {
1102
1920
  if (input.followUp?.kind !== 'drilldown')
1103
1921
  return undefined;
@@ -1110,295 +1928,232 @@ function generatedTrustExplanation(input) {
1110
1928
  ];
1111
1929
  const grain = filters.length ? ` at the requested ${filters.join('/')} grain` : ' at the requested drilldown grain';
1112
1930
  const draft = input.draftBlock
1113
- ? ` The draft was saved at \`${input.draftBlock.path}\` for review.`
1114
- : ' The generated SQL still needs analyst review before certification.';
1115
- return `This is an uncertified drilldown.${source} then generated new SQL${grain}.${draft}`;
1931
+ ? ` The DQL draft was saved at \`${input.draftBlock.path}\` for review.`
1932
+ : ' The review-required DQL artifact and SQL preview still need analyst review before certification.';
1933
+ return `This is an uncertified drilldown.${source} then prepared a SQL preview${grain}.${draft}`;
1116
1934
  }
1117
1935
  function cleanGeneratedSummary(text) {
1118
1936
  return text
1119
1937
  .trim()
1120
1938
  .replace(/^(?:the user (?:is asking|asked|wants|requested)[^.]*\.\s*)+/i, '')
1121
1939
  .replace(/\s*(?:therefore,\s*)?i will generate review-required sql[^.]*\.\s*/gi, ' ')
1940
+ .replace(/\s*(?:therefore,\s*)?i will prepare (?:a )?review-required dql artifact[^.]*\.\s*/gi, ' ')
1122
1941
  .replace(/\s*(?:therefore,\s*)?i will generate[^.]*\.\s*/gi, ' ')
1123
1942
  .trim();
1124
1943
  }
1125
1944
  /**
1126
- * Public for tests. Pulls the first ```sql block and an optional Viz: line
1127
- * out of an LLM response.
1945
+ * Public for tests. Prefer the structured W2.7 JSON proposal contract, then
1946
+ * fall back to the legacy prose + ```sql block + Viz line format.
1128
1947
  */
1129
1948
  export function parseProposal(raw) {
1949
+ const structured = parseStructuredProposal(raw);
1950
+ if (structured)
1951
+ return structured;
1130
1952
  const sqlMatch = raw.match(/```sql\s*([\s\S]*?)```/i);
1131
1953
  const sql = sqlMatch ? sqlMatch[1].trim() : undefined;
1132
1954
  const vizMatch = raw.match(/^Viz:\s*([a-z_]+)/im);
1133
1955
  const viz = vizMatch ? vizMatch[1].trim().toLowerCase() : undefined;
1134
1956
  // Strip the SQL block + Viz line from the prose to keep the summary clean.
1135
1957
  const text = raw
1958
+ .replace(/```json[\s\S]*?```/gi, '')
1136
1959
  .replace(/```sql[\s\S]*?```/gi, '')
1137
1960
  .replace(/^Viz:.*$/gim, '')
1138
1961
  .trim();
1139
1962
  return { text, sql, viz };
1140
1963
  }
1141
- function buildSchemaAwareProposal(input) {
1142
- if (!isGeneratedAgentIntent(input.intent))
1143
- return undefined;
1144
- const schemaContext = schemaContextWithAllowedSqlContext(input.schemaContext, input.contextPack);
1145
- const drilldownProposal = buildMatchedEntityDrilldownProposal({ ...input, schemaContext });
1146
- if (drilldownProposal)
1147
- return drilldownProposal;
1148
- const profileProposal = buildEntityProfileProposal({ ...input, schemaContext });
1149
- if (profileProposal)
1150
- return profileProposal;
1151
- if (isFilteredEntityQuestion(input.question))
1152
- return undefined;
1153
- const lower = input.question.toLowerCase();
1154
- const asksForCustomerPerformance = /\bcustomers?\b/.test(lower)
1155
- && /\border|orders|spend|revenue|perform|performed|better|top|best|rank|ranking|bottom|least|fewest|lowest|less|worst|underperform/.test(lower)
1156
- && !/\b(order details|specific orders|each order|all orders|order line|line item)\b/.test(lower);
1157
- if (asksForCustomerPerformance) {
1158
- const direction = customerRankingDirectionFromText(input.question);
1159
- const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
1160
- const orderFocused = /\border|orders|order count|ordered\b/.test(lower)
1161
- && !/\b(spend|revenue|sales|amount|lifetime spend|value)\b/.test(lower);
1162
- const customers = findSchemaTable(schemaContext, ['customers', 'customer']);
1163
- if (customers) {
1164
- const customerName = findSchemaColumn(customers, ['customer_name', 'name', 'full_name']);
1165
- const orderCount = findSchemaColumn(customers, ['count_lifetime_orders', 'lifetime_orders', 'order_count', 'orders_count', 'orders']);
1166
- const spend = findSchemaColumn(customers, ['lifetime_spend', 'total_lifetime_spend', 'customer_lifetime_value', 'total_revenue', 'revenue']);
1167
- if (customerName && orderCount && spend) {
1168
- const primarySort = orderFocused ? orderCount : spend;
1169
- const secondarySort = orderFocused ? spend : orderCount;
1170
- const rankingLabel = direction === 'bottom'
1171
- ? orderFocused ? 'Customers with the fewest orders' : 'Lowest performing customers'
1172
- : orderFocused ? 'Top customers by order count' : 'Top performing customers';
1173
- return {
1174
- text: `${rankingLabel}, with ${businessMeasurePhrase(orderCount)} and ${businessMeasurePhrase(spend)} for context. This is AI-generated and needs analyst review before certification.`,
1175
- sql: [
1176
- 'SELECT',
1177
- ` ${sqlIdentifier(customerName)} AS customer_name,`,
1178
- ` ${sqlIdentifier(orderCount)} AS orders,`,
1179
- ` ROUND(${sqlIdentifier(spend)}, 2) AS lifetime_spend`,
1180
- `FROM ${sqlRelation(customers.relation)}`,
1181
- `ORDER BY ${sqlIdentifier(primarySort)} ${orderDirection}, ${sqlIdentifier(secondarySort)} ${orderDirection}`,
1182
- 'LIMIT 10',
1183
- ].join('\n'),
1184
- viz: 'table',
1185
- };
1186
- }
1187
- const customerId = findSchemaColumn(customers, ['customer_id', 'id']);
1188
- const orders = findSchemaTable(schemaContext, ['orders', 'order']);
1189
- const orderCustomerId = orders ? findSchemaColumn(orders, ['customer_id', 'customer']) : undefined;
1190
- const orderTotal = orders ? findSchemaColumn(orders, ['order_total', 'total_order_amount', 'total_amount', 'amount', 'subtotal']) : undefined;
1191
- const orderId = orders ? findSchemaColumn(orders, ['order_id', 'id']) : undefined;
1192
- if (orders && customerName && customerId && orderCustomerId && orderTotal) {
1193
- const countExpression = orderId ? `COUNT(DISTINCT o.${sqlIdentifier(orderId)})` : 'COUNT(*)';
1194
- const primarySort = orderFocused ? 'orders' : 'lifetime_spend';
1195
- const secondarySort = orderFocused ? 'lifetime_spend' : 'orders';
1196
- const rankingLabel = direction === 'bottom'
1197
- ? orderFocused ? 'Customers with the fewest orders' : 'Lowest performing customers'
1198
- : orderFocused ? 'Top customers by order count' : 'Top performing customers';
1199
- return {
1200
- text: `${rankingLabel} from order totals, with order count for context. This is AI-generated and needs analyst review before certification.`,
1201
- sql: [
1202
- 'SELECT',
1203
- ` c.${sqlIdentifier(customerName)} AS customer_name,`,
1204
- ` ${countExpression} AS orders,`,
1205
- ` ROUND(SUM(o.${sqlIdentifier(orderTotal)}), 2) AS lifetime_spend`,
1206
- `FROM ${sqlRelation(orders.relation)} AS o`,
1207
- `JOIN ${sqlRelation(customers.relation)} AS c ON o.${sqlIdentifier(orderCustomerId)} = c.${sqlIdentifier(customerId)}`,
1208
- `GROUP BY c.${sqlIdentifier(customerName)}`,
1209
- `ORDER BY ${sqlIdentifier(primarySort)} ${orderDirection}, ${sqlIdentifier(secondarySort)} ${orderDirection}`,
1210
- 'LIMIT 10',
1211
- ].join('\n'),
1212
- viz: 'table',
1213
- };
1214
- }
1964
+ function parseStructuredProposal(raw) {
1965
+ const trimmed = raw.trim();
1966
+ const fenced = raw.match(/```json\s*([\s\S]*?)```/i)?.[1]?.trim();
1967
+ const candidates = [
1968
+ fenced,
1969
+ trimmed.startsWith('{') && trimmed.endsWith('}') ? trimmed : undefined,
1970
+ ].filter((candidate) => Boolean(candidate));
1971
+ for (const candidate of candidates) {
1972
+ try {
1973
+ const parsed = JSON.parse(candidate);
1974
+ const proposal = parsedProposalFromJson(parsed);
1975
+ if (proposal)
1976
+ return proposal;
1977
+ }
1978
+ catch {
1979
+ // Invalid structured output falls through to the legacy SQL-fence parser.
1215
1980
  }
1216
1981
  }
1217
- const genericJoinProposal = buildGenericJoinProposal({ ...input, schemaContext });
1218
- if (genericJoinProposal)
1219
- return genericJoinProposal;
1220
- const genericProposal = buildGenericSingleTableProposal({ ...input, schemaContext });
1221
- if (genericProposal)
1222
- return genericProposal;
1223
1982
  return undefined;
1224
1983
  }
1225
- function customerRankingDirectionFromText(text) {
1226
- const lower = text.toLowerCase();
1227
- if (/\b(less|lesser)\s+(?:order|orders|ordering)\b/.test(lower) || /\borders?\s+(?:less|least|fewest|lowest)\b/.test(lower)) {
1228
- return 'bottom';
1229
- }
1230
- return rankingDirectionFromText(text) ?? 'top';
1231
- }
1232
- function buildGenericJoinProposal(input) {
1233
- const planMode = input.contextPack?.questionPlan?.mode;
1234
- if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
1984
+ function parsedProposalFromJson(value) {
1985
+ if (!value || typeof value !== 'object' || Array.isArray(value))
1235
1986
  return undefined;
1236
- }
1237
- const lower = input.question.toLowerCase();
1238
- if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
1239
- return undefined;
1240
- }
1241
- if (!/\b(by\s+[a-z]|per\s+[a-z]|for each|group(?:ed)? by|split|segment|break\s*down|breakdown|compare)\b/i.test(lower)) {
1987
+ const record = value;
1988
+ const sql = firstJsonString(record.sql, record.query)?.trim();
1989
+ const text = firstJsonString(record.summary, record.text, record.answer, record.description)?.trim() ?? '';
1990
+ const viz = firstJsonString(record.viz, record.visualization, record.chartType)
1991
+ ?.trim()
1992
+ .toLowerCase()
1993
+ .replace(/[^a-z_]/g, '');
1994
+ const dql = jsonRecord(record.dql);
1995
+ const outputs = firstJsonStringList(record.outputs, dql?.outputs);
1996
+ const proposedEntity = firstJsonString(dql?.entity, dql?.proposedEntity, dql?.proposed_entity, record.proposedEntity, record.proposed_entity, record.entity)?.replace(/\s+/g, ' ').trim().slice(0, 160);
1997
+ const requestedFilters = firstJsonStringList(dql?.filters, dql?.requestedFilters, dql?.requested_filters, record.requestedFilters, record.requested_filters, record.filters);
1998
+ const requestedDimensions = firstJsonStringList(dql?.dimensions, dql?.requestedDimensions, dql?.requested_dimensions, record.requestedDimensions, record.requested_dimensions, record.dimensions);
1999
+ if (!sql && !text)
1242
2000
  return undefined;
1243
- }
1244
- const candidate = pickGenericJoinCandidate(input.schemaContext, input.question);
1245
- if (!candidate)
1246
- return undefined;
1247
- const direction = rankingDirectionFromText(input.question);
1248
- const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
1249
- const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
1250
- const chart = isTimeLikeGenericColumn(candidate.dimensionColumn) ? 'line' : 'bar';
1251
- const metricExpression = qualifiedMetricExpression(candidate.metric, 'f');
1252
- const metricAlias = candidate.metric.alias;
1253
- const dimensionAlias = candidate.dimensionColumn;
1254
2001
  return {
1255
- text: `Prepared a review-required ${businessMeasurePhrase(candidate.metric.column)} breakdown by ${humanizeIdentifier(candidate.dimensionColumn)} by joining ${candidate.metricTable.relation} to ${candidate.dimensionTable.relation}. This result is uncertified until reviewed and promoted.`,
1256
- sql: [
1257
- 'SELECT',
1258
- ` d.${sqlIdentifier(candidate.dimensionColumn)} AS ${sqlIdentifier(dimensionAlias)},`,
1259
- ` ${metricExpression} AS ${sqlIdentifier(metricAlias)}`,
1260
- `FROM ${sqlRelation(candidate.metricTable.relation)} AS f`,
1261
- `JOIN ${sqlRelation(candidate.dimensionTable.relation)} AS d ON f.${sqlIdentifier(candidate.metricJoinColumn)} = d.${sqlIdentifier(candidate.dimensionJoinColumn)}`,
1262
- `GROUP BY d.${sqlIdentifier(candidate.dimensionColumn)}`,
1263
- `ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
1264
- `LIMIT ${limit}`,
1265
- ].join('\n'),
1266
- viz: chart,
2002
+ text,
2003
+ ...(sql ? { sql } : {}),
2004
+ ...(viz ? { viz } : {}),
2005
+ ...(outputs?.length ? { outputs } : {}),
2006
+ ...(proposedEntity ? { proposedEntity } : {}),
2007
+ ...(requestedFilters?.length ? { requestedFilters } : {}),
2008
+ ...(requestedDimensions?.length ? { requestedDimensions } : {}),
1267
2009
  };
1268
2010
  }
1269
- function buildGenericSingleTableProposal(input) {
1270
- const planMode = input.contextPack?.questionPlan?.mode;
1271
- if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
1272
- return undefined;
1273
- }
1274
- const lower = input.question.toLowerCase();
1275
- if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
1276
- return undefined;
2011
+ function firstJsonString(...values) {
2012
+ return values.find((value) => typeof value === 'string' && value.trim().length > 0);
2013
+ }
2014
+ function jsonRecord(value) {
2015
+ return value && typeof value === 'object' && !Array.isArray(value)
2016
+ ? value
2017
+ : undefined;
2018
+ }
2019
+ function firstJsonStringList(...values) {
2020
+ for (const value of values) {
2021
+ const list = normalizeProposalStringList(value);
2022
+ if (list.length > 0)
2023
+ return list;
1277
2024
  }
1278
- if (!/\b(show|list|top|bottom|best|worst|highest|lowest|least|fewest|rank|ranking|compare|trend|by\s+[a-z]|how many|number of|revenue|sales|orders?|points?|score|count|total|average|avg|sum)\b/i.test(lower)) {
1279
- return undefined;
2025
+ return undefined;
2026
+ }
2027
+ function normalizeProposalStringList(value) {
2028
+ const rawValues = Array.isArray(value)
2029
+ ? value
2030
+ : typeof value === 'string'
2031
+ ? [value]
2032
+ : [];
2033
+ const seen = new Set();
2034
+ const out = [];
2035
+ for (const raw of rawValues) {
2036
+ if (typeof raw !== 'string')
2037
+ continue;
2038
+ const cleaned = raw.replace(/\s+/g, ' ').trim().slice(0, 160);
2039
+ if (!cleaned)
2040
+ continue;
2041
+ const key = cleaned.toLowerCase();
2042
+ if (seen.has(key))
2043
+ continue;
2044
+ seen.add(key);
2045
+ out.push(cleaned);
2046
+ if (out.length >= 20)
2047
+ break;
1280
2048
  }
1281
- const table = pickGenericAnalysisTable(input.schemaContext, input.question);
1282
- if (!table)
1283
- return undefined;
1284
- const metric = inferGenericMetric(table, input.question);
1285
- if (!metric)
2049
+ return out;
2050
+ }
2051
+ function mergeProposalStringLists(...lists) {
2052
+ const merged = normalizeProposalStringList(lists.flatMap((list) => list ?? []));
2053
+ return merged.length > 0 ? merged : undefined;
2054
+ }
2055
+ function applyParsedProposalMetadata(target, source) {
2056
+ target.outputs = source.outputs ?? target.outputs;
2057
+ target.proposedEntity = source.proposedEntity ?? target.proposedEntity;
2058
+ target.requestedFilters = source.requestedFilters ?? target.requestedFilters;
2059
+ target.requestedDimensions = source.requestedDimensions ?? target.requestedDimensions;
2060
+ }
2061
+ function buildCertifiedBlockDqlArtifact(block, result) {
2062
+ if (block.kind !== 'block')
1286
2063
  return undefined;
1287
- const dimensions = augmentGenericRankingDimensions(table, inferGenericDimensions(table, input.question, metric.column), metric, input.question).slice(0, 2);
1288
- const direction = rankingDirectionFromText(input.question);
1289
- const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
1290
- const metricAlias = metric.alias;
1291
- if (dimensions.length === 0) {
1292
- return {
1293
- text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} summary from ${table.relation}. This result is uncertified until reviewed and promoted.`,
1294
- sql: [
1295
- 'SELECT',
1296
- ` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
1297
- `FROM ${sqlRelation(table.relation)}`,
1298
- ].join('\n'),
1299
- viz: 'single_value',
1300
- };
1301
- }
1302
- if (direction && metric.preAggregated) {
2064
+ const sql = block.sql?.trim() ?? result?.sql?.trim() ?? block.examples?.find((example) => example.sql?.trim())?.sql?.trim();
2065
+ if (!sql) {
2066
+ // The block answered but its SQL was not inlined in the index (navigation /
2067
+ // reference artifacts). Still hand back a reference artifact pointing at the
2068
+ // governed source file so every certified-block answer carries a DQL artifact
2069
+ // the UI can open or save, rather than dropping it silently.
2070
+ if (!block.name)
2071
+ return undefined;
1303
2072
  return {
1304
- text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} ranking by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
1305
- sql: [
1306
- 'SELECT',
1307
- ...dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`),
1308
- ` ${sqlIdentifier(metric.column)} AS ${sqlIdentifier(metric.column)}`,
1309
- `FROM ${sqlRelation(table.relation)}`,
1310
- `ORDER BY ${sqlIdentifier(metric.column)} ${direction === 'bottom' ? 'ASC' : 'DESC'}`,
1311
- `LIMIT ${limit}`,
1312
- ].join('\n'),
1313
- viz: 'table',
2073
+ kind: 'certified_block',
2074
+ name: block.name,
2075
+ sourcePath: block.sourcePath,
2076
+ source: `// Certified DQL block "${escapeDqlArtifactString(block.name)}"`
2077
+ + `${block.sourcePath ? `\n// source: ${block.sourcePath}` : ''}`
2078
+ + `\n// SQL is not inlined in the index — open the source file to view or edit the query.`,
1314
2079
  };
1315
2080
  }
1316
- const selectDimensions = dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`);
1317
- const groupBy = dimensions.map(sqlIdentifier).join(', ');
1318
- const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
1319
- const chart = dimensions.some(isTimeLikeGenericColumn) ? 'line' : 'bar';
2081
+ const domain = block.domain ?? 'misc';
2082
+ const description = block.description ?? `Certified DQL block ${block.name}`;
2083
+ const sourcePathComment = block.sourcePath ? `\n // source: ${block.sourcePath}` : '';
2084
+ const source = `block "${escapeDqlArtifactString(block.name)}" {
2085
+ domain = "${escapeDqlArtifactString(domain)}"
2086
+ type = "custom"
2087
+ status = "certified"${block.owner ? `\n owner = "${escapeDqlArtifactString(block.owner)}"` : ''}
2088
+ description = """${escapeDqlArtifactTripleString(description)}"""${sourcePathComment}
2089
+
2090
+ query = """
2091
+ ${sql.replace(/"""/g, '\\"\\"\\"').split('\n').join('\n ')}
2092
+ """
2093
+ }
2094
+ `;
1320
2095
  return {
1321
- text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} breakdown by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
1322
- sql: [
1323
- 'SELECT',
1324
- ...selectDimensions,
1325
- ` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
1326
- `FROM ${sqlRelation(table.relation)}`,
1327
- `GROUP BY ${groupBy}`,
1328
- `ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
1329
- `LIMIT ${limit}`,
1330
- ].join('\n'),
1331
- viz: chart,
2096
+ kind: 'certified_block',
2097
+ name: block.name,
2098
+ sourcePath: block.sourcePath,
2099
+ source,
1332
2100
  };
1333
2101
  }
1334
- function pickGenericAnalysisTable(schemaContext, question) {
1335
- return schemaContext
1336
- .filter((table) => table.columns.length > 0)
1337
- .map((table, index) => ({
1338
- table,
1339
- metric: inferGenericMetric(table, question),
1340
- dimensions: inferGenericDimensions(table, question),
1341
- index,
1342
- }))
1343
- .filter((candidate) => candidate.metric)
1344
- .sort((a, b) => genericTableScore(b.table, question, b.metric, b.dimensions) -
1345
- genericTableScore(a.table, question, a.metric, a.dimensions) ||
1346
- a.index - b.index)[0]?.table;
1347
- }
1348
- function genericTableScore(table, question, metric, dimensions) {
1349
- const questionTokens = meaningfulTokens(question);
1350
- const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
1351
- const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
1352
- const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
1353
- const selectedRelationBoost = table.selectionRank
1354
- ? Math.max(0, 56 - table.selectionRank * 3) + Math.min(table.selectionScore ?? 0, 24)
1355
- : 0;
1356
- return selectedRelationBoost + overlap * 8 + (metric?.score ?? 0) + dimensions.length * 8 + Math.min(table.columns.length, 20) * 0.3;
1357
- }
1358
- function pickGenericJoinCandidate(schemaContext, question) {
1359
- const requestedDimensions = requestedDimensionPhrases(question);
1360
- if (requestedDimensions.length === 0)
2102
+ function buildGeneratedSqlDqlArtifact(input) {
2103
+ const sql = input.sql?.trim();
2104
+ if (!sql)
1361
2105
  return undefined;
1362
- const metricCandidates = schemaContext
1363
- .filter((table) => table.columns.length > 0)
1364
- .map((table, index) => ({
1365
- table,
1366
- metric: inferGenericMetric(table, question),
1367
- localDimensions: inferGenericDimensions(table, question),
1368
- index,
1369
- }))
1370
- .filter((candidate) => Boolean(candidate.metric))
1371
- .filter((candidate) => !candidate.localDimensions.some((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension)));
1372
- const candidates = [];
1373
- for (const metricCandidate of metricCandidates) {
1374
- for (const dimensionTable of schemaContext) {
1375
- if (normalizeRelationKey(dimensionTable.relation) === normalizeRelationKey(metricCandidate.table.relation))
1376
- continue;
1377
- const dimensionColumn = pickRequestedDimensionColumn(dimensionTable, question, requestedDimensions);
1378
- if (!dimensionColumn || namesEqualLoose(dimensionColumn, metricCandidate.metric.column))
1379
- continue;
1380
- const join = pickJoinColumns(metricCandidate.table, dimensionTable);
1381
- if (!join)
1382
- continue;
1383
- const dimensionScore = genericDimensionScore({ name: dimensionColumn, type: findColumnType(dimensionTable, dimensionColumn) }, question, Math.max(0, dimensionTable.columns.findIndex((column) => namesEqualLoose(column.name, dimensionColumn))), metricCandidate.metric.column);
1384
- candidates.push({
1385
- metricTable: metricCandidate.table,
1386
- dimensionTable,
1387
- metric: metricCandidate.metric,
1388
- dimensionColumn,
1389
- metricJoinColumn: join.leftColumn,
1390
- dimensionJoinColumn: join.rightColumn,
1391
- score: genericTableScore(metricCandidate.table, question, metricCandidate.metric, metricCandidate.localDimensions) +
1392
- dimensionScore +
1393
- join.score +
1394
- selectedRelationWeight(dimensionTable),
1395
- });
1396
- }
1397
- }
1398
- return candidates
1399
- .sort((a, b) => b.score - a.score ||
1400
- (a.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
1401
- a.metricTable.relation.localeCompare(b.metricTable.relation))[0];
2106
+ const slug = deriveGeneratedDraftSlug(input.question);
2107
+ const proposedDomain = input.sourceBlock?.domain
2108
+ ?? input.contextPack?.objects.find((object) => object.domain)?.domain
2109
+ ?? input.domain
2110
+ ?? 'misc';
2111
+ return {
2112
+ kind: 'sql_block',
2113
+ name: slug,
2114
+ source: renderGeneratedSqlDqlArtifact({
2115
+ slug,
2116
+ question: input.question,
2117
+ proposedSql: sql,
2118
+ proposedContractId: `${proposedDomain}.Unknown.${slug}`,
2119
+ proposedDomain,
2120
+ sourceQuestion: input.followUp?.sourceQuestion,
2121
+ sourceBlock: input.followUp?.sourceBlockName ?? input.sourceBlock?.name,
2122
+ sourceDqlArtifact: input.sourceDqlArtifact,
2123
+ followupKind: input.followUp?.kind,
2124
+ proposedEntity: input.proposedEntity,
2125
+ requestedFilters: input.requestedFilters,
2126
+ requestedDimensions: input.requestedDimensions,
2127
+ outputs: input.outputs,
2128
+ contextPackId: input.contextPack?.id,
2129
+ routeIntent: String(input.intent),
2130
+ validationWarnings: input.validationWarnings,
2131
+ }),
2132
+ };
2133
+ }
2134
+ function escapeDqlArtifactString(value) {
2135
+ return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
2136
+ }
2137
+ function escapeDqlArtifactTripleString(value) {
2138
+ return value.replace(/"""/g, '\\"\\"\\"');
2139
+ }
2140
+ /**
2141
+ * Generic partial-shape signal: the SQL executed but dropped MULTIPLE explicitly
2142
+ * requested output columns. Domain-agnostic — this downgrades trust and attaches a
2143
+ * "partial answer, missing <cols>" warning (and biases deep-candidate selection
2144
+ * toward the complete-shape candidate) for ANY project, not just product/customer/
2145
+ * revenue. It never refuses; the executed rows are always returned.
2146
+ */
2147
+ function generatedResultShapeIsPartial(resultShape) {
2148
+ return resultShape.missingOutputs.length >= 2;
2149
+ }
2150
+ function trustedFollowUpFilterValues(followUp) {
2151
+ const filters = new Set((followUp?.filters ?? []).map((value) => value.trim()).filter(Boolean));
2152
+ if (!filters.size || !followUp?.priorResultValues)
2153
+ return undefined;
2154
+ const priorValues = Object.values(followUp.priorResultValues).flat().map((value) => value.trim()).filter(Boolean);
2155
+ const trusted = priorValues.filter((value) => filters.has(value));
2156
+ return trusted.length > 0 ? uniqueDrilldownStrings(trusted) : undefined;
1402
2157
  }
1403
2158
  function buildCandidateJoinPaths(schemaContext) {
1404
2159
  const tables = [...schemaContext]
@@ -1446,542 +2201,63 @@ function joinReason(leftColumn, rightColumn, score) {
1446
2201
  return 'foreign-key style id match';
1447
2202
  return 'join-key style column match';
1448
2203
  }
1449
- function requestedDimensionPhrases(question) {
1450
- const lower = question.toLowerCase();
1451
- const phrases = [
1452
- ...Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
1453
- ...Array.from(lower.matchAll(/\bper\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
1454
- ...Array.from(lower.matchAll(/\bfor each\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
1455
- ];
1456
- return uniqueDrilldownStrings(phrases
1457
- .flatMap((phrase) => phrase.split(/\band\b|,|\//i))
1458
- .map((phrase) => phrase
1459
- .replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time|where|for|with|from|in|during|last|this|next)\b.*$/gi, '')
1460
- .replace(/[^a-z0-9_ -]+/gi, ' ')
1461
- .replace(/\s+/g, ' ')
1462
- .trim())
1463
- .filter((phrase) => phrase.length > 0));
1464
- }
1465
- function pickRequestedDimensionColumn(table, question, requestedDimensions) {
1466
- for (const phrase of requestedDimensions) {
1467
- const direct = findSchemaColumn(table, dimensionColumnCandidatesForPhrase(phrase));
1468
- if (direct && !isJoinKeyColumn(direct) && !isNumericLikeColumn({ name: direct, type: findColumnType(table, direct) })) {
1469
- return direct;
1470
- }
1471
- }
1472
- const inferred = inferGenericDimensions(table, question)
1473
- .find((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension));
1474
- return inferred;
1475
- }
1476
- function dimensionColumnCandidatesForPhrase(phrase) {
1477
- const tokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
1478
- .flatMap((token) => token.split('_'))
1479
- .map(normalizeToken)
1480
- .filter((token) => token.length > 0);
1481
- if (tokens.length === 0)
1482
- return [];
1483
- const joined = tokens.join('_');
1484
- const tail = tokens.slice(1).join('_');
1485
- return uniqueDrilldownStrings([
1486
- joined,
1487
- phrase.replace(/\s+/g, '_').toLowerCase(),
1488
- tail,
1489
- tokens.at(-1) ?? '',
1490
- ...tokens,
1491
- ].filter(Boolean));
1492
- }
1493
- function requestedDimensionMatchesColumn(requestedDimensions, column) {
1494
- const columnTokens = exactMatchTokens(column);
1495
- return requestedDimensions.some((phrase) => {
1496
- const phraseTokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
1497
- .flatMap((token) => token.split('_'))
1498
- .map(normalizeToken)
1499
- .filter((token) => token.length > 0);
1500
- return phraseTokens.some((token) => columnTokens.has(token));
1501
- });
1502
- }
1503
- function selectedRelationWeight(table) {
1504
- return table.selectionRank
1505
- ? Math.max(0, 30 - table.selectionRank * 2) + Math.min(table.selectionScore ?? 0, 16)
1506
- : 0;
1507
- }
1508
2204
  function pickJoinColumns(left, right) {
1509
2205
  const candidates = [];
1510
2206
  const leftColumns = left.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
1511
2207
  const rightColumns = right.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
1512
2208
  for (const leftColumn of leftColumns) {
1513
2209
  for (const rightColumn of rightColumns) {
1514
- const score = joinColumnScore(leftColumn.name, rightColumn.name, left, right);
1515
- if (score <= 0)
1516
- continue;
1517
- candidates.push({ leftColumn: leftColumn.name, rightColumn: rightColumn.name, score });
1518
- }
1519
- }
1520
- return candidates.sort((a, b) => b.score - a.score || a.leftColumn.localeCompare(b.leftColumn))[0];
1521
- }
1522
- function joinColumnScore(leftColumn, rightColumn, leftTable, rightTable) {
1523
- const leftKey = normalizeRelationKey(leftColumn).replace(/\./g, '_');
1524
- const rightKey = normalizeRelationKey(rightColumn).replace(/\./g, '_');
1525
- const leftJoinLike = isJoinKeyColumn(leftColumn);
1526
- const rightJoinLike = isJoinKeyColumn(rightColumn);
1527
- if (!leftJoinLike || !rightJoinLike)
1528
- return 0;
1529
- if (leftKey === rightKey)
1530
- return 70;
1531
- const leftSubject = joinSubjectForColumn(leftColumn);
1532
- const rightSubject = joinSubjectForColumn(rightColumn);
1533
- if (leftSubject && rightSubject && leftSubject === rightSubject)
1534
- return 62;
1535
- const leftTableTokens = tableEntityTokens(leftTable);
1536
- const rightTableTokens = tableEntityTokens(rightTable);
1537
- if (leftSubject && rightKey === 'id' && rightTableTokens.has(leftSubject))
1538
- return 58;
1539
- if (rightSubject && leftKey === 'id' && leftTableTokens.has(rightSubject))
1540
- return 58;
1541
- if (leftSubject && rightTableTokens.has(leftSubject) && rightKey.endsWith('_key'))
1542
- return 42;
1543
- if (rightSubject && leftTableTokens.has(rightSubject) && leftKey.endsWith('_key'))
1544
- return 42;
1545
- return 0;
1546
- }
1547
- function isJoinKeyColumn(column) {
1548
- const normalized = column.toLowerCase();
1549
- return normalized === 'id' ||
1550
- /(^|_)(id|key|uuid|sk)$/.test(normalized) ||
1551
- /_(id|key|uuid|sk)$/.test(normalized);
1552
- }
1553
- function joinSubjectForColumn(column) {
1554
- const normalized = column.toLowerCase();
1555
- const subject = normalized.replace(/_(id|key|uuid|sk)$/i, '');
1556
- if (!subject || subject === normalized || subject === 'id' || subject === 'key')
1557
- return undefined;
1558
- return normalizeToken(subject.split('_').at(-1) ?? subject);
1559
- }
1560
- function tableEntityTokens(table) {
1561
- const tokens = exactMatchTokens([table.name, table.relation.split('.').at(-1) ?? table.relation].join(' '));
1562
- for (const generic of ['dim', 'fct', 'fact', 'stg', 'stage', 'model', 'table'])
1563
- tokens.delete(generic);
1564
- return tokens;
1565
- }
1566
- function findColumnType(table, columnName) {
1567
- return table.columns.find((column) => namesEqualLoose(column.name, columnName))?.type;
1568
- }
1569
- function qualifiedMetricExpression(metric, tableAlias) {
1570
- const column = `${tableAlias}.${sqlIdentifier(metric.column)}`;
1571
- if (metric.column === 'rows')
1572
- return 'COUNT(*)';
1573
- if (/^COUNT\s*\(\s*DISTINCT\b/i.test(metric.expression))
1574
- return `COUNT(DISTINCT ${column})`;
1575
- if (metric.preAggregated)
1576
- return column;
1577
- const aggregate = metric.expression.match(/^(SUM|AVG|MIN|MAX|MEDIAN)\s*\(/i)?.[1]?.toUpperCase()
1578
- ?? (/avg|average|mean/i.test(metric.alias) ? 'AVG' : 'SUM');
1579
- return `${aggregate}(${column})`;
1580
- }
1581
- function inferGenericMetric(table, question) {
1582
- const lower = question.toLowerCase();
1583
- const wantsCount = /\b(count|how many|number of|volume)\b/i.test(lower);
1584
- const candidates = table.columns
1585
- .map((column) => {
1586
- const name = column.name.toLowerCase();
1587
- const tokens = meaningfulTokens(column.name);
1588
- let score = isNumericLikeColumn(column) ? 20 : 0;
1589
- for (const token of meaningfulTokens(question)) {
1590
- if (tokens.has(token))
1591
- score += 18;
1592
- }
1593
- if (/\b(revenue|sales|amount|total|spend|cost|margin|profit|value|points?|score|orders?|count|quantity|duration|minutes?|rate|average|avg)\b/i.test(name.replace(/_/g, ' '))) {
1594
- score += 18;
1595
- }
1596
- if (/\b(id|key|code|zip|postal|phone|season|year|month|week|day|date|time)\b/i.test(name.replace(/_/g, ' '))) {
1597
- score -= 14;
1598
- }
1599
- return { column: column.name, score };
1600
- })
1601
- .filter((candidate) => candidate.score > 0)
1602
- .sort((a, b) => b.score - a.score);
1603
- const countSubject = wantsCount ? inferCountSubject(question) : undefined;
1604
- if (wantsCount) {
1605
- const precomputed = candidates.find((candidate) => isPrecomputedCountMetric(candidate.column, countSubject));
1606
- if (precomputed) {
1607
- return {
1608
- column: precomputed.column,
1609
- expression: isPreAggregatedMetricColumn(precomputed.column) ? sqlIdentifier(precomputed.column) : `SUM(${sqlIdentifier(precomputed.column)})`,
1610
- alias: isPreAggregatedMetricColumn(precomputed.column) ? precomputed.column : `${precomputed.column}_sum`,
1611
- score: precomputed.score + 8,
1612
- preAggregated: isPreAggregatedMetricColumn(precomputed.column),
1613
- };
1614
- }
1615
- const distinctColumn = countSubject ? findDistinctCountColumn(table, countSubject) : undefined;
1616
- if (distinctColumn) {
1617
- return {
1618
- column: distinctColumn,
1619
- expression: `COUNT(DISTINCT ${sqlIdentifier(distinctColumn)})`,
1620
- alias: `${countSubject}_count`,
1621
- score: 34,
1622
- preAggregated: false,
1623
- };
1624
- }
1625
- }
1626
- const selected = candidates[0];
1627
- if (!selected && !wantsCount)
1628
- return undefined;
1629
- if (!selected || wantsCount && selected.score < 26) {
1630
- return {
1631
- column: 'rows',
1632
- expression: 'COUNT(*)',
1633
- alias: 'row_count',
1634
- score: 18,
1635
- preAggregated: true,
1636
- };
1637
- }
1638
- const aggregate = /\b(avg|average|mean)\b/i.test(lower) ? 'AVG' : 'SUM';
1639
- const aliasBase = `${selected.column}_${aggregate.toLowerCase()}`;
1640
- const preAggregated = isPreAggregatedMetricColumn(selected.column);
1641
- return {
1642
- column: selected.column,
1643
- expression: preAggregated ? sqlIdentifier(selected.column) : `${aggregate}(${sqlIdentifier(selected.column)})`,
1644
- alias: preAggregated ? selected.column : aliasBase,
1645
- score: selected.score,
1646
- preAggregated,
1647
- };
1648
- }
1649
- function inferCountSubject(question) {
1650
- const lower = question.toLowerCase();
1651
- const match = lower.match(/\b(?:how many|number of|count of|count|total)\s+(?:distinct\s+|unique\s+)?([a-z][a-z0-9_-]*)/);
1652
- const raw = match?.[1] ?? lower.match(/\b(customers?|accounts?|users?|members?|orders?|products?|players?|teams?|transactions?|sessions?|events?|records?|rows?)\b/)?.[1];
1653
- if (!raw)
1654
- return undefined;
1655
- const normalized = normalizeToken(raw.replace(/[^a-z0-9_]/g, ''));
1656
- if (!normalized || normalized === 'row' || normalized === 'record')
1657
- return undefined;
1658
- return normalized;
1659
- }
1660
- function findDistinctCountColumn(table, subject) {
1661
- const preferred = [
1662
- `${subject}_id`,
1663
- `${subject}_key`,
1664
- `${subject}_uuid`,
1665
- `${subject}_sk`,
1666
- `${subject}_name`,
1667
- subject,
1668
- ];
1669
- const direct = findSchemaColumn(table, preferred);
1670
- if (direct)
1671
- return direct;
1672
- const tableTokens = meaningfulTokens([table.name, table.relation].join(' '));
1673
- if (!tableTokens.has(subject))
1674
- return undefined;
1675
- return findSchemaColumn(table, ['id', 'key', 'uuid']);
1676
- }
1677
- function isPrecomputedCountMetric(column, subject) {
1678
- const normalized = column.toLowerCase();
1679
- const text = normalized.replace(/_/g, ' ');
1680
- if (/\b(id|key|uuid|code)\b/i.test(text))
1681
- return false;
1682
- if (/\b(count|cnt|number|num)\b/i.test(text)) {
1683
- return !subject || text.includes(subject) || text.includes(pluralizeSimple(subject));
1684
- }
1685
- return Boolean(subject && (normalized === `total_${subject}` ||
1686
- normalized === `total_${pluralizeSimple(subject)}`));
1687
- }
1688
- function pluralizeSimple(value) {
1689
- if (value.endsWith('y'))
1690
- return `${value.slice(0, -1)}ies`;
1691
- if (value.endsWith('s'))
1692
- return value;
1693
- return `${value}s`;
1694
- }
1695
- function augmentGenericRankingDimensions(table, dimensions, metric, question) {
1696
- const direction = rankingDirectionFromText(question);
1697
- if (!direction || !metric.preAggregated || /\bby\s+/i.test(question))
1698
- return dimensions;
1699
- const timeColumn = table.columns
1700
- .map((column) => column.name)
1701
- .find((name) => isTimeLikeGenericColumn(name) && !dimensions.some((dimension) => namesEqualLoose(dimension, name)));
1702
- return timeColumn ? [...dimensions, timeColumn] : dimensions;
1703
- }
1704
- function inferGenericDimensions(table, question, metricColumn) {
1705
- const lower = question.toLowerCase();
1706
- const requested = Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g))
1707
- .map((match) => match[1] ?? '')
1708
- .flatMap((value) => value.split(/\band\b|,|\//i))
1709
- .map((value) => value.replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time)\b/gi, '').trim())
1710
- .filter(Boolean);
1711
- const direct = requested
1712
- .map((value) => findSchemaColumn(table, [value, value.replace(/\s+/g, '_')]))
1713
- .filter((value) => Boolean(value));
1714
- if (direct.length > 0)
1715
- return uniqueDrilldownStrings(direct);
1716
- const domainHints = [
1717
- /\bcustomers?\b/.test(lower) ? ['customer_name', 'customer', 'customer_id'] : [],
1718
- /\bproducts?\b/.test(lower) ? ['product_name', 'product', 'sku', 'product_id'] : [],
1719
- /\bplayers?\b/.test(lower) ? ['player_name', 'player', 'player_id'] : [],
1720
- /\bteams?\b/.test(lower) ? ['team_name', 'team', 'team_id'] : [],
1721
- /\bregions?\b|\bmarkets?\b/.test(lower) ? ['region', 'market', 'geo'] : [],
1722
- /\bsegments?\b/.test(lower) ? ['segment', 'customer_segment', 'type'] : [],
1723
- /\bchannels?\b/.test(lower) ? ['channel', 'source_channel'] : [],
1724
- /\b(month|monthly)\b/.test(lower) ? ['month', 'order_month', 'created_month'] : [],
1725
- /\b(week|weekly)\b/.test(lower) ? ['week', 'order_week', 'created_week'] : [],
1726
- /\b(year|season|yearly)\b/.test(lower) ? ['season', 'year', 'order_year'] : [],
1727
- ].flat();
1728
- const hinted = findSchemaColumn(table, domainHints);
1729
- if (hinted)
1730
- return [hinted];
1731
- const fallback = table.columns
1732
- .map((column, index) => ({
1733
- column: column.name,
1734
- score: genericDimensionScore(column, question, index, metricColumn),
1735
- }))
1736
- .filter((candidate) => candidate.score > 0)
1737
- .sort((a, b) => b.score - a.score)
1738
- .map((candidate) => candidate.column);
1739
- return uniqueDrilldownStrings(fallback).slice(0, 1);
1740
- }
1741
- function genericDimensionScore(column, question, index, metricColumn) {
1742
- if (metricColumn && namesEqualLoose(column.name, metricColumn))
1743
- return -100;
1744
- const name = column.name.toLowerCase();
1745
- const text = name.replace(/_/g, ' ');
1746
- let score = Math.max(0, 10 - index * 0.05);
1747
- if (/\b(name|category|type|segment|region|market|channel|status|team|player|customer|product|vendor|account|month|week|year|season|date)\b/i.test(text)) {
1748
- score += 24;
1749
- }
1750
- if (isNumericLikeColumn(column) && !isTimeLikeGenericColumn(column.name))
1751
- score -= 18;
1752
- for (const token of meaningfulTokens(question)) {
1753
- if (meaningfulTokens(column.name).has(token))
1754
- score += 10;
1755
- }
1756
- return score;
1757
- }
1758
- function isNumericLikeColumn(column) {
1759
- const type = column.type ?? '';
1760
- const name = column.name.toLowerCase();
1761
- return /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(type)
1762
- || /\b(total|count|amount|revenue|sales|spend|cost|margin|profit|value|points?|score|quantity|duration|minutes?|rate|avg|average)\b/i.test(name.replace(/_/g, ' '));
1763
- }
1764
- function isPreAggregatedMetricColumn(name) {
1765
- return /^(total|count|avg|average|median|min|max)_/i.test(name)
1766
- || /_(total|count|avg|average|median|min|max)$/i.test(name);
1767
- }
1768
- function isTimeLikeGenericColumn(name) {
1769
- return /\b(date|time|day|week|month|quarter|year|season|period)\b/i.test(name);
1770
- }
1771
- function buildEntityProfileProposal(input) {
1772
- const entityTexts = entityMentionsForProfile(input.question, input.contextPack);
1773
- if (entityTexts.length === 0 || !isEntityProfileQuestion(input.question, input.contextPack))
1774
- return undefined;
1775
- const candidates = input.schemaContext
1776
- .map((table) => profileTableCandidate(table, input.question, input.followUp, entityTexts))
1777
- .filter((candidate) => Boolean(candidate))
1778
- .sort((a, b) => b.score - a.score);
1779
- const selected = candidates[0];
1780
- if (!selected)
1781
- return undefined;
1782
- const where = selected.filterValue
1783
- ? [`WHERE ${sqlIdentifier(selected.entityColumn)} = ${sqlStringLiteral(selected.filterValue)}`]
1784
- : [];
1785
- const columns = selected.selectColumns.map((column) => ` ${sqlIdentifier(column)}`).join(',\n');
1786
- const text = [
1787
- `Prepared a review-required profile query for ${selected.filterValue ?? entityTexts[0]} from ${selected.table.relation}.`,
1788
- selected.usedSampleValue
1789
- ? `The entity filter uses an inspected value match on ${selected.entityColumn}.`
1790
- : `The entity filter uses the likely entity column ${selected.entityColumn} from inspected schema metadata.`,
1791
- 'This result is uncertified until reviewed and promoted.',
1792
- ].join(' ');
1793
- return {
1794
- text,
1795
- sql: [
1796
- 'SELECT',
1797
- columns,
1798
- `FROM ${sqlRelation(selected.table.relation)}`,
1799
- ...where,
1800
- 'LIMIT 50',
1801
- ].join('\n'),
1802
- viz: 'table',
1803
- };
1804
- }
1805
- function profileTableCandidate(table, question, followUp, entityTexts) {
1806
- if (table.columns.length === 0)
1807
- return undefined;
1808
- const matchedFilters = matchedEntityFiltersForQuestion(table, question, followUp);
1809
- const sampled = matchedFilters.find((filter) => entityTexts.some((entity) => normalizeForEntityMatch(entity) === normalizeForEntityMatch(filter.value))) ?? matchedFilters[0];
1810
- const entityColumn = sampled?.column ?? pickEntityProfileColumn(table, question);
1811
- if (!entityColumn)
1812
- return undefined;
1813
- const filterValue = sampled?.value ?? entityTexts[0];
1814
- if (!filterValue)
1815
- return undefined;
1816
- const selectColumns = orderedProfileColumns(table, entityColumn, question);
1817
- if (selectColumns.length === 0)
1818
- return undefined;
1819
- const questionTokens = meaningfulTokens(question);
1820
- const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
1821
- const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
1822
- const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
1823
- const profileSignal = selectColumns.filter((column) => isProfileMeasureColumn(column, table)).length;
1824
- const score = (sampled ? 80 : 42) +
1825
- overlap * 6 +
1826
- profileSignal * 3 +
1827
- (/\b(player|customer|account|user|member|person|entity)\b/i.test(table.name) ? 8 : 0);
1828
- return {
1829
- table,
1830
- entityColumn,
1831
- filterValue,
1832
- selectColumns,
1833
- usedSampleValue: Boolean(sampled),
1834
- score,
1835
- };
1836
- }
1837
- function isEntityProfileQuestion(question, contextPack) {
1838
- const mode = contextPack?.questionPlan?.mode;
1839
- if (mode === 'entity_profile')
1840
- return true;
1841
- return /\b(profile|overview|360|complete\s+(?:stats|statistics|view)|full\s+(?:stats|statistics|view)|all\s+(?:stats|statistics|metrics)|research|reserach)\b/i.test(question);
1842
- }
1843
- function entityMentionsForProfile(question, contextPack) {
1844
- const values = [
1845
- ...(contextPack?.questionPlan?.entities ?? []).map((entity) => entity.text),
1846
- ...Array.from(question.matchAll(/\b(?:for|on|profile\s+for|research\s+on|reserach\s+on)\s+([A-Z][A-Za-z0-9&.'-]+(?:\s+[A-Z][A-Za-z0-9&.'-]+){0,5})/g))
1847
- .map((match) => match[1] ?? ''),
1848
- ];
1849
- if (isEntityProfileQuestion(question, contextPack)) {
1850
- values.push(...Array.from(question.matchAll(/\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,5})\b/g)).map((match) => match[1] ?? ''));
2210
+ const score = joinColumnScore(leftColumn.name, rightColumn.name, left, right);
2211
+ if (score <= 0)
2212
+ continue;
2213
+ candidates.push({ leftColumn: leftColumn.name, rightColumn: rightColumn.name, score });
2214
+ }
1851
2215
  }
1852
- return uniqueDrilldownStrings(values
1853
- .map((value) => value.replace(/\b(profile|stats|statistics|details|summary)\b.*$/i, '').trim())
1854
- .filter((value) => value.length > 1 && !/^(Can You|Could You|Tell Me|Show Me)$/i.test(value)))
1855
- .slice(0, 6);
2216
+ return candidates.sort((a, b) => b.score - a.score || a.leftColumn.localeCompare(b.leftColumn))[0];
1856
2217
  }
1857
- function pickEntityProfileColumn(table, question) {
1858
- const lower = question.toLowerCase();
1859
- const domainHints = [
1860
- /\bplayers?\b/.test(lower) ? 'player' : '',
1861
- /\bcustomers?\b/.test(lower) ? 'customer' : '',
1862
- /\baccounts?\b/.test(lower) ? 'account' : '',
1863
- /\busers?\b/.test(lower) ? 'user' : '',
1864
- /\bmembers?\b/.test(lower) ? 'member' : '',
1865
- /\bteams?\b/.test(lower) ? 'team' : '',
1866
- /\bproducts?\b/.test(lower) ? 'product' : '',
1867
- ].filter(Boolean);
1868
- const preferred = [
1869
- ...domainHints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
1870
- 'name',
1871
- 'full_name',
1872
- 'display_name',
1873
- 'title',
1874
- ];
1875
- const direct = findSchemaColumn(table, preferred);
1876
- if (direct)
1877
- return direct;
1878
- const scored = table.columns
1879
- .map((column) => {
1880
- const name = column.name.toLowerCase();
1881
- let score = 0;
1882
- if (name.endsWith('_name') || name === 'name')
1883
- score += 30;
1884
- if (/\b(name|title|email)\b/i.test(name.replace(/_/g, ' ')))
1885
- score += 20;
1886
- if (domainHints.some((hint) => name.includes(hint)))
1887
- score += 16;
1888
- if (/\b(id|key|code|date|time|amount|total|count|score|points|revenue)\b/i.test(name.replace(/_/g, ' ')))
1889
- score -= 12;
1890
- return { column: column.name, score };
1891
- })
1892
- .filter((item) => item.score > 0)
1893
- .sort((a, b) => b.score - a.score);
1894
- return scored[0]?.column;
1895
- }
1896
- function orderedProfileColumns(table, entityColumn, question) {
1897
- const scored = table.columns
1898
- .map((column, index) => ({
1899
- column: column.name,
1900
- score: profileColumnScore(column, question, index),
1901
- index,
1902
- }))
1903
- .sort((a, b) => b.score - a.score || a.index - b.index);
1904
- const selected = uniqueDrilldownStrings([
1905
- entityColumn,
1906
- ...scored.map((item) => item.column),
1907
- ]).slice(0, 24);
1908
- return selected.length > 0 ? selected : [entityColumn];
1909
- }
1910
- function profileColumnScore(column, question, index) {
1911
- const name = column.name.toLowerCase();
1912
- const text = name.replace(/_/g, ' ');
1913
- const lowerQuestion = question.toLowerCase();
1914
- let score = Math.max(0, 12 - index * 0.1);
1915
- if (isProfileMeasureColumn(column.name, { relation: '', name: '', columns: [column] }))
1916
- score += 35;
1917
- if (/\b(name|title|team|season|year|date|type|category|segment|status)\b/i.test(text))
1918
- score += 18;
1919
- if (/\b(id|key|code)\b/i.test(text))
1920
- score += 2;
1921
- for (const token of meaningfulTokens(lowerQuestion)) {
1922
- if (meaningfulTokens(column.name).has(token))
1923
- score += 12;
1924
- }
1925
- if (column.description && meaningfulTokens(column.description).size > 0)
1926
- score += 4;
1927
- return score;
2218
+ function joinColumnScore(leftColumn, rightColumn, leftTable, rightTable) {
2219
+ const leftKey = normalizeRelationKey(leftColumn).replace(/\./g, '_');
2220
+ const rightKey = normalizeRelationKey(rightColumn).replace(/\./g, '_');
2221
+ const leftJoinLike = isJoinKeyColumn(leftColumn);
2222
+ const rightJoinLike = isJoinKeyColumn(rightColumn);
2223
+ if (!leftJoinLike || !rightJoinLike)
2224
+ return 0;
2225
+ if (leftKey === rightKey)
2226
+ return 70;
2227
+ const leftSubject = joinSubjectForColumn(leftColumn);
2228
+ const rightSubject = joinSubjectForColumn(rightColumn);
2229
+ if (leftSubject && rightSubject && leftSubject === rightSubject)
2230
+ return 62;
2231
+ const leftTableTokens = tableEntityTokens(leftTable);
2232
+ const rightTableTokens = tableEntityTokens(rightTable);
2233
+ if (leftSubject && rightKey === 'id' && rightTableTokens.has(leftSubject))
2234
+ return 58;
2235
+ if (rightSubject && leftKey === 'id' && leftTableTokens.has(rightSubject))
2236
+ return 58;
2237
+ if (leftSubject && rightTableTokens.has(leftSubject) && rightKey.endsWith('_key'))
2238
+ return 42;
2239
+ if (rightSubject && leftTableTokens.has(rightSubject) && leftKey.endsWith('_key'))
2240
+ return 42;
2241
+ return 0;
1928
2242
  }
1929
- function isProfileMeasureColumn(columnName, table) {
1930
- const column = table.columns.find((item) => namesEqualLoose(item.name, columnName));
1931
- const name = columnName.toLowerCase();
1932
- const text = name.replace(/_/g, ' ');
1933
- return /\b(total|count|avg|average|sum|rate|score|points|pts|rebounds?|assists?|steals?|blocks?|turnovers?|minutes?|games?|wins?|losses?|revenue|amount|spend|orders?|quantity|margin|cost|value|stat)\b/i.test(text)
1934
- || /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(column?.type ?? '');
2243
+ function isJoinKeyColumn(column) {
2244
+ const normalized = column.toLowerCase();
2245
+ return normalized === 'id' ||
2246
+ /(^|_)(id|key|uuid|sk)$/.test(normalized) ||
2247
+ /_(id|key|uuid|sk)$/.test(normalized);
1935
2248
  }
1936
- function buildMatchedEntityDrilldownProposal(input) {
1937
- if (input.intent !== 'entity_drilldown' && input.followUp?.kind !== 'drilldown')
1938
- return undefined;
1939
- const table = pickDrilldownTable(input.schemaContext, input.question, input.followUp);
1940
- if (!table)
1941
- return undefined;
1942
- const dimension = inferDrilldownDimension(table, input.question, input.followUp);
1943
- if (!dimension)
1944
- return undefined;
1945
- const entityFilters = matchedEntityFiltersForQuestion(table, input.question, input.followUp);
1946
- if (entityFilters.length === 0)
1947
- return undefined;
1948
- if (entityFilters.some((filter) => namesEqualLoose(filter.column, dimension)))
1949
- return undefined;
1950
- const sourceSql = selectSourceBlockSql(input.contextPack, input.followUp?.sourceBlockName);
1951
- const metric = inferDrilldownMetric(table, input.question, sourceSql);
1952
- if (!metric)
1953
- return undefined;
1954
- const timePredicates = drilldownTimePredicates({
1955
- question: input.question,
1956
- followUp: input.followUp,
1957
- table,
1958
- sourceSql,
1959
- });
1960
- if (mentionsRelativeTime(input.question, input.followUp) && timePredicates.length === 0)
2249
+ function joinSubjectForColumn(column) {
2250
+ const normalized = column.toLowerCase();
2251
+ const subject = normalized.replace(/_(id|key|uuid|sk)$/i, '');
2252
+ if (!subject || subject === normalized || subject === 'id' || subject === 'key')
1961
2253
  return undefined;
1962
- const predicates = [
1963
- ...entityFilters.map((filter) => `${sqlIdentifier(filter.column)} = ${sqlStringLiteral(filter.value)}`),
1964
- ...timePredicates,
1965
- ];
1966
- const where = predicates.length ? [`WHERE ${predicates.join(' AND ')}`] : [];
1967
- return {
1968
- text: [
1969
- `Prepared a review-required ${humanizeIdentifier(dimension)} drilldown from inspected metadata.`,
1970
- `The entity filter uses ${entityFilters.map((filter) => `${filter.column} = ${filter.value}`).join(', ')} from matched sample values.`,
1971
- 'This result is uncertified until reviewed and promoted.',
1972
- ].join(' '),
1973
- sql: [
1974
- 'SELECT',
1975
- ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`,
1976
- ` ${metric.expression} AS ${sqlIdentifier(metric.alias)}`,
1977
- `FROM ${sqlRelation(table.relation)}`,
1978
- ...where,
1979
- `GROUP BY ${sqlIdentifier(dimension)}`,
1980
- `ORDER BY ${sqlIdentifier(metric.alias)} DESC`,
1981
- 'LIMIT 50',
1982
- ].join('\n'),
1983
- viz: 'bar',
1984
- };
2254
+ return normalizeToken(subject.split('_').at(-1) ?? subject);
2255
+ }
2256
+ function tableEntityTokens(table) {
2257
+ const tokens = exactMatchTokens([table.name, table.relation.split('.').at(-1) ?? table.relation].join(' '));
2258
+ for (const generic of ['dim', 'fct', 'fact', 'stg', 'stage', 'model', 'table'])
2259
+ tokens.delete(generic);
2260
+ return tokens;
1985
2261
  }
1986
2262
  function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
1987
2263
  const byRelation = new Map();
@@ -2059,169 +2335,6 @@ function relationSelectionFor(relation, lookup) {
2059
2335
  }
2060
2336
  return undefined;
2061
2337
  }
2062
- function pickDrilldownTable(schemaContext, question, followUp) {
2063
- const scored = schemaContext
2064
- .map((table) => ({
2065
- table,
2066
- filters: matchedEntityFiltersForQuestion(table, question, followUp).length,
2067
- dimension: inferDrilldownDimension(table, question, followUp) ? 1 : 0,
2068
- measure: inferDrilldownMetric(table, question, undefined) ? 1 : 0,
2069
- }))
2070
- .filter((candidate) => candidate.filters > 0 && candidate.dimension > 0 && candidate.measure > 0)
2071
- .sort((a, b) => (b.filters + b.dimension + b.measure) - (a.filters + a.dimension + a.measure));
2072
- return scored[0]?.table;
2073
- }
2074
- function inferDrilldownDimension(table, question, followUp) {
2075
- const lower = question.toLowerCase();
2076
- const requested = [
2077
- ...(followUp?.dimensions ?? []),
2078
- ...Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g)).map((match) => match[1] ?? ''),
2079
- ]
2080
- .flatMap((value) => value.split(/\band\b|,|\//i))
2081
- .map((value) => value.replace(/\b(last|this|next|previous|prior|current)\s+(day|week|month|quarter|year)\b/gi, '').trim())
2082
- .filter(Boolean);
2083
- const direct = findSchemaColumn(table, requested);
2084
- if (direct)
2085
- return direct;
2086
- if (/\bcustomers?\b/.test(lower)) {
2087
- const customer = findSchemaColumn(table, ['customer', 'customer_name', 'account', 'account_name']);
2088
- if (customer)
2089
- return customer;
2090
- }
2091
- if (/\bsegments?\b/.test(lower)) {
2092
- const segment = findSchemaColumn(table, ['segment', 'customer_segment', 'market_segment']);
2093
- if (segment)
2094
- return segment;
2095
- }
2096
- if (/\bproducts?\b/.test(lower)) {
2097
- const product = findSchemaColumn(table, ['product', 'product_name', 'sku']);
2098
- if (product)
2099
- return product;
2100
- }
2101
- if (/\bregions?\b/.test(lower)) {
2102
- const region = findSchemaColumn(table, ['region', 'market', 'geo']);
2103
- if (region)
2104
- return region;
2105
- }
2106
- return undefined;
2107
- }
2108
- function matchedEntityFiltersForQuestion(table, question, followUp) {
2109
- const text = normalizeForEntityMatch([
2110
- question,
2111
- ...(followUp?.filters ?? []),
2112
- ].join(' '));
2113
- const filters = [];
2114
- for (const column of table.columns) {
2115
- for (const sampleValue of column.sampleValues ?? []) {
2116
- if (isTemporalDrilldownValue(sampleValue))
2117
- continue;
2118
- const needle = normalizeForEntityMatch(sampleValue);
2119
- if (!needle || !text.includes(needle))
2120
- continue;
2121
- filters.push({ column: column.name, value: sampleValue });
2122
- }
2123
- }
2124
- return uniqueMatchedEntityFilters(filters);
2125
- }
2126
- function inferDrilldownMetric(table, question, sourceSql) {
2127
- const sourceMetric = sourceSql ? aggregateMetricFromSourceSql(table, sourceSql) : undefined;
2128
- if (sourceMetric)
2129
- return sourceMetric;
2130
- const lower = question.toLowerCase();
2131
- const candidates = /\brevenue|arr|mrr|sales\b/.test(lower)
2132
- ? ['revenue', 'net_revenue', 'gross_revenue', 'amount', 'order_total', 'total_amount', 'sales', 'arr', 'mrr']
2133
- : ['amount', 'revenue', 'order_total', 'total_amount', 'value', 'spend'];
2134
- const column = findSchemaColumn(table, candidates);
2135
- if (!column)
2136
- return undefined;
2137
- const alias = /\brevenue|arr|mrr|sales\b/.test(lower) ? 'revenue_total' : `${column}_total`;
2138
- return {
2139
- expression: `SUM(${sqlIdentifier(column)})`,
2140
- alias,
2141
- };
2142
- }
2143
- function aggregateMetricFromSourceSql(table, sourceSql) {
2144
- for (const column of table.columns) {
2145
- const columnPattern = sqlIdentifierPattern(column.name);
2146
- const aggregatePattern = new RegExp(`\\b(SUM|COUNT|AVG|MIN|MAX)\\s*\\(\\s*(?:["\`]?\\w+["\`]?\\s*\\.\\s*)?(${columnPattern}|\\*)\\s*\\)\\s*(?:AS\\s+(["\`]?\\w+["\`]?))?`, 'i');
2147
- const match = sourceSql.match(aggregatePattern);
2148
- if (!match)
2149
- continue;
2150
- const fn = (match[1] ?? 'SUM').toUpperCase();
2151
- const target = match[2] === '*' ? '*' : sqlIdentifier(column.name);
2152
- const alias = cleanSqlIdentifier(match[3] ?? defaultMetricAlias(fn, column.name));
2153
- return {
2154
- expression: `${fn}(${target})`,
2155
- alias,
2156
- };
2157
- }
2158
- return undefined;
2159
- }
2160
- function drilldownTimePredicates(input) {
2161
- if (!mentionsRelativeTime(input.question, input.followUp))
2162
- return [];
2163
- if (!input.sourceSql)
2164
- return [];
2165
- const timeColumns = input.table.columns.map((column) => column.name).filter(isTimeLikeDrilldownColumn);
2166
- if (timeColumns.length === 0)
2167
- return [];
2168
- return extractWherePredicates(input.sourceSql)
2169
- .map(stripSqlAliasQualifiers)
2170
- .filter((predicate) => isReusableSqlPredicate(predicate))
2171
- .filter((predicate) => timeColumns.some((column) => predicateReferencesColumn(predicate, column)))
2172
- .slice(0, 3);
2173
- }
2174
- function selectSourceBlockSql(contextPack, sourceBlockName) {
2175
- const sourceSql = contextPack?.allowedSqlContext?.sourceBlockSql ?? [];
2176
- if (sourceSql.length === 0)
2177
- return undefined;
2178
- const preferred = sourceBlockName
2179
- ? sourceSql.find((source) => namesEqualLoose(source.name, sourceBlockName))
2180
- : undefined;
2181
- return (preferred ?? sourceSql.find((source) => source.status === 'certified') ?? sourceSql[0])?.sql;
2182
- }
2183
- function extractWherePredicates(sql) {
2184
- const match = sql.match(/\bWHERE\b([\s\S]*?)(\bGROUP\s+BY\b|\bHAVING\b|\bORDER\s+BY\b|\bLIMIT\b|$)/i);
2185
- if (!match)
2186
- return [];
2187
- return (match[1] ?? '')
2188
- .split(/\s+AND\s+/i)
2189
- .map((part) => part.trim().replace(/^\(+|\)+$/g, '').replace(/\s+/g, ' '))
2190
- .filter(Boolean);
2191
- }
2192
- function mentionsRelativeTime(question, followUp) {
2193
- const text = [question, ...(followUp?.filters ?? [])].join(' ');
2194
- return /\b(last|this|next|previous|prior|current)\s+(day|week|month|quarter|year)\b/i.test(text)
2195
- || /\b(today|yesterday|tomorrow|ytd|mtd|qtd|wtd)\b/i.test(text);
2196
- }
2197
- function isTimeLikeDrilldownColumn(name) {
2198
- return /\b(date|time|day|week|month|quarter|year|period|created_at|updated_at)\b/i.test(name);
2199
- }
2200
- function stripSqlAliasQualifiers(predicate) {
2201
- return predicate.replace(/\b["`]?\w+["`]?\s*\.\s*(["`]?\w+["`]?)/g, '$1');
2202
- }
2203
- function isReusableSqlPredicate(predicate) {
2204
- if (!predicate || predicate.length > 240)
2205
- return false;
2206
- if (/[;]/.test(predicate) || /--|\/\*/.test(predicate))
2207
- return false;
2208
- return !/\b(SELECT|INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|COPY|PRAGMA|SET)\b/i.test(predicate);
2209
- }
2210
- function predicateReferencesColumn(predicate, column) {
2211
- return new RegExp(`(^|[^\\w])${sqlIdentifierPattern(column)}([^\\w]|$)`, 'i').test(predicate);
2212
- }
2213
- function sqlIdentifierPattern(identifier) {
2214
- const escaped = escapeRegExp(identifier);
2215
- return `(?:"${escaped}"|\`${escaped}\`|${escaped})`;
2216
- }
2217
- function sqlStringLiteral(value) {
2218
- return `'${value.replace(/'/g, "''")}'`;
2219
- }
2220
- function defaultMetricAlias(fn, column) {
2221
- if (fn === 'SUM' && /revenue|amount|sales|arr|mrr/i.test(column))
2222
- return 'revenue_total';
2223
- return `${column}_${fn.toLowerCase()}`;
2224
- }
2225
2338
  function cleanSqlIdentifier(identifier) {
2226
2339
  return identifier.replace(/^["`]|["`]$/g, '').trim();
2227
2340
  }
@@ -2243,200 +2356,22 @@ function relationLookupKeys(relation) {
2243
2356
  keys.add(parts[parts.length - 1]);
2244
2357
  return Array.from(keys);
2245
2358
  }
2246
- function normalizeForEntityMatch(value) {
2247
- return value.toLowerCase().replace(/[^a-z0-9.%+-]+/g, ' ').replace(/\s+/g, ' ').trim();
2248
- }
2249
- function uniqueMatchedEntityFilters(filters) {
2250
- const seen = new Set();
2251
- const unique = [];
2252
- for (const filter of filters) {
2253
- const key = `${filter.column.toLowerCase()}\0${filter.value.toLowerCase()}`;
2254
- if (seen.has(key))
2255
- continue;
2256
- seen.add(key);
2257
- unique.push(filter);
2258
- }
2259
- return unique;
2260
- }
2261
2359
  function uniqueDrilldownStrings(values) {
2262
2360
  return Array.from(new Set(values));
2263
2361
  }
2264
- function isTemporalDrilldownValue(value) {
2265
- return mentionsRelativeTime(value, undefined) || /^\d{4}-\d{2}-\d{2}/.test(value);
2266
- }
2267
- function escapeRegExp(value) {
2268
- return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
2269
- }
2270
- function buildContextPackAwareProposal(input) {
2271
- if (!isGeneratedAgentIntent(input.intent))
2272
- return undefined;
2273
- if (!input.contextPack)
2274
- return undefined;
2275
- const profileProposal = buildContextPackProfileProposal(input.question, input.contextPack);
2276
- if (profileProposal)
2277
- return profileProposal;
2278
- const lower = input.question.toLowerCase();
2279
- if (!/\b(least|lowest|fewest|bottom|min(?:imum)?)\b/.test(lower))
2280
- return undefined;
2281
- for (const object of input.contextPack.objects) {
2282
- if (object.objectType !== 'dql_block' || object.status !== 'certified')
2283
- continue;
2284
- const sql = typeof object.payload?.sql === 'string' ? object.payload.sql.trim() : '';
2285
- if (!sql || !/\border\s+by\b/i.test(sql) || !/\bdesc\b/i.test(sql))
2286
- continue;
2287
- const inverted = invertRankingSql(sql);
2288
- if (!inverted || inverted === sql)
2289
- continue;
2290
- return {
2291
- text: `Generated a review-required least-ranking query by using certified block "${object.name}" as context and reversing its ranking direction. This result is uncertified until reviewed and promoted.`,
2292
- sql: ensurePreviewLimit(inverted, 10),
2293
- viz: 'table',
2294
- };
2295
- }
2296
- return undefined;
2297
- }
2298
- function buildContextPackProfileProposal(question, contextPack) {
2299
- const entityTexts = entityMentionsForProfile(question, contextPack);
2300
- if (entityTexts.length === 0 || !isEntityProfileQuestion(question, contextPack))
2301
- return undefined;
2302
- const sources = [...(contextPack.allowedSqlContext?.sourceBlockSql ?? [])]
2303
- .sort((a, b) => {
2304
- const preferred = contextPack.routeDecision.certifiedApplicability?.objectKey;
2305
- if (preferred && a.objectKey === preferred)
2306
- return -1;
2307
- if (preferred && b.objectKey === preferred)
2308
- return 1;
2309
- return 0;
2310
- });
2311
- for (const source of sources) {
2312
- const shape = extractSimpleSelectShape(source.sql);
2313
- if (!shape)
2314
- continue;
2315
- const entityColumn = pickEntityColumnFromSelectExpressions(shape.selectExpressions, question);
2316
- if (!entityColumn)
2317
- continue;
2318
- const selectExpressions = uniqueSqlSelectExpressions(shape.selectExpressions).slice(0, 24);
2319
- if (selectExpressions.length === 0)
2320
- continue;
2321
- const orderColumn = selectExpressions
2322
- .map(selectExpressionOutputName)
2323
- .find((column) => column && /\b(season|year|date|month|week|game_date|created_at)\b/i.test(column));
2324
- const sql = [
2325
- 'SELECT',
2326
- selectExpressions.map((expression) => ` ${expression}`).join(',\n'),
2327
- `FROM ${shape.relation}`,
2328
- `WHERE ${sqlIdentifier(entityColumn)} = ${sqlStringLiteral(entityTexts[0])}`,
2329
- orderColumn ? `ORDER BY ${sqlIdentifier(orderColumn)} DESC` : '',
2330
- 'LIMIT 50',
2331
- ].filter(Boolean).join('\n');
2332
- return {
2333
- text: `Prepared a review-required profile query for ${entityTexts[0]} by using certified block "${source.name}" as SQL-shape context. This result is uncertified until reviewed and promoted.`,
2334
- sql,
2335
- viz: 'table',
2336
- };
2337
- }
2338
- return undefined;
2339
- }
2340
- function pickEntityColumnFromSelectExpressions(expressions, question) {
2341
- const outputNames = expressions
2342
- .map(selectExpressionOutputName)
2343
- .filter((value) => Boolean(value));
2344
- const lower = question.toLowerCase();
2345
- const hints = [
2346
- /\bplayers?\b/.test(lower) ? 'player' : '',
2347
- /\bcustomers?\b/.test(lower) ? 'customer' : '',
2348
- /\baccounts?\b/.test(lower) ? 'account' : '',
2349
- /\busers?\b/.test(lower) ? 'user' : '',
2350
- /\bteams?\b/.test(lower) ? 'team' : '',
2351
- /\bproducts?\b/.test(lower) ? 'product' : '',
2352
- ].filter(Boolean);
2353
- const preferred = [
2354
- ...hints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
2355
- 'name',
2356
- 'full_name',
2357
- 'display_name',
2358
- ];
2359
- for (const wanted of preferred) {
2360
- const match = outputNames.find((name) => namesEqualLoose(name, wanted));
2361
- if (match)
2362
- return match;
2363
- }
2364
- return outputNames.find((name) => /(^|_)(name|title|email)$/.test(name.toLowerCase()));
2365
- }
2366
- function uniqueSqlSelectExpressions(expressions) {
2367
- const seen = new Set();
2368
- const unique = [];
2369
- for (const expression of expressions) {
2370
- const normalized = expression.replace(/\s+/g, ' ').trim().toLowerCase();
2371
- if (!normalized || seen.has(normalized))
2372
- continue;
2373
- seen.add(normalized);
2374
- unique.push(expression.replace(/\s+/g, ' ').trim());
2375
- }
2376
- return unique;
2377
- }
2378
- function invertRankingSql(sql) {
2379
- const withoutTrailingSemicolon = sql.replace(/;\s*$/, '').trim();
2380
- const inverted = withoutTrailingSemicolon.replace(/\border\s+by\s+([\s\S]*?)(\blimit\b|$)/i, (match, orderExpr, limitKeyword) => {
2381
- if (!/\bdesc\b/i.test(orderExpr))
2382
- return match;
2383
- const nextExpr = orderExpr
2384
- .replace(/\bDESC\b/gi, 'ASC')
2385
- .replace(/\bNULLS\s+FIRST\b/gi, 'NULLS LAST');
2386
- return `ORDER BY ${nextExpr}${limitKeyword}`;
2387
- });
2388
- return inverted !== withoutTrailingSemicolon ? inverted : undefined;
2389
- }
2390
- function ensurePreviewLimit(sql, limit) {
2391
- if (/\blimit\s+\d+\b/i.test(sql))
2392
- return sql;
2393
- return `${sql.replace(/;\s*$/, '').trim()}\nLIMIT ${limit}`;
2394
- }
2395
- function findSchemaTable(schemaContext, names) {
2396
- return schemaContext.find((table) => {
2397
- const tableNames = new Set([table.name, table.relation.split('.').at(-1) ?? table.relation].map((name) => name.toLowerCase()));
2398
- return names.some((name) => tableNames.has(name.toLowerCase()));
2399
- });
2400
- }
2401
- function findSchemaColumn(table, names) {
2402
- const byLower = new Map(table.columns.map((column) => [column.name.toLowerCase(), column.name]));
2403
- for (const name of names) {
2404
- const exact = byLower.get(name.toLowerCase());
2405
- if (exact)
2406
- return exact;
2407
- }
2408
- return undefined;
2409
- }
2410
- function sqlRelation(relation) {
2411
- return relation.split('.').map(sqlIdentifier).join('.');
2412
- }
2413
- function sqlIdentifier(identifier) {
2414
- return /^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)
2415
- ? identifier
2416
- : `"${identifier.replace(/"/g, '""')}"`;
2417
- }
2418
2362
  function humanizeIdentifier(identifier) {
2419
2363
  return identifier.replace(/[_-]+/g, ' ');
2420
2364
  }
2421
- function businessMeasurePhrase(identifier) {
2422
- const lower = identifier.toLowerCase();
2423
- if (lower.includes('lifetime_spend'))
2424
- return 'lifetime spend';
2425
- if (lower.includes('count_lifetime_orders') || lower.includes('lifetime_orders') || lower.includes('order_count')) {
2426
- return 'lifetime order count';
2427
- }
2428
- return humanizeIdentifier(identifier);
2429
- }
2430
2365
  function pickCertifiedArtifact(input) {
2431
2366
  // Hint match wins immediately: the active Skill's vocabulary points the
2432
2367
  // user at a specific block. We still validate it's certified.
2433
2368
  for (const hint of input.blockHints) {
2434
2369
  const node = input.kg.getNode(`block:${hint}`);
2435
- if (node && node.status === 'certified' && hasCompatibleCertifiedBlockMatch(input.question, node)) {
2370
+ if (node && node.status === 'certified' && hasCertifiedNodeFit(input.question, input.questionPlan, node)) {
2436
2371
  return { node, score: 1, snippet: undefined };
2437
2372
  }
2438
2373
  }
2439
- const executableHit = pickFirstCertifiedHit(input.executableArtifactHits, input.kg, input.excludedArtifactIds, input.question);
2374
+ const executableHit = pickFirstCertifiedHit(input.executableArtifactHits, input.kg, input.excludedArtifactIds, input.question, input.questionPlan);
2440
2375
  if (isBusinessDefinitionQuestion(input.question)) {
2441
2376
  if (executableHit && hasExactExecutableArtifactSignal(input.question, executableHit.node)) {
2442
2377
  return executableHit;
@@ -2464,7 +2399,7 @@ function pickCertifiedArtifact(input) {
2464
2399
  }
2465
2400
  return null;
2466
2401
  }
2467
- function pickFirstCertifiedHit(hits, kg, excludedNodeIds, question) {
2402
+ function pickFirstCertifiedHit(hits, kg, excludedNodeIds, question, questionPlan) {
2468
2403
  for (const hit of hits) {
2469
2404
  if (hit.score < CERTIFIED_HIT_THRESHOLD)
2470
2405
  break;
@@ -2472,7 +2407,9 @@ function pickFirstCertifiedHit(hits, kg, excludedNodeIds, question) {
2472
2407
  continue;
2473
2408
  if (!isCertifiedHit(hit, kg))
2474
2409
  continue;
2475
- if (question && hit.node.kind === 'block' && !hasCompatibleCertifiedBlockMatch(question, hit.node))
2410
+ if (question && questionPlan && hit.node.kind === 'block' && !hasCertifiedNodeFit(question, questionPlan, hit.node))
2411
+ continue;
2412
+ if (question && !questionPlan && hit.node.kind === 'block' && !hasCompatibleCertifiedBlockMatch(question, hit.node))
2476
2413
  continue;
2477
2414
  return hit;
2478
2415
  }
@@ -2493,7 +2430,7 @@ function pickCertifiedDrilldownArtifact(input) {
2493
2430
  continue;
2494
2431
  if (!isCertifiedHit(hit, input.kg))
2495
2432
  continue;
2496
- if (!hasCompatibleCertifiedBlockMatch(input.question, hit.node))
2433
+ if (!hasCertifiedNodeFit(input.question, input.questionPlan, hit.node, { allowInferredContract: true }))
2497
2434
  continue;
2498
2435
  if (!hasRequestedDrilldownOverlap(hit.node, requestedTerms))
2499
2436
  continue;
@@ -2593,6 +2530,40 @@ function hasCompatibleCertifiedBlockMatch(question, node) {
2593
2530
  return hasMeaningfulCertifiedBlockSignal(question, node)
2594
2531
  && hasCompatibleRankingDirection(question, node);
2595
2532
  }
2533
+ function hasCertifiedNodeFit(question, plan, node, options = {}) {
2534
+ if (!hasCompatibleRankingDirection(question, node))
2535
+ return false;
2536
+ const definitionLookup = isBusinessDefinitionQuestion(question) && objectNameInQuestion(question, node);
2537
+ const exactExampleMatch = (node.examples ?? []).some((example) => normalizeQuestion(example.question) === normalizeQuestion(question));
2538
+ const exactObjectRequest = objectNameInQuestion(question, node)
2539
+ && /\b(run|use|open|show|execute|certified|saved|block)\b/i.test(question);
2540
+ if (!definitionLookup && !exactExampleMatch && !exactObjectRequest && !hasMeaningfulCertifiedBlockSignal(question, node))
2541
+ return false;
2542
+ const fit = evaluateCertifiedBlockFit({
2543
+ question,
2544
+ plan,
2545
+ block: node,
2546
+ exactExampleMatch: exactExampleMatch || exactObjectRequest,
2547
+ definitionLookup,
2548
+ });
2549
+ if (certifiedFitAllowsTier1(fit))
2550
+ return true;
2551
+ return Boolean(options.allowInferredContract
2552
+ && fit.kind === 'exact'
2553
+ && fit.confidence === 'medium'
2554
+ && fit.missingDimensions.length === 0
2555
+ && fit.missingOutputs.length === 0
2556
+ && fit.unsupportedFilters.length === 0
2557
+ && !fit.grainMismatch);
2558
+ }
2559
+ function objectNameInQuestion(question, node) {
2560
+ const questionText = normalizeQuestion(question);
2561
+ const name = normalizeQuestion(node.name);
2562
+ return Boolean(name && questionText.includes(name));
2563
+ }
2564
+ function normalizeQuestion(value) {
2565
+ return value.toLowerCase().replace(/[_-]+/g, ' ').replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
2566
+ }
2596
2567
  function hasCompatibleRankingDirection(question, node) {
2597
2568
  const questionDirection = rankingDirectionFromText(question);
2598
2569
  if (!questionDirection)
@@ -2805,8 +2776,9 @@ function isFilteredEntityQuestion(question) {
2805
2776
  if (/\b(for|where|only|specific|single|individual|named|called)\b.+\b(account|accounts|customer|customers|product|products|sku|user|users)\b/i.test(lower)) {
2806
2777
  return true;
2807
2778
  }
2808
- if (/\b(account|customer|product|sku|user)\s+(?:id|name|email)\b/i.test(lower))
2809
- return true;
2779
+ if (/\b(account|customer|product|sku|user)\s+(?:id|name|email)\b/i.test(lower)) {
2780
+ return !mentionsEntityIdentifierAsRequestedOutput(lower);
2781
+ }
2810
2782
  if (/[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3}/.test(question) && /\b(revenue|sales|order|orders|spend|value|churn|usage|activity|performance|performed|metric|kpi)\b/i.test(lower)) {
2811
2783
  return true;
2812
2784
  }
@@ -2814,6 +2786,9 @@ function isFilteredEntityQuestion(question) {
2814
2786
  return true;
2815
2787
  return false;
2816
2788
  }
2789
+ function mentionsEntityIdentifierAsRequestedOutput(lowerQuestion) {
2790
+ return /\b(?:with|including|include|return|show|list|give|provide|columns?|fields?|results?)\b[^.?!]{0,100}\b(?:account|customer|product|sku|user)\s+(?:id|name|email)\b/i.test(lowerQuestion);
2791
+ }
2817
2792
  function looksLikeDataQuestion(question) {
2818
2793
  return /\b(show|list|find|what|which|how many|how much|compare|trend|revenue|account|accounts|customer|customers|order|orders|product|products|sales|metric|kpi|dashboard|performance|performed|user|users)\b/i.test(question);
2819
2794
  }
@@ -2971,10 +2946,10 @@ async function requestSqlRepair(input) {
2971
2946
  {
2972
2947
  role: 'user',
2973
2948
  content: [
2974
- 'The generated SQL failed during bounded preview execution.',
2949
+ 'The SQL preview for the review-required DQL artifact failed during bounded preview execution.',
2975
2950
  `Question: ${input.question}`,
2976
2951
  `Execution error: ${input.executionError}`,
2977
- 'Return one corrected read-only SQL query using only the runtime schema below.',
2952
+ 'Return one corrected read-only SQL query using only the runtime schema below, as a single ```json fenced object with summary, sql, viz, outputs, and optional dql metadata fields.',
2978
2953
  schema,
2979
2954
  ].join('\n\n'),
2980
2955
  },
@@ -2987,6 +2962,253 @@ async function requestSqlRepair(input) {
2987
2962
  reasoningEffort: input.reasoningEffort,
2988
2963
  });
2989
2964
  }
2965
+ async function generateProposalWithOptionalTools(input) {
2966
+ const tools = input.tools?.filter((tool) => tool.name && tool.description) ?? [];
2967
+ const options = {
2968
+ signal: input.signal,
2969
+ reasoningEffort: input.reasoningEffort,
2970
+ };
2971
+ if (!input.provider.generateWithTools || tools.length === 0) {
2972
+ return input.provider.generate(input.messages, options);
2973
+ }
2974
+ const toolBudget = proposalToolBudgetForQuestion(input.questionPlan, input.intent, {
2975
+ analysisDepth: input.analysisDepth,
2976
+ reasoningEffort: input.reasoningEffort,
2977
+ });
2978
+ const toolPolicy = {
2979
+ role: 'system',
2980
+ content: [
2981
+ 'You may use the supplied DQL tools to inspect semantic members, certified context, metadata context, and bounded repair options.',
2982
+ `Tool budget for this question: ${toolBudget.maxToolCalls} call(s) (${toolBudget.effortClass}: ${toolBudget.reason}). Stop as soon as a lane can answer.`,
2983
+ 'Prefer semantic compile before deep warehouse search when the semantic layer contains the requested metric/dimensions/time grain.',
2984
+ 'Use context expansion only to repair a named, known relation/column gap; do not loop on the same failed context.',
2985
+ 'Final response must be a single ```json fenced object with summary, sql, viz, outputs, and optional dql metadata fields.',
2986
+ ].join('\n'),
2987
+ };
2988
+ return input.provider.generateWithTools([...input.messages, toolPolicy], tools, {
2989
+ ...options,
2990
+ maxToolCalls: toolBudget.maxToolCalls,
2991
+ onToolCall: (event) => {
2992
+ const sink = input.toolCalls;
2993
+ if (sink)
2994
+ sink.push(evidenceToolCallFromEvent(event, sink.length + 1));
2995
+ },
2996
+ });
2997
+ }
2998
+ async function selectDeepGeneratedProposalCandidate(input) {
2999
+ const initial = await scoreDeepGeneratedProposalCandidate(input, {
3000
+ raw: input.initial.raw,
3001
+ parsed: input.initial.parsed,
3002
+ index: 1,
3003
+ });
3004
+ // Deep mode diversifies whenever we can COMPARE candidate results (an executor
3005
+ // is available) — so a valid-but-subtly-wrong first candidate can be out-voted
3006
+ // by execution-result agreement — or when the first candidate failed and needs a
3007
+ // repair alternative. Without an executor, a validated first candidate is the
3008
+ // most we can assess, so return it and skip the extra generations.
3009
+ const shouldDiversify = Boolean(input.executeGeneratedSql) || !initial.validationOk;
3010
+ if (!shouldDiversify) {
3011
+ return { selected: initial, notes: [] };
3012
+ }
3013
+ const candidates = [initial];
3014
+ const alternatives = await generateDeepAlternativeProposals(input);
3015
+ for (const [offset, raw] of alternatives.entries()) {
3016
+ if (!raw.trim())
3017
+ continue;
3018
+ candidates.push(await scoreDeepGeneratedProposalCandidate(input, {
3019
+ raw,
3020
+ parsed: parseProposal(raw),
3021
+ index: offset + 2,
3022
+ }));
3023
+ if (candidates.length >= 5)
3024
+ break;
3025
+ }
3026
+ const signatureCounts = new Map();
3027
+ for (const candidate of candidates) {
3028
+ if (!candidate.resultSignature)
3029
+ continue;
3030
+ signatureCounts.set(candidate.resultSignature, (signatureCounts.get(candidate.resultSignature) ?? 0) + 1);
3031
+ }
3032
+ for (const candidate of candidates) {
3033
+ const equivalenceCount = candidate.resultSignature ? signatureCounts.get(candidate.resultSignature) ?? 0 : 0;
3034
+ if (equivalenceCount > 1)
3035
+ candidate.score += equivalenceCount * 10;
3036
+ }
3037
+ const selected = candidates
3038
+ .slice()
3039
+ .sort((a, b) => b.score - a.score ||
3040
+ Number(b.validationOk) - Number(a.validationOk) ||
3041
+ Number(Boolean(b.result && !b.executionError)) - Number(Boolean(a.result && !a.executionError)) ||
3042
+ a.index - b.index)[0];
3043
+ if (!selected)
3044
+ return { notes: [] };
3045
+ const status = selected.validationOk
3046
+ ? selected.executionError
3047
+ ? `validated but preview failed: ${selected.executionError}`
3048
+ : selected.result
3049
+ ? `validated and previewed ${selected.result.rowCount.toLocaleString()} row(s)`
3050
+ : 'validated'
3051
+ : `failed validation: ${selected.validationError ?? 'unknown validation issue'}`;
3052
+ // Surface how much the candidates disagreed: distinct execution-result
3053
+ // signatures among the candidates that executed cleanly.
3054
+ const executedCandidates = candidates.filter((candidate) => candidate.resultSignature);
3055
+ const distinctResults = new Set(executedCandidates.map((candidate) => candidate.resultSignature)).size;
3056
+ const agreementNote = executedCandidates.length > 1
3057
+ ? distinctResults === 1
3058
+ ? ` All ${executedCandidates.length} executed candidates agreed on the result.`
3059
+ : ` Candidates disagreed: ${distinctResults} distinct results across ${executedCandidates.length} executed candidates — selected the highest-scoring.`
3060
+ : '';
3061
+ return {
3062
+ selected,
3063
+ notes: [
3064
+ `Deep candidate selection reviewed ${candidates.length} candidate${candidates.length === 1 ? '' : 's'} and selected candidate ${selected.index} (${status}).${agreementNote}`,
3065
+ ],
3066
+ };
3067
+ }
3068
+ async function scoreDeepGeneratedProposalCandidate(input, candidate) {
3069
+ const parsed = cloneParsedProposal(candidate.parsed);
3070
+ if (parsed.sql)
3071
+ parsed.sql = input.contextLedger.qualifySql(parsed.sql).sql;
3072
+ let validationOk = false;
3073
+ let validationError;
3074
+ let result;
3075
+ let executionError;
3076
+ let score = parsed.sql ? 10 : -200;
3077
+ if (parsed.sql) {
3078
+ const validation = input.contextLedger.validateSql(parsed.sql, {
3079
+ question: input.question,
3080
+ intent: input.intent,
3081
+ });
3082
+ validationOk = validation.ok;
3083
+ if (validation.ok) {
3084
+ score += 100;
3085
+ if (input.executeGeneratedSql) {
3086
+ try {
3087
+ result = await input.executeGeneratedSql(parsed.sql);
3088
+ score += 40;
3089
+ if (result.rowCount > 0)
3090
+ score += 8;
3091
+ const resultShape = validateAnswerResultShape(input.questionPlan, result);
3092
+ score -= resultShape.warnings.length * 6;
3093
+ if (generatedResultShapeIsPartial(resultShape))
3094
+ score -= 120;
3095
+ }
3096
+ catch (error) {
3097
+ executionError = error instanceof Error ? error.message : String(error);
3098
+ score -= 20;
3099
+ }
3100
+ }
3101
+ }
3102
+ else {
3103
+ validationError = validation.error;
3104
+ score -= 100;
3105
+ }
3106
+ if (parsed.outputs?.length)
3107
+ score += Math.min(6, parsed.outputs.length);
3108
+ }
3109
+ return {
3110
+ raw: candidate.raw,
3111
+ parsed,
3112
+ index: candidate.index,
3113
+ validationOk,
3114
+ validationError,
3115
+ result,
3116
+ executionError,
3117
+ resultSignature: result && !executionError ? resultEquivalenceSignature(result) : undefined,
3118
+ score,
3119
+ };
3120
+ }
3121
+ async function generateDeepAlternativeProposals(input) {
3122
+ const previousSql = input.initial.parsed.sql
3123
+ ? `\nInitial SQL candidate:\n\`\`\`sql\n${input.initial.parsed.sql}\n\`\`\``
3124
+ : '';
3125
+ // Diverse candidate styles (CHASE-SQL-style): each explores the solution space
3126
+ // differently so execution-result agreement between styles is a strong signal.
3127
+ const variants = [
3128
+ 'Create a second candidate that favors the most direct inspected relations and explicit joins.',
3129
+ 'Create a third candidate using QUERY-PLAN reasoning: first outline the grain, measures, dimensions, and join path as steps, then write SQL that follows that plan exactly.',
3130
+ 'Create a fourth candidate by DECOMPOSITION: break the question into sub-questions, solve each as a CTE, then compose the final SELECT — avoiding assumptions hidden in the first candidate.',
3131
+ ];
3132
+ const temperatures = [0.2, 0.35, 0.5];
3133
+ const out = [];
3134
+ for (const [index, instruction] of variants.entries()) {
3135
+ try {
3136
+ const raw = await input.provider.generate([
3137
+ ...input.messages,
3138
+ { role: 'assistant', content: input.initial.raw },
3139
+ {
3140
+ role: 'user',
3141
+ content: [
3142
+ 'Deep mode is allowed to compare multiple review-required SQL candidates before choosing one.',
3143
+ `Question: ${input.question}`,
3144
+ instruction,
3145
+ previousSql,
3146
+ 'Return only one ```json fenced object with summary, sql, viz, outputs, and optional dql metadata fields.',
3147
+ ].filter(Boolean).join('\n\n'),
3148
+ },
3149
+ ], {
3150
+ signal: input.signal,
3151
+ reasoningEffort: input.reasoningEffort,
3152
+ temperature: temperatures[index] ?? 0.4,
3153
+ });
3154
+ if (raw.trim())
3155
+ out.push(raw);
3156
+ }
3157
+ catch {
3158
+ // Alternative candidates are opportunistic; the initial candidate remains.
3159
+ }
3160
+ }
3161
+ return out;
3162
+ }
3163
+ function cloneParsedProposal(proposal) {
3164
+ return {
3165
+ text: proposal.text,
3166
+ ...(proposal.sql ? { sql: proposal.sql } : {}),
3167
+ ...(proposal.viz ? { viz: proposal.viz } : {}),
3168
+ ...(proposal.outputs?.length ? { outputs: proposal.outputs.slice() } : {}),
3169
+ ...(proposal.proposedEntity ? { proposedEntity: proposal.proposedEntity } : {}),
3170
+ ...(proposal.requestedFilters?.length ? { requestedFilters: proposal.requestedFilters.slice() } : {}),
3171
+ ...(proposal.requestedDimensions?.length ? { requestedDimensions: proposal.requestedDimensions.slice() } : {}),
3172
+ };
3173
+ }
3174
+ function resultEquivalenceSignature(result) {
3175
+ const columns = result.columns.map((column) => String(column)).join('|');
3176
+ const rows = result.rows.slice(0, 25).map((row) => stableResultRow(row)).join('\n');
3177
+ return `${columns}\n${rows}`;
3178
+ }
3179
+ function stableResultRow(row) {
3180
+ if (!row || typeof row !== 'object' || Array.isArray(row)) {
3181
+ return JSON.stringify(row);
3182
+ }
3183
+ const record = row;
3184
+ return JSON.stringify(Object.keys(record).sort().map((key) => [key, record[key]]));
3185
+ }
3186
+ function evidenceToolCallFromEvent(event, order) {
3187
+ return {
3188
+ name: event.name,
3189
+ status: event.isError ? 'failed' : 'checked',
3190
+ inputSummary: summarizeEvidencePayload(event.input),
3191
+ outputSummary: summarizeEvidencePayload(event.output),
3192
+ order,
3193
+ };
3194
+ }
3195
+ function summarizeEvidencePayload(value, maxLength = 700) {
3196
+ if (value === undefined)
3197
+ return undefined;
3198
+ let raw;
3199
+ try {
3200
+ raw = typeof value === 'string' ? value : JSON.stringify(value);
3201
+ }
3202
+ catch {
3203
+ raw = String(value);
3204
+ }
3205
+ if (!raw)
3206
+ return undefined;
3207
+ const compact = raw.replace(/\s+/g, ' ').trim();
3208
+ if (!compact)
3209
+ return undefined;
3210
+ return compact.length <= maxLength ? compact : `${compact.slice(0, maxLength - 3)}...`;
3211
+ }
2990
3212
  function isRetryableGeneratedSqlError(error) {
2991
3213
  return !/\b(read-only|readonly|select or with|unsafe|delete|insert|update|drop|alter|create|attach|copy|pragma)\b/i.test(error);
2992
3214
  }
@@ -3130,6 +3352,21 @@ function buildCertifiedEvidence(input) {
3130
3352
  .slice(0, 4);
3131
3353
  return {
3132
3354
  route: [
3355
+ ...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({
3356
+ terminalLane: 'certified',
3357
+ lanes: {
3358
+ certified: {
3359
+ label: `Lane 1 answered from certified ${input.artifact.kind.replace('_', ' ')}`,
3360
+ detail: input.artifact.name,
3361
+ },
3362
+ semantic: {
3363
+ label: 'Lane 2 semantic compile skipped because certified context already answered',
3364
+ },
3365
+ generated: {
3366
+ label: 'Lane 3 generated DQL artifact skipped because certified context already answered',
3367
+ },
3368
+ },
3369
+ })),
3133
3370
  {
3134
3371
  tool: 'search_certified_artifacts',
3135
3372
  status: 'selected',
@@ -3193,10 +3430,12 @@ function buildCertifiedEvidence(input) {
3193
3430
  sourceTables,
3194
3431
  semanticObjects,
3195
3432
  validation: {
3196
- status: input.executionError ? 'failed' : 'passed',
3433
+ status: input.executionError ? 'failed' : input.resultShapeWarnings?.length ? 'warning' : 'passed',
3197
3434
  message: input.executionError
3198
3435
  ? 'The certified artifact matched, but execution returned an error.'
3199
- : 'Certified artifact routing passed; no generated SQL was promoted.',
3436
+ : input.resultShapeWarnings?.length
3437
+ ? `Certified artifact executed, but the result shape needs review: ${input.resultShapeWarnings.join(' ')}`
3438
+ : 'Certified artifact routing passed; no review-required DQL artifact was promoted.',
3200
3439
  },
3201
3440
  execution: executionEvidence(input.artifact, input.result, input.executionError, input.executorWasAvailable),
3202
3441
  citations: input.citations,
@@ -3219,8 +3458,35 @@ function buildGeneratedEvidence(input) {
3219
3458
  ]).slice(0, 6);
3220
3459
  const selectedAssets = uniqueAssets(selectedNodes.map(assetFromNode)).slice(0, 4);
3221
3460
  const selectedSemantic = input.activeTier === 'semantic_layer' && semanticObjects.length > 0;
3461
+ const certifiedFitStep = certifiedFitEvidenceStep(input.contextPack);
3462
+ const certifiedCandidateFitSteps = certifiedCandidateFitEvidenceSteps(input.contextPack);
3222
3463
  return {
3223
3464
  route: [
3465
+ ...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({
3466
+ terminalLane: input.terminalLane,
3467
+ lanes: {
3468
+ certified: {
3469
+ label: input.followUp?.kind === 'drilldown'
3470
+ ? 'Lane 1 checked for a distinct certified drilldown block'
3471
+ : 'Lane 1 checked certified blocks for exact answer fit',
3472
+ },
3473
+ semantic: {
3474
+ label: input.terminalLane === 'semantic'
3475
+ ? 'Lane 2 answered through semantic metric compile'
3476
+ : input.semanticHits.length > 0
3477
+ ? 'Lane 2 semantic context was checked but did not fully answer'
3478
+ : 'Lane 2 semantic context had no strong match',
3479
+ ...(input.terminalLane === 'semantic' && input.terminalDetail ? { detail: input.terminalDetail } : {}),
3480
+ },
3481
+ generated: {
3482
+ label: input.terminalLane === 'generated'
3483
+ ? 'Lane 3 prepared review-required DQL artifact with SQL preview'
3484
+ : 'Lane 3 generated SQL skipped because semantic compile answered',
3485
+ ...(input.terminalLane === 'generated' && input.terminalDetail ? { detail: input.terminalDetail } : {}),
3486
+ },
3487
+ },
3488
+ })),
3489
+ ...cascadeBudgetEvidenceRouteSteps(input.budgetTrace),
3224
3490
  {
3225
3491
  tool: 'search_certified_artifacts',
3226
3492
  status: 'checked',
@@ -3231,6 +3497,8 @@ function buildGeneratedEvidence(input) {
3231
3497
  : 'No certified artifact was strong enough for this question',
3232
3498
  detail: input.followUp?.sourceBlockName,
3233
3499
  },
3500
+ ...(certifiedFitStep ? [certifiedFitStep] : []),
3501
+ ...certifiedCandidateFitSteps,
3234
3502
  {
3235
3503
  tool: 'propose_drilldown',
3236
3504
  status: input.followUp?.kind === 'drilldown' ? 'checked' : 'skipped',
@@ -3241,6 +3509,7 @@ function buildGeneratedEvidence(input) {
3241
3509
  ? [...(input.followUp.filters ?? []), ...(input.followUp.dimensions ?? [])].join(', ')
3242
3510
  : undefined,
3243
3511
  },
3512
+ ...providerToolEvidenceRouteSteps(input.toolCalls),
3244
3513
  {
3245
3514
  tool: 'search_business_context',
3246
3515
  status: businessAssets.length > 0 ? 'checked' : 'skipped',
@@ -3269,7 +3538,7 @@ function buildGeneratedEvidence(input) {
3269
3538
  {
3270
3539
  tool: 'validate_sql',
3271
3540
  status: 'checked',
3272
- label: 'SQL is generated and requires host validation before certification',
3541
+ label: 'SQL preview for DQL artifact requires host validation before certification',
3273
3542
  },
3274
3543
  {
3275
3544
  tool: 'execute_generated_sql',
@@ -3281,10 +3550,10 @@ function buildGeneratedEvidence(input) {
3281
3550
  ? 'skipped'
3282
3551
  : 'skipped',
3283
3552
  label: input.executionError
3284
- ? 'Generated SQL preview failed'
3553
+ ? 'SQL preview failed'
3285
3554
  : input.result
3286
- ? 'Executed generated SQL as bounded preview'
3287
- : 'Generated SQL preview not requested',
3555
+ ? 'Executed SQL preview for DQL artifact'
3556
+ : 'SQL preview not requested',
3288
3557
  detail: input.executionError ?? (input.result ? `${input.result.rowCount} rows` : undefined),
3289
3558
  },
3290
3559
  {
@@ -3318,19 +3587,20 @@ function buildGeneratedEvidence(input) {
3318
3587
  selectedAssets,
3319
3588
  sourceTables,
3320
3589
  semanticObjects,
3590
+ ...(input.toolCalls?.length ? { toolCalls: input.toolCalls } : {}),
3321
3591
  validation: {
3322
3592
  status: 'warning',
3323
3593
  message: input.followUp?.kind === 'drilldown'
3324
- ? 'Generated drilldown SQL is not certified. It should be validated, reviewed, and promoted only after analyst approval.'
3325
- : 'Generated SQL is not certified. It should be validated, reviewed, and promoted only after analyst approval.',
3594
+ ? 'Review-required drilldown DQL artifact is not certified. Its SQL preview should be validated, reviewed, and promoted only after analyst approval.'
3595
+ : 'Review-required DQL artifact is not certified. Its SQL preview should be validated, reviewed, and promoted only after analyst approval.',
3326
3596
  },
3327
3597
  execution: {
3328
3598
  status: input.executionError ? 'failed' : input.result ? 'executed' : 'not_requested',
3329
3599
  message: input.executionError
3330
3600
  ? input.executionError
3331
3601
  : input.result
3332
- ? 'Executed generated SQL as an uncertified bounded preview.'
3333
- : 'Generated SQL was returned for review; execution is handled by the host after validation.',
3602
+ ? 'Executed SQL preview as an uncertified bounded preview for the DQL artifact.'
3603
+ : 'Review-required DQL artifact was returned for review; SQL preview execution is handled by the host after validation.',
3334
3604
  rowCount: input.result?.rowCount,
3335
3605
  executionTime: input.result?.executionTime,
3336
3606
  },
@@ -3338,9 +3608,110 @@ function buildGeneratedEvidence(input) {
3338
3608
  analysisPlan: input.analysisPlan,
3339
3609
  };
3340
3610
  }
3611
+ function certifiedFitEvidenceStep(contextPack) {
3612
+ const fit = contextPack?.routeDecision?.blockFit;
3613
+ if (!fit)
3614
+ return undefined;
3615
+ const applicability = contextPack?.routeDecision?.certifiedApplicability;
3616
+ const allowed = certifiedFitAllowsTier1(fit);
3617
+ return {
3618
+ tool: 'check_certified_fit',
3619
+ status: allowed ? 'selected' : 'checked',
3620
+ label: allowed
3621
+ ? `Certified block fit passed${applicability?.name ? ` for ${applicability.name}` : ''}`
3622
+ : `Certified block kept as context${applicability?.name ? `: ${applicability.name}` : ''}`,
3623
+ detail: fit.reasons.length > 0
3624
+ ? fit.reasons.join('; ')
3625
+ : allowed
3626
+ ? 'Certified block covers the requested answer contract.'
3627
+ : 'Certified block did not prove an exact answer contract match.',
3628
+ };
3629
+ }
3630
+ function certifiedCandidateFitEvidenceSteps(contextPack) {
3631
+ const candidates = contextPack?.retrievalDiagnostics.certifiedCandidateFits ?? [];
3632
+ return candidates
3633
+ .slice(0, 4)
3634
+ .map((candidate) => ({
3635
+ tool: 'check_certified_candidate_fit',
3636
+ status: candidate.action === 'certified_answer'
3637
+ ? 'selected'
3638
+ : 'checked',
3639
+ label: certifiedCandidateFitLabel(candidate.name, candidate.action),
3640
+ detail: [
3641
+ `applicability=${candidate.applicabilityKind}`,
3642
+ `fit=${candidate.fit.kind}/${candidate.fit.confidence}`,
3643
+ candidate.fit.reasons.join('; '),
3644
+ ].filter(Boolean).join(' | '),
3645
+ }));
3646
+ }
3647
+ function certifiedCandidateFitLabel(name, action) {
3648
+ switch (action) {
3649
+ case 'certified_answer':
3650
+ return `Certified candidate selected: ${name}`;
3651
+ case 'context_only':
3652
+ return `Certified candidate used as context only: ${name}`;
3653
+ case 'eligible_not_selected':
3654
+ return `Certified candidate fit passed but was not selected: ${name}`;
3655
+ case 'rejected_for_fit':
3656
+ return `Certified candidate rejected for answer fit: ${name}`;
3657
+ }
3658
+ }
3659
+ function providerToolEvidenceRouteSteps(toolCalls) {
3660
+ return (toolCalls ?? []).slice(0, 8).map((call) => ({
3661
+ tool: call.name,
3662
+ status: call.status,
3663
+ label: call.status === 'failed'
3664
+ ? `Provider tool failed: ${call.name}`
3665
+ : `Provider tool observed: ${call.name}`,
3666
+ detail: [
3667
+ call.inputSummary ? `input=${truncateEvidenceDetail(call.inputSummary)}` : '',
3668
+ call.outputSummary ? `output=${truncateEvidenceDetail(call.outputSummary)}` : '',
3669
+ ].filter(Boolean).join(' | ') || undefined,
3670
+ }));
3671
+ }
3672
+ function cascadeBudgetEvidenceRouteSteps(trace) {
3673
+ if (!trace)
3674
+ return [];
3675
+ const { usage, limits } = trace;
3676
+ const used = usage.laneRegroundAttemptsUsed + usage.laneExecutionAttemptsUsed + usage.engineEscalationsUsed;
3677
+ if (used === 0)
3678
+ return [];
3679
+ return [{
3680
+ tool: 'cascade_budget',
3681
+ status: 'checked',
3682
+ label: `Repair budget used: re-ground ${usage.laneRegroundAttemptsUsed}/${limits.lane.reground}, execution ${usage.laneExecutionAttemptsUsed}/${limits.lane.execution}`,
3683
+ detail: `engine escalations ${usage.engineEscalationsUsed}/${limits.engineEscalations}`,
3684
+ }];
3685
+ }
3686
+ function truncateEvidenceDetail(value, maxLength = 240) {
3687
+ return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`;
3688
+ }
3341
3689
  function buildNoAnswerEvidence(input) {
3342
3690
  return {
3343
3691
  route: [
3692
+ ...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({
3693
+ terminalLane: 'refusal',
3694
+ lanes: {
3695
+ certified: {
3696
+ label: input.artifactHits.length > 0
3697
+ ? 'Lane 1 certified blocks were checked but not selected'
3698
+ : 'Lane 1 found no certified block match',
3699
+ },
3700
+ semantic: {
3701
+ label: input.semanticHits.length > 0
3702
+ ? 'Lane 2 semantic context was checked but did not answer'
3703
+ : 'Lane 2 found no semantic match',
3704
+ },
3705
+ generated: {
3706
+ label: 'Lane 3 could not produce a validated review-required DQL artifact',
3707
+ },
3708
+ refusal: {
3709
+ label: 'Lane 4 returned an honest refusal or clarification',
3710
+ detail: input.reason,
3711
+ },
3712
+ },
3713
+ })),
3714
+ ...cascadeBudgetEvidenceRouteSteps(input.budgetTrace),
3344
3715
  {
3345
3716
  tool: 'search_certified_artifacts',
3346
3717
  status: input.artifactHits.length > 0 ? 'checked' : 'skipped',
@@ -3361,6 +3732,7 @@ function buildNoAnswerEvidence(input) {
3361
3732
  status: input.manifestHits.length > 0 ? 'checked' : 'skipped',
3362
3733
  label: input.manifestHits.length > 0 ? 'dbt context considered' : 'No dbt match',
3363
3734
  },
3735
+ ...providerToolEvidenceRouteSteps(input.toolCalls),
3364
3736
  {
3365
3737
  tool: 'validate_sql',
3366
3738
  status: 'failed',
@@ -3382,6 +3754,7 @@ function buildNoAnswerEvidence(input) {
3382
3754
  selectedAssets: [],
3383
3755
  sourceTables: uniqueAssets(input.manifestHits.map((hit) => assetFromNode(hit.node))).slice(0, 6),
3384
3756
  semanticObjects: uniqueAssets(input.semanticHits.map((hit) => assetFromNode(hit.node))).slice(0, 6),
3757
+ ...(input.toolCalls?.length ? { toolCalls: input.toolCalls } : {}),
3385
3758
  validation: {
3386
3759
  status: 'failed',
3387
3760
  message: input.reason,
@@ -3431,7 +3804,7 @@ function schemaContextAssets(schemaContext) {
3431
3804
  nodeId: `runtime_schema:${table.relation}`,
3432
3805
  kind: 'runtime_schema',
3433
3806
  name: table.relation,
3434
- description: table.description ?? `${table.columns.length} runtime column${table.columns.length === 1 ? '' : 's'} available for generated SQL.`,
3807
+ description: table.description ?? `${table.columns.length} runtime column${table.columns.length === 1 ? '' : 's'} available for DQL artifact SQL previews.`,
3435
3808
  sourceTier: 'project',
3436
3809
  certification: 'ai_generated',
3437
3810
  provenance: table.source ?? 'runtime information_schema',
@@ -3450,11 +3823,18 @@ function schemaCitations(schemaContext, limit) {
3450
3823
  }));
3451
3824
  }
3452
3825
  function certificationForNode(node) {
3453
- if (node.status === 'certified' || node.certification === 'certified')
3826
+ const certification = node.certification;
3827
+ if (node.status === 'certified' || certification === 'certified')
3454
3828
  return 'certified';
3455
- if (node.certification === 'analyst_review_required')
3829
+ if (certification === 'reviewed' || node.status === 'review' || node.status === 'reviewed')
3830
+ return 'reviewed';
3831
+ if (certification === 'conflict')
3832
+ return 'conflict';
3833
+ if (certification === 'insufficient_context')
3834
+ return 'insufficient_context';
3835
+ if (certification === 'analyst_review_required')
3456
3836
  return 'analyst_review_required';
3457
- if (node.certification === 'ai_generated' || node.certification === 'uncertified')
3837
+ if (certification === 'ai_generated' || certification === 'uncertified')
3458
3838
  return 'ai_generated';
3459
3839
  return undefined;
3460
3840
  }