@duckcodeailabs/dql-agent 1.6.30 → 1.6.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/dist/agent-run-engine.d.ts +52 -6
  2. package/dist/agent-run-engine.d.ts.map +1 -1
  3. package/dist/agent-run-engine.js +159 -79
  4. package/dist/agent-run-engine.js.map +1 -1
  5. package/dist/agent-run-gates.d.ts.map +1 -1
  6. package/dist/agent-run-gates.js +72 -5
  7. package/dist/agent-run-gates.js.map +1 -1
  8. package/dist/agent-run-planner.d.ts.map +1 -1
  9. package/dist/agent-run-planner.js +13 -0
  10. package/dist/agent-run-planner.js.map +1 -1
  11. package/dist/answer-loop.d.ts +117 -8
  12. package/dist/answer-loop.d.ts.map +1 -1
  13. package/dist/answer-loop.js +1690 -1302
  14. package/dist/answer-loop.js.map +1 -1
  15. package/dist/answer-shape.d.ts +16 -0
  16. package/dist/answer-shape.d.ts.map +1 -0
  17. package/dist/answer-shape.js +85 -0
  18. package/dist/answer-shape.js.map +1 -0
  19. package/dist/cascade/budgets.d.ts +101 -0
  20. package/dist/cascade/budgets.d.ts.map +1 -0
  21. package/dist/cascade/budgets.js +184 -0
  22. package/dist/cascade/budgets.js.map +1 -0
  23. package/dist/cascade/cascade.d.ts +86 -0
  24. package/dist/cascade/cascade.d.ts.map +1 -0
  25. package/dist/cascade/cascade.js +115 -0
  26. package/dist/cascade/cascade.js.map +1 -0
  27. package/dist/cascade/route-policy.d.ts +14 -0
  28. package/dist/cascade/route-policy.d.ts.map +1 -0
  29. package/dist/cascade/route-policy.js +48 -0
  30. package/dist/cascade/route-policy.js.map +1 -0
  31. package/dist/cascade/triage.d.ts +20 -0
  32. package/dist/cascade/triage.d.ts.map +1 -0
  33. package/dist/cascade/triage.js +25 -0
  34. package/dist/cascade/triage.js.map +1 -0
  35. package/dist/conversation/rolling-summary.d.ts +13 -0
  36. package/dist/conversation/rolling-summary.d.ts.map +1 -0
  37. package/dist/conversation/rolling-summary.js +43 -0
  38. package/dist/conversation/rolling-summary.js.map +1 -0
  39. package/dist/conversation/session-store.d.ts +110 -0
  40. package/dist/conversation/session-store.d.ts.map +1 -0
  41. package/dist/conversation/session-store.js +387 -0
  42. package/dist/conversation/session-store.js.map +1 -0
  43. package/dist/conversation/snapshot.d.ts +59 -0
  44. package/dist/conversation/snapshot.d.ts.map +1 -0
  45. package/dist/conversation/snapshot.js +153 -0
  46. package/dist/conversation/snapshot.js.map +1 -0
  47. package/dist/conversation/working-state.d.ts +49 -0
  48. package/dist/conversation/working-state.d.ts.map +1 -0
  49. package/dist/conversation/working-state.js +249 -0
  50. package/dist/conversation/working-state.js.map +1 -0
  51. package/dist/embeddings/provider.d.ts +62 -0
  52. package/dist/embeddings/provider.d.ts.map +1 -1
  53. package/dist/embeddings/provider.js +98 -0
  54. package/dist/embeddings/provider.js.map +1 -1
  55. package/dist/grounding/context-ledger.d.ts +30 -0
  56. package/dist/grounding/context-ledger.d.ts.map +1 -0
  57. package/dist/grounding/context-ledger.js +54 -0
  58. package/dist/grounding/context-ledger.js.map +1 -0
  59. package/dist/grounding/regrounding.d.ts +24 -0
  60. package/dist/grounding/regrounding.d.ts.map +1 -0
  61. package/dist/grounding/regrounding.js +219 -0
  62. package/dist/grounding/regrounding.js.map +1 -0
  63. package/dist/grounding/value-index.d.ts +32 -0
  64. package/dist/grounding/value-index.d.ts.map +1 -0
  65. package/dist/grounding/value-index.js +67 -0
  66. package/dist/grounding/value-index.js.map +1 -0
  67. package/dist/index.d.ts +47 -16
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +104 -13
  70. package/dist/index.js.map +1 -1
  71. package/dist/intent-controller.d.ts +7 -0
  72. package/dist/intent-controller.d.ts.map +1 -1
  73. package/dist/intent-controller.js +22 -1
  74. package/dist/intent-controller.js.map +1 -1
  75. package/dist/kg/build.d.ts.map +1 -1
  76. package/dist/kg/build.js +44 -17
  77. package/dist/kg/build.js.map +1 -1
  78. package/dist/kg/sqlite-fts.d.ts +3 -1
  79. package/dist/kg/sqlite-fts.d.ts.map +1 -1
  80. package/dist/kg/sqlite-fts.js +16 -2
  81. package/dist/kg/sqlite-fts.js.map +1 -1
  82. package/dist/kg/types.d.ts +23 -1
  83. package/dist/kg/types.d.ts.map +1 -1
  84. package/dist/memory/fts-query.d.ts +7 -0
  85. package/dist/memory/fts-query.d.ts.map +1 -0
  86. package/dist/memory/fts-query.js +36 -0
  87. package/dist/memory/fts-query.js.map +1 -0
  88. package/dist/memory/sqlite-memory.d.ts.map +1 -1
  89. package/dist/memory/sqlite-memory.js +1 -30
  90. package/dist/memory/sqlite-memory.js.map +1 -1
  91. package/dist/metadata/analysis-planner.d.ts +23 -0
  92. package/dist/metadata/analysis-planner.d.ts.map +1 -1
  93. package/dist/metadata/analysis-planner.js +432 -2
  94. package/dist/metadata/analysis-planner.js.map +1 -1
  95. package/dist/metadata/block-fit.d.ts +24 -0
  96. package/dist/metadata/block-fit.d.ts.map +1 -0
  97. package/dist/metadata/block-fit.js +373 -0
  98. package/dist/metadata/block-fit.js.map +1 -0
  99. package/dist/metadata/catalog.d.ts +72 -3
  100. package/dist/metadata/catalog.d.ts.map +1 -1
  101. package/dist/metadata/catalog.js +1452 -72
  102. package/dist/metadata/catalog.js.map +1 -1
  103. package/dist/metadata/drafts.d.ts +25 -0
  104. package/dist/metadata/drafts.d.ts.map +1 -1
  105. package/dist/metadata/drafts.js +112 -2
  106. package/dist/metadata/drafts.js.map +1 -1
  107. package/dist/metadata/metric-match.d.ts +55 -6
  108. package/dist/metadata/metric-match.d.ts.map +1 -1
  109. package/dist/metadata/metric-match.js +158 -13
  110. package/dist/metadata/metric-match.js.map +1 -1
  111. package/dist/metadata/sql-context-validation.d.ts +13 -1
  112. package/dist/metadata/sql-context-validation.d.ts.map +1 -1
  113. package/dist/metadata/sql-context-validation.js +172 -19
  114. package/dist/metadata/sql-context-validation.js.map +1 -1
  115. package/dist/metadata/sql-retrieval.d.ts +3 -2
  116. package/dist/metadata/sql-retrieval.d.ts.map +1 -1
  117. package/dist/metadata/sql-retrieval.js +6 -4
  118. package/dist/metadata/sql-retrieval.js.map +1 -1
  119. package/dist/providers/claude.d.ts +2 -1
  120. package/dist/providers/claude.d.ts.map +1 -1
  121. package/dist/providers/claude.js +151 -28
  122. package/dist/providers/claude.js.map +1 -1
  123. package/dist/providers/gemini.d.ts.map +1 -1
  124. package/dist/providers/gemini.js +20 -0
  125. package/dist/providers/gemini.js.map +1 -1
  126. package/dist/providers/index.d.ts +3 -1
  127. package/dist/providers/index.d.ts.map +1 -1
  128. package/dist/providers/index.js +1 -0
  129. package/dist/providers/index.js.map +1 -1
  130. package/dist/providers/openai.d.ts +2 -1
  131. package/dist/providers/openai.d.ts.map +1 -1
  132. package/dist/providers/openai.js +156 -0
  133. package/dist/providers/openai.js.map +1 -1
  134. package/dist/providers/reasoning-effort.d.ts +46 -0
  135. package/dist/providers/reasoning-effort.d.ts.map +1 -0
  136. package/dist/providers/reasoning-effort.js +81 -0
  137. package/dist/providers/reasoning-effort.js.map +1 -0
  138. package/dist/providers/tool-output.d.ts +2 -0
  139. package/dist/providers/tool-output.d.ts.map +1 -0
  140. package/dist/providers/tool-output.js +22 -0
  141. package/dist/providers/tool-output.js.map +1 -0
  142. package/dist/providers/types.d.ts +29 -0
  143. package/dist/providers/types.d.ts.map +1 -1
  144. package/dist/providers/types.js.map +1 -1
  145. package/dist/semantic-bridge/compose.d.ts +89 -0
  146. package/dist/semantic-bridge/compose.d.ts.map +1 -0
  147. package/dist/semantic-bridge/compose.js +528 -0
  148. package/dist/semantic-bridge/compose.js.map +1 -0
  149. package/dist/semantic-bridge/member-select.d.ts +23 -0
  150. package/dist/semantic-bridge/member-select.d.ts.map +1 -0
  151. package/dist/semantic-bridge/member-select.js +106 -0
  152. package/dist/semantic-bridge/member-select.js.map +1 -0
  153. package/dist/skills/defaults.d.ts.map +1 -1
  154. package/dist/skills/defaults.js +5 -0
  155. package/dist/skills/defaults.js.map +1 -1
  156. package/dist/synthesize.d.ts +26 -0
  157. package/dist/synthesize.d.ts.map +1 -1
  158. package/dist/synthesize.js +72 -3
  159. package/dist/synthesize.js.map +1 -1
  160. package/dist/tools/registry.d.ts +15 -0
  161. package/dist/tools/registry.d.ts.map +1 -0
  162. package/dist/tools/registry.js +692 -0
  163. package/dist/tools/registry.js.map +1 -0
  164. package/dist/trust/stamp.d.ts +26 -0
  165. package/dist/trust/stamp.d.ts.map +1 -0
  166. package/dist/trust/stamp.js +33 -0
  167. package/dist/trust/stamp.js.map +1 -0
  168. package/package.json +4 -4
@@ -14,52 +14,132 @@
14
14
  * step. Tests can mock the provider with a canned response and exercise the
15
15
  * full pipeline.
16
16
  */
17
- import { composeEffectiveTrust, } from '@duckcodeailabs/dql-core';
18
17
  import { buildSkillBlockHints, buildSkillsPrompt, selectRelevantSkills } from './skills/loader.js';
19
- import { matchSemanticMetric, resolveGovernedMetricSql } from './metadata/metric-match.js';
18
+ import { deriveGeneratedDraftSlug, renderGeneratedSqlDqlArtifact } from './metadata/drafts.js';
19
+ import { buildAnalysisQuestionPlan } from './metadata/analysis-planner.js';
20
+ import { certifiedFitAllowsTier1, evaluateCertifiedBlockFit } from './metadata/block-fit.js';
21
+ import { buildGovernedMetricFirstSql, matchSemanticMetric, resolveGovernedMetricSql } from './metadata/metric-match.js';
20
22
  import { decideAgentAction } from './intent-controller.js';
21
- import { validateSqlAgainstLocalContext } from './metadata/sql-context-validation.js';
22
- import { buildGroundingFromRuntimeRelations, resolveRelationsInSql, validateSqlAgainstGrounding } from './metadata/sql-grounding.js';
23
+ import { createContextLedger } from './grounding/context-ledger.js';
24
+ import { validateAnswerResultShape } from './answer-shape.js';
23
25
  import { compactSqlSnippet, extractSimpleSelectShape, selectExpressionOutputName, } from './metadata/sql-shape.js';
26
+ import { composeSemanticQueryForQuestion, composeSemanticQueryFromMembers } from './semantic-bridge/compose.js';
27
+ import { selectSemanticMembersViaLlm } from './semantic-bridge/member-select.js';
28
+ import { normalizeValueIndexText } from './grounding/value-index.js';
29
+ import { cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, } from './cascade/cascade.js';
30
+ import { shouldClarifyBeforeGeneration } from './cascade/triage.js';
31
+ import { stampTrustLabel } from './trust/stamp.js';
32
+ import { QUICK_PROMPT_CONTEXT_BUDGET, canUseLaneRepair, cascadeBudgetTrace, createCascadeBudgetState, promptContextBudgetForQuestion, proposalToolBudgetForQuestion, recordLaneRepair, } from './cascade/budgets.js';
33
+ /**
34
+ * Physical column names whose sampled runtime values include `value`. Used by the
35
+ * semantic bridge to bind a filter literal to the dimension that actually carries
36
+ * it — a generic, project-agnostic replacement for hard-coded value→dimension maps.
37
+ */
38
+ function resolveFilterValueColumns(value, schemaContext) {
39
+ const needle = normalizeValueIndexText(value);
40
+ if (!needle)
41
+ return [];
42
+ const columns = [];
43
+ for (const table of schemaContext) {
44
+ for (const column of table.columns) {
45
+ if (column.sampleValues?.some((sample) => normalizeValueIndexText(sample) === needle)) {
46
+ columns.push(column.name);
47
+ }
48
+ }
49
+ }
50
+ return columns;
51
+ }
24
52
  const CERTIFIED_HIT_THRESHOLD = 0.18;
25
53
  const HARD_NEGATIVE_RATIO = 0.5;
26
54
  const EXECUTABLE_ARTIFACT_KINDS = ['block', 'dashboard', 'app', 'notebook'];
55
+ // Dashboards, apps, and notebooks are governed NAVIGATION targets — collections
56
+ // of tiles / a standing surface, not an executable data query. They can ground or
57
+ // be cited, but they never produce the row-level answer to an analytical question,
58
+ // so they must not terminate a data ask as a "certified answer" with no data.
59
+ const NAVIGATION_ARTIFACT_KINDS = ['dashboard', 'app', 'notebook'];
27
60
  const BUSINESS_CONTEXT_KINDS = ['term', 'business_view'];
28
61
  const ARTIFACT_KINDS = [...EXECUTABLE_ARTIFACT_KINDS, ...BUSINESS_CONTEXT_KINDS];
29
62
  const SEMANTIC_KINDS = ['metric', 'dimension', 'measure', 'entity', 'semantic_model', 'saved_query'];
30
63
  const MANIFEST_KINDS = ['dbt_model', 'dbt_source'];
64
+ function refusalCodeForValidation(code) {
65
+ if (code === 'unknown_relation' || code === 'unknown_column' || code === 'insufficient_context' || code === 'missing_baseline') {
66
+ return 'grounding_gap';
67
+ }
68
+ if (code === 'ambiguous_filter')
69
+ return 'ambiguous';
70
+ return 'model_declined';
71
+ }
72
+ function formatOffendingValidationToken(offending) {
73
+ if (!offending?.relation && !offending?.column)
74
+ return '';
75
+ const parts = [
76
+ offending.relation ? `relation=${offending.relation}` : undefined,
77
+ offending.column ? `column=${offending.column}` : undefined,
78
+ ].filter((part) => Boolean(part));
79
+ return `Offending metadata token: ${parts.join(', ')}`;
80
+ }
81
+ function formatValidationWarningsForPrompt(warnings) {
82
+ const useful = warnings.filter((warning) => warning.trim().length > 0).slice(0, 6);
83
+ return useful.length > 0 ? `Additional inspected-context notes:\n${useful.map((warning) => `- ${warning}`).join('\n')}` : '';
84
+ }
31
85
  /**
32
- * Map an answer-loop result's source tier + certification + review state to a
33
- * canonical trust-label id from the one shared vocabulary in dql-core.
34
- * Additive and lenient keeps the legacy `trustLabel` string untouched.
86
+ * Central honesty gate applied once at the single answer() exit so EVERY return
87
+ * site (certified, semantic-metric, generated) is covered by construction. A DATA
88
+ * answer must have PRODUCED ROWS: when execution was attempted (a result or an
89
+ * execution error came back) but no rows resulted, it may never read as a confident
90
+ * answer.
91
+ * - A generated / governed-metric (uncertified) answer downgrades to a low-
92
+ * confidence, review-required no-data state with an honest message — the SQL is
93
+ * still surfaced so the user can inspect and fix it.
94
+ * - A CERTIFIED block keeps its badge (an empty certified result can be a correct
95
+ * "none matched" answer) but gains a non-blocking note to verify data currency.
96
+ * An UN-executed answer (offline SQL preview / certified citation with no executor)
97
+ * is left untouched — that is a legitimate preview, not a hollow answer.
35
98
  */
36
- function canonicalTrustLabelId(result) {
99
+ function applyHollowAnswerGate(result) {
37
100
  if (result.kind === 'no_answer')
38
- return 'insufficient_context';
39
- if (result.certification === 'certified' || result.kind === 'certified')
40
- return 'certified';
41
- if (result.sourceTier === 'business_context' && result.reviewStatus === 'certified')
42
- return 'reviewed';
43
- if (result.certification === 'ai_generated' ||
44
- result.certification === 'analyst_review_required' ||
45
- result.reviewStatus === 'analyst_review_required' ||
46
- result.reviewStatus === 'draft_ready') {
47
- return 'ai_generated';
101
+ return result;
102
+ const attempted = result.result !== undefined || result.executionError !== undefined;
103
+ const producedRows = Boolean(result.result && typeof result.result.rowCount === 'number' && result.result.rowCount > 0);
104
+ if (!attempted || producedRows)
105
+ return result;
106
+ if (result.kind === 'certified') {
107
+ const note = result.executionError
108
+ ? 'This certified block failed to execute — review the source data before relying on it.'
109
+ : 'This certified block returned 0 rows — verify the source data is current before relying on it.';
110
+ return {
111
+ ...result,
112
+ text: [result.text, note].filter(Boolean).join('\n\n'),
113
+ answer: [result.answer ?? result.text, note].filter(Boolean).join('\n\n'),
114
+ validationWarnings: [...(result.validationWarnings ?? []), note],
115
+ };
48
116
  }
49
- return 'insufficient_context';
117
+ const honestText = [
118
+ result.executionError
119
+ ? `The governed query could not be executed (${result.executionError}).`
120
+ : 'The governed query executed but returned no rows.',
121
+ 'This usually means a filter, grain, or join is off — review the SQL preview and refine before reuse.',
122
+ ].join('\n\n');
123
+ const warning = result.executionError
124
+ ? 'The governed query failed to execute and returned no data — review before reuse.'
125
+ : 'The governed query executed but returned no rows — review the SQL, filters, and joins before reuse.';
126
+ return {
127
+ ...result,
128
+ reviewStatus: 'analyst_review_required',
129
+ confidence: Math.min(result.confidence ?? 0.2, 0.2),
130
+ text: honestText,
131
+ answer: honestText,
132
+ validationWarnings: [...(result.validationWarnings ?? []), warning],
133
+ };
50
134
  }
51
135
  export async function answer(input) {
52
- const result = await runAnswerLoop(input);
136
+ const result = applyHollowAnswerGate(await runAnswerLoop(input));
53
137
  // Attach the canonical trust label once, at the single exit point, so every
54
138
  // return site inside runAnswerLoop stays untouched and backward compatible.
55
139
  // Freshness-aware trust: for a certified answer, fold the source block's data
56
140
  // health (stale/failed upstream) into the label so it reads "Certified ·
57
141
  // stale data" / "Certified · upstream failed". Non-certified or fresh answers
58
142
  // are unaffected.
59
- const id = canonicalTrustLabelId(result);
60
- const dataState = id === 'certified'
61
- ? (result.block?.dataState)
62
- : undefined;
63
143
  const { _semanticMetricMatch, ...publicResult } = result;
64
144
  const chosenRoute = result.route ?? deriveAiRoute(result, _semanticMetricMatch);
65
145
  // P0 — record the high-level action this turn warranted, so callers can route
@@ -80,7 +160,21 @@ export async function answer(input) {
80
160
  return {
81
161
  ...publicResult,
82
162
  intentDecision,
83
- trustLabelInfo: composeEffectiveTrust({ id, dataState }),
163
+ trustLabelInfo: stampTrustLabel(result),
164
+ cascade: publicResult.cascade ?? createCascadeAnswerResult({
165
+ routeTier: chosenRoute.tier,
166
+ label: chosenRoute.label,
167
+ ref: chosenRoute.ref,
168
+ artifactKind: result.dqlArtifact?.kind,
169
+ refusalCode: result.refusalCode,
170
+ reason: result.refusalDetails?.message ?? (result.kind === 'no_answer' ? result.text : undefined),
171
+ rowCount: result.result?.rowCount,
172
+ executionStatus: cascadeExecutionStatus(result),
173
+ draftBlockId: result.draftBlockId ?? result.draftBlock?.path,
174
+ metrics: result.dqlArtifact?.metrics,
175
+ dimensions: result.dqlArtifact?.dimensions,
176
+ hasSqlPreview: Boolean(result.proposedSql ?? result.sql),
177
+ }),
84
178
  // Stamp the SELECTED skills that shaped the answer (transparency). Computed
85
179
  // here so every return site inside runAnswerLoop stays untouched.
86
180
  appliedSkills: result.appliedSkills ??
@@ -93,6 +187,17 @@ export async function answer(input) {
93
187
  route: chosenRoute,
94
188
  };
95
189
  }
190
+ function cascadeExecutionStatus(result) {
191
+ if (result.kind === 'no_answer')
192
+ return 'not_applicable';
193
+ if (result.executionError)
194
+ return 'failed';
195
+ if (result.result)
196
+ return 'executed';
197
+ if (result.block || result.dqlArtifact || result.proposedSql || result.sql)
198
+ return 'not_requested';
199
+ return 'not_applicable';
200
+ }
96
201
  /**
97
202
  * Derive the UI-facing route from a finished answer (spec 17, part C). The
98
203
  * semantic-metric tier is named explicitly when the loop matched a governed
@@ -118,7 +223,7 @@ function deriveAiRoute(result, metricMatch) {
118
223
  ref,
119
224
  };
120
225
  }
121
- // Uncertified: a governed metric matched → semantic_metric; else generated SQL.
226
+ // Uncertified: a governed metric matched → semantic_metric; else generated preview.
122
227
  if (result.sourceTier === 'semantic_layer' && metricMatch) {
123
228
  return {
124
229
  tier: 'semantic_metric',
@@ -126,7 +231,9 @@ function deriveAiRoute(result, metricMatch) {
126
231
  ref: metricMatch.metric.name,
127
232
  };
128
233
  }
129
- return { tier: 'generated_sql', label: 'Answered with generated SQL (review required).' };
234
+ return result.dqlArtifact
235
+ ? { tier: 'generated_sql', label: 'Prepared review-required DQL artifact with SQL preview.' }
236
+ : { tier: 'generated_sql', label: 'Prepared review-required SQL preview.' };
130
237
  }
131
238
  async function runAnswerLoop(input) {
132
239
  const { question, userId, domain, provider, kg, skills = [], blockHints = [] } = input;
@@ -152,6 +259,10 @@ async function runAnswerLoop(input) {
152
259
  const considered = mergeHits(artifactHits, semanticHits, manifestHits, kg.search({ query: question, domain, limit: 10 })).slice(0, 30);
153
260
  const schemaContext = schemaContextWithAllowedSqlContext(input.schemaContext ?? [], input.contextPack);
154
261
  const catalogRoute = input.contextPack?.routeDecision;
262
+ const questionPlan = input.contextPack?.questionPlan?.requestedShape
263
+ ? input.contextPack.questionPlan
264
+ : buildAnalysisQuestionPlan(question, input.followUp);
265
+ const repairBudgetState = createCascadeBudgetState(input.cascadeBudgetModel);
155
266
  const fallbackIntent = classifyAgentIntent({
156
267
  question,
157
268
  followUp: input.followUp,
@@ -160,13 +271,14 @@ async function runAnswerLoop(input) {
160
271
  manifestHits,
161
272
  schemaContext,
162
273
  });
163
- const intent = catalogRoute ? agentIntentFromCatalogRoute(catalogRoute) : fallbackIntent;
274
+ let intent = catalogRoute ? agentIntentFromCatalogRoute(catalogRoute) : fallbackIntent;
164
275
  // Stage 1: certified artifact match. Blocks can be executed; dashboards,
165
276
  // Apps, and notebooks are returned as governed citations/navigation targets.
166
277
  const drilldownCertifiedHit = input.followUp?.kind === 'drilldown'
167
278
  ? pickCertifiedDrilldownArtifact({
168
279
  executableArtifactHits,
169
280
  question,
281
+ questionPlan,
170
282
  followUp: input.followUp,
171
283
  excludedArtifactIds,
172
284
  kg,
@@ -179,23 +291,74 @@ async function runAnswerLoop(input) {
179
291
  executableArtifactHits,
180
292
  businessHits,
181
293
  question,
294
+ questionPlan,
182
295
  blockHints: input.followUp?.kind === 'drilldown' ? [] : effectiveBlockHints,
183
296
  excludedArtifactIds,
184
297
  kg,
185
298
  })
186
299
  : null);
187
- if (artifactHit) {
300
+ // A certified TERM / BUSINESS VIEW is documentation, not data: it can be the
301
+ // terminal answer only for a definition-style question with no data ask. A
302
+ // question that requests a data shape (dimensions/measures/outputs/top-N) or
303
+ // continues an ongoing data conversation must be ANSWERED WITH DATA — the
304
+ // matched business context falls through as grounding for the executable
305
+ // tiers (certified block → governed metric → generated SQL), never instead
306
+ // of them. "Certified" always means "a governed, executable definition
307
+ // produced this result", not "a certified document sounded related".
308
+ const businessContextTerminal = (() => {
309
+ if (!artifactHit || artifactHit.node.kind === 'block')
310
+ return true;
311
+ // An ongoing data conversation (prior turn produced result columns/values)
312
+ // must be answered with data even when the phrasing looks definitional
313
+ // ("so Matthew is the top — what is his 360 profile view?").
314
+ if (input.followUp?.priorResultValues || input.followUp?.priorResultColumns?.length)
315
+ return false;
316
+ // A dashboard / app / notebook is a NAVIGATION target, not executable data.
317
+ // It can only terminate a question that explicitly asks to open/see THAT
318
+ // artifact by name ("open the Jaffle Growth Command Center"); any analytical
319
+ // ask ("top customers who bought the top products with revenue") must fall
320
+ // through to the executable + generated tiers and use the artifact only as
321
+ // grounding — never returned as a no-data "certified answer".
322
+ if (NAVIGATION_ARTIFACT_KINDS.includes(artifactHit.node.kind)) {
323
+ return objectNameInQuestion(question, artifactHit.node);
324
+ }
325
+ // Cold questions fall through only on STRONG data-shape signals — bare
326
+ // measure/dimension words are normal in definition asks ("what is revenue
327
+ // health?") and must keep answering from the certified business context.
328
+ // "Strong" = outputs requested BEYOND the measure/dimension words themselves
329
+ // (e.g. product_name), explicit filters, or a top-N.
330
+ const requested = questionPlan.requestedShape;
331
+ const nonTrivialOutputs = requested.requiredOutputs.filter((output) => !requested.measures.includes(output) && !requested.dimensions.includes(output));
332
+ const strongShape = nonTrivialOutputs.length > 0
333
+ || requested.filters.length > 0
334
+ || Boolean(requested.topN);
335
+ return !strongShape;
336
+ })();
337
+ if (artifactHit && businessContextTerminal) {
188
338
  let result;
189
339
  let executionError;
190
340
  if (artifactHit.node.kind === 'block' && input.executeCertifiedBlock) {
191
341
  try {
192
342
  result = await input.executeCertifiedBlock(artifactHit.node);
343
+ result = trimResultToRequestedTopN(result, questionPlan);
193
344
  }
194
345
  catch (err) {
195
346
  executionError = err instanceof Error ? err.message : String(err);
196
347
  }
197
348
  }
198
- const text = composeCertifiedAnswer(artifactHit.node, question, result, executionError);
349
+ const resultShapeWarnings = result ? validateAnswerResultShape(questionPlan, result).warnings : [];
350
+ // When a certified block's execution was ATTEMPTED and FAILED, the answer
351
+ // cannot wear the certified badge — a failed run has no data to stand behind.
352
+ // Downgrade to analyst_review_required (the error is surfaced in the text).
353
+ // Note: a matched-but-unexecuted block (no executor / dry-run) legitimately
354
+ // stays a certified *citation* — non-execution is a separate axis surfaced via
355
+ // cascade.executionStatus ('not_requested') and freshness dataState, not a
356
+ // reason to strip source trust.
357
+ const certifiedShapePassed = executionError === undefined && resultShapeWarnings.length === 0;
358
+ const certifiedText = composeCertifiedAnswer(artifactHit.node, question, result, executionError);
359
+ const text = resultShapeWarnings.length > 0
360
+ ? `${certifiedText}\n\nReview required: ${resultShapeWarnings.join(' ')}`
361
+ : certifiedText;
199
362
  const sourceTier = artifactHit.node.sourceTier === 'business_context'
200
363
  ? 'business_context'
201
364
  : 'certified_artifact';
@@ -218,21 +381,24 @@ async function runAnswerLoop(input) {
218
381
  sql: result?.sql,
219
382
  suggestedViz: result?.chartConfig ? chartNameFromConfig(result.chartConfig) : undefined,
220
383
  });
384
+ const dqlArtifact = buildCertifiedBlockDqlArtifact(artifactHit.node, result);
221
385
  return {
222
- kind: 'certified',
386
+ kind: certifiedShapePassed ? 'certified' : 'uncertified',
223
387
  sourceTier,
224
- certification: 'certified',
225
- reviewStatus: 'certified',
226
- confidence: 0.95,
388
+ certification: certifiedShapePassed ? 'certified' : 'analyst_review_required',
389
+ reviewStatus: certifiedShapePassed ? 'certified' : 'analyst_review_required',
390
+ confidence: certifiedShapePassed ? 0.95 : 0.45,
227
391
  text,
228
392
  answer: text,
229
393
  block: artifactHit.node.kind === 'block' ? artifactHit.node : undefined,
230
394
  result,
231
395
  executionError,
232
396
  sql: result?.sql,
233
- trustLabel: input.contextPack?.trustLabel ?? 'certified',
397
+ dqlArtifact,
398
+ trustLabel: certifiedShapePassed ? input.contextPack?.trustLabel ?? 'certified' : 'mixed',
234
399
  sourceCertifiedBlock: artifactHit.node.kind === 'block' ? artifactHit.node.name : undefined,
235
400
  contextPackId: input.contextPack?.id,
401
+ validationWarnings: resultShapeWarnings,
236
402
  selectedEvidence: input.contextPack?.evidenceRoles?.slice(0, 12),
237
403
  citations,
238
404
  memoryContext: input.memoryContext,
@@ -246,6 +412,7 @@ async function runAnswerLoop(input) {
246
412
  considered,
247
413
  result,
248
414
  executionError,
415
+ resultShapeWarnings,
249
416
  executorWasAvailable: Boolean(input.executeCertifiedBlock),
250
417
  citations,
251
418
  memoryContext: input.memoryContext ?? [],
@@ -264,8 +431,19 @@ async function runAnswerLoop(input) {
264
431
  // cumulative_revenue). Certified-first is still preserved (checked above).
265
432
  const semanticMetricNodes = collectMetricCandidates(semanticHits, considered, kg);
266
433
  let semanticMetricMatch = await matchSemanticMetric(question, semanticMetricNodes).catch(() => null);
267
- // Clarify only when there is ALSO no confident governed-metric match.
268
- if ((intent === 'clarify' || catalogRoute?.route === 'clarify') && !semanticMetricMatch) {
434
+ const clarifyBeforeGeneration = shouldClarifyBeforeGeneration({
435
+ intent,
436
+ routeDecision: catalogRoute,
437
+ hasSemanticMetricMatch: Boolean(semanticMetricMatch),
438
+ schemaContextCount: schemaContext.length,
439
+ allowedRelationCount: input.contextPack?.allowedSqlContext?.relations.length ?? 0,
440
+ sourceBlockSqlCount: input.contextPack?.allowedSqlContext?.sourceBlockSql.length ?? 0,
441
+ metadataObjectCount: considered.length,
442
+ });
443
+ // Clarify only when there is ALSO no confident governed-metric match and no
444
+ // usable schema/catalog context. A conservative catalog "clarify" decision is
445
+ // not a terminal lane if runtime or SQL context can still answer with review.
446
+ if (clarifyBeforeGeneration) {
269
447
  const text = composeCatalogClarificationText(question, catalogRoute) ?? composeClarificationText(question, considered, schemaContext);
270
448
  const analysisPlan = buildAnalysisPlan({
271
449
  question,
@@ -298,12 +476,15 @@ async function runAnswerLoop(input) {
298
476
  considered,
299
477
  memoryContext: input.memoryContext ?? [],
300
478
  analysisPlan,
479
+ budgetTrace: cascadeBudgetTrace(repairBudgetState),
301
480
  }),
302
481
  contextPack: input.contextPack,
303
482
  considered,
304
483
  providerUsed: provider.name,
305
484
  };
306
485
  }
486
+ if (intent === 'clarify')
487
+ intent = questionPlan.routeIntent;
307
488
  // Stage 2/3: generate only after certified artifacts miss. Semantic context
308
489
  // wins over raw dbt manifest context; memory is appended last as advisory.
309
490
  // A confident metric match forces the semantic tier even when FTS returned no
@@ -329,9 +510,15 @@ async function runAnswerLoop(input) {
329
510
  ? [...matchedMetricHit, ...trustedArtifactContext, ...reviewRequiredArtifactHits, ...businessHits.slice(0, 4), ...semanticHits, ...manifestHits].slice(0, 14)
330
511
  : [...trustedArtifactContext, ...reviewRequiredArtifactHits, ...businessHits.slice(0, 4), ...manifestHits].slice(0, 14);
331
512
  const contextNodes = mergeNodes(followUpSourceBlock && input.followUp?.kind === 'drilldown' ? [followUpSourceBlock] : [], (contextHits.length > 0 ? contextHits : considered.slice(0, 6)).map((h) => h.node));
513
+ const kgJoinPathHints = buildKgJoinPathHints(kg, contextNodes, questionPlan);
332
514
  const contextBlocks = contextNodes.filter((n) => n.kind === 'block');
333
515
  const contextBusiness = contextNodes.filter((n) => BUSINESS_CONTEXT_KINDS.includes(n.kind));
334
516
  const contextOther = contextNodes.filter((n) => n.kind !== 'block' && !BUSINESS_CONTEXT_KINDS.includes(n.kind));
517
+ const promptBudget = promptContextBudgetForQuestion({
518
+ questionPlan,
519
+ requestedDepth: input.analysisDepth,
520
+ reasoningEffort: input.reasoningEffort,
521
+ });
335
522
  const messages = [
336
523
  { role: 'system', content: SYSTEM_PROMPT },
337
524
  ];
@@ -340,28 +527,123 @@ async function runAnswerLoop(input) {
340
527
  messages.push({ role: 'system', content: skillsPrompt });
341
528
  messages.push({
342
529
  role: 'system',
343
- content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp, schemaContext, intent, input.contextPack),
530
+ content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp, schemaContext, intent, input.contextPack, input.conversationSnapshot, kgJoinPathHints, promptBudget),
344
531
  });
345
532
  messages.push({ role: 'user', content: question });
346
- const localProposal = buildSchemaAwareProposal({
347
- question,
348
- intent,
349
- schemaContext,
350
- followUp: input.followUp,
351
- contextPack: input.contextPack,
352
- }) ?? buildContextPackAwareProposal({
353
- question,
354
- intent,
533
+ // ── Tier 2: semantic-layer metrics + dimensions (governed hierarchy) ──────
534
+ // Certified blocks already missed (Stage 1). Before LLM generation, a
535
+ // confidently matched governed metric whose definition can
536
+ // express the question's full requested shape (scalar KPI, or a group-by whose
537
+ // dimensions all resolve to columns on the metric's own table) answers
538
+ // deterministically. Only when no metric fits does the question fall through
539
+ // to SQL generation, where the metric still grounds the prompt as context.
540
+ let governedMetricAnswer = false;
541
+ const semanticBridgeToolCalls = [];
542
+ let semanticBridgeAnswer;
543
+ if (input.semanticLayer && semanticMetricMatch) {
544
+ semanticBridgeAnswer = composeSemanticQueryForQuestion({
545
+ semanticLayer: input.semanticLayer,
546
+ question,
547
+ questionPlan,
548
+ matchedMetric: semanticMetricMatch.metric,
549
+ filterValueColumns: (value) => resolveFilterValueColumns(value, schemaContext),
550
+ ...(input.semanticDriver ? { driver: input.semanticDriver } : {}),
551
+ ...(input.semanticTableMapping ? { tableMapping: input.semanticTableMapping } : {}),
552
+ });
553
+ // Lane-2 LLM fallback: the deterministic token matcher missed, but a metric
554
+ // matched so the semantic layer IS relevant. Spend ONE call to have the model
555
+ // pick members (the query_semantic_model contract); the compiler still owns the
556
+ // SQL, so a paraphrased metric question stays governed instead of dropping to
557
+ // Lane-3 generation.
558
+ if (!semanticBridgeAnswer) {
559
+ const selection = await selectSemanticMembersViaLlm({
560
+ provider,
561
+ semanticLayer: input.semanticLayer,
562
+ question,
563
+ signal: input.signal,
564
+ reasoningEffort: input.reasoningEffort,
565
+ });
566
+ if (selection) {
567
+ const composed = composeSemanticQueryFromMembers({
568
+ semanticLayer: input.semanticLayer,
569
+ question,
570
+ selection,
571
+ ...(input.semanticDriver ? { driver: input.semanticDriver } : {}),
572
+ ...(input.semanticTableMapping ? { tableMapping: input.semanticTableMapping } : {}),
573
+ });
574
+ // Coverage guard: if the question asked for a breakdown but the LLM
575
+ // selection produced none, the governed answer would silently DROP the
576
+ // requested grouping (governed-but-wrong). Fall through to Lane-3
577
+ // generation, which can express the breakdown.
578
+ const wantedBreakdown = questionPlan.requestedShape.dimensions.length > 0
579
+ || questionPlan.dimensionTerms.length > 0;
580
+ const dropsBreakdown = Boolean(composed)
581
+ && wantedBreakdown
582
+ && composed.dimensions.length === 0
583
+ && !composed.timeDimension;
584
+ if (composed && !dropsBreakdown) {
585
+ semanticBridgeAnswer = composed;
586
+ semanticBridgeToolCalls.push({
587
+ name: 'query_semantic_model',
588
+ status: 'checked',
589
+ inputSummary: `metrics: ${selection.metrics.join(', ')}${selection.dimensions?.length ? `; by ${selection.dimensions.join(', ')}` : ''}`,
590
+ outputSummary: 'LLM-selected semantic members compiled via composeQuery',
591
+ order: semanticBridgeToolCalls.length + 1,
592
+ });
593
+ }
594
+ }
595
+ }
596
+ }
597
+ const metricFirst = semanticMetricMatch
598
+ ? buildGovernedMetricFirstSql({
599
+ metric: semanticMetricMatch.metric,
600
+ pool: semanticMetricNodes,
601
+ requestedShape: questionPlan.requestedShape,
602
+ schemaTables: schemaContext.map((table) => ({
603
+ relation: table.relation,
604
+ name: table.name,
605
+ columns: table.columns.map((column) => ({ name: column.name })),
606
+ })),
607
+ semanticLayer: input.semanticLayer,
608
+ })
609
+ : undefined;
610
+ let contextLedger = createContextLedger({
355
611
  contextPack: input.contextPack,
612
+ schemaContext,
356
613
  });
357
614
  let proposed = '';
358
615
  let parsed;
359
- if (localProposal) {
360
- parsed = localProposal;
616
+ const proposalToolCalls = [...semanticBridgeToolCalls];
617
+ if (semanticBridgeAnswer) {
618
+ governedMetricAnswer = true;
619
+ parsed = {
620
+ sql: semanticBridgeAnswer.sql,
621
+ text: `Answered from governed semantic metric${semanticBridgeAnswer.metrics.length === 1 ? '' : 's'} ${semanticBridgeAnswer.metrics.join(', ')}${semanticBridgeAnswer.dimensions.length > 0 ? ` by ${semanticBridgeAnswer.dimensions.join(', ')}` : ''}. This result is uncertified until reviewed and promoted.`,
622
+ viz: semanticBridgeAnswer.dimensions.length === 0 && !semanticBridgeAnswer.timeDimension ? 'single_value' : undefined,
623
+ };
624
+ }
625
+ else if (metricFirst) {
626
+ semanticMetricMatch = { ...semanticMetricMatch, metric: metricFirst.metric };
627
+ governedMetricAnswer = true;
628
+ parsed = {
629
+ sql: metricFirst.sql,
630
+ text: `Answered from the governed metric ${metricFirst.metric.name}${metricFirst.dimensions.length > 0 ? ` by ${metricFirst.dimensions.join(', ')}` : ''}. This result is uncertified until reviewed and promoted.`,
631
+ viz: metricFirst.dimensions.length === 0 ? 'single_value' : undefined,
632
+ };
361
633
  }
362
634
  else {
363
635
  try {
364
- proposed = await provider.generate(messages, { signal: input.signal });
636
+ proposed = await generateProposalWithOptionalTools({
637
+ provider,
638
+ messages,
639
+ tools: input.answerLoopTools,
640
+ questionPlan,
641
+ intent,
642
+ signal: input.signal,
643
+ reasoningEffort: input.reasoningEffort,
644
+ analysisDepth: input.analysisDepth,
645
+ toolCalls: proposalToolCalls,
646
+ });
365
647
  }
366
648
  catch (err) {
367
649
  const text = `Provider error: ${err.message}`;
@@ -373,6 +655,8 @@ async function runAnswerLoop(input) {
373
655
  confidence: 0,
374
656
  text,
375
657
  answer: text,
658
+ refusalCode: 'provider_error',
659
+ refusalDetails: { code: 'provider_error', message: text },
376
660
  citations: [],
377
661
  memoryContext: input.memoryContext,
378
662
  evidence: buildNoAnswerEvidence({
@@ -384,6 +668,7 @@ async function runAnswerLoop(input) {
384
668
  manifestHits,
385
669
  considered,
386
670
  memoryContext: input.memoryContext ?? [],
671
+ toolCalls: proposalToolCalls,
387
672
  }),
388
673
  contextPack: input.contextPack,
389
674
  considered,
@@ -392,11 +677,35 @@ async function runAnswerLoop(input) {
392
677
  }
393
678
  parsed = parseProposal(proposed);
394
679
  }
395
- // True when `parsed.sql` was synthesized deterministically from a governed
396
- // semantic-layer metric (not the LLM). Such SQL is trusted and grounded against
397
- // the runtime schema, so it skips the hallucination-guard context validation
398
- // that exists to catch model-invented relations/columns.
399
- let governedMetricAnswer = false;
680
+ let deepCandidateResult;
681
+ let deepCandidateExecutionError;
682
+ let deepCandidateNotes = [];
683
+ if (!governedMetricAnswer && input.analysisDepth === 'deep' && parsed.sql) {
684
+ const selection = await selectDeepGeneratedProposalCandidate({
685
+ provider,
686
+ messages,
687
+ question,
688
+ questionPlan,
689
+ intent,
690
+ initial: { raw: proposed, parsed },
691
+ contextLedger,
692
+ executeGeneratedSql: input.executeGeneratedSql,
693
+ signal: input.signal,
694
+ reasoningEffort: input.reasoningEffort,
695
+ });
696
+ if (selection.selected) {
697
+ proposed = selection.selected.raw;
698
+ parsed = selection.selected.parsed;
699
+ deepCandidateResult = selection.selected.result;
700
+ deepCandidateExecutionError = selection.selected.executionError;
701
+ }
702
+ deepCandidateNotes = selection.notes;
703
+ }
704
+ // `governedMetricAnswer` (declared above): true when `parsed.sql` was
705
+ // synthesized deterministically from a governed semantic-layer metric (not the
706
+ // LLM). Such SQL is trusted and grounded against the runtime schema, so it
707
+ // skips the hallucination-guard context validation that exists to catch
708
+ // model-invented relations/columns.
400
709
  if (!parsed.sql) {
401
710
  // Spec 17, part C — if a governed metric matched confidently but the model
402
711
  // declined SQL, answer from the metric definition (deterministic, offline)
@@ -406,7 +715,7 @@ async function runAnswerLoop(input) {
406
715
  // (`order_item.revenue`). resolveGovernedMetricSql resolves a thin metric to
407
716
  // that synthesizable sibling so the route lands on a real number, not refusal.
408
717
  const resolved = activeTier === 'semantic_layer' && semanticMetricMatch
409
- ? resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes)
718
+ ? resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes, input.semanticLayer)
410
719
  : undefined;
411
720
  if (resolved) {
412
721
  // Point the match at the metric we actually answered from so the route
@@ -421,8 +730,54 @@ async function runAnswerLoop(input) {
421
730
  };
422
731
  }
423
732
  }
733
+ // Forced-join retry. The model declined SQL (rule 8 escape) — but a composite
734
+ // "top X who did top Y" that spans several concepts often trips the model into
735
+ // "there's no combined dataset — show them separately", which is a FALSE refusal
736
+ // when the grounded tables and KG join routes to connect them are right there in
737
+ // context. If the question genuinely wants generated data and we DO have usable
738
+ // relations/schema to build the join, re-issue ONCE with an explicit instruction
739
+ // to compose the join rather than refuse. A truly absent table/column/join key
740
+ // still falls through to the honest no_answer below.
741
+ const wantsGeneratedData = questionPlan.requestedShape.measures.length > 0
742
+ || questionPlan.requestedShape.dimensions.length > 0
743
+ || questionPlan.requestedShape.requiredOutputs.length > 0
744
+ || Boolean(questionPlan.requestedShape.topN);
745
+ const hasGeneratableContext = schemaContext.length > 0
746
+ || (input.contextPack?.allowedSqlContext?.relations.length ?? 0) > 0
747
+ || (input.contextPack?.allowedSqlContext?.sourceBlockSql.length ?? 0) > 0
748
+ || contextBlocks.length > 0;
749
+ if (!parsed.sql && !governedMetricAnswer && wantsGeneratedData && hasGeneratableContext) {
750
+ try {
751
+ proposed = await generateProposalWithOptionalTools({
752
+ provider,
753
+ messages: [...messages, { role: 'system', content: FORCE_JOIN_INSTRUCTION }],
754
+ tools: input.answerLoopTools,
755
+ questionPlan,
756
+ intent,
757
+ signal: input.signal,
758
+ reasoningEffort: input.reasoningEffort,
759
+ analysisDepth: input.analysisDepth,
760
+ toolCalls: proposalToolCalls,
761
+ });
762
+ parsed = parseProposal(proposed);
763
+ }
764
+ catch {
765
+ // keep the original decline; fall through to the honest no_answer.
766
+ }
767
+ }
424
768
  if (!parsed.sql) {
425
- const text = parsed.text || 'No answer (the model declined to propose SQL).';
769
+ // Deterministic honest refusal. The model's own decline prose is STOCHASTIC
770
+ // ("there's no combined dataset — show them separately" one run, a different
771
+ // phrasing the next), which reads as flaky and inconsistent across surfaces.
772
+ // When the question wanted data AND usable context existed (a groundable ask
773
+ // the model still declined even after the forced-join retry), surface ONE
774
+ // consistent, actionable message so the same question yields the same outcome
775
+ // every run and every surface — instead of passing through the model's varying
776
+ // text. A genuinely context-less ask keeps the plain honest message.
777
+ const declinedDespiteContext = wantsGeneratedData && hasGeneratableContext;
778
+ const text = declinedDespiteContext
779
+ ? 'I could not compose a governed query for this from the available tables and metrics. This usually needs a clearer join path or an explicit metric and grouping — name the specific measure and how to break it down, and I can generate a review-required draft.'
780
+ : parsed.text || 'No answer (the model declined to propose SQL).';
426
781
  return {
427
782
  kind: 'no_answer',
428
783
  sourceTier: 'no_answer',
@@ -430,6 +785,11 @@ async function runAnswerLoop(input) {
430
785
  reviewStatus: 'none',
431
786
  confidence: 0.1,
432
787
  text,
788
+ refusalCode: 'model_declined',
789
+ refusalDetails: {
790
+ code: 'model_declined',
791
+ message: text,
792
+ },
433
793
  answer: text,
434
794
  citations: [],
435
795
  memoryContext: input.memoryContext,
@@ -452,32 +812,30 @@ async function runAnswerLoop(input) {
452
812
  // model emitted to its real warehouse relation from the runtime schema BEFORE
453
813
  // governance validation. Same resolver the build path uses — one grounding,
454
814
  // no weak path. `allowedSqlContext` relations are already qualified.
455
- let grounding;
456
- if (parsed.sql && schemaContext.length > 0) {
457
- grounding = buildGroundingFromRuntimeRelations(schemaContext.map((table) => ({
458
- relation: table.relation,
459
- name: table.name,
460
- columns: table.columns.map((column) => ({ name: column.name, type: column.type, description: column.description })),
461
- })));
462
- parsed.sql = resolveRelationsInSql(parsed.sql, grounding, { prefer: 'qualified' }).sql;
815
+ if (parsed.sql) {
816
+ parsed.sql = contextLedger.qualifySql(parsed.sql).sql;
463
817
  }
464
818
  // Validation gate. Governed metric SQL synthesized from the semantic layer is
465
819
  // already trusted (deterministic + grounded); model SQL is validated against the
466
820
  // inspected context to catch hallucinated relations/columns.
467
821
  const semanticMetricRoute = activeTier === 'semantic_layer' && Boolean(semanticMetricMatch);
822
+ const scalarGovernedMetricRecoveryAllowed = questionPlan.requestedShape.dimensions.length === 0
823
+ && questionPlan.requestedShape.filters.length === 0
824
+ && !questionPlan.requestedShape.topN;
468
825
  let contextValidation;
469
826
  if (governedMetricAnswer) {
470
827
  contextValidation = { ok: true, warnings: [] };
471
828
  }
472
829
  else {
473
- const c = validateSqlAgainstLocalContext(parsed.sql, input.contextPack, {
830
+ const c = contextLedger.validateSql(parsed.sql, {
474
831
  question,
475
832
  intent,
476
833
  filterValues: input.followUp?.filters,
834
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
477
835
  });
478
836
  contextValidation = c.ok
479
837
  ? { ok: true, warnings: c.warnings }
480
- : { ok: false, error: c.error, warnings: c.warnings };
838
+ : { ok: false, code: c.code, error: c.error, warnings: c.warnings, offending: c.offending };
481
839
  }
482
840
  // Spec 17, part C — semantic-metric route recovery. A metric question is often
483
841
  // catalog-routed to clarify, leaving a thin contextPack that rejects otherwise-valid
@@ -489,41 +847,132 @@ async function runAnswerLoop(input) {
489
847
  // 2) Otherwise fall back to a CLEAN governed-metric definition (direct or exact
490
848
  // leaf-measure; no fuzzy family guess that could answer the wrong measure).
491
849
  if (!contextValidation.ok && semanticMetricRoute && semanticMetricMatch) {
492
- const grounded = grounding ? validateSqlAgainstGrounding(parsed.sql, grounding) : undefined;
850
+ const grounded = contextLedger.validateRuntimeGrounding(parsed.sql);
493
851
  if (grounded?.ok) {
494
852
  contextValidation = { ok: true, warnings: grounded.warnings };
495
853
  }
496
- else {
497
- const recovered = resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes);
854
+ else if (scalarGovernedMetricRecoveryAllowed) {
855
+ const recovered = resolveGovernedMetricSql(semanticMetricMatch.metric, semanticMetricNodes, input.semanticLayer);
498
856
  if (recovered) {
499
857
  semanticMetricMatch = { ...semanticMetricMatch, metric: recovered.metric };
500
- parsed.sql = grounding
501
- ? resolveRelationsInSql(recovered.sql, grounding, { prefer: 'qualified' }).sql
502
- : recovered.sql;
858
+ parsed.sql = contextLedger.qualifySql(recovered.sql).sql;
503
859
  parsed.viz = parsed.viz ?? 'single_value';
504
860
  parsed.text = parsed.text
505
861
  || `Answered from the governed metric ${recovered.metric.name}. This result is uncertified until reviewed and promoted.`;
506
862
  governedMetricAnswer = true;
507
863
  contextValidation = {
508
864
  ok: true,
509
- warnings: ['Generated SQL failed context validation; answered from the governed metric definition instead.'],
865
+ warnings: ['SQL preview failed context validation; answered from the governed metric definition instead.'],
510
866
  };
511
867
  }
512
868
  }
513
869
  }
870
+ if (!contextValidation.ok && !governedMetricAnswer && input.expandGroundingContext && canUseLaneRepair(repairBudgetState, 'reground')) {
871
+ recordLaneRepair(repairBudgetState, 'reground');
872
+ try {
873
+ const expansion = await input.expandGroundingContext({
874
+ question,
875
+ sql: parsed.sql,
876
+ code: contextValidation.code ?? 'insufficient_context',
877
+ offending: contextValidation.offending,
878
+ contextPack: contextLedger.contextPack,
879
+ schemaContext: contextLedger.schemaContext,
880
+ });
881
+ const merged = contextLedger.withExpansion(expansion);
882
+ if (merged.notes.length > 0) {
883
+ contextLedger = merged.ledger;
884
+ parsed.sql = contextLedger.qualifySql(parsed.sql).sql;
885
+ const revalidated = contextLedger.validateSql(parsed.sql, {
886
+ question,
887
+ intent,
888
+ filterValues: input.followUp?.filters,
889
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
890
+ });
891
+ contextValidation = revalidated.ok
892
+ ? {
893
+ ok: true,
894
+ warnings: [
895
+ ...revalidated.warnings,
896
+ ...merged.notes.map((note) => `Re-grounded metadata context before repair: ${note}`),
897
+ ],
898
+ }
899
+ : {
900
+ ok: false,
901
+ code: revalidated.code,
902
+ error: revalidated.error,
903
+ warnings: [
904
+ ...revalidated.warnings,
905
+ ...merged.notes.map((note) => `Re-grounded metadata context before repair: ${note}`),
906
+ ],
907
+ offending: revalidated.offending,
908
+ };
909
+ }
910
+ }
911
+ catch {
912
+ // Re-grounding is best-effort; the bounded self-repair below still runs.
913
+ }
914
+ }
915
+ // One bounded self-repair before refusing: hand the model the EXACT guard
916
+ // error (e.g. `column "product_name" outside the inspected columns for
917
+ // order_items`) so it can correct itself — usually by joining the relation
918
+ // that actually carries the column. Deterministic paths that never called the
919
+ // provider get the same single chance; any provider failure keeps the honest
920
+ // refusal below. This mirrors the engine-level repair loop, applied to the
921
+ // context-validation gate that previously refused on first failure.
922
+ if (!contextValidation.ok && !governedMetricAnswer && canUseLaneRepair(repairBudgetState, 'reground')) {
923
+ recordLaneRepair(repairBudgetState, 'reground');
924
+ try {
925
+ const failedSql = parsed.sql ?? '';
926
+ const repairPrompt = [
927
+ `Your SQL was rejected before execution: ${contextValidation.error}`,
928
+ formatOffendingValidationToken(contextValidation.offending),
929
+ formatValidationWarningsForPrompt(contextValidation.warnings),
930
+ 'Correct it using ONLY the relations and columns from the inspected context above.',
931
+ 'If a needed column lives on a different relation, JOIN that relation using the suggested join paths.',
932
+ 'If the requested column does not exist anywhere in the inspected context, return the closest answerable SQL and say what is missing.',
933
+ 'Return a single ```json fenced object: {"summary":"...","sql":"SELECT ...","viz":"table","outputs":["column"],"dql":{"entity":"...","dimensions":["..."],"filters":["..."]}}.',
934
+ ].join('\n');
935
+ const repairRaw = await provider.generate([...messages, { role: 'assistant', content: failedSql }, { role: 'user', content: repairPrompt }], { signal: input.signal, reasoningEffort: input.reasoningEffort });
936
+ const reparsed = parseProposal(repairRaw);
937
+ if (reparsed.sql) {
938
+ reparsed.sql = contextLedger.qualifySql(reparsed.sql).sql;
939
+ const revalidated = contextLedger.validateSql(reparsed.sql, {
940
+ question,
941
+ intent,
942
+ filterValues: input.followUp?.filters,
943
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
944
+ });
945
+ if (revalidated.ok) {
946
+ const repairedSql = reparsed.sql;
947
+ parsed.sql = repairedSql;
948
+ parsed.text = reparsed.text || parsed.text;
949
+ parsed.viz = reparsed.viz ?? parsed.viz;
950
+ applyParsedProposalMetadata(parsed, reparsed);
951
+ contextValidation = {
952
+ ok: true,
953
+ warnings: [...revalidated.warnings, 'Repaired after context-validation failure (1 repair).'],
954
+ };
955
+ }
956
+ }
957
+ }
958
+ catch {
959
+ // Repair is best-effort — the refusal below stays the honest fallback.
960
+ }
961
+ }
514
962
  if (!contextValidation.ok) {
515
- const text = `I could not safely prepare this generated SQL from the inspected context. ${contextValidation.error}`;
963
+ const text = `I could not safely prepare this review-required DQL artifact from the inspected context. The SQL preview failed validation: ${contextValidation.error}`;
516
964
  const analysisPlan = buildAnalysisPlan({
517
965
  question,
518
966
  intent,
519
- routeReason: catalogRoute?.reason ?? 'Generated SQL failed metadata context validation before preview execution or draft capture.',
967
+ routeReason: catalogRoute?.reason ?? 'SQL preview for the review-required DQL artifact failed metadata context validation before preview execution or draft capture.',
520
968
  selectedNodes: contextNodes,
521
969
  schemaContext,
522
970
  sql: parsed.sql,
523
971
  suggestedViz: parsed.viz ?? 'table',
524
972
  assumptions: [
525
- 'Generated SQL was rejected before execution because it did not match inspected metadata context.',
973
+ 'SQL preview was rejected before execution because it did not match inspected metadata context.',
526
974
  ...contextValidation.warnings,
975
+ ...deepCandidateNotes,
527
976
  ],
528
977
  });
529
978
  return {
@@ -533,13 +982,19 @@ async function runAnswerLoop(input) {
533
982
  reviewStatus: 'none',
534
983
  confidence: 0.15,
535
984
  text,
985
+ refusalCode: refusalCodeForValidation(contextValidation.code),
986
+ refusalDetails: {
987
+ code: contextValidation.code ?? 'insufficient_context',
988
+ message: contextValidation.error,
989
+ offending: contextValidation.offending,
990
+ },
536
991
  answer: text,
537
992
  proposedSql: parsed.sql,
538
993
  sql: parsed.sql,
539
994
  trustLabel: input.contextPack?.trustLabel,
540
995
  sourceCertifiedBlock: followUpSourceBlock?.name ?? input.followUp?.sourceBlockName,
541
996
  contextPackId: input.contextPack?.id,
542
- validationWarnings: contextValidation.warnings,
997
+ validationWarnings: [...contextValidation.warnings, ...deepCandidateNotes],
543
998
  selectedEvidence: input.contextPack?.evidenceRoles?.slice(0, 12),
544
999
  citations: [],
545
1000
  memoryContext: input.memoryContext,
@@ -557,7 +1012,7 @@ async function runAnswerLoop(input) {
557
1012
  }),
558
1013
  contextPack: input.contextPack,
559
1014
  considered,
560
- providerUsed: localProposal ? 'schema_planner' : provider.name,
1015
+ providerUsed: provider.name,
561
1016
  };
562
1017
  }
563
1018
  const generatedCitations = [
@@ -585,26 +1040,56 @@ async function runAnswerLoop(input) {
585
1040
  // The repair turn returns error-recovery prose ("the column X was not recognized…"),
586
1041
  // NOT a user-facing answer. Keep it for the trace only — never as the answer text.
587
1042
  let repairNarrative;
588
- if (input.executeGeneratedSql) {
1043
+ if (deepCandidateResult) {
1044
+ result = deepCandidateResult;
1045
+ }
1046
+ if (deepCandidateExecutionError) {
1047
+ executionError = deepCandidateExecutionError;
1048
+ }
1049
+ if (input.executeGeneratedSql && !result) {
589
1050
  try {
590
- result = await input.executeGeneratedSql(parsed.sql);
1051
+ if (!executionError)
1052
+ result = await input.executeGeneratedSql(parsed.sql);
591
1053
  }
592
1054
  catch (err) {
593
1055
  executionError = err instanceof Error ? err.message : String(err);
1056
+ }
1057
+ if (executionError) {
594
1058
  if (isRetryableGeneratedSqlError(executionError)) {
595
1059
  const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError, schemaContext);
596
- if (localRepairSql) {
597
- repairAttempts = 1;
598
- parsed.sql = localRepairSql;
599
- try {
600
- result = await input.executeGeneratedSql(parsed.sql);
601
- executionError = undefined;
1060
+ if (localRepairSql && canUseLaneRepair(repairBudgetState, 'execution')) {
1061
+ recordLaneRepair(repairBudgetState, 'execution');
1062
+ const qualifiedLocalRepairSql = contextLedger.qualifySql(localRepairSql).sql;
1063
+ const localRepairValidation = contextLedger.validateSql(qualifiedLocalRepairSql, {
1064
+ question,
1065
+ intent,
1066
+ filterValues: input.followUp?.filters,
1067
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
1068
+ });
1069
+ if (localRepairValidation.ok) {
1070
+ repairAttempts = 1;
1071
+ parsed.sql = qualifiedLocalRepairSql;
1072
+ contextValidation = {
1073
+ ok: true,
1074
+ warnings: [
1075
+ ...localRepairValidation.warnings,
1076
+ 'Execution-repaired SQL passed context validation.',
1077
+ ],
1078
+ };
1079
+ try {
1080
+ result = await input.executeGeneratedSql(parsed.sql);
1081
+ executionError = undefined;
1082
+ }
1083
+ catch (retryErr) {
1084
+ executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1085
+ }
602
1086
  }
603
- catch (retryErr) {
604
- executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1087
+ else {
1088
+ executionError = localRepairValidation.error;
605
1089
  }
606
1090
  }
607
- if (executionError) {
1091
+ if (executionError && canUseLaneRepair(repairBudgetState, 'execution')) {
1092
+ recordLaneRepair(repairBudgetState, 'execution');
608
1093
  const repairedRaw = await requestSqlRepair({
609
1094
  provider,
610
1095
  baseMessages: messages,
@@ -613,20 +1098,41 @@ async function runAnswerLoop(input) {
613
1098
  executionError,
614
1099
  schemaContext,
615
1100
  signal: input.signal,
1101
+ reasoningEffort: input.reasoningEffort,
616
1102
  });
617
1103
  const repaired = parseProposal(repairedRaw);
618
1104
  if (repaired.sql) {
619
- repairAttempts += 1;
620
- // Adopt the corrected SQL, but do NOT let the repair prose become the answer.
621
- repairNarrative = repaired.text?.trim() || undefined;
622
- parsed.sql = repaired.sql;
623
- parsed.viz = repaired.viz ?? parsed.viz;
624
- try {
625
- result = await input.executeGeneratedSql(parsed.sql);
626
- executionError = undefined;
1105
+ repaired.sql = contextLedger.qualifySql(repaired.sql).sql;
1106
+ const repairedValidation = contextLedger.validateSql(repaired.sql, {
1107
+ question,
1108
+ intent,
1109
+ filterValues: input.followUp?.filters,
1110
+ trustedFilterValues: trustedFollowUpFilterValues(input.followUp),
1111
+ });
1112
+ if (repairedValidation.ok) {
1113
+ repairAttempts += 1;
1114
+ // Adopt the corrected SQL, but do NOT let the repair prose become the answer.
1115
+ repairNarrative = repaired.text?.trim() || undefined;
1116
+ parsed.sql = repaired.sql;
1117
+ parsed.viz = repaired.viz ?? parsed.viz;
1118
+ applyParsedProposalMetadata(parsed, repaired);
1119
+ contextValidation = {
1120
+ ok: true,
1121
+ warnings: [
1122
+ ...repairedValidation.warnings,
1123
+ 'Execution-repaired SQL passed context validation.',
1124
+ ],
1125
+ };
1126
+ try {
1127
+ result = await input.executeGeneratedSql(parsed.sql);
1128
+ executionError = undefined;
1129
+ }
1130
+ catch (retryErr) {
1131
+ executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1132
+ }
627
1133
  }
628
- catch (retryErr) {
629
- executionError = retryErr instanceof Error ? retryErr.message : String(retryErr);
1134
+ else {
1135
+ executionError = repairedValidation.error;
630
1136
  }
631
1137
  }
632
1138
  }
@@ -637,26 +1143,73 @@ async function runAnswerLoop(input) {
637
1143
  question,
638
1144
  intent,
639
1145
  routeReason: catalogRoute?.reason ?? (intent === 'drillthrough'
640
- ? 'The user asked for a drill-through or follow-up, so DQL generated review-required SQL from the prior context and current metadata.'
1146
+ ? 'The user asked for a drill-through or follow-up, so DQL prepared a review-required artifact with a SQL preview from the prior context and current metadata.'
641
1147
  : 'The question asks for a custom analysis, ranking, breakdown, comparison, or grain that should not be answered by a loose certified block match.'),
642
1148
  selectedNodes: contextNodes,
643
1149
  schemaContext,
644
1150
  sql: parsed.sql,
645
1151
  suggestedViz: parsed.viz ?? 'table',
646
1152
  assumptions: [
647
- 'Generated SQL is an uncertified preview until an analyst reviews and promotes it.',
648
- ...(localProposal ? ['A local metadata planner selected a review-required SQL grain before provider generation.'] : []),
1153
+ 'The SQL preview is uncertified until an analyst reviews and promotes the DQL artifact.',
649
1154
  ...(repairNarrative ? [`Auto-corrected the query after an execution error: ${repairNarrative}`] : []),
650
1155
  ...contextValidation.warnings,
651
1156
  ...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
652
1157
  ],
653
1158
  repairAttempts,
654
1159
  });
1160
+ // Global top-N asks must return exactly N rows even when the generated SQL
1161
+ // returned more (missing/oversized LIMIT) — mirrors the certified path so a
1162
+ // "top 10" question never shows 200 rows. per_group scope is left intact by
1163
+ // trimResultToRequestedTopN. Domain-agnostic.
1164
+ let topNTrimNote;
1165
+ if (result) {
1166
+ const beforeRows = Array.isArray(result.rows) ? result.rows.length : result.rowCount;
1167
+ result = trimResultToRequestedTopN(result, questionPlan);
1168
+ const afterRows = Array.isArray(result.rows) ? result.rows.length : result.rowCount;
1169
+ if (afterRows < beforeRows) {
1170
+ topNTrimNote = `Showed the top ${questionPlan.requestedShape.topN?.n ?? afterRows} of ${beforeRows} rows the query returned.`;
1171
+ }
1172
+ }
1173
+ const resultShape = result ? validateAnswerResultShape(questionPlan, result) : undefined;
1174
+ // ANY question whose SQL executed but dropped multiple requested columns used to
1175
+ // REFUSE outright ("no governed answer"), throwing away a result that actually
1176
+ // ran. Instead, SURFACE the partial result (review-required) with a warning that
1177
+ // names the missing columns — a partial table the user can see and refine beats a
1178
+ // blank clarify. Domain-agnostic: works for every project, not just jaffle. The
1179
+ // shape gate downgrades trust rather than blocking the answer.
1180
+ const partialShapeMismatch = Boolean(resultShape && result && result.rowCount > 0
1181
+ && generatedResultShapeIsPartial(resultShape));
1182
+ const partialShapeWarning = partialShapeMismatch && resultShape
1183
+ ? `Partial answer: the result is missing ${resultShape.missingOutputs.join(', ')}`
1184
+ + `. Showing the closest table that executed — review before reuse, or narrow the question to the columns shown.`
1185
+ : undefined;
655
1186
  const validationWarnings = [
656
1187
  ...(input.contextPack?.warnings ?? []),
657
1188
  ...contextValidation.warnings,
1189
+ ...deepCandidateNotes,
1190
+ ...(resultShape?.warnings ?? []),
1191
+ ...(partialShapeWarning ? [partialShapeWarning] : []),
1192
+ ...(topNTrimNote ? [topNTrimNote] : []),
658
1193
  ...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
659
1194
  ];
1195
+ const generatedOutputs = parsed.outputs?.length ? parsed.outputs : resultColumnNames(result);
1196
+ const generatedRequestedFilters = mergeProposalStringLists(input.followUp?.filters, parsed.requestedFilters);
1197
+ const generatedRequestedDimensions = mergeProposalStringLists(input.followUp?.dimensions, parsed.requestedDimensions);
1198
+ const dqlArtifact = semanticBridgeAnswer?.dqlArtifact ?? buildGeneratedSqlDqlArtifact({
1199
+ question,
1200
+ sql: parsed.sql,
1201
+ intent,
1202
+ domain,
1203
+ followUp: input.followUp,
1204
+ contextPack: input.contextPack,
1205
+ sourceBlock: followUpSourceBlock ?? undefined,
1206
+ sourceDqlArtifact: input.followUp?.priorDqlArtifact,
1207
+ proposedEntity: parsed.proposedEntity,
1208
+ requestedFilters: generatedRequestedFilters,
1209
+ requestedDimensions: generatedRequestedDimensions,
1210
+ validationWarnings,
1211
+ outputs: generatedOutputs,
1212
+ });
660
1213
  let draftBlock;
661
1214
  let draftCaptureError;
662
1215
  if (input.captureGeneratedDraft && parsed.sql) {
@@ -668,7 +1221,13 @@ async function runAnswerLoop(input) {
668
1221
  followUp: input.followUp,
669
1222
  contextPack: input.contextPack,
670
1223
  sourceBlock: followUpSourceBlock ?? undefined,
1224
+ sourceDqlArtifact: input.followUp?.priorDqlArtifact,
1225
+ dqlArtifact,
1226
+ proposedEntity: parsed.proposedEntity,
1227
+ requestedFilters: generatedRequestedFilters,
1228
+ requestedDimensions: generatedRequestedDimensions,
671
1229
  validationWarnings,
1230
+ outputs: generatedOutputs,
672
1231
  });
673
1232
  }
674
1233
  catch (err) {
@@ -676,6 +1235,9 @@ async function runAnswerLoop(input) {
676
1235
  validationWarnings.push(`Draft capture failed: ${draftCaptureError}`);
677
1236
  }
678
1237
  }
1238
+ const answerDqlArtifact = dqlArtifact && draftBlock?.path
1239
+ ? { ...dqlArtifact, sourcePath: dqlArtifact.sourcePath ?? draftBlock.path }
1240
+ : dqlArtifact;
679
1241
  const sourceCertifiedBlock = followUpSourceBlock?.name ?? input.followUp?.sourceBlockName;
680
1242
  const trustExplanation = generatedTrustExplanation({
681
1243
  followUp: input.followUp,
@@ -683,15 +1245,20 @@ async function runAnswerLoop(input) {
683
1245
  draftBlock,
684
1246
  });
685
1247
  const cleanedSummary = cleanGeneratedSummary(parsed.text);
686
- const generatedText = trustExplanation
687
- ? [trustExplanation, cleanedSummary].filter(Boolean).join('\n\n')
688
- : cleanedSummary;
1248
+ const generatedText = [partialShapeWarning, trustExplanation, cleanedSummary]
1249
+ .filter(Boolean)
1250
+ .join('\n\n');
1251
+ const semanticMetricCertification = governedMetricAnswer && activeTier === 'semantic_layer'
1252
+ ? semanticMetricMatch?.metric.certification
1253
+ : undefined;
1254
+ const certifiedMetricAnswer = semanticMetricCertification === 'certified' || semanticMetricCertification === 'reviewed';
689
1255
  return {
690
1256
  kind: 'uncertified',
691
1257
  sourceTier: activeTier,
692
1258
  certification: 'ai_generated',
693
1259
  reviewStatus: 'draft_ready',
694
- confidence: activeTier === 'semantic_layer' ? 0.72 : 0.55,
1260
+ semanticMetricCertification,
1261
+ confidence: certifiedMetricAnswer ? 0.8 : governedMetricAnswer && activeTier === 'semantic_layer' ? 0.72 : 0.55,
695
1262
  text: generatedText,
696
1263
  answer: generatedText,
697
1264
  proposedSql: parsed.sql,
@@ -699,6 +1266,7 @@ async function runAnswerLoop(input) {
699
1266
  result,
700
1267
  executionError,
701
1268
  suggestedViz: parsed.viz ?? 'table',
1269
+ dqlArtifact: answerDqlArtifact,
702
1270
  draftBlock,
703
1271
  draftBlockId: draftBlock?.path,
704
1272
  promoteCommand: draftBlock ? `dql certify --from-draft ${draftBlock.path}` : undefined,
@@ -713,10 +1281,15 @@ async function runAnswerLoop(input) {
713
1281
  evidence: buildGeneratedEvidence({
714
1282
  question,
715
1283
  activeTier,
1284
+ terminalLane: governedMetricAnswer ? 'semantic' : 'generated',
1285
+ terminalDetail: governedMetricAnswer
1286
+ ? semanticBridgeAnswer?.metrics.join(', ') ?? semanticMetricMatch?.metric.name
1287
+ : answerDqlArtifact?.name ?? draftBlock?.path,
716
1288
  intent,
717
1289
  contextNodes,
718
1290
  schemaContext,
719
1291
  followUp: input.followUp,
1292
+ contextPack: input.contextPack,
720
1293
  businessHits,
721
1294
  semanticHits,
722
1295
  manifestHits,
@@ -727,15 +1300,42 @@ async function runAnswerLoop(input) {
727
1300
  executionError,
728
1301
  executorWasAvailable: Boolean(input.executeGeneratedSql),
729
1302
  analysisPlan,
1303
+ toolCalls: proposalToolCalls,
1304
+ budgetTrace: cascadeBudgetTrace(repairBudgetState),
730
1305
  }),
731
1306
  contextPack: input.contextPack,
732
1307
  considered,
733
- providerUsed: localProposal ? 'schema_planner' : provider.name,
1308
+ providerUsed: provider.name,
734
1309
  // Carry the governed metric match so the exit point can name a
735
1310
  // `semantic_metric` route (spec 17, part C).
736
- _semanticMetricMatch: activeTier === 'semantic_layer' ? semanticMetricMatch ?? undefined : undefined,
1311
+ _semanticMetricMatch: governedMetricAnswer && activeTier === 'semantic_layer' ? semanticMetricMatch ?? undefined : undefined,
1312
+ };
1313
+ }
1314
+ function trimResultToRequestedTopN(result, plan) {
1315
+ const topN = plan.requestedShape.topN;
1316
+ if (!topN || topN.scope === 'per_group' || !Array.isArray(result.rows) || result.rows.length <= topN.n)
1317
+ return result;
1318
+ return {
1319
+ ...result,
1320
+ rows: result.rows.slice(0, topN.n),
1321
+ rowCount: Math.min(result.rowCount, topN.n),
737
1322
  };
738
1323
  }
1324
+ function resultColumnNames(result) {
1325
+ if (!result?.columns)
1326
+ return undefined;
1327
+ const columns = result.columns
1328
+ .map((column) => {
1329
+ if (typeof column === 'string')
1330
+ return column.trim();
1331
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
1332
+ return column.name.trim();
1333
+ }
1334
+ return '';
1335
+ })
1336
+ .filter(Boolean);
1337
+ return columns?.length ? columns : undefined;
1338
+ }
739
1339
  const SYSTEM_PROMPT = `You are the DQL Analytics Agent.
740
1340
 
741
1341
  Rules:
@@ -745,16 +1345,21 @@ Rules:
745
1345
  2. Use certified DQL blocks only when the user's question exactly asks for that
746
1346
  saved block, direct KPI, or definition. For single-user/customer/account,
747
1347
  custom filters, rankings, breakdowns, comparisons, drill-throughs, or custom
748
- grains, generate review-required SQL from supplied metadata and cite
749
- certified context as evidence.
750
- 3. If you must generate SQL, return it inside a single \`\`\`sql code block.
751
- 4. Provide a one-paragraph natural-language summary BEFORE the SQL block. In it,
752
- state your QUERY PLAN first: the grain (one row per WHAT), the measures and how
753
- they aggregate, the dimensions/filters, and the exact join path + join keys
754
- between the grounded tables. Then write SQL that matches that plan — an explicit
755
- grain and join path prevents wrong-grain answers and fan-out (row-multiplying) joins.
756
- 5. Suggest a visualization type from this list, on a line starting with "Viz:":
757
- line, bar, area, pie, single_value, table, pivot, kpi.
1348
+ grains, prepare a review-required DQL artifact with a SQL preview from
1349
+ supplied metadata and cite certified context as evidence.
1350
+ 3. If you must create a SQL preview, return a single \`\`\`json fenced object with:
1351
+ {"summary": "...", "sql": "SELECT ...", "viz": "table", "outputs": ["column_a"], "dql": {"entity": "orders", "dimensions": ["region"], "filters": ["last 30 days"]}}.
1352
+ The SQL string must be one read-only SELECT/WITH statement. Do not include a
1353
+ separate SQL code fence when using JSON. The optional "dql" object is
1354
+ metadata for the deterministic draft-DQL renderer; do not hand-write a full
1355
+ DQL block.
1356
+ 4. In summary, state your QUERY PLAN first: the grain (one row per WHAT), the
1357
+ measures and how they aggregate, the dimensions/filters, and the exact join
1358
+ path + join keys between the grounded tables. Then make the SQL match that
1359
+ plan — an explicit grain and join path prevents wrong-grain answers and
1360
+ fan-out (row-multiplying) joins.
1361
+ 5. Use a visualization type from this list in the "viz" JSON field: line, bar,
1362
+ area, pie, single_value, table, pivot, kpi.
758
1363
  6. NEVER fabricate column names that are not present in the supplied schema,
759
1364
  dbt metadata, or certified source SQL shape context. If a requested filter
760
1365
  value is supplied as a matched value, prefer the table and column that
@@ -764,18 +1369,32 @@ Rules:
764
1369
  proposed SQL. Do not emit INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, COPY,
765
1370
  PRAGMA, SET, or multiple statements.
766
1371
  8. If the schema is insufficient to answer, say so explicitly and ask a
767
- clarifying question instead of guessing.
1372
+ clarifying question instead of guessing. But a MULTI-ENTITY question ("top
1373
+ customers who bought the top products", "accounts with the most overdue
1374
+ invoices") is NOT insufficient context when the grounded tables and a join
1375
+ route between them are supplied (see any "Knowledge graph join routes"
1376
+ section). In that case you MUST compose the joined SELECT that answers it
1377
+ directly — never refuse it or offer to show the parts as separate datasets.
1378
+ Reserve the clarifying question for a genuinely absent table, column, or join
1379
+ key.
768
1380
  9. Write directly to the analyst. Do not say "the user is asking", "the user
769
1381
  requested", "I will generate", or describe internal routing. State the
770
- answer, the certified context used, and the review requirement.
1382
+ answer, the certified context used, the DQL artifact expectation, and the
1383
+ review requirement.
771
1384
  10. For notebook research, SQL build or repair, DQL import/build, or DQL reuse
772
1385
  checks, start with one line in this form: "Outcome: <decision>". Use one of
773
- these decisions: Reuse certified block, Use existing draft, Generate SQL
774
- cell, Fix SQL, Create DQL draft, Needs review, Cannot answer yet.`;
775
- function renderContextPrompt(blocks, businessContext, others, activeTier, memoryContext, extraContext, followUp, schemaContext = [], intent = 'ad_hoc_analysis', contextPack) {
1386
+ these decisions: Reuse certified block, Use existing draft, Review SQL preview,
1387
+ Fix SQL, Create DQL draft, Needs review, Cannot answer yet.`;
1388
+ // Appended on a single retry when the model declined SQL for a question that the
1389
+ // grounded schema + KG join routes can actually answer. Turns a false "no
1390
+ // combined dataset — show them separately" refusal into the join the user asked
1391
+ // for, while still allowing an honest refusal if context is truly missing.
1392
+ const FORCE_JOIN_INSTRUCTION = `Your previous attempt declined to produce SQL. Re-read the supplied schema, metadata, and any "Knowledge graph join routes": this question CAN be answered by joining the grounded tables along their documented keys. Do NOT refuse, and do NOT suggest showing the datasets separately — compose ONE read-only SELECT/WITH that joins the relevant tables to answer it directly, following the JSON contract from rule 3. State the grain and the exact join path in the summary. Only if a required table, column, or join key is truly absent from the supplied context may you still ask a clarifying question.`;
1393
+ function renderContextPrompt(blocks, businessContext, others, activeTier, memoryContext, extraContext, followUp, schemaContext = [], intent = 'ad_hoc_analysis', contextPack, conversationSnapshot, kgJoinPathHints = [], budget = QUICK_PROMPT_CONTEXT_BUDGET) {
776
1394
  const intentSection = `## Routing intent\n\nintent: ${intent}\n${intent === 'exact_certified_lookup'
777
1395
  ? 'Use a certified artifact only if it exactly answers the question.'
778
- : 'Generate review-required SQL for this question. Certified blocks are trusted context, not a reason to answer the wrong grain.'}`;
1396
+ : 'Prepare a review-required DQL artifact with a SQL preview for this question. Certified blocks are trusted context, not a reason to answer the wrong grain.'}`;
1397
+ const budgetSection = `\n\n## Context budget\n\nContext budget: ${budget.label} (relations=${budget.relationCardLimit}, relation_columns=${budget.relationColumnLimit}, objects=${budget.contextObjectLimit}, edges=${budget.edgeLimit}).`;
779
1398
  const blockSection = blocks.length > 0
780
1399
  ? `## Relevant DQL blocks\n\n${blocks
781
1400
  .map((b) => `- \`${b.nodeId}\` (${b.domain ?? 'unscoped'}, ${b.status ?? b.certification ?? 'review_required'}): ${b.description ?? b.llmContext ?? '(no description)'}`)
@@ -791,12 +1410,15 @@ function renderContextPrompt(blocks, businessContext, others, activeTier, memory
791
1410
  .map((n) => `- ${n.kind} \`${n.name}\`${n.domain ? ` (domain: ${n.domain})` : ''}${n.description ? ` — ${n.description}` : ''}${n.llmContext ? `\n ${n.llmContext.replace(/\n/g, '\n ')}` : ''}`)
792
1411
  .join('\n')}`
793
1412
  : '';
1413
+ const kgJoinSection = kgJoinPathHints.length > 0
1414
+ ? `\n\n## Knowledge graph join routes\n\nUse these as relationship/navigation evidence for multi-entity analysis. SQL must still use inspected relation columns from the schema or metadata context.\n${kgJoinPathHints.map((hint) => `- ${hint}`).join('\n')}`
1415
+ : '';
794
1416
  const schemaSection = schemaContext.length > 0
795
1417
  ? `\n\n## Runtime schema context\n\nUse only these runtime relations and columns when generating SQL unless the dbt manifest context gives an equivalent relation.\n${schemaContext
796
- .slice(0, 12)
1418
+ .slice(0, budget.schemaTableLimit)
797
1419
  .map((table) => {
798
1420
  const cols = table.columns
799
- .slice(0, 50)
1421
+ .slice(0, budget.schemaColumnLimit)
800
1422
  .map((col) => {
801
1423
  const sampleValues = col.sampleValues?.length
802
1424
  ? `; matched values: ${col.sampleValues.slice(0, 4).map(formatPromptValue).join(', ')}`
@@ -816,27 +1438,118 @@ function renderContextPrompt(blocks, businessContext, others, activeTier, memory
816
1438
  : '';
817
1439
  // Approved, scope-matched correction hints from the Hint Graph. These are
818
1440
  // human-reviewed lessons (often from a prior wrong→right SQL correction), so
819
- // they carry more weight than advisory memory when generating SQL — but they
1441
+ // they carry more weight than advisory memory when preparing a SQL preview — but they
820
1442
  // still must NOT override a certified artifact that already answers exactly.
821
1443
  const appliedHints = contextPack?.appliedHints ?? [];
822
1444
  const hintsSection = appliedHints.length > 0
823
- ? `\n\n## Applied governed hints (human-approved corrections)\n\nReviewed, scope-matched corrections from your team. Apply them when generating SQL to avoid known mistakes; they refine generated SQL but MUST NOT override a certified artifact that already answers the question.\n${appliedHints
1445
+ ? `\n\n## Applied governed hints (human-approved corrections)\n\nReviewed, scope-matched corrections from your team. Apply them when preparing a SQL preview to avoid known mistakes; they refine the DQL artifact's SQL preview but MUST NOT override a certified artifact that already answers the question.\n${appliedHints
824
1446
  .slice(0, 6)
825
1447
  .map((h) => `- \`${h.title}\`: ${h.guidance}${h.correctedSql ? `\n corrected SQL pattern: ${h.correctedSql.replace(/\s+/g, ' ').trim().slice(0, 240)}` : ''}`)
826
1448
  .join('\n')}`
827
1449
  : '';
828
1450
  const extraSection = extraContext?.trim()
829
- ? `\n\n## Current notebook/app context\n\nThis context may help interpret the user's request, but it MUST NOT override certified artifacts, semantic metrics, dbt metadata, or generated SQL validation.\n\n${extraContext.trim()}`
1451
+ ? `\n\n## Current notebook/app context\n\nThis context may help interpret the user's request, but it MUST NOT override certified artifacts, semantic metrics, dbt metadata, or SQL preview validation.\n\n${extraContext.trim()}`
830
1452
  : '';
831
1453
  const followUpSection = followUp
832
1454
  ? `\n\n## Follow-up routing context\n\n${renderFollowUpContext(followUp)}`
833
1455
  : '';
1456
+ const conversationSection = conversationSnapshot
1457
+ ? `\n\n## Conversation state (session)\n\n${renderConversationSnapshot(conversationSnapshot)}`
1458
+ : '';
834
1459
  const contextPackSection = contextPack
835
- ? `\n\n## Local metadata context pack\n\n${renderContextPackForPrompt(contextPack)}`
1460
+ ? `\n\n## Local metadata context pack\n\n${renderContextPackForPrompt(contextPack, budget)}`
836
1461
  : '';
837
- return `${intentSection}\n\n${blockSection}${businessSection}${otherSection}${schemaSection}${contextPackSection}${memorySection}${hintsSection}${extraSection}${followUpSection}`;
1462
+ return `${intentSection}${budgetSection}\n\n${blockSection}${businessSection}${otherSection}${kgJoinSection}${schemaSection}${contextPackSection}${memorySection}${hintsSection}${conversationSection}${extraSection}${followUpSection}`;
838
1463
  }
839
- function renderContextPackForPrompt(contextPack) {
1464
+ function buildKgJoinPathHints(kg, nodes, questionPlan) {
1465
+ const candidates = nodes
1466
+ .filter((node) => ['block', 'business_view', 'metric', 'entity', 'semantic_model', 'dbt_model', 'dbt_source'].includes(node.kind))
1467
+ .slice(0, 8);
1468
+ if (candidates.length < 2)
1469
+ return [];
1470
+ const domains = new Set(candidates.map((node) => node.domain).filter((domain) => Boolean(domain)));
1471
+ const shouldSearch = domains.size > 1 || questionPlan.entities.length > 1 || questionPlan.dimensionTerms.length > 1;
1472
+ if (!shouldSearch)
1473
+ return [];
1474
+ const hints = [];
1475
+ const seen = new Set();
1476
+ for (let leftIndex = 0; leftIndex < candidates.length; leftIndex += 1) {
1477
+ for (let rightIndex = leftIndex + 1; rightIndex < candidates.length; rightIndex += 1) {
1478
+ if (hints.length >= 6)
1479
+ return hints;
1480
+ const left = candidates[leftIndex];
1481
+ const right = candidates[rightIndex];
1482
+ const path = kg.findJoinPath(left.nodeId, right.nodeId, 4);
1483
+ if (!path || path.length < 2)
1484
+ continue;
1485
+ const key = path.join('>');
1486
+ if (seen.has(key))
1487
+ continue;
1488
+ seen.add(key);
1489
+ hints.push(path.map((nodeId) => formatKgPathNode(kg, nodeId)).join(' -> '));
1490
+ }
1491
+ }
1492
+ return hints;
1493
+ }
1494
+ function formatKgPathNode(kg, nodeId) {
1495
+ const node = kg.getNode(nodeId);
1496
+ if (!node)
1497
+ return nodeId;
1498
+ return `${node.kind}:${node.name}`;
1499
+ }
1500
+ /**
1501
+ * Bounded rendering of the persisted conversation snapshot: working state one-liner
1502
+ * (≤300 chars), compacted rolling summary (≤600), and up to 4 recent turns (≤280 each).
1503
+ */
1504
+ function renderConversationSnapshot(snapshot) {
1505
+ const parts = [];
1506
+ const state = snapshot.workingState;
1507
+ if (state) {
1508
+ const stateLine = [
1509
+ state.entities.length ? `entities=${state.entities.join(',')}` : '',
1510
+ state.measures.length ? `measures=${state.measures.join(',')}` : '',
1511
+ state.dimensions.length ? `dimensions=${state.dimensions.join(',')}` : '',
1512
+ state.filters.length ? `filters=${state.filters.map((f) => f.value).join(',')}` : '',
1513
+ state.limit ? `limit=${state.limit}` : '',
1514
+ state.timeframe ? `timeframe=${state.timeframe}` : '',
1515
+ snapshot.topicRelation ? `topic=${snapshot.topicRelation}` : '',
1516
+ ].filter(Boolean).join('; ');
1517
+ if (stateLine)
1518
+ parts.push(`Working state: ${stateLine.slice(0, 300)}`);
1519
+ }
1520
+ else if (snapshot.topicRelation) {
1521
+ parts.push(`Working state: topic=${snapshot.topicRelation}`);
1522
+ }
1523
+ if (snapshot.rollingSummary) {
1524
+ parts.push(`Earlier in this conversation (compacted):\n${snapshot.rollingSummary.slice(0, 600)}`);
1525
+ }
1526
+ if (snapshot.recentTurns.length > 0) {
1527
+ const turns = snapshot.recentTurns.slice(-4).map((turn, index) => {
1528
+ return `${index + 1}. ${renderConversationSnapshotTurnLine(turn, 520)}`;
1529
+ });
1530
+ parts.push(`Recent turns:\n${turns.join('\n')}`);
1531
+ }
1532
+ if (snapshot.recalledTurns?.length) {
1533
+ parts.push(`Recalled earlier turns (semantic match):\n${snapshot.recalledTurns.slice(0, 3)
1534
+ .map((turn) => `- ${renderConversationSnapshotTurnLine(turn, 520)}`)
1535
+ .join('\n')}`);
1536
+ }
1537
+ parts.push('Use this only where the question refers to it. On a new topic, answer fresh.');
1538
+ return parts.join('\n\n');
1539
+ }
1540
+ function renderConversationSnapshotTurnLine(turn, max) {
1541
+ const line = [
1542
+ `Q: ${turn.question}`,
1543
+ turn.answerSummary ? `A: ${turn.answerSummary}` : '',
1544
+ turn.resultColumns?.length ? `cols: ${turn.resultColumns.slice(0, 6).join(', ')}` : '',
1545
+ typeof turn.resultRowCount === 'number' ? `rows: ${turn.resultRowCount}` : '',
1546
+ turn.sourceSql ? `sql: ${compactSqlSnippet(turn.sourceSql, 180)}` : '',
1547
+ turn.dqlArtifact?.name ? `dql: ${turn.dqlArtifact.name}` : '',
1548
+ turn.sourceCertifiedBlock ? `block: ${turn.sourceCertifiedBlock}` : '',
1549
+ ].filter(Boolean).join(' | ');
1550
+ return line.slice(0, max);
1551
+ }
1552
+ function renderContextPackForPrompt(contextPack, budget) {
840
1553
  const questionPlan = contextPack.questionPlan
841
1554
  ? [
842
1555
  `Question plan: ${contextPack.questionPlan.mode} -> ${contextPack.questionPlan.routeIntent}`,
@@ -851,10 +1564,13 @@ function renderContextPackForPrompt(contextPack) {
851
1564
  const certifiedApplicability = contextPack.routeDecision?.certifiedApplicability
852
1565
  ? `\nCertified applicability: ${contextPack.routeDecision.certifiedApplicability.name} is ${contextPack.routeDecision.certifiedApplicability.kind} (${contextPack.routeDecision.certifiedApplicability.reasons.join('; ')})`
853
1566
  : '';
1567
+ const blockFit = contextPack.routeDecision?.blockFit
1568
+ ? `\nCertified block fit: ${contextPack.routeDecision.blockFit.kind} / ${contextPack.routeDecision.blockFit.confidence} (${contextPack.routeDecision.blockFit.reasons.join('; ')})`
1569
+ : '';
854
1570
  const warnings = contextPack.warnings.length
855
- ? `Warnings:\n${contextPack.warnings.slice(0, 8).map((warning) => `- ${warning}`).join('\n')}\n`
1571
+ ? `Warnings:\n${contextPack.warnings.slice(0, budget.warningLimit).map((warning) => `- ${warning}`).join('\n')}\n`
856
1572
  : '';
857
- const objects = contextPack.objects.slice(0, 18).map((object) => {
1573
+ const objects = contextPack.objects.slice(0, budget.contextObjectLimit).map((object) => {
858
1574
  const detail = [
859
1575
  object.objectType,
860
1576
  object.domain ? `domain: ${object.domain}` : '',
@@ -867,38 +1583,43 @@ function renderContextPackForPrompt(contextPack) {
867
1583
  ? `\nCandidate conflicts:\n${contextPack.retrievalDiagnostics.candidateConflicts.slice(0, 4).map((conflict) => `- ${conflict.reason} ${conflict.prompt}`).join('\n')}`
868
1584
  : '';
869
1585
  const route = contextPack.routeDecision
870
- ? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}${certifiedApplicability}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
1586
+ ? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}${certifiedApplicability}${blockFit}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
871
1587
  : '';
872
- const allowed = renderAllowedSqlRelationsForPrompt(contextPack);
873
- const joins = renderCandidateJoinsForPrompt(contextPack);
1588
+ const allowed = renderAllowedSqlRelationsForPrompt(contextPack, budget);
1589
+ const joins = renderCandidateJoinsForPrompt(contextPack, budget);
1590
+ const lineage = renderColumnLineageForPrompt(contextPack, budget);
1591
+ const edges = renderContextEdgesForPrompt(contextPack, budget);
874
1592
  const relationDiagnostics = contextPack.retrievalDiagnostics?.selectedRelations?.length
875
- ? `\nSelected relation reasoning:\n${contextPack.retrievalDiagnostics.selectedRelations.slice(0, 8).map((relation) => `- ${relation.relation} (score ${relation.score.toFixed(1)}): ${relation.reason}`).join('\n')}`
1593
+ ? `\nSelected relation reasoning:\n${contextPack.retrievalDiagnostics.selectedRelations.slice(0, budget.selectedRelationReasonLimit).map((relation) => `- ${relation.relation} (score ${relation.score.toFixed(1)}): ${relation.reason}`).join('\n')}`
876
1594
  : '';
877
- const sourceSql = renderSourceBlockSqlContext(contextPack);
1595
+ const sourceSql = renderSourceBlockSqlContext(contextPack, budget);
878
1596
  return [
879
1597
  `context_pack_id: ${contextPack.id}`,
880
1598
  `trust_label: ${contextPack.trustLabel}`,
1599
+ contextPack.trustLabelInfo ? `trust_label_canonical: ${contextPack.trustLabelInfo.display}` : '',
881
1600
  questionPlan.trim(),
882
1601
  route.trim(),
883
1602
  warnings.trim(),
884
1603
  `Selected evidence:\n${objects || '- none'}`,
885
1604
  allowed.trim(),
886
1605
  joins.trim(),
1606
+ edges.trim(),
1607
+ lineage.trim(),
887
1608
  relationDiagnostics.trim(),
888
1609
  sourceSql.trim(),
889
1610
  conflicts.trim(),
890
1611
  ].filter(Boolean).join('\n');
891
1612
  }
892
- function renderCandidateJoinsForPrompt(contextPack) {
1613
+ function renderCandidateJoinsForPrompt(contextPack, budget) {
893
1614
  const joins = contextPack.retrievalDiagnostics.selectedJoinPaths?.length
894
- ? contextPack.retrievalDiagnostics.selectedJoinPaths.slice(0, 8).map((join) => ({
1615
+ ? contextPack.retrievalDiagnostics.selectedJoinPaths.slice(0, budget.joinPathLimit).map((join) => ({
895
1616
  leftRelation: join.leftRelation,
896
1617
  leftColumn: join.leftColumn,
897
1618
  rightRelation: join.rightRelation,
898
1619
  rightColumn: join.rightColumn,
899
1620
  reason: join.reason,
900
1621
  }))
901
- : buildCandidateJoinPaths(schemaContextWithAllowedSqlContext([], contextPack)).slice(0, 8);
1622
+ : buildCandidateJoinPaths(schemaContextWithAllowedSqlContext([], contextPack)).slice(0, budget.joinPathLimit);
902
1623
  if (joins.length === 0)
903
1624
  return '';
904
1625
  return [
@@ -906,18 +1627,59 @@ function renderCandidateJoinsForPrompt(contextPack) {
906
1627
  ...joins.map((join) => `- ${join.leftRelation}.${join.leftColumn} -> ${join.rightRelation}.${join.rightColumn}${join.reason ? ` (${join.reason})` : ''}`),
907
1628
  ].join('\n');
908
1629
  }
909
- function renderAllowedSqlRelationsForPrompt(contextPack) {
1630
+ function renderContextEdgesForPrompt(contextPack, budget) {
1631
+ if (budget.edgeLimit <= 0)
1632
+ return '';
1633
+ const edges = contextPack.edges.slice(0, budget.edgeLimit);
1634
+ if (edges.length === 0)
1635
+ return '';
1636
+ return [
1637
+ 'Context graph edges:',
1638
+ 'Use these object relationships as metadata evidence; SQL still must use inspected relation columns and join keys.',
1639
+ ...edges.map((edge) => {
1640
+ const confidence = typeof edge.confidence === 'number' ? ` (confidence ${edge.confidence.toFixed(2)})` : '';
1641
+ return `- ${edge.edgeType}: ${edge.fromKey} -> ${edge.toKey}${confidence}`;
1642
+ }),
1643
+ ].join('\n');
1644
+ }
1645
+ function renderColumnLineageForPrompt(contextPack, budget) {
1646
+ const lineageEdges = contextPack.edges
1647
+ .filter((edge) => edge.edgeType === 'derives_from')
1648
+ .slice(0, budget.lineageEdgeLimit);
1649
+ if (lineageEdges.length === 0)
1650
+ return '';
1651
+ const objectsByKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
1652
+ return [
1653
+ 'Column lineage from governed metadata:',
1654
+ 'Use this when adapting certified blocks or dbt models into a DQL artifact SQL preview so derived output columns map back to their physical source columns.',
1655
+ ...lineageEdges.map((edge) => {
1656
+ const from = objectsByKey.get(edge.fromKey);
1657
+ const to = objectsByKey.get(edge.toKey);
1658
+ const aggregateFn = typeof edge.payload?.aggregateFn === 'string' && edge.payload.aggregateFn.trim()
1659
+ ? ` via ${edge.payload.aggregateFn.trim().toUpperCase()}`
1660
+ : '';
1661
+ const confidence = typeof edge.confidence === 'number' ? `, confidence ${edge.confidence.toFixed(2)}` : '';
1662
+ return `- ${formatLineageObjectForPrompt(from, edge.fromKey)} derives from ${formatLineageObjectForPrompt(to, edge.toKey)}${aggregateFn}${confidence}`;
1663
+ }),
1664
+ ].join('\n');
1665
+ }
1666
+ function formatLineageObjectForPrompt(object, fallbackKey) {
1667
+ if (!object)
1668
+ return fallbackKey;
1669
+ return object.fullName ?? object.name ?? object.objectKey;
1670
+ }
1671
+ function renderAllowedSqlRelationsForPrompt(contextPack, budget) {
910
1672
  const relations = contextPack.allowedSqlContext?.relations ?? [];
911
1673
  if (relations.length === 0)
912
1674
  return '';
913
1675
  const selectedLookup = selectedRelationLookup(contextPack);
914
- const cards = relations.slice(0, 12).map((relation, index) => {
1676
+ const cards = relations.slice(0, budget.relationCardLimit).map((relation, index) => {
915
1677
  const selection = relationSelectionFor(relation.relation, selectedLookup);
916
1678
  const rank = selection?.rank ?? index + 1;
917
1679
  const score = typeof selection?.score === 'number' ? `, score ${selection.score.toFixed(1)}` : '';
918
1680
  const reason = selection?.reason ? `\n why selected: ${selection.reason}` : '';
919
1681
  const columns = relation.columns.length
920
- ? relation.columns.slice(0, 32).map(formatRelationColumnForPrompt).join(', ')
1682
+ ? relation.columns.slice(0, budget.relationColumnLimit).map(formatRelationColumnForPrompt).join(', ')
921
1683
  : '(columns unavailable; use certified source SQL shape or inspect metadata before inventing columns)';
922
1684
  return [
923
1685
  `- [rank ${rank}${score}] ${relation.relation} (${relation.source})`,
@@ -925,11 +1687,37 @@ function renderAllowedSqlRelationsForPrompt(contextPack) {
925
1687
  `\n columns: ${columns}`,
926
1688
  ].join('');
927
1689
  });
1690
+ const otherRelations = renderOtherAvailableRelationsForPrompt(contextPack, budget);
928
1691
  return [
929
1692
  'Selected SQL relation context:',
930
1693
  'Use these ranked relations and columns as the primary SQL-generation boundary. Prefer lower rank when multiple relations look plausible.',
931
1694
  ...cards,
932
- ].join('\n');
1695
+ otherRelations,
1696
+ ].filter(Boolean).join('\n');
1697
+ }
1698
+ function renderOtherAvailableRelationsForPrompt(contextPack, budget) {
1699
+ const names = new Map();
1700
+ for (const relation of (contextPack.allowedSqlContext?.relations ?? []).slice(budget.otherRelationStart, budget.otherRelationEnd)) {
1701
+ names.set(normalizePromptRelationName(relation.relation), relation.relation);
1702
+ }
1703
+ for (const item of contextPack.retrievalDiagnostics.topRejected ?? []) {
1704
+ if (!isRelationLikeRejectedObject(item.objectType))
1705
+ continue;
1706
+ names.set(normalizePromptRelationName(item.name), item.name);
1707
+ }
1708
+ const values = [...names.values()].slice(0, budget.otherRelationLimit);
1709
+ if (values.length === 0)
1710
+ return '';
1711
+ return `Other available relations (names only - expand context before using columns): ${values.join(', ')}`;
1712
+ }
1713
+ function isRelationLikeRejectedObject(objectType) {
1714
+ return objectType === 'dbt_model'
1715
+ || objectType === 'dbt_source'
1716
+ || objectType === 'warehouse_table'
1717
+ || objectType === 'runtime_table';
1718
+ }
1719
+ function normalizePromptRelationName(name) {
1720
+ return name.toLowerCase().replace(/["`]/g, '').trim();
933
1721
  }
934
1722
  function formatRelationColumnForPrompt(column) {
935
1723
  const type = column.type ? ` ${column.type}` : '';
@@ -939,17 +1727,17 @@ function formatRelationColumnForPrompt(column) {
939
1727
  : '';
940
1728
  return `${column.name}${type}${description}${samples}`;
941
1729
  }
942
- function renderSourceBlockSqlContext(contextPack) {
1730
+ function renderSourceBlockSqlContext(contextPack, budget) {
943
1731
  const sources = contextPack.allowedSqlContext?.sourceBlockSql ?? [];
944
1732
  if (sources.length === 0)
945
1733
  return '';
946
- const lines = sources.slice(0, 5).map((source) => {
1734
+ const lines = sources.slice(0, budget.sourceSqlLimit).map((source) => {
947
1735
  const shape = extractSimpleSelectShape(source.sql);
948
1736
  const projectedColumns = shape
949
1737
  ? shape.selectExpressions
950
1738
  .map(selectExpressionOutputName)
951
1739
  .filter((value) => Boolean(value))
952
- .slice(0, 24)
1740
+ .slice(0, budget.sourceSqlColumnLimit)
953
1741
  .join(', ')
954
1742
  : '';
955
1743
  const snippet = compactSqlSnippet(source.sql, 280);
@@ -965,7 +1753,7 @@ function renderSourceBlockSqlContext(contextPack) {
965
1753
  snippet ? ` sql: ${snippet}` : '',
966
1754
  ].filter(Boolean).join('\n');
967
1755
  });
968
- return `## Worked examples from certified blocks (few-shot patterns)\n\nThese certified blocks already answer similar questions. Learn their join paths, grain, and filters and ADAPT them to the question — do not copy blindly, and do not relabel generated SQL as certified.\n${lines.join('\n')}`;
1756
+ return `## Worked examples from certified blocks (few-shot patterns)\n\nThese certified blocks already answer similar questions. Learn their join paths, grain, and filters and ADAPT them to the question — do not copy blindly, and do not relabel a review-required DQL artifact or SQL preview as certified.\n${lines.join('\n')}`;
969
1757
  }
970
1758
  function contextPackCitations(contextPack, limit) {
971
1759
  if (!contextPack)
@@ -991,6 +1779,8 @@ function shouldUseCertifiedRoute(route, intent) {
991
1779
  return intent === 'exact_certified_lookup' || intent === 'definition_lookup';
992
1780
  }
993
1781
  function certifiedHitFromContextPack(contextPack, kg) {
1782
+ if (contextPack?.routeDecision.blockFit && !certifiedFitAllowsTier1(contextPack.routeDecision.blockFit))
1783
+ return null;
994
1784
  const key = contextPack?.routeDecision.exactObjectKey;
995
1785
  if (!key)
996
1786
  return null;
@@ -1039,16 +1829,6 @@ function sourceTierFromContextPack(contextPack) {
1039
1829
  return 'certified_artifact';
1040
1830
  return undefined;
1041
1831
  }
1042
- function isGeneratedAgentIntent(intent) {
1043
- return intent === 'ad_hoc_analysis'
1044
- || intent === 'drillthrough'
1045
- || intent === 'ad_hoc_ranking'
1046
- || intent === 'driver_breakdown'
1047
- || intent === 'diagnose_change'
1048
- || intent === 'segment_compare'
1049
- || intent === 'entity_drilldown'
1050
- || intent === 'anomaly_investigation';
1051
- }
1052
1832
  function metadataObjectKindForCitation(objectType) {
1053
1833
  if (objectType === 'dql_block')
1054
1834
  return 'block';
@@ -1091,12 +1871,51 @@ function renderFollowUpContext(followUp) {
1091
1871
  followUp.sourceAnswer ? `source answer: ${followUp.sourceAnswer.slice(0, 700)}` : '',
1092
1872
  followUp.filters?.length ? `requested filters: ${followUp.filters.join(', ')}` : '',
1093
1873
  followUp.dimensions?.length ? `requested dimensions: ${followUp.dimensions.join(', ')}` : '',
1874
+ followUp.priorResultColumns?.length ? `prior result columns: ${followUp.priorResultColumns.join(', ')}` : '',
1875
+ followUp.priorResultValues ? `prior result values: ${formatPriorResultValues(followUp.priorResultValues)}` : '',
1876
+ followUp.priorResultRef ? renderPriorResultReference(followUp.priorResultRef) : '',
1877
+ followUp.priorDqlArtifact ? renderPriorDqlArtifactReference(followUp.priorDqlArtifact) : '',
1878
+ followUp.priorLimit ? `prior result limit: ${followUp.priorLimit}` : '',
1094
1879
  ].filter(Boolean);
1095
1880
  const rule = followUp.kind === 'drilldown'
1096
- ? 'routing rule: find a distinct certified drilldown block first; if none exists, generate review-required SQL as a draft drilldown. Do not silently re-run the source block unless it explicitly supports the requested filter or dimension.'
1097
- : 'routing rule: reuse the prior certified block when the user asks a generic follow-up.';
1881
+ ? 'routing rule: find a distinct certified drilldown block first; if none exists, prepare a review-required DQL artifact with a SQL preview as a draft drilldown. Do not silently re-run the source block unless it explicitly supports the requested filter or dimension.'
1882
+ : followUp.kind === 'contextual'
1883
+ ? 'routing rule: this is prior-turn context (advisory). The user may be continuing from it — resolve references like "these"/"those"/"same" against it when the question refers to it. If this question starts a genuinely new topic, ignore the prior-turn context entirely and answer fresh.'
1884
+ : 'routing rule: reuse the prior certified block when the user asks a generic follow-up.';
1098
1885
  return [...parts, rule].join('\n');
1099
1886
  }
1887
+ function renderPriorResultReference(ref) {
1888
+ const lines = [
1889
+ `prior result ref: result:${ref.id}`,
1890
+ ref.question ? `prior result question: ${ref.question.slice(0, 220)}` : '',
1891
+ ref.columns.length ? `prior result schema: ${ref.columns.slice(0, 24).join(', ')}` : '',
1892
+ typeof ref.rowCount === 'number' ? `prior result row count: ${ref.rowCount}` : '',
1893
+ ref.sourceSql ? `prior result source SQL: ${compactSqlSnippet(ref.sourceSql, 500)}` : '',
1894
+ ].filter(Boolean);
1895
+ return lines.join('\n');
1896
+ }
1897
+ function renderPriorDqlArtifactReference(artifact) {
1898
+ const lines = [
1899
+ `prior DQL artifact kind: ${artifact.kind}`,
1900
+ artifact.name ? `prior DQL artifact name: ${artifact.name}` : '',
1901
+ artifact.sourcePath ? `prior DQL artifact path: ${artifact.sourcePath}` : '',
1902
+ artifact.metrics?.length ? `prior DQL metrics: ${artifact.metrics.slice(0, 12).join(', ')}` : '',
1903
+ artifact.dimensions?.length ? `prior DQL dimensions: ${artifact.dimensions.slice(0, 12).join(', ')}` : '',
1904
+ artifact.filters?.length ? `prior DQL filters: ${artifact.filters.slice(0, 8).map(formatDqlArtifactFilter).join('; ')}` : '',
1905
+ artifact.timeDimension ? `prior DQL time: ${artifact.timeDimension.name} / ${artifact.timeDimension.granularity}` : '',
1906
+ artifact.source.trim() ? `prior DQL artifact source:\n${compactSqlSnippet(artifact.source, 1400)}` : '',
1907
+ ].filter(Boolean);
1908
+ return lines.join('\n');
1909
+ }
1910
+ function formatDqlArtifactFilter(filter) {
1911
+ return `${filter.dimension} ${filter.operator} ${filter.values.slice(0, 8).join(', ')}`;
1912
+ }
1913
+ function formatPriorResultValues(values) {
1914
+ return Object.entries(values)
1915
+ .slice(0, 8)
1916
+ .map(([key, vals]) => `${key}=[${vals.slice(0, 12).join(', ')}]`)
1917
+ .join('; ');
1918
+ }
1100
1919
  function generatedTrustExplanation(input) {
1101
1920
  if (input.followUp?.kind !== 'drilldown')
1102
1921
  return undefined;
@@ -1109,295 +1928,232 @@ function generatedTrustExplanation(input) {
1109
1928
  ];
1110
1929
  const grain = filters.length ? ` at the requested ${filters.join('/')} grain` : ' at the requested drilldown grain';
1111
1930
  const draft = input.draftBlock
1112
- ? ` The draft was saved at \`${input.draftBlock.path}\` for review.`
1113
- : ' The generated SQL still needs analyst review before certification.';
1114
- return `This is an uncertified drilldown.${source} then generated new SQL${grain}.${draft}`;
1931
+ ? ` The DQL draft was saved at \`${input.draftBlock.path}\` for review.`
1932
+ : ' The review-required DQL artifact and SQL preview still need analyst review before certification.';
1933
+ return `This is an uncertified drilldown.${source} then prepared a SQL preview${grain}.${draft}`;
1115
1934
  }
1116
1935
  function cleanGeneratedSummary(text) {
1117
1936
  return text
1118
1937
  .trim()
1119
1938
  .replace(/^(?:the user (?:is asking|asked|wants|requested)[^.]*\.\s*)+/i, '')
1120
1939
  .replace(/\s*(?:therefore,\s*)?i will generate review-required sql[^.]*\.\s*/gi, ' ')
1940
+ .replace(/\s*(?:therefore,\s*)?i will prepare (?:a )?review-required dql artifact[^.]*\.\s*/gi, ' ')
1121
1941
  .replace(/\s*(?:therefore,\s*)?i will generate[^.]*\.\s*/gi, ' ')
1122
1942
  .trim();
1123
1943
  }
1124
1944
  /**
1125
- * Public for tests. Pulls the first ```sql block and an optional Viz: line
1126
- * out of an LLM response.
1945
+ * Public for tests. Prefer the structured W2.7 JSON proposal contract, then
1946
+ * fall back to the legacy prose + ```sql block + Viz line format.
1127
1947
  */
1128
1948
  export function parseProposal(raw) {
1949
+ const structured = parseStructuredProposal(raw);
1950
+ if (structured)
1951
+ return structured;
1129
1952
  const sqlMatch = raw.match(/```sql\s*([\s\S]*?)```/i);
1130
1953
  const sql = sqlMatch ? sqlMatch[1].trim() : undefined;
1131
1954
  const vizMatch = raw.match(/^Viz:\s*([a-z_]+)/im);
1132
1955
  const viz = vizMatch ? vizMatch[1].trim().toLowerCase() : undefined;
1133
1956
  // Strip the SQL block + Viz line from the prose to keep the summary clean.
1134
1957
  const text = raw
1958
+ .replace(/```json[\s\S]*?```/gi, '')
1135
1959
  .replace(/```sql[\s\S]*?```/gi, '')
1136
1960
  .replace(/^Viz:.*$/gim, '')
1137
1961
  .trim();
1138
1962
  return { text, sql, viz };
1139
1963
  }
1140
- function buildSchemaAwareProposal(input) {
1141
- if (!isGeneratedAgentIntent(input.intent))
1142
- return undefined;
1143
- const schemaContext = schemaContextWithAllowedSqlContext(input.schemaContext, input.contextPack);
1144
- const drilldownProposal = buildMatchedEntityDrilldownProposal({ ...input, schemaContext });
1145
- if (drilldownProposal)
1146
- return drilldownProposal;
1147
- const profileProposal = buildEntityProfileProposal({ ...input, schemaContext });
1148
- if (profileProposal)
1149
- return profileProposal;
1150
- if (isFilteredEntityQuestion(input.question))
1151
- return undefined;
1152
- const lower = input.question.toLowerCase();
1153
- const asksForCustomerPerformance = /\bcustomers?\b/.test(lower)
1154
- && /\border|orders|spend|revenue|perform|performed|better|top|best|rank|ranking|bottom|least|fewest|lowest|less|worst|underperform/.test(lower)
1155
- && !/\b(order details|specific orders|each order|all orders|order line|line item)\b/.test(lower);
1156
- if (asksForCustomerPerformance) {
1157
- const direction = customerRankingDirectionFromText(input.question);
1158
- const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
1159
- const orderFocused = /\border|orders|order count|ordered\b/.test(lower)
1160
- && !/\b(spend|revenue|sales|amount|lifetime spend|value)\b/.test(lower);
1161
- const customers = findSchemaTable(schemaContext, ['customers', 'customer']);
1162
- if (customers) {
1163
- const customerName = findSchemaColumn(customers, ['customer_name', 'name', 'full_name']);
1164
- const orderCount = findSchemaColumn(customers, ['count_lifetime_orders', 'lifetime_orders', 'order_count', 'orders_count', 'orders']);
1165
- const spend = findSchemaColumn(customers, ['lifetime_spend', 'total_lifetime_spend', 'customer_lifetime_value', 'total_revenue', 'revenue']);
1166
- if (customerName && orderCount && spend) {
1167
- const primarySort = orderFocused ? orderCount : spend;
1168
- const secondarySort = orderFocused ? spend : orderCount;
1169
- const rankingLabel = direction === 'bottom'
1170
- ? orderFocused ? 'Customers with the fewest orders' : 'Lowest performing customers'
1171
- : orderFocused ? 'Top customers by order count' : 'Top performing customers';
1172
- return {
1173
- text: `${rankingLabel}, with ${businessMeasurePhrase(orderCount)} and ${businessMeasurePhrase(spend)} for context. This is AI-generated and needs analyst review before certification.`,
1174
- sql: [
1175
- 'SELECT',
1176
- ` ${sqlIdentifier(customerName)} AS customer_name,`,
1177
- ` ${sqlIdentifier(orderCount)} AS orders,`,
1178
- ` ROUND(${sqlIdentifier(spend)}, 2) AS lifetime_spend`,
1179
- `FROM ${sqlRelation(customers.relation)}`,
1180
- `ORDER BY ${sqlIdentifier(primarySort)} ${orderDirection}, ${sqlIdentifier(secondarySort)} ${orderDirection}`,
1181
- 'LIMIT 10',
1182
- ].join('\n'),
1183
- viz: 'table',
1184
- };
1185
- }
1186
- const customerId = findSchemaColumn(customers, ['customer_id', 'id']);
1187
- const orders = findSchemaTable(schemaContext, ['orders', 'order']);
1188
- const orderCustomerId = orders ? findSchemaColumn(orders, ['customer_id', 'customer']) : undefined;
1189
- const orderTotal = orders ? findSchemaColumn(orders, ['order_total', 'total_order_amount', 'total_amount', 'amount', 'subtotal']) : undefined;
1190
- const orderId = orders ? findSchemaColumn(orders, ['order_id', 'id']) : undefined;
1191
- if (orders && customerName && customerId && orderCustomerId && orderTotal) {
1192
- const countExpression = orderId ? `COUNT(DISTINCT o.${sqlIdentifier(orderId)})` : 'COUNT(*)';
1193
- const primarySort = orderFocused ? 'orders' : 'lifetime_spend';
1194
- const secondarySort = orderFocused ? 'lifetime_spend' : 'orders';
1195
- const rankingLabel = direction === 'bottom'
1196
- ? orderFocused ? 'Customers with the fewest orders' : 'Lowest performing customers'
1197
- : orderFocused ? 'Top customers by order count' : 'Top performing customers';
1198
- return {
1199
- text: `${rankingLabel} from order totals, with order count for context. This is AI-generated and needs analyst review before certification.`,
1200
- sql: [
1201
- 'SELECT',
1202
- ` c.${sqlIdentifier(customerName)} AS customer_name,`,
1203
- ` ${countExpression} AS orders,`,
1204
- ` ROUND(SUM(o.${sqlIdentifier(orderTotal)}), 2) AS lifetime_spend`,
1205
- `FROM ${sqlRelation(orders.relation)} AS o`,
1206
- `JOIN ${sqlRelation(customers.relation)} AS c ON o.${sqlIdentifier(orderCustomerId)} = c.${sqlIdentifier(customerId)}`,
1207
- `GROUP BY c.${sqlIdentifier(customerName)}`,
1208
- `ORDER BY ${sqlIdentifier(primarySort)} ${orderDirection}, ${sqlIdentifier(secondarySort)} ${orderDirection}`,
1209
- 'LIMIT 10',
1210
- ].join('\n'),
1211
- viz: 'table',
1212
- };
1213
- }
1964
+ function parseStructuredProposal(raw) {
1965
+ const trimmed = raw.trim();
1966
+ const fenced = raw.match(/```json\s*([\s\S]*?)```/i)?.[1]?.trim();
1967
+ const candidates = [
1968
+ fenced,
1969
+ trimmed.startsWith('{') && trimmed.endsWith('}') ? trimmed : undefined,
1970
+ ].filter((candidate) => Boolean(candidate));
1971
+ for (const candidate of candidates) {
1972
+ try {
1973
+ const parsed = JSON.parse(candidate);
1974
+ const proposal = parsedProposalFromJson(parsed);
1975
+ if (proposal)
1976
+ return proposal;
1977
+ }
1978
+ catch {
1979
+ // Invalid structured output falls through to the legacy SQL-fence parser.
1214
1980
  }
1215
1981
  }
1216
- const genericJoinProposal = buildGenericJoinProposal({ ...input, schemaContext });
1217
- if (genericJoinProposal)
1218
- return genericJoinProposal;
1219
- const genericProposal = buildGenericSingleTableProposal({ ...input, schemaContext });
1220
- if (genericProposal)
1221
- return genericProposal;
1222
1982
  return undefined;
1223
1983
  }
1224
- function customerRankingDirectionFromText(text) {
1225
- const lower = text.toLowerCase();
1226
- if (/\b(less|lesser)\s+(?:order|orders|ordering)\b/.test(lower) || /\borders?\s+(?:less|least|fewest|lowest)\b/.test(lower)) {
1227
- return 'bottom';
1228
- }
1229
- return rankingDirectionFromText(text) ?? 'top';
1230
- }
1231
- function buildGenericJoinProposal(input) {
1232
- const planMode = input.contextPack?.questionPlan?.mode;
1233
- if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
1234
- return undefined;
1235
- }
1236
- const lower = input.question.toLowerCase();
1237
- if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
1984
+ function parsedProposalFromJson(value) {
1985
+ if (!value || typeof value !== 'object' || Array.isArray(value))
1238
1986
  return undefined;
1239
- }
1240
- if (!/\b(by\s+[a-z]|per\s+[a-z]|for each|group(?:ed)? by|split|segment|break\s*down|breakdown|compare)\b/i.test(lower)) {
1241
- return undefined;
1242
- }
1243
- const candidate = pickGenericJoinCandidate(input.schemaContext, input.question);
1244
- if (!candidate)
1987
+ const record = value;
1988
+ const sql = firstJsonString(record.sql, record.query)?.trim();
1989
+ const text = firstJsonString(record.summary, record.text, record.answer, record.description)?.trim() ?? '';
1990
+ const viz = firstJsonString(record.viz, record.visualization, record.chartType)
1991
+ ?.trim()
1992
+ .toLowerCase()
1993
+ .replace(/[^a-z_]/g, '');
1994
+ const dql = jsonRecord(record.dql);
1995
+ const outputs = firstJsonStringList(record.outputs, dql?.outputs);
1996
+ const proposedEntity = firstJsonString(dql?.entity, dql?.proposedEntity, dql?.proposed_entity, record.proposedEntity, record.proposed_entity, record.entity)?.replace(/\s+/g, ' ').trim().slice(0, 160);
1997
+ const requestedFilters = firstJsonStringList(dql?.filters, dql?.requestedFilters, dql?.requested_filters, record.requestedFilters, record.requested_filters, record.filters);
1998
+ const requestedDimensions = firstJsonStringList(dql?.dimensions, dql?.requestedDimensions, dql?.requested_dimensions, record.requestedDimensions, record.requested_dimensions, record.dimensions);
1999
+ if (!sql && !text)
1245
2000
  return undefined;
1246
- const direction = rankingDirectionFromText(input.question);
1247
- const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
1248
- const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
1249
- const chart = isTimeLikeGenericColumn(candidate.dimensionColumn) ? 'line' : 'bar';
1250
- const metricExpression = qualifiedMetricExpression(candidate.metric, 'f');
1251
- const metricAlias = candidate.metric.alias;
1252
- const dimensionAlias = candidate.dimensionColumn;
1253
2001
  return {
1254
- text: `Prepared a review-required ${businessMeasurePhrase(candidate.metric.column)} breakdown by ${humanizeIdentifier(candidate.dimensionColumn)} by joining ${candidate.metricTable.relation} to ${candidate.dimensionTable.relation}. This result is uncertified until reviewed and promoted.`,
1255
- sql: [
1256
- 'SELECT',
1257
- ` d.${sqlIdentifier(candidate.dimensionColumn)} AS ${sqlIdentifier(dimensionAlias)},`,
1258
- ` ${metricExpression} AS ${sqlIdentifier(metricAlias)}`,
1259
- `FROM ${sqlRelation(candidate.metricTable.relation)} AS f`,
1260
- `JOIN ${sqlRelation(candidate.dimensionTable.relation)} AS d ON f.${sqlIdentifier(candidate.metricJoinColumn)} = d.${sqlIdentifier(candidate.dimensionJoinColumn)}`,
1261
- `GROUP BY d.${sqlIdentifier(candidate.dimensionColumn)}`,
1262
- `ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
1263
- `LIMIT ${limit}`,
1264
- ].join('\n'),
1265
- viz: chart,
2002
+ text,
2003
+ ...(sql ? { sql } : {}),
2004
+ ...(viz ? { viz } : {}),
2005
+ ...(outputs?.length ? { outputs } : {}),
2006
+ ...(proposedEntity ? { proposedEntity } : {}),
2007
+ ...(requestedFilters?.length ? { requestedFilters } : {}),
2008
+ ...(requestedDimensions?.length ? { requestedDimensions } : {}),
1266
2009
  };
1267
2010
  }
1268
- function buildGenericSingleTableProposal(input) {
1269
- const planMode = input.contextPack?.questionPlan?.mode;
1270
- if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
1271
- return undefined;
1272
- }
1273
- const lower = input.question.toLowerCase();
1274
- if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
1275
- return undefined;
2011
+ function firstJsonString(...values) {
2012
+ return values.find((value) => typeof value === 'string' && value.trim().length > 0);
2013
+ }
2014
+ function jsonRecord(value) {
2015
+ return value && typeof value === 'object' && !Array.isArray(value)
2016
+ ? value
2017
+ : undefined;
2018
+ }
2019
+ function firstJsonStringList(...values) {
2020
+ for (const value of values) {
2021
+ const list = normalizeProposalStringList(value);
2022
+ if (list.length > 0)
2023
+ return list;
1276
2024
  }
1277
- if (!/\b(show|list|top|bottom|best|worst|highest|lowest|least|fewest|rank|ranking|compare|trend|by\s+[a-z]|how many|number of|revenue|sales|orders?|points?|score|count|total|average|avg|sum)\b/i.test(lower)) {
1278
- return undefined;
2025
+ return undefined;
2026
+ }
2027
+ function normalizeProposalStringList(value) {
2028
+ const rawValues = Array.isArray(value)
2029
+ ? value
2030
+ : typeof value === 'string'
2031
+ ? [value]
2032
+ : [];
2033
+ const seen = new Set();
2034
+ const out = [];
2035
+ for (const raw of rawValues) {
2036
+ if (typeof raw !== 'string')
2037
+ continue;
2038
+ const cleaned = raw.replace(/\s+/g, ' ').trim().slice(0, 160);
2039
+ if (!cleaned)
2040
+ continue;
2041
+ const key = cleaned.toLowerCase();
2042
+ if (seen.has(key))
2043
+ continue;
2044
+ seen.add(key);
2045
+ out.push(cleaned);
2046
+ if (out.length >= 20)
2047
+ break;
1279
2048
  }
1280
- const table = pickGenericAnalysisTable(input.schemaContext, input.question);
1281
- if (!table)
1282
- return undefined;
1283
- const metric = inferGenericMetric(table, input.question);
1284
- if (!metric)
2049
+ return out;
2050
+ }
2051
+ function mergeProposalStringLists(...lists) {
2052
+ const merged = normalizeProposalStringList(lists.flatMap((list) => list ?? []));
2053
+ return merged.length > 0 ? merged : undefined;
2054
+ }
2055
+ function applyParsedProposalMetadata(target, source) {
2056
+ target.outputs = source.outputs ?? target.outputs;
2057
+ target.proposedEntity = source.proposedEntity ?? target.proposedEntity;
2058
+ target.requestedFilters = source.requestedFilters ?? target.requestedFilters;
2059
+ target.requestedDimensions = source.requestedDimensions ?? target.requestedDimensions;
2060
+ }
2061
+ function buildCertifiedBlockDqlArtifact(block, result) {
2062
+ if (block.kind !== 'block')
1285
2063
  return undefined;
1286
- const dimensions = augmentGenericRankingDimensions(table, inferGenericDimensions(table, input.question, metric.column), metric, input.question).slice(0, 2);
1287
- const direction = rankingDirectionFromText(input.question);
1288
- const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
1289
- const metricAlias = metric.alias;
1290
- if (dimensions.length === 0) {
1291
- return {
1292
- text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} summary from ${table.relation}. This result is uncertified until reviewed and promoted.`,
1293
- sql: [
1294
- 'SELECT',
1295
- ` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
1296
- `FROM ${sqlRelation(table.relation)}`,
1297
- ].join('\n'),
1298
- viz: 'single_value',
1299
- };
1300
- }
1301
- if (direction && metric.preAggregated) {
2064
+ const sql = block.sql?.trim() ?? result?.sql?.trim() ?? block.examples?.find((example) => example.sql?.trim())?.sql?.trim();
2065
+ if (!sql) {
2066
+ // The block answered but its SQL was not inlined in the index (navigation /
2067
+ // reference artifacts). Still hand back a reference artifact pointing at the
2068
+ // governed source file so every certified-block answer carries a DQL artifact
2069
+ // the UI can open or save, rather than dropping it silently.
2070
+ if (!block.name)
2071
+ return undefined;
1302
2072
  return {
1303
- text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} ranking by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
1304
- sql: [
1305
- 'SELECT',
1306
- ...dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`),
1307
- ` ${sqlIdentifier(metric.column)} AS ${sqlIdentifier(metric.column)}`,
1308
- `FROM ${sqlRelation(table.relation)}`,
1309
- `ORDER BY ${sqlIdentifier(metric.column)} ${direction === 'bottom' ? 'ASC' : 'DESC'}`,
1310
- `LIMIT ${limit}`,
1311
- ].join('\n'),
1312
- viz: 'table',
2073
+ kind: 'certified_block',
2074
+ name: block.name,
2075
+ sourcePath: block.sourcePath,
2076
+ source: `// Certified DQL block "${escapeDqlArtifactString(block.name)}"`
2077
+ + `${block.sourcePath ? `\n// source: ${block.sourcePath}` : ''}`
2078
+ + `\n// SQL is not inlined in the index — open the source file to view or edit the query.`,
1313
2079
  };
1314
2080
  }
1315
- const selectDimensions = dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`);
1316
- const groupBy = dimensions.map(sqlIdentifier).join(', ');
1317
- const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
1318
- const chart = dimensions.some(isTimeLikeGenericColumn) ? 'line' : 'bar';
2081
+ const domain = block.domain ?? 'misc';
2082
+ const description = block.description ?? `Certified DQL block ${block.name}`;
2083
+ const sourcePathComment = block.sourcePath ? `\n // source: ${block.sourcePath}` : '';
2084
+ const source = `block "${escapeDqlArtifactString(block.name)}" {
2085
+ domain = "${escapeDqlArtifactString(domain)}"
2086
+ type = "custom"
2087
+ status = "certified"${block.owner ? `\n owner = "${escapeDqlArtifactString(block.owner)}"` : ''}
2088
+ description = """${escapeDqlArtifactTripleString(description)}"""${sourcePathComment}
2089
+
2090
+ query = """
2091
+ ${sql.replace(/"""/g, '\\"\\"\\"').split('\n').join('\n ')}
2092
+ """
2093
+ }
2094
+ `;
1319
2095
  return {
1320
- text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} breakdown by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
1321
- sql: [
1322
- 'SELECT',
1323
- ...selectDimensions,
1324
- ` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
1325
- `FROM ${sqlRelation(table.relation)}`,
1326
- `GROUP BY ${groupBy}`,
1327
- `ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
1328
- `LIMIT ${limit}`,
1329
- ].join('\n'),
1330
- viz: chart,
2096
+ kind: 'certified_block',
2097
+ name: block.name,
2098
+ sourcePath: block.sourcePath,
2099
+ source,
1331
2100
  };
1332
2101
  }
1333
- function pickGenericAnalysisTable(schemaContext, question) {
1334
- return schemaContext
1335
- .filter((table) => table.columns.length > 0)
1336
- .map((table, index) => ({
1337
- table,
1338
- metric: inferGenericMetric(table, question),
1339
- dimensions: inferGenericDimensions(table, question),
1340
- index,
1341
- }))
1342
- .filter((candidate) => candidate.metric)
1343
- .sort((a, b) => genericTableScore(b.table, question, b.metric, b.dimensions) -
1344
- genericTableScore(a.table, question, a.metric, a.dimensions) ||
1345
- a.index - b.index)[0]?.table;
1346
- }
1347
- function genericTableScore(table, question, metric, dimensions) {
1348
- const questionTokens = meaningfulTokens(question);
1349
- const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
1350
- const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
1351
- const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
1352
- const selectedRelationBoost = table.selectionRank
1353
- ? Math.max(0, 56 - table.selectionRank * 3) + Math.min(table.selectionScore ?? 0, 24)
1354
- : 0;
1355
- return selectedRelationBoost + overlap * 8 + (metric?.score ?? 0) + dimensions.length * 8 + Math.min(table.columns.length, 20) * 0.3;
1356
- }
1357
- function pickGenericJoinCandidate(schemaContext, question) {
1358
- const requestedDimensions = requestedDimensionPhrases(question);
1359
- if (requestedDimensions.length === 0)
2102
+ function buildGeneratedSqlDqlArtifact(input) {
2103
+ const sql = input.sql?.trim();
2104
+ if (!sql)
1360
2105
  return undefined;
1361
- const metricCandidates = schemaContext
1362
- .filter((table) => table.columns.length > 0)
1363
- .map((table, index) => ({
1364
- table,
1365
- metric: inferGenericMetric(table, question),
1366
- localDimensions: inferGenericDimensions(table, question),
1367
- index,
1368
- }))
1369
- .filter((candidate) => Boolean(candidate.metric))
1370
- .filter((candidate) => !candidate.localDimensions.some((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension)));
1371
- const candidates = [];
1372
- for (const metricCandidate of metricCandidates) {
1373
- for (const dimensionTable of schemaContext) {
1374
- if (normalizeRelationKey(dimensionTable.relation) === normalizeRelationKey(metricCandidate.table.relation))
1375
- continue;
1376
- const dimensionColumn = pickRequestedDimensionColumn(dimensionTable, question, requestedDimensions);
1377
- if (!dimensionColumn || namesEqualLoose(dimensionColumn, metricCandidate.metric.column))
1378
- continue;
1379
- const join = pickJoinColumns(metricCandidate.table, dimensionTable);
1380
- if (!join)
1381
- continue;
1382
- const dimensionScore = genericDimensionScore({ name: dimensionColumn, type: findColumnType(dimensionTable, dimensionColumn) }, question, Math.max(0, dimensionTable.columns.findIndex((column) => namesEqualLoose(column.name, dimensionColumn))), metricCandidate.metric.column);
1383
- candidates.push({
1384
- metricTable: metricCandidate.table,
1385
- dimensionTable,
1386
- metric: metricCandidate.metric,
1387
- dimensionColumn,
1388
- metricJoinColumn: join.leftColumn,
1389
- dimensionJoinColumn: join.rightColumn,
1390
- score: genericTableScore(metricCandidate.table, question, metricCandidate.metric, metricCandidate.localDimensions) +
1391
- dimensionScore +
1392
- join.score +
1393
- selectedRelationWeight(dimensionTable),
1394
- });
1395
- }
1396
- }
1397
- return candidates
1398
- .sort((a, b) => b.score - a.score ||
1399
- (a.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
1400
- a.metricTable.relation.localeCompare(b.metricTable.relation))[0];
2106
+ const slug = deriveGeneratedDraftSlug(input.question);
2107
+ const proposedDomain = input.sourceBlock?.domain
2108
+ ?? input.contextPack?.objects.find((object) => object.domain)?.domain
2109
+ ?? input.domain
2110
+ ?? 'misc';
2111
+ return {
2112
+ kind: 'sql_block',
2113
+ name: slug,
2114
+ source: renderGeneratedSqlDqlArtifact({
2115
+ slug,
2116
+ question: input.question,
2117
+ proposedSql: sql,
2118
+ proposedContractId: `${proposedDomain}.Unknown.${slug}`,
2119
+ proposedDomain,
2120
+ sourceQuestion: input.followUp?.sourceQuestion,
2121
+ sourceBlock: input.followUp?.sourceBlockName ?? input.sourceBlock?.name,
2122
+ sourceDqlArtifact: input.sourceDqlArtifact,
2123
+ followupKind: input.followUp?.kind,
2124
+ proposedEntity: input.proposedEntity,
2125
+ requestedFilters: input.requestedFilters,
2126
+ requestedDimensions: input.requestedDimensions,
2127
+ outputs: input.outputs,
2128
+ contextPackId: input.contextPack?.id,
2129
+ routeIntent: String(input.intent),
2130
+ validationWarnings: input.validationWarnings,
2131
+ }),
2132
+ };
2133
+ }
2134
+ function escapeDqlArtifactString(value) {
2135
+ return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
2136
+ }
2137
+ function escapeDqlArtifactTripleString(value) {
2138
+ return value.replace(/"""/g, '\\"\\"\\"');
2139
+ }
2140
+ /**
2141
+ * Generic partial-shape signal: the SQL executed but dropped MULTIPLE explicitly
2142
+ * requested output columns. Domain-agnostic — this downgrades trust and attaches a
2143
+ * "partial answer, missing <cols>" warning (and biases deep-candidate selection
2144
+ * toward the complete-shape candidate) for ANY project, not just product/customer/
2145
+ * revenue. It never refuses; the executed rows are always returned.
2146
+ */
2147
+ function generatedResultShapeIsPartial(resultShape) {
2148
+ return resultShape.missingOutputs.length >= 2;
2149
+ }
2150
+ function trustedFollowUpFilterValues(followUp) {
2151
+ const filters = new Set((followUp?.filters ?? []).map((value) => value.trim()).filter(Boolean));
2152
+ if (!filters.size || !followUp?.priorResultValues)
2153
+ return undefined;
2154
+ const priorValues = Object.values(followUp.priorResultValues).flat().map((value) => value.trim()).filter(Boolean);
2155
+ const trusted = priorValues.filter((value) => filters.has(value));
2156
+ return trusted.length > 0 ? uniqueDrilldownStrings(trusted) : undefined;
1401
2157
  }
1402
2158
  function buildCandidateJoinPaths(schemaContext) {
1403
2159
  const tables = [...schemaContext]
@@ -1445,542 +2201,63 @@ function joinReason(leftColumn, rightColumn, score) {
1445
2201
  return 'foreign-key style id match';
1446
2202
  return 'join-key style column match';
1447
2203
  }
1448
- function requestedDimensionPhrases(question) {
1449
- const lower = question.toLowerCase();
1450
- const phrases = [
1451
- ...Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
1452
- ...Array.from(lower.matchAll(/\bper\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
1453
- ...Array.from(lower.matchAll(/\bfor each\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
1454
- ];
1455
- return uniqueDrilldownStrings(phrases
1456
- .flatMap((phrase) => phrase.split(/\band\b|,|\//i))
1457
- .map((phrase) => phrase
1458
- .replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time|where|for|with|from|in|during|last|this|next)\b.*$/gi, '')
1459
- .replace(/[^a-z0-9_ -]+/gi, ' ')
1460
- .replace(/\s+/g, ' ')
1461
- .trim())
1462
- .filter((phrase) => phrase.length > 0));
1463
- }
1464
- function pickRequestedDimensionColumn(table, question, requestedDimensions) {
1465
- for (const phrase of requestedDimensions) {
1466
- const direct = findSchemaColumn(table, dimensionColumnCandidatesForPhrase(phrase));
1467
- if (direct && !isJoinKeyColumn(direct) && !isNumericLikeColumn({ name: direct, type: findColumnType(table, direct) })) {
1468
- return direct;
1469
- }
1470
- }
1471
- const inferred = inferGenericDimensions(table, question)
1472
- .find((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension));
1473
- return inferred;
1474
- }
1475
- function dimensionColumnCandidatesForPhrase(phrase) {
1476
- const tokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
1477
- .flatMap((token) => token.split('_'))
1478
- .map(normalizeToken)
1479
- .filter((token) => token.length > 0);
1480
- if (tokens.length === 0)
1481
- return [];
1482
- const joined = tokens.join('_');
1483
- const tail = tokens.slice(1).join('_');
1484
- return uniqueDrilldownStrings([
1485
- joined,
1486
- phrase.replace(/\s+/g, '_').toLowerCase(),
1487
- tail,
1488
- tokens.at(-1) ?? '',
1489
- ...tokens,
1490
- ].filter(Boolean));
1491
- }
1492
- function requestedDimensionMatchesColumn(requestedDimensions, column) {
1493
- const columnTokens = exactMatchTokens(column);
1494
- return requestedDimensions.some((phrase) => {
1495
- const phraseTokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
1496
- .flatMap((token) => token.split('_'))
1497
- .map(normalizeToken)
1498
- .filter((token) => token.length > 0);
1499
- return phraseTokens.some((token) => columnTokens.has(token));
1500
- });
1501
- }
1502
- function selectedRelationWeight(table) {
1503
- return table.selectionRank
1504
- ? Math.max(0, 30 - table.selectionRank * 2) + Math.min(table.selectionScore ?? 0, 16)
1505
- : 0;
1506
- }
1507
2204
  function pickJoinColumns(left, right) {
1508
2205
  const candidates = [];
1509
2206
  const leftColumns = left.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
1510
2207
  const rightColumns = right.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
1511
- for (const leftColumn of leftColumns) {
1512
- for (const rightColumn of rightColumns) {
1513
- const score = joinColumnScore(leftColumn.name, rightColumn.name, left, right);
1514
- if (score <= 0)
1515
- continue;
1516
- candidates.push({ leftColumn: leftColumn.name, rightColumn: rightColumn.name, score });
1517
- }
1518
- }
1519
- return candidates.sort((a, b) => b.score - a.score || a.leftColumn.localeCompare(b.leftColumn))[0];
1520
- }
1521
- function joinColumnScore(leftColumn, rightColumn, leftTable, rightTable) {
1522
- const leftKey = normalizeRelationKey(leftColumn).replace(/\./g, '_');
1523
- const rightKey = normalizeRelationKey(rightColumn).replace(/\./g, '_');
1524
- const leftJoinLike = isJoinKeyColumn(leftColumn);
1525
- const rightJoinLike = isJoinKeyColumn(rightColumn);
1526
- if (!leftJoinLike || !rightJoinLike)
1527
- return 0;
1528
- if (leftKey === rightKey)
1529
- return 70;
1530
- const leftSubject = joinSubjectForColumn(leftColumn);
1531
- const rightSubject = joinSubjectForColumn(rightColumn);
1532
- if (leftSubject && rightSubject && leftSubject === rightSubject)
1533
- return 62;
1534
- const leftTableTokens = tableEntityTokens(leftTable);
1535
- const rightTableTokens = tableEntityTokens(rightTable);
1536
- if (leftSubject && rightKey === 'id' && rightTableTokens.has(leftSubject))
1537
- return 58;
1538
- if (rightSubject && leftKey === 'id' && leftTableTokens.has(rightSubject))
1539
- return 58;
1540
- if (leftSubject && rightTableTokens.has(leftSubject) && rightKey.endsWith('_key'))
1541
- return 42;
1542
- if (rightSubject && leftTableTokens.has(rightSubject) && leftKey.endsWith('_key'))
1543
- return 42;
1544
- return 0;
1545
- }
1546
- function isJoinKeyColumn(column) {
1547
- const normalized = column.toLowerCase();
1548
- return normalized === 'id' ||
1549
- /(^|_)(id|key|uuid|sk)$/.test(normalized) ||
1550
- /_(id|key|uuid|sk)$/.test(normalized);
1551
- }
1552
- function joinSubjectForColumn(column) {
1553
- const normalized = column.toLowerCase();
1554
- const subject = normalized.replace(/_(id|key|uuid|sk)$/i, '');
1555
- if (!subject || subject === normalized || subject === 'id' || subject === 'key')
1556
- return undefined;
1557
- return normalizeToken(subject.split('_').at(-1) ?? subject);
1558
- }
1559
- function tableEntityTokens(table) {
1560
- const tokens = exactMatchTokens([table.name, table.relation.split('.').at(-1) ?? table.relation].join(' '));
1561
- for (const generic of ['dim', 'fct', 'fact', 'stg', 'stage', 'model', 'table'])
1562
- tokens.delete(generic);
1563
- return tokens;
1564
- }
1565
- function findColumnType(table, columnName) {
1566
- return table.columns.find((column) => namesEqualLoose(column.name, columnName))?.type;
1567
- }
1568
- function qualifiedMetricExpression(metric, tableAlias) {
1569
- const column = `${tableAlias}.${sqlIdentifier(metric.column)}`;
1570
- if (metric.column === 'rows')
1571
- return 'COUNT(*)';
1572
- if (/^COUNT\s*\(\s*DISTINCT\b/i.test(metric.expression))
1573
- return `COUNT(DISTINCT ${column})`;
1574
- if (metric.preAggregated)
1575
- return column;
1576
- const aggregate = metric.expression.match(/^(SUM|AVG|MIN|MAX|MEDIAN)\s*\(/i)?.[1]?.toUpperCase()
1577
- ?? (/avg|average|mean/i.test(metric.alias) ? 'AVG' : 'SUM');
1578
- return `${aggregate}(${column})`;
1579
- }
1580
- function inferGenericMetric(table, question) {
1581
- const lower = question.toLowerCase();
1582
- const wantsCount = /\b(count|how many|number of|volume)\b/i.test(lower);
1583
- const candidates = table.columns
1584
- .map((column) => {
1585
- const name = column.name.toLowerCase();
1586
- const tokens = meaningfulTokens(column.name);
1587
- let score = isNumericLikeColumn(column) ? 20 : 0;
1588
- for (const token of meaningfulTokens(question)) {
1589
- if (tokens.has(token))
1590
- score += 18;
1591
- }
1592
- if (/\b(revenue|sales|amount|total|spend|cost|margin|profit|value|points?|score|orders?|count|quantity|duration|minutes?|rate|average|avg)\b/i.test(name.replace(/_/g, ' '))) {
1593
- score += 18;
1594
- }
1595
- if (/\b(id|key|code|zip|postal|phone|season|year|month|week|day|date|time)\b/i.test(name.replace(/_/g, ' '))) {
1596
- score -= 14;
1597
- }
1598
- return { column: column.name, score };
1599
- })
1600
- .filter((candidate) => candidate.score > 0)
1601
- .sort((a, b) => b.score - a.score);
1602
- const countSubject = wantsCount ? inferCountSubject(question) : undefined;
1603
- if (wantsCount) {
1604
- const precomputed = candidates.find((candidate) => isPrecomputedCountMetric(candidate.column, countSubject));
1605
- if (precomputed) {
1606
- return {
1607
- column: precomputed.column,
1608
- expression: isPreAggregatedMetricColumn(precomputed.column) ? sqlIdentifier(precomputed.column) : `SUM(${sqlIdentifier(precomputed.column)})`,
1609
- alias: isPreAggregatedMetricColumn(precomputed.column) ? precomputed.column : `${precomputed.column}_sum`,
1610
- score: precomputed.score + 8,
1611
- preAggregated: isPreAggregatedMetricColumn(precomputed.column),
1612
- };
1613
- }
1614
- const distinctColumn = countSubject ? findDistinctCountColumn(table, countSubject) : undefined;
1615
- if (distinctColumn) {
1616
- return {
1617
- column: distinctColumn,
1618
- expression: `COUNT(DISTINCT ${sqlIdentifier(distinctColumn)})`,
1619
- alias: `${countSubject}_count`,
1620
- score: 34,
1621
- preAggregated: false,
1622
- };
1623
- }
1624
- }
1625
- const selected = candidates[0];
1626
- if (!selected && !wantsCount)
1627
- return undefined;
1628
- if (!selected || wantsCount && selected.score < 26) {
1629
- return {
1630
- column: 'rows',
1631
- expression: 'COUNT(*)',
1632
- alias: 'row_count',
1633
- score: 18,
1634
- preAggregated: true,
1635
- };
1636
- }
1637
- const aggregate = /\b(avg|average|mean)\b/i.test(lower) ? 'AVG' : 'SUM';
1638
- const aliasBase = `${selected.column}_${aggregate.toLowerCase()}`;
1639
- const preAggregated = isPreAggregatedMetricColumn(selected.column);
1640
- return {
1641
- column: selected.column,
1642
- expression: preAggregated ? sqlIdentifier(selected.column) : `${aggregate}(${sqlIdentifier(selected.column)})`,
1643
- alias: preAggregated ? selected.column : aliasBase,
1644
- score: selected.score,
1645
- preAggregated,
1646
- };
1647
- }
1648
- function inferCountSubject(question) {
1649
- const lower = question.toLowerCase();
1650
- const match = lower.match(/\b(?:how many|number of|count of|count|total)\s+(?:distinct\s+|unique\s+)?([a-z][a-z0-9_-]*)/);
1651
- const raw = match?.[1] ?? lower.match(/\b(customers?|accounts?|users?|members?|orders?|products?|players?|teams?|transactions?|sessions?|events?|records?|rows?)\b/)?.[1];
1652
- if (!raw)
1653
- return undefined;
1654
- const normalized = normalizeToken(raw.replace(/[^a-z0-9_]/g, ''));
1655
- if (!normalized || normalized === 'row' || normalized === 'record')
1656
- return undefined;
1657
- return normalized;
1658
- }
1659
- function findDistinctCountColumn(table, subject) {
1660
- const preferred = [
1661
- `${subject}_id`,
1662
- `${subject}_key`,
1663
- `${subject}_uuid`,
1664
- `${subject}_sk`,
1665
- `${subject}_name`,
1666
- subject,
1667
- ];
1668
- const direct = findSchemaColumn(table, preferred);
1669
- if (direct)
1670
- return direct;
1671
- const tableTokens = meaningfulTokens([table.name, table.relation].join(' '));
1672
- if (!tableTokens.has(subject))
1673
- return undefined;
1674
- return findSchemaColumn(table, ['id', 'key', 'uuid']);
1675
- }
1676
- function isPrecomputedCountMetric(column, subject) {
1677
- const normalized = column.toLowerCase();
1678
- const text = normalized.replace(/_/g, ' ');
1679
- if (/\b(id|key|uuid|code)\b/i.test(text))
1680
- return false;
1681
- if (/\b(count|cnt|number|num)\b/i.test(text)) {
1682
- return !subject || text.includes(subject) || text.includes(pluralizeSimple(subject));
1683
- }
1684
- return Boolean(subject && (normalized === `total_${subject}` ||
1685
- normalized === `total_${pluralizeSimple(subject)}`));
1686
- }
1687
- function pluralizeSimple(value) {
1688
- if (value.endsWith('y'))
1689
- return `${value.slice(0, -1)}ies`;
1690
- if (value.endsWith('s'))
1691
- return value;
1692
- return `${value}s`;
1693
- }
1694
- function augmentGenericRankingDimensions(table, dimensions, metric, question) {
1695
- const direction = rankingDirectionFromText(question);
1696
- if (!direction || !metric.preAggregated || /\bby\s+/i.test(question))
1697
- return dimensions;
1698
- const timeColumn = table.columns
1699
- .map((column) => column.name)
1700
- .find((name) => isTimeLikeGenericColumn(name) && !dimensions.some((dimension) => namesEqualLoose(dimension, name)));
1701
- return timeColumn ? [...dimensions, timeColumn] : dimensions;
1702
- }
1703
- function inferGenericDimensions(table, question, metricColumn) {
1704
- const lower = question.toLowerCase();
1705
- const requested = Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g))
1706
- .map((match) => match[1] ?? '')
1707
- .flatMap((value) => value.split(/\band\b|,|\//i))
1708
- .map((value) => value.replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time)\b/gi, '').trim())
1709
- .filter(Boolean);
1710
- const direct = requested
1711
- .map((value) => findSchemaColumn(table, [value, value.replace(/\s+/g, '_')]))
1712
- .filter((value) => Boolean(value));
1713
- if (direct.length > 0)
1714
- return uniqueDrilldownStrings(direct);
1715
- const domainHints = [
1716
- /\bcustomers?\b/.test(lower) ? ['customer_name', 'customer', 'customer_id'] : [],
1717
- /\bproducts?\b/.test(lower) ? ['product_name', 'product', 'sku', 'product_id'] : [],
1718
- /\bplayers?\b/.test(lower) ? ['player_name', 'player', 'player_id'] : [],
1719
- /\bteams?\b/.test(lower) ? ['team_name', 'team', 'team_id'] : [],
1720
- /\bregions?\b|\bmarkets?\b/.test(lower) ? ['region', 'market', 'geo'] : [],
1721
- /\bsegments?\b/.test(lower) ? ['segment', 'customer_segment', 'type'] : [],
1722
- /\bchannels?\b/.test(lower) ? ['channel', 'source_channel'] : [],
1723
- /\b(month|monthly)\b/.test(lower) ? ['month', 'order_month', 'created_month'] : [],
1724
- /\b(week|weekly)\b/.test(lower) ? ['week', 'order_week', 'created_week'] : [],
1725
- /\b(year|season|yearly)\b/.test(lower) ? ['season', 'year', 'order_year'] : [],
1726
- ].flat();
1727
- const hinted = findSchemaColumn(table, domainHints);
1728
- if (hinted)
1729
- return [hinted];
1730
- const fallback = table.columns
1731
- .map((column, index) => ({
1732
- column: column.name,
1733
- score: genericDimensionScore(column, question, index, metricColumn),
1734
- }))
1735
- .filter((candidate) => candidate.score > 0)
1736
- .sort((a, b) => b.score - a.score)
1737
- .map((candidate) => candidate.column);
1738
- return uniqueDrilldownStrings(fallback).slice(0, 1);
1739
- }
1740
- function genericDimensionScore(column, question, index, metricColumn) {
1741
- if (metricColumn && namesEqualLoose(column.name, metricColumn))
1742
- return -100;
1743
- const name = column.name.toLowerCase();
1744
- const text = name.replace(/_/g, ' ');
1745
- let score = Math.max(0, 10 - index * 0.05);
1746
- if (/\b(name|category|type|segment|region|market|channel|status|team|player|customer|product|vendor|account|month|week|year|season|date)\b/i.test(text)) {
1747
- score += 24;
1748
- }
1749
- if (isNumericLikeColumn(column) && !isTimeLikeGenericColumn(column.name))
1750
- score -= 18;
1751
- for (const token of meaningfulTokens(question)) {
1752
- if (meaningfulTokens(column.name).has(token))
1753
- score += 10;
1754
- }
1755
- return score;
1756
- }
1757
- function isNumericLikeColumn(column) {
1758
- const type = column.type ?? '';
1759
- const name = column.name.toLowerCase();
1760
- return /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(type)
1761
- || /\b(total|count|amount|revenue|sales|spend|cost|margin|profit|value|points?|score|quantity|duration|minutes?|rate|avg|average)\b/i.test(name.replace(/_/g, ' '));
1762
- }
1763
- function isPreAggregatedMetricColumn(name) {
1764
- return /^(total|count|avg|average|median|min|max)_/i.test(name)
1765
- || /_(total|count|avg|average|median|min|max)$/i.test(name);
1766
- }
1767
- function isTimeLikeGenericColumn(name) {
1768
- return /\b(date|time|day|week|month|quarter|year|season|period)\b/i.test(name);
1769
- }
1770
- function buildEntityProfileProposal(input) {
1771
- const entityTexts = entityMentionsForProfile(input.question, input.contextPack);
1772
- if (entityTexts.length === 0 || !isEntityProfileQuestion(input.question, input.contextPack))
1773
- return undefined;
1774
- const candidates = input.schemaContext
1775
- .map((table) => profileTableCandidate(table, input.question, input.followUp, entityTexts))
1776
- .filter((candidate) => Boolean(candidate))
1777
- .sort((a, b) => b.score - a.score);
1778
- const selected = candidates[0];
1779
- if (!selected)
1780
- return undefined;
1781
- const where = selected.filterValue
1782
- ? [`WHERE ${sqlIdentifier(selected.entityColumn)} = ${sqlStringLiteral(selected.filterValue)}`]
1783
- : [];
1784
- const columns = selected.selectColumns.map((column) => ` ${sqlIdentifier(column)}`).join(',\n');
1785
- const text = [
1786
- `Prepared a review-required profile query for ${selected.filterValue ?? entityTexts[0]} from ${selected.table.relation}.`,
1787
- selected.usedSampleValue
1788
- ? `The entity filter uses an inspected value match on ${selected.entityColumn}.`
1789
- : `The entity filter uses the likely entity column ${selected.entityColumn} from inspected schema metadata.`,
1790
- 'This result is uncertified until reviewed and promoted.',
1791
- ].join(' ');
1792
- return {
1793
- text,
1794
- sql: [
1795
- 'SELECT',
1796
- columns,
1797
- `FROM ${sqlRelation(selected.table.relation)}`,
1798
- ...where,
1799
- 'LIMIT 50',
1800
- ].join('\n'),
1801
- viz: 'table',
1802
- };
1803
- }
1804
- function profileTableCandidate(table, question, followUp, entityTexts) {
1805
- if (table.columns.length === 0)
1806
- return undefined;
1807
- const matchedFilters = matchedEntityFiltersForQuestion(table, question, followUp);
1808
- const sampled = matchedFilters.find((filter) => entityTexts.some((entity) => normalizeForEntityMatch(entity) === normalizeForEntityMatch(filter.value))) ?? matchedFilters[0];
1809
- const entityColumn = sampled?.column ?? pickEntityProfileColumn(table, question);
1810
- if (!entityColumn)
1811
- return undefined;
1812
- const filterValue = sampled?.value ?? entityTexts[0];
1813
- if (!filterValue)
1814
- return undefined;
1815
- const selectColumns = orderedProfileColumns(table, entityColumn, question);
1816
- if (selectColumns.length === 0)
1817
- return undefined;
1818
- const questionTokens = meaningfulTokens(question);
1819
- const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
1820
- const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
1821
- const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
1822
- const profileSignal = selectColumns.filter((column) => isProfileMeasureColumn(column, table)).length;
1823
- const score = (sampled ? 80 : 42) +
1824
- overlap * 6 +
1825
- profileSignal * 3 +
1826
- (/\b(player|customer|account|user|member|person|entity)\b/i.test(table.name) ? 8 : 0);
1827
- return {
1828
- table,
1829
- entityColumn,
1830
- filterValue,
1831
- selectColumns,
1832
- usedSampleValue: Boolean(sampled),
1833
- score,
1834
- };
1835
- }
1836
- function isEntityProfileQuestion(question, contextPack) {
1837
- const mode = contextPack?.questionPlan?.mode;
1838
- if (mode === 'entity_profile')
1839
- return true;
1840
- return /\b(profile|overview|360|complete\s+(?:stats|statistics|view)|full\s+(?:stats|statistics|view)|all\s+(?:stats|statistics|metrics)|research|reserach)\b/i.test(question);
1841
- }
1842
- function entityMentionsForProfile(question, contextPack) {
1843
- const values = [
1844
- ...(contextPack?.questionPlan?.entities ?? []).map((entity) => entity.text),
1845
- ...Array.from(question.matchAll(/\b(?:for|on|profile\s+for|research\s+on|reserach\s+on)\s+([A-Z][A-Za-z0-9&.'-]+(?:\s+[A-Z][A-Za-z0-9&.'-]+){0,5})/g))
1846
- .map((match) => match[1] ?? ''),
1847
- ];
1848
- if (isEntityProfileQuestion(question, contextPack)) {
1849
- values.push(...Array.from(question.matchAll(/\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,5})\b/g)).map((match) => match[1] ?? ''));
2208
+ for (const leftColumn of leftColumns) {
2209
+ for (const rightColumn of rightColumns) {
2210
+ const score = joinColumnScore(leftColumn.name, rightColumn.name, left, right);
2211
+ if (score <= 0)
2212
+ continue;
2213
+ candidates.push({ leftColumn: leftColumn.name, rightColumn: rightColumn.name, score });
2214
+ }
1850
2215
  }
1851
- return uniqueDrilldownStrings(values
1852
- .map((value) => value.replace(/\b(profile|stats|statistics|details|summary)\b.*$/i, '').trim())
1853
- .filter((value) => value.length > 1 && !/^(Can You|Could You|Tell Me|Show Me)$/i.test(value)))
1854
- .slice(0, 6);
2216
+ return candidates.sort((a, b) => b.score - a.score || a.leftColumn.localeCompare(b.leftColumn))[0];
1855
2217
  }
1856
- function pickEntityProfileColumn(table, question) {
1857
- const lower = question.toLowerCase();
1858
- const domainHints = [
1859
- /\bplayers?\b/.test(lower) ? 'player' : '',
1860
- /\bcustomers?\b/.test(lower) ? 'customer' : '',
1861
- /\baccounts?\b/.test(lower) ? 'account' : '',
1862
- /\busers?\b/.test(lower) ? 'user' : '',
1863
- /\bmembers?\b/.test(lower) ? 'member' : '',
1864
- /\bteams?\b/.test(lower) ? 'team' : '',
1865
- /\bproducts?\b/.test(lower) ? 'product' : '',
1866
- ].filter(Boolean);
1867
- const preferred = [
1868
- ...domainHints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
1869
- 'name',
1870
- 'full_name',
1871
- 'display_name',
1872
- 'title',
1873
- ];
1874
- const direct = findSchemaColumn(table, preferred);
1875
- if (direct)
1876
- return direct;
1877
- const scored = table.columns
1878
- .map((column) => {
1879
- const name = column.name.toLowerCase();
1880
- let score = 0;
1881
- if (name.endsWith('_name') || name === 'name')
1882
- score += 30;
1883
- if (/\b(name|title|email)\b/i.test(name.replace(/_/g, ' ')))
1884
- score += 20;
1885
- if (domainHints.some((hint) => name.includes(hint)))
1886
- score += 16;
1887
- if (/\b(id|key|code|date|time|amount|total|count|score|points|revenue)\b/i.test(name.replace(/_/g, ' ')))
1888
- score -= 12;
1889
- return { column: column.name, score };
1890
- })
1891
- .filter((item) => item.score > 0)
1892
- .sort((a, b) => b.score - a.score);
1893
- return scored[0]?.column;
1894
- }
1895
- function orderedProfileColumns(table, entityColumn, question) {
1896
- const scored = table.columns
1897
- .map((column, index) => ({
1898
- column: column.name,
1899
- score: profileColumnScore(column, question, index),
1900
- index,
1901
- }))
1902
- .sort((a, b) => b.score - a.score || a.index - b.index);
1903
- const selected = uniqueDrilldownStrings([
1904
- entityColumn,
1905
- ...scored.map((item) => item.column),
1906
- ]).slice(0, 24);
1907
- return selected.length > 0 ? selected : [entityColumn];
1908
- }
1909
- function profileColumnScore(column, question, index) {
1910
- const name = column.name.toLowerCase();
1911
- const text = name.replace(/_/g, ' ');
1912
- const lowerQuestion = question.toLowerCase();
1913
- let score = Math.max(0, 12 - index * 0.1);
1914
- if (isProfileMeasureColumn(column.name, { relation: '', name: '', columns: [column] }))
1915
- score += 35;
1916
- if (/\b(name|title|team|season|year|date|type|category|segment|status)\b/i.test(text))
1917
- score += 18;
1918
- if (/\b(id|key|code)\b/i.test(text))
1919
- score += 2;
1920
- for (const token of meaningfulTokens(lowerQuestion)) {
1921
- if (meaningfulTokens(column.name).has(token))
1922
- score += 12;
1923
- }
1924
- if (column.description && meaningfulTokens(column.description).size > 0)
1925
- score += 4;
1926
- return score;
2218
+ function joinColumnScore(leftColumn, rightColumn, leftTable, rightTable) {
2219
+ const leftKey = normalizeRelationKey(leftColumn).replace(/\./g, '_');
2220
+ const rightKey = normalizeRelationKey(rightColumn).replace(/\./g, '_');
2221
+ const leftJoinLike = isJoinKeyColumn(leftColumn);
2222
+ const rightJoinLike = isJoinKeyColumn(rightColumn);
2223
+ if (!leftJoinLike || !rightJoinLike)
2224
+ return 0;
2225
+ if (leftKey === rightKey)
2226
+ return 70;
2227
+ const leftSubject = joinSubjectForColumn(leftColumn);
2228
+ const rightSubject = joinSubjectForColumn(rightColumn);
2229
+ if (leftSubject && rightSubject && leftSubject === rightSubject)
2230
+ return 62;
2231
+ const leftTableTokens = tableEntityTokens(leftTable);
2232
+ const rightTableTokens = tableEntityTokens(rightTable);
2233
+ if (leftSubject && rightKey === 'id' && rightTableTokens.has(leftSubject))
2234
+ return 58;
2235
+ if (rightSubject && leftKey === 'id' && leftTableTokens.has(rightSubject))
2236
+ return 58;
2237
+ if (leftSubject && rightTableTokens.has(leftSubject) && rightKey.endsWith('_key'))
2238
+ return 42;
2239
+ if (rightSubject && leftTableTokens.has(rightSubject) && leftKey.endsWith('_key'))
2240
+ return 42;
2241
+ return 0;
1927
2242
  }
1928
- function isProfileMeasureColumn(columnName, table) {
1929
- const column = table.columns.find((item) => namesEqualLoose(item.name, columnName));
1930
- const name = columnName.toLowerCase();
1931
- const text = name.replace(/_/g, ' ');
1932
- return /\b(total|count|avg|average|sum|rate|score|points|pts|rebounds?|assists?|steals?|blocks?|turnovers?|minutes?|games?|wins?|losses?|revenue|amount|spend|orders?|quantity|margin|cost|value|stat)\b/i.test(text)
1933
- || /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(column?.type ?? '');
2243
+ function isJoinKeyColumn(column) {
2244
+ const normalized = column.toLowerCase();
2245
+ return normalized === 'id' ||
2246
+ /(^|_)(id|key|uuid|sk)$/.test(normalized) ||
2247
+ /_(id|key|uuid|sk)$/.test(normalized);
1934
2248
  }
1935
- function buildMatchedEntityDrilldownProposal(input) {
1936
- if (input.intent !== 'entity_drilldown' && input.followUp?.kind !== 'drilldown')
1937
- return undefined;
1938
- const table = pickDrilldownTable(input.schemaContext, input.question, input.followUp);
1939
- if (!table)
1940
- return undefined;
1941
- const dimension = inferDrilldownDimension(table, input.question, input.followUp);
1942
- if (!dimension)
1943
- return undefined;
1944
- const entityFilters = matchedEntityFiltersForQuestion(table, input.question, input.followUp);
1945
- if (entityFilters.length === 0)
1946
- return undefined;
1947
- if (entityFilters.some((filter) => namesEqualLoose(filter.column, dimension)))
1948
- return undefined;
1949
- const sourceSql = selectSourceBlockSql(input.contextPack, input.followUp?.sourceBlockName);
1950
- const metric = inferDrilldownMetric(table, input.question, sourceSql);
1951
- if (!metric)
1952
- return undefined;
1953
- const timePredicates = drilldownTimePredicates({
1954
- question: input.question,
1955
- followUp: input.followUp,
1956
- table,
1957
- sourceSql,
1958
- });
1959
- if (mentionsRelativeTime(input.question, input.followUp) && timePredicates.length === 0)
2249
+ function joinSubjectForColumn(column) {
2250
+ const normalized = column.toLowerCase();
2251
+ const subject = normalized.replace(/_(id|key|uuid|sk)$/i, '');
2252
+ if (!subject || subject === normalized || subject === 'id' || subject === 'key')
1960
2253
  return undefined;
1961
- const predicates = [
1962
- ...entityFilters.map((filter) => `${sqlIdentifier(filter.column)} = ${sqlStringLiteral(filter.value)}`),
1963
- ...timePredicates,
1964
- ];
1965
- const where = predicates.length ? [`WHERE ${predicates.join(' AND ')}`] : [];
1966
- return {
1967
- text: [
1968
- `Prepared a review-required ${humanizeIdentifier(dimension)} drilldown from inspected metadata.`,
1969
- `The entity filter uses ${entityFilters.map((filter) => `${filter.column} = ${filter.value}`).join(', ')} from matched sample values.`,
1970
- 'This result is uncertified until reviewed and promoted.',
1971
- ].join(' '),
1972
- sql: [
1973
- 'SELECT',
1974
- ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`,
1975
- ` ${metric.expression} AS ${sqlIdentifier(metric.alias)}`,
1976
- `FROM ${sqlRelation(table.relation)}`,
1977
- ...where,
1978
- `GROUP BY ${sqlIdentifier(dimension)}`,
1979
- `ORDER BY ${sqlIdentifier(metric.alias)} DESC`,
1980
- 'LIMIT 50',
1981
- ].join('\n'),
1982
- viz: 'bar',
1983
- };
2254
+ return normalizeToken(subject.split('_').at(-1) ?? subject);
2255
+ }
2256
+ function tableEntityTokens(table) {
2257
+ const tokens = exactMatchTokens([table.name, table.relation.split('.').at(-1) ?? table.relation].join(' '));
2258
+ for (const generic of ['dim', 'fct', 'fact', 'stg', 'stage', 'model', 'table'])
2259
+ tokens.delete(generic);
2260
+ return tokens;
1984
2261
  }
1985
2262
  function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
1986
2263
  const byRelation = new Map();
@@ -2058,169 +2335,6 @@ function relationSelectionFor(relation, lookup) {
2058
2335
  }
2059
2336
  return undefined;
2060
2337
  }
2061
- function pickDrilldownTable(schemaContext, question, followUp) {
2062
- const scored = schemaContext
2063
- .map((table) => ({
2064
- table,
2065
- filters: matchedEntityFiltersForQuestion(table, question, followUp).length,
2066
- dimension: inferDrilldownDimension(table, question, followUp) ? 1 : 0,
2067
- measure: inferDrilldownMetric(table, question, undefined) ? 1 : 0,
2068
- }))
2069
- .filter((candidate) => candidate.filters > 0 && candidate.dimension > 0 && candidate.measure > 0)
2070
- .sort((a, b) => (b.filters + b.dimension + b.measure) - (a.filters + a.dimension + a.measure));
2071
- return scored[0]?.table;
2072
- }
2073
- function inferDrilldownDimension(table, question, followUp) {
2074
- const lower = question.toLowerCase();
2075
- const requested = [
2076
- ...(followUp?.dimensions ?? []),
2077
- ...Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g)).map((match) => match[1] ?? ''),
2078
- ]
2079
- .flatMap((value) => value.split(/\band\b|,|\//i))
2080
- .map((value) => value.replace(/\b(last|this|next|previous|prior|current)\s+(day|week|month|quarter|year)\b/gi, '').trim())
2081
- .filter(Boolean);
2082
- const direct = findSchemaColumn(table, requested);
2083
- if (direct)
2084
- return direct;
2085
- if (/\bcustomers?\b/.test(lower)) {
2086
- const customer = findSchemaColumn(table, ['customer', 'customer_name', 'account', 'account_name']);
2087
- if (customer)
2088
- return customer;
2089
- }
2090
- if (/\bsegments?\b/.test(lower)) {
2091
- const segment = findSchemaColumn(table, ['segment', 'customer_segment', 'market_segment']);
2092
- if (segment)
2093
- return segment;
2094
- }
2095
- if (/\bproducts?\b/.test(lower)) {
2096
- const product = findSchemaColumn(table, ['product', 'product_name', 'sku']);
2097
- if (product)
2098
- return product;
2099
- }
2100
- if (/\bregions?\b/.test(lower)) {
2101
- const region = findSchemaColumn(table, ['region', 'market', 'geo']);
2102
- if (region)
2103
- return region;
2104
- }
2105
- return undefined;
2106
- }
2107
- function matchedEntityFiltersForQuestion(table, question, followUp) {
2108
- const text = normalizeForEntityMatch([
2109
- question,
2110
- ...(followUp?.filters ?? []),
2111
- ].join(' '));
2112
- const filters = [];
2113
- for (const column of table.columns) {
2114
- for (const sampleValue of column.sampleValues ?? []) {
2115
- if (isTemporalDrilldownValue(sampleValue))
2116
- continue;
2117
- const needle = normalizeForEntityMatch(sampleValue);
2118
- if (!needle || !text.includes(needle))
2119
- continue;
2120
- filters.push({ column: column.name, value: sampleValue });
2121
- }
2122
- }
2123
- return uniqueMatchedEntityFilters(filters);
2124
- }
2125
- function inferDrilldownMetric(table, question, sourceSql) {
2126
- const sourceMetric = sourceSql ? aggregateMetricFromSourceSql(table, sourceSql) : undefined;
2127
- if (sourceMetric)
2128
- return sourceMetric;
2129
- const lower = question.toLowerCase();
2130
- const candidates = /\brevenue|arr|mrr|sales\b/.test(lower)
2131
- ? ['revenue', 'net_revenue', 'gross_revenue', 'amount', 'order_total', 'total_amount', 'sales', 'arr', 'mrr']
2132
- : ['amount', 'revenue', 'order_total', 'total_amount', 'value', 'spend'];
2133
- const column = findSchemaColumn(table, candidates);
2134
- if (!column)
2135
- return undefined;
2136
- const alias = /\brevenue|arr|mrr|sales\b/.test(lower) ? 'revenue_total' : `${column}_total`;
2137
- return {
2138
- expression: `SUM(${sqlIdentifier(column)})`,
2139
- alias,
2140
- };
2141
- }
2142
- function aggregateMetricFromSourceSql(table, sourceSql) {
2143
- for (const column of table.columns) {
2144
- const columnPattern = sqlIdentifierPattern(column.name);
2145
- const aggregatePattern = new RegExp(`\\b(SUM|COUNT|AVG|MIN|MAX)\\s*\\(\\s*(?:["\`]?\\w+["\`]?\\s*\\.\\s*)?(${columnPattern}|\\*)\\s*\\)\\s*(?:AS\\s+(["\`]?\\w+["\`]?))?`, 'i');
2146
- const match = sourceSql.match(aggregatePattern);
2147
- if (!match)
2148
- continue;
2149
- const fn = (match[1] ?? 'SUM').toUpperCase();
2150
- const target = match[2] === '*' ? '*' : sqlIdentifier(column.name);
2151
- const alias = cleanSqlIdentifier(match[3] ?? defaultMetricAlias(fn, column.name));
2152
- return {
2153
- expression: `${fn}(${target})`,
2154
- alias,
2155
- };
2156
- }
2157
- return undefined;
2158
- }
2159
- function drilldownTimePredicates(input) {
2160
- if (!mentionsRelativeTime(input.question, input.followUp))
2161
- return [];
2162
- if (!input.sourceSql)
2163
- return [];
2164
- const timeColumns = input.table.columns.map((column) => column.name).filter(isTimeLikeDrilldownColumn);
2165
- if (timeColumns.length === 0)
2166
- return [];
2167
- return extractWherePredicates(input.sourceSql)
2168
- .map(stripSqlAliasQualifiers)
2169
- .filter((predicate) => isReusableSqlPredicate(predicate))
2170
- .filter((predicate) => timeColumns.some((column) => predicateReferencesColumn(predicate, column)))
2171
- .slice(0, 3);
2172
- }
2173
- function selectSourceBlockSql(contextPack, sourceBlockName) {
2174
- const sourceSql = contextPack?.allowedSqlContext?.sourceBlockSql ?? [];
2175
- if (sourceSql.length === 0)
2176
- return undefined;
2177
- const preferred = sourceBlockName
2178
- ? sourceSql.find((source) => namesEqualLoose(source.name, sourceBlockName))
2179
- : undefined;
2180
- return (preferred ?? sourceSql.find((source) => source.status === 'certified') ?? sourceSql[0])?.sql;
2181
- }
2182
- function extractWherePredicates(sql) {
2183
- const match = sql.match(/\bWHERE\b([\s\S]*?)(\bGROUP\s+BY\b|\bHAVING\b|\bORDER\s+BY\b|\bLIMIT\b|$)/i);
2184
- if (!match)
2185
- return [];
2186
- return (match[1] ?? '')
2187
- .split(/\s+AND\s+/i)
2188
- .map((part) => part.trim().replace(/^\(+|\)+$/g, '').replace(/\s+/g, ' '))
2189
- .filter(Boolean);
2190
- }
2191
- function mentionsRelativeTime(question, followUp) {
2192
- const text = [question, ...(followUp?.filters ?? [])].join(' ');
2193
- return /\b(last|this|next|previous|prior|current)\s+(day|week|month|quarter|year)\b/i.test(text)
2194
- || /\b(today|yesterday|tomorrow|ytd|mtd|qtd|wtd)\b/i.test(text);
2195
- }
2196
- function isTimeLikeDrilldownColumn(name) {
2197
- return /\b(date|time|day|week|month|quarter|year|period|created_at|updated_at)\b/i.test(name);
2198
- }
2199
- function stripSqlAliasQualifiers(predicate) {
2200
- return predicate.replace(/\b["`]?\w+["`]?\s*\.\s*(["`]?\w+["`]?)/g, '$1');
2201
- }
2202
- function isReusableSqlPredicate(predicate) {
2203
- if (!predicate || predicate.length > 240)
2204
- return false;
2205
- if (/[;]/.test(predicate) || /--|\/\*/.test(predicate))
2206
- return false;
2207
- return !/\b(SELECT|INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|COPY|PRAGMA|SET)\b/i.test(predicate);
2208
- }
2209
- function predicateReferencesColumn(predicate, column) {
2210
- return new RegExp(`(^|[^\\w])${sqlIdentifierPattern(column)}([^\\w]|$)`, 'i').test(predicate);
2211
- }
2212
- function sqlIdentifierPattern(identifier) {
2213
- const escaped = escapeRegExp(identifier);
2214
- return `(?:"${escaped}"|\`${escaped}\`|${escaped})`;
2215
- }
2216
- function sqlStringLiteral(value) {
2217
- return `'${value.replace(/'/g, "''")}'`;
2218
- }
2219
- function defaultMetricAlias(fn, column) {
2220
- if (fn === 'SUM' && /revenue|amount|sales|arr|mrr/i.test(column))
2221
- return 'revenue_total';
2222
- return `${column}_${fn.toLowerCase()}`;
2223
- }
2224
2338
  function cleanSqlIdentifier(identifier) {
2225
2339
  return identifier.replace(/^["`]|["`]$/g, '').trim();
2226
2340
  }
@@ -2242,200 +2356,22 @@ function relationLookupKeys(relation) {
2242
2356
  keys.add(parts[parts.length - 1]);
2243
2357
  return Array.from(keys);
2244
2358
  }
2245
- function normalizeForEntityMatch(value) {
2246
- return value.toLowerCase().replace(/[^a-z0-9.%+-]+/g, ' ').replace(/\s+/g, ' ').trim();
2247
- }
2248
- function uniqueMatchedEntityFilters(filters) {
2249
- const seen = new Set();
2250
- const unique = [];
2251
- for (const filter of filters) {
2252
- const key = `${filter.column.toLowerCase()}\0${filter.value.toLowerCase()}`;
2253
- if (seen.has(key))
2254
- continue;
2255
- seen.add(key);
2256
- unique.push(filter);
2257
- }
2258
- return unique;
2259
- }
2260
2359
  function uniqueDrilldownStrings(values) {
2261
2360
  return Array.from(new Set(values));
2262
2361
  }
2263
- function isTemporalDrilldownValue(value) {
2264
- return mentionsRelativeTime(value, undefined) || /^\d{4}-\d{2}-\d{2}/.test(value);
2265
- }
2266
- function escapeRegExp(value) {
2267
- return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
2268
- }
2269
- function buildContextPackAwareProposal(input) {
2270
- if (!isGeneratedAgentIntent(input.intent))
2271
- return undefined;
2272
- if (!input.contextPack)
2273
- return undefined;
2274
- const profileProposal = buildContextPackProfileProposal(input.question, input.contextPack);
2275
- if (profileProposal)
2276
- return profileProposal;
2277
- const lower = input.question.toLowerCase();
2278
- if (!/\b(least|lowest|fewest|bottom|min(?:imum)?)\b/.test(lower))
2279
- return undefined;
2280
- for (const object of input.contextPack.objects) {
2281
- if (object.objectType !== 'dql_block' || object.status !== 'certified')
2282
- continue;
2283
- const sql = typeof object.payload?.sql === 'string' ? object.payload.sql.trim() : '';
2284
- if (!sql || !/\border\s+by\b/i.test(sql) || !/\bdesc\b/i.test(sql))
2285
- continue;
2286
- const inverted = invertRankingSql(sql);
2287
- if (!inverted || inverted === sql)
2288
- continue;
2289
- return {
2290
- text: `Generated a review-required least-ranking query by using certified block "${object.name}" as context and reversing its ranking direction. This result is uncertified until reviewed and promoted.`,
2291
- sql: ensurePreviewLimit(inverted, 10),
2292
- viz: 'table',
2293
- };
2294
- }
2295
- return undefined;
2296
- }
2297
- function buildContextPackProfileProposal(question, contextPack) {
2298
- const entityTexts = entityMentionsForProfile(question, contextPack);
2299
- if (entityTexts.length === 0 || !isEntityProfileQuestion(question, contextPack))
2300
- return undefined;
2301
- const sources = [...(contextPack.allowedSqlContext?.sourceBlockSql ?? [])]
2302
- .sort((a, b) => {
2303
- const preferred = contextPack.routeDecision.certifiedApplicability?.objectKey;
2304
- if (preferred && a.objectKey === preferred)
2305
- return -1;
2306
- if (preferred && b.objectKey === preferred)
2307
- return 1;
2308
- return 0;
2309
- });
2310
- for (const source of sources) {
2311
- const shape = extractSimpleSelectShape(source.sql);
2312
- if (!shape)
2313
- continue;
2314
- const entityColumn = pickEntityColumnFromSelectExpressions(shape.selectExpressions, question);
2315
- if (!entityColumn)
2316
- continue;
2317
- const selectExpressions = uniqueSqlSelectExpressions(shape.selectExpressions).slice(0, 24);
2318
- if (selectExpressions.length === 0)
2319
- continue;
2320
- const orderColumn = selectExpressions
2321
- .map(selectExpressionOutputName)
2322
- .find((column) => column && /\b(season|year|date|month|week|game_date|created_at)\b/i.test(column));
2323
- const sql = [
2324
- 'SELECT',
2325
- selectExpressions.map((expression) => ` ${expression}`).join(',\n'),
2326
- `FROM ${shape.relation}`,
2327
- `WHERE ${sqlIdentifier(entityColumn)} = ${sqlStringLiteral(entityTexts[0])}`,
2328
- orderColumn ? `ORDER BY ${sqlIdentifier(orderColumn)} DESC` : '',
2329
- 'LIMIT 50',
2330
- ].filter(Boolean).join('\n');
2331
- return {
2332
- text: `Prepared a review-required profile query for ${entityTexts[0]} by using certified block "${source.name}" as SQL-shape context. This result is uncertified until reviewed and promoted.`,
2333
- sql,
2334
- viz: 'table',
2335
- };
2336
- }
2337
- return undefined;
2338
- }
2339
- function pickEntityColumnFromSelectExpressions(expressions, question) {
2340
- const outputNames = expressions
2341
- .map(selectExpressionOutputName)
2342
- .filter((value) => Boolean(value));
2343
- const lower = question.toLowerCase();
2344
- const hints = [
2345
- /\bplayers?\b/.test(lower) ? 'player' : '',
2346
- /\bcustomers?\b/.test(lower) ? 'customer' : '',
2347
- /\baccounts?\b/.test(lower) ? 'account' : '',
2348
- /\busers?\b/.test(lower) ? 'user' : '',
2349
- /\bteams?\b/.test(lower) ? 'team' : '',
2350
- /\bproducts?\b/.test(lower) ? 'product' : '',
2351
- ].filter(Boolean);
2352
- const preferred = [
2353
- ...hints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
2354
- 'name',
2355
- 'full_name',
2356
- 'display_name',
2357
- ];
2358
- for (const wanted of preferred) {
2359
- const match = outputNames.find((name) => namesEqualLoose(name, wanted));
2360
- if (match)
2361
- return match;
2362
- }
2363
- return outputNames.find((name) => /(^|_)(name|title|email)$/.test(name.toLowerCase()));
2364
- }
2365
- function uniqueSqlSelectExpressions(expressions) {
2366
- const seen = new Set();
2367
- const unique = [];
2368
- for (const expression of expressions) {
2369
- const normalized = expression.replace(/\s+/g, ' ').trim().toLowerCase();
2370
- if (!normalized || seen.has(normalized))
2371
- continue;
2372
- seen.add(normalized);
2373
- unique.push(expression.replace(/\s+/g, ' ').trim());
2374
- }
2375
- return unique;
2376
- }
2377
- function invertRankingSql(sql) {
2378
- const withoutTrailingSemicolon = sql.replace(/;\s*$/, '').trim();
2379
- const inverted = withoutTrailingSemicolon.replace(/\border\s+by\s+([\s\S]*?)(\blimit\b|$)/i, (match, orderExpr, limitKeyword) => {
2380
- if (!/\bdesc\b/i.test(orderExpr))
2381
- return match;
2382
- const nextExpr = orderExpr
2383
- .replace(/\bDESC\b/gi, 'ASC')
2384
- .replace(/\bNULLS\s+FIRST\b/gi, 'NULLS LAST');
2385
- return `ORDER BY ${nextExpr}${limitKeyword}`;
2386
- });
2387
- return inverted !== withoutTrailingSemicolon ? inverted : undefined;
2388
- }
2389
- function ensurePreviewLimit(sql, limit) {
2390
- if (/\blimit\s+\d+\b/i.test(sql))
2391
- return sql;
2392
- return `${sql.replace(/;\s*$/, '').trim()}\nLIMIT ${limit}`;
2393
- }
2394
- function findSchemaTable(schemaContext, names) {
2395
- return schemaContext.find((table) => {
2396
- const tableNames = new Set([table.name, table.relation.split('.').at(-1) ?? table.relation].map((name) => name.toLowerCase()));
2397
- return names.some((name) => tableNames.has(name.toLowerCase()));
2398
- });
2399
- }
2400
- function findSchemaColumn(table, names) {
2401
- const byLower = new Map(table.columns.map((column) => [column.name.toLowerCase(), column.name]));
2402
- for (const name of names) {
2403
- const exact = byLower.get(name.toLowerCase());
2404
- if (exact)
2405
- return exact;
2406
- }
2407
- return undefined;
2408
- }
2409
- function sqlRelation(relation) {
2410
- return relation.split('.').map(sqlIdentifier).join('.');
2411
- }
2412
- function sqlIdentifier(identifier) {
2413
- return /^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)
2414
- ? identifier
2415
- : `"${identifier.replace(/"/g, '""')}"`;
2416
- }
2417
2362
  function humanizeIdentifier(identifier) {
2418
2363
  return identifier.replace(/[_-]+/g, ' ');
2419
2364
  }
2420
- function businessMeasurePhrase(identifier) {
2421
- const lower = identifier.toLowerCase();
2422
- if (lower.includes('lifetime_spend'))
2423
- return 'lifetime spend';
2424
- if (lower.includes('count_lifetime_orders') || lower.includes('lifetime_orders') || lower.includes('order_count')) {
2425
- return 'lifetime order count';
2426
- }
2427
- return humanizeIdentifier(identifier);
2428
- }
2429
2365
  function pickCertifiedArtifact(input) {
2430
2366
  // Hint match wins immediately: the active Skill's vocabulary points the
2431
2367
  // user at a specific block. We still validate it's certified.
2432
2368
  for (const hint of input.blockHints) {
2433
2369
  const node = input.kg.getNode(`block:${hint}`);
2434
- if (node && node.status === 'certified' && hasCompatibleCertifiedBlockMatch(input.question, node)) {
2370
+ if (node && node.status === 'certified' && hasCertifiedNodeFit(input.question, input.questionPlan, node)) {
2435
2371
  return { node, score: 1, snippet: undefined };
2436
2372
  }
2437
2373
  }
2438
- const executableHit = pickFirstCertifiedHit(input.executableArtifactHits, input.kg, input.excludedArtifactIds, input.question);
2374
+ const executableHit = pickFirstCertifiedHit(input.executableArtifactHits, input.kg, input.excludedArtifactIds, input.question, input.questionPlan);
2439
2375
  if (isBusinessDefinitionQuestion(input.question)) {
2440
2376
  if (executableHit && hasExactExecutableArtifactSignal(input.question, executableHit.node)) {
2441
2377
  return executableHit;
@@ -2463,7 +2399,7 @@ function pickCertifiedArtifact(input) {
2463
2399
  }
2464
2400
  return null;
2465
2401
  }
2466
- function pickFirstCertifiedHit(hits, kg, excludedNodeIds, question) {
2402
+ function pickFirstCertifiedHit(hits, kg, excludedNodeIds, question, questionPlan) {
2467
2403
  for (const hit of hits) {
2468
2404
  if (hit.score < CERTIFIED_HIT_THRESHOLD)
2469
2405
  break;
@@ -2471,7 +2407,9 @@ function pickFirstCertifiedHit(hits, kg, excludedNodeIds, question) {
2471
2407
  continue;
2472
2408
  if (!isCertifiedHit(hit, kg))
2473
2409
  continue;
2474
- if (question && hit.node.kind === 'block' && !hasCompatibleCertifiedBlockMatch(question, hit.node))
2410
+ if (question && questionPlan && hit.node.kind === 'block' && !hasCertifiedNodeFit(question, questionPlan, hit.node))
2411
+ continue;
2412
+ if (question && !questionPlan && hit.node.kind === 'block' && !hasCompatibleCertifiedBlockMatch(question, hit.node))
2475
2413
  continue;
2476
2414
  return hit;
2477
2415
  }
@@ -2492,7 +2430,7 @@ function pickCertifiedDrilldownArtifact(input) {
2492
2430
  continue;
2493
2431
  if (!isCertifiedHit(hit, input.kg))
2494
2432
  continue;
2495
- if (!hasCompatibleCertifiedBlockMatch(input.question, hit.node))
2433
+ if (!hasCertifiedNodeFit(input.question, input.questionPlan, hit.node, { allowInferredContract: true }))
2496
2434
  continue;
2497
2435
  if (!hasRequestedDrilldownOverlap(hit.node, requestedTerms))
2498
2436
  continue;
@@ -2592,6 +2530,40 @@ function hasCompatibleCertifiedBlockMatch(question, node) {
2592
2530
  return hasMeaningfulCertifiedBlockSignal(question, node)
2593
2531
  && hasCompatibleRankingDirection(question, node);
2594
2532
  }
2533
+ function hasCertifiedNodeFit(question, plan, node, options = {}) {
2534
+ if (!hasCompatibleRankingDirection(question, node))
2535
+ return false;
2536
+ const definitionLookup = isBusinessDefinitionQuestion(question) && objectNameInQuestion(question, node);
2537
+ const exactExampleMatch = (node.examples ?? []).some((example) => normalizeQuestion(example.question) === normalizeQuestion(question));
2538
+ const exactObjectRequest = objectNameInQuestion(question, node)
2539
+ && /\b(run|use|open|show|execute|certified|saved|block)\b/i.test(question);
2540
+ if (!definitionLookup && !exactExampleMatch && !exactObjectRequest && !hasMeaningfulCertifiedBlockSignal(question, node))
2541
+ return false;
2542
+ const fit = evaluateCertifiedBlockFit({
2543
+ question,
2544
+ plan,
2545
+ block: node,
2546
+ exactExampleMatch: exactExampleMatch || exactObjectRequest,
2547
+ definitionLookup,
2548
+ });
2549
+ if (certifiedFitAllowsTier1(fit))
2550
+ return true;
2551
+ return Boolean(options.allowInferredContract
2552
+ && fit.kind === 'exact'
2553
+ && fit.confidence === 'medium'
2554
+ && fit.missingDimensions.length === 0
2555
+ && fit.missingOutputs.length === 0
2556
+ && fit.unsupportedFilters.length === 0
2557
+ && !fit.grainMismatch);
2558
+ }
2559
+ function objectNameInQuestion(question, node) {
2560
+ const questionText = normalizeQuestion(question);
2561
+ const name = normalizeQuestion(node.name);
2562
+ return Boolean(name && questionText.includes(name));
2563
+ }
2564
+ function normalizeQuestion(value) {
2565
+ return value.toLowerCase().replace(/[_-]+/g, ' ').replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
2566
+ }
2595
2567
  function hasCompatibleRankingDirection(question, node) {
2596
2568
  const questionDirection = rankingDirectionFromText(question);
2597
2569
  if (!questionDirection)
@@ -2804,8 +2776,9 @@ function isFilteredEntityQuestion(question) {
2804
2776
  if (/\b(for|where|only|specific|single|individual|named|called)\b.+\b(account|accounts|customer|customers|product|products|sku|user|users)\b/i.test(lower)) {
2805
2777
  return true;
2806
2778
  }
2807
- if (/\b(account|customer|product|sku|user)\s+(?:id|name|email)\b/i.test(lower))
2808
- return true;
2779
+ if (/\b(account|customer|product|sku|user)\s+(?:id|name|email)\b/i.test(lower)) {
2780
+ return !mentionsEntityIdentifierAsRequestedOutput(lower);
2781
+ }
2809
2782
  if (/[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3}/.test(question) && /\b(revenue|sales|order|orders|spend|value|churn|usage|activity|performance|performed|metric|kpi)\b/i.test(lower)) {
2810
2783
  return true;
2811
2784
  }
@@ -2813,6 +2786,9 @@ function isFilteredEntityQuestion(question) {
2813
2786
  return true;
2814
2787
  return false;
2815
2788
  }
2789
+ function mentionsEntityIdentifierAsRequestedOutput(lowerQuestion) {
2790
+ return /\b(?:with|including|include|return|show|list|give|provide|columns?|fields?|results?)\b[^.?!]{0,100}\b(?:account|customer|product|sku|user)\s+(?:id|name|email)\b/i.test(lowerQuestion);
2791
+ }
2816
2792
  function looksLikeDataQuestion(question) {
2817
2793
  return /\b(show|list|find|what|which|how many|how much|compare|trend|revenue|account|accounts|customer|customers|order|orders|product|products|sales|metric|kpi|dashboard|performance|performed|user|users)\b/i.test(question);
2818
2794
  }
@@ -2970,14 +2946,268 @@ async function requestSqlRepair(input) {
2970
2946
  {
2971
2947
  role: 'user',
2972
2948
  content: [
2973
- 'The generated SQL failed during bounded preview execution.',
2949
+ 'The SQL preview for the review-required DQL artifact failed during bounded preview execution.',
2974
2950
  `Question: ${input.question}`,
2975
2951
  `Execution error: ${input.executionError}`,
2976
- 'Return one corrected read-only SQL query using only the runtime schema below.',
2952
+ 'Return one corrected read-only SQL query using only the runtime schema below, as a single ```json fenced object with summary, sql, viz, outputs, and optional dql metadata fields.',
2977
2953
  schema,
2978
2954
  ].join('\n\n'),
2979
2955
  },
2980
- ], { signal: input.signal });
2956
+ ], {
2957
+ signal: input.signal,
2958
+ // Reuse the run's already-ceiling-clamped effort. We deliberately do NOT bump
2959
+ // here: the effort was clamped to the user's Settings ceiling upstream, and
2960
+ // bumping would let an internal preview-repair exceed that cap. Escalation-level
2961
+ // repairs bump-then-clamp in the host (resolveRunReasoningEffort).
2962
+ reasoningEffort: input.reasoningEffort,
2963
+ });
2964
+ }
2965
+ async function generateProposalWithOptionalTools(input) {
2966
+ const tools = input.tools?.filter((tool) => tool.name && tool.description) ?? [];
2967
+ const options = {
2968
+ signal: input.signal,
2969
+ reasoningEffort: input.reasoningEffort,
2970
+ };
2971
+ if (!input.provider.generateWithTools || tools.length === 0) {
2972
+ return input.provider.generate(input.messages, options);
2973
+ }
2974
+ const toolBudget = proposalToolBudgetForQuestion(input.questionPlan, input.intent, {
2975
+ analysisDepth: input.analysisDepth,
2976
+ reasoningEffort: input.reasoningEffort,
2977
+ });
2978
+ const toolPolicy = {
2979
+ role: 'system',
2980
+ content: [
2981
+ 'You may use the supplied DQL tools to inspect semantic members, certified context, metadata context, and bounded repair options.',
2982
+ `Tool budget for this question: ${toolBudget.maxToolCalls} call(s) (${toolBudget.effortClass}: ${toolBudget.reason}). Stop as soon as a lane can answer.`,
2983
+ 'Prefer semantic compile before deep warehouse search when the semantic layer contains the requested metric/dimensions/time grain.',
2984
+ 'Use context expansion only to repair a named, known relation/column gap; do not loop on the same failed context.',
2985
+ 'Final response must be a single ```json fenced object with summary, sql, viz, outputs, and optional dql metadata fields.',
2986
+ ].join('\n'),
2987
+ };
2988
+ return input.provider.generateWithTools([...input.messages, toolPolicy], tools, {
2989
+ ...options,
2990
+ maxToolCalls: toolBudget.maxToolCalls,
2991
+ onToolCall: (event) => {
2992
+ const sink = input.toolCalls;
2993
+ if (sink)
2994
+ sink.push(evidenceToolCallFromEvent(event, sink.length + 1));
2995
+ },
2996
+ });
2997
+ }
2998
+ async function selectDeepGeneratedProposalCandidate(input) {
2999
+ const initial = await scoreDeepGeneratedProposalCandidate(input, {
3000
+ raw: input.initial.raw,
3001
+ parsed: input.initial.parsed,
3002
+ index: 1,
3003
+ });
3004
+ // Deep mode diversifies whenever we can COMPARE candidate results (an executor
3005
+ // is available) — so a valid-but-subtly-wrong first candidate can be out-voted
3006
+ // by execution-result agreement — or when the first candidate failed and needs a
3007
+ // repair alternative. Without an executor, a validated first candidate is the
3008
+ // most we can assess, so return it and skip the extra generations.
3009
+ const shouldDiversify = Boolean(input.executeGeneratedSql) || !initial.validationOk;
3010
+ if (!shouldDiversify) {
3011
+ return { selected: initial, notes: [] };
3012
+ }
3013
+ const candidates = [initial];
3014
+ const alternatives = await generateDeepAlternativeProposals(input);
3015
+ for (const [offset, raw] of alternatives.entries()) {
3016
+ if (!raw.trim())
3017
+ continue;
3018
+ candidates.push(await scoreDeepGeneratedProposalCandidate(input, {
3019
+ raw,
3020
+ parsed: parseProposal(raw),
3021
+ index: offset + 2,
3022
+ }));
3023
+ if (candidates.length >= 5)
3024
+ break;
3025
+ }
3026
+ const signatureCounts = new Map();
3027
+ for (const candidate of candidates) {
3028
+ if (!candidate.resultSignature)
3029
+ continue;
3030
+ signatureCounts.set(candidate.resultSignature, (signatureCounts.get(candidate.resultSignature) ?? 0) + 1);
3031
+ }
3032
+ for (const candidate of candidates) {
3033
+ const equivalenceCount = candidate.resultSignature ? signatureCounts.get(candidate.resultSignature) ?? 0 : 0;
3034
+ if (equivalenceCount > 1)
3035
+ candidate.score += equivalenceCount * 10;
3036
+ }
3037
+ const selected = candidates
3038
+ .slice()
3039
+ .sort((a, b) => b.score - a.score ||
3040
+ Number(b.validationOk) - Number(a.validationOk) ||
3041
+ Number(Boolean(b.result && !b.executionError)) - Number(Boolean(a.result && !a.executionError)) ||
3042
+ a.index - b.index)[0];
3043
+ if (!selected)
3044
+ return { notes: [] };
3045
+ const status = selected.validationOk
3046
+ ? selected.executionError
3047
+ ? `validated but preview failed: ${selected.executionError}`
3048
+ : selected.result
3049
+ ? `validated and previewed ${selected.result.rowCount.toLocaleString()} row(s)`
3050
+ : 'validated'
3051
+ : `failed validation: ${selected.validationError ?? 'unknown validation issue'}`;
3052
+ // Surface how much the candidates disagreed: distinct execution-result
3053
+ // signatures among the candidates that executed cleanly.
3054
+ const executedCandidates = candidates.filter((candidate) => candidate.resultSignature);
3055
+ const distinctResults = new Set(executedCandidates.map((candidate) => candidate.resultSignature)).size;
3056
+ const agreementNote = executedCandidates.length > 1
3057
+ ? distinctResults === 1
3058
+ ? ` All ${executedCandidates.length} executed candidates agreed on the result.`
3059
+ : ` Candidates disagreed: ${distinctResults} distinct results across ${executedCandidates.length} executed candidates — selected the highest-scoring.`
3060
+ : '';
3061
+ return {
3062
+ selected,
3063
+ notes: [
3064
+ `Deep candidate selection reviewed ${candidates.length} candidate${candidates.length === 1 ? '' : 's'} and selected candidate ${selected.index} (${status}).${agreementNote}`,
3065
+ ],
3066
+ };
3067
+ }
3068
+ async function scoreDeepGeneratedProposalCandidate(input, candidate) {
3069
+ const parsed = cloneParsedProposal(candidate.parsed);
3070
+ if (parsed.sql)
3071
+ parsed.sql = input.contextLedger.qualifySql(parsed.sql).sql;
3072
+ let validationOk = false;
3073
+ let validationError;
3074
+ let result;
3075
+ let executionError;
3076
+ let score = parsed.sql ? 10 : -200;
3077
+ if (parsed.sql) {
3078
+ const validation = input.contextLedger.validateSql(parsed.sql, {
3079
+ question: input.question,
3080
+ intent: input.intent,
3081
+ });
3082
+ validationOk = validation.ok;
3083
+ if (validation.ok) {
3084
+ score += 100;
3085
+ if (input.executeGeneratedSql) {
3086
+ try {
3087
+ result = await input.executeGeneratedSql(parsed.sql);
3088
+ score += 40;
3089
+ if (result.rowCount > 0)
3090
+ score += 8;
3091
+ const resultShape = validateAnswerResultShape(input.questionPlan, result);
3092
+ score -= resultShape.warnings.length * 6;
3093
+ if (generatedResultShapeIsPartial(resultShape))
3094
+ score -= 120;
3095
+ }
3096
+ catch (error) {
3097
+ executionError = error instanceof Error ? error.message : String(error);
3098
+ score -= 20;
3099
+ }
3100
+ }
3101
+ }
3102
+ else {
3103
+ validationError = validation.error;
3104
+ score -= 100;
3105
+ }
3106
+ if (parsed.outputs?.length)
3107
+ score += Math.min(6, parsed.outputs.length);
3108
+ }
3109
+ return {
3110
+ raw: candidate.raw,
3111
+ parsed,
3112
+ index: candidate.index,
3113
+ validationOk,
3114
+ validationError,
3115
+ result,
3116
+ executionError,
3117
+ resultSignature: result && !executionError ? resultEquivalenceSignature(result) : undefined,
3118
+ score,
3119
+ };
3120
+ }
3121
+ async function generateDeepAlternativeProposals(input) {
3122
+ const previousSql = input.initial.parsed.sql
3123
+ ? `\nInitial SQL candidate:\n\`\`\`sql\n${input.initial.parsed.sql}\n\`\`\``
3124
+ : '';
3125
+ // Diverse candidate styles (CHASE-SQL-style): each explores the solution space
3126
+ // differently so execution-result agreement between styles is a strong signal.
3127
+ const variants = [
3128
+ 'Create a second candidate that favors the most direct inspected relations and explicit joins.',
3129
+ 'Create a third candidate using QUERY-PLAN reasoning: first outline the grain, measures, dimensions, and join path as steps, then write SQL that follows that plan exactly.',
3130
+ 'Create a fourth candidate by DECOMPOSITION: break the question into sub-questions, solve each as a CTE, then compose the final SELECT — avoiding assumptions hidden in the first candidate.',
3131
+ ];
3132
+ const temperatures = [0.2, 0.35, 0.5];
3133
+ const out = [];
3134
+ for (const [index, instruction] of variants.entries()) {
3135
+ try {
3136
+ const raw = await input.provider.generate([
3137
+ ...input.messages,
3138
+ { role: 'assistant', content: input.initial.raw },
3139
+ {
3140
+ role: 'user',
3141
+ content: [
3142
+ 'Deep mode is allowed to compare multiple review-required SQL candidates before choosing one.',
3143
+ `Question: ${input.question}`,
3144
+ instruction,
3145
+ previousSql,
3146
+ 'Return only one ```json fenced object with summary, sql, viz, outputs, and optional dql metadata fields.',
3147
+ ].filter(Boolean).join('\n\n'),
3148
+ },
3149
+ ], {
3150
+ signal: input.signal,
3151
+ reasoningEffort: input.reasoningEffort,
3152
+ temperature: temperatures[index] ?? 0.4,
3153
+ });
3154
+ if (raw.trim())
3155
+ out.push(raw);
3156
+ }
3157
+ catch {
3158
+ // Alternative candidates are opportunistic; the initial candidate remains.
3159
+ }
3160
+ }
3161
+ return out;
3162
+ }
3163
+ function cloneParsedProposal(proposal) {
3164
+ return {
3165
+ text: proposal.text,
3166
+ ...(proposal.sql ? { sql: proposal.sql } : {}),
3167
+ ...(proposal.viz ? { viz: proposal.viz } : {}),
3168
+ ...(proposal.outputs?.length ? { outputs: proposal.outputs.slice() } : {}),
3169
+ ...(proposal.proposedEntity ? { proposedEntity: proposal.proposedEntity } : {}),
3170
+ ...(proposal.requestedFilters?.length ? { requestedFilters: proposal.requestedFilters.slice() } : {}),
3171
+ ...(proposal.requestedDimensions?.length ? { requestedDimensions: proposal.requestedDimensions.slice() } : {}),
3172
+ };
3173
+ }
3174
+ function resultEquivalenceSignature(result) {
3175
+ const columns = result.columns.map((column) => String(column)).join('|');
3176
+ const rows = result.rows.slice(0, 25).map((row) => stableResultRow(row)).join('\n');
3177
+ return `${columns}\n${rows}`;
3178
+ }
3179
+ function stableResultRow(row) {
3180
+ if (!row || typeof row !== 'object' || Array.isArray(row)) {
3181
+ return JSON.stringify(row);
3182
+ }
3183
+ const record = row;
3184
+ return JSON.stringify(Object.keys(record).sort().map((key) => [key, record[key]]));
3185
+ }
3186
+ function evidenceToolCallFromEvent(event, order) {
3187
+ return {
3188
+ name: event.name,
3189
+ status: event.isError ? 'failed' : 'checked',
3190
+ inputSummary: summarizeEvidencePayload(event.input),
3191
+ outputSummary: summarizeEvidencePayload(event.output),
3192
+ order,
3193
+ };
3194
+ }
3195
+ function summarizeEvidencePayload(value, maxLength = 700) {
3196
+ if (value === undefined)
3197
+ return undefined;
3198
+ let raw;
3199
+ try {
3200
+ raw = typeof value === 'string' ? value : JSON.stringify(value);
3201
+ }
3202
+ catch {
3203
+ raw = String(value);
3204
+ }
3205
+ if (!raw)
3206
+ return undefined;
3207
+ const compact = raw.replace(/\s+/g, ' ').trim();
3208
+ if (!compact)
3209
+ return undefined;
3210
+ return compact.length <= maxLength ? compact : `${compact.slice(0, maxLength - 3)}...`;
2981
3211
  }
2982
3212
  function isRetryableGeneratedSqlError(error) {
2983
3213
  return !/\b(read-only|readonly|select or with|unsafe|delete|insert|update|drop|alter|create|attach|copy|pragma)\b/i.test(error);
@@ -3122,6 +3352,21 @@ function buildCertifiedEvidence(input) {
3122
3352
  .slice(0, 4);
3123
3353
  return {
3124
3354
  route: [
3355
+ ...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({
3356
+ terminalLane: 'certified',
3357
+ lanes: {
3358
+ certified: {
3359
+ label: `Lane 1 answered from certified ${input.artifact.kind.replace('_', ' ')}`,
3360
+ detail: input.artifact.name,
3361
+ },
3362
+ semantic: {
3363
+ label: 'Lane 2 semantic compile skipped because certified context already answered',
3364
+ },
3365
+ generated: {
3366
+ label: 'Lane 3 generated DQL artifact skipped because certified context already answered',
3367
+ },
3368
+ },
3369
+ })),
3125
3370
  {
3126
3371
  tool: 'search_certified_artifacts',
3127
3372
  status: 'selected',
@@ -3185,10 +3430,12 @@ function buildCertifiedEvidence(input) {
3185
3430
  sourceTables,
3186
3431
  semanticObjects,
3187
3432
  validation: {
3188
- status: input.executionError ? 'failed' : 'passed',
3433
+ status: input.executionError ? 'failed' : input.resultShapeWarnings?.length ? 'warning' : 'passed',
3189
3434
  message: input.executionError
3190
3435
  ? 'The certified artifact matched, but execution returned an error.'
3191
- : 'Certified artifact routing passed; no generated SQL was promoted.',
3436
+ : input.resultShapeWarnings?.length
3437
+ ? `Certified artifact executed, but the result shape needs review: ${input.resultShapeWarnings.join(' ')}`
3438
+ : 'Certified artifact routing passed; no review-required DQL artifact was promoted.',
3192
3439
  },
3193
3440
  execution: executionEvidence(input.artifact, input.result, input.executionError, input.executorWasAvailable),
3194
3441
  citations: input.citations,
@@ -3211,8 +3458,35 @@ function buildGeneratedEvidence(input) {
3211
3458
  ]).slice(0, 6);
3212
3459
  const selectedAssets = uniqueAssets(selectedNodes.map(assetFromNode)).slice(0, 4);
3213
3460
  const selectedSemantic = input.activeTier === 'semantic_layer' && semanticObjects.length > 0;
3461
+ const certifiedFitStep = certifiedFitEvidenceStep(input.contextPack);
3462
+ const certifiedCandidateFitSteps = certifiedCandidateFitEvidenceSteps(input.contextPack);
3214
3463
  return {
3215
3464
  route: [
3465
+ ...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({
3466
+ terminalLane: input.terminalLane,
3467
+ lanes: {
3468
+ certified: {
3469
+ label: input.followUp?.kind === 'drilldown'
3470
+ ? 'Lane 1 checked for a distinct certified drilldown block'
3471
+ : 'Lane 1 checked certified blocks for exact answer fit',
3472
+ },
3473
+ semantic: {
3474
+ label: input.terminalLane === 'semantic'
3475
+ ? 'Lane 2 answered through semantic metric compile'
3476
+ : input.semanticHits.length > 0
3477
+ ? 'Lane 2 semantic context was checked but did not fully answer'
3478
+ : 'Lane 2 semantic context had no strong match',
3479
+ ...(input.terminalLane === 'semantic' && input.terminalDetail ? { detail: input.terminalDetail } : {}),
3480
+ },
3481
+ generated: {
3482
+ label: input.terminalLane === 'generated'
3483
+ ? 'Lane 3 prepared review-required DQL artifact with SQL preview'
3484
+ : 'Lane 3 generated SQL skipped because semantic compile answered',
3485
+ ...(input.terminalLane === 'generated' && input.terminalDetail ? { detail: input.terminalDetail } : {}),
3486
+ },
3487
+ },
3488
+ })),
3489
+ ...cascadeBudgetEvidenceRouteSteps(input.budgetTrace),
3216
3490
  {
3217
3491
  tool: 'search_certified_artifacts',
3218
3492
  status: 'checked',
@@ -3223,6 +3497,8 @@ function buildGeneratedEvidence(input) {
3223
3497
  : 'No certified artifact was strong enough for this question',
3224
3498
  detail: input.followUp?.sourceBlockName,
3225
3499
  },
3500
+ ...(certifiedFitStep ? [certifiedFitStep] : []),
3501
+ ...certifiedCandidateFitSteps,
3226
3502
  {
3227
3503
  tool: 'propose_drilldown',
3228
3504
  status: input.followUp?.kind === 'drilldown' ? 'checked' : 'skipped',
@@ -3233,6 +3509,7 @@ function buildGeneratedEvidence(input) {
3233
3509
  ? [...(input.followUp.filters ?? []), ...(input.followUp.dimensions ?? [])].join(', ')
3234
3510
  : undefined,
3235
3511
  },
3512
+ ...providerToolEvidenceRouteSteps(input.toolCalls),
3236
3513
  {
3237
3514
  tool: 'search_business_context',
3238
3515
  status: businessAssets.length > 0 ? 'checked' : 'skipped',
@@ -3261,7 +3538,7 @@ function buildGeneratedEvidence(input) {
3261
3538
  {
3262
3539
  tool: 'validate_sql',
3263
3540
  status: 'checked',
3264
- label: 'SQL is generated and requires host validation before certification',
3541
+ label: 'SQL preview for DQL artifact requires host validation before certification',
3265
3542
  },
3266
3543
  {
3267
3544
  tool: 'execute_generated_sql',
@@ -3273,10 +3550,10 @@ function buildGeneratedEvidence(input) {
3273
3550
  ? 'skipped'
3274
3551
  : 'skipped',
3275
3552
  label: input.executionError
3276
- ? 'Generated SQL preview failed'
3553
+ ? 'SQL preview failed'
3277
3554
  : input.result
3278
- ? 'Executed generated SQL as bounded preview'
3279
- : 'Generated SQL preview not requested',
3555
+ ? 'Executed SQL preview for DQL artifact'
3556
+ : 'SQL preview not requested',
3280
3557
  detail: input.executionError ?? (input.result ? `${input.result.rowCount} rows` : undefined),
3281
3558
  },
3282
3559
  {
@@ -3310,19 +3587,20 @@ function buildGeneratedEvidence(input) {
3310
3587
  selectedAssets,
3311
3588
  sourceTables,
3312
3589
  semanticObjects,
3590
+ ...(input.toolCalls?.length ? { toolCalls: input.toolCalls } : {}),
3313
3591
  validation: {
3314
3592
  status: 'warning',
3315
3593
  message: input.followUp?.kind === 'drilldown'
3316
- ? 'Generated drilldown SQL is not certified. It should be validated, reviewed, and promoted only after analyst approval.'
3317
- : 'Generated SQL is not certified. It should be validated, reviewed, and promoted only after analyst approval.',
3594
+ ? 'Review-required drilldown DQL artifact is not certified. Its SQL preview should be validated, reviewed, and promoted only after analyst approval.'
3595
+ : 'Review-required DQL artifact is not certified. Its SQL preview should be validated, reviewed, and promoted only after analyst approval.',
3318
3596
  },
3319
3597
  execution: {
3320
3598
  status: input.executionError ? 'failed' : input.result ? 'executed' : 'not_requested',
3321
3599
  message: input.executionError
3322
3600
  ? input.executionError
3323
3601
  : input.result
3324
- ? 'Executed generated SQL as an uncertified bounded preview.'
3325
- : 'Generated SQL was returned for review; execution is handled by the host after validation.',
3602
+ ? 'Executed SQL preview as an uncertified bounded preview for the DQL artifact.'
3603
+ : 'Review-required DQL artifact was returned for review; SQL preview execution is handled by the host after validation.',
3326
3604
  rowCount: input.result?.rowCount,
3327
3605
  executionTime: input.result?.executionTime,
3328
3606
  },
@@ -3330,9 +3608,110 @@ function buildGeneratedEvidence(input) {
3330
3608
  analysisPlan: input.analysisPlan,
3331
3609
  };
3332
3610
  }
3611
+ function certifiedFitEvidenceStep(contextPack) {
3612
+ const fit = contextPack?.routeDecision?.blockFit;
3613
+ if (!fit)
3614
+ return undefined;
3615
+ const applicability = contextPack?.routeDecision?.certifiedApplicability;
3616
+ const allowed = certifiedFitAllowsTier1(fit);
3617
+ return {
3618
+ tool: 'check_certified_fit',
3619
+ status: allowed ? 'selected' : 'checked',
3620
+ label: allowed
3621
+ ? `Certified block fit passed${applicability?.name ? ` for ${applicability.name}` : ''}`
3622
+ : `Certified block kept as context${applicability?.name ? `: ${applicability.name}` : ''}`,
3623
+ detail: fit.reasons.length > 0
3624
+ ? fit.reasons.join('; ')
3625
+ : allowed
3626
+ ? 'Certified block covers the requested answer contract.'
3627
+ : 'Certified block did not prove an exact answer contract match.',
3628
+ };
3629
+ }
3630
+ function certifiedCandidateFitEvidenceSteps(contextPack) {
3631
+ const candidates = contextPack?.retrievalDiagnostics.certifiedCandidateFits ?? [];
3632
+ return candidates
3633
+ .slice(0, 4)
3634
+ .map((candidate) => ({
3635
+ tool: 'check_certified_candidate_fit',
3636
+ status: candidate.action === 'certified_answer'
3637
+ ? 'selected'
3638
+ : 'checked',
3639
+ label: certifiedCandidateFitLabel(candidate.name, candidate.action),
3640
+ detail: [
3641
+ `applicability=${candidate.applicabilityKind}`,
3642
+ `fit=${candidate.fit.kind}/${candidate.fit.confidence}`,
3643
+ candidate.fit.reasons.join('; '),
3644
+ ].filter(Boolean).join(' | '),
3645
+ }));
3646
+ }
3647
+ function certifiedCandidateFitLabel(name, action) {
3648
+ switch (action) {
3649
+ case 'certified_answer':
3650
+ return `Certified candidate selected: ${name}`;
3651
+ case 'context_only':
3652
+ return `Certified candidate used as context only: ${name}`;
3653
+ case 'eligible_not_selected':
3654
+ return `Certified candidate fit passed but was not selected: ${name}`;
3655
+ case 'rejected_for_fit':
3656
+ return `Certified candidate rejected for answer fit: ${name}`;
3657
+ }
3658
+ }
3659
+ function providerToolEvidenceRouteSteps(toolCalls) {
3660
+ return (toolCalls ?? []).slice(0, 8).map((call) => ({
3661
+ tool: call.name,
3662
+ status: call.status,
3663
+ label: call.status === 'failed'
3664
+ ? `Provider tool failed: ${call.name}`
3665
+ : `Provider tool observed: ${call.name}`,
3666
+ detail: [
3667
+ call.inputSummary ? `input=${truncateEvidenceDetail(call.inputSummary)}` : '',
3668
+ call.outputSummary ? `output=${truncateEvidenceDetail(call.outputSummary)}` : '',
3669
+ ].filter(Boolean).join(' | ') || undefined,
3670
+ }));
3671
+ }
3672
+ function cascadeBudgetEvidenceRouteSteps(trace) {
3673
+ if (!trace)
3674
+ return [];
3675
+ const { usage, limits } = trace;
3676
+ const used = usage.laneRegroundAttemptsUsed + usage.laneExecutionAttemptsUsed + usage.engineEscalationsUsed;
3677
+ if (used === 0)
3678
+ return [];
3679
+ return [{
3680
+ tool: 'cascade_budget',
3681
+ status: 'checked',
3682
+ label: `Repair budget used: re-ground ${usage.laneRegroundAttemptsUsed}/${limits.lane.reground}, execution ${usage.laneExecutionAttemptsUsed}/${limits.lane.execution}`,
3683
+ detail: `engine escalations ${usage.engineEscalationsUsed}/${limits.engineEscalations}`,
3684
+ }];
3685
+ }
3686
+ function truncateEvidenceDetail(value, maxLength = 240) {
3687
+ return value.length <= maxLength ? value : `${value.slice(0, maxLength - 3)}...`;
3688
+ }
3333
3689
  function buildNoAnswerEvidence(input) {
3334
3690
  return {
3335
3691
  route: [
3692
+ ...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({
3693
+ terminalLane: 'refusal',
3694
+ lanes: {
3695
+ certified: {
3696
+ label: input.artifactHits.length > 0
3697
+ ? 'Lane 1 certified blocks were checked but not selected'
3698
+ : 'Lane 1 found no certified block match',
3699
+ },
3700
+ semantic: {
3701
+ label: input.semanticHits.length > 0
3702
+ ? 'Lane 2 semantic context was checked but did not answer'
3703
+ : 'Lane 2 found no semantic match',
3704
+ },
3705
+ generated: {
3706
+ label: 'Lane 3 could not produce a validated review-required DQL artifact',
3707
+ },
3708
+ refusal: {
3709
+ label: 'Lane 4 returned an honest refusal or clarification',
3710
+ detail: input.reason,
3711
+ },
3712
+ },
3713
+ })),
3714
+ ...cascadeBudgetEvidenceRouteSteps(input.budgetTrace),
3336
3715
  {
3337
3716
  tool: 'search_certified_artifacts',
3338
3717
  status: input.artifactHits.length > 0 ? 'checked' : 'skipped',
@@ -3353,6 +3732,7 @@ function buildNoAnswerEvidence(input) {
3353
3732
  status: input.manifestHits.length > 0 ? 'checked' : 'skipped',
3354
3733
  label: input.manifestHits.length > 0 ? 'dbt context considered' : 'No dbt match',
3355
3734
  },
3735
+ ...providerToolEvidenceRouteSteps(input.toolCalls),
3356
3736
  {
3357
3737
  tool: 'validate_sql',
3358
3738
  status: 'failed',
@@ -3374,6 +3754,7 @@ function buildNoAnswerEvidence(input) {
3374
3754
  selectedAssets: [],
3375
3755
  sourceTables: uniqueAssets(input.manifestHits.map((hit) => assetFromNode(hit.node))).slice(0, 6),
3376
3756
  semanticObjects: uniqueAssets(input.semanticHits.map((hit) => assetFromNode(hit.node))).slice(0, 6),
3757
+ ...(input.toolCalls?.length ? { toolCalls: input.toolCalls } : {}),
3377
3758
  validation: {
3378
3759
  status: 'failed',
3379
3760
  message: input.reason,
@@ -3423,7 +3804,7 @@ function schemaContextAssets(schemaContext) {
3423
3804
  nodeId: `runtime_schema:${table.relation}`,
3424
3805
  kind: 'runtime_schema',
3425
3806
  name: table.relation,
3426
- description: table.description ?? `${table.columns.length} runtime column${table.columns.length === 1 ? '' : 's'} available for generated SQL.`,
3807
+ description: table.description ?? `${table.columns.length} runtime column${table.columns.length === 1 ? '' : 's'} available for DQL artifact SQL previews.`,
3427
3808
  sourceTier: 'project',
3428
3809
  certification: 'ai_generated',
3429
3810
  provenance: table.source ?? 'runtime information_schema',
@@ -3442,11 +3823,18 @@ function schemaCitations(schemaContext, limit) {
3442
3823
  }));
3443
3824
  }
3444
3825
  function certificationForNode(node) {
3445
- if (node.status === 'certified' || node.certification === 'certified')
3826
+ const certification = node.certification;
3827
+ if (node.status === 'certified' || certification === 'certified')
3446
3828
  return 'certified';
3447
- if (node.certification === 'analyst_review_required')
3829
+ if (certification === 'reviewed' || node.status === 'review' || node.status === 'reviewed')
3830
+ return 'reviewed';
3831
+ if (certification === 'conflict')
3832
+ return 'conflict';
3833
+ if (certification === 'insufficient_context')
3834
+ return 'insufficient_context';
3835
+ if (certification === 'analyst_review_required')
3448
3836
  return 'analyst_review_required';
3449
- if (node.certification === 'ai_generated' || node.certification === 'uncertified')
3837
+ if (certification === 'ai_generated' || certification === 'uncertified')
3450
3838
  return 'ai_generated';
3451
3839
  return undefined;
3452
3840
  }