@duckcodeailabs/dql-cli 1.6.31 → 1.6.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/dist/apps-api.d.ts.map +1 -1
  2. package/dist/apps-api.js +3 -1
  3. package/dist/apps-api.js.map +1 -1
  4. package/dist/args.d.ts +18 -0
  5. package/dist/args.d.ts.map +1 -1
  6. package/dist/args.js +37 -0
  7. package/dist/args.js.map +1 -1
  8. package/dist/assets/dql-notebook/assets/index-B6x1cRJo.js +5698 -0
  9. package/dist/assets/dql-notebook/index.html +1 -1
  10. package/dist/commands/agent.d.ts +47 -1
  11. package/dist/commands/agent.d.ts.map +1 -1
  12. package/dist/commands/agent.js +635 -27
  13. package/dist/commands/agent.js.map +1 -1
  14. package/dist/commands/app.d.ts.map +1 -1
  15. package/dist/commands/app.js +3 -1
  16. package/dist/commands/app.js.map +1 -1
  17. package/dist/commands/diff.d.ts.map +1 -1
  18. package/dist/commands/diff.js +111 -22
  19. package/dist/commands/diff.js.map +1 -1
  20. package/dist/commands/eval-judge.d.ts +37 -0
  21. package/dist/commands/eval-judge.d.ts.map +1 -0
  22. package/dist/commands/eval-judge.js +99 -0
  23. package/dist/commands/eval-judge.js.map +1 -0
  24. package/dist/commands/eval.d.ts +27 -1
  25. package/dist/commands/eval.d.ts.map +1 -1
  26. package/dist/commands/eval.js +143 -6
  27. package/dist/commands/eval.js.map +1 -1
  28. package/dist/commands/mcp.js +27 -3
  29. package/dist/commands/mcp.js.map +1 -1
  30. package/dist/index.js +6 -0
  31. package/dist/index.js.map +1 -1
  32. package/dist/llm/analytics-tools.d.ts.map +1 -1
  33. package/dist/llm/analytics-tools.js +12 -7
  34. package/dist/llm/analytics-tools.js.map +1 -1
  35. package/dist/llm/answer-loop-tools.d.ts +7 -0
  36. package/dist/llm/answer-loop-tools.d.ts.map +1 -0
  37. package/dist/llm/answer-loop-tools.js +67 -0
  38. package/dist/llm/answer-loop-tools.js.map +1 -0
  39. package/dist/llm/proposal-metadata.d.ts +5 -0
  40. package/dist/llm/proposal-metadata.d.ts.map +1 -0
  41. package/dist/llm/proposal-metadata.js +27 -0
  42. package/dist/llm/proposal-metadata.js.map +1 -0
  43. package/dist/llm/providers/claude-agent-sdk.d.ts +6 -0
  44. package/dist/llm/providers/claude-agent-sdk.d.ts.map +1 -1
  45. package/dist/llm/providers/claude-agent-sdk.js +12 -2
  46. package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
  47. package/dist/llm/providers/claude-code.d.ts +7 -1
  48. package/dist/llm/providers/claude-code.d.ts.map +1 -1
  49. package/dist/llm/providers/claude-code.js +11 -2
  50. package/dist/llm/providers/claude-code.js.map +1 -1
  51. package/dist/llm/providers/dql-agent-provider.d.ts +29 -0
  52. package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
  53. package/dist/llm/providers/dql-agent-provider.js +757 -26
  54. package/dist/llm/providers/dql-agent-provider.js.map +1 -1
  55. package/dist/llm/tools.d.ts.map +1 -1
  56. package/dist/llm/tools.js +34 -137
  57. package/dist/llm/tools.js.map +1 -1
  58. package/dist/llm/types.d.ts +67 -2
  59. package/dist/llm/types.d.ts.map +1 -1
  60. package/dist/local-runtime.d.ts +160 -5
  61. package/dist/local-runtime.d.ts.map +1 -1
  62. package/dist/local-runtime.js +1753 -181
  63. package/dist/local-runtime.js.map +1 -1
  64. package/dist/package.json +10 -10
  65. package/dist/providers/oauth/claude-oauth.d.ts +3 -3
  66. package/dist/providers/oauth/claude-oauth.d.ts.map +1 -1
  67. package/dist/providers/oauth/claude-oauth.js +3 -3
  68. package/dist/providers/oauth/claude-oauth.js.map +1 -1
  69. package/dist/providers/oauth/codex-oauth.d.ts +2 -3
  70. package/dist/providers/oauth/codex-oauth.d.ts.map +1 -1
  71. package/dist/providers/oauth/codex-oauth.js +6 -3
  72. package/dist/providers/oauth/codex-oauth.js.map +1 -1
  73. package/dist/settings/provider-settings.d.ts +1 -0
  74. package/dist/settings/provider-settings.d.ts.map +1 -1
  75. package/dist/settings/provider-settings.js +1 -1
  76. package/dist/settings/provider-settings.js.map +1 -1
  77. package/package.json +11 -11
  78. package/dist/assets/dql-notebook/assets/index-BpPs6_rz.js +0 -5648
@@ -1,4 +1,5 @@
1
1
  import { execFileSync, execSync } from 'node:child_process';
2
+ import { createHash } from 'node:crypto';
2
3
  import { createServer } from 'node:http';
3
4
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
4
5
  import { createRequire } from 'node:module';
@@ -7,19 +8,19 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
7
8
  import Anthropic from '@anthropic-ai/sdk';
8
9
  import OpenAI from 'openai';
9
10
  import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
10
- import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
11
+ import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
11
12
  import { load as loadYaml } from 'js-yaml';
12
13
  import { listBlockTemplates } from './block-templates.js';
13
14
  import { getRunner as getLLMRunner } from './llm/index.js';
14
15
  import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
15
16
  import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
16
- import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, clampReasoningEffort, bumpReasoningEffort, } from '@duckcodeailabs/dql-agent';
17
+ import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
17
18
  import { gatherProposeEnrichment } from './propose-enrich.js';
18
19
  import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
19
- import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
20
+ import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
20
21
  import { ClaudeCodeCliProvider, CodexCliProvider } from './providers/subscription-cli.js';
21
- import { ClaudeOAuthManager, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
22
- import { CodexOAuthManager, codexOAuthConnected, CODEX_OAUTH_MODELS, CODEX_OAUTH_DEFAULT_MODEL, } from './providers/oauth/codex-oauth.js';
22
+ import { ClaudeOAuthManager, ClaudeOAuthProvider, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
23
+ import { CodexOAuthManager, CodexOAuthProvider, codexOAuthConnected, CODEX_OAUTH_MODELS, CODEX_OAUTH_DEFAULT_MODEL, } from './providers/oauth/codex-oauth.js';
23
24
  import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
24
25
  import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
25
26
  import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
@@ -78,6 +79,16 @@ function parseAgentRunRequestedMode(value) {
78
79
  ? value
79
80
  : undefined;
80
81
  }
82
+ function parseAgentRunReasoningEffort(value) {
83
+ return value === 'low' || value === 'medium' || value === 'high'
84
+ ? value
85
+ : undefined;
86
+ }
87
+ function parseAgentRunAnalysisDepth(value) {
88
+ return value === 'quick' || value === 'deep'
89
+ ? value
90
+ : undefined;
91
+ }
81
92
  function parseAgentRunSelectedObject(value) {
82
93
  const record = agentRunRecord(value);
83
94
  if (!record)
@@ -105,7 +116,7 @@ function parseAgentRunHistory(value) {
105
116
  });
106
117
  return history.length > 0 ? history.slice(-20) : undefined;
107
118
  }
108
- function parseAgentRunRequestBody(body) {
119
+ export function parseAgentRunRequestBody(body) {
109
120
  const record = agentRunRecord(body);
110
121
  if (!record)
111
122
  return { error: 'Invalid JSON body.' };
@@ -128,11 +139,181 @@ function parseAgentRunRequestBody(body) {
128
139
  signals: signals,
129
140
  selectedObject,
130
141
  workspaceContext,
142
+ conversationContext: agentRunRecord(record.conversationContext),
131
143
  history: parseAgentRunHistory(record.history),
144
+ threadId: agentRunString(record.threadId),
132
145
  runId: agentRunString(record.runId),
146
+ reasoningEffort: parseAgentRunReasoningEffort(record.reasoningEffort),
147
+ analysisDepth: parseAgentRunAnalysisDepth(record.analysisDepth) ?? parseAgentRunAnalysisDepth(record.depth),
133
148
  },
134
149
  };
135
150
  }
151
+ export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
152
+ if (governedAnswer.kind === 'no_answer')
153
+ return false;
154
+ if (governedAnswer.kind === 'certified' || governedAnswer.certification === 'certified')
155
+ return false;
156
+ const finalText = (governedAnswer.answer ?? governedAnswer.text ?? '').trim();
157
+ if (governedAnswer.dqlArtifact && finalText)
158
+ return false;
159
+ return true;
160
+ }
161
+ // ── Server-side conversation threads ────────────────────────────────────────
162
+ // When a run carries a threadId, the persisted thread is the authoritative
163
+ // source of prior turns (survives refresh); the client-built context remains
164
+ // the fallback for embedders that never send a threadId.
165
+ async function conversationContextFromThread(store, threadId, clientContext, question) {
166
+ const turns = store.recentTurns(threadId, 8).map((turn) => {
167
+ const contract = turn.contract ?? {};
168
+ const topN = contract.topN;
169
+ const topNValue = typeof topN === 'number'
170
+ ? topN
171
+ : topN && typeof topN === 'object' && typeof topN.n === 'number'
172
+ ? topN.n
173
+ : undefined;
174
+ return compactConversationRecord({
175
+ id: turn.id,
176
+ question: turn.question,
177
+ answerSummary: turn.answerSummary,
178
+ sourceCertifiedBlock: turn.sourceCertifiedBlock,
179
+ route: turn.route,
180
+ trustLabel: turn.trustLabel,
181
+ certification: turn.certification,
182
+ contextPackId: turn.contextPackId,
183
+ dqlArtifact: turn.dqlArtifact,
184
+ cascade: turn.cascade,
185
+ sourceSql: turn.sql,
186
+ requestedFilters: conversationStringArray(contract.filters),
187
+ requestedDimensions: conversationStringArray(contract.dimensions),
188
+ requestedMeasures: conversationStringArray(contract.measures),
189
+ topN: topNValue,
190
+ result: turn.result
191
+ ? compactConversationRecord({
192
+ columns: turn.result.columns,
193
+ rowsSample: turn.result.rowsSample,
194
+ dimensionValues: turn.result.dimensionValues,
195
+ measureColumns: turn.result.measureColumns,
196
+ rowCount: turn.result.rowCount,
197
+ })
198
+ : undefined,
199
+ });
200
+ }).filter((turn) => Boolean(turn));
201
+ const thread = store.getThread(threadId);
202
+ // Bounded structured snapshot (working state + rolling summary + topic relation)
203
+ // for the answer loop's conversation-state prompt section.
204
+ const serverSnapshot = buildConversationSnapshot(store, threadId, { question });
205
+ if (serverSnapshot && question) {
206
+ // Semantic recall over OLDER turns (the recent window is already verbatim).
207
+ serverSnapshot.recalledTurns = await recallRelevantTurns(store, threadId, question, {
208
+ limit: 3,
209
+ excludeTurnIds: serverSnapshot.recentTurns.map((turn) => turn.id),
210
+ });
211
+ }
212
+ return {
213
+ ...(clientContext ?? {}),
214
+ conversationStateVersion: 1,
215
+ threadId,
216
+ ...(serverSnapshot ? { serverSnapshot } : {}),
217
+ ...(thread?.rollingSummary ? { conversationSummary: thread.rollingSummary } : {}),
218
+ ...(turns.length > 0 ? { turns, activeTurnId: turns[turns.length - 1].id } : {}),
219
+ };
220
+ }
221
+ /** Best-effort: persist a completed run as a conversation turn (never throws). */
222
+ function recordConversationTurn(store, threadId, run) {
223
+ if (!store || !threadId)
224
+ return;
225
+ try {
226
+ if (!store.getThread(threadId))
227
+ return;
228
+ const turn = store.appendTurn(threadId, conversationTurnInputFromRun(run));
229
+ // Fold the turn into the thread's working state + rolling summary (incremental).
230
+ advanceThreadState(store, threadId, turn);
231
+ }
232
+ catch {
233
+ // Conversation persistence is additive; a failed write must not fail the run.
234
+ }
235
+ }
236
+ function conversationTurnInputFromRun(run) {
237
+ const artifact = run.artifacts.find((candidate) => candidate.kind === 'answer')
238
+ ?? run.artifacts.find((candidate) => candidate.kind === 'research_run')
239
+ ?? run.artifacts[0];
240
+ const payload = agentRunRecord(artifact?.payload);
241
+ const result = agentRunRecord(payload?.result);
242
+ const columns = conversationResultColumns(result?.columns);
243
+ const rows = Array.isArray(result?.rows)
244
+ ? result.rows.filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row))).slice(0, 8)
245
+ : [];
246
+ const rowsSample = rows.map((row) => columns.map((column) => row[column]));
247
+ const contextPack = agentRunRecord(payload?.contextPack);
248
+ const questionPlan = agentRunRecord(contextPack?.questionPlan);
249
+ const requestedShape = agentRunRecord(questionPlan?.requestedShape);
250
+ const rowCountRaw = result?.rowCount;
251
+ const measureColumns = conversationMeasureColumns(columns, requestedShape, rows);
252
+ return {
253
+ question: run.question,
254
+ answerSummary: run.answer ?? run.summary,
255
+ route: run.route,
256
+ trustLabel: agentRunString(payload?.trustLabel) ?? run.trustState,
257
+ certification: agentRunString(payload?.certification),
258
+ sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
259
+ ?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
260
+ contextPackId: agentRunString(payload?.contextPackId),
261
+ sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
262
+ dqlArtifact: agentRunRecord(payload?.dqlArtifact),
263
+ cascade: agentRunRecord(payload?.cascade),
264
+ result: columns.length > 0 || rows.length > 0
265
+ ? {
266
+ columns,
267
+ rowsSample,
268
+ dimensionValues: conversationDimensionValues(columns, rows),
269
+ measureColumns,
270
+ rowCount: typeof rowCountRaw === 'number' ? rowCountRaw : rows.length || undefined,
271
+ }
272
+ : undefined,
273
+ contract: requestedShape,
274
+ };
275
+ }
276
+ function conversationResultColumns(value) {
277
+ if (!Array.isArray(value))
278
+ return [];
279
+ return value
280
+ .map((column) => typeof column === 'string'
281
+ ? column
282
+ : agentRunString(column?.name) ?? '')
283
+ .filter(Boolean)
284
+ .slice(0, 24);
285
+ }
286
+ function conversationMeasureColumns(columns, requestedShape, rows) {
287
+ const requested = conversationStringArray(requestedShape?.measures) ?? [];
288
+ const numericColumns = columns.filter((column) => rows.some((row) => typeof row[column] === 'number' && Number.isFinite(row[column])));
289
+ const metricNamedColumns = columns.filter((column) => /\b(revenue|sales|amount|total|count|average|avg|sum|spend|cost|margin|profit|value|points?|score|quantity|units?|rate|volume)\b/i.test(column.replace(/_/g, ' ')));
290
+ const unique = Array.from(new Set([...requested, ...numericColumns, ...metricNamedColumns]
291
+ .map((value) => value.trim())
292
+ .filter(Boolean)));
293
+ return unique.length > 0 ? unique.slice(0, 24) : undefined;
294
+ }
295
+ /** Distinct string values per low-cardinality column — the deictic-resolution fuel. */
296
+ function conversationDimensionValues(columns, rows) {
297
+ if (columns.length === 0 || rows.length === 0)
298
+ return undefined;
299
+ const out = {};
300
+ for (const column of columns.slice(0, 8)) {
301
+ const values = Array.from(new Set(rows.map((row) => row[column]).filter((value) => typeof value === 'string' && value.trim().length > 0))).slice(0, 24);
302
+ if (values.length > 0 && values.length <= 24)
303
+ out[column] = values;
304
+ }
305
+ return Object.keys(out).length > 0 ? out : undefined;
306
+ }
307
+ function conversationStringArray(value) {
308
+ if (!Array.isArray(value))
309
+ return undefined;
310
+ const values = value.filter((item) => typeof item === 'string' && item.trim().length > 0);
311
+ return values.length > 0 ? values : undefined;
312
+ }
313
+ function compactConversationRecord(record) {
314
+ const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
315
+ return entries.length > 0 ? Object.fromEntries(entries) : undefined;
316
+ }
136
317
  export async function startLocalServer(opts) {
137
318
  const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
138
319
  const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
@@ -378,7 +559,10 @@ export async function startLocalServer(opts) {
378
559
  let governedAnswer;
379
560
  let providerError;
380
561
  const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
381
- const reasoningEffort = resolveRunReasoningEffort(projectRoot, resolvedProvider, route, isRepair);
562
+ const reasoningEffort = request.reasoningEffort
563
+ ? resolveRequestedRunReasoningEffort(projectRoot, resolvedProvider, request.reasoningEffort, isRepair)
564
+ : resolveRunReasoningEffort(projectRoot, resolvedProvider, route, isRepair);
565
+ const analysisDepth = request.analysisDepth ?? (route === 'research' ? 'deep' : undefined);
382
566
  const contextEnvelope = {
383
567
  mode: 'agent_run',
384
568
  selectedObject: request.selectedObject,
@@ -386,24 +570,37 @@ export async function startLocalServer(opts) {
386
570
  instruction: [
387
571
  'Route through the governed DQL answer loop.',
388
572
  'Prefer certified DQL blocks when they exactly cover the question.',
389
- 'Generated SQL remains review-required and must use the bounded preview executor.',
573
+ 'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
390
574
  'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
391
575
  ...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
392
576
  ].join(' '),
393
577
  };
394
578
  const controller = new AbortController();
579
+ // Best-effort active warehouse dialect so Lane-2 semantic compiles emit
580
+ // dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
581
+ // connection is configured — the compiler then uses its default dialect.
582
+ let semanticDriver;
583
+ try {
584
+ semanticDriver = requireActiveConnection().driver;
585
+ }
586
+ catch {
587
+ semanticDriver = undefined;
588
+ }
395
589
  await runner.run({
396
590
  provider: resolvedProvider,
397
591
  messages: [
398
592
  ...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
399
593
  { role: 'user', content: isRepair ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}` : request.question },
400
594
  ],
595
+ conversationContext: request.conversationContext,
401
596
  upstream: {
402
597
  cellId: `agent-run:${request.selectedObject?.kind ?? 'workspace'}:${request.selectedObject?.id ?? request.runId ?? 'auto'}`,
403
598
  sql: JSON.stringify(contextEnvelope, null, 2),
404
599
  },
405
600
  reasoningEffort,
601
+ ...(analysisDepth ? { analysisDepth } : {}),
406
602
  projectRoot,
603
+ ...(semanticDriver ? { semanticDriver } : {}),
407
604
  executeCertifiedBlock: executeCertifiedBlockForAgent,
408
605
  executeGeneratedSql: executeGeneratedSqlForAgent,
409
606
  getSchemaContext: getSchemaContextForAgent,
@@ -420,6 +617,22 @@ export async function startLocalServer(opts) {
420
617
  }
421
618
  return governedAnswer;
422
619
  }
620
+ function resolvedRunRouteFromAnswer(governedAnswer) {
621
+ return routeForCascadeAnswerTier(governedAnswer.route?.tier);
622
+ }
623
+ function groundingGapRepairHint(governedAnswer) {
624
+ const details = governedAnswer.refusalDetails;
625
+ if (!details)
626
+ return governedAnswer.text;
627
+ const offending = details.offending;
628
+ const tokens = [
629
+ details.code ? `code=${details.code}` : undefined,
630
+ offending?.relation ? `relation=${offending.relation}` : undefined,
631
+ offending?.column ? `column=${offending.column}` : undefined,
632
+ details.message ? `message=${details.message}` : undefined,
633
+ ].filter((token) => Boolean(token));
634
+ return tokens.length > 0 ? tokens.join('; ') : governedAnswer.text;
635
+ }
423
636
  const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emitAnswerDelta }) => {
424
637
  let governedAnswer;
425
638
  try {
@@ -447,14 +660,20 @@ export async function startLocalServer(opts) {
447
660
  ],
448
661
  };
449
662
  }
663
+ const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
450
664
  const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
451
- const needsClarification = governedAnswer.kind === 'no_answer';
665
+ const isGroundingGap = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'grounding_gap';
666
+ const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
667
+ const groundingRepairHint = isGroundingGap ? groundingGapRepairHint(governedAnswer) : undefined;
668
+ // A provider outage is a retryable infrastructure failure, not a question the
669
+ // user needs to clarify — surface it as blocked so the UI offers a retry.
670
+ const needsClarification = governedAnswer.kind === 'no_answer' && !isGroundingGap && !isProviderError;
452
671
  const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
453
- // Synthesis: for a genuinely generated (non-certified) answer, compose an
454
- // adaptive, well-formatted reply over the executed rows streamed — instead of
455
- // the loop's raw draft. Certified answers keep their fast path (no extra call).
672
+ const runnableSql = governedAnswer.kind === 'no_answer' ? undefined : sql;
673
+ // Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
674
+ // that already produced DQL-first final prose keep the fast path.
456
675
  let synthesizedAnswer;
457
- if (!needsClarification && !isCertified) {
676
+ if (shouldSynthesizeAgentRunAnswer(governedAnswer)) {
458
677
  try {
459
678
  const provider = await createBlockStudioAssistProvider(projectRoot);
460
679
  if (provider) {
@@ -481,32 +700,55 @@ export async function startLocalServer(opts) {
481
700
  synthesizedAnswer = undefined;
482
701
  }
483
702
  }
484
- const status = needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
485
- const trustState = needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
486
- const stopReason = needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
703
+ const status = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
704
+ const trustState = isProviderError ? 'blocked' : needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
705
+ const stopReason = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
487
706
  const nextActions = needsClarification
488
707
  ? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
489
708
  : [
490
- ...(sql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
709
+ { id: 'create-block', label: governedAnswer.dqlArtifact ? 'Review DQL draft' : 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
491
710
  { id: 'research-gap', label: 'Research deeper', route: 'research' },
492
- { id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
711
+ ...(runnableSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
493
712
  ];
494
713
  return {
714
+ resolvedRoute,
715
+ answerTier: governedAnswer.route?.tier,
495
716
  summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
496
717
  answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
497
718
  status,
498
719
  trustState,
499
720
  stopReason,
500
- artifacts: needsClarification
501
- ? []
721
+ artifacts: governedAnswer.kind === 'no_answer'
722
+ // A refusal still keeps the DQL draft the answer loop produced (when any),
723
+ // so the "Review DQL draft" next-action isn't a dead link and the user can
724
+ // see the SQL that was about to run. Provider outages carry no draft.
725
+ ? (governedAnswer.dqlArtifact && !isProviderError
726
+ ? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
727
+ : [])
502
728
  : [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
503
729
  evaluations: [
504
- agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'),
730
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
731
+ plannedRoute: route,
732
+ resolvedRoute,
733
+ aiRoute: governedAnswer.route,
734
+ }),
505
735
  agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
506
736
  ? 'The answer came from certified DQL context.'
507
737
  : needsClarification
508
738
  ? 'The answer loop needs more context before producing a governed answer.'
509
739
  : 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
740
+ ...(isGroundingGap ? [
741
+ {
742
+ ...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
743
+ refusalCode: governedAnswer.refusalCode,
744
+ refusalDetails: governedAnswer.refusalDetails,
745
+ validationWarnings: governedAnswer.validationWarnings,
746
+ route: governedAnswer.route,
747
+ }),
748
+ suggestedRepair: groundingRepairHint,
749
+ repairAction: { kind: 'retry', hint: groundingRepairHint },
750
+ },
751
+ ] : []),
510
752
  ...(governedAnswer.executionError ? [
511
753
  agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
512
754
  ] : []),
@@ -529,8 +771,10 @@ export async function startLocalServer(opts) {
529
771
  const provider = await createBlockStudioAssistProvider(projectRoot);
530
772
  if (provider) {
531
773
  const system = buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext, request.audience ?? 'analyst');
774
+ const conversationMemory = renderConversationMemoryForPrompt(request.conversationContext);
532
775
  const messages = [
533
776
  { role: 'system', content: system },
777
+ ...(conversationMemory ? [{ role: 'system', content: conversationMemory }] : []),
534
778
  ...(request.history ?? []).slice(-6).map((message) => ({ role: message.role, content: message.text })),
535
779
  { role: 'user', content: request.question },
536
780
  ];
@@ -545,7 +789,8 @@ export async function startLocalServer(opts) {
545
789
  text = undefined;
546
790
  }
547
791
  if (!text) {
548
- text = buildConversationalFallback(kind, isGeneralKnowledge, request.question, suggestions);
792
+ text = buildConversationContextRecap(request.conversationContext)
793
+ ?? buildConversationalFallback(kind, isGeneralKnowledge, request.question, suggestions);
549
794
  emitAnswerDelta?.(text);
550
795
  }
551
796
  return {
@@ -564,7 +809,8 @@ export async function startLocalServer(opts) {
564
809
  conversation: conversationRunExecutor,
565
810
  certified_answer: answerRunExecutor,
566
811
  generated_answer: answerRunExecutor,
567
- research: async ({ runId, request, routeDecision, emit }) => {
812
+ research: async (researchContext) => {
813
+ const { runId, request, routeDecision, emit } = researchContext;
568
814
  const metrics = loadSemanticMetrics(projectRoot);
569
815
  let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
570
816
  const usedCertifiedOnly = blocks.length > 0;
@@ -585,6 +831,13 @@ export async function startLocalServer(opts) {
585
831
  // When the user explicitly picked research, investigate — don't collapse to one step.
586
832
  forceInvestigate: request.requestedMode === 'research',
587
833
  });
834
+ // Direct governed answer ("the metric answers this directly"): don't wrap it in
835
+ // a review-required research dossier — run the query through the answer loop and
836
+ // return the executed result table + DQL artifact. A dossier is only for genuine
837
+ // multi-step investigation (plan.done === false) or explicit research mode.
838
+ if (plan.done && !plan.followUp && request.requestedMode !== 'research') {
839
+ return answerRunExecutor(researchContext);
840
+ }
588
841
  const needsClarification = Boolean(plan.followUp);
589
842
  const notebookPath = agentRunNotebookPath(request, runId);
590
843
  const researchIntent = agentRunResearchIntent(request);
@@ -650,9 +903,15 @@ export async function startLocalServer(opts) {
650
903
  researchWorkspaceError = formatNotebookResearchStorageError(error);
651
904
  }
652
905
  }
653
- const researchResultData = coerceNarrateResultData(researchRun?.resultPreview);
654
- // Executed cleanly against real data (rows present, no workspace/exec error) → grounded, not review-required.
655
- const researchExecutedCleanly = Boolean(researchResultData) && !researchWorkspaceError && researchRun?.status !== 'error';
906
+ const researchResultPreview = researchRun?.resultPreview;
907
+ const researchResultData = coerceNarrateResultData(researchResultPreview);
908
+ const researchResultRecord = agentRunRecord(researchResultPreview);
909
+ // A query that ran and matched 0 rows STILL executed — treat it as a clean,
910
+ // grounded execution (not "no result"), so an empty answer is surfaced as
911
+ // "0 rows matched" rather than silently downgraded to review-required.
912
+ const researchDidExecute = Boolean(researchResultData) || Boolean(researchResultRecord && Array.isArray(researchResultRecord.rows));
913
+ const researchZeroRows = !researchResultData && researchDidExecute;
914
+ const researchExecutedCleanly = researchDidExecute && !researchWorkspaceError && researchRun?.status !== 'error';
656
915
  const narration = !needsClarification && researchResultData
657
916
  ? await narrateForAgentRun({
658
917
  question: request.question,
@@ -665,18 +924,22 @@ export async function startLocalServer(opts) {
665
924
  const summary = needsClarification
666
925
  ? 'Needs clarification before running deeper research.'
667
926
  : narration?.summary
668
- ?? (researchRun?.status === 'ready'
669
- ? 'Saved a grounded research dossier with context evidence and next review actions.'
670
- : researchRun?.status === 'error'
671
- ? 'Saved a research dossier, but the preview needs review before promotion.'
672
- : researchWorkspaceError
673
- ? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
674
- : plan.done
675
- ? 'Prepared a direct grounded-answer plan.'
676
- : 'Prepared a grounded research plan over real DQL assets.');
927
+ ?? (researchZeroRows
928
+ ? 'The query executed cleanly against real data and matched 0 rows.'
929
+ : researchRun?.status === 'ready'
930
+ ? 'Saved a grounded research dossier with context evidence and next review actions.'
931
+ : researchRun?.status === 'error'
932
+ ? 'Saved a research dossier, but the preview needs review before promotion.'
933
+ : researchWorkspaceError
934
+ ? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
935
+ : plan.done
936
+ ? 'Prepared a direct grounded-answer plan.'
937
+ : 'Prepared a grounded research plan over real DQL assets.');
677
938
  return {
678
939
  summary,
679
- answer: plan.followUp?.question ?? narration?.summary ?? researchRun?.summary,
940
+ answer: plan.followUp?.question ?? narration?.summary
941
+ ?? (researchZeroRows ? 'The query executed cleanly and matched 0 rows.' : undefined)
942
+ ?? researchRun?.summary,
680
943
  status: needsClarification ? 'needs_clarification' : 'needs_review',
681
944
  trustState: needsClarification ? 'not_applicable' : (researchExecutedCleanly ? 'grounded' : 'review_required'),
682
945
  stopReason: needsClarification ? 'needs_clarification' : 'human_review_required',
@@ -706,15 +969,17 @@ export async function startLocalServer(opts) {
706
969
  ? 'Research workspace storage is unavailable; the plan remains available in this agent run.'
707
970
  : 'No durable research record was created because the run needs clarification first.', { notebookPath, researchRunId: researchRun?.id, error: researchWorkspaceError }),
708
971
  agentRunEvaluation('result-executed', 'Executed against data', researchExecutedCleanly, researchExecutedCleanly ? 'info' : 'warning', researchExecutedCleanly
709
- ? 'The query executed cleanly against real data and returned rows.'
710
- : 'No executed result rows were available; the output stays exploratory pending review.', { rowCount: researchResultData && typeof researchResultData === 'object' && Array.isArray(researchResultData.rows) ? researchResultData.rows.length : 0 }),
972
+ ? (researchZeroRows
973
+ ? 'The query executed cleanly against real data and matched 0 rows.'
974
+ : 'The query executed cleanly against real data and returned rows.')
975
+ : 'No executed result was available; the output stays exploratory pending review.', { rowCount: Array.isArray(researchResultRecord?.rows) ? researchResultRecord.rows.length : 0 }),
711
976
  ],
712
977
  nextActions: needsClarification
713
978
  ? [{ id: 'answer-follow-up', label: 'Answer follow-up', route: 'research' }]
714
979
  : [
715
980
  ...(researchRun?.id ? [{ id: 'open-research', label: 'Open research dossier', artifactKind: 'research_run' }] : []),
716
- ...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
717
- { id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
981
+ { id: 'create-block', label: 'Review DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
982
+ ...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
718
983
  ],
719
984
  };
720
985
  },
@@ -729,12 +994,67 @@ export async function startLocalServer(opts) {
729
994
  agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', 'Generated SQL must be reviewed before it becomes certified analytics.'),
730
995
  ],
731
996
  nextActions: [
732
- { id: 'insert-sql', label: 'Insert SQL cell', artifactKind: 'sql_cell' },
733
- { id: 'create-block', label: 'Promote to DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
997
+ { id: 'insert-sql', label: 'Insert SQL preview', artifactKind: 'sql_cell' },
998
+ { id: 'create-block', label: 'Review as DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
734
999
  ],
735
1000
  };
736
1001
  },
737
1002
  dql_block_draft: async ({ request, routeDecision, attempt, repairHint }) => {
1003
+ const contextRecord = agentRunRecord(request.conversationContext);
1004
+ const carriedDqlArtifact = normalizeDqlArtifactReference(contextRecord?.dqlArtifact)
1005
+ ?? normalizeDqlArtifactReference(request.workspaceContext?.dqlArtifact);
1006
+ if (carriedDqlArtifact && (attempt ?? 0) === 0) {
1007
+ const sourceQuestion = agentRunString(contextRecord?.sourceQuestion) ?? request.question;
1008
+ const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
1009
+ question: sourceQuestion,
1010
+ dqlArtifact: carriedDqlArtifact,
1011
+ inputMode: 'upload',
1012
+ inputPath: carriedDqlArtifact.sourcePath,
1013
+ domain: agentRunWorkspaceValue(request, 'domain'),
1014
+ owner: agentRunWorkspaceValue(request, 'owner'),
1015
+ tags: ['agent-run', 'review-required'],
1016
+ contextPackId: agentRunString(contextRecord?.contextPackId),
1017
+ routeIntent: agentRunString(contextRecord?.route),
1018
+ sourceBlock: carriedDqlArtifact.sourcePath,
1019
+ }, semanticLayer);
1020
+ const candidate = session.candidates[0];
1021
+ const validation = candidate?.validation && typeof candidate.validation === 'object'
1022
+ ? candidate.validation
1023
+ : undefined;
1024
+ const ready = validation?.valid !== false;
1025
+ const payload = {
1026
+ target: 'block',
1027
+ name: candidate?.name ?? carriedDqlArtifact.name ?? 'DQL block draft',
1028
+ path: candidate?.draftSave?.path ?? candidate?.savedPath,
1029
+ importId: session.id,
1030
+ candidateId: candidate?.id,
1031
+ source: candidate?.dqlSource,
1032
+ dqlSource: candidate?.dqlSource,
1033
+ dqlArtifact: carriedDqlArtifact,
1034
+ certifierVerdict: {
1035
+ ready,
1036
+ diagnostics: validation?.diagnostics ?? [],
1037
+ },
1038
+ generation: session.generation,
1039
+ };
1040
+ return {
1041
+ summary: ready
1042
+ ? 'Saved the generated DQL artifact as a review-required block draft.'
1043
+ : 'Saved the generated DQL artifact as a block draft with review blockers or warnings.',
1044
+ artifacts: [agentRunArtifact('dql_block_draft', payload.name, payload, payload.path)],
1045
+ evaluations: [
1046
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to DQL block draft generation.'),
1047
+ agentRunEvaluation('dql-artifact-reuse', 'Used generated DQL artifact', true, 'info', 'Persisted the existing DQL artifact without regenerating it from SQL.', { importId: session.id, path: payload.path }),
1048
+ agentRunEvaluation('certification-boundary', 'Certification boundary', ready, 'warning', ready
1049
+ ? 'The draft has no automatic validation blockers, but certification still requires human review.'
1050
+ : 'The draft has validation blockers that must be resolved before certification review.', payload),
1051
+ ],
1052
+ nextActions: [
1053
+ { id: 'open-review', label: 'Open review checklist', artifactKind: 'dql_block_draft' },
1054
+ { id: 'build-app', label: 'Build app from block', route: 'app_build', artifactKind: 'app_draft' },
1055
+ ],
1056
+ };
1057
+ }
738
1058
  const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
739
1059
  const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
740
1060
  return {
@@ -902,9 +1222,27 @@ export async function startLocalServer(opts) {
902
1222
  getCatalogContext: () => buildAgentRunCatalogContext(),
903
1223
  });
904
1224
  const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
1225
+ // Server-side conversation threads: persisted multi-turn state (survives refresh).
1226
+ // Lazy so a failed native-sqlite load never blocks the runtime; conversation
1227
+ // persistence degrades to the per-request context instead.
1228
+ let conversationStoreInstance;
1229
+ const getConversationStore = () => {
1230
+ if (conversationStoreInstance !== undefined)
1231
+ return conversationStoreInstance;
1232
+ try {
1233
+ conversationStoreInstance = new ConversationStore(defaultConversationPath(projectRoot));
1234
+ }
1235
+ catch {
1236
+ conversationStoreInstance = null;
1237
+ }
1238
+ return conversationStoreInstance;
1239
+ };
905
1240
  const agentRunEngine = new AgentRunEngine({
906
1241
  store: agentRunStore,
907
- executors: agentRunExecutors,
1242
+ executors: {
1243
+ ...agentRunExecutors,
1244
+ ...(opts.agentRunExecutors ?? {}),
1245
+ },
908
1246
  gates: defaultAgentRunGates,
909
1247
  planner: agentRunPlanner,
910
1248
  router: agentRunRouter,
@@ -1063,25 +1401,42 @@ export async function startLocalServer(opts) {
1063
1401
  };
1064
1402
  };
1065
1403
  const getSchemaContextForAgent = async (question) => {
1404
+ const scanRuntimeSchema = async () => {
1405
+ if (!connection)
1406
+ return { ranked: [], snapshot: [] };
1407
+ const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
1408
+ FROM information_schema.columns
1409
+ WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
1410
+ ORDER BY table_schema, table_name, ordinal_position
1411
+ LIMIT 2000`, [], runtimeVariables({}), connection);
1412
+ return {
1413
+ ranked: buildAgentSchemaContext(question, result.rows),
1414
+ snapshot: buildAgentSchemaContext(question, result.rows, { includeUnscored: true, limit: 500 }),
1415
+ };
1416
+ };
1066
1417
  const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question).catch(() => []);
1067
1418
  if (catalogContext.length > 0) {
1068
1419
  if (!connection)
1069
1420
  return catalogContext;
1070
- const enriched = await enrichAgentSchemaContextWithValueMatches(question, catalogContext, executor, connection);
1071
- recordAgentRuntimeSchemaSnapshot(projectRoot, enriched, 'catalog enriched runtime schema');
1421
+ const runtimeScan = shouldAugmentAgentRuntimeSchema(question)
1422
+ ? await scanRuntimeSchema().catch(() => undefined)
1423
+ : undefined;
1424
+ const runtimeContext = runtimeScan?.ranked ?? [];
1425
+ const merged = mergeAgentSchemaContexts(catalogContext, runtimeContext);
1426
+ const enriched = await enrichAgentSchemaContextWithValueMatches(question, merged, executor, connection);
1427
+ recordAgentRuntimeSchemaSnapshot(projectRoot, !runtimeScan?.snapshot.length
1428
+ ? enriched
1429
+ : mergeAgentSchemaSampleValues(runtimeScan.snapshot, enriched), runtimeContext.length > 0
1430
+ ? 'full information_schema runtime scan for composite Ask AI question'
1431
+ : 'catalog enriched runtime schema');
1072
1432
  return enriched;
1073
1433
  }
1074
1434
  if (!connection)
1075
1435
  return [];
1076
1436
  try {
1077
- const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
1078
- FROM information_schema.columns
1079
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
1080
- ORDER BY table_schema, table_name, ordinal_position
1081
- LIMIT 2000`, [], runtimeVariables({}), connection);
1082
- const schemaContext = buildAgentSchemaContext(question, result.rows);
1083
- const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection);
1084
- recordAgentRuntimeSchemaSnapshot(projectRoot, enriched, 'information_schema runtime scan');
1437
+ const schemaContext = await scanRuntimeSchema();
1438
+ const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext.ranked, executor, connection);
1439
+ recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext.snapshot, 'full information_schema runtime scan');
1085
1440
  return enriched;
1086
1441
  }
1087
1442
  catch {
@@ -1147,6 +1502,7 @@ export async function startLocalServer(opts) {
1147
1502
  sql: JSON.stringify(contextEnvelope, null, 2),
1148
1503
  },
1149
1504
  reasoningEffort,
1505
+ analysisDepth: 'deep',
1150
1506
  projectRoot,
1151
1507
  executeCertifiedBlock: executeCertifiedBlockForAgent,
1152
1508
  executeGeneratedSql: executeGeneratedSqlForAgent,
@@ -1223,6 +1579,21 @@ export async function startLocalServer(opts) {
1223
1579
  intents: 0,
1224
1580
  notebooks: 0,
1225
1581
  },
1582
+ reviewMetrics: {
1583
+ totalReviewCount: 0,
1584
+ openReviewCount: 0,
1585
+ terminalReviewCount: 0,
1586
+ draftCreatedCount: 0,
1587
+ certifiedCount: 0,
1588
+ completedCount: 0,
1589
+ rejectedCount: 0,
1590
+ draftCreationRate: null,
1591
+ certifyConversionRate: null,
1592
+ medianOpenReviewAgeMs: null,
1593
+ medianTimeToDraftMs: null,
1594
+ medianTimeToCertificationMs: null,
1595
+ medianTimeToTerminalMs: null,
1596
+ },
1226
1597
  limit: input.limit,
1227
1598
  offset: input.offset ?? 0,
1228
1599
  });
@@ -1265,6 +1636,7 @@ export async function startLocalServer(opts) {
1265
1636
  const context = input.context === undefined ? run.context : input.context;
1266
1637
  let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
1267
1638
  let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
1639
+ let dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? run.dqlArtifact;
1268
1640
  const startedAt = new Date().toISOString();
1269
1641
  storage.updateRun(run.id, {
1270
1642
  domain,
@@ -1275,6 +1647,7 @@ export async function startLocalServer(opts) {
1275
1647
  context,
1276
1648
  generatedSql,
1277
1649
  reviewedSql,
1650
+ dqlArtifact,
1278
1651
  status: 'running',
1279
1652
  reviewStatus: 'needs_review',
1280
1653
  error: '',
@@ -1291,8 +1664,8 @@ export async function startLocalServer(opts) {
1291
1664
  intent,
1292
1665
  researchPattern: notebookResearchIntentPattern(intent),
1293
1666
  },
1294
- strictness: 'balanced',
1295
- limit: 100,
1667
+ strictness: 'exploratory',
1668
+ limit: 160,
1296
1669
  }).catch((error) => {
1297
1670
  const message = error instanceof Error ? error.message : String(error);
1298
1671
  return {
@@ -1334,6 +1707,7 @@ export async function startLocalServer(opts) {
1334
1707
  }, null, 2),
1335
1708
  },
1336
1709
  reasoningEffort: resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false),
1710
+ analysisDepth: 'deep',
1337
1711
  projectRoot,
1338
1712
  executeCertifiedBlock: executeCertifiedBlockForAgent,
1339
1713
  executeGeneratedSql: executeGeneratedSqlForAgent,
@@ -1353,8 +1727,9 @@ export async function startLocalServer(opts) {
1353
1727
  generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
1354
1728
  }
1355
1729
  else {
1730
+ dqlArtifact = normalizeDqlArtifactReference(governedAnswer.dqlArtifact) ?? dqlArtifact;
1356
1731
  generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
1357
- if (!generatedSql) {
1732
+ if (!generatedSql && !dqlArtifact) {
1358
1733
  generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
1359
1734
  }
1360
1735
  }
@@ -1416,7 +1791,7 @@ export async function startLocalServer(opts) {
1416
1791
  const evidence = {
1417
1792
  trustStatus: {
1418
1793
  label: governedAnswer?.trustLabel
1419
- ?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
1794
+ ?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : dqlArtifact ? 'AI-generated DQL artifact' : 'Metadata-grounded research plan'),
1420
1795
  reviewRequired: true,
1421
1796
  },
1422
1797
  contextPackId: contextPack.id,
@@ -1455,9 +1830,11 @@ export async function startLocalServer(opts) {
1455
1830
  ?? notebookResearchSummary(question, resultPreview, previewError);
1456
1831
  const recommendation = previewError
1457
1832
  ? 'Review the SQL, selected metadata, and connection context before rerunning.'
1458
- : sqlForPreview
1459
- ? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
1460
- : 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
1833
+ : dqlArtifact && !reviewedSql
1834
+ ? 'Review the DQL artifact, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
1835
+ : sqlForPreview
1836
+ ? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
1837
+ : 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
1461
1838
  const researchPlan = buildNotebookResearchPlan({
1462
1839
  run,
1463
1840
  evidence,
@@ -1465,6 +1842,7 @@ export async function startLocalServer(opts) {
1465
1842
  previewError,
1466
1843
  generatedSql,
1467
1844
  reviewedSql,
1845
+ dqlArtifact,
1468
1846
  routeDecision,
1469
1847
  });
1470
1848
  return storage.updateRun(run.id, {
@@ -1482,6 +1860,7 @@ export async function startLocalServer(opts) {
1482
1860
  researchPlan,
1483
1861
  generatedSql,
1484
1862
  reviewedSql,
1863
+ dqlArtifact,
1485
1864
  display: display && display.ok ? display.display : undefined,
1486
1865
  contextPackId: contextPack.id,
1487
1866
  routeDecision,
@@ -1508,10 +1887,48 @@ export async function startLocalServer(opts) {
1508
1887
  }
1509
1888
  };
1510
1889
  const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
1511
- const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
1512
- if (!sql)
1513
- throw new Error('Notebook research needs generated or reviewed SQL before DQL draft promotion.');
1890
+ const reviewedSql = notebookResearchString(run.reviewedSql);
1891
+ const dqlArtifact = normalizeDqlArtifactReference(run.dqlArtifact);
1892
+ const sql = reviewedSql ?? notebookResearchString(run.generatedSql);
1514
1893
  const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
1894
+ if (!reviewedSql && dqlArtifact) {
1895
+ const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
1896
+ question: run.question,
1897
+ dqlArtifact,
1898
+ inputMode: 'upload',
1899
+ inputPath: dqlArtifact.sourcePath ?? sourcePath,
1900
+ domain: notebookResearchString(input.domain) ?? run.domain,
1901
+ owner: notebookResearchString(input.owner) ?? run.owner,
1902
+ tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
1903
+ generatedSql: notebookResearchString(run.generatedSql),
1904
+ contextPackId: run.contextPackId,
1905
+ routeIntent: run.intent,
1906
+ sourceBlock: dqlArtifact.sourcePath,
1907
+ }, semanticLayer);
1908
+ const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
1909
+ ?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
1910
+ const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
1911
+ const updated = storage.markPromoted(run.id, {
1912
+ draftBlockPath: draftPath,
1913
+ dqlImportId: session.id,
1914
+ dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
1915
+ dqlPromotion: promotion,
1916
+ }) ?? run;
1917
+ const planned = storage.updateRun(updated.id, {
1918
+ researchPlan: buildNotebookResearchPlan({
1919
+ run: updated,
1920
+ evidence: updated.evidence,
1921
+ resultPreview: updated.resultPreview,
1922
+ generatedSql: updated.generatedSql,
1923
+ reviewedSql: updated.reviewedSql,
1924
+ dqlArtifact: updated.dqlArtifact,
1925
+ routeDecision: updated.routeDecision,
1926
+ }),
1927
+ }) ?? updated;
1928
+ return { run: planned, session };
1929
+ }
1930
+ if (!sql)
1931
+ throw new Error('Notebook research needs reviewed SQL, generated SQL, or a DQL artifact before DQL draft promotion.');
1515
1932
  const session = await createDqlGenerationSessionForProject(projectRoot, {
1516
1933
  inputMode: 'upload',
1517
1934
  sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
@@ -1537,6 +1954,7 @@ export async function startLocalServer(opts) {
1537
1954
  resultPreview: updated.resultPreview,
1538
1955
  generatedSql: updated.generatedSql,
1539
1956
  reviewedSql: updated.reviewedSql,
1957
+ dqlArtifact: updated.dqlArtifact,
1540
1958
  routeDecision: updated.routeDecision,
1541
1959
  }),
1542
1960
  }) ?? updated;
@@ -2061,10 +2479,14 @@ export async function startLocalServer(opts) {
2061
2479
  res.end(serializeJSON({ error: parsed.error ?? 'Invalid agent run request.' }));
2062
2480
  return;
2063
2481
  }
2064
- // The UI does not send retrieval signals; probe the local catalog so the
2065
- // router can short-circuit obvious governed matches without an LLM call.
2066
- if (!parsed.request.signals && !parsed.request.requestedMode) {
2067
- parsed.request.signals = computeAgentRunSignals(projectRoot, parsed.request.question);
2482
+ // The answer-loop cascade now proves certified/semantic/generated tiers
2483
+ // after retrieval and execution. Avoid pre-routing ordinary Ask runs from
2484
+ // token-overlap signals; explicit callers may still provide signals.
2485
+ // Thread-scoped runs: the persisted thread supplies prior turns (server
2486
+ // state wins; the client-built context stays the no-threadId fallback).
2487
+ const conversationStore = parsed.request.threadId ? getConversationStore() : null;
2488
+ if (conversationStore && parsed.request.threadId && conversationStore.getThread(parsed.request.threadId)) {
2489
+ parsed.request.conversationContext = await conversationContextFromThread(conversationStore, parsed.request.threadId, parsed.request.conversationContext, parsed.request.question);
2068
2490
  }
2069
2491
  const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
2070
2492
  if (wantsStream) {
@@ -2079,11 +2501,13 @@ export async function startLocalServer(opts) {
2079
2501
  }, (delta) => {
2080
2502
  writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
2081
2503
  });
2504
+ recordConversationTurn(conversationStore, parsed.request.threadId, run);
2082
2505
  writeAgentRunSse(res, 'agent-run-complete', run);
2083
2506
  res.end();
2084
2507
  return;
2085
2508
  }
2086
2509
  const run = await agentRunEngine.run(parsed.request);
2510
+ recordConversationTurn(conversationStore, parsed.request.threadId, run);
2087
2511
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
2088
2512
  res.end(serializeJSON({ run }));
2089
2513
  }
@@ -2115,6 +2539,7 @@ export async function startLocalServer(opts) {
2115
2539
  const notebookPath = (record && agentRunString(record.notebookPath))
2116
2540
  ?? `notebooks/certification-requests/${Date.now()}.dqlnb`;
2117
2541
  const generatedSql = record ? agentRunString(record.generatedSql) : undefined;
2542
+ const dqlArtifact = record ? normalizeDqlArtifactReference(record.dqlArtifact) : undefined;
2118
2543
  try {
2119
2544
  const storage = openNotebookResearchStorage();
2120
2545
  try {
@@ -2126,9 +2551,11 @@ export async function startLocalServer(opts) {
2126
2551
  domain: record ? agentRunString(record.domain) : undefined,
2127
2552
  owner: record ? agentRunString(record.owner) : undefined,
2128
2553
  generatedSql,
2554
+ dqlArtifact,
2129
2555
  context: {
2130
2556
  surface: 'stakeholder_request_certification',
2131
2557
  requestedContext: agentRunRecord(record?.context) ?? null,
2558
+ dqlArtifact: dqlArtifact ?? null,
2132
2559
  },
2133
2560
  });
2134
2561
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -2166,6 +2593,23 @@ export async function startLocalServer(opts) {
2166
2593
  // DRAFT proposals (each with its stored Certifier verdict) so the notebook
2167
2594
  // "Get Started" surface can route them into human review. dryRun preview —
2168
2595
  // nothing is written or certified by this call.
2596
+ if (req.method === 'GET' && path === '/api/agent-runs/tier-distribution') {
2597
+ try {
2598
+ const store = getConversationStore();
2599
+ const limitParam = Number(url.searchParams.get('limit'));
2600
+ const limit = Number.isFinite(limitParam) && limitParam > 0 ? limitParam : undefined;
2601
+ const distribution = store
2602
+ ? store.tierDistribution({ ...(limit ? { limit } : {}) })
2603
+ : { total: 0, byRouteTier: {}, byTerminalLane: {} };
2604
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2605
+ res.end(JSON.stringify(distribution));
2606
+ }
2607
+ catch (error) {
2608
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2609
+ res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
2610
+ }
2611
+ return;
2612
+ }
2169
2613
  if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose') {
2170
2614
  try {
2171
2615
  let owner;
@@ -2196,6 +2640,69 @@ export async function startLocalServer(opts) {
2196
2640
  }
2197
2641
  return;
2198
2642
  }
2643
+ // Composting preview: mine certified block clusters for semantic metric
2644
+ // candidates. Read-only; returns draft YAML + PR body text for review.
2645
+ if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose/composting') {
2646
+ try {
2647
+ let owner;
2648
+ let limit;
2649
+ let minSupport;
2650
+ if (req.method === 'POST') {
2651
+ const body = await readJSON(req).catch(() => ({}));
2652
+ if (typeof body?.owner === 'string')
2653
+ owner = body.owner;
2654
+ if (typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0)
2655
+ limit = body.limit;
2656
+ if (typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1)
2657
+ minSupport = body.minSupport;
2658
+ }
2659
+ else {
2660
+ const ownerParam = url.searchParams.get('owner');
2661
+ if (ownerParam)
2662
+ owner = ownerParam;
2663
+ const limitParam = Number(url.searchParams.get('limit'));
2664
+ if (Number.isFinite(limitParam) && limitParam > 0)
2665
+ limit = limitParam;
2666
+ const minSupportParam = Number(url.searchParams.get('minSupport'));
2667
+ if (Number.isFinite(minSupportParam) && minSupportParam > 1)
2668
+ minSupport = minSupportParam;
2669
+ }
2670
+ const changeset = buildSemanticCompostingChangeset(projectRoot, { owner, limit, minSupport });
2671
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2672
+ res.end(serializeJSON(changeset));
2673
+ }
2674
+ catch (error) {
2675
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2676
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2677
+ }
2678
+ return;
2679
+ }
2680
+ // Composting generation: writes only approved metric candidate ids under
2681
+ // semantic-layer/metrics/_drafts plus a PR-body artifact. Nothing certified.
2682
+ if (req.method === 'POST' && path === '/api/propose/composting/generate') {
2683
+ try {
2684
+ const body = (await readJSON(req).catch(() => ({})));
2685
+ const ids = Array.isArray(body?.ids)
2686
+ ? body.ids.filter((id) => typeof id === 'string' && id.trim().length > 0)
2687
+ : [];
2688
+ if (ids.length === 0) {
2689
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2690
+ res.end(serializeJSON({ error: 'Provide non-empty { ids } to generate semantic composting drafts.' }));
2691
+ return;
2692
+ }
2693
+ const owner = typeof body?.owner === 'string' ? body.owner : undefined;
2694
+ const limit = typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0 ? body.limit : undefined;
2695
+ const minSupport = typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1 ? body.minSupport : undefined;
2696
+ const result = generateSemanticCompostingDrafts(projectRoot, ids, { owner, limit, minSupport });
2697
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2698
+ res.end(serializeJSON(result));
2699
+ }
2700
+ catch (error) {
2701
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2702
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2703
+ }
2704
+ return;
2705
+ }
2199
2706
  // Deterministic PLAN only (classify → plan). Writes NOTHING. Same data the
2200
2707
  // readiness endpoint embeds, exposed standalone for the approve gate.
2201
2708
  if (req.method === 'POST' && path === '/api/propose/plan') {
@@ -2724,6 +3231,117 @@ export async function startLocalServer(opts) {
2724
3231
  }
2725
3232
  return;
2726
3233
  }
3234
+ // ── Conversation threads (server-side session persistence) ────────────
3235
+ if (path === '/api/agent/threads' || path.startsWith('/api/agent/threads/')) {
3236
+ const store = getConversationStore();
3237
+ if (!store) {
3238
+ res.writeHead(503, { 'Content-Type': 'application/json; charset=utf-8' });
3239
+ res.end(serializeJSON({ error: 'Conversation store is unavailable.' }));
3240
+ return;
3241
+ }
3242
+ try {
3243
+ if (req.method === 'GET' && path === '/api/agent/threads') {
3244
+ const limit = Number(url.searchParams.get('limit') ?? '50');
3245
+ const includeArchived = url.searchParams.get('archived') === '1';
3246
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3247
+ res.end(serializeJSON({ threads: store.listThreads({ limit: Number.isFinite(limit) ? limit : 50, includeArchived }) }));
3248
+ return;
3249
+ }
3250
+ if (req.method === 'POST' && path === '/api/agent/threads') {
3251
+ const body = await readJSON(req).catch(() => null);
3252
+ const record = agentRunRecord(body) ?? {};
3253
+ const thread = store.createThread({
3254
+ surface: agentRunString(record.surface),
3255
+ title: agentRunString(record.title),
3256
+ notebookPath: agentRunString(record.notebookPath),
3257
+ });
3258
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
3259
+ res.end(serializeJSON({ thread }));
3260
+ return;
3261
+ }
3262
+ // Cross-thread turn search ("search conversations") — must precede :id.
3263
+ if (req.method === 'GET' && path === '/api/agent/threads/search') {
3264
+ const query = url.searchParams.get('q') ?? '';
3265
+ const limit = Number(url.searchParams.get('limit') ?? '10');
3266
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3267
+ res.end(serializeJSON({
3268
+ turns: query.trim()
3269
+ ? store.searchTurns({ query, limit: Number.isFinite(limit) ? limit : 10 })
3270
+ : [],
3271
+ }));
3272
+ return;
3273
+ }
3274
+ const threadMatch = path.match(/^\/api\/agent\/threads\/([^/]+)(?:\/(archive|promote))?$/);
3275
+ if (threadMatch) {
3276
+ const threadId = decodeURIComponent(threadMatch[1]);
3277
+ const action = threadMatch[2];
3278
+ const thread = store.getThread(threadId);
3279
+ if (!thread) {
3280
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
3281
+ res.end(serializeJSON({ error: 'Unknown thread id.' }));
3282
+ return;
3283
+ }
3284
+ if (req.method === 'POST' && action === 'archive') {
3285
+ store.archiveThread(threadId);
3286
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3287
+ res.end(serializeJSON({ ok: true }));
3288
+ return;
3289
+ }
3290
+ // Explicit promotion — the ONLY path from conversation into durable
3291
+ // memory ("remember this"). Deliberate user act, tagged for audit.
3292
+ if (req.method === 'POST' && action === 'promote') {
3293
+ const body = await readJSON(req).catch(() => null);
3294
+ const record = agentRunRecord(body) ?? {};
3295
+ const turnId = agentRunString(record.turnId);
3296
+ const turn = turnId
3297
+ ? store.recentTurns(threadId, 200).find((candidate) => candidate.id === turnId)
3298
+ : undefined;
3299
+ if (!turn) {
3300
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3301
+ res.end(serializeJSON({ error: 'turnId is required and must belong to this thread.' }));
3302
+ return;
3303
+ }
3304
+ const scopeRaw = agentRunString(record.scope);
3305
+ const scope = scopeRaw === 'notebook' || scopeRaw === 'project' || scopeRaw === 'user' ? scopeRaw : 'project';
3306
+ const memory = new MemoryStore(defaultMemoryPath(projectRoot));
3307
+ try {
3308
+ const saved = memory.upsert({
3309
+ scope,
3310
+ title: agentRunString(record.title) ?? turn.question.slice(0, 120),
3311
+ content: `Q: ${turn.question}\nA: ${turn.answerSummary ?? turn.answerText ?? '(no answer summary)'}`,
3312
+ tags: ['conversation', threadId],
3313
+ source: 'conversation',
3314
+ confidence: 0.6,
3315
+ importance: 0.5,
3316
+ enabled: true,
3317
+ });
3318
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3319
+ res.end(serializeJSON({ ok: true, memory: saved }));
3320
+ }
3321
+ finally {
3322
+ memory.close();
3323
+ }
3324
+ return;
3325
+ }
3326
+ if (req.method === 'GET' && !action) {
3327
+ const limit = Number(url.searchParams.get('limit') ?? '50');
3328
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3329
+ res.end(serializeJSON({
3330
+ thread,
3331
+ turns: store.recentTurns(threadId, Number.isFinite(limit) ? limit : 50),
3332
+ }));
3333
+ return;
3334
+ }
3335
+ }
3336
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
3337
+ res.end(serializeJSON({ error: 'Method not allowed' }));
3338
+ }
3339
+ catch (error) {
3340
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3341
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3342
+ }
3343
+ return;
3344
+ }
2727
3345
  if (req.method === 'GET' && path === '/api/agent/memory') {
2728
3346
  const memory = new MemoryStore(defaultMemoryPath(projectRoot));
2729
3347
  try {
@@ -3247,6 +3865,7 @@ export async function startLocalServer(opts) {
3247
3865
  const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
3248
3866
  const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
3249
3867
  const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
3868
+ const dqlArtifact = normalizeDqlArtifactReference(body.dqlArtifact);
3250
3869
  if (!notebookPath || !question) {
3251
3870
  res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3252
3871
  res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
@@ -3266,6 +3885,7 @@ export async function startLocalServer(opts) {
3266
3885
  context: body.context,
3267
3886
  generatedSql: notebookResearchString(body.generatedSql),
3268
3887
  reviewedSql: notebookResearchString(body.reviewedSql),
3888
+ dqlArtifact,
3269
3889
  });
3270
3890
  const run = body.run === true
3271
3891
  ? await runNotebookResearch(storage, created, {
@@ -3276,6 +3896,7 @@ export async function startLocalServer(opts) {
3276
3896
  context: body.context,
3277
3897
  generatedSql: notebookResearchString(body.generatedSql),
3278
3898
  reviewedSql: notebookResearchString(body.reviewedSql),
3899
+ dqlArtifact,
3279
3900
  })
3280
3901
  : created;
3281
3902
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -3318,8 +3939,8 @@ export async function startLocalServer(opts) {
3318
3939
  intent,
3319
3940
  researchPattern: notebookResearchIntentPattern(intent),
3320
3941
  },
3321
- strictness: 'balanced',
3322
- limit: 100,
3942
+ strictness: 'exploratory',
3943
+ limit: 160,
3323
3944
  });
3324
3945
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3325
3946
  res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
@@ -3543,6 +4164,7 @@ export async function startLocalServer(opts) {
3543
4164
  routeDecision: body.routeDecision,
3544
4165
  generatedSql: notebookResearchString(body.generatedSql),
3545
4166
  reviewedSql: notebookResearchString(body.reviewedSql),
4167
+ dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
3546
4168
  warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
3547
4169
  reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
3548
4170
  dqlPromotionAction: notebookResearchPromotionAction(body.dqlPromotionAction),
@@ -3555,6 +4177,7 @@ export async function startLocalServer(opts) {
3555
4177
  previewError: updated.error,
3556
4178
  generatedSql: updated.generatedSql,
3557
4179
  reviewedSql: updated.reviewedSql,
4180
+ dqlArtifact: updated.dqlArtifact,
3558
4181
  routeDecision: updated.routeDecision,
3559
4182
  }),
3560
4183
  }) ?? updated;
@@ -3574,6 +4197,7 @@ export async function startLocalServer(opts) {
3574
4197
  context: body.context,
3575
4198
  generatedSql: notebookResearchString(body.generatedSql),
3576
4199
  reviewedSql: notebookResearchString(body.reviewedSql),
4200
+ dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
3577
4201
  });
3578
4202
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3579
4203
  res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
@@ -6729,16 +7353,27 @@ export function resolveDefaultLLMProvider(projectRoot) {
6729
7353
  * answer-loop runner — never the MCP `claudeCodeRunner`, which doesn't emit a governed
6730
7354
  * answer envelope. Everything else uses the Settings-resolved default runner.
6731
7355
  */
6732
- function resolveGovernedAnswerRunner(projectRoot) {
7356
+ export function resolveGovernedAnswerRunner(projectRoot) {
6733
7357
  const active = getActiveProvider(projectRoot);
6734
- if (active === 'claude-code' || active === 'codex') {
7358
+ if (isGovernedAnswerProviderId(active)) {
6735
7359
  return { provider: active, runner: createDqlAgentProviderRunner(active) };
6736
7360
  }
6737
7361
  const resolved = resolveDefaultLLMProvider(projectRoot);
6738
7362
  if (!resolved)
6739
7363
  return null;
6740
- const runner = getLLMRunner(resolved);
6741
- return runner ? { provider: resolved, runner } : null;
7364
+ if (isGovernedAnswerProviderId(resolved)) {
7365
+ return { provider: resolved, runner: createDqlAgentProviderRunner(resolved) };
7366
+ }
7367
+ return null;
7368
+ }
7369
+ function isGovernedAnswerProviderId(value) {
7370
+ return value === 'anthropic'
7371
+ || value === 'openai'
7372
+ || value === 'gemini'
7373
+ || value === 'ollama'
7374
+ || value === 'custom-openai'
7375
+ || value === 'claude-code'
7376
+ || value === 'codex';
6742
7377
  }
6743
7378
  /** Map a runner provider id to the settings id whose reasoning ceiling applies. */
6744
7379
  function reasoningSettingsIdFor(provider) {
@@ -6783,6 +7418,11 @@ function resolveRunReasoningEffort(projectRoot, provider, route, isRepair) {
6783
7418
  desired = bumpReasoningEffort(desired);
6784
7419
  return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
6785
7420
  }
7421
+ function resolveRequestedRunReasoningEffort(projectRoot, provider, requested, isRepair) {
7422
+ const ceiling = getEffectiveProviderConfig(projectRoot, reasoningSettingsIdFor(provider)).reasoningEffort;
7423
+ const desired = isRepair ? bumpReasoningEffort(requested) : requested;
7424
+ return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
7425
+ }
6786
7426
  function loadAppDashboard(projectRoot, appId, dashboardId) {
6787
7427
  for (const p of findAppDocuments(projectRoot)) {
6788
7428
  const { document: app } = loadAppDocument(p);
@@ -7107,6 +7747,11 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
7107
7747
  if (proposal.certification.errors.length === 0)
7108
7748
  readyForReview += 1;
7109
7749
  }
7750
+ const reviewableProposals = buildReviewableProposals(projectRoot, manifestPath, projectConfig, summary.proposals, options.owner);
7751
+ const reviewAges = reviewableProposals
7752
+ .map((proposal) => proposal.review.reviewAgeHours)
7753
+ .filter((value) => typeof value === 'number' && Number.isFinite(value));
7754
+ const estimatedReviewMinutes = reviewableProposals.reduce((sum, proposal) => sum + proposal.review.estimatedReviewMinutes, 0);
7110
7755
  return {
7111
7756
  ready: true,
7112
7757
  summary: {
@@ -7121,11 +7766,595 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
7121
7766
  readyForReview,
7122
7767
  blockingTotal,
7123
7768
  warningTotal,
7769
+ reviewTelemetry: {
7770
+ existingDrafts: reviewableProposals.filter((proposal) => proposal.review.draftExists).length,
7771
+ medianReviewAgeHours: median(reviewAges),
7772
+ readyForReviewRate: ratio(readyForReview, summary.proposalsRanked),
7773
+ estimatedReviewMinutes,
7774
+ },
7124
7775
  },
7125
7776
  plan,
7126
- proposals: summary.proposals,
7777
+ proposals: reviewableProposals,
7778
+ };
7779
+ }
7780
+ function buildReviewableProposals(projectRoot, dbtManifestPath, projectConfig, proposals, owner) {
7781
+ const manifest = safeBuildProjectManifest(projectRoot);
7782
+ return proposals.map((proposal, index) => {
7783
+ const preview = buildProposePreview(projectRoot, dbtManifestPath, proposal.slug, {
7784
+ config: projectConfig.propose,
7785
+ owner,
7786
+ });
7787
+ const draftPath = proposal.path ?? resolveProposeDraftPath(projectRoot, proposal.domain, proposal.slug);
7788
+ const draftFile = join(projectRoot, draftPath);
7789
+ const draftExists = existsSync(draftFile);
7790
+ const stats = draftExists ? statSync(draftFile) : undefined;
7791
+ const firstSeenAt = stats ? stableFsTime(stats.birthtimeMs, stats.ctimeMs) : undefined;
7792
+ const lastUpdatedAt = stats ? stableFsTime(stats.mtimeMs, stats.ctimeMs) : undefined;
7793
+ const reviewAgeHours = firstSeenAt ? hoursSince(firstSeenAt) : undefined;
7794
+ const blockingCount = proposal.certification.errors.length;
7795
+ const warningCount = proposal.certification.warnings.length;
7796
+ const outputs = preview?.outputs ?? proposal.inference.declaredOutputs;
7797
+ const status = blockingCount === 0
7798
+ ? 'ready_for_review'
7799
+ : draftExists
7800
+ ? 'draft_exists'
7801
+ : 'new';
7802
+ return {
7803
+ ...proposal,
7804
+ review: {
7805
+ queueRank: index + 1,
7806
+ status,
7807
+ priorityScore: proposal.ranking.score,
7808
+ blockingCount,
7809
+ warningCount,
7810
+ estimatedReviewMinutes: estimateProposalReviewMinutes(blockingCount, warningCount, Boolean(preview?.sqlPreview)),
7811
+ draftPath,
7812
+ draftExists,
7813
+ firstSeenAt,
7814
+ lastUpdatedAt,
7815
+ reviewAgeHours,
7816
+ certifyCommand: `dql certify --from-draft ${quoteCliArg(draftPath)} --owner you@example.com`,
7817
+ payload: {
7818
+ question: preview?.examples?.[0] ?? proposal.inference.examples[0]?.question ?? `Review ${proposal.slug}`,
7819
+ model: proposal.model,
7820
+ domain: proposal.domain,
7821
+ sqlPreview: preview?.sqlPreview,
7822
+ outputs,
7823
+ grain: preview?.grain ?? proposal.inference.grain,
7824
+ pattern: preview?.pattern ?? proposal.inference.pattern,
7825
+ evidence: proposal.evidence,
7826
+ resultSample: { status: 'not_run', rows: [] },
7827
+ nearestCertifiedBlock: findNearestCertifiedBlock(manifest, proposal, outputs),
7828
+ },
7829
+ },
7830
+ };
7831
+ });
7832
+ }
7833
+ function safeBuildProjectManifest(projectRoot) {
7834
+ try {
7835
+ return buildManifest({ projectRoot });
7836
+ }
7837
+ catch {
7838
+ return { blocks: {} };
7839
+ }
7840
+ }
7841
+ function resolveProposeDraftPath(projectRoot, domain, slug) {
7842
+ const safeDomain = domain
7843
+ .trim()
7844
+ .toLowerCase()
7845
+ .replace(/[^a-z0-9/_-]+/g, '-')
7846
+ .replace(/^[-/]+|[-/]+$/g, '');
7847
+ return safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))
7848
+ ? `domains/${safeDomain}/blocks/_drafts/${slug}.dql`
7849
+ : `blocks/_drafts/${slug}.dql`;
7850
+ }
7851
+ function findNearestCertifiedBlock(manifest, proposal, outputs) {
7852
+ const outputSet = new Set(outputs.map((output) => output.toLowerCase()));
7853
+ let best;
7854
+ for (const block of Object.values(manifest.blocks ?? {})) {
7855
+ if (String(block.status ?? '').toLowerCase() !== 'certified')
7856
+ continue;
7857
+ const blockOutputs = (block.declaredOutputs ?? []).map((output) => output.toLowerCase());
7858
+ const sharedOutputs = blockOutputs.filter((output) => outputSet.has(output));
7859
+ const sameDomain = block.domain === proposal.domain;
7860
+ const overlapScore = sharedOutputs.length * 10 + (sameDomain ? 5 : 0);
7861
+ if (overlapScore <= 0)
7862
+ continue;
7863
+ if (!best || overlapScore > best.overlapScore) {
7864
+ best = {
7865
+ name: block.name,
7866
+ path: block.filePath,
7867
+ domain: block.domain,
7868
+ overlapScore,
7869
+ sharedOutputs,
7870
+ };
7871
+ }
7872
+ }
7873
+ return best;
7874
+ }
7875
+ function estimateProposalReviewMinutes(blockingCount, warningCount, hasSqlPreview) {
7876
+ const base = hasSqlPreview ? 12 : 18;
7877
+ return base + blockingCount * 8 + warningCount * 4;
7878
+ }
7879
+ function stableFsTime(primaryMs, fallbackMs) {
7880
+ const ms = Number.isFinite(primaryMs) && primaryMs > 0 ? primaryMs : fallbackMs;
7881
+ return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : undefined;
7882
+ }
7883
+ function hoursSince(iso) {
7884
+ const ts = Date.parse(iso);
7885
+ if (!Number.isFinite(ts))
7886
+ return 0;
7887
+ return Number(Math.max(0, (Date.now() - ts) / 3_600_000).toFixed(2));
7888
+ }
7889
+ function median(values) {
7890
+ if (values.length === 0)
7891
+ return null;
7892
+ const sorted = [...values].sort((a, b) => a - b);
7893
+ const mid = Math.floor(sorted.length / 2);
7894
+ const value = sorted.length % 2 === 1
7895
+ ? sorted[mid]
7896
+ : (sorted[mid - 1] + sorted[mid]) / 2;
7897
+ return Number(value.toFixed(2));
7898
+ }
7899
+ function ratio(numerator, denominator) {
7900
+ if (denominator === 0)
7901
+ return null;
7902
+ return Number((numerator / denominator).toFixed(4));
7903
+ }
7904
+ function quoteCliArg(value) {
7905
+ if (/^[A-Za-z0-9_./:-]+$/.test(value))
7906
+ return value;
7907
+ return `'${value.replace(/'/g, "'\\''")}'`;
7908
+ }
7909
+ /**
7910
+ * Mine certified block clusters for recurring metric definitions and render a
7911
+ * reviewable semantic-layer changeset. This is read-only: callers must confirm
7912
+ * candidate ids through generateSemanticCompostingDrafts before files are written.
7913
+ */
7914
+ export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
7915
+ let manifest;
7916
+ try {
7917
+ manifest = buildManifest({ projectRoot });
7918
+ }
7919
+ catch (error) {
7920
+ return {
7921
+ ready: false,
7922
+ reason: error instanceof Error ? error.message : String(error),
7923
+ summary: {
7924
+ certifiedBlocksScanned: 0,
7925
+ eligibleMetricClusters: 0,
7926
+ candidatesRanked: 0,
7927
+ existingDrafts: 0,
7928
+ donorBlocksUsed: 0,
7929
+ minSupport: Math.max(2, Math.floor(options.minSupport ?? 2)),
7930
+ },
7931
+ candidates: [],
7932
+ prBody: '',
7933
+ };
7934
+ }
7935
+ const minSupport = Math.max(2, Math.floor(options.minSupport ?? 2));
7936
+ const limit = Number.isFinite(options.limit) && Number(options.limit) > 0 ? Math.floor(Number(options.limit)) : 10;
7937
+ const owner = options.owner?.trim() || resolveLocalOwner(projectRoot, { persist: false });
7938
+ const existingMetricNames = new Set(Object.entries(manifest.metrics ?? {})
7939
+ .filter(([, metric]) => !isSemanticDraftMetricPath(metric.filePath))
7940
+ .map(([name]) => name.toLowerCase()));
7941
+ const certifiedBlocks = Object.values(manifest.blocks ?? {})
7942
+ .filter((block) => String(block.status ?? '').toLowerCase() === 'certified');
7943
+ const groups = new Map();
7944
+ for (const block of certifiedBlocks) {
7945
+ const table = singleTableDependency(block);
7946
+ if (!table)
7947
+ continue;
7948
+ const where = extractCompostingWhereClause(block.sql);
7949
+ const normalizedFilter = where ? normalizeCompostingSql(where) : undefined;
7950
+ const metrics = extractCompostingAggregateMetrics(block.sql);
7951
+ if (metrics.length === 0)
7952
+ continue;
7953
+ const donor = {
7954
+ name: block.name,
7955
+ path: block.filePath,
7956
+ domain: block.domain,
7957
+ outputs: blockOutputNames(block),
7958
+ filters: where ? splitCompostingFilters(where) : [],
7959
+ };
7960
+ for (const metric of metrics) {
7961
+ const normalizedExpression = normalizeCompostingSql(metric.expression);
7962
+ const domain = block.domain?.trim() || 'misc';
7963
+ const key = [
7964
+ domain.toLowerCase(),
7965
+ table.toLowerCase(),
7966
+ metric.type,
7967
+ normalizedExpression,
7968
+ normalizedFilter ?? '',
7969
+ ].join('\0');
7970
+ const existing = groups.get(key);
7971
+ if (existing) {
7972
+ existing.aliases.push(metric.alias ?? '');
7973
+ existing.donors.set(block.filePath, donor);
7974
+ }
7975
+ else {
7976
+ groups.set(key, {
7977
+ domain,
7978
+ table,
7979
+ type: metric.type,
7980
+ expression: metric.expression,
7981
+ normalizedExpression,
7982
+ filter: where,
7983
+ normalizedFilter,
7984
+ aliases: [metric.alias ?? ''],
7985
+ donors: new Map([[block.filePath, donor]]),
7986
+ });
7987
+ }
7988
+ }
7989
+ }
7990
+ const usedNames = new Set(existingMetricNames);
7991
+ const candidates = Array.from(groups.values())
7992
+ .filter((group) => group.donors.size >= minSupport)
7993
+ .sort((a, b) => b.donors.size - a.donors.size || a.normalizedExpression.localeCompare(b.normalizedExpression))
7994
+ .map((group) => buildCompostingMetricCandidate(projectRoot, group, owner, usedNames))
7995
+ .filter((candidate) => Boolean(candidate))
7996
+ .slice(0, limit);
7997
+ const donorBlocksUsed = new Set(candidates.flatMap((candidate) => candidate.donorBlocks.map((donor) => donor.path))).size;
7998
+ return {
7999
+ ready: true,
8000
+ summary: {
8001
+ certifiedBlocksScanned: certifiedBlocks.length,
8002
+ eligibleMetricClusters: Array.from(groups.values()).filter((group) => group.donors.size >= minSupport).length,
8003
+ candidatesRanked: candidates.length,
8004
+ existingDrafts: candidates.filter((candidate) => candidate.draftExists).length,
8005
+ donorBlocksUsed,
8006
+ minSupport,
8007
+ },
8008
+ candidates,
8009
+ prBody: renderCompostingPrBody(candidates),
8010
+ };
8011
+ }
8012
+ export function generateSemanticCompostingDrafts(projectRoot, candidateIds, options = {}) {
8013
+ const changeset = buildSemanticCompostingChangeset(projectRoot, options);
8014
+ if (!changeset.ready) {
8015
+ return {
8016
+ ready: false,
8017
+ reason: changeset.reason,
8018
+ draftsWritten: 0,
8019
+ draftsSkipped: 0,
8020
+ candidates: [],
8021
+ paths: [],
8022
+ prBody: '',
8023
+ };
8024
+ }
8025
+ const wanted = candidateIds && candidateIds.length > 0 ? new Set(candidateIds) : undefined;
8026
+ const selected = wanted ? changeset.candidates.filter((candidate) => wanted.has(candidate.id)) : changeset.candidates;
8027
+ const paths = [];
8028
+ let draftsWritten = 0;
8029
+ let draftsSkipped = 0;
8030
+ for (const candidate of selected) {
8031
+ const absolutePath = join(projectRoot, candidate.draftPath);
8032
+ if (existsSync(absolutePath)) {
8033
+ draftsSkipped += 1;
8034
+ }
8035
+ else {
8036
+ mkdirSync(dirname(absolutePath), { recursive: true });
8037
+ writeFileSync(absolutePath, candidate.yaml, 'utf-8');
8038
+ draftsWritten += 1;
8039
+ }
8040
+ paths.push(candidate.draftPath);
8041
+ }
8042
+ let prBodyPath;
8043
+ const prBody = renderCompostingPrBody(selected);
8044
+ if (selected.length > 0) {
8045
+ prBodyPath = 'semantic-layer/metrics/_drafts/PR_BODY.md';
8046
+ const absolutePrBodyPath = join(projectRoot, prBodyPath);
8047
+ mkdirSync(dirname(absolutePrBodyPath), { recursive: true });
8048
+ writeFileSync(absolutePrBodyPath, prBody, 'utf-8');
8049
+ }
8050
+ return {
8051
+ ready: true,
8052
+ draftsWritten,
8053
+ draftsSkipped,
8054
+ candidates: selected,
8055
+ paths,
8056
+ prBody,
8057
+ prBodyPath,
8058
+ };
8059
+ }
8060
+ function buildCompostingMetricCandidate(projectRoot, group, owner, usedNames) {
8061
+ const baseName = preferredCompostingMetricName(group);
8062
+ const name = dedupeCompostingName(baseName, usedNames);
8063
+ if (!name)
8064
+ return undefined;
8065
+ usedNames.add(name.toLowerCase());
8066
+ const label = titleFromSlug(name);
8067
+ const donors = Array.from(group.donors.values()).sort((a, b) => a.path.localeCompare(b.path));
8068
+ const support = donors.length;
8069
+ const domainSlug = slugForPath(group.domain);
8070
+ const draftPath = `semantic-layer/metrics/_drafts/${domainSlug}/${name}.yaml`;
8071
+ const recurringFilters = group.filter ? splitCompostingFilters(group.filter) : [];
8072
+ const metric = {
8073
+ name,
8074
+ label,
8075
+ description: `Draft metric composted from ${support} certified DQL blocks. Review the expression, filter, grain, and owner before certification.`,
8076
+ domain: group.domain,
8077
+ status: 'draft',
8078
+ sql: group.expression,
8079
+ type: group.type,
8080
+ table: group.table,
8081
+ filter: group.filter,
8082
+ tags: ['composted', 'human-review'],
8083
+ owner,
8084
+ source: {
8085
+ provider: 'dql',
8086
+ objectType: 'composted_metric',
8087
+ objectId: `composted:${name}`,
8088
+ objectName: label,
8089
+ importedAt: new Date().toISOString(),
8090
+ extra: {
8091
+ support,
8092
+ donorBlocks: donors.map((donor) => donor.name),
8093
+ donorPaths: donors.map((donor) => donor.path),
8094
+ normalizedExpression: group.normalizedExpression,
8095
+ normalizedFilter: group.normalizedFilter,
8096
+ },
8097
+ },
8098
+ };
8099
+ return {
8100
+ id: `metric:${name}:${stableShortHash(`${group.domain}|${group.table}|${group.normalizedExpression}|${group.normalizedFilter ?? ''}`)}`,
8101
+ kind: 'metric',
8102
+ name,
8103
+ label,
8104
+ description: metric.description,
8105
+ domain: group.domain,
8106
+ table: group.table,
8107
+ type: group.type,
8108
+ sql: group.expression,
8109
+ filter: group.filter,
8110
+ status: 'draft',
8111
+ support,
8112
+ donorBlocks: donors,
8113
+ draftPath,
8114
+ draftExists: existsSync(join(projectRoot, draftPath)),
8115
+ yaml: serializeMetricDefinitionToYaml(metric),
8116
+ provenance: {
8117
+ normalizedExpression: group.normalizedExpression,
8118
+ normalizedFilter: group.normalizedFilter,
8119
+ recurringFilters,
8120
+ },
8121
+ review: {
8122
+ priorityScore: support * 10 + recurringFilters.length * 2,
8123
+ rationale: `Recurring ${group.type} metric found in ${support} certified blocks${group.filter ? ' with the same filter' : ''}.`,
8124
+ },
8125
+ };
8126
+ }
8127
+ function singleTableDependency(block) {
8128
+ const dependencies = Array.from(new Set((block.tableDependencies ?? []).filter((table) => table && !table.startsWith('ref:'))));
8129
+ if (dependencies.length === 1)
8130
+ return dependencies[0];
8131
+ const rawRefs = Array.from(new Set((block.rawTableRefs ?? []).filter(Boolean)));
8132
+ return rawRefs.length === 1 ? rawRefs[0] : undefined;
8133
+ }
8134
+ function blockOutputNames(block) {
8135
+ if (Array.isArray(block.declaredOutputs) && block.declaredOutputs.length > 0)
8136
+ return block.declaredOutputs;
8137
+ if (Array.isArray(block.outputContract) && block.outputContract.length > 0)
8138
+ return block.outputContract.map((output) => output.name);
8139
+ if (Array.isArray(block.outputs) && block.outputs.length > 0)
8140
+ return block.outputs.map((output) => output.name);
8141
+ return [];
8142
+ }
8143
+ function isSemanticDraftMetricPath(filePath) {
8144
+ return Boolean(filePath?.replaceAll('\\', '/').includes('semantic-layer/metrics/_drafts/'));
8145
+ }
8146
+ function extractCompostingAggregateMetrics(sql) {
8147
+ const select = extractCompostingSelectClause(sql);
8148
+ if (!select)
8149
+ return [];
8150
+ return splitCompostingTopLevel(select, ',').flatMap((item) => {
8151
+ const parsed = parseCompostingSelectItem(item);
8152
+ if (!parsed)
8153
+ return [];
8154
+ return [parsed];
8155
+ });
8156
+ }
8157
+ function extractCompostingSelectClause(sql) {
8158
+ const source = stripSqlTerminator(sql);
8159
+ const selectMatch = /\bselect\b/i.exec(source);
8160
+ if (!selectMatch)
8161
+ return undefined;
8162
+ const fromIndex = findTopLevelKeyword(source, 'from', selectMatch.index + selectMatch[0].length);
8163
+ if (fromIndex < 0)
8164
+ return undefined;
8165
+ return source.slice(selectMatch.index + selectMatch[0].length, fromIndex).trim();
8166
+ }
8167
+ function parseCompostingSelectItem(item) {
8168
+ const { expression, alias } = splitCompostingAlias(item.trim());
8169
+ const normalizedExpression = expression.trim();
8170
+ const distinctCount = normalizedExpression.match(/^count\s*\(\s*distinct\s+[\s\S]+\)$/i);
8171
+ if (distinctCount) {
8172
+ return { expression: normalizedExpression, alias, type: 'count_distinct' };
8173
+ }
8174
+ const count = normalizedExpression.match(/^count\s*\([\s\S]*\)$/i);
8175
+ if (count)
8176
+ return { expression: normalizedExpression, alias, type: 'count' };
8177
+ const aggregate = normalizedExpression.match(/^(sum|avg|min|max)\s*\([\s\S]+\)$/i);
8178
+ if (!aggregate)
8179
+ return undefined;
8180
+ return {
8181
+ expression: normalizedExpression,
8182
+ alias,
8183
+ type: aggregate[1].toLowerCase(),
7127
8184
  };
7128
8185
  }
8186
+ function splitCompostingAlias(item) {
8187
+ const asMatch = item.match(/^([\s\S]+?)\s+as\s+("[^"]+"|`[^`]+`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_$]*)$/i);
8188
+ if (asMatch)
8189
+ return { expression: asMatch[1].trim(), alias: stripIdentifierQuotes(asMatch[2]) };
8190
+ return { expression: item.trim() };
8191
+ }
8192
+ function extractCompostingWhereClause(sql) {
8193
+ const source = stripSqlTerminator(sql);
8194
+ const whereIndex = findTopLevelKeyword(source, 'where', 0);
8195
+ if (whereIndex < 0)
8196
+ return undefined;
8197
+ const start = whereIndex + 'where'.length;
8198
+ const endCandidates = ['group by', 'having', 'order by', 'limit', 'qualify']
8199
+ .map((keyword) => findTopLevelKeyword(source, keyword, start))
8200
+ .filter((index) => index >= 0);
8201
+ const end = endCandidates.length > 0 ? Math.min(...endCandidates) : source.length;
8202
+ const where = source.slice(start, end).trim();
8203
+ return where || undefined;
8204
+ }
8205
+ function splitCompostingFilters(where) {
8206
+ return splitCompostingTopLevel(where, 'and')
8207
+ .map((part) => part.trim())
8208
+ .filter(Boolean);
8209
+ }
8210
+ function splitCompostingTopLevel(input, delimiter) {
8211
+ const out = [];
8212
+ let start = 0;
8213
+ let depth = 0;
8214
+ let quote;
8215
+ for (let i = 0; i < input.length; i += 1) {
8216
+ const char = input[i];
8217
+ if (quote) {
8218
+ if (char === quote && input[i - 1] !== '\\')
8219
+ quote = undefined;
8220
+ continue;
8221
+ }
8222
+ if (char === '"' || char === "'" || char === '`') {
8223
+ quote = char;
8224
+ continue;
8225
+ }
8226
+ if (char === '(')
8227
+ depth += 1;
8228
+ else if (char === ')')
8229
+ depth = Math.max(0, depth - 1);
8230
+ if (depth !== 0)
8231
+ continue;
8232
+ if (delimiter === ',' && char === ',') {
8233
+ out.push(input.slice(start, i).trim());
8234
+ start = i + 1;
8235
+ }
8236
+ else if (delimiter === 'and'
8237
+ && input.slice(i, i + 3).toLowerCase() === 'and'
8238
+ && !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
8239
+ && !/[A-Za-z0-9_]/.test(input[i + 3] ?? '')) {
8240
+ out.push(input.slice(start, i).trim());
8241
+ start = i + 3;
8242
+ i += 2;
8243
+ }
8244
+ }
8245
+ out.push(input.slice(start).trim());
8246
+ return out.filter(Boolean);
8247
+ }
8248
+ function findTopLevelKeyword(input, keyword, fromIndex) {
8249
+ const lower = input.toLowerCase();
8250
+ const target = keyword.toLowerCase();
8251
+ let depth = 0;
8252
+ let quote;
8253
+ for (let i = Math.max(0, fromIndex); i <= input.length - target.length; i += 1) {
8254
+ const char = input[i];
8255
+ if (quote) {
8256
+ if (char === quote && input[i - 1] !== '\\')
8257
+ quote = undefined;
8258
+ continue;
8259
+ }
8260
+ if (char === '"' || char === "'" || char === '`') {
8261
+ quote = char;
8262
+ continue;
8263
+ }
8264
+ if (char === '(') {
8265
+ depth += 1;
8266
+ continue;
8267
+ }
8268
+ if (char === ')') {
8269
+ depth = Math.max(0, depth - 1);
8270
+ continue;
8271
+ }
8272
+ if (depth !== 0)
8273
+ continue;
8274
+ if (lower.slice(i, i + target.length) === target
8275
+ && !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
8276
+ && !/[A-Za-z0-9_]/.test(input[i + target.length] ?? '')) {
8277
+ return i;
8278
+ }
8279
+ }
8280
+ return -1;
8281
+ }
8282
+ function preferredCompostingMetricName(group) {
8283
+ const aliases = group.aliases.map((alias) => slugForIdentifier(alias)).filter(Boolean);
8284
+ const counts = new Map();
8285
+ for (const alias of aliases)
8286
+ counts.set(alias, (counts.get(alias) ?? 0) + 1);
8287
+ const preferredAlias = Array.from(counts.entries()).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]?.[0];
8288
+ if (preferredAlias)
8289
+ return preferredAlias;
8290
+ const expressionName = slugForIdentifier(group.expression.replace(/\b(sum|avg|min|max|count|distinct)\b/gi, ''));
8291
+ return expressionName ? `${expressionName}_${group.type}` : `${group.type}_metric`;
8292
+ }
8293
+ function dedupeCompostingName(baseName, usedNames) {
8294
+ const base = slugForIdentifier(baseName).slice(0, 72).replace(/_+$/g, '');
8295
+ if (!base)
8296
+ return undefined;
8297
+ if (!usedNames.has(base.toLowerCase()))
8298
+ return base;
8299
+ let index = 2;
8300
+ while (usedNames.has(`${base}_${index}`.toLowerCase()))
8301
+ index += 1;
8302
+ return `${base}_${index}`;
8303
+ }
8304
+ function normalizeCompostingSql(value) {
8305
+ return value
8306
+ .replace(/["`]/g, '')
8307
+ .replace(/\s+/g, ' ')
8308
+ .trim()
8309
+ .toLowerCase();
8310
+ }
8311
+ function slugForIdentifier(value) {
8312
+ return value
8313
+ .trim()
8314
+ .replace(/["`[\]]/g, '')
8315
+ .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
8316
+ .toLowerCase()
8317
+ .replace(/[^a-z0-9]+/g, '_')
8318
+ .replace(/^_+|_+$/g, '');
8319
+ }
8320
+ function slugForPath(value) {
8321
+ return slugForIdentifier(value) || 'misc';
8322
+ }
8323
+ function stripIdentifierQuotes(value) {
8324
+ return value.replace(/^["`\[]|["`\]]$/g, '').trim();
8325
+ }
8326
+ function titleFromSlug(value) {
8327
+ return value.split('_').filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(' ');
8328
+ }
8329
+ function stableShortHash(value) {
8330
+ return createHash('sha1').update(value).digest('hex').slice(0, 10);
8331
+ }
8332
+ function renderCompostingPrBody(candidates) {
8333
+ const lines = [
8334
+ '# Semantic Composting Changeset',
8335
+ '',
8336
+ 'This changeset proposes draft semantic metrics mined from recurring logic in certified DQL blocks.',
8337
+ 'Review each metric expression, filter, owner, grain compatibility, and downstream impact before marking it certified.',
8338
+ '',
8339
+ ];
8340
+ if (candidates.length === 0) {
8341
+ lines.push('No recurring certified-block metric clusters met the current support threshold.');
8342
+ return `${lines.join('\n')}\n`;
8343
+ }
8344
+ for (const candidate of candidates) {
8345
+ lines.push(`## ${candidate.label}`);
8346
+ lines.push('');
8347
+ lines.push(`- Draft path: \`${candidate.draftPath}\``);
8348
+ lines.push(`- Domain/table: \`${candidate.domain}\` / \`${candidate.table}\``);
8349
+ lines.push(`- Expression: \`${candidate.sql}\``);
8350
+ if (candidate.filter)
8351
+ lines.push(`- Filter: \`${candidate.filter}\``);
8352
+ lines.push(`- Support: ${candidate.support} certified block(s)`);
8353
+ lines.push(`- Donors: ${candidate.donorBlocks.map((donor) => `\`${donor.path}\``).join(', ')}`);
8354
+ lines.push('');
8355
+ }
8356
+ return `${lines.join('\n')}\n`;
8357
+ }
7129
8358
  export async function generateProposeDrafts(projectRoot, slugs, projectConfig = loadProjectConfig(projectRoot), options = {}) {
7130
8359
  const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
7131
8360
  if (!manifestPath) {
@@ -8259,6 +9488,20 @@ function parseBlockStudioArrayField(source, key) {
8259
9488
  function parseBlockStudioStringField(source, key) {
8260
9489
  return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
8261
9490
  }
9491
+ function parseSemanticRequestedFilters(source) {
9492
+ return parseBlockStudioArrayField(source, 'requested_filters')
9493
+ .map((entry) => {
9494
+ const match = entry.match(/^([^=<>!]+)\s*=\s*(.+)$/);
9495
+ if (!match)
9496
+ return undefined;
9497
+ const dimension = match[1]?.trim();
9498
+ const value = match[2]?.trim();
9499
+ if (!dimension || !value)
9500
+ return undefined;
9501
+ return { dimension, operator: 'equals', values: [value] };
9502
+ })
9503
+ .filter((filter) => Boolean(filter));
9504
+ }
8262
9505
  function parseSemanticBlockConfig(source) {
8263
9506
  const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
8264
9507
  ? 'semantic'
@@ -8266,6 +9509,7 @@ function parseSemanticBlockConfig(source) {
8266
9509
  const metric = parseBlockStudioStringField(source, 'metric');
8267
9510
  const metrics = parseBlockStudioArrayField(source, 'metrics');
8268
9511
  const dimensions = parseBlockStudioArrayField(source, 'dimensions');
9512
+ const filters = parseSemanticRequestedFilters(source);
8269
9513
  const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
8270
9514
  const granularity = parseBlockStudioStringField(source, 'granularity');
8271
9515
  const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
@@ -8274,6 +9518,7 @@ function parseSemanticBlockConfig(source) {
8274
9518
  metric,
8275
9519
  metrics,
8276
9520
  dimensions,
9521
+ filters,
8277
9522
  timeDimension,
8278
9523
  granularity,
8279
9524
  limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
@@ -8419,6 +9664,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
8419
9664
  ? composeRuntimeSemanticQuery({
8420
9665
  metrics,
8421
9666
  dimensions: config.dimensions,
9667
+ filters: config.filters,
8422
9668
  timeDimension: config.timeDimension && config.granularity
8423
9669
  ? { name: config.timeDimension, granularity: config.granularity }
8424
9670
  : undefined,
@@ -8433,6 +9679,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
8433
9679
  : semanticLayer.composeQuery({
8434
9680
  metrics,
8435
9681
  dimensions: config.dimensions,
9682
+ filters: config.filters,
8436
9683
  timeDimension: config.timeDimension && config.granularity
8437
9684
  ? { name: config.timeDimension, granularity: config.granularity }
8438
9685
  : undefined,
@@ -8599,6 +9846,101 @@ export function validateBlockStudioSource(source, semanticLayer) {
8599
9846
  executableSql,
8600
9847
  };
8601
9848
  }
9849
+ export async function createDqlArtifactGenerationSessionForProject(projectRoot, options, semanticLayer) {
9850
+ const dqlArtifact = normalizeDqlArtifactReference(options.dqlArtifact);
9851
+ if (!dqlArtifact)
9852
+ throw new Error('A DQL artifact with source is required for DQL draft promotion.');
9853
+ const question = notebookResearchString(options.question) ?? dqlArtifact.name ?? 'Generated DQL artifact';
9854
+ const slug = deriveGeneratedDraftSlug(dqlArtifact.name ?? question);
9855
+ const domain = notebookResearchString(options.domain) ?? 'misc';
9856
+ const owner = notebookResearchString(options.owner) ?? '';
9857
+ const tags = Array.from(new Set(['notebook-research', 'review-required', 'dql-artifact', ...(Array.isArray(options.tags) ? options.tags : [])]
9858
+ .map((tag) => notebookResearchString(tag))
9859
+ .filter((tag) => Boolean(tag))));
9860
+ const proposedContractId = `${domain}.Unknown.${slug}`;
9861
+ const draft = upsertGeneratedDqlArtifactDraft(projectRoot, {
9862
+ slug,
9863
+ question,
9864
+ proposedContractId,
9865
+ owner,
9866
+ proposedDomain: domain,
9867
+ dqlArtifact,
9868
+ sourceQuestion: question,
9869
+ sourceBlock: options.sourceBlock,
9870
+ contextPackId: options.contextPackId,
9871
+ routeIntent: options.routeIntent,
9872
+ outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
9873
+ });
9874
+ const now = new Date().toISOString();
9875
+ const uniqueSuffix = createHash('sha1').update(`${now}:${draft.path}:${dqlArtifact.source}`).digest('hex').slice(0, 12);
9876
+ const importId = `imp_dql_artifact_${uniqueSuffix}`;
9877
+ const candidateId = `cand_${slug.replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') || 'dql_artifact'}_${uniqueSuffix.slice(0, 6)}`;
9878
+ const sourcePath = dqlArtifact.sourcePath ?? options.inputPath ?? draft.path;
9879
+ const savedSource = readFileSync(join(projectRoot, draft.path), 'utf-8');
9880
+ const candidate = {
9881
+ id: candidateId,
9882
+ sourceKind: 'raw-sql-file',
9883
+ sourcePath,
9884
+ name: dqlArtifact.name ?? slug,
9885
+ domain,
9886
+ description: question,
9887
+ owner,
9888
+ tags,
9889
+ grain: dqlArtifact.timeDimension ? `${dqlArtifact.timeDimension.name}.${dqlArtifact.timeDimension.granularity}` : undefined,
9890
+ outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
9891
+ dimensions: dqlArtifact.dimensions,
9892
+ allowedFilters: dqlArtifact.filters?.map((filter) => filter.dimension),
9893
+ parameterPolicy: dqlArtifact.filters?.map((filter) => ({ name: filter.dimension, policy: 'dynamic' })),
9894
+ sql: notebookResearchString(options.generatedSql) ?? '',
9895
+ dqlSource: savedSource,
9896
+ validation: validateBlockStudioSource(savedSource, semanticLayer),
9897
+ preview: null,
9898
+ lineage: {
9899
+ sourceTables: [],
9900
+ parameters: [],
9901
+ warnings: [],
9902
+ statementIndex: 1,
9903
+ totalStatements: 1,
9904
+ },
9905
+ confidence: 0.95,
9906
+ splitStrategy: 'manual',
9907
+ warnings: [],
9908
+ conversionNotes: ['Persisted the DQL artifact returned by the governed answer; no SQL-to-DQL regeneration was required.'],
9909
+ aiAssistance: [],
9910
+ reviewStatus: 'draft',
9911
+ savedPath: draft.path,
9912
+ generationMode: 'deterministic',
9913
+ generationProvider: 'dql-artifact',
9914
+ llmContext: `Persisted generated DQL artifact for: ${question}`,
9915
+ evidence: [],
9916
+ draftSave: { status: 'saved', path: draft.path, savedAt: now },
9917
+ recommendedAction: 'create_new',
9918
+ similarityMatches: [],
9919
+ };
9920
+ const session = {
9921
+ id: importId,
9922
+ sourceKind: 'raw-sql-file',
9923
+ inputPath: sourcePath,
9924
+ inputMode: options.inputMode ?? 'upload',
9925
+ sourceFiles: [sourcePath],
9926
+ createdAt: now,
9927
+ updatedAt: now,
9928
+ defaults: { domain, owner, tags },
9929
+ candidateIds: [candidate.id],
9930
+ mode: 'ai-import',
9931
+ generation: {
9932
+ provider: 'dql-artifact',
9933
+ aiEnabled: false,
9934
+ contextObjectCount: 0,
9935
+ createdDrafts: 1,
9936
+ warnings: [],
9937
+ },
9938
+ candidates: [candidate],
9939
+ };
9940
+ writeBlockStudioImportSession(projectRoot, session);
9941
+ await refreshLocalMetadataCatalog(projectRoot);
9942
+ return session;
9943
+ }
8602
9944
  export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
8603
9945
  const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
8604
9946
  const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
@@ -9597,7 +10939,13 @@ function inferDqlGenerationPattern(sql) {
9597
10939
  return 'trend';
9598
10940
  if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
9599
10941
  return 'entity_rollup';
9600
- if (!/\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql) && /\b(dim|profile|customer|account|player|product|user|entity)\b/i.test(sql)) {
10942
+ // Entity profile = a non-aggregated, un-grouped row lookup filtered by an
10943
+ // identifier (…WHERE <something>_id = / IN …). Structural + domain-agnostic —
10944
+ // no hard-coded entity vocabulary.
10945
+ const hasAggregate = /\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql);
10946
+ const hasGroupBy = /\bgroup\s+by\b/i.test(sql);
10947
+ const filtersByIdentifier = /\bwhere\b[\s\S]*?\b[A-Za-z_][A-Za-z0-9_]*(?:_id|_key|id)\b\s*(?:=|in\b)/i.test(sql);
10948
+ if (!hasAggregate && !hasGroupBy && filtersByIdentifier) {
9601
10949
  return 'entity_profile';
9602
10950
  }
9603
10951
  return 'custom';
@@ -10195,10 +11543,15 @@ async function createBlockStudioAssistProvider(projectRoot, requestedProvider) {
10195
11543
  provider = new OpenAIProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model, allowNoApiKey: true });
10196
11544
  break;
10197
11545
  case 'claude-code':
10198
- provider = new ClaudeCodeCliProvider({ model: config.model });
11546
+ // OAuth-first: use the browser-login token when connected; fall back to the CLI passthrough.
11547
+ provider = claudeOAuthConnected(projectRoot)
11548
+ ? new ClaudeOAuthProvider({ projectRoot, model: config.model })
11549
+ : new ClaudeCodeCliProvider({ model: config.model });
10199
11550
  break;
10200
11551
  case 'codex':
10201
- provider = new CodexCliProvider({ model: config.model });
11552
+ provider = codexOAuthConnected(projectRoot)
11553
+ ? new CodexOAuthProvider({ projectRoot, model: config.model })
11554
+ : new CodexCliProvider({ model: config.model });
10202
11555
  break;
10203
11556
  default:
10204
11557
  return null;
@@ -10212,8 +11565,7 @@ function agentResultToSynthesisPreview(result) {
10212
11565
  const columns = Array.isArray(result.columns)
10213
11566
  ? result.columns.map((col) => typeof col === 'string' ? col : col?.name ?? String(col))
10214
11567
  : [];
10215
- const rawRows = Array.isArray(result.rows) ? result.rows.slice(0, 20) : [];
10216
- const rows = rawRows.map((row) => {
11568
+ const normalizeRow = (row) => {
10217
11569
  if (row && typeof row === 'object' && !Array.isArray(row))
10218
11570
  return row;
10219
11571
  // Array-shaped rows → zip with column names.
@@ -10223,69 +11575,20 @@ function agentResultToSynthesisPreview(result) {
10223
11575
  return obj;
10224
11576
  }
10225
11577
  return { value: row };
10226
- });
11578
+ };
11579
+ // Normalize ALL rows so verified statistics cover the full result — the
11580
+ // narrator sees only a 20-row sample, and sample-derived aggregate claims
11581
+ // ("spend ranges from X to Y") were the last hallucination surface.
11582
+ const allRows = Array.isArray(result.rows) ? result.rows.map(normalizeRow) : [];
11583
+ const rows = allRows.slice(0, 20);
11584
+ const resolvedColumns = columns.length > 0 ? columns : Object.keys(rows[0] ?? {});
10227
11585
  return {
10228
- columns: columns.length > 0 ? columns : Object.keys(rows[0] ?? {}),
11586
+ columns: resolvedColumns,
10229
11587
  rows,
10230
- rowCount: typeof result.rowCount === 'number' ? result.rowCount : rows.length,
11588
+ rowCount: typeof result.rowCount === 'number' ? result.rowCount : allRows.length,
11589
+ stats: computeResultStats(resolvedColumns, allRows),
10231
11590
  };
10232
11591
  }
10233
- const SIGNAL_STOPWORDS = new Set([
10234
- 'the', 'a', 'an', 'of', 'for', 'to', 'by', 'in', 'on', 'and', 'or', 'is', 'are', 'was', 'were',
10235
- 'what', 'which', 'how', 'show', 'me', 'my', 'our', 'this', 'that', 'top', 'give', 'get', 'list',
10236
- ]);
10237
- function signalTokens(text) {
10238
- return new Set(text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/)
10239
- .filter((token) => token.length > 2 && !SIGNAL_STOPWORDS.has(token)));
10240
- }
10241
- /** Best token-overlap ratio between the question and any candidate name/description. */
10242
- function bestOverlap(questionTokens, candidates) {
10243
- if (questionTokens.size === 0)
10244
- return 0;
10245
- let best = 0;
10246
- for (const candidate of candidates) {
10247
- const candTokens = signalTokens(candidate);
10248
- if (candTokens.size === 0)
10249
- continue;
10250
- let hits = 0;
10251
- for (const token of candTokens)
10252
- if (questionTokens.has(token))
10253
- hits += 1;
10254
- if (hits === 0)
10255
- continue;
10256
- // Ratio of the candidate's tokens the question covers — rewards a tight match.
10257
- const ratio = hits / Math.min(candTokens.size, questionTokens.size);
10258
- if (ratio > best)
10259
- best = ratio;
10260
- }
10261
- return Math.min(1, best);
10262
- }
10263
- /**
10264
- * Cheap, offline retrieval-signal probe. Fills certifiedScore/metricScore/hasRetrieval
10265
- * from local catalog name/description overlap so the deterministic router can reach
10266
- * confidence (and skip the LLM) on obvious governed matches. Best-effort — any failure
10267
- * yields empty signals and the router simply leans on its LLM classification.
10268
- */
10269
- function computeAgentRunSignals(projectRoot, question) {
10270
- try {
10271
- const tokens = signalTokens(question);
10272
- if (tokens.size === 0)
10273
- return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
10274
- const certifiedBlocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
10275
- const blockStrings = certifiedBlocks.flatMap((block) => [block.name, block.description ?? ''].filter(Boolean));
10276
- const metrics = loadSemanticMetrics(projectRoot);
10277
- const metricStrings = metrics.map((metric) => {
10278
- const node = metric;
10279
- return node.name ?? node.label ?? node.id ?? '';
10280
- }).filter(Boolean);
10281
- const certifiedScore = bestOverlap(tokens, blockStrings);
10282
- const metricScore = bestOverlap(tokens, metricStrings);
10283
- return { certifiedScore, metricScore, hasRetrieval: certifiedScore > 0 || metricScore > 0 };
10284
- }
10285
- catch {
10286
- return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
10287
- }
10288
- }
10289
11592
  /** Up to three example questions built from real catalog blocks (falls back to generic asks). */
10290
11593
  function buildConversationSuggestions(projectRoot, kind) {
10291
11594
  if (kind === 'gratitude')
@@ -10340,6 +11643,141 @@ function buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext,
10340
11643
  }
10341
11644
  return lines.join('\n');
10342
11645
  }
11646
+ function renderConversationMemoryForPrompt(context) {
11647
+ const recap = buildConversationContextRecap(context);
11648
+ if (!recap)
11649
+ return undefined;
11650
+ return [
11651
+ 'Current conversation context:',
11652
+ recap,
11653
+ 'If the user asks what this conversation is about, summarize this context directly. Do not query data or invent values.',
11654
+ ].join('\n');
11655
+ }
11656
+ function buildConversationContextRecap(context) {
11657
+ if (!context)
11658
+ return undefined;
11659
+ const turnRecap = buildConversationTurnsRecap(context);
11660
+ if (turnRecap)
11661
+ return turnRecap;
11662
+ const sourceQuestion = agentRunString(context.sourceQuestion);
11663
+ const sourceAnswerSummary = agentRunString(context.sourceAnswerSummary);
11664
+ const columns = agentRunStringArray(context.resultColumns ?? context.outputColumns).slice(0, 8);
11665
+ const values = agentRunRecord(context.resultDimensionValues);
11666
+ const rows = Array.isArray(context.resultRowsSample)
11667
+ ? context.resultRowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
11668
+ : [];
11669
+ const firstRow = rows[0];
11670
+ const leadParts = buildPriorValueLeadParts(values);
11671
+ const rowFacts = firstRow
11672
+ ? columns
11673
+ .slice(0, 5)
11674
+ .map((column) => {
11675
+ const value = firstRow[column];
11676
+ return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
11677
+ })
11678
+ .filter(Boolean)
11679
+ : [];
11680
+ const contextBits = [
11681
+ sourceQuestion ? `the question "${sourceQuestion}"` : '',
11682
+ sourceAnswerSummary ? `the latest answer summary: ${sourceAnswerSummary}` : '',
11683
+ columns.length ? `result columns: ${columns.join(', ')}` : '',
11684
+ leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
11685
+ rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
11686
+ ].filter(Boolean);
11687
+ if (contextBits.length === 0)
11688
+ return undefined;
11689
+ return `We were talking about ${contextBits.join('. ')}.`;
11690
+ }
11691
+ function buildConversationTurnsRecap(context) {
11692
+ const turns = Array.isArray(context.turns)
11693
+ ? context.turns.map(agentRunRecord).filter((turn) => Boolean(turn)).slice(-3)
11694
+ : [];
11695
+ if (turns.length === 0)
11696
+ return undefined;
11697
+ const activeTurnId = agentRunString(context.activeTurnId);
11698
+ const active = activeTurnId
11699
+ ? turns.find((turn) => agentRunString(turn.id) === activeTurnId)
11700
+ : undefined;
11701
+ const selected = active ?? turns[turns.length - 1];
11702
+ if (!selected)
11703
+ return undefined;
11704
+ const result = agentRunRecord(selected.result);
11705
+ const values = agentRunRecord(result?.dimensionValues);
11706
+ const columns = agentRunStringArray(result?.columns).slice(0, 8);
11707
+ const rows = Array.isArray(result?.rowsSample)
11708
+ ? result.rowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
11709
+ : [];
11710
+ const firstRow = rows[0];
11711
+ const leadParts = buildPriorValueLeadParts(values);
11712
+ const rowFacts = firstRow
11713
+ ? columns
11714
+ .slice(0, 5)
11715
+ .map((column) => {
11716
+ const value = firstRow[column];
11717
+ return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
11718
+ })
11719
+ .filter(Boolean)
11720
+ : [];
11721
+ const previous = turns
11722
+ .filter((turn) => turn !== selected)
11723
+ .map((turn) => agentRunString(turn.question))
11724
+ .filter((value) => Boolean(value))
11725
+ .slice(-2);
11726
+ const bits = [
11727
+ agentRunString(selected.question) ? `the latest question "${agentRunString(selected.question)}"` : '',
11728
+ agentRunString(selected.answerSummary) ? `latest answer summary: ${agentRunString(selected.answerSummary)}` : '',
11729
+ columns.length ? `result columns: ${columns.join(', ')}` : '',
11730
+ leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
11731
+ rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
11732
+ previous.length ? `earlier in this thread: ${previous.join(' / ')}` : '',
11733
+ ].filter(Boolean);
11734
+ return bits.length > 0 ? `We were talking about ${bits.join('. ')}.` : undefined;
11735
+ }
11736
+ function agentRunStringArray(value) {
11737
+ if (!Array.isArray(value))
11738
+ return [];
11739
+ return Array.from(new Set(value.map(agentRunString).filter((item) => Boolean(item))));
11740
+ }
11741
+ function firstPriorValue(values, keys) {
11742
+ if (!values)
11743
+ return undefined;
11744
+ for (const key of keys) {
11745
+ const list = Array.isArray(values[key]) ? values[key] : [];
11746
+ const first = list.map(agentRunString).find(Boolean);
11747
+ if (first)
11748
+ return first;
11749
+ }
11750
+ return undefined;
11751
+ }
11752
+ /**
11753
+ * Build human-readable "notable prior values" for conversation recap from whatever
11754
+ * dimension columns the prior result actually had. Domain-agnostic: name/label-ish
11755
+ * columns are surfaced first, then any other dimension — no hard-coded jaffle entity
11756
+ * columns — so follow-up questions get meaningful context on ANY repo.
11757
+ */
11758
+ function buildPriorValueLeadParts(values) {
11759
+ if (!values)
11760
+ return [];
11761
+ const keys = Object.keys(values);
11762
+ const nameLike = keys.filter((key) => /(?:^|_)(?:name|label|title|nm)$/i.test(key));
11763
+ const ordered = [...nameLike, ...keys.filter((key) => !nameLike.includes(key))];
11764
+ const parts = [];
11765
+ const seenLabels = new Set();
11766
+ for (const key of ordered) {
11767
+ if (parts.length >= 3)
11768
+ break;
11769
+ const value = firstPriorValue(values, [key]);
11770
+ if (!value)
11771
+ continue;
11772
+ const label = (key.replace(/_?(?:name|id|key|label|title|nm)$/i, '').replace(/[_-]+/g, ' ').trim()
11773
+ || key.replace(/[_-]+/g, ' ').trim()).toLowerCase();
11774
+ if (seenLabels.has(label))
11775
+ continue;
11776
+ seenLabels.add(label);
11777
+ parts.push(label ? `${label} "${value}"` : `"${value}"`);
11778
+ }
11779
+ return parts;
11780
+ }
10343
11781
  /** Deterministic, warm reply when no AI provider is configured — never governance-speak. */
10344
11782
  function buildConversationalFallback(kind, isGeneralKnowledge, question, suggestions) {
10345
11783
  const examples = suggestions.length > 0
@@ -12104,13 +13542,6 @@ function collectSettingsEnvStatus() {
12104
13542
  },
12105
13543
  ];
12106
13544
  }
12107
- function isProviderSettingsId(value) {
12108
- return value === 'anthropic'
12109
- || value === 'openai'
12110
- || value === 'gemini'
12111
- || value === 'ollama'
12112
- || value === 'custom-openai';
12113
- }
12114
13545
  function isDeterministicDqlGenerationProvider(value) {
12115
13546
  if (typeof value !== 'string')
12116
13547
  return false;
@@ -12131,8 +13562,8 @@ async function testProviderConfig(projectRoot, id, overrides) {
12131
13562
  const label = providerSettingsLabel(id);
12132
13563
  const details = providerConfigDetails(id, config);
12133
13564
  const isSubscriptionCli = id === 'claude-code' || id === 'codex';
12134
- // Subscription CLI providers are always testable — detection (installed + logged in)
12135
- // needs no saved config; the others require enabling/saving first.
13565
+ // Subscription providers are always testable — the connection (browser OAuth, or
13566
+ // an installed+logged-in CLI) needs no saved config; the others require enabling first.
12136
13567
  if (!config.enabled && !isSubscriptionCli) {
12137
13568
  return { ok: false, message: `${label} is disabled. Enable it (or Save) before testing.` };
12138
13569
  }
@@ -12140,6 +13571,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
12140
13571
  return testOpenAIProviderConfig(config, label, details);
12141
13572
  if (id === 'anthropic')
12142
13573
  return testAnthropicProviderConfig(config, label, details);
13574
+ // OAuth-first for subscriptions: when the user signed in via the browser, test the
13575
+ // OAuth-backed provider (no CLI required); otherwise fall back to the CLI passthrough.
13576
+ // Mirrors the runtime selection in dql-agent-provider / createBlockStudioAssistProvider.
13577
+ const claudeOAuth = id === 'claude-code' && claudeOAuthConnected(projectRoot);
13578
+ const codexOAuth = id === 'codex' && codexOAuthConnected(projectRoot);
12143
13579
  let provider;
12144
13580
  switch (id) {
12145
13581
  case 'gemini':
@@ -12149,10 +13585,14 @@ async function testProviderConfig(projectRoot, id, overrides) {
12149
13585
  provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
12150
13586
  break;
12151
13587
  case 'claude-code':
12152
- provider = new ClaudeCodeCliProvider({ model: config.model });
13588
+ provider = claudeOAuth
13589
+ ? new ClaudeOAuthProvider({ projectRoot, model: config.model })
13590
+ : new ClaudeCodeCliProvider({ model: config.model });
12153
13591
  break;
12154
13592
  case 'codex':
12155
- provider = new CodexCliProvider({ model: config.model });
13593
+ provider = codexOAuth
13594
+ ? new CodexOAuthProvider({ projectRoot, model: config.model })
13595
+ : new CodexCliProvider({ model: config.model });
12156
13596
  break;
12157
13597
  case 'custom-openai':
12158
13598
  default:
@@ -12162,10 +13602,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
12162
13602
  const available = await provider.available().catch(() => false);
12163
13603
  if (!available) {
12164
13604
  const cli = id === 'claude-code' ? 'claude' : 'codex';
13605
+ const signIn = id === 'claude-code' ? 'Sign in with Claude' : 'Sign in with ChatGPT';
12165
13606
  return {
12166
13607
  ok: false,
12167
13608
  message: isSubscriptionCli
12168
- ? `${label} is not ready. Install the \`${cli}\` CLI and run \`${cli} ${id === 'claude-code' ? '/login' : 'login'}\` with your subscription, then test again.`
13609
+ ? `${label} is not ready. Click "${signIn}" to sign in with your subscription, or install the \`${cli}\` CLI and run \`${cli} ${id === 'claude-code' ? '/login' : 'login'}\`, then test again.`
12169
13610
  : `${label} is not configured or reachable${details}. Check API key, base URL, and local service state.`,
12170
13611
  };
12171
13612
  }
@@ -12248,10 +13689,16 @@ function providerSettingsLabel(id) {
12248
13689
  }
12249
13690
  }
12250
13691
  function providerConfigDetails(id, config) {
13692
+ // Subscription providers (Claude Code / Codex) authenticate via OAuth/CLI, not an
13693
+ // API key — omit the key clause so we don't report a misleading "API key missing".
13694
+ const isSubscription = id === 'claude-code' || id === 'codex';
13695
+ const authNote = isSubscription
13696
+ ? ''
13697
+ : id === 'ollama' ? 'local endpoint' : config.apiKey ? 'API key present' : 'API key missing';
12251
13698
  const parts = [
12252
13699
  config.model ? `model ${config.model}` : '',
12253
13700
  config.baseUrl ? `base URL ${config.baseUrl}` : '',
12254
- id === 'ollama' ? 'local endpoint' : config.apiKey ? 'API key present' : 'API key missing',
13701
+ authNote,
12255
13702
  ].filter(Boolean);
12256
13703
  return parts.length ? ` (${parts.join(', ')})` : '';
12257
13704
  }
@@ -12273,7 +13720,7 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
12273
13720
  try {
12274
13721
  recordRuntimeSchemaSnapshot(projectRoot, {
12275
13722
  source,
12276
- tables: schemaContext.slice(0, 80).map((table) => ({
13723
+ tables: schemaContext.map((table) => ({
12277
13724
  relation: table.relation,
12278
13725
  schema: table.schema,
12279
13726
  name: table.name,
@@ -12427,7 +13874,9 @@ function buildNotebookResearchReviewChecklist(run) {
12427
13874
  const questionReady = Boolean(notebookResearchString(run.question));
12428
13875
  const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
12429
13876
  const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
13877
+ const hasDqlArtifact = Boolean(normalizeDqlArtifactReference(run.dqlArtifact)?.source);
12430
13878
  const hasSql = hasReviewedSql || hasGeneratedSql;
13879
+ const hasPromotableSource = hasReviewedSql || hasDqlArtifact;
12431
13880
  const preview = notebookResearchPreviewInfo(run.resultPreview);
12432
13881
  const previewReady = run.status === 'ready' && preview.hasPreview;
12433
13882
  const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
@@ -12446,8 +13895,8 @@ function buildNotebookResearchReviewChecklist(run) {
12446
13895
  const warnings = [];
12447
13896
  if (!questionReady)
12448
13897
  blockers.push('Add a business question before review.');
12449
- if (!hasSql && !reuseRecommended)
12450
- blockers.push('Add reviewed SQL or generate SQL before promotion.');
13898
+ if (!hasSql && !hasDqlArtifact && !reuseRecommended)
13899
+ blockers.push('Add reviewed SQL, generate SQL, or attach a DQL artifact before promotion.');
12451
13900
  if (run.status === 'error')
12452
13901
  blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
12453
13902
  if (!previewReady && run.status !== 'error' && !reuseRecommended)
@@ -12473,15 +13922,17 @@ function buildNotebookResearchReviewChecklist(run) {
12473
13922
  },
12474
13923
  {
12475
13924
  id: 'sql',
12476
- label: 'Reviewed SQL',
12477
- status: hasReviewedSql || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
13925
+ label: hasDqlArtifact && !hasReviewedSql ? 'DQL artifact' : 'Reviewed SQL',
13926
+ status: hasReviewedSql || hasDqlArtifact || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
12478
13927
  detail: reuseRecommended && !hasSql
12479
13928
  ? 'Existing certified DQL should be reused; no new SQL is required.'
12480
- : hasReviewedSql
12481
- ? 'Reviewer SQL is saved.'
12482
- : hasGeneratedSql
12483
- ? 'Generated SQL exists; review it before promotion.'
12484
- : 'No SQL is available for preview or DQL generation.',
13929
+ : hasDqlArtifact && !hasReviewedSql
13930
+ ? 'Generated DQL artifact is saved for reviewer promotion.'
13931
+ : hasReviewedSql
13932
+ ? 'Reviewer SQL is saved.'
13933
+ : hasGeneratedSql
13934
+ ? 'Generated SQL exists; review it before promotion.'
13935
+ : 'No SQL is available for preview or DQL generation.',
12485
13936
  },
12486
13937
  {
12487
13938
  id: 'preview',
@@ -12518,33 +13969,35 @@ function buildNotebookResearchReviewChecklist(run) {
12518
13969
  {
12519
13970
  id: 'parameters',
12520
13971
  label: 'Parameter review',
12521
- status: reuseRecommended && !hasSql ? 'passed' : !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
13972
+ status: reuseRecommended && !hasSql ? 'passed' : !hasSql && !hasDqlArtifact ? 'pending' : hasDqlArtifact && !hasSql ? 'passed' : dynamicParameters.length > 0 ? 'passed' : 'warning',
12522
13973
  detail: reuseRecommended && !hasSql
12523
13974
  ? 'Use the certified block parameters; no new SQL parameterization is required.'
12524
- : !hasSql
12525
- ? 'Add SQL before checking runtime parameters.'
12526
- : dynamicParameters.length > 0
12527
- ? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
12528
- : staticParameters.length > 0
12529
- ? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
12530
- : 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
13975
+ : !hasSql && hasDqlArtifact
13976
+ ? 'Review parameters declared in the DQL artifact.'
13977
+ : !hasSql
13978
+ ? 'Add SQL or a DQL artifact before checking runtime parameters.'
13979
+ : dynamicParameters.length > 0
13980
+ ? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
13981
+ : staticParameters.length > 0
13982
+ ? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
13983
+ : 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
12531
13984
  },
12532
13985
  {
12533
13986
  id: 'dql_draft',
12534
13987
  label: 'DQL draft',
12535
- status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasReviewedSql && evidenceReady ? 'pending' : 'blocked',
13988
+ status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasPromotableSource && evidenceReady ? 'pending' : 'blocked',
12536
13989
  detail: reuseRecommended && !hasDraft
12537
13990
  ? 'No new draft is required because an existing DQL block should be reused.'
12538
13991
  : hasDraft
12539
13992
  ? `Draft saved at ${run.draftBlockPath}.`
12540
- : !hasReviewedSql
12541
- ? 'Save reviewed SQL before DQL draft creation.'
13993
+ : !hasPromotableSource
13994
+ ? 'Save reviewed SQL or review the generated DQL artifact before DQL draft creation.'
12542
13995
  : evidenceReady
12543
- ? 'Create a draft after SQL review.'
13996
+ ? (hasDqlArtifact && !hasReviewedSql ? 'Create a draft after DQL artifact review.' : 'Create a draft after SQL review.')
12544
13997
  : 'Save metadata evidence before creating a reusable DQL draft.',
12545
13998
  },
12546
13999
  ];
12547
- const readyForDqlDraft = questionReady && hasReviewedSql && evidenceReady && run.status !== 'error';
14000
+ const readyForDqlDraft = questionReady && hasPromotableSource && evidenceReady && run.status !== 'error';
12548
14001
  const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
12549
14002
  return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
12550
14003
  }
@@ -12552,6 +14005,8 @@ function buildNotebookResearchPlan(input) {
12552
14005
  const reviewedSql = notebookResearchString(input.reviewedSql);
12553
14006
  const generatedSql = notebookResearchString(input.generatedSql);
12554
14007
  const sql = reviewedSql ?? generatedSql;
14008
+ const dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? input.run.dqlArtifact;
14009
+ const hasDqlArtifact = Boolean(dqlArtifact?.source);
12555
14010
  const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
12556
14011
  const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
12557
14012
  ? input.evidence
@@ -12590,6 +14045,7 @@ function buildNotebookResearchPlan(input) {
12590
14045
  promotion: {
12591
14046
  path: notebookResearchPlanPromotionPath({
12592
14047
  hasSql: Boolean(sql),
14048
+ hasDqlArtifact,
12593
14049
  hasEvidence,
12594
14050
  hasPreview,
12595
14051
  previewError: input.previewError,
@@ -12600,6 +14056,7 @@ function buildNotebookResearchPlan(input) {
12600
14056
  },
12601
14057
  reviewFocus: notebookResearchPlanFocus(input.run.intent, {
12602
14058
  hasSql: Boolean(sql),
14059
+ hasDqlArtifact,
12603
14060
  hasEvidence,
12604
14061
  hasPreview,
12605
14062
  dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
@@ -12612,7 +14069,7 @@ function buildNotebookResearchPlan(input) {
12612
14069
  function notebookResearchPlanPromotionPath(input) {
12613
14070
  if (input.promotionAction === 'reuse_existing')
12614
14071
  return 'reuse_existing';
12615
- if (!input.hasSql)
14072
+ if (!input.hasSql && !input.hasDqlArtifact)
12616
14073
  return 'needs_sql';
12617
14074
  if (!input.hasEvidence)
12618
14075
  return 'review_context';
@@ -12624,8 +14081,8 @@ function notebookResearchPlanPromotionPath(input) {
12624
14081
  }
12625
14082
  function notebookResearchPlanFocus(intent, input) {
12626
14083
  const focus = new Set();
12627
- if (!input.hasSql)
12628
- focus.add('Add reviewed SQL or configure AI SQL generation.');
14084
+ if (!input.hasSql && !input.hasDqlArtifact)
14085
+ focus.add('Add reviewed SQL, a DQL artifact, or configure AI generation.');
12629
14086
  if (!input.hasEvidence)
12630
14087
  focus.add('Save metadata context evidence before DQL promotion.');
12631
14088
  if (!input.hasPreview)
@@ -13096,7 +14553,7 @@ function dedupeAgentSchemaColumns(columns) {
13096
14553
  }
13097
14554
  return Array.from(byName.values());
13098
14555
  }
13099
- export function buildAgentSchemaContext(question, rows) {
14556
+ export function buildAgentSchemaContext(question, rows, options = {}) {
13100
14557
  const byRelation = new Map();
13101
14558
  for (const row of rows) {
13102
14559
  if (!row || typeof row !== 'object')
@@ -13125,16 +14582,82 @@ export function buildAgentSchemaContext(question, rows) {
13125
14582
  }
13126
14583
  const tokens = agentSchemaTokens(question);
13127
14584
  const shouldProbeValues = extractAgentValueSearchTerms(question).length > 0;
14585
+ const limit = Math.max(1, Math.trunc(options.limit ?? 12));
13128
14586
  return Array.from(byRelation.values())
13129
14587
  .map((table) => ({
13130
14588
  table,
13131
14589
  score: scoreAgentSchemaTable(table, tokens) + (shouldProbeValues ? scoreAgentValueProbeTable(table) : 0),
13132
14590
  }))
13133
- .filter((entry) => entry.score > 0)
14591
+ .filter((entry) => options.includeUnscored || entry.score > 0)
13134
14592
  .sort((a, b) => b.score - a.score || a.table.relation.localeCompare(b.table.relation))
13135
- .slice(0, 12)
14593
+ .slice(0, limit)
13136
14594
  .map((entry) => entry.table);
13137
14595
  }
14596
+ function mergeAgentSchemaContexts(...contexts) {
14597
+ const byRelation = new Map();
14598
+ for (const table of contexts.flat()) {
14599
+ const key = table.relation.toLowerCase();
14600
+ const existing = byRelation.get(key);
14601
+ if (!existing) {
14602
+ byRelation.set(key, {
14603
+ ...table,
14604
+ columns: dedupeAgentSchemaColumns(table.columns).slice(0, 80),
14605
+ });
14606
+ continue;
14607
+ }
14608
+ byRelation.set(key, {
14609
+ ...existing,
14610
+ description: existing.description ?? table.description,
14611
+ source: existing.source === table.source ? existing.source : 'catalog and runtime schema',
14612
+ columns: dedupeAgentSchemaColumns([...existing.columns, ...table.columns]).slice(0, 80),
14613
+ });
14614
+ }
14615
+ return Array.from(byRelation.values()).slice(0, 16);
14616
+ }
14617
+ function mergeAgentSchemaSampleValues(snapshot, enriched) {
14618
+ const samples = new Map();
14619
+ for (const table of enriched) {
14620
+ for (const column of table.columns) {
14621
+ if (!column.sampleValues?.length)
14622
+ continue;
14623
+ samples.set(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`, column.sampleValues);
14624
+ }
14625
+ }
14626
+ if (samples.size === 0)
14627
+ return snapshot;
14628
+ return snapshot.map((table) => ({
14629
+ ...table,
14630
+ columns: table.columns.map((column) => {
14631
+ const sampleValues = samples.get(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`);
14632
+ return sampleValues?.length
14633
+ ? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0, 8) }
14634
+ : column;
14635
+ }),
14636
+ }));
14637
+ }
14638
+ // Common English plurals that are NOT entity nouns, so a multi-entity heuristic
14639
+ // doesn't count them. Domain-agnostic (no project-specific vocabulary).
14640
+ const NON_ENTITY_PLURALS = new Set([
14641
+ 'details', 'values', 'results', 'rows', 'numbers', 'records', 'items', 'things',
14642
+ 'totals', 'amounts', 'metrics', 'columns', 'fields', 'names', 'ids',
14643
+ ]);
14644
+ /**
14645
+ * Decide whether to run the (advisory, best-effort) runtime value-scan + schema
14646
+ * enrichment for a question. Generic multi-entity / detail intent — no hard-coded
14647
+ * project vocabulary — so it fires for ANY repo's join-shaped questions, not just
14648
+ * jaffle. Over-triggering only costs a few extra bounded probes (all in try/catch).
14649
+ */
14650
+ function shouldAugmentAgentRuntimeSchema(question) {
14651
+ const lower = question.toLowerCase();
14652
+ const wantsMetric = /\brevenu|\bsales\b|\bamount\b|\bspend\b|\bcost\b|\bcount\b|\btotal\b|\bsum\b|\bavg\b|\baverage\b|\bvalue\b|\bprice\b/.test(lower);
14653
+ const wantsDetail = /\bdetails?\b|\bcomplete\b|\bbreakdown\b|\ball\s+values?\b|\beach\b|\bevery\b/.test(lower);
14654
+ const referencesPriorRows = /\b(?:above|previous|prior|these|those|same)\s+\w+/.test(lower);
14655
+ const explicitJoin = /\bjoin(?:ed|ing)?\b|\bcombine[ds]?\b|\bcross[- ]?reference\b|\balong\s+with\b|\btogether\s+with\b/.test(lower);
14656
+ // Multi-entity intent: a linking word + >= 2 distinct plural content nouns.
14657
+ const nouns = new Set((lower.match(/\b[a-z]{4,}s\b/g) ?? []).filter((word) => !NON_ENTITY_PLURALS.has(word)));
14658
+ const multiEntity = /\b(?:and|with|plus|per|for\s+each|by)\b/.test(lower) && nouns.size >= 2;
14659
+ return explicitJoin || referencesPriorRows || (multiEntity && (wantsMetric || wantsDetail));
14660
+ }
13138
14661
  async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
13139
14662
  const searchTerms = extractAgentValueSearchTerms(question);
13140
14663
  if (schemaContext.length === 0 || searchTerms.length === 0)
@@ -13186,8 +14709,57 @@ function scoreAgentSchemaTable(table, tokens) {
13186
14709
  score += 3;
13187
14710
  }
13188
14711
  }
13189
- if (/(customer|order|revenue|product|location|date|month)/i.test(table.name))
14712
+ score += scoreCompositeAgentSchemaTable(table, tokens);
14713
+ // Small tiebreaker toward modeled analytics tables (fct_/dim_/fact_/mart_ …),
14714
+ // a warehouse-modeling convention that generalizes across repos — not a fixed
14715
+ // list of jaffle entity names.
14716
+ if (/^(?:fct|fact|dim|dimension|mart|rpt|report|agg|f|d)[_-]/i.test(table.name)
14717
+ || /(?:^|[_-])(?:fact|dim|mart|summary|report)(?:$|[_-])/i.test(table.name)) {
13190
14718
  score += 1;
14719
+ }
14720
+ return score;
14721
+ }
14722
+ const COMPOSITE_METRIC_TOKENS = new Set([
14723
+ 'revenue', 'sales', 'amount', 'spend', 'price', 'cost', 'total', 'sum',
14724
+ 'count', 'quantity', 'qty', 'value', 'subtotal', 'order', 'orders',
14725
+ ]);
14726
+ const COMPOSITE_STOPWORD_TOKENS = new Set([
14727
+ 'the', 'and', 'with', 'for', 'each', 'per', 'all', 'top', 'show', 'list',
14728
+ 'give', 'get', 'find', 'plus', 'along', 'together', 'combined', 'combine',
14729
+ 'detail', 'details', 'complete', 'breakdown', 'values', 'name', 'names',
14730
+ ...COMPOSITE_METRIC_TOKENS,
14731
+ ]);
14732
+ /**
14733
+ * Boost the fact/junction table for a multi-entity ("composite") question. Fully
14734
+ * domain-agnostic: the ask is detected from >= 2 distinct entity tokens + a metric,
14735
+ * and the winning table is identified STRUCTURALLY (>= 2 foreign-key-style id columns
14736
+ * plus a numeric measure column) rather than by hard-coded jaffle table names — so it
14737
+ * ranks the right join table for any repo, while jaffle's fct_orders still qualifies.
14738
+ */
14739
+ function scoreCompositeAgentSchemaTable(table, tokens) {
14740
+ const entityTokens = [...tokens].filter((token) => token.length >= 3 && !COMPOSITE_STOPWORD_TOKENS.has(token));
14741
+ const hasMetric = [...tokens].some((token) => COMPOSITE_METRIC_TOKENS.has(token));
14742
+ const hasCompositeAsk = entityTokens.length >= 2 && hasMetric;
14743
+ const hasPriorRowsAsk = ['above', 'previous', 'prior', 'these', 'those', 'same'].some((token) => tokens.has(token))
14744
+ && entityTokens.length >= 1;
14745
+ if (!hasCompositeAsk && !hasPriorRowsAsk)
14746
+ return 0;
14747
+ const columns = table.columns.map((column) => column.name.toLowerCase());
14748
+ let score = 0;
14749
+ // Structural fact/junction-table signal: multiple FK-style id columns + a measure.
14750
+ const idColumnCount = columns.filter((name) => /(?:^id$|_id$|_key$|_sk$|_uuid$|_fk$)/.test(name)).length;
14751
+ const hasMeasureColumn = columns.some((name) => /revenue|sales|amount|price|spend|cost|subtotal|total|quantity|\bqty\b|count|value/.test(name));
14752
+ if (idColumnCount >= 2 && hasMeasureColumn)
14753
+ score += 18;
14754
+ else if (idColumnCount >= 2)
14755
+ score += 8;
14756
+ if (hasMeasureColumn)
14757
+ score += 8;
14758
+ // Reward columns whose names overlap the question's actual entity tokens (any domain).
14759
+ for (const token of entityTokens) {
14760
+ if (columns.some((name) => name.includes(token)))
14761
+ score += 4;
14762
+ }
13191
14763
  return score;
13192
14764
  }
13193
14765
  function scoreAgentValueProbeTable(table) {