@duckcodeailabs/dql-cli 1.6.30 → 1.6.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/apps-api.d.ts.map +1 -1
  2. package/dist/apps-api.js +3 -1
  3. package/dist/apps-api.js.map +1 -1
  4. package/dist/args.d.ts +18 -0
  5. package/dist/args.d.ts.map +1 -1
  6. package/dist/args.js +37 -0
  7. package/dist/args.js.map +1 -1
  8. package/dist/assets/dql-notebook/assets/index-B6x1cRJo.js +5698 -0
  9. package/dist/assets/dql-notebook/index.html +1 -1
  10. package/dist/commands/agent.d.ts +47 -1
  11. package/dist/commands/agent.d.ts.map +1 -1
  12. package/dist/commands/agent.js +635 -27
  13. package/dist/commands/agent.js.map +1 -1
  14. package/dist/commands/app.d.ts.map +1 -1
  15. package/dist/commands/app.js +3 -1
  16. package/dist/commands/app.js.map +1 -1
  17. package/dist/commands/diff.d.ts.map +1 -1
  18. package/dist/commands/diff.js +111 -22
  19. package/dist/commands/diff.js.map +1 -1
  20. package/dist/commands/eval-judge.d.ts +37 -0
  21. package/dist/commands/eval-judge.d.ts.map +1 -0
  22. package/dist/commands/eval-judge.js +99 -0
  23. package/dist/commands/eval-judge.js.map +1 -0
  24. package/dist/commands/eval.d.ts +27 -1
  25. package/dist/commands/eval.d.ts.map +1 -1
  26. package/dist/commands/eval.js +143 -6
  27. package/dist/commands/eval.js.map +1 -1
  28. package/dist/commands/mcp.js +27 -3
  29. package/dist/commands/mcp.js.map +1 -1
  30. package/dist/index.js +6 -0
  31. package/dist/index.js.map +1 -1
  32. package/dist/llm/analytics-tools.d.ts.map +1 -1
  33. package/dist/llm/analytics-tools.js +12 -7
  34. package/dist/llm/analytics-tools.js.map +1 -1
  35. package/dist/llm/answer-loop-tools.d.ts +7 -0
  36. package/dist/llm/answer-loop-tools.d.ts.map +1 -0
  37. package/dist/llm/answer-loop-tools.js +67 -0
  38. package/dist/llm/answer-loop-tools.js.map +1 -0
  39. package/dist/llm/proposal-metadata.d.ts +5 -0
  40. package/dist/llm/proposal-metadata.d.ts.map +1 -0
  41. package/dist/llm/proposal-metadata.js +27 -0
  42. package/dist/llm/proposal-metadata.js.map +1 -0
  43. package/dist/llm/providers/claude-agent-sdk.d.ts +6 -0
  44. package/dist/llm/providers/claude-agent-sdk.d.ts.map +1 -1
  45. package/dist/llm/providers/claude-agent-sdk.js +17 -5
  46. package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
  47. package/dist/llm/providers/claude-code.d.ts +7 -1
  48. package/dist/llm/providers/claude-code.d.ts.map +1 -1
  49. package/dist/llm/providers/claude-code.js +11 -2
  50. package/dist/llm/providers/claude-code.js.map +1 -1
  51. package/dist/llm/providers/dql-agent-provider.d.ts +29 -0
  52. package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
  53. package/dist/llm/providers/dql-agent-provider.js +777 -32
  54. package/dist/llm/providers/dql-agent-provider.js.map +1 -1
  55. package/dist/llm/providers/native-sdk-provider.d.ts.map +1 -1
  56. package/dist/llm/providers/native-sdk-provider.js +7 -1
  57. package/dist/llm/providers/native-sdk-provider.js.map +1 -1
  58. package/dist/llm/tools.d.ts.map +1 -1
  59. package/dist/llm/tools.js +34 -137
  60. package/dist/llm/tools.js.map +1 -1
  61. package/dist/llm/types.d.ts +73 -2
  62. package/dist/llm/types.d.ts.map +1 -1
  63. package/dist/local-runtime.d.ts +160 -5
  64. package/dist/local-runtime.d.ts.map +1 -1
  65. package/dist/local-runtime.js +1862 -181
  66. package/dist/local-runtime.js.map +1 -1
  67. package/dist/package.json +10 -10
  68. package/dist/providers/oauth/claude-oauth.d.ts +80 -0
  69. package/dist/providers/oauth/claude-oauth.d.ts.map +1 -0
  70. package/dist/providers/oauth/claude-oauth.js +352 -0
  71. package/dist/providers/oauth/claude-oauth.js.map +1 -0
  72. package/dist/providers/oauth/codex-oauth.d.ts +70 -0
  73. package/dist/providers/oauth/codex-oauth.d.ts.map +1 -0
  74. package/dist/providers/oauth/codex-oauth.js +413 -0
  75. package/dist/providers/oauth/codex-oauth.js.map +1 -0
  76. package/dist/providers/oauth/oauth-store.d.ts +31 -0
  77. package/dist/providers/oauth/oauth-store.d.ts.map +1 -0
  78. package/dist/providers/oauth/oauth-store.js +61 -0
  79. package/dist/providers/oauth/oauth-store.js.map +1 -0
  80. package/dist/settings/provider-settings.d.ts +18 -0
  81. package/dist/settings/provider-settings.d.ts.map +1 -1
  82. package/dist/settings/provider-settings.js +44 -10
  83. package/dist/settings/provider-settings.js.map +1 -1
  84. package/package.json +11 -11
  85. package/dist/assets/dql-notebook/assets/index-QJKCrLXe.js +0 -5648
@@ -1,4 +1,5 @@
1
1
  import { execFileSync, execSync } from 'node:child_process';
2
+ import { createHash } from 'node:crypto';
2
3
  import { createServer } from 'node:http';
3
4
  import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
4
5
  import { createRequire } from 'node:module';
@@ -7,17 +8,19 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
7
8
  import Anthropic from '@anthropic-ai/sdk';
8
9
  import OpenAI from 'openai';
9
10
  import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
10
- import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
11
+ import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
11
12
  import { load as loadYaml } from 'js-yaml';
12
13
  import { listBlockTemplates } from './block-templates.js';
13
14
  import { getRunner as getLLMRunner } from './llm/index.js';
14
15
  import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
15
16
  import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
16
- import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, } from '@duckcodeailabs/dql-agent';
17
+ import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
17
18
  import { gatherProposeEnrichment } from './propose-enrich.js';
18
19
  import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
19
- import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
20
+ import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
20
21
  import { ClaudeCodeCliProvider, CodexCliProvider } from './providers/subscription-cli.js';
22
+ import { ClaudeOAuthManager, ClaudeOAuthProvider, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
23
+ import { CodexOAuthManager, CodexOAuthProvider, codexOAuthConnected, CODEX_OAUTH_MODELS, CODEX_OAUTH_DEFAULT_MODEL, } from './providers/oauth/codex-oauth.js';
21
24
  import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
22
25
  import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
23
26
  import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
@@ -76,6 +79,16 @@ function parseAgentRunRequestedMode(value) {
76
79
  ? value
77
80
  : undefined;
78
81
  }
82
+ function parseAgentRunReasoningEffort(value) {
83
+ return value === 'low' || value === 'medium' || value === 'high'
84
+ ? value
85
+ : undefined;
86
+ }
87
+ function parseAgentRunAnalysisDepth(value) {
88
+ return value === 'quick' || value === 'deep'
89
+ ? value
90
+ : undefined;
91
+ }
79
92
  function parseAgentRunSelectedObject(value) {
80
93
  const record = agentRunRecord(value);
81
94
  if (!record)
@@ -103,7 +116,7 @@ function parseAgentRunHistory(value) {
103
116
  });
104
117
  return history.length > 0 ? history.slice(-20) : undefined;
105
118
  }
106
- function parseAgentRunRequestBody(body) {
119
+ export function parseAgentRunRequestBody(body) {
107
120
  const record = agentRunRecord(body);
108
121
  if (!record)
109
122
  return { error: 'Invalid JSON body.' };
@@ -126,11 +139,181 @@ function parseAgentRunRequestBody(body) {
126
139
  signals: signals,
127
140
  selectedObject,
128
141
  workspaceContext,
142
+ conversationContext: agentRunRecord(record.conversationContext),
129
143
  history: parseAgentRunHistory(record.history),
144
+ threadId: agentRunString(record.threadId),
130
145
  runId: agentRunString(record.runId),
146
+ reasoningEffort: parseAgentRunReasoningEffort(record.reasoningEffort),
147
+ analysisDepth: parseAgentRunAnalysisDepth(record.analysisDepth) ?? parseAgentRunAnalysisDepth(record.depth),
131
148
  },
132
149
  };
133
150
  }
151
+ export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
152
+ if (governedAnswer.kind === 'no_answer')
153
+ return false;
154
+ if (governedAnswer.kind === 'certified' || governedAnswer.certification === 'certified')
155
+ return false;
156
+ const finalText = (governedAnswer.answer ?? governedAnswer.text ?? '').trim();
157
+ if (governedAnswer.dqlArtifact && finalText)
158
+ return false;
159
+ return true;
160
+ }
161
+ // ── Server-side conversation threads ────────────────────────────────────────
162
+ // When a run carries a threadId, the persisted thread is the authoritative
163
+ // source of prior turns (survives refresh); the client-built context remains
164
+ // the fallback for embedders that never send a threadId.
165
+ async function conversationContextFromThread(store, threadId, clientContext, question) {
166
+ const turns = store.recentTurns(threadId, 8).map((turn) => {
167
+ const contract = turn.contract ?? {};
168
+ const topN = contract.topN;
169
+ const topNValue = typeof topN === 'number'
170
+ ? topN
171
+ : topN && typeof topN === 'object' && typeof topN.n === 'number'
172
+ ? topN.n
173
+ : undefined;
174
+ return compactConversationRecord({
175
+ id: turn.id,
176
+ question: turn.question,
177
+ answerSummary: turn.answerSummary,
178
+ sourceCertifiedBlock: turn.sourceCertifiedBlock,
179
+ route: turn.route,
180
+ trustLabel: turn.trustLabel,
181
+ certification: turn.certification,
182
+ contextPackId: turn.contextPackId,
183
+ dqlArtifact: turn.dqlArtifact,
184
+ cascade: turn.cascade,
185
+ sourceSql: turn.sql,
186
+ requestedFilters: conversationStringArray(contract.filters),
187
+ requestedDimensions: conversationStringArray(contract.dimensions),
188
+ requestedMeasures: conversationStringArray(contract.measures),
189
+ topN: topNValue,
190
+ result: turn.result
191
+ ? compactConversationRecord({
192
+ columns: turn.result.columns,
193
+ rowsSample: turn.result.rowsSample,
194
+ dimensionValues: turn.result.dimensionValues,
195
+ measureColumns: turn.result.measureColumns,
196
+ rowCount: turn.result.rowCount,
197
+ })
198
+ : undefined,
199
+ });
200
+ }).filter((turn) => Boolean(turn));
201
+ const thread = store.getThread(threadId);
202
+ // Bounded structured snapshot (working state + rolling summary + topic relation)
203
+ // for the answer loop's conversation-state prompt section.
204
+ const serverSnapshot = buildConversationSnapshot(store, threadId, { question });
205
+ if (serverSnapshot && question) {
206
+ // Semantic recall over OLDER turns (the recent window is already verbatim).
207
+ serverSnapshot.recalledTurns = await recallRelevantTurns(store, threadId, question, {
208
+ limit: 3,
209
+ excludeTurnIds: serverSnapshot.recentTurns.map((turn) => turn.id),
210
+ });
211
+ }
212
+ return {
213
+ ...(clientContext ?? {}),
214
+ conversationStateVersion: 1,
215
+ threadId,
216
+ ...(serverSnapshot ? { serverSnapshot } : {}),
217
+ ...(thread?.rollingSummary ? { conversationSummary: thread.rollingSummary } : {}),
218
+ ...(turns.length > 0 ? { turns, activeTurnId: turns[turns.length - 1].id } : {}),
219
+ };
220
+ }
221
+ /** Best-effort: persist a completed run as a conversation turn (never throws). */
222
+ function recordConversationTurn(store, threadId, run) {
223
+ if (!store || !threadId)
224
+ return;
225
+ try {
226
+ if (!store.getThread(threadId))
227
+ return;
228
+ const turn = store.appendTurn(threadId, conversationTurnInputFromRun(run));
229
+ // Fold the turn into the thread's working state + rolling summary (incremental).
230
+ advanceThreadState(store, threadId, turn);
231
+ }
232
+ catch {
233
+ // Conversation persistence is additive; a failed write must not fail the run.
234
+ }
235
+ }
236
+ function conversationTurnInputFromRun(run) {
237
+ const artifact = run.artifacts.find((candidate) => candidate.kind === 'answer')
238
+ ?? run.artifacts.find((candidate) => candidate.kind === 'research_run')
239
+ ?? run.artifacts[0];
240
+ const payload = agentRunRecord(artifact?.payload);
241
+ const result = agentRunRecord(payload?.result);
242
+ const columns = conversationResultColumns(result?.columns);
243
+ const rows = Array.isArray(result?.rows)
244
+ ? result.rows.filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row))).slice(0, 8)
245
+ : [];
246
+ const rowsSample = rows.map((row) => columns.map((column) => row[column]));
247
+ const contextPack = agentRunRecord(payload?.contextPack);
248
+ const questionPlan = agentRunRecord(contextPack?.questionPlan);
249
+ const requestedShape = agentRunRecord(questionPlan?.requestedShape);
250
+ const rowCountRaw = result?.rowCount;
251
+ const measureColumns = conversationMeasureColumns(columns, requestedShape, rows);
252
+ return {
253
+ question: run.question,
254
+ answerSummary: run.answer ?? run.summary,
255
+ route: run.route,
256
+ trustLabel: agentRunString(payload?.trustLabel) ?? run.trustState,
257
+ certification: agentRunString(payload?.certification),
258
+ sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
259
+ ?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
260
+ contextPackId: agentRunString(payload?.contextPackId),
261
+ sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
262
+ dqlArtifact: agentRunRecord(payload?.dqlArtifact),
263
+ cascade: agentRunRecord(payload?.cascade),
264
+ result: columns.length > 0 || rows.length > 0
265
+ ? {
266
+ columns,
267
+ rowsSample,
268
+ dimensionValues: conversationDimensionValues(columns, rows),
269
+ measureColumns,
270
+ rowCount: typeof rowCountRaw === 'number' ? rowCountRaw : rows.length || undefined,
271
+ }
272
+ : undefined,
273
+ contract: requestedShape,
274
+ };
275
+ }
276
+ function conversationResultColumns(value) {
277
+ if (!Array.isArray(value))
278
+ return [];
279
+ return value
280
+ .map((column) => typeof column === 'string'
281
+ ? column
282
+ : agentRunString(column?.name) ?? '')
283
+ .filter(Boolean)
284
+ .slice(0, 24);
285
+ }
286
+ function conversationMeasureColumns(columns, requestedShape, rows) {
287
+ const requested = conversationStringArray(requestedShape?.measures) ?? [];
288
+ const numericColumns = columns.filter((column) => rows.some((row) => typeof row[column] === 'number' && Number.isFinite(row[column])));
289
+ const metricNamedColumns = columns.filter((column) => /\b(revenue|sales|amount|total|count|average|avg|sum|spend|cost|margin|profit|value|points?|score|quantity|units?|rate|volume)\b/i.test(column.replace(/_/g, ' ')));
290
+ const unique = Array.from(new Set([...requested, ...numericColumns, ...metricNamedColumns]
291
+ .map((value) => value.trim())
292
+ .filter(Boolean)));
293
+ return unique.length > 0 ? unique.slice(0, 24) : undefined;
294
+ }
295
+ /** Distinct string values per low-cardinality column — the deictic-resolution fuel. */
296
+ function conversationDimensionValues(columns, rows) {
297
+ if (columns.length === 0 || rows.length === 0)
298
+ return undefined;
299
+ const out = {};
300
+ for (const column of columns.slice(0, 8)) {
301
+ const values = Array.from(new Set(rows.map((row) => row[column]).filter((value) => typeof value === 'string' && value.trim().length > 0))).slice(0, 24);
302
+ if (values.length > 0 && values.length <= 24)
303
+ out[column] = values;
304
+ }
305
+ return Object.keys(out).length > 0 ? out : undefined;
306
+ }
307
+ function conversationStringArray(value) {
308
+ if (!Array.isArray(value))
309
+ return undefined;
310
+ const values = value.filter((item) => typeof item === 'string' && item.trim().length > 0);
311
+ return values.length > 0 ? values : undefined;
312
+ }
313
+ function compactConversationRecord(record) {
314
+ const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
315
+ return entries.length > 0 ? Object.fromEntries(entries) : undefined;
316
+ }
134
317
  export async function startLocalServer(opts) {
135
318
  const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
136
319
  const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
@@ -366,7 +549,7 @@ export async function startLocalServer(opts) {
366
549
  return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
367
550
  },
368
551
  });
369
- async function runGovernedAgentAnswerForRun(request, repair) {
552
+ async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer') {
370
553
  const governed = resolveGovernedAnswerRunner(projectRoot);
371
554
  const resolvedProvider = governed?.provider ?? null;
372
555
  const runner = governed?.runner ?? null;
@@ -376,6 +559,10 @@ export async function startLocalServer(opts) {
376
559
  let governedAnswer;
377
560
  let providerError;
378
561
  const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
562
+ const reasoningEffort = request.reasoningEffort
563
+ ? resolveRequestedRunReasoningEffort(projectRoot, resolvedProvider, request.reasoningEffort, isRepair)
564
+ : resolveRunReasoningEffort(projectRoot, resolvedProvider, route, isRepair);
565
+ const analysisDepth = request.analysisDepth ?? (route === 'research' ? 'deep' : undefined);
379
566
  const contextEnvelope = {
380
567
  mode: 'agent_run',
381
568
  selectedObject: request.selectedObject,
@@ -383,23 +570,37 @@ export async function startLocalServer(opts) {
383
570
  instruction: [
384
571
  'Route through the governed DQL answer loop.',
385
572
  'Prefer certified DQL blocks when they exactly cover the question.',
386
- 'Generated SQL remains review-required and must use the bounded preview executor.',
573
+ 'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
387
574
  'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
388
575
  ...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
389
576
  ].join(' '),
390
577
  };
391
578
  const controller = new AbortController();
579
+ // Best-effort active warehouse dialect so Lane-2 semantic compiles emit
580
+ // dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
581
+ // connection is configured — the compiler then uses its default dialect.
582
+ let semanticDriver;
583
+ try {
584
+ semanticDriver = requireActiveConnection().driver;
585
+ }
586
+ catch {
587
+ semanticDriver = undefined;
588
+ }
392
589
  await runner.run({
393
590
  provider: resolvedProvider,
394
591
  messages: [
395
592
  ...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
396
593
  { role: 'user', content: isRepair ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}` : request.question },
397
594
  ],
595
+ conversationContext: request.conversationContext,
398
596
  upstream: {
399
597
  cellId: `agent-run:${request.selectedObject?.kind ?? 'workspace'}:${request.selectedObject?.id ?? request.runId ?? 'auto'}`,
400
598
  sql: JSON.stringify(contextEnvelope, null, 2),
401
599
  },
600
+ reasoningEffort,
601
+ ...(analysisDepth ? { analysisDepth } : {}),
402
602
  projectRoot,
603
+ ...(semanticDriver ? { semanticDriver } : {}),
403
604
  executeCertifiedBlock: executeCertifiedBlockForAgent,
404
605
  executeGeneratedSql: executeGeneratedSqlForAgent,
405
606
  getSchemaContext: getSchemaContextForAgent,
@@ -416,10 +617,26 @@ export async function startLocalServer(opts) {
416
617
  }
417
618
  return governedAnswer;
418
619
  }
419
- const answerRunExecutor = async ({ request, routeDecision, attempt, repairHint, emitAnswerDelta }) => {
620
+ function resolvedRunRouteFromAnswer(governedAnswer) {
621
+ return routeForCascadeAnswerTier(governedAnswer.route?.tier);
622
+ }
623
+ function groundingGapRepairHint(governedAnswer) {
624
+ const details = governedAnswer.refusalDetails;
625
+ if (!details)
626
+ return governedAnswer.text;
627
+ const offending = details.offending;
628
+ const tokens = [
629
+ details.code ? `code=${details.code}` : undefined,
630
+ offending?.relation ? `relation=${offending.relation}` : undefined,
631
+ offending?.column ? `column=${offending.column}` : undefined,
632
+ details.message ? `message=${details.message}` : undefined,
633
+ ].filter((token) => Boolean(token));
634
+ return tokens.length > 0 ? tokens.join('; ') : governedAnswer.text;
635
+ }
636
+ const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emitAnswerDelta }) => {
420
637
  let governedAnswer;
421
638
  try {
422
- governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint });
639
+ governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route);
423
640
  // Surface the approved Hint-Graph corrections that shaped this answer so the
424
641
  // UI can show an "applied learnings" chip (memoryContext is already on the answer).
425
642
  if (!governedAnswer.appliedHints) {
@@ -443,14 +660,20 @@ export async function startLocalServer(opts) {
443
660
  ],
444
661
  };
445
662
  }
663
+ const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
446
664
  const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
447
- const needsClarification = governedAnswer.kind === 'no_answer';
665
+ const isGroundingGap = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'grounding_gap';
666
+ const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
667
+ const groundingRepairHint = isGroundingGap ? groundingGapRepairHint(governedAnswer) : undefined;
668
+ // A provider outage is a retryable infrastructure failure, not a question the
669
+ // user needs to clarify — surface it as blocked so the UI offers a retry.
670
+ const needsClarification = governedAnswer.kind === 'no_answer' && !isGroundingGap && !isProviderError;
448
671
  const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
449
- // Synthesis: for a genuinely generated (non-certified) answer, compose an
450
- // adaptive, well-formatted reply over the executed rows streamed — instead of
451
- // the loop's raw draft. Certified answers keep their fast path (no extra call).
672
+ const runnableSql = governedAnswer.kind === 'no_answer' ? undefined : sql;
673
+ // Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
674
+ // that already produced DQL-first final prose keep the fast path.
452
675
  let synthesizedAnswer;
453
- if (!needsClarification && !isCertified) {
676
+ if (shouldSynthesizeAgentRunAnswer(governedAnswer)) {
454
677
  try {
455
678
  const provider = await createBlockStudioAssistProvider(projectRoot);
456
679
  if (provider) {
@@ -477,32 +700,55 @@ export async function startLocalServer(opts) {
477
700
  synthesizedAnswer = undefined;
478
701
  }
479
702
  }
480
- const status = needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
481
- const trustState = needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
482
- const stopReason = needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
703
+ const status = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
704
+ const trustState = isProviderError ? 'blocked' : needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
705
+ const stopReason = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
483
706
  const nextActions = needsClarification
484
707
  ? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
485
708
  : [
486
- ...(sql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
709
+ { id: 'create-block', label: governedAnswer.dqlArtifact ? 'Review DQL draft' : 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
487
710
  { id: 'research-gap', label: 'Research deeper', route: 'research' },
488
- { id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
711
+ ...(runnableSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
489
712
  ];
490
713
  return {
714
+ resolvedRoute,
715
+ answerTier: governedAnswer.route?.tier,
491
716
  summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
492
717
  answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
493
718
  status,
494
719
  trustState,
495
720
  stopReason,
496
- artifacts: needsClarification
497
- ? []
721
+ artifacts: governedAnswer.kind === 'no_answer'
722
+ // A refusal still keeps the DQL draft the answer loop produced (when any),
723
+ // so the "Review DQL draft" next-action isn't a dead link and the user can
724
+ // see the SQL that was about to run. Provider outages carry no draft.
725
+ ? (governedAnswer.dqlArtifact && !isProviderError
726
+ ? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
727
+ : [])
498
728
  : [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
499
729
  evaluations: [
500
- agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'),
730
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
731
+ plannedRoute: route,
732
+ resolvedRoute,
733
+ aiRoute: governedAnswer.route,
734
+ }),
501
735
  agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
502
736
  ? 'The answer came from certified DQL context.'
503
737
  : needsClarification
504
738
  ? 'The answer loop needs more context before producing a governed answer.'
505
739
  : 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
740
+ ...(isGroundingGap ? [
741
+ {
742
+ ...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
743
+ refusalCode: governedAnswer.refusalCode,
744
+ refusalDetails: governedAnswer.refusalDetails,
745
+ validationWarnings: governedAnswer.validationWarnings,
746
+ route: governedAnswer.route,
747
+ }),
748
+ suggestedRepair: groundingRepairHint,
749
+ repairAction: { kind: 'retry', hint: groundingRepairHint },
750
+ },
751
+ ] : []),
506
752
  ...(governedAnswer.executionError ? [
507
753
  agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
508
754
  ] : []),
@@ -525,8 +771,10 @@ export async function startLocalServer(opts) {
525
771
  const provider = await createBlockStudioAssistProvider(projectRoot);
526
772
  if (provider) {
527
773
  const system = buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext, request.audience ?? 'analyst');
774
+ const conversationMemory = renderConversationMemoryForPrompt(request.conversationContext);
528
775
  const messages = [
529
776
  { role: 'system', content: system },
777
+ ...(conversationMemory ? [{ role: 'system', content: conversationMemory }] : []),
530
778
  ...(request.history ?? []).slice(-6).map((message) => ({ role: message.role, content: message.text })),
531
779
  { role: 'user', content: request.question },
532
780
  ];
@@ -541,7 +789,8 @@ export async function startLocalServer(opts) {
541
789
  text = undefined;
542
790
  }
543
791
  if (!text) {
544
- text = buildConversationalFallback(kind, isGeneralKnowledge, request.question, suggestions);
792
+ text = buildConversationContextRecap(request.conversationContext)
793
+ ?? buildConversationalFallback(kind, isGeneralKnowledge, request.question, suggestions);
545
794
  emitAnswerDelta?.(text);
546
795
  }
547
796
  return {
@@ -560,7 +809,8 @@ export async function startLocalServer(opts) {
560
809
  conversation: conversationRunExecutor,
561
810
  certified_answer: answerRunExecutor,
562
811
  generated_answer: answerRunExecutor,
563
- research: async ({ runId, request, routeDecision, emit }) => {
812
+ research: async (researchContext) => {
813
+ const { runId, request, routeDecision, emit } = researchContext;
564
814
  const metrics = loadSemanticMetrics(projectRoot);
565
815
  let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
566
816
  const usedCertifiedOnly = blocks.length > 0;
@@ -581,6 +831,13 @@ export async function startLocalServer(opts) {
581
831
  // When the user explicitly picked research, investigate — don't collapse to one step.
582
832
  forceInvestigate: request.requestedMode === 'research',
583
833
  });
834
+ // Direct governed answer ("the metric answers this directly"): don't wrap it in
835
+ // a review-required research dossier — run the query through the answer loop and
836
+ // return the executed result table + DQL artifact. A dossier is only for genuine
837
+ // multi-step investigation (plan.done === false) or explicit research mode.
838
+ if (plan.done && !plan.followUp && request.requestedMode !== 'research') {
839
+ return answerRunExecutor(researchContext);
840
+ }
584
841
  const needsClarification = Boolean(plan.followUp);
585
842
  const notebookPath = agentRunNotebookPath(request, runId);
586
843
  const researchIntent = agentRunResearchIntent(request);
@@ -646,9 +903,15 @@ export async function startLocalServer(opts) {
646
903
  researchWorkspaceError = formatNotebookResearchStorageError(error);
647
904
  }
648
905
  }
649
- const researchResultData = coerceNarrateResultData(researchRun?.resultPreview);
650
- // Executed cleanly against real data (rows present, no workspace/exec error) → grounded, not review-required.
651
- const researchExecutedCleanly = Boolean(researchResultData) && !researchWorkspaceError && researchRun?.status !== 'error';
906
+ const researchResultPreview = researchRun?.resultPreview;
907
+ const researchResultData = coerceNarrateResultData(researchResultPreview);
908
+ const researchResultRecord = agentRunRecord(researchResultPreview);
909
+ // A query that ran and matched 0 rows STILL executed — treat it as a clean,
910
+ // grounded execution (not "no result"), so an empty answer is surfaced as
911
+ // "0 rows matched" rather than silently downgraded to review-required.
912
+ const researchDidExecute = Boolean(researchResultData) || Boolean(researchResultRecord && Array.isArray(researchResultRecord.rows));
913
+ const researchZeroRows = !researchResultData && researchDidExecute;
914
+ const researchExecutedCleanly = researchDidExecute && !researchWorkspaceError && researchRun?.status !== 'error';
652
915
  const narration = !needsClarification && researchResultData
653
916
  ? await narrateForAgentRun({
654
917
  question: request.question,
@@ -661,18 +924,22 @@ export async function startLocalServer(opts) {
661
924
  const summary = needsClarification
662
925
  ? 'Needs clarification before running deeper research.'
663
926
  : narration?.summary
664
- ?? (researchRun?.status === 'ready'
665
- ? 'Saved a grounded research dossier with context evidence and next review actions.'
666
- : researchRun?.status === 'error'
667
- ? 'Saved a research dossier, but the preview needs review before promotion.'
668
- : researchWorkspaceError
669
- ? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
670
- : plan.done
671
- ? 'Prepared a direct grounded-answer plan.'
672
- : 'Prepared a grounded research plan over real DQL assets.');
927
+ ?? (researchZeroRows
928
+ ? 'The query executed cleanly against real data and matched 0 rows.'
929
+ : researchRun?.status === 'ready'
930
+ ? 'Saved a grounded research dossier with context evidence and next review actions.'
931
+ : researchRun?.status === 'error'
932
+ ? 'Saved a research dossier, but the preview needs review before promotion.'
933
+ : researchWorkspaceError
934
+ ? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
935
+ : plan.done
936
+ ? 'Prepared a direct grounded-answer plan.'
937
+ : 'Prepared a grounded research plan over real DQL assets.');
673
938
  return {
674
939
  summary,
675
- answer: plan.followUp?.question ?? narration?.summary ?? researchRun?.summary,
940
+ answer: plan.followUp?.question ?? narration?.summary
941
+ ?? (researchZeroRows ? 'The query executed cleanly and matched 0 rows.' : undefined)
942
+ ?? researchRun?.summary,
676
943
  status: needsClarification ? 'needs_clarification' : 'needs_review',
677
944
  trustState: needsClarification ? 'not_applicable' : (researchExecutedCleanly ? 'grounded' : 'review_required'),
678
945
  stopReason: needsClarification ? 'needs_clarification' : 'human_review_required',
@@ -702,15 +969,17 @@ export async function startLocalServer(opts) {
702
969
  ? 'Research workspace storage is unavailable; the plan remains available in this agent run.'
703
970
  : 'No durable research record was created because the run needs clarification first.', { notebookPath, researchRunId: researchRun?.id, error: researchWorkspaceError }),
704
971
  agentRunEvaluation('result-executed', 'Executed against data', researchExecutedCleanly, researchExecutedCleanly ? 'info' : 'warning', researchExecutedCleanly
705
- ? 'The query executed cleanly against real data and returned rows.'
706
- : 'No executed result rows were available; the output stays exploratory pending review.', { rowCount: researchResultData && typeof researchResultData === 'object' && Array.isArray(researchResultData.rows) ? researchResultData.rows.length : 0 }),
972
+ ? (researchZeroRows
973
+ ? 'The query executed cleanly against real data and matched 0 rows.'
974
+ : 'The query executed cleanly against real data and returned rows.')
975
+ : 'No executed result was available; the output stays exploratory pending review.', { rowCount: Array.isArray(researchResultRecord?.rows) ? researchResultRecord.rows.length : 0 }),
707
976
  ],
708
977
  nextActions: needsClarification
709
978
  ? [{ id: 'answer-follow-up', label: 'Answer follow-up', route: 'research' }]
710
979
  : [
711
980
  ...(researchRun?.id ? [{ id: 'open-research', label: 'Open research dossier', artifactKind: 'research_run' }] : []),
712
- ...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
713
- { id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
981
+ { id: 'create-block', label: 'Review DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
982
+ ...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL preview', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
714
983
  ],
715
984
  };
716
985
  },
@@ -725,12 +994,67 @@ export async function startLocalServer(opts) {
725
994
  agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', 'Generated SQL must be reviewed before it becomes certified analytics.'),
726
995
  ],
727
996
  nextActions: [
728
- { id: 'insert-sql', label: 'Insert SQL cell', artifactKind: 'sql_cell' },
729
- { id: 'create-block', label: 'Promote to DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
997
+ { id: 'insert-sql', label: 'Insert SQL preview', artifactKind: 'sql_cell' },
998
+ { id: 'create-block', label: 'Review as DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
730
999
  ],
731
1000
  };
732
1001
  },
733
1002
  dql_block_draft: async ({ request, routeDecision, attempt, repairHint }) => {
1003
+ const contextRecord = agentRunRecord(request.conversationContext);
1004
+ const carriedDqlArtifact = normalizeDqlArtifactReference(contextRecord?.dqlArtifact)
1005
+ ?? normalizeDqlArtifactReference(request.workspaceContext?.dqlArtifact);
1006
+ if (carriedDqlArtifact && (attempt ?? 0) === 0) {
1007
+ const sourceQuestion = agentRunString(contextRecord?.sourceQuestion) ?? request.question;
1008
+ const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
1009
+ question: sourceQuestion,
1010
+ dqlArtifact: carriedDqlArtifact,
1011
+ inputMode: 'upload',
1012
+ inputPath: carriedDqlArtifact.sourcePath,
1013
+ domain: agentRunWorkspaceValue(request, 'domain'),
1014
+ owner: agentRunWorkspaceValue(request, 'owner'),
1015
+ tags: ['agent-run', 'review-required'],
1016
+ contextPackId: agentRunString(contextRecord?.contextPackId),
1017
+ routeIntent: agentRunString(contextRecord?.route),
1018
+ sourceBlock: carriedDqlArtifact.sourcePath,
1019
+ }, semanticLayer);
1020
+ const candidate = session.candidates[0];
1021
+ const validation = candidate?.validation && typeof candidate.validation === 'object'
1022
+ ? candidate.validation
1023
+ : undefined;
1024
+ const ready = validation?.valid !== false;
1025
+ const payload = {
1026
+ target: 'block',
1027
+ name: candidate?.name ?? carriedDqlArtifact.name ?? 'DQL block draft',
1028
+ path: candidate?.draftSave?.path ?? candidate?.savedPath,
1029
+ importId: session.id,
1030
+ candidateId: candidate?.id,
1031
+ source: candidate?.dqlSource,
1032
+ dqlSource: candidate?.dqlSource,
1033
+ dqlArtifact: carriedDqlArtifact,
1034
+ certifierVerdict: {
1035
+ ready,
1036
+ diagnostics: validation?.diagnostics ?? [],
1037
+ },
1038
+ generation: session.generation,
1039
+ };
1040
+ return {
1041
+ summary: ready
1042
+ ? 'Saved the generated DQL artifact as a review-required block draft.'
1043
+ : 'Saved the generated DQL artifact as a block draft with review blockers or warnings.',
1044
+ artifacts: [agentRunArtifact('dql_block_draft', payload.name, payload, payload.path)],
1045
+ evaluations: [
1046
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to DQL block draft generation.'),
1047
+ agentRunEvaluation('dql-artifact-reuse', 'Used generated DQL artifact', true, 'info', 'Persisted the existing DQL artifact without regenerating it from SQL.', { importId: session.id, path: payload.path }),
1048
+ agentRunEvaluation('certification-boundary', 'Certification boundary', ready, 'warning', ready
1049
+ ? 'The draft has no automatic validation blockers, but certification still requires human review.'
1050
+ : 'The draft has validation blockers that must be resolved before certification review.', payload),
1051
+ ],
1052
+ nextActions: [
1053
+ { id: 'open-review', label: 'Open review checklist', artifactKind: 'dql_block_draft' },
1054
+ { id: 'build-app', label: 'Build app from block', route: 'app_build', artifactKind: 'app_draft' },
1055
+ ],
1056
+ };
1057
+ }
734
1058
  const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
735
1059
  const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
736
1060
  return {
@@ -898,9 +1222,27 @@ export async function startLocalServer(opts) {
898
1222
  getCatalogContext: () => buildAgentRunCatalogContext(),
899
1223
  });
900
1224
  const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
1225
+ // Server-side conversation threads: persisted multi-turn state (survives refresh).
1226
+ // Lazy so a failed native-sqlite load never blocks the runtime; conversation
1227
+ // persistence degrades to the per-request context instead.
1228
+ let conversationStoreInstance;
1229
+ const getConversationStore = () => {
1230
+ if (conversationStoreInstance !== undefined)
1231
+ return conversationStoreInstance;
1232
+ try {
1233
+ conversationStoreInstance = new ConversationStore(defaultConversationPath(projectRoot));
1234
+ }
1235
+ catch {
1236
+ conversationStoreInstance = null;
1237
+ }
1238
+ return conversationStoreInstance;
1239
+ };
901
1240
  const agentRunEngine = new AgentRunEngine({
902
1241
  store: agentRunStore,
903
- executors: agentRunExecutors,
1242
+ executors: {
1243
+ ...agentRunExecutors,
1244
+ ...(opts.agentRunExecutors ?? {}),
1245
+ },
904
1246
  gates: defaultAgentRunGates,
905
1247
  planner: agentRunPlanner,
906
1248
  router: agentRunRouter,
@@ -1059,25 +1401,42 @@ export async function startLocalServer(opts) {
1059
1401
  };
1060
1402
  };
1061
1403
  const getSchemaContextForAgent = async (question) => {
1404
+ const scanRuntimeSchema = async () => {
1405
+ if (!connection)
1406
+ return { ranked: [], snapshot: [] };
1407
+ const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
1408
+ FROM information_schema.columns
1409
+ WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
1410
+ ORDER BY table_schema, table_name, ordinal_position
1411
+ LIMIT 2000`, [], runtimeVariables({}), connection);
1412
+ return {
1413
+ ranked: buildAgentSchemaContext(question, result.rows),
1414
+ snapshot: buildAgentSchemaContext(question, result.rows, { includeUnscored: true, limit: 500 }),
1415
+ };
1416
+ };
1062
1417
  const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question).catch(() => []);
1063
1418
  if (catalogContext.length > 0) {
1064
1419
  if (!connection)
1065
1420
  return catalogContext;
1066
- const enriched = await enrichAgentSchemaContextWithValueMatches(question, catalogContext, executor, connection);
1067
- recordAgentRuntimeSchemaSnapshot(projectRoot, enriched, 'catalog enriched runtime schema');
1421
+ const runtimeScan = shouldAugmentAgentRuntimeSchema(question)
1422
+ ? await scanRuntimeSchema().catch(() => undefined)
1423
+ : undefined;
1424
+ const runtimeContext = runtimeScan?.ranked ?? [];
1425
+ const merged = mergeAgentSchemaContexts(catalogContext, runtimeContext);
1426
+ const enriched = await enrichAgentSchemaContextWithValueMatches(question, merged, executor, connection);
1427
+ recordAgentRuntimeSchemaSnapshot(projectRoot, !runtimeScan?.snapshot.length
1428
+ ? enriched
1429
+ : mergeAgentSchemaSampleValues(runtimeScan.snapshot, enriched), runtimeContext.length > 0
1430
+ ? 'full information_schema runtime scan for composite Ask AI question'
1431
+ : 'catalog enriched runtime schema');
1068
1432
  return enriched;
1069
1433
  }
1070
1434
  if (!connection)
1071
1435
  return [];
1072
1436
  try {
1073
- const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
1074
- FROM information_schema.columns
1075
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
1076
- ORDER BY table_schema, table_name, ordinal_position
1077
- LIMIT 2000`, [], runtimeVariables({}), connection);
1078
- const schemaContext = buildAgentSchemaContext(question, result.rows);
1079
- const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection);
1080
- recordAgentRuntimeSchemaSnapshot(projectRoot, enriched, 'information_schema runtime scan');
1437
+ const schemaContext = await scanRuntimeSchema();
1438
+ const enriched = await enrichAgentSchemaContextWithValueMatches(question, schemaContext.ranked, executor, connection);
1439
+ recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext.snapshot, 'full information_schema runtime scan');
1081
1440
  return enriched;
1082
1441
  }
1083
1442
  catch {
@@ -1094,6 +1453,7 @@ export async function startLocalServer(opts) {
1094
1453
  let governedAnswer;
1095
1454
  let providerError;
1096
1455
  const memoOnly = input.mode === 'memo_only';
1456
+ const reasoningEffort = resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false);
1097
1457
  const contextEnvelope = {
1098
1458
  mode: 'app_research',
1099
1459
  generationMode: input.mode ?? 'sql_and_memo',
@@ -1141,6 +1501,8 @@ export async function startLocalServer(opts) {
1141
1501
  cellId: `app-research:${input.appId}:${input.dashboardId ?? 'app'}`,
1142
1502
  sql: JSON.stringify(contextEnvelope, null, 2),
1143
1503
  },
1504
+ reasoningEffort,
1505
+ analysisDepth: 'deep',
1144
1506
  projectRoot,
1145
1507
  executeCertifiedBlock: executeCertifiedBlockForAgent,
1146
1508
  executeGeneratedSql: executeGeneratedSqlForAgent,
@@ -1217,6 +1579,21 @@ export async function startLocalServer(opts) {
1217
1579
  intents: 0,
1218
1580
  notebooks: 0,
1219
1581
  },
1582
+ reviewMetrics: {
1583
+ totalReviewCount: 0,
1584
+ openReviewCount: 0,
1585
+ terminalReviewCount: 0,
1586
+ draftCreatedCount: 0,
1587
+ certifiedCount: 0,
1588
+ completedCount: 0,
1589
+ rejectedCount: 0,
1590
+ draftCreationRate: null,
1591
+ certifyConversionRate: null,
1592
+ medianOpenReviewAgeMs: null,
1593
+ medianTimeToDraftMs: null,
1594
+ medianTimeToCertificationMs: null,
1595
+ medianTimeToTerminalMs: null,
1596
+ },
1220
1597
  limit: input.limit,
1221
1598
  offset: input.offset ?? 0,
1222
1599
  });
@@ -1259,6 +1636,7 @@ export async function startLocalServer(opts) {
1259
1636
  const context = input.context === undefined ? run.context : input.context;
1260
1637
  let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
1261
1638
  let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
1639
+ let dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? run.dqlArtifact;
1262
1640
  const startedAt = new Date().toISOString();
1263
1641
  storage.updateRun(run.id, {
1264
1642
  domain,
@@ -1269,6 +1647,7 @@ export async function startLocalServer(opts) {
1269
1647
  context,
1270
1648
  generatedSql,
1271
1649
  reviewedSql,
1650
+ dqlArtifact,
1272
1651
  status: 'running',
1273
1652
  reviewStatus: 'needs_review',
1274
1653
  error: '',
@@ -1285,8 +1664,8 @@ export async function startLocalServer(opts) {
1285
1664
  intent,
1286
1665
  researchPattern: notebookResearchIntentPattern(intent),
1287
1666
  },
1288
- strictness: 'balanced',
1289
- limit: 100,
1667
+ strictness: 'exploratory',
1668
+ limit: 160,
1290
1669
  }).catch((error) => {
1291
1670
  const message = error instanceof Error ? error.message : String(error);
1292
1671
  return {
@@ -1327,6 +1706,8 @@ export async function startLocalServer(opts) {
1327
1706
  context,
1328
1707
  }, null, 2),
1329
1708
  },
1709
+ reasoningEffort: resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false),
1710
+ analysisDepth: 'deep',
1330
1711
  projectRoot,
1331
1712
  executeCertifiedBlock: executeCertifiedBlockForAgent,
1332
1713
  executeGeneratedSql: executeGeneratedSqlForAgent,
@@ -1346,8 +1727,9 @@ export async function startLocalServer(opts) {
1346
1727
  generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
1347
1728
  }
1348
1729
  else {
1730
+ dqlArtifact = normalizeDqlArtifactReference(governedAnswer.dqlArtifact) ?? dqlArtifact;
1349
1731
  generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
1350
- if (!generatedSql) {
1732
+ if (!generatedSql && !dqlArtifact) {
1351
1733
  generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
1352
1734
  }
1353
1735
  }
@@ -1409,7 +1791,7 @@ export async function startLocalServer(opts) {
1409
1791
  const evidence = {
1410
1792
  trustStatus: {
1411
1793
  label: governedAnswer?.trustLabel
1412
- ?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
1794
+ ?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : dqlArtifact ? 'AI-generated DQL artifact' : 'Metadata-grounded research plan'),
1413
1795
  reviewRequired: true,
1414
1796
  },
1415
1797
  contextPackId: contextPack.id,
@@ -1448,9 +1830,11 @@ export async function startLocalServer(opts) {
1448
1830
  ?? notebookResearchSummary(question, resultPreview, previewError);
1449
1831
  const recommendation = previewError
1450
1832
  ? 'Review the SQL, selected metadata, and connection context before rerunning.'
1451
- : sqlForPreview
1452
- ? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
1453
- : 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
1833
+ : dqlArtifact && !reviewedSql
1834
+ ? 'Review the DQL artifact, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
1835
+ : sqlForPreview
1836
+ ? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
1837
+ : 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
1454
1838
  const researchPlan = buildNotebookResearchPlan({
1455
1839
  run,
1456
1840
  evidence,
@@ -1458,6 +1842,7 @@ export async function startLocalServer(opts) {
1458
1842
  previewError,
1459
1843
  generatedSql,
1460
1844
  reviewedSql,
1845
+ dqlArtifact,
1461
1846
  routeDecision,
1462
1847
  });
1463
1848
  return storage.updateRun(run.id, {
@@ -1475,6 +1860,7 @@ export async function startLocalServer(opts) {
1475
1860
  researchPlan,
1476
1861
  generatedSql,
1477
1862
  reviewedSql,
1863
+ dqlArtifact,
1478
1864
  display: display && display.ok ? display.display : undefined,
1479
1865
  contextPackId: contextPack.id,
1480
1866
  routeDecision,
@@ -1501,10 +1887,48 @@ export async function startLocalServer(opts) {
1501
1887
  }
1502
1888
  };
1503
1889
  const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
1504
- const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
1505
- if (!sql)
1506
- throw new Error('Notebook research needs generated or reviewed SQL before DQL draft promotion.');
1890
+ const reviewedSql = notebookResearchString(run.reviewedSql);
1891
+ const dqlArtifact = normalizeDqlArtifactReference(run.dqlArtifact);
1892
+ const sql = reviewedSql ?? notebookResearchString(run.generatedSql);
1507
1893
  const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
1894
+ if (!reviewedSql && dqlArtifact) {
1895
+ const session = await createDqlArtifactGenerationSessionForProject(projectRoot, {
1896
+ question: run.question,
1897
+ dqlArtifact,
1898
+ inputMode: 'upload',
1899
+ inputPath: dqlArtifact.sourcePath ?? sourcePath,
1900
+ domain: notebookResearchString(input.domain) ?? run.domain,
1901
+ owner: notebookResearchString(input.owner) ?? run.owner,
1902
+ tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
1903
+ generatedSql: notebookResearchString(run.generatedSql),
1904
+ contextPackId: run.contextPackId,
1905
+ routeIntent: run.intent,
1906
+ sourceBlock: dqlArtifact.sourcePath,
1907
+ }, semanticLayer);
1908
+ const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
1909
+ ?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
1910
+ const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
1911
+ const updated = storage.markPromoted(run.id, {
1912
+ draftBlockPath: draftPath,
1913
+ dqlImportId: session.id,
1914
+ dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
1915
+ dqlPromotion: promotion,
1916
+ }) ?? run;
1917
+ const planned = storage.updateRun(updated.id, {
1918
+ researchPlan: buildNotebookResearchPlan({
1919
+ run: updated,
1920
+ evidence: updated.evidence,
1921
+ resultPreview: updated.resultPreview,
1922
+ generatedSql: updated.generatedSql,
1923
+ reviewedSql: updated.reviewedSql,
1924
+ dqlArtifact: updated.dqlArtifact,
1925
+ routeDecision: updated.routeDecision,
1926
+ }),
1927
+ }) ?? updated;
1928
+ return { run: planned, session };
1929
+ }
1930
+ if (!sql)
1931
+ throw new Error('Notebook research needs reviewed SQL, generated SQL, or a DQL artifact before DQL draft promotion.');
1508
1932
  const session = await createDqlGenerationSessionForProject(projectRoot, {
1509
1933
  inputMode: 'upload',
1510
1934
  sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
@@ -1530,6 +1954,7 @@ export async function startLocalServer(opts) {
1530
1954
  resultPreview: updated.resultPreview,
1531
1955
  generatedSql: updated.generatedSql,
1532
1956
  reviewedSql: updated.reviewedSql,
1957
+ dqlArtifact: updated.dqlArtifact,
1533
1958
  routeDecision: updated.routeDecision,
1534
1959
  }),
1535
1960
  }) ?? updated;
@@ -2054,10 +2479,14 @@ export async function startLocalServer(opts) {
2054
2479
  res.end(serializeJSON({ error: parsed.error ?? 'Invalid agent run request.' }));
2055
2480
  return;
2056
2481
  }
2057
- // The UI does not send retrieval signals; probe the local catalog so the
2058
- // router can short-circuit obvious governed matches without an LLM call.
2059
- if (!parsed.request.signals && !parsed.request.requestedMode) {
2060
- parsed.request.signals = computeAgentRunSignals(projectRoot, parsed.request.question);
2482
+ // The answer-loop cascade now proves certified/semantic/generated tiers
2483
+ // after retrieval and execution. Avoid pre-routing ordinary Ask runs from
2484
+ // token-overlap signals; explicit callers may still provide signals.
2485
+ // Thread-scoped runs: the persisted thread supplies prior turns (server
2486
+ // state wins; the client-built context stays the no-threadId fallback).
2487
+ const conversationStore = parsed.request.threadId ? getConversationStore() : null;
2488
+ if (conversationStore && parsed.request.threadId && conversationStore.getThread(parsed.request.threadId)) {
2489
+ parsed.request.conversationContext = await conversationContextFromThread(conversationStore, parsed.request.threadId, parsed.request.conversationContext, parsed.request.question);
2061
2490
  }
2062
2491
  const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
2063
2492
  if (wantsStream) {
@@ -2072,11 +2501,13 @@ export async function startLocalServer(opts) {
2072
2501
  }, (delta) => {
2073
2502
  writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
2074
2503
  });
2504
+ recordConversationTurn(conversationStore, parsed.request.threadId, run);
2075
2505
  writeAgentRunSse(res, 'agent-run-complete', run);
2076
2506
  res.end();
2077
2507
  return;
2078
2508
  }
2079
2509
  const run = await agentRunEngine.run(parsed.request);
2510
+ recordConversationTurn(conversationStore, parsed.request.threadId, run);
2080
2511
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
2081
2512
  res.end(serializeJSON({ run }));
2082
2513
  }
@@ -2108,6 +2539,7 @@ export async function startLocalServer(opts) {
2108
2539
  const notebookPath = (record && agentRunString(record.notebookPath))
2109
2540
  ?? `notebooks/certification-requests/${Date.now()}.dqlnb`;
2110
2541
  const generatedSql = record ? agentRunString(record.generatedSql) : undefined;
2542
+ const dqlArtifact = record ? normalizeDqlArtifactReference(record.dqlArtifact) : undefined;
2111
2543
  try {
2112
2544
  const storage = openNotebookResearchStorage();
2113
2545
  try {
@@ -2119,9 +2551,11 @@ export async function startLocalServer(opts) {
2119
2551
  domain: record ? agentRunString(record.domain) : undefined,
2120
2552
  owner: record ? agentRunString(record.owner) : undefined,
2121
2553
  generatedSql,
2554
+ dqlArtifact,
2122
2555
  context: {
2123
2556
  surface: 'stakeholder_request_certification',
2124
2557
  requestedContext: agentRunRecord(record?.context) ?? null,
2558
+ dqlArtifact: dqlArtifact ?? null,
2125
2559
  },
2126
2560
  });
2127
2561
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -2159,6 +2593,23 @@ export async function startLocalServer(opts) {
2159
2593
  // DRAFT proposals (each with its stored Certifier verdict) so the notebook
2160
2594
  // "Get Started" surface can route them into human review. dryRun preview —
2161
2595
  // nothing is written or certified by this call.
2596
+ if (req.method === 'GET' && path === '/api/agent-runs/tier-distribution') {
2597
+ try {
2598
+ const store = getConversationStore();
2599
+ const limitParam = Number(url.searchParams.get('limit'));
2600
+ const limit = Number.isFinite(limitParam) && limitParam > 0 ? limitParam : undefined;
2601
+ const distribution = store
2602
+ ? store.tierDistribution({ ...(limit ? { limit } : {}) })
2603
+ : { total: 0, byRouteTier: {}, byTerminalLane: {} };
2604
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2605
+ res.end(JSON.stringify(distribution));
2606
+ }
2607
+ catch (error) {
2608
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2609
+ res.end(JSON.stringify({ error: error instanceof Error ? error.message : String(error) }));
2610
+ }
2611
+ return;
2612
+ }
2162
2613
  if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose') {
2163
2614
  try {
2164
2615
  let owner;
@@ -2189,6 +2640,69 @@ export async function startLocalServer(opts) {
2189
2640
  }
2190
2641
  return;
2191
2642
  }
2643
+ // Composting preview: mine certified block clusters for semantic metric
2644
+ // candidates. Read-only; returns draft YAML + PR body text for review.
2645
+ if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose/composting') {
2646
+ try {
2647
+ let owner;
2648
+ let limit;
2649
+ let minSupport;
2650
+ if (req.method === 'POST') {
2651
+ const body = await readJSON(req).catch(() => ({}));
2652
+ if (typeof body?.owner === 'string')
2653
+ owner = body.owner;
2654
+ if (typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0)
2655
+ limit = body.limit;
2656
+ if (typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1)
2657
+ minSupport = body.minSupport;
2658
+ }
2659
+ else {
2660
+ const ownerParam = url.searchParams.get('owner');
2661
+ if (ownerParam)
2662
+ owner = ownerParam;
2663
+ const limitParam = Number(url.searchParams.get('limit'));
2664
+ if (Number.isFinite(limitParam) && limitParam > 0)
2665
+ limit = limitParam;
2666
+ const minSupportParam = Number(url.searchParams.get('minSupport'));
2667
+ if (Number.isFinite(minSupportParam) && minSupportParam > 1)
2668
+ minSupport = minSupportParam;
2669
+ }
2670
+ const changeset = buildSemanticCompostingChangeset(projectRoot, { owner, limit, minSupport });
2671
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2672
+ res.end(serializeJSON(changeset));
2673
+ }
2674
+ catch (error) {
2675
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2676
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2677
+ }
2678
+ return;
2679
+ }
2680
+ // Composting generation: writes only approved metric candidate ids under
2681
+ // semantic-layer/metrics/_drafts plus a PR-body artifact. Nothing certified.
2682
+ if (req.method === 'POST' && path === '/api/propose/composting/generate') {
2683
+ try {
2684
+ const body = (await readJSON(req).catch(() => ({})));
2685
+ const ids = Array.isArray(body?.ids)
2686
+ ? body.ids.filter((id) => typeof id === 'string' && id.trim().length > 0)
2687
+ : [];
2688
+ if (ids.length === 0) {
2689
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2690
+ res.end(serializeJSON({ error: 'Provide non-empty { ids } to generate semantic composting drafts.' }));
2691
+ return;
2692
+ }
2693
+ const owner = typeof body?.owner === 'string' ? body.owner : undefined;
2694
+ const limit = typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0 ? body.limit : undefined;
2695
+ const minSupport = typeof body?.minSupport === 'number' && Number.isFinite(body.minSupport) && body.minSupport > 1 ? body.minSupport : undefined;
2696
+ const result = generateSemanticCompostingDrafts(projectRoot, ids, { owner, limit, minSupport });
2697
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2698
+ res.end(serializeJSON(result));
2699
+ }
2700
+ catch (error) {
2701
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2702
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2703
+ }
2704
+ return;
2705
+ }
2192
2706
  // Deterministic PLAN only (classify → plan). Writes NOTHING. Same data the
2193
2707
  // readiness endpoint embeds, exposed standalone for the approve gate.
2194
2708
  if (req.method === 'POST' && path === '/api/propose/plan') {
@@ -2612,6 +3126,7 @@ export async function startLocalServer(opts) {
2612
3126
  apiKey: typeof body.apiKey === 'string' ? body.apiKey : undefined,
2613
3127
  baseUrl: typeof body.baseUrl === 'string' ? body.baseUrl : undefined,
2614
3128
  model: typeof body.model === 'string' ? body.model : undefined,
3129
+ reasoningEffort: isReasoningEffortSetting(body.reasoningEffort) ? body.reasoningEffort : undefined,
2615
3130
  });
2616
3131
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2617
3132
  res.end(serializeJSON({ ok: true, providers }));
@@ -2622,6 +3137,60 @@ export async function startLocalServer(opts) {
2622
3137
  }
2623
3138
  return;
2624
3139
  }
3140
+ // ── Subscription OAuth (Claude Pro/Max, ChatGPT Plus/Pro) ──────────────
3141
+ {
3142
+ const authMatch = path.match(/^\/api\/auth\/(claude|codex)\/(start|status|signout)$/);
3143
+ if (authMatch) {
3144
+ const oauthProvider = authMatch[1];
3145
+ const action = authMatch[2];
3146
+ const info = oauthProvider === 'claude'
3147
+ ? { models: CLAUDE_OAUTH_MODELS, defaultModel: CLAUDE_OAUTH_DEFAULT_MODEL }
3148
+ : { models: CODEX_OAUTH_MODELS, defaultModel: CODEX_OAUTH_DEFAULT_MODEL };
3149
+ const manager = getOAuthManager(projectRoot, oauthProvider);
3150
+ const statusPayload = () => {
3151
+ const connected = oauthProvider === 'claude' ? claudeOAuthConnected(projectRoot) : codexOAuthConnected(projectRoot);
3152
+ return {
3153
+ provider: oauthProvider,
3154
+ connected,
3155
+ email: connected ? manager.getEmail() : null,
3156
+ models: info.models,
3157
+ defaultModel: info.defaultModel,
3158
+ pending: manager.isPending(),
3159
+ };
3160
+ };
3161
+ try {
3162
+ if (action === 'status' && req.method === 'GET') {
3163
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3164
+ res.end(serializeJSON(statusPayload()));
3165
+ return;
3166
+ }
3167
+ if (action === 'signout' && req.method === 'POST') {
3168
+ manager.signOut();
3169
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3170
+ res.end(serializeJSON(statusPayload()));
3171
+ return;
3172
+ }
3173
+ if (action === 'start' && req.method === 'POST') {
3174
+ // Begin a fresh flow: build the authorize URL and start the loopback
3175
+ // callback server. The client opens the URL; the server captures the
3176
+ // redirect and persists tokens. The pending state lives on the manager,
3177
+ // so a superseded flow settles cleanly. Errors surface on the next poll.
3178
+ const url = manager.startAuthorizationFlow();
3179
+ void manager.waitForCallback().catch(() => { });
3180
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3181
+ res.end(serializeJSON({ url, ...statusPayload() }));
3182
+ return;
3183
+ }
3184
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
3185
+ res.end(serializeJSON({ error: 'Method not allowed' }));
3186
+ }
3187
+ catch (error) {
3188
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3189
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3190
+ }
3191
+ return;
3192
+ }
3193
+ }
2625
3194
  if (req.method === 'POST' && path === '/api/settings/providers/test') {
2626
3195
  try {
2627
3196
  const body = await readJSON(req);
@@ -2662,6 +3231,117 @@ export async function startLocalServer(opts) {
2662
3231
  }
2663
3232
  return;
2664
3233
  }
3234
+ // ── Conversation threads (server-side session persistence) ────────────
3235
+ if (path === '/api/agent/threads' || path.startsWith('/api/agent/threads/')) {
3236
+ const store = getConversationStore();
3237
+ if (!store) {
3238
+ res.writeHead(503, { 'Content-Type': 'application/json; charset=utf-8' });
3239
+ res.end(serializeJSON({ error: 'Conversation store is unavailable.' }));
3240
+ return;
3241
+ }
3242
+ try {
3243
+ if (req.method === 'GET' && path === '/api/agent/threads') {
3244
+ const limit = Number(url.searchParams.get('limit') ?? '50');
3245
+ const includeArchived = url.searchParams.get('archived') === '1';
3246
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3247
+ res.end(serializeJSON({ threads: store.listThreads({ limit: Number.isFinite(limit) ? limit : 50, includeArchived }) }));
3248
+ return;
3249
+ }
3250
+ if (req.method === 'POST' && path === '/api/agent/threads') {
3251
+ const body = await readJSON(req).catch(() => null);
3252
+ const record = agentRunRecord(body) ?? {};
3253
+ const thread = store.createThread({
3254
+ surface: agentRunString(record.surface),
3255
+ title: agentRunString(record.title),
3256
+ notebookPath: agentRunString(record.notebookPath),
3257
+ });
3258
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
3259
+ res.end(serializeJSON({ thread }));
3260
+ return;
3261
+ }
3262
+ // Cross-thread turn search ("search conversations") — must precede :id.
3263
+ if (req.method === 'GET' && path === '/api/agent/threads/search') {
3264
+ const query = url.searchParams.get('q') ?? '';
3265
+ const limit = Number(url.searchParams.get('limit') ?? '10');
3266
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3267
+ res.end(serializeJSON({
3268
+ turns: query.trim()
3269
+ ? store.searchTurns({ query, limit: Number.isFinite(limit) ? limit : 10 })
3270
+ : [],
3271
+ }));
3272
+ return;
3273
+ }
3274
+ const threadMatch = path.match(/^\/api\/agent\/threads\/([^/]+)(?:\/(archive|promote))?$/);
3275
+ if (threadMatch) {
3276
+ const threadId = decodeURIComponent(threadMatch[1]);
3277
+ const action = threadMatch[2];
3278
+ const thread = store.getThread(threadId);
3279
+ if (!thread) {
3280
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
3281
+ res.end(serializeJSON({ error: 'Unknown thread id.' }));
3282
+ return;
3283
+ }
3284
+ if (req.method === 'POST' && action === 'archive') {
3285
+ store.archiveThread(threadId);
3286
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3287
+ res.end(serializeJSON({ ok: true }));
3288
+ return;
3289
+ }
3290
+ // Explicit promotion — the ONLY path from conversation into durable
3291
+ // memory ("remember this"). Deliberate user act, tagged for audit.
3292
+ if (req.method === 'POST' && action === 'promote') {
3293
+ const body = await readJSON(req).catch(() => null);
3294
+ const record = agentRunRecord(body) ?? {};
3295
+ const turnId = agentRunString(record.turnId);
3296
+ const turn = turnId
3297
+ ? store.recentTurns(threadId, 200).find((candidate) => candidate.id === turnId)
3298
+ : undefined;
3299
+ if (!turn) {
3300
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3301
+ res.end(serializeJSON({ error: 'turnId is required and must belong to this thread.' }));
3302
+ return;
3303
+ }
3304
+ const scopeRaw = agentRunString(record.scope);
3305
+ const scope = scopeRaw === 'notebook' || scopeRaw === 'project' || scopeRaw === 'user' ? scopeRaw : 'project';
3306
+ const memory = new MemoryStore(defaultMemoryPath(projectRoot));
3307
+ try {
3308
+ const saved = memory.upsert({
3309
+ scope,
3310
+ title: agentRunString(record.title) ?? turn.question.slice(0, 120),
3311
+ content: `Q: ${turn.question}\nA: ${turn.answerSummary ?? turn.answerText ?? '(no answer summary)'}`,
3312
+ tags: ['conversation', threadId],
3313
+ source: 'conversation',
3314
+ confidence: 0.6,
3315
+ importance: 0.5,
3316
+ enabled: true,
3317
+ });
3318
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3319
+ res.end(serializeJSON({ ok: true, memory: saved }));
3320
+ }
3321
+ finally {
3322
+ memory.close();
3323
+ }
3324
+ return;
3325
+ }
3326
+ if (req.method === 'GET' && !action) {
3327
+ const limit = Number(url.searchParams.get('limit') ?? '50');
3328
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3329
+ res.end(serializeJSON({
3330
+ thread,
3331
+ turns: store.recentTurns(threadId, Number.isFinite(limit) ? limit : 50),
3332
+ }));
3333
+ return;
3334
+ }
3335
+ }
3336
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
3337
+ res.end(serializeJSON({ error: 'Method not allowed' }));
3338
+ }
3339
+ catch (error) {
3340
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3341
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3342
+ }
3343
+ return;
3344
+ }
2665
3345
  if (req.method === 'GET' && path === '/api/agent/memory') {
2666
3346
  const memory = new MemoryStore(defaultMemoryPath(projectRoot));
2667
3347
  try {
@@ -3185,6 +3865,7 @@ export async function startLocalServer(opts) {
3185
3865
  const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
3186
3866
  const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
3187
3867
  const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
3868
+ const dqlArtifact = normalizeDqlArtifactReference(body.dqlArtifact);
3188
3869
  if (!notebookPath || !question) {
3189
3870
  res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3190
3871
  res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
@@ -3204,6 +3885,7 @@ export async function startLocalServer(opts) {
3204
3885
  context: body.context,
3205
3886
  generatedSql: notebookResearchString(body.generatedSql),
3206
3887
  reviewedSql: notebookResearchString(body.reviewedSql),
3888
+ dqlArtifact,
3207
3889
  });
3208
3890
  const run = body.run === true
3209
3891
  ? await runNotebookResearch(storage, created, {
@@ -3214,6 +3896,7 @@ export async function startLocalServer(opts) {
3214
3896
  context: body.context,
3215
3897
  generatedSql: notebookResearchString(body.generatedSql),
3216
3898
  reviewedSql: notebookResearchString(body.reviewedSql),
3899
+ dqlArtifact,
3217
3900
  })
3218
3901
  : created;
3219
3902
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -3256,8 +3939,8 @@ export async function startLocalServer(opts) {
3256
3939
  intent,
3257
3940
  researchPattern: notebookResearchIntentPattern(intent),
3258
3941
  },
3259
- strictness: 'balanced',
3260
- limit: 100,
3942
+ strictness: 'exploratory',
3943
+ limit: 160,
3261
3944
  });
3262
3945
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3263
3946
  res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
@@ -3481,6 +4164,7 @@ export async function startLocalServer(opts) {
3481
4164
  routeDecision: body.routeDecision,
3482
4165
  generatedSql: notebookResearchString(body.generatedSql),
3483
4166
  reviewedSql: notebookResearchString(body.reviewedSql),
4167
+ dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
3484
4168
  warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
3485
4169
  reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
3486
4170
  dqlPromotionAction: notebookResearchPromotionAction(body.dqlPromotionAction),
@@ -3493,6 +4177,7 @@ export async function startLocalServer(opts) {
3493
4177
  previewError: updated.error,
3494
4178
  generatedSql: updated.generatedSql,
3495
4179
  reviewedSql: updated.reviewedSql,
4180
+ dqlArtifact: updated.dqlArtifact,
3496
4181
  routeDecision: updated.routeDecision,
3497
4182
  }),
3498
4183
  }) ?? updated;
@@ -3512,6 +4197,7 @@ export async function startLocalServer(opts) {
3512
4197
  context: body.context,
3513
4198
  generatedSql: notebookResearchString(body.generatedSql),
3514
4199
  reviewedSql: notebookResearchString(body.reviewedSql),
4200
+ dqlArtifact: normalizeDqlArtifactReference(body.dqlArtifact),
3515
4201
  });
3516
4202
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3517
4203
  res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
@@ -5676,6 +6362,10 @@ export async function startLocalServer(opts) {
5676
6362
  conversationContext: conversationContext && typeof conversationContext === 'object' && !Array.isArray(conversationContext)
5677
6363
  ? conversationContext
5678
6364
  : undefined,
6365
+ // Intentionally NOT threading a route effort here: this raw block/chat
6366
+ // authoring endpoint keeps each SDK runner's own tuned default (the
6367
+ // claude-agent-sdk runs `xhigh` for deep authoring). Governed answer /
6368
+ // research paths carry task-adaptive effort; this one shouldn't cap it.
5679
6369
  projectRoot,
5680
6370
  executeCertifiedBlock: executeCertifiedBlockForAgent,
5681
6371
  executeGeneratedSql: executeGeneratedSqlForAgent,
@@ -6663,16 +7353,75 @@ export function resolveDefaultLLMProvider(projectRoot) {
6663
7353
  * answer-loop runner — never the MCP `claudeCodeRunner`, which doesn't emit a governed
6664
7354
  * answer envelope. Everything else uses the Settings-resolved default runner.
6665
7355
  */
6666
- function resolveGovernedAnswerRunner(projectRoot) {
7356
+ export function resolveGovernedAnswerRunner(projectRoot) {
6667
7357
  const active = getActiveProvider(projectRoot);
6668
- if (active === 'claude-code' || active === 'codex') {
7358
+ if (isGovernedAnswerProviderId(active)) {
6669
7359
  return { provider: active, runner: createDqlAgentProviderRunner(active) };
6670
7360
  }
6671
7361
  const resolved = resolveDefaultLLMProvider(projectRoot);
6672
7362
  if (!resolved)
6673
7363
  return null;
6674
- const runner = getLLMRunner(resolved);
6675
- return runner ? { provider: resolved, runner } : null;
7364
+ if (isGovernedAnswerProviderId(resolved)) {
7365
+ return { provider: resolved, runner: createDqlAgentProviderRunner(resolved) };
7366
+ }
7367
+ return null;
7368
+ }
7369
+ function isGovernedAnswerProviderId(value) {
7370
+ return value === 'anthropic'
7371
+ || value === 'openai'
7372
+ || value === 'gemini'
7373
+ || value === 'ollama'
7374
+ || value === 'custom-openai'
7375
+ || value === 'claude-code'
7376
+ || value === 'codex';
7377
+ }
7378
+ /** Map a runner provider id to the settings id whose reasoning ceiling applies. */
7379
+ function reasoningSettingsIdFor(provider) {
7380
+ return provider === 'claude-agent-sdk' ? 'anthropic' : provider;
7381
+ }
7382
+ /** Type-guard for the reasoning-effort settings value from an untrusted request body. */
7383
+ function isReasoningEffortSetting(value) {
7384
+ return value === 'auto' || value === 'low' || value === 'medium' || value === 'high';
7385
+ }
7386
+ // Per-project singleton OAuth managers — they hold pending-auth state that must
7387
+ // survive between the `/start` request and the loopback callback. Keyed by
7388
+ // project root. "Pending" is derived from the manager itself (isPending) so a
7389
+ // superseded double-start can't desync a separate flag.
7390
+ const claudeOAuthManagers = new Map();
7391
+ const codexOAuthManagers = new Map();
7392
+ function getOAuthManager(projectRoot, provider) {
7393
+ if (provider === 'claude') {
7394
+ let m = claudeOAuthManagers.get(projectRoot);
7395
+ if (!m) {
7396
+ m = new ClaudeOAuthManager(projectRoot);
7397
+ claudeOAuthManagers.set(projectRoot, m);
7398
+ }
7399
+ return m;
7400
+ }
7401
+ let m = codexOAuthManagers.get(projectRoot);
7402
+ if (!m) {
7403
+ m = new CodexOAuthManager(projectRoot);
7404
+ codexOAuthManagers.set(projectRoot, m);
7405
+ }
7406
+ return m;
7407
+ }
7408
+ /**
7409
+ * Resolve the concrete reasoning effort for a run: the route's task-adaptive
7410
+ * effort (bumped one level on a repair attempt), clamped by the provider's
7411
+ * Settings ceiling (`auto` = no cap). Providers no-op when their model has no
7412
+ * reasoning surface, so returning a concrete level here is always safe.
7413
+ */
7414
+ function resolveRunReasoningEffort(projectRoot, provider, route, isRepair) {
7415
+ const ceiling = getEffectiveProviderConfig(projectRoot, reasoningSettingsIdFor(provider)).reasoningEffort;
7416
+ let desired = route ? routeReasoningEffort(route) : 'medium';
7417
+ if (isRepair)
7418
+ desired = bumpReasoningEffort(desired);
7419
+ return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
7420
+ }
7421
+ function resolveRequestedRunReasoningEffort(projectRoot, provider, requested, isRepair) {
7422
+ const ceiling = getEffectiveProviderConfig(projectRoot, reasoningSettingsIdFor(provider)).reasoningEffort;
7423
+ const desired = isRepair ? bumpReasoningEffort(requested) : requested;
7424
+ return ceiling ? clampReasoningEffort(desired, ceiling) : desired;
6676
7425
  }
6677
7426
  function loadAppDashboard(projectRoot, appId, dashboardId) {
6678
7427
  for (const p of findAppDocuments(projectRoot)) {
@@ -6998,6 +7747,11 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
6998
7747
  if (proposal.certification.errors.length === 0)
6999
7748
  readyForReview += 1;
7000
7749
  }
7750
+ const reviewableProposals = buildReviewableProposals(projectRoot, manifestPath, projectConfig, summary.proposals, options.owner);
7751
+ const reviewAges = reviewableProposals
7752
+ .map((proposal) => proposal.review.reviewAgeHours)
7753
+ .filter((value) => typeof value === 'number' && Number.isFinite(value));
7754
+ const estimatedReviewMinutes = reviewableProposals.reduce((sum, proposal) => sum + proposal.review.estimatedReviewMinutes, 0);
7001
7755
  return {
7002
7756
  ready: true,
7003
7757
  summary: {
@@ -7012,11 +7766,595 @@ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectCo
7012
7766
  readyForReview,
7013
7767
  blockingTotal,
7014
7768
  warningTotal,
7769
+ reviewTelemetry: {
7770
+ existingDrafts: reviewableProposals.filter((proposal) => proposal.review.draftExists).length,
7771
+ medianReviewAgeHours: median(reviewAges),
7772
+ readyForReviewRate: ratio(readyForReview, summary.proposalsRanked),
7773
+ estimatedReviewMinutes,
7774
+ },
7015
7775
  },
7016
7776
  plan,
7017
- proposals: summary.proposals,
7777
+ proposals: reviewableProposals,
7018
7778
  };
7019
7779
  }
7780
+ function buildReviewableProposals(projectRoot, dbtManifestPath, projectConfig, proposals, owner) {
7781
+ const manifest = safeBuildProjectManifest(projectRoot);
7782
+ return proposals.map((proposal, index) => {
7783
+ const preview = buildProposePreview(projectRoot, dbtManifestPath, proposal.slug, {
7784
+ config: projectConfig.propose,
7785
+ owner,
7786
+ });
7787
+ const draftPath = proposal.path ?? resolveProposeDraftPath(projectRoot, proposal.domain, proposal.slug);
7788
+ const draftFile = join(projectRoot, draftPath);
7789
+ const draftExists = existsSync(draftFile);
7790
+ const stats = draftExists ? statSync(draftFile) : undefined;
7791
+ const firstSeenAt = stats ? stableFsTime(stats.birthtimeMs, stats.ctimeMs) : undefined;
7792
+ const lastUpdatedAt = stats ? stableFsTime(stats.mtimeMs, stats.ctimeMs) : undefined;
7793
+ const reviewAgeHours = firstSeenAt ? hoursSince(firstSeenAt) : undefined;
7794
+ const blockingCount = proposal.certification.errors.length;
7795
+ const warningCount = proposal.certification.warnings.length;
7796
+ const outputs = preview?.outputs ?? proposal.inference.declaredOutputs;
7797
+ const status = blockingCount === 0
7798
+ ? 'ready_for_review'
7799
+ : draftExists
7800
+ ? 'draft_exists'
7801
+ : 'new';
7802
+ return {
7803
+ ...proposal,
7804
+ review: {
7805
+ queueRank: index + 1,
7806
+ status,
7807
+ priorityScore: proposal.ranking.score,
7808
+ blockingCount,
7809
+ warningCount,
7810
+ estimatedReviewMinutes: estimateProposalReviewMinutes(blockingCount, warningCount, Boolean(preview?.sqlPreview)),
7811
+ draftPath,
7812
+ draftExists,
7813
+ firstSeenAt,
7814
+ lastUpdatedAt,
7815
+ reviewAgeHours,
7816
+ certifyCommand: `dql certify --from-draft ${quoteCliArg(draftPath)} --owner you@example.com`,
7817
+ payload: {
7818
+ question: preview?.examples?.[0] ?? proposal.inference.examples[0]?.question ?? `Review ${proposal.slug}`,
7819
+ model: proposal.model,
7820
+ domain: proposal.domain,
7821
+ sqlPreview: preview?.sqlPreview,
7822
+ outputs,
7823
+ grain: preview?.grain ?? proposal.inference.grain,
7824
+ pattern: preview?.pattern ?? proposal.inference.pattern,
7825
+ evidence: proposal.evidence,
7826
+ resultSample: { status: 'not_run', rows: [] },
7827
+ nearestCertifiedBlock: findNearestCertifiedBlock(manifest, proposal, outputs),
7828
+ },
7829
+ },
7830
+ };
7831
+ });
7832
+ }
7833
+ function safeBuildProjectManifest(projectRoot) {
7834
+ try {
7835
+ return buildManifest({ projectRoot });
7836
+ }
7837
+ catch {
7838
+ return { blocks: {} };
7839
+ }
7840
+ }
7841
+ function resolveProposeDraftPath(projectRoot, domain, slug) {
7842
+ const safeDomain = domain
7843
+ .trim()
7844
+ .toLowerCase()
7845
+ .replace(/[^a-z0-9/_-]+/g, '-')
7846
+ .replace(/^[-/]+|[-/]+$/g, '');
7847
+ return safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))
7848
+ ? `domains/${safeDomain}/blocks/_drafts/${slug}.dql`
7849
+ : `blocks/_drafts/${slug}.dql`;
7850
+ }
7851
+ function findNearestCertifiedBlock(manifest, proposal, outputs) {
7852
+ const outputSet = new Set(outputs.map((output) => output.toLowerCase()));
7853
+ let best;
7854
+ for (const block of Object.values(manifest.blocks ?? {})) {
7855
+ if (String(block.status ?? '').toLowerCase() !== 'certified')
7856
+ continue;
7857
+ const blockOutputs = (block.declaredOutputs ?? []).map((output) => output.toLowerCase());
7858
+ const sharedOutputs = blockOutputs.filter((output) => outputSet.has(output));
7859
+ const sameDomain = block.domain === proposal.domain;
7860
+ const overlapScore = sharedOutputs.length * 10 + (sameDomain ? 5 : 0);
7861
+ if (overlapScore <= 0)
7862
+ continue;
7863
+ if (!best || overlapScore > best.overlapScore) {
7864
+ best = {
7865
+ name: block.name,
7866
+ path: block.filePath,
7867
+ domain: block.domain,
7868
+ overlapScore,
7869
+ sharedOutputs,
7870
+ };
7871
+ }
7872
+ }
7873
+ return best;
7874
+ }
7875
+ function estimateProposalReviewMinutes(blockingCount, warningCount, hasSqlPreview) {
7876
+ const base = hasSqlPreview ? 12 : 18;
7877
+ return base + blockingCount * 8 + warningCount * 4;
7878
+ }
7879
+ function stableFsTime(primaryMs, fallbackMs) {
7880
+ const ms = Number.isFinite(primaryMs) && primaryMs > 0 ? primaryMs : fallbackMs;
7881
+ return Number.isFinite(ms) && ms > 0 ? new Date(ms).toISOString() : undefined;
7882
+ }
7883
+ function hoursSince(iso) {
7884
+ const ts = Date.parse(iso);
7885
+ if (!Number.isFinite(ts))
7886
+ return 0;
7887
+ return Number(Math.max(0, (Date.now() - ts) / 3_600_000).toFixed(2));
7888
+ }
7889
+ function median(values) {
7890
+ if (values.length === 0)
7891
+ return null;
7892
+ const sorted = [...values].sort((a, b) => a - b);
7893
+ const mid = Math.floor(sorted.length / 2);
7894
+ const value = sorted.length % 2 === 1
7895
+ ? sorted[mid]
7896
+ : (sorted[mid - 1] + sorted[mid]) / 2;
7897
+ return Number(value.toFixed(2));
7898
+ }
7899
+ function ratio(numerator, denominator) {
7900
+ if (denominator === 0)
7901
+ return null;
7902
+ return Number((numerator / denominator).toFixed(4));
7903
+ }
7904
+ function quoteCliArg(value) {
7905
+ if (/^[A-Za-z0-9_./:-]+$/.test(value))
7906
+ return value;
7907
+ return `'${value.replace(/'/g, "'\\''")}'`;
7908
+ }
7909
+ /**
7910
+ * Mine certified block clusters for recurring metric definitions and render a
7911
+ * reviewable semantic-layer changeset. This is read-only: callers must confirm
7912
+ * candidate ids through generateSemanticCompostingDrafts before files are written.
7913
+ */
7914
+ export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
7915
+ let manifest;
7916
+ try {
7917
+ manifest = buildManifest({ projectRoot });
7918
+ }
7919
+ catch (error) {
7920
+ return {
7921
+ ready: false,
7922
+ reason: error instanceof Error ? error.message : String(error),
7923
+ summary: {
7924
+ certifiedBlocksScanned: 0,
7925
+ eligibleMetricClusters: 0,
7926
+ candidatesRanked: 0,
7927
+ existingDrafts: 0,
7928
+ donorBlocksUsed: 0,
7929
+ minSupport: Math.max(2, Math.floor(options.minSupport ?? 2)),
7930
+ },
7931
+ candidates: [],
7932
+ prBody: '',
7933
+ };
7934
+ }
7935
+ const minSupport = Math.max(2, Math.floor(options.minSupport ?? 2));
7936
+ const limit = Number.isFinite(options.limit) && Number(options.limit) > 0 ? Math.floor(Number(options.limit)) : 10;
7937
+ const owner = options.owner?.trim() || resolveLocalOwner(projectRoot, { persist: false });
7938
+ const existingMetricNames = new Set(Object.entries(manifest.metrics ?? {})
7939
+ .filter(([, metric]) => !isSemanticDraftMetricPath(metric.filePath))
7940
+ .map(([name]) => name.toLowerCase()));
7941
+ const certifiedBlocks = Object.values(manifest.blocks ?? {})
7942
+ .filter((block) => String(block.status ?? '').toLowerCase() === 'certified');
7943
+ const groups = new Map();
7944
+ for (const block of certifiedBlocks) {
7945
+ const table = singleTableDependency(block);
7946
+ if (!table)
7947
+ continue;
7948
+ const where = extractCompostingWhereClause(block.sql);
7949
+ const normalizedFilter = where ? normalizeCompostingSql(where) : undefined;
7950
+ const metrics = extractCompostingAggregateMetrics(block.sql);
7951
+ if (metrics.length === 0)
7952
+ continue;
7953
+ const donor = {
7954
+ name: block.name,
7955
+ path: block.filePath,
7956
+ domain: block.domain,
7957
+ outputs: blockOutputNames(block),
7958
+ filters: where ? splitCompostingFilters(where) : [],
7959
+ };
7960
+ for (const metric of metrics) {
7961
+ const normalizedExpression = normalizeCompostingSql(metric.expression);
7962
+ const domain = block.domain?.trim() || 'misc';
7963
+ const key = [
7964
+ domain.toLowerCase(),
7965
+ table.toLowerCase(),
7966
+ metric.type,
7967
+ normalizedExpression,
7968
+ normalizedFilter ?? '',
7969
+ ].join('\0');
7970
+ const existing = groups.get(key);
7971
+ if (existing) {
7972
+ existing.aliases.push(metric.alias ?? '');
7973
+ existing.donors.set(block.filePath, donor);
7974
+ }
7975
+ else {
7976
+ groups.set(key, {
7977
+ domain,
7978
+ table,
7979
+ type: metric.type,
7980
+ expression: metric.expression,
7981
+ normalizedExpression,
7982
+ filter: where,
7983
+ normalizedFilter,
7984
+ aliases: [metric.alias ?? ''],
7985
+ donors: new Map([[block.filePath, donor]]),
7986
+ });
7987
+ }
7988
+ }
7989
+ }
7990
+ const usedNames = new Set(existingMetricNames);
7991
+ const candidates = Array.from(groups.values())
7992
+ .filter((group) => group.donors.size >= minSupport)
7993
+ .sort((a, b) => b.donors.size - a.donors.size || a.normalizedExpression.localeCompare(b.normalizedExpression))
7994
+ .map((group) => buildCompostingMetricCandidate(projectRoot, group, owner, usedNames))
7995
+ .filter((candidate) => Boolean(candidate))
7996
+ .slice(0, limit);
7997
+ const donorBlocksUsed = new Set(candidates.flatMap((candidate) => candidate.donorBlocks.map((donor) => donor.path))).size;
7998
+ return {
7999
+ ready: true,
8000
+ summary: {
8001
+ certifiedBlocksScanned: certifiedBlocks.length,
8002
+ eligibleMetricClusters: Array.from(groups.values()).filter((group) => group.donors.size >= minSupport).length,
8003
+ candidatesRanked: candidates.length,
8004
+ existingDrafts: candidates.filter((candidate) => candidate.draftExists).length,
8005
+ donorBlocksUsed,
8006
+ minSupport,
8007
+ },
8008
+ candidates,
8009
+ prBody: renderCompostingPrBody(candidates),
8010
+ };
8011
+ }
8012
+ export function generateSemanticCompostingDrafts(projectRoot, candidateIds, options = {}) {
8013
+ const changeset = buildSemanticCompostingChangeset(projectRoot, options);
8014
+ if (!changeset.ready) {
8015
+ return {
8016
+ ready: false,
8017
+ reason: changeset.reason,
8018
+ draftsWritten: 0,
8019
+ draftsSkipped: 0,
8020
+ candidates: [],
8021
+ paths: [],
8022
+ prBody: '',
8023
+ };
8024
+ }
8025
+ const wanted = candidateIds && candidateIds.length > 0 ? new Set(candidateIds) : undefined;
8026
+ const selected = wanted ? changeset.candidates.filter((candidate) => wanted.has(candidate.id)) : changeset.candidates;
8027
+ const paths = [];
8028
+ let draftsWritten = 0;
8029
+ let draftsSkipped = 0;
8030
+ for (const candidate of selected) {
8031
+ const absolutePath = join(projectRoot, candidate.draftPath);
8032
+ if (existsSync(absolutePath)) {
8033
+ draftsSkipped += 1;
8034
+ }
8035
+ else {
8036
+ mkdirSync(dirname(absolutePath), { recursive: true });
8037
+ writeFileSync(absolutePath, candidate.yaml, 'utf-8');
8038
+ draftsWritten += 1;
8039
+ }
8040
+ paths.push(candidate.draftPath);
8041
+ }
8042
+ let prBodyPath;
8043
+ const prBody = renderCompostingPrBody(selected);
8044
+ if (selected.length > 0) {
8045
+ prBodyPath = 'semantic-layer/metrics/_drafts/PR_BODY.md';
8046
+ const absolutePrBodyPath = join(projectRoot, prBodyPath);
8047
+ mkdirSync(dirname(absolutePrBodyPath), { recursive: true });
8048
+ writeFileSync(absolutePrBodyPath, prBody, 'utf-8');
8049
+ }
8050
+ return {
8051
+ ready: true,
8052
+ draftsWritten,
8053
+ draftsSkipped,
8054
+ candidates: selected,
8055
+ paths,
8056
+ prBody,
8057
+ prBodyPath,
8058
+ };
8059
+ }
8060
+ function buildCompostingMetricCandidate(projectRoot, group, owner, usedNames) {
8061
+ const baseName = preferredCompostingMetricName(group);
8062
+ const name = dedupeCompostingName(baseName, usedNames);
8063
+ if (!name)
8064
+ return undefined;
8065
+ usedNames.add(name.toLowerCase());
8066
+ const label = titleFromSlug(name);
8067
+ const donors = Array.from(group.donors.values()).sort((a, b) => a.path.localeCompare(b.path));
8068
+ const support = donors.length;
8069
+ const domainSlug = slugForPath(group.domain);
8070
+ const draftPath = `semantic-layer/metrics/_drafts/${domainSlug}/${name}.yaml`;
8071
+ const recurringFilters = group.filter ? splitCompostingFilters(group.filter) : [];
8072
+ const metric = {
8073
+ name,
8074
+ label,
8075
+ description: `Draft metric composted from ${support} certified DQL blocks. Review the expression, filter, grain, and owner before certification.`,
8076
+ domain: group.domain,
8077
+ status: 'draft',
8078
+ sql: group.expression,
8079
+ type: group.type,
8080
+ table: group.table,
8081
+ filter: group.filter,
8082
+ tags: ['composted', 'human-review'],
8083
+ owner,
8084
+ source: {
8085
+ provider: 'dql',
8086
+ objectType: 'composted_metric',
8087
+ objectId: `composted:${name}`,
8088
+ objectName: label,
8089
+ importedAt: new Date().toISOString(),
8090
+ extra: {
8091
+ support,
8092
+ donorBlocks: donors.map((donor) => donor.name),
8093
+ donorPaths: donors.map((donor) => donor.path),
8094
+ normalizedExpression: group.normalizedExpression,
8095
+ normalizedFilter: group.normalizedFilter,
8096
+ },
8097
+ },
8098
+ };
8099
+ return {
8100
+ id: `metric:${name}:${stableShortHash(`${group.domain}|${group.table}|${group.normalizedExpression}|${group.normalizedFilter ?? ''}`)}`,
8101
+ kind: 'metric',
8102
+ name,
8103
+ label,
8104
+ description: metric.description,
8105
+ domain: group.domain,
8106
+ table: group.table,
8107
+ type: group.type,
8108
+ sql: group.expression,
8109
+ filter: group.filter,
8110
+ status: 'draft',
8111
+ support,
8112
+ donorBlocks: donors,
8113
+ draftPath,
8114
+ draftExists: existsSync(join(projectRoot, draftPath)),
8115
+ yaml: serializeMetricDefinitionToYaml(metric),
8116
+ provenance: {
8117
+ normalizedExpression: group.normalizedExpression,
8118
+ normalizedFilter: group.normalizedFilter,
8119
+ recurringFilters,
8120
+ },
8121
+ review: {
8122
+ priorityScore: support * 10 + recurringFilters.length * 2,
8123
+ rationale: `Recurring ${group.type} metric found in ${support} certified blocks${group.filter ? ' with the same filter' : ''}.`,
8124
+ },
8125
+ };
8126
+ }
8127
+ function singleTableDependency(block) {
8128
+ const dependencies = Array.from(new Set((block.tableDependencies ?? []).filter((table) => table && !table.startsWith('ref:'))));
8129
+ if (dependencies.length === 1)
8130
+ return dependencies[0];
8131
+ const rawRefs = Array.from(new Set((block.rawTableRefs ?? []).filter(Boolean)));
8132
+ return rawRefs.length === 1 ? rawRefs[0] : undefined;
8133
+ }
8134
+ function blockOutputNames(block) {
8135
+ if (Array.isArray(block.declaredOutputs) && block.declaredOutputs.length > 0)
8136
+ return block.declaredOutputs;
8137
+ if (Array.isArray(block.outputContract) && block.outputContract.length > 0)
8138
+ return block.outputContract.map((output) => output.name);
8139
+ if (Array.isArray(block.outputs) && block.outputs.length > 0)
8140
+ return block.outputs.map((output) => output.name);
8141
+ return [];
8142
+ }
8143
+ function isSemanticDraftMetricPath(filePath) {
8144
+ return Boolean(filePath?.replaceAll('\\', '/').includes('semantic-layer/metrics/_drafts/'));
8145
+ }
8146
+ function extractCompostingAggregateMetrics(sql) {
8147
+ const select = extractCompostingSelectClause(sql);
8148
+ if (!select)
8149
+ return [];
8150
+ return splitCompostingTopLevel(select, ',').flatMap((item) => {
8151
+ const parsed = parseCompostingSelectItem(item);
8152
+ if (!parsed)
8153
+ return [];
8154
+ return [parsed];
8155
+ });
8156
+ }
8157
+ function extractCompostingSelectClause(sql) {
8158
+ const source = stripSqlTerminator(sql);
8159
+ const selectMatch = /\bselect\b/i.exec(source);
8160
+ if (!selectMatch)
8161
+ return undefined;
8162
+ const fromIndex = findTopLevelKeyword(source, 'from', selectMatch.index + selectMatch[0].length);
8163
+ if (fromIndex < 0)
8164
+ return undefined;
8165
+ return source.slice(selectMatch.index + selectMatch[0].length, fromIndex).trim();
8166
+ }
8167
+ function parseCompostingSelectItem(item) {
8168
+ const { expression, alias } = splitCompostingAlias(item.trim());
8169
+ const normalizedExpression = expression.trim();
8170
+ const distinctCount = normalizedExpression.match(/^count\s*\(\s*distinct\s+[\s\S]+\)$/i);
8171
+ if (distinctCount) {
8172
+ return { expression: normalizedExpression, alias, type: 'count_distinct' };
8173
+ }
8174
+ const count = normalizedExpression.match(/^count\s*\([\s\S]*\)$/i);
8175
+ if (count)
8176
+ return { expression: normalizedExpression, alias, type: 'count' };
8177
+ const aggregate = normalizedExpression.match(/^(sum|avg|min|max)\s*\([\s\S]+\)$/i);
8178
+ if (!aggregate)
8179
+ return undefined;
8180
+ return {
8181
+ expression: normalizedExpression,
8182
+ alias,
8183
+ type: aggregate[1].toLowerCase(),
8184
+ };
8185
+ }
8186
+ function splitCompostingAlias(item) {
8187
+ const asMatch = item.match(/^([\s\S]+?)\s+as\s+("[^"]+"|`[^`]+`|\[[^\]]+\]|[A-Za-z_][A-Za-z0-9_$]*)$/i);
8188
+ if (asMatch)
8189
+ return { expression: asMatch[1].trim(), alias: stripIdentifierQuotes(asMatch[2]) };
8190
+ return { expression: item.trim() };
8191
+ }
8192
+ function extractCompostingWhereClause(sql) {
8193
+ const source = stripSqlTerminator(sql);
8194
+ const whereIndex = findTopLevelKeyword(source, 'where', 0);
8195
+ if (whereIndex < 0)
8196
+ return undefined;
8197
+ const start = whereIndex + 'where'.length;
8198
+ const endCandidates = ['group by', 'having', 'order by', 'limit', 'qualify']
8199
+ .map((keyword) => findTopLevelKeyword(source, keyword, start))
8200
+ .filter((index) => index >= 0);
8201
+ const end = endCandidates.length > 0 ? Math.min(...endCandidates) : source.length;
8202
+ const where = source.slice(start, end).trim();
8203
+ return where || undefined;
8204
+ }
8205
+ function splitCompostingFilters(where) {
8206
+ return splitCompostingTopLevel(where, 'and')
8207
+ .map((part) => part.trim())
8208
+ .filter(Boolean);
8209
+ }
8210
+ function splitCompostingTopLevel(input, delimiter) {
8211
+ const out = [];
8212
+ let start = 0;
8213
+ let depth = 0;
8214
+ let quote;
8215
+ for (let i = 0; i < input.length; i += 1) {
8216
+ const char = input[i];
8217
+ if (quote) {
8218
+ if (char === quote && input[i - 1] !== '\\')
8219
+ quote = undefined;
8220
+ continue;
8221
+ }
8222
+ if (char === '"' || char === "'" || char === '`') {
8223
+ quote = char;
8224
+ continue;
8225
+ }
8226
+ if (char === '(')
8227
+ depth += 1;
8228
+ else if (char === ')')
8229
+ depth = Math.max(0, depth - 1);
8230
+ if (depth !== 0)
8231
+ continue;
8232
+ if (delimiter === ',' && char === ',') {
8233
+ out.push(input.slice(start, i).trim());
8234
+ start = i + 1;
8235
+ }
8236
+ else if (delimiter === 'and'
8237
+ && input.slice(i, i + 3).toLowerCase() === 'and'
8238
+ && !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
8239
+ && !/[A-Za-z0-9_]/.test(input[i + 3] ?? '')) {
8240
+ out.push(input.slice(start, i).trim());
8241
+ start = i + 3;
8242
+ i += 2;
8243
+ }
8244
+ }
8245
+ out.push(input.slice(start).trim());
8246
+ return out.filter(Boolean);
8247
+ }
8248
+ function findTopLevelKeyword(input, keyword, fromIndex) {
8249
+ const lower = input.toLowerCase();
8250
+ const target = keyword.toLowerCase();
8251
+ let depth = 0;
8252
+ let quote;
8253
+ for (let i = Math.max(0, fromIndex); i <= input.length - target.length; i += 1) {
8254
+ const char = input[i];
8255
+ if (quote) {
8256
+ if (char === quote && input[i - 1] !== '\\')
8257
+ quote = undefined;
8258
+ continue;
8259
+ }
8260
+ if (char === '"' || char === "'" || char === '`') {
8261
+ quote = char;
8262
+ continue;
8263
+ }
8264
+ if (char === '(') {
8265
+ depth += 1;
8266
+ continue;
8267
+ }
8268
+ if (char === ')') {
8269
+ depth = Math.max(0, depth - 1);
8270
+ continue;
8271
+ }
8272
+ if (depth !== 0)
8273
+ continue;
8274
+ if (lower.slice(i, i + target.length) === target
8275
+ && !/[A-Za-z0-9_]/.test(input[i - 1] ?? '')
8276
+ && !/[A-Za-z0-9_]/.test(input[i + target.length] ?? '')) {
8277
+ return i;
8278
+ }
8279
+ }
8280
+ return -1;
8281
+ }
8282
+ function preferredCompostingMetricName(group) {
8283
+ const aliases = group.aliases.map((alias) => slugForIdentifier(alias)).filter(Boolean);
8284
+ const counts = new Map();
8285
+ for (const alias of aliases)
8286
+ counts.set(alias, (counts.get(alias) ?? 0) + 1);
8287
+ const preferredAlias = Array.from(counts.entries()).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0]?.[0];
8288
+ if (preferredAlias)
8289
+ return preferredAlias;
8290
+ const expressionName = slugForIdentifier(group.expression.replace(/\b(sum|avg|min|max|count|distinct)\b/gi, ''));
8291
+ return expressionName ? `${expressionName}_${group.type}` : `${group.type}_metric`;
8292
+ }
8293
+ function dedupeCompostingName(baseName, usedNames) {
8294
+ const base = slugForIdentifier(baseName).slice(0, 72).replace(/_+$/g, '');
8295
+ if (!base)
8296
+ return undefined;
8297
+ if (!usedNames.has(base.toLowerCase()))
8298
+ return base;
8299
+ let index = 2;
8300
+ while (usedNames.has(`${base}_${index}`.toLowerCase()))
8301
+ index += 1;
8302
+ return `${base}_${index}`;
8303
+ }
8304
+ function normalizeCompostingSql(value) {
8305
+ return value
8306
+ .replace(/["`]/g, '')
8307
+ .replace(/\s+/g, ' ')
8308
+ .trim()
8309
+ .toLowerCase();
8310
+ }
8311
+ function slugForIdentifier(value) {
8312
+ return value
8313
+ .trim()
8314
+ .replace(/["`[\]]/g, '')
8315
+ .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
8316
+ .toLowerCase()
8317
+ .replace(/[^a-z0-9]+/g, '_')
8318
+ .replace(/^_+|_+$/g, '');
8319
+ }
8320
+ function slugForPath(value) {
8321
+ return slugForIdentifier(value) || 'misc';
8322
+ }
8323
+ function stripIdentifierQuotes(value) {
8324
+ return value.replace(/^["`\[]|["`\]]$/g, '').trim();
8325
+ }
8326
+ function titleFromSlug(value) {
8327
+ return value.split('_').filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(' ');
8328
+ }
8329
+ function stableShortHash(value) {
8330
+ return createHash('sha1').update(value).digest('hex').slice(0, 10);
8331
+ }
8332
+ function renderCompostingPrBody(candidates) {
8333
+ const lines = [
8334
+ '# Semantic Composting Changeset',
8335
+ '',
8336
+ 'This changeset proposes draft semantic metrics mined from recurring logic in certified DQL blocks.',
8337
+ 'Review each metric expression, filter, owner, grain compatibility, and downstream impact before marking it certified.',
8338
+ '',
8339
+ ];
8340
+ if (candidates.length === 0) {
8341
+ lines.push('No recurring certified-block metric clusters met the current support threshold.');
8342
+ return `${lines.join('\n')}\n`;
8343
+ }
8344
+ for (const candidate of candidates) {
8345
+ lines.push(`## ${candidate.label}`);
8346
+ lines.push('');
8347
+ lines.push(`- Draft path: \`${candidate.draftPath}\``);
8348
+ lines.push(`- Domain/table: \`${candidate.domain}\` / \`${candidate.table}\``);
8349
+ lines.push(`- Expression: \`${candidate.sql}\``);
8350
+ if (candidate.filter)
8351
+ lines.push(`- Filter: \`${candidate.filter}\``);
8352
+ lines.push(`- Support: ${candidate.support} certified block(s)`);
8353
+ lines.push(`- Donors: ${candidate.donorBlocks.map((donor) => `\`${donor.path}\``).join(', ')}`);
8354
+ lines.push('');
8355
+ }
8356
+ return `${lines.join('\n')}\n`;
8357
+ }
7020
8358
  export async function generateProposeDrafts(projectRoot, slugs, projectConfig = loadProjectConfig(projectRoot), options = {}) {
7021
8359
  const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
7022
8360
  if (!manifestPath) {
@@ -8150,6 +9488,20 @@ function parseBlockStudioArrayField(source, key) {
8150
9488
  function parseBlockStudioStringField(source, key) {
8151
9489
  return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
8152
9490
  }
9491
+ function parseSemanticRequestedFilters(source) {
9492
+ return parseBlockStudioArrayField(source, 'requested_filters')
9493
+ .map((entry) => {
9494
+ const match = entry.match(/^([^=<>!]+)\s*=\s*(.+)$/);
9495
+ if (!match)
9496
+ return undefined;
9497
+ const dimension = match[1]?.trim();
9498
+ const value = match[2]?.trim();
9499
+ if (!dimension || !value)
9500
+ return undefined;
9501
+ return { dimension, operator: 'equals', values: [value] };
9502
+ })
9503
+ .filter((filter) => Boolean(filter));
9504
+ }
8153
9505
  function parseSemanticBlockConfig(source) {
8154
9506
  const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
8155
9507
  ? 'semantic'
@@ -8157,6 +9509,7 @@ function parseSemanticBlockConfig(source) {
8157
9509
  const metric = parseBlockStudioStringField(source, 'metric');
8158
9510
  const metrics = parseBlockStudioArrayField(source, 'metrics');
8159
9511
  const dimensions = parseBlockStudioArrayField(source, 'dimensions');
9512
+ const filters = parseSemanticRequestedFilters(source);
8160
9513
  const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
8161
9514
  const granularity = parseBlockStudioStringField(source, 'granularity');
8162
9515
  const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
@@ -8165,6 +9518,7 @@ function parseSemanticBlockConfig(source) {
8165
9518
  metric,
8166
9519
  metrics,
8167
9520
  dimensions,
9521
+ filters,
8168
9522
  timeDimension,
8169
9523
  granularity,
8170
9524
  limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
@@ -8310,6 +9664,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
8310
9664
  ? composeRuntimeSemanticQuery({
8311
9665
  metrics,
8312
9666
  dimensions: config.dimensions,
9667
+ filters: config.filters,
8313
9668
  timeDimension: config.timeDimension && config.granularity
8314
9669
  ? { name: config.timeDimension, granularity: config.granularity }
8315
9670
  : undefined,
@@ -8324,6 +9679,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
8324
9679
  : semanticLayer.composeQuery({
8325
9680
  metrics,
8326
9681
  dimensions: config.dimensions,
9682
+ filters: config.filters,
8327
9683
  timeDimension: config.timeDimension && config.granularity
8328
9684
  ? { name: config.timeDimension, granularity: config.granularity }
8329
9685
  : undefined,
@@ -8490,6 +9846,101 @@ export function validateBlockStudioSource(source, semanticLayer) {
8490
9846
  executableSql,
8491
9847
  };
8492
9848
  }
9849
+ export async function createDqlArtifactGenerationSessionForProject(projectRoot, options, semanticLayer) {
9850
+ const dqlArtifact = normalizeDqlArtifactReference(options.dqlArtifact);
9851
+ if (!dqlArtifact)
9852
+ throw new Error('A DQL artifact with source is required for DQL draft promotion.');
9853
+ const question = notebookResearchString(options.question) ?? dqlArtifact.name ?? 'Generated DQL artifact';
9854
+ const slug = deriveGeneratedDraftSlug(dqlArtifact.name ?? question);
9855
+ const domain = notebookResearchString(options.domain) ?? 'misc';
9856
+ const owner = notebookResearchString(options.owner) ?? '';
9857
+ const tags = Array.from(new Set(['notebook-research', 'review-required', 'dql-artifact', ...(Array.isArray(options.tags) ? options.tags : [])]
9858
+ .map((tag) => notebookResearchString(tag))
9859
+ .filter((tag) => Boolean(tag))));
9860
+ const proposedContractId = `${domain}.Unknown.${slug}`;
9861
+ const draft = upsertGeneratedDqlArtifactDraft(projectRoot, {
9862
+ slug,
9863
+ question,
9864
+ proposedContractId,
9865
+ owner,
9866
+ proposedDomain: domain,
9867
+ dqlArtifact,
9868
+ sourceQuestion: question,
9869
+ sourceBlock: options.sourceBlock,
9870
+ contextPackId: options.contextPackId,
9871
+ routeIntent: options.routeIntent,
9872
+ outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
9873
+ });
9874
+ const now = new Date().toISOString();
9875
+ const uniqueSuffix = createHash('sha1').update(`${now}:${draft.path}:${dqlArtifact.source}`).digest('hex').slice(0, 12);
9876
+ const importId = `imp_dql_artifact_${uniqueSuffix}`;
9877
+ const candidateId = `cand_${slug.replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') || 'dql_artifact'}_${uniqueSuffix.slice(0, 6)}`;
9878
+ const sourcePath = dqlArtifact.sourcePath ?? options.inputPath ?? draft.path;
9879
+ const savedSource = readFileSync(join(projectRoot, draft.path), 'utf-8');
9880
+ const candidate = {
9881
+ id: candidateId,
9882
+ sourceKind: 'raw-sql-file',
9883
+ sourcePath,
9884
+ name: dqlArtifact.name ?? slug,
9885
+ domain,
9886
+ description: question,
9887
+ owner,
9888
+ tags,
9889
+ grain: dqlArtifact.timeDimension ? `${dqlArtifact.timeDimension.name}.${dqlArtifact.timeDimension.granularity}` : undefined,
9890
+ outputs: Array.from(new Set([...(dqlArtifact.dimensions ?? []), ...(dqlArtifact.metrics ?? [])])),
9891
+ dimensions: dqlArtifact.dimensions,
9892
+ allowedFilters: dqlArtifact.filters?.map((filter) => filter.dimension),
9893
+ parameterPolicy: dqlArtifact.filters?.map((filter) => ({ name: filter.dimension, policy: 'dynamic' })),
9894
+ sql: notebookResearchString(options.generatedSql) ?? '',
9895
+ dqlSource: savedSource,
9896
+ validation: validateBlockStudioSource(savedSource, semanticLayer),
9897
+ preview: null,
9898
+ lineage: {
9899
+ sourceTables: [],
9900
+ parameters: [],
9901
+ warnings: [],
9902
+ statementIndex: 1,
9903
+ totalStatements: 1,
9904
+ },
9905
+ confidence: 0.95,
9906
+ splitStrategy: 'manual',
9907
+ warnings: [],
9908
+ conversionNotes: ['Persisted the DQL artifact returned by the governed answer; no SQL-to-DQL regeneration was required.'],
9909
+ aiAssistance: [],
9910
+ reviewStatus: 'draft',
9911
+ savedPath: draft.path,
9912
+ generationMode: 'deterministic',
9913
+ generationProvider: 'dql-artifact',
9914
+ llmContext: `Persisted generated DQL artifact for: ${question}`,
9915
+ evidence: [],
9916
+ draftSave: { status: 'saved', path: draft.path, savedAt: now },
9917
+ recommendedAction: 'create_new',
9918
+ similarityMatches: [],
9919
+ };
9920
+ const session = {
9921
+ id: importId,
9922
+ sourceKind: 'raw-sql-file',
9923
+ inputPath: sourcePath,
9924
+ inputMode: options.inputMode ?? 'upload',
9925
+ sourceFiles: [sourcePath],
9926
+ createdAt: now,
9927
+ updatedAt: now,
9928
+ defaults: { domain, owner, tags },
9929
+ candidateIds: [candidate.id],
9930
+ mode: 'ai-import',
9931
+ generation: {
9932
+ provider: 'dql-artifact',
9933
+ aiEnabled: false,
9934
+ contextObjectCount: 0,
9935
+ createdDrafts: 1,
9936
+ warnings: [],
9937
+ },
9938
+ candidates: [candidate],
9939
+ };
9940
+ writeBlockStudioImportSession(projectRoot, session);
9941
+ await refreshLocalMetadataCatalog(projectRoot);
9942
+ return session;
9943
+ }
8493
9944
  export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
8494
9945
  const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
8495
9946
  const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
@@ -9488,7 +10939,13 @@ function inferDqlGenerationPattern(sql) {
9488
10939
  return 'trend';
9489
10940
  if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
9490
10941
  return 'entity_rollup';
9491
- if (!/\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql) && /\b(dim|profile|customer|account|player|product|user|entity)\b/i.test(sql)) {
10942
+ // Entity profile = a non-aggregated, un-grouped row lookup filtered by an
10943
+ // identifier (…WHERE <something>_id = / IN …). Structural + domain-agnostic —
10944
+ // no hard-coded entity vocabulary.
10945
+ const hasAggregate = /\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql);
10946
+ const hasGroupBy = /\bgroup\s+by\b/i.test(sql);
10947
+ const filtersByIdentifier = /\bwhere\b[\s\S]*?\b[A-Za-z_][A-Za-z0-9_]*(?:_id|_key|id)\b\s*(?:=|in\b)/i.test(sql);
10948
+ if (!hasAggregate && !hasGroupBy && filtersByIdentifier) {
9492
10949
  return 'entity_profile';
9493
10950
  }
9494
10951
  return 'custom';
@@ -10086,10 +11543,15 @@ async function createBlockStudioAssistProvider(projectRoot, requestedProvider) {
10086
11543
  provider = new OpenAIProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model, allowNoApiKey: true });
10087
11544
  break;
10088
11545
  case 'claude-code':
10089
- provider = new ClaudeCodeCliProvider({ model: config.model });
11546
+ // OAuth-first: use the browser-login token when connected; fall back to the CLI passthrough.
11547
+ provider = claudeOAuthConnected(projectRoot)
11548
+ ? new ClaudeOAuthProvider({ projectRoot, model: config.model })
11549
+ : new ClaudeCodeCliProvider({ model: config.model });
10090
11550
  break;
10091
11551
  case 'codex':
10092
- provider = new CodexCliProvider({ model: config.model });
11552
+ provider = codexOAuthConnected(projectRoot)
11553
+ ? new CodexOAuthProvider({ projectRoot, model: config.model })
11554
+ : new CodexCliProvider({ model: config.model });
10093
11555
  break;
10094
11556
  default:
10095
11557
  return null;
@@ -10103,8 +11565,7 @@ function agentResultToSynthesisPreview(result) {
10103
11565
  const columns = Array.isArray(result.columns)
10104
11566
  ? result.columns.map((col) => typeof col === 'string' ? col : col?.name ?? String(col))
10105
11567
  : [];
10106
- const rawRows = Array.isArray(result.rows) ? result.rows.slice(0, 20) : [];
10107
- const rows = rawRows.map((row) => {
11568
+ const normalizeRow = (row) => {
10108
11569
  if (row && typeof row === 'object' && !Array.isArray(row))
10109
11570
  return row;
10110
11571
  // Array-shaped rows → zip with column names.
@@ -10114,69 +11575,20 @@ function agentResultToSynthesisPreview(result) {
10114
11575
  return obj;
10115
11576
  }
10116
11577
  return { value: row };
10117
- });
11578
+ };
11579
+ // Normalize ALL rows so verified statistics cover the full result — the
11580
+ // narrator sees only a 20-row sample, and sample-derived aggregate claims
11581
+ // ("spend ranges from X to Y") were the last hallucination surface.
11582
+ const allRows = Array.isArray(result.rows) ? result.rows.map(normalizeRow) : [];
11583
+ const rows = allRows.slice(0, 20);
11584
+ const resolvedColumns = columns.length > 0 ? columns : Object.keys(rows[0] ?? {});
10118
11585
  return {
10119
- columns: columns.length > 0 ? columns : Object.keys(rows[0] ?? {}),
11586
+ columns: resolvedColumns,
10120
11587
  rows,
10121
- rowCount: typeof result.rowCount === 'number' ? result.rowCount : rows.length,
11588
+ rowCount: typeof result.rowCount === 'number' ? result.rowCount : allRows.length,
11589
+ stats: computeResultStats(resolvedColumns, allRows),
10122
11590
  };
10123
11591
  }
10124
- const SIGNAL_STOPWORDS = new Set([
10125
- 'the', 'a', 'an', 'of', 'for', 'to', 'by', 'in', 'on', 'and', 'or', 'is', 'are', 'was', 'were',
10126
- 'what', 'which', 'how', 'show', 'me', 'my', 'our', 'this', 'that', 'top', 'give', 'get', 'list',
10127
- ]);
10128
- function signalTokens(text) {
10129
- return new Set(text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ').split(/\s+/)
10130
- .filter((token) => token.length > 2 && !SIGNAL_STOPWORDS.has(token)));
10131
- }
10132
- /** Best token-overlap ratio between the question and any candidate name/description. */
10133
- function bestOverlap(questionTokens, candidates) {
10134
- if (questionTokens.size === 0)
10135
- return 0;
10136
- let best = 0;
10137
- for (const candidate of candidates) {
10138
- const candTokens = signalTokens(candidate);
10139
- if (candTokens.size === 0)
10140
- continue;
10141
- let hits = 0;
10142
- for (const token of candTokens)
10143
- if (questionTokens.has(token))
10144
- hits += 1;
10145
- if (hits === 0)
10146
- continue;
10147
- // Ratio of the candidate's tokens the question covers — rewards a tight match.
10148
- const ratio = hits / Math.min(candTokens.size, questionTokens.size);
10149
- if (ratio > best)
10150
- best = ratio;
10151
- }
10152
- return Math.min(1, best);
10153
- }
10154
- /**
10155
- * Cheap, offline retrieval-signal probe. Fills certifiedScore/metricScore/hasRetrieval
10156
- * from local catalog name/description overlap so the deterministic router can reach
10157
- * confidence (and skip the LLM) on obvious governed matches. Best-effort — any failure
10158
- * yields empty signals and the router simply leans on its LLM classification.
10159
- */
10160
- function computeAgentRunSignals(projectRoot, question) {
10161
- try {
10162
- const tokens = signalTokens(question);
10163
- if (tokens.size === 0)
10164
- return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
10165
- const certifiedBlocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
10166
- const blockStrings = certifiedBlocks.flatMap((block) => [block.name, block.description ?? ''].filter(Boolean));
10167
- const metrics = loadSemanticMetrics(projectRoot);
10168
- const metricStrings = metrics.map((metric) => {
10169
- const node = metric;
10170
- return node.name ?? node.label ?? node.id ?? '';
10171
- }).filter(Boolean);
10172
- const certifiedScore = bestOverlap(tokens, blockStrings);
10173
- const metricScore = bestOverlap(tokens, metricStrings);
10174
- return { certifiedScore, metricScore, hasRetrieval: certifiedScore > 0 || metricScore > 0 };
10175
- }
10176
- catch {
10177
- return { certifiedScore: 0, metricScore: 0, hasRetrieval: false };
10178
- }
10179
- }
10180
11592
  /** Up to three example questions built from real catalog blocks (falls back to generic asks). */
10181
11593
  function buildConversationSuggestions(projectRoot, kind) {
10182
11594
  if (kind === 'gratitude')
@@ -10231,6 +11643,141 @@ function buildConversationSystemPrompt(kind, isGeneralKnowledge, catalogContext,
10231
11643
  }
10232
11644
  return lines.join('\n');
10233
11645
  }
11646
+ function renderConversationMemoryForPrompt(context) {
11647
+ const recap = buildConversationContextRecap(context);
11648
+ if (!recap)
11649
+ return undefined;
11650
+ return [
11651
+ 'Current conversation context:',
11652
+ recap,
11653
+ 'If the user asks what this conversation is about, summarize this context directly. Do not query data or invent values.',
11654
+ ].join('\n');
11655
+ }
11656
+ function buildConversationContextRecap(context) {
11657
+ if (!context)
11658
+ return undefined;
11659
+ const turnRecap = buildConversationTurnsRecap(context);
11660
+ if (turnRecap)
11661
+ return turnRecap;
11662
+ const sourceQuestion = agentRunString(context.sourceQuestion);
11663
+ const sourceAnswerSummary = agentRunString(context.sourceAnswerSummary);
11664
+ const columns = agentRunStringArray(context.resultColumns ?? context.outputColumns).slice(0, 8);
11665
+ const values = agentRunRecord(context.resultDimensionValues);
11666
+ const rows = Array.isArray(context.resultRowsSample)
11667
+ ? context.resultRowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
11668
+ : [];
11669
+ const firstRow = rows[0];
11670
+ const leadParts = buildPriorValueLeadParts(values);
11671
+ const rowFacts = firstRow
11672
+ ? columns
11673
+ .slice(0, 5)
11674
+ .map((column) => {
11675
+ const value = firstRow[column];
11676
+ return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
11677
+ })
11678
+ .filter(Boolean)
11679
+ : [];
11680
+ const contextBits = [
11681
+ sourceQuestion ? `the question "${sourceQuestion}"` : '',
11682
+ sourceAnswerSummary ? `the latest answer summary: ${sourceAnswerSummary}` : '',
11683
+ columns.length ? `result columns: ${columns.join(', ')}` : '',
11684
+ leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
11685
+ rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
11686
+ ].filter(Boolean);
11687
+ if (contextBits.length === 0)
11688
+ return undefined;
11689
+ return `We were talking about ${contextBits.join('. ')}.`;
11690
+ }
11691
+ function buildConversationTurnsRecap(context) {
11692
+ const turns = Array.isArray(context.turns)
11693
+ ? context.turns.map(agentRunRecord).filter((turn) => Boolean(turn)).slice(-3)
11694
+ : [];
11695
+ if (turns.length === 0)
11696
+ return undefined;
11697
+ const activeTurnId = agentRunString(context.activeTurnId);
11698
+ const active = activeTurnId
11699
+ ? turns.find((turn) => agentRunString(turn.id) === activeTurnId)
11700
+ : undefined;
11701
+ const selected = active ?? turns[turns.length - 1];
11702
+ if (!selected)
11703
+ return undefined;
11704
+ const result = agentRunRecord(selected.result);
11705
+ const values = agentRunRecord(result?.dimensionValues);
11706
+ const columns = agentRunStringArray(result?.columns).slice(0, 8);
11707
+ const rows = Array.isArray(result?.rowsSample)
11708
+ ? result.rowsSample.map(agentRunRecord).filter((row) => Boolean(row)).slice(0, 1)
11709
+ : [];
11710
+ const firstRow = rows[0];
11711
+ const leadParts = buildPriorValueLeadParts(values);
11712
+ const rowFacts = firstRow
11713
+ ? columns
11714
+ .slice(0, 5)
11715
+ .map((column) => {
11716
+ const value = firstRow[column];
11717
+ return value === undefined || value === null || typeof value === 'object' ? '' : `${column}=${String(value)}`;
11718
+ })
11719
+ .filter(Boolean)
11720
+ : [];
11721
+ const previous = turns
11722
+ .filter((turn) => turn !== selected)
11723
+ .map((turn) => agentRunString(turn.question))
11724
+ .filter((value) => Boolean(value))
11725
+ .slice(-2);
11726
+ const bits = [
11727
+ agentRunString(selected.question) ? `the latest question "${agentRunString(selected.question)}"` : '',
11728
+ agentRunString(selected.answerSummary) ? `latest answer summary: ${agentRunString(selected.answerSummary)}` : '',
11729
+ columns.length ? `result columns: ${columns.join(', ')}` : '',
11730
+ leadParts.length ? `notable prior values: ${leadParts.join(', ')}` : '',
11731
+ rowFacts.length ? `first result row: ${rowFacts.join(', ')}` : '',
11732
+ previous.length ? `earlier in this thread: ${previous.join(' / ')}` : '',
11733
+ ].filter(Boolean);
11734
+ return bits.length > 0 ? `We were talking about ${bits.join('. ')}.` : undefined;
11735
+ }
11736
+ function agentRunStringArray(value) {
11737
+ if (!Array.isArray(value))
11738
+ return [];
11739
+ return Array.from(new Set(value.map(agentRunString).filter((item) => Boolean(item))));
11740
+ }
11741
+ function firstPriorValue(values, keys) {
11742
+ if (!values)
11743
+ return undefined;
11744
+ for (const key of keys) {
11745
+ const list = Array.isArray(values[key]) ? values[key] : [];
11746
+ const first = list.map(agentRunString).find(Boolean);
11747
+ if (first)
11748
+ return first;
11749
+ }
11750
+ return undefined;
11751
+ }
11752
+ /**
11753
+ * Build human-readable "notable prior values" for conversation recap from whatever
11754
+ * dimension columns the prior result actually had. Domain-agnostic: name/label-ish
11755
+ * columns are surfaced first, then any other dimension — no hard-coded jaffle entity
11756
+ * columns — so follow-up questions get meaningful context on ANY repo.
11757
+ */
11758
+ function buildPriorValueLeadParts(values) {
11759
+ if (!values)
11760
+ return [];
11761
+ const keys = Object.keys(values);
11762
+ const nameLike = keys.filter((key) => /(?:^|_)(?:name|label|title|nm)$/i.test(key));
11763
+ const ordered = [...nameLike, ...keys.filter((key) => !nameLike.includes(key))];
11764
+ const parts = [];
11765
+ const seenLabels = new Set();
11766
+ for (const key of ordered) {
11767
+ if (parts.length >= 3)
11768
+ break;
11769
+ const value = firstPriorValue(values, [key]);
11770
+ if (!value)
11771
+ continue;
11772
+ const label = (key.replace(/_?(?:name|id|key|label|title|nm)$/i, '').replace(/[_-]+/g, ' ').trim()
11773
+ || key.replace(/[_-]+/g, ' ').trim()).toLowerCase();
11774
+ if (seenLabels.has(label))
11775
+ continue;
11776
+ seenLabels.add(label);
11777
+ parts.push(label ? `${label} "${value}"` : `"${value}"`);
11778
+ }
11779
+ return parts;
11780
+ }
10234
11781
  /** Deterministic, warm reply when no AI provider is configured — never governance-speak. */
10235
11782
  function buildConversationalFallback(kind, isGeneralKnowledge, question, suggestions) {
10236
11783
  const examples = suggestions.length > 0
@@ -11995,13 +13542,6 @@ function collectSettingsEnvStatus() {
11995
13542
  },
11996
13543
  ];
11997
13544
  }
11998
- function isProviderSettingsId(value) {
11999
- return value === 'anthropic'
12000
- || value === 'openai'
12001
- || value === 'gemini'
12002
- || value === 'ollama'
12003
- || value === 'custom-openai';
12004
- }
12005
13545
  function isDeterministicDqlGenerationProvider(value) {
12006
13546
  if (typeof value !== 'string')
12007
13547
  return false;
@@ -12022,8 +13562,8 @@ async function testProviderConfig(projectRoot, id, overrides) {
12022
13562
  const label = providerSettingsLabel(id);
12023
13563
  const details = providerConfigDetails(id, config);
12024
13564
  const isSubscriptionCli = id === 'claude-code' || id === 'codex';
12025
- // Subscription CLI providers are always testable — detection (installed + logged in)
12026
- // needs no saved config; the others require enabling/saving first.
13565
+ // Subscription providers are always testable — the connection (browser OAuth, or
13566
+ // an installed+logged-in CLI) needs no saved config; the others require enabling first.
12027
13567
  if (!config.enabled && !isSubscriptionCli) {
12028
13568
  return { ok: false, message: `${label} is disabled. Enable it (or Save) before testing.` };
12029
13569
  }
@@ -12031,6 +13571,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
12031
13571
  return testOpenAIProviderConfig(config, label, details);
12032
13572
  if (id === 'anthropic')
12033
13573
  return testAnthropicProviderConfig(config, label, details);
13574
+ // OAuth-first for subscriptions: when the user signed in via the browser, test the
13575
+ // OAuth-backed provider (no CLI required); otherwise fall back to the CLI passthrough.
13576
+ // Mirrors the runtime selection in dql-agent-provider / createBlockStudioAssistProvider.
13577
+ const claudeOAuth = id === 'claude-code' && claudeOAuthConnected(projectRoot);
13578
+ const codexOAuth = id === 'codex' && codexOAuthConnected(projectRoot);
12034
13579
  let provider;
12035
13580
  switch (id) {
12036
13581
  case 'gemini':
@@ -12040,10 +13585,14 @@ async function testProviderConfig(projectRoot, id, overrides) {
12040
13585
  provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
12041
13586
  break;
12042
13587
  case 'claude-code':
12043
- provider = new ClaudeCodeCliProvider({ model: config.model });
13588
+ provider = claudeOAuth
13589
+ ? new ClaudeOAuthProvider({ projectRoot, model: config.model })
13590
+ : new ClaudeCodeCliProvider({ model: config.model });
12044
13591
  break;
12045
13592
  case 'codex':
12046
- provider = new CodexCliProvider({ model: config.model });
13593
+ provider = codexOAuth
13594
+ ? new CodexOAuthProvider({ projectRoot, model: config.model })
13595
+ : new CodexCliProvider({ model: config.model });
12047
13596
  break;
12048
13597
  case 'custom-openai':
12049
13598
  default:
@@ -12053,10 +13602,11 @@ async function testProviderConfig(projectRoot, id, overrides) {
12053
13602
  const available = await provider.available().catch(() => false);
12054
13603
  if (!available) {
12055
13604
  const cli = id === 'claude-code' ? 'claude' : 'codex';
13605
+ const signIn = id === 'claude-code' ? 'Sign in with Claude' : 'Sign in with ChatGPT';
12056
13606
  return {
12057
13607
  ok: false,
12058
13608
  message: isSubscriptionCli
12059
- ? `${label} is not ready. Install the \`${cli}\` CLI and run \`${cli} ${id === 'claude-code' ? '/login' : 'login'}\` with your subscription, then test again.`
13609
+ ? `${label} is not ready. Click "${signIn}" to sign in with your subscription, or install the \`${cli}\` CLI and run \`${cli} ${id === 'claude-code' ? '/login' : 'login'}\`, then test again.`
12060
13610
  : `${label} is not configured or reachable${details}. Check API key, base URL, and local service state.`,
12061
13611
  };
12062
13612
  }
@@ -12139,10 +13689,16 @@ function providerSettingsLabel(id) {
12139
13689
  }
12140
13690
  }
12141
13691
  function providerConfigDetails(id, config) {
13692
+ // Subscription providers (Claude Code / Codex) authenticate via OAuth/CLI, not an
13693
+ // API key — omit the key clause so we don't report a misleading "API key missing".
13694
+ const isSubscription = id === 'claude-code' || id === 'codex';
13695
+ const authNote = isSubscription
13696
+ ? ''
13697
+ : id === 'ollama' ? 'local endpoint' : config.apiKey ? 'API key present' : 'API key missing';
12142
13698
  const parts = [
12143
13699
  config.model ? `model ${config.model}` : '',
12144
13700
  config.baseUrl ? `base URL ${config.baseUrl}` : '',
12145
- id === 'ollama' ? 'local endpoint' : config.apiKey ? 'API key present' : 'API key missing',
13701
+ authNote,
12146
13702
  ].filter(Boolean);
12147
13703
  return parts.length ? ` (${parts.join(', ')})` : '';
12148
13704
  }
@@ -12164,7 +13720,7 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
12164
13720
  try {
12165
13721
  recordRuntimeSchemaSnapshot(projectRoot, {
12166
13722
  source,
12167
- tables: schemaContext.slice(0, 80).map((table) => ({
13723
+ tables: schemaContext.map((table) => ({
12168
13724
  relation: table.relation,
12169
13725
  schema: table.schema,
12170
13726
  name: table.name,
@@ -12318,7 +13874,9 @@ function buildNotebookResearchReviewChecklist(run) {
12318
13874
  const questionReady = Boolean(notebookResearchString(run.question));
12319
13875
  const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
12320
13876
  const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
13877
+ const hasDqlArtifact = Boolean(normalizeDqlArtifactReference(run.dqlArtifact)?.source);
12321
13878
  const hasSql = hasReviewedSql || hasGeneratedSql;
13879
+ const hasPromotableSource = hasReviewedSql || hasDqlArtifact;
12322
13880
  const preview = notebookResearchPreviewInfo(run.resultPreview);
12323
13881
  const previewReady = run.status === 'ready' && preview.hasPreview;
12324
13882
  const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
@@ -12337,8 +13895,8 @@ function buildNotebookResearchReviewChecklist(run) {
12337
13895
  const warnings = [];
12338
13896
  if (!questionReady)
12339
13897
  blockers.push('Add a business question before review.');
12340
- if (!hasSql && !reuseRecommended)
12341
- blockers.push('Add reviewed SQL or generate SQL before promotion.');
13898
+ if (!hasSql && !hasDqlArtifact && !reuseRecommended)
13899
+ blockers.push('Add reviewed SQL, generate SQL, or attach a DQL artifact before promotion.');
12342
13900
  if (run.status === 'error')
12343
13901
  blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
12344
13902
  if (!previewReady && run.status !== 'error' && !reuseRecommended)
@@ -12364,15 +13922,17 @@ function buildNotebookResearchReviewChecklist(run) {
12364
13922
  },
12365
13923
  {
12366
13924
  id: 'sql',
12367
- label: 'Reviewed SQL',
12368
- status: hasReviewedSql || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
13925
+ label: hasDqlArtifact && !hasReviewedSql ? 'DQL artifact' : 'Reviewed SQL',
13926
+ status: hasReviewedSql || hasDqlArtifact || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
12369
13927
  detail: reuseRecommended && !hasSql
12370
13928
  ? 'Existing certified DQL should be reused; no new SQL is required.'
12371
- : hasReviewedSql
12372
- ? 'Reviewer SQL is saved.'
12373
- : hasGeneratedSql
12374
- ? 'Generated SQL exists; review it before promotion.'
12375
- : 'No SQL is available for preview or DQL generation.',
13929
+ : hasDqlArtifact && !hasReviewedSql
13930
+ ? 'Generated DQL artifact is saved for reviewer promotion.'
13931
+ : hasReviewedSql
13932
+ ? 'Reviewer SQL is saved.'
13933
+ : hasGeneratedSql
13934
+ ? 'Generated SQL exists; review it before promotion.'
13935
+ : 'No SQL is available for preview or DQL generation.',
12376
13936
  },
12377
13937
  {
12378
13938
  id: 'preview',
@@ -12409,33 +13969,35 @@ function buildNotebookResearchReviewChecklist(run) {
12409
13969
  {
12410
13970
  id: 'parameters',
12411
13971
  label: 'Parameter review',
12412
- status: reuseRecommended && !hasSql ? 'passed' : !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
13972
+ status: reuseRecommended && !hasSql ? 'passed' : !hasSql && !hasDqlArtifact ? 'pending' : hasDqlArtifact && !hasSql ? 'passed' : dynamicParameters.length > 0 ? 'passed' : 'warning',
12413
13973
  detail: reuseRecommended && !hasSql
12414
13974
  ? 'Use the certified block parameters; no new SQL parameterization is required.'
12415
- : !hasSql
12416
- ? 'Add SQL before checking runtime parameters.'
12417
- : dynamicParameters.length > 0
12418
- ? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
12419
- : staticParameters.length > 0
12420
- ? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
12421
- : 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
13975
+ : !hasSql && hasDqlArtifact
13976
+ ? 'Review parameters declared in the DQL artifact.'
13977
+ : !hasSql
13978
+ ? 'Add SQL or a DQL artifact before checking runtime parameters.'
13979
+ : dynamicParameters.length > 0
13980
+ ? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
13981
+ : staticParameters.length > 0
13982
+ ? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
13983
+ : 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
12422
13984
  },
12423
13985
  {
12424
13986
  id: 'dql_draft',
12425
13987
  label: 'DQL draft',
12426
- status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasReviewedSql && evidenceReady ? 'pending' : 'blocked',
13988
+ status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasPromotableSource && evidenceReady ? 'pending' : 'blocked',
12427
13989
  detail: reuseRecommended && !hasDraft
12428
13990
  ? 'No new draft is required because an existing DQL block should be reused.'
12429
13991
  : hasDraft
12430
13992
  ? `Draft saved at ${run.draftBlockPath}.`
12431
- : !hasReviewedSql
12432
- ? 'Save reviewed SQL before DQL draft creation.'
13993
+ : !hasPromotableSource
13994
+ ? 'Save reviewed SQL or review the generated DQL artifact before DQL draft creation.'
12433
13995
  : evidenceReady
12434
- ? 'Create a draft after SQL review.'
13996
+ ? (hasDqlArtifact && !hasReviewedSql ? 'Create a draft after DQL artifact review.' : 'Create a draft after SQL review.')
12435
13997
  : 'Save metadata evidence before creating a reusable DQL draft.',
12436
13998
  },
12437
13999
  ];
12438
- const readyForDqlDraft = questionReady && hasReviewedSql && evidenceReady && run.status !== 'error';
14000
+ const readyForDqlDraft = questionReady && hasPromotableSource && evidenceReady && run.status !== 'error';
12439
14001
  const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
12440
14002
  return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
12441
14003
  }
@@ -12443,6 +14005,8 @@ function buildNotebookResearchPlan(input) {
12443
14005
  const reviewedSql = notebookResearchString(input.reviewedSql);
12444
14006
  const generatedSql = notebookResearchString(input.generatedSql);
12445
14007
  const sql = reviewedSql ?? generatedSql;
14008
+ const dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? input.run.dqlArtifact;
14009
+ const hasDqlArtifact = Boolean(dqlArtifact?.source);
12446
14010
  const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
12447
14011
  const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
12448
14012
  ? input.evidence
@@ -12481,6 +14045,7 @@ function buildNotebookResearchPlan(input) {
12481
14045
  promotion: {
12482
14046
  path: notebookResearchPlanPromotionPath({
12483
14047
  hasSql: Boolean(sql),
14048
+ hasDqlArtifact,
12484
14049
  hasEvidence,
12485
14050
  hasPreview,
12486
14051
  previewError: input.previewError,
@@ -12491,6 +14056,7 @@ function buildNotebookResearchPlan(input) {
12491
14056
  },
12492
14057
  reviewFocus: notebookResearchPlanFocus(input.run.intent, {
12493
14058
  hasSql: Boolean(sql),
14059
+ hasDqlArtifact,
12494
14060
  hasEvidence,
12495
14061
  hasPreview,
12496
14062
  dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
@@ -12503,7 +14069,7 @@ function buildNotebookResearchPlan(input) {
12503
14069
  function notebookResearchPlanPromotionPath(input) {
12504
14070
  if (input.promotionAction === 'reuse_existing')
12505
14071
  return 'reuse_existing';
12506
- if (!input.hasSql)
14072
+ if (!input.hasSql && !input.hasDqlArtifact)
12507
14073
  return 'needs_sql';
12508
14074
  if (!input.hasEvidence)
12509
14075
  return 'review_context';
@@ -12515,8 +14081,8 @@ function notebookResearchPlanPromotionPath(input) {
12515
14081
  }
12516
14082
  function notebookResearchPlanFocus(intent, input) {
12517
14083
  const focus = new Set();
12518
- if (!input.hasSql)
12519
- focus.add('Add reviewed SQL or configure AI SQL generation.');
14084
+ if (!input.hasSql && !input.hasDqlArtifact)
14085
+ focus.add('Add reviewed SQL, a DQL artifact, or configure AI generation.');
12520
14086
  if (!input.hasEvidence)
12521
14087
  focus.add('Save metadata context evidence before DQL promotion.');
12522
14088
  if (!input.hasPreview)
@@ -12987,7 +14553,7 @@ function dedupeAgentSchemaColumns(columns) {
12987
14553
  }
12988
14554
  return Array.from(byName.values());
12989
14555
  }
12990
- export function buildAgentSchemaContext(question, rows) {
14556
+ export function buildAgentSchemaContext(question, rows, options = {}) {
12991
14557
  const byRelation = new Map();
12992
14558
  for (const row of rows) {
12993
14559
  if (!row || typeof row !== 'object')
@@ -13016,16 +14582,82 @@ export function buildAgentSchemaContext(question, rows) {
13016
14582
  }
13017
14583
  const tokens = agentSchemaTokens(question);
13018
14584
  const shouldProbeValues = extractAgentValueSearchTerms(question).length > 0;
14585
+ const limit = Math.max(1, Math.trunc(options.limit ?? 12));
13019
14586
  return Array.from(byRelation.values())
13020
14587
  .map((table) => ({
13021
14588
  table,
13022
14589
  score: scoreAgentSchemaTable(table, tokens) + (shouldProbeValues ? scoreAgentValueProbeTable(table) : 0),
13023
14590
  }))
13024
- .filter((entry) => entry.score > 0)
14591
+ .filter((entry) => options.includeUnscored || entry.score > 0)
13025
14592
  .sort((a, b) => b.score - a.score || a.table.relation.localeCompare(b.table.relation))
13026
- .slice(0, 12)
14593
+ .slice(0, limit)
13027
14594
  .map((entry) => entry.table);
13028
14595
  }
14596
+ function mergeAgentSchemaContexts(...contexts) {
14597
+ const byRelation = new Map();
14598
+ for (const table of contexts.flat()) {
14599
+ const key = table.relation.toLowerCase();
14600
+ const existing = byRelation.get(key);
14601
+ if (!existing) {
14602
+ byRelation.set(key, {
14603
+ ...table,
14604
+ columns: dedupeAgentSchemaColumns(table.columns).slice(0, 80),
14605
+ });
14606
+ continue;
14607
+ }
14608
+ byRelation.set(key, {
14609
+ ...existing,
14610
+ description: existing.description ?? table.description,
14611
+ source: existing.source === table.source ? existing.source : 'catalog and runtime schema',
14612
+ columns: dedupeAgentSchemaColumns([...existing.columns, ...table.columns]).slice(0, 80),
14613
+ });
14614
+ }
14615
+ return Array.from(byRelation.values()).slice(0, 16);
14616
+ }
14617
+ function mergeAgentSchemaSampleValues(snapshot, enriched) {
14618
+ const samples = new Map();
14619
+ for (const table of enriched) {
14620
+ for (const column of table.columns) {
14621
+ if (!column.sampleValues?.length)
14622
+ continue;
14623
+ samples.set(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`, column.sampleValues);
14624
+ }
14625
+ }
14626
+ if (samples.size === 0)
14627
+ return snapshot;
14628
+ return snapshot.map((table) => ({
14629
+ ...table,
14630
+ columns: table.columns.map((column) => {
14631
+ const sampleValues = samples.get(`${table.relation.toLowerCase()}\u0000${column.name.toLowerCase()}`);
14632
+ return sampleValues?.length
14633
+ ? { ...column, sampleValues: uniqueStrings([...(column.sampleValues ?? []), ...sampleValues]).slice(0, 8) }
14634
+ : column;
14635
+ }),
14636
+ }));
14637
+ }
14638
+ // Common English plurals that are NOT entity nouns, so a multi-entity heuristic
14639
+ // doesn't count them. Domain-agnostic (no project-specific vocabulary).
14640
+ const NON_ENTITY_PLURALS = new Set([
14641
+ 'details', 'values', 'results', 'rows', 'numbers', 'records', 'items', 'things',
14642
+ 'totals', 'amounts', 'metrics', 'columns', 'fields', 'names', 'ids',
14643
+ ]);
14644
+ /**
14645
+ * Decide whether to run the (advisory, best-effort) runtime value-scan + schema
14646
+ * enrichment for a question. Generic multi-entity / detail intent — no hard-coded
14647
+ * project vocabulary — so it fires for ANY repo's join-shaped questions, not just
14648
+ * jaffle. Over-triggering only costs a few extra bounded probes (all in try/catch).
14649
+ */
14650
+ function shouldAugmentAgentRuntimeSchema(question) {
14651
+ const lower = question.toLowerCase();
14652
+ const wantsMetric = /\brevenu|\bsales\b|\bamount\b|\bspend\b|\bcost\b|\bcount\b|\btotal\b|\bsum\b|\bavg\b|\baverage\b|\bvalue\b|\bprice\b/.test(lower);
14653
+ const wantsDetail = /\bdetails?\b|\bcomplete\b|\bbreakdown\b|\ball\s+values?\b|\beach\b|\bevery\b/.test(lower);
14654
+ const referencesPriorRows = /\b(?:above|previous|prior|these|those|same)\s+\w+/.test(lower);
14655
+ const explicitJoin = /\bjoin(?:ed|ing)?\b|\bcombine[ds]?\b|\bcross[- ]?reference\b|\balong\s+with\b|\btogether\s+with\b/.test(lower);
14656
+ // Multi-entity intent: a linking word + >= 2 distinct plural content nouns.
14657
+ const nouns = new Set((lower.match(/\b[a-z]{4,}s\b/g) ?? []).filter((word) => !NON_ENTITY_PLURALS.has(word)));
14658
+ const multiEntity = /\b(?:and|with|plus|per|for\s+each|by)\b/.test(lower) && nouns.size >= 2;
14659
+ return explicitJoin || referencesPriorRows || (multiEntity && (wantsMetric || wantsDetail));
14660
+ }
13029
14661
  async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
13030
14662
  const searchTerms = extractAgentValueSearchTerms(question);
13031
14663
  if (schemaContext.length === 0 || searchTerms.length === 0)
@@ -13077,8 +14709,57 @@ function scoreAgentSchemaTable(table, tokens) {
13077
14709
  score += 3;
13078
14710
  }
13079
14711
  }
13080
- if (/(customer|order|revenue|product|location|date|month)/i.test(table.name))
14712
+ score += scoreCompositeAgentSchemaTable(table, tokens);
14713
+ // Small tiebreaker toward modeled analytics tables (fct_/dim_/fact_/mart_ …),
14714
+ // a warehouse-modeling convention that generalizes across repos — not a fixed
14715
+ // list of jaffle entity names.
14716
+ if (/^(?:fct|fact|dim|dimension|mart|rpt|report|agg|f|d)[_-]/i.test(table.name)
14717
+ || /(?:^|[_-])(?:fact|dim|mart|summary|report)(?:$|[_-])/i.test(table.name)) {
13081
14718
  score += 1;
14719
+ }
14720
+ return score;
14721
+ }
14722
+ const COMPOSITE_METRIC_TOKENS = new Set([
14723
+ 'revenue', 'sales', 'amount', 'spend', 'price', 'cost', 'total', 'sum',
14724
+ 'count', 'quantity', 'qty', 'value', 'subtotal', 'order', 'orders',
14725
+ ]);
14726
+ const COMPOSITE_STOPWORD_TOKENS = new Set([
14727
+ 'the', 'and', 'with', 'for', 'each', 'per', 'all', 'top', 'show', 'list',
14728
+ 'give', 'get', 'find', 'plus', 'along', 'together', 'combined', 'combine',
14729
+ 'detail', 'details', 'complete', 'breakdown', 'values', 'name', 'names',
14730
+ ...COMPOSITE_METRIC_TOKENS,
14731
+ ]);
14732
+ /**
14733
+ * Boost the fact/junction table for a multi-entity ("composite") question. Fully
14734
+ * domain-agnostic: the ask is detected from >= 2 distinct entity tokens + a metric,
14735
+ * and the winning table is identified STRUCTURALLY (>= 2 foreign-key-style id columns
14736
+ * plus a numeric measure column) rather than by hard-coded jaffle table names — so it
14737
+ * ranks the right join table for any repo, while jaffle's fct_orders still qualifies.
14738
+ */
14739
+ function scoreCompositeAgentSchemaTable(table, tokens) {
14740
+ const entityTokens = [...tokens].filter((token) => token.length >= 3 && !COMPOSITE_STOPWORD_TOKENS.has(token));
14741
+ const hasMetric = [...tokens].some((token) => COMPOSITE_METRIC_TOKENS.has(token));
14742
+ const hasCompositeAsk = entityTokens.length >= 2 && hasMetric;
14743
+ const hasPriorRowsAsk = ['above', 'previous', 'prior', 'these', 'those', 'same'].some((token) => tokens.has(token))
14744
+ && entityTokens.length >= 1;
14745
+ if (!hasCompositeAsk && !hasPriorRowsAsk)
14746
+ return 0;
14747
+ const columns = table.columns.map((column) => column.name.toLowerCase());
14748
+ let score = 0;
14749
+ // Structural fact/junction-table signal: multiple FK-style id columns + a measure.
14750
+ const idColumnCount = columns.filter((name) => /(?:^id$|_id$|_key$|_sk$|_uuid$|_fk$)/.test(name)).length;
14751
+ const hasMeasureColumn = columns.some((name) => /revenue|sales|amount|price|spend|cost|subtotal|total|quantity|\bqty\b|count|value/.test(name));
14752
+ if (idColumnCount >= 2 && hasMeasureColumn)
14753
+ score += 18;
14754
+ else if (idColumnCount >= 2)
14755
+ score += 8;
14756
+ if (hasMeasureColumn)
14757
+ score += 8;
14758
+ // Reward columns whose names overlap the question's actual entity tokens (any domain).
14759
+ for (const token of entityTokens) {
14760
+ if (columns.some((name) => name.includes(token)))
14761
+ score += 4;
14762
+ }
13082
14763
  return score;
13083
14764
  }
13084
14765
  function scoreAgentValueProbeTable(table) {