@duckcodeailabs/dql-cli 1.6.20 → 1.6.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/apps-api.d.ts +216 -2
  2. package/dist/apps-api.d.ts.map +1 -1
  3. package/dist/apps-api.js +1767 -46
  4. package/dist/apps-api.js.map +1 -1
  5. package/dist/args.d.ts +10 -0
  6. package/dist/args.d.ts.map +1 -1
  7. package/dist/args.js +19 -0
  8. package/dist/args.js.map +1 -1
  9. package/dist/assets/dql-notebook/assets/{codemirror-DJYUkPr1.js → codemirror-Co0dCEDr.js} +1 -1
  10. package/dist/assets/dql-notebook/assets/index-D9Owd4EN.css +1 -0
  11. package/dist/assets/dql-notebook/assets/index-nDZHVgfg.js +5636 -0
  12. package/dist/assets/dql-notebook/assets/{react-CRB3T2We.js → react-BRY8hXab.js} +2 -2
  13. package/dist/assets/dql-notebook/index.html +4 -4
  14. package/dist/commands/agent.d.ts +1 -1
  15. package/dist/commands/agent.d.ts.map +1 -1
  16. package/dist/commands/agent.js +50 -10
  17. package/dist/commands/agent.js.map +1 -1
  18. package/dist/commands/app.d.ts +4 -4
  19. package/dist/commands/app.d.ts.map +1 -1
  20. package/dist/commands/app.js +22 -19
  21. package/dist/commands/app.js.map +1 -1
  22. package/dist/commands/certify.d.ts.map +1 -1
  23. package/dist/commands/certify.js +70 -25
  24. package/dist/commands/certify.js.map +1 -1
  25. package/dist/commands/diff.d.ts.map +1 -1
  26. package/dist/commands/diff.js +108 -5
  27. package/dist/commands/diff.js.map +1 -1
  28. package/dist/commands/doctor.d.ts +1 -1
  29. package/dist/commands/doctor.d.ts.map +1 -1
  30. package/dist/commands/doctor.js +68 -7
  31. package/dist/commands/doctor.js.map +1 -1
  32. package/dist/commands/eval.d.ts +121 -0
  33. package/dist/commands/eval.d.ts.map +1 -0
  34. package/dist/commands/eval.js +350 -0
  35. package/dist/commands/eval.js.map +1 -0
  36. package/dist/commands/init.d.ts.map +1 -1
  37. package/dist/commands/init.js +50 -10
  38. package/dist/commands/init.js.map +1 -1
  39. package/dist/commands/notebook.d.ts +15 -0
  40. package/dist/commands/notebook.d.ts.map +1 -1
  41. package/dist/commands/notebook.js +32 -11
  42. package/dist/commands/notebook.js.map +1 -1
  43. package/dist/commands/propose.d.ts +23 -0
  44. package/dist/commands/propose.d.ts.map +1 -0
  45. package/dist/commands/propose.js +184 -0
  46. package/dist/commands/propose.js.map +1 -0
  47. package/dist/index.js +64 -11
  48. package/dist/index.js.map +1 -1
  49. package/dist/llm/analytics-tools.d.ts.map +1 -1
  50. package/dist/llm/analytics-tools.js +9 -4
  51. package/dist/llm/analytics-tools.js.map +1 -1
  52. package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
  53. package/dist/llm/providers/dql-agent-provider.js +21 -32
  54. package/dist/llm/providers/dql-agent-provider.js.map +1 -1
  55. package/dist/llm/providers/native-sdk-provider.d.ts.map +1 -1
  56. package/dist/llm/providers/native-sdk-provider.js +5 -2
  57. package/dist/llm/providers/native-sdk-provider.js.map +1 -1
  58. package/dist/local-runtime.d.ts +132 -4
  59. package/dist/local-runtime.d.ts.map +1 -1
  60. package/dist/local-runtime.js +2120 -46
  61. package/dist/local-runtime.js.map +1 -1
  62. package/dist/package.json +10 -10
  63. package/dist/propose-enrich.d.ts +13 -0
  64. package/dist/propose-enrich.d.ts.map +1 -0
  65. package/dist/propose-enrich.js +34 -0
  66. package/dist/propose-enrich.js.map +1 -0
  67. package/dist/settings/provider-settings.js +2 -2
  68. package/dist/settings/provider-settings.js.map +1 -1
  69. package/package.json +11 -11
  70. package/dist/assets/dql-notebook/assets/index-Dgqvyn5s.js +0 -3886
  71. package/dist/assets/dql-notebook/assets/index-RaDW1A5g.css +0 -1
@@ -7,21 +7,23 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
7
7
  import Anthropic from '@anthropic-ai/sdk';
8
8
  import OpenAI from 'openai';
9
9
  import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
10
- import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
10
+ import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
11
11
  import { load as loadYaml } from 'js-yaml';
12
12
  import { listBlockTemplates } from './block-templates.js';
13
13
  import { getRunner as getLLMRunner } from './llm/index.js';
14
14
  import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
15
- import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordQueryRun, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
16
- import { handleAppsApi, recommendVisualization } from './apps-api.js';
15
+ import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, } from '@duckcodeailabs/dql-agent';
16
+ import { gatherProposeEnrichment } from './propose-enrich.js';
17
+ import { createAppAiBuildSession, handleAppsApi, recommendVisualization } from './apps-api.js';
17
18
  import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
18
19
  import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
19
20
  import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
20
- import { Certifier, ENTERPRISE_RULES } from '@duckcodeailabs/dql-governance';
21
+ import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
21
22
  import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
22
23
  import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
23
24
  import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
24
25
  const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
26
+ const NOTEBOOK_FAVICON_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="7" fill="#6d5dfc"/><path d="M9 9h14v14H9z" fill="none" stroke="#fff" stroke-width="2"/><path d="M13 13h6M13 17h6M13 21h4" stroke="#fff" stroke-width="2" stroke-linecap="round"/></svg>';
25
27
  export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
26
28
  const configured = projectConfig.semanticLayer;
27
29
  const dbtProjectDir = projectConfig.dbt?.projectDir;
@@ -51,6 +53,82 @@ function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
51
53
  }
52
54
  return false;
53
55
  }
56
+ const AGENT_RUN_REQUESTED_MODES = new Set(['auto', 'ask', 'research', 'sql', 'block', 'app']);
57
+ const AGENT_RUN_SELECTED_OBJECT_KINDS = new Set([
58
+ 'notebook',
59
+ 'cell',
60
+ 'block',
61
+ 'app',
62
+ 'dashboard',
63
+ 'research',
64
+ 'workspace',
65
+ ]);
66
+ function agentRunRecord(value) {
67
+ return value && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
68
+ }
69
+ function agentRunString(value) {
70
+ return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
71
+ }
72
+ function parseAgentRunRequestedMode(value) {
73
+ return typeof value === 'string' && AGENT_RUN_REQUESTED_MODES.has(value)
74
+ ? value
75
+ : undefined;
76
+ }
77
+ function parseAgentRunSelectedObject(value) {
78
+ const record = agentRunRecord(value);
79
+ if (!record)
80
+ return undefined;
81
+ const kind = agentRunString(record.kind);
82
+ if (!kind || !AGENT_RUN_SELECTED_OBJECT_KINDS.has(kind))
83
+ return undefined;
84
+ return {
85
+ kind: kind,
86
+ id: agentRunString(record.id),
87
+ title: agentRunString(record.title),
88
+ path: agentRunString(record.path),
89
+ };
90
+ }
91
+ function parseAgentRunHistory(value) {
92
+ if (!Array.isArray(value))
93
+ return undefined;
94
+ const history = value.flatMap((item) => {
95
+ const record = agentRunRecord(item);
96
+ if (!record)
97
+ return [];
98
+ const role = record.role === 'user' || record.role === 'assistant' ? record.role : undefined;
99
+ const text = agentRunString(record.text) ?? agentRunString(record.content);
100
+ return role && text ? [{ role, text }] : [];
101
+ });
102
+ return history.length > 0 ? history.slice(-20) : undefined;
103
+ }
104
+ function parseAgentRunRequestBody(body) {
105
+ const record = agentRunRecord(body);
106
+ if (!record)
107
+ return { error: 'Invalid JSON body.' };
108
+ const question = agentRunString(record.question) ?? agentRunString(record.prompt) ?? agentRunString(record.message);
109
+ if (!question)
110
+ return { error: 'question is required.' };
111
+ const selectedObject = parseAgentRunSelectedObject(record.selectedObject);
112
+ const workspaceContext = agentRunRecord(record.workspaceContext) ?? agentRunRecord(record.context);
113
+ const signals = agentRunRecord(record.signals);
114
+ const requestedMode = parseAgentRunRequestedMode(record.requestedMode) ?? parseAgentRunRequestedMode(record.mode);
115
+ const audience = record.audience === 'stakeholder' || record.audience === 'analyst'
116
+ ? record.audience
117
+ : undefined;
118
+ return {
119
+ request: {
120
+ question,
121
+ requestedMode,
122
+ audience,
123
+ intent: agentRunString(record.intent),
124
+ signals: signals,
125
+ selectedObject,
126
+ workspaceContext,
127
+ history: parseAgentRunHistory(record.history),
128
+ runId: agentRunString(record.runId),
129
+ },
130
+ };
131
+ }
54
132
  export async function startLocalServer(opts) {
55
133
  const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
56
134
  const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
@@ -126,6 +204,582 @@ export async function startLocalServer(opts) {
126
204
  const result = await executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
127
205
  return normalizeQueryResult(result, semantic.semanticRefs);
128
206
  };
207
+ const runBlockReflectionProbe = async ({ sql, invariants }) => {
208
+ const activeConnection = requireActiveConnection();
209
+ const prepared = prepareLocalExecution(sql, activeConnection, projectRoot, projectConfig);
210
+ const probeSql = `SELECT * FROM (${stripSqlTerminator(prepared.sql)}) _dql_probe LIMIT 2000`;
211
+ const probeResult = await executor.executeQuery(probeSql, [], runtimeVariables({}), prepared.connection);
212
+ const rows = (Array.isArray(probeResult?.rows) ? probeResult.rows : []);
213
+ const rawColumns = Array.isArray(probeResult?.columns)
214
+ ? probeResult.columns
215
+ : [];
216
+ const actualColumns = rawColumns.length > 0
217
+ ? rawColumns.map((c) => (typeof c === 'string' ? c : c?.name ?? String(c)))
218
+ : (rows[0] ? Object.keys(rows[0]) : []);
219
+ const invariantResults = evaluateInvariants(invariants, { columns: actualColumns, rows });
220
+ const passed = invariantResults.filter((r) => r.passed && !r.uncheckable).length;
221
+ const failed = invariantResults.filter((r) => !r.passed && !r.uncheckable).length;
222
+ return {
223
+ actualColumns,
224
+ invariantResults,
225
+ tests: invariants.length > 0
226
+ ? { passed, failed, assertionCount: invariantResults.length }
227
+ : undefined,
228
+ };
229
+ };
230
+ const agentRunWorkspaceValue = (request, key) => {
231
+ const workspace = request.workspaceContext ?? {};
232
+ const nested = agentRunRecord(workspace.context);
233
+ return agentRunString(workspace[key]) ?? (nested ? agentRunString(nested[key]) : undefined);
234
+ };
235
+ const agentRunNotebookPath = (request, runId) => (agentRunWorkspaceValue(request, 'notebookPath')
236
+ ?? (request.selectedObject?.kind === 'notebook' || request.selectedObject?.kind === 'cell' ? request.selectedObject.path : undefined)
237
+ ?? `notebooks/agent-research/${runId}.dqlnb`);
238
+ const agentRunResearchIntent = (request) => {
239
+ switch (request.intent) {
240
+ case 'diagnose_change':
241
+ return 'diagnose_change';
242
+ case 'driver_breakdown':
243
+ return 'driver_breakdown';
244
+ case 'segment_compare':
245
+ return 'segment_compare';
246
+ case 'entity_drilldown':
247
+ return 'entity_drilldown';
248
+ case 'anomaly_investigation':
249
+ return 'anomaly_investigation';
250
+ case 'trust_gap_review':
251
+ return 'trust_gap_review';
252
+ default:
253
+ if (/\b(driver|why|cause|contributor|breakdown)\b/i.test(request.question))
254
+ return 'driver_breakdown';
255
+ if (/\b(anomaly|spike|drop|outlier)\b/i.test(request.question))
256
+ return 'anomaly_investigation';
257
+ if (/\b(compare|segment|cohort)\b/i.test(request.question))
258
+ return 'segment_compare';
259
+ return 'ad_hoc_analysis';
260
+ }
261
+ };
262
+ const agentRunSourceCell = (request) => {
263
+ const sourceCellId = agentRunWorkspaceValue(request, 'sourceCellId') ?? request.selectedObject?.id;
264
+ if (!sourceCellId)
265
+ return undefined;
266
+ return {
267
+ id: sourceCellId,
268
+ sourceCellId,
269
+ name: agentRunWorkspaceValue(request, 'sourceCellName') ?? request.selectedObject?.title,
270
+ sourceCellName: agentRunWorkspaceValue(request, 'sourceCellName') ?? request.selectedObject?.title,
271
+ type: agentRunWorkspaceValue(request, 'sourceCellType'),
272
+ sql: agentRunWorkspaceValue(request, 'cellSql'),
273
+ fingerprint: agentRunWorkspaceValue(request, 'sourceCellFingerprint'),
274
+ };
275
+ };
276
+ const agentRunTitle = (question, fallback) => {
277
+ const cleaned = question.replace(/\s+/g, ' ').trim();
278
+ if (!cleaned)
279
+ return fallback;
280
+ return cleaned.length > 90 ? `${cleaned.slice(0, 87)}...` : cleaned;
281
+ };
282
+ const parseAgentRunSelectedBlockIds = (request) => {
283
+ const workspace = request.workspaceContext ?? {};
284
+ const value = workspace.selectedBlockIds ?? workspace.blockIds;
285
+ if (!Array.isArray(value))
286
+ return [];
287
+ return Array.from(new Set(value.flatMap((item) => {
288
+ const id = agentRunString(item);
289
+ return id ? [id] : [];
290
+ })));
291
+ };
292
+ const formatAgentRunInfrastructureError = (error, scope) => {
293
+ const message = error instanceof Error ? error.message : String(error);
294
+ if (/Could not locate the bindings file/i.test(message) || /better[-_]sqlite3/i.test(message)) {
295
+ return `${scope} is unavailable because the local SQLite native bindings are not installed for this Node.js runtime.`;
296
+ }
297
+ return message;
298
+ };
299
+ const formatNotebookResearchStorageError = (error) => (formatAgentRunInfrastructureError(error, 'Notebook research storage'));
300
+ const buildAgentPromptArtifact = async (request, target, repair) => {
301
+ try {
302
+ await reindexProject(projectRoot, { kgPath: defaultKgPath(projectRoot) });
303
+ }
304
+ catch {
305
+ // Best-effort: buildFromPrompt can still use any existing KG/cache state.
306
+ }
307
+ const skills = loadSkills(projectRoot).skills;
308
+ // On a repair re-run, target the prior failure and (for blocks) revise in place.
309
+ const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
310
+ const mode = target === 'block' && (isRepair || agentRunWorkspaceValue(request, 'mode') === 'edit')
311
+ ? 'edit'
312
+ : 'create';
313
+ const prompt = isRepair
314
+ ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}`
315
+ : request.question;
316
+ return buildFromPrompt({
317
+ projectRoot,
318
+ prompt,
319
+ context: {
320
+ cellSql: agentRunWorkspaceValue(request, 'cellSql'),
321
+ selection: agentRunWorkspaceValue(request, 'selection'),
322
+ },
323
+ target,
324
+ mode,
325
+ blockPath: target === 'block'
326
+ ? agentRunWorkspaceValue(request, 'blockPath') ?? request.selectedObject?.path
327
+ : undefined,
328
+ owner: agentRunWorkspaceValue(request, 'owner'),
329
+ domain: target === 'block' ? agentRunWorkspaceValue(request, 'domain') : undefined,
330
+ userId: agentRunWorkspaceValue(request, 'userId'),
331
+ skills,
332
+ dbtManifestPath: resolveDbtManifestPath(projectRoot, projectConfig),
333
+ executionProbe: target === 'block' ? runBlockReflectionProbe : undefined,
334
+ });
335
+ };
336
+ const agentRunEvaluation = (id, label, passed, severity, message, evidence) => ({ id, label, passed, severity, message, evidence });
337
+ const agentRunArtifact = (kind, title, payload, ref, trustState = 'review_required') => ({
338
+ id: `${kind}:${Date.now()}`,
339
+ kind,
340
+ title,
341
+ trustState,
342
+ ref,
343
+ payload,
344
+ });
345
+ const coerceNarrateResultData = (value) => {
346
+ const record = agentRunRecord(value);
347
+ if (!record)
348
+ return undefined;
349
+ const columns = Array.isArray(record.columns)
350
+ ? record.columns.map((c) => (typeof c === 'string' ? c : agentRunRecord(c)?.name ?? String(c)))
351
+ : [];
352
+ const rows = Array.isArray(record.rows) ? record.rows.filter((r) => Boolean(agentRunRecord(r))) : [];
353
+ if (rows.length === 0)
354
+ return undefined;
355
+ return { columns: columns.length > 0 ? columns : Object.keys(rows[0]), rows };
356
+ };
357
+ // Provider-backed narration for stakeholder stories. Reuses the same provider
358
+ // adapter as the planner; narrateResult always returns (deterministic fallback).
359
+ const narrateForAgentRun = async (input) => narrateResult(input, {
360
+ complete: async ({ system, user, signal }) => {
361
+ const provider = await createBlockStudioAssistProvider(projectRoot);
362
+ if (!provider)
363
+ throw new Error('No AI provider configured for narration.');
364
+ return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
365
+ },
366
+ });
367
+ async function runGovernedAgentAnswerForRun(request, repair) {
368
+ const resolvedProvider = resolveDefaultLLMProvider(projectRoot);
369
+ const runner = resolvedProvider ? getLLMRunner(resolvedProvider) : null;
370
+ if (!resolvedProvider || !runner) {
371
+ throw new Error('No AI provider is configured. Configure OpenAI, Gemini, Ollama, or a custom OpenAI-compatible endpoint in Settings.');
372
+ }
373
+ let governedAnswer;
374
+ let providerError;
375
+ const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
376
+ const contextEnvelope = {
377
+ mode: 'agent_run',
378
+ selectedObject: request.selectedObject,
379
+ workspaceContext: request.workspaceContext,
380
+ instruction: [
381
+ 'Route through the governed DQL answer loop.',
382
+ 'Prefer certified DQL blocks when they exactly cover the question.',
383
+ 'Generated SQL remains review-required and must use the bounded preview executor.',
384
+ 'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
385
+ ...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
386
+ ].join(' '),
387
+ };
388
+ const controller = new AbortController();
389
+ await runner.run({
390
+ provider: resolvedProvider,
391
+ messages: [
392
+ ...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
393
+ { role: 'user', content: isRepair ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}` : request.question },
394
+ ],
395
+ upstream: {
396
+ cellId: `agent-run:${request.selectedObject?.kind ?? 'workspace'}:${request.selectedObject?.id ?? request.runId ?? 'auto'}`,
397
+ sql: JSON.stringify(contextEnvelope, null, 2),
398
+ },
399
+ projectRoot,
400
+ executeCertifiedBlock: executeCertifiedBlockForAgent,
401
+ executeGeneratedSql: executeGeneratedSqlForAgent,
402
+ getSchemaContext: getSchemaContextForAgent,
403
+ }, (turn) => {
404
+ if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
405
+ governedAnswer = turn.output;
406
+ }
407
+ if (turn.kind === 'error') {
408
+ providerError = turn.message;
409
+ }
410
+ }, controller.signal);
411
+ if (!governedAnswer) {
412
+ throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
413
+ }
414
+ return governedAnswer;
415
+ }
416
+ const answerRunExecutor = async ({ request, routeDecision, attempt, repairHint }) => {
417
+ let governedAnswer;
418
+ try {
419
+ governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint });
420
+ // Surface the approved Hint-Graph corrections that shaped this answer so the
421
+ // UI can show an "applied learnings" chip (memoryContext is already on the answer).
422
+ if (!governedAnswer.appliedHints) {
423
+ governedAnswer.appliedHints = governedAnswer.contextPack?.appliedHints;
424
+ }
425
+ }
426
+ catch (error) {
427
+ const message = formatAgentRunInfrastructureError(error, 'AI answer provider');
428
+ return {
429
+ summary: message,
430
+ status: 'blocked',
431
+ trustState: 'blocked',
432
+ stopReason: 'blocked',
433
+ evaluations: [
434
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'),
435
+ agentRunEvaluation('ai-provider', 'AI provider', false, 'blocking', message, { originalErrorType: error instanceof Error ? error.name : typeof error }),
436
+ ],
437
+ nextActions: [
438
+ { id: 'retry-ask-after-provider', label: 'Retry after provider setup', route: 'generated_answer' },
439
+ { id: 'research-without-answer', label: 'Research with available metadata', route: 'research', artifactKind: 'research_run' },
440
+ ],
441
+ };
442
+ }
443
+ const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
444
+ const needsClarification = governedAnswer.kind === 'no_answer';
445
+ const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
446
+ const status = needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
447
+ const trustState = needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
448
+ const stopReason = needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
449
+ const nextActions = needsClarification
450
+ ? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
451
+ : [
452
+ ...(sql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
453
+ { id: 'research-gap', label: 'Research deeper', route: 'research' },
454
+ { id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
455
+ ];
456
+ return {
457
+ summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
458
+ answer: governedAnswer.answer ?? governedAnswer.text,
459
+ status,
460
+ trustState,
461
+ stopReason,
462
+ artifacts: needsClarification
463
+ ? []
464
+ : [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
465
+ evaluations: [
466
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'),
467
+ agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
468
+ ? 'The answer came from certified DQL context.'
469
+ : needsClarification
470
+ ? 'The answer loop needs more context before producing a governed answer.'
471
+ : 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
472
+ ...(governedAnswer.executionError ? [
473
+ agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
474
+ ] : []),
475
+ ],
476
+ nextActions,
477
+ };
478
+ };
479
+ const agentRunExecutors = {
480
+ certified_answer: answerRunExecutor,
481
+ generated_answer: answerRunExecutor,
482
+ research: async ({ runId, request, routeDecision, emit }) => {
483
+ const metrics = loadSemanticMetrics(projectRoot);
484
+ let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
485
+ const usedCertifiedOnly = blocks.length > 0;
486
+ if (blocks.length === 0)
487
+ blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
488
+ emit({
489
+ type: 'executor.started',
490
+ message: 'Building catalog-grounded research plan.',
491
+ route: 'research',
492
+ });
493
+ const plan = await planResearch({
494
+ question: request.question,
495
+ metrics,
496
+ blocks,
497
+ intent: request.intent,
498
+ isFollowUp: Boolean(request.history?.length),
499
+ history: request.history,
500
+ });
501
+ const needsClarification = Boolean(plan.followUp);
502
+ const notebookPath = agentRunNotebookPath(request, runId);
503
+ const researchIntent = agentRunResearchIntent(request);
504
+ let researchRun;
505
+ let researchWorkspaceError;
506
+ if (!needsClarification) {
507
+ try {
508
+ const storage = openNotebookResearchStorage();
509
+ try {
510
+ const sourceCell = agentRunSourceCell(request);
511
+ const sourceCellId = notebookResearchSourceCellId(sourceCell);
512
+ const sourceCellName = notebookResearchSourceCellName(sourceCell);
513
+ const sourceCellFingerprint = notebookResearchSourceCellFingerprint(sourceCell);
514
+ const created = storage.createRun({
515
+ notebookPath,
516
+ title: agentRunTitle(request.question, 'Agent research'),
517
+ question: request.question,
518
+ sourceCell,
519
+ sourceCellId,
520
+ sourceCellName,
521
+ sourceCellFingerprint,
522
+ intent: researchIntent,
523
+ domain: agentRunWorkspaceValue(request, 'domain'),
524
+ owner: agentRunWorkspaceValue(request, 'owner'),
525
+ context: {
526
+ surface: 'unified_agent_run',
527
+ agentRunId: runId,
528
+ routeDecision,
529
+ selectedObject: request.selectedObject,
530
+ workspaceContext: request.workspaceContext,
531
+ plan,
532
+ },
533
+ });
534
+ emit({
535
+ type: 'artifact.created',
536
+ message: 'Saved notebook research workspace record.',
537
+ route: 'research',
538
+ trustState: 'review_required',
539
+ payload: { researchRunId: created.id, notebookPath },
540
+ });
541
+ const executed = await runNotebookResearch(storage, created, {
542
+ domain: agentRunWorkspaceValue(request, 'domain'),
543
+ owner: agentRunWorkspaceValue(request, 'owner'),
544
+ sourceCellFingerprint,
545
+ question: request.question,
546
+ intent: researchIntent,
547
+ context: {
548
+ surface: 'unified_agent_run',
549
+ agentRunId: runId,
550
+ routeDecision,
551
+ selectedObject: request.selectedObject,
552
+ workspaceContext: request.workspaceContext,
553
+ plan,
554
+ },
555
+ });
556
+ researchRun = withNotebookResearchChecklist(executed);
557
+ }
558
+ finally {
559
+ storage.close();
560
+ }
561
+ }
562
+ catch (error) {
563
+ researchWorkspaceError = formatNotebookResearchStorageError(error);
564
+ }
565
+ }
566
+ const researchResultData = coerceNarrateResultData(researchRun?.resultPreview);
567
+ const narration = !needsClarification && researchResultData
568
+ ? await narrateForAgentRun({
569
+ question: request.question,
570
+ intent: request.intent,
571
+ result: researchResultData,
572
+ evidence: plan.sources,
573
+ reviewRequired: true,
574
+ })
575
+ : undefined;
576
+ const summary = needsClarification
577
+ ? 'Needs clarification before running deeper research.'
578
+ : narration?.summary
579
+ ?? (researchRun?.status === 'ready'
580
+ ? 'Saved a grounded research dossier with context evidence and next review actions.'
581
+ : researchRun?.status === 'error'
582
+ ? 'Saved a research dossier, but the preview needs review before promotion.'
583
+ : researchWorkspaceError
584
+ ? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
585
+ : plan.done
586
+ ? 'Prepared a direct grounded-answer plan.'
587
+ : 'Prepared a grounded research plan over real DQL assets.');
588
+ return {
589
+ summary,
590
+ answer: plan.followUp?.question ?? narration?.summary ?? researchRun?.summary,
591
+ status: needsClarification ? 'needs_clarification' : 'needs_review',
592
+ trustState: needsClarification ? 'not_applicable' : 'review_required',
593
+ stopReason: needsClarification ? 'needs_clarification' : 'human_review_required',
594
+ artifacts: needsClarification
595
+ ? []
596
+ : [agentRunArtifact('research_run', 'Research plan', {
597
+ plan,
598
+ researchRun,
599
+ researchRunId: researchRun?.id,
600
+ notebookPath,
601
+ workspaceError: researchWorkspaceError,
602
+ routeDecision,
603
+ narration,
604
+ resultPreview: researchResultData,
605
+ blockCount: blocks.length,
606
+ metricCount: metrics.length,
607
+ certifiedOnly: usedCertifiedOnly,
608
+ }, researchRun?.id)],
609
+ evaluations: [
610
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to research.'),
611
+ agentRunEvaluation('catalog-grounding', 'Catalog grounding', plan.sources.length > 0 || Boolean(researchRun?.evidence), plan.sources.length > 0 || Boolean(researchRun?.evidence) ? 'info' : 'warning', plan.sources.length > 0 || Boolean(researchRun?.evidence)
612
+ ? 'Research dossier is grounded to catalog or context-pack evidence.'
613
+ : 'No certified catalog source was found; output remains exploratory.', { sources: plan.sources, researchRunId: researchRun?.id, evidence: researchRun?.evidence }),
614
+ agentRunEvaluation('research-workspace', 'Research workspace', Boolean(researchRun?.id), researchRun?.id ? 'info' : 'warning', researchRun?.id
615
+ ? 'A durable notebook research record was saved for review and DQL promotion.'
616
+ : researchWorkspaceError
617
+ ? 'Research workspace storage is unavailable; the plan remains available in this agent run.'
618
+ : 'No durable research record was created because the run needs clarification first.', { notebookPath, researchRunId: researchRun?.id, error: researchWorkspaceError }),
619
+ ],
620
+ nextActions: needsClarification
621
+ ? [{ id: 'answer-follow-up', label: 'Answer follow-up', route: 'research' }]
622
+ : [
623
+ ...(researchRun?.id ? [{ id: 'open-research', label: 'Open research dossier', artifactKind: 'research_run' }] : []),
624
+ ...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
625
+ { id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
626
+ ],
627
+ };
628
+ },
629
+ sql_cell: async ({ request, routeDecision, attempt, repairHint }) => {
630
+ const result = await buildAgentPromptArtifact(request, 'cell', { attempt, repairHint });
631
+ return {
632
+ summary: 'Created a review-required SQL cell draft.',
633
+ answer: result.target === 'cell' ? result.explanation : undefined,
634
+ artifacts: [agentRunArtifact('sql_cell', 'Generated SQL cell', result)],
635
+ evaluations: [
636
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to SQL cell generation.'),
637
+ agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', 'Generated SQL must be reviewed before it becomes certified analytics.'),
638
+ ],
639
+ nextActions: [
640
+ { id: 'insert-sql', label: 'Insert SQL cell', artifactKind: 'sql_cell' },
641
+ { id: 'create-block', label: 'Promote to DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
642
+ ],
643
+ };
644
+ },
645
+ dql_block_draft: async ({ request, routeDecision, attempt, repairHint }) => {
646
+ const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
647
+ const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
648
+ return {
649
+ summary: ready
650
+ ? 'Created a DQL block draft that is ready for human certification review.'
651
+ : 'Created a DQL block draft with review blockers or warnings.',
652
+ artifacts: [agentRunArtifact('dql_block_draft', result.target === 'block' ? result.name : 'DQL block draft', result, result.target === 'block' ? result.path : undefined)],
653
+ evaluations: [
654
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to DQL block draft generation.'),
655
+ agentRunEvaluation('certification-boundary', 'Certification boundary', ready, 'warning', ready
656
+ ? 'The draft has no automatic certifier blockers, but certification still requires human review.'
657
+ : 'The draft has certifier blockers that must be resolved before certification review.', result),
658
+ ],
659
+ nextActions: [
660
+ { id: 'open-review', label: 'Open review checklist', artifactKind: 'dql_block_draft' },
661
+ { id: 'build-app', label: 'Build app from block', route: 'app_build', artifactKind: 'app_draft' },
662
+ ],
663
+ };
664
+ },
665
+ app_build: async ({ request, routeDecision, emit }) => {
666
+ emit({
667
+ type: 'executor.started',
668
+ message: 'Creating app build session from governed app builder.',
669
+ route: 'app_build',
670
+ });
671
+ let session;
672
+ try {
673
+ session = await createAppAiBuildSession(projectRoot, {
674
+ prompt: request.question,
675
+ domain: agentRunWorkspaceValue(request, 'domain'),
676
+ owner: agentRunWorkspaceValue(request, 'owner'),
677
+ notebookPath: agentRunWorkspaceValue(request, 'notebookPath') ?? request.selectedObject?.path,
678
+ selectedBlockIds: parseAgentRunSelectedBlockIds(request),
679
+ plannerMode: 'deterministic',
680
+ });
681
+ }
682
+ catch (error) {
683
+ const message = formatAgentRunInfrastructureError(error, 'App build storage');
684
+ return {
685
+ summary: message,
686
+ status: 'blocked',
687
+ trustState: 'blocked',
688
+ stopReason: 'blocked',
689
+ evaluations: [
690
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to app build.'),
691
+ agentRunEvaluation('app-build-storage', 'App build storage', false, 'blocking', message, { originalErrorType: error instanceof Error ? error.name : typeof error }),
692
+ ],
693
+ nextActions: [
694
+ { id: 'research-coverage', label: 'Research missing coverage', route: 'research', artifactKind: 'research_run' },
695
+ { id: 'create-gap-blocks', label: 'Create DQL drafts for gaps', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
696
+ ],
697
+ };
698
+ }
699
+ const ready = session.status === 'ready';
700
+ const sessionPlan = agentRunRecord(session.plan);
701
+ const appTitle = agentRunString(sessionPlan?.name) ?? 'App draft';
702
+ // A coverage gap is NOT terminal — leave status open so the gate can escalate to
703
+ // drafting the missing blocks. Only genuine infra errors (the catch above) block.
704
+ return {
705
+ summary: ready
706
+ ? 'Created a review-required app draft session from certified DQL assets.'
707
+ : 'App build needs more certified DQL coverage before files can be generated.',
708
+ status: ready ? 'needs_review' : undefined,
709
+ trustState: ready ? 'review_required' : undefined,
710
+ stopReason: ready ? 'human_review_required' : undefined,
711
+ artifacts: ready ? [agentRunArtifact('app_draft', appTitle, {
712
+ session,
713
+ sessionId: session.id,
714
+ appId: session.appId,
715
+ dashboardId: session.dashboardId,
716
+ generatedPaths: session.generatedPaths,
717
+ plan: session.plan,
718
+ validation: session.validation,
719
+ }, session.appId)] : [],
720
+ evaluations: [
721
+ agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to app build.'),
722
+ agentRunEvaluation('app-coverage', 'Certified coverage', ready, ready ? 'info' : 'blocking', ready
723
+ ? 'Generated app files are backed by certified block tiles and saved as a draft app session.'
724
+ : session.error ?? 'No certified app tiles matched the request.', session),
725
+ ],
726
+ nextActions: ready
727
+ ? [
728
+ { id: 'open-app', label: 'Open app draft', artifactKind: 'app_draft' },
729
+ { id: 'create-gap-blocks', label: 'Create DQL drafts for gaps', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
730
+ ]
731
+ : [
732
+ { id: 'research-coverage', label: 'Research missing coverage', route: 'research', artifactKind: 'research_run' },
733
+ { id: 'create-gap-blocks', label: 'Create DQL drafts for gaps', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
734
+ ],
735
+ };
736
+ },
737
+ };
738
+ // Compact, catalog-grounded context the LLM planner decomposes `auto` turns against.
739
+ const buildAgentRunCatalogContext = () => {
740
+ try {
741
+ const blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
742
+ const sourceBlocks = blocks.length > 0 ? blocks : collectPlanBlocks(projectRoot, { certifiedOnly: false });
743
+ const blockLines = sourceBlocks.slice(0, 24).map((block) => {
744
+ const domain = block.domain ? ` [${block.domain}]` : '';
745
+ const detail = block.description ? `: ${block.description}` : '';
746
+ return `- ${block.name}${domain}${detail}`;
747
+ });
748
+ const metrics = loadSemanticMetrics(projectRoot).slice(0, 24);
749
+ const metricLines = metrics.map((metric) => {
750
+ const node = metric;
751
+ return `- ${node.name ?? node.label ?? node.id ?? 'metric'}`;
752
+ });
753
+ return [
754
+ blockLines.length > 0 ? `Available DQL blocks:\n${blockLines.join('\n')}` : 'Available DQL blocks: none',
755
+ metricLines.length > 0 ? `Governed metrics:\n${metricLines.join('\n')}` : '',
756
+ ].filter(Boolean).join('\n\n');
757
+ }
758
+ catch {
759
+ return '';
760
+ }
761
+ };
762
+ // Provider-agnostic completion the planner injects. Reuses the configured provider
763
+ // adapter; throwing here makes the planner fall back to its deterministic path.
764
+ const agentRunPlanner = createLlmAgentRunPlanner({
765
+ complete: async ({ system, user, signal }) => {
766
+ const provider = await createBlockStudioAssistProvider(projectRoot);
767
+ if (!provider)
768
+ throw new Error('No AI provider configured for planning.');
769
+ return provider.generate([
770
+ { role: 'system', content: system },
771
+ { role: 'user', content: user },
772
+ ], { maxTokens: 700, temperature: 0.1, signal });
773
+ },
774
+ getCatalogContext: buildAgentRunCatalogContext,
775
+ });
776
+ const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
777
+ const agentRunEngine = new AgentRunEngine({
778
+ store: agentRunStore,
779
+ executors: agentRunExecutors,
780
+ gates: defaultAgentRunGates,
781
+ planner: agentRunPlanner,
782
+ });
129
783
  const runNotebookForApp = async (appId, notebookPath) => {
130
784
  const absPath = safeJoin(projectRoot, notebookPath);
131
785
  if (!absPath || !existsSync(absPath) || statSync(absPath).isDirectory() || !absPath.endsWith('.dqlnb')) {
@@ -313,21 +967,50 @@ export async function startLocalServer(opts) {
313
967
  }
314
968
  let governedAnswer;
315
969
  let providerError;
970
+ const memoOnly = input.mode === 'memo_only';
316
971
  const contextEnvelope = {
317
972
  mode: 'app_research',
973
+ generationMode: input.mode ?? 'sql_and_memo',
318
974
  intent: input.intent,
319
975
  appId: input.appId,
320
976
  dashboardId: input.dashboardId,
321
977
  sourceTileId: input.sourceTileId,
322
978
  sourceBlockId: input.sourceBlockId,
323
979
  title: input.title,
324
- instruction: 'Generate review-required read-only SQL when certified blocks do not exactly answer the requested research grain. Execute only through the bounded generated SQL preview path.',
980
+ instruction: [
981
+ 'Answer as an app-scoped analyst. Start from certified block/result context, active filters, dbt/semantic metadata, and schema evidence.',
982
+ memoOnly
983
+ ? 'Write the stakeholder memo from the supplied selected result, preview rows, generated SQL, metrics, drivers, filters, and caveats. Do not generate replacement SQL unless the user explicitly asks to fix SQL.'
984
+ : 'Generate review-required read-only SQL only when the certified result does not exactly answer the requested analysis grain. Execute only through the bounded generated SQL preview path.',
985
+ 'In the natural-language answer, write a stakeholder analysis memo, not a generic chat answer. Use Markdown headings chosen for the question, usually Executive answer, Key numbers, Drivers or Business readout, Caveats, and Next action.',
986
+ 'Use concrete numbers from the selected result or preview when available. If baseline, segment, grain, lineage, or source proof is missing, say that explicitly instead of inventing a driver story.',
987
+ 'Keep raw SQL in proposedSql/sql. Do not put SQL code fences or implementation trace in the memo body.',
988
+ ].join(' '),
989
+ generatedSql: input.generatedSql,
990
+ metrics: input.metrics,
991
+ drivers: input.drivers,
992
+ resultPreviews: input.resultPreviews,
993
+ summaryHint: input.summaryHint,
994
+ recommendationHint: input.recommendationHint,
995
+ sqlError: input.sqlError,
996
+ sqlErrorKind: input.sqlErrorKind,
997
+ hasReportEvidence: input.hasReportEvidence,
325
998
  context: input.context,
326
999
  };
327
1000
  const controller = new AbortController();
328
1001
  await runner.run({
329
1002
  provider: resolvedProvider,
330
- messages: [{ role: 'user', content: input.question }],
1003
+ messages: [{
1004
+ role: 'user',
1005
+ content: memoOnly
1006
+ ? [
1007
+ `Research question: ${input.question}`,
1008
+ 'Write the business memo now using only the evidence envelope supplied as upstream context.',
1009
+ 'Return Markdown sections. Keep SQL, query text, implementation trace, and raw routing details out of the memo body.',
1010
+ 'If the evidence is insufficient, state the gap and the next reviewer action instead of filling the story with generic text.',
1011
+ ].join('\n')
1012
+ : input.question,
1013
+ }],
331
1014
  upstream: {
332
1015
  cellId: `app-research:${input.appId}:${input.dashboardId ?? 'app'}`,
333
1016
  sql: JSON.stringify(contextEnvelope, null, 2),
@@ -348,7 +1031,7 @@ export async function startLocalServer(opts) {
348
1031
  throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
349
1032
  }
350
1033
  return {
351
- sql: governedAnswer.proposedSql ?? governedAnswer.sql,
1034
+ sql: memoOnly ? undefined : governedAnswer.proposedSql ?? governedAnswer.sql,
352
1035
  answer: governedAnswer.answer ?? governedAnswer.text,
353
1036
  result: governedAnswer.result,
354
1037
  analysisPlan: governedAnswer.analysisPlan,
@@ -360,6 +1043,87 @@ export async function startLocalServer(opts) {
360
1043
  };
361
1044
  };
362
1045
  const openNotebookResearchStorage = () => new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
1046
+ const notebookResearchStorageUnavailableMessage = 'Notebook research storage is unavailable because the local SQLite native bindings are not installed for this Node.js runtime.';
1047
+ const notebookResearchNextActionFilters = [
1048
+ 'fix_blockers',
1049
+ 'review_sql',
1050
+ 'review_context',
1051
+ 'run_preview',
1052
+ 'reuse_existing',
1053
+ 'create_dql_draft',
1054
+ 'open_certification',
1055
+ 'complete_review',
1056
+ 'continue_review',
1057
+ ];
1058
+ const isNotebookResearchStorageUnavailable = (error) => {
1059
+ const message = error instanceof Error ? error.message : String(error);
1060
+ return /better[-_]sqlite3/i.test(message) || /Could not locate the bindings file/i.test(message);
1061
+ };
1062
+ const emptyNotebookResearchNextActionCounts = () => Object.fromEntries(notebookResearchNextActionFilters.map((action) => [action, 0]));
1063
+ const emptyNotebookResearchListPage = (input = {}) => ({
1064
+ runs: [],
1065
+ total: 0,
1066
+ domains: [],
1067
+ owners: [],
1068
+ intents: [],
1069
+ notebooks: [],
1070
+ counts: {
1071
+ total: 0,
1072
+ ready: 0,
1073
+ needsReview: 0,
1074
+ dqlDrafts: 0,
1075
+ errors: 0,
1076
+ reuseExisting: 0,
1077
+ extendExisting: 0,
1078
+ replacements: 0,
1079
+ createNew: 0,
1080
+ draftReady: 0,
1081
+ certificationReady: 0,
1082
+ blocked: 0,
1083
+ staleOpen: 0,
1084
+ expiredOpen: 0,
1085
+ sourceLinked: 0,
1086
+ nextActions: emptyNotebookResearchNextActionCounts(),
1087
+ },
1088
+ groupCounts: {
1089
+ domains: 0,
1090
+ owners: 0,
1091
+ intents: 0,
1092
+ notebooks: 0,
1093
+ },
1094
+ limit: input.limit,
1095
+ offset: input.offset ?? 0,
1096
+ });
1097
+ const emptyNotebookResearchDiagnostics = () => ({
1098
+ counts: {
1099
+ totalRuns: 0,
1100
+ activeRuns: 0,
1101
+ closedRuns: 0,
1102
+ notebooks: 0,
1103
+ domains: 0,
1104
+ owners: 0,
1105
+ sourceLinkedRuns: 0,
1106
+ },
1107
+ health: {
1108
+ staleOpenRuns: 0,
1109
+ expiredOpenRuns: 0,
1110
+ staleThresholdDays: 7,
1111
+ expiredThresholdDays: 30,
1112
+ },
1113
+ search: {
1114
+ indexed: false,
1115
+ indexRows: 0,
1116
+ stale: false,
1117
+ },
1118
+ updatedAt: {},
1119
+ limits: {
1120
+ pageSize: 50,
1121
+ maxPageSize: 500,
1122
+ sourceCoverageLimit: 10_000,
1123
+ seedCellLimit: 1000,
1124
+ },
1125
+ warnings: [notebookResearchStorageUnavailableMessage],
1126
+ });
363
1127
  const runNotebookResearch = async (storage, run, input = {}) => {
364
1128
  const question = notebookResearchString(input.question) || run.question;
365
1129
  const domain = notebookResearchString(input.domain) ?? run.domain;
@@ -936,7 +1700,19 @@ export async function startLocalServer(opts) {
936
1700
  if (tests.length === 0) {
937
1701
  return { passed: 0, failed: 0, skipped: 0, duration: Date.now() - start, assertions: [], runAt: new Date() };
938
1702
  }
939
- const prepared = prepareLocalExecution(plan.sql, activeConnection, projectRoot, projectConfig);
1703
+ // Run tests against the SAME SQL the preview runs: for a semantic block with a
1704
+ // pre-compiled query, that's the query (not a recompiled metric), so the test's
1705
+ // output columns match the block's declared outputs.
1706
+ const semanticCompose = semanticLayer
1707
+ ? composeSemanticBlockSql(source, semanticLayer, {
1708
+ driver: activeConnection.driver,
1709
+ tableMapping,
1710
+ projectRoot,
1711
+ projectConfig,
1712
+ detectedProvider: semanticDetectedProvider,
1713
+ })
1714
+ : null;
1715
+ const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, activeConnection, projectRoot, projectConfig);
940
1716
  const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams ?? [], runtimeVariables(plan.variables ?? {}), prepared.connection);
941
1717
  const rows = Array.isArray(rawResult?.rows) ? rawResult.rows : [];
942
1718
  const columns = Array.isArray(rawResult?.columns)
@@ -1070,12 +1846,25 @@ export async function startLocalServer(opts) {
1070
1846
  sourceSystems: parsed.sourceSystems,
1071
1847
  replacementFor: parsed.replacementFor,
1072
1848
  reviewCadence: parsed.reviewCadence,
1849
+ metricRef: parsed.metricRef || undefined,
1850
+ metricsRef: parsed.metricsRef.length > 0 ? parsed.metricsRef : undefined,
1073
1851
  dependencies: [],
1074
1852
  usedInCount: 0,
1075
1853
  createdAt: new Date(),
1076
1854
  updatedAt: new Date(),
1077
1855
  };
1078
- const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined);
1856
+ // Evaluate declared invariants against the preview result so the
1857
+ // `invariants-hold` certifier rule can enforce them. Best-effort: when the
1858
+ // preview failed there is no result to check, and the rule then blocks
1859
+ // certification (in enterprise mode) because the guarantees are unverified.
1860
+ const invariantEval = preview
1861
+ ? evaluateBlockInvariants(source, {
1862
+ columns: preview.result.columns,
1863
+ rows: preview.result.rows,
1864
+ })
1865
+ : null;
1866
+ record.invariants = extractBlockInvariants(source);
1867
+ const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined, invariantEval ? { invariantResults: invariantEval.invariantResults } : undefined);
1079
1868
  const checklist = buildBlockStudioCertificationChecklist({
1080
1869
  source,
1081
1870
  validation,
@@ -1084,7 +1873,19 @@ export async function startLocalServer(opts) {
1084
1873
  certificationErrors: certification.errors,
1085
1874
  extraBlockers: blockers,
1086
1875
  });
1087
- return { certification, checklist, validation, preview, testResults };
1876
+ return {
1877
+ certification,
1878
+ checklist,
1879
+ validation,
1880
+ preview,
1881
+ testResults,
1882
+ invariantResults: invariantEval?.invariantResults ?? [],
1883
+ invariantViolation: invariantEval?.invariantViolation ?? false,
1884
+ };
1885
+ };
1886
+ const writeAgentRunSse = (response, event, data) => {
1887
+ response.write(`event: ${event}\n`);
1888
+ response.write(`data: ${serializeJSON(data)}\n\n`);
1088
1889
  };
1089
1890
  const server = createServer(async (req, res) => {
1090
1891
  const requestUrl = req.url || '/';
@@ -1099,9 +1900,540 @@ export async function startLocalServer(opts) {
1099
1900
  res.end();
1100
1901
  return;
1101
1902
  }
1102
- if (req.method === 'GET' && path === '/api/health') {
1103
- res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1104
- res.end(serializeJSON({ status: 'ok' }));
1903
+ if (req.method === 'GET' && path === '/api/health') {
1904
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1905
+ res.end(serializeJSON({ status: 'ok' }));
1906
+ return;
1907
+ }
1908
+ if (req.method === 'GET' && path === '/api/agent-runs') {
1909
+ const rawLimit = Number(url.searchParams.get('limit'));
1910
+ const limit = Number.isFinite(rawLimit) && rawLimit > 0
1911
+ ? Math.min(200, Math.floor(rawLimit))
1912
+ : 50;
1913
+ const runs = agentRunStore
1914
+ .list()
1915
+ .sort((a, b) => b.startedAt.localeCompare(a.startedAt))
1916
+ .slice(0, limit);
1917
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1918
+ res.end(serializeJSON({ runs, total: agentRunStore.list().length, limit }));
1919
+ return;
1920
+ }
1921
+ if (req.method === 'POST' && path === '/api/agent-runs') {
1922
+ try {
1923
+ const body = await readJSON(req).catch(() => null);
1924
+ const parsed = parseAgentRunRequestBody(body);
1925
+ if (!parsed.request) {
1926
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1927
+ res.end(serializeJSON({ error: parsed.error ?? 'Invalid agent run request.' }));
1928
+ return;
1929
+ }
1930
+ const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
1931
+ if (wantsStream) {
1932
+ res.writeHead(200, {
1933
+ 'Content-Type': 'text/event-stream; charset=utf-8',
1934
+ 'Cache-Control': 'no-cache',
1935
+ 'Connection': 'keep-alive',
1936
+ 'X-Accel-Buffering': 'no',
1937
+ });
1938
+ const run = await agentRunEngine.run(parsed.request, (event) => {
1939
+ writeAgentRunSse(res, 'agent-run-event', event);
1940
+ });
1941
+ writeAgentRunSse(res, 'agent-run-complete', run);
1942
+ res.end();
1943
+ return;
1944
+ }
1945
+ const run = await agentRunEngine.run(parsed.request);
1946
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
1947
+ res.end(serializeJSON({ run }));
1948
+ }
1949
+ catch (error) {
1950
+ const message = error instanceof Error ? error.message : String(error);
1951
+ if (res.headersSent) {
1952
+ writeAgentRunSse(res, 'agent-run-error', { error: message });
1953
+ res.end();
1954
+ }
1955
+ else {
1956
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1957
+ res.end(serializeJSON({ error: message }));
1958
+ }
1959
+ }
1960
+ return;
1961
+ }
1962
+ // Stakeholder → analyst handoff: turn a review-required output into a draft
1963
+ // research run in the analyst notebook queue (no authoring by the stakeholder).
1964
+ if (req.method === 'POST' && path === '/api/agent-runs/request-certification') {
1965
+ try {
1966
+ const body = await readJSON(req).catch(() => null);
1967
+ const record = agentRunRecord(body);
1968
+ const question = record ? agentRunString(record.question) : undefined;
1969
+ if (!question) {
1970
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1971
+ res.end(serializeJSON({ ok: false, error: 'question is required.' }));
1972
+ return;
1973
+ }
1974
+ const notebookPath = (record && agentRunString(record.notebookPath))
1975
+ ?? `notebooks/certification-requests/${Date.now()}.dqlnb`;
1976
+ const generatedSql = record ? agentRunString(record.generatedSql) : undefined;
1977
+ try {
1978
+ const storage = openNotebookResearchStorage();
1979
+ try {
1980
+ const created = storage.createRun({
1981
+ notebookPath,
1982
+ title: agentRunTitle(question, 'Certification request'),
1983
+ question,
1984
+ intent: 'ad_hoc_analysis',
1985
+ domain: record ? agentRunString(record.domain) : undefined,
1986
+ owner: record ? agentRunString(record.owner) : undefined,
1987
+ generatedSql,
1988
+ context: {
1989
+ surface: 'stakeholder_request_certification',
1990
+ requestedContext: agentRunRecord(record?.context) ?? null,
1991
+ },
1992
+ });
1993
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
1994
+ res.end(serializeJSON({ ok: true, researchRunId: created.id, notebookPath }));
1995
+ }
1996
+ finally {
1997
+ storage.close();
1998
+ }
1999
+ }
2000
+ catch (error) {
2001
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2002
+ res.end(serializeJSON({ ok: false, error: formatNotebookResearchStorageError(error) }));
2003
+ }
2004
+ }
2005
+ catch (error) {
2006
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2007
+ res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
2008
+ }
2009
+ return;
2010
+ }
2011
+ const agentRunMatch = /^\/api\/agent-runs\/([^/]+)$/.exec(path);
2012
+ if (req.method === 'GET' && agentRunMatch) {
2013
+ const id = decodeURIComponent(agentRunMatch[1]);
2014
+ const run = await agentRunStore.get(id);
2015
+ if (!run) {
2016
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
2017
+ res.end(serializeJSON({ error: 'Agent run not found.' }));
2018
+ return;
2019
+ }
2020
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2021
+ res.end(serializeJSON({ run }));
2022
+ return;
2023
+ }
2024
+ // Readiness → propose backbone. Returns a readiness summary plus the ranked
2025
+ // DRAFT proposals (each with its stored Certifier verdict) so the notebook
2026
+ // "Get Started" surface can route them into human review. dryRun preview —
2027
+ // nothing is written or certified by this call.
2028
+ if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose') {
2029
+ try {
2030
+ let owner;
2031
+ let limit;
2032
+ if (req.method === 'POST') {
2033
+ const body = await readJSON(req).catch(() => ({}));
2034
+ if (typeof body?.owner === 'string')
2035
+ owner = body.owner;
2036
+ if (typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0) {
2037
+ limit = body.limit;
2038
+ }
2039
+ }
2040
+ else {
2041
+ const ownerParam = url.searchParams.get('owner');
2042
+ if (ownerParam)
2043
+ owner = ownerParam;
2044
+ const limitParam = Number(url.searchParams.get('limit'));
2045
+ if (Number.isFinite(limitParam) && limitParam > 0)
2046
+ limit = limitParam;
2047
+ }
2048
+ const readiness = buildProposeReadiness(projectRoot, loadProjectConfig(projectRoot), { owner, limit });
2049
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2050
+ res.end(serializeJSON(readiness));
2051
+ }
2052
+ catch (error) {
2053
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2054
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2055
+ }
2056
+ return;
2057
+ }
2058
+ // Deterministic PLAN only (classify → plan). Writes NOTHING. Same data the
2059
+ // readiness endpoint embeds, exposed standalone for the approve gate.
2060
+ if (req.method === 'POST' && path === '/api/propose/plan') {
2061
+ try {
2062
+ const readiness = buildProposeReadiness(projectRoot, loadProjectConfig(projectRoot));
2063
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2064
+ res.end(serializeJSON(readiness.ready ? readiness.plan : { ready: false, reason: readiness.reason }));
2065
+ }
2066
+ catch (error) {
2067
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2068
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2069
+ }
2070
+ return;
2071
+ }
2072
+ // Materialize drafts for an APPROVED scope { slugs } (or { domains }).
2073
+ // This is the only propose endpoint that writes — and only for the approved,
2074
+ // business-only selection. Nothing is ever certified.
2075
+ if (req.method === 'POST' && path === '/api/propose/generate') {
2076
+ try {
2077
+ const body = (await readJSON(req).catch(() => ({})));
2078
+ const config = loadProjectConfig(projectRoot);
2079
+ let slugs = Array.isArray(body?.slugs)
2080
+ ? body.slugs.filter((s) => typeof s === 'string')
2081
+ : [];
2082
+ // { domains } → resolve to the plan's slugs for those domains.
2083
+ if (slugs.length === 0 && Array.isArray(body?.domains)) {
2084
+ const wanted = new Set(body.domains.filter((d) => typeof d === 'string'));
2085
+ const readiness = buildProposeReadiness(projectRoot, config);
2086
+ slugs = readiness.ready
2087
+ ? readiness.plan.domains
2088
+ .filter((d) => wanted.has(d.name))
2089
+ .flatMap((d) => d.candidates.map((c) => c.slug))
2090
+ : [];
2091
+ }
2092
+ if (slugs.length === 0) {
2093
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2094
+ res.end(serializeJSON({ error: 'Provide a non-empty { slugs } or { domains } scope to generate.' }));
2095
+ return;
2096
+ }
2097
+ const owner = typeof body?.owner === 'string' ? body.owner : undefined;
2098
+ const result = await generateProposeDrafts(projectRoot, slugs, config, { owner });
2099
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2100
+ res.end(serializeJSON(result));
2101
+ }
2102
+ catch (error) {
2103
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2104
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2105
+ }
2106
+ return;
2107
+ }
2108
+ // Materialize a single approved draft { slug } and return it. Convenience for
2109
+ // the per-block "Review & Certify" affordance.
2110
+ if (req.method === 'POST' && path === '/api/propose/draft') {
2111
+ try {
2112
+ const body = (await readJSON(req).catch(() => ({})));
2113
+ const slug = typeof body?.slug === 'string' ? body.slug : '';
2114
+ if (!slug) {
2115
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2116
+ res.end(serializeJSON({ error: 'Provide { slug } to draft.' }));
2117
+ return;
2118
+ }
2119
+ const owner = typeof body?.owner === 'string' ? body.owner : undefined;
2120
+ const result = await generateProposeDrafts(projectRoot, [slug], loadProjectConfig(projectRoot), { owner });
2121
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2122
+ res.end(serializeJSON(result));
2123
+ }
2124
+ catch (error) {
2125
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2126
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2127
+ }
2128
+ return;
2129
+ }
2130
+ // Transparent PLAN PREVIEW for ONE candidate (spec 14, part A). Lazy +
2131
+ // expensive: builds the real SQL + Certifier verdict + best-effort AI
2132
+ // enrichment for a single slug, so the UI shows the actual logic before a
2133
+ // human commits. Writes NOTHING.
2134
+ if (req.method === 'GET' && path === '/api/propose/preview') {
2135
+ try {
2136
+ const slug = url.searchParams.get('slug')?.trim();
2137
+ if (!slug) {
2138
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2139
+ res.end(serializeJSON({ error: 'Provide a ?slug= query parameter.' }));
2140
+ return;
2141
+ }
2142
+ const candidate = await buildProposeCandidatePreview(projectRoot, slug, url.searchParams.get('owner') ?? undefined);
2143
+ if (!candidate) {
2144
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
2145
+ res.end(serializeJSON({ error: `No proposed candidate found for slug "${slug}".` }));
2146
+ return;
2147
+ }
2148
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2149
+ res.end(serializeJSON({ candidate }));
2150
+ }
2151
+ catch (error) {
2152
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2153
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2154
+ }
2155
+ return;
2156
+ }
2157
+ // Unified AI BUILD (spec 14, part B). ONE engine, two targets:
2158
+ // target:'cell' → generate SQL from the prompt (+ context). Writes nothing.
2159
+ // target:'block' → assemble a COMPLETE draft, WRITE it, return preview fields.
2160
+ // Never routes through the governed Q&A answer-loop.
2161
+ if (req.method === 'POST' && path === '/api/ai/build') {
2162
+ try {
2163
+ const body = (await readJSON(req).catch(() => ({})));
2164
+ const prompt = typeof body?.prompt === 'string' ? body.prompt.trim() : '';
2165
+ if (!prompt) {
2166
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2167
+ res.end(serializeJSON({ error: 'Provide a non-empty { prompt }.' }));
2168
+ return;
2169
+ }
2170
+ const target = body?.target === 'cell' || body?.target === 'block' ? body.target : undefined;
2171
+ if (!target) {
2172
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2173
+ res.end(serializeJSON({ error: "Provide { target: 'cell' | 'block' }." }));
2174
+ return;
2175
+ }
2176
+ const context = {
2177
+ cellSql: typeof body?.context?.cellSql === 'string' ? body.context.cellSql : undefined,
2178
+ selection: typeof body?.context?.selection === 'string' ? body.context.selection : undefined,
2179
+ };
2180
+ const owner = typeof body?.owner === 'string' ? body.owner : undefined;
2181
+ const userId = typeof body?.userId === 'string'
2182
+ ? body.userId
2183
+ : undefined;
2184
+ // Edit mode (spec 17, part A): modify the block at `blockPath` in place.
2185
+ const mode = body?.mode === 'edit' ? 'edit' : 'create';
2186
+ const blockPath = typeof body?.blockPath === 'string' && body.blockPath.trim()
2187
+ ? body.blockPath.trim()
2188
+ : undefined;
2189
+ if (mode === 'edit' && (target !== 'block' || !blockPath)) {
2190
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2191
+ res.end(serializeJSON({ error: "Edit mode requires target 'block' and a { blockPath }." }));
2192
+ return;
2193
+ }
2194
+ // Ensure the agent knowledge graph is built before generating, so the Build
2195
+ // path's semantic-metric routing sees the governed metrics on the very first
2196
+ // call (a cold Build-before-Ask otherwise reads an unbuilt KG and misses them).
2197
+ // Mirrors what the Ask path does; reindex is idempotent and closes its write
2198
+ // connection, so the read-only metric load observes committed data.
2199
+ try {
2200
+ await reindexProject(projectRoot, { kgPath: defaultKgPath(projectRoot) });
2201
+ }
2202
+ catch {
2203
+ // Best-effort: a failed reindex falls back to whatever KG exists (or none).
2204
+ }
2205
+ // Inject user-authored Skills as business context; the engine selects the
2206
+ // relevant subset and stamps `appliedSkills` on the result.
2207
+ const skills = loadSkills(projectRoot).skills;
2208
+ const result = await buildFromPrompt({
2209
+ projectRoot,
2210
+ prompt,
2211
+ context,
2212
+ target,
2213
+ mode,
2214
+ blockPath,
2215
+ owner,
2216
+ userId,
2217
+ skills,
2218
+ dbtManifestPath: resolveDbtManifestPath(projectRoot, loadProjectConfig(projectRoot)),
2219
+ // Reflect-before-certify probe (P2): run the candidate block's SQL to learn
2220
+ // its REAL output columns and evaluate the declared invariants, so the agent
2221
+ // can reconcile the output contract + produce a grounded verdict before a
2222
+ // human reviews. Best-effort — buildFromPrompt falls back to a static reflection.
2223
+ executionProbe: async ({ sql, invariants }) => {
2224
+ const activeConnection = requireActiveConnection();
2225
+ const prepared = prepareLocalExecution(sql, activeConnection, projectRoot, projectConfig);
2226
+ const probeSql = `SELECT * FROM (${stripSqlTerminator(prepared.sql)}) _dql_probe LIMIT 2000`;
2227
+ const probeResult = await executor.executeQuery(probeSql, [], runtimeVariables({}), prepared.connection);
2228
+ const rows = (Array.isArray(probeResult?.rows) ? probeResult.rows : []);
2229
+ const rawColumns = Array.isArray(probeResult?.columns)
2230
+ ? probeResult.columns
2231
+ : [];
2232
+ const actualColumns = rawColumns.length > 0
2233
+ ? rawColumns.map((c) => (typeof c === 'string' ? c : c?.name ?? String(c)))
2234
+ : (rows[0] ? Object.keys(rows[0]) : []);
2235
+ const invariantResults = evaluateInvariants(invariants, { columns: actualColumns, rows });
2236
+ const passed = invariantResults.filter((r) => r.passed && !r.uncheckable).length;
2237
+ const failed = invariantResults.filter((r) => !r.passed && !r.uncheckable).length;
2238
+ return {
2239
+ actualColumns,
2240
+ invariantResults,
2241
+ tests: invariants.length > 0
2242
+ ? { passed, failed, assertionCount: invariantResults.length }
2243
+ : undefined,
2244
+ };
2245
+ },
2246
+ });
2247
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2248
+ res.end(serializeJSON(result));
2249
+ }
2250
+ catch (error) {
2251
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2252
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2253
+ }
2254
+ return;
2255
+ }
2256
+ // Resolved local OSS owner (spec 14, part C). Stamps drafts so a new block is
2257
+ // never born with a "Missing owner" Certifier strike.
2258
+ if (req.method === 'GET' && path === '/api/identity') {
2259
+ try {
2260
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2261
+ res.end(serializeJSON({ owner: resolveLocalOwner(projectRoot) }));
2262
+ }
2263
+ catch (error) {
2264
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2265
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2266
+ }
2267
+ return;
2268
+ }
2269
+ // ── Skills (spec 16) — user-authored business context. AI drafts, humans
2270
+ // certify; skills never carry certification. PROJECT skills (user empty)
2271
+ // are shared; PERSONAL skills (user set) are user-bound. ────────────────
2272
+ if (req.method === 'GET' && path === '/api/skills') {
2273
+ try {
2274
+ const skills = loadSkills(projectRoot).skills.map(serializeSkill);
2275
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2276
+ res.end(serializeJSON({ skills }));
2277
+ }
2278
+ catch (error) {
2279
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2280
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2281
+ }
2282
+ return;
2283
+ }
2284
+ // Form pickers: metrics from the semantic layer + certified block ids.
2285
+ if (req.method === 'GET' && path === '/api/skills/options') {
2286
+ try {
2287
+ const metrics = semanticLayer
2288
+ ? semanticLayer.listMetrics().map((m) => m.name).sort()
2289
+ : [];
2290
+ const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
2291
+ const blocks = Object.values(manifest.blocks)
2292
+ .filter((b) => b.status === 'certified')
2293
+ .map((b) => b.name)
2294
+ .sort();
2295
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2296
+ res.end(serializeJSON({ metrics, blocks }));
2297
+ }
2298
+ catch (error) {
2299
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2300
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2301
+ }
2302
+ return;
2303
+ }
2304
+ if (req.method === 'POST' && path === '/api/skills') {
2305
+ try {
2306
+ const body = (await readJSON(req).catch(() => ({})));
2307
+ const input = parseSkillInput(body?.skill);
2308
+ if (!input) {
2309
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2310
+ res.end(serializeJSON({ error: 'Provide { skill } with id, scope, and body.' }));
2311
+ return;
2312
+ }
2313
+ const skill = writeSkill(projectRoot, input);
2314
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2315
+ res.end(serializeJSON({ skill: serializeSkill(skill) }));
2316
+ }
2317
+ catch (error) {
2318
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2319
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2320
+ }
2321
+ return;
2322
+ }
2323
+ if (req.method === 'PUT' && path.startsWith('/api/skills/')) {
2324
+ try {
2325
+ const id = decodeURIComponent(path.slice('/api/skills/'.length));
2326
+ const body = (await readJSON(req).catch(() => ({})));
2327
+ const input = parseSkillInput(body?.skill, id);
2328
+ if (!input) {
2329
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2330
+ res.end(serializeJSON({ error: 'Provide { skill } with scope and body.' }));
2331
+ return;
2332
+ }
2333
+ const skill = writeSkill(projectRoot, input);
2334
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2335
+ res.end(serializeJSON({ skill: serializeSkill(skill) }));
2336
+ }
2337
+ catch (error) {
2338
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2339
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2340
+ }
2341
+ return;
2342
+ }
2343
+ if (req.method === 'DELETE' && path.startsWith('/api/skills/')) {
2344
+ try {
2345
+ const id = decodeURIComponent(path.slice('/api/skills/'.length));
2346
+ deleteSkill(projectRoot, id);
2347
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2348
+ res.end(serializeJSON({ ok: true }));
2349
+ }
2350
+ catch (error) {
2351
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2352
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2353
+ }
2354
+ return;
2355
+ }
2356
+ // ── Domains (spec 17, part B) — first-class business domain declarations.
2357
+ // Authoring here satisfies `dql doctor`'s "missing domain declaration"
2358
+ // warning. AI drafts, humans certify; domains carry no certification. ────
2359
+ if (req.method === 'GET' && path === '/api/domains') {
2360
+ try {
2361
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2362
+ res.end(serializeJSON({ domains: listDomains(projectRoot) }));
2363
+ }
2364
+ catch (error) {
2365
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2366
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2367
+ }
2368
+ return;
2369
+ }
2370
+ if (req.method === 'POST' && path === '/api/domains') {
2371
+ try {
2372
+ const body = (await readJSON(req).catch(() => ({})));
2373
+ const input = parseDomainInput(body?.domain);
2374
+ if (!input) {
2375
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2376
+ res.end(serializeJSON({ error: 'Provide { domain } with a non-empty name.' }));
2377
+ return;
2378
+ }
2379
+ writeDomainDeclaration(projectRoot, input);
2380
+ await refreshLocalMetadataCatalog(projectRoot);
2381
+ const domain = findDomain(projectRoot, input.name);
2382
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2383
+ res.end(serializeJSON({ domain }));
2384
+ }
2385
+ catch (error) {
2386
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2387
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2388
+ }
2389
+ return;
2390
+ }
2391
+ if (req.method === 'PUT' && path.startsWith('/api/domains/')) {
2392
+ try {
2393
+ const id = decodeURIComponent(path.slice('/api/domains/'.length));
2394
+ const body = (await readJSON(req).catch(() => ({})));
2395
+ const input = parseDomainInput(body?.domain, id);
2396
+ if (!input) {
2397
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2398
+ res.end(serializeJSON({ error: 'Provide { domain } with a name.' }));
2399
+ return;
2400
+ }
2401
+ // If the name changed, remove the old declaration so we never orphan one.
2402
+ if (domainFolderSlug(id) !== domainFolderSlug(input.name)) {
2403
+ deleteDomainDeclaration(projectRoot, id);
2404
+ }
2405
+ writeDomainDeclaration(projectRoot, input);
2406
+ await refreshLocalMetadataCatalog(projectRoot);
2407
+ const domain = findDomain(projectRoot, input.name);
2408
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2409
+ res.end(serializeJSON({ domain }));
2410
+ }
2411
+ catch (error) {
2412
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2413
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2414
+ }
2415
+ return;
2416
+ }
2417
+ if (req.method === 'DELETE' && path.startsWith('/api/domains/')) {
2418
+ try {
2419
+ const id = decodeURIComponent(path.slice('/api/domains/'.length));
2420
+ deleteDomainDeclaration(projectRoot, id);
2421
+ await refreshLocalMetadataCatalog(projectRoot);
2422
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2423
+ res.end(serializeJSON({ ok: true }));
2424
+ }
2425
+ catch (error) {
2426
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2427
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2428
+ }
2429
+ return;
2430
+ }
2431
+ if (req.method === 'GET' && path === '/favicon.ico') {
2432
+ res.writeHead(200, {
2433
+ 'Content-Type': 'image/svg+xml',
2434
+ 'Cache-Control': 'public, max-age=86400',
2435
+ });
2436
+ res.end(NOTEBOOK_FAVICON_SVG);
1105
2437
  return;
1106
2438
  }
1107
2439
  if (req.method === 'GET' && path === '/api/settings/env-status') {
@@ -1242,6 +2574,80 @@ export async function startLocalServer(opts) {
1242
2574
  res.end(serializeJSON({ ok: true, files }));
1243
2575
  return;
1244
2576
  }
2577
+ // Local learning loop (OSS): record an analyst's wrong→right correction as a
2578
+ // scope-matched Hint-Graph hint plus an advisory memory, so future similar
2579
+ // questions avoid the same mistake. Single-user self-serve — the correction IS
2580
+ // the approval, so the derived candidate is approved immediately unless the
2581
+ // caller opts out. Advisory only: never overrides certified routing. The
2582
+ // multi-tenant review workflow + automated distillation stay a cloud feature.
2583
+ if (req.method === 'POST' && path === '/api/agent/learnings/correction') {
2584
+ const body = await readJSON(req).catch(() => null);
2585
+ const question = body && typeof body.question === 'string' ? body.question.trim() : '';
2586
+ const correctedSql = body && typeof body.correctedSql === 'string' ? body.correctedSql.trim() : '';
2587
+ if (!body || !question || !correctedSql) {
2588
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2589
+ res.end(serializeJSON({ error: 'question and correctedSql are required.' }));
2590
+ return;
2591
+ }
2592
+ const rawScope = body.scope && typeof body.scope === 'object' ? body.scope : {};
2593
+ const scopeStr = (key) => (typeof rawScope[key] === 'string' && rawScope[key].trim() ? rawScope[key].trim() : undefined);
2594
+ const scope = {
2595
+ metric: scopeStr('metric'),
2596
+ dbtModel: scopeStr('dbtModel'),
2597
+ domain: scopeStr('domain'),
2598
+ dialect: scopeStr('dialect'),
2599
+ term: scopeStr('term'),
2600
+ block: scopeStr('block'),
2601
+ };
2602
+ const wrongSql = typeof body.wrongSql === 'string' ? body.wrongSql.trim() : '';
2603
+ const rationale = typeof body.rationale === 'string' && body.rationale.trim() ? body.rationale.trim() : undefined;
2604
+ const author = typeof body.author === 'string' ? body.author : (resolveLocalOwner(projectRoot) ?? undefined);
2605
+ try {
2606
+ const { trace, hint } = recordCorrectionTrace(projectRoot, {
2607
+ question,
2608
+ scope,
2609
+ wrongAnswer: wrongSql || '(no prior SQL captured)',
2610
+ correction: correctedSql,
2611
+ correctedSql,
2612
+ rationale,
2613
+ author,
2614
+ hintTitle: typeof body.title === 'string' && body.title.trim() ? body.title.trim() : undefined,
2615
+ hintGuidance: typeof body.guidance === 'string' && body.guidance.trim() ? body.guidance.trim() : undefined,
2616
+ tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
2617
+ });
2618
+ let approvedHint = hint;
2619
+ if (body.approve !== false) {
2620
+ reviewHint(projectRoot, { hintId: hint.id, decision: 'approved', reviewer: author ?? 'local', note: 'Self-approved (OSS single-user).' });
2621
+ approvedHint = { ...hint, status: 'approved' };
2622
+ }
2623
+ // Plain-language advisory memory mirroring the lesson, for transparency + recall.
2624
+ try {
2625
+ const memory = new MemoryStore(defaultMemoryPath(projectRoot));
2626
+ memory.upsert({
2627
+ id: `mem_${hint.id}`,
2628
+ scope: 'project',
2629
+ title: approvedHint.title,
2630
+ content: `${approvedHint.guidance}${rationale ? ` (${rationale})` : ''}`,
2631
+ tags: [scope.metric, scope.domain, scope.dbtModel].filter((x) => Boolean(x)),
2632
+ source: 'correction',
2633
+ confidence: 0.9,
2634
+ importance: 0.85,
2635
+ enabled: true,
2636
+ });
2637
+ memory.close();
2638
+ }
2639
+ catch {
2640
+ /* best-effort */
2641
+ }
2642
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2643
+ res.end(serializeJSON({ ok: true, trace, hint: approvedHint }));
2644
+ }
2645
+ catch (error) {
2646
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2647
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2648
+ }
2649
+ return;
2650
+ }
1245
2651
  const appDashRun = path.match(/^\/api\/apps\/([^/]+)\/dashboards\/([^/]+)\/run$/);
1246
2652
  if (req.method === 'POST' && appDashRun) {
1247
2653
  try {
@@ -1436,6 +2842,14 @@ export async function startLocalServer(opts) {
1436
2842
  executeSql: executeLocalSqlForStoredResult,
1437
2843
  generateInvestigationSql: generateInvestigationSqlForApp,
1438
2844
  runNotebook: (appId, notebookPath) => runNotebookForApp(appId, notebookPath),
2845
+ // P4: give the App ask lane a grounded research planner over the catalog.
2846
+ planResearch: async ({ question, isFollowUp }) => {
2847
+ const metrics = loadSemanticMetrics(projectRoot);
2848
+ let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
2849
+ if (blocks.length === 0)
2850
+ blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
2851
+ return planResearch({ question, metrics, blocks, isFollowUp });
2852
+ },
1439
2853
  });
1440
2854
  if (handled)
1441
2855
  return;
@@ -1560,8 +2974,11 @@ export async function startLocalServer(opts) {
1560
2974
  return;
1561
2975
  }
1562
2976
  if (req.method === 'GET' && path === '/api/notebook/research') {
1563
- const storage = openNotebookResearchStorage();
2977
+ let storage;
2978
+ const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
2979
+ const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
1564
2980
  try {
2981
+ storage = openNotebookResearchStorage();
1565
2982
  const notebookPath = notebookResearchString(url.searchParams.get('path'));
1566
2983
  const sourceCellId = notebookResearchString(url.searchParams.get('sourceCellId') ?? url.searchParams.get('cellId'));
1567
2984
  const domain = notebookResearchString(url.searchParams.get('domain'));
@@ -1577,24 +2994,28 @@ export async function startLocalServer(opts) {
1577
2994
  const activeOnlyParam = url.searchParams.get('activeOnly') ?? url.searchParams.get('active');
1578
2995
  const activeOnly = activeOnlyParam === 'true' || activeOnlyParam === '1';
1579
2996
  const sort = notebookResearchSort(url.searchParams.get('sort'));
1580
- const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
1581
- const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
1582
2997
  const page = storage.listRunsPage({ notebookPath, sourceCellId, domain, owner, intent, search, status, reviewStatus, promotionAction, readiness, age, nextAction, activeOnly, sort, limit, offset });
1583
2998
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1584
2999
  res.end(serializeJSON(withNotebookResearchChecklistPage(page)));
1585
3000
  }
1586
3001
  catch (error) {
3002
+ if (isNotebookResearchStorageUnavailable(error)) {
3003
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3004
+ res.end(serializeJSON(withNotebookResearchChecklistPage(emptyNotebookResearchListPage({ limit, offset }))));
3005
+ return;
3006
+ }
1587
3007
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1588
3008
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1589
3009
  }
1590
3010
  finally {
1591
- storage.close();
3011
+ storage?.close();
1592
3012
  }
1593
3013
  return;
1594
3014
  }
1595
3015
  if (req.method === 'POST' && path === '/api/notebook/research') {
1596
- const storage = openNotebookResearchStorage();
3016
+ let storage;
1597
3017
  try {
3018
+ storage = openNotebookResearchStorage();
1598
3019
  const body = await readJSON(req);
1599
3020
  const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1600
3021
  const question = notebookResearchString(body.question);
@@ -1641,7 +3062,7 @@ export async function startLocalServer(opts) {
1641
3062
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1642
3063
  }
1643
3064
  finally {
1644
- storage.close();
3065
+ storage?.close();
1645
3066
  }
1646
3067
  return;
1647
3068
  }
@@ -1686,8 +3107,9 @@ export async function startLocalServer(opts) {
1686
3107
  return;
1687
3108
  }
1688
3109
  if (req.method === 'POST' && path === '/api/notebook/research/seed-cells') {
1689
- const storage = openNotebookResearchStorage();
3110
+ let storage;
1690
3111
  try {
3112
+ storage = openNotebookResearchStorage();
1691
3113
  const body = await readJSON(req);
1692
3114
  const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1693
3115
  if (!notebookPath) {
@@ -1738,12 +3160,14 @@ export async function startLocalServer(opts) {
1738
3160
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1739
3161
  }
1740
3162
  finally {
1741
- storage.close();
3163
+ storage?.close();
1742
3164
  }
1743
3165
  return;
1744
3166
  }
1745
3167
  if (req.method === 'POST' && path === '/api/notebook/research/source-coverage') {
1746
- const storage = openNotebookResearchStorage();
3168
+ let storage;
3169
+ let requestedSourceCellCount = 0;
3170
+ let limit = 10_000;
1747
3171
  try {
1748
3172
  const body = await readJSON(req);
1749
3173
  const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
@@ -1784,15 +3208,16 @@ export async function startLocalServer(opts) {
1784
3208
  const sourceCellIds = Array.from(new Set(requestedIds
1785
3209
  .map((id) => notebookResearchString(id))
1786
3210
  .filter((id) => Boolean(id))));
1787
- const requestedSourceCellCount = new Set([
3211
+ requestedSourceCellCount = new Set([
1788
3212
  ...sourceCellIds,
1789
3213
  ...sourceCells
1790
3214
  .map((cell) => notebookResearchSourceCellId(cell))
1791
3215
  .filter((id) => Boolean(id)),
1792
3216
  ]).size;
1793
- const limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
3217
+ limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
1794
3218
  ? Math.max(1, Math.min(10_000, Math.floor(body.limit)))
1795
3219
  : 10_000;
3220
+ storage = openNotebookResearchStorage();
1796
3221
  const linkedRuns = storage.listLatestRunsBySourceCell({
1797
3222
  notebookPath,
1798
3223
  sourceCellIds,
@@ -1815,26 +3240,42 @@ export async function startLocalServer(opts) {
1815
3240
  }));
1816
3241
  }
1817
3242
  catch (error) {
3243
+ if (isNotebookResearchStorageUnavailable(error)) {
3244
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3245
+ res.end(serializeJSON({
3246
+ runs: [],
3247
+ requestedCount: requestedSourceCellCount,
3248
+ matchedCount: 0,
3249
+ limitApplied: requestedSourceCellCount > limit,
3250
+ }));
3251
+ return;
3252
+ }
1818
3253
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1819
3254
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1820
3255
  }
1821
3256
  finally {
1822
- storage.close();
3257
+ storage?.close();
1823
3258
  }
1824
3259
  return;
1825
3260
  }
1826
3261
  if (req.method === 'GET' && path === '/api/notebook/research/diagnostics') {
1827
- const storage = openNotebookResearchStorage();
3262
+ let storage;
1828
3263
  try {
3264
+ storage = openNotebookResearchStorage();
1829
3265
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1830
3266
  res.end(serializeJSON(storage.getDiagnostics()));
1831
3267
  }
1832
3268
  catch (error) {
3269
+ if (isNotebookResearchStorageUnavailable(error)) {
3270
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3271
+ res.end(serializeJSON(emptyNotebookResearchDiagnostics()));
3272
+ return;
3273
+ }
1833
3274
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1834
3275
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1835
3276
  }
1836
3277
  finally {
1837
- storage.close();
3278
+ storage?.close();
1838
3279
  }
1839
3280
  return;
1840
3281
  }
@@ -1842,8 +3283,9 @@ export async function startLocalServer(opts) {
1842
3283
  if (notebookResearchMatch) {
1843
3284
  const id = decodeURIComponent(notebookResearchMatch[1]);
1844
3285
  const action = notebookResearchMatch[2];
1845
- const storage = openNotebookResearchStorage();
3286
+ let storage;
1846
3287
  try {
3288
+ storage = openNotebookResearchStorage();
1847
3289
  const run = storage.getRun(id);
1848
3290
  if (!run) {
1849
3291
  res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -1943,7 +3385,7 @@ export async function startLocalServer(opts) {
1943
3385
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1944
3386
  }
1945
3387
  finally {
1946
- storage.close();
3388
+ storage?.close();
1947
3389
  }
1948
3390
  return;
1949
3391
  }
@@ -2447,6 +3889,144 @@ export async function startLocalServer(opts) {
2447
3889
  }
2448
3890
  return;
2449
3891
  }
3892
+ // ── Distinct values for a block column → app/dashboard filter dropdowns ──
3893
+ if (req.method === 'GET' && path === '/api/dashboard/filter-options') {
3894
+ try {
3895
+ const blockIdParam = url.searchParams.get('block');
3896
+ const blockPath = url.searchParams.get('path')
3897
+ ?? (blockIdParam ? resolveBlockPathById(projectRoot, blockIdParam) : null);
3898
+ const column = (url.searchParams.get('column') ?? '').trim();
3899
+ const limit = Math.min(Math.max(Number(url.searchParams.get('limit') ?? 50) || 50, 1), 200);
3900
+ if (!blockPath || !column || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(column)) {
3901
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3902
+ res.end(serializeJSON({ error: 'path and a valid column are required' }));
3903
+ return;
3904
+ }
3905
+ const absolutePath = resolve(projectRoot, blockPath);
3906
+ if (!absolutePath.startsWith(projectRoot + '/') && absolutePath !== projectRoot) {
3907
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3908
+ res.end(serializeJSON({ error: 'path escapes project root' }));
3909
+ return;
3910
+ }
3911
+ if (!existsSync(absolutePath)) {
3912
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
3913
+ res.end(serializeJSON({ error: 'block not found' }));
3914
+ return;
3915
+ }
3916
+ const source = readFileSync(absolutePath, 'utf-8');
3917
+ // Only expose distinct values for a DECLARED output column — keeps the probe
3918
+ // inside the governed block contract (no arbitrary column scanning).
3919
+ const parsedMeta = parseBlockSourceMetadata(source);
3920
+ if (parsedMeta.outputs.length > 0 && !parsedMeta.outputs.includes(column)) {
3921
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3922
+ res.end(serializeJSON({ error: `"${column}" is not a declared output of this block` }));
3923
+ return;
3924
+ }
3925
+ const activeConnection = requireActiveConnection();
3926
+ const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
3927
+ const semanticCompose = semanticLayer
3928
+ ? composeSemanticBlockSql(source, semanticLayer, {
3929
+ driver: activeConnection.driver,
3930
+ tableMapping,
3931
+ projectRoot,
3932
+ projectConfig,
3933
+ detectedProvider: semanticDetectedProvider,
3934
+ })
3935
+ : null;
3936
+ const validation = validateBlockStudioSource(source, semanticLayer);
3937
+ const baseSql = semanticCompose?.sql ?? validation.executableSql;
3938
+ if (!baseSql) {
3939
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3940
+ res.end(serializeJSON({ error: 'block has no executable SQL' }));
3941
+ return;
3942
+ }
3943
+ const prepared = prepareLocalExecution(baseSql, activeConnection, projectRoot, projectConfig);
3944
+ const q = quoteAgentIdentifier(column, prepared.connection);
3945
+ const wrapped = `SELECT DISTINCT ${q} AS value FROM (${stripSqlTerminator(prepared.sql)}) _dql_opt WHERE ${q} IS NOT NULL ORDER BY 1 LIMIT ${limit + 1}`;
3946
+ const result = await executor.executeQuery(wrapped, [], runtimeVariables({}), prepared.connection);
3947
+ const rows = Array.isArray(result?.rows) ? result.rows : [];
3948
+ const truncated = rows.length > limit;
3949
+ const options = rows
3950
+ .slice(0, limit)
3951
+ .map((row) => row?.value)
3952
+ .filter((value) => value !== null && value !== undefined)
3953
+ .map((value) => String(value));
3954
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3955
+ res.end(serializeJSON({ column, options, truncated }));
3956
+ }
3957
+ catch (error) {
3958
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3959
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3960
+ }
3961
+ return;
3962
+ }
3963
+ // ── Plan an app from a goal (P1: plan → critique → show gaps) ──────────
3964
+ // The agent decomposes the goal into the questions an app should answer
3965
+ // (KPI + trend + breakdowns), matches each to a CERTIFIED block, derives the
3966
+ // shared filters that refresh every tile, and reports coverage + gaps BEFORE
3967
+ // anything is built — so the human reviews the plan, not a blank canvas.
3968
+ if (req.method === 'POST' && path === '/api/app-plan') {
3969
+ try {
3970
+ const body = await readJSON(req);
3971
+ const goal = typeof body.goal === 'string' ? body.goal.trim() : '';
3972
+ if (!goal) {
3973
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3974
+ res.end(serializeJSON({ error: 'goal is required' }));
3975
+ return;
3976
+ }
3977
+ const certifiedOnly = body.certifiedOnly !== false;
3978
+ const metrics = loadSemanticMetrics(projectRoot);
3979
+ let blocks = collectPlanBlocks(projectRoot, { certifiedOnly });
3980
+ // If nothing is certified yet, fall back to all drafts so the plan still
3981
+ // shows what COULD be assembled (every section then reads as a gap to certify).
3982
+ if (blocks.length === 0 && certifiedOnly)
3983
+ blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
3984
+ const plan = await planApp({ goal, metrics, blocks });
3985
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3986
+ res.end(serializeJSON({ plan, blockCount: blocks.length, metricCount: metrics.length, certifiedOnly }));
3987
+ }
3988
+ catch (error) {
3989
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3990
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3991
+ }
3992
+ return;
3993
+ }
3994
+ // ── Research / follow-up planning (P4: ReAct over the catalog) ─────────
3995
+ // Decide whether to answer, research across grounded steps, or ask a smart
3996
+ // follow-up — so the agent behaves like a real assistant instead of always
3997
+ // generating one query. Every step + option is bound to a real metric/block.
3998
+ if (req.method === 'POST' && path === '/api/research-plan') {
3999
+ try {
4000
+ const body = await readJSON(req);
4001
+ const question = typeof body.question === 'string' ? body.question.trim() : '';
4002
+ if (!question) {
4003
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
4004
+ res.end(serializeJSON({ error: 'question is required' }));
4005
+ return;
4006
+ }
4007
+ const metrics = loadSemanticMetrics(projectRoot);
4008
+ let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
4009
+ if (blocks.length === 0)
4010
+ blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
4011
+ const plan = await planResearch({
4012
+ question,
4013
+ metrics,
4014
+ blocks,
4015
+ intent: typeof body.intent === 'string'
4016
+ ? body.intent
4017
+ : undefined,
4018
+ isFollowUp: body.isFollowUp === true,
4019
+ history: Array.isArray(body.history) ? body.history : undefined,
4020
+ });
4021
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
4022
+ res.end(serializeJSON({ plan, blockCount: blocks.length, metricCount: metrics.length }));
4023
+ }
4024
+ catch (error) {
4025
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
4026
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
4027
+ }
4028
+ return;
4029
+ }
2450
4030
  // ── Run block tests ────────────────────────────────────────────────
2451
4031
  if (req.method === 'POST' && path === '/api/blocks/run-tests') {
2452
4032
  try {
@@ -2485,7 +4065,11 @@ export async function startLocalServer(opts) {
2485
4065
  res.end(serializeJSON({ error: 'source is required' }));
2486
4066
  return;
2487
4067
  }
2488
- const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise !== false });
4068
+ // OSS local-first certifies in non-enterprise mode: owner + passing tests +
4069
+ // a successful run are the gate; grain/outputs/pattern/lineage/cadence are
4070
+ // AI-filled advisory warnings, not hard blockers. Enterprise-grade certification
4071
+ // (all of those required) is an opt-in for the cloud tier — request it explicitly.
4072
+ const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise === true });
2489
4073
  const blockers = Array.from(new Set(result.checklist.blockers));
2490
4074
  if (!result.certification.certified || blockers.length > 0) {
2491
4075
  res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -2514,6 +4098,32 @@ export async function startLocalServer(opts) {
2514
4098
  certifiedPayload = openBlockStudioDocument(projectRoot, normalizedBlockPath, semanticLayer);
2515
4099
  }
2516
4100
  }
4101
+ // Auto-capture (OSS local learning loop): certifying a block teaches the
4102
+ // agent to prefer it. Write a scoped, advisory project memory — deterministic,
4103
+ // best-effort, and it never blocks certification.
4104
+ try {
4105
+ const learned = parseBlockSourceMetadata(certifiedSource);
4106
+ if (learned.name) {
4107
+ const learnedOutputs = Array.isArray(learned.outputs)
4108
+ ? learned.outputs.filter((o) => typeof o === 'string')
4109
+ : [];
4110
+ const memory = new MemoryStore(defaultMemoryPath(projectRoot));
4111
+ memory.upsert({
4112
+ id: `mem_certify_${learned.name}`,
4113
+ scope: 'project',
4114
+ title: `Certified block: ${learned.name}`,
4115
+ content: `Prefer the certified block "${learned.name}" for ${learned.description?.trim() || `questions in the ${learned.domain ?? 'analytics'} domain`}.${learned.grain ? ` Grain: ${learned.grain}.` : ''}${learnedOutputs.length ? ` Outputs: ${learnedOutputs.slice(0, 8).join(', ')}.` : ''} It is the trusted source — reuse it instead of generating new SQL.`,
4116
+ tags: [learned.domain, learned.name, ...learnedOutputs.slice(0, 4)].filter((x) => Boolean(x)),
4117
+ source: 'certify',
4118
+ confidence: 0.95,
4119
+ importance: 0.85,
4120
+ enabled: true,
4121
+ });
4122
+ }
4123
+ }
4124
+ catch {
4125
+ /* best-effort: learning capture must never block certification */
4126
+ }
2517
4127
  await refreshLocalMetadataCatalog(projectRoot);
2518
4128
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2519
4129
  res.end(serializeJSON({
@@ -2642,7 +4252,8 @@ export async function startLocalServer(opts) {
2642
4252
  return;
2643
4253
  }
2644
4254
  const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
2645
- const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: true });
4255
+ // OSS certify = non-enterprise (owner-gated); enterprise depth is the cloud tier.
4256
+ const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: false });
2646
4257
  const blockers = Array.from(new Set(certification.checklist.blockers));
2647
4258
  if (!certification.certification.certified || blockers.length > 0) {
2648
4259
  const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
@@ -4576,6 +6187,14 @@ export async function startLocalServer(opts) {
4576
6187
  assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
4577
6188
  const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
4578
6189
  const normalized = normalizeQueryResult(rawResult);
6190
+ // Enforce the block's declared invariants against the result set. This
6191
+ // is additive: blocks without invariants produce `null` and the
6192
+ // response is unchanged. The agent surface (`query_via_block`) reads
6193
+ // these fields to downgrade the trust label on violation.
6194
+ const invariants = evaluateBlockInvariants(executableCell.source || cell.source || '', {
6195
+ columns: normalized.columns,
6196
+ rows: normalized.rows,
6197
+ });
4579
6198
  if (execContext) {
4580
6199
  recordNotebookQueryRun(projectRoot, {
4581
6200
  notebookPath: execContext.notebookPath,
@@ -4604,6 +6223,12 @@ export async function startLocalServer(opts) {
4604
6223
  chartConfig: plan.chartConfig,
4605
6224
  tests: plan.tests,
4606
6225
  result: normalized,
6226
+ ...(invariants
6227
+ ? {
6228
+ invariantResults: invariants.invariantResults,
6229
+ invariantViolation: invariants.invariantViolation,
6230
+ }
6231
+ : {}),
4607
6232
  }));
4608
6233
  }
4609
6234
  catch (error) {
@@ -4679,6 +6304,7 @@ table: ${table}${tagList}
4679
6304
  res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
4680
6305
  res.end(content);
4681
6306
  });
6307
+ opts.captureServer?.(server);
4682
6308
  return new Promise((resolvePromise, reject) => {
4683
6309
  let retriedWithRandomPort = false;
4684
6310
  server.on('error', (error) => {
@@ -4913,6 +6539,145 @@ export function serializeJSON(value) {
4913
6539
  return current;
4914
6540
  });
4915
6541
  }
6542
+ /** Serialize a Skill to the shared API contract shape (spec 16). */
6543
+ function serializeSkill(skill) {
6544
+ return {
6545
+ id: skill.id,
6546
+ scope: skill.scope,
6547
+ user: skill.user,
6548
+ domain: skill.domain,
6549
+ description: skill.description,
6550
+ body: skill.body,
6551
+ preferredMetrics: skill.preferredMetrics,
6552
+ preferredBlocks: skill.preferredBlocks,
6553
+ vocabulary: skill.vocabulary,
6554
+ sourcePath: skill.sourcePath,
6555
+ isStarter: skill.isStarter,
6556
+ };
6557
+ }
6558
+ /**
6559
+ * Validate + normalize an inbound `{ skill }` body into a WriteSkillInput.
6560
+ * `id` + `scope` + `body` are required; `fallbackId` supplies the id on PUT
6561
+ * (from the URL). Returns null when the payload is invalid.
6562
+ */
6563
+ function parseSkillInput(raw, fallbackId) {
6564
+ if (!raw || typeof raw !== 'object')
6565
+ return null;
6566
+ const skill = raw;
6567
+ const id = typeof skill.id === 'string' && skill.id.trim() ? skill.id.trim() : fallbackId;
6568
+ if (!id)
6569
+ return null;
6570
+ const scope = skill.scope === 'personal' ? 'personal' : skill.scope === 'project' ? 'project' : undefined;
6571
+ if (!scope)
6572
+ return null;
6573
+ if (typeof skill.body !== 'string')
6574
+ return null;
6575
+ const asStrings = (value) => Array.isArray(value) ? value.filter((v) => typeof v === 'string') : [];
6576
+ const asMap = (value) => {
6577
+ if (!value || typeof value !== 'object' || Array.isArray(value))
6578
+ return {};
6579
+ const out = {};
6580
+ for (const [k, v] of Object.entries(value)) {
6581
+ if (typeof v === 'string')
6582
+ out[k] = v;
6583
+ }
6584
+ return out;
6585
+ };
6586
+ return {
6587
+ id,
6588
+ scope,
6589
+ user: scope === 'personal' && typeof skill.user === 'string' ? skill.user : undefined,
6590
+ domain: typeof skill.domain === 'string' && skill.domain.trim() ? skill.domain.trim() : undefined,
6591
+ description: typeof skill.description === 'string' ? skill.description : undefined,
6592
+ body: skill.body,
6593
+ preferredMetrics: asStrings(skill.preferredMetrics),
6594
+ preferredBlocks: asStrings(skill.preferredBlocks),
6595
+ vocabulary: asMap(skill.vocabulary),
6596
+ isStarter: skill.isStarter === true ? true : undefined,
6597
+ };
6598
+ }
6599
+ /** Loose, case/slug-insensitive domain key for counting membership. */
6600
+ function domainKey(value) {
6601
+ if (!value)
6602
+ return '';
6603
+ return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
6604
+ }
6605
+ function manifestDomainToDto(domain, counts) {
6606
+ return {
6607
+ id: domain.name,
6608
+ name: domain.name,
6609
+ owner: domain.owner,
6610
+ boundedContext: domain.boundedContext,
6611
+ sourceSystems: domain.sourceSystems,
6612
+ description: domain.description,
6613
+ sourcePath: domain.filePath,
6614
+ blockCount: counts.blockCount,
6615
+ skillCount: counts.skillCount,
6616
+ termCount: counts.termCount,
6617
+ };
6618
+ }
6619
+ /** List authored domains with per-domain block/skill/term counts. */
6620
+ export function listDomains(projectRoot) {
6621
+ const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
6622
+ const domains = manifest.domains ?? {};
6623
+ const skills = loadSkills(projectRoot).skills;
6624
+ const blockCounts = new Map();
6625
+ for (const block of Object.values(manifest.blocks)) {
6626
+ const key = domainKey(block.domain);
6627
+ if (key)
6628
+ blockCounts.set(key, (blockCounts.get(key) ?? 0) + 1);
6629
+ }
6630
+ const termCounts = new Map();
6631
+ for (const term of Object.values(manifest.terms ?? {})) {
6632
+ const key = domainKey(term.domain);
6633
+ if (key)
6634
+ termCounts.set(key, (termCounts.get(key) ?? 0) + 1);
6635
+ }
6636
+ const skillCounts = new Map();
6637
+ for (const skill of skills) {
6638
+ const key = domainKey(skill.domain);
6639
+ if (key)
6640
+ skillCounts.set(key, (skillCounts.get(key) ?? 0) + 1);
6641
+ }
6642
+ return Object.values(domains)
6643
+ .map((domain) => {
6644
+ const key = domainKey(domain.name);
6645
+ return manifestDomainToDto(domain, {
6646
+ blockCount: blockCounts.get(key) ?? 0,
6647
+ skillCount: skillCounts.get(key) ?? 0,
6648
+ termCount: termCounts.get(key) ?? 0,
6649
+ });
6650
+ })
6651
+ .sort((a, b) => a.name.localeCompare(b.name));
6652
+ }
6653
+ /** Find a single authored domain by name/id (case/slug-insensitive). */
6654
+ function findDomain(projectRoot, nameOrId) {
6655
+ const key = domainKey(nameOrId);
6656
+ return listDomains(projectRoot).find((domain) => domainKey(domain.name) === key);
6657
+ }
6658
+ /** Validate + normalize an inbound `{ domain }` body into a DomainInput. */
6659
+ export function parseDomainInput(raw, fallbackId) {
6660
+ if (!raw || typeof raw !== 'object')
6661
+ return null;
6662
+ const domain = raw;
6663
+ const name = typeof domain.name === 'string' && domain.name.trim()
6664
+ ? domain.name.trim()
6665
+ : typeof domain.id === 'string' && domain.id.trim()
6666
+ ? domain.id.trim()
6667
+ : fallbackId;
6668
+ if (!name)
6669
+ return null;
6670
+ const asStrings = (value) => Array.isArray(value)
6671
+ ? value.filter((v) => typeof v === 'string' && v.trim().length > 0)
6672
+ : undefined;
6673
+ return {
6674
+ name,
6675
+ owner: typeof domain.owner === 'string' ? domain.owner : undefined,
6676
+ boundedContext: typeof domain.boundedContext === 'string' ? domain.boundedContext : undefined,
6677
+ sourceSystems: asStrings(domain.sourceSystems),
6678
+ description: typeof domain.description === 'string' ? domain.description : undefined,
6679
+ };
6680
+ }
4916
6681
  async function refreshLocalMetadataCatalog(projectRoot) {
4917
6682
  try {
4918
6683
  await ensureMetadataCatalogFresh(projectRoot, { force: true });
@@ -4985,6 +6750,153 @@ export function loadProjectConfig(projectRoot) {
4985
6750
  }
4986
6751
  return config;
4987
6752
  }
6753
+ /**
6754
+ * Core of the `/api/propose` endpoint, factored out as a pure function so it can
6755
+ * be unit-tested without standing up an HTTP server.
6756
+ *
6757
+ * It reuses the existing `propose` engine from `@duckcodeailabs/dql-agent`
6758
+ * verbatim (no inference/ranking logic is duplicated here) in `dryRun` mode so a
6759
+ * readiness preview never mutates the project. Every returned proposal is a
6760
+ * `status: draft` block with the engine's stored Certifier verdict attached.
6761
+ */
6762
+ export function buildProposeReadiness(projectRoot, projectConfig = loadProjectConfig(projectRoot), options = {}) {
6763
+ const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
6764
+ if (!manifestPath) {
6765
+ return {
6766
+ ready: false,
6767
+ reason: 'No dbt manifest found. Run `dbt parse` (or `dbt compile`) in your dbt project, then reopen Get Started.',
6768
+ summary: {
6769
+ modelsScanned: 0,
6770
+ businessModels: 0,
6771
+ plumbingExcluded: 0,
6772
+ metricsFound: 0,
6773
+ proposalsRanked: 0,
6774
+ draftsExisting: 0,
6775
+ readyForReview: 0,
6776
+ blockingTotal: 0,
6777
+ warningTotal: 0,
6778
+ },
6779
+ plan: {
6780
+ totals: { modelsScanned: 0, businessModels: 0, plumbingExcluded: 0, metricsFound: 0 },
6781
+ willGenerate: 0,
6782
+ willSkip: 0,
6783
+ domains: [],
6784
+ config: {
6785
+ businessLayers: [],
6786
+ excludeLayers: [],
6787
+ maxPerDomain: 0,
6788
+ minScore: 0,
6789
+ aiEnrichment: 'auto',
6790
+ },
6791
+ },
6792
+ proposals: [],
6793
+ };
6794
+ }
6795
+ const proposeConfig = projectConfig.propose;
6796
+ // PLAN: deterministic, business-only, bounded. Writes nothing.
6797
+ const plan = proposePlan(projectRoot, manifestPath, { config: proposeConfig });
6798
+ // dryRun: rank + certify the selected scope only. Never writes from a preview.
6799
+ // Stamp the resolved local OSS owner when none was passed so the stored verdict
6800
+ // does not carry a phantom "Missing owner" strike. Read-only resolution — the
6801
+ // preview must not mutate the project.
6802
+ const summary = propose({
6803
+ projectRoot,
6804
+ dbtManifestPath: manifestPath,
6805
+ owner: options.owner || resolveLocalOwner(projectRoot, { persist: false }),
6806
+ limit: options.limit,
6807
+ dryRun: true,
6808
+ config: proposeConfig,
6809
+ });
6810
+ let readyForReview = 0;
6811
+ let blockingTotal = 0;
6812
+ let warningTotal = 0;
6813
+ for (const proposal of summary.proposals) {
6814
+ blockingTotal += proposal.certification.errors.length;
6815
+ warningTotal += proposal.certification.warnings.length;
6816
+ if (proposal.certification.errors.length === 0)
6817
+ readyForReview += 1;
6818
+ }
6819
+ return {
6820
+ ready: true,
6821
+ summary: {
6822
+ projectName: summary.projectName,
6823
+ modelsScanned: summary.modelsScanned,
6824
+ businessModels: summary.businessModels,
6825
+ plumbingExcluded: summary.plumbingExcluded,
6826
+ metricsFound: summary.metricsFound,
6827
+ proposalsRanked: summary.proposalsRanked,
6828
+ // In dryRun the engine marks already-present blocks as skipped.
6829
+ draftsExisting: summary.draftsSkipped,
6830
+ readyForReview,
6831
+ blockingTotal,
6832
+ warningTotal,
6833
+ },
6834
+ plan,
6835
+ proposals: summary.proposals,
6836
+ };
6837
+ }
6838
+ export async function generateProposeDrafts(projectRoot, slugs, projectConfig = loadProjectConfig(projectRoot), options = {}) {
6839
+ const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
6840
+ if (!manifestPath) {
6841
+ return {
6842
+ ready: false,
6843
+ reason: 'No dbt manifest found. Run `dbt parse` (or `dbt compile`) first.',
6844
+ draftsWritten: 0,
6845
+ draftsSkipped: 0,
6846
+ proposals: [],
6847
+ };
6848
+ }
6849
+ // Structure deterministic, content AI-optional: optionally pre-compute AI
6850
+ // enrichment (description / llmContext / examples) for the approved slugs, then
6851
+ // hand it to the deterministic engine as data. Best-effort — any failure or a
6852
+ // missing provider falls back to dbt-derived content.
6853
+ let enrichedBySlug;
6854
+ const proposeConfig = resolveProposeConfig(projectConfig.propose);
6855
+ if (proposeConfig.aiEnrichment !== 'off' && slugs.length > 0) {
6856
+ enrichedBySlug = await gatherProposeEnrichment(projectRoot, manifestPath, projectConfig.propose, slugs).catch(() => undefined);
6857
+ }
6858
+ const summary = propose({
6859
+ projectRoot,
6860
+ dbtManifestPath: manifestPath,
6861
+ // Stamp the resolved local OSS owner when none was passed so drafts are not
6862
+ // born with a "Missing owner" Certifier strike.
6863
+ owner: options.owner || resolveLocalOwner(projectRoot),
6864
+ config: projectConfig.propose,
6865
+ onlySlugs: slugs,
6866
+ enrichedBySlug,
6867
+ });
6868
+ return {
6869
+ ready: true,
6870
+ draftsWritten: summary.draftsWritten,
6871
+ draftsSkipped: summary.draftsSkipped,
6872
+ proposals: summary.proposals,
6873
+ };
6874
+ }
6875
+ /**
6876
+ * Build the FILLED transparent preview for ONE proposed candidate slug (spec 14,
6877
+ * part A). Reuses the deterministic `buildProposePreview` engine (real SQL +
6878
+ * Certifier verdict) and best-effort AI enrichment (description/llmContext/
6879
+ * examples) when a provider is available. Writes NOTHING. Returns `undefined`
6880
+ * when the slug is not part of the bounded, business-only selection.
6881
+ */
6882
+ export async function buildProposeCandidatePreview(projectRoot, slug, owner, projectConfig = loadProjectConfig(projectRoot)) {
6883
+ const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
6884
+ if (!manifestPath)
6885
+ return undefined;
6886
+ // Best-effort AI enrichment for this one slug (content only). Any miss falls
6887
+ // back to the deterministic dbt-derived content inside buildProposePreview.
6888
+ let enriched;
6889
+ const proposeConfig = resolveProposeConfig(projectConfig.propose);
6890
+ if (proposeConfig.aiEnrichment !== 'off') {
6891
+ const map = await gatherProposeEnrichment(projectRoot, manifestPath, projectConfig.propose, [slug]).catch(() => undefined);
6892
+ enriched = map?.get(slug);
6893
+ }
6894
+ return buildProposePreview(projectRoot, manifestPath, slug, {
6895
+ config: projectConfig.propose,
6896
+ owner: owner || resolveLocalOwner(projectRoot),
6897
+ enriched,
6898
+ });
6899
+ }
4988
6900
  function getProjectConnectionsForApi(config) {
4989
6901
  const connections = getStoredConnections(config);
4990
6902
  if (Object.keys(connections).length === 0 && isConnectionLike(config.defaultConnection)) {
@@ -4997,6 +6909,11 @@ const CONNECTOR_INSTALLS = {
4997
6909
  driver: 'duckdb',
4998
6910
  label: 'DuckDB',
4999
6911
  packageName: 'duckdb',
6912
+ // Latest 1.x. An earlier pin to 1.1.3 worked around a BIGINT serialization crash,
6913
+ // but that only bites a naive `JSON.stringify` — the DQL driver
6914
+ // (`normalizeDuckDBValue`) coerces BIGINT→number before marshaling and
6915
+ // `serializeJSON` has a BigInt replacer, so 1.4.x is verified-good on the local
6916
+ // DuckDB path (COUNT/AVG/SELECT * over UUID/BIGINT/decimal/datetime on real data).
5000
6917
  packageSpec: 'duckdb@^1.1.0',
5001
6918
  builtIn: false,
5002
6919
  },
@@ -5345,6 +7262,10 @@ function buildDashboardFilterPredicate(input) {
5345
7262
  const end = addDashboardFilterParam(input, 'end', range.end);
5346
7263
  return `${input.expression} BETWEEN $${start.position} AND $${end.position}`;
5347
7264
  }
7265
+ // A daterange with only one bound set is incomplete — skip it rather than binding
7266
+ // the partial object as a scalar equality (which would produce invalid SQL).
7267
+ if (input.filterType === 'daterange')
7268
+ return null;
5348
7269
  const values = Array.isArray(input.value) ? input.value.filter((item) => !isEmptyDashboardFilterValue(item)) : [input.value];
5349
7270
  if (values.length === 0)
5350
7271
  return null;
@@ -6191,6 +8112,16 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
6191
8112
  if (diagnostics.some((diagnostic) => diagnostic.severity === 'error')) {
6192
8113
  return { sql: null, diagnostics, semanticRefs };
6193
8114
  }
8115
+ // Import-adapter shape: a semantic block may carry a pre-compiled `query` (the
8116
+ // governed metric already expanded to runnable SQL). When present, RUN THAT —
8117
+ // the `metric` field is provenance/governance, not something to recompile. This
8118
+ // keeps metric-bound blocks runnable offline (the full MetricFlow engine may be
8119
+ // unavailable). We still validated the metric exists above. Returning the query
8120
+ // as the composed SQL lets the normal {{ ref() }} resolution + execution apply.
8121
+ const precompiledQuery = extractBlockStudioSql(source);
8122
+ if (precompiledQuery) {
8123
+ return { sql: precompiledQuery, diagnostics, semanticRefs };
8124
+ }
6194
8125
  let composed;
6195
8126
  try {
6196
8127
  composed = options?.projectRoot && options.projectConfig
@@ -6536,7 +8467,7 @@ function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
6536
8467
  }
6537
8468
  }
6538
8469
  export function saveBlockStudioArtifacts(projectRoot, options) {
6539
- const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
8470
+ const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
6540
8471
  const safeDomain = (options.domain ?? '')
6541
8472
  .trim()
6542
8473
  .toLowerCase()
@@ -6576,7 +8507,7 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
6576
8507
  return targetRelativePath;
6577
8508
  }
6578
8509
  export function saveBlockStudioDraftArtifacts(projectRoot, options) {
6579
- const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
8510
+ const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
6580
8511
  const safeDomain = (options.domain ?? '')
6581
8512
  .trim()
6582
8513
  .toLowerCase()
@@ -6719,8 +8650,132 @@ function readBlockCompanionFile(projectRoot, relativePath) {
6719
8650
  return null;
6720
8651
  }
6721
8652
  }
8653
+ /**
8654
+ * Extract the declared `invariants` from a block's DQL source using the core
8655
+ * parser (the same path that populates the manifest). Returns an empty array
8656
+ * when the source has no invariants or cannot be parsed — invariant evaluation
8657
+ * is best-effort and must never break a run.
8658
+ */
8659
+ export function extractBlockInvariants(source) {
8660
+ // Only DQL block sources declare invariants. Never hand a non-DQL cell source
8661
+ // (e.g. a raw SQL notebook cell like "SELECT 1") to the DQL parser: it can loop
8662
+ // on input it doesn't recognize and OOM-crash the runtime on every execute.
8663
+ // Require a `block "..."` declaration before parsing.
8664
+ if (!/(?:^|\n)\s*block\s+["']/.test(source))
8665
+ return [];
8666
+ try {
8667
+ const ast = new Parser(source).parse();
8668
+ const block = ast.statements.find((statement) => statement.kind === 'BlockDecl');
8669
+ return Array.isArray(block?.invariants) ? block.invariants : [];
8670
+ }
8671
+ catch {
8672
+ return [];
8673
+ }
8674
+ }
8675
+ /**
8676
+ * Evaluate a block's declared invariants against a normalized query result.
8677
+ * Returns `null` when the block declares no invariants so callers can omit the
8678
+ * field entirely (blocks without invariants behave exactly as before).
8679
+ */
8680
+ export function evaluateBlockInvariants(source, result) {
8681
+ const invariants = extractBlockInvariants(source);
8682
+ if (invariants.length === 0)
8683
+ return null;
8684
+ const invariantResults = evaluateInvariants(invariants, {
8685
+ columns: result.columns,
8686
+ rows: result.rows,
8687
+ });
8688
+ return { invariantResults, invariantViolation: hasInvariantViolation(invariantResults) };
8689
+ }
8690
+ /** Resolve a block id/slug to its `.dql` path (filename slug first, then `block "<id>"`). */
8691
+ function resolveBlockPathById(projectRoot, blockId) {
8692
+ const wanted = blockId.trim().toLowerCase();
8693
+ if (!wanted)
8694
+ return null;
8695
+ const leaf = wanted.split('.').pop() ?? wanted;
8696
+ const escaped = leaf.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
8697
+ const nameRe = new RegExp(`^\\s*block\\s+"${escaped}"`, 'im');
8698
+ const stack = ['blocks', 'domains'].map((dir) => join(projectRoot, dir));
8699
+ let nameMatch = null;
8700
+ while (stack.length > 0) {
8701
+ const dir = stack.pop();
8702
+ if (!existsSync(dir))
8703
+ continue;
8704
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
8705
+ const filePath = join(dir, entry.name);
8706
+ if (entry.isDirectory()) {
8707
+ stack.push(filePath);
8708
+ continue;
8709
+ }
8710
+ if (!entry.name.endsWith('.dql'))
8711
+ continue;
8712
+ const rel = relative(projectRoot, filePath).replaceAll('\\', '/');
8713
+ if (entry.name.replace(/\.dql$/, '').toLowerCase() === leaf)
8714
+ return rel; // fast path: filename slug
8715
+ if (!nameMatch) {
8716
+ try {
8717
+ if (nameRe.test(readFileSync(filePath, 'utf-8')))
8718
+ nameMatch = rel;
8719
+ }
8720
+ catch { /* skip */ }
8721
+ }
8722
+ }
8723
+ }
8724
+ return nameMatch;
8725
+ }
8726
+ /**
8727
+ * Collect the workspace's blocks as the App planner needs to see them (name +
8728
+ * governed metric + filterable dimensions). Walks blocks/ and domains/, parsing
8729
+ * each `.dql`. Certified-only by default — the planner builds an app from trusted
8730
+ * material and reports the rest as gaps.
8731
+ */
8732
+ function collectPlanBlocks(projectRoot, opts = {}) {
8733
+ const out = [];
8734
+ const seen = new Set();
8735
+ const stack = ['blocks', 'domains'].map((dir) => join(projectRoot, dir));
8736
+ while (stack.length > 0) {
8737
+ const dir = stack.pop();
8738
+ if (!existsSync(dir))
8739
+ continue;
8740
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
8741
+ const filePath = join(dir, entry.name);
8742
+ if (entry.isDirectory()) {
8743
+ stack.push(filePath);
8744
+ continue;
8745
+ }
8746
+ if (!entry.name.endsWith('.dql'))
8747
+ continue;
8748
+ let source;
8749
+ try {
8750
+ source = readFileSync(filePath, 'utf-8');
8751
+ }
8752
+ catch {
8753
+ continue;
8754
+ }
8755
+ const meta = parseBlockSourceMetadata(source);
8756
+ if (!meta.name || seen.has(meta.name))
8757
+ continue;
8758
+ if (opts.certifiedOnly && meta.status !== 'certified')
8759
+ continue;
8760
+ seen.add(meta.name);
8761
+ out.push({
8762
+ name: meta.name,
8763
+ domain: meta.domain || undefined,
8764
+ description: meta.description || undefined,
8765
+ metricRef: meta.metricRef || meta.metricsRef[0] || undefined,
8766
+ allowedFilters: meta.allowedFilters,
8767
+ dimensions: meta.dimensions,
8768
+ });
8769
+ }
8770
+ }
8771
+ return out;
8772
+ }
6722
8773
  export function parseBlockSourceMetadata(source) {
6723
- const name = source.match(/^\s*block\s+"([^"]+)"/i)?.[1] ?? '';
8774
+ // Multiline: a `.dql` typically opens with a comment header (e.g. a proposed-draft
8775
+ // banner), so the `block "<name>"` declaration is NOT at the start of the string.
8776
+ // Without `m` the name parsed empty and every certified block was saved as the
8777
+ // generic `block.dql` (slug fallback) — colliding on the 2nd certify (BLOCK_EXISTS).
8778
+ const name = source.match(/^\s*block\s+"([^"]+)"/im)?.[1] ?? '';
6724
8779
  const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
6725
8780
  const extractStringArray = (key) => {
6726
8781
  const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'i'));
@@ -6761,6 +8816,8 @@ export function parseBlockSourceMetadata(source) {
6761
8816
  sourceSystems: extractStringArray('sourceSystems'),
6762
8817
  replacementFor: extractStringArray('replacementFor'),
6763
8818
  reviewCadence: extractString('reviewCadence'),
8819
+ metricRef: extractString('metric'),
8820
+ metricsRef: extractStringArray('metrics'),
6764
8821
  };
6765
8822
  }
6766
8823
  function compareBlockStudioValues(actual, operator, expected) {
@@ -6810,14 +8867,15 @@ function buildBlockStudioCertificationChecklist(input) {
6810
8867
  blockers.add(blocker);
6811
8868
  if (!input.previewSucceeded)
6812
8869
  blockers.add('Block has not run successfully');
6813
- if (!input.testResults || input.testResults.failed > 0)
8870
+ // Only a REAL test failure blocks. "No tests" and "no chart" are advisory review
8871
+ // items the AI fills at draft time — never hard gates (owner is the gate). This is
8872
+ // what keeps "build a DQL" from demanding ~13 fields per block at 300-query scale.
8873
+ if (input.testResults && input.testResults.failed > 0)
6814
8874
  blockers.add('Tests must pass before certification');
6815
- if (!input.testResults || input.testResults.assertions.length === 0)
6816
- blockers.add('At least one test assertion is required before certification');
6817
- if (!input.validation.chartConfig?.chart)
6818
- blockers.add('Visualization config is missing');
6819
8875
  return {
6820
- metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
8876
+ // The metadata tab is green once the one required field (owner) is present.
8877
+ // Description/domain are AI-filled + editable and surface as review items, not gates.
8878
+ metadata: Boolean(parsed.owner.trim()),
6821
8879
  validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
6822
8880
  run: input.previewSucceeded,
6823
8881
  tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
@@ -7828,13 +9886,13 @@ async function createBlockStudioAssistProvider(projectRoot, requestedProvider) {
7828
9886
  let provider;
7829
9887
  switch (selected.id) {
7830
9888
  case 'anthropic':
7831
- provider = new ClaudeProvider({ apiKey: config.apiKey, model: config.model });
9889
+ provider = new ClaudeProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
7832
9890
  break;
7833
9891
  case 'openai':
7834
9892
  provider = new OpenAIProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
7835
9893
  break;
7836
9894
  case 'gemini':
7837
- provider = new GeminiProvider({ apiKey: config.apiKey, model: config.model });
9895
+ provider = new GeminiProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
7838
9896
  break;
7839
9897
  case 'ollama':
7840
9898
  provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
@@ -7975,7 +10033,7 @@ function resolveBlockWriteTarget(projectRoot, safeDomain, slug) {
7975
10033
  return { relativePath, absPath: join(projectRoot, relativePath) };
7976
10034
  }
7977
10035
  export function createBlockArtifacts(projectRoot, options) {
7978
- const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
10036
+ const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
7979
10037
  const safeDomain = (options.domain ?? '')
7980
10038
  .trim()
7981
10039
  .toLowerCase()
@@ -8030,7 +10088,7 @@ export function createBlockArtifacts(projectRoot, options) {
8030
10088
  };
8031
10089
  }
8032
10090
  export function createSemanticBuilderBlock(projectRoot, options) {
8033
- const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
10091
+ const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
8034
10092
  const safeDomain = (options.domain ?? '')
8035
10093
  .trim()
8036
10094
  .toLowerCase()
@@ -8493,6 +10551,19 @@ export function discoverDbtProfileConnections(projectRoot, projectConfig) {
8493
10551
  const mapped = mapDbtProfileOutput(output);
8494
10552
  if (!mapped)
8495
10553
  continue;
10554
+ // dbt resolves a relative duckdb `path` against the dbt project dir (where it runs),
10555
+ // NOT the DQL workspace. Resolve to an absolute path here so the imported connection
10556
+ // opens the real warehouse instead of silently creating an empty db next to dql.config.json.
10557
+ if (mapped.adapter === 'duckdb' &&
10558
+ mapped.connection.filepath &&
10559
+ mapped.connection.filepath !== ':memory:') {
10560
+ if (!isAbsoluteLikePath(mapped.connection.filepath)) {
10561
+ mapped.connection.filepath = resolve(dbtProjectPath, mapped.connection.filepath);
10562
+ }
10563
+ if (!existsSync(mapped.connection.filepath)) {
10564
+ mapped.warnings.push(`DuckDB file not found at ${mapped.connection.filepath} — run \`dbt build\` (or \`dbt seed\`) to create it before querying.`);
10565
+ }
10566
+ }
8496
10567
  const warnings = [...mapped.warnings];
8497
10568
  if (targetName !== defaultTarget) {
8498
10569
  warnings.push(`Not the default dbt target "${defaultTarget}".`);
@@ -9605,7 +11676,7 @@ async function testProviderConfig(projectRoot, id) {
9605
11676
  let provider;
9606
11677
  switch (id) {
9607
11678
  case 'gemini':
9608
- provider = new GeminiProvider({ apiKey: config.apiKey, model: config.model });
11679
+ provider = new GeminiProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
9609
11680
  break;
9610
11681
  case 'ollama':
9611
11682
  provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
@@ -9666,7 +11737,10 @@ async function testAnthropicProviderConfig(config, label, details) {
9666
11737
  return { ok: false, message: `${label} is not configured${details}. Add an API key in Settings or ANTHROPIC_API_KEY.` };
9667
11738
  }
9668
11739
  try {
9669
- const client = new Anthropic({ apiKey: config.apiKey });
11740
+ const client = new Anthropic({
11741
+ apiKey: config.apiKey,
11742
+ ...(config.baseUrl ? { baseURL: normalizeAnthropicBaseUrl(config.baseUrl) } : {}),
11743
+ });
9670
11744
  const response = await client.messages.create({
9671
11745
  model: config.model ?? 'claude-opus-4-8',
9672
11746
  max_tokens: 16,