@duckcodeailabs/dql-cli 1.6.20 → 1.6.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts +216 -2
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +1767 -46
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +10 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +19 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/{codemirror-DJYUkPr1.js → codemirror-Co0dCEDr.js} +1 -1
- package/dist/assets/dql-notebook/assets/index-D9Owd4EN.css +1 -0
- package/dist/assets/dql-notebook/assets/index-nDZHVgfg.js +5636 -0
- package/dist/assets/dql-notebook/assets/{react-CRB3T2We.js → react-BRY8hXab.js} +2 -2
- package/dist/assets/dql-notebook/index.html +4 -4
- package/dist/commands/agent.d.ts +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +50 -10
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts +4 -4
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +22 -19
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/certify.d.ts.map +1 -1
- package/dist/commands/certify.js +70 -25
- package/dist/commands/certify.js.map +1 -1
- package/dist/commands/diff.d.ts.map +1 -1
- package/dist/commands/diff.js +108 -5
- package/dist/commands/diff.js.map +1 -1
- package/dist/commands/doctor.d.ts +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +68 -7
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/eval.d.ts +121 -0
- package/dist/commands/eval.d.ts.map +1 -0
- package/dist/commands/eval.js +350 -0
- package/dist/commands/eval.js.map +1 -0
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +50 -10
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/notebook.d.ts +15 -0
- package/dist/commands/notebook.d.ts.map +1 -1
- package/dist/commands/notebook.js +32 -11
- package/dist/commands/notebook.js.map +1 -1
- package/dist/commands/propose.d.ts +23 -0
- package/dist/commands/propose.d.ts.map +1 -0
- package/dist/commands/propose.js +184 -0
- package/dist/commands/propose.js.map +1 -0
- package/dist/index.js +64 -11
- package/dist/index.js.map +1 -1
- package/dist/llm/analytics-tools.d.ts.map +1 -1
- package/dist/llm/analytics-tools.js +9 -4
- package/dist/llm/analytics-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +21 -32
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/providers/native-sdk-provider.d.ts.map +1 -1
- package/dist/llm/providers/native-sdk-provider.js +5 -2
- package/dist/llm/providers/native-sdk-provider.js.map +1 -1
- package/dist/local-runtime.d.ts +132 -4
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +2120 -46
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/propose-enrich.d.ts +13 -0
- package/dist/propose-enrich.d.ts.map +1 -0
- package/dist/propose-enrich.js +34 -0
- package/dist/propose-enrich.js.map +1 -0
- package/dist/settings/provider-settings.js +2 -2
- package/dist/settings/provider-settings.js.map +1 -1
- package/package.json +11 -11
- package/dist/assets/dql-notebook/assets/index-Dgqvyn5s.js +0 -3886
- package/dist/assets/dql-notebook/assets/index-RaDW1A5g.css +0 -1
package/dist/local-runtime.js
CHANGED
|
@@ -7,21 +7,23 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
|
|
|
7
7
|
import Anthropic from '@anthropic-ai/sdk';
|
|
8
8
|
import OpenAI from 'openai';
|
|
9
9
|
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
10
|
-
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
|
|
10
|
+
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, writeDomainDeclaration, deleteDomainDeclaration, domainFolderSlug, } from '@duckcodeailabs/dql-core';
|
|
11
11
|
import { load as loadYaml } from 'js-yaml';
|
|
12
12
|
import { listBlockTemplates } from './block-templates.js';
|
|
13
13
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
14
14
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
15
|
-
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordQueryRun, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
|
|
16
|
-
import {
|
|
15
|
+
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, } from '@duckcodeailabs/dql-agent';
|
|
16
|
+
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
17
|
+
import { createAppAiBuildSession, handleAppsApi, recommendVisualization } from './apps-api.js';
|
|
17
18
|
import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
18
19
|
import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
|
|
19
20
|
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
20
|
-
import { Certifier, ENTERPRISE_RULES } from '@duckcodeailabs/dql-governance';
|
|
21
|
+
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
21
22
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
22
23
|
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
23
24
|
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
|
|
24
25
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
26
|
+
const NOTEBOOK_FAVICON_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="7" fill="#6d5dfc"/><path d="M9 9h14v14H9z" fill="none" stroke="#fff" stroke-width="2"/><path d="M13 13h6M13 17h6M13 21h4" stroke="#fff" stroke-width="2" stroke-linecap="round"/></svg>';
|
|
25
27
|
export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
|
|
26
28
|
const configured = projectConfig.semanticLayer;
|
|
27
29
|
const dbtProjectDir = projectConfig.dbt?.projectDir;
|
|
@@ -51,6 +53,82 @@ function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
|
|
|
51
53
|
}
|
|
52
54
|
return false;
|
|
53
55
|
}
|
|
56
|
+
const AGENT_RUN_REQUESTED_MODES = new Set(['auto', 'ask', 'research', 'sql', 'block', 'app']);
|
|
57
|
+
const AGENT_RUN_SELECTED_OBJECT_KINDS = new Set([
|
|
58
|
+
'notebook',
|
|
59
|
+
'cell',
|
|
60
|
+
'block',
|
|
61
|
+
'app',
|
|
62
|
+
'dashboard',
|
|
63
|
+
'research',
|
|
64
|
+
'workspace',
|
|
65
|
+
]);
|
|
66
|
+
function agentRunRecord(value) {
|
|
67
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
|
|
68
|
+
}
|
|
69
|
+
function agentRunString(value) {
|
|
70
|
+
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
|
71
|
+
}
|
|
72
|
+
function parseAgentRunRequestedMode(value) {
|
|
73
|
+
return typeof value === 'string' && AGENT_RUN_REQUESTED_MODES.has(value)
|
|
74
|
+
? value
|
|
75
|
+
: undefined;
|
|
76
|
+
}
|
|
77
|
+
function parseAgentRunSelectedObject(value) {
|
|
78
|
+
const record = agentRunRecord(value);
|
|
79
|
+
if (!record)
|
|
80
|
+
return undefined;
|
|
81
|
+
const kind = agentRunString(record.kind);
|
|
82
|
+
if (!kind || !AGENT_RUN_SELECTED_OBJECT_KINDS.has(kind))
|
|
83
|
+
return undefined;
|
|
84
|
+
return {
|
|
85
|
+
kind: kind,
|
|
86
|
+
id: agentRunString(record.id),
|
|
87
|
+
title: agentRunString(record.title),
|
|
88
|
+
path: agentRunString(record.path),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function parseAgentRunHistory(value) {
|
|
92
|
+
if (!Array.isArray(value))
|
|
93
|
+
return undefined;
|
|
94
|
+
const history = value.flatMap((item) => {
|
|
95
|
+
const record = agentRunRecord(item);
|
|
96
|
+
if (!record)
|
|
97
|
+
return [];
|
|
98
|
+
const role = record.role === 'user' || record.role === 'assistant' ? record.role : undefined;
|
|
99
|
+
const text = agentRunString(record.text) ?? agentRunString(record.content);
|
|
100
|
+
return role && text ? [{ role, text }] : [];
|
|
101
|
+
});
|
|
102
|
+
return history.length > 0 ? history.slice(-20) : undefined;
|
|
103
|
+
}
|
|
104
|
+
function parseAgentRunRequestBody(body) {
|
|
105
|
+
const record = agentRunRecord(body);
|
|
106
|
+
if (!record)
|
|
107
|
+
return { error: 'Invalid JSON body.' };
|
|
108
|
+
const question = agentRunString(record.question) ?? agentRunString(record.prompt) ?? agentRunString(record.message);
|
|
109
|
+
if (!question)
|
|
110
|
+
return { error: 'question is required.' };
|
|
111
|
+
const selectedObject = parseAgentRunSelectedObject(record.selectedObject);
|
|
112
|
+
const workspaceContext = agentRunRecord(record.workspaceContext) ?? agentRunRecord(record.context);
|
|
113
|
+
const signals = agentRunRecord(record.signals);
|
|
114
|
+
const requestedMode = parseAgentRunRequestedMode(record.requestedMode) ?? parseAgentRunRequestedMode(record.mode);
|
|
115
|
+
const audience = record.audience === 'stakeholder' || record.audience === 'analyst'
|
|
116
|
+
? record.audience
|
|
117
|
+
: undefined;
|
|
118
|
+
return {
|
|
119
|
+
request: {
|
|
120
|
+
question,
|
|
121
|
+
requestedMode,
|
|
122
|
+
audience,
|
|
123
|
+
intent: agentRunString(record.intent),
|
|
124
|
+
signals: signals,
|
|
125
|
+
selectedObject,
|
|
126
|
+
workspaceContext,
|
|
127
|
+
history: parseAgentRunHistory(record.history),
|
|
128
|
+
runId: agentRunString(record.runId),
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
}
|
|
54
132
|
export async function startLocalServer(opts) {
|
|
55
133
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
56
134
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
@@ -126,6 +204,582 @@ export async function startLocalServer(opts) {
|
|
|
126
204
|
const result = await executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
|
|
127
205
|
return normalizeQueryResult(result, semantic.semanticRefs);
|
|
128
206
|
};
|
|
207
|
+
const runBlockReflectionProbe = async ({ sql, invariants }) => {
|
|
208
|
+
const activeConnection = requireActiveConnection();
|
|
209
|
+
const prepared = prepareLocalExecution(sql, activeConnection, projectRoot, projectConfig);
|
|
210
|
+
const probeSql = `SELECT * FROM (${stripSqlTerminator(prepared.sql)}) _dql_probe LIMIT 2000`;
|
|
211
|
+
const probeResult = await executor.executeQuery(probeSql, [], runtimeVariables({}), prepared.connection);
|
|
212
|
+
const rows = (Array.isArray(probeResult?.rows) ? probeResult.rows : []);
|
|
213
|
+
const rawColumns = Array.isArray(probeResult?.columns)
|
|
214
|
+
? probeResult.columns
|
|
215
|
+
: [];
|
|
216
|
+
const actualColumns = rawColumns.length > 0
|
|
217
|
+
? rawColumns.map((c) => (typeof c === 'string' ? c : c?.name ?? String(c)))
|
|
218
|
+
: (rows[0] ? Object.keys(rows[0]) : []);
|
|
219
|
+
const invariantResults = evaluateInvariants(invariants, { columns: actualColumns, rows });
|
|
220
|
+
const passed = invariantResults.filter((r) => r.passed && !r.uncheckable).length;
|
|
221
|
+
const failed = invariantResults.filter((r) => !r.passed && !r.uncheckable).length;
|
|
222
|
+
return {
|
|
223
|
+
actualColumns,
|
|
224
|
+
invariantResults,
|
|
225
|
+
tests: invariants.length > 0
|
|
226
|
+
? { passed, failed, assertionCount: invariantResults.length }
|
|
227
|
+
: undefined,
|
|
228
|
+
};
|
|
229
|
+
};
|
|
230
|
+
const agentRunWorkspaceValue = (request, key) => {
|
|
231
|
+
const workspace = request.workspaceContext ?? {};
|
|
232
|
+
const nested = agentRunRecord(workspace.context);
|
|
233
|
+
return agentRunString(workspace[key]) ?? (nested ? agentRunString(nested[key]) : undefined);
|
|
234
|
+
};
|
|
235
|
+
const agentRunNotebookPath = (request, runId) => (agentRunWorkspaceValue(request, 'notebookPath')
|
|
236
|
+
?? (request.selectedObject?.kind === 'notebook' || request.selectedObject?.kind === 'cell' ? request.selectedObject.path : undefined)
|
|
237
|
+
?? `notebooks/agent-research/${runId}.dqlnb`);
|
|
238
|
+
const agentRunResearchIntent = (request) => {
|
|
239
|
+
switch (request.intent) {
|
|
240
|
+
case 'diagnose_change':
|
|
241
|
+
return 'diagnose_change';
|
|
242
|
+
case 'driver_breakdown':
|
|
243
|
+
return 'driver_breakdown';
|
|
244
|
+
case 'segment_compare':
|
|
245
|
+
return 'segment_compare';
|
|
246
|
+
case 'entity_drilldown':
|
|
247
|
+
return 'entity_drilldown';
|
|
248
|
+
case 'anomaly_investigation':
|
|
249
|
+
return 'anomaly_investigation';
|
|
250
|
+
case 'trust_gap_review':
|
|
251
|
+
return 'trust_gap_review';
|
|
252
|
+
default:
|
|
253
|
+
if (/\b(driver|why|cause|contributor|breakdown)\b/i.test(request.question))
|
|
254
|
+
return 'driver_breakdown';
|
|
255
|
+
if (/\b(anomaly|spike|drop|outlier)\b/i.test(request.question))
|
|
256
|
+
return 'anomaly_investigation';
|
|
257
|
+
if (/\b(compare|segment|cohort)\b/i.test(request.question))
|
|
258
|
+
return 'segment_compare';
|
|
259
|
+
return 'ad_hoc_analysis';
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
const agentRunSourceCell = (request) => {
|
|
263
|
+
const sourceCellId = agentRunWorkspaceValue(request, 'sourceCellId') ?? request.selectedObject?.id;
|
|
264
|
+
if (!sourceCellId)
|
|
265
|
+
return undefined;
|
|
266
|
+
return {
|
|
267
|
+
id: sourceCellId,
|
|
268
|
+
sourceCellId,
|
|
269
|
+
name: agentRunWorkspaceValue(request, 'sourceCellName') ?? request.selectedObject?.title,
|
|
270
|
+
sourceCellName: agentRunWorkspaceValue(request, 'sourceCellName') ?? request.selectedObject?.title,
|
|
271
|
+
type: agentRunWorkspaceValue(request, 'sourceCellType'),
|
|
272
|
+
sql: agentRunWorkspaceValue(request, 'cellSql'),
|
|
273
|
+
fingerprint: agentRunWorkspaceValue(request, 'sourceCellFingerprint'),
|
|
274
|
+
};
|
|
275
|
+
};
|
|
276
|
+
const agentRunTitle = (question, fallback) => {
|
|
277
|
+
const cleaned = question.replace(/\s+/g, ' ').trim();
|
|
278
|
+
if (!cleaned)
|
|
279
|
+
return fallback;
|
|
280
|
+
return cleaned.length > 90 ? `${cleaned.slice(0, 87)}...` : cleaned;
|
|
281
|
+
};
|
|
282
|
+
const parseAgentRunSelectedBlockIds = (request) => {
|
|
283
|
+
const workspace = request.workspaceContext ?? {};
|
|
284
|
+
const value = workspace.selectedBlockIds ?? workspace.blockIds;
|
|
285
|
+
if (!Array.isArray(value))
|
|
286
|
+
return [];
|
|
287
|
+
return Array.from(new Set(value.flatMap((item) => {
|
|
288
|
+
const id = agentRunString(item);
|
|
289
|
+
return id ? [id] : [];
|
|
290
|
+
})));
|
|
291
|
+
};
|
|
292
|
+
const formatAgentRunInfrastructureError = (error, scope) => {
|
|
293
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
294
|
+
if (/Could not locate the bindings file/i.test(message) || /better[-_]sqlite3/i.test(message)) {
|
|
295
|
+
return `${scope} is unavailable because the local SQLite native bindings are not installed for this Node.js runtime.`;
|
|
296
|
+
}
|
|
297
|
+
return message;
|
|
298
|
+
};
|
|
299
|
+
const formatNotebookResearchStorageError = (error) => (formatAgentRunInfrastructureError(error, 'Notebook research storage'));
|
|
300
|
+
const buildAgentPromptArtifact = async (request, target, repair) => {
|
|
301
|
+
try {
|
|
302
|
+
await reindexProject(projectRoot, { kgPath: defaultKgPath(projectRoot) });
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
// Best-effort: buildFromPrompt can still use any existing KG/cache state.
|
|
306
|
+
}
|
|
307
|
+
const skills = loadSkills(projectRoot).skills;
|
|
308
|
+
// On a repair re-run, target the prior failure and (for blocks) revise in place.
|
|
309
|
+
const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
|
|
310
|
+
const mode = target === 'block' && (isRepair || agentRunWorkspaceValue(request, 'mode') === 'edit')
|
|
311
|
+
? 'edit'
|
|
312
|
+
: 'create';
|
|
313
|
+
const prompt = isRepair
|
|
314
|
+
? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}`
|
|
315
|
+
: request.question;
|
|
316
|
+
return buildFromPrompt({
|
|
317
|
+
projectRoot,
|
|
318
|
+
prompt,
|
|
319
|
+
context: {
|
|
320
|
+
cellSql: agentRunWorkspaceValue(request, 'cellSql'),
|
|
321
|
+
selection: agentRunWorkspaceValue(request, 'selection'),
|
|
322
|
+
},
|
|
323
|
+
target,
|
|
324
|
+
mode,
|
|
325
|
+
blockPath: target === 'block'
|
|
326
|
+
? agentRunWorkspaceValue(request, 'blockPath') ?? request.selectedObject?.path
|
|
327
|
+
: undefined,
|
|
328
|
+
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
329
|
+
domain: target === 'block' ? agentRunWorkspaceValue(request, 'domain') : undefined,
|
|
330
|
+
userId: agentRunWorkspaceValue(request, 'userId'),
|
|
331
|
+
skills,
|
|
332
|
+
dbtManifestPath: resolveDbtManifestPath(projectRoot, projectConfig),
|
|
333
|
+
executionProbe: target === 'block' ? runBlockReflectionProbe : undefined,
|
|
334
|
+
});
|
|
335
|
+
};
|
|
336
|
+
const agentRunEvaluation = (id, label, passed, severity, message, evidence) => ({ id, label, passed, severity, message, evidence });
|
|
337
|
+
const agentRunArtifact = (kind, title, payload, ref, trustState = 'review_required') => ({
|
|
338
|
+
id: `${kind}:${Date.now()}`,
|
|
339
|
+
kind,
|
|
340
|
+
title,
|
|
341
|
+
trustState,
|
|
342
|
+
ref,
|
|
343
|
+
payload,
|
|
344
|
+
});
|
|
345
|
+
const coerceNarrateResultData = (value) => {
|
|
346
|
+
const record = agentRunRecord(value);
|
|
347
|
+
if (!record)
|
|
348
|
+
return undefined;
|
|
349
|
+
const columns = Array.isArray(record.columns)
|
|
350
|
+
? record.columns.map((c) => (typeof c === 'string' ? c : agentRunRecord(c)?.name ?? String(c)))
|
|
351
|
+
: [];
|
|
352
|
+
const rows = Array.isArray(record.rows) ? record.rows.filter((r) => Boolean(agentRunRecord(r))) : [];
|
|
353
|
+
if (rows.length === 0)
|
|
354
|
+
return undefined;
|
|
355
|
+
return { columns: columns.length > 0 ? columns : Object.keys(rows[0]), rows };
|
|
356
|
+
};
|
|
357
|
+
// Provider-backed narration for stakeholder stories. Reuses the same provider
|
|
358
|
+
// adapter as the planner; narrateResult always returns (deterministic fallback).
|
|
359
|
+
const narrateForAgentRun = async (input) => narrateResult(input, {
|
|
360
|
+
complete: async ({ system, user, signal }) => {
|
|
361
|
+
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
362
|
+
if (!provider)
|
|
363
|
+
throw new Error('No AI provider configured for narration.');
|
|
364
|
+
return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
|
|
365
|
+
},
|
|
366
|
+
});
|
|
367
|
+
async function runGovernedAgentAnswerForRun(request, repair) {
|
|
368
|
+
const resolvedProvider = resolveDefaultLLMProvider(projectRoot);
|
|
369
|
+
const runner = resolvedProvider ? getLLMRunner(resolvedProvider) : null;
|
|
370
|
+
if (!resolvedProvider || !runner) {
|
|
371
|
+
throw new Error('No AI provider is configured. Configure OpenAI, Gemini, Ollama, or a custom OpenAI-compatible endpoint in Settings.');
|
|
372
|
+
}
|
|
373
|
+
let governedAnswer;
|
|
374
|
+
let providerError;
|
|
375
|
+
const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
|
|
376
|
+
const contextEnvelope = {
|
|
377
|
+
mode: 'agent_run',
|
|
378
|
+
selectedObject: request.selectedObject,
|
|
379
|
+
workspaceContext: request.workspaceContext,
|
|
380
|
+
instruction: [
|
|
381
|
+
'Route through the governed DQL answer loop.',
|
|
382
|
+
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
383
|
+
'Generated SQL remains review-required and must use the bounded preview executor.',
|
|
384
|
+
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
385
|
+
...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
|
|
386
|
+
].join(' '),
|
|
387
|
+
};
|
|
388
|
+
const controller = new AbortController();
|
|
389
|
+
await runner.run({
|
|
390
|
+
provider: resolvedProvider,
|
|
391
|
+
messages: [
|
|
392
|
+
...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
|
|
393
|
+
{ role: 'user', content: isRepair ? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}` : request.question },
|
|
394
|
+
],
|
|
395
|
+
upstream: {
|
|
396
|
+
cellId: `agent-run:${request.selectedObject?.kind ?? 'workspace'}:${request.selectedObject?.id ?? request.runId ?? 'auto'}`,
|
|
397
|
+
sql: JSON.stringify(contextEnvelope, null, 2),
|
|
398
|
+
},
|
|
399
|
+
projectRoot,
|
|
400
|
+
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
401
|
+
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
402
|
+
getSchemaContext: getSchemaContextForAgent,
|
|
403
|
+
}, (turn) => {
|
|
404
|
+
if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
|
|
405
|
+
governedAnswer = turn.output;
|
|
406
|
+
}
|
|
407
|
+
if (turn.kind === 'error') {
|
|
408
|
+
providerError = turn.message;
|
|
409
|
+
}
|
|
410
|
+
}, controller.signal);
|
|
411
|
+
if (!governedAnswer) {
|
|
412
|
+
throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
|
|
413
|
+
}
|
|
414
|
+
return governedAnswer;
|
|
415
|
+
}
|
|
416
|
+
const answerRunExecutor = async ({ request, routeDecision, attempt, repairHint }) => {
|
|
417
|
+
let governedAnswer;
|
|
418
|
+
try {
|
|
419
|
+
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint });
|
|
420
|
+
// Surface the approved Hint-Graph corrections that shaped this answer so the
|
|
421
|
+
// UI can show an "applied learnings" chip (memoryContext is already on the answer).
|
|
422
|
+
if (!governedAnswer.appliedHints) {
|
|
423
|
+
governedAnswer.appliedHints = governedAnswer.contextPack?.appliedHints;
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
catch (error) {
|
|
427
|
+
const message = formatAgentRunInfrastructureError(error, 'AI answer provider');
|
|
428
|
+
return {
|
|
429
|
+
summary: message,
|
|
430
|
+
status: 'blocked',
|
|
431
|
+
trustState: 'blocked',
|
|
432
|
+
stopReason: 'blocked',
|
|
433
|
+
evaluations: [
|
|
434
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'),
|
|
435
|
+
agentRunEvaluation('ai-provider', 'AI provider', false, 'blocking', message, { originalErrorType: error instanceof Error ? error.name : typeof error }),
|
|
436
|
+
],
|
|
437
|
+
nextActions: [
|
|
438
|
+
{ id: 'retry-ask-after-provider', label: 'Retry after provider setup', route: 'generated_answer' },
|
|
439
|
+
{ id: 'research-without-answer', label: 'Research with available metadata', route: 'research', artifactKind: 'research_run' },
|
|
440
|
+
],
|
|
441
|
+
};
|
|
442
|
+
}
|
|
443
|
+
const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
444
|
+
const needsClarification = governedAnswer.kind === 'no_answer';
|
|
445
|
+
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
446
|
+
const status = needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
|
|
447
|
+
const trustState = needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
|
|
448
|
+
const stopReason = needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
|
|
449
|
+
const nextActions = needsClarification
|
|
450
|
+
? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
|
|
451
|
+
: [
|
|
452
|
+
...(sql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
|
|
453
|
+
{ id: 'research-gap', label: 'Research deeper', route: 'research' },
|
|
454
|
+
{ id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
455
|
+
];
|
|
456
|
+
return {
|
|
457
|
+
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
|
|
458
|
+
answer: governedAnswer.answer ?? governedAnswer.text,
|
|
459
|
+
status,
|
|
460
|
+
trustState,
|
|
461
|
+
stopReason,
|
|
462
|
+
artifacts: needsClarification
|
|
463
|
+
? []
|
|
464
|
+
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
|
|
465
|
+
evaluations: [
|
|
466
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.'),
|
|
467
|
+
agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
|
|
468
|
+
? 'The answer came from certified DQL context.'
|
|
469
|
+
: needsClarification
|
|
470
|
+
? 'The answer loop needs more context before producing a governed answer.'
|
|
471
|
+
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
472
|
+
...(governedAnswer.executionError ? [
|
|
473
|
+
agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
|
|
474
|
+
] : []),
|
|
475
|
+
],
|
|
476
|
+
nextActions,
|
|
477
|
+
};
|
|
478
|
+
};
|
|
479
|
+
const agentRunExecutors = {
|
|
480
|
+
certified_answer: answerRunExecutor,
|
|
481
|
+
generated_answer: answerRunExecutor,
|
|
482
|
+
research: async ({ runId, request, routeDecision, emit }) => {
|
|
483
|
+
const metrics = loadSemanticMetrics(projectRoot);
|
|
484
|
+
let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
485
|
+
const usedCertifiedOnly = blocks.length > 0;
|
|
486
|
+
if (blocks.length === 0)
|
|
487
|
+
blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
|
|
488
|
+
emit({
|
|
489
|
+
type: 'executor.started',
|
|
490
|
+
message: 'Building catalog-grounded research plan.',
|
|
491
|
+
route: 'research',
|
|
492
|
+
});
|
|
493
|
+
const plan = await planResearch({
|
|
494
|
+
question: request.question,
|
|
495
|
+
metrics,
|
|
496
|
+
blocks,
|
|
497
|
+
intent: request.intent,
|
|
498
|
+
isFollowUp: Boolean(request.history?.length),
|
|
499
|
+
history: request.history,
|
|
500
|
+
});
|
|
501
|
+
const needsClarification = Boolean(plan.followUp);
|
|
502
|
+
const notebookPath = agentRunNotebookPath(request, runId);
|
|
503
|
+
const researchIntent = agentRunResearchIntent(request);
|
|
504
|
+
let researchRun;
|
|
505
|
+
let researchWorkspaceError;
|
|
506
|
+
if (!needsClarification) {
|
|
507
|
+
try {
|
|
508
|
+
const storage = openNotebookResearchStorage();
|
|
509
|
+
try {
|
|
510
|
+
const sourceCell = agentRunSourceCell(request);
|
|
511
|
+
const sourceCellId = notebookResearchSourceCellId(sourceCell);
|
|
512
|
+
const sourceCellName = notebookResearchSourceCellName(sourceCell);
|
|
513
|
+
const sourceCellFingerprint = notebookResearchSourceCellFingerprint(sourceCell);
|
|
514
|
+
const created = storage.createRun({
|
|
515
|
+
notebookPath,
|
|
516
|
+
title: agentRunTitle(request.question, 'Agent research'),
|
|
517
|
+
question: request.question,
|
|
518
|
+
sourceCell,
|
|
519
|
+
sourceCellId,
|
|
520
|
+
sourceCellName,
|
|
521
|
+
sourceCellFingerprint,
|
|
522
|
+
intent: researchIntent,
|
|
523
|
+
domain: agentRunWorkspaceValue(request, 'domain'),
|
|
524
|
+
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
525
|
+
context: {
|
|
526
|
+
surface: 'unified_agent_run',
|
|
527
|
+
agentRunId: runId,
|
|
528
|
+
routeDecision,
|
|
529
|
+
selectedObject: request.selectedObject,
|
|
530
|
+
workspaceContext: request.workspaceContext,
|
|
531
|
+
plan,
|
|
532
|
+
},
|
|
533
|
+
});
|
|
534
|
+
emit({
|
|
535
|
+
type: 'artifact.created',
|
|
536
|
+
message: 'Saved notebook research workspace record.',
|
|
537
|
+
route: 'research',
|
|
538
|
+
trustState: 'review_required',
|
|
539
|
+
payload: { researchRunId: created.id, notebookPath },
|
|
540
|
+
});
|
|
541
|
+
const executed = await runNotebookResearch(storage, created, {
|
|
542
|
+
domain: agentRunWorkspaceValue(request, 'domain'),
|
|
543
|
+
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
544
|
+
sourceCellFingerprint,
|
|
545
|
+
question: request.question,
|
|
546
|
+
intent: researchIntent,
|
|
547
|
+
context: {
|
|
548
|
+
surface: 'unified_agent_run',
|
|
549
|
+
agentRunId: runId,
|
|
550
|
+
routeDecision,
|
|
551
|
+
selectedObject: request.selectedObject,
|
|
552
|
+
workspaceContext: request.workspaceContext,
|
|
553
|
+
plan,
|
|
554
|
+
},
|
|
555
|
+
});
|
|
556
|
+
researchRun = withNotebookResearchChecklist(executed);
|
|
557
|
+
}
|
|
558
|
+
finally {
|
|
559
|
+
storage.close();
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
catch (error) {
|
|
563
|
+
researchWorkspaceError = formatNotebookResearchStorageError(error);
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
const researchResultData = coerceNarrateResultData(researchRun?.resultPreview);
|
|
567
|
+
const narration = !needsClarification && researchResultData
|
|
568
|
+
? await narrateForAgentRun({
|
|
569
|
+
question: request.question,
|
|
570
|
+
intent: request.intent,
|
|
571
|
+
result: researchResultData,
|
|
572
|
+
evidence: plan.sources,
|
|
573
|
+
reviewRequired: true,
|
|
574
|
+
})
|
|
575
|
+
: undefined;
|
|
576
|
+
const summary = needsClarification
|
|
577
|
+
? 'Needs clarification before running deeper research.'
|
|
578
|
+
: narration?.summary
|
|
579
|
+
?? (researchRun?.status === 'ready'
|
|
580
|
+
? 'Saved a grounded research dossier with context evidence and next review actions.'
|
|
581
|
+
: researchRun?.status === 'error'
|
|
582
|
+
? 'Saved a research dossier, but the preview needs review before promotion.'
|
|
583
|
+
: researchWorkspaceError
|
|
584
|
+
? 'Prepared a grounded research plan; durable research storage is unavailable in this runtime.'
|
|
585
|
+
: plan.done
|
|
586
|
+
? 'Prepared a direct grounded-answer plan.'
|
|
587
|
+
: 'Prepared a grounded research plan over real DQL assets.');
|
|
588
|
+
return {
|
|
589
|
+
summary,
|
|
590
|
+
answer: plan.followUp?.question ?? narration?.summary ?? researchRun?.summary,
|
|
591
|
+
status: needsClarification ? 'needs_clarification' : 'needs_review',
|
|
592
|
+
trustState: needsClarification ? 'not_applicable' : 'review_required',
|
|
593
|
+
stopReason: needsClarification ? 'needs_clarification' : 'human_review_required',
|
|
594
|
+
artifacts: needsClarification
|
|
595
|
+
? []
|
|
596
|
+
: [agentRunArtifact('research_run', 'Research plan', {
|
|
597
|
+
plan,
|
|
598
|
+
researchRun,
|
|
599
|
+
researchRunId: researchRun?.id,
|
|
600
|
+
notebookPath,
|
|
601
|
+
workspaceError: researchWorkspaceError,
|
|
602
|
+
routeDecision,
|
|
603
|
+
narration,
|
|
604
|
+
resultPreview: researchResultData,
|
|
605
|
+
blockCount: blocks.length,
|
|
606
|
+
metricCount: metrics.length,
|
|
607
|
+
certifiedOnly: usedCertifiedOnly,
|
|
608
|
+
}, researchRun?.id)],
|
|
609
|
+
evaluations: [
|
|
610
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to research.'),
|
|
611
|
+
agentRunEvaluation('catalog-grounding', 'Catalog grounding', plan.sources.length > 0 || Boolean(researchRun?.evidence), plan.sources.length > 0 || Boolean(researchRun?.evidence) ? 'info' : 'warning', plan.sources.length > 0 || Boolean(researchRun?.evidence)
|
|
612
|
+
? 'Research dossier is grounded to catalog or context-pack evidence.'
|
|
613
|
+
: 'No certified catalog source was found; output remains exploratory.', { sources: plan.sources, researchRunId: researchRun?.id, evidence: researchRun?.evidence }),
|
|
614
|
+
agentRunEvaluation('research-workspace', 'Research workspace', Boolean(researchRun?.id), researchRun?.id ? 'info' : 'warning', researchRun?.id
|
|
615
|
+
? 'A durable notebook research record was saved for review and DQL promotion.'
|
|
616
|
+
: researchWorkspaceError
|
|
617
|
+
? 'Research workspace storage is unavailable; the plan remains available in this agent run.'
|
|
618
|
+
: 'No durable research record was created because the run needs clarification first.', { notebookPath, researchRunId: researchRun?.id, error: researchWorkspaceError }),
|
|
619
|
+
],
|
|
620
|
+
nextActions: needsClarification
|
|
621
|
+
? [{ id: 'answer-follow-up', label: 'Answer follow-up', route: 'research' }]
|
|
622
|
+
: [
|
|
623
|
+
...(researchRun?.id ? [{ id: 'open-research', label: 'Open research dossier', artifactKind: 'research_run' }] : []),
|
|
624
|
+
...(researchRun?.generatedSql || researchRun?.reviewedSql ? [{ id: 'insert-sql', label: 'Insert SQL cell', route: 'sql_cell', artifactKind: 'sql_cell' }] : []),
|
|
625
|
+
{ id: 'create-block', label: 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
626
|
+
],
|
|
627
|
+
};
|
|
628
|
+
},
|
|
629
|
+
sql_cell: async ({ request, routeDecision, attempt, repairHint }) => {
|
|
630
|
+
const result = await buildAgentPromptArtifact(request, 'cell', { attempt, repairHint });
|
|
631
|
+
return {
|
|
632
|
+
summary: 'Created a review-required SQL cell draft.',
|
|
633
|
+
answer: result.target === 'cell' ? result.explanation : undefined,
|
|
634
|
+
artifacts: [agentRunArtifact('sql_cell', 'Generated SQL cell', result)],
|
|
635
|
+
evaluations: [
|
|
636
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to SQL cell generation.'),
|
|
637
|
+
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', 'Generated SQL must be reviewed before it becomes certified analytics.'),
|
|
638
|
+
],
|
|
639
|
+
nextActions: [
|
|
640
|
+
{ id: 'insert-sql', label: 'Insert SQL cell', artifactKind: 'sql_cell' },
|
|
641
|
+
{ id: 'create-block', label: 'Promote to DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
642
|
+
],
|
|
643
|
+
};
|
|
644
|
+
},
|
|
645
|
+
dql_block_draft: async ({ request, routeDecision, attempt, repairHint }) => {
|
|
646
|
+
const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
|
|
647
|
+
const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
|
|
648
|
+
return {
|
|
649
|
+
summary: ready
|
|
650
|
+
? 'Created a DQL block draft that is ready for human certification review.'
|
|
651
|
+
: 'Created a DQL block draft with review blockers or warnings.',
|
|
652
|
+
artifacts: [agentRunArtifact('dql_block_draft', result.target === 'block' ? result.name : 'DQL block draft', result, result.target === 'block' ? result.path : undefined)],
|
|
653
|
+
evaluations: [
|
|
654
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to DQL block draft generation.'),
|
|
655
|
+
agentRunEvaluation('certification-boundary', 'Certification boundary', ready, 'warning', ready
|
|
656
|
+
? 'The draft has no automatic certifier blockers, but certification still requires human review.'
|
|
657
|
+
: 'The draft has certifier blockers that must be resolved before certification review.', result),
|
|
658
|
+
],
|
|
659
|
+
nextActions: [
|
|
660
|
+
{ id: 'open-review', label: 'Open review checklist', artifactKind: 'dql_block_draft' },
|
|
661
|
+
{ id: 'build-app', label: 'Build app from block', route: 'app_build', artifactKind: 'app_draft' },
|
|
662
|
+
],
|
|
663
|
+
};
|
|
664
|
+
},
|
|
665
|
+
app_build: async ({ request, routeDecision, emit }) => {
|
|
666
|
+
emit({
|
|
667
|
+
type: 'executor.started',
|
|
668
|
+
message: 'Creating app build session from governed app builder.',
|
|
669
|
+
route: 'app_build',
|
|
670
|
+
});
|
|
671
|
+
let session;
|
|
672
|
+
try {
|
|
673
|
+
session = await createAppAiBuildSession(projectRoot, {
|
|
674
|
+
prompt: request.question,
|
|
675
|
+
domain: agentRunWorkspaceValue(request, 'domain'),
|
|
676
|
+
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
677
|
+
notebookPath: agentRunWorkspaceValue(request, 'notebookPath') ?? request.selectedObject?.path,
|
|
678
|
+
selectedBlockIds: parseAgentRunSelectedBlockIds(request),
|
|
679
|
+
plannerMode: 'deterministic',
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
catch (error) {
|
|
683
|
+
const message = formatAgentRunInfrastructureError(error, 'App build storage');
|
|
684
|
+
return {
|
|
685
|
+
summary: message,
|
|
686
|
+
status: 'blocked',
|
|
687
|
+
trustState: 'blocked',
|
|
688
|
+
stopReason: 'blocked',
|
|
689
|
+
evaluations: [
|
|
690
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to app build.'),
|
|
691
|
+
agentRunEvaluation('app-build-storage', 'App build storage', false, 'blocking', message, { originalErrorType: error instanceof Error ? error.name : typeof error }),
|
|
692
|
+
],
|
|
693
|
+
nextActions: [
|
|
694
|
+
{ id: 'research-coverage', label: 'Research missing coverage', route: 'research', artifactKind: 'research_run' },
|
|
695
|
+
{ id: 'create-gap-blocks', label: 'Create DQL drafts for gaps', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
696
|
+
],
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
const ready = session.status === 'ready';
|
|
700
|
+
const sessionPlan = agentRunRecord(session.plan);
|
|
701
|
+
const appTitle = agentRunString(sessionPlan?.name) ?? 'App draft';
|
|
702
|
+
// A coverage gap is NOT terminal — leave status open so the gate can escalate to
|
|
703
|
+
// drafting the missing blocks. Only genuine infra errors (the catch above) block.
|
|
704
|
+
return {
|
|
705
|
+
summary: ready
|
|
706
|
+
? 'Created a review-required app draft session from certified DQL assets.'
|
|
707
|
+
: 'App build needs more certified DQL coverage before files can be generated.',
|
|
708
|
+
status: ready ? 'needs_review' : undefined,
|
|
709
|
+
trustState: ready ? 'review_required' : undefined,
|
|
710
|
+
stopReason: ready ? 'human_review_required' : undefined,
|
|
711
|
+
artifacts: ready ? [agentRunArtifact('app_draft', appTitle, {
|
|
712
|
+
session,
|
|
713
|
+
sessionId: session.id,
|
|
714
|
+
appId: session.appId,
|
|
715
|
+
dashboardId: session.dashboardId,
|
|
716
|
+
generatedPaths: session.generatedPaths,
|
|
717
|
+
plan: session.plan,
|
|
718
|
+
validation: session.validation,
|
|
719
|
+
}, session.appId)] : [],
|
|
720
|
+
evaluations: [
|
|
721
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to app build.'),
|
|
722
|
+
agentRunEvaluation('app-coverage', 'Certified coverage', ready, ready ? 'info' : 'blocking', ready
|
|
723
|
+
? 'Generated app files are backed by certified block tiles and saved as a draft app session.'
|
|
724
|
+
: session.error ?? 'No certified app tiles matched the request.', session),
|
|
725
|
+
],
|
|
726
|
+
nextActions: ready
|
|
727
|
+
? [
|
|
728
|
+
{ id: 'open-app', label: 'Open app draft', artifactKind: 'app_draft' },
|
|
729
|
+
{ id: 'create-gap-blocks', label: 'Create DQL drafts for gaps', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
730
|
+
]
|
|
731
|
+
: [
|
|
732
|
+
{ id: 'research-coverage', label: 'Research missing coverage', route: 'research', artifactKind: 'research_run' },
|
|
733
|
+
{ id: 'create-gap-blocks', label: 'Create DQL drafts for gaps', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
734
|
+
],
|
|
735
|
+
};
|
|
736
|
+
},
|
|
737
|
+
};
|
|
738
|
+
// Compact, catalog-grounded context the LLM planner decomposes `auto` turns against.
|
|
739
|
+
const buildAgentRunCatalogContext = () => {
|
|
740
|
+
try {
|
|
741
|
+
const blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
742
|
+
const sourceBlocks = blocks.length > 0 ? blocks : collectPlanBlocks(projectRoot, { certifiedOnly: false });
|
|
743
|
+
const blockLines = sourceBlocks.slice(0, 24).map((block) => {
|
|
744
|
+
const domain = block.domain ? ` [${block.domain}]` : '';
|
|
745
|
+
const detail = block.description ? `: ${block.description}` : '';
|
|
746
|
+
return `- ${block.name}${domain}${detail}`;
|
|
747
|
+
});
|
|
748
|
+
const metrics = loadSemanticMetrics(projectRoot).slice(0, 24);
|
|
749
|
+
const metricLines = metrics.map((metric) => {
|
|
750
|
+
const node = metric;
|
|
751
|
+
return `- ${node.name ?? node.label ?? node.id ?? 'metric'}`;
|
|
752
|
+
});
|
|
753
|
+
return [
|
|
754
|
+
blockLines.length > 0 ? `Available DQL blocks:\n${blockLines.join('\n')}` : 'Available DQL blocks: none',
|
|
755
|
+
metricLines.length > 0 ? `Governed metrics:\n${metricLines.join('\n')}` : '',
|
|
756
|
+
].filter(Boolean).join('\n\n');
|
|
757
|
+
}
|
|
758
|
+
catch {
|
|
759
|
+
return '';
|
|
760
|
+
}
|
|
761
|
+
};
|
|
762
|
+
// Provider-agnostic completion the planner injects. Reuses the configured provider
|
|
763
|
+
// adapter; throwing here makes the planner fall back to its deterministic path.
|
|
764
|
+
const agentRunPlanner = createLlmAgentRunPlanner({
|
|
765
|
+
complete: async ({ system, user, signal }) => {
|
|
766
|
+
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
767
|
+
if (!provider)
|
|
768
|
+
throw new Error('No AI provider configured for planning.');
|
|
769
|
+
return provider.generate([
|
|
770
|
+
{ role: 'system', content: system },
|
|
771
|
+
{ role: 'user', content: user },
|
|
772
|
+
], { maxTokens: 700, temperature: 0.1, signal });
|
|
773
|
+
},
|
|
774
|
+
getCatalogContext: buildAgentRunCatalogContext,
|
|
775
|
+
});
|
|
776
|
+
const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
|
|
777
|
+
const agentRunEngine = new AgentRunEngine({
|
|
778
|
+
store: agentRunStore,
|
|
779
|
+
executors: agentRunExecutors,
|
|
780
|
+
gates: defaultAgentRunGates,
|
|
781
|
+
planner: agentRunPlanner,
|
|
782
|
+
});
|
|
129
783
|
const runNotebookForApp = async (appId, notebookPath) => {
|
|
130
784
|
const absPath = safeJoin(projectRoot, notebookPath);
|
|
131
785
|
if (!absPath || !existsSync(absPath) || statSync(absPath).isDirectory() || !absPath.endsWith('.dqlnb')) {
|
|
@@ -313,21 +967,50 @@ export async function startLocalServer(opts) {
|
|
|
313
967
|
}
|
|
314
968
|
let governedAnswer;
|
|
315
969
|
let providerError;
|
|
970
|
+
const memoOnly = input.mode === 'memo_only';
|
|
316
971
|
const contextEnvelope = {
|
|
317
972
|
mode: 'app_research',
|
|
973
|
+
generationMode: input.mode ?? 'sql_and_memo',
|
|
318
974
|
intent: input.intent,
|
|
319
975
|
appId: input.appId,
|
|
320
976
|
dashboardId: input.dashboardId,
|
|
321
977
|
sourceTileId: input.sourceTileId,
|
|
322
978
|
sourceBlockId: input.sourceBlockId,
|
|
323
979
|
title: input.title,
|
|
324
|
-
instruction:
|
|
980
|
+
instruction: [
|
|
981
|
+
'Answer as an app-scoped analyst. Start from certified block/result context, active filters, dbt/semantic metadata, and schema evidence.',
|
|
982
|
+
memoOnly
|
|
983
|
+
? 'Write the stakeholder memo from the supplied selected result, preview rows, generated SQL, metrics, drivers, filters, and caveats. Do not generate replacement SQL unless the user explicitly asks to fix SQL.'
|
|
984
|
+
: 'Generate review-required read-only SQL only when the certified result does not exactly answer the requested analysis grain. Execute only through the bounded generated SQL preview path.',
|
|
985
|
+
'In the natural-language answer, write a stakeholder analysis memo, not a generic chat answer. Use Markdown headings chosen for the question, usually Executive answer, Key numbers, Drivers or Business readout, Caveats, and Next action.',
|
|
986
|
+
'Use concrete numbers from the selected result or preview when available. If baseline, segment, grain, lineage, or source proof is missing, say that explicitly instead of inventing a driver story.',
|
|
987
|
+
'Keep raw SQL in proposedSql/sql. Do not put SQL code fences or implementation trace in the memo body.',
|
|
988
|
+
].join(' '),
|
|
989
|
+
generatedSql: input.generatedSql,
|
|
990
|
+
metrics: input.metrics,
|
|
991
|
+
drivers: input.drivers,
|
|
992
|
+
resultPreviews: input.resultPreviews,
|
|
993
|
+
summaryHint: input.summaryHint,
|
|
994
|
+
recommendationHint: input.recommendationHint,
|
|
995
|
+
sqlError: input.sqlError,
|
|
996
|
+
sqlErrorKind: input.sqlErrorKind,
|
|
997
|
+
hasReportEvidence: input.hasReportEvidence,
|
|
325
998
|
context: input.context,
|
|
326
999
|
};
|
|
327
1000
|
const controller = new AbortController();
|
|
328
1001
|
await runner.run({
|
|
329
1002
|
provider: resolvedProvider,
|
|
330
|
-
messages: [{
|
|
1003
|
+
messages: [{
|
|
1004
|
+
role: 'user',
|
|
1005
|
+
content: memoOnly
|
|
1006
|
+
? [
|
|
1007
|
+
`Research question: ${input.question}`,
|
|
1008
|
+
'Write the business memo now using only the evidence envelope supplied as upstream context.',
|
|
1009
|
+
'Return Markdown sections. Keep SQL, query text, implementation trace, and raw routing details out of the memo body.',
|
|
1010
|
+
'If the evidence is insufficient, state the gap and the next reviewer action instead of filling the story with generic text.',
|
|
1011
|
+
].join('\n')
|
|
1012
|
+
: input.question,
|
|
1013
|
+
}],
|
|
331
1014
|
upstream: {
|
|
332
1015
|
cellId: `app-research:${input.appId}:${input.dashboardId ?? 'app'}`,
|
|
333
1016
|
sql: JSON.stringify(contextEnvelope, null, 2),
|
|
@@ -348,7 +1031,7 @@ export async function startLocalServer(opts) {
|
|
|
348
1031
|
throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
|
|
349
1032
|
}
|
|
350
1033
|
return {
|
|
351
|
-
sql: governedAnswer.proposedSql ?? governedAnswer.sql,
|
|
1034
|
+
sql: memoOnly ? undefined : governedAnswer.proposedSql ?? governedAnswer.sql,
|
|
352
1035
|
answer: governedAnswer.answer ?? governedAnswer.text,
|
|
353
1036
|
result: governedAnswer.result,
|
|
354
1037
|
analysisPlan: governedAnswer.analysisPlan,
|
|
@@ -360,6 +1043,87 @@ export async function startLocalServer(opts) {
|
|
|
360
1043
|
};
|
|
361
1044
|
};
|
|
362
1045
|
const openNotebookResearchStorage = () => new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
|
|
1046
|
+
const notebookResearchStorageUnavailableMessage = 'Notebook research storage is unavailable because the local SQLite native bindings are not installed for this Node.js runtime.';
|
|
1047
|
+
const notebookResearchNextActionFilters = [
|
|
1048
|
+
'fix_blockers',
|
|
1049
|
+
'review_sql',
|
|
1050
|
+
'review_context',
|
|
1051
|
+
'run_preview',
|
|
1052
|
+
'reuse_existing',
|
|
1053
|
+
'create_dql_draft',
|
|
1054
|
+
'open_certification',
|
|
1055
|
+
'complete_review',
|
|
1056
|
+
'continue_review',
|
|
1057
|
+
];
|
|
1058
|
+
const isNotebookResearchStorageUnavailable = (error) => {
|
|
1059
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1060
|
+
return /better[-_]sqlite3/i.test(message) || /Could not locate the bindings file/i.test(message);
|
|
1061
|
+
};
|
|
1062
|
+
const emptyNotebookResearchNextActionCounts = () => Object.fromEntries(notebookResearchNextActionFilters.map((action) => [action, 0]));
|
|
1063
|
+
const emptyNotebookResearchListPage = (input = {}) => ({
|
|
1064
|
+
runs: [],
|
|
1065
|
+
total: 0,
|
|
1066
|
+
domains: [],
|
|
1067
|
+
owners: [],
|
|
1068
|
+
intents: [],
|
|
1069
|
+
notebooks: [],
|
|
1070
|
+
counts: {
|
|
1071
|
+
total: 0,
|
|
1072
|
+
ready: 0,
|
|
1073
|
+
needsReview: 0,
|
|
1074
|
+
dqlDrafts: 0,
|
|
1075
|
+
errors: 0,
|
|
1076
|
+
reuseExisting: 0,
|
|
1077
|
+
extendExisting: 0,
|
|
1078
|
+
replacements: 0,
|
|
1079
|
+
createNew: 0,
|
|
1080
|
+
draftReady: 0,
|
|
1081
|
+
certificationReady: 0,
|
|
1082
|
+
blocked: 0,
|
|
1083
|
+
staleOpen: 0,
|
|
1084
|
+
expiredOpen: 0,
|
|
1085
|
+
sourceLinked: 0,
|
|
1086
|
+
nextActions: emptyNotebookResearchNextActionCounts(),
|
|
1087
|
+
},
|
|
1088
|
+
groupCounts: {
|
|
1089
|
+
domains: 0,
|
|
1090
|
+
owners: 0,
|
|
1091
|
+
intents: 0,
|
|
1092
|
+
notebooks: 0,
|
|
1093
|
+
},
|
|
1094
|
+
limit: input.limit,
|
|
1095
|
+
offset: input.offset ?? 0,
|
|
1096
|
+
});
|
|
1097
|
+
const emptyNotebookResearchDiagnostics = () => ({
|
|
1098
|
+
counts: {
|
|
1099
|
+
totalRuns: 0,
|
|
1100
|
+
activeRuns: 0,
|
|
1101
|
+
closedRuns: 0,
|
|
1102
|
+
notebooks: 0,
|
|
1103
|
+
domains: 0,
|
|
1104
|
+
owners: 0,
|
|
1105
|
+
sourceLinkedRuns: 0,
|
|
1106
|
+
},
|
|
1107
|
+
health: {
|
|
1108
|
+
staleOpenRuns: 0,
|
|
1109
|
+
expiredOpenRuns: 0,
|
|
1110
|
+
staleThresholdDays: 7,
|
|
1111
|
+
expiredThresholdDays: 30,
|
|
1112
|
+
},
|
|
1113
|
+
search: {
|
|
1114
|
+
indexed: false,
|
|
1115
|
+
indexRows: 0,
|
|
1116
|
+
stale: false,
|
|
1117
|
+
},
|
|
1118
|
+
updatedAt: {},
|
|
1119
|
+
limits: {
|
|
1120
|
+
pageSize: 50,
|
|
1121
|
+
maxPageSize: 500,
|
|
1122
|
+
sourceCoverageLimit: 10_000,
|
|
1123
|
+
seedCellLimit: 1000,
|
|
1124
|
+
},
|
|
1125
|
+
warnings: [notebookResearchStorageUnavailableMessage],
|
|
1126
|
+
});
|
|
363
1127
|
const runNotebookResearch = async (storage, run, input = {}) => {
|
|
364
1128
|
const question = notebookResearchString(input.question) || run.question;
|
|
365
1129
|
const domain = notebookResearchString(input.domain) ?? run.domain;
|
|
@@ -936,7 +1700,19 @@ export async function startLocalServer(opts) {
|
|
|
936
1700
|
if (tests.length === 0) {
|
|
937
1701
|
return { passed: 0, failed: 0, skipped: 0, duration: Date.now() - start, assertions: [], runAt: new Date() };
|
|
938
1702
|
}
|
|
939
|
-
|
|
1703
|
+
// Run tests against the SAME SQL the preview runs: for a semantic block with a
|
|
1704
|
+
// pre-compiled query, that's the query (not a recompiled metric), so the test's
|
|
1705
|
+
// output columns match the block's declared outputs.
|
|
1706
|
+
const semanticCompose = semanticLayer
|
|
1707
|
+
? composeSemanticBlockSql(source, semanticLayer, {
|
|
1708
|
+
driver: activeConnection.driver,
|
|
1709
|
+
tableMapping,
|
|
1710
|
+
projectRoot,
|
|
1711
|
+
projectConfig,
|
|
1712
|
+
detectedProvider: semanticDetectedProvider,
|
|
1713
|
+
})
|
|
1714
|
+
: null;
|
|
1715
|
+
const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, activeConnection, projectRoot, projectConfig);
|
|
940
1716
|
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams ?? [], runtimeVariables(plan.variables ?? {}), prepared.connection);
|
|
941
1717
|
const rows = Array.isArray(rawResult?.rows) ? rawResult.rows : [];
|
|
942
1718
|
const columns = Array.isArray(rawResult?.columns)
|
|
@@ -1070,12 +1846,25 @@ export async function startLocalServer(opts) {
|
|
|
1070
1846
|
sourceSystems: parsed.sourceSystems,
|
|
1071
1847
|
replacementFor: parsed.replacementFor,
|
|
1072
1848
|
reviewCadence: parsed.reviewCadence,
|
|
1849
|
+
metricRef: parsed.metricRef || undefined,
|
|
1850
|
+
metricsRef: parsed.metricsRef.length > 0 ? parsed.metricsRef : undefined,
|
|
1073
1851
|
dependencies: [],
|
|
1074
1852
|
usedInCount: 0,
|
|
1075
1853
|
createdAt: new Date(),
|
|
1076
1854
|
updatedAt: new Date(),
|
|
1077
1855
|
};
|
|
1078
|
-
|
|
1856
|
+
// Evaluate declared invariants against the preview result so the
|
|
1857
|
+
// `invariants-hold` certifier rule can enforce them. Best-effort: when the
|
|
1858
|
+
// preview failed there is no result to check, and the rule then blocks
|
|
1859
|
+
// certification (in enterprise mode) because the guarantees are unverified.
|
|
1860
|
+
const invariantEval = preview
|
|
1861
|
+
? evaluateBlockInvariants(source, {
|
|
1862
|
+
columns: preview.result.columns,
|
|
1863
|
+
rows: preview.result.rows,
|
|
1864
|
+
})
|
|
1865
|
+
: null;
|
|
1866
|
+
record.invariants = extractBlockInvariants(source);
|
|
1867
|
+
const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined, invariantEval ? { invariantResults: invariantEval.invariantResults } : undefined);
|
|
1079
1868
|
const checklist = buildBlockStudioCertificationChecklist({
|
|
1080
1869
|
source,
|
|
1081
1870
|
validation,
|
|
@@ -1084,7 +1873,19 @@ export async function startLocalServer(opts) {
|
|
|
1084
1873
|
certificationErrors: certification.errors,
|
|
1085
1874
|
extraBlockers: blockers,
|
|
1086
1875
|
});
|
|
1087
|
-
return {
|
|
1876
|
+
return {
|
|
1877
|
+
certification,
|
|
1878
|
+
checklist,
|
|
1879
|
+
validation,
|
|
1880
|
+
preview,
|
|
1881
|
+
testResults,
|
|
1882
|
+
invariantResults: invariantEval?.invariantResults ?? [],
|
|
1883
|
+
invariantViolation: invariantEval?.invariantViolation ?? false,
|
|
1884
|
+
};
|
|
1885
|
+
};
|
|
1886
|
+
const writeAgentRunSse = (response, event, data) => {
|
|
1887
|
+
response.write(`event: ${event}\n`);
|
|
1888
|
+
response.write(`data: ${serializeJSON(data)}\n\n`);
|
|
1088
1889
|
};
|
|
1089
1890
|
const server = createServer(async (req, res) => {
|
|
1090
1891
|
const requestUrl = req.url || '/';
|
|
@@ -1099,9 +1900,540 @@ export async function startLocalServer(opts) {
|
|
|
1099
1900
|
res.end();
|
|
1100
1901
|
return;
|
|
1101
1902
|
}
|
|
1102
|
-
if (req.method === 'GET' && path === '/api/health') {
|
|
1103
|
-
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1104
|
-
res.end(serializeJSON({ status: 'ok' }));
|
|
1903
|
+
if (req.method === 'GET' && path === '/api/health') {
|
|
1904
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1905
|
+
res.end(serializeJSON({ status: 'ok' }));
|
|
1906
|
+
return;
|
|
1907
|
+
}
|
|
1908
|
+
if (req.method === 'GET' && path === '/api/agent-runs') {
|
|
1909
|
+
const rawLimit = Number(url.searchParams.get('limit'));
|
|
1910
|
+
const limit = Number.isFinite(rawLimit) && rawLimit > 0
|
|
1911
|
+
? Math.min(200, Math.floor(rawLimit))
|
|
1912
|
+
: 50;
|
|
1913
|
+
const runs = agentRunStore
|
|
1914
|
+
.list()
|
|
1915
|
+
.sort((a, b) => b.startedAt.localeCompare(a.startedAt))
|
|
1916
|
+
.slice(0, limit);
|
|
1917
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1918
|
+
res.end(serializeJSON({ runs, total: agentRunStore.list().length, limit }));
|
|
1919
|
+
return;
|
|
1920
|
+
}
|
|
1921
|
+
if (req.method === 'POST' && path === '/api/agent-runs') {
|
|
1922
|
+
try {
|
|
1923
|
+
const body = await readJSON(req).catch(() => null);
|
|
1924
|
+
const parsed = parseAgentRunRequestBody(body);
|
|
1925
|
+
if (!parsed.request) {
|
|
1926
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1927
|
+
res.end(serializeJSON({ error: parsed.error ?? 'Invalid agent run request.' }));
|
|
1928
|
+
return;
|
|
1929
|
+
}
|
|
1930
|
+
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
1931
|
+
if (wantsStream) {
|
|
1932
|
+
res.writeHead(200, {
|
|
1933
|
+
'Content-Type': 'text/event-stream; charset=utf-8',
|
|
1934
|
+
'Cache-Control': 'no-cache',
|
|
1935
|
+
'Connection': 'keep-alive',
|
|
1936
|
+
'X-Accel-Buffering': 'no',
|
|
1937
|
+
});
|
|
1938
|
+
const run = await agentRunEngine.run(parsed.request, (event) => {
|
|
1939
|
+
writeAgentRunSse(res, 'agent-run-event', event);
|
|
1940
|
+
});
|
|
1941
|
+
writeAgentRunSse(res, 'agent-run-complete', run);
|
|
1942
|
+
res.end();
|
|
1943
|
+
return;
|
|
1944
|
+
}
|
|
1945
|
+
const run = await agentRunEngine.run(parsed.request);
|
|
1946
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1947
|
+
res.end(serializeJSON({ run }));
|
|
1948
|
+
}
|
|
1949
|
+
catch (error) {
|
|
1950
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1951
|
+
if (res.headersSent) {
|
|
1952
|
+
writeAgentRunSse(res, 'agent-run-error', { error: message });
|
|
1953
|
+
res.end();
|
|
1954
|
+
}
|
|
1955
|
+
else {
|
|
1956
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1957
|
+
res.end(serializeJSON({ error: message }));
|
|
1958
|
+
}
|
|
1959
|
+
}
|
|
1960
|
+
return;
|
|
1961
|
+
}
|
|
1962
|
+
// Stakeholder → analyst handoff: turn a review-required output into a draft
|
|
1963
|
+
// research run in the analyst notebook queue (no authoring by the stakeholder).
|
|
1964
|
+
if (req.method === 'POST' && path === '/api/agent-runs/request-certification') {
|
|
1965
|
+
try {
|
|
1966
|
+
const body = await readJSON(req).catch(() => null);
|
|
1967
|
+
const record = agentRunRecord(body);
|
|
1968
|
+
const question = record ? agentRunString(record.question) : undefined;
|
|
1969
|
+
if (!question) {
|
|
1970
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1971
|
+
res.end(serializeJSON({ ok: false, error: 'question is required.' }));
|
|
1972
|
+
return;
|
|
1973
|
+
}
|
|
1974
|
+
const notebookPath = (record && agentRunString(record.notebookPath))
|
|
1975
|
+
?? `notebooks/certification-requests/${Date.now()}.dqlnb`;
|
|
1976
|
+
const generatedSql = record ? agentRunString(record.generatedSql) : undefined;
|
|
1977
|
+
try {
|
|
1978
|
+
const storage = openNotebookResearchStorage();
|
|
1979
|
+
try {
|
|
1980
|
+
const created = storage.createRun({
|
|
1981
|
+
notebookPath,
|
|
1982
|
+
title: agentRunTitle(question, 'Certification request'),
|
|
1983
|
+
question,
|
|
1984
|
+
intent: 'ad_hoc_analysis',
|
|
1985
|
+
domain: record ? agentRunString(record.domain) : undefined,
|
|
1986
|
+
owner: record ? agentRunString(record.owner) : undefined,
|
|
1987
|
+
generatedSql,
|
|
1988
|
+
context: {
|
|
1989
|
+
surface: 'stakeholder_request_certification',
|
|
1990
|
+
requestedContext: agentRunRecord(record?.context) ?? null,
|
|
1991
|
+
},
|
|
1992
|
+
});
|
|
1993
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1994
|
+
res.end(serializeJSON({ ok: true, researchRunId: created.id, notebookPath }));
|
|
1995
|
+
}
|
|
1996
|
+
finally {
|
|
1997
|
+
storage.close();
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
catch (error) {
|
|
2001
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2002
|
+
res.end(serializeJSON({ ok: false, error: formatNotebookResearchStorageError(error) }));
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
catch (error) {
|
|
2006
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2007
|
+
res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
|
|
2008
|
+
}
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
const agentRunMatch = /^\/api\/agent-runs\/([^/]+)$/.exec(path);
|
|
2012
|
+
if (req.method === 'GET' && agentRunMatch) {
|
|
2013
|
+
const id = decodeURIComponent(agentRunMatch[1]);
|
|
2014
|
+
const run = await agentRunStore.get(id);
|
|
2015
|
+
if (!run) {
|
|
2016
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2017
|
+
res.end(serializeJSON({ error: 'Agent run not found.' }));
|
|
2018
|
+
return;
|
|
2019
|
+
}
|
|
2020
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2021
|
+
res.end(serializeJSON({ run }));
|
|
2022
|
+
return;
|
|
2023
|
+
}
|
|
2024
|
+
// Readiness → propose backbone. Returns a readiness summary plus the ranked
|
|
2025
|
+
// DRAFT proposals (each with its stored Certifier verdict) so the notebook
|
|
2026
|
+
// "Get Started" surface can route them into human review. dryRun preview —
|
|
2027
|
+
// nothing is written or certified by this call.
|
|
2028
|
+
if ((req.method === 'GET' || req.method === 'POST') && path === '/api/propose') {
|
|
2029
|
+
try {
|
|
2030
|
+
let owner;
|
|
2031
|
+
let limit;
|
|
2032
|
+
if (req.method === 'POST') {
|
|
2033
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
2034
|
+
if (typeof body?.owner === 'string')
|
|
2035
|
+
owner = body.owner;
|
|
2036
|
+
if (typeof body?.limit === 'number' && Number.isFinite(body.limit) && body.limit > 0) {
|
|
2037
|
+
limit = body.limit;
|
|
2038
|
+
}
|
|
2039
|
+
}
|
|
2040
|
+
else {
|
|
2041
|
+
const ownerParam = url.searchParams.get('owner');
|
|
2042
|
+
if (ownerParam)
|
|
2043
|
+
owner = ownerParam;
|
|
2044
|
+
const limitParam = Number(url.searchParams.get('limit'));
|
|
2045
|
+
if (Number.isFinite(limitParam) && limitParam > 0)
|
|
2046
|
+
limit = limitParam;
|
|
2047
|
+
}
|
|
2048
|
+
const readiness = buildProposeReadiness(projectRoot, loadProjectConfig(projectRoot), { owner, limit });
|
|
2049
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2050
|
+
res.end(serializeJSON(readiness));
|
|
2051
|
+
}
|
|
2052
|
+
catch (error) {
|
|
2053
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2054
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2055
|
+
}
|
|
2056
|
+
return;
|
|
2057
|
+
}
|
|
2058
|
+
// Deterministic PLAN only (classify → plan). Writes NOTHING. Same data the
|
|
2059
|
+
// readiness endpoint embeds, exposed standalone for the approve gate.
|
|
2060
|
+
if (req.method === 'POST' && path === '/api/propose/plan') {
|
|
2061
|
+
try {
|
|
2062
|
+
const readiness = buildProposeReadiness(projectRoot, loadProjectConfig(projectRoot));
|
|
2063
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2064
|
+
res.end(serializeJSON(readiness.ready ? readiness.plan : { ready: false, reason: readiness.reason }));
|
|
2065
|
+
}
|
|
2066
|
+
catch (error) {
|
|
2067
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2068
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2069
|
+
}
|
|
2070
|
+
return;
|
|
2071
|
+
}
|
|
2072
|
+
// Materialize drafts for an APPROVED scope { slugs } (or { domains }).
|
|
2073
|
+
// This is the only propose endpoint that writes — and only for the approved,
|
|
2074
|
+
// business-only selection. Nothing is ever certified.
|
|
2075
|
+
if (req.method === 'POST' && path === '/api/propose/generate') {
|
|
2076
|
+
try {
|
|
2077
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2078
|
+
const config = loadProjectConfig(projectRoot);
|
|
2079
|
+
let slugs = Array.isArray(body?.slugs)
|
|
2080
|
+
? body.slugs.filter((s) => typeof s === 'string')
|
|
2081
|
+
: [];
|
|
2082
|
+
// { domains } → resolve to the plan's slugs for those domains.
|
|
2083
|
+
if (slugs.length === 0 && Array.isArray(body?.domains)) {
|
|
2084
|
+
const wanted = new Set(body.domains.filter((d) => typeof d === 'string'));
|
|
2085
|
+
const readiness = buildProposeReadiness(projectRoot, config);
|
|
2086
|
+
slugs = readiness.ready
|
|
2087
|
+
? readiness.plan.domains
|
|
2088
|
+
.filter((d) => wanted.has(d.name))
|
|
2089
|
+
.flatMap((d) => d.candidates.map((c) => c.slug))
|
|
2090
|
+
: [];
|
|
2091
|
+
}
|
|
2092
|
+
if (slugs.length === 0) {
|
|
2093
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2094
|
+
res.end(serializeJSON({ error: 'Provide a non-empty { slugs } or { domains } scope to generate.' }));
|
|
2095
|
+
return;
|
|
2096
|
+
}
|
|
2097
|
+
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
2098
|
+
const result = await generateProposeDrafts(projectRoot, slugs, config, { owner });
|
|
2099
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2100
|
+
res.end(serializeJSON(result));
|
|
2101
|
+
}
|
|
2102
|
+
catch (error) {
|
|
2103
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2104
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2105
|
+
}
|
|
2106
|
+
return;
|
|
2107
|
+
}
|
|
2108
|
+
// Materialize a single approved draft { slug } and return it. Convenience for
|
|
2109
|
+
// the per-block "Review & Certify" affordance.
|
|
2110
|
+
if (req.method === 'POST' && path === '/api/propose/draft') {
|
|
2111
|
+
try {
|
|
2112
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2113
|
+
const slug = typeof body?.slug === 'string' ? body.slug : '';
|
|
2114
|
+
if (!slug) {
|
|
2115
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2116
|
+
res.end(serializeJSON({ error: 'Provide { slug } to draft.' }));
|
|
2117
|
+
return;
|
|
2118
|
+
}
|
|
2119
|
+
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
2120
|
+
const result = await generateProposeDrafts(projectRoot, [slug], loadProjectConfig(projectRoot), { owner });
|
|
2121
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2122
|
+
res.end(serializeJSON(result));
|
|
2123
|
+
}
|
|
2124
|
+
catch (error) {
|
|
2125
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2126
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2127
|
+
}
|
|
2128
|
+
return;
|
|
2129
|
+
}
|
|
2130
|
+
// Transparent PLAN PREVIEW for ONE candidate (spec 14, part A). Lazy +
|
|
2131
|
+
// expensive: builds the real SQL + Certifier verdict + best-effort AI
|
|
2132
|
+
// enrichment for a single slug, so the UI shows the actual logic before a
|
|
2133
|
+
// human commits. Writes NOTHING.
|
|
2134
|
+
if (req.method === 'GET' && path === '/api/propose/preview') {
|
|
2135
|
+
try {
|
|
2136
|
+
const slug = url.searchParams.get('slug')?.trim();
|
|
2137
|
+
if (!slug) {
|
|
2138
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2139
|
+
res.end(serializeJSON({ error: 'Provide a ?slug= query parameter.' }));
|
|
2140
|
+
return;
|
|
2141
|
+
}
|
|
2142
|
+
const candidate = await buildProposeCandidatePreview(projectRoot, slug, url.searchParams.get('owner') ?? undefined);
|
|
2143
|
+
if (!candidate) {
|
|
2144
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2145
|
+
res.end(serializeJSON({ error: `No proposed candidate found for slug "${slug}".` }));
|
|
2146
|
+
return;
|
|
2147
|
+
}
|
|
2148
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2149
|
+
res.end(serializeJSON({ candidate }));
|
|
2150
|
+
}
|
|
2151
|
+
catch (error) {
|
|
2152
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2153
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2154
|
+
}
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
2157
|
+
// Unified AI BUILD (spec 14, part B). ONE engine, two targets:
|
|
2158
|
+
// target:'cell' → generate SQL from the prompt (+ context). Writes nothing.
|
|
2159
|
+
// target:'block' → assemble a COMPLETE draft, WRITE it, return preview fields.
|
|
2160
|
+
// Never routes through the governed Q&A answer-loop.
|
|
2161
|
+
if (req.method === 'POST' && path === '/api/ai/build') {
|
|
2162
|
+
try {
|
|
2163
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2164
|
+
const prompt = typeof body?.prompt === 'string' ? body.prompt.trim() : '';
|
|
2165
|
+
if (!prompt) {
|
|
2166
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2167
|
+
res.end(serializeJSON({ error: 'Provide a non-empty { prompt }.' }));
|
|
2168
|
+
return;
|
|
2169
|
+
}
|
|
2170
|
+
const target = body?.target === 'cell' || body?.target === 'block' ? body.target : undefined;
|
|
2171
|
+
if (!target) {
|
|
2172
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2173
|
+
res.end(serializeJSON({ error: "Provide { target: 'cell' | 'block' }." }));
|
|
2174
|
+
return;
|
|
2175
|
+
}
|
|
2176
|
+
const context = {
|
|
2177
|
+
cellSql: typeof body?.context?.cellSql === 'string' ? body.context.cellSql : undefined,
|
|
2178
|
+
selection: typeof body?.context?.selection === 'string' ? body.context.selection : undefined,
|
|
2179
|
+
};
|
|
2180
|
+
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
2181
|
+
const userId = typeof body?.userId === 'string'
|
|
2182
|
+
? body.userId
|
|
2183
|
+
: undefined;
|
|
2184
|
+
// Edit mode (spec 17, part A): modify the block at `blockPath` in place.
|
|
2185
|
+
const mode = body?.mode === 'edit' ? 'edit' : 'create';
|
|
2186
|
+
const blockPath = typeof body?.blockPath === 'string' && body.blockPath.trim()
|
|
2187
|
+
? body.blockPath.trim()
|
|
2188
|
+
: undefined;
|
|
2189
|
+
if (mode === 'edit' && (target !== 'block' || !blockPath)) {
|
|
2190
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2191
|
+
res.end(serializeJSON({ error: "Edit mode requires target 'block' and a { blockPath }." }));
|
|
2192
|
+
return;
|
|
2193
|
+
}
|
|
2194
|
+
// Ensure the agent knowledge graph is built before generating, so the Build
|
|
2195
|
+
// path's semantic-metric routing sees the governed metrics on the very first
|
|
2196
|
+
// call (a cold Build-before-Ask otherwise reads an unbuilt KG and misses them).
|
|
2197
|
+
// Mirrors what the Ask path does; reindex is idempotent and closes its write
|
|
2198
|
+
// connection, so the read-only metric load observes committed data.
|
|
2199
|
+
try {
|
|
2200
|
+
await reindexProject(projectRoot, { kgPath: defaultKgPath(projectRoot) });
|
|
2201
|
+
}
|
|
2202
|
+
catch {
|
|
2203
|
+
// Best-effort: a failed reindex falls back to whatever KG exists (or none).
|
|
2204
|
+
}
|
|
2205
|
+
// Inject user-authored Skills as business context; the engine selects the
|
|
2206
|
+
// relevant subset and stamps `appliedSkills` on the result.
|
|
2207
|
+
const skills = loadSkills(projectRoot).skills;
|
|
2208
|
+
const result = await buildFromPrompt({
|
|
2209
|
+
projectRoot,
|
|
2210
|
+
prompt,
|
|
2211
|
+
context,
|
|
2212
|
+
target,
|
|
2213
|
+
mode,
|
|
2214
|
+
blockPath,
|
|
2215
|
+
owner,
|
|
2216
|
+
userId,
|
|
2217
|
+
skills,
|
|
2218
|
+
dbtManifestPath: resolveDbtManifestPath(projectRoot, loadProjectConfig(projectRoot)),
|
|
2219
|
+
// Reflect-before-certify probe (P2): run the candidate block's SQL to learn
|
|
2220
|
+
// its REAL output columns and evaluate the declared invariants, so the agent
|
|
2221
|
+
// can reconcile the output contract + produce a grounded verdict before a
|
|
2222
|
+
// human reviews. Best-effort — buildFromPrompt falls back to a static reflection.
|
|
2223
|
+
executionProbe: async ({ sql, invariants }) => {
|
|
2224
|
+
const activeConnection = requireActiveConnection();
|
|
2225
|
+
const prepared = prepareLocalExecution(sql, activeConnection, projectRoot, projectConfig);
|
|
2226
|
+
const probeSql = `SELECT * FROM (${stripSqlTerminator(prepared.sql)}) _dql_probe LIMIT 2000`;
|
|
2227
|
+
const probeResult = await executor.executeQuery(probeSql, [], runtimeVariables({}), prepared.connection);
|
|
2228
|
+
const rows = (Array.isArray(probeResult?.rows) ? probeResult.rows : []);
|
|
2229
|
+
const rawColumns = Array.isArray(probeResult?.columns)
|
|
2230
|
+
? probeResult.columns
|
|
2231
|
+
: [];
|
|
2232
|
+
const actualColumns = rawColumns.length > 0
|
|
2233
|
+
? rawColumns.map((c) => (typeof c === 'string' ? c : c?.name ?? String(c)))
|
|
2234
|
+
: (rows[0] ? Object.keys(rows[0]) : []);
|
|
2235
|
+
const invariantResults = evaluateInvariants(invariants, { columns: actualColumns, rows });
|
|
2236
|
+
const passed = invariantResults.filter((r) => r.passed && !r.uncheckable).length;
|
|
2237
|
+
const failed = invariantResults.filter((r) => !r.passed && !r.uncheckable).length;
|
|
2238
|
+
return {
|
|
2239
|
+
actualColumns,
|
|
2240
|
+
invariantResults,
|
|
2241
|
+
tests: invariants.length > 0
|
|
2242
|
+
? { passed, failed, assertionCount: invariantResults.length }
|
|
2243
|
+
: undefined,
|
|
2244
|
+
};
|
|
2245
|
+
},
|
|
2246
|
+
});
|
|
2247
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2248
|
+
res.end(serializeJSON(result));
|
|
2249
|
+
}
|
|
2250
|
+
catch (error) {
|
|
2251
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2252
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2253
|
+
}
|
|
2254
|
+
return;
|
|
2255
|
+
}
|
|
2256
|
+
// Resolved local OSS owner (spec 14, part C). Stamps drafts so a new block is
|
|
2257
|
+
// never born with a "Missing owner" Certifier strike.
|
|
2258
|
+
if (req.method === 'GET' && path === '/api/identity') {
|
|
2259
|
+
try {
|
|
2260
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2261
|
+
res.end(serializeJSON({ owner: resolveLocalOwner(projectRoot) }));
|
|
2262
|
+
}
|
|
2263
|
+
catch (error) {
|
|
2264
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2265
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2266
|
+
}
|
|
2267
|
+
return;
|
|
2268
|
+
}
|
|
2269
|
+
// ── Skills (spec 16) — user-authored business context. AI drafts, humans
|
|
2270
|
+
// certify; skills never carry certification. PROJECT skills (user empty)
|
|
2271
|
+
// are shared; PERSONAL skills (user set) are user-bound. ────────────────
|
|
2272
|
+
if (req.method === 'GET' && path === '/api/skills') {
|
|
2273
|
+
try {
|
|
2274
|
+
const skills = loadSkills(projectRoot).skills.map(serializeSkill);
|
|
2275
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2276
|
+
res.end(serializeJSON({ skills }));
|
|
2277
|
+
}
|
|
2278
|
+
catch (error) {
|
|
2279
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2280
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2281
|
+
}
|
|
2282
|
+
return;
|
|
2283
|
+
}
|
|
2284
|
+
// Form pickers: metrics from the semantic layer + certified block ids.
|
|
2285
|
+
if (req.method === 'GET' && path === '/api/skills/options') {
|
|
2286
|
+
try {
|
|
2287
|
+
const metrics = semanticLayer
|
|
2288
|
+
? semanticLayer.listMetrics().map((m) => m.name).sort()
|
|
2289
|
+
: [];
|
|
2290
|
+
const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
|
|
2291
|
+
const blocks = Object.values(manifest.blocks)
|
|
2292
|
+
.filter((b) => b.status === 'certified')
|
|
2293
|
+
.map((b) => b.name)
|
|
2294
|
+
.sort();
|
|
2295
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2296
|
+
res.end(serializeJSON({ metrics, blocks }));
|
|
2297
|
+
}
|
|
2298
|
+
catch (error) {
|
|
2299
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2300
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2301
|
+
}
|
|
2302
|
+
return;
|
|
2303
|
+
}
|
|
2304
|
+
if (req.method === 'POST' && path === '/api/skills') {
|
|
2305
|
+
try {
|
|
2306
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2307
|
+
const input = parseSkillInput(body?.skill);
|
|
2308
|
+
if (!input) {
|
|
2309
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2310
|
+
res.end(serializeJSON({ error: 'Provide { skill } with id, scope, and body.' }));
|
|
2311
|
+
return;
|
|
2312
|
+
}
|
|
2313
|
+
const skill = writeSkill(projectRoot, input);
|
|
2314
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2315
|
+
res.end(serializeJSON({ skill: serializeSkill(skill) }));
|
|
2316
|
+
}
|
|
2317
|
+
catch (error) {
|
|
2318
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2319
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2320
|
+
}
|
|
2321
|
+
return;
|
|
2322
|
+
}
|
|
2323
|
+
if (req.method === 'PUT' && path.startsWith('/api/skills/')) {
|
|
2324
|
+
try {
|
|
2325
|
+
const id = decodeURIComponent(path.slice('/api/skills/'.length));
|
|
2326
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2327
|
+
const input = parseSkillInput(body?.skill, id);
|
|
2328
|
+
if (!input) {
|
|
2329
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2330
|
+
res.end(serializeJSON({ error: 'Provide { skill } with scope and body.' }));
|
|
2331
|
+
return;
|
|
2332
|
+
}
|
|
2333
|
+
const skill = writeSkill(projectRoot, input);
|
|
2334
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2335
|
+
res.end(serializeJSON({ skill: serializeSkill(skill) }));
|
|
2336
|
+
}
|
|
2337
|
+
catch (error) {
|
|
2338
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2339
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2340
|
+
}
|
|
2341
|
+
return;
|
|
2342
|
+
}
|
|
2343
|
+
if (req.method === 'DELETE' && path.startsWith('/api/skills/')) {
|
|
2344
|
+
try {
|
|
2345
|
+
const id = decodeURIComponent(path.slice('/api/skills/'.length));
|
|
2346
|
+
deleteSkill(projectRoot, id);
|
|
2347
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2348
|
+
res.end(serializeJSON({ ok: true }));
|
|
2349
|
+
}
|
|
2350
|
+
catch (error) {
|
|
2351
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2352
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2353
|
+
}
|
|
2354
|
+
return;
|
|
2355
|
+
}
|
|
2356
|
+
// ── Domains (spec 17, part B) — first-class business domain declarations.
|
|
2357
|
+
// Authoring here satisfies `dql doctor`'s "missing domain declaration"
|
|
2358
|
+
// warning. AI drafts, humans certify; domains carry no certification. ────
|
|
2359
|
+
if (req.method === 'GET' && path === '/api/domains') {
|
|
2360
|
+
try {
|
|
2361
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2362
|
+
res.end(serializeJSON({ domains: listDomains(projectRoot) }));
|
|
2363
|
+
}
|
|
2364
|
+
catch (error) {
|
|
2365
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2366
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2367
|
+
}
|
|
2368
|
+
return;
|
|
2369
|
+
}
|
|
2370
|
+
if (req.method === 'POST' && path === '/api/domains') {
|
|
2371
|
+
try {
|
|
2372
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2373
|
+
const input = parseDomainInput(body?.domain);
|
|
2374
|
+
if (!input) {
|
|
2375
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2376
|
+
res.end(serializeJSON({ error: 'Provide { domain } with a non-empty name.' }));
|
|
2377
|
+
return;
|
|
2378
|
+
}
|
|
2379
|
+
writeDomainDeclaration(projectRoot, input);
|
|
2380
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2381
|
+
const domain = findDomain(projectRoot, input.name);
|
|
2382
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2383
|
+
res.end(serializeJSON({ domain }));
|
|
2384
|
+
}
|
|
2385
|
+
catch (error) {
|
|
2386
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2387
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2388
|
+
}
|
|
2389
|
+
return;
|
|
2390
|
+
}
|
|
2391
|
+
if (req.method === 'PUT' && path.startsWith('/api/domains/')) {
|
|
2392
|
+
try {
|
|
2393
|
+
const id = decodeURIComponent(path.slice('/api/domains/'.length));
|
|
2394
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
2395
|
+
const input = parseDomainInput(body?.domain, id);
|
|
2396
|
+
if (!input) {
|
|
2397
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2398
|
+
res.end(serializeJSON({ error: 'Provide { domain } with a name.' }));
|
|
2399
|
+
return;
|
|
2400
|
+
}
|
|
2401
|
+
// If the name changed, remove the old declaration so we never orphan one.
|
|
2402
|
+
if (domainFolderSlug(id) !== domainFolderSlug(input.name)) {
|
|
2403
|
+
deleteDomainDeclaration(projectRoot, id);
|
|
2404
|
+
}
|
|
2405
|
+
writeDomainDeclaration(projectRoot, input);
|
|
2406
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2407
|
+
const domain = findDomain(projectRoot, input.name);
|
|
2408
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2409
|
+
res.end(serializeJSON({ domain }));
|
|
2410
|
+
}
|
|
2411
|
+
catch (error) {
|
|
2412
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2413
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2414
|
+
}
|
|
2415
|
+
return;
|
|
2416
|
+
}
|
|
2417
|
+
if (req.method === 'DELETE' && path.startsWith('/api/domains/')) {
|
|
2418
|
+
try {
|
|
2419
|
+
const id = decodeURIComponent(path.slice('/api/domains/'.length));
|
|
2420
|
+
deleteDomainDeclaration(projectRoot, id);
|
|
2421
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2422
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2423
|
+
res.end(serializeJSON({ ok: true }));
|
|
2424
|
+
}
|
|
2425
|
+
catch (error) {
|
|
2426
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2427
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2428
|
+
}
|
|
2429
|
+
return;
|
|
2430
|
+
}
|
|
2431
|
+
if (req.method === 'GET' && path === '/favicon.ico') {
|
|
2432
|
+
res.writeHead(200, {
|
|
2433
|
+
'Content-Type': 'image/svg+xml',
|
|
2434
|
+
'Cache-Control': 'public, max-age=86400',
|
|
2435
|
+
});
|
|
2436
|
+
res.end(NOTEBOOK_FAVICON_SVG);
|
|
1105
2437
|
return;
|
|
1106
2438
|
}
|
|
1107
2439
|
if (req.method === 'GET' && path === '/api/settings/env-status') {
|
|
@@ -1242,6 +2574,80 @@ export async function startLocalServer(opts) {
|
|
|
1242
2574
|
res.end(serializeJSON({ ok: true, files }));
|
|
1243
2575
|
return;
|
|
1244
2576
|
}
|
|
2577
|
+
// Local learning loop (OSS): record an analyst's wrong→right correction as a
|
|
2578
|
+
// scope-matched Hint-Graph hint plus an advisory memory, so future similar
|
|
2579
|
+
// questions avoid the same mistake. Single-user self-serve — the correction IS
|
|
2580
|
+
// the approval, so the derived candidate is approved immediately unless the
|
|
2581
|
+
// caller opts out. Advisory only: never overrides certified routing. The
|
|
2582
|
+
// multi-tenant review workflow + automated distillation stay a cloud feature.
|
|
2583
|
+
if (req.method === 'POST' && path === '/api/agent/learnings/correction') {
|
|
2584
|
+
const body = await readJSON(req).catch(() => null);
|
|
2585
|
+
const question = body && typeof body.question === 'string' ? body.question.trim() : '';
|
|
2586
|
+
const correctedSql = body && typeof body.correctedSql === 'string' ? body.correctedSql.trim() : '';
|
|
2587
|
+
if (!body || !question || !correctedSql) {
|
|
2588
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2589
|
+
res.end(serializeJSON({ error: 'question and correctedSql are required.' }));
|
|
2590
|
+
return;
|
|
2591
|
+
}
|
|
2592
|
+
const rawScope = body.scope && typeof body.scope === 'object' ? body.scope : {};
|
|
2593
|
+
const scopeStr = (key) => (typeof rawScope[key] === 'string' && rawScope[key].trim() ? rawScope[key].trim() : undefined);
|
|
2594
|
+
const scope = {
|
|
2595
|
+
metric: scopeStr('metric'),
|
|
2596
|
+
dbtModel: scopeStr('dbtModel'),
|
|
2597
|
+
domain: scopeStr('domain'),
|
|
2598
|
+
dialect: scopeStr('dialect'),
|
|
2599
|
+
term: scopeStr('term'),
|
|
2600
|
+
block: scopeStr('block'),
|
|
2601
|
+
};
|
|
2602
|
+
const wrongSql = typeof body.wrongSql === 'string' ? body.wrongSql.trim() : '';
|
|
2603
|
+
const rationale = typeof body.rationale === 'string' && body.rationale.trim() ? body.rationale.trim() : undefined;
|
|
2604
|
+
const author = typeof body.author === 'string' ? body.author : (resolveLocalOwner(projectRoot) ?? undefined);
|
|
2605
|
+
try {
|
|
2606
|
+
const { trace, hint } = recordCorrectionTrace(projectRoot, {
|
|
2607
|
+
question,
|
|
2608
|
+
scope,
|
|
2609
|
+
wrongAnswer: wrongSql || '(no prior SQL captured)',
|
|
2610
|
+
correction: correctedSql,
|
|
2611
|
+
correctedSql,
|
|
2612
|
+
rationale,
|
|
2613
|
+
author,
|
|
2614
|
+
hintTitle: typeof body.title === 'string' && body.title.trim() ? body.title.trim() : undefined,
|
|
2615
|
+
hintGuidance: typeof body.guidance === 'string' && body.guidance.trim() ? body.guidance.trim() : undefined,
|
|
2616
|
+
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
2617
|
+
});
|
|
2618
|
+
let approvedHint = hint;
|
|
2619
|
+
if (body.approve !== false) {
|
|
2620
|
+
reviewHint(projectRoot, { hintId: hint.id, decision: 'approved', reviewer: author ?? 'local', note: 'Self-approved (OSS single-user).' });
|
|
2621
|
+
approvedHint = { ...hint, status: 'approved' };
|
|
2622
|
+
}
|
|
2623
|
+
// Plain-language advisory memory mirroring the lesson, for transparency + recall.
|
|
2624
|
+
try {
|
|
2625
|
+
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
2626
|
+
memory.upsert({
|
|
2627
|
+
id: `mem_${hint.id}`,
|
|
2628
|
+
scope: 'project',
|
|
2629
|
+
title: approvedHint.title,
|
|
2630
|
+
content: `${approvedHint.guidance}${rationale ? ` (${rationale})` : ''}`,
|
|
2631
|
+
tags: [scope.metric, scope.domain, scope.dbtModel].filter((x) => Boolean(x)),
|
|
2632
|
+
source: 'correction',
|
|
2633
|
+
confidence: 0.9,
|
|
2634
|
+
importance: 0.85,
|
|
2635
|
+
enabled: true,
|
|
2636
|
+
});
|
|
2637
|
+
memory.close();
|
|
2638
|
+
}
|
|
2639
|
+
catch {
|
|
2640
|
+
/* best-effort */
|
|
2641
|
+
}
|
|
2642
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2643
|
+
res.end(serializeJSON({ ok: true, trace, hint: approvedHint }));
|
|
2644
|
+
}
|
|
2645
|
+
catch (error) {
|
|
2646
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2647
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2648
|
+
}
|
|
2649
|
+
return;
|
|
2650
|
+
}
|
|
1245
2651
|
const appDashRun = path.match(/^\/api\/apps\/([^/]+)\/dashboards\/([^/]+)\/run$/);
|
|
1246
2652
|
if (req.method === 'POST' && appDashRun) {
|
|
1247
2653
|
try {
|
|
@@ -1436,6 +2842,14 @@ export async function startLocalServer(opts) {
|
|
|
1436
2842
|
executeSql: executeLocalSqlForStoredResult,
|
|
1437
2843
|
generateInvestigationSql: generateInvestigationSqlForApp,
|
|
1438
2844
|
runNotebook: (appId, notebookPath) => runNotebookForApp(appId, notebookPath),
|
|
2845
|
+
// P4: give the App ask lane a grounded research planner over the catalog.
|
|
2846
|
+
planResearch: async ({ question, isFollowUp }) => {
|
|
2847
|
+
const metrics = loadSemanticMetrics(projectRoot);
|
|
2848
|
+
let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
2849
|
+
if (blocks.length === 0)
|
|
2850
|
+
blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
|
|
2851
|
+
return planResearch({ question, metrics, blocks, isFollowUp });
|
|
2852
|
+
},
|
|
1439
2853
|
});
|
|
1440
2854
|
if (handled)
|
|
1441
2855
|
return;
|
|
@@ -1560,8 +2974,11 @@ export async function startLocalServer(opts) {
|
|
|
1560
2974
|
return;
|
|
1561
2975
|
}
|
|
1562
2976
|
if (req.method === 'GET' && path === '/api/notebook/research') {
|
|
1563
|
-
|
|
2977
|
+
let storage;
|
|
2978
|
+
const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
|
|
2979
|
+
const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
|
|
1564
2980
|
try {
|
|
2981
|
+
storage = openNotebookResearchStorage();
|
|
1565
2982
|
const notebookPath = notebookResearchString(url.searchParams.get('path'));
|
|
1566
2983
|
const sourceCellId = notebookResearchString(url.searchParams.get('sourceCellId') ?? url.searchParams.get('cellId'));
|
|
1567
2984
|
const domain = notebookResearchString(url.searchParams.get('domain'));
|
|
@@ -1577,24 +2994,28 @@ export async function startLocalServer(opts) {
|
|
|
1577
2994
|
const activeOnlyParam = url.searchParams.get('activeOnly') ?? url.searchParams.get('active');
|
|
1578
2995
|
const activeOnly = activeOnlyParam === 'true' || activeOnlyParam === '1';
|
|
1579
2996
|
const sort = notebookResearchSort(url.searchParams.get('sort'));
|
|
1580
|
-
const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
|
|
1581
|
-
const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
|
|
1582
2997
|
const page = storage.listRunsPage({ notebookPath, sourceCellId, domain, owner, intent, search, status, reviewStatus, promotionAction, readiness, age, nextAction, activeOnly, sort, limit, offset });
|
|
1583
2998
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1584
2999
|
res.end(serializeJSON(withNotebookResearchChecklistPage(page)));
|
|
1585
3000
|
}
|
|
1586
3001
|
catch (error) {
|
|
3002
|
+
if (isNotebookResearchStorageUnavailable(error)) {
|
|
3003
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3004
|
+
res.end(serializeJSON(withNotebookResearchChecklistPage(emptyNotebookResearchListPage({ limit, offset }))));
|
|
3005
|
+
return;
|
|
3006
|
+
}
|
|
1587
3007
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1588
3008
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1589
3009
|
}
|
|
1590
3010
|
finally {
|
|
1591
|
-
storage
|
|
3011
|
+
storage?.close();
|
|
1592
3012
|
}
|
|
1593
3013
|
return;
|
|
1594
3014
|
}
|
|
1595
3015
|
if (req.method === 'POST' && path === '/api/notebook/research') {
|
|
1596
|
-
|
|
3016
|
+
let storage;
|
|
1597
3017
|
try {
|
|
3018
|
+
storage = openNotebookResearchStorage();
|
|
1598
3019
|
const body = await readJSON(req);
|
|
1599
3020
|
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1600
3021
|
const question = notebookResearchString(body.question);
|
|
@@ -1641,7 +3062,7 @@ export async function startLocalServer(opts) {
|
|
|
1641
3062
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1642
3063
|
}
|
|
1643
3064
|
finally {
|
|
1644
|
-
storage
|
|
3065
|
+
storage?.close();
|
|
1645
3066
|
}
|
|
1646
3067
|
return;
|
|
1647
3068
|
}
|
|
@@ -1686,8 +3107,9 @@ export async function startLocalServer(opts) {
|
|
|
1686
3107
|
return;
|
|
1687
3108
|
}
|
|
1688
3109
|
if (req.method === 'POST' && path === '/api/notebook/research/seed-cells') {
|
|
1689
|
-
|
|
3110
|
+
let storage;
|
|
1690
3111
|
try {
|
|
3112
|
+
storage = openNotebookResearchStorage();
|
|
1691
3113
|
const body = await readJSON(req);
|
|
1692
3114
|
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1693
3115
|
if (!notebookPath) {
|
|
@@ -1738,12 +3160,14 @@ export async function startLocalServer(opts) {
|
|
|
1738
3160
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1739
3161
|
}
|
|
1740
3162
|
finally {
|
|
1741
|
-
storage
|
|
3163
|
+
storage?.close();
|
|
1742
3164
|
}
|
|
1743
3165
|
return;
|
|
1744
3166
|
}
|
|
1745
3167
|
if (req.method === 'POST' && path === '/api/notebook/research/source-coverage') {
|
|
1746
|
-
|
|
3168
|
+
let storage;
|
|
3169
|
+
let requestedSourceCellCount = 0;
|
|
3170
|
+
let limit = 10_000;
|
|
1747
3171
|
try {
|
|
1748
3172
|
const body = await readJSON(req);
|
|
1749
3173
|
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
@@ -1784,15 +3208,16 @@ export async function startLocalServer(opts) {
|
|
|
1784
3208
|
const sourceCellIds = Array.from(new Set(requestedIds
|
|
1785
3209
|
.map((id) => notebookResearchString(id))
|
|
1786
3210
|
.filter((id) => Boolean(id))));
|
|
1787
|
-
|
|
3211
|
+
requestedSourceCellCount = new Set([
|
|
1788
3212
|
...sourceCellIds,
|
|
1789
3213
|
...sourceCells
|
|
1790
3214
|
.map((cell) => notebookResearchSourceCellId(cell))
|
|
1791
3215
|
.filter((id) => Boolean(id)),
|
|
1792
3216
|
]).size;
|
|
1793
|
-
|
|
3217
|
+
limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
|
|
1794
3218
|
? Math.max(1, Math.min(10_000, Math.floor(body.limit)))
|
|
1795
3219
|
: 10_000;
|
|
3220
|
+
storage = openNotebookResearchStorage();
|
|
1796
3221
|
const linkedRuns = storage.listLatestRunsBySourceCell({
|
|
1797
3222
|
notebookPath,
|
|
1798
3223
|
sourceCellIds,
|
|
@@ -1815,26 +3240,42 @@ export async function startLocalServer(opts) {
|
|
|
1815
3240
|
}));
|
|
1816
3241
|
}
|
|
1817
3242
|
catch (error) {
|
|
3243
|
+
if (isNotebookResearchStorageUnavailable(error)) {
|
|
3244
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3245
|
+
res.end(serializeJSON({
|
|
3246
|
+
runs: [],
|
|
3247
|
+
requestedCount: requestedSourceCellCount,
|
|
3248
|
+
matchedCount: 0,
|
|
3249
|
+
limitApplied: requestedSourceCellCount > limit,
|
|
3250
|
+
}));
|
|
3251
|
+
return;
|
|
3252
|
+
}
|
|
1818
3253
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1819
3254
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1820
3255
|
}
|
|
1821
3256
|
finally {
|
|
1822
|
-
storage
|
|
3257
|
+
storage?.close();
|
|
1823
3258
|
}
|
|
1824
3259
|
return;
|
|
1825
3260
|
}
|
|
1826
3261
|
if (req.method === 'GET' && path === '/api/notebook/research/diagnostics') {
|
|
1827
|
-
|
|
3262
|
+
let storage;
|
|
1828
3263
|
try {
|
|
3264
|
+
storage = openNotebookResearchStorage();
|
|
1829
3265
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1830
3266
|
res.end(serializeJSON(storage.getDiagnostics()));
|
|
1831
3267
|
}
|
|
1832
3268
|
catch (error) {
|
|
3269
|
+
if (isNotebookResearchStorageUnavailable(error)) {
|
|
3270
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3271
|
+
res.end(serializeJSON(emptyNotebookResearchDiagnostics()));
|
|
3272
|
+
return;
|
|
3273
|
+
}
|
|
1833
3274
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1834
3275
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1835
3276
|
}
|
|
1836
3277
|
finally {
|
|
1837
|
-
storage
|
|
3278
|
+
storage?.close();
|
|
1838
3279
|
}
|
|
1839
3280
|
return;
|
|
1840
3281
|
}
|
|
@@ -1842,8 +3283,9 @@ export async function startLocalServer(opts) {
|
|
|
1842
3283
|
if (notebookResearchMatch) {
|
|
1843
3284
|
const id = decodeURIComponent(notebookResearchMatch[1]);
|
|
1844
3285
|
const action = notebookResearchMatch[2];
|
|
1845
|
-
|
|
3286
|
+
let storage;
|
|
1846
3287
|
try {
|
|
3288
|
+
storage = openNotebookResearchStorage();
|
|
1847
3289
|
const run = storage.getRun(id);
|
|
1848
3290
|
if (!run) {
|
|
1849
3291
|
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -1943,7 +3385,7 @@ export async function startLocalServer(opts) {
|
|
|
1943
3385
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1944
3386
|
}
|
|
1945
3387
|
finally {
|
|
1946
|
-
storage
|
|
3388
|
+
storage?.close();
|
|
1947
3389
|
}
|
|
1948
3390
|
return;
|
|
1949
3391
|
}
|
|
@@ -2447,6 +3889,144 @@ export async function startLocalServer(opts) {
|
|
|
2447
3889
|
}
|
|
2448
3890
|
return;
|
|
2449
3891
|
}
|
|
3892
|
+
// ── Distinct values for a block column → app/dashboard filter dropdowns ──
|
|
3893
|
+
if (req.method === 'GET' && path === '/api/dashboard/filter-options') {
|
|
3894
|
+
try {
|
|
3895
|
+
const blockIdParam = url.searchParams.get('block');
|
|
3896
|
+
const blockPath = url.searchParams.get('path')
|
|
3897
|
+
?? (blockIdParam ? resolveBlockPathById(projectRoot, blockIdParam) : null);
|
|
3898
|
+
const column = (url.searchParams.get('column') ?? '').trim();
|
|
3899
|
+
const limit = Math.min(Math.max(Number(url.searchParams.get('limit') ?? 50) || 50, 1), 200);
|
|
3900
|
+
if (!blockPath || !column || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(column)) {
|
|
3901
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3902
|
+
res.end(serializeJSON({ error: 'path and a valid column are required' }));
|
|
3903
|
+
return;
|
|
3904
|
+
}
|
|
3905
|
+
const absolutePath = resolve(projectRoot, blockPath);
|
|
3906
|
+
if (!absolutePath.startsWith(projectRoot + '/') && absolutePath !== projectRoot) {
|
|
3907
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3908
|
+
res.end(serializeJSON({ error: 'path escapes project root' }));
|
|
3909
|
+
return;
|
|
3910
|
+
}
|
|
3911
|
+
if (!existsSync(absolutePath)) {
|
|
3912
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3913
|
+
res.end(serializeJSON({ error: 'block not found' }));
|
|
3914
|
+
return;
|
|
3915
|
+
}
|
|
3916
|
+
const source = readFileSync(absolutePath, 'utf-8');
|
|
3917
|
+
// Only expose distinct values for a DECLARED output column — keeps the probe
|
|
3918
|
+
// inside the governed block contract (no arbitrary column scanning).
|
|
3919
|
+
const parsedMeta = parseBlockSourceMetadata(source);
|
|
3920
|
+
if (parsedMeta.outputs.length > 0 && !parsedMeta.outputs.includes(column)) {
|
|
3921
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3922
|
+
res.end(serializeJSON({ error: `"${column}" is not a declared output of this block` }));
|
|
3923
|
+
return;
|
|
3924
|
+
}
|
|
3925
|
+
const activeConnection = requireActiveConnection();
|
|
3926
|
+
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
3927
|
+
const semanticCompose = semanticLayer
|
|
3928
|
+
? composeSemanticBlockSql(source, semanticLayer, {
|
|
3929
|
+
driver: activeConnection.driver,
|
|
3930
|
+
tableMapping,
|
|
3931
|
+
projectRoot,
|
|
3932
|
+
projectConfig,
|
|
3933
|
+
detectedProvider: semanticDetectedProvider,
|
|
3934
|
+
})
|
|
3935
|
+
: null;
|
|
3936
|
+
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
3937
|
+
const baseSql = semanticCompose?.sql ?? validation.executableSql;
|
|
3938
|
+
if (!baseSql) {
|
|
3939
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3940
|
+
res.end(serializeJSON({ error: 'block has no executable SQL' }));
|
|
3941
|
+
return;
|
|
3942
|
+
}
|
|
3943
|
+
const prepared = prepareLocalExecution(baseSql, activeConnection, projectRoot, projectConfig);
|
|
3944
|
+
const q = quoteAgentIdentifier(column, prepared.connection);
|
|
3945
|
+
const wrapped = `SELECT DISTINCT ${q} AS value FROM (${stripSqlTerminator(prepared.sql)}) _dql_opt WHERE ${q} IS NOT NULL ORDER BY 1 LIMIT ${limit + 1}`;
|
|
3946
|
+
const result = await executor.executeQuery(wrapped, [], runtimeVariables({}), prepared.connection);
|
|
3947
|
+
const rows = Array.isArray(result?.rows) ? result.rows : [];
|
|
3948
|
+
const truncated = rows.length > limit;
|
|
3949
|
+
const options = rows
|
|
3950
|
+
.slice(0, limit)
|
|
3951
|
+
.map((row) => row?.value)
|
|
3952
|
+
.filter((value) => value !== null && value !== undefined)
|
|
3953
|
+
.map((value) => String(value));
|
|
3954
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3955
|
+
res.end(serializeJSON({ column, options, truncated }));
|
|
3956
|
+
}
|
|
3957
|
+
catch (error) {
|
|
3958
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3959
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3960
|
+
}
|
|
3961
|
+
return;
|
|
3962
|
+
}
|
|
3963
|
+
// ── Plan an app from a goal (P1: plan → critique → show gaps) ──────────
|
|
3964
|
+
// The agent decomposes the goal into the questions an app should answer
|
|
3965
|
+
// (KPI + trend + breakdowns), matches each to a CERTIFIED block, derives the
|
|
3966
|
+
// shared filters that refresh every tile, and reports coverage + gaps BEFORE
|
|
3967
|
+
// anything is built — so the human reviews the plan, not a blank canvas.
|
|
3968
|
+
if (req.method === 'POST' && path === '/api/app-plan') {
|
|
3969
|
+
try {
|
|
3970
|
+
const body = await readJSON(req);
|
|
3971
|
+
const goal = typeof body.goal === 'string' ? body.goal.trim() : '';
|
|
3972
|
+
if (!goal) {
|
|
3973
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3974
|
+
res.end(serializeJSON({ error: 'goal is required' }));
|
|
3975
|
+
return;
|
|
3976
|
+
}
|
|
3977
|
+
const certifiedOnly = body.certifiedOnly !== false;
|
|
3978
|
+
const metrics = loadSemanticMetrics(projectRoot);
|
|
3979
|
+
let blocks = collectPlanBlocks(projectRoot, { certifiedOnly });
|
|
3980
|
+
// If nothing is certified yet, fall back to all drafts so the plan still
|
|
3981
|
+
// shows what COULD be assembled (every section then reads as a gap to certify).
|
|
3982
|
+
if (blocks.length === 0 && certifiedOnly)
|
|
3983
|
+
blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
|
|
3984
|
+
const plan = await planApp({ goal, metrics, blocks });
|
|
3985
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3986
|
+
res.end(serializeJSON({ plan, blockCount: blocks.length, metricCount: metrics.length, certifiedOnly }));
|
|
3987
|
+
}
|
|
3988
|
+
catch (error) {
|
|
3989
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3990
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3991
|
+
}
|
|
3992
|
+
return;
|
|
3993
|
+
}
|
|
3994
|
+
// ── Research / follow-up planning (P4: ReAct over the catalog) ─────────
|
|
3995
|
+
// Decide whether to answer, research across grounded steps, or ask a smart
|
|
3996
|
+
// follow-up — so the agent behaves like a real assistant instead of always
|
|
3997
|
+
// generating one query. Every step + option is bound to a real metric/block.
|
|
3998
|
+
if (req.method === 'POST' && path === '/api/research-plan') {
|
|
3999
|
+
try {
|
|
4000
|
+
const body = await readJSON(req);
|
|
4001
|
+
const question = typeof body.question === 'string' ? body.question.trim() : '';
|
|
4002
|
+
if (!question) {
|
|
4003
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4004
|
+
res.end(serializeJSON({ error: 'question is required' }));
|
|
4005
|
+
return;
|
|
4006
|
+
}
|
|
4007
|
+
const metrics = loadSemanticMetrics(projectRoot);
|
|
4008
|
+
let blocks = collectPlanBlocks(projectRoot, { certifiedOnly: true });
|
|
4009
|
+
if (blocks.length === 0)
|
|
4010
|
+
blocks = collectPlanBlocks(projectRoot, { certifiedOnly: false });
|
|
4011
|
+
const plan = await planResearch({
|
|
4012
|
+
question,
|
|
4013
|
+
metrics,
|
|
4014
|
+
blocks,
|
|
4015
|
+
intent: typeof body.intent === 'string'
|
|
4016
|
+
? body.intent
|
|
4017
|
+
: undefined,
|
|
4018
|
+
isFollowUp: body.isFollowUp === true,
|
|
4019
|
+
history: Array.isArray(body.history) ? body.history : undefined,
|
|
4020
|
+
});
|
|
4021
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4022
|
+
res.end(serializeJSON({ plan, blockCount: blocks.length, metricCount: metrics.length }));
|
|
4023
|
+
}
|
|
4024
|
+
catch (error) {
|
|
4025
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4026
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
4027
|
+
}
|
|
4028
|
+
return;
|
|
4029
|
+
}
|
|
2450
4030
|
// ── Run block tests ────────────────────────────────────────────────
|
|
2451
4031
|
if (req.method === 'POST' && path === '/api/blocks/run-tests') {
|
|
2452
4032
|
try {
|
|
@@ -2485,7 +4065,11 @@ export async function startLocalServer(opts) {
|
|
|
2485
4065
|
res.end(serializeJSON({ error: 'source is required' }));
|
|
2486
4066
|
return;
|
|
2487
4067
|
}
|
|
2488
|
-
|
|
4068
|
+
// OSS local-first certifies in non-enterprise mode: owner + passing tests +
|
|
4069
|
+
// a successful run are the gate; grain/outputs/pattern/lineage/cadence are
|
|
4070
|
+
// AI-filled advisory warnings, not hard blockers. Enterprise-grade certification
|
|
4071
|
+
// (all of those required) is an opt-in for the cloud tier — request it explicitly.
|
|
4072
|
+
const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise === true });
|
|
2489
4073
|
const blockers = Array.from(new Set(result.checklist.blockers));
|
|
2490
4074
|
if (!result.certification.certified || blockers.length > 0) {
|
|
2491
4075
|
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -2514,6 +4098,32 @@ export async function startLocalServer(opts) {
|
|
|
2514
4098
|
certifiedPayload = openBlockStudioDocument(projectRoot, normalizedBlockPath, semanticLayer);
|
|
2515
4099
|
}
|
|
2516
4100
|
}
|
|
4101
|
+
// Auto-capture (OSS local learning loop): certifying a block teaches the
|
|
4102
|
+
// agent to prefer it. Write a scoped, advisory project memory — deterministic,
|
|
4103
|
+
// best-effort, and it never blocks certification.
|
|
4104
|
+
try {
|
|
4105
|
+
const learned = parseBlockSourceMetadata(certifiedSource);
|
|
4106
|
+
if (learned.name) {
|
|
4107
|
+
const learnedOutputs = Array.isArray(learned.outputs)
|
|
4108
|
+
? learned.outputs.filter((o) => typeof o === 'string')
|
|
4109
|
+
: [];
|
|
4110
|
+
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
4111
|
+
memory.upsert({
|
|
4112
|
+
id: `mem_certify_${learned.name}`,
|
|
4113
|
+
scope: 'project',
|
|
4114
|
+
title: `Certified block: ${learned.name}`,
|
|
4115
|
+
content: `Prefer the certified block "${learned.name}" for ${learned.description?.trim() || `questions in the ${learned.domain ?? 'analytics'} domain`}.${learned.grain ? ` Grain: ${learned.grain}.` : ''}${learnedOutputs.length ? ` Outputs: ${learnedOutputs.slice(0, 8).join(', ')}.` : ''} It is the trusted source — reuse it instead of generating new SQL.`,
|
|
4116
|
+
tags: [learned.domain, learned.name, ...learnedOutputs.slice(0, 4)].filter((x) => Boolean(x)),
|
|
4117
|
+
source: 'certify',
|
|
4118
|
+
confidence: 0.95,
|
|
4119
|
+
importance: 0.85,
|
|
4120
|
+
enabled: true,
|
|
4121
|
+
});
|
|
4122
|
+
}
|
|
4123
|
+
}
|
|
4124
|
+
catch {
|
|
4125
|
+
/* best-effort: learning capture must never block certification */
|
|
4126
|
+
}
|
|
2517
4127
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
2518
4128
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2519
4129
|
res.end(serializeJSON({
|
|
@@ -2642,7 +4252,8 @@ export async function startLocalServer(opts) {
|
|
|
2642
4252
|
return;
|
|
2643
4253
|
}
|
|
2644
4254
|
const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
|
|
2645
|
-
|
|
4255
|
+
// OSS certify = non-enterprise (owner-gated); enterprise depth is the cloud tier.
|
|
4256
|
+
const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: false });
|
|
2646
4257
|
const blockers = Array.from(new Set(certification.checklist.blockers));
|
|
2647
4258
|
if (!certification.certification.certified || blockers.length > 0) {
|
|
2648
4259
|
const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
|
|
@@ -4576,6 +6187,14 @@ export async function startLocalServer(opts) {
|
|
|
4576
6187
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
4577
6188
|
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
|
|
4578
6189
|
const normalized = normalizeQueryResult(rawResult);
|
|
6190
|
+
// Enforce the block's declared invariants against the result set. This
|
|
6191
|
+
// is additive: blocks without invariants produce `null` and the
|
|
6192
|
+
// response is unchanged. The agent surface (`query_via_block`) reads
|
|
6193
|
+
// these fields to downgrade the trust label on violation.
|
|
6194
|
+
const invariants = evaluateBlockInvariants(executableCell.source || cell.source || '', {
|
|
6195
|
+
columns: normalized.columns,
|
|
6196
|
+
rows: normalized.rows,
|
|
6197
|
+
});
|
|
4579
6198
|
if (execContext) {
|
|
4580
6199
|
recordNotebookQueryRun(projectRoot, {
|
|
4581
6200
|
notebookPath: execContext.notebookPath,
|
|
@@ -4604,6 +6223,12 @@ export async function startLocalServer(opts) {
|
|
|
4604
6223
|
chartConfig: plan.chartConfig,
|
|
4605
6224
|
tests: plan.tests,
|
|
4606
6225
|
result: normalized,
|
|
6226
|
+
...(invariants
|
|
6227
|
+
? {
|
|
6228
|
+
invariantResults: invariants.invariantResults,
|
|
6229
|
+
invariantViolation: invariants.invariantViolation,
|
|
6230
|
+
}
|
|
6231
|
+
: {}),
|
|
4607
6232
|
}));
|
|
4608
6233
|
}
|
|
4609
6234
|
catch (error) {
|
|
@@ -4679,6 +6304,7 @@ table: ${table}${tagList}
|
|
|
4679
6304
|
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
4680
6305
|
res.end(content);
|
|
4681
6306
|
});
|
|
6307
|
+
opts.captureServer?.(server);
|
|
4682
6308
|
return new Promise((resolvePromise, reject) => {
|
|
4683
6309
|
let retriedWithRandomPort = false;
|
|
4684
6310
|
server.on('error', (error) => {
|
|
@@ -4913,6 +6539,145 @@ export function serializeJSON(value) {
|
|
|
4913
6539
|
return current;
|
|
4914
6540
|
});
|
|
4915
6541
|
}
|
|
6542
|
+
/** Serialize a Skill to the shared API contract shape (spec 16). */
|
|
6543
|
+
function serializeSkill(skill) {
|
|
6544
|
+
return {
|
|
6545
|
+
id: skill.id,
|
|
6546
|
+
scope: skill.scope,
|
|
6547
|
+
user: skill.user,
|
|
6548
|
+
domain: skill.domain,
|
|
6549
|
+
description: skill.description,
|
|
6550
|
+
body: skill.body,
|
|
6551
|
+
preferredMetrics: skill.preferredMetrics,
|
|
6552
|
+
preferredBlocks: skill.preferredBlocks,
|
|
6553
|
+
vocabulary: skill.vocabulary,
|
|
6554
|
+
sourcePath: skill.sourcePath,
|
|
6555
|
+
isStarter: skill.isStarter,
|
|
6556
|
+
};
|
|
6557
|
+
}
|
|
6558
|
+
/**
|
|
6559
|
+
* Validate + normalize an inbound `{ skill }` body into a WriteSkillInput.
|
|
6560
|
+
* `id` + `scope` + `body` are required; `fallbackId` supplies the id on PUT
|
|
6561
|
+
* (from the URL). Returns null when the payload is invalid.
|
|
6562
|
+
*/
|
|
6563
|
+
function parseSkillInput(raw, fallbackId) {
|
|
6564
|
+
if (!raw || typeof raw !== 'object')
|
|
6565
|
+
return null;
|
|
6566
|
+
const skill = raw;
|
|
6567
|
+
const id = typeof skill.id === 'string' && skill.id.trim() ? skill.id.trim() : fallbackId;
|
|
6568
|
+
if (!id)
|
|
6569
|
+
return null;
|
|
6570
|
+
const scope = skill.scope === 'personal' ? 'personal' : skill.scope === 'project' ? 'project' : undefined;
|
|
6571
|
+
if (!scope)
|
|
6572
|
+
return null;
|
|
6573
|
+
if (typeof skill.body !== 'string')
|
|
6574
|
+
return null;
|
|
6575
|
+
const asStrings = (value) => Array.isArray(value) ? value.filter((v) => typeof v === 'string') : [];
|
|
6576
|
+
const asMap = (value) => {
|
|
6577
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
6578
|
+
return {};
|
|
6579
|
+
const out = {};
|
|
6580
|
+
for (const [k, v] of Object.entries(value)) {
|
|
6581
|
+
if (typeof v === 'string')
|
|
6582
|
+
out[k] = v;
|
|
6583
|
+
}
|
|
6584
|
+
return out;
|
|
6585
|
+
};
|
|
6586
|
+
return {
|
|
6587
|
+
id,
|
|
6588
|
+
scope,
|
|
6589
|
+
user: scope === 'personal' && typeof skill.user === 'string' ? skill.user : undefined,
|
|
6590
|
+
domain: typeof skill.domain === 'string' && skill.domain.trim() ? skill.domain.trim() : undefined,
|
|
6591
|
+
description: typeof skill.description === 'string' ? skill.description : undefined,
|
|
6592
|
+
body: skill.body,
|
|
6593
|
+
preferredMetrics: asStrings(skill.preferredMetrics),
|
|
6594
|
+
preferredBlocks: asStrings(skill.preferredBlocks),
|
|
6595
|
+
vocabulary: asMap(skill.vocabulary),
|
|
6596
|
+
isStarter: skill.isStarter === true ? true : undefined,
|
|
6597
|
+
};
|
|
6598
|
+
}
|
|
6599
|
+
/** Loose, case/slug-insensitive domain key for counting membership. */
|
|
6600
|
+
function domainKey(value) {
|
|
6601
|
+
if (!value)
|
|
6602
|
+
return '';
|
|
6603
|
+
return value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
|
|
6604
|
+
}
|
|
6605
|
+
function manifestDomainToDto(domain, counts) {
|
|
6606
|
+
return {
|
|
6607
|
+
id: domain.name,
|
|
6608
|
+
name: domain.name,
|
|
6609
|
+
owner: domain.owner,
|
|
6610
|
+
boundedContext: domain.boundedContext,
|
|
6611
|
+
sourceSystems: domain.sourceSystems,
|
|
6612
|
+
description: domain.description,
|
|
6613
|
+
sourcePath: domain.filePath,
|
|
6614
|
+
blockCount: counts.blockCount,
|
|
6615
|
+
skillCount: counts.skillCount,
|
|
6616
|
+
termCount: counts.termCount,
|
|
6617
|
+
};
|
|
6618
|
+
}
|
|
6619
|
+
/** List authored domains with per-domain block/skill/term counts. */
|
|
6620
|
+
export function listDomains(projectRoot) {
|
|
6621
|
+
const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
|
|
6622
|
+
const domains = manifest.domains ?? {};
|
|
6623
|
+
const skills = loadSkills(projectRoot).skills;
|
|
6624
|
+
const blockCounts = new Map();
|
|
6625
|
+
for (const block of Object.values(manifest.blocks)) {
|
|
6626
|
+
const key = domainKey(block.domain);
|
|
6627
|
+
if (key)
|
|
6628
|
+
blockCounts.set(key, (blockCounts.get(key) ?? 0) + 1);
|
|
6629
|
+
}
|
|
6630
|
+
const termCounts = new Map();
|
|
6631
|
+
for (const term of Object.values(manifest.terms ?? {})) {
|
|
6632
|
+
const key = domainKey(term.domain);
|
|
6633
|
+
if (key)
|
|
6634
|
+
termCounts.set(key, (termCounts.get(key) ?? 0) + 1);
|
|
6635
|
+
}
|
|
6636
|
+
const skillCounts = new Map();
|
|
6637
|
+
for (const skill of skills) {
|
|
6638
|
+
const key = domainKey(skill.domain);
|
|
6639
|
+
if (key)
|
|
6640
|
+
skillCounts.set(key, (skillCounts.get(key) ?? 0) + 1);
|
|
6641
|
+
}
|
|
6642
|
+
return Object.values(domains)
|
|
6643
|
+
.map((domain) => {
|
|
6644
|
+
const key = domainKey(domain.name);
|
|
6645
|
+
return manifestDomainToDto(domain, {
|
|
6646
|
+
blockCount: blockCounts.get(key) ?? 0,
|
|
6647
|
+
skillCount: skillCounts.get(key) ?? 0,
|
|
6648
|
+
termCount: termCounts.get(key) ?? 0,
|
|
6649
|
+
});
|
|
6650
|
+
})
|
|
6651
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
6652
|
+
}
|
|
6653
|
+
/** Find a single authored domain by name/id (case/slug-insensitive). */
|
|
6654
|
+
function findDomain(projectRoot, nameOrId) {
|
|
6655
|
+
const key = domainKey(nameOrId);
|
|
6656
|
+
return listDomains(projectRoot).find((domain) => domainKey(domain.name) === key);
|
|
6657
|
+
}
|
|
6658
|
+
/** Validate + normalize an inbound `{ domain }` body into a DomainInput. */
|
|
6659
|
+
export function parseDomainInput(raw, fallbackId) {
|
|
6660
|
+
if (!raw || typeof raw !== 'object')
|
|
6661
|
+
return null;
|
|
6662
|
+
const domain = raw;
|
|
6663
|
+
const name = typeof domain.name === 'string' && domain.name.trim()
|
|
6664
|
+
? domain.name.trim()
|
|
6665
|
+
: typeof domain.id === 'string' && domain.id.trim()
|
|
6666
|
+
? domain.id.trim()
|
|
6667
|
+
: fallbackId;
|
|
6668
|
+
if (!name)
|
|
6669
|
+
return null;
|
|
6670
|
+
const asStrings = (value) => Array.isArray(value)
|
|
6671
|
+
? value.filter((v) => typeof v === 'string' && v.trim().length > 0)
|
|
6672
|
+
: undefined;
|
|
6673
|
+
return {
|
|
6674
|
+
name,
|
|
6675
|
+
owner: typeof domain.owner === 'string' ? domain.owner : undefined,
|
|
6676
|
+
boundedContext: typeof domain.boundedContext === 'string' ? domain.boundedContext : undefined,
|
|
6677
|
+
sourceSystems: asStrings(domain.sourceSystems),
|
|
6678
|
+
description: typeof domain.description === 'string' ? domain.description : undefined,
|
|
6679
|
+
};
|
|
6680
|
+
}
|
|
4916
6681
|
async function refreshLocalMetadataCatalog(projectRoot) {
|
|
4917
6682
|
try {
|
|
4918
6683
|
await ensureMetadataCatalogFresh(projectRoot, { force: true });
|
|
@@ -4985,6 +6750,153 @@ export function loadProjectConfig(projectRoot) {
|
|
|
4985
6750
|
}
|
|
4986
6751
|
return config;
|
|
4987
6752
|
}
|
|
6753
|
+
/**
|
|
6754
|
+
* Core of the `/api/propose` endpoint, factored out as a pure function so it can
|
|
6755
|
+
* be unit-tested without standing up an HTTP server.
|
|
6756
|
+
*
|
|
6757
|
+
* It reuses the existing `propose` engine from `@duckcodeailabs/dql-agent`
|
|
6758
|
+
* verbatim (no inference/ranking logic is duplicated here) in `dryRun` mode so a
|
|
6759
|
+
* readiness preview never mutates the project. Every returned proposal is a
|
|
6760
|
+
* `status: draft` block with the engine's stored Certifier verdict attached.
|
|
6761
|
+
*/
|
|
6762
|
+
export function buildProposeReadiness(projectRoot, projectConfig = loadProjectConfig(projectRoot), options = {}) {
|
|
6763
|
+
const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
6764
|
+
if (!manifestPath) {
|
|
6765
|
+
return {
|
|
6766
|
+
ready: false,
|
|
6767
|
+
reason: 'No dbt manifest found. Run `dbt parse` (or `dbt compile`) in your dbt project, then reopen Get Started.',
|
|
6768
|
+
summary: {
|
|
6769
|
+
modelsScanned: 0,
|
|
6770
|
+
businessModels: 0,
|
|
6771
|
+
plumbingExcluded: 0,
|
|
6772
|
+
metricsFound: 0,
|
|
6773
|
+
proposalsRanked: 0,
|
|
6774
|
+
draftsExisting: 0,
|
|
6775
|
+
readyForReview: 0,
|
|
6776
|
+
blockingTotal: 0,
|
|
6777
|
+
warningTotal: 0,
|
|
6778
|
+
},
|
|
6779
|
+
plan: {
|
|
6780
|
+
totals: { modelsScanned: 0, businessModels: 0, plumbingExcluded: 0, metricsFound: 0 },
|
|
6781
|
+
willGenerate: 0,
|
|
6782
|
+
willSkip: 0,
|
|
6783
|
+
domains: [],
|
|
6784
|
+
config: {
|
|
6785
|
+
businessLayers: [],
|
|
6786
|
+
excludeLayers: [],
|
|
6787
|
+
maxPerDomain: 0,
|
|
6788
|
+
minScore: 0,
|
|
6789
|
+
aiEnrichment: 'auto',
|
|
6790
|
+
},
|
|
6791
|
+
},
|
|
6792
|
+
proposals: [],
|
|
6793
|
+
};
|
|
6794
|
+
}
|
|
6795
|
+
const proposeConfig = projectConfig.propose;
|
|
6796
|
+
// PLAN: deterministic, business-only, bounded. Writes nothing.
|
|
6797
|
+
const plan = proposePlan(projectRoot, manifestPath, { config: proposeConfig });
|
|
6798
|
+
// dryRun: rank + certify the selected scope only. Never writes from a preview.
|
|
6799
|
+
// Stamp the resolved local OSS owner when none was passed so the stored verdict
|
|
6800
|
+
// does not carry a phantom "Missing owner" strike. Read-only resolution — the
|
|
6801
|
+
// preview must not mutate the project.
|
|
6802
|
+
const summary = propose({
|
|
6803
|
+
projectRoot,
|
|
6804
|
+
dbtManifestPath: manifestPath,
|
|
6805
|
+
owner: options.owner || resolveLocalOwner(projectRoot, { persist: false }),
|
|
6806
|
+
limit: options.limit,
|
|
6807
|
+
dryRun: true,
|
|
6808
|
+
config: proposeConfig,
|
|
6809
|
+
});
|
|
6810
|
+
let readyForReview = 0;
|
|
6811
|
+
let blockingTotal = 0;
|
|
6812
|
+
let warningTotal = 0;
|
|
6813
|
+
for (const proposal of summary.proposals) {
|
|
6814
|
+
blockingTotal += proposal.certification.errors.length;
|
|
6815
|
+
warningTotal += proposal.certification.warnings.length;
|
|
6816
|
+
if (proposal.certification.errors.length === 0)
|
|
6817
|
+
readyForReview += 1;
|
|
6818
|
+
}
|
|
6819
|
+
return {
|
|
6820
|
+
ready: true,
|
|
6821
|
+
summary: {
|
|
6822
|
+
projectName: summary.projectName,
|
|
6823
|
+
modelsScanned: summary.modelsScanned,
|
|
6824
|
+
businessModels: summary.businessModels,
|
|
6825
|
+
plumbingExcluded: summary.plumbingExcluded,
|
|
6826
|
+
metricsFound: summary.metricsFound,
|
|
6827
|
+
proposalsRanked: summary.proposalsRanked,
|
|
6828
|
+
// In dryRun the engine marks already-present blocks as skipped.
|
|
6829
|
+
draftsExisting: summary.draftsSkipped,
|
|
6830
|
+
readyForReview,
|
|
6831
|
+
blockingTotal,
|
|
6832
|
+
warningTotal,
|
|
6833
|
+
},
|
|
6834
|
+
plan,
|
|
6835
|
+
proposals: summary.proposals,
|
|
6836
|
+
};
|
|
6837
|
+
}
|
|
6838
|
+
export async function generateProposeDrafts(projectRoot, slugs, projectConfig = loadProjectConfig(projectRoot), options = {}) {
|
|
6839
|
+
const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
6840
|
+
if (!manifestPath) {
|
|
6841
|
+
return {
|
|
6842
|
+
ready: false,
|
|
6843
|
+
reason: 'No dbt manifest found. Run `dbt parse` (or `dbt compile`) first.',
|
|
6844
|
+
draftsWritten: 0,
|
|
6845
|
+
draftsSkipped: 0,
|
|
6846
|
+
proposals: [],
|
|
6847
|
+
};
|
|
6848
|
+
}
|
|
6849
|
+
// Structure deterministic, content AI-optional: optionally pre-compute AI
|
|
6850
|
+
// enrichment (description / llmContext / examples) for the approved slugs, then
|
|
6851
|
+
// hand it to the deterministic engine as data. Best-effort — any failure or a
|
|
6852
|
+
// missing provider falls back to dbt-derived content.
|
|
6853
|
+
let enrichedBySlug;
|
|
6854
|
+
const proposeConfig = resolveProposeConfig(projectConfig.propose);
|
|
6855
|
+
if (proposeConfig.aiEnrichment !== 'off' && slugs.length > 0) {
|
|
6856
|
+
enrichedBySlug = await gatherProposeEnrichment(projectRoot, manifestPath, projectConfig.propose, slugs).catch(() => undefined);
|
|
6857
|
+
}
|
|
6858
|
+
const summary = propose({
|
|
6859
|
+
projectRoot,
|
|
6860
|
+
dbtManifestPath: manifestPath,
|
|
6861
|
+
// Stamp the resolved local OSS owner when none was passed so drafts are not
|
|
6862
|
+
// born with a "Missing owner" Certifier strike.
|
|
6863
|
+
owner: options.owner || resolveLocalOwner(projectRoot),
|
|
6864
|
+
config: projectConfig.propose,
|
|
6865
|
+
onlySlugs: slugs,
|
|
6866
|
+
enrichedBySlug,
|
|
6867
|
+
});
|
|
6868
|
+
return {
|
|
6869
|
+
ready: true,
|
|
6870
|
+
draftsWritten: summary.draftsWritten,
|
|
6871
|
+
draftsSkipped: summary.draftsSkipped,
|
|
6872
|
+
proposals: summary.proposals,
|
|
6873
|
+
};
|
|
6874
|
+
}
|
|
6875
|
+
/**
|
|
6876
|
+
* Build the FILLED transparent preview for ONE proposed candidate slug (spec 14,
|
|
6877
|
+
* part A). Reuses the deterministic `buildProposePreview` engine (real SQL +
|
|
6878
|
+
* Certifier verdict) and best-effort AI enrichment (description/llmContext/
|
|
6879
|
+
* examples) when a provider is available. Writes NOTHING. Returns `undefined`
|
|
6880
|
+
* when the slug is not part of the bounded, business-only selection.
|
|
6881
|
+
*/
|
|
6882
|
+
export async function buildProposeCandidatePreview(projectRoot, slug, owner, projectConfig = loadProjectConfig(projectRoot)) {
|
|
6883
|
+
const manifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
6884
|
+
if (!manifestPath)
|
|
6885
|
+
return undefined;
|
|
6886
|
+
// Best-effort AI enrichment for this one slug (content only). Any miss falls
|
|
6887
|
+
// back to the deterministic dbt-derived content inside buildProposePreview.
|
|
6888
|
+
let enriched;
|
|
6889
|
+
const proposeConfig = resolveProposeConfig(projectConfig.propose);
|
|
6890
|
+
if (proposeConfig.aiEnrichment !== 'off') {
|
|
6891
|
+
const map = await gatherProposeEnrichment(projectRoot, manifestPath, projectConfig.propose, [slug]).catch(() => undefined);
|
|
6892
|
+
enriched = map?.get(slug);
|
|
6893
|
+
}
|
|
6894
|
+
return buildProposePreview(projectRoot, manifestPath, slug, {
|
|
6895
|
+
config: projectConfig.propose,
|
|
6896
|
+
owner: owner || resolveLocalOwner(projectRoot),
|
|
6897
|
+
enriched,
|
|
6898
|
+
});
|
|
6899
|
+
}
|
|
4988
6900
|
function getProjectConnectionsForApi(config) {
|
|
4989
6901
|
const connections = getStoredConnections(config);
|
|
4990
6902
|
if (Object.keys(connections).length === 0 && isConnectionLike(config.defaultConnection)) {
|
|
@@ -4997,6 +6909,11 @@ const CONNECTOR_INSTALLS = {
|
|
|
4997
6909
|
driver: 'duckdb',
|
|
4998
6910
|
label: 'DuckDB',
|
|
4999
6911
|
packageName: 'duckdb',
|
|
6912
|
+
// Latest 1.x. An earlier pin to 1.1.3 worked around a BIGINT serialization crash,
|
|
6913
|
+
// but that only bites a naive `JSON.stringify` — the DQL driver
|
|
6914
|
+
// (`normalizeDuckDBValue`) coerces BIGINT→number before marshaling and
|
|
6915
|
+
// `serializeJSON` has a BigInt replacer, so 1.4.x is verified-good on the local
|
|
6916
|
+
// DuckDB path (COUNT/AVG/SELECT * over UUID/BIGINT/decimal/datetime on real data).
|
|
5000
6917
|
packageSpec: 'duckdb@^1.1.0',
|
|
5001
6918
|
builtIn: false,
|
|
5002
6919
|
},
|
|
@@ -5345,6 +7262,10 @@ function buildDashboardFilterPredicate(input) {
|
|
|
5345
7262
|
const end = addDashboardFilterParam(input, 'end', range.end);
|
|
5346
7263
|
return `${input.expression} BETWEEN $${start.position} AND $${end.position}`;
|
|
5347
7264
|
}
|
|
7265
|
+
// A daterange with only one bound set is incomplete — skip it rather than binding
|
|
7266
|
+
// the partial object as a scalar equality (which would produce invalid SQL).
|
|
7267
|
+
if (input.filterType === 'daterange')
|
|
7268
|
+
return null;
|
|
5348
7269
|
const values = Array.isArray(input.value) ? input.value.filter((item) => !isEmptyDashboardFilterValue(item)) : [input.value];
|
|
5349
7270
|
if (values.length === 0)
|
|
5350
7271
|
return null;
|
|
@@ -6191,6 +8112,16 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
6191
8112
|
if (diagnostics.some((diagnostic) => diagnostic.severity === 'error')) {
|
|
6192
8113
|
return { sql: null, diagnostics, semanticRefs };
|
|
6193
8114
|
}
|
|
8115
|
+
// Import-adapter shape: a semantic block may carry a pre-compiled `query` (the
|
|
8116
|
+
// governed metric already expanded to runnable SQL). When present, RUN THAT —
|
|
8117
|
+
// the `metric` field is provenance/governance, not something to recompile. This
|
|
8118
|
+
// keeps metric-bound blocks runnable offline (the full MetricFlow engine may be
|
|
8119
|
+
// unavailable). We still validated the metric exists above. Returning the query
|
|
8120
|
+
// as the composed SQL lets the normal {{ ref() }} resolution + execution apply.
|
|
8121
|
+
const precompiledQuery = extractBlockStudioSql(source);
|
|
8122
|
+
if (precompiledQuery) {
|
|
8123
|
+
return { sql: precompiledQuery, diagnostics, semanticRefs };
|
|
8124
|
+
}
|
|
6194
8125
|
let composed;
|
|
6195
8126
|
try {
|
|
6196
8127
|
composed = options?.projectRoot && options.projectConfig
|
|
@@ -6536,7 +8467,7 @@ function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
|
|
|
6536
8467
|
}
|
|
6537
8468
|
}
|
|
6538
8469
|
export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
6539
|
-
const slug = options.name.toLowerCase().replace(/[^a-z0-
|
|
8470
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
|
|
6540
8471
|
const safeDomain = (options.domain ?? '')
|
|
6541
8472
|
.trim()
|
|
6542
8473
|
.toLowerCase()
|
|
@@ -6576,7 +8507,7 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
|
6576
8507
|
return targetRelativePath;
|
|
6577
8508
|
}
|
|
6578
8509
|
export function saveBlockStudioDraftArtifacts(projectRoot, options) {
|
|
6579
|
-
const slug = options.name.toLowerCase().replace(/[^a-z0-
|
|
8510
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
|
|
6580
8511
|
const safeDomain = (options.domain ?? '')
|
|
6581
8512
|
.trim()
|
|
6582
8513
|
.toLowerCase()
|
|
@@ -6719,8 +8650,132 @@ function readBlockCompanionFile(projectRoot, relativePath) {
|
|
|
6719
8650
|
return null;
|
|
6720
8651
|
}
|
|
6721
8652
|
}
|
|
8653
|
+
/**
|
|
8654
|
+
* Extract the declared `invariants` from a block's DQL source using the core
|
|
8655
|
+
* parser (the same path that populates the manifest). Returns an empty array
|
|
8656
|
+
* when the source has no invariants or cannot be parsed — invariant evaluation
|
|
8657
|
+
* is best-effort and must never break a run.
|
|
8658
|
+
*/
|
|
8659
|
+
export function extractBlockInvariants(source) {
|
|
8660
|
+
// Only DQL block sources declare invariants. Never hand a non-DQL cell source
|
|
8661
|
+
// (e.g. a raw SQL notebook cell like "SELECT 1") to the DQL parser: it can loop
|
|
8662
|
+
// on input it doesn't recognize and OOM-crash the runtime on every execute.
|
|
8663
|
+
// Require a `block "..."` declaration before parsing.
|
|
8664
|
+
if (!/(?:^|\n)\s*block\s+["']/.test(source))
|
|
8665
|
+
return [];
|
|
8666
|
+
try {
|
|
8667
|
+
const ast = new Parser(source).parse();
|
|
8668
|
+
const block = ast.statements.find((statement) => statement.kind === 'BlockDecl');
|
|
8669
|
+
return Array.isArray(block?.invariants) ? block.invariants : [];
|
|
8670
|
+
}
|
|
8671
|
+
catch {
|
|
8672
|
+
return [];
|
|
8673
|
+
}
|
|
8674
|
+
}
|
|
8675
|
+
/**
|
|
8676
|
+
* Evaluate a block's declared invariants against a normalized query result.
|
|
8677
|
+
* Returns `null` when the block declares no invariants so callers can omit the
|
|
8678
|
+
* field entirely (blocks without invariants behave exactly as before).
|
|
8679
|
+
*/
|
|
8680
|
+
export function evaluateBlockInvariants(source, result) {
|
|
8681
|
+
const invariants = extractBlockInvariants(source);
|
|
8682
|
+
if (invariants.length === 0)
|
|
8683
|
+
return null;
|
|
8684
|
+
const invariantResults = evaluateInvariants(invariants, {
|
|
8685
|
+
columns: result.columns,
|
|
8686
|
+
rows: result.rows,
|
|
8687
|
+
});
|
|
8688
|
+
return { invariantResults, invariantViolation: hasInvariantViolation(invariantResults) };
|
|
8689
|
+
}
|
|
8690
|
+
/** Resolve a block id/slug to its `.dql` path (filename slug first, then `block "<id>"`). */
|
|
8691
|
+
function resolveBlockPathById(projectRoot, blockId) {
|
|
8692
|
+
const wanted = blockId.trim().toLowerCase();
|
|
8693
|
+
if (!wanted)
|
|
8694
|
+
return null;
|
|
8695
|
+
const leaf = wanted.split('.').pop() ?? wanted;
|
|
8696
|
+
const escaped = leaf.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
8697
|
+
const nameRe = new RegExp(`^\\s*block\\s+"${escaped}"`, 'im');
|
|
8698
|
+
const stack = ['blocks', 'domains'].map((dir) => join(projectRoot, dir));
|
|
8699
|
+
let nameMatch = null;
|
|
8700
|
+
while (stack.length > 0) {
|
|
8701
|
+
const dir = stack.pop();
|
|
8702
|
+
if (!existsSync(dir))
|
|
8703
|
+
continue;
|
|
8704
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
8705
|
+
const filePath = join(dir, entry.name);
|
|
8706
|
+
if (entry.isDirectory()) {
|
|
8707
|
+
stack.push(filePath);
|
|
8708
|
+
continue;
|
|
8709
|
+
}
|
|
8710
|
+
if (!entry.name.endsWith('.dql'))
|
|
8711
|
+
continue;
|
|
8712
|
+
const rel = relative(projectRoot, filePath).replaceAll('\\', '/');
|
|
8713
|
+
if (entry.name.replace(/\.dql$/, '').toLowerCase() === leaf)
|
|
8714
|
+
return rel; // fast path: filename slug
|
|
8715
|
+
if (!nameMatch) {
|
|
8716
|
+
try {
|
|
8717
|
+
if (nameRe.test(readFileSync(filePath, 'utf-8')))
|
|
8718
|
+
nameMatch = rel;
|
|
8719
|
+
}
|
|
8720
|
+
catch { /* skip */ }
|
|
8721
|
+
}
|
|
8722
|
+
}
|
|
8723
|
+
}
|
|
8724
|
+
return nameMatch;
|
|
8725
|
+
}
|
|
8726
|
+
/**
|
|
8727
|
+
* Collect the workspace's blocks as the App planner needs to see them (name +
|
|
8728
|
+
* governed metric + filterable dimensions). Walks blocks/ and domains/, parsing
|
|
8729
|
+
* each `.dql`. Certified-only by default — the planner builds an app from trusted
|
|
8730
|
+
* material and reports the rest as gaps.
|
|
8731
|
+
*/
|
|
8732
|
+
function collectPlanBlocks(projectRoot, opts = {}) {
|
|
8733
|
+
const out = [];
|
|
8734
|
+
const seen = new Set();
|
|
8735
|
+
const stack = ['blocks', 'domains'].map((dir) => join(projectRoot, dir));
|
|
8736
|
+
while (stack.length > 0) {
|
|
8737
|
+
const dir = stack.pop();
|
|
8738
|
+
if (!existsSync(dir))
|
|
8739
|
+
continue;
|
|
8740
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
8741
|
+
const filePath = join(dir, entry.name);
|
|
8742
|
+
if (entry.isDirectory()) {
|
|
8743
|
+
stack.push(filePath);
|
|
8744
|
+
continue;
|
|
8745
|
+
}
|
|
8746
|
+
if (!entry.name.endsWith('.dql'))
|
|
8747
|
+
continue;
|
|
8748
|
+
let source;
|
|
8749
|
+
try {
|
|
8750
|
+
source = readFileSync(filePath, 'utf-8');
|
|
8751
|
+
}
|
|
8752
|
+
catch {
|
|
8753
|
+
continue;
|
|
8754
|
+
}
|
|
8755
|
+
const meta = parseBlockSourceMetadata(source);
|
|
8756
|
+
if (!meta.name || seen.has(meta.name))
|
|
8757
|
+
continue;
|
|
8758
|
+
if (opts.certifiedOnly && meta.status !== 'certified')
|
|
8759
|
+
continue;
|
|
8760
|
+
seen.add(meta.name);
|
|
8761
|
+
out.push({
|
|
8762
|
+
name: meta.name,
|
|
8763
|
+
domain: meta.domain || undefined,
|
|
8764
|
+
description: meta.description || undefined,
|
|
8765
|
+
metricRef: meta.metricRef || meta.metricsRef[0] || undefined,
|
|
8766
|
+
allowedFilters: meta.allowedFilters,
|
|
8767
|
+
dimensions: meta.dimensions,
|
|
8768
|
+
});
|
|
8769
|
+
}
|
|
8770
|
+
}
|
|
8771
|
+
return out;
|
|
8772
|
+
}
|
|
6722
8773
|
export function parseBlockSourceMetadata(source) {
|
|
6723
|
-
|
|
8774
|
+
// Multiline: a `.dql` typically opens with a comment header (e.g. a proposed-draft
|
|
8775
|
+
// banner), so the `block "<name>"` declaration is NOT at the start of the string.
|
|
8776
|
+
// Without `m` the name parsed empty and every certified block was saved as the
|
|
8777
|
+
// generic `block.dql` (slug fallback) — colliding on the 2nd certify (BLOCK_EXISTS).
|
|
8778
|
+
const name = source.match(/^\s*block\s+"([^"]+)"/im)?.[1] ?? '';
|
|
6724
8779
|
const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
|
|
6725
8780
|
const extractStringArray = (key) => {
|
|
6726
8781
|
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'i'));
|
|
@@ -6761,6 +8816,8 @@ export function parseBlockSourceMetadata(source) {
|
|
|
6761
8816
|
sourceSystems: extractStringArray('sourceSystems'),
|
|
6762
8817
|
replacementFor: extractStringArray('replacementFor'),
|
|
6763
8818
|
reviewCadence: extractString('reviewCadence'),
|
|
8819
|
+
metricRef: extractString('metric'),
|
|
8820
|
+
metricsRef: extractStringArray('metrics'),
|
|
6764
8821
|
};
|
|
6765
8822
|
}
|
|
6766
8823
|
function compareBlockStudioValues(actual, operator, expected) {
|
|
@@ -6810,14 +8867,15 @@ function buildBlockStudioCertificationChecklist(input) {
|
|
|
6810
8867
|
blockers.add(blocker);
|
|
6811
8868
|
if (!input.previewSucceeded)
|
|
6812
8869
|
blockers.add('Block has not run successfully');
|
|
6813
|
-
|
|
8870
|
+
// Only a REAL test failure blocks. "No tests" and "no chart" are advisory review
|
|
8871
|
+
// items the AI fills at draft time — never hard gates (owner is the gate). This is
|
|
8872
|
+
// what keeps "build a DQL" from demanding ~13 fields per block at 300-query scale.
|
|
8873
|
+
if (input.testResults && input.testResults.failed > 0)
|
|
6814
8874
|
blockers.add('Tests must pass before certification');
|
|
6815
|
-
if (!input.testResults || input.testResults.assertions.length === 0)
|
|
6816
|
-
blockers.add('At least one test assertion is required before certification');
|
|
6817
|
-
if (!input.validation.chartConfig?.chart)
|
|
6818
|
-
blockers.add('Visualization config is missing');
|
|
6819
8875
|
return {
|
|
6820
|
-
metadata
|
|
8876
|
+
// The metadata tab is green once the one required field (owner) is present.
|
|
8877
|
+
// Description/domain are AI-filled + editable and surface as review items, not gates.
|
|
8878
|
+
metadata: Boolean(parsed.owner.trim()),
|
|
6821
8879
|
validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
6822
8880
|
run: input.previewSucceeded,
|
|
6823
8881
|
tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
|
|
@@ -7828,13 +9886,13 @@ async function createBlockStudioAssistProvider(projectRoot, requestedProvider) {
|
|
|
7828
9886
|
let provider;
|
|
7829
9887
|
switch (selected.id) {
|
|
7830
9888
|
case 'anthropic':
|
|
7831
|
-
provider = new ClaudeProvider({ apiKey: config.apiKey, model: config.model });
|
|
9889
|
+
provider = new ClaudeProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
|
|
7832
9890
|
break;
|
|
7833
9891
|
case 'openai':
|
|
7834
9892
|
provider = new OpenAIProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
|
|
7835
9893
|
break;
|
|
7836
9894
|
case 'gemini':
|
|
7837
|
-
provider = new GeminiProvider({ apiKey: config.apiKey, model: config.model });
|
|
9895
|
+
provider = new GeminiProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
|
|
7838
9896
|
break;
|
|
7839
9897
|
case 'ollama':
|
|
7840
9898
|
provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
|
|
@@ -7975,7 +10033,7 @@ function resolveBlockWriteTarget(projectRoot, safeDomain, slug) {
|
|
|
7975
10033
|
return { relativePath, absPath: join(projectRoot, relativePath) };
|
|
7976
10034
|
}
|
|
7977
10035
|
export function createBlockArtifacts(projectRoot, options) {
|
|
7978
|
-
const slug = options.name.toLowerCase().replace(/[^a-z0-
|
|
10036
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
|
|
7979
10037
|
const safeDomain = (options.domain ?? '')
|
|
7980
10038
|
.trim()
|
|
7981
10039
|
.toLowerCase()
|
|
@@ -8030,7 +10088,7 @@ export function createBlockArtifacts(projectRoot, options) {
|
|
|
8030
10088
|
};
|
|
8031
10089
|
}
|
|
8032
10090
|
export function createSemanticBuilderBlock(projectRoot, options) {
|
|
8033
|
-
const slug = options.name.toLowerCase().replace(/[^a-z0-
|
|
10091
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9_]+/g, '-').replace(/^[-_]+|[-_]+$/g, '') || 'block';
|
|
8034
10092
|
const safeDomain = (options.domain ?? '')
|
|
8035
10093
|
.trim()
|
|
8036
10094
|
.toLowerCase()
|
|
@@ -8493,6 +10551,19 @@ export function discoverDbtProfileConnections(projectRoot, projectConfig) {
|
|
|
8493
10551
|
const mapped = mapDbtProfileOutput(output);
|
|
8494
10552
|
if (!mapped)
|
|
8495
10553
|
continue;
|
|
10554
|
+
// dbt resolves a relative duckdb `path` against the dbt project dir (where it runs),
|
|
10555
|
+
// NOT the DQL workspace. Resolve to an absolute path here so the imported connection
|
|
10556
|
+
// opens the real warehouse instead of silently creating an empty db next to dql.config.json.
|
|
10557
|
+
if (mapped.adapter === 'duckdb' &&
|
|
10558
|
+
mapped.connection.filepath &&
|
|
10559
|
+
mapped.connection.filepath !== ':memory:') {
|
|
10560
|
+
if (!isAbsoluteLikePath(mapped.connection.filepath)) {
|
|
10561
|
+
mapped.connection.filepath = resolve(dbtProjectPath, mapped.connection.filepath);
|
|
10562
|
+
}
|
|
10563
|
+
if (!existsSync(mapped.connection.filepath)) {
|
|
10564
|
+
mapped.warnings.push(`DuckDB file not found at ${mapped.connection.filepath} — run \`dbt build\` (or \`dbt seed\`) to create it before querying.`);
|
|
10565
|
+
}
|
|
10566
|
+
}
|
|
8496
10567
|
const warnings = [...mapped.warnings];
|
|
8497
10568
|
if (targetName !== defaultTarget) {
|
|
8498
10569
|
warnings.push(`Not the default dbt target "${defaultTarget}".`);
|
|
@@ -9605,7 +11676,7 @@ async function testProviderConfig(projectRoot, id) {
|
|
|
9605
11676
|
let provider;
|
|
9606
11677
|
switch (id) {
|
|
9607
11678
|
case 'gemini':
|
|
9608
|
-
provider = new GeminiProvider({ apiKey: config.apiKey, model: config.model });
|
|
11679
|
+
provider = new GeminiProvider({ apiKey: config.apiKey, baseUrl: config.baseUrl, model: config.model });
|
|
9609
11680
|
break;
|
|
9610
11681
|
case 'ollama':
|
|
9611
11682
|
provider = new OllamaProvider({ baseUrl: config.baseUrl, model: config.model });
|
|
@@ -9666,7 +11737,10 @@ async function testAnthropicProviderConfig(config, label, details) {
|
|
|
9666
11737
|
return { ok: false, message: `${label} is not configured${details}. Add an API key in Settings or ANTHROPIC_API_KEY.` };
|
|
9667
11738
|
}
|
|
9668
11739
|
try {
|
|
9669
|
-
const client = new Anthropic({
|
|
11740
|
+
const client = new Anthropic({
|
|
11741
|
+
apiKey: config.apiKey,
|
|
11742
|
+
...(config.baseUrl ? { baseURL: normalizeAnthropicBaseUrl(config.baseUrl) } : {}),
|
|
11743
|
+
});
|
|
9670
11744
|
const response = await client.messages.create({
|
|
9671
11745
|
model: config.model ?? 'claude-opus-4-8',
|
|
9672
11746
|
max_tokens: 16,
|