@duckcodeailabs/dql-cli 1.6.18 → 1.6.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts +41 -1
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +297 -18
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +4 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +10 -1
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-Dgqvyn5s.js +3886 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/block-studio-import.d.ts +99 -3
- package/dist/block-studio-import.d.ts.map +1 -1
- package/dist/block-studio-import.js +772 -27
- package/dist/block-studio-import.js.map +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +12 -1
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts +2 -1
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +22 -4
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/certify.d.ts.map +1 -1
- package/dist/commands/certify.js +104 -10
- package/dist/commands/certify.js.map +1 -1
- package/dist/commands/compile.d.ts +1 -0
- package/dist/commands/compile.d.ts.map +1 -1
- package/dist/commands/compile.js +26 -4
- package/dist/commands/compile.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +345 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/import.d.ts.map +1 -1
- package/dist/commands/import.js +45 -70
- package/dist/commands/import.js.map +1 -1
- package/dist/commands/lineage.d.ts +2 -0
- package/dist/commands/lineage.d.ts.map +1 -1
- package/dist/commands/lineage.js +111 -0
- package/dist/commands/lineage.js.map +1 -1
- package/dist/commands/migrate.d.ts +1 -0
- package/dist/commands/migrate.d.ts.map +1 -1
- package/dist/commands/migrate.js +106 -2
- package/dist/commands/migrate.js.map +1 -1
- package/dist/commands/new.d.ts.map +1 -1
- package/dist/commands/new.js +246 -9
- package/dist/commands/new.js.map +1 -1
- package/dist/commands/promote.d.ts +11 -0
- package/dist/commands/promote.d.ts.map +1 -0
- package/dist/commands/promote.js +236 -0
- package/dist/commands/promote.js.map +1 -0
- package/dist/commands/semantic.js +2 -2
- package/dist/commands/semantic.js.map +1 -1
- package/dist/index.js +48 -8
- package/dist/index.js.map +1 -1
- package/dist/llm/providers/claude-agent-sdk.js +1 -1
- package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
- package/dist/llm/tools.js +1 -1
- package/dist/llm/tools.js.map +1 -1
- package/dist/local-runtime.d.ts +176 -5
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +3695 -146
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/promote-from-draft.d.ts +11 -10
- package/dist/promote-from-draft.d.ts.map +1 -1
- package/dist/promote-from-draft.js +53 -30
- package/dist/promote-from-draft.js.map +1 -1
- package/dist/schedule/runner.js +2 -2
- package/dist/schedule/runner.js.map +1 -1
- package/dist/semantic-import.d.ts.map +1 -1
- package/dist/semantic-import.js +28 -12
- package/dist/semantic-import.js.map +1 -1
- package/package.json +11 -11
- package/dist/assets/dql-notebook/assets/index-BbGJ8gZk.js +0 -3639
- package/dist/assets/dql-notebook/assets/index-C4oZK5xm.js +0 -3612
- package/dist/assets/dql-notebook/assets/index-CEMXuPm0.js +0 -3802
- package/dist/assets/dql-notebook/assets/index-CIMLd3Cb.js +0 -3289
- package/dist/assets/dql-notebook/assets/index-DISMWw19.js +0 -3780
- package/dist/assets/dql-notebook/assets/index-DPcxWQuS.js +0 -3859
- package/dist/assets/dql-notebook/assets/index-McpLeeN0.js +0 -3802
package/dist/local-runtime.js
CHANGED
|
@@ -7,21 +7,50 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
|
|
|
7
7
|
import Anthropic from '@anthropic-ai/sdk';
|
|
8
8
|
import OpenAI from 'openai';
|
|
9
9
|
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
10
|
-
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
|
|
10
|
+
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
|
|
11
11
|
import { load as loadYaml } from 'js-yaml';
|
|
12
12
|
import { listBlockTemplates } from './block-templates.js';
|
|
13
13
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
14
14
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
15
|
-
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
|
|
16
|
-
import { handleAppsApi } from './apps-api.js';
|
|
15
|
+
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordQueryRun, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
|
|
16
|
+
import { handleAppsApi, recommendVisualization } from './apps-api.js';
|
|
17
17
|
import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
18
18
|
import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
|
|
19
|
-
import { LocalAppStorage, defaultLocalAppsDbPath } from '@duckcodeailabs/dql-project';
|
|
20
|
-
import { Certifier } from '@duckcodeailabs/dql-governance';
|
|
19
|
+
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
20
|
+
import { Certifier, ENTERPRISE_RULES } from '@duckcodeailabs/dql-governance';
|
|
21
21
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
22
|
-
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
22
|
+
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
23
23
|
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
|
|
24
24
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
25
|
+
export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
|
|
26
|
+
const configured = projectConfig.semanticLayer;
|
|
27
|
+
const dbtProjectDir = projectConfig.dbt?.projectDir;
|
|
28
|
+
if (dbtProjectDir
|
|
29
|
+
&& (!configured || configured.provider === 'dql')
|
|
30
|
+
&& hasDbtSemanticArtifacts(projectRoot, dbtProjectDir)) {
|
|
31
|
+
return { provider: 'dbt', projectPath: dbtProjectDir };
|
|
32
|
+
}
|
|
33
|
+
return configured;
|
|
34
|
+
}
|
|
35
|
+
function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
|
|
36
|
+
const dbtRoot = resolve(projectRoot, dbtProjectDir);
|
|
37
|
+
if (existsSync(join(dbtRoot, 'target', 'semantic_manifest.json')))
|
|
38
|
+
return true;
|
|
39
|
+
const manifestPath = join(dbtRoot, 'target', 'manifest.json');
|
|
40
|
+
if (!existsSync(manifestPath))
|
|
41
|
+
return false;
|
|
42
|
+
try {
|
|
43
|
+
const parsed = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
44
|
+
if (parsed.semantic_models && typeof parsed.semantic_models === 'object')
|
|
45
|
+
return true;
|
|
46
|
+
if (parsed.metrics && typeof parsed.metrics === 'object')
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
25
54
|
export async function startLocalServer(opts) {
|
|
26
55
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
27
56
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
@@ -40,7 +69,7 @@ export async function startLocalServer(opts) {
|
|
|
40
69
|
const semanticLayerDir = join(projectRoot, 'semantic-layer');
|
|
41
70
|
let semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
42
71
|
const userPrefsPath = join(projectRoot, '.dql-user-prefs.json');
|
|
43
|
-
const semanticConfig = projectConfig
|
|
72
|
+
const semanticConfig = resolveProjectSemanticConfig(projectConfig, projectRoot);
|
|
44
73
|
let semanticLastSyncTime = null;
|
|
45
74
|
{
|
|
46
75
|
const semanticConnection = connection;
|
|
@@ -330,6 +359,430 @@ export async function startLocalServer(opts) {
|
|
|
330
359
|
providerUsed: governedAnswer.providerUsed,
|
|
331
360
|
};
|
|
332
361
|
};
|
|
362
|
+
const openNotebookResearchStorage = () => new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
|
|
363
|
+
const runNotebookResearch = async (storage, run, input = {}) => {
|
|
364
|
+
const question = notebookResearchString(input.question) || run.question;
|
|
365
|
+
const domain = notebookResearchString(input.domain) ?? run.domain;
|
|
366
|
+
const owner = notebookResearchString(input.owner) ?? run.owner;
|
|
367
|
+
const sourceCellFingerprint = notebookResearchString(input.sourceCellFingerprint) ?? run.sourceCellFingerprint;
|
|
368
|
+
const intent = input.intent ?? run.intent;
|
|
369
|
+
const context = input.context === undefined ? run.context : input.context;
|
|
370
|
+
let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
|
|
371
|
+
let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
|
|
372
|
+
const startedAt = new Date().toISOString();
|
|
373
|
+
storage.updateRun(run.id, {
|
|
374
|
+
domain,
|
|
375
|
+
owner,
|
|
376
|
+
sourceCellFingerprint,
|
|
377
|
+
question,
|
|
378
|
+
intent,
|
|
379
|
+
context,
|
|
380
|
+
generatedSql,
|
|
381
|
+
reviewedSql,
|
|
382
|
+
status: 'running',
|
|
383
|
+
reviewStatus: 'needs_review',
|
|
384
|
+
error: '',
|
|
385
|
+
});
|
|
386
|
+
try {
|
|
387
|
+
const contextPack = await buildLocalContextPack(projectRoot, {
|
|
388
|
+
question,
|
|
389
|
+
mode: 'question',
|
|
390
|
+
surface: 'notebook',
|
|
391
|
+
selectedContext: {
|
|
392
|
+
...notebookResearchSelectedContext(run, context),
|
|
393
|
+
domain,
|
|
394
|
+
owner,
|
|
395
|
+
intent,
|
|
396
|
+
researchPattern: notebookResearchIntentPattern(intent),
|
|
397
|
+
},
|
|
398
|
+
strictness: 'balanced',
|
|
399
|
+
limit: 100,
|
|
400
|
+
}).catch((error) => {
|
|
401
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
402
|
+
return {
|
|
403
|
+
id: '',
|
|
404
|
+
routeDecision: undefined,
|
|
405
|
+
evidenceRoles: [],
|
|
406
|
+
warnings: [`Context pack failed: ${message}`],
|
|
407
|
+
retrievalDiagnostics: { selectedEvidence: [] },
|
|
408
|
+
};
|
|
409
|
+
});
|
|
410
|
+
let governedAnswer;
|
|
411
|
+
let providerError;
|
|
412
|
+
const generationWarnings = [];
|
|
413
|
+
if (!generatedSql && !reviewedSql) {
|
|
414
|
+
const resolvedProvider = resolveDefaultLLMProvider(projectRoot);
|
|
415
|
+
const runner = resolvedProvider ? getLLMRunner(resolvedProvider) : null;
|
|
416
|
+
if (!resolvedProvider || !runner) {
|
|
417
|
+
generationWarnings.push('No AI provider is configured. Metadata context was saved as a research plan; paste SQL or configure an AI provider to generate candidate SQL.');
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
const controller = new AbortController();
|
|
421
|
+
try {
|
|
422
|
+
await runner.run({
|
|
423
|
+
provider: resolvedProvider,
|
|
424
|
+
messages: [{ role: 'user', content: notebookResearchAgentPrompt(question, intent) }],
|
|
425
|
+
upstream: {
|
|
426
|
+
cellId: `notebook-research:${run.notebookPath}:${run.id}`,
|
|
427
|
+
sql: JSON.stringify({
|
|
428
|
+
mode: 'notebook_research',
|
|
429
|
+
notebookPath: run.notebookPath,
|
|
430
|
+
sourceCellId: run.sourceCellId,
|
|
431
|
+
sourceCellName: run.sourceCellName,
|
|
432
|
+
owner,
|
|
433
|
+
intent,
|
|
434
|
+
researchPattern: notebookResearchIntentPattern(intent),
|
|
435
|
+
instruction: 'Generate review-required read-only SQL from the inspected metadata context. Execute only through bounded preview and keep the result uncertified until promoted to a DQL draft.',
|
|
436
|
+
context,
|
|
437
|
+
}, null, 2),
|
|
438
|
+
},
|
|
439
|
+
projectRoot,
|
|
440
|
+
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
441
|
+
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
442
|
+
getSchemaContext: getSchemaContextForAgent,
|
|
443
|
+
}, (turn) => {
|
|
444
|
+
if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
|
|
445
|
+
governedAnswer = turn.output;
|
|
446
|
+
}
|
|
447
|
+
if (turn.kind === 'error')
|
|
448
|
+
providerError = turn.message;
|
|
449
|
+
}, controller.signal);
|
|
450
|
+
}
|
|
451
|
+
catch (error) {
|
|
452
|
+
providerError = error instanceof Error ? error.message : String(error);
|
|
453
|
+
}
|
|
454
|
+
if (!governedAnswer) {
|
|
455
|
+
generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
|
|
456
|
+
}
|
|
457
|
+
else {
|
|
458
|
+
generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
|
|
459
|
+
if (!generatedSql) {
|
|
460
|
+
generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
const sqlForPreview = reviewedSql || generatedSql;
|
|
466
|
+
const warnings = [
|
|
467
|
+
...(contextPack.warnings ?? []),
|
|
468
|
+
...(governedAnswer?.validationWarnings ?? []),
|
|
469
|
+
...generationWarnings,
|
|
470
|
+
].filter(Boolean);
|
|
471
|
+
let resultPreview;
|
|
472
|
+
let previewError;
|
|
473
|
+
if (governedAnswer?.result?.rows && !reviewedSql) {
|
|
474
|
+
resultPreview = normalizeNotebookAgentResult(governedAnswer.result);
|
|
475
|
+
}
|
|
476
|
+
else if (sqlForPreview) {
|
|
477
|
+
try {
|
|
478
|
+
const previewSql = buildAgentPreviewSql(sqlForPreview);
|
|
479
|
+
const previewStart = Date.now();
|
|
480
|
+
resultPreview = await executeLocalSqlForStoredResult(previewSql);
|
|
481
|
+
recordNotebookQueryRun(projectRoot, {
|
|
482
|
+
notebookPath: run.notebookPath,
|
|
483
|
+
cellId: run.sourceCellId,
|
|
484
|
+
cellName: run.sourceCellName,
|
|
485
|
+
researchRunId: run.id,
|
|
486
|
+
source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
|
|
487
|
+
status: 'success',
|
|
488
|
+
rowCount: resultPreview.rowCount ?? resultPreview.rows.length,
|
|
489
|
+
durationMs: Date.now() - previewStart,
|
|
490
|
+
sql: sqlForPreview,
|
|
491
|
+
contextPackId: contextPack.id,
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
catch (error) {
|
|
495
|
+
previewError = error instanceof Error ? error.message : String(error);
|
|
496
|
+
recordNotebookQueryRun(projectRoot, {
|
|
497
|
+
notebookPath: run.notebookPath,
|
|
498
|
+
cellId: run.sourceCellId,
|
|
499
|
+
cellName: run.sourceCellName,
|
|
500
|
+
researchRunId: run.id,
|
|
501
|
+
source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
|
|
502
|
+
status: 'error',
|
|
503
|
+
errorCode: previewError,
|
|
504
|
+
sql: sqlForPreview,
|
|
505
|
+
contextPackId: contextPack.id,
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
const routeDecision = notebookResearchRouteDecisionForRun(run, contextPack.routeDecision, sqlForPreview);
|
|
510
|
+
const display = resultPreview
|
|
511
|
+
? recommendVisualization(projectRoot, {
|
|
512
|
+
prompt: question,
|
|
513
|
+
resultSchema: resultPreview.columns,
|
|
514
|
+
rowSample: resultPreview.rows.slice(0, 25),
|
|
515
|
+
defaultVisualization: governedAnswer?.suggestedViz,
|
|
516
|
+
})
|
|
517
|
+
: undefined;
|
|
518
|
+
const evidence = {
|
|
519
|
+
trustStatus: {
|
|
520
|
+
label: governedAnswer?.trustLabel
|
|
521
|
+
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
|
|
522
|
+
reviewRequired: true,
|
|
523
|
+
},
|
|
524
|
+
contextPackId: contextPack.id,
|
|
525
|
+
routeDecision,
|
|
526
|
+
selectedEvidence: contextPack.evidenceRoles?.slice(0, 24) ?? [],
|
|
527
|
+
evidenceRoles: contextPack.evidenceRoles?.slice(0, 24) ?? [],
|
|
528
|
+
evidenceSummaries: contextPack.evidenceSummaries?.slice(0, 16) ?? [],
|
|
529
|
+
allowedSqlContext: {
|
|
530
|
+
relations: (contextPack.allowedSqlContext?.relations ?? []).slice(0, 24).map((relation) => ({
|
|
531
|
+
relation: relation.relation,
|
|
532
|
+
name: relation.name,
|
|
533
|
+
source: relation.source,
|
|
534
|
+
columns: relation.columns.slice(0, 32).map((column) => {
|
|
535
|
+
if (typeof column === 'string')
|
|
536
|
+
return column;
|
|
537
|
+
if (column && typeof column === 'object' && typeof column.name === 'string') {
|
|
538
|
+
return String(column.name);
|
|
539
|
+
}
|
|
540
|
+
return String(column);
|
|
541
|
+
}),
|
|
542
|
+
})),
|
|
543
|
+
sourceBlockSql: contextPack.allowedSqlContext?.sourceBlockSql?.slice(0, 12) ?? [],
|
|
544
|
+
},
|
|
545
|
+
missingContext: contextPack.missingContext?.slice(0, 16) ?? [],
|
|
546
|
+
warnings: contextPack.warnings?.slice(0, 16) ?? [],
|
|
547
|
+
retrievalDiagnostics: contextPack.retrievalDiagnostics,
|
|
548
|
+
agentEvidence: governedAnswer?.evidence,
|
|
549
|
+
analysisPlan: governedAnswer?.analysisPlan,
|
|
550
|
+
citations: [
|
|
551
|
+
...(contextPack.citations ?? []),
|
|
552
|
+
...(governedAnswer?.citations ?? []),
|
|
553
|
+
].slice(0, 40),
|
|
554
|
+
};
|
|
555
|
+
const summary = notebookResearchString(governedAnswer?.answer)
|
|
556
|
+
?? notebookResearchString(governedAnswer?.text)
|
|
557
|
+
?? notebookResearchSummary(question, resultPreview, previewError);
|
|
558
|
+
const recommendation = previewError
|
|
559
|
+
? 'Review the SQL, selected metadata, and connection context before rerunning.'
|
|
560
|
+
: sqlForPreview
|
|
561
|
+
? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
|
|
562
|
+
: 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
|
|
563
|
+
const researchPlan = buildNotebookResearchPlan({
|
|
564
|
+
run,
|
|
565
|
+
evidence,
|
|
566
|
+
resultPreview,
|
|
567
|
+
previewError,
|
|
568
|
+
generatedSql,
|
|
569
|
+
reviewedSql,
|
|
570
|
+
routeDecision,
|
|
571
|
+
});
|
|
572
|
+
return storage.updateRun(run.id, {
|
|
573
|
+
domain,
|
|
574
|
+
owner,
|
|
575
|
+
sourceCellFingerprint,
|
|
576
|
+
question,
|
|
577
|
+
intent,
|
|
578
|
+
context,
|
|
579
|
+
status: previewError ? 'error' : 'ready',
|
|
580
|
+
summary,
|
|
581
|
+
recommendation,
|
|
582
|
+
resultPreview,
|
|
583
|
+
evidence,
|
|
584
|
+
researchPlan,
|
|
585
|
+
generatedSql,
|
|
586
|
+
reviewedSql,
|
|
587
|
+
display: display && display.ok ? display.display : undefined,
|
|
588
|
+
contextPackId: contextPack.id,
|
|
589
|
+
routeDecision,
|
|
590
|
+
warnings: [
|
|
591
|
+
...warnings,
|
|
592
|
+
...(display && !display.ok ? [display.error] : []),
|
|
593
|
+
...(display && display.ok ? display.warnings : []),
|
|
594
|
+
],
|
|
595
|
+
reviewStatus: 'needs_review',
|
|
596
|
+
error: previewError,
|
|
597
|
+
lastRunAt: startedAt,
|
|
598
|
+
}) ?? run;
|
|
599
|
+
}
|
|
600
|
+
catch (error) {
|
|
601
|
+
return storage.updateRun(run.id, {
|
|
602
|
+
domain,
|
|
603
|
+
owner,
|
|
604
|
+
sourceCellFingerprint,
|
|
605
|
+
status: 'error',
|
|
606
|
+
error: error instanceof Error ? error.message : String(error),
|
|
607
|
+
reviewStatus: 'needs_review',
|
|
608
|
+
lastRunAt: startedAt,
|
|
609
|
+
}) ?? run;
|
|
610
|
+
}
|
|
611
|
+
};
|
|
612
|
+
const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
|
|
613
|
+
const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
|
|
614
|
+
if (!sql)
|
|
615
|
+
throw new Error('Notebook research needs generated or reviewed SQL before DQL draft promotion.');
|
|
616
|
+
const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
|
|
617
|
+
const session = await createDqlGenerationSessionForProject(projectRoot, {
|
|
618
|
+
inputMode: 'upload',
|
|
619
|
+
sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
|
|
620
|
+
sourceKind: 'raw-sql-file',
|
|
621
|
+
domain: notebookResearchString(input.domain) ?? run.domain,
|
|
622
|
+
owner: notebookResearchString(input.owner),
|
|
623
|
+
tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
|
|
624
|
+
provider: input.provider,
|
|
625
|
+
}, semanticLayer);
|
|
626
|
+
const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
|
|
627
|
+
?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
|
|
628
|
+
const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
|
|
629
|
+
const updated = storage.markPromoted(run.id, {
|
|
630
|
+
draftBlockPath: draftPath,
|
|
631
|
+
dqlImportId: session.id,
|
|
632
|
+
dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
|
|
633
|
+
dqlPromotion: promotion,
|
|
634
|
+
}) ?? run;
|
|
635
|
+
const planned = storage.updateRun(updated.id, {
|
|
636
|
+
researchPlan: buildNotebookResearchPlan({
|
|
637
|
+
run: updated,
|
|
638
|
+
evidence: updated.evidence,
|
|
639
|
+
resultPreview: updated.resultPreview,
|
|
640
|
+
generatedSql: updated.generatedSql,
|
|
641
|
+
reviewedSql: updated.reviewedSql,
|
|
642
|
+
routeDecision: updated.routeDecision,
|
|
643
|
+
}),
|
|
644
|
+
}) ?? updated;
|
|
645
|
+
return { run: planned, session };
|
|
646
|
+
};
|
|
647
|
+
const checkNotebookResearchReuse = async (storage, run, input = {}) => {
|
|
648
|
+
const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
|
|
649
|
+
if (!sql)
|
|
650
|
+
throw new Error('Notebook research needs generated or reviewed SQL before reuse checking.');
|
|
651
|
+
const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}.sql`;
|
|
652
|
+
const match = await matchSqlForDqlReuse({
|
|
653
|
+
sql,
|
|
654
|
+
sourcePath,
|
|
655
|
+
name: run.title || 'Notebook research SQL',
|
|
656
|
+
domain: notebookResearchString(input.domain) ?? run.domain ?? 'uncategorized',
|
|
657
|
+
owner: notebookResearchString(input.owner) ?? 'analytics',
|
|
658
|
+
});
|
|
659
|
+
const promotion = {
|
|
660
|
+
importId: `reuse-check:${run.id}:${Date.now()}`,
|
|
661
|
+
candidateIds: ['reuse_check'],
|
|
662
|
+
recommendedAction: match.recommendedAction,
|
|
663
|
+
similarityMatches: match.similarityMatches.slice(0, 8).map(toNotebookPromotionMatch),
|
|
664
|
+
candidates: [{
|
|
665
|
+
id: 'reuse_check',
|
|
666
|
+
name: run.title || 'Notebook research SQL',
|
|
667
|
+
domain: notebookResearchString(input.domain) ?? run.domain,
|
|
668
|
+
reviewStatus: 'reuse_checked',
|
|
669
|
+
recommendedAction: match.recommendedAction,
|
|
670
|
+
similarityMatches: match.similarityMatches.slice(0, 5).map(toNotebookPromotionMatch),
|
|
671
|
+
parameterPolicy: match.parameterPolicy.slice(0, 16).map((entry) => ({
|
|
672
|
+
name: entry.name,
|
|
673
|
+
policy: entry.policy,
|
|
674
|
+
})),
|
|
675
|
+
allowedFilters: match.allowedFilters.slice(0, 16),
|
|
676
|
+
warnings: match.parameterDecisions
|
|
677
|
+
.filter((decision) => decision.reason)
|
|
678
|
+
.map((decision) => `${decision.name}: ${decision.reason}`)
|
|
679
|
+
.slice(0, 12),
|
|
680
|
+
}],
|
|
681
|
+
createdAt: new Date().toISOString(),
|
|
682
|
+
};
|
|
683
|
+
const checked = storage.updateRun(run.id, {
|
|
684
|
+
dqlPromotionAction: match.recommendedAction,
|
|
685
|
+
dqlPromotion: promotion,
|
|
686
|
+
researchPlan: buildNotebookResearchPlan({
|
|
687
|
+
run: {
|
|
688
|
+
...run,
|
|
689
|
+
dqlPromotionAction: match.recommendedAction,
|
|
690
|
+
dqlPromotion: promotion,
|
|
691
|
+
},
|
|
692
|
+
evidence: run.evidence,
|
|
693
|
+
resultPreview: run.resultPreview,
|
|
694
|
+
generatedSql: run.generatedSql,
|
|
695
|
+
reviewedSql: run.reviewedSql,
|
|
696
|
+
routeDecision: run.routeDecision,
|
|
697
|
+
}),
|
|
698
|
+
}) ?? run;
|
|
699
|
+
return { run: checked, promotion, match };
|
|
700
|
+
};
|
|
701
|
+
const matchSqlForDqlReuse = async (input) => {
|
|
702
|
+
const sql = input.sql.trim();
|
|
703
|
+
if (!sql)
|
|
704
|
+
throw new Error('Missing SQL for reuse check.');
|
|
705
|
+
const sourceTables = extractSqlTablesLight(sql);
|
|
706
|
+
const candidate = {
|
|
707
|
+
id: 'match_sql',
|
|
708
|
+
sourceKind: 'raw-sql-file',
|
|
709
|
+
sourcePath: input.sourcePath ?? 'pasted.sql',
|
|
710
|
+
name: input.name ?? 'SQL match preview',
|
|
711
|
+
domain: input.domain ?? 'imported',
|
|
712
|
+
description: 'SQL match preview candidate.',
|
|
713
|
+
owner: input.owner ?? 'analytics',
|
|
714
|
+
tags: [],
|
|
715
|
+
sql,
|
|
716
|
+
dqlSource: '',
|
|
717
|
+
validation: null,
|
|
718
|
+
preview: null,
|
|
719
|
+
lineage: {
|
|
720
|
+
sourceTables,
|
|
721
|
+
parameters: [],
|
|
722
|
+
warnings: [],
|
|
723
|
+
statementIndex: 1,
|
|
724
|
+
totalStatements: 1,
|
|
725
|
+
},
|
|
726
|
+
confidence: 0.8,
|
|
727
|
+
splitStrategy: 'manual',
|
|
728
|
+
warnings: [],
|
|
729
|
+
conversionNotes: [],
|
|
730
|
+
aiAssistance: [],
|
|
731
|
+
reviewStatus: 'draft',
|
|
732
|
+
};
|
|
733
|
+
const evidence = deterministicDqlGenerationEvidence(candidate);
|
|
734
|
+
const patch = deterministicDqlGenerationPatch(candidate, evidence);
|
|
735
|
+
const contextPack = await buildDqlGenerationContextPack(projectRoot, { ...candidate, sql: patch.sql ?? sql }).catch(() => null);
|
|
736
|
+
const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
|
|
737
|
+
return {
|
|
738
|
+
parameterDecisions: patch.parameterDecisions ?? [],
|
|
739
|
+
parameterPolicy: patch.parameterPolicy ?? [],
|
|
740
|
+
filterBindings: patch.filterBindings ?? [],
|
|
741
|
+
allowedFilters: patch.allowedFilters ?? [],
|
|
742
|
+
parameterizedSql: patch.sql ?? sql,
|
|
743
|
+
similarityMatches: similarity.matches,
|
|
744
|
+
recommendedAction: similarity.recommendedAction,
|
|
745
|
+
};
|
|
746
|
+
};
|
|
747
|
+
const buildNotebookDqlPromotionSummary = (session, draftBlockPath) => {
|
|
748
|
+
const primary = draftBlockPath
|
|
749
|
+
? session.candidates.find((candidate) => candidate.draftSave?.path === draftBlockPath || candidate.savedPath === draftBlockPath) ?? session.candidates[0]
|
|
750
|
+
: session.candidates[0];
|
|
751
|
+
return {
|
|
752
|
+
importId: session.id,
|
|
753
|
+
candidateIds: session.candidates.map((candidate) => candidate.id),
|
|
754
|
+
draftBlockPath,
|
|
755
|
+
recommendedAction: primary?.recommendedAction,
|
|
756
|
+
similarityMatches: (primary?.similarityMatches ?? []).slice(0, 8).map(toNotebookPromotionMatch),
|
|
757
|
+
candidates: session.candidates.map((candidate) => ({
|
|
758
|
+
id: candidate.id,
|
|
759
|
+
name: candidate.name,
|
|
760
|
+
domain: candidate.domain,
|
|
761
|
+
draftPath: candidate.draftSave?.path,
|
|
762
|
+
savedPath: candidate.savedPath,
|
|
763
|
+
reviewStatus: candidate.reviewStatus,
|
|
764
|
+
recommendedAction: candidate.recommendedAction,
|
|
765
|
+
similarityMatches: (candidate.similarityMatches ?? []).slice(0, 5).map(toNotebookPromotionMatch),
|
|
766
|
+
parameterPolicy: (candidate.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
|
|
767
|
+
name: entry.name,
|
|
768
|
+
policy: entry.policy,
|
|
769
|
+
})),
|
|
770
|
+
allowedFilters: (candidate.allowedFilters ?? []).slice(0, 16),
|
|
771
|
+
warnings: [...(candidate.warnings ?? []), ...(candidate.lineage.warnings ?? [])].slice(0, 12),
|
|
772
|
+
})),
|
|
773
|
+
createdAt: new Date().toISOString(),
|
|
774
|
+
};
|
|
775
|
+
};
|
|
776
|
+
const toNotebookPromotionMatch = (match) => ({
|
|
777
|
+
kind: match.kind,
|
|
778
|
+
objectKey: match.objectKey,
|
|
779
|
+
name: match.name,
|
|
780
|
+
status: match.status,
|
|
781
|
+
source: match.source,
|
|
782
|
+
score: match.score,
|
|
783
|
+
reason: match.reason,
|
|
784
|
+
recommendedAction: match.recommendedAction,
|
|
785
|
+
});
|
|
333
786
|
// SSE clients for /api/watch hot-reload
|
|
334
787
|
const sseClients = new Set();
|
|
335
788
|
// Watch notebooks/, workbooks/, semantic-layer/, and data/ dirs for changes
|
|
@@ -398,8 +851,28 @@ export async function startLocalServer(opts) {
|
|
|
398
851
|
if (validated.reviewStatus === 'rejected') {
|
|
399
852
|
errors.unshift('Candidate was rejected.');
|
|
400
853
|
}
|
|
854
|
+
const duplicateBlocker = duplicateCertificationBlocker(validated);
|
|
855
|
+
if (duplicateBlocker)
|
|
856
|
+
errors.unshift(duplicateBlocker);
|
|
401
857
|
return { candidate: validated, errors };
|
|
402
858
|
};
|
|
859
|
+
const duplicateCertificationBlocker = (candidate) => {
|
|
860
|
+
const match = candidate.similarityMatches?.[0];
|
|
861
|
+
if (!match)
|
|
862
|
+
return null;
|
|
863
|
+
const duplicateKind = match.kind === 'exact_sql_match'
|
|
864
|
+
|| match.kind === 'parameterized_duplicate'
|
|
865
|
+
|| (match.kind === 'business_duplicate' && match.status === 'certified' && match.score >= 0.76);
|
|
866
|
+
if (!duplicateKind)
|
|
867
|
+
return null;
|
|
868
|
+
const replacementFor = new Set((candidate.replacementFor ?? []).map((value) => value.toLowerCase()));
|
|
869
|
+
const documentedReplacement = candidate.recommendedAction === 'create_replacement'
|
|
870
|
+
|| replacementFor.has(match.name.toLowerCase())
|
|
871
|
+
|| (match.objectKey && replacementFor.has(match.objectKey.toLowerCase()));
|
|
872
|
+
if (documentedReplacement)
|
|
873
|
+
return null;
|
|
874
|
+
return `Likely duplicate of ${match.name} (${match.kind}, ${Math.round(match.score * 100)}% match). Reuse the existing block, extend it, or document this draft as a replacement before certification.`;
|
|
875
|
+
};
|
|
403
876
|
const runBlockStudioPreviewSource = async (source, targetConnection) => {
|
|
404
877
|
const activeConnection = requireActiveConnection(targetConnection);
|
|
405
878
|
let tableMapping;
|
|
@@ -504,7 +977,50 @@ export async function startLocalServer(opts) {
|
|
|
504
977
|
}
|
|
505
978
|
return { passed, failed, skipped: 0, duration: Date.now() - start, assertions, runAt: new Date() };
|
|
506
979
|
};
|
|
507
|
-
const
|
|
980
|
+
const saveDqlGenerationDraft = (importId, candidate) => saveDqlGenerationDraftForProject(projectRoot, importId, candidate);
|
|
981
|
+
const createDqlGenerationSessionFromBody = async (body) => {
|
|
982
|
+
return createDqlGenerationSessionForProject(projectRoot, {
|
|
983
|
+
inputPath: typeof body.path === 'string' ? body.path : '',
|
|
984
|
+
inputMode: body.inputMode === 'paste' || body.inputMode === 'upload' || body.inputMode === 'path' ? body.inputMode : undefined,
|
|
985
|
+
sources: Array.isArray(body.sources)
|
|
986
|
+
? body.sources.map((source, index) => ({
|
|
987
|
+
path: typeof source?.path === 'string' ? source.path : `source-${index + 1}.sql`,
|
|
988
|
+
content: typeof source?.content === 'string' ? source.content : '',
|
|
989
|
+
}))
|
|
990
|
+
: undefined,
|
|
991
|
+
sourceKind: typeof body.sourceKind === 'string' ? body.sourceKind : 'raw-sql',
|
|
992
|
+
domain: typeof body.domain === 'string' ? body.domain : undefined,
|
|
993
|
+
owner: typeof body.owner === 'string' ? body.owner : undefined,
|
|
994
|
+
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
995
|
+
provider: typeof body.provider === 'string' ? body.provider : undefined,
|
|
996
|
+
}, semanticLayer);
|
|
997
|
+
};
|
|
998
|
+
const loadDqlGenerationSession = (importId) => {
|
|
999
|
+
const session = loadBlockStudioImportSession(projectRoot, importId);
|
|
1000
|
+
const candidates = session.candidates.map((candidate) => ({
|
|
1001
|
+
...candidate,
|
|
1002
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
1003
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
1004
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
1005
|
+
evidence: candidate.evidence ?? deterministicDqlGenerationEvidence(candidate),
|
|
1006
|
+
draftSave: candidate.draftSave ?? (isDraftBlockPath(candidate.savedPath)
|
|
1007
|
+
? { status: 'saved', path: candidate.savedPath }
|
|
1008
|
+
: { status: 'pending' }),
|
|
1009
|
+
}));
|
|
1010
|
+
return {
|
|
1011
|
+
...session,
|
|
1012
|
+
mode: 'ai-import',
|
|
1013
|
+
candidates,
|
|
1014
|
+
generation: {
|
|
1015
|
+
provider: candidates.find((candidate) => candidate.generationProvider)?.generationProvider ?? 'local-deterministic',
|
|
1016
|
+
aiEnabled: candidates.some((candidate) => candidate.generationMode === 'ai'),
|
|
1017
|
+
contextObjectCount: candidates.reduce((sum, candidate) => sum + (candidate.evidence?.length ?? 0), 0),
|
|
1018
|
+
createdDrafts: candidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
|
|
1019
|
+
warnings: candidates.flatMap((candidate) => candidate.draftSave.status === 'error' && candidate.draftSave.error ? [candidate.draftSave.error] : []),
|
|
1020
|
+
},
|
|
1021
|
+
};
|
|
1022
|
+
};
|
|
1023
|
+
const certifyBlockStudioSource = async (source, blockPath, options = {}) => {
|
|
508
1024
|
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
509
1025
|
let preview = null;
|
|
510
1026
|
let testResults = null;
|
|
@@ -542,12 +1058,24 @@ export async function startLocalServer(opts) {
|
|
|
542
1058
|
description: parsed.description,
|
|
543
1059
|
owner: parsed.owner,
|
|
544
1060
|
tags: parsed.tags,
|
|
1061
|
+
llmContext: parsed.llmContext,
|
|
1062
|
+
pattern: parsed.pattern,
|
|
1063
|
+
grain: parsed.grain,
|
|
1064
|
+
entities: parsed.entities,
|
|
1065
|
+
declaredOutputs: parsed.outputs,
|
|
1066
|
+
dimensions: parsed.dimensions,
|
|
1067
|
+
allowedFilters: parsed.allowedFilters,
|
|
1068
|
+
parameterPolicy: parsed.parameterPolicy,
|
|
1069
|
+
filterBindings: parsed.filterBindings,
|
|
1070
|
+
sourceSystems: parsed.sourceSystems,
|
|
1071
|
+
replacementFor: parsed.replacementFor,
|
|
1072
|
+
reviewCadence: parsed.reviewCadence,
|
|
545
1073
|
dependencies: [],
|
|
546
1074
|
usedInCount: 0,
|
|
547
1075
|
createdAt: new Date(),
|
|
548
1076
|
updatedAt: new Date(),
|
|
549
1077
|
};
|
|
550
|
-
const certification = new Certifier().evaluate(record, testResults ?? undefined);
|
|
1078
|
+
const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined);
|
|
551
1079
|
const checklist = buildBlockStudioCertificationChecklist({
|
|
552
1080
|
source,
|
|
553
1081
|
validation,
|
|
@@ -730,6 +1258,7 @@ export async function startLocalServer(opts) {
|
|
|
730
1258
|
const variables = body.variables && typeof body.variables === 'object'
|
|
731
1259
|
? body.variables
|
|
732
1260
|
: {};
|
|
1261
|
+
const dashboardVariables = dashboardRuntimeVariables(loaded.dashboard, variables);
|
|
733
1262
|
const tiles = [];
|
|
734
1263
|
let localApps = null;
|
|
735
1264
|
for (const item of loaded.dashboard.layout.items) {
|
|
@@ -831,8 +1360,15 @@ export async function startLocalServer(opts) {
|
|
|
831
1360
|
});
|
|
832
1361
|
continue;
|
|
833
1362
|
}
|
|
834
|
-
const
|
|
835
|
-
|
|
1363
|
+
const filterApplication = applyDashboardFiltersToBlockExecution({
|
|
1364
|
+
sql: semanticCompose?.sql ?? plan.sql,
|
|
1365
|
+
sqlParams: plan?.sqlParams ?? [],
|
|
1366
|
+
variables: { ...(plan?.variables ?? {}), ...dashboardVariables },
|
|
1367
|
+
block,
|
|
1368
|
+
dashboard: loaded.dashboard,
|
|
1369
|
+
});
|
|
1370
|
+
const prepared = prepareLocalExecution(filterApplication.sql, targetConnection, projectRoot, projectConfig);
|
|
1371
|
+
const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables(filterApplication.variables), prepared.connection);
|
|
836
1372
|
tiles.push({
|
|
837
1373
|
tileId: item.i,
|
|
838
1374
|
status: 'ok',
|
|
@@ -843,6 +1379,10 @@ export async function startLocalServer(opts) {
|
|
|
843
1379
|
viz: item.viz,
|
|
844
1380
|
chartConfig: mergeDashboardChartConfig(plan?.chartConfig, item),
|
|
845
1381
|
result: normalizeQueryResult(result),
|
|
1382
|
+
filters: {
|
|
1383
|
+
applied: filterApplication.appliedFilters,
|
|
1384
|
+
skipped: filterApplication.skippedFilters,
|
|
1385
|
+
},
|
|
846
1386
|
citation: {
|
|
847
1387
|
kind: 'block',
|
|
848
1388
|
name: block.name,
|
|
@@ -885,11 +1425,11 @@ export async function startLocalServer(opts) {
|
|
|
885
1425
|
return;
|
|
886
1426
|
}
|
|
887
1427
|
// Apps, dashboards, persona — see apps-api.ts. Returns true if handled.
|
|
888
|
-
if (path.startsWith('/api/apps') || path === '/api/persona') {
|
|
1428
|
+
if (path.startsWith('/api/apps') || path.startsWith('/api/visualizations') || path === '/api/persona') {
|
|
889
1429
|
try {
|
|
890
1430
|
const handled = await handleAppsApi({
|
|
891
1431
|
req,
|
|
892
|
-
res,
|
|
1432
|
+
res: res,
|
|
893
1433
|
url,
|
|
894
1434
|
path,
|
|
895
1435
|
projectRoot,
|
|
@@ -1019,6 +1559,394 @@ export async function startLocalServer(opts) {
|
|
|
1019
1559
|
}
|
|
1020
1560
|
return;
|
|
1021
1561
|
}
|
|
1562
|
+
if (req.method === 'GET' && path === '/api/notebook/research') {
|
|
1563
|
+
const storage = openNotebookResearchStorage();
|
|
1564
|
+
try {
|
|
1565
|
+
const notebookPath = notebookResearchString(url.searchParams.get('path'));
|
|
1566
|
+
const sourceCellId = notebookResearchString(url.searchParams.get('sourceCellId') ?? url.searchParams.get('cellId'));
|
|
1567
|
+
const domain = notebookResearchString(url.searchParams.get('domain'));
|
|
1568
|
+
const owner = notebookResearchString(url.searchParams.get('owner'));
|
|
1569
|
+
const intent = notebookResearchIntent(url.searchParams.get('intent'));
|
|
1570
|
+
const search = notebookResearchString(url.searchParams.get('q')) ?? notebookResearchString(url.searchParams.get('search'));
|
|
1571
|
+
const status = notebookResearchStatus(url.searchParams.get('status'));
|
|
1572
|
+
const reviewStatus = notebookResearchReviewStatus(url.searchParams.get('reviewStatus'));
|
|
1573
|
+
const promotionAction = notebookResearchPromotionAction(url.searchParams.get('promotionAction') ?? url.searchParams.get('action'));
|
|
1574
|
+
const readiness = notebookResearchReadiness(url.searchParams.get('readiness') ?? url.searchParams.get('ready'));
|
|
1575
|
+
const age = notebookResearchAge(url.searchParams.get('age'));
|
|
1576
|
+
const nextAction = notebookResearchNextAction(url.searchParams.get('nextAction') ?? url.searchParams.get('next'));
|
|
1577
|
+
const activeOnlyParam = url.searchParams.get('activeOnly') ?? url.searchParams.get('active');
|
|
1578
|
+
const activeOnly = activeOnlyParam === 'true' || activeOnlyParam === '1';
|
|
1579
|
+
const sort = notebookResearchSort(url.searchParams.get('sort'));
|
|
1580
|
+
const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
|
|
1581
|
+
const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
|
|
1582
|
+
const page = storage.listRunsPage({ notebookPath, sourceCellId, domain, owner, intent, search, status, reviewStatus, promotionAction, readiness, age, nextAction, activeOnly, sort, limit, offset });
|
|
1583
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1584
|
+
res.end(serializeJSON(withNotebookResearchChecklistPage(page)));
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1588
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1589
|
+
}
|
|
1590
|
+
finally {
|
|
1591
|
+
storage.close();
|
|
1592
|
+
}
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
if (req.method === 'POST' && path === '/api/notebook/research') {
|
|
1596
|
+
const storage = openNotebookResearchStorage();
|
|
1597
|
+
try {
|
|
1598
|
+
const body = await readJSON(req);
|
|
1599
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1600
|
+
const question = notebookResearchString(body.question);
|
|
1601
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1602
|
+
const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
|
|
1603
|
+
const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
|
|
1604
|
+
const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
|
|
1605
|
+
if (!notebookPath || !question) {
|
|
1606
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1607
|
+
res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
const created = storage.createRun({
|
|
1611
|
+
notebookPath,
|
|
1612
|
+
domain: notebookResearchString(body.domain),
|
|
1613
|
+
owner: notebookResearchString(body.owner),
|
|
1614
|
+
sourceCell,
|
|
1615
|
+
sourceCellId,
|
|
1616
|
+
sourceCellName,
|
|
1617
|
+
sourceCellFingerprint,
|
|
1618
|
+
title: notebookResearchString(body.title),
|
|
1619
|
+
question,
|
|
1620
|
+
intent: notebookResearchIntent(body.intent),
|
|
1621
|
+
context: body.context,
|
|
1622
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1623
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1624
|
+
});
|
|
1625
|
+
const run = body.run === true
|
|
1626
|
+
? await runNotebookResearch(storage, created, {
|
|
1627
|
+
domain: notebookResearchString(body.domain),
|
|
1628
|
+
sourceCellFingerprint,
|
|
1629
|
+
question,
|
|
1630
|
+
intent: notebookResearchIntent(body.intent),
|
|
1631
|
+
context: body.context,
|
|
1632
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1633
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1634
|
+
})
|
|
1635
|
+
: created;
|
|
1636
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1637
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
|
|
1638
|
+
}
|
|
1639
|
+
catch (error) {
|
|
1640
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1641
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1642
|
+
}
|
|
1643
|
+
finally {
|
|
1644
|
+
storage.close();
|
|
1645
|
+
}
|
|
1646
|
+
return;
|
|
1647
|
+
}
|
|
1648
|
+
if (req.method === 'POST' && path === '/api/notebook/research/context-preview') {
|
|
1649
|
+
try {
|
|
1650
|
+
const body = await readJSON(req);
|
|
1651
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1652
|
+
const question = notebookResearchString(body.question);
|
|
1653
|
+
if (!notebookPath || !question) {
|
|
1654
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1655
|
+
res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
|
|
1656
|
+
return;
|
|
1657
|
+
}
|
|
1658
|
+
const intent = notebookResearchIntent(body.intent) ?? 'ad_hoc_analysis';
|
|
1659
|
+
const domain = notebookResearchString(body.domain);
|
|
1660
|
+
const context = body.context;
|
|
1661
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1662
|
+
const contextPack = await buildLocalContextPack(projectRoot, {
|
|
1663
|
+
question,
|
|
1664
|
+
mode: 'question',
|
|
1665
|
+
surface: 'notebook',
|
|
1666
|
+
selectedContext: {
|
|
1667
|
+
activeSurface: 'notebook',
|
|
1668
|
+
notebookPath,
|
|
1669
|
+
domain,
|
|
1670
|
+
sourceCellId: notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell),
|
|
1671
|
+
sourceCellName: notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell),
|
|
1672
|
+
context,
|
|
1673
|
+
intent,
|
|
1674
|
+
researchPattern: notebookResearchIntentPattern(intent),
|
|
1675
|
+
},
|
|
1676
|
+
strictness: 'balanced',
|
|
1677
|
+
limit: 100,
|
|
1678
|
+
});
|
|
1679
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1680
|
+
res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
|
|
1681
|
+
}
|
|
1682
|
+
catch (error) {
|
|
1683
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1684
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1685
|
+
}
|
|
1686
|
+
return;
|
|
1687
|
+
}
|
|
1688
|
+
if (req.method === 'POST' && path === '/api/notebook/research/seed-cells') {
|
|
1689
|
+
const storage = openNotebookResearchStorage();
|
|
1690
|
+
try {
|
|
1691
|
+
const body = await readJSON(req);
|
|
1692
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1693
|
+
if (!notebookPath) {
|
|
1694
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1695
|
+
res.end(serializeJSON({ error: 'notebookPath is required.' }));
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
const cells = Array.isArray(body.cells) ? body.cells : [];
|
|
1699
|
+
if (cells.length === 0) {
|
|
1700
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1701
|
+
res.end(serializeJSON({ error: 'At least one source cell is required.' }));
|
|
1702
|
+
return;
|
|
1703
|
+
}
|
|
1704
|
+
const validCells = cells.filter((cell) => Boolean(cell && typeof cell === 'object' && !Array.isArray(cell)));
|
|
1705
|
+
const invalidCellCount = cells.length - validCells.length;
|
|
1706
|
+
const seeded = storage.seedRunsFromCells({
|
|
1707
|
+
notebookPath,
|
|
1708
|
+
domain: notebookResearchString(body.domain),
|
|
1709
|
+
owner: notebookResearchString(body.owner),
|
|
1710
|
+
notebookTitle: notebookResearchString(body.notebookTitle),
|
|
1711
|
+
cells: validCells.map((cell) => ({
|
|
1712
|
+
sourceCell: notebookResearchSourceCellPayload(cell),
|
|
1713
|
+
id: notebookResearchString(cell.id),
|
|
1714
|
+
sourceCellId: notebookResearchString(cell.sourceCellId),
|
|
1715
|
+
name: notebookResearchString(cell.name),
|
|
1716
|
+
sourceCellName: notebookResearchString(cell.sourceCellName),
|
|
1717
|
+
sourceCellFingerprint: notebookResearchString(cell.sourceCellFingerprint),
|
|
1718
|
+
title: notebookResearchString(cell.title),
|
|
1719
|
+
type: notebookResearchString(cell.type),
|
|
1720
|
+
sql: notebookResearchString(cell.sql),
|
|
1721
|
+
content: notebookResearchString(cell.content),
|
|
1722
|
+
source: notebookResearchString(cell.source),
|
|
1723
|
+
question: notebookResearchString(cell.question),
|
|
1724
|
+
intent: notebookResearchIntent(cell.intent),
|
|
1725
|
+
})),
|
|
1726
|
+
limit: 1000,
|
|
1727
|
+
});
|
|
1728
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1729
|
+
res.end(serializeJSON({
|
|
1730
|
+
created: seeded.created.map(withNotebookResearchChecklist),
|
|
1731
|
+
createdCount: seeded.createdCount,
|
|
1732
|
+
skippedCount: seeded.skippedCount + invalidCellCount,
|
|
1733
|
+
limitApplied: seeded.limitApplied,
|
|
1734
|
+
}));
|
|
1735
|
+
}
|
|
1736
|
+
catch (error) {
|
|
1737
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1738
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1739
|
+
}
|
|
1740
|
+
finally {
|
|
1741
|
+
storage.close();
|
|
1742
|
+
}
|
|
1743
|
+
return;
|
|
1744
|
+
}
|
|
1745
|
+
if (req.method === 'POST' && path === '/api/notebook/research/source-coverage') {
|
|
1746
|
+
const storage = openNotebookResearchStorage();
|
|
1747
|
+
try {
|
|
1748
|
+
const body = await readJSON(req);
|
|
1749
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1750
|
+
if (!notebookPath) {
|
|
1751
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1752
|
+
res.end(serializeJSON({ error: 'notebookPath is required.' }));
|
|
1753
|
+
return;
|
|
1754
|
+
}
|
|
1755
|
+
const rawSourceCells = Array.isArray(body.sourceCells)
|
|
1756
|
+
? body.sourceCells
|
|
1757
|
+
: Array.isArray(body.cells)
|
|
1758
|
+
? body.cells
|
|
1759
|
+
: [];
|
|
1760
|
+
const sourceCells = rawSourceCells.flatMap((cell) => {
|
|
1761
|
+
if (!cell || typeof cell !== 'object' || Array.isArray(cell))
|
|
1762
|
+
return [];
|
|
1763
|
+
const record = cell;
|
|
1764
|
+
const nested = notebookResearchSourceCellPayload(record) ?? {};
|
|
1765
|
+
return [{
|
|
1766
|
+
...nested,
|
|
1767
|
+
id: notebookResearchString(record.id) ?? nested.id,
|
|
1768
|
+
sourceCellId: notebookResearchString(record.sourceCellId) ?? nested.sourceCellId,
|
|
1769
|
+
cellId: notebookResearchString(record.cellId) ?? nested.cellId,
|
|
1770
|
+
name: notebookResearchString(record.name) ?? nested.name,
|
|
1771
|
+
sourceCellName: notebookResearchString(record.sourceCellName) ?? nested.sourceCellName,
|
|
1772
|
+
title: notebookResearchString(record.title) ?? nested.title,
|
|
1773
|
+
fingerprint: notebookResearchString(record.fingerprint) ?? nested.fingerprint,
|
|
1774
|
+
sourceCellFingerprint: notebookResearchString(record.sourceCellFingerprint) ?? nested.sourceCellFingerprint,
|
|
1775
|
+
sqlFingerprint: notebookResearchString(record.sqlFingerprint) ?? nested.sqlFingerprint,
|
|
1776
|
+
type: notebookResearchString(record.type) ?? nested.type,
|
|
1777
|
+
}];
|
|
1778
|
+
});
|
|
1779
|
+
const requestedIds = Array.isArray(body.sourceCellIds)
|
|
1780
|
+
? body.sourceCellIds
|
|
1781
|
+
: rawSourceCells.length > 0
|
|
1782
|
+
? sourceCells.map((cell) => notebookResearchSourceCellId(cell))
|
|
1783
|
+
: [];
|
|
1784
|
+
const sourceCellIds = Array.from(new Set(requestedIds
|
|
1785
|
+
.map((id) => notebookResearchString(id))
|
|
1786
|
+
.filter((id) => Boolean(id))));
|
|
1787
|
+
const requestedSourceCellCount = new Set([
|
|
1788
|
+
...sourceCellIds,
|
|
1789
|
+
...sourceCells
|
|
1790
|
+
.map((cell) => notebookResearchSourceCellId(cell))
|
|
1791
|
+
.filter((id) => Boolean(id)),
|
|
1792
|
+
]).size;
|
|
1793
|
+
const limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
|
|
1794
|
+
? Math.max(1, Math.min(10_000, Math.floor(body.limit)))
|
|
1795
|
+
: 10_000;
|
|
1796
|
+
const linkedRuns = storage.listLatestRunsBySourceCell({
|
|
1797
|
+
notebookPath,
|
|
1798
|
+
sourceCellIds,
|
|
1799
|
+
sourceCells,
|
|
1800
|
+
limit,
|
|
1801
|
+
});
|
|
1802
|
+
const missingRuns = storage.listLatestRunsForMissingSourceCells({
|
|
1803
|
+
notebookPath,
|
|
1804
|
+
sourceCellIds,
|
|
1805
|
+
sourceCells,
|
|
1806
|
+
limit,
|
|
1807
|
+
});
|
|
1808
|
+
const runs = [...linkedRuns, ...missingRuns];
|
|
1809
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1810
|
+
res.end(serializeJSON({
|
|
1811
|
+
runs: runs.map(withNotebookResearchChecklist),
|
|
1812
|
+
requestedCount: requestedSourceCellCount,
|
|
1813
|
+
matchedCount: runs.length,
|
|
1814
|
+
limitApplied: requestedSourceCellCount > limit,
|
|
1815
|
+
}));
|
|
1816
|
+
}
|
|
1817
|
+
catch (error) {
|
|
1818
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1819
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1820
|
+
}
|
|
1821
|
+
finally {
|
|
1822
|
+
storage.close();
|
|
1823
|
+
}
|
|
1824
|
+
return;
|
|
1825
|
+
}
|
|
1826
|
+
if (req.method === 'GET' && path === '/api/notebook/research/diagnostics') {
|
|
1827
|
+
const storage = openNotebookResearchStorage();
|
|
1828
|
+
try {
|
|
1829
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1830
|
+
res.end(serializeJSON(storage.getDiagnostics()));
|
|
1831
|
+
}
|
|
1832
|
+
catch (error) {
|
|
1833
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1834
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1835
|
+
}
|
|
1836
|
+
finally {
|
|
1837
|
+
storage.close();
|
|
1838
|
+
}
|
|
1839
|
+
return;
|
|
1840
|
+
}
|
|
1841
|
+
const notebookResearchMatch = /^\/api\/notebook\/research\/([^/]+)(?:\/([^/]+))?$/.exec(path);
|
|
1842
|
+
if (notebookResearchMatch) {
|
|
1843
|
+
const id = decodeURIComponent(notebookResearchMatch[1]);
|
|
1844
|
+
const action = notebookResearchMatch[2];
|
|
1845
|
+
const storage = openNotebookResearchStorage();
|
|
1846
|
+
try {
|
|
1847
|
+
const run = storage.getRun(id);
|
|
1848
|
+
if (!run) {
|
|
1849
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1850
|
+
res.end(serializeJSON({ error: 'Notebook research run not found.' }));
|
|
1851
|
+
return;
|
|
1852
|
+
}
|
|
1853
|
+
if (req.method === 'GET' && !action) {
|
|
1854
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1855
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
if (req.method === 'PATCH' && !action) {
|
|
1859
|
+
const body = await readJSON(req);
|
|
1860
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1861
|
+
const sourceCellIdPatch = notebookResearchPatchString(body, 'sourceCellId');
|
|
1862
|
+
const sourceCellNamePatch = notebookResearchPatchString(body, 'sourceCellName');
|
|
1863
|
+
const sourceCellFingerprintPatch = notebookResearchPatchString(body, 'sourceCellFingerprint');
|
|
1864
|
+
const updated = storage.updateRun(id, {
|
|
1865
|
+
domain: notebookResearchString(body.domain),
|
|
1866
|
+
owner: notebookResearchString(body.owner),
|
|
1867
|
+
sourceCellId: sourceCellIdPatch !== undefined ? sourceCellIdPatch : notebookResearchSourceCellId(sourceCell),
|
|
1868
|
+
sourceCellName: sourceCellNamePatch !== undefined ? sourceCellNamePatch : notebookResearchSourceCellName(sourceCell),
|
|
1869
|
+
sourceCellFingerprint: sourceCellFingerprintPatch !== undefined ? sourceCellFingerprintPatch : notebookResearchSourceCellFingerprint(sourceCell),
|
|
1870
|
+
title: notebookResearchString(body.title),
|
|
1871
|
+
question: notebookResearchString(body.question),
|
|
1872
|
+
intent: notebookResearchIntent(body.intent),
|
|
1873
|
+
context: body.context,
|
|
1874
|
+
recommendation: notebookResearchString(body.recommendation),
|
|
1875
|
+
evidence: body.evidence,
|
|
1876
|
+
contextPackId: notebookResearchString(body.contextPackId),
|
|
1877
|
+
routeDecision: body.routeDecision,
|
|
1878
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1879
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1880
|
+
warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
|
|
1881
|
+
reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
|
|
1882
|
+
dqlPromotionAction: notebookResearchPromotionAction(body.dqlPromotionAction),
|
|
1883
|
+
}) ?? run;
|
|
1884
|
+
const planned = storage.updateRun(updated.id, {
|
|
1885
|
+
researchPlan: buildNotebookResearchPlan({
|
|
1886
|
+
run: updated,
|
|
1887
|
+
evidence: updated.evidence,
|
|
1888
|
+
resultPreview: updated.resultPreview,
|
|
1889
|
+
previewError: updated.error,
|
|
1890
|
+
generatedSql: updated.generatedSql,
|
|
1891
|
+
reviewedSql: updated.reviewedSql,
|
|
1892
|
+
routeDecision: updated.routeDecision,
|
|
1893
|
+
}),
|
|
1894
|
+
}) ?? updated;
|
|
1895
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1896
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(planned) }));
|
|
1897
|
+
return;
|
|
1898
|
+
}
|
|
1899
|
+
if (req.method === 'POST' && action === 'run') {
|
|
1900
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
1901
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1902
|
+
const updated = await runNotebookResearch(storage, run, {
|
|
1903
|
+
domain: notebookResearchString(body.domain),
|
|
1904
|
+
owner: notebookResearchString(body.owner),
|
|
1905
|
+
sourceCellFingerprint: notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell),
|
|
1906
|
+
question: notebookResearchString(body.question),
|
|
1907
|
+
intent: notebookResearchIntent(body.intent),
|
|
1908
|
+
context: body.context,
|
|
1909
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1910
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1911
|
+
});
|
|
1912
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1913
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
|
|
1914
|
+
return;
|
|
1915
|
+
}
|
|
1916
|
+
if (req.method === 'POST' && action === 'reuse-check') {
|
|
1917
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
1918
|
+
const payload = await checkNotebookResearchReuse(storage, run, {
|
|
1919
|
+
domain: notebookResearchString(body.domain),
|
|
1920
|
+
owner: notebookResearchString(body.owner),
|
|
1921
|
+
});
|
|
1922
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1923
|
+
res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
|
|
1924
|
+
return;
|
|
1925
|
+
}
|
|
1926
|
+
if (req.method === 'POST' && action === 'promote-dql') {
|
|
1927
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
1928
|
+
const payload = await promoteNotebookResearchToDql(storage, run, {
|
|
1929
|
+
domain: notebookResearchString(body.domain),
|
|
1930
|
+
owner: notebookResearchString(body.owner),
|
|
1931
|
+
provider: notebookResearchString(body.provider),
|
|
1932
|
+
tags: Array.isArray(body.tags) ? body.tags.filter((item) => typeof item === 'string') : undefined,
|
|
1933
|
+
});
|
|
1934
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1935
|
+
res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
|
|
1936
|
+
return;
|
|
1937
|
+
}
|
|
1938
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1939
|
+
res.end(serializeJSON({ error: 'Unsupported notebook research operation.' }));
|
|
1940
|
+
}
|
|
1941
|
+
catch (error) {
|
|
1942
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1943
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1944
|
+
}
|
|
1945
|
+
finally {
|
|
1946
|
+
storage.close();
|
|
1947
|
+
}
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1022
1950
|
// ── run snapshots (v0.11) ───────────────────────────────────────────────
|
|
1023
1951
|
// Captures executed notebook state (query results + timings) in a
|
|
1024
1952
|
// sibling `.run.json` so notebooks can show last-run output without
|
|
@@ -1304,49 +2232,51 @@ export async function startLocalServer(opts) {
|
|
|
1304
2232
|
// ── Block library (list all blocks with metadata) ────────────────────
|
|
1305
2233
|
if (req.method === 'GET' && path === '/api/blocks/library') {
|
|
1306
2234
|
try {
|
|
1307
|
-
const blocksDir = join(projectRoot, 'blocks');
|
|
1308
2235
|
const blocks = [];
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
2236
|
+
const seen = new Set();
|
|
2237
|
+
const scanDir = (dir) => {
|
|
2238
|
+
if (!existsSync(dir))
|
|
2239
|
+
return;
|
|
2240
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
2241
|
+
const filePath = join(dir, entry.name);
|
|
2242
|
+
if (entry.isDirectory()) {
|
|
2243
|
+
scanDir(filePath);
|
|
2244
|
+
}
|
|
2245
|
+
else if (entry.name.endsWith('.dql') && !seen.has(filePath)) {
|
|
2246
|
+
seen.add(filePath);
|
|
2247
|
+
const relPath = relative(projectRoot, filePath).replaceAll('\\', '/');
|
|
2248
|
+
try {
|
|
2249
|
+
const source = readFileSync(filePath, 'utf-8');
|
|
2250
|
+
const stat = statSync(filePath);
|
|
2251
|
+
// Quick regex parse for key block fields
|
|
2252
|
+
const nameMatch = /block\s+"([^"]+)"/.exec(source);
|
|
2253
|
+
const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
|
|
2254
|
+
const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
|
|
2255
|
+
const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
|
|
2256
|
+
const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
|
|
2257
|
+
const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
|
|
2258
|
+
const parsedTags = tagsMatch
|
|
2259
|
+
? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
|
|
2260
|
+
: [];
|
|
2261
|
+
const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
|
|
2262
|
+
blocks.push({
|
|
2263
|
+
name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
|
|
2264
|
+
domain: (domainMatch?.[1] ?? inferBlockStudioPathDomain(relPath)) || 'uncategorized',
|
|
2265
|
+
status: statusMatch?.[1] ?? 'draft',
|
|
2266
|
+
owner: ownerMatch?.[1] ?? null,
|
|
2267
|
+
tags: parsedTags,
|
|
2268
|
+
path: relPath,
|
|
2269
|
+
lastModified: stat.mtime.toISOString(),
|
|
2270
|
+
description: descMatch?.[1] ?? '',
|
|
2271
|
+
llmContext: llmMatch?.[1] ?? null,
|
|
2272
|
+
});
|
|
1345
2273
|
}
|
|
2274
|
+
catch { /* skip unreadable files */ }
|
|
1346
2275
|
}
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
|
|
2276
|
+
}
|
|
2277
|
+
};
|
|
2278
|
+
scanDir(join(projectRoot, 'blocks'));
|
|
2279
|
+
scanDir(join(projectRoot, 'domains'));
|
|
1350
2280
|
blocks.sort((a, b) => new Date(b.lastModified).getTime() - new Date(a.lastModified).getTime());
|
|
1351
2281
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1352
2282
|
res.end(serializeJSON({ blocks }));
|
|
@@ -1555,20 +2485,216 @@ export async function startLocalServer(opts) {
|
|
|
1555
2485
|
res.end(serializeJSON({ error: 'source is required' }));
|
|
1556
2486
|
return;
|
|
1557
2487
|
}
|
|
1558
|
-
const result = await certifyBlockStudioSource(source, blockPath);
|
|
2488
|
+
const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise !== false });
|
|
1559
2489
|
const blockers = Array.from(new Set(result.checklist.blockers));
|
|
1560
2490
|
if (!result.certification.certified || blockers.length > 0) {
|
|
1561
2491
|
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1562
2492
|
res.end(serializeJSON({ ok: false, ...result, blockers }));
|
|
1563
2493
|
return;
|
|
1564
2494
|
}
|
|
1565
|
-
|
|
1566
|
-
|
|
2495
|
+
let certifiedPayload = null;
|
|
2496
|
+
const certifiedSource = setBlockStudioStatusInSource(source, 'certified');
|
|
2497
|
+
if (blockPath) {
|
|
2498
|
+
const normalizedBlockPath = normalize(blockPath).replace(/^\/+/, '');
|
|
2499
|
+
const parsed = parseBlockSourceMetadata(certifiedSource);
|
|
2500
|
+
if (isDraftBlockPath(normalizedBlockPath)) {
|
|
2501
|
+
const savedPath = saveBlockStudioArtifacts(projectRoot, {
|
|
2502
|
+
currentPath: normalizedBlockPath,
|
|
2503
|
+
source: certifiedSource,
|
|
2504
|
+
name: parsed.name,
|
|
2505
|
+
domain: parsed.domain,
|
|
2506
|
+
description: parsed.description,
|
|
2507
|
+
owner: parsed.owner,
|
|
2508
|
+
tags: parsed.tags,
|
|
2509
|
+
});
|
|
2510
|
+
certifiedPayload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
2511
|
+
}
|
|
2512
|
+
else {
|
|
2513
|
+
setBlockStudioStatus(projectRoot, normalizedBlockPath, 'certified');
|
|
2514
|
+
certifiedPayload = openBlockStudioDocument(projectRoot, normalizedBlockPath, semanticLayer);
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
1567
2517
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
1568
2518
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1569
|
-
res.end(serializeJSON({
|
|
2519
|
+
res.end(serializeJSON({
|
|
2520
|
+
ok: true,
|
|
2521
|
+
status: 'certified',
|
|
2522
|
+
...result,
|
|
2523
|
+
path: certifiedPayload?.path ?? blockPath ?? null,
|
|
2524
|
+
source: certifiedPayload?.source ?? certifiedSource,
|
|
2525
|
+
metadata: certifiedPayload?.metadata,
|
|
2526
|
+
companionPath: certifiedPayload?.companionPath ?? null,
|
|
2527
|
+
validation: certifiedPayload?.validation ?? result.validation,
|
|
2528
|
+
}));
|
|
2529
|
+
}
|
|
2530
|
+
catch (error) {
|
|
2531
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2532
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2533
|
+
}
|
|
2534
|
+
return;
|
|
2535
|
+
}
|
|
2536
|
+
if (req.method === 'POST' && path === '/api/block-studio/ai-imports') {
|
|
2537
|
+
try {
|
|
2538
|
+
const body = await readJSON(req);
|
|
2539
|
+
const session = await createDqlGenerationSessionFromBody(body);
|
|
2540
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2541
|
+
res.end(serializeJSON(session));
|
|
2542
|
+
}
|
|
2543
|
+
catch (error) {
|
|
2544
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2545
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2546
|
+
}
|
|
2547
|
+
return;
|
|
2548
|
+
}
|
|
2549
|
+
if (req.method === 'POST' && path === '/api/block-studio/match-sql') {
|
|
2550
|
+
try {
|
|
2551
|
+
const body = await readJSON(req);
|
|
2552
|
+
if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
|
|
2553
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2554
|
+
res.end(serializeJSON({ error: 'Missing SQL in request body.' }));
|
|
2555
|
+
return;
|
|
2556
|
+
}
|
|
2557
|
+
const sql = body.sql.trim();
|
|
2558
|
+
const match = await matchSqlForDqlReuse({
|
|
2559
|
+
sql,
|
|
2560
|
+
sourcePath: typeof body.sourcePath === 'string' ? body.sourcePath : 'pasted.sql',
|
|
2561
|
+
name: typeof body.name === 'string' ? body.name : 'SQL match preview',
|
|
2562
|
+
domain: typeof body.domain === 'string' ? body.domain : 'imported',
|
|
2563
|
+
owner: typeof body.owner === 'string' ? body.owner : 'analytics',
|
|
2564
|
+
});
|
|
2565
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2566
|
+
res.end(serializeJSON(match));
|
|
2567
|
+
}
|
|
2568
|
+
catch (error) {
|
|
2569
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2570
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2571
|
+
}
|
|
2572
|
+
return;
|
|
2573
|
+
}
|
|
2574
|
+
const aiImportPathMatch = path.match(/^\/api\/block-studio\/ai-imports\/([^/]+)(?:\/candidates\/([^/]+)(?:\/(preview|certify))?)?$/);
|
|
2575
|
+
if (aiImportPathMatch) {
|
|
2576
|
+
const importId = decodeURIComponent(aiImportPathMatch[1]);
|
|
2577
|
+
const candidateId = aiImportPathMatch[2] ? decodeURIComponent(aiImportPathMatch[2]) : null;
|
|
2578
|
+
const action = aiImportPathMatch[3] ?? null;
|
|
2579
|
+
try {
|
|
2580
|
+
if (req.method === 'GET' && !candidateId) {
|
|
2581
|
+
const session = loadDqlGenerationSession(importId);
|
|
2582
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2583
|
+
res.end(serializeJSON(session));
|
|
2584
|
+
return;
|
|
2585
|
+
}
|
|
2586
|
+
if (req.method === 'PATCH' && candidateId && !action) {
|
|
2587
|
+
const body = await readJSON(req);
|
|
2588
|
+
const candidate = updateBlockStudioImportCandidate(projectRoot, importId, candidateId, {
|
|
2589
|
+
name: typeof body.name === 'string' ? body.name : undefined,
|
|
2590
|
+
domain: typeof body.domain === 'string' ? body.domain : undefined,
|
|
2591
|
+
description: typeof body.description === 'string' ? body.description : undefined,
|
|
2592
|
+
owner: typeof body.owner === 'string' ? body.owner : undefined,
|
|
2593
|
+
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
2594
|
+
terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
|
|
2595
|
+
pattern: typeof body.pattern === 'string' ? body.pattern : undefined,
|
|
2596
|
+
grain: typeof body.grain === 'string' ? body.grain : undefined,
|
|
2597
|
+
entities: Array.isArray(body.entities) ? body.entities.map(String) : undefined,
|
|
2598
|
+
outputs: Array.isArray(body.outputs) ? body.outputs.map(String) : undefined,
|
|
2599
|
+
dimensions: Array.isArray(body.dimensions) ? body.dimensions.map(String) : undefined,
|
|
2600
|
+
allowedFilters: Array.isArray(body.allowedFilters) ? body.allowedFilters.map(String) : undefined,
|
|
2601
|
+
parameterPolicy: Array.isArray(body.parameterPolicy) ? body.parameterPolicy : undefined,
|
|
2602
|
+
filterBindings: Array.isArray(body.filterBindings) ? body.filterBindings : undefined,
|
|
2603
|
+
sourceSystems: Array.isArray(body.sourceSystems) ? body.sourceSystems.map(String) : undefined,
|
|
2604
|
+
replacementFor: Array.isArray(body.replacementFor) ? body.replacementFor.map(String) : undefined,
|
|
2605
|
+
reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
|
|
2606
|
+
sql: typeof body.sql === 'string' ? body.sql : undefined,
|
|
2607
|
+
llmContext: typeof body.llmContext === 'string' ? body.llmContext : undefined,
|
|
2608
|
+
});
|
|
2609
|
+
const validated = validateImportCandidate(candidate);
|
|
2610
|
+
const savedDraft = saveDqlGenerationDraft(importId, validated);
|
|
2611
|
+
writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
|
|
2612
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2613
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2614
|
+
res.end(serializeJSON(savedDraft));
|
|
2615
|
+
return;
|
|
2616
|
+
}
|
|
2617
|
+
if (req.method === 'POST' && candidateId && action === 'preview') {
|
|
2618
|
+
const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
|
|
2619
|
+
const preview = await runBlockStudioPreviewSource(candidate.dqlSource);
|
|
2620
|
+
const next = saveDqlGenerationDraft(importId, {
|
|
2621
|
+
...candidate,
|
|
2622
|
+
preview,
|
|
2623
|
+
validation: validateBlockStudioSource(candidate.dqlSource, semanticLayer),
|
|
2624
|
+
});
|
|
2625
|
+
writeBlockStudioImportCandidate(projectRoot, importId, next);
|
|
2626
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2627
|
+
res.end(serializeJSON(next));
|
|
2628
|
+
return;
|
|
2629
|
+
}
|
|
2630
|
+
if (req.method === 'POST' && candidateId && action === 'certify') {
|
|
2631
|
+
const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
|
|
2632
|
+
const readiness = validateImportCandidateForSave(candidate);
|
|
2633
|
+
if (readiness.errors.length > 0) {
|
|
2634
|
+
const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
|
|
2635
|
+
writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
|
|
2636
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2637
|
+
res.end(serializeJSON({
|
|
2638
|
+
error: readiness.errors.join(' '),
|
|
2639
|
+
candidate: savedDraft,
|
|
2640
|
+
diagnostics: savedDraft.validation?.diagnostics ?? [],
|
|
2641
|
+
}));
|
|
2642
|
+
return;
|
|
2643
|
+
}
|
|
2644
|
+
const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
|
|
2645
|
+
const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: true });
|
|
2646
|
+
const blockers = Array.from(new Set(certification.checklist.blockers));
|
|
2647
|
+
if (!certification.certification.certified || blockers.length > 0) {
|
|
2648
|
+
const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
|
|
2649
|
+
writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
|
|
2650
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2651
|
+
res.end(serializeJSON({ ok: false, candidate: savedDraft, blockers, ...certification }));
|
|
2652
|
+
return;
|
|
2653
|
+
}
|
|
2654
|
+
const savedPath = saveBlockStudioArtifacts(projectRoot, {
|
|
2655
|
+
currentPath: readiness.candidate.savedPath,
|
|
2656
|
+
source: certifiedSource,
|
|
2657
|
+
name: readiness.candidate.name,
|
|
2658
|
+
domain: readiness.candidate.domain,
|
|
2659
|
+
description: readiness.candidate.description,
|
|
2660
|
+
owner: readiness.candidate.owner,
|
|
2661
|
+
tags: readiness.candidate.tags,
|
|
2662
|
+
lineage: readiness.candidate.lineage.sourceTables,
|
|
2663
|
+
importMeta: {
|
|
2664
|
+
importId,
|
|
2665
|
+
candidateId,
|
|
2666
|
+
sourceKind: readiness.candidate.sourceKind,
|
|
2667
|
+
sourcePath: readiness.candidate.sourcePath,
|
|
2668
|
+
},
|
|
2669
|
+
});
|
|
2670
|
+
const next = {
|
|
2671
|
+
...readiness.candidate,
|
|
2672
|
+
dqlSource: certifiedSource,
|
|
2673
|
+
reviewStatus: 'saved',
|
|
2674
|
+
savedPath,
|
|
2675
|
+
validation: validateBlockStudioSource(certifiedSource, semanticLayer),
|
|
2676
|
+
generationMode: readiness.candidate.generationMode ?? 'deterministic',
|
|
2677
|
+
generationProvider: readiness.candidate.generationProvider ?? 'local-deterministic',
|
|
2678
|
+
llmContext: readiness.candidate.llmContext ?? deterministicDqlGenerationContext(readiness.candidate, readiness.candidate.evidence ?? []),
|
|
2679
|
+
evidence: readiness.candidate.evidence ?? [],
|
|
2680
|
+
draftSave: readiness.candidate.draftSave ?? { status: 'pending' },
|
|
2681
|
+
};
|
|
2682
|
+
writeBlockStudioImportCandidate(projectRoot, importId, next);
|
|
2683
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2684
|
+
const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
2685
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2686
|
+
res.end(serializeJSON({ candidate: next, block: payload, certification }));
|
|
2687
|
+
return;
|
|
2688
|
+
}
|
|
2689
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2690
|
+
res.end(serializeJSON({ error: 'Unsupported AI import operation.' }));
|
|
1570
2691
|
}
|
|
1571
2692
|
catch (error) {
|
|
2693
|
+
if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
|
|
2694
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2695
|
+
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
2696
|
+
return;
|
|
2697
|
+
}
|
|
1572
2698
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1573
2699
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1574
2700
|
}
|
|
@@ -1708,6 +2834,8 @@ export async function startLocalServer(opts) {
|
|
|
1708
2834
|
description: typeof body.description === 'string' ? body.description : undefined,
|
|
1709
2835
|
owner: typeof body.owner === 'string' ? body.owner : undefined,
|
|
1710
2836
|
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
2837
|
+
terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
|
|
2838
|
+
reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
|
|
1711
2839
|
sql: typeof body.sql === 'string' ? body.sql : undefined,
|
|
1712
2840
|
reviewStatus,
|
|
1713
2841
|
});
|
|
@@ -1844,6 +2972,55 @@ export async function startLocalServer(opts) {
|
|
|
1844
2972
|
}
|
|
1845
2973
|
return;
|
|
1846
2974
|
}
|
|
2975
|
+
if (req.method === 'GET' && path === '/api/semantic-layer/diagnostics') {
|
|
2976
|
+
try {
|
|
2977
|
+
const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
|
|
2978
|
+
semanticLayer,
|
|
2979
|
+
semanticErrors: semanticLayerErrors,
|
|
2980
|
+
semanticConfig,
|
|
2981
|
+
detectedProvider: semanticDetectedProvider,
|
|
2982
|
+
lastSyncTime: semanticLastSyncTime,
|
|
2983
|
+
});
|
|
2984
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2985
|
+
res.end(serializeJSON(diagnostics));
|
|
2986
|
+
}
|
|
2987
|
+
catch (error) {
|
|
2988
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2989
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2990
|
+
}
|
|
2991
|
+
return;
|
|
2992
|
+
}
|
|
2993
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/reload') {
|
|
2994
|
+
try {
|
|
2995
|
+
const semanticConnection = connection;
|
|
2996
|
+
const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
|
|
2997
|
+
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
|
|
2998
|
+
: undefined;
|
|
2999
|
+
const refreshed = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
|
|
3000
|
+
semanticLayer = refreshed.layer;
|
|
3001
|
+
semanticLayerErrors = refreshed.errors;
|
|
3002
|
+
semanticDetectedProvider = refreshed.detectedProvider;
|
|
3003
|
+
semanticLastSyncTime = refreshed.layer ? new Date().toISOString() : null;
|
|
3004
|
+
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
3005
|
+
const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
|
|
3006
|
+
semanticLayer,
|
|
3007
|
+
semanticErrors: semanticLayerErrors,
|
|
3008
|
+
semanticConfig,
|
|
3009
|
+
detectedProvider: semanticDetectedProvider,
|
|
3010
|
+
lastSyncTime: semanticLastSyncTime,
|
|
3011
|
+
});
|
|
3012
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3013
|
+
res.end(serializeJSON({
|
|
3014
|
+
ok: Boolean(refreshed.layer),
|
|
3015
|
+
...diagnostics,
|
|
3016
|
+
}));
|
|
3017
|
+
}
|
|
3018
|
+
catch (error) {
|
|
3019
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3020
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3021
|
+
}
|
|
3022
|
+
return;
|
|
3023
|
+
}
|
|
1847
3024
|
if (req.method === 'GET' && path === '/api/block-studio/open') {
|
|
1848
3025
|
try {
|
|
1849
3026
|
const relativePath = url.searchParams.get('path');
|
|
@@ -1908,8 +3085,9 @@ export async function startLocalServer(opts) {
|
|
|
1908
3085
|
res.end(serializeJSON({ error: 'Block name is required.' }));
|
|
1909
3086
|
return;
|
|
1910
3087
|
}
|
|
1911
|
-
const
|
|
1912
|
-
|
|
3088
|
+
const currentPath = typeof body.path === 'string' ? body.path : undefined;
|
|
3089
|
+
const saveOptions = {
|
|
3090
|
+
currentPath,
|
|
1913
3091
|
source,
|
|
1914
3092
|
name: metadata.name,
|
|
1915
3093
|
domain: metadata.domain,
|
|
@@ -1925,7 +3103,13 @@ export async function startLocalServer(opts) {
|
|
|
1925
3103
|
sourcePath: metadata.sourcePath,
|
|
1926
3104
|
}
|
|
1927
3105
|
: undefined,
|
|
1928
|
-
}
|
|
3106
|
+
};
|
|
3107
|
+
const savedPath = isDraftBlockPath(currentPath)
|
|
3108
|
+
? saveBlockStudioDraftArtifacts(projectRoot, {
|
|
3109
|
+
...saveOptions,
|
|
3110
|
+
stableSuffix: metadata.candidateId,
|
|
3111
|
+
})
|
|
3112
|
+
: saveBlockStudioArtifacts(projectRoot, saveOptions);
|
|
1929
3113
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
1930
3114
|
const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
1931
3115
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -2051,7 +3235,7 @@ export async function startLocalServer(opts) {
|
|
|
2051
3235
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2052
3236
|
res.end(serializeJSON({
|
|
2053
3237
|
available: false,
|
|
2054
|
-
provider:
|
|
3238
|
+
provider: semanticConfig?.provider ?? semanticDetectedProvider ?? null,
|
|
2055
3239
|
errors: semanticLayerErrors,
|
|
2056
3240
|
metrics: [],
|
|
2057
3241
|
measures: [],
|
|
@@ -2180,9 +3364,9 @@ export async function startLocalServer(opts) {
|
|
|
2180
3364
|
owner: q.owner ?? null,
|
|
2181
3365
|
source: q.source ?? null,
|
|
2182
3366
|
}));
|
|
2183
|
-
const provider =
|
|
3367
|
+
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
2184
3368
|
const dbtExecutionReady = provider === 'dbt'
|
|
2185
|
-
? hasDbtSemanticManifest(projectRoot,
|
|
3369
|
+
? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
|
|
2186
3370
|
: false;
|
|
2187
3371
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2188
3372
|
res.end(serializeJSON({
|
|
@@ -2754,8 +3938,12 @@ export async function startLocalServer(opts) {
|
|
|
2754
3938
|
return;
|
|
2755
3939
|
}
|
|
2756
3940
|
if (req.method === 'POST' && path === '/api/query') {
|
|
3941
|
+
let body;
|
|
3942
|
+
let execContext = null;
|
|
3943
|
+
const start = Date.now();
|
|
2757
3944
|
try {
|
|
2758
|
-
|
|
3945
|
+
body = await readJSON(req);
|
|
3946
|
+
execContext = notebookExecutionContext(body.executionContext);
|
|
2759
3947
|
if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
|
|
2760
3948
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2761
3949
|
res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
|
|
@@ -2778,7 +3966,26 @@ export async function startLocalServer(opts) {
|
|
|
2778
3966
|
const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
|
|
2779
3967
|
assertAppAccess({ app, domain, level: 'execute' });
|
|
2780
3968
|
const result = await executor.executeQuery(prepared.sql, Array.isArray(body.sqlParams) ? body.sqlParams : [], runtimeVariables(body.variables && typeof body.variables === 'object' ? body.variables : {}), prepared.connection);
|
|
2781
|
-
const
|
|
3969
|
+
const normalized = normalizeQueryResult(result, semantic.semanticRefs);
|
|
3970
|
+
if (execContext) {
|
|
3971
|
+
recordNotebookQueryRun(projectRoot, {
|
|
3972
|
+
notebookPath: execContext.notebookPath,
|
|
3973
|
+
cellId: execContext.cellId,
|
|
3974
|
+
cellName: execContext.cellName,
|
|
3975
|
+
researchRunId: execContext.researchRunId,
|
|
3976
|
+
source: execContext.source ?? 'notebook_sql_cell',
|
|
3977
|
+
status: 'success',
|
|
3978
|
+
rowCount: normalized.rowCount ?? normalized.rows.length,
|
|
3979
|
+
durationMs: Date.now() - start,
|
|
3980
|
+
sql: body.sql,
|
|
3981
|
+
});
|
|
3982
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
3983
|
+
status: 'success',
|
|
3984
|
+
resultPreview: normalized,
|
|
3985
|
+
sql: body.sql,
|
|
3986
|
+
});
|
|
3987
|
+
}
|
|
3988
|
+
const payload = serializeJSON(normalized);
|
|
2782
3989
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2783
3990
|
res.end(payload);
|
|
2784
3991
|
}
|
|
@@ -2797,6 +4004,24 @@ export async function startLocalServer(opts) {
|
|
|
2797
4004
|
}));
|
|
2798
4005
|
return;
|
|
2799
4006
|
}
|
|
4007
|
+
if (execContext) {
|
|
4008
|
+
recordNotebookQueryRun(projectRoot, {
|
|
4009
|
+
notebookPath: execContext.notebookPath,
|
|
4010
|
+
cellId: execContext.cellId,
|
|
4011
|
+
cellName: execContext.cellName,
|
|
4012
|
+
researchRunId: execContext.researchRunId,
|
|
4013
|
+
source: execContext.source ?? 'notebook_sql_cell',
|
|
4014
|
+
status: 'error',
|
|
4015
|
+
durationMs: Date.now() - start,
|
|
4016
|
+
errorCode: error instanceof Error ? error.message : String(error),
|
|
4017
|
+
sql: typeof body?.sql === 'string' ? body.sql : undefined,
|
|
4018
|
+
});
|
|
4019
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
4020
|
+
status: 'error',
|
|
4021
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4022
|
+
sql: typeof body?.sql === 'string' ? body.sql : undefined,
|
|
4023
|
+
});
|
|
4024
|
+
}
|
|
2800
4025
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2801
4026
|
res.end(serializeJSON({
|
|
2802
4027
|
columns: [],
|
|
@@ -3138,6 +4363,25 @@ export async function startLocalServer(opts) {
|
|
|
3138
4363
|
}
|
|
3139
4364
|
return;
|
|
3140
4365
|
}
|
|
4366
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/business-360/')) {
|
|
4367
|
+
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/business-360/'.length));
|
|
4368
|
+
try {
|
|
4369
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
4370
|
+
const result = queryBusiness360(graph, rawNodeId);
|
|
4371
|
+
if (!result) {
|
|
4372
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4373
|
+
res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
|
|
4374
|
+
return;
|
|
4375
|
+
}
|
|
4376
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4377
|
+
res.end(serializeJSON(result));
|
|
4378
|
+
}
|
|
4379
|
+
catch (error) {
|
|
4380
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4381
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
4382
|
+
}
|
|
4383
|
+
return;
|
|
4384
|
+
}
|
|
3141
4385
|
if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
|
|
3142
4386
|
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
|
|
3143
4387
|
try {
|
|
@@ -3303,8 +4547,12 @@ export async function startLocalServer(opts) {
|
|
|
3303
4547
|
return;
|
|
3304
4548
|
}
|
|
3305
4549
|
if (req.method === 'POST' && path === '/api/notebook/execute') {
|
|
4550
|
+
let body;
|
|
4551
|
+
let execContext = null;
|
|
4552
|
+
const start = Date.now();
|
|
3306
4553
|
try {
|
|
3307
|
-
|
|
4554
|
+
body = await readJSON(req);
|
|
4555
|
+
execContext = notebookExecutionContext(body.executionContext);
|
|
3308
4556
|
const cell = normalizeNotebookCell(body.cell);
|
|
3309
4557
|
if (!cell) {
|
|
3310
4558
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -3327,6 +4575,26 @@ export async function startLocalServer(opts) {
|
|
|
3327
4575
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
3328
4576
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
3329
4577
|
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
|
|
4578
|
+
const normalized = normalizeQueryResult(rawResult);
|
|
4579
|
+
if (execContext) {
|
|
4580
|
+
recordNotebookQueryRun(projectRoot, {
|
|
4581
|
+
notebookPath: execContext.notebookPath,
|
|
4582
|
+
cellId: execContext.cellId ?? cell.id,
|
|
4583
|
+
cellName: execContext.cellName ?? plan.title ?? resolved.blockName,
|
|
4584
|
+
researchRunId: execContext.researchRunId,
|
|
4585
|
+
source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_cell'),
|
|
4586
|
+
status: 'success',
|
|
4587
|
+
rowCount: normalized.rowCount ?? normalized.rows.length,
|
|
4588
|
+
durationMs: Date.now() - start,
|
|
4589
|
+
sql: plan.sql,
|
|
4590
|
+
objectKey: resolved.blockPath,
|
|
4591
|
+
});
|
|
4592
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
4593
|
+
status: 'success',
|
|
4594
|
+
resultPreview: normalized,
|
|
4595
|
+
sql: plan.sql,
|
|
4596
|
+
});
|
|
4597
|
+
}
|
|
3330
4598
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3331
4599
|
res.end(serializeJSON({
|
|
3332
4600
|
cellType: cell.type,
|
|
@@ -3335,10 +4603,26 @@ export async function startLocalServer(opts) {
|
|
|
3335
4603
|
blockPath: resolved.blockPath,
|
|
3336
4604
|
chartConfig: plan.chartConfig,
|
|
3337
4605
|
tests: plan.tests,
|
|
3338
|
-
result:
|
|
4606
|
+
result: normalized,
|
|
3339
4607
|
}));
|
|
3340
4608
|
}
|
|
3341
4609
|
catch (error) {
|
|
4610
|
+
if (execContext) {
|
|
4611
|
+
recordNotebookQueryRun(projectRoot, {
|
|
4612
|
+
notebookPath: execContext.notebookPath,
|
|
4613
|
+
cellId: execContext.cellId,
|
|
4614
|
+
cellName: execContext.cellName,
|
|
4615
|
+
researchRunId: execContext.researchRunId,
|
|
4616
|
+
source: execContext.source ?? 'notebook_cell',
|
|
4617
|
+
status: 'error',
|
|
4618
|
+
durationMs: Date.now() - start,
|
|
4619
|
+
errorCode: error instanceof Error ? error.message : String(error),
|
|
4620
|
+
});
|
|
4621
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
4622
|
+
status: 'error',
|
|
4623
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4624
|
+
});
|
|
4625
|
+
}
|
|
3342
4626
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3343
4627
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3344
4628
|
}
|
|
@@ -3867,6 +5151,245 @@ export function prepareLocalExecution(sql, connection, projectRoot, projectConfi
|
|
|
3867
5151
|
connection: normalizedConnection,
|
|
3868
5152
|
};
|
|
3869
5153
|
}
|
|
5154
|
+
export function dashboardRuntimeVariables(dashboard, overrides = {}) {
|
|
5155
|
+
const variables = {};
|
|
5156
|
+
for (const param of dashboard.params ?? []) {
|
|
5157
|
+
if (param.default !== undefined)
|
|
5158
|
+
variables[param.id] = param.default;
|
|
5159
|
+
}
|
|
5160
|
+
for (const filter of dashboard.filters ?? []) {
|
|
5161
|
+
if (filter.default !== undefined)
|
|
5162
|
+
variables[filter.id] = filter.default;
|
|
5163
|
+
}
|
|
5164
|
+
return { ...variables, ...overrides };
|
|
5165
|
+
}
|
|
5166
|
+
export function applyDashboardFiltersToBlockExecution(input) {
|
|
5167
|
+
const variables = { ...input.variables };
|
|
5168
|
+
const sqlParams = [...input.sqlParams];
|
|
5169
|
+
const appliedFilters = [];
|
|
5170
|
+
const skippedFilters = [];
|
|
5171
|
+
const clauses = [];
|
|
5172
|
+
let nextPosition = sqlParams.reduce((max, param) => Math.max(max, param.position), 0);
|
|
5173
|
+
for (const filter of input.dashboard.filters ?? []) {
|
|
5174
|
+
const value = dashboardFilterValue(filter, variables);
|
|
5175
|
+
if (isEmptyDashboardFilterValue(value)) {
|
|
5176
|
+
skippedFilters.push({ filter: filter.id, reason: 'no value supplied' });
|
|
5177
|
+
continue;
|
|
5178
|
+
}
|
|
5179
|
+
const paramNames = bindDashboardFilterToExistingParams(filter, value, input.block, sqlParams, variables);
|
|
5180
|
+
if (paramNames.length > 0) {
|
|
5181
|
+
appliedFilters.push({
|
|
5182
|
+
filter: filter.id,
|
|
5183
|
+
mode: 'parameter',
|
|
5184
|
+
paramNames,
|
|
5185
|
+
});
|
|
5186
|
+
continue;
|
|
5187
|
+
}
|
|
5188
|
+
const binding = resolveDashboardFilterBinding(filter, input.block);
|
|
5189
|
+
if (!binding) {
|
|
5190
|
+
skippedFilters.push({ filter: filter.id, reason: `block "${input.block.name}" does not declare a compatible filter binding` });
|
|
5191
|
+
continue;
|
|
5192
|
+
}
|
|
5193
|
+
const expression = dashboardFilterExpression(binding);
|
|
5194
|
+
if (!expression) {
|
|
5195
|
+
skippedFilters.push({ filter: filter.id, reason: `filter binding "${binding}" is not safe for runtime predicate injection` });
|
|
5196
|
+
continue;
|
|
5197
|
+
}
|
|
5198
|
+
const predicate = buildDashboardFilterPredicate({
|
|
5199
|
+
expression,
|
|
5200
|
+
filterId: filter.id,
|
|
5201
|
+
filterType: filter.type,
|
|
5202
|
+
value,
|
|
5203
|
+
params: sqlParams,
|
|
5204
|
+
nextPosition: () => {
|
|
5205
|
+
nextPosition += 1;
|
|
5206
|
+
return nextPosition;
|
|
5207
|
+
},
|
|
5208
|
+
variables,
|
|
5209
|
+
});
|
|
5210
|
+
if (!predicate) {
|
|
5211
|
+
skippedFilters.push({ filter: filter.id, reason: 'filter value could not be converted into a predicate' });
|
|
5212
|
+
continue;
|
|
5213
|
+
}
|
|
5214
|
+
clauses.push(predicate);
|
|
5215
|
+
appliedFilters.push({
|
|
5216
|
+
filter: filter.id,
|
|
5217
|
+
binding,
|
|
5218
|
+
mode: 'predicate',
|
|
5219
|
+
paramNames: sqlParams
|
|
5220
|
+
.filter((param) => param.name.startsWith(`__dashboard_filter_${normalizeDashboardFilterName(filter.id)}_`))
|
|
5221
|
+
.map((param) => param.name),
|
|
5222
|
+
});
|
|
5223
|
+
}
|
|
5224
|
+
if (clauses.length === 0) {
|
|
5225
|
+
return { sql: input.sql, sqlParams, variables, appliedFilters, skippedFilters };
|
|
5226
|
+
}
|
|
5227
|
+
return {
|
|
5228
|
+
sql: `SELECT * FROM (${stripSqlTerminator(input.sql)}) _dql_filter WHERE ${clauses.join(' AND ')}`,
|
|
5229
|
+
sqlParams,
|
|
5230
|
+
variables,
|
|
5231
|
+
appliedFilters,
|
|
5232
|
+
skippedFilters,
|
|
5233
|
+
};
|
|
5234
|
+
}
|
|
5235
|
+
function dashboardFilterValue(filter, variables) {
|
|
5236
|
+
if (Object.prototype.hasOwnProperty.call(variables, filter.id))
|
|
5237
|
+
return variables[filter.id];
|
|
5238
|
+
if (filter.bindsTo && Object.prototype.hasOwnProperty.call(variables, filter.bindsTo))
|
|
5239
|
+
return variables[filter.bindsTo];
|
|
5240
|
+
return filter.default;
|
|
5241
|
+
}
|
|
5242
|
+
function isEmptyDashboardFilterValue(value) {
|
|
5243
|
+
if (value === undefined || value === null)
|
|
5244
|
+
return true;
|
|
5245
|
+
if (typeof value === 'string' && value.trim() === '')
|
|
5246
|
+
return true;
|
|
5247
|
+
if (Array.isArray(value) && value.length === 0)
|
|
5248
|
+
return true;
|
|
5249
|
+
return false;
|
|
5250
|
+
}
|
|
5251
|
+
function bindDashboardFilterToExistingParams(filter, value, block, sqlParams, variables) {
|
|
5252
|
+
const availableParamNames = new Set(sqlParams.map((param) => param.name));
|
|
5253
|
+
const declaredParamNames = new Set((block.parameterPolicy ?? []).map((entry) => entry.name));
|
|
5254
|
+
const applied = [];
|
|
5255
|
+
const directCandidates = uniqueDashboardStrings([
|
|
5256
|
+
filter.id,
|
|
5257
|
+
filter.bindsTo ?? '',
|
|
5258
|
+
normalizeDashboardFilterName(filter.id),
|
|
5259
|
+
filter.bindsTo ? normalizeDashboardFilterName(filter.bindsTo) : '',
|
|
5260
|
+
]);
|
|
5261
|
+
for (const name of directCandidates) {
|
|
5262
|
+
if (!name || !availableParamNames.has(name))
|
|
5263
|
+
continue;
|
|
5264
|
+
variables[name] = value;
|
|
5265
|
+
applied.push(name);
|
|
5266
|
+
}
|
|
5267
|
+
if (applied.length > 0)
|
|
5268
|
+
return applied;
|
|
5269
|
+
const range = dashboardRangeValue(value);
|
|
5270
|
+
if (range) {
|
|
5271
|
+
const baseNames = uniqueDashboardStrings([
|
|
5272
|
+
filter.id,
|
|
5273
|
+
filter.id.replace(/_?range$/i, ''),
|
|
5274
|
+
filter.bindsTo ?? '',
|
|
5275
|
+
filter.bindsTo ? filter.bindsTo.replace(/_?range$/i, '') : '',
|
|
5276
|
+
].map(normalizeDashboardFilterName));
|
|
5277
|
+
for (const base of baseNames) {
|
|
5278
|
+
const pairs = [
|
|
5279
|
+
[`${base}_start`, `${base}_end`],
|
|
5280
|
+
[`${base}_from`, `${base}_to`],
|
|
5281
|
+
[`start_${base}`, `end_${base}`],
|
|
5282
|
+
];
|
|
5283
|
+
for (const [startName, endName] of pairs) {
|
|
5284
|
+
if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
|
|
5285
|
+
variables[startName] = range.start;
|
|
5286
|
+
variables[endName] = range.end;
|
|
5287
|
+
return [startName, endName];
|
|
5288
|
+
}
|
|
5289
|
+
}
|
|
5290
|
+
}
|
|
5291
|
+
for (const [startName, endName] of [['start_date', 'end_date'], ['date_start', 'date_end'], ['season_start', 'season_end'], ['year_start', 'year_end']]) {
|
|
5292
|
+
if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
|
|
5293
|
+
variables[startName] = range.start;
|
|
5294
|
+
variables[endName] = range.end;
|
|
5295
|
+
return [startName, endName];
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
}
|
|
5299
|
+
for (const name of declaredParamNames) {
|
|
5300
|
+
if (!availableParamNames.has(name))
|
|
5301
|
+
continue;
|
|
5302
|
+
const normalized = normalizeDashboardFilterName(name);
|
|
5303
|
+
if (normalized === normalizeDashboardFilterName(filter.id) || normalized === normalizeDashboardFilterName(filter.bindsTo ?? '')) {
|
|
5304
|
+
variables[name] = value;
|
|
5305
|
+
return [name];
|
|
5306
|
+
}
|
|
5307
|
+
}
|
|
5308
|
+
return [];
|
|
5309
|
+
}
|
|
5310
|
+
function resolveDashboardFilterBinding(filter, block) {
|
|
5311
|
+
const candidates = uniqueDashboardStrings([filter.id, filter.bindsTo ?? '']).map(normalizeDashboardFilterName);
|
|
5312
|
+
for (const entry of block.filterBindings ?? []) {
|
|
5313
|
+
if (candidates.includes(normalizeDashboardFilterName(entry.filter)))
|
|
5314
|
+
return entry.binding;
|
|
5315
|
+
}
|
|
5316
|
+
if (filter.bindsTo && (block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.bindsTo ?? ''))) {
|
|
5317
|
+
return filter.bindsTo;
|
|
5318
|
+
}
|
|
5319
|
+
if ((block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.id))) {
|
|
5320
|
+
return filter.id;
|
|
5321
|
+
}
|
|
5322
|
+
return null;
|
|
5323
|
+
}
|
|
5324
|
+
function dashboardFilterExpression(binding) {
|
|
5325
|
+
const trimmed = binding.trim();
|
|
5326
|
+
const yearMatch = trimmed.match(/^([A-Za-z_][A-Za-z0-9_.]*)\.year$/i);
|
|
5327
|
+
if (yearMatch) {
|
|
5328
|
+
const base = dashboardOutputColumn(yearMatch[1]);
|
|
5329
|
+
return base ? `EXTRACT(YEAR FROM _dql_filter.${base})` : null;
|
|
5330
|
+
}
|
|
5331
|
+
const column = dashboardOutputColumn(trimmed);
|
|
5332
|
+
return column ? `_dql_filter.${column}` : null;
|
|
5333
|
+
}
|
|
5334
|
+
function dashboardOutputColumn(binding) {
|
|
5335
|
+
const cleaned = binding.replace(/[`"[\]]/g, '').trim();
|
|
5336
|
+
if (!/^[A-Za-z_][A-Za-z0-9_.]*$/.test(cleaned))
|
|
5337
|
+
return null;
|
|
5338
|
+
const column = cleaned.split('.').filter(Boolean).at(-1);
|
|
5339
|
+
return column && /^[A-Za-z_][A-Za-z0-9_]*$/.test(column) ? column : null;
|
|
5340
|
+
}
|
|
5341
|
+
function buildDashboardFilterPredicate(input) {
|
|
5342
|
+
const range = input.filterType === 'daterange' ? dashboardRangeValue(input.value) : null;
|
|
5343
|
+
if (range) {
|
|
5344
|
+
const start = addDashboardFilterParam(input, 'start', range.start);
|
|
5345
|
+
const end = addDashboardFilterParam(input, 'end', range.end);
|
|
5346
|
+
return `${input.expression} BETWEEN $${start.position} AND $${end.position}`;
|
|
5347
|
+
}
|
|
5348
|
+
const values = Array.isArray(input.value) ? input.value.filter((item) => !isEmptyDashboardFilterValue(item)) : [input.value];
|
|
5349
|
+
if (values.length === 0)
|
|
5350
|
+
return null;
|
|
5351
|
+
if (values.length === 1) {
|
|
5352
|
+
const param = addDashboardFilterParam(input, 'value', values[0]);
|
|
5353
|
+
return `${input.expression} = $${param.position}`;
|
|
5354
|
+
}
|
|
5355
|
+
const placeholders = values.map((value, index) => {
|
|
5356
|
+
const param = addDashboardFilterParam(input, `value_${index + 1}`, value);
|
|
5357
|
+
return `$${param.position}`;
|
|
5358
|
+
});
|
|
5359
|
+
return `${input.expression} IN (${placeholders.join(', ')})`;
|
|
5360
|
+
}
|
|
5361
|
+
function addDashboardFilterParam(input, suffix, value) {
|
|
5362
|
+
const name = `__dashboard_filter_${normalizeDashboardFilterName(input.filterId)}_${suffix}`;
|
|
5363
|
+
const uniqueName = input.variables[name] === undefined && !input.params.some((param) => param.name === name)
|
|
5364
|
+
? name
|
|
5365
|
+
: `${name}_${input.params.length + 1}`;
|
|
5366
|
+
const param = { name: uniqueName, position: input.nextPosition() };
|
|
5367
|
+
input.params.push(param);
|
|
5368
|
+
input.variables[uniqueName] = value;
|
|
5369
|
+
return param;
|
|
5370
|
+
}
|
|
5371
|
+
function dashboardRangeValue(value) {
|
|
5372
|
+
if (Array.isArray(value) && value.length >= 2 && !isEmptyDashboardFilterValue(value[0]) && !isEmptyDashboardFilterValue(value[1])) {
|
|
5373
|
+
return { start: value[0], end: value[1] };
|
|
5374
|
+
}
|
|
5375
|
+
if (value && typeof value === 'object') {
|
|
5376
|
+
const record = value;
|
|
5377
|
+
const start = record.start ?? record.from ?? record.min;
|
|
5378
|
+
const end = record.end ?? record.to ?? record.max;
|
|
5379
|
+
if (!isEmptyDashboardFilterValue(start) && !isEmptyDashboardFilterValue(end))
|
|
5380
|
+
return { start, end };
|
|
5381
|
+
}
|
|
5382
|
+
return null;
|
|
5383
|
+
}
|
|
5384
|
+
function normalizeDashboardFilterName(value) {
|
|
5385
|
+
return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
5386
|
+
}
|
|
5387
|
+
function stripSqlTerminator(sql) {
|
|
5388
|
+
return sql.trim().replace(/;\s*$/, '');
|
|
5389
|
+
}
|
|
5390
|
+
function uniqueDashboardStrings(values) {
|
|
5391
|
+
return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
|
|
5392
|
+
}
|
|
3870
5393
|
export function resolveDbtMacrosForExecution(sql, projectRoot, projectConfig = {}) {
|
|
3871
5394
|
if (!/\{\{\s*(?:ref|source)\s*\(/i.test(sql))
|
|
3872
5395
|
return sql;
|
|
@@ -4484,9 +6007,9 @@ function buildDatabaseSchemaTree(projectRoot, executor, connection) {
|
|
|
4484
6007
|
return databaseNodes;
|
|
4485
6008
|
})();
|
|
4486
6009
|
}
|
|
4487
|
-
function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
6010
|
+
export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
4488
6011
|
const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
|
|
4489
|
-
if (!normalizedPath
|
|
6012
|
+
if (!isBlockStudioBlockPath(normalizedPath)) {
|
|
4490
6013
|
throw new Error('Invalid block path');
|
|
4491
6014
|
}
|
|
4492
6015
|
const absPath = join(projectRoot, normalizedPath);
|
|
@@ -4501,7 +6024,7 @@ function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
|
4501
6024
|
const metadata = {
|
|
4502
6025
|
name: parsedMetadata.name || companion?.name || fileName,
|
|
4503
6026
|
path: normalizedPath,
|
|
4504
|
-
domain: parsedMetadata.domain || companion?.domain || normalizedPath
|
|
6027
|
+
domain: parsedMetadata.domain || companion?.domain || inferBlockStudioPathDomain(normalizedPath) || 'uncategorized',
|
|
4505
6028
|
description: parsedMetadata.description || companion?.description || '',
|
|
4506
6029
|
owner: parsedMetadata.owner || companion?.owner || '',
|
|
4507
6030
|
tags: parsedMetadata.tags.length > 0 ? parsedMetadata.tags : companion?.tags ?? [],
|
|
@@ -4584,7 +6107,7 @@ export function buildSemanticTableMapping(semanticLayer, rows) {
|
|
|
4584
6107
|
return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
|
|
4585
6108
|
}
|
|
4586
6109
|
function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
|
|
4587
|
-
if (projectConfig.semanticLayer?.provider === 'dbt' || detectedProvider === 'dbt')
|
|
6110
|
+
if (projectConfig.semanticLayer?.provider === 'dbt' || projectConfig.dbt?.projectDir || detectedProvider === 'dbt')
|
|
4588
6111
|
return true;
|
|
4589
6112
|
return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
|
|
4590
6113
|
}
|
|
@@ -4592,7 +6115,10 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
|
4592
6115
|
const useMetricFlow = request.engine === 'metricflow' || (request.engine !== 'native' &&
|
|
4593
6116
|
isDbtSemanticRuntime(context.projectConfig, context.detectedProvider, semanticLayer));
|
|
4594
6117
|
if (useMetricFlow) {
|
|
4595
|
-
const
|
|
6118
|
+
const effectiveSemanticConfig = resolveProjectSemanticConfig(context.projectConfig, context.projectRoot);
|
|
6119
|
+
const dbtProjectPath = effectiveSemanticConfig?.provider === 'dbt'
|
|
6120
|
+
? effectiveSemanticConfig.projectPath
|
|
6121
|
+
: context.projectConfig.dbt?.projectDir;
|
|
4596
6122
|
const compiled = compileMetricFlowQuery({
|
|
4597
6123
|
projectRoot: context.projectRoot,
|
|
4598
6124
|
dbtProjectPath,
|
|
@@ -4851,6 +6377,164 @@ export function validateBlockStudioSource(source, semanticLayer) {
|
|
|
4851
6377
|
executableSql,
|
|
4852
6378
|
};
|
|
4853
6379
|
}
|
|
6380
|
+
export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
6381
|
+
const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
|
|
6382
|
+
const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
|
|
6383
|
+
const provider = deterministicOnly ? null : await createBlockStudioAssistProvider(projectRoot, requestedProvider);
|
|
6384
|
+
const session = createBlockStudioImportSession(projectRoot, {
|
|
6385
|
+
inputPath: options.inputPath ?? '',
|
|
6386
|
+
inputMode: options.inputMode,
|
|
6387
|
+
sources: options.sources,
|
|
6388
|
+
sourceKind: options.sourceKind ?? 'raw-sql',
|
|
6389
|
+
domain: options.domain,
|
|
6390
|
+
owner: options.owner,
|
|
6391
|
+
tags: options.tags,
|
|
6392
|
+
});
|
|
6393
|
+
const warnings = [];
|
|
6394
|
+
const nextCandidates = [];
|
|
6395
|
+
let contextObjectCount = 0;
|
|
6396
|
+
for (const candidate of session.candidates) {
|
|
6397
|
+
const contextPack = await buildDqlGenerationContextPack(projectRoot, candidate).catch((error) => {
|
|
6398
|
+
warnings.push(`Context pack failed for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
6399
|
+
return null;
|
|
6400
|
+
});
|
|
6401
|
+
contextObjectCount += contextPack?.objects.length ?? 0;
|
|
6402
|
+
const evidence = contextPack ? dqlGenerationEvidenceFromContext(contextPack, candidate) : deterministicDqlGenerationEvidence(candidate);
|
|
6403
|
+
let patch = deterministicDqlGenerationPatch(candidate, evidence);
|
|
6404
|
+
let generatorName = 'local-deterministic';
|
|
6405
|
+
let generationMode = 'deterministic';
|
|
6406
|
+
if (provider) {
|
|
6407
|
+
const aiPatch = await buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack).catch((error) => {
|
|
6408
|
+
warnings.push(`AI generation fell back for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
6409
|
+
return null;
|
|
6410
|
+
});
|
|
6411
|
+
if (aiPatch) {
|
|
6412
|
+
patch = mergeDqlGenerationPatch(patch, aiPatch, candidate, evidence);
|
|
6413
|
+
generatorName = provider.name;
|
|
6414
|
+
generationMode = 'ai';
|
|
6415
|
+
}
|
|
6416
|
+
}
|
|
6417
|
+
const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
|
|
6418
|
+
patch = {
|
|
6419
|
+
...patch,
|
|
6420
|
+
similarityMatches: similarity.matches,
|
|
6421
|
+
recommendedAction: similarity.recommendedAction,
|
|
6422
|
+
};
|
|
6423
|
+
for (const warning of patch.warnings ?? [])
|
|
6424
|
+
warnings.push(`${candidate.name}: ${warning}`);
|
|
6425
|
+
const enriched = updateBlockStudioImportCandidate(projectRoot, session.id, candidate.id, {
|
|
6426
|
+
name: patch.name,
|
|
6427
|
+
domain: patch.domain,
|
|
6428
|
+
description: patch.description,
|
|
6429
|
+
owner: patch.owner,
|
|
6430
|
+
tags: patch.tags,
|
|
6431
|
+
terms: patch.terms,
|
|
6432
|
+
pattern: patch.pattern,
|
|
6433
|
+
grain: patch.grain,
|
|
6434
|
+
entities: patch.entities,
|
|
6435
|
+
outputs: patch.outputs,
|
|
6436
|
+
dimensions: patch.dimensions,
|
|
6437
|
+
allowedFilters: patch.allowedFilters,
|
|
6438
|
+
parameterPolicy: patch.parameterPolicy,
|
|
6439
|
+
filterBindings: patch.filterBindings,
|
|
6440
|
+
parameterDecisions: patch.parameterDecisions,
|
|
6441
|
+
similarityMatches: patch.similarityMatches,
|
|
6442
|
+
recommendedAction: patch.recommendedAction,
|
|
6443
|
+
sourceSystems: patch.sourceSystems,
|
|
6444
|
+
replacementFor: patch.replacementFor,
|
|
6445
|
+
reviewCadence: patch.reviewCadence,
|
|
6446
|
+
sql: patch.sql,
|
|
6447
|
+
llmContext: patch.llmContext,
|
|
6448
|
+
evidence,
|
|
6449
|
+
conversionNotes: dqlGenerationConversionNotes(generatorName),
|
|
6450
|
+
generationMode,
|
|
6451
|
+
generationProvider: generatorName,
|
|
6452
|
+
});
|
|
6453
|
+
const validated = {
|
|
6454
|
+
...enriched,
|
|
6455
|
+
validation: validateBlockStudioSource(enriched.dqlSource, semanticLayer),
|
|
6456
|
+
};
|
|
6457
|
+
const savedDraft = saveDqlGenerationDraftForProject(projectRoot, session.id, validated);
|
|
6458
|
+
writeBlockStudioImportCandidate(projectRoot, session.id, savedDraft);
|
|
6459
|
+
nextCandidates.push(savedDraft);
|
|
6460
|
+
}
|
|
6461
|
+
const generationSession = {
|
|
6462
|
+
...session,
|
|
6463
|
+
mode: 'ai-import',
|
|
6464
|
+
candidates: nextCandidates,
|
|
6465
|
+
updatedAt: new Date().toISOString(),
|
|
6466
|
+
generation: {
|
|
6467
|
+
provider: nextCandidates.find((candidate) => candidate.generationMode === 'ai')?.generationProvider ?? 'local-deterministic',
|
|
6468
|
+
aiEnabled: nextCandidates.some((candidate) => candidate.generationMode === 'ai'),
|
|
6469
|
+
contextObjectCount,
|
|
6470
|
+
createdDrafts: nextCandidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
|
|
6471
|
+
warnings,
|
|
6472
|
+
},
|
|
6473
|
+
};
|
|
6474
|
+
writeBlockStudioImportSession(projectRoot, generationSession);
|
|
6475
|
+
if (generationSession.generation.createdDrafts > 0)
|
|
6476
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
6477
|
+
return generationSession;
|
|
6478
|
+
}
|
|
6479
|
+
function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
|
|
6480
|
+
if (candidate.recommendedAction === 'reuse_existing') {
|
|
6481
|
+
const topMatch = candidate.similarityMatches?.[0];
|
|
6482
|
+
return {
|
|
6483
|
+
...candidate,
|
|
6484
|
+
reviewStatus: 'review',
|
|
6485
|
+
draftSave: {
|
|
6486
|
+
status: 'skipped',
|
|
6487
|
+
reason: topMatch
|
|
6488
|
+
? `Reuse recommended: ${topMatch.name} (${topMatch.kind}, ${(topMatch.score * 100).toFixed(0)}%).`
|
|
6489
|
+
: 'Reuse recommended; no new draft block was needed.',
|
|
6490
|
+
},
|
|
6491
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
6492
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
6493
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
6494
|
+
evidence: candidate.evidence ?? [],
|
|
6495
|
+
};
|
|
6496
|
+
}
|
|
6497
|
+
try {
|
|
6498
|
+
const savedPath = saveBlockStudioDraftArtifacts(projectRoot, {
|
|
6499
|
+
currentPath: isDraftBlockPath(candidate.savedPath) ? candidate.savedPath : undefined,
|
|
6500
|
+
source: candidate.dqlSource,
|
|
6501
|
+
name: candidate.name,
|
|
6502
|
+
domain: candidate.domain,
|
|
6503
|
+
description: candidate.description,
|
|
6504
|
+
owner: candidate.owner,
|
|
6505
|
+
tags: candidate.tags,
|
|
6506
|
+
lineage: candidate.lineage.sourceTables,
|
|
6507
|
+
stableSuffix: candidate.id.replace(/^cand_/, ''),
|
|
6508
|
+
importMeta: {
|
|
6509
|
+
importId,
|
|
6510
|
+
candidateId: candidate.id,
|
|
6511
|
+
sourceKind: candidate.sourceKind,
|
|
6512
|
+
sourcePath: candidate.sourcePath,
|
|
6513
|
+
},
|
|
6514
|
+
});
|
|
6515
|
+
const draftSave = { status: 'saved', path: savedPath, savedAt: new Date().toISOString() };
|
|
6516
|
+
return {
|
|
6517
|
+
...candidate,
|
|
6518
|
+
reviewStatus: candidate.reviewStatus === 'rejected' ? 'rejected' : 'draft',
|
|
6519
|
+
savedPath,
|
|
6520
|
+
draftSave,
|
|
6521
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
6522
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
6523
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
6524
|
+
evidence: candidate.evidence ?? [],
|
|
6525
|
+
};
|
|
6526
|
+
}
|
|
6527
|
+
catch (error) {
|
|
6528
|
+
return {
|
|
6529
|
+
...candidate,
|
|
6530
|
+
draftSave: { status: 'error', error: error instanceof Error ? error.message : String(error) },
|
|
6531
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
6532
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
6533
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
6534
|
+
evidence: candidate.evidence ?? [],
|
|
6535
|
+
};
|
|
6536
|
+
}
|
|
6537
|
+
}
|
|
4854
6538
|
export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
4855
6539
|
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
4856
6540
|
const safeDomain = (options.domain ?? '')
|
|
@@ -4858,9 +6542,9 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
|
4858
6542
|
.toLowerCase()
|
|
4859
6543
|
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
4860
6544
|
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
4861
|
-
const targetRelativePath = `blocks/${safeDomain}/${slug}.dql`;
|
|
4862
|
-
const targetPath = join(projectRoot, targetRelativePath);
|
|
4863
6545
|
const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
|
|
6546
|
+
const targetRelativePath = canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath);
|
|
6547
|
+
const targetPath = join(projectRoot, targetRelativePath);
|
|
4864
6548
|
if (existsSync(targetPath) && previousPath !== targetRelativePath) {
|
|
4865
6549
|
throw new Error('BLOCK_EXISTS');
|
|
4866
6550
|
}
|
|
@@ -4891,12 +6575,103 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
|
4891
6575
|
}
|
|
4892
6576
|
return targetRelativePath;
|
|
4893
6577
|
}
|
|
6578
|
+
export function saveBlockStudioDraftArtifacts(projectRoot, options) {
|
|
6579
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
6580
|
+
const safeDomain = (options.domain ?? '')
|
|
6581
|
+
.trim()
|
|
6582
|
+
.toLowerCase()
|
|
6583
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
6584
|
+
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
6585
|
+
const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
|
|
6586
|
+
const domainFirstRoot = join(projectRoot, 'domains');
|
|
6587
|
+
const useDomainFirstDrafts = existsSync(domainFirstRoot);
|
|
6588
|
+
const draftPrefix = useDomainFirstDrafts
|
|
6589
|
+
? `domains/${safeDomain}/blocks/_drafts/`
|
|
6590
|
+
: `blocks/_drafts/${safeDomain}/`;
|
|
6591
|
+
const stableSuffix = options.stableSuffix?.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 18);
|
|
6592
|
+
let targetRelativePath = previousPath && isDraftBlockPath(previousPath)
|
|
6593
|
+
? previousPath
|
|
6594
|
+
: `${draftPrefix}${slug}${stableSuffix ? `-${stableSuffix}` : ''}.dql`;
|
|
6595
|
+
let targetPath = join(projectRoot, targetRelativePath);
|
|
6596
|
+
if (!previousPath && existsSync(targetPath)) {
|
|
6597
|
+
for (let index = 2; index < 1000; index += 1) {
|
|
6598
|
+
const candidatePath = `${draftPrefix}${slug}-${index}.dql`;
|
|
6599
|
+
if (!existsSync(join(projectRoot, candidatePath))) {
|
|
6600
|
+
targetRelativePath = candidatePath;
|
|
6601
|
+
targetPath = join(projectRoot, targetRelativePath);
|
|
6602
|
+
break;
|
|
6603
|
+
}
|
|
6604
|
+
}
|
|
6605
|
+
}
|
|
6606
|
+
mkdirSync(dirname(targetPath), { recursive: true });
|
|
6607
|
+
writeFileSync(targetPath, options.source.trimEnd() + '\n', 'utf-8');
|
|
6608
|
+
const companionSlug = targetRelativePath.split('/').pop()?.replace(/\.dql$/, '') || slug;
|
|
6609
|
+
writeBlockCompanionFile(projectRoot, {
|
|
6610
|
+
slug: companionSlug,
|
|
6611
|
+
name: options.name,
|
|
6612
|
+
domain: `_drafts/${safeDomain}`,
|
|
6613
|
+
description: options.description,
|
|
6614
|
+
owner: options.owner,
|
|
6615
|
+
tags: options.tags,
|
|
6616
|
+
provider: 'dql',
|
|
6617
|
+
content: options.source,
|
|
6618
|
+
lineage: options.lineage,
|
|
6619
|
+
importMeta: options.importMeta,
|
|
6620
|
+
});
|
|
6621
|
+
return targetRelativePath;
|
|
6622
|
+
}
|
|
6623
|
+
function canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath) {
|
|
6624
|
+
const previousDomainFirst = previousPath?.match(/^domains\/([^/]+)\/blocks\/(?:_drafts\/)?[^/]+\.dql$/);
|
|
6625
|
+
if (previousDomainFirst) {
|
|
6626
|
+
return `domains/${previousDomainFirst[1]}/blocks/${slug}.dql`;
|
|
6627
|
+
}
|
|
6628
|
+
if (existsSync(join(projectRoot, 'domains', safeDomain))) {
|
|
6629
|
+
return `domains/${safeDomain}/blocks/${slug}.dql`;
|
|
6630
|
+
}
|
|
6631
|
+
return `blocks/${safeDomain}/${slug}.dql`;
|
|
6632
|
+
}
|
|
6633
|
+
function isDraftBlockPath(value) {
|
|
6634
|
+
if (!value)
|
|
6635
|
+
return false;
|
|
6636
|
+
const normalized = normalize(value).replace(/^\/+/, '');
|
|
6637
|
+
return normalized.startsWith('blocks/_drafts/') || /^domains\/[^/]+\/blocks\/_drafts\//.test(normalized);
|
|
6638
|
+
}
|
|
6639
|
+
function isBlockStudioBlockPath(value) {
|
|
6640
|
+
if (!value)
|
|
6641
|
+
return false;
|
|
6642
|
+
const normalized = normalize(value).replace(/^\/+/, '');
|
|
6643
|
+
return normalized.startsWith('blocks/') || /^domains\/[^/]+\/blocks\//.test(normalized);
|
|
6644
|
+
}
|
|
6645
|
+
function inferBlockStudioPathDomain(blockPath) {
|
|
6646
|
+
const normalized = normalize(blockPath).replace(/^\/+/, '');
|
|
6647
|
+
if (normalized.startsWith('blocks/')) {
|
|
6648
|
+
return normalized.split('/').slice(1, -1).join('/');
|
|
6649
|
+
}
|
|
6650
|
+
const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)$/);
|
|
6651
|
+
if (!domainFirst)
|
|
6652
|
+
return '';
|
|
6653
|
+
const domain = domainFirst[1];
|
|
6654
|
+
const blockSubpath = domainFirst[2];
|
|
6655
|
+
if (blockSubpath.startsWith('_drafts/'))
|
|
6656
|
+
return `_drafts/${domain}`;
|
|
6657
|
+
return domain;
|
|
6658
|
+
}
|
|
4894
6659
|
function blockCompanionRelativePath(blockPath) {
|
|
4895
6660
|
const normalized = normalize(blockPath).replace(/^\/+/, '');
|
|
4896
|
-
if (
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
6661
|
+
if (normalized.startsWith('blocks/')) {
|
|
6662
|
+
const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
|
|
6663
|
+
return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
|
|
6664
|
+
}
|
|
6665
|
+
const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)\.dql$/);
|
|
6666
|
+
if (domainFirst) {
|
|
6667
|
+
const domain = domainFirst[1];
|
|
6668
|
+
const blockPath = domainFirst[2];
|
|
6669
|
+
const companionBlockPath = blockPath.startsWith('_drafts/')
|
|
6670
|
+
? join('_drafts', domain, blockPath.slice('_drafts/'.length))
|
|
6671
|
+
: join(domain, blockPath);
|
|
6672
|
+
return join('semantic-layer', 'blocks', `${companionBlockPath}.yaml`).replaceAll('\\', '/');
|
|
6673
|
+
}
|
|
6674
|
+
return null;
|
|
4900
6675
|
}
|
|
4901
6676
|
function readBlockCompanionFile(projectRoot, relativePath) {
|
|
4902
6677
|
const absPath = join(projectRoot, relativePath);
|
|
@@ -4944,18 +6719,48 @@ function readBlockCompanionFile(projectRoot, relativePath) {
|
|
|
4944
6719
|
return null;
|
|
4945
6720
|
}
|
|
4946
6721
|
}
|
|
4947
|
-
function parseBlockSourceMetadata(source) {
|
|
6722
|
+
export function parseBlockSourceMetadata(source) {
|
|
4948
6723
|
const name = source.match(/^\s*block\s+"([^"]+)"/i)?.[1] ?? '';
|
|
4949
6724
|
const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
|
|
4950
|
-
const
|
|
6725
|
+
const extractStringArray = (key) => {
|
|
6726
|
+
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'i'));
|
|
6727
|
+
return match ? (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [];
|
|
6728
|
+
};
|
|
6729
|
+
const extractStringMapSection = (key) => {
|
|
6730
|
+
const match = source.match(new RegExp(`\\b${key}\\s*\\{([\\s\\S]*?)\\n\\s*\\}`, 'i'));
|
|
6731
|
+
if (!match)
|
|
6732
|
+
return [];
|
|
6733
|
+
return match[1]
|
|
6734
|
+
.split(/\r?\n/)
|
|
6735
|
+
.map((line) => line.trim())
|
|
6736
|
+
.map((line) => line.match(/^([A-Za-z_][\w.]*)\s*=\s*"([^"]*)"\s*$/))
|
|
6737
|
+
.filter((entry) => Boolean(entry))
|
|
6738
|
+
.map((entry) => ({ key: entry[1], value: entry[2] }));
|
|
6739
|
+
};
|
|
6740
|
+
const parameterPolicy = extractStringMapSection('parameterPolicy')
|
|
6741
|
+
.map((entry) => ({ name: entry.key, policy: entry.value }));
|
|
6742
|
+
const filterBindings = extractStringMapSection('filterBindings')
|
|
6743
|
+
.map((entry) => ({ filter: entry.key, binding: entry.value }));
|
|
4951
6744
|
return {
|
|
4952
6745
|
name,
|
|
4953
6746
|
domain: extractString('domain'),
|
|
4954
6747
|
description: extractString('description'),
|
|
4955
6748
|
owner: extractString('owner'),
|
|
4956
|
-
tags:
|
|
6749
|
+
tags: extractStringArray('tags'),
|
|
4957
6750
|
status: extractString('status') || 'draft',
|
|
4958
6751
|
blockType: extractString('type') || 'custom',
|
|
6752
|
+
llmContext: extractString('llmContext'),
|
|
6753
|
+
pattern: extractString('pattern'),
|
|
6754
|
+
grain: extractString('grain'),
|
|
6755
|
+
entities: extractStringArray('entities'),
|
|
6756
|
+
outputs: extractStringArray('outputs'),
|
|
6757
|
+
dimensions: extractStringArray('dimensions'),
|
|
6758
|
+
allowedFilters: extractStringArray('allowedFilters'),
|
|
6759
|
+
parameterPolicy,
|
|
6760
|
+
filterBindings,
|
|
6761
|
+
sourceSystems: extractStringArray('sourceSystems'),
|
|
6762
|
+
replacementFor: extractStringArray('replacementFor'),
|
|
6763
|
+
reviewCadence: extractString('reviewCadence'),
|
|
4959
6764
|
};
|
|
4960
6765
|
}
|
|
4961
6766
|
function compareBlockStudioValues(actual, operator, expected) {
|
|
@@ -5012,58 +6817,948 @@ function buildBlockStudioCertificationChecklist(input) {
|
|
|
5012
6817
|
if (!input.validation.chartConfig?.chart)
|
|
5013
6818
|
blockers.add('Visualization config is missing');
|
|
5014
6819
|
return {
|
|
5015
|
-
metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
|
|
5016
|
-
validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
5017
|
-
run: input.previewSucceeded,
|
|
5018
|
-
tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
|
|
5019
|
-
chart: Boolean(input.validation.chartConfig?.chart),
|
|
5020
|
-
lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
|
|
5021
|
-
aiReviewed: true,
|
|
5022
|
-
blockers: Array.from(blockers),
|
|
5023
|
-
checkedAt: new Date().toISOString(),
|
|
6820
|
+
metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
|
|
6821
|
+
validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
6822
|
+
run: input.previewSucceeded,
|
|
6823
|
+
tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
|
|
6824
|
+
chart: Boolean(input.validation.chartConfig?.chart),
|
|
6825
|
+
lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
|
|
6826
|
+
aiReviewed: true,
|
|
6827
|
+
blockers: Array.from(blockers),
|
|
6828
|
+
checkedAt: new Date().toISOString(),
|
|
6829
|
+
};
|
|
6830
|
+
}
|
|
6831
|
+
function extractSqlTablesLight(sql) {
|
|
6832
|
+
const tables = new Set();
|
|
6833
|
+
const cleaned = sql
|
|
6834
|
+
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
6835
|
+
.replace(/--[^\n\r]*/g, ' ')
|
|
6836
|
+
.replace(/extract\s*\(\s*\w+\s+from\s+[^)]+\)/gi, 'EXTRACT_VALUE');
|
|
6837
|
+
const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
|
|
6838
|
+
let match;
|
|
6839
|
+
while ((match = regex.exec(cleaned))) {
|
|
6840
|
+
const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
|
|
6841
|
+
if (raw && !raw.startsWith('(') && !/^(select|values|unnest|lateral)$/i.test(raw))
|
|
6842
|
+
tables.add(raw);
|
|
6843
|
+
}
|
|
6844
|
+
return Array.from(tables);
|
|
6845
|
+
}
|
|
6846
|
+
function setBlockStudioStatusInSource(source, newStatus) {
|
|
6847
|
+
if (/status\s*=\s*"[^"]*"/.test(source)) {
|
|
6848
|
+
return source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
|
|
6849
|
+
}
|
|
6850
|
+
return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
|
|
6851
|
+
}
|
|
6852
|
+
export function setBlockStudioStatus(projectRoot, blockPath, newStatus) {
|
|
6853
|
+
const normalizedPath = normalize(blockPath).replace(/^\/+/, '');
|
|
6854
|
+
if (!isBlockStudioBlockPath(normalizedPath))
|
|
6855
|
+
throw new Error('Invalid block path');
|
|
6856
|
+
const absPath = join(projectRoot, normalizedPath);
|
|
6857
|
+
if (!existsSync(absPath))
|
|
6858
|
+
throw new Error('Block file not found');
|
|
6859
|
+
const source = setBlockStudioStatusInSource(readFileSync(absPath, 'utf-8'), newStatus);
|
|
6860
|
+
writeFileSync(absPath, source, 'utf-8');
|
|
6861
|
+
const companionPath = blockCompanionRelativePath(normalizedPath);
|
|
6862
|
+
if (!companionPath)
|
|
6863
|
+
return;
|
|
6864
|
+
const absCompanionPath = join(projectRoot, companionPath);
|
|
6865
|
+
if (!existsSync(absCompanionPath))
|
|
6866
|
+
return;
|
|
6867
|
+
let companion = readFileSync(absCompanionPath, 'utf-8');
|
|
6868
|
+
if (/^reviewStatus:\s*.+$/m.test(companion)) {
|
|
6869
|
+
companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
|
|
6870
|
+
}
|
|
6871
|
+
else {
|
|
6872
|
+
companion = `${companion.trimEnd()}\nreviewStatus: ${newStatus}\n`;
|
|
6873
|
+
}
|
|
6874
|
+
writeFileSync(absCompanionPath, companion, 'utf-8');
|
|
6875
|
+
}
|
|
6876
|
+
async function buildDqlGenerationContextPack(projectRoot, candidate) {
|
|
6877
|
+
const tables = candidate.lineage.sourceTables.join(', ');
|
|
6878
|
+
const question = [
|
|
6879
|
+
`Generate a governed DQL block draft for SQL import candidate "${candidate.name}".`,
|
|
6880
|
+
candidate.description,
|
|
6881
|
+
tables ? `Source tables: ${tables}.` : '',
|
|
6882
|
+
candidate.sql.slice(0, 1200),
|
|
6883
|
+
].filter(Boolean).join('\n');
|
|
6884
|
+
return buildLocalContextPack(projectRoot, {
|
|
6885
|
+
question,
|
|
6886
|
+
mode: 'build',
|
|
6887
|
+
surface: 'block-studio',
|
|
6888
|
+
limit: 80,
|
|
6889
|
+
objectTypes: [
|
|
6890
|
+
'dql_block',
|
|
6891
|
+
'dql_term',
|
|
6892
|
+
'business_view',
|
|
6893
|
+
'domain',
|
|
6894
|
+
'semantic_metric',
|
|
6895
|
+
'semantic_model',
|
|
6896
|
+
'semantic_dimension',
|
|
6897
|
+
'semantic_measure',
|
|
6898
|
+
'dbt_model',
|
|
6899
|
+
'dbt_source',
|
|
6900
|
+
'dbt_column',
|
|
6901
|
+
'warehouse_table',
|
|
6902
|
+
'datalex_domain',
|
|
6903
|
+
'datalex_entity',
|
|
6904
|
+
'datalex_contract',
|
|
6905
|
+
'datalex_term',
|
|
6906
|
+
],
|
|
6907
|
+
strictness: 'balanced',
|
|
6908
|
+
});
|
|
6909
|
+
}
|
|
6910
|
+
function dqlGenerationEvidenceFromContext(contextPack, candidate) {
|
|
6911
|
+
const directEvidence = contextPack.objects
|
|
6912
|
+
.filter((object) => isDqlGenerationEvidenceObject(object))
|
|
6913
|
+
.filter((object) => metadataObjectMatchesImportedSource(object, candidate.lineage.sourceTables))
|
|
6914
|
+
.map((object) => metadataObjectToDqlGenerationEvidence(object, 'Directly matches an imported SQL source table.'));
|
|
6915
|
+
const selected = contextPack.retrievalDiagnostics.selectedEvidence.slice(0, 12);
|
|
6916
|
+
const byKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
|
|
6917
|
+
const evidence = selected.flatMap((item) => {
|
|
6918
|
+
const object = byKey.get(item.objectKey);
|
|
6919
|
+
if (!object || !isDqlGenerationEvidenceObject(object))
|
|
6920
|
+
return [];
|
|
6921
|
+
return [metadataObjectToDqlGenerationEvidence(object, item.reason, Math.max(0.35, Math.min(0.98, item.score / Math.max(item.score, 10))))];
|
|
6922
|
+
});
|
|
6923
|
+
const ranked = uniqueDqlGenerationEvidence([...directEvidence, ...evidence]).slice(0, 12);
|
|
6924
|
+
return ranked.length > 0 ? ranked : deterministicDqlGenerationEvidence(candidate);
|
|
6925
|
+
}
|
|
6926
|
+
function deterministicDqlGenerationEvidence(candidate) {
|
|
6927
|
+
return candidate.lineage.sourceTables.map((table) => ({
|
|
6928
|
+
kind: 'warehouse_table',
|
|
6929
|
+
name: table,
|
|
6930
|
+
source: candidate.sourcePath,
|
|
6931
|
+
reason: 'Detected in the imported SQL source.',
|
|
6932
|
+
confidence: 0.7,
|
|
6933
|
+
}));
|
|
6934
|
+
}
|
|
6935
|
+
function isDqlGenerationEvidenceObject(object) {
|
|
6936
|
+
if (object.objectType === 'dql_block') {
|
|
6937
|
+
if (object.status !== 'certified')
|
|
6938
|
+
return false;
|
|
6939
|
+
if (object.sourcePath?.includes('/_drafts/') || object.sourcePath?.includes('\\_drafts\\'))
|
|
6940
|
+
return false;
|
|
6941
|
+
}
|
|
6942
|
+
return true;
|
|
6943
|
+
}
|
|
6944
|
+
function metadataObjectToDqlGenerationEvidence(object, reason, confidence = 0.82) {
|
|
6945
|
+
return {
|
|
6946
|
+
kind: dqlGenerationEvidenceKind(object.objectType),
|
|
6947
|
+
name: object.fullName ?? object.name,
|
|
6948
|
+
description: object.description,
|
|
6949
|
+
objectKey: object.objectKey,
|
|
6950
|
+
source: object.sourcePath ?? object.sourceSystem,
|
|
6951
|
+
reason,
|
|
6952
|
+
confidence,
|
|
6953
|
+
};
|
|
6954
|
+
}
|
|
6955
|
+
function metadataObjectMatchesImportedSource(object, sourceTables) {
|
|
6956
|
+
if (sourceTables.length === 0)
|
|
6957
|
+
return false;
|
|
6958
|
+
const haystack = [
|
|
6959
|
+
object.fullName,
|
|
6960
|
+
object.name,
|
|
6961
|
+
object.objectKey,
|
|
6962
|
+
object.sourcePath,
|
|
6963
|
+
object.sourceSystem,
|
|
6964
|
+
typeof object.payload?.relation === 'string' ? object.payload.relation : '',
|
|
6965
|
+
typeof object.payload?.relationName === 'string' ? object.payload.relationName : '',
|
|
6966
|
+
typeof object.payload?.relation_name === 'string' ? object.payload.relation_name : '',
|
|
6967
|
+
].filter(Boolean).join(' ').toLowerCase();
|
|
6968
|
+
return sourceTables.some((table) => relationLookupTokens(table).some((token) => token && haystack.includes(token)));
|
|
6969
|
+
}
|
|
6970
|
+
function relationLookupTokens(relation) {
|
|
6971
|
+
const normalized = relation.replace(/[`"[\]]/g, '').toLowerCase();
|
|
6972
|
+
const parts = normalized.split('.').filter(Boolean);
|
|
6973
|
+
return Array.from(new Set([
|
|
6974
|
+
normalized,
|
|
6975
|
+
parts.slice(-2).join('.'),
|
|
6976
|
+
parts.at(-1) ?? '',
|
|
6977
|
+
].filter((item) => item.length >= 3)));
|
|
6978
|
+
}
|
|
6979
|
+
function uniqueDqlGenerationEvidence(evidence) {
|
|
6980
|
+
const seen = new Set();
|
|
6981
|
+
const result = [];
|
|
6982
|
+
for (const item of evidence) {
|
|
6983
|
+
const key = item.objectKey || `${item.kind}:${item.name}`.toLowerCase();
|
|
6984
|
+
if (seen.has(key))
|
|
6985
|
+
continue;
|
|
6986
|
+
seen.add(key);
|
|
6987
|
+
result.push(item);
|
|
6988
|
+
}
|
|
6989
|
+
return result;
|
|
6990
|
+
}
|
|
6991
|
+
function dqlGenerationEvidenceKind(objectType) {
|
|
6992
|
+
if (objectType === 'dql_block')
|
|
6993
|
+
return 'dql_block';
|
|
6994
|
+
if (objectType === 'dql_term')
|
|
6995
|
+
return 'dql_term';
|
|
6996
|
+
if (objectType === 'business_view')
|
|
6997
|
+
return 'business_view';
|
|
6998
|
+
if (objectType === 'domain')
|
|
6999
|
+
return 'domain';
|
|
7000
|
+
if (objectType === 'semantic_metric')
|
|
7001
|
+
return 'semantic_metric';
|
|
7002
|
+
if (objectType === 'semantic_model')
|
|
7003
|
+
return 'semantic_model';
|
|
7004
|
+
if (objectType === 'dbt_model' || objectType === 'dbt_source' || objectType === 'dbt_column')
|
|
7005
|
+
return 'dbt_model';
|
|
7006
|
+
if (objectType === 'warehouse_table')
|
|
7007
|
+
return 'warehouse_table';
|
|
7008
|
+
if (objectType === 'datalex_contract')
|
|
7009
|
+
return 'datalex_contract';
|
|
7010
|
+
if (objectType === 'datalex_entity')
|
|
7011
|
+
return 'datalex_entity';
|
|
7012
|
+
if (objectType === 'datalex_domain')
|
|
7013
|
+
return 'datalex_domain';
|
|
7014
|
+
if (objectType === 'datalex_term')
|
|
7015
|
+
return 'datalex_term';
|
|
7016
|
+
if (objectType.includes('lineage'))
|
|
7017
|
+
return 'lineage';
|
|
7018
|
+
return 'metadata';
|
|
7019
|
+
}
|
|
7020
|
+
function deterministicDqlGenerationPatch(candidate, evidence) {
|
|
7021
|
+
const domain = inferDqlGenerationDomain(candidate);
|
|
7022
|
+
const parameterized = parameterizeSqlForDqlImport(candidate.sql);
|
|
7023
|
+
const grain = extractDqlGenerationGroupByFields(parameterized.sql)[0];
|
|
7024
|
+
const outputs = extractDqlGenerationSelectOutputs(parameterized.sql);
|
|
7025
|
+
const sourceSystems = candidate.lineage.sourceTables.map((table) => table.split('.').filter(Boolean).slice(-2, -1)[0] ?? '').filter(Boolean);
|
|
7026
|
+
const description = candidate.description && !candidate.description.startsWith('Imported from ')
|
|
7027
|
+
? candidate.description
|
|
7028
|
+
: deterministicDqlGenerationDescription(candidate, evidence);
|
|
7029
|
+
const allowedFilters = Array.from(new Set([
|
|
7030
|
+
...parameterized.allowedFilters,
|
|
7031
|
+
...extractDqlGenerationFilterFields(parameterized.sql),
|
|
7032
|
+
])).slice(0, 16);
|
|
7033
|
+
return {
|
|
7034
|
+
name: candidate.name,
|
|
7035
|
+
domain,
|
|
7036
|
+
description,
|
|
7037
|
+
owner: candidate.owner,
|
|
7038
|
+
tags: dqlGenerationBusinessTags(candidate, evidence, domain),
|
|
7039
|
+
terms: inferDqlGenerationTerms(evidence),
|
|
7040
|
+
llmContext: deterministicDqlGenerationContext({ ...candidate, sql: parameterized.sql }, evidence),
|
|
7041
|
+
pattern: inferDqlGenerationPattern(parameterized.sql),
|
|
7042
|
+
grain,
|
|
7043
|
+
entities: inferDqlGenerationEntities({
|
|
7044
|
+
grain,
|
|
7045
|
+
outputs,
|
|
7046
|
+
sourceTables: candidate.lineage.sourceTables,
|
|
7047
|
+
evidence,
|
|
7048
|
+
}),
|
|
7049
|
+
outputs,
|
|
7050
|
+
dimensions: extractDqlGenerationDimensions(parameterized.sql, grain, outputs),
|
|
7051
|
+
allowedFilters,
|
|
7052
|
+
parameterPolicy: parameterized.parameterPolicy,
|
|
7053
|
+
filterBindings: parameterized.filterBindings,
|
|
7054
|
+
parameterDecisions: parameterized.parameterDecisions,
|
|
7055
|
+
sql: parameterized.sql,
|
|
7056
|
+
warnings: parameterized.warnings,
|
|
7057
|
+
sourceSystems,
|
|
7058
|
+
reviewCadence: 'monthly',
|
|
7059
|
+
};
|
|
7060
|
+
}
|
|
7061
|
+
function inferDqlGenerationTerms(evidence) {
|
|
7062
|
+
return Array.from(new Set(evidence
|
|
7063
|
+
.filter((item) => item.kind === 'dql_term' || item.kind === 'datalex_term')
|
|
7064
|
+
.map((item) => item.name.trim())
|
|
7065
|
+
.filter(Boolean))).slice(0, 16);
|
|
7066
|
+
}
|
|
7067
|
+
function deterministicDqlGenerationDescription(candidate, evidence) {
|
|
7068
|
+
const tables = candidate.lineage.sourceTables.length > 0
|
|
7069
|
+
? candidate.lineage.sourceTables.join(', ')
|
|
7070
|
+
: candidate.sourcePath;
|
|
7071
|
+
const sourceContext = evidence.find((item) => (item.kind === 'dbt_model' || item.kind === 'semantic_model' || item.kind === 'warehouse_table') &&
|
|
7072
|
+
item.description &&
|
|
7073
|
+
candidate.lineage.sourceTables.some((table) => relationLookupTokens(table).some((token) => item.name.toLowerCase().includes(token))));
|
|
7074
|
+
const aggregate = /\b(sum|count|avg|min|max)\s*\(/i.test(candidate.sql);
|
|
7075
|
+
const grouped = /\bgroup\s+by\b/i.test(candidate.sql);
|
|
7076
|
+
const sourcePhrase = sourceContext?.description
|
|
7077
|
+
? ` using ${sourceContext.name} (${sourceContext.description})`
|
|
7078
|
+
: ` using ${tables}`;
|
|
7079
|
+
if (aggregate && grouped)
|
|
7080
|
+
return `Summarizes imported SQL${sourcePhrase}.`;
|
|
7081
|
+
if (aggregate)
|
|
7082
|
+
return `Calculates aggregate values${sourcePhrase}.`;
|
|
7083
|
+
return `Exposes imported SQL${sourcePhrase}.`;
|
|
7084
|
+
}
|
|
7085
|
+
function dqlGenerationBusinessTags(candidate, evidence, domain) {
|
|
7086
|
+
const generic = new Set(['imported', 'raw-sql', 'ai-generated']);
|
|
7087
|
+
const stopTags = new Set(['int', 'stg', 'src', 'fct', 'dim', 'name', 'id', 'key', 'model', 'source', 'table']);
|
|
7088
|
+
const sourceTokens = candidate.lineage.sourceTables
|
|
7089
|
+
.flatMap((table) => table.split('.').slice(-2))
|
|
7090
|
+
.flatMap((part) => part.split(/[_\s.-]+/));
|
|
7091
|
+
const tags = [
|
|
7092
|
+
...candidate.tags.filter((tag) => !generic.has(tag)),
|
|
7093
|
+
domain,
|
|
7094
|
+
...sourceTokens,
|
|
7095
|
+
'review-required',
|
|
7096
|
+
].map((tag) => tag.trim().toLowerCase())
|
|
7097
|
+
.filter((tag) => tag.length > 1 && !generic.has(tag) && !stopTags.has(tag))
|
|
7098
|
+
.map((tag) => tag.replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, ''))
|
|
7099
|
+
.filter(Boolean);
|
|
7100
|
+
return Array.from(new Set(tags)).slice(0, 12);
|
|
7101
|
+
}
|
|
7102
|
+
function deterministicDqlGenerationContext(candidate, evidence) {
|
|
7103
|
+
const tables = candidate.lineage.sourceTables.length > 0
|
|
7104
|
+
? candidate.lineage.sourceTables.join(', ')
|
|
7105
|
+
: 'the imported SQL source';
|
|
7106
|
+
const evidenceNames = evidence
|
|
7107
|
+
.filter((item) => item.kind !== 'dql_block')
|
|
7108
|
+
.slice(0, 3)
|
|
7109
|
+
.map((item) => item.name)
|
|
7110
|
+
.filter(Boolean);
|
|
7111
|
+
const grain = extractDqlGenerationGroupByFields(candidate.sql);
|
|
7112
|
+
const years = extractDqlGenerationYearFilters(candidate.sql);
|
|
7113
|
+
return [
|
|
7114
|
+
`Use after review for questions matching "${candidate.name}".`,
|
|
7115
|
+
grain.length > 0 ? `Grain: ${grain.join(', ')}.` : '',
|
|
7116
|
+
years.length > 0 ? `Filters: year ${years.join(', ')}.` : '',
|
|
7117
|
+
`Source: ${tables}.`,
|
|
7118
|
+
evidenceNames.length > 0 ? `Grounded by: ${evidenceNames.join(', ')}.` : '',
|
|
7119
|
+
'Certification must confirm grain, filters, owner, tags, and tests.',
|
|
7120
|
+
].filter(Boolean).join(' ');
|
|
7121
|
+
}
|
|
7122
|
+
function extractDqlGenerationGroupByFields(sql) {
|
|
7123
|
+
const match = sql.match(/\bgroup\s+by\b([\s\S]+?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i);
|
|
7124
|
+
if (!match)
|
|
7125
|
+
return [];
|
|
7126
|
+
return splitDqlGenerationSqlList(match[1])
|
|
7127
|
+
.map((item) => item.replace(/[`"[\]]/g, '').trim())
|
|
7128
|
+
.filter((item) => item && !/^\d+$/.test(item))
|
|
7129
|
+
.slice(0, 4);
|
|
7130
|
+
}
|
|
7131
|
+
function extractDqlGenerationSelectOutputs(sql) {
|
|
7132
|
+
const selectMatch = sql.match(/\bselect\b([\s\S]+?)\bfrom\b/i);
|
|
7133
|
+
if (!selectMatch)
|
|
7134
|
+
return [];
|
|
7135
|
+
return splitDqlGenerationSqlList(selectMatch[1])
|
|
7136
|
+
.map((expr) => {
|
|
7137
|
+
const alias = expr.match(/\bas\s+([A-Za-z_][A-Za-z0-9_]*)\b/i)?.[1]
|
|
7138
|
+
?? expr.match(/([A-Za-z_][A-Za-z0-9_]*)\s*$/)?.[1]
|
|
7139
|
+
?? '';
|
|
7140
|
+
return alias.replace(/[`"[\]]/g, '');
|
|
7141
|
+
})
|
|
7142
|
+
.filter(Boolean)
|
|
7143
|
+
.filter((name) => !/^(from|where|group|order|limit)$/i.test(name))
|
|
7144
|
+
.slice(0, 24);
|
|
7145
|
+
}
|
|
7146
|
+
function extractDqlGenerationDimensions(sql, grain, outputs) {
|
|
7147
|
+
const outputSet = new Set(outputs.map(normalizedTerm));
|
|
7148
|
+
return extractDqlGenerationGroupByFields(sql)
|
|
7149
|
+
.map((field) => field.split('.').pop() ?? field)
|
|
7150
|
+
.filter((field) => normalizedTerm(field) !== normalizedTerm(grain ?? ''))
|
|
7151
|
+
.filter((field) => !outputSet.has(normalizedTerm(field)))
|
|
7152
|
+
.filter((field) => !/\b(total|count|sum|avg|average|min|max|rate|pct|percent|amount|revenue|points?|score)\b/i.test(field))
|
|
7153
|
+
.slice(0, 16);
|
|
7154
|
+
}
|
|
7155
|
+
function inferDqlGenerationEntities(input) {
|
|
7156
|
+
const entities = new Set();
|
|
7157
|
+
const add = (value) => {
|
|
7158
|
+
const entity = businessEntityFromDqlGenerationIdentifier(value ?? '');
|
|
7159
|
+
if (entity)
|
|
7160
|
+
entities.add(entity);
|
|
7161
|
+
};
|
|
7162
|
+
add(input.grain);
|
|
7163
|
+
for (const output of input.outputs)
|
|
7164
|
+
add(output);
|
|
7165
|
+
for (const table of input.sourceTables)
|
|
7166
|
+
add(table.split('.').pop());
|
|
7167
|
+
for (const item of input.evidence.slice(0, 12)) {
|
|
7168
|
+
if (item.kind === 'datalex_entity' || item.kind === 'semantic_model' || item.kind === 'dbt_model' || item.kind === 'warehouse_table') {
|
|
7169
|
+
add(item.name.split('.').pop());
|
|
7170
|
+
}
|
|
7171
|
+
}
|
|
7172
|
+
return [...entities].slice(0, 8);
|
|
7173
|
+
}
|
|
7174
|
+
function businessEntityFromDqlGenerationIdentifier(value) {
|
|
7175
|
+
const raw = value
|
|
7176
|
+
.split('.').pop()
|
|
7177
|
+
?.trim()
|
|
7178
|
+
.replace(/[`"[\]]/g, '')
|
|
7179
|
+
.replace(/^(dim|fact|fct|stg|src|int)_/i, '')
|
|
7180
|
+
.replace(/_(id|key|name|code|uuid|number|num|abbreviation|abbr)$/i, '')
|
|
7181
|
+
.replace(/s$/i, '') ?? '';
|
|
7182
|
+
if (!raw)
|
|
7183
|
+
return '';
|
|
7184
|
+
const tokens = raw
|
|
7185
|
+
.split(/[^A-Za-z0-9]+/)
|
|
7186
|
+
.map((token) => token.trim().toLowerCase())
|
|
7187
|
+
.filter(Boolean)
|
|
7188
|
+
.filter((token) => !isGenericDqlGenerationEntityToken(token));
|
|
7189
|
+
if (tokens.length === 0)
|
|
7190
|
+
return '';
|
|
7191
|
+
return titleizeDqlGenerationName(tokens.join(' '));
|
|
7192
|
+
}
|
|
7193
|
+
function isGenericDqlGenerationEntityToken(token) {
|
|
7194
|
+
return new Set([
|
|
7195
|
+
'total',
|
|
7196
|
+
'count',
|
|
7197
|
+
'sum',
|
|
7198
|
+
'avg',
|
|
7199
|
+
'average',
|
|
7200
|
+
'min',
|
|
7201
|
+
'max',
|
|
7202
|
+
'metric',
|
|
7203
|
+
'measure',
|
|
7204
|
+
'value',
|
|
7205
|
+
'amount',
|
|
7206
|
+
'revenue',
|
|
7207
|
+
'point',
|
|
7208
|
+
'points',
|
|
7209
|
+
'score',
|
|
7210
|
+
'date',
|
|
7211
|
+
'day',
|
|
7212
|
+
'week',
|
|
7213
|
+
'month',
|
|
7214
|
+
'quarter',
|
|
7215
|
+
'year',
|
|
7216
|
+
'season',
|
|
7217
|
+
'period',
|
|
7218
|
+
'time',
|
|
7219
|
+
'row',
|
|
7220
|
+
'record',
|
|
7221
|
+
'detail',
|
|
7222
|
+
'stat',
|
|
7223
|
+
'stats',
|
|
7224
|
+
'analytic',
|
|
7225
|
+
'analytics',
|
|
7226
|
+
'transformed',
|
|
7227
|
+
]).has(token);
|
|
7228
|
+
}
|
|
7229
|
+
function titleizeDqlGenerationName(value) {
|
|
7230
|
+
return value
|
|
7231
|
+
.replace(/[_-]+/g, ' ')
|
|
7232
|
+
.replace(/\s+/g, ' ')
|
|
7233
|
+
.trim()
|
|
7234
|
+
.replace(/\b\w/g, (char) => char.toUpperCase());
|
|
7235
|
+
}
|
|
7236
|
+
function inferDqlGenerationPattern(sql) {
|
|
7237
|
+
const groupFields = extractDqlGenerationGroupByFields(sql);
|
|
7238
|
+
if (/@metric\s*\(/i.test(sql))
|
|
7239
|
+
return 'metric_wrapper';
|
|
7240
|
+
if (/\bjoin\b/i.test(sql)) {
|
|
7241
|
+
const systems = new Set(extractDqlGenerationSourceSystems(sql));
|
|
7242
|
+
if (systems.size > 1)
|
|
7243
|
+
return 'bridge';
|
|
7244
|
+
}
|
|
7245
|
+
if (hasDqlGenerationRankingLimit(sql))
|
|
7246
|
+
return 'ranking';
|
|
7247
|
+
if (groupFields.some((field) => /\b(date|day|week|month|quarter|year|period|time)\b/i.test(field)))
|
|
7248
|
+
return 'trend';
|
|
7249
|
+
if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
|
|
7250
|
+
return 'entity_rollup';
|
|
7251
|
+
if (!/\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql) && /\b(dim|profile|customer|account|player|product|user|entity)\b/i.test(sql)) {
|
|
7252
|
+
return 'entity_profile';
|
|
7253
|
+
}
|
|
7254
|
+
return 'custom';
|
|
7255
|
+
}
|
|
7256
|
+
function hasDqlGenerationRankingLimit(sql) {
|
|
7257
|
+
return /\border\s+by\b[\s\S]*\blimit\s+(?:\d+|\$\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}|[:?][A-Za-z_][A-Za-z0-9_]*)/i.test(sql);
|
|
7258
|
+
}
|
|
7259
|
+
function extractDqlGenerationSourceSystems(sql) {
|
|
7260
|
+
const tables = new Set();
|
|
7261
|
+
const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
|
|
7262
|
+
const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
|
|
7263
|
+
let match;
|
|
7264
|
+
while ((match = regex.exec(cleaned))) {
|
|
7265
|
+
const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
|
|
7266
|
+
const parts = raw.split('.').filter(Boolean);
|
|
7267
|
+
const system = parts.slice(-2, -1)[0] ?? parts[0];
|
|
7268
|
+
if (system)
|
|
7269
|
+
tables.add(system.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, '-'));
|
|
7270
|
+
}
|
|
7271
|
+
return [...tables].filter(Boolean).slice(0, 12);
|
|
7272
|
+
}
|
|
7273
|
+
function extractDqlGenerationFilterFields(sql) {
|
|
7274
|
+
const filters = new Set();
|
|
7275
|
+
const where = sql.match(/\bwhere\b([\s\S]+?)(?:\bgroup\s+by\b|\border\s+by\b|\blimit\b|$)/i)?.[1] ?? '';
|
|
7276
|
+
const addFilter = (value) => {
|
|
7277
|
+
const name = value?.split('.').pop()?.replace(/[`"[\]]/g, '');
|
|
7278
|
+
if (name && !/^(and|or|not|null|year|month|day)$/i.test(name))
|
|
7279
|
+
filters.add(name);
|
|
7280
|
+
};
|
|
7281
|
+
const extractRegex = /\bextract\s*\([^)]*\bfrom\s+([A-Za-z_][A-Za-z0-9_.]*)\s*\)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
|
|
7282
|
+
const isNullRegex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s+is\s+(?:not\s+)?null\b/gi;
|
|
7283
|
+
const regex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
|
|
7284
|
+
let match;
|
|
7285
|
+
while ((match = extractRegex.exec(where)))
|
|
7286
|
+
addFilter(match[1]);
|
|
7287
|
+
while ((match = isNullRegex.exec(where)))
|
|
7288
|
+
addFilter(match[1]);
|
|
7289
|
+
while ((match = regex.exec(where))) {
|
|
7290
|
+
addFilter(match[1]);
|
|
7291
|
+
}
|
|
7292
|
+
return [...filters].slice(0, 16);
|
|
7293
|
+
}
|
|
7294
|
+
function extractDqlGenerationYearFilters(sql) {
|
|
7295
|
+
const years = new Set();
|
|
7296
|
+
const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
|
|
7297
|
+
let match;
|
|
7298
|
+
const extractYear = /extract\s*\(\s*year\s+from\s+[^)]+\)\s*(?:=\s*([12][0-9]{3})|in\s*\(([^)]*)\))/gi;
|
|
7299
|
+
while ((match = extractYear.exec(cleaned))) {
|
|
7300
|
+
for (const year of (match[1] || match[2] || '').match(/[12][0-9]{3}/g) ?? [])
|
|
7301
|
+
years.add(year);
|
|
7302
|
+
}
|
|
7303
|
+
return Array.from(years).sort();
|
|
7304
|
+
}
|
|
7305
|
+
function splitDqlGenerationSqlList(value) {
|
|
7306
|
+
const parts = [];
|
|
7307
|
+
let start = 0;
|
|
7308
|
+
let depth = 0;
|
|
7309
|
+
let single = false;
|
|
7310
|
+
let double = false;
|
|
7311
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
7312
|
+
const char = value[i];
|
|
7313
|
+
if (!double && char === "'" && value[i - 1] !== '\\')
|
|
7314
|
+
single = !single;
|
|
7315
|
+
else if (!single && char === '"' && value[i - 1] !== '\\')
|
|
7316
|
+
double = !double;
|
|
7317
|
+
else if (!single && !double && char === '(')
|
|
7318
|
+
depth += 1;
|
|
7319
|
+
else if (!single && !double && char === ')' && depth > 0)
|
|
7320
|
+
depth -= 1;
|
|
7321
|
+
else if (!single && !double && depth === 0 && char === ',') {
|
|
7322
|
+
parts.push(value.slice(start, i));
|
|
7323
|
+
start = i + 1;
|
|
7324
|
+
}
|
|
7325
|
+
}
|
|
7326
|
+
parts.push(value.slice(start));
|
|
7327
|
+
return parts.map((part) => part.trim()).filter(Boolean);
|
|
7328
|
+
}
|
|
7329
|
+
function inferDqlGenerationDomain(candidate) {
|
|
7330
|
+
const current = candidate.domain?.trim();
|
|
7331
|
+
if (current && current !== 'imported' && current !== 'general')
|
|
7332
|
+
return current;
|
|
7333
|
+
const firstTable = candidate.lineage.sourceTables[0];
|
|
7334
|
+
const schema = firstTable?.split('.').filter(Boolean).slice(-2, -1)[0];
|
|
7335
|
+
return schema?.replace(/[^a-z0-9/_-]+/gi, '-').toLowerCase() || current || 'imported';
|
|
7336
|
+
}
|
|
7337
|
+
async function buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack) {
|
|
7338
|
+
const payload = {
|
|
7339
|
+
candidate: {
|
|
7340
|
+
name: candidate.name,
|
|
7341
|
+
domain: candidate.domain,
|
|
7342
|
+
description: candidate.description,
|
|
7343
|
+
owner: candidate.owner,
|
|
7344
|
+
tags: candidate.tags,
|
|
7345
|
+
sql: candidate.sql,
|
|
7346
|
+
sourceTables: candidate.lineage.sourceTables,
|
|
7347
|
+
parameters: candidate.lineage.parameters,
|
|
7348
|
+
warnings: candidate.warnings ?? candidate.lineage.warnings,
|
|
7349
|
+
grain: extractDqlGenerationGroupByFields(candidate.sql),
|
|
7350
|
+
yearFilters: extractDqlGenerationYearFilters(candidate.sql),
|
|
7351
|
+
parameterization: parameterizeSqlForDqlImport(candidate.sql),
|
|
7352
|
+
},
|
|
7353
|
+
context: {
|
|
7354
|
+
evidence,
|
|
7355
|
+
selectedObjects: contextPack?.objects.slice(0, 24).map((object) => ({
|
|
7356
|
+
objectKey: object.objectKey,
|
|
7357
|
+
objectType: object.objectType,
|
|
7358
|
+
name: object.fullName ?? object.name,
|
|
7359
|
+
domain: object.domain,
|
|
7360
|
+
status: object.status,
|
|
7361
|
+
description: object.description,
|
|
7362
|
+
})) ?? [],
|
|
7363
|
+
allowedRelations: contextPack?.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
|
|
7364
|
+
relation: relation.relation,
|
|
7365
|
+
source: relation.source,
|
|
7366
|
+
columns: relation.columns.slice(0, 16).map((column) => column.name),
|
|
7367
|
+
})) ?? [],
|
|
7368
|
+
},
|
|
7369
|
+
};
|
|
7370
|
+
const text = await provider.generate([
|
|
7371
|
+
{
|
|
7372
|
+
role: 'system',
|
|
7373
|
+
content: [
|
|
7374
|
+
'You generate DQL Block Studio metadata for a local draft.',
|
|
7375
|
+
'Return only a compact JSON object with optional keys: name, domain, description, owner, tags, terms, llmContext, pattern, grain, entities, outputs, dimensions, allowedFilters, sourceSystems, replacementFor, reviewCadence.',
|
|
7376
|
+
'Do not return markdown. Do not mark the block certified. Do not change SQL.',
|
|
7377
|
+
'Use only directly relevant dbt, semantic, warehouse, certified block, and SQL-shape evidence from the payload.',
|
|
7378
|
+
'terms must reference existing DQL/DataLex glossary terms from the payload. Do not invent term names.',
|
|
7379
|
+
'If metadata descriptions are missing, describe the observable SQL intent instead of inventing business meaning.',
|
|
7380
|
+
'Descriptions and llmContext must be specific, business-readable, concise, and review-required.',
|
|
7381
|
+
'reviewCadence should usually be monthly or quarterly unless evidence strongly suggests a different cadence.',
|
|
7382
|
+
'Tags must be business/search tags. Avoid generic tags such as raw-sql, imported, ai-generated.',
|
|
7383
|
+
].join('\n'),
|
|
7384
|
+
},
|
|
7385
|
+
{ role: 'user', content: JSON.stringify(payload) },
|
|
7386
|
+
], { maxTokens: 900, temperature: 0.1 });
|
|
7387
|
+
return normalizeDqlGenerationPatch(parseFirstJsonObject(text));
|
|
7388
|
+
}
|
|
7389
|
+
function parseFirstJsonObject(text) {
|
|
7390
|
+
const trimmed = text.trim();
|
|
7391
|
+
if (trimmed.startsWith('{') && trimmed.endsWith('}'))
|
|
7392
|
+
return JSON.parse(trimmed);
|
|
7393
|
+
const match = trimmed.match(/\{[\s\S]*\}/);
|
|
7394
|
+
if (!match)
|
|
7395
|
+
return null;
|
|
7396
|
+
return JSON.parse(match[0]);
|
|
7397
|
+
}
|
|
7398
|
+
function normalizeDqlGenerationPatch(value) {
|
|
7399
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7400
|
+
return null;
|
|
7401
|
+
const record = value;
|
|
7402
|
+
const patch = {};
|
|
7403
|
+
if (typeof record.name === 'string' && record.name.trim())
|
|
7404
|
+
patch.name = record.name.trim().slice(0, 120);
|
|
7405
|
+
if (typeof record.domain === 'string' && record.domain.trim())
|
|
7406
|
+
patch.domain = record.domain.trim().slice(0, 80);
|
|
7407
|
+
if (typeof record.description === 'string' && record.description.trim())
|
|
7408
|
+
patch.description = record.description.trim().slice(0, 500);
|
|
7409
|
+
if (typeof record.owner === 'string' && record.owner.trim())
|
|
7410
|
+
patch.owner = record.owner.trim().slice(0, 120);
|
|
7411
|
+
if (Array.isArray(record.tags))
|
|
7412
|
+
patch.tags = normalizeDqlGenerationStringArray(record.tags, 12);
|
|
7413
|
+
if (Array.isArray(record.terms))
|
|
7414
|
+
patch.terms = normalizeDqlGenerationStringArray(record.terms, 16);
|
|
7415
|
+
if (typeof record.llmContext === 'string' && record.llmContext.trim())
|
|
7416
|
+
patch.llmContext = record.llmContext.trim().slice(0, 1000);
|
|
7417
|
+
if (typeof record.pattern === 'string' && record.pattern.trim())
|
|
7418
|
+
patch.pattern = normalizeDqlGenerationPattern(record.pattern);
|
|
7419
|
+
if (typeof record.grain === 'string' && record.grain.trim())
|
|
7420
|
+
patch.grain = record.grain.trim().slice(0, 120);
|
|
7421
|
+
if (Array.isArray(record.entities))
|
|
7422
|
+
patch.entities = normalizeDqlGenerationStringArray(record.entities, 12);
|
|
7423
|
+
if (Array.isArray(record.outputs))
|
|
7424
|
+
patch.outputs = normalizeDqlGenerationStringArray(record.outputs, 24);
|
|
7425
|
+
if (Array.isArray(record.dimensions))
|
|
7426
|
+
patch.dimensions = normalizeDqlGenerationStringArray(record.dimensions, 16);
|
|
7427
|
+
if (Array.isArray(record.allowedFilters))
|
|
7428
|
+
patch.allowedFilters = normalizeDqlGenerationStringArray(record.allowedFilters, 16);
|
|
7429
|
+
if (Array.isArray(record.sourceSystems))
|
|
7430
|
+
patch.sourceSystems = normalizeDqlGenerationStringArray(record.sourceSystems, 12);
|
|
7431
|
+
if (Array.isArray(record.replacementFor))
|
|
7432
|
+
patch.replacementFor = normalizeDqlGenerationStringArray(record.replacementFor, 12);
|
|
7433
|
+
if (typeof record.reviewCadence === 'string' && record.reviewCadence.trim()) {
|
|
7434
|
+
const reviewCadence = normalizeDqlGenerationReviewCadence(record.reviewCadence);
|
|
7435
|
+
if (reviewCadence)
|
|
7436
|
+
patch.reviewCadence = reviewCadence;
|
|
7437
|
+
}
|
|
7438
|
+
return Object.keys(patch).length > 0 ? patch : null;
|
|
7439
|
+
}
|
|
7440
|
+
function normalizeDqlGenerationStringArray(value, limit) {
|
|
7441
|
+
const result = [];
|
|
7442
|
+
for (const item of value) {
|
|
7443
|
+
const normalized = normalizeDqlGenerationStringItem(item);
|
|
7444
|
+
if (normalized)
|
|
7445
|
+
result.push(normalized);
|
|
7446
|
+
}
|
|
7447
|
+
return Array.from(new Set(result)).slice(0, limit);
|
|
7448
|
+
}
|
|
7449
|
+
function normalizeDqlGenerationStringItem(value) {
|
|
7450
|
+
if (typeof value === 'string')
|
|
7451
|
+
return value.trim();
|
|
7452
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7453
|
+
return '';
|
|
7454
|
+
const record = value;
|
|
7455
|
+
for (const key of ['name', 'field', 'column', 'id', 'value', 'label', 'sourceSystem', 'source_system']) {
|
|
7456
|
+
const nested = record[key];
|
|
7457
|
+
if (typeof nested === 'string' && nested.trim())
|
|
7458
|
+
return nested.trim();
|
|
7459
|
+
}
|
|
7460
|
+
return '';
|
|
7461
|
+
}
|
|
7462
|
+
function normalizeDqlGenerationPattern(value) {
|
|
7463
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
7464
|
+
const allowed = new Set([
|
|
7465
|
+
'metric_wrapper',
|
|
7466
|
+
'entity_profile',
|
|
7467
|
+
'entity_rollup',
|
|
7468
|
+
'ranking',
|
|
7469
|
+
'trend',
|
|
7470
|
+
'bridge',
|
|
7471
|
+
'drilldown',
|
|
7472
|
+
'replacement',
|
|
7473
|
+
'custom',
|
|
7474
|
+
]);
|
|
7475
|
+
return allowed.has(normalized) ? normalized : undefined;
|
|
7476
|
+
}
|
|
7477
|
+
function normalizeDqlGenerationReviewCadence(value) {
|
|
7478
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_ -]+/g, '').replace(/\s+/g, '_');
|
|
7479
|
+
const allowed = new Set(['daily', 'weekly', 'biweekly', 'monthly', 'quarterly', 'semiannual', 'annual']);
|
|
7480
|
+
return allowed.has(normalized) ? normalized : undefined;
|
|
7481
|
+
}
|
|
7482
|
+
function mergeGroundedDqlGenerationTerms(baseTerms, overrideTerms, evidence) {
|
|
7483
|
+
const grounded = new Map();
|
|
7484
|
+
for (const item of evidence) {
|
|
7485
|
+
if (item.kind !== 'dql_term' && item.kind !== 'datalex_term')
|
|
7486
|
+
continue;
|
|
7487
|
+
const normalized = normalizedTerm(item.name);
|
|
7488
|
+
if (normalized)
|
|
7489
|
+
grounded.set(normalized, item.name.trim());
|
|
7490
|
+
}
|
|
7491
|
+
const merged = [...(baseTerms ?? []), ...(overrideTerms ?? [])]
|
|
7492
|
+
.map((term) => grounded.get(normalizedTerm(term)))
|
|
7493
|
+
.filter((term) => Boolean(term));
|
|
7494
|
+
const result = Array.from(new Set(merged)).slice(0, 16);
|
|
7495
|
+
return result.length > 0 ? result : undefined;
|
|
7496
|
+
}
|
|
7497
|
+
function mergeDqlGenerationPatch(base, override, candidate, evidence) {
|
|
7498
|
+
const name = override.name && !isWeakDqlGenerationName(override.name) ? override.name : base.name;
|
|
7499
|
+
const description = override.description
|
|
7500
|
+
&& !isWeakDqlGenerationDescription(override.description)
|
|
7501
|
+
&& !isUngroundedDqlGenerationDescription(override.description, candidate, evidence)
|
|
7502
|
+
? override.description
|
|
7503
|
+
: base.description;
|
|
7504
|
+
const llmContext = override.llmContext
|
|
7505
|
+
&& !isUngroundedDqlGenerationDescription(override.llmContext, candidate, evidence)
|
|
7506
|
+
? override.llmContext
|
|
7507
|
+
: base.llmContext;
|
|
7508
|
+
const overrideTags = override.tags
|
|
7509
|
+
? override.tags.filter((tag) => !/^(raw-sql|imported|ai-generated)$/i.test(tag))
|
|
7510
|
+
: [];
|
|
7511
|
+
return {
|
|
7512
|
+
...base,
|
|
7513
|
+
...override,
|
|
7514
|
+
name,
|
|
7515
|
+
description,
|
|
7516
|
+
llmContext,
|
|
7517
|
+
tags: Array.from(new Set([...(base.tags ?? []), ...overrideTags])).slice(0, 12),
|
|
7518
|
+
terms: mergeGroundedDqlGenerationTerms(base.terms, override.terms, evidence),
|
|
7519
|
+
pattern: override.pattern ?? base.pattern,
|
|
7520
|
+
grain: override.grain ?? base.grain,
|
|
7521
|
+
entities: override.entities?.length ? override.entities : base.entities,
|
|
7522
|
+
outputs: override.outputs?.length ? override.outputs : base.outputs,
|
|
7523
|
+
dimensions: override.dimensions?.length ? override.dimensions : base.dimensions,
|
|
7524
|
+
allowedFilters: override.allowedFilters?.length ? override.allowedFilters : base.allowedFilters,
|
|
7525
|
+
parameterPolicy: base.parameterPolicy,
|
|
7526
|
+
filterBindings: base.filterBindings,
|
|
7527
|
+
parameterDecisions: base.parameterDecisions,
|
|
7528
|
+
similarityMatches: base.similarityMatches,
|
|
7529
|
+
recommendedAction: base.recommendedAction,
|
|
7530
|
+
sql: base.sql,
|
|
7531
|
+
warnings: base.warnings,
|
|
7532
|
+
sourceSystems: override.sourceSystems?.length ? override.sourceSystems : base.sourceSystems,
|
|
7533
|
+
replacementFor: override.replacementFor?.length ? override.replacementFor : base.replacementFor,
|
|
7534
|
+
reviewCadence: override.reviewCadence ?? base.reviewCadence,
|
|
5024
7535
|
};
|
|
5025
7536
|
}
|
|
5026
|
-
function
|
|
5027
|
-
const
|
|
5028
|
-
const
|
|
5029
|
-
const
|
|
5030
|
-
|
|
5031
|
-
|
|
5032
|
-
|
|
5033
|
-
|
|
5034
|
-
|
|
5035
|
-
|
|
5036
|
-
|
|
7537
|
+
function buildDqlGenerationSimilarityMatches(candidate, patch, contextPack) {
|
|
7538
|
+
const candidateSql = patch.sql ?? candidate.sql;
|
|
7539
|
+
const candidateFingerprints = buildBlockSqlFingerprints(candidateSql);
|
|
7540
|
+
const candidateBusinessFingerprint = buildBlockBusinessFingerprint({
|
|
7541
|
+
name: patch.name ?? candidate.name,
|
|
7542
|
+
domain: patch.domain ?? candidate.domain,
|
|
7543
|
+
pattern: patch.pattern,
|
|
7544
|
+
grain: patch.grain,
|
|
7545
|
+
entities: patch.entities,
|
|
7546
|
+
terms: patch.terms,
|
|
7547
|
+
outputs: patch.outputs,
|
|
7548
|
+
dimensions: patch.dimensions,
|
|
7549
|
+
filters: patch.allowedFilters,
|
|
7550
|
+
sources: candidate.lineage.sourceTables,
|
|
7551
|
+
sourceSystems: patch.sourceSystems,
|
|
7552
|
+
});
|
|
7553
|
+
const candidateShape = {
|
|
7554
|
+
name: candidate.name,
|
|
7555
|
+
pattern: patch.pattern,
|
|
7556
|
+
grain: patch.grain,
|
|
7557
|
+
outputs: new Set((patch.outputs ?? []).map(normalizedTerm)),
|
|
7558
|
+
terms: new Set((patch.terms ?? []).map(normalizedTerm)),
|
|
7559
|
+
dimensions: new Set((patch.dimensions ?? []).map(normalizedTerm)),
|
|
7560
|
+
filters: new Set((patch.allowedFilters ?? []).map(normalizedTerm)),
|
|
7561
|
+
entities: new Set((patch.entities ?? []).map(normalizedTerm)),
|
|
7562
|
+
sources: new Set(candidate.lineage.sourceTables.flatMap(relationLookupTokens).map(normalizedTerm)),
|
|
7563
|
+
};
|
|
7564
|
+
const objects = (contextPack?.objects ?? []).filter((object) => object.objectType === 'dql_block');
|
|
7565
|
+
const matches = objects.flatMap((object) => {
|
|
7566
|
+
const payload = object.payload ?? {};
|
|
7567
|
+
const objectSql = typeof payload.sql === 'string' ? payload.sql : '';
|
|
7568
|
+
const objectFingerprints = metadataSqlFingerprints(payload.sqlFingerprints)
|
|
7569
|
+
?? (objectSql ? buildBlockSqlFingerprints(objectSql) : null);
|
|
7570
|
+
const objectBusinessFingerprint = metadataBusinessFingerprint(payload.businessFingerprint);
|
|
7571
|
+
const objectShape = {
|
|
7572
|
+
pattern: metadataString(payload.pattern),
|
|
7573
|
+
grain: metadataString(payload.grain),
|
|
7574
|
+
outputs: new Set(metadataStringArray(payload.declaredOutputs ?? payload.outputs)),
|
|
7575
|
+
terms: new Set(metadataStringArray(payload.termRefs ?? payload.terms)),
|
|
7576
|
+
dimensions: new Set(metadataStringArray(payload.dimensions)),
|
|
7577
|
+
filters: new Set(metadataStringArray(payload.allowedFilters)),
|
|
7578
|
+
entities: new Set(metadataStringArray(payload.entities)),
|
|
7579
|
+
sources: new Set(metadataStringArray(payload.tableDependencies ?? payload.rawTableRefs).flatMap(relationLookupTokens).map(normalizedTerm)),
|
|
7580
|
+
};
|
|
7581
|
+
if (objectFingerprints?.exact && objectFingerprints.exact === candidateFingerprints.exact) {
|
|
7582
|
+
return [{
|
|
7583
|
+
kind: 'exact_sql_match',
|
|
7584
|
+
objectKey: object.objectKey,
|
|
7585
|
+
name: object.fullName ?? object.name,
|
|
7586
|
+
status: object.status,
|
|
7587
|
+
source: object.sourcePath,
|
|
7588
|
+
score: 0.99,
|
|
7589
|
+
reason: 'Normalized SQL exactly matches an existing DQL block.',
|
|
7590
|
+
recommendedAction: 'reuse_existing',
|
|
7591
|
+
}];
|
|
7592
|
+
}
|
|
7593
|
+
if (objectFingerprints?.parameterized && objectFingerprints.parameterized === candidateFingerprints.parameterized) {
|
|
7594
|
+
return [{
|
|
7595
|
+
kind: 'parameterized_duplicate',
|
|
7596
|
+
objectKey: object.objectKey,
|
|
7597
|
+
name: object.fullName ?? object.name,
|
|
7598
|
+
status: object.status,
|
|
7599
|
+
source: object.sourcePath,
|
|
7600
|
+
score: 0.94,
|
|
7601
|
+
reason: 'SQL shape matches after replacing literal values with parameters.',
|
|
7602
|
+
recommendedAction: 'reuse_existing',
|
|
7603
|
+
}];
|
|
7604
|
+
}
|
|
7605
|
+
const score = businessShapeSimilarity(candidateShape, objectShape, object);
|
|
7606
|
+
if (objectBusinessFingerprint?.hash
|
|
7607
|
+
&& objectBusinessFingerprint.hash === candidateBusinessFingerprint.hash
|
|
7608
|
+
&& object.status === 'certified') {
|
|
7609
|
+
return [{
|
|
7610
|
+
kind: 'business_duplicate',
|
|
7611
|
+
objectKey: object.objectKey,
|
|
7612
|
+
name: object.fullName ?? object.name,
|
|
7613
|
+
status: object.status,
|
|
7614
|
+
source: object.sourcePath,
|
|
7615
|
+
score: Math.max(score, 0.86),
|
|
7616
|
+
reason: 'Certified block has the same persisted business-shape fingerprint.',
|
|
7617
|
+
recommendedAction: 'reuse_existing',
|
|
7618
|
+
}];
|
|
7619
|
+
}
|
|
7620
|
+
if (score >= 0.76 && object.status === 'certified') {
|
|
7621
|
+
return [{
|
|
7622
|
+
kind: 'business_duplicate',
|
|
7623
|
+
objectKey: object.objectKey,
|
|
7624
|
+
name: object.fullName ?? object.name,
|
|
7625
|
+
status: object.status,
|
|
7626
|
+
source: object.sourcePath,
|
|
7627
|
+
score,
|
|
7628
|
+
reason: 'Certified block has the same business shape: entity, grain, filters, outputs, or source family overlap.',
|
|
7629
|
+
recommendedAction: 'reuse_existing',
|
|
7630
|
+
}];
|
|
7631
|
+
}
|
|
7632
|
+
if (score >= 0.58) {
|
|
7633
|
+
return [{
|
|
7634
|
+
kind: 'near_variant',
|
|
7635
|
+
objectKey: object.objectKey,
|
|
7636
|
+
name: object.fullName ?? object.name,
|
|
7637
|
+
status: object.status,
|
|
7638
|
+
source: object.sourcePath,
|
|
7639
|
+
score,
|
|
7640
|
+
reason: 'Existing block is close but appears to need a reviewed filter, output, grain, or source extension.',
|
|
7641
|
+
recommendedAction: 'extend_existing',
|
|
7642
|
+
}];
|
|
7643
|
+
}
|
|
7644
|
+
return [];
|
|
7645
|
+
}).sort((a, b) => b.score - a.score).slice(0, 8);
|
|
7646
|
+
const top = matches[0];
|
|
7647
|
+
return {
|
|
7648
|
+
matches,
|
|
7649
|
+
recommendedAction: top?.recommendedAction ?? 'create_new',
|
|
7650
|
+
};
|
|
5037
7651
|
}
|
|
5038
|
-
function
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
|
|
5042
|
-
const
|
|
5043
|
-
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
7652
|
+
function metadataSqlFingerprints(value) {
|
|
7653
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7654
|
+
return null;
|
|
7655
|
+
const record = value;
|
|
7656
|
+
const exact = typeof record.exact === 'string' ? record.exact : '';
|
|
7657
|
+
const parameterized = typeof record.parameterized === 'string' ? record.parameterized : '';
|
|
7658
|
+
if (!exact || !parameterized)
|
|
7659
|
+
return null;
|
|
7660
|
+
return { exact, parameterized };
|
|
7661
|
+
}
|
|
7662
|
+
function metadataBusinessFingerprint(value) {
|
|
7663
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7664
|
+
return null;
|
|
7665
|
+
const record = value;
|
|
7666
|
+
const hash = typeof record.hash === 'string' ? record.hash : '';
|
|
7667
|
+
if (!hash)
|
|
7668
|
+
return null;
|
|
7669
|
+
const tokens = Array.isArray(record.tokens)
|
|
7670
|
+
? record.tokens.map((item) => String(item)).filter(Boolean)
|
|
7671
|
+
: [];
|
|
7672
|
+
return { hash, tokens };
|
|
7673
|
+
}
|
|
7674
|
+
function businessShapeSimilarity(candidate, object, metadata) {
|
|
7675
|
+
let score = 0;
|
|
7676
|
+
if (candidate.pattern && object.pattern && normalizedTerm(candidate.pattern) === normalizedTerm(object.pattern))
|
|
7677
|
+
score += 0.12;
|
|
7678
|
+
if (candidate.grain && object.grain && normalizedTerm(candidate.grain) === normalizedTerm(object.grain))
|
|
7679
|
+
score += 0.15;
|
|
7680
|
+
score += overlapScore(candidate.outputs, object.outputs) * 0.18;
|
|
7681
|
+
score += overlapScore(candidate.terms, object.terms) * 0.08;
|
|
7682
|
+
score += overlapScore(candidate.dimensions, object.dimensions) * 0.13;
|
|
7683
|
+
score += overlapScore(candidate.filters, object.filters) * 0.14;
|
|
7684
|
+
score += overlapScore(candidate.entities, object.entities) * 0.12;
|
|
7685
|
+
score += overlapScore(candidate.sources, object.sources) * 0.1;
|
|
7686
|
+
const nameTokens = new Set(candidate.name.toLowerCase().split(/[^a-z0-9]+/).filter((value) => value.length > 2));
|
|
7687
|
+
const targetTokens = new Set([
|
|
7688
|
+
...metadata.name.toLowerCase().split(/[^a-z0-9]+/),
|
|
7689
|
+
...(metadata.description ?? '').toLowerCase().split(/[^a-z0-9]+/),
|
|
7690
|
+
].filter((value) => value.length > 2));
|
|
7691
|
+
score += overlapScore(nameTokens, targetTokens) * 0.04;
|
|
7692
|
+
return Number(Math.min(0.98, score).toFixed(3));
|
|
7693
|
+
}
|
|
7694
|
+
function overlapScore(left, right) {
|
|
7695
|
+
if (left.size === 0 || right.size === 0)
|
|
7696
|
+
return 0;
|
|
7697
|
+
let overlap = 0;
|
|
7698
|
+
for (const value of left) {
|
|
7699
|
+
if (value && right.has(value))
|
|
7700
|
+
overlap += 1;
|
|
7701
|
+
}
|
|
7702
|
+
return overlap / Math.max(1, Math.min(left.size, right.size));
|
|
7703
|
+
}
|
|
7704
|
+
function normalizedTerm(value) {
|
|
7705
|
+
return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
7706
|
+
}
|
|
7707
|
+
function metadataString(value) {
|
|
7708
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
7709
|
+
}
|
|
7710
|
+
function metadataStringArray(value) {
|
|
7711
|
+
if (!Array.isArray(value))
|
|
7712
|
+
return [];
|
|
7713
|
+
return value.map((item) => String(item).trim()).filter(Boolean).map(normalizedTerm);
|
|
7714
|
+
}
|
|
7715
|
+
function isWeakDqlGenerationName(name) {
|
|
7716
|
+
return /^(pasted|query|source|imported query|sql import)(?:\s+\d+)?$/i.test(name.trim());
|
|
7717
|
+
}
|
|
7718
|
+
function isWeakDqlGenerationDescription(description) {
|
|
7719
|
+
return /^draft dql block summarizing imported sql/i.test(description.trim())
|
|
7720
|
+
|| /^draft dql block generated from imported sql/i.test(description.trim())
|
|
7721
|
+
|| /^imported from /i.test(description.trim())
|
|
7722
|
+
|| /^this query (shows|returns|provides)/i.test(description.trim());
|
|
7723
|
+
}
|
|
7724
|
+
function isUngroundedDqlGenerationDescription(description, candidate, evidence) {
|
|
7725
|
+
const text = description.toLowerCase();
|
|
7726
|
+
const tokens = new Set();
|
|
7727
|
+
for (const tag of candidate.tags) {
|
|
7728
|
+
if (!/^(raw-sql|imported|ai-generated|review-required)$/i.test(tag) && tag.length > 2)
|
|
7729
|
+
tokens.add(tag.toLowerCase());
|
|
5048
7730
|
}
|
|
5049
|
-
|
|
5050
|
-
|
|
7731
|
+
for (const table of candidate.lineage.sourceTables) {
|
|
7732
|
+
for (const token of table.split(/[._\s/-]+/)) {
|
|
7733
|
+
if (token.length > 2)
|
|
7734
|
+
tokens.add(token.toLowerCase());
|
|
7735
|
+
}
|
|
5051
7736
|
}
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
if (!existsSync(absCompanionPath))
|
|
5058
|
-
return;
|
|
5059
|
-
let companion = readFileSync(absCompanionPath, 'utf-8');
|
|
5060
|
-
if (/^reviewStatus:\s*.+$/m.test(companion)) {
|
|
5061
|
-
companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
|
|
7737
|
+
for (const item of evidence.slice(0, 6)) {
|
|
7738
|
+
for (const token of item.name.split(/[._\s/-]+/)) {
|
|
7739
|
+
if (token.length > 3 && !/^(model|source|semantic|table)$/i.test(token))
|
|
7740
|
+
tokens.add(token.toLowerCase());
|
|
7741
|
+
}
|
|
5062
7742
|
}
|
|
5063
|
-
|
|
5064
|
-
|
|
7743
|
+
if (tokens.size === 0)
|
|
7744
|
+
return false;
|
|
7745
|
+
return !Array.from(tokens).some((token) => text.includes(token));
|
|
7746
|
+
}
|
|
7747
|
+
function dqlGenerationConversionNotes(providerName) {
|
|
7748
|
+
return [
|
|
7749
|
+
`Generated with ${providerName} using local project context when available.`,
|
|
7750
|
+
'Runtime-scope SQL literals are converted into DQL params when safe; business constants stay static for review.',
|
|
7751
|
+
'Certified and draft block similarity is checked before a new draft is saved.',
|
|
7752
|
+
'SQL semantics are preserved while reusable parameter placeholders may replace literal values.',
|
|
7753
|
+
'Drafts are autosaved under blocks/_drafts or domains/<domain>/blocks/_drafts before certification.',
|
|
7754
|
+
'Preview results and tests gate certification before promotion.',
|
|
7755
|
+
];
|
|
7756
|
+
}
|
|
7757
|
+
function setBlockStudioSourceStatus(source, status) {
|
|
7758
|
+
if (/status\s*=\s*"[^"]*"/.test(source)) {
|
|
7759
|
+
return source.replace(/status\s*=\s*"[^"]*"/, `status = "${status}"`);
|
|
5065
7760
|
}
|
|
5066
|
-
|
|
7761
|
+
return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${status}"`);
|
|
5067
7762
|
}
|
|
5068
7763
|
async function buildBlockStudioAiAssistSummary(projectRoot, action, candidate, validation, requestedProvider) {
|
|
5069
7764
|
const fallback = buildDeterministicAiAssistSummary(action, candidate, validation);
|
|
@@ -5271,6 +7966,14 @@ export function readGitMetadata(projectRoot) {
|
|
|
5271
7966
|
return null;
|
|
5272
7967
|
}
|
|
5273
7968
|
}
|
|
7969
|
+
function resolveBlockWriteTarget(projectRoot, safeDomain, slug) {
|
|
7970
|
+
if (safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))) {
|
|
7971
|
+
const relativePath = `domains/${safeDomain}/blocks/${slug}.dql`;
|
|
7972
|
+
return { relativePath, absPath: join(projectRoot, relativePath) };
|
|
7973
|
+
}
|
|
7974
|
+
const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
|
|
7975
|
+
return { relativePath, absPath: join(projectRoot, relativePath) };
|
|
7976
|
+
}
|
|
5274
7977
|
export function createBlockArtifacts(projectRoot, options) {
|
|
5275
7978
|
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
5276
7979
|
const safeDomain = (options.domain ?? '')
|
|
@@ -5278,16 +7981,16 @@ export function createBlockArtifacts(projectRoot, options) {
|
|
|
5278
7981
|
.toLowerCase()
|
|
5279
7982
|
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
5280
7983
|
.replace(/^\/+|\/+$/g, '');
|
|
5281
|
-
const
|
|
5282
|
-
mkdirSync(
|
|
5283
|
-
const blockPath =
|
|
7984
|
+
const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
|
|
7985
|
+
mkdirSync(dirname(target.absPath), { recursive: true });
|
|
7986
|
+
const blockPath = target.absPath;
|
|
5284
7987
|
if (existsSync(blockPath)) {
|
|
5285
7988
|
throw new Error('BLOCK_EXISTS');
|
|
5286
7989
|
}
|
|
5287
7990
|
const templateContent = options.template
|
|
5288
7991
|
? listBlockTemplates().find((template) => template.id === options.template)?.content
|
|
5289
7992
|
: undefined;
|
|
5290
|
-
const relativePath =
|
|
7993
|
+
const relativePath = target.relativePath;
|
|
5291
7994
|
const fileContent = canonicalizeSafe(options.blockType === 'semantic' && !options.content?.trim() && !templateContent
|
|
5292
7995
|
? buildBlankSemanticBlockContent({
|
|
5293
7996
|
name: options.name,
|
|
@@ -5333,9 +8036,9 @@ export function createSemanticBuilderBlock(projectRoot, options) {
|
|
|
5333
8036
|
.toLowerCase()
|
|
5334
8037
|
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
5335
8038
|
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
5336
|
-
const
|
|
5337
|
-
mkdirSync(
|
|
5338
|
-
const blockPath =
|
|
8039
|
+
const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
|
|
8040
|
+
mkdirSync(dirname(target.absPath), { recursive: true });
|
|
8041
|
+
const blockPath = target.absPath;
|
|
5339
8042
|
if (existsSync(blockPath)) {
|
|
5340
8043
|
throw new Error('BLOCK_EXISTS');
|
|
5341
8044
|
}
|
|
@@ -5360,7 +8063,7 @@ export function createSemanticBuilderBlock(projectRoot, options) {
|
|
|
5360
8063
|
],
|
|
5361
8064
|
});
|
|
5362
8065
|
return {
|
|
5363
|
-
path:
|
|
8066
|
+
path: target.relativePath,
|
|
5364
8067
|
content,
|
|
5365
8068
|
companionPath,
|
|
5366
8069
|
};
|
|
@@ -6122,18 +8825,16 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
|
|
|
6122
8825
|
const sourceCount = manifest?.sources && typeof manifest.sources === 'object'
|
|
6123
8826
|
? Object.keys(manifest.sources).length
|
|
6124
8827
|
: 0;
|
|
6125
|
-
const manifestMetricCount = manifest?.metrics
|
|
6126
|
-
|
|
6127
|
-
|
|
6128
|
-
const semanticMetricCount = Array.isArray(semanticManifest?.metrics)
|
|
6129
|
-
? semanticManifest.metrics.length
|
|
8828
|
+
const manifestMetricCount = artifactCollectionCount(manifest?.metrics);
|
|
8829
|
+
const semanticMetricCount = semanticManifest
|
|
8830
|
+
? artifactCollectionCount(semanticManifest.metrics)
|
|
6130
8831
|
: manifestMetricCount;
|
|
6131
|
-
const semanticModelCount =
|
|
6132
|
-
? semanticManifest.semantic_models
|
|
6133
|
-
:
|
|
6134
|
-
const savedQueryCount =
|
|
6135
|
-
? semanticManifest.saved_queries.
|
|
6136
|
-
:
|
|
8832
|
+
const semanticModelCount = semanticManifest
|
|
8833
|
+
? artifactCollectionCount(semanticManifest.semantic_models)
|
|
8834
|
+
: artifactCollectionCount(manifest?.semantic_models);
|
|
8835
|
+
const savedQueryCount = semanticManifest
|
|
8836
|
+
? artifactCollectionCount(semanticManifest.saved_queries ?? semanticManifest.savedQueries)
|
|
8837
|
+
: artifactCollectionCount(manifest?.saved_queries ?? manifest?.savedQueries);
|
|
6137
8838
|
const configured = existsSync(join(dbtProjectPath, 'dbt_project.yml'))
|
|
6138
8839
|
|| Boolean(projectConfig.dbt?.projectDir)
|
|
6139
8840
|
|| Boolean(projectConfig.semanticLayer?.provider === 'dbt' && projectConfig.semanticLayer.projectPath);
|
|
@@ -6145,7 +8846,9 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
|
|
|
6145
8846
|
? 'Run `dbt parse`, `dbt compile`, or `dbt build`, then run `dql sync dbt`.'
|
|
6146
8847
|
: !semanticExists
|
|
6147
8848
|
? 'dbt manifest is ready. Run `dbt parse` or `dbt build` if you use dbt Semantic Layer metrics.'
|
|
6148
|
-
:
|
|
8849
|
+
: semanticMetricCount === 0 && semanticModelCount === 0 && savedQueryCount === 0
|
|
8850
|
+
? 'dbt manifest is ready, but MetricFlow semantic_manifest.json is empty. Add dbt semantic_models and metrics, then rerun dbt parse/build.'
|
|
8851
|
+
: 'dbt artifacts are ready. Build SQL blocks from models or semantic blocks from metrics.';
|
|
6149
8852
|
return {
|
|
6150
8853
|
configured,
|
|
6151
8854
|
provider: projectConfig.semanticLayer?.provider ?? null,
|
|
@@ -6168,6 +8871,185 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
|
|
|
6168
8871
|
setupHint,
|
|
6169
8872
|
};
|
|
6170
8873
|
}
|
|
8874
|
+
export function buildSemanticLayerDiagnostics(projectRoot, projectConfig, options) {
|
|
8875
|
+
const provider = options.semanticConfig?.provider ?? options.detectedProvider ?? null;
|
|
8876
|
+
const dbt = buildDbtStatus(projectRoot, projectConfig, options.lastSyncTime);
|
|
8877
|
+
const counts = semanticLayerCounts(options.semanticLayer);
|
|
8878
|
+
const uncategorizedObjects = countUncategorizedSemanticObjects(options.semanticLayer);
|
|
8879
|
+
const totalObjects = counts.metrics
|
|
8880
|
+
+ counts.measures
|
|
8881
|
+
+ counts.dimensions
|
|
8882
|
+
+ counts.timeDimensions
|
|
8883
|
+
+ counts.entities
|
|
8884
|
+
+ counts.hierarchies
|
|
8885
|
+
+ counts.semanticModels
|
|
8886
|
+
+ counts.savedQueries;
|
|
8887
|
+
const issues = [];
|
|
8888
|
+
for (const error of options.semanticErrors ?? []) {
|
|
8889
|
+
issues.push({
|
|
8890
|
+
severity: 'error',
|
|
8891
|
+
code: 'semantic_load_error',
|
|
8892
|
+
message: error,
|
|
8893
|
+
action: 'Open the semantic source configuration, fix the provider error, then reload the semantic layer.',
|
|
8894
|
+
});
|
|
8895
|
+
}
|
|
8896
|
+
if (!provider) {
|
|
8897
|
+
issues.push({
|
|
8898
|
+
severity: existsSync(join(projectRoot, 'semantic-layer')) ? 'info' : 'warning',
|
|
8899
|
+
code: 'semantic_provider_not_configured',
|
|
8900
|
+
message: 'No semantic layer provider is configured.',
|
|
8901
|
+
action: 'Configure dbt MetricFlow, local semantic-layer YAML, or Snowflake Semantic Views from the setup flow.',
|
|
8902
|
+
});
|
|
8903
|
+
}
|
|
8904
|
+
if (provider === 'dbt' || dbt.configured) {
|
|
8905
|
+
if (!dbt.configured) {
|
|
8906
|
+
issues.push({
|
|
8907
|
+
severity: 'warning',
|
|
8908
|
+
code: 'dbt_project_not_detected',
|
|
8909
|
+
message: 'dbt is selected as the semantic source, but no dbt project was detected.',
|
|
8910
|
+
action: 'Set dbt.projectDir or run DQL from a repository containing dbt_project.yml.',
|
|
8911
|
+
});
|
|
8912
|
+
}
|
|
8913
|
+
if (!dbt.artifacts.manifest.exists) {
|
|
8914
|
+
issues.push({
|
|
8915
|
+
severity: 'error',
|
|
8916
|
+
code: 'dbt_manifest_missing',
|
|
8917
|
+
message: 'dbt manifest.json is missing, so DQL cannot build dbt model context.',
|
|
8918
|
+
action: 'Run dbt parse, dbt compile, or dbt build, then reload the semantic layer.',
|
|
8919
|
+
path: dbt.artifacts.manifest.path,
|
|
8920
|
+
});
|
|
8921
|
+
}
|
|
8922
|
+
if (!dbt.artifacts.semanticManifest.exists) {
|
|
8923
|
+
issues.push({
|
|
8924
|
+
severity: 'warning',
|
|
8925
|
+
code: 'metricflow_semantic_manifest_missing',
|
|
8926
|
+
message: 'dbt MetricFlow semantic_manifest.json is missing. DQL can still inspect dbt models, but MetricFlow metrics and saved queries may be incomplete.',
|
|
8927
|
+
action: 'Run dbt parse or dbt build with semantic models configured, then reload. For execution, ensure MetricFlow is installed and mf is on PATH.',
|
|
8928
|
+
path: dbt.artifacts.semanticManifest.path,
|
|
8929
|
+
});
|
|
8930
|
+
}
|
|
8931
|
+
else if (dbt.counts.metrics === 0 && dbt.counts.semanticModels === 0 && dbt.counts.savedQueries === 0) {
|
|
8932
|
+
issues.push({
|
|
8933
|
+
severity: 'warning',
|
|
8934
|
+
code: 'metricflow_semantic_manifest_empty',
|
|
8935
|
+
message: 'semantic_manifest.json exists, but it does not contain metrics, semantic models, or saved queries.',
|
|
8936
|
+
action: 'Check dbt semantic model YAML definitions and rerun dbt parse.',
|
|
8937
|
+
path: dbt.artifacts.semanticManifest.path,
|
|
8938
|
+
});
|
|
8939
|
+
}
|
|
8940
|
+
if (options.semanticLayer && totalObjects === 0 && (dbt.counts.models > 0 || dbt.counts.sources > 0)) {
|
|
8941
|
+
issues.push({
|
|
8942
|
+
severity: 'warning',
|
|
8943
|
+
code: 'dbt_model_inventory_only',
|
|
8944
|
+
message: 'DQL loaded dbt model inventory, but no MetricFlow semantic metrics were found.',
|
|
8945
|
+
action: 'Create dbt semantic_models and metrics, or use AI Import to build DQL blocks from dbt model metadata.',
|
|
8946
|
+
});
|
|
8947
|
+
}
|
|
8948
|
+
}
|
|
8949
|
+
if (provider === 'snowflake') {
|
|
8950
|
+
if (!options.semanticLayer && (options.semanticErrors ?? []).length === 0) {
|
|
8951
|
+
issues.push({
|
|
8952
|
+
severity: 'warning',
|
|
8953
|
+
code: 'snowflake_semantic_views_not_loaded',
|
|
8954
|
+
message: 'Snowflake semantic provider did not load semantic views.',
|
|
8955
|
+
action: 'Confirm the active Snowflake connection can query semantic views, then reload. Warehouse tables still feed database context separately.',
|
|
8956
|
+
});
|
|
8957
|
+
}
|
|
8958
|
+
else if (options.semanticLayer && totalObjects === 0) {
|
|
8959
|
+
issues.push({
|
|
8960
|
+
severity: 'info',
|
|
8961
|
+
code: 'snowflake_semantic_views_empty',
|
|
8962
|
+
message: 'Snowflake semantic provider loaded, but no semantic view objects were discovered.',
|
|
8963
|
+
action: 'Check that Snowflake Semantic Views exist and the active role can see them.',
|
|
8964
|
+
});
|
|
8965
|
+
}
|
|
8966
|
+
}
|
|
8967
|
+
if (options.semanticLayer && totalObjects === 0 && issues.every((issue) => issue.code !== 'dbt_model_inventory_only')) {
|
|
8968
|
+
issues.push({
|
|
8969
|
+
severity: 'warning',
|
|
8970
|
+
code: 'semantic_layer_empty',
|
|
8971
|
+
message: 'Semantic layer is available, but no semantic objects are loaded.',
|
|
8972
|
+
action: 'Import semantic definitions or run dbt parse/build, then reload.',
|
|
8973
|
+
});
|
|
8974
|
+
}
|
|
8975
|
+
if (uncategorizedObjects > 0) {
|
|
8976
|
+
issues.push({
|
|
8977
|
+
severity: 'info',
|
|
8978
|
+
code: 'semantic_uncategorized_domain',
|
|
8979
|
+
message: `${uncategorizedObjects} semantic object${uncategorizedObjects === 1 ? '' : 's'} had no domain and are visible under "uncategorized".`,
|
|
8980
|
+
action: 'Add domain metadata in dbt semantic YAML or local semantic-layer files when domain ownership matters.',
|
|
8981
|
+
});
|
|
8982
|
+
}
|
|
8983
|
+
const warnings = Array.from(new Set([
|
|
8984
|
+
...(options.semanticErrors ?? []),
|
|
8985
|
+
...issues
|
|
8986
|
+
.filter((issue) => issue.severity !== 'info')
|
|
8987
|
+
.map((issue) => issue.message),
|
|
8988
|
+
]));
|
|
8989
|
+
return {
|
|
8990
|
+
available: Boolean(options.semanticLayer),
|
|
8991
|
+
provider,
|
|
8992
|
+
sourceOfTruth: provider === 'dbt'
|
|
8993
|
+
? 'dbt MetricFlow semantic_manifest.json, dbt manifest.json semantic nodes, then dbt semantic YAML fallback'
|
|
8994
|
+
: provider === 'snowflake'
|
|
8995
|
+
? 'Snowflake Semantic Views only; warehouse tables remain database catalog context'
|
|
8996
|
+
: provider === 'dql'
|
|
8997
|
+
? 'local semantic-layer YAML'
|
|
8998
|
+
: 'not configured',
|
|
8999
|
+
errors: options.semanticErrors ?? [],
|
|
9000
|
+
lastSyncTime: options.lastSyncTime,
|
|
9001
|
+
counts,
|
|
9002
|
+
dbt,
|
|
9003
|
+
issues,
|
|
9004
|
+
warnings,
|
|
9005
|
+
};
|
|
9006
|
+
}
|
|
9007
|
+
function semanticLayerCounts(layer) {
|
|
9008
|
+
return layer
|
|
9009
|
+
? {
|
|
9010
|
+
domains: layer.listDomains().length,
|
|
9011
|
+
metrics: layer.listMetrics().length,
|
|
9012
|
+
measures: layer.listMeasures().length,
|
|
9013
|
+
dimensions: layer.listDimensions().length,
|
|
9014
|
+
timeDimensions: layer.listTimeDimensions().length,
|
|
9015
|
+
entities: layer.listEntities().length,
|
|
9016
|
+
hierarchies: layer.listHierarchies().length,
|
|
9017
|
+
semanticModels: layer.listSemanticModels().length,
|
|
9018
|
+
savedQueries: layer.listSavedQueries().length,
|
|
9019
|
+
}
|
|
9020
|
+
: {
|
|
9021
|
+
domains: 0,
|
|
9022
|
+
metrics: 0,
|
|
9023
|
+
measures: 0,
|
|
9024
|
+
dimensions: 0,
|
|
9025
|
+
timeDimensions: 0,
|
|
9026
|
+
entities: 0,
|
|
9027
|
+
hierarchies: 0,
|
|
9028
|
+
semanticModels: 0,
|
|
9029
|
+
savedQueries: 0,
|
|
9030
|
+
};
|
|
9031
|
+
}
|
|
9032
|
+
function countUncategorizedSemanticObjects(layer) {
|
|
9033
|
+
if (!layer)
|
|
9034
|
+
return 0;
|
|
9035
|
+
return [
|
|
9036
|
+
...layer.listMetrics(),
|
|
9037
|
+
...layer.listMeasures(),
|
|
9038
|
+
...layer.listDimensions(),
|
|
9039
|
+
...layer.listTimeDimensions(),
|
|
9040
|
+
...layer.listEntities(),
|
|
9041
|
+
...layer.listHierarchies(),
|
|
9042
|
+
...layer.listSemanticModels(),
|
|
9043
|
+
...layer.listSavedQueries(),
|
|
9044
|
+
].filter((object) => !object.domain || !object.domain.trim()).length;
|
|
9045
|
+
}
|
|
9046
|
+
function artifactCollectionCount(value) {
|
|
9047
|
+
if (Array.isArray(value))
|
|
9048
|
+
return value.length;
|
|
9049
|
+
if (value && typeof value === 'object')
|
|
9050
|
+
return Object.keys(value).length;
|
|
9051
|
+
return 0;
|
|
9052
|
+
}
|
|
6171
9053
|
function describeArtifact(path, count, generatedAt) {
|
|
6172
9054
|
return {
|
|
6173
9055
|
path,
|
|
@@ -6704,6 +9586,11 @@ function isProviderSettingsId(value) {
|
|
|
6704
9586
|
|| value === 'ollama'
|
|
6705
9587
|
|| value === 'custom-openai';
|
|
6706
9588
|
}
|
|
9589
|
+
function isDeterministicDqlGenerationProvider(value) {
|
|
9590
|
+
if (typeof value !== 'string')
|
|
9591
|
+
return false;
|
|
9592
|
+
return value === 'none' || value === 'deterministic' || value === 'local-deterministic';
|
|
9593
|
+
}
|
|
6707
9594
|
async function testProviderConfig(projectRoot, id) {
|
|
6708
9595
|
const config = getEffectiveProviderConfig(projectRoot, id);
|
|
6709
9596
|
const label = providerSettingsLabel(id);
|
|
@@ -6853,6 +9740,668 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
|
|
|
6853
9740
|
// Runtime schema snapshots are advisory local metadata and must not block answers.
|
|
6854
9741
|
}
|
|
6855
9742
|
}
|
|
9743
|
+
function notebookResearchString(value) {
|
|
9744
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
9745
|
+
}
|
|
9746
|
+
function notebookResearchRecord(value) {
|
|
9747
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
9748
|
+
? value
|
|
9749
|
+
: null;
|
|
9750
|
+
}
|
|
9751
|
+
function notebookResearchSourceCellPayload(body) {
|
|
9752
|
+
const sourceCell = notebookResearchRecord(body.sourceCell)
|
|
9753
|
+
?? notebookResearchRecord(notebookResearchRecord(body.context)?.sourceCell);
|
|
9754
|
+
if (!sourceCell)
|
|
9755
|
+
return undefined;
|
|
9756
|
+
const payload = {
|
|
9757
|
+
id: notebookResearchString(sourceCell.id),
|
|
9758
|
+
sourceCellId: notebookResearchString(sourceCell.sourceCellId),
|
|
9759
|
+
cellId: notebookResearchString(sourceCell.cellId),
|
|
9760
|
+
name: notebookResearchString(sourceCell.name),
|
|
9761
|
+
sourceCellName: notebookResearchString(sourceCell.sourceCellName),
|
|
9762
|
+
title: notebookResearchString(sourceCell.title),
|
|
9763
|
+
fingerprint: notebookResearchString(sourceCell.fingerprint),
|
|
9764
|
+
sourceCellFingerprint: notebookResearchString(sourceCell.sourceCellFingerprint),
|
|
9765
|
+
sqlFingerprint: notebookResearchString(sourceCell.sqlFingerprint),
|
|
9766
|
+
type: notebookResearchString(sourceCell.type),
|
|
9767
|
+
sql: notebookResearchString(sourceCell.sql),
|
|
9768
|
+
content: notebookResearchString(sourceCell.content),
|
|
9769
|
+
source: notebookResearchString(sourceCell.source),
|
|
9770
|
+
};
|
|
9771
|
+
return Object.values(payload).some(Boolean) ? payload : undefined;
|
|
9772
|
+
}
|
|
9773
|
+
function notebookResearchSourceCellId(sourceCell) {
|
|
9774
|
+
return notebookResearchString(sourceCell?.id)
|
|
9775
|
+
?? notebookResearchString(sourceCell?.sourceCellId)
|
|
9776
|
+
?? notebookResearchString(sourceCell?.cellId);
|
|
9777
|
+
}
|
|
9778
|
+
function notebookResearchSourceCellName(sourceCell) {
|
|
9779
|
+
return notebookResearchString(sourceCell?.name)
|
|
9780
|
+
?? notebookResearchString(sourceCell?.sourceCellName)
|
|
9781
|
+
?? notebookResearchString(sourceCell?.title);
|
|
9782
|
+
}
|
|
9783
|
+
function notebookResearchSourceCellFingerprint(sourceCell) {
|
|
9784
|
+
return notebookResearchString(sourceCell?.sourceCellFingerprint)
|
|
9785
|
+
?? notebookResearchString(sourceCell?.fingerprint)
|
|
9786
|
+
?? notebookResearchString(sourceCell?.sqlFingerprint);
|
|
9787
|
+
}
|
|
9788
|
+
function notebookResearchPatchString(body, key) {
|
|
9789
|
+
if (!Object.prototype.hasOwnProperty.call(body, key))
|
|
9790
|
+
return undefined;
|
|
9791
|
+
return notebookResearchString(body[key]) ?? null;
|
|
9792
|
+
}
|
|
9793
|
+
function notebookResearchIntent(value) {
|
|
9794
|
+
return value === 'ad_hoc_analysis'
|
|
9795
|
+
|| value === 'diagnose_change'
|
|
9796
|
+
|| value === 'driver_breakdown'
|
|
9797
|
+
|| value === 'segment_compare'
|
|
9798
|
+
|| value === 'entity_drilldown'
|
|
9799
|
+
|| value === 'anomaly_investigation'
|
|
9800
|
+
|| value === 'trust_gap_review'
|
|
9801
|
+
? value
|
|
9802
|
+
: undefined;
|
|
9803
|
+
}
|
|
9804
|
+
function notebookResearchReviewStatus(value) {
|
|
9805
|
+
return value === 'needs_review'
|
|
9806
|
+
|| value === 'draft_created'
|
|
9807
|
+
|| value === 'completed'
|
|
9808
|
+
|| value === 'certified'
|
|
9809
|
+
|| value === 'rejected'
|
|
9810
|
+
? value
|
|
9811
|
+
: undefined;
|
|
9812
|
+
}
|
|
9813
|
+
function notebookResearchStatus(value) {
|
|
9814
|
+
return value === 'draft'
|
|
9815
|
+
|| value === 'running'
|
|
9816
|
+
|| value === 'ready'
|
|
9817
|
+
|| value === 'error'
|
|
9818
|
+
? value
|
|
9819
|
+
: undefined;
|
|
9820
|
+
}
|
|
9821
|
+
function notebookResearchPromotionAction(value) {
|
|
9822
|
+
return value === 'reuse_existing'
|
|
9823
|
+
|| value === 'extend_existing'
|
|
9824
|
+
|| value === 'create_replacement'
|
|
9825
|
+
|| value === 'create_new'
|
|
9826
|
+
|| value === 'review_required'
|
|
9827
|
+
? value
|
|
9828
|
+
: undefined;
|
|
9829
|
+
}
|
|
9830
|
+
function notebookResearchReadiness(value) {
|
|
9831
|
+
return value === 'draft_ready' || value === 'certification_ready' || value === 'blocked' ? value : undefined;
|
|
9832
|
+
}
|
|
9833
|
+
function notebookResearchAge(value) {
|
|
9834
|
+
return value === 'stale_open' || value === 'expired_open' ? value : undefined;
|
|
9835
|
+
}
|
|
9836
|
+
function notebookResearchNextAction(value) {
|
|
9837
|
+
return value === 'fix_blockers'
|
|
9838
|
+
|| value === 'review_sql'
|
|
9839
|
+
|| value === 'review_context'
|
|
9840
|
+
|| value === 'run_preview'
|
|
9841
|
+
|| value === 'reuse_existing'
|
|
9842
|
+
|| value === 'create_dql_draft'
|
|
9843
|
+
|| value === 'open_certification'
|
|
9844
|
+
|| value === 'complete_review'
|
|
9845
|
+
|| value === 'continue_review'
|
|
9846
|
+
? value
|
|
9847
|
+
: undefined;
|
|
9848
|
+
}
|
|
9849
|
+
function notebookResearchSort(value) {
|
|
9850
|
+
return value === 'priority' || value === 'updated_desc' ? value : undefined;
|
|
9851
|
+
}
|
|
9852
|
+
function withNotebookResearchChecklistPage(page) {
|
|
9853
|
+
return {
|
|
9854
|
+
...page,
|
|
9855
|
+
runs: page.runs.map(withNotebookResearchChecklist),
|
|
9856
|
+
};
|
|
9857
|
+
}
|
|
9858
|
+
function withNotebookResearchChecklist(run) {
|
|
9859
|
+
const researchPlan = run.researchPlan ?? buildNotebookResearchPlan({
|
|
9860
|
+
run,
|
|
9861
|
+
evidence: run.evidence,
|
|
9862
|
+
resultPreview: run.resultPreview,
|
|
9863
|
+
previewError: run.status === 'error' ? run.error : undefined,
|
|
9864
|
+
generatedSql: run.generatedSql,
|
|
9865
|
+
reviewedSql: run.reviewedSql,
|
|
9866
|
+
routeDecision: run.routeDecision,
|
|
9867
|
+
});
|
|
9868
|
+
return {
|
|
9869
|
+
...run,
|
|
9870
|
+
researchPlan,
|
|
9871
|
+
reviewChecklist: buildNotebookResearchReviewChecklist(run),
|
|
9872
|
+
};
|
|
9873
|
+
}
|
|
9874
|
+
function buildNotebookResearchReviewChecklist(run) {
|
|
9875
|
+
const questionReady = Boolean(notebookResearchString(run.question));
|
|
9876
|
+
const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
|
|
9877
|
+
const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
|
|
9878
|
+
const hasSql = hasReviewedSql || hasGeneratedSql;
|
|
9879
|
+
const preview = notebookResearchPreviewInfo(run.resultPreview);
|
|
9880
|
+
const previewReady = run.status === 'ready' && preview.hasPreview;
|
|
9881
|
+
const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
|
|
9882
|
+
const promoted = Boolean(run.dqlPromotion || run.dqlImportId || run.draftBlockPath);
|
|
9883
|
+
const duplicateChecked = Boolean(run.dqlPromotionAction || run.dqlPromotion);
|
|
9884
|
+
const promotionAction = run.dqlPromotionAction ?? run.dqlPromotion?.recommendedAction;
|
|
9885
|
+
const reuseRecommended = promotionAction === 'reuse_existing';
|
|
9886
|
+
const promotionReviewRequired = promotionAction === 'review_required';
|
|
9887
|
+
const reuseClosed = reuseRecommended && (run.reviewStatus === 'completed' || run.reviewStatus === 'certified');
|
|
9888
|
+
const hasDraft = Boolean(notebookResearchString(run.draftBlockPath));
|
|
9889
|
+
const sqlForParameterReview = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
|
|
9890
|
+
const parameterReview = sqlForParameterReview ? parameterizeSqlForDqlImport(sqlForParameterReview) : null;
|
|
9891
|
+
const dynamicParameters = parameterReview?.parameterPolicy.filter((item) => item.policy === 'dynamic') ?? [];
|
|
9892
|
+
const staticParameters = parameterReview?.parameterPolicy.filter((item) => item.policy !== 'dynamic') ?? [];
|
|
9893
|
+
const blockers = [];
|
|
9894
|
+
const warnings = [];
|
|
9895
|
+
if (!questionReady)
|
|
9896
|
+
blockers.push('Add a business question before review.');
|
|
9897
|
+
if (!hasSql && !reuseRecommended)
|
|
9898
|
+
blockers.push('Add reviewed SQL or generate SQL before promotion.');
|
|
9899
|
+
if (run.status === 'error')
|
|
9900
|
+
blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
|
|
9901
|
+
if (!previewReady && run.status !== 'error' && !reuseRecommended)
|
|
9902
|
+
warnings.push('Run a bounded preview before certification review.');
|
|
9903
|
+
if (!evidenceReady && !reuseRecommended)
|
|
9904
|
+
warnings.push('Inspect metadata evidence before turning this into a reusable block.');
|
|
9905
|
+
if (hasSql && dynamicParameters.length === 0)
|
|
9906
|
+
warnings.push('No dynamic parameters were detected; confirm this should be a static block before certification.');
|
|
9907
|
+
for (const warning of parameterReview?.warnings ?? [])
|
|
9908
|
+
warnings.push(warning);
|
|
9909
|
+
if (promoted && !duplicateChecked)
|
|
9910
|
+
warnings.push('Promotion exists but duplicate/reuse evidence is missing.');
|
|
9911
|
+
if (reuseRecommended && !reuseClosed)
|
|
9912
|
+
warnings.push('Reuse the matching block or explicitly document a replacement before certification.');
|
|
9913
|
+
if (promotionReviewRequired)
|
|
9914
|
+
warnings.push('Resolve the review-required promotion decision before certification.');
|
|
9915
|
+
const items = [
|
|
9916
|
+
{
|
|
9917
|
+
id: 'question',
|
|
9918
|
+
label: 'Question',
|
|
9919
|
+
status: questionReady ? 'passed' : 'blocked',
|
|
9920
|
+
detail: questionReady ? 'Business question is captured.' : 'A reusable block needs a clear business question.',
|
|
9921
|
+
},
|
|
9922
|
+
{
|
|
9923
|
+
id: 'sql',
|
|
9924
|
+
label: 'Reviewed SQL',
|
|
9925
|
+
status: hasReviewedSql || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
|
|
9926
|
+
detail: reuseRecommended && !hasSql
|
|
9927
|
+
? 'Existing certified DQL should be reused; no new SQL is required.'
|
|
9928
|
+
: hasReviewedSql
|
|
9929
|
+
? 'Reviewer SQL is saved.'
|
|
9930
|
+
: hasGeneratedSql
|
|
9931
|
+
? 'Generated SQL exists; review it before promotion.'
|
|
9932
|
+
: 'No SQL is available for preview or DQL generation.',
|
|
9933
|
+
},
|
|
9934
|
+
{
|
|
9935
|
+
id: 'preview',
|
|
9936
|
+
label: 'Preview',
|
|
9937
|
+
status: previewReady || (reuseRecommended && !hasSql) ? 'passed' : run.status === 'error' ? 'blocked' : 'pending',
|
|
9938
|
+
detail: reuseRecommended && !hasSql
|
|
9939
|
+
? 'Certified block reuse does not require a new raw SQL preview.'
|
|
9940
|
+
: previewReady
|
|
9941
|
+
? `Preview returned ${preview.rowCount.toLocaleString()} row${preview.rowCount === 1 ? '' : 's'}.`
|
|
9942
|
+
: run.status === 'error'
|
|
9943
|
+
? (run.error ?? 'Preview failed.')
|
|
9944
|
+
: 'Run a bounded preview to validate the query shape.',
|
|
9945
|
+
},
|
|
9946
|
+
{
|
|
9947
|
+
id: 'evidence',
|
|
9948
|
+
label: 'Evidence',
|
|
9949
|
+
status: evidenceReady || reuseRecommended ? 'passed' : 'warning',
|
|
9950
|
+
detail: evidenceReady
|
|
9951
|
+
? 'Metadata evidence or context pack is attached.'
|
|
9952
|
+
: reuseRecommended
|
|
9953
|
+
? 'Reuse evidence is captured in the AI recommendation.'
|
|
9954
|
+
: 'No context evidence is attached yet.',
|
|
9955
|
+
},
|
|
9956
|
+
{
|
|
9957
|
+
id: 'duplicates',
|
|
9958
|
+
label: 'Reuse check',
|
|
9959
|
+
status: duplicateChecked ? (reuseRecommended || promotionReviewRequired ? 'warning' : 'passed') : promoted ? 'warning' : 'pending',
|
|
9960
|
+
detail: duplicateChecked
|
|
9961
|
+
? `Promotion decision: ${promotionAction ?? 'review required'}.`
|
|
9962
|
+
: promoted
|
|
9963
|
+
? 'Draft was created without stored duplicate evidence.'
|
|
9964
|
+
: 'Duplicate check runs when creating a DQL draft.',
|
|
9965
|
+
},
|
|
9966
|
+
{
|
|
9967
|
+
id: 'parameters',
|
|
9968
|
+
label: 'Parameter review',
|
|
9969
|
+
status: reuseRecommended && !hasSql ? 'passed' : !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
|
|
9970
|
+
detail: reuseRecommended && !hasSql
|
|
9971
|
+
? 'Use the certified block parameters; no new SQL parameterization is required.'
|
|
9972
|
+
: !hasSql
|
|
9973
|
+
? 'Add SQL before checking runtime parameters.'
|
|
9974
|
+
: dynamicParameters.length > 0
|
|
9975
|
+
? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
|
|
9976
|
+
: staticParameters.length > 0
|
|
9977
|
+
? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
|
|
9978
|
+
: 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
|
|
9979
|
+
},
|
|
9980
|
+
{
|
|
9981
|
+
id: 'dql_draft',
|
|
9982
|
+
label: 'DQL draft',
|
|
9983
|
+
status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasReviewedSql && evidenceReady ? 'pending' : 'blocked',
|
|
9984
|
+
detail: reuseRecommended && !hasDraft
|
|
9985
|
+
? 'No new draft is required because an existing DQL block should be reused.'
|
|
9986
|
+
: hasDraft
|
|
9987
|
+
? `Draft saved at ${run.draftBlockPath}.`
|
|
9988
|
+
: !hasReviewedSql
|
|
9989
|
+
? 'Save reviewed SQL before DQL draft creation.'
|
|
9990
|
+
: evidenceReady
|
|
9991
|
+
? 'Create a draft after SQL review.'
|
|
9992
|
+
: 'Save metadata evidence before creating a reusable DQL draft.',
|
|
9993
|
+
},
|
|
9994
|
+
];
|
|
9995
|
+
const readyForDqlDraft = questionReady && hasReviewedSql && evidenceReady && run.status !== 'error';
|
|
9996
|
+
const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
|
|
9997
|
+
return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
|
|
9998
|
+
}
|
|
9999
|
+
function buildNotebookResearchPlan(input) {
|
|
10000
|
+
const reviewedSql = notebookResearchString(input.reviewedSql);
|
|
10001
|
+
const generatedSql = notebookResearchString(input.generatedSql);
|
|
10002
|
+
const sql = reviewedSql ?? generatedSql;
|
|
10003
|
+
const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
|
|
10004
|
+
const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
|
|
10005
|
+
? input.evidence
|
|
10006
|
+
: {};
|
|
10007
|
+
const trustStatus = evidence.trustStatus && typeof evidence.trustStatus === 'object' && !Array.isArray(evidence.trustStatus)
|
|
10008
|
+
? evidence.trustStatus
|
|
10009
|
+
: {};
|
|
10010
|
+
const allowedSqlContext = evidence.allowedSqlContext && typeof evidence.allowedSqlContext === 'object' && !Array.isArray(evidence.allowedSqlContext)
|
|
10011
|
+
? evidence.allowedSqlContext
|
|
10012
|
+
: {};
|
|
10013
|
+
const previewRows = input.resultPreview?.rowCount ?? input.resultPreview?.rows.length;
|
|
10014
|
+
const hasEvidence = notebookResearchEvidenceCount(input.evidence) > 0 || Boolean(input.run.contextPackId);
|
|
10015
|
+
const hasPreview = previewRows !== undefined && (previewRows > 0 || Boolean(input.resultPreview?.columns.length));
|
|
10016
|
+
const promotionAction = input.run.dqlPromotionAction ?? input.run.dqlPromotion?.recommendedAction;
|
|
10017
|
+
const missingContext = Array.isArray(evidence.missingContext) ? evidence.missingContext : [];
|
|
10018
|
+
const relations = Array.isArray(allowedSqlContext.relations) ? allowedSqlContext.relations : [];
|
|
10019
|
+
return {
|
|
10020
|
+
sqlState: reviewedSql ? 'reviewed' : generatedSql ? 'generated' : 'missing',
|
|
10021
|
+
grain: sql ? extractDqlGenerationGroupByFields(sql)[0] : undefined,
|
|
10022
|
+
parameterPolicy: (parameterized?.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
|
|
10023
|
+
name: entry.name,
|
|
10024
|
+
policy: entry.policy,
|
|
10025
|
+
})),
|
|
10026
|
+
allowedFilters: (parameterized?.allowedFilters ?? []).slice(0, 16),
|
|
10027
|
+
evidence: {
|
|
10028
|
+
trustLabel: notebookResearchString(trustStatus.label),
|
|
10029
|
+
contextPackId: notebookResearchString(evidence.contextPackId) ?? input.run.contextPackId,
|
|
10030
|
+
evidenceCount: notebookResearchEvidenceCount(input.evidence),
|
|
10031
|
+
relationCount: relations.length,
|
|
10032
|
+
missingContextCount: missingContext.length,
|
|
10033
|
+
},
|
|
10034
|
+
preview: {
|
|
10035
|
+
status: input.previewError ? 'error' : hasPreview ? 'ready' : 'not_run',
|
|
10036
|
+
...(previewRows === undefined ? {} : { rowCount: previewRows }),
|
|
10037
|
+
},
|
|
10038
|
+
promotion: {
|
|
10039
|
+
path: notebookResearchPlanPromotionPath({
|
|
10040
|
+
hasSql: Boolean(sql),
|
|
10041
|
+
hasEvidence,
|
|
10042
|
+
hasPreview,
|
|
10043
|
+
previewError: input.previewError,
|
|
10044
|
+
draftBlockPath: input.run.draftBlockPath,
|
|
10045
|
+
promotionAction,
|
|
10046
|
+
}),
|
|
10047
|
+
duplicateDecision: promotionAction,
|
|
10048
|
+
},
|
|
10049
|
+
reviewFocus: notebookResearchPlanFocus(input.run.intent, {
|
|
10050
|
+
hasSql: Boolean(sql),
|
|
10051
|
+
hasEvidence,
|
|
10052
|
+
hasPreview,
|
|
10053
|
+
dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
|
|
10054
|
+
missingContextCount: missingContext.length,
|
|
10055
|
+
promotionAction,
|
|
10056
|
+
}),
|
|
10057
|
+
generatedAt: new Date().toISOString(),
|
|
10058
|
+
};
|
|
10059
|
+
}
|
|
10060
|
+
function notebookResearchPlanPromotionPath(input) {
|
|
10061
|
+
if (input.promotionAction === 'reuse_existing')
|
|
10062
|
+
return 'reuse_existing';
|
|
10063
|
+
if (!input.hasSql)
|
|
10064
|
+
return 'needs_sql';
|
|
10065
|
+
if (!input.hasEvidence)
|
|
10066
|
+
return 'review_context';
|
|
10067
|
+
if (input.previewError || !input.hasPreview)
|
|
10068
|
+
return 'run_preview';
|
|
10069
|
+
if (input.draftBlockPath)
|
|
10070
|
+
return 'open_certification';
|
|
10071
|
+
return 'create_dql_draft';
|
|
10072
|
+
}
|
|
10073
|
+
function notebookResearchPlanFocus(intent, input) {
|
|
10074
|
+
const focus = new Set();
|
|
10075
|
+
if (!input.hasSql)
|
|
10076
|
+
focus.add('Add reviewed SQL or configure AI SQL generation.');
|
|
10077
|
+
if (!input.hasEvidence)
|
|
10078
|
+
focus.add('Save metadata context evidence before DQL promotion.');
|
|
10079
|
+
if (!input.hasPreview)
|
|
10080
|
+
focus.add('Run a bounded preview and inspect result shape.');
|
|
10081
|
+
if (input.dynamicParameterCount === 0)
|
|
10082
|
+
focus.add('Confirm whether this should be static or parameterized.');
|
|
10083
|
+
if (input.missingContextCount > 0)
|
|
10084
|
+
focus.add('Resolve missing metadata context before certification.');
|
|
10085
|
+
if (input.promotionAction === 'reuse_existing')
|
|
10086
|
+
focus.add('Reuse the existing certified block or document replacement rationale.');
|
|
10087
|
+
const pattern = notebookResearchIntentPattern(intent);
|
|
10088
|
+
const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus : [];
|
|
10089
|
+
for (const item of reviewFocus.slice(0, 4))
|
|
10090
|
+
focus.add(String(item));
|
|
10091
|
+
return Array.from(focus).slice(0, 8);
|
|
10092
|
+
}
|
|
10093
|
+
function notebookResearchPreviewInfo(value) {
|
|
10094
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
10095
|
+
return { hasPreview: false, rowCount: 0 };
|
|
10096
|
+
const record = value;
|
|
10097
|
+
const rowCount = typeof record.rowCount === 'number' && Number.isFinite(record.rowCount)
|
|
10098
|
+
? record.rowCount
|
|
10099
|
+
: Array.isArray(record.rows)
|
|
10100
|
+
? record.rows.length
|
|
10101
|
+
: 0;
|
|
10102
|
+
const hasColumns = Array.isArray(record.columns) && record.columns.length > 0;
|
|
10103
|
+
return { hasPreview: hasColumns || rowCount > 0, rowCount };
|
|
10104
|
+
}
|
|
10105
|
+
function notebookResearchEvidenceCount(value) {
|
|
10106
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
10107
|
+
return 0;
|
|
10108
|
+
const record = value;
|
|
10109
|
+
const selected = Array.isArray(record.selectedEvidence) ? record.selectedEvidence.length : 0;
|
|
10110
|
+
const citations = Array.isArray(record.citations) ? record.citations.length : 0;
|
|
10111
|
+
const roles = Array.isArray(record.evidenceRoles) ? record.evidenceRoles.length : 0;
|
|
10112
|
+
return selected + citations + roles;
|
|
10113
|
+
}
|
|
10114
|
+
function notebookResearchInteger(value, fallback, min, max) {
|
|
10115
|
+
if (typeof value !== 'string' || !value.trim())
|
|
10116
|
+
return fallback;
|
|
10117
|
+
const parsed = Number(value);
|
|
10118
|
+
if (!Number.isFinite(parsed))
|
|
10119
|
+
return fallback;
|
|
10120
|
+
return Math.max(min, Math.min(max, Math.floor(parsed)));
|
|
10121
|
+
}
|
|
10122
|
+
function notebookExecutionContext(value) {
|
|
10123
|
+
if (!value || typeof value !== 'object')
|
|
10124
|
+
return null;
|
|
10125
|
+
const raw = value;
|
|
10126
|
+
const notebookPath = notebookResearchString(raw.notebookPath);
|
|
10127
|
+
if (!notebookPath)
|
|
10128
|
+
return null;
|
|
10129
|
+
return {
|
|
10130
|
+
notebookPath,
|
|
10131
|
+
cellId: notebookResearchString(raw.cellId),
|
|
10132
|
+
cellName: notebookResearchString(raw.cellName),
|
|
10133
|
+
researchRunId: notebookResearchString(raw.researchRunId),
|
|
10134
|
+
source: notebookResearchString(raw.source),
|
|
10135
|
+
};
|
|
10136
|
+
}
|
|
10137
|
+
function notebookResearchSelectedContext(run, context) {
|
|
10138
|
+
return {
|
|
10139
|
+
activeSurface: 'notebook',
|
|
10140
|
+
notebookPath: run.notebookPath,
|
|
10141
|
+
domain: run.domain,
|
|
10142
|
+
sourceCellId: run.sourceCellId,
|
|
10143
|
+
sourceCellName: run.sourceCellName,
|
|
10144
|
+
context,
|
|
10145
|
+
};
|
|
10146
|
+
}
|
|
10147
|
+
function notebookResearchIntentPattern(intent) {
|
|
10148
|
+
switch (intent) {
|
|
10149
|
+
case 'diagnose_change':
|
|
10150
|
+
return {
|
|
10151
|
+
label: 'Change diagnosis',
|
|
10152
|
+
dqlTarget: 'driver or trend block',
|
|
10153
|
+
sqlFocus: ['baseline period', 'comparison period', 'metric delta', 'driver dimensions'],
|
|
10154
|
+
reviewFocus: ['time grain', 'comparison filters', 'metric definition', 'explainability evidence'],
|
|
10155
|
+
};
|
|
10156
|
+
case 'driver_breakdown':
|
|
10157
|
+
return {
|
|
10158
|
+
label: 'Driver breakdown',
|
|
10159
|
+
dqlTarget: 'ranking or contribution block',
|
|
10160
|
+
sqlFocus: ['contribution metric', 'ranking', 'tie breaker', 'segment dimensions'],
|
|
10161
|
+
reviewFocus: ['metric grain', 'ranking direction', 'parameterized filters', 'duplicate block match'],
|
|
10162
|
+
};
|
|
10163
|
+
case 'segment_compare':
|
|
10164
|
+
return {
|
|
10165
|
+
label: 'Segment compare',
|
|
10166
|
+
dqlTarget: 'comparison block',
|
|
10167
|
+
sqlFocus: ['shared metric', 'segment fields', 'normalized comparisons', 'deltas'],
|
|
10168
|
+
reviewFocus: ['segment definitions', 'filter compatibility', 'comparison grain', 'semantic metric alignment'],
|
|
10169
|
+
};
|
|
10170
|
+
case 'entity_drilldown':
|
|
10171
|
+
return {
|
|
10172
|
+
label: 'Entity drilldown',
|
|
10173
|
+
dqlTarget: 'entity profile or drilldown block',
|
|
10174
|
+
sqlFocus: ['entity grain', 'stable identifier', 'detail fields', 'supporting metrics'],
|
|
10175
|
+
reviewFocus: ['entity contract', 'primary key', 'allowed filters', 'downstream drill path'],
|
|
10176
|
+
};
|
|
10177
|
+
case 'anomaly_investigation':
|
|
10178
|
+
return {
|
|
10179
|
+
label: 'Anomaly investigation',
|
|
10180
|
+
dqlTarget: 'monitoring or exception block',
|
|
10181
|
+
sqlFocus: ['observed value', 'expected range', 'anomaly rule', 'supporting dimensions'],
|
|
10182
|
+
reviewFocus: ['threshold definition', 'time window', 'false-positive risk', 'evidence fields'],
|
|
10183
|
+
};
|
|
10184
|
+
case 'trust_gap_review':
|
|
10185
|
+
return {
|
|
10186
|
+
label: 'Trust review',
|
|
10187
|
+
dqlTarget: 'replacement or validation block',
|
|
10188
|
+
sqlFocus: ['definition comparison', 'lineage trace', 'duplicate candidates', 'conflict evidence'],
|
|
10189
|
+
reviewFocus: ['certified block match', 'semantic metric match', 'replacement reason', 'governance blockers'],
|
|
10190
|
+
};
|
|
10191
|
+
default:
|
|
10192
|
+
return {
|
|
10193
|
+
label: 'Ad hoc analysis',
|
|
10194
|
+
dqlTarget: 'custom block',
|
|
10195
|
+
sqlFocus: ['business question', 'grain', 'metrics', 'dimensions', 'filters'],
|
|
10196
|
+
reviewFocus: ['reusability', 'parameterization', 'lineage evidence', 'DQL promotion path'],
|
|
10197
|
+
};
|
|
10198
|
+
}
|
|
10199
|
+
}
|
|
10200
|
+
function notebookResearchAgentPrompt(question, intent) {
|
|
10201
|
+
const pattern = notebookResearchIntentPattern(intent);
|
|
10202
|
+
const sqlFocus = Array.isArray(pattern.sqlFocus) ? pattern.sqlFocus.join(', ') : 'business logic';
|
|
10203
|
+
const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus.join(', ') : 'review evidence';
|
|
10204
|
+
return [
|
|
10205
|
+
`Research question: ${question}`,
|
|
10206
|
+
`Research pattern: ${pattern.label ?? intent}`,
|
|
10207
|
+
`DQL target: ${pattern.dqlTarget ?? 'custom block'}`,
|
|
10208
|
+
`SQL focus: ${sqlFocus}`,
|
|
10209
|
+
`Review focus: ${reviewFocus}`,
|
|
10210
|
+
'Generate read-only SQL that can become a reusable, parameterized DQL block after human review.',
|
|
10211
|
+
].join('\n');
|
|
10212
|
+
}
|
|
10213
|
+
function notebookResearchRouteDecisionForRun(run, routeDecision, sqlForPreview) {
|
|
10214
|
+
if (!routeDecision)
|
|
10215
|
+
return undefined;
|
|
10216
|
+
const hasSelectedNotebookSql = Boolean(run.sourceCellId && notebookResearchString(sqlForPreview));
|
|
10217
|
+
if (!hasSelectedNotebookSql || routeDecision.route !== 'certified')
|
|
10218
|
+
return routeDecision;
|
|
10219
|
+
const certifiedEvidence = routeDecision.selectedEvidence.find((item) => item.role === 'exact_certified_answer')
|
|
10220
|
+
?? routeDecision.selectedEvidence.find((item) => item.role === 'certified_context');
|
|
10221
|
+
const certifiedName = certifiedEvidence?.name ?? routeDecision.certifiedApplicability?.objectKey ?? 'matching certified artifact';
|
|
10222
|
+
return {
|
|
10223
|
+
...routeDecision,
|
|
10224
|
+
route: 'research',
|
|
10225
|
+
trustLabel: routeDecision.trustLabel === 'certified' ? 'mixed' : routeDecision.trustLabel,
|
|
10226
|
+
reviewStatus: 'needs_review',
|
|
10227
|
+
exactObjectKey: undefined,
|
|
10228
|
+
reason: `Selected notebook SQL from "${run.sourceCellName ?? run.sourceCellId}" is the source for this research run. Certified artifact "${certifiedName}" is duplicate/reuse evidence until the reviewed SQL, grain, parameters, and preview are accepted.`,
|
|
10229
|
+
selectedEvidence: routeDecision.selectedEvidence.map((item) => item.role === 'exact_certified_answer'
|
|
10230
|
+
? {
|
|
10231
|
+
...item,
|
|
10232
|
+
role: 'certified_context',
|
|
10233
|
+
reason: 'Certified block is reuse or duplicate evidence for the selected notebook SQL; it is not an automatic answer for this research run.',
|
|
10234
|
+
}
|
|
10235
|
+
: item),
|
|
10236
|
+
missingContext: routeDecision.missingContext.length > 0
|
|
10237
|
+
? routeDecision.missingContext
|
|
10238
|
+
: [{
|
|
10239
|
+
kind: 'metadata',
|
|
10240
|
+
severity: 'warning',
|
|
10241
|
+
message: 'Selected notebook SQL requires human review before DQL promotion, even when certified context exists.',
|
|
10242
|
+
}],
|
|
10243
|
+
};
|
|
10244
|
+
}
|
|
10245
|
+
function notebookResearchContextPreview(contextPack) {
|
|
10246
|
+
return {
|
|
10247
|
+
contextPackId: contextPack.id,
|
|
10248
|
+
trustLabel: contextPack.trustLabel,
|
|
10249
|
+
routeDecision: contextPack.routeDecision
|
|
10250
|
+
? {
|
|
10251
|
+
route: contextPack.routeDecision.route,
|
|
10252
|
+
intent: contextPack.routeDecision.intent,
|
|
10253
|
+
reason: contextPack.routeDecision.reason,
|
|
10254
|
+
}
|
|
10255
|
+
: undefined,
|
|
10256
|
+
evidence: contextPack.evidenceRoles.slice(0, 16).map((item) => ({
|
|
10257
|
+
objectKey: item.objectKey,
|
|
10258
|
+
objectType: item.objectType,
|
|
10259
|
+
name: item.name,
|
|
10260
|
+
role: item.role,
|
|
10261
|
+
reason: item.reason,
|
|
10262
|
+
})),
|
|
10263
|
+
summaries: contextPack.evidenceSummaries.slice(0, 12).map((item) => ({
|
|
10264
|
+
title: item.title,
|
|
10265
|
+
detail: item.detail,
|
|
10266
|
+
objectType: item.objectType,
|
|
10267
|
+
reason: item.reason,
|
|
10268
|
+
})),
|
|
10269
|
+
relations: contextPack.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
|
|
10270
|
+
relation: relation.relation,
|
|
10271
|
+
name: relation.name,
|
|
10272
|
+
source: relation.source,
|
|
10273
|
+
columns: relation.columns.slice(0, 24).map((column) => {
|
|
10274
|
+
if (typeof column === 'string')
|
|
10275
|
+
return column;
|
|
10276
|
+
if (column && typeof column === 'object' && typeof column.name === 'string') {
|
|
10277
|
+
return String(column.name);
|
|
10278
|
+
}
|
|
10279
|
+
return String(column);
|
|
10280
|
+
}),
|
|
10281
|
+
})),
|
|
10282
|
+
missingContext: contextPack.missingContext.slice(0, 12).map((item) => ({
|
|
10283
|
+
kind: item.kind,
|
|
10284
|
+
message: item.message,
|
|
10285
|
+
severity: item.severity,
|
|
10286
|
+
})),
|
|
10287
|
+
warnings: contextPack.warnings.slice(0, 12),
|
|
10288
|
+
topRejected: contextPack.retrievalDiagnostics.topRejected.slice(0, 8).map((item) => ({
|
|
10289
|
+
name: item.name,
|
|
10290
|
+
objectType: item.objectType,
|
|
10291
|
+
reason: item.reason,
|
|
10292
|
+
score: item.score,
|
|
10293
|
+
})),
|
|
10294
|
+
counts: {
|
|
10295
|
+
objects: contextPack.objects.length,
|
|
10296
|
+
evidence: contextPack.evidenceRoles.length,
|
|
10297
|
+
relations: contextPack.allowedSqlContext.relations.length,
|
|
10298
|
+
warnings: contextPack.warnings.length + contextPack.missingContext.length,
|
|
10299
|
+
},
|
|
10300
|
+
};
|
|
10301
|
+
}
|
|
10302
|
+
function normalizeNotebookAgentResult(result) {
|
|
10303
|
+
const columns = Array.isArray(result.columns)
|
|
10304
|
+
? result.columns.map((column) => {
|
|
10305
|
+
if (typeof column === 'string')
|
|
10306
|
+
return column;
|
|
10307
|
+
if (column && typeof column === 'object' && typeof column.name === 'string') {
|
|
10308
|
+
return String(column.name);
|
|
10309
|
+
}
|
|
10310
|
+
return String(column);
|
|
10311
|
+
})
|
|
10312
|
+
: [];
|
|
10313
|
+
const rows = Array.isArray(result.rows)
|
|
10314
|
+
? result.rows
|
|
10315
|
+
.filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row)))
|
|
10316
|
+
.map((row) => row)
|
|
10317
|
+
: [];
|
|
10318
|
+
return {
|
|
10319
|
+
columns,
|
|
10320
|
+
rows,
|
|
10321
|
+
rowCount: typeof result.rowCount === 'number' ? result.rowCount : rows.length,
|
|
10322
|
+
executionTime: typeof result.executionTime === 'number' ? result.executionTime : 0,
|
|
10323
|
+
};
|
|
10324
|
+
}
|
|
10325
|
+
function notebookResearchSummary(question, result, error) {
|
|
10326
|
+
if (error)
|
|
10327
|
+
return `Research could not preview results for "${question}". ${error}`;
|
|
10328
|
+
if (result) {
|
|
10329
|
+
const rowCount = result.rowCount ?? result.rows.length;
|
|
10330
|
+
return `Research preview for "${question}" returned ${rowCount.toLocaleString()} row${rowCount === 1 ? '' : 's'} across ${result.columns.length.toLocaleString()} column${result.columns.length === 1 ? '' : 's'}.`;
|
|
10331
|
+
}
|
|
10332
|
+
return `Research plan created for "${question}". Add or generate SQL, then run a bounded preview.`;
|
|
10333
|
+
}
|
|
10334
|
+
function recordNotebookQueryRun(projectRoot, input) {
|
|
10335
|
+
try {
|
|
10336
|
+
recordQueryRun(projectRoot, {
|
|
10337
|
+
objectKey: input.objectKey,
|
|
10338
|
+
source: input.source,
|
|
10339
|
+
status: input.status,
|
|
10340
|
+
rowCount: input.rowCount,
|
|
10341
|
+
durationMs: input.durationMs,
|
|
10342
|
+
errorCode: input.errorCode,
|
|
10343
|
+
payload: {
|
|
10344
|
+
notebookPath: input.notebookPath,
|
|
10345
|
+
cellId: input.cellId,
|
|
10346
|
+
cellName: input.cellName,
|
|
10347
|
+
researchRunId: input.researchRunId,
|
|
10348
|
+
contextPackId: input.contextPackId,
|
|
10349
|
+
sql: input.sql ? compactSqlForRunHistory(input.sql) : undefined,
|
|
10350
|
+
},
|
|
10351
|
+
});
|
|
10352
|
+
}
|
|
10353
|
+
catch {
|
|
10354
|
+
// Query history is advisory metadata and must not block notebook execution.
|
|
10355
|
+
}
|
|
10356
|
+
}
|
|
10357
|
+
function updateNotebookResearchFromCellExecution(projectRoot, execContext, input) {
|
|
10358
|
+
if (!execContext.notebookPath || (!execContext.cellId && !execContext.researchRunId))
|
|
10359
|
+
return;
|
|
10360
|
+
const storage = new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
|
|
10361
|
+
try {
|
|
10362
|
+
const page = storage.listRunsPage({
|
|
10363
|
+
notebookPath: execContext.notebookPath,
|
|
10364
|
+
sort: 'updated_desc',
|
|
10365
|
+
limit: 500,
|
|
10366
|
+
});
|
|
10367
|
+
const matches = page.runs.filter((run) => {
|
|
10368
|
+
if (execContext.researchRunId && run.id === execContext.researchRunId)
|
|
10369
|
+
return true;
|
|
10370
|
+
return Boolean(execContext.cellId && run.sourceCellId === execContext.cellId);
|
|
10371
|
+
});
|
|
10372
|
+
const now = new Date().toISOString();
|
|
10373
|
+
for (const run of matches.slice(0, 20)) {
|
|
10374
|
+
if (input.status === 'success') {
|
|
10375
|
+
storage.updateRun(run.id, {
|
|
10376
|
+
status: 'ready',
|
|
10377
|
+
resultPreview: input.resultPreview,
|
|
10378
|
+
summary: notebookResearchSummary(run.question, input.resultPreview, undefined),
|
|
10379
|
+
recommendation: run.recommendation ?? 'Review SQL, metadata evidence, grain, filters, and duplicate matches before DQL draft promotion.',
|
|
10380
|
+
error: '',
|
|
10381
|
+
lastRunAt: now,
|
|
10382
|
+
});
|
|
10383
|
+
}
|
|
10384
|
+
else {
|
|
10385
|
+
storage.updateRun(run.id, {
|
|
10386
|
+
status: 'error',
|
|
10387
|
+
error: input.error,
|
|
10388
|
+
summary: notebookResearchSummary(run.question, undefined, input.error),
|
|
10389
|
+
lastRunAt: now,
|
|
10390
|
+
});
|
|
10391
|
+
}
|
|
10392
|
+
}
|
|
10393
|
+
}
|
|
10394
|
+
catch {
|
|
10395
|
+
// Research run sync is advisory metadata and must not block notebook execution.
|
|
10396
|
+
}
|
|
10397
|
+
finally {
|
|
10398
|
+
storage.close();
|
|
10399
|
+
}
|
|
10400
|
+
}
|
|
10401
|
+
function compactSqlForRunHistory(sql) {
|
|
10402
|
+
const clean = sql.replace(/\s+/g, ' ').trim();
|
|
10403
|
+
return clean.length > 1200 ? `${clean.slice(0, 1197)}...` : clean;
|
|
10404
|
+
}
|
|
6856
10405
|
function buildAgentSchemaContextFromContextPack(question, contextPack) {
|
|
6857
10406
|
const byRelation = new Map();
|
|
6858
10407
|
const objectsByKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
|