@duckcodeailabs/dql-cli 1.6.17 → 1.6.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts +41 -1
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +297 -18
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +4 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +10 -1
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-CcpAv4lQ.js +3864 -0
- package/dist/assets/dql-notebook/assets/index-rJXLSheJ.js +3910 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/block-studio-import.d.ts +99 -3
- package/dist/block-studio-import.d.ts.map +1 -1
- package/dist/block-studio-import.js +772 -27
- package/dist/block-studio-import.js.map +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +12 -1
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts +2 -1
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +22 -4
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/certify.d.ts.map +1 -1
- package/dist/commands/certify.js +104 -10
- package/dist/commands/certify.js.map +1 -1
- package/dist/commands/compile.d.ts +1 -0
- package/dist/commands/compile.d.ts.map +1 -1
- package/dist/commands/compile.js +26 -4
- package/dist/commands/compile.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +345 -6
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/import.d.ts.map +1 -1
- package/dist/commands/import.js +45 -70
- package/dist/commands/import.js.map +1 -1
- package/dist/commands/lineage.d.ts +2 -0
- package/dist/commands/lineage.d.ts.map +1 -1
- package/dist/commands/lineage.js +111 -0
- package/dist/commands/lineage.js.map +1 -1
- package/dist/commands/migrate.d.ts +1 -0
- package/dist/commands/migrate.d.ts.map +1 -1
- package/dist/commands/migrate.js +106 -2
- package/dist/commands/migrate.js.map +1 -1
- package/dist/commands/new.d.ts.map +1 -1
- package/dist/commands/new.js +246 -9
- package/dist/commands/new.js.map +1 -1
- package/dist/commands/promote.d.ts +11 -0
- package/dist/commands/promote.d.ts.map +1 -0
- package/dist/commands/promote.js +236 -0
- package/dist/commands/promote.js.map +1 -0
- package/dist/commands/semantic.js +2 -2
- package/dist/commands/semantic.js.map +1 -1
- package/dist/index.js +48 -8
- package/dist/index.js.map +1 -1
- package/dist/llm/providers/claude-agent-sdk.js +1 -1
- package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
- package/dist/llm/tools.js +1 -1
- package/dist/llm/tools.js.map +1 -1
- package/dist/local-runtime.d.ts +176 -5
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +3703 -146
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/promote-from-draft.d.ts +11 -10
- package/dist/promote-from-draft.d.ts.map +1 -1
- package/dist/promote-from-draft.js +53 -30
- package/dist/promote-from-draft.js.map +1 -1
- package/dist/schedule/runner.js +2 -2
- package/dist/schedule/runner.js.map +1 -1
- package/dist/semantic-import.d.ts.map +1 -1
- package/dist/semantic-import.js +28 -12
- package/dist/semantic-import.js.map +1 -1
- package/package.json +11 -11
- package/dist/assets/dql-notebook/assets/index-BFTvljNh.js +0 -3835
package/dist/local-runtime.js
CHANGED
|
@@ -7,21 +7,50 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
|
|
|
7
7
|
import Anthropic from '@anthropic-ai/sdk';
|
|
8
8
|
import OpenAI from 'openai';
|
|
9
9
|
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
10
|
-
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
|
|
10
|
+
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
|
|
11
11
|
import { load as loadYaml } from 'js-yaml';
|
|
12
12
|
import { listBlockTemplates } from './block-templates.js';
|
|
13
13
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
14
14
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
15
|
-
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
|
|
16
|
-
import { handleAppsApi } from './apps-api.js';
|
|
15
|
+
import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordQueryRun, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
|
|
16
|
+
import { handleAppsApi, recommendVisualization } from './apps-api.js';
|
|
17
17
|
import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
18
18
|
import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
|
|
19
|
-
import { LocalAppStorage, defaultLocalAppsDbPath } from '@duckcodeailabs/dql-project';
|
|
20
|
-
import { Certifier } from '@duckcodeailabs/dql-governance';
|
|
19
|
+
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
20
|
+
import { Certifier, ENTERPRISE_RULES } from '@duckcodeailabs/dql-governance';
|
|
21
21
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
22
|
-
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
22
|
+
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
23
23
|
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
|
|
24
24
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
25
|
+
export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
|
|
26
|
+
const configured = projectConfig.semanticLayer;
|
|
27
|
+
const dbtProjectDir = projectConfig.dbt?.projectDir;
|
|
28
|
+
if (dbtProjectDir
|
|
29
|
+
&& (!configured || configured.provider === 'dql')
|
|
30
|
+
&& hasDbtSemanticArtifacts(projectRoot, dbtProjectDir)) {
|
|
31
|
+
return { provider: 'dbt', projectPath: dbtProjectDir };
|
|
32
|
+
}
|
|
33
|
+
return configured;
|
|
34
|
+
}
|
|
35
|
+
function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
|
|
36
|
+
const dbtRoot = resolve(projectRoot, dbtProjectDir);
|
|
37
|
+
if (existsSync(join(dbtRoot, 'target', 'semantic_manifest.json')))
|
|
38
|
+
return true;
|
|
39
|
+
const manifestPath = join(dbtRoot, 'target', 'manifest.json');
|
|
40
|
+
if (!existsSync(manifestPath))
|
|
41
|
+
return false;
|
|
42
|
+
try {
|
|
43
|
+
const parsed = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
44
|
+
if (parsed.semantic_models && typeof parsed.semantic_models === 'object')
|
|
45
|
+
return true;
|
|
46
|
+
if (parsed.metrics && typeof parsed.metrics === 'object')
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
25
54
|
export async function startLocalServer(opts) {
|
|
26
55
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
27
56
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
@@ -40,7 +69,7 @@ export async function startLocalServer(opts) {
|
|
|
40
69
|
const semanticLayerDir = join(projectRoot, 'semantic-layer');
|
|
41
70
|
let semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
42
71
|
const userPrefsPath = join(projectRoot, '.dql-user-prefs.json');
|
|
43
|
-
const semanticConfig = projectConfig
|
|
72
|
+
const semanticConfig = resolveProjectSemanticConfig(projectConfig, projectRoot);
|
|
44
73
|
let semanticLastSyncTime = null;
|
|
45
74
|
{
|
|
46
75
|
const semanticConnection = connection;
|
|
@@ -330,6 +359,430 @@ export async function startLocalServer(opts) {
|
|
|
330
359
|
providerUsed: governedAnswer.providerUsed,
|
|
331
360
|
};
|
|
332
361
|
};
|
|
362
|
+
const openNotebookResearchStorage = () => new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
|
|
363
|
+
const runNotebookResearch = async (storage, run, input = {}) => {
|
|
364
|
+
const question = notebookResearchString(input.question) || run.question;
|
|
365
|
+
const domain = notebookResearchString(input.domain) ?? run.domain;
|
|
366
|
+
const owner = notebookResearchString(input.owner) ?? run.owner;
|
|
367
|
+
const sourceCellFingerprint = notebookResearchString(input.sourceCellFingerprint) ?? run.sourceCellFingerprint;
|
|
368
|
+
const intent = input.intent ?? run.intent;
|
|
369
|
+
const context = input.context === undefined ? run.context : input.context;
|
|
370
|
+
let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
|
|
371
|
+
let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
|
|
372
|
+
const startedAt = new Date().toISOString();
|
|
373
|
+
storage.updateRun(run.id, {
|
|
374
|
+
domain,
|
|
375
|
+
owner,
|
|
376
|
+
sourceCellFingerprint,
|
|
377
|
+
question,
|
|
378
|
+
intent,
|
|
379
|
+
context,
|
|
380
|
+
generatedSql,
|
|
381
|
+
reviewedSql,
|
|
382
|
+
status: 'running',
|
|
383
|
+
reviewStatus: 'needs_review',
|
|
384
|
+
error: '',
|
|
385
|
+
});
|
|
386
|
+
try {
|
|
387
|
+
const contextPack = await buildLocalContextPack(projectRoot, {
|
|
388
|
+
question,
|
|
389
|
+
mode: 'question',
|
|
390
|
+
surface: 'notebook',
|
|
391
|
+
selectedContext: {
|
|
392
|
+
...notebookResearchSelectedContext(run, context),
|
|
393
|
+
domain,
|
|
394
|
+
owner,
|
|
395
|
+
intent,
|
|
396
|
+
researchPattern: notebookResearchIntentPattern(intent),
|
|
397
|
+
},
|
|
398
|
+
strictness: 'balanced',
|
|
399
|
+
limit: 100,
|
|
400
|
+
}).catch((error) => {
|
|
401
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
402
|
+
return {
|
|
403
|
+
id: '',
|
|
404
|
+
routeDecision: undefined,
|
|
405
|
+
evidenceRoles: [],
|
|
406
|
+
warnings: [`Context pack failed: ${message}`],
|
|
407
|
+
retrievalDiagnostics: { selectedEvidence: [] },
|
|
408
|
+
};
|
|
409
|
+
});
|
|
410
|
+
let governedAnswer;
|
|
411
|
+
let providerError;
|
|
412
|
+
const generationWarnings = [];
|
|
413
|
+
if (!generatedSql && !reviewedSql) {
|
|
414
|
+
const resolvedProvider = resolveDefaultLLMProvider(projectRoot);
|
|
415
|
+
const runner = resolvedProvider ? getLLMRunner(resolvedProvider) : null;
|
|
416
|
+
if (!resolvedProvider || !runner) {
|
|
417
|
+
generationWarnings.push('No AI provider is configured. Metadata context was saved as a research plan; paste SQL or configure an AI provider to generate candidate SQL.');
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
const controller = new AbortController();
|
|
421
|
+
try {
|
|
422
|
+
await runner.run({
|
|
423
|
+
provider: resolvedProvider,
|
|
424
|
+
messages: [{ role: 'user', content: notebookResearchAgentPrompt(question, intent) }],
|
|
425
|
+
upstream: {
|
|
426
|
+
cellId: `notebook-research:${run.notebookPath}:${run.id}`,
|
|
427
|
+
sql: JSON.stringify({
|
|
428
|
+
mode: 'notebook_research',
|
|
429
|
+
notebookPath: run.notebookPath,
|
|
430
|
+
sourceCellId: run.sourceCellId,
|
|
431
|
+
sourceCellName: run.sourceCellName,
|
|
432
|
+
owner,
|
|
433
|
+
intent,
|
|
434
|
+
researchPattern: notebookResearchIntentPattern(intent),
|
|
435
|
+
instruction: 'Generate review-required read-only SQL from the inspected metadata context. Execute only through bounded preview and keep the result uncertified until promoted to a DQL draft.',
|
|
436
|
+
context,
|
|
437
|
+
}, null, 2),
|
|
438
|
+
},
|
|
439
|
+
projectRoot,
|
|
440
|
+
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
441
|
+
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
442
|
+
getSchemaContext: getSchemaContextForAgent,
|
|
443
|
+
}, (turn) => {
|
|
444
|
+
if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
|
|
445
|
+
governedAnswer = turn.output;
|
|
446
|
+
}
|
|
447
|
+
if (turn.kind === 'error')
|
|
448
|
+
providerError = turn.message;
|
|
449
|
+
}, controller.signal);
|
|
450
|
+
}
|
|
451
|
+
catch (error) {
|
|
452
|
+
providerError = error instanceof Error ? error.message : String(error);
|
|
453
|
+
}
|
|
454
|
+
if (!governedAnswer) {
|
|
455
|
+
generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
|
|
456
|
+
}
|
|
457
|
+
else {
|
|
458
|
+
generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
|
|
459
|
+
if (!generatedSql) {
|
|
460
|
+
generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
const sqlForPreview = reviewedSql || generatedSql;
|
|
466
|
+
const warnings = [
|
|
467
|
+
...(contextPack.warnings ?? []),
|
|
468
|
+
...(governedAnswer?.validationWarnings ?? []),
|
|
469
|
+
...generationWarnings,
|
|
470
|
+
].filter(Boolean);
|
|
471
|
+
let resultPreview;
|
|
472
|
+
let previewError;
|
|
473
|
+
if (governedAnswer?.result?.rows && !reviewedSql) {
|
|
474
|
+
resultPreview = normalizeNotebookAgentResult(governedAnswer.result);
|
|
475
|
+
}
|
|
476
|
+
else if (sqlForPreview) {
|
|
477
|
+
try {
|
|
478
|
+
const previewSql = buildAgentPreviewSql(sqlForPreview);
|
|
479
|
+
const previewStart = Date.now();
|
|
480
|
+
resultPreview = await executeLocalSqlForStoredResult(previewSql);
|
|
481
|
+
recordNotebookQueryRun(projectRoot, {
|
|
482
|
+
notebookPath: run.notebookPath,
|
|
483
|
+
cellId: run.sourceCellId,
|
|
484
|
+
cellName: run.sourceCellName,
|
|
485
|
+
researchRunId: run.id,
|
|
486
|
+
source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
|
|
487
|
+
status: 'success',
|
|
488
|
+
rowCount: resultPreview.rowCount ?? resultPreview.rows.length,
|
|
489
|
+
durationMs: Date.now() - previewStart,
|
|
490
|
+
sql: sqlForPreview,
|
|
491
|
+
contextPackId: contextPack.id,
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
catch (error) {
|
|
495
|
+
previewError = error instanceof Error ? error.message : String(error);
|
|
496
|
+
recordNotebookQueryRun(projectRoot, {
|
|
497
|
+
notebookPath: run.notebookPath,
|
|
498
|
+
cellId: run.sourceCellId,
|
|
499
|
+
cellName: run.sourceCellName,
|
|
500
|
+
researchRunId: run.id,
|
|
501
|
+
source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
|
|
502
|
+
status: 'error',
|
|
503
|
+
errorCode: previewError,
|
|
504
|
+
sql: sqlForPreview,
|
|
505
|
+
contextPackId: contextPack.id,
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
const routeDecision = notebookResearchRouteDecisionForRun(run, contextPack.routeDecision, sqlForPreview);
|
|
510
|
+
const display = resultPreview
|
|
511
|
+
? recommendVisualization(projectRoot, {
|
|
512
|
+
prompt: question,
|
|
513
|
+
resultSchema: resultPreview.columns,
|
|
514
|
+
rowSample: resultPreview.rows.slice(0, 25),
|
|
515
|
+
defaultVisualization: governedAnswer?.suggestedViz,
|
|
516
|
+
})
|
|
517
|
+
: undefined;
|
|
518
|
+
const evidence = {
|
|
519
|
+
trustStatus: {
|
|
520
|
+
label: governedAnswer?.trustLabel
|
|
521
|
+
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
|
|
522
|
+
reviewRequired: true,
|
|
523
|
+
},
|
|
524
|
+
contextPackId: contextPack.id,
|
|
525
|
+
routeDecision,
|
|
526
|
+
selectedEvidence: contextPack.evidenceRoles?.slice(0, 24) ?? [],
|
|
527
|
+
evidenceRoles: contextPack.evidenceRoles?.slice(0, 24) ?? [],
|
|
528
|
+
evidenceSummaries: contextPack.evidenceSummaries?.slice(0, 16) ?? [],
|
|
529
|
+
allowedSqlContext: {
|
|
530
|
+
relations: (contextPack.allowedSqlContext?.relations ?? []).slice(0, 24).map((relation) => ({
|
|
531
|
+
relation: relation.relation,
|
|
532
|
+
name: relation.name,
|
|
533
|
+
source: relation.source,
|
|
534
|
+
columns: relation.columns.slice(0, 32).map((column) => {
|
|
535
|
+
if (typeof column === 'string')
|
|
536
|
+
return column;
|
|
537
|
+
if (column && typeof column === 'object' && typeof column.name === 'string') {
|
|
538
|
+
return String(column.name);
|
|
539
|
+
}
|
|
540
|
+
return String(column);
|
|
541
|
+
}),
|
|
542
|
+
})),
|
|
543
|
+
sourceBlockSql: contextPack.allowedSqlContext?.sourceBlockSql?.slice(0, 12) ?? [],
|
|
544
|
+
},
|
|
545
|
+
missingContext: contextPack.missingContext?.slice(0, 16) ?? [],
|
|
546
|
+
warnings: contextPack.warnings?.slice(0, 16) ?? [],
|
|
547
|
+
retrievalDiagnostics: contextPack.retrievalDiagnostics,
|
|
548
|
+
agentEvidence: governedAnswer?.evidence,
|
|
549
|
+
analysisPlan: governedAnswer?.analysisPlan,
|
|
550
|
+
citations: [
|
|
551
|
+
...(contextPack.citations ?? []),
|
|
552
|
+
...(governedAnswer?.citations ?? []),
|
|
553
|
+
].slice(0, 40),
|
|
554
|
+
};
|
|
555
|
+
const summary = notebookResearchString(governedAnswer?.answer)
|
|
556
|
+
?? notebookResearchString(governedAnswer?.text)
|
|
557
|
+
?? notebookResearchSummary(question, resultPreview, previewError);
|
|
558
|
+
const recommendation = previewError
|
|
559
|
+
? 'Review the SQL, selected metadata, and connection context before rerunning.'
|
|
560
|
+
: sqlForPreview
|
|
561
|
+
? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
|
|
562
|
+
: 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
|
|
563
|
+
const researchPlan = buildNotebookResearchPlan({
|
|
564
|
+
run,
|
|
565
|
+
evidence,
|
|
566
|
+
resultPreview,
|
|
567
|
+
previewError,
|
|
568
|
+
generatedSql,
|
|
569
|
+
reviewedSql,
|
|
570
|
+
routeDecision,
|
|
571
|
+
});
|
|
572
|
+
return storage.updateRun(run.id, {
|
|
573
|
+
domain,
|
|
574
|
+
owner,
|
|
575
|
+
sourceCellFingerprint,
|
|
576
|
+
question,
|
|
577
|
+
intent,
|
|
578
|
+
context,
|
|
579
|
+
status: previewError ? 'error' : 'ready',
|
|
580
|
+
summary,
|
|
581
|
+
recommendation,
|
|
582
|
+
resultPreview,
|
|
583
|
+
evidence,
|
|
584
|
+
researchPlan,
|
|
585
|
+
generatedSql,
|
|
586
|
+
reviewedSql,
|
|
587
|
+
display: display && display.ok ? display.display : undefined,
|
|
588
|
+
contextPackId: contextPack.id,
|
|
589
|
+
routeDecision,
|
|
590
|
+
warnings: [
|
|
591
|
+
...warnings,
|
|
592
|
+
...(display && !display.ok ? [display.error] : []),
|
|
593
|
+
...(display && display.ok ? display.warnings : []),
|
|
594
|
+
],
|
|
595
|
+
reviewStatus: 'needs_review',
|
|
596
|
+
error: previewError,
|
|
597
|
+
lastRunAt: startedAt,
|
|
598
|
+
}) ?? run;
|
|
599
|
+
}
|
|
600
|
+
catch (error) {
|
|
601
|
+
return storage.updateRun(run.id, {
|
|
602
|
+
domain,
|
|
603
|
+
owner,
|
|
604
|
+
sourceCellFingerprint,
|
|
605
|
+
status: 'error',
|
|
606
|
+
error: error instanceof Error ? error.message : String(error),
|
|
607
|
+
reviewStatus: 'needs_review',
|
|
608
|
+
lastRunAt: startedAt,
|
|
609
|
+
}) ?? run;
|
|
610
|
+
}
|
|
611
|
+
};
|
|
612
|
+
const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
|
|
613
|
+
const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
|
|
614
|
+
if (!sql)
|
|
615
|
+
throw new Error('Notebook research needs generated or reviewed SQL before DQL draft promotion.');
|
|
616
|
+
const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
|
|
617
|
+
const session = await createDqlGenerationSessionForProject(projectRoot, {
|
|
618
|
+
inputMode: 'upload',
|
|
619
|
+
sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
|
|
620
|
+
sourceKind: 'raw-sql-file',
|
|
621
|
+
domain: notebookResearchString(input.domain) ?? run.domain,
|
|
622
|
+
owner: notebookResearchString(input.owner),
|
|
623
|
+
tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
|
|
624
|
+
provider: input.provider,
|
|
625
|
+
}, semanticLayer);
|
|
626
|
+
const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
|
|
627
|
+
?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
|
|
628
|
+
const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
|
|
629
|
+
const updated = storage.markPromoted(run.id, {
|
|
630
|
+
draftBlockPath: draftPath,
|
|
631
|
+
dqlImportId: session.id,
|
|
632
|
+
dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
|
|
633
|
+
dqlPromotion: promotion,
|
|
634
|
+
}) ?? run;
|
|
635
|
+
const planned = storage.updateRun(updated.id, {
|
|
636
|
+
researchPlan: buildNotebookResearchPlan({
|
|
637
|
+
run: updated,
|
|
638
|
+
evidence: updated.evidence,
|
|
639
|
+
resultPreview: updated.resultPreview,
|
|
640
|
+
generatedSql: updated.generatedSql,
|
|
641
|
+
reviewedSql: updated.reviewedSql,
|
|
642
|
+
routeDecision: updated.routeDecision,
|
|
643
|
+
}),
|
|
644
|
+
}) ?? updated;
|
|
645
|
+
return { run: planned, session };
|
|
646
|
+
};
|
|
647
|
+
const checkNotebookResearchReuse = async (storage, run, input = {}) => {
|
|
648
|
+
const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
|
|
649
|
+
if (!sql)
|
|
650
|
+
throw new Error('Notebook research needs generated or reviewed SQL before reuse checking.');
|
|
651
|
+
const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}.sql`;
|
|
652
|
+
const match = await matchSqlForDqlReuse({
|
|
653
|
+
sql,
|
|
654
|
+
sourcePath,
|
|
655
|
+
name: run.title || 'Notebook research SQL',
|
|
656
|
+
domain: notebookResearchString(input.domain) ?? run.domain ?? 'uncategorized',
|
|
657
|
+
owner: notebookResearchString(input.owner) ?? 'analytics',
|
|
658
|
+
});
|
|
659
|
+
const promotion = {
|
|
660
|
+
importId: `reuse-check:${run.id}:${Date.now()}`,
|
|
661
|
+
candidateIds: ['reuse_check'],
|
|
662
|
+
recommendedAction: match.recommendedAction,
|
|
663
|
+
similarityMatches: match.similarityMatches.slice(0, 8).map(toNotebookPromotionMatch),
|
|
664
|
+
candidates: [{
|
|
665
|
+
id: 'reuse_check',
|
|
666
|
+
name: run.title || 'Notebook research SQL',
|
|
667
|
+
domain: notebookResearchString(input.domain) ?? run.domain,
|
|
668
|
+
reviewStatus: 'reuse_checked',
|
|
669
|
+
recommendedAction: match.recommendedAction,
|
|
670
|
+
similarityMatches: match.similarityMatches.slice(0, 5).map(toNotebookPromotionMatch),
|
|
671
|
+
parameterPolicy: match.parameterPolicy.slice(0, 16).map((entry) => ({
|
|
672
|
+
name: entry.name,
|
|
673
|
+
policy: entry.policy,
|
|
674
|
+
})),
|
|
675
|
+
allowedFilters: match.allowedFilters.slice(0, 16),
|
|
676
|
+
warnings: match.parameterDecisions
|
|
677
|
+
.filter((decision) => decision.reason)
|
|
678
|
+
.map((decision) => `${decision.name}: ${decision.reason}`)
|
|
679
|
+
.slice(0, 12),
|
|
680
|
+
}],
|
|
681
|
+
createdAt: new Date().toISOString(),
|
|
682
|
+
};
|
|
683
|
+
const checked = storage.updateRun(run.id, {
|
|
684
|
+
dqlPromotionAction: match.recommendedAction,
|
|
685
|
+
dqlPromotion: promotion,
|
|
686
|
+
researchPlan: buildNotebookResearchPlan({
|
|
687
|
+
run: {
|
|
688
|
+
...run,
|
|
689
|
+
dqlPromotionAction: match.recommendedAction,
|
|
690
|
+
dqlPromotion: promotion,
|
|
691
|
+
},
|
|
692
|
+
evidence: run.evidence,
|
|
693
|
+
resultPreview: run.resultPreview,
|
|
694
|
+
generatedSql: run.generatedSql,
|
|
695
|
+
reviewedSql: run.reviewedSql,
|
|
696
|
+
routeDecision: run.routeDecision,
|
|
697
|
+
}),
|
|
698
|
+
}) ?? run;
|
|
699
|
+
return { run: checked, promotion, match };
|
|
700
|
+
};
|
|
701
|
+
const matchSqlForDqlReuse = async (input) => {
|
|
702
|
+
const sql = input.sql.trim();
|
|
703
|
+
if (!sql)
|
|
704
|
+
throw new Error('Missing SQL for reuse check.');
|
|
705
|
+
const sourceTables = extractSqlTablesLight(sql);
|
|
706
|
+
const candidate = {
|
|
707
|
+
id: 'match_sql',
|
|
708
|
+
sourceKind: 'raw-sql-file',
|
|
709
|
+
sourcePath: input.sourcePath ?? 'pasted.sql',
|
|
710
|
+
name: input.name ?? 'SQL match preview',
|
|
711
|
+
domain: input.domain ?? 'imported',
|
|
712
|
+
description: 'SQL match preview candidate.',
|
|
713
|
+
owner: input.owner ?? 'analytics',
|
|
714
|
+
tags: [],
|
|
715
|
+
sql,
|
|
716
|
+
dqlSource: '',
|
|
717
|
+
validation: null,
|
|
718
|
+
preview: null,
|
|
719
|
+
lineage: {
|
|
720
|
+
sourceTables,
|
|
721
|
+
parameters: [],
|
|
722
|
+
warnings: [],
|
|
723
|
+
statementIndex: 1,
|
|
724
|
+
totalStatements: 1,
|
|
725
|
+
},
|
|
726
|
+
confidence: 0.8,
|
|
727
|
+
splitStrategy: 'manual',
|
|
728
|
+
warnings: [],
|
|
729
|
+
conversionNotes: [],
|
|
730
|
+
aiAssistance: [],
|
|
731
|
+
reviewStatus: 'draft',
|
|
732
|
+
};
|
|
733
|
+
const evidence = deterministicDqlGenerationEvidence(candidate);
|
|
734
|
+
const patch = deterministicDqlGenerationPatch(candidate, evidence);
|
|
735
|
+
const contextPack = await buildDqlGenerationContextPack(projectRoot, { ...candidate, sql: patch.sql ?? sql }).catch(() => null);
|
|
736
|
+
const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
|
|
737
|
+
return {
|
|
738
|
+
parameterDecisions: patch.parameterDecisions ?? [],
|
|
739
|
+
parameterPolicy: patch.parameterPolicy ?? [],
|
|
740
|
+
filterBindings: patch.filterBindings ?? [],
|
|
741
|
+
allowedFilters: patch.allowedFilters ?? [],
|
|
742
|
+
parameterizedSql: patch.sql ?? sql,
|
|
743
|
+
similarityMatches: similarity.matches,
|
|
744
|
+
recommendedAction: similarity.recommendedAction,
|
|
745
|
+
};
|
|
746
|
+
};
|
|
747
|
+
const buildNotebookDqlPromotionSummary = (session, draftBlockPath) => {
|
|
748
|
+
const primary = draftBlockPath
|
|
749
|
+
? session.candidates.find((candidate) => candidate.draftSave?.path === draftBlockPath || candidate.savedPath === draftBlockPath) ?? session.candidates[0]
|
|
750
|
+
: session.candidates[0];
|
|
751
|
+
return {
|
|
752
|
+
importId: session.id,
|
|
753
|
+
candidateIds: session.candidates.map((candidate) => candidate.id),
|
|
754
|
+
draftBlockPath,
|
|
755
|
+
recommendedAction: primary?.recommendedAction,
|
|
756
|
+
similarityMatches: (primary?.similarityMatches ?? []).slice(0, 8).map(toNotebookPromotionMatch),
|
|
757
|
+
candidates: session.candidates.map((candidate) => ({
|
|
758
|
+
id: candidate.id,
|
|
759
|
+
name: candidate.name,
|
|
760
|
+
domain: candidate.domain,
|
|
761
|
+
draftPath: candidate.draftSave?.path,
|
|
762
|
+
savedPath: candidate.savedPath,
|
|
763
|
+
reviewStatus: candidate.reviewStatus,
|
|
764
|
+
recommendedAction: candidate.recommendedAction,
|
|
765
|
+
similarityMatches: (candidate.similarityMatches ?? []).slice(0, 5).map(toNotebookPromotionMatch),
|
|
766
|
+
parameterPolicy: (candidate.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
|
|
767
|
+
name: entry.name,
|
|
768
|
+
policy: entry.policy,
|
|
769
|
+
})),
|
|
770
|
+
allowedFilters: (candidate.allowedFilters ?? []).slice(0, 16),
|
|
771
|
+
warnings: [...(candidate.warnings ?? []), ...(candidate.lineage.warnings ?? [])].slice(0, 12),
|
|
772
|
+
})),
|
|
773
|
+
createdAt: new Date().toISOString(),
|
|
774
|
+
};
|
|
775
|
+
};
|
|
776
|
+
const toNotebookPromotionMatch = (match) => ({
|
|
777
|
+
kind: match.kind,
|
|
778
|
+
objectKey: match.objectKey,
|
|
779
|
+
name: match.name,
|
|
780
|
+
status: match.status,
|
|
781
|
+
source: match.source,
|
|
782
|
+
score: match.score,
|
|
783
|
+
reason: match.reason,
|
|
784
|
+
recommendedAction: match.recommendedAction,
|
|
785
|
+
});
|
|
333
786
|
// SSE clients for /api/watch hot-reload
|
|
334
787
|
const sseClients = new Set();
|
|
335
788
|
// Watch notebooks/, workbooks/, semantic-layer/, and data/ dirs for changes
|
|
@@ -398,8 +851,28 @@ export async function startLocalServer(opts) {
|
|
|
398
851
|
if (validated.reviewStatus === 'rejected') {
|
|
399
852
|
errors.unshift('Candidate was rejected.');
|
|
400
853
|
}
|
|
854
|
+
const duplicateBlocker = duplicateCertificationBlocker(validated);
|
|
855
|
+
if (duplicateBlocker)
|
|
856
|
+
errors.unshift(duplicateBlocker);
|
|
401
857
|
return { candidate: validated, errors };
|
|
402
858
|
};
|
|
859
|
+
const duplicateCertificationBlocker = (candidate) => {
|
|
860
|
+
const match = candidate.similarityMatches?.[0];
|
|
861
|
+
if (!match)
|
|
862
|
+
return null;
|
|
863
|
+
const duplicateKind = match.kind === 'exact_sql_match'
|
|
864
|
+
|| match.kind === 'parameterized_duplicate'
|
|
865
|
+
|| (match.kind === 'business_duplicate' && match.status === 'certified' && match.score >= 0.76);
|
|
866
|
+
if (!duplicateKind)
|
|
867
|
+
return null;
|
|
868
|
+
const replacementFor = new Set((candidate.replacementFor ?? []).map((value) => value.toLowerCase()));
|
|
869
|
+
const documentedReplacement = candidate.recommendedAction === 'create_replacement'
|
|
870
|
+
|| replacementFor.has(match.name.toLowerCase())
|
|
871
|
+
|| (match.objectKey && replacementFor.has(match.objectKey.toLowerCase()));
|
|
872
|
+
if (documentedReplacement)
|
|
873
|
+
return null;
|
|
874
|
+
return `Likely duplicate of ${match.name} (${match.kind}, ${Math.round(match.score * 100)}% match). Reuse the existing block, extend it, or document this draft as a replacement before certification.`;
|
|
875
|
+
};
|
|
403
876
|
const runBlockStudioPreviewSource = async (source, targetConnection) => {
|
|
404
877
|
const activeConnection = requireActiveConnection(targetConnection);
|
|
405
878
|
let tableMapping;
|
|
@@ -504,7 +977,50 @@ export async function startLocalServer(opts) {
|
|
|
504
977
|
}
|
|
505
978
|
return { passed, failed, skipped: 0, duration: Date.now() - start, assertions, runAt: new Date() };
|
|
506
979
|
};
|
|
507
|
-
const
|
|
980
|
+
const saveDqlGenerationDraft = (importId, candidate) => saveDqlGenerationDraftForProject(projectRoot, importId, candidate);
|
|
981
|
+
const createDqlGenerationSessionFromBody = async (body) => {
|
|
982
|
+
return createDqlGenerationSessionForProject(projectRoot, {
|
|
983
|
+
inputPath: typeof body.path === 'string' ? body.path : '',
|
|
984
|
+
inputMode: body.inputMode === 'paste' || body.inputMode === 'upload' || body.inputMode === 'path' ? body.inputMode : undefined,
|
|
985
|
+
sources: Array.isArray(body.sources)
|
|
986
|
+
? body.sources.map((source, index) => ({
|
|
987
|
+
path: typeof source?.path === 'string' ? source.path : `source-${index + 1}.sql`,
|
|
988
|
+
content: typeof source?.content === 'string' ? source.content : '',
|
|
989
|
+
}))
|
|
990
|
+
: undefined,
|
|
991
|
+
sourceKind: typeof body.sourceKind === 'string' ? body.sourceKind : 'raw-sql',
|
|
992
|
+
domain: typeof body.domain === 'string' ? body.domain : undefined,
|
|
993
|
+
owner: typeof body.owner === 'string' ? body.owner : undefined,
|
|
994
|
+
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
995
|
+
provider: typeof body.provider === 'string' ? body.provider : undefined,
|
|
996
|
+
}, semanticLayer);
|
|
997
|
+
};
|
|
998
|
+
const loadDqlGenerationSession = (importId) => {
|
|
999
|
+
const session = loadBlockStudioImportSession(projectRoot, importId);
|
|
1000
|
+
const candidates = session.candidates.map((candidate) => ({
|
|
1001
|
+
...candidate,
|
|
1002
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
1003
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
1004
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
1005
|
+
evidence: candidate.evidence ?? deterministicDqlGenerationEvidence(candidate),
|
|
1006
|
+
draftSave: candidate.draftSave ?? (isDraftBlockPath(candidate.savedPath)
|
|
1007
|
+
? { status: 'saved', path: candidate.savedPath }
|
|
1008
|
+
: { status: 'pending' }),
|
|
1009
|
+
}));
|
|
1010
|
+
return {
|
|
1011
|
+
...session,
|
|
1012
|
+
mode: 'ai-import',
|
|
1013
|
+
candidates,
|
|
1014
|
+
generation: {
|
|
1015
|
+
provider: candidates.find((candidate) => candidate.generationProvider)?.generationProvider ?? 'local-deterministic',
|
|
1016
|
+
aiEnabled: candidates.some((candidate) => candidate.generationMode === 'ai'),
|
|
1017
|
+
contextObjectCount: candidates.reduce((sum, candidate) => sum + (candidate.evidence?.length ?? 0), 0),
|
|
1018
|
+
createdDrafts: candidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
|
|
1019
|
+
warnings: candidates.flatMap((candidate) => candidate.draftSave.status === 'error' && candidate.draftSave.error ? [candidate.draftSave.error] : []),
|
|
1020
|
+
},
|
|
1021
|
+
};
|
|
1022
|
+
};
|
|
1023
|
+
const certifyBlockStudioSource = async (source, blockPath, options = {}) => {
|
|
508
1024
|
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
509
1025
|
let preview = null;
|
|
510
1026
|
let testResults = null;
|
|
@@ -542,12 +1058,24 @@ export async function startLocalServer(opts) {
|
|
|
542
1058
|
description: parsed.description,
|
|
543
1059
|
owner: parsed.owner,
|
|
544
1060
|
tags: parsed.tags,
|
|
1061
|
+
llmContext: parsed.llmContext,
|
|
1062
|
+
pattern: parsed.pattern,
|
|
1063
|
+
grain: parsed.grain,
|
|
1064
|
+
entities: parsed.entities,
|
|
1065
|
+
declaredOutputs: parsed.outputs,
|
|
1066
|
+
dimensions: parsed.dimensions,
|
|
1067
|
+
allowedFilters: parsed.allowedFilters,
|
|
1068
|
+
parameterPolicy: parsed.parameterPolicy,
|
|
1069
|
+
filterBindings: parsed.filterBindings,
|
|
1070
|
+
sourceSystems: parsed.sourceSystems,
|
|
1071
|
+
replacementFor: parsed.replacementFor,
|
|
1072
|
+
reviewCadence: parsed.reviewCadence,
|
|
545
1073
|
dependencies: [],
|
|
546
1074
|
usedInCount: 0,
|
|
547
1075
|
createdAt: new Date(),
|
|
548
1076
|
updatedAt: new Date(),
|
|
549
1077
|
};
|
|
550
|
-
const certification = new Certifier().evaluate(record, testResults ?? undefined);
|
|
1078
|
+
const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined);
|
|
551
1079
|
const checklist = buildBlockStudioCertificationChecklist({
|
|
552
1080
|
source,
|
|
553
1081
|
validation,
|
|
@@ -730,6 +1258,7 @@ export async function startLocalServer(opts) {
|
|
|
730
1258
|
const variables = body.variables && typeof body.variables === 'object'
|
|
731
1259
|
? body.variables
|
|
732
1260
|
: {};
|
|
1261
|
+
const dashboardVariables = dashboardRuntimeVariables(loaded.dashboard, variables);
|
|
733
1262
|
const tiles = [];
|
|
734
1263
|
let localApps = null;
|
|
735
1264
|
for (const item of loaded.dashboard.layout.items) {
|
|
@@ -831,8 +1360,15 @@ export async function startLocalServer(opts) {
|
|
|
831
1360
|
});
|
|
832
1361
|
continue;
|
|
833
1362
|
}
|
|
834
|
-
const
|
|
835
|
-
|
|
1363
|
+
const filterApplication = applyDashboardFiltersToBlockExecution({
|
|
1364
|
+
sql: semanticCompose?.sql ?? plan.sql,
|
|
1365
|
+
sqlParams: plan?.sqlParams ?? [],
|
|
1366
|
+
variables: { ...(plan?.variables ?? {}), ...dashboardVariables },
|
|
1367
|
+
block,
|
|
1368
|
+
dashboard: loaded.dashboard,
|
|
1369
|
+
});
|
|
1370
|
+
const prepared = prepareLocalExecution(filterApplication.sql, targetConnection, projectRoot, projectConfig);
|
|
1371
|
+
const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables(filterApplication.variables), prepared.connection);
|
|
836
1372
|
tiles.push({
|
|
837
1373
|
tileId: item.i,
|
|
838
1374
|
status: 'ok',
|
|
@@ -843,6 +1379,10 @@ export async function startLocalServer(opts) {
|
|
|
843
1379
|
viz: item.viz,
|
|
844
1380
|
chartConfig: mergeDashboardChartConfig(plan?.chartConfig, item),
|
|
845
1381
|
result: normalizeQueryResult(result),
|
|
1382
|
+
filters: {
|
|
1383
|
+
applied: filterApplication.appliedFilters,
|
|
1384
|
+
skipped: filterApplication.skippedFilters,
|
|
1385
|
+
},
|
|
846
1386
|
citation: {
|
|
847
1387
|
kind: 'block',
|
|
848
1388
|
name: block.name,
|
|
@@ -885,11 +1425,11 @@ export async function startLocalServer(opts) {
|
|
|
885
1425
|
return;
|
|
886
1426
|
}
|
|
887
1427
|
// Apps, dashboards, persona — see apps-api.ts. Returns true if handled.
|
|
888
|
-
if (path.startsWith('/api/apps') || path === '/api/persona') {
|
|
1428
|
+
if (path.startsWith('/api/apps') || path.startsWith('/api/visualizations') || path === '/api/persona') {
|
|
889
1429
|
try {
|
|
890
1430
|
const handled = await handleAppsApi({
|
|
891
1431
|
req,
|
|
892
|
-
res,
|
|
1432
|
+
res: res,
|
|
893
1433
|
url,
|
|
894
1434
|
path,
|
|
895
1435
|
projectRoot,
|
|
@@ -1019,6 +1559,393 @@ export async function startLocalServer(opts) {
|
|
|
1019
1559
|
}
|
|
1020
1560
|
return;
|
|
1021
1561
|
}
|
|
1562
|
+
if (req.method === 'GET' && path === '/api/notebook/research') {
|
|
1563
|
+
const storage = openNotebookResearchStorage();
|
|
1564
|
+
try {
|
|
1565
|
+
const notebookPath = notebookResearchString(url.searchParams.get('path'));
|
|
1566
|
+
const sourceCellId = notebookResearchString(url.searchParams.get('sourceCellId') ?? url.searchParams.get('cellId'));
|
|
1567
|
+
const domain = notebookResearchString(url.searchParams.get('domain'));
|
|
1568
|
+
const owner = notebookResearchString(url.searchParams.get('owner'));
|
|
1569
|
+
const intent = notebookResearchIntent(url.searchParams.get('intent'));
|
|
1570
|
+
const search = notebookResearchString(url.searchParams.get('q')) ?? notebookResearchString(url.searchParams.get('search'));
|
|
1571
|
+
const status = notebookResearchStatus(url.searchParams.get('status'));
|
|
1572
|
+
const reviewStatus = notebookResearchReviewStatus(url.searchParams.get('reviewStatus'));
|
|
1573
|
+
const promotionAction = notebookResearchPromotionAction(url.searchParams.get('promotionAction') ?? url.searchParams.get('action'));
|
|
1574
|
+
const readiness = notebookResearchReadiness(url.searchParams.get('readiness') ?? url.searchParams.get('ready'));
|
|
1575
|
+
const age = notebookResearchAge(url.searchParams.get('age'));
|
|
1576
|
+
const nextAction = notebookResearchNextAction(url.searchParams.get('nextAction') ?? url.searchParams.get('next'));
|
|
1577
|
+
const activeOnlyParam = url.searchParams.get('activeOnly') ?? url.searchParams.get('active');
|
|
1578
|
+
const activeOnly = activeOnlyParam === 'true' || activeOnlyParam === '1';
|
|
1579
|
+
const sort = notebookResearchSort(url.searchParams.get('sort'));
|
|
1580
|
+
const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
|
|
1581
|
+
const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
|
|
1582
|
+
const page = storage.listRunsPage({ notebookPath, sourceCellId, domain, owner, intent, search, status, reviewStatus, promotionAction, readiness, age, nextAction, activeOnly, sort, limit, offset });
|
|
1583
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1584
|
+
res.end(serializeJSON(withNotebookResearchChecklistPage(page)));
|
|
1585
|
+
}
|
|
1586
|
+
catch (error) {
|
|
1587
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1588
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1589
|
+
}
|
|
1590
|
+
finally {
|
|
1591
|
+
storage.close();
|
|
1592
|
+
}
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
if (req.method === 'POST' && path === '/api/notebook/research') {
|
|
1596
|
+
const storage = openNotebookResearchStorage();
|
|
1597
|
+
try {
|
|
1598
|
+
const body = await readJSON(req);
|
|
1599
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1600
|
+
const question = notebookResearchString(body.question);
|
|
1601
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1602
|
+
const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
|
|
1603
|
+
const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
|
|
1604
|
+
const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
|
|
1605
|
+
if (!notebookPath || !question) {
|
|
1606
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1607
|
+
res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
|
|
1608
|
+
return;
|
|
1609
|
+
}
|
|
1610
|
+
const created = storage.createRun({
|
|
1611
|
+
notebookPath,
|
|
1612
|
+
domain: notebookResearchString(body.domain),
|
|
1613
|
+
owner: notebookResearchString(body.owner),
|
|
1614
|
+
sourceCell,
|
|
1615
|
+
sourceCellId,
|
|
1616
|
+
sourceCellName,
|
|
1617
|
+
sourceCellFingerprint,
|
|
1618
|
+
title: notebookResearchString(body.title),
|
|
1619
|
+
question,
|
|
1620
|
+
intent: notebookResearchIntent(body.intent),
|
|
1621
|
+
context: body.context,
|
|
1622
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1623
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1624
|
+
});
|
|
1625
|
+
const run = body.run === true
|
|
1626
|
+
? await runNotebookResearch(storage, created, {
|
|
1627
|
+
domain: notebookResearchString(body.domain),
|
|
1628
|
+
sourceCellFingerprint,
|
|
1629
|
+
question,
|
|
1630
|
+
intent: notebookResearchIntent(body.intent),
|
|
1631
|
+
context: body.context,
|
|
1632
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1633
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1634
|
+
})
|
|
1635
|
+
: created;
|
|
1636
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1637
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
|
|
1638
|
+
}
|
|
1639
|
+
catch (error) {
|
|
1640
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1641
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1642
|
+
}
|
|
1643
|
+
finally {
|
|
1644
|
+
storage.close();
|
|
1645
|
+
}
|
|
1646
|
+
return;
|
|
1647
|
+
}
|
|
1648
|
+
if (req.method === 'POST' && path === '/api/notebook/research/context-preview') {
|
|
1649
|
+
try {
|
|
1650
|
+
const body = await readJSON(req);
|
|
1651
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1652
|
+
const question = notebookResearchString(body.question);
|
|
1653
|
+
if (!notebookPath || !question) {
|
|
1654
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1655
|
+
res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
|
|
1656
|
+
return;
|
|
1657
|
+
}
|
|
1658
|
+
const intent = notebookResearchIntent(body.intent) ?? 'ad_hoc_analysis';
|
|
1659
|
+
const domain = notebookResearchString(body.domain);
|
|
1660
|
+
const context = body.context;
|
|
1661
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1662
|
+
const contextPack = await buildLocalContextPack(projectRoot, {
|
|
1663
|
+
question,
|
|
1664
|
+
mode: 'question',
|
|
1665
|
+
surface: 'notebook',
|
|
1666
|
+
selectedContext: {
|
|
1667
|
+
activeSurface: 'notebook',
|
|
1668
|
+
notebookPath,
|
|
1669
|
+
domain,
|
|
1670
|
+
sourceCellId: notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell),
|
|
1671
|
+
sourceCellName: notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell),
|
|
1672
|
+
context,
|
|
1673
|
+
intent,
|
|
1674
|
+
researchPattern: notebookResearchIntentPattern(intent),
|
|
1675
|
+
},
|
|
1676
|
+
strictness: 'balanced',
|
|
1677
|
+
limit: 100,
|
|
1678
|
+
});
|
|
1679
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1680
|
+
res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
|
|
1681
|
+
}
|
|
1682
|
+
catch (error) {
|
|
1683
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1684
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1685
|
+
}
|
|
1686
|
+
return;
|
|
1687
|
+
}
|
|
1688
|
+
if (req.method === 'POST' && path === '/api/notebook/research/seed-cells') {
|
|
1689
|
+
const storage = openNotebookResearchStorage();
|
|
1690
|
+
try {
|
|
1691
|
+
const body = await readJSON(req);
|
|
1692
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1693
|
+
if (!notebookPath) {
|
|
1694
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1695
|
+
res.end(serializeJSON({ error: 'notebookPath is required.' }));
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
const cells = Array.isArray(body.cells) ? body.cells : [];
|
|
1699
|
+
if (cells.length === 0) {
|
|
1700
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1701
|
+
res.end(serializeJSON({ error: 'At least one source cell is required.' }));
|
|
1702
|
+
return;
|
|
1703
|
+
}
|
|
1704
|
+
const validCells = cells.filter((cell) => Boolean(cell && typeof cell === 'object' && !Array.isArray(cell)));
|
|
1705
|
+
const invalidCellCount = cells.length - validCells.length;
|
|
1706
|
+
const seeded = storage.seedRunsFromCells({
|
|
1707
|
+
notebookPath,
|
|
1708
|
+
domain: notebookResearchString(body.domain),
|
|
1709
|
+
owner: notebookResearchString(body.owner),
|
|
1710
|
+
notebookTitle: notebookResearchString(body.notebookTitle),
|
|
1711
|
+
cells: validCells.map((cell) => ({
|
|
1712
|
+
sourceCell: notebookResearchSourceCellPayload(cell),
|
|
1713
|
+
id: notebookResearchString(cell.id),
|
|
1714
|
+
sourceCellId: notebookResearchString(cell.sourceCellId),
|
|
1715
|
+
name: notebookResearchString(cell.name),
|
|
1716
|
+
sourceCellName: notebookResearchString(cell.sourceCellName),
|
|
1717
|
+
sourceCellFingerprint: notebookResearchString(cell.sourceCellFingerprint),
|
|
1718
|
+
title: notebookResearchString(cell.title),
|
|
1719
|
+
type: notebookResearchString(cell.type),
|
|
1720
|
+
sql: notebookResearchString(cell.sql),
|
|
1721
|
+
content: notebookResearchString(cell.content),
|
|
1722
|
+
source: notebookResearchString(cell.source),
|
|
1723
|
+
question: notebookResearchString(cell.question),
|
|
1724
|
+
intent: notebookResearchIntent(cell.intent),
|
|
1725
|
+
})),
|
|
1726
|
+
limit: 1000,
|
|
1727
|
+
});
|
|
1728
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1729
|
+
res.end(serializeJSON({
|
|
1730
|
+
created: seeded.created.map(withNotebookResearchChecklist),
|
|
1731
|
+
createdCount: seeded.createdCount,
|
|
1732
|
+
skippedCount: seeded.skippedCount + invalidCellCount,
|
|
1733
|
+
limitApplied: seeded.limitApplied,
|
|
1734
|
+
}));
|
|
1735
|
+
}
|
|
1736
|
+
catch (error) {
|
|
1737
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1738
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1739
|
+
}
|
|
1740
|
+
finally {
|
|
1741
|
+
storage.close();
|
|
1742
|
+
}
|
|
1743
|
+
return;
|
|
1744
|
+
}
|
|
1745
|
+
if (req.method === 'POST' && path === '/api/notebook/research/source-coverage') {
|
|
1746
|
+
const storage = openNotebookResearchStorage();
|
|
1747
|
+
try {
|
|
1748
|
+
const body = await readJSON(req);
|
|
1749
|
+
const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
|
|
1750
|
+
if (!notebookPath) {
|
|
1751
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1752
|
+
res.end(serializeJSON({ error: 'notebookPath is required.' }));
|
|
1753
|
+
return;
|
|
1754
|
+
}
|
|
1755
|
+
const rawSourceCells = Array.isArray(body.sourceCells)
|
|
1756
|
+
? body.sourceCells
|
|
1757
|
+
: Array.isArray(body.cells)
|
|
1758
|
+
? body.cells
|
|
1759
|
+
: [];
|
|
1760
|
+
const sourceCells = rawSourceCells.flatMap((cell) => {
|
|
1761
|
+
if (!cell || typeof cell !== 'object' || Array.isArray(cell))
|
|
1762
|
+
return [];
|
|
1763
|
+
const record = cell;
|
|
1764
|
+
const nested = notebookResearchSourceCellPayload(record) ?? {};
|
|
1765
|
+
return [{
|
|
1766
|
+
...nested,
|
|
1767
|
+
id: notebookResearchString(record.id) ?? nested.id,
|
|
1768
|
+
sourceCellId: notebookResearchString(record.sourceCellId) ?? nested.sourceCellId,
|
|
1769
|
+
cellId: notebookResearchString(record.cellId) ?? nested.cellId,
|
|
1770
|
+
name: notebookResearchString(record.name) ?? nested.name,
|
|
1771
|
+
sourceCellName: notebookResearchString(record.sourceCellName) ?? nested.sourceCellName,
|
|
1772
|
+
title: notebookResearchString(record.title) ?? nested.title,
|
|
1773
|
+
fingerprint: notebookResearchString(record.fingerprint) ?? nested.fingerprint,
|
|
1774
|
+
sourceCellFingerprint: notebookResearchString(record.sourceCellFingerprint) ?? nested.sourceCellFingerprint,
|
|
1775
|
+
sqlFingerprint: notebookResearchString(record.sqlFingerprint) ?? nested.sqlFingerprint,
|
|
1776
|
+
type: notebookResearchString(record.type) ?? nested.type,
|
|
1777
|
+
}];
|
|
1778
|
+
});
|
|
1779
|
+
const requestedIds = Array.isArray(body.sourceCellIds)
|
|
1780
|
+
? body.sourceCellIds
|
|
1781
|
+
: rawSourceCells.length > 0
|
|
1782
|
+
? sourceCells.map((cell) => notebookResearchSourceCellId(cell))
|
|
1783
|
+
: [];
|
|
1784
|
+
const sourceCellIds = Array.from(new Set(requestedIds
|
|
1785
|
+
.map((id) => notebookResearchString(id))
|
|
1786
|
+
.filter((id) => Boolean(id))));
|
|
1787
|
+
const requestedSourceCellCount = new Set([
|
|
1788
|
+
...sourceCellIds,
|
|
1789
|
+
...sourceCells
|
|
1790
|
+
.map((cell) => notebookResearchSourceCellId(cell))
|
|
1791
|
+
.filter((id) => Boolean(id)),
|
|
1792
|
+
]).size;
|
|
1793
|
+
const limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
|
|
1794
|
+
? Math.max(1, Math.min(10_000, Math.floor(body.limit)))
|
|
1795
|
+
: 10_000;
|
|
1796
|
+
const linkedRuns = storage.listLatestRunsBySourceCell({
|
|
1797
|
+
notebookPath,
|
|
1798
|
+
sourceCellIds,
|
|
1799
|
+
sourceCells,
|
|
1800
|
+
limit,
|
|
1801
|
+
});
|
|
1802
|
+
const missingRuns = storage.listLatestRunsForMissingSourceCells({
|
|
1803
|
+
notebookPath,
|
|
1804
|
+
sourceCellIds,
|
|
1805
|
+
sourceCells,
|
|
1806
|
+
limit,
|
|
1807
|
+
});
|
|
1808
|
+
const runs = [...linkedRuns, ...missingRuns];
|
|
1809
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1810
|
+
res.end(serializeJSON({
|
|
1811
|
+
runs: runs.map(withNotebookResearchChecklist),
|
|
1812
|
+
requestedCount: requestedSourceCellCount,
|
|
1813
|
+
matchedCount: runs.length,
|
|
1814
|
+
limitApplied: requestedSourceCellCount > limit,
|
|
1815
|
+
}));
|
|
1816
|
+
}
|
|
1817
|
+
catch (error) {
|
|
1818
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1819
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1820
|
+
}
|
|
1821
|
+
finally {
|
|
1822
|
+
storage.close();
|
|
1823
|
+
}
|
|
1824
|
+
return;
|
|
1825
|
+
}
|
|
1826
|
+
if (req.method === 'GET' && path === '/api/notebook/research/diagnostics') {
|
|
1827
|
+
const storage = openNotebookResearchStorage();
|
|
1828
|
+
try {
|
|
1829
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1830
|
+
res.end(serializeJSON(storage.getDiagnostics()));
|
|
1831
|
+
}
|
|
1832
|
+
catch (error) {
|
|
1833
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1834
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1835
|
+
}
|
|
1836
|
+
finally {
|
|
1837
|
+
storage.close();
|
|
1838
|
+
}
|
|
1839
|
+
return;
|
|
1840
|
+
}
|
|
1841
|
+
const notebookResearchMatch = /^\/api\/notebook\/research\/([^/]+)(?:\/([^/]+))?$/.exec(path);
|
|
1842
|
+
if (notebookResearchMatch) {
|
|
1843
|
+
const id = decodeURIComponent(notebookResearchMatch[1]);
|
|
1844
|
+
const action = notebookResearchMatch[2];
|
|
1845
|
+
const storage = openNotebookResearchStorage();
|
|
1846
|
+
try {
|
|
1847
|
+
const run = storage.getRun(id);
|
|
1848
|
+
if (!run) {
|
|
1849
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1850
|
+
res.end(serializeJSON({ error: 'Notebook research run not found.' }));
|
|
1851
|
+
return;
|
|
1852
|
+
}
|
|
1853
|
+
if (req.method === 'GET' && !action) {
|
|
1854
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1855
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
if (req.method === 'PATCH' && !action) {
|
|
1859
|
+
const body = await readJSON(req);
|
|
1860
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1861
|
+
const sourceCellIdPatch = notebookResearchPatchString(body, 'sourceCellId');
|
|
1862
|
+
const sourceCellNamePatch = notebookResearchPatchString(body, 'sourceCellName');
|
|
1863
|
+
const sourceCellFingerprintPatch = notebookResearchPatchString(body, 'sourceCellFingerprint');
|
|
1864
|
+
const updated = storage.updateRun(id, {
|
|
1865
|
+
domain: notebookResearchString(body.domain),
|
|
1866
|
+
owner: notebookResearchString(body.owner),
|
|
1867
|
+
sourceCellId: sourceCellIdPatch !== undefined ? sourceCellIdPatch : notebookResearchSourceCellId(sourceCell),
|
|
1868
|
+
sourceCellName: sourceCellNamePatch !== undefined ? sourceCellNamePatch : notebookResearchSourceCellName(sourceCell),
|
|
1869
|
+
sourceCellFingerprint: sourceCellFingerprintPatch !== undefined ? sourceCellFingerprintPatch : notebookResearchSourceCellFingerprint(sourceCell),
|
|
1870
|
+
title: notebookResearchString(body.title),
|
|
1871
|
+
question: notebookResearchString(body.question),
|
|
1872
|
+
intent: notebookResearchIntent(body.intent),
|
|
1873
|
+
context: body.context,
|
|
1874
|
+
recommendation: notebookResearchString(body.recommendation),
|
|
1875
|
+
evidence: body.evidence,
|
|
1876
|
+
contextPackId: notebookResearchString(body.contextPackId),
|
|
1877
|
+
routeDecision: body.routeDecision,
|
|
1878
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1879
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1880
|
+
warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
|
|
1881
|
+
reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
|
|
1882
|
+
}) ?? run;
|
|
1883
|
+
const planned = storage.updateRun(updated.id, {
|
|
1884
|
+
researchPlan: buildNotebookResearchPlan({
|
|
1885
|
+
run: updated,
|
|
1886
|
+
evidence: updated.evidence,
|
|
1887
|
+
resultPreview: updated.resultPreview,
|
|
1888
|
+
previewError: updated.error,
|
|
1889
|
+
generatedSql: updated.generatedSql,
|
|
1890
|
+
reviewedSql: updated.reviewedSql,
|
|
1891
|
+
routeDecision: updated.routeDecision,
|
|
1892
|
+
}),
|
|
1893
|
+
}) ?? updated;
|
|
1894
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1895
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(planned) }));
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
if (req.method === 'POST' && action === 'run') {
|
|
1899
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
1900
|
+
const sourceCell = notebookResearchSourceCellPayload(body);
|
|
1901
|
+
const updated = await runNotebookResearch(storage, run, {
|
|
1902
|
+
domain: notebookResearchString(body.domain),
|
|
1903
|
+
owner: notebookResearchString(body.owner),
|
|
1904
|
+
sourceCellFingerprint: notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell),
|
|
1905
|
+
question: notebookResearchString(body.question),
|
|
1906
|
+
intent: notebookResearchIntent(body.intent),
|
|
1907
|
+
context: body.context,
|
|
1908
|
+
generatedSql: notebookResearchString(body.generatedSql),
|
|
1909
|
+
reviewedSql: notebookResearchString(body.reviewedSql),
|
|
1910
|
+
});
|
|
1911
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1912
|
+
res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
|
|
1913
|
+
return;
|
|
1914
|
+
}
|
|
1915
|
+
if (req.method === 'POST' && action === 'reuse-check') {
|
|
1916
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
1917
|
+
const payload = await checkNotebookResearchReuse(storage, run, {
|
|
1918
|
+
domain: notebookResearchString(body.domain),
|
|
1919
|
+
owner: notebookResearchString(body.owner),
|
|
1920
|
+
});
|
|
1921
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1922
|
+
res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
|
|
1923
|
+
return;
|
|
1924
|
+
}
|
|
1925
|
+
if (req.method === 'POST' && action === 'promote-dql') {
|
|
1926
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
1927
|
+
const payload = await promoteNotebookResearchToDql(storage, run, {
|
|
1928
|
+
domain: notebookResearchString(body.domain),
|
|
1929
|
+
owner: notebookResearchString(body.owner),
|
|
1930
|
+
provider: notebookResearchString(body.provider),
|
|
1931
|
+
tags: Array.isArray(body.tags) ? body.tags.filter((item) => typeof item === 'string') : undefined,
|
|
1932
|
+
});
|
|
1933
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1934
|
+
res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
|
|
1935
|
+
return;
|
|
1936
|
+
}
|
|
1937
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1938
|
+
res.end(serializeJSON({ error: 'Unsupported notebook research operation.' }));
|
|
1939
|
+
}
|
|
1940
|
+
catch (error) {
|
|
1941
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1942
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1943
|
+
}
|
|
1944
|
+
finally {
|
|
1945
|
+
storage.close();
|
|
1946
|
+
}
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1022
1949
|
// ── run snapshots (v0.11) ───────────────────────────────────────────────
|
|
1023
1950
|
// Captures executed notebook state (query results + timings) in a
|
|
1024
1951
|
// sibling `.run.json` so notebooks can show last-run output without
|
|
@@ -1304,49 +2231,51 @@ export async function startLocalServer(opts) {
|
|
|
1304
2231
|
// ── Block library (list all blocks with metadata) ────────────────────
|
|
1305
2232
|
if (req.method === 'GET' && path === '/api/blocks/library') {
|
|
1306
2233
|
try {
|
|
1307
|
-
const blocksDir = join(projectRoot, 'blocks');
|
|
1308
2234
|
const blocks = [];
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
2235
|
+
const seen = new Set();
|
|
2236
|
+
const scanDir = (dir) => {
|
|
2237
|
+
if (!existsSync(dir))
|
|
2238
|
+
return;
|
|
2239
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
2240
|
+
const filePath = join(dir, entry.name);
|
|
2241
|
+
if (entry.isDirectory()) {
|
|
2242
|
+
scanDir(filePath);
|
|
2243
|
+
}
|
|
2244
|
+
else if (entry.name.endsWith('.dql') && !seen.has(filePath)) {
|
|
2245
|
+
seen.add(filePath);
|
|
2246
|
+
const relPath = relative(projectRoot, filePath).replaceAll('\\', '/');
|
|
2247
|
+
try {
|
|
2248
|
+
const source = readFileSync(filePath, 'utf-8');
|
|
2249
|
+
const stat = statSync(filePath);
|
|
2250
|
+
// Quick regex parse for key block fields
|
|
2251
|
+
const nameMatch = /block\s+"([^"]+)"/.exec(source);
|
|
2252
|
+
const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
|
|
2253
|
+
const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
|
|
2254
|
+
const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
|
|
2255
|
+
const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
|
|
2256
|
+
const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
|
|
2257
|
+
const parsedTags = tagsMatch
|
|
2258
|
+
? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
|
|
2259
|
+
: [];
|
|
2260
|
+
const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
|
|
2261
|
+
blocks.push({
|
|
2262
|
+
name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
|
|
2263
|
+
domain: (domainMatch?.[1] ?? inferBlockStudioPathDomain(relPath)) || 'uncategorized',
|
|
2264
|
+
status: statusMatch?.[1] ?? 'draft',
|
|
2265
|
+
owner: ownerMatch?.[1] ?? null,
|
|
2266
|
+
tags: parsedTags,
|
|
2267
|
+
path: relPath,
|
|
2268
|
+
lastModified: stat.mtime.toISOString(),
|
|
2269
|
+
description: descMatch?.[1] ?? '',
|
|
2270
|
+
llmContext: llmMatch?.[1] ?? null,
|
|
2271
|
+
});
|
|
1345
2272
|
}
|
|
2273
|
+
catch { /* skip unreadable files */ }
|
|
1346
2274
|
}
|
|
1347
|
-
}
|
|
1348
|
-
|
|
1349
|
-
|
|
2275
|
+
}
|
|
2276
|
+
};
|
|
2277
|
+
scanDir(join(projectRoot, 'blocks'));
|
|
2278
|
+
scanDir(join(projectRoot, 'domains'));
|
|
1350
2279
|
blocks.sort((a, b) => new Date(b.lastModified).getTime() - new Date(a.lastModified).getTime());
|
|
1351
2280
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1352
2281
|
res.end(serializeJSON({ blocks }));
|
|
@@ -1555,20 +2484,216 @@ export async function startLocalServer(opts) {
|
|
|
1555
2484
|
res.end(serializeJSON({ error: 'source is required' }));
|
|
1556
2485
|
return;
|
|
1557
2486
|
}
|
|
1558
|
-
const result = await certifyBlockStudioSource(source, blockPath);
|
|
2487
|
+
const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise !== false });
|
|
1559
2488
|
const blockers = Array.from(new Set(result.checklist.blockers));
|
|
1560
2489
|
if (!result.certification.certified || blockers.length > 0) {
|
|
1561
2490
|
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1562
2491
|
res.end(serializeJSON({ ok: false, ...result, blockers }));
|
|
1563
2492
|
return;
|
|
1564
2493
|
}
|
|
1565
|
-
|
|
1566
|
-
|
|
2494
|
+
let certifiedPayload = null;
|
|
2495
|
+
const certifiedSource = setBlockStudioStatusInSource(source, 'certified');
|
|
2496
|
+
if (blockPath) {
|
|
2497
|
+
const normalizedBlockPath = normalize(blockPath).replace(/^\/+/, '');
|
|
2498
|
+
const parsed = parseBlockSourceMetadata(certifiedSource);
|
|
2499
|
+
if (isDraftBlockPath(normalizedBlockPath)) {
|
|
2500
|
+
const savedPath = saveBlockStudioArtifacts(projectRoot, {
|
|
2501
|
+
currentPath: normalizedBlockPath,
|
|
2502
|
+
source: certifiedSource,
|
|
2503
|
+
name: parsed.name,
|
|
2504
|
+
domain: parsed.domain,
|
|
2505
|
+
description: parsed.description,
|
|
2506
|
+
owner: parsed.owner,
|
|
2507
|
+
tags: parsed.tags,
|
|
2508
|
+
});
|
|
2509
|
+
certifiedPayload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
2510
|
+
}
|
|
2511
|
+
else {
|
|
2512
|
+
setBlockStudioStatus(projectRoot, normalizedBlockPath, 'certified');
|
|
2513
|
+
certifiedPayload = openBlockStudioDocument(projectRoot, normalizedBlockPath, semanticLayer);
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
1567
2516
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
1568
2517
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1569
|
-
res.end(serializeJSON({
|
|
2518
|
+
res.end(serializeJSON({
|
|
2519
|
+
ok: true,
|
|
2520
|
+
status: 'certified',
|
|
2521
|
+
...result,
|
|
2522
|
+
path: certifiedPayload?.path ?? blockPath ?? null,
|
|
2523
|
+
source: certifiedPayload?.source ?? certifiedSource,
|
|
2524
|
+
metadata: certifiedPayload?.metadata,
|
|
2525
|
+
companionPath: certifiedPayload?.companionPath ?? null,
|
|
2526
|
+
validation: certifiedPayload?.validation ?? result.validation,
|
|
2527
|
+
}));
|
|
2528
|
+
}
|
|
2529
|
+
catch (error) {
|
|
2530
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2531
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2532
|
+
}
|
|
2533
|
+
return;
|
|
2534
|
+
}
|
|
2535
|
+
if (req.method === 'POST' && path === '/api/block-studio/ai-imports') {
|
|
2536
|
+
try {
|
|
2537
|
+
const body = await readJSON(req);
|
|
2538
|
+
const session = await createDqlGenerationSessionFromBody(body);
|
|
2539
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2540
|
+
res.end(serializeJSON(session));
|
|
2541
|
+
}
|
|
2542
|
+
catch (error) {
|
|
2543
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2544
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2545
|
+
}
|
|
2546
|
+
return;
|
|
2547
|
+
}
|
|
2548
|
+
if (req.method === 'POST' && path === '/api/block-studio/match-sql') {
|
|
2549
|
+
try {
|
|
2550
|
+
const body = await readJSON(req);
|
|
2551
|
+
if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
|
|
2552
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2553
|
+
res.end(serializeJSON({ error: 'Missing SQL in request body.' }));
|
|
2554
|
+
return;
|
|
2555
|
+
}
|
|
2556
|
+
const sql = body.sql.trim();
|
|
2557
|
+
const match = await matchSqlForDqlReuse({
|
|
2558
|
+
sql,
|
|
2559
|
+
sourcePath: typeof body.sourcePath === 'string' ? body.sourcePath : 'pasted.sql',
|
|
2560
|
+
name: typeof body.name === 'string' ? body.name : 'SQL match preview',
|
|
2561
|
+
domain: typeof body.domain === 'string' ? body.domain : 'imported',
|
|
2562
|
+
owner: typeof body.owner === 'string' ? body.owner : 'analytics',
|
|
2563
|
+
});
|
|
2564
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2565
|
+
res.end(serializeJSON(match));
|
|
2566
|
+
}
|
|
2567
|
+
catch (error) {
|
|
2568
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2569
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2570
|
+
}
|
|
2571
|
+
return;
|
|
2572
|
+
}
|
|
2573
|
+
const aiImportPathMatch = path.match(/^\/api\/block-studio\/ai-imports\/([^/]+)(?:\/candidates\/([^/]+)(?:\/(preview|certify))?)?$/);
|
|
2574
|
+
if (aiImportPathMatch) {
|
|
2575
|
+
const importId = decodeURIComponent(aiImportPathMatch[1]);
|
|
2576
|
+
const candidateId = aiImportPathMatch[2] ? decodeURIComponent(aiImportPathMatch[2]) : null;
|
|
2577
|
+
const action = aiImportPathMatch[3] ?? null;
|
|
2578
|
+
try {
|
|
2579
|
+
if (req.method === 'GET' && !candidateId) {
|
|
2580
|
+
const session = loadDqlGenerationSession(importId);
|
|
2581
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2582
|
+
res.end(serializeJSON(session));
|
|
2583
|
+
return;
|
|
2584
|
+
}
|
|
2585
|
+
if (req.method === 'PATCH' && candidateId && !action) {
|
|
2586
|
+
const body = await readJSON(req);
|
|
2587
|
+
const candidate = updateBlockStudioImportCandidate(projectRoot, importId, candidateId, {
|
|
2588
|
+
name: typeof body.name === 'string' ? body.name : undefined,
|
|
2589
|
+
domain: typeof body.domain === 'string' ? body.domain : undefined,
|
|
2590
|
+
description: typeof body.description === 'string' ? body.description : undefined,
|
|
2591
|
+
owner: typeof body.owner === 'string' ? body.owner : undefined,
|
|
2592
|
+
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
2593
|
+
terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
|
|
2594
|
+
pattern: typeof body.pattern === 'string' ? body.pattern : undefined,
|
|
2595
|
+
grain: typeof body.grain === 'string' ? body.grain : undefined,
|
|
2596
|
+
entities: Array.isArray(body.entities) ? body.entities.map(String) : undefined,
|
|
2597
|
+
outputs: Array.isArray(body.outputs) ? body.outputs.map(String) : undefined,
|
|
2598
|
+
dimensions: Array.isArray(body.dimensions) ? body.dimensions.map(String) : undefined,
|
|
2599
|
+
allowedFilters: Array.isArray(body.allowedFilters) ? body.allowedFilters.map(String) : undefined,
|
|
2600
|
+
parameterPolicy: Array.isArray(body.parameterPolicy) ? body.parameterPolicy : undefined,
|
|
2601
|
+
filterBindings: Array.isArray(body.filterBindings) ? body.filterBindings : undefined,
|
|
2602
|
+
sourceSystems: Array.isArray(body.sourceSystems) ? body.sourceSystems.map(String) : undefined,
|
|
2603
|
+
replacementFor: Array.isArray(body.replacementFor) ? body.replacementFor.map(String) : undefined,
|
|
2604
|
+
reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
|
|
2605
|
+
sql: typeof body.sql === 'string' ? body.sql : undefined,
|
|
2606
|
+
llmContext: typeof body.llmContext === 'string' ? body.llmContext : undefined,
|
|
2607
|
+
});
|
|
2608
|
+
const validated = validateImportCandidate(candidate);
|
|
2609
|
+
const savedDraft = saveDqlGenerationDraft(importId, validated);
|
|
2610
|
+
writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
|
|
2611
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2612
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2613
|
+
res.end(serializeJSON(savedDraft));
|
|
2614
|
+
return;
|
|
2615
|
+
}
|
|
2616
|
+
if (req.method === 'POST' && candidateId && action === 'preview') {
|
|
2617
|
+
const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
|
|
2618
|
+
const preview = await runBlockStudioPreviewSource(candidate.dqlSource);
|
|
2619
|
+
const next = saveDqlGenerationDraft(importId, {
|
|
2620
|
+
...candidate,
|
|
2621
|
+
preview,
|
|
2622
|
+
validation: validateBlockStudioSource(candidate.dqlSource, semanticLayer),
|
|
2623
|
+
});
|
|
2624
|
+
writeBlockStudioImportCandidate(projectRoot, importId, next);
|
|
2625
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2626
|
+
res.end(serializeJSON(next));
|
|
2627
|
+
return;
|
|
2628
|
+
}
|
|
2629
|
+
if (req.method === 'POST' && candidateId && action === 'certify') {
|
|
2630
|
+
const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
|
|
2631
|
+
const readiness = validateImportCandidateForSave(candidate);
|
|
2632
|
+
if (readiness.errors.length > 0) {
|
|
2633
|
+
const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
|
|
2634
|
+
writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
|
|
2635
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2636
|
+
res.end(serializeJSON({
|
|
2637
|
+
error: readiness.errors.join(' '),
|
|
2638
|
+
candidate: savedDraft,
|
|
2639
|
+
diagnostics: savedDraft.validation?.diagnostics ?? [],
|
|
2640
|
+
}));
|
|
2641
|
+
return;
|
|
2642
|
+
}
|
|
2643
|
+
const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
|
|
2644
|
+
const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: true });
|
|
2645
|
+
const blockers = Array.from(new Set(certification.checklist.blockers));
|
|
2646
|
+
if (!certification.certification.certified || blockers.length > 0) {
|
|
2647
|
+
const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
|
|
2648
|
+
writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
|
|
2649
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2650
|
+
res.end(serializeJSON({ ok: false, candidate: savedDraft, blockers, ...certification }));
|
|
2651
|
+
return;
|
|
2652
|
+
}
|
|
2653
|
+
const savedPath = saveBlockStudioArtifacts(projectRoot, {
|
|
2654
|
+
currentPath: readiness.candidate.savedPath,
|
|
2655
|
+
source: certifiedSource,
|
|
2656
|
+
name: readiness.candidate.name,
|
|
2657
|
+
domain: readiness.candidate.domain,
|
|
2658
|
+
description: readiness.candidate.description,
|
|
2659
|
+
owner: readiness.candidate.owner,
|
|
2660
|
+
tags: readiness.candidate.tags,
|
|
2661
|
+
lineage: readiness.candidate.lineage.sourceTables,
|
|
2662
|
+
importMeta: {
|
|
2663
|
+
importId,
|
|
2664
|
+
candidateId,
|
|
2665
|
+
sourceKind: readiness.candidate.sourceKind,
|
|
2666
|
+
sourcePath: readiness.candidate.sourcePath,
|
|
2667
|
+
},
|
|
2668
|
+
});
|
|
2669
|
+
const next = {
|
|
2670
|
+
...readiness.candidate,
|
|
2671
|
+
dqlSource: certifiedSource,
|
|
2672
|
+
reviewStatus: 'saved',
|
|
2673
|
+
savedPath,
|
|
2674
|
+
validation: validateBlockStudioSource(certifiedSource, semanticLayer),
|
|
2675
|
+
generationMode: readiness.candidate.generationMode ?? 'deterministic',
|
|
2676
|
+
generationProvider: readiness.candidate.generationProvider ?? 'local-deterministic',
|
|
2677
|
+
llmContext: readiness.candidate.llmContext ?? deterministicDqlGenerationContext(readiness.candidate, readiness.candidate.evidence ?? []),
|
|
2678
|
+
evidence: readiness.candidate.evidence ?? [],
|
|
2679
|
+
draftSave: readiness.candidate.draftSave ?? { status: 'pending' },
|
|
2680
|
+
};
|
|
2681
|
+
writeBlockStudioImportCandidate(projectRoot, importId, next);
|
|
2682
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
2683
|
+
const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
2684
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2685
|
+
res.end(serializeJSON({ candidate: next, block: payload, certification }));
|
|
2686
|
+
return;
|
|
2687
|
+
}
|
|
2688
|
+
res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2689
|
+
res.end(serializeJSON({ error: 'Unsupported AI import operation.' }));
|
|
1570
2690
|
}
|
|
1571
2691
|
catch (error) {
|
|
2692
|
+
if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
|
|
2693
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2694
|
+
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
2695
|
+
return;
|
|
2696
|
+
}
|
|
1572
2697
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1573
2698
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1574
2699
|
}
|
|
@@ -1708,6 +2833,8 @@ export async function startLocalServer(opts) {
|
|
|
1708
2833
|
description: typeof body.description === 'string' ? body.description : undefined,
|
|
1709
2834
|
owner: typeof body.owner === 'string' ? body.owner : undefined,
|
|
1710
2835
|
tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
|
|
2836
|
+
terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
|
|
2837
|
+
reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
|
|
1711
2838
|
sql: typeof body.sql === 'string' ? body.sql : undefined,
|
|
1712
2839
|
reviewStatus,
|
|
1713
2840
|
});
|
|
@@ -1844,6 +2971,55 @@ export async function startLocalServer(opts) {
|
|
|
1844
2971
|
}
|
|
1845
2972
|
return;
|
|
1846
2973
|
}
|
|
2974
|
+
if (req.method === 'GET' && path === '/api/semantic-layer/diagnostics') {
|
|
2975
|
+
try {
|
|
2976
|
+
const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
|
|
2977
|
+
semanticLayer,
|
|
2978
|
+
semanticErrors: semanticLayerErrors,
|
|
2979
|
+
semanticConfig,
|
|
2980
|
+
detectedProvider: semanticDetectedProvider,
|
|
2981
|
+
lastSyncTime: semanticLastSyncTime,
|
|
2982
|
+
});
|
|
2983
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2984
|
+
res.end(serializeJSON(diagnostics));
|
|
2985
|
+
}
|
|
2986
|
+
catch (error) {
|
|
2987
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2988
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2989
|
+
}
|
|
2990
|
+
return;
|
|
2991
|
+
}
|
|
2992
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/reload') {
|
|
2993
|
+
try {
|
|
2994
|
+
const semanticConnection = connection;
|
|
2995
|
+
const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
|
|
2996
|
+
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
|
|
2997
|
+
: undefined;
|
|
2998
|
+
const refreshed = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
|
|
2999
|
+
semanticLayer = refreshed.layer;
|
|
3000
|
+
semanticLayerErrors = refreshed.errors;
|
|
3001
|
+
semanticDetectedProvider = refreshed.detectedProvider;
|
|
3002
|
+
semanticLastSyncTime = refreshed.layer ? new Date().toISOString() : null;
|
|
3003
|
+
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
3004
|
+
const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
|
|
3005
|
+
semanticLayer,
|
|
3006
|
+
semanticErrors: semanticLayerErrors,
|
|
3007
|
+
semanticConfig,
|
|
3008
|
+
detectedProvider: semanticDetectedProvider,
|
|
3009
|
+
lastSyncTime: semanticLastSyncTime,
|
|
3010
|
+
});
|
|
3011
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3012
|
+
res.end(serializeJSON({
|
|
3013
|
+
ok: Boolean(refreshed.layer),
|
|
3014
|
+
...diagnostics,
|
|
3015
|
+
}));
|
|
3016
|
+
}
|
|
3017
|
+
catch (error) {
|
|
3018
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3019
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3020
|
+
}
|
|
3021
|
+
return;
|
|
3022
|
+
}
|
|
1847
3023
|
if (req.method === 'GET' && path === '/api/block-studio/open') {
|
|
1848
3024
|
try {
|
|
1849
3025
|
const relativePath = url.searchParams.get('path');
|
|
@@ -1908,8 +3084,9 @@ export async function startLocalServer(opts) {
|
|
|
1908
3084
|
res.end(serializeJSON({ error: 'Block name is required.' }));
|
|
1909
3085
|
return;
|
|
1910
3086
|
}
|
|
1911
|
-
const
|
|
1912
|
-
|
|
3087
|
+
const currentPath = typeof body.path === 'string' ? body.path : undefined;
|
|
3088
|
+
const saveOptions = {
|
|
3089
|
+
currentPath,
|
|
1913
3090
|
source,
|
|
1914
3091
|
name: metadata.name,
|
|
1915
3092
|
domain: metadata.domain,
|
|
@@ -1925,7 +3102,13 @@ export async function startLocalServer(opts) {
|
|
|
1925
3102
|
sourcePath: metadata.sourcePath,
|
|
1926
3103
|
}
|
|
1927
3104
|
: undefined,
|
|
1928
|
-
}
|
|
3105
|
+
};
|
|
3106
|
+
const savedPath = isDraftBlockPath(currentPath)
|
|
3107
|
+
? saveBlockStudioDraftArtifacts(projectRoot, {
|
|
3108
|
+
...saveOptions,
|
|
3109
|
+
stableSuffix: metadata.candidateId,
|
|
3110
|
+
})
|
|
3111
|
+
: saveBlockStudioArtifacts(projectRoot, saveOptions);
|
|
1929
3112
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
1930
3113
|
const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
1931
3114
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -2051,7 +3234,7 @@ export async function startLocalServer(opts) {
|
|
|
2051
3234
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2052
3235
|
res.end(serializeJSON({
|
|
2053
3236
|
available: false,
|
|
2054
|
-
provider:
|
|
3237
|
+
provider: semanticConfig?.provider ?? semanticDetectedProvider ?? null,
|
|
2055
3238
|
errors: semanticLayerErrors,
|
|
2056
3239
|
metrics: [],
|
|
2057
3240
|
measures: [],
|
|
@@ -2180,9 +3363,9 @@ export async function startLocalServer(opts) {
|
|
|
2180
3363
|
owner: q.owner ?? null,
|
|
2181
3364
|
source: q.source ?? null,
|
|
2182
3365
|
}));
|
|
2183
|
-
const provider =
|
|
3366
|
+
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
2184
3367
|
const dbtExecutionReady = provider === 'dbt'
|
|
2185
|
-
? hasDbtSemanticManifest(projectRoot,
|
|
3368
|
+
? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
|
|
2186
3369
|
: false;
|
|
2187
3370
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2188
3371
|
res.end(serializeJSON({
|
|
@@ -2731,9 +3914,35 @@ export async function startLocalServer(opts) {
|
|
|
2731
3914
|
res.end();
|
|
2732
3915
|
return;
|
|
2733
3916
|
}
|
|
2734
|
-
if (req.method === 'POST' && path === '/api/
|
|
3917
|
+
if (req.method === 'POST' && path === '/api/ai/sql-draft/preview') {
|
|
2735
3918
|
try {
|
|
2736
3919
|
const body = await readJSON(req);
|
|
3920
|
+
if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
|
|
3921
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3922
|
+
res.end(serializeJSON({ error: 'Missing SQL in request body.' }));
|
|
3923
|
+
return;
|
|
3924
|
+
}
|
|
3925
|
+
const previewSql = buildAgentPreviewSql(body.sql);
|
|
3926
|
+
const result = await executeLocalSqlForStoredResult(previewSql);
|
|
3927
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3928
|
+
res.end(serializeJSON({ ok: true, result }));
|
|
3929
|
+
}
|
|
3930
|
+
catch (error) {
|
|
3931
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3932
|
+
res.end(serializeJSON({
|
|
3933
|
+
ok: false,
|
|
3934
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3935
|
+
}));
|
|
3936
|
+
}
|
|
3937
|
+
return;
|
|
3938
|
+
}
|
|
3939
|
+
if (req.method === 'POST' && path === '/api/query') {
|
|
3940
|
+
let body;
|
|
3941
|
+
let execContext = null;
|
|
3942
|
+
const start = Date.now();
|
|
3943
|
+
try {
|
|
3944
|
+
body = await readJSON(req);
|
|
3945
|
+
execContext = notebookExecutionContext(body.executionContext);
|
|
2737
3946
|
if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
|
|
2738
3947
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2739
3948
|
res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
|
|
@@ -2756,7 +3965,26 @@ export async function startLocalServer(opts) {
|
|
|
2756
3965
|
const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
|
|
2757
3966
|
assertAppAccess({ app, domain, level: 'execute' });
|
|
2758
3967
|
const result = await executor.executeQuery(prepared.sql, Array.isArray(body.sqlParams) ? body.sqlParams : [], runtimeVariables(body.variables && typeof body.variables === 'object' ? body.variables : {}), prepared.connection);
|
|
2759
|
-
const
|
|
3968
|
+
const normalized = normalizeQueryResult(result, semantic.semanticRefs);
|
|
3969
|
+
if (execContext) {
|
|
3970
|
+
recordNotebookQueryRun(projectRoot, {
|
|
3971
|
+
notebookPath: execContext.notebookPath,
|
|
3972
|
+
cellId: execContext.cellId,
|
|
3973
|
+
cellName: execContext.cellName,
|
|
3974
|
+
researchRunId: execContext.researchRunId,
|
|
3975
|
+
source: execContext.source ?? 'notebook_sql_cell',
|
|
3976
|
+
status: 'success',
|
|
3977
|
+
rowCount: normalized.rowCount ?? normalized.rows.length,
|
|
3978
|
+
durationMs: Date.now() - start,
|
|
3979
|
+
sql: body.sql,
|
|
3980
|
+
});
|
|
3981
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
3982
|
+
status: 'success',
|
|
3983
|
+
resultPreview: normalized,
|
|
3984
|
+
sql: body.sql,
|
|
3985
|
+
});
|
|
3986
|
+
}
|
|
3987
|
+
const payload = serializeJSON(normalized);
|
|
2760
3988
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2761
3989
|
res.end(payload);
|
|
2762
3990
|
}
|
|
@@ -2775,6 +4003,24 @@ export async function startLocalServer(opts) {
|
|
|
2775
4003
|
}));
|
|
2776
4004
|
return;
|
|
2777
4005
|
}
|
|
4006
|
+
if (execContext) {
|
|
4007
|
+
recordNotebookQueryRun(projectRoot, {
|
|
4008
|
+
notebookPath: execContext.notebookPath,
|
|
4009
|
+
cellId: execContext.cellId,
|
|
4010
|
+
cellName: execContext.cellName,
|
|
4011
|
+
researchRunId: execContext.researchRunId,
|
|
4012
|
+
source: execContext.source ?? 'notebook_sql_cell',
|
|
4013
|
+
status: 'error',
|
|
4014
|
+
durationMs: Date.now() - start,
|
|
4015
|
+
errorCode: error instanceof Error ? error.message : String(error),
|
|
4016
|
+
sql: typeof body?.sql === 'string' ? body.sql : undefined,
|
|
4017
|
+
});
|
|
4018
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
4019
|
+
status: 'error',
|
|
4020
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4021
|
+
sql: typeof body?.sql === 'string' ? body.sql : undefined,
|
|
4022
|
+
});
|
|
4023
|
+
}
|
|
2778
4024
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2779
4025
|
res.end(serializeJSON({
|
|
2780
4026
|
columns: [],
|
|
@@ -3116,6 +4362,25 @@ export async function startLocalServer(opts) {
|
|
|
3116
4362
|
}
|
|
3117
4363
|
return;
|
|
3118
4364
|
}
|
|
4365
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/business-360/')) {
|
|
4366
|
+
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/business-360/'.length));
|
|
4367
|
+
try {
|
|
4368
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
4369
|
+
const result = queryBusiness360(graph, rawNodeId);
|
|
4370
|
+
if (!result) {
|
|
4371
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4372
|
+
res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
|
|
4373
|
+
return;
|
|
4374
|
+
}
|
|
4375
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4376
|
+
res.end(serializeJSON(result));
|
|
4377
|
+
}
|
|
4378
|
+
catch (error) {
|
|
4379
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4380
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
4381
|
+
}
|
|
4382
|
+
return;
|
|
4383
|
+
}
|
|
3119
4384
|
if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
|
|
3120
4385
|
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
|
|
3121
4386
|
try {
|
|
@@ -3281,8 +4546,12 @@ export async function startLocalServer(opts) {
|
|
|
3281
4546
|
return;
|
|
3282
4547
|
}
|
|
3283
4548
|
if (req.method === 'POST' && path === '/api/notebook/execute') {
|
|
4549
|
+
let body;
|
|
4550
|
+
let execContext = null;
|
|
4551
|
+
const start = Date.now();
|
|
3284
4552
|
try {
|
|
3285
|
-
|
|
4553
|
+
body = await readJSON(req);
|
|
4554
|
+
execContext = notebookExecutionContext(body.executionContext);
|
|
3286
4555
|
const cell = normalizeNotebookCell(body.cell);
|
|
3287
4556
|
if (!cell) {
|
|
3288
4557
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -3305,6 +4574,26 @@ export async function startLocalServer(opts) {
|
|
|
3305
4574
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
3306
4575
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
3307
4576
|
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
|
|
4577
|
+
const normalized = normalizeQueryResult(rawResult);
|
|
4578
|
+
if (execContext) {
|
|
4579
|
+
recordNotebookQueryRun(projectRoot, {
|
|
4580
|
+
notebookPath: execContext.notebookPath,
|
|
4581
|
+
cellId: execContext.cellId ?? cell.id,
|
|
4582
|
+
cellName: execContext.cellName ?? plan.title ?? resolved.blockName,
|
|
4583
|
+
researchRunId: execContext.researchRunId,
|
|
4584
|
+
source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_cell'),
|
|
4585
|
+
status: 'success',
|
|
4586
|
+
rowCount: normalized.rowCount ?? normalized.rows.length,
|
|
4587
|
+
durationMs: Date.now() - start,
|
|
4588
|
+
sql: plan.sql,
|
|
4589
|
+
objectKey: resolved.blockPath,
|
|
4590
|
+
});
|
|
4591
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
4592
|
+
status: 'success',
|
|
4593
|
+
resultPreview: normalized,
|
|
4594
|
+
sql: plan.sql,
|
|
4595
|
+
});
|
|
4596
|
+
}
|
|
3308
4597
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3309
4598
|
res.end(serializeJSON({
|
|
3310
4599
|
cellType: cell.type,
|
|
@@ -3313,10 +4602,26 @@ export async function startLocalServer(opts) {
|
|
|
3313
4602
|
blockPath: resolved.blockPath,
|
|
3314
4603
|
chartConfig: plan.chartConfig,
|
|
3315
4604
|
tests: plan.tests,
|
|
3316
|
-
result:
|
|
4605
|
+
result: normalized,
|
|
3317
4606
|
}));
|
|
3318
4607
|
}
|
|
3319
4608
|
catch (error) {
|
|
4609
|
+
if (execContext) {
|
|
4610
|
+
recordNotebookQueryRun(projectRoot, {
|
|
4611
|
+
notebookPath: execContext.notebookPath,
|
|
4612
|
+
cellId: execContext.cellId,
|
|
4613
|
+
cellName: execContext.cellName,
|
|
4614
|
+
researchRunId: execContext.researchRunId,
|
|
4615
|
+
source: execContext.source ?? 'notebook_cell',
|
|
4616
|
+
status: 'error',
|
|
4617
|
+
durationMs: Date.now() - start,
|
|
4618
|
+
errorCode: error instanceof Error ? error.message : String(error),
|
|
4619
|
+
});
|
|
4620
|
+
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
4621
|
+
status: 'error',
|
|
4622
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4623
|
+
});
|
|
4624
|
+
}
|
|
3320
4625
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3321
4626
|
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
3322
4627
|
}
|
|
@@ -3845,6 +5150,245 @@ export function prepareLocalExecution(sql, connection, projectRoot, projectConfi
|
|
|
3845
5150
|
connection: normalizedConnection,
|
|
3846
5151
|
};
|
|
3847
5152
|
}
|
|
5153
|
+
export function dashboardRuntimeVariables(dashboard, overrides = {}) {
|
|
5154
|
+
const variables = {};
|
|
5155
|
+
for (const param of dashboard.params ?? []) {
|
|
5156
|
+
if (param.default !== undefined)
|
|
5157
|
+
variables[param.id] = param.default;
|
|
5158
|
+
}
|
|
5159
|
+
for (const filter of dashboard.filters ?? []) {
|
|
5160
|
+
if (filter.default !== undefined)
|
|
5161
|
+
variables[filter.id] = filter.default;
|
|
5162
|
+
}
|
|
5163
|
+
return { ...variables, ...overrides };
|
|
5164
|
+
}
|
|
5165
|
+
export function applyDashboardFiltersToBlockExecution(input) {
|
|
5166
|
+
const variables = { ...input.variables };
|
|
5167
|
+
const sqlParams = [...input.sqlParams];
|
|
5168
|
+
const appliedFilters = [];
|
|
5169
|
+
const skippedFilters = [];
|
|
5170
|
+
const clauses = [];
|
|
5171
|
+
let nextPosition = sqlParams.reduce((max, param) => Math.max(max, param.position), 0);
|
|
5172
|
+
for (const filter of input.dashboard.filters ?? []) {
|
|
5173
|
+
const value = dashboardFilterValue(filter, variables);
|
|
5174
|
+
if (isEmptyDashboardFilterValue(value)) {
|
|
5175
|
+
skippedFilters.push({ filter: filter.id, reason: 'no value supplied' });
|
|
5176
|
+
continue;
|
|
5177
|
+
}
|
|
5178
|
+
const paramNames = bindDashboardFilterToExistingParams(filter, value, input.block, sqlParams, variables);
|
|
5179
|
+
if (paramNames.length > 0) {
|
|
5180
|
+
appliedFilters.push({
|
|
5181
|
+
filter: filter.id,
|
|
5182
|
+
mode: 'parameter',
|
|
5183
|
+
paramNames,
|
|
5184
|
+
});
|
|
5185
|
+
continue;
|
|
5186
|
+
}
|
|
5187
|
+
const binding = resolveDashboardFilterBinding(filter, input.block);
|
|
5188
|
+
if (!binding) {
|
|
5189
|
+
skippedFilters.push({ filter: filter.id, reason: `block "${input.block.name}" does not declare a compatible filter binding` });
|
|
5190
|
+
continue;
|
|
5191
|
+
}
|
|
5192
|
+
const expression = dashboardFilterExpression(binding);
|
|
5193
|
+
if (!expression) {
|
|
5194
|
+
skippedFilters.push({ filter: filter.id, reason: `filter binding "${binding}" is not safe for runtime predicate injection` });
|
|
5195
|
+
continue;
|
|
5196
|
+
}
|
|
5197
|
+
const predicate = buildDashboardFilterPredicate({
|
|
5198
|
+
expression,
|
|
5199
|
+
filterId: filter.id,
|
|
5200
|
+
filterType: filter.type,
|
|
5201
|
+
value,
|
|
5202
|
+
params: sqlParams,
|
|
5203
|
+
nextPosition: () => {
|
|
5204
|
+
nextPosition += 1;
|
|
5205
|
+
return nextPosition;
|
|
5206
|
+
},
|
|
5207
|
+
variables,
|
|
5208
|
+
});
|
|
5209
|
+
if (!predicate) {
|
|
5210
|
+
skippedFilters.push({ filter: filter.id, reason: 'filter value could not be converted into a predicate' });
|
|
5211
|
+
continue;
|
|
5212
|
+
}
|
|
5213
|
+
clauses.push(predicate);
|
|
5214
|
+
appliedFilters.push({
|
|
5215
|
+
filter: filter.id,
|
|
5216
|
+
binding,
|
|
5217
|
+
mode: 'predicate',
|
|
5218
|
+
paramNames: sqlParams
|
|
5219
|
+
.filter((param) => param.name.startsWith(`__dashboard_filter_${normalizeDashboardFilterName(filter.id)}_`))
|
|
5220
|
+
.map((param) => param.name),
|
|
5221
|
+
});
|
|
5222
|
+
}
|
|
5223
|
+
if (clauses.length === 0) {
|
|
5224
|
+
return { sql: input.sql, sqlParams, variables, appliedFilters, skippedFilters };
|
|
5225
|
+
}
|
|
5226
|
+
return {
|
|
5227
|
+
sql: `SELECT * FROM (${stripSqlTerminator(input.sql)}) _dql_filter WHERE ${clauses.join(' AND ')}`,
|
|
5228
|
+
sqlParams,
|
|
5229
|
+
variables,
|
|
5230
|
+
appliedFilters,
|
|
5231
|
+
skippedFilters,
|
|
5232
|
+
};
|
|
5233
|
+
}
|
|
5234
|
+
function dashboardFilterValue(filter, variables) {
|
|
5235
|
+
if (Object.prototype.hasOwnProperty.call(variables, filter.id))
|
|
5236
|
+
return variables[filter.id];
|
|
5237
|
+
if (filter.bindsTo && Object.prototype.hasOwnProperty.call(variables, filter.bindsTo))
|
|
5238
|
+
return variables[filter.bindsTo];
|
|
5239
|
+
return filter.default;
|
|
5240
|
+
}
|
|
5241
|
+
function isEmptyDashboardFilterValue(value) {
|
|
5242
|
+
if (value === undefined || value === null)
|
|
5243
|
+
return true;
|
|
5244
|
+
if (typeof value === 'string' && value.trim() === '')
|
|
5245
|
+
return true;
|
|
5246
|
+
if (Array.isArray(value) && value.length === 0)
|
|
5247
|
+
return true;
|
|
5248
|
+
return false;
|
|
5249
|
+
}
|
|
5250
|
+
function bindDashboardFilterToExistingParams(filter, value, block, sqlParams, variables) {
|
|
5251
|
+
const availableParamNames = new Set(sqlParams.map((param) => param.name));
|
|
5252
|
+
const declaredParamNames = new Set((block.parameterPolicy ?? []).map((entry) => entry.name));
|
|
5253
|
+
const applied = [];
|
|
5254
|
+
const directCandidates = uniqueDashboardStrings([
|
|
5255
|
+
filter.id,
|
|
5256
|
+
filter.bindsTo ?? '',
|
|
5257
|
+
normalizeDashboardFilterName(filter.id),
|
|
5258
|
+
filter.bindsTo ? normalizeDashboardFilterName(filter.bindsTo) : '',
|
|
5259
|
+
]);
|
|
5260
|
+
for (const name of directCandidates) {
|
|
5261
|
+
if (!name || !availableParamNames.has(name))
|
|
5262
|
+
continue;
|
|
5263
|
+
variables[name] = value;
|
|
5264
|
+
applied.push(name);
|
|
5265
|
+
}
|
|
5266
|
+
if (applied.length > 0)
|
|
5267
|
+
return applied;
|
|
5268
|
+
const range = dashboardRangeValue(value);
|
|
5269
|
+
if (range) {
|
|
5270
|
+
const baseNames = uniqueDashboardStrings([
|
|
5271
|
+
filter.id,
|
|
5272
|
+
filter.id.replace(/_?range$/i, ''),
|
|
5273
|
+
filter.bindsTo ?? '',
|
|
5274
|
+
filter.bindsTo ? filter.bindsTo.replace(/_?range$/i, '') : '',
|
|
5275
|
+
].map(normalizeDashboardFilterName));
|
|
5276
|
+
for (const base of baseNames) {
|
|
5277
|
+
const pairs = [
|
|
5278
|
+
[`${base}_start`, `${base}_end`],
|
|
5279
|
+
[`${base}_from`, `${base}_to`],
|
|
5280
|
+
[`start_${base}`, `end_${base}`],
|
|
5281
|
+
];
|
|
5282
|
+
for (const [startName, endName] of pairs) {
|
|
5283
|
+
if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
|
|
5284
|
+
variables[startName] = range.start;
|
|
5285
|
+
variables[endName] = range.end;
|
|
5286
|
+
return [startName, endName];
|
|
5287
|
+
}
|
|
5288
|
+
}
|
|
5289
|
+
}
|
|
5290
|
+
for (const [startName, endName] of [['start_date', 'end_date'], ['date_start', 'date_end'], ['season_start', 'season_end'], ['year_start', 'year_end']]) {
|
|
5291
|
+
if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
|
|
5292
|
+
variables[startName] = range.start;
|
|
5293
|
+
variables[endName] = range.end;
|
|
5294
|
+
return [startName, endName];
|
|
5295
|
+
}
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
for (const name of declaredParamNames) {
|
|
5299
|
+
if (!availableParamNames.has(name))
|
|
5300
|
+
continue;
|
|
5301
|
+
const normalized = normalizeDashboardFilterName(name);
|
|
5302
|
+
if (normalized === normalizeDashboardFilterName(filter.id) || normalized === normalizeDashboardFilterName(filter.bindsTo ?? '')) {
|
|
5303
|
+
variables[name] = value;
|
|
5304
|
+
return [name];
|
|
5305
|
+
}
|
|
5306
|
+
}
|
|
5307
|
+
return [];
|
|
5308
|
+
}
|
|
5309
|
+
function resolveDashboardFilterBinding(filter, block) {
|
|
5310
|
+
const candidates = uniqueDashboardStrings([filter.id, filter.bindsTo ?? '']).map(normalizeDashboardFilterName);
|
|
5311
|
+
for (const entry of block.filterBindings ?? []) {
|
|
5312
|
+
if (candidates.includes(normalizeDashboardFilterName(entry.filter)))
|
|
5313
|
+
return entry.binding;
|
|
5314
|
+
}
|
|
5315
|
+
if (filter.bindsTo && (block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.bindsTo ?? ''))) {
|
|
5316
|
+
return filter.bindsTo;
|
|
5317
|
+
}
|
|
5318
|
+
if ((block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.id))) {
|
|
5319
|
+
return filter.id;
|
|
5320
|
+
}
|
|
5321
|
+
return null;
|
|
5322
|
+
}
|
|
5323
|
+
function dashboardFilterExpression(binding) {
|
|
5324
|
+
const trimmed = binding.trim();
|
|
5325
|
+
const yearMatch = trimmed.match(/^([A-Za-z_][A-Za-z0-9_.]*)\.year$/i);
|
|
5326
|
+
if (yearMatch) {
|
|
5327
|
+
const base = dashboardOutputColumn(yearMatch[1]);
|
|
5328
|
+
return base ? `EXTRACT(YEAR FROM _dql_filter.${base})` : null;
|
|
5329
|
+
}
|
|
5330
|
+
const column = dashboardOutputColumn(trimmed);
|
|
5331
|
+
return column ? `_dql_filter.${column}` : null;
|
|
5332
|
+
}
|
|
5333
|
+
function dashboardOutputColumn(binding) {
|
|
5334
|
+
const cleaned = binding.replace(/[`"[\]]/g, '').trim();
|
|
5335
|
+
if (!/^[A-Za-z_][A-Za-z0-9_.]*$/.test(cleaned))
|
|
5336
|
+
return null;
|
|
5337
|
+
const column = cleaned.split('.').filter(Boolean).at(-1);
|
|
5338
|
+
return column && /^[A-Za-z_][A-Za-z0-9_]*$/.test(column) ? column : null;
|
|
5339
|
+
}
|
|
5340
|
+
function buildDashboardFilterPredicate(input) {
|
|
5341
|
+
const range = input.filterType === 'daterange' ? dashboardRangeValue(input.value) : null;
|
|
5342
|
+
if (range) {
|
|
5343
|
+
const start = addDashboardFilterParam(input, 'start', range.start);
|
|
5344
|
+
const end = addDashboardFilterParam(input, 'end', range.end);
|
|
5345
|
+
return `${input.expression} BETWEEN $${start.position} AND $${end.position}`;
|
|
5346
|
+
}
|
|
5347
|
+
const values = Array.isArray(input.value) ? input.value.filter((item) => !isEmptyDashboardFilterValue(item)) : [input.value];
|
|
5348
|
+
if (values.length === 0)
|
|
5349
|
+
return null;
|
|
5350
|
+
if (values.length === 1) {
|
|
5351
|
+
const param = addDashboardFilterParam(input, 'value', values[0]);
|
|
5352
|
+
return `${input.expression} = $${param.position}`;
|
|
5353
|
+
}
|
|
5354
|
+
const placeholders = values.map((value, index) => {
|
|
5355
|
+
const param = addDashboardFilterParam(input, `value_${index + 1}`, value);
|
|
5356
|
+
return `$${param.position}`;
|
|
5357
|
+
});
|
|
5358
|
+
return `${input.expression} IN (${placeholders.join(', ')})`;
|
|
5359
|
+
}
|
|
5360
|
+
function addDashboardFilterParam(input, suffix, value) {
|
|
5361
|
+
const name = `__dashboard_filter_${normalizeDashboardFilterName(input.filterId)}_${suffix}`;
|
|
5362
|
+
const uniqueName = input.variables[name] === undefined && !input.params.some((param) => param.name === name)
|
|
5363
|
+
? name
|
|
5364
|
+
: `${name}_${input.params.length + 1}`;
|
|
5365
|
+
const param = { name: uniqueName, position: input.nextPosition() };
|
|
5366
|
+
input.params.push(param);
|
|
5367
|
+
input.variables[uniqueName] = value;
|
|
5368
|
+
return param;
|
|
5369
|
+
}
|
|
5370
|
+
function dashboardRangeValue(value) {
|
|
5371
|
+
if (Array.isArray(value) && value.length >= 2 && !isEmptyDashboardFilterValue(value[0]) && !isEmptyDashboardFilterValue(value[1])) {
|
|
5372
|
+
return { start: value[0], end: value[1] };
|
|
5373
|
+
}
|
|
5374
|
+
if (value && typeof value === 'object') {
|
|
5375
|
+
const record = value;
|
|
5376
|
+
const start = record.start ?? record.from ?? record.min;
|
|
5377
|
+
const end = record.end ?? record.to ?? record.max;
|
|
5378
|
+
if (!isEmptyDashboardFilterValue(start) && !isEmptyDashboardFilterValue(end))
|
|
5379
|
+
return { start, end };
|
|
5380
|
+
}
|
|
5381
|
+
return null;
|
|
5382
|
+
}
|
|
5383
|
+
function normalizeDashboardFilterName(value) {
|
|
5384
|
+
return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
5385
|
+
}
|
|
5386
|
+
function stripSqlTerminator(sql) {
|
|
5387
|
+
return sql.trim().replace(/;\s*$/, '');
|
|
5388
|
+
}
|
|
5389
|
+
function uniqueDashboardStrings(values) {
|
|
5390
|
+
return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
|
|
5391
|
+
}
|
|
3848
5392
|
export function resolveDbtMacrosForExecution(sql, projectRoot, projectConfig = {}) {
|
|
3849
5393
|
if (!/\{\{\s*(?:ref|source)\s*\(/i.test(sql))
|
|
3850
5394
|
return sql;
|
|
@@ -4462,9 +6006,9 @@ function buildDatabaseSchemaTree(projectRoot, executor, connection) {
|
|
|
4462
6006
|
return databaseNodes;
|
|
4463
6007
|
})();
|
|
4464
6008
|
}
|
|
4465
|
-
function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
6009
|
+
export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
4466
6010
|
const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
|
|
4467
|
-
if (!normalizedPath
|
|
6011
|
+
if (!isBlockStudioBlockPath(normalizedPath)) {
|
|
4468
6012
|
throw new Error('Invalid block path');
|
|
4469
6013
|
}
|
|
4470
6014
|
const absPath = join(projectRoot, normalizedPath);
|
|
@@ -4479,7 +6023,7 @@ function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
|
4479
6023
|
const metadata = {
|
|
4480
6024
|
name: parsedMetadata.name || companion?.name || fileName,
|
|
4481
6025
|
path: normalizedPath,
|
|
4482
|
-
domain: parsedMetadata.domain || companion?.domain || normalizedPath
|
|
6026
|
+
domain: parsedMetadata.domain || companion?.domain || inferBlockStudioPathDomain(normalizedPath) || 'uncategorized',
|
|
4483
6027
|
description: parsedMetadata.description || companion?.description || '',
|
|
4484
6028
|
owner: parsedMetadata.owner || companion?.owner || '',
|
|
4485
6029
|
tags: parsedMetadata.tags.length > 0 ? parsedMetadata.tags : companion?.tags ?? [],
|
|
@@ -4562,7 +6106,7 @@ export function buildSemanticTableMapping(semanticLayer, rows) {
|
|
|
4562
6106
|
return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
|
|
4563
6107
|
}
|
|
4564
6108
|
function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
|
|
4565
|
-
if (projectConfig.semanticLayer?.provider === 'dbt' || detectedProvider === 'dbt')
|
|
6109
|
+
if (projectConfig.semanticLayer?.provider === 'dbt' || projectConfig.dbt?.projectDir || detectedProvider === 'dbt')
|
|
4566
6110
|
return true;
|
|
4567
6111
|
return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
|
|
4568
6112
|
}
|
|
@@ -4570,7 +6114,10 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
|
4570
6114
|
const useMetricFlow = request.engine === 'metricflow' || (request.engine !== 'native' &&
|
|
4571
6115
|
isDbtSemanticRuntime(context.projectConfig, context.detectedProvider, semanticLayer));
|
|
4572
6116
|
if (useMetricFlow) {
|
|
4573
|
-
const
|
|
6117
|
+
const effectiveSemanticConfig = resolveProjectSemanticConfig(context.projectConfig, context.projectRoot);
|
|
6118
|
+
const dbtProjectPath = effectiveSemanticConfig?.provider === 'dbt'
|
|
6119
|
+
? effectiveSemanticConfig.projectPath
|
|
6120
|
+
: context.projectConfig.dbt?.projectDir;
|
|
4574
6121
|
const compiled = compileMetricFlowQuery({
|
|
4575
6122
|
projectRoot: context.projectRoot,
|
|
4576
6123
|
dbtProjectPath,
|
|
@@ -4829,6 +6376,164 @@ export function validateBlockStudioSource(source, semanticLayer) {
|
|
|
4829
6376
|
executableSql,
|
|
4830
6377
|
};
|
|
4831
6378
|
}
|
|
6379
|
+
export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
6380
|
+
const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
|
|
6381
|
+
const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
|
|
6382
|
+
const provider = deterministicOnly ? null : await createBlockStudioAssistProvider(projectRoot, requestedProvider);
|
|
6383
|
+
const session = createBlockStudioImportSession(projectRoot, {
|
|
6384
|
+
inputPath: options.inputPath ?? '',
|
|
6385
|
+
inputMode: options.inputMode,
|
|
6386
|
+
sources: options.sources,
|
|
6387
|
+
sourceKind: options.sourceKind ?? 'raw-sql',
|
|
6388
|
+
domain: options.domain,
|
|
6389
|
+
owner: options.owner,
|
|
6390
|
+
tags: options.tags,
|
|
6391
|
+
});
|
|
6392
|
+
const warnings = [];
|
|
6393
|
+
const nextCandidates = [];
|
|
6394
|
+
let contextObjectCount = 0;
|
|
6395
|
+
for (const candidate of session.candidates) {
|
|
6396
|
+
const contextPack = await buildDqlGenerationContextPack(projectRoot, candidate).catch((error) => {
|
|
6397
|
+
warnings.push(`Context pack failed for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
6398
|
+
return null;
|
|
6399
|
+
});
|
|
6400
|
+
contextObjectCount += contextPack?.objects.length ?? 0;
|
|
6401
|
+
const evidence = contextPack ? dqlGenerationEvidenceFromContext(contextPack, candidate) : deterministicDqlGenerationEvidence(candidate);
|
|
6402
|
+
let patch = deterministicDqlGenerationPatch(candidate, evidence);
|
|
6403
|
+
let generatorName = 'local-deterministic';
|
|
6404
|
+
let generationMode = 'deterministic';
|
|
6405
|
+
if (provider) {
|
|
6406
|
+
const aiPatch = await buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack).catch((error) => {
|
|
6407
|
+
warnings.push(`AI generation fell back for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
6408
|
+
return null;
|
|
6409
|
+
});
|
|
6410
|
+
if (aiPatch) {
|
|
6411
|
+
patch = mergeDqlGenerationPatch(patch, aiPatch, candidate, evidence);
|
|
6412
|
+
generatorName = provider.name;
|
|
6413
|
+
generationMode = 'ai';
|
|
6414
|
+
}
|
|
6415
|
+
}
|
|
6416
|
+
const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
|
|
6417
|
+
patch = {
|
|
6418
|
+
...patch,
|
|
6419
|
+
similarityMatches: similarity.matches,
|
|
6420
|
+
recommendedAction: similarity.recommendedAction,
|
|
6421
|
+
};
|
|
6422
|
+
for (const warning of patch.warnings ?? [])
|
|
6423
|
+
warnings.push(`${candidate.name}: ${warning}`);
|
|
6424
|
+
const enriched = updateBlockStudioImportCandidate(projectRoot, session.id, candidate.id, {
|
|
6425
|
+
name: patch.name,
|
|
6426
|
+
domain: patch.domain,
|
|
6427
|
+
description: patch.description,
|
|
6428
|
+
owner: patch.owner,
|
|
6429
|
+
tags: patch.tags,
|
|
6430
|
+
terms: patch.terms,
|
|
6431
|
+
pattern: patch.pattern,
|
|
6432
|
+
grain: patch.grain,
|
|
6433
|
+
entities: patch.entities,
|
|
6434
|
+
outputs: patch.outputs,
|
|
6435
|
+
dimensions: patch.dimensions,
|
|
6436
|
+
allowedFilters: patch.allowedFilters,
|
|
6437
|
+
parameterPolicy: patch.parameterPolicy,
|
|
6438
|
+
filterBindings: patch.filterBindings,
|
|
6439
|
+
parameterDecisions: patch.parameterDecisions,
|
|
6440
|
+
similarityMatches: patch.similarityMatches,
|
|
6441
|
+
recommendedAction: patch.recommendedAction,
|
|
6442
|
+
sourceSystems: patch.sourceSystems,
|
|
6443
|
+
replacementFor: patch.replacementFor,
|
|
6444
|
+
reviewCadence: patch.reviewCadence,
|
|
6445
|
+
sql: patch.sql,
|
|
6446
|
+
llmContext: patch.llmContext,
|
|
6447
|
+
evidence,
|
|
6448
|
+
conversionNotes: dqlGenerationConversionNotes(generatorName),
|
|
6449
|
+
generationMode,
|
|
6450
|
+
generationProvider: generatorName,
|
|
6451
|
+
});
|
|
6452
|
+
const validated = {
|
|
6453
|
+
...enriched,
|
|
6454
|
+
validation: validateBlockStudioSource(enriched.dqlSource, semanticLayer),
|
|
6455
|
+
};
|
|
6456
|
+
const savedDraft = saveDqlGenerationDraftForProject(projectRoot, session.id, validated);
|
|
6457
|
+
writeBlockStudioImportCandidate(projectRoot, session.id, savedDraft);
|
|
6458
|
+
nextCandidates.push(savedDraft);
|
|
6459
|
+
}
|
|
6460
|
+
const generationSession = {
|
|
6461
|
+
...session,
|
|
6462
|
+
mode: 'ai-import',
|
|
6463
|
+
candidates: nextCandidates,
|
|
6464
|
+
updatedAt: new Date().toISOString(),
|
|
6465
|
+
generation: {
|
|
6466
|
+
provider: nextCandidates.find((candidate) => candidate.generationMode === 'ai')?.generationProvider ?? 'local-deterministic',
|
|
6467
|
+
aiEnabled: nextCandidates.some((candidate) => candidate.generationMode === 'ai'),
|
|
6468
|
+
contextObjectCount,
|
|
6469
|
+
createdDrafts: nextCandidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
|
|
6470
|
+
warnings,
|
|
6471
|
+
},
|
|
6472
|
+
};
|
|
6473
|
+
writeBlockStudioImportSession(projectRoot, generationSession);
|
|
6474
|
+
if (generationSession.generation.createdDrafts > 0)
|
|
6475
|
+
await refreshLocalMetadataCatalog(projectRoot);
|
|
6476
|
+
return generationSession;
|
|
6477
|
+
}
|
|
6478
|
+
function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
|
|
6479
|
+
if (candidate.recommendedAction === 'reuse_existing') {
|
|
6480
|
+
const topMatch = candidate.similarityMatches?.[0];
|
|
6481
|
+
return {
|
|
6482
|
+
...candidate,
|
|
6483
|
+
reviewStatus: 'review',
|
|
6484
|
+
draftSave: {
|
|
6485
|
+
status: 'skipped',
|
|
6486
|
+
reason: topMatch
|
|
6487
|
+
? `Reuse recommended: ${topMatch.name} (${topMatch.kind}, ${(topMatch.score * 100).toFixed(0)}%).`
|
|
6488
|
+
: 'Reuse recommended; no new draft block was needed.',
|
|
6489
|
+
},
|
|
6490
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
6491
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
6492
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
6493
|
+
evidence: candidate.evidence ?? [],
|
|
6494
|
+
};
|
|
6495
|
+
}
|
|
6496
|
+
try {
|
|
6497
|
+
const savedPath = saveBlockStudioDraftArtifacts(projectRoot, {
|
|
6498
|
+
currentPath: isDraftBlockPath(candidate.savedPath) ? candidate.savedPath : undefined,
|
|
6499
|
+
source: candidate.dqlSource,
|
|
6500
|
+
name: candidate.name,
|
|
6501
|
+
domain: candidate.domain,
|
|
6502
|
+
description: candidate.description,
|
|
6503
|
+
owner: candidate.owner,
|
|
6504
|
+
tags: candidate.tags,
|
|
6505
|
+
lineage: candidate.lineage.sourceTables,
|
|
6506
|
+
stableSuffix: candidate.id.replace(/^cand_/, ''),
|
|
6507
|
+
importMeta: {
|
|
6508
|
+
importId,
|
|
6509
|
+
candidateId: candidate.id,
|
|
6510
|
+
sourceKind: candidate.sourceKind,
|
|
6511
|
+
sourcePath: candidate.sourcePath,
|
|
6512
|
+
},
|
|
6513
|
+
});
|
|
6514
|
+
const draftSave = { status: 'saved', path: savedPath, savedAt: new Date().toISOString() };
|
|
6515
|
+
return {
|
|
6516
|
+
...candidate,
|
|
6517
|
+
reviewStatus: candidate.reviewStatus === 'rejected' ? 'rejected' : 'draft',
|
|
6518
|
+
savedPath,
|
|
6519
|
+
draftSave,
|
|
6520
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
6521
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
6522
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
6523
|
+
evidence: candidate.evidence ?? [],
|
|
6524
|
+
};
|
|
6525
|
+
}
|
|
6526
|
+
catch (error) {
|
|
6527
|
+
return {
|
|
6528
|
+
...candidate,
|
|
6529
|
+
draftSave: { status: 'error', error: error instanceof Error ? error.message : String(error) },
|
|
6530
|
+
generationMode: candidate.generationMode ?? 'deterministic',
|
|
6531
|
+
generationProvider: candidate.generationProvider ?? 'local-deterministic',
|
|
6532
|
+
llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
|
|
6533
|
+
evidence: candidate.evidence ?? [],
|
|
6534
|
+
};
|
|
6535
|
+
}
|
|
6536
|
+
}
|
|
4832
6537
|
export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
4833
6538
|
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
4834
6539
|
const safeDomain = (options.domain ?? '')
|
|
@@ -4836,9 +6541,9 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
|
4836
6541
|
.toLowerCase()
|
|
4837
6542
|
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
4838
6543
|
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
4839
|
-
const targetRelativePath = `blocks/${safeDomain}/${slug}.dql`;
|
|
4840
|
-
const targetPath = join(projectRoot, targetRelativePath);
|
|
4841
6544
|
const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
|
|
6545
|
+
const targetRelativePath = canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath);
|
|
6546
|
+
const targetPath = join(projectRoot, targetRelativePath);
|
|
4842
6547
|
if (existsSync(targetPath) && previousPath !== targetRelativePath) {
|
|
4843
6548
|
throw new Error('BLOCK_EXISTS');
|
|
4844
6549
|
}
|
|
@@ -4869,12 +6574,103 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
|
|
|
4869
6574
|
}
|
|
4870
6575
|
return targetRelativePath;
|
|
4871
6576
|
}
|
|
6577
|
+
export function saveBlockStudioDraftArtifacts(projectRoot, options) {
|
|
6578
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
6579
|
+
const safeDomain = (options.domain ?? '')
|
|
6580
|
+
.trim()
|
|
6581
|
+
.toLowerCase()
|
|
6582
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
6583
|
+
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
6584
|
+
const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
|
|
6585
|
+
const domainFirstRoot = join(projectRoot, 'domains');
|
|
6586
|
+
const useDomainFirstDrafts = existsSync(domainFirstRoot);
|
|
6587
|
+
const draftPrefix = useDomainFirstDrafts
|
|
6588
|
+
? `domains/${safeDomain}/blocks/_drafts/`
|
|
6589
|
+
: `blocks/_drafts/${safeDomain}/`;
|
|
6590
|
+
const stableSuffix = options.stableSuffix?.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 18);
|
|
6591
|
+
let targetRelativePath = previousPath && isDraftBlockPath(previousPath)
|
|
6592
|
+
? previousPath
|
|
6593
|
+
: `${draftPrefix}${slug}${stableSuffix ? `-${stableSuffix}` : ''}.dql`;
|
|
6594
|
+
let targetPath = join(projectRoot, targetRelativePath);
|
|
6595
|
+
if (!previousPath && existsSync(targetPath)) {
|
|
6596
|
+
for (let index = 2; index < 1000; index += 1) {
|
|
6597
|
+
const candidatePath = `${draftPrefix}${slug}-${index}.dql`;
|
|
6598
|
+
if (!existsSync(join(projectRoot, candidatePath))) {
|
|
6599
|
+
targetRelativePath = candidatePath;
|
|
6600
|
+
targetPath = join(projectRoot, targetRelativePath);
|
|
6601
|
+
break;
|
|
6602
|
+
}
|
|
6603
|
+
}
|
|
6604
|
+
}
|
|
6605
|
+
mkdirSync(dirname(targetPath), { recursive: true });
|
|
6606
|
+
writeFileSync(targetPath, options.source.trimEnd() + '\n', 'utf-8');
|
|
6607
|
+
const companionSlug = targetRelativePath.split('/').pop()?.replace(/\.dql$/, '') || slug;
|
|
6608
|
+
writeBlockCompanionFile(projectRoot, {
|
|
6609
|
+
slug: companionSlug,
|
|
6610
|
+
name: options.name,
|
|
6611
|
+
domain: `_drafts/${safeDomain}`,
|
|
6612
|
+
description: options.description,
|
|
6613
|
+
owner: options.owner,
|
|
6614
|
+
tags: options.tags,
|
|
6615
|
+
provider: 'dql',
|
|
6616
|
+
content: options.source,
|
|
6617
|
+
lineage: options.lineage,
|
|
6618
|
+
importMeta: options.importMeta,
|
|
6619
|
+
});
|
|
6620
|
+
return targetRelativePath;
|
|
6621
|
+
}
|
|
6622
|
+
function canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath) {
|
|
6623
|
+
const previousDomainFirst = previousPath?.match(/^domains\/([^/]+)\/blocks\/(?:_drafts\/)?[^/]+\.dql$/);
|
|
6624
|
+
if (previousDomainFirst) {
|
|
6625
|
+
return `domains/${previousDomainFirst[1]}/blocks/${slug}.dql`;
|
|
6626
|
+
}
|
|
6627
|
+
if (existsSync(join(projectRoot, 'domains', safeDomain))) {
|
|
6628
|
+
return `domains/${safeDomain}/blocks/${slug}.dql`;
|
|
6629
|
+
}
|
|
6630
|
+
return `blocks/${safeDomain}/${slug}.dql`;
|
|
6631
|
+
}
|
|
6632
|
+
function isDraftBlockPath(value) {
|
|
6633
|
+
if (!value)
|
|
6634
|
+
return false;
|
|
6635
|
+
const normalized = normalize(value).replace(/^\/+/, '');
|
|
6636
|
+
return normalized.startsWith('blocks/_drafts/') || /^domains\/[^/]+\/blocks\/_drafts\//.test(normalized);
|
|
6637
|
+
}
|
|
6638
|
+
function isBlockStudioBlockPath(value) {
|
|
6639
|
+
if (!value)
|
|
6640
|
+
return false;
|
|
6641
|
+
const normalized = normalize(value).replace(/^\/+/, '');
|
|
6642
|
+
return normalized.startsWith('blocks/') || /^domains\/[^/]+\/blocks\//.test(normalized);
|
|
6643
|
+
}
|
|
6644
|
+
function inferBlockStudioPathDomain(blockPath) {
|
|
6645
|
+
const normalized = normalize(blockPath).replace(/^\/+/, '');
|
|
6646
|
+
if (normalized.startsWith('blocks/')) {
|
|
6647
|
+
return normalized.split('/').slice(1, -1).join('/');
|
|
6648
|
+
}
|
|
6649
|
+
const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)$/);
|
|
6650
|
+
if (!domainFirst)
|
|
6651
|
+
return '';
|
|
6652
|
+
const domain = domainFirst[1];
|
|
6653
|
+
const blockSubpath = domainFirst[2];
|
|
6654
|
+
if (blockSubpath.startsWith('_drafts/'))
|
|
6655
|
+
return `_drafts/${domain}`;
|
|
6656
|
+
return domain;
|
|
6657
|
+
}
|
|
4872
6658
|
function blockCompanionRelativePath(blockPath) {
|
|
4873
6659
|
const normalized = normalize(blockPath).replace(/^\/+/, '');
|
|
4874
|
-
if (
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
|
|
6660
|
+
if (normalized.startsWith('blocks/')) {
|
|
6661
|
+
const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
|
|
6662
|
+
return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
|
|
6663
|
+
}
|
|
6664
|
+
const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)\.dql$/);
|
|
6665
|
+
if (domainFirst) {
|
|
6666
|
+
const domain = domainFirst[1];
|
|
6667
|
+
const blockPath = domainFirst[2];
|
|
6668
|
+
const companionBlockPath = blockPath.startsWith('_drafts/')
|
|
6669
|
+
? join('_drafts', domain, blockPath.slice('_drafts/'.length))
|
|
6670
|
+
: join(domain, blockPath);
|
|
6671
|
+
return join('semantic-layer', 'blocks', `${companionBlockPath}.yaml`).replaceAll('\\', '/');
|
|
6672
|
+
}
|
|
6673
|
+
return null;
|
|
4878
6674
|
}
|
|
4879
6675
|
function readBlockCompanionFile(projectRoot, relativePath) {
|
|
4880
6676
|
const absPath = join(projectRoot, relativePath);
|
|
@@ -4922,18 +6718,48 @@ function readBlockCompanionFile(projectRoot, relativePath) {
|
|
|
4922
6718
|
return null;
|
|
4923
6719
|
}
|
|
4924
6720
|
}
|
|
4925
|
-
function parseBlockSourceMetadata(source) {
|
|
6721
|
+
export function parseBlockSourceMetadata(source) {
|
|
4926
6722
|
const name = source.match(/^\s*block\s+"([^"]+)"/i)?.[1] ?? '';
|
|
4927
6723
|
const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
|
|
4928
|
-
const
|
|
6724
|
+
const extractStringArray = (key) => {
|
|
6725
|
+
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'i'));
|
|
6726
|
+
return match ? (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [];
|
|
6727
|
+
};
|
|
6728
|
+
const extractStringMapSection = (key) => {
|
|
6729
|
+
const match = source.match(new RegExp(`\\b${key}\\s*\\{([\\s\\S]*?)\\n\\s*\\}`, 'i'));
|
|
6730
|
+
if (!match)
|
|
6731
|
+
return [];
|
|
6732
|
+
return match[1]
|
|
6733
|
+
.split(/\r?\n/)
|
|
6734
|
+
.map((line) => line.trim())
|
|
6735
|
+
.map((line) => line.match(/^([A-Za-z_][\w.]*)\s*=\s*"([^"]*)"\s*$/))
|
|
6736
|
+
.filter((entry) => Boolean(entry))
|
|
6737
|
+
.map((entry) => ({ key: entry[1], value: entry[2] }));
|
|
6738
|
+
};
|
|
6739
|
+
const parameterPolicy = extractStringMapSection('parameterPolicy')
|
|
6740
|
+
.map((entry) => ({ name: entry.key, policy: entry.value }));
|
|
6741
|
+
const filterBindings = extractStringMapSection('filterBindings')
|
|
6742
|
+
.map((entry) => ({ filter: entry.key, binding: entry.value }));
|
|
4929
6743
|
return {
|
|
4930
6744
|
name,
|
|
4931
6745
|
domain: extractString('domain'),
|
|
4932
6746
|
description: extractString('description'),
|
|
4933
6747
|
owner: extractString('owner'),
|
|
4934
|
-
tags:
|
|
6748
|
+
tags: extractStringArray('tags'),
|
|
4935
6749
|
status: extractString('status') || 'draft',
|
|
4936
6750
|
blockType: extractString('type') || 'custom',
|
|
6751
|
+
llmContext: extractString('llmContext'),
|
|
6752
|
+
pattern: extractString('pattern'),
|
|
6753
|
+
grain: extractString('grain'),
|
|
6754
|
+
entities: extractStringArray('entities'),
|
|
6755
|
+
outputs: extractStringArray('outputs'),
|
|
6756
|
+
dimensions: extractStringArray('dimensions'),
|
|
6757
|
+
allowedFilters: extractStringArray('allowedFilters'),
|
|
6758
|
+
parameterPolicy,
|
|
6759
|
+
filterBindings,
|
|
6760
|
+
sourceSystems: extractStringArray('sourceSystems'),
|
|
6761
|
+
replacementFor: extractStringArray('replacementFor'),
|
|
6762
|
+
reviewCadence: extractString('reviewCadence'),
|
|
4937
6763
|
};
|
|
4938
6764
|
}
|
|
4939
6765
|
function compareBlockStudioValues(actual, operator, expected) {
|
|
@@ -4990,58 +6816,948 @@ function buildBlockStudioCertificationChecklist(input) {
|
|
|
4990
6816
|
if (!input.validation.chartConfig?.chart)
|
|
4991
6817
|
blockers.add('Visualization config is missing');
|
|
4992
6818
|
return {
|
|
4993
|
-
metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
|
|
4994
|
-
validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
4995
|
-
run: input.previewSucceeded,
|
|
4996
|
-
tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
|
|
4997
|
-
chart: Boolean(input.validation.chartConfig?.chart),
|
|
4998
|
-
lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
|
|
4999
|
-
aiReviewed: true,
|
|
5000
|
-
blockers: Array.from(blockers),
|
|
5001
|
-
checkedAt: new Date().toISOString(),
|
|
6819
|
+
metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
|
|
6820
|
+
validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
6821
|
+
run: input.previewSucceeded,
|
|
6822
|
+
tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
|
|
6823
|
+
chart: Boolean(input.validation.chartConfig?.chart),
|
|
6824
|
+
lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
|
|
6825
|
+
aiReviewed: true,
|
|
6826
|
+
blockers: Array.from(blockers),
|
|
6827
|
+
checkedAt: new Date().toISOString(),
|
|
6828
|
+
};
|
|
6829
|
+
}
|
|
6830
|
+
function extractSqlTablesLight(sql) {
|
|
6831
|
+
const tables = new Set();
|
|
6832
|
+
const cleaned = sql
|
|
6833
|
+
.replace(/\/\*[\s\S]*?\*\//g, ' ')
|
|
6834
|
+
.replace(/--[^\n\r]*/g, ' ')
|
|
6835
|
+
.replace(/extract\s*\(\s*\w+\s+from\s+[^)]+\)/gi, 'EXTRACT_VALUE');
|
|
6836
|
+
const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
|
|
6837
|
+
let match;
|
|
6838
|
+
while ((match = regex.exec(cleaned))) {
|
|
6839
|
+
const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
|
|
6840
|
+
if (raw && !raw.startsWith('(') && !/^(select|values|unnest|lateral)$/i.test(raw))
|
|
6841
|
+
tables.add(raw);
|
|
6842
|
+
}
|
|
6843
|
+
return Array.from(tables);
|
|
6844
|
+
}
|
|
6845
|
+
function setBlockStudioStatusInSource(source, newStatus) {
|
|
6846
|
+
if (/status\s*=\s*"[^"]*"/.test(source)) {
|
|
6847
|
+
return source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
|
|
6848
|
+
}
|
|
6849
|
+
return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
|
|
6850
|
+
}
|
|
6851
|
+
export function setBlockStudioStatus(projectRoot, blockPath, newStatus) {
|
|
6852
|
+
const normalizedPath = normalize(blockPath).replace(/^\/+/, '');
|
|
6853
|
+
if (!isBlockStudioBlockPath(normalizedPath))
|
|
6854
|
+
throw new Error('Invalid block path');
|
|
6855
|
+
const absPath = join(projectRoot, normalizedPath);
|
|
6856
|
+
if (!existsSync(absPath))
|
|
6857
|
+
throw new Error('Block file not found');
|
|
6858
|
+
const source = setBlockStudioStatusInSource(readFileSync(absPath, 'utf-8'), newStatus);
|
|
6859
|
+
writeFileSync(absPath, source, 'utf-8');
|
|
6860
|
+
const companionPath = blockCompanionRelativePath(normalizedPath);
|
|
6861
|
+
if (!companionPath)
|
|
6862
|
+
return;
|
|
6863
|
+
const absCompanionPath = join(projectRoot, companionPath);
|
|
6864
|
+
if (!existsSync(absCompanionPath))
|
|
6865
|
+
return;
|
|
6866
|
+
let companion = readFileSync(absCompanionPath, 'utf-8');
|
|
6867
|
+
if (/^reviewStatus:\s*.+$/m.test(companion)) {
|
|
6868
|
+
companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
|
|
6869
|
+
}
|
|
6870
|
+
else {
|
|
6871
|
+
companion = `${companion.trimEnd()}\nreviewStatus: ${newStatus}\n`;
|
|
6872
|
+
}
|
|
6873
|
+
writeFileSync(absCompanionPath, companion, 'utf-8');
|
|
6874
|
+
}
|
|
6875
|
+
async function buildDqlGenerationContextPack(projectRoot, candidate) {
|
|
6876
|
+
const tables = candidate.lineage.sourceTables.join(', ');
|
|
6877
|
+
const question = [
|
|
6878
|
+
`Generate a governed DQL block draft for SQL import candidate "${candidate.name}".`,
|
|
6879
|
+
candidate.description,
|
|
6880
|
+
tables ? `Source tables: ${tables}.` : '',
|
|
6881
|
+
candidate.sql.slice(0, 1200),
|
|
6882
|
+
].filter(Boolean).join('\n');
|
|
6883
|
+
return buildLocalContextPack(projectRoot, {
|
|
6884
|
+
question,
|
|
6885
|
+
mode: 'build',
|
|
6886
|
+
surface: 'block-studio',
|
|
6887
|
+
limit: 80,
|
|
6888
|
+
objectTypes: [
|
|
6889
|
+
'dql_block',
|
|
6890
|
+
'dql_term',
|
|
6891
|
+
'business_view',
|
|
6892
|
+
'domain',
|
|
6893
|
+
'semantic_metric',
|
|
6894
|
+
'semantic_model',
|
|
6895
|
+
'semantic_dimension',
|
|
6896
|
+
'semantic_measure',
|
|
6897
|
+
'dbt_model',
|
|
6898
|
+
'dbt_source',
|
|
6899
|
+
'dbt_column',
|
|
6900
|
+
'warehouse_table',
|
|
6901
|
+
'datalex_domain',
|
|
6902
|
+
'datalex_entity',
|
|
6903
|
+
'datalex_contract',
|
|
6904
|
+
'datalex_term',
|
|
6905
|
+
],
|
|
6906
|
+
strictness: 'balanced',
|
|
6907
|
+
});
|
|
6908
|
+
}
|
|
6909
|
+
function dqlGenerationEvidenceFromContext(contextPack, candidate) {
|
|
6910
|
+
const directEvidence = contextPack.objects
|
|
6911
|
+
.filter((object) => isDqlGenerationEvidenceObject(object))
|
|
6912
|
+
.filter((object) => metadataObjectMatchesImportedSource(object, candidate.lineage.sourceTables))
|
|
6913
|
+
.map((object) => metadataObjectToDqlGenerationEvidence(object, 'Directly matches an imported SQL source table.'));
|
|
6914
|
+
const selected = contextPack.retrievalDiagnostics.selectedEvidence.slice(0, 12);
|
|
6915
|
+
const byKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
|
|
6916
|
+
const evidence = selected.flatMap((item) => {
|
|
6917
|
+
const object = byKey.get(item.objectKey);
|
|
6918
|
+
if (!object || !isDqlGenerationEvidenceObject(object))
|
|
6919
|
+
return [];
|
|
6920
|
+
return [metadataObjectToDqlGenerationEvidence(object, item.reason, Math.max(0.35, Math.min(0.98, item.score / Math.max(item.score, 10))))];
|
|
6921
|
+
});
|
|
6922
|
+
const ranked = uniqueDqlGenerationEvidence([...directEvidence, ...evidence]).slice(0, 12);
|
|
6923
|
+
return ranked.length > 0 ? ranked : deterministicDqlGenerationEvidence(candidate);
|
|
6924
|
+
}
|
|
6925
|
+
function deterministicDqlGenerationEvidence(candidate) {
|
|
6926
|
+
return candidate.lineage.sourceTables.map((table) => ({
|
|
6927
|
+
kind: 'warehouse_table',
|
|
6928
|
+
name: table,
|
|
6929
|
+
source: candidate.sourcePath,
|
|
6930
|
+
reason: 'Detected in the imported SQL source.',
|
|
6931
|
+
confidence: 0.7,
|
|
6932
|
+
}));
|
|
6933
|
+
}
|
|
6934
|
+
function isDqlGenerationEvidenceObject(object) {
|
|
6935
|
+
if (object.objectType === 'dql_block') {
|
|
6936
|
+
if (object.status !== 'certified')
|
|
6937
|
+
return false;
|
|
6938
|
+
if (object.sourcePath?.includes('/_drafts/') || object.sourcePath?.includes('\\_drafts\\'))
|
|
6939
|
+
return false;
|
|
6940
|
+
}
|
|
6941
|
+
return true;
|
|
6942
|
+
}
|
|
6943
|
+
function metadataObjectToDqlGenerationEvidence(object, reason, confidence = 0.82) {
|
|
6944
|
+
return {
|
|
6945
|
+
kind: dqlGenerationEvidenceKind(object.objectType),
|
|
6946
|
+
name: object.fullName ?? object.name,
|
|
6947
|
+
description: object.description,
|
|
6948
|
+
objectKey: object.objectKey,
|
|
6949
|
+
source: object.sourcePath ?? object.sourceSystem,
|
|
6950
|
+
reason,
|
|
6951
|
+
confidence,
|
|
6952
|
+
};
|
|
6953
|
+
}
|
|
6954
|
+
function metadataObjectMatchesImportedSource(object, sourceTables) {
|
|
6955
|
+
if (sourceTables.length === 0)
|
|
6956
|
+
return false;
|
|
6957
|
+
const haystack = [
|
|
6958
|
+
object.fullName,
|
|
6959
|
+
object.name,
|
|
6960
|
+
object.objectKey,
|
|
6961
|
+
object.sourcePath,
|
|
6962
|
+
object.sourceSystem,
|
|
6963
|
+
typeof object.payload?.relation === 'string' ? object.payload.relation : '',
|
|
6964
|
+
typeof object.payload?.relationName === 'string' ? object.payload.relationName : '',
|
|
6965
|
+
typeof object.payload?.relation_name === 'string' ? object.payload.relation_name : '',
|
|
6966
|
+
].filter(Boolean).join(' ').toLowerCase();
|
|
6967
|
+
return sourceTables.some((table) => relationLookupTokens(table).some((token) => token && haystack.includes(token)));
|
|
6968
|
+
}
|
|
6969
|
+
function relationLookupTokens(relation) {
|
|
6970
|
+
const normalized = relation.replace(/[`"[\]]/g, '').toLowerCase();
|
|
6971
|
+
const parts = normalized.split('.').filter(Boolean);
|
|
6972
|
+
return Array.from(new Set([
|
|
6973
|
+
normalized,
|
|
6974
|
+
parts.slice(-2).join('.'),
|
|
6975
|
+
parts.at(-1) ?? '',
|
|
6976
|
+
].filter((item) => item.length >= 3)));
|
|
6977
|
+
}
|
|
6978
|
+
function uniqueDqlGenerationEvidence(evidence) {
|
|
6979
|
+
const seen = new Set();
|
|
6980
|
+
const result = [];
|
|
6981
|
+
for (const item of evidence) {
|
|
6982
|
+
const key = item.objectKey || `${item.kind}:${item.name}`.toLowerCase();
|
|
6983
|
+
if (seen.has(key))
|
|
6984
|
+
continue;
|
|
6985
|
+
seen.add(key);
|
|
6986
|
+
result.push(item);
|
|
6987
|
+
}
|
|
6988
|
+
return result;
|
|
6989
|
+
}
|
|
6990
|
+
function dqlGenerationEvidenceKind(objectType) {
|
|
6991
|
+
if (objectType === 'dql_block')
|
|
6992
|
+
return 'dql_block';
|
|
6993
|
+
if (objectType === 'dql_term')
|
|
6994
|
+
return 'dql_term';
|
|
6995
|
+
if (objectType === 'business_view')
|
|
6996
|
+
return 'business_view';
|
|
6997
|
+
if (objectType === 'domain')
|
|
6998
|
+
return 'domain';
|
|
6999
|
+
if (objectType === 'semantic_metric')
|
|
7000
|
+
return 'semantic_metric';
|
|
7001
|
+
if (objectType === 'semantic_model')
|
|
7002
|
+
return 'semantic_model';
|
|
7003
|
+
if (objectType === 'dbt_model' || objectType === 'dbt_source' || objectType === 'dbt_column')
|
|
7004
|
+
return 'dbt_model';
|
|
7005
|
+
if (objectType === 'warehouse_table')
|
|
7006
|
+
return 'warehouse_table';
|
|
7007
|
+
if (objectType === 'datalex_contract')
|
|
7008
|
+
return 'datalex_contract';
|
|
7009
|
+
if (objectType === 'datalex_entity')
|
|
7010
|
+
return 'datalex_entity';
|
|
7011
|
+
if (objectType === 'datalex_domain')
|
|
7012
|
+
return 'datalex_domain';
|
|
7013
|
+
if (objectType === 'datalex_term')
|
|
7014
|
+
return 'datalex_term';
|
|
7015
|
+
if (objectType.includes('lineage'))
|
|
7016
|
+
return 'lineage';
|
|
7017
|
+
return 'metadata';
|
|
7018
|
+
}
|
|
7019
|
+
function deterministicDqlGenerationPatch(candidate, evidence) {
|
|
7020
|
+
const domain = inferDqlGenerationDomain(candidate);
|
|
7021
|
+
const parameterized = parameterizeSqlForDqlImport(candidate.sql);
|
|
7022
|
+
const grain = extractDqlGenerationGroupByFields(parameterized.sql)[0];
|
|
7023
|
+
const outputs = extractDqlGenerationSelectOutputs(parameterized.sql);
|
|
7024
|
+
const sourceSystems = candidate.lineage.sourceTables.map((table) => table.split('.').filter(Boolean).slice(-2, -1)[0] ?? '').filter(Boolean);
|
|
7025
|
+
const description = candidate.description && !candidate.description.startsWith('Imported from ')
|
|
7026
|
+
? candidate.description
|
|
7027
|
+
: deterministicDqlGenerationDescription(candidate, evidence);
|
|
7028
|
+
const allowedFilters = Array.from(new Set([
|
|
7029
|
+
...parameterized.allowedFilters,
|
|
7030
|
+
...extractDqlGenerationFilterFields(parameterized.sql),
|
|
7031
|
+
])).slice(0, 16);
|
|
7032
|
+
return {
|
|
7033
|
+
name: candidate.name,
|
|
7034
|
+
domain,
|
|
7035
|
+
description,
|
|
7036
|
+
owner: candidate.owner,
|
|
7037
|
+
tags: dqlGenerationBusinessTags(candidate, evidence, domain),
|
|
7038
|
+
terms: inferDqlGenerationTerms(evidence),
|
|
7039
|
+
llmContext: deterministicDqlGenerationContext({ ...candidate, sql: parameterized.sql }, evidence),
|
|
7040
|
+
pattern: inferDqlGenerationPattern(parameterized.sql),
|
|
7041
|
+
grain,
|
|
7042
|
+
entities: inferDqlGenerationEntities({
|
|
7043
|
+
grain,
|
|
7044
|
+
outputs,
|
|
7045
|
+
sourceTables: candidate.lineage.sourceTables,
|
|
7046
|
+
evidence,
|
|
7047
|
+
}),
|
|
7048
|
+
outputs,
|
|
7049
|
+
dimensions: extractDqlGenerationDimensions(parameterized.sql, grain, outputs),
|
|
7050
|
+
allowedFilters,
|
|
7051
|
+
parameterPolicy: parameterized.parameterPolicy,
|
|
7052
|
+
filterBindings: parameterized.filterBindings,
|
|
7053
|
+
parameterDecisions: parameterized.parameterDecisions,
|
|
7054
|
+
sql: parameterized.sql,
|
|
7055
|
+
warnings: parameterized.warnings,
|
|
7056
|
+
sourceSystems,
|
|
7057
|
+
reviewCadence: 'monthly',
|
|
7058
|
+
};
|
|
7059
|
+
}
|
|
7060
|
+
function inferDqlGenerationTerms(evidence) {
|
|
7061
|
+
return Array.from(new Set(evidence
|
|
7062
|
+
.filter((item) => item.kind === 'dql_term' || item.kind === 'datalex_term')
|
|
7063
|
+
.map((item) => item.name.trim())
|
|
7064
|
+
.filter(Boolean))).slice(0, 16);
|
|
7065
|
+
}
|
|
7066
|
+
function deterministicDqlGenerationDescription(candidate, evidence) {
|
|
7067
|
+
const tables = candidate.lineage.sourceTables.length > 0
|
|
7068
|
+
? candidate.lineage.sourceTables.join(', ')
|
|
7069
|
+
: candidate.sourcePath;
|
|
7070
|
+
const sourceContext = evidence.find((item) => (item.kind === 'dbt_model' || item.kind === 'semantic_model' || item.kind === 'warehouse_table') &&
|
|
7071
|
+
item.description &&
|
|
7072
|
+
candidate.lineage.sourceTables.some((table) => relationLookupTokens(table).some((token) => item.name.toLowerCase().includes(token))));
|
|
7073
|
+
const aggregate = /\b(sum|count|avg|min|max)\s*\(/i.test(candidate.sql);
|
|
7074
|
+
const grouped = /\bgroup\s+by\b/i.test(candidate.sql);
|
|
7075
|
+
const sourcePhrase = sourceContext?.description
|
|
7076
|
+
? ` using ${sourceContext.name} (${sourceContext.description})`
|
|
7077
|
+
: ` using ${tables}`;
|
|
7078
|
+
if (aggregate && grouped)
|
|
7079
|
+
return `Summarizes imported SQL${sourcePhrase}.`;
|
|
7080
|
+
if (aggregate)
|
|
7081
|
+
return `Calculates aggregate values${sourcePhrase}.`;
|
|
7082
|
+
return `Exposes imported SQL${sourcePhrase}.`;
|
|
7083
|
+
}
|
|
7084
|
+
function dqlGenerationBusinessTags(candidate, evidence, domain) {
|
|
7085
|
+
const generic = new Set(['imported', 'raw-sql', 'ai-generated']);
|
|
7086
|
+
const stopTags = new Set(['int', 'stg', 'src', 'fct', 'dim', 'name', 'id', 'key', 'model', 'source', 'table']);
|
|
7087
|
+
const sourceTokens = candidate.lineage.sourceTables
|
|
7088
|
+
.flatMap((table) => table.split('.').slice(-2))
|
|
7089
|
+
.flatMap((part) => part.split(/[_\s.-]+/));
|
|
7090
|
+
const tags = [
|
|
7091
|
+
...candidate.tags.filter((tag) => !generic.has(tag)),
|
|
7092
|
+
domain,
|
|
7093
|
+
...sourceTokens,
|
|
7094
|
+
'review-required',
|
|
7095
|
+
].map((tag) => tag.trim().toLowerCase())
|
|
7096
|
+
.filter((tag) => tag.length > 1 && !generic.has(tag) && !stopTags.has(tag))
|
|
7097
|
+
.map((tag) => tag.replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, ''))
|
|
7098
|
+
.filter(Boolean);
|
|
7099
|
+
return Array.from(new Set(tags)).slice(0, 12);
|
|
7100
|
+
}
|
|
7101
|
+
function deterministicDqlGenerationContext(candidate, evidence) {
|
|
7102
|
+
const tables = candidate.lineage.sourceTables.length > 0
|
|
7103
|
+
? candidate.lineage.sourceTables.join(', ')
|
|
7104
|
+
: 'the imported SQL source';
|
|
7105
|
+
const evidenceNames = evidence
|
|
7106
|
+
.filter((item) => item.kind !== 'dql_block')
|
|
7107
|
+
.slice(0, 3)
|
|
7108
|
+
.map((item) => item.name)
|
|
7109
|
+
.filter(Boolean);
|
|
7110
|
+
const grain = extractDqlGenerationGroupByFields(candidate.sql);
|
|
7111
|
+
const years = extractDqlGenerationYearFilters(candidate.sql);
|
|
7112
|
+
return [
|
|
7113
|
+
`Use after review for questions matching "${candidate.name}".`,
|
|
7114
|
+
grain.length > 0 ? `Grain: ${grain.join(', ')}.` : '',
|
|
7115
|
+
years.length > 0 ? `Filters: year ${years.join(', ')}.` : '',
|
|
7116
|
+
`Source: ${tables}.`,
|
|
7117
|
+
evidenceNames.length > 0 ? `Grounded by: ${evidenceNames.join(', ')}.` : '',
|
|
7118
|
+
'Certification must confirm grain, filters, owner, tags, and tests.',
|
|
7119
|
+
].filter(Boolean).join(' ');
|
|
7120
|
+
}
|
|
7121
|
+
function extractDqlGenerationGroupByFields(sql) {
|
|
7122
|
+
const match = sql.match(/\bgroup\s+by\b([\s\S]+?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i);
|
|
7123
|
+
if (!match)
|
|
7124
|
+
return [];
|
|
7125
|
+
return splitDqlGenerationSqlList(match[1])
|
|
7126
|
+
.map((item) => item.replace(/[`"[\]]/g, '').trim())
|
|
7127
|
+
.filter((item) => item && !/^\d+$/.test(item))
|
|
7128
|
+
.slice(0, 4);
|
|
7129
|
+
}
|
|
7130
|
+
function extractDqlGenerationSelectOutputs(sql) {
|
|
7131
|
+
const selectMatch = sql.match(/\bselect\b([\s\S]+?)\bfrom\b/i);
|
|
7132
|
+
if (!selectMatch)
|
|
7133
|
+
return [];
|
|
7134
|
+
return splitDqlGenerationSqlList(selectMatch[1])
|
|
7135
|
+
.map((expr) => {
|
|
7136
|
+
const alias = expr.match(/\bas\s+([A-Za-z_][A-Za-z0-9_]*)\b/i)?.[1]
|
|
7137
|
+
?? expr.match(/([A-Za-z_][A-Za-z0-9_]*)\s*$/)?.[1]
|
|
7138
|
+
?? '';
|
|
7139
|
+
return alias.replace(/[`"[\]]/g, '');
|
|
7140
|
+
})
|
|
7141
|
+
.filter(Boolean)
|
|
7142
|
+
.filter((name) => !/^(from|where|group|order|limit)$/i.test(name))
|
|
7143
|
+
.slice(0, 24);
|
|
7144
|
+
}
|
|
7145
|
+
function extractDqlGenerationDimensions(sql, grain, outputs) {
|
|
7146
|
+
const outputSet = new Set(outputs.map(normalizedTerm));
|
|
7147
|
+
return extractDqlGenerationGroupByFields(sql)
|
|
7148
|
+
.map((field) => field.split('.').pop() ?? field)
|
|
7149
|
+
.filter((field) => normalizedTerm(field) !== normalizedTerm(grain ?? ''))
|
|
7150
|
+
.filter((field) => !outputSet.has(normalizedTerm(field)))
|
|
7151
|
+
.filter((field) => !/\b(total|count|sum|avg|average|min|max|rate|pct|percent|amount|revenue|points?|score)\b/i.test(field))
|
|
7152
|
+
.slice(0, 16);
|
|
7153
|
+
}
|
|
7154
|
+
function inferDqlGenerationEntities(input) {
|
|
7155
|
+
const entities = new Set();
|
|
7156
|
+
const add = (value) => {
|
|
7157
|
+
const entity = businessEntityFromDqlGenerationIdentifier(value ?? '');
|
|
7158
|
+
if (entity)
|
|
7159
|
+
entities.add(entity);
|
|
7160
|
+
};
|
|
7161
|
+
add(input.grain);
|
|
7162
|
+
for (const output of input.outputs)
|
|
7163
|
+
add(output);
|
|
7164
|
+
for (const table of input.sourceTables)
|
|
7165
|
+
add(table.split('.').pop());
|
|
7166
|
+
for (const item of input.evidence.slice(0, 12)) {
|
|
7167
|
+
if (item.kind === 'datalex_entity' || item.kind === 'semantic_model' || item.kind === 'dbt_model' || item.kind === 'warehouse_table') {
|
|
7168
|
+
add(item.name.split('.').pop());
|
|
7169
|
+
}
|
|
7170
|
+
}
|
|
7171
|
+
return [...entities].slice(0, 8);
|
|
7172
|
+
}
|
|
7173
|
+
function businessEntityFromDqlGenerationIdentifier(value) {
|
|
7174
|
+
const raw = value
|
|
7175
|
+
.split('.').pop()
|
|
7176
|
+
?.trim()
|
|
7177
|
+
.replace(/[`"[\]]/g, '')
|
|
7178
|
+
.replace(/^(dim|fact|fct|stg|src|int)_/i, '')
|
|
7179
|
+
.replace(/_(id|key|name|code|uuid|number|num|abbreviation|abbr)$/i, '')
|
|
7180
|
+
.replace(/s$/i, '') ?? '';
|
|
7181
|
+
if (!raw)
|
|
7182
|
+
return '';
|
|
7183
|
+
const tokens = raw
|
|
7184
|
+
.split(/[^A-Za-z0-9]+/)
|
|
7185
|
+
.map((token) => token.trim().toLowerCase())
|
|
7186
|
+
.filter(Boolean)
|
|
7187
|
+
.filter((token) => !isGenericDqlGenerationEntityToken(token));
|
|
7188
|
+
if (tokens.length === 0)
|
|
7189
|
+
return '';
|
|
7190
|
+
return titleizeDqlGenerationName(tokens.join(' '));
|
|
7191
|
+
}
|
|
7192
|
+
function isGenericDqlGenerationEntityToken(token) {
|
|
7193
|
+
return new Set([
|
|
7194
|
+
'total',
|
|
7195
|
+
'count',
|
|
7196
|
+
'sum',
|
|
7197
|
+
'avg',
|
|
7198
|
+
'average',
|
|
7199
|
+
'min',
|
|
7200
|
+
'max',
|
|
7201
|
+
'metric',
|
|
7202
|
+
'measure',
|
|
7203
|
+
'value',
|
|
7204
|
+
'amount',
|
|
7205
|
+
'revenue',
|
|
7206
|
+
'point',
|
|
7207
|
+
'points',
|
|
7208
|
+
'score',
|
|
7209
|
+
'date',
|
|
7210
|
+
'day',
|
|
7211
|
+
'week',
|
|
7212
|
+
'month',
|
|
7213
|
+
'quarter',
|
|
7214
|
+
'year',
|
|
7215
|
+
'season',
|
|
7216
|
+
'period',
|
|
7217
|
+
'time',
|
|
7218
|
+
'row',
|
|
7219
|
+
'record',
|
|
7220
|
+
'detail',
|
|
7221
|
+
'stat',
|
|
7222
|
+
'stats',
|
|
7223
|
+
'analytic',
|
|
7224
|
+
'analytics',
|
|
7225
|
+
'transformed',
|
|
7226
|
+
]).has(token);
|
|
7227
|
+
}
|
|
7228
|
+
function titleizeDqlGenerationName(value) {
|
|
7229
|
+
return value
|
|
7230
|
+
.replace(/[_-]+/g, ' ')
|
|
7231
|
+
.replace(/\s+/g, ' ')
|
|
7232
|
+
.trim()
|
|
7233
|
+
.replace(/\b\w/g, (char) => char.toUpperCase());
|
|
7234
|
+
}
|
|
7235
|
+
function inferDqlGenerationPattern(sql) {
|
|
7236
|
+
const groupFields = extractDqlGenerationGroupByFields(sql);
|
|
7237
|
+
if (/@metric\s*\(/i.test(sql))
|
|
7238
|
+
return 'metric_wrapper';
|
|
7239
|
+
if (/\bjoin\b/i.test(sql)) {
|
|
7240
|
+
const systems = new Set(extractDqlGenerationSourceSystems(sql));
|
|
7241
|
+
if (systems.size > 1)
|
|
7242
|
+
return 'bridge';
|
|
7243
|
+
}
|
|
7244
|
+
if (hasDqlGenerationRankingLimit(sql))
|
|
7245
|
+
return 'ranking';
|
|
7246
|
+
if (groupFields.some((field) => /\b(date|day|week|month|quarter|year|period|time)\b/i.test(field)))
|
|
7247
|
+
return 'trend';
|
|
7248
|
+
if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
|
|
7249
|
+
return 'entity_rollup';
|
|
7250
|
+
if (!/\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql) && /\b(dim|profile|customer|account|player|product|user|entity)\b/i.test(sql)) {
|
|
7251
|
+
return 'entity_profile';
|
|
7252
|
+
}
|
|
7253
|
+
return 'custom';
|
|
7254
|
+
}
|
|
7255
|
+
function hasDqlGenerationRankingLimit(sql) {
|
|
7256
|
+
return /\border\s+by\b[\s\S]*\blimit\s+(?:\d+|\$\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}|[:?][A-Za-z_][A-Za-z0-9_]*)/i.test(sql);
|
|
7257
|
+
}
|
|
7258
|
+
function extractDqlGenerationSourceSystems(sql) {
|
|
7259
|
+
const tables = new Set();
|
|
7260
|
+
const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
|
|
7261
|
+
const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
|
|
7262
|
+
let match;
|
|
7263
|
+
while ((match = regex.exec(cleaned))) {
|
|
7264
|
+
const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
|
|
7265
|
+
const parts = raw.split('.').filter(Boolean);
|
|
7266
|
+
const system = parts.slice(-2, -1)[0] ?? parts[0];
|
|
7267
|
+
if (system)
|
|
7268
|
+
tables.add(system.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, '-'));
|
|
7269
|
+
}
|
|
7270
|
+
return [...tables].filter(Boolean).slice(0, 12);
|
|
7271
|
+
}
|
|
7272
|
+
function extractDqlGenerationFilterFields(sql) {
|
|
7273
|
+
const filters = new Set();
|
|
7274
|
+
const where = sql.match(/\bwhere\b([\s\S]+?)(?:\bgroup\s+by\b|\border\s+by\b|\blimit\b|$)/i)?.[1] ?? '';
|
|
7275
|
+
const addFilter = (value) => {
|
|
7276
|
+
const name = value?.split('.').pop()?.replace(/[`"[\]]/g, '');
|
|
7277
|
+
if (name && !/^(and|or|not|null|year|month|day)$/i.test(name))
|
|
7278
|
+
filters.add(name);
|
|
7279
|
+
};
|
|
7280
|
+
const extractRegex = /\bextract\s*\([^)]*\bfrom\s+([A-Za-z_][A-Za-z0-9_.]*)\s*\)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
|
|
7281
|
+
const isNullRegex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s+is\s+(?:not\s+)?null\b/gi;
|
|
7282
|
+
const regex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
|
|
7283
|
+
let match;
|
|
7284
|
+
while ((match = extractRegex.exec(where)))
|
|
7285
|
+
addFilter(match[1]);
|
|
7286
|
+
while ((match = isNullRegex.exec(where)))
|
|
7287
|
+
addFilter(match[1]);
|
|
7288
|
+
while ((match = regex.exec(where))) {
|
|
7289
|
+
addFilter(match[1]);
|
|
7290
|
+
}
|
|
7291
|
+
return [...filters].slice(0, 16);
|
|
7292
|
+
}
|
|
7293
|
+
function extractDqlGenerationYearFilters(sql) {
|
|
7294
|
+
const years = new Set();
|
|
7295
|
+
const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
|
|
7296
|
+
let match;
|
|
7297
|
+
const extractYear = /extract\s*\(\s*year\s+from\s+[^)]+\)\s*(?:=\s*([12][0-9]{3})|in\s*\(([^)]*)\))/gi;
|
|
7298
|
+
while ((match = extractYear.exec(cleaned))) {
|
|
7299
|
+
for (const year of (match[1] || match[2] || '').match(/[12][0-9]{3}/g) ?? [])
|
|
7300
|
+
years.add(year);
|
|
7301
|
+
}
|
|
7302
|
+
return Array.from(years).sort();
|
|
7303
|
+
}
|
|
7304
|
+
function splitDqlGenerationSqlList(value) {
|
|
7305
|
+
const parts = [];
|
|
7306
|
+
let start = 0;
|
|
7307
|
+
let depth = 0;
|
|
7308
|
+
let single = false;
|
|
7309
|
+
let double = false;
|
|
7310
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
7311
|
+
const char = value[i];
|
|
7312
|
+
if (!double && char === "'" && value[i - 1] !== '\\')
|
|
7313
|
+
single = !single;
|
|
7314
|
+
else if (!single && char === '"' && value[i - 1] !== '\\')
|
|
7315
|
+
double = !double;
|
|
7316
|
+
else if (!single && !double && char === '(')
|
|
7317
|
+
depth += 1;
|
|
7318
|
+
else if (!single && !double && char === ')' && depth > 0)
|
|
7319
|
+
depth -= 1;
|
|
7320
|
+
else if (!single && !double && depth === 0 && char === ',') {
|
|
7321
|
+
parts.push(value.slice(start, i));
|
|
7322
|
+
start = i + 1;
|
|
7323
|
+
}
|
|
7324
|
+
}
|
|
7325
|
+
parts.push(value.slice(start));
|
|
7326
|
+
return parts.map((part) => part.trim()).filter(Boolean);
|
|
7327
|
+
}
|
|
7328
|
+
function inferDqlGenerationDomain(candidate) {
|
|
7329
|
+
const current = candidate.domain?.trim();
|
|
7330
|
+
if (current && current !== 'imported' && current !== 'general')
|
|
7331
|
+
return current;
|
|
7332
|
+
const firstTable = candidate.lineage.sourceTables[0];
|
|
7333
|
+
const schema = firstTable?.split('.').filter(Boolean).slice(-2, -1)[0];
|
|
7334
|
+
return schema?.replace(/[^a-z0-9/_-]+/gi, '-').toLowerCase() || current || 'imported';
|
|
7335
|
+
}
|
|
7336
|
+
async function buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack) {
|
|
7337
|
+
const payload = {
|
|
7338
|
+
candidate: {
|
|
7339
|
+
name: candidate.name,
|
|
7340
|
+
domain: candidate.domain,
|
|
7341
|
+
description: candidate.description,
|
|
7342
|
+
owner: candidate.owner,
|
|
7343
|
+
tags: candidate.tags,
|
|
7344
|
+
sql: candidate.sql,
|
|
7345
|
+
sourceTables: candidate.lineage.sourceTables,
|
|
7346
|
+
parameters: candidate.lineage.parameters,
|
|
7347
|
+
warnings: candidate.warnings ?? candidate.lineage.warnings,
|
|
7348
|
+
grain: extractDqlGenerationGroupByFields(candidate.sql),
|
|
7349
|
+
yearFilters: extractDqlGenerationYearFilters(candidate.sql),
|
|
7350
|
+
parameterization: parameterizeSqlForDqlImport(candidate.sql),
|
|
7351
|
+
},
|
|
7352
|
+
context: {
|
|
7353
|
+
evidence,
|
|
7354
|
+
selectedObjects: contextPack?.objects.slice(0, 24).map((object) => ({
|
|
7355
|
+
objectKey: object.objectKey,
|
|
7356
|
+
objectType: object.objectType,
|
|
7357
|
+
name: object.fullName ?? object.name,
|
|
7358
|
+
domain: object.domain,
|
|
7359
|
+
status: object.status,
|
|
7360
|
+
description: object.description,
|
|
7361
|
+
})) ?? [],
|
|
7362
|
+
allowedRelations: contextPack?.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
|
|
7363
|
+
relation: relation.relation,
|
|
7364
|
+
source: relation.source,
|
|
7365
|
+
columns: relation.columns.slice(0, 16).map((column) => column.name),
|
|
7366
|
+
})) ?? [],
|
|
7367
|
+
},
|
|
7368
|
+
};
|
|
7369
|
+
const text = await provider.generate([
|
|
7370
|
+
{
|
|
7371
|
+
role: 'system',
|
|
7372
|
+
content: [
|
|
7373
|
+
'You generate DQL Block Studio metadata for a local draft.',
|
|
7374
|
+
'Return only a compact JSON object with optional keys: name, domain, description, owner, tags, terms, llmContext, pattern, grain, entities, outputs, dimensions, allowedFilters, sourceSystems, replacementFor, reviewCadence.',
|
|
7375
|
+
'Do not return markdown. Do not mark the block certified. Do not change SQL.',
|
|
7376
|
+
'Use only directly relevant dbt, semantic, warehouse, certified block, and SQL-shape evidence from the payload.',
|
|
7377
|
+
'terms must reference existing DQL/DataLex glossary terms from the payload. Do not invent term names.',
|
|
7378
|
+
'If metadata descriptions are missing, describe the observable SQL intent instead of inventing business meaning.',
|
|
7379
|
+
'Descriptions and llmContext must be specific, business-readable, concise, and review-required.',
|
|
7380
|
+
'reviewCadence should usually be monthly or quarterly unless evidence strongly suggests a different cadence.',
|
|
7381
|
+
'Tags must be business/search tags. Avoid generic tags such as raw-sql, imported, ai-generated.',
|
|
7382
|
+
].join('\n'),
|
|
7383
|
+
},
|
|
7384
|
+
{ role: 'user', content: JSON.stringify(payload) },
|
|
7385
|
+
], { maxTokens: 900, temperature: 0.1 });
|
|
7386
|
+
return normalizeDqlGenerationPatch(parseFirstJsonObject(text));
|
|
7387
|
+
}
|
|
7388
|
+
function parseFirstJsonObject(text) {
|
|
7389
|
+
const trimmed = text.trim();
|
|
7390
|
+
if (trimmed.startsWith('{') && trimmed.endsWith('}'))
|
|
7391
|
+
return JSON.parse(trimmed);
|
|
7392
|
+
const match = trimmed.match(/\{[\s\S]*\}/);
|
|
7393
|
+
if (!match)
|
|
7394
|
+
return null;
|
|
7395
|
+
return JSON.parse(match[0]);
|
|
7396
|
+
}
|
|
7397
|
+
function normalizeDqlGenerationPatch(value) {
|
|
7398
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7399
|
+
return null;
|
|
7400
|
+
const record = value;
|
|
7401
|
+
const patch = {};
|
|
7402
|
+
if (typeof record.name === 'string' && record.name.trim())
|
|
7403
|
+
patch.name = record.name.trim().slice(0, 120);
|
|
7404
|
+
if (typeof record.domain === 'string' && record.domain.trim())
|
|
7405
|
+
patch.domain = record.domain.trim().slice(0, 80);
|
|
7406
|
+
if (typeof record.description === 'string' && record.description.trim())
|
|
7407
|
+
patch.description = record.description.trim().slice(0, 500);
|
|
7408
|
+
if (typeof record.owner === 'string' && record.owner.trim())
|
|
7409
|
+
patch.owner = record.owner.trim().slice(0, 120);
|
|
7410
|
+
if (Array.isArray(record.tags))
|
|
7411
|
+
patch.tags = normalizeDqlGenerationStringArray(record.tags, 12);
|
|
7412
|
+
if (Array.isArray(record.terms))
|
|
7413
|
+
patch.terms = normalizeDqlGenerationStringArray(record.terms, 16);
|
|
7414
|
+
if (typeof record.llmContext === 'string' && record.llmContext.trim())
|
|
7415
|
+
patch.llmContext = record.llmContext.trim().slice(0, 1000);
|
|
7416
|
+
if (typeof record.pattern === 'string' && record.pattern.trim())
|
|
7417
|
+
patch.pattern = normalizeDqlGenerationPattern(record.pattern);
|
|
7418
|
+
if (typeof record.grain === 'string' && record.grain.trim())
|
|
7419
|
+
patch.grain = record.grain.trim().slice(0, 120);
|
|
7420
|
+
if (Array.isArray(record.entities))
|
|
7421
|
+
patch.entities = normalizeDqlGenerationStringArray(record.entities, 12);
|
|
7422
|
+
if (Array.isArray(record.outputs))
|
|
7423
|
+
patch.outputs = normalizeDqlGenerationStringArray(record.outputs, 24);
|
|
7424
|
+
if (Array.isArray(record.dimensions))
|
|
7425
|
+
patch.dimensions = normalizeDqlGenerationStringArray(record.dimensions, 16);
|
|
7426
|
+
if (Array.isArray(record.allowedFilters))
|
|
7427
|
+
patch.allowedFilters = normalizeDqlGenerationStringArray(record.allowedFilters, 16);
|
|
7428
|
+
if (Array.isArray(record.sourceSystems))
|
|
7429
|
+
patch.sourceSystems = normalizeDqlGenerationStringArray(record.sourceSystems, 12);
|
|
7430
|
+
if (Array.isArray(record.replacementFor))
|
|
7431
|
+
patch.replacementFor = normalizeDqlGenerationStringArray(record.replacementFor, 12);
|
|
7432
|
+
if (typeof record.reviewCadence === 'string' && record.reviewCadence.trim()) {
|
|
7433
|
+
const reviewCadence = normalizeDqlGenerationReviewCadence(record.reviewCadence);
|
|
7434
|
+
if (reviewCadence)
|
|
7435
|
+
patch.reviewCadence = reviewCadence;
|
|
7436
|
+
}
|
|
7437
|
+
return Object.keys(patch).length > 0 ? patch : null;
|
|
7438
|
+
}
|
|
7439
|
+
function normalizeDqlGenerationStringArray(value, limit) {
|
|
7440
|
+
const result = [];
|
|
7441
|
+
for (const item of value) {
|
|
7442
|
+
const normalized = normalizeDqlGenerationStringItem(item);
|
|
7443
|
+
if (normalized)
|
|
7444
|
+
result.push(normalized);
|
|
7445
|
+
}
|
|
7446
|
+
return Array.from(new Set(result)).slice(0, limit);
|
|
7447
|
+
}
|
|
7448
|
+
function normalizeDqlGenerationStringItem(value) {
|
|
7449
|
+
if (typeof value === 'string')
|
|
7450
|
+
return value.trim();
|
|
7451
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7452
|
+
return '';
|
|
7453
|
+
const record = value;
|
|
7454
|
+
for (const key of ['name', 'field', 'column', 'id', 'value', 'label', 'sourceSystem', 'source_system']) {
|
|
7455
|
+
const nested = record[key];
|
|
7456
|
+
if (typeof nested === 'string' && nested.trim())
|
|
7457
|
+
return nested.trim();
|
|
7458
|
+
}
|
|
7459
|
+
return '';
|
|
7460
|
+
}
|
|
7461
|
+
function normalizeDqlGenerationPattern(value) {
|
|
7462
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
7463
|
+
const allowed = new Set([
|
|
7464
|
+
'metric_wrapper',
|
|
7465
|
+
'entity_profile',
|
|
7466
|
+
'entity_rollup',
|
|
7467
|
+
'ranking',
|
|
7468
|
+
'trend',
|
|
7469
|
+
'bridge',
|
|
7470
|
+
'drilldown',
|
|
7471
|
+
'replacement',
|
|
7472
|
+
'custom',
|
|
7473
|
+
]);
|
|
7474
|
+
return allowed.has(normalized) ? normalized : undefined;
|
|
7475
|
+
}
|
|
7476
|
+
function normalizeDqlGenerationReviewCadence(value) {
|
|
7477
|
+
const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_ -]+/g, '').replace(/\s+/g, '_');
|
|
7478
|
+
const allowed = new Set(['daily', 'weekly', 'biweekly', 'monthly', 'quarterly', 'semiannual', 'annual']);
|
|
7479
|
+
return allowed.has(normalized) ? normalized : undefined;
|
|
7480
|
+
}
|
|
7481
|
+
function mergeGroundedDqlGenerationTerms(baseTerms, overrideTerms, evidence) {
|
|
7482
|
+
const grounded = new Map();
|
|
7483
|
+
for (const item of evidence) {
|
|
7484
|
+
if (item.kind !== 'dql_term' && item.kind !== 'datalex_term')
|
|
7485
|
+
continue;
|
|
7486
|
+
const normalized = normalizedTerm(item.name);
|
|
7487
|
+
if (normalized)
|
|
7488
|
+
grounded.set(normalized, item.name.trim());
|
|
7489
|
+
}
|
|
7490
|
+
const merged = [...(baseTerms ?? []), ...(overrideTerms ?? [])]
|
|
7491
|
+
.map((term) => grounded.get(normalizedTerm(term)))
|
|
7492
|
+
.filter((term) => Boolean(term));
|
|
7493
|
+
const result = Array.from(new Set(merged)).slice(0, 16);
|
|
7494
|
+
return result.length > 0 ? result : undefined;
|
|
7495
|
+
}
|
|
7496
|
+
function mergeDqlGenerationPatch(base, override, candidate, evidence) {
|
|
7497
|
+
const name = override.name && !isWeakDqlGenerationName(override.name) ? override.name : base.name;
|
|
7498
|
+
const description = override.description
|
|
7499
|
+
&& !isWeakDqlGenerationDescription(override.description)
|
|
7500
|
+
&& !isUngroundedDqlGenerationDescription(override.description, candidate, evidence)
|
|
7501
|
+
? override.description
|
|
7502
|
+
: base.description;
|
|
7503
|
+
const llmContext = override.llmContext
|
|
7504
|
+
&& !isUngroundedDqlGenerationDescription(override.llmContext, candidate, evidence)
|
|
7505
|
+
? override.llmContext
|
|
7506
|
+
: base.llmContext;
|
|
7507
|
+
const overrideTags = override.tags
|
|
7508
|
+
? override.tags.filter((tag) => !/^(raw-sql|imported|ai-generated)$/i.test(tag))
|
|
7509
|
+
: [];
|
|
7510
|
+
return {
|
|
7511
|
+
...base,
|
|
7512
|
+
...override,
|
|
7513
|
+
name,
|
|
7514
|
+
description,
|
|
7515
|
+
llmContext,
|
|
7516
|
+
tags: Array.from(new Set([...(base.tags ?? []), ...overrideTags])).slice(0, 12),
|
|
7517
|
+
terms: mergeGroundedDqlGenerationTerms(base.terms, override.terms, evidence),
|
|
7518
|
+
pattern: override.pattern ?? base.pattern,
|
|
7519
|
+
grain: override.grain ?? base.grain,
|
|
7520
|
+
entities: override.entities?.length ? override.entities : base.entities,
|
|
7521
|
+
outputs: override.outputs?.length ? override.outputs : base.outputs,
|
|
7522
|
+
dimensions: override.dimensions?.length ? override.dimensions : base.dimensions,
|
|
7523
|
+
allowedFilters: override.allowedFilters?.length ? override.allowedFilters : base.allowedFilters,
|
|
7524
|
+
parameterPolicy: base.parameterPolicy,
|
|
7525
|
+
filterBindings: base.filterBindings,
|
|
7526
|
+
parameterDecisions: base.parameterDecisions,
|
|
7527
|
+
similarityMatches: base.similarityMatches,
|
|
7528
|
+
recommendedAction: base.recommendedAction,
|
|
7529
|
+
sql: base.sql,
|
|
7530
|
+
warnings: base.warnings,
|
|
7531
|
+
sourceSystems: override.sourceSystems?.length ? override.sourceSystems : base.sourceSystems,
|
|
7532
|
+
replacementFor: override.replacementFor?.length ? override.replacementFor : base.replacementFor,
|
|
7533
|
+
reviewCadence: override.reviewCadence ?? base.reviewCadence,
|
|
5002
7534
|
};
|
|
5003
7535
|
}
|
|
5004
|
-
function
|
|
5005
|
-
const
|
|
5006
|
-
const
|
|
5007
|
-
const
|
|
5008
|
-
|
|
5009
|
-
|
|
5010
|
-
|
|
5011
|
-
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
7536
|
+
function buildDqlGenerationSimilarityMatches(candidate, patch, contextPack) {
|
|
7537
|
+
const candidateSql = patch.sql ?? candidate.sql;
|
|
7538
|
+
const candidateFingerprints = buildBlockSqlFingerprints(candidateSql);
|
|
7539
|
+
const candidateBusinessFingerprint = buildBlockBusinessFingerprint({
|
|
7540
|
+
name: patch.name ?? candidate.name,
|
|
7541
|
+
domain: patch.domain ?? candidate.domain,
|
|
7542
|
+
pattern: patch.pattern,
|
|
7543
|
+
grain: patch.grain,
|
|
7544
|
+
entities: patch.entities,
|
|
7545
|
+
terms: patch.terms,
|
|
7546
|
+
outputs: patch.outputs,
|
|
7547
|
+
dimensions: patch.dimensions,
|
|
7548
|
+
filters: patch.allowedFilters,
|
|
7549
|
+
sources: candidate.lineage.sourceTables,
|
|
7550
|
+
sourceSystems: patch.sourceSystems,
|
|
7551
|
+
});
|
|
7552
|
+
const candidateShape = {
|
|
7553
|
+
name: candidate.name,
|
|
7554
|
+
pattern: patch.pattern,
|
|
7555
|
+
grain: patch.grain,
|
|
7556
|
+
outputs: new Set((patch.outputs ?? []).map(normalizedTerm)),
|
|
7557
|
+
terms: new Set((patch.terms ?? []).map(normalizedTerm)),
|
|
7558
|
+
dimensions: new Set((patch.dimensions ?? []).map(normalizedTerm)),
|
|
7559
|
+
filters: new Set((patch.allowedFilters ?? []).map(normalizedTerm)),
|
|
7560
|
+
entities: new Set((patch.entities ?? []).map(normalizedTerm)),
|
|
7561
|
+
sources: new Set(candidate.lineage.sourceTables.flatMap(relationLookupTokens).map(normalizedTerm)),
|
|
7562
|
+
};
|
|
7563
|
+
const objects = (contextPack?.objects ?? []).filter((object) => object.objectType === 'dql_block');
|
|
7564
|
+
const matches = objects.flatMap((object) => {
|
|
7565
|
+
const payload = object.payload ?? {};
|
|
7566
|
+
const objectSql = typeof payload.sql === 'string' ? payload.sql : '';
|
|
7567
|
+
const objectFingerprints = metadataSqlFingerprints(payload.sqlFingerprints)
|
|
7568
|
+
?? (objectSql ? buildBlockSqlFingerprints(objectSql) : null);
|
|
7569
|
+
const objectBusinessFingerprint = metadataBusinessFingerprint(payload.businessFingerprint);
|
|
7570
|
+
const objectShape = {
|
|
7571
|
+
pattern: metadataString(payload.pattern),
|
|
7572
|
+
grain: metadataString(payload.grain),
|
|
7573
|
+
outputs: new Set(metadataStringArray(payload.declaredOutputs ?? payload.outputs)),
|
|
7574
|
+
terms: new Set(metadataStringArray(payload.termRefs ?? payload.terms)),
|
|
7575
|
+
dimensions: new Set(metadataStringArray(payload.dimensions)),
|
|
7576
|
+
filters: new Set(metadataStringArray(payload.allowedFilters)),
|
|
7577
|
+
entities: new Set(metadataStringArray(payload.entities)),
|
|
7578
|
+
sources: new Set(metadataStringArray(payload.tableDependencies ?? payload.rawTableRefs).flatMap(relationLookupTokens).map(normalizedTerm)),
|
|
7579
|
+
};
|
|
7580
|
+
if (objectFingerprints?.exact && objectFingerprints.exact === candidateFingerprints.exact) {
|
|
7581
|
+
return [{
|
|
7582
|
+
kind: 'exact_sql_match',
|
|
7583
|
+
objectKey: object.objectKey,
|
|
7584
|
+
name: object.fullName ?? object.name,
|
|
7585
|
+
status: object.status,
|
|
7586
|
+
source: object.sourcePath,
|
|
7587
|
+
score: 0.99,
|
|
7588
|
+
reason: 'Normalized SQL exactly matches an existing DQL block.',
|
|
7589
|
+
recommendedAction: 'reuse_existing',
|
|
7590
|
+
}];
|
|
7591
|
+
}
|
|
7592
|
+
if (objectFingerprints?.parameterized && objectFingerprints.parameterized === candidateFingerprints.parameterized) {
|
|
7593
|
+
return [{
|
|
7594
|
+
kind: 'parameterized_duplicate',
|
|
7595
|
+
objectKey: object.objectKey,
|
|
7596
|
+
name: object.fullName ?? object.name,
|
|
7597
|
+
status: object.status,
|
|
7598
|
+
source: object.sourcePath,
|
|
7599
|
+
score: 0.94,
|
|
7600
|
+
reason: 'SQL shape matches after replacing literal values with parameters.',
|
|
7601
|
+
recommendedAction: 'reuse_existing',
|
|
7602
|
+
}];
|
|
7603
|
+
}
|
|
7604
|
+
const score = businessShapeSimilarity(candidateShape, objectShape, object);
|
|
7605
|
+
if (objectBusinessFingerprint?.hash
|
|
7606
|
+
&& objectBusinessFingerprint.hash === candidateBusinessFingerprint.hash
|
|
7607
|
+
&& object.status === 'certified') {
|
|
7608
|
+
return [{
|
|
7609
|
+
kind: 'business_duplicate',
|
|
7610
|
+
objectKey: object.objectKey,
|
|
7611
|
+
name: object.fullName ?? object.name,
|
|
7612
|
+
status: object.status,
|
|
7613
|
+
source: object.sourcePath,
|
|
7614
|
+
score: Math.max(score, 0.86),
|
|
7615
|
+
reason: 'Certified block has the same persisted business-shape fingerprint.',
|
|
7616
|
+
recommendedAction: 'reuse_existing',
|
|
7617
|
+
}];
|
|
7618
|
+
}
|
|
7619
|
+
if (score >= 0.76 && object.status === 'certified') {
|
|
7620
|
+
return [{
|
|
7621
|
+
kind: 'business_duplicate',
|
|
7622
|
+
objectKey: object.objectKey,
|
|
7623
|
+
name: object.fullName ?? object.name,
|
|
7624
|
+
status: object.status,
|
|
7625
|
+
source: object.sourcePath,
|
|
7626
|
+
score,
|
|
7627
|
+
reason: 'Certified block has the same business shape: entity, grain, filters, outputs, or source family overlap.',
|
|
7628
|
+
recommendedAction: 'reuse_existing',
|
|
7629
|
+
}];
|
|
7630
|
+
}
|
|
7631
|
+
if (score >= 0.58) {
|
|
7632
|
+
return [{
|
|
7633
|
+
kind: 'near_variant',
|
|
7634
|
+
objectKey: object.objectKey,
|
|
7635
|
+
name: object.fullName ?? object.name,
|
|
7636
|
+
status: object.status,
|
|
7637
|
+
source: object.sourcePath,
|
|
7638
|
+
score,
|
|
7639
|
+
reason: 'Existing block is close but appears to need a reviewed filter, output, grain, or source extension.',
|
|
7640
|
+
recommendedAction: 'extend_existing',
|
|
7641
|
+
}];
|
|
7642
|
+
}
|
|
7643
|
+
return [];
|
|
7644
|
+
}).sort((a, b) => b.score - a.score).slice(0, 8);
|
|
7645
|
+
const top = matches[0];
|
|
7646
|
+
return {
|
|
7647
|
+
matches,
|
|
7648
|
+
recommendedAction: top?.recommendedAction ?? 'create_new',
|
|
7649
|
+
};
|
|
5015
7650
|
}
|
|
5016
|
-
function
|
|
5017
|
-
|
|
5018
|
-
|
|
5019
|
-
|
|
5020
|
-
const
|
|
5021
|
-
|
|
5022
|
-
|
|
5023
|
-
|
|
5024
|
-
|
|
5025
|
-
|
|
7651
|
+
function metadataSqlFingerprints(value) {
|
|
7652
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7653
|
+
return null;
|
|
7654
|
+
const record = value;
|
|
7655
|
+
const exact = typeof record.exact === 'string' ? record.exact : '';
|
|
7656
|
+
const parameterized = typeof record.parameterized === 'string' ? record.parameterized : '';
|
|
7657
|
+
if (!exact || !parameterized)
|
|
7658
|
+
return null;
|
|
7659
|
+
return { exact, parameterized };
|
|
7660
|
+
}
|
|
7661
|
+
function metadataBusinessFingerprint(value) {
|
|
7662
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
7663
|
+
return null;
|
|
7664
|
+
const record = value;
|
|
7665
|
+
const hash = typeof record.hash === 'string' ? record.hash : '';
|
|
7666
|
+
if (!hash)
|
|
7667
|
+
return null;
|
|
7668
|
+
const tokens = Array.isArray(record.tokens)
|
|
7669
|
+
? record.tokens.map((item) => String(item)).filter(Boolean)
|
|
7670
|
+
: [];
|
|
7671
|
+
return { hash, tokens };
|
|
7672
|
+
}
|
|
7673
|
+
function businessShapeSimilarity(candidate, object, metadata) {
|
|
7674
|
+
let score = 0;
|
|
7675
|
+
if (candidate.pattern && object.pattern && normalizedTerm(candidate.pattern) === normalizedTerm(object.pattern))
|
|
7676
|
+
score += 0.12;
|
|
7677
|
+
if (candidate.grain && object.grain && normalizedTerm(candidate.grain) === normalizedTerm(object.grain))
|
|
7678
|
+
score += 0.15;
|
|
7679
|
+
score += overlapScore(candidate.outputs, object.outputs) * 0.18;
|
|
7680
|
+
score += overlapScore(candidate.terms, object.terms) * 0.08;
|
|
7681
|
+
score += overlapScore(candidate.dimensions, object.dimensions) * 0.13;
|
|
7682
|
+
score += overlapScore(candidate.filters, object.filters) * 0.14;
|
|
7683
|
+
score += overlapScore(candidate.entities, object.entities) * 0.12;
|
|
7684
|
+
score += overlapScore(candidate.sources, object.sources) * 0.1;
|
|
7685
|
+
const nameTokens = new Set(candidate.name.toLowerCase().split(/[^a-z0-9]+/).filter((value) => value.length > 2));
|
|
7686
|
+
const targetTokens = new Set([
|
|
7687
|
+
...metadata.name.toLowerCase().split(/[^a-z0-9]+/),
|
|
7688
|
+
...(metadata.description ?? '').toLowerCase().split(/[^a-z0-9]+/),
|
|
7689
|
+
].filter((value) => value.length > 2));
|
|
7690
|
+
score += overlapScore(nameTokens, targetTokens) * 0.04;
|
|
7691
|
+
return Number(Math.min(0.98, score).toFixed(3));
|
|
7692
|
+
}
|
|
7693
|
+
function overlapScore(left, right) {
|
|
7694
|
+
if (left.size === 0 || right.size === 0)
|
|
7695
|
+
return 0;
|
|
7696
|
+
let overlap = 0;
|
|
7697
|
+
for (const value of left) {
|
|
7698
|
+
if (value && right.has(value))
|
|
7699
|
+
overlap += 1;
|
|
7700
|
+
}
|
|
7701
|
+
return overlap / Math.max(1, Math.min(left.size, right.size));
|
|
7702
|
+
}
|
|
7703
|
+
function normalizedTerm(value) {
|
|
7704
|
+
return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
7705
|
+
}
|
|
7706
|
+
function metadataString(value) {
|
|
7707
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
7708
|
+
}
|
|
7709
|
+
function metadataStringArray(value) {
|
|
7710
|
+
if (!Array.isArray(value))
|
|
7711
|
+
return [];
|
|
7712
|
+
return value.map((item) => String(item).trim()).filter(Boolean).map(normalizedTerm);
|
|
7713
|
+
}
|
|
7714
|
+
function isWeakDqlGenerationName(name) {
|
|
7715
|
+
return /^(pasted|query|source|imported query|sql import)(?:\s+\d+)?$/i.test(name.trim());
|
|
7716
|
+
}
|
|
7717
|
+
function isWeakDqlGenerationDescription(description) {
|
|
7718
|
+
return /^draft dql block summarizing imported sql/i.test(description.trim())
|
|
7719
|
+
|| /^draft dql block generated from imported sql/i.test(description.trim())
|
|
7720
|
+
|| /^imported from /i.test(description.trim())
|
|
7721
|
+
|| /^this query (shows|returns|provides)/i.test(description.trim());
|
|
7722
|
+
}
|
|
7723
|
+
function isUngroundedDqlGenerationDescription(description, candidate, evidence) {
|
|
7724
|
+
const text = description.toLowerCase();
|
|
7725
|
+
const tokens = new Set();
|
|
7726
|
+
for (const tag of candidate.tags) {
|
|
7727
|
+
if (!/^(raw-sql|imported|ai-generated|review-required)$/i.test(tag) && tag.length > 2)
|
|
7728
|
+
tokens.add(tag.toLowerCase());
|
|
5026
7729
|
}
|
|
5027
|
-
|
|
5028
|
-
|
|
7730
|
+
for (const table of candidate.lineage.sourceTables) {
|
|
7731
|
+
for (const token of table.split(/[._\s/-]+/)) {
|
|
7732
|
+
if (token.length > 2)
|
|
7733
|
+
tokens.add(token.toLowerCase());
|
|
7734
|
+
}
|
|
5029
7735
|
}
|
|
5030
|
-
|
|
5031
|
-
|
|
5032
|
-
|
|
5033
|
-
|
|
5034
|
-
|
|
5035
|
-
if (!existsSync(absCompanionPath))
|
|
5036
|
-
return;
|
|
5037
|
-
let companion = readFileSync(absCompanionPath, 'utf-8');
|
|
5038
|
-
if (/^reviewStatus:\s*.+$/m.test(companion)) {
|
|
5039
|
-
companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
|
|
7736
|
+
for (const item of evidence.slice(0, 6)) {
|
|
7737
|
+
for (const token of item.name.split(/[._\s/-]+/)) {
|
|
7738
|
+
if (token.length > 3 && !/^(model|source|semantic|table)$/i.test(token))
|
|
7739
|
+
tokens.add(token.toLowerCase());
|
|
7740
|
+
}
|
|
5040
7741
|
}
|
|
5041
|
-
|
|
5042
|
-
|
|
7742
|
+
if (tokens.size === 0)
|
|
7743
|
+
return false;
|
|
7744
|
+
return !Array.from(tokens).some((token) => text.includes(token));
|
|
7745
|
+
}
|
|
7746
|
+
function dqlGenerationConversionNotes(providerName) {
|
|
7747
|
+
return [
|
|
7748
|
+
`Generated with ${providerName} using local project context when available.`,
|
|
7749
|
+
'Runtime-scope SQL literals are converted into DQL params when safe; business constants stay static for review.',
|
|
7750
|
+
'Certified and draft block similarity is checked before a new draft is saved.',
|
|
7751
|
+
'SQL semantics are preserved while reusable parameter placeholders may replace literal values.',
|
|
7752
|
+
'Drafts are autosaved under blocks/_drafts or domains/<domain>/blocks/_drafts before certification.',
|
|
7753
|
+
'Preview results and tests gate certification before promotion.',
|
|
7754
|
+
];
|
|
7755
|
+
}
|
|
7756
|
+
function setBlockStudioSourceStatus(source, status) {
|
|
7757
|
+
if (/status\s*=\s*"[^"]*"/.test(source)) {
|
|
7758
|
+
return source.replace(/status\s*=\s*"[^"]*"/, `status = "${status}"`);
|
|
5043
7759
|
}
|
|
5044
|
-
|
|
7760
|
+
return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${status}"`);
|
|
5045
7761
|
}
|
|
5046
7762
|
async function buildBlockStudioAiAssistSummary(projectRoot, action, candidate, validation, requestedProvider) {
|
|
5047
7763
|
const fallback = buildDeterministicAiAssistSummary(action, candidate, validation);
|
|
@@ -5249,6 +7965,14 @@ export function readGitMetadata(projectRoot) {
|
|
|
5249
7965
|
return null;
|
|
5250
7966
|
}
|
|
5251
7967
|
}
|
|
7968
|
+
function resolveBlockWriteTarget(projectRoot, safeDomain, slug) {
|
|
7969
|
+
if (safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))) {
|
|
7970
|
+
const relativePath = `domains/${safeDomain}/blocks/${slug}.dql`;
|
|
7971
|
+
return { relativePath, absPath: join(projectRoot, relativePath) };
|
|
7972
|
+
}
|
|
7973
|
+
const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
|
|
7974
|
+
return { relativePath, absPath: join(projectRoot, relativePath) };
|
|
7975
|
+
}
|
|
5252
7976
|
export function createBlockArtifacts(projectRoot, options) {
|
|
5253
7977
|
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
5254
7978
|
const safeDomain = (options.domain ?? '')
|
|
@@ -5256,16 +7980,16 @@ export function createBlockArtifacts(projectRoot, options) {
|
|
|
5256
7980
|
.toLowerCase()
|
|
5257
7981
|
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
5258
7982
|
.replace(/^\/+|\/+$/g, '');
|
|
5259
|
-
const
|
|
5260
|
-
mkdirSync(
|
|
5261
|
-
const blockPath =
|
|
7983
|
+
const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
|
|
7984
|
+
mkdirSync(dirname(target.absPath), { recursive: true });
|
|
7985
|
+
const blockPath = target.absPath;
|
|
5262
7986
|
if (existsSync(blockPath)) {
|
|
5263
7987
|
throw new Error('BLOCK_EXISTS');
|
|
5264
7988
|
}
|
|
5265
7989
|
const templateContent = options.template
|
|
5266
7990
|
? listBlockTemplates().find((template) => template.id === options.template)?.content
|
|
5267
7991
|
: undefined;
|
|
5268
|
-
const relativePath =
|
|
7992
|
+
const relativePath = target.relativePath;
|
|
5269
7993
|
const fileContent = canonicalizeSafe(options.blockType === 'semantic' && !options.content?.trim() && !templateContent
|
|
5270
7994
|
? buildBlankSemanticBlockContent({
|
|
5271
7995
|
name: options.name,
|
|
@@ -5311,9 +8035,9 @@ export function createSemanticBuilderBlock(projectRoot, options) {
|
|
|
5311
8035
|
.toLowerCase()
|
|
5312
8036
|
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
5313
8037
|
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
5314
|
-
const
|
|
5315
|
-
mkdirSync(
|
|
5316
|
-
const blockPath =
|
|
8038
|
+
const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
|
|
8039
|
+
mkdirSync(dirname(target.absPath), { recursive: true });
|
|
8040
|
+
const blockPath = target.absPath;
|
|
5317
8041
|
if (existsSync(blockPath)) {
|
|
5318
8042
|
throw new Error('BLOCK_EXISTS');
|
|
5319
8043
|
}
|
|
@@ -5338,7 +8062,7 @@ export function createSemanticBuilderBlock(projectRoot, options) {
|
|
|
5338
8062
|
],
|
|
5339
8063
|
});
|
|
5340
8064
|
return {
|
|
5341
|
-
path:
|
|
8065
|
+
path: target.relativePath,
|
|
5342
8066
|
content,
|
|
5343
8067
|
companionPath,
|
|
5344
8068
|
};
|
|
@@ -6100,18 +8824,16 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
|
|
|
6100
8824
|
const sourceCount = manifest?.sources && typeof manifest.sources === 'object'
|
|
6101
8825
|
? Object.keys(manifest.sources).length
|
|
6102
8826
|
: 0;
|
|
6103
|
-
const manifestMetricCount = manifest?.metrics
|
|
6104
|
-
|
|
6105
|
-
|
|
6106
|
-
const semanticMetricCount = Array.isArray(semanticManifest?.metrics)
|
|
6107
|
-
? semanticManifest.metrics.length
|
|
8827
|
+
const manifestMetricCount = artifactCollectionCount(manifest?.metrics);
|
|
8828
|
+
const semanticMetricCount = semanticManifest
|
|
8829
|
+
? artifactCollectionCount(semanticManifest.metrics)
|
|
6108
8830
|
: manifestMetricCount;
|
|
6109
|
-
const semanticModelCount =
|
|
6110
|
-
? semanticManifest.semantic_models
|
|
6111
|
-
:
|
|
6112
|
-
const savedQueryCount =
|
|
6113
|
-
? semanticManifest.saved_queries.
|
|
6114
|
-
:
|
|
8831
|
+
const semanticModelCount = semanticManifest
|
|
8832
|
+
? artifactCollectionCount(semanticManifest.semantic_models)
|
|
8833
|
+
: artifactCollectionCount(manifest?.semantic_models);
|
|
8834
|
+
const savedQueryCount = semanticManifest
|
|
8835
|
+
? artifactCollectionCount(semanticManifest.saved_queries ?? semanticManifest.savedQueries)
|
|
8836
|
+
: artifactCollectionCount(manifest?.saved_queries ?? manifest?.savedQueries);
|
|
6115
8837
|
const configured = existsSync(join(dbtProjectPath, 'dbt_project.yml'))
|
|
6116
8838
|
|| Boolean(projectConfig.dbt?.projectDir)
|
|
6117
8839
|
|| Boolean(projectConfig.semanticLayer?.provider === 'dbt' && projectConfig.semanticLayer.projectPath);
|
|
@@ -6123,7 +8845,9 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
|
|
|
6123
8845
|
? 'Run `dbt parse`, `dbt compile`, or `dbt build`, then run `dql sync dbt`.'
|
|
6124
8846
|
: !semanticExists
|
|
6125
8847
|
? 'dbt manifest is ready. Run `dbt parse` or `dbt build` if you use dbt Semantic Layer metrics.'
|
|
6126
|
-
:
|
|
8848
|
+
: semanticMetricCount === 0 && semanticModelCount === 0 && savedQueryCount === 0
|
|
8849
|
+
? 'dbt manifest is ready, but MetricFlow semantic_manifest.json is empty. Add dbt semantic_models and metrics, then rerun dbt parse/build.'
|
|
8850
|
+
: 'dbt artifacts are ready. Build SQL blocks from models or semantic blocks from metrics.';
|
|
6127
8851
|
return {
|
|
6128
8852
|
configured,
|
|
6129
8853
|
provider: projectConfig.semanticLayer?.provider ?? null,
|
|
@@ -6146,6 +8870,185 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
|
|
|
6146
8870
|
setupHint,
|
|
6147
8871
|
};
|
|
6148
8872
|
}
|
|
8873
|
+
export function buildSemanticLayerDiagnostics(projectRoot, projectConfig, options) {
|
|
8874
|
+
const provider = options.semanticConfig?.provider ?? options.detectedProvider ?? null;
|
|
8875
|
+
const dbt = buildDbtStatus(projectRoot, projectConfig, options.lastSyncTime);
|
|
8876
|
+
const counts = semanticLayerCounts(options.semanticLayer);
|
|
8877
|
+
const uncategorizedObjects = countUncategorizedSemanticObjects(options.semanticLayer);
|
|
8878
|
+
const totalObjects = counts.metrics
|
|
8879
|
+
+ counts.measures
|
|
8880
|
+
+ counts.dimensions
|
|
8881
|
+
+ counts.timeDimensions
|
|
8882
|
+
+ counts.entities
|
|
8883
|
+
+ counts.hierarchies
|
|
8884
|
+
+ counts.semanticModels
|
|
8885
|
+
+ counts.savedQueries;
|
|
8886
|
+
const issues = [];
|
|
8887
|
+
for (const error of options.semanticErrors ?? []) {
|
|
8888
|
+
issues.push({
|
|
8889
|
+
severity: 'error',
|
|
8890
|
+
code: 'semantic_load_error',
|
|
8891
|
+
message: error,
|
|
8892
|
+
action: 'Open the semantic source configuration, fix the provider error, then reload the semantic layer.',
|
|
8893
|
+
});
|
|
8894
|
+
}
|
|
8895
|
+
if (!provider) {
|
|
8896
|
+
issues.push({
|
|
8897
|
+
severity: existsSync(join(projectRoot, 'semantic-layer')) ? 'info' : 'warning',
|
|
8898
|
+
code: 'semantic_provider_not_configured',
|
|
8899
|
+
message: 'No semantic layer provider is configured.',
|
|
8900
|
+
action: 'Configure dbt MetricFlow, local semantic-layer YAML, or Snowflake Semantic Views from the setup flow.',
|
|
8901
|
+
});
|
|
8902
|
+
}
|
|
8903
|
+
if (provider === 'dbt' || dbt.configured) {
|
|
8904
|
+
if (!dbt.configured) {
|
|
8905
|
+
issues.push({
|
|
8906
|
+
severity: 'warning',
|
|
8907
|
+
code: 'dbt_project_not_detected',
|
|
8908
|
+
message: 'dbt is selected as the semantic source, but no dbt project was detected.',
|
|
8909
|
+
action: 'Set dbt.projectDir or run DQL from a repository containing dbt_project.yml.',
|
|
8910
|
+
});
|
|
8911
|
+
}
|
|
8912
|
+
if (!dbt.artifacts.manifest.exists) {
|
|
8913
|
+
issues.push({
|
|
8914
|
+
severity: 'error',
|
|
8915
|
+
code: 'dbt_manifest_missing',
|
|
8916
|
+
message: 'dbt manifest.json is missing, so DQL cannot build dbt model context.',
|
|
8917
|
+
action: 'Run dbt parse, dbt compile, or dbt build, then reload the semantic layer.',
|
|
8918
|
+
path: dbt.artifacts.manifest.path,
|
|
8919
|
+
});
|
|
8920
|
+
}
|
|
8921
|
+
if (!dbt.artifacts.semanticManifest.exists) {
|
|
8922
|
+
issues.push({
|
|
8923
|
+
severity: 'warning',
|
|
8924
|
+
code: 'metricflow_semantic_manifest_missing',
|
|
8925
|
+
message: 'dbt MetricFlow semantic_manifest.json is missing. DQL can still inspect dbt models, but MetricFlow metrics and saved queries may be incomplete.',
|
|
8926
|
+
action: 'Run dbt parse or dbt build with semantic models configured, then reload. For execution, ensure MetricFlow is installed and mf is on PATH.',
|
|
8927
|
+
path: dbt.artifacts.semanticManifest.path,
|
|
8928
|
+
});
|
|
8929
|
+
}
|
|
8930
|
+
else if (dbt.counts.metrics === 0 && dbt.counts.semanticModels === 0 && dbt.counts.savedQueries === 0) {
|
|
8931
|
+
issues.push({
|
|
8932
|
+
severity: 'warning',
|
|
8933
|
+
code: 'metricflow_semantic_manifest_empty',
|
|
8934
|
+
message: 'semantic_manifest.json exists, but it does not contain metrics, semantic models, or saved queries.',
|
|
8935
|
+
action: 'Check dbt semantic model YAML definitions and rerun dbt parse.',
|
|
8936
|
+
path: dbt.artifacts.semanticManifest.path,
|
|
8937
|
+
});
|
|
8938
|
+
}
|
|
8939
|
+
if (options.semanticLayer && totalObjects === 0 && (dbt.counts.models > 0 || dbt.counts.sources > 0)) {
|
|
8940
|
+
issues.push({
|
|
8941
|
+
severity: 'warning',
|
|
8942
|
+
code: 'dbt_model_inventory_only',
|
|
8943
|
+
message: 'DQL loaded dbt model inventory, but no MetricFlow semantic metrics were found.',
|
|
8944
|
+
action: 'Create dbt semantic_models and metrics, or use AI Import to build DQL blocks from dbt model metadata.',
|
|
8945
|
+
});
|
|
8946
|
+
}
|
|
8947
|
+
}
|
|
8948
|
+
if (provider === 'snowflake') {
|
|
8949
|
+
if (!options.semanticLayer && (options.semanticErrors ?? []).length === 0) {
|
|
8950
|
+
issues.push({
|
|
8951
|
+
severity: 'warning',
|
|
8952
|
+
code: 'snowflake_semantic_views_not_loaded',
|
|
8953
|
+
message: 'Snowflake semantic provider did not load semantic views.',
|
|
8954
|
+
action: 'Confirm the active Snowflake connection can query semantic views, then reload. Warehouse tables still feed database context separately.',
|
|
8955
|
+
});
|
|
8956
|
+
}
|
|
8957
|
+
else if (options.semanticLayer && totalObjects === 0) {
|
|
8958
|
+
issues.push({
|
|
8959
|
+
severity: 'info',
|
|
8960
|
+
code: 'snowflake_semantic_views_empty',
|
|
8961
|
+
message: 'Snowflake semantic provider loaded, but no semantic view objects were discovered.',
|
|
8962
|
+
action: 'Check that Snowflake Semantic Views exist and the active role can see them.',
|
|
8963
|
+
});
|
|
8964
|
+
}
|
|
8965
|
+
}
|
|
8966
|
+
if (options.semanticLayer && totalObjects === 0 && issues.every((issue) => issue.code !== 'dbt_model_inventory_only')) {
|
|
8967
|
+
issues.push({
|
|
8968
|
+
severity: 'warning',
|
|
8969
|
+
code: 'semantic_layer_empty',
|
|
8970
|
+
message: 'Semantic layer is available, but no semantic objects are loaded.',
|
|
8971
|
+
action: 'Import semantic definitions or run dbt parse/build, then reload.',
|
|
8972
|
+
});
|
|
8973
|
+
}
|
|
8974
|
+
if (uncategorizedObjects > 0) {
|
|
8975
|
+
issues.push({
|
|
8976
|
+
severity: 'info',
|
|
8977
|
+
code: 'semantic_uncategorized_domain',
|
|
8978
|
+
message: `${uncategorizedObjects} semantic object${uncategorizedObjects === 1 ? '' : 's'} had no domain and are visible under "uncategorized".`,
|
|
8979
|
+
action: 'Add domain metadata in dbt semantic YAML or local semantic-layer files when domain ownership matters.',
|
|
8980
|
+
});
|
|
8981
|
+
}
|
|
8982
|
+
const warnings = Array.from(new Set([
|
|
8983
|
+
...(options.semanticErrors ?? []),
|
|
8984
|
+
...issues
|
|
8985
|
+
.filter((issue) => issue.severity !== 'info')
|
|
8986
|
+
.map((issue) => issue.message),
|
|
8987
|
+
]));
|
|
8988
|
+
return {
|
|
8989
|
+
available: Boolean(options.semanticLayer),
|
|
8990
|
+
provider,
|
|
8991
|
+
sourceOfTruth: provider === 'dbt'
|
|
8992
|
+
? 'dbt MetricFlow semantic_manifest.json, dbt manifest.json semantic nodes, then dbt semantic YAML fallback'
|
|
8993
|
+
: provider === 'snowflake'
|
|
8994
|
+
? 'Snowflake Semantic Views only; warehouse tables remain database catalog context'
|
|
8995
|
+
: provider === 'dql'
|
|
8996
|
+
? 'local semantic-layer YAML'
|
|
8997
|
+
: 'not configured',
|
|
8998
|
+
errors: options.semanticErrors ?? [],
|
|
8999
|
+
lastSyncTime: options.lastSyncTime,
|
|
9000
|
+
counts,
|
|
9001
|
+
dbt,
|
|
9002
|
+
issues,
|
|
9003
|
+
warnings,
|
|
9004
|
+
};
|
|
9005
|
+
}
|
|
9006
|
+
function semanticLayerCounts(layer) {
|
|
9007
|
+
return layer
|
|
9008
|
+
? {
|
|
9009
|
+
domains: layer.listDomains().length,
|
|
9010
|
+
metrics: layer.listMetrics().length,
|
|
9011
|
+
measures: layer.listMeasures().length,
|
|
9012
|
+
dimensions: layer.listDimensions().length,
|
|
9013
|
+
timeDimensions: layer.listTimeDimensions().length,
|
|
9014
|
+
entities: layer.listEntities().length,
|
|
9015
|
+
hierarchies: layer.listHierarchies().length,
|
|
9016
|
+
semanticModels: layer.listSemanticModels().length,
|
|
9017
|
+
savedQueries: layer.listSavedQueries().length,
|
|
9018
|
+
}
|
|
9019
|
+
: {
|
|
9020
|
+
domains: 0,
|
|
9021
|
+
metrics: 0,
|
|
9022
|
+
measures: 0,
|
|
9023
|
+
dimensions: 0,
|
|
9024
|
+
timeDimensions: 0,
|
|
9025
|
+
entities: 0,
|
|
9026
|
+
hierarchies: 0,
|
|
9027
|
+
semanticModels: 0,
|
|
9028
|
+
savedQueries: 0,
|
|
9029
|
+
};
|
|
9030
|
+
}
|
|
9031
|
+
function countUncategorizedSemanticObjects(layer) {
|
|
9032
|
+
if (!layer)
|
|
9033
|
+
return 0;
|
|
9034
|
+
return [
|
|
9035
|
+
...layer.listMetrics(),
|
|
9036
|
+
...layer.listMeasures(),
|
|
9037
|
+
...layer.listDimensions(),
|
|
9038
|
+
...layer.listTimeDimensions(),
|
|
9039
|
+
...layer.listEntities(),
|
|
9040
|
+
...layer.listHierarchies(),
|
|
9041
|
+
...layer.listSemanticModels(),
|
|
9042
|
+
...layer.listSavedQueries(),
|
|
9043
|
+
].filter((object) => !object.domain || !object.domain.trim()).length;
|
|
9044
|
+
}
|
|
9045
|
+
function artifactCollectionCount(value) {
|
|
9046
|
+
if (Array.isArray(value))
|
|
9047
|
+
return value.length;
|
|
9048
|
+
if (value && typeof value === 'object')
|
|
9049
|
+
return Object.keys(value).length;
|
|
9050
|
+
return 0;
|
|
9051
|
+
}
|
|
6149
9052
|
function describeArtifact(path, count, generatedAt) {
|
|
6150
9053
|
return {
|
|
6151
9054
|
path,
|
|
@@ -6682,6 +9585,11 @@ function isProviderSettingsId(value) {
|
|
|
6682
9585
|
|| value === 'ollama'
|
|
6683
9586
|
|| value === 'custom-openai';
|
|
6684
9587
|
}
|
|
9588
|
+
function isDeterministicDqlGenerationProvider(value) {
|
|
9589
|
+
if (typeof value !== 'string')
|
|
9590
|
+
return false;
|
|
9591
|
+
return value === 'none' || value === 'deterministic' || value === 'local-deterministic';
|
|
9592
|
+
}
|
|
6685
9593
|
async function testProviderConfig(projectRoot, id) {
|
|
6686
9594
|
const config = getEffectiveProviderConfig(projectRoot, id);
|
|
6687
9595
|
const label = providerSettingsLabel(id);
|
|
@@ -6831,6 +9739,655 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
|
|
|
6831
9739
|
// Runtime schema snapshots are advisory local metadata and must not block answers.
|
|
6832
9740
|
}
|
|
6833
9741
|
}
|
|
9742
|
+
function notebookResearchString(value) {
|
|
9743
|
+
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
9744
|
+
}
|
|
9745
|
+
function notebookResearchRecord(value) {
|
|
9746
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
9747
|
+
? value
|
|
9748
|
+
: null;
|
|
9749
|
+
}
|
|
9750
|
+
function notebookResearchSourceCellPayload(body) {
|
|
9751
|
+
const sourceCell = notebookResearchRecord(body.sourceCell)
|
|
9752
|
+
?? notebookResearchRecord(notebookResearchRecord(body.context)?.sourceCell);
|
|
9753
|
+
if (!sourceCell)
|
|
9754
|
+
return undefined;
|
|
9755
|
+
const payload = {
|
|
9756
|
+
id: notebookResearchString(sourceCell.id),
|
|
9757
|
+
sourceCellId: notebookResearchString(sourceCell.sourceCellId),
|
|
9758
|
+
cellId: notebookResearchString(sourceCell.cellId),
|
|
9759
|
+
name: notebookResearchString(sourceCell.name),
|
|
9760
|
+
sourceCellName: notebookResearchString(sourceCell.sourceCellName),
|
|
9761
|
+
title: notebookResearchString(sourceCell.title),
|
|
9762
|
+
fingerprint: notebookResearchString(sourceCell.fingerprint),
|
|
9763
|
+
sourceCellFingerprint: notebookResearchString(sourceCell.sourceCellFingerprint),
|
|
9764
|
+
sqlFingerprint: notebookResearchString(sourceCell.sqlFingerprint),
|
|
9765
|
+
type: notebookResearchString(sourceCell.type),
|
|
9766
|
+
sql: notebookResearchString(sourceCell.sql),
|
|
9767
|
+
content: notebookResearchString(sourceCell.content),
|
|
9768
|
+
source: notebookResearchString(sourceCell.source),
|
|
9769
|
+
};
|
|
9770
|
+
return Object.values(payload).some(Boolean) ? payload : undefined;
|
|
9771
|
+
}
|
|
9772
|
+
function notebookResearchSourceCellId(sourceCell) {
|
|
9773
|
+
return notebookResearchString(sourceCell?.id)
|
|
9774
|
+
?? notebookResearchString(sourceCell?.sourceCellId)
|
|
9775
|
+
?? notebookResearchString(sourceCell?.cellId);
|
|
9776
|
+
}
|
|
9777
|
+
function notebookResearchSourceCellName(sourceCell) {
|
|
9778
|
+
return notebookResearchString(sourceCell?.name)
|
|
9779
|
+
?? notebookResearchString(sourceCell?.sourceCellName)
|
|
9780
|
+
?? notebookResearchString(sourceCell?.title);
|
|
9781
|
+
}
|
|
9782
|
+
function notebookResearchSourceCellFingerprint(sourceCell) {
|
|
9783
|
+
return notebookResearchString(sourceCell?.sourceCellFingerprint)
|
|
9784
|
+
?? notebookResearchString(sourceCell?.fingerprint)
|
|
9785
|
+
?? notebookResearchString(sourceCell?.sqlFingerprint);
|
|
9786
|
+
}
|
|
9787
|
+
function notebookResearchPatchString(body, key) {
|
|
9788
|
+
if (!Object.prototype.hasOwnProperty.call(body, key))
|
|
9789
|
+
return undefined;
|
|
9790
|
+
return notebookResearchString(body[key]) ?? null;
|
|
9791
|
+
}
|
|
9792
|
+
function notebookResearchIntent(value) {
|
|
9793
|
+
return value === 'ad_hoc_analysis'
|
|
9794
|
+
|| value === 'diagnose_change'
|
|
9795
|
+
|| value === 'driver_breakdown'
|
|
9796
|
+
|| value === 'segment_compare'
|
|
9797
|
+
|| value === 'entity_drilldown'
|
|
9798
|
+
|| value === 'anomaly_investigation'
|
|
9799
|
+
|| value === 'trust_gap_review'
|
|
9800
|
+
? value
|
|
9801
|
+
: undefined;
|
|
9802
|
+
}
|
|
9803
|
+
function notebookResearchReviewStatus(value) {
|
|
9804
|
+
return value === 'needs_review'
|
|
9805
|
+
|| value === 'draft_created'
|
|
9806
|
+
|| value === 'completed'
|
|
9807
|
+
|| value === 'certified'
|
|
9808
|
+
|| value === 'rejected'
|
|
9809
|
+
? value
|
|
9810
|
+
: undefined;
|
|
9811
|
+
}
|
|
9812
|
+
function notebookResearchStatus(value) {
|
|
9813
|
+
return value === 'draft'
|
|
9814
|
+
|| value === 'running'
|
|
9815
|
+
|| value === 'ready'
|
|
9816
|
+
|| value === 'error'
|
|
9817
|
+
? value
|
|
9818
|
+
: undefined;
|
|
9819
|
+
}
|
|
9820
|
+
function notebookResearchPromotionAction(value) {
|
|
9821
|
+
return value === 'reuse_existing'
|
|
9822
|
+
|| value === 'extend_existing'
|
|
9823
|
+
|| value === 'create_replacement'
|
|
9824
|
+
|| value === 'create_new'
|
|
9825
|
+
|| value === 'review_required'
|
|
9826
|
+
? value
|
|
9827
|
+
: undefined;
|
|
9828
|
+
}
|
|
9829
|
+
function notebookResearchReadiness(value) {
|
|
9830
|
+
return value === 'draft_ready' || value === 'certification_ready' || value === 'blocked' ? value : undefined;
|
|
9831
|
+
}
|
|
9832
|
+
function notebookResearchAge(value) {
|
|
9833
|
+
return value === 'stale_open' || value === 'expired_open' ? value : undefined;
|
|
9834
|
+
}
|
|
9835
|
+
function notebookResearchNextAction(value) {
|
|
9836
|
+
return value === 'fix_blockers'
|
|
9837
|
+
|| value === 'review_sql'
|
|
9838
|
+
|| value === 'review_context'
|
|
9839
|
+
|| value === 'run_preview'
|
|
9840
|
+
|| value === 'reuse_existing'
|
|
9841
|
+
|| value === 'create_dql_draft'
|
|
9842
|
+
|| value === 'open_certification'
|
|
9843
|
+
|| value === 'complete_review'
|
|
9844
|
+
|| value === 'continue_review'
|
|
9845
|
+
? value
|
|
9846
|
+
: undefined;
|
|
9847
|
+
}
|
|
9848
|
+
function notebookResearchSort(value) {
|
|
9849
|
+
return value === 'priority' || value === 'updated_desc' ? value : undefined;
|
|
9850
|
+
}
|
|
9851
|
+
function withNotebookResearchChecklistPage(page) {
|
|
9852
|
+
return {
|
|
9853
|
+
...page,
|
|
9854
|
+
runs: page.runs.map(withNotebookResearchChecklist),
|
|
9855
|
+
};
|
|
9856
|
+
}
|
|
9857
|
+
function withNotebookResearchChecklist(run) {
|
|
9858
|
+
const researchPlan = run.researchPlan ?? buildNotebookResearchPlan({
|
|
9859
|
+
run,
|
|
9860
|
+
evidence: run.evidence,
|
|
9861
|
+
resultPreview: run.resultPreview,
|
|
9862
|
+
previewError: run.status === 'error' ? run.error : undefined,
|
|
9863
|
+
generatedSql: run.generatedSql,
|
|
9864
|
+
reviewedSql: run.reviewedSql,
|
|
9865
|
+
routeDecision: run.routeDecision,
|
|
9866
|
+
});
|
|
9867
|
+
return {
|
|
9868
|
+
...run,
|
|
9869
|
+
researchPlan,
|
|
9870
|
+
reviewChecklist: buildNotebookResearchReviewChecklist(run),
|
|
9871
|
+
};
|
|
9872
|
+
}
|
|
9873
|
+
function buildNotebookResearchReviewChecklist(run) {
|
|
9874
|
+
const questionReady = Boolean(notebookResearchString(run.question));
|
|
9875
|
+
const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
|
|
9876
|
+
const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
|
|
9877
|
+
const hasSql = hasReviewedSql || hasGeneratedSql;
|
|
9878
|
+
const preview = notebookResearchPreviewInfo(run.resultPreview);
|
|
9879
|
+
const previewReady = run.status === 'ready' && preview.hasPreview;
|
|
9880
|
+
const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
|
|
9881
|
+
const promoted = Boolean(run.dqlPromotion || run.dqlImportId || run.draftBlockPath);
|
|
9882
|
+
const duplicateChecked = Boolean(run.dqlPromotionAction || run.dqlPromotion);
|
|
9883
|
+
const promotionAction = run.dqlPromotionAction ?? run.dqlPromotion?.recommendedAction;
|
|
9884
|
+
const reuseRecommended = promotionAction === 'reuse_existing';
|
|
9885
|
+
const promotionReviewRequired = promotionAction === 'review_required';
|
|
9886
|
+
const hasDraft = Boolean(notebookResearchString(run.draftBlockPath));
|
|
9887
|
+
const sqlForParameterReview = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
|
|
9888
|
+
const parameterReview = sqlForParameterReview ? parameterizeSqlForDqlImport(sqlForParameterReview) : null;
|
|
9889
|
+
const dynamicParameters = parameterReview?.parameterPolicy.filter((item) => item.policy === 'dynamic') ?? [];
|
|
9890
|
+
const staticParameters = parameterReview?.parameterPolicy.filter((item) => item.policy !== 'dynamic') ?? [];
|
|
9891
|
+
const blockers = [];
|
|
9892
|
+
const warnings = [];
|
|
9893
|
+
if (!questionReady)
|
|
9894
|
+
blockers.push('Add a business question before review.');
|
|
9895
|
+
if (!hasSql)
|
|
9896
|
+
blockers.push('Add reviewed SQL or generate SQL before promotion.');
|
|
9897
|
+
if (run.status === 'error')
|
|
9898
|
+
blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
|
|
9899
|
+
if (!previewReady && run.status !== 'error')
|
|
9900
|
+
warnings.push('Run a bounded preview before certification review.');
|
|
9901
|
+
if (!evidenceReady)
|
|
9902
|
+
warnings.push('Inspect metadata evidence before turning this into a reusable block.');
|
|
9903
|
+
if (hasSql && dynamicParameters.length === 0)
|
|
9904
|
+
warnings.push('No dynamic parameters were detected; confirm this should be a static block before certification.');
|
|
9905
|
+
for (const warning of parameterReview?.warnings ?? [])
|
|
9906
|
+
warnings.push(warning);
|
|
9907
|
+
if (promoted && !duplicateChecked)
|
|
9908
|
+
warnings.push('Promotion exists but duplicate/reuse evidence is missing.');
|
|
9909
|
+
if (reuseRecommended)
|
|
9910
|
+
warnings.push('Reuse the matching block or explicitly document a replacement before certification.');
|
|
9911
|
+
if (promotionReviewRequired)
|
|
9912
|
+
warnings.push('Resolve the review-required promotion decision before certification.');
|
|
9913
|
+
const items = [
|
|
9914
|
+
{
|
|
9915
|
+
id: 'question',
|
|
9916
|
+
label: 'Question',
|
|
9917
|
+
status: questionReady ? 'passed' : 'blocked',
|
|
9918
|
+
detail: questionReady ? 'Business question is captured.' : 'A reusable block needs a clear business question.',
|
|
9919
|
+
},
|
|
9920
|
+
{
|
|
9921
|
+
id: 'sql',
|
|
9922
|
+
label: 'Reviewed SQL',
|
|
9923
|
+
status: hasReviewedSql ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
|
|
9924
|
+
detail: hasReviewedSql
|
|
9925
|
+
? 'Reviewer SQL is saved.'
|
|
9926
|
+
: hasGeneratedSql
|
|
9927
|
+
? 'Generated SQL exists; review it before promotion.'
|
|
9928
|
+
: 'No SQL is available for preview or DQL generation.',
|
|
9929
|
+
},
|
|
9930
|
+
{
|
|
9931
|
+
id: 'preview',
|
|
9932
|
+
label: 'Preview',
|
|
9933
|
+
status: previewReady ? 'passed' : run.status === 'error' ? 'blocked' : 'pending',
|
|
9934
|
+
detail: previewReady
|
|
9935
|
+
? `Preview returned ${preview.rowCount.toLocaleString()} row${preview.rowCount === 1 ? '' : 's'}.`
|
|
9936
|
+
: run.status === 'error'
|
|
9937
|
+
? (run.error ?? 'Preview failed.')
|
|
9938
|
+
: 'Run a bounded preview to validate the query shape.',
|
|
9939
|
+
},
|
|
9940
|
+
{
|
|
9941
|
+
id: 'evidence',
|
|
9942
|
+
label: 'Evidence',
|
|
9943
|
+
status: evidenceReady ? 'passed' : 'warning',
|
|
9944
|
+
detail: evidenceReady ? 'Metadata evidence or context pack is attached.' : 'No context evidence is attached yet.',
|
|
9945
|
+
},
|
|
9946
|
+
{
|
|
9947
|
+
id: 'duplicates',
|
|
9948
|
+
label: 'Reuse check',
|
|
9949
|
+
status: duplicateChecked ? (reuseRecommended || promotionReviewRequired ? 'warning' : 'passed') : promoted ? 'warning' : 'pending',
|
|
9950
|
+
detail: duplicateChecked
|
|
9951
|
+
? `Promotion decision: ${promotionAction ?? 'review required'}.`
|
|
9952
|
+
: promoted
|
|
9953
|
+
? 'Draft was created without stored duplicate evidence.'
|
|
9954
|
+
: 'Duplicate check runs when creating a DQL draft.',
|
|
9955
|
+
},
|
|
9956
|
+
{
|
|
9957
|
+
id: 'parameters',
|
|
9958
|
+
label: 'Parameter review',
|
|
9959
|
+
status: !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
|
|
9960
|
+
detail: !hasSql
|
|
9961
|
+
? 'Add SQL before checking runtime parameters.'
|
|
9962
|
+
: dynamicParameters.length > 0
|
|
9963
|
+
? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
|
|
9964
|
+
: staticParameters.length > 0
|
|
9965
|
+
? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
|
|
9966
|
+
: 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
|
|
9967
|
+
},
|
|
9968
|
+
{
|
|
9969
|
+
id: 'dql_draft',
|
|
9970
|
+
label: 'DQL draft',
|
|
9971
|
+
status: hasDraft ? 'passed' : hasReviewedSql && evidenceReady ? 'pending' : 'blocked',
|
|
9972
|
+
detail: hasDraft
|
|
9973
|
+
? `Draft saved at ${run.draftBlockPath}.`
|
|
9974
|
+
: !hasReviewedSql
|
|
9975
|
+
? 'Save reviewed SQL before DQL draft creation.'
|
|
9976
|
+
: evidenceReady
|
|
9977
|
+
? 'Create a draft after SQL review.'
|
|
9978
|
+
: 'Save metadata evidence before creating a reusable DQL draft.',
|
|
9979
|
+
},
|
|
9980
|
+
];
|
|
9981
|
+
const readyForDqlDraft = questionReady && hasReviewedSql && evidenceReady && run.status !== 'error';
|
|
9982
|
+
const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
|
|
9983
|
+
return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
|
|
9984
|
+
}
|
|
9985
|
+
function buildNotebookResearchPlan(input) {
|
|
9986
|
+
const reviewedSql = notebookResearchString(input.reviewedSql);
|
|
9987
|
+
const generatedSql = notebookResearchString(input.generatedSql);
|
|
9988
|
+
const sql = reviewedSql ?? generatedSql;
|
|
9989
|
+
const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
|
|
9990
|
+
const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
|
|
9991
|
+
? input.evidence
|
|
9992
|
+
: {};
|
|
9993
|
+
const trustStatus = evidence.trustStatus && typeof evidence.trustStatus === 'object' && !Array.isArray(evidence.trustStatus)
|
|
9994
|
+
? evidence.trustStatus
|
|
9995
|
+
: {};
|
|
9996
|
+
const allowedSqlContext = evidence.allowedSqlContext && typeof evidence.allowedSqlContext === 'object' && !Array.isArray(evidence.allowedSqlContext)
|
|
9997
|
+
? evidence.allowedSqlContext
|
|
9998
|
+
: {};
|
|
9999
|
+
const previewRows = input.resultPreview?.rowCount ?? input.resultPreview?.rows.length;
|
|
10000
|
+
const hasEvidence = notebookResearchEvidenceCount(input.evidence) > 0 || Boolean(input.run.contextPackId);
|
|
10001
|
+
const hasPreview = previewRows !== undefined && (previewRows > 0 || Boolean(input.resultPreview?.columns.length));
|
|
10002
|
+
const promotionAction = input.run.dqlPromotionAction ?? input.run.dqlPromotion?.recommendedAction;
|
|
10003
|
+
const missingContext = Array.isArray(evidence.missingContext) ? evidence.missingContext : [];
|
|
10004
|
+
const relations = Array.isArray(allowedSqlContext.relations) ? allowedSqlContext.relations : [];
|
|
10005
|
+
return {
|
|
10006
|
+
sqlState: reviewedSql ? 'reviewed' : generatedSql ? 'generated' : 'missing',
|
|
10007
|
+
grain: sql ? extractDqlGenerationGroupByFields(sql)[0] : undefined,
|
|
10008
|
+
parameterPolicy: (parameterized?.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
|
|
10009
|
+
name: entry.name,
|
|
10010
|
+
policy: entry.policy,
|
|
10011
|
+
})),
|
|
10012
|
+
allowedFilters: (parameterized?.allowedFilters ?? []).slice(0, 16),
|
|
10013
|
+
evidence: {
|
|
10014
|
+
trustLabel: notebookResearchString(trustStatus.label),
|
|
10015
|
+
contextPackId: notebookResearchString(evidence.contextPackId) ?? input.run.contextPackId,
|
|
10016
|
+
evidenceCount: notebookResearchEvidenceCount(input.evidence),
|
|
10017
|
+
relationCount: relations.length,
|
|
10018
|
+
missingContextCount: missingContext.length,
|
|
10019
|
+
},
|
|
10020
|
+
preview: {
|
|
10021
|
+
status: input.previewError ? 'error' : hasPreview ? 'ready' : 'not_run',
|
|
10022
|
+
...(previewRows === undefined ? {} : { rowCount: previewRows }),
|
|
10023
|
+
},
|
|
10024
|
+
promotion: {
|
|
10025
|
+
path: notebookResearchPlanPromotionPath({
|
|
10026
|
+
hasSql: Boolean(sql),
|
|
10027
|
+
hasEvidence,
|
|
10028
|
+
hasPreview,
|
|
10029
|
+
previewError: input.previewError,
|
|
10030
|
+
draftBlockPath: input.run.draftBlockPath,
|
|
10031
|
+
promotionAction,
|
|
10032
|
+
}),
|
|
10033
|
+
duplicateDecision: promotionAction,
|
|
10034
|
+
},
|
|
10035
|
+
reviewFocus: notebookResearchPlanFocus(input.run.intent, {
|
|
10036
|
+
hasSql: Boolean(sql),
|
|
10037
|
+
hasEvidence,
|
|
10038
|
+
hasPreview,
|
|
10039
|
+
dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
|
|
10040
|
+
missingContextCount: missingContext.length,
|
|
10041
|
+
promotionAction,
|
|
10042
|
+
}),
|
|
10043
|
+
generatedAt: new Date().toISOString(),
|
|
10044
|
+
};
|
|
10045
|
+
}
|
|
10046
|
+
function notebookResearchPlanPromotionPath(input) {
|
|
10047
|
+
if (!input.hasSql)
|
|
10048
|
+
return 'needs_sql';
|
|
10049
|
+
if (!input.hasEvidence)
|
|
10050
|
+
return 'review_context';
|
|
10051
|
+
if (input.previewError || !input.hasPreview)
|
|
10052
|
+
return 'run_preview';
|
|
10053
|
+
if (input.promotionAction === 'reuse_existing')
|
|
10054
|
+
return 'reuse_existing';
|
|
10055
|
+
if (input.draftBlockPath)
|
|
10056
|
+
return 'open_certification';
|
|
10057
|
+
return 'create_dql_draft';
|
|
10058
|
+
}
|
|
10059
|
+
function notebookResearchPlanFocus(intent, input) {
|
|
10060
|
+
const focus = new Set();
|
|
10061
|
+
if (!input.hasSql)
|
|
10062
|
+
focus.add('Add reviewed SQL or configure AI SQL generation.');
|
|
10063
|
+
if (!input.hasEvidence)
|
|
10064
|
+
focus.add('Save metadata context evidence before DQL promotion.');
|
|
10065
|
+
if (!input.hasPreview)
|
|
10066
|
+
focus.add('Run a bounded preview and inspect result shape.');
|
|
10067
|
+
if (input.dynamicParameterCount === 0)
|
|
10068
|
+
focus.add('Confirm whether this should be static or parameterized.');
|
|
10069
|
+
if (input.missingContextCount > 0)
|
|
10070
|
+
focus.add('Resolve missing metadata context before certification.');
|
|
10071
|
+
if (input.promotionAction === 'reuse_existing')
|
|
10072
|
+
focus.add('Reuse the existing certified block or document replacement rationale.');
|
|
10073
|
+
const pattern = notebookResearchIntentPattern(intent);
|
|
10074
|
+
const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus : [];
|
|
10075
|
+
for (const item of reviewFocus.slice(0, 4))
|
|
10076
|
+
focus.add(String(item));
|
|
10077
|
+
return Array.from(focus).slice(0, 8);
|
|
10078
|
+
}
|
|
10079
|
+
function notebookResearchPreviewInfo(value) {
|
|
10080
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
10081
|
+
return { hasPreview: false, rowCount: 0 };
|
|
10082
|
+
const record = value;
|
|
10083
|
+
const rowCount = typeof record.rowCount === 'number' && Number.isFinite(record.rowCount)
|
|
10084
|
+
? record.rowCount
|
|
10085
|
+
: Array.isArray(record.rows)
|
|
10086
|
+
? record.rows.length
|
|
10087
|
+
: 0;
|
|
10088
|
+
const hasColumns = Array.isArray(record.columns) && record.columns.length > 0;
|
|
10089
|
+
return { hasPreview: hasColumns || rowCount > 0, rowCount };
|
|
10090
|
+
}
|
|
10091
|
+
function notebookResearchEvidenceCount(value) {
|
|
10092
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
10093
|
+
return 0;
|
|
10094
|
+
const record = value;
|
|
10095
|
+
const selected = Array.isArray(record.selectedEvidence) ? record.selectedEvidence.length : 0;
|
|
10096
|
+
const citations = Array.isArray(record.citations) ? record.citations.length : 0;
|
|
10097
|
+
const roles = Array.isArray(record.evidenceRoles) ? record.evidenceRoles.length : 0;
|
|
10098
|
+
return selected + citations + roles;
|
|
10099
|
+
}
|
|
10100
|
+
function notebookResearchInteger(value, fallback, min, max) {
|
|
10101
|
+
if (typeof value !== 'string' || !value.trim())
|
|
10102
|
+
return fallback;
|
|
10103
|
+
const parsed = Number(value);
|
|
10104
|
+
if (!Number.isFinite(parsed))
|
|
10105
|
+
return fallback;
|
|
10106
|
+
return Math.max(min, Math.min(max, Math.floor(parsed)));
|
|
10107
|
+
}
|
|
10108
|
+
function notebookExecutionContext(value) {
|
|
10109
|
+
if (!value || typeof value !== 'object')
|
|
10110
|
+
return null;
|
|
10111
|
+
const raw = value;
|
|
10112
|
+
const notebookPath = notebookResearchString(raw.notebookPath);
|
|
10113
|
+
if (!notebookPath)
|
|
10114
|
+
return null;
|
|
10115
|
+
return {
|
|
10116
|
+
notebookPath,
|
|
10117
|
+
cellId: notebookResearchString(raw.cellId),
|
|
10118
|
+
cellName: notebookResearchString(raw.cellName),
|
|
10119
|
+
researchRunId: notebookResearchString(raw.researchRunId),
|
|
10120
|
+
source: notebookResearchString(raw.source),
|
|
10121
|
+
};
|
|
10122
|
+
}
|
|
10123
|
+
function notebookResearchSelectedContext(run, context) {
|
|
10124
|
+
return {
|
|
10125
|
+
activeSurface: 'notebook',
|
|
10126
|
+
notebookPath: run.notebookPath,
|
|
10127
|
+
domain: run.domain,
|
|
10128
|
+
sourceCellId: run.sourceCellId,
|
|
10129
|
+
sourceCellName: run.sourceCellName,
|
|
10130
|
+
context,
|
|
10131
|
+
};
|
|
10132
|
+
}
|
|
10133
|
+
function notebookResearchIntentPattern(intent) {
|
|
10134
|
+
switch (intent) {
|
|
10135
|
+
case 'diagnose_change':
|
|
10136
|
+
return {
|
|
10137
|
+
label: 'Change diagnosis',
|
|
10138
|
+
dqlTarget: 'driver or trend block',
|
|
10139
|
+
sqlFocus: ['baseline period', 'comparison period', 'metric delta', 'driver dimensions'],
|
|
10140
|
+
reviewFocus: ['time grain', 'comparison filters', 'metric definition', 'explainability evidence'],
|
|
10141
|
+
};
|
|
10142
|
+
case 'driver_breakdown':
|
|
10143
|
+
return {
|
|
10144
|
+
label: 'Driver breakdown',
|
|
10145
|
+
dqlTarget: 'ranking or contribution block',
|
|
10146
|
+
sqlFocus: ['contribution metric', 'ranking', 'tie breaker', 'segment dimensions'],
|
|
10147
|
+
reviewFocus: ['metric grain', 'ranking direction', 'parameterized filters', 'duplicate block match'],
|
|
10148
|
+
};
|
|
10149
|
+
case 'segment_compare':
|
|
10150
|
+
return {
|
|
10151
|
+
label: 'Segment compare',
|
|
10152
|
+
dqlTarget: 'comparison block',
|
|
10153
|
+
sqlFocus: ['shared metric', 'segment fields', 'normalized comparisons', 'deltas'],
|
|
10154
|
+
reviewFocus: ['segment definitions', 'filter compatibility', 'comparison grain', 'semantic metric alignment'],
|
|
10155
|
+
};
|
|
10156
|
+
case 'entity_drilldown':
|
|
10157
|
+
return {
|
|
10158
|
+
label: 'Entity drilldown',
|
|
10159
|
+
dqlTarget: 'entity profile or drilldown block',
|
|
10160
|
+
sqlFocus: ['entity grain', 'stable identifier', 'detail fields', 'supporting metrics'],
|
|
10161
|
+
reviewFocus: ['entity contract', 'primary key', 'allowed filters', 'downstream drill path'],
|
|
10162
|
+
};
|
|
10163
|
+
case 'anomaly_investigation':
|
|
10164
|
+
return {
|
|
10165
|
+
label: 'Anomaly investigation',
|
|
10166
|
+
dqlTarget: 'monitoring or exception block',
|
|
10167
|
+
sqlFocus: ['observed value', 'expected range', 'anomaly rule', 'supporting dimensions'],
|
|
10168
|
+
reviewFocus: ['threshold definition', 'time window', 'false-positive risk', 'evidence fields'],
|
|
10169
|
+
};
|
|
10170
|
+
case 'trust_gap_review':
|
|
10171
|
+
return {
|
|
10172
|
+
label: 'Trust review',
|
|
10173
|
+
dqlTarget: 'replacement or validation block',
|
|
10174
|
+
sqlFocus: ['definition comparison', 'lineage trace', 'duplicate candidates', 'conflict evidence'],
|
|
10175
|
+
reviewFocus: ['certified block match', 'semantic metric match', 'replacement reason', 'governance blockers'],
|
|
10176
|
+
};
|
|
10177
|
+
default:
|
|
10178
|
+
return {
|
|
10179
|
+
label: 'Ad hoc analysis',
|
|
10180
|
+
dqlTarget: 'custom block',
|
|
10181
|
+
sqlFocus: ['business question', 'grain', 'metrics', 'dimensions', 'filters'],
|
|
10182
|
+
reviewFocus: ['reusability', 'parameterization', 'lineage evidence', 'DQL promotion path'],
|
|
10183
|
+
};
|
|
10184
|
+
}
|
|
10185
|
+
}
|
|
10186
|
+
function notebookResearchAgentPrompt(question, intent) {
|
|
10187
|
+
const pattern = notebookResearchIntentPattern(intent);
|
|
10188
|
+
const sqlFocus = Array.isArray(pattern.sqlFocus) ? pattern.sqlFocus.join(', ') : 'business logic';
|
|
10189
|
+
const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus.join(', ') : 'review evidence';
|
|
10190
|
+
return [
|
|
10191
|
+
`Research question: ${question}`,
|
|
10192
|
+
`Research pattern: ${pattern.label ?? intent}`,
|
|
10193
|
+
`DQL target: ${pattern.dqlTarget ?? 'custom block'}`,
|
|
10194
|
+
`SQL focus: ${sqlFocus}`,
|
|
10195
|
+
`Review focus: ${reviewFocus}`,
|
|
10196
|
+
'Generate read-only SQL that can become a reusable, parameterized DQL block after human review.',
|
|
10197
|
+
].join('\n');
|
|
10198
|
+
}
|
|
10199
|
+
function notebookResearchRouteDecisionForRun(run, routeDecision, sqlForPreview) {
|
|
10200
|
+
if (!routeDecision)
|
|
10201
|
+
return undefined;
|
|
10202
|
+
const hasSelectedNotebookSql = Boolean(run.sourceCellId && notebookResearchString(sqlForPreview));
|
|
10203
|
+
if (!hasSelectedNotebookSql || routeDecision.route !== 'certified')
|
|
10204
|
+
return routeDecision;
|
|
10205
|
+
const certifiedEvidence = routeDecision.selectedEvidence.find((item) => item.role === 'exact_certified_answer')
|
|
10206
|
+
?? routeDecision.selectedEvidence.find((item) => item.role === 'certified_context');
|
|
10207
|
+
const certifiedName = certifiedEvidence?.name ?? routeDecision.certifiedApplicability?.objectKey ?? 'matching certified artifact';
|
|
10208
|
+
return {
|
|
10209
|
+
...routeDecision,
|
|
10210
|
+
route: 'research',
|
|
10211
|
+
trustLabel: routeDecision.trustLabel === 'certified' ? 'mixed' : routeDecision.trustLabel,
|
|
10212
|
+
reviewStatus: 'needs_review',
|
|
10213
|
+
exactObjectKey: undefined,
|
|
10214
|
+
reason: `Selected notebook SQL from "${run.sourceCellName ?? run.sourceCellId}" is the source for this research run. Certified artifact "${certifiedName}" is duplicate/reuse evidence until the reviewed SQL, grain, parameters, and preview are accepted.`,
|
|
10215
|
+
selectedEvidence: routeDecision.selectedEvidence.map((item) => item.role === 'exact_certified_answer'
|
|
10216
|
+
? {
|
|
10217
|
+
...item,
|
|
10218
|
+
role: 'certified_context',
|
|
10219
|
+
reason: 'Certified block is reuse or duplicate evidence for the selected notebook SQL; it is not an automatic answer for this research run.',
|
|
10220
|
+
}
|
|
10221
|
+
: item),
|
|
10222
|
+
missingContext: routeDecision.missingContext.length > 0
|
|
10223
|
+
? routeDecision.missingContext
|
|
10224
|
+
: [{
|
|
10225
|
+
kind: 'metadata',
|
|
10226
|
+
severity: 'warning',
|
|
10227
|
+
message: 'Selected notebook SQL requires human review before DQL promotion, even when certified context exists.',
|
|
10228
|
+
}],
|
|
10229
|
+
};
|
|
10230
|
+
}
|
|
10231
|
+
function notebookResearchContextPreview(contextPack) {
|
|
10232
|
+
return {
|
|
10233
|
+
contextPackId: contextPack.id,
|
|
10234
|
+
trustLabel: contextPack.trustLabel,
|
|
10235
|
+
routeDecision: contextPack.routeDecision
|
|
10236
|
+
? {
|
|
10237
|
+
route: contextPack.routeDecision.route,
|
|
10238
|
+
intent: contextPack.routeDecision.intent,
|
|
10239
|
+
reason: contextPack.routeDecision.reason,
|
|
10240
|
+
}
|
|
10241
|
+
: undefined,
|
|
10242
|
+
evidence: contextPack.evidenceRoles.slice(0, 16).map((item) => ({
|
|
10243
|
+
objectKey: item.objectKey,
|
|
10244
|
+
objectType: item.objectType,
|
|
10245
|
+
name: item.name,
|
|
10246
|
+
role: item.role,
|
|
10247
|
+
reason: item.reason,
|
|
10248
|
+
})),
|
|
10249
|
+
summaries: contextPack.evidenceSummaries.slice(0, 12).map((item) => ({
|
|
10250
|
+
title: item.title,
|
|
10251
|
+
detail: item.detail,
|
|
10252
|
+
objectType: item.objectType,
|
|
10253
|
+
reason: item.reason,
|
|
10254
|
+
})),
|
|
10255
|
+
relations: contextPack.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
|
|
10256
|
+
relation: relation.relation,
|
|
10257
|
+
name: relation.name,
|
|
10258
|
+
source: relation.source,
|
|
10259
|
+
columns: relation.columns.slice(0, 24).map((column) => {
|
|
10260
|
+
if (typeof column === 'string')
|
|
10261
|
+
return column;
|
|
10262
|
+
if (column && typeof column === 'object' && typeof column.name === 'string') {
|
|
10263
|
+
return String(column.name);
|
|
10264
|
+
}
|
|
10265
|
+
return String(column);
|
|
10266
|
+
}),
|
|
10267
|
+
})),
|
|
10268
|
+
missingContext: contextPack.missingContext.slice(0, 12).map((item) => ({
|
|
10269
|
+
kind: item.kind,
|
|
10270
|
+
message: item.message,
|
|
10271
|
+
severity: item.severity,
|
|
10272
|
+
})),
|
|
10273
|
+
warnings: contextPack.warnings.slice(0, 12),
|
|
10274
|
+
topRejected: contextPack.retrievalDiagnostics.topRejected.slice(0, 8).map((item) => ({
|
|
10275
|
+
name: item.name,
|
|
10276
|
+
objectType: item.objectType,
|
|
10277
|
+
reason: item.reason,
|
|
10278
|
+
score: item.score,
|
|
10279
|
+
})),
|
|
10280
|
+
counts: {
|
|
10281
|
+
objects: contextPack.objects.length,
|
|
10282
|
+
evidence: contextPack.evidenceRoles.length,
|
|
10283
|
+
relations: contextPack.allowedSqlContext.relations.length,
|
|
10284
|
+
warnings: contextPack.warnings.length + contextPack.missingContext.length,
|
|
10285
|
+
},
|
|
10286
|
+
};
|
|
10287
|
+
}
|
|
10288
|
+
function normalizeNotebookAgentResult(result) {
|
|
10289
|
+
const columns = Array.isArray(result.columns)
|
|
10290
|
+
? result.columns.map((column) => {
|
|
10291
|
+
if (typeof column === 'string')
|
|
10292
|
+
return column;
|
|
10293
|
+
if (column && typeof column === 'object' && typeof column.name === 'string') {
|
|
10294
|
+
return String(column.name);
|
|
10295
|
+
}
|
|
10296
|
+
return String(column);
|
|
10297
|
+
})
|
|
10298
|
+
: [];
|
|
10299
|
+
const rows = Array.isArray(result.rows)
|
|
10300
|
+
? result.rows
|
|
10301
|
+
.filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row)))
|
|
10302
|
+
.map((row) => row)
|
|
10303
|
+
: [];
|
|
10304
|
+
return {
|
|
10305
|
+
columns,
|
|
10306
|
+
rows,
|
|
10307
|
+
rowCount: typeof result.rowCount === 'number' ? result.rowCount : rows.length,
|
|
10308
|
+
executionTime: typeof result.executionTime === 'number' ? result.executionTime : 0,
|
|
10309
|
+
};
|
|
10310
|
+
}
|
|
10311
|
+
function notebookResearchSummary(question, result, error) {
|
|
10312
|
+
if (error)
|
|
10313
|
+
return `Research could not preview results for "${question}". ${error}`;
|
|
10314
|
+
if (result) {
|
|
10315
|
+
const rowCount = result.rowCount ?? result.rows.length;
|
|
10316
|
+
return `Research preview for "${question}" returned ${rowCount.toLocaleString()} row${rowCount === 1 ? '' : 's'} across ${result.columns.length.toLocaleString()} column${result.columns.length === 1 ? '' : 's'}.`;
|
|
10317
|
+
}
|
|
10318
|
+
return `Research plan created for "${question}". Add or generate SQL, then run a bounded preview.`;
|
|
10319
|
+
}
|
|
10320
|
+
function recordNotebookQueryRun(projectRoot, input) {
|
|
10321
|
+
try {
|
|
10322
|
+
recordQueryRun(projectRoot, {
|
|
10323
|
+
objectKey: input.objectKey,
|
|
10324
|
+
source: input.source,
|
|
10325
|
+
status: input.status,
|
|
10326
|
+
rowCount: input.rowCount,
|
|
10327
|
+
durationMs: input.durationMs,
|
|
10328
|
+
errorCode: input.errorCode,
|
|
10329
|
+
payload: {
|
|
10330
|
+
notebookPath: input.notebookPath,
|
|
10331
|
+
cellId: input.cellId,
|
|
10332
|
+
cellName: input.cellName,
|
|
10333
|
+
researchRunId: input.researchRunId,
|
|
10334
|
+
contextPackId: input.contextPackId,
|
|
10335
|
+
sql: input.sql ? compactSqlForRunHistory(input.sql) : undefined,
|
|
10336
|
+
},
|
|
10337
|
+
});
|
|
10338
|
+
}
|
|
10339
|
+
catch {
|
|
10340
|
+
// Query history is advisory metadata and must not block notebook execution.
|
|
10341
|
+
}
|
|
10342
|
+
}
|
|
10343
|
+
function updateNotebookResearchFromCellExecution(projectRoot, execContext, input) {
|
|
10344
|
+
if (!execContext.notebookPath || (!execContext.cellId && !execContext.researchRunId))
|
|
10345
|
+
return;
|
|
10346
|
+
const storage = new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
|
|
10347
|
+
try {
|
|
10348
|
+
const page = storage.listRunsPage({
|
|
10349
|
+
notebookPath: execContext.notebookPath,
|
|
10350
|
+
sort: 'updated_desc',
|
|
10351
|
+
limit: 500,
|
|
10352
|
+
});
|
|
10353
|
+
const matches = page.runs.filter((run) => {
|
|
10354
|
+
if (execContext.researchRunId && run.id === execContext.researchRunId)
|
|
10355
|
+
return true;
|
|
10356
|
+
return Boolean(execContext.cellId && run.sourceCellId === execContext.cellId);
|
|
10357
|
+
});
|
|
10358
|
+
const now = new Date().toISOString();
|
|
10359
|
+
for (const run of matches.slice(0, 20)) {
|
|
10360
|
+
if (input.status === 'success') {
|
|
10361
|
+
storage.updateRun(run.id, {
|
|
10362
|
+
status: 'ready',
|
|
10363
|
+
resultPreview: input.resultPreview,
|
|
10364
|
+
summary: notebookResearchSummary(run.question, input.resultPreview, undefined),
|
|
10365
|
+
recommendation: run.recommendation ?? 'Review SQL, metadata evidence, grain, filters, and duplicate matches before DQL draft promotion.',
|
|
10366
|
+
error: '',
|
|
10367
|
+
lastRunAt: now,
|
|
10368
|
+
});
|
|
10369
|
+
}
|
|
10370
|
+
else {
|
|
10371
|
+
storage.updateRun(run.id, {
|
|
10372
|
+
status: 'error',
|
|
10373
|
+
error: input.error,
|
|
10374
|
+
summary: notebookResearchSummary(run.question, undefined, input.error),
|
|
10375
|
+
lastRunAt: now,
|
|
10376
|
+
});
|
|
10377
|
+
}
|
|
10378
|
+
}
|
|
10379
|
+
}
|
|
10380
|
+
catch {
|
|
10381
|
+
// Research run sync is advisory metadata and must not block notebook execution.
|
|
10382
|
+
}
|
|
10383
|
+
finally {
|
|
10384
|
+
storage.close();
|
|
10385
|
+
}
|
|
10386
|
+
}
|
|
10387
|
+
function compactSqlForRunHistory(sql) {
|
|
10388
|
+
const clean = sql.replace(/\s+/g, ' ').trim();
|
|
10389
|
+
return clean.length > 1200 ? `${clean.slice(0, 1197)}...` : clean;
|
|
10390
|
+
}
|
|
6834
10391
|
function buildAgentSchemaContextFromContextPack(question, contextPack) {
|
|
6835
10392
|
const byRelation = new Map();
|
|
6836
10393
|
const objectsByKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
|