@duckcodeailabs/dql-cli 1.6.17 → 1.6.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/apps-api.d.ts +41 -1
  2. package/dist/apps-api.d.ts.map +1 -1
  3. package/dist/apps-api.js +297 -18
  4. package/dist/apps-api.js.map +1 -1
  5. package/dist/args.d.ts +4 -0
  6. package/dist/args.d.ts.map +1 -1
  7. package/dist/args.js +10 -1
  8. package/dist/args.js.map +1 -1
  9. package/dist/assets/dql-notebook/assets/index-CcpAv4lQ.js +3864 -0
  10. package/dist/assets/dql-notebook/assets/index-rJXLSheJ.js +3910 -0
  11. package/dist/assets/dql-notebook/index.html +1 -1
  12. package/dist/block-studio-import.d.ts +99 -3
  13. package/dist/block-studio-import.d.ts.map +1 -1
  14. package/dist/block-studio-import.js +772 -27
  15. package/dist/block-studio-import.js.map +1 -1
  16. package/dist/commands/agent.d.ts.map +1 -1
  17. package/dist/commands/agent.js +12 -1
  18. package/dist/commands/agent.js.map +1 -1
  19. package/dist/commands/app.d.ts +2 -1
  20. package/dist/commands/app.d.ts.map +1 -1
  21. package/dist/commands/app.js +22 -4
  22. package/dist/commands/app.js.map +1 -1
  23. package/dist/commands/certify.d.ts.map +1 -1
  24. package/dist/commands/certify.js +104 -10
  25. package/dist/commands/certify.js.map +1 -1
  26. package/dist/commands/compile.d.ts +1 -0
  27. package/dist/commands/compile.d.ts.map +1 -1
  28. package/dist/commands/compile.js +26 -4
  29. package/dist/commands/compile.js.map +1 -1
  30. package/dist/commands/doctor.d.ts.map +1 -1
  31. package/dist/commands/doctor.js +345 -6
  32. package/dist/commands/doctor.js.map +1 -1
  33. package/dist/commands/import.d.ts.map +1 -1
  34. package/dist/commands/import.js +45 -70
  35. package/dist/commands/import.js.map +1 -1
  36. package/dist/commands/lineage.d.ts +2 -0
  37. package/dist/commands/lineage.d.ts.map +1 -1
  38. package/dist/commands/lineage.js +111 -0
  39. package/dist/commands/lineage.js.map +1 -1
  40. package/dist/commands/migrate.d.ts +1 -0
  41. package/dist/commands/migrate.d.ts.map +1 -1
  42. package/dist/commands/migrate.js +106 -2
  43. package/dist/commands/migrate.js.map +1 -1
  44. package/dist/commands/new.d.ts.map +1 -1
  45. package/dist/commands/new.js +246 -9
  46. package/dist/commands/new.js.map +1 -1
  47. package/dist/commands/promote.d.ts +11 -0
  48. package/dist/commands/promote.d.ts.map +1 -0
  49. package/dist/commands/promote.js +236 -0
  50. package/dist/commands/promote.js.map +1 -0
  51. package/dist/commands/semantic.js +2 -2
  52. package/dist/commands/semantic.js.map +1 -1
  53. package/dist/index.js +48 -8
  54. package/dist/index.js.map +1 -1
  55. package/dist/llm/providers/claude-agent-sdk.js +1 -1
  56. package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
  57. package/dist/llm/tools.js +1 -1
  58. package/dist/llm/tools.js.map +1 -1
  59. package/dist/local-runtime.d.ts +176 -5
  60. package/dist/local-runtime.d.ts.map +1 -1
  61. package/dist/local-runtime.js +3703 -146
  62. package/dist/local-runtime.js.map +1 -1
  63. package/dist/package.json +10 -10
  64. package/dist/promote-from-draft.d.ts +11 -10
  65. package/dist/promote-from-draft.d.ts.map +1 -1
  66. package/dist/promote-from-draft.js +53 -30
  67. package/dist/promote-from-draft.js.map +1 -1
  68. package/dist/schedule/runner.js +2 -2
  69. package/dist/schedule/runner.js.map +1 -1
  70. package/dist/semantic-import.d.ts.map +1 -1
  71. package/dist/semantic-import.js +28 -12
  72. package/dist/semantic-import.js.map +1 -1
  73. package/package.json +11 -11
  74. package/dist/assets/dql-notebook/assets/index-BFTvljNh.js +0 -3835
@@ -7,21 +7,50 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
7
7
  import Anthropic from '@anthropic-ai/sdk';
8
8
  import OpenAI from 'openai';
9
9
  import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
10
- import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
10
+ import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
11
11
  import { load as loadYaml } from 'js-yaml';
12
12
  import { listBlockTemplates } from './block-templates.js';
13
13
  import { getRunner as getLLMRunner } from './llm/index.js';
14
14
  import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
15
- import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
16
- import { handleAppsApi } from './apps-api.js';
15
+ import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordQueryRun, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
16
+ import { handleAppsApi, recommendVisualization } from './apps-api.js';
17
17
  import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
18
18
  import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
19
- import { LocalAppStorage, defaultLocalAppsDbPath } from '@duckcodeailabs/dql-project';
20
- import { Certifier } from '@duckcodeailabs/dql-governance';
19
+ import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
20
+ import { Certifier, ENTERPRISE_RULES } from '@duckcodeailabs/dql-governance';
21
21
  import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
22
- import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
22
+ import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
23
23
  import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
24
24
  const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
25
+ export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
26
+ const configured = projectConfig.semanticLayer;
27
+ const dbtProjectDir = projectConfig.dbt?.projectDir;
28
+ if (dbtProjectDir
29
+ && (!configured || configured.provider === 'dql')
30
+ && hasDbtSemanticArtifacts(projectRoot, dbtProjectDir)) {
31
+ return { provider: 'dbt', projectPath: dbtProjectDir };
32
+ }
33
+ return configured;
34
+ }
35
+ function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
36
+ const dbtRoot = resolve(projectRoot, dbtProjectDir);
37
+ if (existsSync(join(dbtRoot, 'target', 'semantic_manifest.json')))
38
+ return true;
39
+ const manifestPath = join(dbtRoot, 'target', 'manifest.json');
40
+ if (!existsSync(manifestPath))
41
+ return false;
42
+ try {
43
+ const parsed = JSON.parse(readFileSync(manifestPath, 'utf-8'));
44
+ if (parsed.semantic_models && typeof parsed.semantic_models === 'object')
45
+ return true;
46
+ if (parsed.metrics && typeof parsed.metrics === 'object')
47
+ return true;
48
+ }
49
+ catch {
50
+ return false;
51
+ }
52
+ return false;
53
+ }
25
54
  export async function startLocalServer(opts) {
26
55
  const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
27
56
  const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
@@ -40,7 +69,7 @@ export async function startLocalServer(opts) {
40
69
  const semanticLayerDir = join(projectRoot, 'semantic-layer');
41
70
  let semanticImportManifest = loadSemanticImportManifest(projectRoot);
42
71
  const userPrefsPath = join(projectRoot, '.dql-user-prefs.json');
43
- const semanticConfig = projectConfig.semanticLayer;
72
+ const semanticConfig = resolveProjectSemanticConfig(projectConfig, projectRoot);
44
73
  let semanticLastSyncTime = null;
45
74
  {
46
75
  const semanticConnection = connection;
@@ -330,6 +359,430 @@ export async function startLocalServer(opts) {
330
359
  providerUsed: governedAnswer.providerUsed,
331
360
  };
332
361
  };
362
+ const openNotebookResearchStorage = () => new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
363
+ const runNotebookResearch = async (storage, run, input = {}) => {
364
+ const question = notebookResearchString(input.question) || run.question;
365
+ const domain = notebookResearchString(input.domain) ?? run.domain;
366
+ const owner = notebookResearchString(input.owner) ?? run.owner;
367
+ const sourceCellFingerprint = notebookResearchString(input.sourceCellFingerprint) ?? run.sourceCellFingerprint;
368
+ const intent = input.intent ?? run.intent;
369
+ const context = input.context === undefined ? run.context : input.context;
370
+ let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
371
+ let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
372
+ const startedAt = new Date().toISOString();
373
+ storage.updateRun(run.id, {
374
+ domain,
375
+ owner,
376
+ sourceCellFingerprint,
377
+ question,
378
+ intent,
379
+ context,
380
+ generatedSql,
381
+ reviewedSql,
382
+ status: 'running',
383
+ reviewStatus: 'needs_review',
384
+ error: '',
385
+ });
386
+ try {
387
+ const contextPack = await buildLocalContextPack(projectRoot, {
388
+ question,
389
+ mode: 'question',
390
+ surface: 'notebook',
391
+ selectedContext: {
392
+ ...notebookResearchSelectedContext(run, context),
393
+ domain,
394
+ owner,
395
+ intent,
396
+ researchPattern: notebookResearchIntentPattern(intent),
397
+ },
398
+ strictness: 'balanced',
399
+ limit: 100,
400
+ }).catch((error) => {
401
+ const message = error instanceof Error ? error.message : String(error);
402
+ return {
403
+ id: '',
404
+ routeDecision: undefined,
405
+ evidenceRoles: [],
406
+ warnings: [`Context pack failed: ${message}`],
407
+ retrievalDiagnostics: { selectedEvidence: [] },
408
+ };
409
+ });
410
+ let governedAnswer;
411
+ let providerError;
412
+ const generationWarnings = [];
413
+ if (!generatedSql && !reviewedSql) {
414
+ const resolvedProvider = resolveDefaultLLMProvider(projectRoot);
415
+ const runner = resolvedProvider ? getLLMRunner(resolvedProvider) : null;
416
+ if (!resolvedProvider || !runner) {
417
+ generationWarnings.push('No AI provider is configured. Metadata context was saved as a research plan; paste SQL or configure an AI provider to generate candidate SQL.');
418
+ }
419
+ else {
420
+ const controller = new AbortController();
421
+ try {
422
+ await runner.run({
423
+ provider: resolvedProvider,
424
+ messages: [{ role: 'user', content: notebookResearchAgentPrompt(question, intent) }],
425
+ upstream: {
426
+ cellId: `notebook-research:${run.notebookPath}:${run.id}`,
427
+ sql: JSON.stringify({
428
+ mode: 'notebook_research',
429
+ notebookPath: run.notebookPath,
430
+ sourceCellId: run.sourceCellId,
431
+ sourceCellName: run.sourceCellName,
432
+ owner,
433
+ intent,
434
+ researchPattern: notebookResearchIntentPattern(intent),
435
+ instruction: 'Generate review-required read-only SQL from the inspected metadata context. Execute only through bounded preview and keep the result uncertified until promoted to a DQL draft.',
436
+ context,
437
+ }, null, 2),
438
+ },
439
+ projectRoot,
440
+ executeCertifiedBlock: executeCertifiedBlockForAgent,
441
+ executeGeneratedSql: executeGeneratedSqlForAgent,
442
+ getSchemaContext: getSchemaContextForAgent,
443
+ }, (turn) => {
444
+ if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
445
+ governedAnswer = turn.output;
446
+ }
447
+ if (turn.kind === 'error')
448
+ providerError = turn.message;
449
+ }, controller.signal);
450
+ }
451
+ catch (error) {
452
+ providerError = error instanceof Error ? error.message : String(error);
453
+ }
454
+ if (!governedAnswer) {
455
+ generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
456
+ }
457
+ else {
458
+ generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
459
+ if (!generatedSql) {
460
+ generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
461
+ }
462
+ }
463
+ }
464
+ }
465
+ const sqlForPreview = reviewedSql || generatedSql;
466
+ const warnings = [
467
+ ...(contextPack.warnings ?? []),
468
+ ...(governedAnswer?.validationWarnings ?? []),
469
+ ...generationWarnings,
470
+ ].filter(Boolean);
471
+ let resultPreview;
472
+ let previewError;
473
+ if (governedAnswer?.result?.rows && !reviewedSql) {
474
+ resultPreview = normalizeNotebookAgentResult(governedAnswer.result);
475
+ }
476
+ else if (sqlForPreview) {
477
+ try {
478
+ const previewSql = buildAgentPreviewSql(sqlForPreview);
479
+ const previewStart = Date.now();
480
+ resultPreview = await executeLocalSqlForStoredResult(previewSql);
481
+ recordNotebookQueryRun(projectRoot, {
482
+ notebookPath: run.notebookPath,
483
+ cellId: run.sourceCellId,
484
+ cellName: run.sourceCellName,
485
+ researchRunId: run.id,
486
+ source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
487
+ status: 'success',
488
+ rowCount: resultPreview.rowCount ?? resultPreview.rows.length,
489
+ durationMs: Date.now() - previewStart,
490
+ sql: sqlForPreview,
491
+ contextPackId: contextPack.id,
492
+ });
493
+ }
494
+ catch (error) {
495
+ previewError = error instanceof Error ? error.message : String(error);
496
+ recordNotebookQueryRun(projectRoot, {
497
+ notebookPath: run.notebookPath,
498
+ cellId: run.sourceCellId,
499
+ cellName: run.sourceCellName,
500
+ researchRunId: run.id,
501
+ source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
502
+ status: 'error',
503
+ errorCode: previewError,
504
+ sql: sqlForPreview,
505
+ contextPackId: contextPack.id,
506
+ });
507
+ }
508
+ }
509
+ const routeDecision = notebookResearchRouteDecisionForRun(run, contextPack.routeDecision, sqlForPreview);
510
+ const display = resultPreview
511
+ ? recommendVisualization(projectRoot, {
512
+ prompt: question,
513
+ resultSchema: resultPreview.columns,
514
+ rowSample: resultPreview.rows.slice(0, 25),
515
+ defaultVisualization: governedAnswer?.suggestedViz,
516
+ })
517
+ : undefined;
518
+ const evidence = {
519
+ trustStatus: {
520
+ label: governedAnswer?.trustLabel
521
+ ?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
522
+ reviewRequired: true,
523
+ },
524
+ contextPackId: contextPack.id,
525
+ routeDecision,
526
+ selectedEvidence: contextPack.evidenceRoles?.slice(0, 24) ?? [],
527
+ evidenceRoles: contextPack.evidenceRoles?.slice(0, 24) ?? [],
528
+ evidenceSummaries: contextPack.evidenceSummaries?.slice(0, 16) ?? [],
529
+ allowedSqlContext: {
530
+ relations: (contextPack.allowedSqlContext?.relations ?? []).slice(0, 24).map((relation) => ({
531
+ relation: relation.relation,
532
+ name: relation.name,
533
+ source: relation.source,
534
+ columns: relation.columns.slice(0, 32).map((column) => {
535
+ if (typeof column === 'string')
536
+ return column;
537
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
538
+ return String(column.name);
539
+ }
540
+ return String(column);
541
+ }),
542
+ })),
543
+ sourceBlockSql: contextPack.allowedSqlContext?.sourceBlockSql?.slice(0, 12) ?? [],
544
+ },
545
+ missingContext: contextPack.missingContext?.slice(0, 16) ?? [],
546
+ warnings: contextPack.warnings?.slice(0, 16) ?? [],
547
+ retrievalDiagnostics: contextPack.retrievalDiagnostics,
548
+ agentEvidence: governedAnswer?.evidence,
549
+ analysisPlan: governedAnswer?.analysisPlan,
550
+ citations: [
551
+ ...(contextPack.citations ?? []),
552
+ ...(governedAnswer?.citations ?? []),
553
+ ].slice(0, 40),
554
+ };
555
+ const summary = notebookResearchString(governedAnswer?.answer)
556
+ ?? notebookResearchString(governedAnswer?.text)
557
+ ?? notebookResearchSummary(question, resultPreview, previewError);
558
+ const recommendation = previewError
559
+ ? 'Review the SQL, selected metadata, and connection context before rerunning.'
560
+ : sqlForPreview
561
+ ? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
562
+ : 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
563
+ const researchPlan = buildNotebookResearchPlan({
564
+ run,
565
+ evidence,
566
+ resultPreview,
567
+ previewError,
568
+ generatedSql,
569
+ reviewedSql,
570
+ routeDecision,
571
+ });
572
+ return storage.updateRun(run.id, {
573
+ domain,
574
+ owner,
575
+ sourceCellFingerprint,
576
+ question,
577
+ intent,
578
+ context,
579
+ status: previewError ? 'error' : 'ready',
580
+ summary,
581
+ recommendation,
582
+ resultPreview,
583
+ evidence,
584
+ researchPlan,
585
+ generatedSql,
586
+ reviewedSql,
587
+ display: display && display.ok ? display.display : undefined,
588
+ contextPackId: contextPack.id,
589
+ routeDecision,
590
+ warnings: [
591
+ ...warnings,
592
+ ...(display && !display.ok ? [display.error] : []),
593
+ ...(display && display.ok ? display.warnings : []),
594
+ ],
595
+ reviewStatus: 'needs_review',
596
+ error: previewError,
597
+ lastRunAt: startedAt,
598
+ }) ?? run;
599
+ }
600
+ catch (error) {
601
+ return storage.updateRun(run.id, {
602
+ domain,
603
+ owner,
604
+ sourceCellFingerprint,
605
+ status: 'error',
606
+ error: error instanceof Error ? error.message : String(error),
607
+ reviewStatus: 'needs_review',
608
+ lastRunAt: startedAt,
609
+ }) ?? run;
610
+ }
611
+ };
612
+ const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
613
+ const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
614
+ if (!sql)
615
+ throw new Error('Notebook research needs generated or reviewed SQL before DQL draft promotion.');
616
+ const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
617
+ const session = await createDqlGenerationSessionForProject(projectRoot, {
618
+ inputMode: 'upload',
619
+ sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
620
+ sourceKind: 'raw-sql-file',
621
+ domain: notebookResearchString(input.domain) ?? run.domain,
622
+ owner: notebookResearchString(input.owner),
623
+ tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
624
+ provider: input.provider,
625
+ }, semanticLayer);
626
+ const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
627
+ ?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
628
+ const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
629
+ const updated = storage.markPromoted(run.id, {
630
+ draftBlockPath: draftPath,
631
+ dqlImportId: session.id,
632
+ dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
633
+ dqlPromotion: promotion,
634
+ }) ?? run;
635
+ const planned = storage.updateRun(updated.id, {
636
+ researchPlan: buildNotebookResearchPlan({
637
+ run: updated,
638
+ evidence: updated.evidence,
639
+ resultPreview: updated.resultPreview,
640
+ generatedSql: updated.generatedSql,
641
+ reviewedSql: updated.reviewedSql,
642
+ routeDecision: updated.routeDecision,
643
+ }),
644
+ }) ?? updated;
645
+ return { run: planned, session };
646
+ };
647
+ const checkNotebookResearchReuse = async (storage, run, input = {}) => {
648
+ const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
649
+ if (!sql)
650
+ throw new Error('Notebook research needs generated or reviewed SQL before reuse checking.');
651
+ const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}.sql`;
652
+ const match = await matchSqlForDqlReuse({
653
+ sql,
654
+ sourcePath,
655
+ name: run.title || 'Notebook research SQL',
656
+ domain: notebookResearchString(input.domain) ?? run.domain ?? 'uncategorized',
657
+ owner: notebookResearchString(input.owner) ?? 'analytics',
658
+ });
659
+ const promotion = {
660
+ importId: `reuse-check:${run.id}:${Date.now()}`,
661
+ candidateIds: ['reuse_check'],
662
+ recommendedAction: match.recommendedAction,
663
+ similarityMatches: match.similarityMatches.slice(0, 8).map(toNotebookPromotionMatch),
664
+ candidates: [{
665
+ id: 'reuse_check',
666
+ name: run.title || 'Notebook research SQL',
667
+ domain: notebookResearchString(input.domain) ?? run.domain,
668
+ reviewStatus: 'reuse_checked',
669
+ recommendedAction: match.recommendedAction,
670
+ similarityMatches: match.similarityMatches.slice(0, 5).map(toNotebookPromotionMatch),
671
+ parameterPolicy: match.parameterPolicy.slice(0, 16).map((entry) => ({
672
+ name: entry.name,
673
+ policy: entry.policy,
674
+ })),
675
+ allowedFilters: match.allowedFilters.slice(0, 16),
676
+ warnings: match.parameterDecisions
677
+ .filter((decision) => decision.reason)
678
+ .map((decision) => `${decision.name}: ${decision.reason}`)
679
+ .slice(0, 12),
680
+ }],
681
+ createdAt: new Date().toISOString(),
682
+ };
683
+ const checked = storage.updateRun(run.id, {
684
+ dqlPromotionAction: match.recommendedAction,
685
+ dqlPromotion: promotion,
686
+ researchPlan: buildNotebookResearchPlan({
687
+ run: {
688
+ ...run,
689
+ dqlPromotionAction: match.recommendedAction,
690
+ dqlPromotion: promotion,
691
+ },
692
+ evidence: run.evidence,
693
+ resultPreview: run.resultPreview,
694
+ generatedSql: run.generatedSql,
695
+ reviewedSql: run.reviewedSql,
696
+ routeDecision: run.routeDecision,
697
+ }),
698
+ }) ?? run;
699
+ return { run: checked, promotion, match };
700
+ };
701
+ const matchSqlForDqlReuse = async (input) => {
702
+ const sql = input.sql.trim();
703
+ if (!sql)
704
+ throw new Error('Missing SQL for reuse check.');
705
+ const sourceTables = extractSqlTablesLight(sql);
706
+ const candidate = {
707
+ id: 'match_sql',
708
+ sourceKind: 'raw-sql-file',
709
+ sourcePath: input.sourcePath ?? 'pasted.sql',
710
+ name: input.name ?? 'SQL match preview',
711
+ domain: input.domain ?? 'imported',
712
+ description: 'SQL match preview candidate.',
713
+ owner: input.owner ?? 'analytics',
714
+ tags: [],
715
+ sql,
716
+ dqlSource: '',
717
+ validation: null,
718
+ preview: null,
719
+ lineage: {
720
+ sourceTables,
721
+ parameters: [],
722
+ warnings: [],
723
+ statementIndex: 1,
724
+ totalStatements: 1,
725
+ },
726
+ confidence: 0.8,
727
+ splitStrategy: 'manual',
728
+ warnings: [],
729
+ conversionNotes: [],
730
+ aiAssistance: [],
731
+ reviewStatus: 'draft',
732
+ };
733
+ const evidence = deterministicDqlGenerationEvidence(candidate);
734
+ const patch = deterministicDqlGenerationPatch(candidate, evidence);
735
+ const contextPack = await buildDqlGenerationContextPack(projectRoot, { ...candidate, sql: patch.sql ?? sql }).catch(() => null);
736
+ const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
737
+ return {
738
+ parameterDecisions: patch.parameterDecisions ?? [],
739
+ parameterPolicy: patch.parameterPolicy ?? [],
740
+ filterBindings: patch.filterBindings ?? [],
741
+ allowedFilters: patch.allowedFilters ?? [],
742
+ parameterizedSql: patch.sql ?? sql,
743
+ similarityMatches: similarity.matches,
744
+ recommendedAction: similarity.recommendedAction,
745
+ };
746
+ };
747
+ const buildNotebookDqlPromotionSummary = (session, draftBlockPath) => {
748
+ const primary = draftBlockPath
749
+ ? session.candidates.find((candidate) => candidate.draftSave?.path === draftBlockPath || candidate.savedPath === draftBlockPath) ?? session.candidates[0]
750
+ : session.candidates[0];
751
+ return {
752
+ importId: session.id,
753
+ candidateIds: session.candidates.map((candidate) => candidate.id),
754
+ draftBlockPath,
755
+ recommendedAction: primary?.recommendedAction,
756
+ similarityMatches: (primary?.similarityMatches ?? []).slice(0, 8).map(toNotebookPromotionMatch),
757
+ candidates: session.candidates.map((candidate) => ({
758
+ id: candidate.id,
759
+ name: candidate.name,
760
+ domain: candidate.domain,
761
+ draftPath: candidate.draftSave?.path,
762
+ savedPath: candidate.savedPath,
763
+ reviewStatus: candidate.reviewStatus,
764
+ recommendedAction: candidate.recommendedAction,
765
+ similarityMatches: (candidate.similarityMatches ?? []).slice(0, 5).map(toNotebookPromotionMatch),
766
+ parameterPolicy: (candidate.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
767
+ name: entry.name,
768
+ policy: entry.policy,
769
+ })),
770
+ allowedFilters: (candidate.allowedFilters ?? []).slice(0, 16),
771
+ warnings: [...(candidate.warnings ?? []), ...(candidate.lineage.warnings ?? [])].slice(0, 12),
772
+ })),
773
+ createdAt: new Date().toISOString(),
774
+ };
775
+ };
776
+ const toNotebookPromotionMatch = (match) => ({
777
+ kind: match.kind,
778
+ objectKey: match.objectKey,
779
+ name: match.name,
780
+ status: match.status,
781
+ source: match.source,
782
+ score: match.score,
783
+ reason: match.reason,
784
+ recommendedAction: match.recommendedAction,
785
+ });
333
786
  // SSE clients for /api/watch hot-reload
334
787
  const sseClients = new Set();
335
788
  // Watch notebooks/, workbooks/, semantic-layer/, and data/ dirs for changes
@@ -398,8 +851,28 @@ export async function startLocalServer(opts) {
398
851
  if (validated.reviewStatus === 'rejected') {
399
852
  errors.unshift('Candidate was rejected.');
400
853
  }
854
+ const duplicateBlocker = duplicateCertificationBlocker(validated);
855
+ if (duplicateBlocker)
856
+ errors.unshift(duplicateBlocker);
401
857
  return { candidate: validated, errors };
402
858
  };
859
+ const duplicateCertificationBlocker = (candidate) => {
860
+ const match = candidate.similarityMatches?.[0];
861
+ if (!match)
862
+ return null;
863
+ const duplicateKind = match.kind === 'exact_sql_match'
864
+ || match.kind === 'parameterized_duplicate'
865
+ || (match.kind === 'business_duplicate' && match.status === 'certified' && match.score >= 0.76);
866
+ if (!duplicateKind)
867
+ return null;
868
+ const replacementFor = new Set((candidate.replacementFor ?? []).map((value) => value.toLowerCase()));
869
+ const documentedReplacement = candidate.recommendedAction === 'create_replacement'
870
+ || replacementFor.has(match.name.toLowerCase())
871
+ || (match.objectKey && replacementFor.has(match.objectKey.toLowerCase()));
872
+ if (documentedReplacement)
873
+ return null;
874
+ return `Likely duplicate of ${match.name} (${match.kind}, ${Math.round(match.score * 100)}% match). Reuse the existing block, extend it, or document this draft as a replacement before certification.`;
875
+ };
403
876
  const runBlockStudioPreviewSource = async (source, targetConnection) => {
404
877
  const activeConnection = requireActiveConnection(targetConnection);
405
878
  let tableMapping;
@@ -504,7 +977,50 @@ export async function startLocalServer(opts) {
504
977
  }
505
978
  return { passed, failed, skipped: 0, duration: Date.now() - start, assertions, runAt: new Date() };
506
979
  };
507
- const certifyBlockStudioSource = async (source, blockPath) => {
980
+ const saveDqlGenerationDraft = (importId, candidate) => saveDqlGenerationDraftForProject(projectRoot, importId, candidate);
981
+ const createDqlGenerationSessionFromBody = async (body) => {
982
+ return createDqlGenerationSessionForProject(projectRoot, {
983
+ inputPath: typeof body.path === 'string' ? body.path : '',
984
+ inputMode: body.inputMode === 'paste' || body.inputMode === 'upload' || body.inputMode === 'path' ? body.inputMode : undefined,
985
+ sources: Array.isArray(body.sources)
986
+ ? body.sources.map((source, index) => ({
987
+ path: typeof source?.path === 'string' ? source.path : `source-${index + 1}.sql`,
988
+ content: typeof source?.content === 'string' ? source.content : '',
989
+ }))
990
+ : undefined,
991
+ sourceKind: typeof body.sourceKind === 'string' ? body.sourceKind : 'raw-sql',
992
+ domain: typeof body.domain === 'string' ? body.domain : undefined,
993
+ owner: typeof body.owner === 'string' ? body.owner : undefined,
994
+ tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
995
+ provider: typeof body.provider === 'string' ? body.provider : undefined,
996
+ }, semanticLayer);
997
+ };
998
+ const loadDqlGenerationSession = (importId) => {
999
+ const session = loadBlockStudioImportSession(projectRoot, importId);
1000
+ const candidates = session.candidates.map((candidate) => ({
1001
+ ...candidate,
1002
+ generationMode: candidate.generationMode ?? 'deterministic',
1003
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
1004
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
1005
+ evidence: candidate.evidence ?? deterministicDqlGenerationEvidence(candidate),
1006
+ draftSave: candidate.draftSave ?? (isDraftBlockPath(candidate.savedPath)
1007
+ ? { status: 'saved', path: candidate.savedPath }
1008
+ : { status: 'pending' }),
1009
+ }));
1010
+ return {
1011
+ ...session,
1012
+ mode: 'ai-import',
1013
+ candidates,
1014
+ generation: {
1015
+ provider: candidates.find((candidate) => candidate.generationProvider)?.generationProvider ?? 'local-deterministic',
1016
+ aiEnabled: candidates.some((candidate) => candidate.generationMode === 'ai'),
1017
+ contextObjectCount: candidates.reduce((sum, candidate) => sum + (candidate.evidence?.length ?? 0), 0),
1018
+ createdDrafts: candidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
1019
+ warnings: candidates.flatMap((candidate) => candidate.draftSave.status === 'error' && candidate.draftSave.error ? [candidate.draftSave.error] : []),
1020
+ },
1021
+ };
1022
+ };
1023
+ const certifyBlockStudioSource = async (source, blockPath, options = {}) => {
508
1024
  const validation = validateBlockStudioSource(source, semanticLayer);
509
1025
  let preview = null;
510
1026
  let testResults = null;
@@ -542,12 +1058,24 @@ export async function startLocalServer(opts) {
542
1058
  description: parsed.description,
543
1059
  owner: parsed.owner,
544
1060
  tags: parsed.tags,
1061
+ llmContext: parsed.llmContext,
1062
+ pattern: parsed.pattern,
1063
+ grain: parsed.grain,
1064
+ entities: parsed.entities,
1065
+ declaredOutputs: parsed.outputs,
1066
+ dimensions: parsed.dimensions,
1067
+ allowedFilters: parsed.allowedFilters,
1068
+ parameterPolicy: parsed.parameterPolicy,
1069
+ filterBindings: parsed.filterBindings,
1070
+ sourceSystems: parsed.sourceSystems,
1071
+ replacementFor: parsed.replacementFor,
1072
+ reviewCadence: parsed.reviewCadence,
545
1073
  dependencies: [],
546
1074
  usedInCount: 0,
547
1075
  createdAt: new Date(),
548
1076
  updatedAt: new Date(),
549
1077
  };
550
- const certification = new Certifier().evaluate(record, testResults ?? undefined);
1078
+ const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined);
551
1079
  const checklist = buildBlockStudioCertificationChecklist({
552
1080
  source,
553
1081
  validation,
@@ -730,6 +1258,7 @@ export async function startLocalServer(opts) {
730
1258
  const variables = body.variables && typeof body.variables === 'object'
731
1259
  ? body.variables
732
1260
  : {};
1261
+ const dashboardVariables = dashboardRuntimeVariables(loaded.dashboard, variables);
733
1262
  const tiles = [];
734
1263
  let localApps = null;
735
1264
  for (const item of loaded.dashboard.layout.items) {
@@ -831,8 +1360,15 @@ export async function startLocalServer(opts) {
831
1360
  });
832
1361
  continue;
833
1362
  }
834
- const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, targetConnection, projectRoot, projectConfig);
835
- const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...variables }), prepared.connection);
1363
+ const filterApplication = applyDashboardFiltersToBlockExecution({
1364
+ sql: semanticCompose?.sql ?? plan.sql,
1365
+ sqlParams: plan?.sqlParams ?? [],
1366
+ variables: { ...(plan?.variables ?? {}), ...dashboardVariables },
1367
+ block,
1368
+ dashboard: loaded.dashboard,
1369
+ });
1370
+ const prepared = prepareLocalExecution(filterApplication.sql, targetConnection, projectRoot, projectConfig);
1371
+ const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables(filterApplication.variables), prepared.connection);
836
1372
  tiles.push({
837
1373
  tileId: item.i,
838
1374
  status: 'ok',
@@ -843,6 +1379,10 @@ export async function startLocalServer(opts) {
843
1379
  viz: item.viz,
844
1380
  chartConfig: mergeDashboardChartConfig(plan?.chartConfig, item),
845
1381
  result: normalizeQueryResult(result),
1382
+ filters: {
1383
+ applied: filterApplication.appliedFilters,
1384
+ skipped: filterApplication.skippedFilters,
1385
+ },
846
1386
  citation: {
847
1387
  kind: 'block',
848
1388
  name: block.name,
@@ -885,11 +1425,11 @@ export async function startLocalServer(opts) {
885
1425
  return;
886
1426
  }
887
1427
  // Apps, dashboards, persona — see apps-api.ts. Returns true if handled.
888
- if (path.startsWith('/api/apps') || path === '/api/persona') {
1428
+ if (path.startsWith('/api/apps') || path.startsWith('/api/visualizations') || path === '/api/persona') {
889
1429
  try {
890
1430
  const handled = await handleAppsApi({
891
1431
  req,
892
- res,
1432
+ res: res,
893
1433
  url,
894
1434
  path,
895
1435
  projectRoot,
@@ -1019,6 +1559,393 @@ export async function startLocalServer(opts) {
1019
1559
  }
1020
1560
  return;
1021
1561
  }
1562
+ if (req.method === 'GET' && path === '/api/notebook/research') {
1563
+ const storage = openNotebookResearchStorage();
1564
+ try {
1565
+ const notebookPath = notebookResearchString(url.searchParams.get('path'));
1566
+ const sourceCellId = notebookResearchString(url.searchParams.get('sourceCellId') ?? url.searchParams.get('cellId'));
1567
+ const domain = notebookResearchString(url.searchParams.get('domain'));
1568
+ const owner = notebookResearchString(url.searchParams.get('owner'));
1569
+ const intent = notebookResearchIntent(url.searchParams.get('intent'));
1570
+ const search = notebookResearchString(url.searchParams.get('q')) ?? notebookResearchString(url.searchParams.get('search'));
1571
+ const status = notebookResearchStatus(url.searchParams.get('status'));
1572
+ const reviewStatus = notebookResearchReviewStatus(url.searchParams.get('reviewStatus'));
1573
+ const promotionAction = notebookResearchPromotionAction(url.searchParams.get('promotionAction') ?? url.searchParams.get('action'));
1574
+ const readiness = notebookResearchReadiness(url.searchParams.get('readiness') ?? url.searchParams.get('ready'));
1575
+ const age = notebookResearchAge(url.searchParams.get('age'));
1576
+ const nextAction = notebookResearchNextAction(url.searchParams.get('nextAction') ?? url.searchParams.get('next'));
1577
+ const activeOnlyParam = url.searchParams.get('activeOnly') ?? url.searchParams.get('active');
1578
+ const activeOnly = activeOnlyParam === 'true' || activeOnlyParam === '1';
1579
+ const sort = notebookResearchSort(url.searchParams.get('sort'));
1580
+ const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
1581
+ const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
1582
+ const page = storage.listRunsPage({ notebookPath, sourceCellId, domain, owner, intent, search, status, reviewStatus, promotionAction, readiness, age, nextAction, activeOnly, sort, limit, offset });
1583
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1584
+ res.end(serializeJSON(withNotebookResearchChecklistPage(page)));
1585
+ }
1586
+ catch (error) {
1587
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1588
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1589
+ }
1590
+ finally {
1591
+ storage.close();
1592
+ }
1593
+ return;
1594
+ }
1595
+ if (req.method === 'POST' && path === '/api/notebook/research') {
1596
+ const storage = openNotebookResearchStorage();
1597
+ try {
1598
+ const body = await readJSON(req);
1599
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1600
+ const question = notebookResearchString(body.question);
1601
+ const sourceCell = notebookResearchSourceCellPayload(body);
1602
+ const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
1603
+ const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
1604
+ const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
1605
+ if (!notebookPath || !question) {
1606
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1607
+ res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
1608
+ return;
1609
+ }
1610
+ const created = storage.createRun({
1611
+ notebookPath,
1612
+ domain: notebookResearchString(body.domain),
1613
+ owner: notebookResearchString(body.owner),
1614
+ sourceCell,
1615
+ sourceCellId,
1616
+ sourceCellName,
1617
+ sourceCellFingerprint,
1618
+ title: notebookResearchString(body.title),
1619
+ question,
1620
+ intent: notebookResearchIntent(body.intent),
1621
+ context: body.context,
1622
+ generatedSql: notebookResearchString(body.generatedSql),
1623
+ reviewedSql: notebookResearchString(body.reviewedSql),
1624
+ });
1625
+ const run = body.run === true
1626
+ ? await runNotebookResearch(storage, created, {
1627
+ domain: notebookResearchString(body.domain),
1628
+ sourceCellFingerprint,
1629
+ question,
1630
+ intent: notebookResearchIntent(body.intent),
1631
+ context: body.context,
1632
+ generatedSql: notebookResearchString(body.generatedSql),
1633
+ reviewedSql: notebookResearchString(body.reviewedSql),
1634
+ })
1635
+ : created;
1636
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
1637
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
1638
+ }
1639
+ catch (error) {
1640
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1641
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1642
+ }
1643
+ finally {
1644
+ storage.close();
1645
+ }
1646
+ return;
1647
+ }
1648
+ if (req.method === 'POST' && path === '/api/notebook/research/context-preview') {
1649
+ try {
1650
+ const body = await readJSON(req);
1651
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1652
+ const question = notebookResearchString(body.question);
1653
+ if (!notebookPath || !question) {
1654
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1655
+ res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
1656
+ return;
1657
+ }
1658
+ const intent = notebookResearchIntent(body.intent) ?? 'ad_hoc_analysis';
1659
+ const domain = notebookResearchString(body.domain);
1660
+ const context = body.context;
1661
+ const sourceCell = notebookResearchSourceCellPayload(body);
1662
+ const contextPack = await buildLocalContextPack(projectRoot, {
1663
+ question,
1664
+ mode: 'question',
1665
+ surface: 'notebook',
1666
+ selectedContext: {
1667
+ activeSurface: 'notebook',
1668
+ notebookPath,
1669
+ domain,
1670
+ sourceCellId: notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell),
1671
+ sourceCellName: notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell),
1672
+ context,
1673
+ intent,
1674
+ researchPattern: notebookResearchIntentPattern(intent),
1675
+ },
1676
+ strictness: 'balanced',
1677
+ limit: 100,
1678
+ });
1679
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1680
+ res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
1681
+ }
1682
+ catch (error) {
1683
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1684
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1685
+ }
1686
+ return;
1687
+ }
1688
+ if (req.method === 'POST' && path === '/api/notebook/research/seed-cells') {
1689
+ const storage = openNotebookResearchStorage();
1690
+ try {
1691
+ const body = await readJSON(req);
1692
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1693
+ if (!notebookPath) {
1694
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1695
+ res.end(serializeJSON({ error: 'notebookPath is required.' }));
1696
+ return;
1697
+ }
1698
+ const cells = Array.isArray(body.cells) ? body.cells : [];
1699
+ if (cells.length === 0) {
1700
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1701
+ res.end(serializeJSON({ error: 'At least one source cell is required.' }));
1702
+ return;
1703
+ }
1704
+ const validCells = cells.filter((cell) => Boolean(cell && typeof cell === 'object' && !Array.isArray(cell)));
1705
+ const invalidCellCount = cells.length - validCells.length;
1706
+ const seeded = storage.seedRunsFromCells({
1707
+ notebookPath,
1708
+ domain: notebookResearchString(body.domain),
1709
+ owner: notebookResearchString(body.owner),
1710
+ notebookTitle: notebookResearchString(body.notebookTitle),
1711
+ cells: validCells.map((cell) => ({
1712
+ sourceCell: notebookResearchSourceCellPayload(cell),
1713
+ id: notebookResearchString(cell.id),
1714
+ sourceCellId: notebookResearchString(cell.sourceCellId),
1715
+ name: notebookResearchString(cell.name),
1716
+ sourceCellName: notebookResearchString(cell.sourceCellName),
1717
+ sourceCellFingerprint: notebookResearchString(cell.sourceCellFingerprint),
1718
+ title: notebookResearchString(cell.title),
1719
+ type: notebookResearchString(cell.type),
1720
+ sql: notebookResearchString(cell.sql),
1721
+ content: notebookResearchString(cell.content),
1722
+ source: notebookResearchString(cell.source),
1723
+ question: notebookResearchString(cell.question),
1724
+ intent: notebookResearchIntent(cell.intent),
1725
+ })),
1726
+ limit: 1000,
1727
+ });
1728
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
1729
+ res.end(serializeJSON({
1730
+ created: seeded.created.map(withNotebookResearchChecklist),
1731
+ createdCount: seeded.createdCount,
1732
+ skippedCount: seeded.skippedCount + invalidCellCount,
1733
+ limitApplied: seeded.limitApplied,
1734
+ }));
1735
+ }
1736
+ catch (error) {
1737
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1738
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1739
+ }
1740
+ finally {
1741
+ storage.close();
1742
+ }
1743
+ return;
1744
+ }
1745
+ if (req.method === 'POST' && path === '/api/notebook/research/source-coverage') {
1746
+ const storage = openNotebookResearchStorage();
1747
+ try {
1748
+ const body = await readJSON(req);
1749
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1750
+ if (!notebookPath) {
1751
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1752
+ res.end(serializeJSON({ error: 'notebookPath is required.' }));
1753
+ return;
1754
+ }
1755
+ const rawSourceCells = Array.isArray(body.sourceCells)
1756
+ ? body.sourceCells
1757
+ : Array.isArray(body.cells)
1758
+ ? body.cells
1759
+ : [];
1760
+ const sourceCells = rawSourceCells.flatMap((cell) => {
1761
+ if (!cell || typeof cell !== 'object' || Array.isArray(cell))
1762
+ return [];
1763
+ const record = cell;
1764
+ const nested = notebookResearchSourceCellPayload(record) ?? {};
1765
+ return [{
1766
+ ...nested,
1767
+ id: notebookResearchString(record.id) ?? nested.id,
1768
+ sourceCellId: notebookResearchString(record.sourceCellId) ?? nested.sourceCellId,
1769
+ cellId: notebookResearchString(record.cellId) ?? nested.cellId,
1770
+ name: notebookResearchString(record.name) ?? nested.name,
1771
+ sourceCellName: notebookResearchString(record.sourceCellName) ?? nested.sourceCellName,
1772
+ title: notebookResearchString(record.title) ?? nested.title,
1773
+ fingerprint: notebookResearchString(record.fingerprint) ?? nested.fingerprint,
1774
+ sourceCellFingerprint: notebookResearchString(record.sourceCellFingerprint) ?? nested.sourceCellFingerprint,
1775
+ sqlFingerprint: notebookResearchString(record.sqlFingerprint) ?? nested.sqlFingerprint,
1776
+ type: notebookResearchString(record.type) ?? nested.type,
1777
+ }];
1778
+ });
1779
+ const requestedIds = Array.isArray(body.sourceCellIds)
1780
+ ? body.sourceCellIds
1781
+ : rawSourceCells.length > 0
1782
+ ? sourceCells.map((cell) => notebookResearchSourceCellId(cell))
1783
+ : [];
1784
+ const sourceCellIds = Array.from(new Set(requestedIds
1785
+ .map((id) => notebookResearchString(id))
1786
+ .filter((id) => Boolean(id))));
1787
+ const requestedSourceCellCount = new Set([
1788
+ ...sourceCellIds,
1789
+ ...sourceCells
1790
+ .map((cell) => notebookResearchSourceCellId(cell))
1791
+ .filter((id) => Boolean(id)),
1792
+ ]).size;
1793
+ const limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
1794
+ ? Math.max(1, Math.min(10_000, Math.floor(body.limit)))
1795
+ : 10_000;
1796
+ const linkedRuns = storage.listLatestRunsBySourceCell({
1797
+ notebookPath,
1798
+ sourceCellIds,
1799
+ sourceCells,
1800
+ limit,
1801
+ });
1802
+ const missingRuns = storage.listLatestRunsForMissingSourceCells({
1803
+ notebookPath,
1804
+ sourceCellIds,
1805
+ sourceCells,
1806
+ limit,
1807
+ });
1808
+ const runs = [...linkedRuns, ...missingRuns];
1809
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1810
+ res.end(serializeJSON({
1811
+ runs: runs.map(withNotebookResearchChecklist),
1812
+ requestedCount: requestedSourceCellCount,
1813
+ matchedCount: runs.length,
1814
+ limitApplied: requestedSourceCellCount > limit,
1815
+ }));
1816
+ }
1817
+ catch (error) {
1818
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1819
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1820
+ }
1821
+ finally {
1822
+ storage.close();
1823
+ }
1824
+ return;
1825
+ }
1826
+ if (req.method === 'GET' && path === '/api/notebook/research/diagnostics') {
1827
+ const storage = openNotebookResearchStorage();
1828
+ try {
1829
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1830
+ res.end(serializeJSON(storage.getDiagnostics()));
1831
+ }
1832
+ catch (error) {
1833
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1834
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1835
+ }
1836
+ finally {
1837
+ storage.close();
1838
+ }
1839
+ return;
1840
+ }
1841
+ const notebookResearchMatch = /^\/api\/notebook\/research\/([^/]+)(?:\/([^/]+))?$/.exec(path);
1842
+ if (notebookResearchMatch) {
1843
+ const id = decodeURIComponent(notebookResearchMatch[1]);
1844
+ const action = notebookResearchMatch[2];
1845
+ const storage = openNotebookResearchStorage();
1846
+ try {
1847
+ const run = storage.getRun(id);
1848
+ if (!run) {
1849
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
1850
+ res.end(serializeJSON({ error: 'Notebook research run not found.' }));
1851
+ return;
1852
+ }
1853
+ if (req.method === 'GET' && !action) {
1854
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1855
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
1856
+ return;
1857
+ }
1858
+ if (req.method === 'PATCH' && !action) {
1859
+ const body = await readJSON(req);
1860
+ const sourceCell = notebookResearchSourceCellPayload(body);
1861
+ const sourceCellIdPatch = notebookResearchPatchString(body, 'sourceCellId');
1862
+ const sourceCellNamePatch = notebookResearchPatchString(body, 'sourceCellName');
1863
+ const sourceCellFingerprintPatch = notebookResearchPatchString(body, 'sourceCellFingerprint');
1864
+ const updated = storage.updateRun(id, {
1865
+ domain: notebookResearchString(body.domain),
1866
+ owner: notebookResearchString(body.owner),
1867
+ sourceCellId: sourceCellIdPatch !== undefined ? sourceCellIdPatch : notebookResearchSourceCellId(sourceCell),
1868
+ sourceCellName: sourceCellNamePatch !== undefined ? sourceCellNamePatch : notebookResearchSourceCellName(sourceCell),
1869
+ sourceCellFingerprint: sourceCellFingerprintPatch !== undefined ? sourceCellFingerprintPatch : notebookResearchSourceCellFingerprint(sourceCell),
1870
+ title: notebookResearchString(body.title),
1871
+ question: notebookResearchString(body.question),
1872
+ intent: notebookResearchIntent(body.intent),
1873
+ context: body.context,
1874
+ recommendation: notebookResearchString(body.recommendation),
1875
+ evidence: body.evidence,
1876
+ contextPackId: notebookResearchString(body.contextPackId),
1877
+ routeDecision: body.routeDecision,
1878
+ generatedSql: notebookResearchString(body.generatedSql),
1879
+ reviewedSql: notebookResearchString(body.reviewedSql),
1880
+ warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
1881
+ reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
1882
+ }) ?? run;
1883
+ const planned = storage.updateRun(updated.id, {
1884
+ researchPlan: buildNotebookResearchPlan({
1885
+ run: updated,
1886
+ evidence: updated.evidence,
1887
+ resultPreview: updated.resultPreview,
1888
+ previewError: updated.error,
1889
+ generatedSql: updated.generatedSql,
1890
+ reviewedSql: updated.reviewedSql,
1891
+ routeDecision: updated.routeDecision,
1892
+ }),
1893
+ }) ?? updated;
1894
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1895
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(planned) }));
1896
+ return;
1897
+ }
1898
+ if (req.method === 'POST' && action === 'run') {
1899
+ const body = await readJSON(req).catch(() => ({}));
1900
+ const sourceCell = notebookResearchSourceCellPayload(body);
1901
+ const updated = await runNotebookResearch(storage, run, {
1902
+ domain: notebookResearchString(body.domain),
1903
+ owner: notebookResearchString(body.owner),
1904
+ sourceCellFingerprint: notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell),
1905
+ question: notebookResearchString(body.question),
1906
+ intent: notebookResearchIntent(body.intent),
1907
+ context: body.context,
1908
+ generatedSql: notebookResearchString(body.generatedSql),
1909
+ reviewedSql: notebookResearchString(body.reviewedSql),
1910
+ });
1911
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1912
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
1913
+ return;
1914
+ }
1915
+ if (req.method === 'POST' && action === 'reuse-check') {
1916
+ const body = await readJSON(req).catch(() => ({}));
1917
+ const payload = await checkNotebookResearchReuse(storage, run, {
1918
+ domain: notebookResearchString(body.domain),
1919
+ owner: notebookResearchString(body.owner),
1920
+ });
1921
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1922
+ res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
1923
+ return;
1924
+ }
1925
+ if (req.method === 'POST' && action === 'promote-dql') {
1926
+ const body = await readJSON(req).catch(() => ({}));
1927
+ const payload = await promoteNotebookResearchToDql(storage, run, {
1928
+ domain: notebookResearchString(body.domain),
1929
+ owner: notebookResearchString(body.owner),
1930
+ provider: notebookResearchString(body.provider),
1931
+ tags: Array.isArray(body.tags) ? body.tags.filter((item) => typeof item === 'string') : undefined,
1932
+ });
1933
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1934
+ res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
1935
+ return;
1936
+ }
1937
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
1938
+ res.end(serializeJSON({ error: 'Unsupported notebook research operation.' }));
1939
+ }
1940
+ catch (error) {
1941
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1942
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1943
+ }
1944
+ finally {
1945
+ storage.close();
1946
+ }
1947
+ return;
1948
+ }
1022
1949
  // ── run snapshots (v0.11) ───────────────────────────────────────────────
1023
1950
  // Captures executed notebook state (query results + timings) in a
1024
1951
  // sibling `.run.json` so notebooks can show last-run output without
@@ -1304,49 +2231,51 @@ export async function startLocalServer(opts) {
1304
2231
  // ── Block library (list all blocks with metadata) ────────────────────
1305
2232
  if (req.method === 'GET' && path === '/api/blocks/library') {
1306
2233
  try {
1307
- const blocksDir = join(projectRoot, 'blocks');
1308
2234
  const blocks = [];
1309
- if (existsSync(blocksDir)) {
1310
- const scanDir = (dir) => {
1311
- for (const entry of readdirSync(dir, { withFileTypes: true })) {
1312
- if (entry.isDirectory()) {
1313
- scanDir(join(dir, entry.name));
1314
- }
1315
- else if (entry.name.endsWith('.dql')) {
1316
- const filePath = join(dir, entry.name);
1317
- const relPath = relative(projectRoot, filePath);
1318
- try {
1319
- const source = readFileSync(filePath, 'utf-8');
1320
- const stat = statSync(filePath);
1321
- // Quick regex parse for key block fields
1322
- const nameMatch = /block\s+"([^"]+)"/.exec(source);
1323
- const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
1324
- const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
1325
- const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
1326
- const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
1327
- const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
1328
- const parsedTags = tagsMatch
1329
- ? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
1330
- : [];
1331
- const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
1332
- blocks.push({
1333
- name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
1334
- domain: domainMatch?.[1] ?? 'uncategorized',
1335
- status: statusMatch?.[1] ?? 'draft',
1336
- owner: ownerMatch?.[1] ?? null,
1337
- tags: parsedTags,
1338
- path: relPath,
1339
- lastModified: stat.mtime.toISOString(),
1340
- description: descMatch?.[1] ?? '',
1341
- llmContext: llmMatch?.[1] ?? null,
1342
- });
1343
- }
1344
- catch { /* skip unreadable files */ }
2235
+ const seen = new Set();
2236
+ const scanDir = (dir) => {
2237
+ if (!existsSync(dir))
2238
+ return;
2239
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
2240
+ const filePath = join(dir, entry.name);
2241
+ if (entry.isDirectory()) {
2242
+ scanDir(filePath);
2243
+ }
2244
+ else if (entry.name.endsWith('.dql') && !seen.has(filePath)) {
2245
+ seen.add(filePath);
2246
+ const relPath = relative(projectRoot, filePath).replaceAll('\\', '/');
2247
+ try {
2248
+ const source = readFileSync(filePath, 'utf-8');
2249
+ const stat = statSync(filePath);
2250
+ // Quick regex parse for key block fields
2251
+ const nameMatch = /block\s+"([^"]+)"/.exec(source);
2252
+ const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
2253
+ const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
2254
+ const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
2255
+ const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
2256
+ const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
2257
+ const parsedTags = tagsMatch
2258
+ ? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
2259
+ : [];
2260
+ const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
2261
+ blocks.push({
2262
+ name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
2263
+ domain: (domainMatch?.[1] ?? inferBlockStudioPathDomain(relPath)) || 'uncategorized',
2264
+ status: statusMatch?.[1] ?? 'draft',
2265
+ owner: ownerMatch?.[1] ?? null,
2266
+ tags: parsedTags,
2267
+ path: relPath,
2268
+ lastModified: stat.mtime.toISOString(),
2269
+ description: descMatch?.[1] ?? '',
2270
+ llmContext: llmMatch?.[1] ?? null,
2271
+ });
1345
2272
  }
2273
+ catch { /* skip unreadable files */ }
1346
2274
  }
1347
- };
1348
- scanDir(blocksDir);
1349
- }
2275
+ }
2276
+ };
2277
+ scanDir(join(projectRoot, 'blocks'));
2278
+ scanDir(join(projectRoot, 'domains'));
1350
2279
  blocks.sort((a, b) => new Date(b.lastModified).getTime() - new Date(a.lastModified).getTime());
1351
2280
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1352
2281
  res.end(serializeJSON({ blocks }));
@@ -1555,20 +2484,216 @@ export async function startLocalServer(opts) {
1555
2484
  res.end(serializeJSON({ error: 'source is required' }));
1556
2485
  return;
1557
2486
  }
1558
- const result = await certifyBlockStudioSource(source, blockPath);
2487
+ const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise !== false });
1559
2488
  const blockers = Array.from(new Set(result.checklist.blockers));
1560
2489
  if (!result.certification.certified || blockers.length > 0) {
1561
2490
  res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
1562
2491
  res.end(serializeJSON({ ok: false, ...result, blockers }));
1563
2492
  return;
1564
2493
  }
1565
- if (blockPath)
1566
- setBlockStudioStatus(projectRoot, blockPath, 'certified');
2494
+ let certifiedPayload = null;
2495
+ const certifiedSource = setBlockStudioStatusInSource(source, 'certified');
2496
+ if (blockPath) {
2497
+ const normalizedBlockPath = normalize(blockPath).replace(/^\/+/, '');
2498
+ const parsed = parseBlockSourceMetadata(certifiedSource);
2499
+ if (isDraftBlockPath(normalizedBlockPath)) {
2500
+ const savedPath = saveBlockStudioArtifacts(projectRoot, {
2501
+ currentPath: normalizedBlockPath,
2502
+ source: certifiedSource,
2503
+ name: parsed.name,
2504
+ domain: parsed.domain,
2505
+ description: parsed.description,
2506
+ owner: parsed.owner,
2507
+ tags: parsed.tags,
2508
+ });
2509
+ certifiedPayload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
2510
+ }
2511
+ else {
2512
+ setBlockStudioStatus(projectRoot, normalizedBlockPath, 'certified');
2513
+ certifiedPayload = openBlockStudioDocument(projectRoot, normalizedBlockPath, semanticLayer);
2514
+ }
2515
+ }
1567
2516
  await refreshLocalMetadataCatalog(projectRoot);
1568
2517
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1569
- res.end(serializeJSON({ ok: true, status: 'certified', ...result }));
2518
+ res.end(serializeJSON({
2519
+ ok: true,
2520
+ status: 'certified',
2521
+ ...result,
2522
+ path: certifiedPayload?.path ?? blockPath ?? null,
2523
+ source: certifiedPayload?.source ?? certifiedSource,
2524
+ metadata: certifiedPayload?.metadata,
2525
+ companionPath: certifiedPayload?.companionPath ?? null,
2526
+ validation: certifiedPayload?.validation ?? result.validation,
2527
+ }));
2528
+ }
2529
+ catch (error) {
2530
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2531
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2532
+ }
2533
+ return;
2534
+ }
2535
+ if (req.method === 'POST' && path === '/api/block-studio/ai-imports') {
2536
+ try {
2537
+ const body = await readJSON(req);
2538
+ const session = await createDqlGenerationSessionFromBody(body);
2539
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2540
+ res.end(serializeJSON(session));
2541
+ }
2542
+ catch (error) {
2543
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2544
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2545
+ }
2546
+ return;
2547
+ }
2548
+ if (req.method === 'POST' && path === '/api/block-studio/match-sql') {
2549
+ try {
2550
+ const body = await readJSON(req);
2551
+ if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
2552
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2553
+ res.end(serializeJSON({ error: 'Missing SQL in request body.' }));
2554
+ return;
2555
+ }
2556
+ const sql = body.sql.trim();
2557
+ const match = await matchSqlForDqlReuse({
2558
+ sql,
2559
+ sourcePath: typeof body.sourcePath === 'string' ? body.sourcePath : 'pasted.sql',
2560
+ name: typeof body.name === 'string' ? body.name : 'SQL match preview',
2561
+ domain: typeof body.domain === 'string' ? body.domain : 'imported',
2562
+ owner: typeof body.owner === 'string' ? body.owner : 'analytics',
2563
+ });
2564
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2565
+ res.end(serializeJSON(match));
2566
+ }
2567
+ catch (error) {
2568
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2569
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2570
+ }
2571
+ return;
2572
+ }
2573
+ const aiImportPathMatch = path.match(/^\/api\/block-studio\/ai-imports\/([^/]+)(?:\/candidates\/([^/]+)(?:\/(preview|certify))?)?$/);
2574
+ if (aiImportPathMatch) {
2575
+ const importId = decodeURIComponent(aiImportPathMatch[1]);
2576
+ const candidateId = aiImportPathMatch[2] ? decodeURIComponent(aiImportPathMatch[2]) : null;
2577
+ const action = aiImportPathMatch[3] ?? null;
2578
+ try {
2579
+ if (req.method === 'GET' && !candidateId) {
2580
+ const session = loadDqlGenerationSession(importId);
2581
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2582
+ res.end(serializeJSON(session));
2583
+ return;
2584
+ }
2585
+ if (req.method === 'PATCH' && candidateId && !action) {
2586
+ const body = await readJSON(req);
2587
+ const candidate = updateBlockStudioImportCandidate(projectRoot, importId, candidateId, {
2588
+ name: typeof body.name === 'string' ? body.name : undefined,
2589
+ domain: typeof body.domain === 'string' ? body.domain : undefined,
2590
+ description: typeof body.description === 'string' ? body.description : undefined,
2591
+ owner: typeof body.owner === 'string' ? body.owner : undefined,
2592
+ tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
2593
+ terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
2594
+ pattern: typeof body.pattern === 'string' ? body.pattern : undefined,
2595
+ grain: typeof body.grain === 'string' ? body.grain : undefined,
2596
+ entities: Array.isArray(body.entities) ? body.entities.map(String) : undefined,
2597
+ outputs: Array.isArray(body.outputs) ? body.outputs.map(String) : undefined,
2598
+ dimensions: Array.isArray(body.dimensions) ? body.dimensions.map(String) : undefined,
2599
+ allowedFilters: Array.isArray(body.allowedFilters) ? body.allowedFilters.map(String) : undefined,
2600
+ parameterPolicy: Array.isArray(body.parameterPolicy) ? body.parameterPolicy : undefined,
2601
+ filterBindings: Array.isArray(body.filterBindings) ? body.filterBindings : undefined,
2602
+ sourceSystems: Array.isArray(body.sourceSystems) ? body.sourceSystems.map(String) : undefined,
2603
+ replacementFor: Array.isArray(body.replacementFor) ? body.replacementFor.map(String) : undefined,
2604
+ reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
2605
+ sql: typeof body.sql === 'string' ? body.sql : undefined,
2606
+ llmContext: typeof body.llmContext === 'string' ? body.llmContext : undefined,
2607
+ });
2608
+ const validated = validateImportCandidate(candidate);
2609
+ const savedDraft = saveDqlGenerationDraft(importId, validated);
2610
+ writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
2611
+ await refreshLocalMetadataCatalog(projectRoot);
2612
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2613
+ res.end(serializeJSON(savedDraft));
2614
+ return;
2615
+ }
2616
+ if (req.method === 'POST' && candidateId && action === 'preview') {
2617
+ const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
2618
+ const preview = await runBlockStudioPreviewSource(candidate.dqlSource);
2619
+ const next = saveDqlGenerationDraft(importId, {
2620
+ ...candidate,
2621
+ preview,
2622
+ validation: validateBlockStudioSource(candidate.dqlSource, semanticLayer),
2623
+ });
2624
+ writeBlockStudioImportCandidate(projectRoot, importId, next);
2625
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2626
+ res.end(serializeJSON(next));
2627
+ return;
2628
+ }
2629
+ if (req.method === 'POST' && candidateId && action === 'certify') {
2630
+ const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
2631
+ const readiness = validateImportCandidateForSave(candidate);
2632
+ if (readiness.errors.length > 0) {
2633
+ const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
2634
+ writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
2635
+ res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
2636
+ res.end(serializeJSON({
2637
+ error: readiness.errors.join(' '),
2638
+ candidate: savedDraft,
2639
+ diagnostics: savedDraft.validation?.diagnostics ?? [],
2640
+ }));
2641
+ return;
2642
+ }
2643
+ const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
2644
+ const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: true });
2645
+ const blockers = Array.from(new Set(certification.checklist.blockers));
2646
+ if (!certification.certification.certified || blockers.length > 0) {
2647
+ const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
2648
+ writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
2649
+ res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
2650
+ res.end(serializeJSON({ ok: false, candidate: savedDraft, blockers, ...certification }));
2651
+ return;
2652
+ }
2653
+ const savedPath = saveBlockStudioArtifacts(projectRoot, {
2654
+ currentPath: readiness.candidate.savedPath,
2655
+ source: certifiedSource,
2656
+ name: readiness.candidate.name,
2657
+ domain: readiness.candidate.domain,
2658
+ description: readiness.candidate.description,
2659
+ owner: readiness.candidate.owner,
2660
+ tags: readiness.candidate.tags,
2661
+ lineage: readiness.candidate.lineage.sourceTables,
2662
+ importMeta: {
2663
+ importId,
2664
+ candidateId,
2665
+ sourceKind: readiness.candidate.sourceKind,
2666
+ sourcePath: readiness.candidate.sourcePath,
2667
+ },
2668
+ });
2669
+ const next = {
2670
+ ...readiness.candidate,
2671
+ dqlSource: certifiedSource,
2672
+ reviewStatus: 'saved',
2673
+ savedPath,
2674
+ validation: validateBlockStudioSource(certifiedSource, semanticLayer),
2675
+ generationMode: readiness.candidate.generationMode ?? 'deterministic',
2676
+ generationProvider: readiness.candidate.generationProvider ?? 'local-deterministic',
2677
+ llmContext: readiness.candidate.llmContext ?? deterministicDqlGenerationContext(readiness.candidate, readiness.candidate.evidence ?? []),
2678
+ evidence: readiness.candidate.evidence ?? [],
2679
+ draftSave: readiness.candidate.draftSave ?? { status: 'pending' },
2680
+ };
2681
+ writeBlockStudioImportCandidate(projectRoot, importId, next);
2682
+ await refreshLocalMetadataCatalog(projectRoot);
2683
+ const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
2684
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2685
+ res.end(serializeJSON({ candidate: next, block: payload, certification }));
2686
+ return;
2687
+ }
2688
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
2689
+ res.end(serializeJSON({ error: 'Unsupported AI import operation.' }));
1570
2690
  }
1571
2691
  catch (error) {
2692
+ if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
2693
+ res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
2694
+ res.end(serializeJSON({ error: 'Block already exists' }));
2695
+ return;
2696
+ }
1572
2697
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1573
2698
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1574
2699
  }
@@ -1708,6 +2833,8 @@ export async function startLocalServer(opts) {
1708
2833
  description: typeof body.description === 'string' ? body.description : undefined,
1709
2834
  owner: typeof body.owner === 'string' ? body.owner : undefined,
1710
2835
  tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
2836
+ terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
2837
+ reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
1711
2838
  sql: typeof body.sql === 'string' ? body.sql : undefined,
1712
2839
  reviewStatus,
1713
2840
  });
@@ -1844,6 +2971,55 @@ export async function startLocalServer(opts) {
1844
2971
  }
1845
2972
  return;
1846
2973
  }
2974
+ if (req.method === 'GET' && path === '/api/semantic-layer/diagnostics') {
2975
+ try {
2976
+ const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
2977
+ semanticLayer,
2978
+ semanticErrors: semanticLayerErrors,
2979
+ semanticConfig,
2980
+ detectedProvider: semanticDetectedProvider,
2981
+ lastSyncTime: semanticLastSyncTime,
2982
+ });
2983
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2984
+ res.end(serializeJSON(diagnostics));
2985
+ }
2986
+ catch (error) {
2987
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2988
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2989
+ }
2990
+ return;
2991
+ }
2992
+ if (req.method === 'POST' && path === '/api/semantic-layer/reload') {
2993
+ try {
2994
+ const semanticConnection = connection;
2995
+ const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
2996
+ ? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
2997
+ : undefined;
2998
+ const refreshed = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
2999
+ semanticLayer = refreshed.layer;
3000
+ semanticLayerErrors = refreshed.errors;
3001
+ semanticDetectedProvider = refreshed.detectedProvider;
3002
+ semanticLastSyncTime = refreshed.layer ? new Date().toISOString() : null;
3003
+ semanticImportManifest = loadSemanticImportManifest(projectRoot);
3004
+ const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
3005
+ semanticLayer,
3006
+ semanticErrors: semanticLayerErrors,
3007
+ semanticConfig,
3008
+ detectedProvider: semanticDetectedProvider,
3009
+ lastSyncTime: semanticLastSyncTime,
3010
+ });
3011
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3012
+ res.end(serializeJSON({
3013
+ ok: Boolean(refreshed.layer),
3014
+ ...diagnostics,
3015
+ }));
3016
+ }
3017
+ catch (error) {
3018
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3019
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3020
+ }
3021
+ return;
3022
+ }
1847
3023
  if (req.method === 'GET' && path === '/api/block-studio/open') {
1848
3024
  try {
1849
3025
  const relativePath = url.searchParams.get('path');
@@ -1908,8 +3084,9 @@ export async function startLocalServer(opts) {
1908
3084
  res.end(serializeJSON({ error: 'Block name is required.' }));
1909
3085
  return;
1910
3086
  }
1911
- const savedPath = saveBlockStudioArtifacts(projectRoot, {
1912
- currentPath: typeof body.path === 'string' ? body.path : undefined,
3087
+ const currentPath = typeof body.path === 'string' ? body.path : undefined;
3088
+ const saveOptions = {
3089
+ currentPath,
1913
3090
  source,
1914
3091
  name: metadata.name,
1915
3092
  domain: metadata.domain,
@@ -1925,7 +3102,13 @@ export async function startLocalServer(opts) {
1925
3102
  sourcePath: metadata.sourcePath,
1926
3103
  }
1927
3104
  : undefined,
1928
- });
3105
+ };
3106
+ const savedPath = isDraftBlockPath(currentPath)
3107
+ ? saveBlockStudioDraftArtifacts(projectRoot, {
3108
+ ...saveOptions,
3109
+ stableSuffix: metadata.candidateId,
3110
+ })
3111
+ : saveBlockStudioArtifacts(projectRoot, saveOptions);
1929
3112
  await refreshLocalMetadataCatalog(projectRoot);
1930
3113
  const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
1931
3114
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -2051,7 +3234,7 @@ export async function startLocalServer(opts) {
2051
3234
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2052
3235
  res.end(serializeJSON({
2053
3236
  available: false,
2054
- provider: projectConfig.semanticLayer?.provider ?? semanticDetectedProvider ?? null,
3237
+ provider: semanticConfig?.provider ?? semanticDetectedProvider ?? null,
2055
3238
  errors: semanticLayerErrors,
2056
3239
  metrics: [],
2057
3240
  measures: [],
@@ -2180,9 +3363,9 @@ export async function startLocalServer(opts) {
2180
3363
  owner: q.owner ?? null,
2181
3364
  source: q.source ?? null,
2182
3365
  }));
2183
- const provider = projectConfig.semanticLayer?.provider ?? semanticDetectedProvider ?? 'dql';
3366
+ const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
2184
3367
  const dbtExecutionReady = provider === 'dbt'
2185
- ? hasDbtSemanticManifest(projectRoot, projectConfig.semanticLayer?.projectPath)
3368
+ ? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
2186
3369
  : false;
2187
3370
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2188
3371
  res.end(serializeJSON({
@@ -2731,9 +3914,35 @@ export async function startLocalServer(opts) {
2731
3914
  res.end();
2732
3915
  return;
2733
3916
  }
2734
- if (req.method === 'POST' && path === '/api/query') {
3917
+ if (req.method === 'POST' && path === '/api/ai/sql-draft/preview') {
2735
3918
  try {
2736
3919
  const body = await readJSON(req);
3920
+ if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
3921
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
3922
+ res.end(serializeJSON({ error: 'Missing SQL in request body.' }));
3923
+ return;
3924
+ }
3925
+ const previewSql = buildAgentPreviewSql(body.sql);
3926
+ const result = await executeLocalSqlForStoredResult(previewSql);
3927
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3928
+ res.end(serializeJSON({ ok: true, result }));
3929
+ }
3930
+ catch (error) {
3931
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3932
+ res.end(serializeJSON({
3933
+ ok: false,
3934
+ error: error instanceof Error ? error.message : String(error),
3935
+ }));
3936
+ }
3937
+ return;
3938
+ }
3939
+ if (req.method === 'POST' && path === '/api/query') {
3940
+ let body;
3941
+ let execContext = null;
3942
+ const start = Date.now();
3943
+ try {
3944
+ body = await readJSON(req);
3945
+ execContext = notebookExecutionContext(body.executionContext);
2737
3946
  if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
2738
3947
  res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2739
3948
  res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
@@ -2756,7 +3965,26 @@ export async function startLocalServer(opts) {
2756
3965
  const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
2757
3966
  assertAppAccess({ app, domain, level: 'execute' });
2758
3967
  const result = await executor.executeQuery(prepared.sql, Array.isArray(body.sqlParams) ? body.sqlParams : [], runtimeVariables(body.variables && typeof body.variables === 'object' ? body.variables : {}), prepared.connection);
2759
- const payload = serializeJSON(normalizeQueryResult(result, semantic.semanticRefs));
3968
+ const normalized = normalizeQueryResult(result, semantic.semanticRefs);
3969
+ if (execContext) {
3970
+ recordNotebookQueryRun(projectRoot, {
3971
+ notebookPath: execContext.notebookPath,
3972
+ cellId: execContext.cellId,
3973
+ cellName: execContext.cellName,
3974
+ researchRunId: execContext.researchRunId,
3975
+ source: execContext.source ?? 'notebook_sql_cell',
3976
+ status: 'success',
3977
+ rowCount: normalized.rowCount ?? normalized.rows.length,
3978
+ durationMs: Date.now() - start,
3979
+ sql: body.sql,
3980
+ });
3981
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
3982
+ status: 'success',
3983
+ resultPreview: normalized,
3984
+ sql: body.sql,
3985
+ });
3986
+ }
3987
+ const payload = serializeJSON(normalized);
2760
3988
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2761
3989
  res.end(payload);
2762
3990
  }
@@ -2775,6 +4003,24 @@ export async function startLocalServer(opts) {
2775
4003
  }));
2776
4004
  return;
2777
4005
  }
4006
+ if (execContext) {
4007
+ recordNotebookQueryRun(projectRoot, {
4008
+ notebookPath: execContext.notebookPath,
4009
+ cellId: execContext.cellId,
4010
+ cellName: execContext.cellName,
4011
+ researchRunId: execContext.researchRunId,
4012
+ source: execContext.source ?? 'notebook_sql_cell',
4013
+ status: 'error',
4014
+ durationMs: Date.now() - start,
4015
+ errorCode: error instanceof Error ? error.message : String(error),
4016
+ sql: typeof body?.sql === 'string' ? body.sql : undefined,
4017
+ });
4018
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
4019
+ status: 'error',
4020
+ error: error instanceof Error ? error.message : String(error),
4021
+ sql: typeof body?.sql === 'string' ? body.sql : undefined,
4022
+ });
4023
+ }
2778
4024
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2779
4025
  res.end(serializeJSON({
2780
4026
  columns: [],
@@ -3116,6 +4362,25 @@ export async function startLocalServer(opts) {
3116
4362
  }
3117
4363
  return;
3118
4364
  }
4365
+ if (req.method === 'GET' && path.startsWith('/api/lineage/business-360/')) {
4366
+ const rawNodeId = decodeURIComponent(path.slice('/api/lineage/business-360/'.length));
4367
+ try {
4368
+ const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
4369
+ const result = queryBusiness360(graph, rawNodeId);
4370
+ if (!result) {
4371
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
4372
+ res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
4373
+ return;
4374
+ }
4375
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
4376
+ res.end(serializeJSON(result));
4377
+ }
4378
+ catch (error) {
4379
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
4380
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
4381
+ }
4382
+ return;
4383
+ }
3119
4384
  if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
3120
4385
  const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
3121
4386
  try {
@@ -3281,8 +4546,12 @@ export async function startLocalServer(opts) {
3281
4546
  return;
3282
4547
  }
3283
4548
  if (req.method === 'POST' && path === '/api/notebook/execute') {
4549
+ let body;
4550
+ let execContext = null;
4551
+ const start = Date.now();
3284
4552
  try {
3285
- const body = await readJSON(req);
4553
+ body = await readJSON(req);
4554
+ execContext = notebookExecutionContext(body.executionContext);
3286
4555
  const cell = normalizeNotebookCell(body.cell);
3287
4556
  if (!cell) {
3288
4557
  res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -3305,6 +4574,26 @@ export async function startLocalServer(opts) {
3305
4574
  const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
3306
4575
  assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
3307
4576
  const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
4577
+ const normalized = normalizeQueryResult(rawResult);
4578
+ if (execContext) {
4579
+ recordNotebookQueryRun(projectRoot, {
4580
+ notebookPath: execContext.notebookPath,
4581
+ cellId: execContext.cellId ?? cell.id,
4582
+ cellName: execContext.cellName ?? plan.title ?? resolved.blockName,
4583
+ researchRunId: execContext.researchRunId,
4584
+ source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_cell'),
4585
+ status: 'success',
4586
+ rowCount: normalized.rowCount ?? normalized.rows.length,
4587
+ durationMs: Date.now() - start,
4588
+ sql: plan.sql,
4589
+ objectKey: resolved.blockPath,
4590
+ });
4591
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
4592
+ status: 'success',
4593
+ resultPreview: normalized,
4594
+ sql: plan.sql,
4595
+ });
4596
+ }
3308
4597
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3309
4598
  res.end(serializeJSON({
3310
4599
  cellType: cell.type,
@@ -3313,10 +4602,26 @@ export async function startLocalServer(opts) {
3313
4602
  blockPath: resolved.blockPath,
3314
4603
  chartConfig: plan.chartConfig,
3315
4604
  tests: plan.tests,
3316
- result: normalizeQueryResult(rawResult),
4605
+ result: normalized,
3317
4606
  }));
3318
4607
  }
3319
4608
  catch (error) {
4609
+ if (execContext) {
4610
+ recordNotebookQueryRun(projectRoot, {
4611
+ notebookPath: execContext.notebookPath,
4612
+ cellId: execContext.cellId,
4613
+ cellName: execContext.cellName,
4614
+ researchRunId: execContext.researchRunId,
4615
+ source: execContext.source ?? 'notebook_cell',
4616
+ status: 'error',
4617
+ durationMs: Date.now() - start,
4618
+ errorCode: error instanceof Error ? error.message : String(error),
4619
+ });
4620
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
4621
+ status: 'error',
4622
+ error: error instanceof Error ? error.message : String(error),
4623
+ });
4624
+ }
3320
4625
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3321
4626
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3322
4627
  }
@@ -3845,6 +5150,245 @@ export function prepareLocalExecution(sql, connection, projectRoot, projectConfi
3845
5150
  connection: normalizedConnection,
3846
5151
  };
3847
5152
  }
5153
+ export function dashboardRuntimeVariables(dashboard, overrides = {}) {
5154
+ const variables = {};
5155
+ for (const param of dashboard.params ?? []) {
5156
+ if (param.default !== undefined)
5157
+ variables[param.id] = param.default;
5158
+ }
5159
+ for (const filter of dashboard.filters ?? []) {
5160
+ if (filter.default !== undefined)
5161
+ variables[filter.id] = filter.default;
5162
+ }
5163
+ return { ...variables, ...overrides };
5164
+ }
5165
+ export function applyDashboardFiltersToBlockExecution(input) {
5166
+ const variables = { ...input.variables };
5167
+ const sqlParams = [...input.sqlParams];
5168
+ const appliedFilters = [];
5169
+ const skippedFilters = [];
5170
+ const clauses = [];
5171
+ let nextPosition = sqlParams.reduce((max, param) => Math.max(max, param.position), 0);
5172
+ for (const filter of input.dashboard.filters ?? []) {
5173
+ const value = dashboardFilterValue(filter, variables);
5174
+ if (isEmptyDashboardFilterValue(value)) {
5175
+ skippedFilters.push({ filter: filter.id, reason: 'no value supplied' });
5176
+ continue;
5177
+ }
5178
+ const paramNames = bindDashboardFilterToExistingParams(filter, value, input.block, sqlParams, variables);
5179
+ if (paramNames.length > 0) {
5180
+ appliedFilters.push({
5181
+ filter: filter.id,
5182
+ mode: 'parameter',
5183
+ paramNames,
5184
+ });
5185
+ continue;
5186
+ }
5187
+ const binding = resolveDashboardFilterBinding(filter, input.block);
5188
+ if (!binding) {
5189
+ skippedFilters.push({ filter: filter.id, reason: `block "${input.block.name}" does not declare a compatible filter binding` });
5190
+ continue;
5191
+ }
5192
+ const expression = dashboardFilterExpression(binding);
5193
+ if (!expression) {
5194
+ skippedFilters.push({ filter: filter.id, reason: `filter binding "${binding}" is not safe for runtime predicate injection` });
5195
+ continue;
5196
+ }
5197
+ const predicate = buildDashboardFilterPredicate({
5198
+ expression,
5199
+ filterId: filter.id,
5200
+ filterType: filter.type,
5201
+ value,
5202
+ params: sqlParams,
5203
+ nextPosition: () => {
5204
+ nextPosition += 1;
5205
+ return nextPosition;
5206
+ },
5207
+ variables,
5208
+ });
5209
+ if (!predicate) {
5210
+ skippedFilters.push({ filter: filter.id, reason: 'filter value could not be converted into a predicate' });
5211
+ continue;
5212
+ }
5213
+ clauses.push(predicate);
5214
+ appliedFilters.push({
5215
+ filter: filter.id,
5216
+ binding,
5217
+ mode: 'predicate',
5218
+ paramNames: sqlParams
5219
+ .filter((param) => param.name.startsWith(`__dashboard_filter_${normalizeDashboardFilterName(filter.id)}_`))
5220
+ .map((param) => param.name),
5221
+ });
5222
+ }
5223
+ if (clauses.length === 0) {
5224
+ return { sql: input.sql, sqlParams, variables, appliedFilters, skippedFilters };
5225
+ }
5226
+ return {
5227
+ sql: `SELECT * FROM (${stripSqlTerminator(input.sql)}) _dql_filter WHERE ${clauses.join(' AND ')}`,
5228
+ sqlParams,
5229
+ variables,
5230
+ appliedFilters,
5231
+ skippedFilters,
5232
+ };
5233
+ }
5234
+ function dashboardFilterValue(filter, variables) {
5235
+ if (Object.prototype.hasOwnProperty.call(variables, filter.id))
5236
+ return variables[filter.id];
5237
+ if (filter.bindsTo && Object.prototype.hasOwnProperty.call(variables, filter.bindsTo))
5238
+ return variables[filter.bindsTo];
5239
+ return filter.default;
5240
+ }
5241
+ function isEmptyDashboardFilterValue(value) {
5242
+ if (value === undefined || value === null)
5243
+ return true;
5244
+ if (typeof value === 'string' && value.trim() === '')
5245
+ return true;
5246
+ if (Array.isArray(value) && value.length === 0)
5247
+ return true;
5248
+ return false;
5249
+ }
5250
+ function bindDashboardFilterToExistingParams(filter, value, block, sqlParams, variables) {
5251
+ const availableParamNames = new Set(sqlParams.map((param) => param.name));
5252
+ const declaredParamNames = new Set((block.parameterPolicy ?? []).map((entry) => entry.name));
5253
+ const applied = [];
5254
+ const directCandidates = uniqueDashboardStrings([
5255
+ filter.id,
5256
+ filter.bindsTo ?? '',
5257
+ normalizeDashboardFilterName(filter.id),
5258
+ filter.bindsTo ? normalizeDashboardFilterName(filter.bindsTo) : '',
5259
+ ]);
5260
+ for (const name of directCandidates) {
5261
+ if (!name || !availableParamNames.has(name))
5262
+ continue;
5263
+ variables[name] = value;
5264
+ applied.push(name);
5265
+ }
5266
+ if (applied.length > 0)
5267
+ return applied;
5268
+ const range = dashboardRangeValue(value);
5269
+ if (range) {
5270
+ const baseNames = uniqueDashboardStrings([
5271
+ filter.id,
5272
+ filter.id.replace(/_?range$/i, ''),
5273
+ filter.bindsTo ?? '',
5274
+ filter.bindsTo ? filter.bindsTo.replace(/_?range$/i, '') : '',
5275
+ ].map(normalizeDashboardFilterName));
5276
+ for (const base of baseNames) {
5277
+ const pairs = [
5278
+ [`${base}_start`, `${base}_end`],
5279
+ [`${base}_from`, `${base}_to`],
5280
+ [`start_${base}`, `end_${base}`],
5281
+ ];
5282
+ for (const [startName, endName] of pairs) {
5283
+ if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
5284
+ variables[startName] = range.start;
5285
+ variables[endName] = range.end;
5286
+ return [startName, endName];
5287
+ }
5288
+ }
5289
+ }
5290
+ for (const [startName, endName] of [['start_date', 'end_date'], ['date_start', 'date_end'], ['season_start', 'season_end'], ['year_start', 'year_end']]) {
5291
+ if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
5292
+ variables[startName] = range.start;
5293
+ variables[endName] = range.end;
5294
+ return [startName, endName];
5295
+ }
5296
+ }
5297
+ }
5298
+ for (const name of declaredParamNames) {
5299
+ if (!availableParamNames.has(name))
5300
+ continue;
5301
+ const normalized = normalizeDashboardFilterName(name);
5302
+ if (normalized === normalizeDashboardFilterName(filter.id) || normalized === normalizeDashboardFilterName(filter.bindsTo ?? '')) {
5303
+ variables[name] = value;
5304
+ return [name];
5305
+ }
5306
+ }
5307
+ return [];
5308
+ }
5309
+ function resolveDashboardFilterBinding(filter, block) {
5310
+ const candidates = uniqueDashboardStrings([filter.id, filter.bindsTo ?? '']).map(normalizeDashboardFilterName);
5311
+ for (const entry of block.filterBindings ?? []) {
5312
+ if (candidates.includes(normalizeDashboardFilterName(entry.filter)))
5313
+ return entry.binding;
5314
+ }
5315
+ if (filter.bindsTo && (block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.bindsTo ?? ''))) {
5316
+ return filter.bindsTo;
5317
+ }
5318
+ if ((block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.id))) {
5319
+ return filter.id;
5320
+ }
5321
+ return null;
5322
+ }
5323
+ function dashboardFilterExpression(binding) {
5324
+ const trimmed = binding.trim();
5325
+ const yearMatch = trimmed.match(/^([A-Za-z_][A-Za-z0-9_.]*)\.year$/i);
5326
+ if (yearMatch) {
5327
+ const base = dashboardOutputColumn(yearMatch[1]);
5328
+ return base ? `EXTRACT(YEAR FROM _dql_filter.${base})` : null;
5329
+ }
5330
+ const column = dashboardOutputColumn(trimmed);
5331
+ return column ? `_dql_filter.${column}` : null;
5332
+ }
5333
+ function dashboardOutputColumn(binding) {
5334
+ const cleaned = binding.replace(/[`"[\]]/g, '').trim();
5335
+ if (!/^[A-Za-z_][A-Za-z0-9_.]*$/.test(cleaned))
5336
+ return null;
5337
+ const column = cleaned.split('.').filter(Boolean).at(-1);
5338
+ return column && /^[A-Za-z_][A-Za-z0-9_]*$/.test(column) ? column : null;
5339
+ }
5340
+ function buildDashboardFilterPredicate(input) {
5341
+ const range = input.filterType === 'daterange' ? dashboardRangeValue(input.value) : null;
5342
+ if (range) {
5343
+ const start = addDashboardFilterParam(input, 'start', range.start);
5344
+ const end = addDashboardFilterParam(input, 'end', range.end);
5345
+ return `${input.expression} BETWEEN $${start.position} AND $${end.position}`;
5346
+ }
5347
+ const values = Array.isArray(input.value) ? input.value.filter((item) => !isEmptyDashboardFilterValue(item)) : [input.value];
5348
+ if (values.length === 0)
5349
+ return null;
5350
+ if (values.length === 1) {
5351
+ const param = addDashboardFilterParam(input, 'value', values[0]);
5352
+ return `${input.expression} = $${param.position}`;
5353
+ }
5354
+ const placeholders = values.map((value, index) => {
5355
+ const param = addDashboardFilterParam(input, `value_${index + 1}`, value);
5356
+ return `$${param.position}`;
5357
+ });
5358
+ return `${input.expression} IN (${placeholders.join(', ')})`;
5359
+ }
5360
+ function addDashboardFilterParam(input, suffix, value) {
5361
+ const name = `__dashboard_filter_${normalizeDashboardFilterName(input.filterId)}_${suffix}`;
5362
+ const uniqueName = input.variables[name] === undefined && !input.params.some((param) => param.name === name)
5363
+ ? name
5364
+ : `${name}_${input.params.length + 1}`;
5365
+ const param = { name: uniqueName, position: input.nextPosition() };
5366
+ input.params.push(param);
5367
+ input.variables[uniqueName] = value;
5368
+ return param;
5369
+ }
5370
+ function dashboardRangeValue(value) {
5371
+ if (Array.isArray(value) && value.length >= 2 && !isEmptyDashboardFilterValue(value[0]) && !isEmptyDashboardFilterValue(value[1])) {
5372
+ return { start: value[0], end: value[1] };
5373
+ }
5374
+ if (value && typeof value === 'object') {
5375
+ const record = value;
5376
+ const start = record.start ?? record.from ?? record.min;
5377
+ const end = record.end ?? record.to ?? record.max;
5378
+ if (!isEmptyDashboardFilterValue(start) && !isEmptyDashboardFilterValue(end))
5379
+ return { start, end };
5380
+ }
5381
+ return null;
5382
+ }
5383
+ function normalizeDashboardFilterName(value) {
5384
+ return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
5385
+ }
5386
+ function stripSqlTerminator(sql) {
5387
+ return sql.trim().replace(/;\s*$/, '');
5388
+ }
5389
+ function uniqueDashboardStrings(values) {
5390
+ return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
5391
+ }
3848
5392
  export function resolveDbtMacrosForExecution(sql, projectRoot, projectConfig = {}) {
3849
5393
  if (!/\{\{\s*(?:ref|source)\s*\(/i.test(sql))
3850
5394
  return sql;
@@ -4462,9 +6006,9 @@ function buildDatabaseSchemaTree(projectRoot, executor, connection) {
4462
6006
  return databaseNodes;
4463
6007
  })();
4464
6008
  }
4465
- function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
6009
+ export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
4466
6010
  const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
4467
- if (!normalizedPath.startsWith('blocks/')) {
6011
+ if (!isBlockStudioBlockPath(normalizedPath)) {
4468
6012
  throw new Error('Invalid block path');
4469
6013
  }
4470
6014
  const absPath = join(projectRoot, normalizedPath);
@@ -4479,7 +6023,7 @@ function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
4479
6023
  const metadata = {
4480
6024
  name: parsedMetadata.name || companion?.name || fileName,
4481
6025
  path: normalizedPath,
4482
- domain: parsedMetadata.domain || companion?.domain || normalizedPath.split('/').slice(1, -1).join('/') || 'uncategorized',
6026
+ domain: parsedMetadata.domain || companion?.domain || inferBlockStudioPathDomain(normalizedPath) || 'uncategorized',
4483
6027
  description: parsedMetadata.description || companion?.description || '',
4484
6028
  owner: parsedMetadata.owner || companion?.owner || '',
4485
6029
  tags: parsedMetadata.tags.length > 0 ? parsedMetadata.tags : companion?.tags ?? [],
@@ -4562,7 +6106,7 @@ export function buildSemanticTableMapping(semanticLayer, rows) {
4562
6106
  return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
4563
6107
  }
4564
6108
  function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
4565
- if (projectConfig.semanticLayer?.provider === 'dbt' || detectedProvider === 'dbt')
6109
+ if (projectConfig.semanticLayer?.provider === 'dbt' || projectConfig.dbt?.projectDir || detectedProvider === 'dbt')
4566
6110
  return true;
4567
6111
  return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
4568
6112
  }
@@ -4570,7 +6114,10 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
4570
6114
  const useMetricFlow = request.engine === 'metricflow' || (request.engine !== 'native' &&
4571
6115
  isDbtSemanticRuntime(context.projectConfig, context.detectedProvider, semanticLayer));
4572
6116
  if (useMetricFlow) {
4573
- const dbtProjectPath = context.projectConfig.semanticLayer?.projectPath;
6117
+ const effectiveSemanticConfig = resolveProjectSemanticConfig(context.projectConfig, context.projectRoot);
6118
+ const dbtProjectPath = effectiveSemanticConfig?.provider === 'dbt'
6119
+ ? effectiveSemanticConfig.projectPath
6120
+ : context.projectConfig.dbt?.projectDir;
4574
6121
  const compiled = compileMetricFlowQuery({
4575
6122
  projectRoot: context.projectRoot,
4576
6123
  dbtProjectPath,
@@ -4829,6 +6376,164 @@ export function validateBlockStudioSource(source, semanticLayer) {
4829
6376
  executableSql,
4830
6377
  };
4831
6378
  }
6379
+ export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
6380
+ const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
6381
+ const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
6382
+ const provider = deterministicOnly ? null : await createBlockStudioAssistProvider(projectRoot, requestedProvider);
6383
+ const session = createBlockStudioImportSession(projectRoot, {
6384
+ inputPath: options.inputPath ?? '',
6385
+ inputMode: options.inputMode,
6386
+ sources: options.sources,
6387
+ sourceKind: options.sourceKind ?? 'raw-sql',
6388
+ domain: options.domain,
6389
+ owner: options.owner,
6390
+ tags: options.tags,
6391
+ });
6392
+ const warnings = [];
6393
+ const nextCandidates = [];
6394
+ let contextObjectCount = 0;
6395
+ for (const candidate of session.candidates) {
6396
+ const contextPack = await buildDqlGenerationContextPack(projectRoot, candidate).catch((error) => {
6397
+ warnings.push(`Context pack failed for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
6398
+ return null;
6399
+ });
6400
+ contextObjectCount += contextPack?.objects.length ?? 0;
6401
+ const evidence = contextPack ? dqlGenerationEvidenceFromContext(contextPack, candidate) : deterministicDqlGenerationEvidence(candidate);
6402
+ let patch = deterministicDqlGenerationPatch(candidate, evidence);
6403
+ let generatorName = 'local-deterministic';
6404
+ let generationMode = 'deterministic';
6405
+ if (provider) {
6406
+ const aiPatch = await buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack).catch((error) => {
6407
+ warnings.push(`AI generation fell back for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
6408
+ return null;
6409
+ });
6410
+ if (aiPatch) {
6411
+ patch = mergeDqlGenerationPatch(patch, aiPatch, candidate, evidence);
6412
+ generatorName = provider.name;
6413
+ generationMode = 'ai';
6414
+ }
6415
+ }
6416
+ const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
6417
+ patch = {
6418
+ ...patch,
6419
+ similarityMatches: similarity.matches,
6420
+ recommendedAction: similarity.recommendedAction,
6421
+ };
6422
+ for (const warning of patch.warnings ?? [])
6423
+ warnings.push(`${candidate.name}: ${warning}`);
6424
+ const enriched = updateBlockStudioImportCandidate(projectRoot, session.id, candidate.id, {
6425
+ name: patch.name,
6426
+ domain: patch.domain,
6427
+ description: patch.description,
6428
+ owner: patch.owner,
6429
+ tags: patch.tags,
6430
+ terms: patch.terms,
6431
+ pattern: patch.pattern,
6432
+ grain: patch.grain,
6433
+ entities: patch.entities,
6434
+ outputs: patch.outputs,
6435
+ dimensions: patch.dimensions,
6436
+ allowedFilters: patch.allowedFilters,
6437
+ parameterPolicy: patch.parameterPolicy,
6438
+ filterBindings: patch.filterBindings,
6439
+ parameterDecisions: patch.parameterDecisions,
6440
+ similarityMatches: patch.similarityMatches,
6441
+ recommendedAction: patch.recommendedAction,
6442
+ sourceSystems: patch.sourceSystems,
6443
+ replacementFor: patch.replacementFor,
6444
+ reviewCadence: patch.reviewCadence,
6445
+ sql: patch.sql,
6446
+ llmContext: patch.llmContext,
6447
+ evidence,
6448
+ conversionNotes: dqlGenerationConversionNotes(generatorName),
6449
+ generationMode,
6450
+ generationProvider: generatorName,
6451
+ });
6452
+ const validated = {
6453
+ ...enriched,
6454
+ validation: validateBlockStudioSource(enriched.dqlSource, semanticLayer),
6455
+ };
6456
+ const savedDraft = saveDqlGenerationDraftForProject(projectRoot, session.id, validated);
6457
+ writeBlockStudioImportCandidate(projectRoot, session.id, savedDraft);
6458
+ nextCandidates.push(savedDraft);
6459
+ }
6460
+ const generationSession = {
6461
+ ...session,
6462
+ mode: 'ai-import',
6463
+ candidates: nextCandidates,
6464
+ updatedAt: new Date().toISOString(),
6465
+ generation: {
6466
+ provider: nextCandidates.find((candidate) => candidate.generationMode === 'ai')?.generationProvider ?? 'local-deterministic',
6467
+ aiEnabled: nextCandidates.some((candidate) => candidate.generationMode === 'ai'),
6468
+ contextObjectCount,
6469
+ createdDrafts: nextCandidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
6470
+ warnings,
6471
+ },
6472
+ };
6473
+ writeBlockStudioImportSession(projectRoot, generationSession);
6474
+ if (generationSession.generation.createdDrafts > 0)
6475
+ await refreshLocalMetadataCatalog(projectRoot);
6476
+ return generationSession;
6477
+ }
6478
+ function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
6479
+ if (candidate.recommendedAction === 'reuse_existing') {
6480
+ const topMatch = candidate.similarityMatches?.[0];
6481
+ return {
6482
+ ...candidate,
6483
+ reviewStatus: 'review',
6484
+ draftSave: {
6485
+ status: 'skipped',
6486
+ reason: topMatch
6487
+ ? `Reuse recommended: ${topMatch.name} (${topMatch.kind}, ${(topMatch.score * 100).toFixed(0)}%).`
6488
+ : 'Reuse recommended; no new draft block was needed.',
6489
+ },
6490
+ generationMode: candidate.generationMode ?? 'deterministic',
6491
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
6492
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
6493
+ evidence: candidate.evidence ?? [],
6494
+ };
6495
+ }
6496
+ try {
6497
+ const savedPath = saveBlockStudioDraftArtifacts(projectRoot, {
6498
+ currentPath: isDraftBlockPath(candidate.savedPath) ? candidate.savedPath : undefined,
6499
+ source: candidate.dqlSource,
6500
+ name: candidate.name,
6501
+ domain: candidate.domain,
6502
+ description: candidate.description,
6503
+ owner: candidate.owner,
6504
+ tags: candidate.tags,
6505
+ lineage: candidate.lineage.sourceTables,
6506
+ stableSuffix: candidate.id.replace(/^cand_/, ''),
6507
+ importMeta: {
6508
+ importId,
6509
+ candidateId: candidate.id,
6510
+ sourceKind: candidate.sourceKind,
6511
+ sourcePath: candidate.sourcePath,
6512
+ },
6513
+ });
6514
+ const draftSave = { status: 'saved', path: savedPath, savedAt: new Date().toISOString() };
6515
+ return {
6516
+ ...candidate,
6517
+ reviewStatus: candidate.reviewStatus === 'rejected' ? 'rejected' : 'draft',
6518
+ savedPath,
6519
+ draftSave,
6520
+ generationMode: candidate.generationMode ?? 'deterministic',
6521
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
6522
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
6523
+ evidence: candidate.evidence ?? [],
6524
+ };
6525
+ }
6526
+ catch (error) {
6527
+ return {
6528
+ ...candidate,
6529
+ draftSave: { status: 'error', error: error instanceof Error ? error.message : String(error) },
6530
+ generationMode: candidate.generationMode ?? 'deterministic',
6531
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
6532
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
6533
+ evidence: candidate.evidence ?? [],
6534
+ };
6535
+ }
6536
+ }
4832
6537
  export function saveBlockStudioArtifacts(projectRoot, options) {
4833
6538
  const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
4834
6539
  const safeDomain = (options.domain ?? '')
@@ -4836,9 +6541,9 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
4836
6541
  .toLowerCase()
4837
6542
  .replace(/[^a-z0-9/_-]+/g, '-')
4838
6543
  .replace(/^\/+|\/+$/g, '') || 'uncategorized';
4839
- const targetRelativePath = `blocks/${safeDomain}/${slug}.dql`;
4840
- const targetPath = join(projectRoot, targetRelativePath);
4841
6544
  const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
6545
+ const targetRelativePath = canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath);
6546
+ const targetPath = join(projectRoot, targetRelativePath);
4842
6547
  if (existsSync(targetPath) && previousPath !== targetRelativePath) {
4843
6548
  throw new Error('BLOCK_EXISTS');
4844
6549
  }
@@ -4869,12 +6574,103 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
4869
6574
  }
4870
6575
  return targetRelativePath;
4871
6576
  }
6577
+ export function saveBlockStudioDraftArtifacts(projectRoot, options) {
6578
+ const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
6579
+ const safeDomain = (options.domain ?? '')
6580
+ .trim()
6581
+ .toLowerCase()
6582
+ .replace(/[^a-z0-9/_-]+/g, '-')
6583
+ .replace(/^\/+|\/+$/g, '') || 'uncategorized';
6584
+ const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
6585
+ const domainFirstRoot = join(projectRoot, 'domains');
6586
+ const useDomainFirstDrafts = existsSync(domainFirstRoot);
6587
+ const draftPrefix = useDomainFirstDrafts
6588
+ ? `domains/${safeDomain}/blocks/_drafts/`
6589
+ : `blocks/_drafts/${safeDomain}/`;
6590
+ const stableSuffix = options.stableSuffix?.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 18);
6591
+ let targetRelativePath = previousPath && isDraftBlockPath(previousPath)
6592
+ ? previousPath
6593
+ : `${draftPrefix}${slug}${stableSuffix ? `-${stableSuffix}` : ''}.dql`;
6594
+ let targetPath = join(projectRoot, targetRelativePath);
6595
+ if (!previousPath && existsSync(targetPath)) {
6596
+ for (let index = 2; index < 1000; index += 1) {
6597
+ const candidatePath = `${draftPrefix}${slug}-${index}.dql`;
6598
+ if (!existsSync(join(projectRoot, candidatePath))) {
6599
+ targetRelativePath = candidatePath;
6600
+ targetPath = join(projectRoot, targetRelativePath);
6601
+ break;
6602
+ }
6603
+ }
6604
+ }
6605
+ mkdirSync(dirname(targetPath), { recursive: true });
6606
+ writeFileSync(targetPath, options.source.trimEnd() + '\n', 'utf-8');
6607
+ const companionSlug = targetRelativePath.split('/').pop()?.replace(/\.dql$/, '') || slug;
6608
+ writeBlockCompanionFile(projectRoot, {
6609
+ slug: companionSlug,
6610
+ name: options.name,
6611
+ domain: `_drafts/${safeDomain}`,
6612
+ description: options.description,
6613
+ owner: options.owner,
6614
+ tags: options.tags,
6615
+ provider: 'dql',
6616
+ content: options.source,
6617
+ lineage: options.lineage,
6618
+ importMeta: options.importMeta,
6619
+ });
6620
+ return targetRelativePath;
6621
+ }
6622
+ function canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath) {
6623
+ const previousDomainFirst = previousPath?.match(/^domains\/([^/]+)\/blocks\/(?:_drafts\/)?[^/]+\.dql$/);
6624
+ if (previousDomainFirst) {
6625
+ return `domains/${previousDomainFirst[1]}/blocks/${slug}.dql`;
6626
+ }
6627
+ if (existsSync(join(projectRoot, 'domains', safeDomain))) {
6628
+ return `domains/${safeDomain}/blocks/${slug}.dql`;
6629
+ }
6630
+ return `blocks/${safeDomain}/${slug}.dql`;
6631
+ }
6632
+ function isDraftBlockPath(value) {
6633
+ if (!value)
6634
+ return false;
6635
+ const normalized = normalize(value).replace(/^\/+/, '');
6636
+ return normalized.startsWith('blocks/_drafts/') || /^domains\/[^/]+\/blocks\/_drafts\//.test(normalized);
6637
+ }
6638
+ function isBlockStudioBlockPath(value) {
6639
+ if (!value)
6640
+ return false;
6641
+ const normalized = normalize(value).replace(/^\/+/, '');
6642
+ return normalized.startsWith('blocks/') || /^domains\/[^/]+\/blocks\//.test(normalized);
6643
+ }
6644
+ function inferBlockStudioPathDomain(blockPath) {
6645
+ const normalized = normalize(blockPath).replace(/^\/+/, '');
6646
+ if (normalized.startsWith('blocks/')) {
6647
+ return normalized.split('/').slice(1, -1).join('/');
6648
+ }
6649
+ const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)$/);
6650
+ if (!domainFirst)
6651
+ return '';
6652
+ const domain = domainFirst[1];
6653
+ const blockSubpath = domainFirst[2];
6654
+ if (blockSubpath.startsWith('_drafts/'))
6655
+ return `_drafts/${domain}`;
6656
+ return domain;
6657
+ }
4872
6658
  function blockCompanionRelativePath(blockPath) {
4873
6659
  const normalized = normalize(blockPath).replace(/^\/+/, '');
4874
- if (!normalized.startsWith('blocks/'))
4875
- return null;
4876
- const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
4877
- return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
6660
+ if (normalized.startsWith('blocks/')) {
6661
+ const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
6662
+ return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
6663
+ }
6664
+ const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)\.dql$/);
6665
+ if (domainFirst) {
6666
+ const domain = domainFirst[1];
6667
+ const blockPath = domainFirst[2];
6668
+ const companionBlockPath = blockPath.startsWith('_drafts/')
6669
+ ? join('_drafts', domain, blockPath.slice('_drafts/'.length))
6670
+ : join(domain, blockPath);
6671
+ return join('semantic-layer', 'blocks', `${companionBlockPath}.yaml`).replaceAll('\\', '/');
6672
+ }
6673
+ return null;
4878
6674
  }
4879
6675
  function readBlockCompanionFile(projectRoot, relativePath) {
4880
6676
  const absPath = join(projectRoot, relativePath);
@@ -4922,18 +6718,48 @@ function readBlockCompanionFile(projectRoot, relativePath) {
4922
6718
  return null;
4923
6719
  }
4924
6720
  }
4925
- function parseBlockSourceMetadata(source) {
6721
+ export function parseBlockSourceMetadata(source) {
4926
6722
  const name = source.match(/^\s*block\s+"([^"]+)"/i)?.[1] ?? '';
4927
6723
  const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
4928
- const tags = source.match(/\btags\s*=\s*\[([^\]]*)\]/i);
6724
+ const extractStringArray = (key) => {
6725
+ const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'i'));
6726
+ return match ? (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [];
6727
+ };
6728
+ const extractStringMapSection = (key) => {
6729
+ const match = source.match(new RegExp(`\\b${key}\\s*\\{([\\s\\S]*?)\\n\\s*\\}`, 'i'));
6730
+ if (!match)
6731
+ return [];
6732
+ return match[1]
6733
+ .split(/\r?\n/)
6734
+ .map((line) => line.trim())
6735
+ .map((line) => line.match(/^([A-Za-z_][\w.]*)\s*=\s*"([^"]*)"\s*$/))
6736
+ .filter((entry) => Boolean(entry))
6737
+ .map((entry) => ({ key: entry[1], value: entry[2] }));
6738
+ };
6739
+ const parameterPolicy = extractStringMapSection('parameterPolicy')
6740
+ .map((entry) => ({ name: entry.key, policy: entry.value }));
6741
+ const filterBindings = extractStringMapSection('filterBindings')
6742
+ .map((entry) => ({ filter: entry.key, binding: entry.value }));
4929
6743
  return {
4930
6744
  name,
4931
6745
  domain: extractString('domain'),
4932
6746
  description: extractString('description'),
4933
6747
  owner: extractString('owner'),
4934
- tags: tags ? (tags[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [],
6748
+ tags: extractStringArray('tags'),
4935
6749
  status: extractString('status') || 'draft',
4936
6750
  blockType: extractString('type') || 'custom',
6751
+ llmContext: extractString('llmContext'),
6752
+ pattern: extractString('pattern'),
6753
+ grain: extractString('grain'),
6754
+ entities: extractStringArray('entities'),
6755
+ outputs: extractStringArray('outputs'),
6756
+ dimensions: extractStringArray('dimensions'),
6757
+ allowedFilters: extractStringArray('allowedFilters'),
6758
+ parameterPolicy,
6759
+ filterBindings,
6760
+ sourceSystems: extractStringArray('sourceSystems'),
6761
+ replacementFor: extractStringArray('replacementFor'),
6762
+ reviewCadence: extractString('reviewCadence'),
4937
6763
  };
4938
6764
  }
4939
6765
  function compareBlockStudioValues(actual, operator, expected) {
@@ -4990,58 +6816,948 @@ function buildBlockStudioCertificationChecklist(input) {
4990
6816
  if (!input.validation.chartConfig?.chart)
4991
6817
  blockers.add('Visualization config is missing');
4992
6818
  return {
4993
- metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
4994
- validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
4995
- run: input.previewSucceeded,
4996
- tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
4997
- chart: Boolean(input.validation.chartConfig?.chart),
4998
- lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
4999
- aiReviewed: true,
5000
- blockers: Array.from(blockers),
5001
- checkedAt: new Date().toISOString(),
6819
+ metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
6820
+ validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
6821
+ run: input.previewSucceeded,
6822
+ tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
6823
+ chart: Boolean(input.validation.chartConfig?.chart),
6824
+ lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
6825
+ aiReviewed: true,
6826
+ blockers: Array.from(blockers),
6827
+ checkedAt: new Date().toISOString(),
6828
+ };
6829
+ }
6830
+ function extractSqlTablesLight(sql) {
6831
+ const tables = new Set();
6832
+ const cleaned = sql
6833
+ .replace(/\/\*[\s\S]*?\*\//g, ' ')
6834
+ .replace(/--[^\n\r]*/g, ' ')
6835
+ .replace(/extract\s*\(\s*\w+\s+from\s+[^)]+\)/gi, 'EXTRACT_VALUE');
6836
+ const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
6837
+ let match;
6838
+ while ((match = regex.exec(cleaned))) {
6839
+ const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
6840
+ if (raw && !raw.startsWith('(') && !/^(select|values|unnest|lateral)$/i.test(raw))
6841
+ tables.add(raw);
6842
+ }
6843
+ return Array.from(tables);
6844
+ }
6845
+ function setBlockStudioStatusInSource(source, newStatus) {
6846
+ if (/status\s*=\s*"[^"]*"/.test(source)) {
6847
+ return source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
6848
+ }
6849
+ return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
6850
+ }
6851
+ export function setBlockStudioStatus(projectRoot, blockPath, newStatus) {
6852
+ const normalizedPath = normalize(blockPath).replace(/^\/+/, '');
6853
+ if (!isBlockStudioBlockPath(normalizedPath))
6854
+ throw new Error('Invalid block path');
6855
+ const absPath = join(projectRoot, normalizedPath);
6856
+ if (!existsSync(absPath))
6857
+ throw new Error('Block file not found');
6858
+ const source = setBlockStudioStatusInSource(readFileSync(absPath, 'utf-8'), newStatus);
6859
+ writeFileSync(absPath, source, 'utf-8');
6860
+ const companionPath = blockCompanionRelativePath(normalizedPath);
6861
+ if (!companionPath)
6862
+ return;
6863
+ const absCompanionPath = join(projectRoot, companionPath);
6864
+ if (!existsSync(absCompanionPath))
6865
+ return;
6866
+ let companion = readFileSync(absCompanionPath, 'utf-8');
6867
+ if (/^reviewStatus:\s*.+$/m.test(companion)) {
6868
+ companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
6869
+ }
6870
+ else {
6871
+ companion = `${companion.trimEnd()}\nreviewStatus: ${newStatus}\n`;
6872
+ }
6873
+ writeFileSync(absCompanionPath, companion, 'utf-8');
6874
+ }
6875
+ async function buildDqlGenerationContextPack(projectRoot, candidate) {
6876
+ const tables = candidate.lineage.sourceTables.join(', ');
6877
+ const question = [
6878
+ `Generate a governed DQL block draft for SQL import candidate "${candidate.name}".`,
6879
+ candidate.description,
6880
+ tables ? `Source tables: ${tables}.` : '',
6881
+ candidate.sql.slice(0, 1200),
6882
+ ].filter(Boolean).join('\n');
6883
+ return buildLocalContextPack(projectRoot, {
6884
+ question,
6885
+ mode: 'build',
6886
+ surface: 'block-studio',
6887
+ limit: 80,
6888
+ objectTypes: [
6889
+ 'dql_block',
6890
+ 'dql_term',
6891
+ 'business_view',
6892
+ 'domain',
6893
+ 'semantic_metric',
6894
+ 'semantic_model',
6895
+ 'semantic_dimension',
6896
+ 'semantic_measure',
6897
+ 'dbt_model',
6898
+ 'dbt_source',
6899
+ 'dbt_column',
6900
+ 'warehouse_table',
6901
+ 'datalex_domain',
6902
+ 'datalex_entity',
6903
+ 'datalex_contract',
6904
+ 'datalex_term',
6905
+ ],
6906
+ strictness: 'balanced',
6907
+ });
6908
+ }
6909
+ function dqlGenerationEvidenceFromContext(contextPack, candidate) {
6910
+ const directEvidence = contextPack.objects
6911
+ .filter((object) => isDqlGenerationEvidenceObject(object))
6912
+ .filter((object) => metadataObjectMatchesImportedSource(object, candidate.lineage.sourceTables))
6913
+ .map((object) => metadataObjectToDqlGenerationEvidence(object, 'Directly matches an imported SQL source table.'));
6914
+ const selected = contextPack.retrievalDiagnostics.selectedEvidence.slice(0, 12);
6915
+ const byKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
6916
+ const evidence = selected.flatMap((item) => {
6917
+ const object = byKey.get(item.objectKey);
6918
+ if (!object || !isDqlGenerationEvidenceObject(object))
6919
+ return [];
6920
+ return [metadataObjectToDqlGenerationEvidence(object, item.reason, Math.max(0.35, Math.min(0.98, item.score / Math.max(item.score, 10))))];
6921
+ });
6922
+ const ranked = uniqueDqlGenerationEvidence([...directEvidence, ...evidence]).slice(0, 12);
6923
+ return ranked.length > 0 ? ranked : deterministicDqlGenerationEvidence(candidate);
6924
+ }
6925
+ function deterministicDqlGenerationEvidence(candidate) {
6926
+ return candidate.lineage.sourceTables.map((table) => ({
6927
+ kind: 'warehouse_table',
6928
+ name: table,
6929
+ source: candidate.sourcePath,
6930
+ reason: 'Detected in the imported SQL source.',
6931
+ confidence: 0.7,
6932
+ }));
6933
+ }
6934
+ function isDqlGenerationEvidenceObject(object) {
6935
+ if (object.objectType === 'dql_block') {
6936
+ if (object.status !== 'certified')
6937
+ return false;
6938
+ if (object.sourcePath?.includes('/_drafts/') || object.sourcePath?.includes('\\_drafts\\'))
6939
+ return false;
6940
+ }
6941
+ return true;
6942
+ }
6943
+ function metadataObjectToDqlGenerationEvidence(object, reason, confidence = 0.82) {
6944
+ return {
6945
+ kind: dqlGenerationEvidenceKind(object.objectType),
6946
+ name: object.fullName ?? object.name,
6947
+ description: object.description,
6948
+ objectKey: object.objectKey,
6949
+ source: object.sourcePath ?? object.sourceSystem,
6950
+ reason,
6951
+ confidence,
6952
+ };
6953
+ }
6954
+ function metadataObjectMatchesImportedSource(object, sourceTables) {
6955
+ if (sourceTables.length === 0)
6956
+ return false;
6957
+ const haystack = [
6958
+ object.fullName,
6959
+ object.name,
6960
+ object.objectKey,
6961
+ object.sourcePath,
6962
+ object.sourceSystem,
6963
+ typeof object.payload?.relation === 'string' ? object.payload.relation : '',
6964
+ typeof object.payload?.relationName === 'string' ? object.payload.relationName : '',
6965
+ typeof object.payload?.relation_name === 'string' ? object.payload.relation_name : '',
6966
+ ].filter(Boolean).join(' ').toLowerCase();
6967
+ return sourceTables.some((table) => relationLookupTokens(table).some((token) => token && haystack.includes(token)));
6968
+ }
6969
+ function relationLookupTokens(relation) {
6970
+ const normalized = relation.replace(/[`"[\]]/g, '').toLowerCase();
6971
+ const parts = normalized.split('.').filter(Boolean);
6972
+ return Array.from(new Set([
6973
+ normalized,
6974
+ parts.slice(-2).join('.'),
6975
+ parts.at(-1) ?? '',
6976
+ ].filter((item) => item.length >= 3)));
6977
+ }
6978
+ function uniqueDqlGenerationEvidence(evidence) {
6979
+ const seen = new Set();
6980
+ const result = [];
6981
+ for (const item of evidence) {
6982
+ const key = item.objectKey || `${item.kind}:${item.name}`.toLowerCase();
6983
+ if (seen.has(key))
6984
+ continue;
6985
+ seen.add(key);
6986
+ result.push(item);
6987
+ }
6988
+ return result;
6989
+ }
6990
+ function dqlGenerationEvidenceKind(objectType) {
6991
+ if (objectType === 'dql_block')
6992
+ return 'dql_block';
6993
+ if (objectType === 'dql_term')
6994
+ return 'dql_term';
6995
+ if (objectType === 'business_view')
6996
+ return 'business_view';
6997
+ if (objectType === 'domain')
6998
+ return 'domain';
6999
+ if (objectType === 'semantic_metric')
7000
+ return 'semantic_metric';
7001
+ if (objectType === 'semantic_model')
7002
+ return 'semantic_model';
7003
+ if (objectType === 'dbt_model' || objectType === 'dbt_source' || objectType === 'dbt_column')
7004
+ return 'dbt_model';
7005
+ if (objectType === 'warehouse_table')
7006
+ return 'warehouse_table';
7007
+ if (objectType === 'datalex_contract')
7008
+ return 'datalex_contract';
7009
+ if (objectType === 'datalex_entity')
7010
+ return 'datalex_entity';
7011
+ if (objectType === 'datalex_domain')
7012
+ return 'datalex_domain';
7013
+ if (objectType === 'datalex_term')
7014
+ return 'datalex_term';
7015
+ if (objectType.includes('lineage'))
7016
+ return 'lineage';
7017
+ return 'metadata';
7018
+ }
7019
+ function deterministicDqlGenerationPatch(candidate, evidence) {
7020
+ const domain = inferDqlGenerationDomain(candidate);
7021
+ const parameterized = parameterizeSqlForDqlImport(candidate.sql);
7022
+ const grain = extractDqlGenerationGroupByFields(parameterized.sql)[0];
7023
+ const outputs = extractDqlGenerationSelectOutputs(parameterized.sql);
7024
+ const sourceSystems = candidate.lineage.sourceTables.map((table) => table.split('.').filter(Boolean).slice(-2, -1)[0] ?? '').filter(Boolean);
7025
+ const description = candidate.description && !candidate.description.startsWith('Imported from ')
7026
+ ? candidate.description
7027
+ : deterministicDqlGenerationDescription(candidate, evidence);
7028
+ const allowedFilters = Array.from(new Set([
7029
+ ...parameterized.allowedFilters,
7030
+ ...extractDqlGenerationFilterFields(parameterized.sql),
7031
+ ])).slice(0, 16);
7032
+ return {
7033
+ name: candidate.name,
7034
+ domain,
7035
+ description,
7036
+ owner: candidate.owner,
7037
+ tags: dqlGenerationBusinessTags(candidate, evidence, domain),
7038
+ terms: inferDqlGenerationTerms(evidence),
7039
+ llmContext: deterministicDqlGenerationContext({ ...candidate, sql: parameterized.sql }, evidence),
7040
+ pattern: inferDqlGenerationPattern(parameterized.sql),
7041
+ grain,
7042
+ entities: inferDqlGenerationEntities({
7043
+ grain,
7044
+ outputs,
7045
+ sourceTables: candidate.lineage.sourceTables,
7046
+ evidence,
7047
+ }),
7048
+ outputs,
7049
+ dimensions: extractDqlGenerationDimensions(parameterized.sql, grain, outputs),
7050
+ allowedFilters,
7051
+ parameterPolicy: parameterized.parameterPolicy,
7052
+ filterBindings: parameterized.filterBindings,
7053
+ parameterDecisions: parameterized.parameterDecisions,
7054
+ sql: parameterized.sql,
7055
+ warnings: parameterized.warnings,
7056
+ sourceSystems,
7057
+ reviewCadence: 'monthly',
7058
+ };
7059
+ }
7060
+ function inferDqlGenerationTerms(evidence) {
7061
+ return Array.from(new Set(evidence
7062
+ .filter((item) => item.kind === 'dql_term' || item.kind === 'datalex_term')
7063
+ .map((item) => item.name.trim())
7064
+ .filter(Boolean))).slice(0, 16);
7065
+ }
7066
+ function deterministicDqlGenerationDescription(candidate, evidence) {
7067
+ const tables = candidate.lineage.sourceTables.length > 0
7068
+ ? candidate.lineage.sourceTables.join(', ')
7069
+ : candidate.sourcePath;
7070
+ const sourceContext = evidence.find((item) => (item.kind === 'dbt_model' || item.kind === 'semantic_model' || item.kind === 'warehouse_table') &&
7071
+ item.description &&
7072
+ candidate.lineage.sourceTables.some((table) => relationLookupTokens(table).some((token) => item.name.toLowerCase().includes(token))));
7073
+ const aggregate = /\b(sum|count|avg|min|max)\s*\(/i.test(candidate.sql);
7074
+ const grouped = /\bgroup\s+by\b/i.test(candidate.sql);
7075
+ const sourcePhrase = sourceContext?.description
7076
+ ? ` using ${sourceContext.name} (${sourceContext.description})`
7077
+ : ` using ${tables}`;
7078
+ if (aggregate && grouped)
7079
+ return `Summarizes imported SQL${sourcePhrase}.`;
7080
+ if (aggregate)
7081
+ return `Calculates aggregate values${sourcePhrase}.`;
7082
+ return `Exposes imported SQL${sourcePhrase}.`;
7083
+ }
7084
+ function dqlGenerationBusinessTags(candidate, evidence, domain) {
7085
+ const generic = new Set(['imported', 'raw-sql', 'ai-generated']);
7086
+ const stopTags = new Set(['int', 'stg', 'src', 'fct', 'dim', 'name', 'id', 'key', 'model', 'source', 'table']);
7087
+ const sourceTokens = candidate.lineage.sourceTables
7088
+ .flatMap((table) => table.split('.').slice(-2))
7089
+ .flatMap((part) => part.split(/[_\s.-]+/));
7090
+ const tags = [
7091
+ ...candidate.tags.filter((tag) => !generic.has(tag)),
7092
+ domain,
7093
+ ...sourceTokens,
7094
+ 'review-required',
7095
+ ].map((tag) => tag.trim().toLowerCase())
7096
+ .filter((tag) => tag.length > 1 && !generic.has(tag) && !stopTags.has(tag))
7097
+ .map((tag) => tag.replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, ''))
7098
+ .filter(Boolean);
7099
+ return Array.from(new Set(tags)).slice(0, 12);
7100
+ }
7101
+ function deterministicDqlGenerationContext(candidate, evidence) {
7102
+ const tables = candidate.lineage.sourceTables.length > 0
7103
+ ? candidate.lineage.sourceTables.join(', ')
7104
+ : 'the imported SQL source';
7105
+ const evidenceNames = evidence
7106
+ .filter((item) => item.kind !== 'dql_block')
7107
+ .slice(0, 3)
7108
+ .map((item) => item.name)
7109
+ .filter(Boolean);
7110
+ const grain = extractDqlGenerationGroupByFields(candidate.sql);
7111
+ const years = extractDqlGenerationYearFilters(candidate.sql);
7112
+ return [
7113
+ `Use after review for questions matching "${candidate.name}".`,
7114
+ grain.length > 0 ? `Grain: ${grain.join(', ')}.` : '',
7115
+ years.length > 0 ? `Filters: year ${years.join(', ')}.` : '',
7116
+ `Source: ${tables}.`,
7117
+ evidenceNames.length > 0 ? `Grounded by: ${evidenceNames.join(', ')}.` : '',
7118
+ 'Certification must confirm grain, filters, owner, tags, and tests.',
7119
+ ].filter(Boolean).join(' ');
7120
+ }
7121
+ function extractDqlGenerationGroupByFields(sql) {
7122
+ const match = sql.match(/\bgroup\s+by\b([\s\S]+?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i);
7123
+ if (!match)
7124
+ return [];
7125
+ return splitDqlGenerationSqlList(match[1])
7126
+ .map((item) => item.replace(/[`"[\]]/g, '').trim())
7127
+ .filter((item) => item && !/^\d+$/.test(item))
7128
+ .slice(0, 4);
7129
+ }
7130
+ function extractDqlGenerationSelectOutputs(sql) {
7131
+ const selectMatch = sql.match(/\bselect\b([\s\S]+?)\bfrom\b/i);
7132
+ if (!selectMatch)
7133
+ return [];
7134
+ return splitDqlGenerationSqlList(selectMatch[1])
7135
+ .map((expr) => {
7136
+ const alias = expr.match(/\bas\s+([A-Za-z_][A-Za-z0-9_]*)\b/i)?.[1]
7137
+ ?? expr.match(/([A-Za-z_][A-Za-z0-9_]*)\s*$/)?.[1]
7138
+ ?? '';
7139
+ return alias.replace(/[`"[\]]/g, '');
7140
+ })
7141
+ .filter(Boolean)
7142
+ .filter((name) => !/^(from|where|group|order|limit)$/i.test(name))
7143
+ .slice(0, 24);
7144
+ }
7145
+ function extractDqlGenerationDimensions(sql, grain, outputs) {
7146
+ const outputSet = new Set(outputs.map(normalizedTerm));
7147
+ return extractDqlGenerationGroupByFields(sql)
7148
+ .map((field) => field.split('.').pop() ?? field)
7149
+ .filter((field) => normalizedTerm(field) !== normalizedTerm(grain ?? ''))
7150
+ .filter((field) => !outputSet.has(normalizedTerm(field)))
7151
+ .filter((field) => !/\b(total|count|sum|avg|average|min|max|rate|pct|percent|amount|revenue|points?|score)\b/i.test(field))
7152
+ .slice(0, 16);
7153
+ }
7154
+ function inferDqlGenerationEntities(input) {
7155
+ const entities = new Set();
7156
+ const add = (value) => {
7157
+ const entity = businessEntityFromDqlGenerationIdentifier(value ?? '');
7158
+ if (entity)
7159
+ entities.add(entity);
7160
+ };
7161
+ add(input.grain);
7162
+ for (const output of input.outputs)
7163
+ add(output);
7164
+ for (const table of input.sourceTables)
7165
+ add(table.split('.').pop());
7166
+ for (const item of input.evidence.slice(0, 12)) {
7167
+ if (item.kind === 'datalex_entity' || item.kind === 'semantic_model' || item.kind === 'dbt_model' || item.kind === 'warehouse_table') {
7168
+ add(item.name.split('.').pop());
7169
+ }
7170
+ }
7171
+ return [...entities].slice(0, 8);
7172
+ }
7173
+ function businessEntityFromDqlGenerationIdentifier(value) {
7174
+ const raw = value
7175
+ .split('.').pop()
7176
+ ?.trim()
7177
+ .replace(/[`"[\]]/g, '')
7178
+ .replace(/^(dim|fact|fct|stg|src|int)_/i, '')
7179
+ .replace(/_(id|key|name|code|uuid|number|num|abbreviation|abbr)$/i, '')
7180
+ .replace(/s$/i, '') ?? '';
7181
+ if (!raw)
7182
+ return '';
7183
+ const tokens = raw
7184
+ .split(/[^A-Za-z0-9]+/)
7185
+ .map((token) => token.trim().toLowerCase())
7186
+ .filter(Boolean)
7187
+ .filter((token) => !isGenericDqlGenerationEntityToken(token));
7188
+ if (tokens.length === 0)
7189
+ return '';
7190
+ return titleizeDqlGenerationName(tokens.join(' '));
7191
+ }
7192
+ function isGenericDqlGenerationEntityToken(token) {
7193
+ return new Set([
7194
+ 'total',
7195
+ 'count',
7196
+ 'sum',
7197
+ 'avg',
7198
+ 'average',
7199
+ 'min',
7200
+ 'max',
7201
+ 'metric',
7202
+ 'measure',
7203
+ 'value',
7204
+ 'amount',
7205
+ 'revenue',
7206
+ 'point',
7207
+ 'points',
7208
+ 'score',
7209
+ 'date',
7210
+ 'day',
7211
+ 'week',
7212
+ 'month',
7213
+ 'quarter',
7214
+ 'year',
7215
+ 'season',
7216
+ 'period',
7217
+ 'time',
7218
+ 'row',
7219
+ 'record',
7220
+ 'detail',
7221
+ 'stat',
7222
+ 'stats',
7223
+ 'analytic',
7224
+ 'analytics',
7225
+ 'transformed',
7226
+ ]).has(token);
7227
+ }
7228
+ function titleizeDqlGenerationName(value) {
7229
+ return value
7230
+ .replace(/[_-]+/g, ' ')
7231
+ .replace(/\s+/g, ' ')
7232
+ .trim()
7233
+ .replace(/\b\w/g, (char) => char.toUpperCase());
7234
+ }
7235
+ function inferDqlGenerationPattern(sql) {
7236
+ const groupFields = extractDqlGenerationGroupByFields(sql);
7237
+ if (/@metric\s*\(/i.test(sql))
7238
+ return 'metric_wrapper';
7239
+ if (/\bjoin\b/i.test(sql)) {
7240
+ const systems = new Set(extractDqlGenerationSourceSystems(sql));
7241
+ if (systems.size > 1)
7242
+ return 'bridge';
7243
+ }
7244
+ if (hasDqlGenerationRankingLimit(sql))
7245
+ return 'ranking';
7246
+ if (groupFields.some((field) => /\b(date|day|week|month|quarter|year|period|time)\b/i.test(field)))
7247
+ return 'trend';
7248
+ if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
7249
+ return 'entity_rollup';
7250
+ if (!/\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql) && /\b(dim|profile|customer|account|player|product|user|entity)\b/i.test(sql)) {
7251
+ return 'entity_profile';
7252
+ }
7253
+ return 'custom';
7254
+ }
7255
+ function hasDqlGenerationRankingLimit(sql) {
7256
+ return /\border\s+by\b[\s\S]*\blimit\s+(?:\d+|\$\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}|[:?][A-Za-z_][A-Za-z0-9_]*)/i.test(sql);
7257
+ }
7258
+ function extractDqlGenerationSourceSystems(sql) {
7259
+ const tables = new Set();
7260
+ const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
7261
+ const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
7262
+ let match;
7263
+ while ((match = regex.exec(cleaned))) {
7264
+ const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
7265
+ const parts = raw.split('.').filter(Boolean);
7266
+ const system = parts.slice(-2, -1)[0] ?? parts[0];
7267
+ if (system)
7268
+ tables.add(system.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, '-'));
7269
+ }
7270
+ return [...tables].filter(Boolean).slice(0, 12);
7271
+ }
7272
+ function extractDqlGenerationFilterFields(sql) {
7273
+ const filters = new Set();
7274
+ const where = sql.match(/\bwhere\b([\s\S]+?)(?:\bgroup\s+by\b|\border\s+by\b|\blimit\b|$)/i)?.[1] ?? '';
7275
+ const addFilter = (value) => {
7276
+ const name = value?.split('.').pop()?.replace(/[`"[\]]/g, '');
7277
+ if (name && !/^(and|or|not|null|year|month|day)$/i.test(name))
7278
+ filters.add(name);
7279
+ };
7280
+ const extractRegex = /\bextract\s*\([^)]*\bfrom\s+([A-Za-z_][A-Za-z0-9_.]*)\s*\)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
7281
+ const isNullRegex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s+is\s+(?:not\s+)?null\b/gi;
7282
+ const regex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
7283
+ let match;
7284
+ while ((match = extractRegex.exec(where)))
7285
+ addFilter(match[1]);
7286
+ while ((match = isNullRegex.exec(where)))
7287
+ addFilter(match[1]);
7288
+ while ((match = regex.exec(where))) {
7289
+ addFilter(match[1]);
7290
+ }
7291
+ return [...filters].slice(0, 16);
7292
+ }
7293
+ function extractDqlGenerationYearFilters(sql) {
7294
+ const years = new Set();
7295
+ const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
7296
+ let match;
7297
+ const extractYear = /extract\s*\(\s*year\s+from\s+[^)]+\)\s*(?:=\s*([12][0-9]{3})|in\s*\(([^)]*)\))/gi;
7298
+ while ((match = extractYear.exec(cleaned))) {
7299
+ for (const year of (match[1] || match[2] || '').match(/[12][0-9]{3}/g) ?? [])
7300
+ years.add(year);
7301
+ }
7302
+ return Array.from(years).sort();
7303
+ }
7304
+ function splitDqlGenerationSqlList(value) {
7305
+ const parts = [];
7306
+ let start = 0;
7307
+ let depth = 0;
7308
+ let single = false;
7309
+ let double = false;
7310
+ for (let i = 0; i < value.length; i += 1) {
7311
+ const char = value[i];
7312
+ if (!double && char === "'" && value[i - 1] !== '\\')
7313
+ single = !single;
7314
+ else if (!single && char === '"' && value[i - 1] !== '\\')
7315
+ double = !double;
7316
+ else if (!single && !double && char === '(')
7317
+ depth += 1;
7318
+ else if (!single && !double && char === ')' && depth > 0)
7319
+ depth -= 1;
7320
+ else if (!single && !double && depth === 0 && char === ',') {
7321
+ parts.push(value.slice(start, i));
7322
+ start = i + 1;
7323
+ }
7324
+ }
7325
+ parts.push(value.slice(start));
7326
+ return parts.map((part) => part.trim()).filter(Boolean);
7327
+ }
7328
+ function inferDqlGenerationDomain(candidate) {
7329
+ const current = candidate.domain?.trim();
7330
+ if (current && current !== 'imported' && current !== 'general')
7331
+ return current;
7332
+ const firstTable = candidate.lineage.sourceTables[0];
7333
+ const schema = firstTable?.split('.').filter(Boolean).slice(-2, -1)[0];
7334
+ return schema?.replace(/[^a-z0-9/_-]+/gi, '-').toLowerCase() || current || 'imported';
7335
+ }
7336
+ async function buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack) {
7337
+ const payload = {
7338
+ candidate: {
7339
+ name: candidate.name,
7340
+ domain: candidate.domain,
7341
+ description: candidate.description,
7342
+ owner: candidate.owner,
7343
+ tags: candidate.tags,
7344
+ sql: candidate.sql,
7345
+ sourceTables: candidate.lineage.sourceTables,
7346
+ parameters: candidate.lineage.parameters,
7347
+ warnings: candidate.warnings ?? candidate.lineage.warnings,
7348
+ grain: extractDqlGenerationGroupByFields(candidate.sql),
7349
+ yearFilters: extractDqlGenerationYearFilters(candidate.sql),
7350
+ parameterization: parameterizeSqlForDqlImport(candidate.sql),
7351
+ },
7352
+ context: {
7353
+ evidence,
7354
+ selectedObjects: contextPack?.objects.slice(0, 24).map((object) => ({
7355
+ objectKey: object.objectKey,
7356
+ objectType: object.objectType,
7357
+ name: object.fullName ?? object.name,
7358
+ domain: object.domain,
7359
+ status: object.status,
7360
+ description: object.description,
7361
+ })) ?? [],
7362
+ allowedRelations: contextPack?.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
7363
+ relation: relation.relation,
7364
+ source: relation.source,
7365
+ columns: relation.columns.slice(0, 16).map((column) => column.name),
7366
+ })) ?? [],
7367
+ },
7368
+ };
7369
+ const text = await provider.generate([
7370
+ {
7371
+ role: 'system',
7372
+ content: [
7373
+ 'You generate DQL Block Studio metadata for a local draft.',
7374
+ 'Return only a compact JSON object with optional keys: name, domain, description, owner, tags, terms, llmContext, pattern, grain, entities, outputs, dimensions, allowedFilters, sourceSystems, replacementFor, reviewCadence.',
7375
+ 'Do not return markdown. Do not mark the block certified. Do not change SQL.',
7376
+ 'Use only directly relevant dbt, semantic, warehouse, certified block, and SQL-shape evidence from the payload.',
7377
+ 'terms must reference existing DQL/DataLex glossary terms from the payload. Do not invent term names.',
7378
+ 'If metadata descriptions are missing, describe the observable SQL intent instead of inventing business meaning.',
7379
+ 'Descriptions and llmContext must be specific, business-readable, concise, and review-required.',
7380
+ 'reviewCadence should usually be monthly or quarterly unless evidence strongly suggests a different cadence.',
7381
+ 'Tags must be business/search tags. Avoid generic tags such as raw-sql, imported, ai-generated.',
7382
+ ].join('\n'),
7383
+ },
7384
+ { role: 'user', content: JSON.stringify(payload) },
7385
+ ], { maxTokens: 900, temperature: 0.1 });
7386
+ return normalizeDqlGenerationPatch(parseFirstJsonObject(text));
7387
+ }
7388
+ function parseFirstJsonObject(text) {
7389
+ const trimmed = text.trim();
7390
+ if (trimmed.startsWith('{') && trimmed.endsWith('}'))
7391
+ return JSON.parse(trimmed);
7392
+ const match = trimmed.match(/\{[\s\S]*\}/);
7393
+ if (!match)
7394
+ return null;
7395
+ return JSON.parse(match[0]);
7396
+ }
7397
+ function normalizeDqlGenerationPatch(value) {
7398
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7399
+ return null;
7400
+ const record = value;
7401
+ const patch = {};
7402
+ if (typeof record.name === 'string' && record.name.trim())
7403
+ patch.name = record.name.trim().slice(0, 120);
7404
+ if (typeof record.domain === 'string' && record.domain.trim())
7405
+ patch.domain = record.domain.trim().slice(0, 80);
7406
+ if (typeof record.description === 'string' && record.description.trim())
7407
+ patch.description = record.description.trim().slice(0, 500);
7408
+ if (typeof record.owner === 'string' && record.owner.trim())
7409
+ patch.owner = record.owner.trim().slice(0, 120);
7410
+ if (Array.isArray(record.tags))
7411
+ patch.tags = normalizeDqlGenerationStringArray(record.tags, 12);
7412
+ if (Array.isArray(record.terms))
7413
+ patch.terms = normalizeDqlGenerationStringArray(record.terms, 16);
7414
+ if (typeof record.llmContext === 'string' && record.llmContext.trim())
7415
+ patch.llmContext = record.llmContext.trim().slice(0, 1000);
7416
+ if (typeof record.pattern === 'string' && record.pattern.trim())
7417
+ patch.pattern = normalizeDqlGenerationPattern(record.pattern);
7418
+ if (typeof record.grain === 'string' && record.grain.trim())
7419
+ patch.grain = record.grain.trim().slice(0, 120);
7420
+ if (Array.isArray(record.entities))
7421
+ patch.entities = normalizeDqlGenerationStringArray(record.entities, 12);
7422
+ if (Array.isArray(record.outputs))
7423
+ patch.outputs = normalizeDqlGenerationStringArray(record.outputs, 24);
7424
+ if (Array.isArray(record.dimensions))
7425
+ patch.dimensions = normalizeDqlGenerationStringArray(record.dimensions, 16);
7426
+ if (Array.isArray(record.allowedFilters))
7427
+ patch.allowedFilters = normalizeDqlGenerationStringArray(record.allowedFilters, 16);
7428
+ if (Array.isArray(record.sourceSystems))
7429
+ patch.sourceSystems = normalizeDqlGenerationStringArray(record.sourceSystems, 12);
7430
+ if (Array.isArray(record.replacementFor))
7431
+ patch.replacementFor = normalizeDqlGenerationStringArray(record.replacementFor, 12);
7432
+ if (typeof record.reviewCadence === 'string' && record.reviewCadence.trim()) {
7433
+ const reviewCadence = normalizeDqlGenerationReviewCadence(record.reviewCadence);
7434
+ if (reviewCadence)
7435
+ patch.reviewCadence = reviewCadence;
7436
+ }
7437
+ return Object.keys(patch).length > 0 ? patch : null;
7438
+ }
7439
+ function normalizeDqlGenerationStringArray(value, limit) {
7440
+ const result = [];
7441
+ for (const item of value) {
7442
+ const normalized = normalizeDqlGenerationStringItem(item);
7443
+ if (normalized)
7444
+ result.push(normalized);
7445
+ }
7446
+ return Array.from(new Set(result)).slice(0, limit);
7447
+ }
7448
+ function normalizeDqlGenerationStringItem(value) {
7449
+ if (typeof value === 'string')
7450
+ return value.trim();
7451
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7452
+ return '';
7453
+ const record = value;
7454
+ for (const key of ['name', 'field', 'column', 'id', 'value', 'label', 'sourceSystem', 'source_system']) {
7455
+ const nested = record[key];
7456
+ if (typeof nested === 'string' && nested.trim())
7457
+ return nested.trim();
7458
+ }
7459
+ return '';
7460
+ }
7461
+ function normalizeDqlGenerationPattern(value) {
7462
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
7463
+ const allowed = new Set([
7464
+ 'metric_wrapper',
7465
+ 'entity_profile',
7466
+ 'entity_rollup',
7467
+ 'ranking',
7468
+ 'trend',
7469
+ 'bridge',
7470
+ 'drilldown',
7471
+ 'replacement',
7472
+ 'custom',
7473
+ ]);
7474
+ return allowed.has(normalized) ? normalized : undefined;
7475
+ }
7476
+ function normalizeDqlGenerationReviewCadence(value) {
7477
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_ -]+/g, '').replace(/\s+/g, '_');
7478
+ const allowed = new Set(['daily', 'weekly', 'biweekly', 'monthly', 'quarterly', 'semiannual', 'annual']);
7479
+ return allowed.has(normalized) ? normalized : undefined;
7480
+ }
7481
+ function mergeGroundedDqlGenerationTerms(baseTerms, overrideTerms, evidence) {
7482
+ const grounded = new Map();
7483
+ for (const item of evidence) {
7484
+ if (item.kind !== 'dql_term' && item.kind !== 'datalex_term')
7485
+ continue;
7486
+ const normalized = normalizedTerm(item.name);
7487
+ if (normalized)
7488
+ grounded.set(normalized, item.name.trim());
7489
+ }
7490
+ const merged = [...(baseTerms ?? []), ...(overrideTerms ?? [])]
7491
+ .map((term) => grounded.get(normalizedTerm(term)))
7492
+ .filter((term) => Boolean(term));
7493
+ const result = Array.from(new Set(merged)).slice(0, 16);
7494
+ return result.length > 0 ? result : undefined;
7495
+ }
7496
+ function mergeDqlGenerationPatch(base, override, candidate, evidence) {
7497
+ const name = override.name && !isWeakDqlGenerationName(override.name) ? override.name : base.name;
7498
+ const description = override.description
7499
+ && !isWeakDqlGenerationDescription(override.description)
7500
+ && !isUngroundedDqlGenerationDescription(override.description, candidate, evidence)
7501
+ ? override.description
7502
+ : base.description;
7503
+ const llmContext = override.llmContext
7504
+ && !isUngroundedDqlGenerationDescription(override.llmContext, candidate, evidence)
7505
+ ? override.llmContext
7506
+ : base.llmContext;
7507
+ const overrideTags = override.tags
7508
+ ? override.tags.filter((tag) => !/^(raw-sql|imported|ai-generated)$/i.test(tag))
7509
+ : [];
7510
+ return {
7511
+ ...base,
7512
+ ...override,
7513
+ name,
7514
+ description,
7515
+ llmContext,
7516
+ tags: Array.from(new Set([...(base.tags ?? []), ...overrideTags])).slice(0, 12),
7517
+ terms: mergeGroundedDqlGenerationTerms(base.terms, override.terms, evidence),
7518
+ pattern: override.pattern ?? base.pattern,
7519
+ grain: override.grain ?? base.grain,
7520
+ entities: override.entities?.length ? override.entities : base.entities,
7521
+ outputs: override.outputs?.length ? override.outputs : base.outputs,
7522
+ dimensions: override.dimensions?.length ? override.dimensions : base.dimensions,
7523
+ allowedFilters: override.allowedFilters?.length ? override.allowedFilters : base.allowedFilters,
7524
+ parameterPolicy: base.parameterPolicy,
7525
+ filterBindings: base.filterBindings,
7526
+ parameterDecisions: base.parameterDecisions,
7527
+ similarityMatches: base.similarityMatches,
7528
+ recommendedAction: base.recommendedAction,
7529
+ sql: base.sql,
7530
+ warnings: base.warnings,
7531
+ sourceSystems: override.sourceSystems?.length ? override.sourceSystems : base.sourceSystems,
7532
+ replacementFor: override.replacementFor?.length ? override.replacementFor : base.replacementFor,
7533
+ reviewCadence: override.reviewCadence ?? base.reviewCadence,
5002
7534
  };
5003
7535
  }
5004
- function extractSqlTablesLight(sql) {
5005
- const tables = new Set();
5006
- const cleaned = sql.replace(/\/\*[\s\S]*?\*\//g, ' ').replace(/--[^\n\r]*/g, ' ');
5007
- const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
5008
- let match;
5009
- while ((match = regex.exec(cleaned))) {
5010
- const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
5011
- if (raw && !raw.startsWith('(') && !/^(select|values|unnest|lateral)$/i.test(raw))
5012
- tables.add(raw);
5013
- }
5014
- return Array.from(tables);
7536
+ function buildDqlGenerationSimilarityMatches(candidate, patch, contextPack) {
7537
+ const candidateSql = patch.sql ?? candidate.sql;
7538
+ const candidateFingerprints = buildBlockSqlFingerprints(candidateSql);
7539
+ const candidateBusinessFingerprint = buildBlockBusinessFingerprint({
7540
+ name: patch.name ?? candidate.name,
7541
+ domain: patch.domain ?? candidate.domain,
7542
+ pattern: patch.pattern,
7543
+ grain: patch.grain,
7544
+ entities: patch.entities,
7545
+ terms: patch.terms,
7546
+ outputs: patch.outputs,
7547
+ dimensions: patch.dimensions,
7548
+ filters: patch.allowedFilters,
7549
+ sources: candidate.lineage.sourceTables,
7550
+ sourceSystems: patch.sourceSystems,
7551
+ });
7552
+ const candidateShape = {
7553
+ name: candidate.name,
7554
+ pattern: patch.pattern,
7555
+ grain: patch.grain,
7556
+ outputs: new Set((patch.outputs ?? []).map(normalizedTerm)),
7557
+ terms: new Set((patch.terms ?? []).map(normalizedTerm)),
7558
+ dimensions: new Set((patch.dimensions ?? []).map(normalizedTerm)),
7559
+ filters: new Set((patch.allowedFilters ?? []).map(normalizedTerm)),
7560
+ entities: new Set((patch.entities ?? []).map(normalizedTerm)),
7561
+ sources: new Set(candidate.lineage.sourceTables.flatMap(relationLookupTokens).map(normalizedTerm)),
7562
+ };
7563
+ const objects = (contextPack?.objects ?? []).filter((object) => object.objectType === 'dql_block');
7564
+ const matches = objects.flatMap((object) => {
7565
+ const payload = object.payload ?? {};
7566
+ const objectSql = typeof payload.sql === 'string' ? payload.sql : '';
7567
+ const objectFingerprints = metadataSqlFingerprints(payload.sqlFingerprints)
7568
+ ?? (objectSql ? buildBlockSqlFingerprints(objectSql) : null);
7569
+ const objectBusinessFingerprint = metadataBusinessFingerprint(payload.businessFingerprint);
7570
+ const objectShape = {
7571
+ pattern: metadataString(payload.pattern),
7572
+ grain: metadataString(payload.grain),
7573
+ outputs: new Set(metadataStringArray(payload.declaredOutputs ?? payload.outputs)),
7574
+ terms: new Set(metadataStringArray(payload.termRefs ?? payload.terms)),
7575
+ dimensions: new Set(metadataStringArray(payload.dimensions)),
7576
+ filters: new Set(metadataStringArray(payload.allowedFilters)),
7577
+ entities: new Set(metadataStringArray(payload.entities)),
7578
+ sources: new Set(metadataStringArray(payload.tableDependencies ?? payload.rawTableRefs).flatMap(relationLookupTokens).map(normalizedTerm)),
7579
+ };
7580
+ if (objectFingerprints?.exact && objectFingerprints.exact === candidateFingerprints.exact) {
7581
+ return [{
7582
+ kind: 'exact_sql_match',
7583
+ objectKey: object.objectKey,
7584
+ name: object.fullName ?? object.name,
7585
+ status: object.status,
7586
+ source: object.sourcePath,
7587
+ score: 0.99,
7588
+ reason: 'Normalized SQL exactly matches an existing DQL block.',
7589
+ recommendedAction: 'reuse_existing',
7590
+ }];
7591
+ }
7592
+ if (objectFingerprints?.parameterized && objectFingerprints.parameterized === candidateFingerprints.parameterized) {
7593
+ return [{
7594
+ kind: 'parameterized_duplicate',
7595
+ objectKey: object.objectKey,
7596
+ name: object.fullName ?? object.name,
7597
+ status: object.status,
7598
+ source: object.sourcePath,
7599
+ score: 0.94,
7600
+ reason: 'SQL shape matches after replacing literal values with parameters.',
7601
+ recommendedAction: 'reuse_existing',
7602
+ }];
7603
+ }
7604
+ const score = businessShapeSimilarity(candidateShape, objectShape, object);
7605
+ if (objectBusinessFingerprint?.hash
7606
+ && objectBusinessFingerprint.hash === candidateBusinessFingerprint.hash
7607
+ && object.status === 'certified') {
7608
+ return [{
7609
+ kind: 'business_duplicate',
7610
+ objectKey: object.objectKey,
7611
+ name: object.fullName ?? object.name,
7612
+ status: object.status,
7613
+ source: object.sourcePath,
7614
+ score: Math.max(score, 0.86),
7615
+ reason: 'Certified block has the same persisted business-shape fingerprint.',
7616
+ recommendedAction: 'reuse_existing',
7617
+ }];
7618
+ }
7619
+ if (score >= 0.76 && object.status === 'certified') {
7620
+ return [{
7621
+ kind: 'business_duplicate',
7622
+ objectKey: object.objectKey,
7623
+ name: object.fullName ?? object.name,
7624
+ status: object.status,
7625
+ source: object.sourcePath,
7626
+ score,
7627
+ reason: 'Certified block has the same business shape: entity, grain, filters, outputs, or source family overlap.',
7628
+ recommendedAction: 'reuse_existing',
7629
+ }];
7630
+ }
7631
+ if (score >= 0.58) {
7632
+ return [{
7633
+ kind: 'near_variant',
7634
+ objectKey: object.objectKey,
7635
+ name: object.fullName ?? object.name,
7636
+ status: object.status,
7637
+ source: object.sourcePath,
7638
+ score,
7639
+ reason: 'Existing block is close but appears to need a reviewed filter, output, grain, or source extension.',
7640
+ recommendedAction: 'extend_existing',
7641
+ }];
7642
+ }
7643
+ return [];
7644
+ }).sort((a, b) => b.score - a.score).slice(0, 8);
7645
+ const top = matches[0];
7646
+ return {
7647
+ matches,
7648
+ recommendedAction: top?.recommendedAction ?? 'create_new',
7649
+ };
5015
7650
  }
5016
- function setBlockStudioStatus(projectRoot, blockPath, newStatus) {
5017
- const normalizedPath = normalize(blockPath).replace(/^\/+/, '');
5018
- if (!normalizedPath.startsWith('blocks/'))
5019
- throw new Error('Invalid block path');
5020
- const absPath = join(projectRoot, normalizedPath);
5021
- if (!existsSync(absPath))
5022
- throw new Error('Block file not found');
5023
- let source = readFileSync(absPath, 'utf-8');
5024
- if (/status\s*=\s*"[^"]*"/.test(source)) {
5025
- source = source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
7651
+ function metadataSqlFingerprints(value) {
7652
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7653
+ return null;
7654
+ const record = value;
7655
+ const exact = typeof record.exact === 'string' ? record.exact : '';
7656
+ const parameterized = typeof record.parameterized === 'string' ? record.parameterized : '';
7657
+ if (!exact || !parameterized)
7658
+ return null;
7659
+ return { exact, parameterized };
7660
+ }
7661
+ function metadataBusinessFingerprint(value) {
7662
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7663
+ return null;
7664
+ const record = value;
7665
+ const hash = typeof record.hash === 'string' ? record.hash : '';
7666
+ if (!hash)
7667
+ return null;
7668
+ const tokens = Array.isArray(record.tokens)
7669
+ ? record.tokens.map((item) => String(item)).filter(Boolean)
7670
+ : [];
7671
+ return { hash, tokens };
7672
+ }
7673
+ function businessShapeSimilarity(candidate, object, metadata) {
7674
+ let score = 0;
7675
+ if (candidate.pattern && object.pattern && normalizedTerm(candidate.pattern) === normalizedTerm(object.pattern))
7676
+ score += 0.12;
7677
+ if (candidate.grain && object.grain && normalizedTerm(candidate.grain) === normalizedTerm(object.grain))
7678
+ score += 0.15;
7679
+ score += overlapScore(candidate.outputs, object.outputs) * 0.18;
7680
+ score += overlapScore(candidate.terms, object.terms) * 0.08;
7681
+ score += overlapScore(candidate.dimensions, object.dimensions) * 0.13;
7682
+ score += overlapScore(candidate.filters, object.filters) * 0.14;
7683
+ score += overlapScore(candidate.entities, object.entities) * 0.12;
7684
+ score += overlapScore(candidate.sources, object.sources) * 0.1;
7685
+ const nameTokens = new Set(candidate.name.toLowerCase().split(/[^a-z0-9]+/).filter((value) => value.length > 2));
7686
+ const targetTokens = new Set([
7687
+ ...metadata.name.toLowerCase().split(/[^a-z0-9]+/),
7688
+ ...(metadata.description ?? '').toLowerCase().split(/[^a-z0-9]+/),
7689
+ ].filter((value) => value.length > 2));
7690
+ score += overlapScore(nameTokens, targetTokens) * 0.04;
7691
+ return Number(Math.min(0.98, score).toFixed(3));
7692
+ }
7693
+ function overlapScore(left, right) {
7694
+ if (left.size === 0 || right.size === 0)
7695
+ return 0;
7696
+ let overlap = 0;
7697
+ for (const value of left) {
7698
+ if (value && right.has(value))
7699
+ overlap += 1;
7700
+ }
7701
+ return overlap / Math.max(1, Math.min(left.size, right.size));
7702
+ }
7703
+ function normalizedTerm(value) {
7704
+ return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
7705
+ }
7706
+ function metadataString(value) {
7707
+ return typeof value === 'string' && value.trim() ? value.trim() : undefined;
7708
+ }
7709
+ function metadataStringArray(value) {
7710
+ if (!Array.isArray(value))
7711
+ return [];
7712
+ return value.map((item) => String(item).trim()).filter(Boolean).map(normalizedTerm);
7713
+ }
7714
+ function isWeakDqlGenerationName(name) {
7715
+ return /^(pasted|query|source|imported query|sql import)(?:\s+\d+)?$/i.test(name.trim());
7716
+ }
7717
+ function isWeakDqlGenerationDescription(description) {
7718
+ return /^draft dql block summarizing imported sql/i.test(description.trim())
7719
+ || /^draft dql block generated from imported sql/i.test(description.trim())
7720
+ || /^imported from /i.test(description.trim())
7721
+ || /^this query (shows|returns|provides)/i.test(description.trim());
7722
+ }
7723
+ function isUngroundedDqlGenerationDescription(description, candidate, evidence) {
7724
+ const text = description.toLowerCase();
7725
+ const tokens = new Set();
7726
+ for (const tag of candidate.tags) {
7727
+ if (!/^(raw-sql|imported|ai-generated|review-required)$/i.test(tag) && tag.length > 2)
7728
+ tokens.add(tag.toLowerCase());
5026
7729
  }
5027
- else {
5028
- source = source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
7730
+ for (const table of candidate.lineage.sourceTables) {
7731
+ for (const token of table.split(/[._\s/-]+/)) {
7732
+ if (token.length > 2)
7733
+ tokens.add(token.toLowerCase());
7734
+ }
5029
7735
  }
5030
- writeFileSync(absPath, source, 'utf-8');
5031
- const companionPath = blockCompanionRelativePath(normalizedPath);
5032
- if (!companionPath)
5033
- return;
5034
- const absCompanionPath = join(projectRoot, companionPath);
5035
- if (!existsSync(absCompanionPath))
5036
- return;
5037
- let companion = readFileSync(absCompanionPath, 'utf-8');
5038
- if (/^reviewStatus:\s*.+$/m.test(companion)) {
5039
- companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
7736
+ for (const item of evidence.slice(0, 6)) {
7737
+ for (const token of item.name.split(/[._\s/-]+/)) {
7738
+ if (token.length > 3 && !/^(model|source|semantic|table)$/i.test(token))
7739
+ tokens.add(token.toLowerCase());
7740
+ }
5040
7741
  }
5041
- else {
5042
- companion = `${companion.trimEnd()}\nreviewStatus: ${newStatus}\n`;
7742
+ if (tokens.size === 0)
7743
+ return false;
7744
+ return !Array.from(tokens).some((token) => text.includes(token));
7745
+ }
7746
+ function dqlGenerationConversionNotes(providerName) {
7747
+ return [
7748
+ `Generated with ${providerName} using local project context when available.`,
7749
+ 'Runtime-scope SQL literals are converted into DQL params when safe; business constants stay static for review.',
7750
+ 'Certified and draft block similarity is checked before a new draft is saved.',
7751
+ 'SQL semantics are preserved while reusable parameter placeholders may replace literal values.',
7752
+ 'Drafts are autosaved under blocks/_drafts or domains/<domain>/blocks/_drafts before certification.',
7753
+ 'Preview results and tests gate certification before promotion.',
7754
+ ];
7755
+ }
7756
+ function setBlockStudioSourceStatus(source, status) {
7757
+ if (/status\s*=\s*"[^"]*"/.test(source)) {
7758
+ return source.replace(/status\s*=\s*"[^"]*"/, `status = "${status}"`);
5043
7759
  }
5044
- writeFileSync(absCompanionPath, companion, 'utf-8');
7760
+ return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${status}"`);
5045
7761
  }
5046
7762
  async function buildBlockStudioAiAssistSummary(projectRoot, action, candidate, validation, requestedProvider) {
5047
7763
  const fallback = buildDeterministicAiAssistSummary(action, candidate, validation);
@@ -5249,6 +7965,14 @@ export function readGitMetadata(projectRoot) {
5249
7965
  return null;
5250
7966
  }
5251
7967
  }
7968
+ function resolveBlockWriteTarget(projectRoot, safeDomain, slug) {
7969
+ if (safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))) {
7970
+ const relativePath = `domains/${safeDomain}/blocks/${slug}.dql`;
7971
+ return { relativePath, absPath: join(projectRoot, relativePath) };
7972
+ }
7973
+ const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
7974
+ return { relativePath, absPath: join(projectRoot, relativePath) };
7975
+ }
5252
7976
  export function createBlockArtifacts(projectRoot, options) {
5253
7977
  const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
5254
7978
  const safeDomain = (options.domain ?? '')
@@ -5256,16 +7980,16 @@ export function createBlockArtifacts(projectRoot, options) {
5256
7980
  .toLowerCase()
5257
7981
  .replace(/[^a-z0-9/_-]+/g, '-')
5258
7982
  .replace(/^\/+|\/+$/g, '');
5259
- const blocksDir = safeDomain ? join(projectRoot, 'blocks', safeDomain) : join(projectRoot, 'blocks');
5260
- mkdirSync(blocksDir, { recursive: true });
5261
- const blockPath = join(blocksDir, `${slug}.dql`);
7983
+ const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
7984
+ mkdirSync(dirname(target.absPath), { recursive: true });
7985
+ const blockPath = target.absPath;
5262
7986
  if (existsSync(blockPath)) {
5263
7987
  throw new Error('BLOCK_EXISTS');
5264
7988
  }
5265
7989
  const templateContent = options.template
5266
7990
  ? listBlockTemplates().find((template) => template.id === options.template)?.content
5267
7991
  : undefined;
5268
- const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
7992
+ const relativePath = target.relativePath;
5269
7993
  const fileContent = canonicalizeSafe(options.blockType === 'semantic' && !options.content?.trim() && !templateContent
5270
7994
  ? buildBlankSemanticBlockContent({
5271
7995
  name: options.name,
@@ -5311,9 +8035,9 @@ export function createSemanticBuilderBlock(projectRoot, options) {
5311
8035
  .toLowerCase()
5312
8036
  .replace(/[^a-z0-9/_-]+/g, '-')
5313
8037
  .replace(/^\/+|\/+$/g, '') || 'uncategorized';
5314
- const blocksDir = join(projectRoot, 'blocks', safeDomain);
5315
- mkdirSync(blocksDir, { recursive: true });
5316
- const blockPath = join(blocksDir, `${slug}.dql`);
8038
+ const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
8039
+ mkdirSync(dirname(target.absPath), { recursive: true });
8040
+ const blockPath = target.absPath;
5317
8041
  if (existsSync(blockPath)) {
5318
8042
  throw new Error('BLOCK_EXISTS');
5319
8043
  }
@@ -5338,7 +8062,7 @@ export function createSemanticBuilderBlock(projectRoot, options) {
5338
8062
  ],
5339
8063
  });
5340
8064
  return {
5341
- path: `blocks/${safeDomain}/${slug}.dql`,
8065
+ path: target.relativePath,
5342
8066
  content,
5343
8067
  companionPath,
5344
8068
  };
@@ -6100,18 +8824,16 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
6100
8824
  const sourceCount = manifest?.sources && typeof manifest.sources === 'object'
6101
8825
  ? Object.keys(manifest.sources).length
6102
8826
  : 0;
6103
- const manifestMetricCount = manifest?.metrics && typeof manifest.metrics === 'object'
6104
- ? Object.keys(manifest.metrics).length
6105
- : 0;
6106
- const semanticMetricCount = Array.isArray(semanticManifest?.metrics)
6107
- ? semanticManifest.metrics.length
8827
+ const manifestMetricCount = artifactCollectionCount(manifest?.metrics);
8828
+ const semanticMetricCount = semanticManifest
8829
+ ? artifactCollectionCount(semanticManifest.metrics)
6108
8830
  : manifestMetricCount;
6109
- const semanticModelCount = Array.isArray(semanticManifest?.semantic_models)
6110
- ? semanticManifest.semantic_models.length
6111
- : 0;
6112
- const savedQueryCount = Array.isArray(semanticManifest?.saved_queries)
6113
- ? semanticManifest.saved_queries.length
6114
- : 0;
8831
+ const semanticModelCount = semanticManifest
8832
+ ? artifactCollectionCount(semanticManifest.semantic_models)
8833
+ : artifactCollectionCount(manifest?.semantic_models);
8834
+ const savedQueryCount = semanticManifest
8835
+ ? artifactCollectionCount(semanticManifest.saved_queries ?? semanticManifest.savedQueries)
8836
+ : artifactCollectionCount(manifest?.saved_queries ?? manifest?.savedQueries);
6115
8837
  const configured = existsSync(join(dbtProjectPath, 'dbt_project.yml'))
6116
8838
  || Boolean(projectConfig.dbt?.projectDir)
6117
8839
  || Boolean(projectConfig.semanticLayer?.provider === 'dbt' && projectConfig.semanticLayer.projectPath);
@@ -6123,7 +8845,9 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
6123
8845
  ? 'Run `dbt parse`, `dbt compile`, or `dbt build`, then run `dql sync dbt`.'
6124
8846
  : !semanticExists
6125
8847
  ? 'dbt manifest is ready. Run `dbt parse` or `dbt build` if you use dbt Semantic Layer metrics.'
6126
- : 'dbt artifacts are ready. Build SQL blocks from models or semantic blocks from metrics.';
8848
+ : semanticMetricCount === 0 && semanticModelCount === 0 && savedQueryCount === 0
8849
+ ? 'dbt manifest is ready, but MetricFlow semantic_manifest.json is empty. Add dbt semantic_models and metrics, then rerun dbt parse/build.'
8850
+ : 'dbt artifacts are ready. Build SQL blocks from models or semantic blocks from metrics.';
6127
8851
  return {
6128
8852
  configured,
6129
8853
  provider: projectConfig.semanticLayer?.provider ?? null,
@@ -6146,6 +8870,185 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
6146
8870
  setupHint,
6147
8871
  };
6148
8872
  }
8873
+ export function buildSemanticLayerDiagnostics(projectRoot, projectConfig, options) {
8874
+ const provider = options.semanticConfig?.provider ?? options.detectedProvider ?? null;
8875
+ const dbt = buildDbtStatus(projectRoot, projectConfig, options.lastSyncTime);
8876
+ const counts = semanticLayerCounts(options.semanticLayer);
8877
+ const uncategorizedObjects = countUncategorizedSemanticObjects(options.semanticLayer);
8878
+ const totalObjects = counts.metrics
8879
+ + counts.measures
8880
+ + counts.dimensions
8881
+ + counts.timeDimensions
8882
+ + counts.entities
8883
+ + counts.hierarchies
8884
+ + counts.semanticModels
8885
+ + counts.savedQueries;
8886
+ const issues = [];
8887
+ for (const error of options.semanticErrors ?? []) {
8888
+ issues.push({
8889
+ severity: 'error',
8890
+ code: 'semantic_load_error',
8891
+ message: error,
8892
+ action: 'Open the semantic source configuration, fix the provider error, then reload the semantic layer.',
8893
+ });
8894
+ }
8895
+ if (!provider) {
8896
+ issues.push({
8897
+ severity: existsSync(join(projectRoot, 'semantic-layer')) ? 'info' : 'warning',
8898
+ code: 'semantic_provider_not_configured',
8899
+ message: 'No semantic layer provider is configured.',
8900
+ action: 'Configure dbt MetricFlow, local semantic-layer YAML, or Snowflake Semantic Views from the setup flow.',
8901
+ });
8902
+ }
8903
+ if (provider === 'dbt' || dbt.configured) {
8904
+ if (!dbt.configured) {
8905
+ issues.push({
8906
+ severity: 'warning',
8907
+ code: 'dbt_project_not_detected',
8908
+ message: 'dbt is selected as the semantic source, but no dbt project was detected.',
8909
+ action: 'Set dbt.projectDir or run DQL from a repository containing dbt_project.yml.',
8910
+ });
8911
+ }
8912
+ if (!dbt.artifacts.manifest.exists) {
8913
+ issues.push({
8914
+ severity: 'error',
8915
+ code: 'dbt_manifest_missing',
8916
+ message: 'dbt manifest.json is missing, so DQL cannot build dbt model context.',
8917
+ action: 'Run dbt parse, dbt compile, or dbt build, then reload the semantic layer.',
8918
+ path: dbt.artifacts.manifest.path,
8919
+ });
8920
+ }
8921
+ if (!dbt.artifacts.semanticManifest.exists) {
8922
+ issues.push({
8923
+ severity: 'warning',
8924
+ code: 'metricflow_semantic_manifest_missing',
8925
+ message: 'dbt MetricFlow semantic_manifest.json is missing. DQL can still inspect dbt models, but MetricFlow metrics and saved queries may be incomplete.',
8926
+ action: 'Run dbt parse or dbt build with semantic models configured, then reload. For execution, ensure MetricFlow is installed and mf is on PATH.',
8927
+ path: dbt.artifacts.semanticManifest.path,
8928
+ });
8929
+ }
8930
+ else if (dbt.counts.metrics === 0 && dbt.counts.semanticModels === 0 && dbt.counts.savedQueries === 0) {
8931
+ issues.push({
8932
+ severity: 'warning',
8933
+ code: 'metricflow_semantic_manifest_empty',
8934
+ message: 'semantic_manifest.json exists, but it does not contain metrics, semantic models, or saved queries.',
8935
+ action: 'Check dbt semantic model YAML definitions and rerun dbt parse.',
8936
+ path: dbt.artifacts.semanticManifest.path,
8937
+ });
8938
+ }
8939
+ if (options.semanticLayer && totalObjects === 0 && (dbt.counts.models > 0 || dbt.counts.sources > 0)) {
8940
+ issues.push({
8941
+ severity: 'warning',
8942
+ code: 'dbt_model_inventory_only',
8943
+ message: 'DQL loaded dbt model inventory, but no MetricFlow semantic metrics were found.',
8944
+ action: 'Create dbt semantic_models and metrics, or use AI Import to build DQL blocks from dbt model metadata.',
8945
+ });
8946
+ }
8947
+ }
8948
+ if (provider === 'snowflake') {
8949
+ if (!options.semanticLayer && (options.semanticErrors ?? []).length === 0) {
8950
+ issues.push({
8951
+ severity: 'warning',
8952
+ code: 'snowflake_semantic_views_not_loaded',
8953
+ message: 'Snowflake semantic provider did not load semantic views.',
8954
+ action: 'Confirm the active Snowflake connection can query semantic views, then reload. Warehouse tables still feed database context separately.',
8955
+ });
8956
+ }
8957
+ else if (options.semanticLayer && totalObjects === 0) {
8958
+ issues.push({
8959
+ severity: 'info',
8960
+ code: 'snowflake_semantic_views_empty',
8961
+ message: 'Snowflake semantic provider loaded, but no semantic view objects were discovered.',
8962
+ action: 'Check that Snowflake Semantic Views exist and the active role can see them.',
8963
+ });
8964
+ }
8965
+ }
8966
+ if (options.semanticLayer && totalObjects === 0 && issues.every((issue) => issue.code !== 'dbt_model_inventory_only')) {
8967
+ issues.push({
8968
+ severity: 'warning',
8969
+ code: 'semantic_layer_empty',
8970
+ message: 'Semantic layer is available, but no semantic objects are loaded.',
8971
+ action: 'Import semantic definitions or run dbt parse/build, then reload.',
8972
+ });
8973
+ }
8974
+ if (uncategorizedObjects > 0) {
8975
+ issues.push({
8976
+ severity: 'info',
8977
+ code: 'semantic_uncategorized_domain',
8978
+ message: `${uncategorizedObjects} semantic object${uncategorizedObjects === 1 ? '' : 's'} had no domain and are visible under "uncategorized".`,
8979
+ action: 'Add domain metadata in dbt semantic YAML or local semantic-layer files when domain ownership matters.',
8980
+ });
8981
+ }
8982
+ const warnings = Array.from(new Set([
8983
+ ...(options.semanticErrors ?? []),
8984
+ ...issues
8985
+ .filter((issue) => issue.severity !== 'info')
8986
+ .map((issue) => issue.message),
8987
+ ]));
8988
+ return {
8989
+ available: Boolean(options.semanticLayer),
8990
+ provider,
8991
+ sourceOfTruth: provider === 'dbt'
8992
+ ? 'dbt MetricFlow semantic_manifest.json, dbt manifest.json semantic nodes, then dbt semantic YAML fallback'
8993
+ : provider === 'snowflake'
8994
+ ? 'Snowflake Semantic Views only; warehouse tables remain database catalog context'
8995
+ : provider === 'dql'
8996
+ ? 'local semantic-layer YAML'
8997
+ : 'not configured',
8998
+ errors: options.semanticErrors ?? [],
8999
+ lastSyncTime: options.lastSyncTime,
9000
+ counts,
9001
+ dbt,
9002
+ issues,
9003
+ warnings,
9004
+ };
9005
+ }
9006
+ function semanticLayerCounts(layer) {
9007
+ return layer
9008
+ ? {
9009
+ domains: layer.listDomains().length,
9010
+ metrics: layer.listMetrics().length,
9011
+ measures: layer.listMeasures().length,
9012
+ dimensions: layer.listDimensions().length,
9013
+ timeDimensions: layer.listTimeDimensions().length,
9014
+ entities: layer.listEntities().length,
9015
+ hierarchies: layer.listHierarchies().length,
9016
+ semanticModels: layer.listSemanticModels().length,
9017
+ savedQueries: layer.listSavedQueries().length,
9018
+ }
9019
+ : {
9020
+ domains: 0,
9021
+ metrics: 0,
9022
+ measures: 0,
9023
+ dimensions: 0,
9024
+ timeDimensions: 0,
9025
+ entities: 0,
9026
+ hierarchies: 0,
9027
+ semanticModels: 0,
9028
+ savedQueries: 0,
9029
+ };
9030
+ }
9031
+ function countUncategorizedSemanticObjects(layer) {
9032
+ if (!layer)
9033
+ return 0;
9034
+ return [
9035
+ ...layer.listMetrics(),
9036
+ ...layer.listMeasures(),
9037
+ ...layer.listDimensions(),
9038
+ ...layer.listTimeDimensions(),
9039
+ ...layer.listEntities(),
9040
+ ...layer.listHierarchies(),
9041
+ ...layer.listSemanticModels(),
9042
+ ...layer.listSavedQueries(),
9043
+ ].filter((object) => !object.domain || !object.domain.trim()).length;
9044
+ }
9045
+ function artifactCollectionCount(value) {
9046
+ if (Array.isArray(value))
9047
+ return value.length;
9048
+ if (value && typeof value === 'object')
9049
+ return Object.keys(value).length;
9050
+ return 0;
9051
+ }
6149
9052
  function describeArtifact(path, count, generatedAt) {
6150
9053
  return {
6151
9054
  path,
@@ -6682,6 +9585,11 @@ function isProviderSettingsId(value) {
6682
9585
  || value === 'ollama'
6683
9586
  || value === 'custom-openai';
6684
9587
  }
9588
+ function isDeterministicDqlGenerationProvider(value) {
9589
+ if (typeof value !== 'string')
9590
+ return false;
9591
+ return value === 'none' || value === 'deterministic' || value === 'local-deterministic';
9592
+ }
6685
9593
  async function testProviderConfig(projectRoot, id) {
6686
9594
  const config = getEffectiveProviderConfig(projectRoot, id);
6687
9595
  const label = providerSettingsLabel(id);
@@ -6831,6 +9739,655 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
6831
9739
  // Runtime schema snapshots are advisory local metadata and must not block answers.
6832
9740
  }
6833
9741
  }
9742
+ function notebookResearchString(value) {
9743
+ return typeof value === 'string' && value.trim() ? value.trim() : undefined;
9744
+ }
9745
+ function notebookResearchRecord(value) {
9746
+ return value && typeof value === 'object' && !Array.isArray(value)
9747
+ ? value
9748
+ : null;
9749
+ }
9750
+ function notebookResearchSourceCellPayload(body) {
9751
+ const sourceCell = notebookResearchRecord(body.sourceCell)
9752
+ ?? notebookResearchRecord(notebookResearchRecord(body.context)?.sourceCell);
9753
+ if (!sourceCell)
9754
+ return undefined;
9755
+ const payload = {
9756
+ id: notebookResearchString(sourceCell.id),
9757
+ sourceCellId: notebookResearchString(sourceCell.sourceCellId),
9758
+ cellId: notebookResearchString(sourceCell.cellId),
9759
+ name: notebookResearchString(sourceCell.name),
9760
+ sourceCellName: notebookResearchString(sourceCell.sourceCellName),
9761
+ title: notebookResearchString(sourceCell.title),
9762
+ fingerprint: notebookResearchString(sourceCell.fingerprint),
9763
+ sourceCellFingerprint: notebookResearchString(sourceCell.sourceCellFingerprint),
9764
+ sqlFingerprint: notebookResearchString(sourceCell.sqlFingerprint),
9765
+ type: notebookResearchString(sourceCell.type),
9766
+ sql: notebookResearchString(sourceCell.sql),
9767
+ content: notebookResearchString(sourceCell.content),
9768
+ source: notebookResearchString(sourceCell.source),
9769
+ };
9770
+ return Object.values(payload).some(Boolean) ? payload : undefined;
9771
+ }
9772
+ function notebookResearchSourceCellId(sourceCell) {
9773
+ return notebookResearchString(sourceCell?.id)
9774
+ ?? notebookResearchString(sourceCell?.sourceCellId)
9775
+ ?? notebookResearchString(sourceCell?.cellId);
9776
+ }
9777
+ function notebookResearchSourceCellName(sourceCell) {
9778
+ return notebookResearchString(sourceCell?.name)
9779
+ ?? notebookResearchString(sourceCell?.sourceCellName)
9780
+ ?? notebookResearchString(sourceCell?.title);
9781
+ }
9782
+ function notebookResearchSourceCellFingerprint(sourceCell) {
9783
+ return notebookResearchString(sourceCell?.sourceCellFingerprint)
9784
+ ?? notebookResearchString(sourceCell?.fingerprint)
9785
+ ?? notebookResearchString(sourceCell?.sqlFingerprint);
9786
+ }
9787
+ function notebookResearchPatchString(body, key) {
9788
+ if (!Object.prototype.hasOwnProperty.call(body, key))
9789
+ return undefined;
9790
+ return notebookResearchString(body[key]) ?? null;
9791
+ }
9792
+ function notebookResearchIntent(value) {
9793
+ return value === 'ad_hoc_analysis'
9794
+ || value === 'diagnose_change'
9795
+ || value === 'driver_breakdown'
9796
+ || value === 'segment_compare'
9797
+ || value === 'entity_drilldown'
9798
+ || value === 'anomaly_investigation'
9799
+ || value === 'trust_gap_review'
9800
+ ? value
9801
+ : undefined;
9802
+ }
9803
+ function notebookResearchReviewStatus(value) {
9804
+ return value === 'needs_review'
9805
+ || value === 'draft_created'
9806
+ || value === 'completed'
9807
+ || value === 'certified'
9808
+ || value === 'rejected'
9809
+ ? value
9810
+ : undefined;
9811
+ }
9812
+ function notebookResearchStatus(value) {
9813
+ return value === 'draft'
9814
+ || value === 'running'
9815
+ || value === 'ready'
9816
+ || value === 'error'
9817
+ ? value
9818
+ : undefined;
9819
+ }
9820
+ function notebookResearchPromotionAction(value) {
9821
+ return value === 'reuse_existing'
9822
+ || value === 'extend_existing'
9823
+ || value === 'create_replacement'
9824
+ || value === 'create_new'
9825
+ || value === 'review_required'
9826
+ ? value
9827
+ : undefined;
9828
+ }
9829
+ function notebookResearchReadiness(value) {
9830
+ return value === 'draft_ready' || value === 'certification_ready' || value === 'blocked' ? value : undefined;
9831
+ }
9832
+ function notebookResearchAge(value) {
9833
+ return value === 'stale_open' || value === 'expired_open' ? value : undefined;
9834
+ }
9835
+ function notebookResearchNextAction(value) {
9836
+ return value === 'fix_blockers'
9837
+ || value === 'review_sql'
9838
+ || value === 'review_context'
9839
+ || value === 'run_preview'
9840
+ || value === 'reuse_existing'
9841
+ || value === 'create_dql_draft'
9842
+ || value === 'open_certification'
9843
+ || value === 'complete_review'
9844
+ || value === 'continue_review'
9845
+ ? value
9846
+ : undefined;
9847
+ }
9848
+ function notebookResearchSort(value) {
9849
+ return value === 'priority' || value === 'updated_desc' ? value : undefined;
9850
+ }
9851
+ function withNotebookResearchChecklistPage(page) {
9852
+ return {
9853
+ ...page,
9854
+ runs: page.runs.map(withNotebookResearchChecklist),
9855
+ };
9856
+ }
9857
+ function withNotebookResearchChecklist(run) {
9858
+ const researchPlan = run.researchPlan ?? buildNotebookResearchPlan({
9859
+ run,
9860
+ evidence: run.evidence,
9861
+ resultPreview: run.resultPreview,
9862
+ previewError: run.status === 'error' ? run.error : undefined,
9863
+ generatedSql: run.generatedSql,
9864
+ reviewedSql: run.reviewedSql,
9865
+ routeDecision: run.routeDecision,
9866
+ });
9867
+ return {
9868
+ ...run,
9869
+ researchPlan,
9870
+ reviewChecklist: buildNotebookResearchReviewChecklist(run),
9871
+ };
9872
+ }
9873
+ function buildNotebookResearchReviewChecklist(run) {
9874
+ const questionReady = Boolean(notebookResearchString(run.question));
9875
+ const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
9876
+ const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
9877
+ const hasSql = hasReviewedSql || hasGeneratedSql;
9878
+ const preview = notebookResearchPreviewInfo(run.resultPreview);
9879
+ const previewReady = run.status === 'ready' && preview.hasPreview;
9880
+ const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
9881
+ const promoted = Boolean(run.dqlPromotion || run.dqlImportId || run.draftBlockPath);
9882
+ const duplicateChecked = Boolean(run.dqlPromotionAction || run.dqlPromotion);
9883
+ const promotionAction = run.dqlPromotionAction ?? run.dqlPromotion?.recommendedAction;
9884
+ const reuseRecommended = promotionAction === 'reuse_existing';
9885
+ const promotionReviewRequired = promotionAction === 'review_required';
9886
+ const hasDraft = Boolean(notebookResearchString(run.draftBlockPath));
9887
+ const sqlForParameterReview = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
9888
+ const parameterReview = sqlForParameterReview ? parameterizeSqlForDqlImport(sqlForParameterReview) : null;
9889
+ const dynamicParameters = parameterReview?.parameterPolicy.filter((item) => item.policy === 'dynamic') ?? [];
9890
+ const staticParameters = parameterReview?.parameterPolicy.filter((item) => item.policy !== 'dynamic') ?? [];
9891
+ const blockers = [];
9892
+ const warnings = [];
9893
+ if (!questionReady)
9894
+ blockers.push('Add a business question before review.');
9895
+ if (!hasSql)
9896
+ blockers.push('Add reviewed SQL or generate SQL before promotion.');
9897
+ if (run.status === 'error')
9898
+ blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
9899
+ if (!previewReady && run.status !== 'error')
9900
+ warnings.push('Run a bounded preview before certification review.');
9901
+ if (!evidenceReady)
9902
+ warnings.push('Inspect metadata evidence before turning this into a reusable block.');
9903
+ if (hasSql && dynamicParameters.length === 0)
9904
+ warnings.push('No dynamic parameters were detected; confirm this should be a static block before certification.');
9905
+ for (const warning of parameterReview?.warnings ?? [])
9906
+ warnings.push(warning);
9907
+ if (promoted && !duplicateChecked)
9908
+ warnings.push('Promotion exists but duplicate/reuse evidence is missing.');
9909
+ if (reuseRecommended)
9910
+ warnings.push('Reuse the matching block or explicitly document a replacement before certification.');
9911
+ if (promotionReviewRequired)
9912
+ warnings.push('Resolve the review-required promotion decision before certification.');
9913
+ const items = [
9914
+ {
9915
+ id: 'question',
9916
+ label: 'Question',
9917
+ status: questionReady ? 'passed' : 'blocked',
9918
+ detail: questionReady ? 'Business question is captured.' : 'A reusable block needs a clear business question.',
9919
+ },
9920
+ {
9921
+ id: 'sql',
9922
+ label: 'Reviewed SQL',
9923
+ status: hasReviewedSql ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
9924
+ detail: hasReviewedSql
9925
+ ? 'Reviewer SQL is saved.'
9926
+ : hasGeneratedSql
9927
+ ? 'Generated SQL exists; review it before promotion.'
9928
+ : 'No SQL is available for preview or DQL generation.',
9929
+ },
9930
+ {
9931
+ id: 'preview',
9932
+ label: 'Preview',
9933
+ status: previewReady ? 'passed' : run.status === 'error' ? 'blocked' : 'pending',
9934
+ detail: previewReady
9935
+ ? `Preview returned ${preview.rowCount.toLocaleString()} row${preview.rowCount === 1 ? '' : 's'}.`
9936
+ : run.status === 'error'
9937
+ ? (run.error ?? 'Preview failed.')
9938
+ : 'Run a bounded preview to validate the query shape.',
9939
+ },
9940
+ {
9941
+ id: 'evidence',
9942
+ label: 'Evidence',
9943
+ status: evidenceReady ? 'passed' : 'warning',
9944
+ detail: evidenceReady ? 'Metadata evidence or context pack is attached.' : 'No context evidence is attached yet.',
9945
+ },
9946
+ {
9947
+ id: 'duplicates',
9948
+ label: 'Reuse check',
9949
+ status: duplicateChecked ? (reuseRecommended || promotionReviewRequired ? 'warning' : 'passed') : promoted ? 'warning' : 'pending',
9950
+ detail: duplicateChecked
9951
+ ? `Promotion decision: ${promotionAction ?? 'review required'}.`
9952
+ : promoted
9953
+ ? 'Draft was created without stored duplicate evidence.'
9954
+ : 'Duplicate check runs when creating a DQL draft.',
9955
+ },
9956
+ {
9957
+ id: 'parameters',
9958
+ label: 'Parameter review',
9959
+ status: !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
9960
+ detail: !hasSql
9961
+ ? 'Add SQL before checking runtime parameters.'
9962
+ : dynamicParameters.length > 0
9963
+ ? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
9964
+ : staticParameters.length > 0
9965
+ ? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
9966
+ : 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
9967
+ },
9968
+ {
9969
+ id: 'dql_draft',
9970
+ label: 'DQL draft',
9971
+ status: hasDraft ? 'passed' : hasReviewedSql && evidenceReady ? 'pending' : 'blocked',
9972
+ detail: hasDraft
9973
+ ? `Draft saved at ${run.draftBlockPath}.`
9974
+ : !hasReviewedSql
9975
+ ? 'Save reviewed SQL before DQL draft creation.'
9976
+ : evidenceReady
9977
+ ? 'Create a draft after SQL review.'
9978
+ : 'Save metadata evidence before creating a reusable DQL draft.',
9979
+ },
9980
+ ];
9981
+ const readyForDqlDraft = questionReady && hasReviewedSql && evidenceReady && run.status !== 'error';
9982
+ const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
9983
+ return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
9984
+ }
9985
+ function buildNotebookResearchPlan(input) {
9986
+ const reviewedSql = notebookResearchString(input.reviewedSql);
9987
+ const generatedSql = notebookResearchString(input.generatedSql);
9988
+ const sql = reviewedSql ?? generatedSql;
9989
+ const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
9990
+ const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
9991
+ ? input.evidence
9992
+ : {};
9993
+ const trustStatus = evidence.trustStatus && typeof evidence.trustStatus === 'object' && !Array.isArray(evidence.trustStatus)
9994
+ ? evidence.trustStatus
9995
+ : {};
9996
+ const allowedSqlContext = evidence.allowedSqlContext && typeof evidence.allowedSqlContext === 'object' && !Array.isArray(evidence.allowedSqlContext)
9997
+ ? evidence.allowedSqlContext
9998
+ : {};
9999
+ const previewRows = input.resultPreview?.rowCount ?? input.resultPreview?.rows.length;
10000
+ const hasEvidence = notebookResearchEvidenceCount(input.evidence) > 0 || Boolean(input.run.contextPackId);
10001
+ const hasPreview = previewRows !== undefined && (previewRows > 0 || Boolean(input.resultPreview?.columns.length));
10002
+ const promotionAction = input.run.dqlPromotionAction ?? input.run.dqlPromotion?.recommendedAction;
10003
+ const missingContext = Array.isArray(evidence.missingContext) ? evidence.missingContext : [];
10004
+ const relations = Array.isArray(allowedSqlContext.relations) ? allowedSqlContext.relations : [];
10005
+ return {
10006
+ sqlState: reviewedSql ? 'reviewed' : generatedSql ? 'generated' : 'missing',
10007
+ grain: sql ? extractDqlGenerationGroupByFields(sql)[0] : undefined,
10008
+ parameterPolicy: (parameterized?.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
10009
+ name: entry.name,
10010
+ policy: entry.policy,
10011
+ })),
10012
+ allowedFilters: (parameterized?.allowedFilters ?? []).slice(0, 16),
10013
+ evidence: {
10014
+ trustLabel: notebookResearchString(trustStatus.label),
10015
+ contextPackId: notebookResearchString(evidence.contextPackId) ?? input.run.contextPackId,
10016
+ evidenceCount: notebookResearchEvidenceCount(input.evidence),
10017
+ relationCount: relations.length,
10018
+ missingContextCount: missingContext.length,
10019
+ },
10020
+ preview: {
10021
+ status: input.previewError ? 'error' : hasPreview ? 'ready' : 'not_run',
10022
+ ...(previewRows === undefined ? {} : { rowCount: previewRows }),
10023
+ },
10024
+ promotion: {
10025
+ path: notebookResearchPlanPromotionPath({
10026
+ hasSql: Boolean(sql),
10027
+ hasEvidence,
10028
+ hasPreview,
10029
+ previewError: input.previewError,
10030
+ draftBlockPath: input.run.draftBlockPath,
10031
+ promotionAction,
10032
+ }),
10033
+ duplicateDecision: promotionAction,
10034
+ },
10035
+ reviewFocus: notebookResearchPlanFocus(input.run.intent, {
10036
+ hasSql: Boolean(sql),
10037
+ hasEvidence,
10038
+ hasPreview,
10039
+ dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
10040
+ missingContextCount: missingContext.length,
10041
+ promotionAction,
10042
+ }),
10043
+ generatedAt: new Date().toISOString(),
10044
+ };
10045
+ }
10046
+ function notebookResearchPlanPromotionPath(input) {
10047
+ if (!input.hasSql)
10048
+ return 'needs_sql';
10049
+ if (!input.hasEvidence)
10050
+ return 'review_context';
10051
+ if (input.previewError || !input.hasPreview)
10052
+ return 'run_preview';
10053
+ if (input.promotionAction === 'reuse_existing')
10054
+ return 'reuse_existing';
10055
+ if (input.draftBlockPath)
10056
+ return 'open_certification';
10057
+ return 'create_dql_draft';
10058
+ }
10059
+ function notebookResearchPlanFocus(intent, input) {
10060
+ const focus = new Set();
10061
+ if (!input.hasSql)
10062
+ focus.add('Add reviewed SQL or configure AI SQL generation.');
10063
+ if (!input.hasEvidence)
10064
+ focus.add('Save metadata context evidence before DQL promotion.');
10065
+ if (!input.hasPreview)
10066
+ focus.add('Run a bounded preview and inspect result shape.');
10067
+ if (input.dynamicParameterCount === 0)
10068
+ focus.add('Confirm whether this should be static or parameterized.');
10069
+ if (input.missingContextCount > 0)
10070
+ focus.add('Resolve missing metadata context before certification.');
10071
+ if (input.promotionAction === 'reuse_existing')
10072
+ focus.add('Reuse the existing certified block or document replacement rationale.');
10073
+ const pattern = notebookResearchIntentPattern(intent);
10074
+ const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus : [];
10075
+ for (const item of reviewFocus.slice(0, 4))
10076
+ focus.add(String(item));
10077
+ return Array.from(focus).slice(0, 8);
10078
+ }
10079
+ function notebookResearchPreviewInfo(value) {
10080
+ if (!value || typeof value !== 'object' || Array.isArray(value))
10081
+ return { hasPreview: false, rowCount: 0 };
10082
+ const record = value;
10083
+ const rowCount = typeof record.rowCount === 'number' && Number.isFinite(record.rowCount)
10084
+ ? record.rowCount
10085
+ : Array.isArray(record.rows)
10086
+ ? record.rows.length
10087
+ : 0;
10088
+ const hasColumns = Array.isArray(record.columns) && record.columns.length > 0;
10089
+ return { hasPreview: hasColumns || rowCount > 0, rowCount };
10090
+ }
10091
+ function notebookResearchEvidenceCount(value) {
10092
+ if (!value || typeof value !== 'object' || Array.isArray(value))
10093
+ return 0;
10094
+ const record = value;
10095
+ const selected = Array.isArray(record.selectedEvidence) ? record.selectedEvidence.length : 0;
10096
+ const citations = Array.isArray(record.citations) ? record.citations.length : 0;
10097
+ const roles = Array.isArray(record.evidenceRoles) ? record.evidenceRoles.length : 0;
10098
+ return selected + citations + roles;
10099
+ }
10100
+ function notebookResearchInteger(value, fallback, min, max) {
10101
+ if (typeof value !== 'string' || !value.trim())
10102
+ return fallback;
10103
+ const parsed = Number(value);
10104
+ if (!Number.isFinite(parsed))
10105
+ return fallback;
10106
+ return Math.max(min, Math.min(max, Math.floor(parsed)));
10107
+ }
10108
+ function notebookExecutionContext(value) {
10109
+ if (!value || typeof value !== 'object')
10110
+ return null;
10111
+ const raw = value;
10112
+ const notebookPath = notebookResearchString(raw.notebookPath);
10113
+ if (!notebookPath)
10114
+ return null;
10115
+ return {
10116
+ notebookPath,
10117
+ cellId: notebookResearchString(raw.cellId),
10118
+ cellName: notebookResearchString(raw.cellName),
10119
+ researchRunId: notebookResearchString(raw.researchRunId),
10120
+ source: notebookResearchString(raw.source),
10121
+ };
10122
+ }
10123
+ function notebookResearchSelectedContext(run, context) {
10124
+ return {
10125
+ activeSurface: 'notebook',
10126
+ notebookPath: run.notebookPath,
10127
+ domain: run.domain,
10128
+ sourceCellId: run.sourceCellId,
10129
+ sourceCellName: run.sourceCellName,
10130
+ context,
10131
+ };
10132
+ }
10133
+ function notebookResearchIntentPattern(intent) {
10134
+ switch (intent) {
10135
+ case 'diagnose_change':
10136
+ return {
10137
+ label: 'Change diagnosis',
10138
+ dqlTarget: 'driver or trend block',
10139
+ sqlFocus: ['baseline period', 'comparison period', 'metric delta', 'driver dimensions'],
10140
+ reviewFocus: ['time grain', 'comparison filters', 'metric definition', 'explainability evidence'],
10141
+ };
10142
+ case 'driver_breakdown':
10143
+ return {
10144
+ label: 'Driver breakdown',
10145
+ dqlTarget: 'ranking or contribution block',
10146
+ sqlFocus: ['contribution metric', 'ranking', 'tie breaker', 'segment dimensions'],
10147
+ reviewFocus: ['metric grain', 'ranking direction', 'parameterized filters', 'duplicate block match'],
10148
+ };
10149
+ case 'segment_compare':
10150
+ return {
10151
+ label: 'Segment compare',
10152
+ dqlTarget: 'comparison block',
10153
+ sqlFocus: ['shared metric', 'segment fields', 'normalized comparisons', 'deltas'],
10154
+ reviewFocus: ['segment definitions', 'filter compatibility', 'comparison grain', 'semantic metric alignment'],
10155
+ };
10156
+ case 'entity_drilldown':
10157
+ return {
10158
+ label: 'Entity drilldown',
10159
+ dqlTarget: 'entity profile or drilldown block',
10160
+ sqlFocus: ['entity grain', 'stable identifier', 'detail fields', 'supporting metrics'],
10161
+ reviewFocus: ['entity contract', 'primary key', 'allowed filters', 'downstream drill path'],
10162
+ };
10163
+ case 'anomaly_investigation':
10164
+ return {
10165
+ label: 'Anomaly investigation',
10166
+ dqlTarget: 'monitoring or exception block',
10167
+ sqlFocus: ['observed value', 'expected range', 'anomaly rule', 'supporting dimensions'],
10168
+ reviewFocus: ['threshold definition', 'time window', 'false-positive risk', 'evidence fields'],
10169
+ };
10170
+ case 'trust_gap_review':
10171
+ return {
10172
+ label: 'Trust review',
10173
+ dqlTarget: 'replacement or validation block',
10174
+ sqlFocus: ['definition comparison', 'lineage trace', 'duplicate candidates', 'conflict evidence'],
10175
+ reviewFocus: ['certified block match', 'semantic metric match', 'replacement reason', 'governance blockers'],
10176
+ };
10177
+ default:
10178
+ return {
10179
+ label: 'Ad hoc analysis',
10180
+ dqlTarget: 'custom block',
10181
+ sqlFocus: ['business question', 'grain', 'metrics', 'dimensions', 'filters'],
10182
+ reviewFocus: ['reusability', 'parameterization', 'lineage evidence', 'DQL promotion path'],
10183
+ };
10184
+ }
10185
+ }
10186
+ function notebookResearchAgentPrompt(question, intent) {
10187
+ const pattern = notebookResearchIntentPattern(intent);
10188
+ const sqlFocus = Array.isArray(pattern.sqlFocus) ? pattern.sqlFocus.join(', ') : 'business logic';
10189
+ const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus.join(', ') : 'review evidence';
10190
+ return [
10191
+ `Research question: ${question}`,
10192
+ `Research pattern: ${pattern.label ?? intent}`,
10193
+ `DQL target: ${pattern.dqlTarget ?? 'custom block'}`,
10194
+ `SQL focus: ${sqlFocus}`,
10195
+ `Review focus: ${reviewFocus}`,
10196
+ 'Generate read-only SQL that can become a reusable, parameterized DQL block after human review.',
10197
+ ].join('\n');
10198
+ }
10199
+ function notebookResearchRouteDecisionForRun(run, routeDecision, sqlForPreview) {
10200
+ if (!routeDecision)
10201
+ return undefined;
10202
+ const hasSelectedNotebookSql = Boolean(run.sourceCellId && notebookResearchString(sqlForPreview));
10203
+ if (!hasSelectedNotebookSql || routeDecision.route !== 'certified')
10204
+ return routeDecision;
10205
+ const certifiedEvidence = routeDecision.selectedEvidence.find((item) => item.role === 'exact_certified_answer')
10206
+ ?? routeDecision.selectedEvidence.find((item) => item.role === 'certified_context');
10207
+ const certifiedName = certifiedEvidence?.name ?? routeDecision.certifiedApplicability?.objectKey ?? 'matching certified artifact';
10208
+ return {
10209
+ ...routeDecision,
10210
+ route: 'research',
10211
+ trustLabel: routeDecision.trustLabel === 'certified' ? 'mixed' : routeDecision.trustLabel,
10212
+ reviewStatus: 'needs_review',
10213
+ exactObjectKey: undefined,
10214
+ reason: `Selected notebook SQL from "${run.sourceCellName ?? run.sourceCellId}" is the source for this research run. Certified artifact "${certifiedName}" is duplicate/reuse evidence until the reviewed SQL, grain, parameters, and preview are accepted.`,
10215
+ selectedEvidence: routeDecision.selectedEvidence.map((item) => item.role === 'exact_certified_answer'
10216
+ ? {
10217
+ ...item,
10218
+ role: 'certified_context',
10219
+ reason: 'Certified block is reuse or duplicate evidence for the selected notebook SQL; it is not an automatic answer for this research run.',
10220
+ }
10221
+ : item),
10222
+ missingContext: routeDecision.missingContext.length > 0
10223
+ ? routeDecision.missingContext
10224
+ : [{
10225
+ kind: 'metadata',
10226
+ severity: 'warning',
10227
+ message: 'Selected notebook SQL requires human review before DQL promotion, even when certified context exists.',
10228
+ }],
10229
+ };
10230
+ }
10231
+ function notebookResearchContextPreview(contextPack) {
10232
+ return {
10233
+ contextPackId: contextPack.id,
10234
+ trustLabel: contextPack.trustLabel,
10235
+ routeDecision: contextPack.routeDecision
10236
+ ? {
10237
+ route: contextPack.routeDecision.route,
10238
+ intent: contextPack.routeDecision.intent,
10239
+ reason: contextPack.routeDecision.reason,
10240
+ }
10241
+ : undefined,
10242
+ evidence: contextPack.evidenceRoles.slice(0, 16).map((item) => ({
10243
+ objectKey: item.objectKey,
10244
+ objectType: item.objectType,
10245
+ name: item.name,
10246
+ role: item.role,
10247
+ reason: item.reason,
10248
+ })),
10249
+ summaries: contextPack.evidenceSummaries.slice(0, 12).map((item) => ({
10250
+ title: item.title,
10251
+ detail: item.detail,
10252
+ objectType: item.objectType,
10253
+ reason: item.reason,
10254
+ })),
10255
+ relations: contextPack.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
10256
+ relation: relation.relation,
10257
+ name: relation.name,
10258
+ source: relation.source,
10259
+ columns: relation.columns.slice(0, 24).map((column) => {
10260
+ if (typeof column === 'string')
10261
+ return column;
10262
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
10263
+ return String(column.name);
10264
+ }
10265
+ return String(column);
10266
+ }),
10267
+ })),
10268
+ missingContext: contextPack.missingContext.slice(0, 12).map((item) => ({
10269
+ kind: item.kind,
10270
+ message: item.message,
10271
+ severity: item.severity,
10272
+ })),
10273
+ warnings: contextPack.warnings.slice(0, 12),
10274
+ topRejected: contextPack.retrievalDiagnostics.topRejected.slice(0, 8).map((item) => ({
10275
+ name: item.name,
10276
+ objectType: item.objectType,
10277
+ reason: item.reason,
10278
+ score: item.score,
10279
+ })),
10280
+ counts: {
10281
+ objects: contextPack.objects.length,
10282
+ evidence: contextPack.evidenceRoles.length,
10283
+ relations: contextPack.allowedSqlContext.relations.length,
10284
+ warnings: contextPack.warnings.length + contextPack.missingContext.length,
10285
+ },
10286
+ };
10287
+ }
10288
+ function normalizeNotebookAgentResult(result) {
10289
+ const columns = Array.isArray(result.columns)
10290
+ ? result.columns.map((column) => {
10291
+ if (typeof column === 'string')
10292
+ return column;
10293
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
10294
+ return String(column.name);
10295
+ }
10296
+ return String(column);
10297
+ })
10298
+ : [];
10299
+ const rows = Array.isArray(result.rows)
10300
+ ? result.rows
10301
+ .filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row)))
10302
+ .map((row) => row)
10303
+ : [];
10304
+ return {
10305
+ columns,
10306
+ rows,
10307
+ rowCount: typeof result.rowCount === 'number' ? result.rowCount : rows.length,
10308
+ executionTime: typeof result.executionTime === 'number' ? result.executionTime : 0,
10309
+ };
10310
+ }
10311
+ function notebookResearchSummary(question, result, error) {
10312
+ if (error)
10313
+ return `Research could not preview results for "${question}". ${error}`;
10314
+ if (result) {
10315
+ const rowCount = result.rowCount ?? result.rows.length;
10316
+ return `Research preview for "${question}" returned ${rowCount.toLocaleString()} row${rowCount === 1 ? '' : 's'} across ${result.columns.length.toLocaleString()} column${result.columns.length === 1 ? '' : 's'}.`;
10317
+ }
10318
+ return `Research plan created for "${question}". Add or generate SQL, then run a bounded preview.`;
10319
+ }
10320
+ function recordNotebookQueryRun(projectRoot, input) {
10321
+ try {
10322
+ recordQueryRun(projectRoot, {
10323
+ objectKey: input.objectKey,
10324
+ source: input.source,
10325
+ status: input.status,
10326
+ rowCount: input.rowCount,
10327
+ durationMs: input.durationMs,
10328
+ errorCode: input.errorCode,
10329
+ payload: {
10330
+ notebookPath: input.notebookPath,
10331
+ cellId: input.cellId,
10332
+ cellName: input.cellName,
10333
+ researchRunId: input.researchRunId,
10334
+ contextPackId: input.contextPackId,
10335
+ sql: input.sql ? compactSqlForRunHistory(input.sql) : undefined,
10336
+ },
10337
+ });
10338
+ }
10339
+ catch {
10340
+ // Query history is advisory metadata and must not block notebook execution.
10341
+ }
10342
+ }
10343
+ function updateNotebookResearchFromCellExecution(projectRoot, execContext, input) {
10344
+ if (!execContext.notebookPath || (!execContext.cellId && !execContext.researchRunId))
10345
+ return;
10346
+ const storage = new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
10347
+ try {
10348
+ const page = storage.listRunsPage({
10349
+ notebookPath: execContext.notebookPath,
10350
+ sort: 'updated_desc',
10351
+ limit: 500,
10352
+ });
10353
+ const matches = page.runs.filter((run) => {
10354
+ if (execContext.researchRunId && run.id === execContext.researchRunId)
10355
+ return true;
10356
+ return Boolean(execContext.cellId && run.sourceCellId === execContext.cellId);
10357
+ });
10358
+ const now = new Date().toISOString();
10359
+ for (const run of matches.slice(0, 20)) {
10360
+ if (input.status === 'success') {
10361
+ storage.updateRun(run.id, {
10362
+ status: 'ready',
10363
+ resultPreview: input.resultPreview,
10364
+ summary: notebookResearchSummary(run.question, input.resultPreview, undefined),
10365
+ recommendation: run.recommendation ?? 'Review SQL, metadata evidence, grain, filters, and duplicate matches before DQL draft promotion.',
10366
+ error: '',
10367
+ lastRunAt: now,
10368
+ });
10369
+ }
10370
+ else {
10371
+ storage.updateRun(run.id, {
10372
+ status: 'error',
10373
+ error: input.error,
10374
+ summary: notebookResearchSummary(run.question, undefined, input.error),
10375
+ lastRunAt: now,
10376
+ });
10377
+ }
10378
+ }
10379
+ }
10380
+ catch {
10381
+ // Research run sync is advisory metadata and must not block notebook execution.
10382
+ }
10383
+ finally {
10384
+ storage.close();
10385
+ }
10386
+ }
10387
+ function compactSqlForRunHistory(sql) {
10388
+ const clean = sql.replace(/\s+/g, ' ').trim();
10389
+ return clean.length > 1200 ? `${clean.slice(0, 1197)}...` : clean;
10390
+ }
6834
10391
  function buildAgentSchemaContextFromContextPack(question, contextPack) {
6835
10392
  const byRelation = new Map();
6836
10393
  const objectsByKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));