@duckcodeailabs/dql-cli 1.6.18 → 1.6.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/apps-api.d.ts +41 -1
  2. package/dist/apps-api.d.ts.map +1 -1
  3. package/dist/apps-api.js +297 -18
  4. package/dist/apps-api.js.map +1 -1
  5. package/dist/args.d.ts +4 -0
  6. package/dist/args.d.ts.map +1 -1
  7. package/dist/args.js +10 -1
  8. package/dist/args.js.map +1 -1
  9. package/dist/assets/dql-notebook/assets/index-Dgqvyn5s.js +3886 -0
  10. package/dist/assets/dql-notebook/index.html +1 -1
  11. package/dist/block-studio-import.d.ts +99 -3
  12. package/dist/block-studio-import.d.ts.map +1 -1
  13. package/dist/block-studio-import.js +772 -27
  14. package/dist/block-studio-import.js.map +1 -1
  15. package/dist/commands/agent.d.ts.map +1 -1
  16. package/dist/commands/agent.js +12 -1
  17. package/dist/commands/agent.js.map +1 -1
  18. package/dist/commands/app.d.ts +2 -1
  19. package/dist/commands/app.d.ts.map +1 -1
  20. package/dist/commands/app.js +22 -4
  21. package/dist/commands/app.js.map +1 -1
  22. package/dist/commands/certify.d.ts.map +1 -1
  23. package/dist/commands/certify.js +104 -10
  24. package/dist/commands/certify.js.map +1 -1
  25. package/dist/commands/compile.d.ts +1 -0
  26. package/dist/commands/compile.d.ts.map +1 -1
  27. package/dist/commands/compile.js +26 -4
  28. package/dist/commands/compile.js.map +1 -1
  29. package/dist/commands/doctor.d.ts.map +1 -1
  30. package/dist/commands/doctor.js +345 -6
  31. package/dist/commands/doctor.js.map +1 -1
  32. package/dist/commands/import.d.ts.map +1 -1
  33. package/dist/commands/import.js +45 -70
  34. package/dist/commands/import.js.map +1 -1
  35. package/dist/commands/lineage.d.ts +2 -0
  36. package/dist/commands/lineage.d.ts.map +1 -1
  37. package/dist/commands/lineage.js +111 -0
  38. package/dist/commands/lineage.js.map +1 -1
  39. package/dist/commands/migrate.d.ts +1 -0
  40. package/dist/commands/migrate.d.ts.map +1 -1
  41. package/dist/commands/migrate.js +106 -2
  42. package/dist/commands/migrate.js.map +1 -1
  43. package/dist/commands/new.d.ts.map +1 -1
  44. package/dist/commands/new.js +246 -9
  45. package/dist/commands/new.js.map +1 -1
  46. package/dist/commands/promote.d.ts +11 -0
  47. package/dist/commands/promote.d.ts.map +1 -0
  48. package/dist/commands/promote.js +236 -0
  49. package/dist/commands/promote.js.map +1 -0
  50. package/dist/commands/semantic.js +2 -2
  51. package/dist/commands/semantic.js.map +1 -1
  52. package/dist/index.js +48 -8
  53. package/dist/index.js.map +1 -1
  54. package/dist/llm/providers/claude-agent-sdk.js +1 -1
  55. package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
  56. package/dist/llm/tools.js +1 -1
  57. package/dist/llm/tools.js.map +1 -1
  58. package/dist/local-runtime.d.ts +176 -5
  59. package/dist/local-runtime.d.ts.map +1 -1
  60. package/dist/local-runtime.js +3695 -146
  61. package/dist/local-runtime.js.map +1 -1
  62. package/dist/package.json +10 -10
  63. package/dist/promote-from-draft.d.ts +11 -10
  64. package/dist/promote-from-draft.d.ts.map +1 -1
  65. package/dist/promote-from-draft.js +53 -30
  66. package/dist/promote-from-draft.js.map +1 -1
  67. package/dist/schedule/runner.js +2 -2
  68. package/dist/schedule/runner.js.map +1 -1
  69. package/dist/semantic-import.d.ts.map +1 -1
  70. package/dist/semantic-import.js +28 -12
  71. package/dist/semantic-import.js.map +1 -1
  72. package/package.json +11 -11
  73. package/dist/assets/dql-notebook/assets/index-BbGJ8gZk.js +0 -3639
  74. package/dist/assets/dql-notebook/assets/index-C4oZK5xm.js +0 -3612
  75. package/dist/assets/dql-notebook/assets/index-CEMXuPm0.js +0 -3802
  76. package/dist/assets/dql-notebook/assets/index-CIMLd3Cb.js +0 -3289
  77. package/dist/assets/dql-notebook/assets/index-DISMWw19.js +0 -3780
  78. package/dist/assets/dql-notebook/assets/index-DPcxWQuS.js +0 -3859
  79. package/dist/assets/dql-notebook/assets/index-McpLeeN0.js +0 -3802
@@ -7,21 +7,50 @@ import { dirname, extname, join, normalize, relative, resolve } from 'node:path'
7
7
  import Anthropic from '@anthropic-ai/sdk';
8
8
  import OpenAI from 'openai';
9
9
  import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
10
- import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
10
+ import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, getDialect, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
11
11
  import { load as loadYaml } from 'js-yaml';
12
12
  import { listBlockTemplates } from './block-templates.js';
13
13
  import { getRunner as getLLMRunner } from './llm/index.js';
14
14
  import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
15
- import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
16
- import { handleAppsApi } from './apps-api.js';
15
+ import { ClaudeProvider, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, recordQueryRun, recordRuntimeSchemaSnapshot, } from '@duckcodeailabs/dql-agent';
16
+ import { handleAppsApi, recommendVisualization } from './apps-api.js';
17
17
  import { getActiveProvider, getEffectiveProviderConfig, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
18
18
  import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
19
- import { LocalAppStorage, defaultLocalAppsDbPath } from '@duckcodeailabs/dql-project';
20
- import { Certifier } from '@duckcodeailabs/dql-governance';
19
+ import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
20
+ import { Certifier, ENTERPRISE_RULES } from '@duckcodeailabs/dql-governance';
21
21
  import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
22
- import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
22
+ import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
23
23
  import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
24
24
  const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
25
+ export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
26
+ const configured = projectConfig.semanticLayer;
27
+ const dbtProjectDir = projectConfig.dbt?.projectDir;
28
+ if (dbtProjectDir
29
+ && (!configured || configured.provider === 'dql')
30
+ && hasDbtSemanticArtifacts(projectRoot, dbtProjectDir)) {
31
+ return { provider: 'dbt', projectPath: dbtProjectDir };
32
+ }
33
+ return configured;
34
+ }
35
+ function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir) {
36
+ const dbtRoot = resolve(projectRoot, dbtProjectDir);
37
+ if (existsSync(join(dbtRoot, 'target', 'semantic_manifest.json')))
38
+ return true;
39
+ const manifestPath = join(dbtRoot, 'target', 'manifest.json');
40
+ if (!existsSync(manifestPath))
41
+ return false;
42
+ try {
43
+ const parsed = JSON.parse(readFileSync(manifestPath, 'utf-8'));
44
+ if (parsed.semantic_models && typeof parsed.semantic_models === 'object')
45
+ return true;
46
+ if (parsed.metrics && typeof parsed.metrics === 'object')
47
+ return true;
48
+ }
49
+ catch {
50
+ return false;
51
+ }
52
+ return false;
53
+ }
25
54
  export async function startLocalServer(opts) {
26
55
  const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
27
56
  const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
@@ -40,7 +69,7 @@ export async function startLocalServer(opts) {
40
69
  const semanticLayerDir = join(projectRoot, 'semantic-layer');
41
70
  let semanticImportManifest = loadSemanticImportManifest(projectRoot);
42
71
  const userPrefsPath = join(projectRoot, '.dql-user-prefs.json');
43
- const semanticConfig = projectConfig.semanticLayer;
72
+ const semanticConfig = resolveProjectSemanticConfig(projectConfig, projectRoot);
44
73
  let semanticLastSyncTime = null;
45
74
  {
46
75
  const semanticConnection = connection;
@@ -330,6 +359,430 @@ export async function startLocalServer(opts) {
330
359
  providerUsed: governedAnswer.providerUsed,
331
360
  };
332
361
  };
362
+ const openNotebookResearchStorage = () => new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
363
+ const runNotebookResearch = async (storage, run, input = {}) => {
364
+ const question = notebookResearchString(input.question) || run.question;
365
+ const domain = notebookResearchString(input.domain) ?? run.domain;
366
+ const owner = notebookResearchString(input.owner) ?? run.owner;
367
+ const sourceCellFingerprint = notebookResearchString(input.sourceCellFingerprint) ?? run.sourceCellFingerprint;
368
+ const intent = input.intent ?? run.intent;
369
+ const context = input.context === undefined ? run.context : input.context;
370
+ let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
371
+ let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
372
+ const startedAt = new Date().toISOString();
373
+ storage.updateRun(run.id, {
374
+ domain,
375
+ owner,
376
+ sourceCellFingerprint,
377
+ question,
378
+ intent,
379
+ context,
380
+ generatedSql,
381
+ reviewedSql,
382
+ status: 'running',
383
+ reviewStatus: 'needs_review',
384
+ error: '',
385
+ });
386
+ try {
387
+ const contextPack = await buildLocalContextPack(projectRoot, {
388
+ question,
389
+ mode: 'question',
390
+ surface: 'notebook',
391
+ selectedContext: {
392
+ ...notebookResearchSelectedContext(run, context),
393
+ domain,
394
+ owner,
395
+ intent,
396
+ researchPattern: notebookResearchIntentPattern(intent),
397
+ },
398
+ strictness: 'balanced',
399
+ limit: 100,
400
+ }).catch((error) => {
401
+ const message = error instanceof Error ? error.message : String(error);
402
+ return {
403
+ id: '',
404
+ routeDecision: undefined,
405
+ evidenceRoles: [],
406
+ warnings: [`Context pack failed: ${message}`],
407
+ retrievalDiagnostics: { selectedEvidence: [] },
408
+ };
409
+ });
410
+ let governedAnswer;
411
+ let providerError;
412
+ const generationWarnings = [];
413
+ if (!generatedSql && !reviewedSql) {
414
+ const resolvedProvider = resolveDefaultLLMProvider(projectRoot);
415
+ const runner = resolvedProvider ? getLLMRunner(resolvedProvider) : null;
416
+ if (!resolvedProvider || !runner) {
417
+ generationWarnings.push('No AI provider is configured. Metadata context was saved as a research plan; paste SQL or configure an AI provider to generate candidate SQL.');
418
+ }
419
+ else {
420
+ const controller = new AbortController();
421
+ try {
422
+ await runner.run({
423
+ provider: resolvedProvider,
424
+ messages: [{ role: 'user', content: notebookResearchAgentPrompt(question, intent) }],
425
+ upstream: {
426
+ cellId: `notebook-research:${run.notebookPath}:${run.id}`,
427
+ sql: JSON.stringify({
428
+ mode: 'notebook_research',
429
+ notebookPath: run.notebookPath,
430
+ sourceCellId: run.sourceCellId,
431
+ sourceCellName: run.sourceCellName,
432
+ owner,
433
+ intent,
434
+ researchPattern: notebookResearchIntentPattern(intent),
435
+ instruction: 'Generate review-required read-only SQL from the inspected metadata context. Execute only through bounded preview and keep the result uncertified until promoted to a DQL draft.',
436
+ context,
437
+ }, null, 2),
438
+ },
439
+ projectRoot,
440
+ executeCertifiedBlock: executeCertifiedBlockForAgent,
441
+ executeGeneratedSql: executeGeneratedSqlForAgent,
442
+ getSchemaContext: getSchemaContextForAgent,
443
+ }, (turn) => {
444
+ if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
445
+ governedAnswer = turn.output;
446
+ }
447
+ if (turn.kind === 'error')
448
+ providerError = turn.message;
449
+ }, controller.signal);
450
+ }
451
+ catch (error) {
452
+ providerError = error instanceof Error ? error.message : String(error);
453
+ }
454
+ if (!governedAnswer) {
455
+ generationWarnings.push(`AI SQL generation did not return a governed answer. Metadata context was saved for review.${providerError ? ` ${providerError}` : ''}`);
456
+ }
457
+ else {
458
+ generatedSql = notebookResearchString(governedAnswer.proposedSql) ?? notebookResearchString(governedAnswer.sql);
459
+ if (!generatedSql) {
460
+ generationWarnings.push('AI returned a governed answer without SQL. Metadata context was saved; add reviewed SQL before DQL promotion.');
461
+ }
462
+ }
463
+ }
464
+ }
465
+ const sqlForPreview = reviewedSql || generatedSql;
466
+ const warnings = [
467
+ ...(contextPack.warnings ?? []),
468
+ ...(governedAnswer?.validationWarnings ?? []),
469
+ ...generationWarnings,
470
+ ].filter(Boolean);
471
+ let resultPreview;
472
+ let previewError;
473
+ if (governedAnswer?.result?.rows && !reviewedSql) {
474
+ resultPreview = normalizeNotebookAgentResult(governedAnswer.result);
475
+ }
476
+ else if (sqlForPreview) {
477
+ try {
478
+ const previewSql = buildAgentPreviewSql(sqlForPreview);
479
+ const previewStart = Date.now();
480
+ resultPreview = await executeLocalSqlForStoredResult(previewSql);
481
+ recordNotebookQueryRun(projectRoot, {
482
+ notebookPath: run.notebookPath,
483
+ cellId: run.sourceCellId,
484
+ cellName: run.sourceCellName,
485
+ researchRunId: run.id,
486
+ source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
487
+ status: 'success',
488
+ rowCount: resultPreview.rowCount ?? resultPreview.rows.length,
489
+ durationMs: Date.now() - previewStart,
490
+ sql: sqlForPreview,
491
+ contextPackId: contextPack.id,
492
+ });
493
+ }
494
+ catch (error) {
495
+ previewError = error instanceof Error ? error.message : String(error);
496
+ recordNotebookQueryRun(projectRoot, {
497
+ notebookPath: run.notebookPath,
498
+ cellId: run.sourceCellId,
499
+ cellName: run.sourceCellName,
500
+ researchRunId: run.id,
501
+ source: reviewedSql ? 'notebook_research_reviewed_sql' : 'notebook_research_ai_sql',
502
+ status: 'error',
503
+ errorCode: previewError,
504
+ sql: sqlForPreview,
505
+ contextPackId: contextPack.id,
506
+ });
507
+ }
508
+ }
509
+ const routeDecision = notebookResearchRouteDecisionForRun(run, contextPack.routeDecision, sqlForPreview);
510
+ const display = resultPreview
511
+ ? recommendVisualization(projectRoot, {
512
+ prompt: question,
513
+ resultSchema: resultPreview.columns,
514
+ rowSample: resultPreview.rows.slice(0, 25),
515
+ defaultVisualization: governedAnswer?.suggestedViz,
516
+ })
517
+ : undefined;
518
+ const evidence = {
519
+ trustStatus: {
520
+ label: governedAnswer?.trustLabel
521
+ ?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : 'Metadata-grounded research plan'),
522
+ reviewRequired: true,
523
+ },
524
+ contextPackId: contextPack.id,
525
+ routeDecision,
526
+ selectedEvidence: contextPack.evidenceRoles?.slice(0, 24) ?? [],
527
+ evidenceRoles: contextPack.evidenceRoles?.slice(0, 24) ?? [],
528
+ evidenceSummaries: contextPack.evidenceSummaries?.slice(0, 16) ?? [],
529
+ allowedSqlContext: {
530
+ relations: (contextPack.allowedSqlContext?.relations ?? []).slice(0, 24).map((relation) => ({
531
+ relation: relation.relation,
532
+ name: relation.name,
533
+ source: relation.source,
534
+ columns: relation.columns.slice(0, 32).map((column) => {
535
+ if (typeof column === 'string')
536
+ return column;
537
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
538
+ return String(column.name);
539
+ }
540
+ return String(column);
541
+ }),
542
+ })),
543
+ sourceBlockSql: contextPack.allowedSqlContext?.sourceBlockSql?.slice(0, 12) ?? [],
544
+ },
545
+ missingContext: contextPack.missingContext?.slice(0, 16) ?? [],
546
+ warnings: contextPack.warnings?.slice(0, 16) ?? [],
547
+ retrievalDiagnostics: contextPack.retrievalDiagnostics,
548
+ agentEvidence: governedAnswer?.evidence,
549
+ analysisPlan: governedAnswer?.analysisPlan,
550
+ citations: [
551
+ ...(contextPack.citations ?? []),
552
+ ...(governedAnswer?.citations ?? []),
553
+ ].slice(0, 40),
554
+ };
555
+ const summary = notebookResearchString(governedAnswer?.answer)
556
+ ?? notebookResearchString(governedAnswer?.text)
557
+ ?? notebookResearchSummary(question, resultPreview, previewError);
558
+ const recommendation = previewError
559
+ ? 'Review the SQL, selected metadata, and connection context before rerunning.'
560
+ : sqlForPreview
561
+ ? 'Review the SQL, parameter choices, grain, and evidence before promoting this research into a DQL draft block.'
562
+ : 'Review the selected metadata context, then paste reviewed SQL or configure an AI provider before DQL draft promotion.';
563
+ const researchPlan = buildNotebookResearchPlan({
564
+ run,
565
+ evidence,
566
+ resultPreview,
567
+ previewError,
568
+ generatedSql,
569
+ reviewedSql,
570
+ routeDecision,
571
+ });
572
+ return storage.updateRun(run.id, {
573
+ domain,
574
+ owner,
575
+ sourceCellFingerprint,
576
+ question,
577
+ intent,
578
+ context,
579
+ status: previewError ? 'error' : 'ready',
580
+ summary,
581
+ recommendation,
582
+ resultPreview,
583
+ evidence,
584
+ researchPlan,
585
+ generatedSql,
586
+ reviewedSql,
587
+ display: display && display.ok ? display.display : undefined,
588
+ contextPackId: contextPack.id,
589
+ routeDecision,
590
+ warnings: [
591
+ ...warnings,
592
+ ...(display && !display.ok ? [display.error] : []),
593
+ ...(display && display.ok ? display.warnings : []),
594
+ ],
595
+ reviewStatus: 'needs_review',
596
+ error: previewError,
597
+ lastRunAt: startedAt,
598
+ }) ?? run;
599
+ }
600
+ catch (error) {
601
+ return storage.updateRun(run.id, {
602
+ domain,
603
+ owner,
604
+ sourceCellFingerprint,
605
+ status: 'error',
606
+ error: error instanceof Error ? error.message : String(error),
607
+ reviewStatus: 'needs_review',
608
+ lastRunAt: startedAt,
609
+ }) ?? run;
610
+ }
611
+ };
612
+ const promoteNotebookResearchToDql = async (storage, run, input = {}) => {
613
+ const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
614
+ if (!sql)
615
+ throw new Error('Notebook research needs generated or reviewed SQL before DQL draft promotion.');
616
+ const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}`;
617
+ const session = await createDqlGenerationSessionForProject(projectRoot, {
618
+ inputMode: 'upload',
619
+ sources: [{ path: sourcePath.endsWith('.sql') ? sourcePath : `${sourcePath}.sql`, content: sql }],
620
+ sourceKind: 'raw-sql-file',
621
+ domain: notebookResearchString(input.domain) ?? run.domain,
622
+ owner: notebookResearchString(input.owner),
623
+ tags: ['notebook-research', 'review-required', ...(Array.isArray(input.tags) ? input.tags : [])],
624
+ provider: input.provider,
625
+ }, semanticLayer);
626
+ const draftPath = session.candidates.find((candidate) => candidate.draftSave?.path)?.draftSave?.path
627
+ ?? session.candidates.find((candidate) => candidate.savedPath)?.savedPath;
628
+ const promotion = buildNotebookDqlPromotionSummary(session, draftPath);
629
+ const updated = storage.markPromoted(run.id, {
630
+ draftBlockPath: draftPath,
631
+ dqlImportId: session.id,
632
+ dqlCandidateIds: session.candidates.map((candidate) => candidate.id),
633
+ dqlPromotion: promotion,
634
+ }) ?? run;
635
+ const planned = storage.updateRun(updated.id, {
636
+ researchPlan: buildNotebookResearchPlan({
637
+ run: updated,
638
+ evidence: updated.evidence,
639
+ resultPreview: updated.resultPreview,
640
+ generatedSql: updated.generatedSql,
641
+ reviewedSql: updated.reviewedSql,
642
+ routeDecision: updated.routeDecision,
643
+ }),
644
+ }) ?? updated;
645
+ return { run: planned, session };
646
+ };
647
+ const checkNotebookResearchReuse = async (storage, run, input = {}) => {
648
+ const sql = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
649
+ if (!sql)
650
+ throw new Error('Notebook research needs generated or reviewed SQL before reuse checking.');
651
+ const sourcePath = `${run.notebookPath}${run.sourceCellId ? `#${run.sourceCellId}` : ''}.sql`;
652
+ const match = await matchSqlForDqlReuse({
653
+ sql,
654
+ sourcePath,
655
+ name: run.title || 'Notebook research SQL',
656
+ domain: notebookResearchString(input.domain) ?? run.domain ?? 'uncategorized',
657
+ owner: notebookResearchString(input.owner) ?? 'analytics',
658
+ });
659
+ const promotion = {
660
+ importId: `reuse-check:${run.id}:${Date.now()}`,
661
+ candidateIds: ['reuse_check'],
662
+ recommendedAction: match.recommendedAction,
663
+ similarityMatches: match.similarityMatches.slice(0, 8).map(toNotebookPromotionMatch),
664
+ candidates: [{
665
+ id: 'reuse_check',
666
+ name: run.title || 'Notebook research SQL',
667
+ domain: notebookResearchString(input.domain) ?? run.domain,
668
+ reviewStatus: 'reuse_checked',
669
+ recommendedAction: match.recommendedAction,
670
+ similarityMatches: match.similarityMatches.slice(0, 5).map(toNotebookPromotionMatch),
671
+ parameterPolicy: match.parameterPolicy.slice(0, 16).map((entry) => ({
672
+ name: entry.name,
673
+ policy: entry.policy,
674
+ })),
675
+ allowedFilters: match.allowedFilters.slice(0, 16),
676
+ warnings: match.parameterDecisions
677
+ .filter((decision) => decision.reason)
678
+ .map((decision) => `${decision.name}: ${decision.reason}`)
679
+ .slice(0, 12),
680
+ }],
681
+ createdAt: new Date().toISOString(),
682
+ };
683
+ const checked = storage.updateRun(run.id, {
684
+ dqlPromotionAction: match.recommendedAction,
685
+ dqlPromotion: promotion,
686
+ researchPlan: buildNotebookResearchPlan({
687
+ run: {
688
+ ...run,
689
+ dqlPromotionAction: match.recommendedAction,
690
+ dqlPromotion: promotion,
691
+ },
692
+ evidence: run.evidence,
693
+ resultPreview: run.resultPreview,
694
+ generatedSql: run.generatedSql,
695
+ reviewedSql: run.reviewedSql,
696
+ routeDecision: run.routeDecision,
697
+ }),
698
+ }) ?? run;
699
+ return { run: checked, promotion, match };
700
+ };
701
+ const matchSqlForDqlReuse = async (input) => {
702
+ const sql = input.sql.trim();
703
+ if (!sql)
704
+ throw new Error('Missing SQL for reuse check.');
705
+ const sourceTables = extractSqlTablesLight(sql);
706
+ const candidate = {
707
+ id: 'match_sql',
708
+ sourceKind: 'raw-sql-file',
709
+ sourcePath: input.sourcePath ?? 'pasted.sql',
710
+ name: input.name ?? 'SQL match preview',
711
+ domain: input.domain ?? 'imported',
712
+ description: 'SQL match preview candidate.',
713
+ owner: input.owner ?? 'analytics',
714
+ tags: [],
715
+ sql,
716
+ dqlSource: '',
717
+ validation: null,
718
+ preview: null,
719
+ lineage: {
720
+ sourceTables,
721
+ parameters: [],
722
+ warnings: [],
723
+ statementIndex: 1,
724
+ totalStatements: 1,
725
+ },
726
+ confidence: 0.8,
727
+ splitStrategy: 'manual',
728
+ warnings: [],
729
+ conversionNotes: [],
730
+ aiAssistance: [],
731
+ reviewStatus: 'draft',
732
+ };
733
+ const evidence = deterministicDqlGenerationEvidence(candidate);
734
+ const patch = deterministicDqlGenerationPatch(candidate, evidence);
735
+ const contextPack = await buildDqlGenerationContextPack(projectRoot, { ...candidate, sql: patch.sql ?? sql }).catch(() => null);
736
+ const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
737
+ return {
738
+ parameterDecisions: patch.parameterDecisions ?? [],
739
+ parameterPolicy: patch.parameterPolicy ?? [],
740
+ filterBindings: patch.filterBindings ?? [],
741
+ allowedFilters: patch.allowedFilters ?? [],
742
+ parameterizedSql: patch.sql ?? sql,
743
+ similarityMatches: similarity.matches,
744
+ recommendedAction: similarity.recommendedAction,
745
+ };
746
+ };
747
+ const buildNotebookDqlPromotionSummary = (session, draftBlockPath) => {
748
+ const primary = draftBlockPath
749
+ ? session.candidates.find((candidate) => candidate.draftSave?.path === draftBlockPath || candidate.savedPath === draftBlockPath) ?? session.candidates[0]
750
+ : session.candidates[0];
751
+ return {
752
+ importId: session.id,
753
+ candidateIds: session.candidates.map((candidate) => candidate.id),
754
+ draftBlockPath,
755
+ recommendedAction: primary?.recommendedAction,
756
+ similarityMatches: (primary?.similarityMatches ?? []).slice(0, 8).map(toNotebookPromotionMatch),
757
+ candidates: session.candidates.map((candidate) => ({
758
+ id: candidate.id,
759
+ name: candidate.name,
760
+ domain: candidate.domain,
761
+ draftPath: candidate.draftSave?.path,
762
+ savedPath: candidate.savedPath,
763
+ reviewStatus: candidate.reviewStatus,
764
+ recommendedAction: candidate.recommendedAction,
765
+ similarityMatches: (candidate.similarityMatches ?? []).slice(0, 5).map(toNotebookPromotionMatch),
766
+ parameterPolicy: (candidate.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
767
+ name: entry.name,
768
+ policy: entry.policy,
769
+ })),
770
+ allowedFilters: (candidate.allowedFilters ?? []).slice(0, 16),
771
+ warnings: [...(candidate.warnings ?? []), ...(candidate.lineage.warnings ?? [])].slice(0, 12),
772
+ })),
773
+ createdAt: new Date().toISOString(),
774
+ };
775
+ };
776
+ const toNotebookPromotionMatch = (match) => ({
777
+ kind: match.kind,
778
+ objectKey: match.objectKey,
779
+ name: match.name,
780
+ status: match.status,
781
+ source: match.source,
782
+ score: match.score,
783
+ reason: match.reason,
784
+ recommendedAction: match.recommendedAction,
785
+ });
333
786
  // SSE clients for /api/watch hot-reload
334
787
  const sseClients = new Set();
335
788
  // Watch notebooks/, workbooks/, semantic-layer/, and data/ dirs for changes
@@ -398,8 +851,28 @@ export async function startLocalServer(opts) {
398
851
  if (validated.reviewStatus === 'rejected') {
399
852
  errors.unshift('Candidate was rejected.');
400
853
  }
854
+ const duplicateBlocker = duplicateCertificationBlocker(validated);
855
+ if (duplicateBlocker)
856
+ errors.unshift(duplicateBlocker);
401
857
  return { candidate: validated, errors };
402
858
  };
859
+ const duplicateCertificationBlocker = (candidate) => {
860
+ const match = candidate.similarityMatches?.[0];
861
+ if (!match)
862
+ return null;
863
+ const duplicateKind = match.kind === 'exact_sql_match'
864
+ || match.kind === 'parameterized_duplicate'
865
+ || (match.kind === 'business_duplicate' && match.status === 'certified' && match.score >= 0.76);
866
+ if (!duplicateKind)
867
+ return null;
868
+ const replacementFor = new Set((candidate.replacementFor ?? []).map((value) => value.toLowerCase()));
869
+ const documentedReplacement = candidate.recommendedAction === 'create_replacement'
870
+ || replacementFor.has(match.name.toLowerCase())
871
+ || (match.objectKey && replacementFor.has(match.objectKey.toLowerCase()));
872
+ if (documentedReplacement)
873
+ return null;
874
+ return `Likely duplicate of ${match.name} (${match.kind}, ${Math.round(match.score * 100)}% match). Reuse the existing block, extend it, or document this draft as a replacement before certification.`;
875
+ };
403
876
  const runBlockStudioPreviewSource = async (source, targetConnection) => {
404
877
  const activeConnection = requireActiveConnection(targetConnection);
405
878
  let tableMapping;
@@ -504,7 +977,50 @@ export async function startLocalServer(opts) {
504
977
  }
505
978
  return { passed, failed, skipped: 0, duration: Date.now() - start, assertions, runAt: new Date() };
506
979
  };
507
- const certifyBlockStudioSource = async (source, blockPath) => {
980
+ const saveDqlGenerationDraft = (importId, candidate) => saveDqlGenerationDraftForProject(projectRoot, importId, candidate);
981
+ const createDqlGenerationSessionFromBody = async (body) => {
982
+ return createDqlGenerationSessionForProject(projectRoot, {
983
+ inputPath: typeof body.path === 'string' ? body.path : '',
984
+ inputMode: body.inputMode === 'paste' || body.inputMode === 'upload' || body.inputMode === 'path' ? body.inputMode : undefined,
985
+ sources: Array.isArray(body.sources)
986
+ ? body.sources.map((source, index) => ({
987
+ path: typeof source?.path === 'string' ? source.path : `source-${index + 1}.sql`,
988
+ content: typeof source?.content === 'string' ? source.content : '',
989
+ }))
990
+ : undefined,
991
+ sourceKind: typeof body.sourceKind === 'string' ? body.sourceKind : 'raw-sql',
992
+ domain: typeof body.domain === 'string' ? body.domain : undefined,
993
+ owner: typeof body.owner === 'string' ? body.owner : undefined,
994
+ tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
995
+ provider: typeof body.provider === 'string' ? body.provider : undefined,
996
+ }, semanticLayer);
997
+ };
998
+ const loadDqlGenerationSession = (importId) => {
999
+ const session = loadBlockStudioImportSession(projectRoot, importId);
1000
+ const candidates = session.candidates.map((candidate) => ({
1001
+ ...candidate,
1002
+ generationMode: candidate.generationMode ?? 'deterministic',
1003
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
1004
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
1005
+ evidence: candidate.evidence ?? deterministicDqlGenerationEvidence(candidate),
1006
+ draftSave: candidate.draftSave ?? (isDraftBlockPath(candidate.savedPath)
1007
+ ? { status: 'saved', path: candidate.savedPath }
1008
+ : { status: 'pending' }),
1009
+ }));
1010
+ return {
1011
+ ...session,
1012
+ mode: 'ai-import',
1013
+ candidates,
1014
+ generation: {
1015
+ provider: candidates.find((candidate) => candidate.generationProvider)?.generationProvider ?? 'local-deterministic',
1016
+ aiEnabled: candidates.some((candidate) => candidate.generationMode === 'ai'),
1017
+ contextObjectCount: candidates.reduce((sum, candidate) => sum + (candidate.evidence?.length ?? 0), 0),
1018
+ createdDrafts: candidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
1019
+ warnings: candidates.flatMap((candidate) => candidate.draftSave.status === 'error' && candidate.draftSave.error ? [candidate.draftSave.error] : []),
1020
+ },
1021
+ };
1022
+ };
1023
+ const certifyBlockStudioSource = async (source, blockPath, options = {}) => {
508
1024
  const validation = validateBlockStudioSource(source, semanticLayer);
509
1025
  let preview = null;
510
1026
  let testResults = null;
@@ -542,12 +1058,24 @@ export async function startLocalServer(opts) {
542
1058
  description: parsed.description,
543
1059
  owner: parsed.owner,
544
1060
  tags: parsed.tags,
1061
+ llmContext: parsed.llmContext,
1062
+ pattern: parsed.pattern,
1063
+ grain: parsed.grain,
1064
+ entities: parsed.entities,
1065
+ declaredOutputs: parsed.outputs,
1066
+ dimensions: parsed.dimensions,
1067
+ allowedFilters: parsed.allowedFilters,
1068
+ parameterPolicy: parsed.parameterPolicy,
1069
+ filterBindings: parsed.filterBindings,
1070
+ sourceSystems: parsed.sourceSystems,
1071
+ replacementFor: parsed.replacementFor,
1072
+ reviewCadence: parsed.reviewCadence,
545
1073
  dependencies: [],
546
1074
  usedInCount: 0,
547
1075
  createdAt: new Date(),
548
1076
  updatedAt: new Date(),
549
1077
  };
550
- const certification = new Certifier().evaluate(record, testResults ?? undefined);
1078
+ const certification = new Certifier(options.enterprise ? ENTERPRISE_RULES : undefined).evaluate(record, testResults ?? undefined);
551
1079
  const checklist = buildBlockStudioCertificationChecklist({
552
1080
  source,
553
1081
  validation,
@@ -730,6 +1258,7 @@ export async function startLocalServer(opts) {
730
1258
  const variables = body.variables && typeof body.variables === 'object'
731
1259
  ? body.variables
732
1260
  : {};
1261
+ const dashboardVariables = dashboardRuntimeVariables(loaded.dashboard, variables);
733
1262
  const tiles = [];
734
1263
  let localApps = null;
735
1264
  for (const item of loaded.dashboard.layout.items) {
@@ -831,8 +1360,15 @@ export async function startLocalServer(opts) {
831
1360
  });
832
1361
  continue;
833
1362
  }
834
- const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, targetConnection, projectRoot, projectConfig);
835
- const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...variables }), prepared.connection);
1363
+ const filterApplication = applyDashboardFiltersToBlockExecution({
1364
+ sql: semanticCompose?.sql ?? plan.sql,
1365
+ sqlParams: plan?.sqlParams ?? [],
1366
+ variables: { ...(plan?.variables ?? {}), ...dashboardVariables },
1367
+ block,
1368
+ dashboard: loaded.dashboard,
1369
+ });
1370
+ const prepared = prepareLocalExecution(filterApplication.sql, targetConnection, projectRoot, projectConfig);
1371
+ const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables(filterApplication.variables), prepared.connection);
836
1372
  tiles.push({
837
1373
  tileId: item.i,
838
1374
  status: 'ok',
@@ -843,6 +1379,10 @@ export async function startLocalServer(opts) {
843
1379
  viz: item.viz,
844
1380
  chartConfig: mergeDashboardChartConfig(plan?.chartConfig, item),
845
1381
  result: normalizeQueryResult(result),
1382
+ filters: {
1383
+ applied: filterApplication.appliedFilters,
1384
+ skipped: filterApplication.skippedFilters,
1385
+ },
846
1386
  citation: {
847
1387
  kind: 'block',
848
1388
  name: block.name,
@@ -885,11 +1425,11 @@ export async function startLocalServer(opts) {
885
1425
  return;
886
1426
  }
887
1427
  // Apps, dashboards, persona — see apps-api.ts. Returns true if handled.
888
- if (path.startsWith('/api/apps') || path === '/api/persona') {
1428
+ if (path.startsWith('/api/apps') || path.startsWith('/api/visualizations') || path === '/api/persona') {
889
1429
  try {
890
1430
  const handled = await handleAppsApi({
891
1431
  req,
892
- res,
1432
+ res: res,
893
1433
  url,
894
1434
  path,
895
1435
  projectRoot,
@@ -1019,6 +1559,394 @@ export async function startLocalServer(opts) {
1019
1559
  }
1020
1560
  return;
1021
1561
  }
1562
+ if (req.method === 'GET' && path === '/api/notebook/research') {
1563
+ const storage = openNotebookResearchStorage();
1564
+ try {
1565
+ const notebookPath = notebookResearchString(url.searchParams.get('path'));
1566
+ const sourceCellId = notebookResearchString(url.searchParams.get('sourceCellId') ?? url.searchParams.get('cellId'));
1567
+ const domain = notebookResearchString(url.searchParams.get('domain'));
1568
+ const owner = notebookResearchString(url.searchParams.get('owner'));
1569
+ const intent = notebookResearchIntent(url.searchParams.get('intent'));
1570
+ const search = notebookResearchString(url.searchParams.get('q')) ?? notebookResearchString(url.searchParams.get('search'));
1571
+ const status = notebookResearchStatus(url.searchParams.get('status'));
1572
+ const reviewStatus = notebookResearchReviewStatus(url.searchParams.get('reviewStatus'));
1573
+ const promotionAction = notebookResearchPromotionAction(url.searchParams.get('promotionAction') ?? url.searchParams.get('action'));
1574
+ const readiness = notebookResearchReadiness(url.searchParams.get('readiness') ?? url.searchParams.get('ready'));
1575
+ const age = notebookResearchAge(url.searchParams.get('age'));
1576
+ const nextAction = notebookResearchNextAction(url.searchParams.get('nextAction') ?? url.searchParams.get('next'));
1577
+ const activeOnlyParam = url.searchParams.get('activeOnly') ?? url.searchParams.get('active');
1578
+ const activeOnly = activeOnlyParam === 'true' || activeOnlyParam === '1';
1579
+ const sort = notebookResearchSort(url.searchParams.get('sort'));
1580
+ const limit = notebookResearchInteger(url.searchParams.get('limit'), 50, 1, 500);
1581
+ const offset = notebookResearchInteger(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER);
1582
+ const page = storage.listRunsPage({ notebookPath, sourceCellId, domain, owner, intent, search, status, reviewStatus, promotionAction, readiness, age, nextAction, activeOnly, sort, limit, offset });
1583
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1584
+ res.end(serializeJSON(withNotebookResearchChecklistPage(page)));
1585
+ }
1586
+ catch (error) {
1587
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1588
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1589
+ }
1590
+ finally {
1591
+ storage.close();
1592
+ }
1593
+ return;
1594
+ }
1595
+ if (req.method === 'POST' && path === '/api/notebook/research') {
1596
+ const storage = openNotebookResearchStorage();
1597
+ try {
1598
+ const body = await readJSON(req);
1599
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1600
+ const question = notebookResearchString(body.question);
1601
+ const sourceCell = notebookResearchSourceCellPayload(body);
1602
+ const sourceCellId = notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell);
1603
+ const sourceCellName = notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell);
1604
+ const sourceCellFingerprint = notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell);
1605
+ if (!notebookPath || !question) {
1606
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1607
+ res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
1608
+ return;
1609
+ }
1610
+ const created = storage.createRun({
1611
+ notebookPath,
1612
+ domain: notebookResearchString(body.domain),
1613
+ owner: notebookResearchString(body.owner),
1614
+ sourceCell,
1615
+ sourceCellId,
1616
+ sourceCellName,
1617
+ sourceCellFingerprint,
1618
+ title: notebookResearchString(body.title),
1619
+ question,
1620
+ intent: notebookResearchIntent(body.intent),
1621
+ context: body.context,
1622
+ generatedSql: notebookResearchString(body.generatedSql),
1623
+ reviewedSql: notebookResearchString(body.reviewedSql),
1624
+ });
1625
+ const run = body.run === true
1626
+ ? await runNotebookResearch(storage, created, {
1627
+ domain: notebookResearchString(body.domain),
1628
+ sourceCellFingerprint,
1629
+ question,
1630
+ intent: notebookResearchIntent(body.intent),
1631
+ context: body.context,
1632
+ generatedSql: notebookResearchString(body.generatedSql),
1633
+ reviewedSql: notebookResearchString(body.reviewedSql),
1634
+ })
1635
+ : created;
1636
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
1637
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
1638
+ }
1639
+ catch (error) {
1640
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1641
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1642
+ }
1643
+ finally {
1644
+ storage.close();
1645
+ }
1646
+ return;
1647
+ }
1648
+ if (req.method === 'POST' && path === '/api/notebook/research/context-preview') {
1649
+ try {
1650
+ const body = await readJSON(req);
1651
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1652
+ const question = notebookResearchString(body.question);
1653
+ if (!notebookPath || !question) {
1654
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1655
+ res.end(serializeJSON({ error: 'notebookPath and question are required.' }));
1656
+ return;
1657
+ }
1658
+ const intent = notebookResearchIntent(body.intent) ?? 'ad_hoc_analysis';
1659
+ const domain = notebookResearchString(body.domain);
1660
+ const context = body.context;
1661
+ const sourceCell = notebookResearchSourceCellPayload(body);
1662
+ const contextPack = await buildLocalContextPack(projectRoot, {
1663
+ question,
1664
+ mode: 'question',
1665
+ surface: 'notebook',
1666
+ selectedContext: {
1667
+ activeSurface: 'notebook',
1668
+ notebookPath,
1669
+ domain,
1670
+ sourceCellId: notebookResearchString(body.sourceCellId) ?? notebookResearchSourceCellId(sourceCell),
1671
+ sourceCellName: notebookResearchString(body.sourceCellName) ?? notebookResearchSourceCellName(sourceCell),
1672
+ context,
1673
+ intent,
1674
+ researchPattern: notebookResearchIntentPattern(intent),
1675
+ },
1676
+ strictness: 'balanced',
1677
+ limit: 100,
1678
+ });
1679
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1680
+ res.end(serializeJSON(notebookResearchContextPreview(contextPack)));
1681
+ }
1682
+ catch (error) {
1683
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1684
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1685
+ }
1686
+ return;
1687
+ }
1688
+ if (req.method === 'POST' && path === '/api/notebook/research/seed-cells') {
1689
+ const storage = openNotebookResearchStorage();
1690
+ try {
1691
+ const body = await readJSON(req);
1692
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1693
+ if (!notebookPath) {
1694
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1695
+ res.end(serializeJSON({ error: 'notebookPath is required.' }));
1696
+ return;
1697
+ }
1698
+ const cells = Array.isArray(body.cells) ? body.cells : [];
1699
+ if (cells.length === 0) {
1700
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1701
+ res.end(serializeJSON({ error: 'At least one source cell is required.' }));
1702
+ return;
1703
+ }
1704
+ const validCells = cells.filter((cell) => Boolean(cell && typeof cell === 'object' && !Array.isArray(cell)));
1705
+ const invalidCellCount = cells.length - validCells.length;
1706
+ const seeded = storage.seedRunsFromCells({
1707
+ notebookPath,
1708
+ domain: notebookResearchString(body.domain),
1709
+ owner: notebookResearchString(body.owner),
1710
+ notebookTitle: notebookResearchString(body.notebookTitle),
1711
+ cells: validCells.map((cell) => ({
1712
+ sourceCell: notebookResearchSourceCellPayload(cell),
1713
+ id: notebookResearchString(cell.id),
1714
+ sourceCellId: notebookResearchString(cell.sourceCellId),
1715
+ name: notebookResearchString(cell.name),
1716
+ sourceCellName: notebookResearchString(cell.sourceCellName),
1717
+ sourceCellFingerprint: notebookResearchString(cell.sourceCellFingerprint),
1718
+ title: notebookResearchString(cell.title),
1719
+ type: notebookResearchString(cell.type),
1720
+ sql: notebookResearchString(cell.sql),
1721
+ content: notebookResearchString(cell.content),
1722
+ source: notebookResearchString(cell.source),
1723
+ question: notebookResearchString(cell.question),
1724
+ intent: notebookResearchIntent(cell.intent),
1725
+ })),
1726
+ limit: 1000,
1727
+ });
1728
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
1729
+ res.end(serializeJSON({
1730
+ created: seeded.created.map(withNotebookResearchChecklist),
1731
+ createdCount: seeded.createdCount,
1732
+ skippedCount: seeded.skippedCount + invalidCellCount,
1733
+ limitApplied: seeded.limitApplied,
1734
+ }));
1735
+ }
1736
+ catch (error) {
1737
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1738
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1739
+ }
1740
+ finally {
1741
+ storage.close();
1742
+ }
1743
+ return;
1744
+ }
1745
+ if (req.method === 'POST' && path === '/api/notebook/research/source-coverage') {
1746
+ const storage = openNotebookResearchStorage();
1747
+ try {
1748
+ const body = await readJSON(req);
1749
+ const notebookPath = notebookResearchString(body.notebookPath) ?? notebookResearchString(body.path);
1750
+ if (!notebookPath) {
1751
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
1752
+ res.end(serializeJSON({ error: 'notebookPath is required.' }));
1753
+ return;
1754
+ }
1755
+ const rawSourceCells = Array.isArray(body.sourceCells)
1756
+ ? body.sourceCells
1757
+ : Array.isArray(body.cells)
1758
+ ? body.cells
1759
+ : [];
1760
+ const sourceCells = rawSourceCells.flatMap((cell) => {
1761
+ if (!cell || typeof cell !== 'object' || Array.isArray(cell))
1762
+ return [];
1763
+ const record = cell;
1764
+ const nested = notebookResearchSourceCellPayload(record) ?? {};
1765
+ return [{
1766
+ ...nested,
1767
+ id: notebookResearchString(record.id) ?? nested.id,
1768
+ sourceCellId: notebookResearchString(record.sourceCellId) ?? nested.sourceCellId,
1769
+ cellId: notebookResearchString(record.cellId) ?? nested.cellId,
1770
+ name: notebookResearchString(record.name) ?? nested.name,
1771
+ sourceCellName: notebookResearchString(record.sourceCellName) ?? nested.sourceCellName,
1772
+ title: notebookResearchString(record.title) ?? nested.title,
1773
+ fingerprint: notebookResearchString(record.fingerprint) ?? nested.fingerprint,
1774
+ sourceCellFingerprint: notebookResearchString(record.sourceCellFingerprint) ?? nested.sourceCellFingerprint,
1775
+ sqlFingerprint: notebookResearchString(record.sqlFingerprint) ?? nested.sqlFingerprint,
1776
+ type: notebookResearchString(record.type) ?? nested.type,
1777
+ }];
1778
+ });
1779
+ const requestedIds = Array.isArray(body.sourceCellIds)
1780
+ ? body.sourceCellIds
1781
+ : rawSourceCells.length > 0
1782
+ ? sourceCells.map((cell) => notebookResearchSourceCellId(cell))
1783
+ : [];
1784
+ const sourceCellIds = Array.from(new Set(requestedIds
1785
+ .map((id) => notebookResearchString(id))
1786
+ .filter((id) => Boolean(id))));
1787
+ const requestedSourceCellCount = new Set([
1788
+ ...sourceCellIds,
1789
+ ...sourceCells
1790
+ .map((cell) => notebookResearchSourceCellId(cell))
1791
+ .filter((id) => Boolean(id)),
1792
+ ]).size;
1793
+ const limit = typeof body.limit === 'number' && Number.isFinite(body.limit)
1794
+ ? Math.max(1, Math.min(10_000, Math.floor(body.limit)))
1795
+ : 10_000;
1796
+ const linkedRuns = storage.listLatestRunsBySourceCell({
1797
+ notebookPath,
1798
+ sourceCellIds,
1799
+ sourceCells,
1800
+ limit,
1801
+ });
1802
+ const missingRuns = storage.listLatestRunsForMissingSourceCells({
1803
+ notebookPath,
1804
+ sourceCellIds,
1805
+ sourceCells,
1806
+ limit,
1807
+ });
1808
+ const runs = [...linkedRuns, ...missingRuns];
1809
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1810
+ res.end(serializeJSON({
1811
+ runs: runs.map(withNotebookResearchChecklist),
1812
+ requestedCount: requestedSourceCellCount,
1813
+ matchedCount: runs.length,
1814
+ limitApplied: requestedSourceCellCount > limit,
1815
+ }));
1816
+ }
1817
+ catch (error) {
1818
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1819
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1820
+ }
1821
+ finally {
1822
+ storage.close();
1823
+ }
1824
+ return;
1825
+ }
1826
+ if (req.method === 'GET' && path === '/api/notebook/research/diagnostics') {
1827
+ const storage = openNotebookResearchStorage();
1828
+ try {
1829
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1830
+ res.end(serializeJSON(storage.getDiagnostics()));
1831
+ }
1832
+ catch (error) {
1833
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1834
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1835
+ }
1836
+ finally {
1837
+ storage.close();
1838
+ }
1839
+ return;
1840
+ }
1841
+ const notebookResearchMatch = /^\/api\/notebook\/research\/([^/]+)(?:\/([^/]+))?$/.exec(path);
1842
+ if (notebookResearchMatch) {
1843
+ const id = decodeURIComponent(notebookResearchMatch[1]);
1844
+ const action = notebookResearchMatch[2];
1845
+ const storage = openNotebookResearchStorage();
1846
+ try {
1847
+ const run = storage.getRun(id);
1848
+ if (!run) {
1849
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
1850
+ res.end(serializeJSON({ error: 'Notebook research run not found.' }));
1851
+ return;
1852
+ }
1853
+ if (req.method === 'GET' && !action) {
1854
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1855
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(run) }));
1856
+ return;
1857
+ }
1858
+ if (req.method === 'PATCH' && !action) {
1859
+ const body = await readJSON(req);
1860
+ const sourceCell = notebookResearchSourceCellPayload(body);
1861
+ const sourceCellIdPatch = notebookResearchPatchString(body, 'sourceCellId');
1862
+ const sourceCellNamePatch = notebookResearchPatchString(body, 'sourceCellName');
1863
+ const sourceCellFingerprintPatch = notebookResearchPatchString(body, 'sourceCellFingerprint');
1864
+ const updated = storage.updateRun(id, {
1865
+ domain: notebookResearchString(body.domain),
1866
+ owner: notebookResearchString(body.owner),
1867
+ sourceCellId: sourceCellIdPatch !== undefined ? sourceCellIdPatch : notebookResearchSourceCellId(sourceCell),
1868
+ sourceCellName: sourceCellNamePatch !== undefined ? sourceCellNamePatch : notebookResearchSourceCellName(sourceCell),
1869
+ sourceCellFingerprint: sourceCellFingerprintPatch !== undefined ? sourceCellFingerprintPatch : notebookResearchSourceCellFingerprint(sourceCell),
1870
+ title: notebookResearchString(body.title),
1871
+ question: notebookResearchString(body.question),
1872
+ intent: notebookResearchIntent(body.intent),
1873
+ context: body.context,
1874
+ recommendation: notebookResearchString(body.recommendation),
1875
+ evidence: body.evidence,
1876
+ contextPackId: notebookResearchString(body.contextPackId),
1877
+ routeDecision: body.routeDecision,
1878
+ generatedSql: notebookResearchString(body.generatedSql),
1879
+ reviewedSql: notebookResearchString(body.reviewedSql),
1880
+ warnings: Array.isArray(body.warnings) ? body.warnings.filter((item) => typeof item === 'string') : undefined,
1881
+ reviewStatus: notebookResearchReviewStatus(body.reviewStatus),
1882
+ dqlPromotionAction: notebookResearchPromotionAction(body.dqlPromotionAction),
1883
+ }) ?? run;
1884
+ const planned = storage.updateRun(updated.id, {
1885
+ researchPlan: buildNotebookResearchPlan({
1886
+ run: updated,
1887
+ evidence: updated.evidence,
1888
+ resultPreview: updated.resultPreview,
1889
+ previewError: updated.error,
1890
+ generatedSql: updated.generatedSql,
1891
+ reviewedSql: updated.reviewedSql,
1892
+ routeDecision: updated.routeDecision,
1893
+ }),
1894
+ }) ?? updated;
1895
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1896
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(planned) }));
1897
+ return;
1898
+ }
1899
+ if (req.method === 'POST' && action === 'run') {
1900
+ const body = await readJSON(req).catch(() => ({}));
1901
+ const sourceCell = notebookResearchSourceCellPayload(body);
1902
+ const updated = await runNotebookResearch(storage, run, {
1903
+ domain: notebookResearchString(body.domain),
1904
+ owner: notebookResearchString(body.owner),
1905
+ sourceCellFingerprint: notebookResearchString(body.sourceCellFingerprint) ?? notebookResearchSourceCellFingerprint(sourceCell),
1906
+ question: notebookResearchString(body.question),
1907
+ intent: notebookResearchIntent(body.intent),
1908
+ context: body.context,
1909
+ generatedSql: notebookResearchString(body.generatedSql),
1910
+ reviewedSql: notebookResearchString(body.reviewedSql),
1911
+ });
1912
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1913
+ res.end(serializeJSON({ run: withNotebookResearchChecklist(updated) }));
1914
+ return;
1915
+ }
1916
+ if (req.method === 'POST' && action === 'reuse-check') {
1917
+ const body = await readJSON(req).catch(() => ({}));
1918
+ const payload = await checkNotebookResearchReuse(storage, run, {
1919
+ domain: notebookResearchString(body.domain),
1920
+ owner: notebookResearchString(body.owner),
1921
+ });
1922
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1923
+ res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
1924
+ return;
1925
+ }
1926
+ if (req.method === 'POST' && action === 'promote-dql') {
1927
+ const body = await readJSON(req).catch(() => ({}));
1928
+ const payload = await promoteNotebookResearchToDql(storage, run, {
1929
+ domain: notebookResearchString(body.domain),
1930
+ owner: notebookResearchString(body.owner),
1931
+ provider: notebookResearchString(body.provider),
1932
+ tags: Array.isArray(body.tags) ? body.tags.filter((item) => typeof item === 'string') : undefined,
1933
+ });
1934
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1935
+ res.end(serializeJSON({ ...payload, run: withNotebookResearchChecklist(payload.run) }));
1936
+ return;
1937
+ }
1938
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
1939
+ res.end(serializeJSON({ error: 'Unsupported notebook research operation.' }));
1940
+ }
1941
+ catch (error) {
1942
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1943
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1944
+ }
1945
+ finally {
1946
+ storage.close();
1947
+ }
1948
+ return;
1949
+ }
1022
1950
  // ── run snapshots (v0.11) ───────────────────────────────────────────────
1023
1951
  // Captures executed notebook state (query results + timings) in a
1024
1952
  // sibling `.run.json` so notebooks can show last-run output without
@@ -1304,49 +2232,51 @@ export async function startLocalServer(opts) {
1304
2232
  // ── Block library (list all blocks with metadata) ────────────────────
1305
2233
  if (req.method === 'GET' && path === '/api/blocks/library') {
1306
2234
  try {
1307
- const blocksDir = join(projectRoot, 'blocks');
1308
2235
  const blocks = [];
1309
- if (existsSync(blocksDir)) {
1310
- const scanDir = (dir) => {
1311
- for (const entry of readdirSync(dir, { withFileTypes: true })) {
1312
- if (entry.isDirectory()) {
1313
- scanDir(join(dir, entry.name));
1314
- }
1315
- else if (entry.name.endsWith('.dql')) {
1316
- const filePath = join(dir, entry.name);
1317
- const relPath = relative(projectRoot, filePath);
1318
- try {
1319
- const source = readFileSync(filePath, 'utf-8');
1320
- const stat = statSync(filePath);
1321
- // Quick regex parse for key block fields
1322
- const nameMatch = /block\s+"([^"]+)"/.exec(source);
1323
- const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
1324
- const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
1325
- const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
1326
- const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
1327
- const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
1328
- const parsedTags = tagsMatch
1329
- ? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
1330
- : [];
1331
- const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
1332
- blocks.push({
1333
- name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
1334
- domain: domainMatch?.[1] ?? 'uncategorized',
1335
- status: statusMatch?.[1] ?? 'draft',
1336
- owner: ownerMatch?.[1] ?? null,
1337
- tags: parsedTags,
1338
- path: relPath,
1339
- lastModified: stat.mtime.toISOString(),
1340
- description: descMatch?.[1] ?? '',
1341
- llmContext: llmMatch?.[1] ?? null,
1342
- });
1343
- }
1344
- catch { /* skip unreadable files */ }
2236
+ const seen = new Set();
2237
+ const scanDir = (dir) => {
2238
+ if (!existsSync(dir))
2239
+ return;
2240
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
2241
+ const filePath = join(dir, entry.name);
2242
+ if (entry.isDirectory()) {
2243
+ scanDir(filePath);
2244
+ }
2245
+ else if (entry.name.endsWith('.dql') && !seen.has(filePath)) {
2246
+ seen.add(filePath);
2247
+ const relPath = relative(projectRoot, filePath).replaceAll('\\', '/');
2248
+ try {
2249
+ const source = readFileSync(filePath, 'utf-8');
2250
+ const stat = statSync(filePath);
2251
+ // Quick regex parse for key block fields
2252
+ const nameMatch = /block\s+"([^"]+)"/.exec(source);
2253
+ const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
2254
+ const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
2255
+ const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
2256
+ const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
2257
+ const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
2258
+ const parsedTags = tagsMatch
2259
+ ? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
2260
+ : [];
2261
+ const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
2262
+ blocks.push({
2263
+ name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
2264
+ domain: (domainMatch?.[1] ?? inferBlockStudioPathDomain(relPath)) || 'uncategorized',
2265
+ status: statusMatch?.[1] ?? 'draft',
2266
+ owner: ownerMatch?.[1] ?? null,
2267
+ tags: parsedTags,
2268
+ path: relPath,
2269
+ lastModified: stat.mtime.toISOString(),
2270
+ description: descMatch?.[1] ?? '',
2271
+ llmContext: llmMatch?.[1] ?? null,
2272
+ });
1345
2273
  }
2274
+ catch { /* skip unreadable files */ }
1346
2275
  }
1347
- };
1348
- scanDir(blocksDir);
1349
- }
2276
+ }
2277
+ };
2278
+ scanDir(join(projectRoot, 'blocks'));
2279
+ scanDir(join(projectRoot, 'domains'));
1350
2280
  blocks.sort((a, b) => new Date(b.lastModified).getTime() - new Date(a.lastModified).getTime());
1351
2281
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1352
2282
  res.end(serializeJSON({ blocks }));
@@ -1555,20 +2485,216 @@ export async function startLocalServer(opts) {
1555
2485
  res.end(serializeJSON({ error: 'source is required' }));
1556
2486
  return;
1557
2487
  }
1558
- const result = await certifyBlockStudioSource(source, blockPath);
2488
+ const result = await certifyBlockStudioSource(source, blockPath, { enterprise: body.enterprise !== false });
1559
2489
  const blockers = Array.from(new Set(result.checklist.blockers));
1560
2490
  if (!result.certification.certified || blockers.length > 0) {
1561
2491
  res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
1562
2492
  res.end(serializeJSON({ ok: false, ...result, blockers }));
1563
2493
  return;
1564
2494
  }
1565
- if (blockPath)
1566
- setBlockStudioStatus(projectRoot, blockPath, 'certified');
2495
+ let certifiedPayload = null;
2496
+ const certifiedSource = setBlockStudioStatusInSource(source, 'certified');
2497
+ if (blockPath) {
2498
+ const normalizedBlockPath = normalize(blockPath).replace(/^\/+/, '');
2499
+ const parsed = parseBlockSourceMetadata(certifiedSource);
2500
+ if (isDraftBlockPath(normalizedBlockPath)) {
2501
+ const savedPath = saveBlockStudioArtifacts(projectRoot, {
2502
+ currentPath: normalizedBlockPath,
2503
+ source: certifiedSource,
2504
+ name: parsed.name,
2505
+ domain: parsed.domain,
2506
+ description: parsed.description,
2507
+ owner: parsed.owner,
2508
+ tags: parsed.tags,
2509
+ });
2510
+ certifiedPayload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
2511
+ }
2512
+ else {
2513
+ setBlockStudioStatus(projectRoot, normalizedBlockPath, 'certified');
2514
+ certifiedPayload = openBlockStudioDocument(projectRoot, normalizedBlockPath, semanticLayer);
2515
+ }
2516
+ }
1567
2517
  await refreshLocalMetadataCatalog(projectRoot);
1568
2518
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
1569
- res.end(serializeJSON({ ok: true, status: 'certified', ...result }));
2519
+ res.end(serializeJSON({
2520
+ ok: true,
2521
+ status: 'certified',
2522
+ ...result,
2523
+ path: certifiedPayload?.path ?? blockPath ?? null,
2524
+ source: certifiedPayload?.source ?? certifiedSource,
2525
+ metadata: certifiedPayload?.metadata,
2526
+ companionPath: certifiedPayload?.companionPath ?? null,
2527
+ validation: certifiedPayload?.validation ?? result.validation,
2528
+ }));
2529
+ }
2530
+ catch (error) {
2531
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2532
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2533
+ }
2534
+ return;
2535
+ }
2536
+ if (req.method === 'POST' && path === '/api/block-studio/ai-imports') {
2537
+ try {
2538
+ const body = await readJSON(req);
2539
+ const session = await createDqlGenerationSessionFromBody(body);
2540
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2541
+ res.end(serializeJSON(session));
2542
+ }
2543
+ catch (error) {
2544
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2545
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2546
+ }
2547
+ return;
2548
+ }
2549
+ if (req.method === 'POST' && path === '/api/block-studio/match-sql') {
2550
+ try {
2551
+ const body = await readJSON(req);
2552
+ if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
2553
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2554
+ res.end(serializeJSON({ error: 'Missing SQL in request body.' }));
2555
+ return;
2556
+ }
2557
+ const sql = body.sql.trim();
2558
+ const match = await matchSqlForDqlReuse({
2559
+ sql,
2560
+ sourcePath: typeof body.sourcePath === 'string' ? body.sourcePath : 'pasted.sql',
2561
+ name: typeof body.name === 'string' ? body.name : 'SQL match preview',
2562
+ domain: typeof body.domain === 'string' ? body.domain : 'imported',
2563
+ owner: typeof body.owner === 'string' ? body.owner : 'analytics',
2564
+ });
2565
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2566
+ res.end(serializeJSON(match));
2567
+ }
2568
+ catch (error) {
2569
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2570
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2571
+ }
2572
+ return;
2573
+ }
2574
+ const aiImportPathMatch = path.match(/^\/api\/block-studio\/ai-imports\/([^/]+)(?:\/candidates\/([^/]+)(?:\/(preview|certify))?)?$/);
2575
+ if (aiImportPathMatch) {
2576
+ const importId = decodeURIComponent(aiImportPathMatch[1]);
2577
+ const candidateId = aiImportPathMatch[2] ? decodeURIComponent(aiImportPathMatch[2]) : null;
2578
+ const action = aiImportPathMatch[3] ?? null;
2579
+ try {
2580
+ if (req.method === 'GET' && !candidateId) {
2581
+ const session = loadDqlGenerationSession(importId);
2582
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2583
+ res.end(serializeJSON(session));
2584
+ return;
2585
+ }
2586
+ if (req.method === 'PATCH' && candidateId && !action) {
2587
+ const body = await readJSON(req);
2588
+ const candidate = updateBlockStudioImportCandidate(projectRoot, importId, candidateId, {
2589
+ name: typeof body.name === 'string' ? body.name : undefined,
2590
+ domain: typeof body.domain === 'string' ? body.domain : undefined,
2591
+ description: typeof body.description === 'string' ? body.description : undefined,
2592
+ owner: typeof body.owner === 'string' ? body.owner : undefined,
2593
+ tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
2594
+ terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
2595
+ pattern: typeof body.pattern === 'string' ? body.pattern : undefined,
2596
+ grain: typeof body.grain === 'string' ? body.grain : undefined,
2597
+ entities: Array.isArray(body.entities) ? body.entities.map(String) : undefined,
2598
+ outputs: Array.isArray(body.outputs) ? body.outputs.map(String) : undefined,
2599
+ dimensions: Array.isArray(body.dimensions) ? body.dimensions.map(String) : undefined,
2600
+ allowedFilters: Array.isArray(body.allowedFilters) ? body.allowedFilters.map(String) : undefined,
2601
+ parameterPolicy: Array.isArray(body.parameterPolicy) ? body.parameterPolicy : undefined,
2602
+ filterBindings: Array.isArray(body.filterBindings) ? body.filterBindings : undefined,
2603
+ sourceSystems: Array.isArray(body.sourceSystems) ? body.sourceSystems.map(String) : undefined,
2604
+ replacementFor: Array.isArray(body.replacementFor) ? body.replacementFor.map(String) : undefined,
2605
+ reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
2606
+ sql: typeof body.sql === 'string' ? body.sql : undefined,
2607
+ llmContext: typeof body.llmContext === 'string' ? body.llmContext : undefined,
2608
+ });
2609
+ const validated = validateImportCandidate(candidate);
2610
+ const savedDraft = saveDqlGenerationDraft(importId, validated);
2611
+ writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
2612
+ await refreshLocalMetadataCatalog(projectRoot);
2613
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2614
+ res.end(serializeJSON(savedDraft));
2615
+ return;
2616
+ }
2617
+ if (req.method === 'POST' && candidateId && action === 'preview') {
2618
+ const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
2619
+ const preview = await runBlockStudioPreviewSource(candidate.dqlSource);
2620
+ const next = saveDqlGenerationDraft(importId, {
2621
+ ...candidate,
2622
+ preview,
2623
+ validation: validateBlockStudioSource(candidate.dqlSource, semanticLayer),
2624
+ });
2625
+ writeBlockStudioImportCandidate(projectRoot, importId, next);
2626
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2627
+ res.end(serializeJSON(next));
2628
+ return;
2629
+ }
2630
+ if (req.method === 'POST' && candidateId && action === 'certify') {
2631
+ const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
2632
+ const readiness = validateImportCandidateForSave(candidate);
2633
+ if (readiness.errors.length > 0) {
2634
+ const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
2635
+ writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
2636
+ res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
2637
+ res.end(serializeJSON({
2638
+ error: readiness.errors.join(' '),
2639
+ candidate: savedDraft,
2640
+ diagnostics: savedDraft.validation?.diagnostics ?? [],
2641
+ }));
2642
+ return;
2643
+ }
2644
+ const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
2645
+ const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: true });
2646
+ const blockers = Array.from(new Set(certification.checklist.blockers));
2647
+ if (!certification.certification.certified || blockers.length > 0) {
2648
+ const savedDraft = saveDqlGenerationDraft(importId, readiness.candidate);
2649
+ writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
2650
+ res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
2651
+ res.end(serializeJSON({ ok: false, candidate: savedDraft, blockers, ...certification }));
2652
+ return;
2653
+ }
2654
+ const savedPath = saveBlockStudioArtifacts(projectRoot, {
2655
+ currentPath: readiness.candidate.savedPath,
2656
+ source: certifiedSource,
2657
+ name: readiness.candidate.name,
2658
+ domain: readiness.candidate.domain,
2659
+ description: readiness.candidate.description,
2660
+ owner: readiness.candidate.owner,
2661
+ tags: readiness.candidate.tags,
2662
+ lineage: readiness.candidate.lineage.sourceTables,
2663
+ importMeta: {
2664
+ importId,
2665
+ candidateId,
2666
+ sourceKind: readiness.candidate.sourceKind,
2667
+ sourcePath: readiness.candidate.sourcePath,
2668
+ },
2669
+ });
2670
+ const next = {
2671
+ ...readiness.candidate,
2672
+ dqlSource: certifiedSource,
2673
+ reviewStatus: 'saved',
2674
+ savedPath,
2675
+ validation: validateBlockStudioSource(certifiedSource, semanticLayer),
2676
+ generationMode: readiness.candidate.generationMode ?? 'deterministic',
2677
+ generationProvider: readiness.candidate.generationProvider ?? 'local-deterministic',
2678
+ llmContext: readiness.candidate.llmContext ?? deterministicDqlGenerationContext(readiness.candidate, readiness.candidate.evidence ?? []),
2679
+ evidence: readiness.candidate.evidence ?? [],
2680
+ draftSave: readiness.candidate.draftSave ?? { status: 'pending' },
2681
+ };
2682
+ writeBlockStudioImportCandidate(projectRoot, importId, next);
2683
+ await refreshLocalMetadataCatalog(projectRoot);
2684
+ const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
2685
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2686
+ res.end(serializeJSON({ candidate: next, block: payload, certification }));
2687
+ return;
2688
+ }
2689
+ res.writeHead(405, { 'Content-Type': 'application/json; charset=utf-8' });
2690
+ res.end(serializeJSON({ error: 'Unsupported AI import operation.' }));
1570
2691
  }
1571
2692
  catch (error) {
2693
+ if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
2694
+ res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
2695
+ res.end(serializeJSON({ error: 'Block already exists' }));
2696
+ return;
2697
+ }
1572
2698
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
1573
2699
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
1574
2700
  }
@@ -1708,6 +2834,8 @@ export async function startLocalServer(opts) {
1708
2834
  description: typeof body.description === 'string' ? body.description : undefined,
1709
2835
  owner: typeof body.owner === 'string' ? body.owner : undefined,
1710
2836
  tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
2837
+ terms: Array.isArray(body.terms) ? body.terms.map(String) : undefined,
2838
+ reviewCadence: typeof body.reviewCadence === 'string' ? body.reviewCadence : undefined,
1711
2839
  sql: typeof body.sql === 'string' ? body.sql : undefined,
1712
2840
  reviewStatus,
1713
2841
  });
@@ -1844,6 +2972,55 @@ export async function startLocalServer(opts) {
1844
2972
  }
1845
2973
  return;
1846
2974
  }
2975
+ if (req.method === 'GET' && path === '/api/semantic-layer/diagnostics') {
2976
+ try {
2977
+ const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
2978
+ semanticLayer,
2979
+ semanticErrors: semanticLayerErrors,
2980
+ semanticConfig,
2981
+ detectedProvider: semanticDetectedProvider,
2982
+ lastSyncTime: semanticLastSyncTime,
2983
+ });
2984
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2985
+ res.end(serializeJSON(diagnostics));
2986
+ }
2987
+ catch (error) {
2988
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2989
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
2990
+ }
2991
+ return;
2992
+ }
2993
+ if (req.method === 'POST' && path === '/api/semantic-layer/reload') {
2994
+ try {
2995
+ const semanticConnection = connection;
2996
+ const executeQuery = semanticConfig?.provider === 'snowflake' && semanticConnection
2997
+ ? async (sql) => { const r = await executor.executeQuery(sql, [], {}, semanticConnection); return { rows: r.rows }; }
2998
+ : undefined;
2999
+ const refreshed = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
3000
+ semanticLayer = refreshed.layer;
3001
+ semanticLayerErrors = refreshed.errors;
3002
+ semanticDetectedProvider = refreshed.detectedProvider;
3003
+ semanticLastSyncTime = refreshed.layer ? new Date().toISOString() : null;
3004
+ semanticImportManifest = loadSemanticImportManifest(projectRoot);
3005
+ const diagnostics = buildSemanticLayerDiagnostics(projectRoot, projectConfig, {
3006
+ semanticLayer,
3007
+ semanticErrors: semanticLayerErrors,
3008
+ semanticConfig,
3009
+ detectedProvider: semanticDetectedProvider,
3010
+ lastSyncTime: semanticLastSyncTime,
3011
+ });
3012
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3013
+ res.end(serializeJSON({
3014
+ ok: Boolean(refreshed.layer),
3015
+ ...diagnostics,
3016
+ }));
3017
+ }
3018
+ catch (error) {
3019
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3020
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3021
+ }
3022
+ return;
3023
+ }
1847
3024
  if (req.method === 'GET' && path === '/api/block-studio/open') {
1848
3025
  try {
1849
3026
  const relativePath = url.searchParams.get('path');
@@ -1908,8 +3085,9 @@ export async function startLocalServer(opts) {
1908
3085
  res.end(serializeJSON({ error: 'Block name is required.' }));
1909
3086
  return;
1910
3087
  }
1911
- const savedPath = saveBlockStudioArtifacts(projectRoot, {
1912
- currentPath: typeof body.path === 'string' ? body.path : undefined,
3088
+ const currentPath = typeof body.path === 'string' ? body.path : undefined;
3089
+ const saveOptions = {
3090
+ currentPath,
1913
3091
  source,
1914
3092
  name: metadata.name,
1915
3093
  domain: metadata.domain,
@@ -1925,7 +3103,13 @@ export async function startLocalServer(opts) {
1925
3103
  sourcePath: metadata.sourcePath,
1926
3104
  }
1927
3105
  : undefined,
1928
- });
3106
+ };
3107
+ const savedPath = isDraftBlockPath(currentPath)
3108
+ ? saveBlockStudioDraftArtifacts(projectRoot, {
3109
+ ...saveOptions,
3110
+ stableSuffix: metadata.candidateId,
3111
+ })
3112
+ : saveBlockStudioArtifacts(projectRoot, saveOptions);
1929
3113
  await refreshLocalMetadataCatalog(projectRoot);
1930
3114
  const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
1931
3115
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -2051,7 +3235,7 @@ export async function startLocalServer(opts) {
2051
3235
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2052
3236
  res.end(serializeJSON({
2053
3237
  available: false,
2054
- provider: projectConfig.semanticLayer?.provider ?? semanticDetectedProvider ?? null,
3238
+ provider: semanticConfig?.provider ?? semanticDetectedProvider ?? null,
2055
3239
  errors: semanticLayerErrors,
2056
3240
  metrics: [],
2057
3241
  measures: [],
@@ -2180,9 +3364,9 @@ export async function startLocalServer(opts) {
2180
3364
  owner: q.owner ?? null,
2181
3365
  source: q.source ?? null,
2182
3366
  }));
2183
- const provider = projectConfig.semanticLayer?.provider ?? semanticDetectedProvider ?? 'dql';
3367
+ const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
2184
3368
  const dbtExecutionReady = provider === 'dbt'
2185
- ? hasDbtSemanticManifest(projectRoot, projectConfig.semanticLayer?.projectPath)
3369
+ ? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
2186
3370
  : false;
2187
3371
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2188
3372
  res.end(serializeJSON({
@@ -2754,8 +3938,12 @@ export async function startLocalServer(opts) {
2754
3938
  return;
2755
3939
  }
2756
3940
  if (req.method === 'POST' && path === '/api/query') {
3941
+ let body;
3942
+ let execContext = null;
3943
+ const start = Date.now();
2757
3944
  try {
2758
- const body = await readJSON(req);
3945
+ body = await readJSON(req);
3946
+ execContext = notebookExecutionContext(body.executionContext);
2759
3947
  if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
2760
3948
  res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
2761
3949
  res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
@@ -2778,7 +3966,26 @@ export async function startLocalServer(opts) {
2778
3966
  const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
2779
3967
  assertAppAccess({ app, domain, level: 'execute' });
2780
3968
  const result = await executor.executeQuery(prepared.sql, Array.isArray(body.sqlParams) ? body.sqlParams : [], runtimeVariables(body.variables && typeof body.variables === 'object' ? body.variables : {}), prepared.connection);
2781
- const payload = serializeJSON(normalizeQueryResult(result, semantic.semanticRefs));
3969
+ const normalized = normalizeQueryResult(result, semantic.semanticRefs);
3970
+ if (execContext) {
3971
+ recordNotebookQueryRun(projectRoot, {
3972
+ notebookPath: execContext.notebookPath,
3973
+ cellId: execContext.cellId,
3974
+ cellName: execContext.cellName,
3975
+ researchRunId: execContext.researchRunId,
3976
+ source: execContext.source ?? 'notebook_sql_cell',
3977
+ status: 'success',
3978
+ rowCount: normalized.rowCount ?? normalized.rows.length,
3979
+ durationMs: Date.now() - start,
3980
+ sql: body.sql,
3981
+ });
3982
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
3983
+ status: 'success',
3984
+ resultPreview: normalized,
3985
+ sql: body.sql,
3986
+ });
3987
+ }
3988
+ const payload = serializeJSON(normalized);
2782
3989
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2783
3990
  res.end(payload);
2784
3991
  }
@@ -2797,6 +4004,24 @@ export async function startLocalServer(opts) {
2797
4004
  }));
2798
4005
  return;
2799
4006
  }
4007
+ if (execContext) {
4008
+ recordNotebookQueryRun(projectRoot, {
4009
+ notebookPath: execContext.notebookPath,
4010
+ cellId: execContext.cellId,
4011
+ cellName: execContext.cellName,
4012
+ researchRunId: execContext.researchRunId,
4013
+ source: execContext.source ?? 'notebook_sql_cell',
4014
+ status: 'error',
4015
+ durationMs: Date.now() - start,
4016
+ errorCode: error instanceof Error ? error.message : String(error),
4017
+ sql: typeof body?.sql === 'string' ? body.sql : undefined,
4018
+ });
4019
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
4020
+ status: 'error',
4021
+ error: error instanceof Error ? error.message : String(error),
4022
+ sql: typeof body?.sql === 'string' ? body.sql : undefined,
4023
+ });
4024
+ }
2800
4025
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
2801
4026
  res.end(serializeJSON({
2802
4027
  columns: [],
@@ -3138,6 +4363,25 @@ export async function startLocalServer(opts) {
3138
4363
  }
3139
4364
  return;
3140
4365
  }
4366
+ if (req.method === 'GET' && path.startsWith('/api/lineage/business-360/')) {
4367
+ const rawNodeId = decodeURIComponent(path.slice('/api/lineage/business-360/'.length));
4368
+ try {
4369
+ const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
4370
+ const result = queryBusiness360(graph, rawNodeId);
4371
+ if (!result) {
4372
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
4373
+ res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
4374
+ return;
4375
+ }
4376
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
4377
+ res.end(serializeJSON(result));
4378
+ }
4379
+ catch (error) {
4380
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
4381
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
4382
+ }
4383
+ return;
4384
+ }
3141
4385
  if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
3142
4386
  const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
3143
4387
  try {
@@ -3303,8 +4547,12 @@ export async function startLocalServer(opts) {
3303
4547
  return;
3304
4548
  }
3305
4549
  if (req.method === 'POST' && path === '/api/notebook/execute') {
4550
+ let body;
4551
+ let execContext = null;
4552
+ const start = Date.now();
3306
4553
  try {
3307
- const body = await readJSON(req);
4554
+ body = await readJSON(req);
4555
+ execContext = notebookExecutionContext(body.executionContext);
3308
4556
  const cell = normalizeNotebookCell(body.cell);
3309
4557
  if (!cell) {
3310
4558
  res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -3327,6 +4575,26 @@ export async function startLocalServer(opts) {
3327
4575
  const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
3328
4576
  assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
3329
4577
  const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
4578
+ const normalized = normalizeQueryResult(rawResult);
4579
+ if (execContext) {
4580
+ recordNotebookQueryRun(projectRoot, {
4581
+ notebookPath: execContext.notebookPath,
4582
+ cellId: execContext.cellId ?? cell.id,
4583
+ cellName: execContext.cellName ?? plan.title ?? resolved.blockName,
4584
+ researchRunId: execContext.researchRunId,
4585
+ source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_cell'),
4586
+ status: 'success',
4587
+ rowCount: normalized.rowCount ?? normalized.rows.length,
4588
+ durationMs: Date.now() - start,
4589
+ sql: plan.sql,
4590
+ objectKey: resolved.blockPath,
4591
+ });
4592
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
4593
+ status: 'success',
4594
+ resultPreview: normalized,
4595
+ sql: plan.sql,
4596
+ });
4597
+ }
3330
4598
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3331
4599
  res.end(serializeJSON({
3332
4600
  cellType: cell.type,
@@ -3335,10 +4603,26 @@ export async function startLocalServer(opts) {
3335
4603
  blockPath: resolved.blockPath,
3336
4604
  chartConfig: plan.chartConfig,
3337
4605
  tests: plan.tests,
3338
- result: normalizeQueryResult(rawResult),
4606
+ result: normalized,
3339
4607
  }));
3340
4608
  }
3341
4609
  catch (error) {
4610
+ if (execContext) {
4611
+ recordNotebookQueryRun(projectRoot, {
4612
+ notebookPath: execContext.notebookPath,
4613
+ cellId: execContext.cellId,
4614
+ cellName: execContext.cellName,
4615
+ researchRunId: execContext.researchRunId,
4616
+ source: execContext.source ?? 'notebook_cell',
4617
+ status: 'error',
4618
+ durationMs: Date.now() - start,
4619
+ errorCode: error instanceof Error ? error.message : String(error),
4620
+ });
4621
+ updateNotebookResearchFromCellExecution(projectRoot, execContext, {
4622
+ status: 'error',
4623
+ error: error instanceof Error ? error.message : String(error),
4624
+ });
4625
+ }
3342
4626
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3343
4627
  res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3344
4628
  }
@@ -3867,6 +5151,245 @@ export function prepareLocalExecution(sql, connection, projectRoot, projectConfi
3867
5151
  connection: normalizedConnection,
3868
5152
  };
3869
5153
  }
5154
+ export function dashboardRuntimeVariables(dashboard, overrides = {}) {
5155
+ const variables = {};
5156
+ for (const param of dashboard.params ?? []) {
5157
+ if (param.default !== undefined)
5158
+ variables[param.id] = param.default;
5159
+ }
5160
+ for (const filter of dashboard.filters ?? []) {
5161
+ if (filter.default !== undefined)
5162
+ variables[filter.id] = filter.default;
5163
+ }
5164
+ return { ...variables, ...overrides };
5165
+ }
5166
+ export function applyDashboardFiltersToBlockExecution(input) {
5167
+ const variables = { ...input.variables };
5168
+ const sqlParams = [...input.sqlParams];
5169
+ const appliedFilters = [];
5170
+ const skippedFilters = [];
5171
+ const clauses = [];
5172
+ let nextPosition = sqlParams.reduce((max, param) => Math.max(max, param.position), 0);
5173
+ for (const filter of input.dashboard.filters ?? []) {
5174
+ const value = dashboardFilterValue(filter, variables);
5175
+ if (isEmptyDashboardFilterValue(value)) {
5176
+ skippedFilters.push({ filter: filter.id, reason: 'no value supplied' });
5177
+ continue;
5178
+ }
5179
+ const paramNames = bindDashboardFilterToExistingParams(filter, value, input.block, sqlParams, variables);
5180
+ if (paramNames.length > 0) {
5181
+ appliedFilters.push({
5182
+ filter: filter.id,
5183
+ mode: 'parameter',
5184
+ paramNames,
5185
+ });
5186
+ continue;
5187
+ }
5188
+ const binding = resolveDashboardFilterBinding(filter, input.block);
5189
+ if (!binding) {
5190
+ skippedFilters.push({ filter: filter.id, reason: `block "${input.block.name}" does not declare a compatible filter binding` });
5191
+ continue;
5192
+ }
5193
+ const expression = dashboardFilterExpression(binding);
5194
+ if (!expression) {
5195
+ skippedFilters.push({ filter: filter.id, reason: `filter binding "${binding}" is not safe for runtime predicate injection` });
5196
+ continue;
5197
+ }
5198
+ const predicate = buildDashboardFilterPredicate({
5199
+ expression,
5200
+ filterId: filter.id,
5201
+ filterType: filter.type,
5202
+ value,
5203
+ params: sqlParams,
5204
+ nextPosition: () => {
5205
+ nextPosition += 1;
5206
+ return nextPosition;
5207
+ },
5208
+ variables,
5209
+ });
5210
+ if (!predicate) {
5211
+ skippedFilters.push({ filter: filter.id, reason: 'filter value could not be converted into a predicate' });
5212
+ continue;
5213
+ }
5214
+ clauses.push(predicate);
5215
+ appliedFilters.push({
5216
+ filter: filter.id,
5217
+ binding,
5218
+ mode: 'predicate',
5219
+ paramNames: sqlParams
5220
+ .filter((param) => param.name.startsWith(`__dashboard_filter_${normalizeDashboardFilterName(filter.id)}_`))
5221
+ .map((param) => param.name),
5222
+ });
5223
+ }
5224
+ if (clauses.length === 0) {
5225
+ return { sql: input.sql, sqlParams, variables, appliedFilters, skippedFilters };
5226
+ }
5227
+ return {
5228
+ sql: `SELECT * FROM (${stripSqlTerminator(input.sql)}) _dql_filter WHERE ${clauses.join(' AND ')}`,
5229
+ sqlParams,
5230
+ variables,
5231
+ appliedFilters,
5232
+ skippedFilters,
5233
+ };
5234
+ }
5235
+ function dashboardFilterValue(filter, variables) {
5236
+ if (Object.prototype.hasOwnProperty.call(variables, filter.id))
5237
+ return variables[filter.id];
5238
+ if (filter.bindsTo && Object.prototype.hasOwnProperty.call(variables, filter.bindsTo))
5239
+ return variables[filter.bindsTo];
5240
+ return filter.default;
5241
+ }
5242
+ function isEmptyDashboardFilterValue(value) {
5243
+ if (value === undefined || value === null)
5244
+ return true;
5245
+ if (typeof value === 'string' && value.trim() === '')
5246
+ return true;
5247
+ if (Array.isArray(value) && value.length === 0)
5248
+ return true;
5249
+ return false;
5250
+ }
5251
+ function bindDashboardFilterToExistingParams(filter, value, block, sqlParams, variables) {
5252
+ const availableParamNames = new Set(sqlParams.map((param) => param.name));
5253
+ const declaredParamNames = new Set((block.parameterPolicy ?? []).map((entry) => entry.name));
5254
+ const applied = [];
5255
+ const directCandidates = uniqueDashboardStrings([
5256
+ filter.id,
5257
+ filter.bindsTo ?? '',
5258
+ normalizeDashboardFilterName(filter.id),
5259
+ filter.bindsTo ? normalizeDashboardFilterName(filter.bindsTo) : '',
5260
+ ]);
5261
+ for (const name of directCandidates) {
5262
+ if (!name || !availableParamNames.has(name))
5263
+ continue;
5264
+ variables[name] = value;
5265
+ applied.push(name);
5266
+ }
5267
+ if (applied.length > 0)
5268
+ return applied;
5269
+ const range = dashboardRangeValue(value);
5270
+ if (range) {
5271
+ const baseNames = uniqueDashboardStrings([
5272
+ filter.id,
5273
+ filter.id.replace(/_?range$/i, ''),
5274
+ filter.bindsTo ?? '',
5275
+ filter.bindsTo ? filter.bindsTo.replace(/_?range$/i, '') : '',
5276
+ ].map(normalizeDashboardFilterName));
5277
+ for (const base of baseNames) {
5278
+ const pairs = [
5279
+ [`${base}_start`, `${base}_end`],
5280
+ [`${base}_from`, `${base}_to`],
5281
+ [`start_${base}`, `end_${base}`],
5282
+ ];
5283
+ for (const [startName, endName] of pairs) {
5284
+ if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
5285
+ variables[startName] = range.start;
5286
+ variables[endName] = range.end;
5287
+ return [startName, endName];
5288
+ }
5289
+ }
5290
+ }
5291
+ for (const [startName, endName] of [['start_date', 'end_date'], ['date_start', 'date_end'], ['season_start', 'season_end'], ['year_start', 'year_end']]) {
5292
+ if (availableParamNames.has(startName) && availableParamNames.has(endName)) {
5293
+ variables[startName] = range.start;
5294
+ variables[endName] = range.end;
5295
+ return [startName, endName];
5296
+ }
5297
+ }
5298
+ }
5299
+ for (const name of declaredParamNames) {
5300
+ if (!availableParamNames.has(name))
5301
+ continue;
5302
+ const normalized = normalizeDashboardFilterName(name);
5303
+ if (normalized === normalizeDashboardFilterName(filter.id) || normalized === normalizeDashboardFilterName(filter.bindsTo ?? '')) {
5304
+ variables[name] = value;
5305
+ return [name];
5306
+ }
5307
+ }
5308
+ return [];
5309
+ }
5310
+ function resolveDashboardFilterBinding(filter, block) {
5311
+ const candidates = uniqueDashboardStrings([filter.id, filter.bindsTo ?? '']).map(normalizeDashboardFilterName);
5312
+ for (const entry of block.filterBindings ?? []) {
5313
+ if (candidates.includes(normalizeDashboardFilterName(entry.filter)))
5314
+ return entry.binding;
5315
+ }
5316
+ if (filter.bindsTo && (block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.bindsTo ?? ''))) {
5317
+ return filter.bindsTo;
5318
+ }
5319
+ if ((block.allowedFilters ?? []).some((item) => normalizeDashboardFilterName(item) === normalizeDashboardFilterName(filter.id))) {
5320
+ return filter.id;
5321
+ }
5322
+ return null;
5323
+ }
5324
+ function dashboardFilterExpression(binding) {
5325
+ const trimmed = binding.trim();
5326
+ const yearMatch = trimmed.match(/^([A-Za-z_][A-Za-z0-9_.]*)\.year$/i);
5327
+ if (yearMatch) {
5328
+ const base = dashboardOutputColumn(yearMatch[1]);
5329
+ return base ? `EXTRACT(YEAR FROM _dql_filter.${base})` : null;
5330
+ }
5331
+ const column = dashboardOutputColumn(trimmed);
5332
+ return column ? `_dql_filter.${column}` : null;
5333
+ }
5334
+ function dashboardOutputColumn(binding) {
5335
+ const cleaned = binding.replace(/[`"[\]]/g, '').trim();
5336
+ if (!/^[A-Za-z_][A-Za-z0-9_.]*$/.test(cleaned))
5337
+ return null;
5338
+ const column = cleaned.split('.').filter(Boolean).at(-1);
5339
+ return column && /^[A-Za-z_][A-Za-z0-9_]*$/.test(column) ? column : null;
5340
+ }
5341
+ function buildDashboardFilterPredicate(input) {
5342
+ const range = input.filterType === 'daterange' ? dashboardRangeValue(input.value) : null;
5343
+ if (range) {
5344
+ const start = addDashboardFilterParam(input, 'start', range.start);
5345
+ const end = addDashboardFilterParam(input, 'end', range.end);
5346
+ return `${input.expression} BETWEEN $${start.position} AND $${end.position}`;
5347
+ }
5348
+ const values = Array.isArray(input.value) ? input.value.filter((item) => !isEmptyDashboardFilterValue(item)) : [input.value];
5349
+ if (values.length === 0)
5350
+ return null;
5351
+ if (values.length === 1) {
5352
+ const param = addDashboardFilterParam(input, 'value', values[0]);
5353
+ return `${input.expression} = $${param.position}`;
5354
+ }
5355
+ const placeholders = values.map((value, index) => {
5356
+ const param = addDashboardFilterParam(input, `value_${index + 1}`, value);
5357
+ return `$${param.position}`;
5358
+ });
5359
+ return `${input.expression} IN (${placeholders.join(', ')})`;
5360
+ }
5361
+ function addDashboardFilterParam(input, suffix, value) {
5362
+ const name = `__dashboard_filter_${normalizeDashboardFilterName(input.filterId)}_${suffix}`;
5363
+ const uniqueName = input.variables[name] === undefined && !input.params.some((param) => param.name === name)
5364
+ ? name
5365
+ : `${name}_${input.params.length + 1}`;
5366
+ const param = { name: uniqueName, position: input.nextPosition() };
5367
+ input.params.push(param);
5368
+ input.variables[uniqueName] = value;
5369
+ return param;
5370
+ }
5371
+ function dashboardRangeValue(value) {
5372
+ if (Array.isArray(value) && value.length >= 2 && !isEmptyDashboardFilterValue(value[0]) && !isEmptyDashboardFilterValue(value[1])) {
5373
+ return { start: value[0], end: value[1] };
5374
+ }
5375
+ if (value && typeof value === 'object') {
5376
+ const record = value;
5377
+ const start = record.start ?? record.from ?? record.min;
5378
+ const end = record.end ?? record.to ?? record.max;
5379
+ if (!isEmptyDashboardFilterValue(start) && !isEmptyDashboardFilterValue(end))
5380
+ return { start, end };
5381
+ }
5382
+ return null;
5383
+ }
5384
+ function normalizeDashboardFilterName(value) {
5385
+ return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
5386
+ }
5387
+ function stripSqlTerminator(sql) {
5388
+ return sql.trim().replace(/;\s*$/, '');
5389
+ }
5390
+ function uniqueDashboardStrings(values) {
5391
+ return Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
5392
+ }
3870
5393
  export function resolveDbtMacrosForExecution(sql, projectRoot, projectConfig = {}) {
3871
5394
  if (!/\{\{\s*(?:ref|source)\s*\(/i.test(sql))
3872
5395
  return sql;
@@ -4484,9 +6007,9 @@ function buildDatabaseSchemaTree(projectRoot, executor, connection) {
4484
6007
  return databaseNodes;
4485
6008
  })();
4486
6009
  }
4487
- function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
6010
+ export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
4488
6011
  const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
4489
- if (!normalizedPath.startsWith('blocks/')) {
6012
+ if (!isBlockStudioBlockPath(normalizedPath)) {
4490
6013
  throw new Error('Invalid block path');
4491
6014
  }
4492
6015
  const absPath = join(projectRoot, normalizedPath);
@@ -4501,7 +6024,7 @@ function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
4501
6024
  const metadata = {
4502
6025
  name: parsedMetadata.name || companion?.name || fileName,
4503
6026
  path: normalizedPath,
4504
- domain: parsedMetadata.domain || companion?.domain || normalizedPath.split('/').slice(1, -1).join('/') || 'uncategorized',
6027
+ domain: parsedMetadata.domain || companion?.domain || inferBlockStudioPathDomain(normalizedPath) || 'uncategorized',
4505
6028
  description: parsedMetadata.description || companion?.description || '',
4506
6029
  owner: parsedMetadata.owner || companion?.owner || '',
4507
6030
  tags: parsedMetadata.tags.length > 0 ? parsedMetadata.tags : companion?.tags ?? [],
@@ -4584,7 +6107,7 @@ export function buildSemanticTableMapping(semanticLayer, rows) {
4584
6107
  return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
4585
6108
  }
4586
6109
  function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
4587
- if (projectConfig.semanticLayer?.provider === 'dbt' || detectedProvider === 'dbt')
6110
+ if (projectConfig.semanticLayer?.provider === 'dbt' || projectConfig.dbt?.projectDir || detectedProvider === 'dbt')
4588
6111
  return true;
4589
6112
  return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
4590
6113
  }
@@ -4592,7 +6115,10 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
4592
6115
  const useMetricFlow = request.engine === 'metricflow' || (request.engine !== 'native' &&
4593
6116
  isDbtSemanticRuntime(context.projectConfig, context.detectedProvider, semanticLayer));
4594
6117
  if (useMetricFlow) {
4595
- const dbtProjectPath = context.projectConfig.semanticLayer?.projectPath;
6118
+ const effectiveSemanticConfig = resolveProjectSemanticConfig(context.projectConfig, context.projectRoot);
6119
+ const dbtProjectPath = effectiveSemanticConfig?.provider === 'dbt'
6120
+ ? effectiveSemanticConfig.projectPath
6121
+ : context.projectConfig.dbt?.projectDir;
4596
6122
  const compiled = compileMetricFlowQuery({
4597
6123
  projectRoot: context.projectRoot,
4598
6124
  dbtProjectPath,
@@ -4851,6 +6377,164 @@ export function validateBlockStudioSource(source, semanticLayer) {
4851
6377
  executableSql,
4852
6378
  };
4853
6379
  }
6380
+ export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
6381
+ const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
6382
+ const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
6383
+ const provider = deterministicOnly ? null : await createBlockStudioAssistProvider(projectRoot, requestedProvider);
6384
+ const session = createBlockStudioImportSession(projectRoot, {
6385
+ inputPath: options.inputPath ?? '',
6386
+ inputMode: options.inputMode,
6387
+ sources: options.sources,
6388
+ sourceKind: options.sourceKind ?? 'raw-sql',
6389
+ domain: options.domain,
6390
+ owner: options.owner,
6391
+ tags: options.tags,
6392
+ });
6393
+ const warnings = [];
6394
+ const nextCandidates = [];
6395
+ let contextObjectCount = 0;
6396
+ for (const candidate of session.candidates) {
6397
+ const contextPack = await buildDqlGenerationContextPack(projectRoot, candidate).catch((error) => {
6398
+ warnings.push(`Context pack failed for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
6399
+ return null;
6400
+ });
6401
+ contextObjectCount += contextPack?.objects.length ?? 0;
6402
+ const evidence = contextPack ? dqlGenerationEvidenceFromContext(contextPack, candidate) : deterministicDqlGenerationEvidence(candidate);
6403
+ let patch = deterministicDqlGenerationPatch(candidate, evidence);
6404
+ let generatorName = 'local-deterministic';
6405
+ let generationMode = 'deterministic';
6406
+ if (provider) {
6407
+ const aiPatch = await buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack).catch((error) => {
6408
+ warnings.push(`AI generation fell back for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
6409
+ return null;
6410
+ });
6411
+ if (aiPatch) {
6412
+ patch = mergeDqlGenerationPatch(patch, aiPatch, candidate, evidence);
6413
+ generatorName = provider.name;
6414
+ generationMode = 'ai';
6415
+ }
6416
+ }
6417
+ const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
6418
+ patch = {
6419
+ ...patch,
6420
+ similarityMatches: similarity.matches,
6421
+ recommendedAction: similarity.recommendedAction,
6422
+ };
6423
+ for (const warning of patch.warnings ?? [])
6424
+ warnings.push(`${candidate.name}: ${warning}`);
6425
+ const enriched = updateBlockStudioImportCandidate(projectRoot, session.id, candidate.id, {
6426
+ name: patch.name,
6427
+ domain: patch.domain,
6428
+ description: patch.description,
6429
+ owner: patch.owner,
6430
+ tags: patch.tags,
6431
+ terms: patch.terms,
6432
+ pattern: patch.pattern,
6433
+ grain: patch.grain,
6434
+ entities: patch.entities,
6435
+ outputs: patch.outputs,
6436
+ dimensions: patch.dimensions,
6437
+ allowedFilters: patch.allowedFilters,
6438
+ parameterPolicy: patch.parameterPolicy,
6439
+ filterBindings: patch.filterBindings,
6440
+ parameterDecisions: patch.parameterDecisions,
6441
+ similarityMatches: patch.similarityMatches,
6442
+ recommendedAction: patch.recommendedAction,
6443
+ sourceSystems: patch.sourceSystems,
6444
+ replacementFor: patch.replacementFor,
6445
+ reviewCadence: patch.reviewCadence,
6446
+ sql: patch.sql,
6447
+ llmContext: patch.llmContext,
6448
+ evidence,
6449
+ conversionNotes: dqlGenerationConversionNotes(generatorName),
6450
+ generationMode,
6451
+ generationProvider: generatorName,
6452
+ });
6453
+ const validated = {
6454
+ ...enriched,
6455
+ validation: validateBlockStudioSource(enriched.dqlSource, semanticLayer),
6456
+ };
6457
+ const savedDraft = saveDqlGenerationDraftForProject(projectRoot, session.id, validated);
6458
+ writeBlockStudioImportCandidate(projectRoot, session.id, savedDraft);
6459
+ nextCandidates.push(savedDraft);
6460
+ }
6461
+ const generationSession = {
6462
+ ...session,
6463
+ mode: 'ai-import',
6464
+ candidates: nextCandidates,
6465
+ updatedAt: new Date().toISOString(),
6466
+ generation: {
6467
+ provider: nextCandidates.find((candidate) => candidate.generationMode === 'ai')?.generationProvider ?? 'local-deterministic',
6468
+ aiEnabled: nextCandidates.some((candidate) => candidate.generationMode === 'ai'),
6469
+ contextObjectCount,
6470
+ createdDrafts: nextCandidates.filter((candidate) => candidate.draftSave.status === 'saved').length,
6471
+ warnings,
6472
+ },
6473
+ };
6474
+ writeBlockStudioImportSession(projectRoot, generationSession);
6475
+ if (generationSession.generation.createdDrafts > 0)
6476
+ await refreshLocalMetadataCatalog(projectRoot);
6477
+ return generationSession;
6478
+ }
6479
+ function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
6480
+ if (candidate.recommendedAction === 'reuse_existing') {
6481
+ const topMatch = candidate.similarityMatches?.[0];
6482
+ return {
6483
+ ...candidate,
6484
+ reviewStatus: 'review',
6485
+ draftSave: {
6486
+ status: 'skipped',
6487
+ reason: topMatch
6488
+ ? `Reuse recommended: ${topMatch.name} (${topMatch.kind}, ${(topMatch.score * 100).toFixed(0)}%).`
6489
+ : 'Reuse recommended; no new draft block was needed.',
6490
+ },
6491
+ generationMode: candidate.generationMode ?? 'deterministic',
6492
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
6493
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
6494
+ evidence: candidate.evidence ?? [],
6495
+ };
6496
+ }
6497
+ try {
6498
+ const savedPath = saveBlockStudioDraftArtifacts(projectRoot, {
6499
+ currentPath: isDraftBlockPath(candidate.savedPath) ? candidate.savedPath : undefined,
6500
+ source: candidate.dqlSource,
6501
+ name: candidate.name,
6502
+ domain: candidate.domain,
6503
+ description: candidate.description,
6504
+ owner: candidate.owner,
6505
+ tags: candidate.tags,
6506
+ lineage: candidate.lineage.sourceTables,
6507
+ stableSuffix: candidate.id.replace(/^cand_/, ''),
6508
+ importMeta: {
6509
+ importId,
6510
+ candidateId: candidate.id,
6511
+ sourceKind: candidate.sourceKind,
6512
+ sourcePath: candidate.sourcePath,
6513
+ },
6514
+ });
6515
+ const draftSave = { status: 'saved', path: savedPath, savedAt: new Date().toISOString() };
6516
+ return {
6517
+ ...candidate,
6518
+ reviewStatus: candidate.reviewStatus === 'rejected' ? 'rejected' : 'draft',
6519
+ savedPath,
6520
+ draftSave,
6521
+ generationMode: candidate.generationMode ?? 'deterministic',
6522
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
6523
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
6524
+ evidence: candidate.evidence ?? [],
6525
+ };
6526
+ }
6527
+ catch (error) {
6528
+ return {
6529
+ ...candidate,
6530
+ draftSave: { status: 'error', error: error instanceof Error ? error.message : String(error) },
6531
+ generationMode: candidate.generationMode ?? 'deterministic',
6532
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
6533
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
6534
+ evidence: candidate.evidence ?? [],
6535
+ };
6536
+ }
6537
+ }
4854
6538
  export function saveBlockStudioArtifacts(projectRoot, options) {
4855
6539
  const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
4856
6540
  const safeDomain = (options.domain ?? '')
@@ -4858,9 +6542,9 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
4858
6542
  .toLowerCase()
4859
6543
  .replace(/[^a-z0-9/_-]+/g, '-')
4860
6544
  .replace(/^\/+|\/+$/g, '') || 'uncategorized';
4861
- const targetRelativePath = `blocks/${safeDomain}/${slug}.dql`;
4862
- const targetPath = join(projectRoot, targetRelativePath);
4863
6545
  const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
6546
+ const targetRelativePath = canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath);
6547
+ const targetPath = join(projectRoot, targetRelativePath);
4864
6548
  if (existsSync(targetPath) && previousPath !== targetRelativePath) {
4865
6549
  throw new Error('BLOCK_EXISTS');
4866
6550
  }
@@ -4891,12 +6575,103 @@ export function saveBlockStudioArtifacts(projectRoot, options) {
4891
6575
  }
4892
6576
  return targetRelativePath;
4893
6577
  }
6578
+ export function saveBlockStudioDraftArtifacts(projectRoot, options) {
6579
+ const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
6580
+ const safeDomain = (options.domain ?? '')
6581
+ .trim()
6582
+ .toLowerCase()
6583
+ .replace(/[^a-z0-9/_-]+/g, '-')
6584
+ .replace(/^\/+|\/+$/g, '') || 'uncategorized';
6585
+ const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
6586
+ const domainFirstRoot = join(projectRoot, 'domains');
6587
+ const useDomainFirstDrafts = existsSync(domainFirstRoot);
6588
+ const draftPrefix = useDomainFirstDrafts
6589
+ ? `domains/${safeDomain}/blocks/_drafts/`
6590
+ : `blocks/_drafts/${safeDomain}/`;
6591
+ const stableSuffix = options.stableSuffix?.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 18);
6592
+ let targetRelativePath = previousPath && isDraftBlockPath(previousPath)
6593
+ ? previousPath
6594
+ : `${draftPrefix}${slug}${stableSuffix ? `-${stableSuffix}` : ''}.dql`;
6595
+ let targetPath = join(projectRoot, targetRelativePath);
6596
+ if (!previousPath && existsSync(targetPath)) {
6597
+ for (let index = 2; index < 1000; index += 1) {
6598
+ const candidatePath = `${draftPrefix}${slug}-${index}.dql`;
6599
+ if (!existsSync(join(projectRoot, candidatePath))) {
6600
+ targetRelativePath = candidatePath;
6601
+ targetPath = join(projectRoot, targetRelativePath);
6602
+ break;
6603
+ }
6604
+ }
6605
+ }
6606
+ mkdirSync(dirname(targetPath), { recursive: true });
6607
+ writeFileSync(targetPath, options.source.trimEnd() + '\n', 'utf-8');
6608
+ const companionSlug = targetRelativePath.split('/').pop()?.replace(/\.dql$/, '') || slug;
6609
+ writeBlockCompanionFile(projectRoot, {
6610
+ slug: companionSlug,
6611
+ name: options.name,
6612
+ domain: `_drafts/${safeDomain}`,
6613
+ description: options.description,
6614
+ owner: options.owner,
6615
+ tags: options.tags,
6616
+ provider: 'dql',
6617
+ content: options.source,
6618
+ lineage: options.lineage,
6619
+ importMeta: options.importMeta,
6620
+ });
6621
+ return targetRelativePath;
6622
+ }
6623
+ function canonicalBlockRelativePath(projectRoot, safeDomain, slug, previousPath) {
6624
+ const previousDomainFirst = previousPath?.match(/^domains\/([^/]+)\/blocks\/(?:_drafts\/)?[^/]+\.dql$/);
6625
+ if (previousDomainFirst) {
6626
+ return `domains/${previousDomainFirst[1]}/blocks/${slug}.dql`;
6627
+ }
6628
+ if (existsSync(join(projectRoot, 'domains', safeDomain))) {
6629
+ return `domains/${safeDomain}/blocks/${slug}.dql`;
6630
+ }
6631
+ return `blocks/${safeDomain}/${slug}.dql`;
6632
+ }
6633
+ function isDraftBlockPath(value) {
6634
+ if (!value)
6635
+ return false;
6636
+ const normalized = normalize(value).replace(/^\/+/, '');
6637
+ return normalized.startsWith('blocks/_drafts/') || /^domains\/[^/]+\/blocks\/_drafts\//.test(normalized);
6638
+ }
6639
+ function isBlockStudioBlockPath(value) {
6640
+ if (!value)
6641
+ return false;
6642
+ const normalized = normalize(value).replace(/^\/+/, '');
6643
+ return normalized.startsWith('blocks/') || /^domains\/[^/]+\/blocks\//.test(normalized);
6644
+ }
6645
+ function inferBlockStudioPathDomain(blockPath) {
6646
+ const normalized = normalize(blockPath).replace(/^\/+/, '');
6647
+ if (normalized.startsWith('blocks/')) {
6648
+ return normalized.split('/').slice(1, -1).join('/');
6649
+ }
6650
+ const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)$/);
6651
+ if (!domainFirst)
6652
+ return '';
6653
+ const domain = domainFirst[1];
6654
+ const blockSubpath = domainFirst[2];
6655
+ if (blockSubpath.startsWith('_drafts/'))
6656
+ return `_drafts/${domain}`;
6657
+ return domain;
6658
+ }
4894
6659
  function blockCompanionRelativePath(blockPath) {
4895
6660
  const normalized = normalize(blockPath).replace(/^\/+/, '');
4896
- if (!normalized.startsWith('blocks/'))
4897
- return null;
4898
- const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
4899
- return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
6661
+ if (normalized.startsWith('blocks/')) {
6662
+ const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
6663
+ return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
6664
+ }
6665
+ const domainFirst = normalized.match(/^domains\/([^/]+)\/blocks\/(.+)\.dql$/);
6666
+ if (domainFirst) {
6667
+ const domain = domainFirst[1];
6668
+ const blockPath = domainFirst[2];
6669
+ const companionBlockPath = blockPath.startsWith('_drafts/')
6670
+ ? join('_drafts', domain, blockPath.slice('_drafts/'.length))
6671
+ : join(domain, blockPath);
6672
+ return join('semantic-layer', 'blocks', `${companionBlockPath}.yaml`).replaceAll('\\', '/');
6673
+ }
6674
+ return null;
4900
6675
  }
4901
6676
  function readBlockCompanionFile(projectRoot, relativePath) {
4902
6677
  const absPath = join(projectRoot, relativePath);
@@ -4944,18 +6719,48 @@ function readBlockCompanionFile(projectRoot, relativePath) {
4944
6719
  return null;
4945
6720
  }
4946
6721
  }
4947
- function parseBlockSourceMetadata(source) {
6722
+ export function parseBlockSourceMetadata(source) {
4948
6723
  const name = source.match(/^\s*block\s+"([^"]+)"/i)?.[1] ?? '';
4949
6724
  const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
4950
- const tags = source.match(/\btags\s*=\s*\[([^\]]*)\]/i);
6725
+ const extractStringArray = (key) => {
6726
+ const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([^\\]]*)\\]`, 'i'));
6727
+ return match ? (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [];
6728
+ };
6729
+ const extractStringMapSection = (key) => {
6730
+ const match = source.match(new RegExp(`\\b${key}\\s*\\{([\\s\\S]*?)\\n\\s*\\}`, 'i'));
6731
+ if (!match)
6732
+ return [];
6733
+ return match[1]
6734
+ .split(/\r?\n/)
6735
+ .map((line) => line.trim())
6736
+ .map((line) => line.match(/^([A-Za-z_][\w.]*)\s*=\s*"([^"]*)"\s*$/))
6737
+ .filter((entry) => Boolean(entry))
6738
+ .map((entry) => ({ key: entry[1], value: entry[2] }));
6739
+ };
6740
+ const parameterPolicy = extractStringMapSection('parameterPolicy')
6741
+ .map((entry) => ({ name: entry.key, policy: entry.value }));
6742
+ const filterBindings = extractStringMapSection('filterBindings')
6743
+ .map((entry) => ({ filter: entry.key, binding: entry.value }));
4951
6744
  return {
4952
6745
  name,
4953
6746
  domain: extractString('domain'),
4954
6747
  description: extractString('description'),
4955
6748
  owner: extractString('owner'),
4956
- tags: tags ? (tags[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [],
6749
+ tags: extractStringArray('tags'),
4957
6750
  status: extractString('status') || 'draft',
4958
6751
  blockType: extractString('type') || 'custom',
6752
+ llmContext: extractString('llmContext'),
6753
+ pattern: extractString('pattern'),
6754
+ grain: extractString('grain'),
6755
+ entities: extractStringArray('entities'),
6756
+ outputs: extractStringArray('outputs'),
6757
+ dimensions: extractStringArray('dimensions'),
6758
+ allowedFilters: extractStringArray('allowedFilters'),
6759
+ parameterPolicy,
6760
+ filterBindings,
6761
+ sourceSystems: extractStringArray('sourceSystems'),
6762
+ replacementFor: extractStringArray('replacementFor'),
6763
+ reviewCadence: extractString('reviewCadence'),
4959
6764
  };
4960
6765
  }
4961
6766
  function compareBlockStudioValues(actual, operator, expected) {
@@ -5012,58 +6817,948 @@ function buildBlockStudioCertificationChecklist(input) {
5012
6817
  if (!input.validation.chartConfig?.chart)
5013
6818
  blockers.add('Visualization config is missing');
5014
6819
  return {
5015
- metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
5016
- validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
5017
- run: input.previewSucceeded,
5018
- tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
5019
- chart: Boolean(input.validation.chartConfig?.chart),
5020
- lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
5021
- aiReviewed: true,
5022
- blockers: Array.from(blockers),
5023
- checkedAt: new Date().toISOString(),
6820
+ metadata: Boolean(parsed.domain.trim() && parsed.owner.trim() && parsed.description.trim()),
6821
+ validation: input.validation.diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
6822
+ run: input.previewSucceeded,
6823
+ tests: Boolean(input.testResults && input.testResults.failed === 0 && input.testResults.assertions.length > 0),
6824
+ chart: Boolean(input.validation.chartConfig?.chart),
6825
+ lineage: extractSqlTablesLight(sql).length > 0 || input.validation.semanticRefs.metrics.length > 0,
6826
+ aiReviewed: true,
6827
+ blockers: Array.from(blockers),
6828
+ checkedAt: new Date().toISOString(),
6829
+ };
6830
+ }
6831
+ function extractSqlTablesLight(sql) {
6832
+ const tables = new Set();
6833
+ const cleaned = sql
6834
+ .replace(/\/\*[\s\S]*?\*\//g, ' ')
6835
+ .replace(/--[^\n\r]*/g, ' ')
6836
+ .replace(/extract\s*\(\s*\w+\s+from\s+[^)]+\)/gi, 'EXTRACT_VALUE');
6837
+ const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
6838
+ let match;
6839
+ while ((match = regex.exec(cleaned))) {
6840
+ const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
6841
+ if (raw && !raw.startsWith('(') && !/^(select|values|unnest|lateral)$/i.test(raw))
6842
+ tables.add(raw);
6843
+ }
6844
+ return Array.from(tables);
6845
+ }
6846
+ function setBlockStudioStatusInSource(source, newStatus) {
6847
+ if (/status\s*=\s*"[^"]*"/.test(source)) {
6848
+ return source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
6849
+ }
6850
+ return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
6851
+ }
6852
+ export function setBlockStudioStatus(projectRoot, blockPath, newStatus) {
6853
+ const normalizedPath = normalize(blockPath).replace(/^\/+/, '');
6854
+ if (!isBlockStudioBlockPath(normalizedPath))
6855
+ throw new Error('Invalid block path');
6856
+ const absPath = join(projectRoot, normalizedPath);
6857
+ if (!existsSync(absPath))
6858
+ throw new Error('Block file not found');
6859
+ const source = setBlockStudioStatusInSource(readFileSync(absPath, 'utf-8'), newStatus);
6860
+ writeFileSync(absPath, source, 'utf-8');
6861
+ const companionPath = blockCompanionRelativePath(normalizedPath);
6862
+ if (!companionPath)
6863
+ return;
6864
+ const absCompanionPath = join(projectRoot, companionPath);
6865
+ if (!existsSync(absCompanionPath))
6866
+ return;
6867
+ let companion = readFileSync(absCompanionPath, 'utf-8');
6868
+ if (/^reviewStatus:\s*.+$/m.test(companion)) {
6869
+ companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
6870
+ }
6871
+ else {
6872
+ companion = `${companion.trimEnd()}\nreviewStatus: ${newStatus}\n`;
6873
+ }
6874
+ writeFileSync(absCompanionPath, companion, 'utf-8');
6875
+ }
6876
+ async function buildDqlGenerationContextPack(projectRoot, candidate) {
6877
+ const tables = candidate.lineage.sourceTables.join(', ');
6878
+ const question = [
6879
+ `Generate a governed DQL block draft for SQL import candidate "${candidate.name}".`,
6880
+ candidate.description,
6881
+ tables ? `Source tables: ${tables}.` : '',
6882
+ candidate.sql.slice(0, 1200),
6883
+ ].filter(Boolean).join('\n');
6884
+ return buildLocalContextPack(projectRoot, {
6885
+ question,
6886
+ mode: 'build',
6887
+ surface: 'block-studio',
6888
+ limit: 80,
6889
+ objectTypes: [
6890
+ 'dql_block',
6891
+ 'dql_term',
6892
+ 'business_view',
6893
+ 'domain',
6894
+ 'semantic_metric',
6895
+ 'semantic_model',
6896
+ 'semantic_dimension',
6897
+ 'semantic_measure',
6898
+ 'dbt_model',
6899
+ 'dbt_source',
6900
+ 'dbt_column',
6901
+ 'warehouse_table',
6902
+ 'datalex_domain',
6903
+ 'datalex_entity',
6904
+ 'datalex_contract',
6905
+ 'datalex_term',
6906
+ ],
6907
+ strictness: 'balanced',
6908
+ });
6909
+ }
6910
+ function dqlGenerationEvidenceFromContext(contextPack, candidate) {
6911
+ const directEvidence = contextPack.objects
6912
+ .filter((object) => isDqlGenerationEvidenceObject(object))
6913
+ .filter((object) => metadataObjectMatchesImportedSource(object, candidate.lineage.sourceTables))
6914
+ .map((object) => metadataObjectToDqlGenerationEvidence(object, 'Directly matches an imported SQL source table.'));
6915
+ const selected = contextPack.retrievalDiagnostics.selectedEvidence.slice(0, 12);
6916
+ const byKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));
6917
+ const evidence = selected.flatMap((item) => {
6918
+ const object = byKey.get(item.objectKey);
6919
+ if (!object || !isDqlGenerationEvidenceObject(object))
6920
+ return [];
6921
+ return [metadataObjectToDqlGenerationEvidence(object, item.reason, Math.max(0.35, Math.min(0.98, item.score / Math.max(item.score, 10))))];
6922
+ });
6923
+ const ranked = uniqueDqlGenerationEvidence([...directEvidence, ...evidence]).slice(0, 12);
6924
+ return ranked.length > 0 ? ranked : deterministicDqlGenerationEvidence(candidate);
6925
+ }
6926
+ function deterministicDqlGenerationEvidence(candidate) {
6927
+ return candidate.lineage.sourceTables.map((table) => ({
6928
+ kind: 'warehouse_table',
6929
+ name: table,
6930
+ source: candidate.sourcePath,
6931
+ reason: 'Detected in the imported SQL source.',
6932
+ confidence: 0.7,
6933
+ }));
6934
+ }
6935
+ function isDqlGenerationEvidenceObject(object) {
6936
+ if (object.objectType === 'dql_block') {
6937
+ if (object.status !== 'certified')
6938
+ return false;
6939
+ if (object.sourcePath?.includes('/_drafts/') || object.sourcePath?.includes('\\_drafts\\'))
6940
+ return false;
6941
+ }
6942
+ return true;
6943
+ }
6944
+ function metadataObjectToDqlGenerationEvidence(object, reason, confidence = 0.82) {
6945
+ return {
6946
+ kind: dqlGenerationEvidenceKind(object.objectType),
6947
+ name: object.fullName ?? object.name,
6948
+ description: object.description,
6949
+ objectKey: object.objectKey,
6950
+ source: object.sourcePath ?? object.sourceSystem,
6951
+ reason,
6952
+ confidence,
6953
+ };
6954
+ }
6955
+ function metadataObjectMatchesImportedSource(object, sourceTables) {
6956
+ if (sourceTables.length === 0)
6957
+ return false;
6958
+ const haystack = [
6959
+ object.fullName,
6960
+ object.name,
6961
+ object.objectKey,
6962
+ object.sourcePath,
6963
+ object.sourceSystem,
6964
+ typeof object.payload?.relation === 'string' ? object.payload.relation : '',
6965
+ typeof object.payload?.relationName === 'string' ? object.payload.relationName : '',
6966
+ typeof object.payload?.relation_name === 'string' ? object.payload.relation_name : '',
6967
+ ].filter(Boolean).join(' ').toLowerCase();
6968
+ return sourceTables.some((table) => relationLookupTokens(table).some((token) => token && haystack.includes(token)));
6969
+ }
6970
+ function relationLookupTokens(relation) {
6971
+ const normalized = relation.replace(/[`"[\]]/g, '').toLowerCase();
6972
+ const parts = normalized.split('.').filter(Boolean);
6973
+ return Array.from(new Set([
6974
+ normalized,
6975
+ parts.slice(-2).join('.'),
6976
+ parts.at(-1) ?? '',
6977
+ ].filter((item) => item.length >= 3)));
6978
+ }
6979
+ function uniqueDqlGenerationEvidence(evidence) {
6980
+ const seen = new Set();
6981
+ const result = [];
6982
+ for (const item of evidence) {
6983
+ const key = item.objectKey || `${item.kind}:${item.name}`.toLowerCase();
6984
+ if (seen.has(key))
6985
+ continue;
6986
+ seen.add(key);
6987
+ result.push(item);
6988
+ }
6989
+ return result;
6990
+ }
6991
+ function dqlGenerationEvidenceKind(objectType) {
6992
+ if (objectType === 'dql_block')
6993
+ return 'dql_block';
6994
+ if (objectType === 'dql_term')
6995
+ return 'dql_term';
6996
+ if (objectType === 'business_view')
6997
+ return 'business_view';
6998
+ if (objectType === 'domain')
6999
+ return 'domain';
7000
+ if (objectType === 'semantic_metric')
7001
+ return 'semantic_metric';
7002
+ if (objectType === 'semantic_model')
7003
+ return 'semantic_model';
7004
+ if (objectType === 'dbt_model' || objectType === 'dbt_source' || objectType === 'dbt_column')
7005
+ return 'dbt_model';
7006
+ if (objectType === 'warehouse_table')
7007
+ return 'warehouse_table';
7008
+ if (objectType === 'datalex_contract')
7009
+ return 'datalex_contract';
7010
+ if (objectType === 'datalex_entity')
7011
+ return 'datalex_entity';
7012
+ if (objectType === 'datalex_domain')
7013
+ return 'datalex_domain';
7014
+ if (objectType === 'datalex_term')
7015
+ return 'datalex_term';
7016
+ if (objectType.includes('lineage'))
7017
+ return 'lineage';
7018
+ return 'metadata';
7019
+ }
7020
+ function deterministicDqlGenerationPatch(candidate, evidence) {
7021
+ const domain = inferDqlGenerationDomain(candidate);
7022
+ const parameterized = parameterizeSqlForDqlImport(candidate.sql);
7023
+ const grain = extractDqlGenerationGroupByFields(parameterized.sql)[0];
7024
+ const outputs = extractDqlGenerationSelectOutputs(parameterized.sql);
7025
+ const sourceSystems = candidate.lineage.sourceTables.map((table) => table.split('.').filter(Boolean).slice(-2, -1)[0] ?? '').filter(Boolean);
7026
+ const description = candidate.description && !candidate.description.startsWith('Imported from ')
7027
+ ? candidate.description
7028
+ : deterministicDqlGenerationDescription(candidate, evidence);
7029
+ const allowedFilters = Array.from(new Set([
7030
+ ...parameterized.allowedFilters,
7031
+ ...extractDqlGenerationFilterFields(parameterized.sql),
7032
+ ])).slice(0, 16);
7033
+ return {
7034
+ name: candidate.name,
7035
+ domain,
7036
+ description,
7037
+ owner: candidate.owner,
7038
+ tags: dqlGenerationBusinessTags(candidate, evidence, domain),
7039
+ terms: inferDqlGenerationTerms(evidence),
7040
+ llmContext: deterministicDqlGenerationContext({ ...candidate, sql: parameterized.sql }, evidence),
7041
+ pattern: inferDqlGenerationPattern(parameterized.sql),
7042
+ grain,
7043
+ entities: inferDqlGenerationEntities({
7044
+ grain,
7045
+ outputs,
7046
+ sourceTables: candidate.lineage.sourceTables,
7047
+ evidence,
7048
+ }),
7049
+ outputs,
7050
+ dimensions: extractDqlGenerationDimensions(parameterized.sql, grain, outputs),
7051
+ allowedFilters,
7052
+ parameterPolicy: parameterized.parameterPolicy,
7053
+ filterBindings: parameterized.filterBindings,
7054
+ parameterDecisions: parameterized.parameterDecisions,
7055
+ sql: parameterized.sql,
7056
+ warnings: parameterized.warnings,
7057
+ sourceSystems,
7058
+ reviewCadence: 'monthly',
7059
+ };
7060
+ }
7061
+ function inferDqlGenerationTerms(evidence) {
7062
+ return Array.from(new Set(evidence
7063
+ .filter((item) => item.kind === 'dql_term' || item.kind === 'datalex_term')
7064
+ .map((item) => item.name.trim())
7065
+ .filter(Boolean))).slice(0, 16);
7066
+ }
7067
+ function deterministicDqlGenerationDescription(candidate, evidence) {
7068
+ const tables = candidate.lineage.sourceTables.length > 0
7069
+ ? candidate.lineage.sourceTables.join(', ')
7070
+ : candidate.sourcePath;
7071
+ const sourceContext = evidence.find((item) => (item.kind === 'dbt_model' || item.kind === 'semantic_model' || item.kind === 'warehouse_table') &&
7072
+ item.description &&
7073
+ candidate.lineage.sourceTables.some((table) => relationLookupTokens(table).some((token) => item.name.toLowerCase().includes(token))));
7074
+ const aggregate = /\b(sum|count|avg|min|max)\s*\(/i.test(candidate.sql);
7075
+ const grouped = /\bgroup\s+by\b/i.test(candidate.sql);
7076
+ const sourcePhrase = sourceContext?.description
7077
+ ? ` using ${sourceContext.name} (${sourceContext.description})`
7078
+ : ` using ${tables}`;
7079
+ if (aggregate && grouped)
7080
+ return `Summarizes imported SQL${sourcePhrase}.`;
7081
+ if (aggregate)
7082
+ return `Calculates aggregate values${sourcePhrase}.`;
7083
+ return `Exposes imported SQL${sourcePhrase}.`;
7084
+ }
7085
+ function dqlGenerationBusinessTags(candidate, evidence, domain) {
7086
+ const generic = new Set(['imported', 'raw-sql', 'ai-generated']);
7087
+ const stopTags = new Set(['int', 'stg', 'src', 'fct', 'dim', 'name', 'id', 'key', 'model', 'source', 'table']);
7088
+ const sourceTokens = candidate.lineage.sourceTables
7089
+ .flatMap((table) => table.split('.').slice(-2))
7090
+ .flatMap((part) => part.split(/[_\s.-]+/));
7091
+ const tags = [
7092
+ ...candidate.tags.filter((tag) => !generic.has(tag)),
7093
+ domain,
7094
+ ...sourceTokens,
7095
+ 'review-required',
7096
+ ].map((tag) => tag.trim().toLowerCase())
7097
+ .filter((tag) => tag.length > 1 && !generic.has(tag) && !stopTags.has(tag))
7098
+ .map((tag) => tag.replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, ''))
7099
+ .filter(Boolean);
7100
+ return Array.from(new Set(tags)).slice(0, 12);
7101
+ }
7102
+ function deterministicDqlGenerationContext(candidate, evidence) {
7103
+ const tables = candidate.lineage.sourceTables.length > 0
7104
+ ? candidate.lineage.sourceTables.join(', ')
7105
+ : 'the imported SQL source';
7106
+ const evidenceNames = evidence
7107
+ .filter((item) => item.kind !== 'dql_block')
7108
+ .slice(0, 3)
7109
+ .map((item) => item.name)
7110
+ .filter(Boolean);
7111
+ const grain = extractDqlGenerationGroupByFields(candidate.sql);
7112
+ const years = extractDqlGenerationYearFilters(candidate.sql);
7113
+ return [
7114
+ `Use after review for questions matching "${candidate.name}".`,
7115
+ grain.length > 0 ? `Grain: ${grain.join(', ')}.` : '',
7116
+ years.length > 0 ? `Filters: year ${years.join(', ')}.` : '',
7117
+ `Source: ${tables}.`,
7118
+ evidenceNames.length > 0 ? `Grounded by: ${evidenceNames.join(', ')}.` : '',
7119
+ 'Certification must confirm grain, filters, owner, tags, and tests.',
7120
+ ].filter(Boolean).join(' ');
7121
+ }
7122
+ function extractDqlGenerationGroupByFields(sql) {
7123
+ const match = sql.match(/\bgroup\s+by\b([\s\S]+?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i);
7124
+ if (!match)
7125
+ return [];
7126
+ return splitDqlGenerationSqlList(match[1])
7127
+ .map((item) => item.replace(/[`"[\]]/g, '').trim())
7128
+ .filter((item) => item && !/^\d+$/.test(item))
7129
+ .slice(0, 4);
7130
+ }
7131
+ function extractDqlGenerationSelectOutputs(sql) {
7132
+ const selectMatch = sql.match(/\bselect\b([\s\S]+?)\bfrom\b/i);
7133
+ if (!selectMatch)
7134
+ return [];
7135
+ return splitDqlGenerationSqlList(selectMatch[1])
7136
+ .map((expr) => {
7137
+ const alias = expr.match(/\bas\s+([A-Za-z_][A-Za-z0-9_]*)\b/i)?.[1]
7138
+ ?? expr.match(/([A-Za-z_][A-Za-z0-9_]*)\s*$/)?.[1]
7139
+ ?? '';
7140
+ return alias.replace(/[`"[\]]/g, '');
7141
+ })
7142
+ .filter(Boolean)
7143
+ .filter((name) => !/^(from|where|group|order|limit)$/i.test(name))
7144
+ .slice(0, 24);
7145
+ }
7146
+ function extractDqlGenerationDimensions(sql, grain, outputs) {
7147
+ const outputSet = new Set(outputs.map(normalizedTerm));
7148
+ return extractDqlGenerationGroupByFields(sql)
7149
+ .map((field) => field.split('.').pop() ?? field)
7150
+ .filter((field) => normalizedTerm(field) !== normalizedTerm(grain ?? ''))
7151
+ .filter((field) => !outputSet.has(normalizedTerm(field)))
7152
+ .filter((field) => !/\b(total|count|sum|avg|average|min|max|rate|pct|percent|amount|revenue|points?|score)\b/i.test(field))
7153
+ .slice(0, 16);
7154
+ }
7155
+ function inferDqlGenerationEntities(input) {
7156
+ const entities = new Set();
7157
+ const add = (value) => {
7158
+ const entity = businessEntityFromDqlGenerationIdentifier(value ?? '');
7159
+ if (entity)
7160
+ entities.add(entity);
7161
+ };
7162
+ add(input.grain);
7163
+ for (const output of input.outputs)
7164
+ add(output);
7165
+ for (const table of input.sourceTables)
7166
+ add(table.split('.').pop());
7167
+ for (const item of input.evidence.slice(0, 12)) {
7168
+ if (item.kind === 'datalex_entity' || item.kind === 'semantic_model' || item.kind === 'dbt_model' || item.kind === 'warehouse_table') {
7169
+ add(item.name.split('.').pop());
7170
+ }
7171
+ }
7172
+ return [...entities].slice(0, 8);
7173
+ }
7174
+ function businessEntityFromDqlGenerationIdentifier(value) {
7175
+ const raw = value
7176
+ .split('.').pop()
7177
+ ?.trim()
7178
+ .replace(/[`"[\]]/g, '')
7179
+ .replace(/^(dim|fact|fct|stg|src|int)_/i, '')
7180
+ .replace(/_(id|key|name|code|uuid|number|num|abbreviation|abbr)$/i, '')
7181
+ .replace(/s$/i, '') ?? '';
7182
+ if (!raw)
7183
+ return '';
7184
+ const tokens = raw
7185
+ .split(/[^A-Za-z0-9]+/)
7186
+ .map((token) => token.trim().toLowerCase())
7187
+ .filter(Boolean)
7188
+ .filter((token) => !isGenericDqlGenerationEntityToken(token));
7189
+ if (tokens.length === 0)
7190
+ return '';
7191
+ return titleizeDqlGenerationName(tokens.join(' '));
7192
+ }
7193
+ function isGenericDqlGenerationEntityToken(token) {
7194
+ return new Set([
7195
+ 'total',
7196
+ 'count',
7197
+ 'sum',
7198
+ 'avg',
7199
+ 'average',
7200
+ 'min',
7201
+ 'max',
7202
+ 'metric',
7203
+ 'measure',
7204
+ 'value',
7205
+ 'amount',
7206
+ 'revenue',
7207
+ 'point',
7208
+ 'points',
7209
+ 'score',
7210
+ 'date',
7211
+ 'day',
7212
+ 'week',
7213
+ 'month',
7214
+ 'quarter',
7215
+ 'year',
7216
+ 'season',
7217
+ 'period',
7218
+ 'time',
7219
+ 'row',
7220
+ 'record',
7221
+ 'detail',
7222
+ 'stat',
7223
+ 'stats',
7224
+ 'analytic',
7225
+ 'analytics',
7226
+ 'transformed',
7227
+ ]).has(token);
7228
+ }
7229
+ function titleizeDqlGenerationName(value) {
7230
+ return value
7231
+ .replace(/[_-]+/g, ' ')
7232
+ .replace(/\s+/g, ' ')
7233
+ .trim()
7234
+ .replace(/\b\w/g, (char) => char.toUpperCase());
7235
+ }
7236
+ function inferDqlGenerationPattern(sql) {
7237
+ const groupFields = extractDqlGenerationGroupByFields(sql);
7238
+ if (/@metric\s*\(/i.test(sql))
7239
+ return 'metric_wrapper';
7240
+ if (/\bjoin\b/i.test(sql)) {
7241
+ const systems = new Set(extractDqlGenerationSourceSystems(sql));
7242
+ if (systems.size > 1)
7243
+ return 'bridge';
7244
+ }
7245
+ if (hasDqlGenerationRankingLimit(sql))
7246
+ return 'ranking';
7247
+ if (groupFields.some((field) => /\b(date|day|week|month|quarter|year|period|time)\b/i.test(field)))
7248
+ return 'trend';
7249
+ if (groupFields.length === 1 && /_id$|_key$/i.test(groupFields[0]))
7250
+ return 'entity_rollup';
7251
+ if (!/\b(sum|count|avg|min|max|median|percentile|rank)\s*\(/i.test(sql) && /\b(dim|profile|customer|account|player|product|user|entity)\b/i.test(sql)) {
7252
+ return 'entity_profile';
7253
+ }
7254
+ return 'custom';
7255
+ }
7256
+ function hasDqlGenerationRankingLimit(sql) {
7257
+ return /\border\s+by\b[\s\S]*\blimit\s+(?:\d+|\$\{\s*[A-Za-z_][A-Za-z0-9_]*\s*\}|[:?][A-Za-z_][A-Za-z0-9_]*)/i.test(sql);
7258
+ }
7259
+ function extractDqlGenerationSourceSystems(sql) {
7260
+ const tables = new Set();
7261
+ const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
7262
+ const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
7263
+ let match;
7264
+ while ((match = regex.exec(cleaned))) {
7265
+ const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
7266
+ const parts = raw.split('.').filter(Boolean);
7267
+ const system = parts.slice(-2, -1)[0] ?? parts[0];
7268
+ if (system)
7269
+ tables.add(system.trim().toLowerCase().replace(/[^a-z0-9_-]+/g, '-'));
7270
+ }
7271
+ return [...tables].filter(Boolean).slice(0, 12);
7272
+ }
7273
+ function extractDqlGenerationFilterFields(sql) {
7274
+ const filters = new Set();
7275
+ const where = sql.match(/\bwhere\b([\s\S]+?)(?:\bgroup\s+by\b|\border\s+by\b|\blimit\b|$)/i)?.[1] ?? '';
7276
+ const addFilter = (value) => {
7277
+ const name = value?.split('.').pop()?.replace(/[`"[\]]/g, '');
7278
+ if (name && !/^(and|or|not|null|year|month|day)$/i.test(name))
7279
+ filters.add(name);
7280
+ };
7281
+ const extractRegex = /\bextract\s*\([^)]*\bfrom\s+([A-Za-z_][A-Za-z0-9_.]*)\s*\)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
7282
+ const isNullRegex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s+is\s+(?:not\s+)?null\b/gi;
7283
+ const regex = /\b([A-Za-z_][A-Za-z0-9_.]*)\s*(?:=|<>|!=|>|<|>=|<=|\bin\b|\blike\b)/gi;
7284
+ let match;
7285
+ while ((match = extractRegex.exec(where)))
7286
+ addFilter(match[1]);
7287
+ while ((match = isNullRegex.exec(where)))
7288
+ addFilter(match[1]);
7289
+ while ((match = regex.exec(where))) {
7290
+ addFilter(match[1]);
7291
+ }
7292
+ return [...filters].slice(0, 16);
7293
+ }
7294
+ function extractDqlGenerationYearFilters(sql) {
7295
+ const years = new Set();
7296
+ const cleaned = sql.replace(/--[^\n\r]*/g, ' ').replace(/\/\*[\s\S]*?\*\//g, ' ');
7297
+ let match;
7298
+ const extractYear = /extract\s*\(\s*year\s+from\s+[^)]+\)\s*(?:=\s*([12][0-9]{3})|in\s*\(([^)]*)\))/gi;
7299
+ while ((match = extractYear.exec(cleaned))) {
7300
+ for (const year of (match[1] || match[2] || '').match(/[12][0-9]{3}/g) ?? [])
7301
+ years.add(year);
7302
+ }
7303
+ return Array.from(years).sort();
7304
+ }
7305
+ function splitDqlGenerationSqlList(value) {
7306
+ const parts = [];
7307
+ let start = 0;
7308
+ let depth = 0;
7309
+ let single = false;
7310
+ let double = false;
7311
+ for (let i = 0; i < value.length; i += 1) {
7312
+ const char = value[i];
7313
+ if (!double && char === "'" && value[i - 1] !== '\\')
7314
+ single = !single;
7315
+ else if (!single && char === '"' && value[i - 1] !== '\\')
7316
+ double = !double;
7317
+ else if (!single && !double && char === '(')
7318
+ depth += 1;
7319
+ else if (!single && !double && char === ')' && depth > 0)
7320
+ depth -= 1;
7321
+ else if (!single && !double && depth === 0 && char === ',') {
7322
+ parts.push(value.slice(start, i));
7323
+ start = i + 1;
7324
+ }
7325
+ }
7326
+ parts.push(value.slice(start));
7327
+ return parts.map((part) => part.trim()).filter(Boolean);
7328
+ }
7329
+ function inferDqlGenerationDomain(candidate) {
7330
+ const current = candidate.domain?.trim();
7331
+ if (current && current !== 'imported' && current !== 'general')
7332
+ return current;
7333
+ const firstTable = candidate.lineage.sourceTables[0];
7334
+ const schema = firstTable?.split('.').filter(Boolean).slice(-2, -1)[0];
7335
+ return schema?.replace(/[^a-z0-9/_-]+/gi, '-').toLowerCase() || current || 'imported';
7336
+ }
7337
+ async function buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack) {
7338
+ const payload = {
7339
+ candidate: {
7340
+ name: candidate.name,
7341
+ domain: candidate.domain,
7342
+ description: candidate.description,
7343
+ owner: candidate.owner,
7344
+ tags: candidate.tags,
7345
+ sql: candidate.sql,
7346
+ sourceTables: candidate.lineage.sourceTables,
7347
+ parameters: candidate.lineage.parameters,
7348
+ warnings: candidate.warnings ?? candidate.lineage.warnings,
7349
+ grain: extractDqlGenerationGroupByFields(candidate.sql),
7350
+ yearFilters: extractDqlGenerationYearFilters(candidate.sql),
7351
+ parameterization: parameterizeSqlForDqlImport(candidate.sql),
7352
+ },
7353
+ context: {
7354
+ evidence,
7355
+ selectedObjects: contextPack?.objects.slice(0, 24).map((object) => ({
7356
+ objectKey: object.objectKey,
7357
+ objectType: object.objectType,
7358
+ name: object.fullName ?? object.name,
7359
+ domain: object.domain,
7360
+ status: object.status,
7361
+ description: object.description,
7362
+ })) ?? [],
7363
+ allowedRelations: contextPack?.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
7364
+ relation: relation.relation,
7365
+ source: relation.source,
7366
+ columns: relation.columns.slice(0, 16).map((column) => column.name),
7367
+ })) ?? [],
7368
+ },
7369
+ };
7370
+ const text = await provider.generate([
7371
+ {
7372
+ role: 'system',
7373
+ content: [
7374
+ 'You generate DQL Block Studio metadata for a local draft.',
7375
+ 'Return only a compact JSON object with optional keys: name, domain, description, owner, tags, terms, llmContext, pattern, grain, entities, outputs, dimensions, allowedFilters, sourceSystems, replacementFor, reviewCadence.',
7376
+ 'Do not return markdown. Do not mark the block certified. Do not change SQL.',
7377
+ 'Use only directly relevant dbt, semantic, warehouse, certified block, and SQL-shape evidence from the payload.',
7378
+ 'terms must reference existing DQL/DataLex glossary terms from the payload. Do not invent term names.',
7379
+ 'If metadata descriptions are missing, describe the observable SQL intent instead of inventing business meaning.',
7380
+ 'Descriptions and llmContext must be specific, business-readable, concise, and review-required.',
7381
+ 'reviewCadence should usually be monthly or quarterly unless evidence strongly suggests a different cadence.',
7382
+ 'Tags must be business/search tags. Avoid generic tags such as raw-sql, imported, ai-generated.',
7383
+ ].join('\n'),
7384
+ },
7385
+ { role: 'user', content: JSON.stringify(payload) },
7386
+ ], { maxTokens: 900, temperature: 0.1 });
7387
+ return normalizeDqlGenerationPatch(parseFirstJsonObject(text));
7388
+ }
7389
+ function parseFirstJsonObject(text) {
7390
+ const trimmed = text.trim();
7391
+ if (trimmed.startsWith('{') && trimmed.endsWith('}'))
7392
+ return JSON.parse(trimmed);
7393
+ const match = trimmed.match(/\{[\s\S]*\}/);
7394
+ if (!match)
7395
+ return null;
7396
+ return JSON.parse(match[0]);
7397
+ }
7398
+ function normalizeDqlGenerationPatch(value) {
7399
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7400
+ return null;
7401
+ const record = value;
7402
+ const patch = {};
7403
+ if (typeof record.name === 'string' && record.name.trim())
7404
+ patch.name = record.name.trim().slice(0, 120);
7405
+ if (typeof record.domain === 'string' && record.domain.trim())
7406
+ patch.domain = record.domain.trim().slice(0, 80);
7407
+ if (typeof record.description === 'string' && record.description.trim())
7408
+ patch.description = record.description.trim().slice(0, 500);
7409
+ if (typeof record.owner === 'string' && record.owner.trim())
7410
+ patch.owner = record.owner.trim().slice(0, 120);
7411
+ if (Array.isArray(record.tags))
7412
+ patch.tags = normalizeDqlGenerationStringArray(record.tags, 12);
7413
+ if (Array.isArray(record.terms))
7414
+ patch.terms = normalizeDqlGenerationStringArray(record.terms, 16);
7415
+ if (typeof record.llmContext === 'string' && record.llmContext.trim())
7416
+ patch.llmContext = record.llmContext.trim().slice(0, 1000);
7417
+ if (typeof record.pattern === 'string' && record.pattern.trim())
7418
+ patch.pattern = normalizeDqlGenerationPattern(record.pattern);
7419
+ if (typeof record.grain === 'string' && record.grain.trim())
7420
+ patch.grain = record.grain.trim().slice(0, 120);
7421
+ if (Array.isArray(record.entities))
7422
+ patch.entities = normalizeDqlGenerationStringArray(record.entities, 12);
7423
+ if (Array.isArray(record.outputs))
7424
+ patch.outputs = normalizeDqlGenerationStringArray(record.outputs, 24);
7425
+ if (Array.isArray(record.dimensions))
7426
+ patch.dimensions = normalizeDqlGenerationStringArray(record.dimensions, 16);
7427
+ if (Array.isArray(record.allowedFilters))
7428
+ patch.allowedFilters = normalizeDqlGenerationStringArray(record.allowedFilters, 16);
7429
+ if (Array.isArray(record.sourceSystems))
7430
+ patch.sourceSystems = normalizeDqlGenerationStringArray(record.sourceSystems, 12);
7431
+ if (Array.isArray(record.replacementFor))
7432
+ patch.replacementFor = normalizeDqlGenerationStringArray(record.replacementFor, 12);
7433
+ if (typeof record.reviewCadence === 'string' && record.reviewCadence.trim()) {
7434
+ const reviewCadence = normalizeDqlGenerationReviewCadence(record.reviewCadence);
7435
+ if (reviewCadence)
7436
+ patch.reviewCadence = reviewCadence;
7437
+ }
7438
+ return Object.keys(patch).length > 0 ? patch : null;
7439
+ }
7440
+ function normalizeDqlGenerationStringArray(value, limit) {
7441
+ const result = [];
7442
+ for (const item of value) {
7443
+ const normalized = normalizeDqlGenerationStringItem(item);
7444
+ if (normalized)
7445
+ result.push(normalized);
7446
+ }
7447
+ return Array.from(new Set(result)).slice(0, limit);
7448
+ }
7449
+ function normalizeDqlGenerationStringItem(value) {
7450
+ if (typeof value === 'string')
7451
+ return value.trim();
7452
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7453
+ return '';
7454
+ const record = value;
7455
+ for (const key of ['name', 'field', 'column', 'id', 'value', 'label', 'sourceSystem', 'source_system']) {
7456
+ const nested = record[key];
7457
+ if (typeof nested === 'string' && nested.trim())
7458
+ return nested.trim();
7459
+ }
7460
+ return '';
7461
+ }
7462
+ function normalizeDqlGenerationPattern(value) {
7463
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
7464
+ const allowed = new Set([
7465
+ 'metric_wrapper',
7466
+ 'entity_profile',
7467
+ 'entity_rollup',
7468
+ 'ranking',
7469
+ 'trend',
7470
+ 'bridge',
7471
+ 'drilldown',
7472
+ 'replacement',
7473
+ 'custom',
7474
+ ]);
7475
+ return allowed.has(normalized) ? normalized : undefined;
7476
+ }
7477
+ function normalizeDqlGenerationReviewCadence(value) {
7478
+ const normalized = value.trim().toLowerCase().replace(/[^a-z0-9_ -]+/g, '').replace(/\s+/g, '_');
7479
+ const allowed = new Set(['daily', 'weekly', 'biweekly', 'monthly', 'quarterly', 'semiannual', 'annual']);
7480
+ return allowed.has(normalized) ? normalized : undefined;
7481
+ }
7482
+ function mergeGroundedDqlGenerationTerms(baseTerms, overrideTerms, evidence) {
7483
+ const grounded = new Map();
7484
+ for (const item of evidence) {
7485
+ if (item.kind !== 'dql_term' && item.kind !== 'datalex_term')
7486
+ continue;
7487
+ const normalized = normalizedTerm(item.name);
7488
+ if (normalized)
7489
+ grounded.set(normalized, item.name.trim());
7490
+ }
7491
+ const merged = [...(baseTerms ?? []), ...(overrideTerms ?? [])]
7492
+ .map((term) => grounded.get(normalizedTerm(term)))
7493
+ .filter((term) => Boolean(term));
7494
+ const result = Array.from(new Set(merged)).slice(0, 16);
7495
+ return result.length > 0 ? result : undefined;
7496
+ }
7497
+ function mergeDqlGenerationPatch(base, override, candidate, evidence) {
7498
+ const name = override.name && !isWeakDqlGenerationName(override.name) ? override.name : base.name;
7499
+ const description = override.description
7500
+ && !isWeakDqlGenerationDescription(override.description)
7501
+ && !isUngroundedDqlGenerationDescription(override.description, candidate, evidence)
7502
+ ? override.description
7503
+ : base.description;
7504
+ const llmContext = override.llmContext
7505
+ && !isUngroundedDqlGenerationDescription(override.llmContext, candidate, evidence)
7506
+ ? override.llmContext
7507
+ : base.llmContext;
7508
+ const overrideTags = override.tags
7509
+ ? override.tags.filter((tag) => !/^(raw-sql|imported|ai-generated)$/i.test(tag))
7510
+ : [];
7511
+ return {
7512
+ ...base,
7513
+ ...override,
7514
+ name,
7515
+ description,
7516
+ llmContext,
7517
+ tags: Array.from(new Set([...(base.tags ?? []), ...overrideTags])).slice(0, 12),
7518
+ terms: mergeGroundedDqlGenerationTerms(base.terms, override.terms, evidence),
7519
+ pattern: override.pattern ?? base.pattern,
7520
+ grain: override.grain ?? base.grain,
7521
+ entities: override.entities?.length ? override.entities : base.entities,
7522
+ outputs: override.outputs?.length ? override.outputs : base.outputs,
7523
+ dimensions: override.dimensions?.length ? override.dimensions : base.dimensions,
7524
+ allowedFilters: override.allowedFilters?.length ? override.allowedFilters : base.allowedFilters,
7525
+ parameterPolicy: base.parameterPolicy,
7526
+ filterBindings: base.filterBindings,
7527
+ parameterDecisions: base.parameterDecisions,
7528
+ similarityMatches: base.similarityMatches,
7529
+ recommendedAction: base.recommendedAction,
7530
+ sql: base.sql,
7531
+ warnings: base.warnings,
7532
+ sourceSystems: override.sourceSystems?.length ? override.sourceSystems : base.sourceSystems,
7533
+ replacementFor: override.replacementFor?.length ? override.replacementFor : base.replacementFor,
7534
+ reviewCadence: override.reviewCadence ?? base.reviewCadence,
5024
7535
  };
5025
7536
  }
5026
- function extractSqlTablesLight(sql) {
5027
- const tables = new Set();
5028
- const cleaned = sql.replace(/\/\*[\s\S]*?\*\//g, ' ').replace(/--[^\n\r]*/g, ' ');
5029
- const regex = /\b(?:from|join|update|into)\s+([`"[]?[A-Za-z0-9_./:-]+(?:\.[A-Za-z0-9_./:-]+)*[`"\]]?)/gi;
5030
- let match;
5031
- while ((match = regex.exec(cleaned))) {
5032
- const raw = match[1].replace(/^[`"[]|[`"\]]$/g, '');
5033
- if (raw && !raw.startsWith('(') && !/^(select|values|unnest|lateral)$/i.test(raw))
5034
- tables.add(raw);
5035
- }
5036
- return Array.from(tables);
7537
+ function buildDqlGenerationSimilarityMatches(candidate, patch, contextPack) {
7538
+ const candidateSql = patch.sql ?? candidate.sql;
7539
+ const candidateFingerprints = buildBlockSqlFingerprints(candidateSql);
7540
+ const candidateBusinessFingerprint = buildBlockBusinessFingerprint({
7541
+ name: patch.name ?? candidate.name,
7542
+ domain: patch.domain ?? candidate.domain,
7543
+ pattern: patch.pattern,
7544
+ grain: patch.grain,
7545
+ entities: patch.entities,
7546
+ terms: patch.terms,
7547
+ outputs: patch.outputs,
7548
+ dimensions: patch.dimensions,
7549
+ filters: patch.allowedFilters,
7550
+ sources: candidate.lineage.sourceTables,
7551
+ sourceSystems: patch.sourceSystems,
7552
+ });
7553
+ const candidateShape = {
7554
+ name: candidate.name,
7555
+ pattern: patch.pattern,
7556
+ grain: patch.grain,
7557
+ outputs: new Set((patch.outputs ?? []).map(normalizedTerm)),
7558
+ terms: new Set((patch.terms ?? []).map(normalizedTerm)),
7559
+ dimensions: new Set((patch.dimensions ?? []).map(normalizedTerm)),
7560
+ filters: new Set((patch.allowedFilters ?? []).map(normalizedTerm)),
7561
+ entities: new Set((patch.entities ?? []).map(normalizedTerm)),
7562
+ sources: new Set(candidate.lineage.sourceTables.flatMap(relationLookupTokens).map(normalizedTerm)),
7563
+ };
7564
+ const objects = (contextPack?.objects ?? []).filter((object) => object.objectType === 'dql_block');
7565
+ const matches = objects.flatMap((object) => {
7566
+ const payload = object.payload ?? {};
7567
+ const objectSql = typeof payload.sql === 'string' ? payload.sql : '';
7568
+ const objectFingerprints = metadataSqlFingerprints(payload.sqlFingerprints)
7569
+ ?? (objectSql ? buildBlockSqlFingerprints(objectSql) : null);
7570
+ const objectBusinessFingerprint = metadataBusinessFingerprint(payload.businessFingerprint);
7571
+ const objectShape = {
7572
+ pattern: metadataString(payload.pattern),
7573
+ grain: metadataString(payload.grain),
7574
+ outputs: new Set(metadataStringArray(payload.declaredOutputs ?? payload.outputs)),
7575
+ terms: new Set(metadataStringArray(payload.termRefs ?? payload.terms)),
7576
+ dimensions: new Set(metadataStringArray(payload.dimensions)),
7577
+ filters: new Set(metadataStringArray(payload.allowedFilters)),
7578
+ entities: new Set(metadataStringArray(payload.entities)),
7579
+ sources: new Set(metadataStringArray(payload.tableDependencies ?? payload.rawTableRefs).flatMap(relationLookupTokens).map(normalizedTerm)),
7580
+ };
7581
+ if (objectFingerprints?.exact && objectFingerprints.exact === candidateFingerprints.exact) {
7582
+ return [{
7583
+ kind: 'exact_sql_match',
7584
+ objectKey: object.objectKey,
7585
+ name: object.fullName ?? object.name,
7586
+ status: object.status,
7587
+ source: object.sourcePath,
7588
+ score: 0.99,
7589
+ reason: 'Normalized SQL exactly matches an existing DQL block.',
7590
+ recommendedAction: 'reuse_existing',
7591
+ }];
7592
+ }
7593
+ if (objectFingerprints?.parameterized && objectFingerprints.parameterized === candidateFingerprints.parameterized) {
7594
+ return [{
7595
+ kind: 'parameterized_duplicate',
7596
+ objectKey: object.objectKey,
7597
+ name: object.fullName ?? object.name,
7598
+ status: object.status,
7599
+ source: object.sourcePath,
7600
+ score: 0.94,
7601
+ reason: 'SQL shape matches after replacing literal values with parameters.',
7602
+ recommendedAction: 'reuse_existing',
7603
+ }];
7604
+ }
7605
+ const score = businessShapeSimilarity(candidateShape, objectShape, object);
7606
+ if (objectBusinessFingerprint?.hash
7607
+ && objectBusinessFingerprint.hash === candidateBusinessFingerprint.hash
7608
+ && object.status === 'certified') {
7609
+ return [{
7610
+ kind: 'business_duplicate',
7611
+ objectKey: object.objectKey,
7612
+ name: object.fullName ?? object.name,
7613
+ status: object.status,
7614
+ source: object.sourcePath,
7615
+ score: Math.max(score, 0.86),
7616
+ reason: 'Certified block has the same persisted business-shape fingerprint.',
7617
+ recommendedAction: 'reuse_existing',
7618
+ }];
7619
+ }
7620
+ if (score >= 0.76 && object.status === 'certified') {
7621
+ return [{
7622
+ kind: 'business_duplicate',
7623
+ objectKey: object.objectKey,
7624
+ name: object.fullName ?? object.name,
7625
+ status: object.status,
7626
+ source: object.sourcePath,
7627
+ score,
7628
+ reason: 'Certified block has the same business shape: entity, grain, filters, outputs, or source family overlap.',
7629
+ recommendedAction: 'reuse_existing',
7630
+ }];
7631
+ }
7632
+ if (score >= 0.58) {
7633
+ return [{
7634
+ kind: 'near_variant',
7635
+ objectKey: object.objectKey,
7636
+ name: object.fullName ?? object.name,
7637
+ status: object.status,
7638
+ source: object.sourcePath,
7639
+ score,
7640
+ reason: 'Existing block is close but appears to need a reviewed filter, output, grain, or source extension.',
7641
+ recommendedAction: 'extend_existing',
7642
+ }];
7643
+ }
7644
+ return [];
7645
+ }).sort((a, b) => b.score - a.score).slice(0, 8);
7646
+ const top = matches[0];
7647
+ return {
7648
+ matches,
7649
+ recommendedAction: top?.recommendedAction ?? 'create_new',
7650
+ };
5037
7651
  }
5038
- function setBlockStudioStatus(projectRoot, blockPath, newStatus) {
5039
- const normalizedPath = normalize(blockPath).replace(/^\/+/, '');
5040
- if (!normalizedPath.startsWith('blocks/'))
5041
- throw new Error('Invalid block path');
5042
- const absPath = join(projectRoot, normalizedPath);
5043
- if (!existsSync(absPath))
5044
- throw new Error('Block file not found');
5045
- let source = readFileSync(absPath, 'utf-8');
5046
- if (/status\s*=\s*"[^"]*"/.test(source)) {
5047
- source = source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
7652
+ function metadataSqlFingerprints(value) {
7653
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7654
+ return null;
7655
+ const record = value;
7656
+ const exact = typeof record.exact === 'string' ? record.exact : '';
7657
+ const parameterized = typeof record.parameterized === 'string' ? record.parameterized : '';
7658
+ if (!exact || !parameterized)
7659
+ return null;
7660
+ return { exact, parameterized };
7661
+ }
7662
+ function metadataBusinessFingerprint(value) {
7663
+ if (!value || typeof value !== 'object' || Array.isArray(value))
7664
+ return null;
7665
+ const record = value;
7666
+ const hash = typeof record.hash === 'string' ? record.hash : '';
7667
+ if (!hash)
7668
+ return null;
7669
+ const tokens = Array.isArray(record.tokens)
7670
+ ? record.tokens.map((item) => String(item)).filter(Boolean)
7671
+ : [];
7672
+ return { hash, tokens };
7673
+ }
7674
+ function businessShapeSimilarity(candidate, object, metadata) {
7675
+ let score = 0;
7676
+ if (candidate.pattern && object.pattern && normalizedTerm(candidate.pattern) === normalizedTerm(object.pattern))
7677
+ score += 0.12;
7678
+ if (candidate.grain && object.grain && normalizedTerm(candidate.grain) === normalizedTerm(object.grain))
7679
+ score += 0.15;
7680
+ score += overlapScore(candidate.outputs, object.outputs) * 0.18;
7681
+ score += overlapScore(candidate.terms, object.terms) * 0.08;
7682
+ score += overlapScore(candidate.dimensions, object.dimensions) * 0.13;
7683
+ score += overlapScore(candidate.filters, object.filters) * 0.14;
7684
+ score += overlapScore(candidate.entities, object.entities) * 0.12;
7685
+ score += overlapScore(candidate.sources, object.sources) * 0.1;
7686
+ const nameTokens = new Set(candidate.name.toLowerCase().split(/[^a-z0-9]+/).filter((value) => value.length > 2));
7687
+ const targetTokens = new Set([
7688
+ ...metadata.name.toLowerCase().split(/[^a-z0-9]+/),
7689
+ ...(metadata.description ?? '').toLowerCase().split(/[^a-z0-9]+/),
7690
+ ].filter((value) => value.length > 2));
7691
+ score += overlapScore(nameTokens, targetTokens) * 0.04;
7692
+ return Number(Math.min(0.98, score).toFixed(3));
7693
+ }
7694
+ function overlapScore(left, right) {
7695
+ if (left.size === 0 || right.size === 0)
7696
+ return 0;
7697
+ let overlap = 0;
7698
+ for (const value of left) {
7699
+ if (value && right.has(value))
7700
+ overlap += 1;
7701
+ }
7702
+ return overlap / Math.max(1, Math.min(left.size, right.size));
7703
+ }
7704
+ function normalizedTerm(value) {
7705
+ return String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
7706
+ }
7707
+ function metadataString(value) {
7708
+ return typeof value === 'string' && value.trim() ? value.trim() : undefined;
7709
+ }
7710
+ function metadataStringArray(value) {
7711
+ if (!Array.isArray(value))
7712
+ return [];
7713
+ return value.map((item) => String(item).trim()).filter(Boolean).map(normalizedTerm);
7714
+ }
7715
+ function isWeakDqlGenerationName(name) {
7716
+ return /^(pasted|query|source|imported query|sql import)(?:\s+\d+)?$/i.test(name.trim());
7717
+ }
7718
+ function isWeakDqlGenerationDescription(description) {
7719
+ return /^draft dql block summarizing imported sql/i.test(description.trim())
7720
+ || /^draft dql block generated from imported sql/i.test(description.trim())
7721
+ || /^imported from /i.test(description.trim())
7722
+ || /^this query (shows|returns|provides)/i.test(description.trim());
7723
+ }
7724
+ function isUngroundedDqlGenerationDescription(description, candidate, evidence) {
7725
+ const text = description.toLowerCase();
7726
+ const tokens = new Set();
7727
+ for (const tag of candidate.tags) {
7728
+ if (!/^(raw-sql|imported|ai-generated|review-required)$/i.test(tag) && tag.length > 2)
7729
+ tokens.add(tag.toLowerCase());
5048
7730
  }
5049
- else {
5050
- source = source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
7731
+ for (const table of candidate.lineage.sourceTables) {
7732
+ for (const token of table.split(/[._\s/-]+/)) {
7733
+ if (token.length > 2)
7734
+ tokens.add(token.toLowerCase());
7735
+ }
5051
7736
  }
5052
- writeFileSync(absPath, source, 'utf-8');
5053
- const companionPath = blockCompanionRelativePath(normalizedPath);
5054
- if (!companionPath)
5055
- return;
5056
- const absCompanionPath = join(projectRoot, companionPath);
5057
- if (!existsSync(absCompanionPath))
5058
- return;
5059
- let companion = readFileSync(absCompanionPath, 'utf-8');
5060
- if (/^reviewStatus:\s*.+$/m.test(companion)) {
5061
- companion = companion.replace(/^reviewStatus:\s*.+$/m, `reviewStatus: ${newStatus}`);
7737
+ for (const item of evidence.slice(0, 6)) {
7738
+ for (const token of item.name.split(/[._\s/-]+/)) {
7739
+ if (token.length > 3 && !/^(model|source|semantic|table)$/i.test(token))
7740
+ tokens.add(token.toLowerCase());
7741
+ }
5062
7742
  }
5063
- else {
5064
- companion = `${companion.trimEnd()}\nreviewStatus: ${newStatus}\n`;
7743
+ if (tokens.size === 0)
7744
+ return false;
7745
+ return !Array.from(tokens).some((token) => text.includes(token));
7746
+ }
7747
+ function dqlGenerationConversionNotes(providerName) {
7748
+ return [
7749
+ `Generated with ${providerName} using local project context when available.`,
7750
+ 'Runtime-scope SQL literals are converted into DQL params when safe; business constants stay static for review.',
7751
+ 'Certified and draft block similarity is checked before a new draft is saved.',
7752
+ 'SQL semantics are preserved while reusable parameter placeholders may replace literal values.',
7753
+ 'Drafts are autosaved under blocks/_drafts or domains/<domain>/blocks/_drafts before certification.',
7754
+ 'Preview results and tests gate certification before promotion.',
7755
+ ];
7756
+ }
7757
+ function setBlockStudioSourceStatus(source, status) {
7758
+ if (/status\s*=\s*"[^"]*"/.test(source)) {
7759
+ return source.replace(/status\s*=\s*"[^"]*"/, `status = "${status}"`);
5065
7760
  }
5066
- writeFileSync(absCompanionPath, companion, 'utf-8');
7761
+ return source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${status}"`);
5067
7762
  }
5068
7763
  async function buildBlockStudioAiAssistSummary(projectRoot, action, candidate, validation, requestedProvider) {
5069
7764
  const fallback = buildDeterministicAiAssistSummary(action, candidate, validation);
@@ -5271,6 +7966,14 @@ export function readGitMetadata(projectRoot) {
5271
7966
  return null;
5272
7967
  }
5273
7968
  }
7969
+ function resolveBlockWriteTarget(projectRoot, safeDomain, slug) {
7970
+ if (safeDomain && existsSync(join(projectRoot, 'domains', safeDomain))) {
7971
+ const relativePath = `domains/${safeDomain}/blocks/${slug}.dql`;
7972
+ return { relativePath, absPath: join(projectRoot, relativePath) };
7973
+ }
7974
+ const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
7975
+ return { relativePath, absPath: join(projectRoot, relativePath) };
7976
+ }
5274
7977
  export function createBlockArtifacts(projectRoot, options) {
5275
7978
  const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
5276
7979
  const safeDomain = (options.domain ?? '')
@@ -5278,16 +7981,16 @@ export function createBlockArtifacts(projectRoot, options) {
5278
7981
  .toLowerCase()
5279
7982
  .replace(/[^a-z0-9/_-]+/g, '-')
5280
7983
  .replace(/^\/+|\/+$/g, '');
5281
- const blocksDir = safeDomain ? join(projectRoot, 'blocks', safeDomain) : join(projectRoot, 'blocks');
5282
- mkdirSync(blocksDir, { recursive: true });
5283
- const blockPath = join(blocksDir, `${slug}.dql`);
7984
+ const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
7985
+ mkdirSync(dirname(target.absPath), { recursive: true });
7986
+ const blockPath = target.absPath;
5284
7987
  if (existsSync(blockPath)) {
5285
7988
  throw new Error('BLOCK_EXISTS');
5286
7989
  }
5287
7990
  const templateContent = options.template
5288
7991
  ? listBlockTemplates().find((template) => template.id === options.template)?.content
5289
7992
  : undefined;
5290
- const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
7993
+ const relativePath = target.relativePath;
5291
7994
  const fileContent = canonicalizeSafe(options.blockType === 'semantic' && !options.content?.trim() && !templateContent
5292
7995
  ? buildBlankSemanticBlockContent({
5293
7996
  name: options.name,
@@ -5333,9 +8036,9 @@ export function createSemanticBuilderBlock(projectRoot, options) {
5333
8036
  .toLowerCase()
5334
8037
  .replace(/[^a-z0-9/_-]+/g, '-')
5335
8038
  .replace(/^\/+|\/+$/g, '') || 'uncategorized';
5336
- const blocksDir = join(projectRoot, 'blocks', safeDomain);
5337
- mkdirSync(blocksDir, { recursive: true });
5338
- const blockPath = join(blocksDir, `${slug}.dql`);
8039
+ const target = resolveBlockWriteTarget(projectRoot, safeDomain, slug);
8040
+ mkdirSync(dirname(target.absPath), { recursive: true });
8041
+ const blockPath = target.absPath;
5339
8042
  if (existsSync(blockPath)) {
5340
8043
  throw new Error('BLOCK_EXISTS');
5341
8044
  }
@@ -5360,7 +8063,7 @@ export function createSemanticBuilderBlock(projectRoot, options) {
5360
8063
  ],
5361
8064
  });
5362
8065
  return {
5363
- path: `blocks/${safeDomain}/${slug}.dql`,
8066
+ path: target.relativePath,
5364
8067
  content,
5365
8068
  companionPath,
5366
8069
  };
@@ -6122,18 +8825,16 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
6122
8825
  const sourceCount = manifest?.sources && typeof manifest.sources === 'object'
6123
8826
  ? Object.keys(manifest.sources).length
6124
8827
  : 0;
6125
- const manifestMetricCount = manifest?.metrics && typeof manifest.metrics === 'object'
6126
- ? Object.keys(manifest.metrics).length
6127
- : 0;
6128
- const semanticMetricCount = Array.isArray(semanticManifest?.metrics)
6129
- ? semanticManifest.metrics.length
8828
+ const manifestMetricCount = artifactCollectionCount(manifest?.metrics);
8829
+ const semanticMetricCount = semanticManifest
8830
+ ? artifactCollectionCount(semanticManifest.metrics)
6130
8831
  : manifestMetricCount;
6131
- const semanticModelCount = Array.isArray(semanticManifest?.semantic_models)
6132
- ? semanticManifest.semantic_models.length
6133
- : 0;
6134
- const savedQueryCount = Array.isArray(semanticManifest?.saved_queries)
6135
- ? semanticManifest.saved_queries.length
6136
- : 0;
8832
+ const semanticModelCount = semanticManifest
8833
+ ? artifactCollectionCount(semanticManifest.semantic_models)
8834
+ : artifactCollectionCount(manifest?.semantic_models);
8835
+ const savedQueryCount = semanticManifest
8836
+ ? artifactCollectionCount(semanticManifest.saved_queries ?? semanticManifest.savedQueries)
8837
+ : artifactCollectionCount(manifest?.saved_queries ?? manifest?.savedQueries);
6137
8838
  const configured = existsSync(join(dbtProjectPath, 'dbt_project.yml'))
6138
8839
  || Boolean(projectConfig.dbt?.projectDir)
6139
8840
  || Boolean(projectConfig.semanticLayer?.provider === 'dbt' && projectConfig.semanticLayer.projectPath);
@@ -6145,7 +8846,9 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
6145
8846
  ? 'Run `dbt parse`, `dbt compile`, or `dbt build`, then run `dql sync dbt`.'
6146
8847
  : !semanticExists
6147
8848
  ? 'dbt manifest is ready. Run `dbt parse` or `dbt build` if you use dbt Semantic Layer metrics.'
6148
- : 'dbt artifacts are ready. Build SQL blocks from models or semantic blocks from metrics.';
8849
+ : semanticMetricCount === 0 && semanticModelCount === 0 && savedQueryCount === 0
8850
+ ? 'dbt manifest is ready, but MetricFlow semantic_manifest.json is empty. Add dbt semantic_models and metrics, then rerun dbt parse/build.'
8851
+ : 'dbt artifacts are ready. Build SQL blocks from models or semantic blocks from metrics.';
6149
8852
  return {
6150
8853
  configured,
6151
8854
  provider: projectConfig.semanticLayer?.provider ?? null,
@@ -6168,6 +8871,185 @@ export function buildDbtStatus(projectRoot, projectConfig, lastSyncTime) {
6168
8871
  setupHint,
6169
8872
  };
6170
8873
  }
8874
+ export function buildSemanticLayerDiagnostics(projectRoot, projectConfig, options) {
8875
+ const provider = options.semanticConfig?.provider ?? options.detectedProvider ?? null;
8876
+ const dbt = buildDbtStatus(projectRoot, projectConfig, options.lastSyncTime);
8877
+ const counts = semanticLayerCounts(options.semanticLayer);
8878
+ const uncategorizedObjects = countUncategorizedSemanticObjects(options.semanticLayer);
8879
+ const totalObjects = counts.metrics
8880
+ + counts.measures
8881
+ + counts.dimensions
8882
+ + counts.timeDimensions
8883
+ + counts.entities
8884
+ + counts.hierarchies
8885
+ + counts.semanticModels
8886
+ + counts.savedQueries;
8887
+ const issues = [];
8888
+ for (const error of options.semanticErrors ?? []) {
8889
+ issues.push({
8890
+ severity: 'error',
8891
+ code: 'semantic_load_error',
8892
+ message: error,
8893
+ action: 'Open the semantic source configuration, fix the provider error, then reload the semantic layer.',
8894
+ });
8895
+ }
8896
+ if (!provider) {
8897
+ issues.push({
8898
+ severity: existsSync(join(projectRoot, 'semantic-layer')) ? 'info' : 'warning',
8899
+ code: 'semantic_provider_not_configured',
8900
+ message: 'No semantic layer provider is configured.',
8901
+ action: 'Configure dbt MetricFlow, local semantic-layer YAML, or Snowflake Semantic Views from the setup flow.',
8902
+ });
8903
+ }
8904
+ if (provider === 'dbt' || dbt.configured) {
8905
+ if (!dbt.configured) {
8906
+ issues.push({
8907
+ severity: 'warning',
8908
+ code: 'dbt_project_not_detected',
8909
+ message: 'dbt is selected as the semantic source, but no dbt project was detected.',
8910
+ action: 'Set dbt.projectDir or run DQL from a repository containing dbt_project.yml.',
8911
+ });
8912
+ }
8913
+ if (!dbt.artifacts.manifest.exists) {
8914
+ issues.push({
8915
+ severity: 'error',
8916
+ code: 'dbt_manifest_missing',
8917
+ message: 'dbt manifest.json is missing, so DQL cannot build dbt model context.',
8918
+ action: 'Run dbt parse, dbt compile, or dbt build, then reload the semantic layer.',
8919
+ path: dbt.artifacts.manifest.path,
8920
+ });
8921
+ }
8922
+ if (!dbt.artifacts.semanticManifest.exists) {
8923
+ issues.push({
8924
+ severity: 'warning',
8925
+ code: 'metricflow_semantic_manifest_missing',
8926
+ message: 'dbt MetricFlow semantic_manifest.json is missing. DQL can still inspect dbt models, but MetricFlow metrics and saved queries may be incomplete.',
8927
+ action: 'Run dbt parse or dbt build with semantic models configured, then reload. For execution, ensure MetricFlow is installed and mf is on PATH.',
8928
+ path: dbt.artifacts.semanticManifest.path,
8929
+ });
8930
+ }
8931
+ else if (dbt.counts.metrics === 0 && dbt.counts.semanticModels === 0 && dbt.counts.savedQueries === 0) {
8932
+ issues.push({
8933
+ severity: 'warning',
8934
+ code: 'metricflow_semantic_manifest_empty',
8935
+ message: 'semantic_manifest.json exists, but it does not contain metrics, semantic models, or saved queries.',
8936
+ action: 'Check dbt semantic model YAML definitions and rerun dbt parse.',
8937
+ path: dbt.artifacts.semanticManifest.path,
8938
+ });
8939
+ }
8940
+ if (options.semanticLayer && totalObjects === 0 && (dbt.counts.models > 0 || dbt.counts.sources > 0)) {
8941
+ issues.push({
8942
+ severity: 'warning',
8943
+ code: 'dbt_model_inventory_only',
8944
+ message: 'DQL loaded dbt model inventory, but no MetricFlow semantic metrics were found.',
8945
+ action: 'Create dbt semantic_models and metrics, or use AI Import to build DQL blocks from dbt model metadata.',
8946
+ });
8947
+ }
8948
+ }
8949
+ if (provider === 'snowflake') {
8950
+ if (!options.semanticLayer && (options.semanticErrors ?? []).length === 0) {
8951
+ issues.push({
8952
+ severity: 'warning',
8953
+ code: 'snowflake_semantic_views_not_loaded',
8954
+ message: 'Snowflake semantic provider did not load semantic views.',
8955
+ action: 'Confirm the active Snowflake connection can query semantic views, then reload. Warehouse tables still feed database context separately.',
8956
+ });
8957
+ }
8958
+ else if (options.semanticLayer && totalObjects === 0) {
8959
+ issues.push({
8960
+ severity: 'info',
8961
+ code: 'snowflake_semantic_views_empty',
8962
+ message: 'Snowflake semantic provider loaded, but no semantic view objects were discovered.',
8963
+ action: 'Check that Snowflake Semantic Views exist and the active role can see them.',
8964
+ });
8965
+ }
8966
+ }
8967
+ if (options.semanticLayer && totalObjects === 0 && issues.every((issue) => issue.code !== 'dbt_model_inventory_only')) {
8968
+ issues.push({
8969
+ severity: 'warning',
8970
+ code: 'semantic_layer_empty',
8971
+ message: 'Semantic layer is available, but no semantic objects are loaded.',
8972
+ action: 'Import semantic definitions or run dbt parse/build, then reload.',
8973
+ });
8974
+ }
8975
+ if (uncategorizedObjects > 0) {
8976
+ issues.push({
8977
+ severity: 'info',
8978
+ code: 'semantic_uncategorized_domain',
8979
+ message: `${uncategorizedObjects} semantic object${uncategorizedObjects === 1 ? '' : 's'} had no domain and are visible under "uncategorized".`,
8980
+ action: 'Add domain metadata in dbt semantic YAML or local semantic-layer files when domain ownership matters.',
8981
+ });
8982
+ }
8983
+ const warnings = Array.from(new Set([
8984
+ ...(options.semanticErrors ?? []),
8985
+ ...issues
8986
+ .filter((issue) => issue.severity !== 'info')
8987
+ .map((issue) => issue.message),
8988
+ ]));
8989
+ return {
8990
+ available: Boolean(options.semanticLayer),
8991
+ provider,
8992
+ sourceOfTruth: provider === 'dbt'
8993
+ ? 'dbt MetricFlow semantic_manifest.json, dbt manifest.json semantic nodes, then dbt semantic YAML fallback'
8994
+ : provider === 'snowflake'
8995
+ ? 'Snowflake Semantic Views only; warehouse tables remain database catalog context'
8996
+ : provider === 'dql'
8997
+ ? 'local semantic-layer YAML'
8998
+ : 'not configured',
8999
+ errors: options.semanticErrors ?? [],
9000
+ lastSyncTime: options.lastSyncTime,
9001
+ counts,
9002
+ dbt,
9003
+ issues,
9004
+ warnings,
9005
+ };
9006
+ }
9007
+ function semanticLayerCounts(layer) {
9008
+ return layer
9009
+ ? {
9010
+ domains: layer.listDomains().length,
9011
+ metrics: layer.listMetrics().length,
9012
+ measures: layer.listMeasures().length,
9013
+ dimensions: layer.listDimensions().length,
9014
+ timeDimensions: layer.listTimeDimensions().length,
9015
+ entities: layer.listEntities().length,
9016
+ hierarchies: layer.listHierarchies().length,
9017
+ semanticModels: layer.listSemanticModels().length,
9018
+ savedQueries: layer.listSavedQueries().length,
9019
+ }
9020
+ : {
9021
+ domains: 0,
9022
+ metrics: 0,
9023
+ measures: 0,
9024
+ dimensions: 0,
9025
+ timeDimensions: 0,
9026
+ entities: 0,
9027
+ hierarchies: 0,
9028
+ semanticModels: 0,
9029
+ savedQueries: 0,
9030
+ };
9031
+ }
9032
+ function countUncategorizedSemanticObjects(layer) {
9033
+ if (!layer)
9034
+ return 0;
9035
+ return [
9036
+ ...layer.listMetrics(),
9037
+ ...layer.listMeasures(),
9038
+ ...layer.listDimensions(),
9039
+ ...layer.listTimeDimensions(),
9040
+ ...layer.listEntities(),
9041
+ ...layer.listHierarchies(),
9042
+ ...layer.listSemanticModels(),
9043
+ ...layer.listSavedQueries(),
9044
+ ].filter((object) => !object.domain || !object.domain.trim()).length;
9045
+ }
9046
+ function artifactCollectionCount(value) {
9047
+ if (Array.isArray(value))
9048
+ return value.length;
9049
+ if (value && typeof value === 'object')
9050
+ return Object.keys(value).length;
9051
+ return 0;
9052
+ }
6171
9053
  function describeArtifact(path, count, generatedAt) {
6172
9054
  return {
6173
9055
  path,
@@ -6704,6 +9586,11 @@ function isProviderSettingsId(value) {
6704
9586
  || value === 'ollama'
6705
9587
  || value === 'custom-openai';
6706
9588
  }
9589
+ function isDeterministicDqlGenerationProvider(value) {
9590
+ if (typeof value !== 'string')
9591
+ return false;
9592
+ return value === 'none' || value === 'deterministic' || value === 'local-deterministic';
9593
+ }
6707
9594
  async function testProviderConfig(projectRoot, id) {
6708
9595
  const config = getEffectiveProviderConfig(projectRoot, id);
6709
9596
  const label = providerSettingsLabel(id);
@@ -6853,6 +9740,668 @@ function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
6853
9740
  // Runtime schema snapshots are advisory local metadata and must not block answers.
6854
9741
  }
6855
9742
  }
9743
+ function notebookResearchString(value) {
9744
+ return typeof value === 'string' && value.trim() ? value.trim() : undefined;
9745
+ }
9746
+ function notebookResearchRecord(value) {
9747
+ return value && typeof value === 'object' && !Array.isArray(value)
9748
+ ? value
9749
+ : null;
9750
+ }
9751
+ function notebookResearchSourceCellPayload(body) {
9752
+ const sourceCell = notebookResearchRecord(body.sourceCell)
9753
+ ?? notebookResearchRecord(notebookResearchRecord(body.context)?.sourceCell);
9754
+ if (!sourceCell)
9755
+ return undefined;
9756
+ const payload = {
9757
+ id: notebookResearchString(sourceCell.id),
9758
+ sourceCellId: notebookResearchString(sourceCell.sourceCellId),
9759
+ cellId: notebookResearchString(sourceCell.cellId),
9760
+ name: notebookResearchString(sourceCell.name),
9761
+ sourceCellName: notebookResearchString(sourceCell.sourceCellName),
9762
+ title: notebookResearchString(sourceCell.title),
9763
+ fingerprint: notebookResearchString(sourceCell.fingerprint),
9764
+ sourceCellFingerprint: notebookResearchString(sourceCell.sourceCellFingerprint),
9765
+ sqlFingerprint: notebookResearchString(sourceCell.sqlFingerprint),
9766
+ type: notebookResearchString(sourceCell.type),
9767
+ sql: notebookResearchString(sourceCell.sql),
9768
+ content: notebookResearchString(sourceCell.content),
9769
+ source: notebookResearchString(sourceCell.source),
9770
+ };
9771
+ return Object.values(payload).some(Boolean) ? payload : undefined;
9772
+ }
9773
+ function notebookResearchSourceCellId(sourceCell) {
9774
+ return notebookResearchString(sourceCell?.id)
9775
+ ?? notebookResearchString(sourceCell?.sourceCellId)
9776
+ ?? notebookResearchString(sourceCell?.cellId);
9777
+ }
9778
+ function notebookResearchSourceCellName(sourceCell) {
9779
+ return notebookResearchString(sourceCell?.name)
9780
+ ?? notebookResearchString(sourceCell?.sourceCellName)
9781
+ ?? notebookResearchString(sourceCell?.title);
9782
+ }
9783
+ function notebookResearchSourceCellFingerprint(sourceCell) {
9784
+ return notebookResearchString(sourceCell?.sourceCellFingerprint)
9785
+ ?? notebookResearchString(sourceCell?.fingerprint)
9786
+ ?? notebookResearchString(sourceCell?.sqlFingerprint);
9787
+ }
9788
+ function notebookResearchPatchString(body, key) {
9789
+ if (!Object.prototype.hasOwnProperty.call(body, key))
9790
+ return undefined;
9791
+ return notebookResearchString(body[key]) ?? null;
9792
+ }
9793
+ function notebookResearchIntent(value) {
9794
+ return value === 'ad_hoc_analysis'
9795
+ || value === 'diagnose_change'
9796
+ || value === 'driver_breakdown'
9797
+ || value === 'segment_compare'
9798
+ || value === 'entity_drilldown'
9799
+ || value === 'anomaly_investigation'
9800
+ || value === 'trust_gap_review'
9801
+ ? value
9802
+ : undefined;
9803
+ }
9804
+ function notebookResearchReviewStatus(value) {
9805
+ return value === 'needs_review'
9806
+ || value === 'draft_created'
9807
+ || value === 'completed'
9808
+ || value === 'certified'
9809
+ || value === 'rejected'
9810
+ ? value
9811
+ : undefined;
9812
+ }
9813
+ function notebookResearchStatus(value) {
9814
+ return value === 'draft'
9815
+ || value === 'running'
9816
+ || value === 'ready'
9817
+ || value === 'error'
9818
+ ? value
9819
+ : undefined;
9820
+ }
9821
+ function notebookResearchPromotionAction(value) {
9822
+ return value === 'reuse_existing'
9823
+ || value === 'extend_existing'
9824
+ || value === 'create_replacement'
9825
+ || value === 'create_new'
9826
+ || value === 'review_required'
9827
+ ? value
9828
+ : undefined;
9829
+ }
9830
+ function notebookResearchReadiness(value) {
9831
+ return value === 'draft_ready' || value === 'certification_ready' || value === 'blocked' ? value : undefined;
9832
+ }
9833
+ function notebookResearchAge(value) {
9834
+ return value === 'stale_open' || value === 'expired_open' ? value : undefined;
9835
+ }
9836
+ function notebookResearchNextAction(value) {
9837
+ return value === 'fix_blockers'
9838
+ || value === 'review_sql'
9839
+ || value === 'review_context'
9840
+ || value === 'run_preview'
9841
+ || value === 'reuse_existing'
9842
+ || value === 'create_dql_draft'
9843
+ || value === 'open_certification'
9844
+ || value === 'complete_review'
9845
+ || value === 'continue_review'
9846
+ ? value
9847
+ : undefined;
9848
+ }
9849
+ function notebookResearchSort(value) {
9850
+ return value === 'priority' || value === 'updated_desc' ? value : undefined;
9851
+ }
9852
+ function withNotebookResearchChecklistPage(page) {
9853
+ return {
9854
+ ...page,
9855
+ runs: page.runs.map(withNotebookResearchChecklist),
9856
+ };
9857
+ }
9858
+ function withNotebookResearchChecklist(run) {
9859
+ const researchPlan = run.researchPlan ?? buildNotebookResearchPlan({
9860
+ run,
9861
+ evidence: run.evidence,
9862
+ resultPreview: run.resultPreview,
9863
+ previewError: run.status === 'error' ? run.error : undefined,
9864
+ generatedSql: run.generatedSql,
9865
+ reviewedSql: run.reviewedSql,
9866
+ routeDecision: run.routeDecision,
9867
+ });
9868
+ return {
9869
+ ...run,
9870
+ researchPlan,
9871
+ reviewChecklist: buildNotebookResearchReviewChecklist(run),
9872
+ };
9873
+ }
9874
+ function buildNotebookResearchReviewChecklist(run) {
9875
+ const questionReady = Boolean(notebookResearchString(run.question));
9876
+ const hasReviewedSql = Boolean(notebookResearchString(run.reviewedSql));
9877
+ const hasGeneratedSql = Boolean(notebookResearchString(run.generatedSql));
9878
+ const hasSql = hasReviewedSql || hasGeneratedSql;
9879
+ const preview = notebookResearchPreviewInfo(run.resultPreview);
9880
+ const previewReady = run.status === 'ready' && preview.hasPreview;
9881
+ const evidenceReady = notebookResearchEvidenceCount(run.evidence) > 0 || Boolean(run.contextPackId);
9882
+ const promoted = Boolean(run.dqlPromotion || run.dqlImportId || run.draftBlockPath);
9883
+ const duplicateChecked = Boolean(run.dqlPromotionAction || run.dqlPromotion);
9884
+ const promotionAction = run.dqlPromotionAction ?? run.dqlPromotion?.recommendedAction;
9885
+ const reuseRecommended = promotionAction === 'reuse_existing';
9886
+ const promotionReviewRequired = promotionAction === 'review_required';
9887
+ const reuseClosed = reuseRecommended && (run.reviewStatus === 'completed' || run.reviewStatus === 'certified');
9888
+ const hasDraft = Boolean(notebookResearchString(run.draftBlockPath));
9889
+ const sqlForParameterReview = notebookResearchString(run.reviewedSql) ?? notebookResearchString(run.generatedSql);
9890
+ const parameterReview = sqlForParameterReview ? parameterizeSqlForDqlImport(sqlForParameterReview) : null;
9891
+ const dynamicParameters = parameterReview?.parameterPolicy.filter((item) => item.policy === 'dynamic') ?? [];
9892
+ const staticParameters = parameterReview?.parameterPolicy.filter((item) => item.policy !== 'dynamic') ?? [];
9893
+ const blockers = [];
9894
+ const warnings = [];
9895
+ if (!questionReady)
9896
+ blockers.push('Add a business question before review.');
9897
+ if (!hasSql && !reuseRecommended)
9898
+ blockers.push('Add reviewed SQL or generate SQL before promotion.');
9899
+ if (run.status === 'error')
9900
+ blockers.push(run.error ? `Fix preview error: ${run.error}` : 'Fix the failed preview before promotion.');
9901
+ if (!previewReady && run.status !== 'error' && !reuseRecommended)
9902
+ warnings.push('Run a bounded preview before certification review.');
9903
+ if (!evidenceReady && !reuseRecommended)
9904
+ warnings.push('Inspect metadata evidence before turning this into a reusable block.');
9905
+ if (hasSql && dynamicParameters.length === 0)
9906
+ warnings.push('No dynamic parameters were detected; confirm this should be a static block before certification.');
9907
+ for (const warning of parameterReview?.warnings ?? [])
9908
+ warnings.push(warning);
9909
+ if (promoted && !duplicateChecked)
9910
+ warnings.push('Promotion exists but duplicate/reuse evidence is missing.');
9911
+ if (reuseRecommended && !reuseClosed)
9912
+ warnings.push('Reuse the matching block or explicitly document a replacement before certification.');
9913
+ if (promotionReviewRequired)
9914
+ warnings.push('Resolve the review-required promotion decision before certification.');
9915
+ const items = [
9916
+ {
9917
+ id: 'question',
9918
+ label: 'Question',
9919
+ status: questionReady ? 'passed' : 'blocked',
9920
+ detail: questionReady ? 'Business question is captured.' : 'A reusable block needs a clear business question.',
9921
+ },
9922
+ {
9923
+ id: 'sql',
9924
+ label: 'Reviewed SQL',
9925
+ status: hasReviewedSql || (reuseRecommended && !hasSql) ? 'passed' : hasGeneratedSql ? 'warning' : 'blocked',
9926
+ detail: reuseRecommended && !hasSql
9927
+ ? 'Existing certified DQL should be reused; no new SQL is required.'
9928
+ : hasReviewedSql
9929
+ ? 'Reviewer SQL is saved.'
9930
+ : hasGeneratedSql
9931
+ ? 'Generated SQL exists; review it before promotion.'
9932
+ : 'No SQL is available for preview or DQL generation.',
9933
+ },
9934
+ {
9935
+ id: 'preview',
9936
+ label: 'Preview',
9937
+ status: previewReady || (reuseRecommended && !hasSql) ? 'passed' : run.status === 'error' ? 'blocked' : 'pending',
9938
+ detail: reuseRecommended && !hasSql
9939
+ ? 'Certified block reuse does not require a new raw SQL preview.'
9940
+ : previewReady
9941
+ ? `Preview returned ${preview.rowCount.toLocaleString()} row${preview.rowCount === 1 ? '' : 's'}.`
9942
+ : run.status === 'error'
9943
+ ? (run.error ?? 'Preview failed.')
9944
+ : 'Run a bounded preview to validate the query shape.',
9945
+ },
9946
+ {
9947
+ id: 'evidence',
9948
+ label: 'Evidence',
9949
+ status: evidenceReady || reuseRecommended ? 'passed' : 'warning',
9950
+ detail: evidenceReady
9951
+ ? 'Metadata evidence or context pack is attached.'
9952
+ : reuseRecommended
9953
+ ? 'Reuse evidence is captured in the AI recommendation.'
9954
+ : 'No context evidence is attached yet.',
9955
+ },
9956
+ {
9957
+ id: 'duplicates',
9958
+ label: 'Reuse check',
9959
+ status: duplicateChecked ? (reuseRecommended || promotionReviewRequired ? 'warning' : 'passed') : promoted ? 'warning' : 'pending',
9960
+ detail: duplicateChecked
9961
+ ? `Promotion decision: ${promotionAction ?? 'review required'}.`
9962
+ : promoted
9963
+ ? 'Draft was created without stored duplicate evidence.'
9964
+ : 'Duplicate check runs when creating a DQL draft.',
9965
+ },
9966
+ {
9967
+ id: 'parameters',
9968
+ label: 'Parameter review',
9969
+ status: reuseRecommended && !hasSql ? 'passed' : !hasSql ? 'pending' : dynamicParameters.length > 0 ? 'passed' : 'warning',
9970
+ detail: reuseRecommended && !hasSql
9971
+ ? 'Use the certified block parameters; no new SQL parameterization is required.'
9972
+ : !hasSql
9973
+ ? 'Add SQL before checking runtime parameters.'
9974
+ : dynamicParameters.length > 0
9975
+ ? `Detected ${dynamicParameters.length.toLocaleString()} dynamic parameter${dynamicParameters.length === 1 ? '' : 's'}: ${dynamicParameters.slice(0, 6).map((item) => item.name).join(', ')}.`
9976
+ : staticParameters.length > 0
9977
+ ? `Only static or review-required literals were detected: ${staticParameters.slice(0, 6).map((item) => `${item.name} (${item.policy})`).join(', ')}.`
9978
+ : 'No reusable runtime parameters were detected. Certify as static only if the business question is intentionally fixed.',
9979
+ },
9980
+ {
9981
+ id: 'dql_draft',
9982
+ label: 'DQL draft',
9983
+ status: reuseRecommended && !hasDraft ? 'passed' : hasDraft ? 'passed' : hasReviewedSql && evidenceReady ? 'pending' : 'blocked',
9984
+ detail: reuseRecommended && !hasDraft
9985
+ ? 'No new draft is required because an existing DQL block should be reused.'
9986
+ : hasDraft
9987
+ ? `Draft saved at ${run.draftBlockPath}.`
9988
+ : !hasReviewedSql
9989
+ ? 'Save reviewed SQL before DQL draft creation.'
9990
+ : evidenceReady
9991
+ ? 'Create a draft after SQL review.'
9992
+ : 'Save metadata evidence before creating a reusable DQL draft.',
9993
+ },
9994
+ ];
9995
+ const readyForDqlDraft = questionReady && hasReviewedSql && evidenceReady && run.status !== 'error';
9996
+ const readyForCertificationReview = readyForDqlDraft && previewReady && hasDraft && !reuseRecommended && !promotionReviewRequired;
9997
+ return { readyForDqlDraft, readyForCertificationReview, blockers, warnings, items };
9998
+ }
9999
+ function buildNotebookResearchPlan(input) {
10000
+ const reviewedSql = notebookResearchString(input.reviewedSql);
10001
+ const generatedSql = notebookResearchString(input.generatedSql);
10002
+ const sql = reviewedSql ?? generatedSql;
10003
+ const parameterized = sql ? parameterizeSqlForDqlImport(sql) : null;
10004
+ const evidence = input.evidence && typeof input.evidence === 'object' && !Array.isArray(input.evidence)
10005
+ ? input.evidence
10006
+ : {};
10007
+ const trustStatus = evidence.trustStatus && typeof evidence.trustStatus === 'object' && !Array.isArray(evidence.trustStatus)
10008
+ ? evidence.trustStatus
10009
+ : {};
10010
+ const allowedSqlContext = evidence.allowedSqlContext && typeof evidence.allowedSqlContext === 'object' && !Array.isArray(evidence.allowedSqlContext)
10011
+ ? evidence.allowedSqlContext
10012
+ : {};
10013
+ const previewRows = input.resultPreview?.rowCount ?? input.resultPreview?.rows.length;
10014
+ const hasEvidence = notebookResearchEvidenceCount(input.evidence) > 0 || Boolean(input.run.contextPackId);
10015
+ const hasPreview = previewRows !== undefined && (previewRows > 0 || Boolean(input.resultPreview?.columns.length));
10016
+ const promotionAction = input.run.dqlPromotionAction ?? input.run.dqlPromotion?.recommendedAction;
10017
+ const missingContext = Array.isArray(evidence.missingContext) ? evidence.missingContext : [];
10018
+ const relations = Array.isArray(allowedSqlContext.relations) ? allowedSqlContext.relations : [];
10019
+ return {
10020
+ sqlState: reviewedSql ? 'reviewed' : generatedSql ? 'generated' : 'missing',
10021
+ grain: sql ? extractDqlGenerationGroupByFields(sql)[0] : undefined,
10022
+ parameterPolicy: (parameterized?.parameterPolicy ?? []).slice(0, 16).map((entry) => ({
10023
+ name: entry.name,
10024
+ policy: entry.policy,
10025
+ })),
10026
+ allowedFilters: (parameterized?.allowedFilters ?? []).slice(0, 16),
10027
+ evidence: {
10028
+ trustLabel: notebookResearchString(trustStatus.label),
10029
+ contextPackId: notebookResearchString(evidence.contextPackId) ?? input.run.contextPackId,
10030
+ evidenceCount: notebookResearchEvidenceCount(input.evidence),
10031
+ relationCount: relations.length,
10032
+ missingContextCount: missingContext.length,
10033
+ },
10034
+ preview: {
10035
+ status: input.previewError ? 'error' : hasPreview ? 'ready' : 'not_run',
10036
+ ...(previewRows === undefined ? {} : { rowCount: previewRows }),
10037
+ },
10038
+ promotion: {
10039
+ path: notebookResearchPlanPromotionPath({
10040
+ hasSql: Boolean(sql),
10041
+ hasEvidence,
10042
+ hasPreview,
10043
+ previewError: input.previewError,
10044
+ draftBlockPath: input.run.draftBlockPath,
10045
+ promotionAction,
10046
+ }),
10047
+ duplicateDecision: promotionAction,
10048
+ },
10049
+ reviewFocus: notebookResearchPlanFocus(input.run.intent, {
10050
+ hasSql: Boolean(sql),
10051
+ hasEvidence,
10052
+ hasPreview,
10053
+ dynamicParameterCount: parameterized?.parameterPolicy.filter((entry) => entry.policy === 'dynamic').length ?? 0,
10054
+ missingContextCount: missingContext.length,
10055
+ promotionAction,
10056
+ }),
10057
+ generatedAt: new Date().toISOString(),
10058
+ };
10059
+ }
10060
+ function notebookResearchPlanPromotionPath(input) {
10061
+ if (input.promotionAction === 'reuse_existing')
10062
+ return 'reuse_existing';
10063
+ if (!input.hasSql)
10064
+ return 'needs_sql';
10065
+ if (!input.hasEvidence)
10066
+ return 'review_context';
10067
+ if (input.previewError || !input.hasPreview)
10068
+ return 'run_preview';
10069
+ if (input.draftBlockPath)
10070
+ return 'open_certification';
10071
+ return 'create_dql_draft';
10072
+ }
10073
+ function notebookResearchPlanFocus(intent, input) {
10074
+ const focus = new Set();
10075
+ if (!input.hasSql)
10076
+ focus.add('Add reviewed SQL or configure AI SQL generation.');
10077
+ if (!input.hasEvidence)
10078
+ focus.add('Save metadata context evidence before DQL promotion.');
10079
+ if (!input.hasPreview)
10080
+ focus.add('Run a bounded preview and inspect result shape.');
10081
+ if (input.dynamicParameterCount === 0)
10082
+ focus.add('Confirm whether this should be static or parameterized.');
10083
+ if (input.missingContextCount > 0)
10084
+ focus.add('Resolve missing metadata context before certification.');
10085
+ if (input.promotionAction === 'reuse_existing')
10086
+ focus.add('Reuse the existing certified block or document replacement rationale.');
10087
+ const pattern = notebookResearchIntentPattern(intent);
10088
+ const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus : [];
10089
+ for (const item of reviewFocus.slice(0, 4))
10090
+ focus.add(String(item));
10091
+ return Array.from(focus).slice(0, 8);
10092
+ }
10093
+ function notebookResearchPreviewInfo(value) {
10094
+ if (!value || typeof value !== 'object' || Array.isArray(value))
10095
+ return { hasPreview: false, rowCount: 0 };
10096
+ const record = value;
10097
+ const rowCount = typeof record.rowCount === 'number' && Number.isFinite(record.rowCount)
10098
+ ? record.rowCount
10099
+ : Array.isArray(record.rows)
10100
+ ? record.rows.length
10101
+ : 0;
10102
+ const hasColumns = Array.isArray(record.columns) && record.columns.length > 0;
10103
+ return { hasPreview: hasColumns || rowCount > 0, rowCount };
10104
+ }
10105
+ function notebookResearchEvidenceCount(value) {
10106
+ if (!value || typeof value !== 'object' || Array.isArray(value))
10107
+ return 0;
10108
+ const record = value;
10109
+ const selected = Array.isArray(record.selectedEvidence) ? record.selectedEvidence.length : 0;
10110
+ const citations = Array.isArray(record.citations) ? record.citations.length : 0;
10111
+ const roles = Array.isArray(record.evidenceRoles) ? record.evidenceRoles.length : 0;
10112
+ return selected + citations + roles;
10113
+ }
10114
+ function notebookResearchInteger(value, fallback, min, max) {
10115
+ if (typeof value !== 'string' || !value.trim())
10116
+ return fallback;
10117
+ const parsed = Number(value);
10118
+ if (!Number.isFinite(parsed))
10119
+ return fallback;
10120
+ return Math.max(min, Math.min(max, Math.floor(parsed)));
10121
+ }
10122
+ function notebookExecutionContext(value) {
10123
+ if (!value || typeof value !== 'object')
10124
+ return null;
10125
+ const raw = value;
10126
+ const notebookPath = notebookResearchString(raw.notebookPath);
10127
+ if (!notebookPath)
10128
+ return null;
10129
+ return {
10130
+ notebookPath,
10131
+ cellId: notebookResearchString(raw.cellId),
10132
+ cellName: notebookResearchString(raw.cellName),
10133
+ researchRunId: notebookResearchString(raw.researchRunId),
10134
+ source: notebookResearchString(raw.source),
10135
+ };
10136
+ }
10137
+ function notebookResearchSelectedContext(run, context) {
10138
+ return {
10139
+ activeSurface: 'notebook',
10140
+ notebookPath: run.notebookPath,
10141
+ domain: run.domain,
10142
+ sourceCellId: run.sourceCellId,
10143
+ sourceCellName: run.sourceCellName,
10144
+ context,
10145
+ };
10146
+ }
10147
+ function notebookResearchIntentPattern(intent) {
10148
+ switch (intent) {
10149
+ case 'diagnose_change':
10150
+ return {
10151
+ label: 'Change diagnosis',
10152
+ dqlTarget: 'driver or trend block',
10153
+ sqlFocus: ['baseline period', 'comparison period', 'metric delta', 'driver dimensions'],
10154
+ reviewFocus: ['time grain', 'comparison filters', 'metric definition', 'explainability evidence'],
10155
+ };
10156
+ case 'driver_breakdown':
10157
+ return {
10158
+ label: 'Driver breakdown',
10159
+ dqlTarget: 'ranking or contribution block',
10160
+ sqlFocus: ['contribution metric', 'ranking', 'tie breaker', 'segment dimensions'],
10161
+ reviewFocus: ['metric grain', 'ranking direction', 'parameterized filters', 'duplicate block match'],
10162
+ };
10163
+ case 'segment_compare':
10164
+ return {
10165
+ label: 'Segment compare',
10166
+ dqlTarget: 'comparison block',
10167
+ sqlFocus: ['shared metric', 'segment fields', 'normalized comparisons', 'deltas'],
10168
+ reviewFocus: ['segment definitions', 'filter compatibility', 'comparison grain', 'semantic metric alignment'],
10169
+ };
10170
+ case 'entity_drilldown':
10171
+ return {
10172
+ label: 'Entity drilldown',
10173
+ dqlTarget: 'entity profile or drilldown block',
10174
+ sqlFocus: ['entity grain', 'stable identifier', 'detail fields', 'supporting metrics'],
10175
+ reviewFocus: ['entity contract', 'primary key', 'allowed filters', 'downstream drill path'],
10176
+ };
10177
+ case 'anomaly_investigation':
10178
+ return {
10179
+ label: 'Anomaly investigation',
10180
+ dqlTarget: 'monitoring or exception block',
10181
+ sqlFocus: ['observed value', 'expected range', 'anomaly rule', 'supporting dimensions'],
10182
+ reviewFocus: ['threshold definition', 'time window', 'false-positive risk', 'evidence fields'],
10183
+ };
10184
+ case 'trust_gap_review':
10185
+ return {
10186
+ label: 'Trust review',
10187
+ dqlTarget: 'replacement or validation block',
10188
+ sqlFocus: ['definition comparison', 'lineage trace', 'duplicate candidates', 'conflict evidence'],
10189
+ reviewFocus: ['certified block match', 'semantic metric match', 'replacement reason', 'governance blockers'],
10190
+ };
10191
+ default:
10192
+ return {
10193
+ label: 'Ad hoc analysis',
10194
+ dqlTarget: 'custom block',
10195
+ sqlFocus: ['business question', 'grain', 'metrics', 'dimensions', 'filters'],
10196
+ reviewFocus: ['reusability', 'parameterization', 'lineage evidence', 'DQL promotion path'],
10197
+ };
10198
+ }
10199
+ }
10200
+ function notebookResearchAgentPrompt(question, intent) {
10201
+ const pattern = notebookResearchIntentPattern(intent);
10202
+ const sqlFocus = Array.isArray(pattern.sqlFocus) ? pattern.sqlFocus.join(', ') : 'business logic';
10203
+ const reviewFocus = Array.isArray(pattern.reviewFocus) ? pattern.reviewFocus.join(', ') : 'review evidence';
10204
+ return [
10205
+ `Research question: ${question}`,
10206
+ `Research pattern: ${pattern.label ?? intent}`,
10207
+ `DQL target: ${pattern.dqlTarget ?? 'custom block'}`,
10208
+ `SQL focus: ${sqlFocus}`,
10209
+ `Review focus: ${reviewFocus}`,
10210
+ 'Generate read-only SQL that can become a reusable, parameterized DQL block after human review.',
10211
+ ].join('\n');
10212
+ }
10213
+ function notebookResearchRouteDecisionForRun(run, routeDecision, sqlForPreview) {
10214
+ if (!routeDecision)
10215
+ return undefined;
10216
+ const hasSelectedNotebookSql = Boolean(run.sourceCellId && notebookResearchString(sqlForPreview));
10217
+ if (!hasSelectedNotebookSql || routeDecision.route !== 'certified')
10218
+ return routeDecision;
10219
+ const certifiedEvidence = routeDecision.selectedEvidence.find((item) => item.role === 'exact_certified_answer')
10220
+ ?? routeDecision.selectedEvidence.find((item) => item.role === 'certified_context');
10221
+ const certifiedName = certifiedEvidence?.name ?? routeDecision.certifiedApplicability?.objectKey ?? 'matching certified artifact';
10222
+ return {
10223
+ ...routeDecision,
10224
+ route: 'research',
10225
+ trustLabel: routeDecision.trustLabel === 'certified' ? 'mixed' : routeDecision.trustLabel,
10226
+ reviewStatus: 'needs_review',
10227
+ exactObjectKey: undefined,
10228
+ reason: `Selected notebook SQL from "${run.sourceCellName ?? run.sourceCellId}" is the source for this research run. Certified artifact "${certifiedName}" is duplicate/reuse evidence until the reviewed SQL, grain, parameters, and preview are accepted.`,
10229
+ selectedEvidence: routeDecision.selectedEvidence.map((item) => item.role === 'exact_certified_answer'
10230
+ ? {
10231
+ ...item,
10232
+ role: 'certified_context',
10233
+ reason: 'Certified block is reuse or duplicate evidence for the selected notebook SQL; it is not an automatic answer for this research run.',
10234
+ }
10235
+ : item),
10236
+ missingContext: routeDecision.missingContext.length > 0
10237
+ ? routeDecision.missingContext
10238
+ : [{
10239
+ kind: 'metadata',
10240
+ severity: 'warning',
10241
+ message: 'Selected notebook SQL requires human review before DQL promotion, even when certified context exists.',
10242
+ }],
10243
+ };
10244
+ }
10245
+ function notebookResearchContextPreview(contextPack) {
10246
+ return {
10247
+ contextPackId: contextPack.id,
10248
+ trustLabel: contextPack.trustLabel,
10249
+ routeDecision: contextPack.routeDecision
10250
+ ? {
10251
+ route: contextPack.routeDecision.route,
10252
+ intent: contextPack.routeDecision.intent,
10253
+ reason: contextPack.routeDecision.reason,
10254
+ }
10255
+ : undefined,
10256
+ evidence: contextPack.evidenceRoles.slice(0, 16).map((item) => ({
10257
+ objectKey: item.objectKey,
10258
+ objectType: item.objectType,
10259
+ name: item.name,
10260
+ role: item.role,
10261
+ reason: item.reason,
10262
+ })),
10263
+ summaries: contextPack.evidenceSummaries.slice(0, 12).map((item) => ({
10264
+ title: item.title,
10265
+ detail: item.detail,
10266
+ objectType: item.objectType,
10267
+ reason: item.reason,
10268
+ })),
10269
+ relations: contextPack.allowedSqlContext.relations.slice(0, 12).map((relation) => ({
10270
+ relation: relation.relation,
10271
+ name: relation.name,
10272
+ source: relation.source,
10273
+ columns: relation.columns.slice(0, 24).map((column) => {
10274
+ if (typeof column === 'string')
10275
+ return column;
10276
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
10277
+ return String(column.name);
10278
+ }
10279
+ return String(column);
10280
+ }),
10281
+ })),
10282
+ missingContext: contextPack.missingContext.slice(0, 12).map((item) => ({
10283
+ kind: item.kind,
10284
+ message: item.message,
10285
+ severity: item.severity,
10286
+ })),
10287
+ warnings: contextPack.warnings.slice(0, 12),
10288
+ topRejected: contextPack.retrievalDiagnostics.topRejected.slice(0, 8).map((item) => ({
10289
+ name: item.name,
10290
+ objectType: item.objectType,
10291
+ reason: item.reason,
10292
+ score: item.score,
10293
+ })),
10294
+ counts: {
10295
+ objects: contextPack.objects.length,
10296
+ evidence: contextPack.evidenceRoles.length,
10297
+ relations: contextPack.allowedSqlContext.relations.length,
10298
+ warnings: contextPack.warnings.length + contextPack.missingContext.length,
10299
+ },
10300
+ };
10301
+ }
10302
+ function normalizeNotebookAgentResult(result) {
10303
+ const columns = Array.isArray(result.columns)
10304
+ ? result.columns.map((column) => {
10305
+ if (typeof column === 'string')
10306
+ return column;
10307
+ if (column && typeof column === 'object' && typeof column.name === 'string') {
10308
+ return String(column.name);
10309
+ }
10310
+ return String(column);
10311
+ })
10312
+ : [];
10313
+ const rows = Array.isArray(result.rows)
10314
+ ? result.rows
10315
+ .filter((row) => Boolean(row && typeof row === 'object' && !Array.isArray(row)))
10316
+ .map((row) => row)
10317
+ : [];
10318
+ return {
10319
+ columns,
10320
+ rows,
10321
+ rowCount: typeof result.rowCount === 'number' ? result.rowCount : rows.length,
10322
+ executionTime: typeof result.executionTime === 'number' ? result.executionTime : 0,
10323
+ };
10324
+ }
10325
+ function notebookResearchSummary(question, result, error) {
10326
+ if (error)
10327
+ return `Research could not preview results for "${question}". ${error}`;
10328
+ if (result) {
10329
+ const rowCount = result.rowCount ?? result.rows.length;
10330
+ return `Research preview for "${question}" returned ${rowCount.toLocaleString()} row${rowCount === 1 ? '' : 's'} across ${result.columns.length.toLocaleString()} column${result.columns.length === 1 ? '' : 's'}.`;
10331
+ }
10332
+ return `Research plan created for "${question}". Add or generate SQL, then run a bounded preview.`;
10333
+ }
10334
+ function recordNotebookQueryRun(projectRoot, input) {
10335
+ try {
10336
+ recordQueryRun(projectRoot, {
10337
+ objectKey: input.objectKey,
10338
+ source: input.source,
10339
+ status: input.status,
10340
+ rowCount: input.rowCount,
10341
+ durationMs: input.durationMs,
10342
+ errorCode: input.errorCode,
10343
+ payload: {
10344
+ notebookPath: input.notebookPath,
10345
+ cellId: input.cellId,
10346
+ cellName: input.cellName,
10347
+ researchRunId: input.researchRunId,
10348
+ contextPackId: input.contextPackId,
10349
+ sql: input.sql ? compactSqlForRunHistory(input.sql) : undefined,
10350
+ },
10351
+ });
10352
+ }
10353
+ catch {
10354
+ // Query history is advisory metadata and must not block notebook execution.
10355
+ }
10356
+ }
10357
+ function updateNotebookResearchFromCellExecution(projectRoot, execContext, input) {
10358
+ if (!execContext.notebookPath || (!execContext.cellId && !execContext.researchRunId))
10359
+ return;
10360
+ const storage = new LocalNotebookResearchStorage(defaultNotebookResearchDbPath(projectRoot));
10361
+ try {
10362
+ const page = storage.listRunsPage({
10363
+ notebookPath: execContext.notebookPath,
10364
+ sort: 'updated_desc',
10365
+ limit: 500,
10366
+ });
10367
+ const matches = page.runs.filter((run) => {
10368
+ if (execContext.researchRunId && run.id === execContext.researchRunId)
10369
+ return true;
10370
+ return Boolean(execContext.cellId && run.sourceCellId === execContext.cellId);
10371
+ });
10372
+ const now = new Date().toISOString();
10373
+ for (const run of matches.slice(0, 20)) {
10374
+ if (input.status === 'success') {
10375
+ storage.updateRun(run.id, {
10376
+ status: 'ready',
10377
+ resultPreview: input.resultPreview,
10378
+ summary: notebookResearchSummary(run.question, input.resultPreview, undefined),
10379
+ recommendation: run.recommendation ?? 'Review SQL, metadata evidence, grain, filters, and duplicate matches before DQL draft promotion.',
10380
+ error: '',
10381
+ lastRunAt: now,
10382
+ });
10383
+ }
10384
+ else {
10385
+ storage.updateRun(run.id, {
10386
+ status: 'error',
10387
+ error: input.error,
10388
+ summary: notebookResearchSummary(run.question, undefined, input.error),
10389
+ lastRunAt: now,
10390
+ });
10391
+ }
10392
+ }
10393
+ }
10394
+ catch {
10395
+ // Research run sync is advisory metadata and must not block notebook execution.
10396
+ }
10397
+ finally {
10398
+ storage.close();
10399
+ }
10400
+ }
10401
+ function compactSqlForRunHistory(sql) {
10402
+ const clean = sql.replace(/\s+/g, ' ').trim();
10403
+ return clean.length > 1200 ? `${clean.slice(0, 1197)}...` : clean;
10404
+ }
6856
10405
  function buildAgentSchemaContextFromContextPack(question, contextPack) {
6857
10406
  const byRelation = new Map();
6858
10407
  const objectsByKey = new Map(contextPack.objects.map((object) => [object.objectKey, object]));