@duckcodeailabs/dql-cli 1.6.32 → 1.6.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/apps-api.d.ts.map +1 -1
  2. package/dist/apps-api.js +46 -3
  3. package/dist/apps-api.js.map +1 -1
  4. package/dist/assets/dql-notebook/assets/index-CYQrnwxa.css +1 -0
  5. package/dist/assets/dql-notebook/assets/index-GofggEGB.js +5777 -0
  6. package/dist/assets/dql-notebook/index.html +2 -2
  7. package/dist/block-studio-import.d.ts +2 -0
  8. package/dist/block-studio-import.d.ts.map +1 -1
  9. package/dist/block-studio-import.js.map +1 -1
  10. package/dist/commands/agent.d.ts.map +1 -1
  11. package/dist/commands/agent.js +4 -3
  12. package/dist/commands/agent.js.map +1 -1
  13. package/dist/commands/init.d.ts.map +1 -1
  14. package/dist/commands/init.js +22 -7
  15. package/dist/commands/init.js.map +1 -1
  16. package/dist/llm/answer-loop-tools.d.ts +1 -4
  17. package/dist/llm/answer-loop-tools.d.ts.map +1 -1
  18. package/dist/llm/answer-loop-tools.js +143 -37
  19. package/dist/llm/answer-loop-tools.js.map +1 -1
  20. package/dist/llm/providers/dql-agent-provider.d.ts +13 -1
  21. package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
  22. package/dist/llm/providers/dql-agent-provider.js +116 -20
  23. package/dist/llm/providers/dql-agent-provider.js.map +1 -1
  24. package/dist/llm/tools.d.ts +7 -0
  25. package/dist/llm/tools.d.ts.map +1 -1
  26. package/dist/llm/tools.js +63 -2
  27. package/dist/llm/tools.js.map +1 -1
  28. package/dist/llm/types.d.ts +2 -2
  29. package/dist/llm/types.d.ts.map +1 -1
  30. package/dist/local-runtime.d.ts +44 -1
  31. package/dist/local-runtime.d.ts.map +1 -1
  32. package/dist/local-runtime.js +1302 -136
  33. package/dist/local-runtime.js.map +1 -1
  34. package/dist/package.json +10 -10
  35. package/package.json +11 -11
  36. package/dist/assets/dql-notebook/assets/index-B6x1cRJo.js +0 -5698
  37. package/dist/assets/dql-notebook/assets/index-D9Owd4EN.css +0 -1
@@ -14,7 +14,7 @@ import { listBlockTemplates } from './block-templates.js';
14
14
  import { getRunner as getLLMRunner } from './llm/index.js';
15
15
  import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
16
16
  import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
17
- import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, loadSkills, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
17
+ import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, probeLocalOllamaEmbeddings, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
18
18
  import { gatherProposeEnrichment } from './propose-enrich.js';
19
19
  import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
20
20
  import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
@@ -145,6 +145,7 @@ export function parseAgentRunRequestBody(body) {
145
145
  runId: agentRunString(record.runId),
146
146
  reasoningEffort: parseAgentRunReasoningEffort(record.reasoningEffort),
147
147
  analysisDepth: parseAgentRunAnalysisDepth(record.analysisDepth) ?? parseAgentRunAnalysisDepth(record.depth),
148
+ thinkingMode: coerceThinkingMode(record.thinkingMode),
148
149
  },
149
150
  };
150
151
  }
@@ -325,6 +326,32 @@ export async function startLocalServer(opts) {
325
326
  }
326
327
  return candidate;
327
328
  };
329
+ // Zero-config semantic search: if the user is running Ollama with an embedding model
330
+ // but hasn't set an embed env var, auto-detect it so retrieval + metric/block matching
331
+ // get real semantic recall out of the box (match by meaning, not just keywords) — the
332
+ // biggest lever for "find the right metric/block instead of jumping to raw SQL".
333
+ // Explicit config always wins; when nothing is found we stay on the deterministic
334
+ // keyword matcher. Scoped to the app server (not eval/CI) and fully best-effort.
335
+ if (!process.env.DQL_OLLAMA_EMBED_URL && !process.env.DQL_OPENAI_API_KEY && !process.env.OPENAI_API_KEY) {
336
+ try {
337
+ const detected = await probeLocalOllamaEmbeddings();
338
+ if (detected) {
339
+ process.env.DQL_OLLAMA_EMBED_URL = detected.endpoint;
340
+ process.env.DQL_OLLAMA_EMBED_MODEL = detected.model;
341
+ console.log(`[dql] Semantic search on: local Ollama embeddings (${detected.model}) — matching by meaning, not just keywords.`);
342
+ }
343
+ else {
344
+ console.log('[dql] Semantic search: keyword-only. For higher matching accuracy, install Ollama and run `ollama pull nomic-embed-text` — DQL detects and uses it automatically, fully local & free.');
345
+ }
346
+ }
347
+ catch {
348
+ // Probe failure never blocks startup — fall back to the keyword matcher.
349
+ }
350
+ }
351
+ // Auto-ensure the active connection's driver so a configured connection is never
352
+ // left "broken" after a fresh clone, a CLI upgrade, or a Node version change (the
353
+ // driver lives in gitignored, per-project .dql/connectors). Best-effort + non-fatal.
354
+ ensureConnectorInstalledForStartup(projectRoot, connection?.driver);
328
355
  // Load semantic layer via provider system (dql native, dbt, cubejs, etc.)
329
356
  let semanticLayer;
330
357
  let semanticLayerErrors = [];
@@ -549,7 +576,7 @@ export async function startLocalServer(opts) {
549
576
  return provider.generate([{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 600, temperature: 0.2, signal });
550
577
  },
551
578
  });
552
- async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer') {
579
+ async function runGovernedAgentAnswerForRun(request, repair, route = 'generated_answer', onProgress) {
553
580
  const governed = resolveGovernedAnswerRunner(projectRoot);
554
581
  const resolvedProvider = governed?.provider ?? null;
555
582
  const runner = governed?.runner ?? null;
@@ -559,10 +586,16 @@ export async function startLocalServer(opts) {
559
586
  let governedAnswer;
560
587
  let providerError;
561
588
  const isRepair = (repair?.attempt ?? 0) > 0 && Boolean(repair?.repairHint);
562
- const reasoningEffort = request.reasoningEffort
563
- ? resolveRequestedRunReasoningEffort(projectRoot, resolvedProvider, request.reasoningEffort, isRepair)
589
+ // The chat composer sends a `thinkingMode` (auto/low/medium/high); resolve it
590
+ // into the effort+depth bundle it stands for. An explicit `reasoningEffort` /
591
+ // `analysisDepth` on the request (e.g. a CLI flag) still wins over the mode.
592
+ const thinkingResolved = request.thinkingMode ? resolveThinkingMode(request.thinkingMode) : {};
593
+ const requestedEffort = request.reasoningEffort ?? thinkingResolved.reasoningEffort;
594
+ const requestedDepth = request.analysisDepth ?? thinkingResolved.analysisDepth;
595
+ const reasoningEffort = requestedEffort
596
+ ? resolveRequestedRunReasoningEffort(projectRoot, resolvedProvider, requestedEffort, isRepair)
564
597
  : resolveRunReasoningEffort(projectRoot, resolvedProvider, route, isRepair);
565
- const analysisDepth = request.analysisDepth ?? (route === 'research' ? 'deep' : undefined);
598
+ const analysisDepth = requestedDepth ?? (route === 'research' ? 'deep' : undefined);
566
599
  const contextEnvelope = {
567
600
  mode: 'agent_run',
568
601
  selectedObject: request.selectedObject,
@@ -575,7 +608,9 @@ export async function startLocalServer(opts) {
575
608
  ...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
576
609
  ].join(' '),
577
610
  };
578
- const controller = new AbortController();
611
+ const controller = request.runId
612
+ ? activeAgentRunControllers.get(request.runId) ?? new AbortController()
613
+ : new AbortController();
579
614
  // Best-effort active warehouse dialect so Lane-2 semantic compiles emit
580
615
  // dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
581
616
  // connection is configured — the compiler then uses its default dialect.
@@ -605,6 +640,8 @@ export async function startLocalServer(opts) {
605
640
  executeGeneratedSql: executeGeneratedSqlForAgent,
606
641
  getSchemaContext: getSchemaContextForAgent,
607
642
  }, (turn) => {
643
+ if (turn.kind === 'thinking')
644
+ onProgress?.(turn.text);
608
645
  if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
609
646
  governedAnswer = turn.output;
610
647
  }
@@ -633,15 +670,82 @@ export async function startLocalServer(opts) {
633
670
  ].filter((token) => Boolean(token));
634
671
  return tokens.length > 0 ? tokens.join('; ') : governedAnswer.text;
635
672
  }
636
- const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emitAnswerDelta }) => {
673
+ /**
674
+ * The model picks a visualization while composing the answer, but only the
675
+ * executed rows know whether that choice is viable. Preserve an authored DQL
676
+ * chart; otherwise validate the model preference against the actual result and
677
+ * attach a compact, explainable display contract for every consumer.
678
+ */
679
+ function applySmartVisualization(governedAnswer, question) {
680
+ const result = governedAnswer.result;
681
+ if (!result || !Array.isArray(result.rows))
682
+ return;
683
+ const rows = result.rows.filter((row) => Boolean(agentRunRecord(row)));
684
+ const columns = Array.isArray(result.columns)
685
+ ? result.columns.map((column) => typeof column === 'string'
686
+ ? column
687
+ : agentRunString(agentRunRecord(column)?.name) ?? String(column))
688
+ : (rows.length > 0 ? Object.keys(rows[0]) : []);
689
+ if (rows.length === 0 || columns.length === 0)
690
+ return;
691
+ const existing = agentRunRecord(result.chartConfig) ?? {};
692
+ // A declared chart on the execution result came from authored DQL and is a
693
+ // governed display contract. Agent `suggestedViz` remains a soft preference.
694
+ if (typeof existing.chart === 'string')
695
+ return;
696
+ const recommendation = recommendVisualization(projectRoot, {
697
+ blockRef: governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name,
698
+ prompt: question,
699
+ resultSchema: columns,
700
+ rowSample: rows.slice(0, 25),
701
+ defaultVisualization: governedAnswer.suggestedViz,
702
+ });
703
+ if (!recommendation.ok)
704
+ return;
705
+ const fieldHints = recommendation.display.fieldHints ?? {};
706
+ const chart = recommendation.display.defaultVisualization.replace(/_/g, '-');
707
+ const agentChoice = typeof governedAnswer.suggestedViz === 'string'
708
+ ? governedAnswer.suggestedViz.toLowerCase().replace(/_/g, '-')
709
+ : undefined;
710
+ result.chartConfig = {
711
+ ...existing,
712
+ chart,
713
+ decisionSource: agentChoice === chart ? 'agent' : 'data',
714
+ rationale: recommendation.display.rationale,
715
+ ...(typeof fieldHints.x === 'string' ? { x: fieldHints.x } : {}),
716
+ ...(typeof fieldHints.y === 'string' ? { y: fieldHints.y } : {}),
717
+ };
718
+ governedAnswer.suggestedViz = chart;
719
+ const evidence = governedAnswer.evidence ?? {
720
+ route: [],
721
+ lineage: [],
722
+ businessContext: [],
723
+ selectedAssets: [],
724
+ sourceTables: [],
725
+ semanticObjects: [],
726
+ citations: [],
727
+ };
728
+ governedAnswer.evidence = evidence;
729
+ evidence.route = [
730
+ ...evidence.route,
731
+ {
732
+ tool: 'recommend_visualization',
733
+ status: 'selected',
734
+ label: `Smart visualization: ${chart.replace(/-/g, ' ')}`,
735
+ detail: recommendation.display.rationale,
736
+ },
737
+ ];
738
+ }
739
+ const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emit, emitAnswerDelta }) => {
637
740
  let governedAnswer;
638
741
  try {
639
- governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route);
742
+ governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }));
640
743
  // Surface the approved Hint-Graph corrections that shaped this answer so the
641
744
  // UI can show an "applied learnings" chip (memoryContext is already on the answer).
642
745
  if (!governedAnswer.appliedHints) {
643
746
  governedAnswer.appliedHints = governedAnswer.contextPack?.appliedHints;
644
747
  }
748
+ applySmartVisualization(governedAnswer, request.question);
645
749
  }
646
750
  catch (error) {
647
751
  const message = formatAgentRunInfrastructureError(error, 'AI answer provider');
@@ -662,12 +766,24 @@ export async function startLocalServer(opts) {
662
766
  }
663
767
  const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
664
768
  const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
769
+ const isSemantic = governedAnswer.route?.tier === 'semantic_metric';
665
770
  const isGroundingGap = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'grounding_gap';
666
771
  const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
772
+ // The model tried to compose a governed query and declined despite having usable
773
+ // context (e.g. it wasn't confident about a multi-table join). That is NOT a
774
+ // question for the USER to clarify — it's a case to retry harder: escalate to a
775
+ // deeper research pass (higher reasoning effort + deep analysis) through the
776
+ // engine's bounded loop, exactly as a grounding gap is retried today.
777
+ const isModelDeclined = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'model_declined';
667
778
  const groundingRepairHint = isGroundingGap ? groundingGapRepairHint(governedAnswer) : undefined;
668
- // A provider outage is a retryable infrastructure failure, not a question the
669
- // user needs to clarify surface it as blocked so the UI offers a retry.
670
- const needsClarification = governedAnswer.kind === 'no_answer' && !isGroundingGap && !isProviderError;
779
+ const declinedRepairHint = isModelDeclined
780
+ ? 'The first attempt declined to compose a governed query despite available context. Investigate the join path across the requested entities/metrics and compose a review-required query rather than declining.'
781
+ : undefined;
782
+ // Only a genuinely AMBIGUOUS question is surfaced as "needs clarification". A
783
+ // grounding gap or a model decline is retried/escalated by the engine, and a
784
+ // provider outage is surfaced as blocked so the UI offers a retry.
785
+ const needsClarification = governedAnswer.kind === 'no_answer'
786
+ && !isGroundingGap && !isProviderError && !isModelDeclined;
671
787
  const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
672
788
  const runnableSql = governedAnswer.kind === 'no_answer' ? undefined : sql;
673
789
  // Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
@@ -700,9 +816,31 @@ export async function startLocalServer(opts) {
700
816
  synthesizedAnswer = undefined;
701
817
  }
702
818
  }
703
- const status = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'completed' : 'needs_review';
704
- const trustState = isProviderError ? 'blocked' : needsClarification ? 'not_applicable' : isCertified ? 'certified' : 'review_required';
705
- const stopReason = isProviderError ? 'blocked' : needsClarification ? 'needs_clarification' : isCertified ? 'certified_answer_found' : 'human_review_required';
819
+ const status = isProviderError
820
+ ? 'blocked'
821
+ : needsClarification
822
+ ? 'needs_clarification'
823
+ : isCertified || isSemantic
824
+ ? 'completed'
825
+ : 'needs_review';
826
+ const trustState = isProviderError
827
+ ? 'blocked'
828
+ : needsClarification
829
+ ? 'not_applicable'
830
+ : isCertified
831
+ ? 'certified'
832
+ : isSemantic
833
+ ? 'governed'
834
+ : 'review_required';
835
+ const stopReason = isProviderError
836
+ ? 'blocked'
837
+ : needsClarification
838
+ ? 'needs_clarification'
839
+ : isCertified
840
+ ? 'certified_answer_found'
841
+ : isSemantic
842
+ ? 'governed_semantic_answer'
843
+ : 'human_review_required';
706
844
  const nextActions = needsClarification
707
845
  ? [{ id: 'clarify', label: 'Clarify question', route: 'generated_answer' }]
708
846
  : [
@@ -712,6 +850,7 @@ export async function startLocalServer(opts) {
712
850
  ];
713
851
  return {
714
852
  resolvedRoute,
853
+ answerRefusalCode: governedAnswer.kind === 'no_answer' ? governedAnswer.refusalCode : undefined,
715
854
  answerTier: governedAnswer.route?.tier,
716
855
  summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
717
856
  answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
@@ -725,18 +864,20 @@ export async function startLocalServer(opts) {
725
864
  ? (governedAnswer.dqlArtifact && !isProviderError
726
865
  ? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
727
866
  : [])
728
- : [agentRunArtifact('answer', isCertified ? 'Certified answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : 'review_required')],
867
+ : [agentRunArtifact('answer', isCertified ? 'Certified answer' : isSemantic ? 'Governed semantic answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : isSemantic ? 'governed' : 'review_required')],
729
868
  evaluations: [
730
869
  agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
731
870
  plannedRoute: route,
732
871
  resolvedRoute,
733
872
  aiRoute: governedAnswer.route,
734
873
  }),
735
- agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified, isCertified ? 'info' : 'warning', isCertified
874
+ agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified || isSemantic, isCertified || isSemantic ? 'info' : 'warning', isCertified
736
875
  ? 'The answer came from certified DQL context.'
737
- : needsClarification
738
- ? 'The answer loop needs more context before producing a governed answer.'
739
- : 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
876
+ : isSemantic
877
+ ? 'The answer was compiled and executed from governed semantic definitions; it is not a human-certified reusable block.'
878
+ : needsClarification
879
+ ? 'The answer loop needs more context before producing a governed answer.'
880
+ : 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
740
881
  ...(isGroundingGap ? [
741
882
  {
742
883
  ...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
@@ -749,6 +890,18 @@ export async function startLocalServer(opts) {
749
890
  repairAction: { kind: 'retry', hint: groundingRepairHint },
750
891
  },
751
892
  ] : []),
893
+ ...(isModelDeclined ? [
894
+ {
895
+ ...agentRunEvaluation('declined-despite-context', 'Answer grounding', false, 'blocking', 'The model declined to compose a governed query despite available context — escalating to a deeper investigation before accepting a refusal.', {
896
+ refusalCode: governedAnswer.refusalCode,
897
+ refusalDetails: governedAnswer.refusalDetails,
898
+ validationWarnings: governedAnswer.validationWarnings,
899
+ route: governedAnswer.route,
900
+ }),
901
+ suggestedRepair: declinedRepairHint,
902
+ repairAction: { kind: 'escalate', route: 'research', hint: declinedRepairHint },
903
+ },
904
+ ] : []),
752
905
  ...(governedAnswer.executionError ? [
753
906
  agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
754
907
  ] : []),
@@ -808,6 +961,7 @@ export async function startLocalServer(opts) {
808
961
  const agentRunExecutors = {
809
962
  conversation: conversationRunExecutor,
810
963
  certified_answer: answerRunExecutor,
964
+ semantic_answer: answerRunExecutor,
811
965
  generated_answer: answerRunExecutor,
812
966
  research: async (researchContext) => {
813
967
  const { runId, request, routeDecision, emit } = researchContext;
@@ -984,10 +1138,26 @@ export async function startLocalServer(opts) {
984
1138
  };
985
1139
  },
986
1140
  sql_cell: async ({ request, routeDecision, attempt, repairHint }) => {
987
- const result = await buildAgentPromptArtifact(request, 'cell', { attempt, repairHint });
1141
+ // P4: route Notebook SQL-cell generation through the SAME tool-rich governed
1142
+ // pipeline Ask AI uses (schema-discovery tools P3 + declined-retry P1 + budget
1143
+ // recovery P2), instead of the tool-less buildFromPrompt path. A SQL cell only
1144
+ // needs the SQL, which the governed answer produces directly as proposedSql.
1145
+ const governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt: attempt ?? 0, repairHint });
1146
+ const sql = governedAnswer.proposedSql ?? governedAnswer.sql ?? '';
1147
+ const explanation = governedAnswer.answer ?? governedAnswer.text;
1148
+ const hasSql = Boolean(sql.trim());
1149
+ // Preserve the BuildCellResult shape the notebook UI already consumes.
1150
+ const result = {
1151
+ target: 'cell',
1152
+ sql,
1153
+ explanation,
1154
+ ...(governedAnswer.appliedSkills ? { appliedSkills: governedAnswer.appliedSkills } : {}),
1155
+ };
988
1156
  return {
989
- summary: 'Created a review-required SQL cell draft.',
990
- answer: result.target === 'cell' ? result.explanation : undefined,
1157
+ summary: hasSql
1158
+ ? 'Created a review-required SQL cell draft.'
1159
+ : (explanation || 'No SQL could be generated for this request.'),
1160
+ answer: explanation,
991
1161
  artifacts: [agentRunArtifact('sql_cell', 'Generated SQL cell', result)],
992
1162
  evaluations: [
993
1163
  agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to SQL cell generation.'),
@@ -1055,6 +1225,15 @@ export async function startLocalServer(opts) {
1055
1225
  ],
1056
1226
  };
1057
1227
  }
1228
+ // P4 (partial): a direct "build a block from a cold prompt" still uses the
1229
+ // tool-less buildFromPrompt path. Unlike a SQL cell, a block needs SQL→DQL-block
1230
+ // CONSTRUCTION (name, outputs, grain, certifier verdict, reflection), which
1231
+ // buildFromPrompt owns — routing this through answer() cleanly requires driving
1232
+ // that construction from the tool-rich pipeline (thread captureGeneratedDraft
1233
+ // through the governed answer), a larger change deferred to its own session.
1234
+ // NOTE: the two-step "ask a question → Create DQL draft" flow ALREADY routes
1235
+ // block creation through the tool-rich governed pipeline via the carried-artifact
1236
+ // branch above, so only this cold-start direct-build case remains tool-less.
1058
1237
  const result = await buildAgentPromptArtifact(request, 'block', { attempt, repairHint });
1059
1238
  const ready = result.target === 'block' ? result.certifierVerdict.ready : false;
1060
1239
  return {
@@ -1222,6 +1401,9 @@ export async function startLocalServer(opts) {
1222
1401
  getCatalogContext: () => buildAgentRunCatalogContext(),
1223
1402
  });
1224
1403
  const agentRunStore = new FileAgentRunStore({ path: defaultAgentRunStorePath(projectRoot) });
1404
+ // A run may outlive its streaming browser connection, so cancellation is
1405
+ // server-owned and keyed by run id rather than relying on fetch abort alone.
1406
+ const activeAgentRunControllers = new Map();
1225
1407
  // Server-side conversation threads: persisted multi-turn state (survives refresh).
1226
1408
  // Lazy so a failed native-sqlite load never blocks the runtime; conversation
1227
1409
  // persistence degrades to the per-request context instead.
@@ -1400,13 +1582,13 @@ export async function startLocalServer(opts) {
1400
1582
  sql: prepared.sql,
1401
1583
  };
1402
1584
  };
1403
- const getSchemaContextForAgent = async (question) => {
1585
+ const getSchemaContextForAgent = async (question, preparedContextPack) => {
1404
1586
  const scanRuntimeSchema = async () => {
1405
1587
  if (!connection)
1406
1588
  return { ranked: [], snapshot: [] };
1407
1589
  const result = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
1408
1590
  FROM information_schema.columns
1409
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
1591
+ WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
1410
1592
  ORDER BY table_schema, table_name, ordinal_position
1411
1593
  LIMIT 2000`, [], runtimeVariables({}), connection);
1412
1594
  return {
@@ -1414,11 +1596,14 @@ export async function startLocalServer(opts) {
1414
1596
  snapshot: buildAgentSchemaContext(question, result.rows, { includeUnscored: true, limit: 500 }),
1415
1597
  };
1416
1598
  };
1417
- const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question).catch(() => []);
1599
+ const catalogContext = await buildAgentSchemaContextFromCatalog(projectRoot, question, preparedContextPack).catch(() => []);
1418
1600
  if (catalogContext.length > 0) {
1419
1601
  if (!connection)
1420
1602
  return catalogContext;
1421
- const runtimeScan = shouldAugmentAgentRuntimeSchema(question)
1603
+ // Rescan live when the question shape calls for it OR the stored snapshot is
1604
+ // stale/absent (P6) — otherwise a warehouse schema change between sessions is
1605
+ // silently reasoned over from a cached snapshot that never expires.
1606
+ const runtimeScan = (shouldAugmentAgentRuntimeSchema(question) || runtimeSnapshotStale(projectRoot))
1422
1607
  ? await scanRuntimeSchema().catch(() => undefined)
1423
1608
  : undefined;
1424
1609
  const runtimeContext = runtimeScan?.ranked ?? [];
@@ -2196,7 +2381,7 @@ export async function startLocalServer(opts) {
2196
2381
  try {
2197
2382
  const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
2198
2383
  FROM information_schema.tables
2199
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, activeConnection);
2384
+ WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, activeConnection);
2200
2385
  tableMapping = buildSemanticTableMapping(semanticLayer, tablesResult.rows);
2201
2386
  }
2202
2387
  catch {
@@ -2321,6 +2506,8 @@ export async function startLocalServer(opts) {
2321
2506
  owner: typeof body.owner === 'string' ? body.owner : undefined,
2322
2507
  tags: Array.isArray(body.tags) ? body.tags.map(String) : undefined,
2323
2508
  provider: typeof body.provider === 'string' ? body.provider : undefined,
2509
+ async: body.async === true,
2510
+ persistence: body.persistence === 'session-only' || body.persistence === 'draft-files' ? body.persistence : undefined,
2324
2511
  }, semanticLayer);
2325
2512
  };
2326
2513
  const loadDqlGenerationSession = (importId) => {
@@ -2470,6 +2657,20 @@ export async function startLocalServer(opts) {
2470
2657
  res.end(serializeJSON({ runs, total: agentRunStore.list().length, limit }));
2471
2658
  return;
2472
2659
  }
2660
+ if (req.method === 'POST' && /^\/api\/agent-runs\/[^/]+\/cancel$/.test(path)) {
2661
+ const match = path.match(/^\/api\/agent-runs\/([^/]+)\/cancel$/);
2662
+ const id = decodeURIComponent(match?.[1] ?? '');
2663
+ const controller = activeAgentRunControllers.get(id);
2664
+ if (!controller) {
2665
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
2666
+ res.end(serializeJSON({ ok: false, error: 'This run is no longer active.' }));
2667
+ return;
2668
+ }
2669
+ controller.abort(new Error('Stopped by user.'));
2670
+ res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
2671
+ res.end(serializeJSON({ ok: true, id }));
2672
+ return;
2673
+ }
2473
2674
  if (req.method === 'POST' && path === '/api/agent-runs') {
2474
2675
  try {
2475
2676
  const body = await readJSON(req).catch(() => null);
@@ -2489,27 +2690,36 @@ export async function startLocalServer(opts) {
2489
2690
  parsed.request.conversationContext = await conversationContextFromThread(conversationStore, parsed.request.threadId, parsed.request.conversationContext, parsed.request.question);
2490
2691
  }
2491
2692
  const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
2492
- if (wantsStream) {
2493
- res.writeHead(200, {
2494
- 'Content-Type': 'text/event-stream; charset=utf-8',
2495
- 'Cache-Control': 'no-cache',
2496
- 'Connection': 'keep-alive',
2497
- 'X-Accel-Buffering': 'no',
2498
- });
2499
- const run = await agentRunEngine.run(parsed.request, (event) => {
2500
- writeAgentRunSse(res, 'agent-run-event', event);
2501
- }, (delta) => {
2502
- writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
2503
- });
2693
+ const runId = parsed.request.runId;
2694
+ if (runId)
2695
+ activeAgentRunControllers.set(runId, new AbortController());
2696
+ try {
2697
+ if (wantsStream) {
2698
+ res.writeHead(200, {
2699
+ 'Content-Type': 'text/event-stream; charset=utf-8',
2700
+ 'Cache-Control': 'no-cache',
2701
+ 'Connection': 'keep-alive',
2702
+ 'X-Accel-Buffering': 'no',
2703
+ });
2704
+ const run = await agentRunEngine.run(parsed.request, (event) => {
2705
+ writeAgentRunSse(res, 'agent-run-event', event);
2706
+ }, (delta) => {
2707
+ writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
2708
+ });
2709
+ recordConversationTurn(conversationStore, parsed.request.threadId, run);
2710
+ writeAgentRunSse(res, 'agent-run-complete', run);
2711
+ res.end();
2712
+ return;
2713
+ }
2714
+ const run = await agentRunEngine.run(parsed.request);
2504
2715
  recordConversationTurn(conversationStore, parsed.request.threadId, run);
2505
- writeAgentRunSse(res, 'agent-run-complete', run);
2506
- res.end();
2507
- return;
2716
+ res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
2717
+ res.end(serializeJSON({ run }));
2718
+ }
2719
+ finally {
2720
+ if (runId)
2721
+ activeAgentRunControllers.delete(runId);
2508
2722
  }
2509
- const run = await agentRunEngine.run(parsed.request);
2510
- recordConversationTurn(conversationStore, parsed.request.threadId, run);
2511
- res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
2512
- res.end(serializeJSON({ run }));
2513
2723
  }
2514
2724
  catch (error) {
2515
2725
  const message = error instanceof Error ? error.message : String(error);
@@ -2919,7 +3129,9 @@ export async function startLocalServer(opts) {
2919
3129
  // are shared; PERSONAL skills (user set) are user-bound. ────────────────
2920
3130
  if (req.method === 'GET' && path === '/api/skills') {
2921
3131
  try {
2922
- const skills = loadSkills(projectRoot).skills.map(serializeSkill);
3132
+ const skills = loadSkills(projectRoot).skills
3133
+ .filter((skill) => skill.scope === 'project')
3134
+ .map(serializeSkill);
2923
3135
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2924
3136
  res.end(serializeJSON({ skills }));
2925
3137
  }
@@ -2932,16 +3144,20 @@ export async function startLocalServer(opts) {
2932
3144
  // Form pickers: metrics from the semantic layer + certified block ids.
2933
3145
  if (req.method === 'GET' && path === '/api/skills/options') {
2934
3146
  try {
3147
+ const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
3148
+ const limit = Math.min(50, Math.max(1, Number(url.searchParams.get('limit') ?? 30) || 30));
3149
+ const matches = (value) => !query || value.toLowerCase().includes(query);
2935
3150
  const metrics = semanticLayer
2936
- ? semanticLayer.listMetrics().map((m) => m.name).sort()
3151
+ ? semanticLayer.listMetrics().map((m) => m.name).filter(matches).sort().slice(0, limit)
2937
3152
  : [];
2938
3153
  const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
2939
3154
  const blocks = Object.values(manifest.blocks)
2940
3155
  .filter((b) => b.status === 'certified')
2941
3156
  .map((b) => b.name)
3157
+ .filter(matches)
2942
3158
  .sort();
2943
3159
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
2944
- res.end(serializeJSON({ metrics, blocks }));
3160
+ res.end(serializeJSON({ metrics, blocks: blocks.slice(0, limit) }));
2945
3161
  }
2946
3162
  catch (error) {
2947
3163
  res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -3076,6 +3292,117 @@ export async function startLocalServer(opts) {
3076
3292
  }
3077
3293
  return;
3078
3294
  }
3295
+ // ── Domain + Skill bootstrap — evidence-first, session-only until save. ──
3296
+ if (req.method === 'POST' && path === '/api/context-bootstrap') {
3297
+ try {
3298
+ const body = (await readJSON(req).catch(() => ({})));
3299
+ const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
3300
+ const existingSkills = new Set(loadSkills(projectRoot).skills.map((skill) => skill.id));
3301
+ const candidates = draftDomainSkillBootstrap(manifest).map((candidate) => (candidate.kind === 'skill' && candidate.skill && existingSkills.has(candidate.skill.id)
3302
+ ? { ...candidate, action: 'unchanged', evidence: [...candidate.evidence, 'Existing skill preserved'] }
3303
+ : candidate));
3304
+ const session = writeContextBootstrapSession(projectRoot, {
3305
+ candidates,
3306
+ status: 'queued',
3307
+ ai: { requested: body.ai !== false, mode: 'pending' },
3308
+ progress: bootstrapProgress(candidates, new Set(), 4, 'Queued repository inventory for domain and skill drafts…', ['Draft session created.']),
3309
+ });
3310
+ void enrichContextBootstrapSession(projectRoot, session.id).catch((error) => {
3311
+ updateContextBootstrapSession(projectRoot, session.id, (current) => ({
3312
+ ...current,
3313
+ status: 'needs_attention',
3314
+ progress: bootstrapProgress(current.candidates, new Set(), 100, 'Repository draft is ready, but AI enrichment could not start.', [...current.progress.events, 'AI enrichment could not start.']),
3315
+ warnings: [...(current.warnings ?? []), `AI enrichment could not start: ${error instanceof Error ? error.message : String(error)}`],
3316
+ }));
3317
+ });
3318
+ res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
3319
+ res.end(serializeJSON(session));
3320
+ }
3321
+ catch (error) {
3322
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3323
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3324
+ }
3325
+ return;
3326
+ }
3327
+ if (req.method === 'GET' && path === '/api/context-bootstrap/latest') {
3328
+ try {
3329
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3330
+ res.end(serializeJSON({ session: latestOpenContextBootstrapSession(projectRoot) }));
3331
+ }
3332
+ catch (error) {
3333
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3334
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3335
+ }
3336
+ return;
3337
+ }
3338
+ if (req.method === 'GET' && path.startsWith('/api/context-bootstrap/')) {
3339
+ try {
3340
+ const id = decodeURIComponent(path.slice('/api/context-bootstrap/'.length));
3341
+ const session = readContextBootstrapSession(projectRoot, id);
3342
+ if (!session) {
3343
+ res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
3344
+ res.end(serializeJSON({ error: 'Bootstrap session not found.' }));
3345
+ return;
3346
+ }
3347
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3348
+ res.end(serializeJSON(session));
3349
+ }
3350
+ catch (error) {
3351
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3352
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3353
+ }
3354
+ return;
3355
+ }
3356
+ if (req.method === 'POST' && /^\/api\/context-bootstrap\/[^/]+\/save-selected$/.test(path)) {
3357
+ try {
3358
+ const match = path.match(/^\/api\/context-bootstrap\/([^/]+)\/save-selected$/);
3359
+ const id = decodeURIComponent(match?.[1] ?? '');
3360
+ const session = readContextBootstrapSession(projectRoot, id);
3361
+ if (!session)
3362
+ throw new Error('Bootstrap session not found.');
3363
+ if (!['ready', 'needs_attention'].includes(session.status)) {
3364
+ res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
3365
+ res.end(serializeJSON({ error: 'Wait for the repository draft to finish before saving selected candidates.' }));
3366
+ return;
3367
+ }
3368
+ const body = (await readJSON(req).catch(() => ({})));
3369
+ const selected = new Set(Array.isArray(body.candidateIds) ? body.candidateIds.filter((value) => typeof value === 'string') : []);
3370
+ const saved = [];
3371
+ for (const candidate of session.candidates) {
3372
+ if (!selected.has(candidate.id))
3373
+ continue;
3374
+ if (candidate.action === 'unchanged') {
3375
+ saved.push({ id: candidate.id, status: 'skipped' });
3376
+ continue;
3377
+ }
3378
+ try {
3379
+ if (candidate.kind === 'domain' && candidate.domain) {
3380
+ const written = writeDomainDeclaration(projectRoot, candidate.domain);
3381
+ saved.push({ id: candidate.id, path: written.path, status: 'saved' });
3382
+ }
3383
+ else if (candidate.kind === 'skill' && candidate.skill) {
3384
+ const skill = writeSkill(projectRoot, { ...candidate.skill, scope: 'project', status: 'draft' });
3385
+ saved.push({ id: candidate.id, path: relative(projectRoot, skill.sourcePath), status: 'saved' });
3386
+ }
3387
+ else {
3388
+ saved.push({ id: candidate.id, status: 'blocked', blockers: ['Candidate has no writable artifact.'] });
3389
+ }
3390
+ }
3391
+ catch (error) {
3392
+ saved.push({ id: candidate.id, status: 'blocked', blockers: [error instanceof Error ? error.message : String(error)] });
3393
+ }
3394
+ }
3395
+ await refreshLocalMetadataCatalog(projectRoot);
3396
+ updateContextBootstrapSession(projectRoot, id, (current) => ({ ...current, closedAt: new Date().toISOString() }));
3397
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
3398
+ res.end(serializeJSON({ id, saved }));
3399
+ }
3400
+ catch (error) {
3401
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
3402
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
3403
+ }
3404
+ return;
3405
+ }
3079
3406
  if (req.method === 'GET' && path === '/favicon.ico') {
3080
3407
  res.writeHead(200, {
3081
3408
  'Content-Type': 'image/svg+xml',
@@ -3456,6 +3783,14 @@ export async function startLocalServer(opts) {
3456
3783
  reviewHint(projectRoot, { hintId: hint.id, decision: 'approved', reviewer: author ?? 'local', note: 'Self-approved (OSS single-user).' });
3457
3784
  approvedHint = { ...hint, status: 'approved' };
3458
3785
  }
3786
+ // W4.3 — turn the correction into a durable regression eval case so the wrong
3787
+ // answer can never silently return. Best-effort; never blocks the correction.
3788
+ try {
3789
+ emitCorrectionEvalCase(projectRoot, { question, correctedSql });
3790
+ }
3791
+ catch {
3792
+ /* best-effort */
3793
+ }
3459
3794
  // Plain-language advisory memory mirroring the lesson, for transparency + recall.
3460
3795
  try {
3461
3796
  const memory = new MemoryStore(defaultMemoryPath(projectRoot));
@@ -4297,6 +4632,23 @@ export async function startLocalServer(opts) {
4297
4632
  res.end(serializeJSON(await readGitRemote(projectRoot)));
4298
4633
  return;
4299
4634
  }
4635
+ if (req.method === 'GET' && path === '/api/git/governed-context') {
4636
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
4637
+ res.end(serializeJSON(await readGitGovernedContext(projectRoot)));
4638
+ return;
4639
+ }
4640
+ if (req.method === 'POST' && path === '/api/git/governed-context/enable') {
4641
+ try {
4642
+ const result = await enableGitGovernedContextTracking(projectRoot);
4643
+ res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
4644
+ res.end(serializeJSON(result));
4645
+ }
4646
+ catch (e) {
4647
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
4648
+ res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
4649
+ }
4650
+ return;
4651
+ }
4300
4652
  if (req.method === 'POST' && path === '/api/git/stage') {
4301
4653
  try {
4302
4654
  const body = (await readJSON(req));
@@ -4361,6 +4713,24 @@ export async function startLocalServer(opts) {
4361
4713
  }
4362
4714
  return;
4363
4715
  }
4716
+ if (req.method === 'POST' && path === '/api/git/review') {
4717
+ try {
4718
+ const body = (await readJSON(req));
4719
+ const result = await gitCreateReview(projectRoot, {
4720
+ paths: body.paths ?? [],
4721
+ title: body.title ?? '',
4722
+ body: body.body ?? '',
4723
+ base: body.base ?? 'main',
4724
+ });
4725
+ res.writeHead(result.ok ? 201 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
4726
+ res.end(serializeJSON(result));
4727
+ }
4728
+ catch (e) {
4729
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
4730
+ res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
4731
+ }
4732
+ return;
4733
+ }
4364
4734
  if (req.method === 'POST' && path === '/api/git/pull') {
4365
4735
  try {
4366
4736
  const result = await gitPull(projectRoot);
@@ -4471,13 +4841,13 @@ export async function startLocalServer(opts) {
4471
4841
  res.end(serializeJSON({ error: 'name and content are required' }));
4472
4842
  return;
4473
4843
  }
4844
+ // OSS keeps the save flow deliberately small: the person saving the
4845
+ // reusable answer is accountable for it. Domain/description/tags are
4846
+ // helpful defaults, but not reasons to send a single-user back to a
4847
+ // review queue.
4474
4848
  const missing = [];
4475
4849
  if (!owner || !owner.trim())
4476
4850
  missing.push('owner');
4477
- if (!domain || !domain.trim())
4478
- missing.push('domain');
4479
- if (!description || !description.trim())
4480
- missing.push('description');
4481
4851
  if (missing.length > 0) {
4482
4852
  res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
4483
4853
  res.end(serializeJSON({
@@ -4500,8 +4870,34 @@ export async function startLocalServer(opts) {
4500
4870
  invariants,
4501
4871
  gitMetadata: readGitMetadata(projectRoot),
4502
4872
  });
4873
+ // Save first, then certify against the same local runtime used for the
4874
+ // answer. Passing blocks become certified immediately; anything that
4875
+ // cannot prove itself remains a clearly labelled draft at this path.
4876
+ setBlockStudioStatus(projectRoot, created.path, 'draft');
4877
+ let source = readFileSync(join(projectRoot, created.path), 'utf-8');
4878
+ let status = 'draft';
4879
+ let blockers = [];
4880
+ let certification;
4881
+ try {
4882
+ const verdict = await certifyBlockStudioSource(source, created.path);
4883
+ blockers = Array.from(new Set(verdict.checklist.blockers));
4884
+ certification = {
4885
+ certified: verdict.certification.certified && blockers.length === 0,
4886
+ errors: verdict.certification.errors,
4887
+ warnings: verdict.certification.warnings,
4888
+ };
4889
+ if (certification.certified) {
4890
+ setBlockStudioStatus(projectRoot, created.path, 'certified');
4891
+ status = 'certified';
4892
+ source = readFileSync(join(projectRoot, created.path), 'utf-8');
4893
+ }
4894
+ }
4895
+ catch (error) {
4896
+ blockers = [error instanceof Error ? error.message : String(error)];
4897
+ }
4898
+ await refreshLocalMetadataCatalog(projectRoot);
4503
4899
  res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
4504
- res.end(serializeJSON(created));
4900
+ res.end(serializeJSON({ ...created, content: source, status, blockers, certification }));
4505
4901
  }
4506
4902
  catch (error) {
4507
4903
  if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
@@ -4540,6 +4936,11 @@ export async function startLocalServer(opts) {
4540
4936
  const stat = statSync(filePath);
4541
4937
  // Quick regex parse for key block fields
4542
4938
  const nameMatch = /block\s+"([^"]+)"/.exec(source);
4939
+ // `domains/**/domain.dql` is a business-boundary declaration,
4940
+ // not a reusable block. Only a real block declaration belongs
4941
+ // in the Blocks library.
4942
+ if (!nameMatch)
4943
+ continue;
4543
4944
  const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
4544
4945
  const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
4545
4946
  const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
@@ -5029,6 +5430,115 @@ export async function startLocalServer(opts) {
5029
5430
  }
5030
5431
  return;
5031
5432
  }
5433
+ const aiImportSaveSelectedMatch = path.match(/^\/api\/block-studio\/ai-imports\/([^/]+)\/save-selected$/);
5434
+ if (req.method === 'POST' && aiImportSaveSelectedMatch) {
5435
+ const importId = decodeURIComponent(aiImportSaveSelectedMatch[1]);
5436
+ try {
5437
+ const body = await readJSON(req);
5438
+ const candidateIds = Array.from(new Set(Array.isArray(body.candidateIds) ? body.candidateIds.map((value) => String(value)) : []));
5439
+ const owner = typeof body.owner === 'string' ? body.owner.trim() : '';
5440
+ if (candidateIds.length === 0 || !owner) {
5441
+ res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
5442
+ res.end(serializeJSON({ error: 'Select at least one candidate and provide an owner.' }));
5443
+ return;
5444
+ }
5445
+ const results = [];
5446
+ for (const candidateId of candidateIds) {
5447
+ try {
5448
+ let candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
5449
+ if (candidate.reviewStatus === 'saved' && candidate.savedPath && existsSync(join(projectRoot, candidate.savedPath))) {
5450
+ results.push({
5451
+ candidateId,
5452
+ path: candidate.savedPath,
5453
+ status: 'certified',
5454
+ blockers: [],
5455
+ candidate: candidate,
5456
+ block: openBlockStudioDocument(projectRoot, candidate.savedPath, semanticLayer),
5457
+ });
5458
+ continue;
5459
+ }
5460
+ if (candidate.recommendedAction === 'reuse_existing') {
5461
+ results.push({
5462
+ candidateId,
5463
+ status: 'error',
5464
+ blockers: ['An equivalent governed block already exists. Open and reuse it instead of saving a duplicate.'],
5465
+ candidate: candidate,
5466
+ });
5467
+ continue;
5468
+ }
5469
+ candidate = updateBlockStudioImportCandidate(projectRoot, importId, candidateId, { owner });
5470
+ const readiness = validateImportCandidateForSave(candidate);
5471
+ const validationBlockers = readiness.errors;
5472
+ if (validationBlockers.length === 0) {
5473
+ const certifiedSource = setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'certified');
5474
+ const certification = await certifyBlockStudioSource(certifiedSource, readiness.candidate.savedPath, { enterprise: false });
5475
+ const blockers = Array.from(new Set(certification.checklist.blockers));
5476
+ if (certification.certification.certified && blockers.length === 0) {
5477
+ const savedPath = saveBlockStudioArtifacts(projectRoot, {
5478
+ currentPath: readiness.candidate.savedPath,
5479
+ source: certifiedSource,
5480
+ name: readiness.candidate.name,
5481
+ domain: readiness.candidate.domain,
5482
+ description: readiness.candidate.description,
5483
+ owner,
5484
+ tags: readiness.candidate.tags,
5485
+ lineage: readiness.candidate.lineage.sourceTables,
5486
+ importMeta: { importId, candidateId, sourceKind: readiness.candidate.sourceKind, sourcePath: readiness.candidate.sourcePath },
5487
+ });
5488
+ const savedCandidate = {
5489
+ ...readiness.candidate,
5490
+ dqlSource: certifiedSource,
5491
+ reviewStatus: 'saved',
5492
+ savedPath,
5493
+ analysisStatus: 'ready',
5494
+ validation: validateBlockStudioSource(certifiedSource, semanticLayer),
5495
+ generationMode: readiness.candidate.generationMode ?? 'deterministic',
5496
+ generationProvider: readiness.candidate.generationProvider ?? 'local-deterministic',
5497
+ llmContext: readiness.candidate.llmContext ?? deterministicDqlGenerationContext(readiness.candidate, readiness.candidate.evidence ?? []),
5498
+ evidence: readiness.candidate.evidence ?? [],
5499
+ draftSave: { status: 'saved', path: savedPath, savedAt: new Date().toISOString() },
5500
+ };
5501
+ writeBlockStudioImportCandidate(projectRoot, importId, savedCandidate);
5502
+ results.push({ candidateId, path: savedPath, status: 'certified', blockers: [], candidate: savedCandidate, block: openBlockStudioDocument(projectRoot, savedPath, semanticLayer) });
5503
+ continue;
5504
+ }
5505
+ validationBlockers.push(...blockers);
5506
+ }
5507
+ const draftCandidate = saveDqlGenerationDraft(importId, {
5508
+ ...readiness.candidate,
5509
+ dqlSource: setBlockStudioSourceStatus(readiness.candidate.dqlSource, 'draft'),
5510
+ analysisStatus: 'needs_attention',
5511
+ });
5512
+ writeBlockStudioImportCandidate(projectRoot, importId, draftCandidate);
5513
+ results.push({
5514
+ candidateId,
5515
+ path: draftCandidate.savedPath,
5516
+ status: 'draft',
5517
+ blockers: Array.from(new Set(validationBlockers.length > 0 ? validationBlockers : ['Certification gates did not pass.'])),
5518
+ candidate: draftCandidate,
5519
+ block: draftCandidate.savedPath ? openBlockStudioDocument(projectRoot, draftCandidate.savedPath, semanticLayer) : undefined,
5520
+ });
5521
+ }
5522
+ catch (error) {
5523
+ results.push({
5524
+ candidateId,
5525
+ status: 'error',
5526
+ blockers: [],
5527
+ error: error instanceof Error ? error.message : String(error),
5528
+ });
5529
+ }
5530
+ }
5531
+ await refreshLocalMetadataCatalog(projectRoot);
5532
+ const nextSession = loadDqlGenerationSession(importId);
5533
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
5534
+ res.end(serializeJSON({ ok: results.every((result) => result.status !== 'error'), session: nextSession, results }));
5535
+ }
5536
+ catch (error) {
5537
+ res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
5538
+ res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
5539
+ }
5540
+ return;
5541
+ }
5032
5542
  const aiImportPathMatch = path.match(/^\/api\/block-studio\/ai-imports\/([^/]+)(?:\/candidates\/([^/]+)(?:\/(preview|certify))?)?$/);
5033
5543
  if (aiImportPathMatch) {
5034
5544
  const importId = decodeURIComponent(aiImportPathMatch[1]);
@@ -5065,9 +5575,20 @@ export async function startLocalServer(opts) {
5065
5575
  llmContext: typeof body.llmContext === 'string' ? body.llmContext : undefined,
5066
5576
  });
5067
5577
  const validated = validateImportCandidate(candidate);
5068
- const savedDraft = saveDqlGenerationDraft(importId, validated);
5578
+ const savedDraft = validated.draftSave?.status === 'pending' && !isDraftBlockPath(validated.savedPath)
5579
+ ? {
5580
+ ...validated,
5581
+ analysisStatus: 'ready',
5582
+ generationMode: validated.generationMode ?? 'deterministic',
5583
+ generationProvider: validated.generationProvider ?? 'local-deterministic',
5584
+ llmContext: validated.llmContext ?? deterministicDqlGenerationContext(validated, validated.evidence ?? []),
5585
+ evidence: validated.evidence ?? [],
5586
+ draftSave: { status: 'pending' },
5587
+ }
5588
+ : saveDqlGenerationDraft(importId, validated);
5069
5589
  writeBlockStudioImportCandidate(projectRoot, importId, savedDraft);
5070
- await refreshLocalMetadataCatalog(projectRoot);
5590
+ if (savedDraft.draftSave.status === 'saved')
5591
+ await refreshLocalMetadataCatalog(projectRoot);
5071
5592
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
5072
5593
  res.end(serializeJSON(savedDraft));
5073
5594
  return;
@@ -5075,11 +5596,22 @@ export async function startLocalServer(opts) {
5075
5596
  if (req.method === 'POST' && candidateId && action === 'preview') {
5076
5597
  const candidate = readBlockStudioImportCandidate(projectRoot, importId, candidateId);
5077
5598
  const preview = await runBlockStudioPreviewSource(candidate.dqlSource);
5078
- const next = saveDqlGenerationDraft(importId, {
5599
+ const previewed = {
5079
5600
  ...candidate,
5080
5601
  preview,
5602
+ analysisStatus: 'ready',
5081
5603
  validation: validateBlockStudioSource(candidate.dqlSource, semanticLayer),
5082
- });
5604
+ };
5605
+ const next = candidate.draftSave?.status === 'pending' && !isDraftBlockPath(candidate.savedPath)
5606
+ ? {
5607
+ ...previewed,
5608
+ generationMode: candidate.generationMode ?? 'deterministic',
5609
+ generationProvider: candidate.generationProvider ?? 'local-deterministic',
5610
+ llmContext: candidate.llmContext ?? deterministicDqlGenerationContext(candidate, candidate.evidence ?? []),
5611
+ evidence: candidate.evidence ?? [],
5612
+ draftSave: { status: 'pending' },
5613
+ }
5614
+ : saveDqlGenerationDraft(importId, previewed);
5083
5615
  writeBlockStudioImportCandidate(projectRoot, importId, next);
5084
5616
  res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
5085
5617
  res.end(serializeJSON(next));
@@ -6510,7 +7042,7 @@ export async function startLocalServer(opts) {
6510
7042
  // Build table mapping: resolve semantic model names to actual DB table names
6511
7043
  let tableMapping;
6512
7044
  try {
6513
- const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
7045
+ const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
6514
7046
  const dbTableNames = new Set();
6515
7047
  const schemaQualified = new Map();
6516
7048
  for (const row of tablesResult.rows) {
@@ -6601,7 +7133,7 @@ export async function startLocalServer(opts) {
6601
7133
  const driver = targetConnection.driver;
6602
7134
  let tableMapping;
6603
7135
  try {
6604
- const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
7136
+ const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
6605
7137
  const schemaQualified = new Map();
6606
7138
  for (const row of tablesResult.rows) {
6607
7139
  const schema = String(row['table_schema'] ?? '');
@@ -7474,13 +8006,24 @@ export function serializeJSON(value) {
7474
8006
  function serializeSkill(skill) {
7475
8007
  return {
7476
8008
  id: skill.id,
7477
- scope: skill.scope,
7478
- user: skill.user,
8009
+ scope: 'project',
8010
+ user: undefined,
7479
8011
  domain: skill.domain,
8012
+ domains: skill.domains,
8013
+ kind: skill.kind,
8014
+ status: skill.status,
8015
+ owner: skill.owner,
8016
+ triggers: skill.triggers,
8017
+ exclusions: skill.exclusions,
7480
8018
  description: skill.description,
7481
8019
  body: skill.body,
7482
8020
  preferredMetrics: skill.preferredMetrics,
7483
8021
  preferredBlocks: skill.preferredBlocks,
8022
+ preferredDimensions: skill.preferredDimensions,
8023
+ requiredFilters: skill.requiredFilters,
8024
+ clarifyWhen: skill.clarifyWhen,
8025
+ examples: skill.examples,
8026
+ sourceRefs: skill.sourceRefs,
7484
8027
  vocabulary: skill.vocabulary,
7485
8028
  sourcePath: skill.sourcePath,
7486
8029
  isStarter: skill.isStarter,
@@ -7498,7 +8041,7 @@ function parseSkillInput(raw, fallbackId) {
7498
8041
  const id = typeof skill.id === 'string' && skill.id.trim() ? skill.id.trim() : fallbackId;
7499
8042
  if (!id)
7500
8043
  return null;
7501
- const scope = skill.scope === 'personal' ? 'personal' : skill.scope === 'project' ? 'project' : undefined;
8044
+ const scope = skill.scope === 'project' ? 'project' : undefined;
7502
8045
  if (!scope)
7503
8046
  return null;
7504
8047
  if (typeof skill.body !== 'string')
@@ -7516,13 +8059,24 @@ function parseSkillInput(raw, fallbackId) {
7516
8059
  };
7517
8060
  return {
7518
8061
  id,
7519
- scope,
7520
- user: scope === 'personal' && typeof skill.user === 'string' ? skill.user : undefined,
8062
+ scope: 'project',
8063
+ user: undefined,
7521
8064
  domain: typeof skill.domain === 'string' && skill.domain.trim() ? skill.domain.trim() : undefined,
8065
+ domains: asStrings(skill.domains),
8066
+ kind: skill.kind === 'domain_reference' || skill.kind === 'metric_policy' || skill.kind === 'glossary' || skill.kind === 'analysis_pattern' || skill.kind === 'sql_policy' ? skill.kind : 'custom',
8067
+ status: skill.status === 'draft' || skill.status === 'deprecated' ? skill.status : 'active',
8068
+ owner: typeof skill.owner === 'string' ? skill.owner : undefined,
8069
+ triggers: asStrings(skill.triggers),
8070
+ exclusions: asStrings(skill.exclusions),
7522
8071
  description: typeof skill.description === 'string' ? skill.description : undefined,
7523
8072
  body: skill.body,
7524
8073
  preferredMetrics: asStrings(skill.preferredMetrics),
7525
8074
  preferredBlocks: asStrings(skill.preferredBlocks),
8075
+ preferredDimensions: asStrings(skill.preferredDimensions),
8076
+ requiredFilters: asStrings(skill.requiredFilters),
8077
+ clarifyWhen: asStrings(skill.clarifyWhen),
8078
+ examples: asStrings(skill.examples),
8079
+ sourceRefs: asStrings(skill.sourceRefs),
7526
8080
  vocabulary: asMap(skill.vocabulary),
7527
8081
  isStarter: skill.isStarter === true ? true : undefined,
7528
8082
  };
@@ -7537,10 +8091,23 @@ function manifestDomainToDto(domain, counts) {
7537
8091
  return {
7538
8092
  id: domain.name,
7539
8093
  name: domain.name,
8094
+ parent: domain.parent,
7540
8095
  owner: domain.owner,
8096
+ businessOwner: domain.businessOwner,
7541
8097
  boundedContext: domain.boundedContext,
7542
8098
  sourceSystems: domain.sourceSystems,
7543
8099
  description: domain.description,
8100
+ primaryTerms: domain.primaryTerms,
8101
+ tags: domain.tags,
8102
+ businessOutcome: domain.businessOutcome,
8103
+ reviewCadence: domain.reviewCadence,
8104
+ inScope: domain.inScope,
8105
+ outOfScope: domain.outOfScope,
8106
+ dbtGroups: domain.dbtGroups,
8107
+ dbtPaths: domain.dbtPaths,
8108
+ dbtTags: domain.dbtTags,
8109
+ semanticDomains: domain.semanticDomains,
8110
+ semanticTags: domain.semanticTags,
7544
8111
  sourcePath: domain.filePath,
7545
8112
  blockCount: counts.blockCount,
7546
8113
  skillCount: counts.skillCount,
@@ -7603,11 +8170,168 @@ export function parseDomainInput(raw, fallbackId) {
7603
8170
  : undefined;
7604
8171
  return {
7605
8172
  name,
8173
+ parent: typeof domain.parent === 'string' ? domain.parent : undefined,
7606
8174
  owner: typeof domain.owner === 'string' ? domain.owner : undefined,
8175
+ businessOwner: typeof domain.businessOwner === 'string' ? domain.businessOwner : undefined,
7607
8176
  boundedContext: typeof domain.boundedContext === 'string' ? domain.boundedContext : undefined,
7608
8177
  sourceSystems: asStrings(domain.sourceSystems),
7609
8178
  description: typeof domain.description === 'string' ? domain.description : undefined,
8179
+ primaryTerms: asStrings(domain.primaryTerms),
8180
+ tags: asStrings(domain.tags),
8181
+ businessOutcome: typeof domain.businessOutcome === 'string' ? domain.businessOutcome : undefined,
8182
+ reviewCadence: typeof domain.reviewCadence === 'string' ? domain.reviewCadence : undefined,
8183
+ inScope: asStrings(domain.inScope),
8184
+ outOfScope: asStrings(domain.outOfScope),
8185
+ dbtGroups: asStrings(domain.dbtGroups),
8186
+ dbtPaths: asStrings(domain.dbtPaths),
8187
+ dbtTags: asStrings(domain.dbtTags),
8188
+ semanticDomains: asStrings(domain.semanticDomains),
8189
+ semanticTags: asStrings(domain.semanticTags),
8190
+ sourcePath: typeof domain.sourcePath === 'string' ? domain.sourcePath : undefined,
8191
+ };
8192
+ }
8193
+ function contextBootstrapDir(projectRoot) {
8194
+ return join(projectRoot, '.dql', 'local', 'context-bootstrap');
8195
+ }
8196
+ function contextBootstrapPath(projectRoot, id) {
8197
+ const safe = id.replace(/[^a-zA-Z0-9_-]/g, '');
8198
+ return join(contextBootstrapDir(projectRoot), `${safe}.json`);
8199
+ }
8200
+ function writeContextBootstrapSession(projectRoot, input) {
8201
+ const id = `ctx_${Date.now().toString(36)}_${createHash('sha1').update(JSON.stringify(input.candidates)).digest('hex').slice(0, 8)}`;
8202
+ const session = { id, createdAt: new Date().toISOString(), persistence: 'session-only', ...input };
8203
+ const dir = contextBootstrapDir(projectRoot);
8204
+ if (!existsSync(dir))
8205
+ mkdirSync(dir, { recursive: true });
8206
+ writeFileSync(contextBootstrapPath(projectRoot, id), JSON.stringify(session, null, 2), 'utf-8');
8207
+ return session;
8208
+ }
8209
+ function updateContextBootstrapSession(projectRoot, id, update) {
8210
+ const current = readContextBootstrapSession(projectRoot, id);
8211
+ if (!current)
8212
+ return null;
8213
+ const next = update(current);
8214
+ writeFileSync(contextBootstrapPath(projectRoot, id), JSON.stringify(next, null, 2), 'utf-8');
8215
+ return next;
8216
+ }
8217
+ function bootstrapProgress(candidates, readyCandidateIds, percent, message, events) {
8218
+ const count = (kind) => {
8219
+ const all = candidates.filter((candidate) => candidate.kind === kind);
8220
+ return { total: all.length, ready: all.filter((candidate) => readyCandidateIds.has(candidate.id)).length };
7610
8221
  };
8222
+ return { percent, message, domains: count('domain'), skills: count('skill'), events: events.slice(-6) };
8223
+ }
8224
+ async function enrichContextBootstrapSession(projectRoot, id) {
8225
+ // One compiled snapshot serves the entire session; do not re-search a large
8226
+ // repository for every domain/skill candidate.
8227
+ const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook' });
8228
+ let session = updateContextBootstrapSession(projectRoot, id, (current) => ({
8229
+ ...current,
8230
+ status: 'inventory',
8231
+ progress: bootstrapProgress(current.candidates, new Set(), 12, 'Repository inventory is ready. Checking the configured AI provider…', ['Repository inventory complete.']),
8232
+ }));
8233
+ if (!session)
8234
+ return;
8235
+ const provider = session.ai.requested ? await createBlockStudioAssistProvider(projectRoot) : null;
8236
+ if (!provider) {
8237
+ updateContextBootstrapSession(projectRoot, id, (current) => ({
8238
+ ...current,
8239
+ status: 'ready',
8240
+ ai: { ...current.ai, mode: 'evidence_only' },
8241
+ progress: bootstrapProgress(current.candidates, new Set(current.candidates.map((candidate) => candidate.id)), 100, 'Evidence-first draft is ready. Configure an AI provider to enrich its prose and guidance.', ['Repository inventory complete.', 'No configured AI provider was available; retained evidence-only drafts.']),
8242
+ warnings: [...(current.warnings ?? []), 'No configured AI provider was available. Candidates were drafted from repository evidence only.'],
8243
+ }));
8244
+ return;
8245
+ }
8246
+ session = updateContextBootstrapSession(projectRoot, id, (current) => ({
8247
+ ...current,
8248
+ status: 'grounding',
8249
+ ai: { ...current.ai, mode: 'provider', provider: provider.name },
8250
+ progress: bootstrapProgress(current.candidates, new Set(), 25, `Grounding ${current.candidates.length} candidate(s) in governed repository metadata…`, ['Repository inventory complete.', `Using ${provider.name} to enrich bounded guidance.`]),
8251
+ }));
8252
+ if (!session)
8253
+ return;
8254
+ const chunks = [];
8255
+ for (let index = 0; index < session.candidates.length; index += 4)
8256
+ chunks.push(session.candidates.slice(index, index + 4));
8257
+ const complete = new Set();
8258
+ const warnings = [];
8259
+ for (let index = 0; index < chunks.length; index += 1) {
8260
+ const before = readContextBootstrapSession(projectRoot, id);
8261
+ if (!before)
8262
+ return;
8263
+ updateContextBootstrapSession(projectRoot, id, (current) => ({
8264
+ ...current,
8265
+ status: 'generating',
8266
+ progress: bootstrapProgress(current.candidates, complete, 25 + Math.floor((index / Math.max(chunks.length, 1)) * 60), `Writing grounded guidance for ${index * 4 + 1}–${Math.min((index + 1) * 4, current.candidates.length)} of ${current.candidates.length} candidates…`, [...current.progress.events, `AI enrichment batch ${index + 1} of ${chunks.length}.`]),
8267
+ }));
8268
+ const prompt = buildDomainSkillBootstrapPrompt(manifest, chunks[index]);
8269
+ try {
8270
+ const response = await provider.generate([
8271
+ { role: 'system', content: prompt.system },
8272
+ { role: 'user', content: prompt.user },
8273
+ ], { maxTokens: 1800, temperature: 0.1 });
8274
+ updateContextBootstrapSession(projectRoot, id, (current) => {
8275
+ const candidateById = new Map(current.candidates.map((candidate) => [candidate.id, candidate]));
8276
+ const currentChunk = chunks[index].map((candidate) => candidateById.get(candidate.id) ?? candidate);
8277
+ const merged = mergeDomainSkillBootstrapEnrichment(currentChunk, response);
8278
+ for (const candidate of merged.candidates)
8279
+ candidateById.set(candidate.id, candidate);
8280
+ for (const candidate of chunks[index])
8281
+ complete.add(candidate.id);
8282
+ warnings.push(...merged.rejected);
8283
+ const candidates = current.candidates.map((candidate) => candidateById.get(candidate.id) ?? candidate);
8284
+ return {
8285
+ ...current,
8286
+ candidates,
8287
+ progress: bootstrapProgress(candidates, complete, 25 + Math.floor(((index + 1) / Math.max(chunks.length, 1)) * 60), `Grounded guidance ready for ${complete.size} of ${candidates.length} candidates.`, [...current.progress.events, `AI enrichment batch ${index + 1} complete.`]),
8288
+ };
8289
+ });
8290
+ }
8291
+ catch (error) {
8292
+ for (const candidate of chunks[index])
8293
+ complete.add(candidate.id);
8294
+ warnings.push(`AI enrichment batch ${index + 1} failed: ${error instanceof Error ? error.message : String(error)}`);
8295
+ }
8296
+ }
8297
+ updateContextBootstrapSession(projectRoot, id, (current) => ({
8298
+ ...current,
8299
+ status: warnings.length ? 'needs_attention' : 'ready',
8300
+ progress: bootstrapProgress(current.candidates, new Set(current.candidates.map((candidate) => candidate.id)), 100, warnings.length ? 'Drafts are ready; review the noted AI enrichment gaps before saving.' : 'Grounded domain and skill drafts are ready for review.', [...current.progress.events, warnings.length ? 'Some AI enrichment needs attention.' : 'Validation complete.']),
8301
+ warnings: warnings.length ? [...(current.warnings ?? []), ...warnings] : current.warnings,
8302
+ }));
8303
+ }
8304
+ function readContextBootstrapSession(projectRoot, id) {
8305
+ const path = contextBootstrapPath(projectRoot, id);
8306
+ if (!existsSync(path))
8307
+ return null;
8308
+ try {
8309
+ const parsed = JSON.parse(readFileSync(path, 'utf-8'));
8310
+ return parsed?.id === id && Array.isArray(parsed.candidates) ? parsed : null;
8311
+ }
8312
+ catch {
8313
+ return null;
8314
+ }
8315
+ }
8316
+ function latestOpenContextBootstrapSession(projectRoot) {
8317
+ const dir = contextBootstrapDir(projectRoot);
8318
+ if (!existsSync(dir))
8319
+ return null;
8320
+ const paths = readdirSync(dir)
8321
+ .filter((name) => name.endsWith('.json'))
8322
+ .map((name) => join(dir, name))
8323
+ .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs);
8324
+ for (const path of paths) {
8325
+ try {
8326
+ const session = JSON.parse(readFileSync(path, 'utf-8'));
8327
+ if (session?.id && Array.isArray(session.candidates) && !session.closedAt)
8328
+ return session;
8329
+ }
8330
+ catch {
8331
+ // A truncated local session should not hide an earlier valid session.
8332
+ }
8333
+ }
8334
+ return null;
7611
8335
  }
7612
8336
  async function refreshLocalMetadataCatalog(projectRoot) {
7613
8337
  try {
@@ -7929,6 +8653,7 @@ export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
7929
8653
  minSupport: Math.max(2, Math.floor(options.minSupport ?? 2)),
7930
8654
  },
7931
8655
  candidates: [],
8656
+ joinCandidates: [],
7932
8657
  prBody: '',
7933
8658
  };
7934
8659
  }
@@ -7995,6 +8720,8 @@ export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
7995
8720
  .filter((candidate) => Boolean(candidate))
7996
8721
  .slice(0, limit);
7997
8722
  const donorBlocksUsed = new Set(candidates.flatMap((candidate) => candidate.donorBlocks.map((donor) => donor.path))).size;
8723
+ // Composting v2 (W4.4) — recurring join shapes across certified blocks.
8724
+ const joinCandidates = mineJoinPatterns(certifiedBlocks.map((block) => ({ name: block.name, sql: block.sql })), minSupport).slice(0, limit);
7998
8725
  return {
7999
8726
  ready: true,
8000
8727
  summary: {
@@ -8006,6 +8733,7 @@ export function buildSemanticCompostingChangeset(projectRoot, options = {}) {
8006
8733
  minSupport,
8007
8734
  },
8008
8735
  candidates,
8736
+ joinCandidates,
8009
8737
  prBody: renderCompostingPrBody(candidates),
8010
8738
  };
8011
8739
  }
@@ -8513,6 +9241,37 @@ function installConnectorPackage(projectRoot, driver) {
8513
9241
  });
8514
9242
  return getConnectorInstallStatuses(projectRoot).find((status) => status.driver === definition.driver);
8515
9243
  }
9244
+ /**
9245
+ * Ensure the active connection's driver package is present at startup.
9246
+ *
9247
+ * The connection CONFIG persists in dql.config.json, but the driver package
9248
+ * (duckdb, snowflake-sdk, …) lives in the gitignored, per-project .dql/connectors
9249
+ * — so a fresh clone, a `npm i -g` upgrade of the CLI, or even a Node version bump
9250
+ * (native bindings are Node-version-specific) leaves a configured connection with
9251
+ * no loadable driver, and the user has to reinstall it by hand from the Connections
9252
+ * page. This installs it once, at boot, so connections just work after any
9253
+ * install/upgrade. Best-effort and non-fatal: an offline machine keeps the manual
9254
+ * "Install" button and a clear message rather than a failed startup.
9255
+ */
9256
+ export function ensureConnectorInstalledForStartup(projectRoot, driver) {
9257
+ if (!driver)
9258
+ return;
9259
+ const definition = CONNECTOR_INSTALLS[driver];
9260
+ // Built-in drivers (e.g. the local file driver) and unknown drivers need nothing.
9261
+ if (!definition || definition.builtIn || !definition.packageSpec || !definition.packageName)
9262
+ return;
9263
+ if (isConnectorPackageInstalled(projectRoot, definition.packageName))
9264
+ return;
9265
+ console.log(`[dql] Installing the ${definition.label} driver into .dql/connectors (first run after install/upgrade)…`);
9266
+ try {
9267
+ installConnectorPackage(projectRoot, driver);
9268
+ console.log(`[dql] ${definition.label} driver ready — connection restored without manual setup.`);
9269
+ }
9270
+ catch (err) {
9271
+ const detail = err instanceof Error ? err.message : String(err);
9272
+ console.log(`[dql] Could not auto-install the ${definition.label} driver (${detail}). Open the Connections page and click "Install" once you have network access.`);
9273
+ }
9274
+ }
8516
9275
  function getStoredConnections(raw) {
8517
9276
  const value = raw.connections;
8518
9277
  if (!value || typeof value !== 'object' || Array.isArray(value)) {
@@ -9313,7 +10072,7 @@ async function introspectSchema(executor, connection) {
9313
10072
  try {
9314
10073
  const catalogRows = await executor.executeQuery(`SELECT table_schema, table_name, table_type
9315
10074
  FROM information_schema.tables
9316
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
10075
+ WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
9317
10076
  ORDER BY table_schema, table_name`, [], {}, connection);
9318
10077
  tables = catalogRows.rows.map((row) => {
9319
10078
  const schema = String(row['table_schema'] ?? row['TABLE_SCHEMA'] ?? 'default');
@@ -9324,7 +10083,7 @@ async function introspectSchema(executor, connection) {
9324
10083
  });
9325
10084
  const columnRows = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
9326
10085
  FROM information_schema.columns
9327
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
10086
+ WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
9328
10087
  ORDER BY table_schema, table_name, ordinal_position`, [], {}, connection);
9329
10088
  columnsByPath = columnRows.rows.reduce((map, row) => {
9330
10089
  const schema = String(row['table_schema'] ?? row['TABLE_SCHEMA'] ?? 'default');
@@ -9530,7 +10289,7 @@ export async function resolveSemanticTableMapping(executor, connection, semantic
9530
10289
  try {
9531
10290
  const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
9532
10291
  FROM information_schema.tables
9533
- WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
10292
+ WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
9534
10293
  ORDER BY table_schema, table_name
9535
10294
  LIMIT 2000`, [], {}, connection);
9536
10295
  return buildSemanticTableMapping(semanticLayer, tablesResult.rows);
@@ -9942,9 +10701,6 @@ export async function createDqlArtifactGenerationSessionForProject(projectRoot,
9942
10701
  return session;
9943
10702
  }
9944
10703
  export async function createDqlGenerationSessionForProject(projectRoot, options, semanticLayer) {
9945
- const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
9946
- const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
9947
- const provider = deterministicOnly ? null : await createBlockStudioAssistProvider(projectRoot, requestedProvider);
9948
10704
  const session = createBlockStudioImportSession(projectRoot, {
9949
10705
  inputPath: options.inputPath ?? '',
9950
10706
  inputMode: options.inputMode,
@@ -9954,77 +10710,183 @@ export async function createDqlGenerationSessionForProject(projectRoot, options,
9954
10710
  owner: options.owner,
9955
10711
  tags: options.tags,
9956
10712
  });
10713
+ const persistence = options.persistence ?? 'draft-files';
10714
+ if (options.async) {
10715
+ const queuedCandidates = session.candidates.map((candidate) => ({
10716
+ ...candidate,
10717
+ analysisStatus: 'queued',
10718
+ generationMode: 'deterministic',
10719
+ generationProvider: 'pending',
10720
+ llmContext: '',
10721
+ evidence: [],
10722
+ draftSave: { status: 'pending' },
10723
+ }));
10724
+ const queuedSession = {
10725
+ ...session,
10726
+ mode: 'ai-import',
10727
+ persistence,
10728
+ candidates: queuedCandidates,
10729
+ generation: {
10730
+ provider: 'pending',
10731
+ aiEnabled: false,
10732
+ contextObjectCount: 0,
10733
+ createdDrafts: 0,
10734
+ warnings: [],
10735
+ },
10736
+ };
10737
+ writeBlockStudioImportSession(projectRoot, queuedSession);
10738
+ void processDqlGenerationSessionForProject(projectRoot, session, { ...options, async: false, persistence }, semanticLayer)
10739
+ .catch((error) => {
10740
+ const message = error instanceof Error ? error.message : String(error);
10741
+ for (const candidate of queuedCandidates) {
10742
+ const current = readBlockStudioImportCandidate(projectRoot, session.id, candidate.id);
10743
+ if (current.analysisStatus === 'ready')
10744
+ continue;
10745
+ writeBlockStudioImportCandidate(projectRoot, session.id, {
10746
+ ...current,
10747
+ analysisStatus: 'needs_attention',
10748
+ warnings: Array.from(new Set([...(current.warnings ?? []), message])),
10749
+ });
10750
+ }
10751
+ });
10752
+ return queuedSession;
10753
+ }
10754
+ return processDqlGenerationSessionForProject(projectRoot, session, { ...options, persistence }, semanticLayer);
10755
+ }
10756
+ async function processDqlGenerationSessionForProject(projectRoot, session, options, semanticLayer) {
10757
+ const deterministicOnly = isDeterministicDqlGenerationProvider(options.provider);
10758
+ const requestedProvider = !deterministicOnly && isProviderSettingsId(options.provider) ? options.provider : undefined;
10759
+ const provider = deterministicOnly ? null : await createBlockStudioAssistProvider(projectRoot, requestedProvider);
10760
+ const persistence = options.persistence ?? 'draft-files';
9957
10761
  const warnings = [];
9958
- const nextCandidates = [];
9959
- let contextObjectCount = 0;
9960
10762
  for (const candidate of session.candidates) {
9961
- const contextPack = await buildDqlGenerationContextPack(projectRoot, candidate).catch((error) => {
9962
- warnings.push(`Context pack failed for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
9963
- return null;
10763
+ writeBlockStudioImportCandidate(projectRoot, session.id, {
10764
+ ...readBlockStudioImportCandidate(projectRoot, session.id, candidate.id),
10765
+ analysisStatus: 'retrieving',
10766
+ draftSave: { status: 'pending' },
9964
10767
  });
9965
- contextObjectCount += contextPack?.objects.length ?? 0;
9966
- const evidence = contextPack ? dqlGenerationEvidenceFromContext(contextPack, candidate) : deterministicDqlGenerationEvidence(candidate);
9967
- let patch = deterministicDqlGenerationPatch(candidate, evidence);
9968
- let generatorName = 'local-deterministic';
9969
- let generationMode = 'deterministic';
9970
- if (provider) {
9971
- const aiPatch = await buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack).catch((error) => {
9972
- warnings.push(`AI generation fell back for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
9973
- return null;
10768
+ }
10769
+ const sharedContextPack = await buildDqlGenerationSessionContextPack(projectRoot, session.candidates).catch((error) => {
10770
+ warnings.push(`Shared context pack failed: ${error instanceof Error ? error.message : String(error)}`);
10771
+ return null;
10772
+ });
10773
+ const contextObjectCount = sharedContextPack?.objects.length ?? 0;
10774
+ const nextCandidates = await mapWithConcurrency(session.candidates, 3, async (candidate) => {
10775
+ try {
10776
+ const contextPack = sharedContextPack;
10777
+ const evidence = contextPack ? dqlGenerationEvidenceFromContext(contextPack, candidate) : deterministicDqlGenerationEvidence(candidate);
10778
+ writeBlockStudioImportCandidate(projectRoot, session.id, {
10779
+ ...readBlockStudioImportCandidate(projectRoot, session.id, candidate.id),
10780
+ analysisStatus: 'reviewing',
10781
+ evidence,
9974
10782
  });
9975
- if (aiPatch) {
9976
- patch = mergeDqlGenerationPatch(patch, aiPatch, candidate, evidence);
9977
- generatorName = provider.name;
9978
- generationMode = 'ai';
10783
+ let patch = deterministicDqlGenerationPatch(candidate, evidence);
10784
+ let generatorName = 'local-deterministic';
10785
+ let generationMode = 'deterministic';
10786
+ const deterministicSimilarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
10787
+ if (deterministicSimilarity.recommendedAction === 'reuse_existing') {
10788
+ patch = {
10789
+ ...patch,
10790
+ similarityMatches: deterministicSimilarity.matches,
10791
+ recommendedAction: 'reuse_existing',
10792
+ };
10793
+ generatorName = 'certified-reuse';
9979
10794
  }
10795
+ else if (provider) {
10796
+ const aiPatch = await buildAiDqlGenerationPatch(provider, candidate, evidence, contextPack).catch((error) => {
10797
+ warnings.push(`AI generation fell back for ${candidate.name}: ${error instanceof Error ? error.message : String(error)}`);
10798
+ return null;
10799
+ });
10800
+ if (aiPatch) {
10801
+ patch = mergeDqlGenerationPatch(patch, aiPatch, candidate, evidence);
10802
+ generatorName = provider.name;
10803
+ generationMode = 'ai';
10804
+ }
10805
+ }
10806
+ const similarity = patch.recommendedAction === 'reuse_existing'
10807
+ ? deterministicSimilarity
10808
+ : buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
10809
+ patch = {
10810
+ ...patch,
10811
+ similarityMatches: similarity.matches,
10812
+ recommendedAction: similarity.recommendedAction,
10813
+ };
10814
+ for (const warning of patch.warnings ?? [])
10815
+ warnings.push(`${candidate.name}: ${warning}`);
10816
+ const enriched = updateBlockStudioImportCandidate(projectRoot, session.id, candidate.id, {
10817
+ name: patch.name,
10818
+ domain: patch.domain,
10819
+ description: patch.description,
10820
+ owner: patch.owner,
10821
+ tags: patch.tags,
10822
+ terms: patch.terms,
10823
+ pattern: patch.pattern,
10824
+ grain: patch.grain,
10825
+ entities: patch.entities,
10826
+ outputs: patch.outputs,
10827
+ dimensions: patch.dimensions,
10828
+ allowedFilters: patch.allowedFilters,
10829
+ parameterPolicy: patch.parameterPolicy,
10830
+ filterBindings: patch.filterBindings,
10831
+ parameterDecisions: patch.parameterDecisions,
10832
+ similarityMatches: patch.similarityMatches,
10833
+ recommendedAction: patch.recommendedAction,
10834
+ sourceSystems: patch.sourceSystems,
10835
+ replacementFor: patch.replacementFor,
10836
+ reviewCadence: patch.reviewCadence,
10837
+ sql: patch.sql,
10838
+ llmContext: patch.llmContext,
10839
+ evidence,
10840
+ conversionNotes: dqlGenerationConversionNotes(generatorName),
10841
+ generationMode,
10842
+ generationProvider: generatorName,
10843
+ });
10844
+ const validated = {
10845
+ ...enriched,
10846
+ dqlSource: buildSemanticImportCandidateSource(enriched, patch, semanticLayer) ?? enriched.dqlSource,
10847
+ validation: validateBlockStudioSource(enriched.dqlSource, semanticLayer),
10848
+ analysisStatus: 'ready',
10849
+ };
10850
+ validated.validation = validateBlockStudioSource(validated.dqlSource, semanticLayer);
10851
+ const result = persistence === 'draft-files'
10852
+ ? saveDqlGenerationDraftForProject(projectRoot, session.id, validated)
10853
+ : {
10854
+ ...validated,
10855
+ reviewStatus: validated.recommendedAction === 'reuse_existing' ? 'review' : 'draft',
10856
+ generationMode: validated.generationMode ?? 'deterministic',
10857
+ generationProvider: validated.generationProvider ?? 'local-deterministic',
10858
+ llmContext: validated.llmContext ?? deterministicDqlGenerationContext(validated, validated.evidence ?? []),
10859
+ evidence: validated.evidence ?? [],
10860
+ draftSave: {
10861
+ status: validated.recommendedAction === 'reuse_existing' ? 'skipped' : 'pending',
10862
+ reason: validated.recommendedAction === 'reuse_existing' ? 'Reuse an existing governed block; no draft file was created.' : undefined,
10863
+ },
10864
+ };
10865
+ writeBlockStudioImportCandidate(projectRoot, session.id, result);
10866
+ return result;
9980
10867
  }
9981
- const similarity = buildDqlGenerationSimilarityMatches(candidate, patch, contextPack);
9982
- patch = {
9983
- ...patch,
9984
- similarityMatches: similarity.matches,
9985
- recommendedAction: similarity.recommendedAction,
9986
- };
9987
- for (const warning of patch.warnings ?? [])
9988
- warnings.push(`${candidate.name}: ${warning}`);
9989
- const enriched = updateBlockStudioImportCandidate(projectRoot, session.id, candidate.id, {
9990
- name: patch.name,
9991
- domain: patch.domain,
9992
- description: patch.description,
9993
- owner: patch.owner,
9994
- tags: patch.tags,
9995
- terms: patch.terms,
9996
- pattern: patch.pattern,
9997
- grain: patch.grain,
9998
- entities: patch.entities,
9999
- outputs: patch.outputs,
10000
- dimensions: patch.dimensions,
10001
- allowedFilters: patch.allowedFilters,
10002
- parameterPolicy: patch.parameterPolicy,
10003
- filterBindings: patch.filterBindings,
10004
- parameterDecisions: patch.parameterDecisions,
10005
- similarityMatches: patch.similarityMatches,
10006
- recommendedAction: patch.recommendedAction,
10007
- sourceSystems: patch.sourceSystems,
10008
- replacementFor: patch.replacementFor,
10009
- reviewCadence: patch.reviewCadence,
10010
- sql: patch.sql,
10011
- llmContext: patch.llmContext,
10012
- evidence,
10013
- conversionNotes: dqlGenerationConversionNotes(generatorName),
10014
- generationMode,
10015
- generationProvider: generatorName,
10016
- });
10017
- const validated = {
10018
- ...enriched,
10019
- validation: validateBlockStudioSource(enriched.dqlSource, semanticLayer),
10020
- };
10021
- const savedDraft = saveDqlGenerationDraftForProject(projectRoot, session.id, validated);
10022
- writeBlockStudioImportCandidate(projectRoot, session.id, savedDraft);
10023
- nextCandidates.push(savedDraft);
10024
- }
10868
+ catch (error) {
10869
+ const message = error instanceof Error ? error.message : String(error);
10870
+ warnings.push(`${candidate.name}: ${message}`);
10871
+ const current = readBlockStudioImportCandidate(projectRoot, session.id, candidate.id);
10872
+ const failed = {
10873
+ ...current,
10874
+ analysisStatus: 'needs_attention',
10875
+ warnings: Array.from(new Set([...(current.warnings ?? []), message])),
10876
+ generationMode: current.generationMode ?? 'deterministic',
10877
+ generationProvider: current.generationProvider ?? 'local-deterministic',
10878
+ llmContext: current.llmContext ?? deterministicDqlGenerationContext(current, current.evidence ?? []),
10879
+ evidence: current.evidence ?? deterministicDqlGenerationEvidence(current),
10880
+ draftSave: current.draftSave ?? { status: 'pending' },
10881
+ };
10882
+ writeBlockStudioImportCandidate(projectRoot, session.id, failed);
10883
+ return failed;
10884
+ }
10885
+ });
10025
10886
  const generationSession = {
10026
10887
  ...session,
10027
10888
  mode: 'ai-import',
10889
+ persistence,
10028
10890
  candidates: nextCandidates,
10029
10891
  updatedAt: new Date().toISOString(),
10030
10892
  generation: {
@@ -10040,6 +10902,19 @@ export async function createDqlGenerationSessionForProject(projectRoot, options,
10040
10902
  await refreshLocalMetadataCatalog(projectRoot);
10041
10903
  return generationSession;
10042
10904
  }
10905
+ async function mapWithConcurrency(items, limit, mapper) {
10906
+ const results = new Array(items.length);
10907
+ let cursor = 0;
10908
+ const workers = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
10909
+ while (cursor < items.length) {
10910
+ const index = cursor;
10911
+ cursor += 1;
10912
+ results[index] = await mapper(items[index], index);
10913
+ }
10914
+ });
10915
+ await Promise.all(workers);
10916
+ return results;
10917
+ }
10043
10918
  function saveDqlGenerationDraftForProject(projectRoot, importId, candidate) {
10044
10919
  if (candidate.recommendedAction === 'reuse_existing') {
10045
10920
  const topMatch = candidate.similarityMatches?.[0];
@@ -10598,6 +11473,43 @@ async function buildDqlGenerationContextPack(projectRoot, candidate) {
10598
11473
  strictness: 'balanced',
10599
11474
  });
10600
11475
  }
11476
+ async function buildDqlGenerationSessionContextPack(projectRoot, candidates) {
11477
+ const tables = Array.from(new Set(candidates.flatMap((candidate) => candidate.lineage.sourceTables))).join(', ');
11478
+ const question = [
11479
+ `Analyze ${candidates.length} SQL import candidate${candidates.length === 1 ? '' : 's'} for governed DQL reuse or composition.`,
11480
+ tables ? `Source tables: ${tables}.` : '',
11481
+ ...candidates.slice(0, 20).map((candidate, index) => [
11482
+ `Candidate ${index + 1}: ${candidate.name}`,
11483
+ candidate.description,
11484
+ candidate.sql.slice(0, 1000),
11485
+ ].filter(Boolean).join('\n')),
11486
+ ].filter(Boolean).join('\n\n');
11487
+ return buildLocalContextPack(projectRoot, {
11488
+ question,
11489
+ mode: 'build',
11490
+ surface: 'block-studio',
11491
+ limit: 160,
11492
+ objectTypes: [
11493
+ 'dql_block',
11494
+ 'dql_term',
11495
+ 'business_view',
11496
+ 'domain',
11497
+ 'semantic_metric',
11498
+ 'semantic_model',
11499
+ 'semantic_dimension',
11500
+ 'semantic_measure',
11501
+ 'dbt_model',
11502
+ 'dbt_source',
11503
+ 'dbt_column',
11504
+ 'warehouse_table',
11505
+ 'datalex_domain',
11506
+ 'datalex_entity',
11507
+ 'datalex_contract',
11508
+ 'datalex_term',
11509
+ ],
11510
+ strictness: 'balanced',
11511
+ });
11512
+ }
10601
11513
  function dqlGenerationEvidenceFromContext(contextPack, candidate) {
10602
11514
  const directEvidence = contextPack.objects
10603
11515
  .filter((object) => isDqlGenerationEvidenceObject(object))
@@ -10749,6 +11661,54 @@ function deterministicDqlGenerationPatch(candidate, evidence) {
10749
11661
  reviewCadence: 'monthly',
10750
11662
  };
10751
11663
  }
11664
+ function buildSemanticImportCandidateSource(candidate, patch, semanticLayer) {
11665
+ if (!semanticLayer || candidate.recommendedAction === 'reuse_existing')
11666
+ return null;
11667
+ const aggregateAliases = Array.from(candidate.sql.matchAll(/\b(?:sum|count|avg|min|max)\s*\([^)]*\)\s+(?:as\s+)?([A-Za-z_][A-Za-z0-9_]*)/gi))
11668
+ .map((match) => match[1]);
11669
+ if (aggregateAliases.length === 0)
11670
+ return null;
11671
+ const metrics = semanticLayer.listMetrics();
11672
+ const matchedMetrics = aggregateAliases.map((alias) => metrics.find((metric) => normalizedTerm(metric.name) === normalizedTerm(alias)));
11673
+ if (matchedMetrics.some((metric) => !metric))
11674
+ return null;
11675
+ const metricNames = Array.from(new Set(matchedMetrics.map((metric) => metric.name)));
11676
+ const compatibleDimensions = semanticLayer.listCompatibleDimensions(metricNames);
11677
+ const compatibleByName = new Map(compatibleDimensions.map((dimension) => [normalizedTerm(dimension.name), dimension]));
11678
+ const groupFields = extractDqlGenerationGroupByFields(candidate.sql)
11679
+ .map((field) => field.split('.').pop() ?? field)
11680
+ .filter((field) => !/^\d+$/.test(field));
11681
+ const matchedDimensions = groupFields.map((field) => compatibleByName.get(normalizedTerm(field)));
11682
+ if (groupFields.length > 0 && matchedDimensions.some((dimension) => !dimension))
11683
+ return null;
11684
+ const dimensionNames = Array.from(new Set(matchedDimensions.map((dimension) => dimension.name)));
11685
+ const allowedFilters = (patch.allowedFilters ?? [])
11686
+ .map((filter) => compatibleByName.get(normalizedTerm(filter))?.name)
11687
+ .filter((filter) => Boolean(filter));
11688
+ const quote = (value) => `"${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
11689
+ const metricLine = metricNames.length === 1
11690
+ ? ` metric = ${quote(metricNames[0])}`
11691
+ : ` metrics = [${metricNames.map(quote).join(', ')}]`;
11692
+ const lines = [
11693
+ `block ${quote(candidate.name)} {`,
11694
+ ' status = "draft"',
11695
+ ` domain = ${quote(candidate.domain)}`,
11696
+ ' type = "semantic"',
11697
+ ` description = ${quote(candidate.description)}`,
11698
+ ` owner = ${quote(candidate.owner)}`,
11699
+ ` tags = [${candidate.tags.map(quote).join(', ')}]`,
11700
+ metricLine,
11701
+ dimensionNames.length > 0 ? ` dimensions = [${dimensionNames.map(quote).join(', ')}]` : '',
11702
+ allowedFilters.length > 0 ? ` requested_filters = [${allowedFilters.map(quote).join(', ')}]` : '',
11703
+ '',
11704
+ ' visualization {',
11705
+ ` chart = "${dimensionNames.length > 0 ? 'bar' : 'kpi'}"`,
11706
+ ' }',
11707
+ '}',
11708
+ '',
11709
+ ];
11710
+ return lines.filter((line, index) => line !== '' || (index > 0 && lines[index - 1] !== '')).join('\n');
11711
+ }
10752
11712
  function inferDqlGenerationTerms(evidence) {
10753
11713
  return Array.from(new Set(evidence
10754
11714
  .filter((item) => item.kind === 'dql_term' || item.kind === 'datalex_term')
@@ -12210,7 +13170,7 @@ function indentBlock(value, spaces) {
12210
13170
  function normalizeBlockStudioContent(options) {
12211
13171
  const content = options.content?.trim();
12212
13172
  if (content && /^\s*block\s+"/i.test(content)) {
12213
- return `${content.trimEnd()}\n`;
13173
+ return `${applySavedBlockMetadata(content, options).trimEnd()}\n`;
12214
13174
  }
12215
13175
  return buildBlankBlockContent({
12216
13176
  name: options.name,
@@ -12224,6 +13184,29 @@ function normalizeBlockStudioContent(options) {
12224
13184
  sql: content || 'SELECT 1 AS value',
12225
13185
  });
12226
13186
  }
13187
+ /** Apply the fields supplied by the save dialog to an AI/DQL artifact as well. */
13188
+ function applySavedBlockMetadata(source, options) {
13189
+ let next = source.trim();
13190
+ next = next.replace(/block\s+"[^"]*"\s*\{/, `block "${escapeDqlString(options.name)}" {`);
13191
+ const upsert = (key, value) => {
13192
+ const pattern = new RegExp(`^(\\s*)${key}\\s*=.*$`, 'm');
13193
+ if (pattern.test(next)) {
13194
+ next = next.replace(pattern, (_match, indent) => `${indent}${key} = ${value}`);
13195
+ }
13196
+ else {
13197
+ next = next.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n ${key} = ${value}`);
13198
+ }
13199
+ };
13200
+ upsert('domain', `"${escapeDqlString(options.domain)}"`);
13201
+ if (options.owner?.trim())
13202
+ upsert('owner', `"${escapeDqlString(options.owner.trim())}"`);
13203
+ if (options.description?.trim())
13204
+ upsert('description', `"${escapeDqlString(options.description.trim())}"`);
13205
+ if (options.tags && options.tags.length > 0) {
13206
+ upsert('tags', `[${options.tags.map((tag) => `"${escapeDqlString(tag)}"`).join(', ')}]`);
13207
+ }
13208
+ return next;
13209
+ }
12227
13210
  function buildBlankBlockContent(options) {
12228
13211
  const lines = [
12229
13212
  `block "${escapeDqlString(options.name)}" {`,
@@ -13399,6 +14382,93 @@ async function gitPush(cwd) {
13399
14382
  ? { ok: true, output: gitErrorOutput(res) }
13400
14383
  : { ok: false, error: gitErrorOutput(res) };
13401
14384
  }
14385
+ async function execGh(cwd, args) {
14386
+ const { execFile } = await import('node:child_process');
14387
+ return new Promise((resolve) => {
14388
+ execFile('gh', args, { cwd, maxBuffer: 8 * 1024 * 1024 }, (err, stdout, stderr) => {
14389
+ resolve({
14390
+ stdout: String(stdout ?? ''),
14391
+ stderr: String(stderr ?? ''),
14392
+ code: err ? (err.code ? 1 : err.code ?? 1) : 0,
14393
+ });
14394
+ });
14395
+ });
14396
+ }
14397
+ function gitReviewBranchName(title) {
14398
+ const slug = title.toLowerCase().trim()
14399
+ .replace(/[^a-z0-9]+/g, '-')
14400
+ .replace(/^-+|-+$/g, '')
14401
+ .slice(0, 46) || 'governed-update';
14402
+ return `dql/review/${slug}-${Date.now().toString(36)}`;
14403
+ }
14404
+ function gitHubCompareUrl(remoteUrl, branch, base) {
14405
+ if (!remoteUrl)
14406
+ return undefined;
14407
+ const normalized = remoteUrl
14408
+ .replace(/^git@github\.com:/, 'https://github.com/')
14409
+ .replace(/\.git$/, '');
14410
+ const match = /^https?:\/\/github\.com\/([^/]+\/[^/]+)$/.exec(normalized);
14411
+ return match ? `https://github.com/${match[1]}/compare/${encodeURIComponent(base)}...${encodeURIComponent(branch)}?expand=1` : undefined;
14412
+ }
14413
+ /**
14414
+ * The only Git write workflow that can create a PR from the UI. It explicitly
14415
+ * creates/switches a review branch when starting from the base branch and never
14416
+ * calls merge. Existing staged changes outside the requested files are blocked
14417
+ * so a review cannot accidentally include unrelated work.
14418
+ */
14419
+ async function gitCreateReview(cwd, input) {
14420
+ const gitRoot = await resolveGitRoot(cwd);
14421
+ if (!gitRoot)
14422
+ return { ok: false, error: 'Not a git repository' };
14423
+ const title = input.title.trim();
14424
+ if (!title)
14425
+ return { ok: false, error: 'Review title required' };
14426
+ const validPaths = validatePaths(gitRoot, input.paths);
14427
+ if (!validPaths.ok)
14428
+ return { ok: false, error: validPaths.error };
14429
+ const base = input.base.trim() || 'main';
14430
+ const status = await readGitStatus(gitRoot);
14431
+ const stagedOutsideReview = status.changes
14432
+ .filter((change) => change.status[0] !== ' ' && change.status[0] !== '?' && !validPaths.paths.includes(change.path))
14433
+ .map((change) => change.path);
14434
+ if (stagedOutsideReview.length > 0) {
14435
+ return { ok: false, error: `Unstage unrelated files before requesting review: ${stagedOutsideReview.slice(0, 3).join(', ')}` };
14436
+ }
14437
+ const remoteBeforeReview = await readGitRemote(gitRoot);
14438
+ if (!remoteBeforeReview.url) {
14439
+ return { ok: false, error: 'Add a Git remote before requesting review. No branch or commit was created.' };
14440
+ }
14441
+ let branch = status.branch;
14442
+ if (!branch || branch === 'HEAD' || branch === base || branch === 'master') {
14443
+ branch = gitReviewBranchName(title);
14444
+ const created = await gitCreateBranch(gitRoot, branch, true);
14445
+ if (!created.ok)
14446
+ return created;
14447
+ }
14448
+ const stage = await gitStage(gitRoot, validPaths.paths);
14449
+ if (!stage.ok)
14450
+ return stage;
14451
+ const commit = await gitCommit(gitRoot, title, false);
14452
+ if (!commit.ok)
14453
+ return commit;
14454
+ const pushed = await gitPush(gitRoot);
14455
+ if (!pushed.ok)
14456
+ return { ...pushed, branch, hash: commit.hash };
14457
+ const remote = await readGitRemote(gitRoot);
14458
+ const compareUrl = gitHubCompareUrl(remote.url, branch, base);
14459
+ const pr = await execGh(gitRoot, ['pr', 'create', '--base', base, '--head', branch, '--title', title, '--body', input.body.trim() || `Review governed analytics changes: ${title}`]);
14460
+ if (pr.code === 0) {
14461
+ const prUrl = (pr.stdout.match(/https:\/\/github\.com\/[^\s]+\/pull\/\d+/)?.[0]) ?? compareUrl;
14462
+ return { ok: true, branch, hash: commit.hash, prUrl };
14463
+ }
14464
+ return {
14465
+ ok: true,
14466
+ branch,
14467
+ hash: commit.hash,
14468
+ prUrl: compareUrl,
14469
+ warning: `Branch pushed. Open the compare page to create the PR: ${gitErrorOutput(pr) || 'GitHub CLI is not available or authenticated.'}`,
14470
+ };
14471
+ }
13402
14472
  async function gitPull(cwd) {
13403
14473
  const gitRoot = await resolveGitRoot(cwd);
13404
14474
  if (!gitRoot)
@@ -13431,6 +14501,81 @@ async function readGitRemote(cwd) {
13431
14501
  const url = await execGit(gitRoot, ['remote', 'get-url', name]);
13432
14502
  return { inRepo: true, url: url.code === 0 ? url.stdout.trim() : null, name };
13433
14503
  }
14504
+ function collectGovernedSourcePaths(root, folder, predicate) {
14505
+ const start = join(root, folder);
14506
+ if (!existsSync(start))
14507
+ return [];
14508
+ const paths = [];
14509
+ const walk = (dir) => {
14510
+ if (paths.length >= 400)
14511
+ return;
14512
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
14513
+ if (paths.length >= 400)
14514
+ return;
14515
+ const absolute = join(dir, entry.name);
14516
+ if (entry.isDirectory())
14517
+ walk(absolute);
14518
+ else if (entry.isFile() && predicate(entry.name))
14519
+ paths.push(absolute);
14520
+ }
14521
+ };
14522
+ walk(start);
14523
+ return paths;
14524
+ }
14525
+ async function gitGovernedFileState(gitRoot, path) {
14526
+ const tracked = await execGit(gitRoot, ['ls-files', '--error-unmatch', '--', path]);
14527
+ const status = await execGit(gitRoot, ['status', '--porcelain=v1', '--untracked-files=all', '--', path]);
14528
+ if (tracked.code === 0)
14529
+ return status.stdout.trim() ? 'changed' : 'tracked';
14530
+ const ignored = await execGit(gitRoot, ['check-ignore', '-q', '--', path]);
14531
+ return ignored.code === 0 ? 'ignored' : 'untracked';
14532
+ }
14533
+ async function readGitGovernedContext(cwd) {
14534
+ const gitRoot = await resolveGitRoot(cwd);
14535
+ const empty = () => ({ total: 0, tracked: 0, changed: 0, untracked: 0, ignored: 0, paths: [] });
14536
+ if (!gitRoot)
14537
+ return { inRepo: false, trackingReady: false, domains: empty(), skills: empty() };
14538
+ const summarize = async (files) => {
14539
+ const group = empty();
14540
+ for (const absolute of files) {
14541
+ const path = relative(gitRoot, absolute).replace(/\\/g, '/');
14542
+ const state = await gitGovernedFileState(gitRoot, path);
14543
+ group.total += 1;
14544
+ group[state] += 1;
14545
+ if (group.paths.length < 24)
14546
+ group.paths.push({ path, state });
14547
+ }
14548
+ return group;
14549
+ };
14550
+ const [domains, skills] = await Promise.all([
14551
+ summarize(collectGovernedSourcePaths(cwd, 'domains', (name) => name === 'domain.dql')),
14552
+ summarize([
14553
+ ...collectGovernedSourcePaths(cwd, 'skills', (name) => name.endsWith('.skill.md')),
14554
+ ...collectGovernedSourcePaths(cwd, '.dql/skills', (name) => name.endsWith('.skill.md')),
14555
+ ]),
14556
+ ]);
14557
+ return { inRepo: true, trackingReady: domains.ignored === 0 && skills.ignored === 0, domains, skills };
14558
+ }
14559
+ /** Replace only the legacy broad `.dql/` ignore with cache/runtime-specific rules. */
14560
+ async function enableGitGovernedContextTracking(cwd) {
14561
+ const gitRoot = await resolveGitRoot(cwd);
14562
+ if (!gitRoot)
14563
+ return { ok: false, error: 'Not a git repository' };
14564
+ const ignorePath = join(cwd, '.gitignore');
14565
+ const existing = existsSync(ignorePath) ? readFileSync(ignorePath, 'utf-8') : '';
14566
+ const lines = existing.split(/\r?\n/);
14567
+ const next = lines.filter((line) => !/^\s*\/?\.dql\/\s*$/.test(line));
14568
+ const keepLocal = ['.dql/runs/', '.dql/cache/', '.dql/imports/', '.dql/local/', '.dql/docker-starter/'];
14569
+ for (const rule of keepLocal) {
14570
+ if (!next.some((line) => line.trim() === rule))
14571
+ next.push(rule);
14572
+ }
14573
+ const rendered = `${next.join('\n').replace(/\n+$/, '')}\n`;
14574
+ if (rendered !== existing)
14575
+ writeFileSync(ignorePath, rendered, 'utf-8');
14576
+ const migration = migrateLegacySkills(cwd);
14577
+ return { ok: true, changed: rendered !== existing || migration.moved.length > 0 };
14578
+ }
13434
14579
  async function gitCreateBranch(cwd, name, checkout) {
13435
14580
  const gitRoot = await resolveGitRoot(cwd);
13436
14581
  if (!gitRoot)
@@ -13710,10 +14855,31 @@ function isAiPinRefreshDue(lastRefreshedAt) {
13710
14855
  return true;
13711
14856
  return Date.now() - last >= 24 * 60 * 60 * 1000;
13712
14857
  }
13713
- async function buildAgentSchemaContextFromCatalog(projectRoot, question) {
13714
- const contextPack = await buildLocalContextPack(projectRoot, { question, limit: 80 });
14858
+ async function buildAgentSchemaContextFromCatalog(projectRoot, question, preparedContextPack) {
14859
+ const contextPack = preparedContextPack ?? await buildLocalContextPack(projectRoot, { question, limit: 80 });
13715
14860
  return buildAgentSchemaContextFromContextPack(question, contextPack);
13716
14861
  }
14862
+ /** How long a stored live-warehouse schema snapshot is trusted before a rescan (P6). */
14863
+ const RUNTIME_SNAPSHOT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
14864
+ /**
14865
+ * Whether the project's stored live-schema snapshot is missing or older than the
14866
+ * freshness window (P6). Used to force a fresh information_schema scan even when the
14867
+ * question-shape heuristic wouldn't otherwise trigger one — so a warehouse schema
14868
+ * change between sessions isn't silently reasoned over. Best-effort: a catalog error
14869
+ * returns false so it never causes a rescan storm.
14870
+ */
14871
+ export function runtimeSnapshotStale(projectRoot, maxAgeMs = RUNTIME_SNAPSHOT_MAX_AGE_MS) {
14872
+ try {
14873
+ const snapshot = latestRuntimeSchemaSnapshotForProject(projectRoot);
14874
+ if (!snapshot?.capturedAt)
14875
+ return true;
14876
+ const age = Date.now() - new Date(snapshot.capturedAt).getTime();
14877
+ return !Number.isFinite(age) || age > maxAgeMs;
14878
+ }
14879
+ catch {
14880
+ return false;
14881
+ }
14882
+ }
13717
14883
  function recordAgentRuntimeSchemaSnapshot(projectRoot, schemaContext, source) {
13718
14884
  if (schemaContext.length === 0)
13719
14885
  return;