@duckcodeailabs/dql-cli 1.12.10 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts +151 -3
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +675 -97
- package/dist/apps-api.js.map +1 -1
- package/dist/assets/dql-notebook/assets/AgentAnswerCard-Cp0eh6zU.js +5 -0
- package/dist/assets/dql-notebook/assets/{AgentLogPage-uvVs284-.js → AgentLogPage-CBGEoGmU.js} +1 -1
- package/dist/assets/dql-notebook/assets/{AiBuildDialog-DEUS19WF.js → AiBuildDialog-CwAD3tXE.js} +1 -1
- package/dist/assets/dql-notebook/assets/AiBuildResult-CyqTuQe5.js +33 -0
- package/dist/assets/dql-notebook/assets/AiSidePanel-DjB8PWHG.js +6 -0
- package/dist/assets/dql-notebook/assets/AnalyticsHome-B7BkvA-R.js +6 -0
- package/dist/assets/dql-notebook/assets/AppsView-CUrqtEGC.js +4583 -0
- package/dist/assets/dql-notebook/assets/BlockStudio-Bl0aGQJR.js +60 -0
- package/dist/assets/dql-notebook/assets/{BusinessArtifactView-D8FIgrkF.js → BusinessArtifactView-C4SZVNW-.js} +1 -1
- package/dist/assets/dql-notebook/assets/DbtFirstModelingPage-B0uxitnx.js +22 -0
- package/dist/assets/dql-notebook/assets/GitPage-BfVD4tVq.js +7 -0
- package/dist/assets/dql-notebook/assets/GlobalAiRail-DUwdy3fB.js +1 -0
- package/dist/assets/dql-notebook/assets/GovernedContextPage-D9f26ZFb.js +7 -0
- package/dist/assets/dql-notebook/assets/{HelpDocsPage-Bc5pLakt.js → HelpDocsPage-BDPNVot1.js} +3 -3
- package/dist/assets/dql-notebook/assets/{HomePage-DQSo-K4y.js → HomePage-DiOnRhEG.js} +9 -4
- package/dist/assets/dql-notebook/assets/LineageDAG-BiUFX2KL.js +1 -0
- package/dist/assets/dql-notebook/assets/{LineageDetailView-CZEx2r1H.js → LineageDetailView-CfbSvlPJ.js} +1 -1
- package/dist/assets/dql-notebook/assets/{LineageDrawer-P7ZdOW8-.js → LineageDrawer-CgTScaGB.js} +1 -1
- package/dist/assets/dql-notebook/assets/LineagePathBreadcrumb-D7ZA7k68.js +14 -0
- package/dist/assets/dql-notebook/assets/{MiniLineageGraph-Ctp_yFh8.js → MiniLineageGraph-GkD4y17x.js} +1 -1
- package/dist/assets/dql-notebook/assets/NewBlockModal-B9TfRjt7.js +1 -0
- package/dist/assets/dql-notebook/assets/NewNotebookModal-BbO9NKUl.js +46 -0
- package/dist/assets/dql-notebook/assets/NotebookEditor-CuMpvXR9.js +200 -0
- package/dist/assets/dql-notebook/assets/ReadinessPage-BROJ9YBM.js +1 -0
- package/dist/assets/dql-notebook/assets/{SetupOnboarding-DX2f53J2.js → SetupOnboarding-BCcZlpuQ.js} +2 -2
- package/dist/assets/dql-notebook/assets/{TrustBadge-DPnKzRSY.js → TrustBadge-vQQh8d0D.js} +1 -1
- package/dist/assets/dql-notebook/assets/UnifiedAgentRunPanel-Dj4AXAhT.js +83 -0
- package/dist/assets/dql-notebook/assets/{answer-to-notebook-Bm9oByWA.js → answer-to-notebook-DuqCcDDl.js} +1 -1
- package/dist/assets/dql-notebook/assets/{arrow-left-D2FbLWqh.js → arrow-left-CMe1kkOL.js} +1 -1
- package/dist/assets/dql-notebook/assets/{arrow-right-OYItAZ-s.js → arrow-right-DvB_DONr.js} +2 -2
- package/dist/assets/dql-notebook/assets/{book-open-text-BnnirfAc.js → book-open-text-CJ3NAvrs.js} +1 -1
- package/dist/assets/dql-notebook/assets/{bot-BJNxPYFm.js → bot-CV8DYRrt.js} +1 -1
- package/dist/assets/dql-notebook/assets/{chevron-up-DM5LX-26.js → chevron-up-HxU1l85K.js} +2 -2
- package/dist/assets/dql-notebook/assets/{circle-x-Dic_-4tF.js → circle-x-BCgtha1W.js} +1 -1
- package/dist/assets/dql-notebook/assets/{codemirror-1_I9gm6Z.js → codemirror-DXt7RW72.js} +1 -1
- package/dist/assets/dql-notebook/assets/{dagre.esm-C9zxjHHX.js → dagre.esm-GKp7qoUL.js} +1 -1
- package/dist/assets/dql-notebook/assets/{external-link-DHxAcoCE.js → external-link-C-PXokWV.js} +1 -1
- package/dist/assets/dql-notebook/assets/{grip-vertical-02I-EOjL.js → grip-vertical-B4EHbGXW.js} +1 -1
- package/dist/assets/dql-notebook/assets/index-1mIC_XX4.js +634 -0
- package/dist/assets/dql-notebook/assets/{link-2-Delh-lai.js → link-2-BLx-goet.js} +2 -2
- package/dist/assets/dql-notebook/assets/{list-tree-BrIZg6l6.js → list-tree-BRFpEmvV.js} +1 -1
- package/dist/assets/dql-notebook/assets/{maximize-2-CQOA-bbN.js → maximize-2-CCd5uRIZ.js} +1 -1
- package/dist/assets/dql-notebook/assets/{minimize-2-CUDBPkwe.js → minimize-2-CDwSCGlO.js} +1 -1
- package/dist/assets/dql-notebook/assets/{panel-right-open-CwhZfkKJ.js → panel-right-open-CunA1xO7.js} +2 -2
- package/dist/assets/dql-notebook/assets/{pencil-Cf6-MgpT.js → pencil-D_I14Wyk.js} +2 -2
- package/dist/assets/dql-notebook/assets/{rotate-ccw-DndbJAiz.js → rotate-ccw-DOrfTZk8.js} +2 -2
- package/dist/assets/dql-notebook/assets/{semantic-fields-DwKPY_mU.js → semantic-fields-AfQvjt5R.js} +1 -1
- package/dist/assets/dql-notebook/assets/{star-DvcMC9wZ.js → star-DDhv4e8M.js} +2 -2
- package/dist/assets/dql-notebook/assets/{triangle-alert-DUf4Ook0.js → triangle-alert-2ZXx8wAu.js} +1 -1
- package/dist/assets/dql-notebook/assets/usePersistedAgentThreadId-3rzkZrqc.js +18 -0
- package/dist/assets/dql-notebook/assets/{user-round-DSt0xU3K.js → user-round-BA074kiQ.js} +3 -3
- package/dist/assets/dql-notebook/assets/{wand-sparkles-ro8Cq0ro.js → wand-sparkles-CEYHLt9b.js} +1 -1
- package/dist/assets/dql-notebook/assets/{workflow-DVJXehjU.js → workflow-CZtyrcR-.js} +2 -2
- package/dist/assets/dql-notebook/assets/{wrench-DxUnoHtz.js → wrench-Ong23-wu.js} +1 -1
- package/dist/assets/dql-notebook/assets/{x-9Gtb3EAd.js → x-CSo1M3tu.js} +2 -2
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts +4 -3
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +13 -10
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/local-operation-coordinator.d.ts +59 -0
- package/dist/local-operation-coordinator.d.ts.map +1 -0
- package/dist/local-operation-coordinator.js +256 -0
- package/dist/local-operation-coordinator.js.map +1 -0
- package/dist/local-runtime.d.ts +89 -2
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +2158 -461
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +12 -11
- package/dist/project-refresh-coordinator.d.ts +19 -0
- package/dist/project-refresh-coordinator.d.ts.map +1 -0
- package/dist/project-refresh-coordinator.js +134 -0
- package/dist/project-refresh-coordinator.js.map +1 -0
- package/dist/project-refresh-worker.d.ts +13 -0
- package/dist/project-refresh-worker.d.ts.map +1 -0
- package/dist/project-refresh-worker.js +58 -0
- package/dist/project-refresh-worker.js.map +1 -0
- package/package.json +13 -11
- package/dist/assets/dql-notebook/assets/AgentAnswerCard-Dp4odOLE.js +0 -5
- package/dist/assets/dql-notebook/assets/AiBuildResult-B9KmBRsh.js +0 -33
- package/dist/assets/dql-notebook/assets/AiSidePanel-C7BZVSkM.js +0 -6
- package/dist/assets/dql-notebook/assets/AnalyticsHome-4TgtH6SB.js +0 -6
- package/dist/assets/dql-notebook/assets/AppsView-CnfNtk5W.js +0 -4466
- package/dist/assets/dql-notebook/assets/BlockStudio-DUXN3MKz.js +0 -60
- package/dist/assets/dql-notebook/assets/DbtFirstModelingPage-DR8hTZ95.js +0 -22
- package/dist/assets/dql-notebook/assets/GitPage-BS2sm9zh.js +0 -7
- package/dist/assets/dql-notebook/assets/GlobalAiRail-B_MZsB0z.js +0 -1
- package/dist/assets/dql-notebook/assets/GovernedContextPage-CNM5UCAH.js +0 -7
- package/dist/assets/dql-notebook/assets/LineageDAG-RdpWmGmR.js +0 -1
- package/dist/assets/dql-notebook/assets/LineagePathBreadcrumb-DTUMFkJ8.js +0 -14
- package/dist/assets/dql-notebook/assets/NewBlockModal-BLtjuWGy.js +0 -1
- package/dist/assets/dql-notebook/assets/NewNotebookModal-xgaKB2kI.js +0 -46
- package/dist/assets/dql-notebook/assets/NotebookEditor-D6RgLQw-.js +0 -202
- package/dist/assets/dql-notebook/assets/ReadinessPage-C8hATaT_.js +0 -1
- package/dist/assets/dql-notebook/assets/UnifiedAgentRunPanel-CwxQ4T19.js +0 -83
- package/dist/assets/dql-notebook/assets/index-Ca3UeUWI.js +0 -644
- package/dist/assets/dql-notebook/assets/usePersistedAgentThreadId-DbdBi0oH.js +0 -18
package/dist/local-runtime.js
CHANGED
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
import { execFileSync, execSync } from "node:child_process";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
|
-
import {
|
|
3
|
+
import { gzip } from "node:zlib";
|
|
4
4
|
import { createServer } from "node:http";
|
|
5
5
|
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, watch, writeFileSync, } from "node:fs";
|
|
6
|
+
import { mkdir as mkdirAsync, rename as renameAsync, rm as rmAsync, writeFile as writeFileAsync, } from 'node:fs/promises';
|
|
6
7
|
import { createRequire } from "node:module";
|
|
7
8
|
import { homedir } from "node:os";
|
|
8
9
|
import { fileURLToPath } from "node:url";
|
|
9
10
|
import { dirname, extname, join, normalize, relative, resolve, sep, } from "node:path";
|
|
10
11
|
import { Readable } from "node:stream";
|
|
12
|
+
import { LocalOperationCoordinator, } from './local-operation-coordinator.js';
|
|
13
|
+
import { ProjectRefreshCoordinator } from './project-refresh-coordinator.js';
|
|
11
14
|
import { buildMixedSourceWarehouseFallbackSql, findMentionedNotebookDataset, planMixedSourceNotebookSql, planMixedSourceSql, } from "./mixed-source-sql.js";
|
|
12
15
|
import { resolveNpmInvocation } from './npm-runtime.js';
|
|
13
16
|
import { ensureDqlGitignore, isGovernedSourceFile, isLegacyBroadDqlIgnore, } from './git-contract.js';
|
|
@@ -24,7 +27,7 @@ import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.j
|
|
|
24
27
|
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, conversationHistoryFromContext, recallRelevantTurns, renderConversationEnvelopeForPrompt, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, prepareConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, normalizeWarehouseSqlFailure, parseProposal, propose, proposePlan, recordGovernedCorrection, HintStore, defaultHintIndexPath, ensureHintIndexFresh, listHintsFromGit, getHintEvaluationFromGit, getCorrectionTraceFromGit, inspectGovernedHint, editGovernedHintCandidate, reopenGovernedHint, retireHint, supersedeHint, hintsConflict, mineJoinPatterns, reviewGovernedHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, internalRelationIdsInSql, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, projectEmbeddingProvider, isHashedEmbeddingProvider, clearProjectEmbeddingCache, upgradeVectorIndexForProject, openMetadataCatalog, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, isTrustedConversationTurn, resolveInternalRelationIds, analyticalError, tagAnalyticalError, withAnalyticalErrorOrigin, withAnalyticalErrorOriginSync, } from '@duckcodeailabs/dql-agent';
|
|
25
28
|
import { addSqlResultFilter, filterableResultColumns, replaceBlockStudioSql } from './sql-result-filter.js';
|
|
26
29
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
27
|
-
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
30
|
+
import { handleAppsApi, proposeAppAiBuild, recommendVisualization, } from './apps-api.js';
|
|
28
31
|
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
29
32
|
import { ClaudeCodeCliProvider, CodexCliProvider } from './providers/subscription-cli.js';
|
|
30
33
|
import { ClaudeOAuthManager, ClaudeOAuthProvider, claudeOAuthConnected, CLAUDE_OAUTH_MODELS, CLAUDE_OAUTH_DEFAULT_MODEL, } from './providers/oauth/claude-oauth.js';
|
|
@@ -85,6 +88,63 @@ function hasDbtSemanticArtifacts(projectRoot, dbtProjectDir, configuredManifestP
|
|
|
85
88
|
return false;
|
|
86
89
|
}
|
|
87
90
|
const AGENT_RUN_REQUESTED_MODES = new Set(['auto', 'ask', 'research', 'sql', 'block', 'app']);
|
|
91
|
+
/** AGT-007 / AGT-022: App-specific context adapter into the uniform AgentRun. */
|
|
92
|
+
export function buildAppCopilotResearchAgentRequest(input) {
|
|
93
|
+
const memoOnly = input.mode === 'memo_only';
|
|
94
|
+
const appResearch = {
|
|
95
|
+
mode: 'app_research',
|
|
96
|
+
generationMode: input.mode ?? 'sql_and_memo',
|
|
97
|
+
intent: input.intent,
|
|
98
|
+
appId: input.appId,
|
|
99
|
+
dashboardId: input.dashboardId,
|
|
100
|
+
sourceTileId: input.sourceTileId,
|
|
101
|
+
sourceBlockId: input.sourceBlockId,
|
|
102
|
+
title: input.title,
|
|
103
|
+
instruction: [
|
|
104
|
+
'Answer as an app-scoped analyst. Start from certified block/result context, active filters, dbt/semantic metadata, and schema evidence.',
|
|
105
|
+
memoOnly
|
|
106
|
+
? 'Write the stakeholder memo from the supplied selected result, preview rows, generated SQL, metrics, drivers, filters, and caveats. Do not generate replacement SQL unless the user explicitly asks to fix SQL.'
|
|
107
|
+
: 'Generate review-required read-only SQL only when the certified result does not exactly answer the requested analysis grain. Execute only through the bounded generated SQL preview path.',
|
|
108
|
+
'In the natural-language answer, write a stakeholder analysis memo, not a generic chat answer. Use Markdown headings chosen for the question, usually Executive answer, Key numbers, Drivers or Business readout, Caveats, and Next action.',
|
|
109
|
+
'Use concrete numbers from the selected result or preview when available. If baseline, segment, grain, lineage, or source proof is missing, say that explicitly instead of inventing a driver story.',
|
|
110
|
+
'Keep raw SQL in proposedSql/sql. Do not put SQL code fences or implementation trace in the memo body.',
|
|
111
|
+
].join(' '),
|
|
112
|
+
generatedSql: input.generatedSql,
|
|
113
|
+
metrics: input.metrics,
|
|
114
|
+
drivers: input.drivers,
|
|
115
|
+
resultPreviews: input.resultPreviews,
|
|
116
|
+
summaryHint: input.summaryHint,
|
|
117
|
+
recommendationHint: input.recommendationHint,
|
|
118
|
+
sqlError: input.sqlError,
|
|
119
|
+
sqlErrorKind: input.sqlErrorKind,
|
|
120
|
+
hasReportEvidence: input.hasReportEvidence,
|
|
121
|
+
context: input.context,
|
|
122
|
+
};
|
|
123
|
+
return {
|
|
124
|
+
question: memoOnly
|
|
125
|
+
? [
|
|
126
|
+
`Research question: ${input.question}`,
|
|
127
|
+
'Write the business memo now using only the evidence envelope supplied as upstream context.',
|
|
128
|
+
'Return Markdown sections. Keep SQL, query text, implementation trace, and raw routing details out of the memo body.',
|
|
129
|
+
'If the evidence is insufficient, state the gap and the next reviewer action instead of filling the story with generic text.',
|
|
130
|
+
].join('\n')
|
|
131
|
+
: input.question,
|
|
132
|
+
requestedMode: 'research',
|
|
133
|
+
audience: 'stakeholder',
|
|
134
|
+
selectedObject: {
|
|
135
|
+
kind: 'app',
|
|
136
|
+
id: input.appId,
|
|
137
|
+
title: input.title,
|
|
138
|
+
},
|
|
139
|
+
analysisDepth: 'deep',
|
|
140
|
+
workspaceContext: {
|
|
141
|
+
surface: 'app_copilot',
|
|
142
|
+
appId: input.appId,
|
|
143
|
+
dashboardId: input.dashboardId,
|
|
144
|
+
appResearch,
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
}
|
|
88
148
|
const AGENT_RUN_SELECTED_OBJECT_KINDS = new Set([
|
|
89
149
|
'notebook',
|
|
90
150
|
'cell',
|
|
@@ -228,6 +288,7 @@ export function parseAgentRunRequestBody(body) {
|
|
|
228
288
|
request: {
|
|
229
289
|
question,
|
|
230
290
|
selectedEvidenceId: agentRunString(record.selectedEvidenceId),
|
|
291
|
+
clarificationSourceQuestion: agentRunString(record.clarificationSourceQuestion),
|
|
231
292
|
requestedMode,
|
|
232
293
|
audience,
|
|
233
294
|
intent: agentRunString(record.intent),
|
|
@@ -324,7 +385,16 @@ export function agentRunDeadlineMs(request, env = process.env, activeProviderId)
|
|
|
324
385
|
if (request.requestedMode === 'research' || effectiveDepth === 'deep') {
|
|
325
386
|
return scale(researchDeadline);
|
|
326
387
|
}
|
|
327
|
-
|
|
388
|
+
// A structured clarification chip displays a compact metric/path label, but
|
|
389
|
+
// the engine restores and executes the original analytical question. Budget
|
|
390
|
+
// from that same original question as well. Otherwise a complex ranking or
|
|
391
|
+
// cross-model request can incorrectly inherit the 45s scalar-lookup budget
|
|
392
|
+
// merely because the selected label is short, then time out after the user
|
|
393
|
+
// already supplied the exact governed meaning.
|
|
394
|
+
const deadlineQuestion = request.selectedEvidenceId
|
|
395
|
+
? request.clarificationSourceQuestion?.trim() || request.question
|
|
396
|
+
: request.question;
|
|
397
|
+
const plan = buildAnalysisQuestionPlan(deadlineQuestion);
|
|
328
398
|
if (plan.needsResearchWorkspace || isAnalyticalBuildQuestion(plan)) {
|
|
329
399
|
return scale(researchDeadline);
|
|
330
400
|
}
|
|
@@ -418,6 +488,41 @@ export function repairableSqlFromAgentRun(run) {
|
|
|
418
488
|
}
|
|
419
489
|
return undefined;
|
|
420
490
|
}
|
|
491
|
+
/** Editable DQL retained on an immutable failed run, including its bound values. */
|
|
492
|
+
export function repairableDqlArtifactFromAgentRun(run) {
|
|
493
|
+
for (const artifact of run.artifacts) {
|
|
494
|
+
const payload = agentRunRecord(artifact.payload);
|
|
495
|
+
if (!payload)
|
|
496
|
+
continue;
|
|
497
|
+
const dqlArtifact = normalizeDqlArtifactReference(payload.dqlArtifact);
|
|
498
|
+
// Only an ownerless/custom SQL block is editable in place. Certified and
|
|
499
|
+
// semantic artifacts remain immutable; their retained compiled SQL may
|
|
500
|
+
// still produce a separate review-required repair result below.
|
|
501
|
+
if (dqlArtifact?.kind === 'sql_block'
|
|
502
|
+
&& dqlArtifact.source?.trim()
|
|
503
|
+
&& notebookDqlSourceAllowsBackgroundRepair(dqlArtifact.source)) {
|
|
504
|
+
return dqlArtifact;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
return undefined;
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* Keep the parameter surface immutable while repairing malformed DQL. A parser
|
|
511
|
+
* cannot reliably read a broken wrapper, so collect both declared parameter
|
|
512
|
+
* names and SQL interpolation names with a deliberately small lexical pass.
|
|
513
|
+
*/
|
|
514
|
+
export function dqlRepairParameterContract(source) {
|
|
515
|
+
const names = new Set();
|
|
516
|
+
for (const match of source.matchAll(/\$\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}/g)) {
|
|
517
|
+
names.add(match[1]);
|
|
518
|
+
}
|
|
519
|
+
for (const match of source.matchAll(/\bparams\s*\{([\s\S]*?)\}/gi)) {
|
|
520
|
+
for (const declaration of match[1].matchAll(/(?:^|\n)\s*([A-Za-z_][A-Za-z0-9_]*)\s*:/g)) {
|
|
521
|
+
names.add(declaration[1]);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return [...names].sort((left, right) => left.localeCompare(right));
|
|
525
|
+
}
|
|
421
526
|
/** Structured producer failure retained on the source run. */
|
|
422
527
|
export function warehouseFailureFromAgentRun(run) {
|
|
423
528
|
for (const artifact of run.artifacts) {
|
|
@@ -429,10 +534,49 @@ export function warehouseFailureFromAgentRun(run) {
|
|
|
429
534
|
}
|
|
430
535
|
return undefined;
|
|
431
536
|
}
|
|
537
|
+
/**
|
|
538
|
+
* Retain planning and retrieval evidence for a successful derived repair while
|
|
539
|
+
* deliberately dropping the source failure and any result-bound facts. This
|
|
540
|
+
* is the presentation contract used by "How it was answered" and by grounded
|
|
541
|
+
* follow-ups after repair.
|
|
542
|
+
*/
|
|
543
|
+
export function repairPresentationContextFromAgentRun(run) {
|
|
544
|
+
const retainedKeys = [
|
|
545
|
+
'resolvedAnalyticalPlan',
|
|
546
|
+
'analyticalExecutionGraph',
|
|
547
|
+
'analyticalFreshnessObservation',
|
|
548
|
+
'governedCompilationReceipt',
|
|
549
|
+
'governedAnalyticalGraphCompilationReceipt',
|
|
550
|
+
'contextPack',
|
|
551
|
+
'contextPackId',
|
|
552
|
+
'cascade',
|
|
553
|
+
'citations',
|
|
554
|
+
'lineage',
|
|
555
|
+
];
|
|
556
|
+
const context = {};
|
|
557
|
+
for (const artifact of run.artifacts) {
|
|
558
|
+
const payload = agentRunRecord(artifact.payload);
|
|
559
|
+
if (!payload)
|
|
560
|
+
continue;
|
|
561
|
+
for (const key of retainedKeys) {
|
|
562
|
+
if (context[key] === undefined && payload[key] !== undefined)
|
|
563
|
+
context[key] = payload[key];
|
|
564
|
+
}
|
|
565
|
+
const semanticTrace = agentRunRecord(payload.semanticExecutionTrace);
|
|
566
|
+
if (context.semanticExecutionTrace === undefined && semanticTrace && !semanticTrace.failure) {
|
|
567
|
+
context.semanticExecutionTrace = semanticTrace;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
return context;
|
|
571
|
+
}
|
|
432
572
|
/** Permission, authentication, policy and unsafe-query failures are never AI-repaired. */
|
|
433
573
|
export function analyticalFailureAllowsDeterministicRetry(failure) {
|
|
434
574
|
return !failure || !['permission', 'authentication', 'unsafe', 'cancelled'].includes(failure.category);
|
|
435
575
|
}
|
|
576
|
+
/** Structured policy/access refusals stay terminal even when retained SQL exists. */
|
|
577
|
+
export function analyticalFailureAllowsAppRepair(failure) {
|
|
578
|
+
return !failure || !['PERMISSION_DENIED', 'POLICY_DENIED'].includes(failure.code);
|
|
579
|
+
}
|
|
436
580
|
function businessNarrativeGaps(warnings) {
|
|
437
581
|
const businessRelevant = (warnings ?? []).filter((warning) => !(/missing requested output column/i.test(warning)
|
|
438
582
|
|| /\bquery plan\b/i.test(warning)
|
|
@@ -1011,11 +1155,11 @@ export async function startLocalServer(opts) {
|
|
|
1011
1155
|
const resolveNamedConnection = (name) => {
|
|
1012
1156
|
if (!name)
|
|
1013
1157
|
return null;
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1158
|
+
// `GET /api/connections` exposes the active legacy/default connection as
|
|
1159
|
+
// `default` so every execution surface can present one uniform selector.
|
|
1160
|
+
// Resolve that selector through the same project contract used by metadata
|
|
1161
|
+
// APIs instead of looking only in the newer named `connections` map.
|
|
1162
|
+
return projectConnectionById(projectRoot, projectConfig, name);
|
|
1019
1163
|
};
|
|
1020
1164
|
const resolveExecutionConnection = async (body) => {
|
|
1021
1165
|
const target = body.executionTarget && typeof body.executionTarget === "object"
|
|
@@ -1104,6 +1248,35 @@ export async function startLocalServer(opts) {
|
|
|
1104
1248
|
// unchanged. Explicit refresh, save, and certification paths still request
|
|
1105
1249
|
// the default forced rebuild below.
|
|
1106
1250
|
await refreshLocalMetadataCatalog(projectRoot, undefined, undefined, false);
|
|
1251
|
+
// PERF-001 / API-001: project mutations acknowledge after their source file
|
|
1252
|
+
// is durable. Rebuildable manifest, lineage, metadata, and KG projections run
|
|
1253
|
+
// in one coalesced worker lane and remain observable across page navigation.
|
|
1254
|
+
const operationCoordinator = new LocalOperationCoordinator(join(projectRoot, '.dql', 'cache', 'operations.sqlite'));
|
|
1255
|
+
const operationSseClients = new Set();
|
|
1256
|
+
const operationIdempotency = new Map();
|
|
1257
|
+
const projectRefreshCoordinator = new ProjectRefreshCoordinator(operationCoordinator, () => {
|
|
1258
|
+
projectSnapshots.invalidate();
|
|
1259
|
+
dbtNodeDetailCache.clear();
|
|
1260
|
+
invalidateAgentProjectState(projectRoot);
|
|
1261
|
+
_lineageCache.delete(projectRoot);
|
|
1262
|
+
});
|
|
1263
|
+
const scheduleProjectRefresh = (reason, writeManifest = true) => (projectRefreshCoordinator.schedule({
|
|
1264
|
+
projectRoot,
|
|
1265
|
+
dbtManifestPath: resolveDbtManifestPath(projectRoot, projectConfig) ?? undefined,
|
|
1266
|
+
writeManifest,
|
|
1267
|
+
reason,
|
|
1268
|
+
}));
|
|
1269
|
+
const unsubscribeOperationEvents = operationCoordinator.subscribe((operation) => {
|
|
1270
|
+
const payload = serializeJSON(operation);
|
|
1271
|
+
for (const client of operationSseClients) {
|
|
1272
|
+
try {
|
|
1273
|
+
client.write(`event: operation\ndata: ${payload}\n\n`);
|
|
1274
|
+
}
|
|
1275
|
+
catch {
|
|
1276
|
+
operationSseClients.delete(client);
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
});
|
|
1107
1280
|
const startupDbtManifestPath = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
1108
1281
|
if (startupDbtManifestPath && !isAgentProjectIndexReady(projectRoot)) {
|
|
1109
1282
|
try {
|
|
@@ -1185,8 +1358,8 @@ export async function startLocalServer(opts) {
|
|
|
1185
1358
|
catch { /* non-fatal */ }
|
|
1186
1359
|
}
|
|
1187
1360
|
}
|
|
1188
|
-
const executeLocalSqlForStoredResult = async (sql) => {
|
|
1189
|
-
const activeConnection = requireActiveConnection();
|
|
1361
|
+
const executeLocalSqlForStoredResult = async (sql, executionConnection) => {
|
|
1362
|
+
const activeConnection = requireActiveConnection(executionConnection);
|
|
1190
1363
|
const semantic = prepareSemanticSql(sql, semanticLayer);
|
|
1191
1364
|
if (semantic.unresolvedRefs.length > 0) {
|
|
1192
1365
|
throw new Error(`Unknown semantic reference${semantic.unresolvedRefs.length > 1 ? 's' : ''}: ${semantic.unresolvedRefs.join(', ')}`);
|
|
@@ -2453,6 +2626,23 @@ export async function startLocalServer(opts) {
|
|
|
2453
2626
|
const needsClarification = Boolean(plan.followUp);
|
|
2454
2627
|
const notebookPath = agentRunNotebookPath(request, runId);
|
|
2455
2628
|
const researchIntent = agentRunResearchIntent(request);
|
|
2629
|
+
const researchSource = agentRunRecord(request.workspaceContext?.researchSource);
|
|
2630
|
+
const researchExecutionConnection = !needsClarification && request.executionTarget
|
|
2631
|
+
? await resolveAgentRunExecutionConnection(request)
|
|
2632
|
+
: undefined;
|
|
2633
|
+
const researchExecutionConnectionName = request.executionTarget?.target === 'connection'
|
|
2634
|
+
? request.executionTarget.connectionName
|
|
2635
|
+
: undefined;
|
|
2636
|
+
const researchContextEnvelope = {
|
|
2637
|
+
surface: 'unified_agent_run',
|
|
2638
|
+
agentRunId: runId,
|
|
2639
|
+
routeDecision,
|
|
2640
|
+
selectedObject: request.selectedObject,
|
|
2641
|
+
executionTarget: request.executionTarget,
|
|
2642
|
+
workspaceContext: request.workspaceContext,
|
|
2643
|
+
researchSource,
|
|
2644
|
+
plan,
|
|
2645
|
+
};
|
|
2456
2646
|
let researchRun;
|
|
2457
2647
|
let researchWorkspaceError;
|
|
2458
2648
|
if (!needsClarification) {
|
|
@@ -2474,14 +2664,7 @@ export async function startLocalServer(opts) {
|
|
|
2474
2664
|
intent: researchIntent,
|
|
2475
2665
|
domain: agentRunWorkspaceValue(request, 'domain'),
|
|
2476
2666
|
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
2477
|
-
context:
|
|
2478
|
-
surface: 'unified_agent_run',
|
|
2479
|
-
agentRunId: runId,
|
|
2480
|
-
routeDecision,
|
|
2481
|
-
selectedObject: request.selectedObject,
|
|
2482
|
-
workspaceContext: request.workspaceContext,
|
|
2483
|
-
plan,
|
|
2484
|
-
},
|
|
2667
|
+
context: researchContextEnvelope,
|
|
2485
2668
|
});
|
|
2486
2669
|
emit({
|
|
2487
2670
|
type: 'artifact.created',
|
|
@@ -2496,14 +2679,13 @@ export async function startLocalServer(opts) {
|
|
|
2496
2679
|
sourceCellFingerprint,
|
|
2497
2680
|
question: request.question,
|
|
2498
2681
|
intent: researchIntent,
|
|
2499
|
-
context:
|
|
2500
|
-
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2506
|
-
},
|
|
2682
|
+
context: researchContextEnvelope,
|
|
2683
|
+
executionConnection: researchExecutionConnection,
|
|
2684
|
+
executionConnectionName: researchExecutionConnectionName,
|
|
2685
|
+
signal: request.signal,
|
|
2686
|
+
baselineSql: agentRunString(researchSource?.sql),
|
|
2687
|
+
baselineDqlArtifact: researchSource?.dqlArtifact,
|
|
2688
|
+
baselineRunId: agentRunString(researchSource?.runId),
|
|
2507
2689
|
});
|
|
2508
2690
|
researchRun = withNotebookResearchChecklist(executed);
|
|
2509
2691
|
}
|
|
@@ -2816,7 +2998,7 @@ export async function startLocalServer(opts) {
|
|
|
2816
2998
|
message: `Generating review-required SQL for an uncovered question: ${question}`,
|
|
2817
2999
|
route: 'app_build',
|
|
2818
3000
|
});
|
|
2819
|
-
return
|
|
3001
|
+
return generateAppBuildAnswerWithRepair(question);
|
|
2820
3002
|
},
|
|
2821
3003
|
});
|
|
2822
3004
|
}
|
|
@@ -3474,7 +3656,7 @@ export async function startLocalServer(opts) {
|
|
|
3474
3656
|
* string, so the only remaining reasons to differ are the connection and the
|
|
3475
3657
|
* governance gates, both of which report themselves honestly.
|
|
3476
3658
|
*/
|
|
3477
|
-
const executeGeneratedSqlDirect = async (question, sql, seed, executionConnection) => {
|
|
3659
|
+
const executeGeneratedSqlDirect = async (question, sql, seed, executionConnection, bindings) => {
|
|
3478
3660
|
const activeConnection = requireActiveConnection(executionConnection);
|
|
3479
3661
|
const rowBound = clampAnalyticalRowBound(seed?.limit ?? 200);
|
|
3480
3662
|
const trimmed = sql.trim().replace(/;\s*$/, '').trim();
|
|
@@ -3535,7 +3717,7 @@ export async function startLocalServer(opts) {
|
|
|
3535
3717
|
})
|
|
3536
3718
|
: null;
|
|
3537
3719
|
const rawResult = semanticExecution?.result
|
|
3538
|
-
?? await executor.executeQuery(preparation.executedSql, [], runtimeVariables({}), preparation.connection);
|
|
3720
|
+
?? await executor.executeQuery(preparation.executedSql, bindings?.sqlParams ?? [], runtimeVariables(bindings?.variables ?? {}), preparation.connection);
|
|
3539
3721
|
const normalized = normalizeQueryResult(rawResult, semantic.semanticRefs);
|
|
3540
3722
|
// The authoritative bound. `buildRowBoundedSql` may legitimately decline to
|
|
3541
3723
|
// touch the statement, and only one of 15 drivers honours a maxRows hint, so
|
|
@@ -3716,13 +3898,15 @@ export async function startLocalServer(opts) {
|
|
|
3716
3898
|
* evidence. The refusal can say the relation is not visible to this
|
|
3717
3899
|
* connection, rather than only that DQL had not looked at it.
|
|
3718
3900
|
*/
|
|
3719
|
-
const probeNamedRelationsForAgent = async (relations, probeConnection) => {
|
|
3901
|
+
const probeNamedRelationsForAgent = async (relations, probeConnection, verifyExecutionSchema = false) => {
|
|
3720
3902
|
const probeTarget = probeConnection ?? connection;
|
|
3721
3903
|
if (!probeTarget)
|
|
3722
3904
|
return undefined;
|
|
3723
|
-
//
|
|
3724
|
-
//
|
|
3725
|
-
|
|
3905
|
+
// Named relation discovery remains behind the operator's live-grounding
|
|
3906
|
+
// switch. Executable schema verification is different: it is a bounded,
|
|
3907
|
+
// read-only metadata point lookup for relations already selected from dbt,
|
|
3908
|
+
// and never reads row values or mutates the active target.
|
|
3909
|
+
if (!verifyExecutionSchema && resolveAgentRuntimeValueGrounding(projectConfig).mode !== 'safe_automatic')
|
|
3726
3910
|
return undefined;
|
|
3727
3911
|
const sql = buildNamedRelationProbeSql(relations);
|
|
3728
3912
|
if (!sql)
|
|
@@ -3775,12 +3959,29 @@ export async function startLocalServer(opts) {
|
|
|
3775
3959
|
&& activeRuntimeSnapshot.status !== 'stale');
|
|
3776
3960
|
const valueGrounding = resolveAgentRuntimeValueGrounding(projectConfig);
|
|
3777
3961
|
if (catalogContext.length > 0) {
|
|
3778
|
-
if (!schemaConnection
|
|
3962
|
+
if (!schemaConnection)
|
|
3779
3963
|
return catalogContext;
|
|
3780
|
-
//
|
|
3781
|
-
//
|
|
3782
|
-
//
|
|
3783
|
-
|
|
3964
|
+
// dbt metadata is the semantic authority, but generated SQL must use the
|
|
3965
|
+
// physical columns visible on THIS execution target. Verify only the few
|
|
3966
|
+
// question-ranked relations with a bounded information_schema point lookup;
|
|
3967
|
+
// this is read-only and never mutates DuckDB, Snowflake, or any other target.
|
|
3968
|
+
let executionSchema = catalogContext;
|
|
3969
|
+
try {
|
|
3970
|
+
const live = await probeNamedRelationsForAgent(catalogContext.slice(0, 4).map((table) => table.relation), schemaConnection, true);
|
|
3971
|
+
if (live?.schemaContext?.length) {
|
|
3972
|
+
executionSchema = reconcileAgentSchemaContextWithLive(catalogContext, live.schemaContext);
|
|
3973
|
+
}
|
|
3974
|
+
}
|
|
3975
|
+
catch {
|
|
3976
|
+
// Live verification is additive. Existing metadata remains available if
|
|
3977
|
+
// the connection cannot expose information_schema to this user.
|
|
3978
|
+
}
|
|
3979
|
+
// The immutable dbt/DQL snapshot owns schema discovery. Live verification
|
|
3980
|
+
// only confirms those named relations; row values remain governed by the
|
|
3981
|
+
// separate allowlisted grounding policy below.
|
|
3982
|
+
if (valueGrounding.mode !== 'safe_automatic')
|
|
3983
|
+
return executionSchema;
|
|
3984
|
+
const enriched = await enrichAgentSchemaContextWithValueMatches(question, executionSchema, executor, schemaConnection, valueGrounding.searchSafeColumns);
|
|
3784
3985
|
return enriched;
|
|
3785
3986
|
}
|
|
3786
3987
|
// An activated, scope-qualified warehouse generation is the complete warm
|
|
@@ -3803,83 +4004,474 @@ export async function startLocalServer(opts) {
|
|
|
3803
4004
|
return [];
|
|
3804
4005
|
}
|
|
3805
4006
|
};
|
|
3806
|
-
const
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
4007
|
+
const boundedRepairError = (message, status, warehouseFailure) => Object.assign(new Error(message), { status, ...(warehouseFailure ? { warehouseFailure } : {}) });
|
|
4008
|
+
/**
|
|
4009
|
+
* Shared one-attempt SQL repair for Ask and Notebook. It never creates a
|
|
4010
|
+
* conversation, writes a DQL file, changes targets, or promotes trust.
|
|
4011
|
+
*/
|
|
4012
|
+
const executeBoundedSqlRepair = async (input) => {
|
|
4013
|
+
let result;
|
|
4014
|
+
let repairedSql = input.sourceSql;
|
|
4015
|
+
let repairMode = 'deterministic';
|
|
4016
|
+
let firstFailure;
|
|
4017
|
+
try {
|
|
4018
|
+
result = await executeGeneratedSqlDirect(input.question, input.sourceSql, undefined, input.targetConnection, { sqlParams: input.sqlParams, variables: input.variables });
|
|
3812
4019
|
}
|
|
3813
|
-
|
|
3814
|
-
|
|
3815
|
-
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3821
|
-
|
|
3822
|
-
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3826
|
-
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
4020
|
+
catch (error) {
|
|
4021
|
+
firstFailure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
4022
|
+
}
|
|
4023
|
+
if (!result) {
|
|
4024
|
+
if (firstFailure?.retryDisposition !== 'model_repair') {
|
|
4025
|
+
throw boundedRepairError(firstFailure?.redactedMessage ?? 'The deterministic repair did not produce an executable query.', 422, firstFailure);
|
|
4026
|
+
}
|
|
4027
|
+
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
4028
|
+
if (!provider) {
|
|
4029
|
+
throw boundedRepairError('Configure an available AI provider in Settings to repair this query.', 409);
|
|
4030
|
+
}
|
|
4031
|
+
let schemaContext;
|
|
4032
|
+
try {
|
|
4033
|
+
schemaContext = await getSchemaContextForAgent(input.question, undefined, input.targetConnection, input.targetConnectionName);
|
|
4034
|
+
}
|
|
4035
|
+
catch (error) {
|
|
4036
|
+
const failure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
4037
|
+
throw boundedRepairError(failure.redactedMessage, 422, failure);
|
|
4038
|
+
}
|
|
4039
|
+
const boundedSchema = schemaContext.slice(0, 16).map((table) => ({
|
|
4040
|
+
relation: table.relation,
|
|
4041
|
+
columns: table.columns.slice(0, 40).map((column) => ({ name: column.name, type: column.type })),
|
|
4042
|
+
}));
|
|
4043
|
+
let raw;
|
|
4044
|
+
try {
|
|
4045
|
+
raw = await provider.generate([
|
|
4046
|
+
{
|
|
4047
|
+
role: 'system',
|
|
4048
|
+
content: [
|
|
4049
|
+
'Repair one failed read-only analytical SQL statement.',
|
|
4050
|
+
'Return exactly one JSON object in a ```json fence with keys summary, sql, viz, outputs.',
|
|
4051
|
+
'Preserve the requested grain, filters, output columns, and aggregation semantics.',
|
|
4052
|
+
input.sqlParams?.length
|
|
4053
|
+
? 'Preserve every positional parameter placeholder exactly; do not add, remove, or renumber placeholders.'
|
|
4054
|
+
: '',
|
|
4055
|
+
'Use only relations and columns present in the supplied schema context.',
|
|
4056
|
+
'Never emit internal DQL relation identities such as source::, model::, seed::, or snapshot::.',
|
|
4057
|
+
'Do not add writes, DDL, multiple statements, or commentary outside the JSON fence.',
|
|
4058
|
+
].join(' '),
|
|
4059
|
+
},
|
|
4060
|
+
{
|
|
4061
|
+
role: 'user',
|
|
4062
|
+
content: JSON.stringify({
|
|
4063
|
+
question: input.question,
|
|
4064
|
+
driver: input.targetConnection.driver,
|
|
4065
|
+
failure: firstFailure.redactedMessage,
|
|
4066
|
+
sql: input.sourceSql,
|
|
4067
|
+
parameters: input.sqlParams?.map((parameter) => ({
|
|
4068
|
+
name: parameter.name,
|
|
4069
|
+
placeholder: `$${parameter.position}`,
|
|
4070
|
+
})),
|
|
4071
|
+
schema: boundedSchema,
|
|
4072
|
+
}),
|
|
4073
|
+
},
|
|
4074
|
+
], { maxTokens: 1200, temperature: 0 });
|
|
4075
|
+
}
|
|
4076
|
+
catch {
|
|
4077
|
+
throw boundedRepairError('The AI provider could not complete this repair. Check Provider Settings and retry.', 502);
|
|
4078
|
+
}
|
|
4079
|
+
const proposal = parseProposal(raw);
|
|
4080
|
+
if (!proposal.sql?.trim()) {
|
|
4081
|
+
throw boundedRepairError('The AI repair did not return an executable SQL statement.', 422);
|
|
4082
|
+
}
|
|
4083
|
+
repairedSql = proposal.sql.trim();
|
|
4084
|
+
const requiredPlaceholders = (input.sqlParams ?? []).map((parameter) => `$${parameter.position}`);
|
|
4085
|
+
if (requiredPlaceholders.some((placeholder) => !new RegExp(`${placeholder.replace('$', '\\$')}\\b`).test(repairedSql))) {
|
|
4086
|
+
throw boundedRepairError('The repair changed this cell\'s parameter contract, so DQL stopped it before execution.', 422);
|
|
4087
|
+
}
|
|
4088
|
+
repairMode = 'ai';
|
|
4089
|
+
try {
|
|
4090
|
+
result = await executeGeneratedSqlDirect(input.question, repairedSql, undefined, input.targetConnection, { sqlParams: input.sqlParams, variables: input.variables });
|
|
4091
|
+
}
|
|
4092
|
+
catch (error) {
|
|
4093
|
+
const failure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
4094
|
+
throw boundedRepairError(failure.redactedMessage, 422, failure);
|
|
4095
|
+
}
|
|
4096
|
+
}
|
|
4097
|
+
// Keep preview-only row bounds out of editable notebook source. The one
|
|
4098
|
+
// deterministic source rewrite we do persist is decoding internal graph
|
|
4099
|
+
// identities, because those can never execute as warehouse SQL.
|
|
4100
|
+
let rewrites = [];
|
|
4101
|
+
if (internalRelationIdsInSql(repairedSql).length > 0) {
|
|
4102
|
+
const normalizedSource = await prepareAnalyticalExecutionSql({
|
|
4103
|
+
sql: repairedSql,
|
|
4104
|
+
subject: 'Repaired SQL',
|
|
4105
|
+
executor,
|
|
4106
|
+
connection: input.targetConnection,
|
|
4107
|
+
projectRoot,
|
|
4108
|
+
projectConfig,
|
|
4109
|
+
enforceReadOnly: true,
|
|
4110
|
+
});
|
|
4111
|
+
repairedSql = normalizedSource.decodedSql;
|
|
4112
|
+
rewrites = normalizedSource.rewrites;
|
|
4113
|
+
}
|
|
4114
|
+
return { result, repairedSql, repairMode, firstFailure, rewrites };
|
|
4115
|
+
};
|
|
4116
|
+
/**
|
|
4117
|
+
* Repair a DQL wrapper that cannot compile far enough to expose executable
|
|
4118
|
+
* SQL. This is the DQL-side half of the same bounded repair contract used by
|
|
4119
|
+
* Ask and Notebook: one provider call, one block, identical parameters, one
|
|
4120
|
+
* read-only execution, and the exact target selected by the failed run.
|
|
4121
|
+
*/
|
|
4122
|
+
const executeBoundedDqlRepair = async (input) => {
|
|
4123
|
+
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
4124
|
+
if (!provider) {
|
|
4125
|
+
throw boundedRepairError('Configure an available AI provider in Settings to repair this DQL cell.', 409);
|
|
4126
|
+
}
|
|
4127
|
+
let schemaContext;
|
|
4128
|
+
try {
|
|
4129
|
+
// Include source identifiers in retrieval so a terse cell title still
|
|
4130
|
+
// resolves the physical dbt relations and live target columns it uses.
|
|
4131
|
+
schemaContext = await getSchemaContextForAgent(`${input.question}\n${input.source.slice(0, 6_000)}`, undefined, input.targetConnection, input.targetConnectionName);
|
|
4132
|
+
}
|
|
4133
|
+
catch (error) {
|
|
4134
|
+
const failure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
4135
|
+
throw boundedRepairError(failure.redactedMessage, 422, failure);
|
|
4136
|
+
}
|
|
4137
|
+
const boundedSchema = schemaContext.slice(0, 16).map((table) => ({
|
|
4138
|
+
relation: table.relation,
|
|
4139
|
+
columns: table.columns.slice(0, 40).map((column) => ({ name: column.name, type: column.type })),
|
|
4140
|
+
}));
|
|
4141
|
+
let raw;
|
|
4142
|
+
try {
|
|
4143
|
+
raw = await provider.generate([
|
|
4144
|
+
{
|
|
4145
|
+
role: 'system',
|
|
4146
|
+
content: [
|
|
4147
|
+
'Repair one malformed executable DQL custom block and its embedded read-only SQL.',
|
|
4148
|
+
'Return only one JSON object with exactly one key named dql. Do not use markdown.',
|
|
4149
|
+
'The DQL shape is: block "Name" { type = "custom" params { name: type = default } query = """SELECT ...""" visualization { ... } tests { ... } }.',
|
|
4150
|
+
'Preserve the block name, business fields, visualization, tests, requested grain, filters, output columns, and aggregation semantics.',
|
|
4151
|
+
'Preserve every declared parameter and every ${parameter} interpolation exactly. Do not add or remove parameters.',
|
|
4152
|
+
'Use only relations and columns present in the supplied schema context.',
|
|
4153
|
+
'Never emit internal relation identities such as source::, model::, seed::, or snapshot::.',
|
|
4154
|
+
'Return exactly one custom block. Never certify it, bind a saved block, add writes, DDL, or multiple SQL statements.',
|
|
4155
|
+
].join(' '),
|
|
4156
|
+
},
|
|
4157
|
+
{
|
|
3850
4158
|
role: 'user',
|
|
3851
|
-
content:
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
}
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
4159
|
+
content: JSON.stringify({
|
|
4160
|
+
question: input.question,
|
|
4161
|
+
driver: input.targetConnection.driver,
|
|
4162
|
+
failure: input.failure.slice(0, 2_000),
|
|
4163
|
+
dql: input.source.slice(0, 40_000),
|
|
4164
|
+
parameters: input.parameters ?? {},
|
|
4165
|
+
schema: boundedSchema,
|
|
4166
|
+
}),
|
|
4167
|
+
},
|
|
4168
|
+
], { maxTokens: 2400, temperature: 0 });
|
|
4169
|
+
}
|
|
4170
|
+
catch {
|
|
4171
|
+
throw boundedRepairError('The AI provider could not complete this DQL repair. Check Provider Settings and retry.', 502);
|
|
4172
|
+
}
|
|
4173
|
+
let repairedSource;
|
|
4174
|
+
try {
|
|
4175
|
+
const parsed = parseFirstJsonObject(raw);
|
|
4176
|
+
repairedSource = parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
4177
|
+
&& typeof parsed.dql === 'string'
|
|
4178
|
+
? String(parsed.dql).trim()
|
|
4179
|
+
: undefined;
|
|
4180
|
+
}
|
|
4181
|
+
catch {
|
|
4182
|
+
repairedSource = undefined;
|
|
4183
|
+
}
|
|
4184
|
+
if (!repairedSource) {
|
|
4185
|
+
throw boundedRepairError('The AI repair did not return a complete DQL block.', 422);
|
|
4186
|
+
}
|
|
4187
|
+
if (repairedSource.length > 50_000) {
|
|
4188
|
+
throw boundedRepairError('The repaired DQL exceeded the safe editable-cell size limit.', 422);
|
|
4189
|
+
}
|
|
4190
|
+
if (/^\s*@block\(/i.test(repairedSource) || /\btype\s*=\s*"semantic"/i.test(repairedSource)) {
|
|
4191
|
+
throw boundedRepairError('The repair changed this editable custom block into a governed reference or semantic block.', 422);
|
|
4192
|
+
}
|
|
4193
|
+
if (!/\bstatus\s*=\s*"certified"/i.test(input.source)
|
|
4194
|
+
&& /\bstatus\s*=\s*"certified"/i.test(repairedSource)) {
|
|
4195
|
+
throw boundedRepairError('The repair attempted to certify an AI-edited block, so DQL stopped it.', 422);
|
|
4196
|
+
}
|
|
4197
|
+
const beforeParameters = dqlRepairParameterContract(input.source);
|
|
4198
|
+
const afterParameters = dqlRepairParameterContract(repairedSource);
|
|
4199
|
+
if (beforeParameters.join('|') !== afterParameters.join('|')) {
|
|
4200
|
+
throw boundedRepairError('The repair changed this block\'s parameter contract, so DQL stopped it before execution.', 422);
|
|
4201
|
+
}
|
|
4202
|
+
let invocation;
|
|
4203
|
+
let plan;
|
|
4204
|
+
try {
|
|
4205
|
+
const program = new Parser(repairedSource, '<bounded-dql-repair>').parse();
|
|
4206
|
+
const blocks = program.statements.filter((statement) => statement.kind === NodeKind.BlockDecl);
|
|
4207
|
+
if (blocks.length !== 1 || program.statements.length !== 1) {
|
|
4208
|
+
throw new Error('The repaired source must contain exactly one DQL block.');
|
|
4209
|
+
}
|
|
4210
|
+
const block = blocks[0];
|
|
4211
|
+
if (block.kind !== NodeKind.BlockDecl || block.blockType !== 'custom') {
|
|
4212
|
+
throw new Error('The repaired source must remain a custom DQL block.');
|
|
4213
|
+
}
|
|
4214
|
+
invocation = prepareBlockInvocation({
|
|
4215
|
+
source: repairedSource,
|
|
4216
|
+
parameters: input.parameters,
|
|
4217
|
+
question: input.question,
|
|
4218
|
+
surface: 'notebook',
|
|
4219
|
+
});
|
|
4220
|
+
if (invocation.errors.length)
|
|
4221
|
+
throw new Error(invocation.errors.join(' '));
|
|
4222
|
+
if (invocation.unresolvedParameters.length) {
|
|
4223
|
+
throw new Error(`Provide required parameters before retrying: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
4224
|
+
}
|
|
4225
|
+
plan = buildExecutionPlan({
|
|
4226
|
+
id: 'bounded-dql-repair',
|
|
4227
|
+
type: 'dql',
|
|
4228
|
+
source: repairedSource,
|
|
4229
|
+
title: input.artifactName,
|
|
4230
|
+
}, {
|
|
4231
|
+
semanticLayer,
|
|
4232
|
+
driver: input.targetConnection.driver,
|
|
4233
|
+
parameters: invocation.values,
|
|
4234
|
+
});
|
|
4235
|
+
if (!plan?.sql?.trim())
|
|
4236
|
+
throw new Error('The repaired DQL did not compile to SQL.');
|
|
4237
|
+
}
|
|
4238
|
+
catch (error) {
|
|
4239
|
+
throw boundedRepairError(`The repaired DQL did not compile safely: ${error instanceof Error ? error.message : String(error)}`, 422);
|
|
4240
|
+
}
|
|
4241
|
+
const seed = {
|
|
4242
|
+
kind: 'sql_block',
|
|
4243
|
+
source: repairedSource,
|
|
4244
|
+
name: input.artifactName ?? plan.title,
|
|
4245
|
+
persistence: 'transient',
|
|
4246
|
+
trustState: 'review_required',
|
|
4247
|
+
compiledSql: plan.sql,
|
|
4248
|
+
...(invocation.parameters.length ? { parameters: invocation.parameters } : {}),
|
|
4249
|
+
...(Object.keys(invocation.values).length ? { parameterValues: invocation.values } : {}),
|
|
4250
|
+
};
|
|
4251
|
+
let result;
|
|
4252
|
+
try {
|
|
4253
|
+
result = await executeGeneratedSqlDirect(input.question, plan.sql, seed, input.targetConnection, { sqlParams: plan.sqlParams, variables: { ...plan.variables, ...invocation.values } });
|
|
4254
|
+
}
|
|
4255
|
+
catch (error) {
|
|
4256
|
+
const failure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
4257
|
+
throw boundedRepairError(failure.redactedMessage, 422, failure);
|
|
4258
|
+
}
|
|
4259
|
+
result = {
|
|
4260
|
+
...result,
|
|
4261
|
+
dqlArtifact: {
|
|
4262
|
+
...(result.dqlArtifact ?? seed),
|
|
4263
|
+
source: repairedSource,
|
|
4264
|
+
compiledSql: result.sql ?? plan.sql,
|
|
4265
|
+
trustState: 'review_required',
|
|
4266
|
+
persistence: 'transient',
|
|
4267
|
+
...(result.executionReceipt ? { executionReceipt: result.executionReceipt } : {}),
|
|
3863
4268
|
},
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
4269
|
+
};
|
|
4270
|
+
return {
|
|
4271
|
+
result,
|
|
4272
|
+
repairedSql: plan.sql,
|
|
4273
|
+
repairedSource,
|
|
4274
|
+
repairMode: 'ai',
|
|
4275
|
+
rewrites: [],
|
|
4276
|
+
sqlParams: plan.sqlParams,
|
|
4277
|
+
variables: { ...plan.variables, ...invocation.values },
|
|
4278
|
+
};
|
|
4279
|
+
};
|
|
4280
|
+
/**
|
|
4281
|
+
* App proposal parity with Ask/Notebook repair. The first governed attempt is
|
|
4282
|
+
* retained unchanged. Only an execution failure with retained read-only SQL
|
|
4283
|
+
* may spend the shared one-attempt repair budget, on the same connection.
|
|
4284
|
+
* The repaired candidate is still exploratory and review-required.
|
|
4285
|
+
*/
|
|
4286
|
+
const generateAppBuildAnswerWithRepair = async (question, appContext) => {
|
|
4287
|
+
const request = {
|
|
4288
|
+
question,
|
|
4289
|
+
audience: 'stakeholder',
|
|
4290
|
+
workspaceContext: appContext
|
|
4291
|
+
? { surface: 'app_builder', appContext }
|
|
4292
|
+
: { surface: 'app_builder' },
|
|
4293
|
+
};
|
|
4294
|
+
const answer = await runGovernedAgentAnswerForRun(request);
|
|
4295
|
+
const sourceSql = answer.proposedSql?.trim() || answer.sql?.trim();
|
|
4296
|
+
const hasExecutionFailure = agentAnswerHasExecutionFailure(answer)
|
|
4297
|
+
|| Boolean(answer.warehouseFailure)
|
|
4298
|
+
|| Boolean(answer.analyticalFailure);
|
|
4299
|
+
if (answer.result || !sourceSql || !hasExecutionFailure)
|
|
4300
|
+
return answer;
|
|
4301
|
+
const attemptedAt = new Date().toISOString();
|
|
4302
|
+
const originalFailure = answer.warehouseFailure?.redactedMessage
|
|
4303
|
+
?? answer.executionError
|
|
4304
|
+
?? 'The generated query did not execute.';
|
|
4305
|
+
const originalSqlFingerprint = `sha256:${createHash('sha256').update(sourceSql).digest('hex')}`;
|
|
4306
|
+
const failedTrace = (message) => ({
|
|
4307
|
+
version: 1,
|
|
4308
|
+
status: 'failed',
|
|
4309
|
+
source: 'query_generator',
|
|
4310
|
+
attemptedAt,
|
|
4311
|
+
originalFailure,
|
|
4312
|
+
originalSqlFingerprint,
|
|
4313
|
+
approvalEligible: false,
|
|
4314
|
+
message,
|
|
4315
|
+
});
|
|
4316
|
+
if (!analyticalFailureAllowsDeterministicRetry(answer.warehouseFailure)
|
|
4317
|
+
|| !analyticalFailureAllowsAppRepair(answer.analyticalFailure)) {
|
|
4318
|
+
return {
|
|
4319
|
+
...answer,
|
|
4320
|
+
appBuilderRepair: failedTrace('This failure needs an access, policy, or connection decision and was not sent to AI repair.'),
|
|
4321
|
+
};
|
|
4322
|
+
}
|
|
4323
|
+
try {
|
|
4324
|
+
const targetConnection = await resolveAgentRunExecutionConnection(request);
|
|
4325
|
+
const targetConnectionName = request.executionTarget?.target === 'connection'
|
|
4326
|
+
? request.executionTarget.connectionName
|
|
4327
|
+
: request.executionTarget?.target === 'local'
|
|
4328
|
+
? 'local'
|
|
4329
|
+
: projectConfig.defaultConnectionName;
|
|
4330
|
+
const repaired = await executeBoundedSqlRepair({
|
|
4331
|
+
question,
|
|
4332
|
+
sourceSql,
|
|
4333
|
+
targetConnection,
|
|
4334
|
+
targetConnectionName,
|
|
4335
|
+
});
|
|
4336
|
+
const repair = {
|
|
4337
|
+
version: 1,
|
|
4338
|
+
status: 'repaired',
|
|
4339
|
+
source: 'query_generator',
|
|
4340
|
+
mode: repaired.repairMode,
|
|
4341
|
+
attemptedAt,
|
|
4342
|
+
originalFailure,
|
|
4343
|
+
originalSqlFingerprint,
|
|
4344
|
+
repairedSqlFingerprint: `sha256:${createHash('sha256').update(repaired.repairedSql).digest('hex')}`,
|
|
4345
|
+
approvalEligible: false,
|
|
4346
|
+
message: `${repaired.repairMode === 'ai' ? 'AI' : 'Deterministic'} repair executed successfully on the same data target. Review the repaired query before adding it to the private App draft.`,
|
|
4347
|
+
};
|
|
4348
|
+
return {
|
|
4349
|
+
...answer,
|
|
4350
|
+
result: repaired.result,
|
|
4351
|
+
proposedSql: repaired.repairedSql,
|
|
4352
|
+
sql: repaired.repairedSql,
|
|
4353
|
+
executionError: undefined,
|
|
4354
|
+
warehouseFailure: undefined,
|
|
4355
|
+
analyticalFailure: undefined,
|
|
4356
|
+
validationWarnings: [
|
|
4357
|
+
...(answer.validationWarnings ?? []),
|
|
4358
|
+
repair.message,
|
|
4359
|
+
],
|
|
4360
|
+
appBuilderRepair: repair,
|
|
4361
|
+
};
|
|
4362
|
+
}
|
|
4363
|
+
catch (error) {
|
|
4364
|
+
return {
|
|
4365
|
+
...answer,
|
|
4366
|
+
appBuilderRepair: failedTrace(error instanceof Error
|
|
4367
|
+
? `The bounded repair could not complete: ${error.message}`
|
|
4368
|
+
: 'The bounded repair could not complete.'),
|
|
4369
|
+
};
|
|
4370
|
+
}
|
|
4371
|
+
};
|
|
4372
|
+
const repairFailedAppTileExecution = async (input) => {
|
|
4373
|
+
const attemptedAt = new Date().toISOString();
|
|
4374
|
+
const driver = input.targetConnection?.driver ?? connection?.driver ?? 'duckdb';
|
|
4375
|
+
const failure = normalizeWarehouseSqlFailure(input.error, driver);
|
|
4376
|
+
const originalFailure = failure.redactedMessage;
|
|
4377
|
+
const originalSqlFingerprint = input.sourceSql
|
|
4378
|
+
? `sha256:${createHash('sha256').update(input.sourceSql).digest('hex')}`
|
|
4379
|
+
: undefined;
|
|
4380
|
+
const failed = (message) => ({
|
|
4381
|
+
ok: false,
|
|
4382
|
+
repair: {
|
|
4383
|
+
version: 1,
|
|
4384
|
+
status: 'failed',
|
|
4385
|
+
source: input.source,
|
|
4386
|
+
attemptedAt,
|
|
4387
|
+
originalFailure,
|
|
4388
|
+
...(originalSqlFingerprint ? { originalSqlFingerprint } : {}),
|
|
4389
|
+
approvalEligible: false,
|
|
4390
|
+
message,
|
|
4391
|
+
},
|
|
4392
|
+
});
|
|
4393
|
+
if (!input.targetConnection) {
|
|
4394
|
+
return failed('Configure the original data target before retrying this App tile.');
|
|
4395
|
+
}
|
|
4396
|
+
if (!input.sourceSql?.trim() && input.sourceDql?.trim()) {
|
|
4397
|
+
try {
|
|
4398
|
+
const repaired = await executeBoundedDqlRepair({
|
|
4399
|
+
question: input.question,
|
|
4400
|
+
source: input.sourceDql,
|
|
4401
|
+
failure: originalFailure,
|
|
4402
|
+
targetConnection: input.targetConnection,
|
|
4403
|
+
targetConnectionName: input.targetConnectionName,
|
|
4404
|
+
parameters: input.variables,
|
|
4405
|
+
artifactName: input.artifactName,
|
|
4406
|
+
});
|
|
4407
|
+
const repair = {
|
|
4408
|
+
version: 1,
|
|
4409
|
+
status: 'repaired',
|
|
4410
|
+
source: input.source,
|
|
4411
|
+
mode: repaired.repairMode,
|
|
4412
|
+
attemptedAt,
|
|
4413
|
+
originalFailure,
|
|
4414
|
+
repairedSqlFingerprint: `sha256:${createHash('sha256').update(repaired.repairedSql).digest('hex')}`,
|
|
4415
|
+
approvalEligible: false,
|
|
4416
|
+
message: `${repaired.repairMode === 'ai' ? 'AI' : 'Deterministic'} DQL repair executed successfully on the same data target. The App result remains review-required and the source file was not changed.`,
|
|
4417
|
+
};
|
|
4418
|
+
return {
|
|
4419
|
+
ok: true,
|
|
4420
|
+
result: repaired.result,
|
|
4421
|
+
repairedSql: repaired.repairedSql,
|
|
4422
|
+
repairedDql: repaired.repairedSource,
|
|
4423
|
+
repair,
|
|
4424
|
+
};
|
|
3873
4425
|
}
|
|
3874
|
-
|
|
3875
|
-
|
|
4426
|
+
catch (error) {
|
|
4427
|
+
return failed(error instanceof Error
|
|
4428
|
+
? `The bounded DQL repair could not complete: ${error.message}`
|
|
4429
|
+
: 'The bounded DQL repair could not complete.');
|
|
3876
4430
|
}
|
|
3877
|
-
}, controller.signal);
|
|
3878
|
-
if (!governedAnswer) {
|
|
3879
|
-
throw new Error(providerError ?? 'The AI provider did not return a governed answer.');
|
|
3880
4431
|
}
|
|
4432
|
+
if (!input.sourceSql?.trim()) {
|
|
4433
|
+
return failed('DQL retained the failure, but no read-only SQL or editable DQL was available for bounded repair.');
|
|
4434
|
+
}
|
|
4435
|
+
if (!analyticalFailureAllowsDeterministicRetry(failure)) {
|
|
4436
|
+
return failed('This failure needs an access, policy, or connection decision and was not sent to AI repair.');
|
|
4437
|
+
}
|
|
4438
|
+
try {
|
|
4439
|
+
const repaired = await executeBoundedSqlRepair({
|
|
4440
|
+
question: input.question,
|
|
4441
|
+
sourceSql: input.sourceSql,
|
|
4442
|
+
targetConnection: input.targetConnection,
|
|
4443
|
+
targetConnectionName: input.targetConnectionName,
|
|
4444
|
+
sqlParams: input.sqlParams,
|
|
4445
|
+
variables: input.variables,
|
|
4446
|
+
});
|
|
4447
|
+
const repair = {
|
|
4448
|
+
version: 1,
|
|
4449
|
+
status: 'repaired',
|
|
4450
|
+
source: input.source,
|
|
4451
|
+
mode: repaired.repairMode,
|
|
4452
|
+
attemptedAt,
|
|
4453
|
+
originalFailure,
|
|
4454
|
+
...(originalSqlFingerprint ? { originalSqlFingerprint } : {}),
|
|
4455
|
+
repairedSqlFingerprint: `sha256:${createHash('sha256').update(repaired.repairedSql).digest('hex')}`,
|
|
4456
|
+
approvalEligible: false,
|
|
4457
|
+
message: `${repaired.repairMode === 'ai' ? 'AI' : 'Deterministic'} repair executed successfully on the same data target. The result remains review-required.`,
|
|
4458
|
+
};
|
|
4459
|
+
return { ok: true, result: repaired.result, repairedSql: repaired.repairedSql, repair };
|
|
4460
|
+
}
|
|
4461
|
+
catch (error) {
|
|
4462
|
+
return failed(error instanceof Error
|
|
4463
|
+
? `The bounded repair could not complete: ${error.message}`
|
|
4464
|
+
: 'The bounded repair could not complete.');
|
|
4465
|
+
}
|
|
4466
|
+
};
|
|
4467
|
+
const generateInvestigationSqlForApp = async (input) => {
|
|
4468
|
+
const request = buildAppCopilotResearchAgentRequest(input);
|
|
4469
|
+
// AGT-007 / AGT-022: App Copilot keeps its server-owned App evidence
|
|
4470
|
+
// adapter, while routing the actual investigation through the uniform
|
|
4471
|
+
// snapshot, retrieval, trust, execution, and bounded-repair orchestration.
|
|
4472
|
+
const governedAnswer = await runGovernedAgentAnswerForRun(request, undefined, 'research');
|
|
3881
4473
|
return {
|
|
3882
|
-
sql:
|
|
4474
|
+
sql: input.mode === 'memo_only' ? undefined : governedAnswer.proposedSql ?? governedAnswer.sql,
|
|
3883
4475
|
answer: governedAnswer.answer ?? governedAnswer.text,
|
|
3884
4476
|
result: governedAnswer.result,
|
|
3885
4477
|
analysisPlan: governedAnswer.analysisPlan,
|
|
@@ -3997,6 +4589,9 @@ export async function startLocalServer(opts) {
|
|
|
3997
4589
|
let generatedSql = notebookResearchString(input.generatedSql) ?? run.generatedSql;
|
|
3998
4590
|
let reviewedSql = notebookResearchString(input.reviewedSql) ?? run.reviewedSql;
|
|
3999
4591
|
let dqlArtifact = normalizeDqlArtifactReference(input.dqlArtifact) ?? run.dqlArtifact;
|
|
4592
|
+
const baselineSql = notebookResearchString(input.baselineSql);
|
|
4593
|
+
const baselineDqlArtifact = normalizeDqlArtifactReference(input.baselineDqlArtifact);
|
|
4594
|
+
let usedBaselineFallback = false;
|
|
4000
4595
|
const startedAt = new Date().toISOString();
|
|
4001
4596
|
storage.updateRun(run.id, {
|
|
4002
4597
|
domain,
|
|
@@ -4047,7 +4642,7 @@ export async function startLocalServer(opts) {
|
|
|
4047
4642
|
generationWarnings.push('No AI provider is configured. Metadata context was saved as a research plan; paste SQL or configure an AI provider to generate candidate SQL.');
|
|
4048
4643
|
}
|
|
4049
4644
|
else {
|
|
4050
|
-
const
|
|
4645
|
+
const researchSignal = input.signal ?? new AbortController().signal;
|
|
4051
4646
|
try {
|
|
4052
4647
|
await runner.run({
|
|
4053
4648
|
provider: resolvedProvider,
|
|
@@ -4069,18 +4664,19 @@ export async function startLocalServer(opts) {
|
|
|
4069
4664
|
reasoningEffort: resolveRunReasoningEffort(projectRoot, resolvedProvider, 'research', false),
|
|
4070
4665
|
analysisDepth: 'deep',
|
|
4071
4666
|
projectRoot,
|
|
4072
|
-
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
4073
|
-
executeGeneratedSql:
|
|
4074
|
-
getSchemaContext: getSchemaContextForAgent,
|
|
4667
|
+
executeCertifiedBlock: (node, invocation) => executeCertifiedBlockForAgent(node, invocation, input.executionConnection, input.executionConnectionName),
|
|
4668
|
+
executeGeneratedSql: (sql, artifact) => executeGeneratedArtifactForAgent(question, sql, artifact, input.executionConnection, input.executionConnectionName),
|
|
4669
|
+
getSchemaContext: (schemaQuestion, preparedContextPack) => getSchemaContextForAgent(schemaQuestion, preparedContextPack, input.executionConnection, input.executionConnectionName),
|
|
4075
4670
|
}, (turn) => {
|
|
4076
4671
|
if (turn.kind === 'tool_result' && turn.id === 'governed_answer') {
|
|
4077
4672
|
governedAnswer = turn.output;
|
|
4078
4673
|
}
|
|
4079
4674
|
if (turn.kind === 'error')
|
|
4080
4675
|
providerError = turn.message;
|
|
4081
|
-
},
|
|
4676
|
+
}, researchSignal);
|
|
4082
4677
|
}
|
|
4083
4678
|
catch (error) {
|
|
4679
|
+
rethrowIfCancelled(error, input.signal);
|
|
4084
4680
|
providerError = error instanceof Error ? error.message : String(error);
|
|
4085
4681
|
}
|
|
4086
4682
|
if (!governedAnswer) {
|
|
@@ -4095,6 +4691,25 @@ export async function startLocalServer(opts) {
|
|
|
4095
4691
|
}
|
|
4096
4692
|
}
|
|
4097
4693
|
}
|
|
4694
|
+
const deeperCompositionFailed = !governedAnswer
|
|
4695
|
+
|| governedAnswer.kind === 'no_answer'
|
|
4696
|
+
|| agentAnswerHasExecutionFailure(governedAnswer)
|
|
4697
|
+
|| Boolean(governedAnswer.analyticalFailure)
|
|
4698
|
+
|| (!governedAnswer.result && !generatedSql && !dqlArtifact);
|
|
4699
|
+
if (!reviewedSql && baselineSql && deeperCompositionFailed) {
|
|
4700
|
+
const failure = governedAnswer?.executionError
|
|
4701
|
+
?? governedAnswer?.analyticalFailure?.message
|
|
4702
|
+
?? governedAnswer?.answer
|
|
4703
|
+
?? governedAnswer?.text
|
|
4704
|
+
?? providerError;
|
|
4705
|
+
generatedSql = baselineSql;
|
|
4706
|
+
dqlArtifact = baselineDqlArtifact ?? dqlArtifact;
|
|
4707
|
+
usedBaselineFallback = true;
|
|
4708
|
+
generationWarnings.push([
|
|
4709
|
+
'The deeper composition did not produce a safer executable query, so DQL kept and revalidated the successful Ask baseline on the same data target.',
|
|
4710
|
+
failure ? `Research detail: ${failure}` : '',
|
|
4711
|
+
].filter(Boolean).join(' '));
|
|
4712
|
+
}
|
|
4098
4713
|
const sqlForPreview = reviewedSql || generatedSql;
|
|
4099
4714
|
const warnings = [
|
|
4100
4715
|
...(contextPack.warnings ?? []),
|
|
@@ -4103,14 +4718,14 @@ export async function startLocalServer(opts) {
|
|
|
4103
4718
|
].filter(Boolean);
|
|
4104
4719
|
let resultPreview;
|
|
4105
4720
|
let previewError;
|
|
4106
|
-
if (governedAnswer?.result?.rows && !reviewedSql) {
|
|
4721
|
+
if (!usedBaselineFallback && governedAnswer?.result?.rows && !reviewedSql) {
|
|
4107
4722
|
resultPreview = normalizeNotebookAgentResult(governedAnswer.result);
|
|
4108
4723
|
}
|
|
4109
4724
|
else if (sqlForPreview) {
|
|
4110
4725
|
try {
|
|
4111
4726
|
const previewSql = buildAgentPreviewSql(sqlForPreview);
|
|
4112
4727
|
const previewStart = Date.now();
|
|
4113
|
-
resultPreview = await executeLocalSqlForStoredResult(previewSql);
|
|
4728
|
+
resultPreview = await executeLocalSqlForStoredResult(previewSql, input.executionConnection);
|
|
4114
4729
|
recordNotebookQueryRun(projectRoot, {
|
|
4115
4730
|
notebookPath: run.notebookPath,
|
|
4116
4731
|
cellId: run.sourceCellId,
|
|
@@ -4150,8 +4765,10 @@ export async function startLocalServer(opts) {
|
|
|
4150
4765
|
: undefined;
|
|
4151
4766
|
const evidence = {
|
|
4152
4767
|
trustStatus: {
|
|
4153
|
-
label:
|
|
4154
|
-
|
|
4768
|
+
label: usedBaselineFallback
|
|
4769
|
+
? 'Successful Ask baseline · revalidated on the same target'
|
|
4770
|
+
: governedAnswer?.trustLabel
|
|
4771
|
+
?? (reviewedSql ? 'Reviewed notebook SQL' : generatedSql ? 'AI-generated research SQL' : dqlArtifact ? 'AI-generated DQL artifact' : 'Metadata-grounded research plan'),
|
|
4155
4772
|
reviewRequired: true,
|
|
4156
4773
|
},
|
|
4157
4774
|
contextPackId: contextPack.id,
|
|
@@ -4185,9 +4802,11 @@ export async function startLocalServer(opts) {
|
|
|
4185
4802
|
...(governedAnswer?.citations ?? []),
|
|
4186
4803
|
].slice(0, 40),
|
|
4187
4804
|
};
|
|
4188
|
-
const summary =
|
|
4189
|
-
|
|
4190
|
-
|
|
4805
|
+
const summary = usedBaselineFallback && resultPreview && !previewError
|
|
4806
|
+
? 'Research retained the successful Ask baseline and revalidated it on the same data target. No unverified replacement query was accepted; use the dossier to add the next breakdown or comparison.'
|
|
4807
|
+
: notebookResearchString(governedAnswer?.answer)
|
|
4808
|
+
?? notebookResearchString(governedAnswer?.text)
|
|
4809
|
+
?? notebookResearchSummary(question, resultPreview, previewError);
|
|
4191
4810
|
const recommendation = previewError
|
|
4192
4811
|
? 'Review the SQL, selected metadata, and connection context before rerunning.'
|
|
4193
4812
|
: dqlArtifact && !reviewedSql
|
|
@@ -4225,6 +4844,9 @@ export async function startLocalServer(opts) {
|
|
|
4225
4844
|
contextPackId: contextPack.id,
|
|
4226
4845
|
routeDecision,
|
|
4227
4846
|
warnings: [
|
|
4847
|
+
...(usedBaselineFallback && input.baselineRunId
|
|
4848
|
+
? [`Baseline Ask run: ${input.baselineRunId}`]
|
|
4849
|
+
: []),
|
|
4228
4850
|
...warnings,
|
|
4229
4851
|
...(display && !display.ok ? [display.error] : []),
|
|
4230
4852
|
...(display && display.ok ? display.warnings : []),
|
|
@@ -4235,6 +4857,7 @@ export async function startLocalServer(opts) {
|
|
|
4235
4857
|
}) ?? run;
|
|
4236
4858
|
}
|
|
4237
4859
|
catch (error) {
|
|
4860
|
+
rethrowIfCancelled(error, input.signal);
|
|
4238
4861
|
return storage.updateRun(run.id, {
|
|
4239
4862
|
domain,
|
|
4240
4863
|
owner,
|
|
@@ -4461,16 +5084,19 @@ export async function startLocalServer(opts) {
|
|
|
4461
5084
|
});
|
|
4462
5085
|
// SSE clients for /api/watch hot-reload
|
|
4463
5086
|
const sseClients = new Set();
|
|
5087
|
+
const projectWatchers = [];
|
|
5088
|
+
let runtimeClosing = false;
|
|
4464
5089
|
// Watch all Git-owned authoring roots. Domain Packages are recursive and are
|
|
4465
5090
|
// the canonical home for modeling, skills, blocks, notebooks, Apps, and evals.
|
|
4466
5091
|
if (projectRoot) {
|
|
4467
|
-
let sourceRefreshTimer;
|
|
4468
5092
|
for (const dir of ['domains', 'skills', 'tests', 'notebooks', 'workbooks', 'blocks', 'apps', 'dashboards', 'semantic-layer', 'data']) {
|
|
4469
5093
|
const watchDir = join(projectRoot, dir);
|
|
4470
5094
|
if (!existsSync(watchDir))
|
|
4471
5095
|
continue;
|
|
4472
5096
|
try {
|
|
4473
|
-
watch(watchDir, { persistent: false, recursive: true }, (eventType, filename) => {
|
|
5097
|
+
const watcher = watch(watchDir, { persistent: false, recursive: true }, (eventType, filename) => {
|
|
5098
|
+
if (runtimeClosing)
|
|
5099
|
+
return;
|
|
4474
5100
|
if (!filename)
|
|
4475
5101
|
return;
|
|
4476
5102
|
const path = `${dir}/${filename}`;
|
|
@@ -4485,11 +5111,7 @@ export async function startLocalServer(opts) {
|
|
|
4485
5111
|
}
|
|
4486
5112
|
if (['domains', 'skills', 'tests', 'blocks', 'apps', 'notebooks'].includes(dir)) {
|
|
4487
5113
|
invalidateAgentProjectState(projectRoot);
|
|
4488
|
-
|
|
4489
|
-
clearTimeout(sourceRefreshTimer);
|
|
4490
|
-
sourceRefreshTimer = setTimeout(() => {
|
|
4491
|
-
void refreshUnifiedProjectIndexes(projectRoot).catch(() => undefined);
|
|
4492
|
-
}, 250);
|
|
5114
|
+
scheduleProjectRefresh(`file:${path}`);
|
|
4493
5115
|
}
|
|
4494
5116
|
// Hot-reload semantic layer on change and notify frontend
|
|
4495
5117
|
if (dir === 'semantic-layer') {
|
|
@@ -4507,21 +5129,20 @@ export async function startLocalServer(opts) {
|
|
|
4507
5129
|
}).catch(() => { });
|
|
4508
5130
|
}
|
|
4509
5131
|
});
|
|
5132
|
+
projectWatchers.push(watcher);
|
|
4510
5133
|
}
|
|
4511
5134
|
catch { /* dir not watchable */ }
|
|
4512
5135
|
}
|
|
4513
5136
|
const configuredDbtManifest = resolveDbtManifestPath(projectRoot, projectConfig);
|
|
4514
5137
|
if (configuredDbtManifest && existsSync(dirname(configuredDbtManifest))) {
|
|
4515
5138
|
try {
|
|
4516
|
-
watch(dirname(configuredDbtManifest), { persistent: false }, (_eventType, filename) => {
|
|
5139
|
+
const watcher = watch(dirname(configuredDbtManifest), { persistent: false }, (_eventType, filename) => {
|
|
5140
|
+
if (runtimeClosing)
|
|
5141
|
+
return;
|
|
4517
5142
|
if (!filename || !['manifest.json', 'catalog.json', 'semantic_manifest.json', 'run_results.json'].includes(String(filename)))
|
|
4518
5143
|
return;
|
|
4519
5144
|
invalidateAgentProjectState(projectRoot);
|
|
4520
|
-
|
|
4521
|
-
clearTimeout(sourceRefreshTimer);
|
|
4522
|
-
sourceRefreshTimer = setTimeout(() => {
|
|
4523
|
-
void refreshUnifiedProjectIndexes(projectRoot).catch(() => undefined);
|
|
4524
|
-
}, 250);
|
|
5145
|
+
scheduleProjectRefresh(`dbt:${String(filename)}`);
|
|
4525
5146
|
const payload = JSON.stringify({ type: 'dbt-artifacts-changed', path: String(filename) });
|
|
4526
5147
|
for (const client of sseClients) {
|
|
4527
5148
|
try {
|
|
@@ -4543,6 +5164,7 @@ export async function startLocalServer(opts) {
|
|
|
4543
5164
|
}
|
|
4544
5165
|
}).catch(() => { });
|
|
4545
5166
|
});
|
|
5167
|
+
projectWatchers.push(watcher);
|
|
4546
5168
|
}
|
|
4547
5169
|
catch { /* dbt artifact directory not watchable */ }
|
|
4548
5170
|
}
|
|
@@ -4625,11 +5247,15 @@ export async function startLocalServer(opts) {
|
|
|
4625
5247
|
result: normalizeQueryResult(result),
|
|
4626
5248
|
chartConfig: plan?.chartConfig ?? validation.chartConfig ?? null,
|
|
4627
5249
|
invocation,
|
|
5250
|
+
tests: plan?.tests ?? [],
|
|
4628
5251
|
};
|
|
4629
5252
|
};
|
|
4630
|
-
const runBlockStudioTestSummary = async (source, targetConnection) => {
|
|
4631
|
-
const activeConnection = requireActiveConnection(targetConnection);
|
|
5253
|
+
const runBlockStudioTestSummary = async (source, targetConnection, executedPreview) => {
|
|
4632
5254
|
const start = Date.now();
|
|
5255
|
+
if (executedPreview) {
|
|
5256
|
+
return evaluateBlockStudioTests(executedPreview.tests, executedPreview.result.rows, executedPreview.result.columns, start);
|
|
5257
|
+
}
|
|
5258
|
+
const activeConnection = requireActiveConnection(targetConnection);
|
|
4633
5259
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer, projectRoot);
|
|
4634
5260
|
// Run tests against the SAME SQL the preview runs: for a semantic block with a
|
|
4635
5261
|
// pre-compiled query, that's the query (not a recompiled metric), so the test's
|
|
@@ -4674,40 +5300,7 @@ export async function startLocalServer(opts) {
|
|
|
4674
5300
|
const columns = Array.isArray(rawResult?.columns)
|
|
4675
5301
|
? rawResult.columns.map((column) => typeof column === 'string' ? column : column?.name ?? String(column))
|
|
4676
5302
|
: [];
|
|
4677
|
-
|
|
4678
|
-
let passed = 0;
|
|
4679
|
-
let failed = 0;
|
|
4680
|
-
for (const test of tests) {
|
|
4681
|
-
const name = `assert ${test.field} ${test.operator} ${formatBlockStudioExpected(test.expected)}`;
|
|
4682
|
-
let actual;
|
|
4683
|
-
if (test.field === 'row_count') {
|
|
4684
|
-
actual = rows.length;
|
|
4685
|
-
}
|
|
4686
|
-
else if (!columns.includes(test.field)) {
|
|
4687
|
-
assertions.push({ name, passed: false, expected: test.expected, error: `Column '${test.field}' not found in results` });
|
|
4688
|
-
failed += 1;
|
|
4689
|
-
continue;
|
|
4690
|
-
}
|
|
4691
|
-
else {
|
|
4692
|
-
actual = rows[0]?.[test.field];
|
|
4693
|
-
}
|
|
4694
|
-
const ok = compareBlockStudioValues(actual, test.operator, test.expected);
|
|
4695
|
-
if (ok) {
|
|
4696
|
-
assertions.push({ name, passed: true, actual, expected: test.expected });
|
|
4697
|
-
passed += 1;
|
|
4698
|
-
}
|
|
4699
|
-
else {
|
|
4700
|
-
assertions.push({
|
|
4701
|
-
name,
|
|
4702
|
-
passed: false,
|
|
4703
|
-
actual,
|
|
4704
|
-
expected: test.expected,
|
|
4705
|
-
error: `${String(actual)} ${test.operator} ${formatBlockStudioExpected(test.expected)} is false`,
|
|
4706
|
-
});
|
|
4707
|
-
failed += 1;
|
|
4708
|
-
}
|
|
4709
|
-
}
|
|
4710
|
-
return { passed, failed, skipped: 0, duration: Date.now() - start, assertions, runAt: new Date() };
|
|
5303
|
+
return evaluateBlockStudioTests(tests, rows, columns, start);
|
|
4711
5304
|
};
|
|
4712
5305
|
const saveDqlGenerationDraft = (importId, candidate) => saveDqlGenerationDraftForProject(projectRoot, importId, candidate);
|
|
4713
5306
|
const createDqlGenerationSessionFromBody = async (body) => {
|
|
@@ -4766,7 +5359,9 @@ export async function startLocalServer(opts) {
|
|
|
4766
5359
|
blockers.push(error instanceof Error ? error.message : String(error));
|
|
4767
5360
|
}
|
|
4768
5361
|
try {
|
|
4769
|
-
testResults =
|
|
5362
|
+
testResults = preview
|
|
5363
|
+
? await runBlockStudioTestSummary(source, undefined, preview)
|
|
5364
|
+
: await runBlockStudioTestSummary(source);
|
|
4770
5365
|
}
|
|
4771
5366
|
catch (error) {
|
|
4772
5367
|
testResults = {
|
|
@@ -4923,15 +5518,24 @@ export async function startLocalServer(opts) {
|
|
|
4923
5518
|
writeHead(status, { ...headers, ...validatorHeaders });
|
|
4924
5519
|
return end(raw);
|
|
4925
5520
|
}
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
5521
|
+
gzip(raw, (error, gz) => {
|
|
5522
|
+
if (response.destroyed || response.writableEnded)
|
|
5523
|
+
return;
|
|
5524
|
+
if (error) {
|
|
5525
|
+
writeHead(status, { ...headers, ...validatorHeaders });
|
|
5526
|
+
end(raw);
|
|
5527
|
+
return;
|
|
5528
|
+
}
|
|
5529
|
+
writeHead(status, {
|
|
5530
|
+
...headers,
|
|
5531
|
+
...validatorHeaders,
|
|
5532
|
+
'Content-Encoding': 'gzip',
|
|
5533
|
+
'Content-Length': gz.byteLength,
|
|
5534
|
+
Vary: 'Accept-Encoding, Origin',
|
|
5535
|
+
});
|
|
5536
|
+
end(gz);
|
|
4933
5537
|
});
|
|
4934
|
-
return
|
|
5538
|
+
return response;
|
|
4935
5539
|
});
|
|
4936
5540
|
return rawResponse;
|
|
4937
5541
|
};
|
|
@@ -4950,7 +5554,7 @@ export async function startLocalServer(opts) {
|
|
|
4950
5554
|
res.setHeader('Access-Control-Allow-Origin', requestOrigin);
|
|
4951
5555
|
res.setHeader('Vary', 'Origin');
|
|
4952
5556
|
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
|
|
4953
|
-
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
|
5557
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Idempotency-Key');
|
|
4954
5558
|
if (!originAllowed && path.startsWith('/api/')) {
|
|
4955
5559
|
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4956
5560
|
res.end(serializeJSON({ error: 'Origin is not allowed.' }));
|
|
@@ -4985,6 +5589,40 @@ export async function startLocalServer(opts) {
|
|
|
4985
5589
|
res.end(serializeJSON({ status: 'ok', version: runtimeVersion, versionStatus, retrievalHealth }));
|
|
4986
5590
|
return;
|
|
4987
5591
|
}
|
|
5592
|
+
if (req.method === 'GET' && path === '/api/operations/events') {
|
|
5593
|
+
res.writeHead(200, {
|
|
5594
|
+
'Content-Type': 'text/event-stream',
|
|
5595
|
+
'Cache-Control': 'no-cache',
|
|
5596
|
+
'Connection': 'keep-alive',
|
|
5597
|
+
'X-Accel-Buffering': 'no',
|
|
5598
|
+
});
|
|
5599
|
+
res.write(': connected\n\n');
|
|
5600
|
+
operationSseClients.add(res);
|
|
5601
|
+
for (const operation of operationCoordinator.list(20).reverse()) {
|
|
5602
|
+
res.write(`event: operation\ndata: ${serializeJSON(operation)}\n\n`);
|
|
5603
|
+
}
|
|
5604
|
+
req.on('close', () => operationSseClients.delete(res));
|
|
5605
|
+
return;
|
|
5606
|
+
}
|
|
5607
|
+
if (req.method === 'GET' && path === '/api/operations') {
|
|
5608
|
+
const limit = Number(url.searchParams.get('limit') ?? 50);
|
|
5609
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5610
|
+
res.end(serializeJSON({ operations: operationCoordinator.list(limit) }));
|
|
5611
|
+
return;
|
|
5612
|
+
}
|
|
5613
|
+
const operationMatch = path.match(/^\/api\/operations\/([^/]+)$/);
|
|
5614
|
+
if (operationMatch && req.method === 'GET') {
|
|
5615
|
+
const operation = operationCoordinator.get(decodeURIComponent(operationMatch[1]));
|
|
5616
|
+
res.writeHead(operation ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5617
|
+
res.end(serializeJSON(operation ?? { error: 'Operation not found.' }));
|
|
5618
|
+
return;
|
|
5619
|
+
}
|
|
5620
|
+
if (operationMatch && req.method === 'DELETE') {
|
|
5621
|
+
const operation = operationCoordinator.cancel(decodeURIComponent(operationMatch[1]));
|
|
5622
|
+
res.writeHead(operation ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5623
|
+
res.end(serializeJSON(operation ?? { error: 'Operation not found.' }));
|
|
5624
|
+
return;
|
|
5625
|
+
}
|
|
4988
5626
|
if (req.method === 'GET' && path === '/api/onboarding/launch') {
|
|
4989
5627
|
const requestId = apiRequestId('onboarding-launch');
|
|
4990
5628
|
const prefs = readUserPrefs(userPrefsPath);
|
|
@@ -5593,15 +6231,79 @@ export async function startLocalServer(opts) {
|
|
|
5593
6231
|
}
|
|
5594
6232
|
if (req.method === 'GET' && path === '/api/modeling/dbt-first/inventory') {
|
|
5595
6233
|
const requestId = apiRequestId('modeling-inventory');
|
|
5596
|
-
const
|
|
6234
|
+
const physicalOnly = url.searchParams.get('physicalOnly') === 'true';
|
|
6235
|
+
const modelingEnabled = projectConfig.manifestVersion === 3 && projectConfig.modeling?.mode === 'dbt-first';
|
|
6236
|
+
if (physicalOnly && !modelingEnabled) {
|
|
6237
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6238
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
6239
|
+
requestId,
|
|
6240
|
+
code: 'DBT_FIRST_NOT_ENABLED',
|
|
6241
|
+
message: 'dbt-first modeling is not enabled. Connect and apply a dbt project from Settings → Project & dbt.',
|
|
6242
|
+
nextActions: ['Open Settings → Project & dbt and apply a valid dbt project.'],
|
|
6243
|
+
})));
|
|
6244
|
+
return;
|
|
6245
|
+
}
|
|
6246
|
+
const dbtProjectDir = findDbtProjectPath(projectRoot, projectConfig);
|
|
6247
|
+
const configuredManifestPath = projectConfig.dbt?.manifestPath ?? 'target/manifest.json';
|
|
6248
|
+
const expectedManifestPath = resolve(dbtProjectDir, configuredManifestPath);
|
|
6249
|
+
if (physicalOnly && !existsSync(expectedManifestPath)) {
|
|
6250
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6251
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
6252
|
+
requestId,
|
|
6253
|
+
code: 'DBT_MANIFEST_NOT_FOUND',
|
|
6254
|
+
message: `The configured dbt manifest was not found at ${configuredManifestPath}.`,
|
|
6255
|
+
details: { manifestPath: configuredManifestPath },
|
|
6256
|
+
nextActions: ['Run dbt parse, dbt compile, or dbt build, then refresh the database catalog.'],
|
|
6257
|
+
})));
|
|
6258
|
+
return;
|
|
6259
|
+
}
|
|
6260
|
+
let snapshot;
|
|
6261
|
+
if (physicalOnly) {
|
|
6262
|
+
try {
|
|
6263
|
+
snapshot = projectSnapshot();
|
|
6264
|
+
}
|
|
6265
|
+
catch (error) {
|
|
6266
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6267
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
6268
|
+
requestId,
|
|
6269
|
+
code: 'DBT_MANIFEST_COMPILE_FAILED',
|
|
6270
|
+
message: apiErrorMessage(error),
|
|
6271
|
+
details: { manifestPath: configuredManifestPath },
|
|
6272
|
+
nextActions: ['Fix or rebuild the dbt manifest, then refresh the database catalog.'],
|
|
6273
|
+
})));
|
|
6274
|
+
return;
|
|
6275
|
+
}
|
|
6276
|
+
}
|
|
6277
|
+
else {
|
|
6278
|
+
snapshot = projectSnapshot();
|
|
6279
|
+
}
|
|
5597
6280
|
const manifest = snapshot.manifest;
|
|
5598
|
-
|
|
6281
|
+
if (physicalOnly && (manifest.manifestVersion !== 3 || !manifest.modeling || !manifest.dbtProvenance)) {
|
|
6282
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6283
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
6284
|
+
requestId,
|
|
6285
|
+
snapshotId: snapshot.snapshotId,
|
|
6286
|
+
code: 'DBT_MANIFEST_COMPILE_FAILED',
|
|
6287
|
+
message: snapshot.error ?? 'DQL could not compile the dbt database catalog.',
|
|
6288
|
+
details: { manifestPath: configuredManifestPath },
|
|
6289
|
+
nextActions: ['Review the dbt artifact, rebuild it, then refresh the database catalog.'],
|
|
6290
|
+
})));
|
|
6291
|
+
return;
|
|
6292
|
+
}
|
|
6293
|
+
// PERF-001 / CTX-005: only the Database browser opts into the smaller,
|
|
6294
|
+
// physical-relation contract. Existing modeling consumers retain their
|
|
6295
|
+
// original inventory scope and pagination defaults.
|
|
6296
|
+
const maxLimit = physicalOnly ? 100 : 200;
|
|
6297
|
+
const defaultLimit = physicalOnly ? 25 : 50;
|
|
6298
|
+
const limit = Math.min(maxLimit, Math.max(1, Number(url.searchParams.get('limit')) || defaultLimit));
|
|
5599
6299
|
const cursor = Math.max(0, Number(url.searchParams.get('cursor')) || 0);
|
|
5600
6300
|
const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
|
|
5601
6301
|
const queryTokens = query.split(/[^a-z0-9]+/).filter(Boolean);
|
|
5602
6302
|
const domain = (url.searchParams.get('domain') ?? '').trim();
|
|
5603
6303
|
const boundByDbtId = new Map(Object.values(manifest.modeling?.entities ?? {}).map((entity) => [entity.dbtUniqueId, entity]));
|
|
6304
|
+
const physicalIds = physicalOnly ? physicalDbtRelationIds(expectedManifestPath) : undefined;
|
|
5604
6305
|
const nodes = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
6306
|
+
.filter((node) => !physicalOnly || (Boolean(node.relation) && (!physicalIds || physicalIds.has(node.uniqueId))))
|
|
5605
6307
|
.filter((node) => {
|
|
5606
6308
|
if (!queryTokens.length)
|
|
5607
6309
|
return true;
|
|
@@ -5634,6 +6336,11 @@ export async function startLocalServer(opts) {
|
|
|
5634
6336
|
nextCursor: cursor + limit < nodes.length ? cursor + limit : null,
|
|
5635
6337
|
total: nodes.length,
|
|
5636
6338
|
snapshotId: snapshot.snapshotId,
|
|
6339
|
+
...(physicalOnly ? {
|
|
6340
|
+
manifestPath: manifest.dbtProvenance.manifestPath,
|
|
6341
|
+
projectName: manifest.dbtProvenance.projectName,
|
|
6342
|
+
scope: 'dbt_relations',
|
|
6343
|
+
} : {}),
|
|
5637
6344
|
}));
|
|
5638
6345
|
return;
|
|
5639
6346
|
}
|
|
@@ -5840,13 +6547,14 @@ export async function startLocalServer(opts) {
|
|
|
5840
6547
|
return;
|
|
5841
6548
|
}
|
|
5842
6549
|
const sourceSql = repairableSqlFromAgentRun(sourceRun);
|
|
6550
|
+
const sourceDqlArtifact = repairableDqlArtifactFromAgentRun(sourceRun);
|
|
5843
6551
|
const retainedFailure = warehouseFailureFromAgentRun(sourceRun);
|
|
5844
|
-
if (!sourceSql || !analyticalFailureAllowsDeterministicRetry(retainedFailure)) {
|
|
6552
|
+
if ((!sourceSql && !sourceDqlArtifact) || !analyticalFailureAllowsDeterministicRetry(retainedFailure)) {
|
|
5845
6553
|
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5846
6554
|
res.end(serializeJSON({
|
|
5847
|
-
error: sourceSql
|
|
6555
|
+
error: sourceSql || sourceDqlArtifact
|
|
5848
6556
|
? 'This failure requires authorized access or a policy change and cannot be repaired by AI.'
|
|
5849
|
-
: 'This run has no retained SQL statement to repair.',
|
|
6557
|
+
: 'This run has no retained SQL statement or editable DQL artifact to repair.',
|
|
5850
6558
|
...(retainedFailure ? { warehouseFailure: retainedFailure } : {}),
|
|
5851
6559
|
}));
|
|
5852
6560
|
return;
|
|
@@ -5870,97 +6578,40 @@ export async function startLocalServer(opts) {
|
|
|
5870
6578
|
: sourceRun.executionTarget?.target === 'local'
|
|
5871
6579
|
? 'local'
|
|
5872
6580
|
: projectConfig.defaultConnectionName;
|
|
5873
|
-
let
|
|
5874
|
-
let repairedSql = sourceSql;
|
|
5875
|
-
let repairMode = 'deterministic';
|
|
5876
|
-
let firstFailure;
|
|
6581
|
+
let repaired;
|
|
5877
6582
|
try {
|
|
5878
|
-
|
|
6583
|
+
// Prefer the editable DQL contract even when the failed run also
|
|
6584
|
+
// retained compiled SQL. This keeps parameters, visualization, tests,
|
|
6585
|
+
// and the corrected wrapper attached to the repaired Ask result.
|
|
6586
|
+
repaired = sourceDqlArtifact
|
|
6587
|
+
? await executeBoundedDqlRepair({
|
|
6588
|
+
question: sourceRun.question,
|
|
6589
|
+
source: sourceDqlArtifact.source,
|
|
6590
|
+
failure: retainedFailure?.redactedMessage
|
|
6591
|
+
?? sourceRun.diagnosticReceipt?.failure?.message
|
|
6592
|
+
?? sourceRun.summary,
|
|
6593
|
+
targetConnection,
|
|
6594
|
+
targetConnectionName,
|
|
6595
|
+
parameters: sourceDqlArtifact.parameterValues,
|
|
6596
|
+
artifactName: sourceDqlArtifact.name,
|
|
6597
|
+
})
|
|
6598
|
+
: await executeBoundedSqlRepair({
|
|
6599
|
+
question: sourceRun.question,
|
|
6600
|
+
sourceSql: sourceSql,
|
|
6601
|
+
targetConnection,
|
|
6602
|
+
targetConnectionName,
|
|
6603
|
+
});
|
|
5879
6604
|
}
|
|
5880
6605
|
catch (error) {
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
|
|
5885
|
-
|
|
5886
|
-
res.end(serializeJSON({
|
|
5887
|
-
error: firstFailure?.redactedMessage ?? 'The deterministic repair did not produce an executable query.',
|
|
5888
|
-
...(firstFailure ? { warehouseFailure: firstFailure } : {}),
|
|
5889
|
-
}));
|
|
5890
|
-
return;
|
|
5891
|
-
}
|
|
5892
|
-
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
5893
|
-
if (!provider) {
|
|
5894
|
-
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5895
|
-
res.end(serializeJSON({ error: 'Configure an available AI provider in Settings to repair this query.' }));
|
|
5896
|
-
return;
|
|
5897
|
-
}
|
|
5898
|
-
let schemaContext;
|
|
5899
|
-
try {
|
|
5900
|
-
schemaContext = await getSchemaContextForAgent(sourceRun.question, undefined, targetConnection, targetConnectionName);
|
|
5901
|
-
}
|
|
5902
|
-
catch (error) {
|
|
5903
|
-
const failure = normalizeWarehouseSqlFailure(error, targetConnection.driver);
|
|
5904
|
-
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5905
|
-
res.end(serializeJSON({ error: failure.redactedMessage, warehouseFailure: failure }));
|
|
5906
|
-
return;
|
|
5907
|
-
}
|
|
5908
|
-
const boundedSchema = schemaContext.slice(0, 16).map((table) => ({
|
|
5909
|
-
relation: table.relation,
|
|
5910
|
-
columns: table.columns.slice(0, 40).map((column) => ({ name: column.name, type: column.type })),
|
|
6606
|
+
const repairError = error;
|
|
6607
|
+
res.writeHead(repairError.status ?? 500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6608
|
+
res.end(serializeJSON({
|
|
6609
|
+
error: repairError.message,
|
|
6610
|
+
...(repairError.warehouseFailure ? { warehouseFailure: repairError.warehouseFailure } : {}),
|
|
5911
6611
|
}));
|
|
5912
|
-
|
|
5913
|
-
try {
|
|
5914
|
-
raw = await provider.generate([
|
|
5915
|
-
{
|
|
5916
|
-
role: 'system',
|
|
5917
|
-
content: [
|
|
5918
|
-
'Repair one failed read-only analytical SQL statement.',
|
|
5919
|
-
'Return exactly one JSON object in a ```json fence with keys summary, sql, viz, outputs.',
|
|
5920
|
-
'Preserve the user question, selected grain, filters, and aggregation semantics.',
|
|
5921
|
-
'Use only relations and columns present in the supplied schema context.',
|
|
5922
|
-
'Never emit internal DQL relation identities such as source::, model::, seed::, or snapshot::.',
|
|
5923
|
-
'Do not add writes, DDL, multiple statements, or commentary outside the JSON fence.',
|
|
5924
|
-
].join(' '),
|
|
5925
|
-
},
|
|
5926
|
-
{
|
|
5927
|
-
role: 'user',
|
|
5928
|
-
content: JSON.stringify({
|
|
5929
|
-
question: sourceRun.question,
|
|
5930
|
-
driver: targetConnection.driver,
|
|
5931
|
-
failure: firstFailure.redactedMessage,
|
|
5932
|
-
sql: sourceSql,
|
|
5933
|
-
schema: boundedSchema,
|
|
5934
|
-
}),
|
|
5935
|
-
},
|
|
5936
|
-
], { maxTokens: 1200, temperature: 0 });
|
|
5937
|
-
}
|
|
5938
|
-
catch {
|
|
5939
|
-
res.writeHead(502, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5940
|
-
res.end(serializeJSON({ error: 'The AI provider could not complete this repair. Check Provider Settings and retry.' }));
|
|
5941
|
-
return;
|
|
5942
|
-
}
|
|
5943
|
-
const proposal = parseProposal(raw);
|
|
5944
|
-
if (!proposal.sql?.trim()) {
|
|
5945
|
-
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5946
|
-
res.end(serializeJSON({ error: 'The AI repair did not return an executable SQL statement.' }));
|
|
5947
|
-
return;
|
|
5948
|
-
}
|
|
5949
|
-
repairedSql = proposal.sql.trim();
|
|
5950
|
-
repairMode = 'ai';
|
|
5951
|
-
try {
|
|
5952
|
-
result = await executeGeneratedSqlDirect(sourceRun.question, repairedSql, undefined, targetConnection);
|
|
5953
|
-
}
|
|
5954
|
-
catch (error) {
|
|
5955
|
-
const failure = normalizeWarehouseSqlFailure(error, targetConnection.driver);
|
|
5956
|
-
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5957
|
-
res.end(serializeJSON({
|
|
5958
|
-
error: failure.redactedMessage,
|
|
5959
|
-
warehouseFailure: failure,
|
|
5960
|
-
}));
|
|
5961
|
-
return;
|
|
5962
|
-
}
|
|
6612
|
+
return;
|
|
5963
6613
|
}
|
|
6614
|
+
const { result, repairedSql, repairMode, firstFailure } = repaired;
|
|
5964
6615
|
const completedAt = new Date().toISOString();
|
|
5965
6616
|
const preview = agentResultToSynthesisPreview(result);
|
|
5966
6617
|
const synthesis = await synthesizeAnswer({
|
|
@@ -5972,6 +6623,8 @@ export async function startLocalServer(opts) {
|
|
|
5972
6623
|
...(preview ? { columnFormats: agentColumnDisplayFormats(projectRoot, preview.columns) } : {}),
|
|
5973
6624
|
});
|
|
5974
6625
|
const sourceFailure = analyticalFailedRunFromAgentRun(sourceRun)?.failure;
|
|
6626
|
+
const presentationContext = repairPresentationContextFromAgentRun(sourceRun);
|
|
6627
|
+
const executionResultFingerprint = result.executionReceipt?.resultFingerprint;
|
|
5975
6628
|
const artifact = agentRunArtifact('answer', 'Repaired analytical answer', {
|
|
5976
6629
|
kind: 'uncertified',
|
|
5977
6630
|
certification: 'analyst_review_required',
|
|
@@ -5979,13 +6632,31 @@ export async function startLocalServer(opts) {
|
|
|
5979
6632
|
trustLabel: 'AI-generated · review required',
|
|
5980
6633
|
answer: synthesis.text,
|
|
5981
6634
|
text: synthesis.text,
|
|
6635
|
+
...presentationContext,
|
|
5982
6636
|
result,
|
|
5983
6637
|
proposedSql: repairedSql,
|
|
5984
6638
|
sql: result.sql ?? repairedSql,
|
|
5985
6639
|
dqlArtifact: result.dqlArtifact,
|
|
6640
|
+
diagnosticReceipt: {
|
|
6641
|
+
version: 1,
|
|
6642
|
+
phase: 'run.completed',
|
|
6643
|
+
route: 'generated_answer',
|
|
6644
|
+
repair: {
|
|
6645
|
+
attempt: 1,
|
|
6646
|
+
mode: repairMode,
|
|
6647
|
+
sourceRunId: sourceRun.id,
|
|
6648
|
+
targetPreserved: true,
|
|
6649
|
+
readOnly: true,
|
|
6650
|
+
},
|
|
6651
|
+
execution: {
|
|
6652
|
+
rowCount: result.rowCount,
|
|
6653
|
+
...(executionResultFingerprint ? { resultFingerprint: executionResultFingerprint } : {}),
|
|
6654
|
+
},
|
|
6655
|
+
},
|
|
5986
6656
|
previousAttempt: {
|
|
5987
6657
|
sourceRunId: sourceRun.id,
|
|
5988
|
-
sql: sourceSql,
|
|
6658
|
+
...(sourceSql ? { sql: sourceSql } : {}),
|
|
6659
|
+
...(sourceDqlArtifact?.source ? { dql: sourceDqlArtifact.source } : {}),
|
|
5989
6660
|
failure: retainedFailure ?? firstFailure,
|
|
5990
6661
|
},
|
|
5991
6662
|
repair: { attempt: 1, mode: repairMode },
|
|
@@ -6168,39 +6839,83 @@ export async function startLocalServer(opts) {
|
|
|
6168
6839
|
const wantsStream = url.searchParams.get('stream') === '1' || url.searchParams.get('stream') === 'true';
|
|
6169
6840
|
const runId = parsed.request.runId;
|
|
6170
6841
|
const runController = new AbortController();
|
|
6171
|
-
parsed.request.signal = AbortSignal.any([
|
|
6172
|
-
runController.signal,
|
|
6173
|
-
AbortSignal.timeout(agentRunDeadlineMs(parsed.request, process.env, getActiveProvider(projectRoot))),
|
|
6174
|
-
]);
|
|
6175
6842
|
if (runId)
|
|
6176
6843
|
activeAgentRunControllers.set(runId, runController);
|
|
6844
|
+
let streamConnected = true;
|
|
6845
|
+
res.on('close', () => { streamConnected = false; });
|
|
6846
|
+
const writeStream = (event, data) => {
|
|
6847
|
+
if (!streamConnected || res.destroyed || res.writableEnded)
|
|
6848
|
+
return;
|
|
6849
|
+
try {
|
|
6850
|
+
writeAgentRunSse(res, event, data);
|
|
6851
|
+
}
|
|
6852
|
+
catch {
|
|
6853
|
+
streamConnected = false;
|
|
6854
|
+
}
|
|
6855
|
+
};
|
|
6856
|
+
if (wantsStream) {
|
|
6857
|
+
res.writeHead(200, {
|
|
6858
|
+
'Content-Type': 'text/event-stream; charset=utf-8',
|
|
6859
|
+
'Cache-Control': 'no-cache',
|
|
6860
|
+
'Connection': 'keep-alive',
|
|
6861
|
+
'X-Accel-Buffering': 'no',
|
|
6862
|
+
});
|
|
6863
|
+
}
|
|
6864
|
+
const operation = runId ? operationCoordinator.create({
|
|
6865
|
+
type: 'agent_run',
|
|
6866
|
+
scope: `agent-run:${runId}`,
|
|
6867
|
+
resourceRevision: parsed.request.threadId,
|
|
6868
|
+
message: 'AI request accepted. You can change pages while it runs.',
|
|
6869
|
+
cancellable: true,
|
|
6870
|
+
}) : null;
|
|
6871
|
+
if (wantsStream)
|
|
6872
|
+
writeStream('agent-run-accepted', { runId, operationId: operation?.id });
|
|
6873
|
+
let completedRun;
|
|
6874
|
+
let runError;
|
|
6177
6875
|
try {
|
|
6876
|
+
const executeRun = async (operationSignal, report) => {
|
|
6877
|
+
parsed.request.signal = AbortSignal.any([
|
|
6878
|
+
runController.signal,
|
|
6879
|
+
...(operationSignal ? [operationSignal] : []),
|
|
6880
|
+
AbortSignal.timeout(agentRunDeadlineMs(parsed.request, process.env, getActiveProvider(projectRoot))),
|
|
6881
|
+
]);
|
|
6882
|
+
report?.({ phase: 'reasoning', progress: 8, message: 'Grounding the request and selecting the governed route.' });
|
|
6883
|
+
try {
|
|
6884
|
+
const run = await agentRunEngine.run(parsed.request, (event) => {
|
|
6885
|
+
if (wantsStream)
|
|
6886
|
+
writeStream('agent-run-event', event);
|
|
6887
|
+
report?.({ phase: event.type, progress: 45, message: event.message });
|
|
6888
|
+
}, (delta) => {
|
|
6889
|
+
if (wantsStream)
|
|
6890
|
+
writeStream('agent-run-answer-delta', { runId: parsed.request.runId, delta });
|
|
6891
|
+
});
|
|
6892
|
+
completedRun = run;
|
|
6893
|
+
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
6894
|
+
return { runId: run.id, threadId: parsed.request.threadId, status: run.status };
|
|
6895
|
+
}
|
|
6896
|
+
catch (error) {
|
|
6897
|
+
runError = error;
|
|
6898
|
+
throw error;
|
|
6899
|
+
}
|
|
6900
|
+
};
|
|
6901
|
+
if (operation)
|
|
6902
|
+
await operationCoordinator.run(operation.id, (signal, report) => executeRun(signal, report));
|
|
6903
|
+
else
|
|
6904
|
+
await executeRun();
|
|
6905
|
+
if (runError)
|
|
6906
|
+
throw runError;
|
|
6907
|
+
if (!completedRun)
|
|
6908
|
+
throw new Error('The AI request ended before producing a stored run.');
|
|
6178
6909
|
if (wantsStream) {
|
|
6179
|
-
|
|
6180
|
-
|
|
6181
|
-
|
|
6182
|
-
|
|
6183
|
-
|
|
6184
|
-
});
|
|
6185
|
-
const run = await agentRunEngine.run(parsed.request, (event) => {
|
|
6186
|
-
writeAgentRunSse(res, 'agent-run-event', event);
|
|
6187
|
-
}, (delta) => {
|
|
6188
|
-
writeAgentRunSse(res, 'agent-run-answer-delta', { runId: parsed.request.runId, delta });
|
|
6189
|
-
});
|
|
6190
|
-
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
6191
|
-
// The terminal frame carried the whole stored run — 2.4 MB on a real
|
|
6192
|
-
// question, of which the renderable part was ~3 KB. The rest was the
|
|
6193
|
-
// same artifacts repeated four times over. The client renders from
|
|
6194
|
-
// the projection and fetches the full record from the run-state
|
|
6195
|
-
// route only if something needs it.
|
|
6196
|
-
writeAgentRunSse(res, 'agent-run-complete', slimAgentRunForTransport(run));
|
|
6197
|
-
res.end();
|
|
6910
|
+
// The complete stored run stays in the run-state route; the live
|
|
6911
|
+
// stream ships only the renderable transport projection.
|
|
6912
|
+
writeStream('agent-run-complete', slimAgentRunForTransport(completedRun));
|
|
6913
|
+
if (streamConnected && !res.writableEnded)
|
|
6914
|
+
res.end();
|
|
6198
6915
|
return;
|
|
6199
6916
|
}
|
|
6200
|
-
const run = await agentRunEngine.run(parsed.request);
|
|
6201
|
-
recordConversationTurn(conversationStore, parsed.request.threadId, run);
|
|
6202
6917
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6203
|
-
res.end(serializeJSON({ run }));
|
|
6918
|
+
res.end(serializeJSON({ run: completedRun }));
|
|
6204
6919
|
}
|
|
6205
6920
|
finally {
|
|
6206
6921
|
if (runId)
|
|
@@ -6210,8 +6925,10 @@ export async function startLocalServer(opts) {
|
|
|
6210
6925
|
catch (error) {
|
|
6211
6926
|
const message = error instanceof Error ? error.message : String(error);
|
|
6212
6927
|
if (res.headersSent) {
|
|
6213
|
-
|
|
6214
|
-
|
|
6928
|
+
if (!res.destroyed && !res.writableEnded) {
|
|
6929
|
+
writeAgentRunSse(res, 'agent-run-error', { error: message });
|
|
6930
|
+
res.end();
|
|
6931
|
+
}
|
|
6215
6932
|
}
|
|
6216
6933
|
else {
|
|
6217
6934
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -7883,9 +8600,14 @@ export async function startLocalServer(opts) {
|
|
|
7883
8600
|
? body.variables
|
|
7884
8601
|
: {};
|
|
7885
8602
|
const dashboardVariables = dashboardRuntimeVariables(loaded.dashboard, variables);
|
|
8603
|
+
const requestedTileId = typeof body.tileId === 'string' && body.tileId.trim()
|
|
8604
|
+
? body.tileId.trim()
|
|
8605
|
+
: undefined;
|
|
7886
8606
|
const tiles = [];
|
|
7887
8607
|
let localApps = null;
|
|
7888
8608
|
for (const item of loaded.dashboard.layout.items) {
|
|
8609
|
+
if (requestedTileId && item.i !== requestedTileId)
|
|
8610
|
+
continue;
|
|
7889
8611
|
if (item.text) {
|
|
7890
8612
|
tiles.push({
|
|
7891
8613
|
tileId: item.i,
|
|
@@ -7973,9 +8695,194 @@ export async function startLocalServer(opts) {
|
|
|
7973
8695
|
kind: 'ai_pin',
|
|
7974
8696
|
name: pin.title,
|
|
7975
8697
|
},
|
|
8698
|
+
artifact: {
|
|
8699
|
+
version: 1,
|
|
8700
|
+
sourceKind: 'ai_pin',
|
|
8701
|
+
name: pinArtifact?.name ?? pin.title,
|
|
8702
|
+
...(pinArtifact?.sourcePath ? { sourcePath: pinArtifact.sourcePath } : {}),
|
|
8703
|
+
...(pinArtifact?.source ? { dql: pinArtifact.source.slice(0, 40_000) } : {}),
|
|
8704
|
+
...(pinArtifact?.compiledSql || pin.sql ? { sql: (pinArtifact?.compiledSql ?? pin.sql).slice(0, 40_000) } : {}),
|
|
8705
|
+
trustState: pinArtifact?.trustState === 'certified' || pin.certification === 'certified'
|
|
8706
|
+
? 'certified'
|
|
8707
|
+
: 'review_required',
|
|
8708
|
+
explanation: [
|
|
8709
|
+
'This result comes from a saved App insight.',
|
|
8710
|
+
pinArtifact ? 'The DQL artifact was executed again with the current App parameters.' : 'The saved bounded result is displayed with its original trust state.',
|
|
8711
|
+
],
|
|
8712
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8713
|
+
},
|
|
7976
8714
|
});
|
|
7977
8715
|
continue;
|
|
7978
8716
|
}
|
|
8717
|
+
if (item.draftAnalysis) {
|
|
8718
|
+
let draftSource;
|
|
8719
|
+
let draftAppRelativePath;
|
|
8720
|
+
let draftParameters = {};
|
|
8721
|
+
let draftTargetConnection;
|
|
8722
|
+
let draftRepairPlan;
|
|
8723
|
+
try {
|
|
8724
|
+
const draftPath = join(loaded.appDir, item.draftAnalysis.ref);
|
|
8725
|
+
draftAppRelativePath = relative(loaded.appDir, draftPath);
|
|
8726
|
+
if (draftAppRelativePath.startsWith('..') || draftAppRelativePath.startsWith('/')) {
|
|
8727
|
+
throw new Error('App-scoped analysis reference escapes the App directory.');
|
|
8728
|
+
}
|
|
8729
|
+
draftSource = readFileSync(draftPath, 'utf-8');
|
|
8730
|
+
const fingerprint = `sha256:${createHash('sha256').update(draftSource).digest('hex')}`;
|
|
8731
|
+
if (fingerprint !== item.draftAnalysis.artifactFingerprint) {
|
|
8732
|
+
throw new Error('App-scoped analysis changed after approval. Review the draft again.');
|
|
8733
|
+
}
|
|
8734
|
+
draftParameters = dashboardTileParameterValues({
|
|
8735
|
+
item,
|
|
8736
|
+
dashboardValues: dashboardVariables,
|
|
8737
|
+
requestValues: variables,
|
|
8738
|
+
});
|
|
8739
|
+
draftTargetConnection = isConnectionConfig(body.connection)
|
|
8740
|
+
? body.connection
|
|
8741
|
+
: connection ?? undefined;
|
|
8742
|
+
draftRepairPlan = buildExecutionPlan({ id: item.i, type: 'dql', source: draftSource, title: item.title ?? item.i }, {
|
|
8743
|
+
semanticLayer,
|
|
8744
|
+
driver: draftTargetConnection?.driver ?? 'duckdb',
|
|
8745
|
+
parameters: draftParameters,
|
|
8746
|
+
});
|
|
8747
|
+
let result;
|
|
8748
|
+
let repair;
|
|
8749
|
+
let repairedSql;
|
|
8750
|
+
let repairedDql;
|
|
8751
|
+
try {
|
|
8752
|
+
result = await executeDqlArtifactSourceForAgent(draftSource, { name: item.title ?? item.i, path: draftAppRelativePath }, { question: item.title, parameters: draftParameters });
|
|
8753
|
+
}
|
|
8754
|
+
catch (executionError) {
|
|
8755
|
+
const repaired = await repairFailedAppTileExecution({
|
|
8756
|
+
source: 'draft_analysis',
|
|
8757
|
+
question: item.title ?? item.i,
|
|
8758
|
+
sourceSql: draftRepairPlan?.sql,
|
|
8759
|
+
sourceDql: draftSource,
|
|
8760
|
+
artifactName: item.title ?? item.i,
|
|
8761
|
+
error: executionError,
|
|
8762
|
+
targetConnection: draftTargetConnection,
|
|
8763
|
+
targetConnectionName: projectConfig.defaultConnectionName,
|
|
8764
|
+
sqlParams: draftRepairPlan?.sqlParams,
|
|
8765
|
+
variables: { ...(draftRepairPlan?.variables ?? {}), ...draftParameters },
|
|
8766
|
+
});
|
|
8767
|
+
if (!repaired.ok) {
|
|
8768
|
+
tiles.push({
|
|
8769
|
+
tileId: item.i,
|
|
8770
|
+
status: 'error',
|
|
8771
|
+
tileType: 'draftAnalysis',
|
|
8772
|
+
title: item.title ?? item.i,
|
|
8773
|
+
error: executionError instanceof Error ? executionError.message : String(executionError),
|
|
8774
|
+
trustState: 'review_required',
|
|
8775
|
+
repair: repaired.repair,
|
|
8776
|
+
artifact: {
|
|
8777
|
+
version: 1,
|
|
8778
|
+
sourceKind: 'draft_analysis',
|
|
8779
|
+
name: item.title ?? item.i,
|
|
8780
|
+
sourcePath: draftAppRelativePath,
|
|
8781
|
+
dql: draftSource.slice(0, 40_000),
|
|
8782
|
+
...(draftRepairPlan?.sql ? { sql: draftRepairPlan.sql.slice(0, 40_000) } : {}),
|
|
8783
|
+
trustState: 'review_required',
|
|
8784
|
+
explanation: ['This App-scoped analysis remains review-required.', 'Automatic repair could not produce a safe executable result.'],
|
|
8785
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8786
|
+
},
|
|
8787
|
+
});
|
|
8788
|
+
continue;
|
|
8789
|
+
}
|
|
8790
|
+
result = repaired.result;
|
|
8791
|
+
repair = repaired.repair;
|
|
8792
|
+
repairedSql = repaired.repairedSql;
|
|
8793
|
+
repairedDql = repaired.repairedDql;
|
|
8794
|
+
}
|
|
8795
|
+
tiles.push({
|
|
8796
|
+
tileId: item.i,
|
|
8797
|
+
status: 'ok',
|
|
8798
|
+
tileType: 'draftAnalysis',
|
|
8799
|
+
title: item.title ?? item.i,
|
|
8800
|
+
viz: item.viz,
|
|
8801
|
+
chartConfig: mergeDashboardChartConfig(result.chartConfig, item),
|
|
8802
|
+
result,
|
|
8803
|
+
trustState: 'review_required',
|
|
8804
|
+
citation: { kind: repair ? 'app_draft_analysis_repaired' : 'app_draft_analysis', name: item.title ?? item.i, path: draftAppRelativePath },
|
|
8805
|
+
...(repair ? { repair } : {}),
|
|
8806
|
+
artifact: {
|
|
8807
|
+
version: 1,
|
|
8808
|
+
sourceKind: 'draft_analysis',
|
|
8809
|
+
name: item.title ?? item.i,
|
|
8810
|
+
sourcePath: draftAppRelativePath,
|
|
8811
|
+
dql: (repairedDql ?? draftSource).slice(0, 40_000),
|
|
8812
|
+
...(repairedSql ?? draftRepairPlan?.sql ? { sql: (repairedSql ?? draftRepairPlan.sql).slice(0, 40_000) } : {}),
|
|
8813
|
+
trustState: 'review_required',
|
|
8814
|
+
explanation: [
|
|
8815
|
+
'This is private App analysis and remains review-required.',
|
|
8816
|
+
repair ? 'A bounded repair ran on the original data target without changing the saved source file.' : 'The saved App-scoped DQL was executed with the current filters.',
|
|
8817
|
+
],
|
|
8818
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8819
|
+
},
|
|
8820
|
+
});
|
|
8821
|
+
}
|
|
8822
|
+
catch (err) {
|
|
8823
|
+
const repaired = await repairFailedAppTileExecution({
|
|
8824
|
+
source: 'draft_analysis',
|
|
8825
|
+
question: item.title ?? item.i,
|
|
8826
|
+
sourceSql: draftRepairPlan?.sql,
|
|
8827
|
+
sourceDql: draftSource,
|
|
8828
|
+
artifactName: item.title ?? item.i,
|
|
8829
|
+
error: err,
|
|
8830
|
+
targetConnection: draftTargetConnection,
|
|
8831
|
+
targetConnectionName: projectConfig.defaultConnectionName,
|
|
8832
|
+
sqlParams: draftRepairPlan?.sqlParams,
|
|
8833
|
+
variables: { ...(draftRepairPlan?.variables ?? {}), ...draftParameters },
|
|
8834
|
+
});
|
|
8835
|
+
if (repaired.ok && draftSource) {
|
|
8836
|
+
tiles.push({
|
|
8837
|
+
tileId: item.i,
|
|
8838
|
+
status: 'ok',
|
|
8839
|
+
tileType: 'draftAnalysis',
|
|
8840
|
+
title: item.title ?? item.i,
|
|
8841
|
+
viz: item.viz,
|
|
8842
|
+
chartConfig: mergeDashboardChartConfig(repaired.result.chartConfig, item),
|
|
8843
|
+
result: repaired.result,
|
|
8844
|
+
trustState: 'review_required',
|
|
8845
|
+
citation: { kind: 'app_draft_analysis_repaired', name: item.title ?? item.i, path: draftAppRelativePath },
|
|
8846
|
+
repair: repaired.repair,
|
|
8847
|
+
artifact: {
|
|
8848
|
+
version: 1,
|
|
8849
|
+
sourceKind: 'draft_analysis',
|
|
8850
|
+
name: item.title ?? item.i,
|
|
8851
|
+
...(draftAppRelativePath ? { sourcePath: draftAppRelativePath } : {}),
|
|
8852
|
+
dql: (repaired.repairedDql ?? draftSource).slice(0, 40_000),
|
|
8853
|
+
sql: repaired.repairedSql.slice(0, 40_000),
|
|
8854
|
+
trustState: 'review_required',
|
|
8855
|
+
explanation: ['A bounded DQL repair executed on the original data target. The saved App analysis was not changed.'],
|
|
8856
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8857
|
+
},
|
|
8858
|
+
});
|
|
8859
|
+
continue;
|
|
8860
|
+
}
|
|
8861
|
+
tiles.push({
|
|
8862
|
+
tileId: item.i,
|
|
8863
|
+
status: 'error',
|
|
8864
|
+
tileType: 'draftAnalysis',
|
|
8865
|
+
title: item.title ?? item.i,
|
|
8866
|
+
error: err instanceof Error ? err.message : String(err),
|
|
8867
|
+
trustState: 'review_required',
|
|
8868
|
+
repair: repaired.repair,
|
|
8869
|
+
...(draftSource ? {
|
|
8870
|
+
artifact: {
|
|
8871
|
+
version: 1,
|
|
8872
|
+
sourceKind: 'draft_analysis',
|
|
8873
|
+
name: item.title ?? item.i,
|
|
8874
|
+
...(draftAppRelativePath ? { sourcePath: draftAppRelativePath } : {}),
|
|
8875
|
+
dql: draftSource.slice(0, 40_000),
|
|
8876
|
+
...(draftRepairPlan?.sql ? { sql: draftRepairPlan.sql.slice(0, 40_000) } : {}),
|
|
8877
|
+
trustState: 'review_required',
|
|
8878
|
+
explanation: ['Automatic repair could not produce a safe executable result. The saved App analysis was not changed.'],
|
|
8879
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8880
|
+
},
|
|
8881
|
+
} : {}),
|
|
8882
|
+
});
|
|
8883
|
+
}
|
|
8884
|
+
continue;
|
|
8885
|
+
}
|
|
7979
8886
|
if (item.semantic) {
|
|
7980
8887
|
try {
|
|
7981
8888
|
if (!semanticLayer)
|
|
@@ -8016,16 +8923,69 @@ export async function startLocalServer(opts) {
|
|
|
8016
8923
|
if (!composed)
|
|
8017
8924
|
throw new Error('The governed semantic query could not be composed.');
|
|
8018
8925
|
const prepared = prepareLocalExecution(composed.sql, targetConnection, projectRoot, projectConfig);
|
|
8019
|
-
|
|
8926
|
+
let normalizedResult;
|
|
8927
|
+
let repair;
|
|
8928
|
+
let executedSql = composed.sql;
|
|
8929
|
+
try {
|
|
8930
|
+
const result = await executor.executeQuery(prepared.sql, [], {}, prepared.connection);
|
|
8931
|
+
normalizedResult = normalizeQueryResult(result);
|
|
8932
|
+
}
|
|
8933
|
+
catch (executionError) {
|
|
8934
|
+
const repaired = await repairFailedAppTileExecution({
|
|
8935
|
+
source: 'semantic_query',
|
|
8936
|
+
question: item.title ?? item.semantic.id,
|
|
8937
|
+
sourceSql: composed.sql,
|
|
8938
|
+
error: executionError,
|
|
8939
|
+
targetConnection,
|
|
8940
|
+
});
|
|
8941
|
+
if (!repaired.ok) {
|
|
8942
|
+
tiles.push({
|
|
8943
|
+
tileId: item.i,
|
|
8944
|
+
status: 'error',
|
|
8945
|
+
tileType: 'semantic',
|
|
8946
|
+
title: item.title ?? item.semantic.id,
|
|
8947
|
+
error: executionError instanceof Error ? executionError.message : String(executionError),
|
|
8948
|
+
trustState: 'review_required',
|
|
8949
|
+
repair: repaired.repair,
|
|
8950
|
+
artifact: {
|
|
8951
|
+
version: 1,
|
|
8952
|
+
sourceKind: 'semantic_query',
|
|
8953
|
+
name: item.title ?? item.semantic.id,
|
|
8954
|
+
sql: composed.sql.slice(0, 40_000),
|
|
8955
|
+
trustState: 'review_required',
|
|
8956
|
+
explanation: ['Automatic repair could not execute the governed semantic query safely.'],
|
|
8957
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8958
|
+
},
|
|
8959
|
+
});
|
|
8960
|
+
continue;
|
|
8961
|
+
}
|
|
8962
|
+
normalizedResult = normalizeQueryResult(repaired.result);
|
|
8963
|
+
repair = repaired.repair;
|
|
8964
|
+
executedSql = repaired.repairedSql;
|
|
8965
|
+
}
|
|
8020
8966
|
tiles.push({
|
|
8021
8967
|
tileId: item.i,
|
|
8022
8968
|
status: 'ok',
|
|
8023
8969
|
tileType: 'semantic',
|
|
8024
8970
|
title: item.title ?? item.semantic.id,
|
|
8025
8971
|
viz: item.viz,
|
|
8026
|
-
result:
|
|
8972
|
+
result: normalizedResult,
|
|
8027
8973
|
trustState: 'review_required',
|
|
8028
|
-
citation: { kind: 'semantic_query', name: item.semantic.id },
|
|
8974
|
+
citation: { kind: repair ? 'semantic_query_repaired' : 'semantic_query', name: item.semantic.id },
|
|
8975
|
+
...(repair ? { repair } : {}),
|
|
8976
|
+
artifact: {
|
|
8977
|
+
version: 1,
|
|
8978
|
+
sourceKind: 'semantic_query',
|
|
8979
|
+
name: item.title ?? item.semantic.id,
|
|
8980
|
+
sql: executedSql.slice(0, 40_000),
|
|
8981
|
+
trustState: 'review_required',
|
|
8982
|
+
explanation: [
|
|
8983
|
+
`Metrics: ${item.semantic.metrics.join(', ') || 'none'}.`,
|
|
8984
|
+
`Dimensions: ${item.semantic.dimensions?.join(', ') || 'none'}.`,
|
|
8985
|
+
repair ? 'The composed SQL needed bounded repair and therefore remains review-required.' : 'The governed semantic intent was composed for the active warehouse and remains review-required until approval.',
|
|
8986
|
+
],
|
|
8987
|
+
executionTarget: executionTargetDescriptor(body),
|
|
8988
|
+
},
|
|
8029
8989
|
});
|
|
8030
8990
|
}
|
|
8031
8991
|
catch (err) {
|
|
@@ -8050,6 +9010,11 @@ export async function startLocalServer(opts) {
|
|
|
8050
9010
|
});
|
|
8051
9011
|
continue;
|
|
8052
9012
|
}
|
|
9013
|
+
let blockSource;
|
|
9014
|
+
let blockTargetConnection;
|
|
9015
|
+
let blockPlan;
|
|
9016
|
+
let blockFilterApplication;
|
|
9017
|
+
let blockInvocation;
|
|
8053
9018
|
try {
|
|
8054
9019
|
assertAppAccess({
|
|
8055
9020
|
app: loaded.app,
|
|
@@ -8057,55 +9022,55 @@ export async function startLocalServer(opts) {
|
|
|
8057
9022
|
level: 'execute',
|
|
8058
9023
|
});
|
|
8059
9024
|
const absBlockPath = join(projectRoot, block.filePath);
|
|
8060
|
-
|
|
8061
|
-
|
|
8062
|
-
const tableMapping = await resolveSemanticTableMapping(executor,
|
|
9025
|
+
blockSource = readFileSync(absBlockPath, 'utf-8');
|
|
9026
|
+
blockTargetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
9027
|
+
const tableMapping = await resolveSemanticTableMapping(executor, blockTargetConnection, semanticLayer, projectRoot);
|
|
8063
9028
|
const boundParameters = dashboardTileParameterValues({
|
|
8064
9029
|
item,
|
|
8065
9030
|
dashboardValues: dashboardVariables,
|
|
8066
9031
|
requestValues: variables,
|
|
8067
9032
|
});
|
|
8068
|
-
|
|
9033
|
+
blockInvocation = prepareBlockInvocation({
|
|
8069
9034
|
block: block.name,
|
|
8070
|
-
source,
|
|
9035
|
+
source: blockSource,
|
|
8071
9036
|
parameters: { ...dashboardVariables, ...boundParameters },
|
|
8072
9037
|
parameterSources: Object.fromEntries(Object.keys(boundParameters).map((name) => [name, 'surface'])),
|
|
8073
9038
|
surface: 'app',
|
|
8074
9039
|
});
|
|
8075
|
-
if (
|
|
9040
|
+
if (blockInvocation.errors.length || blockInvocation.unresolvedParameters.length) {
|
|
8076
9041
|
tiles.push({
|
|
8077
9042
|
tileId: item.i,
|
|
8078
9043
|
status: 'unresolved',
|
|
8079
9044
|
tileType: 'block',
|
|
8080
9045
|
blockId: block.name,
|
|
8081
9046
|
title: item.title ?? block.name,
|
|
8082
|
-
error:
|
|
9047
|
+
error: blockInvocation.errors[0] ?? `Needs values for: ${blockInvocation.unresolvedParameters.join(', ')}`,
|
|
8083
9048
|
invocation: {
|
|
8084
|
-
resolvedParameters:
|
|
8085
|
-
unresolvedParameters:
|
|
8086
|
-
auditId:
|
|
9049
|
+
resolvedParameters: blockInvocation.resolvedParameters,
|
|
9050
|
+
unresolvedParameters: blockInvocation.unresolvedParameters,
|
|
9051
|
+
auditId: blockInvocation.auditId,
|
|
8087
9052
|
},
|
|
8088
9053
|
});
|
|
8089
9054
|
continue;
|
|
8090
9055
|
}
|
|
8091
9056
|
const semanticCompose = semanticLayer
|
|
8092
|
-
? await composeSemanticBlockSqlForRuntime(
|
|
8093
|
-
driver:
|
|
9057
|
+
? await composeSemanticBlockSqlForRuntime(blockSource, semanticLayer, {
|
|
9058
|
+
driver: blockTargetConnection.driver,
|
|
8094
9059
|
tableMapping,
|
|
8095
9060
|
projectRoot,
|
|
8096
9061
|
projectConfig,
|
|
8097
9062
|
detectedProvider: semanticDetectedProvider,
|
|
8098
|
-
parameters:
|
|
9063
|
+
parameters: blockInvocation.values,
|
|
8099
9064
|
})
|
|
8100
9065
|
: null;
|
|
8101
|
-
|
|
9066
|
+
blockPlan = buildExecutionPlan({ id: item.i, type: 'dql', source: blockSource, title: item.title ?? block.name }, {
|
|
8102
9067
|
semanticLayer,
|
|
8103
|
-
driver:
|
|
9068
|
+
driver: blockTargetConnection.driver,
|
|
8104
9069
|
tableMapping,
|
|
8105
|
-
parameters:
|
|
9070
|
+
parameters: blockInvocation.values,
|
|
8106
9071
|
semanticSql: semanticCompose?.sql ?? undefined,
|
|
8107
9072
|
});
|
|
8108
|
-
if (!
|
|
9073
|
+
if (!blockPlan && !semanticCompose?.sql) {
|
|
8109
9074
|
tiles.push({
|
|
8110
9075
|
tileId: item.i,
|
|
8111
9076
|
status: 'error',
|
|
@@ -8114,15 +9079,15 @@ export async function startLocalServer(opts) {
|
|
|
8114
9079
|
});
|
|
8115
9080
|
continue;
|
|
8116
9081
|
}
|
|
8117
|
-
|
|
8118
|
-
sql: semanticCompose?.sql ??
|
|
8119
|
-
sqlParams:
|
|
8120
|
-
variables: { ...(
|
|
9082
|
+
blockFilterApplication = applyDashboardFiltersToBlockExecution({
|
|
9083
|
+
sql: semanticCompose?.sql ?? blockPlan.sql,
|
|
9084
|
+
sqlParams: blockPlan?.sqlParams ?? [],
|
|
9085
|
+
variables: { ...(blockPlan?.variables ?? {}), ...dashboardVariables },
|
|
8121
9086
|
block,
|
|
8122
9087
|
dashboard: loaded.dashboard,
|
|
8123
9088
|
});
|
|
8124
|
-
const prepared = prepareLocalExecution(
|
|
8125
|
-
const result = await executor.executeQuery(prepared.sql,
|
|
9089
|
+
const prepared = prepareLocalExecution(blockFilterApplication.sql, blockTargetConnection, projectRoot, projectConfig);
|
|
9090
|
+
const result = await executor.executeQuery(prepared.sql, blockFilterApplication.sqlParams, runtimeVariables({ ...blockFilterApplication.variables, ...blockInvocation.values }), prepared.connection);
|
|
8126
9091
|
tiles.push({
|
|
8127
9092
|
tileId: item.i,
|
|
8128
9093
|
status: 'ok',
|
|
@@ -8131,22 +9096,37 @@ export async function startLocalServer(opts) {
|
|
|
8131
9096
|
certificationStatus: block.status ?? null,
|
|
8132
9097
|
title: item.title ?? block.name,
|
|
8133
9098
|
viz: item.viz,
|
|
8134
|
-
chartConfig: mergeDashboardChartConfig(
|
|
9099
|
+
chartConfig: mergeDashboardChartConfig(blockPlan?.chartConfig, item),
|
|
8135
9100
|
result: normalizeQueryResult(result),
|
|
8136
9101
|
filters: {
|
|
8137
|
-
applied:
|
|
8138
|
-
skipped:
|
|
9102
|
+
applied: blockFilterApplication.appliedFilters,
|
|
9103
|
+
skipped: blockFilterApplication.skippedFilters,
|
|
8139
9104
|
},
|
|
8140
9105
|
invocation: {
|
|
8141
|
-
resolvedParameters:
|
|
8142
|
-
unresolvedParameters:
|
|
8143
|
-
auditId:
|
|
9106
|
+
resolvedParameters: blockInvocation.resolvedParameters,
|
|
9107
|
+
unresolvedParameters: blockInvocation.unresolvedParameters,
|
|
9108
|
+
auditId: blockInvocation.auditId,
|
|
8144
9109
|
},
|
|
9110
|
+
trustState: block.status === 'certified' ? 'certified' : 'review_required',
|
|
8145
9111
|
citation: {
|
|
8146
9112
|
kind: 'block',
|
|
8147
9113
|
name: block.name,
|
|
8148
9114
|
path: block.filePath,
|
|
8149
9115
|
},
|
|
9116
|
+
artifact: {
|
|
9117
|
+
version: 1,
|
|
9118
|
+
sourceKind: 'certified_block',
|
|
9119
|
+
name: block.name,
|
|
9120
|
+
sourcePath: block.filePath,
|
|
9121
|
+
dql: blockSource.slice(0, 40_000),
|
|
9122
|
+
sql: blockFilterApplication.sql.slice(0, 40_000),
|
|
9123
|
+
trustState: block.status === 'certified' ? 'certified' : 'review_required',
|
|
9124
|
+
explanation: [
|
|
9125
|
+
block.status === 'certified' ? 'This tile executed a certified project block.' : 'This project block is not certified and remains review-required.',
|
|
9126
|
+
'Dashboard filters and typed parameters were bound before read-only execution.',
|
|
9127
|
+
],
|
|
9128
|
+
executionTarget: executionTargetDescriptor(body),
|
|
9129
|
+
},
|
|
8150
9130
|
});
|
|
8151
9131
|
}
|
|
8152
9132
|
catch (err) {
|
|
@@ -8159,11 +9139,87 @@ export async function startLocalServer(opts) {
|
|
|
8159
9139
|
});
|
|
8160
9140
|
}
|
|
8161
9141
|
else {
|
|
9142
|
+
const repaired = await repairFailedAppTileExecution({
|
|
9143
|
+
source: 'certified_block',
|
|
9144
|
+
question: item.title ?? block.name,
|
|
9145
|
+
sourceSql: blockFilterApplication?.sql ?? blockPlan?.sql,
|
|
9146
|
+
sourceDql: blockSource,
|
|
9147
|
+
artifactName: block.name,
|
|
9148
|
+
error: err,
|
|
9149
|
+
targetConnection: blockTargetConnection,
|
|
9150
|
+
targetConnectionName: projectConfig.defaultConnectionName,
|
|
9151
|
+
sqlParams: blockFilterApplication?.sqlParams ?? blockPlan?.sqlParams,
|
|
9152
|
+
variables: {
|
|
9153
|
+
...(blockPlan?.variables ?? {}),
|
|
9154
|
+
...(blockFilterApplication?.variables ?? {}),
|
|
9155
|
+
...(blockInvocation?.values ?? {}),
|
|
9156
|
+
},
|
|
9157
|
+
});
|
|
9158
|
+
if (repaired.ok) {
|
|
9159
|
+
tiles.push({
|
|
9160
|
+
tileId: item.i,
|
|
9161
|
+
status: 'ok',
|
|
9162
|
+
tileType: 'block',
|
|
9163
|
+
blockId: block.name,
|
|
9164
|
+
blockPath: block.filePath,
|
|
9165
|
+
certificationStatus: block.status ?? null,
|
|
9166
|
+
title: item.title ?? block.name,
|
|
9167
|
+
viz: item.viz,
|
|
9168
|
+
chartConfig: mergeDashboardChartConfig(blockPlan?.chartConfig, item),
|
|
9169
|
+
result: normalizeQueryResult(repaired.result),
|
|
9170
|
+
...(blockFilterApplication ? {
|
|
9171
|
+
filters: {
|
|
9172
|
+
applied: blockFilterApplication.appliedFilters,
|
|
9173
|
+
skipped: blockFilterApplication.skippedFilters,
|
|
9174
|
+
},
|
|
9175
|
+
} : {}),
|
|
9176
|
+
...(blockInvocation ? {
|
|
9177
|
+
invocation: {
|
|
9178
|
+
resolvedParameters: blockInvocation.resolvedParameters,
|
|
9179
|
+
unresolvedParameters: blockInvocation.unresolvedParameters,
|
|
9180
|
+
auditId: blockInvocation.auditId,
|
|
9181
|
+
},
|
|
9182
|
+
} : {}),
|
|
9183
|
+
trustState: 'review_required',
|
|
9184
|
+
repair: repaired.repair,
|
|
9185
|
+
citation: { kind: 'block_repaired', name: block.name, path: block.filePath },
|
|
9186
|
+
artifact: {
|
|
9187
|
+
version: 1,
|
|
9188
|
+
sourceKind: 'certified_block',
|
|
9189
|
+
name: block.name,
|
|
9190
|
+
sourcePath: block.filePath,
|
|
9191
|
+
...(repaired.repairedDql ?? blockSource ? { dql: (repaired.repairedDql ?? blockSource).slice(0, 40_000) } : {}),
|
|
9192
|
+
sql: repaired.repairedSql.slice(0, 40_000),
|
|
9193
|
+
trustState: 'review_required',
|
|
9194
|
+
explanation: [
|
|
9195
|
+
'The saved certified block was not changed.',
|
|
9196
|
+
'A bounded repair executed on the original data target; this repaired result requires human review.',
|
|
9197
|
+
],
|
|
9198
|
+
executionTarget: executionTargetDescriptor(body),
|
|
9199
|
+
},
|
|
9200
|
+
});
|
|
9201
|
+
continue;
|
|
9202
|
+
}
|
|
8162
9203
|
tiles.push({
|
|
8163
9204
|
tileId: item.i,
|
|
8164
9205
|
status: 'error',
|
|
9206
|
+
tileType: 'block',
|
|
8165
9207
|
blockId: block.name,
|
|
8166
9208
|
error: err instanceof Error ? err.message : String(err),
|
|
9209
|
+
repair: repaired.repair,
|
|
9210
|
+
...(blockSource ? {
|
|
9211
|
+
artifact: {
|
|
9212
|
+
version: 1,
|
|
9213
|
+
sourceKind: 'certified_block',
|
|
9214
|
+
name: block.name,
|
|
9215
|
+
sourcePath: block.filePath,
|
|
9216
|
+
dql: blockSource.slice(0, 40_000),
|
|
9217
|
+
...(blockFilterApplication?.sql ?? blockPlan?.sql ? { sql: (blockFilterApplication?.sql ?? blockPlan.sql).slice(0, 40_000) } : {}),
|
|
9218
|
+
trustState: 'review_required',
|
|
9219
|
+
explanation: ['Automatic repair could not produce a safe executable result. The saved block was not changed.'],
|
|
9220
|
+
executionTarget: executionTargetDescriptor(body),
|
|
9221
|
+
},
|
|
9222
|
+
} : {}),
|
|
8167
9223
|
});
|
|
8168
9224
|
}
|
|
8169
9225
|
}
|
|
@@ -8198,6 +9254,10 @@ export async function startLocalServer(opts) {
|
|
|
8198
9254
|
filterFingerprint,
|
|
8199
9255
|
resultFingerprint,
|
|
8200
9256
|
personaFingerprint,
|
|
9257
|
+
successfulTileIds: tiles.filter((tile) => tile.status === 'ok').map((tile) => tile.tileId),
|
|
9258
|
+
semanticApprovalEligibleTileIds: tiles.filter((tile) => (tile.status === 'ok'
|
|
9259
|
+
&& tile.tileType === 'semantic'
|
|
9260
|
+
&& (!('repair' in tile) || tile.repair?.approvalEligible !== false))).map((tile) => tile.tileId),
|
|
8201
9261
|
facts: storyResult.facts,
|
|
8202
9262
|
story: storyResult.story,
|
|
8203
9263
|
expiresAt: Date.now() + 15 * 60_000,
|
|
@@ -8249,7 +9309,30 @@ export async function startLocalServer(opts) {
|
|
|
8249
9309
|
// Two-phase app build gap-fill: bounded governed answers for uncovered
|
|
8250
9310
|
// questions. Throws when no provider is configured — propose degrades
|
|
8251
9311
|
// gracefully by listing the gap instead.
|
|
8252
|
-
generateGovernedAnswer: (question) =>
|
|
9312
|
+
generateGovernedAnswer: (question, appContext) => generateAppBuildAnswerWithRepair(question, appContext),
|
|
9313
|
+
queueOperation: (input, task) => {
|
|
9314
|
+
const operation = operationCoordinator.create(input);
|
|
9315
|
+
void operationCoordinator.run(operation.id, task);
|
|
9316
|
+
return operation;
|
|
9317
|
+
},
|
|
9318
|
+
scheduleProjectRefresh: (reason) => { scheduleProjectRefresh(reason); },
|
|
9319
|
+
verifyDashboardRun: ({ runId, appId, dashboardId, tileIds }) => {
|
|
9320
|
+
const evidence = dashboardRunEvidence.get(runId);
|
|
9321
|
+
if (!evidence || evidence.expiresAt < Date.now())
|
|
9322
|
+
return { ok: false, error: 'The dashboard run receipt expired. Run the dashboard again.' };
|
|
9323
|
+
if (evidence.appId !== appId || evidence.dashboardId !== dashboardId)
|
|
9324
|
+
return { ok: false, error: 'The dashboard run receipt does not match this App page.' };
|
|
9325
|
+
const successful = new Set(evidence.successfulTileIds);
|
|
9326
|
+
const failed = tileIds.filter((tileId) => !successful.has(tileId));
|
|
9327
|
+
if (failed.length > 0)
|
|
9328
|
+
return { ok: false, error: `These semantic tiles did not complete successfully: ${failed.join(', ')}` };
|
|
9329
|
+
const approvalEligible = new Set(evidence.semanticApprovalEligibleTileIds);
|
|
9330
|
+
const repaired = tileIds.filter((tileId) => !approvalEligible.has(tileId));
|
|
9331
|
+
if (repaired.length > 0) {
|
|
9332
|
+
return { ok: false, error: `These semantic previews required SQL repair and cannot be approved as governed semantic results: ${repaired.join(', ')}. Review them as exploratory App analysis instead.` };
|
|
9333
|
+
}
|
|
9334
|
+
return { ok: true, snapshotId: evidence.snapshotId, resultFingerprint: evidence.resultFingerprint };
|
|
9335
|
+
},
|
|
8253
9336
|
// Story narration for commit — LLM-backed with deterministic fallback.
|
|
8254
9337
|
narrate: narrateForAgentRun,
|
|
8255
9338
|
});
|
|
@@ -8425,13 +9508,21 @@ export async function startLocalServer(opts) {
|
|
|
8425
9508
|
res.end(serializeJSON({ error: 'Invalid path' }));
|
|
8426
9509
|
return;
|
|
8427
9510
|
}
|
|
8428
|
-
|
|
9511
|
+
await mkdirAsync(dirname(absPath), { recursive: true });
|
|
8429
9512
|
const toWrite = absPath.endsWith('.dql')
|
|
8430
9513
|
? canonicalizeSafe(content)
|
|
8431
9514
|
: absPath.endsWith('.dqlnb')
|
|
8432
9515
|
? canonicalizeNotebookSafe(content)
|
|
8433
9516
|
: content;
|
|
8434
|
-
|
|
9517
|
+
const tempPath = `${absPath}.tmp-${process.pid}-${Date.now()}`;
|
|
9518
|
+
try {
|
|
9519
|
+
await writeFileAsync(tempPath, toWrite, 'utf-8');
|
|
9520
|
+
await renameAsync(tempPath, absPath);
|
|
9521
|
+
}
|
|
9522
|
+
catch (error) {
|
|
9523
|
+
await rmAsync(tempPath, { force: true }).catch(() => undefined);
|
|
9524
|
+
throw error;
|
|
9525
|
+
}
|
|
8435
9526
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8436
9527
|
res.end(serializeJSON({ ok: true }));
|
|
8437
9528
|
}
|
|
@@ -9720,21 +10811,194 @@ export async function startLocalServer(opts) {
|
|
|
9720
10811
|
res.end(serializeJSON({ error: 'source is required' }));
|
|
9721
10812
|
return;
|
|
9722
10813
|
}
|
|
9723
|
-
const summary = await runBlockStudioTestSummary(source);
|
|
9724
|
-
const results = summary.assertions.map((assertion) => ({
|
|
9725
|
-
name: assertion.name,
|
|
9726
|
-
field: assertion.name.match(/^assert\s+(\S+)/)?.[1] ?? assertion.name,
|
|
9727
|
-
operator: assertion.name.match(/^assert\s+\S+\s+(\S+)/)?.[1] ?? '',
|
|
9728
|
-
expected: assertion.expected !== undefined ? String(assertion.expected) : assertion.name.replace(/^assert\s+\S+\s+\S+\s*/, ''),
|
|
9729
|
-
passed: assertion.passed,
|
|
9730
|
-
actual: assertion.error ?? (assertion.actual !== undefined ? String(assertion.actual) : undefined),
|
|
10814
|
+
const summary = await runBlockStudioTestSummary(source);
|
|
10815
|
+
const results = summary.assertions.map((assertion) => ({
|
|
10816
|
+
name: assertion.name,
|
|
10817
|
+
field: assertion.name.match(/^assert\s+(\S+)/)?.[1] ?? assertion.name,
|
|
10818
|
+
operator: assertion.name.match(/^assert\s+\S+\s+(\S+)/)?.[1] ?? '',
|
|
10819
|
+
expected: assertion.expected !== undefined ? String(assertion.expected) : assertion.name.replace(/^assert\s+\S+\s+\S+\s*/, ''),
|
|
10820
|
+
passed: assertion.passed,
|
|
10821
|
+
actual: assertion.error ?? (assertion.actual !== undefined ? String(assertion.actual) : undefined),
|
|
10822
|
+
}));
|
|
10823
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10824
|
+
res.end(serializeJSON({ assertions: results, passed: summary.passed, failed: summary.failed, duration: summary.duration }));
|
|
10825
|
+
}
|
|
10826
|
+
catch (error) {
|
|
10827
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10828
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
10829
|
+
}
|
|
10830
|
+
return;
|
|
10831
|
+
}
|
|
10832
|
+
if (req.method === 'POST' && path === '/api/block-studio/certifications') {
|
|
10833
|
+
try {
|
|
10834
|
+
const body = await readJSON(req);
|
|
10835
|
+
const source = typeof body.source === 'string' ? body.source : '';
|
|
10836
|
+
const suppliedMetadata = body.metadata && typeof body.metadata === 'object'
|
|
10837
|
+
? body.metadata
|
|
10838
|
+
: {};
|
|
10839
|
+
const currentPath = typeof body.path === 'string'
|
|
10840
|
+
? normalize(body.path).replace(/^\/+/, '').replaceAll('\\', '/')
|
|
10841
|
+
: undefined;
|
|
10842
|
+
const expectedSourceFingerprint = typeof body.expectedSourceFingerprint === 'string'
|
|
10843
|
+
? body.expectedSourceFingerprint
|
|
10844
|
+
: undefined;
|
|
10845
|
+
const idempotencyKey = typeof req.headers['idempotency-key'] === 'string'
|
|
10846
|
+
? req.headers['idempotency-key'].trim()
|
|
10847
|
+
: '';
|
|
10848
|
+
if (!source.trim()) {
|
|
10849
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10850
|
+
res.end(serializeJSON({ error: 'source is required' }));
|
|
10851
|
+
return;
|
|
10852
|
+
}
|
|
10853
|
+
if (idempotencyKey) {
|
|
10854
|
+
const existingId = operationIdempotency.get(idempotencyKey);
|
|
10855
|
+
const existing = existingId ? operationCoordinator.get(existingId) : null;
|
|
10856
|
+
if (existing) {
|
|
10857
|
+
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10858
|
+
res.end(serializeJSON({ operation: existing, deduplicated: true }));
|
|
10859
|
+
return;
|
|
10860
|
+
}
|
|
10861
|
+
}
|
|
10862
|
+
if (currentPath && expectedSourceFingerprint) {
|
|
10863
|
+
const absoluteCurrentPath = safeJoin(projectRoot, currentPath);
|
|
10864
|
+
if (!absoluteCurrentPath || !existsSync(absoluteCurrentPath)) {
|
|
10865
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10866
|
+
res.end(serializeJSON({ error: 'The block changed or moved. Reopen it before certifying.', code: 'SOURCE_CHANGED' }));
|
|
10867
|
+
return;
|
|
10868
|
+
}
|
|
10869
|
+
const actualFingerprint = createHash('sha256')
|
|
10870
|
+
.update(readFileSync(absoluteCurrentPath, 'utf-8'))
|
|
10871
|
+
.digest('hex');
|
|
10872
|
+
if (actualFingerprint !== expectedSourceFingerprint) {
|
|
10873
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10874
|
+
res.end(serializeJSON({ error: 'The block changed after it was opened. Reopen it before certifying.', code: 'SOURCE_CHANGED' }));
|
|
10875
|
+
return;
|
|
10876
|
+
}
|
|
10877
|
+
}
|
|
10878
|
+
const draftSource = setBlockStudioStatusInSource(source, 'draft');
|
|
10879
|
+
const parsed = parseBlockSourceMetadata(draftSource);
|
|
10880
|
+
const name = typeof suppliedMetadata.name === 'string' && suppliedMetadata.name.trim()
|
|
10881
|
+
? suppliedMetadata.name.trim()
|
|
10882
|
+
: parsed.name;
|
|
10883
|
+
if (!name) {
|
|
10884
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10885
|
+
res.end(serializeJSON({ error: 'Block name is required.' }));
|
|
10886
|
+
return;
|
|
10887
|
+
}
|
|
10888
|
+
const canonicalPath = currentPath && !isDraftBlockPath(currentPath) ? currentPath : undefined;
|
|
10889
|
+
const stableSuffix = createHash('sha256')
|
|
10890
|
+
.update(currentPath ?? `${name}:${draftSource}`)
|
|
10891
|
+
.digest('hex')
|
|
10892
|
+
.slice(0, 16);
|
|
10893
|
+
const draftPath = saveBlockStudioDraftArtifacts(projectRoot, {
|
|
10894
|
+
currentPath: currentPath && isDraftBlockPath(currentPath) ? currentPath : undefined,
|
|
10895
|
+
source: draftSource,
|
|
10896
|
+
name,
|
|
10897
|
+
domain: typeof suppliedMetadata.domain === 'string' ? suppliedMetadata.domain : parsed.domain,
|
|
10898
|
+
description: typeof suppliedMetadata.description === 'string' ? suppliedMetadata.description : parsed.description,
|
|
10899
|
+
owner: typeof suppliedMetadata.owner === 'string' ? suppliedMetadata.owner : parsed.owner,
|
|
10900
|
+
tags: Array.isArray(suppliedMetadata.tags) ? suppliedMetadata.tags.map(String) : parsed.tags,
|
|
10901
|
+
lineage: Array.isArray(suppliedMetadata.lineage) ? suppliedMetadata.lineage.map(String) : undefined,
|
|
10902
|
+
stableSuffix,
|
|
10903
|
+
});
|
|
10904
|
+
const operation = operationCoordinator.create({
|
|
10905
|
+
type: 'block_certification',
|
|
10906
|
+
scope: `block:${draftPath}`,
|
|
10907
|
+
resourceRevision: typeof body.clientRevision === 'string'
|
|
10908
|
+
? body.clientRevision
|
|
10909
|
+
: createHash('sha256').update(draftSource).digest('hex'),
|
|
10910
|
+
message: `Certification queued for ${name}.`,
|
|
10911
|
+
cancellable: true,
|
|
10912
|
+
});
|
|
10913
|
+
if (idempotencyKey) {
|
|
10914
|
+
operationIdempotency.set(idempotencyKey, operation.id);
|
|
10915
|
+
while (operationIdempotency.size > 100) {
|
|
10916
|
+
const oldest = operationIdempotency.keys().next().value;
|
|
10917
|
+
if (typeof oldest !== 'string')
|
|
10918
|
+
break;
|
|
10919
|
+
operationIdempotency.delete(oldest);
|
|
10920
|
+
}
|
|
10921
|
+
}
|
|
10922
|
+
operationCoordinator.run(operation.id, async (signal, report) => {
|
|
10923
|
+
report({ phase: 'validating', progress: 15, message: 'Validating and running the saved draft once.' });
|
|
10924
|
+
const result = await certifyBlockStudioSource(draftSource, draftPath, { enterprise: body.enterprise === true });
|
|
10925
|
+
const blockers = Array.from(new Set(result.checklist.blockers));
|
|
10926
|
+
if (signal.aborted)
|
|
10927
|
+
throw Object.assign(new Error('Certification cancelled.'), { code: 'OPERATION_CANCELLED' });
|
|
10928
|
+
if (!result.certification.certified || blockers.length > 0) {
|
|
10929
|
+
return {
|
|
10930
|
+
outcome: 'draft_saved_with_blockers',
|
|
10931
|
+
oldPath: currentPath ?? draftPath,
|
|
10932
|
+
draftPath,
|
|
10933
|
+
blockers,
|
|
10934
|
+
checklist: result.checklist,
|
|
10935
|
+
block: openBlockStudioDocument(projectRoot, draftPath, semanticLayer),
|
|
10936
|
+
};
|
|
10937
|
+
}
|
|
10938
|
+
report({ phase: 'publishing', progress: 80, message: 'Publishing the certified block.' });
|
|
10939
|
+
const certifiedSource = setBlockStudioStatusInSource(draftSource, 'certified');
|
|
10940
|
+
const certifiedMetadata = parseBlockSourceMetadata(certifiedSource);
|
|
10941
|
+
const savedPath = saveBlockStudioArtifacts(projectRoot, {
|
|
10942
|
+
currentPath: canonicalPath ?? draftPath,
|
|
10943
|
+
source: certifiedSource,
|
|
10944
|
+
name: certifiedMetadata.name || name,
|
|
10945
|
+
domain: certifiedMetadata.domain,
|
|
10946
|
+
description: certifiedMetadata.description,
|
|
10947
|
+
owner: certifiedMetadata.owner,
|
|
10948
|
+
tags: certifiedMetadata.tags,
|
|
10949
|
+
});
|
|
10950
|
+
if (canonicalPath && draftPath !== savedPath) {
|
|
10951
|
+
try {
|
|
10952
|
+
deleteBlockStudioArtifacts(projectRoot, draftPath);
|
|
10953
|
+
}
|
|
10954
|
+
catch { /* certified source is already durable */ }
|
|
10955
|
+
}
|
|
10956
|
+
// Shared learning remains the same deterministic, advisory write used
|
|
10957
|
+
// by the synchronous compatibility route. It never gates publication.
|
|
10958
|
+
try {
|
|
10959
|
+
const learnedOutputs = Array.isArray(certifiedMetadata.outputs)
|
|
10960
|
+
? certifiedMetadata.outputs.filter((output) => typeof output === 'string')
|
|
10961
|
+
: [];
|
|
10962
|
+
if (certifiedMetadata.name) {
|
|
10963
|
+
const memory = new MemoryStore(defaultMemoryPath(projectRoot));
|
|
10964
|
+
memory.upsert({
|
|
10965
|
+
id: `mem_certify_${certifiedMetadata.name}`,
|
|
10966
|
+
scope: 'project',
|
|
10967
|
+
title: `Certified block: ${certifiedMetadata.name}`,
|
|
10968
|
+
content: `Prefer the certified block "${certifiedMetadata.name}" for ${certifiedMetadata.description?.trim() || `questions in the ${certifiedMetadata.domain ?? 'analytics'} domain`}.${certifiedMetadata.grain ? ` Grain: ${certifiedMetadata.grain}.` : ''}${learnedOutputs.length ? ` Outputs: ${learnedOutputs.slice(0, 8).join(', ')}.` : ''} It is the trusted source — reuse it instead of generating new SQL.`,
|
|
10969
|
+
tags: [certifiedMetadata.domain, certifiedMetadata.name, ...learnedOutputs.slice(0, 4)].filter((value) => Boolean(value)),
|
|
10970
|
+
source: 'certify',
|
|
10971
|
+
confidence: 0.95,
|
|
10972
|
+
importance: 0.85,
|
|
10973
|
+
enabled: true,
|
|
10974
|
+
});
|
|
10975
|
+
}
|
|
10976
|
+
}
|
|
10977
|
+
catch {
|
|
10978
|
+
/* best-effort: learning capture must never block certification */
|
|
10979
|
+
}
|
|
10980
|
+
const refreshOperation = scheduleProjectRefresh('block-studio-certification');
|
|
10981
|
+
return {
|
|
10982
|
+
outcome: 'certified',
|
|
10983
|
+
oldPath: currentPath ?? draftPath,
|
|
10984
|
+
draftPath,
|
|
10985
|
+
newPath: savedPath,
|
|
10986
|
+
block: openBlockStudioDocument(projectRoot, savedPath, semanticLayer),
|
|
10987
|
+
checklist: result.checklist,
|
|
10988
|
+
indexRefresh: { status: 'queued', operationId: refreshOperation.id },
|
|
10989
|
+
};
|
|
10990
|
+
});
|
|
10991
|
+
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10992
|
+
res.end(serializeJSON({
|
|
10993
|
+
operation: operationCoordinator.get(operation.id) ?? operation,
|
|
10994
|
+
draft: openBlockStudioDocument(projectRoot, draftPath, semanticLayer),
|
|
9731
10995
|
}));
|
|
9732
|
-
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9733
|
-
res.end(serializeJSON({ assertions: results, passed: summary.passed, failed: summary.failed, duration: summary.duration }));
|
|
9734
10996
|
}
|
|
9735
10997
|
catch (error) {
|
|
9736
|
-
|
|
9737
|
-
|
|
10998
|
+
const code = apiErrorCode(error, 'CERTIFICATION_REQUEST_FAILED');
|
|
10999
|
+
const status = code === 'SOURCE_CHANGED' || apiErrorMessage(error) === 'BLOCK_EXISTS' ? 409 : 500;
|
|
11000
|
+
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
11001
|
+
res.end(serializeJSON({ error: apiErrorMessage(error), code }));
|
|
9738
11002
|
}
|
|
9739
11003
|
return;
|
|
9740
11004
|
}
|
|
@@ -9807,7 +11071,7 @@ export async function startLocalServer(opts) {
|
|
|
9807
11071
|
catch {
|
|
9808
11072
|
/* best-effort: learning capture must never block certification */
|
|
9809
11073
|
}
|
|
9810
|
-
|
|
11074
|
+
const refreshOperation = scheduleProjectRefresh('block-studio-certify');
|
|
9811
11075
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9812
11076
|
res.end(serializeJSON({
|
|
9813
11077
|
ok: true,
|
|
@@ -9818,6 +11082,7 @@ export async function startLocalServer(opts) {
|
|
|
9818
11082
|
metadata: certifiedPayload?.metadata,
|
|
9819
11083
|
companionPath: certifiedPayload?.companionPath ?? null,
|
|
9820
11084
|
validation: certifiedPayload?.validation ?? result.validation,
|
|
11085
|
+
lineageRefresh: { status: 'queued', operationId: refreshOperation.id },
|
|
9821
11086
|
}));
|
|
9822
11087
|
}
|
|
9823
11088
|
catch (error) {
|
|
@@ -10399,10 +11664,20 @@ export async function startLocalServer(opts) {
|
|
|
10399
11664
|
// of the same 7,500+ object catalog while keeping the route compatible
|
|
10400
11665
|
// for older clients.
|
|
10401
11666
|
const includeSemantic = url.searchParams.get('includeSemantic') !== 'false';
|
|
11667
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, cfg);
|
|
11668
|
+
let databaseTree = [];
|
|
11669
|
+
if (cfg.manifestVersion === 3 && cfg.modeling?.mode === 'dbt-first' && dbtManifestPath) {
|
|
11670
|
+
try {
|
|
11671
|
+
databaseTree = buildDbtDatabaseSchemaTree(projectSnapshot().manifest, 25, physicalDbtRelationIds(dbtManifestPath));
|
|
11672
|
+
}
|
|
11673
|
+
catch {
|
|
11674
|
+
// Keep the rest of Block Studio available while dbt is rebuilt.
|
|
11675
|
+
}
|
|
11676
|
+
}
|
|
10402
11677
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10403
11678
|
res.end(serializeJSON({
|
|
10404
11679
|
semanticTree: includeSemantic && semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
10405
|
-
databaseTree
|
|
11680
|
+
databaseTree,
|
|
10406
11681
|
connection: {
|
|
10407
11682
|
default: defaultKey,
|
|
10408
11683
|
current: defaultKey,
|
|
@@ -10657,25 +11932,13 @@ export async function startLocalServer(opts) {
|
|
|
10657
11932
|
stableSuffix: metadata.candidateId,
|
|
10658
11933
|
})
|
|
10659
11934
|
: saveBlockStudioArtifacts(projectRoot, saveOptions);
|
|
10660
|
-
|
|
10661
|
-
let lineageRefresh;
|
|
10662
|
-
try {
|
|
10663
|
-
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
10664
|
-
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
10665
|
-
}
|
|
10666
|
-
catch (error) {
|
|
10667
|
-
// The block is already safely stored. Preserve it and report the
|
|
10668
|
-
// rebuild failure separately instead of turning a successful save
|
|
10669
|
-
// into a misleading 500 response.
|
|
10670
|
-
lineageRefresh = {
|
|
10671
|
-
status: 'failed',
|
|
10672
|
-
message: error instanceof Error ? error.message : String(error),
|
|
10673
|
-
};
|
|
10674
|
-
}
|
|
10675
|
-
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
11935
|
+
const refreshOperation = scheduleProjectRefresh('block-studio-save');
|
|
10676
11936
|
const payload = {
|
|
10677
11937
|
...openBlockStudioDocument(projectRoot, savedPath, semanticLayer),
|
|
10678
|
-
lineageRefresh
|
|
11938
|
+
lineageRefresh: {
|
|
11939
|
+
status: 'queued',
|
|
11940
|
+
operationId: refreshOperation.id,
|
|
11941
|
+
},
|
|
10679
11942
|
};
|
|
10680
11943
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10681
11944
|
res.end(serializeJSON(payload));
|
|
@@ -10695,21 +11958,13 @@ export async function startLocalServer(opts) {
|
|
|
10695
11958
|
try {
|
|
10696
11959
|
const blockPath = url.searchParams.get('path');
|
|
10697
11960
|
const deleted = deleteBlockStudioArtifacts(projectRoot, blockPath ?? '');
|
|
10698
|
-
|
|
10699
|
-
let lineageRefresh;
|
|
10700
|
-
try {
|
|
10701
|
-
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
10702
|
-
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
10703
|
-
}
|
|
10704
|
-
catch (error) {
|
|
10705
|
-
lineageRefresh = {
|
|
10706
|
-
status: 'failed',
|
|
10707
|
-
message: error instanceof Error ? error.message : String(error),
|
|
10708
|
-
};
|
|
10709
|
-
}
|
|
10710
|
-
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
11961
|
+
const refreshOperation = scheduleProjectRefresh('block-studio-delete');
|
|
10711
11962
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10712
|
-
res.end(serializeJSON({
|
|
11963
|
+
res.end(serializeJSON({
|
|
11964
|
+
ok: true,
|
|
11965
|
+
...deleted,
|
|
11966
|
+
lineageRefresh: { status: 'queued', operationId: refreshOperation.id },
|
|
11967
|
+
}));
|
|
10713
11968
|
}
|
|
10714
11969
|
catch (error) {
|
|
10715
11970
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -12173,13 +13428,17 @@ export async function startLocalServer(opts) {
|
|
|
12173
13428
|
// GET /api/describe-table?table=schema.table — returns columns for a specific table
|
|
12174
13429
|
if (req.method === 'GET' && path === '/api/describe-table') {
|
|
12175
13430
|
try {
|
|
12176
|
-
const
|
|
12177
|
-
const
|
|
12178
|
-
|
|
13431
|
+
const requestedRelation = url.searchParams.get('relation')?.trim() ?? '';
|
|
13432
|
+
const legacyTable = url.searchParams.get('table')?.trim() ?? '';
|
|
13433
|
+
const legacySchema = url.searchParams.get('schema')?.trim() ?? '';
|
|
13434
|
+
const relationParts = splitQualifiedRelationIdentifier(requestedRelation || [legacySchema, legacyTable].filter(Boolean).join('.'));
|
|
13435
|
+
if (!relationParts?.length) {
|
|
12179
13436
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12180
|
-
res.end(serializeJSON({ error: '
|
|
13437
|
+
res.end(serializeJSON({ error: 'A safe dbt relation is required.' }));
|
|
12181
13438
|
return;
|
|
12182
13439
|
}
|
|
13440
|
+
const tablePath = relationParts.at(-1);
|
|
13441
|
+
const schemaName = relationParts.length > 1 ? relationParts.slice(0, -1).join('.') : undefined;
|
|
12183
13442
|
// Try connector.listColumns() first
|
|
12184
13443
|
let columns = [];
|
|
12185
13444
|
const activeConnection = connection;
|
|
@@ -12188,7 +13447,11 @@ export async function startLocalServer(opts) {
|
|
|
12188
13447
|
throw new Error('No active connection');
|
|
12189
13448
|
const connector = await executor.getConnector(activeConnection);
|
|
12190
13449
|
if (typeof connector.listColumns === 'function') {
|
|
12191
|
-
|
|
13450
|
+
// Connector contracts accept one schema plus one table. For a
|
|
13451
|
+
// three-part dbt relation, the active connection owns the database
|
|
13452
|
+
// and the final namespace segment is the schema.
|
|
13453
|
+
const connectorSchema = schemaName?.split('.').filter(Boolean).at(-1);
|
|
13454
|
+
const rawCols = await connector.listColumns(connectorSchema, tablePath);
|
|
12192
13455
|
columns = rawCols.map((c) => ({ name: c.name, type: c.dataType }));
|
|
12193
13456
|
}
|
|
12194
13457
|
}
|
|
@@ -12200,17 +13463,15 @@ export async function startLocalServer(opts) {
|
|
|
12200
13463
|
try {
|
|
12201
13464
|
const isFile = /\.(csv|parquet|json)$/i.test(tablePath) || tablePath.startsWith('data/');
|
|
12202
13465
|
const safePath = tablePath.replace(/'/g, "''");
|
|
12203
|
-
|
|
13466
|
+
if (!activeConnection)
|
|
13467
|
+
throw new Error('No active connection');
|
|
13468
|
+
const dialect = getDialect(activeConnection.driver);
|
|
13469
|
+
const qualifiedIdentifier = relationParts.map((part) => dialect.quoteIdentifier(part)).join('.');
|
|
12204
13470
|
const sql = isFile
|
|
12205
13471
|
? `DESCRIBE SELECT * FROM read_csv_auto('${safePath}') LIMIT 0`
|
|
12206
13472
|
: `DESCRIBE ${qualifiedIdentifier}`;
|
|
12207
|
-
if (!activeConnection)
|
|
12208
|
-
throw new Error('No active connection');
|
|
12209
13473
|
const result = await executor.executeQuery(sql, [], {}, activeConnection);
|
|
12210
|
-
columns = result.rows
|
|
12211
|
-
name: String(row['column_name'] ?? row['Field'] ?? ''),
|
|
12212
|
-
type: String(row['column_type'] ?? row['Type'] ?? ''),
|
|
12213
|
-
}));
|
|
13474
|
+
columns = schemaColumnsFromDescribeRows(result.rows);
|
|
12214
13475
|
}
|
|
12215
13476
|
catch {
|
|
12216
13477
|
// empty columns
|
|
@@ -12299,6 +13560,214 @@ export async function startLocalServer(opts) {
|
|
|
12299
13560
|
}
|
|
12300
13561
|
return;
|
|
12301
13562
|
}
|
|
13563
|
+
/**
|
|
13564
|
+
* Repair an editable notebook cell without starting Notebook AI research.
|
|
13565
|
+
* The returned source is applied only to the in-memory notebook cell by the
|
|
13566
|
+
* client; this endpoint never writes a block/domain/dbt source file.
|
|
13567
|
+
*/
|
|
13568
|
+
if (req.method === 'POST' && path === '/api/notebook/repair-execution') {
|
|
13569
|
+
let body = {};
|
|
13570
|
+
let execContext = null;
|
|
13571
|
+
const start = Date.now();
|
|
13572
|
+
try {
|
|
13573
|
+
body = await readJSON(req);
|
|
13574
|
+
execContext = notebookExecutionContext(body.executionContext);
|
|
13575
|
+
const cell = normalizeNotebookCell(body.cell);
|
|
13576
|
+
if (!cell || (cell.type !== 'sql' && cell.type !== 'dql')) {
|
|
13577
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13578
|
+
res.end(serializeJSON({ error: 'An editable SQL or DQL notebook cell is required.' }));
|
|
13579
|
+
return;
|
|
13580
|
+
}
|
|
13581
|
+
if (cell.type === 'dql' && /^\s*@block\(/i.test(cell.source)) {
|
|
13582
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13583
|
+
res.end(serializeJSON({ error: 'Customize the bound block as a notebook copy before repairing it.' }));
|
|
13584
|
+
return;
|
|
13585
|
+
}
|
|
13586
|
+
const embeddedSourceSql = notebookCellRepairSql(cell);
|
|
13587
|
+
if (!embeddedSourceSql && (cell.type !== 'dql' || !notebookDqlSourceAllowsBackgroundRepair(cell.source))) {
|
|
13588
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13589
|
+
res.end(serializeJSON({ error: 'Background repair supports SQL cells and editable custom DQL blocks. Open governed references or semantic definitions in their owning editor.' }));
|
|
13590
|
+
return;
|
|
13591
|
+
}
|
|
13592
|
+
if (cell.type === 'sql' && /\{\{[^}]+\}\}/.test(cell.source)) {
|
|
13593
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13594
|
+
res.end(serializeJSON({ error: 'Run or repair the referenced upstream cell before repairing this dependent query.' }));
|
|
13595
|
+
return;
|
|
13596
|
+
}
|
|
13597
|
+
const targetConnection = await resolveExecutionConnection(body);
|
|
13598
|
+
const submittedError = typeof body.error === 'string' ? body.error.trim() : '';
|
|
13599
|
+
const submittedFailure = body.failure && typeof body.failure === 'object' && !Array.isArray(body.failure)
|
|
13600
|
+
? body.failure
|
|
13601
|
+
: undefined;
|
|
13602
|
+
const failureCode = typeof submittedFailure?.code === 'string' ? submittedFailure.code : undefined;
|
|
13603
|
+
if (!notebookFailureAllowsBackgroundRepair({ code: failureCode, message: submittedError })) {
|
|
13604
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13605
|
+
res.end(serializeJSON({ error: 'This failure needs a data, access, parameter, or notebook dependency change and cannot be auto-fixed safely.' }));
|
|
13606
|
+
return;
|
|
13607
|
+
}
|
|
13608
|
+
const retainedFailure = submittedError
|
|
13609
|
+
? normalizeWarehouseSqlFailure({ code: failureCode, message: submittedError }, targetConnection.driver)
|
|
13610
|
+
: undefined;
|
|
13611
|
+
if (!analyticalFailureAllowsDeterministicRetry(retainedFailure)) {
|
|
13612
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13613
|
+
res.end(serializeJSON({
|
|
13614
|
+
error: 'This failure requires authorized access or a policy change and cannot be repaired in the background.',
|
|
13615
|
+
warehouseFailure: retainedFailure,
|
|
13616
|
+
}));
|
|
13617
|
+
return;
|
|
13618
|
+
}
|
|
13619
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
13620
|
+
? body.parameters
|
|
13621
|
+
: {};
|
|
13622
|
+
let invocation = null;
|
|
13623
|
+
let plan = null;
|
|
13624
|
+
let dqlCompilationError;
|
|
13625
|
+
if (cell.type === 'dql') {
|
|
13626
|
+
try {
|
|
13627
|
+
invocation = prepareBlockInvocation({ source: cell.source, parameters, surface: 'notebook' });
|
|
13628
|
+
if (invocation.errors.length)
|
|
13629
|
+
throw boundedRepairError(invocation.errors.join(' '), 409);
|
|
13630
|
+
if (invocation.unresolvedParameters.length) {
|
|
13631
|
+
throw boundedRepairError(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'} before retrying: ${invocation.unresolvedParameters.join(', ')}.`, 409);
|
|
13632
|
+
}
|
|
13633
|
+
plan = buildExecutionPlan(cell, {
|
|
13634
|
+
semanticLayer,
|
|
13635
|
+
driver: targetConnection.driver,
|
|
13636
|
+
parameters: invocation.values,
|
|
13637
|
+
});
|
|
13638
|
+
}
|
|
13639
|
+
catch (error) {
|
|
13640
|
+
const repairError = error;
|
|
13641
|
+
if (repairError.status === 409)
|
|
13642
|
+
throw error;
|
|
13643
|
+
dqlCompilationError = repairError.message || String(error);
|
|
13644
|
+
}
|
|
13645
|
+
}
|
|
13646
|
+
const attemptedSql = typeof body.attemptedSql === 'string' && body.attemptedSql.trim()
|
|
13647
|
+
? body.attemptedSql.trim()
|
|
13648
|
+
: embeddedSourceSql;
|
|
13649
|
+
const executionTarget = executionTargetDescriptor(body);
|
|
13650
|
+
const targetConnectionName = executionTarget.target === 'connection'
|
|
13651
|
+
? executionTarget.connectionName
|
|
13652
|
+
: executionTarget.target === 'local'
|
|
13653
|
+
? 'local'
|
|
13654
|
+
: projectConfig.defaultConnectionName;
|
|
13655
|
+
const question = [
|
|
13656
|
+
`Repair notebook cell ${cell.title?.trim() || cell.id}.`,
|
|
13657
|
+
submittedError ? `Failure: ${submittedError.slice(0, 1_000)}` : '',
|
|
13658
|
+
].filter(Boolean).join(' ');
|
|
13659
|
+
let repaired;
|
|
13660
|
+
let repairedSource;
|
|
13661
|
+
if (cell.type === 'dql' && dqlCompilationError) {
|
|
13662
|
+
const repairedDql = await executeBoundedDqlRepair({
|
|
13663
|
+
question,
|
|
13664
|
+
source: cell.source,
|
|
13665
|
+
failure: [submittedError, dqlCompilationError].filter(Boolean).join('\n'),
|
|
13666
|
+
targetConnection,
|
|
13667
|
+
targetConnectionName,
|
|
13668
|
+
parameters,
|
|
13669
|
+
artifactName: cell.title,
|
|
13670
|
+
});
|
|
13671
|
+
repaired = repairedDql;
|
|
13672
|
+
repairedSource = repairedDql.repairedSource;
|
|
13673
|
+
}
|
|
13674
|
+
else {
|
|
13675
|
+
const sourceSql = plan?.sql ?? attemptedSql;
|
|
13676
|
+
if (!sourceSql?.trim()) {
|
|
13677
|
+
throw boundedRepairError('DQL could not identify an embedded query to repair.', 422);
|
|
13678
|
+
}
|
|
13679
|
+
const sqlParams = plan?.sqlParams ?? [];
|
|
13680
|
+
repaired = await executeBoundedSqlRepair({
|
|
13681
|
+
question,
|
|
13682
|
+
sourceSql,
|
|
13683
|
+
targetConnection,
|
|
13684
|
+
targetConnectionName,
|
|
13685
|
+
sqlParams,
|
|
13686
|
+
variables: {
|
|
13687
|
+
...(plan?.variables ?? {}),
|
|
13688
|
+
...(invocation?.values ?? {}),
|
|
13689
|
+
},
|
|
13690
|
+
});
|
|
13691
|
+
if (cell.type === 'dql') {
|
|
13692
|
+
const originalQuery = notebookCellRepairSql(cell);
|
|
13693
|
+
const repairedQuery = repaired.repairMode === 'deterministic'
|
|
13694
|
+
? applyNotebookRepairRewrites(originalQuery, repaired.rewrites)
|
|
13695
|
+
: restoreNotebookDqlParameterInterpolations(repaired.repairedSql, sqlParams);
|
|
13696
|
+
repairedSource = replaceNotebookDqlQueryForRepair(cell.source, repairedQuery);
|
|
13697
|
+
}
|
|
13698
|
+
else {
|
|
13699
|
+
repairedSource = repaired.repairedSql;
|
|
13700
|
+
}
|
|
13701
|
+
}
|
|
13702
|
+
if (!repairedSource) {
|
|
13703
|
+
throw boundedRepairError('DQL could not safely replace the embedded query in this cell.', 422);
|
|
13704
|
+
}
|
|
13705
|
+
// Prove the repaired DQL wrapper still compiles. SQL execution above
|
|
13706
|
+
// already proved the corrected statement on the selected target.
|
|
13707
|
+
if (cell.type === 'dql') {
|
|
13708
|
+
const validationPlan = buildExecutionPlan({ ...cell, source: repairedSource }, {
|
|
13709
|
+
semanticLayer,
|
|
13710
|
+
driver: targetConnection.driver,
|
|
13711
|
+
parameters: invocation?.values ?? parameters,
|
|
13712
|
+
});
|
|
13713
|
+
if (!validationPlan?.sql) {
|
|
13714
|
+
throw boundedRepairError('The repaired SQL ran, but the DQL wrapper no longer compiles.', 422);
|
|
13715
|
+
}
|
|
13716
|
+
const beforeParameters = (plan?.sqlParams ?? ('sqlParams' in repaired ? repaired.sqlParams : []))
|
|
13717
|
+
.map((parameter) => `${parameter.position}:${parameter.name}`);
|
|
13718
|
+
const afterParameters = validationPlan.sqlParams.map((parameter) => `${parameter.position}:${parameter.name}`);
|
|
13719
|
+
if (beforeParameters.join('|') !== afterParameters.join('|')) {
|
|
13720
|
+
throw boundedRepairError('The repaired DQL changed the cell parameter contract, so the update was stopped.', 422);
|
|
13721
|
+
}
|
|
13722
|
+
}
|
|
13723
|
+
if (execContext?.notebookPath) {
|
|
13724
|
+
recordNotebookQueryRun(projectRoot, {
|
|
13725
|
+
notebookPath: execContext.notebookPath,
|
|
13726
|
+
cellId: execContext.cellId ?? cell.id,
|
|
13727
|
+
cellName: execContext.cellName ?? cell.title,
|
|
13728
|
+
source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_sql_cell'),
|
|
13729
|
+
status: 'success',
|
|
13730
|
+
rowCount: repaired.result.rowCount ?? repaired.result.rows.length,
|
|
13731
|
+
durationMs: Date.now() - start,
|
|
13732
|
+
sql: repaired.repairedSql,
|
|
13733
|
+
});
|
|
13734
|
+
}
|
|
13735
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13736
|
+
res.end(serializeJSON({
|
|
13737
|
+
ok: true,
|
|
13738
|
+
source: repairedSource,
|
|
13739
|
+
repairedSql: repaired.repairedSql,
|
|
13740
|
+
repairMode: repaired.repairMode,
|
|
13741
|
+
result: repaired.result,
|
|
13742
|
+
executionTarget,
|
|
13743
|
+
executionContext: notebookExecutionIdentity(execContext, cell.id),
|
|
13744
|
+
compiledSql: repaired.repairedSql,
|
|
13745
|
+
executedSql: repaired.result.sql ?? repaired.repairedSql,
|
|
13746
|
+
executionReceipt: repaired.result.executionReceipt,
|
|
13747
|
+
}));
|
|
13748
|
+
}
|
|
13749
|
+
catch (error) {
|
|
13750
|
+
const repairError = error;
|
|
13751
|
+
if (execContext?.notebookPath) {
|
|
13752
|
+
recordNotebookQueryRun(projectRoot, {
|
|
13753
|
+
notebookPath: execContext.notebookPath,
|
|
13754
|
+
cellId: execContext.cellId,
|
|
13755
|
+
cellName: execContext.cellName,
|
|
13756
|
+
source: execContext.source ?? 'notebook_cell_repair',
|
|
13757
|
+
status: 'error',
|
|
13758
|
+
durationMs: Date.now() - start,
|
|
13759
|
+
errorCode: repairError.warehouseFailure?.category ?? repairError.message,
|
|
13760
|
+
sql: typeof body.attemptedSql === 'string' ? body.attemptedSql : undefined,
|
|
13761
|
+
});
|
|
13762
|
+
}
|
|
13763
|
+
res.writeHead(repairError.status ?? 500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
13764
|
+
res.end(serializeJSON({
|
|
13765
|
+
error: repairError.message,
|
|
13766
|
+
...(repairError.warehouseFailure ? { warehouseFailure: repairError.warehouseFailure } : {}),
|
|
13767
|
+
}));
|
|
13768
|
+
}
|
|
13769
|
+
return;
|
|
13770
|
+
}
|
|
12302
13771
|
if (req.method === 'POST' && path === '/api/query') {
|
|
12303
13772
|
let body;
|
|
12304
13773
|
let execContext = null;
|
|
@@ -13511,6 +14980,22 @@ table: ${table}${tagList}
|
|
|
13511
14980
|
})));
|
|
13512
14981
|
});
|
|
13513
14982
|
});
|
|
14983
|
+
server.once('close', () => {
|
|
14984
|
+
runtimeClosing = true;
|
|
14985
|
+
for (const watcher of projectWatchers)
|
|
14986
|
+
watcher.close();
|
|
14987
|
+
projectWatchers.length = 0;
|
|
14988
|
+
unsubscribeOperationEvents();
|
|
14989
|
+
projectRefreshCoordinator.close();
|
|
14990
|
+
for (const client of operationSseClients) {
|
|
14991
|
+
try {
|
|
14992
|
+
client.end();
|
|
14993
|
+
}
|
|
14994
|
+
catch { /* connection already closed */ }
|
|
14995
|
+
}
|
|
14996
|
+
operationSseClients.clear();
|
|
14997
|
+
operationCoordinator.close();
|
|
14998
|
+
});
|
|
13514
14999
|
opts.captureServer?.(server);
|
|
13515
15000
|
return new Promise((resolvePromise, reject) => {
|
|
13516
15001
|
let retriedWithRandomPort = false;
|
|
@@ -13952,7 +15437,7 @@ function loadAppDashboard(projectRoot, appId, dashboardId) {
|
|
|
13952
15437
|
for (const d of findDashboardsForApp(appDir)) {
|
|
13953
15438
|
const { document: dashboard } = loadDashboardDocument(d);
|
|
13954
15439
|
if (dashboard?.id === dashboardId)
|
|
13955
|
-
return { app, dashboard };
|
|
15440
|
+
return { app, dashboard, appDir };
|
|
13956
15441
|
}
|
|
13957
15442
|
}
|
|
13958
15443
|
return null;
|
|
@@ -16846,6 +18331,60 @@ function normalizeNotebookCell(value) {
|
|
|
16846
18331
|
config: candidate.config,
|
|
16847
18332
|
};
|
|
16848
18333
|
}
|
|
18334
|
+
export function notebookCellRepairSql(cell) {
|
|
18335
|
+
if (cell.type === 'sql')
|
|
18336
|
+
return cell.source.trim() || undefined;
|
|
18337
|
+
if (cell.type !== 'dql')
|
|
18338
|
+
return undefined;
|
|
18339
|
+
const match = /\bquery\s*=\s*"""([\s\S]*?)"""/i.exec(cell.source);
|
|
18340
|
+
return match?.[1]?.trim() || undefined;
|
|
18341
|
+
}
|
|
18342
|
+
export function notebookDqlSourceAllowsBackgroundRepair(source) {
|
|
18343
|
+
if (/^\s*@block\(/i.test(source) || /\btype\s*=\s*"semantic"/i.test(source))
|
|
18344
|
+
return false;
|
|
18345
|
+
return /\bblock\b/i.test(source) && /\bquery\b/i.test(source);
|
|
18346
|
+
}
|
|
18347
|
+
export function replaceNotebookDqlQueryForRepair(source, sql) {
|
|
18348
|
+
const queryField = /\b(query\s*=\s*""")([\s\S]*?)(""")/i;
|
|
18349
|
+
if (!queryField.test(source) || !sql.trim())
|
|
18350
|
+
return undefined;
|
|
18351
|
+
return source.replace(queryField, (_match, open, _body, close) => (`${open}\n${sql.trim()}\n ${close}`));
|
|
18352
|
+
}
|
|
18353
|
+
const NOTEBOOK_BACKGROUND_REPAIR_BLOCKED_CODES = new Set([
|
|
18354
|
+
'AMBIGUOUS_NOTEBOOK_DEPENDENCY',
|
|
18355
|
+
'CROSS_ENGINE_JOIN_REQUIRED',
|
|
18356
|
+
'EXECUTION_TARGET_MISMATCH',
|
|
18357
|
+
'SEMANTIC_SOURCE_DRIFT',
|
|
18358
|
+
'STALE_CELL_EXECUTION',
|
|
18359
|
+
'UPSTREAM_RESULT_UNAVAILABLE',
|
|
18360
|
+
'UNAUTHORIZED',
|
|
18361
|
+
]);
|
|
18362
|
+
/**
|
|
18363
|
+
* Notebook background repair is deliberately narrower than conversational
|
|
18364
|
+
* repair. Access, policy, target, parameter, and dependency failures require a
|
|
18365
|
+
* user decision; only query-shape failures may enter the one-attempt loop.
|
|
18366
|
+
* Acceptance: API-006, API-007, UI-012, UI-013, SEC-004, E2E-014.
|
|
18367
|
+
*/
|
|
18368
|
+
export function notebookFailureAllowsBackgroundRepair(input) {
|
|
18369
|
+
const code = input.code?.trim().toUpperCase();
|
|
18370
|
+
const message = input.message?.trim() ?? '';
|
|
18371
|
+
if (!message)
|
|
18372
|
+
return false;
|
|
18373
|
+
if (code && NOTEBOOK_BACKGROUND_REPAIR_BLOCKED_CODES.has(code))
|
|
18374
|
+
return false;
|
|
18375
|
+
return !/(?:permission|not authorized|unauthori[sz]ed|access denied|authentication|credential|policy|unsafe|read[- ]only|cancel(?:led|ed)|required parameters?|provide required|cross[- ]engine|upstream (?:cell|result)|notebook dependency|target mismatch|source drift|matched more than one schema)/i.test(message);
|
|
18376
|
+
}
|
|
18377
|
+
export function applyNotebookRepairRewrites(sql, rewrites) {
|
|
18378
|
+
return rewrites.reduce((current, rewrite) => {
|
|
18379
|
+
const pattern = rewrite.from.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
18380
|
+
return current.replace(new RegExp(pattern, 'g'), rewrite.to);
|
|
18381
|
+
}, sql);
|
|
18382
|
+
}
|
|
18383
|
+
export function restoreNotebookDqlParameterInterpolations(sql, parameters) {
|
|
18384
|
+
return [...parameters]
|
|
18385
|
+
.sort((left, right) => right.position - left.position)
|
|
18386
|
+
.reduce((current, parameter) => current.replace(new RegExp(`\\$${parameter.position}\\b`, 'g'), `\${${parameter.name}}`), sql);
|
|
18387
|
+
}
|
|
16849
18388
|
function resolveNotebookBlockReferenceCell(cell, projectRoot) {
|
|
16850
18389
|
if (cell.type !== 'dql')
|
|
16851
18390
|
return { cell };
|
|
@@ -17039,6 +18578,68 @@ function readUserPrefs(userPrefsPath) {
|
|
|
17039
18578
|
function writeUserPrefs(userPrefsPath, prefs) {
|
|
17040
18579
|
writeFileSync(userPrefsPath, JSON.stringify(prefs, null, 2) + '\n', 'utf-8');
|
|
17041
18580
|
}
|
|
18581
|
+
/** Normalize DESCRIBE output across Snowflake, DuckDB, and PostgreSQL shapes. */
|
|
18582
|
+
export function schemaColumnsFromDescribeRows(rows) {
|
|
18583
|
+
return rows.map((row) => ({
|
|
18584
|
+
name: String(row['column_name'] ?? row['COLUMN_NAME'] ?? row['Field'] ?? row['FIELD'] ?? row['name'] ?? row['NAME'] ?? ''),
|
|
18585
|
+
type: String(row['column_type'] ?? row['COLUMN_TYPE'] ?? row['data_type'] ?? row['DATA_TYPE'] ?? row['Type'] ?? row['TYPE'] ?? row['type'] ?? ''),
|
|
18586
|
+
})).filter((column) => Boolean(column.name));
|
|
18587
|
+
}
|
|
18588
|
+
/** Parse dbt relation strings without letting relation text become SQL syntax. */
|
|
18589
|
+
export function splitQualifiedRelationIdentifier(value) {
|
|
18590
|
+
const rawParts = [];
|
|
18591
|
+
let current = '';
|
|
18592
|
+
let quote = null;
|
|
18593
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
18594
|
+
const char = value[index];
|
|
18595
|
+
if (quote) {
|
|
18596
|
+
current += char;
|
|
18597
|
+
if (char === quote) {
|
|
18598
|
+
const next = value[index + 1];
|
|
18599
|
+
if ((quote === '"' || quote === '`') && next === quote) {
|
|
18600
|
+
current += next;
|
|
18601
|
+
index += 1;
|
|
18602
|
+
}
|
|
18603
|
+
else {
|
|
18604
|
+
quote = null;
|
|
18605
|
+
}
|
|
18606
|
+
}
|
|
18607
|
+
continue;
|
|
18608
|
+
}
|
|
18609
|
+
if (char === '"' || char === '`') {
|
|
18610
|
+
quote = char;
|
|
18611
|
+
current += char;
|
|
18612
|
+
continue;
|
|
18613
|
+
}
|
|
18614
|
+
if (char === '[') {
|
|
18615
|
+
quote = ']';
|
|
18616
|
+
current += char;
|
|
18617
|
+
continue;
|
|
18618
|
+
}
|
|
18619
|
+
if (char === '.') {
|
|
18620
|
+
rawParts.push(current);
|
|
18621
|
+
current = '';
|
|
18622
|
+
continue;
|
|
18623
|
+
}
|
|
18624
|
+
current += char;
|
|
18625
|
+
}
|
|
18626
|
+
if (quote)
|
|
18627
|
+
return null;
|
|
18628
|
+
rawParts.push(current);
|
|
18629
|
+
if (rawParts.length === 0 || rawParts.length > 4)
|
|
18630
|
+
return null;
|
|
18631
|
+
const decoded = rawParts.map((raw) => {
|
|
18632
|
+
const token = raw.trim();
|
|
18633
|
+
if (/^"(?:[^"]|"")+"$/.test(token))
|
|
18634
|
+
return token.slice(1, -1).replace(/""/g, '"');
|
|
18635
|
+
if (/^`(?:[^`]|``)+`$/.test(token))
|
|
18636
|
+
return token.slice(1, -1).replace(/``/g, '`');
|
|
18637
|
+
if (/^\[[^\]]+\]$/.test(token))
|
|
18638
|
+
return token.slice(1, -1).replace(/\]\]/g, ']');
|
|
18639
|
+
return /^[A-Za-z_][A-Za-z0-9_$]*$/.test(token) ? token : null;
|
|
18640
|
+
});
|
|
18641
|
+
return decoded.every((part) => Boolean(part)) ? decoded : null;
|
|
18642
|
+
}
|
|
17042
18643
|
async function introspectSchema(executor, connection, options = {}) {
|
|
17043
18644
|
let tables = [];
|
|
17044
18645
|
const columnsByPath = new Map();
|
|
@@ -17093,77 +18694,93 @@ async function introspectSchema(executor, connection, options = {}) {
|
|
|
17093
18694
|
}
|
|
17094
18695
|
return { tables, columnsByPath };
|
|
17095
18696
|
}
|
|
17096
|
-
|
|
17097
|
-
|
|
17098
|
-
|
|
17099
|
-
|
|
17100
|
-
|
|
17101
|
-
|
|
17102
|
-
|
|
17103
|
-
|
|
17104
|
-
|
|
17105
|
-
|
|
17106
|
-
|
|
17107
|
-
|
|
17108
|
-
|
|
17109
|
-
const
|
|
17110
|
-
|
|
17111
|
-
|
|
17112
|
-
|
|
17113
|
-
|
|
17114
|
-
|
|
17115
|
-
|
|
17116
|
-
|
|
17117
|
-
|
|
17118
|
-
|
|
17119
|
-
|
|
17120
|
-
|
|
17121
|
-
|
|
17122
|
-
|
|
17123
|
-
|
|
17124
|
-
|
|
17125
|
-
|
|
17126
|
-
|
|
17127
|
-
// Eagerly resolve file columns via DuckDB DESCRIBE
|
|
17128
|
-
if (dataFiles.length > 0) {
|
|
17129
|
-
const fileChildren = [];
|
|
17130
|
-
for (const file of dataFiles) {
|
|
17131
|
-
let columns = [];
|
|
17132
|
-
if (connection) {
|
|
17133
|
-
try {
|
|
17134
|
-
const ext = file.name.split('.').pop()?.toLowerCase();
|
|
17135
|
-
const readFn = ext === 'parquet' ? 'read_parquet' : ext === 'json' ? 'read_json_auto' : 'read_csv_auto';
|
|
17136
|
-
const descResult = await executor.executeQuery(`DESCRIBE SELECT * FROM ${readFn}('${file.path.replace(/'/g, "''")}') LIMIT 0`, [], {}, connection);
|
|
17137
|
-
columns = descResult.rows.map((row) => ({
|
|
17138
|
-
id: `db-column:${file.path}:${String(row['column_name'] ?? '')}`,
|
|
17139
|
-
label: String(row['column_name'] ?? ''),
|
|
17140
|
-
kind: 'column',
|
|
17141
|
-
path: file.path,
|
|
17142
|
-
type: String(row['column_type'] ?? ''),
|
|
17143
|
-
}));
|
|
17144
|
-
}
|
|
17145
|
-
catch {
|
|
17146
|
-
// file column discovery failed — empty children is fine
|
|
17147
|
-
}
|
|
17148
|
-
}
|
|
17149
|
-
fileChildren.push({
|
|
17150
|
-
id: `db-table:${file.path}`,
|
|
17151
|
-
label: file.name,
|
|
17152
|
-
kind: 'table',
|
|
17153
|
-
path: file.path,
|
|
17154
|
-
type: 'FILE',
|
|
17155
|
-
children: columns,
|
|
17156
|
-
});
|
|
18697
|
+
const dbtPhysicalRelationCache = new Map();
|
|
18698
|
+
/**
|
|
18699
|
+
* Display-only physical relation filter sourced from the raw dbt artifact.
|
|
18700
|
+
* This deliberately does not mutate DQLManifest or the agent metadata/indexes.
|
|
18701
|
+
*/
|
|
18702
|
+
function physicalDbtRelationIds(manifestPath) {
|
|
18703
|
+
try {
|
|
18704
|
+
const stat = statSync(manifestPath);
|
|
18705
|
+
const version = `${stat.size}:${stat.mtimeMs}`;
|
|
18706
|
+
const cached = dbtPhysicalRelationCache.get(manifestPath);
|
|
18707
|
+
if (cached?.version === version)
|
|
18708
|
+
return cached.ids;
|
|
18709
|
+
const raw = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
18710
|
+
const ids = new Set();
|
|
18711
|
+
for (const collectionName of ['nodes', 'sources']) {
|
|
18712
|
+
const collection = raw[collectionName];
|
|
18713
|
+
if (!collection || typeof collection !== 'object' || Array.isArray(collection))
|
|
18714
|
+
continue;
|
|
18715
|
+
for (const [fallbackId, value] of Object.entries(collection)) {
|
|
18716
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
18717
|
+
continue;
|
|
18718
|
+
const node = value;
|
|
18719
|
+
const resourceType = String(node.resource_type ?? (collectionName === 'sources' ? 'source' : ''));
|
|
18720
|
+
if (resourceType !== 'model' && resourceType !== 'source')
|
|
18721
|
+
continue;
|
|
18722
|
+
const config = node.config && typeof node.config === 'object' && !Array.isArray(node.config)
|
|
18723
|
+
? node.config
|
|
18724
|
+
: {};
|
|
18725
|
+
if (resourceType === 'model' && String(config.materialized ?? '').toLowerCase() === 'ephemeral')
|
|
18726
|
+
continue;
|
|
18727
|
+
ids.add(typeof node.unique_id === 'string' ? node.unique_id : fallbackId);
|
|
17157
18728
|
}
|
|
17158
|
-
databaseNodes.unshift({
|
|
17159
|
-
id: 'db-schema:files',
|
|
17160
|
-
label: 'files',
|
|
17161
|
-
kind: 'schema',
|
|
17162
|
-
children: fileChildren,
|
|
17163
|
-
});
|
|
17164
18729
|
}
|
|
17165
|
-
|
|
17166
|
-
|
|
18730
|
+
dbtPhysicalRelationCache.set(manifestPath, { version, ids });
|
|
18731
|
+
return ids;
|
|
18732
|
+
}
|
|
18733
|
+
catch {
|
|
18734
|
+
return undefined;
|
|
18735
|
+
}
|
|
18736
|
+
}
|
|
18737
|
+
export function buildDbtDatabaseSchemaTree(manifest, limit = 25, physicalIds) {
|
|
18738
|
+
// PERF-001 / CTX-005: Block Studio must not enumerate the connected account.
|
|
18739
|
+
// Its initial editor context is the first bounded page of physical relations
|
|
18740
|
+
// declared by the connected dbt artifact; columns remain lazy point-lookups.
|
|
18741
|
+
const boundedLimit = Math.min(100, Math.max(1, Math.floor(limit)));
|
|
18742
|
+
const seen = new Set();
|
|
18743
|
+
const tables = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
18744
|
+
.filter((node) => Boolean(node.relation) && (!physicalIds || physicalIds.has(node.uniqueId)))
|
|
18745
|
+
.sort((a, b) => a.uniqueId.localeCompare(b.uniqueId))
|
|
18746
|
+
.flatMap((node) => {
|
|
18747
|
+
const path = node.relation;
|
|
18748
|
+
const key = path.toLowerCase();
|
|
18749
|
+
if (seen.has(key))
|
|
18750
|
+
return [];
|
|
18751
|
+
seen.add(key);
|
|
18752
|
+
const parts = path.split('.').filter(Boolean);
|
|
18753
|
+
return [{
|
|
18754
|
+
schema: parts.length > 1 ? parts.slice(0, -1).join('.') : 'default',
|
|
18755
|
+
name: parts.at(-1) ?? node.name,
|
|
18756
|
+
path,
|
|
18757
|
+
type: node.resourceType === 'source' ? 'DBT_SOURCE' : 'DBT_MODEL',
|
|
18758
|
+
}];
|
|
18759
|
+
})
|
|
18760
|
+
.slice(0, boundedLimit);
|
|
18761
|
+
const bySchema = new Map();
|
|
18762
|
+
for (const table of tables) {
|
|
18763
|
+
const current = bySchema.get(table.schema) ?? [];
|
|
18764
|
+
current.push(table);
|
|
18765
|
+
bySchema.set(table.schema, current);
|
|
18766
|
+
}
|
|
18767
|
+
return Array.from(bySchema.entries())
|
|
18768
|
+
.sort((a, b) => a[0].localeCompare(b[0]))
|
|
18769
|
+
.map(([schema, schemaTables]) => ({
|
|
18770
|
+
id: `db-schema:${schema}`,
|
|
18771
|
+
label: schema,
|
|
18772
|
+
kind: 'schema',
|
|
18773
|
+
children: schemaTables
|
|
18774
|
+
.sort((a, b) => a.name.localeCompare(b.name))
|
|
18775
|
+
.map((table) => ({
|
|
18776
|
+
id: `db-table:${table.path}`,
|
|
18777
|
+
label: table.name,
|
|
18778
|
+
kind: 'table',
|
|
18779
|
+
path: table.path,
|
|
18780
|
+
type: table.type,
|
|
18781
|
+
children: [],
|
|
18782
|
+
})),
|
|
18783
|
+
}));
|
|
17167
18784
|
}
|
|
17168
18785
|
export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
17169
18786
|
const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
|
|
@@ -17188,6 +18805,7 @@ export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer
|
|
|
17188
18805
|
owner: parsedMetadata.owner || companion?.owner || '',
|
|
17189
18806
|
tags: parsedMetadata.tags.length > 0 ? parsedMetadata.tags : companion?.tags ?? [],
|
|
17190
18807
|
reviewStatus: parsedMetadata.status || companion?.reviewStatus || 'draft',
|
|
18808
|
+
sourceFingerprint: createHash('sha256').update(source).digest('hex'),
|
|
17191
18809
|
};
|
|
17192
18810
|
return {
|
|
17193
18811
|
path: normalizedPath,
|
|
@@ -18752,6 +20370,45 @@ function formatBlockStudioExpected(expected) {
|
|
|
18752
20370
|
return String(normalized);
|
|
18753
20371
|
return JSON.stringify(normalized);
|
|
18754
20372
|
}
|
|
20373
|
+
function evaluateBlockStudioTests(tests, rows, columns, startedAt) {
|
|
20374
|
+
if (tests.length === 0) {
|
|
20375
|
+
return { passed: 0, failed: 0, skipped: 0, duration: Date.now() - startedAt, assertions: [], runAt: new Date() };
|
|
20376
|
+
}
|
|
20377
|
+
const assertions = [];
|
|
20378
|
+
let passed = 0;
|
|
20379
|
+
let failed = 0;
|
|
20380
|
+
for (const test of tests) {
|
|
20381
|
+
const name = `assert ${test.field} ${test.operator} ${formatBlockStudioExpected(test.expected)}`;
|
|
20382
|
+
let actual;
|
|
20383
|
+
if (test.field === 'row_count') {
|
|
20384
|
+
actual = rows.length;
|
|
20385
|
+
}
|
|
20386
|
+
else if (!columns.includes(test.field)) {
|
|
20387
|
+
assertions.push({ name, passed: false, expected: test.expected, error: `Column '${test.field}' not found in results` });
|
|
20388
|
+
failed += 1;
|
|
20389
|
+
continue;
|
|
20390
|
+
}
|
|
20391
|
+
else {
|
|
20392
|
+
actual = rows[0]?.[test.field];
|
|
20393
|
+
}
|
|
20394
|
+
const ok = compareBlockStudioValues(actual, test.operator, test.expected);
|
|
20395
|
+
if (ok) {
|
|
20396
|
+
assertions.push({ name, passed: true, actual, expected: test.expected });
|
|
20397
|
+
passed += 1;
|
|
20398
|
+
}
|
|
20399
|
+
else {
|
|
20400
|
+
assertions.push({
|
|
20401
|
+
name,
|
|
20402
|
+
passed: false,
|
|
20403
|
+
actual,
|
|
20404
|
+
expected: test.expected,
|
|
20405
|
+
error: `${String(actual)} ${test.operator} ${formatBlockStudioExpected(test.expected)} is false`,
|
|
20406
|
+
});
|
|
20407
|
+
failed += 1;
|
|
20408
|
+
}
|
|
20409
|
+
}
|
|
20410
|
+
return { passed, failed, skipped: 0, duration: Date.now() - startedAt, assertions, runAt: new Date() };
|
|
20411
|
+
}
|
|
18755
20412
|
function buildBlockStudioCertificationChecklist(input) {
|
|
18756
20413
|
const parsed = parseBlockSourceMetadata(input.source);
|
|
18757
20414
|
const sql = extractBlockStudioSql(input.source) ?? '';
|
|
@@ -23842,6 +25499,46 @@ function mergeAgentSchemaContexts(...contexts) {
|
|
|
23842
25499
|
}
|
|
23843
25500
|
return Array.from(byRelation.values()).slice(0, 16);
|
|
23844
25501
|
}
|
|
25502
|
+
function agentRelationMatchKey(relation) {
|
|
25503
|
+
return relation
|
|
25504
|
+
.replace(/["`\[\]]/g, '')
|
|
25505
|
+
.split('.')
|
|
25506
|
+
.filter(Boolean)
|
|
25507
|
+
.slice(-2)
|
|
25508
|
+
.join('.')
|
|
25509
|
+
.toLowerCase();
|
|
25510
|
+
}
|
|
25511
|
+
/**
|
|
25512
|
+
* Keep dbt descriptions and semantic meaning, but make a complete live point
|
|
25513
|
+
* lookup authoritative for executable column names on the selected connection.
|
|
25514
|
+
* No fuzzy aliases are invented: a stale `customer_name` declaration disappears
|
|
25515
|
+
* when the live relation exposes only `name`.
|
|
25516
|
+
*/
|
|
25517
|
+
export function reconcileAgentSchemaContextWithLive(catalog, live) {
|
|
25518
|
+
const liveByRelation = new Map(live
|
|
25519
|
+
.filter((table) => table.columns.length > 0)
|
|
25520
|
+
.map((table) => [agentRelationMatchKey(table.relation), table]));
|
|
25521
|
+
return catalog.map((table) => {
|
|
25522
|
+
const verified = liveByRelation.get(agentRelationMatchKey(table.relation));
|
|
25523
|
+
if (!verified)
|
|
25524
|
+
return table;
|
|
25525
|
+
const metadataColumns = new Map(table.columns.map((column) => [column.name.toLowerCase(), column]));
|
|
25526
|
+
return {
|
|
25527
|
+
...table,
|
|
25528
|
+
source: 'dbt metadata verified against live runtime schema',
|
|
25529
|
+
columnCompleteness: 'complete',
|
|
25530
|
+
columns: verified.columns.map((column) => {
|
|
25531
|
+
const metadata = metadataColumns.get(column.name.toLowerCase());
|
|
25532
|
+
return {
|
|
25533
|
+
...column,
|
|
25534
|
+
type: column.type ?? metadata?.type,
|
|
25535
|
+
description: metadata?.description,
|
|
25536
|
+
sampleValues: metadata?.sampleValues,
|
|
25537
|
+
};
|
|
25538
|
+
}),
|
|
25539
|
+
};
|
|
25540
|
+
});
|
|
25541
|
+
}
|
|
23845
25542
|
function mergeAgentSchemaSampleValues(snapshot, enriched) {
|
|
23846
25543
|
const samples = new Map();
|
|
23847
25544
|
for (const table of enriched) {
|