@duckcodeailabs/dql-cli 1.12.9 → 1.12.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/dql-notebook/assets/{AgentAnswerCard-DbI0CAsX.js → AgentAnswerCard-CpL2kDCV.js} +1 -1
- package/dist/assets/dql-notebook/assets/{AgentLogPage-BbMhuJps.js → AgentLogPage-hzHqpD3-.js} +1 -1
- package/dist/assets/dql-notebook/assets/{AiBuildDialog-DNgkam6H.js → AiBuildDialog-BQRWoxp9.js} +1 -1
- package/dist/assets/dql-notebook/assets/AiBuildResult-D9t_4sQl.js +33 -0
- package/dist/assets/dql-notebook/assets/{AiSidePanel-D-EhCOEd.js → AiSidePanel-BCQB9rVF.js} +1 -1
- package/dist/assets/dql-notebook/assets/{AnalyticsHome-BTjkolBn.js → AnalyticsHome-Ba4Ve_Gd.js} +1 -1
- package/dist/assets/dql-notebook/assets/AppsView-CdAT9BOq.js +4466 -0
- package/dist/assets/dql-notebook/assets/{BlockStudio-DTFHMR3y.js → BlockStudio-Brg4fwn5.js} +13 -13
- package/dist/assets/dql-notebook/assets/{BusinessArtifactView-DGiHB7OS.js → BusinessArtifactView-CnTbFVGT.js} +1 -1
- package/dist/assets/dql-notebook/assets/DbtFirstModelingPage-CQe2FUZq.js +22 -0
- package/dist/assets/dql-notebook/assets/{GitPage-BWzPKyws.js → GitPage-DHyDBChc.js} +1 -1
- package/dist/assets/dql-notebook/assets/{GlobalAiRail-CGvBUci2.js → GlobalAiRail-Cys4n_rq.js} +1 -1
- package/dist/assets/dql-notebook/assets/GovernedContextPage-DyKoDSZ2.js +7 -0
- package/dist/assets/dql-notebook/assets/{HelpDocsPage-C7IX_7xh.js → HelpDocsPage-CL0vbSER.js} +3 -3
- package/dist/assets/dql-notebook/assets/{HomePage-e0CObjOb.js → HomePage-DzAucwBW.js} +9 -4
- package/dist/assets/dql-notebook/assets/{LineageDAG-DV6T_0ha.js → LineageDAG-QHqd_AM5.js} +1 -1
- package/dist/assets/dql-notebook/assets/{LineageDetailView-ZVJiCqua.js → LineageDetailView-DdVH-nz6.js} +1 -1
- package/dist/assets/dql-notebook/assets/{LineageDrawer-DQdzGIoL.js → LineageDrawer-BvrVZNKY.js} +1 -1
- package/dist/assets/dql-notebook/assets/LineagePathBreadcrumb-CSut-zvm.js +14 -0
- package/dist/assets/dql-notebook/assets/{MiniLineageGraph-BlJYiAY0.js → MiniLineageGraph-zg_1wW3M.js} +1 -1
- package/dist/assets/dql-notebook/assets/NewBlockModal-9cDletl-.js +1 -0
- package/dist/assets/dql-notebook/assets/NewNotebookModal-V6de8L4d.js +46 -0
- package/dist/assets/dql-notebook/assets/NotebookEditor-EZ0pfWVs.js +200 -0
- package/dist/assets/dql-notebook/assets/{ReadinessPage-m8jlEKzQ.js → ReadinessPage-BaGvjyfd.js} +1 -1
- package/dist/assets/dql-notebook/assets/{SetupOnboarding-ClldPV4C.js → SetupOnboarding-Bg-MjbC9.js} +2 -2
- package/dist/assets/dql-notebook/assets/{TrustBadge-C0FyL1ZO.js → TrustBadge-DGuHB8kP.js} +1 -1
- package/dist/assets/dql-notebook/assets/UnifiedAgentRunPanel-BlWAVEwO.js +83 -0
- package/dist/assets/dql-notebook/assets/{answer-to-notebook-CgVDBH0o.js → answer-to-notebook-BDzY4MuQ.js} +1 -1
- package/dist/assets/dql-notebook/assets/{arrow-left-XTRkwgdW.js → arrow-left-IjlnAihE.js} +1 -1
- package/dist/assets/dql-notebook/assets/{arrow-right-BUYb7B8y.js → arrow-right-BI4cXgOY.js} +2 -2
- package/dist/assets/dql-notebook/assets/{book-open-text-CQHKAXcN.js → book-open-text-i6xhxu_I.js} +1 -1
- package/dist/assets/dql-notebook/assets/{bot-D9ODKCwi.js → bot-C6DE7Q6m.js} +1 -1
- package/dist/assets/dql-notebook/assets/{chevron-up-BR57nCns.js → chevron-up-DSEILnDs.js} +2 -2
- package/dist/assets/dql-notebook/assets/{circle-x-B3MKf4hs.js → circle-x-BoAorVSm.js} +1 -1
- package/dist/assets/dql-notebook/assets/{codemirror-1_I9gm6Z.js → codemirror-DXt7RW72.js} +1 -1
- package/dist/assets/dql-notebook/assets/{dagre.esm-BWViOEog.js → dagre.esm-CycymVj6.js} +1 -1
- package/dist/assets/dql-notebook/assets/{external-link-D2rxGhQ_.js → external-link-ZwE5iHsX.js} +1 -1
- package/dist/assets/dql-notebook/assets/{grip-vertical-Bzs7bIWP.js → grip-vertical-Dnp2st8l.js} +1 -1
- package/dist/assets/dql-notebook/assets/index-BTuliKhK.js +634 -0
- package/dist/assets/dql-notebook/assets/{link-2-D6nW0_qQ.js → link-2-CTKe6Snr.js} +2 -2
- package/dist/assets/dql-notebook/assets/{list-tree-MH5kYamM.js → list-tree-Ced8e_S1.js} +1 -1
- package/dist/assets/dql-notebook/assets/{maximize-2-C3bEmsdA.js → maximize-2-Cojc6867.js} +1 -1
- package/dist/assets/dql-notebook/assets/{minimize-2-cji4jKmJ.js → minimize-2-CBIAE0XH.js} +1 -1
- package/dist/assets/dql-notebook/assets/{panel-right-open-T-_o8FS9.js → panel-right-open-po3bWSeL.js} +2 -2
- package/dist/assets/dql-notebook/assets/{pencil-BzJ_vZuL.js → pencil-Cb45c6WU.js} +2 -2
- package/dist/assets/dql-notebook/assets/{rotate-ccw-CqSfrg2J.js → rotate-ccw-B9v1LTJ0.js} +2 -2
- package/dist/assets/dql-notebook/assets/{semantic-fields-BXNCrm72.js → semantic-fields-B6ii0an0.js} +1 -1
- package/dist/assets/dql-notebook/assets/{star-C22MG8pV.js → star-CO4tfaDq.js} +2 -2
- package/dist/assets/dql-notebook/assets/{triangle-alert-BtoT0VL-.js → triangle-alert-BSdIu2El.js} +1 -1
- package/dist/assets/dql-notebook/assets/{usePersistedAgentThreadId-BL9P0gFB.js → usePersistedAgentThreadId-CkVFrrjk.js} +2 -2
- package/dist/assets/dql-notebook/assets/{user-round-CpifcAyh.js → user-round-BfrV8U-W.js} +3 -3
- package/dist/assets/dql-notebook/assets/{wand-sparkles-BiOywrQh.js → wand-sparkles-BgTxZNBc.js} +1 -1
- package/dist/assets/dql-notebook/assets/{workflow-DDvVMZ2B.js → workflow--2ycp0yy.js} +2 -2
- package/dist/assets/dql-notebook/assets/{wrench-CoKg2kyO.js → wrench-LMCrpa9_.js} +1 -1
- package/dist/assets/dql-notebook/assets/{x-CCmr0LQ8.js → x-CeNgQwGm.js} +2 -2
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/local-runtime.d.ts +90 -2
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +1005 -152
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/package.json +10 -10
- package/dist/assets/dql-notebook/assets/AiBuildResult-BuTao779.js +0 -33
- package/dist/assets/dql-notebook/assets/AppsView-B3jlWoNm.js +0 -4466
- package/dist/assets/dql-notebook/assets/DbtFirstModelingPage-D-fv_dS2.js +0 -22
- package/dist/assets/dql-notebook/assets/GovernedContextPage-CxkrtqyP.js +0 -7
- package/dist/assets/dql-notebook/assets/LineagePathBreadcrumb-6gFsnWlN.js +0 -14
- package/dist/assets/dql-notebook/assets/NewBlockModal-DaxBAFLu.js +0 -1
- package/dist/assets/dql-notebook/assets/NewNotebookModal-B7GPLsYc.js +0 -46
- package/dist/assets/dql-notebook/assets/NotebookEditor-RuxFQnQC.js +0 -202
- package/dist/assets/dql-notebook/assets/UnifiedAgentRunPanel-Bd1qmRFx.js +0 -83
- package/dist/assets/dql-notebook/assets/index-CDpukm0G.js +0 -644
package/dist/local-runtime.js
CHANGED
|
@@ -21,7 +21,7 @@ import { fetchLatestPublishedDqlVersion, resolveDqlRuntimeVersionStatus } from '
|
|
|
21
21
|
import { resolveRetrievalHealthStatus } from './retrieval-health.js';
|
|
22
22
|
import { createDqlAgentProviderRunner, resolveAgentFollowUpContext } from './llm/providers/dql-agent-provider.js';
|
|
23
23
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
24
|
-
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, conversationHistoryFromContext, recallRelevantTurns, renderConversationEnvelopeForPrompt, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, prepareConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, propose, proposePlan, recordGovernedCorrection, HintStore, defaultHintIndexPath, ensureHintIndexFresh, listHintsFromGit, getHintEvaluationFromGit, getCorrectionTraceFromGit, inspectGovernedHint, editGovernedHintCandidate, reopenGovernedHint, retireHint, supersedeHint, hintsConflict, mineJoinPatterns, reviewGovernedHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, internalRelationIdsInSql, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, projectEmbeddingProvider, isHashedEmbeddingProvider, clearProjectEmbeddingCache, upgradeVectorIndexForProject, openMetadataCatalog, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, isTrustedConversationTurn, resolveInternalRelationIds, analyticalError, withAnalyticalErrorOrigin, withAnalyticalErrorOriginSync, } from '@duckcodeailabs/dql-agent';
|
|
24
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, conversationHistoryFromContext, recallRelevantTurns, renderConversationEnvelopeForPrompt, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildAnalysisQuestionPlan, composeSemanticQueryForQuestion, aggregationIntegrityIssuesForSql, buildLocalContextPack, applyContextPackCompatibility, toAgentRetrievalEvidence, prepareConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureAgentProjectReady, isAgentProjectIndexReady, currentMetadataFingerprint, ensureMetadataCatalogFresh, readIndexedDomainKnowledge, readIndexedKnowledge360, compactSemanticRuntimeFailure, classifyAnalyticalFailure, normalizeWarehouseSqlFailure, parseProposal, propose, proposePlan, recordGovernedCorrection, HintStore, defaultHintIndexPath, ensureHintIndexFresh, listHintsFromGit, getHintEvaluationFromGit, getCorrectionTraceFromGit, inspectGovernedHint, editGovernedHintCandidate, reopenGovernedHint, retireHint, supersedeHint, hintsConflict, mineJoinPatterns, reviewGovernedHint, AgentRunEngine, SqliteAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, buildProposePreview, buildFromPrompt, internalRelationIdsInSql, defaultAgentRunStorePath, defaultAgentRunSqlitePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, deriveAnalyticalRepair, reindexProject, invalidateAgentProjectState, recordAgentRuntimeVersion, resolveDomainContextEnvelope, projectEmbeddingProvider, isHashedEmbeddingProvider, clearProjectEmbeddingCache, upgradeVectorIndexForProject, openMetadataCatalog, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, upsertGeneratedDqlArtifactDraft, loadAgentSemanticLayer, isTrustedConversationTurn, resolveInternalRelationIds, analyticalError, tagAnalyticalError, withAnalyticalErrorOrigin, withAnalyticalErrorOriginSync, } from '@duckcodeailabs/dql-agent';
|
|
25
25
|
import { addSqlResultFilter, filterableResultColumns, replaceBlockStudioSql } from './sql-result-filter.js';
|
|
26
26
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
27
27
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
@@ -403,6 +403,36 @@ export function analyticalFailedRunFromAgentRun(run) {
|
|
|
403
403
|
}
|
|
404
404
|
return undefined;
|
|
405
405
|
}
|
|
406
|
+
/** Exact failed statement retained on an immutable run, if one exists. */
|
|
407
|
+
export function repairableSqlFromAgentRun(run) {
|
|
408
|
+
for (const artifact of run.artifacts) {
|
|
409
|
+
const payload = agentRunRecord(artifact.payload);
|
|
410
|
+
if (!payload)
|
|
411
|
+
continue;
|
|
412
|
+
const dqlArtifact = normalizeDqlArtifactReference(payload.dqlArtifact);
|
|
413
|
+
const sql = agentRunString(payload.proposedSql)
|
|
414
|
+
?? agentRunString(payload.sql)
|
|
415
|
+
?? dqlArtifact?.compiledSql;
|
|
416
|
+
if (sql?.trim())
|
|
417
|
+
return sql.trim();
|
|
418
|
+
}
|
|
419
|
+
return undefined;
|
|
420
|
+
}
|
|
421
|
+
/** Structured producer failure retained on the source run. */
|
|
422
|
+
export function warehouseFailureFromAgentRun(run) {
|
|
423
|
+
for (const artifact of run.artifacts) {
|
|
424
|
+
const payload = agentRunRecord(artifact.payload);
|
|
425
|
+
const failure = agentRunRecord(payload?.warehouseFailure);
|
|
426
|
+
if (failure?.version !== 1 || typeof failure.redactedMessage !== 'string')
|
|
427
|
+
continue;
|
|
428
|
+
return failure;
|
|
429
|
+
}
|
|
430
|
+
return undefined;
|
|
431
|
+
}
|
|
432
|
+
/** Permission, authentication, policy and unsafe-query failures are never AI-repaired. */
|
|
433
|
+
export function analyticalFailureAllowsDeterministicRetry(failure) {
|
|
434
|
+
return !failure || !['permission', 'authentication', 'unsafe', 'cancelled'].includes(failure.category);
|
|
435
|
+
}
|
|
406
436
|
function businessNarrativeGaps(warnings) {
|
|
407
437
|
const businessRelevant = (warnings ?? []).filter((warning) => !(/missing requested output column/i.test(warning)
|
|
408
438
|
|| /\bquery plan\b/i.test(warning)
|
|
@@ -1996,6 +2026,7 @@ export async function startLocalServer(opts) {
|
|
|
1996
2026
|
sql: exploration.sql,
|
|
1997
2027
|
exploratoryCandidate: repairedExploratoryCandidate,
|
|
1998
2028
|
executionError: exploration.error,
|
|
2029
|
+
warehouseFailure: exploration.warehouseFailure,
|
|
1999
2030
|
text: explorationAnswer,
|
|
2000
2031
|
answer: explorationAnswer,
|
|
2001
2032
|
trustLabel: 'Exploratory · DBT-grounded',
|
|
@@ -3242,10 +3273,21 @@ export async function startLocalServer(opts) {
|
|
|
3242
3273
|
const semanticError = semanticCompose?.diagnostics.find((diagnostic) => diagnostic.severity === 'error')?.message;
|
|
3243
3274
|
throw analyticalError(semanticError ?? `DQL artifact "${metadata.name ?? 'draft'}" produced no executable SQL.`, { origin: 'dql_compilation', stage: 'compile' });
|
|
3244
3275
|
}
|
|
3245
|
-
const
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3276
|
+
const preparation = await prepareAnalyticalExecutionSql({
|
|
3277
|
+
sql: semanticCompose?.sql ?? plan.sql,
|
|
3278
|
+
subject: 'DQL artifact query',
|
|
3279
|
+
executor,
|
|
3280
|
+
connection: activeConnection,
|
|
3281
|
+
projectRoot,
|
|
3282
|
+
projectConfig,
|
|
3283
|
+
// Preserve the existing contract: previewed Ask artifacts are explicitly
|
|
3284
|
+
// read-only and bounded; saved/manual DQL execution keeps its compiler-
|
|
3285
|
+
// governed behavior when no preview bound was requested.
|
|
3286
|
+
enforceReadOnly: invocationInput?.rowLimit !== undefined,
|
|
3287
|
+
rowLimit: invocationInput?.rowLimit,
|
|
3288
|
+
});
|
|
3289
|
+
const prepared = { sql: preparation.preparedSql, connection: preparation.connection };
|
|
3290
|
+
const executionSql = preparation.executedSql;
|
|
3249
3291
|
const app = loadRuntimeApp(projectRoot, activePersonaAppId());
|
|
3250
3292
|
const sourceDomain = metadata.domain ?? source.match(/\bdomain\s*=\s*"([^"]+)"/i)?.[1];
|
|
3251
3293
|
assertAppAccess({ app, domain: sourceDomain ?? app?.domain, level: 'execute' });
|
|
@@ -3432,7 +3474,7 @@ export async function startLocalServer(opts) {
|
|
|
3432
3474
|
* string, so the only remaining reasons to differ are the connection and the
|
|
3433
3475
|
* governance gates, both of which report themselves honestly.
|
|
3434
3476
|
*/
|
|
3435
|
-
const executeGeneratedSqlDirect = async (question, sql, seed, executionConnection) => {
|
|
3477
|
+
const executeGeneratedSqlDirect = async (question, sql, seed, executionConnection, bindings) => {
|
|
3436
3478
|
const activeConnection = requireActiveConnection(executionConnection);
|
|
3437
3479
|
const rowBound = clampAnalyticalRowBound(seed?.limit ?? 200);
|
|
3438
3480
|
const trimmed = sql.trim().replace(/;\s*$/, '').trim();
|
|
@@ -3454,13 +3496,18 @@ export async function startLocalServer(opts) {
|
|
|
3454
3496
|
if (semantic.unresolvedRefs.length > 0) {
|
|
3455
3497
|
throw analyticalError(`Unknown semantic reference${semantic.unresolvedRefs.length > 1 ? 's' : ''}: ${semantic.unresolvedRefs.join(', ')}`, { origin: 'dql_compilation', stage: 'compile', code: 'semantic_ref' });
|
|
3456
3498
|
}
|
|
3457
|
-
const
|
|
3458
|
-
|
|
3459
|
-
|
|
3460
|
-
|
|
3461
|
-
|
|
3462
|
-
|
|
3463
|
-
|
|
3499
|
+
const preparation = await prepareAnalyticalExecutionSql({
|
|
3500
|
+
sql: semantic.sql,
|
|
3501
|
+
subject: 'Generated SQL',
|
|
3502
|
+
executor,
|
|
3503
|
+
connection: activeConnection,
|
|
3504
|
+
projectRoot,
|
|
3505
|
+
projectConfig,
|
|
3506
|
+
enforceReadOnly: true,
|
|
3507
|
+
rowLimit: rowBound,
|
|
3508
|
+
});
|
|
3509
|
+
const prepared = { sql: preparation.preparedSql, connection: preparation.connection };
|
|
3510
|
+
const bounded = preparation.rowBound;
|
|
3464
3511
|
const app = loadRuntimeApp(projectRoot, activePersonaAppId());
|
|
3465
3512
|
// Generated SQL is governed as 'uncategorized' unless its seed declares a
|
|
3466
3513
|
// domain — the same value `generatedSqlArtifactContract` stamps into the
|
|
@@ -3488,7 +3535,7 @@ export async function startLocalServer(opts) {
|
|
|
3488
3535
|
})
|
|
3489
3536
|
: null;
|
|
3490
3537
|
const rawResult = semanticExecution?.result
|
|
3491
|
-
?? await executor.executeQuery(
|
|
3538
|
+
?? await executor.executeQuery(preparation.executedSql, bindings?.sqlParams ?? [], runtimeVariables(bindings?.variables ?? {}), preparation.connection);
|
|
3492
3539
|
const normalized = normalizeQueryResult(rawResult, semantic.semanticRefs);
|
|
3493
3540
|
// The authoritative bound. `buildRowBoundedSql` may legitimately decline to
|
|
3494
3541
|
// touch the statement, and only one of 15 drivers honours a maxRows hint, so
|
|
@@ -3528,34 +3575,65 @@ export async function startLocalServer(opts) {
|
|
|
3528
3575
|
*/
|
|
3529
3576
|
const executeExploratoryCandidate = async (candidate, schemaContext = [], question = '', requestedTopN, executionConnection) => {
|
|
3530
3577
|
const activeConnection = requireActiveConnection(executionConnection);
|
|
3531
|
-
const
|
|
3578
|
+
const failed = (message, sql, repairs, proofs = [], code = 'exploratory_validation_failed') => ({
|
|
3579
|
+
proofs,
|
|
3580
|
+
sql,
|
|
3581
|
+
repairs,
|
|
3582
|
+
error: message,
|
|
3583
|
+
warehouseFailure: normalizeWarehouseSqlFailure(analyticalError(message, {
|
|
3584
|
+
origin: 'governance_gate',
|
|
3585
|
+
stage: 'validation',
|
|
3586
|
+
code,
|
|
3587
|
+
}), activeConnection.driver),
|
|
3588
|
+
});
|
|
3589
|
+
let normalizedCandidateSql;
|
|
3590
|
+
try {
|
|
3591
|
+
normalizedCandidateSql = (await prepareAnalyticalExecutionSql({
|
|
3592
|
+
sql: candidate.sql,
|
|
3593
|
+
subject: question || 'this exploratory query',
|
|
3594
|
+
executor,
|
|
3595
|
+
connection: activeConnection,
|
|
3596
|
+
projectRoot,
|
|
3597
|
+
projectConfig,
|
|
3598
|
+
enforceReadOnly: true,
|
|
3599
|
+
})).preparedSql;
|
|
3600
|
+
}
|
|
3601
|
+
catch (error) {
|
|
3602
|
+
const failure = normalizeWarehouseSqlFailure(error, activeConnection.driver);
|
|
3603
|
+
return {
|
|
3604
|
+
proofs: [],
|
|
3605
|
+
sql: candidate.sql,
|
|
3606
|
+
repairs: [],
|
|
3607
|
+
error: failure.redactedMessage,
|
|
3608
|
+
warehouseFailure: failure,
|
|
3609
|
+
};
|
|
3610
|
+
}
|
|
3611
|
+
const preflight = repairExploratorySqlBeforeExecution(normalizedCandidateSql, schemaContext, question, activeConnection.driver);
|
|
3532
3612
|
const boundedSql = applyRequestedTopNToExploratorySql(preflight.sql, requestedTopN);
|
|
3533
3613
|
const repairs = boundedSql === preflight.sql
|
|
3534
3614
|
? [...preflight.repairs]
|
|
3535
3615
|
: [...preflight.repairs, `Applied the requested overall top-${requestedTopN} bound before exploratory execution.`];
|
|
3536
3616
|
if (preflight.blockedReason) {
|
|
3537
|
-
return
|
|
3538
|
-
proofs: [],
|
|
3539
|
-
sql: boundedSql,
|
|
3540
|
-
repairs,
|
|
3541
|
-
error: preflight.blockedReason,
|
|
3542
|
-
};
|
|
3617
|
+
return failed(preflight.blockedReason, boundedSql, repairs, [], 'exploratory_preflight_blocked');
|
|
3543
3618
|
}
|
|
3544
3619
|
const analysis = analyzeSqlReferences(boundedSql, activeConnection.driver);
|
|
3620
|
+
if (!analysis.parsed && sqlMayContainJoin(boundedSql)) {
|
|
3621
|
+
return failed('DQL could not parse the exploratory SQL to validate its join predicates. Nothing was sent to the warehouse.', boundedSql, repairs, [], 'exploratory_join_parse_failed');
|
|
3622
|
+
}
|
|
3545
3623
|
if (!analysis.parsed) {
|
|
3546
|
-
|
|
3624
|
+
repairs.push('The local DQL parser could not inspect this join-free statement; warehouse read-only execution will provide the authoritative syntax check.');
|
|
3547
3625
|
}
|
|
3548
|
-
const probeableJoins = probeableExploratoryJoins(analysis.joins, analysis.ctes, analysis.derivedRelations);
|
|
3626
|
+
const probeableJoins = probeableExploratoryJoins(analysis.parsed ? analysis.joins : [], analysis.parsed ? analysis.ctes : [], analysis.parsed ? analysis.derivedRelations : []);
|
|
3549
3627
|
// A normal customer → order → order-item → product question needs three
|
|
3550
3628
|
// joins. Keep the lane bounded, but do not reject this common dbt-star path
|
|
3551
3629
|
// solely because the old two-join demo limit was too small. CTE-internal
|
|
3552
3630
|
// joins are counted physically (they appear in analysis.joins with their
|
|
3553
3631
|
// physical endpoints); joins ON a CTE are restructuring, not new paths.
|
|
3554
3632
|
if (probeableJoins.length > 4) {
|
|
3555
|
-
return
|
|
3633
|
+
return failed('This exploratory query has more than four joins and requires an analyst to review the relationship path.', boundedSql, repairs, [], 'exploratory_join_limit');
|
|
3556
3634
|
}
|
|
3557
3635
|
if (probeableJoins.some((join) => !join.leftRelation || !join.rightRelation)) {
|
|
3558
|
-
return
|
|
3636
|
+
return failed('This exploratory query has a join endpoint DQL could not resolve for bounded validation.', boundedSql, repairs, [], 'exploratory_join_unresolved');
|
|
3559
3637
|
}
|
|
3560
3638
|
// Structural binding: when the candidate carries a declared draft join path,
|
|
3561
3639
|
// every SQL join must ride one of its edges on the declared key pair. A join
|
|
@@ -3568,22 +3646,12 @@ export async function startLocalServer(opts) {
|
|
|
3568
3646
|
const edge = declaredEdges.find((value) => declaredExploratoryRelationMatches(join.leftRelation, value.fromRelation) && declaredExploratoryRelationMatches(join.rightRelation, value.toRelation)
|
|
3569
3647
|
|| declaredExploratoryRelationMatches(join.leftRelation, value.toRelation) && declaredExploratoryRelationMatches(join.rightRelation, value.fromRelation));
|
|
3570
3648
|
if (!edge) {
|
|
3571
|
-
return {
|
|
3572
|
-
proofs: [],
|
|
3573
|
-
sql: boundedSql,
|
|
3574
|
-
repairs,
|
|
3575
|
-
error: `The join between ${join.leftRelation} and ${join.rightRelation} is not part of the declared relationship path, so DQL did not execute it. Declare the relationship in the DQL model to enable it.`,
|
|
3576
|
-
};
|
|
3649
|
+
return failed(`The join between ${join.leftRelation} and ${join.rightRelation} is not part of the declared relationship path, so DQL did not execute it. Declare the relationship in the DQL model to enable it.`, boundedSql, repairs, [], 'undeclared_relationship');
|
|
3577
3650
|
}
|
|
3578
3651
|
const keyMatches = edge.keys.some((key) => exploratoryColumnsMatch(join.leftColumn, key.from) && exploratoryColumnsMatch(join.rightColumn, key.to)
|
|
3579
3652
|
|| exploratoryColumnsMatch(join.leftColumn, key.to) && exploratoryColumnsMatch(join.rightColumn, key.from));
|
|
3580
3653
|
if (!keyMatches) {
|
|
3581
|
-
return {
|
|
3582
|
-
proofs: [],
|
|
3583
|
-
sql: boundedSql,
|
|
3584
|
-
repairs,
|
|
3585
|
-
error: `The join ${join.leftColumn} = ${join.rightColumn} uses different keys than relationship "${edge.relationshipId}" declares (${edge.keys.map((key) => `${key.from}=${key.to}`).join(', ')}). Review the declared keys before executing.`,
|
|
3586
|
-
};
|
|
3654
|
+
return failed(`The join ${join.leftColumn} = ${join.rightColumn} uses different keys than relationship "${edge.relationshipId}" declares (${edge.keys.map((key) => `${key.from}=${key.to}`).join(', ')}). Review the declared keys before executing.`, boundedSql, repairs, [], 'relationship_key_mismatch');
|
|
3587
3655
|
}
|
|
3588
3656
|
joinEdges.set(index, edge);
|
|
3589
3657
|
}
|
|
@@ -3594,7 +3662,7 @@ export async function startLocalServer(opts) {
|
|
|
3594
3662
|
}
|
|
3595
3663
|
}
|
|
3596
3664
|
}
|
|
3597
|
-
if (probeableJoins.length < analysis.joins.length) {
|
|
3665
|
+
if (analysis.parsed && probeableJoins.length < analysis.joins.length) {
|
|
3598
3666
|
repairs.push(`Skipped join probes for ${analysis.joins.length - probeableJoins.length} join(s) on derived (CTE) relations; the final query execution validates them.`);
|
|
3599
3667
|
}
|
|
3600
3668
|
const proofs = [];
|
|
@@ -3615,14 +3683,21 @@ export async function startLocalServer(opts) {
|
|
|
3615
3683
|
// answer — these gates only inspect the raw key samples.
|
|
3616
3684
|
const contradiction = exploratoryProbeContradiction(raw.rows, join, joinEdges.get(index));
|
|
3617
3685
|
if (contradiction) {
|
|
3618
|
-
return
|
|
3686
|
+
return failed(contradiction, boundedSql, repairs, proofs, 'relationship_probe_contradiction');
|
|
3619
3687
|
}
|
|
3620
3688
|
}
|
|
3621
3689
|
const result = await executeGeneratedArtifactForAgent(question, boundedSql, undefined, activeConnection);
|
|
3622
3690
|
return { result, proofs, sql: boundedSql, repairs };
|
|
3623
3691
|
}
|
|
3624
3692
|
catch (error) {
|
|
3625
|
-
|
|
3693
|
+
const failure = normalizeWarehouseSqlFailure(error, activeConnection.driver);
|
|
3694
|
+
return {
|
|
3695
|
+
proofs,
|
|
3696
|
+
sql: boundedSql,
|
|
3697
|
+
repairs,
|
|
3698
|
+
error: failure.redactedMessage,
|
|
3699
|
+
warehouseFailure: failure,
|
|
3700
|
+
};
|
|
3626
3701
|
}
|
|
3627
3702
|
};
|
|
3628
3703
|
/**
|
|
@@ -3728,6 +3803,115 @@ export async function startLocalServer(opts) {
|
|
|
3728
3803
|
return [];
|
|
3729
3804
|
}
|
|
3730
3805
|
};
|
|
3806
|
+
const boundedRepairError = (message, status, warehouseFailure) => Object.assign(new Error(message), { status, ...(warehouseFailure ? { warehouseFailure } : {}) });
|
|
3807
|
+
/**
|
|
3808
|
+
* Shared one-attempt SQL repair for Ask and Notebook. It never creates a
|
|
3809
|
+
* conversation, writes a DQL file, changes targets, or promotes trust.
|
|
3810
|
+
*/
|
|
3811
|
+
const executeBoundedSqlRepair = async (input) => {
|
|
3812
|
+
let result;
|
|
3813
|
+
let repairedSql = input.sourceSql;
|
|
3814
|
+
let repairMode = 'deterministic';
|
|
3815
|
+
let firstFailure;
|
|
3816
|
+
try {
|
|
3817
|
+
result = await executeGeneratedSqlDirect(input.question, input.sourceSql, undefined, input.targetConnection, { sqlParams: input.sqlParams, variables: input.variables });
|
|
3818
|
+
}
|
|
3819
|
+
catch (error) {
|
|
3820
|
+
firstFailure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
3821
|
+
}
|
|
3822
|
+
if (!result) {
|
|
3823
|
+
if (firstFailure?.retryDisposition !== 'model_repair') {
|
|
3824
|
+
throw boundedRepairError(firstFailure?.redactedMessage ?? 'The deterministic repair did not produce an executable query.', 422, firstFailure);
|
|
3825
|
+
}
|
|
3826
|
+
const provider = await createBlockStudioAssistProvider(projectRoot);
|
|
3827
|
+
if (!provider) {
|
|
3828
|
+
throw boundedRepairError('Configure an available AI provider in Settings to repair this query.', 409);
|
|
3829
|
+
}
|
|
3830
|
+
let schemaContext;
|
|
3831
|
+
try {
|
|
3832
|
+
schemaContext = await getSchemaContextForAgent(input.question, undefined, input.targetConnection, input.targetConnectionName);
|
|
3833
|
+
}
|
|
3834
|
+
catch (error) {
|
|
3835
|
+
const failure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
3836
|
+
throw boundedRepairError(failure.redactedMessage, 422, failure);
|
|
3837
|
+
}
|
|
3838
|
+
const boundedSchema = schemaContext.slice(0, 16).map((table) => ({
|
|
3839
|
+
relation: table.relation,
|
|
3840
|
+
columns: table.columns.slice(0, 40).map((column) => ({ name: column.name, type: column.type })),
|
|
3841
|
+
}));
|
|
3842
|
+
let raw;
|
|
3843
|
+
try {
|
|
3844
|
+
raw = await provider.generate([
|
|
3845
|
+
{
|
|
3846
|
+
role: 'system',
|
|
3847
|
+
content: [
|
|
3848
|
+
'Repair one failed read-only analytical SQL statement.',
|
|
3849
|
+
'Return exactly one JSON object in a ```json fence with keys summary, sql, viz, outputs.',
|
|
3850
|
+
'Preserve the requested grain, filters, output columns, and aggregation semantics.',
|
|
3851
|
+
input.sqlParams?.length
|
|
3852
|
+
? 'Preserve every positional parameter placeholder exactly; do not add, remove, or renumber placeholders.'
|
|
3853
|
+
: '',
|
|
3854
|
+
'Use only relations and columns present in the supplied schema context.',
|
|
3855
|
+
'Never emit internal DQL relation identities such as source::, model::, seed::, or snapshot::.',
|
|
3856
|
+
'Do not add writes, DDL, multiple statements, or commentary outside the JSON fence.',
|
|
3857
|
+
].join(' '),
|
|
3858
|
+
},
|
|
3859
|
+
{
|
|
3860
|
+
role: 'user',
|
|
3861
|
+
content: JSON.stringify({
|
|
3862
|
+
question: input.question,
|
|
3863
|
+
driver: input.targetConnection.driver,
|
|
3864
|
+
failure: firstFailure.redactedMessage,
|
|
3865
|
+
sql: input.sourceSql,
|
|
3866
|
+
parameters: input.sqlParams?.map((parameter) => ({
|
|
3867
|
+
name: parameter.name,
|
|
3868
|
+
placeholder: `$${parameter.position}`,
|
|
3869
|
+
})),
|
|
3870
|
+
schema: boundedSchema,
|
|
3871
|
+
}),
|
|
3872
|
+
},
|
|
3873
|
+
], { maxTokens: 1200, temperature: 0 });
|
|
3874
|
+
}
|
|
3875
|
+
catch {
|
|
3876
|
+
throw boundedRepairError('The AI provider could not complete this repair. Check Provider Settings and retry.', 502);
|
|
3877
|
+
}
|
|
3878
|
+
const proposal = parseProposal(raw);
|
|
3879
|
+
if (!proposal.sql?.trim()) {
|
|
3880
|
+
throw boundedRepairError('The AI repair did not return an executable SQL statement.', 422);
|
|
3881
|
+
}
|
|
3882
|
+
repairedSql = proposal.sql.trim();
|
|
3883
|
+
const requiredPlaceholders = (input.sqlParams ?? []).map((parameter) => `$${parameter.position}`);
|
|
3884
|
+
if (requiredPlaceholders.some((placeholder) => !new RegExp(`${placeholder.replace('$', '\\$')}\\b`).test(repairedSql))) {
|
|
3885
|
+
throw boundedRepairError('The repair changed this cell\'s parameter contract, so DQL stopped it before execution.', 422);
|
|
3886
|
+
}
|
|
3887
|
+
repairMode = 'ai';
|
|
3888
|
+
try {
|
|
3889
|
+
result = await executeGeneratedSqlDirect(input.question, repairedSql, undefined, input.targetConnection, { sqlParams: input.sqlParams, variables: input.variables });
|
|
3890
|
+
}
|
|
3891
|
+
catch (error) {
|
|
3892
|
+
const failure = normalizeWarehouseSqlFailure(error, input.targetConnection.driver);
|
|
3893
|
+
throw boundedRepairError(failure.redactedMessage, 422, failure);
|
|
3894
|
+
}
|
|
3895
|
+
}
|
|
3896
|
+
// Keep preview-only row bounds out of editable notebook source. The one
|
|
3897
|
+
// deterministic source rewrite we do persist is decoding internal graph
|
|
3898
|
+
// identities, because those can never execute as warehouse SQL.
|
|
3899
|
+
let rewrites = [];
|
|
3900
|
+
if (internalRelationIdsInSql(repairedSql).length > 0) {
|
|
3901
|
+
const normalizedSource = await prepareAnalyticalExecutionSql({
|
|
3902
|
+
sql: repairedSql,
|
|
3903
|
+
subject: 'Repaired SQL',
|
|
3904
|
+
executor,
|
|
3905
|
+
connection: input.targetConnection,
|
|
3906
|
+
projectRoot,
|
|
3907
|
+
projectConfig,
|
|
3908
|
+
enforceReadOnly: true,
|
|
3909
|
+
});
|
|
3910
|
+
repairedSql = normalizedSource.decodedSql;
|
|
3911
|
+
rewrites = normalizedSource.rewrites;
|
|
3912
|
+
}
|
|
3913
|
+
return { result, repairedSql, repairMode, firstFailure, rewrites };
|
|
3914
|
+
};
|
|
3731
3915
|
const generateInvestigationSqlForApp = async (input) => {
|
|
3732
3916
|
const governed = resolveGovernedAnswerRunner(projectRoot);
|
|
3733
3917
|
const resolvedProvider = governed?.provider ?? null;
|
|
@@ -4784,9 +4968,10 @@ export async function startLocalServer(opts) {
|
|
|
4784
4968
|
*/
|
|
4785
4969
|
const MIN_COMPRESS_BYTES = 1024;
|
|
4786
4970
|
const COMPRESSIBLE = /^(?:text\/(?!event-stream)|application\/(?:json|javascript|xml|wasm)|image\/svg)/i;
|
|
4787
|
-
const withCompression = (request,
|
|
4971
|
+
const withCompression = (request, rawResponse) => {
|
|
4972
|
+
const response = rawResponse;
|
|
4788
4973
|
if (!/\bgzip\b/i.test(String(request.headers['accept-encoding'] ?? '')))
|
|
4789
|
-
return
|
|
4974
|
+
return rawResponse;
|
|
4790
4975
|
const writeHead = response.writeHead.bind(response);
|
|
4791
4976
|
const end = response.end.bind(response);
|
|
4792
4977
|
const write = response.write.bind(response);
|
|
@@ -4857,7 +5042,7 @@ export async function startLocalServer(opts) {
|
|
|
4857
5042
|
});
|
|
4858
5043
|
return end(gz);
|
|
4859
5044
|
});
|
|
4860
|
-
return
|
|
5045
|
+
return rawResponse;
|
|
4861
5046
|
};
|
|
4862
5047
|
const server = createServer((req, res) => {
|
|
4863
5048
|
res = withCompression(req, res);
|
|
@@ -5517,15 +5702,79 @@ export async function startLocalServer(opts) {
|
|
|
5517
5702
|
}
|
|
5518
5703
|
if (req.method === 'GET' && path === '/api/modeling/dbt-first/inventory') {
|
|
5519
5704
|
const requestId = apiRequestId('modeling-inventory');
|
|
5520
|
-
const
|
|
5705
|
+
const physicalOnly = url.searchParams.get('physicalOnly') === 'true';
|
|
5706
|
+
const modelingEnabled = projectConfig.manifestVersion === 3 && projectConfig.modeling?.mode === 'dbt-first';
|
|
5707
|
+
if (physicalOnly && !modelingEnabled) {
|
|
5708
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5709
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
5710
|
+
requestId,
|
|
5711
|
+
code: 'DBT_FIRST_NOT_ENABLED',
|
|
5712
|
+
message: 'dbt-first modeling is not enabled. Connect and apply a dbt project from Settings → Project & dbt.',
|
|
5713
|
+
nextActions: ['Open Settings → Project & dbt and apply a valid dbt project.'],
|
|
5714
|
+
})));
|
|
5715
|
+
return;
|
|
5716
|
+
}
|
|
5717
|
+
const dbtProjectDir = findDbtProjectPath(projectRoot, projectConfig);
|
|
5718
|
+
const configuredManifestPath = projectConfig.dbt?.manifestPath ?? 'target/manifest.json';
|
|
5719
|
+
const expectedManifestPath = resolve(dbtProjectDir, configuredManifestPath);
|
|
5720
|
+
if (physicalOnly && !existsSync(expectedManifestPath)) {
|
|
5721
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5722
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
5723
|
+
requestId,
|
|
5724
|
+
code: 'DBT_MANIFEST_NOT_FOUND',
|
|
5725
|
+
message: `The configured dbt manifest was not found at ${configuredManifestPath}.`,
|
|
5726
|
+
details: { manifestPath: configuredManifestPath },
|
|
5727
|
+
nextActions: ['Run dbt parse, dbt compile, or dbt build, then refresh the database catalog.'],
|
|
5728
|
+
})));
|
|
5729
|
+
return;
|
|
5730
|
+
}
|
|
5731
|
+
let snapshot;
|
|
5732
|
+
if (physicalOnly) {
|
|
5733
|
+
try {
|
|
5734
|
+
snapshot = projectSnapshot();
|
|
5735
|
+
}
|
|
5736
|
+
catch (error) {
|
|
5737
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5738
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
5739
|
+
requestId,
|
|
5740
|
+
code: 'DBT_MANIFEST_COMPILE_FAILED',
|
|
5741
|
+
message: apiErrorMessage(error),
|
|
5742
|
+
details: { manifestPath: configuredManifestPath },
|
|
5743
|
+
nextActions: ['Fix or rebuild the dbt manifest, then refresh the database catalog.'],
|
|
5744
|
+
})));
|
|
5745
|
+
return;
|
|
5746
|
+
}
|
|
5747
|
+
}
|
|
5748
|
+
else {
|
|
5749
|
+
snapshot = projectSnapshot();
|
|
5750
|
+
}
|
|
5521
5751
|
const manifest = snapshot.manifest;
|
|
5522
|
-
|
|
5752
|
+
if (physicalOnly && (manifest.manifestVersion !== 3 || !manifest.modeling || !manifest.dbtProvenance)) {
|
|
5753
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5754
|
+
res.end(serializeJSON(apiErrorEnvelope({
|
|
5755
|
+
requestId,
|
|
5756
|
+
snapshotId: snapshot.snapshotId,
|
|
5757
|
+
code: 'DBT_MANIFEST_COMPILE_FAILED',
|
|
5758
|
+
message: snapshot.error ?? 'DQL could not compile the dbt database catalog.',
|
|
5759
|
+
details: { manifestPath: configuredManifestPath },
|
|
5760
|
+
nextActions: ['Review the dbt artifact, rebuild it, then refresh the database catalog.'],
|
|
5761
|
+
})));
|
|
5762
|
+
return;
|
|
5763
|
+
}
|
|
5764
|
+
// PERF-001 / CTX-005: only the Database browser opts into the smaller,
|
|
5765
|
+
// physical-relation contract. Existing modeling consumers retain their
|
|
5766
|
+
// original inventory scope and pagination defaults.
|
|
5767
|
+
const maxLimit = physicalOnly ? 100 : 200;
|
|
5768
|
+
const defaultLimit = physicalOnly ? 25 : 50;
|
|
5769
|
+
const limit = Math.min(maxLimit, Math.max(1, Number(url.searchParams.get('limit')) || defaultLimit));
|
|
5523
5770
|
const cursor = Math.max(0, Number(url.searchParams.get('cursor')) || 0);
|
|
5524
5771
|
const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
|
|
5525
5772
|
const queryTokens = query.split(/[^a-z0-9]+/).filter(Boolean);
|
|
5526
5773
|
const domain = (url.searchParams.get('domain') ?? '').trim();
|
|
5527
5774
|
const boundByDbtId = new Map(Object.values(manifest.modeling?.entities ?? {}).map((entity) => [entity.dbtUniqueId, entity]));
|
|
5775
|
+
const physicalIds = physicalOnly ? physicalDbtRelationIds(expectedManifestPath) : undefined;
|
|
5528
5776
|
const nodes = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
5777
|
+
.filter((node) => !physicalOnly || (Boolean(node.relation) && (!physicalIds || physicalIds.has(node.uniqueId))))
|
|
5529
5778
|
.filter((node) => {
|
|
5530
5779
|
if (!queryTokens.length)
|
|
5531
5780
|
return true;
|
|
@@ -5558,6 +5807,11 @@ export async function startLocalServer(opts) {
|
|
|
5558
5807
|
nextCursor: cursor + limit < nodes.length ? cursor + limit : null,
|
|
5559
5808
|
total: nodes.length,
|
|
5560
5809
|
snapshotId: snapshot.snapshotId,
|
|
5810
|
+
...(physicalOnly ? {
|
|
5811
|
+
manifestPath: manifest.dbtProvenance.manifestPath,
|
|
5812
|
+
projectName: manifest.dbtProvenance.projectName,
|
|
5813
|
+
scope: 'dbt_relations',
|
|
5814
|
+
} : {}),
|
|
5561
5815
|
}));
|
|
5562
5816
|
return;
|
|
5563
5817
|
}
|
|
@@ -5710,7 +5964,7 @@ export async function startLocalServer(opts) {
|
|
|
5710
5964
|
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId)
|
|
5711
5965
|
throw Object.assign(new Error('Project sources changed before relationship validation.'), { code: 'SOURCE_CHANGED' });
|
|
5712
5966
|
const manifest = snapshot.manifest;
|
|
5713
|
-
const activeConnection =
|
|
5967
|
+
const activeConnection = await resolveExecutionConnection(body);
|
|
5714
5968
|
const evidence = await validateModelingRelationship(body.relationship, manifest, (sql) => executor.executeQuery(sql, [], {}, activeConnection), (identifier) => getDialect(activeConnection.driver).quoteIdentifier(identifier));
|
|
5715
5969
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5716
5970
|
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, evidence }));
|
|
@@ -5735,6 +5989,195 @@ export async function startLocalServer(opts) {
|
|
|
5735
5989
|
res.end(serializeJSON({ runs, total: agentRunStore.list().length, limit }));
|
|
5736
5990
|
return;
|
|
5737
5991
|
}
|
|
5992
|
+
/**
|
|
5993
|
+
* One-click bounded execution repair.
|
|
5994
|
+
*
|
|
5995
|
+
* The source run is immutable. First, retry its exact SQL through the
|
|
5996
|
+
* shared deterministic preparation boundary (this repairs internal IDs and
|
|
5997
|
+
* dialect-safe row bounds without an LLM). Only a connector-classified
|
|
5998
|
+
* model-repair failure may spend one provider call to rewrite the SQL. The
|
|
5999
|
+
* derived SQL is still read-only, bounded, and executed on the source run's
|
|
6000
|
+
* exact target before a replacement card is returned.
|
|
6001
|
+
*
|
|
6002
|
+
* Acceptance: API-003, API-007, UI-012, UI-013, SEC-004, E2E-014.
|
|
6003
|
+
*/
|
|
6004
|
+
if (req.method === 'POST' && /^\/api\/agent-runs\/[^/]+\/repair-execution$/.test(path)) {
|
|
6005
|
+
const match = path.match(/^\/api\/agent-runs\/([^/]+)\/repair-execution$/);
|
|
6006
|
+
const sourceRunId = decodeURIComponent(match?.[1] ?? '');
|
|
6007
|
+
const sourceRun = await agentRunStore.get(sourceRunId);
|
|
6008
|
+
if (!sourceRun) {
|
|
6009
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6010
|
+
res.end(serializeJSON({ error: 'Agent run not found.' }));
|
|
6011
|
+
return;
|
|
6012
|
+
}
|
|
6013
|
+
const derivedRunId = `${sourceRun.id}:repair:1`;
|
|
6014
|
+
const existing = await agentRunStore.get(derivedRunId);
|
|
6015
|
+
if (existing) {
|
|
6016
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6017
|
+
res.end(serializeJSON({ run: existing }));
|
|
6018
|
+
return;
|
|
6019
|
+
}
|
|
6020
|
+
const sourceSql = repairableSqlFromAgentRun(sourceRun);
|
|
6021
|
+
const retainedFailure = warehouseFailureFromAgentRun(sourceRun);
|
|
6022
|
+
if (!sourceSql || !analyticalFailureAllowsDeterministicRetry(retainedFailure)) {
|
|
6023
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6024
|
+
res.end(serializeJSON({
|
|
6025
|
+
error: sourceSql
|
|
6026
|
+
? 'This failure requires authorized access or a policy change and cannot be repaired by AI.'
|
|
6027
|
+
: 'This run has no retained SQL statement to repair.',
|
|
6028
|
+
...(retainedFailure ? { warehouseFailure: retainedFailure } : {}),
|
|
6029
|
+
}));
|
|
6030
|
+
return;
|
|
6031
|
+
}
|
|
6032
|
+
const body = agentRunRecord(await readJSON(req).catch(() => null));
|
|
6033
|
+
const threadId = agentRunString(body?.threadId);
|
|
6034
|
+
let targetConnection;
|
|
6035
|
+
try {
|
|
6036
|
+
targetConnection = await resolveAgentRunExecutionConnection({
|
|
6037
|
+
question: sourceRun.question,
|
|
6038
|
+
executionTarget: sourceRun.executionTarget,
|
|
6039
|
+
});
|
|
6040
|
+
}
|
|
6041
|
+
catch (error) {
|
|
6042
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6043
|
+
res.end(serializeJSON({ error: apiErrorMessage(error) }));
|
|
6044
|
+
return;
|
|
6045
|
+
}
|
|
6046
|
+
const targetConnectionName = sourceRun.executionTarget?.target === 'connection'
|
|
6047
|
+
? sourceRun.executionTarget.connectionName
|
|
6048
|
+
: sourceRun.executionTarget?.target === 'local'
|
|
6049
|
+
? 'local'
|
|
6050
|
+
: projectConfig.defaultConnectionName;
|
|
6051
|
+
let repaired;
|
|
6052
|
+
try {
|
|
6053
|
+
repaired = await executeBoundedSqlRepair({
|
|
6054
|
+
question: sourceRun.question,
|
|
6055
|
+
sourceSql,
|
|
6056
|
+
targetConnection,
|
|
6057
|
+
targetConnectionName,
|
|
6058
|
+
});
|
|
6059
|
+
}
|
|
6060
|
+
catch (error) {
|
|
6061
|
+
const repairError = error;
|
|
6062
|
+
res.writeHead(repairError.status ?? 500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6063
|
+
res.end(serializeJSON({
|
|
6064
|
+
error: repairError.message,
|
|
6065
|
+
...(repairError.warehouseFailure ? { warehouseFailure: repairError.warehouseFailure } : {}),
|
|
6066
|
+
}));
|
|
6067
|
+
return;
|
|
6068
|
+
}
|
|
6069
|
+
const { result, repairedSql, repairMode, firstFailure } = repaired;
|
|
6070
|
+
const completedAt = new Date().toISOString();
|
|
6071
|
+
const preview = agentResultToSynthesisPreview(result);
|
|
6072
|
+
const synthesis = await synthesizeAnswer({
|
|
6073
|
+
question: sourceRun.question,
|
|
6074
|
+
audience: 'stakeholder',
|
|
6075
|
+
resultPreview: preview,
|
|
6076
|
+
sql: repairedSql,
|
|
6077
|
+
draftText: `The repaired query returned ${result.rowCount} row${result.rowCount === 1 ? '' : 's'}.`,
|
|
6078
|
+
...(preview ? { columnFormats: agentColumnDisplayFormats(projectRoot, preview.columns) } : {}),
|
|
6079
|
+
});
|
|
6080
|
+
const sourceFailure = analyticalFailedRunFromAgentRun(sourceRun)?.failure;
|
|
6081
|
+
const artifact = agentRunArtifact('answer', 'Repaired analytical answer', {
|
|
6082
|
+
kind: 'uncertified',
|
|
6083
|
+
certification: 'analyst_review_required',
|
|
6084
|
+
reviewStatus: 'analyst_review_required',
|
|
6085
|
+
trustLabel: 'AI-generated · review required',
|
|
6086
|
+
answer: synthesis.text,
|
|
6087
|
+
text: synthesis.text,
|
|
6088
|
+
result,
|
|
6089
|
+
proposedSql: repairedSql,
|
|
6090
|
+
sql: result.sql ?? repairedSql,
|
|
6091
|
+
dqlArtifact: result.dqlArtifact,
|
|
6092
|
+
previousAttempt: {
|
|
6093
|
+
sourceRunId: sourceRun.id,
|
|
6094
|
+
sql: sourceSql,
|
|
6095
|
+
failure: retainedFailure ?? firstFailure,
|
|
6096
|
+
},
|
|
6097
|
+
repair: { attempt: 1, mode: repairMode },
|
|
6098
|
+
}, undefined, 'review_required');
|
|
6099
|
+
const evaluation = {
|
|
6100
|
+
id: `${derivedRunId}:execution`,
|
|
6101
|
+
label: 'Repaired query execution',
|
|
6102
|
+
passed: true,
|
|
6103
|
+
severity: 'info',
|
|
6104
|
+
message: `The ${repairMode === 'ai' ? 'AI-repaired' : 'deterministically normalized'} read-only query executed on the original target.`,
|
|
6105
|
+
};
|
|
6106
|
+
const repairStep = {
|
|
6107
|
+
id: `${derivedRunId}:step`,
|
|
6108
|
+
index: 0,
|
|
6109
|
+
route: 'generated_answer',
|
|
6110
|
+
resolvedRoute: 'generated_answer',
|
|
6111
|
+
goal: 'Repair and execute the failed analytical query on its original target.',
|
|
6112
|
+
successCriteria: ['Read-only validation passes.', 'The original execution target returns a bounded result.'],
|
|
6113
|
+
status: 'repaired',
|
|
6114
|
+
attempts: 1,
|
|
6115
|
+
summary: evaluation.message,
|
|
6116
|
+
evaluations: [evaluation],
|
|
6117
|
+
artifacts: [artifact],
|
|
6118
|
+
};
|
|
6119
|
+
const events = [
|
|
6120
|
+
{ id: `${derivedRunId}:repair`, runId: derivedRunId, type: 'repair.attempted', at: completedAt, message: 'Started one bounded query repair.', route: 'generated_answer' },
|
|
6121
|
+
{ id: `${derivedRunId}:artifact`, runId: derivedRunId, type: 'artifact.created', at: completedAt, message: 'Created the repaired result artifact.', route: 'generated_answer', trustState: 'review_required' },
|
|
6122
|
+
{ id: `${derivedRunId}:complete`, runId: derivedRunId, type: 'run.completed', at: completedAt, message: 'The repaired query executed successfully.', route: 'generated_answer', status: 'needs_review', trustState: 'review_required' },
|
|
6123
|
+
];
|
|
6124
|
+
const derivedRun = {
|
|
6125
|
+
id: derivedRunId,
|
|
6126
|
+
question: sourceRun.question,
|
|
6127
|
+
requestedMode: sourceRun.requestedMode,
|
|
6128
|
+
route: 'generated_answer',
|
|
6129
|
+
status: 'needs_review',
|
|
6130
|
+
trustState: 'review_required',
|
|
6131
|
+
stopReason: 'generated_review_required',
|
|
6132
|
+
startedAt: completedAt,
|
|
6133
|
+
completedAt,
|
|
6134
|
+
selectedObject: sourceRun.selectedObject,
|
|
6135
|
+
executionTarget: sourceRun.executionTarget,
|
|
6136
|
+
routeDecision: sourceRun.routeDecision,
|
|
6137
|
+
plan: sourceRun.plan,
|
|
6138
|
+
steps: [repairStep],
|
|
6139
|
+
summary: synthesis.text,
|
|
6140
|
+
answer: synthesis.text,
|
|
6141
|
+
answerKind: 'governed',
|
|
6142
|
+
artifacts: [artifact],
|
|
6143
|
+
evaluations: [evaluation],
|
|
6144
|
+
events,
|
|
6145
|
+
nextActions: sourceRun.nextActions,
|
|
6146
|
+
repairAttempts: 1,
|
|
6147
|
+
lifecycle: {
|
|
6148
|
+
version: 1,
|
|
6149
|
+
state: 'terminal',
|
|
6150
|
+
phase: 'run.completed',
|
|
6151
|
+
revision: 1,
|
|
6152
|
+
eventCursor: events.length,
|
|
6153
|
+
startedAt: completedAt,
|
|
6154
|
+
updatedAt: completedAt,
|
|
6155
|
+
completedAt,
|
|
6156
|
+
},
|
|
6157
|
+
diagnosticReceipt: {
|
|
6158
|
+
version: 1,
|
|
6159
|
+
runId: derivedRunId,
|
|
6160
|
+
phase: 'run.completed',
|
|
6161
|
+
route: 'generated_answer',
|
|
6162
|
+
plan: sourceRun.plan,
|
|
6163
|
+
steps: [repairStep],
|
|
6164
|
+
artifacts: [artifact],
|
|
6165
|
+
evaluations: [evaluation],
|
|
6166
|
+
},
|
|
6167
|
+
derivation: {
|
|
6168
|
+
version: 1,
|
|
6169
|
+
kind: 'analytical_repair',
|
|
6170
|
+
sourceRunId: sourceRun.id,
|
|
6171
|
+
...(sourceFailure?.failureId ? { sourceFailureId: sourceFailure.failureId } : {}),
|
|
6172
|
+
attempt: 1,
|
|
6173
|
+
},
|
|
6174
|
+
};
|
|
6175
|
+
await agentRunStore.save(derivedRun);
|
|
6176
|
+
recordConversationTurn(threadId ? getConversationStore() : null, threadId, derivedRun);
|
|
6177
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6178
|
+
res.end(serializeJSON({ run: derivedRun }));
|
|
6179
|
+
return;
|
|
6180
|
+
}
|
|
5738
6181
|
if (req.method === 'POST' && /^\/api\/agent-runs\/[^/]+\/analytical-repair$/.test(path)) {
|
|
5739
6182
|
const match = path.match(/^\/api\/agent-runs\/([^/]+)\/analytical-repair$/);
|
|
5740
6183
|
const id = decodeURIComponent(match?.[1] ?? '');
|
|
@@ -10062,10 +10505,20 @@ export async function startLocalServer(opts) {
|
|
|
10062
10505
|
// of the same 7,500+ object catalog while keeping the route compatible
|
|
10063
10506
|
// for older clients.
|
|
10064
10507
|
const includeSemantic = url.searchParams.get('includeSemantic') !== 'false';
|
|
10508
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, cfg);
|
|
10509
|
+
let databaseTree = [];
|
|
10510
|
+
if (cfg.manifestVersion === 3 && cfg.modeling?.mode === 'dbt-first' && dbtManifestPath) {
|
|
10511
|
+
try {
|
|
10512
|
+
databaseTree = buildDbtDatabaseSchemaTree(projectSnapshot().manifest, 25, physicalDbtRelationIds(dbtManifestPath));
|
|
10513
|
+
}
|
|
10514
|
+
catch {
|
|
10515
|
+
// Keep the rest of Block Studio available while dbt is rebuilt.
|
|
10516
|
+
}
|
|
10517
|
+
}
|
|
10065
10518
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10066
10519
|
res.end(serializeJSON({
|
|
10067
10520
|
semanticTree: includeSemantic && semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
10068
|
-
databaseTree
|
|
10521
|
+
databaseTree,
|
|
10069
10522
|
connection: {
|
|
10070
10523
|
default: defaultKey,
|
|
10071
10524
|
current: defaultKey,
|
|
@@ -11836,13 +12289,17 @@ export async function startLocalServer(opts) {
|
|
|
11836
12289
|
// GET /api/describe-table?table=schema.table — returns columns for a specific table
|
|
11837
12290
|
if (req.method === 'GET' && path === '/api/describe-table') {
|
|
11838
12291
|
try {
|
|
11839
|
-
const
|
|
11840
|
-
const
|
|
11841
|
-
|
|
12292
|
+
const requestedRelation = url.searchParams.get('relation')?.trim() ?? '';
|
|
12293
|
+
const legacyTable = url.searchParams.get('table')?.trim() ?? '';
|
|
12294
|
+
const legacySchema = url.searchParams.get('schema')?.trim() ?? '';
|
|
12295
|
+
const relationParts = splitQualifiedRelationIdentifier(requestedRelation || [legacySchema, legacyTable].filter(Boolean).join('.'));
|
|
12296
|
+
if (!relationParts?.length) {
|
|
11842
12297
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
11843
|
-
res.end(serializeJSON({ error: '
|
|
12298
|
+
res.end(serializeJSON({ error: 'A safe dbt relation is required.' }));
|
|
11844
12299
|
return;
|
|
11845
12300
|
}
|
|
12301
|
+
const tablePath = relationParts.at(-1);
|
|
12302
|
+
const schemaName = relationParts.length > 1 ? relationParts.slice(0, -1).join('.') : undefined;
|
|
11846
12303
|
// Try connector.listColumns() first
|
|
11847
12304
|
let columns = [];
|
|
11848
12305
|
const activeConnection = connection;
|
|
@@ -11851,7 +12308,11 @@ export async function startLocalServer(opts) {
|
|
|
11851
12308
|
throw new Error('No active connection');
|
|
11852
12309
|
const connector = await executor.getConnector(activeConnection);
|
|
11853
12310
|
if (typeof connector.listColumns === 'function') {
|
|
11854
|
-
|
|
12311
|
+
// Connector contracts accept one schema plus one table. For a
|
|
12312
|
+
// three-part dbt relation, the active connection owns the database
|
|
12313
|
+
// and the final namespace segment is the schema.
|
|
12314
|
+
const connectorSchema = schemaName?.split('.').filter(Boolean).at(-1);
|
|
12315
|
+
const rawCols = await connector.listColumns(connectorSchema, tablePath);
|
|
11855
12316
|
columns = rawCols.map((c) => ({ name: c.name, type: c.dataType }));
|
|
11856
12317
|
}
|
|
11857
12318
|
}
|
|
@@ -11863,17 +12324,15 @@ export async function startLocalServer(opts) {
|
|
|
11863
12324
|
try {
|
|
11864
12325
|
const isFile = /\.(csv|parquet|json)$/i.test(tablePath) || tablePath.startsWith('data/');
|
|
11865
12326
|
const safePath = tablePath.replace(/'/g, "''");
|
|
11866
|
-
|
|
12327
|
+
if (!activeConnection)
|
|
12328
|
+
throw new Error('No active connection');
|
|
12329
|
+
const dialect = getDialect(activeConnection.driver);
|
|
12330
|
+
const qualifiedIdentifier = relationParts.map((part) => dialect.quoteIdentifier(part)).join('.');
|
|
11867
12331
|
const sql = isFile
|
|
11868
12332
|
? `DESCRIBE SELECT * FROM read_csv_auto('${safePath}') LIMIT 0`
|
|
11869
12333
|
: `DESCRIBE ${qualifiedIdentifier}`;
|
|
11870
|
-
if (!activeConnection)
|
|
11871
|
-
throw new Error('No active connection');
|
|
11872
12334
|
const result = await executor.executeQuery(sql, [], {}, activeConnection);
|
|
11873
|
-
columns = result.rows
|
|
11874
|
-
name: String(row['column_name'] ?? row['Field'] ?? ''),
|
|
11875
|
-
type: String(row['column_type'] ?? row['Type'] ?? ''),
|
|
11876
|
-
}));
|
|
12335
|
+
columns = schemaColumnsFromDescribeRows(result.rows);
|
|
11877
12336
|
}
|
|
11878
12337
|
catch {
|
|
11879
12338
|
// empty columns
|
|
@@ -11962,6 +12421,187 @@ export async function startLocalServer(opts) {
|
|
|
11962
12421
|
}
|
|
11963
12422
|
return;
|
|
11964
12423
|
}
|
|
12424
|
+
/**
|
|
12425
|
+
* Repair an editable notebook cell without starting Notebook AI research.
|
|
12426
|
+
* The returned source is applied only to the in-memory notebook cell by the
|
|
12427
|
+
* client; this endpoint never writes a block/domain/dbt source file.
|
|
12428
|
+
*/
|
|
12429
|
+
if (req.method === 'POST' && path === '/api/notebook/repair-execution') {
|
|
12430
|
+
let body = {};
|
|
12431
|
+
let execContext = null;
|
|
12432
|
+
const start = Date.now();
|
|
12433
|
+
try {
|
|
12434
|
+
body = await readJSON(req);
|
|
12435
|
+
execContext = notebookExecutionContext(body.executionContext);
|
|
12436
|
+
const cell = normalizeNotebookCell(body.cell);
|
|
12437
|
+
if (!cell || (cell.type !== 'sql' && cell.type !== 'dql')) {
|
|
12438
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12439
|
+
res.end(serializeJSON({ error: 'An editable SQL or DQL notebook cell is required.' }));
|
|
12440
|
+
return;
|
|
12441
|
+
}
|
|
12442
|
+
if (cell.type === 'dql' && /^\s*@block\(/i.test(cell.source)) {
|
|
12443
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12444
|
+
res.end(serializeJSON({ error: 'Customize the bound block as a notebook copy before repairing it.' }));
|
|
12445
|
+
return;
|
|
12446
|
+
}
|
|
12447
|
+
const embeddedSourceSql = notebookCellRepairSql(cell);
|
|
12448
|
+
if (!embeddedSourceSql) {
|
|
12449
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12450
|
+
res.end(serializeJSON({ error: 'Background repair supports SQL cells and DQL cells with an embedded query. Edit this semantic DQL definition directly.' }));
|
|
12451
|
+
return;
|
|
12452
|
+
}
|
|
12453
|
+
if (cell.type === 'sql' && /\{\{[^}]+\}\}/.test(cell.source)) {
|
|
12454
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12455
|
+
res.end(serializeJSON({ error: 'Run or repair the referenced upstream cell before repairing this dependent query.' }));
|
|
12456
|
+
return;
|
|
12457
|
+
}
|
|
12458
|
+
const targetConnection = await resolveExecutionConnection(body);
|
|
12459
|
+
const submittedError = typeof body.error === 'string' ? body.error.trim() : '';
|
|
12460
|
+
const submittedFailure = body.failure && typeof body.failure === 'object' && !Array.isArray(body.failure)
|
|
12461
|
+
? body.failure
|
|
12462
|
+
: undefined;
|
|
12463
|
+
const failureCode = typeof submittedFailure?.code === 'string' ? submittedFailure.code : undefined;
|
|
12464
|
+
if (!notebookFailureAllowsBackgroundRepair({ code: failureCode, message: submittedError })) {
|
|
12465
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12466
|
+
res.end(serializeJSON({ error: 'This failure needs a data, access, parameter, or notebook dependency change and cannot be auto-fixed safely.' }));
|
|
12467
|
+
return;
|
|
12468
|
+
}
|
|
12469
|
+
const retainedFailure = submittedError
|
|
12470
|
+
? normalizeWarehouseSqlFailure({ code: failureCode, message: submittedError }, targetConnection.driver)
|
|
12471
|
+
: undefined;
|
|
12472
|
+
if (!analyticalFailureAllowsDeterministicRetry(retainedFailure)) {
|
|
12473
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12474
|
+
res.end(serializeJSON({
|
|
12475
|
+
error: 'This failure requires authorized access or a policy change and cannot be repaired in the background.',
|
|
12476
|
+
warehouseFailure: retainedFailure,
|
|
12477
|
+
}));
|
|
12478
|
+
return;
|
|
12479
|
+
}
|
|
12480
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
12481
|
+
? body.parameters
|
|
12482
|
+
: {};
|
|
12483
|
+
const invocation = cell.type === 'dql'
|
|
12484
|
+
? prepareBlockInvocation({ source: cell.source, parameters, surface: 'notebook' })
|
|
12485
|
+
: null;
|
|
12486
|
+
if (invocation?.errors.length) {
|
|
12487
|
+
throw boundedRepairError(invocation.errors.join(' '), 409);
|
|
12488
|
+
}
|
|
12489
|
+
if (invocation?.unresolvedParameters.length) {
|
|
12490
|
+
throw boundedRepairError(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'} before retrying: ${invocation.unresolvedParameters.join(', ')}.`, 409);
|
|
12491
|
+
}
|
|
12492
|
+
const plan = cell.type === 'dql'
|
|
12493
|
+
? buildExecutionPlan(cell, {
|
|
12494
|
+
semanticLayer,
|
|
12495
|
+
driver: targetConnection.driver,
|
|
12496
|
+
parameters: invocation?.values,
|
|
12497
|
+
})
|
|
12498
|
+
: null;
|
|
12499
|
+
const attemptedSql = typeof body.attemptedSql === 'string' && body.attemptedSql.trim()
|
|
12500
|
+
? body.attemptedSql.trim()
|
|
12501
|
+
: embeddedSourceSql;
|
|
12502
|
+
const sourceSql = plan?.sql ?? attemptedSql;
|
|
12503
|
+
const sqlParams = plan?.sqlParams ?? [];
|
|
12504
|
+
const variables = {
|
|
12505
|
+
...(plan?.variables ?? {}),
|
|
12506
|
+
...(invocation?.values ?? {}),
|
|
12507
|
+
};
|
|
12508
|
+
const executionTarget = executionTargetDescriptor(body);
|
|
12509
|
+
const targetConnectionName = executionTarget.target === 'connection'
|
|
12510
|
+
? executionTarget.connectionName
|
|
12511
|
+
: executionTarget.target === 'local'
|
|
12512
|
+
? 'local'
|
|
12513
|
+
: projectConfig.defaultConnectionName;
|
|
12514
|
+
const question = [
|
|
12515
|
+
`Repair notebook cell ${cell.title?.trim() || cell.id}.`,
|
|
12516
|
+
submittedError ? `Failure: ${submittedError.slice(0, 1_000)}` : '',
|
|
12517
|
+
].filter(Boolean).join(' ');
|
|
12518
|
+
const repaired = await executeBoundedSqlRepair({
|
|
12519
|
+
question,
|
|
12520
|
+
sourceSql,
|
|
12521
|
+
targetConnection,
|
|
12522
|
+
targetConnectionName,
|
|
12523
|
+
sqlParams,
|
|
12524
|
+
variables,
|
|
12525
|
+
});
|
|
12526
|
+
let repairedSource;
|
|
12527
|
+
if (cell.type === 'dql') {
|
|
12528
|
+
const originalQuery = notebookCellRepairSql(cell);
|
|
12529
|
+
const repairedQuery = repaired.repairMode === 'deterministic'
|
|
12530
|
+
? applyNotebookRepairRewrites(originalQuery, repaired.rewrites)
|
|
12531
|
+
: restoreNotebookDqlParameterInterpolations(repaired.repairedSql, sqlParams);
|
|
12532
|
+
repairedSource = replaceNotebookDqlQueryForRepair(cell.source, repairedQuery);
|
|
12533
|
+
}
|
|
12534
|
+
else {
|
|
12535
|
+
repairedSource = repaired.repairedSql;
|
|
12536
|
+
}
|
|
12537
|
+
if (!repairedSource) {
|
|
12538
|
+
throw boundedRepairError('DQL could not safely replace the embedded query in this cell.', 422);
|
|
12539
|
+
}
|
|
12540
|
+
// Prove the repaired DQL wrapper still compiles. SQL execution above
|
|
12541
|
+
// already proved the corrected statement on the selected target.
|
|
12542
|
+
if (cell.type === 'dql') {
|
|
12543
|
+
const validationPlan = buildExecutionPlan({ ...cell, source: repairedSource }, {
|
|
12544
|
+
semanticLayer,
|
|
12545
|
+
driver: targetConnection.driver,
|
|
12546
|
+
parameters: invocation?.values,
|
|
12547
|
+
});
|
|
12548
|
+
if (!validationPlan?.sql) {
|
|
12549
|
+
throw boundedRepairError('The repaired SQL ran, but the DQL wrapper no longer compiles.', 422);
|
|
12550
|
+
}
|
|
12551
|
+
const beforeParameters = sqlParams.map((parameter) => `${parameter.position}:${parameter.name}`);
|
|
12552
|
+
const afterParameters = validationPlan.sqlParams.map((parameter) => `${parameter.position}:${parameter.name}`);
|
|
12553
|
+
if (beforeParameters.join('|') !== afterParameters.join('|')) {
|
|
12554
|
+
throw boundedRepairError('The repaired DQL changed the cell parameter contract, so the update was stopped.', 422);
|
|
12555
|
+
}
|
|
12556
|
+
}
|
|
12557
|
+
if (execContext?.notebookPath) {
|
|
12558
|
+
recordNotebookQueryRun(projectRoot, {
|
|
12559
|
+
notebookPath: execContext.notebookPath,
|
|
12560
|
+
cellId: execContext.cellId ?? cell.id,
|
|
12561
|
+
cellName: execContext.cellName ?? cell.title,
|
|
12562
|
+
source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_sql_cell'),
|
|
12563
|
+
status: 'success',
|
|
12564
|
+
rowCount: repaired.result.rowCount ?? repaired.result.rows.length,
|
|
12565
|
+
durationMs: Date.now() - start,
|
|
12566
|
+
sql: repaired.repairedSql,
|
|
12567
|
+
});
|
|
12568
|
+
}
|
|
12569
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12570
|
+
res.end(serializeJSON({
|
|
12571
|
+
ok: true,
|
|
12572
|
+
source: repairedSource,
|
|
12573
|
+
repairedSql: repaired.repairedSql,
|
|
12574
|
+
repairMode: repaired.repairMode,
|
|
12575
|
+
result: repaired.result,
|
|
12576
|
+
executionTarget,
|
|
12577
|
+
executionContext: notebookExecutionIdentity(execContext, cell.id),
|
|
12578
|
+
compiledSql: repaired.repairedSql,
|
|
12579
|
+
executedSql: repaired.result.sql ?? repaired.repairedSql,
|
|
12580
|
+
executionReceipt: repaired.result.executionReceipt,
|
|
12581
|
+
}));
|
|
12582
|
+
}
|
|
12583
|
+
catch (error) {
|
|
12584
|
+
const repairError = error;
|
|
12585
|
+
if (execContext?.notebookPath) {
|
|
12586
|
+
recordNotebookQueryRun(projectRoot, {
|
|
12587
|
+
notebookPath: execContext.notebookPath,
|
|
12588
|
+
cellId: execContext.cellId,
|
|
12589
|
+
cellName: execContext.cellName,
|
|
12590
|
+
source: execContext.source ?? 'notebook_cell_repair',
|
|
12591
|
+
status: 'error',
|
|
12592
|
+
durationMs: Date.now() - start,
|
|
12593
|
+
errorCode: repairError.warehouseFailure?.category ?? repairError.message,
|
|
12594
|
+
sql: typeof body.attemptedSql === 'string' ? body.attemptedSql : undefined,
|
|
12595
|
+
});
|
|
12596
|
+
}
|
|
12597
|
+
res.writeHead(repairError.status ?? 500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12598
|
+
res.end(serializeJSON({
|
|
12599
|
+
error: repairError.message,
|
|
12600
|
+
...(repairError.warehouseFailure ? { warehouseFailure: repairError.warehouseFailure } : {}),
|
|
12601
|
+
}));
|
|
12602
|
+
}
|
|
12603
|
+
return;
|
|
12604
|
+
}
|
|
11965
12605
|
if (req.method === 'POST' && path === '/api/query') {
|
|
11966
12606
|
let body;
|
|
11967
12607
|
let execContext = null;
|
|
@@ -12005,7 +12645,15 @@ export async function startLocalServer(opts) {
|
|
|
12005
12645
|
}));
|
|
12006
12646
|
return;
|
|
12007
12647
|
}
|
|
12008
|
-
const
|
|
12648
|
+
const preparation = await prepareAnalyticalExecutionSql({
|
|
12649
|
+
sql: semantic.sql,
|
|
12650
|
+
subject: 'Notebook query',
|
|
12651
|
+
executor,
|
|
12652
|
+
connection: executionConnection,
|
|
12653
|
+
projectRoot,
|
|
12654
|
+
projectConfig,
|
|
12655
|
+
});
|
|
12656
|
+
const prepared = { sql: preparation.preparedSql, connection: preparation.connection };
|
|
12009
12657
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
12010
12658
|
const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
|
|
12011
12659
|
assertAppAccess({ app, domain, level: 'execute' });
|
|
@@ -12033,8 +12681,8 @@ export async function startLocalServer(opts) {
|
|
|
12033
12681
|
...normalized,
|
|
12034
12682
|
executionContext: notebookExecutionIdentity(execContext),
|
|
12035
12683
|
executionTarget: executionTargetDescriptor(body),
|
|
12036
|
-
compiledSql:
|
|
12037
|
-
executedSql:
|
|
12684
|
+
compiledSql: preparation.decodedSql,
|
|
12685
|
+
executedSql: preparation.executedSql,
|
|
12038
12686
|
});
|
|
12039
12687
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
12040
12688
|
res.end(payload);
|
|
@@ -12705,6 +13353,11 @@ export async function startLocalServer(opts) {
|
|
|
12705
13353
|
if (req.method === 'POST' && path === '/api/dql/artifacts/execute') {
|
|
12706
13354
|
try {
|
|
12707
13355
|
const body = await readJSON(req);
|
|
13356
|
+
const targetConnection = await resolveExecutionConnection(body);
|
|
13357
|
+
const targetDescriptor = executionTargetDescriptor(body);
|
|
13358
|
+
const targetConnectionName = targetDescriptor.target === 'connection'
|
|
13359
|
+
? targetDescriptor.connectionName
|
|
13360
|
+
: 'local';
|
|
12708
13361
|
const requestedArtifact = normalizeDqlArtifactReference(body.artifact);
|
|
12709
13362
|
const requestedBlockName = typeof body.blockName === 'string' ? body.blockName.trim() : '';
|
|
12710
13363
|
let source = requestedArtifact?.source ?? '';
|
|
@@ -12762,7 +13415,7 @@ export async function startLocalServer(opts) {
|
|
|
12762
13415
|
parameters,
|
|
12763
13416
|
question: typeof body.question === 'string' ? body.question : undefined,
|
|
12764
13417
|
rowLimit: requestedArtifact?.limit,
|
|
12765
|
-
});
|
|
13418
|
+
}, targetConnection, targetConnectionName);
|
|
12766
13419
|
const actualReceipt = result.executionReceipt;
|
|
12767
13420
|
// Same reasoning: if the canonical block was reloaded, its compiled SQL
|
|
12768
13421
|
// is EXPECTED to differ, so this determinism check would fire on a
|
|
@@ -12917,26 +13570,16 @@ export async function startLocalServer(opts) {
|
|
|
12917
13570
|
res.end(serializeJSON({ cellType: cell.type, result: null }));
|
|
12918
13571
|
return;
|
|
12919
13572
|
}
|
|
12920
|
-
|
|
12921
|
-
|
|
12922
|
-
|
|
12923
|
-
|
|
12924
|
-
|
|
12925
|
-
|
|
12926
|
-
|
|
12927
|
-
|
|
12928
|
-
|
|
12929
|
-
|
|
12930
|
-
decodedSql = bareResolution.sql;
|
|
12931
|
-
}
|
|
12932
|
-
const internalRelationError = internalDqlRelationIdValidationError(decodedSql, cell.type === 'dql' ? 'DQL query' : 'Notebook query',
|
|
12933
|
-
// A probe that came back empty is PROOF the relation is not visible
|
|
12934
|
-
// to this connection, so the refusal states that instead of sending
|
|
12935
|
-
// the user to an AI repair that cannot conjure a missing table.
|
|
12936
|
-
bareResolution ? { probed: true, ambiguous: bareResolution.ambiguous } : undefined);
|
|
12937
|
-
if (internalRelationError)
|
|
12938
|
-
throw new DqlInternalRelationIdError(internalRelationError);
|
|
12939
|
-
const prepared = prepareLocalExecution(decodedSql, cellConnection, projectRoot, projectConfig);
|
|
13573
|
+
const preparation = await prepareAnalyticalExecutionSql({
|
|
13574
|
+
sql: executableSql,
|
|
13575
|
+
subject: cell.type === 'dql' ? 'DQL query' : 'Notebook query',
|
|
13576
|
+
executor,
|
|
13577
|
+
connection: cellConnection,
|
|
13578
|
+
projectRoot,
|
|
13579
|
+
projectConfig,
|
|
13580
|
+
});
|
|
13581
|
+
const decodedSql = preparation.decodedSql;
|
|
13582
|
+
const prepared = { sql: preparation.preparedSql, connection: preparation.connection };
|
|
12940
13583
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
12941
13584
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
12942
13585
|
const semanticExecution = semanticCompose?.compiled
|
|
@@ -15281,6 +15924,60 @@ export function prepareLocalExecution(sql, connection, projectRoot, projectConfi
|
|
|
15281
15924
|
connection: normalizedConnection,
|
|
15282
15925
|
};
|
|
15283
15926
|
}
|
|
15927
|
+
/**
|
|
15928
|
+
* One host-owned SQL preparation boundary shared by Ask and Notebook.
|
|
15929
|
+
*
|
|
15930
|
+
* Surface-specific compilation happens before this function and surface policy
|
|
15931
|
+
* (for example exploratory join proof) may run around it, but no analytical
|
|
15932
|
+
* entrypoint may decode internal IDs, resolve dbt paths, enforce read-only
|
|
15933
|
+
* execution, or append a preview bound differently anymore.
|
|
15934
|
+
*
|
|
15935
|
+
* Acceptance: API-003, API-006, API-007, EXP-001, E2E-014.
|
|
15936
|
+
*/
|
|
15937
|
+
export async function prepareAnalyticalExecutionSql(input) {
|
|
15938
|
+
const sourceSql = input.sql.trim();
|
|
15939
|
+
const rewrites = [];
|
|
15940
|
+
let decodedSql = resolveInternalRelationIds(sourceSql, EMPTY_SCHEMA_GROUNDING, 'qualified', rewrites);
|
|
15941
|
+
let bareResolution = null;
|
|
15942
|
+
if (internalDqlRelationIdValidationError(decodedSql, 'x')) {
|
|
15943
|
+
bareResolution = await resolveBareInternalRelationIds(decodedSql, input.executor, input.connection);
|
|
15944
|
+
decodedSql = bareResolution.sql;
|
|
15945
|
+
rewrites.push(...bareResolution.resolved);
|
|
15946
|
+
}
|
|
15947
|
+
const internalRelationError = internalDqlRelationIdValidationError(decodedSql, input.subject, bareResolution
|
|
15948
|
+
? { probed: true, ambiguous: bareResolution.ambiguous }
|
|
15949
|
+
: undefined);
|
|
15950
|
+
if (internalRelationError) {
|
|
15951
|
+
throw tagAnalyticalError(new DqlInternalRelationIdError(internalRelationError), {
|
|
15952
|
+
origin: bareResolution?.ambiguous.length ? 'ambiguity' : 'retrieval_gap',
|
|
15953
|
+
stage: 'grounding',
|
|
15954
|
+
code: bareResolution?.ambiguous.length ? 'ambiguous_relation' : 'unknown_relation',
|
|
15955
|
+
});
|
|
15956
|
+
}
|
|
15957
|
+
const prepared = prepareLocalExecution(decodedSql, input.connection, input.projectRoot, input.projectConfig);
|
|
15958
|
+
if (input.enforceReadOnly) {
|
|
15959
|
+
const readOnlyError = readOnlySqlValidationError(prepared.sql, input.subject, input.connection.driver);
|
|
15960
|
+
if (readOnlyError) {
|
|
15961
|
+
throw analyticalError(readOnlyError, {
|
|
15962
|
+
origin: 'governance_gate',
|
|
15963
|
+
stage: 'validation',
|
|
15964
|
+
code: 'unsafe_sql',
|
|
15965
|
+
});
|
|
15966
|
+
}
|
|
15967
|
+
}
|
|
15968
|
+
const rowBound = input.rowLimit === undefined
|
|
15969
|
+
? undefined
|
|
15970
|
+
: buildRowBoundedSql(prepared.sql, input.rowLimit, input.connection.driver);
|
|
15971
|
+
return {
|
|
15972
|
+
sourceSql,
|
|
15973
|
+
decodedSql,
|
|
15974
|
+
preparedSql: prepared.sql,
|
|
15975
|
+
executedSql: rowBound?.sql ?? prepared.sql,
|
|
15976
|
+
connection: prepared.connection,
|
|
15977
|
+
rewrites,
|
|
15978
|
+
...(rowBound ? { rowBound } : {}),
|
|
15979
|
+
};
|
|
15980
|
+
}
|
|
15284
15981
|
export function dashboardRuntimeVariables(dashboard, overrides = {}) {
|
|
15285
15982
|
const variables = {};
|
|
15286
15983
|
for (const param of dashboard.params ?? []) {
|
|
@@ -16452,6 +17149,55 @@ function normalizeNotebookCell(value) {
|
|
|
16452
17149
|
config: candidate.config,
|
|
16453
17150
|
};
|
|
16454
17151
|
}
|
|
17152
|
+
export function notebookCellRepairSql(cell) {
|
|
17153
|
+
if (cell.type === 'sql')
|
|
17154
|
+
return cell.source.trim() || undefined;
|
|
17155
|
+
if (cell.type !== 'dql')
|
|
17156
|
+
return undefined;
|
|
17157
|
+
const match = /\bquery\s*=\s*"""([\s\S]*?)"""/i.exec(cell.source);
|
|
17158
|
+
return match?.[1]?.trim() || undefined;
|
|
17159
|
+
}
|
|
17160
|
+
export function replaceNotebookDqlQueryForRepair(source, sql) {
|
|
17161
|
+
const queryField = /\b(query\s*=\s*""")([\s\S]*?)(""")/i;
|
|
17162
|
+
if (!queryField.test(source) || !sql.trim())
|
|
17163
|
+
return undefined;
|
|
17164
|
+
return source.replace(queryField, (_match, open, _body, close) => (`${open}\n${sql.trim()}\n ${close}`));
|
|
17165
|
+
}
|
|
17166
|
+
const NOTEBOOK_BACKGROUND_REPAIR_BLOCKED_CODES = new Set([
|
|
17167
|
+
'AMBIGUOUS_NOTEBOOK_DEPENDENCY',
|
|
17168
|
+
'CROSS_ENGINE_JOIN_REQUIRED',
|
|
17169
|
+
'EXECUTION_TARGET_MISMATCH',
|
|
17170
|
+
'SEMANTIC_SOURCE_DRIFT',
|
|
17171
|
+
'STALE_CELL_EXECUTION',
|
|
17172
|
+
'UPSTREAM_RESULT_UNAVAILABLE',
|
|
17173
|
+
'UNAUTHORIZED',
|
|
17174
|
+
]);
|
|
17175
|
+
/**
|
|
17176
|
+
* Notebook background repair is deliberately narrower than conversational
|
|
17177
|
+
* repair. Access, policy, target, parameter, and dependency failures require a
|
|
17178
|
+
* user decision; only query-shape failures may enter the one-attempt loop.
|
|
17179
|
+
* Acceptance: API-006, API-007, UI-012, UI-013, SEC-004, E2E-014.
|
|
17180
|
+
*/
|
|
17181
|
+
export function notebookFailureAllowsBackgroundRepair(input) {
|
|
17182
|
+
const code = input.code?.trim().toUpperCase();
|
|
17183
|
+
const message = input.message?.trim() ?? '';
|
|
17184
|
+
if (!message)
|
|
17185
|
+
return false;
|
|
17186
|
+
if (code && NOTEBOOK_BACKGROUND_REPAIR_BLOCKED_CODES.has(code))
|
|
17187
|
+
return false;
|
|
17188
|
+
return !/(?:permission|not authorized|unauthori[sz]ed|access denied|authentication|credential|policy|unsafe|read[- ]only|cancel(?:led|ed)|required parameters?|provide required|cross[- ]engine|upstream (?:cell|result)|notebook dependency|target mismatch|source drift|matched more than one schema)/i.test(message);
|
|
17189
|
+
}
|
|
17190
|
+
export function applyNotebookRepairRewrites(sql, rewrites) {
|
|
17191
|
+
return rewrites.reduce((current, rewrite) => {
|
|
17192
|
+
const pattern = rewrite.from.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
17193
|
+
return current.replace(new RegExp(pattern, 'g'), rewrite.to);
|
|
17194
|
+
}, sql);
|
|
17195
|
+
}
|
|
17196
|
+
export function restoreNotebookDqlParameterInterpolations(sql, parameters) {
|
|
17197
|
+
return [...parameters]
|
|
17198
|
+
.sort((left, right) => right.position - left.position)
|
|
17199
|
+
.reduce((current, parameter) => current.replace(new RegExp(`\\$${parameter.position}\\b`, 'g'), `\${${parameter.name}}`), sql);
|
|
17200
|
+
}
|
|
16455
17201
|
function resolveNotebookBlockReferenceCell(cell, projectRoot) {
|
|
16456
17202
|
if (cell.type !== 'dql')
|
|
16457
17203
|
return { cell };
|
|
@@ -16645,6 +17391,68 @@ function readUserPrefs(userPrefsPath) {
|
|
|
16645
17391
|
function writeUserPrefs(userPrefsPath, prefs) {
|
|
16646
17392
|
writeFileSync(userPrefsPath, JSON.stringify(prefs, null, 2) + '\n', 'utf-8');
|
|
16647
17393
|
}
|
|
17394
|
+
/** Normalize DESCRIBE output across Snowflake, DuckDB, and PostgreSQL shapes. */
|
|
17395
|
+
export function schemaColumnsFromDescribeRows(rows) {
|
|
17396
|
+
return rows.map((row) => ({
|
|
17397
|
+
name: String(row['column_name'] ?? row['COLUMN_NAME'] ?? row['Field'] ?? row['FIELD'] ?? row['name'] ?? row['NAME'] ?? ''),
|
|
17398
|
+
type: String(row['column_type'] ?? row['COLUMN_TYPE'] ?? row['data_type'] ?? row['DATA_TYPE'] ?? row['Type'] ?? row['TYPE'] ?? row['type'] ?? ''),
|
|
17399
|
+
})).filter((column) => Boolean(column.name));
|
|
17400
|
+
}
|
|
17401
|
+
/** Parse dbt relation strings without letting relation text become SQL syntax. */
|
|
17402
|
+
export function splitQualifiedRelationIdentifier(value) {
|
|
17403
|
+
const rawParts = [];
|
|
17404
|
+
let current = '';
|
|
17405
|
+
let quote = null;
|
|
17406
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
17407
|
+
const char = value[index];
|
|
17408
|
+
if (quote) {
|
|
17409
|
+
current += char;
|
|
17410
|
+
if (char === quote) {
|
|
17411
|
+
const next = value[index + 1];
|
|
17412
|
+
if ((quote === '"' || quote === '`') && next === quote) {
|
|
17413
|
+
current += next;
|
|
17414
|
+
index += 1;
|
|
17415
|
+
}
|
|
17416
|
+
else {
|
|
17417
|
+
quote = null;
|
|
17418
|
+
}
|
|
17419
|
+
}
|
|
17420
|
+
continue;
|
|
17421
|
+
}
|
|
17422
|
+
if (char === '"' || char === '`') {
|
|
17423
|
+
quote = char;
|
|
17424
|
+
current += char;
|
|
17425
|
+
continue;
|
|
17426
|
+
}
|
|
17427
|
+
if (char === '[') {
|
|
17428
|
+
quote = ']';
|
|
17429
|
+
current += char;
|
|
17430
|
+
continue;
|
|
17431
|
+
}
|
|
17432
|
+
if (char === '.') {
|
|
17433
|
+
rawParts.push(current);
|
|
17434
|
+
current = '';
|
|
17435
|
+
continue;
|
|
17436
|
+
}
|
|
17437
|
+
current += char;
|
|
17438
|
+
}
|
|
17439
|
+
if (quote)
|
|
17440
|
+
return null;
|
|
17441
|
+
rawParts.push(current);
|
|
17442
|
+
if (rawParts.length === 0 || rawParts.length > 4)
|
|
17443
|
+
return null;
|
|
17444
|
+
const decoded = rawParts.map((raw) => {
|
|
17445
|
+
const token = raw.trim();
|
|
17446
|
+
if (/^"(?:[^"]|"")+"$/.test(token))
|
|
17447
|
+
return token.slice(1, -1).replace(/""/g, '"');
|
|
17448
|
+
if (/^`(?:[^`]|``)+`$/.test(token))
|
|
17449
|
+
return token.slice(1, -1).replace(/``/g, '`');
|
|
17450
|
+
if (/^\[[^\]]+\]$/.test(token))
|
|
17451
|
+
return token.slice(1, -1).replace(/\]\]/g, ']');
|
|
17452
|
+
return /^[A-Za-z_][A-Za-z0-9_$]*$/.test(token) ? token : null;
|
|
17453
|
+
});
|
|
17454
|
+
return decoded.every((part) => Boolean(part)) ? decoded : null;
|
|
17455
|
+
}
|
|
16648
17456
|
async function introspectSchema(executor, connection, options = {}) {
|
|
16649
17457
|
let tables = [];
|
|
16650
17458
|
const columnsByPath = new Map();
|
|
@@ -16699,77 +17507,93 @@ async function introspectSchema(executor, connection, options = {}) {
|
|
|
16699
17507
|
}
|
|
16700
17508
|
return { tables, columnsByPath };
|
|
16701
17509
|
}
|
|
16702
|
-
|
|
16703
|
-
|
|
16704
|
-
|
|
16705
|
-
|
|
16706
|
-
|
|
16707
|
-
|
|
16708
|
-
|
|
16709
|
-
|
|
16710
|
-
|
|
16711
|
-
|
|
16712
|
-
|
|
16713
|
-
|
|
16714
|
-
|
|
16715
|
-
const
|
|
16716
|
-
|
|
16717
|
-
|
|
16718
|
-
|
|
16719
|
-
|
|
16720
|
-
|
|
16721
|
-
|
|
16722
|
-
|
|
16723
|
-
|
|
16724
|
-
|
|
16725
|
-
|
|
16726
|
-
|
|
16727
|
-
|
|
16728
|
-
|
|
16729
|
-
|
|
16730
|
-
|
|
16731
|
-
|
|
16732
|
-
|
|
16733
|
-
// Eagerly resolve file columns via DuckDB DESCRIBE
|
|
16734
|
-
if (dataFiles.length > 0) {
|
|
16735
|
-
const fileChildren = [];
|
|
16736
|
-
for (const file of dataFiles) {
|
|
16737
|
-
let columns = [];
|
|
16738
|
-
if (connection) {
|
|
16739
|
-
try {
|
|
16740
|
-
const ext = file.name.split('.').pop()?.toLowerCase();
|
|
16741
|
-
const readFn = ext === 'parquet' ? 'read_parquet' : ext === 'json' ? 'read_json_auto' : 'read_csv_auto';
|
|
16742
|
-
const descResult = await executor.executeQuery(`DESCRIBE SELECT * FROM ${readFn}('${file.path.replace(/'/g, "''")}') LIMIT 0`, [], {}, connection);
|
|
16743
|
-
columns = descResult.rows.map((row) => ({
|
|
16744
|
-
id: `db-column:${file.path}:${String(row['column_name'] ?? '')}`,
|
|
16745
|
-
label: String(row['column_name'] ?? ''),
|
|
16746
|
-
kind: 'column',
|
|
16747
|
-
path: file.path,
|
|
16748
|
-
type: String(row['column_type'] ?? ''),
|
|
16749
|
-
}));
|
|
16750
|
-
}
|
|
16751
|
-
catch {
|
|
16752
|
-
// file column discovery failed — empty children is fine
|
|
16753
|
-
}
|
|
16754
|
-
}
|
|
16755
|
-
fileChildren.push({
|
|
16756
|
-
id: `db-table:${file.path}`,
|
|
16757
|
-
label: file.name,
|
|
16758
|
-
kind: 'table',
|
|
16759
|
-
path: file.path,
|
|
16760
|
-
type: 'FILE',
|
|
16761
|
-
children: columns,
|
|
16762
|
-
});
|
|
17510
|
+
const dbtPhysicalRelationCache = new Map();
|
|
17511
|
+
/**
|
|
17512
|
+
* Display-only physical relation filter sourced from the raw dbt artifact.
|
|
17513
|
+
* This deliberately does not mutate DQLManifest or the agent metadata/indexes.
|
|
17514
|
+
*/
|
|
17515
|
+
function physicalDbtRelationIds(manifestPath) {
|
|
17516
|
+
try {
|
|
17517
|
+
const stat = statSync(manifestPath);
|
|
17518
|
+
const version = `${stat.size}:${stat.mtimeMs}`;
|
|
17519
|
+
const cached = dbtPhysicalRelationCache.get(manifestPath);
|
|
17520
|
+
if (cached?.version === version)
|
|
17521
|
+
return cached.ids;
|
|
17522
|
+
const raw = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
17523
|
+
const ids = new Set();
|
|
17524
|
+
for (const collectionName of ['nodes', 'sources']) {
|
|
17525
|
+
const collection = raw[collectionName];
|
|
17526
|
+
if (!collection || typeof collection !== 'object' || Array.isArray(collection))
|
|
17527
|
+
continue;
|
|
17528
|
+
for (const [fallbackId, value] of Object.entries(collection)) {
|
|
17529
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
17530
|
+
continue;
|
|
17531
|
+
const node = value;
|
|
17532
|
+
const resourceType = String(node.resource_type ?? (collectionName === 'sources' ? 'source' : ''));
|
|
17533
|
+
if (resourceType !== 'model' && resourceType !== 'source')
|
|
17534
|
+
continue;
|
|
17535
|
+
const config = node.config && typeof node.config === 'object' && !Array.isArray(node.config)
|
|
17536
|
+
? node.config
|
|
17537
|
+
: {};
|
|
17538
|
+
if (resourceType === 'model' && String(config.materialized ?? '').toLowerCase() === 'ephemeral')
|
|
17539
|
+
continue;
|
|
17540
|
+
ids.add(typeof node.unique_id === 'string' ? node.unique_id : fallbackId);
|
|
16763
17541
|
}
|
|
16764
|
-
databaseNodes.unshift({
|
|
16765
|
-
id: 'db-schema:files',
|
|
16766
|
-
label: 'files',
|
|
16767
|
-
kind: 'schema',
|
|
16768
|
-
children: fileChildren,
|
|
16769
|
-
});
|
|
16770
17542
|
}
|
|
16771
|
-
|
|
16772
|
-
|
|
17543
|
+
dbtPhysicalRelationCache.set(manifestPath, { version, ids });
|
|
17544
|
+
return ids;
|
|
17545
|
+
}
|
|
17546
|
+
catch {
|
|
17547
|
+
return undefined;
|
|
17548
|
+
}
|
|
17549
|
+
}
|
|
17550
|
+
export function buildDbtDatabaseSchemaTree(manifest, limit = 25, physicalIds) {
|
|
17551
|
+
// PERF-001 / CTX-005: Block Studio must not enumerate the connected account.
|
|
17552
|
+
// Its initial editor context is the first bounded page of physical relations
|
|
17553
|
+
// declared by the connected dbt artifact; columns remain lazy point-lookups.
|
|
17554
|
+
const boundedLimit = Math.min(100, Math.max(1, Math.floor(limit)));
|
|
17555
|
+
const seen = new Set();
|
|
17556
|
+
const tables = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
17557
|
+
.filter((node) => Boolean(node.relation) && (!physicalIds || physicalIds.has(node.uniqueId)))
|
|
17558
|
+
.sort((a, b) => a.uniqueId.localeCompare(b.uniqueId))
|
|
17559
|
+
.flatMap((node) => {
|
|
17560
|
+
const path = node.relation;
|
|
17561
|
+
const key = path.toLowerCase();
|
|
17562
|
+
if (seen.has(key))
|
|
17563
|
+
return [];
|
|
17564
|
+
seen.add(key);
|
|
17565
|
+
const parts = path.split('.').filter(Boolean);
|
|
17566
|
+
return [{
|
|
17567
|
+
schema: parts.length > 1 ? parts.slice(0, -1).join('.') : 'default',
|
|
17568
|
+
name: parts.at(-1) ?? node.name,
|
|
17569
|
+
path,
|
|
17570
|
+
type: node.resourceType === 'source' ? 'DBT_SOURCE' : 'DBT_MODEL',
|
|
17571
|
+
}];
|
|
17572
|
+
})
|
|
17573
|
+
.slice(0, boundedLimit);
|
|
17574
|
+
const bySchema = new Map();
|
|
17575
|
+
for (const table of tables) {
|
|
17576
|
+
const current = bySchema.get(table.schema) ?? [];
|
|
17577
|
+
current.push(table);
|
|
17578
|
+
bySchema.set(table.schema, current);
|
|
17579
|
+
}
|
|
17580
|
+
return Array.from(bySchema.entries())
|
|
17581
|
+
.sort((a, b) => a[0].localeCompare(b[0]))
|
|
17582
|
+
.map(([schema, schemaTables]) => ({
|
|
17583
|
+
id: `db-schema:${schema}`,
|
|
17584
|
+
label: schema,
|
|
17585
|
+
kind: 'schema',
|
|
17586
|
+
children: schemaTables
|
|
17587
|
+
.sort((a, b) => a.name.localeCompare(b.name))
|
|
17588
|
+
.map((table) => ({
|
|
17589
|
+
id: `db-table:${table.path}`,
|
|
17590
|
+
label: table.name,
|
|
17591
|
+
kind: 'table',
|
|
17592
|
+
path: table.path,
|
|
17593
|
+
type: table.type,
|
|
17594
|
+
children: [],
|
|
17595
|
+
})),
|
|
17596
|
+
}));
|
|
16773
17597
|
}
|
|
16774
17598
|
export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
16775
17599
|
const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
|
|
@@ -23332,6 +24156,35 @@ ${predicates.map((predicate) => ` ${predicate}`).join(' OR\n')}
|
|
|
23332
24156
|
ORDER BY table_schema, table_name, ordinal_position
|
|
23333
24157
|
LIMIT 400`;
|
|
23334
24158
|
}
|
|
24159
|
+
/**
|
|
24160
|
+
* Conservative fallback for deciding whether an unparsed statement still
|
|
24161
|
+
* needs DQL's structural join validation. Join-free SELECT syntax may be
|
|
24162
|
+
* newer than the local parser and can safely continue to the read-only
|
|
24163
|
+
* warehouse boundary; an unparsed join must never bypass relationship proof.
|
|
24164
|
+
*/
|
|
24165
|
+
export function sqlMayContainJoin(sql) {
|
|
24166
|
+
const normalized = stripSqlStringsAndComments(sql)
|
|
24167
|
+
.replace(/\s+/g, ' ')
|
|
24168
|
+
.trim();
|
|
24169
|
+
if (/\bjoin\b/i.test(normalized))
|
|
24170
|
+
return true;
|
|
24171
|
+
const fromMatch = /\bfrom\b/i.exec(normalized);
|
|
24172
|
+
if (!fromMatch)
|
|
24173
|
+
return false;
|
|
24174
|
+
const tail = normalized.slice(fromMatch.index + fromMatch[0].length);
|
|
24175
|
+
const clauseMatch = /\b(where|group\s+by|having|qualify|order\s+by|limit|fetch|union|intersect|except)\b/i.exec(tail);
|
|
24176
|
+
const fromClause = clauseMatch ? tail.slice(0, clauseMatch.index) : tail;
|
|
24177
|
+
let depth = 0;
|
|
24178
|
+
for (const char of fromClause) {
|
|
24179
|
+
if (char === '(')
|
|
24180
|
+
depth += 1;
|
|
24181
|
+
else if (char === ')')
|
|
24182
|
+
depth = Math.max(0, depth - 1);
|
|
24183
|
+
else if (char === ',' && depth === 0)
|
|
24184
|
+
return true;
|
|
24185
|
+
}
|
|
24186
|
+
return false;
|
|
24187
|
+
}
|
|
23335
24188
|
/**
|
|
23336
24189
|
* Resolve a BARE internal graph identity (`source::orders` — a name with no
|
|
23337
24190
|
* database or schema) against the warehouse.
|