@duckcodeailabs/dql-cli 1.6.34 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts +20 -3
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +156 -85
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +4 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +7 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-4DbeLWHA.js +5876 -0
- package/dist/assets/dql-notebook/assets/index-dSyhu_eU.css +1 -0
- package/dist/assets/dql-notebook/index.html +2 -2
- package/dist/block-invocation.d.ts +31 -0
- package/dist/block-invocation.d.ts.map +1 -0
- package/dist/block-invocation.js +82 -0
- package/dist/block-invocation.js.map +1 -0
- package/dist/commands/agent.d.ts +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +12 -3
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts +1 -2
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +6 -9
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/compile.js +1 -1
- package/dist/commands/compile.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +44 -86
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/migrate.d.ts +87 -0
- package/dist/commands/migrate.d.ts.map +1 -1
- package/dist/commands/migrate.js +812 -3
- package/dist/commands/migrate.js.map +1 -1
- package/dist/commands/model.d.ts +4 -0
- package/dist/commands/model.d.ts.map +1 -0
- package/dist/commands/model.js +219 -0
- package/dist/commands/model.js.map +1 -0
- package/dist/commands/new.d.ts.map +1 -1
- package/dist/commands/new.js +28 -4
- package/dist/commands/new.js.map +1 -1
- package/dist/commands/parse.d.ts.map +1 -1
- package/dist/commands/parse.js +32 -0
- package/dist/commands/parse.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +23 -7
- package/dist/commands/sync.js.map +1 -1
- package/dist/index.js +19 -1
- package/dist/index.js.map +1 -1
- package/dist/llm/answer-loop-tools.d.ts +1 -1
- package/dist/llm/answer-loop-tools.d.ts.map +1 -1
- package/dist/llm/answer-loop-tools.js +210 -10
- package/dist/llm/answer-loop-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +30 -18
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/tools.d.ts.map +1 -1
- package/dist/llm/tools.js +3 -1
- package/dist/llm/tools.js.map +1 -1
- package/dist/llm/types.d.ts +12 -2
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +95 -7
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +3365 -172
- package/dist/local-runtime.js.map +1 -1
- package/dist/metricflow.d.ts +2 -0
- package/dist/metricflow.d.ts.map +1 -1
- package/dist/metricflow.js +10 -0
- package/dist/metricflow.js.map +1 -1
- package/dist/mixed-source-sql.d.ts +53 -0
- package/dist/mixed-source-sql.d.ts.map +1 -0
- package/dist/mixed-source-sql.js +311 -0
- package/dist/mixed-source-sql.js.map +1 -0
- package/dist/notebook-datasets.d.ts +120 -0
- package/dist/notebook-datasets.d.ts.map +1 -0
- package/dist/notebook-datasets.js +484 -0
- package/dist/notebook-datasets.js.map +1 -0
- package/dist/package.json +10 -10
- package/package.json +10 -10
- package/dist/assets/dql-notebook/assets/index-CYQrnwxa.css +0 -1
- package/dist/assets/dql-notebook/assets/index-GofggEGB.js +0 -5777
package/dist/local-runtime.js
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
import { execFileSync, execSync } from
|
|
2
|
-
import { createHash } from
|
|
3
|
-
import { createServer } from
|
|
4
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from
|
|
5
|
-
import { createRequire } from
|
|
6
|
-
import { homedir } from
|
|
7
|
-
import { dirname, extname, join, normalize, relative, resolve } from
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import
|
|
11
|
-
import
|
|
1
|
+
import { execFileSync, execSync } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, watch, writeFileSync, } from "node:fs";
|
|
5
|
+
import { createRequire } from "node:module";
|
|
6
|
+
import { homedir } from "node:os";
|
|
7
|
+
import { dirname, extname, join, normalize, relative, resolve, } from "node:path";
|
|
8
|
+
import { Readable } from "node:stream";
|
|
9
|
+
import { buildMixedSourceWarehouseFallbackSql, findMentionedNotebookDataset, planMixedSourceNotebookSql, planMixedSourceSql, } from "./mixed-source-sql.js";
|
|
10
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
11
|
+
import OpenAI from "openai";
|
|
12
|
+
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, hasStandaloneSemanticRef, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
13
|
+
import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, resolveRepoSource, getDialect, Parser, NodeKind, blockParameterDefinitions, buildLineageGraph, buildManifest, collectInputFiles, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, domainFolderSlug, previewModelingChange, applyModelingChange, previewDbtSourcePatch, applyDbtSourcePatch, loadDomainPackageRegistry, loadDbtNodeAuthoringDetail, relationshipValidationProofFingerprint, discoverDbtDomains, renderDomainDeclaration, ProjectSnapshotService, analyzeSqlReferences, } from '@duckcodeailabs/dql-core';
|
|
12
14
|
import { load as loadYaml } from 'js-yaml';
|
|
13
15
|
import { listBlockTemplates } from './block-templates.js';
|
|
14
16
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
15
17
|
import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
|
|
16
18
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
17
|
-
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath,
|
|
19
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, buildDeterministicDashboardStory, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, configuredSkillsPath, skillsDir, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, resolveDomainContextEnvelope, defaultKgPath, planAppFromPrompt, KGStore, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, probeLocalOllamaEmbeddings, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
18
20
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
19
21
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
20
22
|
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
@@ -25,8 +27,10 @@ import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeA
|
|
|
25
27
|
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
26
28
|
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
27
29
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
28
|
-
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
29
|
-
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from
|
|
30
|
+
import { clearBlockStudioImportSessions, candidateToDqlSource, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
31
|
+
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, hasMetricFlowCli, } from "./metricflow.js";
|
|
32
|
+
import { NotebookDatasetWorkspace, } from "./notebook-datasets.js";
|
|
33
|
+
import { prepareBlockInvocation } from './block-invocation.js';
|
|
30
34
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
31
35
|
const NOTEBOOK_FAVICON_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="7" fill="#6d5dfc"/><path d="M9 9h14v14H9z" fill="none" stroke="#fff" stroke-width="2"/><path d="M13 13h6M13 17h6M13 21h4" stroke="#fff" stroke-width="2" stroke-linecap="round"/></svg>';
|
|
32
36
|
export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
|
|
@@ -152,13 +156,33 @@ export function parseAgentRunRequestBody(body) {
|
|
|
152
156
|
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
153
157
|
if (governedAnswer.kind === 'no_answer')
|
|
154
158
|
return false;
|
|
159
|
+
// The executed rows are the presentation boundary: certified, semantic,
|
|
160
|
+
// DQL-first, and exploratory routes all deserve a separate stakeholder-facing
|
|
161
|
+
// explanation grounded in those values. Trust and query mechanics stay in the
|
|
162
|
+
// artifact/inspector instead of leaking into the primary answer.
|
|
163
|
+
if (governedAnswer.result)
|
|
164
|
+
return true;
|
|
155
165
|
if (governedAnswer.kind === 'certified' || governedAnswer.certification === 'certified')
|
|
156
166
|
return false;
|
|
167
|
+
// EXP-001: exploratory prose is part of the trust boundary. A free-form
|
|
168
|
+
// synthesis pass can accidentally relabel an entity-level lifetime measure
|
|
169
|
+
// as a child-level allocation or total repeated values. Keep the host's
|
|
170
|
+
// deterministic grain statement instead.
|
|
171
|
+
if (governedAnswer.exploratoryCandidate)
|
|
172
|
+
return false;
|
|
157
173
|
const finalText = (governedAnswer.answer ?? governedAnswer.text ?? '').trim();
|
|
158
174
|
if (governedAnswer.dqlArtifact && finalText)
|
|
159
175
|
return false;
|
|
160
176
|
return true;
|
|
161
177
|
}
|
|
178
|
+
function businessNarrativeGaps(warnings) {
|
|
179
|
+
const businessRelevant = (warnings ?? []).filter((warning) => !(/missing requested output column/i.test(warning)
|
|
180
|
+
|| /\bquery plan\b/i.test(warning)
|
|
181
|
+
|| /\bgrain\b/i.test(warning)
|
|
182
|
+
|| /\bcolumn(?:s)?\b.*\bmissing\b/i.test(warning)
|
|
183
|
+
|| /\bSQL\b|\bDBT\b|\bjoin probe\b/i.test(warning)));
|
|
184
|
+
return businessRelevant.length > 0 ? businessRelevant : undefined;
|
|
185
|
+
}
|
|
162
186
|
// ── Server-side conversation threads ────────────────────────────────────────
|
|
163
187
|
// When a run carries a threadId, the persisted thread is the authoritative
|
|
164
188
|
// source of prior turns (survives refresh); the client-built context remains
|
|
@@ -258,7 +282,7 @@ function conversationTurnInputFromRun(run) {
|
|
|
258
282
|
certification: agentRunString(payload?.certification),
|
|
259
283
|
sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
|
|
260
284
|
?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
|
|
261
|
-
contextPackId: agentRunString(payload?.contextPackId),
|
|
285
|
+
contextPackId: agentRunString(payload?.contextPackId) ?? agentRunString(contextPack?.id),
|
|
262
286
|
sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
|
|
263
287
|
dqlArtifact: agentRunRecord(payload?.dqlArtifact),
|
|
264
288
|
cascade: agentRunRecord(payload?.cascade),
|
|
@@ -311,15 +335,187 @@ function conversationStringArray(value) {
|
|
|
311
335
|
const values = value.filter((item) => typeof item === 'string' && item.trim().length > 0);
|
|
312
336
|
return values.length > 0 ? values : undefined;
|
|
313
337
|
}
|
|
338
|
+
function isLoopbackOrigin(value) {
|
|
339
|
+
try {
|
|
340
|
+
const hostname = new URL(value).hostname;
|
|
341
|
+
return hostname === '127.0.0.1' || hostname === 'localhost' || hostname === '::1' || hostname === '[::1]';
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
return false;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
314
347
|
function compactConversationRecord(record) {
|
|
315
348
|
const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
|
|
316
349
|
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
317
350
|
}
|
|
351
|
+
function apiRequestId(scope) {
|
|
352
|
+
return `${scope}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
353
|
+
}
|
|
354
|
+
function apiErrorEnvelope(input) {
|
|
355
|
+
return {
|
|
356
|
+
requestId: input.requestId,
|
|
357
|
+
...(input.snapshotId ? { snapshotId: input.snapshotId } : {}),
|
|
358
|
+
code: input.code,
|
|
359
|
+
message: input.message,
|
|
360
|
+
// Retain the legacy field during the compatibility window.
|
|
361
|
+
error: input.message,
|
|
362
|
+
recoverable: input.recoverable ?? input.code !== 'DOMAIN_COLLISION',
|
|
363
|
+
...(input.details !== undefined ? { details: input.details } : {}),
|
|
364
|
+
...(input.nextActions?.length ? { nextActions: input.nextActions } : {}),
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
function apiErrorCode(error, fallback) {
|
|
368
|
+
return typeof error === 'object' && error && 'code' in error
|
|
369
|
+
? String(error.code)
|
|
370
|
+
: fallback;
|
|
371
|
+
}
|
|
372
|
+
function apiErrorMessage(error) {
|
|
373
|
+
return error instanceof Error ? error.message : String(error);
|
|
374
|
+
}
|
|
318
375
|
export async function startLocalServer(opts) {
|
|
319
376
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
320
377
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
321
|
-
|
|
378
|
+
const loopback = bindHost === '127.0.0.1' || bindHost === 'localhost' || bindHost === '::1';
|
|
379
|
+
const authToken = opts.authToken ?? process.env.DQL_SERVER_TOKEN;
|
|
380
|
+
const allowedOrigins = new Set((opts.allowedOrigins ?? (process.env.DQL_ALLOWED_ORIGINS ?? '').split(','))
|
|
381
|
+
.map((value) => value.trim().replace(/\/$/, ''))
|
|
382
|
+
.filter(Boolean));
|
|
383
|
+
if (!loopback && !authToken) {
|
|
384
|
+
throw new Error('Non-loopback DQL server binding requires DQL_SERVER_TOKEN (or LocalServerOptions.authToken).');
|
|
385
|
+
}
|
|
386
|
+
if (!loopback && allowedOrigins.size === 0) {
|
|
387
|
+
throw new Error('Non-loopback DQL server binding requires DQL_ALLOWED_ORIGINS (or LocalServerOptions.allowedOrigins).');
|
|
388
|
+
}
|
|
389
|
+
const gitRoot = await resolveGitRoot(projectRoot);
|
|
390
|
+
if (gitRoot)
|
|
391
|
+
ensureLocalRuntimeGitignore(projectRoot);
|
|
322
392
|
let projectConfig = loadProjectConfig(projectRoot);
|
|
393
|
+
const configuredConnection = rawConnection
|
|
394
|
+
? normalizeProjectConnection(rawConnection, projectRoot)
|
|
395
|
+
: projectConfig.defaultConnection
|
|
396
|
+
? normalizeProjectConnection(projectConfig.defaultConnection, projectRoot)
|
|
397
|
+
: null;
|
|
398
|
+
let connection = configuredConnection ?? resolveDbtProfileRuntimeConnection(projectRoot, projectConfig);
|
|
399
|
+
const projectSnapshots = new ProjectSnapshotService();
|
|
400
|
+
const dashboardRunEvidence = new Map();
|
|
401
|
+
const dbtNodeDetailCache = new Map();
|
|
402
|
+
const onboardingJobs = new Map();
|
|
403
|
+
const projectSnapshot = () => {
|
|
404
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, projectConfig) ?? undefined;
|
|
405
|
+
const inputs = collectInputFiles({ projectRoot, dbtManifestPath });
|
|
406
|
+
const signatureHash = createHash('sha256');
|
|
407
|
+
for (const input of inputs.sort()) {
|
|
408
|
+
try {
|
|
409
|
+
const stats = statSync(input);
|
|
410
|
+
signatureHash.update(`${input}\0${stats.size}\0${stats.mtimeMs}\n`);
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
signatureHash.update(`${input}\0missing\n`);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
const signature = signatureHash.digest('hex');
|
|
417
|
+
const previousId = projectSnapshots.current()?.snapshotId;
|
|
418
|
+
const snapshot = projectSnapshots.refresh(signature, () => buildManifest({ projectRoot, dbtManifestPath }));
|
|
419
|
+
if (snapshot.snapshotId !== previousId)
|
|
420
|
+
dbtNodeDetailCache.clear();
|
|
421
|
+
return {
|
|
422
|
+
signature: snapshot.sourceVersion,
|
|
423
|
+
snapshotId: snapshot.snapshotId,
|
|
424
|
+
manifest: snapshot.value,
|
|
425
|
+
stale: snapshot.stale,
|
|
426
|
+
error: snapshot.error,
|
|
427
|
+
};
|
|
428
|
+
};
|
|
429
|
+
const onboardingDbtPaths = (body = {}) => {
|
|
430
|
+
const repoUrl = typeof body.repoUrl === 'string' && body.repoUrl.trim() ? body.repoUrl.trim() : undefined;
|
|
431
|
+
const branch = typeof body.branch === 'string' && body.branch.trim() ? body.branch.trim() : undefined;
|
|
432
|
+
const subPath = typeof body.subPath === 'string' && body.subPath.trim() ? body.subPath.trim() : undefined;
|
|
433
|
+
const repo = repoUrl ? resolveRepoSource({
|
|
434
|
+
provider: 'dbt', source: repoUrl.includes('gitlab') ? 'gitlab' : 'github', repoUrl, branch, subPath,
|
|
435
|
+
}, projectRoot) : undefined;
|
|
436
|
+
const projectDirInput = repo?.localPath ?? (typeof body.projectDir === 'string' ? body.projectDir : projectConfig.dbt?.projectDir ?? '.');
|
|
437
|
+
const dbtProjectDir = resolve(projectRoot, projectDirInput);
|
|
438
|
+
const manifestInput = typeof body.manifestPath === 'string' ? body.manifestPath : projectConfig.dbt?.manifestPath ?? 'target/manifest.json';
|
|
439
|
+
const manifestPath = resolve(dbtProjectDir, manifestInput);
|
|
440
|
+
if (!(manifestPath === dbtProjectDir || manifestPath.startsWith(`${dbtProjectDir}/`))) {
|
|
441
|
+
throw Object.assign(new Error('dbt manifestPath must stay inside dbt projectDir.'), { code: 'DBT_ARTIFACT_INVALID' });
|
|
442
|
+
}
|
|
443
|
+
const profilesDirInput = typeof body.profilesDir === 'string'
|
|
444
|
+
? body.profilesDir
|
|
445
|
+
: projectConfig.dbt?.profilesDir;
|
|
446
|
+
const configuredProfilesDir = profilesDirInput ? resolve(projectRoot, profilesDirInput) : undefined;
|
|
447
|
+
const profilePath = findDbtProfilePaths(projectRoot, dbtProjectDir, configuredProfilesDir)[0];
|
|
448
|
+
const profilesDir = configuredProfilesDir ?? (profilePath ? dirname(profilePath) : undefined);
|
|
449
|
+
return { dbtProjectDir, manifestPath, projectDirInput, manifestInput, profilesDir, repoUrl, branch, subPath, repoWarnings: repo?.warnings ?? [] };
|
|
450
|
+
};
|
|
451
|
+
const previewDbtOnboarding = (body = {}) => {
|
|
452
|
+
const paths = onboardingDbtPaths(body);
|
|
453
|
+
if (!existsSync(join(paths.dbtProjectDir, 'dbt_project.yml')))
|
|
454
|
+
throw Object.assign(new Error('dbt_project.yml was not found.'), { code: 'DBT_PROJECT_NOT_FOUND' });
|
|
455
|
+
if (!existsSync(paths.manifestPath))
|
|
456
|
+
throw Object.assign(new Error(`dbt manifest is missing at ${paths.manifestPath}. Run dbt parse, then retry.`), { code: 'DBT_MANIFEST_MISSING' });
|
|
457
|
+
let raw;
|
|
458
|
+
try {
|
|
459
|
+
raw = JSON.parse(readFileSync(paths.manifestPath, 'utf8'));
|
|
460
|
+
}
|
|
461
|
+
catch (error) {
|
|
462
|
+
throw Object.assign(new Error(`dbt manifest is invalid: ${error instanceof Error ? error.message : String(error)}`), { code: 'DBT_ARTIFACT_INVALID' });
|
|
463
|
+
}
|
|
464
|
+
const fingerprint = createHash('sha256').update(readFileSync(paths.manifestPath)).digest('hex');
|
|
465
|
+
return {
|
|
466
|
+
projectDir: portableProjectPath(projectRoot, paths.dbtProjectDir),
|
|
467
|
+
manifestPath: relative(paths.dbtProjectDir, paths.manifestPath).replace(/\\/g, '/'),
|
|
468
|
+
...(paths.profilesDir ? { profilesDir: portableProjectPath(projectRoot, paths.profilesDir) } : {}),
|
|
469
|
+
...(paths.repoUrl ? { repoUrl: paths.repoUrl, branch: paths.branch, subPath: paths.subPath } : {}),
|
|
470
|
+
...(paths.repoWarnings.length ? { warnings: paths.repoWarnings } : {}),
|
|
471
|
+
fingerprint,
|
|
472
|
+
projectName: typeof raw.metadata?.project_name === 'string' ? raw.metadata.project_name : undefined,
|
|
473
|
+
counts: {
|
|
474
|
+
models: Object.keys(raw.nodes ?? {}).filter((id) => id.startsWith('model.')).length,
|
|
475
|
+
sources: Object.keys(raw.sources ?? {}).length,
|
|
476
|
+
metrics: Object.keys(raw.metrics ?? {}).length,
|
|
477
|
+
},
|
|
478
|
+
artifacts: {
|
|
479
|
+
manifest: paths.manifestPath,
|
|
480
|
+
catalog: existsSync(join(dirname(paths.manifestPath), 'catalog.json')) ? join(dirname(paths.manifestPath), 'catalog.json') : null,
|
|
481
|
+
semanticManifest: existsSync(join(dirname(paths.manifestPath), 'semantic_manifest.json')) ? join(dirname(paths.manifestPath), 'semantic_manifest.json') : null,
|
|
482
|
+
},
|
|
483
|
+
};
|
|
484
|
+
};
|
|
485
|
+
const datasetWorkspace = new NotebookDatasetWorkspace(projectRoot, executor, connectorModuleSearchPaths(projectRoot));
|
|
486
|
+
let localWorkspaceReady = false;
|
|
487
|
+
const ensureLocalWorkspaceReady = async () => {
|
|
488
|
+
ensureConnectorInstalledForStartup(projectRoot, "duckdb");
|
|
489
|
+
if (!localWorkspaceReady) {
|
|
490
|
+
await datasetWorkspace.initialize();
|
|
491
|
+
localWorkspaceReady = true;
|
|
492
|
+
}
|
|
493
|
+
return datasetWorkspace.localConnection;
|
|
494
|
+
};
|
|
495
|
+
const resolveNamedConnection = (name) => {
|
|
496
|
+
if (!name)
|
|
497
|
+
return null;
|
|
498
|
+
const stored = getStoredConnections(projectConfig);
|
|
499
|
+
const normalized = normalizeStoredConnection(stored[name]);
|
|
500
|
+
return normalized
|
|
501
|
+
? normalizeProjectConnection(normalized, projectRoot)
|
|
502
|
+
: null;
|
|
503
|
+
};
|
|
504
|
+
const resolveExecutionConnection = async (body) => {
|
|
505
|
+
const target = body.executionTarget && typeof body.executionTarget === "object"
|
|
506
|
+
? body.executionTarget
|
|
507
|
+
: null;
|
|
508
|
+
if (target?.target === "local")
|
|
509
|
+
return ensureLocalWorkspaceReady();
|
|
510
|
+
if (target?.target === "connection" &&
|
|
511
|
+
typeof target.connectionName === "string") {
|
|
512
|
+
const named = resolveNamedConnection(target.connectionName);
|
|
513
|
+
if (!named)
|
|
514
|
+
throw new Error(`Connection not found: ${target.connectionName}`);
|
|
515
|
+
return named;
|
|
516
|
+
}
|
|
517
|
+
return requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
518
|
+
};
|
|
323
519
|
const requireActiveConnection = (candidate = connection) => {
|
|
324
520
|
if (!candidate) {
|
|
325
521
|
throw new Error('No database connection is configured yet. Open Connections, add a warehouse or local DuckDB/file connection, then retry.');
|
|
@@ -382,6 +578,45 @@ export async function startLocalServer(opts) {
|
|
|
382
578
|
}
|
|
383
579
|
}
|
|
384
580
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
581
|
+
const recordDatasetMetadataSnapshot = (datasets = datasetWorkspace.list()) => {
|
|
582
|
+
if (datasets.length === 0)
|
|
583
|
+
return;
|
|
584
|
+
try {
|
|
585
|
+
recordRuntimeSchemaSnapshot(projectRoot, {
|
|
586
|
+
source: "notebook_local_datasets",
|
|
587
|
+
tables: datasets.map((dataset) => ({
|
|
588
|
+
relation: dataset.alias,
|
|
589
|
+
schema: "notebook_local",
|
|
590
|
+
name: dataset.name,
|
|
591
|
+
description: [
|
|
592
|
+
dataset.description,
|
|
593
|
+
`${dataset.storageMode} ${dataset.format} dataset`,
|
|
594
|
+
`refreshed ${dataset.refreshedAt}`,
|
|
595
|
+
`trust ${dataset.trustState}`,
|
|
596
|
+
]
|
|
597
|
+
.filter(Boolean)
|
|
598
|
+
.join(" · "),
|
|
599
|
+
source: dataset.storageMode === "project"
|
|
600
|
+
? "project_dataset"
|
|
601
|
+
: "local_dataset",
|
|
602
|
+
columns: dataset.profile.columns.map((column) => ({
|
|
603
|
+
name: column.name,
|
|
604
|
+
type: column.type,
|
|
605
|
+
description: column.flags?.length
|
|
606
|
+
? column.flags.join(", ")
|
|
607
|
+
: undefined,
|
|
608
|
+
})),
|
|
609
|
+
})),
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
// Dataset metadata improves retrieval but must never block notebook startup.
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
if (datasetWorkspace.list().length > 0) {
|
|
617
|
+
await ensureLocalWorkspaceReady().catch(() => undefined);
|
|
618
|
+
recordDatasetMetadataSnapshot();
|
|
619
|
+
}
|
|
385
620
|
// Auto-register data/ CSV and Parquet files as DuckDB views so semantic layer
|
|
386
621
|
// queries like `FROM orders` resolve without requiring read_csv_auto() in SQL.
|
|
387
622
|
if (connection && (connection.driver === 'file' || connection.driver === 'duckdb')) {
|
|
@@ -509,6 +744,38 @@ export async function startLocalServer(opts) {
|
|
|
509
744
|
return message;
|
|
510
745
|
};
|
|
511
746
|
const formatNotebookResearchStorageError = (error) => (formatAgentRunInfrastructureError(error, 'Notebook research storage'));
|
|
747
|
+
/**
|
|
748
|
+
* CTX-001 / SKILL-001 — Build surfaces receive the same server-resolved,
|
|
749
|
+
* snapshot-bound domain envelope and compact evidence pack as Ask. The pack
|
|
750
|
+
* is a retrieval boundary, never an authorization bypass.
|
|
751
|
+
*/
|
|
752
|
+
const buildDirectBuildContext = async (input) => {
|
|
753
|
+
const snapshot = projectSnapshot();
|
|
754
|
+
const domainContext = resolveDomainContextEnvelope({
|
|
755
|
+
manifest: snapshot.manifest,
|
|
756
|
+
activeDomain: input.domain,
|
|
757
|
+
purpose: input.purpose,
|
|
758
|
+
modelAreaId: input.modelAreaId,
|
|
759
|
+
source: input.domain ? 'explicit_ui' : 'inferred',
|
|
760
|
+
snapshotId: snapshot.snapshotId,
|
|
761
|
+
});
|
|
762
|
+
const contextPack = await buildLocalContextPack(projectRoot, {
|
|
763
|
+
question: input.question,
|
|
764
|
+
mode: 'build',
|
|
765
|
+
surface: input.surface,
|
|
766
|
+
selectedContext: {
|
|
767
|
+
activeSurface: input.surface,
|
|
768
|
+
domain: input.domain,
|
|
769
|
+
purpose: input.purpose,
|
|
770
|
+
modelAreaId: input.modelAreaId,
|
|
771
|
+
...input.selectedContext,
|
|
772
|
+
},
|
|
773
|
+
domainContext,
|
|
774
|
+
strictness: 'balanced',
|
|
775
|
+
limit: 120,
|
|
776
|
+
});
|
|
777
|
+
return { domainContext, contextPack };
|
|
778
|
+
};
|
|
512
779
|
const buildAgentPromptArtifact = async (request, target, repair) => {
|
|
513
780
|
try {
|
|
514
781
|
await reindexProject(projectRoot, { kgPath: defaultKgPath(projectRoot) });
|
|
@@ -525,6 +792,17 @@ export async function startLocalServer(opts) {
|
|
|
525
792
|
const prompt = isRepair
|
|
526
793
|
? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}`
|
|
527
794
|
: request.question;
|
|
795
|
+
const domain = target === 'block' ? agentRunWorkspaceValue(request, 'domain') : undefined;
|
|
796
|
+
const modelAreaId = agentRunWorkspaceValue(request, 'modelAreaId');
|
|
797
|
+
const purpose = agentRunWorkspaceValue(request, 'purpose');
|
|
798
|
+
const { domainContext, contextPack } = await buildDirectBuildContext({
|
|
799
|
+
question: prompt,
|
|
800
|
+
domain,
|
|
801
|
+
purpose,
|
|
802
|
+
modelAreaId,
|
|
803
|
+
surface: target === 'cell' ? 'notebook' : 'block',
|
|
804
|
+
selectedContext: { target, mode },
|
|
805
|
+
});
|
|
528
806
|
return buildFromPrompt({
|
|
529
807
|
projectRoot,
|
|
530
808
|
prompt,
|
|
@@ -535,10 +813,16 @@ export async function startLocalServer(opts) {
|
|
|
535
813
|
target,
|
|
536
814
|
mode,
|
|
537
815
|
blockPath: target === 'block'
|
|
538
|
-
? agentRunWorkspaceValue(request, 'blockPath')
|
|
816
|
+
? agentRunWorkspaceValue(request, 'blockPath')
|
|
817
|
+
?? (request.selectedObject?.kind === 'block' && request.selectedObject.path?.endsWith('.dql')
|
|
818
|
+
? request.selectedObject.path
|
|
819
|
+
: undefined)
|
|
539
820
|
: undefined,
|
|
540
821
|
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
541
|
-
domain
|
|
822
|
+
domain,
|
|
823
|
+
modelAreaId,
|
|
824
|
+
contextPack,
|
|
825
|
+
domainContext,
|
|
542
826
|
userId: agentRunWorkspaceValue(request, 'userId'),
|
|
543
827
|
skills,
|
|
544
828
|
dbtManifestPath: resolveDbtManifestPath(projectRoot, projectConfig),
|
|
@@ -605,6 +889,9 @@ export async function startLocalServer(opts) {
|
|
|
605
889
|
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
606
890
|
'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
|
|
607
891
|
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
892
|
+
...(route === 'sql_cell'
|
|
893
|
+
? ['This is a notebook authoring fallback. Reuse certified blocks and semantic definitions as context, but return the requested row-level SQL cell rather than substituting a related aggregate answer.']
|
|
894
|
+
: []),
|
|
608
895
|
...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
|
|
609
896
|
].join(' '),
|
|
610
897
|
};
|
|
@@ -621,11 +908,28 @@ export async function startLocalServer(opts) {
|
|
|
621
908
|
catch {
|
|
622
909
|
semanticDriver = undefined;
|
|
623
910
|
}
|
|
911
|
+
const requestedDomain = agentRunWorkspaceValue(request, 'domain');
|
|
912
|
+
const requestedPurpose = agentRunWorkspaceValue(request, 'purpose');
|
|
913
|
+
const requestedModelAreaId = agentRunWorkspaceValue(request, 'modelAreaId');
|
|
914
|
+
const runProjectSnapshot = projectSnapshot();
|
|
915
|
+
const domainContext = requestedDomain
|
|
916
|
+
? resolveDomainContextEnvelope({
|
|
917
|
+
manifest: runProjectSnapshot.manifest,
|
|
918
|
+
activeDomain: requestedDomain,
|
|
919
|
+
purpose: requestedPurpose,
|
|
920
|
+
modelAreaId: requestedModelAreaId,
|
|
921
|
+
source: 'explicit_ui',
|
|
922
|
+
snapshotId: runProjectSnapshot.snapshotId,
|
|
923
|
+
})
|
|
924
|
+
: undefined;
|
|
624
925
|
await runner.run({
|
|
625
926
|
provider: resolvedProvider,
|
|
626
927
|
messages: [
|
|
627
928
|
...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
|
|
628
|
-
|
|
929
|
+
...(isRepair
|
|
930
|
+
? [{ role: 'assistant', content: `The prior attempt needs repair without changing the original question or requested output grain: ${repair?.repairHint}` }]
|
|
931
|
+
: []),
|
|
932
|
+
{ role: 'user', content: request.question },
|
|
629
933
|
],
|
|
630
934
|
conversationContext: request.conversationContext,
|
|
631
935
|
upstream: {
|
|
@@ -635,6 +939,13 @@ export async function startLocalServer(opts) {
|
|
|
635
939
|
reasoningEffort,
|
|
636
940
|
...(analysisDepth ? { analysisDepth } : {}),
|
|
637
941
|
projectRoot,
|
|
942
|
+
domainContext,
|
|
943
|
+
projectSnapshot: { snapshotId: runProjectSnapshot.snapshotId, manifest: runProjectSnapshot.manifest },
|
|
944
|
+
assertProjectSnapshot: (snapshotId) => {
|
|
945
|
+
// Refresh the service from current source state before the final guard.
|
|
946
|
+
projectSnapshot();
|
|
947
|
+
projectSnapshots.assertCurrent(snapshotId);
|
|
948
|
+
},
|
|
638
949
|
...(semanticDriver ? { semanticDriver } : {}),
|
|
639
950
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
640
951
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -736,7 +1047,71 @@ export async function startLocalServer(opts) {
|
|
|
736
1047
|
},
|
|
737
1048
|
];
|
|
738
1049
|
}
|
|
739
|
-
|
|
1050
|
+
/**
|
|
1051
|
+
* PRD-001 / AGT-002 / AGT-006 / EXP-002 — every answer surface receives the
|
|
1052
|
+
* same executable DQL artifact. Generated SQL is parameterized before the
|
|
1053
|
+
* artifact leaves the runtime; it remains compiled evidence, not the primary
|
|
1054
|
+
* user-facing object. Saving later persists this exact source without another
|
|
1055
|
+
* model call.
|
|
1056
|
+
*/
|
|
1057
|
+
function attachExecutableDqlArtifactContract(governedAnswer, question) {
|
|
1058
|
+
const originalSql = governedAnswer.proposedSql?.trim() || governedAnswer.sql?.trim();
|
|
1059
|
+
let artifact = governedAnswer.dqlArtifact;
|
|
1060
|
+
if ((!artifact || artifact.kind === 'sql_block') && originalSql) {
|
|
1061
|
+
const parameterized = parameterizeSqlForDqlImport(originalSql);
|
|
1062
|
+
const artifactName = artifact?.name
|
|
1063
|
+
?? artifact?.source.match(/\bblock\s+"([^"]+)"/i)?.[1]
|
|
1064
|
+
?? deriveGeneratedDraftSlug(question);
|
|
1065
|
+
const domain = artifact?.source.match(/\bdomain\s*=\s*"([^"]+)"/i)?.[1] ?? 'uncategorized';
|
|
1066
|
+
const source = candidateToDqlSource({
|
|
1067
|
+
name: artifactName,
|
|
1068
|
+
domain,
|
|
1069
|
+
description: question,
|
|
1070
|
+
owner: '',
|
|
1071
|
+
tags: ['ai-generated', 'review-required'],
|
|
1072
|
+
terms: [],
|
|
1073
|
+
pattern: '',
|
|
1074
|
+
grain: '',
|
|
1075
|
+
entities: [],
|
|
1076
|
+
outputs: [],
|
|
1077
|
+
dimensions: [],
|
|
1078
|
+
allowedFilters: parameterized.allowedFilters,
|
|
1079
|
+
parameterPolicy: parameterized.parameterPolicy,
|
|
1080
|
+
filterBindings: parameterized.filterBindings,
|
|
1081
|
+
parameterDecisions: parameterized.parameterDecisions,
|
|
1082
|
+
sourceSystems: [],
|
|
1083
|
+
replacementFor: [],
|
|
1084
|
+
reviewCadence: 'monthly',
|
|
1085
|
+
sql: parameterized.sql,
|
|
1086
|
+
llmContext: `Review-required DQL generated for: ${question}`,
|
|
1087
|
+
});
|
|
1088
|
+
artifact = {
|
|
1089
|
+
...(artifact ?? { kind: 'sql_block' }),
|
|
1090
|
+
kind: 'sql_block',
|
|
1091
|
+
name: artifactName,
|
|
1092
|
+
source,
|
|
1093
|
+
};
|
|
1094
|
+
}
|
|
1095
|
+
if (!artifact?.source)
|
|
1096
|
+
return;
|
|
1097
|
+
const invocation = prepareBlockInvocation({
|
|
1098
|
+
source: artifact.source,
|
|
1099
|
+
parameters: artifact.parameterValues,
|
|
1100
|
+
question,
|
|
1101
|
+
surface: 'ask_ai',
|
|
1102
|
+
});
|
|
1103
|
+
const certified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
1104
|
+
const semantic = governedAnswer.route?.tier === 'semantic_metric';
|
|
1105
|
+
governedAnswer.dqlArtifact = {
|
|
1106
|
+
...artifact,
|
|
1107
|
+
...(invocation.parameters.length > 0 ? { parameters: invocation.parameters } : {}),
|
|
1108
|
+
...(Object.keys(invocation.values).length > 0 ? { parameterValues: invocation.values } : {}),
|
|
1109
|
+
persistence: artifact.sourcePath ? 'saved' : 'transient',
|
|
1110
|
+
trustState: certified ? 'certified' : semantic ? 'governed' : 'review_required',
|
|
1111
|
+
...(originalSql ? { compiledSql: originalSql } : {}),
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
1114
|
+
const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emit }) => {
|
|
740
1115
|
let governedAnswer;
|
|
741
1116
|
try {
|
|
742
1117
|
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }));
|
|
@@ -745,6 +1120,126 @@ export async function startLocalServer(opts) {
|
|
|
745
1120
|
if (!governedAnswer.appliedHints) {
|
|
746
1121
|
governedAnswer.appliedHints = governedAnswer.contextPack?.appliedHints;
|
|
747
1122
|
}
|
|
1123
|
+
// EXP-001 — a missing modeled relationship is not the same as an unsafe
|
|
1124
|
+
// relationship. The answer loop exposes a non-executed candidate only for
|
|
1125
|
+
// that narrow case; the host owns bounded probe/execution and never
|
|
1126
|
+
// weakens the final governed-SQL guard.
|
|
1127
|
+
if (governedAnswer.exploratoryCandidate) {
|
|
1128
|
+
const explorationSchema = governedAnswer.contextPack?.allowedSqlContext.relations.map((table) => ({
|
|
1129
|
+
relation: table.relation,
|
|
1130
|
+
name: table.name,
|
|
1131
|
+
columns: table.columns.map((column) => ({ ...column })),
|
|
1132
|
+
source: table.source,
|
|
1133
|
+
}))
|
|
1134
|
+
?? governedAnswer.analysisPlan?.candidateTables.map((table) => ({
|
|
1135
|
+
relation: table.relation,
|
|
1136
|
+
name: table.relation.split('.').at(-1) ?? table.relation,
|
|
1137
|
+
columns: table.columns.map((name) => ({ name })),
|
|
1138
|
+
source: 'answer analysis plan',
|
|
1139
|
+
}));
|
|
1140
|
+
const exploration = await executeExploratoryCandidate(governedAnswer.exploratoryCandidate, explorationSchema, request.question, governedAnswer.contextPack?.questionPlan.requestedShape.topN?.scope === 'overall'
|
|
1141
|
+
? governedAnswer.contextPack.questionPlan.requestedShape.topN.n
|
|
1142
|
+
: undefined);
|
|
1143
|
+
const probeSummary = exploration.proofs.length > 0
|
|
1144
|
+
? `${exploration.proofs.length} bounded join probe${exploration.proofs.length === 1 ? '' : 's'} completed.`
|
|
1145
|
+
: 'No join probe was required for this query.';
|
|
1146
|
+
const baseWarnings = governedAnswer.validationWarnings ?? [];
|
|
1147
|
+
const evidence = governedAnswer.evidence ?? {
|
|
1148
|
+
route: [], lineage: [], businessContext: [], selectedAssets: [], sourceTables: [], semanticObjects: [], citations: [],
|
|
1149
|
+
};
|
|
1150
|
+
governedAnswer.evidence = evidence;
|
|
1151
|
+
evidence.route.push({
|
|
1152
|
+
tool: 'explore_dbt_grounded_sql',
|
|
1153
|
+
status: exploration.result ? 'checked' : 'failed',
|
|
1154
|
+
label: exploration.result ? 'Executed bounded DBT-grounded exploration' : 'DBT-grounded exploration could not execute',
|
|
1155
|
+
detail: exploration.error ?? probeSummary,
|
|
1156
|
+
});
|
|
1157
|
+
if (exploration.proofs.length > 0) {
|
|
1158
|
+
evidence.businessContext.push({
|
|
1159
|
+
label: 'Exploratory join probes',
|
|
1160
|
+
value: exploration.proofs.map((proof) => proof.summary).join('; '),
|
|
1161
|
+
source: 'bounded DBT-grounded validation',
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
if (exploration.result) {
|
|
1165
|
+
evidence.route = [
|
|
1166
|
+
...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({ terminalLane: 'generated' })),
|
|
1167
|
+
...evidence.route.filter((step) => !step.tool.startsWith('cascade_')),
|
|
1168
|
+
];
|
|
1169
|
+
evidence.validation = {
|
|
1170
|
+
status: 'warning',
|
|
1171
|
+
message: 'The bounded DBT-grounded query executed successfully, but its inferred relationship path still requires analyst review before reuse.',
|
|
1172
|
+
};
|
|
1173
|
+
evidence.execution = {
|
|
1174
|
+
status: 'executed',
|
|
1175
|
+
message: 'Executed the bounded DBT-grounded exploratory SQL preview.',
|
|
1176
|
+
rowCount: exploration.result.rowCount,
|
|
1177
|
+
executionTime: exploration.result.executionTime,
|
|
1178
|
+
};
|
|
1179
|
+
}
|
|
1180
|
+
const repairedExploratoryCandidate = exploration.sql !== governedAnswer.exploratoryCandidate.sql
|
|
1181
|
+
? { ...governedAnswer.exploratoryCandidate, sql: exploration.sql }
|
|
1182
|
+
: governedAnswer.exploratoryCandidate;
|
|
1183
|
+
const lifetimeGrainRepair = exploration.repairs.find((repair) => repair.startsWith('Used lifetime_'));
|
|
1184
|
+
const explorationAnswer = exploration.result
|
|
1185
|
+
? lifetimeGrainRepair
|
|
1186
|
+
? `Exploratory DBT-grounded query executed and returned ${exploration.result.rowCount} row${exploration.result.rowCount === 1 ? '' : 's'}. Each row is an owning-entity and joined-dimension pair. The lifetime measure remains an entity-level attribute repeated across that dimension; it is not allocated to the joined dimension. ${lifetimeGrainRepair}`
|
|
1187
|
+
: `Exploratory DBT-grounded query executed and returned ${exploration.result.rowCount} row${exploration.result.rowCount === 1 ? '' : 's'}. Review the inferred join evidence before reusing this analysis.`
|
|
1188
|
+
: `DQL found a DBT-grounded exploratory path, but the bounded validation could not run: ${exploration.error ?? 'unknown execution error'}.`;
|
|
1189
|
+
governedAnswer = {
|
|
1190
|
+
...governedAnswer,
|
|
1191
|
+
kind: 'uncertified',
|
|
1192
|
+
sourceTier: 'dbt_manifest',
|
|
1193
|
+
certification: 'analyst_review_required',
|
|
1194
|
+
reviewStatus: 'analyst_review_required',
|
|
1195
|
+
confidence: exploration.result ? 0.58 : 0.2,
|
|
1196
|
+
...(exploration.result ? {
|
|
1197
|
+
refusalCode: undefined,
|
|
1198
|
+
refusalDetails: undefined,
|
|
1199
|
+
intentDecision: {
|
|
1200
|
+
...governedAnswer.intentDecision,
|
|
1201
|
+
action: 'answer',
|
|
1202
|
+
confidence: Math.max(governedAnswer.intentDecision?.confidence ?? 0, 0.9),
|
|
1203
|
+
reason: 'Executed a bounded DBT-grounded exploratory answer after certified and semantic routes did not fully cover the request.',
|
|
1204
|
+
clarifyingQuestion: undefined,
|
|
1205
|
+
clarifySoft: undefined,
|
|
1206
|
+
followsUp: governedAnswer.intentDecision?.followsUp ?? false,
|
|
1207
|
+
},
|
|
1208
|
+
} : {}),
|
|
1209
|
+
result: exploration.result,
|
|
1210
|
+
proposedSql: exploration.sql,
|
|
1211
|
+
sql: exploration.sql,
|
|
1212
|
+
exploratoryCandidate: repairedExploratoryCandidate,
|
|
1213
|
+
executionError: exploration.error,
|
|
1214
|
+
text: explorationAnswer,
|
|
1215
|
+
answer: explorationAnswer,
|
|
1216
|
+
trustLabel: 'Exploratory · DBT-grounded',
|
|
1217
|
+
contextPackId: governedAnswer.contextPackId ?? governedAnswer.contextPack?.id,
|
|
1218
|
+
route: {
|
|
1219
|
+
tier: 'generated_sql',
|
|
1220
|
+
label: exploration.result
|
|
1221
|
+
? 'Exploratory · DBT-grounded answer (review required).'
|
|
1222
|
+
: 'Exploratory · DBT-grounded query needs review before execution.',
|
|
1223
|
+
},
|
|
1224
|
+
cascade: exploration.result
|
|
1225
|
+
? createCascadeAnswerResult({
|
|
1226
|
+
routeTier: 'generated_sql',
|
|
1227
|
+
label: 'Exploratory DBT-grounded answer executed with review required.',
|
|
1228
|
+
artifactKind: 'dbt_grounded_exploration',
|
|
1229
|
+
hasSqlPreview: true,
|
|
1230
|
+
executionStatus: 'executed',
|
|
1231
|
+
rowCount: exploration.result.rowCount,
|
|
1232
|
+
})
|
|
1233
|
+
: governedAnswer.cascade,
|
|
1234
|
+
validationWarnings: [
|
|
1235
|
+
...baseWarnings,
|
|
1236
|
+
probeSummary,
|
|
1237
|
+
...exploration.repairs,
|
|
1238
|
+
...(exploration.error ? [`Exploratory execution: ${exploration.error}`] : []),
|
|
1239
|
+
],
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
attachExecutableDqlArtifactContract(governedAnswer, request.question);
|
|
748
1243
|
applySmartVisualization(governedAnswer, request.question);
|
|
749
1244
|
}
|
|
750
1245
|
catch (error) {
|
|
@@ -767,8 +1262,21 @@ export async function startLocalServer(opts) {
|
|
|
767
1262
|
const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
|
|
768
1263
|
const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
769
1264
|
const isSemantic = governedAnswer.route?.tier === 'semantic_metric';
|
|
770
|
-
const
|
|
1265
|
+
const requestedNotebookDataset = findMentionedNotebookDataset(request.question, datasetWorkspace.list());
|
|
1266
|
+
const governedAssetMissesLocalSource = Boolean(requestedNotebookDataset && (isCertified
|
|
1267
|
+
|| isSemantic
|
|
1268
|
+
|| governedAnswer.sourceCertifiedBlock
|
|
1269
|
+
|| governedAnswer.dqlArtifact?.kind === 'certified_block'));
|
|
1270
|
+
const isExploratory = Boolean(governedAnswer.exploratoryCandidate);
|
|
1271
|
+
const isGroundingGap = governedAnswer.kind === 'no_answer'
|
|
1272
|
+
&& governedAnswer.refusalCode === 'grounding_gap'
|
|
1273
|
+
&& !isExploratory;
|
|
771
1274
|
const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
|
|
1275
|
+
// AGT-004: a rejected attribution/export/proof policy is a deliberate
|
|
1276
|
+
// governance boundary, not an ambiguous user question and not a repairable
|
|
1277
|
+
// retrieval miss. Keep the precise policy detail visible, but never burn two
|
|
1278
|
+
// more provider calls retrying the same incompatible candidate.
|
|
1279
|
+
const isPolicyBlocked = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'policy_blocked';
|
|
772
1280
|
// The model tried to compose a governed query and declined despite having usable
|
|
773
1281
|
// context (e.g. it wasn't confident about a multi-table join). That is NOT a
|
|
774
1282
|
// question for the USER to clarify — it's a case to retry harder: escalate to a
|
|
@@ -783,9 +1291,11 @@ export async function startLocalServer(opts) {
|
|
|
783
1291
|
// grounding gap or a model decline is retried/escalated by the engine, and a
|
|
784
1292
|
// provider outage is surfaced as blocked so the UI offers a retry.
|
|
785
1293
|
const needsClarification = governedAnswer.kind === 'no_answer'
|
|
786
|
-
&& !isGroundingGap && !isProviderError && !isModelDeclined;
|
|
1294
|
+
&& !isGroundingGap && !isProviderError && !isModelDeclined && !isPolicyBlocked;
|
|
787
1295
|
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
788
|
-
const runnableSql = governedAnswer.kind === 'no_answer'
|
|
1296
|
+
const runnableSql = governedAnswer.kind === 'no_answer' || (isExploratory && !governedAnswer.result)
|
|
1297
|
+
? undefined
|
|
1298
|
+
: sql;
|
|
789
1299
|
// Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
|
|
790
1300
|
// that already produced DQL-first final prose keep the fast path.
|
|
791
1301
|
let synthesizedAnswer;
|
|
@@ -798,13 +1308,14 @@ export async function startLocalServer(opts) {
|
|
|
798
1308
|
const result = await synthesizeAnswer({
|
|
799
1309
|
question: request.question,
|
|
800
1310
|
category: routeDecision?.category,
|
|
801
|
-
|
|
1311
|
+
// The primary Ask reply is always business-facing. Analysts keep
|
|
1312
|
+
// the full DQL, SQL, lineage, gates, and grain in the inspector.
|
|
1313
|
+
audience: 'stakeholder',
|
|
802
1314
|
resultPreview: preview,
|
|
803
1315
|
sql: sql,
|
|
804
1316
|
draftText: draft,
|
|
805
|
-
gaps: governedAnswer.validationWarnings,
|
|
1317
|
+
gaps: businessNarrativeGaps(governedAnswer.validationWarnings),
|
|
806
1318
|
}, {
|
|
807
|
-
onDelta: emitAnswerDelta,
|
|
808
1319
|
complete: ({ system, user, signal, onDelta }) => streamOrGenerate(provider, [{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 350, temperature: 0.3, signal }, onDelta ?? (() => { })),
|
|
809
1320
|
});
|
|
810
1321
|
if (result.text)
|
|
@@ -846,13 +1357,18 @@ export async function startLocalServer(opts) {
|
|
|
846
1357
|
: [
|
|
847
1358
|
{ id: 'create-block', label: governedAnswer.dqlArtifact ? 'Review DQL draft' : 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
848
1359
|
{ id: 'research-gap', label: 'Research deeper', route: 'research' },
|
|
849
|
-
...(runnableSql ? [{
|
|
1360
|
+
...(runnableSql ? [{
|
|
1361
|
+
id: 'insert-sql',
|
|
1362
|
+
label: governedAnswer.dqlArtifact ? 'Insert as DQL cell' : 'Insert SQL preview',
|
|
1363
|
+
route: 'sql_cell',
|
|
1364
|
+
artifactKind: 'sql_cell',
|
|
1365
|
+
}] : []),
|
|
850
1366
|
];
|
|
851
1367
|
return {
|
|
852
1368
|
resolvedRoute,
|
|
853
1369
|
answerRefusalCode: governedAnswer.kind === 'no_answer' ? governedAnswer.refusalCode : undefined,
|
|
854
1370
|
answerTier: governedAnswer.route?.tier,
|
|
855
|
-
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
|
|
1371
|
+
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : isExploratory ? 'Exploratory DBT-grounded analysis requires review.' : 'Answered with review-required generated analysis.'),
|
|
856
1372
|
answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
|
|
857
1373
|
status,
|
|
858
1374
|
trustState,
|
|
@@ -864,20 +1380,35 @@ export async function startLocalServer(opts) {
|
|
|
864
1380
|
? (governedAnswer.dqlArtifact && !isProviderError
|
|
865
1381
|
? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
|
|
866
1382
|
: [])
|
|
867
|
-
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : isSemantic ? 'Governed semantic answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : isSemantic ? 'governed' : 'review_required')],
|
|
1383
|
+
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : isSemantic ? 'Governed semantic answer' : isExploratory ? 'Exploratory DBT-grounded answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : isSemantic ? 'governed' : 'review_required')],
|
|
868
1384
|
evaluations: [
|
|
869
1385
|
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
|
|
870
1386
|
plannedRoute: route,
|
|
871
1387
|
resolvedRoute,
|
|
872
1388
|
aiRoute: governedAnswer.route,
|
|
873
1389
|
}),
|
|
1390
|
+
...(governedAssetMissesLocalSource ? [
|
|
1391
|
+
{
|
|
1392
|
+
...agentRunEvaluation('requested-local-source', 'Requested local dataset', false, 'warning', `${requestedNotebookDataset.alias} was explicitly requested, but the governed answer did not use local data. Reuse its governed logic only as context and build the mixed-source SQL fallback.`, { datasetId: requestedNotebookDataset.id, alias: requestedNotebookDataset.alias }),
|
|
1393
|
+
suggestedRepair: `Build a notebook SQL fallback that combines the requested ${requestedNotebookDataset.alias} dataset with the required warehouse extraction.`,
|
|
1394
|
+
repairAction: {
|
|
1395
|
+
kind: 'escalate',
|
|
1396
|
+
route: 'sql_cell',
|
|
1397
|
+
hint: `The certified or semantic result is not exact because it omits the explicitly requested local dataset ${requestedNotebookDataset.alias}. Author the warehouse extraction and mixed-source notebook handoff.`,
|
|
1398
|
+
},
|
|
1399
|
+
},
|
|
1400
|
+
] : []),
|
|
874
1401
|
agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified || isSemantic, isCertified || isSemantic ? 'info' : 'warning', isCertified
|
|
875
1402
|
? 'The answer came from certified DQL context.'
|
|
876
1403
|
: isSemantic
|
|
877
1404
|
? 'The answer was compiled and executed from governed semantic definitions; it is not a human-certified reusable block.'
|
|
878
1405
|
: needsClarification
|
|
879
1406
|
? 'The answer loop needs more context before producing a governed answer.'
|
|
880
|
-
:
|
|
1407
|
+
: isPolicyBlocked
|
|
1408
|
+
? `DQL blocked the generated path under its relationship policy: ${governedAnswer.refusalDetails?.message ?? 'the relationship is not authorized for this analysis.'}`
|
|
1409
|
+
: isExploratory
|
|
1410
|
+
? 'The answer used bounded DBT/schema evidence because no certified modeled relationship path covered the request. It is review-required and was not certified.'
|
|
1411
|
+
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
881
1412
|
...(isGroundingGap ? [
|
|
882
1413
|
{
|
|
883
1414
|
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
|
|
@@ -902,6 +1433,15 @@ export async function startLocalServer(opts) {
|
|
|
902
1433
|
repairAction: { kind: 'escalate', route: 'research', hint: declinedRepairHint },
|
|
903
1434
|
},
|
|
904
1435
|
] : []),
|
|
1436
|
+
...(isPolicyBlocked ? [
|
|
1437
|
+
agentRunEvaluation('relationship-policy', 'Relationship policy', false, 'warning', governedAnswer.refusalDetails?.message
|
|
1438
|
+
?? 'The selected relationship is not authorized for this analysis.', {
|
|
1439
|
+
refusalCode: governedAnswer.refusalCode,
|
|
1440
|
+
refusalDetails: governedAnswer.refusalDetails,
|
|
1441
|
+
validationWarnings: governedAnswer.validationWarnings,
|
|
1442
|
+
route: governedAnswer.route,
|
|
1443
|
+
}),
|
|
1444
|
+
] : []),
|
|
905
1445
|
...(governedAnswer.executionError ? [
|
|
906
1446
|
agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
|
|
907
1447
|
] : []),
|
|
@@ -1138,33 +1678,112 @@ export async function startLocalServer(opts) {
|
|
|
1138
1678
|
};
|
|
1139
1679
|
},
|
|
1140
1680
|
sql_cell: async ({ request, routeDecision, attempt, repairHint }) => {
|
|
1681
|
+
const selectedCellSql = agentRunWorkspaceValue(request, 'cellSql');
|
|
1682
|
+
const mixedSourcePlan = selectedCellSql
|
|
1683
|
+
? planMixedSourceSql(selectedCellSql, datasetWorkspace.list().map((dataset) => dataset.alias))
|
|
1684
|
+
: null;
|
|
1685
|
+
if (mixedSourcePlan) {
|
|
1686
|
+
const explanation = [
|
|
1687
|
+
`The original query directly joined warehouse data with the local dataset ${mixedSourcePlan.localDataset}. Those sources execute in different engines, so the CSV is not a missing dbt table.`,
|
|
1688
|
+
`I prepared the warehouse-only extraction and retained ${mixedSourcePlan.warehouseKey}, the key needed to join with ${mixedSourcePlan.localDataset}.${mixedSourcePlan.localKey}.`,
|
|
1689
|
+
'Insert and run this SQL on the warehouse connection. On its result, choose Combine with local data, select the local dataset, and confirm the suggested keys. The final mixed result runs locally and remains review-required.',
|
|
1690
|
+
].join(' ');
|
|
1691
|
+
const result = {
|
|
1692
|
+
target: 'cell',
|
|
1693
|
+
sql: mixedSourcePlan.warehouseSql,
|
|
1694
|
+
explanation,
|
|
1695
|
+
mixedSourcePlan,
|
|
1696
|
+
};
|
|
1697
|
+
return {
|
|
1698
|
+
summary: `Prepared a warehouse extraction for local combination with ${mixedSourcePlan.localDataset}.`,
|
|
1699
|
+
answer: explanation,
|
|
1700
|
+
artifacts: [agentRunArtifact('sql_cell', 'Warehouse extraction for local analysis', result)],
|
|
1701
|
+
evaluations: [
|
|
1702
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Detected a mixed local and warehouse SQL repair.'),
|
|
1703
|
+
agentRunEvaluation('mixed-source-boundary', 'Mixed-source boundary', true, 'warning', 'No direct cross-engine join was executed. The user must explicitly stage the bounded warehouse result.'),
|
|
1704
|
+
],
|
|
1705
|
+
nextActions: [{ id: 'insert-sql', label: 'Insert warehouse extraction', artifactKind: 'sql_cell' }],
|
|
1706
|
+
};
|
|
1707
|
+
}
|
|
1708
|
+
const requestedNotebookDataset = findMentionedNotebookDataset(request.question, datasetWorkspace.list());
|
|
1709
|
+
const authoringRequest = requestedNotebookDataset
|
|
1710
|
+
? {
|
|
1711
|
+
...request,
|
|
1712
|
+
question: [
|
|
1713
|
+
request.question,
|
|
1714
|
+
'',
|
|
1715
|
+
`Notebook SQL fallback contract: ${requestedNotebookDataset.alias} is a local dataset, not a warehouse table. Produce the warehouse-only row-level extraction needed for the requested analysis. Include the warehouse join key that corresponds to the local dataset identifier, plus the requested order/detail columns. Do not return an aggregate certified answer and do not reference ${requestedNotebookDataset.alias} in the warehouse SQL. The UI will stage this bounded result and create the local join cell after confirmation.`,
|
|
1716
|
+
].join('\n'),
|
|
1717
|
+
}
|
|
1718
|
+
: request;
|
|
1141
1719
|
// P4: route Notebook SQL-cell generation through the SAME tool-rich governed
|
|
1142
1720
|
// pipeline Ask AI uses (schema-discovery tools P3 + declined-retry P1 + budget
|
|
1143
1721
|
// recovery P2), instead of the tool-less buildFromPrompt path. A SQL cell only
|
|
1144
1722
|
// needs the SQL, which the governed answer produces directly as proposedSql.
|
|
1145
|
-
const governedAnswer = await runGovernedAgentAnswerForRun(
|
|
1146
|
-
|
|
1723
|
+
const governedAnswer = await runGovernedAgentAnswerForRun(authoringRequest, { attempt: attempt ?? 0, repairHint }, 'sql_cell');
|
|
1724
|
+
attachExecutableDqlArtifactContract(governedAnswer, request.question);
|
|
1725
|
+
const schemaContext = requestedNotebookDataset
|
|
1726
|
+
? await getSchemaContextForAgent(request.question)
|
|
1727
|
+
: [];
|
|
1728
|
+
const generatedSql = governedAnswer.proposedSql ?? governedAnswer.sql ?? '';
|
|
1729
|
+
const generatedPlan = requestedNotebookDataset && generatedSql.trim()
|
|
1730
|
+
? planMixedSourceNotebookSql(generatedSql, {
|
|
1731
|
+
id: requestedNotebookDataset.id,
|
|
1732
|
+
name: requestedNotebookDataset.name,
|
|
1733
|
+
alias: requestedNotebookDataset.alias,
|
|
1734
|
+
columns: requestedNotebookDataset.profile.columns.map((column) => ({
|
|
1735
|
+
name: column.name,
|
|
1736
|
+
flags: column.flags,
|
|
1737
|
+
})),
|
|
1738
|
+
}, schemaContext)
|
|
1739
|
+
: null;
|
|
1740
|
+
const generatedReusedNonExactAsset = Boolean(governedAnswer.sourceCertifiedBlock || governedAnswer.dqlArtifact?.kind === 'certified_block');
|
|
1741
|
+
const fallbackSql = requestedNotebookDataset && (generatedReusedNonExactAsset || !generatedSql.trim())
|
|
1742
|
+
? buildMixedSourceWarehouseFallbackSql(request.question, {
|
|
1743
|
+
id: requestedNotebookDataset.id,
|
|
1744
|
+
name: requestedNotebookDataset.name,
|
|
1745
|
+
alias: requestedNotebookDataset.alias,
|
|
1746
|
+
columns: requestedNotebookDataset.profile.columns.map((column) => ({ name: column.name, flags: column.flags })),
|
|
1747
|
+
}, schemaContext)
|
|
1748
|
+
: undefined;
|
|
1749
|
+
const naturalMixedSourcePlan = requestedNotebookDataset && fallbackSql
|
|
1750
|
+
? planMixedSourceNotebookSql(fallbackSql, {
|
|
1751
|
+
id: requestedNotebookDataset.id,
|
|
1752
|
+
name: requestedNotebookDataset.name,
|
|
1753
|
+
alias: requestedNotebookDataset.alias,
|
|
1754
|
+
columns: requestedNotebookDataset.profile.columns.map((column) => ({ name: column.name, flags: column.flags })),
|
|
1755
|
+
}, schemaContext)
|
|
1756
|
+
: generatedPlan;
|
|
1757
|
+
const sql = naturalMixedSourcePlan?.warehouseSql ?? generatedSql;
|
|
1147
1758
|
const explanation = governedAnswer.answer ?? governedAnswer.text;
|
|
1148
1759
|
const hasSql = Boolean(sql.trim());
|
|
1149
1760
|
// Preserve the BuildCellResult shape the notebook UI already consumes.
|
|
1150
1761
|
const result = {
|
|
1151
1762
|
target: 'cell',
|
|
1152
1763
|
sql,
|
|
1153
|
-
explanation
|
|
1764
|
+
explanation: naturalMixedSourcePlan
|
|
1765
|
+
? `No certified block or semantic result exactly covered all requested sources. The workflow combines ${naturalMixedSourcePlan.warehouseRelations?.join(', ') || 'the requested warehouse tables'} in one bounded warehouse extraction, then joins it locally with ${naturalMixedSourcePlan.localDataset} on ${naturalMixedSourcePlan.warehouseKey} = ${naturalMixedSourcePlan.localKey}. Click Add workflow to notebook below to create the cells. The mixed result remains review-required.`
|
|
1766
|
+
: explanation,
|
|
1767
|
+
...(naturalMixedSourcePlan ? { mixedSourcePlan: naturalMixedSourcePlan } : {}),
|
|
1768
|
+
...(!naturalMixedSourcePlan && governedAnswer.dqlArtifact ? { dqlArtifact: governedAnswer.dqlArtifact } : {}),
|
|
1154
1769
|
...(governedAnswer.appliedSkills ? { appliedSkills: governedAnswer.appliedSkills } : {}),
|
|
1155
1770
|
};
|
|
1156
1771
|
return {
|
|
1157
1772
|
summary: hasSql
|
|
1158
|
-
?
|
|
1773
|
+
? naturalMixedSourcePlan
|
|
1774
|
+
? `Prepared a mixed-source notebook workflow using ${naturalMixedSourcePlan.localDataset}.`
|
|
1775
|
+
: 'Created a review-required DQL query cell.'
|
|
1159
1776
|
: (explanation || 'No SQL could be generated for this request.'),
|
|
1160
|
-
answer: explanation,
|
|
1161
|
-
artifacts: [agentRunArtifact('sql_cell', 'Generated
|
|
1777
|
+
answer: result.explanation,
|
|
1778
|
+
artifacts: [agentRunArtifact('sql_cell', naturalMixedSourcePlan ? 'Warehouse extraction for local analysis' : 'Generated DQL query cell', result)],
|
|
1162
1779
|
evaluations: [
|
|
1163
1780
|
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to SQL cell generation.'),
|
|
1164
|
-
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning',
|
|
1781
|
+
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', naturalMixedSourcePlan
|
|
1782
|
+
? 'The warehouse extraction is staged with limits and the CSV join runs locally. Mixed-source output is never auto-certified.'
|
|
1783
|
+
: 'Generated DQL and its compiled SQL evidence must be reviewed before certification.'),
|
|
1165
1784
|
],
|
|
1166
|
-
nextActions: [
|
|
1167
|
-
{ id: 'insert-sql', label: 'Insert SQL preview', artifactKind: 'sql_cell' },
|
|
1785
|
+
nextActions: naturalMixedSourcePlan ? [] : [
|
|
1786
|
+
{ id: 'insert-sql', label: governedAnswer.dqlArtifact ? 'Insert as DQL cell' : 'Insert SQL preview', artifactKind: 'sql_cell' },
|
|
1168
1787
|
{ id: 'create-block', label: 'Review as DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
1169
1788
|
],
|
|
1170
1789
|
};
|
|
@@ -1302,17 +1921,19 @@ export async function startLocalServer(opts) {
|
|
|
1302
1921
|
// A real app needs a certified anchor: only offer confirmation when at least
|
|
1303
1922
|
// one certified tile is present. Otherwise fall through to the escalate path
|
|
1304
1923
|
// (research / draft the missing blocks) so the user isn't sent to a dead end.
|
|
1305
|
-
const proposed = Boolean(proposal && proposal.tiles.some((tile) => tile.certification === 'certified' && !tile.error));
|
|
1924
|
+
const proposed = Boolean(proposal && proposal.tiles.some((tile) => (tile.certification === 'certified' || tile.certification === 'reviewed_semantic') && !tile.error));
|
|
1306
1925
|
const sessionPlan = agentRunRecord(session.plan);
|
|
1307
1926
|
const appTitle = agentRunString(sessionPlan?.name) ?? 'App proposal';
|
|
1308
1927
|
const certifiedCount = proposal?.coverage.certifiedTiles ?? 0;
|
|
1309
1928
|
const generatedCount = proposal?.coverage.generatedTiles ?? 0;
|
|
1929
|
+
const semanticCount = proposal?.coverage.semanticTiles ?? 0;
|
|
1310
1930
|
const gapCount = proposal?.coverage.gaps ?? 0;
|
|
1311
1931
|
// A coverage gap is NOT terminal — leave status open so the gate can escalate to
|
|
1312
1932
|
// drafting the missing blocks. Only genuine infra errors (the catch above) block.
|
|
1313
1933
|
return {
|
|
1314
1934
|
summary: proposed
|
|
1315
1935
|
? `Proposed ${certifiedCount} certified tile${certifiedCount === 1 ? '' : 's'}`
|
|
1936
|
+
+ (semanticCount > 0 ? ` + ${semanticCount} reviewed semantic tile${semanticCount === 1 ? '' : 's'}` : '')
|
|
1316
1937
|
+ (generatedCount > 0 ? ` + ${generatedCount} AI-generated (review required)` : '')
|
|
1317
1938
|
+ (gapCount > 0 ? `; ${gapCount} question${gapCount === 1 ? '' : 's'} still uncovered` : '')
|
|
1318
1939
|
+ '. Review the list and confirm to create the app.'
|
|
@@ -1326,6 +1947,7 @@ export async function startLocalServer(opts) {
|
|
|
1326
1947
|
dashboardId: session.dashboardId,
|
|
1327
1948
|
plan: session.plan,
|
|
1328
1949
|
proposal,
|
|
1950
|
+
proposalHash: session.proposalHash,
|
|
1329
1951
|
warnings: session.warnings,
|
|
1330
1952
|
}, session.appId)] : [],
|
|
1331
1953
|
evaluations: [
|
|
@@ -1519,19 +2141,21 @@ export async function startLocalServer(opts) {
|
|
|
1519
2141
|
cells: snapshotCells,
|
|
1520
2142
|
});
|
|
1521
2143
|
};
|
|
1522
|
-
const
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
2144
|
+
const executeDqlArtifactSourceForAgent = async (source, metadata = {}, invocationInput) => {
|
|
2145
|
+
const invocation = prepareBlockInvocation({
|
|
2146
|
+
source,
|
|
2147
|
+
parameters: invocationInput?.parameters,
|
|
2148
|
+
question: invocationInput?.question,
|
|
2149
|
+
surface: 'ask_ai',
|
|
2150
|
+
});
|
|
2151
|
+
if (invocation.errors.length > 0)
|
|
2152
|
+
throw new Error(invocation.errors.join(' '));
|
|
2153
|
+
if (invocation.unresolvedParameters.length > 0) {
|
|
2154
|
+
throw new Error(`I need values for: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
1530
2155
|
}
|
|
1531
|
-
const absBlockPath = join(projectRoot, block.filePath);
|
|
1532
|
-
const source = readFileSync(absBlockPath, 'utf-8');
|
|
1533
2156
|
const activeConnection = requireActiveConnection();
|
|
1534
2157
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
2158
|
+
const plan = buildExecutionPlan({ id: `agent-${metadata.name ?? 'dql-artifact'}`, type: 'dql', source, title: metadata.name ?? 'DQL artifact' }, { semanticLayer, driver: activeConnection.driver, tableMapping, parameters: invocation.values });
|
|
1535
2159
|
const semanticCompose = semanticLayer
|
|
1536
2160
|
? composeSemanticBlockSql(source, semanticLayer, {
|
|
1537
2161
|
driver: activeConnection.driver,
|
|
@@ -1539,29 +2163,55 @@ export async function startLocalServer(opts) {
|
|
|
1539
2163
|
projectRoot,
|
|
1540
2164
|
projectConfig,
|
|
1541
2165
|
detectedProvider: semanticDetectedProvider,
|
|
2166
|
+
parameters: invocation.values,
|
|
1542
2167
|
})
|
|
1543
2168
|
: null;
|
|
1544
|
-
const plan = buildExecutionPlan({ id: `agent-${block.name}`, type: 'dql', source, title: block.name }, { semanticLayer, driver: activeConnection.driver, tableMapping });
|
|
1545
2169
|
if (!plan && !semanticCompose?.sql) {
|
|
1546
2170
|
const semanticError = semanticCompose?.diagnostics.find((diagnostic) => diagnostic.severity === 'error')?.message;
|
|
1547
|
-
throw new Error(semanticError ?? `
|
|
2171
|
+
throw new Error(semanticError ?? `DQL artifact "${metadata.name ?? 'draft'}" produced no executable SQL.`);
|
|
1548
2172
|
}
|
|
1549
2173
|
const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, activeConnection, projectRoot, projectConfig);
|
|
1550
2174
|
const app = loadRuntimeApp(projectRoot, activePersonaAppId());
|
|
1551
|
-
|
|
1552
|
-
|
|
2175
|
+
const sourceDomain = metadata.domain ?? source.match(/\bdomain\s*=\s*"([^"]+)"/i)?.[1];
|
|
2176
|
+
assertAppAccess({ app, domain: sourceDomain ?? app?.domain, level: 'execute' });
|
|
2177
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...invocation.values }), prepared.connection);
|
|
1553
2178
|
const normalized = normalizeQueryResult(rawResult);
|
|
1554
2179
|
return {
|
|
1555
2180
|
columns: normalized.columns,
|
|
1556
2181
|
rows: normalized.rows,
|
|
1557
2182
|
rowCount: normalized.rowCount,
|
|
1558
2183
|
executionTime: normalized.executionTime,
|
|
1559
|
-
chartConfig: plan?.chartConfig ?? (
|
|
2184
|
+
chartConfig: plan?.chartConfig ?? (metadata.chartType ? { chart: metadata.chartType } : undefined),
|
|
1560
2185
|
sql: prepared.sql,
|
|
1561
|
-
blockName:
|
|
1562
|
-
blockPath:
|
|
2186
|
+
...(metadata.name ? { blockName: metadata.name } : {}),
|
|
2187
|
+
...(metadata.path ? { blockPath: metadata.path } : {}),
|
|
2188
|
+
parameters: invocation.resolvedParameters,
|
|
2189
|
+
auditId: invocation.auditId,
|
|
1563
2190
|
};
|
|
1564
2191
|
};
|
|
2192
|
+
const executeCertifiedBlockByNameForAgent = async (blockName, invocationInput, requireCertified = false) => {
|
|
2193
|
+
const manifest = buildManifest({ projectRoot });
|
|
2194
|
+
const block = manifest.blocks[blockName];
|
|
2195
|
+
if (!block) {
|
|
2196
|
+
throw new Error(`Matched block "${blockName}" is not present in the project manifest.`);
|
|
2197
|
+
}
|
|
2198
|
+
if (requireCertified && block.status !== 'certified') {
|
|
2199
|
+
throw new Error(`Block "${block.name}" is not certified and cannot be rerun as a certified answer.`);
|
|
2200
|
+
}
|
|
2201
|
+
const source = readFileSync(join(projectRoot, block.filePath), 'utf-8');
|
|
2202
|
+
return executeDqlArtifactSourceForAgent(source, {
|
|
2203
|
+
name: block.name,
|
|
2204
|
+
path: block.filePath,
|
|
2205
|
+
domain: block.domain,
|
|
2206
|
+
chartType: block.chartType,
|
|
2207
|
+
}, invocationInput);
|
|
2208
|
+
};
|
|
2209
|
+
const executeCertifiedBlockForAgent = async (node, invocationInput) => {
|
|
2210
|
+
if (node.kind !== 'block') {
|
|
2211
|
+
throw new Error(`Certified ${node.kind} "${node.name}" is a navigation artifact and cannot be executed as a block.`);
|
|
2212
|
+
}
|
|
2213
|
+
return executeCertifiedBlockByNameForAgent(node.name || node.nodeId.replace(/^block:/, ''), invocationInput);
|
|
2214
|
+
};
|
|
1565
2215
|
const executeGeneratedSqlForAgent = async (sql) => {
|
|
1566
2216
|
const activeConnection = requireActiveConnection();
|
|
1567
2217
|
const boundedSql = buildAgentPreviewSql(sql);
|
|
@@ -1582,6 +2232,62 @@ export async function startLocalServer(opts) {
|
|
|
1582
2232
|
sql: prepared.sql,
|
|
1583
2233
|
};
|
|
1584
2234
|
};
|
|
2235
|
+
/**
|
|
2236
|
+
* EXP-001: execution host for the deliberately narrow non-governed lane.
|
|
2237
|
+
* It does not reinterpret dbt lineage or shared names as relationship proof.
|
|
2238
|
+
* Instead it validates the generated equality predicates against bounded
|
|
2239
|
+
* samples, records the result as exploratory evidence, then executes the
|
|
2240
|
+
* existing read-only 200-row preview. Any malformed, broad, or unsupported
|
|
2241
|
+
* candidate remains review-required and is not executed automatically.
|
|
2242
|
+
*/
|
|
2243
|
+
const executeExploratoryCandidate = async (candidate, schemaContext = [], question = '', requestedTopN) => {
|
|
2244
|
+
const preflight = repairExploratorySqlBeforeExecution(candidate.sql, schemaContext, question);
|
|
2245
|
+
const boundedSql = applyRequestedTopNToExploratorySql(preflight.sql, requestedTopN);
|
|
2246
|
+
const repairs = boundedSql === preflight.sql
|
|
2247
|
+
? preflight.repairs
|
|
2248
|
+
: [...preflight.repairs, `Applied the requested overall top-${requestedTopN} bound before exploratory execution.`];
|
|
2249
|
+
if (preflight.blockedReason) {
|
|
2250
|
+
return {
|
|
2251
|
+
proofs: [],
|
|
2252
|
+
sql: boundedSql,
|
|
2253
|
+
repairs,
|
|
2254
|
+
error: preflight.blockedReason,
|
|
2255
|
+
};
|
|
2256
|
+
}
|
|
2257
|
+
const analysis = analyzeSqlReferences(boundedSql);
|
|
2258
|
+
if (!analysis.parsed) {
|
|
2259
|
+
return { proofs: [], sql: boundedSql, repairs, error: 'DQL could not parse the exploratory SQL to validate its join predicates.' };
|
|
2260
|
+
}
|
|
2261
|
+
// A normal customer → order → order-item → product question needs three
|
|
2262
|
+
// joins. Keep the lane bounded, but do not reject this common dbt-star path
|
|
2263
|
+
// solely because the old two-join demo limit was too small.
|
|
2264
|
+
if (analysis.joins.length > 4) {
|
|
2265
|
+
return { proofs: [], sql: boundedSql, repairs, error: 'This exploratory query has more than four joins and requires an analyst to review the relationship path.' };
|
|
2266
|
+
}
|
|
2267
|
+
if (analysis.joins.some((join) => !join.leftRelation || !join.rightRelation)) {
|
|
2268
|
+
return { proofs: [], sql: boundedSql, repairs, error: 'This exploratory query has a join endpoint DQL could not resolve for bounded validation.' };
|
|
2269
|
+
}
|
|
2270
|
+
const activeConnection = requireActiveConnection();
|
|
2271
|
+
const proofs = [];
|
|
2272
|
+
try {
|
|
2273
|
+
for (const join of analysis.joins) {
|
|
2274
|
+
const proofSql = buildExploratoryJoinProbeSql({
|
|
2275
|
+
leftRelation: join.leftRelation,
|
|
2276
|
+
leftColumn: join.leftColumn,
|
|
2277
|
+
rightRelation: join.rightRelation,
|
|
2278
|
+
rightColumn: join.rightColumn,
|
|
2279
|
+
});
|
|
2280
|
+
const prepared = prepareLocalExecution(proofSql, activeConnection, projectRoot, projectConfig);
|
|
2281
|
+
const raw = await executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
|
|
2282
|
+
proofs.push({ summary: summarizeExploratoryJoinProbe(raw.rows, join.leftRelation, join.leftColumn, join.rightRelation, join.rightColumn) });
|
|
2283
|
+
}
|
|
2284
|
+
const result = await executeGeneratedSqlForAgent(boundedSql);
|
|
2285
|
+
return { result, proofs, sql: boundedSql, repairs };
|
|
2286
|
+
}
|
|
2287
|
+
catch (error) {
|
|
2288
|
+
return { proofs, sql: boundedSql, repairs, error: error instanceof Error ? error.message : String(error) };
|
|
2289
|
+
}
|
|
2290
|
+
};
|
|
1585
2291
|
const getSchemaContextForAgent = async (question, preparedContextPack) => {
|
|
1586
2292
|
const scanRuntimeSchema = async () => {
|
|
1587
2293
|
if (!connection)
|
|
@@ -1603,7 +2309,7 @@ export async function startLocalServer(opts) {
|
|
|
1603
2309
|
// Rescan live when the question shape calls for it OR the stored snapshot is
|
|
1604
2310
|
// stale/absent (P6) — otherwise a warehouse schema change between sessions is
|
|
1605
2311
|
// silently reasoned over from a cached snapshot that never expires.
|
|
1606
|
-
const runtimeScan = (shouldAugmentAgentRuntimeSchema(question) || runtimeSnapshotStale(projectRoot))
|
|
2312
|
+
const runtimeScan = (shouldAugmentAgentRuntimeSchema(question, preparedContextPack?.questionPlan) || runtimeSnapshotStale(projectRoot))
|
|
1607
2313
|
? await scanRuntimeSchema().catch(() => undefined)
|
|
1608
2314
|
: undefined;
|
|
1609
2315
|
const runtimeContext = runtimeScan?.ranked ?? [];
|
|
@@ -2286,14 +2992,16 @@ export async function startLocalServer(opts) {
|
|
|
2286
2992
|
});
|
|
2287
2993
|
// SSE clients for /api/watch hot-reload
|
|
2288
2994
|
const sseClients = new Set();
|
|
2289
|
-
// Watch
|
|
2995
|
+
// Watch all Git-owned authoring roots. Domain Packages are recursive and are
|
|
2996
|
+
// the canonical home for modeling, skills, blocks, notebooks, Apps, and evals.
|
|
2290
2997
|
if (projectRoot) {
|
|
2291
|
-
|
|
2998
|
+
let sourceRefreshTimer;
|
|
2999
|
+
for (const dir of ['domains', 'skills', 'tests', 'notebooks', 'workbooks', 'blocks', 'apps', 'dashboards', 'semantic-layer', 'data']) {
|
|
2292
3000
|
const watchDir = join(projectRoot, dir);
|
|
2293
3001
|
if (!existsSync(watchDir))
|
|
2294
3002
|
continue;
|
|
2295
3003
|
try {
|
|
2296
|
-
watch(watchDir, { persistent: false }, (eventType, filename) => {
|
|
3004
|
+
watch(watchDir, { persistent: false, recursive: true }, (eventType, filename) => {
|
|
2297
3005
|
if (!filename)
|
|
2298
3006
|
return;
|
|
2299
3007
|
const path = `${dir}/${filename}`;
|
|
@@ -2306,6 +3014,14 @@ export async function startLocalServer(opts) {
|
|
|
2306
3014
|
sseClients.delete(client);
|
|
2307
3015
|
}
|
|
2308
3016
|
}
|
|
3017
|
+
if (['domains', 'skills', 'tests', 'blocks', 'apps', 'notebooks'].includes(dir)) {
|
|
3018
|
+
invalidateAgentProjectState(projectRoot);
|
|
3019
|
+
if (sourceRefreshTimer)
|
|
3020
|
+
clearTimeout(sourceRefreshTimer);
|
|
3021
|
+
sourceRefreshTimer = setTimeout(() => {
|
|
3022
|
+
void refreshUnifiedProjectIndexes(projectRoot).catch(() => undefined);
|
|
3023
|
+
}, 250);
|
|
3024
|
+
}
|
|
2309
3025
|
// Hot-reload semantic layer on change and notify frontend
|
|
2310
3026
|
if (dir === 'semantic-layer') {
|
|
2311
3027
|
const semanticConnection = connection;
|
|
@@ -2338,6 +3054,31 @@ export async function startLocalServer(opts) {
|
|
|
2338
3054
|
}
|
|
2339
3055
|
catch { /* dir not watchable */ }
|
|
2340
3056
|
}
|
|
3057
|
+
const configuredDbtManifest = resolveDbtManifestPath(projectRoot);
|
|
3058
|
+
if (configuredDbtManifest && existsSync(dirname(configuredDbtManifest))) {
|
|
3059
|
+
try {
|
|
3060
|
+
watch(dirname(configuredDbtManifest), { persistent: false }, (_eventType, filename) => {
|
|
3061
|
+
if (!filename || !['manifest.json', 'catalog.json', 'semantic_manifest.json', 'run_results.json'].includes(String(filename)))
|
|
3062
|
+
return;
|
|
3063
|
+
invalidateAgentProjectState(projectRoot);
|
|
3064
|
+
if (sourceRefreshTimer)
|
|
3065
|
+
clearTimeout(sourceRefreshTimer);
|
|
3066
|
+
sourceRefreshTimer = setTimeout(() => {
|
|
3067
|
+
void refreshUnifiedProjectIndexes(projectRoot).catch(() => undefined);
|
|
3068
|
+
}, 250);
|
|
3069
|
+
const payload = JSON.stringify({ type: 'dbt-artifacts-changed', path: String(filename) });
|
|
3070
|
+
for (const client of sseClients) {
|
|
3071
|
+
try {
|
|
3072
|
+
client.write(`event: change\ndata: ${payload}\n\n`);
|
|
3073
|
+
}
|
|
3074
|
+
catch {
|
|
3075
|
+
sseClients.delete(client);
|
|
3076
|
+
}
|
|
3077
|
+
}
|
|
3078
|
+
});
|
|
3079
|
+
}
|
|
3080
|
+
catch { /* dbt artifact directory not watchable */ }
|
|
3081
|
+
}
|
|
2341
3082
|
}
|
|
2342
3083
|
const validateImportCandidate = (candidate) => ({
|
|
2343
3084
|
...candidate,
|
|
@@ -2374,7 +3115,7 @@ export async function startLocalServer(opts) {
|
|
|
2374
3115
|
return null;
|
|
2375
3116
|
return `Likely duplicate of ${match.name} (${match.kind}, ${Math.round(match.score * 100)}% match). Reuse the existing block, extend it, or document this draft as a replacement before certification.`;
|
|
2376
3117
|
};
|
|
2377
|
-
const runBlockStudioPreviewSource = async (source, targetConnection) => {
|
|
3118
|
+
const runBlockStudioPreviewSource = async (source, targetConnection, parameters = {}) => {
|
|
2378
3119
|
const activeConnection = requireActiveConnection(targetConnection);
|
|
2379
3120
|
let tableMapping;
|
|
2380
3121
|
if (semanticLayer) {
|
|
@@ -2388,6 +3129,12 @@ export async function startLocalServer(opts) {
|
|
|
2388
3129
|
tableMapping = undefined;
|
|
2389
3130
|
}
|
|
2390
3131
|
}
|
|
3132
|
+
const invocation = prepareBlockInvocation({ source, parameters, surface: 'block_studio' });
|
|
3133
|
+
if (invocation.errors.length > 0)
|
|
3134
|
+
throw new Error(invocation.errors.join(' '));
|
|
3135
|
+
if (invocation.unresolvedParameters.length > 0) {
|
|
3136
|
+
throw new Error(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'}: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
3137
|
+
}
|
|
2391
3138
|
const semanticCompose = semanticLayer
|
|
2392
3139
|
? composeSemanticBlockSql(source, semanticLayer, {
|
|
2393
3140
|
driver: activeConnection.driver,
|
|
@@ -2395,6 +3142,7 @@ export async function startLocalServer(opts) {
|
|
|
2395
3142
|
projectRoot,
|
|
2396
3143
|
projectConfig,
|
|
2397
3144
|
detectedProvider: semanticDetectedProvider,
|
|
3145
|
+
parameters: invocation.values,
|
|
2398
3146
|
})
|
|
2399
3147
|
: null;
|
|
2400
3148
|
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
@@ -2407,11 +3155,12 @@ export async function startLocalServer(opts) {
|
|
|
2407
3155
|
}
|
|
2408
3156
|
const plan = buildExecutionPlan({ id: 'block-studio', type: 'dql', source, title: 'Block Studio' }, { semanticLayer, driver: activeConnection.driver, tableMapping });
|
|
2409
3157
|
const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan?.sql ?? executableSql, activeConnection, projectRoot, projectConfig);
|
|
2410
|
-
const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables(plan?.variables ?? {}), prepared.connection);
|
|
3158
|
+
const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...invocation.values }), prepared.connection);
|
|
2411
3159
|
return {
|
|
2412
3160
|
sql: prepared.sql,
|
|
2413
3161
|
result: normalizeQueryResult(result),
|
|
2414
3162
|
chartConfig: plan?.chartConfig ?? validation.chartConfig ?? null,
|
|
3163
|
+
invocation,
|
|
2415
3164
|
};
|
|
2416
3165
|
};
|
|
2417
3166
|
const runBlockStudioTestSummary = async (source, targetConnection) => {
|
|
@@ -2630,20 +3379,616 @@ export async function startLocalServer(opts) {
|
|
|
2630
3379
|
const requestUrl = req.url || '/';
|
|
2631
3380
|
const url = new URL(requestUrl, 'http://127.0.0.1');
|
|
2632
3381
|
const path = url.pathname || '/';
|
|
2633
|
-
// CORS
|
|
2634
|
-
|
|
3382
|
+
// Exact-origin CORS. Loopback dev origins are accepted; remote serving
|
|
3383
|
+
// requires an explicit allowlist and bearer token.
|
|
3384
|
+
const requestOrigin = typeof req.headers.origin === 'string' ? req.headers.origin.replace(/\/$/, '') : undefined;
|
|
3385
|
+
const loopbackOrigin = requestOrigin ? isLoopbackOrigin(requestOrigin) : false;
|
|
3386
|
+
const originAllowed = !requestOrigin || (loopback ? loopbackOrigin : allowedOrigins.has(requestOrigin));
|
|
3387
|
+
if (requestOrigin && originAllowed)
|
|
3388
|
+
res.setHeader('Access-Control-Allow-Origin', requestOrigin);
|
|
3389
|
+
res.setHeader('Vary', 'Origin');
|
|
2635
3390
|
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
|
|
2636
|
-
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
|
3391
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
|
3392
|
+
if (!originAllowed && path.startsWith('/api/')) {
|
|
3393
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3394
|
+
res.end(serializeJSON({ error: 'Origin is not allowed.' }));
|
|
3395
|
+
return;
|
|
3396
|
+
}
|
|
2637
3397
|
if (req.method === 'OPTIONS') {
|
|
2638
3398
|
res.writeHead(204);
|
|
2639
3399
|
res.end();
|
|
2640
3400
|
return;
|
|
2641
3401
|
}
|
|
3402
|
+
if (!loopback && path.startsWith('/api/') && path !== '/api/health') {
|
|
3403
|
+
const authorization = req.headers.authorization ?? '';
|
|
3404
|
+
if (authorization !== `Bearer ${authToken}`) {
|
|
3405
|
+
res.writeHead(401, { 'Content-Type': 'application/json; charset=utf-8', 'WWW-Authenticate': 'Bearer' });
|
|
3406
|
+
res.end(serializeJSON({ error: 'A valid DQL server bearer token is required.' }));
|
|
3407
|
+
return;
|
|
3408
|
+
}
|
|
3409
|
+
}
|
|
2642
3410
|
if (req.method === 'GET' && path === '/api/health') {
|
|
2643
3411
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2644
3412
|
res.end(serializeJSON({ status: 'ok' }));
|
|
2645
3413
|
return;
|
|
2646
3414
|
}
|
|
3415
|
+
if (req.method === 'GET' && path === '/api/onboarding/status') {
|
|
3416
|
+
const requestId = apiRequestId('onboarding-status');
|
|
3417
|
+
const dbtProjectDir = findDbtProjectPath(projectRoot, projectConfig);
|
|
3418
|
+
const manifestPath = resolve(dbtProjectDir, projectConfig.dbt?.manifestPath ?? 'target/manifest.json');
|
|
3419
|
+
const configuredProfilesDir = projectConfig.dbt?.profilesDir ? resolve(projectRoot, projectConfig.dbt.profilesDir) : undefined;
|
|
3420
|
+
const profilePath = findDbtProfilePaths(projectRoot, dbtProjectDir, configuredProfilesDir)[0];
|
|
3421
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
3422
|
+
const manifestFound = existsSync(manifestPath);
|
|
3423
|
+
let snapshotId;
|
|
3424
|
+
let snapshotError;
|
|
3425
|
+
if (manifestFound) {
|
|
3426
|
+
try {
|
|
3427
|
+
snapshotId = projectSnapshot().snapshotId;
|
|
3428
|
+
}
|
|
3429
|
+
catch (error) {
|
|
3430
|
+
snapshotError = error instanceof Error ? error.message : String(error);
|
|
3431
|
+
}
|
|
3432
|
+
}
|
|
3433
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3434
|
+
res.end(serializeJSON({
|
|
3435
|
+
requestId,
|
|
3436
|
+
snapshotId,
|
|
3437
|
+
dbt: {
|
|
3438
|
+
configured: Boolean(projectConfig.dbt),
|
|
3439
|
+
projectDir: projectConfig.dbt?.projectDir,
|
|
3440
|
+
manifestPath: projectConfig.dbt?.manifestPath,
|
|
3441
|
+
profilesDir: configuredProfilesDir
|
|
3442
|
+
? portableProjectPath(projectRoot, configuredProfilesDir)
|
|
3443
|
+
: profilePath ? portableProjectPath(projectRoot, dirname(profilePath)) : undefined,
|
|
3444
|
+
repoUrl: projectConfig.dbt?.repoUrl,
|
|
3445
|
+
branch: projectConfig.dbt?.branch,
|
|
3446
|
+
subPath: projectConfig.dbt?.subPath,
|
|
3447
|
+
projectFound: existsSync(join(dbtProjectDir, 'dbt_project.yml')),
|
|
3448
|
+
manifestFound,
|
|
3449
|
+
},
|
|
3450
|
+
modeling: {
|
|
3451
|
+
enabled: projectConfig.manifestVersion === 3 && projectConfig.modeling?.mode === 'dbt-first',
|
|
3452
|
+
manifestVersion: projectConfig.manifestVersion ?? 2,
|
|
3453
|
+
mode: projectConfig.modeling?.mode,
|
|
3454
|
+
},
|
|
3455
|
+
domains: { count: registry.values().length, diagnostics: registry.diagnostics },
|
|
3456
|
+
snapshot: { id: snapshotId, error: snapshotError },
|
|
3457
|
+
capabilities: {
|
|
3458
|
+
warehouse: Boolean(connection),
|
|
3459
|
+
ai: Boolean(process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.GEMINI_API_KEY || process.env.OLLAMA_BASE_URL),
|
|
3460
|
+
},
|
|
3461
|
+
}));
|
|
3462
|
+
return;
|
|
3463
|
+
}
|
|
3464
|
+
if (req.method === 'POST' && path === '/api/onboarding/dbt/preview') {
|
|
3465
|
+
const requestId = apiRequestId('onboarding-dbt-preview');
|
|
3466
|
+
try {
|
|
3467
|
+
const preview = previewDbtOnboarding(await readJSON(req));
|
|
3468
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3469
|
+
res.end(serializeJSON({ requestId, ...preview }));
|
|
3470
|
+
}
|
|
3471
|
+
catch (error) {
|
|
3472
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
3473
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3474
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Verify the dbt project and artifact paths, then preview again.'] })));
|
|
3475
|
+
}
|
|
3476
|
+
return;
|
|
3477
|
+
}
|
|
3478
|
+
if (req.method === 'POST' && path === '/api/onboarding/dbt/apply') {
|
|
3479
|
+
const requestId = apiRequestId('onboarding-dbt-apply');
|
|
3480
|
+
try {
|
|
3481
|
+
const body = await readJSON(req);
|
|
3482
|
+
let preview;
|
|
3483
|
+
try {
|
|
3484
|
+
preview = previewDbtOnboarding(body);
|
|
3485
|
+
}
|
|
3486
|
+
catch (error) {
|
|
3487
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : '';
|
|
3488
|
+
if (code !== 'DBT_MANIFEST_MISSING' || body.buildArtifacts !== true)
|
|
3489
|
+
throw error;
|
|
3490
|
+
const { dbtProjectDir } = onboardingDbtPaths(body);
|
|
3491
|
+
try {
|
|
3492
|
+
const { profilesDir } = onboardingDbtPaths(body);
|
|
3493
|
+
const parseArgs = buildDbtParseArgs(dbtProjectDir, profilesDir);
|
|
3494
|
+
execFileSync('dbt', parseArgs, {
|
|
3495
|
+
cwd: dbtProjectDir,
|
|
3496
|
+
timeout: 120_000,
|
|
3497
|
+
maxBuffer: 1024 * 1024,
|
|
3498
|
+
encoding: 'utf8',
|
|
3499
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
3500
|
+
env: { ...process.env, DBT_LOG_FORMAT: 'text' },
|
|
3501
|
+
});
|
|
3502
|
+
}
|
|
3503
|
+
catch (parseError) {
|
|
3504
|
+
throw Object.assign(new Error(`dbt parse failed. Run it in ${dbtProjectDir} to inspect the project error, then retry.`), { code: 'DBT_PARSE_FAILED', cause: parseError });
|
|
3505
|
+
}
|
|
3506
|
+
preview = previewDbtOnboarding(body);
|
|
3507
|
+
}
|
|
3508
|
+
const expectedFingerprint = typeof body.fingerprint === 'string'
|
|
3509
|
+
? body.fingerprint
|
|
3510
|
+
: typeof body.expectedFingerprint === 'string' ? body.expectedFingerprint : undefined;
|
|
3511
|
+
if ((!expectedFingerprint && body.buildArtifacts !== true) || (expectedFingerprint && expectedFingerprint !== preview.fingerprint)) {
|
|
3512
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3513
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed after preview. Review the refreshed preview before applying.', nextActions: ['Refresh the preview and review the new diff.'] })));
|
|
3514
|
+
return;
|
|
3515
|
+
}
|
|
3516
|
+
const nextConfig = {
|
|
3517
|
+
...projectConfig,
|
|
3518
|
+
manifestVersion: 3,
|
|
3519
|
+
modeling: { mode: 'dbt-first' },
|
|
3520
|
+
dbt: {
|
|
3521
|
+
projectDir: preview.projectDir,
|
|
3522
|
+
manifestPath: preview.manifestPath,
|
|
3523
|
+
...(preview.profilesDir ? { profilesDir: preview.profilesDir } : {}),
|
|
3524
|
+
...(preview.repoUrl ? { repoUrl: preview.repoUrl, branch: preview.branch, subPath: preview.subPath } : {}),
|
|
3525
|
+
},
|
|
3526
|
+
semanticLayer: { provider: 'dbt', projectPath: preview.projectDir },
|
|
3527
|
+
};
|
|
3528
|
+
const configPath = join(projectRoot, 'dql.config.json');
|
|
3529
|
+
const previousConfigSource = readFileSync(configPath, 'utf8');
|
|
3530
|
+
const previousProjectConfig = projectConfig;
|
|
3531
|
+
const previousConnection = connection;
|
|
3532
|
+
const tempPath = `${configPath}.tmp-${process.pid}`;
|
|
3533
|
+
writeFileSync(tempPath, `${JSON.stringify(nextConfig, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
3534
|
+
renameSync(tempPath, configPath);
|
|
3535
|
+
projectConfig = nextConfig;
|
|
3536
|
+
if (!configuredConnection)
|
|
3537
|
+
connection = resolveDbtProfileRuntimeConnection(projectRoot, nextConfig);
|
|
3538
|
+
projectSnapshots.invalidate();
|
|
3539
|
+
invalidateAgentProjectState(projectRoot);
|
|
3540
|
+
let snapshotId;
|
|
3541
|
+
try {
|
|
3542
|
+
snapshotId = projectSnapshot().snapshotId;
|
|
3543
|
+
}
|
|
3544
|
+
catch (error) {
|
|
3545
|
+
const rollbackPath = `${configPath}.rollback-${process.pid}`;
|
|
3546
|
+
writeFileSync(rollbackPath, previousConfigSource, { encoding: 'utf8', mode: 0o600 });
|
|
3547
|
+
renameSync(rollbackPath, configPath);
|
|
3548
|
+
projectConfig = previousProjectConfig;
|
|
3549
|
+
connection = previousConnection;
|
|
3550
|
+
projectSnapshots.invalidate();
|
|
3551
|
+
invalidateAgentProjectState(projectRoot);
|
|
3552
|
+
throw Object.assign(new Error(`dbt-first configuration did not compile and was rolled back: ${error instanceof Error ? error.message : String(error)}`), { code: 'SNAPSHOT_BUILD_FAILED' });
|
|
3553
|
+
}
|
|
3554
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3555
|
+
res.end(serializeJSON({ requestId, snapshotId, applied: true, config: { manifestVersion: 3, modeling: nextConfig.modeling, dbt: nextConfig.dbt }, fingerprint: preview.fingerprint }));
|
|
3556
|
+
}
|
|
3557
|
+
catch (error) {
|
|
3558
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
3559
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3560
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Fix the reported dbt issue and run preview again.'] })));
|
|
3561
|
+
}
|
|
3562
|
+
return;
|
|
3563
|
+
}
|
|
3564
|
+
if (req.method === 'POST' && path === '/api/onboarding/refresh') {
|
|
3565
|
+
const requestId = apiRequestId('onboarding-refresh');
|
|
3566
|
+
const id = `dbt-refresh-${Date.now().toString(36)}`;
|
|
3567
|
+
try {
|
|
3568
|
+
const body = await readJSON(req);
|
|
3569
|
+
const currentArtifact = previewDbtOnboarding({});
|
|
3570
|
+
if (!body.expectedFingerprint || body.expectedFingerprint !== currentArtifact.fingerprint) {
|
|
3571
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3572
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed before refresh. Review the current artifact preview.', nextActions: ['Return to the dbt preview step and review the refreshed fingerprint.'] })));
|
|
3573
|
+
return;
|
|
3574
|
+
}
|
|
3575
|
+
projectSnapshots.invalidate();
|
|
3576
|
+
const snapshot = projectSnapshot();
|
|
3577
|
+
invalidateAgentProjectState(projectRoot);
|
|
3578
|
+
await reindexProject(projectRoot, { manifest: snapshot.manifest, kgPath: defaultKgPath(projectRoot) });
|
|
3579
|
+
const job = { id, kind: 'dbt_refresh', status: 'completed', createdAt: new Date().toISOString(), result: { snapshotId: snapshot.snapshotId, diagnostics: snapshot.manifest.diagnostics ?? [] } };
|
|
3580
|
+
onboardingJobs.set(id, job);
|
|
3581
|
+
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3582
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, jobId: id, ...job }));
|
|
3583
|
+
}
|
|
3584
|
+
catch (error) {
|
|
3585
|
+
const job = { id, kind: 'dbt_refresh', status: 'failed', createdAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) };
|
|
3586
|
+
onboardingJobs.set(id, job);
|
|
3587
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3588
|
+
res.end(serializeJSON({ ...apiErrorEnvelope({ requestId, code: 'SNAPSHOT_BUILD_FAILED', message: job.error, nextActions: ['Keep using the previous snapshot while fixing compile diagnostics.'] }), jobId: id, ...job }));
|
|
3589
|
+
}
|
|
3590
|
+
return;
|
|
3591
|
+
}
|
|
3592
|
+
if (req.method === 'GET' && path.startsWith('/api/onboarding/jobs/')) {
|
|
3593
|
+
const requestId = apiRequestId('onboarding-job');
|
|
3594
|
+
const id = decodeURIComponent(path.slice('/api/onboarding/jobs/'.length));
|
|
3595
|
+
const job = onboardingJobs.get(id);
|
|
3596
|
+
res.writeHead(job ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3597
|
+
const jobSnapshotId = typeof job?.result?.snapshotId === 'string'
|
|
3598
|
+
? String((job?.result).snapshotId)
|
|
3599
|
+
: undefined;
|
|
3600
|
+
res.end(serializeJSON(job ? { requestId, ...(jobSnapshotId ? { snapshotId: jobSnapshotId } : {}), ...job } : apiErrorEnvelope({ requestId, code: 'ONBOARDING_JOB_NOT_FOUND', message: `onboarding job not found: ${id}`, recoverable: false })));
|
|
3601
|
+
return;
|
|
3602
|
+
}
|
|
3603
|
+
if (req.method === 'DELETE' && path.startsWith('/api/onboarding/jobs/')) {
|
|
3604
|
+
const requestId = apiRequestId('onboarding-job-cancel');
|
|
3605
|
+
const id = decodeURIComponent(path.slice('/api/onboarding/jobs/'.length));
|
|
3606
|
+
const job = onboardingJobs.get(id);
|
|
3607
|
+
if (job)
|
|
3608
|
+
onboardingJobs.set(id, { ...job, status: 'cancelled' });
|
|
3609
|
+
res.writeHead(job ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3610
|
+
res.end(serializeJSON(job ? { requestId, ...job, status: 'cancelled' } : apiErrorEnvelope({ requestId, code: 'ONBOARDING_JOB_NOT_FOUND', message: `onboarding job not found: ${id}`, recoverable: false })));
|
|
3611
|
+
return;
|
|
3612
|
+
}
|
|
3613
|
+
if (req.method === 'POST' && path === '/api/onboarding/domains/discover') {
|
|
3614
|
+
const requestId = apiRequestId('domain-discovery');
|
|
3615
|
+
try {
|
|
3616
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3617
|
+
if (!dbtManifestPath || !existsSync(dbtManifestPath))
|
|
3618
|
+
throw Object.assign(new Error('No dbt manifest found. Run dbt parse or connect the project first.'), { code: 'DBT_MANIFEST_MISSING' });
|
|
3619
|
+
const report = discoverDbtDomains({ projectRoot, dbtManifestPath });
|
|
3620
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3621
|
+
res.end(serializeJSON({
|
|
3622
|
+
requestId,
|
|
3623
|
+
snapshotId: projectSnapshot().snapshotId,
|
|
3624
|
+
...report,
|
|
3625
|
+
capabilities: { aiEnrichment: false, deterministic: true },
|
|
3626
|
+
}));
|
|
3627
|
+
}
|
|
3628
|
+
catch (error) {
|
|
3629
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
3630
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3631
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Build a valid dbt manifest and retry discovery.'] })));
|
|
3632
|
+
}
|
|
3633
|
+
return;
|
|
3634
|
+
}
|
|
3635
|
+
if (req.method === 'POST' && path === '/api/onboarding/domains/apply') {
|
|
3636
|
+
const requestId = apiRequestId('domain-apply');
|
|
3637
|
+
try {
|
|
3638
|
+
const body = await readJSON(req);
|
|
3639
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3640
|
+
if (!dbtManifestPath || !existsSync(dbtManifestPath))
|
|
3641
|
+
throw Object.assign(new Error('No dbt manifest found.'), { code: 'DBT_MANIFEST_MISSING' });
|
|
3642
|
+
const report = discoverDbtDomains({ projectRoot, dbtManifestPath });
|
|
3643
|
+
const expectedFingerprint = body.sourceFingerprint ?? body.expectedSourceFingerprint;
|
|
3644
|
+
if (!expectedFingerprint || expectedFingerprint !== report.sourceFingerprint) {
|
|
3645
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3646
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: projectSnapshot().snapshotId, code: 'SOURCE_CHANGED', message: 'dbt/domain evidence changed after discovery. Review the refreshed proposals before applying.', nextActions: ['Run domain discovery again and review the updated evidence.'] })));
|
|
3647
|
+
return;
|
|
3648
|
+
}
|
|
3649
|
+
const proposalIds = Array.isArray(body.proposals) ? body.proposals.flatMap((value) => typeof value === 'object' && value && 'id' in value && typeof value.id === 'string' ? [value.id] : []) : [];
|
|
3650
|
+
const requested = proposalIds.length > 0 ? proposalIds : Array.isArray(body.selectedDomains) ? body.selectedDomains : Array.isArray(body.domains) ? body.domains : report.proposals.map((proposal) => proposal.id);
|
|
3651
|
+
const selected = new Set(requested.filter((value) => typeof value === 'string'));
|
|
3652
|
+
if (body.mode === 'preview') {
|
|
3653
|
+
const preview = report.proposals.filter((proposal) => selected.has(proposal.id)).map((proposal) => ({ path: join('domains', ...proposal.id.split('.'), 'domain.dql').replace(/\\/g, '/'), operation: 'create_or_retain', summary: `${proposal.matchedDbtUniqueIds.length} dbt model(s); review required` }));
|
|
3654
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3655
|
+
res.end(serializeJSON({ requestId, snapshotId: projectSnapshot().snapshotId, applied: false, preview }));
|
|
3656
|
+
return;
|
|
3657
|
+
}
|
|
3658
|
+
const existing = loadDomainPackageRegistry(projectRoot);
|
|
3659
|
+
const applied = [];
|
|
3660
|
+
for (const proposal of report.proposals.filter((value) => selected.has(value.id))) {
|
|
3661
|
+
if (existing.get(proposal.id)) {
|
|
3662
|
+
applied.push({ id: proposal.id, status: 'existing', message: 'Existing Domain Package retained unchanged.' });
|
|
3663
|
+
continue;
|
|
3664
|
+
}
|
|
3665
|
+
const segments = proposal.id.split('.');
|
|
3666
|
+
if (segments.length === 0 || segments.some((segment) => !/^[a-z0-9_]+$/.test(segment))) {
|
|
3667
|
+
applied.push({ id: proposal.id, status: 'blocked', message: 'Domain id is not a safe normalized path.' });
|
|
3668
|
+
continue;
|
|
3669
|
+
}
|
|
3670
|
+
const sourcePath = join('domains', ...segments, 'domain.dql').replace(/\\/g, '/');
|
|
3671
|
+
const absolute = join(projectRoot, sourcePath);
|
|
3672
|
+
mkdirSync(dirname(absolute), { recursive: true });
|
|
3673
|
+
const dbtPaths = [...new Set(report.memberships.filter((membership) => membership.proposedDomain === proposal.id).flatMap((membership) => membership.sourcePath ? [membership.sourcePath] : []))].sort();
|
|
3674
|
+
const source = renderDomainDeclaration({
|
|
3675
|
+
id: proposal.id,
|
|
3676
|
+
name: proposal.name,
|
|
3677
|
+
parent: proposal.proposedParent,
|
|
3678
|
+
owner: proposal.owner,
|
|
3679
|
+
dbtPaths,
|
|
3680
|
+
description: 'Draft domain boundary proposed from cited dbt evidence; review before governance use.',
|
|
3681
|
+
});
|
|
3682
|
+
const temp = `${absolute}.tmp-${process.pid}`;
|
|
3683
|
+
writeFileSync(temp, source, { encoding: 'utf8', mode: 0o600 });
|
|
3684
|
+
renameSync(temp, absolute);
|
|
3685
|
+
applied.push({ id: proposal.id, status: 'created', path: sourcePath, message: 'Review-required domain boundary created; no relationships or skills were certified.' });
|
|
3686
|
+
}
|
|
3687
|
+
projectSnapshots.invalidate();
|
|
3688
|
+
invalidateAgentProjectState(projectRoot);
|
|
3689
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3690
|
+
res.end(serializeJSON({ requestId, snapshotId: projectSnapshot().snapshotId, applied: true, results: applied, domains: loadDomainPackageRegistry(projectRoot).values().map((pkg) => ({ id: pkg.id, filePath: pkg.declarationPath, lifecycle: 'draft' })) }));
|
|
3691
|
+
}
|
|
3692
|
+
catch (error) {
|
|
3693
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DOMAIN_COLLISION';
|
|
3694
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3695
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Resolve the collision or invalid proposal and preview again.'] })));
|
|
3696
|
+
}
|
|
3697
|
+
return;
|
|
3698
|
+
}
|
|
3699
|
+
if (req.method === 'GET' && path === '/api/domain-packages') {
|
|
3700
|
+
const snapshot = projectSnapshot();
|
|
3701
|
+
const manifest = snapshot.manifest;
|
|
3702
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
3703
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3704
|
+
res.end(serializeJSON({
|
|
3705
|
+
domains: listDomains(projectRoot),
|
|
3706
|
+
packages: registry.values().map((pkg) => ({
|
|
3707
|
+
id: pkg.id,
|
|
3708
|
+
name: pkg.name,
|
|
3709
|
+
parent: pkg.parent,
|
|
3710
|
+
depth: pkg.depth,
|
|
3711
|
+
ancestry: pkg.ancestry,
|
|
3712
|
+
declarationPath: pkg.declarationPath,
|
|
3713
|
+
root: relative(projectRoot, pkg.root).replace(/\\/g, '/'),
|
|
3714
|
+
owner: pkg.owner,
|
|
3715
|
+
exports: pkg.exports,
|
|
3716
|
+
})),
|
|
3717
|
+
modeling: manifest.modeling,
|
|
3718
|
+
dbtProvenance: manifest.dbtProvenance,
|
|
3719
|
+
domainAssets: collectDomainPackageAssets(projectRoot, registry),
|
|
3720
|
+
diagnostics: manifest.diagnostics ?? [],
|
|
3721
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3722
|
+
}));
|
|
3723
|
+
return;
|
|
3724
|
+
}
|
|
3725
|
+
if (req.method === 'GET' && path === '/api/domain-workspaces') {
|
|
3726
|
+
const requestId = apiRequestId('domain-workspaces');
|
|
3727
|
+
const snapshot = projectSnapshot();
|
|
3728
|
+
const manifest = snapshot.manifest;
|
|
3729
|
+
const packages = Object.values(manifest.modeling?.packages ?? {});
|
|
3730
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3731
|
+
res.end(serializeJSON({
|
|
3732
|
+
requestId,
|
|
3733
|
+
snapshotId: snapshot.snapshotId,
|
|
3734
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3735
|
+
domains: packages.map((pkg) => domainWorkspaceSummary(manifest, pkg.id)),
|
|
3736
|
+
unassignedModels: Object.values(manifest.dbtProvenance?.nodes ?? {}).filter((node) => !Object.values(manifest.modeling?.entities ?? {}).some((entity) => entity.dbtUniqueId === node.uniqueId)).length,
|
|
3737
|
+
}));
|
|
3738
|
+
return;
|
|
3739
|
+
}
|
|
3740
|
+
if (req.method === 'GET' && path.startsWith('/api/domain-workspaces/')) {
|
|
3741
|
+
const requestId = apiRequestId('domain-workspace');
|
|
3742
|
+
const suffix = decodeURIComponent(path.slice('/api/domain-workspaces/'.length));
|
|
3743
|
+
const relatedSuffix = '/related-products';
|
|
3744
|
+
const domainId = suffix.endsWith(relatedSuffix) ? suffix.slice(0, -relatedSuffix.length) : suffix;
|
|
3745
|
+
const snapshot = projectSnapshot();
|
|
3746
|
+
const manifest = snapshot.manifest;
|
|
3747
|
+
if (!manifest.modeling?.packages[domainId]) {
|
|
3748
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3749
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DOMAIN_NOT_FOUND', message: `domain workspace not found: ${domainId}`, recoverable: false })));
|
|
3750
|
+
return;
|
|
3751
|
+
}
|
|
3752
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3753
|
+
res.end(serializeJSON(suffix.endsWith(relatedSuffix)
|
|
3754
|
+
? { requestId, domain: domainId, ...relatedProductsForDomain(manifest, domainId), snapshotId: snapshot.snapshotId }
|
|
3755
|
+
: {
|
|
3756
|
+
requestId,
|
|
3757
|
+
snapshotId: snapshot.snapshotId,
|
|
3758
|
+
...domainWorkspaceSummary(manifest, domainId),
|
|
3759
|
+
relatedProducts: relatedProductsForDomain(manifest, domainId),
|
|
3760
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3761
|
+
}));
|
|
3762
|
+
return;
|
|
3763
|
+
}
|
|
3764
|
+
// Manifest v3 modeling surface. This response intentionally contains dbt
|
|
3765
|
+
// provenance references and the sparse DQL overlay, never copied dbt YAML.
|
|
3766
|
+
if (req.method === 'GET' && path === '/api/modeling/dbt-first') {
|
|
3767
|
+
const requestId = apiRequestId('modeling-dbt-first');
|
|
3768
|
+
const snapshot = projectSnapshot();
|
|
3769
|
+
const manifest = snapshot.manifest;
|
|
3770
|
+
if (manifest.manifestVersion !== 3 || !manifest.modeling || !manifest.dbtProvenance) {
|
|
3771
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3772
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DBT_FIRST_NOT_ENABLED', message: 'dbt-first modeling is not enabled. Set manifestVersion: 3 and modeling.mode: "dbt-first" in dql.config.json.', nextActions: ['Use Setup to connect dbt and enable dbt-first modeling.'] })));
|
|
3773
|
+
return;
|
|
3774
|
+
}
|
|
3775
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3776
|
+
res.end(serializeJSON({
|
|
3777
|
+
requestId,
|
|
3778
|
+
snapshotId: snapshot.snapshotId,
|
|
3779
|
+
manifestVersion: manifest.manifestVersion,
|
|
3780
|
+
dbtProvenance: manifest.dbtProvenance,
|
|
3781
|
+
modeling: manifest.modeling,
|
|
3782
|
+
domainAssets: collectDomainPackageAssets(projectRoot, loadDomainPackageRegistry(projectRoot)),
|
|
3783
|
+
lineage: manifest.lineage,
|
|
3784
|
+
diagnostics: manifest.diagnostics ?? [],
|
|
3785
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3786
|
+
}));
|
|
3787
|
+
return;
|
|
3788
|
+
}
|
|
3789
|
+
if (req.method === 'GET' && path.startsWith('/api/modeling/dbt-first/nodes/')) {
|
|
3790
|
+
const requestId = apiRequestId('modeling-node');
|
|
3791
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3792
|
+
const uniqueId = decodeURIComponent(path.slice('/api/modeling/dbt-first/nodes/'.length));
|
|
3793
|
+
const snapshot = projectSnapshot();
|
|
3794
|
+
const cacheKey = `${snapshot.snapshotId}:${uniqueId}`;
|
|
3795
|
+
if (!dbtNodeDetailCache.has(cacheKey)) {
|
|
3796
|
+
dbtNodeDetailCache.set(cacheKey, dbtManifestPath ? loadDbtNodeAuthoringDetail(dbtManifestPath, uniqueId) : undefined);
|
|
3797
|
+
}
|
|
3798
|
+
const detail = dbtNodeDetailCache.get(cacheKey);
|
|
3799
|
+
if (!detail) {
|
|
3800
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3801
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DBT_NODE_NOT_FOUND', message: `dbt node not found: ${uniqueId}`, recoverable: false })));
|
|
3802
|
+
return;
|
|
3803
|
+
}
|
|
3804
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3805
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, ...detail }));
|
|
3806
|
+
return;
|
|
3807
|
+
}
|
|
3808
|
+
if (req.method === 'GET' && path === '/api/modeling/dbt-first/inventory') {
|
|
3809
|
+
const requestId = apiRequestId('modeling-inventory');
|
|
3810
|
+
const snapshot = projectSnapshot();
|
|
3811
|
+
const manifest = snapshot.manifest;
|
|
3812
|
+
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 50));
|
|
3813
|
+
const cursor = Math.max(0, Number(url.searchParams.get('cursor')) || 0);
|
|
3814
|
+
const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
|
|
3815
|
+
const domain = (url.searchParams.get('domain') ?? '').trim();
|
|
3816
|
+
const boundByDbtId = new Map(Object.values(manifest.modeling?.entities ?? {}).map((entity) => [entity.dbtUniqueId, entity]));
|
|
3817
|
+
const nodes = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
3818
|
+
.filter((node) => !query || `${node.name} ${node.relation ?? ''} ${node.sourcePath ?? ''}`.toLowerCase().includes(query))
|
|
3819
|
+
.filter((node) => !domain || boundByDbtId.get(node.uniqueId)?.domain === domain)
|
|
3820
|
+
.sort((a, b) => a.uniqueId.localeCompare(b.uniqueId));
|
|
3821
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3822
|
+
res.end(serializeJSON({
|
|
3823
|
+
requestId,
|
|
3824
|
+
items: nodes.slice(cursor, cursor + limit).map((node) => ({ ...node, binding: boundByDbtId.get(node.uniqueId) })),
|
|
3825
|
+
nextCursor: cursor + limit < nodes.length ? cursor + limit : null,
|
|
3826
|
+
total: nodes.length,
|
|
3827
|
+
snapshotId: snapshot.snapshotId,
|
|
3828
|
+
}));
|
|
3829
|
+
return;
|
|
3830
|
+
}
|
|
3831
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/nodes/batch') {
|
|
3832
|
+
const requestId = apiRequestId('modeling-node-batch');
|
|
3833
|
+
const body = await readJSON(req);
|
|
3834
|
+
const uniqueIds = Array.isArray(body.uniqueIds) ? body.uniqueIds.filter((value) => typeof value === 'string').slice(0, 100) : [];
|
|
3835
|
+
const snapshot = projectSnapshot();
|
|
3836
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3837
|
+
const details = uniqueIds.map((uniqueId) => {
|
|
3838
|
+
const cacheKey = `${snapshot.snapshotId}:${uniqueId}`;
|
|
3839
|
+
if (!dbtNodeDetailCache.has(cacheKey))
|
|
3840
|
+
dbtNodeDetailCache.set(cacheKey, dbtManifestPath ? loadDbtNodeAuthoringDetail(dbtManifestPath, uniqueId) : undefined);
|
|
3841
|
+
return dbtNodeDetailCache.get(cacheKey);
|
|
3842
|
+
}).filter(Boolean);
|
|
3843
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3844
|
+
res.end(serializeJSON({ requestId, details, snapshotId: snapshot.snapshotId }));
|
|
3845
|
+
return;
|
|
3846
|
+
}
|
|
3847
|
+
if (req.method === 'GET' && path === '/api/modeling/dbt-first/neighborhood') {
|
|
3848
|
+
const requestId = apiRequestId('modeling-neighborhood');
|
|
3849
|
+
const snapshot = projectSnapshot();
|
|
3850
|
+
const manifest = snapshot.manifest;
|
|
3851
|
+
const entityRef = url.searchParams.get('entity') ?? '';
|
|
3852
|
+
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 200));
|
|
3853
|
+
const entities = manifest.modeling?.entities ?? {};
|
|
3854
|
+
const relationships = manifest.modeling?.relationships ?? {};
|
|
3855
|
+
const entityKey = entities[entityRef] ? entityRef : Object.entries(entities).find(([, entity]) => entity.qualifiedId === entityRef)?.[0];
|
|
3856
|
+
if (!entityKey) {
|
|
3857
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3858
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'MODELING_ENTITY_NOT_FOUND', message: `modeling entity not found: ${entityRef}`, recoverable: false })));
|
|
3859
|
+
return;
|
|
3860
|
+
}
|
|
3861
|
+
const edges = Object.values(relationships).filter((relationship) => relationship.from === entityKey || relationship.to === entityKey).slice(0, limit);
|
|
3862
|
+
const keys = new Set([entityKey, ...edges.flatMap((edge) => [edge.from, edge.to])]);
|
|
3863
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3864
|
+
res.end(serializeJSON({
|
|
3865
|
+
requestId,
|
|
3866
|
+
entities: Object.fromEntries([...keys].flatMap((key) => entities[key] ? [[key, entities[key]]] : [])),
|
|
3867
|
+
relationships: Object.fromEntries(edges.map((edge) => [edge.qualifiedId, edge])),
|
|
3868
|
+
snapshotId: snapshot.snapshotId,
|
|
3869
|
+
}));
|
|
3870
|
+
return;
|
|
3871
|
+
}
|
|
3872
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/dbt-source/preview') {
|
|
3873
|
+
const requestId = apiRequestId('modeling-dbt-source-preview');
|
|
3874
|
+
const snapshot = projectSnapshot();
|
|
3875
|
+
try {
|
|
3876
|
+
const body = await readJSON(req);
|
|
3877
|
+
if (!body.change)
|
|
3878
|
+
throw Object.assign(new Error('A dbt source change is required.'), { code: 'INVALID_REQUEST' });
|
|
3879
|
+
if (body.expectedSnapshotId && body.expectedSnapshotId !== snapshot.snapshotId) {
|
|
3880
|
+
throw Object.assign(new Error('Project sources changed after the model was loaded.'), { code: 'SOURCE_CHANGED' });
|
|
3881
|
+
}
|
|
3882
|
+
const { dbtProjectDir, manifestPath } = onboardingDbtPaths({});
|
|
3883
|
+
const preview = previewDbtSourcePatch(dbtProjectDir, manifestPath, body.change);
|
|
3884
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3885
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, ...preview }));
|
|
3886
|
+
}
|
|
3887
|
+
catch (error) {
|
|
3888
|
+
const code = apiErrorCode(error, 'DBT_SOURCE_PATCH_INVALID');
|
|
3889
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3890
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh the model and review the dbt YAML source patch again.'] })));
|
|
3891
|
+
}
|
|
3892
|
+
return;
|
|
3893
|
+
}
|
|
3894
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/dbt-source/apply') {
|
|
3895
|
+
const requestId = apiRequestId('modeling-dbt-source-apply');
|
|
3896
|
+
const snapshot = projectSnapshot();
|
|
3897
|
+
try {
|
|
3898
|
+
const body = await readJSON(req);
|
|
3899
|
+
if (!body.change)
|
|
3900
|
+
throw Object.assign(new Error('A dbt source change is required.'), { code: 'INVALID_REQUEST' });
|
|
3901
|
+
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId) {
|
|
3902
|
+
throw Object.assign(new Error('Project sources changed after the source patch preview.'), { code: 'SOURCE_CHANGED' });
|
|
3903
|
+
}
|
|
3904
|
+
const expectedFingerprint = body.expectedFingerprint ?? body.fingerprint;
|
|
3905
|
+
if (!expectedFingerprint)
|
|
3906
|
+
throw Object.assign(new Error('The reviewed source patch fingerprint is required.'), { code: 'SOURCE_CHANGED' });
|
|
3907
|
+
const { dbtProjectDir, manifestPath } = onboardingDbtPaths({});
|
|
3908
|
+
const applied = applyDbtSourcePatch(dbtProjectDir, manifestPath, body.change, expectedFingerprint);
|
|
3909
|
+
projectSnapshots.invalidate();
|
|
3910
|
+
invalidateAgentProjectState(projectRoot);
|
|
3911
|
+
const nextSnapshot = projectSnapshot();
|
|
3912
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3913
|
+
res.end(serializeJSON({ requestId, snapshotId: nextSnapshot.snapshotId, applied }));
|
|
3914
|
+
}
|
|
3915
|
+
catch (error) {
|
|
3916
|
+
const inferred = /changed after the preview/i.test(apiErrorMessage(error)) ? 'SOURCE_CHANGED' : 'DBT_SOURCE_PATCH_INVALID';
|
|
3917
|
+
const code = apiErrorCode(error, inferred);
|
|
3918
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3919
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh the source patch preview before applying.'] })));
|
|
3920
|
+
}
|
|
3921
|
+
return;
|
|
3922
|
+
}
|
|
3923
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/preview') {
|
|
3924
|
+
const requestId = apiRequestId('modeling-preview');
|
|
3925
|
+
const snapshot = projectSnapshot();
|
|
3926
|
+
try {
|
|
3927
|
+
const body = await readJSON(req);
|
|
3928
|
+
if (!body.change)
|
|
3929
|
+
throw new Error('A modeling change is required.');
|
|
3930
|
+
if (body.expectedSnapshotId && body.expectedSnapshotId !== snapshot.snapshotId)
|
|
3931
|
+
throw Object.assign(new Error('Project sources changed after the model was loaded.'), { code: 'SOURCE_CHANGED' });
|
|
3932
|
+
const preview = previewModelingChange(projectRoot, body.change);
|
|
3933
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3934
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, ...preview }));
|
|
3935
|
+
}
|
|
3936
|
+
catch (error) {
|
|
3937
|
+
const code = apiErrorCode(error, 'MODELING_CHANGE_INVALID');
|
|
3938
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3939
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh Domain Studio and preview the change again.'] })));
|
|
3940
|
+
}
|
|
3941
|
+
return;
|
|
3942
|
+
}
|
|
3943
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/apply') {
|
|
3944
|
+
const requestId = apiRequestId('modeling-apply');
|
|
3945
|
+
const snapshot = projectSnapshot();
|
|
3946
|
+
try {
|
|
3947
|
+
const body = await readJSON(req);
|
|
3948
|
+
if (!body.change)
|
|
3949
|
+
throw new Error('A modeling change is required.');
|
|
3950
|
+
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId)
|
|
3951
|
+
throw Object.assign(new Error('Project sources changed after the modeling preview.'), { code: 'SOURCE_CHANGED' });
|
|
3952
|
+
if (!body.fingerprint)
|
|
3953
|
+
throw Object.assign(new Error('The reviewed modeling source fingerprint is required.'), { code: 'SOURCE_CHANGED' });
|
|
3954
|
+
const applied = applyModelingChange(projectRoot, body.change, body.fingerprint);
|
|
3955
|
+
projectSnapshots.invalidate();
|
|
3956
|
+
const nextSnapshot = projectSnapshot();
|
|
3957
|
+
const manifest = nextSnapshot.manifest;
|
|
3958
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
3959
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3960
|
+
res.end(serializeJSON({ requestId, snapshotId: nextSnapshot.snapshotId, applied, modeling: manifest.modeling, diagnostics: manifest.diagnostics ?? [] }));
|
|
3961
|
+
}
|
|
3962
|
+
catch (error) {
|
|
3963
|
+
const inferred = /changed after the preview/i.test(apiErrorMessage(error)) ? 'SOURCE_CHANGED' : 'MODELING_APPLY_FAILED';
|
|
3964
|
+
const code = apiErrorCode(error, inferred);
|
|
3965
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3966
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh Domain Studio and review the latest source diff.'] })));
|
|
3967
|
+
}
|
|
3968
|
+
return;
|
|
3969
|
+
}
|
|
3970
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/relationships/validate') {
|
|
3971
|
+
const requestId = apiRequestId('modeling-relationship-validate');
|
|
3972
|
+
const snapshot = projectSnapshot();
|
|
3973
|
+
try {
|
|
3974
|
+
const body = await readJSON(req);
|
|
3975
|
+
if (!body.relationship)
|
|
3976
|
+
throw new Error('A relationship is required.');
|
|
3977
|
+
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId)
|
|
3978
|
+
throw Object.assign(new Error('Project sources changed before relationship validation.'), { code: 'SOURCE_CHANGED' });
|
|
3979
|
+
const manifest = snapshot.manifest;
|
|
3980
|
+
const activeConnection = requireActiveConnection();
|
|
3981
|
+
const evidence = await validateModelingRelationship(body.relationship, manifest, (sql) => executor.executeQuery(sql, [], {}, activeConnection), (identifier) => getDialect(activeConnection.driver).quoteIdentifier(identifier));
|
|
3982
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3983
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, evidence }));
|
|
3984
|
+
}
|
|
3985
|
+
catch (error) {
|
|
3986
|
+
const code = apiErrorCode(error, 'RELATIONSHIP_VALIDATION_FAILED');
|
|
3987
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3988
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh the model or configure a warehouse connection, then retry validation.'] })));
|
|
3989
|
+
}
|
|
3990
|
+
return;
|
|
3991
|
+
}
|
|
2647
3992
|
if (req.method === 'GET' && path === '/api/agent-runs') {
|
|
2648
3993
|
const rawLimit = Number(url.searchParams.get('limit'));
|
|
2649
3994
|
const limit = Number.isFinite(rawLimit) && rawLimit > 0
|
|
@@ -3036,6 +4381,11 @@ export async function startLocalServer(opts) {
|
|
|
3036
4381
|
selection: typeof body?.context?.selection === 'string' ? body.context.selection : undefined,
|
|
3037
4382
|
};
|
|
3038
4383
|
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
4384
|
+
const domain = typeof body?.domain === 'string' && body.domain.trim() ? body.domain.trim() : undefined;
|
|
4385
|
+
const modelAreaId = typeof body?.modelAreaId === 'string' && body.modelAreaId.trim() ? body.modelAreaId.trim() : undefined;
|
|
4386
|
+
const purpose = typeof body?.purpose === 'string'
|
|
4387
|
+
? body.purpose?.trim() || undefined
|
|
4388
|
+
: undefined;
|
|
3039
4389
|
const userId = typeof body?.userId === 'string'
|
|
3040
4390
|
? body.userId
|
|
3041
4391
|
: undefined;
|
|
@@ -3060,6 +4410,14 @@ export async function startLocalServer(opts) {
|
|
|
3060
4410
|
catch {
|
|
3061
4411
|
// Best-effort: a failed reindex falls back to whatever KG exists (or none).
|
|
3062
4412
|
}
|
|
4413
|
+
const { domainContext, contextPack } = await buildDirectBuildContext({
|
|
4414
|
+
question: prompt,
|
|
4415
|
+
domain,
|
|
4416
|
+
purpose,
|
|
4417
|
+
modelAreaId,
|
|
4418
|
+
surface: target === 'cell' ? 'notebook' : 'block',
|
|
4419
|
+
selectedContext: { target, mode, blockPath, owner },
|
|
4420
|
+
});
|
|
3063
4421
|
// Inject user-authored Skills as business context; the engine selects the
|
|
3064
4422
|
// relevant subset and stamps `appliedSkills` on the result.
|
|
3065
4423
|
const skills = loadSkills(projectRoot).skills;
|
|
@@ -3071,6 +4429,10 @@ export async function startLocalServer(opts) {
|
|
|
3071
4429
|
mode,
|
|
3072
4430
|
blockPath,
|
|
3073
4431
|
owner,
|
|
4432
|
+
domain,
|
|
4433
|
+
modelAreaId,
|
|
4434
|
+
contextPack,
|
|
4435
|
+
domainContext,
|
|
3074
4436
|
userId,
|
|
3075
4437
|
skills,
|
|
3076
4438
|
dbtManifestPath: resolveDbtManifestPath(projectRoot, loadProjectConfig(projectRoot)),
|
|
@@ -3127,6 +4489,52 @@ export async function startLocalServer(opts) {
|
|
|
3127
4489
|
// ── Skills (spec 16) — user-authored business context. AI drafts, humans
|
|
3128
4490
|
// certify; skills never carry certification. PROJECT skills (user empty)
|
|
3129
4491
|
// are shared; PERSONAL skills (user set) are user-bound. ────────────────
|
|
4492
|
+
if (req.method === 'GET' && path === '/api/skills/settings') {
|
|
4493
|
+
try {
|
|
4494
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4495
|
+
res.end(serializeJSON(buildSkillPathSettings(projectRoot)));
|
|
4496
|
+
}
|
|
4497
|
+
catch (error) {
|
|
4498
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4499
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
4500
|
+
}
|
|
4501
|
+
return;
|
|
4502
|
+
}
|
|
4503
|
+
if (req.method === 'PUT' && path === '/api/skills/settings') {
|
|
4504
|
+
try {
|
|
4505
|
+
const body = (await readJSON(req).catch(() => ({})));
|
|
4506
|
+
const requestedPath = typeof body.path === 'string' ? body.path.trim() : '';
|
|
4507
|
+
if (!requestedPath || requestedPath.includes('\0')) {
|
|
4508
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4509
|
+
res.end(serializeJSON({ error: 'Provide an existing Skills folder path.' }));
|
|
4510
|
+
return;
|
|
4511
|
+
}
|
|
4512
|
+
const resolvedPath = resolve(projectRoot, requestedPath);
|
|
4513
|
+
if (!existsSync(resolvedPath) || !statSync(resolvedPath).isDirectory()) {
|
|
4514
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4515
|
+
res.end(serializeJSON({ error: `Skills folder not found: ${resolvedPath}` }));
|
|
4516
|
+
return;
|
|
4517
|
+
}
|
|
4518
|
+
const configPath = join(projectRoot, 'dql.config.json');
|
|
4519
|
+
const raw = existsSync(configPath)
|
|
4520
|
+
? JSON.parse(readFileSync(configPath, 'utf-8'))
|
|
4521
|
+
: {};
|
|
4522
|
+
const layout = raw.layout && typeof raw.layout === 'object' && !Array.isArray(raw.layout)
|
|
4523
|
+
? raw.layout
|
|
4524
|
+
: {};
|
|
4525
|
+
raw.layout = { ...layout, skillsPath: requestedPath };
|
|
4526
|
+
writeFileSync(configPath, `${JSON.stringify(raw, null, 2)}\n`, 'utf-8');
|
|
4527
|
+
projectConfig = loadProjectConfig(projectRoot);
|
|
4528
|
+
invalidateAgentProjectState(projectRoot);
|
|
4529
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4530
|
+
res.end(serializeJSON(buildSkillPathSettings(projectRoot)));
|
|
4531
|
+
}
|
|
4532
|
+
catch (error) {
|
|
4533
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4534
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
4535
|
+
}
|
|
4536
|
+
return;
|
|
4537
|
+
}
|
|
3130
4538
|
if (req.method === 'GET' && path === '/api/skills') {
|
|
3131
4539
|
try {
|
|
3132
4540
|
const skills = loadSkills(projectRoot).skills
|
|
@@ -3240,9 +4648,12 @@ export async function startLocalServer(opts) {
|
|
|
3240
4648
|
res.end(serializeJSON({ error: 'Provide { domain } with a non-empty name.' }));
|
|
3241
4649
|
return;
|
|
3242
4650
|
}
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
4651
|
+
applyModelingChange(projectRoot, {
|
|
4652
|
+
operation: 'upsert_domain',
|
|
4653
|
+
value: { ...input, id: input.id ?? domainFolderSlug(input.name).replace(/-/g, '_') },
|
|
4654
|
+
});
|
|
4655
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
4656
|
+
const domain = findDomain(projectRoot, input.id ?? input.name);
|
|
3246
4657
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3247
4658
|
res.end(serializeJSON({ domain }));
|
|
3248
4659
|
}
|
|
@@ -3263,12 +4674,15 @@ export async function startLocalServer(opts) {
|
|
|
3263
4674
|
return;
|
|
3264
4675
|
}
|
|
3265
4676
|
// If the name changed, remove the old declaration so we never orphan one.
|
|
3266
|
-
if (
|
|
3267
|
-
|
|
4677
|
+
if (domainKey(id) !== domainKey(input.id ?? input.name)) {
|
|
4678
|
+
deleteCanonicalDomainDeclaration(projectRoot, id);
|
|
3268
4679
|
}
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
4680
|
+
applyModelingChange(projectRoot, {
|
|
4681
|
+
operation: 'upsert_domain',
|
|
4682
|
+
value: { ...input, id: input.id ?? id },
|
|
4683
|
+
});
|
|
4684
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
4685
|
+
const domain = findDomain(projectRoot, input.id ?? input.name);
|
|
3272
4686
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3273
4687
|
res.end(serializeJSON({ domain }));
|
|
3274
4688
|
}
|
|
@@ -3281,8 +4695,8 @@ export async function startLocalServer(opts) {
|
|
|
3281
4695
|
if (req.method === 'DELETE' && path.startsWith('/api/domains/')) {
|
|
3282
4696
|
try {
|
|
3283
4697
|
const id = decodeURIComponent(path.slice('/api/domains/'.length));
|
|
3284
|
-
|
|
3285
|
-
await
|
|
4698
|
+
deleteCanonicalDomainDeclaration(projectRoot, id);
|
|
4699
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
3286
4700
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3287
4701
|
res.end(serializeJSON({ ok: true }));
|
|
3288
4702
|
}
|
|
@@ -3377,8 +4791,12 @@ export async function startLocalServer(opts) {
|
|
|
3377
4791
|
}
|
|
3378
4792
|
try {
|
|
3379
4793
|
if (candidate.kind === 'domain' && candidate.domain) {
|
|
3380
|
-
const
|
|
3381
|
-
|
|
4794
|
+
const domainId = candidate.domain.id ?? domainFolderSlug(candidate.domain.name).replace(/-/g, '_');
|
|
4795
|
+
const written = applyModelingChange(projectRoot, {
|
|
4796
|
+
operation: 'upsert_domain',
|
|
4797
|
+
value: { ...candidate.domain, id: domainId },
|
|
4798
|
+
});
|
|
4799
|
+
saved.push({ id: candidate.id, path: written.patches[0]?.path, status: 'saved' });
|
|
3382
4800
|
}
|
|
3383
4801
|
else if (candidate.kind === 'skill' && candidate.skill) {
|
|
3384
4802
|
const skill = writeSkill(projectRoot, { ...candidate.skill, scope: 'project', status: 'draft' });
|
|
@@ -3392,7 +4810,7 @@ export async function startLocalServer(opts) {
|
|
|
3392
4810
|
saved.push({ id: candidate.id, status: 'blocked', blockers: [error instanceof Error ? error.message : String(error)] });
|
|
3393
4811
|
}
|
|
3394
4812
|
}
|
|
3395
|
-
await
|
|
4813
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
3396
4814
|
updateContextBootstrapSession(projectRoot, id, (current) => ({ ...current, closedAt: new Date().toISOString() }));
|
|
3397
4815
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3398
4816
|
res.end(serializeJSON({ id, saved }));
|
|
@@ -3819,6 +5237,33 @@ export async function startLocalServer(opts) {
|
|
|
3819
5237
|
}
|
|
3820
5238
|
return;
|
|
3821
5239
|
}
|
|
5240
|
+
const appStoryRun = path.match(/^\/api\/apps\/([^/]+)\/dashboards\/([^/]+)\/story$/);
|
|
5241
|
+
if (req.method === 'POST' && appStoryRun) {
|
|
5242
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
5243
|
+
const runId = typeof body.runId === 'string' ? body.runId : '';
|
|
5244
|
+
const evidence = dashboardRunEvidence.get(runId);
|
|
5245
|
+
const appId = decodeURIComponent(appStoryRun[1]);
|
|
5246
|
+
const dashboardId = decodeURIComponent(appStoryRun[2]);
|
|
5247
|
+
if (!evidence || evidence.expiresAt < Date.now() || evidence.appId !== appId || evidence.dashboardId !== dashboardId) {
|
|
5248
|
+
dashboardRunEvidence.delete(runId);
|
|
5249
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5250
|
+
res.end(serializeJSON({ error: 'Dashboard run evidence expired or does not match this App. Run the dashboard again.' }));
|
|
5251
|
+
return;
|
|
5252
|
+
}
|
|
5253
|
+
// The endpoint accepts runId only. Optional provider wording can be added
|
|
5254
|
+
// here later; the claim-checked deterministic brief is always available.
|
|
5255
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5256
|
+
res.end(serializeJSON({
|
|
5257
|
+
runId,
|
|
5258
|
+
snapshotId: evidence.snapshotId,
|
|
5259
|
+
filterFingerprint: evidence.filterFingerprint,
|
|
5260
|
+
resultFingerprint: evidence.resultFingerprint,
|
|
5261
|
+
personaFingerprint: evidence.personaFingerprint,
|
|
5262
|
+
facts: evidence.facts,
|
|
5263
|
+
story: evidence.story,
|
|
5264
|
+
}));
|
|
5265
|
+
return;
|
|
5266
|
+
}
|
|
3822
5267
|
const appDashRun = path.match(/^\/api\/apps\/([^/]+)\/dashboards\/([^/]+)\/run$/);
|
|
3823
5268
|
if (req.method === 'POST' && appDashRun) {
|
|
3824
5269
|
try {
|
|
@@ -3831,6 +5276,12 @@ export async function startLocalServer(opts) {
|
|
|
3831
5276
|
res.end(serializeJSON({ error: `Dashboard "${dashboardId}" not found in app "${appId}"` }));
|
|
3832
5277
|
return;
|
|
3833
5278
|
}
|
|
5279
|
+
const activeApp = activePersonaAppId();
|
|
5280
|
+
if (activeApp && activeApp !== appId) {
|
|
5281
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5282
|
+
res.end(serializeJSON({ error: `The active persona belongs to App "${activeApp}", not "${appId}".` }));
|
|
5283
|
+
return;
|
|
5284
|
+
}
|
|
3834
5285
|
const manifest = buildManifest({ projectRoot });
|
|
3835
5286
|
const variables = body.variables && typeof body.variables === 'object'
|
|
3836
5287
|
? body.variables
|
|
@@ -3873,7 +5324,38 @@ export async function startLocalServer(opts) {
|
|
|
3873
5324
|
});
|
|
3874
5325
|
continue;
|
|
3875
5326
|
}
|
|
3876
|
-
|
|
5327
|
+
const pinPlan = pin.analysisPlan && typeof pin.analysisPlan === 'object' && !Array.isArray(pin.analysisPlan)
|
|
5328
|
+
? pin.analysisPlan
|
|
5329
|
+
: undefined;
|
|
5330
|
+
const pinArtifact = normalizeDqlArtifactReference(pinPlan?.dqlArtifact);
|
|
5331
|
+
let renderedPinResult = pin.result;
|
|
5332
|
+
let renderedPinChartConfig = pin.chartConfig;
|
|
5333
|
+
if (pinArtifact) {
|
|
5334
|
+
try {
|
|
5335
|
+
const boundParameters = dashboardTileParameterValues({
|
|
5336
|
+
item,
|
|
5337
|
+
dashboardValues: dashboardVariables,
|
|
5338
|
+
requestValues: variables,
|
|
5339
|
+
});
|
|
5340
|
+
const executed = await executeDqlArtifactSourceForAgent(pinArtifact.source, { name: pinArtifact.name ?? pin.title, path: pinArtifact.sourcePath }, { question: pin.question, parameters: { ...(pinArtifact.parameterValues ?? {}), ...boundParameters } });
|
|
5341
|
+
renderedPinResult = executed;
|
|
5342
|
+
renderedPinChartConfig = executed.chartConfig && typeof executed.chartConfig === 'object'
|
|
5343
|
+
? executed.chartConfig
|
|
5344
|
+
: renderedPinChartConfig;
|
|
5345
|
+
}
|
|
5346
|
+
catch (err) {
|
|
5347
|
+
tiles.push({
|
|
5348
|
+
tileId: item.i,
|
|
5349
|
+
status: 'error',
|
|
5350
|
+
tileType: 'aiPin',
|
|
5351
|
+
title: item.title ?? pin.title,
|
|
5352
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5353
|
+
trustState: pinArtifact.trustState ?? 'review_required',
|
|
5354
|
+
});
|
|
5355
|
+
continue;
|
|
5356
|
+
}
|
|
5357
|
+
}
|
|
5358
|
+
else if (pin.refreshCadence === 'daily' && pin.sql && isAiPinRefreshDue(pin.lastRefreshedAt)) {
|
|
3877
5359
|
try {
|
|
3878
5360
|
const refreshed = await executeLocalSqlForStoredResult(pin.sql);
|
|
3879
5361
|
pin = localApps.updateAiPinResult(pin.id, refreshed) ?? pin;
|
|
@@ -3888,8 +5370,8 @@ export async function startLocalServer(opts) {
|
|
|
3888
5370
|
tileType: 'aiPin',
|
|
3889
5371
|
title: item.title ?? pin.title,
|
|
3890
5372
|
viz: item.viz,
|
|
3891
|
-
chartConfig: mergeDashboardChartConfig(
|
|
3892
|
-
result:
|
|
5373
|
+
chartConfig: mergeDashboardChartConfig(renderedPinChartConfig, item),
|
|
5374
|
+
result: renderedPinResult,
|
|
3893
5375
|
aiPin: pin,
|
|
3894
5376
|
citation: {
|
|
3895
5377
|
kind: 'ai_pin',
|
|
@@ -3898,6 +5380,70 @@ export async function startLocalServer(opts) {
|
|
|
3898
5380
|
});
|
|
3899
5381
|
continue;
|
|
3900
5382
|
}
|
|
5383
|
+
if (item.semantic) {
|
|
5384
|
+
try {
|
|
5385
|
+
if (!semanticLayer)
|
|
5386
|
+
throw new Error('Semantic layer is not available for this governed query.');
|
|
5387
|
+
const knownMetrics = new Set(semanticLayer.listMetrics().map((metric) => metric.name));
|
|
5388
|
+
const missingMetrics = item.semantic.metrics.filter((metric) => !knownMetrics.has(metric));
|
|
5389
|
+
if (missingMetrics.length)
|
|
5390
|
+
throw new Error(`Semantic definition changed; missing metric(s): ${missingMetrics.join(', ')}`);
|
|
5391
|
+
const knownModels = new Set(semanticLayer.listSemanticModels().map((model) => model.name));
|
|
5392
|
+
const missingModels = item.semantic.semanticModelRefs.filter((model) => !knownModels.has(model));
|
|
5393
|
+
if (missingModels.length)
|
|
5394
|
+
throw new Error(`Semantic definition changed; missing model(s): ${missingModels.join(', ')}`);
|
|
5395
|
+
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
5396
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
5397
|
+
const activeFilters = Object.entries(dashboardVariables)
|
|
5398
|
+
.filter(([field, value]) => item.semantic.dimensions?.includes(field) && value !== undefined && value !== null && value !== '')
|
|
5399
|
+
.map(([dimension, value]) => ({ dimension, operator: 'equals', values: Array.isArray(value) ? value.map(String) : [String(value)] }));
|
|
5400
|
+
const staticFilters = (item.semantic.filters ?? []).map((filter) => ({
|
|
5401
|
+
dimension: filter.field,
|
|
5402
|
+
operator: filter.operator,
|
|
5403
|
+
values: Array.isArray(filter.value) ? filter.value.map(String) : [String(filter.value)],
|
|
5404
|
+
}));
|
|
5405
|
+
const composed = composeRuntimeSemanticQuery({
|
|
5406
|
+
metrics: item.semantic.metrics,
|
|
5407
|
+
dimensions: item.semantic.dimensions ?? [],
|
|
5408
|
+
filters: [...staticFilters, ...activeFilters],
|
|
5409
|
+
...(item.semantic.timeDimension ? { timeDimension: { name: item.semantic.timeDimension, granularity: 'month' } } : {}),
|
|
5410
|
+
orderBy: item.semantic.orderBy?.map((order) => ({ name: order.field, direction: order.direction })),
|
|
5411
|
+
limit: item.semantic.limit,
|
|
5412
|
+
engine: item.semantic.provider,
|
|
5413
|
+
}, semanticLayer, {
|
|
5414
|
+
projectRoot,
|
|
5415
|
+
projectConfig,
|
|
5416
|
+
detectedProvider: semanticDetectedProvider,
|
|
5417
|
+
driver: targetConnection.driver,
|
|
5418
|
+
tableMapping,
|
|
5419
|
+
});
|
|
5420
|
+
if (!composed)
|
|
5421
|
+
throw new Error('The governed semantic query could not be composed.');
|
|
5422
|
+
const prepared = prepareLocalExecution(composed.sql, targetConnection, projectRoot, projectConfig);
|
|
5423
|
+
const result = await executor.executeQuery(prepared.sql, [], {}, prepared.connection);
|
|
5424
|
+
tiles.push({
|
|
5425
|
+
tileId: item.i,
|
|
5426
|
+
status: 'ok',
|
|
5427
|
+
tileType: 'semantic',
|
|
5428
|
+
title: item.title ?? item.semantic.id,
|
|
5429
|
+
viz: item.viz,
|
|
5430
|
+
result: normalizeQueryResult(result),
|
|
5431
|
+
trustState: 'review_required',
|
|
5432
|
+
citation: { kind: 'semantic_query', name: item.semantic.id },
|
|
5433
|
+
});
|
|
5434
|
+
}
|
|
5435
|
+
catch (err) {
|
|
5436
|
+
tiles.push({
|
|
5437
|
+
tileId: item.i,
|
|
5438
|
+
status: 'error',
|
|
5439
|
+
tileType: 'semantic',
|
|
5440
|
+
title: item.title ?? item.semantic.id,
|
|
5441
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5442
|
+
trustState: 'review_required',
|
|
5443
|
+
});
|
|
5444
|
+
}
|
|
5445
|
+
continue;
|
|
5446
|
+
}
|
|
3901
5447
|
const block = resolveDashboardItemBlock(item, manifest);
|
|
3902
5448
|
if (!block) {
|
|
3903
5449
|
tiles.push({
|
|
@@ -3918,6 +5464,34 @@ export async function startLocalServer(opts) {
|
|
|
3918
5464
|
const source = readFileSync(absBlockPath, 'utf-8');
|
|
3919
5465
|
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
3920
5466
|
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
5467
|
+
const boundParameters = dashboardTileParameterValues({
|
|
5468
|
+
item,
|
|
5469
|
+
dashboardValues: dashboardVariables,
|
|
5470
|
+
requestValues: variables,
|
|
5471
|
+
});
|
|
5472
|
+
const invocation = prepareBlockInvocation({
|
|
5473
|
+
block: block.name,
|
|
5474
|
+
source,
|
|
5475
|
+
parameters: { ...dashboardVariables, ...boundParameters },
|
|
5476
|
+
parameterSources: Object.fromEntries(Object.keys(boundParameters).map((name) => [name, 'surface'])),
|
|
5477
|
+
surface: 'app',
|
|
5478
|
+
});
|
|
5479
|
+
if (invocation.errors.length || invocation.unresolvedParameters.length) {
|
|
5480
|
+
tiles.push({
|
|
5481
|
+
tileId: item.i,
|
|
5482
|
+
status: 'unresolved',
|
|
5483
|
+
tileType: 'block',
|
|
5484
|
+
blockId: block.name,
|
|
5485
|
+
title: item.title ?? block.name,
|
|
5486
|
+
error: invocation.errors[0] ?? `Needs values for: ${invocation.unresolvedParameters.join(', ')}`,
|
|
5487
|
+
invocation: {
|
|
5488
|
+
resolvedParameters: invocation.resolvedParameters,
|
|
5489
|
+
unresolvedParameters: invocation.unresolvedParameters,
|
|
5490
|
+
auditId: invocation.auditId,
|
|
5491
|
+
},
|
|
5492
|
+
});
|
|
5493
|
+
continue;
|
|
5494
|
+
}
|
|
3921
5495
|
const semanticCompose = semanticLayer
|
|
3922
5496
|
? composeSemanticBlockSql(source, semanticLayer, {
|
|
3923
5497
|
driver: targetConnection.driver,
|
|
@@ -3925,6 +5499,7 @@ export async function startLocalServer(opts) {
|
|
|
3925
5499
|
projectRoot,
|
|
3926
5500
|
projectConfig,
|
|
3927
5501
|
detectedProvider: semanticDetectedProvider,
|
|
5502
|
+
parameters: invocation.values,
|
|
3928
5503
|
})
|
|
3929
5504
|
: null;
|
|
3930
5505
|
const plan = buildExecutionPlan({ id: item.i, type: 'dql', source, title: item.title ?? block.name }, { semanticLayer, driver: targetConnection.driver, tableMapping });
|
|
@@ -3945,7 +5520,7 @@ export async function startLocalServer(opts) {
|
|
|
3945
5520
|
dashboard: loaded.dashboard,
|
|
3946
5521
|
});
|
|
3947
5522
|
const prepared = prepareLocalExecution(filterApplication.sql, targetConnection, projectRoot, projectConfig);
|
|
3948
|
-
const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables(filterApplication.variables), prepared.connection);
|
|
5523
|
+
const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables({ ...filterApplication.variables, ...invocation.values }), prepared.connection);
|
|
3949
5524
|
tiles.push({
|
|
3950
5525
|
tileId: item.i,
|
|
3951
5526
|
status: 'ok',
|
|
@@ -3960,6 +5535,11 @@ export async function startLocalServer(opts) {
|
|
|
3960
5535
|
applied: filterApplication.appliedFilters,
|
|
3961
5536
|
skipped: filterApplication.skippedFilters,
|
|
3962
5537
|
},
|
|
5538
|
+
invocation: {
|
|
5539
|
+
resolvedParameters: invocation.resolvedParameters,
|
|
5540
|
+
unresolvedParameters: invocation.unresolvedParameters,
|
|
5541
|
+
auditId: invocation.auditId,
|
|
5542
|
+
},
|
|
3963
5543
|
citation: {
|
|
3964
5544
|
kind: 'block',
|
|
3965
5545
|
name: block.name,
|
|
@@ -3987,12 +5567,55 @@ export async function startLocalServer(opts) {
|
|
|
3987
5567
|
}
|
|
3988
5568
|
}
|
|
3989
5569
|
localApps?.close();
|
|
5570
|
+
const snapshot = projectSnapshot();
|
|
5571
|
+
const filterFingerprint = createHash('sha256').update(JSON.stringify(dashboardVariables)).digest('hex');
|
|
5572
|
+
const resultFingerprint = createHash('sha256').update(JSON.stringify(tiles.map((tile) => ({ tileId: tile.tileId, status: tile.status, result: tile.result })))).digest('hex');
|
|
5573
|
+
const personaFingerprint = createHash('sha256').update(activePersonaAppId() ?? 'global').digest('hex');
|
|
5574
|
+
const runId = `app_run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
5575
|
+
const storyResult = buildDeterministicDashboardStory({
|
|
5576
|
+
goal: loaded.dashboard.story?.goal ?? loaded.dashboard.metadata.businessOutcome ?? loaded.dashboard.metadata.title,
|
|
5577
|
+
audience: loaded.dashboard.story?.audience ?? loaded.dashboard.metadata.audience,
|
|
5578
|
+
filters: dashboardVariables,
|
|
5579
|
+
tiles: tiles.map((tile) => ({
|
|
5580
|
+
tileId: tile.tileId,
|
|
5581
|
+
title: typeof tile.title === 'string' ? tile.title : tile.tileId,
|
|
5582
|
+
status: tile.status,
|
|
5583
|
+
trustState: (tile.trustState === 'certified' || tile.trustState === 'review_required' || tile.trustState === 'draft_ready'
|
|
5584
|
+
? tile.trustState
|
|
5585
|
+
: tile.certificationStatus === 'certified' ? 'certified' : 'review_required'),
|
|
5586
|
+
result: tile.result,
|
|
5587
|
+
citation: tile.citation,
|
|
5588
|
+
})),
|
|
5589
|
+
eligibleTileIds: loaded.dashboard.story?.eligibleTileIds,
|
|
5590
|
+
driverTileIds: loaded.dashboard.story?.driverTileIds,
|
|
5591
|
+
});
|
|
5592
|
+
dashboardRunEvidence.set(runId, {
|
|
5593
|
+
appId,
|
|
5594
|
+
dashboardId,
|
|
5595
|
+
snapshotId: snapshot.snapshotId,
|
|
5596
|
+
filterFingerprint,
|
|
5597
|
+
resultFingerprint,
|
|
5598
|
+
personaFingerprint,
|
|
5599
|
+
facts: storyResult.facts,
|
|
5600
|
+
story: storyResult.story,
|
|
5601
|
+
expiresAt: Date.now() + 15 * 60_000,
|
|
5602
|
+
});
|
|
5603
|
+
for (const [id, evidence] of dashboardRunEvidence)
|
|
5604
|
+
if (evidence.expiresAt < Date.now())
|
|
5605
|
+
dashboardRunEvidence.delete(id);
|
|
3990
5606
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3991
5607
|
res.end(serializeJSON({
|
|
3992
5608
|
appId,
|
|
3993
5609
|
dashboardId,
|
|
3994
5610
|
persona: activePersonaAppId() ? { appId: activePersonaAppId() } : null,
|
|
5611
|
+
runId,
|
|
5612
|
+
snapshotId: snapshot.snapshotId,
|
|
5613
|
+
filterFingerprint,
|
|
5614
|
+
resultFingerprint,
|
|
5615
|
+
personaFingerprint,
|
|
3995
5616
|
tiles,
|
|
5617
|
+
facts: storyResult.facts,
|
|
5618
|
+
story: storyResult.story,
|
|
3996
5619
|
}));
|
|
3997
5620
|
}
|
|
3998
5621
|
catch (err) {
|
|
@@ -4731,6 +6354,23 @@ export async function startLocalServer(opts) {
|
|
|
4731
6354
|
}
|
|
4732
6355
|
return;
|
|
4733
6356
|
}
|
|
6357
|
+
if (req.method === 'POST' && path === '/api/git/review/open') {
|
|
6358
|
+
try {
|
|
6359
|
+
const body = (await readJSON(req));
|
|
6360
|
+
const result = await gitOpenReview(projectRoot, {
|
|
6361
|
+
title: body.title ?? '',
|
|
6362
|
+
body: body.body ?? '',
|
|
6363
|
+
base: body.base ?? 'main',
|
|
6364
|
+
});
|
|
6365
|
+
res.writeHead(result.ok ? 201 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6366
|
+
res.end(serializeJSON(result));
|
|
6367
|
+
}
|
|
6368
|
+
catch (e) {
|
|
6369
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6370
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
6371
|
+
}
|
|
6372
|
+
return;
|
|
6373
|
+
}
|
|
4734
6374
|
if (req.method === 'POST' && path === '/api/git/pull') {
|
|
4735
6375
|
try {
|
|
4736
6376
|
const result = await gitPull(projectRoot);
|
|
@@ -4772,6 +6412,21 @@ export async function startLocalServer(opts) {
|
|
|
4772
6412
|
if (req.method === 'GET' && path === '/api/schema') {
|
|
4773
6413
|
try {
|
|
4774
6414
|
const dataFiles = scanDataFiles(projectRoot);
|
|
6415
|
+
const notebookDatasets = datasetWorkspace.list().map((dataset) => ({
|
|
6416
|
+
name: dataset.alias,
|
|
6417
|
+
path: dataset.sourcePath,
|
|
6418
|
+
columns: dataset.profile.columns.map((column) => ({
|
|
6419
|
+
name: column.name,
|
|
6420
|
+
type: column.type,
|
|
6421
|
+
})),
|
|
6422
|
+
source: "file",
|
|
6423
|
+
objectType: dataset.storageMode === "staged" ? "staged_dataset" : "dataset",
|
|
6424
|
+
datasetId: dataset.id,
|
|
6425
|
+
fileFingerprint: dataset.fileFingerprint,
|
|
6426
|
+
storageMode: dataset.storageMode,
|
|
6427
|
+
refreshedAt: dataset.refreshedAt,
|
|
6428
|
+
trustState: dataset.trustState,
|
|
6429
|
+
}));
|
|
4775
6430
|
const { tables, columnsByPath } = connection
|
|
4776
6431
|
? await introspectSchema(executor, connection)
|
|
4777
6432
|
: { tables: [], columnsByPath: new Map() };
|
|
@@ -4782,9 +6437,13 @@ export async function startLocalServer(opts) {
|
|
|
4782
6437
|
source: 'database',
|
|
4783
6438
|
objectType: t.type,
|
|
4784
6439
|
}));
|
|
4785
|
-
const seen = new Set(
|
|
6440
|
+
const seen = new Set([
|
|
6441
|
+
...dataFiles.map((f) => f.name),
|
|
6442
|
+
...notebookDatasets.map((dataset) => dataset.name),
|
|
6443
|
+
]);
|
|
4786
6444
|
const merged = [
|
|
4787
|
-
...
|
|
6445
|
+
...notebookDatasets,
|
|
6446
|
+
...dataFiles.map((f) => ({ ...f, source: "file" })),
|
|
4788
6447
|
...dbTables.filter((t) => !seen.has(t.name)),
|
|
4789
6448
|
];
|
|
4790
6449
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -4793,8 +6452,27 @@ export async function startLocalServer(opts) {
|
|
|
4793
6452
|
catch (error) {
|
|
4794
6453
|
const message = error instanceof Error ? error.message : String(error);
|
|
4795
6454
|
console.warn(`[dql] /api/schema introspection failed: ${message}`);
|
|
4796
|
-
const fallback =
|
|
4797
|
-
|
|
6455
|
+
const fallback = [
|
|
6456
|
+
...datasetWorkspace.list().map((dataset) => ({
|
|
6457
|
+
name: dataset.alias,
|
|
6458
|
+
path: dataset.sourcePath,
|
|
6459
|
+
columns: dataset.profile.columns.map((column) => ({
|
|
6460
|
+
name: column.name,
|
|
6461
|
+
type: column.type,
|
|
6462
|
+
})),
|
|
6463
|
+
source: "file",
|
|
6464
|
+
objectType: dataset.storageMode === "staged" ? "staged_dataset" : "dataset",
|
|
6465
|
+
datasetId: dataset.id,
|
|
6466
|
+
fileFingerprint: dataset.fileFingerprint,
|
|
6467
|
+
storageMode: dataset.storageMode,
|
|
6468
|
+
refreshedAt: dataset.refreshedAt,
|
|
6469
|
+
trustState: dataset.trustState,
|
|
6470
|
+
})),
|
|
6471
|
+
...scanDataFiles(projectRoot).map((f) => ({ ...f, source: "file" })),
|
|
6472
|
+
];
|
|
6473
|
+
res.writeHead(200, {
|
|
6474
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
6475
|
+
});
|
|
4798
6476
|
res.end(serializeJSON(fallback));
|
|
4799
6477
|
}
|
|
4800
6478
|
return;
|
|
@@ -4808,6 +6486,11 @@ export async function startLocalServer(opts) {
|
|
|
4808
6486
|
res.end(serializeJSON({ error: 'Missing block name' }));
|
|
4809
6487
|
return;
|
|
4810
6488
|
}
|
|
6489
|
+
if (dbtFirstOwnsSemanticAuthoring(projectConfig)
|
|
6490
|
+
&& (blockType === 'semantic' || /\btype\s*=\s*["']semantic["']/i.test(content ?? ''))) {
|
|
6491
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
6492
|
+
return;
|
|
6493
|
+
}
|
|
4811
6494
|
const created = createBlockArtifacts(projectRoot, {
|
|
4812
6495
|
name,
|
|
4813
6496
|
domain,
|
|
@@ -4835,12 +6518,16 @@ export async function startLocalServer(opts) {
|
|
|
4835
6518
|
if (req.method === 'POST' && path === '/api/blocks/save-from-cell') {
|
|
4836
6519
|
try {
|
|
4837
6520
|
const body = await readJSON(req);
|
|
4838
|
-
const { name, domain, owner, content, description, tags, metricRefs, template, llmContext, examples, invariants, } = body;
|
|
6521
|
+
const { name, domain, owner, content, description, tags, metricRefs, template, llmContext, examples, invariants, reviewRequired, datasetRefs, lineage, } = body;
|
|
4839
6522
|
if (!name || typeof name !== 'string' || !content || typeof content !== 'string') {
|
|
4840
6523
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4841
6524
|
res.end(serializeJSON({ error: 'name and content are required' }));
|
|
4842
6525
|
return;
|
|
4843
6526
|
}
|
|
6527
|
+
if (dbtFirstOwnsSemanticAuthoring(projectConfig) && /\btype\s*=\s*["']semantic["']/i.test(content)) {
|
|
6528
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
6529
|
+
return;
|
|
6530
|
+
}
|
|
4844
6531
|
// OSS keeps the save flow deliberately small: the person saving the
|
|
4845
6532
|
// reusable answer is accountable for it. Domain/description/tags are
|
|
4846
6533
|
// helpful defaults, but not reasons to send a single-user back to a
|
|
@@ -4873,12 +6560,18 @@ export async function startLocalServer(opts) {
|
|
|
4873
6560
|
// Save first, then certify against the same local runtime used for the
|
|
4874
6561
|
// answer. Passing blocks become certified immediately; anything that
|
|
4875
6562
|
// cannot prove itself remains a clearly labelled draft at this path.
|
|
4876
|
-
setBlockStudioStatus(projectRoot, created.path,
|
|
4877
|
-
let source = readFileSync(join(projectRoot, created.path),
|
|
4878
|
-
let status =
|
|
4879
|
-
let blockers =
|
|
6563
|
+
setBlockStudioStatus(projectRoot, created.path, "draft");
|
|
6564
|
+
let source = readFileSync(join(projectRoot, created.path), "utf-8");
|
|
6565
|
+
let status = "draft";
|
|
6566
|
+
let blockers = reviewRequired
|
|
6567
|
+
? [
|
|
6568
|
+
"Imported or staged datasets require review and reproducibility checks before certification.",
|
|
6569
|
+
]
|
|
6570
|
+
: [];
|
|
4880
6571
|
let certification;
|
|
4881
6572
|
try {
|
|
6573
|
+
if (reviewRequired)
|
|
6574
|
+
throw new Error(blockers[0]);
|
|
4882
6575
|
const verdict = await certifyBlockStudioSource(source, created.path);
|
|
4883
6576
|
blockers = Array.from(new Set(verdict.checklist.blockers));
|
|
4884
6577
|
certification = {
|
|
@@ -4893,7 +6586,10 @@ export async function startLocalServer(opts) {
|
|
|
4893
6586
|
}
|
|
4894
6587
|
}
|
|
4895
6588
|
catch (error) {
|
|
4896
|
-
blockers =
|
|
6589
|
+
blockers = Array.from(new Set([
|
|
6590
|
+
...blockers,
|
|
6591
|
+
error instanceof Error ? error.message : String(error),
|
|
6592
|
+
]));
|
|
4897
6593
|
}
|
|
4898
6594
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
4899
6595
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -5166,7 +6862,9 @@ export async function startLocalServer(opts) {
|
|
|
5166
6862
|
// Only expose distinct values for a DECLARED output column — keeps the probe
|
|
5167
6863
|
// inside the governed block contract (no arbitrary column scanning).
|
|
5168
6864
|
const parsedMeta = parseBlockSourceMetadata(source);
|
|
5169
|
-
|
|
6865
|
+
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
6866
|
+
const parameterNames = new Set((validation.parameters ?? []).map((parameter) => parameter.name));
|
|
6867
|
+
if (parameterNames.has(column) || (parsedMeta.outputs.length > 0 && !parsedMeta.outputs.includes(column))) {
|
|
5170
6868
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5171
6869
|
res.end(serializeJSON({ error: `"${column}" is not a declared output of this block` }));
|
|
5172
6870
|
return;
|
|
@@ -5182,7 +6880,6 @@ export async function startLocalServer(opts) {
|
|
|
5182
6880
|
detectedProvider: semanticDetectedProvider,
|
|
5183
6881
|
})
|
|
5184
6882
|
: null;
|
|
5185
|
-
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
5186
6883
|
const baseSql = semanticCompose?.sql ?? validation.executableSql;
|
|
5187
6884
|
if (!baseSql) {
|
|
5188
6885
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -5223,16 +6920,26 @@ export async function startLocalServer(opts) {
|
|
|
5223
6920
|
res.end(serializeJSON({ error: 'goal is required' }));
|
|
5224
6921
|
return;
|
|
5225
6922
|
}
|
|
5226
|
-
const
|
|
5227
|
-
const
|
|
5228
|
-
|
|
5229
|
-
|
|
5230
|
-
|
|
5231
|
-
|
|
5232
|
-
|
|
5233
|
-
|
|
6923
|
+
const snapshot = projectSnapshot();
|
|
6924
|
+
const kgPath = defaultKgPath(projectRoot);
|
|
6925
|
+
await reindexProject(projectRoot, { kgPath, manifest: snapshot.manifest });
|
|
6926
|
+
const kg = new KGStore(kgPath);
|
|
6927
|
+
let plan;
|
|
6928
|
+
try {
|
|
6929
|
+
plan = planAppFromPrompt({
|
|
6930
|
+
prompt: goal,
|
|
6931
|
+
kg,
|
|
6932
|
+
domain: typeof body.domain === 'string' ? body.domain : undefined,
|
|
6933
|
+
audience: typeof body.audience === 'string' ? body.audience : undefined,
|
|
6934
|
+
mode: body.mode === 'personal' ? 'personal' : 'stakeholder',
|
|
6935
|
+
snapshotId: snapshot.snapshotId,
|
|
6936
|
+
});
|
|
6937
|
+
}
|
|
6938
|
+
finally {
|
|
6939
|
+
kg.close();
|
|
6940
|
+
}
|
|
5234
6941
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5235
|
-
res.end(serializeJSON({ plan,
|
|
6942
|
+
res.end(serializeJSON({ plan, snapshotId: snapshot.snapshotId }));
|
|
5236
6943
|
}
|
|
5237
6944
|
catch (error) {
|
|
5238
6945
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -6049,7 +7756,10 @@ export async function startLocalServer(opts) {
|
|
|
6049
7756
|
const body = await readJSON(req);
|
|
6050
7757
|
const source = typeof body.source === 'string' ? body.source : '';
|
|
6051
7758
|
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
6052
|
-
const
|
|
7759
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
7760
|
+
? body.parameters
|
|
7761
|
+
: {};
|
|
7762
|
+
const preview = await runBlockStudioPreviewSource(source, targetConnection, parameters);
|
|
6053
7763
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6054
7764
|
res.end(serializeJSON(preview));
|
|
6055
7765
|
}
|
|
@@ -6124,12 +7834,294 @@ export async function startLocalServer(opts) {
|
|
|
6124
7834
|
?? Object.keys(connections)[0]
|
|
6125
7835
|
?? 'default';
|
|
6126
7836
|
const dbtProfiles = discoverDbtProfileConnections(projectRoot, cfg);
|
|
7837
|
+
const activeDbtProfile = dbtProfiles.find((candidate) => candidate.missingFields.length === 0
|
|
7838
|
+
&& !candidate.warnings.some((warning) => warning.startsWith('DuckDB file not found'))
|
|
7839
|
+
&& !candidate.warnings.some((warning) => warning.startsWith('Not the default dbt target')));
|
|
7840
|
+
const activeConnection = connection ? {
|
|
7841
|
+
source: Object.keys(connections).length > 0 ? 'dql_config' : activeDbtProfile ? 'dbt_profile' : 'runtime',
|
|
7842
|
+
driver: connection.driver,
|
|
7843
|
+
...(Object.keys(connections).length === 0 && activeDbtProfile ? { profileId: activeDbtProfile.id } : {}),
|
|
7844
|
+
} : null;
|
|
6127
7845
|
const connectorStatus = getConnectorInstallStatuses(projectRoot);
|
|
6128
7846
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6129
|
-
res.end(serializeJSON({ default: defaultKey, connections, dbtProfiles, connectorStatus }));
|
|
7847
|
+
res.end(serializeJSON({ default: defaultKey, connections, dbtProfiles, activeConnection, connectorStatus }));
|
|
7848
|
+
return;
|
|
7849
|
+
}
|
|
7850
|
+
if (req.method === 'POST' && path === '/api/connections/dbt-profiles/preview') {
|
|
7851
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
7852
|
+
const profilePath = typeof body.path === 'string' ? body.path.trim() : '';
|
|
7853
|
+
if (!profilePath) {
|
|
7854
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7855
|
+
res.end(serializeJSON({ error: 'Enter a profiles.yml file or folder path.' }));
|
|
7856
|
+
return;
|
|
7857
|
+
}
|
|
7858
|
+
const dbtProfiles = discoverDbtProfileConnections(projectRoot, loadProjectConfig(projectRoot), profilePath);
|
|
7859
|
+
if (dbtProfiles.length === 0) {
|
|
7860
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7861
|
+
res.end(serializeJSON({ error: 'No supported dbt profile targets were found at that path.' }));
|
|
7862
|
+
return;
|
|
7863
|
+
}
|
|
7864
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7865
|
+
res.end(serializeJSON({ dbtProfiles }));
|
|
7866
|
+
return;
|
|
7867
|
+
}
|
|
7868
|
+
if (req.method === "GET" && path === "/api/datasets") {
|
|
7869
|
+
res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" });
|
|
7870
|
+
res.end(serializeJSON({
|
|
7871
|
+
datasets: datasetWorkspace.list(),
|
|
7872
|
+
workspace: {
|
|
7873
|
+
target: "local",
|
|
7874
|
+
databasePath: relative(projectRoot, datasetWorkspace.databasePath).replaceAll("\\", "/"),
|
|
7875
|
+
},
|
|
7876
|
+
}));
|
|
7877
|
+
return;
|
|
7878
|
+
}
|
|
7879
|
+
if (req.method === "POST" && path === "/api/datasets/import") {
|
|
7880
|
+
try {
|
|
7881
|
+
const multipart = String(req.headers["content-type"] ?? "")
|
|
7882
|
+
.toLowerCase()
|
|
7883
|
+
.startsWith("multipart/form-data")
|
|
7884
|
+
? await readMultipartForm(req)
|
|
7885
|
+
: null;
|
|
7886
|
+
const body = multipart?.fields ?? (await readJSON(req));
|
|
7887
|
+
if (typeof body.contentBase64 === "string" &&
|
|
7888
|
+
body.contentBase64.length > 350_000_000) {
|
|
7889
|
+
throw new Error("Uploaded dataset exceeds the 250 MB local import limit. Link the local path or reduce the file first.");
|
|
7890
|
+
}
|
|
7891
|
+
if (multipart?.file &&
|
|
7892
|
+
multipart.file.content.byteLength > 250_000_000) {
|
|
7893
|
+
throw new Error("Uploaded dataset exceeds the 250 MB local import limit. Link the local path or reduce the file first.");
|
|
7894
|
+
}
|
|
7895
|
+
await ensureLocalWorkspaceReady();
|
|
7896
|
+
const imported = await datasetWorkspace.import({
|
|
7897
|
+
filename: multipart?.file?.filename ??
|
|
7898
|
+
(typeof body.filename === "string" ? body.filename : undefined),
|
|
7899
|
+
sourcePath: typeof body.sourcePath === "string" ? body.sourcePath : undefined,
|
|
7900
|
+
contentBase64: typeof body.contentBase64 === "string"
|
|
7901
|
+
? body.contentBase64
|
|
7902
|
+
: undefined,
|
|
7903
|
+
content: multipart?.file?.content,
|
|
7904
|
+
storageMode: body.storageMode === "project" ? "project" : "local",
|
|
7905
|
+
link: body.link === true || body.link === "true",
|
|
7906
|
+
name: typeof body.name === "string" ? body.name : undefined,
|
|
7907
|
+
alias: typeof body.alias === "string" ? body.alias : undefined,
|
|
7908
|
+
description: typeof body.description === "string" ? body.description : undefined,
|
|
7909
|
+
owner: typeof body.owner === "string" ? body.owner : undefined,
|
|
7910
|
+
tags: Array.isArray(body.tags)
|
|
7911
|
+
? body.tags.map(String)
|
|
7912
|
+
: typeof body.tags === "string"
|
|
7913
|
+
? body.tags
|
|
7914
|
+
.split(",")
|
|
7915
|
+
.map((tag) => tag.trim())
|
|
7916
|
+
.filter(Boolean)
|
|
7917
|
+
: undefined,
|
|
7918
|
+
});
|
|
7919
|
+
recordDatasetMetadataSnapshot();
|
|
7920
|
+
res.writeHead(imported.duplicate ? 200 : 201, {
|
|
7921
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7922
|
+
});
|
|
7923
|
+
res.end(serializeJSON(imported));
|
|
7924
|
+
}
|
|
7925
|
+
catch (error) {
|
|
7926
|
+
res.writeHead(400, {
|
|
7927
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7928
|
+
});
|
|
7929
|
+
res.end(serializeJSON({
|
|
7930
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7931
|
+
}));
|
|
7932
|
+
}
|
|
7933
|
+
return;
|
|
7934
|
+
}
|
|
7935
|
+
if (req.method === "GET" && /^\/api\/datasets\/[^/]+$/.test(path)) {
|
|
7936
|
+
const id = decodeURIComponent(path.slice("/api/datasets/".length));
|
|
7937
|
+
const dataset = datasetWorkspace.get(id);
|
|
7938
|
+
if (!dataset) {
|
|
7939
|
+
res.writeHead(404, {
|
|
7940
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7941
|
+
});
|
|
7942
|
+
res.end(serializeJSON({ error: "Dataset not found." }));
|
|
7943
|
+
}
|
|
7944
|
+
else {
|
|
7945
|
+
res.writeHead(200, {
|
|
7946
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7947
|
+
});
|
|
7948
|
+
res.end(serializeJSON({ dataset }));
|
|
7949
|
+
}
|
|
7950
|
+
return;
|
|
7951
|
+
}
|
|
7952
|
+
if (req.method === "POST" &&
|
|
7953
|
+
/^\/api\/datasets\/[^/]+\/refresh$/.test(path)) {
|
|
7954
|
+
try {
|
|
7955
|
+
const id = decodeURIComponent(path.match(/^\/api\/datasets\/([^/]+)\/refresh$/)?.[1] ?? "");
|
|
7956
|
+
await ensureLocalWorkspaceReady();
|
|
7957
|
+
const dataset = await datasetWorkspace.refresh(id);
|
|
7958
|
+
recordDatasetMetadataSnapshot();
|
|
7959
|
+
res.writeHead(200, {
|
|
7960
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7961
|
+
});
|
|
7962
|
+
res.end(serializeJSON({ dataset }));
|
|
7963
|
+
}
|
|
7964
|
+
catch (error) {
|
|
7965
|
+
res.writeHead(400, {
|
|
7966
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7967
|
+
});
|
|
7968
|
+
res.end(serializeJSON({
|
|
7969
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7970
|
+
}));
|
|
7971
|
+
}
|
|
7972
|
+
return;
|
|
7973
|
+
}
|
|
7974
|
+
if (req.method === "POST" &&
|
|
7975
|
+
/^\/api\/datasets\/[^/]+\/(rename|pin)$/.test(path)) {
|
|
7976
|
+
try {
|
|
7977
|
+
const match = path.match(/^\/api\/datasets\/([^/]+)\/(rename|pin)$/);
|
|
7978
|
+
const body = await readJSON(req);
|
|
7979
|
+
const dataset = match[2] === "pin"
|
|
7980
|
+
? datasetWorkspace.pin(decodeURIComponent(match[1]), body.pinned !== false)
|
|
7981
|
+
: datasetWorkspace.rename(decodeURIComponent(match[1]), String(body.name ?? ""), typeof body.alias === "string" ? body.alias : undefined);
|
|
7982
|
+
recordDatasetMetadataSnapshot();
|
|
7983
|
+
res.writeHead(200, {
|
|
7984
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7985
|
+
});
|
|
7986
|
+
res.end(serializeJSON({ dataset }));
|
|
7987
|
+
}
|
|
7988
|
+
catch (error) {
|
|
7989
|
+
res.writeHead(400, {
|
|
7990
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7991
|
+
});
|
|
7992
|
+
res.end(serializeJSON({
|
|
7993
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7994
|
+
}));
|
|
7995
|
+
}
|
|
7996
|
+
return;
|
|
7997
|
+
}
|
|
7998
|
+
if (req.method === "POST" &&
|
|
7999
|
+
/^\/api\/datasets\/[^/]+\/schema$/.test(path)) {
|
|
8000
|
+
try {
|
|
8001
|
+
const id = decodeURIComponent(path.match(/^\/api\/datasets\/([^/]+)\/schema$/)?.[1] ?? "");
|
|
8002
|
+
const body = await readJSON(req);
|
|
8003
|
+
await ensureLocalWorkspaceReady();
|
|
8004
|
+
const dataset = await datasetWorkspace.updateSchema(id, body.overrides && typeof body.overrides === "object"
|
|
8005
|
+
? body.overrides
|
|
8006
|
+
: {});
|
|
8007
|
+
recordDatasetMetadataSnapshot();
|
|
8008
|
+
res.writeHead(200, {
|
|
8009
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
8010
|
+
});
|
|
8011
|
+
res.end(serializeJSON({ dataset }));
|
|
8012
|
+
}
|
|
8013
|
+
catch (error) {
|
|
8014
|
+
res.writeHead(400, {
|
|
8015
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
8016
|
+
});
|
|
8017
|
+
res.end(serializeJSON({
|
|
8018
|
+
error: error instanceof Error ? error.message : String(error),
|
|
8019
|
+
}));
|
|
8020
|
+
}
|
|
8021
|
+
return;
|
|
8022
|
+
}
|
|
8023
|
+
if (req.method === "DELETE" && /^\/api\/datasets\/[^/]+$/.test(path)) {
|
|
8024
|
+
const id = decodeURIComponent(path.slice("/api/datasets/".length));
|
|
8025
|
+
datasetWorkspace.remove(id);
|
|
8026
|
+
recordDatasetMetadataSnapshot();
|
|
8027
|
+
res.writeHead(204);
|
|
8028
|
+
res.end();
|
|
8029
|
+
return;
|
|
8030
|
+
}
|
|
8031
|
+
if (req.method === "POST" && path === "/api/datasets/stage") {
|
|
8032
|
+
try {
|
|
8033
|
+
const body = await readJSON(req);
|
|
8034
|
+
if (body.confirmed !== true) {
|
|
8035
|
+
res.writeHead(428, {
|
|
8036
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
8037
|
+
});
|
|
8038
|
+
res.end(serializeJSON({
|
|
8039
|
+
error: "Confirm the bounded local copy before staging warehouse data.",
|
|
8040
|
+
}));
|
|
8041
|
+
return;
|
|
8042
|
+
}
|
|
8043
|
+
const configured = projectConfig.notebook?.staging;
|
|
8044
|
+
const maxRows = Math.max(1, Math.min(Number(body.maxRows ?? configured?.maxRows ?? 100_000), 500_000));
|
|
8045
|
+
const maxBytes = Math.max(1_000_000, Math.min(Number(body.maxBytes ?? configured?.maxBytes ?? 250_000_000), 1_000_000_000));
|
|
8046
|
+
const timeoutMs = Math.max(1_000, Math.min(Number(body.timeoutSeconds ?? configured?.timeoutSeconds ?? 120) *
|
|
8047
|
+
1000, 600_000));
|
|
8048
|
+
const sql = typeof body.sql === "string" ? body.sql.trim() : "";
|
|
8049
|
+
const validationError = readOnlySqlValidationError(sql, "Local staging");
|
|
8050
|
+
if (validationError)
|
|
8051
|
+
throw new Error(validationError);
|
|
8052
|
+
const semantic = prepareSemanticSql(sql, semanticLayer);
|
|
8053
|
+
if (semantic.unresolvedRefs.length > 0)
|
|
8054
|
+
throw new Error(`Unknown semantic references: ${semantic.unresolvedRefs.join(", ")}`);
|
|
8055
|
+
const named = typeof body.connectionName === "string"
|
|
8056
|
+
? resolveNamedConnection(body.connectionName)
|
|
8057
|
+
: connection;
|
|
8058
|
+
const sourceConnection = requireActiveConnection(named);
|
|
8059
|
+
const prepared = prepareLocalExecution(`SELECT * FROM (${stripSqlTerminator(semantic.sql)}) AS dql_stage_source LIMIT ${maxRows + 1}`, sourceConnection, projectRoot, projectConfig);
|
|
8060
|
+
const execution = executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
|
|
8061
|
+
const result = await Promise.race([
|
|
8062
|
+
execution,
|
|
8063
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Staging query exceeded ${Math.round(timeoutMs / 1000)} seconds.`)), timeoutMs)),
|
|
8064
|
+
]);
|
|
8065
|
+
const rows = (result.rows ?? []);
|
|
8066
|
+
if (rows.length > maxRows)
|
|
8067
|
+
throw new Error(`Staging would exceed the ${maxRows.toLocaleString()} row limit. Add filters or aggregate the warehouse query.`);
|
|
8068
|
+
const rowPayload = serializeJSON(rows);
|
|
8069
|
+
const byteCount = Buffer.byteLength(rowPayload);
|
|
8070
|
+
if (byteCount > maxBytes)
|
|
8071
|
+
throw new Error(`Staging would exceed the ${(maxBytes / 1_000_000).toFixed(0)} MB limit. Add filters or aggregate the warehouse query.`);
|
|
8072
|
+
await ensureLocalWorkspaceReady();
|
|
8073
|
+
const dataset = await datasetWorkspace.stageRows({
|
|
8074
|
+
name: typeof body.name === "string" && body.name.trim()
|
|
8075
|
+
? body.name.trim()
|
|
8076
|
+
: "Staged warehouse result",
|
|
8077
|
+
rows,
|
|
8078
|
+
expiresInDays: configured?.expiryDays ?? 7,
|
|
8079
|
+
lineage: {
|
|
8080
|
+
connectionName: typeof body.connectionName === "string"
|
|
8081
|
+
? body.connectionName
|
|
8082
|
+
: projectConfig.defaultConnectionName,
|
|
8083
|
+
query: sql,
|
|
8084
|
+
blockPath: typeof body.blockPath === "string" ? body.blockPath : undefined,
|
|
8085
|
+
semanticMetrics: semantic.semanticRefs.metrics,
|
|
8086
|
+
semanticDimensions: semantic.semanticRefs.dimensions,
|
|
8087
|
+
filters: body.filters && typeof body.filters === "object"
|
|
8088
|
+
? body.filters
|
|
8089
|
+
: undefined,
|
|
8090
|
+
parameters: body.parameters && typeof body.parameters === "object"
|
|
8091
|
+
? body.parameters
|
|
8092
|
+
: undefined,
|
|
8093
|
+
extractedAt: new Date().toISOString(),
|
|
8094
|
+
rowCount: rows.length,
|
|
8095
|
+
byteCount,
|
|
8096
|
+
sourceFingerprint: createHash("sha256")
|
|
8097
|
+
.update(`${body.connectionName ?? projectConfig.defaultConnectionName ?? "default"}:${sql}`)
|
|
8098
|
+
.digest("hex"),
|
|
8099
|
+
resultFingerprint: createHash("sha256")
|
|
8100
|
+
.update(rowPayload)
|
|
8101
|
+
.digest("hex"),
|
|
8102
|
+
},
|
|
8103
|
+
});
|
|
8104
|
+
recordDatasetMetadataSnapshot();
|
|
8105
|
+
res.writeHead(201, {
|
|
8106
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
8107
|
+
});
|
|
8108
|
+
res.end(serializeJSON({
|
|
8109
|
+
dataset,
|
|
8110
|
+
limits: { maxRows, maxBytes, timeoutMs },
|
|
8111
|
+
trustLabel: "review_required",
|
|
8112
|
+
}));
|
|
8113
|
+
}
|
|
8114
|
+
catch (error) {
|
|
8115
|
+
res.writeHead(400, {
|
|
8116
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
8117
|
+
});
|
|
8118
|
+
res.end(serializeJSON({
|
|
8119
|
+
error: error instanceof Error ? error.message : String(error),
|
|
8120
|
+
}));
|
|
8121
|
+
}
|
|
6130
8122
|
return;
|
|
6131
8123
|
}
|
|
6132
|
-
if (req.method ===
|
|
8124
|
+
if (req.method === "POST" && path === "/api/connectors/install") {
|
|
6133
8125
|
try {
|
|
6134
8126
|
const body = await readJSON(req);
|
|
6135
8127
|
const driver = typeof body.driver === 'string' ? body.driver : '';
|
|
@@ -6356,9 +8348,18 @@ export async function startLocalServer(opts) {
|
|
|
6356
8348
|
source: q.source ?? null,
|
|
6357
8349
|
}));
|
|
6358
8350
|
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
6359
|
-
const
|
|
8351
|
+
const dbtManifestReady = provider === 'dbt'
|
|
6360
8352
|
? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
|
|
6361
8353
|
: false;
|
|
8354
|
+
const metricFlowReady = provider === 'dbt' ? hasMetricFlowCli() : false;
|
|
8355
|
+
const dbtExecutionReady = dbtManifestReady && metricFlowReady;
|
|
8356
|
+
const dbtExecutionSetup = dbtExecutionReady
|
|
8357
|
+
? null
|
|
8358
|
+
: !dbtManifestReady && !metricFlowReady
|
|
8359
|
+
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.'
|
|
8360
|
+
: !dbtManifestReady
|
|
8361
|
+
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists.'
|
|
8362
|
+
: 'Install MetricFlow so `mf` is on PATH, or set DQL_METRICFLOW_BIN to the MetricFlow executable.';
|
|
6362
8363
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6363
8364
|
res.end(serializeJSON({
|
|
6364
8365
|
available: true,
|
|
@@ -6367,9 +8368,7 @@ export async function startLocalServer(opts) {
|
|
|
6367
8368
|
? {
|
|
6368
8369
|
engine: 'metricflow',
|
|
6369
8370
|
ready: dbtExecutionReady,
|
|
6370
|
-
setup:
|
|
6371
|
-
? null
|
|
6372
|
-
: 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.',
|
|
8371
|
+
setup: dbtExecutionSetup,
|
|
6373
8372
|
}
|
|
6374
8373
|
: { engine: 'native', ready: true, setup: null },
|
|
6375
8374
|
errors: semanticLayerErrors,
|
|
@@ -6771,8 +8770,186 @@ export async function startLocalServer(opts) {
|
|
|
6771
8770
|
}
|
|
6772
8771
|
return;
|
|
6773
8772
|
}
|
|
6774
|
-
// ──
|
|
6775
|
-
if (req.method ===
|
|
8773
|
+
// ── Ranked, lazy editor completions (large-repo safe) ───────────────────────
|
|
8774
|
+
if (req.method === "GET" && path === "/api/editor/completions") {
|
|
8775
|
+
const query = (url.searchParams.get("q") ?? "").trim().toLowerCase();
|
|
8776
|
+
const kind = (url.searchParams.get("kind") ?? "all").trim().toLowerCase();
|
|
8777
|
+
const relationFilter = (url.searchParams.get("relation") ?? "")
|
|
8778
|
+
.trim()
|
|
8779
|
+
.toLowerCase();
|
|
8780
|
+
const selectedDomain = (url.searchParams.get("domain") ?? "")
|
|
8781
|
+
.trim()
|
|
8782
|
+
.toLowerCase();
|
|
8783
|
+
const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 50) || 50));
|
|
8784
|
+
const recent = readUserPrefs(userPrefsPath).recentlyUsed.map((name) => name.toLowerCase());
|
|
8785
|
+
const candidates = [];
|
|
8786
|
+
if (semanticLayer && (kind === "all" || kind === "metric")) {
|
|
8787
|
+
for (const metric of semanticLayer.listMetrics())
|
|
8788
|
+
candidates.push({
|
|
8789
|
+
type: "metric",
|
|
8790
|
+
name: metric.name,
|
|
8791
|
+
label: metric.label,
|
|
8792
|
+
description: metric.description ?? "",
|
|
8793
|
+
domain: metric.domain,
|
|
8794
|
+
tags: metric.tags ?? [],
|
|
8795
|
+
detail: "@metric",
|
|
8796
|
+
apply: metric.name,
|
|
8797
|
+
governance: "semantic",
|
|
8798
|
+
});
|
|
8799
|
+
}
|
|
8800
|
+
if (semanticLayer && (kind === "all" || kind === "dimension")) {
|
|
8801
|
+
for (const dimension of semanticLayer.listDimensions())
|
|
8802
|
+
candidates.push({
|
|
8803
|
+
type: "dimension",
|
|
8804
|
+
name: dimension.name,
|
|
8805
|
+
label: dimension.label,
|
|
8806
|
+
description: dimension.description ?? "",
|
|
8807
|
+
domain: dimension.domain,
|
|
8808
|
+
tags: dimension.tags ?? [],
|
|
8809
|
+
detail: "@dim",
|
|
8810
|
+
apply: dimension.name,
|
|
8811
|
+
governance: "semantic",
|
|
8812
|
+
});
|
|
8813
|
+
}
|
|
8814
|
+
const datasets = datasetWorkspace.list();
|
|
8815
|
+
if (kind === "all" || kind === "relation" || kind === "dataset") {
|
|
8816
|
+
for (const dataset of datasets)
|
|
8817
|
+
candidates.push({
|
|
8818
|
+
type: kind === "dataset" ? "dataset" : "relation",
|
|
8819
|
+
name: dataset.alias,
|
|
8820
|
+
label: dataset.name,
|
|
8821
|
+
description: `${dataset.storageMode} ${dataset.format} · refreshed ${dataset.refreshedAt}`,
|
|
8822
|
+
relation: dataset.alias,
|
|
8823
|
+
detail: dataset.trustState,
|
|
8824
|
+
apply: `"${dataset.alias.replaceAll('"', '""')}"`,
|
|
8825
|
+
governance: dataset.trustState,
|
|
8826
|
+
});
|
|
8827
|
+
}
|
|
8828
|
+
if (kind === "all" || kind === "column") {
|
|
8829
|
+
for (const dataset of datasets) {
|
|
8830
|
+
if (relationFilter &&
|
|
8831
|
+
![dataset.id, dataset.alias, dataset.name].some((value) => value.toLowerCase() === relationFilter))
|
|
8832
|
+
continue;
|
|
8833
|
+
for (const column of dataset.profile.columns)
|
|
8834
|
+
candidates.push({
|
|
8835
|
+
type: "column",
|
|
8836
|
+
name: column.name,
|
|
8837
|
+
label: column.name,
|
|
8838
|
+
description: column.flags?.join(", ") ?? "",
|
|
8839
|
+
relation: dataset.alias,
|
|
8840
|
+
detail: column.type,
|
|
8841
|
+
apply: column.name,
|
|
8842
|
+
governance: dataset.trustState,
|
|
8843
|
+
});
|
|
8844
|
+
}
|
|
8845
|
+
const snapshot = latestRuntimeSchemaSnapshotForProject(projectRoot);
|
|
8846
|
+
for (const table of snapshot?.tables ?? []) {
|
|
8847
|
+
if (relationFilter &&
|
|
8848
|
+
![table.relation, table.name]
|
|
8849
|
+
.filter(Boolean)
|
|
8850
|
+
.some((value) => value.toLowerCase() === relationFilter ||
|
|
8851
|
+
value.toLowerCase().endsWith(`.${relationFilter}`)))
|
|
8852
|
+
continue;
|
|
8853
|
+
for (const column of table.columns)
|
|
8854
|
+
candidates.push({
|
|
8855
|
+
type: "column",
|
|
8856
|
+
name: column.name,
|
|
8857
|
+
label: column.name,
|
|
8858
|
+
description: column.description ?? "",
|
|
8859
|
+
relation: table.relation,
|
|
8860
|
+
detail: column.type,
|
|
8861
|
+
apply: column.name,
|
|
8862
|
+
governance: "runtime_schema",
|
|
8863
|
+
});
|
|
8864
|
+
}
|
|
8865
|
+
}
|
|
8866
|
+
if (kind === "all" || kind === "relation") {
|
|
8867
|
+
const snapshot = latestRuntimeSchemaSnapshotForProject(projectRoot);
|
|
8868
|
+
for (const table of snapshot?.tables ?? [])
|
|
8869
|
+
candidates.push({
|
|
8870
|
+
type: "relation",
|
|
8871
|
+
name: table.relation,
|
|
8872
|
+
label: table.name ?? table.relation,
|
|
8873
|
+
description: table.description ?? "",
|
|
8874
|
+
relation: table.relation,
|
|
8875
|
+
detail: table.schema,
|
|
8876
|
+
apply: table.relation,
|
|
8877
|
+
governance: table.source ?? "runtime_schema",
|
|
8878
|
+
});
|
|
8879
|
+
}
|
|
8880
|
+
if (kind === "all" || kind === "block") {
|
|
8881
|
+
for (const file of scanNotebookFiles(projectRoot).filter((entry) => entry.type === "block"))
|
|
8882
|
+
candidates.push({
|
|
8883
|
+
type: "block",
|
|
8884
|
+
name: file.name.replace(/\.dql$/i, ""),
|
|
8885
|
+
label: file.name.replace(/\.dql$/i, ""),
|
|
8886
|
+
description: file.path,
|
|
8887
|
+
detail: "@block",
|
|
8888
|
+
apply: `@block("${file.name.replace(/\.dql$/i, "")}")`,
|
|
8889
|
+
governance: file.path.includes("_draft") ? "draft" : "governed",
|
|
8890
|
+
});
|
|
8891
|
+
}
|
|
8892
|
+
if (kind === "all" || kind === "connection") {
|
|
8893
|
+
for (const name of Object.keys(getStoredConnections(projectConfig)))
|
|
8894
|
+
candidates.push({
|
|
8895
|
+
type: "connection",
|
|
8896
|
+
name,
|
|
8897
|
+
label: name,
|
|
8898
|
+
description: "Configured connection",
|
|
8899
|
+
detail: name === projectConfig.defaultConnectionName
|
|
8900
|
+
? "default"
|
|
8901
|
+
: undefined,
|
|
8902
|
+
apply: name,
|
|
8903
|
+
});
|
|
8904
|
+
}
|
|
8905
|
+
const uniqueCandidates = [
|
|
8906
|
+
...new Map(candidates.map((candidate) => [
|
|
8907
|
+
`${candidate.type}:${candidate.name.toLowerCase()}:${candidate.relation?.toLowerCase() ?? ""}`,
|
|
8908
|
+
candidate,
|
|
8909
|
+
])).values(),
|
|
8910
|
+
];
|
|
8911
|
+
const scored = uniqueCandidates
|
|
8912
|
+
.map((candidate) => {
|
|
8913
|
+
const name = candidate.name.toLowerCase();
|
|
8914
|
+
const label = candidate.label.toLowerCase();
|
|
8915
|
+
const haystack = `${name} ${label} ${candidate.description.toLowerCase()} ${(candidate.tags ?? []).join(" ").toLowerCase()} ${candidate.domain?.toLowerCase() ?? ""}`;
|
|
8916
|
+
let score = query
|
|
8917
|
+
? name === query
|
|
8918
|
+
? 100
|
|
8919
|
+
: name.startsWith(query)
|
|
8920
|
+
? 80
|
|
8921
|
+
: label.startsWith(query)
|
|
8922
|
+
? 70
|
|
8923
|
+
: haystack.includes(query)
|
|
8924
|
+
? 40
|
|
8925
|
+
: 0
|
|
8926
|
+
: 10;
|
|
8927
|
+
if (selectedDomain &&
|
|
8928
|
+
candidate.domain?.toLowerCase() === selectedDomain)
|
|
8929
|
+
score += 20;
|
|
8930
|
+
if (recent.includes(name))
|
|
8931
|
+
score += 15;
|
|
8932
|
+
if (candidate.governance === "certified" ||
|
|
8933
|
+
candidate.governance === "semantic" ||
|
|
8934
|
+
candidate.governance === "project_controlled")
|
|
8935
|
+
score += 10;
|
|
8936
|
+
return { ...candidate, score };
|
|
8937
|
+
})
|
|
8938
|
+
.filter((candidate) => !query || (candidate.score ?? 0) > 0)
|
|
8939
|
+
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || a.name.localeCompare(b.name))
|
|
8940
|
+
.slice(0, limit);
|
|
8941
|
+
res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" });
|
|
8942
|
+
res.end(serializeJSON({
|
|
8943
|
+
completions: scored,
|
|
8944
|
+
query,
|
|
8945
|
+
kind,
|
|
8946
|
+
limit,
|
|
8947
|
+
totalCandidates: uniqueCandidates.length,
|
|
8948
|
+
}));
|
|
8949
|
+
return;
|
|
8950
|
+
}
|
|
8951
|
+
// ── Legacy semantic completion endpoint ────────────────────────────────────
|
|
8952
|
+
if (req.method === "GET" && path === "/api/semantic-completions") {
|
|
6776
8953
|
const completions = [];
|
|
6777
8954
|
if (semanticLayer) {
|
|
6778
8955
|
for (const m of semanticLayer.listMetrics()) {
|
|
@@ -6944,7 +9121,11 @@ export async function startLocalServer(opts) {
|
|
|
6944
9121
|
res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
|
|
6945
9122
|
return;
|
|
6946
9123
|
}
|
|
6947
|
-
const
|
|
9124
|
+
const executionConnection = await resolveExecutionConnection(body);
|
|
9125
|
+
const tableMapping = hasStandaloneSemanticRef(body.sql)
|
|
9126
|
+
? await resolveSemanticTableMapping(executor, executionConnection, semanticLayer)
|
|
9127
|
+
: undefined;
|
|
9128
|
+
const semantic = prepareSemanticSql(body.sql, semanticLayer, { tableMapping });
|
|
6948
9129
|
if (semantic.unresolvedRefs.length > 0) {
|
|
6949
9130
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6950
9131
|
res.end(serializeJSON({
|
|
@@ -6956,7 +9137,7 @@ export async function startLocalServer(opts) {
|
|
|
6956
9137
|
}));
|
|
6957
9138
|
return;
|
|
6958
9139
|
}
|
|
6959
|
-
const prepared = prepareLocalExecution(semantic.sql,
|
|
9140
|
+
const prepared = prepareLocalExecution(semantic.sql, executionConnection, projectRoot, projectConfig);
|
|
6960
9141
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
6961
9142
|
const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
|
|
6962
9143
|
assertAppAccess({ app, domain, level: 'execute' });
|
|
@@ -7214,6 +9395,10 @@ export async function startLocalServer(opts) {
|
|
|
7214
9395
|
res.end(serializeJSON({ error: 'name and at least one metric are required.' }));
|
|
7215
9396
|
return;
|
|
7216
9397
|
}
|
|
9398
|
+
if (blockType === 'semantic' && dbtFirstOwnsSemanticAuthoring(projectConfig)) {
|
|
9399
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
9400
|
+
return;
|
|
9401
|
+
}
|
|
7217
9402
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
7218
9403
|
const composed = composeRuntimeSemanticQuery({
|
|
7219
9404
|
metrics,
|
|
@@ -7531,14 +9716,128 @@ export async function startLocalServer(opts) {
|
|
|
7531
9716
|
res.end(serializeJSON({ error: 'Missing file path.' }));
|
|
7532
9717
|
return;
|
|
7533
9718
|
}
|
|
7534
|
-
const filePath = safeJoin(projectRoot, relativePath);
|
|
7535
|
-
if (!filePath || !existsSync(filePath) || statSync(filePath).isDirectory()) {
|
|
7536
|
-
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7537
|
-
res.end(serializeJSON({ error: `File not found: ${relativePath}` }));
|
|
7538
|
-
return;
|
|
9719
|
+
const filePath = safeJoin(projectRoot, relativePath);
|
|
9720
|
+
if (!filePath || !existsSync(filePath) || statSync(filePath).isDirectory()) {
|
|
9721
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9722
|
+
res.end(serializeJSON({ error: `File not found: ${relativePath}` }));
|
|
9723
|
+
return;
|
|
9724
|
+
}
|
|
9725
|
+
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
9726
|
+
res.end(readFileSync(filePath));
|
|
9727
|
+
return;
|
|
9728
|
+
}
|
|
9729
|
+
if (req.method === 'POST' && path === '/api/dql/artifacts/execute') {
|
|
9730
|
+
try {
|
|
9731
|
+
const body = await readJSON(req);
|
|
9732
|
+
const requestedArtifact = normalizeDqlArtifactReference(body.artifact);
|
|
9733
|
+
const requestedBlockName = typeof body.blockName === 'string' ? body.blockName.trim() : '';
|
|
9734
|
+
let source = requestedArtifact?.source ?? '';
|
|
9735
|
+
let sourcePath = requestedArtifact?.sourcePath;
|
|
9736
|
+
let blockName = requestedBlockName || requestedArtifact?.name;
|
|
9737
|
+
let domain;
|
|
9738
|
+
let chartType;
|
|
9739
|
+
if (requestedBlockName) {
|
|
9740
|
+
const block = buildManifest({ projectRoot }).blocks[requestedBlockName];
|
|
9741
|
+
if (!block)
|
|
9742
|
+
throw new Error(`Block "${requestedBlockName}" was not found.`);
|
|
9743
|
+
source = readFileSync(join(projectRoot, block.filePath), 'utf-8');
|
|
9744
|
+
sourcePath = block.filePath;
|
|
9745
|
+
blockName = block.name;
|
|
9746
|
+
domain = block.domain;
|
|
9747
|
+
chartType = block.chartType;
|
|
9748
|
+
}
|
|
9749
|
+
else if (sourcePath) {
|
|
9750
|
+
const filePath = safeJoin(projectRoot, sourcePath);
|
|
9751
|
+
if (filePath && existsSync(filePath) && !statSync(filePath).isDirectory())
|
|
9752
|
+
source = readFileSync(filePath, 'utf-8');
|
|
9753
|
+
}
|
|
9754
|
+
if (!source.trim())
|
|
9755
|
+
throw new Error('Provide an executable DQL artifact or block name.');
|
|
9756
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
9757
|
+
? body.parameters
|
|
9758
|
+
: requestedArtifact?.parameterValues ?? {};
|
|
9759
|
+
const result = await executeDqlArtifactSourceForAgent(source, {
|
|
9760
|
+
name: blockName,
|
|
9761
|
+
path: sourcePath,
|
|
9762
|
+
domain,
|
|
9763
|
+
chartType,
|
|
9764
|
+
}, {
|
|
9765
|
+
parameters,
|
|
9766
|
+
question: typeof body.question === 'string' ? body.question : undefined,
|
|
9767
|
+
});
|
|
9768
|
+
const contract = prepareBlockInvocation({ source, parameters, surface: 'ask_ai' });
|
|
9769
|
+
const certified = /\bstatus\s*=\s*"certified"/i.test(source);
|
|
9770
|
+
const semantic = /\btype\s*=\s*"semantic"/i.test(source);
|
|
9771
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9772
|
+
res.end(serializeJSON({
|
|
9773
|
+
result,
|
|
9774
|
+
artifact: {
|
|
9775
|
+
...(requestedArtifact ?? { kind: semantic ? 'semantic_block' : 'sql_block' }),
|
|
9776
|
+
source,
|
|
9777
|
+
...(blockName ? { name: blockName } : {}),
|
|
9778
|
+
...(sourcePath ? { sourcePath } : {}),
|
|
9779
|
+
parameters: contract.parameters,
|
|
9780
|
+
parameterValues: contract.values,
|
|
9781
|
+
persistence: sourcePath ? 'saved' : 'transient',
|
|
9782
|
+
trustState: certified ? 'certified' : semantic ? 'governed' : 'review_required',
|
|
9783
|
+
compiledSql: result.sql,
|
|
9784
|
+
},
|
|
9785
|
+
}));
|
|
9786
|
+
}
|
|
9787
|
+
catch (error) {
|
|
9788
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9789
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
9790
|
+
}
|
|
9791
|
+
return;
|
|
9792
|
+
}
|
|
9793
|
+
if (req.method === 'GET' && path === '/api/blocks/parameters') {
|
|
9794
|
+
try {
|
|
9795
|
+
const blockName = url.searchParams.get('name')?.trim();
|
|
9796
|
+
if (!blockName)
|
|
9797
|
+
throw new Error('Missing certified block name.');
|
|
9798
|
+
const manifest = buildManifest({ projectRoot });
|
|
9799
|
+
const block = manifest.blocks[blockName];
|
|
9800
|
+
if (!block) {
|
|
9801
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9802
|
+
res.end(serializeJSON({ error: `Block "${blockName}" was not found.` }));
|
|
9803
|
+
return;
|
|
9804
|
+
}
|
|
9805
|
+
if (block.status !== 'certified')
|
|
9806
|
+
throw new Error(`Block "${blockName}" is not certified.`);
|
|
9807
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9808
|
+
res.end(serializeJSON({
|
|
9809
|
+
blockName: block.name,
|
|
9810
|
+
blockPath: block.filePath,
|
|
9811
|
+
parameters: block.parameters ?? [],
|
|
9812
|
+
}));
|
|
9813
|
+
}
|
|
9814
|
+
catch (error) {
|
|
9815
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9816
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
9817
|
+
}
|
|
9818
|
+
return;
|
|
9819
|
+
}
|
|
9820
|
+
if (req.method === 'POST' && path === '/api/blocks/invoke') {
|
|
9821
|
+
try {
|
|
9822
|
+
const body = await readJSON(req);
|
|
9823
|
+
const blockName = typeof body.blockName === 'string' ? body.blockName.trim() : '';
|
|
9824
|
+
if (!blockName)
|
|
9825
|
+
throw new Error('Missing certified block name.');
|
|
9826
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
9827
|
+
? body.parameters
|
|
9828
|
+
: {};
|
|
9829
|
+
const result = await executeCertifiedBlockByNameForAgent(blockName, { parameters, question: typeof body.question === 'string' ? body.question : undefined }, true);
|
|
9830
|
+
const manifest = buildManifest({ projectRoot });
|
|
9831
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9832
|
+
res.end(serializeJSON({
|
|
9833
|
+
result,
|
|
9834
|
+
parameters: manifest.blocks[blockName]?.parameters ?? [],
|
|
9835
|
+
}));
|
|
9836
|
+
}
|
|
9837
|
+
catch (error) {
|
|
9838
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9839
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
7539
9840
|
}
|
|
7540
|
-
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
7541
|
-
res.end(readFileSync(filePath));
|
|
7542
9841
|
return;
|
|
7543
9842
|
}
|
|
7544
9843
|
if (req.method === 'POST' && path === '/api/notebook/execute') {
|
|
@@ -7556,11 +9855,30 @@ export async function startLocalServer(opts) {
|
|
|
7556
9855
|
}
|
|
7557
9856
|
const resolved = resolveNotebookBlockReferenceCell(cell, projectRoot);
|
|
7558
9857
|
const executableCell = resolved.cell;
|
|
7559
|
-
const
|
|
9858
|
+
const invocation = executableCell.type === 'dql'
|
|
9859
|
+
? prepareBlockInvocation({
|
|
9860
|
+
source: executableCell.source,
|
|
9861
|
+
parameters: body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
9862
|
+
? body.parameters
|
|
9863
|
+
: {},
|
|
9864
|
+
surface: 'notebook',
|
|
9865
|
+
})
|
|
9866
|
+
: null;
|
|
9867
|
+
if (invocation?.errors.length)
|
|
9868
|
+
throw new Error(invocation.errors.join(' '));
|
|
9869
|
+
if (invocation?.unresolvedParameters.length) {
|
|
9870
|
+
throw new Error(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'}: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
9871
|
+
}
|
|
9872
|
+
const cellConnection = await resolveExecutionConnection(body);
|
|
7560
9873
|
const tableMapping = needsSemanticTableMapping(executableCell)
|
|
7561
9874
|
? await resolveSemanticTableMapping(executor, cellConnection, semanticLayer)
|
|
7562
9875
|
: undefined;
|
|
7563
|
-
const plan = buildExecutionPlan(executableCell, {
|
|
9876
|
+
const plan = buildExecutionPlan(executableCell, {
|
|
9877
|
+
semanticLayer,
|
|
9878
|
+
driver: cellConnection.driver,
|
|
9879
|
+
tableMapping,
|
|
9880
|
+
parameters: invocation?.values,
|
|
9881
|
+
});
|
|
7564
9882
|
if (!plan) {
|
|
7565
9883
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7566
9884
|
res.end(serializeJSON({ cellType: cell.type, result: null }));
|
|
@@ -7569,7 +9887,7 @@ export async function startLocalServer(opts) {
|
|
|
7569
9887
|
const prepared = prepareLocalExecution(plan.sql, cellConnection, projectRoot, projectConfig);
|
|
7570
9888
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
7571
9889
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
7572
|
-
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
|
|
9890
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables({ ...plan.variables, ...(invocation?.values ?? {}) }), prepared.connection);
|
|
7573
9891
|
const normalized = normalizeQueryResult(rawResult);
|
|
7574
9892
|
// Enforce the block's declared invariants against the result set. This
|
|
7575
9893
|
// is additive: blocks without invariants produce `null` and the
|
|
@@ -7607,6 +9925,13 @@ export async function startLocalServer(opts) {
|
|
|
7607
9925
|
chartConfig: plan.chartConfig,
|
|
7608
9926
|
tests: plan.tests,
|
|
7609
9927
|
result: normalized,
|
|
9928
|
+
...(invocation ? {
|
|
9929
|
+
invocation: {
|
|
9930
|
+
resolvedParameters: invocation.resolvedParameters,
|
|
9931
|
+
unresolvedParameters: invocation.unresolvedParameters,
|
|
9932
|
+
auditId: invocation.auditId,
|
|
9933
|
+
},
|
|
9934
|
+
} : {}),
|
|
7610
9935
|
...(invariants
|
|
7611
9936
|
? {
|
|
7612
9937
|
invariantResults: invariants.invariantResults,
|
|
@@ -7640,6 +9965,10 @@ export async function startLocalServer(opts) {
|
|
|
7640
9965
|
// Create a new metric YAML file in semantic-layer/metrics/
|
|
7641
9966
|
if (req.method === 'POST' && path === '/api/semantic-layer/metric') {
|
|
7642
9967
|
try {
|
|
9968
|
+
if (dbtFirstOwnsSemanticAuthoring(projectConfig)) {
|
|
9969
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
9970
|
+
return;
|
|
9971
|
+
}
|
|
7643
9972
|
const body = await readJSON(req);
|
|
7644
9973
|
const { name, label, description, domain, sql, type, table, tags } = body;
|
|
7645
9974
|
if (!name || !sql || !type || !table) {
|
|
@@ -7709,6 +10038,148 @@ table: ${table}${tagList}
|
|
|
7709
10038
|
});
|
|
7710
10039
|
});
|
|
7711
10040
|
}
|
|
10041
|
+
async function validateModelingRelationship(relationship, manifest, execute, quoteIdentifier = quoteSqlIdentifier) {
|
|
10042
|
+
const fromEntity = manifest.modeling?.entities[relationship.from];
|
|
10043
|
+
const toEntity = manifest.modeling?.entities[relationship.to];
|
|
10044
|
+
if (!fromEntity || !toEntity)
|
|
10045
|
+
throw new Error('Both relationship entities must be bound to dbt models before validation.');
|
|
10046
|
+
const fromNode = manifest.dbtProvenance?.nodes[fromEntity.dbtUniqueId];
|
|
10047
|
+
const toNode = manifest.dbtProvenance?.nodes[toEntity.dbtUniqueId];
|
|
10048
|
+
if (!fromNode?.relation || !toNode?.relation)
|
|
10049
|
+
throw new Error('Both dbt models must expose warehouse relation names.');
|
|
10050
|
+
if (!relationship.keys.length)
|
|
10051
|
+
throw new Error('At least one join key pair is required.');
|
|
10052
|
+
const safeQuote = (identifier) => quoteIdentifier(assertSafeSqlIdentifier(identifier));
|
|
10053
|
+
const fromRelation = quoteQualifiedIdentifier(fromNode.relation, safeQuote);
|
|
10054
|
+
const toRelation = quoteQualifiedIdentifier(toNode.relation, safeQuote);
|
|
10055
|
+
const fromNull = relationship.keys.map((key) => `f.${safeQuote(key.from)} IS NULL`).join(' OR ');
|
|
10056
|
+
const toNull = relationship.keys.map((key) => `t.${safeQuote(key.to)} IS NULL`).join(' OR ');
|
|
10057
|
+
const join = relationship.keys.map((key) => `f.${safeQuote(key.from)} = t.${safeQuote(key.to)}`).join(' AND ');
|
|
10058
|
+
const fromKeys = relationship.keys.map((key) => safeQuote(key.from)).join(', ');
|
|
10059
|
+
const toKeys = relationship.keys.map((key) => safeQuote(key.to)).join(', ');
|
|
10060
|
+
const firstToKey = safeQuote(relationship.keys[0].to);
|
|
10061
|
+
const sql = `WITH
|
|
10062
|
+
from_counts AS (SELECT COUNT(*) AS rows, SUM(CASE WHEN ${fromNull} THEN 1 ELSE 0 END) AS null_keys FROM ${fromRelation} f),
|
|
10063
|
+
to_counts AS (SELECT COUNT(*) AS rows, SUM(CASE WHEN ${toNull} THEN 1 ELSE 0 END) AS null_keys FROM ${toRelation} t),
|
|
10064
|
+
from_max AS (SELECT COALESCE(MAX(key_count), 0) AS max_per_key FROM (SELECT COUNT(*) AS key_count FROM ${fromRelation} GROUP BY ${fromKeys}) x),
|
|
10065
|
+
to_max AS (SELECT COALESCE(MAX(key_count), 0) AS max_per_key FROM (SELECT COUNT(*) AS key_count FROM ${toRelation} GROUP BY ${toKeys}) x),
|
|
10066
|
+
joined AS (SELECT COUNT(*) AS rows FROM ${fromRelation} f JOIN ${toRelation} t ON ${join}),
|
|
10067
|
+
unmatched AS (SELECT COUNT(*) AS rows FROM ${fromRelation} f LEFT JOIN ${toRelation} t ON ${join} WHERE t.${firstToKey} IS NULL)
|
|
10068
|
+
SELECT from_counts.rows AS from_rows, to_counts.rows AS to_rows, joined.rows AS joined_rows,
|
|
10069
|
+
from_counts.null_keys AS from_null_keys, to_counts.null_keys AS to_null_keys,
|
|
10070
|
+
unmatched.rows AS unmatched_from, from_max.max_per_key AS max_from_per_key, to_max.max_per_key AS max_to_per_key
|
|
10071
|
+
FROM from_counts, to_counts, joined, unmatched, from_max, to_max`;
|
|
10072
|
+
const result = await execute(sql);
|
|
10073
|
+
const row = result.rows[0] ?? {};
|
|
10074
|
+
const maxFromPerKey = numericValue(row.max_from_per_key);
|
|
10075
|
+
const maxToPerKey = numericValue(row.max_to_per_key);
|
|
10076
|
+
const cardinalityPassed = relationship.cardinality === 'one_to_one'
|
|
10077
|
+
? maxFromPerKey <= 1 && maxToPerKey <= 1
|
|
10078
|
+
: relationship.cardinality === 'one_to_many'
|
|
10079
|
+
? maxFromPerKey <= 1
|
|
10080
|
+
: relationship.cardinality === 'many_to_one'
|
|
10081
|
+
? maxToPerKey <= 1
|
|
10082
|
+
: false;
|
|
10083
|
+
const policyPassed = relationship.fanout === 'safe' && cardinalityPassed;
|
|
10084
|
+
const queryFingerprint = createHash('sha256').update(sql).digest('hex');
|
|
10085
|
+
return {
|
|
10086
|
+
status: policyPassed ? 'passed' : 'failed',
|
|
10087
|
+
checkedAt: new Date().toISOString(),
|
|
10088
|
+
queryFingerprint,
|
|
10089
|
+
proofFingerprint: relationshipValidationProofFingerprint({
|
|
10090
|
+
fromRelation: fromNode.relation,
|
|
10091
|
+
toRelation: toNode.relation,
|
|
10092
|
+
keys: relationship.keys,
|
|
10093
|
+
cardinality: relationship.cardinality,
|
|
10094
|
+
fanout: relationship.fanout,
|
|
10095
|
+
queryFingerprint,
|
|
10096
|
+
}),
|
|
10097
|
+
fromRows: numericValue(row.from_rows),
|
|
10098
|
+
toRows: numericValue(row.to_rows),
|
|
10099
|
+
joinedRows: numericValue(row.joined_rows),
|
|
10100
|
+
fromNullKeys: numericValue(row.from_null_keys),
|
|
10101
|
+
toNullKeys: numericValue(row.to_null_keys),
|
|
10102
|
+
unmatchedFrom: numericValue(row.unmatched_from),
|
|
10103
|
+
maxFromPerKey,
|
|
10104
|
+
maxToPerKey,
|
|
10105
|
+
message: policyPassed
|
|
10106
|
+
? 'Warehouse evidence matches the declared cardinality and safe fanout policy.'
|
|
10107
|
+
: 'Warehouse evidence does not prove the declared cardinality and safe fanout policy.',
|
|
10108
|
+
};
|
|
10109
|
+
}
|
|
10110
|
+
function relatedProductsForDomain(manifest, domainId) {
|
|
10111
|
+
const apps = Object.values(manifest.apps ?? {})
|
|
10112
|
+
.filter((app) => app.ownerDomain === domainId || app.usesDomains.includes(domainId))
|
|
10113
|
+
.map((app) => ({
|
|
10114
|
+
id: app.id,
|
|
10115
|
+
name: app.name,
|
|
10116
|
+
filePath: app.filePath,
|
|
10117
|
+
ownerDomain: app.ownerDomain ?? app.domain,
|
|
10118
|
+
usesDomains: app.usesDomains,
|
|
10119
|
+
purpose: app.purpose,
|
|
10120
|
+
requiredExports: app.requiredExports,
|
|
10121
|
+
classification: app.classification,
|
|
10122
|
+
lifecycle: app.lifecycle,
|
|
10123
|
+
legacyLayout: app.filePath.startsWith('domains/'),
|
|
10124
|
+
}));
|
|
10125
|
+
const notebooks = Object.entries(manifest.notebooks)
|
|
10126
|
+
.filter(([, notebook]) => notebook.ownerDomain === domainId || notebook.usesDomains.includes(domainId))
|
|
10127
|
+
.map(([id, notebook]) => ({
|
|
10128
|
+
id,
|
|
10129
|
+
title: notebook.title,
|
|
10130
|
+
filePath: notebook.filePath,
|
|
10131
|
+
ownerDomain: notebook.ownerDomain,
|
|
10132
|
+
usesDomains: notebook.usesDomains,
|
|
10133
|
+
purpose: notebook.purpose,
|
|
10134
|
+
requiredExports: notebook.requiredExports,
|
|
10135
|
+
classification: notebook.classification,
|
|
10136
|
+
legacyLayout: notebook.filePath.startsWith('domains/'),
|
|
10137
|
+
}));
|
|
10138
|
+
return { apps, notebooks };
|
|
10139
|
+
}
|
|
10140
|
+
function domainWorkspaceSummary(manifest, domainId) {
|
|
10141
|
+
const pkg = manifest.modeling?.packages[domainId];
|
|
10142
|
+
const entities = Object.values(manifest.modeling?.entities ?? {}).filter((entity) => entity.domain === domainId);
|
|
10143
|
+
const entityKeys = new Set(Object.entries(manifest.modeling?.entities ?? {}).filter(([, entity]) => entity.domain === domainId).map(([key]) => key));
|
|
10144
|
+
const relationships = Object.values(manifest.modeling?.relationships ?? {}).filter((relationship) => entityKeys.has(relationship.from) || entityKeys.has(relationship.to));
|
|
10145
|
+
const contracts = Object.values(manifest.modeling?.contracts ?? {}).filter((contract) => contract.domain === domainId);
|
|
10146
|
+
const imports = Object.values(manifest.modeling?.interfaces?.imports ?? {}).filter((value) => value.domain === domainId);
|
|
10147
|
+
const exports = Object.values(manifest.modeling?.interfaces?.exports ?? {}).filter((value) => value.domain === domainId);
|
|
10148
|
+
return {
|
|
10149
|
+
id: domainId,
|
|
10150
|
+
parent: pkg?.parent,
|
|
10151
|
+
owner: pkg?.owner,
|
|
10152
|
+
filePath: pkg?.filePath,
|
|
10153
|
+
counts: {
|
|
10154
|
+
entities: entities.length,
|
|
10155
|
+
relationships: relationships.length,
|
|
10156
|
+
safeRelationships: relationships.filter((relationship) => relationship.automaticJoinAllowed).length,
|
|
10157
|
+
staleRelationships: relationships.filter((relationship) => relationship.staleCertification).length,
|
|
10158
|
+
contracts: contracts.length,
|
|
10159
|
+
imports: imports.length,
|
|
10160
|
+
exports: exports.length,
|
|
10161
|
+
relatedProducts: Object.values(relatedProductsForDomain(manifest, domainId)).reduce((sum, values) => sum + values.length, 0),
|
|
10162
|
+
},
|
|
10163
|
+
readiness: relationships.some((relationship) => relationship.staleCertification) ? 'attention' : entities.length > 0 ? 'ready' : 'setup',
|
|
10164
|
+
};
|
|
10165
|
+
}
|
|
10166
|
+
function quoteQualifiedIdentifier(value, quote = quoteSqlIdentifier) {
|
|
10167
|
+
return value.split('.').map((part) => quote(assertSafeSqlIdentifier(part))).join('.');
|
|
10168
|
+
}
|
|
10169
|
+
function quoteSqlIdentifier(value) {
|
|
10170
|
+
return `"${value.replace(/"/g, '""')}"`;
|
|
10171
|
+
}
|
|
10172
|
+
function assertSafeSqlIdentifier(value) {
|
|
10173
|
+
if (!value || !/^[A-Za-z_][A-Za-z0-9_$]*$/.test(value))
|
|
10174
|
+
throw new Error(`Unsafe SQL identifier: ${value}`);
|
|
10175
|
+
return value;
|
|
10176
|
+
}
|
|
10177
|
+
function numericValue(value) {
|
|
10178
|
+
if (typeof value === 'bigint')
|
|
10179
|
+
return Number(value);
|
|
10180
|
+
const numeric = Number(value ?? 0);
|
|
10181
|
+
return Number.isFinite(numeric) ? numeric : 0;
|
|
10182
|
+
}
|
|
7712
10183
|
export async function assertLocalQueryRuntimeReady(executor, connection) {
|
|
7713
10184
|
try {
|
|
7714
10185
|
const connector = await executor.getConnector(connection);
|
|
@@ -8010,6 +10481,7 @@ function serializeSkill(skill) {
|
|
|
8010
10481
|
user: undefined,
|
|
8011
10482
|
domain: skill.domain,
|
|
8012
10483
|
domains: skill.domains,
|
|
10484
|
+
modelAreaRefs: skill.modelAreaRefs,
|
|
8013
10485
|
kind: skill.kind,
|
|
8014
10486
|
status: skill.status,
|
|
8015
10487
|
owner: skill.owner,
|
|
@@ -8063,6 +10535,7 @@ function parseSkillInput(raw, fallbackId) {
|
|
|
8063
10535
|
user: undefined,
|
|
8064
10536
|
domain: typeof skill.domain === 'string' && skill.domain.trim() ? skill.domain.trim() : undefined,
|
|
8065
10537
|
domains: asStrings(skill.domains),
|
|
10538
|
+
modelAreaRefs: asStrings(skill.modelAreaRefs),
|
|
8066
10539
|
kind: skill.kind === 'domain_reference' || skill.kind === 'metric_policy' || skill.kind === 'glossary' || skill.kind === 'analysis_pattern' || skill.kind === 'sql_policy' ? skill.kind : 'custom',
|
|
8067
10540
|
status: skill.status === 'draft' || skill.status === 'deprecated' ? skill.status : 'active',
|
|
8068
10541
|
owner: typeof skill.owner === 'string' ? skill.owner : undefined,
|
|
@@ -8089,7 +10562,7 @@ function domainKey(value) {
|
|
|
8089
10562
|
}
|
|
8090
10563
|
function manifestDomainToDto(domain, counts) {
|
|
8091
10564
|
return {
|
|
8092
|
-
id: domain.name,
|
|
10565
|
+
id: domain.id ?? domain.name,
|
|
8093
10566
|
name: domain.name,
|
|
8094
10567
|
parent: domain.parent,
|
|
8095
10568
|
owner: domain.owner,
|
|
@@ -8139,7 +10612,7 @@ export function listDomains(projectRoot) {
|
|
|
8139
10612
|
}
|
|
8140
10613
|
return Object.values(domains)
|
|
8141
10614
|
.map((domain) => {
|
|
8142
|
-
const key = domainKey(domain.name);
|
|
10615
|
+
const key = domainKey(domain.id ?? domain.name);
|
|
8143
10616
|
return manifestDomainToDto(domain, {
|
|
8144
10617
|
blockCount: blockCounts.get(key) ?? 0,
|
|
8145
10618
|
skillCount: skillCounts.get(key) ?? 0,
|
|
@@ -8151,7 +10624,56 @@ export function listDomains(projectRoot) {
|
|
|
8151
10624
|
/** Find a single authored domain by name/id (case/slug-insensitive). */
|
|
8152
10625
|
function findDomain(projectRoot, nameOrId) {
|
|
8153
10626
|
const key = domainKey(nameOrId);
|
|
8154
|
-
return listDomains(projectRoot).find((domain) => domainKey(domain.name) === key);
|
|
10627
|
+
return listDomains(projectRoot).find((domain) => domainKey(domain.id) === key || domainKey(domain.name) === key);
|
|
10628
|
+
}
|
|
10629
|
+
function deleteCanonicalDomainDeclaration(projectRoot, nameOrId) {
|
|
10630
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
10631
|
+
const key = domainKey(nameOrId);
|
|
10632
|
+
const pkg = registry.values().find((candidate) => domainKey(candidate.id) === key || domainKey(candidate.name) === key);
|
|
10633
|
+
if (!pkg)
|
|
10634
|
+
return false;
|
|
10635
|
+
const declaration = join(projectRoot, pkg.declarationPath);
|
|
10636
|
+
if (existsSync(declaration))
|
|
10637
|
+
rmSync(declaration, { force: true });
|
|
10638
|
+
if (pkg.legacyYamlPath) {
|
|
10639
|
+
const legacy = join(projectRoot, pkg.legacyYamlPath);
|
|
10640
|
+
if (existsSync(legacy))
|
|
10641
|
+
rmSync(legacy, { force: true });
|
|
10642
|
+
}
|
|
10643
|
+
return true;
|
|
10644
|
+
}
|
|
10645
|
+
async function refreshUnifiedProjectIndexes(projectRoot) {
|
|
10646
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot) ?? undefined;
|
|
10647
|
+
const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook', dbtManifestPath });
|
|
10648
|
+
invalidateAgentProjectState(projectRoot);
|
|
10649
|
+
await reindexProject(projectRoot, {
|
|
10650
|
+
manifest,
|
|
10651
|
+
kgPath: defaultKgPath(projectRoot),
|
|
10652
|
+
forceMetadataCatalog: true,
|
|
10653
|
+
forceKgIndex: true,
|
|
10654
|
+
});
|
|
10655
|
+
}
|
|
10656
|
+
function collectDomainPackageAssets(projectRoot, registry) {
|
|
10657
|
+
const output = {};
|
|
10658
|
+
for (const pkg of registry.values())
|
|
10659
|
+
output[pkg.id] = {};
|
|
10660
|
+
for (const path of listProjectFiles(projectRoot).filter((candidate) => candidate.startsWith('domains/'))) {
|
|
10661
|
+
const pkg = registry.packageForPath(join(projectRoot, path));
|
|
10662
|
+
if (!pkg || path === pkg.declarationPath)
|
|
10663
|
+
continue;
|
|
10664
|
+
const local = relative(pkg.root, join(projectRoot, path)).replace(/\\/g, '/');
|
|
10665
|
+
const top = local.split('/')[0] ?? '';
|
|
10666
|
+
const kind = top === 'business-views' ? 'views' : top;
|
|
10667
|
+
if (!['terms', 'skills', 'blocks', 'views', 'notebooks', 'apps', 'evaluations', 'tests'].includes(kind))
|
|
10668
|
+
continue;
|
|
10669
|
+
const bucket = output[pkg.id] ?? (output[pkg.id] = {});
|
|
10670
|
+
(bucket[kind] ??= []).push(path);
|
|
10671
|
+
}
|
|
10672
|
+
for (const assets of Object.values(output)) {
|
|
10673
|
+
for (const paths of Object.values(assets))
|
|
10674
|
+
paths.sort();
|
|
10675
|
+
}
|
|
10676
|
+
return output;
|
|
8155
10677
|
}
|
|
8156
10678
|
/** Validate + normalize an inbound `{ domain }` body into a DomainInput. */
|
|
8157
10679
|
export function parseDomainInput(raw, fallbackId) {
|
|
@@ -8169,6 +10691,7 @@ export function parseDomainInput(raw, fallbackId) {
|
|
|
8169
10691
|
? value.filter((v) => typeof v === 'string' && v.trim().length > 0)
|
|
8170
10692
|
: undefined;
|
|
8171
10693
|
return {
|
|
10694
|
+
id: typeof domain.id === 'string' && domain.id.trim() ? domain.id.trim() : domainFolderSlug(name).replace(/-/g, '_'),
|
|
8172
10695
|
name,
|
|
8173
10696
|
parent: typeof domain.parent === 'string' ? domain.parent : undefined,
|
|
8174
10697
|
owner: typeof domain.owner === 'string' ? domain.owner : undefined,
|
|
@@ -8405,6 +10928,29 @@ export function loadProjectConfig(projectRoot) {
|
|
|
8405
10928
|
}
|
|
8406
10929
|
return config;
|
|
8407
10930
|
}
|
|
10931
|
+
/** Current Git-backed Skills folder and what the loader can read from it. */
|
|
10932
|
+
export function buildSkillPathSettings(projectRoot) {
|
|
10933
|
+
const resolvedPath = skillsDir(projectRoot);
|
|
10934
|
+
const loaded = loadSkills(projectRoot);
|
|
10935
|
+
return {
|
|
10936
|
+
path: configuredSkillsPath(projectRoot),
|
|
10937
|
+
resolvedPath,
|
|
10938
|
+
exists: existsSync(resolvedPath) && statSync(resolvedPath).isDirectory(),
|
|
10939
|
+
skillCount: loaded.skills.filter((skill) => skill.scope === 'project').length,
|
|
10940
|
+
errorCount: loaded.errors.length,
|
|
10941
|
+
};
|
|
10942
|
+
}
|
|
10943
|
+
function dbtFirstOwnsSemanticAuthoring(config) {
|
|
10944
|
+
return config.manifestVersion === 3 && config.modeling?.mode === 'dbt-first';
|
|
10945
|
+
}
|
|
10946
|
+
function sendDbtOwnedSemanticAuthoringError(res) {
|
|
10947
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10948
|
+
res.end(serializeJSON({
|
|
10949
|
+
error: 'dbt-first projects use dbt/MetricFlow as the physical and metric source of truth. Edit the dbt project and refresh artifacts instead of creating duplicate local semantic definitions.',
|
|
10950
|
+
code: 'DQL_MODELING_DBT_OWNED',
|
|
10951
|
+
nextAction: 'Create or edit the metric in dbt/MetricFlow, run dbt parse or dbt build, then run dql sync dbt.',
|
|
10952
|
+
}));
|
|
10953
|
+
}
|
|
8408
10954
|
/**
|
|
8409
10955
|
* Core of the `/api/propose` endpoint, factored out as a pure function so it can
|
|
8410
10956
|
* be unit-tested without standing up an HTTP server.
|
|
@@ -9359,6 +11905,38 @@ export function dashboardRuntimeVariables(dashboard, overrides = {}) {
|
|
|
9359
11905
|
}
|
|
9360
11906
|
return { ...variables, ...overrides };
|
|
9361
11907
|
}
|
|
11908
|
+
/**
|
|
11909
|
+
* Resolve only declared tile parameter bindings. Dashboard filters remain
|
|
11910
|
+
* values; they never become SQL identifiers or an implicit name-based binding.
|
|
11911
|
+
*/
|
|
11912
|
+
export function dashboardTileParameterValues(input) {
|
|
11913
|
+
const values = {};
|
|
11914
|
+
const requestValues = input.requestValues ?? {};
|
|
11915
|
+
const persona = requestValues.persona && typeof requestValues.persona === 'object' && !Array.isArray(requestValues.persona)
|
|
11916
|
+
? requestValues.persona
|
|
11917
|
+
: {};
|
|
11918
|
+
for (const binding of input.item.parameterBindings ?? []) {
|
|
11919
|
+
const lookup = binding.filter ?? binding.field ?? binding.param;
|
|
11920
|
+
let value;
|
|
11921
|
+
switch (binding.source) {
|
|
11922
|
+
case 'constant':
|
|
11923
|
+
value = binding.value;
|
|
11924
|
+
break;
|
|
11925
|
+
case 'dashboard_filter':
|
|
11926
|
+
value = input.dashboardValues[lookup];
|
|
11927
|
+
break;
|
|
11928
|
+
case 'persona':
|
|
11929
|
+
value = persona[lookup] ?? requestValues[`persona.${lookup}`];
|
|
11930
|
+
break;
|
|
11931
|
+
case 'variable':
|
|
11932
|
+
value = requestValues[lookup];
|
|
11933
|
+
break;
|
|
11934
|
+
}
|
|
11935
|
+
if (value !== undefined)
|
|
11936
|
+
values[binding.param] = value;
|
|
11937
|
+
}
|
|
11938
|
+
return values;
|
|
11939
|
+
}
|
|
9362
11940
|
export function applyDashboardFiltersToBlockExecution(input) {
|
|
9363
11941
|
const variables = { ...input.variables };
|
|
9364
11942
|
const sqlParams = [...input.sqlParams];
|
|
@@ -9723,6 +12301,297 @@ export function buildAgentPreviewSql(sql) {
|
|
|
9723
12301
|
throw new Error(readOnlyError);
|
|
9724
12302
|
return `SELECT * FROM (\n${withoutTrailingSemicolon}\n) AS dql_agent_preview LIMIT 200`;
|
|
9725
12303
|
}
|
|
12304
|
+
/**
|
|
12305
|
+
* EXP-001 preflight for model-authored exploratory SQL.
|
|
12306
|
+
*
|
|
12307
|
+
* This is deliberately narrow and deterministic: it repairs a singular/plural
|
|
12308
|
+
* qualifier typo only when it resolves to exactly one relation already present
|
|
12309
|
+
* in the query, and it changes SUM to MAX for a non-additive parent attribute
|
|
12310
|
+
* only when the owning entity is retained in GROUP BY. It never invents a
|
|
12311
|
+
* relation, join key, or allocation rule.
|
|
12312
|
+
*/
|
|
12313
|
+
export function repairExploratorySqlBeforeExecution(sql, schemaContext, question = '') {
|
|
12314
|
+
const repairs = [];
|
|
12315
|
+
let repairedSql = qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs);
|
|
12316
|
+
repairedSql = repairExploratoryRelationQualifiers(repairedSql, repairs);
|
|
12317
|
+
repairedSql = repairExploratoryLifetimeMeasureSelection(repairedSql, schemaContext, question, repairs);
|
|
12318
|
+
repairedSql = repairExploratoryMisleadingPercentAliases(repairedSql, question, repairs);
|
|
12319
|
+
const grainRepair = repairExploratoryNonAdditiveAggregates(repairedSql, schemaContext, repairs);
|
|
12320
|
+
repairedSql = grainRepair.sql;
|
|
12321
|
+
return {
|
|
12322
|
+
sql: repairedSql,
|
|
12323
|
+
repairs,
|
|
12324
|
+
...(grainRepair.blockedReason ? { blockedReason: grainRepair.blockedReason } : {}),
|
|
12325
|
+
};
|
|
12326
|
+
}
|
|
12327
|
+
/**
|
|
12328
|
+
* Prevent a generated amount/count from being presented as a percentage.
|
|
12329
|
+
* The repair is intentionally conservative: percent-intent questions and
|
|
12330
|
+
* expressions that visibly calculate a ratio/percentage are left untouched.
|
|
12331
|
+
*/
|
|
12332
|
+
function repairExploratoryMisleadingPercentAliases(sql, question, repairs) {
|
|
12333
|
+
if (/\b(percent(?:age)?|pct|share|ratio|rate|proportion)\b/i.test(question))
|
|
12334
|
+
return sql;
|
|
12335
|
+
const aggregateAlias = /((?:SUM|AVG|COUNT|MIN|MAX)\s*\([^)]*\))\s+AS\s+([a-zA-Z_][a-zA-Z0-9_]*?)(?:_pct|_percent|_percentage)\b/gi;
|
|
12336
|
+
const aliases = new Map();
|
|
12337
|
+
for (const match of sql.matchAll(aggregateAlias)) {
|
|
12338
|
+
const expression = match[1];
|
|
12339
|
+
const oldAlias = match[0].match(/\bAS\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*$/i)?.[1];
|
|
12340
|
+
const newAlias = match[2];
|
|
12341
|
+
if (!oldAlias || !newAlias)
|
|
12342
|
+
continue;
|
|
12343
|
+
if (/\/|\bpercent(?:age)?\b|\bratio\b|\brate\b|\*\s*100\b|\b100\s*\*/i.test(expression))
|
|
12344
|
+
continue;
|
|
12345
|
+
aliases.set(oldAlias, newAlias);
|
|
12346
|
+
}
|
|
12347
|
+
let repaired = sql;
|
|
12348
|
+
for (const [oldAlias, newAlias] of aliases) {
|
|
12349
|
+
repaired = repaired.replace(new RegExp(`\\b${escapeExploratoryRegex(oldAlias)}\\b`, 'g'), newAlias);
|
|
12350
|
+
repairs.push(`Renamed misleading percentage alias ${oldAlias} to ${newAlias}; the expression returns an amount, not a percentage.`);
|
|
12351
|
+
}
|
|
12352
|
+
return repaired;
|
|
12353
|
+
}
|
|
12354
|
+
/**
|
|
12355
|
+
* EXP-003 — make the typed overall ranking contract executable before the host
|
|
12356
|
+
* runs a review-required exploratory candidate. The answer-shape gate remains a
|
|
12357
|
+
* backstop, but a provider's generic LIMIT 100 must not trigger a second model
|
|
12358
|
+
* call when the planner already established top 10.
|
|
12359
|
+
*/
|
|
12360
|
+
export function applyRequestedTopNToExploratorySql(sql, requestedTopN) {
|
|
12361
|
+
if (!Number.isInteger(requestedTopN) || requestedTopN <= 0)
|
|
12362
|
+
return sql;
|
|
12363
|
+
const withoutTerminator = sql.trim().replace(/;\s*$/, '');
|
|
12364
|
+
const trailingLimit = /\blimit\s+\d+\s*$/i;
|
|
12365
|
+
if (trailingLimit.test(withoutTerminator)) {
|
|
12366
|
+
return withoutTerminator.replace(trailingLimit, `LIMIT ${requestedTopN}`);
|
|
12367
|
+
}
|
|
12368
|
+
return `${withoutTerminator}\nLIMIT ${requestedTopN}`;
|
|
12369
|
+
}
|
|
12370
|
+
function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs) {
|
|
12371
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12372
|
+
if (!analysis.parsed)
|
|
12373
|
+
return sql;
|
|
12374
|
+
let repaired = sql;
|
|
12375
|
+
for (const relation of analysis.tables) {
|
|
12376
|
+
const cleanRelation = relation.replace(/["`\[\]]/g, '');
|
|
12377
|
+
if (cleanRelation.includes('.'))
|
|
12378
|
+
continue;
|
|
12379
|
+
const basename = cleanRelation.toLowerCase();
|
|
12380
|
+
const candidates = schemaContext
|
|
12381
|
+
.map((table) => table.relation)
|
|
12382
|
+
.filter((candidate) => (candidate.split('.').at(-1) ?? candidate).toLowerCase() === basename)
|
|
12383
|
+
.filter((candidate, index, values) => values.indexOf(candidate) === index);
|
|
12384
|
+
if (candidates.length === 0)
|
|
12385
|
+
continue;
|
|
12386
|
+
const compatible = candidates.every((candidate) => candidates.every((other) => candidate.toLowerCase() === other.toLowerCase()
|
|
12387
|
+
|| candidate.toLowerCase().endsWith(`.${other.toLowerCase()}`)
|
|
12388
|
+
|| other.toLowerCase().endsWith(`.${candidate.toLowerCase()}`)));
|
|
12389
|
+
if (!compatible)
|
|
12390
|
+
continue;
|
|
12391
|
+
const replacement = [...candidates].sort((left, right) => right.split('.').length - left.split('.').length)[0];
|
|
12392
|
+
if (replacement.toLowerCase() === basename)
|
|
12393
|
+
continue;
|
|
12394
|
+
repaired = repaired.replace(new RegExp(`\\b(from|join)\\s+${escapeExploratoryRegex(cleanRelation)}\\b`, 'gi'), (_match, keyword) => `${keyword} ${replacement}`);
|
|
12395
|
+
repairs.push(`Qualified exploratory relation ${cleanRelation} as inspected relation ${replacement}.`);
|
|
12396
|
+
}
|
|
12397
|
+
return repaired;
|
|
12398
|
+
}
|
|
12399
|
+
function repairExploratoryRelationQualifiers(sql, repairs) {
|
|
12400
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12401
|
+
if (!analysis.parsed)
|
|
12402
|
+
return sql;
|
|
12403
|
+
const declared = Object.entries(analysis.aliasToRelation).map(([qualifier, relation]) => ({
|
|
12404
|
+
qualifier,
|
|
12405
|
+
subject: normalizeExploratoryRelationSubject(relation.split('.').at(-1) ?? relation),
|
|
12406
|
+
}));
|
|
12407
|
+
let repaired = sql;
|
|
12408
|
+
const unresolved = analysis.columns
|
|
12409
|
+
.filter((column) => column.tableAlias && column.relation === column.tableAlias && !analysis.aliasToRelation[column.tableAlias])
|
|
12410
|
+
.map((column) => column.tableAlias)
|
|
12411
|
+
.filter((value, index, values) => values.indexOf(value) === index);
|
|
12412
|
+
for (const qualifier of unresolved) {
|
|
12413
|
+
const subject = normalizeExploratoryRelationSubject(qualifier);
|
|
12414
|
+
const matches = declared.filter((candidate) => candidate.subject === subject);
|
|
12415
|
+
if (matches.length !== 1 || matches[0].qualifier === qualifier)
|
|
12416
|
+
continue;
|
|
12417
|
+
const replacement = matches[0].qualifier;
|
|
12418
|
+
repaired = repaired.replace(new RegExp(`\\b${escapeExploratoryRegex(qualifier)}\\s*\\.`, 'gi'), `${replacement}.`);
|
|
12419
|
+
repairs.push(`Corrected unresolved SQL qualifier ${qualifier} to the inspected relation alias ${replacement}.`);
|
|
12420
|
+
}
|
|
12421
|
+
return repaired;
|
|
12422
|
+
}
|
|
12423
|
+
function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question, repairs) {
|
|
12424
|
+
if (!/\b(lifetime|life\s*span|lifespan|customer\s+life)\b/i.test(question))
|
|
12425
|
+
return sql;
|
|
12426
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12427
|
+
if (!analysis.parsed || analysis.joins.length === 0)
|
|
12428
|
+
return sql;
|
|
12429
|
+
const groupClause = sql.match(/\bgroup\s+by\s+([\s\S]*?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i)?.[1] ?? '';
|
|
12430
|
+
let repaired = sql;
|
|
12431
|
+
for (const aggregate of analysis.aggregates) {
|
|
12432
|
+
if (aggregate.func.toLowerCase() !== 'sum' || !aggregate.column)
|
|
12433
|
+
continue;
|
|
12434
|
+
const lifetimeColumn = `lifetime_${aggregate.column.toLowerCase()}`;
|
|
12435
|
+
const owner = resolveUniqueExploratoryColumnOwner(schemaContext, lifetimeColumn);
|
|
12436
|
+
if (!owner)
|
|
12437
|
+
continue;
|
|
12438
|
+
const ownerAliases = Object.entries(analysis.aliasToRelation)
|
|
12439
|
+
.filter(([, relation]) => exploratoryRelationsMatch(relation, owner.relation))
|
|
12440
|
+
.map(([alias]) => alias);
|
|
12441
|
+
const ownerRetained = owner.columns.some((ownerColumn) => new RegExp(`(?:^|[^A-Za-z0-9_$])${escapeExploratoryRegex(ownerColumn.name)}(?:[^A-Za-z0-9_$]|$)`, 'i').test(groupClause));
|
|
12442
|
+
if (!ownerRetained || ownerAliases.length === 0)
|
|
12443
|
+
continue;
|
|
12444
|
+
const sourcePattern = new RegExp(`\\bsum\\s*\\(\\s*(?:[A-Za-z_][A-Za-z0-9_$]*\\s*\\.\\s*)?${escapeExploratoryRegex(aggregate.column)}\\s*\\)`, 'gi');
|
|
12445
|
+
if (!sourcePattern.test(repaired))
|
|
12446
|
+
continue;
|
|
12447
|
+
repaired = repaired.replace(sourcePattern, `MAX(${ownerAliases[0]}.${lifetimeColumn})`);
|
|
12448
|
+
repairs.push(`Used ${lifetimeColumn} from ${owner.name} at the retained entity grain instead of multiplying ${aggregate.column} across lower-grain joined rows.`);
|
|
12449
|
+
}
|
|
12450
|
+
return repaired;
|
|
12451
|
+
}
|
|
12452
|
+
function repairExploratoryNonAdditiveAggregates(sql, schemaContext, repairs) {
|
|
12453
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12454
|
+
if (!analysis.parsed || analysis.joins.length === 0)
|
|
12455
|
+
return { sql };
|
|
12456
|
+
const risky = analysis.aggregates.filter((aggregate) => aggregate.func.toLowerCase() === 'sum'
|
|
12457
|
+
&& Boolean(aggregate.column)
|
|
12458
|
+
&& /(^|_)(lifetime|cumulative|balance|snapshot|rate|ratio|average|avg)(_|$)/i.test(aggregate.column));
|
|
12459
|
+
if (risky.length === 0)
|
|
12460
|
+
return { sql };
|
|
12461
|
+
const groupClause = sql.match(/\bgroup\s+by\s+([\s\S]*?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i)?.[1] ?? '';
|
|
12462
|
+
let repaired = sql;
|
|
12463
|
+
for (const aggregate of risky) {
|
|
12464
|
+
const column = aggregate.column;
|
|
12465
|
+
const owner = resolveUniqueExploratoryColumnOwner(schemaContext, column);
|
|
12466
|
+
if (!owner) {
|
|
12467
|
+
return {
|
|
12468
|
+
sql: repaired,
|
|
12469
|
+
blockedReason: `The exploratory query sums non-additive measure ${column} after a join, but DQL cannot prove its owning grain. Clarify the desired allocation or keep the owning entity in the result.`,
|
|
12470
|
+
};
|
|
12471
|
+
}
|
|
12472
|
+
const ownerAliases = Object.entries(analysis.aliasToRelation)
|
|
12473
|
+
.filter(([, relation]) => exploratoryRelationsMatch(relation, owner.relation))
|
|
12474
|
+
.map(([alias]) => alias);
|
|
12475
|
+
const ownerRetained = owner.columns.some((ownerColumn) => {
|
|
12476
|
+
const name = escapeExploratoryRegex(ownerColumn.name);
|
|
12477
|
+
if (new RegExp(`(?:^|[^A-Za-z0-9_$])${name}(?:[^A-Za-z0-9_$]|$)`, 'i').test(groupClause))
|
|
12478
|
+
return true;
|
|
12479
|
+
return ownerAliases.some((alias) => new RegExp(`\\b${escapeExploratoryRegex(alias)}\\s*\\.\\s*${name}\\b`, 'i').test(groupClause));
|
|
12480
|
+
});
|
|
12481
|
+
if (!ownerRetained) {
|
|
12482
|
+
return {
|
|
12483
|
+
sql: repaired,
|
|
12484
|
+
blockedReason: `The exploratory query would allocate non-additive measure ${column} below its owning ${owner.name} grain. Keep ${owner.name} in the output or provide an allocation policy.`,
|
|
12485
|
+
};
|
|
12486
|
+
}
|
|
12487
|
+
const sumPattern = new RegExp(`\\bsum\\s*\\(\\s*((?:[A-Za-z_][A-Za-z0-9_$]*\\s*\\.\\s*)?${escapeExploratoryRegex(column)})\\s*\\)`, 'gi');
|
|
12488
|
+
if (!sumPattern.test(repaired))
|
|
12489
|
+
continue;
|
|
12490
|
+
repaired = repaired.replace(sumPattern, 'MAX($1)');
|
|
12491
|
+
repairs.push(`Preserved non-additive ${column} at ${owner.name} grain instead of summing repeated values after the join.`);
|
|
12492
|
+
}
|
|
12493
|
+
return { sql: repaired };
|
|
12494
|
+
}
|
|
12495
|
+
function resolveUniqueExploratoryColumnOwner(schemaContext, column) {
|
|
12496
|
+
const candidates = schemaContext.filter((table) => table.columns.some((candidate) => candidate.name.toLowerCase() === column.toLowerCase()));
|
|
12497
|
+
if (candidates.length === 0)
|
|
12498
|
+
return undefined;
|
|
12499
|
+
const compatible = candidates.every((candidate) => candidates.every((other) => exploratoryRelationsMatch(candidate.relation, other.relation)));
|
|
12500
|
+
if (!compatible)
|
|
12501
|
+
return undefined;
|
|
12502
|
+
const preferred = [...candidates].sort((left, right) => right.relation.split('.').length - left.relation.split('.').length)[0];
|
|
12503
|
+
const columns = new Map();
|
|
12504
|
+
for (const candidate of candidates) {
|
|
12505
|
+
for (const candidateColumn of candidate.columns)
|
|
12506
|
+
columns.set(candidateColumn.name.toLowerCase(), candidateColumn);
|
|
12507
|
+
}
|
|
12508
|
+
return { ...preferred, columns: [...columns.values()] };
|
|
12509
|
+
}
|
|
12510
|
+
function normalizeExploratoryRelationSubject(value) {
|
|
12511
|
+
const normalized = value.toLowerCase().replace(/[^a-z0-9_]/g, '');
|
|
12512
|
+
if (normalized.endsWith('ies') && normalized.length > 4)
|
|
12513
|
+
return `${normalized.slice(0, -3)}y`;
|
|
12514
|
+
if (normalized.endsWith('s') && normalized.length > 3)
|
|
12515
|
+
return normalized.slice(0, -1);
|
|
12516
|
+
return normalized;
|
|
12517
|
+
}
|
|
12518
|
+
function exploratoryRelationsMatch(left, right) {
|
|
12519
|
+
const leftLower = left.toLowerCase();
|
|
12520
|
+
const rightLower = right.toLowerCase();
|
|
12521
|
+
return leftLower === rightLower
|
|
12522
|
+
|| leftLower.endsWith(`.${rightLower}`)
|
|
12523
|
+
|| rightLower.endsWith(`.${leftLower}`)
|
|
12524
|
+
|| normalizeExploratoryRelationSubject(leftLower.split('.').at(-1) ?? leftLower)
|
|
12525
|
+
=== normalizeExploratoryRelationSubject(rightLower.split('.').at(-1) ?? rightLower);
|
|
12526
|
+
}
|
|
12527
|
+
function escapeExploratoryRegex(value) {
|
|
12528
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
12529
|
+
}
|
|
12530
|
+
/** Render parser-derived physical identifiers only; exploratory probes never interpolate arbitrary SQL. */
|
|
12531
|
+
function quoteExploratoryIdentifier(value, label) {
|
|
12532
|
+
const parts = value
|
|
12533
|
+
.replace(/["`\[\]]/g, '')
|
|
12534
|
+
.split('.')
|
|
12535
|
+
.map((part) => part.trim())
|
|
12536
|
+
.filter(Boolean);
|
|
12537
|
+
if (parts.length === 0 || parts.some((part) => !/^[A-Za-z_][A-Za-z0-9_$]*$/.test(part))) {
|
|
12538
|
+
throw new Error(`Exploratory ${label} is not a safe physical identifier.`);
|
|
12539
|
+
}
|
|
12540
|
+
return parts.map((part) => `"${part.replace(/"/g, '""')}"`).join('.');
|
|
12541
|
+
}
|
|
12542
|
+
/** One bounded, sampled relationship observation per inferred equality join. */
|
|
12543
|
+
export function buildExploratoryJoinProbeSql(input) {
|
|
12544
|
+
const leftRelation = quoteExploratoryIdentifier(input.leftRelation, 'left relation');
|
|
12545
|
+
const leftColumn = quoteExploratoryIdentifier(input.leftColumn, 'left column');
|
|
12546
|
+
const rightRelation = quoteExploratoryIdentifier(input.rightRelation, 'right relation');
|
|
12547
|
+
const rightColumn = quoteExploratoryIdentifier(input.rightColumn, 'right column');
|
|
12548
|
+
return `WITH
|
|
12549
|
+
left_sample AS (
|
|
12550
|
+
SELECT ${leftColumn} AS join_key
|
|
12551
|
+
FROM ${leftRelation}
|
|
12552
|
+
WHERE ${leftColumn} IS NOT NULL
|
|
12553
|
+
LIMIT 5000
|
|
12554
|
+
),
|
|
12555
|
+
right_sample AS (
|
|
12556
|
+
SELECT ${rightColumn} AS join_key
|
|
12557
|
+
FROM ${rightRelation}
|
|
12558
|
+
WHERE ${rightColumn} IS NOT NULL
|
|
12559
|
+
LIMIT 5000
|
|
12560
|
+
),
|
|
12561
|
+
matches AS (
|
|
12562
|
+
SELECT l.join_key AS left_key, r.join_key AS right_key
|
|
12563
|
+
FROM left_sample AS l
|
|
12564
|
+
INNER JOIN right_sample AS r ON l.join_key = r.join_key
|
|
12565
|
+
),
|
|
12566
|
+
left_counts AS (
|
|
12567
|
+
SELECT left_key, COUNT(*) AS match_count
|
|
12568
|
+
FROM matches
|
|
12569
|
+
GROUP BY left_key
|
|
12570
|
+
),
|
|
12571
|
+
right_counts AS (
|
|
12572
|
+
SELECT right_key, COUNT(*) AS match_count
|
|
12573
|
+
FROM matches
|
|
12574
|
+
GROUP BY right_key
|
|
12575
|
+
)
|
|
12576
|
+
SELECT
|
|
12577
|
+
(SELECT COUNT(*) FROM left_sample) AS left_sample_rows,
|
|
12578
|
+
(SELECT COUNT(*) FROM right_sample) AS right_sample_rows,
|
|
12579
|
+
(SELECT COUNT(*) FROM matches) AS joined_rows,
|
|
12580
|
+
(SELECT COUNT(*) FROM left_sample AS l WHERE NOT EXISTS (SELECT 1 FROM right_sample AS r WHERE r.join_key = l.join_key)) AS unmatched_left_sample_rows,
|
|
12581
|
+
(SELECT COUNT(*) FROM right_sample AS r WHERE NOT EXISTS (SELECT 1 FROM left_sample AS l WHERE l.join_key = r.join_key)) AS unmatched_right_sample_rows,
|
|
12582
|
+
(SELECT COALESCE(MAX(match_count), 0) FROM left_counts) AS max_matches_per_left_key,
|
|
12583
|
+
(SELECT COALESCE(MAX(match_count), 0) FROM right_counts) AS max_matches_per_right_key`;
|
|
12584
|
+
}
|
|
12585
|
+
function summarizeExploratoryJoinProbe(rows, leftRelation, leftColumn, rightRelation, rightColumn) {
|
|
12586
|
+
const row = Array.isArray(rows) && rows[0] && typeof rows[0] === 'object'
|
|
12587
|
+
? rows[0]
|
|
12588
|
+
: {};
|
|
12589
|
+
const number = (key) => {
|
|
12590
|
+
const value = Number(row[key]);
|
|
12591
|
+
return Number.isFinite(value) ? value : 0;
|
|
12592
|
+
};
|
|
12593
|
+
return `${leftRelation}.${leftColumn} ↔ ${rightRelation}.${rightColumn}: sample left=${number('left_sample_rows')}, right=${number('right_sample_rows')}, joined=${number('joined_rows')}, unmatched left/right=${number('unmatched_left_sample_rows')}/${number('unmatched_right_sample_rows')}, max matches left/right=${number('max_matches_per_left_key')}/${number('max_matches_per_right_key')}`;
|
|
12594
|
+
}
|
|
9726
12595
|
function readOnlySqlValidationError(sql, subject) {
|
|
9727
12596
|
const scanSql = stripSqlStringsAndComments(sql).trim();
|
|
9728
12597
|
if (!/^(select|with)\b/i.test(scanSql)) {
|
|
@@ -9793,11 +12662,11 @@ function stripSqlStringsAndComments(sql) {
|
|
|
9793
12662
|
* Used by notebook SQL execution and Block Studio validation so both paths
|
|
9794
12663
|
* behave identically. If the SQL has no refs, returns it unchanged.
|
|
9795
12664
|
*/
|
|
9796
|
-
export function prepareSemanticSql(sql, semanticLayer) {
|
|
12665
|
+
export function prepareSemanticSql(sql, semanticLayer, options) {
|
|
9797
12666
|
if (!hasSemanticRefs(sql)) {
|
|
9798
12667
|
return { sql, semanticRefs: { metrics: [], dimensions: [] }, unresolvedRefs: [] };
|
|
9799
12668
|
}
|
|
9800
|
-
const resolution = resolveSemanticRefs(sql, semanticLayer);
|
|
12669
|
+
const resolution = resolveSemanticRefs(sql, semanticLayer, options);
|
|
9801
12670
|
return {
|
|
9802
12671
|
sql: resolution.resolvedSql,
|
|
9803
12672
|
semanticRefs: {
|
|
@@ -9870,6 +12739,34 @@ function readJSON(req) {
|
|
|
9870
12739
|
req.on('error', reject);
|
|
9871
12740
|
});
|
|
9872
12741
|
}
|
|
12742
|
+
async function readMultipartForm(req) {
|
|
12743
|
+
const headers = new Headers();
|
|
12744
|
+
for (const [key, value] of Object.entries(req.headers)) {
|
|
12745
|
+
if (Array.isArray(value))
|
|
12746
|
+
value.forEach((item) => headers.append(key, item));
|
|
12747
|
+
else if (value !== undefined)
|
|
12748
|
+
headers.set(key, value);
|
|
12749
|
+
}
|
|
12750
|
+
const request = new Request("http://127.0.0.1/upload", {
|
|
12751
|
+
method: "POST",
|
|
12752
|
+
headers,
|
|
12753
|
+
body: Readable.toWeb(req),
|
|
12754
|
+
duplex: "half",
|
|
12755
|
+
});
|
|
12756
|
+
const form = await request.formData();
|
|
12757
|
+
const fields = {};
|
|
12758
|
+
let file;
|
|
12759
|
+
for (const [key, value] of form.entries()) {
|
|
12760
|
+
if (typeof value === "string")
|
|
12761
|
+
fields[key] = value;
|
|
12762
|
+
else if (key === "file")
|
|
12763
|
+
file = {
|
|
12764
|
+
filename: value.name,
|
|
12765
|
+
content: Buffer.from(await value.arrayBuffer()),
|
|
12766
|
+
};
|
|
12767
|
+
}
|
|
12768
|
+
return { fields, file };
|
|
12769
|
+
}
|
|
9873
12770
|
function safeJoin(rootDir, requestPath) {
|
|
9874
12771
|
const normalized = normalize(requestPath).replace(/^([.][.][/\\])+/, '');
|
|
9875
12772
|
const fullPath = resolve(rootDir, `.${normalized.startsWith('/') ? normalized : `/${normalized}`}`);
|
|
@@ -9987,7 +12884,43 @@ function scanNotebookFiles(projectRoot) {
|
|
|
9987
12884
|
continue;
|
|
9988
12885
|
collect(dir, folder, type);
|
|
9989
12886
|
}
|
|
12887
|
+
collectDomainArtifacts(join(projectRoot, 'domains'), 'domains');
|
|
9990
12888
|
return result;
|
|
12889
|
+
function collectDomainArtifacts(currentDir, relativeDir) {
|
|
12890
|
+
if (!existsSync(currentDir))
|
|
12891
|
+
return;
|
|
12892
|
+
try {
|
|
12893
|
+
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
|
|
12894
|
+
if (entry.name.startsWith('.') || entry.name === 'node_modules')
|
|
12895
|
+
continue;
|
|
12896
|
+
const fullPath = join(currentDir, entry.name);
|
|
12897
|
+
const relativePath = `${relativeDir}/${entry.name}`;
|
|
12898
|
+
if (entry.isDirectory()) {
|
|
12899
|
+
collectDomainArtifacts(fullPath, relativePath);
|
|
12900
|
+
continue;
|
|
12901
|
+
}
|
|
12902
|
+
if (!entry.isFile())
|
|
12903
|
+
continue;
|
|
12904
|
+
const segments = relativePath.split('/');
|
|
12905
|
+
const ownerFolder = segments.find((segment) => ['notebooks', 'blocks', 'terms', 'views'].includes(segment));
|
|
12906
|
+
const type = ownerFolder === 'notebooks' ? 'notebook'
|
|
12907
|
+
: ownerFolder === 'blocks' ? 'block'
|
|
12908
|
+
: ownerFolder === 'terms' ? 'term'
|
|
12909
|
+
: ownerFolder === 'views' ? 'business_view'
|
|
12910
|
+
: undefined;
|
|
12911
|
+
if (!type || (!entry.name.endsWith('.dql') && !entry.name.endsWith('.dqlnb')))
|
|
12912
|
+
continue;
|
|
12913
|
+
const fallbackName = entry.name.replace(/\.(dql|dqlnb)$/, '');
|
|
12914
|
+
result.push({
|
|
12915
|
+
name: inferDqlArtifactName(fullPath, type, fallbackName),
|
|
12916
|
+
path: relativePath,
|
|
12917
|
+
type,
|
|
12918
|
+
folder: relativeDir,
|
|
12919
|
+
});
|
|
12920
|
+
}
|
|
12921
|
+
}
|
|
12922
|
+
catch { /* skip unreadable domain package dirs */ }
|
|
12923
|
+
}
|
|
9991
12924
|
function collect(currentDir, relativeDir, type) {
|
|
9992
12925
|
try {
|
|
9993
12926
|
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
|
|
@@ -10315,7 +13248,13 @@ export function buildSemanticTableMapping(semanticLayer, rows) {
|
|
|
10315
13248
|
allSemanticTables.add(metric.table);
|
|
10316
13249
|
for (const dimension of semanticLayer.listDimensions())
|
|
10317
13250
|
allSemanticTables.add(dimension.table);
|
|
13251
|
+
for (const measure of semanticLayer.listMeasures())
|
|
13252
|
+
allSemanticTables.add(measure.table);
|
|
13253
|
+
for (const model of semanticLayer.listSemanticModels())
|
|
13254
|
+
allSemanticTables.add(model.table);
|
|
10318
13255
|
for (const semTable of allSemanticTables) {
|
|
13256
|
+
if (!semTable)
|
|
13257
|
+
continue;
|
|
10319
13258
|
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
10320
13259
|
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
10321
13260
|
}
|
|
@@ -10335,23 +13274,32 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
|
10335
13274
|
const dbtProjectPath = effectiveSemanticConfig?.provider === 'dbt'
|
|
10336
13275
|
? effectiveSemanticConfig.projectPath
|
|
10337
13276
|
: context.projectConfig.dbt?.projectDir;
|
|
10338
|
-
|
|
10339
|
-
|
|
10340
|
-
|
|
10341
|
-
|
|
10342
|
-
|
|
10343
|
-
|
|
10344
|
-
|
|
10345
|
-
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10351
|
-
|
|
10352
|
-
|
|
10353
|
-
|
|
10354
|
-
|
|
13277
|
+
try {
|
|
13278
|
+
const compiled = compileMetricFlowQuery({
|
|
13279
|
+
projectRoot: context.projectRoot,
|
|
13280
|
+
dbtProjectPath,
|
|
13281
|
+
metrics: request.metrics,
|
|
13282
|
+
dimensions: request.dimensions,
|
|
13283
|
+
filters: request.filters,
|
|
13284
|
+
timeDimension: request.timeDimension,
|
|
13285
|
+
orderBy: request.orderBy,
|
|
13286
|
+
limit: request.limit,
|
|
13287
|
+
savedQuery: request.savedQuery,
|
|
13288
|
+
});
|
|
13289
|
+
return {
|
|
13290
|
+
sql: compiled.sql,
|
|
13291
|
+
joins: [],
|
|
13292
|
+
tables: [],
|
|
13293
|
+
engine: 'metricflow',
|
|
13294
|
+
};
|
|
13295
|
+
}
|
|
13296
|
+
catch (error) {
|
|
13297
|
+
// An explicit MetricFlow request must retain its strict engine contract.
|
|
13298
|
+
// For the default dbt path, a missing local CLI may fall back only if the
|
|
13299
|
+
// native composer can safely materialize the requested simple metrics.
|
|
13300
|
+
if (request.engine === 'metricflow' || !(error instanceof MetricFlowUnavailableError))
|
|
13301
|
+
throw error;
|
|
13302
|
+
}
|
|
10355
13303
|
}
|
|
10356
13304
|
const composed = semanticLayer.composeQuery({
|
|
10357
13305
|
metrics: request.metrics,
|
|
@@ -10367,6 +13315,7 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
|
10367
13315
|
}
|
|
10368
13316
|
function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
10369
13317
|
const config = parseSemanticBlockConfig(source);
|
|
13318
|
+
const runtimeBindings = semanticRuntimeParameterBindings(source, options?.parameters ?? {});
|
|
10370
13319
|
const metrics = config.metrics.length > 0
|
|
10371
13320
|
? config.metrics
|
|
10372
13321
|
: config.metric
|
|
@@ -10423,11 +13372,11 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
10423
13372
|
? composeRuntimeSemanticQuery({
|
|
10424
13373
|
metrics,
|
|
10425
13374
|
dimensions: config.dimensions,
|
|
10426
|
-
filters: config.filters,
|
|
13375
|
+
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
10427
13376
|
timeDimension: config.timeDimension && config.granularity
|
|
10428
13377
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
10429
13378
|
: undefined,
|
|
10430
|
-
limit: config.limit,
|
|
13379
|
+
limit: runtimeBindings.limit ?? config.limit,
|
|
10431
13380
|
}, semanticLayer, {
|
|
10432
13381
|
projectRoot: options.projectRoot,
|
|
10433
13382
|
projectConfig: options.projectConfig,
|
|
@@ -10438,11 +13387,11 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
10438
13387
|
: semanticLayer.composeQuery({
|
|
10439
13388
|
metrics,
|
|
10440
13389
|
dimensions: config.dimensions,
|
|
10441
|
-
filters: config.filters,
|
|
13390
|
+
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
10442
13391
|
timeDimension: config.timeDimension && config.granularity
|
|
10443
13392
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
10444
13393
|
: undefined,
|
|
10445
|
-
limit: config.limit,
|
|
13394
|
+
limit: runtimeBindings.limit ?? config.limit,
|
|
10446
13395
|
driver: options?.driver,
|
|
10447
13396
|
tableMapping: options?.tableMapping,
|
|
10448
13397
|
});
|
|
@@ -10469,6 +13418,39 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
10469
13418
|
semanticRefs,
|
|
10470
13419
|
};
|
|
10471
13420
|
}
|
|
13421
|
+
function semanticRuntimeParameterBindings(source, values) {
|
|
13422
|
+
const program = new Parser(source, '<semantic-parameters>').parse();
|
|
13423
|
+
const block = program.statements.find((statement) => statement.kind === NodeKind.BlockDecl);
|
|
13424
|
+
if (!block || block.kind !== NodeKind.BlockDecl || block.blockType !== 'semantic')
|
|
13425
|
+
return { filters: [] };
|
|
13426
|
+
const filters = [];
|
|
13427
|
+
let limit;
|
|
13428
|
+
for (const definition of blockParameterDefinitions(block)) {
|
|
13429
|
+
const value = values[definition.name];
|
|
13430
|
+
if (value === undefined || value === null || value === '')
|
|
13431
|
+
continue;
|
|
13432
|
+
if (definition.binding?.kind === 'semantic_filter') {
|
|
13433
|
+
const rawValues = Array.isArray(value) ? value : [value];
|
|
13434
|
+
filters.push({
|
|
13435
|
+
dimension: definition.binding.field,
|
|
13436
|
+
operator: definition.binding.operator,
|
|
13437
|
+
values: rawValues.map((item) => String(item)),
|
|
13438
|
+
});
|
|
13439
|
+
}
|
|
13440
|
+
else if (definition.binding?.kind === 'limit') {
|
|
13441
|
+
const parsed = typeof value === 'number' ? value : Number(value);
|
|
13442
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
13443
|
+
limit = Math.floor(parsed);
|
|
13444
|
+
}
|
|
13445
|
+
}
|
|
13446
|
+
return { filters, ...(limit !== undefined ? { limit } : {}) };
|
|
13447
|
+
}
|
|
13448
|
+
function mergeSemanticRuntimeFilters(declared, runtime) {
|
|
13449
|
+
if (runtime.length === 0)
|
|
13450
|
+
return declared;
|
|
13451
|
+
const runtimeDimensions = new Set(runtime.map((filter) => filter.dimension));
|
|
13452
|
+
return [...declared.filter((filter) => !runtimeDimensions.has(filter.dimension)), ...runtime];
|
|
13453
|
+
}
|
|
10472
13454
|
function resolveCustomBlockSql(sql, semanticLayer) {
|
|
10473
13455
|
if (!sql) {
|
|
10474
13456
|
return {
|
|
@@ -10597,12 +13579,17 @@ export function validateBlockStudioSource(source, semanticLayer) {
|
|
|
10597
13579
|
message: 'No executable SQL found in the block source.',
|
|
10598
13580
|
});
|
|
10599
13581
|
}
|
|
13582
|
+
const parameterInvocation = prepareBlockInvocation({ source, surface: 'block_studio' });
|
|
13583
|
+
if (parameterInvocation.errors.length > 0) {
|
|
13584
|
+
diagnostics.push(...parameterInvocation.errors.map((message) => ({ severity: 'error', code: 'parameter_parse', message })));
|
|
13585
|
+
}
|
|
10600
13586
|
return {
|
|
10601
13587
|
valid: diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
10602
13588
|
diagnostics,
|
|
10603
13589
|
semanticRefs,
|
|
10604
13590
|
chartConfig: chartConfig ?? undefined,
|
|
10605
13591
|
executableSql,
|
|
13592
|
+
parameters: parameterInvocation.parameters,
|
|
10606
13593
|
};
|
|
10607
13594
|
}
|
|
10608
13595
|
export async function createDqlArtifactGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
@@ -13286,26 +16273,117 @@ function yamlScalar(value) {
|
|
|
13286
16273
|
function buildNotebookTemplate(title, template) {
|
|
13287
16274
|
const id = () => Math.random().toString(36).slice(2, 10);
|
|
13288
16275
|
let cells;
|
|
13289
|
-
if (template ===
|
|
16276
|
+
if (template === "analysis") {
|
|
16277
|
+
cells = [
|
|
16278
|
+
{
|
|
16279
|
+
id: id(),
|
|
16280
|
+
type: "markdown",
|
|
16281
|
+
content: "# TL;DR\n\nSummarize the answer and decision here.",
|
|
16282
|
+
},
|
|
16283
|
+
{
|
|
16284
|
+
id: id(),
|
|
16285
|
+
type: "markdown",
|
|
16286
|
+
content: "## Context and methods\n\nState the business question, scope, definitions, and assumptions.",
|
|
16287
|
+
},
|
|
16288
|
+
{
|
|
16289
|
+
id: id(),
|
|
16290
|
+
type: "dql",
|
|
16291
|
+
name: "analysis_data",
|
|
16292
|
+
content: "",
|
|
16293
|
+
},
|
|
16294
|
+
{
|
|
16295
|
+
id: id(),
|
|
16296
|
+
type: "markdown",
|
|
16297
|
+
content: "## Results\n\nExplain the material findings and uncertainty.",
|
|
16298
|
+
},
|
|
16299
|
+
{
|
|
16300
|
+
id: id(),
|
|
16301
|
+
type: "markdown",
|
|
16302
|
+
content: "## Takeaways\n\nRecord the decision, risks, and next action.",
|
|
16303
|
+
},
|
|
16304
|
+
];
|
|
16305
|
+
}
|
|
16306
|
+
else if (template === "metric_diagnostic") {
|
|
13290
16307
|
cells = [
|
|
13291
|
-
{
|
|
13292
|
-
|
|
13293
|
-
|
|
16308
|
+
{
|
|
16309
|
+
id: id(),
|
|
16310
|
+
type: "markdown",
|
|
16311
|
+
content: "# Metric diagnostic\n\nWhat changed, when, and for whom?",
|
|
16312
|
+
},
|
|
16313
|
+
{
|
|
16314
|
+
id: id(),
|
|
16315
|
+
type: "dql",
|
|
16316
|
+
name: "metric_trend",
|
|
16317
|
+
content: "",
|
|
16318
|
+
},
|
|
16319
|
+
{
|
|
16320
|
+
id: id(),
|
|
16321
|
+
type: "dql",
|
|
16322
|
+
name: "driver_breakdown",
|
|
16323
|
+
content: "",
|
|
16324
|
+
},
|
|
16325
|
+
{
|
|
16326
|
+
id: id(),
|
|
16327
|
+
type: "markdown",
|
|
16328
|
+
content: "## Diagnosis\n\nSeparate validated drivers from hypotheses and data limitations.",
|
|
16329
|
+
},
|
|
13294
16330
|
];
|
|
13295
16331
|
}
|
|
13296
|
-
else if (template ===
|
|
16332
|
+
else if (template === "data_quality") {
|
|
13297
16333
|
cells = [
|
|
13298
|
-
{
|
|
13299
|
-
|
|
16334
|
+
{
|
|
16335
|
+
id: id(),
|
|
16336
|
+
type: "markdown",
|
|
16337
|
+
content: "# Data-quality investigation\n\nDocument the source, owner, freshness expectation, and affected decisions.",
|
|
16338
|
+
},
|
|
16339
|
+
{
|
|
16340
|
+
id: id(),
|
|
16341
|
+
type: "dql",
|
|
16342
|
+
name: "quality_profile",
|
|
16343
|
+
content: "",
|
|
16344
|
+
},
|
|
16345
|
+
{
|
|
16346
|
+
id: id(),
|
|
16347
|
+
type: "dql",
|
|
16348
|
+
name: "quality_checks",
|
|
16349
|
+
content: "",
|
|
16350
|
+
},
|
|
16351
|
+
{
|
|
16352
|
+
id: id(),
|
|
16353
|
+
type: "markdown",
|
|
16354
|
+
content: "## Findings and disposition\n\nRecord severity, impacted metrics, owner, and remediation.",
|
|
16355
|
+
},
|
|
13300
16356
|
];
|
|
13301
16357
|
}
|
|
13302
|
-
else {
|
|
16358
|
+
else if (template === "experiment") {
|
|
13303
16359
|
cells = [
|
|
13304
|
-
{
|
|
13305
|
-
|
|
16360
|
+
{
|
|
16361
|
+
id: id(),
|
|
16362
|
+
type: "markdown",
|
|
16363
|
+
content: "# Experiment log\n\nHypothesis, primary metric, guardrails, population, and dates.",
|
|
16364
|
+
},
|
|
16365
|
+
{
|
|
16366
|
+
id: id(),
|
|
16367
|
+
type: "dql",
|
|
16368
|
+
name: "experiment_results",
|
|
16369
|
+
content: "",
|
|
16370
|
+
},
|
|
16371
|
+
{
|
|
16372
|
+
id: id(),
|
|
16373
|
+
type: "markdown",
|
|
16374
|
+
content: "## Results\n\nReport effect size, uncertainty, guardrails, and data-quality checks.",
|
|
16375
|
+
},
|
|
16376
|
+
{
|
|
16377
|
+
id: id(),
|
|
16378
|
+
type: "markdown",
|
|
16379
|
+
content: "## Decision\n\nShip, iterate, or stop — with rationale.",
|
|
16380
|
+
},
|
|
13306
16381
|
];
|
|
13307
16382
|
}
|
|
13308
|
-
|
|
16383
|
+
else {
|
|
16384
|
+
cells = [];
|
|
16385
|
+
}
|
|
16386
|
+
return JSON.stringify({ dqlnbVersion: 2, version: 1, title, cells }, null, 2);
|
|
13309
16387
|
}
|
|
13310
16388
|
/** Build a lineage graph from the project's blocks and semantic layer. */
|
|
13311
16389
|
// Simple lineage graph cache: rebuilds at most every 5 seconds
|
|
@@ -13401,15 +16479,23 @@ function resolveDbtManifestPath(projectRoot, projectConfig = {}) {
|
|
|
13401
16479
|
candidates.push(join(projectRoot, 'target', 'manifest.json'), join(resolve(projectRoot, '..'), 'target', 'manifest.json'), join(resolve(projectRoot, '../dbt'), 'target', 'manifest.json'), join(resolve(projectRoot, '../../dbt'), 'target', 'manifest.json'));
|
|
13402
16480
|
return candidates.find((candidate, index, list) => list.indexOf(candidate) === index && existsSync(candidate));
|
|
13403
16481
|
}
|
|
13404
|
-
export function discoverDbtProfileConnections(projectRoot, projectConfig) {
|
|
16482
|
+
export function discoverDbtProfileConnections(projectRoot, projectConfig, explicitPath) {
|
|
13405
16483
|
const dbtProjectPath = findDbtProjectPath(projectRoot, projectConfig);
|
|
13406
|
-
const projectProfileName = readDbtProjectProfileName(dbtProjectPath);
|
|
13407
|
-
const
|
|
16484
|
+
const projectProfileName = explicitPath ? null : readDbtProjectProfileName(dbtProjectPath);
|
|
16485
|
+
const configuredProfilesDir = projectConfig.dbt?.profilesDir
|
|
16486
|
+
? resolve(projectRoot, projectConfig.dbt.profilesDir)
|
|
16487
|
+
: undefined;
|
|
16488
|
+
const profilePaths = explicitPath
|
|
16489
|
+
? findExplicitDbtProfilePaths(projectRoot, explicitPath)
|
|
16490
|
+
: findDbtProfilePaths(projectRoot, dbtProjectPath, configuredProfilesDir);
|
|
13408
16491
|
const candidates = [];
|
|
13409
16492
|
for (const profilePath of profilePaths) {
|
|
13410
16493
|
const profiles = readYamlFile(profilePath);
|
|
13411
16494
|
if (!profiles)
|
|
13412
16495
|
continue;
|
|
16496
|
+
const profileRelativeBase = explicitPath && existsSync(join(dirname(profilePath), 'dbt_project.yml'))
|
|
16497
|
+
? dirname(profilePath)
|
|
16498
|
+
: dbtProjectPath;
|
|
13413
16499
|
for (const [profileName, rawProfile] of Object.entries(profiles)) {
|
|
13414
16500
|
if (!rawProfile || typeof rawProfile !== 'object')
|
|
13415
16501
|
continue;
|
|
@@ -13433,7 +16519,7 @@ export function discoverDbtProfileConnections(projectRoot, projectConfig) {
|
|
|
13433
16519
|
mapped.connection.filepath &&
|
|
13434
16520
|
mapped.connection.filepath !== ':memory:') {
|
|
13435
16521
|
if (!isAbsoluteLikePath(mapped.connection.filepath)) {
|
|
13436
|
-
mapped.connection.filepath = resolve(
|
|
16522
|
+
mapped.connection.filepath = resolve(profileRelativeBase, mapped.connection.filepath);
|
|
13437
16523
|
}
|
|
13438
16524
|
if (!existsSync(mapped.connection.filepath)) {
|
|
13439
16525
|
mapped.warnings.push(`DuckDB file not found at ${mapped.connection.filepath} — run \`dbt build\` (or \`dbt seed\`) to create it before querying.`);
|
|
@@ -13458,6 +16544,21 @@ export function discoverDbtProfileConnections(projectRoot, projectConfig) {
|
|
|
13458
16544
|
}
|
|
13459
16545
|
return candidates.slice(0, 20);
|
|
13460
16546
|
}
|
|
16547
|
+
export function buildDbtParseArgs(dbtProjectDir, profilesDir) {
|
|
16548
|
+
return [
|
|
16549
|
+
'parse',
|
|
16550
|
+
'--project-dir', dbtProjectDir,
|
|
16551
|
+
...(profilesDir ? ['--profiles-dir', profilesDir] : []),
|
|
16552
|
+
];
|
|
16553
|
+
}
|
|
16554
|
+
/** CFG-003: use a complete default dbt target when no saved DQL connection exists. */
|
|
16555
|
+
export function resolveDbtProfileRuntimeConnection(projectRoot, projectConfig) {
|
|
16556
|
+
const candidates = discoverDbtProfileConnections(projectRoot, projectConfig);
|
|
16557
|
+
const complete = candidates.filter((candidate) => candidate.missingFields.length === 0
|
|
16558
|
+
&& !candidate.warnings.some((warning) => warning.startsWith('DuckDB file not found')));
|
|
16559
|
+
const defaultTarget = complete.find((candidate) => !candidate.warnings.some((warning) => warning.startsWith('Not the default dbt target')));
|
|
16560
|
+
return defaultTarget?.connection ?? (complete.length === 1 ? complete[0].connection : null);
|
|
16561
|
+
}
|
|
13461
16562
|
function findDbtProjectPath(projectRoot, projectConfig) {
|
|
13462
16563
|
const configuredDbtDir = projectConfig.dbt?.projectDir
|
|
13463
16564
|
? resolve(projectRoot, projectConfig.dbt.projectDir)
|
|
@@ -13478,8 +16579,18 @@ function findDbtProjectPath(projectRoot, projectConfig) {
|
|
|
13478
16579
|
?? semanticDbtDir
|
|
13479
16580
|
?? projectRoot;
|
|
13480
16581
|
}
|
|
13481
|
-
function
|
|
16582
|
+
function portableProjectPath(projectRoot, absolutePath) {
|
|
16583
|
+
const normalizedAbsolute = resolve(absolutePath).replace(/\\/g, '/');
|
|
16584
|
+
const relativePath = relative(projectRoot, absolutePath).replace(/\\/g, '/');
|
|
16585
|
+
return !relativePath
|
|
16586
|
+
? '.'
|
|
16587
|
+
: relativePath === '..' || relativePath.startsWith('../')
|
|
16588
|
+
? normalizedAbsolute
|
|
16589
|
+
: relativePath;
|
|
16590
|
+
}
|
|
16591
|
+
function findDbtProfilePaths(projectRoot, dbtProjectPath, configuredProfilesDir) {
|
|
13482
16592
|
const dirs = [
|
|
16593
|
+
configuredProfilesDir,
|
|
13483
16594
|
process.env.DBT_PROFILES_DIR,
|
|
13484
16595
|
dbtProjectPath,
|
|
13485
16596
|
projectRoot,
|
|
@@ -13487,7 +16598,7 @@ function findDbtProfilePaths(projectRoot, dbtProjectPath) {
|
|
|
13487
16598
|
].filter((value) => Boolean(value));
|
|
13488
16599
|
const paths = [];
|
|
13489
16600
|
for (const dir of dirs) {
|
|
13490
|
-
for (const filename of ['profiles.yml', 'profiles.yaml']) {
|
|
16601
|
+
for (const filename of ['profiles.yml', 'profiles.yaml', 'profile.yml', 'profile.yaml']) {
|
|
13491
16602
|
const profilePath = resolve(dir, filename);
|
|
13492
16603
|
if (existsSync(profilePath) && !paths.includes(profilePath)) {
|
|
13493
16604
|
paths.push(profilePath);
|
|
@@ -13496,6 +16607,24 @@ function findDbtProfilePaths(projectRoot, dbtProjectPath) {
|
|
|
13496
16607
|
}
|
|
13497
16608
|
return paths;
|
|
13498
16609
|
}
|
|
16610
|
+
function findExplicitDbtProfilePaths(projectRoot, input) {
|
|
16611
|
+
const trimmed = input.trim();
|
|
16612
|
+
if (!trimmed)
|
|
16613
|
+
return [];
|
|
16614
|
+
const expanded = trimmed === '~'
|
|
16615
|
+
? homedir()
|
|
16616
|
+
: trimmed.startsWith('~/')
|
|
16617
|
+
? join(homedir(), trimmed.slice(2))
|
|
16618
|
+
: trimmed;
|
|
16619
|
+
const absolute = resolve(projectRoot, expanded);
|
|
16620
|
+
if (!existsSync(absolute))
|
|
16621
|
+
return [];
|
|
16622
|
+
if (!statSync(absolute).isDirectory())
|
|
16623
|
+
return [absolute];
|
|
16624
|
+
return ['profiles.yml', 'profiles.yaml', 'profile.yml', 'profile.yaml']
|
|
16625
|
+
.map((filename) => join(absolute, filename))
|
|
16626
|
+
.filter((profilePath) => existsSync(profilePath));
|
|
16627
|
+
}
|
|
13499
16628
|
function readDbtProjectProfileName(dbtProjectPath) {
|
|
13500
16629
|
const projectFile = join(dbtProjectPath, 'dbt_project.yml');
|
|
13501
16630
|
const projectYaml = readYamlFile(projectFile);
|
|
@@ -14185,6 +17314,33 @@ function ensureGitignoreEntry(projectRoot, pattern) {
|
|
|
14185
17314
|
// Best-effort; failure to write .gitignore shouldn't fail the snapshot.
|
|
14186
17315
|
}
|
|
14187
17316
|
}
|
|
17317
|
+
/**
|
|
17318
|
+
* UI-001, SEC-001, E2E-001: local runtime, credential, and generated artifacts must never be offered as
|
|
17319
|
+
* shareable source-control changes. Project skills are intentionally absent:
|
|
17320
|
+
* `.dql/skills` remains readable during migration and can be governed in Git.
|
|
17321
|
+
*/
|
|
17322
|
+
const LOCAL_RUNTIME_GITIGNORE_RULES = [
|
|
17323
|
+
'dql-manifest.json',
|
|
17324
|
+
'*.duckdb',
|
|
17325
|
+
'*.duckdb.wal',
|
|
17326
|
+
'*.run.json',
|
|
17327
|
+
'**/.dql/runs/',
|
|
17328
|
+
'**/.dql/cache/',
|
|
17329
|
+
'**/.dql/imports/',
|
|
17330
|
+
'**/.dql/local/',
|
|
17331
|
+
'**/.dql/connectors/',
|
|
17332
|
+
'**/.dql/memory/',
|
|
17333
|
+
'**/.dql/migration-staging/',
|
|
17334
|
+
'**/.dql/docker-starter/',
|
|
17335
|
+
'**/.dql/oauth-credentials.json',
|
|
17336
|
+
'**/.dql/provider-settings.json',
|
|
17337
|
+
'**/.dql/mcp-servers.json',
|
|
17338
|
+
'**/.dql-user-prefs.json',
|
|
17339
|
+
];
|
|
17340
|
+
export function ensureLocalRuntimeGitignore(projectRoot) {
|
|
17341
|
+
for (const rule of LOCAL_RUNTIME_GITIGNORE_RULES)
|
|
17342
|
+
ensureGitignoreEntry(projectRoot, rule);
|
|
17343
|
+
}
|
|
14188
17344
|
async function readGitDiff(cwd, filePath, staged = false) {
|
|
14189
17345
|
const gitRoot = await resolveGitRoot(cwd);
|
|
14190
17346
|
if (!gitRoot) {
|
|
@@ -14469,6 +17625,39 @@ async function gitCreateReview(cwd, input) {
|
|
|
14469
17625
|
warning: `Branch pushed. Open the compare page to create the PR: ${gitErrorOutput(pr) || 'GitHub CLI is not available or authenticated.'}`,
|
|
14470
17626
|
};
|
|
14471
17627
|
}
|
|
17628
|
+
/** UI-001, SEC-001, E2E-001: open a PR for a branch already committed and pushed by the guided share flow. */
|
|
17629
|
+
async function gitOpenReview(cwd, input) {
|
|
17630
|
+
const gitRoot = await resolveGitRoot(cwd);
|
|
17631
|
+
if (!gitRoot)
|
|
17632
|
+
return { ok: false, error: 'Not a git repository' };
|
|
17633
|
+
const title = input.title.trim();
|
|
17634
|
+
if (!title)
|
|
17635
|
+
return { ok: false, error: 'Review title required' };
|
|
17636
|
+
const base = input.base.trim() || 'main';
|
|
17637
|
+
const status = await readGitStatus(gitRoot);
|
|
17638
|
+
const branch = status.branch;
|
|
17639
|
+
if (!branch || branch === 'HEAD' || branch === base || branch === 'master') {
|
|
17640
|
+
return { ok: false, error: 'Share the changes to a review branch before opening a review request.' };
|
|
17641
|
+
}
|
|
17642
|
+
const remote = await readGitRemote(gitRoot);
|
|
17643
|
+
if (!remote.url)
|
|
17644
|
+
return { ok: false, error: 'Add a Git remote before requesting review.' };
|
|
17645
|
+
const compareUrl = gitHubCompareUrl(remote.url, branch, base);
|
|
17646
|
+
const pr = await execGh(gitRoot, [
|
|
17647
|
+
'pr', 'create', '--base', base, '--head', branch, '--title', title,
|
|
17648
|
+
'--body', input.body.trim() || `Review governed analytics changes: ${title}`,
|
|
17649
|
+
]);
|
|
17650
|
+
if (pr.code === 0) {
|
|
17651
|
+
const prUrl = pr.stdout.match(/https:\/\/github\.com\/[^\s]+\/pull\/\d+/)?.[0] ?? compareUrl;
|
|
17652
|
+
return { ok: true, branch, prUrl };
|
|
17653
|
+
}
|
|
17654
|
+
return {
|
|
17655
|
+
ok: true,
|
|
17656
|
+
branch,
|
|
17657
|
+
prUrl: compareUrl,
|
|
17658
|
+
warning: `Branch is ready. Open the compare page to create the PR: ${gitErrorOutput(pr) || 'GitHub CLI is not available or authenticated.'}`,
|
|
17659
|
+
};
|
|
17660
|
+
}
|
|
14472
17661
|
async function gitPull(cwd) {
|
|
14473
17662
|
const gitRoot = await resolveGitRoot(cwd);
|
|
14474
17663
|
if (!gitRoot)
|
|
@@ -14565,8 +17754,7 @@ async function enableGitGovernedContextTracking(cwd) {
|
|
|
14565
17754
|
const existing = existsSync(ignorePath) ? readFileSync(ignorePath, 'utf-8') : '';
|
|
14566
17755
|
const lines = existing.split(/\r?\n/);
|
|
14567
17756
|
const next = lines.filter((line) => !/^\s*\/?\.dql\/\s*$/.test(line));
|
|
14568
|
-
const
|
|
14569
|
-
for (const rule of keepLocal) {
|
|
17757
|
+
for (const rule of LOCAL_RUNTIME_GITIGNORE_RULES) {
|
|
14570
17758
|
if (!next.some((line) => line.trim() === rule))
|
|
14571
17759
|
next.push(rule);
|
|
14572
17760
|
}
|
|
@@ -15813,7 +19001,7 @@ const NON_ENTITY_PLURALS = new Set([
|
|
|
15813
19001
|
* project vocabulary — so it fires for ANY repo's join-shaped questions, not just
|
|
15814
19002
|
* jaffle. Over-triggering only costs a few extra bounded probes (all in try/catch).
|
|
15815
19003
|
*/
|
|
15816
|
-
function shouldAugmentAgentRuntimeSchema(question) {
|
|
19004
|
+
export function shouldAugmentAgentRuntimeSchema(question, questionPlan) {
|
|
15817
19005
|
const lower = question.toLowerCase();
|
|
15818
19006
|
const wantsMetric = /\brevenu|\bsales\b|\bamount\b|\bspend\b|\bcost\b|\bcount\b|\btotal\b|\bsum\b|\bavg\b|\baverage\b|\bvalue\b|\bprice\b/.test(lower);
|
|
15819
19007
|
const wantsDetail = /\bdetails?\b|\bcomplete\b|\bbreakdown\b|\ball\s+values?\b|\beach\b|\bevery\b/.test(lower);
|
|
@@ -15822,7 +19010,12 @@ function shouldAugmentAgentRuntimeSchema(question) {
|
|
|
15822
19010
|
// Multi-entity intent: a linking word + >= 2 distinct plural content nouns.
|
|
15823
19011
|
const nouns = new Set((lower.match(/\b[a-z]{4,}s\b/g) ?? []).filter((word) => !NON_ENTITY_PLURALS.has(word)));
|
|
15824
19012
|
const multiEntity = /\b(?:and|with|plus|per|for\s+each|by)\b/.test(lower) && nouns.size >= 2;
|
|
15825
|
-
|
|
19013
|
+
const plannedConcepts = new Set([
|
|
19014
|
+
...(questionPlan?.entities ?? []).map((entity) => typeof entity === 'string' ? entity : entity.typeHint ?? entity.text ?? ''),
|
|
19015
|
+
...(questionPlan?.dimensionTerms ?? []),
|
|
19016
|
+
].map((value) => value.trim().toLowerCase()).filter(Boolean));
|
|
19017
|
+
const plannedCompositeMetric = (questionPlan?.metricTerms?.length ?? 0) > 0 && plannedConcepts.size >= 2;
|
|
19018
|
+
return explicitJoin || referencesPriorRows || plannedCompositeMetric || (multiEntity && (wantsMetric || wantsDetail));
|
|
15826
19019
|
}
|
|
15827
19020
|
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
|
|
15828
19021
|
const searchTerms = extractAgentValueSearchTerms(question);
|