@duckcodeailabs/dql-cli 1.6.34 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts +9 -0
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +65 -10
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +4 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +7 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-2Tszpipm.js +5914 -0
- package/dist/assets/dql-notebook/assets/index-dSyhu_eU.css +1 -0
- package/dist/assets/dql-notebook/index.html +2 -2
- package/dist/block-invocation.d.ts +31 -0
- package/dist/block-invocation.d.ts.map +1 -0
- package/dist/block-invocation.js +82 -0
- package/dist/block-invocation.js.map +1 -0
- package/dist/commands/agent.d.ts +1 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +12 -3
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts +1 -2
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +6 -9
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/compile.js +1 -1
- package/dist/commands/compile.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +44 -86
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/migrate.d.ts +87 -0
- package/dist/commands/migrate.d.ts.map +1 -1
- package/dist/commands/migrate.js +812 -3
- package/dist/commands/migrate.js.map +1 -1
- package/dist/commands/model.d.ts +4 -0
- package/dist/commands/model.d.ts.map +1 -0
- package/dist/commands/model.js +219 -0
- package/dist/commands/model.js.map +1 -0
- package/dist/commands/new.d.ts.map +1 -1
- package/dist/commands/new.js +28 -4
- package/dist/commands/new.js.map +1 -1
- package/dist/commands/parse.d.ts.map +1 -1
- package/dist/commands/parse.js +32 -0
- package/dist/commands/parse.js.map +1 -1
- package/dist/commands/sync.d.ts.map +1 -1
- package/dist/commands/sync.js +23 -7
- package/dist/commands/sync.js.map +1 -1
- package/dist/index.js +19 -1
- package/dist/index.js.map +1 -1
- package/dist/llm/answer-loop-tools.d.ts +1 -1
- package/dist/llm/answer-loop-tools.d.ts.map +1 -1
- package/dist/llm/answer-loop-tools.js +210 -10
- package/dist/llm/answer-loop-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +30 -18
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/tools.d.ts.map +1 -1
- package/dist/llm/tools.js +3 -1
- package/dist/llm/tools.js.map +1 -1
- package/dist/llm/types.d.ts +12 -2
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +78 -6
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +3028 -163
- package/dist/local-runtime.js.map +1 -1
- package/dist/metricflow.d.ts +2 -0
- package/dist/metricflow.d.ts.map +1 -1
- package/dist/metricflow.js +10 -0
- package/dist/metricflow.js.map +1 -1
- package/dist/mixed-source-sql.d.ts +53 -0
- package/dist/mixed-source-sql.d.ts.map +1 -0
- package/dist/mixed-source-sql.js +311 -0
- package/dist/mixed-source-sql.js.map +1 -0
- package/dist/notebook-datasets.d.ts +120 -0
- package/dist/notebook-datasets.d.ts.map +1 -0
- package/dist/notebook-datasets.js +484 -0
- package/dist/notebook-datasets.js.map +1 -0
- package/dist/package.json +10 -10
- package/package.json +10 -10
- package/dist/assets/dql-notebook/assets/index-CYQrnwxa.css +0 -1
- package/dist/assets/dql-notebook/assets/index-GofggEGB.js +0 -5777
package/dist/local-runtime.js
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
import { execFileSync, execSync } from
|
|
2
|
-
import { createHash } from
|
|
3
|
-
import { createServer } from
|
|
4
|
-
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from
|
|
5
|
-
import { createRequire } from
|
|
6
|
-
import { homedir } from
|
|
7
|
-
import { dirname, extname, join, normalize, relative, resolve } from
|
|
8
|
-
import
|
|
9
|
-
import
|
|
10
|
-
import
|
|
11
|
-
import
|
|
1
|
+
import { execFileSync, execSync } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, watch, writeFileSync, } from "node:fs";
|
|
5
|
+
import { createRequire } from "node:module";
|
|
6
|
+
import { homedir } from "node:os";
|
|
7
|
+
import { dirname, extname, join, normalize, relative, resolve, } from "node:path";
|
|
8
|
+
import { Readable } from "node:stream";
|
|
9
|
+
import { buildMixedSourceWarehouseFallbackSql, findMentionedNotebookDataset, planMixedSourceNotebookSql, planMixedSourceSql, } from "./mixed-source-sql.js";
|
|
10
|
+
import Anthropic from "@anthropic-ai/sdk";
|
|
11
|
+
import OpenAI from "openai";
|
|
12
|
+
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, hasStandaloneSemanticRef, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
13
|
+
import { loadSemanticLayerFromDir, normalizeDqlArtifactReference, serializeMetricDefinitionToYaml, resolveSemanticLayerAsync, getDialect, Parser, NodeKind, blockParameterDefinitions, buildLineageGraph, buildManifest, collectInputFiles, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryBusiness360, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, domainFolderSlug, previewModelingChange, applyModelingChange, previewDbtSourcePatch, applyDbtSourcePatch, loadDomainPackageRegistry, loadDbtNodeAuthoringDetail, relationshipValidationProofFingerprint, discoverDbtDomains, renderDomainDeclaration, ProjectSnapshotService, analyzeSqlReferences, } from '@duckcodeailabs/dql-core';
|
|
12
14
|
import { load as loadYaml } from 'js-yaml';
|
|
13
15
|
import { listBlockTemplates } from './block-templates.js';
|
|
14
16
|
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
15
17
|
import { createDqlAgentProviderRunner } from './llm/providers/dql-agent-provider.js';
|
|
16
18
|
import { listRemoteMcpSettings, saveRemoteMcpSettings } from './llm/mcp-config.js';
|
|
17
|
-
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, defaultKgPath, planApp, planResearch, loadSemanticMetrics, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, probeLocalOllamaEmbeddings, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
19
|
+
import { ClaudeProvider, ConversationStore, advanceThreadState, buildConversationSnapshot, recallRelevantTurns, GeminiProvider, MemoryStore, OllamaProvider, OpenAIProvider, buildBlockBusinessFingerprint, buildBlockSqlFingerprints, buildLocalContextPack, defaultConversationPath, defaultMemoryPath, ensureDefaultMemoryFiles, ensureMetadataCatalogFresh, propose, proposePlan, recordCorrectionTrace, emitCorrectionEvalCase, mineJoinPatterns, reviewHint, AgentRunEngine, FileAgentRunStore, defaultAgentRunGates, createLlmAgentRunPlanner, createHybridRouter, computeResultStats, synthesizeAnswer, streamOrGenerate, narrateResult, normalizeAnthropicBaseUrl, buildProposePreview, buildFromPrompt, defaultAgentRunStorePath, resolveLocalOwner, resolveProposeConfig, recordQueryRun, recordRuntimeSchemaSnapshot, latestRuntimeSchemaSnapshotForProject, loadSkills, migrateLegacySkills, draftDomainSkillBootstrap, buildDomainSkillBootstrapPrompt, mergeDomainSkillBootstrapEnrichment, writeSkill, deleteSkill, deriveGeneratedDraftSlug, reindexProject, invalidateAgentProjectState, resolveDomainContextEnvelope, defaultKgPath, planApp, planResearch, loadSemanticMetrics, cascadeTraceToEvidenceRouteSteps, createCascadeAnswerResult, createCascadeTrace, routeReasoningEffort, routeForCascadeAnswerTier, clampReasoningEffort, bumpReasoningEffort, resolveThinkingMode, coerceThinkingMode, probeLocalOllamaEmbeddings, upsertGeneratedDqlArtifactDraft, } from '@duckcodeailabs/dql-agent';
|
|
18
20
|
import { gatherProposeEnrichment } from './propose-enrich.js';
|
|
19
21
|
import { handleAppsApi, proposeAppAiBuild, recommendVisualization } from './apps-api.js';
|
|
20
22
|
import { getActiveProvider, getEffectiveProviderConfig, isProviderSettingsId, listProviderSettings, saveProviderSettings, } from './settings/provider-settings.js';
|
|
@@ -25,8 +27,10 @@ import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeA
|
|
|
25
27
|
import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath, defaultNotebookResearchDbPath } from '@duckcodeailabs/dql-project';
|
|
26
28
|
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
27
29
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
28
|
-
import { clearBlockStudioImportSessions, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
29
|
-
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from
|
|
30
|
+
import { clearBlockStudioImportSessions, candidateToDqlSource, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
31
|
+
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, hasMetricFlowCli, } from "./metricflow.js";
|
|
32
|
+
import { NotebookDatasetWorkspace, } from "./notebook-datasets.js";
|
|
33
|
+
import { prepareBlockInvocation } from './block-invocation.js';
|
|
30
34
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
31
35
|
const NOTEBOOK_FAVICON_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32"><rect width="32" height="32" rx="7" fill="#6d5dfc"/><path d="M9 9h14v14H9z" fill="none" stroke="#fff" stroke-width="2"/><path d="M13 13h6M13 17h6M13 21h4" stroke="#fff" stroke-width="2" stroke-linecap="round"/></svg>';
|
|
32
36
|
export function resolveProjectSemanticConfig(projectConfig, projectRoot) {
|
|
@@ -152,13 +156,33 @@ export function parseAgentRunRequestBody(body) {
|
|
|
152
156
|
export function shouldSynthesizeAgentRunAnswer(governedAnswer) {
|
|
153
157
|
if (governedAnswer.kind === 'no_answer')
|
|
154
158
|
return false;
|
|
159
|
+
// The executed rows are the presentation boundary: certified, semantic,
|
|
160
|
+
// DQL-first, and exploratory routes all deserve a separate stakeholder-facing
|
|
161
|
+
// explanation grounded in those values. Trust and query mechanics stay in the
|
|
162
|
+
// artifact/inspector instead of leaking into the primary answer.
|
|
163
|
+
if (governedAnswer.result)
|
|
164
|
+
return true;
|
|
155
165
|
if (governedAnswer.kind === 'certified' || governedAnswer.certification === 'certified')
|
|
156
166
|
return false;
|
|
167
|
+
// EXP-001: exploratory prose is part of the trust boundary. A free-form
|
|
168
|
+
// synthesis pass can accidentally relabel an entity-level lifetime measure
|
|
169
|
+
// as a child-level allocation or total repeated values. Keep the host's
|
|
170
|
+
// deterministic grain statement instead.
|
|
171
|
+
if (governedAnswer.exploratoryCandidate)
|
|
172
|
+
return false;
|
|
157
173
|
const finalText = (governedAnswer.answer ?? governedAnswer.text ?? '').trim();
|
|
158
174
|
if (governedAnswer.dqlArtifact && finalText)
|
|
159
175
|
return false;
|
|
160
176
|
return true;
|
|
161
177
|
}
|
|
178
|
+
function businessNarrativeGaps(warnings) {
|
|
179
|
+
const businessRelevant = (warnings ?? []).filter((warning) => !(/missing requested output column/i.test(warning)
|
|
180
|
+
|| /\bquery plan\b/i.test(warning)
|
|
181
|
+
|| /\bgrain\b/i.test(warning)
|
|
182
|
+
|| /\bcolumn(?:s)?\b.*\bmissing\b/i.test(warning)
|
|
183
|
+
|| /\bSQL\b|\bDBT\b|\bjoin probe\b/i.test(warning)));
|
|
184
|
+
return businessRelevant.length > 0 ? businessRelevant : undefined;
|
|
185
|
+
}
|
|
162
186
|
// ── Server-side conversation threads ────────────────────────────────────────
|
|
163
187
|
// When a run carries a threadId, the persisted thread is the authoritative
|
|
164
188
|
// source of prior turns (survives refresh); the client-built context remains
|
|
@@ -258,7 +282,7 @@ function conversationTurnInputFromRun(run) {
|
|
|
258
282
|
certification: agentRunString(payload?.certification),
|
|
259
283
|
sourceCertifiedBlock: agentRunString(payload?.sourceCertifiedBlock)
|
|
260
284
|
?? (artifact?.kind === 'answer' ? agentRunString(artifact.ref) : undefined),
|
|
261
|
-
contextPackId: agentRunString(payload?.contextPackId),
|
|
285
|
+
contextPackId: agentRunString(payload?.contextPackId) ?? agentRunString(contextPack?.id),
|
|
262
286
|
sql: agentRunString(payload?.proposedSql) ?? agentRunString(payload?.sql),
|
|
263
287
|
dqlArtifact: agentRunRecord(payload?.dqlArtifact),
|
|
264
288
|
cascade: agentRunRecord(payload?.cascade),
|
|
@@ -311,15 +335,166 @@ function conversationStringArray(value) {
|
|
|
311
335
|
const values = value.filter((item) => typeof item === 'string' && item.trim().length > 0);
|
|
312
336
|
return values.length > 0 ? values : undefined;
|
|
313
337
|
}
|
|
338
|
+
function isLoopbackOrigin(value) {
|
|
339
|
+
try {
|
|
340
|
+
const hostname = new URL(value).hostname;
|
|
341
|
+
return hostname === '127.0.0.1' || hostname === 'localhost' || hostname === '::1' || hostname === '[::1]';
|
|
342
|
+
}
|
|
343
|
+
catch {
|
|
344
|
+
return false;
|
|
345
|
+
}
|
|
346
|
+
}
|
|
314
347
|
function compactConversationRecord(record) {
|
|
315
348
|
const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
|
|
316
349
|
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
317
350
|
}
|
|
351
|
+
function apiRequestId(scope) {
|
|
352
|
+
return `${scope}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
|
353
|
+
}
|
|
354
|
+
function apiErrorEnvelope(input) {
|
|
355
|
+
return {
|
|
356
|
+
requestId: input.requestId,
|
|
357
|
+
...(input.snapshotId ? { snapshotId: input.snapshotId } : {}),
|
|
358
|
+
code: input.code,
|
|
359
|
+
message: input.message,
|
|
360
|
+
// Retain the legacy field during the compatibility window.
|
|
361
|
+
error: input.message,
|
|
362
|
+
recoverable: input.recoverable ?? input.code !== 'DOMAIN_COLLISION',
|
|
363
|
+
...(input.details !== undefined ? { details: input.details } : {}),
|
|
364
|
+
...(input.nextActions?.length ? { nextActions: input.nextActions } : {}),
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
function apiErrorCode(error, fallback) {
|
|
368
|
+
return typeof error === 'object' && error && 'code' in error
|
|
369
|
+
? String(error.code)
|
|
370
|
+
: fallback;
|
|
371
|
+
}
|
|
372
|
+
function apiErrorMessage(error) {
|
|
373
|
+
return error instanceof Error ? error.message : String(error);
|
|
374
|
+
}
|
|
318
375
|
export async function startLocalServer(opts) {
|
|
319
376
|
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
320
377
|
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
378
|
+
const loopback = bindHost === '127.0.0.1' || bindHost === 'localhost' || bindHost === '::1';
|
|
379
|
+
const authToken = opts.authToken ?? process.env.DQL_SERVER_TOKEN;
|
|
380
|
+
const allowedOrigins = new Set((opts.allowedOrigins ?? (process.env.DQL_ALLOWED_ORIGINS ?? '').split(','))
|
|
381
|
+
.map((value) => value.trim().replace(/\/$/, ''))
|
|
382
|
+
.filter(Boolean));
|
|
383
|
+
if (!loopback && !authToken) {
|
|
384
|
+
throw new Error('Non-loopback DQL server binding requires DQL_SERVER_TOKEN (or LocalServerOptions.authToken).');
|
|
385
|
+
}
|
|
386
|
+
if (!loopback && allowedOrigins.size === 0) {
|
|
387
|
+
throw new Error('Non-loopback DQL server binding requires DQL_ALLOWED_ORIGINS (or LocalServerOptions.allowedOrigins).');
|
|
388
|
+
}
|
|
389
|
+
const gitRoot = await resolveGitRoot(projectRoot);
|
|
390
|
+
if (gitRoot)
|
|
391
|
+
ensureLocalRuntimeGitignore(projectRoot);
|
|
321
392
|
let connection = rawConnection ? normalizeProjectConnection(rawConnection, projectRoot) : null;
|
|
322
393
|
let projectConfig = loadProjectConfig(projectRoot);
|
|
394
|
+
const projectSnapshots = new ProjectSnapshotService();
|
|
395
|
+
const dbtNodeDetailCache = new Map();
|
|
396
|
+
const onboardingJobs = new Map();
|
|
397
|
+
const projectSnapshot = () => {
|
|
398
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot) ?? undefined;
|
|
399
|
+
const inputs = collectInputFiles({ projectRoot, dbtManifestPath });
|
|
400
|
+
const signatureHash = createHash('sha256');
|
|
401
|
+
for (const input of inputs.sort()) {
|
|
402
|
+
try {
|
|
403
|
+
const stats = statSync(input);
|
|
404
|
+
signatureHash.update(`${input}\0${stats.size}\0${stats.mtimeMs}\n`);
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
signatureHash.update(`${input}\0missing\n`);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
const signature = signatureHash.digest('hex');
|
|
411
|
+
const previousId = projectSnapshots.current()?.snapshotId;
|
|
412
|
+
const snapshot = projectSnapshots.refresh(signature, () => buildManifest({ projectRoot, dbtManifestPath }));
|
|
413
|
+
if (snapshot.snapshotId !== previousId)
|
|
414
|
+
dbtNodeDetailCache.clear();
|
|
415
|
+
return {
|
|
416
|
+
signature: snapshot.sourceVersion,
|
|
417
|
+
snapshotId: snapshot.snapshotId,
|
|
418
|
+
manifest: snapshot.value,
|
|
419
|
+
stale: snapshot.stale,
|
|
420
|
+
error: snapshot.error,
|
|
421
|
+
};
|
|
422
|
+
};
|
|
423
|
+
const onboardingDbtPaths = (body = {}) => {
|
|
424
|
+
const projectDirInput = typeof body.projectDir === 'string' ? body.projectDir : projectConfig.dbt?.projectDir ?? '.';
|
|
425
|
+
const dbtProjectDir = resolve(projectRoot, projectDirInput);
|
|
426
|
+
const manifestInput = typeof body.manifestPath === 'string' ? body.manifestPath : projectConfig.dbt?.manifestPath ?? 'target/manifest.json';
|
|
427
|
+
const manifestPath = resolve(dbtProjectDir, manifestInput);
|
|
428
|
+
if (!(manifestPath === dbtProjectDir || manifestPath.startsWith(`${dbtProjectDir}/`))) {
|
|
429
|
+
throw Object.assign(new Error('dbt manifestPath must stay inside dbt projectDir.'), { code: 'DBT_ARTIFACT_INVALID' });
|
|
430
|
+
}
|
|
431
|
+
return { dbtProjectDir, manifestPath, projectDirInput, manifestInput };
|
|
432
|
+
};
|
|
433
|
+
const previewDbtOnboarding = (body = {}) => {
|
|
434
|
+
const paths = onboardingDbtPaths(body);
|
|
435
|
+
if (!existsSync(join(paths.dbtProjectDir, 'dbt_project.yml')))
|
|
436
|
+
throw Object.assign(new Error('dbt_project.yml was not found.'), { code: 'DBT_PROJECT_NOT_FOUND' });
|
|
437
|
+
if (!existsSync(paths.manifestPath))
|
|
438
|
+
throw Object.assign(new Error(`dbt manifest is missing at ${paths.manifestPath}. Run dbt parse, then retry.`), { code: 'DBT_MANIFEST_MISSING' });
|
|
439
|
+
let raw;
|
|
440
|
+
try {
|
|
441
|
+
raw = JSON.parse(readFileSync(paths.manifestPath, 'utf8'));
|
|
442
|
+
}
|
|
443
|
+
catch (error) {
|
|
444
|
+
throw Object.assign(new Error(`dbt manifest is invalid: ${error instanceof Error ? error.message : String(error)}`), { code: 'DBT_ARTIFACT_INVALID' });
|
|
445
|
+
}
|
|
446
|
+
const fingerprint = createHash('sha256').update(readFileSync(paths.manifestPath)).digest('hex');
|
|
447
|
+
return {
|
|
448
|
+
projectDir: relative(projectRoot, paths.dbtProjectDir).replace(/\\/g, '/') || '.',
|
|
449
|
+
manifestPath: relative(paths.dbtProjectDir, paths.manifestPath).replace(/\\/g, '/'),
|
|
450
|
+
fingerprint,
|
|
451
|
+
projectName: typeof raw.metadata?.project_name === 'string' ? raw.metadata.project_name : undefined,
|
|
452
|
+
counts: {
|
|
453
|
+
models: Object.keys(raw.nodes ?? {}).filter((id) => id.startsWith('model.')).length,
|
|
454
|
+
sources: Object.keys(raw.sources ?? {}).length,
|
|
455
|
+
metrics: Object.keys(raw.metrics ?? {}).length,
|
|
456
|
+
},
|
|
457
|
+
artifacts: {
|
|
458
|
+
manifest: paths.manifestPath,
|
|
459
|
+
catalog: existsSync(join(dirname(paths.manifestPath), 'catalog.json')) ? join(dirname(paths.manifestPath), 'catalog.json') : null,
|
|
460
|
+
semanticManifest: existsSync(join(dirname(paths.manifestPath), 'semantic_manifest.json')) ? join(dirname(paths.manifestPath), 'semantic_manifest.json') : null,
|
|
461
|
+
},
|
|
462
|
+
};
|
|
463
|
+
};
|
|
464
|
+
const datasetWorkspace = new NotebookDatasetWorkspace(projectRoot, executor, connectorModuleSearchPaths(projectRoot));
|
|
465
|
+
let localWorkspaceReady = false;
|
|
466
|
+
const ensureLocalWorkspaceReady = async () => {
|
|
467
|
+
ensureConnectorInstalledForStartup(projectRoot, "duckdb");
|
|
468
|
+
if (!localWorkspaceReady) {
|
|
469
|
+
await datasetWorkspace.initialize();
|
|
470
|
+
localWorkspaceReady = true;
|
|
471
|
+
}
|
|
472
|
+
return datasetWorkspace.localConnection;
|
|
473
|
+
};
|
|
474
|
+
const resolveNamedConnection = (name) => {
|
|
475
|
+
if (!name)
|
|
476
|
+
return null;
|
|
477
|
+
const stored = getStoredConnections(projectConfig);
|
|
478
|
+
const normalized = normalizeStoredConnection(stored[name]);
|
|
479
|
+
return normalized
|
|
480
|
+
? normalizeProjectConnection(normalized, projectRoot)
|
|
481
|
+
: null;
|
|
482
|
+
};
|
|
483
|
+
const resolveExecutionConnection = async (body) => {
|
|
484
|
+
const target = body.executionTarget && typeof body.executionTarget === "object"
|
|
485
|
+
? body.executionTarget
|
|
486
|
+
: null;
|
|
487
|
+
if (target?.target === "local")
|
|
488
|
+
return ensureLocalWorkspaceReady();
|
|
489
|
+
if (target?.target === "connection" &&
|
|
490
|
+
typeof target.connectionName === "string") {
|
|
491
|
+
const named = resolveNamedConnection(target.connectionName);
|
|
492
|
+
if (!named)
|
|
493
|
+
throw new Error(`Connection not found: ${target.connectionName}`);
|
|
494
|
+
return named;
|
|
495
|
+
}
|
|
496
|
+
return requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
497
|
+
};
|
|
323
498
|
const requireActiveConnection = (candidate = connection) => {
|
|
324
499
|
if (!candidate) {
|
|
325
500
|
throw new Error('No database connection is configured yet. Open Connections, add a warehouse or local DuckDB/file connection, then retry.');
|
|
@@ -382,6 +557,45 @@ export async function startLocalServer(opts) {
|
|
|
382
557
|
}
|
|
383
558
|
}
|
|
384
559
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
560
|
+
const recordDatasetMetadataSnapshot = (datasets = datasetWorkspace.list()) => {
|
|
561
|
+
if (datasets.length === 0)
|
|
562
|
+
return;
|
|
563
|
+
try {
|
|
564
|
+
recordRuntimeSchemaSnapshot(projectRoot, {
|
|
565
|
+
source: "notebook_local_datasets",
|
|
566
|
+
tables: datasets.map((dataset) => ({
|
|
567
|
+
relation: dataset.alias,
|
|
568
|
+
schema: "notebook_local",
|
|
569
|
+
name: dataset.name,
|
|
570
|
+
description: [
|
|
571
|
+
dataset.description,
|
|
572
|
+
`${dataset.storageMode} ${dataset.format} dataset`,
|
|
573
|
+
`refreshed ${dataset.refreshedAt}`,
|
|
574
|
+
`trust ${dataset.trustState}`,
|
|
575
|
+
]
|
|
576
|
+
.filter(Boolean)
|
|
577
|
+
.join(" · "),
|
|
578
|
+
source: dataset.storageMode === "project"
|
|
579
|
+
? "project_dataset"
|
|
580
|
+
: "local_dataset",
|
|
581
|
+
columns: dataset.profile.columns.map((column) => ({
|
|
582
|
+
name: column.name,
|
|
583
|
+
type: column.type,
|
|
584
|
+
description: column.flags?.length
|
|
585
|
+
? column.flags.join(", ")
|
|
586
|
+
: undefined,
|
|
587
|
+
})),
|
|
588
|
+
})),
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
catch {
|
|
592
|
+
// Dataset metadata improves retrieval but must never block notebook startup.
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
if (datasetWorkspace.list().length > 0) {
|
|
596
|
+
await ensureLocalWorkspaceReady().catch(() => undefined);
|
|
597
|
+
recordDatasetMetadataSnapshot();
|
|
598
|
+
}
|
|
385
599
|
// Auto-register data/ CSV and Parquet files as DuckDB views so semantic layer
|
|
386
600
|
// queries like `FROM orders` resolve without requiring read_csv_auto() in SQL.
|
|
387
601
|
if (connection && (connection.driver === 'file' || connection.driver === 'duckdb')) {
|
|
@@ -509,6 +723,38 @@ export async function startLocalServer(opts) {
|
|
|
509
723
|
return message;
|
|
510
724
|
};
|
|
511
725
|
const formatNotebookResearchStorageError = (error) => (formatAgentRunInfrastructureError(error, 'Notebook research storage'));
|
|
726
|
+
/**
|
|
727
|
+
* CTX-001 / SKILL-001 — Build surfaces receive the same server-resolved,
|
|
728
|
+
* snapshot-bound domain envelope and compact evidence pack as Ask. The pack
|
|
729
|
+
* is a retrieval boundary, never an authorization bypass.
|
|
730
|
+
*/
|
|
731
|
+
const buildDirectBuildContext = async (input) => {
|
|
732
|
+
const snapshot = projectSnapshot();
|
|
733
|
+
const domainContext = resolveDomainContextEnvelope({
|
|
734
|
+
manifest: snapshot.manifest,
|
|
735
|
+
activeDomain: input.domain,
|
|
736
|
+
purpose: input.purpose,
|
|
737
|
+
modelAreaId: input.modelAreaId,
|
|
738
|
+
source: input.domain ? 'explicit_ui' : 'inferred',
|
|
739
|
+
snapshotId: snapshot.snapshotId,
|
|
740
|
+
});
|
|
741
|
+
const contextPack = await buildLocalContextPack(projectRoot, {
|
|
742
|
+
question: input.question,
|
|
743
|
+
mode: 'build',
|
|
744
|
+
surface: input.surface,
|
|
745
|
+
selectedContext: {
|
|
746
|
+
activeSurface: input.surface,
|
|
747
|
+
domain: input.domain,
|
|
748
|
+
purpose: input.purpose,
|
|
749
|
+
modelAreaId: input.modelAreaId,
|
|
750
|
+
...input.selectedContext,
|
|
751
|
+
},
|
|
752
|
+
domainContext,
|
|
753
|
+
strictness: 'balanced',
|
|
754
|
+
limit: 120,
|
|
755
|
+
});
|
|
756
|
+
return { domainContext, contextPack };
|
|
757
|
+
};
|
|
512
758
|
const buildAgentPromptArtifact = async (request, target, repair) => {
|
|
513
759
|
try {
|
|
514
760
|
await reindexProject(projectRoot, { kgPath: defaultKgPath(projectRoot) });
|
|
@@ -525,6 +771,17 @@ export async function startLocalServer(opts) {
|
|
|
525
771
|
const prompt = isRepair
|
|
526
772
|
? `${request.question}\n\nFix the previous attempt: ${repair?.repairHint}`
|
|
527
773
|
: request.question;
|
|
774
|
+
const domain = target === 'block' ? agentRunWorkspaceValue(request, 'domain') : undefined;
|
|
775
|
+
const modelAreaId = agentRunWorkspaceValue(request, 'modelAreaId');
|
|
776
|
+
const purpose = agentRunWorkspaceValue(request, 'purpose');
|
|
777
|
+
const { domainContext, contextPack } = await buildDirectBuildContext({
|
|
778
|
+
question: prompt,
|
|
779
|
+
domain,
|
|
780
|
+
purpose,
|
|
781
|
+
modelAreaId,
|
|
782
|
+
surface: target === 'cell' ? 'notebook' : 'block',
|
|
783
|
+
selectedContext: { target, mode },
|
|
784
|
+
});
|
|
528
785
|
return buildFromPrompt({
|
|
529
786
|
projectRoot,
|
|
530
787
|
prompt,
|
|
@@ -535,10 +792,16 @@ export async function startLocalServer(opts) {
|
|
|
535
792
|
target,
|
|
536
793
|
mode,
|
|
537
794
|
blockPath: target === 'block'
|
|
538
|
-
? agentRunWorkspaceValue(request, 'blockPath')
|
|
795
|
+
? agentRunWorkspaceValue(request, 'blockPath')
|
|
796
|
+
?? (request.selectedObject?.kind === 'block' && request.selectedObject.path?.endsWith('.dql')
|
|
797
|
+
? request.selectedObject.path
|
|
798
|
+
: undefined)
|
|
539
799
|
: undefined,
|
|
540
800
|
owner: agentRunWorkspaceValue(request, 'owner'),
|
|
541
|
-
domain
|
|
801
|
+
domain,
|
|
802
|
+
modelAreaId,
|
|
803
|
+
contextPack,
|
|
804
|
+
domainContext,
|
|
542
805
|
userId: agentRunWorkspaceValue(request, 'userId'),
|
|
543
806
|
skills,
|
|
544
807
|
dbtManifestPath: resolveDbtManifestPath(projectRoot, projectConfig),
|
|
@@ -605,6 +868,9 @@ export async function startLocalServer(opts) {
|
|
|
605
868
|
'Prefer certified DQL blocks when they exactly cover the question.',
|
|
606
869
|
'Generated DQL artifacts remain review-required; SQL is only the bounded preview/compiled evidence.',
|
|
607
870
|
'If the question needs investigation, return the clearest answer and next review action without certifying generated work.',
|
|
871
|
+
...(route === 'sql_cell'
|
|
872
|
+
? ['This is a notebook authoring fallback. Reuse certified blocks and semantic definitions as context, but return the requested row-level SQL cell rather than substituting a related aggregate answer.']
|
|
873
|
+
: []),
|
|
608
874
|
...(isRepair ? [`This is a repair attempt — fix the previous failure: ${repair?.repairHint}`] : []),
|
|
609
875
|
].join(' '),
|
|
610
876
|
};
|
|
@@ -621,11 +887,28 @@ export async function startLocalServer(opts) {
|
|
|
621
887
|
catch {
|
|
622
888
|
semanticDriver = undefined;
|
|
623
889
|
}
|
|
890
|
+
const requestedDomain = agentRunWorkspaceValue(request, 'domain');
|
|
891
|
+
const requestedPurpose = agentRunWorkspaceValue(request, 'purpose');
|
|
892
|
+
const requestedModelAreaId = agentRunWorkspaceValue(request, 'modelAreaId');
|
|
893
|
+
const runProjectSnapshot = projectSnapshot();
|
|
894
|
+
const domainContext = requestedDomain
|
|
895
|
+
? resolveDomainContextEnvelope({
|
|
896
|
+
manifest: runProjectSnapshot.manifest,
|
|
897
|
+
activeDomain: requestedDomain,
|
|
898
|
+
purpose: requestedPurpose,
|
|
899
|
+
modelAreaId: requestedModelAreaId,
|
|
900
|
+
source: 'explicit_ui',
|
|
901
|
+
snapshotId: runProjectSnapshot.snapshotId,
|
|
902
|
+
})
|
|
903
|
+
: undefined;
|
|
624
904
|
await runner.run({
|
|
625
905
|
provider: resolvedProvider,
|
|
626
906
|
messages: [
|
|
627
907
|
...(request.history ?? []).map((message) => ({ role: message.role, content: message.text })),
|
|
628
|
-
|
|
908
|
+
...(isRepair
|
|
909
|
+
? [{ role: 'assistant', content: `The prior attempt needs repair without changing the original question or requested output grain: ${repair?.repairHint}` }]
|
|
910
|
+
: []),
|
|
911
|
+
{ role: 'user', content: request.question },
|
|
629
912
|
],
|
|
630
913
|
conversationContext: request.conversationContext,
|
|
631
914
|
upstream: {
|
|
@@ -635,6 +918,13 @@ export async function startLocalServer(opts) {
|
|
|
635
918
|
reasoningEffort,
|
|
636
919
|
...(analysisDepth ? { analysisDepth } : {}),
|
|
637
920
|
projectRoot,
|
|
921
|
+
domainContext,
|
|
922
|
+
projectSnapshot: { snapshotId: runProjectSnapshot.snapshotId, manifest: runProjectSnapshot.manifest },
|
|
923
|
+
assertProjectSnapshot: (snapshotId) => {
|
|
924
|
+
// Refresh the service from current source state before the final guard.
|
|
925
|
+
projectSnapshot();
|
|
926
|
+
projectSnapshots.assertCurrent(snapshotId);
|
|
927
|
+
},
|
|
638
928
|
...(semanticDriver ? { semanticDriver } : {}),
|
|
639
929
|
executeCertifiedBlock: executeCertifiedBlockForAgent,
|
|
640
930
|
executeGeneratedSql: executeGeneratedSqlForAgent,
|
|
@@ -736,7 +1026,71 @@ export async function startLocalServer(opts) {
|
|
|
736
1026
|
},
|
|
737
1027
|
];
|
|
738
1028
|
}
|
|
739
|
-
|
|
1029
|
+
/**
|
|
1030
|
+
* PRD-001 / AGT-002 / AGT-006 / EXP-002 — every answer surface receives the
|
|
1031
|
+
* same executable DQL artifact. Generated SQL is parameterized before the
|
|
1032
|
+
* artifact leaves the runtime; it remains compiled evidence, not the primary
|
|
1033
|
+
* user-facing object. Saving later persists this exact source without another
|
|
1034
|
+
* model call.
|
|
1035
|
+
*/
|
|
1036
|
+
function attachExecutableDqlArtifactContract(governedAnswer, question) {
|
|
1037
|
+
const originalSql = governedAnswer.proposedSql?.trim() || governedAnswer.sql?.trim();
|
|
1038
|
+
let artifact = governedAnswer.dqlArtifact;
|
|
1039
|
+
if ((!artifact || artifact.kind === 'sql_block') && originalSql) {
|
|
1040
|
+
const parameterized = parameterizeSqlForDqlImport(originalSql);
|
|
1041
|
+
const artifactName = artifact?.name
|
|
1042
|
+
?? artifact?.source.match(/\bblock\s+"([^"]+)"/i)?.[1]
|
|
1043
|
+
?? deriveGeneratedDraftSlug(question);
|
|
1044
|
+
const domain = artifact?.source.match(/\bdomain\s*=\s*"([^"]+)"/i)?.[1] ?? 'uncategorized';
|
|
1045
|
+
const source = candidateToDqlSource({
|
|
1046
|
+
name: artifactName,
|
|
1047
|
+
domain,
|
|
1048
|
+
description: question,
|
|
1049
|
+
owner: '',
|
|
1050
|
+
tags: ['ai-generated', 'review-required'],
|
|
1051
|
+
terms: [],
|
|
1052
|
+
pattern: '',
|
|
1053
|
+
grain: '',
|
|
1054
|
+
entities: [],
|
|
1055
|
+
outputs: [],
|
|
1056
|
+
dimensions: [],
|
|
1057
|
+
allowedFilters: parameterized.allowedFilters,
|
|
1058
|
+
parameterPolicy: parameterized.parameterPolicy,
|
|
1059
|
+
filterBindings: parameterized.filterBindings,
|
|
1060
|
+
parameterDecisions: parameterized.parameterDecisions,
|
|
1061
|
+
sourceSystems: [],
|
|
1062
|
+
replacementFor: [],
|
|
1063
|
+
reviewCadence: 'monthly',
|
|
1064
|
+
sql: parameterized.sql,
|
|
1065
|
+
llmContext: `Review-required DQL generated for: ${question}`,
|
|
1066
|
+
});
|
|
1067
|
+
artifact = {
|
|
1068
|
+
...(artifact ?? { kind: 'sql_block' }),
|
|
1069
|
+
kind: 'sql_block',
|
|
1070
|
+
name: artifactName,
|
|
1071
|
+
source,
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
if (!artifact?.source)
|
|
1075
|
+
return;
|
|
1076
|
+
const invocation = prepareBlockInvocation({
|
|
1077
|
+
source: artifact.source,
|
|
1078
|
+
parameters: artifact.parameterValues,
|
|
1079
|
+
question,
|
|
1080
|
+
surface: 'ask_ai',
|
|
1081
|
+
});
|
|
1082
|
+
const certified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
1083
|
+
const semantic = governedAnswer.route?.tier === 'semantic_metric';
|
|
1084
|
+
governedAnswer.dqlArtifact = {
|
|
1085
|
+
...artifact,
|
|
1086
|
+
...(invocation.parameters.length > 0 ? { parameters: invocation.parameters } : {}),
|
|
1087
|
+
...(Object.keys(invocation.values).length > 0 ? { parameterValues: invocation.values } : {}),
|
|
1088
|
+
persistence: artifact.sourcePath ? 'saved' : 'transient',
|
|
1089
|
+
trustState: certified ? 'certified' : semantic ? 'governed' : 'review_required',
|
|
1090
|
+
...(originalSql ? { compiledSql: originalSql } : {}),
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
const answerRunExecutor = async ({ request, route, routeDecision, attempt, repairHint, emit }) => {
|
|
740
1094
|
let governedAnswer;
|
|
741
1095
|
try {
|
|
742
1096
|
governedAnswer = await runGovernedAgentAnswerForRun(request, { attempt, repairHint }, route, (message) => emit({ type: 'executor.started', message, route }));
|
|
@@ -745,6 +1099,126 @@ export async function startLocalServer(opts) {
|
|
|
745
1099
|
if (!governedAnswer.appliedHints) {
|
|
746
1100
|
governedAnswer.appliedHints = governedAnswer.contextPack?.appliedHints;
|
|
747
1101
|
}
|
|
1102
|
+
// EXP-001 — a missing modeled relationship is not the same as an unsafe
|
|
1103
|
+
// relationship. The answer loop exposes a non-executed candidate only for
|
|
1104
|
+
// that narrow case; the host owns bounded probe/execution and never
|
|
1105
|
+
// weakens the final governed-SQL guard.
|
|
1106
|
+
if (governedAnswer.exploratoryCandidate) {
|
|
1107
|
+
const explorationSchema = governedAnswer.contextPack?.allowedSqlContext.relations.map((table) => ({
|
|
1108
|
+
relation: table.relation,
|
|
1109
|
+
name: table.name,
|
|
1110
|
+
columns: table.columns.map((column) => ({ ...column })),
|
|
1111
|
+
source: table.source,
|
|
1112
|
+
}))
|
|
1113
|
+
?? governedAnswer.analysisPlan?.candidateTables.map((table) => ({
|
|
1114
|
+
relation: table.relation,
|
|
1115
|
+
name: table.relation.split('.').at(-1) ?? table.relation,
|
|
1116
|
+
columns: table.columns.map((name) => ({ name })),
|
|
1117
|
+
source: 'answer analysis plan',
|
|
1118
|
+
}));
|
|
1119
|
+
const exploration = await executeExploratoryCandidate(governedAnswer.exploratoryCandidate, explorationSchema, request.question, governedAnswer.contextPack?.questionPlan.requestedShape.topN?.scope === 'overall'
|
|
1120
|
+
? governedAnswer.contextPack.questionPlan.requestedShape.topN.n
|
|
1121
|
+
: undefined);
|
|
1122
|
+
const probeSummary = exploration.proofs.length > 0
|
|
1123
|
+
? `${exploration.proofs.length} bounded join probe${exploration.proofs.length === 1 ? '' : 's'} completed.`
|
|
1124
|
+
: 'No join probe was required for this query.';
|
|
1125
|
+
const baseWarnings = governedAnswer.validationWarnings ?? [];
|
|
1126
|
+
const evidence = governedAnswer.evidence ?? {
|
|
1127
|
+
route: [], lineage: [], businessContext: [], selectedAssets: [], sourceTables: [], semanticObjects: [], citations: [],
|
|
1128
|
+
};
|
|
1129
|
+
governedAnswer.evidence = evidence;
|
|
1130
|
+
evidence.route.push({
|
|
1131
|
+
tool: 'explore_dbt_grounded_sql',
|
|
1132
|
+
status: exploration.result ? 'checked' : 'failed',
|
|
1133
|
+
label: exploration.result ? 'Executed bounded DBT-grounded exploration' : 'DBT-grounded exploration could not execute',
|
|
1134
|
+
detail: exploration.error ?? probeSummary,
|
|
1135
|
+
});
|
|
1136
|
+
if (exploration.proofs.length > 0) {
|
|
1137
|
+
evidence.businessContext.push({
|
|
1138
|
+
label: 'Exploratory join probes',
|
|
1139
|
+
value: exploration.proofs.map((proof) => proof.summary).join('; '),
|
|
1140
|
+
source: 'bounded DBT-grounded validation',
|
|
1141
|
+
});
|
|
1142
|
+
}
|
|
1143
|
+
if (exploration.result) {
|
|
1144
|
+
evidence.route = [
|
|
1145
|
+
...cascadeTraceToEvidenceRouteSteps(createCascadeTrace({ terminalLane: 'generated' })),
|
|
1146
|
+
...evidence.route.filter((step) => !step.tool.startsWith('cascade_')),
|
|
1147
|
+
];
|
|
1148
|
+
evidence.validation = {
|
|
1149
|
+
status: 'warning',
|
|
1150
|
+
message: 'The bounded DBT-grounded query executed successfully, but its inferred relationship path still requires analyst review before reuse.',
|
|
1151
|
+
};
|
|
1152
|
+
evidence.execution = {
|
|
1153
|
+
status: 'executed',
|
|
1154
|
+
message: 'Executed the bounded DBT-grounded exploratory SQL preview.',
|
|
1155
|
+
rowCount: exploration.result.rowCount,
|
|
1156
|
+
executionTime: exploration.result.executionTime,
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1159
|
+
const repairedExploratoryCandidate = exploration.sql !== governedAnswer.exploratoryCandidate.sql
|
|
1160
|
+
? { ...governedAnswer.exploratoryCandidate, sql: exploration.sql }
|
|
1161
|
+
: governedAnswer.exploratoryCandidate;
|
|
1162
|
+
const lifetimeGrainRepair = exploration.repairs.find((repair) => repair.startsWith('Used lifetime_'));
|
|
1163
|
+
const explorationAnswer = exploration.result
|
|
1164
|
+
? lifetimeGrainRepair
|
|
1165
|
+
? `Exploratory DBT-grounded query executed and returned ${exploration.result.rowCount} row${exploration.result.rowCount === 1 ? '' : 's'}. Each row is an owning-entity and joined-dimension pair. The lifetime measure remains an entity-level attribute repeated across that dimension; it is not allocated to the joined dimension. ${lifetimeGrainRepair}`
|
|
1166
|
+
: `Exploratory DBT-grounded query executed and returned ${exploration.result.rowCount} row${exploration.result.rowCount === 1 ? '' : 's'}. Review the inferred join evidence before reusing this analysis.`
|
|
1167
|
+
: `DQL found a DBT-grounded exploratory path, but the bounded validation could not run: ${exploration.error ?? 'unknown execution error'}.`;
|
|
1168
|
+
governedAnswer = {
|
|
1169
|
+
...governedAnswer,
|
|
1170
|
+
kind: 'uncertified',
|
|
1171
|
+
sourceTier: 'dbt_manifest',
|
|
1172
|
+
certification: 'analyst_review_required',
|
|
1173
|
+
reviewStatus: 'analyst_review_required',
|
|
1174
|
+
confidence: exploration.result ? 0.58 : 0.2,
|
|
1175
|
+
...(exploration.result ? {
|
|
1176
|
+
refusalCode: undefined,
|
|
1177
|
+
refusalDetails: undefined,
|
|
1178
|
+
intentDecision: {
|
|
1179
|
+
...governedAnswer.intentDecision,
|
|
1180
|
+
action: 'answer',
|
|
1181
|
+
confidence: Math.max(governedAnswer.intentDecision?.confidence ?? 0, 0.9),
|
|
1182
|
+
reason: 'Executed a bounded DBT-grounded exploratory answer after certified and semantic routes did not fully cover the request.',
|
|
1183
|
+
clarifyingQuestion: undefined,
|
|
1184
|
+
clarifySoft: undefined,
|
|
1185
|
+
followsUp: governedAnswer.intentDecision?.followsUp ?? false,
|
|
1186
|
+
},
|
|
1187
|
+
} : {}),
|
|
1188
|
+
result: exploration.result,
|
|
1189
|
+
proposedSql: exploration.sql,
|
|
1190
|
+
sql: exploration.sql,
|
|
1191
|
+
exploratoryCandidate: repairedExploratoryCandidate,
|
|
1192
|
+
executionError: exploration.error,
|
|
1193
|
+
text: explorationAnswer,
|
|
1194
|
+
answer: explorationAnswer,
|
|
1195
|
+
trustLabel: 'Exploratory · DBT-grounded',
|
|
1196
|
+
contextPackId: governedAnswer.contextPackId ?? governedAnswer.contextPack?.id,
|
|
1197
|
+
route: {
|
|
1198
|
+
tier: 'generated_sql',
|
|
1199
|
+
label: exploration.result
|
|
1200
|
+
? 'Exploratory · DBT-grounded answer (review required).'
|
|
1201
|
+
: 'Exploratory · DBT-grounded query needs review before execution.',
|
|
1202
|
+
},
|
|
1203
|
+
cascade: exploration.result
|
|
1204
|
+
? createCascadeAnswerResult({
|
|
1205
|
+
routeTier: 'generated_sql',
|
|
1206
|
+
label: 'Exploratory DBT-grounded answer executed with review required.',
|
|
1207
|
+
artifactKind: 'dbt_grounded_exploration',
|
|
1208
|
+
hasSqlPreview: true,
|
|
1209
|
+
executionStatus: 'executed',
|
|
1210
|
+
rowCount: exploration.result.rowCount,
|
|
1211
|
+
})
|
|
1212
|
+
: governedAnswer.cascade,
|
|
1213
|
+
validationWarnings: [
|
|
1214
|
+
...baseWarnings,
|
|
1215
|
+
probeSummary,
|
|
1216
|
+
...exploration.repairs,
|
|
1217
|
+
...(exploration.error ? [`Exploratory execution: ${exploration.error}`] : []),
|
|
1218
|
+
],
|
|
1219
|
+
};
|
|
1220
|
+
}
|
|
1221
|
+
attachExecutableDqlArtifactContract(governedAnswer, request.question);
|
|
748
1222
|
applySmartVisualization(governedAnswer, request.question);
|
|
749
1223
|
}
|
|
750
1224
|
catch (error) {
|
|
@@ -767,8 +1241,21 @@ export async function startLocalServer(opts) {
|
|
|
767
1241
|
const resolvedRoute = resolvedRunRouteFromAnswer(governedAnswer) ?? route;
|
|
768
1242
|
const isCertified = governedAnswer.certification === 'certified' || governedAnswer.kind === 'certified';
|
|
769
1243
|
const isSemantic = governedAnswer.route?.tier === 'semantic_metric';
|
|
770
|
-
const
|
|
1244
|
+
const requestedNotebookDataset = findMentionedNotebookDataset(request.question, datasetWorkspace.list());
|
|
1245
|
+
const governedAssetMissesLocalSource = Boolean(requestedNotebookDataset && (isCertified
|
|
1246
|
+
|| isSemantic
|
|
1247
|
+
|| governedAnswer.sourceCertifiedBlock
|
|
1248
|
+
|| governedAnswer.dqlArtifact?.kind === 'certified_block'));
|
|
1249
|
+
const isExploratory = Boolean(governedAnswer.exploratoryCandidate);
|
|
1250
|
+
const isGroundingGap = governedAnswer.kind === 'no_answer'
|
|
1251
|
+
&& governedAnswer.refusalCode === 'grounding_gap'
|
|
1252
|
+
&& !isExploratory;
|
|
771
1253
|
const isProviderError = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'provider_error';
|
|
1254
|
+
// AGT-004: a rejected attribution/export/proof policy is a deliberate
|
|
1255
|
+
// governance boundary, not an ambiguous user question and not a repairable
|
|
1256
|
+
// retrieval miss. Keep the precise policy detail visible, but never burn two
|
|
1257
|
+
// more provider calls retrying the same incompatible candidate.
|
|
1258
|
+
const isPolicyBlocked = governedAnswer.kind === 'no_answer' && governedAnswer.refusalCode === 'policy_blocked';
|
|
772
1259
|
// The model tried to compose a governed query and declined despite having usable
|
|
773
1260
|
// context (e.g. it wasn't confident about a multi-table join). That is NOT a
|
|
774
1261
|
// question for the USER to clarify — it's a case to retry harder: escalate to a
|
|
@@ -783,9 +1270,11 @@ export async function startLocalServer(opts) {
|
|
|
783
1270
|
// grounding gap or a model decline is retried/escalated by the engine, and a
|
|
784
1271
|
// provider outage is surfaced as blocked so the UI offers a retry.
|
|
785
1272
|
const needsClarification = governedAnswer.kind === 'no_answer'
|
|
786
|
-
&& !isGroundingGap && !isProviderError && !isModelDeclined;
|
|
1273
|
+
&& !isGroundingGap && !isProviderError && !isModelDeclined && !isPolicyBlocked;
|
|
787
1274
|
const sql = governedAnswer.proposedSql ?? governedAnswer.sql;
|
|
788
|
-
const runnableSql = governedAnswer.kind === 'no_answer'
|
|
1275
|
+
const runnableSql = governedAnswer.kind === 'no_answer' || (isExploratory && !governedAnswer.result)
|
|
1276
|
+
? undefined
|
|
1277
|
+
: sql;
|
|
789
1278
|
// Synthesis is a legacy polish pass. Certified/no-answer paths and lanes
|
|
790
1279
|
// that already produced DQL-first final prose keep the fast path.
|
|
791
1280
|
let synthesizedAnswer;
|
|
@@ -798,13 +1287,14 @@ export async function startLocalServer(opts) {
|
|
|
798
1287
|
const result = await synthesizeAnswer({
|
|
799
1288
|
question: request.question,
|
|
800
1289
|
category: routeDecision?.category,
|
|
801
|
-
|
|
1290
|
+
// The primary Ask reply is always business-facing. Analysts keep
|
|
1291
|
+
// the full DQL, SQL, lineage, gates, and grain in the inspector.
|
|
1292
|
+
audience: 'stakeholder',
|
|
802
1293
|
resultPreview: preview,
|
|
803
1294
|
sql: sql,
|
|
804
1295
|
draftText: draft,
|
|
805
|
-
gaps: governedAnswer.validationWarnings,
|
|
1296
|
+
gaps: businessNarrativeGaps(governedAnswer.validationWarnings),
|
|
806
1297
|
}, {
|
|
807
|
-
onDelta: emitAnswerDelta,
|
|
808
1298
|
complete: ({ system, user, signal, onDelta }) => streamOrGenerate(provider, [{ role: 'system', content: system }, { role: 'user', content: user }], { maxTokens: 350, temperature: 0.3, signal }, onDelta ?? (() => { })),
|
|
809
1299
|
});
|
|
810
1300
|
if (result.text)
|
|
@@ -846,13 +1336,18 @@ export async function startLocalServer(opts) {
|
|
|
846
1336
|
: [
|
|
847
1337
|
{ id: 'create-block', label: governedAnswer.dqlArtifact ? 'Review DQL draft' : 'Create DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
848
1338
|
{ id: 'research-gap', label: 'Research deeper', route: 'research' },
|
|
849
|
-
...(runnableSql ? [{
|
|
1339
|
+
...(runnableSql ? [{
|
|
1340
|
+
id: 'insert-sql',
|
|
1341
|
+
label: governedAnswer.dqlArtifact ? 'Insert as DQL cell' : 'Insert SQL preview',
|
|
1342
|
+
route: 'sql_cell',
|
|
1343
|
+
artifactKind: 'sql_cell',
|
|
1344
|
+
}] : []),
|
|
850
1345
|
];
|
|
851
1346
|
return {
|
|
852
1347
|
resolvedRoute,
|
|
853
1348
|
answerRefusalCode: governedAnswer.kind === 'no_answer' ? governedAnswer.refusalCode : undefined,
|
|
854
1349
|
answerTier: governedAnswer.route?.tier,
|
|
855
|
-
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : 'Answered with review-required generated analysis.'),
|
|
1350
|
+
summary: governedAnswer.route?.label ?? (isCertified ? 'Answered from certified DQL context.' : isExploratory ? 'Exploratory DBT-grounded analysis requires review.' : 'Answered with review-required generated analysis.'),
|
|
856
1351
|
answer: synthesizedAnswer ?? governedAnswer.answer ?? governedAnswer.text,
|
|
857
1352
|
status,
|
|
858
1353
|
trustState,
|
|
@@ -864,20 +1359,35 @@ export async function startLocalServer(opts) {
|
|
|
864
1359
|
? (governedAnswer.dqlArtifact && !isProviderError
|
|
865
1360
|
? [agentRunArtifact('dql_block_draft', 'DQL draft (review required)', governedAnswer.dqlArtifact, undefined, 'review_required')]
|
|
866
1361
|
: [])
|
|
867
|
-
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : isSemantic ? 'Governed semantic answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : isSemantic ? 'governed' : 'review_required')],
|
|
1362
|
+
: [agentRunArtifact('answer', isCertified ? 'Certified answer' : isSemantic ? 'Governed semantic answer' : isExploratory ? 'Exploratory DBT-grounded answer' : 'Review-required answer', governedAnswer, governedAnswer.sourceCertifiedBlock ?? governedAnswer.block?.name, isCertified ? 'certified' : isSemantic ? 'governed' : 'review_required')],
|
|
868
1363
|
evaluations: [
|
|
869
1364
|
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to governed answer.', {
|
|
870
1365
|
plannedRoute: route,
|
|
871
1366
|
resolvedRoute,
|
|
872
1367
|
aiRoute: governedAnswer.route,
|
|
873
1368
|
}),
|
|
1369
|
+
...(governedAssetMissesLocalSource ? [
|
|
1370
|
+
{
|
|
1371
|
+
...agentRunEvaluation('requested-local-source', 'Requested local dataset', false, 'warning', `${requestedNotebookDataset.alias} was explicitly requested, but the governed answer did not use local data. Reuse its governed logic only as context and build the mixed-source SQL fallback.`, { datasetId: requestedNotebookDataset.id, alias: requestedNotebookDataset.alias }),
|
|
1372
|
+
suggestedRepair: `Build a notebook SQL fallback that combines the requested ${requestedNotebookDataset.alias} dataset with the required warehouse extraction.`,
|
|
1373
|
+
repairAction: {
|
|
1374
|
+
kind: 'escalate',
|
|
1375
|
+
route: 'sql_cell',
|
|
1376
|
+
hint: `The certified or semantic result is not exact because it omits the explicitly requested local dataset ${requestedNotebookDataset.alias}. Author the warehouse extraction and mixed-source notebook handoff.`,
|
|
1377
|
+
},
|
|
1378
|
+
},
|
|
1379
|
+
] : []),
|
|
874
1380
|
agentRunEvaluation('trust-boundary', 'Trust boundary', isCertified || isSemantic, isCertified || isSemantic ? 'info' : 'warning', isCertified
|
|
875
1381
|
? 'The answer came from certified DQL context.'
|
|
876
1382
|
: isSemantic
|
|
877
1383
|
? 'The answer was compiled and executed from governed semantic definitions; it is not a human-certified reusable block.'
|
|
878
1384
|
: needsClarification
|
|
879
1385
|
? 'The answer loop needs more context before producing a governed answer.'
|
|
880
|
-
:
|
|
1386
|
+
: isPolicyBlocked
|
|
1387
|
+
? `DQL blocked the generated path under its relationship policy: ${governedAnswer.refusalDetails?.message ?? 'the relationship is not authorized for this analysis.'}`
|
|
1388
|
+
: isExploratory
|
|
1389
|
+
? 'The answer used bounded DBT/schema evidence because no certified modeled relationship path covered the request. It is review-required and was not certified.'
|
|
1390
|
+
: 'The answer is generated or semantic-layer backed and remains review-required.', governedAnswer.route),
|
|
881
1391
|
...(isGroundingGap ? [
|
|
882
1392
|
{
|
|
883
1393
|
...agentRunEvaluation('grounding-gap', 'Metadata grounding', false, 'warning', 'The answer loop found a metadata grounding gap that can be retried with wider context.', {
|
|
@@ -902,6 +1412,15 @@ export async function startLocalServer(opts) {
|
|
|
902
1412
|
repairAction: { kind: 'escalate', route: 'research', hint: declinedRepairHint },
|
|
903
1413
|
},
|
|
904
1414
|
] : []),
|
|
1415
|
+
...(isPolicyBlocked ? [
|
|
1416
|
+
agentRunEvaluation('relationship-policy', 'Relationship policy', false, 'warning', governedAnswer.refusalDetails?.message
|
|
1417
|
+
?? 'The selected relationship is not authorized for this analysis.', {
|
|
1418
|
+
refusalCode: governedAnswer.refusalCode,
|
|
1419
|
+
refusalDetails: governedAnswer.refusalDetails,
|
|
1420
|
+
validationWarnings: governedAnswer.validationWarnings,
|
|
1421
|
+
route: governedAnswer.route,
|
|
1422
|
+
}),
|
|
1423
|
+
] : []),
|
|
905
1424
|
...(governedAnswer.executionError ? [
|
|
906
1425
|
agentRunEvaluation('execution-error', 'Execution error', false, 'warning', governedAnswer.executionError),
|
|
907
1426
|
] : []),
|
|
@@ -1138,33 +1657,112 @@ export async function startLocalServer(opts) {
|
|
|
1138
1657
|
};
|
|
1139
1658
|
},
|
|
1140
1659
|
sql_cell: async ({ request, routeDecision, attempt, repairHint }) => {
|
|
1660
|
+
const selectedCellSql = agentRunWorkspaceValue(request, 'cellSql');
|
|
1661
|
+
const mixedSourcePlan = selectedCellSql
|
|
1662
|
+
? planMixedSourceSql(selectedCellSql, datasetWorkspace.list().map((dataset) => dataset.alias))
|
|
1663
|
+
: null;
|
|
1664
|
+
if (mixedSourcePlan) {
|
|
1665
|
+
const explanation = [
|
|
1666
|
+
`The original query directly joined warehouse data with the local dataset ${mixedSourcePlan.localDataset}. Those sources execute in different engines, so the CSV is not a missing dbt table.`,
|
|
1667
|
+
`I prepared the warehouse-only extraction and retained ${mixedSourcePlan.warehouseKey}, the key needed to join with ${mixedSourcePlan.localDataset}.${mixedSourcePlan.localKey}.`,
|
|
1668
|
+
'Insert and run this SQL on the warehouse connection. On its result, choose Combine with local data, select the local dataset, and confirm the suggested keys. The final mixed result runs locally and remains review-required.',
|
|
1669
|
+
].join(' ');
|
|
1670
|
+
const result = {
|
|
1671
|
+
target: 'cell',
|
|
1672
|
+
sql: mixedSourcePlan.warehouseSql,
|
|
1673
|
+
explanation,
|
|
1674
|
+
mixedSourcePlan,
|
|
1675
|
+
};
|
|
1676
|
+
return {
|
|
1677
|
+
summary: `Prepared a warehouse extraction for local combination with ${mixedSourcePlan.localDataset}.`,
|
|
1678
|
+
answer: explanation,
|
|
1679
|
+
artifacts: [agentRunArtifact('sql_cell', 'Warehouse extraction for local analysis', result)],
|
|
1680
|
+
evaluations: [
|
|
1681
|
+
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Detected a mixed local and warehouse SQL repair.'),
|
|
1682
|
+
agentRunEvaluation('mixed-source-boundary', 'Mixed-source boundary', true, 'warning', 'No direct cross-engine join was executed. The user must explicitly stage the bounded warehouse result.'),
|
|
1683
|
+
],
|
|
1684
|
+
nextActions: [{ id: 'insert-sql', label: 'Insert warehouse extraction', artifactKind: 'sql_cell' }],
|
|
1685
|
+
};
|
|
1686
|
+
}
|
|
1687
|
+
const requestedNotebookDataset = findMentionedNotebookDataset(request.question, datasetWorkspace.list());
|
|
1688
|
+
const authoringRequest = requestedNotebookDataset
|
|
1689
|
+
? {
|
|
1690
|
+
...request,
|
|
1691
|
+
question: [
|
|
1692
|
+
request.question,
|
|
1693
|
+
'',
|
|
1694
|
+
`Notebook SQL fallback contract: ${requestedNotebookDataset.alias} is a local dataset, not a warehouse table. Produce the warehouse-only row-level extraction needed for the requested analysis. Include the warehouse join key that corresponds to the local dataset identifier, plus the requested order/detail columns. Do not return an aggregate certified answer and do not reference ${requestedNotebookDataset.alias} in the warehouse SQL. The UI will stage this bounded result and create the local join cell after confirmation.`,
|
|
1695
|
+
].join('\n'),
|
|
1696
|
+
}
|
|
1697
|
+
: request;
|
|
1141
1698
|
// P4: route Notebook SQL-cell generation through the SAME tool-rich governed
|
|
1142
1699
|
// pipeline Ask AI uses (schema-discovery tools P3 + declined-retry P1 + budget
|
|
1143
1700
|
// recovery P2), instead of the tool-less buildFromPrompt path. A SQL cell only
|
|
1144
1701
|
// needs the SQL, which the governed answer produces directly as proposedSql.
|
|
1145
|
-
const governedAnswer = await runGovernedAgentAnswerForRun(
|
|
1146
|
-
|
|
1702
|
+
const governedAnswer = await runGovernedAgentAnswerForRun(authoringRequest, { attempt: attempt ?? 0, repairHint }, 'sql_cell');
|
|
1703
|
+
attachExecutableDqlArtifactContract(governedAnswer, request.question);
|
|
1704
|
+
const schemaContext = requestedNotebookDataset
|
|
1705
|
+
? await getSchemaContextForAgent(request.question)
|
|
1706
|
+
: [];
|
|
1707
|
+
const generatedSql = governedAnswer.proposedSql ?? governedAnswer.sql ?? '';
|
|
1708
|
+
const generatedPlan = requestedNotebookDataset && generatedSql.trim()
|
|
1709
|
+
? planMixedSourceNotebookSql(generatedSql, {
|
|
1710
|
+
id: requestedNotebookDataset.id,
|
|
1711
|
+
name: requestedNotebookDataset.name,
|
|
1712
|
+
alias: requestedNotebookDataset.alias,
|
|
1713
|
+
columns: requestedNotebookDataset.profile.columns.map((column) => ({
|
|
1714
|
+
name: column.name,
|
|
1715
|
+
flags: column.flags,
|
|
1716
|
+
})),
|
|
1717
|
+
}, schemaContext)
|
|
1718
|
+
: null;
|
|
1719
|
+
const generatedReusedNonExactAsset = Boolean(governedAnswer.sourceCertifiedBlock || governedAnswer.dqlArtifact?.kind === 'certified_block');
|
|
1720
|
+
const fallbackSql = requestedNotebookDataset && (generatedReusedNonExactAsset || !generatedSql.trim())
|
|
1721
|
+
? buildMixedSourceWarehouseFallbackSql(request.question, {
|
|
1722
|
+
id: requestedNotebookDataset.id,
|
|
1723
|
+
name: requestedNotebookDataset.name,
|
|
1724
|
+
alias: requestedNotebookDataset.alias,
|
|
1725
|
+
columns: requestedNotebookDataset.profile.columns.map((column) => ({ name: column.name, flags: column.flags })),
|
|
1726
|
+
}, schemaContext)
|
|
1727
|
+
: undefined;
|
|
1728
|
+
const naturalMixedSourcePlan = requestedNotebookDataset && fallbackSql
|
|
1729
|
+
? planMixedSourceNotebookSql(fallbackSql, {
|
|
1730
|
+
id: requestedNotebookDataset.id,
|
|
1731
|
+
name: requestedNotebookDataset.name,
|
|
1732
|
+
alias: requestedNotebookDataset.alias,
|
|
1733
|
+
columns: requestedNotebookDataset.profile.columns.map((column) => ({ name: column.name, flags: column.flags })),
|
|
1734
|
+
}, schemaContext)
|
|
1735
|
+
: generatedPlan;
|
|
1736
|
+
const sql = naturalMixedSourcePlan?.warehouseSql ?? generatedSql;
|
|
1147
1737
|
const explanation = governedAnswer.answer ?? governedAnswer.text;
|
|
1148
1738
|
const hasSql = Boolean(sql.trim());
|
|
1149
1739
|
// Preserve the BuildCellResult shape the notebook UI already consumes.
|
|
1150
1740
|
const result = {
|
|
1151
1741
|
target: 'cell',
|
|
1152
1742
|
sql,
|
|
1153
|
-
explanation
|
|
1743
|
+
explanation: naturalMixedSourcePlan
|
|
1744
|
+
? `No certified block or semantic result exactly covered all requested sources. The workflow combines ${naturalMixedSourcePlan.warehouseRelations?.join(', ') || 'the requested warehouse tables'} in one bounded warehouse extraction, then joins it locally with ${naturalMixedSourcePlan.localDataset} on ${naturalMixedSourcePlan.warehouseKey} = ${naturalMixedSourcePlan.localKey}. Click Add workflow to notebook below to create the cells. The mixed result remains review-required.`
|
|
1745
|
+
: explanation,
|
|
1746
|
+
...(naturalMixedSourcePlan ? { mixedSourcePlan: naturalMixedSourcePlan } : {}),
|
|
1747
|
+
...(!naturalMixedSourcePlan && governedAnswer.dqlArtifact ? { dqlArtifact: governedAnswer.dqlArtifact } : {}),
|
|
1154
1748
|
...(governedAnswer.appliedSkills ? { appliedSkills: governedAnswer.appliedSkills } : {}),
|
|
1155
1749
|
};
|
|
1156
1750
|
return {
|
|
1157
1751
|
summary: hasSql
|
|
1158
|
-
?
|
|
1752
|
+
? naturalMixedSourcePlan
|
|
1753
|
+
? `Prepared a mixed-source notebook workflow using ${naturalMixedSourcePlan.localDataset}.`
|
|
1754
|
+
: 'Created a review-required DQL query cell.'
|
|
1159
1755
|
: (explanation || 'No SQL could be generated for this request.'),
|
|
1160
|
-
answer: explanation,
|
|
1161
|
-
artifacts: [agentRunArtifact('sql_cell', 'Generated
|
|
1756
|
+
answer: result.explanation,
|
|
1757
|
+
artifacts: [agentRunArtifact('sql_cell', naturalMixedSourcePlan ? 'Warehouse extraction for local analysis' : 'Generated DQL query cell', result)],
|
|
1162
1758
|
evaluations: [
|
|
1163
1759
|
agentRunEvaluation('route-decision', 'Route decision', true, 'info', routeDecision?.reason ?? 'Routed request to SQL cell generation.'),
|
|
1164
|
-
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning',
|
|
1760
|
+
agentRunEvaluation('review-boundary', 'Review boundary', true, 'warning', naturalMixedSourcePlan
|
|
1761
|
+
? 'The warehouse extraction is staged with limits and the CSV join runs locally. Mixed-source output is never auto-certified.'
|
|
1762
|
+
: 'Generated DQL and its compiled SQL evidence must be reviewed before certification.'),
|
|
1165
1763
|
],
|
|
1166
|
-
nextActions: [
|
|
1167
|
-
{ id: 'insert-sql', label: 'Insert SQL preview', artifactKind: 'sql_cell' },
|
|
1764
|
+
nextActions: naturalMixedSourcePlan ? [] : [
|
|
1765
|
+
{ id: 'insert-sql', label: governedAnswer.dqlArtifact ? 'Insert as DQL cell' : 'Insert SQL preview', artifactKind: 'sql_cell' },
|
|
1168
1766
|
{ id: 'create-block', label: 'Review as DQL draft', route: 'dql_block_draft', artifactKind: 'dql_block_draft' },
|
|
1169
1767
|
],
|
|
1170
1768
|
};
|
|
@@ -1519,19 +2117,21 @@ export async function startLocalServer(opts) {
|
|
|
1519
2117
|
cells: snapshotCells,
|
|
1520
2118
|
});
|
|
1521
2119
|
};
|
|
1522
|
-
const
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
2120
|
+
const executeDqlArtifactSourceForAgent = async (source, metadata = {}, invocationInput) => {
|
|
2121
|
+
const invocation = prepareBlockInvocation({
|
|
2122
|
+
source,
|
|
2123
|
+
parameters: invocationInput?.parameters,
|
|
2124
|
+
question: invocationInput?.question,
|
|
2125
|
+
surface: 'ask_ai',
|
|
2126
|
+
});
|
|
2127
|
+
if (invocation.errors.length > 0)
|
|
2128
|
+
throw new Error(invocation.errors.join(' '));
|
|
2129
|
+
if (invocation.unresolvedParameters.length > 0) {
|
|
2130
|
+
throw new Error(`I need values for: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
1530
2131
|
}
|
|
1531
|
-
const absBlockPath = join(projectRoot, block.filePath);
|
|
1532
|
-
const source = readFileSync(absBlockPath, 'utf-8');
|
|
1533
2132
|
const activeConnection = requireActiveConnection();
|
|
1534
2133
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
2134
|
+
const plan = buildExecutionPlan({ id: `agent-${metadata.name ?? 'dql-artifact'}`, type: 'dql', source, title: metadata.name ?? 'DQL artifact' }, { semanticLayer, driver: activeConnection.driver, tableMapping, parameters: invocation.values });
|
|
1535
2135
|
const semanticCompose = semanticLayer
|
|
1536
2136
|
? composeSemanticBlockSql(source, semanticLayer, {
|
|
1537
2137
|
driver: activeConnection.driver,
|
|
@@ -1539,29 +2139,55 @@ export async function startLocalServer(opts) {
|
|
|
1539
2139
|
projectRoot,
|
|
1540
2140
|
projectConfig,
|
|
1541
2141
|
detectedProvider: semanticDetectedProvider,
|
|
2142
|
+
parameters: invocation.values,
|
|
1542
2143
|
})
|
|
1543
2144
|
: null;
|
|
1544
|
-
const plan = buildExecutionPlan({ id: `agent-${block.name}`, type: 'dql', source, title: block.name }, { semanticLayer, driver: activeConnection.driver, tableMapping });
|
|
1545
2145
|
if (!plan && !semanticCompose?.sql) {
|
|
1546
2146
|
const semanticError = semanticCompose?.diagnostics.find((diagnostic) => diagnostic.severity === 'error')?.message;
|
|
1547
|
-
throw new Error(semanticError ?? `
|
|
2147
|
+
throw new Error(semanticError ?? `DQL artifact "${metadata.name ?? 'draft'}" produced no executable SQL.`);
|
|
1548
2148
|
}
|
|
1549
2149
|
const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, activeConnection, projectRoot, projectConfig);
|
|
1550
2150
|
const app = loadRuntimeApp(projectRoot, activePersonaAppId());
|
|
1551
|
-
|
|
1552
|
-
|
|
2151
|
+
const sourceDomain = metadata.domain ?? source.match(/\bdomain\s*=\s*"([^"]+)"/i)?.[1];
|
|
2152
|
+
assertAppAccess({ app, domain: sourceDomain ?? app?.domain, level: 'execute' });
|
|
2153
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...invocation.values }), prepared.connection);
|
|
1553
2154
|
const normalized = normalizeQueryResult(rawResult);
|
|
1554
2155
|
return {
|
|
1555
2156
|
columns: normalized.columns,
|
|
1556
2157
|
rows: normalized.rows,
|
|
1557
2158
|
rowCount: normalized.rowCount,
|
|
1558
2159
|
executionTime: normalized.executionTime,
|
|
1559
|
-
chartConfig: plan?.chartConfig ?? (
|
|
2160
|
+
chartConfig: plan?.chartConfig ?? (metadata.chartType ? { chart: metadata.chartType } : undefined),
|
|
1560
2161
|
sql: prepared.sql,
|
|
1561
|
-
blockName:
|
|
1562
|
-
blockPath:
|
|
2162
|
+
...(metadata.name ? { blockName: metadata.name } : {}),
|
|
2163
|
+
...(metadata.path ? { blockPath: metadata.path } : {}),
|
|
2164
|
+
parameters: invocation.resolvedParameters,
|
|
2165
|
+
auditId: invocation.auditId,
|
|
1563
2166
|
};
|
|
1564
2167
|
};
|
|
2168
|
+
const executeCertifiedBlockByNameForAgent = async (blockName, invocationInput, requireCertified = false) => {
|
|
2169
|
+
const manifest = buildManifest({ projectRoot });
|
|
2170
|
+
const block = manifest.blocks[blockName];
|
|
2171
|
+
if (!block) {
|
|
2172
|
+
throw new Error(`Matched block "${blockName}" is not present in the project manifest.`);
|
|
2173
|
+
}
|
|
2174
|
+
if (requireCertified && block.status !== 'certified') {
|
|
2175
|
+
throw new Error(`Block "${block.name}" is not certified and cannot be rerun as a certified answer.`);
|
|
2176
|
+
}
|
|
2177
|
+
const source = readFileSync(join(projectRoot, block.filePath), 'utf-8');
|
|
2178
|
+
return executeDqlArtifactSourceForAgent(source, {
|
|
2179
|
+
name: block.name,
|
|
2180
|
+
path: block.filePath,
|
|
2181
|
+
domain: block.domain,
|
|
2182
|
+
chartType: block.chartType,
|
|
2183
|
+
}, invocationInput);
|
|
2184
|
+
};
|
|
2185
|
+
const executeCertifiedBlockForAgent = async (node, invocationInput) => {
|
|
2186
|
+
if (node.kind !== 'block') {
|
|
2187
|
+
throw new Error(`Certified ${node.kind} "${node.name}" is a navigation artifact and cannot be executed as a block.`);
|
|
2188
|
+
}
|
|
2189
|
+
return executeCertifiedBlockByNameForAgent(node.name || node.nodeId.replace(/^block:/, ''), invocationInput);
|
|
2190
|
+
};
|
|
1565
2191
|
const executeGeneratedSqlForAgent = async (sql) => {
|
|
1566
2192
|
const activeConnection = requireActiveConnection();
|
|
1567
2193
|
const boundedSql = buildAgentPreviewSql(sql);
|
|
@@ -1582,6 +2208,62 @@ export async function startLocalServer(opts) {
|
|
|
1582
2208
|
sql: prepared.sql,
|
|
1583
2209
|
};
|
|
1584
2210
|
};
|
|
2211
|
+
/**
|
|
2212
|
+
* EXP-001: execution host for the deliberately narrow non-governed lane.
|
|
2213
|
+
* It does not reinterpret dbt lineage or shared names as relationship proof.
|
|
2214
|
+
* Instead it validates the generated equality predicates against bounded
|
|
2215
|
+
* samples, records the result as exploratory evidence, then executes the
|
|
2216
|
+
* existing read-only 200-row preview. Any malformed, broad, or unsupported
|
|
2217
|
+
* candidate remains review-required and is not executed automatically.
|
|
2218
|
+
*/
|
|
2219
|
+
const executeExploratoryCandidate = async (candidate, schemaContext = [], question = '', requestedTopN) => {
|
|
2220
|
+
const preflight = repairExploratorySqlBeforeExecution(candidate.sql, schemaContext, question);
|
|
2221
|
+
const boundedSql = applyRequestedTopNToExploratorySql(preflight.sql, requestedTopN);
|
|
2222
|
+
const repairs = boundedSql === preflight.sql
|
|
2223
|
+
? preflight.repairs
|
|
2224
|
+
: [...preflight.repairs, `Applied the requested overall top-${requestedTopN} bound before exploratory execution.`];
|
|
2225
|
+
if (preflight.blockedReason) {
|
|
2226
|
+
return {
|
|
2227
|
+
proofs: [],
|
|
2228
|
+
sql: boundedSql,
|
|
2229
|
+
repairs,
|
|
2230
|
+
error: preflight.blockedReason,
|
|
2231
|
+
};
|
|
2232
|
+
}
|
|
2233
|
+
const analysis = analyzeSqlReferences(boundedSql);
|
|
2234
|
+
if (!analysis.parsed) {
|
|
2235
|
+
return { proofs: [], sql: boundedSql, repairs, error: 'DQL could not parse the exploratory SQL to validate its join predicates.' };
|
|
2236
|
+
}
|
|
2237
|
+
// A normal customer → order → order-item → product question needs three
|
|
2238
|
+
// joins. Keep the lane bounded, but do not reject this common dbt-star path
|
|
2239
|
+
// solely because the old two-join demo limit was too small.
|
|
2240
|
+
if (analysis.joins.length > 4) {
|
|
2241
|
+
return { proofs: [], sql: boundedSql, repairs, error: 'This exploratory query has more than four joins and requires an analyst to review the relationship path.' };
|
|
2242
|
+
}
|
|
2243
|
+
if (analysis.joins.some((join) => !join.leftRelation || !join.rightRelation)) {
|
|
2244
|
+
return { proofs: [], sql: boundedSql, repairs, error: 'This exploratory query has a join endpoint DQL could not resolve for bounded validation.' };
|
|
2245
|
+
}
|
|
2246
|
+
const activeConnection = requireActiveConnection();
|
|
2247
|
+
const proofs = [];
|
|
2248
|
+
try {
|
|
2249
|
+
for (const join of analysis.joins) {
|
|
2250
|
+
const proofSql = buildExploratoryJoinProbeSql({
|
|
2251
|
+
leftRelation: join.leftRelation,
|
|
2252
|
+
leftColumn: join.leftColumn,
|
|
2253
|
+
rightRelation: join.rightRelation,
|
|
2254
|
+
rightColumn: join.rightColumn,
|
|
2255
|
+
});
|
|
2256
|
+
const prepared = prepareLocalExecution(proofSql, activeConnection, projectRoot, projectConfig);
|
|
2257
|
+
const raw = await executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
|
|
2258
|
+
proofs.push({ summary: summarizeExploratoryJoinProbe(raw.rows, join.leftRelation, join.leftColumn, join.rightRelation, join.rightColumn) });
|
|
2259
|
+
}
|
|
2260
|
+
const result = await executeGeneratedSqlForAgent(boundedSql);
|
|
2261
|
+
return { result, proofs, sql: boundedSql, repairs };
|
|
2262
|
+
}
|
|
2263
|
+
catch (error) {
|
|
2264
|
+
return { proofs, sql: boundedSql, repairs, error: error instanceof Error ? error.message : String(error) };
|
|
2265
|
+
}
|
|
2266
|
+
};
|
|
1585
2267
|
const getSchemaContextForAgent = async (question, preparedContextPack) => {
|
|
1586
2268
|
const scanRuntimeSchema = async () => {
|
|
1587
2269
|
if (!connection)
|
|
@@ -1603,7 +2285,7 @@ export async function startLocalServer(opts) {
|
|
|
1603
2285
|
// Rescan live when the question shape calls for it OR the stored snapshot is
|
|
1604
2286
|
// stale/absent (P6) — otherwise a warehouse schema change between sessions is
|
|
1605
2287
|
// silently reasoned over from a cached snapshot that never expires.
|
|
1606
|
-
const runtimeScan = (shouldAugmentAgentRuntimeSchema(question) || runtimeSnapshotStale(projectRoot))
|
|
2288
|
+
const runtimeScan = (shouldAugmentAgentRuntimeSchema(question, preparedContextPack?.questionPlan) || runtimeSnapshotStale(projectRoot))
|
|
1607
2289
|
? await scanRuntimeSchema().catch(() => undefined)
|
|
1608
2290
|
: undefined;
|
|
1609
2291
|
const runtimeContext = runtimeScan?.ranked ?? [];
|
|
@@ -2286,14 +2968,16 @@ export async function startLocalServer(opts) {
|
|
|
2286
2968
|
});
|
|
2287
2969
|
// SSE clients for /api/watch hot-reload
|
|
2288
2970
|
const sseClients = new Set();
|
|
2289
|
-
// Watch
|
|
2971
|
+
// Watch all Git-owned authoring roots. Domain Packages are recursive and are
|
|
2972
|
+
// the canonical home for modeling, skills, blocks, notebooks, Apps, and evals.
|
|
2290
2973
|
if (projectRoot) {
|
|
2291
|
-
|
|
2974
|
+
let sourceRefreshTimer;
|
|
2975
|
+
for (const dir of ['domains', 'skills', 'tests', 'notebooks', 'workbooks', 'blocks', 'apps', 'dashboards', 'semantic-layer', 'data']) {
|
|
2292
2976
|
const watchDir = join(projectRoot, dir);
|
|
2293
2977
|
if (!existsSync(watchDir))
|
|
2294
2978
|
continue;
|
|
2295
2979
|
try {
|
|
2296
|
-
watch(watchDir, { persistent: false }, (eventType, filename) => {
|
|
2980
|
+
watch(watchDir, { persistent: false, recursive: true }, (eventType, filename) => {
|
|
2297
2981
|
if (!filename)
|
|
2298
2982
|
return;
|
|
2299
2983
|
const path = `${dir}/${filename}`;
|
|
@@ -2306,6 +2990,14 @@ export async function startLocalServer(opts) {
|
|
|
2306
2990
|
sseClients.delete(client);
|
|
2307
2991
|
}
|
|
2308
2992
|
}
|
|
2993
|
+
if (['domains', 'skills', 'tests', 'blocks', 'apps', 'notebooks'].includes(dir)) {
|
|
2994
|
+
invalidateAgentProjectState(projectRoot);
|
|
2995
|
+
if (sourceRefreshTimer)
|
|
2996
|
+
clearTimeout(sourceRefreshTimer);
|
|
2997
|
+
sourceRefreshTimer = setTimeout(() => {
|
|
2998
|
+
void refreshUnifiedProjectIndexes(projectRoot).catch(() => undefined);
|
|
2999
|
+
}, 250);
|
|
3000
|
+
}
|
|
2309
3001
|
// Hot-reload semantic layer on change and notify frontend
|
|
2310
3002
|
if (dir === 'semantic-layer') {
|
|
2311
3003
|
const semanticConnection = connection;
|
|
@@ -2338,6 +3030,31 @@ export async function startLocalServer(opts) {
|
|
|
2338
3030
|
}
|
|
2339
3031
|
catch { /* dir not watchable */ }
|
|
2340
3032
|
}
|
|
3033
|
+
const configuredDbtManifest = resolveDbtManifestPath(projectRoot);
|
|
3034
|
+
if (configuredDbtManifest && existsSync(dirname(configuredDbtManifest))) {
|
|
3035
|
+
try {
|
|
3036
|
+
watch(dirname(configuredDbtManifest), { persistent: false }, (_eventType, filename) => {
|
|
3037
|
+
if (!filename || !['manifest.json', 'catalog.json', 'semantic_manifest.json', 'run_results.json'].includes(String(filename)))
|
|
3038
|
+
return;
|
|
3039
|
+
invalidateAgentProjectState(projectRoot);
|
|
3040
|
+
if (sourceRefreshTimer)
|
|
3041
|
+
clearTimeout(sourceRefreshTimer);
|
|
3042
|
+
sourceRefreshTimer = setTimeout(() => {
|
|
3043
|
+
void refreshUnifiedProjectIndexes(projectRoot).catch(() => undefined);
|
|
3044
|
+
}, 250);
|
|
3045
|
+
const payload = JSON.stringify({ type: 'dbt-artifacts-changed', path: String(filename) });
|
|
3046
|
+
for (const client of sseClients) {
|
|
3047
|
+
try {
|
|
3048
|
+
client.write(`event: change\ndata: ${payload}\n\n`);
|
|
3049
|
+
}
|
|
3050
|
+
catch {
|
|
3051
|
+
sseClients.delete(client);
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
});
|
|
3055
|
+
}
|
|
3056
|
+
catch { /* dbt artifact directory not watchable */ }
|
|
3057
|
+
}
|
|
2341
3058
|
}
|
|
2342
3059
|
const validateImportCandidate = (candidate) => ({
|
|
2343
3060
|
...candidate,
|
|
@@ -2374,7 +3091,7 @@ export async function startLocalServer(opts) {
|
|
|
2374
3091
|
return null;
|
|
2375
3092
|
return `Likely duplicate of ${match.name} (${match.kind}, ${Math.round(match.score * 100)}% match). Reuse the existing block, extend it, or document this draft as a replacement before certification.`;
|
|
2376
3093
|
};
|
|
2377
|
-
const runBlockStudioPreviewSource = async (source, targetConnection) => {
|
|
3094
|
+
const runBlockStudioPreviewSource = async (source, targetConnection, parameters = {}) => {
|
|
2378
3095
|
const activeConnection = requireActiveConnection(targetConnection);
|
|
2379
3096
|
let tableMapping;
|
|
2380
3097
|
if (semanticLayer) {
|
|
@@ -2388,6 +3105,12 @@ export async function startLocalServer(opts) {
|
|
|
2388
3105
|
tableMapping = undefined;
|
|
2389
3106
|
}
|
|
2390
3107
|
}
|
|
3108
|
+
const invocation = prepareBlockInvocation({ source, parameters, surface: 'block_studio' });
|
|
3109
|
+
if (invocation.errors.length > 0)
|
|
3110
|
+
throw new Error(invocation.errors.join(' '));
|
|
3111
|
+
if (invocation.unresolvedParameters.length > 0) {
|
|
3112
|
+
throw new Error(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'}: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
3113
|
+
}
|
|
2391
3114
|
const semanticCompose = semanticLayer
|
|
2392
3115
|
? composeSemanticBlockSql(source, semanticLayer, {
|
|
2393
3116
|
driver: activeConnection.driver,
|
|
@@ -2395,6 +3118,7 @@ export async function startLocalServer(opts) {
|
|
|
2395
3118
|
projectRoot,
|
|
2396
3119
|
projectConfig,
|
|
2397
3120
|
detectedProvider: semanticDetectedProvider,
|
|
3121
|
+
parameters: invocation.values,
|
|
2398
3122
|
})
|
|
2399
3123
|
: null;
|
|
2400
3124
|
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
@@ -2407,11 +3131,12 @@ export async function startLocalServer(opts) {
|
|
|
2407
3131
|
}
|
|
2408
3132
|
const plan = buildExecutionPlan({ id: 'block-studio', type: 'dql', source, title: 'Block Studio' }, { semanticLayer, driver: activeConnection.driver, tableMapping });
|
|
2409
3133
|
const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan?.sql ?? executableSql, activeConnection, projectRoot, projectConfig);
|
|
2410
|
-
const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables(plan?.variables ?? {}), prepared.connection);
|
|
3134
|
+
const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...invocation.values }), prepared.connection);
|
|
2411
3135
|
return {
|
|
2412
3136
|
sql: prepared.sql,
|
|
2413
3137
|
result: normalizeQueryResult(result),
|
|
2414
3138
|
chartConfig: plan?.chartConfig ?? validation.chartConfig ?? null,
|
|
3139
|
+
invocation,
|
|
2415
3140
|
};
|
|
2416
3141
|
};
|
|
2417
3142
|
const runBlockStudioTestSummary = async (source, targetConnection) => {
|
|
@@ -2630,18 +3355,595 @@ export async function startLocalServer(opts) {
|
|
|
2630
3355
|
const requestUrl = req.url || '/';
|
|
2631
3356
|
const url = new URL(requestUrl, 'http://127.0.0.1');
|
|
2632
3357
|
const path = url.pathname || '/';
|
|
2633
|
-
// CORS
|
|
2634
|
-
|
|
3358
|
+
// Exact-origin CORS. Loopback dev origins are accepted; remote serving
|
|
3359
|
+
// requires an explicit allowlist and bearer token.
|
|
3360
|
+
const requestOrigin = typeof req.headers.origin === 'string' ? req.headers.origin.replace(/\/$/, '') : undefined;
|
|
3361
|
+
const loopbackOrigin = requestOrigin ? isLoopbackOrigin(requestOrigin) : false;
|
|
3362
|
+
const originAllowed = !requestOrigin || (loopback ? loopbackOrigin : allowedOrigins.has(requestOrigin));
|
|
3363
|
+
if (requestOrigin && originAllowed)
|
|
3364
|
+
res.setHeader('Access-Control-Allow-Origin', requestOrigin);
|
|
3365
|
+
res.setHeader('Vary', 'Origin');
|
|
2635
3366
|
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
|
|
2636
|
-
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
|
3367
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
|
3368
|
+
if (!originAllowed && path.startsWith('/api/')) {
|
|
3369
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3370
|
+
res.end(serializeJSON({ error: 'Origin is not allowed.' }));
|
|
3371
|
+
return;
|
|
3372
|
+
}
|
|
2637
3373
|
if (req.method === 'OPTIONS') {
|
|
2638
3374
|
res.writeHead(204);
|
|
2639
3375
|
res.end();
|
|
2640
3376
|
return;
|
|
2641
3377
|
}
|
|
2642
|
-
if (
|
|
2643
|
-
|
|
2644
|
-
|
|
3378
|
+
if (!loopback && path.startsWith('/api/') && path !== '/api/health') {
|
|
3379
|
+
const authorization = req.headers.authorization ?? '';
|
|
3380
|
+
if (authorization !== `Bearer ${authToken}`) {
|
|
3381
|
+
res.writeHead(401, { 'Content-Type': 'application/json; charset=utf-8', 'WWW-Authenticate': 'Bearer' });
|
|
3382
|
+
res.end(serializeJSON({ error: 'A valid DQL server bearer token is required.' }));
|
|
3383
|
+
return;
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3386
|
+
if (req.method === 'GET' && path === '/api/health') {
|
|
3387
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3388
|
+
res.end(serializeJSON({ status: 'ok' }));
|
|
3389
|
+
return;
|
|
3390
|
+
}
|
|
3391
|
+
if (req.method === 'GET' && path === '/api/onboarding/status') {
|
|
3392
|
+
const requestId = apiRequestId('onboarding-status');
|
|
3393
|
+
const dbtProjectDir = resolve(projectRoot, projectConfig.dbt?.projectDir ?? '.');
|
|
3394
|
+
const manifestPath = resolve(dbtProjectDir, projectConfig.dbt?.manifestPath ?? 'target/manifest.json');
|
|
3395
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
3396
|
+
const manifestFound = existsSync(manifestPath);
|
|
3397
|
+
let snapshotId;
|
|
3398
|
+
let snapshotError;
|
|
3399
|
+
if (manifestFound) {
|
|
3400
|
+
try {
|
|
3401
|
+
snapshotId = projectSnapshot().snapshotId;
|
|
3402
|
+
}
|
|
3403
|
+
catch (error) {
|
|
3404
|
+
snapshotError = error instanceof Error ? error.message : String(error);
|
|
3405
|
+
}
|
|
3406
|
+
}
|
|
3407
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3408
|
+
res.end(serializeJSON({
|
|
3409
|
+
requestId,
|
|
3410
|
+
snapshotId,
|
|
3411
|
+
dbt: {
|
|
3412
|
+
configured: Boolean(projectConfig.dbt),
|
|
3413
|
+
projectDir: projectConfig.dbt?.projectDir,
|
|
3414
|
+
manifestPath: projectConfig.dbt?.manifestPath,
|
|
3415
|
+
projectFound: existsSync(join(dbtProjectDir, 'dbt_project.yml')),
|
|
3416
|
+
manifestFound,
|
|
3417
|
+
},
|
|
3418
|
+
modeling: {
|
|
3419
|
+
enabled: projectConfig.manifestVersion === 3 && projectConfig.modeling?.mode === 'dbt-first',
|
|
3420
|
+
manifestVersion: projectConfig.manifestVersion ?? 2,
|
|
3421
|
+
mode: projectConfig.modeling?.mode,
|
|
3422
|
+
},
|
|
3423
|
+
domains: { count: registry.values().length, diagnostics: registry.diagnostics },
|
|
3424
|
+
snapshot: { id: snapshotId, error: snapshotError },
|
|
3425
|
+
capabilities: {
|
|
3426
|
+
warehouse: Boolean(connection),
|
|
3427
|
+
ai: Boolean(process.env.OPENAI_API_KEY || process.env.ANTHROPIC_API_KEY || process.env.GEMINI_API_KEY || process.env.OLLAMA_BASE_URL),
|
|
3428
|
+
},
|
|
3429
|
+
}));
|
|
3430
|
+
return;
|
|
3431
|
+
}
|
|
3432
|
+
if (req.method === 'POST' && path === '/api/onboarding/dbt/preview') {
|
|
3433
|
+
const requestId = apiRequestId('onboarding-dbt-preview');
|
|
3434
|
+
try {
|
|
3435
|
+
const preview = previewDbtOnboarding(await readJSON(req));
|
|
3436
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3437
|
+
res.end(serializeJSON({ requestId, ...preview }));
|
|
3438
|
+
}
|
|
3439
|
+
catch (error) {
|
|
3440
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
3441
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3442
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Verify the dbt project and artifact paths, then preview again.'] })));
|
|
3443
|
+
}
|
|
3444
|
+
return;
|
|
3445
|
+
}
|
|
3446
|
+
if (req.method === 'POST' && path === '/api/onboarding/dbt/apply') {
|
|
3447
|
+
const requestId = apiRequestId('onboarding-dbt-apply');
|
|
3448
|
+
try {
|
|
3449
|
+
const body = await readJSON(req);
|
|
3450
|
+
let preview;
|
|
3451
|
+
try {
|
|
3452
|
+
preview = previewDbtOnboarding(body);
|
|
3453
|
+
}
|
|
3454
|
+
catch (error) {
|
|
3455
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : '';
|
|
3456
|
+
if (code !== 'DBT_MANIFEST_MISSING' || body.buildArtifacts !== true)
|
|
3457
|
+
throw error;
|
|
3458
|
+
const { dbtProjectDir } = onboardingDbtPaths(body);
|
|
3459
|
+
try {
|
|
3460
|
+
execFileSync('dbt', ['parse', '--project-dir', dbtProjectDir], {
|
|
3461
|
+
cwd: dbtProjectDir,
|
|
3462
|
+
timeout: 120_000,
|
|
3463
|
+
maxBuffer: 1024 * 1024,
|
|
3464
|
+
encoding: 'utf8',
|
|
3465
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
3466
|
+
env: { ...process.env, DBT_LOG_FORMAT: 'text' },
|
|
3467
|
+
});
|
|
3468
|
+
}
|
|
3469
|
+
catch (parseError) {
|
|
3470
|
+
throw Object.assign(new Error(`dbt parse failed. Run it in ${dbtProjectDir} to inspect the project error, then retry.`), { code: 'DBT_PARSE_FAILED', cause: parseError });
|
|
3471
|
+
}
|
|
3472
|
+
preview = previewDbtOnboarding(body);
|
|
3473
|
+
}
|
|
3474
|
+
const expectedFingerprint = typeof body.fingerprint === 'string'
|
|
3475
|
+
? body.fingerprint
|
|
3476
|
+
: typeof body.expectedFingerprint === 'string' ? body.expectedFingerprint : undefined;
|
|
3477
|
+
if ((!expectedFingerprint && body.buildArtifacts !== true) || (expectedFingerprint && expectedFingerprint !== preview.fingerprint)) {
|
|
3478
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3479
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed after preview. Review the refreshed preview before applying.', nextActions: ['Refresh the preview and review the new diff.'] })));
|
|
3480
|
+
return;
|
|
3481
|
+
}
|
|
3482
|
+
const nextConfig = {
|
|
3483
|
+
...projectConfig,
|
|
3484
|
+
manifestVersion: 3,
|
|
3485
|
+
modeling: { mode: 'dbt-first' },
|
|
3486
|
+
dbt: { projectDir: preview.projectDir, manifestPath: preview.manifestPath },
|
|
3487
|
+
semanticLayer: { provider: 'dbt', projectPath: preview.projectDir },
|
|
3488
|
+
};
|
|
3489
|
+
const configPath = join(projectRoot, 'dql.config.json');
|
|
3490
|
+
const previousConfigSource = readFileSync(configPath, 'utf8');
|
|
3491
|
+
const previousProjectConfig = projectConfig;
|
|
3492
|
+
const tempPath = `${configPath}.tmp-${process.pid}`;
|
|
3493
|
+
writeFileSync(tempPath, `${JSON.stringify(nextConfig, null, 2)}\n`, { encoding: 'utf8', mode: 0o600 });
|
|
3494
|
+
renameSync(tempPath, configPath);
|
|
3495
|
+
projectConfig = nextConfig;
|
|
3496
|
+
projectSnapshots.invalidate();
|
|
3497
|
+
invalidateAgentProjectState(projectRoot);
|
|
3498
|
+
let snapshotId;
|
|
3499
|
+
try {
|
|
3500
|
+
snapshotId = projectSnapshot().snapshotId;
|
|
3501
|
+
}
|
|
3502
|
+
catch (error) {
|
|
3503
|
+
const rollbackPath = `${configPath}.rollback-${process.pid}`;
|
|
3504
|
+
writeFileSync(rollbackPath, previousConfigSource, { encoding: 'utf8', mode: 0o600 });
|
|
3505
|
+
renameSync(rollbackPath, configPath);
|
|
3506
|
+
projectConfig = previousProjectConfig;
|
|
3507
|
+
projectSnapshots.invalidate();
|
|
3508
|
+
invalidateAgentProjectState(projectRoot);
|
|
3509
|
+
throw Object.assign(new Error(`dbt-first configuration did not compile and was rolled back: ${error instanceof Error ? error.message : String(error)}`), { code: 'SNAPSHOT_BUILD_FAILED' });
|
|
3510
|
+
}
|
|
3511
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3512
|
+
res.end(serializeJSON({ requestId, snapshotId, applied: true, config: { manifestVersion: 3, modeling: nextConfig.modeling, dbt: nextConfig.dbt }, fingerprint: preview.fingerprint }));
|
|
3513
|
+
}
|
|
3514
|
+
catch (error) {
|
|
3515
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
3516
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3517
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Fix the reported dbt issue and run preview again.'] })));
|
|
3518
|
+
}
|
|
3519
|
+
return;
|
|
3520
|
+
}
|
|
3521
|
+
if (req.method === 'POST' && path === '/api/onboarding/refresh') {
|
|
3522
|
+
const requestId = apiRequestId('onboarding-refresh');
|
|
3523
|
+
const id = `dbt-refresh-${Date.now().toString(36)}`;
|
|
3524
|
+
try {
|
|
3525
|
+
const body = await readJSON(req);
|
|
3526
|
+
const currentArtifact = previewDbtOnboarding({});
|
|
3527
|
+
if (!body.expectedFingerprint || body.expectedFingerprint !== currentArtifact.fingerprint) {
|
|
3528
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3529
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code: 'SOURCE_CHANGED', message: 'dbt artifacts changed before refresh. Review the current artifact preview.', nextActions: ['Return to the dbt preview step and review the refreshed fingerprint.'] })));
|
|
3530
|
+
return;
|
|
3531
|
+
}
|
|
3532
|
+
projectSnapshots.invalidate();
|
|
3533
|
+
const snapshot = projectSnapshot();
|
|
3534
|
+
invalidateAgentProjectState(projectRoot);
|
|
3535
|
+
await reindexProject(projectRoot, { manifest: snapshot.manifest, kgPath: defaultKgPath(projectRoot) });
|
|
3536
|
+
const job = { id, kind: 'dbt_refresh', status: 'completed', createdAt: new Date().toISOString(), result: { snapshotId: snapshot.snapshotId, diagnostics: snapshot.manifest.diagnostics ?? [] } };
|
|
3537
|
+
onboardingJobs.set(id, job);
|
|
3538
|
+
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3539
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, jobId: id, ...job }));
|
|
3540
|
+
}
|
|
3541
|
+
catch (error) {
|
|
3542
|
+
const job = { id, kind: 'dbt_refresh', status: 'failed', createdAt: new Date().toISOString(), error: error instanceof Error ? error.message : String(error) };
|
|
3543
|
+
onboardingJobs.set(id, job);
|
|
3544
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3545
|
+
res.end(serializeJSON({ ...apiErrorEnvelope({ requestId, code: 'SNAPSHOT_BUILD_FAILED', message: job.error, nextActions: ['Keep using the previous snapshot while fixing compile diagnostics.'] }), jobId: id, ...job }));
|
|
3546
|
+
}
|
|
3547
|
+
return;
|
|
3548
|
+
}
|
|
3549
|
+
if (req.method === 'GET' && path.startsWith('/api/onboarding/jobs/')) {
|
|
3550
|
+
const requestId = apiRequestId('onboarding-job');
|
|
3551
|
+
const id = decodeURIComponent(path.slice('/api/onboarding/jobs/'.length));
|
|
3552
|
+
const job = onboardingJobs.get(id);
|
|
3553
|
+
res.writeHead(job ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3554
|
+
const jobSnapshotId = typeof job?.result?.snapshotId === 'string'
|
|
3555
|
+
? String((job?.result).snapshotId)
|
|
3556
|
+
: undefined;
|
|
3557
|
+
res.end(serializeJSON(job ? { requestId, ...(jobSnapshotId ? { snapshotId: jobSnapshotId } : {}), ...job } : apiErrorEnvelope({ requestId, code: 'ONBOARDING_JOB_NOT_FOUND', message: `onboarding job not found: ${id}`, recoverable: false })));
|
|
3558
|
+
return;
|
|
3559
|
+
}
|
|
3560
|
+
if (req.method === 'DELETE' && path.startsWith('/api/onboarding/jobs/')) {
|
|
3561
|
+
const requestId = apiRequestId('onboarding-job-cancel');
|
|
3562
|
+
const id = decodeURIComponent(path.slice('/api/onboarding/jobs/'.length));
|
|
3563
|
+
const job = onboardingJobs.get(id);
|
|
3564
|
+
if (job)
|
|
3565
|
+
onboardingJobs.set(id, { ...job, status: 'cancelled' });
|
|
3566
|
+
res.writeHead(job ? 200 : 404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3567
|
+
res.end(serializeJSON(job ? { requestId, ...job, status: 'cancelled' } : apiErrorEnvelope({ requestId, code: 'ONBOARDING_JOB_NOT_FOUND', message: `onboarding job not found: ${id}`, recoverable: false })));
|
|
3568
|
+
return;
|
|
3569
|
+
}
|
|
3570
|
+
if (req.method === 'POST' && path === '/api/onboarding/domains/discover') {
|
|
3571
|
+
const requestId = apiRequestId('domain-discovery');
|
|
3572
|
+
try {
|
|
3573
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3574
|
+
if (!dbtManifestPath || !existsSync(dbtManifestPath))
|
|
3575
|
+
throw Object.assign(new Error('No dbt manifest found. Run dbt parse or connect the project first.'), { code: 'DBT_MANIFEST_MISSING' });
|
|
3576
|
+
const report = discoverDbtDomains({ projectRoot, dbtManifestPath });
|
|
3577
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3578
|
+
res.end(serializeJSON({
|
|
3579
|
+
requestId,
|
|
3580
|
+
snapshotId: projectSnapshot().snapshotId,
|
|
3581
|
+
...report,
|
|
3582
|
+
capabilities: { aiEnrichment: false, deterministic: true },
|
|
3583
|
+
}));
|
|
3584
|
+
}
|
|
3585
|
+
catch (error) {
|
|
3586
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DBT_ARTIFACT_INVALID';
|
|
3587
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3588
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Build a valid dbt manifest and retry discovery.'] })));
|
|
3589
|
+
}
|
|
3590
|
+
return;
|
|
3591
|
+
}
|
|
3592
|
+
if (req.method === 'POST' && path === '/api/onboarding/domains/apply') {
|
|
3593
|
+
const requestId = apiRequestId('domain-apply');
|
|
3594
|
+
try {
|
|
3595
|
+
const body = await readJSON(req);
|
|
3596
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3597
|
+
if (!dbtManifestPath || !existsSync(dbtManifestPath))
|
|
3598
|
+
throw Object.assign(new Error('No dbt manifest found.'), { code: 'DBT_MANIFEST_MISSING' });
|
|
3599
|
+
const report = discoverDbtDomains({ projectRoot, dbtManifestPath });
|
|
3600
|
+
const expectedFingerprint = body.sourceFingerprint ?? body.expectedSourceFingerprint;
|
|
3601
|
+
if (!expectedFingerprint || expectedFingerprint !== report.sourceFingerprint) {
|
|
3602
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3603
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: projectSnapshot().snapshotId, code: 'SOURCE_CHANGED', message: 'dbt/domain evidence changed after discovery. Review the refreshed proposals before applying.', nextActions: ['Run domain discovery again and review the updated evidence.'] })));
|
|
3604
|
+
return;
|
|
3605
|
+
}
|
|
3606
|
+
const proposalIds = Array.isArray(body.proposals) ? body.proposals.flatMap((value) => typeof value === 'object' && value && 'id' in value && typeof value.id === 'string' ? [value.id] : []) : [];
|
|
3607
|
+
const requested = proposalIds.length > 0 ? proposalIds : Array.isArray(body.selectedDomains) ? body.selectedDomains : Array.isArray(body.domains) ? body.domains : report.proposals.map((proposal) => proposal.id);
|
|
3608
|
+
const selected = new Set(requested.filter((value) => typeof value === 'string'));
|
|
3609
|
+
if (body.mode === 'preview') {
|
|
3610
|
+
const preview = report.proposals.filter((proposal) => selected.has(proposal.id)).map((proposal) => ({ path: join('domains', ...proposal.id.split('.'), 'domain.dql').replace(/\\/g, '/'), operation: 'create_or_retain', summary: `${proposal.matchedDbtUniqueIds.length} dbt model(s); review required` }));
|
|
3611
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3612
|
+
res.end(serializeJSON({ requestId, snapshotId: projectSnapshot().snapshotId, applied: false, preview }));
|
|
3613
|
+
return;
|
|
3614
|
+
}
|
|
3615
|
+
const existing = loadDomainPackageRegistry(projectRoot);
|
|
3616
|
+
const applied = [];
|
|
3617
|
+
for (const proposal of report.proposals.filter((value) => selected.has(value.id))) {
|
|
3618
|
+
if (existing.get(proposal.id)) {
|
|
3619
|
+
applied.push({ id: proposal.id, status: 'existing', message: 'Existing Domain Package retained unchanged.' });
|
|
3620
|
+
continue;
|
|
3621
|
+
}
|
|
3622
|
+
const segments = proposal.id.split('.');
|
|
3623
|
+
if (segments.length === 0 || segments.some((segment) => !/^[a-z0-9_]+$/.test(segment))) {
|
|
3624
|
+
applied.push({ id: proposal.id, status: 'blocked', message: 'Domain id is not a safe normalized path.' });
|
|
3625
|
+
continue;
|
|
3626
|
+
}
|
|
3627
|
+
const sourcePath = join('domains', ...segments, 'domain.dql').replace(/\\/g, '/');
|
|
3628
|
+
const absolute = join(projectRoot, sourcePath);
|
|
3629
|
+
mkdirSync(dirname(absolute), { recursive: true });
|
|
3630
|
+
const dbtPaths = [...new Set(report.memberships.filter((membership) => membership.proposedDomain === proposal.id).flatMap((membership) => membership.sourcePath ? [membership.sourcePath] : []))].sort();
|
|
3631
|
+
const source = renderDomainDeclaration({
|
|
3632
|
+
id: proposal.id,
|
|
3633
|
+
name: proposal.name,
|
|
3634
|
+
parent: proposal.proposedParent,
|
|
3635
|
+
owner: proposal.owner,
|
|
3636
|
+
dbtPaths,
|
|
3637
|
+
description: 'Draft domain boundary proposed from cited dbt evidence; review before governance use.',
|
|
3638
|
+
});
|
|
3639
|
+
const temp = `${absolute}.tmp-${process.pid}`;
|
|
3640
|
+
writeFileSync(temp, source, { encoding: 'utf8', mode: 0o600 });
|
|
3641
|
+
renameSync(temp, absolute);
|
|
3642
|
+
applied.push({ id: proposal.id, status: 'created', path: sourcePath, message: 'Review-required domain boundary created; no relationships or skills were certified.' });
|
|
3643
|
+
}
|
|
3644
|
+
projectSnapshots.invalidate();
|
|
3645
|
+
invalidateAgentProjectState(projectRoot);
|
|
3646
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3647
|
+
res.end(serializeJSON({ requestId, snapshotId: projectSnapshot().snapshotId, applied: true, results: applied, domains: loadDomainPackageRegistry(projectRoot).values().map((pkg) => ({ id: pkg.id, filePath: pkg.declarationPath, lifecycle: 'draft' })) }));
|
|
3648
|
+
}
|
|
3649
|
+
catch (error) {
|
|
3650
|
+
const code = typeof error === 'object' && error && 'code' in error ? String(error.code) : 'DOMAIN_COLLISION';
|
|
3651
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3652
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, code, message: apiErrorMessage(error), nextActions: ['Resolve the collision or invalid proposal and preview again.'] })));
|
|
3653
|
+
}
|
|
3654
|
+
return;
|
|
3655
|
+
}
|
|
3656
|
+
if (req.method === 'GET' && path === '/api/domain-packages') {
|
|
3657
|
+
const snapshot = projectSnapshot();
|
|
3658
|
+
const manifest = snapshot.manifest;
|
|
3659
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
3660
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3661
|
+
res.end(serializeJSON({
|
|
3662
|
+
domains: listDomains(projectRoot),
|
|
3663
|
+
packages: registry.values().map((pkg) => ({
|
|
3664
|
+
id: pkg.id,
|
|
3665
|
+
name: pkg.name,
|
|
3666
|
+
parent: pkg.parent,
|
|
3667
|
+
depth: pkg.depth,
|
|
3668
|
+
ancestry: pkg.ancestry,
|
|
3669
|
+
declarationPath: pkg.declarationPath,
|
|
3670
|
+
root: relative(projectRoot, pkg.root).replace(/\\/g, '/'),
|
|
3671
|
+
owner: pkg.owner,
|
|
3672
|
+
exports: pkg.exports,
|
|
3673
|
+
})),
|
|
3674
|
+
modeling: manifest.modeling,
|
|
3675
|
+
dbtProvenance: manifest.dbtProvenance,
|
|
3676
|
+
domainAssets: collectDomainPackageAssets(projectRoot, registry),
|
|
3677
|
+
diagnostics: manifest.diagnostics ?? [],
|
|
3678
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3679
|
+
}));
|
|
3680
|
+
return;
|
|
3681
|
+
}
|
|
3682
|
+
if (req.method === 'GET' && path === '/api/domain-workspaces') {
|
|
3683
|
+
const requestId = apiRequestId('domain-workspaces');
|
|
3684
|
+
const snapshot = projectSnapshot();
|
|
3685
|
+
const manifest = snapshot.manifest;
|
|
3686
|
+
const packages = Object.values(manifest.modeling?.packages ?? {});
|
|
3687
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3688
|
+
res.end(serializeJSON({
|
|
3689
|
+
requestId,
|
|
3690
|
+
snapshotId: snapshot.snapshotId,
|
|
3691
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3692
|
+
domains: packages.map((pkg) => domainWorkspaceSummary(manifest, pkg.id)),
|
|
3693
|
+
unassignedModels: Object.values(manifest.dbtProvenance?.nodes ?? {}).filter((node) => !Object.values(manifest.modeling?.entities ?? {}).some((entity) => entity.dbtUniqueId === node.uniqueId)).length,
|
|
3694
|
+
}));
|
|
3695
|
+
return;
|
|
3696
|
+
}
|
|
3697
|
+
if (req.method === 'GET' && path.startsWith('/api/domain-workspaces/')) {
|
|
3698
|
+
const requestId = apiRequestId('domain-workspace');
|
|
3699
|
+
const suffix = decodeURIComponent(path.slice('/api/domain-workspaces/'.length));
|
|
3700
|
+
const relatedSuffix = '/related-products';
|
|
3701
|
+
const domainId = suffix.endsWith(relatedSuffix) ? suffix.slice(0, -relatedSuffix.length) : suffix;
|
|
3702
|
+
const snapshot = projectSnapshot();
|
|
3703
|
+
const manifest = snapshot.manifest;
|
|
3704
|
+
if (!manifest.modeling?.packages[domainId]) {
|
|
3705
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3706
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DOMAIN_NOT_FOUND', message: `domain workspace not found: ${domainId}`, recoverable: false })));
|
|
3707
|
+
return;
|
|
3708
|
+
}
|
|
3709
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3710
|
+
res.end(serializeJSON(suffix.endsWith(relatedSuffix)
|
|
3711
|
+
? { requestId, domain: domainId, ...relatedProductsForDomain(manifest, domainId), snapshotId: snapshot.snapshotId }
|
|
3712
|
+
: {
|
|
3713
|
+
requestId,
|
|
3714
|
+
snapshotId: snapshot.snapshotId,
|
|
3715
|
+
...domainWorkspaceSummary(manifest, domainId),
|
|
3716
|
+
relatedProducts: relatedProductsForDomain(manifest, domainId),
|
|
3717
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3718
|
+
}));
|
|
3719
|
+
return;
|
|
3720
|
+
}
|
|
3721
|
+
// Manifest v3 modeling surface. This response intentionally contains dbt
|
|
3722
|
+
// provenance references and the sparse DQL overlay, never copied dbt YAML.
|
|
3723
|
+
if (req.method === 'GET' && path === '/api/modeling/dbt-first') {
|
|
3724
|
+
const requestId = apiRequestId('modeling-dbt-first');
|
|
3725
|
+
const snapshot = projectSnapshot();
|
|
3726
|
+
const manifest = snapshot.manifest;
|
|
3727
|
+
if (manifest.manifestVersion !== 3 || !manifest.modeling || !manifest.dbtProvenance) {
|
|
3728
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3729
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DBT_FIRST_NOT_ENABLED', message: 'dbt-first modeling is not enabled. Set manifestVersion: 3 and modeling.mode: "dbt-first" in dql.config.json.', nextActions: ['Use Setup to connect dbt and enable dbt-first modeling.'] })));
|
|
3730
|
+
return;
|
|
3731
|
+
}
|
|
3732
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3733
|
+
res.end(serializeJSON({
|
|
3734
|
+
requestId,
|
|
3735
|
+
snapshotId: snapshot.snapshotId,
|
|
3736
|
+
manifestVersion: manifest.manifestVersion,
|
|
3737
|
+
dbtProvenance: manifest.dbtProvenance,
|
|
3738
|
+
modeling: manifest.modeling,
|
|
3739
|
+
domainAssets: collectDomainPackageAssets(projectRoot, loadDomainPackageRegistry(projectRoot)),
|
|
3740
|
+
lineage: manifest.lineage,
|
|
3741
|
+
diagnostics: manifest.diagnostics ?? [],
|
|
3742
|
+
snapshot: { id: snapshot.snapshotId, stale: snapshot.stale, error: snapshot.error },
|
|
3743
|
+
}));
|
|
3744
|
+
return;
|
|
3745
|
+
}
|
|
3746
|
+
if (req.method === 'GET' && path.startsWith('/api/modeling/dbt-first/nodes/')) {
|
|
3747
|
+
const requestId = apiRequestId('modeling-node');
|
|
3748
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3749
|
+
const uniqueId = decodeURIComponent(path.slice('/api/modeling/dbt-first/nodes/'.length));
|
|
3750
|
+
const snapshot = projectSnapshot();
|
|
3751
|
+
const cacheKey = `${snapshot.snapshotId}:${uniqueId}`;
|
|
3752
|
+
if (!dbtNodeDetailCache.has(cacheKey)) {
|
|
3753
|
+
dbtNodeDetailCache.set(cacheKey, dbtManifestPath ? loadDbtNodeAuthoringDetail(dbtManifestPath, uniqueId) : undefined);
|
|
3754
|
+
}
|
|
3755
|
+
const detail = dbtNodeDetailCache.get(cacheKey);
|
|
3756
|
+
if (!detail) {
|
|
3757
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3758
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'DBT_NODE_NOT_FOUND', message: `dbt node not found: ${uniqueId}`, recoverable: false })));
|
|
3759
|
+
return;
|
|
3760
|
+
}
|
|
3761
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3762
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, ...detail }));
|
|
3763
|
+
return;
|
|
3764
|
+
}
|
|
3765
|
+
if (req.method === 'GET' && path === '/api/modeling/dbt-first/inventory') {
|
|
3766
|
+
const requestId = apiRequestId('modeling-inventory');
|
|
3767
|
+
const snapshot = projectSnapshot();
|
|
3768
|
+
const manifest = snapshot.manifest;
|
|
3769
|
+
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 50));
|
|
3770
|
+
const cursor = Math.max(0, Number(url.searchParams.get('cursor')) || 0);
|
|
3771
|
+
const query = (url.searchParams.get('q') ?? '').trim().toLowerCase();
|
|
3772
|
+
const domain = (url.searchParams.get('domain') ?? '').trim();
|
|
3773
|
+
const boundByDbtId = new Map(Object.values(manifest.modeling?.entities ?? {}).map((entity) => [entity.dbtUniqueId, entity]));
|
|
3774
|
+
const nodes = Object.values(manifest.dbtProvenance?.nodes ?? {})
|
|
3775
|
+
.filter((node) => !query || `${node.name} ${node.relation ?? ''} ${node.sourcePath ?? ''}`.toLowerCase().includes(query))
|
|
3776
|
+
.filter((node) => !domain || boundByDbtId.get(node.uniqueId)?.domain === domain)
|
|
3777
|
+
.sort((a, b) => a.uniqueId.localeCompare(b.uniqueId));
|
|
3778
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3779
|
+
res.end(serializeJSON({
|
|
3780
|
+
requestId,
|
|
3781
|
+
items: nodes.slice(cursor, cursor + limit).map((node) => ({ ...node, binding: boundByDbtId.get(node.uniqueId) })),
|
|
3782
|
+
nextCursor: cursor + limit < nodes.length ? cursor + limit : null,
|
|
3783
|
+
total: nodes.length,
|
|
3784
|
+
snapshotId: snapshot.snapshotId,
|
|
3785
|
+
}));
|
|
3786
|
+
return;
|
|
3787
|
+
}
|
|
3788
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/nodes/batch') {
|
|
3789
|
+
const requestId = apiRequestId('modeling-node-batch');
|
|
3790
|
+
const body = await readJSON(req);
|
|
3791
|
+
const uniqueIds = Array.isArray(body.uniqueIds) ? body.uniqueIds.filter((value) => typeof value === 'string').slice(0, 100) : [];
|
|
3792
|
+
const snapshot = projectSnapshot();
|
|
3793
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3794
|
+
const details = uniqueIds.map((uniqueId) => {
|
|
3795
|
+
const cacheKey = `${snapshot.snapshotId}:${uniqueId}`;
|
|
3796
|
+
if (!dbtNodeDetailCache.has(cacheKey))
|
|
3797
|
+
dbtNodeDetailCache.set(cacheKey, dbtManifestPath ? loadDbtNodeAuthoringDetail(dbtManifestPath, uniqueId) : undefined);
|
|
3798
|
+
return dbtNodeDetailCache.get(cacheKey);
|
|
3799
|
+
}).filter(Boolean);
|
|
3800
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3801
|
+
res.end(serializeJSON({ requestId, details, snapshotId: snapshot.snapshotId }));
|
|
3802
|
+
return;
|
|
3803
|
+
}
|
|
3804
|
+
if (req.method === 'GET' && path === '/api/modeling/dbt-first/neighborhood') {
|
|
3805
|
+
const requestId = apiRequestId('modeling-neighborhood');
|
|
3806
|
+
const snapshot = projectSnapshot();
|
|
3807
|
+
const manifest = snapshot.manifest;
|
|
3808
|
+
const entityRef = url.searchParams.get('entity') ?? '';
|
|
3809
|
+
const limit = Math.min(200, Math.max(1, Number(url.searchParams.get('limit')) || 200));
|
|
3810
|
+
const entities = manifest.modeling?.entities ?? {};
|
|
3811
|
+
const relationships = manifest.modeling?.relationships ?? {};
|
|
3812
|
+
const entityKey = entities[entityRef] ? entityRef : Object.entries(entities).find(([, entity]) => entity.qualifiedId === entityRef)?.[0];
|
|
3813
|
+
if (!entityKey) {
|
|
3814
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3815
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code: 'MODELING_ENTITY_NOT_FOUND', message: `modeling entity not found: ${entityRef}`, recoverable: false })));
|
|
3816
|
+
return;
|
|
3817
|
+
}
|
|
3818
|
+
const edges = Object.values(relationships).filter((relationship) => relationship.from === entityKey || relationship.to === entityKey).slice(0, limit);
|
|
3819
|
+
const keys = new Set([entityKey, ...edges.flatMap((edge) => [edge.from, edge.to])]);
|
|
3820
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3821
|
+
res.end(serializeJSON({
|
|
3822
|
+
requestId,
|
|
3823
|
+
entities: Object.fromEntries([...keys].flatMap((key) => entities[key] ? [[key, entities[key]]] : [])),
|
|
3824
|
+
relationships: Object.fromEntries(edges.map((edge) => [edge.qualifiedId, edge])),
|
|
3825
|
+
snapshotId: snapshot.snapshotId,
|
|
3826
|
+
}));
|
|
3827
|
+
return;
|
|
3828
|
+
}
|
|
3829
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/dbt-source/preview') {
|
|
3830
|
+
const requestId = apiRequestId('modeling-dbt-source-preview');
|
|
3831
|
+
const snapshot = projectSnapshot();
|
|
3832
|
+
try {
|
|
3833
|
+
const body = await readJSON(req);
|
|
3834
|
+
if (!body.change)
|
|
3835
|
+
throw Object.assign(new Error('A dbt source change is required.'), { code: 'INVALID_REQUEST' });
|
|
3836
|
+
if (body.expectedSnapshotId && body.expectedSnapshotId !== snapshot.snapshotId) {
|
|
3837
|
+
throw Object.assign(new Error('Project sources changed after the model was loaded.'), { code: 'SOURCE_CHANGED' });
|
|
3838
|
+
}
|
|
3839
|
+
const { dbtProjectDir, manifestPath } = onboardingDbtPaths({});
|
|
3840
|
+
const preview = previewDbtSourcePatch(dbtProjectDir, manifestPath, body.change);
|
|
3841
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3842
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, ...preview }));
|
|
3843
|
+
}
|
|
3844
|
+
catch (error) {
|
|
3845
|
+
const code = apiErrorCode(error, 'DBT_SOURCE_PATCH_INVALID');
|
|
3846
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3847
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh the model and review the dbt YAML source patch again.'] })));
|
|
3848
|
+
}
|
|
3849
|
+
return;
|
|
3850
|
+
}
|
|
3851
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/dbt-source/apply') {
|
|
3852
|
+
const requestId = apiRequestId('modeling-dbt-source-apply');
|
|
3853
|
+
const snapshot = projectSnapshot();
|
|
3854
|
+
try {
|
|
3855
|
+
const body = await readJSON(req);
|
|
3856
|
+
if (!body.change)
|
|
3857
|
+
throw Object.assign(new Error('A dbt source change is required.'), { code: 'INVALID_REQUEST' });
|
|
3858
|
+
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId) {
|
|
3859
|
+
throw Object.assign(new Error('Project sources changed after the source patch preview.'), { code: 'SOURCE_CHANGED' });
|
|
3860
|
+
}
|
|
3861
|
+
const expectedFingerprint = body.expectedFingerprint ?? body.fingerprint;
|
|
3862
|
+
if (!expectedFingerprint)
|
|
3863
|
+
throw Object.assign(new Error('The reviewed source patch fingerprint is required.'), { code: 'SOURCE_CHANGED' });
|
|
3864
|
+
const { dbtProjectDir, manifestPath } = onboardingDbtPaths({});
|
|
3865
|
+
const applied = applyDbtSourcePatch(dbtProjectDir, manifestPath, body.change, expectedFingerprint);
|
|
3866
|
+
projectSnapshots.invalidate();
|
|
3867
|
+
invalidateAgentProjectState(projectRoot);
|
|
3868
|
+
const nextSnapshot = projectSnapshot();
|
|
3869
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3870
|
+
res.end(serializeJSON({ requestId, snapshotId: nextSnapshot.snapshotId, applied }));
|
|
3871
|
+
}
|
|
3872
|
+
catch (error) {
|
|
3873
|
+
const inferred = /changed after the preview/i.test(apiErrorMessage(error)) ? 'SOURCE_CHANGED' : 'DBT_SOURCE_PATCH_INVALID';
|
|
3874
|
+
const code = apiErrorCode(error, inferred);
|
|
3875
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3876
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh the source patch preview before applying.'] })));
|
|
3877
|
+
}
|
|
3878
|
+
return;
|
|
3879
|
+
}
|
|
3880
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/preview') {
|
|
3881
|
+
const requestId = apiRequestId('modeling-preview');
|
|
3882
|
+
const snapshot = projectSnapshot();
|
|
3883
|
+
try {
|
|
3884
|
+
const body = await readJSON(req);
|
|
3885
|
+
if (!body.change)
|
|
3886
|
+
throw new Error('A modeling change is required.');
|
|
3887
|
+
if (body.expectedSnapshotId && body.expectedSnapshotId !== snapshot.snapshotId)
|
|
3888
|
+
throw Object.assign(new Error('Project sources changed after the model was loaded.'), { code: 'SOURCE_CHANGED' });
|
|
3889
|
+
const preview = previewModelingChange(projectRoot, body.change);
|
|
3890
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3891
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, ...preview }));
|
|
3892
|
+
}
|
|
3893
|
+
catch (error) {
|
|
3894
|
+
const code = apiErrorCode(error, 'MODELING_CHANGE_INVALID');
|
|
3895
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3896
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh Domain Studio and preview the change again.'] })));
|
|
3897
|
+
}
|
|
3898
|
+
return;
|
|
3899
|
+
}
|
|
3900
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/apply') {
|
|
3901
|
+
const requestId = apiRequestId('modeling-apply');
|
|
3902
|
+
const snapshot = projectSnapshot();
|
|
3903
|
+
try {
|
|
3904
|
+
const body = await readJSON(req);
|
|
3905
|
+
if (!body.change)
|
|
3906
|
+
throw new Error('A modeling change is required.');
|
|
3907
|
+
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId)
|
|
3908
|
+
throw Object.assign(new Error('Project sources changed after the modeling preview.'), { code: 'SOURCE_CHANGED' });
|
|
3909
|
+
if (!body.fingerprint)
|
|
3910
|
+
throw Object.assign(new Error('The reviewed modeling source fingerprint is required.'), { code: 'SOURCE_CHANGED' });
|
|
3911
|
+
const applied = applyModelingChange(projectRoot, body.change, body.fingerprint);
|
|
3912
|
+
projectSnapshots.invalidate();
|
|
3913
|
+
const nextSnapshot = projectSnapshot();
|
|
3914
|
+
const manifest = nextSnapshot.manifest;
|
|
3915
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
3916
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3917
|
+
res.end(serializeJSON({ requestId, snapshotId: nextSnapshot.snapshotId, applied, modeling: manifest.modeling, diagnostics: manifest.diagnostics ?? [] }));
|
|
3918
|
+
}
|
|
3919
|
+
catch (error) {
|
|
3920
|
+
const inferred = /changed after the preview/i.test(apiErrorMessage(error)) ? 'SOURCE_CHANGED' : 'MODELING_APPLY_FAILED';
|
|
3921
|
+
const code = apiErrorCode(error, inferred);
|
|
3922
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3923
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh Domain Studio and review the latest source diff.'] })));
|
|
3924
|
+
}
|
|
3925
|
+
return;
|
|
3926
|
+
}
|
|
3927
|
+
if (req.method === 'POST' && path === '/api/modeling/dbt-first/relationships/validate') {
|
|
3928
|
+
const requestId = apiRequestId('modeling-relationship-validate');
|
|
3929
|
+
const snapshot = projectSnapshot();
|
|
3930
|
+
try {
|
|
3931
|
+
const body = await readJSON(req);
|
|
3932
|
+
if (!body.relationship)
|
|
3933
|
+
throw new Error('A relationship is required.');
|
|
3934
|
+
if (!body.expectedSnapshotId || body.expectedSnapshotId !== snapshot.snapshotId)
|
|
3935
|
+
throw Object.assign(new Error('Project sources changed before relationship validation.'), { code: 'SOURCE_CHANGED' });
|
|
3936
|
+
const manifest = snapshot.manifest;
|
|
3937
|
+
const activeConnection = requireActiveConnection();
|
|
3938
|
+
const evidence = await validateModelingRelationship(body.relationship, manifest, (sql) => executor.executeQuery(sql, [], {}, activeConnection), (identifier) => getDialect(activeConnection.driver).quoteIdentifier(identifier));
|
|
3939
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3940
|
+
res.end(serializeJSON({ requestId, snapshotId: snapshot.snapshotId, evidence }));
|
|
3941
|
+
}
|
|
3942
|
+
catch (error) {
|
|
3943
|
+
const code = apiErrorCode(error, 'RELATIONSHIP_VALIDATION_FAILED');
|
|
3944
|
+
res.writeHead(code === 'SOURCE_CHANGED' ? 409 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3945
|
+
res.end(serializeJSON(apiErrorEnvelope({ requestId, snapshotId: snapshot.snapshotId, code, message: apiErrorMessage(error), nextActions: ['Refresh the model or configure a warehouse connection, then retry validation.'] })));
|
|
3946
|
+
}
|
|
2645
3947
|
return;
|
|
2646
3948
|
}
|
|
2647
3949
|
if (req.method === 'GET' && path === '/api/agent-runs') {
|
|
@@ -3036,6 +4338,11 @@ export async function startLocalServer(opts) {
|
|
|
3036
4338
|
selection: typeof body?.context?.selection === 'string' ? body.context.selection : undefined,
|
|
3037
4339
|
};
|
|
3038
4340
|
const owner = typeof body?.owner === 'string' ? body.owner : undefined;
|
|
4341
|
+
const domain = typeof body?.domain === 'string' && body.domain.trim() ? body.domain.trim() : undefined;
|
|
4342
|
+
const modelAreaId = typeof body?.modelAreaId === 'string' && body.modelAreaId.trim() ? body.modelAreaId.trim() : undefined;
|
|
4343
|
+
const purpose = typeof body?.purpose === 'string'
|
|
4344
|
+
? body.purpose?.trim() || undefined
|
|
4345
|
+
: undefined;
|
|
3039
4346
|
const userId = typeof body?.userId === 'string'
|
|
3040
4347
|
? body.userId
|
|
3041
4348
|
: undefined;
|
|
@@ -3060,6 +4367,14 @@ export async function startLocalServer(opts) {
|
|
|
3060
4367
|
catch {
|
|
3061
4368
|
// Best-effort: a failed reindex falls back to whatever KG exists (or none).
|
|
3062
4369
|
}
|
|
4370
|
+
const { domainContext, contextPack } = await buildDirectBuildContext({
|
|
4371
|
+
question: prompt,
|
|
4372
|
+
domain,
|
|
4373
|
+
purpose,
|
|
4374
|
+
modelAreaId,
|
|
4375
|
+
surface: target === 'cell' ? 'notebook' : 'block',
|
|
4376
|
+
selectedContext: { target, mode, blockPath, owner },
|
|
4377
|
+
});
|
|
3063
4378
|
// Inject user-authored Skills as business context; the engine selects the
|
|
3064
4379
|
// relevant subset and stamps `appliedSkills` on the result.
|
|
3065
4380
|
const skills = loadSkills(projectRoot).skills;
|
|
@@ -3071,6 +4386,10 @@ export async function startLocalServer(opts) {
|
|
|
3071
4386
|
mode,
|
|
3072
4387
|
blockPath,
|
|
3073
4388
|
owner,
|
|
4389
|
+
domain,
|
|
4390
|
+
modelAreaId,
|
|
4391
|
+
contextPack,
|
|
4392
|
+
domainContext,
|
|
3074
4393
|
userId,
|
|
3075
4394
|
skills,
|
|
3076
4395
|
dbtManifestPath: resolveDbtManifestPath(projectRoot, loadProjectConfig(projectRoot)),
|
|
@@ -3240,9 +4559,12 @@ export async function startLocalServer(opts) {
|
|
|
3240
4559
|
res.end(serializeJSON({ error: 'Provide { domain } with a non-empty name.' }));
|
|
3241
4560
|
return;
|
|
3242
4561
|
}
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
4562
|
+
applyModelingChange(projectRoot, {
|
|
4563
|
+
operation: 'upsert_domain',
|
|
4564
|
+
value: { ...input, id: input.id ?? domainFolderSlug(input.name).replace(/-/g, '_') },
|
|
4565
|
+
});
|
|
4566
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
4567
|
+
const domain = findDomain(projectRoot, input.id ?? input.name);
|
|
3246
4568
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3247
4569
|
res.end(serializeJSON({ domain }));
|
|
3248
4570
|
}
|
|
@@ -3263,12 +4585,15 @@ export async function startLocalServer(opts) {
|
|
|
3263
4585
|
return;
|
|
3264
4586
|
}
|
|
3265
4587
|
// If the name changed, remove the old declaration so we never orphan one.
|
|
3266
|
-
if (
|
|
3267
|
-
|
|
4588
|
+
if (domainKey(id) !== domainKey(input.id ?? input.name)) {
|
|
4589
|
+
deleteCanonicalDomainDeclaration(projectRoot, id);
|
|
3268
4590
|
}
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
4591
|
+
applyModelingChange(projectRoot, {
|
|
4592
|
+
operation: 'upsert_domain',
|
|
4593
|
+
value: { ...input, id: input.id ?? id },
|
|
4594
|
+
});
|
|
4595
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
4596
|
+
const domain = findDomain(projectRoot, input.id ?? input.name);
|
|
3272
4597
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3273
4598
|
res.end(serializeJSON({ domain }));
|
|
3274
4599
|
}
|
|
@@ -3281,8 +4606,8 @@ export async function startLocalServer(opts) {
|
|
|
3281
4606
|
if (req.method === 'DELETE' && path.startsWith('/api/domains/')) {
|
|
3282
4607
|
try {
|
|
3283
4608
|
const id = decodeURIComponent(path.slice('/api/domains/'.length));
|
|
3284
|
-
|
|
3285
|
-
await
|
|
4609
|
+
deleteCanonicalDomainDeclaration(projectRoot, id);
|
|
4610
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
3286
4611
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3287
4612
|
res.end(serializeJSON({ ok: true }));
|
|
3288
4613
|
}
|
|
@@ -3377,8 +4702,12 @@ export async function startLocalServer(opts) {
|
|
|
3377
4702
|
}
|
|
3378
4703
|
try {
|
|
3379
4704
|
if (candidate.kind === 'domain' && candidate.domain) {
|
|
3380
|
-
const
|
|
3381
|
-
|
|
4705
|
+
const domainId = candidate.domain.id ?? domainFolderSlug(candidate.domain.name).replace(/-/g, '_');
|
|
4706
|
+
const written = applyModelingChange(projectRoot, {
|
|
4707
|
+
operation: 'upsert_domain',
|
|
4708
|
+
value: { ...candidate.domain, id: domainId },
|
|
4709
|
+
});
|
|
4710
|
+
saved.push({ id: candidate.id, path: written.patches[0]?.path, status: 'saved' });
|
|
3382
4711
|
}
|
|
3383
4712
|
else if (candidate.kind === 'skill' && candidate.skill) {
|
|
3384
4713
|
const skill = writeSkill(projectRoot, { ...candidate.skill, scope: 'project', status: 'draft' });
|
|
@@ -3392,7 +4721,7 @@ export async function startLocalServer(opts) {
|
|
|
3392
4721
|
saved.push({ id: candidate.id, status: 'blocked', blockers: [error instanceof Error ? error.message : String(error)] });
|
|
3393
4722
|
}
|
|
3394
4723
|
}
|
|
3395
|
-
await
|
|
4724
|
+
await refreshUnifiedProjectIndexes(projectRoot);
|
|
3396
4725
|
updateContextBootstrapSession(projectRoot, id, (current) => ({ ...current, closedAt: new Date().toISOString() }));
|
|
3397
4726
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
3398
4727
|
res.end(serializeJSON({ id, saved }));
|
|
@@ -3873,7 +5202,38 @@ export async function startLocalServer(opts) {
|
|
|
3873
5202
|
});
|
|
3874
5203
|
continue;
|
|
3875
5204
|
}
|
|
3876
|
-
|
|
5205
|
+
const pinPlan = pin.analysisPlan && typeof pin.analysisPlan === 'object' && !Array.isArray(pin.analysisPlan)
|
|
5206
|
+
? pin.analysisPlan
|
|
5207
|
+
: undefined;
|
|
5208
|
+
const pinArtifact = normalizeDqlArtifactReference(pinPlan?.dqlArtifact);
|
|
5209
|
+
let renderedPinResult = pin.result;
|
|
5210
|
+
let renderedPinChartConfig = pin.chartConfig;
|
|
5211
|
+
if (pinArtifact) {
|
|
5212
|
+
try {
|
|
5213
|
+
const boundParameters = dashboardTileParameterValues({
|
|
5214
|
+
item,
|
|
5215
|
+
dashboardValues: dashboardVariables,
|
|
5216
|
+
requestValues: variables,
|
|
5217
|
+
});
|
|
5218
|
+
const executed = await executeDqlArtifactSourceForAgent(pinArtifact.source, { name: pinArtifact.name ?? pin.title, path: pinArtifact.sourcePath }, { question: pin.question, parameters: { ...(pinArtifact.parameterValues ?? {}), ...boundParameters } });
|
|
5219
|
+
renderedPinResult = executed;
|
|
5220
|
+
renderedPinChartConfig = executed.chartConfig && typeof executed.chartConfig === 'object'
|
|
5221
|
+
? executed.chartConfig
|
|
5222
|
+
: renderedPinChartConfig;
|
|
5223
|
+
}
|
|
5224
|
+
catch (err) {
|
|
5225
|
+
tiles.push({
|
|
5226
|
+
tileId: item.i,
|
|
5227
|
+
status: 'error',
|
|
5228
|
+
tileType: 'aiPin',
|
|
5229
|
+
title: item.title ?? pin.title,
|
|
5230
|
+
error: err instanceof Error ? err.message : String(err),
|
|
5231
|
+
trustState: pinArtifact.trustState ?? 'review_required',
|
|
5232
|
+
});
|
|
5233
|
+
continue;
|
|
5234
|
+
}
|
|
5235
|
+
}
|
|
5236
|
+
else if (pin.refreshCadence === 'daily' && pin.sql && isAiPinRefreshDue(pin.lastRefreshedAt)) {
|
|
3877
5237
|
try {
|
|
3878
5238
|
const refreshed = await executeLocalSqlForStoredResult(pin.sql);
|
|
3879
5239
|
pin = localApps.updateAiPinResult(pin.id, refreshed) ?? pin;
|
|
@@ -3888,8 +5248,8 @@ export async function startLocalServer(opts) {
|
|
|
3888
5248
|
tileType: 'aiPin',
|
|
3889
5249
|
title: item.title ?? pin.title,
|
|
3890
5250
|
viz: item.viz,
|
|
3891
|
-
chartConfig: mergeDashboardChartConfig(
|
|
3892
|
-
result:
|
|
5251
|
+
chartConfig: mergeDashboardChartConfig(renderedPinChartConfig, item),
|
|
5252
|
+
result: renderedPinResult,
|
|
3893
5253
|
aiPin: pin,
|
|
3894
5254
|
citation: {
|
|
3895
5255
|
kind: 'ai_pin',
|
|
@@ -3918,6 +5278,34 @@ export async function startLocalServer(opts) {
|
|
|
3918
5278
|
const source = readFileSync(absBlockPath, 'utf-8');
|
|
3919
5279
|
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
3920
5280
|
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
5281
|
+
const boundParameters = dashboardTileParameterValues({
|
|
5282
|
+
item,
|
|
5283
|
+
dashboardValues: dashboardVariables,
|
|
5284
|
+
requestValues: variables,
|
|
5285
|
+
});
|
|
5286
|
+
const invocation = prepareBlockInvocation({
|
|
5287
|
+
block: block.name,
|
|
5288
|
+
source,
|
|
5289
|
+
parameters: { ...dashboardVariables, ...boundParameters },
|
|
5290
|
+
parameterSources: Object.fromEntries(Object.keys(boundParameters).map((name) => [name, 'surface'])),
|
|
5291
|
+
surface: 'app',
|
|
5292
|
+
});
|
|
5293
|
+
if (invocation.errors.length || invocation.unresolvedParameters.length) {
|
|
5294
|
+
tiles.push({
|
|
5295
|
+
tileId: item.i,
|
|
5296
|
+
status: 'unresolved',
|
|
5297
|
+
tileType: 'block',
|
|
5298
|
+
blockId: block.name,
|
|
5299
|
+
title: item.title ?? block.name,
|
|
5300
|
+
error: invocation.errors[0] ?? `Needs values for: ${invocation.unresolvedParameters.join(', ')}`,
|
|
5301
|
+
invocation: {
|
|
5302
|
+
resolvedParameters: invocation.resolvedParameters,
|
|
5303
|
+
unresolvedParameters: invocation.unresolvedParameters,
|
|
5304
|
+
auditId: invocation.auditId,
|
|
5305
|
+
},
|
|
5306
|
+
});
|
|
5307
|
+
continue;
|
|
5308
|
+
}
|
|
3921
5309
|
const semanticCompose = semanticLayer
|
|
3922
5310
|
? composeSemanticBlockSql(source, semanticLayer, {
|
|
3923
5311
|
driver: targetConnection.driver,
|
|
@@ -3925,6 +5313,7 @@ export async function startLocalServer(opts) {
|
|
|
3925
5313
|
projectRoot,
|
|
3926
5314
|
projectConfig,
|
|
3927
5315
|
detectedProvider: semanticDetectedProvider,
|
|
5316
|
+
parameters: invocation.values,
|
|
3928
5317
|
})
|
|
3929
5318
|
: null;
|
|
3930
5319
|
const plan = buildExecutionPlan({ id: item.i, type: 'dql', source, title: item.title ?? block.name }, { semanticLayer, driver: targetConnection.driver, tableMapping });
|
|
@@ -3945,7 +5334,7 @@ export async function startLocalServer(opts) {
|
|
|
3945
5334
|
dashboard: loaded.dashboard,
|
|
3946
5335
|
});
|
|
3947
5336
|
const prepared = prepareLocalExecution(filterApplication.sql, targetConnection, projectRoot, projectConfig);
|
|
3948
|
-
const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables(filterApplication.variables), prepared.connection);
|
|
5337
|
+
const result = await executor.executeQuery(prepared.sql, filterApplication.sqlParams, runtimeVariables({ ...filterApplication.variables, ...invocation.values }), prepared.connection);
|
|
3949
5338
|
tiles.push({
|
|
3950
5339
|
tileId: item.i,
|
|
3951
5340
|
status: 'ok',
|
|
@@ -3960,6 +5349,11 @@ export async function startLocalServer(opts) {
|
|
|
3960
5349
|
applied: filterApplication.appliedFilters,
|
|
3961
5350
|
skipped: filterApplication.skippedFilters,
|
|
3962
5351
|
},
|
|
5352
|
+
invocation: {
|
|
5353
|
+
resolvedParameters: invocation.resolvedParameters,
|
|
5354
|
+
unresolvedParameters: invocation.unresolvedParameters,
|
|
5355
|
+
auditId: invocation.auditId,
|
|
5356
|
+
},
|
|
3963
5357
|
citation: {
|
|
3964
5358
|
kind: 'block',
|
|
3965
5359
|
name: block.name,
|
|
@@ -4731,6 +6125,23 @@ export async function startLocalServer(opts) {
|
|
|
4731
6125
|
}
|
|
4732
6126
|
return;
|
|
4733
6127
|
}
|
|
6128
|
+
if (req.method === 'POST' && path === '/api/git/review/open') {
|
|
6129
|
+
try {
|
|
6130
|
+
const body = (await readJSON(req));
|
|
6131
|
+
const result = await gitOpenReview(projectRoot, {
|
|
6132
|
+
title: body.title ?? '',
|
|
6133
|
+
body: body.body ?? '',
|
|
6134
|
+
base: body.base ?? 'main',
|
|
6135
|
+
});
|
|
6136
|
+
res.writeHead(result.ok ? 201 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6137
|
+
res.end(serializeJSON(result));
|
|
6138
|
+
}
|
|
6139
|
+
catch (e) {
|
|
6140
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6141
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
6142
|
+
}
|
|
6143
|
+
return;
|
|
6144
|
+
}
|
|
4734
6145
|
if (req.method === 'POST' && path === '/api/git/pull') {
|
|
4735
6146
|
try {
|
|
4736
6147
|
const result = await gitPull(projectRoot);
|
|
@@ -4772,6 +6183,21 @@ export async function startLocalServer(opts) {
|
|
|
4772
6183
|
if (req.method === 'GET' && path === '/api/schema') {
|
|
4773
6184
|
try {
|
|
4774
6185
|
const dataFiles = scanDataFiles(projectRoot);
|
|
6186
|
+
const notebookDatasets = datasetWorkspace.list().map((dataset) => ({
|
|
6187
|
+
name: dataset.alias,
|
|
6188
|
+
path: dataset.sourcePath,
|
|
6189
|
+
columns: dataset.profile.columns.map((column) => ({
|
|
6190
|
+
name: column.name,
|
|
6191
|
+
type: column.type,
|
|
6192
|
+
})),
|
|
6193
|
+
source: "file",
|
|
6194
|
+
objectType: dataset.storageMode === "staged" ? "staged_dataset" : "dataset",
|
|
6195
|
+
datasetId: dataset.id,
|
|
6196
|
+
fileFingerprint: dataset.fileFingerprint,
|
|
6197
|
+
storageMode: dataset.storageMode,
|
|
6198
|
+
refreshedAt: dataset.refreshedAt,
|
|
6199
|
+
trustState: dataset.trustState,
|
|
6200
|
+
}));
|
|
4775
6201
|
const { tables, columnsByPath } = connection
|
|
4776
6202
|
? await introspectSchema(executor, connection)
|
|
4777
6203
|
: { tables: [], columnsByPath: new Map() };
|
|
@@ -4782,9 +6208,13 @@ export async function startLocalServer(opts) {
|
|
|
4782
6208
|
source: 'database',
|
|
4783
6209
|
objectType: t.type,
|
|
4784
6210
|
}));
|
|
4785
|
-
const seen = new Set(
|
|
6211
|
+
const seen = new Set([
|
|
6212
|
+
...dataFiles.map((f) => f.name),
|
|
6213
|
+
...notebookDatasets.map((dataset) => dataset.name),
|
|
6214
|
+
]);
|
|
4786
6215
|
const merged = [
|
|
4787
|
-
...
|
|
6216
|
+
...notebookDatasets,
|
|
6217
|
+
...dataFiles.map((f) => ({ ...f, source: "file" })),
|
|
4788
6218
|
...dbTables.filter((t) => !seen.has(t.name)),
|
|
4789
6219
|
];
|
|
4790
6220
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -4793,8 +6223,27 @@ export async function startLocalServer(opts) {
|
|
|
4793
6223
|
catch (error) {
|
|
4794
6224
|
const message = error instanceof Error ? error.message : String(error);
|
|
4795
6225
|
console.warn(`[dql] /api/schema introspection failed: ${message}`);
|
|
4796
|
-
const fallback =
|
|
4797
|
-
|
|
6226
|
+
const fallback = [
|
|
6227
|
+
...datasetWorkspace.list().map((dataset) => ({
|
|
6228
|
+
name: dataset.alias,
|
|
6229
|
+
path: dataset.sourcePath,
|
|
6230
|
+
columns: dataset.profile.columns.map((column) => ({
|
|
6231
|
+
name: column.name,
|
|
6232
|
+
type: column.type,
|
|
6233
|
+
})),
|
|
6234
|
+
source: "file",
|
|
6235
|
+
objectType: dataset.storageMode === "staged" ? "staged_dataset" : "dataset",
|
|
6236
|
+
datasetId: dataset.id,
|
|
6237
|
+
fileFingerprint: dataset.fileFingerprint,
|
|
6238
|
+
storageMode: dataset.storageMode,
|
|
6239
|
+
refreshedAt: dataset.refreshedAt,
|
|
6240
|
+
trustState: dataset.trustState,
|
|
6241
|
+
})),
|
|
6242
|
+
...scanDataFiles(projectRoot).map((f) => ({ ...f, source: "file" })),
|
|
6243
|
+
];
|
|
6244
|
+
res.writeHead(200, {
|
|
6245
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
6246
|
+
});
|
|
4798
6247
|
res.end(serializeJSON(fallback));
|
|
4799
6248
|
}
|
|
4800
6249
|
return;
|
|
@@ -4808,6 +6257,11 @@ export async function startLocalServer(opts) {
|
|
|
4808
6257
|
res.end(serializeJSON({ error: 'Missing block name' }));
|
|
4809
6258
|
return;
|
|
4810
6259
|
}
|
|
6260
|
+
if (dbtFirstOwnsSemanticAuthoring(projectConfig)
|
|
6261
|
+
&& (blockType === 'semantic' || /\btype\s*=\s*["']semantic["']/i.test(content ?? ''))) {
|
|
6262
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
6263
|
+
return;
|
|
6264
|
+
}
|
|
4811
6265
|
const created = createBlockArtifacts(projectRoot, {
|
|
4812
6266
|
name,
|
|
4813
6267
|
domain,
|
|
@@ -4835,12 +6289,16 @@ export async function startLocalServer(opts) {
|
|
|
4835
6289
|
if (req.method === 'POST' && path === '/api/blocks/save-from-cell') {
|
|
4836
6290
|
try {
|
|
4837
6291
|
const body = await readJSON(req);
|
|
4838
|
-
const { name, domain, owner, content, description, tags, metricRefs, template, llmContext, examples, invariants, } = body;
|
|
6292
|
+
const { name, domain, owner, content, description, tags, metricRefs, template, llmContext, examples, invariants, reviewRequired, datasetRefs, lineage, } = body;
|
|
4839
6293
|
if (!name || typeof name !== 'string' || !content || typeof content !== 'string') {
|
|
4840
6294
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
4841
6295
|
res.end(serializeJSON({ error: 'name and content are required' }));
|
|
4842
6296
|
return;
|
|
4843
6297
|
}
|
|
6298
|
+
if (dbtFirstOwnsSemanticAuthoring(projectConfig) && /\btype\s*=\s*["']semantic["']/i.test(content)) {
|
|
6299
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
6300
|
+
return;
|
|
6301
|
+
}
|
|
4844
6302
|
// OSS keeps the save flow deliberately small: the person saving the
|
|
4845
6303
|
// reusable answer is accountable for it. Domain/description/tags are
|
|
4846
6304
|
// helpful defaults, but not reasons to send a single-user back to a
|
|
@@ -4873,12 +6331,18 @@ export async function startLocalServer(opts) {
|
|
|
4873
6331
|
// Save first, then certify against the same local runtime used for the
|
|
4874
6332
|
// answer. Passing blocks become certified immediately; anything that
|
|
4875
6333
|
// cannot prove itself remains a clearly labelled draft at this path.
|
|
4876
|
-
setBlockStudioStatus(projectRoot, created.path,
|
|
4877
|
-
let source = readFileSync(join(projectRoot, created.path),
|
|
4878
|
-
let status =
|
|
4879
|
-
let blockers =
|
|
6334
|
+
setBlockStudioStatus(projectRoot, created.path, "draft");
|
|
6335
|
+
let source = readFileSync(join(projectRoot, created.path), "utf-8");
|
|
6336
|
+
let status = "draft";
|
|
6337
|
+
let blockers = reviewRequired
|
|
6338
|
+
? [
|
|
6339
|
+
"Imported or staged datasets require review and reproducibility checks before certification.",
|
|
6340
|
+
]
|
|
6341
|
+
: [];
|
|
4880
6342
|
let certification;
|
|
4881
6343
|
try {
|
|
6344
|
+
if (reviewRequired)
|
|
6345
|
+
throw new Error(blockers[0]);
|
|
4882
6346
|
const verdict = await certifyBlockStudioSource(source, created.path);
|
|
4883
6347
|
blockers = Array.from(new Set(verdict.checklist.blockers));
|
|
4884
6348
|
certification = {
|
|
@@ -4893,7 +6357,10 @@ export async function startLocalServer(opts) {
|
|
|
4893
6357
|
}
|
|
4894
6358
|
}
|
|
4895
6359
|
catch (error) {
|
|
4896
|
-
blockers =
|
|
6360
|
+
blockers = Array.from(new Set([
|
|
6361
|
+
...blockers,
|
|
6362
|
+
error instanceof Error ? error.message : String(error),
|
|
6363
|
+
]));
|
|
4897
6364
|
}
|
|
4898
6365
|
await refreshLocalMetadataCatalog(projectRoot);
|
|
4899
6366
|
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -5166,7 +6633,9 @@ export async function startLocalServer(opts) {
|
|
|
5166
6633
|
// Only expose distinct values for a DECLARED output column — keeps the probe
|
|
5167
6634
|
// inside the governed block contract (no arbitrary column scanning).
|
|
5168
6635
|
const parsedMeta = parseBlockSourceMetadata(source);
|
|
5169
|
-
|
|
6636
|
+
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
6637
|
+
const parameterNames = new Set((validation.parameters ?? []).map((parameter) => parameter.name));
|
|
6638
|
+
if (parameterNames.has(column) || (parsedMeta.outputs.length > 0 && !parsedMeta.outputs.includes(column))) {
|
|
5170
6639
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
5171
6640
|
res.end(serializeJSON({ error: `"${column}" is not a declared output of this block` }));
|
|
5172
6641
|
return;
|
|
@@ -5182,7 +6651,6 @@ export async function startLocalServer(opts) {
|
|
|
5182
6651
|
detectedProvider: semanticDetectedProvider,
|
|
5183
6652
|
})
|
|
5184
6653
|
: null;
|
|
5185
|
-
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
5186
6654
|
const baseSql = semanticCompose?.sql ?? validation.executableSql;
|
|
5187
6655
|
if (!baseSql) {
|
|
5188
6656
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -6049,7 +7517,10 @@ export async function startLocalServer(opts) {
|
|
|
6049
7517
|
const body = await readJSON(req);
|
|
6050
7518
|
const source = typeof body.source === 'string' ? body.source : '';
|
|
6051
7519
|
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
6052
|
-
const
|
|
7520
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
7521
|
+
? body.parameters
|
|
7522
|
+
: {};
|
|
7523
|
+
const preview = await runBlockStudioPreviewSource(source, targetConnection, parameters);
|
|
6053
7524
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6054
7525
|
res.end(serializeJSON(preview));
|
|
6055
7526
|
}
|
|
@@ -6129,7 +7600,263 @@ export async function startLocalServer(opts) {
|
|
|
6129
7600
|
res.end(serializeJSON({ default: defaultKey, connections, dbtProfiles, connectorStatus }));
|
|
6130
7601
|
return;
|
|
6131
7602
|
}
|
|
6132
|
-
if (req.method ===
|
|
7603
|
+
if (req.method === "GET" && path === "/api/datasets") {
|
|
7604
|
+
res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" });
|
|
7605
|
+
res.end(serializeJSON({
|
|
7606
|
+
datasets: datasetWorkspace.list(),
|
|
7607
|
+
workspace: {
|
|
7608
|
+
target: "local",
|
|
7609
|
+
databasePath: relative(projectRoot, datasetWorkspace.databasePath).replaceAll("\\", "/"),
|
|
7610
|
+
},
|
|
7611
|
+
}));
|
|
7612
|
+
return;
|
|
7613
|
+
}
|
|
7614
|
+
if (req.method === "POST" && path === "/api/datasets/import") {
|
|
7615
|
+
try {
|
|
7616
|
+
const multipart = String(req.headers["content-type"] ?? "")
|
|
7617
|
+
.toLowerCase()
|
|
7618
|
+
.startsWith("multipart/form-data")
|
|
7619
|
+
? await readMultipartForm(req)
|
|
7620
|
+
: null;
|
|
7621
|
+
const body = multipart?.fields ?? (await readJSON(req));
|
|
7622
|
+
if (typeof body.contentBase64 === "string" &&
|
|
7623
|
+
body.contentBase64.length > 350_000_000) {
|
|
7624
|
+
throw new Error("Uploaded dataset exceeds the 250 MB local import limit. Link the local path or reduce the file first.");
|
|
7625
|
+
}
|
|
7626
|
+
if (multipart?.file &&
|
|
7627
|
+
multipart.file.content.byteLength > 250_000_000) {
|
|
7628
|
+
throw new Error("Uploaded dataset exceeds the 250 MB local import limit. Link the local path or reduce the file first.");
|
|
7629
|
+
}
|
|
7630
|
+
await ensureLocalWorkspaceReady();
|
|
7631
|
+
const imported = await datasetWorkspace.import({
|
|
7632
|
+
filename: multipart?.file?.filename ??
|
|
7633
|
+
(typeof body.filename === "string" ? body.filename : undefined),
|
|
7634
|
+
sourcePath: typeof body.sourcePath === "string" ? body.sourcePath : undefined,
|
|
7635
|
+
contentBase64: typeof body.contentBase64 === "string"
|
|
7636
|
+
? body.contentBase64
|
|
7637
|
+
: undefined,
|
|
7638
|
+
content: multipart?.file?.content,
|
|
7639
|
+
storageMode: body.storageMode === "project" ? "project" : "local",
|
|
7640
|
+
link: body.link === true || body.link === "true",
|
|
7641
|
+
name: typeof body.name === "string" ? body.name : undefined,
|
|
7642
|
+
alias: typeof body.alias === "string" ? body.alias : undefined,
|
|
7643
|
+
description: typeof body.description === "string" ? body.description : undefined,
|
|
7644
|
+
owner: typeof body.owner === "string" ? body.owner : undefined,
|
|
7645
|
+
tags: Array.isArray(body.tags)
|
|
7646
|
+
? body.tags.map(String)
|
|
7647
|
+
: typeof body.tags === "string"
|
|
7648
|
+
? body.tags
|
|
7649
|
+
.split(",")
|
|
7650
|
+
.map((tag) => tag.trim())
|
|
7651
|
+
.filter(Boolean)
|
|
7652
|
+
: undefined,
|
|
7653
|
+
});
|
|
7654
|
+
recordDatasetMetadataSnapshot();
|
|
7655
|
+
res.writeHead(imported.duplicate ? 200 : 201, {
|
|
7656
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7657
|
+
});
|
|
7658
|
+
res.end(serializeJSON(imported));
|
|
7659
|
+
}
|
|
7660
|
+
catch (error) {
|
|
7661
|
+
res.writeHead(400, {
|
|
7662
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7663
|
+
});
|
|
7664
|
+
res.end(serializeJSON({
|
|
7665
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7666
|
+
}));
|
|
7667
|
+
}
|
|
7668
|
+
return;
|
|
7669
|
+
}
|
|
7670
|
+
if (req.method === "GET" && /^\/api\/datasets\/[^/]+$/.test(path)) {
|
|
7671
|
+
const id = decodeURIComponent(path.slice("/api/datasets/".length));
|
|
7672
|
+
const dataset = datasetWorkspace.get(id);
|
|
7673
|
+
if (!dataset) {
|
|
7674
|
+
res.writeHead(404, {
|
|
7675
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7676
|
+
});
|
|
7677
|
+
res.end(serializeJSON({ error: "Dataset not found." }));
|
|
7678
|
+
}
|
|
7679
|
+
else {
|
|
7680
|
+
res.writeHead(200, {
|
|
7681
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7682
|
+
});
|
|
7683
|
+
res.end(serializeJSON({ dataset }));
|
|
7684
|
+
}
|
|
7685
|
+
return;
|
|
7686
|
+
}
|
|
7687
|
+
if (req.method === "POST" &&
|
|
7688
|
+
/^\/api\/datasets\/[^/]+\/refresh$/.test(path)) {
|
|
7689
|
+
try {
|
|
7690
|
+
const id = decodeURIComponent(path.match(/^\/api\/datasets\/([^/]+)\/refresh$/)?.[1] ?? "");
|
|
7691
|
+
await ensureLocalWorkspaceReady();
|
|
7692
|
+
const dataset = await datasetWorkspace.refresh(id);
|
|
7693
|
+
recordDatasetMetadataSnapshot();
|
|
7694
|
+
res.writeHead(200, {
|
|
7695
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7696
|
+
});
|
|
7697
|
+
res.end(serializeJSON({ dataset }));
|
|
7698
|
+
}
|
|
7699
|
+
catch (error) {
|
|
7700
|
+
res.writeHead(400, {
|
|
7701
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7702
|
+
});
|
|
7703
|
+
res.end(serializeJSON({
|
|
7704
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7705
|
+
}));
|
|
7706
|
+
}
|
|
7707
|
+
return;
|
|
7708
|
+
}
|
|
7709
|
+
if (req.method === "POST" &&
|
|
7710
|
+
/^\/api\/datasets\/[^/]+\/(rename|pin)$/.test(path)) {
|
|
7711
|
+
try {
|
|
7712
|
+
const match = path.match(/^\/api\/datasets\/([^/]+)\/(rename|pin)$/);
|
|
7713
|
+
const body = await readJSON(req);
|
|
7714
|
+
const dataset = match[2] === "pin"
|
|
7715
|
+
? datasetWorkspace.pin(decodeURIComponent(match[1]), body.pinned !== false)
|
|
7716
|
+
: datasetWorkspace.rename(decodeURIComponent(match[1]), String(body.name ?? ""), typeof body.alias === "string" ? body.alias : undefined);
|
|
7717
|
+
recordDatasetMetadataSnapshot();
|
|
7718
|
+
res.writeHead(200, {
|
|
7719
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7720
|
+
});
|
|
7721
|
+
res.end(serializeJSON({ dataset }));
|
|
7722
|
+
}
|
|
7723
|
+
catch (error) {
|
|
7724
|
+
res.writeHead(400, {
|
|
7725
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7726
|
+
});
|
|
7727
|
+
res.end(serializeJSON({
|
|
7728
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7729
|
+
}));
|
|
7730
|
+
}
|
|
7731
|
+
return;
|
|
7732
|
+
}
|
|
7733
|
+
if (req.method === "POST" &&
|
|
7734
|
+
/^\/api\/datasets\/[^/]+\/schema$/.test(path)) {
|
|
7735
|
+
try {
|
|
7736
|
+
const id = decodeURIComponent(path.match(/^\/api\/datasets\/([^/]+)\/schema$/)?.[1] ?? "");
|
|
7737
|
+
const body = await readJSON(req);
|
|
7738
|
+
await ensureLocalWorkspaceReady();
|
|
7739
|
+
const dataset = await datasetWorkspace.updateSchema(id, body.overrides && typeof body.overrides === "object"
|
|
7740
|
+
? body.overrides
|
|
7741
|
+
: {});
|
|
7742
|
+
recordDatasetMetadataSnapshot();
|
|
7743
|
+
res.writeHead(200, {
|
|
7744
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7745
|
+
});
|
|
7746
|
+
res.end(serializeJSON({ dataset }));
|
|
7747
|
+
}
|
|
7748
|
+
catch (error) {
|
|
7749
|
+
res.writeHead(400, {
|
|
7750
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7751
|
+
});
|
|
7752
|
+
res.end(serializeJSON({
|
|
7753
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7754
|
+
}));
|
|
7755
|
+
}
|
|
7756
|
+
return;
|
|
7757
|
+
}
|
|
7758
|
+
if (req.method === "DELETE" && /^\/api\/datasets\/[^/]+$/.test(path)) {
|
|
7759
|
+
const id = decodeURIComponent(path.slice("/api/datasets/".length));
|
|
7760
|
+
datasetWorkspace.remove(id);
|
|
7761
|
+
recordDatasetMetadataSnapshot();
|
|
7762
|
+
res.writeHead(204);
|
|
7763
|
+
res.end();
|
|
7764
|
+
return;
|
|
7765
|
+
}
|
|
7766
|
+
if (req.method === "POST" && path === "/api/datasets/stage") {
|
|
7767
|
+
try {
|
|
7768
|
+
const body = await readJSON(req);
|
|
7769
|
+
if (body.confirmed !== true) {
|
|
7770
|
+
res.writeHead(428, {
|
|
7771
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7772
|
+
});
|
|
7773
|
+
res.end(serializeJSON({
|
|
7774
|
+
error: "Confirm the bounded local copy before staging warehouse data.",
|
|
7775
|
+
}));
|
|
7776
|
+
return;
|
|
7777
|
+
}
|
|
7778
|
+
const configured = projectConfig.notebook?.staging;
|
|
7779
|
+
const maxRows = Math.max(1, Math.min(Number(body.maxRows ?? configured?.maxRows ?? 100_000), 500_000));
|
|
7780
|
+
const maxBytes = Math.max(1_000_000, Math.min(Number(body.maxBytes ?? configured?.maxBytes ?? 250_000_000), 1_000_000_000));
|
|
7781
|
+
const timeoutMs = Math.max(1_000, Math.min(Number(body.timeoutSeconds ?? configured?.timeoutSeconds ?? 120) *
|
|
7782
|
+
1000, 600_000));
|
|
7783
|
+
const sql = typeof body.sql === "string" ? body.sql.trim() : "";
|
|
7784
|
+
const validationError = readOnlySqlValidationError(sql, "Local staging");
|
|
7785
|
+
if (validationError)
|
|
7786
|
+
throw new Error(validationError);
|
|
7787
|
+
const semantic = prepareSemanticSql(sql, semanticLayer);
|
|
7788
|
+
if (semantic.unresolvedRefs.length > 0)
|
|
7789
|
+
throw new Error(`Unknown semantic references: ${semantic.unresolvedRefs.join(", ")}`);
|
|
7790
|
+
const named = typeof body.connectionName === "string"
|
|
7791
|
+
? resolveNamedConnection(body.connectionName)
|
|
7792
|
+
: connection;
|
|
7793
|
+
const sourceConnection = requireActiveConnection(named);
|
|
7794
|
+
const prepared = prepareLocalExecution(`SELECT * FROM (${stripSqlTerminator(semantic.sql)}) AS dql_stage_source LIMIT ${maxRows + 1}`, sourceConnection, projectRoot, projectConfig);
|
|
7795
|
+
const execution = executor.executeQuery(prepared.sql, [], runtimeVariables({}), prepared.connection);
|
|
7796
|
+
const result = await Promise.race([
|
|
7797
|
+
execution,
|
|
7798
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Staging query exceeded ${Math.round(timeoutMs / 1000)} seconds.`)), timeoutMs)),
|
|
7799
|
+
]);
|
|
7800
|
+
const rows = (result.rows ?? []);
|
|
7801
|
+
if (rows.length > maxRows)
|
|
7802
|
+
throw new Error(`Staging would exceed the ${maxRows.toLocaleString()} row limit. Add filters or aggregate the warehouse query.`);
|
|
7803
|
+
const rowPayload = serializeJSON(rows);
|
|
7804
|
+
const byteCount = Buffer.byteLength(rowPayload);
|
|
7805
|
+
if (byteCount > maxBytes)
|
|
7806
|
+
throw new Error(`Staging would exceed the ${(maxBytes / 1_000_000).toFixed(0)} MB limit. Add filters or aggregate the warehouse query.`);
|
|
7807
|
+
await ensureLocalWorkspaceReady();
|
|
7808
|
+
const dataset = await datasetWorkspace.stageRows({
|
|
7809
|
+
name: typeof body.name === "string" && body.name.trim()
|
|
7810
|
+
? body.name.trim()
|
|
7811
|
+
: "Staged warehouse result",
|
|
7812
|
+
rows,
|
|
7813
|
+
expiresInDays: configured?.expiryDays ?? 7,
|
|
7814
|
+
lineage: {
|
|
7815
|
+
connectionName: typeof body.connectionName === "string"
|
|
7816
|
+
? body.connectionName
|
|
7817
|
+
: projectConfig.defaultConnectionName,
|
|
7818
|
+
query: sql,
|
|
7819
|
+
blockPath: typeof body.blockPath === "string" ? body.blockPath : undefined,
|
|
7820
|
+
semanticMetrics: semantic.semanticRefs.metrics,
|
|
7821
|
+
semanticDimensions: semantic.semanticRefs.dimensions,
|
|
7822
|
+
filters: body.filters && typeof body.filters === "object"
|
|
7823
|
+
? body.filters
|
|
7824
|
+
: undefined,
|
|
7825
|
+
parameters: body.parameters && typeof body.parameters === "object"
|
|
7826
|
+
? body.parameters
|
|
7827
|
+
: undefined,
|
|
7828
|
+
extractedAt: new Date().toISOString(),
|
|
7829
|
+
rowCount: rows.length,
|
|
7830
|
+
byteCount,
|
|
7831
|
+
sourceFingerprint: createHash("sha256")
|
|
7832
|
+
.update(`${body.connectionName ?? projectConfig.defaultConnectionName ?? "default"}:${sql}`)
|
|
7833
|
+
.digest("hex"),
|
|
7834
|
+
resultFingerprint: createHash("sha256")
|
|
7835
|
+
.update(rowPayload)
|
|
7836
|
+
.digest("hex"),
|
|
7837
|
+
},
|
|
7838
|
+
});
|
|
7839
|
+
recordDatasetMetadataSnapshot();
|
|
7840
|
+
res.writeHead(201, {
|
|
7841
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7842
|
+
});
|
|
7843
|
+
res.end(serializeJSON({
|
|
7844
|
+
dataset,
|
|
7845
|
+
limits: { maxRows, maxBytes, timeoutMs },
|
|
7846
|
+
trustLabel: "review_required",
|
|
7847
|
+
}));
|
|
7848
|
+
}
|
|
7849
|
+
catch (error) {
|
|
7850
|
+
res.writeHead(400, {
|
|
7851
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
7852
|
+
});
|
|
7853
|
+
res.end(serializeJSON({
|
|
7854
|
+
error: error instanceof Error ? error.message : String(error),
|
|
7855
|
+
}));
|
|
7856
|
+
}
|
|
7857
|
+
return;
|
|
7858
|
+
}
|
|
7859
|
+
if (req.method === "POST" && path === "/api/connectors/install") {
|
|
6133
7860
|
try {
|
|
6134
7861
|
const body = await readJSON(req);
|
|
6135
7862
|
const driver = typeof body.driver === 'string' ? body.driver : '';
|
|
@@ -6356,9 +8083,18 @@ export async function startLocalServer(opts) {
|
|
|
6356
8083
|
source: q.source ?? null,
|
|
6357
8084
|
}));
|
|
6358
8085
|
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
6359
|
-
const
|
|
8086
|
+
const dbtManifestReady = provider === 'dbt'
|
|
6360
8087
|
? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
|
|
6361
8088
|
: false;
|
|
8089
|
+
const metricFlowReady = provider === 'dbt' ? hasMetricFlowCli() : false;
|
|
8090
|
+
const dbtExecutionReady = dbtManifestReady && metricFlowReady;
|
|
8091
|
+
const dbtExecutionSetup = dbtExecutionReady
|
|
8092
|
+
? null
|
|
8093
|
+
: !dbtManifestReady && !metricFlowReady
|
|
8094
|
+
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.'
|
|
8095
|
+
: !dbtManifestReady
|
|
8096
|
+
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists.'
|
|
8097
|
+
: 'Install MetricFlow so `mf` is on PATH, or set DQL_METRICFLOW_BIN to the MetricFlow executable.';
|
|
6362
8098
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6363
8099
|
res.end(serializeJSON({
|
|
6364
8100
|
available: true,
|
|
@@ -6367,9 +8103,7 @@ export async function startLocalServer(opts) {
|
|
|
6367
8103
|
? {
|
|
6368
8104
|
engine: 'metricflow',
|
|
6369
8105
|
ready: dbtExecutionReady,
|
|
6370
|
-
setup:
|
|
6371
|
-
? null
|
|
6372
|
-
: 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.',
|
|
8106
|
+
setup: dbtExecutionSetup,
|
|
6373
8107
|
}
|
|
6374
8108
|
: { engine: 'native', ready: true, setup: null },
|
|
6375
8109
|
errors: semanticLayerErrors,
|
|
@@ -6771,8 +8505,186 @@ export async function startLocalServer(opts) {
|
|
|
6771
8505
|
}
|
|
6772
8506
|
return;
|
|
6773
8507
|
}
|
|
6774
|
-
// ──
|
|
6775
|
-
if (req.method ===
|
|
8508
|
+
// ── Ranked, lazy editor completions (large-repo safe) ───────────────────────
|
|
8509
|
+
if (req.method === "GET" && path === "/api/editor/completions") {
|
|
8510
|
+
const query = (url.searchParams.get("q") ?? "").trim().toLowerCase();
|
|
8511
|
+
const kind = (url.searchParams.get("kind") ?? "all").trim().toLowerCase();
|
|
8512
|
+
const relationFilter = (url.searchParams.get("relation") ?? "")
|
|
8513
|
+
.trim()
|
|
8514
|
+
.toLowerCase();
|
|
8515
|
+
const selectedDomain = (url.searchParams.get("domain") ?? "")
|
|
8516
|
+
.trim()
|
|
8517
|
+
.toLowerCase();
|
|
8518
|
+
const limit = Math.max(1, Math.min(100, Number(url.searchParams.get("limit") ?? 50) || 50));
|
|
8519
|
+
const recent = readUserPrefs(userPrefsPath).recentlyUsed.map((name) => name.toLowerCase());
|
|
8520
|
+
const candidates = [];
|
|
8521
|
+
if (semanticLayer && (kind === "all" || kind === "metric")) {
|
|
8522
|
+
for (const metric of semanticLayer.listMetrics())
|
|
8523
|
+
candidates.push({
|
|
8524
|
+
type: "metric",
|
|
8525
|
+
name: metric.name,
|
|
8526
|
+
label: metric.label,
|
|
8527
|
+
description: metric.description ?? "",
|
|
8528
|
+
domain: metric.domain,
|
|
8529
|
+
tags: metric.tags ?? [],
|
|
8530
|
+
detail: "@metric",
|
|
8531
|
+
apply: metric.name,
|
|
8532
|
+
governance: "semantic",
|
|
8533
|
+
});
|
|
8534
|
+
}
|
|
8535
|
+
if (semanticLayer && (kind === "all" || kind === "dimension")) {
|
|
8536
|
+
for (const dimension of semanticLayer.listDimensions())
|
|
8537
|
+
candidates.push({
|
|
8538
|
+
type: "dimension",
|
|
8539
|
+
name: dimension.name,
|
|
8540
|
+
label: dimension.label,
|
|
8541
|
+
description: dimension.description ?? "",
|
|
8542
|
+
domain: dimension.domain,
|
|
8543
|
+
tags: dimension.tags ?? [],
|
|
8544
|
+
detail: "@dim",
|
|
8545
|
+
apply: dimension.name,
|
|
8546
|
+
governance: "semantic",
|
|
8547
|
+
});
|
|
8548
|
+
}
|
|
8549
|
+
const datasets = datasetWorkspace.list();
|
|
8550
|
+
if (kind === "all" || kind === "relation" || kind === "dataset") {
|
|
8551
|
+
for (const dataset of datasets)
|
|
8552
|
+
candidates.push({
|
|
8553
|
+
type: kind === "dataset" ? "dataset" : "relation",
|
|
8554
|
+
name: dataset.alias,
|
|
8555
|
+
label: dataset.name,
|
|
8556
|
+
description: `${dataset.storageMode} ${dataset.format} · refreshed ${dataset.refreshedAt}`,
|
|
8557
|
+
relation: dataset.alias,
|
|
8558
|
+
detail: dataset.trustState,
|
|
8559
|
+
apply: `"${dataset.alias.replaceAll('"', '""')}"`,
|
|
8560
|
+
governance: dataset.trustState,
|
|
8561
|
+
});
|
|
8562
|
+
}
|
|
8563
|
+
if (kind === "all" || kind === "column") {
|
|
8564
|
+
for (const dataset of datasets) {
|
|
8565
|
+
if (relationFilter &&
|
|
8566
|
+
![dataset.id, dataset.alias, dataset.name].some((value) => value.toLowerCase() === relationFilter))
|
|
8567
|
+
continue;
|
|
8568
|
+
for (const column of dataset.profile.columns)
|
|
8569
|
+
candidates.push({
|
|
8570
|
+
type: "column",
|
|
8571
|
+
name: column.name,
|
|
8572
|
+
label: column.name,
|
|
8573
|
+
description: column.flags?.join(", ") ?? "",
|
|
8574
|
+
relation: dataset.alias,
|
|
8575
|
+
detail: column.type,
|
|
8576
|
+
apply: column.name,
|
|
8577
|
+
governance: dataset.trustState,
|
|
8578
|
+
});
|
|
8579
|
+
}
|
|
8580
|
+
const snapshot = latestRuntimeSchemaSnapshotForProject(projectRoot);
|
|
8581
|
+
for (const table of snapshot?.tables ?? []) {
|
|
8582
|
+
if (relationFilter &&
|
|
8583
|
+
![table.relation, table.name]
|
|
8584
|
+
.filter(Boolean)
|
|
8585
|
+
.some((value) => value.toLowerCase() === relationFilter ||
|
|
8586
|
+
value.toLowerCase().endsWith(`.${relationFilter}`)))
|
|
8587
|
+
continue;
|
|
8588
|
+
for (const column of table.columns)
|
|
8589
|
+
candidates.push({
|
|
8590
|
+
type: "column",
|
|
8591
|
+
name: column.name,
|
|
8592
|
+
label: column.name,
|
|
8593
|
+
description: column.description ?? "",
|
|
8594
|
+
relation: table.relation,
|
|
8595
|
+
detail: column.type,
|
|
8596
|
+
apply: column.name,
|
|
8597
|
+
governance: "runtime_schema",
|
|
8598
|
+
});
|
|
8599
|
+
}
|
|
8600
|
+
}
|
|
8601
|
+
if (kind === "all" || kind === "relation") {
|
|
8602
|
+
const snapshot = latestRuntimeSchemaSnapshotForProject(projectRoot);
|
|
8603
|
+
for (const table of snapshot?.tables ?? [])
|
|
8604
|
+
candidates.push({
|
|
8605
|
+
type: "relation",
|
|
8606
|
+
name: table.relation,
|
|
8607
|
+
label: table.name ?? table.relation,
|
|
8608
|
+
description: table.description ?? "",
|
|
8609
|
+
relation: table.relation,
|
|
8610
|
+
detail: table.schema,
|
|
8611
|
+
apply: table.relation,
|
|
8612
|
+
governance: table.source ?? "runtime_schema",
|
|
8613
|
+
});
|
|
8614
|
+
}
|
|
8615
|
+
if (kind === "all" || kind === "block") {
|
|
8616
|
+
for (const file of scanNotebookFiles(projectRoot).filter((entry) => entry.type === "block"))
|
|
8617
|
+
candidates.push({
|
|
8618
|
+
type: "block",
|
|
8619
|
+
name: file.name.replace(/\.dql$/i, ""),
|
|
8620
|
+
label: file.name.replace(/\.dql$/i, ""),
|
|
8621
|
+
description: file.path,
|
|
8622
|
+
detail: "@block",
|
|
8623
|
+
apply: `@block("${file.name.replace(/\.dql$/i, "")}")`,
|
|
8624
|
+
governance: file.path.includes("_draft") ? "draft" : "governed",
|
|
8625
|
+
});
|
|
8626
|
+
}
|
|
8627
|
+
if (kind === "all" || kind === "connection") {
|
|
8628
|
+
for (const name of Object.keys(getStoredConnections(projectConfig)))
|
|
8629
|
+
candidates.push({
|
|
8630
|
+
type: "connection",
|
|
8631
|
+
name,
|
|
8632
|
+
label: name,
|
|
8633
|
+
description: "Configured connection",
|
|
8634
|
+
detail: name === projectConfig.defaultConnectionName
|
|
8635
|
+
? "default"
|
|
8636
|
+
: undefined,
|
|
8637
|
+
apply: name,
|
|
8638
|
+
});
|
|
8639
|
+
}
|
|
8640
|
+
const uniqueCandidates = [
|
|
8641
|
+
...new Map(candidates.map((candidate) => [
|
|
8642
|
+
`${candidate.type}:${candidate.name.toLowerCase()}:${candidate.relation?.toLowerCase() ?? ""}`,
|
|
8643
|
+
candidate,
|
|
8644
|
+
])).values(),
|
|
8645
|
+
];
|
|
8646
|
+
const scored = uniqueCandidates
|
|
8647
|
+
.map((candidate) => {
|
|
8648
|
+
const name = candidate.name.toLowerCase();
|
|
8649
|
+
const label = candidate.label.toLowerCase();
|
|
8650
|
+
const haystack = `${name} ${label} ${candidate.description.toLowerCase()} ${(candidate.tags ?? []).join(" ").toLowerCase()} ${candidate.domain?.toLowerCase() ?? ""}`;
|
|
8651
|
+
let score = query
|
|
8652
|
+
? name === query
|
|
8653
|
+
? 100
|
|
8654
|
+
: name.startsWith(query)
|
|
8655
|
+
? 80
|
|
8656
|
+
: label.startsWith(query)
|
|
8657
|
+
? 70
|
|
8658
|
+
: haystack.includes(query)
|
|
8659
|
+
? 40
|
|
8660
|
+
: 0
|
|
8661
|
+
: 10;
|
|
8662
|
+
if (selectedDomain &&
|
|
8663
|
+
candidate.domain?.toLowerCase() === selectedDomain)
|
|
8664
|
+
score += 20;
|
|
8665
|
+
if (recent.includes(name))
|
|
8666
|
+
score += 15;
|
|
8667
|
+
if (candidate.governance === "certified" ||
|
|
8668
|
+
candidate.governance === "semantic" ||
|
|
8669
|
+
candidate.governance === "project_controlled")
|
|
8670
|
+
score += 10;
|
|
8671
|
+
return { ...candidate, score };
|
|
8672
|
+
})
|
|
8673
|
+
.filter((candidate) => !query || (candidate.score ?? 0) > 0)
|
|
8674
|
+
.sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || a.name.localeCompare(b.name))
|
|
8675
|
+
.slice(0, limit);
|
|
8676
|
+
res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" });
|
|
8677
|
+
res.end(serializeJSON({
|
|
8678
|
+
completions: scored,
|
|
8679
|
+
query,
|
|
8680
|
+
kind,
|
|
8681
|
+
limit,
|
|
8682
|
+
totalCandidates: uniqueCandidates.length,
|
|
8683
|
+
}));
|
|
8684
|
+
return;
|
|
8685
|
+
}
|
|
8686
|
+
// ── Legacy semantic completion endpoint ────────────────────────────────────
|
|
8687
|
+
if (req.method === "GET" && path === "/api/semantic-completions") {
|
|
6776
8688
|
const completions = [];
|
|
6777
8689
|
if (semanticLayer) {
|
|
6778
8690
|
for (const m of semanticLayer.listMetrics()) {
|
|
@@ -6944,7 +8856,11 @@ export async function startLocalServer(opts) {
|
|
|
6944
8856
|
res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
|
|
6945
8857
|
return;
|
|
6946
8858
|
}
|
|
6947
|
-
const
|
|
8859
|
+
const executionConnection = await resolveExecutionConnection(body);
|
|
8860
|
+
const tableMapping = hasStandaloneSemanticRef(body.sql)
|
|
8861
|
+
? await resolveSemanticTableMapping(executor, executionConnection, semanticLayer)
|
|
8862
|
+
: undefined;
|
|
8863
|
+
const semantic = prepareSemanticSql(body.sql, semanticLayer, { tableMapping });
|
|
6948
8864
|
if (semantic.unresolvedRefs.length > 0) {
|
|
6949
8865
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
6950
8866
|
res.end(serializeJSON({
|
|
@@ -6956,7 +8872,7 @@ export async function startLocalServer(opts) {
|
|
|
6956
8872
|
}));
|
|
6957
8873
|
return;
|
|
6958
8874
|
}
|
|
6959
|
-
const prepared = prepareLocalExecution(semantic.sql,
|
|
8875
|
+
const prepared = prepareLocalExecution(semantic.sql, executionConnection, projectRoot, projectConfig);
|
|
6960
8876
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
6961
8877
|
const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
|
|
6962
8878
|
assertAppAccess({ app, domain, level: 'execute' });
|
|
@@ -7214,6 +9130,10 @@ export async function startLocalServer(opts) {
|
|
|
7214
9130
|
res.end(serializeJSON({ error: 'name and at least one metric are required.' }));
|
|
7215
9131
|
return;
|
|
7216
9132
|
}
|
|
9133
|
+
if (blockType === 'semantic' && dbtFirstOwnsSemanticAuthoring(projectConfig)) {
|
|
9134
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
9135
|
+
return;
|
|
9136
|
+
}
|
|
7217
9137
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
7218
9138
|
const composed = composeRuntimeSemanticQuery({
|
|
7219
9139
|
metrics,
|
|
@@ -7524,21 +9444,135 @@ export async function startLocalServer(opts) {
|
|
|
7524
9444
|
}));
|
|
7525
9445
|
return;
|
|
7526
9446
|
}
|
|
7527
|
-
if (req.method === 'GET' && path === '/api/notebook/file') {
|
|
7528
|
-
const relativePath = url.searchParams.get('path');
|
|
7529
|
-
if (!relativePath) {
|
|
7530
|
-
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7531
|
-
res.end(serializeJSON({ error: 'Missing file path.' }));
|
|
7532
|
-
return;
|
|
9447
|
+
if (req.method === 'GET' && path === '/api/notebook/file') {
|
|
9448
|
+
const relativePath = url.searchParams.get('path');
|
|
9449
|
+
if (!relativePath) {
|
|
9450
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9451
|
+
res.end(serializeJSON({ error: 'Missing file path.' }));
|
|
9452
|
+
return;
|
|
9453
|
+
}
|
|
9454
|
+
const filePath = safeJoin(projectRoot, relativePath);
|
|
9455
|
+
if (!filePath || !existsSync(filePath) || statSync(filePath).isDirectory()) {
|
|
9456
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9457
|
+
res.end(serializeJSON({ error: `File not found: ${relativePath}` }));
|
|
9458
|
+
return;
|
|
9459
|
+
}
|
|
9460
|
+
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
9461
|
+
res.end(readFileSync(filePath));
|
|
9462
|
+
return;
|
|
9463
|
+
}
|
|
9464
|
+
if (req.method === 'POST' && path === '/api/dql/artifacts/execute') {
|
|
9465
|
+
try {
|
|
9466
|
+
const body = await readJSON(req);
|
|
9467
|
+
const requestedArtifact = normalizeDqlArtifactReference(body.artifact);
|
|
9468
|
+
const requestedBlockName = typeof body.blockName === 'string' ? body.blockName.trim() : '';
|
|
9469
|
+
let source = requestedArtifact?.source ?? '';
|
|
9470
|
+
let sourcePath = requestedArtifact?.sourcePath;
|
|
9471
|
+
let blockName = requestedBlockName || requestedArtifact?.name;
|
|
9472
|
+
let domain;
|
|
9473
|
+
let chartType;
|
|
9474
|
+
if (requestedBlockName) {
|
|
9475
|
+
const block = buildManifest({ projectRoot }).blocks[requestedBlockName];
|
|
9476
|
+
if (!block)
|
|
9477
|
+
throw new Error(`Block "${requestedBlockName}" was not found.`);
|
|
9478
|
+
source = readFileSync(join(projectRoot, block.filePath), 'utf-8');
|
|
9479
|
+
sourcePath = block.filePath;
|
|
9480
|
+
blockName = block.name;
|
|
9481
|
+
domain = block.domain;
|
|
9482
|
+
chartType = block.chartType;
|
|
9483
|
+
}
|
|
9484
|
+
else if (sourcePath) {
|
|
9485
|
+
const filePath = safeJoin(projectRoot, sourcePath);
|
|
9486
|
+
if (filePath && existsSync(filePath) && !statSync(filePath).isDirectory())
|
|
9487
|
+
source = readFileSync(filePath, 'utf-8');
|
|
9488
|
+
}
|
|
9489
|
+
if (!source.trim())
|
|
9490
|
+
throw new Error('Provide an executable DQL artifact or block name.');
|
|
9491
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
9492
|
+
? body.parameters
|
|
9493
|
+
: requestedArtifact?.parameterValues ?? {};
|
|
9494
|
+
const result = await executeDqlArtifactSourceForAgent(source, {
|
|
9495
|
+
name: blockName,
|
|
9496
|
+
path: sourcePath,
|
|
9497
|
+
domain,
|
|
9498
|
+
chartType,
|
|
9499
|
+
}, {
|
|
9500
|
+
parameters,
|
|
9501
|
+
question: typeof body.question === 'string' ? body.question : undefined,
|
|
9502
|
+
});
|
|
9503
|
+
const contract = prepareBlockInvocation({ source, parameters, surface: 'ask_ai' });
|
|
9504
|
+
const certified = /\bstatus\s*=\s*"certified"/i.test(source);
|
|
9505
|
+
const semantic = /\btype\s*=\s*"semantic"/i.test(source);
|
|
9506
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9507
|
+
res.end(serializeJSON({
|
|
9508
|
+
result,
|
|
9509
|
+
artifact: {
|
|
9510
|
+
...(requestedArtifact ?? { kind: semantic ? 'semantic_block' : 'sql_block' }),
|
|
9511
|
+
source,
|
|
9512
|
+
...(blockName ? { name: blockName } : {}),
|
|
9513
|
+
...(sourcePath ? { sourcePath } : {}),
|
|
9514
|
+
parameters: contract.parameters,
|
|
9515
|
+
parameterValues: contract.values,
|
|
9516
|
+
persistence: sourcePath ? 'saved' : 'transient',
|
|
9517
|
+
trustState: certified ? 'certified' : semantic ? 'governed' : 'review_required',
|
|
9518
|
+
compiledSql: result.sql,
|
|
9519
|
+
},
|
|
9520
|
+
}));
|
|
9521
|
+
}
|
|
9522
|
+
catch (error) {
|
|
9523
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9524
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
9525
|
+
}
|
|
9526
|
+
return;
|
|
9527
|
+
}
|
|
9528
|
+
if (req.method === 'GET' && path === '/api/blocks/parameters') {
|
|
9529
|
+
try {
|
|
9530
|
+
const blockName = url.searchParams.get('name')?.trim();
|
|
9531
|
+
if (!blockName)
|
|
9532
|
+
throw new Error('Missing certified block name.');
|
|
9533
|
+
const manifest = buildManifest({ projectRoot });
|
|
9534
|
+
const block = manifest.blocks[blockName];
|
|
9535
|
+
if (!block) {
|
|
9536
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9537
|
+
res.end(serializeJSON({ error: `Block "${blockName}" was not found.` }));
|
|
9538
|
+
return;
|
|
9539
|
+
}
|
|
9540
|
+
if (block.status !== 'certified')
|
|
9541
|
+
throw new Error(`Block "${blockName}" is not certified.`);
|
|
9542
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9543
|
+
res.end(serializeJSON({
|
|
9544
|
+
blockName: block.name,
|
|
9545
|
+
blockPath: block.filePath,
|
|
9546
|
+
parameters: block.parameters ?? [],
|
|
9547
|
+
}));
|
|
9548
|
+
}
|
|
9549
|
+
catch (error) {
|
|
9550
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9551
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
9552
|
+
}
|
|
9553
|
+
return;
|
|
9554
|
+
}
|
|
9555
|
+
if (req.method === 'POST' && path === '/api/blocks/invoke') {
|
|
9556
|
+
try {
|
|
9557
|
+
const body = await readJSON(req);
|
|
9558
|
+
const blockName = typeof body.blockName === 'string' ? body.blockName.trim() : '';
|
|
9559
|
+
if (!blockName)
|
|
9560
|
+
throw new Error('Missing certified block name.');
|
|
9561
|
+
const parameters = body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
9562
|
+
? body.parameters
|
|
9563
|
+
: {};
|
|
9564
|
+
const result = await executeCertifiedBlockByNameForAgent(blockName, { parameters, question: typeof body.question === 'string' ? body.question : undefined }, true);
|
|
9565
|
+
const manifest = buildManifest({ projectRoot });
|
|
9566
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9567
|
+
res.end(serializeJSON({
|
|
9568
|
+
result,
|
|
9569
|
+
parameters: manifest.blocks[blockName]?.parameters ?? [],
|
|
9570
|
+
}));
|
|
7533
9571
|
}
|
|
7534
|
-
|
|
7535
|
-
|
|
7536
|
-
res.
|
|
7537
|
-
res.end(serializeJSON({ error: `File not found: ${relativePath}` }));
|
|
7538
|
-
return;
|
|
9572
|
+
catch (error) {
|
|
9573
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9574
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
7539
9575
|
}
|
|
7540
|
-
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
7541
|
-
res.end(readFileSync(filePath));
|
|
7542
9576
|
return;
|
|
7543
9577
|
}
|
|
7544
9578
|
if (req.method === 'POST' && path === '/api/notebook/execute') {
|
|
@@ -7556,11 +9590,30 @@ export async function startLocalServer(opts) {
|
|
|
7556
9590
|
}
|
|
7557
9591
|
const resolved = resolveNotebookBlockReferenceCell(cell, projectRoot);
|
|
7558
9592
|
const executableCell = resolved.cell;
|
|
7559
|
-
const
|
|
9593
|
+
const invocation = executableCell.type === 'dql'
|
|
9594
|
+
? prepareBlockInvocation({
|
|
9595
|
+
source: executableCell.source,
|
|
9596
|
+
parameters: body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
9597
|
+
? body.parameters
|
|
9598
|
+
: {},
|
|
9599
|
+
surface: 'notebook',
|
|
9600
|
+
})
|
|
9601
|
+
: null;
|
|
9602
|
+
if (invocation?.errors.length)
|
|
9603
|
+
throw new Error(invocation.errors.join(' '));
|
|
9604
|
+
if (invocation?.unresolvedParameters.length) {
|
|
9605
|
+
throw new Error(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'}: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
9606
|
+
}
|
|
9607
|
+
const cellConnection = await resolveExecutionConnection(body);
|
|
7560
9608
|
const tableMapping = needsSemanticTableMapping(executableCell)
|
|
7561
9609
|
? await resolveSemanticTableMapping(executor, cellConnection, semanticLayer)
|
|
7562
9610
|
: undefined;
|
|
7563
|
-
const plan = buildExecutionPlan(executableCell, {
|
|
9611
|
+
const plan = buildExecutionPlan(executableCell, {
|
|
9612
|
+
semanticLayer,
|
|
9613
|
+
driver: cellConnection.driver,
|
|
9614
|
+
tableMapping,
|
|
9615
|
+
parameters: invocation?.values,
|
|
9616
|
+
});
|
|
7564
9617
|
if (!plan) {
|
|
7565
9618
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7566
9619
|
res.end(serializeJSON({ cellType: cell.type, result: null }));
|
|
@@ -7569,7 +9622,7 @@ export async function startLocalServer(opts) {
|
|
|
7569
9622
|
const prepared = prepareLocalExecution(plan.sql, cellConnection, projectRoot, projectConfig);
|
|
7570
9623
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
7571
9624
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
7572
|
-
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
|
|
9625
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables({ ...plan.variables, ...(invocation?.values ?? {}) }), prepared.connection);
|
|
7573
9626
|
const normalized = normalizeQueryResult(rawResult);
|
|
7574
9627
|
// Enforce the block's declared invariants against the result set. This
|
|
7575
9628
|
// is additive: blocks without invariants produce `null` and the
|
|
@@ -7607,6 +9660,13 @@ export async function startLocalServer(opts) {
|
|
|
7607
9660
|
chartConfig: plan.chartConfig,
|
|
7608
9661
|
tests: plan.tests,
|
|
7609
9662
|
result: normalized,
|
|
9663
|
+
...(invocation ? {
|
|
9664
|
+
invocation: {
|
|
9665
|
+
resolvedParameters: invocation.resolvedParameters,
|
|
9666
|
+
unresolvedParameters: invocation.unresolvedParameters,
|
|
9667
|
+
auditId: invocation.auditId,
|
|
9668
|
+
},
|
|
9669
|
+
} : {}),
|
|
7610
9670
|
...(invariants
|
|
7611
9671
|
? {
|
|
7612
9672
|
invariantResults: invariants.invariantResults,
|
|
@@ -7640,6 +9700,10 @@ export async function startLocalServer(opts) {
|
|
|
7640
9700
|
// Create a new metric YAML file in semantic-layer/metrics/
|
|
7641
9701
|
if (req.method === 'POST' && path === '/api/semantic-layer/metric') {
|
|
7642
9702
|
try {
|
|
9703
|
+
if (dbtFirstOwnsSemanticAuthoring(projectConfig)) {
|
|
9704
|
+
sendDbtOwnedSemanticAuthoringError(res);
|
|
9705
|
+
return;
|
|
9706
|
+
}
|
|
7643
9707
|
const body = await readJSON(req);
|
|
7644
9708
|
const { name, label, description, domain, sql, type, table, tags } = body;
|
|
7645
9709
|
if (!name || !sql || !type || !table) {
|
|
@@ -7709,6 +9773,148 @@ table: ${table}${tagList}
|
|
|
7709
9773
|
});
|
|
7710
9774
|
});
|
|
7711
9775
|
}
|
|
9776
|
+
async function validateModelingRelationship(relationship, manifest, execute, quoteIdentifier = quoteSqlIdentifier) {
|
|
9777
|
+
const fromEntity = manifest.modeling?.entities[relationship.from];
|
|
9778
|
+
const toEntity = manifest.modeling?.entities[relationship.to];
|
|
9779
|
+
if (!fromEntity || !toEntity)
|
|
9780
|
+
throw new Error('Both relationship entities must be bound to dbt models before validation.');
|
|
9781
|
+
const fromNode = manifest.dbtProvenance?.nodes[fromEntity.dbtUniqueId];
|
|
9782
|
+
const toNode = manifest.dbtProvenance?.nodes[toEntity.dbtUniqueId];
|
|
9783
|
+
if (!fromNode?.relation || !toNode?.relation)
|
|
9784
|
+
throw new Error('Both dbt models must expose warehouse relation names.');
|
|
9785
|
+
if (!relationship.keys.length)
|
|
9786
|
+
throw new Error('At least one join key pair is required.');
|
|
9787
|
+
const safeQuote = (identifier) => quoteIdentifier(assertSafeSqlIdentifier(identifier));
|
|
9788
|
+
const fromRelation = quoteQualifiedIdentifier(fromNode.relation, safeQuote);
|
|
9789
|
+
const toRelation = quoteQualifiedIdentifier(toNode.relation, safeQuote);
|
|
9790
|
+
const fromNull = relationship.keys.map((key) => `f.${safeQuote(key.from)} IS NULL`).join(' OR ');
|
|
9791
|
+
const toNull = relationship.keys.map((key) => `t.${safeQuote(key.to)} IS NULL`).join(' OR ');
|
|
9792
|
+
const join = relationship.keys.map((key) => `f.${safeQuote(key.from)} = t.${safeQuote(key.to)}`).join(' AND ');
|
|
9793
|
+
const fromKeys = relationship.keys.map((key) => safeQuote(key.from)).join(', ');
|
|
9794
|
+
const toKeys = relationship.keys.map((key) => safeQuote(key.to)).join(', ');
|
|
9795
|
+
const firstToKey = safeQuote(relationship.keys[0].to);
|
|
9796
|
+
const sql = `WITH
|
|
9797
|
+
from_counts AS (SELECT COUNT(*) AS rows, SUM(CASE WHEN ${fromNull} THEN 1 ELSE 0 END) AS null_keys FROM ${fromRelation} f),
|
|
9798
|
+
to_counts AS (SELECT COUNT(*) AS rows, SUM(CASE WHEN ${toNull} THEN 1 ELSE 0 END) AS null_keys FROM ${toRelation} t),
|
|
9799
|
+
from_max AS (SELECT COALESCE(MAX(key_count), 0) AS max_per_key FROM (SELECT COUNT(*) AS key_count FROM ${fromRelation} GROUP BY ${fromKeys}) x),
|
|
9800
|
+
to_max AS (SELECT COALESCE(MAX(key_count), 0) AS max_per_key FROM (SELECT COUNT(*) AS key_count FROM ${toRelation} GROUP BY ${toKeys}) x),
|
|
9801
|
+
joined AS (SELECT COUNT(*) AS rows FROM ${fromRelation} f JOIN ${toRelation} t ON ${join}),
|
|
9802
|
+
unmatched AS (SELECT COUNT(*) AS rows FROM ${fromRelation} f LEFT JOIN ${toRelation} t ON ${join} WHERE t.${firstToKey} IS NULL)
|
|
9803
|
+
SELECT from_counts.rows AS from_rows, to_counts.rows AS to_rows, joined.rows AS joined_rows,
|
|
9804
|
+
from_counts.null_keys AS from_null_keys, to_counts.null_keys AS to_null_keys,
|
|
9805
|
+
unmatched.rows AS unmatched_from, from_max.max_per_key AS max_from_per_key, to_max.max_per_key AS max_to_per_key
|
|
9806
|
+
FROM from_counts, to_counts, joined, unmatched, from_max, to_max`;
|
|
9807
|
+
const result = await execute(sql);
|
|
9808
|
+
const row = result.rows[0] ?? {};
|
|
9809
|
+
const maxFromPerKey = numericValue(row.max_from_per_key);
|
|
9810
|
+
const maxToPerKey = numericValue(row.max_to_per_key);
|
|
9811
|
+
const cardinalityPassed = relationship.cardinality === 'one_to_one'
|
|
9812
|
+
? maxFromPerKey <= 1 && maxToPerKey <= 1
|
|
9813
|
+
: relationship.cardinality === 'one_to_many'
|
|
9814
|
+
? maxFromPerKey <= 1
|
|
9815
|
+
: relationship.cardinality === 'many_to_one'
|
|
9816
|
+
? maxToPerKey <= 1
|
|
9817
|
+
: false;
|
|
9818
|
+
const policyPassed = relationship.fanout === 'safe' && cardinalityPassed;
|
|
9819
|
+
const queryFingerprint = createHash('sha256').update(sql).digest('hex');
|
|
9820
|
+
return {
|
|
9821
|
+
status: policyPassed ? 'passed' : 'failed',
|
|
9822
|
+
checkedAt: new Date().toISOString(),
|
|
9823
|
+
queryFingerprint,
|
|
9824
|
+
proofFingerprint: relationshipValidationProofFingerprint({
|
|
9825
|
+
fromRelation: fromNode.relation,
|
|
9826
|
+
toRelation: toNode.relation,
|
|
9827
|
+
keys: relationship.keys,
|
|
9828
|
+
cardinality: relationship.cardinality,
|
|
9829
|
+
fanout: relationship.fanout,
|
|
9830
|
+
queryFingerprint,
|
|
9831
|
+
}),
|
|
9832
|
+
fromRows: numericValue(row.from_rows),
|
|
9833
|
+
toRows: numericValue(row.to_rows),
|
|
9834
|
+
joinedRows: numericValue(row.joined_rows),
|
|
9835
|
+
fromNullKeys: numericValue(row.from_null_keys),
|
|
9836
|
+
toNullKeys: numericValue(row.to_null_keys),
|
|
9837
|
+
unmatchedFrom: numericValue(row.unmatched_from),
|
|
9838
|
+
maxFromPerKey,
|
|
9839
|
+
maxToPerKey,
|
|
9840
|
+
message: policyPassed
|
|
9841
|
+
? 'Warehouse evidence matches the declared cardinality and safe fanout policy.'
|
|
9842
|
+
: 'Warehouse evidence does not prove the declared cardinality and safe fanout policy.',
|
|
9843
|
+
};
|
|
9844
|
+
}
|
|
9845
|
+
function relatedProductsForDomain(manifest, domainId) {
|
|
9846
|
+
const apps = Object.values(manifest.apps ?? {})
|
|
9847
|
+
.filter((app) => app.ownerDomain === domainId || app.usesDomains.includes(domainId))
|
|
9848
|
+
.map((app) => ({
|
|
9849
|
+
id: app.id,
|
|
9850
|
+
name: app.name,
|
|
9851
|
+
filePath: app.filePath,
|
|
9852
|
+
ownerDomain: app.ownerDomain ?? app.domain,
|
|
9853
|
+
usesDomains: app.usesDomains,
|
|
9854
|
+
purpose: app.purpose,
|
|
9855
|
+
requiredExports: app.requiredExports,
|
|
9856
|
+
classification: app.classification,
|
|
9857
|
+
lifecycle: app.lifecycle,
|
|
9858
|
+
legacyLayout: app.filePath.startsWith('domains/'),
|
|
9859
|
+
}));
|
|
9860
|
+
const notebooks = Object.entries(manifest.notebooks)
|
|
9861
|
+
.filter(([, notebook]) => notebook.ownerDomain === domainId || notebook.usesDomains.includes(domainId))
|
|
9862
|
+
.map(([id, notebook]) => ({
|
|
9863
|
+
id,
|
|
9864
|
+
title: notebook.title,
|
|
9865
|
+
filePath: notebook.filePath,
|
|
9866
|
+
ownerDomain: notebook.ownerDomain,
|
|
9867
|
+
usesDomains: notebook.usesDomains,
|
|
9868
|
+
purpose: notebook.purpose,
|
|
9869
|
+
requiredExports: notebook.requiredExports,
|
|
9870
|
+
classification: notebook.classification,
|
|
9871
|
+
legacyLayout: notebook.filePath.startsWith('domains/'),
|
|
9872
|
+
}));
|
|
9873
|
+
return { apps, notebooks };
|
|
9874
|
+
}
|
|
9875
|
+
function domainWorkspaceSummary(manifest, domainId) {
|
|
9876
|
+
const pkg = manifest.modeling?.packages[domainId];
|
|
9877
|
+
const entities = Object.values(manifest.modeling?.entities ?? {}).filter((entity) => entity.domain === domainId);
|
|
9878
|
+
const entityKeys = new Set(Object.entries(manifest.modeling?.entities ?? {}).filter(([, entity]) => entity.domain === domainId).map(([key]) => key));
|
|
9879
|
+
const relationships = Object.values(manifest.modeling?.relationships ?? {}).filter((relationship) => entityKeys.has(relationship.from) || entityKeys.has(relationship.to));
|
|
9880
|
+
const contracts = Object.values(manifest.modeling?.contracts ?? {}).filter((contract) => contract.domain === domainId);
|
|
9881
|
+
const imports = Object.values(manifest.modeling?.interfaces?.imports ?? {}).filter((value) => value.domain === domainId);
|
|
9882
|
+
const exports = Object.values(manifest.modeling?.interfaces?.exports ?? {}).filter((value) => value.domain === domainId);
|
|
9883
|
+
return {
|
|
9884
|
+
id: domainId,
|
|
9885
|
+
parent: pkg?.parent,
|
|
9886
|
+
owner: pkg?.owner,
|
|
9887
|
+
filePath: pkg?.filePath,
|
|
9888
|
+
counts: {
|
|
9889
|
+
entities: entities.length,
|
|
9890
|
+
relationships: relationships.length,
|
|
9891
|
+
safeRelationships: relationships.filter((relationship) => relationship.automaticJoinAllowed).length,
|
|
9892
|
+
staleRelationships: relationships.filter((relationship) => relationship.staleCertification).length,
|
|
9893
|
+
contracts: contracts.length,
|
|
9894
|
+
imports: imports.length,
|
|
9895
|
+
exports: exports.length,
|
|
9896
|
+
relatedProducts: Object.values(relatedProductsForDomain(manifest, domainId)).reduce((sum, values) => sum + values.length, 0),
|
|
9897
|
+
},
|
|
9898
|
+
readiness: relationships.some((relationship) => relationship.staleCertification) ? 'attention' : entities.length > 0 ? 'ready' : 'setup',
|
|
9899
|
+
};
|
|
9900
|
+
}
|
|
9901
|
+
function quoteQualifiedIdentifier(value, quote = quoteSqlIdentifier) {
|
|
9902
|
+
return value.split('.').map((part) => quote(assertSafeSqlIdentifier(part))).join('.');
|
|
9903
|
+
}
|
|
9904
|
+
function quoteSqlIdentifier(value) {
|
|
9905
|
+
return `"${value.replace(/"/g, '""')}"`;
|
|
9906
|
+
}
|
|
9907
|
+
function assertSafeSqlIdentifier(value) {
|
|
9908
|
+
if (!value || !/^[A-Za-z_][A-Za-z0-9_$]*$/.test(value))
|
|
9909
|
+
throw new Error(`Unsafe SQL identifier: ${value}`);
|
|
9910
|
+
return value;
|
|
9911
|
+
}
|
|
9912
|
+
function numericValue(value) {
|
|
9913
|
+
if (typeof value === 'bigint')
|
|
9914
|
+
return Number(value);
|
|
9915
|
+
const numeric = Number(value ?? 0);
|
|
9916
|
+
return Number.isFinite(numeric) ? numeric : 0;
|
|
9917
|
+
}
|
|
7712
9918
|
export async function assertLocalQueryRuntimeReady(executor, connection) {
|
|
7713
9919
|
try {
|
|
7714
9920
|
const connector = await executor.getConnector(connection);
|
|
@@ -8010,6 +10216,7 @@ function serializeSkill(skill) {
|
|
|
8010
10216
|
user: undefined,
|
|
8011
10217
|
domain: skill.domain,
|
|
8012
10218
|
domains: skill.domains,
|
|
10219
|
+
modelAreaRefs: skill.modelAreaRefs,
|
|
8013
10220
|
kind: skill.kind,
|
|
8014
10221
|
status: skill.status,
|
|
8015
10222
|
owner: skill.owner,
|
|
@@ -8063,6 +10270,7 @@ function parseSkillInput(raw, fallbackId) {
|
|
|
8063
10270
|
user: undefined,
|
|
8064
10271
|
domain: typeof skill.domain === 'string' && skill.domain.trim() ? skill.domain.trim() : undefined,
|
|
8065
10272
|
domains: asStrings(skill.domains),
|
|
10273
|
+
modelAreaRefs: asStrings(skill.modelAreaRefs),
|
|
8066
10274
|
kind: skill.kind === 'domain_reference' || skill.kind === 'metric_policy' || skill.kind === 'glossary' || skill.kind === 'analysis_pattern' || skill.kind === 'sql_policy' ? skill.kind : 'custom',
|
|
8067
10275
|
status: skill.status === 'draft' || skill.status === 'deprecated' ? skill.status : 'active',
|
|
8068
10276
|
owner: typeof skill.owner === 'string' ? skill.owner : undefined,
|
|
@@ -8089,7 +10297,7 @@ function domainKey(value) {
|
|
|
8089
10297
|
}
|
|
8090
10298
|
function manifestDomainToDto(domain, counts) {
|
|
8091
10299
|
return {
|
|
8092
|
-
id: domain.name,
|
|
10300
|
+
id: domain.id ?? domain.name,
|
|
8093
10301
|
name: domain.name,
|
|
8094
10302
|
parent: domain.parent,
|
|
8095
10303
|
owner: domain.owner,
|
|
@@ -8139,7 +10347,7 @@ export function listDomains(projectRoot) {
|
|
|
8139
10347
|
}
|
|
8140
10348
|
return Object.values(domains)
|
|
8141
10349
|
.map((domain) => {
|
|
8142
|
-
const key = domainKey(domain.name);
|
|
10350
|
+
const key = domainKey(domain.id ?? domain.name);
|
|
8143
10351
|
return manifestDomainToDto(domain, {
|
|
8144
10352
|
blockCount: blockCounts.get(key) ?? 0,
|
|
8145
10353
|
skillCount: skillCounts.get(key) ?? 0,
|
|
@@ -8151,7 +10359,56 @@ export function listDomains(projectRoot) {
|
|
|
8151
10359
|
/** Find a single authored domain by name/id (case/slug-insensitive). */
|
|
8152
10360
|
function findDomain(projectRoot, nameOrId) {
|
|
8153
10361
|
const key = domainKey(nameOrId);
|
|
8154
|
-
return listDomains(projectRoot).find((domain) => domainKey(domain.name) === key);
|
|
10362
|
+
return listDomains(projectRoot).find((domain) => domainKey(domain.id) === key || domainKey(domain.name) === key);
|
|
10363
|
+
}
|
|
10364
|
+
function deleteCanonicalDomainDeclaration(projectRoot, nameOrId) {
|
|
10365
|
+
const registry = loadDomainPackageRegistry(projectRoot);
|
|
10366
|
+
const key = domainKey(nameOrId);
|
|
10367
|
+
const pkg = registry.values().find((candidate) => domainKey(candidate.id) === key || domainKey(candidate.name) === key);
|
|
10368
|
+
if (!pkg)
|
|
10369
|
+
return false;
|
|
10370
|
+
const declaration = join(projectRoot, pkg.declarationPath);
|
|
10371
|
+
if (existsSync(declaration))
|
|
10372
|
+
rmSync(declaration, { force: true });
|
|
10373
|
+
if (pkg.legacyYamlPath) {
|
|
10374
|
+
const legacy = join(projectRoot, pkg.legacyYamlPath);
|
|
10375
|
+
if (existsSync(legacy))
|
|
10376
|
+
rmSync(legacy, { force: true });
|
|
10377
|
+
}
|
|
10378
|
+
return true;
|
|
10379
|
+
}
|
|
10380
|
+
async function refreshUnifiedProjectIndexes(projectRoot) {
|
|
10381
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot) ?? undefined;
|
|
10382
|
+
const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook', dbtManifestPath });
|
|
10383
|
+
invalidateAgentProjectState(projectRoot);
|
|
10384
|
+
await reindexProject(projectRoot, {
|
|
10385
|
+
manifest,
|
|
10386
|
+
kgPath: defaultKgPath(projectRoot),
|
|
10387
|
+
forceMetadataCatalog: true,
|
|
10388
|
+
forceKgIndex: true,
|
|
10389
|
+
});
|
|
10390
|
+
}
|
|
10391
|
+
function collectDomainPackageAssets(projectRoot, registry) {
|
|
10392
|
+
const output = {};
|
|
10393
|
+
for (const pkg of registry.values())
|
|
10394
|
+
output[pkg.id] = {};
|
|
10395
|
+
for (const path of listProjectFiles(projectRoot).filter((candidate) => candidate.startsWith('domains/'))) {
|
|
10396
|
+
const pkg = registry.packageForPath(join(projectRoot, path));
|
|
10397
|
+
if (!pkg || path === pkg.declarationPath)
|
|
10398
|
+
continue;
|
|
10399
|
+
const local = relative(pkg.root, join(projectRoot, path)).replace(/\\/g, '/');
|
|
10400
|
+
const top = local.split('/')[0] ?? '';
|
|
10401
|
+
const kind = top === 'business-views' ? 'views' : top;
|
|
10402
|
+
if (!['terms', 'skills', 'blocks', 'views', 'notebooks', 'apps', 'evaluations', 'tests'].includes(kind))
|
|
10403
|
+
continue;
|
|
10404
|
+
const bucket = output[pkg.id] ?? (output[pkg.id] = {});
|
|
10405
|
+
(bucket[kind] ??= []).push(path);
|
|
10406
|
+
}
|
|
10407
|
+
for (const assets of Object.values(output)) {
|
|
10408
|
+
for (const paths of Object.values(assets))
|
|
10409
|
+
paths.sort();
|
|
10410
|
+
}
|
|
10411
|
+
return output;
|
|
8155
10412
|
}
|
|
8156
10413
|
/** Validate + normalize an inbound `{ domain }` body into a DomainInput. */
|
|
8157
10414
|
export function parseDomainInput(raw, fallbackId) {
|
|
@@ -8169,6 +10426,7 @@ export function parseDomainInput(raw, fallbackId) {
|
|
|
8169
10426
|
? value.filter((v) => typeof v === 'string' && v.trim().length > 0)
|
|
8170
10427
|
: undefined;
|
|
8171
10428
|
return {
|
|
10429
|
+
id: typeof domain.id === 'string' && domain.id.trim() ? domain.id.trim() : domainFolderSlug(name).replace(/-/g, '_'),
|
|
8172
10430
|
name,
|
|
8173
10431
|
parent: typeof domain.parent === 'string' ? domain.parent : undefined,
|
|
8174
10432
|
owner: typeof domain.owner === 'string' ? domain.owner : undefined,
|
|
@@ -8405,6 +10663,17 @@ export function loadProjectConfig(projectRoot) {
|
|
|
8405
10663
|
}
|
|
8406
10664
|
return config;
|
|
8407
10665
|
}
|
|
10666
|
+
function dbtFirstOwnsSemanticAuthoring(config) {
|
|
10667
|
+
return config.manifestVersion === 3 && config.modeling?.mode === 'dbt-first';
|
|
10668
|
+
}
|
|
10669
|
+
function sendDbtOwnedSemanticAuthoringError(res) {
|
|
10670
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10671
|
+
res.end(serializeJSON({
|
|
10672
|
+
error: 'dbt-first projects use dbt/MetricFlow as the physical and metric source of truth. Edit the dbt project and refresh artifacts instead of creating duplicate local semantic definitions.',
|
|
10673
|
+
code: 'DQL_MODELING_DBT_OWNED',
|
|
10674
|
+
nextAction: 'Create or edit the metric in dbt/MetricFlow, run dbt parse or dbt build, then run dql sync dbt.',
|
|
10675
|
+
}));
|
|
10676
|
+
}
|
|
8408
10677
|
/**
|
|
8409
10678
|
* Core of the `/api/propose` endpoint, factored out as a pure function so it can
|
|
8410
10679
|
* be unit-tested without standing up an HTTP server.
|
|
@@ -9359,6 +11628,38 @@ export function dashboardRuntimeVariables(dashboard, overrides = {}) {
|
|
|
9359
11628
|
}
|
|
9360
11629
|
return { ...variables, ...overrides };
|
|
9361
11630
|
}
|
|
11631
|
+
/**
|
|
11632
|
+
* Resolve only declared tile parameter bindings. Dashboard filters remain
|
|
11633
|
+
* values; they never become SQL identifiers or an implicit name-based binding.
|
|
11634
|
+
*/
|
|
11635
|
+
export function dashboardTileParameterValues(input) {
|
|
11636
|
+
const values = {};
|
|
11637
|
+
const requestValues = input.requestValues ?? {};
|
|
11638
|
+
const persona = requestValues.persona && typeof requestValues.persona === 'object' && !Array.isArray(requestValues.persona)
|
|
11639
|
+
? requestValues.persona
|
|
11640
|
+
: {};
|
|
11641
|
+
for (const binding of input.item.parameterBindings ?? []) {
|
|
11642
|
+
const lookup = binding.filter ?? binding.field ?? binding.param;
|
|
11643
|
+
let value;
|
|
11644
|
+
switch (binding.source) {
|
|
11645
|
+
case 'constant':
|
|
11646
|
+
value = binding.value;
|
|
11647
|
+
break;
|
|
11648
|
+
case 'dashboard_filter':
|
|
11649
|
+
value = input.dashboardValues[lookup];
|
|
11650
|
+
break;
|
|
11651
|
+
case 'persona':
|
|
11652
|
+
value = persona[lookup] ?? requestValues[`persona.${lookup}`];
|
|
11653
|
+
break;
|
|
11654
|
+
case 'variable':
|
|
11655
|
+
value = requestValues[lookup];
|
|
11656
|
+
break;
|
|
11657
|
+
}
|
|
11658
|
+
if (value !== undefined)
|
|
11659
|
+
values[binding.param] = value;
|
|
11660
|
+
}
|
|
11661
|
+
return values;
|
|
11662
|
+
}
|
|
9362
11663
|
export function applyDashboardFiltersToBlockExecution(input) {
|
|
9363
11664
|
const variables = { ...input.variables };
|
|
9364
11665
|
const sqlParams = [...input.sqlParams];
|
|
@@ -9723,6 +12024,297 @@ export function buildAgentPreviewSql(sql) {
|
|
|
9723
12024
|
throw new Error(readOnlyError);
|
|
9724
12025
|
return `SELECT * FROM (\n${withoutTrailingSemicolon}\n) AS dql_agent_preview LIMIT 200`;
|
|
9725
12026
|
}
|
|
12027
|
+
/**
|
|
12028
|
+
* EXP-001 preflight for model-authored exploratory SQL.
|
|
12029
|
+
*
|
|
12030
|
+
* This is deliberately narrow and deterministic: it repairs a singular/plural
|
|
12031
|
+
* qualifier typo only when it resolves to exactly one relation already present
|
|
12032
|
+
* in the query, and it changes SUM to MAX for a non-additive parent attribute
|
|
12033
|
+
* only when the owning entity is retained in GROUP BY. It never invents a
|
|
12034
|
+
* relation, join key, or allocation rule.
|
|
12035
|
+
*/
|
|
12036
|
+
export function repairExploratorySqlBeforeExecution(sql, schemaContext, question = '') {
|
|
12037
|
+
const repairs = [];
|
|
12038
|
+
let repairedSql = qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs);
|
|
12039
|
+
repairedSql = repairExploratoryRelationQualifiers(repairedSql, repairs);
|
|
12040
|
+
repairedSql = repairExploratoryLifetimeMeasureSelection(repairedSql, schemaContext, question, repairs);
|
|
12041
|
+
repairedSql = repairExploratoryMisleadingPercentAliases(repairedSql, question, repairs);
|
|
12042
|
+
const grainRepair = repairExploratoryNonAdditiveAggregates(repairedSql, schemaContext, repairs);
|
|
12043
|
+
repairedSql = grainRepair.sql;
|
|
12044
|
+
return {
|
|
12045
|
+
sql: repairedSql,
|
|
12046
|
+
repairs,
|
|
12047
|
+
...(grainRepair.blockedReason ? { blockedReason: grainRepair.blockedReason } : {}),
|
|
12048
|
+
};
|
|
12049
|
+
}
|
|
12050
|
+
/**
|
|
12051
|
+
* Prevent a generated amount/count from being presented as a percentage.
|
|
12052
|
+
* The repair is intentionally conservative: percent-intent questions and
|
|
12053
|
+
* expressions that visibly calculate a ratio/percentage are left untouched.
|
|
12054
|
+
*/
|
|
12055
|
+
function repairExploratoryMisleadingPercentAliases(sql, question, repairs) {
|
|
12056
|
+
if (/\b(percent(?:age)?|pct|share|ratio|rate|proportion)\b/i.test(question))
|
|
12057
|
+
return sql;
|
|
12058
|
+
const aggregateAlias = /((?:SUM|AVG|COUNT|MIN|MAX)\s*\([^)]*\))\s+AS\s+([a-zA-Z_][a-zA-Z0-9_]*?)(?:_pct|_percent|_percentage)\b/gi;
|
|
12059
|
+
const aliases = new Map();
|
|
12060
|
+
for (const match of sql.matchAll(aggregateAlias)) {
|
|
12061
|
+
const expression = match[1];
|
|
12062
|
+
const oldAlias = match[0].match(/\bAS\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*$/i)?.[1];
|
|
12063
|
+
const newAlias = match[2];
|
|
12064
|
+
if (!oldAlias || !newAlias)
|
|
12065
|
+
continue;
|
|
12066
|
+
if (/\/|\bpercent(?:age)?\b|\bratio\b|\brate\b|\*\s*100\b|\b100\s*\*/i.test(expression))
|
|
12067
|
+
continue;
|
|
12068
|
+
aliases.set(oldAlias, newAlias);
|
|
12069
|
+
}
|
|
12070
|
+
let repaired = sql;
|
|
12071
|
+
for (const [oldAlias, newAlias] of aliases) {
|
|
12072
|
+
repaired = repaired.replace(new RegExp(`\\b${escapeExploratoryRegex(oldAlias)}\\b`, 'g'), newAlias);
|
|
12073
|
+
repairs.push(`Renamed misleading percentage alias ${oldAlias} to ${newAlias}; the expression returns an amount, not a percentage.`);
|
|
12074
|
+
}
|
|
12075
|
+
return repaired;
|
|
12076
|
+
}
|
|
12077
|
+
/**
|
|
12078
|
+
* EXP-003 — make the typed overall ranking contract executable before the host
|
|
12079
|
+
* runs a review-required exploratory candidate. The answer-shape gate remains a
|
|
12080
|
+
* backstop, but a provider's generic LIMIT 100 must not trigger a second model
|
|
12081
|
+
* call when the planner already established top 10.
|
|
12082
|
+
*/
|
|
12083
|
+
export function applyRequestedTopNToExploratorySql(sql, requestedTopN) {
|
|
12084
|
+
if (!Number.isInteger(requestedTopN) || requestedTopN <= 0)
|
|
12085
|
+
return sql;
|
|
12086
|
+
const withoutTerminator = sql.trim().replace(/;\s*$/, '');
|
|
12087
|
+
const trailingLimit = /\blimit\s+\d+\s*$/i;
|
|
12088
|
+
if (trailingLimit.test(withoutTerminator)) {
|
|
12089
|
+
return withoutTerminator.replace(trailingLimit, `LIMIT ${requestedTopN}`);
|
|
12090
|
+
}
|
|
12091
|
+
return `${withoutTerminator}\nLIMIT ${requestedTopN}`;
|
|
12092
|
+
}
|
|
12093
|
+
function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs) {
|
|
12094
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12095
|
+
if (!analysis.parsed)
|
|
12096
|
+
return sql;
|
|
12097
|
+
let repaired = sql;
|
|
12098
|
+
for (const relation of analysis.tables) {
|
|
12099
|
+
const cleanRelation = relation.replace(/["`\[\]]/g, '');
|
|
12100
|
+
if (cleanRelation.includes('.'))
|
|
12101
|
+
continue;
|
|
12102
|
+
const basename = cleanRelation.toLowerCase();
|
|
12103
|
+
const candidates = schemaContext
|
|
12104
|
+
.map((table) => table.relation)
|
|
12105
|
+
.filter((candidate) => (candidate.split('.').at(-1) ?? candidate).toLowerCase() === basename)
|
|
12106
|
+
.filter((candidate, index, values) => values.indexOf(candidate) === index);
|
|
12107
|
+
if (candidates.length === 0)
|
|
12108
|
+
continue;
|
|
12109
|
+
const compatible = candidates.every((candidate) => candidates.every((other) => candidate.toLowerCase() === other.toLowerCase()
|
|
12110
|
+
|| candidate.toLowerCase().endsWith(`.${other.toLowerCase()}`)
|
|
12111
|
+
|| other.toLowerCase().endsWith(`.${candidate.toLowerCase()}`)));
|
|
12112
|
+
if (!compatible)
|
|
12113
|
+
continue;
|
|
12114
|
+
const replacement = [...candidates].sort((left, right) => right.split('.').length - left.split('.').length)[0];
|
|
12115
|
+
if (replacement.toLowerCase() === basename)
|
|
12116
|
+
continue;
|
|
12117
|
+
repaired = repaired.replace(new RegExp(`\\b(from|join)\\s+${escapeExploratoryRegex(cleanRelation)}\\b`, 'gi'), (_match, keyword) => `${keyword} ${replacement}`);
|
|
12118
|
+
repairs.push(`Qualified exploratory relation ${cleanRelation} as inspected relation ${replacement}.`);
|
|
12119
|
+
}
|
|
12120
|
+
return repaired;
|
|
12121
|
+
}
|
|
12122
|
+
function repairExploratoryRelationQualifiers(sql, repairs) {
|
|
12123
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12124
|
+
if (!analysis.parsed)
|
|
12125
|
+
return sql;
|
|
12126
|
+
const declared = Object.entries(analysis.aliasToRelation).map(([qualifier, relation]) => ({
|
|
12127
|
+
qualifier,
|
|
12128
|
+
subject: normalizeExploratoryRelationSubject(relation.split('.').at(-1) ?? relation),
|
|
12129
|
+
}));
|
|
12130
|
+
let repaired = sql;
|
|
12131
|
+
const unresolved = analysis.columns
|
|
12132
|
+
.filter((column) => column.tableAlias && column.relation === column.tableAlias && !analysis.aliasToRelation[column.tableAlias])
|
|
12133
|
+
.map((column) => column.tableAlias)
|
|
12134
|
+
.filter((value, index, values) => values.indexOf(value) === index);
|
|
12135
|
+
for (const qualifier of unresolved) {
|
|
12136
|
+
const subject = normalizeExploratoryRelationSubject(qualifier);
|
|
12137
|
+
const matches = declared.filter((candidate) => candidate.subject === subject);
|
|
12138
|
+
if (matches.length !== 1 || matches[0].qualifier === qualifier)
|
|
12139
|
+
continue;
|
|
12140
|
+
const replacement = matches[0].qualifier;
|
|
12141
|
+
repaired = repaired.replace(new RegExp(`\\b${escapeExploratoryRegex(qualifier)}\\s*\\.`, 'gi'), `${replacement}.`);
|
|
12142
|
+
repairs.push(`Corrected unresolved SQL qualifier ${qualifier} to the inspected relation alias ${replacement}.`);
|
|
12143
|
+
}
|
|
12144
|
+
return repaired;
|
|
12145
|
+
}
|
|
12146
|
+
function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question, repairs) {
|
|
12147
|
+
if (!/\b(lifetime|life\s*span|lifespan|customer\s+life)\b/i.test(question))
|
|
12148
|
+
return sql;
|
|
12149
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12150
|
+
if (!analysis.parsed || analysis.joins.length === 0)
|
|
12151
|
+
return sql;
|
|
12152
|
+
const groupClause = sql.match(/\bgroup\s+by\s+([\s\S]*?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i)?.[1] ?? '';
|
|
12153
|
+
let repaired = sql;
|
|
12154
|
+
for (const aggregate of analysis.aggregates) {
|
|
12155
|
+
if (aggregate.func.toLowerCase() !== 'sum' || !aggregate.column)
|
|
12156
|
+
continue;
|
|
12157
|
+
const lifetimeColumn = `lifetime_${aggregate.column.toLowerCase()}`;
|
|
12158
|
+
const owner = resolveUniqueExploratoryColumnOwner(schemaContext, lifetimeColumn);
|
|
12159
|
+
if (!owner)
|
|
12160
|
+
continue;
|
|
12161
|
+
const ownerAliases = Object.entries(analysis.aliasToRelation)
|
|
12162
|
+
.filter(([, relation]) => exploratoryRelationsMatch(relation, owner.relation))
|
|
12163
|
+
.map(([alias]) => alias);
|
|
12164
|
+
const ownerRetained = owner.columns.some((ownerColumn) => new RegExp(`(?:^|[^A-Za-z0-9_$])${escapeExploratoryRegex(ownerColumn.name)}(?:[^A-Za-z0-9_$]|$)`, 'i').test(groupClause));
|
|
12165
|
+
if (!ownerRetained || ownerAliases.length === 0)
|
|
12166
|
+
continue;
|
|
12167
|
+
const sourcePattern = new RegExp(`\\bsum\\s*\\(\\s*(?:[A-Za-z_][A-Za-z0-9_$]*\\s*\\.\\s*)?${escapeExploratoryRegex(aggregate.column)}\\s*\\)`, 'gi');
|
|
12168
|
+
if (!sourcePattern.test(repaired))
|
|
12169
|
+
continue;
|
|
12170
|
+
repaired = repaired.replace(sourcePattern, `MAX(${ownerAliases[0]}.${lifetimeColumn})`);
|
|
12171
|
+
repairs.push(`Used ${lifetimeColumn} from ${owner.name} at the retained entity grain instead of multiplying ${aggregate.column} across lower-grain joined rows.`);
|
|
12172
|
+
}
|
|
12173
|
+
return repaired;
|
|
12174
|
+
}
|
|
12175
|
+
function repairExploratoryNonAdditiveAggregates(sql, schemaContext, repairs) {
|
|
12176
|
+
const analysis = analyzeSqlReferences(sql);
|
|
12177
|
+
if (!analysis.parsed || analysis.joins.length === 0)
|
|
12178
|
+
return { sql };
|
|
12179
|
+
const risky = analysis.aggregates.filter((aggregate) => aggregate.func.toLowerCase() === 'sum'
|
|
12180
|
+
&& Boolean(aggregate.column)
|
|
12181
|
+
&& /(^|_)(lifetime|cumulative|balance|snapshot|rate|ratio|average|avg)(_|$)/i.test(aggregate.column));
|
|
12182
|
+
if (risky.length === 0)
|
|
12183
|
+
return { sql };
|
|
12184
|
+
const groupClause = sql.match(/\bgroup\s+by\s+([\s\S]*?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i)?.[1] ?? '';
|
|
12185
|
+
let repaired = sql;
|
|
12186
|
+
for (const aggregate of risky) {
|
|
12187
|
+
const column = aggregate.column;
|
|
12188
|
+
const owner = resolveUniqueExploratoryColumnOwner(schemaContext, column);
|
|
12189
|
+
if (!owner) {
|
|
12190
|
+
return {
|
|
12191
|
+
sql: repaired,
|
|
12192
|
+
blockedReason: `The exploratory query sums non-additive measure ${column} after a join, but DQL cannot prove its owning grain. Clarify the desired allocation or keep the owning entity in the result.`,
|
|
12193
|
+
};
|
|
12194
|
+
}
|
|
12195
|
+
const ownerAliases = Object.entries(analysis.aliasToRelation)
|
|
12196
|
+
.filter(([, relation]) => exploratoryRelationsMatch(relation, owner.relation))
|
|
12197
|
+
.map(([alias]) => alias);
|
|
12198
|
+
const ownerRetained = owner.columns.some((ownerColumn) => {
|
|
12199
|
+
const name = escapeExploratoryRegex(ownerColumn.name);
|
|
12200
|
+
if (new RegExp(`(?:^|[^A-Za-z0-9_$])${name}(?:[^A-Za-z0-9_$]|$)`, 'i').test(groupClause))
|
|
12201
|
+
return true;
|
|
12202
|
+
return ownerAliases.some((alias) => new RegExp(`\\b${escapeExploratoryRegex(alias)}\\s*\\.\\s*${name}\\b`, 'i').test(groupClause));
|
|
12203
|
+
});
|
|
12204
|
+
if (!ownerRetained) {
|
|
12205
|
+
return {
|
|
12206
|
+
sql: repaired,
|
|
12207
|
+
blockedReason: `The exploratory query would allocate non-additive measure ${column} below its owning ${owner.name} grain. Keep ${owner.name} in the output or provide an allocation policy.`,
|
|
12208
|
+
};
|
|
12209
|
+
}
|
|
12210
|
+
const sumPattern = new RegExp(`\\bsum\\s*\\(\\s*((?:[A-Za-z_][A-Za-z0-9_$]*\\s*\\.\\s*)?${escapeExploratoryRegex(column)})\\s*\\)`, 'gi');
|
|
12211
|
+
if (!sumPattern.test(repaired))
|
|
12212
|
+
continue;
|
|
12213
|
+
repaired = repaired.replace(sumPattern, 'MAX($1)');
|
|
12214
|
+
repairs.push(`Preserved non-additive ${column} at ${owner.name} grain instead of summing repeated values after the join.`);
|
|
12215
|
+
}
|
|
12216
|
+
return { sql: repaired };
|
|
12217
|
+
}
|
|
12218
|
+
function resolveUniqueExploratoryColumnOwner(schemaContext, column) {
|
|
12219
|
+
const candidates = schemaContext.filter((table) => table.columns.some((candidate) => candidate.name.toLowerCase() === column.toLowerCase()));
|
|
12220
|
+
if (candidates.length === 0)
|
|
12221
|
+
return undefined;
|
|
12222
|
+
const compatible = candidates.every((candidate) => candidates.every((other) => exploratoryRelationsMatch(candidate.relation, other.relation)));
|
|
12223
|
+
if (!compatible)
|
|
12224
|
+
return undefined;
|
|
12225
|
+
const preferred = [...candidates].sort((left, right) => right.relation.split('.').length - left.relation.split('.').length)[0];
|
|
12226
|
+
const columns = new Map();
|
|
12227
|
+
for (const candidate of candidates) {
|
|
12228
|
+
for (const candidateColumn of candidate.columns)
|
|
12229
|
+
columns.set(candidateColumn.name.toLowerCase(), candidateColumn);
|
|
12230
|
+
}
|
|
12231
|
+
return { ...preferred, columns: [...columns.values()] };
|
|
12232
|
+
}
|
|
12233
|
+
function normalizeExploratoryRelationSubject(value) {
|
|
12234
|
+
const normalized = value.toLowerCase().replace(/[^a-z0-9_]/g, '');
|
|
12235
|
+
if (normalized.endsWith('ies') && normalized.length > 4)
|
|
12236
|
+
return `${normalized.slice(0, -3)}y`;
|
|
12237
|
+
if (normalized.endsWith('s') && normalized.length > 3)
|
|
12238
|
+
return normalized.slice(0, -1);
|
|
12239
|
+
return normalized;
|
|
12240
|
+
}
|
|
12241
|
+
function exploratoryRelationsMatch(left, right) {
|
|
12242
|
+
const leftLower = left.toLowerCase();
|
|
12243
|
+
const rightLower = right.toLowerCase();
|
|
12244
|
+
return leftLower === rightLower
|
|
12245
|
+
|| leftLower.endsWith(`.${rightLower}`)
|
|
12246
|
+
|| rightLower.endsWith(`.${leftLower}`)
|
|
12247
|
+
|| normalizeExploratoryRelationSubject(leftLower.split('.').at(-1) ?? leftLower)
|
|
12248
|
+
=== normalizeExploratoryRelationSubject(rightLower.split('.').at(-1) ?? rightLower);
|
|
12249
|
+
}
|
|
12250
|
+
function escapeExploratoryRegex(value) {
|
|
12251
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
12252
|
+
}
|
|
12253
|
+
/** Render parser-derived physical identifiers only; exploratory probes never interpolate arbitrary SQL. */
|
|
12254
|
+
function quoteExploratoryIdentifier(value, label) {
|
|
12255
|
+
const parts = value
|
|
12256
|
+
.replace(/["`\[\]]/g, '')
|
|
12257
|
+
.split('.')
|
|
12258
|
+
.map((part) => part.trim())
|
|
12259
|
+
.filter(Boolean);
|
|
12260
|
+
if (parts.length === 0 || parts.some((part) => !/^[A-Za-z_][A-Za-z0-9_$]*$/.test(part))) {
|
|
12261
|
+
throw new Error(`Exploratory ${label} is not a safe physical identifier.`);
|
|
12262
|
+
}
|
|
12263
|
+
return parts.map((part) => `"${part.replace(/"/g, '""')}"`).join('.');
|
|
12264
|
+
}
|
|
12265
|
+
/** One bounded, sampled relationship observation per inferred equality join. */
|
|
12266
|
+
export function buildExploratoryJoinProbeSql(input) {
|
|
12267
|
+
const leftRelation = quoteExploratoryIdentifier(input.leftRelation, 'left relation');
|
|
12268
|
+
const leftColumn = quoteExploratoryIdentifier(input.leftColumn, 'left column');
|
|
12269
|
+
const rightRelation = quoteExploratoryIdentifier(input.rightRelation, 'right relation');
|
|
12270
|
+
const rightColumn = quoteExploratoryIdentifier(input.rightColumn, 'right column');
|
|
12271
|
+
return `WITH
|
|
12272
|
+
left_sample AS (
|
|
12273
|
+
SELECT ${leftColumn} AS join_key
|
|
12274
|
+
FROM ${leftRelation}
|
|
12275
|
+
WHERE ${leftColumn} IS NOT NULL
|
|
12276
|
+
LIMIT 5000
|
|
12277
|
+
),
|
|
12278
|
+
right_sample AS (
|
|
12279
|
+
SELECT ${rightColumn} AS join_key
|
|
12280
|
+
FROM ${rightRelation}
|
|
12281
|
+
WHERE ${rightColumn} IS NOT NULL
|
|
12282
|
+
LIMIT 5000
|
|
12283
|
+
),
|
|
12284
|
+
matches AS (
|
|
12285
|
+
SELECT l.join_key AS left_key, r.join_key AS right_key
|
|
12286
|
+
FROM left_sample AS l
|
|
12287
|
+
INNER JOIN right_sample AS r ON l.join_key = r.join_key
|
|
12288
|
+
),
|
|
12289
|
+
left_counts AS (
|
|
12290
|
+
SELECT left_key, COUNT(*) AS match_count
|
|
12291
|
+
FROM matches
|
|
12292
|
+
GROUP BY left_key
|
|
12293
|
+
),
|
|
12294
|
+
right_counts AS (
|
|
12295
|
+
SELECT right_key, COUNT(*) AS match_count
|
|
12296
|
+
FROM matches
|
|
12297
|
+
GROUP BY right_key
|
|
12298
|
+
)
|
|
12299
|
+
SELECT
|
|
12300
|
+
(SELECT COUNT(*) FROM left_sample) AS left_sample_rows,
|
|
12301
|
+
(SELECT COUNT(*) FROM right_sample) AS right_sample_rows,
|
|
12302
|
+
(SELECT COUNT(*) FROM matches) AS joined_rows,
|
|
12303
|
+
(SELECT COUNT(*) FROM left_sample AS l WHERE NOT EXISTS (SELECT 1 FROM right_sample AS r WHERE r.join_key = l.join_key)) AS unmatched_left_sample_rows,
|
|
12304
|
+
(SELECT COUNT(*) FROM right_sample AS r WHERE NOT EXISTS (SELECT 1 FROM left_sample AS l WHERE l.join_key = r.join_key)) AS unmatched_right_sample_rows,
|
|
12305
|
+
(SELECT COALESCE(MAX(match_count), 0) FROM left_counts) AS max_matches_per_left_key,
|
|
12306
|
+
(SELECT COALESCE(MAX(match_count), 0) FROM right_counts) AS max_matches_per_right_key`;
|
|
12307
|
+
}
|
|
12308
|
+
function summarizeExploratoryJoinProbe(rows, leftRelation, leftColumn, rightRelation, rightColumn) {
|
|
12309
|
+
const row = Array.isArray(rows) && rows[0] && typeof rows[0] === 'object'
|
|
12310
|
+
? rows[0]
|
|
12311
|
+
: {};
|
|
12312
|
+
const number = (key) => {
|
|
12313
|
+
const value = Number(row[key]);
|
|
12314
|
+
return Number.isFinite(value) ? value : 0;
|
|
12315
|
+
};
|
|
12316
|
+
return `${leftRelation}.${leftColumn} ↔ ${rightRelation}.${rightColumn}: sample left=${number('left_sample_rows')}, right=${number('right_sample_rows')}, joined=${number('joined_rows')}, unmatched left/right=${number('unmatched_left_sample_rows')}/${number('unmatched_right_sample_rows')}, max matches left/right=${number('max_matches_per_left_key')}/${number('max_matches_per_right_key')}`;
|
|
12317
|
+
}
|
|
9726
12318
|
function readOnlySqlValidationError(sql, subject) {
|
|
9727
12319
|
const scanSql = stripSqlStringsAndComments(sql).trim();
|
|
9728
12320
|
if (!/^(select|with)\b/i.test(scanSql)) {
|
|
@@ -9793,11 +12385,11 @@ function stripSqlStringsAndComments(sql) {
|
|
|
9793
12385
|
* Used by notebook SQL execution and Block Studio validation so both paths
|
|
9794
12386
|
* behave identically. If the SQL has no refs, returns it unchanged.
|
|
9795
12387
|
*/
|
|
9796
|
-
export function prepareSemanticSql(sql, semanticLayer) {
|
|
12388
|
+
export function prepareSemanticSql(sql, semanticLayer, options) {
|
|
9797
12389
|
if (!hasSemanticRefs(sql)) {
|
|
9798
12390
|
return { sql, semanticRefs: { metrics: [], dimensions: [] }, unresolvedRefs: [] };
|
|
9799
12391
|
}
|
|
9800
|
-
const resolution = resolveSemanticRefs(sql, semanticLayer);
|
|
12392
|
+
const resolution = resolveSemanticRefs(sql, semanticLayer, options);
|
|
9801
12393
|
return {
|
|
9802
12394
|
sql: resolution.resolvedSql,
|
|
9803
12395
|
semanticRefs: {
|
|
@@ -9870,6 +12462,34 @@ function readJSON(req) {
|
|
|
9870
12462
|
req.on('error', reject);
|
|
9871
12463
|
});
|
|
9872
12464
|
}
|
|
12465
|
+
async function readMultipartForm(req) {
|
|
12466
|
+
const headers = new Headers();
|
|
12467
|
+
for (const [key, value] of Object.entries(req.headers)) {
|
|
12468
|
+
if (Array.isArray(value))
|
|
12469
|
+
value.forEach((item) => headers.append(key, item));
|
|
12470
|
+
else if (value !== undefined)
|
|
12471
|
+
headers.set(key, value);
|
|
12472
|
+
}
|
|
12473
|
+
const request = new Request("http://127.0.0.1/upload", {
|
|
12474
|
+
method: "POST",
|
|
12475
|
+
headers,
|
|
12476
|
+
body: Readable.toWeb(req),
|
|
12477
|
+
duplex: "half",
|
|
12478
|
+
});
|
|
12479
|
+
const form = await request.formData();
|
|
12480
|
+
const fields = {};
|
|
12481
|
+
let file;
|
|
12482
|
+
for (const [key, value] of form.entries()) {
|
|
12483
|
+
if (typeof value === "string")
|
|
12484
|
+
fields[key] = value;
|
|
12485
|
+
else if (key === "file")
|
|
12486
|
+
file = {
|
|
12487
|
+
filename: value.name,
|
|
12488
|
+
content: Buffer.from(await value.arrayBuffer()),
|
|
12489
|
+
};
|
|
12490
|
+
}
|
|
12491
|
+
return { fields, file };
|
|
12492
|
+
}
|
|
9873
12493
|
function safeJoin(rootDir, requestPath) {
|
|
9874
12494
|
const normalized = normalize(requestPath).replace(/^([.][.][/\\])+/, '');
|
|
9875
12495
|
const fullPath = resolve(rootDir, `.${normalized.startsWith('/') ? normalized : `/${normalized}`}`);
|
|
@@ -9987,7 +12607,43 @@ function scanNotebookFiles(projectRoot) {
|
|
|
9987
12607
|
continue;
|
|
9988
12608
|
collect(dir, folder, type);
|
|
9989
12609
|
}
|
|
12610
|
+
collectDomainArtifacts(join(projectRoot, 'domains'), 'domains');
|
|
9990
12611
|
return result;
|
|
12612
|
+
function collectDomainArtifacts(currentDir, relativeDir) {
|
|
12613
|
+
if (!existsSync(currentDir))
|
|
12614
|
+
return;
|
|
12615
|
+
try {
|
|
12616
|
+
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
|
|
12617
|
+
if (entry.name.startsWith('.') || entry.name === 'node_modules')
|
|
12618
|
+
continue;
|
|
12619
|
+
const fullPath = join(currentDir, entry.name);
|
|
12620
|
+
const relativePath = `${relativeDir}/${entry.name}`;
|
|
12621
|
+
if (entry.isDirectory()) {
|
|
12622
|
+
collectDomainArtifacts(fullPath, relativePath);
|
|
12623
|
+
continue;
|
|
12624
|
+
}
|
|
12625
|
+
if (!entry.isFile())
|
|
12626
|
+
continue;
|
|
12627
|
+
const segments = relativePath.split('/');
|
|
12628
|
+
const ownerFolder = segments.find((segment) => ['notebooks', 'blocks', 'terms', 'views'].includes(segment));
|
|
12629
|
+
const type = ownerFolder === 'notebooks' ? 'notebook'
|
|
12630
|
+
: ownerFolder === 'blocks' ? 'block'
|
|
12631
|
+
: ownerFolder === 'terms' ? 'term'
|
|
12632
|
+
: ownerFolder === 'views' ? 'business_view'
|
|
12633
|
+
: undefined;
|
|
12634
|
+
if (!type || (!entry.name.endsWith('.dql') && !entry.name.endsWith('.dqlnb')))
|
|
12635
|
+
continue;
|
|
12636
|
+
const fallbackName = entry.name.replace(/\.(dql|dqlnb)$/, '');
|
|
12637
|
+
result.push({
|
|
12638
|
+
name: inferDqlArtifactName(fullPath, type, fallbackName),
|
|
12639
|
+
path: relativePath,
|
|
12640
|
+
type,
|
|
12641
|
+
folder: relativeDir,
|
|
12642
|
+
});
|
|
12643
|
+
}
|
|
12644
|
+
}
|
|
12645
|
+
catch { /* skip unreadable domain package dirs */ }
|
|
12646
|
+
}
|
|
9991
12647
|
function collect(currentDir, relativeDir, type) {
|
|
9992
12648
|
try {
|
|
9993
12649
|
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
|
|
@@ -10315,7 +12971,13 @@ export function buildSemanticTableMapping(semanticLayer, rows) {
|
|
|
10315
12971
|
allSemanticTables.add(metric.table);
|
|
10316
12972
|
for (const dimension of semanticLayer.listDimensions())
|
|
10317
12973
|
allSemanticTables.add(dimension.table);
|
|
12974
|
+
for (const measure of semanticLayer.listMeasures())
|
|
12975
|
+
allSemanticTables.add(measure.table);
|
|
12976
|
+
for (const model of semanticLayer.listSemanticModels())
|
|
12977
|
+
allSemanticTables.add(model.table);
|
|
10318
12978
|
for (const semTable of allSemanticTables) {
|
|
12979
|
+
if (!semTable)
|
|
12980
|
+
continue;
|
|
10319
12981
|
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
10320
12982
|
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
10321
12983
|
}
|
|
@@ -10335,23 +12997,32 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
|
10335
12997
|
const dbtProjectPath = effectiveSemanticConfig?.provider === 'dbt'
|
|
10336
12998
|
? effectiveSemanticConfig.projectPath
|
|
10337
12999
|
: context.projectConfig.dbt?.projectDir;
|
|
10338
|
-
|
|
10339
|
-
|
|
10340
|
-
|
|
10341
|
-
|
|
10342
|
-
|
|
10343
|
-
|
|
10344
|
-
|
|
10345
|
-
|
|
10346
|
-
|
|
10347
|
-
|
|
10348
|
-
|
|
10349
|
-
|
|
10350
|
-
|
|
10351
|
-
|
|
10352
|
-
|
|
10353
|
-
|
|
10354
|
-
|
|
13000
|
+
try {
|
|
13001
|
+
const compiled = compileMetricFlowQuery({
|
|
13002
|
+
projectRoot: context.projectRoot,
|
|
13003
|
+
dbtProjectPath,
|
|
13004
|
+
metrics: request.metrics,
|
|
13005
|
+
dimensions: request.dimensions,
|
|
13006
|
+
filters: request.filters,
|
|
13007
|
+
timeDimension: request.timeDimension,
|
|
13008
|
+
orderBy: request.orderBy,
|
|
13009
|
+
limit: request.limit,
|
|
13010
|
+
savedQuery: request.savedQuery,
|
|
13011
|
+
});
|
|
13012
|
+
return {
|
|
13013
|
+
sql: compiled.sql,
|
|
13014
|
+
joins: [],
|
|
13015
|
+
tables: [],
|
|
13016
|
+
engine: 'metricflow',
|
|
13017
|
+
};
|
|
13018
|
+
}
|
|
13019
|
+
catch (error) {
|
|
13020
|
+
// An explicit MetricFlow request must retain its strict engine contract.
|
|
13021
|
+
// For the default dbt path, a missing local CLI may fall back only if the
|
|
13022
|
+
// native composer can safely materialize the requested simple metrics.
|
|
13023
|
+
if (request.engine === 'metricflow' || !(error instanceof MetricFlowUnavailableError))
|
|
13024
|
+
throw error;
|
|
13025
|
+
}
|
|
10355
13026
|
}
|
|
10356
13027
|
const composed = semanticLayer.composeQuery({
|
|
10357
13028
|
metrics: request.metrics,
|
|
@@ -10367,6 +13038,7 @@ function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
|
10367
13038
|
}
|
|
10368
13039
|
function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
10369
13040
|
const config = parseSemanticBlockConfig(source);
|
|
13041
|
+
const runtimeBindings = semanticRuntimeParameterBindings(source, options?.parameters ?? {});
|
|
10370
13042
|
const metrics = config.metrics.length > 0
|
|
10371
13043
|
? config.metrics
|
|
10372
13044
|
: config.metric
|
|
@@ -10423,11 +13095,11 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
10423
13095
|
? composeRuntimeSemanticQuery({
|
|
10424
13096
|
metrics,
|
|
10425
13097
|
dimensions: config.dimensions,
|
|
10426
|
-
filters: config.filters,
|
|
13098
|
+
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
10427
13099
|
timeDimension: config.timeDimension && config.granularity
|
|
10428
13100
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
10429
13101
|
: undefined,
|
|
10430
|
-
limit: config.limit,
|
|
13102
|
+
limit: runtimeBindings.limit ?? config.limit,
|
|
10431
13103
|
}, semanticLayer, {
|
|
10432
13104
|
projectRoot: options.projectRoot,
|
|
10433
13105
|
projectConfig: options.projectConfig,
|
|
@@ -10438,11 +13110,11 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
10438
13110
|
: semanticLayer.composeQuery({
|
|
10439
13111
|
metrics,
|
|
10440
13112
|
dimensions: config.dimensions,
|
|
10441
|
-
filters: config.filters,
|
|
13113
|
+
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
10442
13114
|
timeDimension: config.timeDimension && config.granularity
|
|
10443
13115
|
? { name: config.timeDimension, granularity: config.granularity }
|
|
10444
13116
|
: undefined,
|
|
10445
|
-
limit: config.limit,
|
|
13117
|
+
limit: runtimeBindings.limit ?? config.limit,
|
|
10446
13118
|
driver: options?.driver,
|
|
10447
13119
|
tableMapping: options?.tableMapping,
|
|
10448
13120
|
});
|
|
@@ -10469,6 +13141,39 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
10469
13141
|
semanticRefs,
|
|
10470
13142
|
};
|
|
10471
13143
|
}
|
|
13144
|
+
function semanticRuntimeParameterBindings(source, values) {
|
|
13145
|
+
const program = new Parser(source, '<semantic-parameters>').parse();
|
|
13146
|
+
const block = program.statements.find((statement) => statement.kind === NodeKind.BlockDecl);
|
|
13147
|
+
if (!block || block.kind !== NodeKind.BlockDecl || block.blockType !== 'semantic')
|
|
13148
|
+
return { filters: [] };
|
|
13149
|
+
const filters = [];
|
|
13150
|
+
let limit;
|
|
13151
|
+
for (const definition of blockParameterDefinitions(block)) {
|
|
13152
|
+
const value = values[definition.name];
|
|
13153
|
+
if (value === undefined || value === null || value === '')
|
|
13154
|
+
continue;
|
|
13155
|
+
if (definition.binding?.kind === 'semantic_filter') {
|
|
13156
|
+
const rawValues = Array.isArray(value) ? value : [value];
|
|
13157
|
+
filters.push({
|
|
13158
|
+
dimension: definition.binding.field,
|
|
13159
|
+
operator: definition.binding.operator,
|
|
13160
|
+
values: rawValues.map((item) => String(item)),
|
|
13161
|
+
});
|
|
13162
|
+
}
|
|
13163
|
+
else if (definition.binding?.kind === 'limit') {
|
|
13164
|
+
const parsed = typeof value === 'number' ? value : Number(value);
|
|
13165
|
+
if (Number.isFinite(parsed) && parsed > 0)
|
|
13166
|
+
limit = Math.floor(parsed);
|
|
13167
|
+
}
|
|
13168
|
+
}
|
|
13169
|
+
return { filters, ...(limit !== undefined ? { limit } : {}) };
|
|
13170
|
+
}
|
|
13171
|
+
function mergeSemanticRuntimeFilters(declared, runtime) {
|
|
13172
|
+
if (runtime.length === 0)
|
|
13173
|
+
return declared;
|
|
13174
|
+
const runtimeDimensions = new Set(runtime.map((filter) => filter.dimension));
|
|
13175
|
+
return [...declared.filter((filter) => !runtimeDimensions.has(filter.dimension)), ...runtime];
|
|
13176
|
+
}
|
|
10472
13177
|
function resolveCustomBlockSql(sql, semanticLayer) {
|
|
10473
13178
|
if (!sql) {
|
|
10474
13179
|
return {
|
|
@@ -10597,12 +13302,17 @@ export function validateBlockStudioSource(source, semanticLayer) {
|
|
|
10597
13302
|
message: 'No executable SQL found in the block source.',
|
|
10598
13303
|
});
|
|
10599
13304
|
}
|
|
13305
|
+
const parameterInvocation = prepareBlockInvocation({ source, surface: 'block_studio' });
|
|
13306
|
+
if (parameterInvocation.errors.length > 0) {
|
|
13307
|
+
diagnostics.push(...parameterInvocation.errors.map((message) => ({ severity: 'error', code: 'parameter_parse', message })));
|
|
13308
|
+
}
|
|
10600
13309
|
return {
|
|
10601
13310
|
valid: diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
10602
13311
|
diagnostics,
|
|
10603
13312
|
semanticRefs,
|
|
10604
13313
|
chartConfig: chartConfig ?? undefined,
|
|
10605
13314
|
executableSql,
|
|
13315
|
+
parameters: parameterInvocation.parameters,
|
|
10606
13316
|
};
|
|
10607
13317
|
}
|
|
10608
13318
|
export async function createDqlArtifactGenerationSessionForProject(projectRoot, options, semanticLayer) {
|
|
@@ -13286,26 +15996,117 @@ function yamlScalar(value) {
|
|
|
13286
15996
|
function buildNotebookTemplate(title, template) {
|
|
13287
15997
|
const id = () => Math.random().toString(36).slice(2, 10);
|
|
13288
15998
|
let cells;
|
|
13289
|
-
if (template ===
|
|
15999
|
+
if (template === "analysis") {
|
|
16000
|
+
cells = [
|
|
16001
|
+
{
|
|
16002
|
+
id: id(),
|
|
16003
|
+
type: "markdown",
|
|
16004
|
+
content: "# TL;DR\n\nSummarize the answer and decision here.",
|
|
16005
|
+
},
|
|
16006
|
+
{
|
|
16007
|
+
id: id(),
|
|
16008
|
+
type: "markdown",
|
|
16009
|
+
content: "## Context and methods\n\nState the business question, scope, definitions, and assumptions.",
|
|
16010
|
+
},
|
|
16011
|
+
{
|
|
16012
|
+
id: id(),
|
|
16013
|
+
type: "dql",
|
|
16014
|
+
name: "analysis_data",
|
|
16015
|
+
content: "",
|
|
16016
|
+
},
|
|
16017
|
+
{
|
|
16018
|
+
id: id(),
|
|
16019
|
+
type: "markdown",
|
|
16020
|
+
content: "## Results\n\nExplain the material findings and uncertainty.",
|
|
16021
|
+
},
|
|
16022
|
+
{
|
|
16023
|
+
id: id(),
|
|
16024
|
+
type: "markdown",
|
|
16025
|
+
content: "## Takeaways\n\nRecord the decision, risks, and next action.",
|
|
16026
|
+
},
|
|
16027
|
+
];
|
|
16028
|
+
}
|
|
16029
|
+
else if (template === "metric_diagnostic") {
|
|
13290
16030
|
cells = [
|
|
13291
|
-
{
|
|
13292
|
-
|
|
13293
|
-
|
|
16031
|
+
{
|
|
16032
|
+
id: id(),
|
|
16033
|
+
type: "markdown",
|
|
16034
|
+
content: "# Metric diagnostic\n\nWhat changed, when, and for whom?",
|
|
16035
|
+
},
|
|
16036
|
+
{
|
|
16037
|
+
id: id(),
|
|
16038
|
+
type: "dql",
|
|
16039
|
+
name: "metric_trend",
|
|
16040
|
+
content: "",
|
|
16041
|
+
},
|
|
16042
|
+
{
|
|
16043
|
+
id: id(),
|
|
16044
|
+
type: "dql",
|
|
16045
|
+
name: "driver_breakdown",
|
|
16046
|
+
content: "",
|
|
16047
|
+
},
|
|
16048
|
+
{
|
|
16049
|
+
id: id(),
|
|
16050
|
+
type: "markdown",
|
|
16051
|
+
content: "## Diagnosis\n\nSeparate validated drivers from hypotheses and data limitations.",
|
|
16052
|
+
},
|
|
13294
16053
|
];
|
|
13295
16054
|
}
|
|
13296
|
-
else if (template ===
|
|
16055
|
+
else if (template === "data_quality") {
|
|
13297
16056
|
cells = [
|
|
13298
|
-
{
|
|
13299
|
-
|
|
16057
|
+
{
|
|
16058
|
+
id: id(),
|
|
16059
|
+
type: "markdown",
|
|
16060
|
+
content: "# Data-quality investigation\n\nDocument the source, owner, freshness expectation, and affected decisions.",
|
|
16061
|
+
},
|
|
16062
|
+
{
|
|
16063
|
+
id: id(),
|
|
16064
|
+
type: "dql",
|
|
16065
|
+
name: "quality_profile",
|
|
16066
|
+
content: "",
|
|
16067
|
+
},
|
|
16068
|
+
{
|
|
16069
|
+
id: id(),
|
|
16070
|
+
type: "dql",
|
|
16071
|
+
name: "quality_checks",
|
|
16072
|
+
content: "",
|
|
16073
|
+
},
|
|
16074
|
+
{
|
|
16075
|
+
id: id(),
|
|
16076
|
+
type: "markdown",
|
|
16077
|
+
content: "## Findings and disposition\n\nRecord severity, impacted metrics, owner, and remediation.",
|
|
16078
|
+
},
|
|
13300
16079
|
];
|
|
13301
16080
|
}
|
|
13302
|
-
else {
|
|
16081
|
+
else if (template === "experiment") {
|
|
13303
16082
|
cells = [
|
|
13304
|
-
{
|
|
13305
|
-
|
|
16083
|
+
{
|
|
16084
|
+
id: id(),
|
|
16085
|
+
type: "markdown",
|
|
16086
|
+
content: "# Experiment log\n\nHypothesis, primary metric, guardrails, population, and dates.",
|
|
16087
|
+
},
|
|
16088
|
+
{
|
|
16089
|
+
id: id(),
|
|
16090
|
+
type: "dql",
|
|
16091
|
+
name: "experiment_results",
|
|
16092
|
+
content: "",
|
|
16093
|
+
},
|
|
16094
|
+
{
|
|
16095
|
+
id: id(),
|
|
16096
|
+
type: "markdown",
|
|
16097
|
+
content: "## Results\n\nReport effect size, uncertainty, guardrails, and data-quality checks.",
|
|
16098
|
+
},
|
|
16099
|
+
{
|
|
16100
|
+
id: id(),
|
|
16101
|
+
type: "markdown",
|
|
16102
|
+
content: "## Decision\n\nShip, iterate, or stop — with rationale.",
|
|
16103
|
+
},
|
|
13306
16104
|
];
|
|
13307
16105
|
}
|
|
13308
|
-
|
|
16106
|
+
else {
|
|
16107
|
+
cells = [];
|
|
16108
|
+
}
|
|
16109
|
+
return JSON.stringify({ dqlnbVersion: 2, version: 1, title, cells }, null, 2);
|
|
13309
16110
|
}
|
|
13310
16111
|
/** Build a lineage graph from the project's blocks and semantic layer. */
|
|
13311
16112
|
// Simple lineage graph cache: rebuilds at most every 5 seconds
|
|
@@ -14185,6 +16986,33 @@ function ensureGitignoreEntry(projectRoot, pattern) {
|
|
|
14185
16986
|
// Best-effort; failure to write .gitignore shouldn't fail the snapshot.
|
|
14186
16987
|
}
|
|
14187
16988
|
}
|
|
16989
|
+
/**
|
|
16990
|
+
* UI-001, SEC-001, E2E-001: local runtime, credential, and generated artifacts must never be offered as
|
|
16991
|
+
* shareable source-control changes. Project skills are intentionally absent:
|
|
16992
|
+
* `.dql/skills` remains readable during migration and can be governed in Git.
|
|
16993
|
+
*/
|
|
16994
|
+
const LOCAL_RUNTIME_GITIGNORE_RULES = [
|
|
16995
|
+
'dql-manifest.json',
|
|
16996
|
+
'*.duckdb',
|
|
16997
|
+
'*.duckdb.wal',
|
|
16998
|
+
'*.run.json',
|
|
16999
|
+
'**/.dql/runs/',
|
|
17000
|
+
'**/.dql/cache/',
|
|
17001
|
+
'**/.dql/imports/',
|
|
17002
|
+
'**/.dql/local/',
|
|
17003
|
+
'**/.dql/connectors/',
|
|
17004
|
+
'**/.dql/memory/',
|
|
17005
|
+
'**/.dql/migration-staging/',
|
|
17006
|
+
'**/.dql/docker-starter/',
|
|
17007
|
+
'**/.dql/oauth-credentials.json',
|
|
17008
|
+
'**/.dql/provider-settings.json',
|
|
17009
|
+
'**/.dql/mcp-servers.json',
|
|
17010
|
+
'**/.dql-user-prefs.json',
|
|
17011
|
+
];
|
|
17012
|
+
export function ensureLocalRuntimeGitignore(projectRoot) {
|
|
17013
|
+
for (const rule of LOCAL_RUNTIME_GITIGNORE_RULES)
|
|
17014
|
+
ensureGitignoreEntry(projectRoot, rule);
|
|
17015
|
+
}
|
|
14188
17016
|
async function readGitDiff(cwd, filePath, staged = false) {
|
|
14189
17017
|
const gitRoot = await resolveGitRoot(cwd);
|
|
14190
17018
|
if (!gitRoot) {
|
|
@@ -14469,6 +17297,39 @@ async function gitCreateReview(cwd, input) {
|
|
|
14469
17297
|
warning: `Branch pushed. Open the compare page to create the PR: ${gitErrorOutput(pr) || 'GitHub CLI is not available or authenticated.'}`,
|
|
14470
17298
|
};
|
|
14471
17299
|
}
|
|
17300
|
+
/** UI-001, SEC-001, E2E-001: open a PR for a branch already committed and pushed by the guided share flow. */
|
|
17301
|
+
async function gitOpenReview(cwd, input) {
|
|
17302
|
+
const gitRoot = await resolveGitRoot(cwd);
|
|
17303
|
+
if (!gitRoot)
|
|
17304
|
+
return { ok: false, error: 'Not a git repository' };
|
|
17305
|
+
const title = input.title.trim();
|
|
17306
|
+
if (!title)
|
|
17307
|
+
return { ok: false, error: 'Review title required' };
|
|
17308
|
+
const base = input.base.trim() || 'main';
|
|
17309
|
+
const status = await readGitStatus(gitRoot);
|
|
17310
|
+
const branch = status.branch;
|
|
17311
|
+
if (!branch || branch === 'HEAD' || branch === base || branch === 'master') {
|
|
17312
|
+
return { ok: false, error: 'Share the changes to a review branch before opening a review request.' };
|
|
17313
|
+
}
|
|
17314
|
+
const remote = await readGitRemote(gitRoot);
|
|
17315
|
+
if (!remote.url)
|
|
17316
|
+
return { ok: false, error: 'Add a Git remote before requesting review.' };
|
|
17317
|
+
const compareUrl = gitHubCompareUrl(remote.url, branch, base);
|
|
17318
|
+
const pr = await execGh(gitRoot, [
|
|
17319
|
+
'pr', 'create', '--base', base, '--head', branch, '--title', title,
|
|
17320
|
+
'--body', input.body.trim() || `Review governed analytics changes: ${title}`,
|
|
17321
|
+
]);
|
|
17322
|
+
if (pr.code === 0) {
|
|
17323
|
+
const prUrl = pr.stdout.match(/https:\/\/github\.com\/[^\s]+\/pull\/\d+/)?.[0] ?? compareUrl;
|
|
17324
|
+
return { ok: true, branch, prUrl };
|
|
17325
|
+
}
|
|
17326
|
+
return {
|
|
17327
|
+
ok: true,
|
|
17328
|
+
branch,
|
|
17329
|
+
prUrl: compareUrl,
|
|
17330
|
+
warning: `Branch is ready. Open the compare page to create the PR: ${gitErrorOutput(pr) || 'GitHub CLI is not available or authenticated.'}`,
|
|
17331
|
+
};
|
|
17332
|
+
}
|
|
14472
17333
|
async function gitPull(cwd) {
|
|
14473
17334
|
const gitRoot = await resolveGitRoot(cwd);
|
|
14474
17335
|
if (!gitRoot)
|
|
@@ -14565,8 +17426,7 @@ async function enableGitGovernedContextTracking(cwd) {
|
|
|
14565
17426
|
const existing = existsSync(ignorePath) ? readFileSync(ignorePath, 'utf-8') : '';
|
|
14566
17427
|
const lines = existing.split(/\r?\n/);
|
|
14567
17428
|
const next = lines.filter((line) => !/^\s*\/?\.dql\/\s*$/.test(line));
|
|
14568
|
-
const
|
|
14569
|
-
for (const rule of keepLocal) {
|
|
17429
|
+
for (const rule of LOCAL_RUNTIME_GITIGNORE_RULES) {
|
|
14570
17430
|
if (!next.some((line) => line.trim() === rule))
|
|
14571
17431
|
next.push(rule);
|
|
14572
17432
|
}
|
|
@@ -15813,7 +18673,7 @@ const NON_ENTITY_PLURALS = new Set([
|
|
|
15813
18673
|
* project vocabulary — so it fires for ANY repo's join-shaped questions, not just
|
|
15814
18674
|
* jaffle. Over-triggering only costs a few extra bounded probes (all in try/catch).
|
|
15815
18675
|
*/
|
|
15816
|
-
function shouldAugmentAgentRuntimeSchema(question) {
|
|
18676
|
+
export function shouldAugmentAgentRuntimeSchema(question, questionPlan) {
|
|
15817
18677
|
const lower = question.toLowerCase();
|
|
15818
18678
|
const wantsMetric = /\brevenu|\bsales\b|\bamount\b|\bspend\b|\bcost\b|\bcount\b|\btotal\b|\bsum\b|\bavg\b|\baverage\b|\bvalue\b|\bprice\b/.test(lower);
|
|
15819
18679
|
const wantsDetail = /\bdetails?\b|\bcomplete\b|\bbreakdown\b|\ball\s+values?\b|\beach\b|\bevery\b/.test(lower);
|
|
@@ -15822,7 +18682,12 @@ function shouldAugmentAgentRuntimeSchema(question) {
|
|
|
15822
18682
|
// Multi-entity intent: a linking word + >= 2 distinct plural content nouns.
|
|
15823
18683
|
const nouns = new Set((lower.match(/\b[a-z]{4,}s\b/g) ?? []).filter((word) => !NON_ENTITY_PLURALS.has(word)));
|
|
15824
18684
|
const multiEntity = /\b(?:and|with|plus|per|for\s+each|by)\b/.test(lower) && nouns.size >= 2;
|
|
15825
|
-
|
|
18685
|
+
const plannedConcepts = new Set([
|
|
18686
|
+
...(questionPlan?.entities ?? []).map((entity) => typeof entity === 'string' ? entity : entity.typeHint ?? entity.text ?? ''),
|
|
18687
|
+
...(questionPlan?.dimensionTerms ?? []),
|
|
18688
|
+
].map((value) => value.trim().toLowerCase()).filter(Boolean));
|
|
18689
|
+
const plannedCompositeMetric = (questionPlan?.metricTerms?.length ?? 0) > 0 && plannedConcepts.size >= 2;
|
|
18690
|
+
return explicitJoin || referencesPriorRows || plannedCompositeMetric || (multiEntity && (wantsMetric || wantsDetail));
|
|
15826
18691
|
}
|
|
15827
18692
|
async function enrichAgentSchemaContextWithValueMatches(question, schemaContext, executor, connection) {
|
|
15828
18693
|
const searchTerms = extractAgentValueSearchTerms(question);
|