@duckcodeailabs/dql-cli 1.8.3 → 1.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/dql-notebook/assets/{index-DGbXmDU6.js → index-CPG4A872.js} +340 -340
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/block-invocation.d.ts +2 -2
- package/dist/block-invocation.d.ts.map +1 -1
- package/dist/dbt-cloud-semantic.d.ts +42 -0
- package/dist/dbt-cloud-semantic.d.ts.map +1 -0
- package/dist/dbt-cloud-semantic.js +185 -0
- package/dist/dbt-cloud-semantic.js.map +1 -0
- package/dist/llm/analytics-tools.d.ts.map +1 -1
- package/dist/llm/analytics-tools.js +13 -1
- package/dist/llm/analytics-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +1 -0
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/types.d.ts +3 -1
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +8 -2
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +340 -200
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/semantic-runtime-settings.d.ts +55 -0
- package/dist/semantic-runtime-settings.d.ts.map +1 -0
- package/dist/semantic-runtime-settings.js +191 -0
- package/dist/semantic-runtime-settings.js.map +1 -0
- package/dist/semantic-runtime.d.ts +80 -0
- package/dist/semantic-runtime.d.ts.map +1 -0
- package/dist/semantic-runtime.js +238 -0
- package/dist/semantic-runtime.js.map +1 -0
- package/package.json +10 -10
package/dist/local-runtime.js
CHANGED
|
@@ -31,7 +31,9 @@ import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath,
|
|
|
31
31
|
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
32
32
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
33
33
|
import { clearBlockStudioImportSessions, candidateToDqlSource, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
34
|
-
import { MetricFlowUnavailableError,
|
|
34
|
+
import { MetricFlowUnavailableError, hasDbtSemanticManifest, } from "./metricflow.js";
|
|
35
|
+
import { compileSemanticRuntimeQuery, getSemanticRuntimeStatus, isSemanticRuntimeError, listRuntimeCompatibleDimensions, semanticMetricExecutionCapability as runtimeMetricExecutionCapability, SemanticRuntimeRequiredError, testSemanticRuntimeDraft, } from './semantic-runtime.js';
|
|
36
|
+
import { getSemanticRuntimeSettings, saveTestedSemanticRuntimeSettings, } from './semantic-runtime-settings.js';
|
|
35
37
|
import { NotebookDatasetWorkspace, } from "./notebook-datasets.js";
|
|
36
38
|
import { prepareBlockInvocation } from './block-invocation.js';
|
|
37
39
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
@@ -1087,12 +1089,17 @@ export async function startLocalServer(opts) {
|
|
|
1087
1089
|
// dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
|
|
1088
1090
|
// connection is configured — the compiler then uses its default dialect.
|
|
1089
1091
|
let semanticDriver;
|
|
1092
|
+
let semanticConnection;
|
|
1090
1093
|
try {
|
|
1091
|
-
|
|
1094
|
+
semanticConnection = requireActiveConnection();
|
|
1095
|
+
semanticDriver = semanticConnection.driver;
|
|
1092
1096
|
}
|
|
1093
1097
|
catch {
|
|
1094
1098
|
semanticDriver = undefined;
|
|
1095
1099
|
}
|
|
1100
|
+
const semanticTableMapping = semanticLayer && semanticConnection
|
|
1101
|
+
? await resolveSemanticTableMapping(executor, semanticConnection, semanticLayer)
|
|
1102
|
+
: undefined;
|
|
1096
1103
|
const requestedDomain = agentRunWorkspaceValue(request, 'domain');
|
|
1097
1104
|
const requestedPurpose = agentRunWorkspaceValue(request, 'purpose');
|
|
1098
1105
|
const requestedModelAreaId = agentRunWorkspaceValue(request, 'modelAreaId');
|
|
@@ -1139,6 +1146,26 @@ export async function startLocalServer(opts) {
|
|
|
1139
1146
|
projectSnapshots.assertCurrent(snapshotId);
|
|
1140
1147
|
},
|
|
1141
1148
|
...(semanticDriver ? { semanticDriver } : {}),
|
|
1149
|
+
...(semanticTableMapping ? { semanticTableMapping } : {}),
|
|
1150
|
+
...(semanticLayer ? {
|
|
1151
|
+
semanticQueryCompiler: async (selection) => {
|
|
1152
|
+
const compiled = await compileSemanticRuntimeQuery({
|
|
1153
|
+
...selection,
|
|
1154
|
+
dimensions: selection.dimensions ?? [],
|
|
1155
|
+
}, {
|
|
1156
|
+
projectRoot,
|
|
1157
|
+
projectConfig,
|
|
1158
|
+
detectedProvider: semanticDetectedProvider,
|
|
1159
|
+
semanticLayer: semanticLayer,
|
|
1160
|
+
driver: semanticDriver,
|
|
1161
|
+
tableMapping: semanticTableMapping,
|
|
1162
|
+
});
|
|
1163
|
+
if (!compiled) {
|
|
1164
|
+
throw new SemanticRuntimeRequiredError('The selected semantic members could not be composed. Configure dbt Cloud Semantic Layer or a compatible local MetricFlow runtime for derived metrics.');
|
|
1165
|
+
}
|
|
1166
|
+
return { sql: compiled.sql, engine: compiled.engine };
|
|
1167
|
+
},
|
|
1168
|
+
} : {}),
|
|
1142
1169
|
...(routeDecision?.meaningResolution?.selectedConceptIds.length
|
|
1143
1170
|
? { preferredEvidenceIds: routeDecision.meaningResolution.selectedConceptIds }
|
|
1144
1171
|
: {}),
|
|
@@ -2458,13 +2485,28 @@ export async function startLocalServer(opts) {
|
|
|
2458
2485
|
const activeConnection = requireActiveConnection();
|
|
2459
2486
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
2460
2487
|
const plan = buildExecutionPlan(resolved.cell, { semanticLayer, driver: activeConnection.driver, tableMapping });
|
|
2461
|
-
|
|
2488
|
+
const semanticCompose = resolved.cell.type === 'dql'
|
|
2489
|
+
&& semanticLayer
|
|
2490
|
+
&& /\btype\s*=\s*"semantic"/i.test(resolved.cell.source)
|
|
2491
|
+
? await composeSemanticBlockSqlForRuntime(resolved.cell.source, semanticLayer, {
|
|
2492
|
+
driver: activeConnection.driver,
|
|
2493
|
+
tableMapping,
|
|
2494
|
+
detectedProvider: semanticDetectedProvider,
|
|
2495
|
+
projectRoot,
|
|
2496
|
+
projectConfig,
|
|
2497
|
+
})
|
|
2498
|
+
: null;
|
|
2499
|
+
if (semanticCompose && !semanticCompose.sql) {
|
|
2500
|
+
throw new Error(semanticCompose.diagnostics.map((diagnostic) => diagnostic.message).join(' '));
|
|
2501
|
+
}
|
|
2502
|
+
const executableSql = semanticCompose?.sql ?? plan?.sql;
|
|
2503
|
+
if (!executableSql) {
|
|
2462
2504
|
snapshotCells.push({ cellId, status: 'idle', executionCount: 0, executedAt });
|
|
2463
2505
|
continue;
|
|
2464
2506
|
}
|
|
2465
|
-
const prepared = prepareLocalExecution(
|
|
2507
|
+
const prepared = prepareLocalExecution(executableSql, activeConnection, projectRoot, projectConfig);
|
|
2466
2508
|
assertAppAccess({ app, domain: resolved.domain ?? app.domain, level: 'execute' });
|
|
2467
|
-
const rawResult = await executor.executeQuery(prepared.sql, plan
|
|
2509
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables(plan?.variables ?? {}), prepared.connection);
|
|
2468
2510
|
const result = normalizeQueryResult(rawResult);
|
|
2469
2511
|
snapshotCells.push({
|
|
2470
2512
|
cellId,
|
|
@@ -2511,6 +2553,7 @@ export async function startLocalServer(opts) {
|
|
|
2511
2553
|
const invocation = prepareBlockInvocation({
|
|
2512
2554
|
source,
|
|
2513
2555
|
parameters: invocationInput?.parameters,
|
|
2556
|
+
parameterSources: invocationInput?.parameterSources,
|
|
2514
2557
|
question: invocationInput?.question,
|
|
2515
2558
|
surface: 'ask_ai',
|
|
2516
2559
|
});
|
|
@@ -2523,7 +2566,7 @@ export async function startLocalServer(opts) {
|
|
|
2523
2566
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
2524
2567
|
const plan = buildExecutionPlan({ id: `agent-${metadata.name ?? 'dql-artifact'}`, type: 'dql', source, title: metadata.name ?? 'DQL artifact' }, { semanticLayer, driver: activeConnection.driver, tableMapping, parameters: invocation.values });
|
|
2525
2568
|
const semanticCompose = semanticLayer
|
|
2526
|
-
?
|
|
2569
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
2527
2570
|
driver: activeConnection.driver,
|
|
2528
2571
|
tableMapping,
|
|
2529
2572
|
projectRoot,
|
|
@@ -2607,7 +2650,8 @@ export async function startLocalServer(opts) {
|
|
|
2607
2650
|
* candidate remains review-required and is not executed automatically.
|
|
2608
2651
|
*/
|
|
2609
2652
|
const executeExploratoryCandidate = async (candidate, schemaContext = [], question = '', requestedTopN) => {
|
|
2610
|
-
const
|
|
2653
|
+
const activeConnection = requireActiveConnection();
|
|
2654
|
+
const preflight = repairExploratorySqlBeforeExecution(candidate.sql, schemaContext, question, activeConnection.driver);
|
|
2611
2655
|
const boundedSql = applyRequestedTopNToExploratorySql(preflight.sql, requestedTopN);
|
|
2612
2656
|
const repairs = boundedSql === preflight.sql
|
|
2613
2657
|
? [...preflight.repairs]
|
|
@@ -2620,7 +2664,7 @@ export async function startLocalServer(opts) {
|
|
|
2620
2664
|
error: preflight.blockedReason,
|
|
2621
2665
|
};
|
|
2622
2666
|
}
|
|
2623
|
-
const analysis = analyzeSqlReferences(boundedSql);
|
|
2667
|
+
const analysis = analyzeSqlReferences(boundedSql, activeConnection.driver);
|
|
2624
2668
|
if (!analysis.parsed) {
|
|
2625
2669
|
return { proofs: [], sql: boundedSql, repairs, error: 'DQL could not parse the exploratory SQL to validate its join predicates.' };
|
|
2626
2670
|
}
|
|
@@ -2676,7 +2720,6 @@ export async function startLocalServer(opts) {
|
|
|
2676
2720
|
if (probeableJoins.length < analysis.joins.length) {
|
|
2677
2721
|
repairs.push(`Skipped join probes for ${analysis.joins.length - probeableJoins.length} join(s) on derived (CTE) relations; the final query execution validates them.`);
|
|
2678
2722
|
}
|
|
2679
|
-
const activeConnection = requireActiveConnection();
|
|
2680
2723
|
const proofs = [];
|
|
2681
2724
|
try {
|
|
2682
2725
|
for (const [index, join] of probeableJoins.entries()) {
|
|
@@ -3545,7 +3588,7 @@ export async function startLocalServer(opts) {
|
|
|
3545
3588
|
throw new Error(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'}: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
3546
3589
|
}
|
|
3547
3590
|
const semanticCompose = semanticLayer
|
|
3548
|
-
?
|
|
3591
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
3549
3592
|
driver: activeConnection.driver,
|
|
3550
3593
|
tableMapping,
|
|
3551
3594
|
projectRoot,
|
|
@@ -3599,7 +3642,7 @@ export async function startLocalServer(opts) {
|
|
|
3599
3642
|
// pre-compiled query, that's the query (not a recompiled metric), so the test's
|
|
3600
3643
|
// output columns match the block's declared outputs.
|
|
3601
3644
|
const semanticCompose = semanticLayer
|
|
3602
|
-
?
|
|
3645
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
3603
3646
|
driver: activeConnection.driver,
|
|
3604
3647
|
tableMapping,
|
|
3605
3648
|
projectRoot,
|
|
@@ -6017,7 +6060,7 @@ export async function startLocalServer(opts) {
|
|
|
6017
6060
|
operator: filter.operator,
|
|
6018
6061
|
values: Array.isArray(filter.value) ? filter.value.map(String) : [String(filter.value)],
|
|
6019
6062
|
}));
|
|
6020
|
-
const composed = composeRuntimeSemanticQuery({
|
|
6063
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
6021
6064
|
metrics: item.semantic.metrics,
|
|
6022
6065
|
dimensions: item.semantic.dimensions ?? [],
|
|
6023
6066
|
filters: [...staticFilters, ...activeFilters],
|
|
@@ -6108,7 +6151,7 @@ export async function startLocalServer(opts) {
|
|
|
6108
6151
|
continue;
|
|
6109
6152
|
}
|
|
6110
6153
|
const semanticCompose = semanticLayer
|
|
6111
|
-
?
|
|
6154
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
6112
6155
|
driver: targetConnection.driver,
|
|
6113
6156
|
tableMapping,
|
|
6114
6157
|
projectRoot,
|
|
@@ -7487,7 +7530,7 @@ export async function startLocalServer(opts) {
|
|
|
7487
7530
|
const activeConnection = requireActiveConnection();
|
|
7488
7531
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
7489
7532
|
const semanticCompose = semanticLayer
|
|
7490
|
-
?
|
|
7533
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
7491
7534
|
driver: activeConnection.driver,
|
|
7492
7535
|
tableMapping,
|
|
7493
7536
|
projectRoot,
|
|
@@ -7993,8 +8036,23 @@ export async function startLocalServer(opts) {
|
|
|
7993
8036
|
draftSave: readiness.candidate.draftSave ?? { status: 'pending' },
|
|
7994
8037
|
};
|
|
7995
8038
|
writeBlockStudioImportCandidate(projectRoot, importId, next);
|
|
7996
|
-
|
|
7997
|
-
|
|
8039
|
+
let compiledManifest;
|
|
8040
|
+
let lineageRefresh;
|
|
8041
|
+
try {
|
|
8042
|
+
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
8043
|
+
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
8044
|
+
}
|
|
8045
|
+
catch (error) {
|
|
8046
|
+
lineageRefresh = {
|
|
8047
|
+
status: 'failed',
|
|
8048
|
+
message: error instanceof Error ? error.message : String(error),
|
|
8049
|
+
};
|
|
8050
|
+
}
|
|
8051
|
+
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
8052
|
+
const payload = {
|
|
8053
|
+
...openBlockStudioDocument(projectRoot, savedPath, semanticLayer),
|
|
8054
|
+
lineageRefresh,
|
|
8055
|
+
};
|
|
7998
8056
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7999
8057
|
res.end(serializeJSON({ candidate: next, block: payload, certification }));
|
|
8000
8058
|
return;
|
|
@@ -8107,10 +8165,23 @@ export async function startLocalServer(opts) {
|
|
|
8107
8165
|
}
|
|
8108
8166
|
const nextSession = { ...session, candidates: nextCandidates, updatedAt: new Date().toISOString() };
|
|
8109
8167
|
writeBlockStudioImportSession(projectRoot, nextSession);
|
|
8110
|
-
|
|
8111
|
-
|
|
8168
|
+
let lineageRefresh;
|
|
8169
|
+
if (saved.length > 0) {
|
|
8170
|
+
let compiledManifest;
|
|
8171
|
+
try {
|
|
8172
|
+
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
8173
|
+
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
8174
|
+
}
|
|
8175
|
+
catch (error) {
|
|
8176
|
+
lineageRefresh = {
|
|
8177
|
+
status: 'failed',
|
|
8178
|
+
message: error instanceof Error ? error.message : String(error),
|
|
8179
|
+
};
|
|
8180
|
+
}
|
|
8181
|
+
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
8182
|
+
}
|
|
8112
8183
|
res.writeHead(errors.length > 0 ? 207 : 200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8113
|
-
res.end(serializeJSON({ ok: errors.length === 0, session: nextSession, saved, errors }));
|
|
8184
|
+
res.end(serializeJSON({ ok: errors.length === 0, session: nextSession, saved, errors, lineageRefresh }));
|
|
8114
8185
|
}
|
|
8115
8186
|
catch (error) {
|
|
8116
8187
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -8254,9 +8325,14 @@ export async function startLocalServer(opts) {
|
|
|
8254
8325
|
const connections = getProjectConnectionsForApi(cfg);
|
|
8255
8326
|
const defaultKey = resolveDefaultConnectionKey(cfg, connections) ?? Object.keys(connections)[0] ?? 'default';
|
|
8256
8327
|
const userPrefs = readUserPrefs(userPrefsPath);
|
|
8328
|
+
// UI-009 / PERF-001: Block Studio already loads the canonical semantic
|
|
8329
|
+
// layer. Let it omit this second, potentially multi-megabyte rendering
|
|
8330
|
+
// of the same 7,500+ object catalog while keeping the route compatible
|
|
8331
|
+
// for older clients.
|
|
8332
|
+
const includeSemantic = url.searchParams.get('includeSemantic') !== 'false';
|
|
8257
8333
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8258
8334
|
res.end(serializeJSON({
|
|
8259
|
-
semanticTree: semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
8335
|
+
semanticTree: includeSemantic && semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
8260
8336
|
databaseTree: await buildDatabaseSchemaTree(projectRoot, executor, connection),
|
|
8261
8337
|
connection: {
|
|
8262
8338
|
default: defaultKey,
|
|
@@ -8417,8 +8493,26 @@ export async function startLocalServer(opts) {
|
|
|
8417
8493
|
stableSuffix: metadata.candidateId,
|
|
8418
8494
|
})
|
|
8419
8495
|
: saveBlockStudioArtifacts(projectRoot, saveOptions);
|
|
8420
|
-
|
|
8421
|
-
|
|
8496
|
+
let compiledManifest;
|
|
8497
|
+
let lineageRefresh;
|
|
8498
|
+
try {
|
|
8499
|
+
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
8500
|
+
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
8501
|
+
}
|
|
8502
|
+
catch (error) {
|
|
8503
|
+
// The block is already safely stored. Preserve it and report the
|
|
8504
|
+
// rebuild failure separately instead of turning a successful save
|
|
8505
|
+
// into a misleading 500 response.
|
|
8506
|
+
lineageRefresh = {
|
|
8507
|
+
status: 'failed',
|
|
8508
|
+
message: error instanceof Error ? error.message : String(error),
|
|
8509
|
+
};
|
|
8510
|
+
}
|
|
8511
|
+
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
8512
|
+
const payload = {
|
|
8513
|
+
...openBlockStudioDocument(projectRoot, savedPath, semanticLayer),
|
|
8514
|
+
lineageRefresh,
|
|
8515
|
+
};
|
|
8422
8516
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8423
8517
|
res.end(serializeJSON(payload));
|
|
8424
8518
|
}
|
|
@@ -8817,6 +8911,47 @@ export async function startLocalServer(opts) {
|
|
|
8817
8911
|
}
|
|
8818
8912
|
return;
|
|
8819
8913
|
}
|
|
8914
|
+
// ── Semantic runtime adapters (API-004 / UI-009 / E2E-008) ───────────────
|
|
8915
|
+
if (req.method === 'GET' && path === '/api/semantic-runtime') {
|
|
8916
|
+
try {
|
|
8917
|
+
const runtime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
8918
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8919
|
+
res.end(serializeJSON({ ...getSemanticRuntimeSettings(projectRoot), runtime }));
|
|
8920
|
+
}
|
|
8921
|
+
catch (error) {
|
|
8922
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8923
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
8924
|
+
}
|
|
8925
|
+
return;
|
|
8926
|
+
}
|
|
8927
|
+
if (req.method === 'POST' && path === '/api/semantic-runtime/dbt-cloud/test') {
|
|
8928
|
+
try {
|
|
8929
|
+
const body = await readJSON(req);
|
|
8930
|
+
const result = await testSemanticRuntimeDraft(projectRoot, body);
|
|
8931
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8932
|
+
res.end(serializeJSON(result));
|
|
8933
|
+
}
|
|
8934
|
+
catch (error) {
|
|
8935
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8936
|
+
res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
|
|
8937
|
+
}
|
|
8938
|
+
return;
|
|
8939
|
+
}
|
|
8940
|
+
if (req.method === 'POST' && path === '/api/semantic-runtime/dbt-cloud/apply') {
|
|
8941
|
+
try {
|
|
8942
|
+
const body = await readJSON(req);
|
|
8943
|
+
const result = await testSemanticRuntimeDraft(projectRoot, body);
|
|
8944
|
+
const settings = saveTestedSemanticRuntimeSettings(projectRoot, body, result);
|
|
8945
|
+
const runtime = await getSemanticRuntimeStatus(projectRoot);
|
|
8946
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8947
|
+
res.end(serializeJSON({ ok: true, ...settings, runtime }));
|
|
8948
|
+
}
|
|
8949
|
+
catch (error) {
|
|
8950
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8951
|
+
res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
|
|
8952
|
+
}
|
|
8953
|
+
return;
|
|
8954
|
+
}
|
|
8820
8955
|
// ── Semantic layer discovery API ─────────────────────────────────────────
|
|
8821
8956
|
if (req.method === 'GET' && path === '/api/semantic-layer') {
|
|
8822
8957
|
const userPrefs = readUserPrefs(userPrefsPath);
|
|
@@ -8846,8 +8981,7 @@ export async function startLocalServer(opts) {
|
|
|
8846
8981
|
const dbtManifestReady = provider === 'dbt'
|
|
8847
8982
|
? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
|
|
8848
8983
|
: false;
|
|
8849
|
-
const
|
|
8850
|
-
const dbtExecutionReady = dbtManifestReady && metricFlowReady;
|
|
8984
|
+
const semanticRuntime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
8851
8985
|
const metrics = semanticLayer.listMetrics().map((m) => ({
|
|
8852
8986
|
name: m.name,
|
|
8853
8987
|
label: m.label,
|
|
@@ -8862,7 +8996,7 @@ export async function startLocalServer(opts) {
|
|
|
8862
8996
|
typeParams: m.typeParams ?? null,
|
|
8863
8997
|
filter: m.filter ?? null,
|
|
8864
8998
|
source: m.source ?? null,
|
|
8865
|
-
execution:
|
|
8999
|
+
execution: runtimeMetricExecutionCapability(m.name, semanticLayer, provider, semanticRuntime),
|
|
8866
9000
|
}));
|
|
8867
9001
|
const measures = semanticLayer.listMeasures().map((m) => ({
|
|
8868
9002
|
name: m.name,
|
|
@@ -8960,24 +9094,21 @@ export async function startLocalServer(opts) {
|
|
|
8960
9094
|
owner: q.owner ?? null,
|
|
8961
9095
|
source: q.source ?? null,
|
|
8962
9096
|
}));
|
|
8963
|
-
const dbtExecutionSetup =
|
|
8964
|
-
?
|
|
8965
|
-
:
|
|
8966
|
-
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.'
|
|
8967
|
-
: !dbtManifestReady
|
|
8968
|
-
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists.'
|
|
8969
|
-
: 'Install MetricFlow so `mf` is on PATH, or set DQL_METRICFLOW_BIN to the MetricFlow executable.';
|
|
9097
|
+
const dbtExecutionSetup = !dbtManifestReady
|
|
9098
|
+
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists.'
|
|
9099
|
+
: semanticRuntime.setup;
|
|
8970
9100
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8971
9101
|
res.end(serializeJSON({
|
|
8972
9102
|
available: true,
|
|
8973
9103
|
provider,
|
|
8974
9104
|
execution: provider === 'dbt'
|
|
8975
9105
|
? {
|
|
8976
|
-
engine:
|
|
8977
|
-
ready:
|
|
9106
|
+
engine: semanticRuntime.active,
|
|
9107
|
+
ready: semanticRuntime.active !== 'native' || metrics.every((metric) => metric.execution.status === 'ready'),
|
|
8978
9108
|
setup: dbtExecutionSetup,
|
|
9109
|
+
adapters: semanticRuntime.adapters,
|
|
8979
9110
|
}
|
|
8980
|
-
: { engine: 'native', ready: true, setup: null },
|
|
9111
|
+
: { engine: 'native', ready: true, setup: null, adapters: semanticRuntime.adapters },
|
|
8981
9112
|
errors: semanticLayerErrors,
|
|
8982
9113
|
metrics,
|
|
8983
9114
|
measures,
|
|
@@ -9314,7 +9445,7 @@ export async function startLocalServer(opts) {
|
|
|
9314
9445
|
.split(',')
|
|
9315
9446
|
.map((value) => value.trim())
|
|
9316
9447
|
.filter(Boolean);
|
|
9317
|
-
const dimensions = semanticLayer
|
|
9448
|
+
const dimensions = (await listRuntimeCompatibleDimensions(projectRoot, semanticLayer, metrics)).map((d) => ({
|
|
9318
9449
|
name: d.name,
|
|
9319
9450
|
label: d.label,
|
|
9320
9451
|
description: d.description,
|
|
@@ -9827,37 +9958,8 @@ export async function startLocalServer(opts) {
|
|
|
9827
9958
|
// Resolve which connection to use — request can override default
|
|
9828
9959
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
9829
9960
|
const driver = targetConnection.driver;
|
|
9830
|
-
|
|
9831
|
-
|
|
9832
|
-
try {
|
|
9833
|
-
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
|
|
9834
|
-
const dbTableNames = new Set();
|
|
9835
|
-
const schemaQualified = new Map();
|
|
9836
|
-
for (const row of tablesResult.rows) {
|
|
9837
|
-
const schema = String(row['table_schema'] ?? '');
|
|
9838
|
-
const name = String(row['table_name'] ?? '');
|
|
9839
|
-
dbTableNames.add(name);
|
|
9840
|
-
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
9841
|
-
}
|
|
9842
|
-
// For each table in the semantic layer, map to qualified name if it exists
|
|
9843
|
-
const allSemanticTables = new Set();
|
|
9844
|
-
for (const m of semanticLayer.listMetrics())
|
|
9845
|
-
allSemanticTables.add(m.table);
|
|
9846
|
-
for (const d of semanticLayer.listDimensions())
|
|
9847
|
-
allSemanticTables.add(d.table);
|
|
9848
|
-
tableMapping = {};
|
|
9849
|
-
for (const semTable of allSemanticTables) {
|
|
9850
|
-
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
9851
|
-
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
9852
|
-
}
|
|
9853
|
-
}
|
|
9854
|
-
if (Object.keys(tableMapping).length === 0)
|
|
9855
|
-
tableMapping = undefined;
|
|
9856
|
-
}
|
|
9857
|
-
catch {
|
|
9858
|
-
// Non-fatal: proceed without table mapping
|
|
9859
|
-
}
|
|
9860
|
-
const composed = composeRuntimeSemanticQuery({
|
|
9961
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
9962
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
9861
9963
|
metrics,
|
|
9862
9964
|
dimensions,
|
|
9863
9965
|
filters,
|
|
@@ -9875,10 +9977,10 @@ export async function startLocalServer(opts) {
|
|
|
9875
9977
|
});
|
|
9876
9978
|
if (!composed) {
|
|
9877
9979
|
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
9878
|
-
const
|
|
9980
|
+
const semanticRuntime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
9879
9981
|
const blocked = metrics.map((metricName) => ({
|
|
9880
9982
|
metric: metricName,
|
|
9881
|
-
...
|
|
9983
|
+
...runtimeMetricExecutionCapability(metricName, semanticLayer, provider, semanticRuntime),
|
|
9882
9984
|
})).filter((capability) => capability.status !== 'ready');
|
|
9883
9985
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9884
9986
|
res.end(serializeJSON({
|
|
@@ -9908,13 +10010,14 @@ export async function startLocalServer(opts) {
|
|
|
9908
10010
|
res.end(serializeJSON({ error: error.message, code: 'unauthorized' }));
|
|
9909
10011
|
return;
|
|
9910
10012
|
}
|
|
9911
|
-
const
|
|
10013
|
+
const runtimeError = isSemanticRuntimeError(error);
|
|
10014
|
+
const status = runtimeError ? 400 : 500;
|
|
9912
10015
|
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9913
10016
|
res.end(serializeJSON({
|
|
9914
10017
|
error: error instanceof Error ? error.message : String(error),
|
|
9915
|
-
code:
|
|
9916
|
-
hint:
|
|
9917
|
-
? '
|
|
10018
|
+
code: runtimeError ? 'SEMANTIC_RUNTIME_REQUIRED' : undefined,
|
|
10019
|
+
hint: runtimeError
|
|
10020
|
+
? 'Configure dbt Cloud Semantic Layer in Project & dbt settings, or install a compatible local MetricFlow runtime.'
|
|
9918
10021
|
: undefined,
|
|
9919
10022
|
}));
|
|
9920
10023
|
}
|
|
@@ -9931,31 +10034,8 @@ export async function startLocalServer(opts) {
|
|
|
9931
10034
|
const { metrics = [], dimensions = [], filters = [], limit, timeDimension, orderBy, savedQuery, engine } = body;
|
|
9932
10035
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
9933
10036
|
const driver = targetConnection.driver;
|
|
9934
|
-
|
|
9935
|
-
|
|
9936
|
-
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
|
|
9937
|
-
const schemaQualified = new Map();
|
|
9938
|
-
for (const row of tablesResult.rows) {
|
|
9939
|
-
const schema = String(row['table_schema'] ?? '');
|
|
9940
|
-
const name = String(row['table_name'] ?? '');
|
|
9941
|
-
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
9942
|
-
}
|
|
9943
|
-
tableMapping = {};
|
|
9944
|
-
for (const metric of semanticLayer.listMetrics()) {
|
|
9945
|
-
if (schemaQualified.has(metric.table))
|
|
9946
|
-
tableMapping[metric.table] = schemaQualified.get(metric.table);
|
|
9947
|
-
}
|
|
9948
|
-
for (const dimension of semanticLayer.listDimensions()) {
|
|
9949
|
-
if (schemaQualified.has(dimension.table))
|
|
9950
|
-
tableMapping[dimension.table] = schemaQualified.get(dimension.table);
|
|
9951
|
-
}
|
|
9952
|
-
if (Object.keys(tableMapping).length === 0)
|
|
9953
|
-
tableMapping = undefined;
|
|
9954
|
-
}
|
|
9955
|
-
catch {
|
|
9956
|
-
tableMapping = undefined;
|
|
9957
|
-
}
|
|
9958
|
-
const composed = composeRuntimeSemanticQuery({
|
|
10037
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
10038
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
9959
10039
|
metrics,
|
|
9960
10040
|
dimensions,
|
|
9961
10041
|
filters,
|
|
@@ -9973,10 +10053,10 @@ export async function startLocalServer(opts) {
|
|
|
9973
10053
|
});
|
|
9974
10054
|
if (!composed) {
|
|
9975
10055
|
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
9976
|
-
const
|
|
10056
|
+
const semanticRuntime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
9977
10057
|
const blocked = metrics.map((metricName) => ({
|
|
9978
10058
|
metric: metricName,
|
|
9979
|
-
...
|
|
10059
|
+
...runtimeMetricExecutionCapability(metricName, semanticLayer, provider, semanticRuntime),
|
|
9980
10060
|
})).filter((capability) => capability.status !== 'ready');
|
|
9981
10061
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9982
10062
|
res.end(serializeJSON({
|
|
@@ -10000,13 +10080,14 @@ export async function startLocalServer(opts) {
|
|
|
10000
10080
|
}));
|
|
10001
10081
|
}
|
|
10002
10082
|
catch (error) {
|
|
10003
|
-
const
|
|
10083
|
+
const runtimeError = isSemanticRuntimeError(error);
|
|
10084
|
+
const status = runtimeError ? 400 : 500;
|
|
10004
10085
|
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10005
10086
|
res.end(serializeJSON({
|
|
10006
10087
|
error: error instanceof Error ? error.message : String(error),
|
|
10007
|
-
code:
|
|
10008
|
-
hint:
|
|
10009
|
-
? '
|
|
10088
|
+
code: runtimeError ? 'SEMANTIC_RUNTIME_REQUIRED' : undefined,
|
|
10089
|
+
hint: runtimeError
|
|
10090
|
+
? 'Configure dbt Cloud Semantic Layer in Project & dbt settings, or install a compatible local MetricFlow runtime.'
|
|
10010
10091
|
: undefined,
|
|
10011
10092
|
}));
|
|
10012
10093
|
}
|
|
@@ -10031,7 +10112,8 @@ export async function startLocalServer(opts) {
|
|
|
10031
10112
|
return;
|
|
10032
10113
|
}
|
|
10033
10114
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
10034
|
-
const
|
|
10115
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
10116
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
10035
10117
|
metrics,
|
|
10036
10118
|
dimensions,
|
|
10037
10119
|
filters,
|
|
@@ -10042,6 +10124,7 @@ export async function startLocalServer(opts) {
|
|
|
10042
10124
|
projectConfig,
|
|
10043
10125
|
detectedProvider: semanticDetectedProvider,
|
|
10044
10126
|
driver: targetConnection.driver,
|
|
10127
|
+
tableMapping,
|
|
10045
10128
|
});
|
|
10046
10129
|
if (!composed) {
|
|
10047
10130
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -10072,13 +10155,14 @@ export async function startLocalServer(opts) {
|
|
|
10072
10155
|
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
10073
10156
|
return;
|
|
10074
10157
|
}
|
|
10075
|
-
const
|
|
10158
|
+
const runtimeError = isSemanticRuntimeError(error);
|
|
10159
|
+
const status = runtimeError ? 400 : 500;
|
|
10076
10160
|
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10077
10161
|
res.end(serializeJSON({
|
|
10078
10162
|
error: error instanceof Error ? error.message : String(error),
|
|
10079
|
-
code:
|
|
10080
|
-
hint:
|
|
10081
|
-
? '
|
|
10163
|
+
code: runtimeError ? 'SEMANTIC_RUNTIME_REQUIRED' : undefined,
|
|
10164
|
+
hint: runtimeError
|
|
10165
|
+
? 'Configure dbt Cloud Semantic Layer in Project & dbt settings, or install a compatible local MetricFlow runtime.'
|
|
10082
10166
|
: undefined,
|
|
10083
10167
|
}));
|
|
10084
10168
|
}
|
|
@@ -10400,7 +10484,12 @@ export async function startLocalServer(opts) {
|
|
|
10400
10484
|
parameters,
|
|
10401
10485
|
question: typeof body.question === 'string' ? body.question : undefined,
|
|
10402
10486
|
});
|
|
10403
|
-
const contract = prepareBlockInvocation({
|
|
10487
|
+
const contract = prepareBlockInvocation({
|
|
10488
|
+
source,
|
|
10489
|
+
parameters,
|
|
10490
|
+
question: typeof body.question === 'string' ? body.question : undefined,
|
|
10491
|
+
surface: 'ask_ai',
|
|
10492
|
+
});
|
|
10404
10493
|
const certified = /\bstatus\s*=\s*"certified"/i.test(source);
|
|
10405
10494
|
const semantic = /\btype\s*=\s*"semantic"/i.test(source);
|
|
10406
10495
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -10496,6 +10585,7 @@ export async function startLocalServer(opts) {
|
|
|
10496
10585
|
parameters: body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
10497
10586
|
? body.parameters
|
|
10498
10587
|
: {},
|
|
10588
|
+
question: typeof body.question === 'string' ? body.question : undefined,
|
|
10499
10589
|
surface: 'notebook',
|
|
10500
10590
|
})
|
|
10501
10591
|
: null;
|
|
@@ -10514,15 +10604,31 @@ export async function startLocalServer(opts) {
|
|
|
10514
10604
|
tableMapping,
|
|
10515
10605
|
parameters: invocation?.values,
|
|
10516
10606
|
});
|
|
10517
|
-
|
|
10607
|
+
const semanticCompose = executableCell.type === 'dql'
|
|
10608
|
+
&& semanticLayer
|
|
10609
|
+
&& /\btype\s*=\s*"semantic"/i.test(executableCell.source)
|
|
10610
|
+
? await composeSemanticBlockSqlForRuntime(executableCell.source, semanticLayer, {
|
|
10611
|
+
driver: cellConnection.driver,
|
|
10612
|
+
tableMapping,
|
|
10613
|
+
parameters: invocation?.values,
|
|
10614
|
+
detectedProvider: semanticDetectedProvider,
|
|
10615
|
+
projectRoot,
|
|
10616
|
+
projectConfig,
|
|
10617
|
+
})
|
|
10618
|
+
: null;
|
|
10619
|
+
if (semanticCompose && !semanticCompose.sql) {
|
|
10620
|
+
throw new Error(semanticCompose.diagnostics.map((diagnostic) => diagnostic.message).join(' '));
|
|
10621
|
+
}
|
|
10622
|
+
const executableSql = semanticCompose?.sql ?? plan?.sql;
|
|
10623
|
+
if (!executableSql) {
|
|
10518
10624
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10519
10625
|
res.end(serializeJSON({ cellType: cell.type, result: null }));
|
|
10520
10626
|
return;
|
|
10521
10627
|
}
|
|
10522
|
-
const prepared = prepareLocalExecution(
|
|
10628
|
+
const prepared = prepareLocalExecution(executableSql, cellConnection, projectRoot, projectConfig);
|
|
10523
10629
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
10524
10630
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
10525
|
-
const rawResult = await executor.executeQuery(prepared.sql, plan
|
|
10631
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...(invocation?.values ?? {}) }), prepared.connection);
|
|
10526
10632
|
const normalized = normalizeQueryResult(rawResult);
|
|
10527
10633
|
// Enforce the block's declared invariants against the result set. This
|
|
10528
10634
|
// is additive: blocks without invariants produce `null` and the
|
|
@@ -10536,29 +10642,29 @@ export async function startLocalServer(opts) {
|
|
|
10536
10642
|
recordNotebookQueryRun(projectRoot, {
|
|
10537
10643
|
notebookPath: execContext.notebookPath,
|
|
10538
10644
|
cellId: execContext.cellId ?? cell.id,
|
|
10539
|
-
cellName: execContext.cellName ?? plan
|
|
10645
|
+
cellName: execContext.cellName ?? plan?.title ?? resolved.blockName,
|
|
10540
10646
|
researchRunId: execContext.researchRunId,
|
|
10541
10647
|
source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_cell'),
|
|
10542
10648
|
status: 'success',
|
|
10543
10649
|
rowCount: normalized.rowCount ?? normalized.rows.length,
|
|
10544
10650
|
durationMs: Date.now() - start,
|
|
10545
|
-
sql:
|
|
10651
|
+
sql: executableSql,
|
|
10546
10652
|
objectKey: resolved.blockPath,
|
|
10547
10653
|
});
|
|
10548
10654
|
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
10549
10655
|
status: 'success',
|
|
10550
10656
|
resultPreview: normalized,
|
|
10551
|
-
sql:
|
|
10657
|
+
sql: executableSql,
|
|
10552
10658
|
});
|
|
10553
10659
|
}
|
|
10554
10660
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10555
10661
|
res.end(serializeJSON({
|
|
10556
10662
|
cellType: cell.type,
|
|
10557
|
-
title: plan
|
|
10663
|
+
title: plan?.title,
|
|
10558
10664
|
blockName: resolved.blockName,
|
|
10559
10665
|
blockPath: resolved.blockPath,
|
|
10560
|
-
chartConfig: plan
|
|
10561
|
-
tests: plan
|
|
10666
|
+
chartConfig: plan?.chartConfig,
|
|
10667
|
+
tests: plan?.tests,
|
|
10562
10668
|
result: normalized,
|
|
10563
10669
|
...(invocation ? {
|
|
10564
10670
|
invocation: {
|
|
@@ -11491,9 +11597,34 @@ function latestOpenContextBootstrapSession(projectRoot) {
|
|
|
11491
11597
|
}
|
|
11492
11598
|
return null;
|
|
11493
11599
|
}
|
|
11494
|
-
|
|
11600
|
+
/**
|
|
11601
|
+
* API-004 / E2E-006: save-time compile for Block Studio. The manifest is
|
|
11602
|
+
* replaced atomically so lineage readers and agent retrieval never observe a
|
|
11603
|
+
* half-written snapshot, and an existing manifest survives compilation errors.
|
|
11604
|
+
*/
|
|
11605
|
+
export function compileBlockStudioManifest(projectRoot, projectConfig = loadProjectConfig(projectRoot)) {
|
|
11606
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, projectConfig) ?? undefined;
|
|
11607
|
+
const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook', dbtManifestPath });
|
|
11608
|
+
const manifestPath = join(projectRoot, 'dql-manifest.json');
|
|
11609
|
+
const tempPath = `${manifestPath}.tmp-${process.pid}-${Date.now()}`;
|
|
11495
11610
|
try {
|
|
11496
|
-
|
|
11611
|
+
writeFileSync(tempPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
|
|
11612
|
+
renameSync(tempPath, manifestPath);
|
|
11613
|
+
}
|
|
11614
|
+
catch (error) {
|
|
11615
|
+
rmSync(tempPath, { force: true });
|
|
11616
|
+
throw error;
|
|
11617
|
+
}
|
|
11618
|
+
_lineageCache.delete(projectRoot);
|
|
11619
|
+
return manifest;
|
|
11620
|
+
}
|
|
11621
|
+
async function refreshLocalMetadataCatalog(projectRoot, manifest, semanticLayer) {
|
|
11622
|
+
try {
|
|
11623
|
+
await ensureMetadataCatalogFresh(projectRoot, {
|
|
11624
|
+
force: true,
|
|
11625
|
+
...(manifest ? { manifest } : {}),
|
|
11626
|
+
...(semanticLayer ? { semanticLayer } : {}),
|
|
11627
|
+
});
|
|
11497
11628
|
}
|
|
11498
11629
|
catch {
|
|
11499
11630
|
// The catalog is a rebuildable local cache. Save/certify flows should not
|
|
@@ -12946,13 +13077,13 @@ export function buildAgentPreviewSql(sql) {
|
|
|
12946
13077
|
* only when the owning entity is retained in GROUP BY. It never invents a
|
|
12947
13078
|
* relation, join key, or allocation rule.
|
|
12948
13079
|
*/
|
|
12949
|
-
export function repairExploratorySqlBeforeExecution(sql, schemaContext, question = '') {
|
|
13080
|
+
export function repairExploratorySqlBeforeExecution(sql, schemaContext, question = '', dialect = 'duckdb') {
|
|
12950
13081
|
const repairs = [];
|
|
12951
|
-
let repairedSql = qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs);
|
|
12952
|
-
repairedSql = repairExploratoryRelationQualifiers(repairedSql, repairs);
|
|
12953
|
-
repairedSql = repairExploratoryLifetimeMeasureSelection(repairedSql, schemaContext, question, repairs);
|
|
13082
|
+
let repairedSql = qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs, dialect);
|
|
13083
|
+
repairedSql = repairExploratoryRelationQualifiers(repairedSql, repairs, dialect);
|
|
13084
|
+
repairedSql = repairExploratoryLifetimeMeasureSelection(repairedSql, schemaContext, question, repairs, dialect);
|
|
12954
13085
|
repairedSql = repairExploratoryMisleadingPercentAliases(repairedSql, question, repairs);
|
|
12955
|
-
const grainRepair = repairExploratoryNonAdditiveAggregates(repairedSql, schemaContext, repairs);
|
|
13086
|
+
const grainRepair = repairExploratoryNonAdditiveAggregates(repairedSql, schemaContext, repairs, dialect);
|
|
12956
13087
|
repairedSql = grainRepair.sql;
|
|
12957
13088
|
return {
|
|
12958
13089
|
sql: repairedSql,
|
|
@@ -13003,8 +13134,8 @@ export function applyRequestedTopNToExploratorySql(sql, requestedTopN) {
|
|
|
13003
13134
|
}
|
|
13004
13135
|
return `${withoutTerminator}\nLIMIT ${requestedTopN}`;
|
|
13005
13136
|
}
|
|
13006
|
-
function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs) {
|
|
13007
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13137
|
+
function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs, dialect = 'duckdb') {
|
|
13138
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13008
13139
|
if (!analysis.parsed)
|
|
13009
13140
|
return sql;
|
|
13010
13141
|
let repaired = sql;
|
|
@@ -13032,8 +13163,8 @@ function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs) {
|
|
|
13032
13163
|
}
|
|
13033
13164
|
return repaired;
|
|
13034
13165
|
}
|
|
13035
|
-
function repairExploratoryRelationQualifiers(sql, repairs) {
|
|
13036
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13166
|
+
function repairExploratoryRelationQualifiers(sql, repairs, dialect = 'duckdb') {
|
|
13167
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13037
13168
|
if (!analysis.parsed)
|
|
13038
13169
|
return sql;
|
|
13039
13170
|
const declared = Object.entries(analysis.aliasToRelation).map(([qualifier, relation]) => ({
|
|
@@ -13056,10 +13187,10 @@ function repairExploratoryRelationQualifiers(sql, repairs) {
|
|
|
13056
13187
|
}
|
|
13057
13188
|
return repaired;
|
|
13058
13189
|
}
|
|
13059
|
-
function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question, repairs) {
|
|
13190
|
+
function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question, repairs, dialect = 'duckdb') {
|
|
13060
13191
|
if (!/\b(lifetime|life\s*span|lifespan|customer\s+life)\b/i.test(question))
|
|
13061
13192
|
return sql;
|
|
13062
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13193
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13063
13194
|
if (!analysis.parsed || analysis.joins.length === 0)
|
|
13064
13195
|
return sql;
|
|
13065
13196
|
const groupClause = sql.match(/\bgroup\s+by\s+([\s\S]*?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i)?.[1] ?? '';
|
|
@@ -13085,8 +13216,8 @@ function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question,
|
|
|
13085
13216
|
}
|
|
13086
13217
|
return repaired;
|
|
13087
13218
|
}
|
|
13088
|
-
function repairExploratoryNonAdditiveAggregates(sql, schemaContext, repairs) {
|
|
13089
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13219
|
+
function repairExploratoryNonAdditiveAggregates(sql, schemaContext, repairs, dialect = 'duckdb') {
|
|
13220
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13090
13221
|
if (!analysis.parsed || analysis.joins.length === 0)
|
|
13091
13222
|
return { sql };
|
|
13092
13223
|
const risky = analysis.aggregates.filter((aggregate) => aggregate.func.toLowerCase() === 'sum'
|
|
@@ -13906,28 +14037,6 @@ export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer
|
|
|
13906
14037
|
validation: validateBlockStudioSource(source, semanticLayer),
|
|
13907
14038
|
};
|
|
13908
14039
|
}
|
|
13909
|
-
function semanticMetricExecutionCapability(metricName, semanticLayer, provider, metricFlowReady, driver) {
|
|
13910
|
-
if (provider === 'dbt' && metricFlowReady) {
|
|
13911
|
-
return { status: 'ready', engine: 'metricflow', reason: null };
|
|
13912
|
-
}
|
|
13913
|
-
const native = semanticLayer.composeQuery({ metrics: [metricName], dimensions: [], driver });
|
|
13914
|
-
if (native)
|
|
13915
|
-
return { status: 'ready', engine: 'native', reason: null };
|
|
13916
|
-
const metric = semanticLayer.getMetric(metricName);
|
|
13917
|
-
if (provider === 'dbt') {
|
|
13918
|
-
const kind = metric?.metricType || metric?.aggregation || metric?.type || 'metric';
|
|
13919
|
-
return {
|
|
13920
|
-
status: 'requires_setup',
|
|
13921
|
-
engine: null,
|
|
13922
|
-
reason: `${kind} metric requires MetricFlow execution. Install/configure the MetricFlow CLI, then refresh the semantic layer.`,
|
|
13923
|
-
};
|
|
13924
|
-
}
|
|
13925
|
-
return {
|
|
13926
|
-
status: 'unsupported',
|
|
13927
|
-
engine: null,
|
|
13928
|
-
reason: 'The metric does not have enough composable measure and relation metadata.',
|
|
13929
|
-
};
|
|
13930
|
-
}
|
|
13931
14040
|
function parseBlockStudioArrayField(source, key) {
|
|
13932
14041
|
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([\\s\\S]*?)\\]`, 'i'));
|
|
13933
14042
|
if (!match)
|
|
@@ -13980,8 +14089,7 @@ export async function resolveSemanticTableMapping(executor, connection, semantic
|
|
|
13980
14089
|
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
|
|
13981
14090
|
FROM information_schema.tables
|
|
13982
14091
|
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
|
|
13983
|
-
ORDER BY table_schema, table_name
|
|
13984
|
-
LIMIT 2000`, [], {}, connection);
|
|
14092
|
+
ORDER BY table_schema, table_name`, [], {}, connection);
|
|
13985
14093
|
return buildSemanticTableMapping(semanticLayer, tablesResult.rows);
|
|
13986
14094
|
}
|
|
13987
14095
|
catch {
|
|
@@ -14023,41 +14131,17 @@ function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
|
|
|
14023
14131
|
return true;
|
|
14024
14132
|
return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
|
|
14025
14133
|
}
|
|
14026
|
-
function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
14027
|
-
|
|
14028
|
-
|
|
14029
|
-
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
|
|
14033
|
-
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
dbtProjectPath,
|
|
14038
|
-
metrics: request.metrics,
|
|
14039
|
-
dimensions: request.dimensions,
|
|
14040
|
-
filters: request.filters,
|
|
14041
|
-
timeDimension: request.timeDimension,
|
|
14042
|
-
orderBy: request.orderBy,
|
|
14043
|
-
limit: request.limit,
|
|
14044
|
-
savedQuery: request.savedQuery,
|
|
14045
|
-
});
|
|
14046
|
-
return {
|
|
14047
|
-
sql: compiled.sql,
|
|
14048
|
-
joins: [],
|
|
14049
|
-
tables: [],
|
|
14050
|
-
engine: 'metricflow',
|
|
14051
|
-
};
|
|
14052
|
-
}
|
|
14053
|
-
catch (error) {
|
|
14054
|
-
// An explicit MetricFlow request must retain its strict engine contract.
|
|
14055
|
-
// For the default dbt path, a missing local CLI may fall back only if the
|
|
14056
|
-
// native composer can safely materialize the requested simple metrics.
|
|
14057
|
-
if (request.engine === 'metricflow' || !(error instanceof MetricFlowUnavailableError))
|
|
14058
|
-
throw error;
|
|
14059
|
-
}
|
|
14060
|
-
}
|
|
14134
|
+
async function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
14135
|
+
return compileSemanticRuntimeQuery(request, {
|
|
14136
|
+
projectRoot: context.projectRoot,
|
|
14137
|
+
projectConfig: context.projectConfig,
|
|
14138
|
+
detectedProvider: context.detectedProvider,
|
|
14139
|
+
semanticLayer,
|
|
14140
|
+
driver: context.driver,
|
|
14141
|
+
tableMapping: context.tableMapping,
|
|
14142
|
+
});
|
|
14143
|
+
}
|
|
14144
|
+
function composeRuntimeSemanticQueryNative(request, semanticLayer, context) {
|
|
14061
14145
|
const composed = semanticLayer.composeQuery({
|
|
14062
14146
|
metrics: request.metrics,
|
|
14063
14147
|
dimensions: request.dimensions,
|
|
@@ -14126,7 +14210,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14126
14210
|
let composed;
|
|
14127
14211
|
try {
|
|
14128
14212
|
composed = options?.projectRoot && options.projectConfig
|
|
14129
|
-
?
|
|
14213
|
+
? composeRuntimeSemanticQueryNative({
|
|
14130
14214
|
metrics,
|
|
14131
14215
|
dimensions: config.dimensions,
|
|
14132
14216
|
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
@@ -14135,9 +14219,6 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14135
14219
|
: undefined,
|
|
14136
14220
|
limit: runtimeBindings.limit ?? config.limit,
|
|
14137
14221
|
}, semanticLayer, {
|
|
14138
|
-
projectRoot: options.projectRoot,
|
|
14139
|
-
projectConfig: options.projectConfig,
|
|
14140
|
-
detectedProvider: options.detectedProvider ?? null,
|
|
14141
14222
|
driver: options.driver,
|
|
14142
14223
|
tableMapping: options.tableMapping,
|
|
14143
14224
|
})
|
|
@@ -14163,10 +14244,14 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14163
14244
|
}
|
|
14164
14245
|
if (!composed) {
|
|
14165
14246
|
const provider = options?.projectConfig && isDbtSemanticRuntime(options.projectConfig, options.detectedProvider, semanticLayer) ? 'dbt' : (options?.detectedProvider ?? 'dql');
|
|
14166
|
-
const metricFlowReady = provider === 'dbt' && hasMetricFlowCli();
|
|
14167
14247
|
const reasons = metrics.map((metricName) => {
|
|
14168
|
-
|
|
14169
|
-
|
|
14248
|
+
if (semanticLayer.canComposeMetric(metricName))
|
|
14249
|
+
return null;
|
|
14250
|
+
const metric = semanticLayer.getMetric(metricName);
|
|
14251
|
+
const kind = metric?.metricType || metric?.aggregation || metric?.type || 'metric';
|
|
14252
|
+
return provider === 'dbt'
|
|
14253
|
+
? `${metricName}: ${kind} metric requires a configured dbt Cloud or local MetricFlow runtime.`
|
|
14254
|
+
: `${metricName}: the metric does not have enough composable measure and relation metadata.`;
|
|
14170
14255
|
}).filter((reason) => Boolean(reason));
|
|
14171
14256
|
diagnostics.push({
|
|
14172
14257
|
severity: 'error',
|
|
@@ -14183,6 +14268,61 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14183
14268
|
semanticRefs,
|
|
14184
14269
|
};
|
|
14185
14270
|
}
|
|
14271
|
+
/**
|
|
14272
|
+
* Runtime-aware semantic block compiler. Static validation remains synchronous
|
|
14273
|
+
* and native-only, while execution surfaces can use the bundled dbt Cloud or
|
|
14274
|
+
* local MetricFlow adapters without changing the persisted semantic identities.
|
|
14275
|
+
*/
|
|
14276
|
+
async function composeSemanticBlockSqlForRuntime(source, semanticLayer, options) {
|
|
14277
|
+
const native = composeSemanticBlockSql(source, semanticLayer, options);
|
|
14278
|
+
if (native.sql)
|
|
14279
|
+
return native;
|
|
14280
|
+
const config = parseSemanticBlockConfig(source);
|
|
14281
|
+
if (config.blockType !== 'semantic')
|
|
14282
|
+
return native;
|
|
14283
|
+
const metrics = config.metrics.length > 0 ? config.metrics : config.metric ? [config.metric] : [];
|
|
14284
|
+
if (metrics.length === 0 || native.diagnostics.some((diagnostic) => diagnostic.code === 'semantic_ref'))
|
|
14285
|
+
return native;
|
|
14286
|
+
const runtimeBindings = semanticRuntimeParameterBindings(source, options.parameters ?? {});
|
|
14287
|
+
try {
|
|
14288
|
+
const compiled = await composeRuntimeSemanticQuery({
|
|
14289
|
+
metrics,
|
|
14290
|
+
dimensions: config.dimensions,
|
|
14291
|
+
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
14292
|
+
timeDimension: config.timeDimension && config.granularity
|
|
14293
|
+
? { name: config.timeDimension, granularity: config.granularity }
|
|
14294
|
+
: undefined,
|
|
14295
|
+
limit: runtimeBindings.limit ?? config.limit,
|
|
14296
|
+
}, semanticLayer, {
|
|
14297
|
+
projectRoot: options.projectRoot,
|
|
14298
|
+
projectConfig: options.projectConfig,
|
|
14299
|
+
detectedProvider: options.detectedProvider ?? null,
|
|
14300
|
+
driver: options.driver,
|
|
14301
|
+
tableMapping: options.tableMapping,
|
|
14302
|
+
});
|
|
14303
|
+
if (!compiled)
|
|
14304
|
+
return native;
|
|
14305
|
+
return {
|
|
14306
|
+
sql: compiled.sql,
|
|
14307
|
+
diagnostics: native.diagnostics.filter((diagnostic) => diagnostic.code !== 'semantic_compose_failed'),
|
|
14308
|
+
semanticRefs: native.semanticRefs,
|
|
14309
|
+
};
|
|
14310
|
+
}
|
|
14311
|
+
catch (error) {
|
|
14312
|
+
return {
|
|
14313
|
+
sql: null,
|
|
14314
|
+
diagnostics: [
|
|
14315
|
+
...native.diagnostics.filter((diagnostic) => diagnostic.code !== 'semantic_compose_failed'),
|
|
14316
|
+
{
|
|
14317
|
+
severity: 'error',
|
|
14318
|
+
code: isSemanticRuntimeError(error) ? 'semantic_runtime_required' : 'semantic_compose_failed',
|
|
14319
|
+
message: error instanceof Error ? error.message : String(error),
|
|
14320
|
+
},
|
|
14321
|
+
],
|
|
14322
|
+
semanticRefs: native.semanticRefs,
|
|
14323
|
+
};
|
|
14324
|
+
}
|
|
14325
|
+
}
|
|
14186
14326
|
function semanticRuntimeParameterBindings(source, values) {
|
|
14187
14327
|
const program = new Parser(source, '<semantic-parameters>').parse();
|
|
14188
14328
|
const block = program.statements.find((statement) => statement.kind === NodeKind.BlockDecl);
|