@duckcodeailabs/dql-cli 1.8.3 → 1.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/dql-notebook/assets/{index-DGbXmDU6.js → index-gPs-oUu4.js} +346 -345
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/block-invocation.d.ts +2 -2
- package/dist/block-invocation.d.ts.map +1 -1
- package/dist/dbt-cloud-semantic.d.ts +42 -0
- package/dist/dbt-cloud-semantic.d.ts.map +1 -0
- package/dist/dbt-cloud-semantic.js +185 -0
- package/dist/dbt-cloud-semantic.js.map +1 -0
- package/dist/llm/analytics-tools.d.ts.map +1 -1
- package/dist/llm/analytics-tools.js +13 -1
- package/dist/llm/analytics-tools.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +1 -0
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/types.d.ts +3 -1
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +8 -2
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +383 -200
- package/dist/local-runtime.js.map +1 -1
- package/dist/metricflow-installer.d.ts +56 -0
- package/dist/metricflow-installer.d.ts.map +1 -0
- package/dist/metricflow-installer.js +276 -0
- package/dist/metricflow-installer.js.map +1 -0
- package/dist/metricflow.d.ts +19 -1
- package/dist/metricflow.d.ts.map +1 -1
- package/dist/metricflow.js +67 -14
- package/dist/metricflow.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/semantic-runtime-settings.d.ts +55 -0
- package/dist/semantic-runtime-settings.d.ts.map +1 -0
- package/dist/semantic-runtime-settings.js +191 -0
- package/dist/semantic-runtime-settings.js.map +1 -0
- package/dist/semantic-runtime.d.ts +81 -0
- package/dist/semantic-runtime.d.ts.map +1 -0
- package/dist/semantic-runtime.js +240 -0
- package/dist/semantic-runtime.js.map +1 -0
- package/package.json +10 -10
package/dist/local-runtime.js
CHANGED
|
@@ -31,7 +31,10 @@ import { LocalAppStorage, LocalNotebookResearchStorage, defaultLocalAppsDbPath,
|
|
|
31
31
|
import { Certifier, ENTERPRISE_RULES, evaluateInvariants, hasInvariantViolation, } from '@duckcodeailabs/dql-governance';
|
|
32
32
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
33
33
|
import { clearBlockStudioImportSessions, candidateToDqlSource, createBlockStudioImportSession, deleteBlockStudioImportSession, listBlockStudioImportSessions, loadBlockStudioImportSession, readBlockStudioImportCandidate, parameterizeSqlForDqlImport, updateBlockStudioImportCandidate, writeBlockStudioImportSession, writeBlockStudioImportCandidate, } from './block-studio-import.js';
|
|
34
|
-
import { MetricFlowUnavailableError,
|
|
34
|
+
import { MetricFlowUnavailableError, hasDbtSemanticManifest, } from "./metricflow.js";
|
|
35
|
+
import { ManagedMetricFlowInstaller, isMetricFlowWarehouseAdapter, metricFlowAdapterForDriver, } from './metricflow-installer.js';
|
|
36
|
+
import { compileSemanticRuntimeQuery, getSemanticRuntimeStatus, isSemanticRuntimeError, listRuntimeCompatibleDimensions, semanticMetricExecutionCapability as runtimeMetricExecutionCapability, SemanticRuntimeRequiredError, testSemanticRuntimeDraft, } from './semantic-runtime.js';
|
|
37
|
+
import { getSemanticRuntimeSettings, saveTestedSemanticRuntimeSettings, } from './semantic-runtime-settings.js';
|
|
35
38
|
import { NotebookDatasetWorkspace, } from "./notebook-datasets.js";
|
|
36
39
|
import { prepareBlockInvocation } from './block-invocation.js';
|
|
37
40
|
const NOTEBOOK_EXECUTE_PREVIEW_ROW_LIMIT = 500;
|
|
@@ -467,6 +470,7 @@ export async function startLocalServer(opts) {
|
|
|
467
470
|
const dashboardRunEvidence = new Map();
|
|
468
471
|
const dbtNodeDetailCache = new Map();
|
|
469
472
|
const onboardingJobs = new Map();
|
|
473
|
+
const metricFlowInstaller = new ManagedMetricFlowInstaller(projectRoot);
|
|
470
474
|
const projectSnapshot = () => {
|
|
471
475
|
const dbtManifestPath = resolveDbtManifestPath(projectRoot, projectConfig) ?? undefined;
|
|
472
476
|
const inputs = collectInputFiles({ projectRoot, dbtManifestPath });
|
|
@@ -1087,12 +1091,17 @@ export async function startLocalServer(opts) {
|
|
|
1087
1091
|
// dialect-correct SQL (e.g. DATE_TRUNC / identifier quoting). Absent when no
|
|
1088
1092
|
// connection is configured — the compiler then uses its default dialect.
|
|
1089
1093
|
let semanticDriver;
|
|
1094
|
+
let semanticConnection;
|
|
1090
1095
|
try {
|
|
1091
|
-
|
|
1096
|
+
semanticConnection = requireActiveConnection();
|
|
1097
|
+
semanticDriver = semanticConnection.driver;
|
|
1092
1098
|
}
|
|
1093
1099
|
catch {
|
|
1094
1100
|
semanticDriver = undefined;
|
|
1095
1101
|
}
|
|
1102
|
+
const semanticTableMapping = semanticLayer && semanticConnection
|
|
1103
|
+
? await resolveSemanticTableMapping(executor, semanticConnection, semanticLayer)
|
|
1104
|
+
: undefined;
|
|
1096
1105
|
const requestedDomain = agentRunWorkspaceValue(request, 'domain');
|
|
1097
1106
|
const requestedPurpose = agentRunWorkspaceValue(request, 'purpose');
|
|
1098
1107
|
const requestedModelAreaId = agentRunWorkspaceValue(request, 'modelAreaId');
|
|
@@ -1139,6 +1148,26 @@ export async function startLocalServer(opts) {
|
|
|
1139
1148
|
projectSnapshots.assertCurrent(snapshotId);
|
|
1140
1149
|
},
|
|
1141
1150
|
...(semanticDriver ? { semanticDriver } : {}),
|
|
1151
|
+
...(semanticTableMapping ? { semanticTableMapping } : {}),
|
|
1152
|
+
...(semanticLayer ? {
|
|
1153
|
+
semanticQueryCompiler: async (selection) => {
|
|
1154
|
+
const compiled = await compileSemanticRuntimeQuery({
|
|
1155
|
+
...selection,
|
|
1156
|
+
dimensions: selection.dimensions ?? [],
|
|
1157
|
+
}, {
|
|
1158
|
+
projectRoot,
|
|
1159
|
+
projectConfig,
|
|
1160
|
+
detectedProvider: semanticDetectedProvider,
|
|
1161
|
+
semanticLayer: semanticLayer,
|
|
1162
|
+
driver: semanticDriver,
|
|
1163
|
+
tableMapping: semanticTableMapping,
|
|
1164
|
+
});
|
|
1165
|
+
if (!compiled) {
|
|
1166
|
+
throw new SemanticRuntimeRequiredError('The selected semantic members could not be composed. Configure dbt Cloud Semantic Layer or a compatible local MetricFlow runtime for derived metrics.');
|
|
1167
|
+
}
|
|
1168
|
+
return { sql: compiled.sql, engine: compiled.engine };
|
|
1169
|
+
},
|
|
1170
|
+
} : {}),
|
|
1142
1171
|
...(routeDecision?.meaningResolution?.selectedConceptIds.length
|
|
1143
1172
|
? { preferredEvidenceIds: routeDecision.meaningResolution.selectedConceptIds }
|
|
1144
1173
|
: {}),
|
|
@@ -2458,13 +2487,28 @@ export async function startLocalServer(opts) {
|
|
|
2458
2487
|
const activeConnection = requireActiveConnection();
|
|
2459
2488
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
2460
2489
|
const plan = buildExecutionPlan(resolved.cell, { semanticLayer, driver: activeConnection.driver, tableMapping });
|
|
2461
|
-
|
|
2490
|
+
const semanticCompose = resolved.cell.type === 'dql'
|
|
2491
|
+
&& semanticLayer
|
|
2492
|
+
&& /\btype\s*=\s*"semantic"/i.test(resolved.cell.source)
|
|
2493
|
+
? await composeSemanticBlockSqlForRuntime(resolved.cell.source, semanticLayer, {
|
|
2494
|
+
driver: activeConnection.driver,
|
|
2495
|
+
tableMapping,
|
|
2496
|
+
detectedProvider: semanticDetectedProvider,
|
|
2497
|
+
projectRoot,
|
|
2498
|
+
projectConfig,
|
|
2499
|
+
})
|
|
2500
|
+
: null;
|
|
2501
|
+
if (semanticCompose && !semanticCompose.sql) {
|
|
2502
|
+
throw new Error(semanticCompose.diagnostics.map((diagnostic) => diagnostic.message).join(' '));
|
|
2503
|
+
}
|
|
2504
|
+
const executableSql = semanticCompose?.sql ?? plan?.sql;
|
|
2505
|
+
if (!executableSql) {
|
|
2462
2506
|
snapshotCells.push({ cellId, status: 'idle', executionCount: 0, executedAt });
|
|
2463
2507
|
continue;
|
|
2464
2508
|
}
|
|
2465
|
-
const prepared = prepareLocalExecution(
|
|
2509
|
+
const prepared = prepareLocalExecution(executableSql, activeConnection, projectRoot, projectConfig);
|
|
2466
2510
|
assertAppAccess({ app, domain: resolved.domain ?? app.domain, level: 'execute' });
|
|
2467
|
-
const rawResult = await executor.executeQuery(prepared.sql, plan
|
|
2511
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables(plan?.variables ?? {}), prepared.connection);
|
|
2468
2512
|
const result = normalizeQueryResult(rawResult);
|
|
2469
2513
|
snapshotCells.push({
|
|
2470
2514
|
cellId,
|
|
@@ -2511,6 +2555,7 @@ export async function startLocalServer(opts) {
|
|
|
2511
2555
|
const invocation = prepareBlockInvocation({
|
|
2512
2556
|
source,
|
|
2513
2557
|
parameters: invocationInput?.parameters,
|
|
2558
|
+
parameterSources: invocationInput?.parameterSources,
|
|
2514
2559
|
question: invocationInput?.question,
|
|
2515
2560
|
surface: 'ask_ai',
|
|
2516
2561
|
});
|
|
@@ -2523,7 +2568,7 @@ export async function startLocalServer(opts) {
|
|
|
2523
2568
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
2524
2569
|
const plan = buildExecutionPlan({ id: `agent-${metadata.name ?? 'dql-artifact'}`, type: 'dql', source, title: metadata.name ?? 'DQL artifact' }, { semanticLayer, driver: activeConnection.driver, tableMapping, parameters: invocation.values });
|
|
2525
2570
|
const semanticCompose = semanticLayer
|
|
2526
|
-
?
|
|
2571
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
2527
2572
|
driver: activeConnection.driver,
|
|
2528
2573
|
tableMapping,
|
|
2529
2574
|
projectRoot,
|
|
@@ -2607,7 +2652,8 @@ export async function startLocalServer(opts) {
|
|
|
2607
2652
|
* candidate remains review-required and is not executed automatically.
|
|
2608
2653
|
*/
|
|
2609
2654
|
const executeExploratoryCandidate = async (candidate, schemaContext = [], question = '', requestedTopN) => {
|
|
2610
|
-
const
|
|
2655
|
+
const activeConnection = requireActiveConnection();
|
|
2656
|
+
const preflight = repairExploratorySqlBeforeExecution(candidate.sql, schemaContext, question, activeConnection.driver);
|
|
2611
2657
|
const boundedSql = applyRequestedTopNToExploratorySql(preflight.sql, requestedTopN);
|
|
2612
2658
|
const repairs = boundedSql === preflight.sql
|
|
2613
2659
|
? [...preflight.repairs]
|
|
@@ -2620,7 +2666,7 @@ export async function startLocalServer(opts) {
|
|
|
2620
2666
|
error: preflight.blockedReason,
|
|
2621
2667
|
};
|
|
2622
2668
|
}
|
|
2623
|
-
const analysis = analyzeSqlReferences(boundedSql);
|
|
2669
|
+
const analysis = analyzeSqlReferences(boundedSql, activeConnection.driver);
|
|
2624
2670
|
if (!analysis.parsed) {
|
|
2625
2671
|
return { proofs: [], sql: boundedSql, repairs, error: 'DQL could not parse the exploratory SQL to validate its join predicates.' };
|
|
2626
2672
|
}
|
|
@@ -2676,7 +2722,6 @@ export async function startLocalServer(opts) {
|
|
|
2676
2722
|
if (probeableJoins.length < analysis.joins.length) {
|
|
2677
2723
|
repairs.push(`Skipped join probes for ${analysis.joins.length - probeableJoins.length} join(s) on derived (CTE) relations; the final query execution validates them.`);
|
|
2678
2724
|
}
|
|
2679
|
-
const activeConnection = requireActiveConnection();
|
|
2680
2725
|
const proofs = [];
|
|
2681
2726
|
try {
|
|
2682
2727
|
for (const [index, join] of probeableJoins.entries()) {
|
|
@@ -3545,7 +3590,7 @@ export async function startLocalServer(opts) {
|
|
|
3545
3590
|
throw new Error(`Provide required parameter${invocation.unresolvedParameters.length === 1 ? '' : 's'}: ${invocation.unresolvedParameters.join(', ')}.`);
|
|
3546
3591
|
}
|
|
3547
3592
|
const semanticCompose = semanticLayer
|
|
3548
|
-
?
|
|
3593
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
3549
3594
|
driver: activeConnection.driver,
|
|
3550
3595
|
tableMapping,
|
|
3551
3596
|
projectRoot,
|
|
@@ -3599,7 +3644,7 @@ export async function startLocalServer(opts) {
|
|
|
3599
3644
|
// pre-compiled query, that's the query (not a recompiled metric), so the test's
|
|
3600
3645
|
// output columns match the block's declared outputs.
|
|
3601
3646
|
const semanticCompose = semanticLayer
|
|
3602
|
-
?
|
|
3647
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
3603
3648
|
driver: activeConnection.driver,
|
|
3604
3649
|
tableMapping,
|
|
3605
3650
|
projectRoot,
|
|
@@ -6017,7 +6062,7 @@ export async function startLocalServer(opts) {
|
|
|
6017
6062
|
operator: filter.operator,
|
|
6018
6063
|
values: Array.isArray(filter.value) ? filter.value.map(String) : [String(filter.value)],
|
|
6019
6064
|
}));
|
|
6020
|
-
const composed = composeRuntimeSemanticQuery({
|
|
6065
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
6021
6066
|
metrics: item.semantic.metrics,
|
|
6022
6067
|
dimensions: item.semantic.dimensions ?? [],
|
|
6023
6068
|
filters: [...staticFilters, ...activeFilters],
|
|
@@ -6108,7 +6153,7 @@ export async function startLocalServer(opts) {
|
|
|
6108
6153
|
continue;
|
|
6109
6154
|
}
|
|
6110
6155
|
const semanticCompose = semanticLayer
|
|
6111
|
-
?
|
|
6156
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
6112
6157
|
driver: targetConnection.driver,
|
|
6113
6158
|
tableMapping,
|
|
6114
6159
|
projectRoot,
|
|
@@ -7487,7 +7532,7 @@ export async function startLocalServer(opts) {
|
|
|
7487
7532
|
const activeConnection = requireActiveConnection();
|
|
7488
7533
|
const tableMapping = await resolveSemanticTableMapping(executor, activeConnection, semanticLayer);
|
|
7489
7534
|
const semanticCompose = semanticLayer
|
|
7490
|
-
?
|
|
7535
|
+
? await composeSemanticBlockSqlForRuntime(source, semanticLayer, {
|
|
7491
7536
|
driver: activeConnection.driver,
|
|
7492
7537
|
tableMapping,
|
|
7493
7538
|
projectRoot,
|
|
@@ -7993,8 +8038,23 @@ export async function startLocalServer(opts) {
|
|
|
7993
8038
|
draftSave: readiness.candidate.draftSave ?? { status: 'pending' },
|
|
7994
8039
|
};
|
|
7995
8040
|
writeBlockStudioImportCandidate(projectRoot, importId, next);
|
|
7996
|
-
|
|
7997
|
-
|
|
8041
|
+
let compiledManifest;
|
|
8042
|
+
let lineageRefresh;
|
|
8043
|
+
try {
|
|
8044
|
+
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
8045
|
+
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
8046
|
+
}
|
|
8047
|
+
catch (error) {
|
|
8048
|
+
lineageRefresh = {
|
|
8049
|
+
status: 'failed',
|
|
8050
|
+
message: error instanceof Error ? error.message : String(error),
|
|
8051
|
+
};
|
|
8052
|
+
}
|
|
8053
|
+
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
8054
|
+
const payload = {
|
|
8055
|
+
...openBlockStudioDocument(projectRoot, savedPath, semanticLayer),
|
|
8056
|
+
lineageRefresh,
|
|
8057
|
+
};
|
|
7998
8058
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
7999
8059
|
res.end(serializeJSON({ candidate: next, block: payload, certification }));
|
|
8000
8060
|
return;
|
|
@@ -8107,10 +8167,23 @@ export async function startLocalServer(opts) {
|
|
|
8107
8167
|
}
|
|
8108
8168
|
const nextSession = { ...session, candidates: nextCandidates, updatedAt: new Date().toISOString() };
|
|
8109
8169
|
writeBlockStudioImportSession(projectRoot, nextSession);
|
|
8110
|
-
|
|
8111
|
-
|
|
8170
|
+
let lineageRefresh;
|
|
8171
|
+
if (saved.length > 0) {
|
|
8172
|
+
let compiledManifest;
|
|
8173
|
+
try {
|
|
8174
|
+
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
8175
|
+
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
8176
|
+
}
|
|
8177
|
+
catch (error) {
|
|
8178
|
+
lineageRefresh = {
|
|
8179
|
+
status: 'failed',
|
|
8180
|
+
message: error instanceof Error ? error.message : String(error),
|
|
8181
|
+
};
|
|
8182
|
+
}
|
|
8183
|
+
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
8184
|
+
}
|
|
8112
8185
|
res.writeHead(errors.length > 0 ? 207 : 200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8113
|
-
res.end(serializeJSON({ ok: errors.length === 0, session: nextSession, saved, errors }));
|
|
8186
|
+
res.end(serializeJSON({ ok: errors.length === 0, session: nextSession, saved, errors, lineageRefresh }));
|
|
8114
8187
|
}
|
|
8115
8188
|
catch (error) {
|
|
8116
8189
|
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -8254,9 +8327,14 @@ export async function startLocalServer(opts) {
|
|
|
8254
8327
|
const connections = getProjectConnectionsForApi(cfg);
|
|
8255
8328
|
const defaultKey = resolveDefaultConnectionKey(cfg, connections) ?? Object.keys(connections)[0] ?? 'default';
|
|
8256
8329
|
const userPrefs = readUserPrefs(userPrefsPath);
|
|
8330
|
+
// UI-009 / PERF-001: Block Studio already loads the canonical semantic
|
|
8331
|
+
// layer. Let it omit this second, potentially multi-megabyte rendering
|
|
8332
|
+
// of the same 7,500+ object catalog while keeping the route compatible
|
|
8333
|
+
// for older clients.
|
|
8334
|
+
const includeSemantic = url.searchParams.get('includeSemantic') !== 'false';
|
|
8257
8335
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8258
8336
|
res.end(serializeJSON({
|
|
8259
|
-
semanticTree: semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
8337
|
+
semanticTree: includeSemantic && semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
8260
8338
|
databaseTree: await buildDatabaseSchemaTree(projectRoot, executor, connection),
|
|
8261
8339
|
connection: {
|
|
8262
8340
|
default: defaultKey,
|
|
@@ -8417,8 +8495,26 @@ export async function startLocalServer(opts) {
|
|
|
8417
8495
|
stableSuffix: metadata.candidateId,
|
|
8418
8496
|
})
|
|
8419
8497
|
: saveBlockStudioArtifacts(projectRoot, saveOptions);
|
|
8420
|
-
|
|
8421
|
-
|
|
8498
|
+
let compiledManifest;
|
|
8499
|
+
let lineageRefresh;
|
|
8500
|
+
try {
|
|
8501
|
+
compiledManifest = compileBlockStudioManifest(projectRoot, projectConfig);
|
|
8502
|
+
lineageRefresh = { status: 'ready', compiledAt: new Date().toISOString() };
|
|
8503
|
+
}
|
|
8504
|
+
catch (error) {
|
|
8505
|
+
// The block is already safely stored. Preserve it and report the
|
|
8506
|
+
// rebuild failure separately instead of turning a successful save
|
|
8507
|
+
// into a misleading 500 response.
|
|
8508
|
+
lineageRefresh = {
|
|
8509
|
+
status: 'failed',
|
|
8510
|
+
message: error instanceof Error ? error.message : String(error),
|
|
8511
|
+
};
|
|
8512
|
+
}
|
|
8513
|
+
await refreshLocalMetadataCatalog(projectRoot, compiledManifest, semanticLayer);
|
|
8514
|
+
const payload = {
|
|
8515
|
+
...openBlockStudioDocument(projectRoot, savedPath, semanticLayer),
|
|
8516
|
+
lineageRefresh,
|
|
8517
|
+
};
|
|
8422
8518
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8423
8519
|
res.end(serializeJSON(payload));
|
|
8424
8520
|
}
|
|
@@ -8817,6 +8913,87 @@ export async function startLocalServer(opts) {
|
|
|
8817
8913
|
}
|
|
8818
8914
|
return;
|
|
8819
8915
|
}
|
|
8916
|
+
// ── Semantic runtime adapters (API-004 / UI-009 / E2E-008) ───────────────
|
|
8917
|
+
if (req.method === 'GET' && path === '/api/semantic-runtime') {
|
|
8918
|
+
try {
|
|
8919
|
+
const runtime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
8920
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8921
|
+
res.end(serializeJSON({ ...getSemanticRuntimeSettings(projectRoot), runtime }));
|
|
8922
|
+
}
|
|
8923
|
+
catch (error) {
|
|
8924
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8925
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
8926
|
+
}
|
|
8927
|
+
return;
|
|
8928
|
+
}
|
|
8929
|
+
if (req.method === 'GET' && path === '/api/semantic-runtime/metricflow/installer') {
|
|
8930
|
+
try {
|
|
8931
|
+
const profileAdapter = discoverDbtProfileConnections(projectRoot, projectConfig)[0]?.adapter;
|
|
8932
|
+
const recommendedAdapter = metricFlowAdapterForDriver(connection?.driver ?? profileAdapter);
|
|
8933
|
+
const dbtProjectDir = findDbtProjectPath(projectRoot, projectConfig);
|
|
8934
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8935
|
+
res.end(serializeJSON({
|
|
8936
|
+
job: metricFlowInstaller.latest(),
|
|
8937
|
+
recommendedAdapter,
|
|
8938
|
+
supportedAdapters: ['duckdb', 'snowflake', 'bigquery', 'databricks', 'redshift', 'postgres', 'trino'],
|
|
8939
|
+
projectConfigured: existsSync(join(dbtProjectDir, 'dbt_project.yml')),
|
|
8940
|
+
semanticManifestFound: existsSync(join(dbtProjectDir, 'target', 'semantic_manifest.json')),
|
|
8941
|
+
}));
|
|
8942
|
+
}
|
|
8943
|
+
catch (error) {
|
|
8944
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8945
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
8946
|
+
}
|
|
8947
|
+
return;
|
|
8948
|
+
}
|
|
8949
|
+
if (req.method === 'POST' && path === '/api/semantic-runtime/metricflow/install') {
|
|
8950
|
+
try {
|
|
8951
|
+
const body = await readJSON(req);
|
|
8952
|
+
const profileAdapter = discoverDbtProfileConnections(projectRoot, projectConfig)[0]?.adapter;
|
|
8953
|
+
const inferred = metricFlowAdapterForDriver(connection?.driver ?? profileAdapter);
|
|
8954
|
+
const adapter = body.adapter === undefined ? inferred : body.adapter;
|
|
8955
|
+
if (!isMetricFlowWarehouseAdapter(adapter)) {
|
|
8956
|
+
throw new Error('Choose a supported warehouse adapter before installing MetricFlow.');
|
|
8957
|
+
}
|
|
8958
|
+
const { dbtProjectDir, profilesDir } = onboardingDbtPaths({});
|
|
8959
|
+
const job = metricFlowInstaller.start({ adapter, dbtProjectDir, profilesDir });
|
|
8960
|
+
res.writeHead(202, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8961
|
+
res.end(serializeJSON({ ok: true, job }));
|
|
8962
|
+
}
|
|
8963
|
+
catch (error) {
|
|
8964
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8965
|
+
res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
|
|
8966
|
+
}
|
|
8967
|
+
return;
|
|
8968
|
+
}
|
|
8969
|
+
if (req.method === 'POST' && path === '/api/semantic-runtime/dbt-cloud/test') {
|
|
8970
|
+
try {
|
|
8971
|
+
const body = await readJSON(req);
|
|
8972
|
+
const result = await testSemanticRuntimeDraft(projectRoot, body);
|
|
8973
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8974
|
+
res.end(serializeJSON(result));
|
|
8975
|
+
}
|
|
8976
|
+
catch (error) {
|
|
8977
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8978
|
+
res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
|
|
8979
|
+
}
|
|
8980
|
+
return;
|
|
8981
|
+
}
|
|
8982
|
+
if (req.method === 'POST' && path === '/api/semantic-runtime/dbt-cloud/apply') {
|
|
8983
|
+
try {
|
|
8984
|
+
const body = await readJSON(req);
|
|
8985
|
+
const result = await testSemanticRuntimeDraft(projectRoot, body);
|
|
8986
|
+
const settings = saveTestedSemanticRuntimeSettings(projectRoot, body, result);
|
|
8987
|
+
const runtime = await getSemanticRuntimeStatus(projectRoot);
|
|
8988
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8989
|
+
res.end(serializeJSON({ ok: true, ...settings, runtime }));
|
|
8990
|
+
}
|
|
8991
|
+
catch (error) {
|
|
8992
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8993
|
+
res.end(serializeJSON({ ok: false, error: error instanceof Error ? error.message : String(error) }));
|
|
8994
|
+
}
|
|
8995
|
+
return;
|
|
8996
|
+
}
|
|
8820
8997
|
// ── Semantic layer discovery API ─────────────────────────────────────────
|
|
8821
8998
|
if (req.method === 'GET' && path === '/api/semantic-layer') {
|
|
8822
8999
|
const userPrefs = readUserPrefs(userPrefsPath);
|
|
@@ -8846,8 +9023,7 @@ export async function startLocalServer(opts) {
|
|
|
8846
9023
|
const dbtManifestReady = provider === 'dbt'
|
|
8847
9024
|
? hasDbtSemanticManifest(projectRoot, semanticConfig?.projectPath)
|
|
8848
9025
|
: false;
|
|
8849
|
-
const
|
|
8850
|
-
const dbtExecutionReady = dbtManifestReady && metricFlowReady;
|
|
9026
|
+
const semanticRuntime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
8851
9027
|
const metrics = semanticLayer.listMetrics().map((m) => ({
|
|
8852
9028
|
name: m.name,
|
|
8853
9029
|
label: m.label,
|
|
@@ -8862,7 +9038,7 @@ export async function startLocalServer(opts) {
|
|
|
8862
9038
|
typeParams: m.typeParams ?? null,
|
|
8863
9039
|
filter: m.filter ?? null,
|
|
8864
9040
|
source: m.source ?? null,
|
|
8865
|
-
execution:
|
|
9041
|
+
execution: runtimeMetricExecutionCapability(m.name, semanticLayer, provider, semanticRuntime),
|
|
8866
9042
|
}));
|
|
8867
9043
|
const measures = semanticLayer.listMeasures().map((m) => ({
|
|
8868
9044
|
name: m.name,
|
|
@@ -8960,24 +9136,21 @@ export async function startLocalServer(opts) {
|
|
|
8960
9136
|
owner: q.owner ?? null,
|
|
8961
9137
|
source: q.source ?? null,
|
|
8962
9138
|
}));
|
|
8963
|
-
const dbtExecutionSetup =
|
|
8964
|
-
?
|
|
8965
|
-
:
|
|
8966
|
-
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.'
|
|
8967
|
-
: !dbtManifestReady
|
|
8968
|
-
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists.'
|
|
8969
|
-
: 'Install MetricFlow so `mf` is on PATH, or set DQL_METRICFLOW_BIN to the MetricFlow executable.';
|
|
9139
|
+
const dbtExecutionSetup = !dbtManifestReady
|
|
9140
|
+
? 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists.'
|
|
9141
|
+
: semanticRuntime.setup;
|
|
8970
9142
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
8971
9143
|
res.end(serializeJSON({
|
|
8972
9144
|
available: true,
|
|
8973
9145
|
provider,
|
|
8974
9146
|
execution: provider === 'dbt'
|
|
8975
9147
|
? {
|
|
8976
|
-
engine:
|
|
8977
|
-
ready:
|
|
9148
|
+
engine: semanticRuntime.active,
|
|
9149
|
+
ready: semanticRuntime.active !== 'native' || metrics.every((metric) => metric.execution.status === 'ready'),
|
|
8978
9150
|
setup: dbtExecutionSetup,
|
|
9151
|
+
adapters: semanticRuntime.adapters,
|
|
8979
9152
|
}
|
|
8980
|
-
: { engine: 'native', ready: true, setup: null },
|
|
9153
|
+
: { engine: 'native', ready: true, setup: null, adapters: semanticRuntime.adapters },
|
|
8981
9154
|
errors: semanticLayerErrors,
|
|
8982
9155
|
metrics,
|
|
8983
9156
|
measures,
|
|
@@ -9314,7 +9487,7 @@ export async function startLocalServer(opts) {
|
|
|
9314
9487
|
.split(',')
|
|
9315
9488
|
.map((value) => value.trim())
|
|
9316
9489
|
.filter(Boolean);
|
|
9317
|
-
const dimensions = semanticLayer
|
|
9490
|
+
const dimensions = (await listRuntimeCompatibleDimensions(projectRoot, semanticLayer, metrics)).map((d) => ({
|
|
9318
9491
|
name: d.name,
|
|
9319
9492
|
label: d.label,
|
|
9320
9493
|
description: d.description,
|
|
@@ -9827,37 +10000,8 @@ export async function startLocalServer(opts) {
|
|
|
9827
10000
|
// Resolve which connection to use — request can override default
|
|
9828
10001
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
9829
10002
|
const driver = targetConnection.driver;
|
|
9830
|
-
|
|
9831
|
-
|
|
9832
|
-
try {
|
|
9833
|
-
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
|
|
9834
|
-
const dbTableNames = new Set();
|
|
9835
|
-
const schemaQualified = new Map();
|
|
9836
|
-
for (const row of tablesResult.rows) {
|
|
9837
|
-
const schema = String(row['table_schema'] ?? '');
|
|
9838
|
-
const name = String(row['table_name'] ?? '');
|
|
9839
|
-
dbTableNames.add(name);
|
|
9840
|
-
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
9841
|
-
}
|
|
9842
|
-
// For each table in the semantic layer, map to qualified name if it exists
|
|
9843
|
-
const allSemanticTables = new Set();
|
|
9844
|
-
for (const m of semanticLayer.listMetrics())
|
|
9845
|
-
allSemanticTables.add(m.table);
|
|
9846
|
-
for (const d of semanticLayer.listDimensions())
|
|
9847
|
-
allSemanticTables.add(d.table);
|
|
9848
|
-
tableMapping = {};
|
|
9849
|
-
for (const semTable of allSemanticTables) {
|
|
9850
|
-
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
9851
|
-
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
9852
|
-
}
|
|
9853
|
-
}
|
|
9854
|
-
if (Object.keys(tableMapping).length === 0)
|
|
9855
|
-
tableMapping = undefined;
|
|
9856
|
-
}
|
|
9857
|
-
catch {
|
|
9858
|
-
// Non-fatal: proceed without table mapping
|
|
9859
|
-
}
|
|
9860
|
-
const composed = composeRuntimeSemanticQuery({
|
|
10003
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
10004
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
9861
10005
|
metrics,
|
|
9862
10006
|
dimensions,
|
|
9863
10007
|
filters,
|
|
@@ -9875,10 +10019,10 @@ export async function startLocalServer(opts) {
|
|
|
9875
10019
|
});
|
|
9876
10020
|
if (!composed) {
|
|
9877
10021
|
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
9878
|
-
const
|
|
10022
|
+
const semanticRuntime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
9879
10023
|
const blocked = metrics.map((metricName) => ({
|
|
9880
10024
|
metric: metricName,
|
|
9881
|
-
...
|
|
10025
|
+
...runtimeMetricExecutionCapability(metricName, semanticLayer, provider, semanticRuntime),
|
|
9882
10026
|
})).filter((capability) => capability.status !== 'ready');
|
|
9883
10027
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9884
10028
|
res.end(serializeJSON({
|
|
@@ -9908,13 +10052,14 @@ export async function startLocalServer(opts) {
|
|
|
9908
10052
|
res.end(serializeJSON({ error: error.message, code: 'unauthorized' }));
|
|
9909
10053
|
return;
|
|
9910
10054
|
}
|
|
9911
|
-
const
|
|
10055
|
+
const runtimeError = isSemanticRuntimeError(error);
|
|
10056
|
+
const status = runtimeError ? 400 : 500;
|
|
9912
10057
|
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9913
10058
|
res.end(serializeJSON({
|
|
9914
10059
|
error: error instanceof Error ? error.message : String(error),
|
|
9915
|
-
code:
|
|
9916
|
-
hint:
|
|
9917
|
-
? '
|
|
10060
|
+
code: runtimeError ? 'SEMANTIC_RUNTIME_REQUIRED' : undefined,
|
|
10061
|
+
hint: runtimeError
|
|
10062
|
+
? 'Configure dbt Cloud Semantic Layer in Project & dbt settings, or install a compatible local MetricFlow runtime.'
|
|
9918
10063
|
: undefined,
|
|
9919
10064
|
}));
|
|
9920
10065
|
}
|
|
@@ -9931,31 +10076,8 @@ export async function startLocalServer(opts) {
|
|
|
9931
10076
|
const { metrics = [], dimensions = [], filters = [], limit, timeDimension, orderBy, savedQuery, engine } = body;
|
|
9932
10077
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
9933
10078
|
const driver = targetConnection.driver;
|
|
9934
|
-
|
|
9935
|
-
|
|
9936
|
-
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')`, [], {}, targetConnection);
|
|
9937
|
-
const schemaQualified = new Map();
|
|
9938
|
-
for (const row of tablesResult.rows) {
|
|
9939
|
-
const schema = String(row['table_schema'] ?? '');
|
|
9940
|
-
const name = String(row['table_name'] ?? '');
|
|
9941
|
-
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
9942
|
-
}
|
|
9943
|
-
tableMapping = {};
|
|
9944
|
-
for (const metric of semanticLayer.listMetrics()) {
|
|
9945
|
-
if (schemaQualified.has(metric.table))
|
|
9946
|
-
tableMapping[metric.table] = schemaQualified.get(metric.table);
|
|
9947
|
-
}
|
|
9948
|
-
for (const dimension of semanticLayer.listDimensions()) {
|
|
9949
|
-
if (schemaQualified.has(dimension.table))
|
|
9950
|
-
tableMapping[dimension.table] = schemaQualified.get(dimension.table);
|
|
9951
|
-
}
|
|
9952
|
-
if (Object.keys(tableMapping).length === 0)
|
|
9953
|
-
tableMapping = undefined;
|
|
9954
|
-
}
|
|
9955
|
-
catch {
|
|
9956
|
-
tableMapping = undefined;
|
|
9957
|
-
}
|
|
9958
|
-
const composed = composeRuntimeSemanticQuery({
|
|
10079
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
10080
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
9959
10081
|
metrics,
|
|
9960
10082
|
dimensions,
|
|
9961
10083
|
filters,
|
|
@@ -9973,10 +10095,10 @@ export async function startLocalServer(opts) {
|
|
|
9973
10095
|
});
|
|
9974
10096
|
if (!composed) {
|
|
9975
10097
|
const provider = semanticConfig?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
9976
|
-
const
|
|
10098
|
+
const semanticRuntime = await getSemanticRuntimeStatus(projectRoot, { probeConfiguredCloud: true });
|
|
9977
10099
|
const blocked = metrics.map((metricName) => ({
|
|
9978
10100
|
metric: metricName,
|
|
9979
|
-
...
|
|
10101
|
+
...runtimeMetricExecutionCapability(metricName, semanticLayer, provider, semanticRuntime),
|
|
9980
10102
|
})).filter((capability) => capability.status !== 'ready');
|
|
9981
10103
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
9982
10104
|
res.end(serializeJSON({
|
|
@@ -10000,13 +10122,14 @@ export async function startLocalServer(opts) {
|
|
|
10000
10122
|
}));
|
|
10001
10123
|
}
|
|
10002
10124
|
catch (error) {
|
|
10003
|
-
const
|
|
10125
|
+
const runtimeError = isSemanticRuntimeError(error);
|
|
10126
|
+
const status = runtimeError ? 400 : 500;
|
|
10004
10127
|
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10005
10128
|
res.end(serializeJSON({
|
|
10006
10129
|
error: error instanceof Error ? error.message : String(error),
|
|
10007
|
-
code:
|
|
10008
|
-
hint:
|
|
10009
|
-
? '
|
|
10130
|
+
code: runtimeError ? 'SEMANTIC_RUNTIME_REQUIRED' : undefined,
|
|
10131
|
+
hint: runtimeError
|
|
10132
|
+
? 'Configure dbt Cloud Semantic Layer in Project & dbt settings, or install a compatible local MetricFlow runtime.'
|
|
10010
10133
|
: undefined,
|
|
10011
10134
|
}));
|
|
10012
10135
|
}
|
|
@@ -10031,7 +10154,8 @@ export async function startLocalServer(opts) {
|
|
|
10031
10154
|
return;
|
|
10032
10155
|
}
|
|
10033
10156
|
const targetConnection = requireActiveConnection(isConnectionConfig(body.connection) ? body.connection : connection);
|
|
10034
|
-
const
|
|
10157
|
+
const tableMapping = await resolveSemanticTableMapping(executor, targetConnection, semanticLayer);
|
|
10158
|
+
const composed = await composeRuntimeSemanticQuery({
|
|
10035
10159
|
metrics,
|
|
10036
10160
|
dimensions,
|
|
10037
10161
|
filters,
|
|
@@ -10042,6 +10166,7 @@ export async function startLocalServer(opts) {
|
|
|
10042
10166
|
projectConfig,
|
|
10043
10167
|
detectedProvider: semanticDetectedProvider,
|
|
10044
10168
|
driver: targetConnection.driver,
|
|
10169
|
+
tableMapping,
|
|
10045
10170
|
});
|
|
10046
10171
|
if (!composed) {
|
|
10047
10172
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -10072,13 +10197,14 @@ export async function startLocalServer(opts) {
|
|
|
10072
10197
|
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
10073
10198
|
return;
|
|
10074
10199
|
}
|
|
10075
|
-
const
|
|
10200
|
+
const runtimeError = isSemanticRuntimeError(error);
|
|
10201
|
+
const status = runtimeError ? 400 : 500;
|
|
10076
10202
|
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10077
10203
|
res.end(serializeJSON({
|
|
10078
10204
|
error: error instanceof Error ? error.message : String(error),
|
|
10079
|
-
code:
|
|
10080
|
-
hint:
|
|
10081
|
-
? '
|
|
10205
|
+
code: runtimeError ? 'SEMANTIC_RUNTIME_REQUIRED' : undefined,
|
|
10206
|
+
hint: runtimeError
|
|
10207
|
+
? 'Configure dbt Cloud Semantic Layer in Project & dbt settings, or install a compatible local MetricFlow runtime.'
|
|
10082
10208
|
: undefined,
|
|
10083
10209
|
}));
|
|
10084
10210
|
}
|
|
@@ -10400,7 +10526,12 @@ export async function startLocalServer(opts) {
|
|
|
10400
10526
|
parameters,
|
|
10401
10527
|
question: typeof body.question === 'string' ? body.question : undefined,
|
|
10402
10528
|
});
|
|
10403
|
-
const contract = prepareBlockInvocation({
|
|
10529
|
+
const contract = prepareBlockInvocation({
|
|
10530
|
+
source,
|
|
10531
|
+
parameters,
|
|
10532
|
+
question: typeof body.question === 'string' ? body.question : undefined,
|
|
10533
|
+
surface: 'ask_ai',
|
|
10534
|
+
});
|
|
10404
10535
|
const certified = /\bstatus\s*=\s*"certified"/i.test(source);
|
|
10405
10536
|
const semantic = /\btype\s*=\s*"semantic"/i.test(source);
|
|
10406
10537
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -10496,6 +10627,7 @@ export async function startLocalServer(opts) {
|
|
|
10496
10627
|
parameters: body.parameters && typeof body.parameters === 'object' && !Array.isArray(body.parameters)
|
|
10497
10628
|
? body.parameters
|
|
10498
10629
|
: {},
|
|
10630
|
+
question: typeof body.question === 'string' ? body.question : undefined,
|
|
10499
10631
|
surface: 'notebook',
|
|
10500
10632
|
})
|
|
10501
10633
|
: null;
|
|
@@ -10514,15 +10646,31 @@ export async function startLocalServer(opts) {
|
|
|
10514
10646
|
tableMapping,
|
|
10515
10647
|
parameters: invocation?.values,
|
|
10516
10648
|
});
|
|
10517
|
-
|
|
10649
|
+
const semanticCompose = executableCell.type === 'dql'
|
|
10650
|
+
&& semanticLayer
|
|
10651
|
+
&& /\btype\s*=\s*"semantic"/i.test(executableCell.source)
|
|
10652
|
+
? await composeSemanticBlockSqlForRuntime(executableCell.source, semanticLayer, {
|
|
10653
|
+
driver: cellConnection.driver,
|
|
10654
|
+
tableMapping,
|
|
10655
|
+
parameters: invocation?.values,
|
|
10656
|
+
detectedProvider: semanticDetectedProvider,
|
|
10657
|
+
projectRoot,
|
|
10658
|
+
projectConfig,
|
|
10659
|
+
})
|
|
10660
|
+
: null;
|
|
10661
|
+
if (semanticCompose && !semanticCompose.sql) {
|
|
10662
|
+
throw new Error(semanticCompose.diagnostics.map((diagnostic) => diagnostic.message).join(' '));
|
|
10663
|
+
}
|
|
10664
|
+
const executableSql = semanticCompose?.sql ?? plan?.sql;
|
|
10665
|
+
if (!executableSql) {
|
|
10518
10666
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10519
10667
|
res.end(serializeJSON({ cellType: cell.type, result: null }));
|
|
10520
10668
|
return;
|
|
10521
10669
|
}
|
|
10522
|
-
const prepared = prepareLocalExecution(
|
|
10670
|
+
const prepared = prepareLocalExecution(executableSql, cellConnection, projectRoot, projectConfig);
|
|
10523
10671
|
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
10524
10672
|
assertAppAccess({ app, domain: resolved.domain ?? app?.domain, level: 'execute' });
|
|
10525
|
-
const rawResult = await executor.executeQuery(prepared.sql, plan
|
|
10673
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...(invocation?.values ?? {}) }), prepared.connection);
|
|
10526
10674
|
const normalized = normalizeQueryResult(rawResult);
|
|
10527
10675
|
// Enforce the block's declared invariants against the result set. This
|
|
10528
10676
|
// is additive: blocks without invariants produce `null` and the
|
|
@@ -10536,29 +10684,29 @@ export async function startLocalServer(opts) {
|
|
|
10536
10684
|
recordNotebookQueryRun(projectRoot, {
|
|
10537
10685
|
notebookPath: execContext.notebookPath,
|
|
10538
10686
|
cellId: execContext.cellId ?? cell.id,
|
|
10539
|
-
cellName: execContext.cellName ?? plan
|
|
10687
|
+
cellName: execContext.cellName ?? plan?.title ?? resolved.blockName,
|
|
10540
10688
|
researchRunId: execContext.researchRunId,
|
|
10541
10689
|
source: execContext.source ?? (cell.type === 'dql' ? 'notebook_dql_cell' : 'notebook_cell'),
|
|
10542
10690
|
status: 'success',
|
|
10543
10691
|
rowCount: normalized.rowCount ?? normalized.rows.length,
|
|
10544
10692
|
durationMs: Date.now() - start,
|
|
10545
|
-
sql:
|
|
10693
|
+
sql: executableSql,
|
|
10546
10694
|
objectKey: resolved.blockPath,
|
|
10547
10695
|
});
|
|
10548
10696
|
updateNotebookResearchFromCellExecution(projectRoot, execContext, {
|
|
10549
10697
|
status: 'success',
|
|
10550
10698
|
resultPreview: normalized,
|
|
10551
|
-
sql:
|
|
10699
|
+
sql: executableSql,
|
|
10552
10700
|
});
|
|
10553
10701
|
}
|
|
10554
10702
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
10555
10703
|
res.end(serializeJSON({
|
|
10556
10704
|
cellType: cell.type,
|
|
10557
|
-
title: plan
|
|
10705
|
+
title: plan?.title,
|
|
10558
10706
|
blockName: resolved.blockName,
|
|
10559
10707
|
blockPath: resolved.blockPath,
|
|
10560
|
-
chartConfig: plan
|
|
10561
|
-
tests: plan
|
|
10708
|
+
chartConfig: plan?.chartConfig,
|
|
10709
|
+
tests: plan?.tests,
|
|
10562
10710
|
result: normalized,
|
|
10563
10711
|
...(invocation ? {
|
|
10564
10712
|
invocation: {
|
|
@@ -11491,9 +11639,34 @@ function latestOpenContextBootstrapSession(projectRoot) {
|
|
|
11491
11639
|
}
|
|
11492
11640
|
return null;
|
|
11493
11641
|
}
|
|
11494
|
-
|
|
11642
|
+
/**
|
|
11643
|
+
* API-004 / E2E-006: save-time compile for Block Studio. The manifest is
|
|
11644
|
+
* replaced atomically so lineage readers and agent retrieval never observe a
|
|
11645
|
+
* half-written snapshot, and an existing manifest survives compilation errors.
|
|
11646
|
+
*/
|
|
11647
|
+
export function compileBlockStudioManifest(projectRoot, projectConfig = loadProjectConfig(projectRoot)) {
|
|
11648
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot, projectConfig) ?? undefined;
|
|
11649
|
+
const manifest = buildManifest({ projectRoot, dqlVersion: 'notebook', dbtManifestPath });
|
|
11650
|
+
const manifestPath = join(projectRoot, 'dql-manifest.json');
|
|
11651
|
+
const tempPath = `${manifestPath}.tmp-${process.pid}-${Date.now()}`;
|
|
11495
11652
|
try {
|
|
11496
|
-
|
|
11653
|
+
writeFileSync(tempPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf-8');
|
|
11654
|
+
renameSync(tempPath, manifestPath);
|
|
11655
|
+
}
|
|
11656
|
+
catch (error) {
|
|
11657
|
+
rmSync(tempPath, { force: true });
|
|
11658
|
+
throw error;
|
|
11659
|
+
}
|
|
11660
|
+
_lineageCache.delete(projectRoot);
|
|
11661
|
+
return manifest;
|
|
11662
|
+
}
|
|
11663
|
+
async function refreshLocalMetadataCatalog(projectRoot, manifest, semanticLayer) {
|
|
11664
|
+
try {
|
|
11665
|
+
await ensureMetadataCatalogFresh(projectRoot, {
|
|
11666
|
+
force: true,
|
|
11667
|
+
...(manifest ? { manifest } : {}),
|
|
11668
|
+
...(semanticLayer ? { semanticLayer } : {}),
|
|
11669
|
+
});
|
|
11497
11670
|
}
|
|
11498
11671
|
catch {
|
|
11499
11672
|
// The catalog is a rebuildable local cache. Save/certify flows should not
|
|
@@ -12946,13 +13119,13 @@ export function buildAgentPreviewSql(sql) {
|
|
|
12946
13119
|
* only when the owning entity is retained in GROUP BY. It never invents a
|
|
12947
13120
|
* relation, join key, or allocation rule.
|
|
12948
13121
|
*/
|
|
12949
|
-
export function repairExploratorySqlBeforeExecution(sql, schemaContext, question = '') {
|
|
13122
|
+
export function repairExploratorySqlBeforeExecution(sql, schemaContext, question = '', dialect = 'duckdb') {
|
|
12950
13123
|
const repairs = [];
|
|
12951
|
-
let repairedSql = qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs);
|
|
12952
|
-
repairedSql = repairExploratoryRelationQualifiers(repairedSql, repairs);
|
|
12953
|
-
repairedSql = repairExploratoryLifetimeMeasureSelection(repairedSql, schemaContext, question, repairs);
|
|
13124
|
+
let repairedSql = qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs, dialect);
|
|
13125
|
+
repairedSql = repairExploratoryRelationQualifiers(repairedSql, repairs, dialect);
|
|
13126
|
+
repairedSql = repairExploratoryLifetimeMeasureSelection(repairedSql, schemaContext, question, repairs, dialect);
|
|
12954
13127
|
repairedSql = repairExploratoryMisleadingPercentAliases(repairedSql, question, repairs);
|
|
12955
|
-
const grainRepair = repairExploratoryNonAdditiveAggregates(repairedSql, schemaContext, repairs);
|
|
13128
|
+
const grainRepair = repairExploratoryNonAdditiveAggregates(repairedSql, schemaContext, repairs, dialect);
|
|
12956
13129
|
repairedSql = grainRepair.sql;
|
|
12957
13130
|
return {
|
|
12958
13131
|
sql: repairedSql,
|
|
@@ -13003,8 +13176,8 @@ export function applyRequestedTopNToExploratorySql(sql, requestedTopN) {
|
|
|
13003
13176
|
}
|
|
13004
13177
|
return `${withoutTerminator}\nLIMIT ${requestedTopN}`;
|
|
13005
13178
|
}
|
|
13006
|
-
function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs) {
|
|
13007
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13179
|
+
function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs, dialect = 'duckdb') {
|
|
13180
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13008
13181
|
if (!analysis.parsed)
|
|
13009
13182
|
return sql;
|
|
13010
13183
|
let repaired = sql;
|
|
@@ -13032,8 +13205,8 @@ function qualifyExploratoryRelationsFromSchema(sql, schemaContext, repairs) {
|
|
|
13032
13205
|
}
|
|
13033
13206
|
return repaired;
|
|
13034
13207
|
}
|
|
13035
|
-
function repairExploratoryRelationQualifiers(sql, repairs) {
|
|
13036
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13208
|
+
function repairExploratoryRelationQualifiers(sql, repairs, dialect = 'duckdb') {
|
|
13209
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13037
13210
|
if (!analysis.parsed)
|
|
13038
13211
|
return sql;
|
|
13039
13212
|
const declared = Object.entries(analysis.aliasToRelation).map(([qualifier, relation]) => ({
|
|
@@ -13056,10 +13229,10 @@ function repairExploratoryRelationQualifiers(sql, repairs) {
|
|
|
13056
13229
|
}
|
|
13057
13230
|
return repaired;
|
|
13058
13231
|
}
|
|
13059
|
-
function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question, repairs) {
|
|
13232
|
+
function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question, repairs, dialect = 'duckdb') {
|
|
13060
13233
|
if (!/\b(lifetime|life\s*span|lifespan|customer\s+life)\b/i.test(question))
|
|
13061
13234
|
return sql;
|
|
13062
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13235
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13063
13236
|
if (!analysis.parsed || analysis.joins.length === 0)
|
|
13064
13237
|
return sql;
|
|
13065
13238
|
const groupClause = sql.match(/\bgroup\s+by\s+([\s\S]*?)(?:\border\s+by\b|\blimit\b|\bqualify\b|\bhaving\b|$)/i)?.[1] ?? '';
|
|
@@ -13085,8 +13258,8 @@ function repairExploratoryLifetimeMeasureSelection(sql, schemaContext, question,
|
|
|
13085
13258
|
}
|
|
13086
13259
|
return repaired;
|
|
13087
13260
|
}
|
|
13088
|
-
function repairExploratoryNonAdditiveAggregates(sql, schemaContext, repairs) {
|
|
13089
|
-
const analysis = analyzeSqlReferences(sql);
|
|
13261
|
+
function repairExploratoryNonAdditiveAggregates(sql, schemaContext, repairs, dialect = 'duckdb') {
|
|
13262
|
+
const analysis = analyzeSqlReferences(sql, dialect);
|
|
13090
13263
|
if (!analysis.parsed || analysis.joins.length === 0)
|
|
13091
13264
|
return { sql };
|
|
13092
13265
|
const risky = analysis.aggregates.filter((aggregate) => aggregate.func.toLowerCase() === 'sum'
|
|
@@ -13906,28 +14079,6 @@ export function openBlockStudioDocument(projectRoot, relativePath, semanticLayer
|
|
|
13906
14079
|
validation: validateBlockStudioSource(source, semanticLayer),
|
|
13907
14080
|
};
|
|
13908
14081
|
}
|
|
13909
|
-
function semanticMetricExecutionCapability(metricName, semanticLayer, provider, metricFlowReady, driver) {
|
|
13910
|
-
if (provider === 'dbt' && metricFlowReady) {
|
|
13911
|
-
return { status: 'ready', engine: 'metricflow', reason: null };
|
|
13912
|
-
}
|
|
13913
|
-
const native = semanticLayer.composeQuery({ metrics: [metricName], dimensions: [], driver });
|
|
13914
|
-
if (native)
|
|
13915
|
-
return { status: 'ready', engine: 'native', reason: null };
|
|
13916
|
-
const metric = semanticLayer.getMetric(metricName);
|
|
13917
|
-
if (provider === 'dbt') {
|
|
13918
|
-
const kind = metric?.metricType || metric?.aggregation || metric?.type || 'metric';
|
|
13919
|
-
return {
|
|
13920
|
-
status: 'requires_setup',
|
|
13921
|
-
engine: null,
|
|
13922
|
-
reason: `${kind} metric requires MetricFlow execution. Install/configure the MetricFlow CLI, then refresh the semantic layer.`,
|
|
13923
|
-
};
|
|
13924
|
-
}
|
|
13925
|
-
return {
|
|
13926
|
-
status: 'unsupported',
|
|
13927
|
-
engine: null,
|
|
13928
|
-
reason: 'The metric does not have enough composable measure and relation metadata.',
|
|
13929
|
-
};
|
|
13930
|
-
}
|
|
13931
14082
|
function parseBlockStudioArrayField(source, key) {
|
|
13932
14083
|
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([\\s\\S]*?)\\]`, 'i'));
|
|
13933
14084
|
if (!match)
|
|
@@ -13980,8 +14131,7 @@ export async function resolveSemanticTableMapping(executor, connection, semantic
|
|
|
13980
14131
|
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
|
|
13981
14132
|
FROM information_schema.tables
|
|
13982
14133
|
WHERE UPPER(table_schema) NOT IN ('INFORMATION_SCHEMA', 'PG_CATALOG')
|
|
13983
|
-
ORDER BY table_schema, table_name
|
|
13984
|
-
LIMIT 2000`, [], {}, connection);
|
|
14134
|
+
ORDER BY table_schema, table_name`, [], {}, connection);
|
|
13985
14135
|
return buildSemanticTableMapping(semanticLayer, tablesResult.rows);
|
|
13986
14136
|
}
|
|
13987
14137
|
catch {
|
|
@@ -14023,41 +14173,17 @@ function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
|
|
|
14023
14173
|
return true;
|
|
14024
14174
|
return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
|
|
14025
14175
|
}
|
|
14026
|
-
function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
14027
|
-
|
|
14028
|
-
|
|
14029
|
-
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
|
|
14033
|
-
|
|
14034
|
-
|
|
14035
|
-
|
|
14036
|
-
|
|
14037
|
-
dbtProjectPath,
|
|
14038
|
-
metrics: request.metrics,
|
|
14039
|
-
dimensions: request.dimensions,
|
|
14040
|
-
filters: request.filters,
|
|
14041
|
-
timeDimension: request.timeDimension,
|
|
14042
|
-
orderBy: request.orderBy,
|
|
14043
|
-
limit: request.limit,
|
|
14044
|
-
savedQuery: request.savedQuery,
|
|
14045
|
-
});
|
|
14046
|
-
return {
|
|
14047
|
-
sql: compiled.sql,
|
|
14048
|
-
joins: [],
|
|
14049
|
-
tables: [],
|
|
14050
|
-
engine: 'metricflow',
|
|
14051
|
-
};
|
|
14052
|
-
}
|
|
14053
|
-
catch (error) {
|
|
14054
|
-
// An explicit MetricFlow request must retain its strict engine contract.
|
|
14055
|
-
// For the default dbt path, a missing local CLI may fall back only if the
|
|
14056
|
-
// native composer can safely materialize the requested simple metrics.
|
|
14057
|
-
if (request.engine === 'metricflow' || !(error instanceof MetricFlowUnavailableError))
|
|
14058
|
-
throw error;
|
|
14059
|
-
}
|
|
14060
|
-
}
|
|
14176
|
+
async function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
14177
|
+
return compileSemanticRuntimeQuery(request, {
|
|
14178
|
+
projectRoot: context.projectRoot,
|
|
14179
|
+
projectConfig: context.projectConfig,
|
|
14180
|
+
detectedProvider: context.detectedProvider,
|
|
14181
|
+
semanticLayer,
|
|
14182
|
+
driver: context.driver,
|
|
14183
|
+
tableMapping: context.tableMapping,
|
|
14184
|
+
});
|
|
14185
|
+
}
|
|
14186
|
+
function composeRuntimeSemanticQueryNative(request, semanticLayer, context) {
|
|
14061
14187
|
const composed = semanticLayer.composeQuery({
|
|
14062
14188
|
metrics: request.metrics,
|
|
14063
14189
|
dimensions: request.dimensions,
|
|
@@ -14126,7 +14252,7 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14126
14252
|
let composed;
|
|
14127
14253
|
try {
|
|
14128
14254
|
composed = options?.projectRoot && options.projectConfig
|
|
14129
|
-
?
|
|
14255
|
+
? composeRuntimeSemanticQueryNative({
|
|
14130
14256
|
metrics,
|
|
14131
14257
|
dimensions: config.dimensions,
|
|
14132
14258
|
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
@@ -14135,9 +14261,6 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14135
14261
|
: undefined,
|
|
14136
14262
|
limit: runtimeBindings.limit ?? config.limit,
|
|
14137
14263
|
}, semanticLayer, {
|
|
14138
|
-
projectRoot: options.projectRoot,
|
|
14139
|
-
projectConfig: options.projectConfig,
|
|
14140
|
-
detectedProvider: options.detectedProvider ?? null,
|
|
14141
14264
|
driver: options.driver,
|
|
14142
14265
|
tableMapping: options.tableMapping,
|
|
14143
14266
|
})
|
|
@@ -14163,10 +14286,14 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14163
14286
|
}
|
|
14164
14287
|
if (!composed) {
|
|
14165
14288
|
const provider = options?.projectConfig && isDbtSemanticRuntime(options.projectConfig, options.detectedProvider, semanticLayer) ? 'dbt' : (options?.detectedProvider ?? 'dql');
|
|
14166
|
-
const metricFlowReady = provider === 'dbt' && hasMetricFlowCli();
|
|
14167
14289
|
const reasons = metrics.map((metricName) => {
|
|
14168
|
-
|
|
14169
|
-
|
|
14290
|
+
if (semanticLayer.canComposeMetric(metricName))
|
|
14291
|
+
return null;
|
|
14292
|
+
const metric = semanticLayer.getMetric(metricName);
|
|
14293
|
+
const kind = metric?.metricType || metric?.aggregation || metric?.type || 'metric';
|
|
14294
|
+
return provider === 'dbt'
|
|
14295
|
+
? `${metricName}: ${kind} metric requires a configured dbt Cloud or local MetricFlow runtime.`
|
|
14296
|
+
: `${metricName}: the metric does not have enough composable measure and relation metadata.`;
|
|
14170
14297
|
}).filter((reason) => Boolean(reason));
|
|
14171
14298
|
diagnostics.push({
|
|
14172
14299
|
severity: 'error',
|
|
@@ -14183,6 +14310,61 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
|
14183
14310
|
semanticRefs,
|
|
14184
14311
|
};
|
|
14185
14312
|
}
|
|
14313
|
+
/**
|
|
14314
|
+
* Runtime-aware semantic block compiler. Static validation remains synchronous
|
|
14315
|
+
* and native-only, while execution surfaces can use the bundled dbt Cloud or
|
|
14316
|
+
* local MetricFlow adapters without changing the persisted semantic identities.
|
|
14317
|
+
*/
|
|
14318
|
+
async function composeSemanticBlockSqlForRuntime(source, semanticLayer, options) {
|
|
14319
|
+
const native = composeSemanticBlockSql(source, semanticLayer, options);
|
|
14320
|
+
if (native.sql)
|
|
14321
|
+
return native;
|
|
14322
|
+
const config = parseSemanticBlockConfig(source);
|
|
14323
|
+
if (config.blockType !== 'semantic')
|
|
14324
|
+
return native;
|
|
14325
|
+
const metrics = config.metrics.length > 0 ? config.metrics : config.metric ? [config.metric] : [];
|
|
14326
|
+
if (metrics.length === 0 || native.diagnostics.some((diagnostic) => diagnostic.code === 'semantic_ref'))
|
|
14327
|
+
return native;
|
|
14328
|
+
const runtimeBindings = semanticRuntimeParameterBindings(source, options.parameters ?? {});
|
|
14329
|
+
try {
|
|
14330
|
+
const compiled = await composeRuntimeSemanticQuery({
|
|
14331
|
+
metrics,
|
|
14332
|
+
dimensions: config.dimensions,
|
|
14333
|
+
filters: mergeSemanticRuntimeFilters(config.filters, runtimeBindings.filters),
|
|
14334
|
+
timeDimension: config.timeDimension && config.granularity
|
|
14335
|
+
? { name: config.timeDimension, granularity: config.granularity }
|
|
14336
|
+
: undefined,
|
|
14337
|
+
limit: runtimeBindings.limit ?? config.limit,
|
|
14338
|
+
}, semanticLayer, {
|
|
14339
|
+
projectRoot: options.projectRoot,
|
|
14340
|
+
projectConfig: options.projectConfig,
|
|
14341
|
+
detectedProvider: options.detectedProvider ?? null,
|
|
14342
|
+
driver: options.driver,
|
|
14343
|
+
tableMapping: options.tableMapping,
|
|
14344
|
+
});
|
|
14345
|
+
if (!compiled)
|
|
14346
|
+
return native;
|
|
14347
|
+
return {
|
|
14348
|
+
sql: compiled.sql,
|
|
14349
|
+
diagnostics: native.diagnostics.filter((diagnostic) => diagnostic.code !== 'semantic_compose_failed'),
|
|
14350
|
+
semanticRefs: native.semanticRefs,
|
|
14351
|
+
};
|
|
14352
|
+
}
|
|
14353
|
+
catch (error) {
|
|
14354
|
+
return {
|
|
14355
|
+
sql: null,
|
|
14356
|
+
diagnostics: [
|
|
14357
|
+
...native.diagnostics.filter((diagnostic) => diagnostic.code !== 'semantic_compose_failed'),
|
|
14358
|
+
{
|
|
14359
|
+
severity: 'error',
|
|
14360
|
+
code: isSemanticRuntimeError(error) ? 'semantic_runtime_required' : 'semantic_compose_failed',
|
|
14361
|
+
message: error instanceof Error ? error.message : String(error),
|
|
14362
|
+
},
|
|
14363
|
+
],
|
|
14364
|
+
semanticRefs: native.semanticRefs,
|
|
14365
|
+
};
|
|
14366
|
+
}
|
|
14367
|
+
}
|
|
14186
14368
|
function semanticRuntimeParameterBindings(source, values) {
|
|
14187
14369
|
const program = new Parser(source, '<semantic-parameters>').parse();
|
|
14188
14370
|
const block = program.statements.find((statement) => statement.kind === NodeKind.BlockDecl);
|
|
@@ -18208,6 +18390,7 @@ const LOCAL_RUNTIME_GITIGNORE_RULES = [
|
|
|
18208
18390
|
'**/.dql/cache/',
|
|
18209
18391
|
'**/.dql/imports/',
|
|
18210
18392
|
'**/.dql/local/',
|
|
18393
|
+
'**/.dql/runtimes/',
|
|
18211
18394
|
'**/.dql/connectors/',
|
|
18212
18395
|
'**/.dql/memory/',
|
|
18213
18396
|
'**/.dql/migration-staging/',
|