@duckcodeailabs/dql-cli 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/dql-notebook/assets/index-Cxj__xjY.js +564 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/commands/lineage.d.ts +4 -4
- package/dist/commands/lineage.d.ts.map +1 -1
- package/dist/commands/lineage.js +126 -246
- package/dist/commands/lineage.js.map +1 -1
- package/dist/local-runtime.d.ts +24 -1
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +387 -146
- package/dist/local-runtime.js.map +1 -1
- package/dist/local-runtime.test.js +130 -1
- package/dist/local-runtime.test.js.map +1 -1
- package/dist/semantic-import.js +11 -11
- package/dist/semantic-import.js.map +1 -1
- package/dist/semantic-import.test.js +89 -272
- package/dist/semantic-import.test.js.map +1 -1
- package/package.json +9 -9
- package/dist/assets/dql-notebook/assets/index-Cp34wXvX.js +0 -558
package/dist/local-runtime.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { createServer } from 'node:http';
|
|
2
2
|
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
|
|
3
3
|
import { dirname, extname, join, normalize, relative, resolve } from 'node:path';
|
|
4
|
-
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, } from '@duckcodeailabs/dql-notebook';
|
|
5
|
-
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage,
|
|
4
|
+
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
5
|
+
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, buildManifest, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, LineageGraph, } from '@duckcodeailabs/dql-core';
|
|
6
6
|
import { listBlockTemplates } from './block-templates.js';
|
|
7
7
|
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
8
8
|
export async function startLocalServer(opts) {
|
|
@@ -595,17 +595,37 @@ export async function startLocalServer(opts) {
|
|
|
595
595
|
try {
|
|
596
596
|
const body = await readJSON(req);
|
|
597
597
|
const source = typeof body.source === 'string' ? body.source : '';
|
|
598
|
+
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
599
|
+
let tableMapping;
|
|
600
|
+
if (semanticLayer) {
|
|
601
|
+
try {
|
|
602
|
+
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
|
|
603
|
+
FROM information_schema.tables
|
|
604
|
+
WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
|
|
605
|
+
tableMapping = buildSemanticTableMapping(semanticLayer, tablesResult.rows);
|
|
606
|
+
}
|
|
607
|
+
catch {
|
|
608
|
+
tableMapping = undefined;
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
const semanticCompose = semanticLayer
|
|
612
|
+
? composeSemanticBlockSql(source, semanticLayer, { driver: targetConnection.driver, tableMapping })
|
|
613
|
+
: null;
|
|
598
614
|
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
599
|
-
|
|
615
|
+
const executableSql = semanticCompose?.sql ?? validation.executableSql;
|
|
616
|
+
if (!executableSql) {
|
|
600
617
|
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
601
|
-
|
|
618
|
+
const message = semanticCompose?.diagnostics.find((item) => item.severity === 'error')?.message
|
|
619
|
+
?? validation.diagnostics.find((item) => item.severity === 'error')?.message
|
|
620
|
+
?? 'No executable SQL found in block source.';
|
|
621
|
+
res.end(serializeJSON({ error: message, diagnostics: validation.diagnostics }));
|
|
602
622
|
return;
|
|
603
623
|
}
|
|
604
|
-
const sql = resolveProjectRelativeSqlPaths(
|
|
605
|
-
const result = await executor.executeQuery(sql, [], {},
|
|
624
|
+
const sql = resolveProjectRelativeSqlPaths(executableSql, projectRoot, projectConfig.dataDir);
|
|
625
|
+
const result = await executor.executeQuery(sql, [], {}, targetConnection);
|
|
606
626
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
607
627
|
res.end(serializeJSON({
|
|
608
|
-
sql:
|
|
628
|
+
sql: executableSql,
|
|
609
629
|
result: normalizeQueryResult(result),
|
|
610
630
|
chartConfig: validation.chartConfig ?? null,
|
|
611
631
|
}));
|
|
@@ -1443,6 +1463,75 @@ export async function startLocalServer(opts) {
|
|
|
1443
1463
|
}
|
|
1444
1464
|
return;
|
|
1445
1465
|
}
|
|
1466
|
+
if (req.method === 'GET' && path === '/api/lineage/search') {
|
|
1467
|
+
const term = url.searchParams.get('q') ?? '';
|
|
1468
|
+
try {
|
|
1469
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
1470
|
+
const result = queryLineage(graph, { search: term });
|
|
1471
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1472
|
+
res.end(serializeJSON({ matches: result.matches ?? [] }));
|
|
1473
|
+
}
|
|
1474
|
+
catch (error) {
|
|
1475
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1476
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1477
|
+
}
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
if (req.method === 'GET' && path === '/api/lineage/query') {
|
|
1481
|
+
try {
|
|
1482
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
1483
|
+
const types = url.searchParams.get('types')
|
|
1484
|
+
?.split(',')
|
|
1485
|
+
.map((value) => value.trim())
|
|
1486
|
+
.filter(Boolean);
|
|
1487
|
+
const upstreamDepthParam = url.searchParams.get('upstreamDepth');
|
|
1488
|
+
const downstreamDepthParam = url.searchParams.get('downstreamDepth');
|
|
1489
|
+
const result = queryLineage(graph, {
|
|
1490
|
+
focus: url.searchParams.get('focus') ?? undefined,
|
|
1491
|
+
search: url.searchParams.get('search') ?? undefined,
|
|
1492
|
+
types,
|
|
1493
|
+
domain: url.searchParams.get('domain') ?? undefined,
|
|
1494
|
+
upstreamDepth: upstreamDepthParam ? Number(upstreamDepthParam) : undefined,
|
|
1495
|
+
downstreamDepth: downstreamDepthParam ? Number(downstreamDepthParam) : undefined,
|
|
1496
|
+
});
|
|
1497
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1498
|
+
res.end(serializeJSON(result));
|
|
1499
|
+
}
|
|
1500
|
+
catch (error) {
|
|
1501
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1502
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1503
|
+
}
|
|
1504
|
+
return;
|
|
1505
|
+
}
|
|
1506
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
|
|
1507
|
+
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
|
|
1508
|
+
try {
|
|
1509
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
1510
|
+
const node = resolveLineageNode(graph, rawNodeId);
|
|
1511
|
+
if (!node) {
|
|
1512
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1513
|
+
res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
|
|
1514
|
+
return;
|
|
1515
|
+
}
|
|
1516
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1517
|
+
res.end(serializeJSON({
|
|
1518
|
+
node,
|
|
1519
|
+
incoming: graph.getIncomingEdges(node.id).map((edge) => ({
|
|
1520
|
+
edge,
|
|
1521
|
+
node: graph.getNode(edge.source),
|
|
1522
|
+
})),
|
|
1523
|
+
outgoing: graph.getOutgoingEdges(node.id).map((edge) => ({
|
|
1524
|
+
edge,
|
|
1525
|
+
node: graph.getNode(edge.target),
|
|
1526
|
+
})),
|
|
1527
|
+
}));
|
|
1528
|
+
}
|
|
1529
|
+
catch (error) {
|
|
1530
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1531
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1532
|
+
}
|
|
1533
|
+
return;
|
|
1534
|
+
}
|
|
1446
1535
|
if (req.method === 'GET' && path.startsWith('/api/lineage/domain/')) {
|
|
1447
1536
|
const domain = decodeURIComponent(path.slice('/api/lineage/domain/'.length));
|
|
1448
1537
|
try {
|
|
@@ -1527,83 +1616,6 @@ export async function startLocalServer(opts) {
|
|
|
1527
1616
|
}
|
|
1528
1617
|
return;
|
|
1529
1618
|
}
|
|
1530
|
-
// ---- Lineage Query API (focused/searchable views) ----
|
|
1531
|
-
if (req.method === 'GET' && path === '/api/lineage/query') {
|
|
1532
|
-
try {
|
|
1533
|
-
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
1534
|
-
const focus = url.searchParams.get('focus') ?? undefined;
|
|
1535
|
-
const search = url.searchParams.get('search') ?? undefined;
|
|
1536
|
-
const typesParam = url.searchParams.get('types');
|
|
1537
|
-
const domain = url.searchParams.get('domain') ?? undefined;
|
|
1538
|
-
const upstreamDepth = url.searchParams.has('upstreamDepth')
|
|
1539
|
-
? Number(url.searchParams.get('upstreamDepth'))
|
|
1540
|
-
: undefined;
|
|
1541
|
-
const downstreamDepth = url.searchParams.has('downstreamDepth')
|
|
1542
|
-
? Number(url.searchParams.get('downstreamDepth'))
|
|
1543
|
-
: undefined;
|
|
1544
|
-
const types = typesParam
|
|
1545
|
-
? typesParam.split(',').map((t) => t.trim())
|
|
1546
|
-
: undefined;
|
|
1547
|
-
const result = queryLineage(graph, {
|
|
1548
|
-
focus,
|
|
1549
|
-
search,
|
|
1550
|
-
types,
|
|
1551
|
-
domain,
|
|
1552
|
-
upstreamDepth,
|
|
1553
|
-
downstreamDepth,
|
|
1554
|
-
});
|
|
1555
|
-
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1556
|
-
res.end(serializeJSON(result));
|
|
1557
|
-
}
|
|
1558
|
-
catch (error) {
|
|
1559
|
-
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1560
|
-
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1561
|
-
}
|
|
1562
|
-
return;
|
|
1563
|
-
}
|
|
1564
|
-
if (req.method === 'GET' && path === '/api/lineage/search') {
|
|
1565
|
-
const q = url.searchParams.get('q') ?? '';
|
|
1566
|
-
if (!q) {
|
|
1567
|
-
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1568
|
-
res.end(serializeJSON({ error: 'Missing "q" query parameter' }));
|
|
1569
|
-
return;
|
|
1570
|
-
}
|
|
1571
|
-
try {
|
|
1572
|
-
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
1573
|
-
const limit = url.searchParams.has('limit') ? Number(url.searchParams.get('limit')) : 20;
|
|
1574
|
-
const matches = searchNodes(graph, q, limit);
|
|
1575
|
-
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1576
|
-
res.end(serializeJSON({ matches }));
|
|
1577
|
-
}
|
|
1578
|
-
catch (error) {
|
|
1579
|
-
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1580
|
-
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1581
|
-
}
|
|
1582
|
-
return;
|
|
1583
|
-
}
|
|
1584
|
-
if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
|
|
1585
|
-
const nodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
|
|
1586
|
-
try {
|
|
1587
|
-
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
1588
|
-
const node = graph.getNode(nodeId);
|
|
1589
|
-
if (!node) {
|
|
1590
|
-
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1591
|
-
res.end(serializeJSON({ error: `Node "${nodeId}" not found` }));
|
|
1592
|
-
return;
|
|
1593
|
-
}
|
|
1594
|
-
const ancestors = graph.ancestors(nodeId);
|
|
1595
|
-
const descendants = graph.descendants(nodeId);
|
|
1596
|
-
const incoming = graph.getIncomingEdges(nodeId);
|
|
1597
|
-
const outgoing = graph.getOutgoingEdges(nodeId);
|
|
1598
|
-
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1599
|
-
res.end(serializeJSON({ node, ancestors, descendants, incoming, outgoing }));
|
|
1600
|
-
}
|
|
1601
|
-
catch (error) {
|
|
1602
|
-
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1603
|
-
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1604
|
-
}
|
|
1605
|
-
return;
|
|
1606
|
-
}
|
|
1607
1619
|
if (req.method === 'GET' && path === '/api/notebook/bootstrap') {
|
|
1608
1620
|
const welcomeNotebook = resolveNotebook(projectRoot, projectConfig.project ?? 'DQL Project');
|
|
1609
1621
|
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
@@ -2215,34 +2227,222 @@ function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
|
2215
2227
|
validation: validateBlockStudioSource(source, semanticLayer),
|
|
2216
2228
|
};
|
|
2217
2229
|
}
|
|
2218
|
-
function
|
|
2230
|
+
function parseBlockStudioArrayField(source, key) {
|
|
2231
|
+
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([\\s\\S]*?)\\]`, 'i'));
|
|
2232
|
+
if (!match)
|
|
2233
|
+
return [];
|
|
2234
|
+
return (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)).filter(Boolean);
|
|
2235
|
+
}
|
|
2236
|
+
function parseBlockStudioStringField(source, key) {
|
|
2237
|
+
return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
|
|
2238
|
+
}
|
|
2239
|
+
function parseSemanticBlockConfig(source) {
|
|
2240
|
+
const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
|
|
2241
|
+
? 'semantic'
|
|
2242
|
+
: 'custom';
|
|
2243
|
+
const metric = parseBlockStudioStringField(source, 'metric');
|
|
2244
|
+
const metrics = parseBlockStudioArrayField(source, 'metrics');
|
|
2245
|
+
const dimensions = parseBlockStudioArrayField(source, 'dimensions');
|
|
2246
|
+
const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
|
|
2247
|
+
const granularity = parseBlockStudioStringField(source, 'granularity');
|
|
2248
|
+
const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
|
|
2249
|
+
return {
|
|
2250
|
+
blockType,
|
|
2251
|
+
metric,
|
|
2252
|
+
metrics,
|
|
2253
|
+
dimensions,
|
|
2254
|
+
timeDimension,
|
|
2255
|
+
granularity,
|
|
2256
|
+
limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
|
|
2257
|
+
};
|
|
2258
|
+
}
|
|
2259
|
+
function buildSemanticTableMapping(semanticLayer, rows) {
|
|
2260
|
+
const dbTableNames = new Set();
|
|
2261
|
+
const schemaQualified = new Map();
|
|
2262
|
+
for (const row of rows) {
|
|
2263
|
+
const schema = String(row['table_schema'] ?? '');
|
|
2264
|
+
const name = String(row['table_name'] ?? '');
|
|
2265
|
+
if (!name)
|
|
2266
|
+
continue;
|
|
2267
|
+
dbTableNames.add(name);
|
|
2268
|
+
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
2269
|
+
}
|
|
2270
|
+
const tableMapping = {};
|
|
2271
|
+
const allSemanticTables = new Set();
|
|
2272
|
+
for (const metric of semanticLayer.listMetrics())
|
|
2273
|
+
allSemanticTables.add(metric.table);
|
|
2274
|
+
for (const dimension of semanticLayer.listDimensions())
|
|
2275
|
+
allSemanticTables.add(dimension.table);
|
|
2276
|
+
for (const semTable of allSemanticTables) {
|
|
2277
|
+
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
2278
|
+
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
2279
|
+
}
|
|
2280
|
+
}
|
|
2281
|
+
return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
|
|
2282
|
+
}
|
|
2283
|
+
function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
2284
|
+
const config = parseSemanticBlockConfig(source);
|
|
2285
|
+
const metrics = config.metrics.length > 0
|
|
2286
|
+
? config.metrics
|
|
2287
|
+
: config.metric
|
|
2288
|
+
? [config.metric]
|
|
2289
|
+
: [];
|
|
2290
|
+
const semanticRefs = {
|
|
2291
|
+
metrics,
|
|
2292
|
+
dimensions: config.dimensions,
|
|
2293
|
+
segments: [],
|
|
2294
|
+
};
|
|
2219
2295
|
const diagnostics = [];
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
parser.parse();
|
|
2296
|
+
if (config.blockType !== 'semantic') {
|
|
2297
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
2223
2298
|
}
|
|
2224
|
-
|
|
2299
|
+
if (metrics.length === 0) {
|
|
2225
2300
|
diagnostics.push({
|
|
2226
2301
|
severity: 'error',
|
|
2227
|
-
code: '
|
|
2228
|
-
message:
|
|
2302
|
+
code: 'semantic_metric_missing',
|
|
2303
|
+
message: 'Semantic block is missing a metric. Add metric = "metric_name" or metrics = ["metric_name"].',
|
|
2229
2304
|
});
|
|
2305
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
2230
2306
|
}
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2307
|
+
if (config.timeDimension && !config.granularity) {
|
|
2308
|
+
diagnostics.push({
|
|
2309
|
+
severity: 'error',
|
|
2310
|
+
code: 'semantic_granularity_missing',
|
|
2311
|
+
message: `Semantic block selects time_dimension = "${config.timeDimension}" but is missing granularity.`,
|
|
2312
|
+
});
|
|
2313
|
+
}
|
|
2314
|
+
const refValidation = semanticLayer.validateReferences([...metrics, ...config.dimensions]);
|
|
2315
|
+
for (const unknown of refValidation.unknown) {
|
|
2316
|
+
diagnostics.push({
|
|
2317
|
+
severity: 'error',
|
|
2318
|
+
code: 'semantic_ref',
|
|
2319
|
+
message: `Unknown semantic reference: ${unknown}`,
|
|
2320
|
+
});
|
|
2321
|
+
}
|
|
2322
|
+
if (diagnostics.some((diagnostic) => diagnostic.severity === 'error')) {
|
|
2323
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
2324
|
+
}
|
|
2325
|
+
const composed = semanticLayer.composeQuery({
|
|
2326
|
+
metrics,
|
|
2327
|
+
dimensions: config.dimensions,
|
|
2328
|
+
timeDimension: config.timeDimension && config.granularity
|
|
2329
|
+
? { name: config.timeDimension, granularity: config.granularity }
|
|
2330
|
+
: undefined,
|
|
2331
|
+
limit: config.limit,
|
|
2332
|
+
driver: options?.driver,
|
|
2333
|
+
tableMapping: options?.tableMapping,
|
|
2334
|
+
});
|
|
2335
|
+
if (!composed) {
|
|
2336
|
+
diagnostics.push({
|
|
2337
|
+
severity: 'error',
|
|
2338
|
+
code: 'semantic_compose_failed',
|
|
2339
|
+
message: `Could not compose SQL for semantic block metrics: [${metrics.join(', ')}].`,
|
|
2340
|
+
});
|
|
2341
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
2342
|
+
}
|
|
2343
|
+
return {
|
|
2344
|
+
sql: composed.sql,
|
|
2345
|
+
diagnostics,
|
|
2346
|
+
semanticRefs,
|
|
2347
|
+
};
|
|
2348
|
+
}
|
|
2349
|
+
function resolveCustomBlockSql(sql, semanticLayer) {
|
|
2350
|
+
if (!sql) {
|
|
2351
|
+
return {
|
|
2352
|
+
sql: null,
|
|
2353
|
+
diagnostics: [],
|
|
2354
|
+
semanticRefs: { metrics: [], dimensions: [], segments: [] },
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
const semanticRefs = extractBlockStudioSemanticReferences(sql);
|
|
2358
|
+
if (!hasSemanticRefs(sql)) {
|
|
2359
|
+
return { sql, diagnostics: [], semanticRefs };
|
|
2360
|
+
}
|
|
2361
|
+
const resolution = resolveSemanticRefs(sql, semanticLayer);
|
|
2362
|
+
if (resolution.unresolvedRefs.length > 0) {
|
|
2363
|
+
return {
|
|
2364
|
+
sql: null,
|
|
2365
|
+
diagnostics: resolution.unresolvedRefs.map((unresolved) => ({
|
|
2240
2366
|
severity: 'error',
|
|
2241
2367
|
code: 'semantic_ref',
|
|
2242
|
-
message: `Unknown semantic reference: ${
|
|
2368
|
+
message: `Unknown semantic reference: ${unresolved}`,
|
|
2369
|
+
})),
|
|
2370
|
+
semanticRefs,
|
|
2371
|
+
};
|
|
2372
|
+
}
|
|
2373
|
+
return {
|
|
2374
|
+
sql: resolution.resolvedSql,
|
|
2375
|
+
diagnostics: [],
|
|
2376
|
+
semanticRefs: {
|
|
2377
|
+
metrics: resolution.resolvedMetrics,
|
|
2378
|
+
dimensions: resolution.resolvedDimensions,
|
|
2379
|
+
segments: semanticRefs.segments,
|
|
2380
|
+
},
|
|
2381
|
+
};
|
|
2382
|
+
}
|
|
2383
|
+
export function validateBlockStudioSource(source, semanticLayer) {
|
|
2384
|
+
const diagnostics = [];
|
|
2385
|
+
const semanticConfig = parseSemanticBlockConfig(source);
|
|
2386
|
+
if (semanticConfig.blockType !== 'semantic') {
|
|
2387
|
+
try {
|
|
2388
|
+
const parser = new Parser(source, '<block-studio>');
|
|
2389
|
+
parser.parse();
|
|
2390
|
+
}
|
|
2391
|
+
catch (error) {
|
|
2392
|
+
diagnostics.push({
|
|
2393
|
+
severity: 'error',
|
|
2394
|
+
code: 'syntax',
|
|
2395
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2243
2396
|
});
|
|
2244
2397
|
}
|
|
2245
2398
|
}
|
|
2399
|
+
else {
|
|
2400
|
+
const hasBlockHeader = /\bblock\s+"[^"]+"\s*\{/i.test(source);
|
|
2401
|
+
const hasClosingBrace = /\}\s*$/m.test(source);
|
|
2402
|
+
if (!hasBlockHeader || !hasClosingBrace) {
|
|
2403
|
+
diagnostics.push({
|
|
2404
|
+
severity: 'error',
|
|
2405
|
+
code: 'semantic_shape',
|
|
2406
|
+
message: 'Semantic block must use block "Name" { ... } structure.',
|
|
2407
|
+
});
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2410
|
+
let semanticRefs = extractBlockStudioSemanticReferences(source);
|
|
2411
|
+
if (semanticConfig.blockType === 'semantic') {
|
|
2412
|
+
const selectedMetrics = semanticConfig.metrics.length > 0
|
|
2413
|
+
? semanticConfig.metrics
|
|
2414
|
+
: semanticConfig.metric
|
|
2415
|
+
? [semanticConfig.metric]
|
|
2416
|
+
: [];
|
|
2417
|
+
semanticRefs = {
|
|
2418
|
+
metrics: selectedMetrics,
|
|
2419
|
+
dimensions: semanticConfig.dimensions,
|
|
2420
|
+
segments: semanticRefs.segments,
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
let executableSql = extractBlockStudioSql(source);
|
|
2424
|
+
if (semanticConfig.blockType === 'semantic') {
|
|
2425
|
+
if (semanticLayer) {
|
|
2426
|
+
const semanticCompose = composeSemanticBlockSql(source, semanticLayer);
|
|
2427
|
+
semanticRefs = semanticCompose.semanticRefs;
|
|
2428
|
+
diagnostics.push(...semanticCompose.diagnostics);
|
|
2429
|
+
executableSql = semanticCompose.sql;
|
|
2430
|
+
}
|
|
2431
|
+
else {
|
|
2432
|
+
diagnostics.push({
|
|
2433
|
+
severity: 'error',
|
|
2434
|
+
code: 'semantic_layer_missing',
|
|
2435
|
+
message: 'Semantic block cannot run because no semantic layer is configured.',
|
|
2436
|
+
});
|
|
2437
|
+
executableSql = null;
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
else if (semanticLayer) {
|
|
2441
|
+
const resolvedCustomSql = resolveCustomBlockSql(executableSql, semanticLayer);
|
|
2442
|
+
semanticRefs = resolvedCustomSql.semanticRefs;
|
|
2443
|
+
diagnostics.push(...resolvedCustomSql.diagnostics);
|
|
2444
|
+
executableSql = resolvedCustomSql.sql;
|
|
2445
|
+
}
|
|
2246
2446
|
const chartConfig = extractBlockStudioChartConfig(source);
|
|
2247
2447
|
if (!chartConfig) {
|
|
2248
2448
|
diagnostics.push({
|
|
@@ -2251,13 +2451,18 @@ function validateBlockStudioSource(source, semanticLayer) {
|
|
|
2251
2451
|
message: 'Block has no visualization section yet.',
|
|
2252
2452
|
});
|
|
2253
2453
|
}
|
|
2254
|
-
const executableSql = extractBlockStudioSql(source);
|
|
2255
2454
|
if (!executableSql) {
|
|
2256
|
-
diagnostics.push(
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2455
|
+
diagnostics.push(semanticConfig.blockType === 'semantic'
|
|
2456
|
+
? {
|
|
2457
|
+
severity: 'warning',
|
|
2458
|
+
code: 'semantic_not_runnable',
|
|
2459
|
+
message: 'Semantic block is not runnable yet. Select a metric and complete any required time settings.',
|
|
2460
|
+
}
|
|
2461
|
+
: {
|
|
2462
|
+
severity: 'warning',
|
|
2463
|
+
code: 'sql_missing',
|
|
2464
|
+
message: 'No executable SQL found in the block source.',
|
|
2465
|
+
});
|
|
2261
2466
|
}
|
|
2262
2467
|
return {
|
|
2263
2468
|
valid: diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
@@ -2747,51 +2952,87 @@ function buildNotebookTemplate(title, template) {
|
|
|
2747
2952
|
}
|
|
2748
2953
|
/** Build a lineage graph from the project's blocks and semantic layer. */
|
|
2749
2954
|
function buildProjectLineageGraph(projectRoot, semanticLayer) {
|
|
2750
|
-
const
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
for (const entry of readdirSync(dirPath, { withFileTypes: true })) {
|
|
2760
|
-
if (!entry.isFile() || extname(entry.name) !== '.dql')
|
|
2761
|
-
continue;
|
|
2762
|
-
try {
|
|
2763
|
-
const source = readFileSync(join(dirPath, entry.name), 'utf-8');
|
|
2764
|
-
const parser = new Parser(source, `${dir}/${entry.name}`);
|
|
2765
|
-
const ast = parser.parse();
|
|
2766
|
-
for (const stmt of ast.statements) {
|
|
2767
|
-
const block = stmt;
|
|
2768
|
-
if (block.kind !== 'BlockDecl')
|
|
2769
|
-
continue;
|
|
2770
|
-
blocks.push({
|
|
2771
|
-
name: block.name,
|
|
2772
|
-
sql: block.query?.rawSQL ?? '',
|
|
2773
|
-
domain: extractProp(block, 'domain'),
|
|
2774
|
-
owner: extractProp(block, 'owner'),
|
|
2775
|
-
status: extractProp(block, 'status'),
|
|
2776
|
-
blockType: block.blockType,
|
|
2777
|
-
metricRef: block.metricRef,
|
|
2778
|
-
chartType: extractVizChart(block),
|
|
2779
|
-
});
|
|
2780
|
-
}
|
|
2955
|
+
const manifestPath = join(projectRoot, 'dql-manifest.json');
|
|
2956
|
+
if (existsSync(manifestPath)) {
|
|
2957
|
+
try {
|
|
2958
|
+
const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
2959
|
+
if (manifest.lineage?.nodes && manifest.lineage?.edges) {
|
|
2960
|
+
return LineageGraph.fromJSON({
|
|
2961
|
+
nodes: manifest.lineage.nodes,
|
|
2962
|
+
edges: manifest.lineage.edges,
|
|
2963
|
+
});
|
|
2781
2964
|
}
|
|
2782
|
-
catch { /* skip unparseable */ }
|
|
2783
2965
|
}
|
|
2966
|
+
catch {
|
|
2967
|
+
// Fall back to a live build.
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
2971
|
+
try {
|
|
2972
|
+
const manifest = buildManifest({
|
|
2973
|
+
projectRoot,
|
|
2974
|
+
dbtManifestPath,
|
|
2975
|
+
});
|
|
2976
|
+
return LineageGraph.fromJSON({
|
|
2977
|
+
nodes: manifest.lineage.nodes,
|
|
2978
|
+
edges: manifest.lineage.edges,
|
|
2979
|
+
});
|
|
2784
2980
|
}
|
|
2785
|
-
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2981
|
+
catch {
|
|
2982
|
+
const blocks = [];
|
|
2983
|
+
const metrics = [];
|
|
2984
|
+
const dimensions = [];
|
|
2985
|
+
const dirs = ['blocks', 'dashboards', 'workbooks'];
|
|
2986
|
+
for (const dir of dirs) {
|
|
2987
|
+
const dirPath = join(projectRoot, dir);
|
|
2988
|
+
if (!existsSync(dirPath))
|
|
2989
|
+
continue;
|
|
2990
|
+
for (const entry of readdirSync(dirPath, { withFileTypes: true })) {
|
|
2991
|
+
if (!entry.isFile() || extname(entry.name) !== '.dql')
|
|
2992
|
+
continue;
|
|
2993
|
+
try {
|
|
2994
|
+
const source = readFileSync(join(dirPath, entry.name), 'utf-8');
|
|
2995
|
+
const parser = new Parser(source, `${dir}/${entry.name}`);
|
|
2996
|
+
const ast = parser.parse();
|
|
2997
|
+
for (const stmt of ast.statements) {
|
|
2998
|
+
const block = stmt;
|
|
2999
|
+
if (block.kind !== 'BlockDecl')
|
|
3000
|
+
continue;
|
|
3001
|
+
blocks.push({
|
|
3002
|
+
name: block.name,
|
|
3003
|
+
sql: block.query?.rawSQL ?? '',
|
|
3004
|
+
domain: extractProp(block, 'domain'),
|
|
3005
|
+
owner: extractProp(block, 'owner'),
|
|
3006
|
+
status: extractProp(block, 'status'),
|
|
3007
|
+
blockType: block.blockType,
|
|
3008
|
+
metricRef: block.metricRef,
|
|
3009
|
+
chartType: extractVizChart(block),
|
|
3010
|
+
});
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
catch { /* skip unparseable */ }
|
|
3014
|
+
}
|
|
2789
3015
|
}
|
|
2790
|
-
|
|
2791
|
-
|
|
3016
|
+
if (semanticLayer) {
|
|
3017
|
+
for (const m of semanticLayer.listMetrics()) {
|
|
3018
|
+
metrics.push({ name: m.name, table: m.table, domain: m.domain, type: m.type });
|
|
3019
|
+
}
|
|
3020
|
+
for (const d of semanticLayer.listDimensions()) {
|
|
3021
|
+
dimensions.push({ name: d.name, table: d.table });
|
|
3022
|
+
}
|
|
2792
3023
|
}
|
|
3024
|
+
return buildLineageGraph(blocks, metrics, dimensions);
|
|
2793
3025
|
}
|
|
2794
|
-
|
|
3026
|
+
}
|
|
3027
|
+
function resolveDbtManifestPath(projectRoot) {
|
|
3028
|
+
const candidate = join(projectRoot, 'target', 'manifest.json');
|
|
3029
|
+
return existsSync(candidate) ? candidate : undefined;
|
|
3030
|
+
}
|
|
3031
|
+
function resolveLineageNode(graph, rawNodeId) {
|
|
3032
|
+
if (graph.getNode(rawNodeId))
|
|
3033
|
+
return graph.getNode(rawNodeId);
|
|
3034
|
+
const result = queryLineage(graph, { focus: rawNodeId });
|
|
3035
|
+
return result.focalNode;
|
|
2795
3036
|
}
|
|
2796
3037
|
function extractProp(block, key) {
|
|
2797
3038
|
// Check direct AST fields first (parser puts domain, owner, type directly on the node)
|