@duckcodeailabs/dql-cli 0.8.8 → 0.8.10

package/dist/local-runtime.js

@@ -2,7 +2,7 @@ import { createServer } from 'node:http';
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
 import { dirname, extname, join, normalize, relative, resolve } from 'node:path';
 import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, } from '@duckcodeailabs/dql-notebook';
-import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, searchNodes, } from '@duckcodeailabs/dql-core';
+import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, } from '@duckcodeailabs/dql-core';
 import { listBlockTemplates } from './block-templates.js';
 import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
 export async function startLocalServer(opts) {
@@ -595,17 +595,37 @@ export async function startLocalServer(opts) {
             try {
                 const body = await readJSON(req);
                 const source = typeof body.source === 'string' ? body.source : '';
+                const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
+                let tableMapping;
+                if (semanticLayer) {
+                    try {
+                        const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
+               FROM information_schema.tables
+               WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
+                        tableMapping = buildSemanticTableMapping(semanticLayer, tablesResult.rows);
+                    }
+                    catch {
+                        tableMapping = undefined;
+                    }
+                }
+                const semanticCompose = semanticLayer
+                    ? composeSemanticBlockSql(source, semanticLayer, { driver: targetConnection.driver, tableMapping })
+                    : null;
                 const validation = validateBlockStudioSource(source, semanticLayer);
-                if (!validation.executableSql) {
+                const executableSql = semanticCompose?.sql ?? validation.executableSql;
+                if (!executableSql) {
                     res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
-                    res.end(serializeJSON({ error: 'No executable SQL found in block source.' }));
+                    const message = semanticCompose?.diagnostics.find((item) => item.severity === 'error')?.message
+                        ?? validation.diagnostics.find((item) => item.severity === 'error')?.message
+                        ?? 'No executable SQL found in block source.';
+                    res.end(serializeJSON({ error: message, diagnostics: validation.diagnostics }));
                     return;
                 }
-                const sql = resolveProjectRelativeSqlPaths(validation.executableSql, projectRoot, projectConfig.dataDir);
-                const result = await executor.executeQuery(sql, [], {}, connection);
+                const sql = resolveProjectRelativeSqlPaths(executableSql, projectRoot, projectConfig.dataDir);
+                const result = await executor.executeQuery(sql, [], {}, targetConnection);
                 res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
                 res.end(serializeJSON({
-                    sql: validation.executableSql,
+                    sql: executableSql,
                     result: normalizeQueryResult(result),
                     chartConfig: validation.chartConfig ?? null,
                 }));
@@ -1527,83 +1547,6 @@ export async function startLocalServer(opts) {
             }
             return;
         }
-        // ---- Lineage Query API (focused/searchable views) ----
-        if (req.method === 'GET' && path === '/api/lineage/query') {
-            try {
-                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
-                const focus = url.searchParams.get('focus') ?? undefined;
-                const search = url.searchParams.get('search') ?? undefined;
-                const typesParam = url.searchParams.get('types');
-                const domain = url.searchParams.get('domain') ?? undefined;
-                const upstreamDepth = url.searchParams.has('upstreamDepth')
-                    ? Number(url.searchParams.get('upstreamDepth'))
-                    : undefined;
-                const downstreamDepth = url.searchParams.has('downstreamDepth')
-                    ? Number(url.searchParams.get('downstreamDepth'))
-                    : undefined;
-                const types = typesParam
-                    ? typesParam.split(',').map((t) => t.trim())
-                    : undefined;
-                const result = queryLineage(graph, {
-                    focus,
-                    search,
-                    types,
-                    domain,
-                    upstreamDepth,
-                    downstreamDepth,
-                });
-                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON(result));
-            }
-            catch (error) {
-                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
-            }
-            return;
-        }
-        if (req.method === 'GET' && path === '/api/lineage/search') {
-            const q = url.searchParams.get('q') ?? '';
-            if (!q) {
-                res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON({ error: 'Missing "q" query parameter' }));
-                return;
-            }
-            try {
-                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
-                const limit = url.searchParams.has('limit') ? Number(url.searchParams.get('limit')) : 20;
-                const matches = searchNodes(graph, q, limit);
-                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON({ matches }));
-            }
-            catch (error) {
-                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
-            }
-            return;
-        }
-        if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
-            const nodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
-            try {
-                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
-                const node = graph.getNode(nodeId);
-                if (!node) {
-                    res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
-                    res.end(serializeJSON({ error: `Node "${nodeId}" not found` }));
-                    return;
-                }
-                const ancestors = graph.ancestors(nodeId);
-                const descendants = graph.descendants(nodeId);
-                const incoming = graph.getIncomingEdges(nodeId);
-                const outgoing = graph.getOutgoingEdges(nodeId);
-                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON({ node, ancestors, descendants, incoming, outgoing }));
-            }
-            catch (error) {
-                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
-            }
-            return;
-        }
         if (req.method === 'GET' && path === '/api/notebook/bootstrap') {
             const welcomeNotebook = resolveNotebook(projectRoot, projectConfig.project ?? 'DQL Project');
             res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
@@ -2215,21 +2158,183 @@ function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
         validation: validateBlockStudioSource(source, semanticLayer),
     };
 }
-function validateBlockStudioSource(source, semanticLayer) {
+function parseBlockStudioArrayField(source, key) {
+    const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([\\s\\S]*?)\\]`, 'i'));
+    if (!match)
+        return [];
+    return (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)).filter(Boolean);
+}
+function parseBlockStudioStringField(source, key) {
+    return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
+}
+function parseSemanticBlockConfig(source) {
+    const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
+        ? 'semantic'
+        : 'custom';
+    const metric = parseBlockStudioStringField(source, 'metric');
+    const metrics = parseBlockStudioArrayField(source, 'metrics');
+    const dimensions = parseBlockStudioArrayField(source, 'dimensions');
+    const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
+    const granularity = parseBlockStudioStringField(source, 'granularity');
+    const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
+    return {
+        blockType,
+        metric,
+        metrics,
+        dimensions,
+        timeDimension,
+        granularity,
+        limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
+    };
+}
+function buildSemanticTableMapping(semanticLayer, rows) {
+    const dbTableNames = new Set();
+    const schemaQualified = new Map();
+    for (const row of rows) {
+        const schema = String(row['table_schema'] ?? '');
+        const name = String(row['table_name'] ?? '');
+        if (!name)
+            continue;
+        dbTableNames.add(name);
+        schemaQualified.set(name, schema ? `${schema}.${name}` : name);
+    }
+    const tableMapping = {};
+    const allSemanticTables = new Set();
+    for (const metric of semanticLayer.listMetrics())
+        allSemanticTables.add(metric.table);
+    for (const dimension of semanticLayer.listDimensions())
+        allSemanticTables.add(dimension.table);
+    for (const semTable of allSemanticTables) {
+        if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
+            tableMapping[semTable] = schemaQualified.get(semTable);
+        }
+    }
+    return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
+}
+function composeSemanticBlockSql(source, semanticLayer, options) {
+    const config = parseSemanticBlockConfig(source);
+    const metrics = config.metrics.length > 0
+        ? config.metrics
+        : config.metric
+            ? [config.metric]
+            : [];
+    const semanticRefs = {
+        metrics,
+        dimensions: config.dimensions,
+        segments: [],
+    };
     const diagnostics = [];
-    try {
-        const parser = new Parser(source, '<block-studio>');
-        parser.parse();
+    if (config.blockType !== 'semantic') {
+        return { sql: null, diagnostics, semanticRefs };
     }
-    catch (error) {
+    if (metrics.length === 0) {
         diagnostics.push({
             severity: 'error',
-            code: 'syntax',
-            message: error instanceof Error ? error.message : String(error),
+            code: 'semantic_metric_missing',
+            message: 'Semantic block is missing a metric. Add metric = "metric_name" or metrics = ["metric_name"].',
         });
+        return { sql: null, diagnostics, semanticRefs };
     }
-    const semanticRefs = extractBlockStudioSemanticReferences(source);
-    if (semanticLayer) {
+    if (config.timeDimension && !config.granularity) {
+        diagnostics.push({
+            severity: 'error',
+            code: 'semantic_granularity_missing',
+            message: `Semantic block selects time_dimension = "${config.timeDimension}" but is missing granularity.`,
+        });
+    }
+    const refValidation = semanticLayer.validateReferences([...metrics, ...config.dimensions]);
+    for (const unknown of refValidation.unknown) {
+        diagnostics.push({
+            severity: 'error',
+            code: 'semantic_ref',
+            message: `Unknown semantic reference: ${unknown}`,
+        });
+    }
+    if (diagnostics.some((diagnostic) => diagnostic.severity === 'error')) {
+        return { sql: null, diagnostics, semanticRefs };
+    }
+    const composed = semanticLayer.composeQuery({
+        metrics,
+        dimensions: config.dimensions,
+        timeDimension: config.timeDimension && config.granularity
+            ? { name: config.timeDimension, granularity: config.granularity }
+            : undefined,
+        limit: config.limit,
+        driver: options?.driver,
+        tableMapping: options?.tableMapping,
+    });
+    if (!composed) {
+        diagnostics.push({
+            severity: 'error',
+            code: 'semantic_compose_failed',
+            message: `Could not compose SQL for semantic block metrics: [${metrics.join(', ')}].`,
+        });
+        return { sql: null, diagnostics, semanticRefs };
+    }
+    return {
+        sql: composed.sql,
+        diagnostics,
+        semanticRefs,
+    };
+}
+export function validateBlockStudioSource(source, semanticLayer) {
+    const diagnostics = [];
+    const semanticConfig = parseSemanticBlockConfig(source);
+    if (semanticConfig.blockType !== 'semantic') {
+        try {
+            const parser = new Parser(source, '<block-studio>');
+            parser.parse();
+        }
+        catch (error) {
+            diagnostics.push({
+                severity: 'error',
+                code: 'syntax',
+                message: error instanceof Error ? error.message : String(error),
+            });
+        }
+    }
+    else {
+        const hasBlockHeader = /\bblock\s+"[^"]+"\s*\{/i.test(source);
+        const hasClosingBrace = /\}\s*$/m.test(source);
+        if (!hasBlockHeader || !hasClosingBrace) {
+            diagnostics.push({
+                severity: 'error',
+                code: 'semantic_shape',
+                message: 'Semantic block must use block "Name" { ... } structure.',
+            });
+        }
+    }
+    let semanticRefs = extractBlockStudioSemanticReferences(source);
+    if (semanticConfig.blockType === 'semantic') {
+        const selectedMetrics = semanticConfig.metrics.length > 0
+            ? semanticConfig.metrics
+            : semanticConfig.metric
+                ? [semanticConfig.metric]
+                : [];
+        semanticRefs = {
+            metrics: selectedMetrics,
+            dimensions: semanticConfig.dimensions,
+            segments: semanticRefs.segments,
+        };
+    }
+    let executableSql = extractBlockStudioSql(source);
+    if (semanticConfig.blockType === 'semantic') {
+        if (semanticLayer) {
+            const semanticCompose = composeSemanticBlockSql(source, semanticLayer);
+            semanticRefs = semanticCompose.semanticRefs;
+            diagnostics.push(...semanticCompose.diagnostics);
+            executableSql = semanticCompose.sql;
+        }
+        else {
+            diagnostics.push({
+                severity: 'error',
+                code: 'semantic_layer_missing',
+                message: 'Semantic block cannot run because no semantic layer is configured.',
+            });
+            executableSql = null;
+        }
+    }
+    else if (semanticLayer) {
         const refValidation = semanticLayer.validateReferences([
             ...semanticRefs.metrics,
             ...semanticRefs.dimensions,
@@ -2251,13 +2356,18 @@ function validateBlockStudioSource(source, semanticLayer) {
             message: 'Block has no visualization section yet.',
         });
     }
-    const executableSql = extractBlockStudioSql(source);
     if (!executableSql) {
-        diagnostics.push({
-            severity: 'warning',
-            code: 'sql_missing',
-            message: 'No executable SQL found in the block source.',
-        });
+        diagnostics.push(semanticConfig.blockType === 'semantic'
+            ? {
+                severity: 'warning',
+                code: 'semantic_not_runnable',
+                message: 'Semantic block is not runnable yet. Select a metric and complete any required time settings.',
+            }
+            : {
+                severity: 'warning',
+                code: 'sql_missing',
+                message: 'No executable SQL found in the block source.',
+            });
     }
     return {
         valid: diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),