@duckcodeailabs/dql-cli 0.8.10 → 0.8.12

package/dist/local-runtime.js

@@ -1,8 +1,8 @@
 import { createServer } from 'node:http';
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
 import { dirname, extname, join, normalize, relative, resolve } from 'node:path';
-import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, } from '@duckcodeailabs/dql-notebook';
-import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, } from '@duckcodeailabs/dql-core';
+import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
+import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, buildManifest, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryCompleteLineagePaths, LineageGraph, } from '@duckcodeailabs/dql-core';
 import { listBlockTemplates } from './block-templates.js';
 import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
 export async function startLocalServer(opts) {
@@ -1463,6 +1463,75 @@ export async function startLocalServer(opts) {
             }
             return;
         }
+        if (req.method === 'GET' && path === '/api/lineage/search') {
+            const term = url.searchParams.get('q') ?? '';
+            try {
+                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
+                const result = queryLineage(graph, { search: term });
+                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ matches: result.matches ?? [] }));
+            }
+            catch (error) {
+                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
+            }
+            return;
+        }
+        if (req.method === 'GET' && path === '/api/lineage/query') {
+            try {
+                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
+                const types = url.searchParams.get('types')
+                    ?.split(',')
+                    .map((value) => value.trim())
+                    .filter(Boolean);
+                const upstreamDepthParam = url.searchParams.get('upstreamDepth');
+                const downstreamDepthParam = url.searchParams.get('downstreamDepth');
+                const result = queryLineage(graph, {
+                    focus: url.searchParams.get('focus') ?? undefined,
+                    search: url.searchParams.get('search') ?? undefined,
+                    types,
+                    domain: url.searchParams.get('domain') ?? undefined,
+                    upstreamDepth: upstreamDepthParam ? Number(upstreamDepthParam) : undefined,
+                    downstreamDepth: downstreamDepthParam ? Number(downstreamDepthParam) : undefined,
+                });
+                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON(result));
+            }
+            catch (error) {
+                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
+            }
+            return;
+        }
+        if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
+            const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
+            try {
+                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
+                const node = resolveLineageNode(graph, rawNodeId);
+                if (!node) {
+                    res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
+                    return;
+                }
+                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({
+                    node,
+                    incoming: graph.getIncomingEdges(node.id).map((edge) => ({
+                        edge,
+                        node: graph.getNode(edge.source),
+                    })),
+                    outgoing: graph.getOutgoingEdges(node.id).map((edge) => ({
+                        edge,
+                        node: graph.getNode(edge.target),
+                    })),
+                }));
+            }
+            catch (error) {
+                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
+            }
+            return;
+        }
         if (req.method === 'GET' && path.startsWith('/api/lineage/domain/')) {
             const domain = decodeURIComponent(path.slice('/api/lineage/domain/'.length));
             try {
@@ -1527,6 +1596,27 @@ export async function startLocalServer(opts) {
             }
             return;
         }
+        if (req.method === 'GET' && path.startsWith('/api/lineage/paths/')) {
+            const rawNodeId = decodeURIComponent(path.slice('/api/lineage/paths/'.length));
+            try {
+                const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
+                const maxDepth = Number(url.searchParams.get('maxDepth') ?? '10') || 10;
+                const maxPaths = Number(url.searchParams.get('maxPaths') ?? '20') || 20;
+                const result = queryCompleteLineagePaths(graph, rawNodeId, { maxDepth, maxPaths });
+                if (!result) {
+                    res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
+                    res.end(serializeJSON({ error: `Node "${rawNodeId}" not found` }));
+                    return;
+                }
+                res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON(result));
+            }
+            catch (error) {
+                res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
+                res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
+            }
+            return;
+        }
         if (req.method === 'GET' && path === '/api/lineage/trust-chain') {
             const from = url.searchParams.get('from');
             const to = url.searchParams.get('to');
@@ -2277,6 +2367,40 @@ function composeSemanticBlockSql(source, semanticLayer, options) {
         semanticRefs,
     };
 }
+function resolveCustomBlockSql(sql, semanticLayer) {
+    if (!sql) {
+        return {
+            sql: null,
+            diagnostics: [],
+            semanticRefs: { metrics: [], dimensions: [], segments: [] },
+        };
+    }
+    const semanticRefs = extractBlockStudioSemanticReferences(sql);
+    if (!hasSemanticRefs(sql)) {
+        return { sql, diagnostics: [], semanticRefs };
+    }
+    const resolution = resolveSemanticRefs(sql, semanticLayer);
+    if (resolution.unresolvedRefs.length > 0) {
+        return {
+            sql: null,
+            diagnostics: resolution.unresolvedRefs.map((unresolved) => ({
+                severity: 'error',
+                code: 'semantic_ref',
+                message: `Unknown semantic reference: ${unresolved}`,
+            })),
+            semanticRefs,
+        };
+    }
+    return {
+        sql: resolution.resolvedSql,
+        diagnostics: [],
+        semanticRefs: {
+            metrics: resolution.resolvedMetrics,
+            dimensions: resolution.resolvedDimensions,
+            segments: semanticRefs.segments,
+        },
+    };
+}
 export function validateBlockStudioSource(source, semanticLayer) {
     const diagnostics = [];
     const semanticConfig = parseSemanticBlockConfig(source);
@@ -2335,18 +2459,10 @@ export function validateBlockStudioSource(source, semanticLayer) {
         }
     }
     else if (semanticLayer) {
-        const refValidation = semanticLayer.validateReferences([
-            ...semanticRefs.metrics,
-            ...semanticRefs.dimensions,
-            ...semanticRefs.segments,
-        ]);
-        for (const unknown of refValidation.unknown) {
-            diagnostics.push({
-                severity: 'error',
-                code: 'semantic_ref',
-                message: `Unknown semantic reference: ${unknown}`,
-            });
-        }
+        const resolvedCustomSql = resolveCustomBlockSql(executableSql, semanticLayer);
+        semanticRefs = resolvedCustomSql.semanticRefs;
+        diagnostics.push(...resolvedCustomSql.diagnostics);
+        executableSql = resolvedCustomSql.sql;
     }
     const chartConfig = extractBlockStudioChartConfig(source);
     if (!chartConfig) {
@@ -2856,52 +2972,99 @@ function buildNotebookTemplate(title, template) {
     return JSON.stringify({ version: 1, title, cells }, null, 2);
 }
 /** Build a lineage graph from the project's blocks and semantic layer. */
+// Simple lineage graph cache: rebuilds at most every 5 seconds
+let _lineageCache = null;
+const LINEAGE_CACHE_TTL_MS = 5000;
 function buildProjectLineageGraph(projectRoot, semanticLayer) {
-    const blocks = [];
-    const metrics = [];
-    const dimensions = [];
-    // Scan .dql files
-    const dirs = ['blocks', 'dashboards', 'workbooks'];
-    for (const dir of dirs) {
-        const dirPath = join(projectRoot, dir);
-        if (!existsSync(dirPath))
-            continue;
-        for (const entry of readdirSync(dirPath, { withFileTypes: true })) {
-            if (!entry.isFile() || extname(entry.name) !== '.dql')
-                continue;
-            try {
-                const source = readFileSync(join(dirPath, entry.name), 'utf-8');
-                const parser = new Parser(source, `${dir}/${entry.name}`);
-                const ast = parser.parse();
-                for (const stmt of ast.statements) {
-                    const block = stmt;
-                    if (block.kind !== 'BlockDecl')
-                        continue;
-                    blocks.push({
-                        name: block.name,
-                        sql: block.query?.rawSQL ?? '',
-                        domain: extractProp(block, 'domain'),
-                        owner: extractProp(block, 'owner'),
-                        status: extractProp(block, 'status'),
-                        blockType: block.blockType,
-                        metricRef: block.metricRef,
-                        chartType: extractVizChart(block),
-                    });
-                }
+    if (_lineageCache && Date.now() - _lineageCache.builtAt < LINEAGE_CACHE_TTL_MS) {
+        return _lineageCache.graph;
+    }
+    const graph = buildProjectLineageGraphUncached(projectRoot, semanticLayer);
+    _lineageCache = { graph, builtAt: Date.now() };
+    return graph;
+}
+function buildProjectLineageGraphUncached(projectRoot, semanticLayer) {
+    const manifestPath = join(projectRoot, 'dql-manifest.json');
+    if (existsSync(manifestPath)) {
+        try {
+            const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+            if (manifest.lineage?.nodes && manifest.lineage?.edges) {
+                return LineageGraph.fromJSON({
+                    nodes: manifest.lineage.nodes,
+                    edges: manifest.lineage.edges,
+                });
             }
-            catch { /* skip unparseable */ }
         }
+        catch {
+            // Fall back to a live build.
+        }
+    }
+    const dbtManifestPath = resolveDbtManifestPath(projectRoot);
+    try {
+        const manifest = buildManifest({
+            projectRoot,
+            dbtManifestPath,
+        });
+        return LineageGraph.fromJSON({
+            nodes: manifest.lineage.nodes,
+            edges: manifest.lineage.edges,
+        });
     }
-    // Load from semantic layer
-    if (semanticLayer) {
-        for (const m of semanticLayer.listMetrics()) {
-            metrics.push({ name: m.name, table: m.table, domain: m.domain, type: m.type });
+    catch {
+        const blocks = [];
+        const metrics = [];
+        const dimensions = [];
+        const dirs = ['blocks', 'dashboards', 'workbooks'];
+        for (const dir of dirs) {
+            const dirPath = join(projectRoot, dir);
+            if (!existsSync(dirPath))
+                continue;
+            for (const entry of readdirSync(dirPath, { withFileTypes: true })) {
+                if (!entry.isFile() || extname(entry.name) !== '.dql')
+                    continue;
+                try {
+                    const source = readFileSync(join(dirPath, entry.name), 'utf-8');
+                    const parser = new Parser(source, `${dir}/${entry.name}`);
+                    const ast = parser.parse();
+                    for (const stmt of ast.statements) {
+                        const block = stmt;
+                        if (block.kind !== 'BlockDecl')
+                            continue;
+                        blocks.push({
+                            name: block.name,
+                            sql: block.query?.rawSQL ?? '',
+                            domain: extractProp(block, 'domain'),
+                            owner: extractProp(block, 'owner'),
+                            status: extractProp(block, 'status'),
+                            blockType: block.blockType,
+                            metricRef: block.metricRef,
+                            chartType: extractVizChart(block),
+                        });
+                    }
+                }
+                catch { /* skip unparseable */ }
+            }
         }
-        for (const d of semanticLayer.listDimensions()) {
-            dimensions.push({ name: d.name, table: d.table });
+        if (semanticLayer) {
+            for (const m of semanticLayer.listMetrics()) {
+                metrics.push({ name: m.name, table: m.table, domain: m.domain, type: m.type });
+            }
+            for (const d of semanticLayer.listDimensions()) {
+                dimensions.push({ name: d.name, table: d.table });
+            }
         }
+        return buildLineageGraph(blocks, metrics, dimensions);
     }
-    return buildLineageGraph(blocks, metrics, dimensions);
+}
+function resolveDbtManifestPath(projectRoot) {
+    const candidate = join(projectRoot, 'target', 'manifest.json');
+    return existsSync(candidate) ? candidate : undefined;
+}
+function resolveLineageNode(graph, rawNodeId) {
+    if (graph.getNode(rawNodeId))
+        return graph.getNode(rawNodeId);
+    const result = queryLineage(graph, { focus: rawNodeId });
+    return result.focalNode;
 }
 function extractProp(block, key) {
     // Check direct AST fields first (parser puts domain, owner, type directly on the node)