@duckcodeailabs/dql-core 1.14.1 → 1.14.3-rc.1

package/dist/manifest/builder.js

@@ -36,6 +36,10 @@ import { loadManifestKnowledgeSkills, manifestKnowledgeSkillInputFiles } from '.
 export function collectInputFiles(options) {
     const { projectRoot } = options;
     const files = new Set();
+    // Cache overlapping directory walks here as well. `sync dbt` asks for these
+    // tracked files before a cache lookup, so repeated block/domain roots must
+    // not turn a warm-cache check into several 4k-file traversals.
+    const scanCache = createManifestScanCache();
     const configPath = join(projectRoot, 'dql.config.json');
     if (existsSync(configPath))
         files.add(configPath);
@@ -46,27 +50,27 @@ export function collectInputFiles(options) {
     }
     const blockDirs = ['blocks', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
     for (const dir of blockDirs) {
-        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql']))
+        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql'], scanCache))
             files.add(f);
     }
     const businessViewDirs = ['blocks', 'business-views', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
     for (const dir of businessViewDirs) {
-        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql']))
+        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql'], scanCache))
             files.add(f);
     }
     const termDirs = ['terms', 'blocks', 'business-views', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
     for (const dir of termDirs) {
-        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql']))
+        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dql'], scanCache))
             files.add(f);
     }
     const notebookDirs = ['notebooks', 'domains', 'blocks', 'dashboards', 'workbooks', ...(options.extraNotebookDirs ?? [])];
     for (const dir of notebookDirs) {
-        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dqlnb']))
+        for (const f of scanFilesRecursive(join(projectRoot, dir), ['.dqlnb'], scanCache))
             files.add(f);
     }
     const semanticDir = resolveSemanticPath(projectRoot, config);
     if (existsSync(semanticDir)) {
-        for (const f of scanFilesRecursive(semanticDir, ['.yaml', '.yml']))
+        for (const f of scanFilesRecursive(semanticDir, ['.yaml', '.yml'], scanCache))
             files.add(f);
     }
     if (options.dbtManifestPath && existsSync(options.dbtManifestPath)) {
@@ -79,7 +83,7 @@ export function collectInputFiles(options) {
             files.add(semanticManifestPath);
     }
     if (config.manifestVersion === 3 && config.modeling?.mode === 'dbt-first') {
-        for (const f of scanFilesRecursive(join(projectRoot, 'domains'), ['.yaml', '.yml']))
+        for (const f of scanFilesRecursive(join(projectRoot, 'domains'), ['.yaml', '.yml'], scanCache))
             files.add(f);
         const registry = loadDomainPackageRegistry(projectRoot);
         for (const f of manifestKnowledgeSkillInputFiles(projectRoot, registry))
@@ -98,6 +102,12 @@ export function collectInputFiles(options) {
 export function buildManifest(options) {
     const { projectRoot, dqlVersion = '0.6.0' } = options;
     const diagnostics = [];
+    // A large project commonly stores blocks, terms, and domains under the same
+    // directory tree. The manifest scanner needs distinct declaration passes,
+    // but it must not re-walk that tree or re-read the same source three times
+    // during one build. Source text is always read freshly; unchanged text can
+    // reuse a bounded parsed AST across adjacent builds.
+    const scanCache = createManifestScanCache(projectRoot);
     // Load project config
     const config = loadProjectConfig(projectRoot);
     const projectName = config.project ?? 'dql-project';
@@ -126,20 +136,20 @@ export function buildManifest(options) {
     const packageRegistry = dbtFirstV3 ? loadDomainPackageRegistry(projectRoot) : undefined;
     const domains = packageRegistry
         ? Object.fromEntries(packageRegistry.values().map((pkg) => [pkg.id, pkg.domain]))
-        : scanDomains(projectRoot, domainDirs, diagnostics);
+        : scanDomains(projectRoot, domainDirs, diagnostics, scanCache);
     if (!packageRegistry)
         validateDomainHierarchy(domains, diagnostics);
     // Scan blocks
     const blockDirs = ['blocks', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
-    const blockScan = scanBlocks(projectRoot, blockDirs, diagnostics, datalexRegistry);
+    const blockScan = scanBlocks(projectRoot, blockDirs, diagnostics, datalexRegistry, scanCache);
     const blocks = blockScan.blocks;
     const knowledgeBlocks = blockScan.all;
     // Scan business composition views
     const businessViewDirs = ['blocks', 'business-views', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
-    const businessViews = scanBusinessViews(projectRoot, businessViewDirs, diagnostics);
+    const businessViews = scanBusinessViews(projectRoot, businessViewDirs, diagnostics, scanCache);
     // Scan business glossary terms
     const termDirs = ['terms', 'blocks', 'business-views', 'domains', 'dashboards', 'workbooks', ...(options.extraBlockDirs ?? [])];
-    const terms = scanTerms(projectRoot, termDirs, diagnostics);
+    const terms = scanTerms(projectRoot, termDirs, diagnostics, scanCache);
     if (packageRegistry) {
         applyDomainPackageOwnership(projectRoot, packageRegistry, blocks, businessViews, terms, diagnostics);
         applyDomainPackageOwnershipToValues(projectRoot, packageRegistry, knowledgeBlocks, 'Block', diagnostics);
@@ -455,11 +465,44 @@ function resolveSemanticPath(projectRoot, config) {
         return join(projectRoot, customPath);
     return join(projectRoot, 'semantic-layer');
 }
-// ---- Recursive File Scanner ----
-function scanFilesRecursive(dir, extensions) {
+// Keep only the most recently compiled project roots. Source text is compared
+// before any AST reuse, so this is a parser cache rather than a stale-source
+// cache; it makes watcher-triggered one-file edits avoid reparsing the other
+// thousands of unchanged DQL declarations.
+const MAX_MANIFEST_PARSE_CACHE_PROJECTS = 4;
+const manifestParseCaches = new Map();
+function createManifestScanCache(projectRoot) {
+    let parsedSources;
+    if (projectRoot) {
+        parsedSources = manifestParseCaches.get(projectRoot);
+        if (parsedSources) {
+            // Map insertion order doubles as a tiny LRU.
+            manifestParseCaches.delete(projectRoot);
+            manifestParseCaches.set(projectRoot, parsedSources);
+        }
+        else {
+            parsedSources = new Map();
+            manifestParseCaches.set(projectRoot, parsedSources);
+            while (manifestParseCaches.size > MAX_MANIFEST_PARSE_CACHE_PROJECTS) {
+                const oldest = manifestParseCaches.keys().next().value;
+                if (!oldest)
+                    break;
+                manifestParseCaches.delete(oldest);
+            }
+        }
+    }
+    return { filesByDirectory: new Map(), sources: new Map(), parsedSources };
+}
+function scanFilesRecursive(dir, extensions, cache) {
+    const cacheKey = `${dir}\u0000${[...extensions].sort().join(',')}`;
+    const cached = cache?.filesByDirectory.get(cacheKey);
+    if (cached)
+        return cached;
     const results = [];
-    if (!existsSync(dir))
+    if (!existsSync(dir)) {
+        cache?.filesByDirectory.set(cacheKey, results);
         return results;
+    }
     const entries = readdirSync(dir, { withFileTypes: true });
     for (const entry of entries) {
         const fullPath = join(dir, entry.name);
@@ -467,32 +510,48 @@ function scanFilesRecursive(dir, extensions) {
             // Skip hidden dirs and node_modules
             if (entry.name.startsWith('.') || entry.name === 'node_modules')
                 continue;
-            results.push(...scanFilesRecursive(fullPath, extensions));
+            results.push(...scanFilesRecursive(fullPath, extensions, cache));
         }
         else if (entry.isFile() && extensions.includes(extname(entry.name))) {
             results.push(fullPath);
         }
     }
+    cache?.filesByDirectory.set(cacheKey, results);
     return results;
 }
+function readManifestSource(filePath, cache) {
+    const cached = cache?.sources.get(filePath);
+    if (cached !== undefined)
+        return cached;
+    const source = readFileSync(filePath, 'utf-8');
+    cache?.sources.set(filePath, source);
+    return source;
+}
+function parseManifestSource(source, relPath, filePath, cache) {
+    const cached = cache?.parsedSources?.get(filePath);
+    if (cached?.source === source)
+        return cached.ast;
+    const ast = new Parser(source, relPath).parse();
+    cache?.parsedSources?.set(filePath, { source, ast });
+    return ast;
+}
 // ---- Block Scanning ----
-function scanDomains(projectRoot, dirs, diagnostics) {
+function scanDomains(projectRoot, dirs, diagnostics, scanCache) {
     const domains = {};
     const seenFiles = new Set();
     for (const dir of dirs) {
         const dirPath = join(projectRoot, dir);
-        const files = scanFilesRecursive(dirPath, ['.dql']);
+        const files = scanFilesRecursive(dirPath, ['.dql'], scanCache);
         for (const filePath of files) {
             if (seenFiles.has(filePath))
                 continue;
             seenFiles.add(filePath);
             const relPath = relative(projectRoot, filePath);
             try {
-                const source = readFileSync(filePath, 'utf-8');
+                const source = readManifestSource(filePath, scanCache);
                 if (!/(^|\n)\s*domain\s+"/.test(source))
                     continue;
-                const parser = new Parser(source, relPath);
-                const ast = parser.parse();
+                const ast = parseManifestSource(source, relPath, filePath, scanCache);
                 for (const stmt of ast.statements) {
                     const domain = stmt;
                     if (domain.kind !== 'DomainDecl')
@@ -557,22 +616,21 @@ function validateDomainHierarchy(domains, diagnostics) {
         }
     }
 }
-function scanBlocks(projectRoot, dirs, diagnostics, datalexRegistry) {
+function scanBlocks(projectRoot, dirs, diagnostics, datalexRegistry, scanCache) {
     const blocks = {};
     const all = [];
     const seenFiles = new Set();
     for (const dir of dirs) {
         const dirPath = join(projectRoot, dir);
-        const files = scanFilesRecursive(dirPath, ['.dql']);
+        const files = scanFilesRecursive(dirPath, ['.dql'], scanCache);
         for (const filePath of files) {
             if (seenFiles.has(filePath))
                 continue;
             seenFiles.add(filePath);
             const relPath = relative(projectRoot, filePath);
             try {
-                const source = readFileSync(filePath, 'utf-8');
-                const parser = new Parser(source, relPath);
-                const ast = parser.parse();
+                const source = readManifestSource(filePath, scanCache);
+                const ast = parseManifestSource(source, relPath, filePath, scanCache);
                 collectContractDiagnostics(ast, relPath, diagnostics, datalexRegistry);
                 for (const stmt of ast.statements) {
                     const block = stmt;
@@ -629,23 +687,22 @@ function duplicateBlockPriority(block) {
     return score;
 }
 // ---- Business View Scanning ----
-function scanBusinessViews(projectRoot, dirs, diagnostics) {
+function scanBusinessViews(projectRoot, dirs, diagnostics, scanCache) {
     const views = {};
     const seenFiles = new Set();
     for (const dir of dirs) {
         const dirPath = join(projectRoot, dir);
-        const files = scanFilesRecursive(dirPath, ['.dql']);
+        const files = scanFilesRecursive(dirPath, ['.dql'], scanCache);
         for (const filePath of files) {
             if (seenFiles.has(filePath))
                 continue;
             seenFiles.add(filePath);
             const relPath = relative(projectRoot, filePath);
             try {
-                const source = readFileSync(filePath, 'utf-8');
+                const source = readManifestSource(filePath, scanCache);
                 if (!source.includes('business_view'))
                     continue;
-                const parser = new Parser(source, relPath);
-                const ast = parser.parse();
+                const ast = parseManifestSource(source, relPath, filePath, scanCache);
                 for (const stmt of ast.statements) {
                     const view = stmt;
                     if (view.kind !== 'BusinessViewDecl')
@@ -676,23 +733,22 @@ function scanBusinessViews(projectRoot, dirs, diagnostics) {
     return views;
 }
 // ---- Business Term Scanning ----
-function scanTerms(projectRoot, dirs, diagnostics) {
+function scanTerms(projectRoot, dirs, diagnostics, scanCache) {
     const terms = {};
     const seenFiles = new Set();
     for (const dir of dirs) {
         const dirPath = join(projectRoot, dir);
-        const files = scanFilesRecursive(dirPath, ['.dql']);
+        const files = scanFilesRecursive(dirPath, ['.dql'], scanCache);
         for (const filePath of files) {
             if (seenFiles.has(filePath))
                 continue;
             seenFiles.add(filePath);
             const relPath = relative(projectRoot, filePath);
             try {
-                const source = readFileSync(filePath, 'utf-8');
+                const source = readManifestSource(filePath, scanCache);
                 if (!source.includes('term'))
                     continue;
-                const parser = new Parser(source, relPath);
-                const ast = parser.parse();
+                const ast = parseManifestSource(source, relPath, filePath, scanCache);
                 for (const stmt of ast.statements) {
                     const term = stmt;
                     if (term.kind !== 'TermDecl')