gitnexus 1.6.2-rc.2 → 1.6.2-rc.4

package/dist/core/group/extractors/manifest-extractor.js

@@ -68,14 +68,29 @@ export function manifestSymbolUid(repo, contractId) {
 }
 export class ManifestExtractor {
     async extractFromManifest(links, dbExecutors) {
-        const contracts = [];
-        const crossLinks = [];
-        for (const link of links) {
+        const resolveCache = new Map();
+        const resolveOnce = (repo, link) => {
+            const key = `${repo}\u0000${link.type}\u0000${link.contract}`;
+            let pending = resolveCache.get(key);
+            if (!pending) {
+                pending = this.resolveSymbol(repo, link, dbExecutors);
+                resolveCache.set(key, pending);
+            }
+            return pending;
+        };
+        const perLink = await Promise.all(links.map(async (link) => {
             const contractId = this.buildContractId(link.type, link.contract);
             const providerRepo = link.role === 'provider' ? link.from : link.to;
             const consumerRepo = link.role === 'provider' ? link.to : link.from;
-            const providerSymbol = await this.resolveSymbol(providerRepo, link, dbExecutors);
-            const consumerSymbol = await this.resolveSymbol(consumerRepo, link, dbExecutors);
+            const [providerSymbol, consumerSymbol] = await Promise.all([
+                resolveOnce(providerRepo, link),
+                resolveOnce(consumerRepo, link),
+            ]);
+            return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol };
+        }));
+        const contracts = [];
+        const crossLinks = [];
+        for (const { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol, } of perLink) {
             const providerRef = providerSymbol || { filePath: '', name: link.contract };
             const consumerRef = consumerSymbol || { filePath: '', name: link.contract };
             // When the resolver finds a real graph symbol we keep its uid, otherwise

package/dist/core/group/sync.js

@@ -6,6 +6,7 @@ import { readRegistry } from '../../storage/repo-manager.js';
 import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
 import { GrpcExtractor } from './extractors/grpc-extractor.js';
 import { TopicExtractor } from './extractors/topic-extractor.js';
+import { ManifestExtractor } from './extractors/manifest-extractor.js';
 import { runExactMatch } from './matching.js';
 import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
 import { writeContractRegistry } from './storage.js';
@@ -34,10 +35,28 @@ function defaultResolveHandle(allEntries) {
         };
     };
 }
+/**
+ * Dedupe cross-links that point from the same consumer endpoint to the same
+ * provider endpoint for the same contract. Preserves first-seen order so the
+ * caller controls precedence (e.g., pass manifest links first).
+ */
+function dedupeCrossLinks(links) {
+    const seen = new Set();
+    const out = [];
+    for (const link of links) {
+        const key = `${link.from.repo}::${link.from.symbolUid}|${link.to.repo}::${link.to.symbolUid}|${link.type}|${link.contractId}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(link);
+    }
+    return out;
+}
 export async function syncGroup(config, opts) {
     const missingRepos = [];
     const repoSnapshots = {};
     let autoContracts = [];
+    let manifestCrossLinks = [];
     let dbExecutors;
     const eo = opts?.extractorOverride;
     if (eo && eo.length === 0) {
@@ -124,8 +143,37 @@ export async function syncGroup(config, opts) {
             }
         }
     }
+    // Process manifest links declared in group.yaml.
+    // ManifestExtractor is fully implemented but was never wired into this
+    // pipeline — config.links were parsed and validated but silently dropped.
+    // Placed after the DB try/finally: resolveSymbol falls back to synthetic
+    // UIDs when dbExecutors is undefined or a pool is closed, so cross-links
+    // are always generated regardless of whether real DB executors are available.
+    if (config.links.length > 0) {
+        // Warn about dangling links that reference repos not declared in config.repos.
+        // They still generate cross-links via synthetic UIDs (determinism is preserved),
+        // but the operator probably meant something that now silently does nothing useful.
+        const knownRepos = new Set(Object.keys(config.repos));
+        for (const link of config.links) {
+            const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
+            if (dangling.length > 0) {
+                console.warn(`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`);
+            }
+        }
+        const manifestEx = new ManifestExtractor();
+        const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors);
+        autoContracts.push(...manifestResult.contracts);
+        manifestCrossLinks = manifestResult.crossLinks;
+        if (opts?.verbose) {
+            console.log(`  manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`);
+        }
+    }
     const { matched, unmatched } = runExactMatch(autoContracts);
-    const crossLinks = matched;
+    // Dedupe cross-links. Manifest contracts participate in runExactMatch, so a
+    // manifest-declared link can also emit a matchType:'exact' CrossLink with the
+    // same endpoints. Prefer the manifest version — it reflects operator intent
+    // and carries matchType:'manifest' which downstream consumers may rely on.
+    const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]);
     const allContracts = autoContracts;
     const registry = {
         version: 1,

package/dist/core/lbug/lbug-adapter.js

@@ -1,6 +1,8 @@
 import fs from 'fs/promises';
 import { createReadStream, createWriteStream } from 'fs';
 import { createInterface } from 'readline';
+import { once } from 'events';
+import { finished } from 'stream/promises';
 import path from 'path';
 import lbug from '@ladybugdb/core';
 import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
@@ -25,100 +27,82 @@ export const splitRelCsvByLabelPair = async (csvPath, csvDir, validTables, getNo
     const pairWriteStreams = new Map();
     let skippedRels = 0;
     let totalValidRels = 0;
-    await new Promise((resolve, reject) => {
-        const inputStream = createReadStream(csvPath, 'utf-8');
-        const rl = createInterface({
-            input: inputStream,
-            crlfDelay: Infinity,
-        });
-        // Track which streams are already waiting for drain to prevent
-        // listener accumulation. rl.pause() is not synchronous — buffered
-        // line events continue firing after pause(), and without this guard
-        // each line targeting the same pairKey would add another drain listener.
-        const waitingForDrain = new Set();
-        let settled = false;
-        const cleanup = (err) => {
-            if (settled)
-                return;
-            settled = true;
-            try {
-                rl.close();
-            }
-            catch { }
-            try {
-                inputStream.destroy();
-            }
-            catch { }
-            for (const ws of pairWriteStreams.values()) {
-                try {
-                    ws.destroy();
-                }
-                catch { }
-            }
-            reject(err);
-        };
+    const inputStream = createReadStream(csvPath, 'utf-8');
+    const rl = createInterface({ input: inputStream, crlfDelay: Infinity });
+    // If any pair WriteStream errors (disk full, EMFILE, etc.) or the input
+    // stream fails, we need to abort the pending `once(ws, 'drain')` await.
+    // An AbortController gives us one signal to cancel all pending waits
+    // without a custom state machine.
+    const abortOnError = new AbortController();
+    let streamError = null;
+    const markStreamError = (err) => {
+        streamError ??= err;
+        abortOnError.abort(err);
+    };
+    try {
+        // `for await (const line of rl)` replaces the old manual
+        // on('line')/pause()/resume()/waitingForDrain state machine: readline's
+        // async iterator naturally serializes line delivery with our awaits, so
+        // at most one ws can be in backpressure at a time and we just await its
+        // 'drain' event.
         let isFirst = true;
-        rl.on('line', (line) => {
+        for await (const line of rl) {
+            if (streamError)
+                throw streamError;
             if (isFirst) {
                 relHeader = line;
                 isFirst = false;
-                return;
+                continue;
             }
             if (!line.trim())
-                return;
+                continue;
             const match = line.match(/"([^"]*)","([^"]*)"/);
             if (!match) {
                 skippedRels++;
-                return;
+                continue;
             }
             const fromLabel = getNodeLabel(match[1]);
             const toLabel = getNodeLabel(match[2]);
             if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
                 skippedRels++;
-                return;
+                continue;
             }
             const pairKey = `${fromLabel}|${toLabel}`;
             let ws = pairWriteStreams.get(pairKey);
             if (!ws) {
                 const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
                 ws = wsFactory(pairCsvPath);
-                // If any per-pair WriteStream errors (disk full, EMFILE, etc.),
-                // tear down everything and reject the Promise. Without this handler,
-                // a stream error while rl is paused waiting for drain would cause
-                // the drain callback to never fire and the Promise to hang forever.
-                ws.on('error', cleanup);
-                ws.write(relHeader + '\n');
+                ws.on('error', markStreamError);
                 pairWriteStreams.set(pairKey, ws);
                 relsByPairMeta.set(pairKey, { csvPath: pairCsvPath, rows: 0 });
+                if (!ws.write(relHeader + '\n')) {
+                    await once(ws, 'drain', { signal: abortOnError.signal });
+                }
+            }
+            if (!ws.write(line + '\n')) {
+                await once(ws, 'drain', { signal: abortOnError.signal });
             }
-            const ok = ws.write(line + '\n');
             relsByPairMeta.get(pairKey).rows++;
             totalValidRels++;
-            // Handle backpressure: pause reading when the write buffer is full,
-            // resume when the stream drains. Prevents unbounded memory growth
-            // on repos with millions of relationships.
-            // Guard with waitingForDrain to ensure only one drain listener is
-            // registered per stream at a time — rl.pause() doesn't stop buffered
-            // line events immediately. Only resume when ALL streams have drained
-            // to avoid writing into still-full streams.
-            if (!ok && !waitingForDrain.has(pairKey)) {
-                waitingForDrain.add(pairKey);
-                rl.pause();
-                ws.once('drain', () => {
-                    waitingForDrain.delete(pairKey);
-                    if (waitingForDrain.size === 0)
-                        rl.resume();
-                });
-            }
-        });
-        rl.on('close', () => {
-            if (!settled) {
-                settled = true;
-                resolve();
-            }
-        });
-        rl.on('error', cleanup);
-    });
+        }
+        if (streamError)
+            throw streamError;
+    }
+    catch (err) {
+        // Tear down everything so no fd is left dangling. If the abort was caused
+        // by a stream error, rethrow that error (more actionable than AbortError).
+        for (const ws of pairWriteStreams.values())
+            ws.destroy();
+        inputStream.destroy();
+        throw streamError ?? err;
+    }
+    finally {
+        // Readline 'close' fires before the underlying fs.ReadStream releases its
+        // fd — on Windows that race caused ENOTEMPTY on the parent dir.
+        // stream/promises.finished is the stdlib "wait until this stream is fully
+        // closed" primitive and handles both success and error paths.
+        await finished(inputStream).catch(() => { });
+    }
     return { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels };
 };
 let db = null;
@@ -332,15 +316,13 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
     }
     // Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
     const { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels } = await splitRelCsvByLabelPair(csvResult.relCsvPath, csvDir, validTables, getNodeLabel);
-    // Close all per-pair write streams before COPY
-    await Promise.all(Array.from(pairWriteStreams.values()).map((ws) => new Promise((resolve, reject) => {
-        const onError = (err) => reject(err);
-        ws.on('error', onError);
-        ws.end(() => {
-            ws.removeListener('error', onError);
-            resolve();
-        });
-    })));
+    // Close all per-pair write streams before COPY. `stream/promises.finished`
+    // resolves on the stream's 'finish' event and rejects on 'error' — replaces
+    // a hand-rolled promisification with the stdlib primitive.
+    await Promise.all(Array.from(pairWriteStreams.values()).map(async (ws) => {
+        ws.end();
+        await finished(ws);
+    }));
     const insertedRels = totalValidRels;
     const warnings = [];
     if (insertedRels > 0) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.2-rc.2",
+  "version": "1.6.2-rc.4",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",