gitnexus 1.6.4-rc.42 → 1.6.4-rc.44

package/dist/core/embeddings/embedder.js

@@ -13,7 +13,6 @@ if (!process.env.ORT_LOG_LEVEL) {
     process.env.ORT_LOG_LEVEL = '3';
 }
 import { pipeline, env } from '@huggingface/transformers';
-import os from 'os';
 import { existsSync } from 'fs';
 import { execFileSync } from 'child_process';
 import { join, dirname } from 'path';
@@ -21,6 +20,7 @@ import { createRequire } from 'module';
 import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
 import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
 import { resolveEmbeddingConfig } from './config.js';
+import { applyHfEnvOverrides } from './hf-env.js';
 /**
  * Check whether the onnxruntime-node package that @huggingface/transformers
  * will actually load at runtime ships the CUDA execution provider.
@@ -132,11 +132,11 @@ export const initEmbedder = async (onProgress, config = {}, forceDevice) => {
         try {
             // Configure transformers.js environment
             env.allowLocalModels = false;
-            // Default cache to user-writable location. transformers.js defaults to
-            // ./node_modules/.cache inside its own install dir, which is unwritable
-            // when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
-            // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
-            env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+            // Bridge user-controlled env vars to transformers.js: HF_HOME →
+            // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
+            // applyHfEnvOverrides so the MCP embedder entry point behaves
+            // identically.
+            applyHfEnvOverrides(env);
             const isDev = process.env.NODE_ENV === 'development';
             if (isDev) {
                 console.log(`🧠 Loading embedding model: ${finalConfig.modelId}`);

package/dist/core/embeddings/hf-env.d.ts

@@ -0,0 +1,46 @@
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Minimal subset of `@huggingface/transformers`' `env` object that gitnexus
+ * mutates. Defining a local structural type keeps this helper free of a
+ * transitive dependency on transformers' generated `.d.ts` while still
+ * giving full type-checking on the two fields we actually touch.
+ */
+export interface HfEnvSubset {
+    cacheDir: string;
+    remoteHost: string;
+}
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Apply user-controlled HuggingFace environment overrides to the
+ * `@huggingface/transformers` `env` object. Centralises the two env-var
+ * bridges so every gitnexus embedder entry point (the analyze pipeline
+ * and the MCP server) behaves identically.
+ *
+ * - **`HF_HOME`** → `env.cacheDir` (default: `~/.cache/huggingface`).
+ *   transformers.js otherwise defaults to `./node_modules/.cache` inside
+ *   its own install dir, which is unwritable when gitnexus is installed
+ *   globally (e.g. `/usr/lib/node_modules/`).
+ *
+ * - **`HF_ENDPOINT`** → `env.remoteHost` (#1205). transformers.js does
+ *   not read `HF_ENDPOINT` on its own — it reads `env.remoteHost` —
+ *   even though `HF_ENDPOINT` is the standard env var the upstream
+ *   `huggingface_hub` Python client and the official HF mirror docs
+ *   tell users to set. Bridging the two unblocks `--embeddings` for
+ *   users behind networks where `huggingface.co` is unreachable
+ *   (corporate proxies, the GFW, air-gapped mirrors). The trailing
+ *   slash is normalised because transformers.js builds URLs by string
+ *   concatenation and a missing slash silently falls through to its
+ *   default `huggingface.co/...` host.
+ *
+ * Mutation rather than return-and-apply because callers already hold a
+ * reference to the live `env` object imported from
+ * `@huggingface/transformers` — passing the same reference in keeps the
+ * call site a single line at each entry point.
+ */
+export declare function applyHfEnvOverrides(env: HfEnvSubset): void;

package/dist/core/embeddings/hf-env.js

@@ -0,0 +1,46 @@
+import os from 'node:os';
+import { join } from 'node:path';
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Apply user-controlled HuggingFace environment overrides to the
+ * `@huggingface/transformers` `env` object. Centralises the two env-var
+ * bridges so every gitnexus embedder entry point (the analyze pipeline
+ * and the MCP server) behaves identically.
+ *
+ * - **`HF_HOME`** → `env.cacheDir` (default: `~/.cache/huggingface`).
+ *   transformers.js otherwise defaults to `./node_modules/.cache` inside
+ *   its own install dir, which is unwritable when gitnexus is installed
+ *   globally (e.g. `/usr/lib/node_modules/`).
+ *
+ * - **`HF_ENDPOINT`** → `env.remoteHost` (#1205). transformers.js does
+ *   not read `HF_ENDPOINT` on its own — it reads `env.remoteHost` —
+ *   even though `HF_ENDPOINT` is the standard env var the upstream
+ *   `huggingface_hub` Python client and the official HF mirror docs
+ *   tell users to set. Bridging the two unblocks `--embeddings` for
+ *   users behind networks where `huggingface.co` is unreachable
+ *   (corporate proxies, the GFW, air-gapped mirrors). The trailing
+ *   slash is normalised because transformers.js builds URLs by string
+ *   concatenation and a missing slash silently falls through to its
+ *   default `huggingface.co/...` host.
+ *
+ * Mutation rather than return-and-apply because callers already hold a
+ * reference to the live `env` object imported from
+ * `@huggingface/transformers` — passing the same reference in keeps the
+ * call site a single line at each entry point.
+ */
+export function applyHfEnvOverrides(env) {
+    env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+    // `.trim()` guards against the common copy-paste failure mode of
+    // `HF_ENDPOINT="  https://hf-mirror.com  "` (leading/trailing whitespace
+    // from shell scripts or docs) — without it, a whitespace-only value
+    // would be truthy and produce an invalid `env.remoteHost = '   /'` that
+    // silently misroutes downloads. Empty string remains falsy in JS so the
+    // truthy guard already handles the unset/empty cases.
+    const endpoint = process.env.HF_ENDPOINT?.trim();
+    if (endpoint) {
+        env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/';
+    }
+}

package/dist/core/group/config-parser.js

@@ -9,6 +9,7 @@ const DEFAULT_DETECT = {
     topics: true,
     shared_libs: true,
     embedding_fallback: true,
+    workspace_deps: true,
 };
 const DEFAULT_MATCHING = {
     bm25_threshold: 0.7,

package/dist/core/group/extractors/rust-workspace-extractor.d.ts

@@ -0,0 +1,30 @@
+import type { CypherExecutor } from '../contract-extractor.js';
+import type { GroupManifestLink } from '../types.js';
+/**
+ * Discover cross-crate contracts in a Rust workspace by reading each
+ * member's `Cargo.toml` dependencies and scanning source files for
+ * `use <workspace_dep>::<Type>` imports.
+ *
+ * Emits `GroupManifestLink[]` with `type: 'custom'` that feed into the
+ * existing ManifestExtractor pipeline — no new matching logic needed.
+ *
+ * Designed for the group-level sync pipeline: it receives all repos in
+ * a group and produces cross-repo links between them.
+ */
+interface CrateMeta {
+    name: string;
+    groupPath: string;
+    repoPath: string;
+    workspaceDeps: string[];
+}
+export interface RustWorkspaceResult {
+    links: GroupManifestLink[];
+    discoveredCrates: Map<string, CrateMeta>;
+}
+/**
+ * Discover cross-crate contracts across all Rust repos in a group.
+ *
+ * Returns `GroupManifestLink[]` ready to feed into `ManifestExtractor`.
+ */
+export declare function extractRustWorkspaceLinks(repos: Record<string, string>, repoPaths: Map<string, string>, _dbExecutors?: Map<string, CypherExecutor>): Promise<RustWorkspaceResult>;
+export {};

package/dist/core/group/extractors/rust-workspace-extractor.js

@@ -0,0 +1,214 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { shouldIgnorePath } from '../../../config/ignore-service.js';
+import { loadIgnoreRules } from '../../../config/ignore-service.js';
+/**
+ * Parse a Cargo.toml to extract the crate name and workspace dependency
+ * names. Uses simple line-based parsing — no TOML library needed for
+ * the subset we care about.
+ */
+async function parseCrateManifest(repoPath) {
+    const cargoPath = path.join(repoPath, 'Cargo.toml');
+    let content;
+    try {
+        content = await fs.readFile(cargoPath, 'utf-8');
+    }
+    catch {
+        return null;
+    }
+    let name = '';
+    const workspaceDeps = [];
+    const nameMatch = content.match(/^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m);
+    if (nameMatch)
+        name = nameMatch[1];
+    // Match dependencies that use workspace = true, which indicates they
+    // are workspace-internal deps:
+    //   dep_name = { workspace = true }
+    //   dep_name.workspace = true
+    //
+    // Also match plain path dependencies:
+    //   dep_name = { path = "../other" }
+    const depSections = content.matchAll(/\[(dependencies|dev-dependencies|build-dependencies)\]\s*\n([\s\S]*?)(?=\n\[|$)/g);
+    for (const section of depSections) {
+        const sectionBody = section[2];
+        // workspace = true style
+        const wsMatches = sectionBody.matchAll(/^(\w[\w-]*)\s*=\s*\{[^}]*workspace\s*=\s*true[^}]*\}/gm);
+        for (const m of wsMatches)
+            workspaceDeps.push(m[1]);
+        // dotted workspace style: dep_name.workspace = true
+        const dottedMatches = sectionBody.matchAll(/^(\w[\w-]*)\.workspace\s*=\s*true/gm);
+        for (const m of dottedMatches)
+            workspaceDeps.push(m[1]);
+        // path = "../other" style (local path deps within workspace)
+        const pathMatches = sectionBody.matchAll(/^(\w[\w-]*)\s*=\s*\{[^}]*path\s*=\s*"[^"]*"[^}]*\}/gm);
+        for (const m of pathMatches)
+            workspaceDeps.push(m[1]);
+    }
+    if (!name)
+        return null;
+    return { name, workspaceDeps: [...new Set(workspaceDeps)] };
+}
+/**
+ * Scan Rust source files for `use <crate>::<path>::<Symbol>` patterns
+ * where <crate> is a known workspace dependency.
+ */
+async function scanImports(repoPath, knownCrates) {
+    const results = [];
+    const normalizedCrates = new Map();
+    for (const c of knownCrates) {
+        normalizedCrates.set(c.replace(/-/g, '_'), c);
+    }
+    const sourceFiles = await findRustFiles(repoPath);
+    for (const relFile of sourceFiles) {
+        const absPath = path.join(repoPath, relFile);
+        let content;
+        try {
+            content = await fs.readFile(absPath, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        // Match patterns:
+        //   use crate_name::Type;
+        //   use crate_name::module::Type;
+        //   use crate_name::{Type1, Type2};
+        //   use crate_name::module::{Type1, Type2};
+        const useRegex = /^use\s+(\w+)::(.+);/gm;
+        let match;
+        while ((match = useRegex.exec(content)) !== null) {
+            const crateName = match[1];
+            const originalCrateName = normalizedCrates.get(crateName);
+            if (!originalCrateName)
+                continue;
+            const importPath = match[2].trim();
+            // Handle grouped imports: {Type1, Type2, module::Type3}
+            const braceMatch = importPath.match(/\{([^}]+)\}/);
+            if (braceMatch) {
+                const items = braceMatch[1].split(',').map((s) => s.trim());
+                for (const item of items) {
+                    const symbolName = extractSymbolName(item);
+                    if (symbolName && isTypeName(symbolName)) {
+                        results.push({ crateName: originalCrateName, symbolName, filePath: relFile });
+                    }
+                }
+            }
+            else {
+                const symbolName = extractSymbolName(importPath);
+                if (symbolName && isTypeName(symbolName)) {
+                    results.push({ crateName: originalCrateName, symbolName, filePath: relFile });
+                }
+            }
+        }
+    }
+    return results;
+}
+/** Extract the final symbol name from a path like `module::submod::TypeName`. */
+function extractSymbolName(importPath) {
+    const trimmed = importPath.trim();
+    if (!trimmed || trimmed === '*' || trimmed === 'self')
+        return null;
+    const parts = trimmed.split('::');
+    return parts[parts.length - 1].trim() || null;
+}
+/**
+ * Heuristic: in Rust, types (structs, enums, traits) are PascalCase.
+ * Functions and modules are snake_case. We only want types as cross-crate
+ * contracts — functions are too granular and modules too broad.
+ */
+function isTypeName(name) {
+    return /^[A-Z][A-Za-z0-9]*$/.test(name);
+}
+async function findRustFiles(repoPath) {
+    const results = [];
+    const ig = await loadIgnoreRules(repoPath);
+    async function walk(dir, rel) {
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+            if (entry.isDirectory()) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel + '/'))
+                    continue;
+                await walk(path.join(dir, entry.name), childRel);
+            }
+            else if (entry.name.endsWith('.rs')) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel))
+                    continue;
+                results.push(childRel);
+            }
+        }
+    }
+    await walk(repoPath, '');
+    return results;
+}
+/**
+ * Discover cross-crate contracts across all Rust repos in a group.
+ *
+ * Returns `GroupManifestLink[]` ready to feed into `ManifestExtractor`.
+ */
+export async function extractRustWorkspaceLinks(repos, repoPaths, _dbExecutors) {
+    // Phase 1: Parse all Cargo.toml files to build crate registry
+    const cratesByName = new Map();
+    const cratesByGroupPath = new Map();
+    for (const [groupPath] of Object.entries(repos)) {
+        const repoPath = repoPaths.get(groupPath);
+        if (!repoPath)
+            continue;
+        const manifest = await parseCrateManifest(repoPath);
+        if (!manifest)
+            continue;
+        const meta = {
+            name: manifest.name,
+            groupPath,
+            repoPath,
+            workspaceDeps: manifest.workspaceDeps,
+        };
+        const existing = cratesByName.get(manifest.name);
+        if (existing) {
+            console.warn(`[rust-workspace-extractor] duplicate crate name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`);
+            continue;
+        }
+        cratesByName.set(manifest.name, meta);
+        cratesByGroupPath.set(groupPath, meta);
+    }
+    // Phase 2: For each crate, identify which of its workspace deps are
+    // also in this group (i.e., repos we can link to)
+    const links = [];
+    const seen = new Set();
+    for (const [, crate] of cratesByGroupPath) {
+        const groupCrateDeps = crate.workspaceDeps.filter((d) => cratesByName.has(d));
+        if (groupCrateDeps.length === 0)
+            continue;
+        // Phase 3: Scan source files for imports from workspace deps
+        const knownCrates = new Set(groupCrateDeps);
+        const imports = await scanImports(crate.repoPath, knownCrates);
+        for (const imp of imports) {
+            const providerCrate = cratesByName.get(imp.crateName);
+            if (!providerCrate)
+                continue;
+            const qualifiedContract = `${imp.crateName}::${imp.symbolName}`;
+            const key = `${crate.groupPath}→${providerCrate.groupPath}::${qualifiedContract}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            const link = {
+                from: providerCrate.groupPath,
+                to: crate.groupPath,
+                type: 'custom',
+                contract: qualifiedContract,
+                role: 'provider',
+            };
+            links.push(link);
+        }
+    }
+    return { links, discoveredCrates: cratesByGroupPath };
+}

package/dist/core/group/sync.js

@@ -7,6 +7,7 @@ import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
 import { GrpcExtractor } from './extractors/grpc-extractor.js';
 import { TopicExtractor } from './extractors/topic-extractor.js';
 import { ManifestExtractor } from './extractors/manifest-extractor.js';
+import { extractRustWorkspaceLinks } from './extractors/rust-workspace-extractor.js';
 import { runExactMatch } from './matching.js';
 import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
 import { writeContractRegistry } from './storage.js';
@@ -58,12 +59,14 @@ export async function syncGroup(config, opts) {
     let autoContracts = [];
     let manifestCrossLinks = [];
     let dbExecutors;
+    let registryEntries;
     const eo = opts?.extractorOverride;
     if (eo && eo.length === 0) {
         autoContracts = await eo();
     }
     else {
-        const entries = await readRegistry();
+        registryEntries = await readRegistry();
+        const entries = registryEntries;
         const resolve = opts?.resolveRepoHandle ?? defaultResolveHandle(entries);
         const httpEx = new HttpRouteExtractor();
         const grpcEx = new GrpcExtractor();
@@ -143,29 +146,47 @@ export async function syncGroup(config, opts) {
             }
         }
     }
-    // Process manifest links declared in group.yaml.
+    // Auto-discover workspace dependency contracts (Rust Cargo workspaces, etc.)
+    // and merge them with explicit manifest links. Discovered links use the same
+    // ManifestExtractor pipeline as hand-written links in group.yaml.
+    let allLinks = [...config.links];
+    if (config.detect.workspace_deps) {
+        const repoPaths = new Map();
+        if (!registryEntries)
+            registryEntries = await readRegistry();
+        for (const [groupPath, regName] of Object.entries(config.repos)) {
+            const e = registryEntries.find((en) => en.name === regName);
+            if (e)
+                repoPaths.set(groupPath, e.path);
+        }
+        const wsResult = await extractRustWorkspaceLinks(config.repos, repoPaths, dbExecutors);
+        if (wsResult.links.length > 0) {
+            allLinks = [...allLinks, ...wsResult.links];
+            if (opts?.verbose) {
+                console.log(`  workspace-deps: discovered ${wsResult.links.length} cross-crate links from ${wsResult.discoveredCrates.size} Rust crates`);
+            }
+        }
+    }
+    // Process manifest links declared in group.yaml (plus any auto-discovered).
     // ManifestExtractor is fully implemented but was never wired into this
     // pipeline — config.links were parsed and validated but silently dropped.
     // Placed after the DB try/finally: resolveSymbol falls back to synthetic
     // UIDs when dbExecutors is undefined or a pool is closed, so cross-links
     // are always generated regardless of whether real DB executors are available.
-    if (config.links.length > 0) {
-        // Warn about dangling links that reference repos not declared in config.repos.
-        // They still generate cross-links via synthetic UIDs (determinism is preserved),
-        // but the operator probably meant something that now silently does nothing useful.
+    if (allLinks.length > 0) {
         const knownRepos = new Set(Object.keys(config.repos));
-        for (const link of config.links) {
+        for (const link of allLinks) {
             const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
             if (dangling.length > 0) {
                 console.warn(`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`);
             }
         }
         const manifestEx = new ManifestExtractor();
-        const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors);
+        const manifestResult = await manifestEx.extractFromManifest(allLinks, dbExecutors);
         autoContracts.push(...manifestResult.contracts);
         manifestCrossLinks = manifestResult.crossLinks;
         if (opts?.verbose) {
-            console.log(`  manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`);
+            console.log(`  manifest: ${manifestCrossLinks.length} cross-links from ${allLinks.length} links (${config.links.length} declared + ${allLinks.length - config.links.length} discovered)`);
         }
     }
     const { matched, unmatched } = runExactMatch(autoContracts, undefined, config.matching);

package/dist/core/group/types.d.ts

@@ -24,6 +24,7 @@ export interface DetectConfig {
     topics: boolean;
     shared_libs: boolean;
     embedding_fallback: boolean;
+    workspace_deps: boolean;
 }
 export interface MatchingConfig {
     bm25_threshold: number;

package/dist/mcp/core/embedder.js

@@ -5,10 +5,9 @@
  * For MCP, we only need to compute query embeddings, not batch embed.
  */
 import { pipeline, env } from '@huggingface/transformers';
-import os from 'os';
-import { join } from 'path';
 import { isHttpMode, getHttpDimensions, httpEmbedQuery, } from '../../core/embeddings/http-client.js';
 import { resolveEmbeddingConfig } from '../../core/embeddings/config.js';
+import { applyHfEnvOverrides } from '../../core/embeddings/hf-env.js';
 import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js';
 // Model config
 const MODEL_ID = 'Snowflake/snowflake-arctic-embed-xs';
@@ -33,11 +32,11 @@ export const initEmbedder = async () => {
     initPromise = (async () => {
         try {
             env.allowLocalModels = false;
-            // Default cache to user-writable location. transformers.js defaults to
-            // ./node_modules/.cache inside its own install dir, which is unwritable
-            // when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
-            // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
-            env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+            // Bridge user-controlled env vars to transformers.js: HF_HOME →
+            // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
+            // applyHfEnvOverrides so this MCP entry point behaves identically to
+            // the analyze pipeline embedder.
+            applyHfEnvOverrides(env);
             const embeddingConfig = resolveEmbeddingConfig();
             console.error('GitNexus: Loading embedding model (first search may take a moment)...');
             const devicesToTry = embeddingConfig.device === 'dml' || embeddingConfig.device === 'cuda'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.4-rc.42",
+  "version": "1.6.4-rc.44",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",