gitnexus 1.6.3-rc.39 → 1.6.3-rc.40

package/dist/cli/analyze.d.ts

@@ -33,5 +33,11 @@ export interface AnalyzeOptions {
      * `allowDuplicateName` option end-to-end.
      */
     allowDuplicateName?: boolean;
+    /**
+     * Override the walker's large-file skip threshold (#991). Value in KB;
+     * clamped downstream to the tree-sitter 32 MB ceiling. Sets
+     * `GITNEXUS_MAX_FILE_SIZE` for the rest of the pipeline.
+     */
+    maxFileSize?: string;
 }
 export declare const analyzeCommand: (inputPath?: string, options?: AnalyzeOptions) => Promise<void>;

package/dist/cli/analyze.js

@@ -15,6 +15,7 @@ import { closeLbug } from '../core/lbug/lbug-adapter.js';
 import { getStoragePaths, getGlobalRegistryPath, RegistryNameCollisionError, } from '../storage/repo-manager.js';
 import { getGitRoot, hasGitDir } from '../storage/git.js';
 import { runFullAnalysis } from '../core/run-analyze.js';
+import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js';
 import fs from 'fs/promises';
 const HEAP_MB = 8192;
 const HEAP_FLAG = `--max-old-space-size=${HEAP_MB}`;
@@ -51,6 +52,9 @@ export const analyzeCommand = async (inputPath, options) => {
     if (options?.verbose) {
         process.env.GITNEXUS_VERBOSE = '1';
     }
+    if (options?.maxFileSize) {
+        process.env.GITNEXUS_MAX_FILE_SIZE = options.maxFileSize;
+    }
     console.log('\n  GitNexus Analyzer\n');
     let repoPath;
     if (inputPath) {
@@ -85,6 +89,10 @@ export const analyzeCommand = async (inputPath, options) => {
     if (process.env.GITNEXUS_NO_GITIGNORE) {
         console.log('  GITNEXUS_NO_GITIGNORE is set — skipping .gitignore (still reading .gitnexusignore)\n');
     }
+    const maxFileSizeBanner = getMaxFileSizeBannerMessage();
+    if (maxFileSizeBanner) {
+        console.log(`${maxFileSizeBanner}\n`);
+    }
     // ── CLI progress bar setup ─────────────────────────────────────────
     const bar = new cliProgress.SingleBar({
         format: '  {bar} {percentage}% | {phase}',

package/dist/cli/index.js

@@ -27,7 +27,10 @@ program
     .option('--allow-duplicate-name', 'Register this repo even if another path already uses the same --name alias. ' +
     'Leaves `-r <name>` ambiguous for the two paths; use -r <path> to disambiguate.')
     .option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)')
-    .addHelpText('after', '\nEnvironment variables:\n  GITNEXUS_NO_GITIGNORE=1  Skip .gitignore parsing (still reads .gitnexusignore)')
+    .option('--max-file-size <kb>', 'Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).')
+    .addHelpText('after', '\nEnvironment variables:\n' +
+    '  GITNEXUS_NO_GITIGNORE=1   Skip .gitignore parsing (still reads .gitnexusignore)\n' +
+    '  GITNEXUS_MAX_FILE_SIZE=N  Override large-file skip threshold (KB). Default 512, max 32768.')
     .action(createLazyAction(() => import('./analyze.js'), 'analyzeCommand'));
 program
     .command('index [path...]')

package/dist/core/ingestion/filesystem-walker.js

@@ -1,17 +1,17 @@
 import { isVerboseIngestionEnabled } from './utils/verbose.js';
+import { DEFAULT_MAX_FILE_SIZE_BYTES, getMaxFileSizeBytes } from './utils/max-file-size.js';
 import fs from 'fs/promises';
 import path from 'path';
 import { glob } from 'glob';
 import { createIgnoreFilter } from '../../config/ignore-service.js';
 const READ_CONCURRENCY = 32;
-/** Skip files larger than 512KB — they're usually generated/vendored and crash tree-sitter */
-const MAX_FILE_SIZE = 512 * 1024;
 /**
  * Phase 1: Scan repository — stat files to get paths + sizes, no content loaded.
  * Memory: ~10MB for 100K files vs ~1GB+ with content.
  */
 export const walkRepositoryPaths = async (repoPath, onProgress) => {
     const ignoreFilter = await createIgnoreFilter(repoPath);
+    const maxFileSizeBytes = getMaxFileSizeBytes();
     const filtered = await glob('**/*', {
         cwd: repoPath,
         nodir: true,
@@ -27,7 +27,7 @@ export const walkRepositoryPaths = async (repoPath, onProgress) => {
         const results = await Promise.allSettled(batch.map(async (relativePath) => {
             const fullPath = path.join(repoPath, relativePath);
             const stat = await fs.stat(fullPath);
-            if (stat.size > MAX_FILE_SIZE) {
+            if (stat.size > maxFileSizeBytes) {
                 skippedLarge++;
                 skippedLargePaths.push(relativePath.replace(/\\/g, '/'));
                 return null;
@@ -46,7 +46,9 @@ export const walkRepositoryPaths = async (repoPath, onProgress) => {
         }
     }
     if (skippedLarge > 0) {
-        console.warn(`  Skipped ${skippedLarge} large files (>${MAX_FILE_SIZE / 1024}KB, likely generated/vendored)`);
+        const isDefault = maxFileSizeBytes === DEFAULT_MAX_FILE_SIZE_BYTES;
+        const suffix = isDefault ? ', likely generated/vendored' : '';
+        console.warn(`  Skipped ${skippedLarge} large files (>${maxFileSizeBytes / 1024}KB${suffix})`);
         if (isVerboseIngestionEnabled()) {
             for (const p of skippedLargePaths) {
                 console.warn(`  - ${p}`);

package/dist/core/ingestion/utils/max-file-size.d.ts

@@ -0,0 +1,20 @@
+/** Default threshold (512 KB). Files larger than this are skipped by the walker. */
+export declare const DEFAULT_MAX_FILE_SIZE_BYTES: number;
+/** Hard upper bound — tree-sitter refuses buffers above this regardless. */
+export declare const MAX_FILE_SIZE_UPPER_BOUND_BYTES: number;
+/**
+ * Resolve the effective file-size skip threshold (bytes) for the walker.
+ * Reads `GITNEXUS_MAX_FILE_SIZE` (KB). Invalid values fall back to the default
+ * and emit a one-time warning. Values above the tree-sitter ceiling are clamped.
+ */
+export declare const getMaxFileSizeBytes: () => number;
+/**
+ * Build the CLI banner message announcing an active file-size override.
+ * Returns `null` when the effective threshold equals the default — the caller
+ * should print nothing in that case. The returned message reflects the
+ * *effective* post-clamp threshold, not the raw env value, so operators reading
+ * startup output see the actual configuration the walker will use.
+ */
+export declare const getMaxFileSizeBannerMessage: () => string | null;
+/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */
+export declare const _resetMaxFileSizeWarnings: () => void;

package/dist/core/ingestion/utils/max-file-size.js

@@ -0,0 +1,52 @@
+import { TREE_SITTER_MAX_BUFFER } from '../constants.js';
+/** Default threshold (512 KB). Files larger than this are skipped by the walker. */
+export const DEFAULT_MAX_FILE_SIZE_BYTES = 512 * 1024;
+/** Hard upper bound — tree-sitter refuses buffers above this regardless. */
+export const MAX_FILE_SIZE_UPPER_BOUND_BYTES = TREE_SITTER_MAX_BUFFER;
+const warned = new Set();
+const warnOnce = (key, message) => {
+    if (warned.has(key))
+        return;
+    warned.add(key);
+    console.warn(message);
+};
+/**
+ * Resolve the effective file-size skip threshold (bytes) for the walker.
+ * Reads `GITNEXUS_MAX_FILE_SIZE` (KB). Invalid values fall back to the default
+ * and emit a one-time warning. Values above the tree-sitter ceiling are clamped.
+ */
+export const getMaxFileSizeBytes = () => {
+    const raw = process.env.GITNEXUS_MAX_FILE_SIZE;
+    if (!raw)
+        return DEFAULT_MAX_FILE_SIZE_BYTES;
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
+        warnOnce(`invalid:${raw}`, `  GITNEXUS_MAX_FILE_SIZE must be a positive integer (KB), got "${raw}" — using default ${DEFAULT_MAX_FILE_SIZE_BYTES / 1024}KB`);
+        return DEFAULT_MAX_FILE_SIZE_BYTES;
+    }
+    const bytes = parsed * 1024;
+    if (bytes > MAX_FILE_SIZE_UPPER_BOUND_BYTES) {
+        warnOnce(`clamp:${raw}`, `  GITNEXUS_MAX_FILE_SIZE=${parsed}KB exceeds tree-sitter ceiling (${MAX_FILE_SIZE_UPPER_BOUND_BYTES / 1024}KB) — clamping`);
+        return MAX_FILE_SIZE_UPPER_BOUND_BYTES;
+    }
+    return bytes;
+};
+/**
+ * Build the CLI banner message announcing an active file-size override.
+ * Returns `null` when the effective threshold equals the default — the caller
+ * should print nothing in that case. The returned message reflects the
+ * *effective* post-clamp threshold, not the raw env value, so operators reading
+ * startup output see the actual configuration the walker will use.
+ */
+export const getMaxFileSizeBannerMessage = () => {
+    const effectiveBytes = getMaxFileSizeBytes();
+    if (effectiveBytes === DEFAULT_MAX_FILE_SIZE_BYTES)
+        return null;
+    const effectiveKb = effectiveBytes / 1024;
+    const defaultKb = DEFAULT_MAX_FILE_SIZE_BYTES / 1024;
+    return `  GITNEXUS_MAX_FILE_SIZE: effective threshold ${effectiveKb}KB (default ${defaultKb}KB)`;
+};
+/** Test-only: reset the warn-once cache so repeated test runs can re-observe warnings. */
+export const _resetMaxFileSizeWarnings = () => {
+    warned.clear();
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.3-rc.39",
+  "version": "1.6.3-rc.40",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",