gitnexus 1.6.6-rc.150 → 1.6.6-rc.151

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/README.md +1 -1
  2. package/dist/_shared/scope-resolution/parsed-file.d.ts +24 -0
  3. package/dist/_shared/scope-resolution/parsed-file.d.ts.map +1 -1
  4. package/dist/cli/analyze.d.ts +10 -1
  5. package/dist/cli/analyze.js +48 -15
  6. package/dist/cli/i18n/en.d.ts +1 -1
  7. package/dist/cli/i18n/en.js +1 -1
  8. package/dist/cli/i18n/resources.d.ts +1 -1
  9. package/dist/cli/i18n/zh-CN.js +1 -1
  10. package/dist/cli/index.js +1 -1
  11. package/dist/core/ingestion/call-processor.d.ts +5 -0
  12. package/dist/core/ingestion/call-processor.js +30 -29
  13. package/dist/core/ingestion/finalize-orchestrator.js +9 -1
  14. package/dist/core/ingestion/language-provider.d.ts +20 -0
  15. package/dist/core/ingestion/languages/c/capture-side-channel.d.ts +62 -0
  16. package/dist/core/ingestion/languages/c/capture-side-channel.js +65 -0
  17. package/dist/core/ingestion/languages/c/import-target.js +52 -9
  18. package/dist/core/ingestion/languages/c/index.d.ts +2 -1
  19. package/dist/core/ingestion/languages/c/index.js +2 -1
  20. package/dist/core/ingestion/languages/c/scope-resolver.js +47 -4
  21. package/dist/core/ingestion/languages/c/static-linkage.d.ts +8 -0
  22. package/dist/core/ingestion/languages/c/static-linkage.js +46 -1
  23. package/dist/core/ingestion/languages/c-cpp.js +16 -2
  24. package/dist/core/ingestion/languages/cpp/adl.d.ts +34 -0
  25. package/dist/core/ingestion/languages/cpp/adl.js +105 -2
  26. package/dist/core/ingestion/languages/cpp/capture-side-channel.d.ts +65 -0
  27. package/dist/core/ingestion/languages/cpp/capture-side-channel.js +78 -0
  28. package/dist/core/ingestion/languages/cpp/file-local-linkage.d.ts +18 -52
  29. package/dist/core/ingestion/languages/cpp/file-local-linkage.js +49 -1
  30. package/dist/core/ingestion/languages/cpp/index.d.ts +1 -0
  31. package/dist/core/ingestion/languages/cpp/index.js +1 -0
  32. package/dist/core/ingestion/languages/cpp/inline-namespaces.d.ts +8 -0
  33. package/dist/core/ingestion/languages/cpp/inline-namespaces.js +21 -0
  34. package/dist/core/ingestion/languages/cpp/scope-resolver.js +46 -4
  35. package/dist/core/ingestion/languages/cpp/two-phase-lookup.d.ts +19 -0
  36. package/dist/core/ingestion/languages/cpp/two-phase-lookup.js +32 -0
  37. package/dist/core/ingestion/languages/csharp/captures.js +5 -4
  38. package/dist/core/ingestion/languages/csharp/namespace-siblings.d.ts +3 -2
  39. package/dist/core/ingestion/languages/kotlin/capture-side-channel.d.ts +57 -0
  40. package/dist/core/ingestion/languages/kotlin/capture-side-channel.js +60 -0
  41. package/dist/core/ingestion/languages/kotlin/companion-scopes.d.ts +7 -0
  42. package/dist/core/ingestion/languages/kotlin/companion-scopes.js +10 -0
  43. package/dist/core/ingestion/languages/kotlin/index.d.ts +1 -0
  44. package/dist/core/ingestion/languages/kotlin/index.js +1 -0
  45. package/dist/core/ingestion/languages/kotlin/scope-resolver.js +17 -0
  46. package/dist/core/ingestion/languages/kotlin.js +8 -1
  47. package/dist/core/ingestion/languages/typescript/captures.js +5 -4
  48. package/dist/core/ingestion/parsing-processor.d.ts +28 -24
  49. package/dist/core/ingestion/parsing-processor.js +76 -754
  50. package/dist/core/ingestion/pipeline-phases/parse-impl.d.ts +16 -22
  51. package/dist/core/ingestion/pipeline-phases/parse-impl.js +360 -304
  52. package/dist/core/ingestion/pipeline-phases/parse.d.ts +9 -30
  53. package/dist/core/ingestion/pipeline-phases/parse.js +2 -2
  54. package/dist/core/ingestion/pipeline.d.ts +13 -18
  55. package/dist/core/ingestion/scope-resolution/contract/scope-resolver.d.ts +38 -0
  56. package/dist/core/ingestion/scope-resolution/passes/free-call-fallback.d.ts +20 -1
  57. package/dist/core/ingestion/scope-resolution/passes/free-call-fallback.js +60 -22
  58. package/dist/core/ingestion/scope-resolution/pipeline/phase.js +114 -17
  59. package/dist/core/ingestion/scope-resolution/pipeline/run.d.ts +28 -5
  60. package/dist/core/ingestion/scope-resolution/pipeline/run.js +67 -10
  61. package/dist/core/ingestion/scope-resolution/scope/walkers.js +10 -21
  62. package/dist/core/ingestion/scope-resolution/workspace-index.d.ts +18 -2
  63. package/dist/core/ingestion/scope-resolution/workspace-index.js +94 -7
  64. package/dist/core/ingestion/utils/heap-probe.d.ts +20 -0
  65. package/dist/core/ingestion/utils/heap-probe.js +39 -0
  66. package/dist/core/ingestion/workers/parse-worker.js +113 -27
  67. package/dist/core/ingestion/workers/worker-pool.d.ts +46 -12
  68. package/dist/core/ingestion/workers/worker-pool.js +47 -12
  69. package/dist/core/lbug/csv-generator.js +20 -2
  70. package/dist/core/run-analyze.d.ts +2 -2
  71. package/dist/core/run-analyze.js +11 -2
  72. package/dist/storage/parse-cache.d.ts +23 -1
  73. package/dist/storage/parse-cache.js +138 -36
  74. package/dist/storage/parsedfile-store.d.ts +142 -0
  75. package/dist/storage/parsedfile-store.js +367 -0
  76. package/dist/storage/scope-index-store.d.ts +114 -0
  77. package/dist/storage/scope-index-store.js +236 -0
  78. package/package.json +1 -1
package/README.md CHANGED
@@ -400,7 +400,7 @@ Values above **32768 KB (32 MB)** are clamped to the tree-sitter parser ceiling;
400
400
 
401
401
  ### Analyze reports a worker timeout
402
402
 
403
- Worker parse timeouts are recoverable. GitNexus retries stalled worker jobs with backoff, splits large jobs to isolate slow files, and falls back to the sequential parser when needed. If a large repository needs more time per worker job, use either:
403
+ Worker parse timeouts are recoverable. GitNexus retries stalled worker jobs with backoff, splits large jobs to isolate slow files, and quarantines a file that repeatedly crashes its worker (respawning the slot so the pool keeps going). If a large repository needs more time per worker job, use either:
404
404
 
405
405
  ```bash
406
406
  # CLI flag, in seconds
@@ -72,5 +72,29 @@ export interface ParsedFile {
72
72
  */
73
73
  readonly localDefs: readonly SymbolDefinition[];
74
74
  readonly referenceSites: readonly ReferenceSite[];
75
+ /**
76
+ * Opaque, language-private serialization of capture-time side-channel
77
+ * state that a provider's `emitScopeCaptures` populates into module-level
78
+ * maps as a SIDE EFFECT (not onto the scopes/defs of this `ParsedFile`).
79
+ *
80
+ * Such state is computed inside the parse worker (where `emitScopeCaptures`
81
+ * runs) and would otherwise be lost across the worker→main MessageChannel
82
+ * and the disk store, because scope-resolution reuses the serialized
83
+ * `ParsedFile` and SKIPS re-extraction on the main thread (#1983 — the
84
+ * whole point is to avoid a main-thread tree-sitter re-parse). Carrying the
85
+ * data here lets the main thread repopulate those maps WITHOUT re-parsing.
86
+ *
87
+ * Shared / ingestion code treats this as opaque (`unknown`) per AGENTS.md
88
+ * (no language names in shared code). The producing language fills it via
89
+ * the `LanguageProvider.collectCaptureSideChannel` hook (worker side) and
90
+ * consumes it via the `ScopeResolver.applyCaptureSideChannel` hook
91
+ * (main-thread resolution side). It MUST be plain JSON-serializable data
92
+ * (objects / arrays / primitives) so it round-trips through the disk-backed
93
+ * `parsedfile-store` (JSON.stringify + interning reviver).
94
+ *
95
+ * Optional: providers whose `emitScopeCaptures` is pure (no module-level
96
+ * side effects — the contract default) leave this undefined.
97
+ */
98
+ readonly captureSideChannel?: unknown;
75
99
  }
76
100
  //# sourceMappingURL=parsed-file.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"parsed-file.d.ts","sourceRoot":"","sources":["../../src/scope-resolution/parsed-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,QAAQ,CAAC,aAAa,EAAE,SAAS,YAAY,EAAE,CAAC;IAChD;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAChD,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;CACnD"}
1
+ {"version":3,"file":"parsed-file.d.ts","sourceRoot":"","sources":["../../src/scope-resolution/parsed-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,QAAQ,CAAC,aAAa,EAAE,SAAS,YAAY,EAAE,CAAC;IAChD;;;;OAIG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAChD,QAAQ,CAAC,cAAc,EAAE,SAAS,aAAa,EAAE,CAAC;IAClD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAC;CACvC"}
@@ -7,6 +7,15 @@
7
7
  * This CLI wrapper handles: heap management, progress bar, SIGINT,
8
8
  * skill generation (--skills), summary output, and process.exit().
9
9
  */
10
+ /**
11
+ * RAM-aware re-exec heap cap (MB): `0.75 × effective RAM`, clamped to
12
+ * `>= DEFAULT_HEAP_MB`. Kept BELOW physical RAM on purpose — a cap `>=` RAM makes
13
+ * V8 collect lazily and inflate the heap into swap-thrash (observed analyzing the
14
+ * Linux kernel at a 30GB cap on a 31GB box). `constrainedBytes` is the cgroup
15
+ * limit or `null`; it is honored only as a real, smaller-than-physical cap, because
16
+ * `process.constrainedMemory()` returns a huge sentinel when UNCONSTRAINED.
17
+ */
18
+ export declare function computeHeapCapMb(totalBytes: number, constrainedBytes: number | null): number;
10
19
  export interface AnalyzeOptions {
11
20
  force?: boolean;
12
21
  repairFts?: boolean;
@@ -78,7 +87,7 @@ export interface AnalyzeOptions {
78
87
  workerTimeout?: string;
79
88
  /** Control LadybugDB WAL auto-checkpoint threshold during analyze. */
80
89
  walCheckpointThreshold?: string;
81
- /** Parse worker pool size; 0 disables workers (sequential fallback). */
90
+ /** Parse worker pool size (>=1); 0 is rejected (no sequential mode). */
82
91
  workers?: string;
83
92
  embeddingThreads?: string;
84
93
  embeddingBatchSize?: string;
@@ -8,6 +8,7 @@
8
8
  * skill generation (--skills), summary output, and process.exit().
9
9
  */
10
10
  import path from 'path';
11
+ import os from 'os';
11
12
  import { spawn } from 'child_process';
12
13
  import v8 from 'v8';
13
14
  import cliProgress from 'cli-progress';
@@ -63,12 +64,41 @@ const installFatalHandlers = () => {
63
64
  process.exit(1);
64
65
  });
65
66
  };
66
- const HEAP_MB = 16384;
67
+ /** Historical floor for the re-exec heap cap — the auto-sizer never goes below
68
+ * this, so small boxes / CI never regress. */
69
+ const DEFAULT_HEAP_MB = 16384;
70
+ /**
71
+ * RAM-aware re-exec heap cap (MB): `0.75 × effective RAM`, clamped to
72
+ * `>= DEFAULT_HEAP_MB`. Kept BELOW physical RAM on purpose — a cap `>=` RAM makes
73
+ * V8 collect lazily and inflate the heap into swap-thrash (observed analyzing the
74
+ * Linux kernel at a 30GB cap on a 31GB box). `constrainedBytes` is the cgroup
75
+ * limit or `null`; it is honored only as a real, smaller-than-physical cap, because
76
+ * `process.constrainedMemory()` returns a huge sentinel when UNCONSTRAINED.
77
+ */
78
+ export function computeHeapCapMb(totalBytes, constrainedBytes) {
79
+ const effectiveBytes = constrainedBytes !== null && constrainedBytes > 0 && constrainedBytes < totalBytes
80
+ ? constrainedBytes
81
+ : totalBytes;
82
+ const effectiveMb = Math.floor(effectiveBytes / (1024 * 1024));
83
+ return Math.max(DEFAULT_HEAP_MB, Math.floor(0.75 * effectiveMb));
84
+ }
85
+ function readConstrainedBytes() {
86
+ if (typeof process.constrainedMemory !== 'function')
87
+ return null;
88
+ const c = process.constrainedMemory();
89
+ return typeof c === 'number' && c > 0 ? c : null;
90
+ }
91
+ const HEAP_MB = computeHeapCapMb(os.totalmem(), readConstrainedBytes());
67
92
  const TEST_RESPAWN_HEAP_MB = Number(process.env.GITNEXUS_TEST_RESPAWN_HEAP_MB);
68
93
  const RESPAWN_HEAP_MB = Number.isFinite(TEST_RESPAWN_HEAP_MB) && TEST_RESPAWN_HEAP_MB > 0
69
94
  ? Math.floor(TEST_RESPAWN_HEAP_MB)
70
95
  : HEAP_MB;
71
96
  const HEAP_FLAG = `--max-old-space-size=${RESPAWN_HEAP_MB}`;
97
+ /** Larger semi-space (young-gen) cuts minor-GC frequency + promotion churn during
98
+ * the multi-million-node graph build/emit. Allowed in NODE_OPTIONS (unlike
99
+ * --stack-size), so it propagates to the re-exec env cleanly. */
100
+ const SEMI_SPACE_MB = 128;
101
+ const SEMI_FLAG = `--max-semi-space-size=${SEMI_SPACE_MB}`;
72
102
  /** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */
73
103
  const STACK_KB = 4096;
74
104
  const STACK_FLAG = `--stack-size=${STACK_KB}`;
@@ -335,7 +365,8 @@ const forceHeapOOMForTestIfEnabled = () => {
335
365
  // the `DEFAULT_WAL_CHECKPOINT_THRESHOLD` constant in
336
366
  // `gitnexus/src/core/lbug/lbug-config.ts` in sync with this value.
337
367
  const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024;
338
- /** Re-exec the process with a 16GB heap and larger stack if we're currently below that. */
368
+ /** Re-exec the process with the RAM-aware auto heap cap + larger semi-space/stack
369
+ * if we're currently below that. A user-supplied NODE_OPTIONS heap wins (no re-exec). */
339
370
  async function ensureHeap() {
340
371
  const nodeOpts = process.env.NODE_OPTIONS || '';
341
372
  if (nodeOpts.includes('--max-old-space-size'))
@@ -343,32 +374,32 @@ async function ensureHeap() {
343
374
  const v8Heap = v8.getHeapStatistics().heap_size_limit;
344
375
  if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9)
345
376
  return false;
346
- // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+,
347
- // so pass it only as a direct CLI argument, not via the environment.
348
- const cliFlags = [HEAP_FLAG];
377
+ // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, so pass it
378
+ // only as a direct CLI argument. --max-semi-space-size IS allowed in NODE_OPTIONS.
379
+ const cliFlags = [HEAP_FLAG, SEMI_FLAG];
349
380
  if (!nodeOpts.includes('--stack-size'))
350
381
  cliFlags.push(STACK_FLAG);
351
382
  const childArgs = [...cliFlags, ...process.argv.slice(1)];
352
383
  const childEnv = {
353
384
  ...process.env,
354
- NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim(),
385
+ NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG} ${SEMI_FLAG}`.trim(),
355
386
  };
356
387
  if (shouldBridgeRespawnProgressTty())
357
388
  childEnv[RESPAWN_PROGRESS_ENV] = '1';
358
389
  const childExit = await runRespawnedAnalyze(childArgs, childEnv);
359
390
  if (childExit.status !== 0 || childExit.signal) {
360
391
  if (childProcessLikelyOom(childExit)) {
361
- cliError(` Analysis likely ran out of memory.\n` +
362
- ` Retry with a larger heap if your machine allows it:\n` +
363
- ` NODE_OPTIONS="--max-old-space-size=24576" gitnexus analyze [your-args]\n` +
364
- ` (Windows: set NODE_OPTIONS=--max-old-space-size=24576 && gitnexus analyze [your-args])\n` +
392
+ cliError(` Analysis likely ran out of memory (heap cap auto-sized to ${RESPAWN_HEAP_MB}MB ≈ 0.75x RAM).\n` +
393
+ ` This repository's working set exceeds available RAM. Use a machine with more RAM,\n` +
394
+ ` or override the cap (a cap above physical RAM causes swap-thrash — use with care):\n` +
395
+ ` NODE_OPTIONS="--max-old-space-size=<MB>" gitnexus analyze [your-args]\n` +
396
+ ` (Windows: set NODE_OPTIONS=--max-old-space-size=<MB> && gitnexus analyze [your-args])\n` +
365
397
  ` If this persists, it may be a native crash unrelated to heap size.\n`, { recoveryHint: 'heap-oom-respawn' });
366
398
  }
367
399
  else if (childProcessLikelyNativeAbort(childExit)) {
368
400
  cliError(` Analysis aborted in a native worker or native binding path.\n` +
369
401
  ` Try one of these recovery paths:\n` +
370
- ` gitnexus analyze --workers 0\n` +
371
- ` npm uninstall -g gitnexus && npm install -g gitnexus@latest\n` +
402
+ ` npm uninstall -g gitnexus && npm install -g gitnexus@latest (rebuilds native bindings)\n` +
372
403
  ` Use Node 22 LTS if you are on a newer non-LTS runtime.\n`, { recoveryHint: 'native-worker-abort' });
373
404
  }
374
405
  const status = typeof childExit.status === 'number' && childExit.status !== 0 ? childExit.status : 1;
@@ -389,6 +420,7 @@ const ANALYZE_CLI_ENV_KEYS = [
389
420
  'GITNEXUS_VERBOSE',
390
421
  'GITNEXUS_PROFILE_DEFERRED',
391
422
  'GITNEXUS_PROFILE_DEFERRED_SLOW_MS',
423
+ 'GITNEXUS_DEBUG_HEAP',
392
424
  'GITNEXUS_MAX_FILE_SIZE',
393
425
  'GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS',
394
426
  'GITNEXUS_WAL_CHECKPOINT_THRESHOLD',
@@ -576,9 +608,10 @@ const analyzeCommandImpl = async (inputPath, cliOptions) => {
576
608
  let workerPoolSize;
577
609
  if (options.workers !== undefined) {
578
610
  const parsedWorkers = Number(options.workers);
579
- if (!Number.isInteger(parsedWorkers) || parsedWorkers < 0) {
580
- cliError(' --workers must be a non-negative integer. ' +
581
- 'Pass 0 to disable the worker pool (sequential fallback).\n');
611
+ if (!Number.isInteger(parsedWorkers) || parsedWorkers < 1) {
612
+ cliError(' --workers must be a positive integer (>= 1). ' +
613
+ 'GitNexus parses through a worker pool only — there is no sequential ' +
614
+ 'mode, so 0 is not allowed. Omit --workers for an auto-sized pool.\n');
582
615
  process.exitCode = 1;
583
616
  return;
584
617
  }
@@ -143,7 +143,7 @@ export declare const en: {
143
143
  readonly 'help.option.analyze.maxFileSize': "Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).";
144
144
  readonly 'help.option.analyze.workerTimeout': "Worker sub-batch idle timeout before retry/fallback. Default: 30.";
145
145
  readonly 'help.option.analyze.walCheckpointThreshold': "LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).";
146
- readonly 'help.option.analyze.workers': "Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).";
146
+ readonly 'help.option.analyze.workers': "Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.";
147
147
  readonly 'help.option.analyze.embeddingThreads': "Limit local ONNX embedding CPU threads";
148
148
  readonly 'help.option.analyze.embeddingBatchSize': "Number of nodes per embedding batch";
149
149
  readonly 'help.option.analyze.embeddingSubBatchSize': "Number of chunks per embedding model call";
@@ -143,7 +143,7 @@ export const en = {
143
143
  'help.option.analyze.maxFileSize': 'Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).',
144
144
  'help.option.analyze.workerTimeout': 'Worker sub-batch idle timeout before retry/fallback. Default: 30.',
145
145
  'help.option.analyze.walCheckpointThreshold': 'LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).',
146
- 'help.option.analyze.workers': 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).',
146
+ 'help.option.analyze.workers': 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.',
147
147
  'help.option.analyze.embeddingThreads': 'Limit local ONNX embedding CPU threads',
148
148
  'help.option.analyze.embeddingBatchSize': 'Number of nodes per embedding batch',
149
149
  'help.option.analyze.embeddingSubBatchSize': 'Number of chunks per embedding model call',
@@ -144,7 +144,7 @@ export declare const cliResources: {
144
144
  readonly 'help.option.analyze.maxFileSize': "Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).";
145
145
  readonly 'help.option.analyze.workerTimeout': "Worker sub-batch idle timeout before retry/fallback. Default: 30.";
146
146
  readonly 'help.option.analyze.walCheckpointThreshold': "LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).";
147
- readonly 'help.option.analyze.workers': "Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).";
147
+ readonly 'help.option.analyze.workers': "Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.";
148
148
  readonly 'help.option.analyze.embeddingThreads': "Limit local ONNX embedding CPU threads";
149
149
  readonly 'help.option.analyze.embeddingBatchSize': "Number of nodes per embedding batch";
150
150
  readonly 'help.option.analyze.embeddingSubBatchSize': "Number of chunks per embedding model call";
@@ -143,7 +143,7 @@ export const zhCN = {
143
143
  'help.option.analyze.maxFileSize': '跳过大于该值的文件(KB)。默认:512。硬上限:32768(tree-sitter 限制)。',
144
144
  'help.option.analyze.workerTimeout': 'Worker 子批次空闲超时,超时后重试/回退。默认:30。',
145
145
  'help.option.analyze.walCheckpointThreshold': 'analyze 期间 LadybugDB WAL 自动 checkpoint 阈值(字节,整数 >= -1;默认:67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。',
146
- 'help.option.analyze.workers': '解析 worker 池大小。默认:cores-1,最多 16。传 0 禁用 worker(顺序执行)。',
146
+ 'help.option.analyze.workers': '解析 worker 池大小(>=1)。默认:cores-1,最多 16,按仓库规模自适应。',
147
147
  'help.option.analyze.embeddingThreads': '限制本地 ONNX 嵌入 CPU 线程数',
148
148
  'help.option.analyze.embeddingBatchSize': '每个嵌入批次的节点数',
149
149
  'help.option.analyze.embeddingSubBatchSize': '每次嵌入模型调用的分块数',
package/dist/cli/index.js CHANGED
@@ -44,7 +44,7 @@ program
44
44
  .option('--worker-timeout <seconds>', 'Worker sub-batch idle timeout before retry/fallback. Default: 30.')
45
45
  .option('--wal-checkpoint-threshold <bytes>', 'LadybugDB WAL auto-checkpoint threshold in bytes during analyze ' +
46
46
  '(integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).')
47
- .option('--workers <n>', 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).')
47
+ .option('--workers <n>', 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.')
48
48
  .option('--embedding-threads <n>', 'Limit local ONNX embedding CPU threads')
49
49
  .option('--embedding-batch-size <n>', 'Number of nodes per embedding batch')
50
50
  .option('--embedding-sub-batch-size <n>', 'Number of chunks per embedding model call')
@@ -22,6 +22,11 @@ import type { ExtractedRoute, ExtractedFetchCall } from './workers/parse-worker.
22
22
  /** Per-file resolved type bindings for exported symbols.
23
23
  * Consumed by the cross-file re-resolution / enrichment pass. */
24
24
  export type ExportedTypeMap = Map<string, Map<string, string>>;
25
+ /** Record one exported graph node into the incremental ExportedTypeMap. */
26
+ export declare const accumulateExportedTypesFromParsedNode: (result: ExportedTypeMap, node: {
27
+ id: string;
28
+ properties?: Record<string, unknown>;
29
+ }, symbolTable: SymbolTableReader) => void;
25
30
  /** Build ExportedTypeMap from graph nodes — used for the worker path where the
26
31
  * sequential TypeEnv is not available in the main thread. Collects
27
32
  * returnType/declaredType from exported symbols with known types. */
@@ -27,41 +27,42 @@ import { normalizeFetchURL, routeMatches } from './route-extractors/nextjs.js';
27
27
  import { extractReturnTypeName } from './type-extractors/shared.js';
28
28
  const MAX_EXPORTS_PER_FILE = 500;
29
29
  const MAX_TYPE_NAME_LENGTH = 256;
30
+ /** Record one exported graph node into the incremental ExportedTypeMap. */
31
+ export const accumulateExportedTypesFromParsedNode = (result, node, symbolTable) => {
32
+ if (!node.properties?.isExported)
33
+ return;
34
+ if (!node.properties?.filePath || !node.properties?.name)
35
+ return;
36
+ const filePath = node.properties.filePath;
37
+ const name = node.properties.name;
38
+ if (!name || name.length > MAX_TYPE_NAME_LENGTH)
39
+ return;
40
+ const defs = symbolTable.lookupExactAll(filePath, name);
41
+ const def = defs.find((d) => d.nodeId === node.id) ?? defs[0];
42
+ if (!def)
43
+ return;
44
+ const typeName = def.returnType ?? def.declaredType;
45
+ if (!typeName || typeName.length > MAX_TYPE_NAME_LENGTH)
46
+ return;
47
+ const simpleType = extractReturnTypeName(typeName) ?? typeName;
48
+ if (!simpleType)
49
+ return;
50
+ let fileExports = result.get(filePath);
51
+ if (!fileExports) {
52
+ fileExports = new Map();
53
+ result.set(filePath, fileExports);
54
+ }
55
+ if (fileExports.size < MAX_EXPORTS_PER_FILE) {
56
+ fileExports.set(name, simpleType);
57
+ }
58
+ };
30
59
  /** Build ExportedTypeMap from graph nodes — used for the worker path where the
31
60
  * sequential TypeEnv is not available in the main thread. Collects
32
61
  * returnType/declaredType from exported symbols with known types. */
33
62
  export function buildExportedTypeMapFromGraph(graph, symbolTable) {
34
63
  const result = new Map();
35
64
  graph.forEachNode((node) => {
36
- if (!node.properties?.isExported)
37
- return;
38
- if (!node.properties?.filePath || !node.properties?.name)
39
- return;
40
- const filePath = node.properties.filePath;
41
- const name = node.properties.name;
42
- if (!name || name.length > MAX_TYPE_NAME_LENGTH)
43
- return;
44
- // For callable symbols, use returnType; for properties/variables, use declaredType.
45
- // Use lookupExactAll + nodeId match to handle same-name methods in different classes.
46
- const defs = symbolTable.lookupExactAll(filePath, name);
47
- const def = defs.find((d) => d.nodeId === node.id) ?? defs[0];
48
- if (!def)
49
- return;
50
- const typeName = def.returnType ?? def.declaredType;
51
- if (!typeName || typeName.length > MAX_TYPE_NAME_LENGTH)
52
- return;
53
- // Extract simple type name (strip Promise<>, etc.) — reuse shared utility
54
- const simpleType = extractReturnTypeName(typeName) ?? typeName;
55
- if (!simpleType)
56
- return;
57
- let fileExports = result.get(filePath);
58
- if (!fileExports) {
59
- fileExports = new Map();
60
- result.set(filePath, fileExports);
61
- }
62
- if (fileExports.size < MAX_EXPORTS_PER_FILE) {
63
- fileExports.set(name, simpleType);
64
- }
65
+ accumulateExportedTypesFromParsedNode(result, node, symbolTable);
65
66
  });
66
67
  return result;
67
68
  }
@@ -32,6 +32,8 @@
32
32
  * safely consult `model.scopes` without branching on presence.
33
33
  */
34
34
  import { buildDefIndex, buildMethodDispatchIndex, buildModuleScopeIndex, buildQualifiedNameIndex, buildScopeTree, finalize, } from '../../_shared/index.js';
35
+ import { parseTruthyEnv } from './utils/env.js';
36
+ import { TransitionalScopeTree } from '../../storage/scope-index-store.js';
35
37
  /**
36
38
  * Produce a fully materialized `ScopeResolutionIndexes` from the
37
39
  * workspace's per-file artifacts.
@@ -66,7 +68,13 @@ export function finalizeScopeModel(parsedFiles, options = {}) {
66
68
  allDefs.push(d);
67
69
  moduleEntries.push({ filePath: file.filePath, moduleScopeId: file.moduleScope });
68
70
  }
69
- const scopeTree = buildScopeTree(allScopes);
71
+ // Out-of-core scope index: when enabled, build a TransitionalScopeTree
72
+ // (validated + fully resident now; sealed to disk by run.ts just before emit so
73
+ // the heavy Scope.bindings payload is reclaimed). Default off → the in-heap
74
+ // buildScopeTree result exactly, byte-identical.
75
+ const scopeTree = parseTruthyEnv(process.env.GITNEXUS_DISK_SCOPE_INDEX)
76
+ ? new TransitionalScopeTree(allScopes)
77
+ : buildScopeTree(allScopes);
70
78
  const defs = buildDefIndex(allDefs);
71
79
  const qualifiedNames = buildQualifiedNameIndex(allDefs);
72
80
  const moduleScopes = buildModuleScopeIndex(moduleEntries);
@@ -241,6 +241,26 @@ interface LanguageProviderConfig {
241
241
  sourceMeta?: {
242
242
  readonly sourceKind?: 'full-file' | 'pre-extracted-script';
243
243
  }) => readonly CaptureMatch[];
244
+ /**
245
+ * Snapshot the capture-time side-channel state that this provider's
246
+ * `emitScopeCaptures` just populated for `filePath` into module-level maps,
247
+ * returning a plain JSON-serializable value (or `undefined` when there is
248
+ * nothing to carry).
249
+ *
250
+ * Called in the parse worker IMMEDIATELY after `emitScopeCaptures` runs for
251
+ * a file (see `parse-worker.ts`), and the result is stored on the produced
252
+ * `ParsedFile.captureSideChannel`. Scope-resolution on the main thread reuses
253
+ * that serialized `ParsedFile` and skips re-extraction (#1983), so this hook
254
+ * is how the worker-computed marks survive the worker→main boundary and the
255
+ * disk store WITHOUT a main-thread re-parse. The main thread restores them
256
+ * via the matching `ScopeResolver.applyCaptureSideChannel` hook.
257
+ *
258
+ * MUST return plain data (objects / arrays / primitives) so it round-trips
259
+ * through `JSON.stringify` + the parsedfile-store interning reviver.
260
+ *
261
+ * Default: undefined (provider has no capture-time module-level side effects).
262
+ */
263
+ readonly collectCaptureSideChannel?: (filePath: string) => unknown;
244
264
  /**
245
265
  * Interpret a raw `@import.statement` capture group into a `ParsedImport`.
246
266
  * The central finalize algorithm resolves `ParsedImport.targetRaw` to a
@@ -0,0 +1,62 @@
1
+ /**
2
+ * C capture-time side-channel serialization (#1983).
3
+ *
4
+ * `emitCScopeCaptures` populates one MODULE-LEVEL, per-file map as a side
5
+ * effect that is NOT part of the returned `ParsedFile`'s scopes/defs:
6
+ *
7
+ * - `staticNames` (static-linkage.ts) — the simple names of functions
8
+ * declared with `static` storage class (file-local / translation-unit
9
+ * linkage in C), recorded via `markStaticName` from the
10
+ * `@declaration.name` capture when the function node has a `static`
11
+ * storage-class specifier.
12
+ *
13
+ * On the worker path that map is filled in the WORKER process and lost across
14
+ * the worker→main MessageChannel (and the disk-backed parsedfile-store),
15
+ * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the
16
+ * main-thread re-extraction (the #1983 fix that avoids a main-thread
17
+ * tree-sitter re-parse / OOM on huge repos — e.g. the Linux kernel). The main
18
+ * thread then reads the map empty in `isStaticName` (consulted by
19
+ * `isFileLocalDef` in `c/scope-resolver.ts` and by `expandCWildcardNames` in
20
+ * static-linkage.ts) — so file-local `static` functions become eligible for
21
+ * cross-file global free-call resolution (false CALLS edges) and `#include`
22
+ * wildcard imports over-expose them.
23
+ *
24
+ * This module snapshots the per-file slice of that map into a plain,
25
+ * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and
26
+ * restores it on the main thread WITHOUT any parse. It mirrors the C++ pattern
27
+ * in `cpp/capture-side-channel.ts` and the Kotlin pattern in
28
+ * `kotlin/capture-side-channel.ts`.
29
+ *
30
+ * The single generic `ParsedFile.captureSideChannel` field is shared with C++
31
+ * and Kotlin, which is safe because each file is one language (a `.c` file uses
32
+ * the C provider). The payload is self-describing (`{ kind: 'c', staticNames }`)
33
+ * so `applyCStaticLinkageSideChannel` only restores C state and ignores a
34
+ * foreign-shaped snapshot.
35
+ */
36
+ import type { ParsedFile } from '../../../../_shared/index.js';
37
+ /**
38
+ * Plain JSON-serializable snapshot of the per-file C capture-time
39
+ * side-channel. Carried opaquely on `ParsedFile.captureSideChannel`. The
40
+ * `kind` tag makes the payload self-describing so `apply` can distinguish a C
41
+ * snapshot from another language's (C++ and Kotlin share the same field).
42
+ */
43
+ export interface CCaptureSideChannel {
44
+ readonly kind: 'c';
45
+ /** Simple names of `static` (file-local linkage) functions in this file. */
46
+ readonly staticNames: readonly string[];
47
+ }
48
+ /**
49
+ * `LanguageProvider.collectCaptureSideChannel` implementation for C.
50
+ * Returns `undefined` when this file recorded no static names at all, so the
51
+ * produced `ParsedFile` carries the field only when there's data to ship.
52
+ */
53
+ export declare function collectCStaticLinkageSideChannel(filePath: string): CCaptureSideChannel | undefined;
54
+ /**
55
+ * `ScopeResolver.applyCaptureSideChannel` implementation for C. Reads the
56
+ * worker-serialized snapshot from `parsed.captureSideChannel` and re-populates
57
+ * the module-level static-linkage map via `markStaticName`. Tolerant of
58
+ * `undefined` (file carried no data) and of an unexpected / foreign shape
59
+ * (defensive — the `kind` tag guards against restoring a non-C payload).
60
+ * Does NO tree-sitter parse.
61
+ */
62
+ export declare function applyCStaticLinkageSideChannel(parsed: ParsedFile): void;
@@ -0,0 +1,65 @@
1
+ /**
2
+ * C capture-time side-channel serialization (#1983).
3
+ *
4
+ * `emitCScopeCaptures` populates one MODULE-LEVEL, per-file map as a side
5
+ * effect that is NOT part of the returned `ParsedFile`'s scopes/defs:
6
+ *
7
+ * - `staticNames` (static-linkage.ts) — the simple names of functions
8
+ * declared with `static` storage class (file-local / translation-unit
9
+ * linkage in C), recorded via `markStaticName` from the
10
+ * `@declaration.name` capture when the function node has a `static`
11
+ * storage-class specifier.
12
+ *
13
+ * On the worker path that map is filled in the WORKER process and lost across
14
+ * the worker→main MessageChannel (and the disk-backed parsedfile-store),
15
+ * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the
16
+ * main-thread re-extraction (the #1983 fix that avoids a main-thread
17
+ * tree-sitter re-parse / OOM on huge repos — e.g. the Linux kernel). The main
18
+ * thread then reads the map empty in `isStaticName` (consulted by
19
+ * `isFileLocalDef` in `c/scope-resolver.ts` and by `expandCWildcardNames` in
20
+ * static-linkage.ts) — so file-local `static` functions become eligible for
21
+ * cross-file global free-call resolution (false CALLS edges) and `#include`
22
+ * wildcard imports over-expose them.
23
+ *
24
+ * This module snapshots the per-file slice of that map into a plain,
25
+ * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and
26
+ * restores it on the main thread WITHOUT any parse. It mirrors the C++ pattern
27
+ * in `cpp/capture-side-channel.ts` and the Kotlin pattern in
28
+ * `kotlin/capture-side-channel.ts`.
29
+ *
30
+ * The single generic `ParsedFile.captureSideChannel` field is shared with C++
31
+ * and Kotlin, which is safe because each file is one language (a `.c` file uses
32
+ * the C provider). The payload is self-describing (`{ kind: 'c', staticNames }`)
33
+ * so `applyCStaticLinkageSideChannel` only restores C state and ignores a
34
+ * foreign-shaped snapshot.
35
+ */
36
+ import { getStaticNamesForFile, markStaticName } from './static-linkage.js';
37
+ /**
38
+ * `LanguageProvider.collectCaptureSideChannel` implementation for C.
39
+ * Returns `undefined` when this file recorded no static names at all, so the
40
+ * produced `ParsedFile` carries the field only when there's data to ship.
41
+ */
42
+ export function collectCStaticLinkageSideChannel(filePath) {
43
+ const staticNames = getStaticNamesForFile(filePath);
44
+ if (staticNames.length === 0)
45
+ return undefined;
46
+ return { kind: 'c', staticNames };
47
+ }
48
+ /**
49
+ * `ScopeResolver.applyCaptureSideChannel` implementation for C. Reads the
50
+ * worker-serialized snapshot from `parsed.captureSideChannel` and re-populates
51
+ * the module-level static-linkage map via `markStaticName`. Tolerant of
52
+ * `undefined` (file carried no data) and of an unexpected / foreign shape
53
+ * (defensive — the `kind` tag guards against restoring a non-C payload).
54
+ * Does NO tree-sitter parse.
55
+ */
56
+ export function applyCStaticLinkageSideChannel(parsed) {
57
+ const data = parsed.captureSideChannel;
58
+ if (data === undefined || data === null || typeof data !== 'object')
59
+ return;
60
+ if (data.kind !== 'c' || !Array.isArray(data.staticNames))
61
+ return;
62
+ for (const name of data.staticNames) {
63
+ markStaticName(parsed.filePath, name);
64
+ }
65
+ }
@@ -1,4 +1,39 @@
1
1
  import { dirname, join } from 'path';
2
+ /**
3
+ * Per-pass memo: workspace paths bucketed by basename (last path segment),
4
+ * keyed on the `allFilePaths` set identity.
5
+ *
6
+ * `resolveCImportTarget` is called once per (quoted) C/C++ `#include` with the
7
+ * same `allFilePaths` set per pass (the augmented set is itself memoized in
8
+ * the C resolver). The old suffix-match fallback scanned ALL workspace paths
9
+ * per include — with a per-element `.replace`/`.split` and no early exit
10
+ * (the fewest-path-components tie-break forces a full scan) — i.e.
11
+ * O(R_suffix × (F+H)). A path can satisfy `endsWith('/'+target)` (or equal
12
+ * the target) ONLY IF its basename equals the target's last segment, so we
13
+ * pre-bucket by basename once (O(F+H), `normalized`/`depth` precomputed) and
14
+ * the fallback inspects a single small bucket → O(F+H) build + ~O(1)/include.
15
+ * `WeakMap`-keyed so it is reclaimed with the pass (no cross-pass staleness).
16
+ * Shared by C and C++ (`resolveCppImportTarget` delegates here).
17
+ */
18
+ const suffixIndexByPaths = new WeakMap();
19
+ function suffixIndex(allFilePaths) {
20
+ let index = suffixIndexByPaths.get(allFilePaths);
21
+ if (index === undefined) {
22
+ index = new Map();
23
+ for (const original of allFilePaths) {
24
+ const normalized = original.replace(/\\/g, '/');
25
+ const basename = normalized.slice(normalized.lastIndexOf('/') + 1);
26
+ let bucket = index.get(basename);
27
+ if (bucket === undefined) {
28
+ bucket = [];
29
+ index.set(basename, bucket);
30
+ }
31
+ bucket.push({ original, normalized, depth: normalized.split('/').length });
32
+ }
33
+ suffixIndexByPaths.set(allFilePaths, index);
34
+ }
35
+ return index;
36
+ }
2
37
  /**
3
38
  * Resolve a C #include path to a file in the workspace.
4
39
  *
@@ -36,20 +71,28 @@ export function resolveCImportTarget(targetRaw, fromFile, allFilePaths) {
36
71
  // Exact match (path as-is in the workspace)
37
72
  if (allFilePaths.has(normalizedTarget))
38
73
  return normalizedTarget;
39
- // Suffix match: find files ending with /targetRaw or equal to targetRaw
74
+ // Suffix match: find files ending with /targetRaw or equal to targetRaw.
75
+ // A path can only match `=== normalizedTarget` or `endsWith('/'+target)` if
76
+ // its basename equals the target's last segment, so we inspect only that
77
+ // basename bucket (built once per pass) instead of scanning every workspace
78
+ // path. Match condition + tie-break (fewest path components, then
79
+ // lexicographic on the normalized path) are byte-identical to the prior scan.
40
80
  const suffix = '/' + normalizedTarget;
81
+ const targetBasename = normalizedTarget.slice(normalizedTarget.lastIndexOf('/') + 1);
82
+ const bucket = suffixIndex(allFilePaths).get(targetBasename);
83
+ if (bucket === undefined)
84
+ return null;
41
85
  let bestMatch = null;
42
86
  let bestDepth = Infinity;
43
87
  let bestNormalized = '';
44
- for (const filePath of allFilePaths) {
45
- const normalized = filePath.replace(/\\/g, '/');
46
- if (normalized === normalizedTarget || normalized.endsWith(suffix)) {
88
+ for (const cand of bucket) {
89
+ if (cand.normalized === normalizedTarget || cand.normalized.endsWith(suffix)) {
47
90
  // Prefer shortest path (closest match)
48
- const depth = normalized.split('/').length;
49
- if (depth < bestDepth || (depth === bestDepth && normalized < bestNormalized)) {
50
- bestDepth = depth;
51
- bestMatch = filePath;
52
- bestNormalized = normalized;
91
+ if (cand.depth < bestDepth ||
92
+ (cand.depth === bestDepth && cand.normalized < bestNormalized)) {
93
+ bestDepth = cand.depth;
94
+ bestMatch = cand.original;
95
+ bestNormalized = cand.normalized;
53
96
  }
54
97
  }
55
98
  }
@@ -8,4 +8,5 @@ export { cArityCompatibility } from './arity.js';
8
8
  export { cMergeBindings } from './merge-bindings.js';
9
9
  export { cBindingScopeFor, cImportOwningScope, cReceiverBinding } from './simple-hooks.js';
10
10
  export { resolveCImportTarget } from './import-target.js';
11
- export { markStaticName, isStaticName, clearStaticNames, expandCWildcardNames, } from './static-linkage.js';
11
+ export { markStaticName, isStaticName, clearStaticNames, expandCWildcardNames, getStaticNamesForFile, } from './static-linkage.js';
12
+ export { collectCStaticLinkageSideChannel, applyCStaticLinkageSideChannel, } from './capture-side-channel.js';
@@ -8,4 +8,5 @@ export { cArityCompatibility } from './arity.js';
8
8
  export { cMergeBindings } from './merge-bindings.js';
9
9
  export { cBindingScopeFor, cImportOwningScope, cReceiverBinding } from './simple-hooks.js';
10
10
  export { resolveCImportTarget } from './import-target.js';
11
- export { markStaticName, isStaticName, clearStaticNames, expandCWildcardNames, } from './static-linkage.js';
11
+ export { markStaticName, isStaticName, clearStaticNames, expandCWildcardNames, getStaticNamesForFile, } from './static-linkage.js';
12
+ export { collectCStaticLinkageSideChannel, applyCStaticLinkageSideChannel, } from './capture-side-channel.js';