gitnexus 1.6.7 → 1.6.8-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -0
- package/dist/cli/analyze-config.js +39 -0
- package/dist/cli/analyze.d.ts +16 -0
- package/dist/cli/analyze.js +34 -6
- package/dist/cli/clean.d.ts +1 -0
- package/dist/cli/clean.js +43 -1
- package/dist/cli/eval-server.js +43 -0
- package/dist/cli/help-i18n.js +6 -0
- package/dist/cli/i18n/en.d.ts +11 -0
- package/dist/cli/i18n/en.js +11 -0
- package/dist/cli/i18n/resources.d.ts +22 -0
- package/dist/cli/i18n/zh-CN.d.ts +11 -0
- package/dist/cli/i18n/zh-CN.js +11 -0
- package/dist/cli/index.js +9 -0
- package/dist/cli/list.js +12 -0
- package/dist/cli/optional-grammars.d.ts +6 -8
- package/dist/cli/optional-grammars.js +8 -11
- package/dist/cli/status.js +26 -5
- package/dist/cli/tool.d.ts +5 -0
- package/dist/cli/tool.js +5 -0
- package/dist/core/embeddings/embedder.js +4 -0
- package/dist/core/embeddings/embedding-pipeline.js +27 -16
- package/dist/core/embeddings/onnxruntime-common-resolver.d.ts +6 -0
- package/dist/core/embeddings/onnxruntime-common-resolver.js +130 -0
- package/dist/core/group/extractors/grpc-patterns/proto.js +10 -6
- package/dist/core/group/extractors/http-patterns/java.js +3 -50
- package/dist/core/group/extractors/http-patterns/kotlin.js +7 -7
- package/dist/core/group/extractors/include-extractor.js +7 -7
- package/dist/core/ingestion/language-provider.d.ts +25 -2
- package/dist/core/ingestion/languages/c-cpp.js +11 -4
- package/dist/core/ingestion/languages/java.js +3 -0
- package/dist/core/ingestion/languages/kotlin/query.js +3 -2
- package/dist/core/ingestion/languages/kotlin.js +5 -1
- package/dist/core/ingestion/parsing-processor.js +21 -0
- package/dist/core/ingestion/pipeline-phases/routes.js +64 -14
- package/dist/core/ingestion/pipeline.d.ts +9 -0
- package/dist/core/ingestion/route-extractors/spring-shared.d.ts +50 -0
- package/dist/core/ingestion/route-extractors/spring-shared.js +80 -0
- package/dist/core/ingestion/route-extractors/spring.d.ts +35 -0
- package/dist/core/ingestion/route-extractors/spring.js +136 -0
- package/dist/core/ingestion/workers/clone-safety.d.ts +109 -0
- package/dist/core/ingestion/workers/clone-safety.js +465 -0
- package/dist/core/ingestion/workers/parse-worker.d.ts +10 -0
- package/dist/core/ingestion/workers/parse-worker.js +27 -52
- package/dist/core/ingestion/workers/post-result.d.ts +22 -0
- package/dist/core/ingestion/workers/post-result.js +87 -0
- package/dist/core/ingestion/workers/result-merge.d.ts +20 -0
- package/dist/core/ingestion/workers/result-merge.js +43 -0
- package/dist/core/ingestion/workers/worker-pool.d.ts +6 -0
- package/dist/core/ingestion/workers/worker-pool.js +28 -14
- package/dist/core/lbug/lbug-adapter.d.ts +19 -0
- package/dist/core/lbug/lbug-adapter.js +56 -1
- package/dist/core/run-analyze.d.ts +45 -0
- package/dist/core/run-analyze.js +175 -24
- package/dist/core/tree-sitter/parser-loader.js +5 -4
- package/dist/core/tree-sitter/vendored-grammars.d.ts +39 -0
- package/dist/core/tree-sitter/vendored-grammars.js +57 -0
- package/dist/mcp/core/embedder.js +4 -0
- package/dist/mcp/local/local-backend.d.ts +55 -2
- package/dist/mcp/local/local-backend.js +363 -25
- package/dist/mcp/tools.js +32 -0
- package/dist/server/analyze-worker-ipc.d.ts +58 -0
- package/dist/server/analyze-worker-ipc.js +16 -0
- package/dist/server/analyze-worker.js +7 -1
- package/dist/storage/branch-index.d.ts +52 -0
- package/dist/storage/branch-index.js +65 -0
- package/dist/storage/git.d.ts +11 -0
- package/dist/storage/git.js +28 -0
- package/dist/storage/parse-cache.js +3 -0
- package/dist/storage/repo-manager.d.ts +57 -2
- package/dist/storage/repo-manager.js +132 -18
- package/hooks/antigravity/gitnexus-antigravity-hook.cjs +24 -4
- package/hooks/claude/gitnexus-hook.cjs +18 -3
- package/package.json +2 -2
- package/scripts/assert-publish-grammar-coverage.cjs +31 -0
- package/scripts/build-tree-sitter-grammars.cjs +16 -10
- package/vendor/tree-sitter-c/package.json +1 -1
- package/vendor/tree-sitter-dart/package.json +1 -1
- package/vendor/tree-sitter-kotlin/package.json +1 -1
- package/vendor/tree-sitter-proto/package.json +1 -1
- package/vendor/tree-sitter-swift/package.json +1 -1
- package/scripts/materialize-vendor-grammars.cjs +0 -97
|
@@ -11,15 +11,17 @@ import Go from 'tree-sitter-go';
|
|
|
11
11
|
import Rust from 'tree-sitter-rust';
|
|
12
12
|
import PHP from 'tree-sitter-php';
|
|
13
13
|
import Ruby from 'tree-sitter-ruby';
|
|
14
|
-
import {
|
|
14
|
+
import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js';
|
|
15
15
|
import { SupportedLanguages } from '../../../_shared/index.js';
|
|
16
16
|
import { getProvider } from '../languages/index.js';
|
|
17
17
|
import { getTreeSitterBufferSize, getTreeSitterContentByteLength, TREE_SITTER_MAX_BUFFER, } from '../constants.js';
|
|
18
18
|
import { ARRAY_METHOD_HOC_BLOCKLIST_SET, DEFAULT_EXPORT_IDENTIFIER_BLOCKLIST_SET, deriveDefaultExportHocName, } from '../ts-js-hoc-utils.js';
|
|
19
19
|
import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
|
|
20
|
+
import { postResultCloneSafe } from './post-result.js';
|
|
21
|
+
import { mergeResult } from './result-merge.js';
|
|
20
22
|
// ── Worker grammar loading — enforcement boundary (#2091/#2093, #2101) ───────
|
|
21
|
-
// The worker maintains its own grammar table (the guarded
|
|
22
|
-
// `languageMap`) and intentionally does NOT consult the runtime
|
|
23
|
+
// The worker maintains its own grammar table (the guarded vendored-grammar
|
|
24
|
+
// loads below + `languageMap`) and intentionally does NOT consult the runtime
|
|
23
25
|
// `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` opt-out. It does not need to: the MAIN
|
|
24
26
|
// THREAD's `parseableScanned` filter (pipeline-phases/parse-impl.ts, gated on
|
|
25
27
|
// `parser-loader.isLanguageAvailable`, which honors the runtime opt-out and a
|
|
@@ -30,33 +32,29 @@ import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
|
|
|
30
32
|
// `isLanguageAvailable` must re-introduce the gate here. (The cleaner end-state
|
|
31
33
|
// — routing this table through `parser-loader.getLanguageGrammar` so there is
|
|
32
34
|
// one loader — is the deferred Tier-1 consolidation.)
|
|
33
|
-
//
|
|
34
|
-
|
|
35
|
+
// Swift/Dart/Kotlin/C are vendored grammars loaded from `vendor/` by absolute
|
|
36
|
+
// path (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Each
|
|
37
|
+
// may be absent on a platform without a prebuild or a toolchain-less /
|
|
38
|
+
// `--ignore-scripts` install, so every load is guarded so a missing binding
|
|
39
|
+
// cannot crash the worker at module-load (#2091/#2093, #2116).
|
|
35
40
|
let Swift = null;
|
|
36
41
|
try {
|
|
37
|
-
Swift =
|
|
42
|
+
Swift = requireVendoredGrammar('tree-sitter-swift');
|
|
38
43
|
}
|
|
39
44
|
catch { }
|
|
40
|
-
// tree-sitter-dart is an optionalDependency — may not be installed
|
|
41
45
|
let Dart = null;
|
|
42
46
|
try {
|
|
43
|
-
Dart =
|
|
47
|
+
Dart = requireVendoredGrammar('tree-sitter-dart');
|
|
44
48
|
}
|
|
45
49
|
catch { }
|
|
46
|
-
// tree-sitter-kotlin is an optionalDependency — may not be installed
|
|
47
50
|
let Kotlin = null;
|
|
48
51
|
try {
|
|
49
|
-
Kotlin =
|
|
52
|
+
Kotlin = requireVendoredGrammar('tree-sitter-kotlin');
|
|
50
53
|
}
|
|
51
54
|
catch { }
|
|
52
|
-
// tree-sitter-c is now vendored prebuild-only (#2116) and may be absent on a
|
|
53
|
-
// toolchain-less / `--ignore-scripts` install. Guard it like Swift/Dart/Kotlin so
|
|
54
|
-
// a missing binding cannot crash the worker at module-load (#2091/#2093); the
|
|
55
|
-
// main-thread `isLanguageAvailable` filter keeps C files from being dispatched
|
|
56
|
-
// here when the entry is absent.
|
|
57
55
|
let C = null;
|
|
58
56
|
try {
|
|
59
|
-
C =
|
|
57
|
+
C = requireVendoredGrammar('tree-sitter-c');
|
|
60
58
|
}
|
|
61
59
|
catch { }
|
|
62
60
|
import { getLanguageFromFilename } from '../../../_shared/index.js';
|
|
@@ -678,6 +676,7 @@ const ROUTE_DECORATOR_NAMES = new Set([
|
|
|
678
676
|
'PostMapping',
|
|
679
677
|
'PutMapping',
|
|
680
678
|
'DeleteMapping',
|
|
679
|
+
'PatchMapping',
|
|
681
680
|
]);
|
|
682
681
|
// ============================================================================
|
|
683
682
|
// ORM Query Detection (Prisma + Supabase)
|
|
@@ -1743,6 +1742,15 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
1743
1742
|
if (language === SupportedLanguages.Python) {
|
|
1744
1743
|
extractFastAPIRouterBindings(file.path, parseContent, result.routerIncludes, result.routerImports, (result.routerModuleAliases ??= []));
|
|
1745
1744
|
}
|
|
1745
|
+
// Language-specific decorator route extraction via provider hook.
|
|
1746
|
+
// The provider's extractDecoratorRoutes walks the AST for framework-specific
|
|
1747
|
+
// route patterns (e.g., Java Spring class-level prefix joining). Routes are
|
|
1748
|
+
// appended to decoratorRoutes for the routes phase to emit as Route nodes.
|
|
1749
|
+
if (provider.extractDecoratorRoutes) {
|
|
1750
|
+
const frameworkRoutes = provider.extractDecoratorRoutes(tree, file.path, lineOffset);
|
|
1751
|
+
for (const r of frameworkRoutes)
|
|
1752
|
+
result.decoratorRoutes.push(r);
|
|
1753
|
+
}
|
|
1746
1754
|
// Vue: emit CALLS edges for components used in <template>
|
|
1747
1755
|
if (language === SupportedLanguages.Vue) {
|
|
1748
1756
|
const templateComponents = extractTemplateComponents(file.content);
|
|
@@ -1783,41 +1791,8 @@ let accumulated = {
|
|
|
1783
1791
|
fileCount: 0,
|
|
1784
1792
|
};
|
|
1785
1793
|
let cumulativeProcessed = 0;
|
|
1786
|
-
//
|
|
1787
|
-
//
|
|
1788
|
-
// and blows the stack when arr has >~65k elements).
|
|
1789
|
-
const appendAll = (target, src) => {
|
|
1790
|
-
for (let i = 0; i < src.length; i++)
|
|
1791
|
-
target.push(src[i]);
|
|
1792
|
-
};
|
|
1793
|
-
const mergeResult = (target, src) => {
|
|
1794
|
-
appendAll(target.nodes, src.nodes);
|
|
1795
|
-
appendAll(target.relationships, src.relationships);
|
|
1796
|
-
appendAll(target.symbols, src.symbols);
|
|
1797
|
-
appendAll(target.calls, src.calls);
|
|
1798
|
-
appendAll(target.assignments, src.assignments);
|
|
1799
|
-
appendAll(target.routes, src.routes);
|
|
1800
|
-
appendAll(target.fetchCalls, src.fetchCalls);
|
|
1801
|
-
appendAll(target.fetchWrapperDefs, src.fetchWrapperDefs);
|
|
1802
|
-
appendAll(target.decoratorRoutes, src.decoratorRoutes);
|
|
1803
|
-
if (src.routerIncludes)
|
|
1804
|
-
appendAll(target.routerIncludes, src.routerIncludes);
|
|
1805
|
-
if (src.routerImports)
|
|
1806
|
-
appendAll(target.routerImports, src.routerImports);
|
|
1807
|
-
if (src.routerModuleAliases) {
|
|
1808
|
-
target.routerModuleAliases ??= [];
|
|
1809
|
-
appendAll(target.routerModuleAliases, src.routerModuleAliases);
|
|
1810
|
-
}
|
|
1811
|
-
appendAll(target.toolDefs, src.toolDefs);
|
|
1812
|
-
appendAll(target.ormQueries, src.ormQueries);
|
|
1813
|
-
appendAll(target.constructorBindings, src.constructorBindings);
|
|
1814
|
-
appendAll(target.fileScopeBindings, src.fileScopeBindings);
|
|
1815
|
-
appendAll(target.parsedFiles, src.parsedFiles);
|
|
1816
|
-
for (const [lang, count] of Object.entries(src.skippedLanguages)) {
|
|
1817
|
-
target.skippedLanguages[lang] = (target.skippedLanguages[lang] || 0) + count;
|
|
1818
|
-
}
|
|
1819
|
-
target.fileCount += src.fileCount;
|
|
1820
|
-
};
|
|
1794
|
+
// `mergeResult` (+ its `appendAll`) lives in ./result-merge.ts (extracted so it
|
|
1795
|
+
// can be unit-tested without importing this entry module).
|
|
1821
1796
|
// Signal the pool that worker-side initialization (parser imports, language
|
|
1822
1797
|
// grammars, type-env setup, all helper modules) is complete and the message
|
|
1823
1798
|
// handler below is about to be attached. The pool's `waitForWorkerReady`
|
|
@@ -1910,7 +1885,7 @@ parentPort.on('message', (msg) => {
|
|
|
1910
1885
|
accumulated.parsedFiles = [];
|
|
1911
1886
|
}
|
|
1912
1887
|
}
|
|
1913
|
-
|
|
1888
|
+
postResultCloneSafe(accumulated);
|
|
1914
1889
|
// Reset for potential reuse
|
|
1915
1890
|
accumulated = {
|
|
1916
1891
|
nodes: [],
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ParseWorkerResult } from './parse-worker.js';
|
|
2
|
+
/**
|
|
3
|
+
* Deliver the accumulated result to the pool, surviving a non-cloneable value
|
|
4
|
+
* (#2112). Fast path: post as-is — on a healthy result this is the only thing
|
|
5
|
+
* that runs, so clone-safety adds zero overhead to normal runs. If structured
|
|
6
|
+
* clone rejects the payload (a function/symbol leaked into an extraction
|
|
7
|
+
* record — the reporter's case was a node `properties` value pointing at a
|
|
8
|
+
* native `toString`), rewrite the boundary-crossing arrays so the result is
|
|
9
|
+
* cloneable, record the affected paths on `result.skippedPaths`, warn the
|
|
10
|
+
* operator naming the offending field + file (so the still-unpinned leak is
|
|
11
|
+
* diagnosable from logs and fixable at source), and re-post.
|
|
12
|
+
*
|
|
13
|
+
* Recovery is attempted for ANY first-post failure, not only a `DataCloneError`.
|
|
14
|
+
* structuredClone invokes getters, and a getter that THROWS surfaces its own
|
|
15
|
+
* error (a `RangeError`, etc.) — NOT a `DataCloneError` (confirmed against a
|
|
16
|
+
* real MessageChannel). Gating recovery on `DataCloneError` let such a throw
|
|
17
|
+
* re-throw past the sanitizer and re-arm, under `POOL_SIZE=1`, the worker-death
|
|
18
|
+
* cascade this net prevents. The recovery path is wrapped in its own try/catch
|
|
19
|
+
* so a still-uncloneable re-post fails closed to a primitive-only
|
|
20
|
+
* `{type:'error'}` DELIBERATELY rather than escaping the worker.
|
|
21
|
+
*/
|
|
22
|
+
export declare function postResultCloneSafe(result: ParseWorkerResult): void;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker → main result delivery with clone-safety (#2112).
|
|
3
|
+
*
|
|
4
|
+
* Extracted from `parse-worker.ts` into its own side-effect-free module so it
|
|
5
|
+
* can be imported and exercised directly (the parse worker is an entry module:
|
|
6
|
+
* importing it would construct the parser, post `ready`, and attach the real
|
|
7
|
+
* message handler). The integration test imports `postResultCloneSafe` from
|
|
8
|
+
* here to cover the production wiring end to end rather than re-implementing it.
|
|
9
|
+
*/
|
|
10
|
+
import { parentPort } from 'node:worker_threads';
|
|
11
|
+
import { makeWorkerResultCloneSafe } from './clone-safety.js';
|
|
12
|
+
/**
|
|
13
|
+
* Strict mode (opt-in via `GITNEXUS_STRICT_CLONE=1`, inherited by workers). When
|
|
14
|
+
* on, a clone failure THROWS with the offending key path instead of silently
|
|
15
|
+
* sanitizing + delivering — so a leak introduced by a future provider/extractor
|
|
16
|
+
* change fails LOUDLY (in CI / dev) at its origin rather than being quietly
|
|
17
|
+
* stripped in production. The silent-recovery behavior is exactly what hid the
|
|
18
|
+
* original #2112 leak; strict mode removes the silence where we want loudness.
|
|
19
|
+
* Off in production, where the net's job is to keep the run alive.
|
|
20
|
+
*/
|
|
21
|
+
const STRICT_CLONE = process.env.GITNEXUS_STRICT_CLONE === '1';
|
|
22
|
+
/**
|
|
23
|
+
* Deliver the accumulated result to the pool, surviving a non-cloneable value
|
|
24
|
+
* (#2112). Fast path: post as-is — on a healthy result this is the only thing
|
|
25
|
+
* that runs, so clone-safety adds zero overhead to normal runs. If structured
|
|
26
|
+
* clone rejects the payload (a function/symbol leaked into an extraction
|
|
27
|
+
* record — the reporter's case was a node `properties` value pointing at a
|
|
28
|
+
* native `toString`), rewrite the boundary-crossing arrays so the result is
|
|
29
|
+
* cloneable, record the affected paths on `result.skippedPaths`, warn the
|
|
30
|
+
* operator naming the offending field + file (so the still-unpinned leak is
|
|
31
|
+
* diagnosable from logs and fixable at source), and re-post.
|
|
32
|
+
*
|
|
33
|
+
* Recovery is attempted for ANY first-post failure, not only a `DataCloneError`.
|
|
34
|
+
* structuredClone invokes getters, and a getter that THROWS surfaces its own
|
|
35
|
+
* error (a `RangeError`, etc.) — NOT a `DataCloneError` (confirmed against a
|
|
36
|
+
* real MessageChannel). Gating recovery on `DataCloneError` let such a throw
|
|
37
|
+
* re-throw past the sanitizer and re-arm, under `POOL_SIZE=1`, the worker-death
|
|
38
|
+
* cascade this net prevents. The recovery path is wrapped in its own try/catch
|
|
39
|
+
* so a still-uncloneable re-post fails closed to a primitive-only
|
|
40
|
+
* `{type:'error'}` DELIBERATELY rather than escaping the worker.
|
|
41
|
+
*/
|
|
42
|
+
export function postResultCloneSafe(result) {
|
|
43
|
+
try {
|
|
44
|
+
parentPort.postMessage({ type: 'result', data: result });
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// Fall through to recovery on ANY failure (DataCloneError OR a throwing
|
|
49
|
+
// getter's own error). A healthy post returned above and never reaches here.
|
|
50
|
+
}
|
|
51
|
+
try {
|
|
52
|
+
// `as unknown as Record<string, unknown>` is the standard widening for a
|
|
53
|
+
// no-index-signature interface (TS rejects a single-step `as`). The field
|
|
54
|
+
// sets are typed to `keyof ParseWorkerResult` so renaming a field is a
|
|
55
|
+
// compile error here, not a silent loss of the drop-whole / skip protection.
|
|
56
|
+
const { skipped } = makeWorkerResultCloneSafe(result, {
|
|
57
|
+
dropWholeElement: new Set(['parsedFiles']),
|
|
58
|
+
skipFields: new Set(['skippedPaths']),
|
|
59
|
+
});
|
|
60
|
+
if (skipped.length > 0) {
|
|
61
|
+
if (STRICT_CLONE) {
|
|
62
|
+
// Surface the leak loudly with its exact key path(s) instead of
|
|
63
|
+
// delivering a sanitized result. Routes to the catch below → a
|
|
64
|
+
// primitive-only {type:'error'} the pool reports, failing CI.
|
|
65
|
+
const detail = skipped.map((s) => `${s.path}: ${s.reason}`).join('; ');
|
|
66
|
+
throw new Error(`GITNEXUS_STRICT_CLONE: worker result was not structured-cloneable — ${detail}`);
|
|
67
|
+
}
|
|
68
|
+
result.skippedPaths = [...(result.skippedPaths ?? []), ...skipped];
|
|
69
|
+
const sample = skipped
|
|
70
|
+
.slice(0, 5)
|
|
71
|
+
.map((s) => `${s.path} (${s.reason})`)
|
|
72
|
+
.join('; ');
|
|
73
|
+
const more = skipped.length > 5 ? ` …and ${skipped.length - 5} more` : '';
|
|
74
|
+
if (parentPort) {
|
|
75
|
+
parentPort.postMessage({
|
|
76
|
+
type: 'warning',
|
|
77
|
+
message: `Sanitized ${skipped.length} file(s) with non-serializable parse output before delivery: ${sample}${more}`,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
parentPort.postMessage({ type: 'result', data: result });
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
const e = err instanceof Error ? err : new Error(String(err));
|
|
85
|
+
parentPort.postMessage({ type: 'error', error: e.message, errorStack: e.stack });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Merge of accumulated parse-worker results (sub-batch result → the conceptual
|
|
3
|
+
* job's running accumulator).
|
|
4
|
+
*
|
|
5
|
+
* Extracted from `parse-worker.ts` into this side-effect-free module so the
|
|
6
|
+
* merge can be imported and unit-tested directly — the parse worker is an entry
|
|
7
|
+
* module (importing it constructs the parser, posts `ready`, and attaches the
|
|
8
|
+
* real MessagePort handler), so a main-thread test cannot import a helper out of
|
|
9
|
+
* it. Mirrors the `post-result.ts` extraction.
|
|
10
|
+
*
|
|
11
|
+
* `import type` of `ParseWorkerResult` is erased at runtime, so there is no
|
|
12
|
+
* import cycle with `parse-worker.ts` (which imports this module's runtime).
|
|
13
|
+
*/
|
|
14
|
+
import type { ParseWorkerResult } from './parse-worker.js';
|
|
15
|
+
/**
|
|
16
|
+
* Merge `src` into `target` in place: append every boundary-crossing array,
|
|
17
|
+
* sum the per-language skip counts, union the clone-safety `skippedPaths`, and
|
|
18
|
+
* add the file count.
|
|
19
|
+
*/
|
|
20
|
+
export declare const mergeResult: (target: ParseWorkerResult, src: ParseWorkerResult) => void;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// Use a loop instead of push(...spread) to avoid hitting V8's argument limit
|
|
2
|
+
// when merging large result sets (push(...arr) calls apply() under the hood
|
|
3
|
+
// and blows the stack when arr has >~65k elements).
|
|
4
|
+
const appendAll = (target, src) => {
|
|
5
|
+
for (let i = 0; i < src.length; i++)
|
|
6
|
+
target.push(src[i]);
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Merge `src` into `target` in place: append every boundary-crossing array,
|
|
10
|
+
* sum the per-language skip counts, union the clone-safety `skippedPaths`, and
|
|
11
|
+
* add the file count.
|
|
12
|
+
*/
|
|
13
|
+
export const mergeResult = (target, src) => {
|
|
14
|
+
appendAll(target.nodes, src.nodes);
|
|
15
|
+
appendAll(target.relationships, src.relationships);
|
|
16
|
+
appendAll(target.symbols, src.symbols);
|
|
17
|
+
appendAll(target.calls, src.calls);
|
|
18
|
+
appendAll(target.assignments, src.assignments);
|
|
19
|
+
appendAll(target.routes, src.routes);
|
|
20
|
+
appendAll(target.fetchCalls, src.fetchCalls);
|
|
21
|
+
appendAll(target.fetchWrapperDefs, src.fetchWrapperDefs);
|
|
22
|
+
appendAll(target.decoratorRoutes, src.decoratorRoutes);
|
|
23
|
+
if (src.routerIncludes)
|
|
24
|
+
appendAll(target.routerIncludes, src.routerIncludes);
|
|
25
|
+
if (src.routerImports)
|
|
26
|
+
appendAll(target.routerImports, src.routerImports);
|
|
27
|
+
if (src.routerModuleAliases) {
|
|
28
|
+
target.routerModuleAliases ??= [];
|
|
29
|
+
appendAll(target.routerModuleAliases, src.routerModuleAliases);
|
|
30
|
+
}
|
|
31
|
+
appendAll(target.toolDefs, src.toolDefs);
|
|
32
|
+
appendAll(target.ormQueries, src.ormQueries);
|
|
33
|
+
appendAll(target.constructorBindings, src.constructorBindings);
|
|
34
|
+
appendAll(target.fileScopeBindings, src.fileScopeBindings);
|
|
35
|
+
appendAll(target.parsedFiles, src.parsedFiles);
|
|
36
|
+
for (const [lang, count] of Object.entries(src.skippedLanguages)) {
|
|
37
|
+
target.skippedLanguages[lang] = (target.skippedLanguages[lang] || 0) + count;
|
|
38
|
+
}
|
|
39
|
+
if (src.skippedPaths && src.skippedPaths.length > 0) {
|
|
40
|
+
(target.skippedPaths ??= []).push(...src.skippedPaths);
|
|
41
|
+
}
|
|
42
|
+
target.fileCount += src.fileCount;
|
|
43
|
+
};
|
|
@@ -273,6 +273,12 @@ export declare function resolveAutoPoolSize(): number;
|
|
|
273
273
|
* time spent across all attempts/splits/retries. When the budget is
|
|
274
274
|
* exhausted, the pool surfaces the in-flight path via `WorkerPoolDispatchError`
|
|
275
275
|
* instead of letting timeouts compound indefinitely.
|
|
276
|
+
*
|
|
277
|
+
* Upstream of these layers, the parse worker self-sanitizes a result that the
|
|
278
|
+
* structured-clone algorithm can't serialize (#2112) — stripping or dropping
|
|
279
|
+
* the offending value and reporting the affected paths on the result — so a
|
|
280
|
+
* single non-cloneable value can't masquerade as a worker death and exhaust a
|
|
281
|
+
* slot's respawn budget here.
|
|
276
282
|
*/
|
|
277
283
|
export declare const createWorkerPool: (workerUrl: URL, poolSize?: number, options?: WorkerPoolOptions) => WorkerPool;
|
|
278
284
|
export {};
|
|
@@ -523,6 +523,12 @@ function createJobs(items, maxItems, maxBytes, timeoutMs, chunkHash) {
|
|
|
523
523
|
* time spent across all attempts/splits/retries. When the budget is
|
|
524
524
|
* exhausted, the pool surfaces the in-flight path via `WorkerPoolDispatchError`
|
|
525
525
|
* instead of letting timeouts compound indefinitely.
|
|
526
|
+
*
|
|
527
|
+
* Upstream of these layers, the parse worker self-sanitizes a result that the
|
|
528
|
+
* structured-clone algorithm can't serialize (#2112) — stripping or dropping
|
|
529
|
+
* the offending value and reporting the affected paths on the result — so a
|
|
530
|
+
* single non-cloneable value can't masquerade as a worker death and exhaust a
|
|
531
|
+
* slot's respawn budget here.
|
|
526
532
|
*/
|
|
527
533
|
export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
528
534
|
// Validate worker script exists before spawning to prevent uncaught
|
|
@@ -1313,14 +1319,19 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
|
1313
1319
|
if (settled || stopped)
|
|
1314
1320
|
return;
|
|
1315
1321
|
// Native postMessage delivers POJO directly via Node's
|
|
1316
|
-
// structured clone.
|
|
1317
|
-
//
|
|
1318
|
-
//
|
|
1319
|
-
//
|
|
1320
|
-
//
|
|
1321
|
-
//
|
|
1322
|
-
// `
|
|
1323
|
-
//
|
|
1322
|
+
// structured clone. Two distinct clone failure modes exist,
|
|
1323
|
+
// and NEITHER reaches this handler: (1) a SENDER-side
|
|
1324
|
+
// non-cloneable value (a function/symbol that leaked into the
|
|
1325
|
+
// result) throws a synchronous `DataCloneError` on the
|
|
1326
|
+
// worker's own postMessage — the parse worker self-sanitizes
|
|
1327
|
+
// such results before delivery (#2112) and falls back to a
|
|
1328
|
+
// primitive-only `{type:'error'}` if it still can't serialize;
|
|
1329
|
+
// (2) a RECEIVER-side deserialization failure surfaces as a
|
|
1330
|
+
// `messageerror` event handled below. The only thing THIS
|
|
1331
|
+
// handler guards is a worker that sends a message without a
|
|
1332
|
+
// `type` discriminant (a worker bug, not a wire-format issue):
|
|
1333
|
+
// without the guard `null.type` would throw a TypeError out of
|
|
1334
|
+
// the EventEmitter listener → uncaughtException on the main
|
|
1324
1335
|
// thread.
|
|
1325
1336
|
const msg = raw;
|
|
1326
1337
|
if (msg === null || typeof msg !== 'object' || typeof msg.type !== 'string') {
|
|
@@ -1417,12 +1428,15 @@ export const createWorkerPool = (workerUrl, poolSize, options) => {
|
|
|
1417
1428
|
`Likely OOM or native addon failure${inFlightSuffix}.`, excludes);
|
|
1418
1429
|
}
|
|
1419
1430
|
};
|
|
1420
|
-
// `messageerror` fires when V8 fails to
|
|
1421
|
-
// payload (
|
|
1422
|
-
//
|
|
1423
|
-
//
|
|
1424
|
-
//
|
|
1425
|
-
//
|
|
1431
|
+
// `messageerror` fires when V8 fails to DESERIALIZE a postMessage
|
|
1432
|
+
// payload on THIS (receiver) side — a value that serialized on the
|
|
1433
|
+
// worker but can't be reconstructed here. (A non-cloneable value on
|
|
1434
|
+
// the SENDER side instead throws a synchronous DataCloneError on the
|
|
1435
|
+
// worker's own postMessage; that path is caught and sanitized
|
|
1436
|
+
// worker-side (#2112) and never arrives here.) The worker stays ALIVE
|
|
1437
|
+
// but the message is lost — without this handler the pool would sit on
|
|
1438
|
+
// the dropped message until the idle timeout expires. Treat it as
|
|
1439
|
+
// worker death so the resilience layers fire:
|
|
1426
1440
|
// requeue the remainder via `recoverAndResume`, attribute the
|
|
1427
1441
|
// in-flight file from the `starting-file` signal (if observed),
|
|
1428
1442
|
// and let the per-slot respawn budget and circuit breaker decide
|
|
@@ -233,6 +233,25 @@ export declare const loadVectorExtension: (targetConn?: lbug.Connection, opts?:
|
|
|
233
233
|
* @param stemmer - Stemming algorithm (default: 'porter')
|
|
234
234
|
*/
|
|
235
235
|
export declare const createFTSIndex: (tableName: string, indexName: string, properties: string[], stemmer?: string) => Promise<void>;
|
|
236
|
+
/**
|
|
237
|
+
* Create the HNSW vector index on the CodeEmbedding table.
|
|
238
|
+
*
|
|
239
|
+
* MUST run via `conn.query()` (here through `queryAndDrain`), NOT through the
|
|
240
|
+
* prepared `executeQuery`/`conn.prepare()` path: `CALL CREATE_VECTOR_INDEX(...)`
|
|
241
|
+
* compiles to multiple statements, which LadybugDB cannot prepare — it fails
|
|
242
|
+
* with "Connection Exception: We do not support prepare multiple statements."
|
|
243
|
+
* Routing index creation through `executeQuery` (prepared) is exactly what
|
|
244
|
+
* broke vector-index creation during `analyze` (#2114; the singleton
|
|
245
|
+
* `executeQuery` was switched to the prepared path in #1655 while FTS index
|
|
246
|
+
* creation kept using `conn.query()`, which is why FTS survived and VECTOR did
|
|
247
|
+
* not). Mirrors `createFTSIndex` above.
|
|
248
|
+
*
|
|
249
|
+
* Returns `true` on success (or when the index already exists — idempotent so
|
|
250
|
+
* incremental re-runs don't spuriously downgrade to exact scan), `false` when
|
|
251
|
+
* the VECTOR extension is unavailable or the connection is read-only. Any other
|
|
252
|
+
* failure propagates so the caller can log it.
|
|
253
|
+
*/
|
|
254
|
+
export declare const createVectorIndex: () => Promise<boolean>;
|
|
236
255
|
/**
|
|
237
256
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
238
257
|
*
|
|
@@ -8,7 +8,7 @@ import os from 'os';
|
|
|
8
8
|
import crypto from 'crypto';
|
|
9
9
|
import lbug from '@ladybugdb/core';
|
|
10
10
|
import { closeQueryResults } from './query-result-utils.js';
|
|
11
|
-
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, STALE_HASH_SENTINEL, } from './schema.js';
|
|
11
|
+
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, CREATE_VECTOR_INDEX_QUERY, STALE_HASH_SENTINEL, } from './schema.js';
|
|
12
12
|
import { streamAllCSVsToDisk } from './csv-generator.js';
|
|
13
13
|
import { extensionManager } from './extension-loader.js';
|
|
14
14
|
import { closeLbugConnection, isDbBusyError, isOpenRetryExhausted, isWalCorruptionError, openLbugConnection, toNativeSafePath, WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, } from './lbug-config.js';
|
|
@@ -119,6 +119,11 @@ let currentDbPath = null;
|
|
|
119
119
|
let currentDbReadOnly = false;
|
|
120
120
|
let ftsLoaded = false;
|
|
121
121
|
let vectorExtensionLoaded = false;
|
|
122
|
+
// In-process guard so a repeated createVectorIndex() within one connection
|
|
123
|
+
// lifetime skips the DB round-trip (mirrors ensuredFTSIndexes). Reset wherever
|
|
124
|
+
// vectorExtensionLoaded resets, so it can never stay true against a swapped or
|
|
125
|
+
// closed connection.
|
|
126
|
+
let vectorIndexEnsured = false;
|
|
122
127
|
/**
|
|
123
128
|
* In-process cache of FTS indexes observed against the current singleton
|
|
124
129
|
* connection. Avoids repeated `CALL CREATE_FTS_INDEX` calls, which can trip
|
|
@@ -494,6 +499,7 @@ const resetOpenConnectionState = () => {
|
|
|
494
499
|
currentDbPath = null;
|
|
495
500
|
ftsLoaded = false;
|
|
496
501
|
vectorExtensionLoaded = false;
|
|
502
|
+
vectorIndexEnsured = false;
|
|
497
503
|
ensuredFTSIndexes.clear();
|
|
498
504
|
};
|
|
499
505
|
const runSchemaCreationQueries = async (dbPath) => {
|
|
@@ -572,6 +578,7 @@ export const withLbugDb = async (dbPath, operation, options = {}) => {
|
|
|
572
578
|
currentDbPath = null;
|
|
573
579
|
ftsLoaded = false;
|
|
574
580
|
vectorExtensionLoaded = false;
|
|
581
|
+
vectorIndexEnsured = false;
|
|
575
582
|
ensuredFTSIndexes.clear();
|
|
576
583
|
});
|
|
577
584
|
// Sleep outside the lock — no need to block others while waiting
|
|
@@ -596,6 +603,7 @@ const doInitLbug = async (dbPath, readOnly = false) => {
|
|
|
596
603
|
currentDbPath = null;
|
|
597
604
|
ftsLoaded = false;
|
|
598
605
|
vectorExtensionLoaded = false;
|
|
606
|
+
vectorIndexEnsured = false;
|
|
599
607
|
ensuredFTSIndexes.clear();
|
|
600
608
|
}
|
|
601
609
|
// ---------------------------------------------------------------------------
|
|
@@ -1476,6 +1484,7 @@ export const closeLbug = async () => {
|
|
|
1476
1484
|
currentDbPath = null;
|
|
1477
1485
|
ftsLoaded = false;
|
|
1478
1486
|
vectorExtensionLoaded = false;
|
|
1487
|
+
vectorIndexEnsured = false;
|
|
1479
1488
|
ensuredFTSIndexes.clear();
|
|
1480
1489
|
};
|
|
1481
1490
|
export const isLbugReady = () => conn !== null && db !== null;
|
|
@@ -1708,6 +1717,52 @@ export const createFTSIndex = async (tableName, indexName, properties, stemmer =
|
|
|
1708
1717
|
throw e;
|
|
1709
1718
|
}
|
|
1710
1719
|
};
|
|
1720
|
+
/**
|
|
1721
|
+
* Create the HNSW vector index on the CodeEmbedding table.
|
|
1722
|
+
*
|
|
1723
|
+
* MUST run via `conn.query()` (here through `queryAndDrain`), NOT through the
|
|
1724
|
+
* prepared `executeQuery`/`conn.prepare()` path: `CALL CREATE_VECTOR_INDEX(...)`
|
|
1725
|
+
* compiles to multiple statements, which LadybugDB cannot prepare — it fails
|
|
1726
|
+
* with "Connection Exception: We do not support prepare multiple statements."
|
|
1727
|
+
* Routing index creation through `executeQuery` (prepared) is exactly what
|
|
1728
|
+
* broke vector-index creation during `analyze` (#2114; the singleton
|
|
1729
|
+
* `executeQuery` was switched to the prepared path in #1655 while FTS index
|
|
1730
|
+
* creation kept using `conn.query()`, which is why FTS survived and VECTOR did
|
|
1731
|
+
* not). Mirrors `createFTSIndex` above.
|
|
1732
|
+
*
|
|
1733
|
+
* Returns `true` on success (or when the index already exists — idempotent so
|
|
1734
|
+
* incremental re-runs don't spuriously downgrade to exact scan), `false` when
|
|
1735
|
+
* the VECTOR extension is unavailable or the connection is read-only. Any other
|
|
1736
|
+
* failure propagates so the caller can log it.
|
|
1737
|
+
*/
|
|
1738
|
+
export const createVectorIndex = async () => {
|
|
1739
|
+
if (!conn) {
|
|
1740
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1741
|
+
}
|
|
1742
|
+
// Already built on this connection — skip the round-trip (mirrors createFTSIndex).
|
|
1743
|
+
if (vectorIndexEnsured)
|
|
1744
|
+
return true;
|
|
1745
|
+
if (!(await loadVectorExtension())) {
|
|
1746
|
+
return false;
|
|
1747
|
+
}
|
|
1748
|
+
try {
|
|
1749
|
+
await queryAndDrain(conn, CREATE_VECTOR_INDEX_QUERY);
|
|
1750
|
+
vectorIndexEnsured = true;
|
|
1751
|
+
return true;
|
|
1752
|
+
}
|
|
1753
|
+
catch (e) {
|
|
1754
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
1755
|
+
// Idempotent: a prior analyze already built the HNSW index.
|
|
1756
|
+
if (msg.includes('already exists')) {
|
|
1757
|
+
vectorIndexEnsured = true;
|
|
1758
|
+
return true;
|
|
1759
|
+
}
|
|
1760
|
+
// Read-only DB (e.g. the MCP query pool): writable analyze owns creation.
|
|
1761
|
+
if (isReadOnlyDbError(e))
|
|
1762
|
+
return false;
|
|
1763
|
+
throw e;
|
|
1764
|
+
}
|
|
1765
|
+
};
|
|
1711
1766
|
/**
|
|
1712
1767
|
* Lazy-create an FTS index, caching the fact in-process.
|
|
1713
1768
|
*
|
|
@@ -55,6 +55,16 @@ export interface AnalyzeOptions {
|
|
|
55
55
|
* "main" fallback for non-CLI callers (e.g. the server analyze worker).
|
|
56
56
|
*/
|
|
57
57
|
defaultBranch?: string;
|
|
58
|
+
/**
|
|
59
|
+
* Index-branch selector (#2106). Distinct from `defaultBranch` (which only
|
|
60
|
+
* affects generated AGENTS.md/CLAUDE.md base_ref text). When set, this run is
|
|
61
|
+
* labelled as that branch and routed to a per-branch index slot unless it is
|
|
62
|
+
* the primary branch. When `undefined`, the branch is auto-detected from the
|
|
63
|
+
* checked-out HEAD (the flat/primary slot for the first-indexed branch, a
|
|
64
|
+
* `branches/<slug>/` sub-directory otherwise). Detached HEAD / non-git always
|
|
65
|
+
* maps to the flat slot.
|
|
66
|
+
*/
|
|
67
|
+
branch?: string;
|
|
58
68
|
/**
|
|
59
69
|
* User-provided alias for the registry `name` (#829). When set,
|
|
60
70
|
* forwarded to `registerRepo` so the indexed repo is stored under
|
|
@@ -78,6 +88,13 @@ export interface AnalyzeOptions {
|
|
|
78
88
|
* removed); `undefined` defers to the env / auto-formula fallback.
|
|
79
89
|
*/
|
|
80
90
|
workerPoolSize?: number;
|
|
91
|
+
/**
|
|
92
|
+
* Extra fetch-wrapper function names to treat as HTTP consumers, forwarded to
|
|
93
|
+
* `PipelineOptions.fetchWrappers` (#1589/#1852 residual). Sourced from the CLI
|
|
94
|
+
* `.gitnexusrc` `fetchWrappers` list. `undefined`/empty leaves the route
|
|
95
|
+
* consumer scan unchanged.
|
|
96
|
+
*/
|
|
97
|
+
fetchWrappers?: string[];
|
|
81
98
|
}
|
|
82
99
|
export interface AnalyzeResult {
|
|
83
100
|
repoName: string;
|
|
@@ -102,6 +119,13 @@ export interface AnalyzeResult {
|
|
|
102
119
|
* the persisted meta surface the degraded state instead of reporting healthy.
|
|
103
120
|
*/
|
|
104
121
|
ftsSkipped?: boolean;
|
|
122
|
+
/**
|
|
123
|
+
* True when the index this run produced/validated is the primary/flat slot
|
|
124
|
+
* (#2106 R2). `false` for a non-primary branch index. Lets the CLI skip
|
|
125
|
+
* repo-root AGENTS.md/CLAUDE.md refreshes (e.g. the base_ref fast-path) for a
|
|
126
|
+
* branch analyze, mirroring the in-pipeline `if (!placement.branch)` gate.
|
|
127
|
+
*/
|
|
128
|
+
isPrimaryBranch?: boolean;
|
|
105
129
|
}
|
|
106
130
|
export { deriveEmbeddingMode, DEFAULT_EMBEDDING_NODE_LIMIT } from './embedding-mode.js';
|
|
107
131
|
export type { EmbeddingMode } from './embedding-mode.js';
|
|
@@ -117,4 +141,25 @@ export declare const PHASE_LABELS: Record<string, string>;
|
|
|
117
141
|
* the {@link AnalyzeCallbacks} interface — it never writes to stdout/stderr
|
|
118
142
|
* directly and never calls `process.exit()`.
|
|
119
143
|
*/
|
|
144
|
+
/**
|
|
145
|
+
* Build the primary-inversion warning (#2106 R8), or `undefined` when there is
|
|
146
|
+
* nothing to warn about. Pure + exported for testing. Both inputs are trimmed
|
|
147
|
+
* (a diagnostic — a missed warning is low-harm; a false warning is the thing to
|
|
148
|
+
* avoid). `defaultBranch` is the repo's `origin/HEAD` branch (null when unset,
|
|
149
|
+
* e.g. fresh clones / CI), `flatOwner` is the branch that owns the flat slot.
|
|
150
|
+
*/
|
|
151
|
+
export declare const primaryInversionWarning: (defaultBranch: string | null | undefined, flatOwner: string | null | undefined) => string | undefined;
|
|
152
|
+
/**
|
|
153
|
+
* Collect the recorded parse-cache chunk keys across the flat + every branch
|
|
154
|
+
* meta under a flat `.gitnexus` storage, EXCLUDING `excludeDir` (the current
|
|
155
|
+
* run's own meta dir) so a single-branch repo collects nothing and its prune
|
|
156
|
+
* stays byte-identical to today (#2106 R6). `complete` is false when a sibling
|
|
157
|
+
* meta.json exists but fails to parse — callers then retain the whole shared
|
|
158
|
+
* cache rather than over-evict another branch's still-live shards. Exported for
|
|
159
|
+
* testing.
|
|
160
|
+
*/
|
|
161
|
+
export declare const collectBranchCacheKeys: (storagePath: string, excludeDir?: string) => Promise<{
|
|
162
|
+
keys: Set<string>;
|
|
163
|
+
complete: boolean;
|
|
164
|
+
}>;
|
|
120
165
|
export declare function runFullAnalysis(repoPath: string, options: AnalyzeOptions, callbacks: AnalyzeCallbacks): Promise<AnalyzeResult>;
|