gitnexus 1.4.10 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dist/cli/ai-context.d.ts +4 -1
- package/dist/cli/ai-context.js +19 -11
- package/dist/cli/analyze.d.ts +6 -0
- package/dist/cli/analyze.js +105 -251
- package/dist/cli/eval-server.js +20 -11
- package/dist/cli/index-repo.js +20 -22
- package/dist/cli/index.js +8 -7
- package/dist/cli/mcp.js +1 -1
- package/dist/cli/serve.js +29 -1
- package/dist/cli/setup.js +9 -9
- package/dist/cli/skill-gen.js +15 -9
- package/dist/cli/wiki.d.ts +2 -0
- package/dist/cli/wiki.js +141 -26
- package/dist/config/ignore-service.js +102 -22
- package/dist/config/supported-languages.d.ts +8 -42
- package/dist/config/supported-languages.js +8 -43
- package/dist/core/augmentation/engine.js +19 -7
- package/dist/core/embeddings/embedder.js +19 -15
- package/dist/core/embeddings/embedding-pipeline.js +6 -6
- package/dist/core/embeddings/http-client.js +3 -3
- package/dist/core/embeddings/text-generator.js +9 -24
- package/dist/core/embeddings/types.d.ts +1 -1
- package/dist/core/embeddings/types.js +1 -7
- package/dist/core/graph/graph.js +6 -2
- package/dist/core/graph/types.d.ts +9 -59
- package/dist/core/ingestion/ast-cache.js +3 -3
- package/dist/core/ingestion/call-processor.d.ts +20 -2
- package/dist/core/ingestion/call-processor.js +347 -144
- package/dist/core/ingestion/call-routing.js +10 -4
- package/dist/core/ingestion/call-sites/extract-language-call-site.d.ts +10 -0
- package/dist/core/ingestion/call-sites/extract-language-call-site.js +22 -0
- package/dist/core/ingestion/call-sites/java.d.ts +9 -0
- package/dist/core/ingestion/call-sites/java.js +30 -0
- package/dist/core/ingestion/cluster-enricher.js +6 -8
- package/dist/core/ingestion/cobol/cobol-copy-expander.js +10 -3
- package/dist/core/ingestion/cobol/cobol-preprocessor.js +287 -81
- package/dist/core/ingestion/cobol/jcl-parser.js +1 -1
- package/dist/core/ingestion/cobol/jcl-processor.js +1 -1
- package/dist/core/ingestion/cobol-processor.js +102 -56
- package/dist/core/ingestion/community-processor.js +21 -15
- package/dist/core/ingestion/entry-point-scoring.d.ts +1 -1
- package/dist/core/ingestion/entry-point-scoring.js +5 -6
- package/dist/core/ingestion/export-detection.js +32 -9
- package/dist/core/ingestion/field-extractor.d.ts +1 -1
- package/dist/core/ingestion/field-extractors/configs/c-cpp.js +8 -12
- package/dist/core/ingestion/field-extractors/configs/csharp.js +45 -2
- package/dist/core/ingestion/field-extractors/configs/dart.js +5 -3
- package/dist/core/ingestion/field-extractors/configs/go.js +3 -7
- package/dist/core/ingestion/field-extractors/configs/helpers.d.ts +5 -0
- package/dist/core/ingestion/field-extractors/configs/helpers.js +14 -0
- package/dist/core/ingestion/field-extractors/configs/jvm.js +7 -7
- package/dist/core/ingestion/field-extractors/configs/php.js +9 -11
- package/dist/core/ingestion/field-extractors/configs/python.js +1 -1
- package/dist/core/ingestion/field-extractors/configs/ruby.js +4 -3
- package/dist/core/ingestion/field-extractors/configs/rust.js +2 -5
- package/dist/core/ingestion/field-extractors/configs/swift.js +9 -7
- package/dist/core/ingestion/field-extractors/configs/typescript-javascript.js +2 -6
- package/dist/core/ingestion/field-extractors/generic.d.ts +5 -2
- package/dist/core/ingestion/field-extractors/generic.js +6 -0
- package/dist/core/ingestion/field-extractors/typescript.d.ts +1 -1
- package/dist/core/ingestion/field-extractors/typescript.js +1 -1
- package/dist/core/ingestion/field-types.d.ts +4 -2
- package/dist/core/ingestion/filesystem-walker.js +3 -3
- package/dist/core/ingestion/framework-detection.d.ts +1 -1
- package/dist/core/ingestion/framework-detection.js +355 -85
- package/dist/core/ingestion/heritage-processor.d.ts +24 -0
- package/dist/core/ingestion/heritage-processor.js +99 -8
- package/dist/core/ingestion/import-processor.js +44 -15
- package/dist/core/ingestion/import-resolvers/csharp.js +7 -3
- package/dist/core/ingestion/import-resolvers/dart.js +1 -1
- package/dist/core/ingestion/import-resolvers/go.js +4 -2
- package/dist/core/ingestion/import-resolvers/jvm.js +4 -4
- package/dist/core/ingestion/import-resolvers/php.js +4 -4
- package/dist/core/ingestion/import-resolvers/python.js +1 -1
- package/dist/core/ingestion/import-resolvers/rust.js +9 -3
- package/dist/core/ingestion/import-resolvers/standard.d.ts +1 -1
- package/dist/core/ingestion/import-resolvers/standard.js +6 -5
- package/dist/core/ingestion/import-resolvers/swift.js +2 -1
- package/dist/core/ingestion/import-resolvers/utils.js +26 -7
- package/dist/core/ingestion/language-config.js +5 -4
- package/dist/core/ingestion/language-provider.d.ts +7 -2
- package/dist/core/ingestion/languages/c-cpp.js +106 -21
- package/dist/core/ingestion/languages/cobol.js +1 -1
- package/dist/core/ingestion/languages/csharp.js +96 -19
- package/dist/core/ingestion/languages/dart.js +23 -7
- package/dist/core/ingestion/languages/go.js +1 -1
- package/dist/core/ingestion/languages/index.d.ts +1 -1
- package/dist/core/ingestion/languages/index.js +2 -3
- package/dist/core/ingestion/languages/java.js +4 -1
- package/dist/core/ingestion/languages/kotlin.js +60 -13
- package/dist/core/ingestion/languages/php.js +102 -25
- package/dist/core/ingestion/languages/python.js +28 -5
- package/dist/core/ingestion/languages/ruby.js +56 -14
- package/dist/core/ingestion/languages/rust.js +55 -11
- package/dist/core/ingestion/languages/swift.js +112 -27
- package/dist/core/ingestion/languages/typescript.js +95 -19
- package/dist/core/ingestion/markdown-processor.js +5 -5
- package/dist/core/ingestion/method-extractors/configs/csharp.d.ts +2 -0
- package/dist/core/ingestion/method-extractors/configs/csharp.js +283 -0
- package/dist/core/ingestion/method-extractors/configs/jvm.d.ts +3 -0
- package/dist/core/ingestion/method-extractors/configs/jvm.js +326 -0
- package/dist/core/ingestion/method-extractors/generic.d.ts +5 -0
- package/dist/core/ingestion/method-extractors/generic.js +137 -0
- package/dist/core/ingestion/method-types.d.ts +61 -0
- package/dist/core/ingestion/method-types.js +2 -0
- package/dist/core/ingestion/mro-processor.d.ts +1 -1
- package/dist/core/ingestion/mro-processor.js +12 -8
- package/dist/core/ingestion/named-binding-processor.js +2 -2
- package/dist/core/ingestion/named-bindings/rust.js +3 -1
- package/dist/core/ingestion/parsing-processor.js +74 -24
- package/dist/core/ingestion/pipeline.d.ts +2 -1
- package/dist/core/ingestion/pipeline.js +208 -102
- package/dist/core/ingestion/process-processor.js +12 -10
- package/dist/core/ingestion/resolution-context.js +3 -3
- package/dist/core/ingestion/route-extractors/middleware.js +31 -7
- package/dist/core/ingestion/route-extractors/php.js +2 -1
- package/dist/core/ingestion/route-extractors/response-shapes.js +8 -4
- package/dist/core/ingestion/structure-processor.d.ts +1 -1
- package/dist/core/ingestion/structure-processor.js +4 -4
- package/dist/core/ingestion/symbol-table.d.ts +1 -1
- package/dist/core/ingestion/symbol-table.js +22 -6
- package/dist/core/ingestion/tree-sitter-queries.d.ts +1 -1
- package/dist/core/ingestion/tree-sitter-queries.js +1 -1
- package/dist/core/ingestion/type-env.d.ts +2 -2
- package/dist/core/ingestion/type-env.js +75 -50
- package/dist/core/ingestion/type-extractors/c-cpp.js +33 -30
- package/dist/core/ingestion/type-extractors/csharp.js +24 -14
- package/dist/core/ingestion/type-extractors/dart.js +6 -8
- package/dist/core/ingestion/type-extractors/go.js +7 -6
- package/dist/core/ingestion/type-extractors/jvm.js +10 -21
- package/dist/core/ingestion/type-extractors/php.js +26 -13
- package/dist/core/ingestion/type-extractors/python.js +11 -15
- package/dist/core/ingestion/type-extractors/ruby.js +8 -3
- package/dist/core/ingestion/type-extractors/rust.js +6 -8
- package/dist/core/ingestion/type-extractors/shared.js +134 -50
- package/dist/core/ingestion/type-extractors/swift.js +16 -13
- package/dist/core/ingestion/type-extractors/typescript.js +23 -15
- package/dist/core/ingestion/utils/ast-helpers.d.ts +8 -8
- package/dist/core/ingestion/utils/ast-helpers.js +72 -35
- package/dist/core/ingestion/utils/call-analysis.d.ts +2 -0
- package/dist/core/ingestion/utils/call-analysis.js +96 -49
- package/dist/core/ingestion/utils/event-loop.js +1 -1
- package/dist/core/ingestion/workers/parse-worker.d.ts +7 -2
- package/dist/core/ingestion/workers/parse-worker.js +364 -84
- package/dist/core/ingestion/workers/worker-pool.js +5 -10
- package/dist/core/lbug/csv-generator.js +54 -15
- package/dist/core/lbug/lbug-adapter.d.ts +5 -0
- package/dist/core/lbug/lbug-adapter.js +86 -23
- package/dist/core/lbug/schema.d.ts +3 -6
- package/dist/core/lbug/schema.js +6 -30
- package/dist/core/run-analyze.d.ts +49 -0
- package/dist/core/run-analyze.js +257 -0
- package/dist/core/tree-sitter/parser-loader.d.ts +1 -1
- package/dist/core/tree-sitter/parser-loader.js +1 -1
- package/dist/core/wiki/cursor-client.js +2 -7
- package/dist/core/wiki/generator.js +38 -23
- package/dist/core/wiki/graph-queries.js +10 -10
- package/dist/core/wiki/html-viewer.js +7 -3
- package/dist/core/wiki/llm-client.d.ts +23 -2
- package/dist/core/wiki/llm-client.js +96 -26
- package/dist/core/wiki/prompts.js +7 -6
- package/dist/mcp/core/embedder.js +1 -1
- package/dist/mcp/core/lbug-adapter.d.ts +4 -1
- package/dist/mcp/core/lbug-adapter.js +17 -7
- package/dist/mcp/local/local-backend.js +247 -95
- package/dist/mcp/resources.js +14 -6
- package/dist/mcp/server.js +13 -5
- package/dist/mcp/staleness.js +5 -1
- package/dist/mcp/tools.js +100 -23
- package/dist/server/analyze-job.d.ts +53 -0
- package/dist/server/analyze-job.js +146 -0
- package/dist/server/analyze-worker.d.ts +13 -0
- package/dist/server/analyze-worker.js +59 -0
- package/dist/server/api.js +795 -44
- package/dist/server/git-clone.d.ts +25 -0
- package/dist/server/git-clone.js +91 -0
- package/dist/storage/git.js +1 -3
- package/dist/storage/repo-manager.d.ts +5 -2
- package/dist/storage/repo-manager.js +4 -4
- package/dist/types/pipeline.d.ts +1 -21
- package/dist/types/pipeline.js +1 -18
- package/hooks/claude/gitnexus-hook.cjs +52 -22
- package/package.json +3 -2
- package/dist/core/ingestion/utils/language-detection.d.ts +0 -9
- package/dist/core/ingestion/utils/language-detection.js +0 -70
|
@@ -92,41 +92,122 @@ const DEFAULT_IGNORE_LIST = new Set([
|
|
|
92
92
|
]);
|
|
93
93
|
const IGNORED_EXTENSIONS = new Set([
|
|
94
94
|
// Images
|
|
95
|
-
'.png',
|
|
96
|
-
'.
|
|
95
|
+
'.png',
|
|
96
|
+
'.jpg',
|
|
97
|
+
'.jpeg',
|
|
98
|
+
'.gif',
|
|
99
|
+
'.svg',
|
|
100
|
+
'.ico',
|
|
101
|
+
'.webp',
|
|
102
|
+
'.bmp',
|
|
103
|
+
'.tiff',
|
|
104
|
+
'.tif',
|
|
105
|
+
'.psd',
|
|
106
|
+
'.ai',
|
|
107
|
+
'.sketch',
|
|
108
|
+
'.fig',
|
|
109
|
+
'.xd',
|
|
97
110
|
// Archives
|
|
98
|
-
'.zip',
|
|
111
|
+
'.zip',
|
|
112
|
+
'.tar',
|
|
113
|
+
'.gz',
|
|
114
|
+
'.rar',
|
|
115
|
+
'.7z',
|
|
116
|
+
'.bz2',
|
|
117
|
+
'.xz',
|
|
118
|
+
'.tgz',
|
|
99
119
|
// Binary/Compiled
|
|
100
|
-
'.exe',
|
|
101
|
-
'.
|
|
102
|
-
'.
|
|
120
|
+
'.exe',
|
|
121
|
+
'.dll',
|
|
122
|
+
'.so',
|
|
123
|
+
'.dylib',
|
|
124
|
+
'.a',
|
|
125
|
+
'.lib',
|
|
126
|
+
'.o',
|
|
127
|
+
'.obj',
|
|
128
|
+
'.class',
|
|
129
|
+
'.jar',
|
|
130
|
+
'.war',
|
|
131
|
+
'.ear',
|
|
132
|
+
'.pyc',
|
|
133
|
+
'.pyo',
|
|
134
|
+
'.pyd',
|
|
103
135
|
'.beam', // Erlang
|
|
104
136
|
'.wasm', // WebAssembly - important!
|
|
105
137
|
'.node', // Native Node addons
|
|
106
138
|
// Documents
|
|
107
|
-
'.pdf',
|
|
108
|
-
'.
|
|
139
|
+
'.pdf',
|
|
140
|
+
'.doc',
|
|
141
|
+
'.docx',
|
|
142
|
+
'.xls',
|
|
143
|
+
'.xlsx',
|
|
144
|
+
'.ppt',
|
|
145
|
+
'.pptx',
|
|
146
|
+
'.odt',
|
|
147
|
+
'.ods',
|
|
148
|
+
'.odp',
|
|
109
149
|
// Media
|
|
110
|
-
'.mp4',
|
|
111
|
-
'.
|
|
150
|
+
'.mp4',
|
|
151
|
+
'.mp3',
|
|
152
|
+
'.wav',
|
|
153
|
+
'.mov',
|
|
154
|
+
'.avi',
|
|
155
|
+
'.mkv',
|
|
156
|
+
'.flv',
|
|
157
|
+
'.wmv',
|
|
158
|
+
'.ogg',
|
|
159
|
+
'.webm',
|
|
160
|
+
'.flac',
|
|
161
|
+
'.aac',
|
|
162
|
+
'.m4a',
|
|
112
163
|
// Fonts
|
|
113
|
-
'.woff',
|
|
164
|
+
'.woff',
|
|
165
|
+
'.woff2',
|
|
166
|
+
'.ttf',
|
|
167
|
+
'.eot',
|
|
168
|
+
'.otf',
|
|
114
169
|
// Databases
|
|
115
|
-
'.db',
|
|
170
|
+
'.db',
|
|
171
|
+
'.sqlite',
|
|
172
|
+
'.sqlite3',
|
|
173
|
+
'.mdb',
|
|
174
|
+
'.accdb',
|
|
116
175
|
// Minified/Bundled files
|
|
117
|
-
'.min.js',
|
|
176
|
+
'.min.js',
|
|
177
|
+
'.min.css',
|
|
178
|
+
'.bundle.js',
|
|
179
|
+
'.chunk.js',
|
|
118
180
|
// Source maps (debug files, not source)
|
|
119
181
|
'.map',
|
|
120
182
|
// Lock files (handled separately, but also here)
|
|
121
183
|
'.lock',
|
|
122
184
|
// Certificates & Keys (security - don't index!)
|
|
123
|
-
'.pem',
|
|
185
|
+
'.pem',
|
|
186
|
+
'.key',
|
|
187
|
+
'.crt',
|
|
188
|
+
'.cer',
|
|
189
|
+
'.p12',
|
|
190
|
+
'.pfx',
|
|
124
191
|
// Data files (often large/binary)
|
|
125
|
-
'.csv',
|
|
126
|
-
'.
|
|
192
|
+
'.csv',
|
|
193
|
+
'.tsv',
|
|
194
|
+
'.parquet',
|
|
195
|
+
'.avro',
|
|
196
|
+
'.feather',
|
|
197
|
+
'.npy',
|
|
198
|
+
'.npz',
|
|
199
|
+
'.pkl',
|
|
200
|
+
'.pickle',
|
|
201
|
+
'.h5',
|
|
202
|
+
'.hdf5',
|
|
127
203
|
// Misc binary
|
|
128
|
-
'.bin',
|
|
129
|
-
'.
|
|
204
|
+
'.bin',
|
|
205
|
+
'.dat',
|
|
206
|
+
'.data',
|
|
207
|
+
'.raw',
|
|
208
|
+
'.iso',
|
|
209
|
+
'.img',
|
|
210
|
+
'.dmg',
|
|
130
211
|
]);
|
|
131
212
|
// Files to ignore by exact name
|
|
132
213
|
const IGNORED_FILES = new Set([
|
|
@@ -208,7 +289,8 @@ export const shouldIgnorePath = (filePath) => {
|
|
|
208
289
|
if (fileNameLower.includes('.bundle.') ||
|
|
209
290
|
fileNameLower.includes('.chunk.') ||
|
|
210
291
|
fileNameLower.includes('.generated.') ||
|
|
211
|
-
fileNameLower.endsWith('.d.ts')) {
|
|
292
|
+
fileNameLower.endsWith('.d.ts')) {
|
|
293
|
+
// TypeScript declaration files
|
|
212
294
|
return true;
|
|
213
295
|
}
|
|
214
296
|
return false;
|
|
@@ -222,9 +304,7 @@ export const loadIgnoreRules = async (repoPath, options) => {
|
|
|
222
304
|
let hasRules = false;
|
|
223
305
|
// Allow users to bypass .gitignore parsing (e.g. when .gitignore accidentally excludes source files)
|
|
224
306
|
const skipGitignore = options?.noGitignore ?? !!process.env.GITNEXUS_NO_GITIGNORE;
|
|
225
|
-
const filenames = skipGitignore
|
|
226
|
-
? ['.gitnexusignore']
|
|
227
|
-
: ['.gitignore', '.gitnexusignore'];
|
|
307
|
+
const filenames = skipGitignore ? ['.gitnexusignore'] : ['.gitignore', '.gitnexusignore'];
|
|
228
308
|
for (const filename of filenames) {
|
|
229
309
|
try {
|
|
230
310
|
const content = await fs.readFile(nodePath.join(repoPath, filename), 'utf-8');
|
|
@@ -1,47 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
+
* Re-export SupportedLanguages from gitnexus-shared (single source of truth).
|
|
3
|
+
*
|
|
2
4
|
* HOW TO ADD A NEW LANGUAGE:
|
|
3
5
|
*
|
|
4
|
-
* 1. Add the enum member
|
|
6
|
+
* 1. Add the enum member in gitnexus-shared/src/languages.ts
|
|
5
7
|
* 2. Run `tsc --noEmit` — compiler errors guide you to every dispatch table
|
|
6
|
-
* 3. Use
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* tree-sitter-queries.ts | Query string + LANGUAGE_QUERIES entry | (required)
|
|
11
|
-
* export-detection.ts | ExportChecker function + table entry | (required)
|
|
12
|
-
* import-resolvers/<lang>.ts | Exported resolve<Lang>Import function | resolveStandard(...)
|
|
13
|
-
* call-routing.ts | CallRouter function (or noRouting) | noRouting
|
|
14
|
-
* entry-point-scoring.ts | ENTRY_POINT_PATTERNS entry | []
|
|
15
|
-
* framework-detection.ts | AST_FRAMEWORK_PATTERNS entry | []
|
|
16
|
-
* type-extractors/<lang>.ts | New file + index.ts import | (required)
|
|
17
|
-
* resolvers/<lang>.ts | Resolver file (if non-standard) | (only if resolveStandard insufficient)
|
|
18
|
-
* named-bindings/<lang>.ts | Extractor (if named imports) | (only if language has named imports)
|
|
19
|
-
*
|
|
20
|
-
* 4. Also check these files for language-specific if-checks (no compile-time guard):
|
|
21
|
-
* - mro-processor.ts (MRO strategy selection)
|
|
22
|
-
* - heritage-processor.ts (extends/implements handling)
|
|
23
|
-
* - parse-worker.ts (AST edge cases)
|
|
24
|
-
* - parsing-processor.ts (node label normalization)
|
|
25
|
-
*
|
|
26
|
-
* 5. Add tree-sitter-<lang> to package.json dependencies
|
|
27
|
-
* 6. Add file extension mapping in utils.ts getLanguageFromFilename()
|
|
28
|
-
* 7. Run full test suite
|
|
8
|
+
* 3. Use the checklist in each ingestion file for what to add
|
|
9
|
+
* 4. Add tree-sitter-<lang> to gitnexus/package.json dependencies
|
|
10
|
+
* 5. Add file extension mapping in utils.ts getLanguageFromFilename()
|
|
11
|
+
* 6. Run full test suite
|
|
29
12
|
*/
|
|
30
|
-
export
|
|
31
|
-
JavaScript = "javascript",
|
|
32
|
-
TypeScript = "typescript",
|
|
33
|
-
Python = "python",
|
|
34
|
-
Java = "java",
|
|
35
|
-
C = "c",
|
|
36
|
-
CPlusPlus = "cpp",
|
|
37
|
-
CSharp = "csharp",
|
|
38
|
-
Go = "go",
|
|
39
|
-
Ruby = "ruby",
|
|
40
|
-
Rust = "rust",
|
|
41
|
-
PHP = "php",
|
|
42
|
-
Kotlin = "kotlin",
|
|
43
|
-
Swift = "swift",
|
|
44
|
-
Dart = "dart",
|
|
45
|
-
/** Standalone regex processor — no tree-sitter, no LanguageProvider. */
|
|
46
|
-
Cobol = "cobol"
|
|
47
|
-
}
|
|
13
|
+
export { SupportedLanguages } from 'gitnexus-shared';
|
|
@@ -1,48 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
+
* Re-export SupportedLanguages from gitnexus-shared (single source of truth).
|
|
3
|
+
*
|
|
2
4
|
* HOW TO ADD A NEW LANGUAGE:
|
|
3
5
|
*
|
|
4
|
-
* 1. Add the enum member
|
|
6
|
+
* 1. Add the enum member in gitnexus-shared/src/languages.ts
|
|
5
7
|
* 2. Run `tsc --noEmit` — compiler errors guide you to every dispatch table
|
|
6
|
-
* 3. Use
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* tree-sitter-queries.ts | Query string + LANGUAGE_QUERIES entry | (required)
|
|
11
|
-
* export-detection.ts | ExportChecker function + table entry | (required)
|
|
12
|
-
* import-resolvers/<lang>.ts | Exported resolve<Lang>Import function | resolveStandard(...)
|
|
13
|
-
* call-routing.ts | CallRouter function (or noRouting) | noRouting
|
|
14
|
-
* entry-point-scoring.ts | ENTRY_POINT_PATTERNS entry | []
|
|
15
|
-
* framework-detection.ts | AST_FRAMEWORK_PATTERNS entry | []
|
|
16
|
-
* type-extractors/<lang>.ts | New file + index.ts import | (required)
|
|
17
|
-
* resolvers/<lang>.ts | Resolver file (if non-standard) | (only if resolveStandard insufficient)
|
|
18
|
-
* named-bindings/<lang>.ts | Extractor (if named imports) | (only if language has named imports)
|
|
19
|
-
*
|
|
20
|
-
* 4. Also check these files for language-specific if-checks (no compile-time guard):
|
|
21
|
-
* - mro-processor.ts (MRO strategy selection)
|
|
22
|
-
* - heritage-processor.ts (extends/implements handling)
|
|
23
|
-
* - parse-worker.ts (AST edge cases)
|
|
24
|
-
* - parsing-processor.ts (node label normalization)
|
|
25
|
-
*
|
|
26
|
-
* 5. Add tree-sitter-<lang> to package.json dependencies
|
|
27
|
-
* 6. Add file extension mapping in utils.ts getLanguageFromFilename()
|
|
28
|
-
* 7. Run full test suite
|
|
8
|
+
* 3. Use the checklist in each ingestion file for what to add
|
|
9
|
+
* 4. Add tree-sitter-<lang> to gitnexus/package.json dependencies
|
|
10
|
+
* 5. Add file extension mapping in utils.ts getLanguageFromFilename()
|
|
11
|
+
* 6. Run full test suite
|
|
29
12
|
*/
|
|
30
|
-
export
|
|
31
|
-
(function (SupportedLanguages) {
|
|
32
|
-
SupportedLanguages["JavaScript"] = "javascript";
|
|
33
|
-
SupportedLanguages["TypeScript"] = "typescript";
|
|
34
|
-
SupportedLanguages["Python"] = "python";
|
|
35
|
-
SupportedLanguages["Java"] = "java";
|
|
36
|
-
SupportedLanguages["C"] = "c";
|
|
37
|
-
SupportedLanguages["CPlusPlus"] = "cpp";
|
|
38
|
-
SupportedLanguages["CSharp"] = "csharp";
|
|
39
|
-
SupportedLanguages["Go"] = "go";
|
|
40
|
-
SupportedLanguages["Ruby"] = "ruby";
|
|
41
|
-
SupportedLanguages["Rust"] = "rust";
|
|
42
|
-
SupportedLanguages["PHP"] = "php";
|
|
43
|
-
SupportedLanguages["Kotlin"] = "kotlin";
|
|
44
|
-
SupportedLanguages["Swift"] = "swift";
|
|
45
|
-
SupportedLanguages["Dart"] = "dart";
|
|
46
|
-
/** Standalone regex processor — no tree-sitter, no LanguageProvider. */
|
|
47
|
-
SupportedLanguages["Cobol"] = "cobol";
|
|
48
|
-
})(SupportedLanguages || (SupportedLanguages = {}));
|
|
13
|
+
export { SupportedLanguages } from 'gitnexus-shared';
|
|
@@ -114,16 +114,20 @@ export async function augment(pattern, cwd) {
|
|
|
114
114
|
});
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
|
-
catch {
|
|
117
|
+
catch {
|
|
118
|
+
/* skip */
|
|
119
|
+
}
|
|
118
120
|
}
|
|
119
121
|
if (symbolMatches.length === 0)
|
|
120
122
|
return '';
|
|
121
123
|
// Step 3: Batch-fetch callers/callees/processes/cohesion for top matches
|
|
122
124
|
// Uses batched WHERE n.id IN [...] queries instead of per-symbol queries
|
|
123
|
-
const uniqueSymbols = symbolMatches
|
|
125
|
+
const uniqueSymbols = symbolMatches
|
|
126
|
+
.slice(0, 5)
|
|
127
|
+
.filter((sym, i, arr) => arr.findIndex((s) => s.nodeId === sym.nodeId) === i);
|
|
124
128
|
if (uniqueSymbols.length === 0)
|
|
125
129
|
return '';
|
|
126
|
-
const idList = uniqueSymbols.map(s => `'${s.nodeId.replace(/'/g, "''")}'`).join(', ');
|
|
130
|
+
const idList = uniqueSymbols.map((s) => `'${s.nodeId.replace(/'/g, "''")}'`).join(', ');
|
|
127
131
|
// Batch fetch callers
|
|
128
132
|
const callersMap = new Map();
|
|
129
133
|
try {
|
|
@@ -143,7 +147,9 @@ export async function augment(pattern, cwd) {
|
|
|
143
147
|
}
|
|
144
148
|
}
|
|
145
149
|
}
|
|
146
|
-
catch {
|
|
150
|
+
catch {
|
|
151
|
+
/* skip */
|
|
152
|
+
}
|
|
147
153
|
// Batch fetch callees
|
|
148
154
|
const calleesMap = new Map();
|
|
149
155
|
try {
|
|
@@ -163,7 +169,9 @@ export async function augment(pattern, cwd) {
|
|
|
163
169
|
}
|
|
164
170
|
}
|
|
165
171
|
}
|
|
166
|
-
catch {
|
|
172
|
+
catch {
|
|
173
|
+
/* skip */
|
|
174
|
+
}
|
|
167
175
|
// Batch fetch processes
|
|
168
176
|
const processesMap = new Map();
|
|
169
177
|
try {
|
|
@@ -184,7 +192,9 @@ export async function augment(pattern, cwd) {
|
|
|
184
192
|
}
|
|
185
193
|
}
|
|
186
194
|
}
|
|
187
|
-
catch {
|
|
195
|
+
catch {
|
|
196
|
+
/* skip */
|
|
197
|
+
}
|
|
188
198
|
// Batch fetch cohesion
|
|
189
199
|
const cohesionMap = new Map();
|
|
190
200
|
try {
|
|
@@ -200,7 +210,9 @@ export async function augment(pattern, cwd) {
|
|
|
200
210
|
cohesionMap.set(nid, coh);
|
|
201
211
|
}
|
|
202
212
|
}
|
|
203
|
-
catch {
|
|
213
|
+
catch {
|
|
214
|
+
/* skip */
|
|
215
|
+
}
|
|
204
216
|
// Assemble enriched results
|
|
205
217
|
const enriched = [];
|
|
206
218
|
for (const sym of uniqueSymbols) {
|
|
@@ -125,8 +125,8 @@ export const initEmbedder = async (onProgress, config = {}, forceDevice) => {
|
|
|
125
125
|
// Probe for CUDA first — ONNX Runtime crashes (uncatchable native error)
|
|
126
126
|
// if we attempt CUDA without the required shared libraries
|
|
127
127
|
const isWindows = process.platform === 'win32';
|
|
128
|
-
const gpuDevice = isWindows ? 'dml' :
|
|
129
|
-
|
|
128
|
+
const gpuDevice = isWindows ? 'dml' : isCudaAvailable() ? 'cuda' : 'cpu';
|
|
129
|
+
const requestedDevice = forceDevice || (finalConfig.device === 'auto' ? gpuDevice : finalConfig.device);
|
|
130
130
|
initPromise = (async () => {
|
|
131
131
|
try {
|
|
132
132
|
// Configure transformers.js environment
|
|
@@ -135,19 +135,21 @@ export const initEmbedder = async (onProgress, config = {}, forceDevice) => {
|
|
|
135
135
|
if (isDev) {
|
|
136
136
|
console.log(`🧠 Loading embedding model: ${finalConfig.modelId}`);
|
|
137
137
|
}
|
|
138
|
-
const progressCallback = onProgress
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
138
|
+
const progressCallback = onProgress
|
|
139
|
+
? (data) => {
|
|
140
|
+
const progress = {
|
|
141
|
+
status: data.status || 'progress',
|
|
142
|
+
file: data.file,
|
|
143
|
+
progress: data.progress,
|
|
144
|
+
loaded: data.loaded,
|
|
145
|
+
total: data.total,
|
|
146
|
+
};
|
|
147
|
+
onProgress(progress);
|
|
148
|
+
}
|
|
149
|
+
: undefined;
|
|
148
150
|
// Try GPU first if auto, fall back to CPU
|
|
149
151
|
// Windows: dml (DirectML/DirectX12), Linux: cuda
|
|
150
|
-
const devicesToTry =
|
|
152
|
+
const devicesToTry = requestedDevice === 'dml' || requestedDevice === 'cuda'
|
|
151
153
|
? [requestedDevice, 'cpu']
|
|
152
154
|
: [requestedDevice];
|
|
153
155
|
for (const device of devicesToTry) {
|
|
@@ -172,8 +174,10 @@ export const initEmbedder = async (onProgress, config = {}, forceDevice) => {
|
|
|
172
174
|
});
|
|
173
175
|
currentDevice = device;
|
|
174
176
|
if (isDev) {
|
|
175
|
-
const label = device === 'dml'
|
|
176
|
-
|
|
177
|
+
const label = device === 'dml'
|
|
178
|
+
? 'GPU (DirectML/DirectX12)'
|
|
179
|
+
: device === 'cuda'
|
|
180
|
+
? 'GPU (CUDA)'
|
|
177
181
|
: device.toUpperCase();
|
|
178
182
|
console.log(`✅ Using ${label} backend`);
|
|
179
183
|
console.log('✅ Embedding model loaded successfully');
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* 4. Update LadybugDB with embeddings
|
|
9
9
|
* 5. Create vector index for semantic search
|
|
10
10
|
*/
|
|
11
|
-
import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady } from './embedder.js';
|
|
11
|
+
import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady, } from './embedder.js';
|
|
12
12
|
import { generateBatchEmbeddingTexts } from './text-generator.js';
|
|
13
13
|
import { DEFAULT_EMBEDDING_CONFIG, EMBEDDABLE_LABELS, } from './types.js';
|
|
14
14
|
const isDev = process.env.NODE_ENV === 'development';
|
|
@@ -69,7 +69,7 @@ const queryEmbeddableNodes = async (executeQuery) => {
|
|
|
69
69
|
const batchInsertEmbeddings = async (executeWithReusedStatement, updates) => {
|
|
70
70
|
// INSERT into separate embedding table - much more memory efficient!
|
|
71
71
|
const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`;
|
|
72
|
-
const paramsList = updates.map(u => ({ nodeId: u.id, embedding: u.embedding }));
|
|
72
|
+
const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding }));
|
|
73
73
|
await executeWithReusedStatement(cypher, paramsList);
|
|
74
74
|
};
|
|
75
75
|
/**
|
|
@@ -144,7 +144,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
144
144
|
// Incremental mode: filter out nodes that already have embeddings
|
|
145
145
|
if (skipNodeIds && skipNodeIds.size > 0) {
|
|
146
146
|
const beforeCount = nodes.length;
|
|
147
|
-
nodes = nodes.filter(n => !skipNodeIds.has(n.id));
|
|
147
|
+
nodes = nodes.filter((n) => !skipNodeIds.has(n.id));
|
|
148
148
|
if (isDev) {
|
|
149
149
|
console.log(`📦 Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`);
|
|
150
150
|
}
|
|
@@ -190,7 +190,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
190
190
|
await batchInsertEmbeddings(executeWithReusedStatement, updates);
|
|
191
191
|
processedNodes += batch.length;
|
|
192
192
|
// Report progress (20-90% for embedding phase)
|
|
193
|
-
const embeddingProgress = 20 + (
|
|
193
|
+
const embeddingProgress = 20 + (processedNodes / totalNodes) * 70;
|
|
194
194
|
onProgress({
|
|
195
195
|
phase: 'embedding',
|
|
196
196
|
percent: Math.round(embeddingProgress),
|
|
@@ -282,7 +282,7 @@ export const semanticSearch = async (executeQuery, query, k = 10, maxDistance =
|
|
|
282
282
|
// Batch-fetch metadata per label
|
|
283
283
|
const results = [];
|
|
284
284
|
for (const [label, items] of byLabel) {
|
|
285
|
-
const idList = items.map(i => `'${i.nodeId.replace(/'/g, "''")}'`).join(', ');
|
|
285
|
+
const idList = items.map((i) => `'${i.nodeId.replace(/'/g, "''")}'`).join(', ');
|
|
286
286
|
try {
|
|
287
287
|
let nodeQuery;
|
|
288
288
|
if (label === 'File') {
|
|
@@ -344,7 +344,7 @@ export const semanticSearchWithContext = async (executeQuery, query, k = 5, _hop
|
|
|
344
344
|
// For multi-table schema, just return semantic search results
|
|
345
345
|
// Graph traversal is complex with separate tables - use execute_vector_cypher instead
|
|
346
346
|
const results = await semanticSearch(executeQuery, query, k, 0.5);
|
|
347
|
-
return results.map(r => ({
|
|
347
|
+
return results.map((r) => ({
|
|
348
348
|
matchId: r.nodeId,
|
|
349
349
|
matchName: r.name,
|
|
350
350
|
matchLabel: r.label,
|
|
@@ -75,7 +75,7 @@ const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0, attempt
|
|
|
75
75
|
signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
|
|
76
76
|
headers: {
|
|
77
77
|
'Content-Type': 'application/json',
|
|
78
|
-
|
|
78
|
+
Authorization: `Bearer ${apiKey}`,
|
|
79
79
|
},
|
|
80
80
|
body: JSON.stringify({ input: batch, model }),
|
|
81
81
|
});
|
|
@@ -90,7 +90,7 @@ const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0, attempt
|
|
|
90
90
|
// DNS, connection errors — retry with backoff
|
|
91
91
|
if (attempt < HTTP_MAX_RETRIES) {
|
|
92
92
|
const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1);
|
|
93
|
-
await new Promise(r => setTimeout(r, delay));
|
|
93
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
94
94
|
return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1);
|
|
95
95
|
}
|
|
96
96
|
const reason = err instanceof Error ? err.message : String(err);
|
|
@@ -100,7 +100,7 @@ const httpEmbedBatch = async (url, batch, model, apiKey, batchIndex = 0, attempt
|
|
|
100
100
|
const status = resp.status;
|
|
101
101
|
if ((status === 429 || status >= 500) && attempt < HTTP_MAX_RETRIES) {
|
|
102
102
|
const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1);
|
|
103
|
-
await new Promise(r => setTimeout(r, delay));
|
|
103
|
+
await new Promise((r) => setTimeout(r, delay));
|
|
104
104
|
return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1);
|
|
105
105
|
}
|
|
106
106
|
throw new Error(`Embedding endpoint returned ${status} (${safeUrl(url)}, batch ${batchIndex})`);
|
|
@@ -40,25 +40,22 @@ const truncateContent = (content, maxLength) => {
|
|
|
40
40
|
* Removes excessive whitespace while preserving structure
|
|
41
41
|
*/
|
|
42
42
|
const cleanContent = (content) => {
|
|
43
|
-
return content
|
|
43
|
+
return (content
|
|
44
44
|
// Normalize line endings
|
|
45
45
|
.replace(/\r\n/g, '\n')
|
|
46
46
|
// Remove excessive blank lines (more than 2)
|
|
47
47
|
.replace(/\n{3,}/g, '\n\n')
|
|
48
48
|
// Trim each line
|
|
49
49
|
.split('\n')
|
|
50
|
-
.map(line => line.trimEnd())
|
|
50
|
+
.map((line) => line.trimEnd())
|
|
51
51
|
.join('\n')
|
|
52
|
-
.trim();
|
|
52
|
+
.trim());
|
|
53
53
|
};
|
|
54
54
|
/**
|
|
55
55
|
* Generate embedding text for a Function node
|
|
56
56
|
*/
|
|
57
57
|
const generateFunctionText = (node, maxSnippetLength) => {
|
|
58
|
-
const parts = [
|
|
59
|
-
`Function: ${node.name}`,
|
|
60
|
-
`File: ${getFileName(node.filePath)}`,
|
|
61
|
-
];
|
|
58
|
+
const parts = [`Function: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
|
62
59
|
const dir = getDirectory(node.filePath);
|
|
63
60
|
if (dir) {
|
|
64
61
|
parts.push(`Directory: ${dir}`);
|
|
@@ -74,10 +71,7 @@ const generateFunctionText = (node, maxSnippetLength) => {
|
|
|
74
71
|
* Generate embedding text for a Class node
|
|
75
72
|
*/
|
|
76
73
|
const generateClassText = (node, maxSnippetLength) => {
|
|
77
|
-
const parts = [
|
|
78
|
-
`Class: ${node.name}`,
|
|
79
|
-
`File: ${getFileName(node.filePath)}`,
|
|
80
|
-
];
|
|
74
|
+
const parts = [`Class: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
|
81
75
|
const dir = getDirectory(node.filePath);
|
|
82
76
|
if (dir) {
|
|
83
77
|
parts.push(`Directory: ${dir}`);
|
|
@@ -93,10 +87,7 @@ const generateClassText = (node, maxSnippetLength) => {
|
|
|
93
87
|
* Generate embedding text for a Method node
|
|
94
88
|
*/
|
|
95
89
|
const generateMethodText = (node, maxSnippetLength) => {
|
|
96
|
-
const parts = [
|
|
97
|
-
`Method: ${node.name}`,
|
|
98
|
-
`File: ${getFileName(node.filePath)}`,
|
|
99
|
-
];
|
|
90
|
+
const parts = [`Method: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
|
100
91
|
const dir = getDirectory(node.filePath);
|
|
101
92
|
if (dir) {
|
|
102
93
|
parts.push(`Directory: ${dir}`);
|
|
@@ -112,10 +103,7 @@ const generateMethodText = (node, maxSnippetLength) => {
|
|
|
112
103
|
* Generate embedding text for an Interface node
|
|
113
104
|
*/
|
|
114
105
|
const generateInterfaceText = (node, maxSnippetLength) => {
|
|
115
|
-
const parts = [
|
|
116
|
-
`Interface: ${node.name}`,
|
|
117
|
-
`File: ${getFileName(node.filePath)}`,
|
|
118
|
-
];
|
|
106
|
+
const parts = [`Interface: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
|
119
107
|
const dir = getDirectory(node.filePath);
|
|
120
108
|
if (dir) {
|
|
121
109
|
parts.push(`Directory: ${dir}`);
|
|
@@ -132,10 +120,7 @@ const generateInterfaceText = (node, maxSnippetLength) => {
|
|
|
132
120
|
* Uses file name and first N characters of content
|
|
133
121
|
*/
|
|
134
122
|
const generateFileText = (node, maxSnippetLength) => {
|
|
135
|
-
const parts = [
|
|
136
|
-
`File: ${node.name}`,
|
|
137
|
-
`Path: ${node.filePath}`,
|
|
138
|
-
];
|
|
123
|
+
const parts = [`File: ${node.name}`, `Path: ${node.filePath}`];
|
|
139
124
|
if (node.content) {
|
|
140
125
|
const cleanedContent = cleanContent(node.content);
|
|
141
126
|
// For files, use a shorter snippet since they can be very long
|
|
@@ -178,5 +163,5 @@ export const generateEmbeddingText = (node, config = {}) => {
|
|
|
178
163
|
* @returns Array of texts in the same order as input nodes
|
|
179
164
|
*/
|
|
180
165
|
export const generateBatchEmbeddingTexts = (nodes, config = {}) => {
|
|
181
|
-
return nodes.map(node => generateEmbeddingText(node, config));
|
|
166
|
+
return nodes.map((node) => generateEmbeddingText(node, config));
|
|
182
167
|
};
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* These are code elements that benefit from semantic matching
|
|
9
9
|
*/
|
|
10
10
|
export declare const EMBEDDABLE_LABELS: readonly ["Function", "Class", "Method", "Interface", "File"];
|
|
11
|
-
export type EmbeddableLabel = typeof EMBEDDABLE_LABELS[number];
|
|
11
|
+
export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
|
|
12
12
|
/**
|
|
13
13
|
* Check if a label should be embedded
|
|
14
14
|
*/
|
|
@@ -7,13 +7,7 @@
|
|
|
7
7
|
* Node labels that should be embedded for semantic search
|
|
8
8
|
* These are code elements that benefit from semantic matching
|
|
9
9
|
*/
|
|
10
|
-
export const EMBEDDABLE_LABELS = [
|
|
11
|
-
'Function',
|
|
12
|
-
'Class',
|
|
13
|
-
'Method',
|
|
14
|
-
'Interface',
|
|
15
|
-
'File',
|
|
16
|
-
];
|
|
10
|
+
export const EMBEDDABLE_LABELS = ['Function', 'Class', 'Method', 'Interface', 'File'];
|
|
17
11
|
/**
|
|
18
12
|
* Check if a label should be embedded
|
|
19
13
|
*/
|
package/dist/core/graph/graph.js
CHANGED
|
@@ -55,8 +55,12 @@ export const createKnowledgeGraph = () => {
|
|
|
55
55
|
},
|
|
56
56
|
iterNodes: () => nodeMap.values(),
|
|
57
57
|
iterRelationships: () => relationshipMap.values(),
|
|
58
|
-
forEachNode(fn) {
|
|
59
|
-
|
|
58
|
+
forEachNode(fn) {
|
|
59
|
+
nodeMap.forEach(fn);
|
|
60
|
+
},
|
|
61
|
+
forEachRelationship(fn) {
|
|
62
|
+
relationshipMap.forEach(fn);
|
|
63
|
+
},
|
|
60
64
|
getNode: (id) => nodeMap.get(id),
|
|
61
65
|
// O(1) count getters - avoid creating arrays just for length
|
|
62
66
|
get nodeCount() {
|