@veewo/gitnexus 1.5.3 → 1.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/clean.d.ts +3 -2
- package/dist/cli/clean.js +33 -12
- package/dist/core/ingestion/parsing-processor.js +4 -2
- package/dist/core/ingestion/workers/parse-worker.js +66 -42
- package/dist/core/tree-sitter/parser-loader.d.ts +7 -3
- package/dist/core/tree-sitter/parser-loader.js +65 -33
- package/package.json +1 -1
- package/skills/gitnexus-cli.md +71 -22
package/dist/cli/clean.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Clean Command
|
|
3
3
|
*
|
|
4
|
-
* Removes the
|
|
5
|
-
* Also unregisters
|
|
4
|
+
* Removes the GitNexus index from the current repository while preserving
|
|
5
|
+
* configuration files (e.g. sync-manifest.txt). Also unregisters the repo
|
|
6
|
+
* from the global registry.
|
|
6
7
|
*/
|
|
7
8
|
export declare const cleanCommand: (options?: {
|
|
8
9
|
force?: boolean;
|
package/dist/cli/clean.js
CHANGED
|
@@ -1,11 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Clean Command
|
|
3
3
|
*
|
|
4
|
-
* Removes the
|
|
5
|
-
* Also unregisters
|
|
4
|
+
* Removes the GitNexus index from the current repository while preserving
|
|
5
|
+
* configuration files (e.g. sync-manifest.txt). Also unregisters the repo
|
|
6
|
+
* from the global registry.
|
|
6
7
|
*/
|
|
7
8
|
import fs from 'fs/promises';
|
|
9
|
+
import path from 'path';
|
|
8
10
|
import { findRepo, unregisterRepo, listRegisteredRepos } from '../storage/repo-manager.js';
|
|
11
|
+
/** Files under .gitnexus/ that are configuration, not index data. */
|
|
12
|
+
const PRESERVE_FILES = new Set(['sync-manifest.txt']);
|
|
13
|
+
async function cleanStoragePath(storagePath) {
|
|
14
|
+
let entries;
|
|
15
|
+
try {
|
|
16
|
+
entries = await fs.readdir(storagePath);
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
if (err.code === 'ENOENT')
|
|
20
|
+
return;
|
|
21
|
+
throw err;
|
|
22
|
+
}
|
|
23
|
+
for (const entry of entries) {
|
|
24
|
+
if (PRESERVE_FILES.has(entry))
|
|
25
|
+
continue;
|
|
26
|
+
const fullPath = path.join(storagePath, entry);
|
|
27
|
+
await fs.rm(fullPath, { recursive: true, force: true });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
9
30
|
export const cleanCommand = async (options) => {
|
|
10
31
|
// --all flag: clean all indexed repos
|
|
11
32
|
if (options?.all) {
|
|
@@ -15,22 +36,22 @@ export const cleanCommand = async (options) => {
|
|
|
15
36
|
console.log('No indexed repositories found.');
|
|
16
37
|
return;
|
|
17
38
|
}
|
|
18
|
-
console.log(`This will
|
|
39
|
+
console.log(`This will clean GitNexus indexes for ${entries.length} repo(s):`);
|
|
19
40
|
for (const entry of entries) {
|
|
20
41
|
console.log(` - ${entry.name} (${entry.path})`);
|
|
21
42
|
}
|
|
22
|
-
console.log('\nRun with --force to confirm
|
|
43
|
+
console.log('\nRun with --force to confirm.');
|
|
23
44
|
return;
|
|
24
45
|
}
|
|
25
46
|
const entries = await listRegisteredRepos();
|
|
26
47
|
for (const entry of entries) {
|
|
27
48
|
try {
|
|
28
|
-
await
|
|
49
|
+
await cleanStoragePath(entry.storagePath);
|
|
29
50
|
await unregisterRepo(entry.path);
|
|
30
|
-
console.log(`
|
|
51
|
+
console.log(`Cleaned: ${entry.name} (${entry.storagePath})`);
|
|
31
52
|
}
|
|
32
53
|
catch (err) {
|
|
33
|
-
console.error(`Failed to
|
|
54
|
+
console.error(`Failed to clean ${entry.name}:`, err);
|
|
34
55
|
}
|
|
35
56
|
}
|
|
36
57
|
return;
|
|
@@ -44,17 +65,17 @@ export const cleanCommand = async (options) => {
|
|
|
44
65
|
}
|
|
45
66
|
const repoName = repo.repoPath.split(/[/\\]/).pop() || repo.repoPath;
|
|
46
67
|
if (!options?.force) {
|
|
47
|
-
console.log(`This will
|
|
68
|
+
console.log(`This will clean the GitNexus index for: ${repoName}`);
|
|
48
69
|
console.log(` Path: ${repo.storagePath}`);
|
|
49
|
-
console.log('\nRun with --force to confirm
|
|
70
|
+
console.log('\nRun with --force to confirm.');
|
|
50
71
|
return;
|
|
51
72
|
}
|
|
52
73
|
try {
|
|
53
|
-
await
|
|
74
|
+
await cleanStoragePath(repo.storagePath);
|
|
54
75
|
await unregisterRepo(repo.repoPath);
|
|
55
|
-
console.log(`
|
|
76
|
+
console.log(`Cleaned: ${repo.storagePath}`);
|
|
56
77
|
}
|
|
57
78
|
catch (err) {
|
|
58
|
-
console.error('Failed to
|
|
79
|
+
console.error('Failed to clean:', err);
|
|
59
80
|
}
|
|
60
81
|
};
|
|
@@ -104,8 +104,10 @@ const processParsingSequential = async (graph, files, symbolTable, astCache, onF
|
|
|
104
104
|
skippedLanguages.set(language, (skippedLanguages.get(language) || 0) + 1);
|
|
105
105
|
continue;
|
|
106
106
|
}
|
|
107
|
-
// Skip files larger than the max tree-sitter buffer (32 MB)
|
|
108
|
-
|
|
107
|
+
// Skip files larger than the max tree-sitter buffer (32 MB).
|
|
108
|
+
// Use UTF-8 bytes, not JS string length, because tree-sitter buffer sizing
|
|
109
|
+
// is byte-oriented and multi-byte source can otherwise slip past the cap.
|
|
110
|
+
if (Buffer.byteLength(file.content, 'utf8') > TREE_SITTER_MAX_BUFFER)
|
|
109
111
|
continue;
|
|
110
112
|
try {
|
|
111
113
|
await loadLanguage(language, file.path);
|
|
@@ -14,26 +14,8 @@ import Ruby from 'tree-sitter-ruby';
|
|
|
14
14
|
import { createRequire } from 'node:module';
|
|
15
15
|
import { SupportedLanguages } from '../../../config/supported-languages.js';
|
|
16
16
|
import { LANGUAGE_QUERIES } from '../tree-sitter-queries.js';
|
|
17
|
-
import { TREE_SITTER_MAX_BUFFER } from '../constants.js';
|
|
17
|
+
import { getTreeSitterBufferSize, TREE_SITTER_MAX_BUFFER } from '../constants.js';
|
|
18
18
|
const _require = createRequire(import.meta.url);
|
|
19
|
-
// tree-sitter-gdscript is an optionalDependency — may not be installed
|
|
20
|
-
let GDScript = null;
|
|
21
|
-
try {
|
|
22
|
-
GDScript = _require('tree-sitter-gdscript');
|
|
23
|
-
}
|
|
24
|
-
catch { }
|
|
25
|
-
// tree-sitter-swift is an optionalDependency — may not be installed
|
|
26
|
-
let Swift = null;
|
|
27
|
-
try {
|
|
28
|
-
Swift = _require('tree-sitter-swift');
|
|
29
|
-
}
|
|
30
|
-
catch { }
|
|
31
|
-
// tree-sitter-kotlin is an optionalDependency — may not be installed
|
|
32
|
-
let Kotlin = null;
|
|
33
|
-
try {
|
|
34
|
-
Kotlin = _require('tree-sitter-kotlin');
|
|
35
|
-
}
|
|
36
|
-
catch { }
|
|
37
19
|
import { getLanguageFromFilename, FUNCTION_NODE_TYPES, extractFunctionName, isBuiltInOrNoise, getDefinitionNodeFromCaptures, findEnclosingClassId, extractMethodSignature, countCallArguments, inferCallForm, extractReceiverName, extractReceiverNode, CALL_EXPRESSION_TYPES, extractCallChain, } from '../utils.js';
|
|
38
20
|
import { buildTypeEnv } from '../type-env.js';
|
|
39
21
|
import { isNodeExported } from '../export-detection.js';
|
|
@@ -47,7 +29,7 @@ import { callRouters } from '../call-routing.js';
|
|
|
47
29
|
// Worker-local parser + language map
|
|
48
30
|
// ============================================================================
|
|
49
31
|
const parser = new Parser();
|
|
50
|
-
const
|
|
32
|
+
const requiredLanguageMap = {
|
|
51
33
|
[SupportedLanguages.JavaScript]: JavaScript,
|
|
52
34
|
[SupportedLanguages.TypeScript]: TypeScript.typescript,
|
|
53
35
|
[`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx,
|
|
@@ -58,11 +40,60 @@ const languageMap = {
|
|
|
58
40
|
[SupportedLanguages.CSharp]: CSharp,
|
|
59
41
|
[SupportedLanguages.Go]: Go,
|
|
60
42
|
[SupportedLanguages.Rust]: Rust,
|
|
61
|
-
...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),
|
|
62
43
|
[SupportedLanguages.PHP]: PHP.php_only,
|
|
63
44
|
[SupportedLanguages.Ruby]: Ruby,
|
|
64
|
-
|
|
65
|
-
|
|
45
|
+
};
|
|
46
|
+
const optionalLanguagePackages = {
|
|
47
|
+
[SupportedLanguages.GDScript]: 'tree-sitter-gdscript',
|
|
48
|
+
[SupportedLanguages.Swift]: 'tree-sitter-swift',
|
|
49
|
+
[SupportedLanguages.Kotlin]: 'tree-sitter-kotlin',
|
|
50
|
+
};
|
|
51
|
+
const optionalLanguageCache = new Map();
|
|
52
|
+
const optionalAvailabilityCache = new Map();
|
|
53
|
+
const isOptionalLanguageInstalled = (language) => {
|
|
54
|
+
if (optionalAvailabilityCache.has(language)) {
|
|
55
|
+
return optionalAvailabilityCache.get(language);
|
|
56
|
+
}
|
|
57
|
+
const packageName = optionalLanguagePackages[language];
|
|
58
|
+
if (!packageName) {
|
|
59
|
+
optionalAvailabilityCache.set(language, false);
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
try {
|
|
63
|
+
_require.resolve(packageName);
|
|
64
|
+
optionalAvailabilityCache.set(language, true);
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
optionalAvailabilityCache.set(language, false);
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
const loadOptionalLanguage = (language) => {
|
|
73
|
+
if (optionalLanguageCache.has(language)) {
|
|
74
|
+
return optionalLanguageCache.get(language);
|
|
75
|
+
}
|
|
76
|
+
const packageName = optionalLanguagePackages[language];
|
|
77
|
+
if (!packageName) {
|
|
78
|
+
optionalLanguageCache.set(language, null);
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
const grammar = _require(packageName);
|
|
83
|
+
optionalLanguageCache.set(language, grammar);
|
|
84
|
+
return grammar;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
optionalLanguageCache.set(language, null);
|
|
88
|
+
optionalAvailabilityCache.set(language, false);
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
const resolveLanguage = (key, language) => {
|
|
93
|
+
if (key in requiredLanguageMap) {
|
|
94
|
+
return requiredLanguageMap[key];
|
|
95
|
+
}
|
|
96
|
+
return loadOptionalLanguage(language);
|
|
66
97
|
};
|
|
67
98
|
/**
|
|
68
99
|
* Check if a language grammar is available in this worker.
|
|
@@ -74,13 +105,13 @@ const isLanguageAvailable = (language, filePath) => {
|
|
|
74
105
|
const key = language === SupportedLanguages.TypeScript && filePath.endsWith('.tsx')
|
|
75
106
|
? `${language}:tsx`
|
|
76
107
|
: language;
|
|
77
|
-
return key in
|
|
108
|
+
return key in requiredLanguageMap || isOptionalLanguageInstalled(language);
|
|
78
109
|
};
|
|
79
110
|
const setLanguage = (language, filePath) => {
|
|
80
111
|
const key = language === SupportedLanguages.TypeScript && filePath.endsWith('.tsx')
|
|
81
112
|
? `${language}:tsx`
|
|
82
113
|
: language;
|
|
83
|
-
const lang =
|
|
114
|
+
const lang = resolveLanguage(key, language);
|
|
84
115
|
if (!lang)
|
|
85
116
|
throw new Error(`Unsupported language: ${language}`);
|
|
86
117
|
parser.setLanguage(lang);
|
|
@@ -720,27 +751,23 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
720
751
|
return;
|
|
721
752
|
}
|
|
722
753
|
for (const file of files) {
|
|
723
|
-
// Skip files larger than the max tree-sitter buffer (32 MB)
|
|
724
|
-
|
|
754
|
+
// Skip files larger than the max tree-sitter buffer (32 MB).
|
|
755
|
+
// tree-sitter buffer sizing is byte-oriented; JS string length undercounts
|
|
756
|
+
// UTF-8 multi-byte source and can route oversized input into native code.
|
|
757
|
+
if (Buffer.byteLength(file.content, 'utf8') > TREE_SITTER_MAX_BUFFER)
|
|
725
758
|
continue;
|
|
726
759
|
let tree;
|
|
727
760
|
let usedRawContentFallback = false;
|
|
728
761
|
try {
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
if (index >= file.content.length)
|
|
732
|
-
return null;
|
|
733
|
-
return file.content.slice(index, index + MAX_CHUNK);
|
|
762
|
+
tree = parser.parse(file.content, null, {
|
|
763
|
+
bufferSize: getTreeSitterBufferSize(Buffer.byteLength(file.content, 'utf8')),
|
|
734
764
|
});
|
|
735
765
|
}
|
|
736
766
|
catch (err) {
|
|
737
767
|
if (file.rawContent && file.rawContent !== file.content) {
|
|
738
768
|
try {
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
if (index >= file.rawContent.length)
|
|
742
|
-
return null;
|
|
743
|
-
return file.rawContent.slice(index, index + MAX_CHUNK);
|
|
769
|
+
tree = parser.parse(file.rawContent, null, {
|
|
770
|
+
bufferSize: getTreeSitterBufferSize(Buffer.byteLength(file.rawContent, 'utf8')),
|
|
744
771
|
});
|
|
745
772
|
usedRawContentFallback = true;
|
|
746
773
|
}
|
|
@@ -756,11 +783,8 @@ const processFileGroup = (files, language, queryString, result, onFileProcessed)
|
|
|
756
783
|
}
|
|
757
784
|
if (file.rawContent && file.rawContent !== file.content && tree.rootNode?.hasError) {
|
|
758
785
|
try {
|
|
759
|
-
const
|
|
760
|
-
|
|
761
|
-
if (index >= file.rawContent.length)
|
|
762
|
-
return null;
|
|
763
|
-
return file.rawContent.slice(index, index + MAX_CHUNK);
|
|
786
|
+
const rawTree = parser.parse(file.rawContent, null, {
|
|
787
|
+
bufferSize: getTreeSitterBufferSize(Buffer.byteLength(file.rawContent, 'utf8')),
|
|
764
788
|
});
|
|
765
789
|
if (!rawTree.rootNode?.hasError) {
|
|
766
790
|
tree = rawTree;
|
|
@@ -4,9 +4,13 @@ export declare const isLanguageAvailable: (language: SupportedLanguages) => bool
|
|
|
4
4
|
export declare const loadParser: () => Promise<Parser>;
|
|
5
5
|
export declare const loadLanguage: (language: SupportedLanguages, filePath?: string) => Promise<void>;
|
|
6
6
|
/**
|
|
7
|
-
* Parse source code using tree-sitter's
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* Parse source code using tree-sitter's string input path with an adaptive
|
|
8
|
+
* native buffer size.
|
|
9
|
+
*
|
|
10
|
+
* The callback input API receives byte offsets. Returning JavaScript string
|
|
11
|
+
* slices from those byte offsets is unsafe for UTF-8/multi-byte content and has
|
|
12
|
+
* caused native tree-sitter crashes in large repositories. Use the stable string
|
|
13
|
+
* input path instead and raise tree-sitter's internal buffer for large files.
|
|
10
14
|
*
|
|
11
15
|
* @param content - Full source file content as UTF-8 string
|
|
12
16
|
* @param oldTree - Optional previous tree for incremental parsing (must call tree.edit() first)
|
|
@@ -12,27 +12,10 @@ import PHP from 'tree-sitter-php';
|
|
|
12
12
|
import Ruby from 'tree-sitter-ruby';
|
|
13
13
|
import { createRequire } from 'node:module';
|
|
14
14
|
import { SupportedLanguages } from '../../config/supported-languages.js';
|
|
15
|
+
import { getTreeSitterBufferSize } from '../ingestion/constants.js';
|
|
15
16
|
const _require = createRequire(import.meta.url);
|
|
16
|
-
// tree-sitter-gdscript is an optionalDependency — may not be installed
|
|
17
|
-
let GDScript = null;
|
|
18
|
-
try {
|
|
19
|
-
GDScript = _require('tree-sitter-gdscript');
|
|
20
|
-
}
|
|
21
|
-
catch { }
|
|
22
|
-
// tree-sitter-swift is an optionalDependency — may not be installed
|
|
23
|
-
let Swift = null;
|
|
24
|
-
try {
|
|
25
|
-
Swift = _require('tree-sitter-swift');
|
|
26
|
-
}
|
|
27
|
-
catch { }
|
|
28
|
-
// tree-sitter-kotlin is an optionalDependency — may not be installed
|
|
29
|
-
let Kotlin = null;
|
|
30
|
-
try {
|
|
31
|
-
Kotlin = _require('tree-sitter-kotlin');
|
|
32
|
-
}
|
|
33
|
-
catch { }
|
|
34
17
|
let parser = null;
|
|
35
|
-
const
|
|
18
|
+
const requiredLanguageMap = {
|
|
36
19
|
[SupportedLanguages.JavaScript]: JavaScript,
|
|
37
20
|
[SupportedLanguages.TypeScript]: TypeScript.typescript,
|
|
38
21
|
[`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx,
|
|
@@ -43,13 +26,62 @@ const languageMap = {
|
|
|
43
26
|
[SupportedLanguages.CSharp]: CSharp,
|
|
44
27
|
[SupportedLanguages.Go]: Go,
|
|
45
28
|
[SupportedLanguages.Rust]: Rust,
|
|
46
|
-
...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),
|
|
47
29
|
[SupportedLanguages.PHP]: PHP.php_only,
|
|
48
30
|
[SupportedLanguages.Ruby]: Ruby,
|
|
49
|
-
...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
|
|
50
|
-
...(GDScript ? { [SupportedLanguages.GDScript]: GDScript } : {}),
|
|
51
31
|
};
|
|
52
|
-
|
|
32
|
+
const optionalLanguagePackages = {
|
|
33
|
+
[SupportedLanguages.GDScript]: 'tree-sitter-gdscript',
|
|
34
|
+
[SupportedLanguages.Swift]: 'tree-sitter-swift',
|
|
35
|
+
[SupportedLanguages.Kotlin]: 'tree-sitter-kotlin',
|
|
36
|
+
};
|
|
37
|
+
const optionalLanguageCache = new Map();
|
|
38
|
+
const optionalAvailabilityCache = new Map();
|
|
39
|
+
const isOptionalLanguageInstalled = (language) => {
|
|
40
|
+
if (optionalAvailabilityCache.has(language)) {
|
|
41
|
+
return optionalAvailabilityCache.get(language);
|
|
42
|
+
}
|
|
43
|
+
const packageName = optionalLanguagePackages[language];
|
|
44
|
+
if (!packageName) {
|
|
45
|
+
optionalAvailabilityCache.set(language, false);
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
_require.resolve(packageName);
|
|
50
|
+
optionalAvailabilityCache.set(language, true);
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
catch {
|
|
54
|
+
optionalAvailabilityCache.set(language, false);
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const loadOptionalLanguage = (language) => {
|
|
59
|
+
if (optionalLanguageCache.has(language)) {
|
|
60
|
+
return optionalLanguageCache.get(language);
|
|
61
|
+
}
|
|
62
|
+
const packageName = optionalLanguagePackages[language];
|
|
63
|
+
if (!packageName) {
|
|
64
|
+
optionalLanguageCache.set(language, null);
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
const grammar = _require(packageName);
|
|
69
|
+
optionalLanguageCache.set(language, grammar);
|
|
70
|
+
return grammar;
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
optionalLanguageCache.set(language, null);
|
|
74
|
+
optionalAvailabilityCache.set(language, false);
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
const resolveLanguage = (key, language) => {
|
|
79
|
+
if (key in requiredLanguageMap) {
|
|
80
|
+
return requiredLanguageMap[key];
|
|
81
|
+
}
|
|
82
|
+
return loadOptionalLanguage(language);
|
|
83
|
+
};
|
|
84
|
+
export const isLanguageAvailable = (language) => language in requiredLanguageMap || isOptionalLanguageInstalled(language);
|
|
53
85
|
export const loadParser = async () => {
|
|
54
86
|
if (parser)
|
|
55
87
|
return parser;
|
|
@@ -62,17 +94,20 @@ export const loadLanguage = async (language, filePath) => {
|
|
|
62
94
|
const key = language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
|
|
63
95
|
? `${language}:tsx`
|
|
64
96
|
: language;
|
|
65
|
-
const lang =
|
|
97
|
+
const lang = resolveLanguage(key, language);
|
|
66
98
|
if (!lang) {
|
|
67
99
|
throw new Error(`Unsupported language: ${language}`);
|
|
68
100
|
}
|
|
69
101
|
parser.setLanguage(lang);
|
|
70
102
|
};
|
|
71
|
-
const MAX_CHUNK = 4096;
|
|
72
103
|
/**
|
|
73
|
-
* Parse source code using tree-sitter's
|
|
74
|
-
*
|
|
75
|
-
*
|
|
104
|
+
* Parse source code using tree-sitter's string input path with an adaptive
|
|
105
|
+
* native buffer size.
|
|
106
|
+
*
|
|
107
|
+
* The callback input API receives byte offsets. Returning JavaScript string
|
|
108
|
+
* slices from those byte offsets is unsafe for UTF-8/multi-byte content and has
|
|
109
|
+
* caused native tree-sitter crashes in large repositories. Use the stable string
|
|
110
|
+
* input path instead and raise tree-sitter's internal buffer for large files.
|
|
76
111
|
*
|
|
77
112
|
* @param content - Full source file content as UTF-8 string
|
|
78
113
|
* @param oldTree - Optional previous tree for incremental parsing (must call tree.edit() first)
|
|
@@ -81,9 +116,6 @@ const MAX_CHUNK = 4096;
|
|
|
81
116
|
export const parseContent = (content, oldTree) => {
|
|
82
117
|
if (!parser)
|
|
83
118
|
throw new Error('Parser not initialized — call loadParser() first');
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
return null;
|
|
87
|
-
return content.slice(index, index + MAX_CHUNK);
|
|
88
|
-
}, oldTree);
|
|
119
|
+
const bufferSize = getTreeSitterBufferSize(Buffer.byteLength(content, 'utf8'));
|
|
120
|
+
return parser.parse(content, oldTree ?? null, { bufferSize });
|
|
89
121
|
};
|
package/package.json
CHANGED
package/skills/gitnexus-cli.md
CHANGED
|
@@ -60,35 +60,83 @@ $GN analyze
|
|
|
60
60
|
|
|
61
61
|
Run from the project root. This parses all source files, builds the knowledge graph, writes it to `.gitnexus/`, and generates CLAUDE.md / AGENTS.md context files.
|
|
62
62
|
|
|
63
|
-
| Flag
|
|
64
|
-
|
|
65
|
-
| `--force`
|
|
66
|
-
| `--embeddings`
|
|
67
|
-
| `--
|
|
68
|
-
| `--csharp-define-csproj <path>` | Load C# `DefineConstants` from a `.csproj` and normalize `#if/#elif/#else/#endif` before parsing |
|
|
69
|
-
| `--scope-prefix <prefix>` | Limit analysis to a path prefix (e.g., `--scope-prefix Assets/` for Unity) |
|
|
70
|
-
| `--scope-manifest <file>` | Read scope rules from a manifest file (e.g., `.gitnexus/sync-manifest.txt`) |
|
|
71
|
-
| `--sync-manifest-policy <policy>` | Drift policy when explicit CLI values differ from manifest directives: `ask|update|keep|error` |
|
|
72
|
-
| `--skills` | Generate repo-specific skill files from detected code communities |
|
|
63
|
+
| Flag | Effect |
|
|
64
|
+
|------|--------|
|
|
65
|
+
| `--force` | Force full re-index even if up to date |
|
|
66
|
+
| `--embeddings` | Enable embedding generation (off by default) |
|
|
67
|
+
| `--skills` | Generate repo-specific skill files from detected communities |
|
|
73
68
|
|
|
74
|
-
**
|
|
69
|
+
**Two mutually exclusive paths. Choose one per rebuild.**
|
|
75
70
|
|
|
76
|
-
|
|
71
|
+
#### Path A: sync-manifest managed (recommended for Unity / monorepo)
|
|
77
72
|
|
|
78
|
-
|
|
73
|
+
If `.gitnexus/sync-manifest.txt` exists, `analyze` **auto-uses** it when you do **not** pass `--scope-prefix` or `--scope-manifest`.
|
|
79
74
|
|
|
80
|
-
|
|
75
|
+
```bash
|
|
76
|
+
# Unity project with an existing manifest — this is the normal rebuild command
|
|
77
|
+
$GN analyze --force
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
The manifest controls scope rules, extensions, and repo alias. Example:
|
|
81
|
+
|
|
82
|
+
```
|
|
83
|
+
Assets/
|
|
84
|
+
Packages/
|
|
85
|
+
@extensions=.cs,.meta
|
|
86
|
+
@repoAlias=neonspark-core
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
- Non-`@` lines = path-prefix scope rules
|
|
90
|
+
- `@extensions=<csv>` = file extension filter
|
|
91
|
+
- `@repoAlias=<name>` = stable repo alias
|
|
92
|
+
- `@embeddings=<true|false>` = embedding toggle
|
|
93
|
+
|
|
94
|
+
**Drift guard:** If you pass `--extensions` / `--repo-alias` / `--embeddings` while a manifest exists, CLI compares them. Use `--sync-manifest-policy` to control: `ask|update|keep|error` (default `ask`; non-TTY requires explicit policy).
|
|
95
|
+
|
|
96
|
+
**Do not mix Path A and Path B.** Passing `--scope-prefix` or `--extensions` when a manifest exists triggers the drift guard and may error out in non-TTY environments.
|
|
97
|
+
|
|
98
|
+
#### Path B: manual CLI flags (first-time or simple projects)
|
|
81
99
|
|
|
82
|
-
|
|
100
|
+
Use when no sync-manifest exists:
|
|
83
101
|
|
|
84
|
-
|
|
85
|
-
|
|
102
|
+
```bash
|
|
103
|
+
# Unity project, first-time index
|
|
104
|
+
$GN analyze --force --extensions ".cs,.meta" --scope-prefix Assets/ --repo-alias neonspark-core
|
|
105
|
+
|
|
106
|
+
# Generic project
|
|
107
|
+
$GN analyze --force --extensions ".ts,.tsx" --scope-prefix src/
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
| Manual flag | Effect |
|
|
111
|
+
|-------------|--------|
|
|
112
|
+
| `--extensions <ext>` | Comma-separated file extensions |
|
|
113
|
+
| `--scope-prefix <prefix>` | Add a path prefix rule (repeatable) |
|
|
114
|
+
| `--scope-manifest <file>` | Read scope rules from a manifest file |
|
|
115
|
+
| `--repo-alias <name>` | Override indexed repository name |
|
|
116
|
+
| `--csharp-define-csproj <path>` | Load C# `DefineConstants` from `.csproj` for `#if` normalization |
|
|
117
|
+
|
|
118
|
+
**C# preprocessing (Unity):** For projects with heavy conditional compilation, add `--csharp-define-csproj /path/to/Assembly-CSharp.csproj` (neonspark: `/Volumes/Shuttle/projects/neonspark/Assembly-CSharp.csproj`). Without it, C# files are parsed raw and tree-sitter may mishandle `#if` branches.
|
|
119
|
+
|
|
120
|
+
#### Rebuild recovery — when analyze hangs or crashes
|
|
121
|
+
|
|
122
|
+
If `analyze --force` hangs (no progress after 5+ minutes) or crashes leaving a corrupted index:
|
|
86
123
|
|
|
87
124
|
```bash
|
|
88
|
-
|
|
89
|
-
$GN
|
|
125
|
+
# 1. Clean the corrupted index (preserves sync-manifest.txt)
|
|
126
|
+
$GN clean --force
|
|
127
|
+
|
|
128
|
+
# 2. Rebuild
|
|
129
|
+
$GN analyze --force
|
|
90
130
|
```
|
|
91
131
|
|
|
132
|
+
**When to clean before rebuild:**
|
|
133
|
+
- Previous run left `.gitnexus/csv/` with no `relations.csv` (crash mid-streaming)
|
|
134
|
+
- `.gitnexus/lbug.wal` exists but `lbug` is tiny (LadybugDB in unrecovered state)
|
|
135
|
+
- Any `analyze` run hangs indefinitely in the "Loading into LadybugDB..." phase
|
|
136
|
+
- After `gitnexus clean`, always run `analyze --force` to rebuild
|
|
137
|
+
|
|
138
|
+
**When to run:** First time in a project, after major code changes, or when `gitnexus://repo/{name}/context` reports the index is stale.
|
|
139
|
+
|
|
92
140
|
### status — Check index freshness
|
|
93
141
|
|
|
94
142
|
```bash
|
|
@@ -97,13 +145,13 @@ $GN status
|
|
|
97
145
|
|
|
98
146
|
Shows whether the current repo has a GitNexus index, when it was last updated, and symbol/relationship counts. Use this to check if re-indexing is needed.
|
|
99
147
|
|
|
100
|
-
### clean — Delete the index
|
|
148
|
+
### clean — Delete the index (preserves config)
|
|
101
149
|
|
|
102
150
|
```bash
|
|
103
|
-
$GN clean
|
|
151
|
+
$GN clean --force
|
|
104
152
|
```
|
|
105
153
|
|
|
106
|
-
|
|
154
|
+
Removes the GitNexus index (graph, CSVs, LadybugDB) from `.gitnexus/` while **preserving `sync-manifest.txt`** and other configuration files. Use this to recover from a corrupted index before re-indexing.
|
|
107
155
|
|
|
108
156
|
| Flag | Effect |
|
|
109
157
|
| --------- | ------------------------------------------------- |
|
|
@@ -189,6 +237,7 @@ $GN unity-ui-trace "Assets/NEON/VeewoUI/Uxml/BarScreen/Patch/PatchItemPreview.ux
|
|
|
189
237
|
- **"Not inside a git repository"**: Run from a directory inside a git repo
|
|
190
238
|
- **Index is stale after re-analyzing**: Restart Claude Code to reload the MCP server
|
|
191
239
|
- **Embeddings slow**: Omit `--embeddings` (it's off by default) or set `OPENAI_API_KEY` for faster API-based embedding
|
|
240
|
+
- **`analyze --force` hangs or crashes**: Run `$GN clean --force` to remove the corrupted index (sync-manifest is preserved), then `$GN analyze --force` to rebuild. Common corruption signatures: `.gitnexus/csv/` exists but `relations.csv` is missing; `.gitnexus/lbug.wal` exists while `lbug` is only a few KB.
|
|
192
241
|
|
|
193
242
|
## Runtime-Chain Closure Guard
|
|
194
243
|
|