@codebolt/codeboltjs 2.0.5 → 2.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/websocket.js +2 -1
- package/dist/index.d.ts +65 -19
- package/dist/index.js +2 -2
- package/dist/modules/agent.js +1 -1
- package/dist/modules/codeparsers.d.ts +32 -3
- package/dist/modules/codeparsers.js +295 -3
- package/dist/modules/codeutils.d.ts +24 -7
- package/dist/modules/codeutils.js +93 -29
- package/dist/modules/fs.d.ts +46 -0
- package/dist/modules/fs.js +70 -1
- package/dist/modules/git.d.ts +7 -14
- package/dist/modules/git.js +16 -37
- package/dist/modules/outputparsers.d.ts +36 -4
- package/dist/modules/outputparsers.js +56 -5
- package/dist/modules/project.d.ts +3 -4
- package/dist/modules/project.js +7 -5
- package/dist/modules/terminal.js +1 -1
- package/dist/{modules → utils}/docutils.d.ts +3 -6
- package/dist/utils/docutils.js +69 -0
- package/dist/utils/parse-source-code/index.d.ts +9 -0
- package/dist/utils/parse-source-code/index.js +233 -0
- package/dist/utils/parse-source-code/languageParser.d.ts +8 -0
- package/dist/utils/parse-source-code/languageParser.js +137 -0
- package/dist/utils/parse-source-code/queries/c-sharp.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/c-sharp.js +25 -0
- package/dist/utils/parse-source-code/queries/c.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/c.js +17 -0
- package/dist/utils/parse-source-code/queries/cpp.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/cpp.js +25 -0
- package/dist/utils/parse-source-code/queries/go.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/go.js +29 -0
- package/dist/utils/parse-source-code/queries/index.d.ts +12 -0
- package/dist/utils/parse-source-code/queries/index.js +30 -0
- package/dist/utils/parse-source-code/queries/java.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/java.js +17 -0
- package/dist/utils/parse-source-code/queries/javascript.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/javascript.js +67 -0
- package/dist/utils/parse-source-code/queries/php.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/php.js +17 -0
- package/dist/utils/parse-source-code/queries/python.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/python.js +13 -0
- package/dist/utils/parse-source-code/queries/ruby.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/ruby.js +54 -0
- package/dist/utils/parse-source-code/queries/rust.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/rust.js +18 -0
- package/dist/utils/parse-source-code/queries/swift.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/swift.js +47 -0
- package/dist/utils/parse-source-code/queries/typescript.d.ts +2 -0
- package/dist/utils/parse-source-code/queries/typescript.js +34 -0
- package/dist/utils/parse-source-code/tree-sitter-c.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-c_sharp.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-cpp.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-go.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-java.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-javascript.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-php.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-python.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-ruby.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-rust.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-swift.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-tsx.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter-typescript.wasm +0 -0
- package/dist/utils/parse-source-code/tree-sitter.wasm +0 -0
- package/dist/utils.d.ts +1 -0
- package/dist/utils.js +6 -1
- package/package.json +6 -2
- package/dist/modules/docutils.js +0 -19
package/dist/modules/terminal.js
CHANGED
|
@@ -1,12 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
* A module for document utility functions.
|
|
3
|
-
*/
|
|
4
|
-
declare const cbdocutils: {
|
|
1
|
+
declare const docutils: {
|
|
5
2
|
/**
|
|
6
3
|
* Converts a PDF document to text.
|
|
7
4
|
* @param pdf_path - The file path to the PDF document to be converted.
|
|
8
5
|
* @returns {Promise<string>} A promise that resolves with the converted text.
|
|
9
6
|
*/
|
|
10
|
-
pdf_to_text: (pdf_path:
|
|
7
|
+
pdf_to_text: (pdf_path: string) => Promise<string>;
|
|
11
8
|
};
|
|
12
|
-
export default
|
|
9
|
+
export default docutils;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
/**
|
|
30
|
+
* A module for document utility functions.
|
|
31
|
+
*/
|
|
32
|
+
const fs = __importStar(require("fs"));
|
|
33
|
+
const pdf_parse_1 = __importDefault(require("pdf-parse"));
|
|
34
|
+
const docutils = {
|
|
35
|
+
/**
|
|
36
|
+
* Converts a PDF document to text.
|
|
37
|
+
* @param pdf_path - The file path to the PDF document to be converted.
|
|
38
|
+
* @returns {Promise<string>} A promise that resolves with the converted text.
|
|
39
|
+
*/
|
|
40
|
+
pdf_to_text: (pdf_path) => {
|
|
41
|
+
return new Promise((resolve, reject) => {
|
|
42
|
+
try {
|
|
43
|
+
// Check if file exists
|
|
44
|
+
if (!fs.existsSync(pdf_path)) {
|
|
45
|
+
return reject(new Error(`PDF file not found at path: ${pdf_path}`));
|
|
46
|
+
}
|
|
47
|
+
// Read the PDF file as buffer
|
|
48
|
+
const dataBuffer = fs.readFileSync(pdf_path);
|
|
49
|
+
// Parse the PDF document
|
|
50
|
+
(0, pdf_parse_1.default)(dataBuffer)
|
|
51
|
+
.then((data) => {
|
|
52
|
+
resolve(data.text);
|
|
53
|
+
})
|
|
54
|
+
.catch((error) => {
|
|
55
|
+
reject(new Error(`Error parsing PDF: ${error.message}`));
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
if (error instanceof Error) {
|
|
60
|
+
reject(new Error(`Error processing PDF: ${error.message}`));
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
reject(new Error('Unknown error processing PDF'));
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
exports.default = docutils;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const LIST_FILES_LIMIT = 200;
|
|
2
|
+
export declare function parseSourceCodeForDefinitionsTopLevel(dirPath: string): Promise<string>;
|
|
3
|
+
export declare function listFiles(dirPath: string, recursive: boolean): Promise<string[]>;
|
|
4
|
+
export declare function globbyLevelByLevel(options: any): Promise<string[]>;
|
|
5
|
+
export declare function separateFiles(allFiles: string[]): {
|
|
6
|
+
filesToParse: string[];
|
|
7
|
+
remainingFiles: string[];
|
|
8
|
+
};
|
|
9
|
+
export declare function parseFile(filePath: string, languageParsers: any): Promise<string | undefined>;
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseFile = exports.separateFiles = exports.globbyLevelByLevel = exports.listFiles = exports.parseSourceCodeForDefinitionsTopLevel = exports.LIST_FILES_LIMIT = void 0;
|
|
4
|
+
const fs = require("fs/promises");
|
|
5
|
+
const os = require("os");
|
|
6
|
+
const path = require("path");
|
|
7
|
+
const { LanguageParser, loadRequiredLanguageParsers } = require("./languageParser");
|
|
8
|
+
// const { globby } = await import("globby");
|
|
9
|
+
const { globby } = require('@codebolt/globby');
|
|
10
|
+
exports.LIST_FILES_LIMIT = 200;
|
|
11
|
+
// TODO: implement caching behavior to avoid having to keep analyzing project for new tasks.
|
|
12
|
+
async function parseSourceCodeForDefinitionsTopLevel(dirPath) {
|
|
13
|
+
// check if the path exists
|
|
14
|
+
const dirExists = await fs
|
|
15
|
+
.access(path.resolve(dirPath))
|
|
16
|
+
.then(() => true)
|
|
17
|
+
.catch(() => false);
|
|
18
|
+
if (!dirExists) {
|
|
19
|
+
return "This directory does not exist or you do not have permission to access it.";
|
|
20
|
+
}
|
|
21
|
+
// Get all files at top level (not gitignored)
|
|
22
|
+
const allFiles = await listFiles(dirPath, false);
|
|
23
|
+
let result = "";
|
|
24
|
+
// Separate files to parse and remaining files
|
|
25
|
+
const { filesToParse, remainingFiles } = separateFiles(allFiles);
|
|
26
|
+
const languageParsers = await loadRequiredLanguageParsers(filesToParse);
|
|
27
|
+
// Parse specific files we have language parsers for
|
|
28
|
+
// const filesWithoutDefinitions: string[] = []
|
|
29
|
+
for (const file of filesToParse) {
|
|
30
|
+
const definitions = await parseFile(file, languageParsers);
|
|
31
|
+
if (definitions) {
|
|
32
|
+
result += `${path.relative(dirPath, file)}\n${definitions}\n`;
|
|
33
|
+
}
|
|
34
|
+
// else {
|
|
35
|
+
// filesWithoutDefinitions.push(file)
|
|
36
|
+
// }
|
|
37
|
+
}
|
|
38
|
+
// List remaining files' paths
|
|
39
|
+
// let didFindUnparsedFiles = false
|
|
40
|
+
// filesWithoutDefinitions
|
|
41
|
+
// .concat(remainingFiles)
|
|
42
|
+
// .sort()
|
|
43
|
+
// .forEach((file) => {
|
|
44
|
+
// if (!didFindUnparsedFiles) {
|
|
45
|
+
// result += "# Unparsed Files\n\n"
|
|
46
|
+
// didFindUnparsedFiles = true
|
|
47
|
+
// }
|
|
48
|
+
// result += `${path.relative(dirPath, file)}\n`
|
|
49
|
+
// })
|
|
50
|
+
return result ? result : "No source code definitions found.";
|
|
51
|
+
}
|
|
52
|
+
exports.parseSourceCodeForDefinitionsTopLevel = parseSourceCodeForDefinitionsTopLevel;
|
|
53
|
+
async function listFiles(dirPath, recursive) {
|
|
54
|
+
const absolutePath = path.resolve(dirPath);
|
|
55
|
+
// Do not allow listing files in root or home directory, which Claude tends to want to do when the user's prompt is vague.
|
|
56
|
+
const root = process.platform === "win32" ? path.parse(absolutePath).root : "/";
|
|
57
|
+
const isRoot = absolutePath === root;
|
|
58
|
+
if (isRoot) {
|
|
59
|
+
return [root];
|
|
60
|
+
}
|
|
61
|
+
const homeDir = os.homedir();
|
|
62
|
+
const isHomeDir = absolutePath === homeDir;
|
|
63
|
+
if (isHomeDir) {
|
|
64
|
+
return [homeDir];
|
|
65
|
+
}
|
|
66
|
+
const dirsToIgnore = [
|
|
67
|
+
"node_modules",
|
|
68
|
+
"__pycache__",
|
|
69
|
+
"env",
|
|
70
|
+
"venv",
|
|
71
|
+
"target/dependency",
|
|
72
|
+
"build/dependencies",
|
|
73
|
+
"dist",
|
|
74
|
+
"out",
|
|
75
|
+
"bundle",
|
|
76
|
+
"vendor",
|
|
77
|
+
"tmp",
|
|
78
|
+
"temp",
|
|
79
|
+
"deps",
|
|
80
|
+
"pkg",
|
|
81
|
+
"Pods",
|
|
82
|
+
".*", // Ignore all hidden directories and files
|
|
83
|
+
].map((dir) => `**/${dir}/**`);
|
|
84
|
+
const options = {
|
|
85
|
+
cwd: dirPath,
|
|
86
|
+
dot: false, // do not ignore hidden files/directories
|
|
87
|
+
absolute: true,
|
|
88
|
+
markDirectories: true, // Append a / on any directories matched
|
|
89
|
+
gitignore: recursive, // globby ignores any files that are gitignored
|
|
90
|
+
ignore: recursive ? dirsToIgnore : undefined, // just in case there is no gitignore, we ignore sensible defaults
|
|
91
|
+
onlyFiles: false, // true by default, false means it will list directories on their own too
|
|
92
|
+
};
|
|
93
|
+
// * globs all files in one dir, ** globs files in nested directories
|
|
94
|
+
const files = recursive
|
|
95
|
+
? await globbyLevelByLevel(options)
|
|
96
|
+
: (await globby("*", options)).slice(0, exports.LIST_FILES_LIMIT);
|
|
97
|
+
return files;
|
|
98
|
+
}
|
|
99
|
+
exports.listFiles = listFiles;
|
|
100
|
+
// globby doesnt natively support top down level by level globbing, so we implement it ourselves
|
|
101
|
+
async function globbyLevelByLevel(options) {
|
|
102
|
+
let results = [];
|
|
103
|
+
const globbingProcess = async () => {
|
|
104
|
+
let currentLevel = 0;
|
|
105
|
+
// const { globby } = await import("globby");
|
|
106
|
+
while (results.length < exports.LIST_FILES_LIMIT) {
|
|
107
|
+
const pattern = currentLevel === 0 ? "*" : `${"*/".repeat(currentLevel)}*`;
|
|
108
|
+
const filesAtLevel = await globby(pattern, options);
|
|
109
|
+
if (filesAtLevel.length === 0) {
|
|
110
|
+
break;
|
|
111
|
+
}
|
|
112
|
+
results.push(...filesAtLevel);
|
|
113
|
+
if (results.length >= exports.LIST_FILES_LIMIT) {
|
|
114
|
+
results = results.slice(0, exports.LIST_FILES_LIMIT);
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
currentLevel++;
|
|
118
|
+
}
|
|
119
|
+
return results;
|
|
120
|
+
};
|
|
121
|
+
// Timeout after 10 seconds and return partial results
|
|
122
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
123
|
+
setTimeout(() => reject(new Error("Globbing timeout")), 10000);
|
|
124
|
+
});
|
|
125
|
+
try {
|
|
126
|
+
return await Promise.race([globbingProcess(), timeoutPromise]);
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
console.log(error);
|
|
130
|
+
console.warn("Globbing timed out, returning partial results");
|
|
131
|
+
return results;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.globbyLevelByLevel = globbyLevelByLevel;
|
|
135
|
+
function separateFiles(allFiles) {
|
|
136
|
+
const extensions = [
|
|
137
|
+
"js",
|
|
138
|
+
"jsx",
|
|
139
|
+
"ts",
|
|
140
|
+
"tsx",
|
|
141
|
+
"py",
|
|
142
|
+
// Rust
|
|
143
|
+
"rs",
|
|
144
|
+
"go",
|
|
145
|
+
// C
|
|
146
|
+
"c",
|
|
147
|
+
"h",
|
|
148
|
+
// C++
|
|
149
|
+
"cpp",
|
|
150
|
+
"hpp",
|
|
151
|
+
// C#
|
|
152
|
+
"cs",
|
|
153
|
+
// Ruby
|
|
154
|
+
"rb",
|
|
155
|
+
"java",
|
|
156
|
+
"php",
|
|
157
|
+
"swift",
|
|
158
|
+
].map((e) => `.${e}`);
|
|
159
|
+
const filesToParse = allFiles.filter((file) => extensions.includes(path.extname(file))).slice(0, 50); // 50 files max
|
|
160
|
+
const remainingFiles = allFiles.filter((file) => !filesToParse.includes(file));
|
|
161
|
+
return { filesToParse, remainingFiles };
|
|
162
|
+
}
|
|
163
|
+
exports.separateFiles = separateFiles;
|
|
164
|
+
/*
|
|
165
|
+
Parsing files using tree-sitter
|
|
166
|
+
|
|
167
|
+
1. Parse the file content into an AST (Abstract Syntax Tree) using the appropriate language grammar (set of rules that define how the components of a language like keywords, expressions, and statements can be combined to create valid programs).
|
|
168
|
+
2. Create a query using a language-specific query string, and run it against the AST's root node to capture specific syntax elements.
|
|
169
|
+
- We use tag queries to identify named entities in a program, and then use a syntax capture to label the entity and its name. A notable example of this is GitHub's search-based code navigation.
|
|
170
|
+
- Our custom tag queries are based on tree-sitter's default tag queries, but modified to only capture definitions.
|
|
171
|
+
3. Sort the captures by their position in the file, output the name of the definition, and format by i.e. adding "|----\n" for gaps between captured sections.
|
|
172
|
+
|
|
173
|
+
This approach allows us to focus on the most relevant parts of the code (defined by our language-specific queries) and provides a concise yet informative view of the file's structure and key elements.
|
|
174
|
+
|
|
175
|
+
- https://github.com/tree-sitter/node-tree-sitter/blob/master/test/query_test.js
|
|
176
|
+
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/query-test.js
|
|
177
|
+
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/helper.js
|
|
178
|
+
- https://tree-sitter.github.io/tree-sitter/code-navigation-systems
|
|
179
|
+
*/
|
|
180
|
+
async function parseFile(filePath, languageParsers) {
|
|
181
|
+
const fileContent = await fs.readFile(filePath, "utf8");
|
|
182
|
+
const ext = path.extname(filePath).toLowerCase().slice(1);
|
|
183
|
+
const { parser, query } = languageParsers[ext] || {};
|
|
184
|
+
if (!parser || !query) {
|
|
185
|
+
return `Unsupported file type: ${filePath}`;
|
|
186
|
+
}
|
|
187
|
+
let formattedOutput = "";
|
|
188
|
+
try {
|
|
189
|
+
// Parse the file content into an Abstract Syntax Tree (AST), a tree-like representation of the code
|
|
190
|
+
const tree = parser.parse(fileContent);
|
|
191
|
+
// Apply the query to the AST and get the captures
|
|
192
|
+
// Captures are specific parts of the AST that match our query patterns, each capture represents a node in the AST that we're interested in.
|
|
193
|
+
const captures = query.captures(tree.rootNode);
|
|
194
|
+
// Sort captures by their start position
|
|
195
|
+
captures.sort((a, b) => a.node.startPosition.row - b.node.startPosition.row);
|
|
196
|
+
// Split the file content into individual lines
|
|
197
|
+
const lines = fileContent.split("\n");
|
|
198
|
+
// Keep track of the last line we've processed
|
|
199
|
+
let lastLine = -1;
|
|
200
|
+
captures.forEach((capture) => {
|
|
201
|
+
const { node, name } = capture;
|
|
202
|
+
// Get the start and end lines of the current AST node
|
|
203
|
+
const startLine = node.startPosition.row;
|
|
204
|
+
const endLine = node.endPosition.row;
|
|
205
|
+
// Once we've retrieved the nodes we care about through the language query, we filter for lines with definition names only.
|
|
206
|
+
// name.startsWith("name.reference.") > refs can be used for ranking purposes, but we don't need them for the output
|
|
207
|
+
// previously we did `name.startsWith("name.definition.")` but this was too strict and excluded some relevant definitions
|
|
208
|
+
// Add separator if there's a gap between captures
|
|
209
|
+
if (lastLine !== -1 && startLine > lastLine + 1) {
|
|
210
|
+
formattedOutput += "|----\n";
|
|
211
|
+
}
|
|
212
|
+
// Only add the first line of the definition
|
|
213
|
+
// query captures includes the definition name and the definition implementation, but we only want the name (I found discrepencies in the naming structure for various languages, i.e. javascript names would be 'name' and typescript names would be 'name.definition)
|
|
214
|
+
if (name.includes("name") && lines[startLine]) {
|
|
215
|
+
formattedOutput += `│${lines[startLine]}\n`;
|
|
216
|
+
}
|
|
217
|
+
// Adds all the captured lines
|
|
218
|
+
// for (let i = startLine; i <= endLine; i++) {
|
|
219
|
+
// formattedOutput += `│${lines[i]}\n`
|
|
220
|
+
// }
|
|
221
|
+
//}
|
|
222
|
+
lastLine = endLine;
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
console.log(`Error parsing file: ${error}\n`);
|
|
227
|
+
}
|
|
228
|
+
if (formattedOutput.length > 0) {
|
|
229
|
+
return `|----\n${formattedOutput}|----\n`;
|
|
230
|
+
}
|
|
231
|
+
return undefined;
|
|
232
|
+
}
|
|
233
|
+
exports.parseFile = parseFile;
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.loadRequiredLanguageParsers = void 0;
|
|
30
|
+
const path = __importStar(require("path"));
|
|
31
|
+
const web_tree_sitter_1 = __importDefault(require("web-tree-sitter"));
|
|
32
|
+
const queries_1 = require("./queries");
|
|
33
|
+
async function loadLanguage(langName) {
|
|
34
|
+
return await web_tree_sitter_1.default.Language.load(path.join(__dirname, `tree-sitter-${langName}.wasm`));
|
|
35
|
+
}
|
|
36
|
+
let isParserInitialized = false;
|
|
37
|
+
async function initializeParser() {
|
|
38
|
+
if (!isParserInitialized) {
|
|
39
|
+
await web_tree_sitter_1.default.init();
|
|
40
|
+
isParserInitialized = true;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/*
|
|
44
|
+
Using node bindings for tree-sitter is problematic in vscode extensions
|
|
45
|
+
because of incompatibility with electron. Going the .wasm route has the
|
|
46
|
+
advantage of not having to build for multiple architectures.
|
|
47
|
+
|
|
48
|
+
We use web-tree-sitter and tree-sitter-wasms which provides auto-updating prebuilt WASM binaries for tree-sitter's language parsers.
|
|
49
|
+
|
|
50
|
+
This function loads WASM modules for relevant language parsers based on input files:
|
|
51
|
+
1. Extracts unique file extensions
|
|
52
|
+
2. Maps extensions to language names
|
|
53
|
+
3. Loads corresponding WASM files (containing grammar rules)
|
|
54
|
+
4. Uses WASM modules to initialize tree-sitter parsers
|
|
55
|
+
|
|
56
|
+
This approach optimizes performance by loading only necessary parsers once for all relevant files.
|
|
57
|
+
|
|
58
|
+
Sources:
|
|
59
|
+
- https://github.com/tree-sitter/node-tree-sitter/issues/169
|
|
60
|
+
- https://github.com/tree-sitter/node-tree-sitter/issues/168
|
|
61
|
+
- https://github.com/Gregoor/tree-sitter-wasms/blob/main/README.md
|
|
62
|
+
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/README.md
|
|
63
|
+
- https://github.com/tree-sitter/tree-sitter/blob/master/lib/binding_web/test/query-test.js
|
|
64
|
+
*/
|
|
65
|
+
async function loadRequiredLanguageParsers(filesToParse) {
|
|
66
|
+
await initializeParser();
|
|
67
|
+
const extensionsToLoad = new Set(filesToParse.map((file) => path.extname(file).toLowerCase().slice(1)));
|
|
68
|
+
const parsers = {};
|
|
69
|
+
for (const ext of extensionsToLoad) {
|
|
70
|
+
let language;
|
|
71
|
+
let query;
|
|
72
|
+
switch (ext) {
|
|
73
|
+
case "js":
|
|
74
|
+
case "jsx":
|
|
75
|
+
language = await loadLanguage("javascript");
|
|
76
|
+
query = language.query(queries_1.javascriptQuery);
|
|
77
|
+
break;
|
|
78
|
+
case "ts":
|
|
79
|
+
language = await loadLanguage("typescript");
|
|
80
|
+
query = language.query(queries_1.typescriptQuery);
|
|
81
|
+
break;
|
|
82
|
+
case "tsx":
|
|
83
|
+
language = await loadLanguage("tsx");
|
|
84
|
+
query = language.query(queries_1.typescriptQuery);
|
|
85
|
+
break;
|
|
86
|
+
case "py":
|
|
87
|
+
language = await loadLanguage("python");
|
|
88
|
+
query = language.query(queries_1.pythonQuery);
|
|
89
|
+
break;
|
|
90
|
+
case "rs":
|
|
91
|
+
language = await loadLanguage("rust");
|
|
92
|
+
query = language.query(queries_1.rustQuery);
|
|
93
|
+
break;
|
|
94
|
+
case "go":
|
|
95
|
+
language = await loadLanguage("go");
|
|
96
|
+
query = language.query(queries_1.goQuery);
|
|
97
|
+
break;
|
|
98
|
+
case "cpp":
|
|
99
|
+
case "hpp":
|
|
100
|
+
language = await loadLanguage("cpp");
|
|
101
|
+
query = language.query(queries_1.cppQuery);
|
|
102
|
+
break;
|
|
103
|
+
case "c":
|
|
104
|
+
case "h":
|
|
105
|
+
language = await loadLanguage("c");
|
|
106
|
+
query = language.query(queries_1.cQuery);
|
|
107
|
+
break;
|
|
108
|
+
case "cs":
|
|
109
|
+
language = await loadLanguage("c_sharp");
|
|
110
|
+
query = language.query(queries_1.csharpQuery);
|
|
111
|
+
break;
|
|
112
|
+
case "rb":
|
|
113
|
+
language = await loadLanguage("ruby");
|
|
114
|
+
query = language.query(queries_1.rubyQuery);
|
|
115
|
+
break;
|
|
116
|
+
case "java":
|
|
117
|
+
language = await loadLanguage("java");
|
|
118
|
+
query = language.query(queries_1.javaQuery);
|
|
119
|
+
break;
|
|
120
|
+
case "php":
|
|
121
|
+
language = await loadLanguage("php");
|
|
122
|
+
query = language.query(queries_1.phpQuery);
|
|
123
|
+
break;
|
|
124
|
+
case "swift":
|
|
125
|
+
language = await loadLanguage("swift");
|
|
126
|
+
query = language.query(queries_1.swiftQuery);
|
|
127
|
+
break;
|
|
128
|
+
default:
|
|
129
|
+
throw new Error(`Unsupported language: ${ext}`);
|
|
130
|
+
}
|
|
131
|
+
const parser = new web_tree_sitter_1.default();
|
|
132
|
+
parser.setLanguage(language);
|
|
133
|
+
parsers[ext] = { parser, query };
|
|
134
|
+
}
|
|
135
|
+
return parsers;
|
|
136
|
+
}
|
|
137
|
+
exports.loadRequiredLanguageParsers = loadRequiredLanguageParsers;
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
declare const _default: "\n(class_declaration\n name: (identifier) @name.definition.class\n) @definition.class\n\n(interface_declaration\n name: (identifier) @name.definition.interface\n) @definition.interface\n\n(method_declaration\n name: (identifier) @name.definition.method\n) @definition.method\n\n(namespace_declaration\n name: (identifier) @name.definition.module\n) @definition.module\n";
|
|
2
|
+
export default _default;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/*
|
|
4
|
+
- class declarations
|
|
5
|
+
- interface declarations
|
|
6
|
+
- method declarations
|
|
7
|
+
- namespace declarations
|
|
8
|
+
*/
|
|
9
|
+
exports.default = `
|
|
10
|
+
(class_declaration
|
|
11
|
+
name: (identifier) @name.definition.class
|
|
12
|
+
) @definition.class
|
|
13
|
+
|
|
14
|
+
(interface_declaration
|
|
15
|
+
name: (identifier) @name.definition.interface
|
|
16
|
+
) @definition.interface
|
|
17
|
+
|
|
18
|
+
(method_declaration
|
|
19
|
+
name: (identifier) @name.definition.method
|
|
20
|
+
) @definition.method
|
|
21
|
+
|
|
22
|
+
(namespace_declaration
|
|
23
|
+
name: (identifier) @name.definition.module
|
|
24
|
+
) @definition.module
|
|
25
|
+
`;
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
declare const _default: "\n(struct_specifier name: (type_identifier) @name.definition.class body:(_)) @definition.class\n\n(declaration type: (union_specifier name: (type_identifier) @name.definition.class)) @definition.class\n\n(function_declarator declarator: (identifier) @name.definition.function) @definition.function\n\n(type_definition declarator: (type_identifier) @name.definition.type) @definition.type\n";
|
|
2
|
+
export default _default;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/*
|
|
4
|
+
- struct declarations
|
|
5
|
+
- union declarations
|
|
6
|
+
- function declarations
|
|
7
|
+
- typedef declarations
|
|
8
|
+
*/
|
|
9
|
+
exports.default = `
|
|
10
|
+
(struct_specifier name: (type_identifier) @name.definition.class body:(_)) @definition.class
|
|
11
|
+
|
|
12
|
+
(declaration type: (union_specifier name: (type_identifier) @name.definition.class)) @definition.class
|
|
13
|
+
|
|
14
|
+
(function_declarator declarator: (identifier) @name.definition.function) @definition.function
|
|
15
|
+
|
|
16
|
+
(type_definition declarator: (type_identifier) @name.definition.type) @definition.type
|
|
17
|
+
`;
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
declare const _default: "\n(struct_specifier name: (type_identifier) @name.definition.class body:(_)) @definition.class\n\n(declaration type: (union_specifier name: (type_identifier) @name.definition.class)) @definition.class\n\n(function_declarator declarator: (identifier) @name.definition.function) @definition.function\n\n(function_declarator declarator: (field_identifier) @name.definition.function) @definition.function\n\n(function_declarator declarator: (qualified_identifier scope: (namespace_identifier) @scope name: (identifier) @name.definition.method)) @definition.method\n\n(type_definition declarator: (type_identifier) @name.definition.type) @definition.type\n\n(class_specifier name: (type_identifier) @name.definition.class) @definition.class\n";
|
|
2
|
+
export default _default;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/*
|
|
4
|
+
- struct declarations
|
|
5
|
+
- union declarations
|
|
6
|
+
- function declarations
|
|
7
|
+
- method declarations (with namespace scope)
|
|
8
|
+
- typedef declarations
|
|
9
|
+
- class declarations
|
|
10
|
+
*/
|
|
11
|
+
exports.default = `
|
|
12
|
+
(struct_specifier name: (type_identifier) @name.definition.class body:(_)) @definition.class
|
|
13
|
+
|
|
14
|
+
(declaration type: (union_specifier name: (type_identifier) @name.definition.class)) @definition.class
|
|
15
|
+
|
|
16
|
+
(function_declarator declarator: (identifier) @name.definition.function) @definition.function
|
|
17
|
+
|
|
18
|
+
(function_declarator declarator: (field_identifier) @name.definition.function) @definition.function
|
|
19
|
+
|
|
20
|
+
(function_declarator declarator: (qualified_identifier scope: (namespace_identifier) @scope name: (identifier) @name.definition.method)) @definition.method
|
|
21
|
+
|
|
22
|
+
(type_definition declarator: (type_identifier) @name.definition.type) @definition.type
|
|
23
|
+
|
|
24
|
+
(class_specifier name: (type_identifier) @name.definition.class) @definition.class
|
|
25
|
+
`;
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
declare const _default: "\n(\n (comment)* @doc\n .\n (function_declaration\n name: (identifier) @name.definition.function) @definition.function\n (#strip! @doc \"^//\\s*\")\n (#set-adjacent! @doc @definition.function)\n)\n\n(\n (comment)* @doc\n .\n (method_declaration\n name: (field_identifier) @name.definition.method) @definition.method\n (#strip! @doc \"^//\\s*\")\n (#set-adjacent! @doc @definition.method)\n)\n\n(type_spec\n name: (type_identifier) @name.definition.type) @definition.type\n";
|
|
2
|
+
export default _default;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/*
|
|
4
|
+
- function declarations (with associated comments)
|
|
5
|
+
- method declarations (with associated comments)
|
|
6
|
+
- type specifications
|
|
7
|
+
*/
|
|
8
|
+
exports.default = `
|
|
9
|
+
(
|
|
10
|
+
(comment)* @doc
|
|
11
|
+
.
|
|
12
|
+
(function_declaration
|
|
13
|
+
name: (identifier) @name.definition.function) @definition.function
|
|
14
|
+
(#strip! @doc "^//\\s*")
|
|
15
|
+
(#set-adjacent! @doc @definition.function)
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
(
|
|
19
|
+
(comment)* @doc
|
|
20
|
+
.
|
|
21
|
+
(method_declaration
|
|
22
|
+
name: (field_identifier) @name.definition.method) @definition.method
|
|
23
|
+
(#strip! @doc "^//\\s*")
|
|
24
|
+
(#set-adjacent! @doc @definition.method)
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
(type_spec
|
|
28
|
+
name: (type_identifier) @name.definition.type) @definition.type
|
|
29
|
+
`;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { default as phpQuery } from "./php";
|
|
2
|
+
export { default as typescriptQuery } from "./typescript";
|
|
3
|
+
export { default as pythonQuery } from "./python";
|
|
4
|
+
export { default as javascriptQuery } from "./javascript";
|
|
5
|
+
export { default as javaQuery } from "./java";
|
|
6
|
+
export { default as rustQuery } from "./rust";
|
|
7
|
+
export { default as rubyQuery } from "./ruby";
|
|
8
|
+
export { default as cppQuery } from "./cpp";
|
|
9
|
+
export { default as cQuery } from "./c";
|
|
10
|
+
export { default as csharpQuery } from "./c-sharp";
|
|
11
|
+
export { default as goQuery } from "./go";
|
|
12
|
+
export { default as swiftQuery } from "./swift";
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.swiftQuery = exports.goQuery = exports.csharpQuery = exports.cQuery = exports.cppQuery = exports.rubyQuery = exports.rustQuery = exports.javaQuery = exports.javascriptQuery = exports.pythonQuery = exports.typescriptQuery = exports.phpQuery = void 0;
|
|
7
|
+
var php_1 = require("./php");
|
|
8
|
+
Object.defineProperty(exports, "phpQuery", { enumerable: true, get: function () { return __importDefault(php_1).default; } });
|
|
9
|
+
var typescript_1 = require("./typescript");
|
|
10
|
+
Object.defineProperty(exports, "typescriptQuery", { enumerable: true, get: function () { return __importDefault(typescript_1).default; } });
|
|
11
|
+
var python_1 = require("./python");
|
|
12
|
+
Object.defineProperty(exports, "pythonQuery", { enumerable: true, get: function () { return __importDefault(python_1).default; } });
|
|
13
|
+
var javascript_1 = require("./javascript");
|
|
14
|
+
Object.defineProperty(exports, "javascriptQuery", { enumerable: true, get: function () { return __importDefault(javascript_1).default; } });
|
|
15
|
+
var java_1 = require("./java");
|
|
16
|
+
Object.defineProperty(exports, "javaQuery", { enumerable: true, get: function () { return __importDefault(java_1).default; } });
|
|
17
|
+
var rust_1 = require("./rust");
|
|
18
|
+
Object.defineProperty(exports, "rustQuery", { enumerable: true, get: function () { return __importDefault(rust_1).default; } });
|
|
19
|
+
var ruby_1 = require("./ruby");
|
|
20
|
+
Object.defineProperty(exports, "rubyQuery", { enumerable: true, get: function () { return __importDefault(ruby_1).default; } });
|
|
21
|
+
var cpp_1 = require("./cpp");
|
|
22
|
+
Object.defineProperty(exports, "cppQuery", { enumerable: true, get: function () { return __importDefault(cpp_1).default; } });
|
|
23
|
+
var c_1 = require("./c");
|
|
24
|
+
Object.defineProperty(exports, "cQuery", { enumerable: true, get: function () { return __importDefault(c_1).default; } });
|
|
25
|
+
var c_sharp_1 = require("./c-sharp");
|
|
26
|
+
Object.defineProperty(exports, "csharpQuery", { enumerable: true, get: function () { return __importDefault(c_sharp_1).default; } });
|
|
27
|
+
var go_1 = require("./go");
|
|
28
|
+
Object.defineProperty(exports, "goQuery", { enumerable: true, get: function () { return __importDefault(go_1).default; } });
|
|
29
|
+
var swift_1 = require("./swift");
|
|
30
|
+
Object.defineProperty(exports, "swiftQuery", { enumerable: true, get: function () { return __importDefault(swift_1).default; } });
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
declare const _default: "\n(class_declaration\n name: (identifier) @name.definition.class) @definition.class\n\n(method_declaration\n name: (identifier) @name.definition.method) @definition.method\n\n(interface_declaration\n name: (identifier) @name.definition.interface) @definition.interface\n";
|
|
2
|
+
export default _default;
|