@mhalder/qdrant-mcp-server 1.3.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codecov.yml +16 -0
- package/CHANGELOG.md +25 -0
- package/README.md +304 -9
- package/build/code/chunker/base.d.ts +19 -0
- package/build/code/chunker/base.d.ts.map +1 -0
- package/build/code/chunker/base.js +5 -0
- package/build/code/chunker/base.js.map +1 -0
- package/build/code/chunker/character-chunker.d.ts +22 -0
- package/build/code/chunker/character-chunker.d.ts.map +1 -0
- package/build/code/chunker/character-chunker.js +111 -0
- package/build/code/chunker/character-chunker.js.map +1 -0
- package/build/code/chunker/tree-sitter-chunker.d.ts +29 -0
- package/build/code/chunker/tree-sitter-chunker.d.ts.map +1 -0
- package/build/code/chunker/tree-sitter-chunker.js +213 -0
- package/build/code/chunker/tree-sitter-chunker.js.map +1 -0
- package/build/code/config.d.ts +11 -0
- package/build/code/config.d.ts.map +1 -0
- package/build/code/config.js +145 -0
- package/build/code/config.js.map +1 -0
- package/build/code/indexer.d.ts +42 -0
- package/build/code/indexer.d.ts.map +1 -0
- package/build/code/indexer.js +508 -0
- package/build/code/indexer.js.map +1 -0
- package/build/code/metadata.d.ts +32 -0
- package/build/code/metadata.d.ts.map +1 -0
- package/build/code/metadata.js +128 -0
- package/build/code/metadata.js.map +1 -0
- package/build/code/scanner.d.ts +35 -0
- package/build/code/scanner.d.ts.map +1 -0
- package/build/code/scanner.js +108 -0
- package/build/code/scanner.js.map +1 -0
- package/build/code/sync/merkle.d.ts +45 -0
- package/build/code/sync/merkle.d.ts.map +1 -0
- package/build/code/sync/merkle.js +116 -0
- package/build/code/sync/merkle.js.map +1 -0
- package/build/code/sync/snapshot.d.ts +41 -0
- package/build/code/sync/snapshot.d.ts.map +1 -0
- package/build/code/sync/snapshot.js +91 -0
- package/build/code/sync/snapshot.js.map +1 -0
- package/build/code/sync/synchronizer.d.ts +53 -0
- package/build/code/sync/synchronizer.d.ts.map +1 -0
- package/build/code/sync/synchronizer.js +132 -0
- package/build/code/sync/synchronizer.js.map +1 -0
- package/build/code/types.d.ts +98 -0
- package/build/code/types.d.ts.map +1 -0
- package/build/code/types.js +5 -0
- package/build/code/types.js.map +1 -0
- package/build/index.js +321 -6
- package/build/index.js.map +1 -1
- package/build/prompts/index.d.ts +7 -0
- package/build/prompts/index.d.ts.map +1 -0
- package/build/prompts/index.js +7 -0
- package/build/prompts/index.js.map +1 -0
- package/build/prompts/index.test.d.ts +2 -0
- package/build/prompts/index.test.d.ts.map +1 -0
- package/build/prompts/index.test.js +25 -0
- package/build/prompts/index.test.js.map +1 -0
- package/build/prompts/loader.d.ts +25 -0
- package/build/prompts/loader.d.ts.map +1 -0
- package/build/prompts/loader.js +81 -0
- package/build/prompts/loader.js.map +1 -0
- package/build/prompts/loader.test.d.ts +2 -0
- package/build/prompts/loader.test.d.ts.map +1 -0
- package/build/prompts/loader.test.js +417 -0
- package/build/prompts/loader.test.js.map +1 -0
- package/build/prompts/template.d.ts +20 -0
- package/build/prompts/template.d.ts.map +1 -0
- package/build/prompts/template.js +52 -0
- package/build/prompts/template.js.map +1 -0
- package/build/prompts/template.test.d.ts +2 -0
- package/build/prompts/template.test.d.ts.map +1 -0
- package/build/prompts/template.test.js +163 -0
- package/build/prompts/template.test.js.map +1 -0
- package/build/prompts/types.d.ts +34 -0
- package/build/prompts/types.d.ts.map +1 -0
- package/build/prompts/types.js +5 -0
- package/build/prompts/types.js.map +1 -0
- package/examples/code-search/README.md +271 -0
- package/package.json +13 -1
- package/prompts.example.json +96 -0
- package/src/code/chunker/base.ts +22 -0
- package/src/code/chunker/character-chunker.ts +131 -0
- package/src/code/chunker/tree-sitter-chunker.ts +250 -0
- package/src/code/config.ts +156 -0
- package/src/code/indexer.ts +613 -0
- package/src/code/metadata.ts +153 -0
- package/src/code/scanner.ts +124 -0
- package/src/code/sync/merkle.ts +136 -0
- package/src/code/sync/snapshot.ts +110 -0
- package/src/code/sync/synchronizer.ts +154 -0
- package/src/code/types.ts +117 -0
- package/src/index.ts +382 -5
- package/src/prompts/index.test.ts +29 -0
- package/src/prompts/index.ts +7 -0
- package/src/prompts/loader.test.ts +494 -0
- package/src/prompts/loader.ts +90 -0
- package/src/prompts/template.test.ts +212 -0
- package/src/prompts/template.ts +69 -0
- package/src/prompts/types.ts +37 -0
- package/tests/code/chunker/character-chunker.test.ts +141 -0
- package/tests/code/chunker/tree-sitter-chunker.test.ts +275 -0
- package/tests/code/fixtures/sample-py/calculator.py +32 -0
- package/tests/code/fixtures/sample-ts/async-operations.ts +120 -0
- package/tests/code/fixtures/sample-ts/auth.ts +31 -0
- package/tests/code/fixtures/sample-ts/config.ts +52 -0
- package/tests/code/fixtures/sample-ts/database.ts +50 -0
- package/tests/code/fixtures/sample-ts/index.ts +39 -0
- package/tests/code/fixtures/sample-ts/types-advanced.ts +132 -0
- package/tests/code/fixtures/sample-ts/utils.ts +105 -0
- package/tests/code/fixtures/sample-ts/validator.ts +169 -0
- package/tests/code/indexer.test.ts +828 -0
- package/tests/code/integration.test.ts +708 -0
- package/tests/code/metadata.test.ts +457 -0
- package/tests/code/scanner.test.ts +131 -0
- package/tests/code/sync/merkle.test.ts +406 -0
- package/tests/code/sync/snapshot.test.ts +360 -0
- package/tests/code/sync/synchronizer.test.ts +501 -0
- package/vitest.config.ts +1 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TreeSitterChunker - AST-aware code chunking using tree-sitter
|
|
3
|
+
* Primary chunking strategy for supported languages
|
|
4
|
+
*/
|
|
5
|
+
import Parser from "tree-sitter";
|
|
6
|
+
// tree-sitter language modules don't have proper types
|
|
7
|
+
import Bash from "tree-sitter-bash";
|
|
8
|
+
import Go from "tree-sitter-go";
|
|
9
|
+
import Java from "tree-sitter-java";
|
|
10
|
+
import JavaScript from "tree-sitter-javascript";
|
|
11
|
+
import Python from "tree-sitter-python";
|
|
12
|
+
import Rust from "tree-sitter-rust";
|
|
13
|
+
import TypeScript from "tree-sitter-typescript";
|
|
14
|
+
import { CharacterChunker } from "./character-chunker.js";
|
|
15
|
+
export class TreeSitterChunker {
|
|
16
|
+
config;
|
|
17
|
+
languages = new Map();
|
|
18
|
+
fallbackChunker;
|
|
19
|
+
constructor(config) {
|
|
20
|
+
this.config = config;
|
|
21
|
+
this.fallbackChunker = new CharacterChunker(config);
|
|
22
|
+
this.initializeParsers();
|
|
23
|
+
}
|
|
24
|
+
initializeParsers() {
|
|
25
|
+
// TypeScript
|
|
26
|
+
const tsParser = new Parser();
|
|
27
|
+
tsParser.setLanguage(TypeScript.typescript);
|
|
28
|
+
this.languages.set("typescript", {
|
|
29
|
+
parser: tsParser,
|
|
30
|
+
chunkableTypes: [
|
|
31
|
+
"function_declaration",
|
|
32
|
+
"method_definition",
|
|
33
|
+
"class_declaration",
|
|
34
|
+
"interface_declaration",
|
|
35
|
+
"type_alias_declaration",
|
|
36
|
+
"enum_declaration",
|
|
37
|
+
],
|
|
38
|
+
});
|
|
39
|
+
// JavaScript
|
|
40
|
+
const jsParser = new Parser();
|
|
41
|
+
jsParser.setLanguage(JavaScript);
|
|
42
|
+
this.languages.set("javascript", {
|
|
43
|
+
parser: jsParser,
|
|
44
|
+
chunkableTypes: [
|
|
45
|
+
"function_declaration",
|
|
46
|
+
"method_definition",
|
|
47
|
+
"class_declaration",
|
|
48
|
+
"export_statement",
|
|
49
|
+
],
|
|
50
|
+
});
|
|
51
|
+
// Python
|
|
52
|
+
const pyParser = new Parser();
|
|
53
|
+
pyParser.setLanguage(Python);
|
|
54
|
+
this.languages.set("python", {
|
|
55
|
+
parser: pyParser,
|
|
56
|
+
chunkableTypes: ["function_definition", "class_definition", "decorated_definition"],
|
|
57
|
+
});
|
|
58
|
+
// Go
|
|
59
|
+
const goParser = new Parser();
|
|
60
|
+
goParser.setLanguage(Go);
|
|
61
|
+
this.languages.set("go", {
|
|
62
|
+
parser: goParser,
|
|
63
|
+
chunkableTypes: [
|
|
64
|
+
"function_declaration",
|
|
65
|
+
"method_declaration",
|
|
66
|
+
"type_declaration",
|
|
67
|
+
"interface_declaration",
|
|
68
|
+
],
|
|
69
|
+
});
|
|
70
|
+
// Rust
|
|
71
|
+
const rustParser = new Parser();
|
|
72
|
+
rustParser.setLanguage(Rust);
|
|
73
|
+
this.languages.set("rust", {
|
|
74
|
+
parser: rustParser,
|
|
75
|
+
chunkableTypes: ["function_item", "impl_item", "trait_item", "struct_item", "enum_item"],
|
|
76
|
+
});
|
|
77
|
+
// Java
|
|
78
|
+
const javaParser = new Parser();
|
|
79
|
+
javaParser.setLanguage(Java);
|
|
80
|
+
this.languages.set("java", {
|
|
81
|
+
parser: javaParser,
|
|
82
|
+
chunkableTypes: [
|
|
83
|
+
"method_declaration",
|
|
84
|
+
"class_declaration",
|
|
85
|
+
"interface_declaration",
|
|
86
|
+
"enum_declaration",
|
|
87
|
+
],
|
|
88
|
+
});
|
|
89
|
+
// Bash
|
|
90
|
+
const bashParser = new Parser();
|
|
91
|
+
bashParser.setLanguage(Bash);
|
|
92
|
+
this.languages.set("bash", {
|
|
93
|
+
parser: bashParser,
|
|
94
|
+
chunkableTypes: ["function_definition", "command"],
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
async chunk(code, filePath, language) {
|
|
98
|
+
const langConfig = this.languages.get(language);
|
|
99
|
+
if (!langConfig) {
|
|
100
|
+
// Fallback to character-based chunking
|
|
101
|
+
return this.fallbackChunker.chunk(code, filePath, language);
|
|
102
|
+
}
|
|
103
|
+
try {
|
|
104
|
+
const tree = langConfig.parser.parse(code);
|
|
105
|
+
const chunks = [];
|
|
106
|
+
// Find all chunkable nodes
|
|
107
|
+
const nodes = this.findChunkableNodes(tree.rootNode, langConfig.chunkableTypes);
|
|
108
|
+
for (const [index, node] of nodes.entries()) {
|
|
109
|
+
const content = code.substring(node.startIndex, node.endIndex);
|
|
110
|
+
// Skip chunks that are too small
|
|
111
|
+
if (content.length < 50) {
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
// If chunk is too large, fall back to character chunking for this node
|
|
115
|
+
if (content.length > this.config.maxChunkSize * 2) {
|
|
116
|
+
const subChunks = await this.fallbackChunker.chunk(content, filePath, language);
|
|
117
|
+
// Adjust line numbers for sub-chunks
|
|
118
|
+
for (const subChunk of subChunks) {
|
|
119
|
+
chunks.push({
|
|
120
|
+
...subChunk,
|
|
121
|
+
startLine: node.startPosition.row + 1 + subChunk.startLine - 1,
|
|
122
|
+
endLine: node.startPosition.row + 1 + subChunk.endLine - 1,
|
|
123
|
+
metadata: {
|
|
124
|
+
...subChunk.metadata,
|
|
125
|
+
chunkIndex: chunks.length,
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
chunks.push({
|
|
132
|
+
content: content.trim(),
|
|
133
|
+
startLine: node.startPosition.row + 1,
|
|
134
|
+
endLine: node.endPosition.row + 1,
|
|
135
|
+
metadata: {
|
|
136
|
+
filePath,
|
|
137
|
+
language,
|
|
138
|
+
chunkIndex: index,
|
|
139
|
+
chunkType: this.getChunkType(node.type),
|
|
140
|
+
name: this.extractName(node, code),
|
|
141
|
+
},
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
// If no chunks found or file is small, use fallback
|
|
145
|
+
if (chunks.length === 0 && code.length > 100) {
|
|
146
|
+
return this.fallbackChunker.chunk(code, filePath, language);
|
|
147
|
+
}
|
|
148
|
+
return chunks;
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
// On parsing error, fallback to character-based chunking
|
|
152
|
+
console.error(`Tree-sitter parsing failed for ${filePath}:`, error);
|
|
153
|
+
return this.fallbackChunker.chunk(code, filePath, language);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
supportsLanguage(language) {
|
|
157
|
+
return this.languages.has(language);
|
|
158
|
+
}
|
|
159
|
+
getStrategyName() {
|
|
160
|
+
return "tree-sitter";
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Find all chunkable nodes in the AST
|
|
164
|
+
*/
|
|
165
|
+
findChunkableNodes(node, chunkableTypes) {
|
|
166
|
+
const nodes = [];
|
|
167
|
+
const traverse = (n) => {
|
|
168
|
+
if (chunkableTypes.includes(n.type)) {
|
|
169
|
+
nodes.push(n);
|
|
170
|
+
// Don't traverse children of chunkable nodes to avoid nested chunks
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
for (const child of n.children) {
|
|
174
|
+
traverse(child);
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
traverse(node);
|
|
178
|
+
return nodes;
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Extract function/class name from AST node
|
|
182
|
+
*/
|
|
183
|
+
extractName(node, code) {
|
|
184
|
+
// Try to find name node
|
|
185
|
+
const nameNode = node.childForFieldName("name");
|
|
186
|
+
if (nameNode) {
|
|
187
|
+
return code.substring(nameNode.startIndex, nameNode.endIndex);
|
|
188
|
+
}
|
|
189
|
+
// For some node types, name might be in a different location
|
|
190
|
+
for (const child of node.children) {
|
|
191
|
+
if (child.type === "identifier" || child.type === "type_identifier") {
|
|
192
|
+
return code.substring(child.startIndex, child.endIndex);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return undefined;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Map AST node type to chunk type
|
|
199
|
+
*/
|
|
200
|
+
getChunkType(nodeType) {
|
|
201
|
+
if (nodeType.includes("function") || nodeType.includes("method")) {
|
|
202
|
+
return "function";
|
|
203
|
+
}
|
|
204
|
+
if (nodeType.includes("class") || nodeType.includes("struct")) {
|
|
205
|
+
return "class";
|
|
206
|
+
}
|
|
207
|
+
if (nodeType.includes("interface") || nodeType.includes("trait")) {
|
|
208
|
+
return "interface";
|
|
209
|
+
}
|
|
210
|
+
return "block";
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
//# sourceMappingURL=tree-sitter-chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tree-sitter-chunker.js","sourceRoot":"","sources":["../../../src/code/chunker/tree-sitter-chunker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,MAAM,MAAM,aAAa,CAAC;AACjC,uDAAuD;AACvD,OAAO,IAAI,MAAM,kBAAkB,CAAC;AACpC,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAChC,OAAO,IAAI,MAAM,kBAAkB,CAAC;AACpC,OAAO,UAAU,MAAM,wBAAwB,CAAC;AAChD,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,IAAI,MAAM,kBAAkB,CAAC;AACpC,OAAO,UAAU,MAAM,wBAAwB,CAAC;AAIhD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAO1D,MAAM,OAAO,iBAAiB;IAIR;IAHZ,SAAS,GAAgC,IAAI,GAAG,EAAE,CAAC;IACnD,eAAe,CAAmB;IAE1C,YAAoB,MAAqB;QAArB,WAAM,GAAN,MAAM,CAAe;QACvC,IAAI,CAAC,eAAe,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEO,iBAAiB;QACvB,aAAa;QACb,MAAM,QAAQ,GAAG,IAAI,MAAM,EAAE,CAAC;QAC9B,QAAQ,CAAC,WAAW,CAAC,UAAU,CAAC,UAAiB,CAAC,CAAC;QACnD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,EAAE;YAC/B,MAAM,EAAE,QAAQ;YAChB,cAAc,EAAE;gBACd,sBAAsB;gBACtB,mBAAmB;gBACnB,mBAAmB;gBACnB,uBAAuB;gBACvB,wBAAwB;gBACxB,kBAAkB;aACnB;SACF,CAAC,CAAC;QAEH,aAAa;QACb,MAAM,QAAQ,GAAG,IAAI,MAAM,EAAE,CAAC;QAC9B,QAAQ,CAAC,WAAW,CAAC,UAAiB,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,EAAE;YAC/B,MAAM,EAAE,QAAQ;YAChB,cAAc,EAAE;gBACd,sBAAsB;gBACtB,mBAAmB;gBACnB,mBAAmB;gBACnB,kBAAkB;aACnB;SACF,CAAC,CAAC;QAEH,SAAS;QACT,MAAM,QAAQ,GAAG,IAAI,MAAM,EAAE,CAAC;QAC9B,QAAQ,CAAC,WAAW,CAAC,MAAa,CAAC,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC3B,MAAM,EAAE,QAAQ;YAChB,cAAc,EAAE,CAAC,qBAAqB,EAAE,kBAAkB,EAAE,sBAAsB,CAAC;SACpF,CAAC,CAAC;QAEH,KAAK;QACL,MAAM,QAAQ,GAAG,IAAI,MAAM,EAAE,CAAC;QAC9B,QAAQ,CAAC,WAAW,CAAC,EAAS,CAAC,CAAC;QAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,QAAQ;YAChB,cAAc,EAAE;gBACd,sBAAsB;gBACtB,oBAAoB;gBACpB,kBAAkB;gBAClB,uBAAuB;aACxB;SACF,CAAC,CAAC;QAEH,OAAO;QACP,MAAM,UAAU,GAAG,IAAI,MAAM,EAAE,CAAC;QAChC,UAAU,CAAC,WAAW,CAAC,IAAW,CAAC,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE;YACzB,MAAM,EAAE,UAAU;YAClB,cAAc,EAAE,CAAC,eAAe,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,CAAC;SACzF,CAAC,CAAC;QAEH,OAAO;QACP,MAAM,UAAU,GAAG,IAAI,MAAM,EAAE,CAAC;QAChC,UAAU,CAAC,WAAW,CAAC,IAAW,CAAC,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE;YACzB,MAAM,EAAE,UAAU;YAClB,cAAc,EAAE;gBACd,oBAAoB;gBACpB,mBAAmB;gBACnB,uBAAuB;gBACvB,kBAAkB;aACnB;SACF,CAAC,CAAC;QAEH,OAAO;QACP,MAAM,UAAU,GAAG,IAAI,MAAM,EAAE,CAAC;QAChC,UAAU,CAAC,WAAW,CAAC,IAAW,CAAC,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE;YACzB,MAAM,EAAE,UAAU;YAClB,cAAc,EAAE,CAAC,qBAAqB,EAAE,SAAS,CAAC;SACnD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAgB,EAAE,QAAgB;QAC1D,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEhD,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,uCAAuC;YACvC,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAgB,EAAE,CAAC;YAE/B,2BAA2B;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC;YAEhF,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;gBAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE/D,iCAAiC;gBACjC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,uEAAuE;gBACvE,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;oBAClD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;oBAChF,qCAAqC;oBACrC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;wBACjC,MAAM,CAAC,IAAI,CAAC;4BACV,GAAG,QAAQ;4BACX,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,SAAS,GAAG,CAAC;4BAC9D,OAAO,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,GAAG,CAAC;4BAC1D,QAAQ,EAAE;gCACR,GAAG,QAAQ,CAAC,QAAQ;gCACpB,UAAU,EAAE,MAAM,CAAC,MAAM;6BAC1B;yBACF,CAAC,CAAC;oBACL,CAAC;oBACD,SAAS;gBACX,CAAC;gBAED,MAAM,CAAC,IAAI,CAAC;oBACV,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;oBACvB,SAAS,EAAE,IAAI,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC;oBACrC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC;oBACjC,QAAQ,EAAE;wBACR,QAAQ;wBACR,QAAQ;wBACR,UAAU,EAAE,KAAK;wBACjB,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;wBACvC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC;qBACnC;iBACF,CAAC,CAAC;YACL,CAAC;YAED,oDAAoD;YACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC7C,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC9D,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yDAAyD;YACzD,OAAO,CAAC,KAAK,CAAC,kCAAkC,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,gBAAgB,CAAC,QAAgB;QAC/B,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED,eAAe;QACb,OAAO,aAAa,CAAC;IACvB,CAAC;IAED;;OAEG;IACK,kBAAkB,CACxB,IAAuB,EACvB,cAAwB;QAExB,MAAM,KAAK,GAAwB,EAAE,CAAC;QAEtC,MAAM,QAAQ,GAAG,CAAC,CAAoB,EAAE,EAAE;YACxC,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACd,oEAAoE;gBACpE,OAAO;YACT,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC/B,QAAQ,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;QACH,CAAC,CAAC;QAEF,QAAQ,CAAC,IAAI,CAAC,CAAC;QACf,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,IAAuB,EAAE,IAAY;QACvD,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChE,CAAC;QAED,6DAA6D;QAC7D,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBACpE,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,QAAgB;QACnC,IAAI,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjE,OAAO,UAAU,CAAC;QACpB,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9D,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,IAAI,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACjE,OAAO,WAAW,CAAC;QACrB,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration and constants for code vectorization
|
|
3
|
+
*/
|
|
4
|
+
export declare const DEFAULT_CODE_EXTENSIONS: string[];
|
|
5
|
+
export declare const DEFAULT_IGNORE_PATTERNS: string[];
|
|
6
|
+
export declare const LANGUAGE_MAP: Record<string, string>;
|
|
7
|
+
export declare const DEFAULT_CHUNK_SIZE = 2500;
|
|
8
|
+
export declare const DEFAULT_CHUNK_OVERLAP = 300;
|
|
9
|
+
export declare const DEFAULT_BATCH_SIZE = 100;
|
|
10
|
+
export declare const DEFAULT_SEARCH_LIMIT = 5;
|
|
11
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/code/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,eAAO,MAAM,uBAAuB,UA6DnC,CAAC;AAEF,eAAO,MAAM,uBAAuB,UAsBnC,CAAC;AAEF,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CA2D/C,CAAC;AAEF,eAAO,MAAM,kBAAkB,OAAO,CAAC;AACvC,eAAO,MAAM,qBAAqB,MAAM,CAAC;AACzC,eAAO,MAAM,kBAAkB,MAAM,CAAC;AACtC,eAAO,MAAM,oBAAoB,IAAI,CAAC"}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration and constants for code vectorization
|
|
3
|
+
*/
|
|
4
|
+
export const DEFAULT_CODE_EXTENSIONS = [
|
|
5
|
+
// TypeScript/JavaScript
|
|
6
|
+
".ts",
|
|
7
|
+
".tsx",
|
|
8
|
+
".js",
|
|
9
|
+
".jsx",
|
|
10
|
+
// Python
|
|
11
|
+
".py",
|
|
12
|
+
// Go
|
|
13
|
+
".go",
|
|
14
|
+
// Rust
|
|
15
|
+
".rs",
|
|
16
|
+
// Java/Kotlin
|
|
17
|
+
".java",
|
|
18
|
+
".kt",
|
|
19
|
+
// C/C++
|
|
20
|
+
".c",
|
|
21
|
+
".cpp",
|
|
22
|
+
".h",
|
|
23
|
+
".hpp",
|
|
24
|
+
".cc",
|
|
25
|
+
".cxx",
|
|
26
|
+
// C#
|
|
27
|
+
".cs",
|
|
28
|
+
// Ruby
|
|
29
|
+
".rb",
|
|
30
|
+
// PHP
|
|
31
|
+
".php",
|
|
32
|
+
// Swift
|
|
33
|
+
".swift",
|
|
34
|
+
// Dart
|
|
35
|
+
".dart",
|
|
36
|
+
// Scala
|
|
37
|
+
".scala",
|
|
38
|
+
// Clojure
|
|
39
|
+
".clj",
|
|
40
|
+
".cljs",
|
|
41
|
+
// Haskell
|
|
42
|
+
".hs",
|
|
43
|
+
// OCaml
|
|
44
|
+
".ml",
|
|
45
|
+
// Shell
|
|
46
|
+
".sh",
|
|
47
|
+
".bash",
|
|
48
|
+
".zsh",
|
|
49
|
+
".fish",
|
|
50
|
+
// SQL/Data
|
|
51
|
+
".sql",
|
|
52
|
+
".proto",
|
|
53
|
+
".graphql",
|
|
54
|
+
// Web
|
|
55
|
+
".vue",
|
|
56
|
+
".svelte",
|
|
57
|
+
// Config/Markup
|
|
58
|
+
".md",
|
|
59
|
+
".markdown",
|
|
60
|
+
".json",
|
|
61
|
+
".yaml",
|
|
62
|
+
".yml",
|
|
63
|
+
".toml",
|
|
64
|
+
".xml",
|
|
65
|
+
];
|
|
66
|
+
export const DEFAULT_IGNORE_PATTERNS = [
|
|
67
|
+
"node_modules/**",
|
|
68
|
+
"dist/**",
|
|
69
|
+
"build/**",
|
|
70
|
+
"out/**",
|
|
71
|
+
"target/**",
|
|
72
|
+
"coverage/**",
|
|
73
|
+
".nyc_output/**",
|
|
74
|
+
".cache/**",
|
|
75
|
+
"__pycache__/**",
|
|
76
|
+
".git/**",
|
|
77
|
+
".svn/**",
|
|
78
|
+
".hg/**",
|
|
79
|
+
".vscode/**",
|
|
80
|
+
".idea/**",
|
|
81
|
+
"*.min.js",
|
|
82
|
+
"*.min.css",
|
|
83
|
+
"*.bundle.js",
|
|
84
|
+
"*.map",
|
|
85
|
+
"*.log",
|
|
86
|
+
".env",
|
|
87
|
+
".env.*",
|
|
88
|
+
];
|
|
89
|
+
export const LANGUAGE_MAP = {
|
|
90
|
+
// TypeScript/JavaScript
|
|
91
|
+
".ts": "typescript",
|
|
92
|
+
".tsx": "typescript",
|
|
93
|
+
".js": "javascript",
|
|
94
|
+
".jsx": "javascript",
|
|
95
|
+
// Backend languages
|
|
96
|
+
".py": "python",
|
|
97
|
+
".java": "java",
|
|
98
|
+
".go": "go",
|
|
99
|
+
".rs": "rust",
|
|
100
|
+
".rb": "ruby",
|
|
101
|
+
".php": "php",
|
|
102
|
+
// Systems languages
|
|
103
|
+
".c": "c",
|
|
104
|
+
".cpp": "cpp",
|
|
105
|
+
".cc": "cpp",
|
|
106
|
+
".cxx": "cpp",
|
|
107
|
+
".h": "c",
|
|
108
|
+
".hpp": "cpp",
|
|
109
|
+
".cs": "c_sharp",
|
|
110
|
+
// Mobile
|
|
111
|
+
".swift": "swift",
|
|
112
|
+
".kt": "kotlin",
|
|
113
|
+
".dart": "dart",
|
|
114
|
+
// Functional
|
|
115
|
+
".scala": "scala",
|
|
116
|
+
".clj": "clojure",
|
|
117
|
+
".cljs": "clojure",
|
|
118
|
+
".hs": "haskell",
|
|
119
|
+
".ml": "ocaml",
|
|
120
|
+
// Scripting
|
|
121
|
+
".sh": "bash",
|
|
122
|
+
".bash": "bash",
|
|
123
|
+
".zsh": "bash",
|
|
124
|
+
".fish": "fish",
|
|
125
|
+
// Data/Query
|
|
126
|
+
".sql": "sql",
|
|
127
|
+
".proto": "proto",
|
|
128
|
+
".graphql": "graphql",
|
|
129
|
+
// Markup/Config
|
|
130
|
+
".md": "markdown",
|
|
131
|
+
".markdown": "markdown",
|
|
132
|
+
".json": "json",
|
|
133
|
+
".yaml": "yaml",
|
|
134
|
+
".yml": "yaml",
|
|
135
|
+
".toml": "toml",
|
|
136
|
+
".xml": "xml",
|
|
137
|
+
// Web
|
|
138
|
+
".vue": "vue",
|
|
139
|
+
".svelte": "svelte",
|
|
140
|
+
};
|
|
141
|
+
export const DEFAULT_CHUNK_SIZE = 2500;
|
|
142
|
+
export const DEFAULT_CHUNK_OVERLAP = 300;
|
|
143
|
+
export const DEFAULT_BATCH_SIZE = 100;
|
|
144
|
+
export const DEFAULT_SEARCH_LIMIT = 5;
|
|
145
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/code/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,wBAAwB;IACxB,KAAK;IACL,MAAM;IACN,KAAK;IACL,MAAM;IACN,SAAS;IACT,KAAK;IACL,KAAK;IACL,KAAK;IACL,OAAO;IACP,KAAK;IACL,cAAc;IACd,OAAO;IACP,KAAK;IACL,QAAQ;IACR,IAAI;IACJ,MAAM;IACN,IAAI;IACJ,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,OAAO;IACP,KAAK;IACL,MAAM;IACN,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,MAAM;IACN,OAAO;IACP,UAAU;IACV,KAAK;IACL,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,KAAK;IACL,OAAO;IACP,MAAM;IACN,OAAO;IACP,WAAW;IACX,MAAM;IACN,QAAQ;IACR,UAAU;IACV,MAAM;IACN,MAAM;IACN,SAAS;IACT,gBAAgB;IAChB,KAAK;IACL,WAAW;IACX,OAAO;IACP,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;CACP,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,iBAAiB;IACjB,SAAS;IACT,UAAU;IACV,QAAQ;IACR,WAAW;IACX,aAAa;IACb,gBAAgB;IAChB,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,SAAS;IACT,QAAQ;IACR,YAAY;IACZ,UAAU;IACV,UAAU;IACV,WAAW;IACX,aAAa;IACb,OAAO;IACP,OAAO;IACP,MAAM;IACN,QAAQ;CACT,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAA2B;IAClD,wBAAwB;IACxB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,YAAY;IAEpB,oBAAoB;IACpB,KAAK,EAAE,QAAQ;IACf,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,IAAI;IACX,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IAEb,oBAAoB;IACpB,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,GAAG;IACT,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,SAAS;IAEhB,SAAS;IACT,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,QAAQ;IACf,OAAO,EAAE,MAAM;IAEf,aAAa;IACb,QAAQ,EAAE,OAAO;IACjB,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,SAAS;IAClB,KAAK,EAAE,SAAS;IAChB,KAAK,EAAE,OAAO;IAEd,YAAY;IACZ,KAAK,EAAE,MAAM;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IAEf,aAAa;IACb,MAAM,EAAE,KAAK;IACb,QAAQ,EAAE,OAAO;IACjB,UAAU,EAAE,SAAS;IAErB,gBAAgB;IAChB,KAAK,EAAE,UAAU;IACjB,WAAW,EAAE,UAAU;IACvB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IAEb,MAAM;IACN,MAAM,EAAE,KAAK;IACb,SAAS,EAAE,QAAQ;CACpB,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AACvC,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AACzC,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AACtC,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CodeIndexer - Main orchestrator for code vectorization
|
|
3
|
+
*/
|
|
4
|
+
import type { EmbeddingProvider } from "../embeddings/base.js";
|
|
5
|
+
import type { QdrantManager } from "../qdrant/client.js";
|
|
6
|
+
import type { ChangeStats, CodeConfig, CodeSearchResult, IndexOptions, IndexStats, IndexStatus, ProgressCallback, SearchOptions } from "./types.js";
|
|
7
|
+
export declare class CodeIndexer {
|
|
8
|
+
private qdrant;
|
|
9
|
+
private embeddings;
|
|
10
|
+
private config;
|
|
11
|
+
constructor(qdrant: QdrantManager, embeddings: EmbeddingProvider, config: CodeConfig);
|
|
12
|
+
/**
|
|
13
|
+
* Validate that a path doesn't attempt directory traversal
|
|
14
|
+
* @throws Error if path traversal is detected
|
|
15
|
+
*/
|
|
16
|
+
private validatePath;
|
|
17
|
+
/**
|
|
18
|
+
* Index a codebase from scratch or force re-index
|
|
19
|
+
*/
|
|
20
|
+
indexCodebase(path: string, options?: IndexOptions, progressCallback?: ProgressCallback): Promise<IndexStats>;
|
|
21
|
+
/**
|
|
22
|
+
* Search code semantically
|
|
23
|
+
*/
|
|
24
|
+
searchCode(path: string, query: string, options?: SearchOptions): Promise<CodeSearchResult[]>;
|
|
25
|
+
/**
|
|
26
|
+
* Get indexing status for a codebase
|
|
27
|
+
*/
|
|
28
|
+
getIndexStatus(path: string): Promise<IndexStatus>;
|
|
29
|
+
/**
|
|
30
|
+
* Incrementally re-index only changed files
|
|
31
|
+
*/
|
|
32
|
+
reindexChanges(path: string, progressCallback?: ProgressCallback): Promise<ChangeStats>;
|
|
33
|
+
/**
|
|
34
|
+
* Clear all indexed data for a codebase
|
|
35
|
+
*/
|
|
36
|
+
clearIndex(path: string): Promise<void>;
|
|
37
|
+
/**
|
|
38
|
+
* Generate deterministic collection name from codebase path
|
|
39
|
+
*/
|
|
40
|
+
private getCollectionName;
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/code/indexer.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKzD,OAAO,KAAK,EACV,WAAW,EAEX,UAAU,EACV,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,WAAW,EACX,gBAAgB,EAChB,aAAa,EACd,MAAM,YAAY,CAAC;AAEpB,qBAAa,WAAW;IAEpB,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,UAAU;IAClB,OAAO,CAAC,MAAM;gBAFN,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,iBAAiB,EAC7B,MAAM,EAAE,UAAU;IAG5B;;;OAGG;YACW,YAAY;IAkB1B;;OAEG;IACG,aAAa,CACjB,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,YAAY,EACtB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,CAAC,UAAU,CAAC;IA4NtB;;OAEG;IACG,UAAU,CACd,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAmF9B;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAoBxD;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC;IAyL7F;;OAEG;IACG,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB7C;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAK1B"}
|