opencode-codebase-index 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -2
- package/dist/cli.cjs +4862 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.js +4867 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.cjs +655 -352
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +657 -353
- package/dist/index.js.map +1 -1
- package/native/codebase-index-native.darwin-arm64.node +0 -0
- package/native/codebase-index-native.darwin-x64.node +0 -0
- package/native/codebase-index-native.linux-arm64-gnu.node +0 -0
- package/native/codebase-index-native.linux-x64-gnu.node +0 -0
- package/native/codebase-index-native.win32-x64-msvc.node +0 -0
- package/package.json +23 -4
package/dist/cli.cjs
ADDED
|
@@ -0,0 +1,4862 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// opencode-codebase-index - Semantic codebase search for OpenCode
|
|
3
|
+
"use strict";
|
|
4
|
+
var __create = Object.create;
|
|
5
|
+
var __defProp = Object.defineProperty;
|
|
6
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
7
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
8
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
9
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
10
|
+
var __commonJS = (cb, mod) => function __require() {
|
|
11
|
+
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
12
|
+
};
|
|
13
|
+
var __copyProps = (to, from, except, desc) => {
|
|
14
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
15
|
+
for (let key of __getOwnPropNames(from))
|
|
16
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
17
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
18
|
+
}
|
|
19
|
+
return to;
|
|
20
|
+
};
|
|
21
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
22
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
23
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
24
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
25
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
26
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
27
|
+
mod
|
|
28
|
+
));
|
|
29
|
+
|
|
30
|
+
// node_modules/eventemitter3/index.js
|
|
31
|
+
var require_eventemitter3 = __commonJS({
|
|
32
|
+
"node_modules/eventemitter3/index.js"(exports2, module2) {
|
|
33
|
+
"use strict";
|
|
34
|
+
var has = Object.prototype.hasOwnProperty;
|
|
35
|
+
var prefix = "~";
|
|
36
|
+
function Events() {
|
|
37
|
+
}
|
|
38
|
+
if (Object.create) {
|
|
39
|
+
Events.prototype = /* @__PURE__ */ Object.create(null);
|
|
40
|
+
if (!new Events().__proto__) prefix = false;
|
|
41
|
+
}
|
|
42
|
+
function EE(fn, context, once) {
|
|
43
|
+
this.fn = fn;
|
|
44
|
+
this.context = context;
|
|
45
|
+
this.once = once || false;
|
|
46
|
+
}
|
|
47
|
+
function addListener(emitter, event, fn, context, once) {
|
|
48
|
+
if (typeof fn !== "function") {
|
|
49
|
+
throw new TypeError("The listener must be a function");
|
|
50
|
+
}
|
|
51
|
+
var listener = new EE(fn, context || emitter, once), evt = prefix ? prefix + event : event;
|
|
52
|
+
if (!emitter._events[evt]) emitter._events[evt] = listener, emitter._eventsCount++;
|
|
53
|
+
else if (!emitter._events[evt].fn) emitter._events[evt].push(listener);
|
|
54
|
+
else emitter._events[evt] = [emitter._events[evt], listener];
|
|
55
|
+
return emitter;
|
|
56
|
+
}
|
|
57
|
+
function clearEvent(emitter, evt) {
|
|
58
|
+
if (--emitter._eventsCount === 0) emitter._events = new Events();
|
|
59
|
+
else delete emitter._events[evt];
|
|
60
|
+
}
|
|
61
|
+
function EventEmitter2() {
|
|
62
|
+
this._events = new Events();
|
|
63
|
+
this._eventsCount = 0;
|
|
64
|
+
}
|
|
65
|
+
EventEmitter2.prototype.eventNames = function eventNames() {
|
|
66
|
+
var names = [], events, name;
|
|
67
|
+
if (this._eventsCount === 0) return names;
|
|
68
|
+
for (name in events = this._events) {
|
|
69
|
+
if (has.call(events, name)) names.push(prefix ? name.slice(1) : name);
|
|
70
|
+
}
|
|
71
|
+
if (Object.getOwnPropertySymbols) {
|
|
72
|
+
return names.concat(Object.getOwnPropertySymbols(events));
|
|
73
|
+
}
|
|
74
|
+
return names;
|
|
75
|
+
};
|
|
76
|
+
EventEmitter2.prototype.listeners = function listeners(event) {
|
|
77
|
+
var evt = prefix ? prefix + event : event, handlers = this._events[evt];
|
|
78
|
+
if (!handlers) return [];
|
|
79
|
+
if (handlers.fn) return [handlers.fn];
|
|
80
|
+
for (var i = 0, l = handlers.length, ee = new Array(l); i < l; i++) {
|
|
81
|
+
ee[i] = handlers[i].fn;
|
|
82
|
+
}
|
|
83
|
+
return ee;
|
|
84
|
+
};
|
|
85
|
+
EventEmitter2.prototype.listenerCount = function listenerCount(event) {
|
|
86
|
+
var evt = prefix ? prefix + event : event, listeners = this._events[evt];
|
|
87
|
+
if (!listeners) return 0;
|
|
88
|
+
if (listeners.fn) return 1;
|
|
89
|
+
return listeners.length;
|
|
90
|
+
};
|
|
91
|
+
EventEmitter2.prototype.emit = function emit(event, a1, a2, a3, a4, a5) {
|
|
92
|
+
var evt = prefix ? prefix + event : event;
|
|
93
|
+
if (!this._events[evt]) return false;
|
|
94
|
+
var listeners = this._events[evt], len = arguments.length, args, i;
|
|
95
|
+
if (listeners.fn) {
|
|
96
|
+
if (listeners.once) this.removeListener(event, listeners.fn, void 0, true);
|
|
97
|
+
switch (len) {
|
|
98
|
+
case 1:
|
|
99
|
+
return listeners.fn.call(listeners.context), true;
|
|
100
|
+
case 2:
|
|
101
|
+
return listeners.fn.call(listeners.context, a1), true;
|
|
102
|
+
case 3:
|
|
103
|
+
return listeners.fn.call(listeners.context, a1, a2), true;
|
|
104
|
+
case 4:
|
|
105
|
+
return listeners.fn.call(listeners.context, a1, a2, a3), true;
|
|
106
|
+
case 5:
|
|
107
|
+
return listeners.fn.call(listeners.context, a1, a2, a3, a4), true;
|
|
108
|
+
case 6:
|
|
109
|
+
return listeners.fn.call(listeners.context, a1, a2, a3, a4, a5), true;
|
|
110
|
+
}
|
|
111
|
+
for (i = 1, args = new Array(len - 1); i < len; i++) {
|
|
112
|
+
args[i - 1] = arguments[i];
|
|
113
|
+
}
|
|
114
|
+
listeners.fn.apply(listeners.context, args);
|
|
115
|
+
} else {
|
|
116
|
+
var length = listeners.length, j;
|
|
117
|
+
for (i = 0; i < length; i++) {
|
|
118
|
+
if (listeners[i].once) this.removeListener(event, listeners[i].fn, void 0, true);
|
|
119
|
+
switch (len) {
|
|
120
|
+
case 1:
|
|
121
|
+
listeners[i].fn.call(listeners[i].context);
|
|
122
|
+
break;
|
|
123
|
+
case 2:
|
|
124
|
+
listeners[i].fn.call(listeners[i].context, a1);
|
|
125
|
+
break;
|
|
126
|
+
case 3:
|
|
127
|
+
listeners[i].fn.call(listeners[i].context, a1, a2);
|
|
128
|
+
break;
|
|
129
|
+
case 4:
|
|
130
|
+
listeners[i].fn.call(listeners[i].context, a1, a2, a3);
|
|
131
|
+
break;
|
|
132
|
+
default:
|
|
133
|
+
if (!args) for (j = 1, args = new Array(len - 1); j < len; j++) {
|
|
134
|
+
args[j - 1] = arguments[j];
|
|
135
|
+
}
|
|
136
|
+
listeners[i].fn.apply(listeners[i].context, args);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return true;
|
|
141
|
+
};
|
|
142
|
+
EventEmitter2.prototype.on = function on(event, fn, context) {
|
|
143
|
+
return addListener(this, event, fn, context, false);
|
|
144
|
+
};
|
|
145
|
+
EventEmitter2.prototype.once = function once(event, fn, context) {
|
|
146
|
+
return addListener(this, event, fn, context, true);
|
|
147
|
+
};
|
|
148
|
+
EventEmitter2.prototype.removeListener = function removeListener(event, fn, context, once) {
|
|
149
|
+
var evt = prefix ? prefix + event : event;
|
|
150
|
+
if (!this._events[evt]) return this;
|
|
151
|
+
if (!fn) {
|
|
152
|
+
clearEvent(this, evt);
|
|
153
|
+
return this;
|
|
154
|
+
}
|
|
155
|
+
var listeners = this._events[evt];
|
|
156
|
+
if (listeners.fn) {
|
|
157
|
+
if (listeners.fn === fn && (!once || listeners.once) && (!context || listeners.context === context)) {
|
|
158
|
+
clearEvent(this, evt);
|
|
159
|
+
}
|
|
160
|
+
} else {
|
|
161
|
+
for (var i = 0, events = [], length = listeners.length; i < length; i++) {
|
|
162
|
+
if (listeners[i].fn !== fn || once && !listeners[i].once || context && listeners[i].context !== context) {
|
|
163
|
+
events.push(listeners[i]);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (events.length) this._events[evt] = events.length === 1 ? events[0] : events;
|
|
167
|
+
else clearEvent(this, evt);
|
|
168
|
+
}
|
|
169
|
+
return this;
|
|
170
|
+
};
|
|
171
|
+
EventEmitter2.prototype.removeAllListeners = function removeAllListeners(event) {
|
|
172
|
+
var evt;
|
|
173
|
+
if (event) {
|
|
174
|
+
evt = prefix ? prefix + event : event;
|
|
175
|
+
if (this._events[evt]) clearEvent(this, evt);
|
|
176
|
+
} else {
|
|
177
|
+
this._events = new Events();
|
|
178
|
+
this._eventsCount = 0;
|
|
179
|
+
}
|
|
180
|
+
return this;
|
|
181
|
+
};
|
|
182
|
+
EventEmitter2.prototype.off = EventEmitter2.prototype.removeListener;
|
|
183
|
+
EventEmitter2.prototype.addListener = EventEmitter2.prototype.on;
|
|
184
|
+
EventEmitter2.prefixed = prefix;
|
|
185
|
+
EventEmitter2.EventEmitter = EventEmitter2;
|
|
186
|
+
if ("undefined" !== typeof module2) {
|
|
187
|
+
module2.exports = EventEmitter2;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
// node_modules/ignore/index.js
|
|
193
|
+
var require_ignore = __commonJS({
|
|
194
|
+
"node_modules/ignore/index.js"(exports2, module2) {
|
|
195
|
+
"use strict";
|
|
196
|
+
function makeArray(subject) {
|
|
197
|
+
return Array.isArray(subject) ? subject : [subject];
|
|
198
|
+
}
|
|
199
|
+
var UNDEFINED = void 0;
|
|
200
|
+
var EMPTY = "";
|
|
201
|
+
var SPACE = " ";
|
|
202
|
+
var ESCAPE = "\\";
|
|
203
|
+
var REGEX_TEST_BLANK_LINE = /^\s+$/;
|
|
204
|
+
var REGEX_INVALID_TRAILING_BACKSLASH = /(?:[^\\]|^)\\$/;
|
|
205
|
+
var REGEX_REPLACE_LEADING_EXCAPED_EXCLAMATION = /^\\!/;
|
|
206
|
+
var REGEX_REPLACE_LEADING_EXCAPED_HASH = /^\\#/;
|
|
207
|
+
var REGEX_SPLITALL_CRLF = /\r?\n/g;
|
|
208
|
+
var REGEX_TEST_INVALID_PATH = /^\.{0,2}\/|^\.{1,2}$/;
|
|
209
|
+
var REGEX_TEST_TRAILING_SLASH = /\/$/;
|
|
210
|
+
var SLASH = "/";
|
|
211
|
+
var TMP_KEY_IGNORE = "node-ignore";
|
|
212
|
+
if (typeof Symbol !== "undefined") {
|
|
213
|
+
TMP_KEY_IGNORE = /* @__PURE__ */ Symbol.for("node-ignore");
|
|
214
|
+
}
|
|
215
|
+
var KEY_IGNORE = TMP_KEY_IGNORE;
|
|
216
|
+
var define = (object, key, value) => {
|
|
217
|
+
Object.defineProperty(object, key, { value });
|
|
218
|
+
return value;
|
|
219
|
+
};
|
|
220
|
+
var REGEX_REGEXP_RANGE = /([0-z])-([0-z])/g;
|
|
221
|
+
var RETURN_FALSE = () => false;
|
|
222
|
+
var sanitizeRange = (range) => range.replace(
|
|
223
|
+
REGEX_REGEXP_RANGE,
|
|
224
|
+
(match, from, to) => from.charCodeAt(0) <= to.charCodeAt(0) ? match : EMPTY
|
|
225
|
+
);
|
|
226
|
+
var cleanRangeBackSlash = (slashes) => {
|
|
227
|
+
const { length } = slashes;
|
|
228
|
+
return slashes.slice(0, length - length % 2);
|
|
229
|
+
};
|
|
230
|
+
var REPLACERS = [
|
|
231
|
+
[
|
|
232
|
+
// Remove BOM
|
|
233
|
+
// TODO:
|
|
234
|
+
// Other similar zero-width characters?
|
|
235
|
+
/^\uFEFF/,
|
|
236
|
+
() => EMPTY
|
|
237
|
+
],
|
|
238
|
+
// > Trailing spaces are ignored unless they are quoted with backslash ("\")
|
|
239
|
+
[
|
|
240
|
+
// (a\ ) -> (a )
|
|
241
|
+
// (a ) -> (a)
|
|
242
|
+
// (a ) -> (a)
|
|
243
|
+
// (a \ ) -> (a )
|
|
244
|
+
/((?:\\\\)*?)(\\?\s+)$/,
|
|
245
|
+
(_, m1, m2) => m1 + (m2.indexOf("\\") === 0 ? SPACE : EMPTY)
|
|
246
|
+
],
|
|
247
|
+
// Replace (\ ) with ' '
|
|
248
|
+
// (\ ) -> ' '
|
|
249
|
+
// (\\ ) -> '\\ '
|
|
250
|
+
// (\\\ ) -> '\\ '
|
|
251
|
+
[
|
|
252
|
+
/(\\+?)\s/g,
|
|
253
|
+
(_, m1) => {
|
|
254
|
+
const { length } = m1;
|
|
255
|
+
return m1.slice(0, length - length % 2) + SPACE;
|
|
256
|
+
}
|
|
257
|
+
],
|
|
258
|
+
// Escape metacharacters
|
|
259
|
+
// which is written down by users but means special for regular expressions.
|
|
260
|
+
// > There are 12 characters with special meanings:
|
|
261
|
+
// > - the backslash \,
|
|
262
|
+
// > - the caret ^,
|
|
263
|
+
// > - the dollar sign $,
|
|
264
|
+
// > - the period or dot .,
|
|
265
|
+
// > - the vertical bar or pipe symbol |,
|
|
266
|
+
// > - the question mark ?,
|
|
267
|
+
// > - the asterisk or star *,
|
|
268
|
+
// > - the plus sign +,
|
|
269
|
+
// > - the opening parenthesis (,
|
|
270
|
+
// > - the closing parenthesis ),
|
|
271
|
+
// > - and the opening square bracket [,
|
|
272
|
+
// > - the opening curly brace {,
|
|
273
|
+
// > These special characters are often called "metacharacters".
|
|
274
|
+
[
|
|
275
|
+
/[\\$.|*+(){^]/g,
|
|
276
|
+
(match) => `\\${match}`
|
|
277
|
+
],
|
|
278
|
+
[
|
|
279
|
+
// > a question mark (?) matches a single character
|
|
280
|
+
/(?!\\)\?/g,
|
|
281
|
+
() => "[^/]"
|
|
282
|
+
],
|
|
283
|
+
// leading slash
|
|
284
|
+
[
|
|
285
|
+
// > A leading slash matches the beginning of the pathname.
|
|
286
|
+
// > For example, "/*.c" matches "cat-file.c" but not "mozilla-sha1/sha1.c".
|
|
287
|
+
// A leading slash matches the beginning of the pathname
|
|
288
|
+
/^\//,
|
|
289
|
+
() => "^"
|
|
290
|
+
],
|
|
291
|
+
// replace special metacharacter slash after the leading slash
|
|
292
|
+
[
|
|
293
|
+
/\//g,
|
|
294
|
+
() => "\\/"
|
|
295
|
+
],
|
|
296
|
+
[
|
|
297
|
+
// > A leading "**" followed by a slash means match in all directories.
|
|
298
|
+
// > For example, "**/foo" matches file or directory "foo" anywhere,
|
|
299
|
+
// > the same as pattern "foo".
|
|
300
|
+
// > "**/foo/bar" matches file or directory "bar" anywhere that is directly
|
|
301
|
+
// > under directory "foo".
|
|
302
|
+
// Notice that the '*'s have been replaced as '\\*'
|
|
303
|
+
/^\^*\\\*\\\*\\\//,
|
|
304
|
+
// '**/foo' <-> 'foo'
|
|
305
|
+
() => "^(?:.*\\/)?"
|
|
306
|
+
],
|
|
307
|
+
// starting
|
|
308
|
+
[
|
|
309
|
+
// there will be no leading '/'
|
|
310
|
+
// (which has been replaced by section "leading slash")
|
|
311
|
+
// If starts with '**', adding a '^' to the regular expression also works
|
|
312
|
+
/^(?=[^^])/,
|
|
313
|
+
function startingReplacer() {
|
|
314
|
+
return !/\/(?!$)/.test(this) ? "(?:^|\\/)" : "^";
|
|
315
|
+
}
|
|
316
|
+
],
|
|
317
|
+
// two globstars
|
|
318
|
+
[
|
|
319
|
+
// Use lookahead assertions so that we could match more than one `'/**'`
|
|
320
|
+
/\\\/\\\*\\\*(?=\\\/|$)/g,
|
|
321
|
+
// Zero, one or several directories
|
|
322
|
+
// should not use '*', or it will be replaced by the next replacer
|
|
323
|
+
// Check if it is not the last `'/**'`
|
|
324
|
+
(_, index, str) => index + 6 < str.length ? "(?:\\/[^\\/]+)*" : "\\/.+"
|
|
325
|
+
],
|
|
326
|
+
// normal intermediate wildcards
|
|
327
|
+
[
|
|
328
|
+
// Never replace escaped '*'
|
|
329
|
+
// ignore rule '\*' will match the path '*'
|
|
330
|
+
// 'abc.*/' -> go
|
|
331
|
+
// 'abc.*' -> skip this rule,
|
|
332
|
+
// coz trailing single wildcard will be handed by [trailing wildcard]
|
|
333
|
+
/(^|[^\\]+)(\\\*)+(?=.+)/g,
|
|
334
|
+
// '*.js' matches '.js'
|
|
335
|
+
// '*.js' doesn't match 'abc'
|
|
336
|
+
(_, p1, p2) => {
|
|
337
|
+
const unescaped = p2.replace(/\\\*/g, "[^\\/]*");
|
|
338
|
+
return p1 + unescaped;
|
|
339
|
+
}
|
|
340
|
+
],
|
|
341
|
+
[
|
|
342
|
+
// unescape, revert step 3 except for back slash
|
|
343
|
+
// For example, if a user escape a '\\*',
|
|
344
|
+
// after step 3, the result will be '\\\\\\*'
|
|
345
|
+
/\\\\\\(?=[$.|*+(){^])/g,
|
|
346
|
+
() => ESCAPE
|
|
347
|
+
],
|
|
348
|
+
[
|
|
349
|
+
// '\\\\' -> '\\'
|
|
350
|
+
/\\\\/g,
|
|
351
|
+
() => ESCAPE
|
|
352
|
+
],
|
|
353
|
+
[
|
|
354
|
+
// > The range notation, e.g. [a-zA-Z],
|
|
355
|
+
// > can be used to match one of the characters in a range.
|
|
356
|
+
// `\` is escaped by step 3
|
|
357
|
+
/(\\)?\[([^\]/]*?)(\\*)($|\])/g,
|
|
358
|
+
(match, leadEscape, range, endEscape, close) => leadEscape === ESCAPE ? `\\[${range}${cleanRangeBackSlash(endEscape)}${close}` : close === "]" ? endEscape.length % 2 === 0 ? `[${sanitizeRange(range)}${endEscape}]` : "[]" : "[]"
|
|
359
|
+
],
|
|
360
|
+
// ending
|
|
361
|
+
[
|
|
362
|
+
// 'js' will not match 'js.'
|
|
363
|
+
// 'ab' will not match 'abc'
|
|
364
|
+
/(?:[^*])$/,
|
|
365
|
+
// WTF!
|
|
366
|
+
// https://git-scm.com/docs/gitignore
|
|
367
|
+
// changes in [2.22.1](https://git-scm.com/docs/gitignore/2.22.1)
|
|
368
|
+
// which re-fixes #24, #38
|
|
369
|
+
// > If there is a separator at the end of the pattern then the pattern
|
|
370
|
+
// > will only match directories, otherwise the pattern can match both
|
|
371
|
+
// > files and directories.
|
|
372
|
+
// 'js*' will not match 'a.js'
|
|
373
|
+
// 'js/' will not match 'a.js'
|
|
374
|
+
// 'js' will match 'a.js' and 'a.js/'
|
|
375
|
+
(match) => /\/$/.test(match) ? `${match}$` : `${match}(?=$|\\/$)`
|
|
376
|
+
]
|
|
377
|
+
];
|
|
378
|
+
var REGEX_REPLACE_TRAILING_WILDCARD = /(^|\\\/)?\\\*$/;
|
|
379
|
+
var MODE_IGNORE = "regex";
|
|
380
|
+
var MODE_CHECK_IGNORE = "checkRegex";
|
|
381
|
+
var UNDERSCORE = "_";
|
|
382
|
+
var TRAILING_WILD_CARD_REPLACERS = {
|
|
383
|
+
[MODE_IGNORE](_, p1) {
|
|
384
|
+
const prefix = p1 ? `${p1}[^/]+` : "[^/]*";
|
|
385
|
+
return `${prefix}(?=$|\\/$)`;
|
|
386
|
+
},
|
|
387
|
+
[MODE_CHECK_IGNORE](_, p1) {
|
|
388
|
+
const prefix = p1 ? `${p1}[^/]*` : "[^/]*";
|
|
389
|
+
return `${prefix}(?=$|\\/$)`;
|
|
390
|
+
}
|
|
391
|
+
};
|
|
392
|
+
var makeRegexPrefix = (pattern) => REPLACERS.reduce(
|
|
393
|
+
(prev, [matcher, replacer]) => prev.replace(matcher, replacer.bind(pattern)),
|
|
394
|
+
pattern
|
|
395
|
+
);
|
|
396
|
+
var isString = (subject) => typeof subject === "string";
|
|
397
|
+
var checkPattern = (pattern) => pattern && isString(pattern) && !REGEX_TEST_BLANK_LINE.test(pattern) && !REGEX_INVALID_TRAILING_BACKSLASH.test(pattern) && pattern.indexOf("#") !== 0;
|
|
398
|
+
var splitPattern = (pattern) => pattern.split(REGEX_SPLITALL_CRLF).filter(Boolean);
|
|
399
|
+
var IgnoreRule = class {
|
|
400
|
+
constructor(pattern, mark, body, ignoreCase, negative, prefix) {
|
|
401
|
+
this.pattern = pattern;
|
|
402
|
+
this.mark = mark;
|
|
403
|
+
this.negative = negative;
|
|
404
|
+
define(this, "body", body);
|
|
405
|
+
define(this, "ignoreCase", ignoreCase);
|
|
406
|
+
define(this, "regexPrefix", prefix);
|
|
407
|
+
}
|
|
408
|
+
get regex() {
|
|
409
|
+
const key = UNDERSCORE + MODE_IGNORE;
|
|
410
|
+
if (this[key]) {
|
|
411
|
+
return this[key];
|
|
412
|
+
}
|
|
413
|
+
return this._make(MODE_IGNORE, key);
|
|
414
|
+
}
|
|
415
|
+
get checkRegex() {
|
|
416
|
+
const key = UNDERSCORE + MODE_CHECK_IGNORE;
|
|
417
|
+
if (this[key]) {
|
|
418
|
+
return this[key];
|
|
419
|
+
}
|
|
420
|
+
return this._make(MODE_CHECK_IGNORE, key);
|
|
421
|
+
}
|
|
422
|
+
_make(mode, key) {
|
|
423
|
+
const str = this.regexPrefix.replace(
|
|
424
|
+
REGEX_REPLACE_TRAILING_WILDCARD,
|
|
425
|
+
// It does not need to bind pattern
|
|
426
|
+
TRAILING_WILD_CARD_REPLACERS[mode]
|
|
427
|
+
);
|
|
428
|
+
const regex = this.ignoreCase ? new RegExp(str, "i") : new RegExp(str);
|
|
429
|
+
return define(this, key, regex);
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
var createRule = ({
|
|
433
|
+
pattern,
|
|
434
|
+
mark
|
|
435
|
+
}, ignoreCase) => {
|
|
436
|
+
let negative = false;
|
|
437
|
+
let body = pattern;
|
|
438
|
+
if (body.indexOf("!") === 0) {
|
|
439
|
+
negative = true;
|
|
440
|
+
body = body.substr(1);
|
|
441
|
+
}
|
|
442
|
+
body = body.replace(REGEX_REPLACE_LEADING_EXCAPED_EXCLAMATION, "!").replace(REGEX_REPLACE_LEADING_EXCAPED_HASH, "#");
|
|
443
|
+
const regexPrefix = makeRegexPrefix(body);
|
|
444
|
+
return new IgnoreRule(
|
|
445
|
+
pattern,
|
|
446
|
+
mark,
|
|
447
|
+
body,
|
|
448
|
+
ignoreCase,
|
|
449
|
+
negative,
|
|
450
|
+
regexPrefix
|
|
451
|
+
);
|
|
452
|
+
};
|
|
453
|
+
var RuleManager = class {
|
|
454
|
+
constructor(ignoreCase) {
|
|
455
|
+
this._ignoreCase = ignoreCase;
|
|
456
|
+
this._rules = [];
|
|
457
|
+
}
|
|
458
|
+
_add(pattern) {
|
|
459
|
+
if (pattern && pattern[KEY_IGNORE]) {
|
|
460
|
+
this._rules = this._rules.concat(pattern._rules._rules);
|
|
461
|
+
this._added = true;
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
if (isString(pattern)) {
|
|
465
|
+
pattern = {
|
|
466
|
+
pattern
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
if (checkPattern(pattern.pattern)) {
|
|
470
|
+
const rule = createRule(pattern, this._ignoreCase);
|
|
471
|
+
this._added = true;
|
|
472
|
+
this._rules.push(rule);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
// @param {Array<string> | string | Ignore} pattern
|
|
476
|
+
add(pattern) {
|
|
477
|
+
this._added = false;
|
|
478
|
+
makeArray(
|
|
479
|
+
isString(pattern) ? splitPattern(pattern) : pattern
|
|
480
|
+
).forEach(this._add, this);
|
|
481
|
+
return this._added;
|
|
482
|
+
}
|
|
483
|
+
// Test one single path without recursively checking parent directories
|
|
484
|
+
//
|
|
485
|
+
// - checkUnignored `boolean` whether should check if the path is unignored,
|
|
486
|
+
// setting `checkUnignored` to `false` could reduce additional
|
|
487
|
+
// path matching.
|
|
488
|
+
// - check `string` either `MODE_IGNORE` or `MODE_CHECK_IGNORE`
|
|
489
|
+
// @returns {TestResult} true if a file is ignored
|
|
490
|
+
test(path7, checkUnignored, mode) {
|
|
491
|
+
let ignored = false;
|
|
492
|
+
let unignored = false;
|
|
493
|
+
let matchedRule;
|
|
494
|
+
this._rules.forEach((rule) => {
|
|
495
|
+
const { negative } = rule;
|
|
496
|
+
if (unignored === negative && ignored !== unignored || negative && !ignored && !unignored && !checkUnignored) {
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
const matched = rule[mode].test(path7);
|
|
500
|
+
if (!matched) {
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
ignored = !negative;
|
|
504
|
+
unignored = negative;
|
|
505
|
+
matchedRule = negative ? UNDEFINED : rule;
|
|
506
|
+
});
|
|
507
|
+
const ret = {
|
|
508
|
+
ignored,
|
|
509
|
+
unignored
|
|
510
|
+
};
|
|
511
|
+
if (matchedRule) {
|
|
512
|
+
ret.rule = matchedRule;
|
|
513
|
+
}
|
|
514
|
+
return ret;
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
var throwError = (message, Ctor) => {
|
|
518
|
+
throw new Ctor(message);
|
|
519
|
+
};
|
|
520
|
+
var checkPath = (path7, originalPath, doThrow) => {
|
|
521
|
+
if (!isString(path7)) {
|
|
522
|
+
return doThrow(
|
|
523
|
+
`path must be a string, but got \`${originalPath}\``,
|
|
524
|
+
TypeError
|
|
525
|
+
);
|
|
526
|
+
}
|
|
527
|
+
if (!path7) {
|
|
528
|
+
return doThrow(`path must not be empty`, TypeError);
|
|
529
|
+
}
|
|
530
|
+
if (checkPath.isNotRelative(path7)) {
|
|
531
|
+
const r = "`path.relative()`d";
|
|
532
|
+
return doThrow(
|
|
533
|
+
`path should be a ${r} string, but got "${originalPath}"`,
|
|
534
|
+
RangeError
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
return true;
|
|
538
|
+
};
|
|
539
|
+
var isNotRelative = (path7) => REGEX_TEST_INVALID_PATH.test(path7);
|
|
540
|
+
checkPath.isNotRelative = isNotRelative;
|
|
541
|
+
checkPath.convert = (p) => p;
|
|
542
|
+
var Ignore2 = class {
|
|
543
|
+
constructor({
|
|
544
|
+
ignorecase = true,
|
|
545
|
+
ignoreCase = ignorecase,
|
|
546
|
+
allowRelativePaths = false
|
|
547
|
+
} = {}) {
|
|
548
|
+
define(this, KEY_IGNORE, true);
|
|
549
|
+
this._rules = new RuleManager(ignoreCase);
|
|
550
|
+
this._strictPathCheck = !allowRelativePaths;
|
|
551
|
+
this._initCache();
|
|
552
|
+
}
|
|
553
|
+
_initCache() {
|
|
554
|
+
this._ignoreCache = /* @__PURE__ */ Object.create(null);
|
|
555
|
+
this._testCache = /* @__PURE__ */ Object.create(null);
|
|
556
|
+
}
|
|
557
|
+
add(pattern) {
|
|
558
|
+
if (this._rules.add(pattern)) {
|
|
559
|
+
this._initCache();
|
|
560
|
+
}
|
|
561
|
+
return this;
|
|
562
|
+
}
|
|
563
|
+
// legacy
|
|
564
|
+
addPattern(pattern) {
|
|
565
|
+
return this.add(pattern);
|
|
566
|
+
}
|
|
567
|
+
// @returns {TestResult}
|
|
568
|
+
_test(originalPath, cache, checkUnignored, slices) {
|
|
569
|
+
const path7 = originalPath && checkPath.convert(originalPath);
|
|
570
|
+
checkPath(
|
|
571
|
+
path7,
|
|
572
|
+
originalPath,
|
|
573
|
+
this._strictPathCheck ? throwError : RETURN_FALSE
|
|
574
|
+
);
|
|
575
|
+
return this._t(path7, cache, checkUnignored, slices);
|
|
576
|
+
}
|
|
577
|
+
checkIgnore(path7) {
|
|
578
|
+
if (!REGEX_TEST_TRAILING_SLASH.test(path7)) {
|
|
579
|
+
return this.test(path7);
|
|
580
|
+
}
|
|
581
|
+
const slices = path7.split(SLASH).filter(Boolean);
|
|
582
|
+
slices.pop();
|
|
583
|
+
if (slices.length) {
|
|
584
|
+
const parent = this._t(
|
|
585
|
+
slices.join(SLASH) + SLASH,
|
|
586
|
+
this._testCache,
|
|
587
|
+
true,
|
|
588
|
+
slices
|
|
589
|
+
);
|
|
590
|
+
if (parent.ignored) {
|
|
591
|
+
return parent;
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
return this._rules.test(path7, false, MODE_CHECK_IGNORE);
|
|
595
|
+
}
|
|
596
|
+
_t(path7, cache, checkUnignored, slices) {
|
|
597
|
+
if (path7 in cache) {
|
|
598
|
+
return cache[path7];
|
|
599
|
+
}
|
|
600
|
+
if (!slices) {
|
|
601
|
+
slices = path7.split(SLASH).filter(Boolean);
|
|
602
|
+
}
|
|
603
|
+
slices.pop();
|
|
604
|
+
if (!slices.length) {
|
|
605
|
+
return cache[path7] = this._rules.test(path7, checkUnignored, MODE_IGNORE);
|
|
606
|
+
}
|
|
607
|
+
const parent = this._t(
|
|
608
|
+
slices.join(SLASH) + SLASH,
|
|
609
|
+
cache,
|
|
610
|
+
checkUnignored,
|
|
611
|
+
slices
|
|
612
|
+
);
|
|
613
|
+
return cache[path7] = parent.ignored ? parent : this._rules.test(path7, checkUnignored, MODE_IGNORE);
|
|
614
|
+
}
|
|
615
|
+
ignores(path7) {
|
|
616
|
+
return this._test(path7, this._ignoreCache, false).ignored;
|
|
617
|
+
}
|
|
618
|
+
createFilter() {
|
|
619
|
+
return (path7) => !this.ignores(path7);
|
|
620
|
+
}
|
|
621
|
+
filter(paths) {
|
|
622
|
+
return makeArray(paths).filter(this.createFilter());
|
|
623
|
+
}
|
|
624
|
+
// @returns {TestResult}
|
|
625
|
+
test(path7) {
|
|
626
|
+
return this._test(path7, this._testCache, true);
|
|
627
|
+
}
|
|
628
|
+
};
|
|
629
|
+
var factory = (options) => new Ignore2(options);
|
|
630
|
+
var isPathValid = (path7) => checkPath(path7 && checkPath.convert(path7), path7, RETURN_FALSE);
|
|
631
|
+
var setupWindows = () => {
|
|
632
|
+
const makePosix = (str) => /^\\\\\?\\/.test(str) || /["<>|\u0000-\u001F]+/u.test(str) ? str : str.replace(/\\/g, "/");
|
|
633
|
+
checkPath.convert = makePosix;
|
|
634
|
+
const REGEX_TEST_WINDOWS_PATH_ABSOLUTE = /^[a-z]:\//i;
|
|
635
|
+
checkPath.isNotRelative = (path7) => REGEX_TEST_WINDOWS_PATH_ABSOLUTE.test(path7) || isNotRelative(path7);
|
|
636
|
+
};
|
|
637
|
+
if (
|
|
638
|
+
// Detect `process` so that it can run in browsers.
|
|
639
|
+
typeof process !== "undefined" && process.platform === "win32"
|
|
640
|
+
) {
|
|
641
|
+
setupWindows();
|
|
642
|
+
}
|
|
643
|
+
module2.exports = factory;
|
|
644
|
+
factory.default = factory;
|
|
645
|
+
module2.exports.isPathValid = isPathValid;
|
|
646
|
+
define(module2.exports, /* @__PURE__ */ Symbol.for("setupWindows"), setupWindows);
|
|
647
|
+
}
|
|
648
|
+
});
|
|
649
|
+
|
|
650
|
+
// src/cli.ts
|
|
651
|
+
var import_stdio = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
652
|
+
var import_fs5 = require("fs");
|
|
653
|
+
var path6 = __toESM(require("path"), 1);
|
|
654
|
+
var os3 = __toESM(require("os"), 1);
|
|
655
|
+
|
|
656
|
+
// src/config/constants.ts
|
|
657
|
+
var DEFAULT_INCLUDE = [
|
|
658
|
+
"**/*.{ts,tsx,js,jsx,mjs,cjs}",
|
|
659
|
+
"**/*.{py,pyi}",
|
|
660
|
+
"**/*.{go,rs,java,kt,scala}",
|
|
661
|
+
"**/*.{c,cpp,cc,h,hpp}",
|
|
662
|
+
"**/*.{rb,php,swift}",
|
|
663
|
+
"**/*.{vue,svelte,astro}",
|
|
664
|
+
"**/*.{sql,graphql,proto}",
|
|
665
|
+
"**/*.{yaml,yml,toml}",
|
|
666
|
+
"**/*.{md,mdx}",
|
|
667
|
+
"**/*.{sh,bash,zsh}"
|
|
668
|
+
];
|
|
669
|
+
var DEFAULT_EXCLUDE = [
|
|
670
|
+
"**/node_modules/**",
|
|
671
|
+
"**/.git/**",
|
|
672
|
+
"**/dist/**",
|
|
673
|
+
"**/build/**",
|
|
674
|
+
"**/*.min.js",
|
|
675
|
+
"**/*.bundle.js",
|
|
676
|
+
"**/vendor/**",
|
|
677
|
+
"**/__pycache__/**",
|
|
678
|
+
"**/target/**",
|
|
679
|
+
"**/coverage/**",
|
|
680
|
+
"**/.next/**",
|
|
681
|
+
"**/.nuxt/**",
|
|
682
|
+
"**/.opencode/**"
|
|
683
|
+
];
|
|
684
|
+
var EMBEDDING_MODELS = {
|
|
685
|
+
"google": {
|
|
686
|
+
// `text-embedding-004` is DEPRECATED - https://ai.google.dev/gemini-api/docs/deprecations
|
|
687
|
+
"text-embedding-005": {
|
|
688
|
+
provider: "google",
|
|
689
|
+
model: "text-embedding-005",
|
|
690
|
+
dimensions: 768,
|
|
691
|
+
maxTokens: 2048,
|
|
692
|
+
costPer1MTokens: 0.025,
|
|
693
|
+
taskAble: false
|
|
694
|
+
// Note: on reality, this model allows for task-specific embeddings. See: https://docs.cloud.google.com/vertex-ai/generative-ai/docs/embeddings/task-types
|
|
695
|
+
},
|
|
696
|
+
"gemini-embedding-001": {
|
|
697
|
+
provider: "google",
|
|
698
|
+
model: "gemini-embedding-001",
|
|
699
|
+
// Native output is 3072D, but we use Matryoshka truncation via outputDimensionality
|
|
700
|
+
// to reduce to 1536D for better storage/search efficiency with minimal quality loss.
|
|
701
|
+
// Google recommends 768, 1536, or 3072. See: https://ai.google.dev/gemini-api/docs/embeddings
|
|
702
|
+
dimensions: 1536,
|
|
703
|
+
maxTokens: 2048,
|
|
704
|
+
costPer1MTokens: 0.15,
|
|
705
|
+
taskAble: true
|
|
706
|
+
}
|
|
707
|
+
},
|
|
708
|
+
"openai": {
|
|
709
|
+
"text-embedding-3-small": {
|
|
710
|
+
provider: "openai",
|
|
711
|
+
model: "text-embedding-3-small",
|
|
712
|
+
dimensions: 1536,
|
|
713
|
+
maxTokens: 8191,
|
|
714
|
+
costPer1MTokens: 0.02
|
|
715
|
+
},
|
|
716
|
+
"text-embedding-3-large": {
|
|
717
|
+
provider: "openai",
|
|
718
|
+
model: "text-embedding-3-large",
|
|
719
|
+
dimensions: 3072,
|
|
720
|
+
maxTokens: 8191,
|
|
721
|
+
costPer1MTokens: 0.13
|
|
722
|
+
}
|
|
723
|
+
},
|
|
724
|
+
"ollama": {
|
|
725
|
+
"nomic-embed-text": {
|
|
726
|
+
provider: "ollama",
|
|
727
|
+
model: "nomic-embed-text",
|
|
728
|
+
dimensions: 768,
|
|
729
|
+
maxTokens: 8192,
|
|
730
|
+
costPer1MTokens: 0
|
|
731
|
+
},
|
|
732
|
+
"mxbai-embed-large": {
|
|
733
|
+
provider: "ollama",
|
|
734
|
+
model: "mxbai-embed-large",
|
|
735
|
+
dimensions: 1024,
|
|
736
|
+
maxTokens: 512,
|
|
737
|
+
costPer1MTokens: 0
|
|
738
|
+
}
|
|
739
|
+
},
|
|
740
|
+
"github-copilot": {
|
|
741
|
+
"text-embedding-3-small": {
|
|
742
|
+
provider: "github-copilot",
|
|
743
|
+
model: "text-embedding-3-small",
|
|
744
|
+
dimensions: 1536,
|
|
745
|
+
maxTokens: 8191,
|
|
746
|
+
costPer1MTokens: 0
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
};
|
|
750
|
+
var DEFAULT_PROVIDER_MODELS = {
|
|
751
|
+
"github-copilot": "text-embedding-3-small",
|
|
752
|
+
"openai": "text-embedding-3-small",
|
|
753
|
+
"google": "text-embedding-005",
|
|
754
|
+
"ollama": "nomic-embed-text"
|
|
755
|
+
};
|
|
756
|
+
|
|
757
|
+
// src/config/schema.ts
|
|
758
|
+
function getDefaultIndexingConfig() {
|
|
759
|
+
return {
|
|
760
|
+
autoIndex: false,
|
|
761
|
+
watchFiles: true,
|
|
762
|
+
maxFileSize: 1048576,
|
|
763
|
+
maxChunksPerFile: 100,
|
|
764
|
+
semanticOnly: false,
|
|
765
|
+
retries: 3,
|
|
766
|
+
retryDelayMs: 1e3,
|
|
767
|
+
autoGc: true,
|
|
768
|
+
gcIntervalDays: 7,
|
|
769
|
+
gcOrphanThreshold: 100,
|
|
770
|
+
requireProjectMarker: true
|
|
771
|
+
};
|
|
772
|
+
}
|
|
773
|
+
function getDefaultSearchConfig() {
|
|
774
|
+
return {
|
|
775
|
+
maxResults: 20,
|
|
776
|
+
minScore: 0.1,
|
|
777
|
+
includeContext: true,
|
|
778
|
+
hybridWeight: 0.5,
|
|
779
|
+
contextLines: 0
|
|
780
|
+
};
|
|
781
|
+
}
|
|
782
|
+
function getDefaultDebugConfig() {
|
|
783
|
+
return {
|
|
784
|
+
enabled: false,
|
|
785
|
+
logLevel: "info",
|
|
786
|
+
logSearch: true,
|
|
787
|
+
logEmbedding: true,
|
|
788
|
+
logCache: true,
|
|
789
|
+
logGc: true,
|
|
790
|
+
logBranch: true,
|
|
791
|
+
metrics: true
|
|
792
|
+
};
|
|
793
|
+
}
|
|
794
|
+
var VALID_SCOPES = ["project", "global"];
|
|
795
|
+
var VALID_LOG_LEVELS = ["error", "warn", "info", "debug"];
|
|
796
|
+
function isValidProvider(value) {
|
|
797
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS).includes(value);
|
|
798
|
+
}
|
|
799
|
+
function isValidModel(value, provider) {
|
|
800
|
+
return typeof value === "string" && Object.keys(EMBEDDING_MODELS[provider]).includes(value);
|
|
801
|
+
}
|
|
802
|
+
function isValidScope(value) {
|
|
803
|
+
return typeof value === "string" && VALID_SCOPES.includes(value);
|
|
804
|
+
}
|
|
805
|
+
function isStringArray(value) {
|
|
806
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
807
|
+
}
|
|
808
|
+
function isValidLogLevel(value) {
|
|
809
|
+
return typeof value === "string" && VALID_LOG_LEVELS.includes(value);
|
|
810
|
+
}
|
|
811
|
+
function parseConfig(raw) {
|
|
812
|
+
const input = raw && typeof raw === "object" ? raw : {};
|
|
813
|
+
const defaultIndexing = getDefaultIndexingConfig();
|
|
814
|
+
const defaultSearch = getDefaultSearchConfig();
|
|
815
|
+
const defaultDebug = getDefaultDebugConfig();
|
|
816
|
+
const rawIndexing = input.indexing && typeof input.indexing === "object" ? input.indexing : {};
|
|
817
|
+
const indexing = {
|
|
818
|
+
autoIndex: typeof rawIndexing.autoIndex === "boolean" ? rawIndexing.autoIndex : defaultIndexing.autoIndex,
|
|
819
|
+
watchFiles: typeof rawIndexing.watchFiles === "boolean" ? rawIndexing.watchFiles : defaultIndexing.watchFiles,
|
|
820
|
+
maxFileSize: typeof rawIndexing.maxFileSize === "number" ? rawIndexing.maxFileSize : defaultIndexing.maxFileSize,
|
|
821
|
+
maxChunksPerFile: typeof rawIndexing.maxChunksPerFile === "number" ? Math.max(1, rawIndexing.maxChunksPerFile) : defaultIndexing.maxChunksPerFile,
|
|
822
|
+
semanticOnly: typeof rawIndexing.semanticOnly === "boolean" ? rawIndexing.semanticOnly : defaultIndexing.semanticOnly,
|
|
823
|
+
retries: typeof rawIndexing.retries === "number" ? rawIndexing.retries : defaultIndexing.retries,
|
|
824
|
+
retryDelayMs: typeof rawIndexing.retryDelayMs === "number" ? rawIndexing.retryDelayMs : defaultIndexing.retryDelayMs,
|
|
825
|
+
autoGc: typeof rawIndexing.autoGc === "boolean" ? rawIndexing.autoGc : defaultIndexing.autoGc,
|
|
826
|
+
gcIntervalDays: typeof rawIndexing.gcIntervalDays === "number" ? Math.max(1, rawIndexing.gcIntervalDays) : defaultIndexing.gcIntervalDays,
|
|
827
|
+
gcOrphanThreshold: typeof rawIndexing.gcOrphanThreshold === "number" ? Math.max(0, rawIndexing.gcOrphanThreshold) : defaultIndexing.gcOrphanThreshold,
|
|
828
|
+
requireProjectMarker: typeof rawIndexing.requireProjectMarker === "boolean" ? rawIndexing.requireProjectMarker : defaultIndexing.requireProjectMarker
|
|
829
|
+
};
|
|
830
|
+
const rawSearch = input.search && typeof input.search === "object" ? input.search : {};
|
|
831
|
+
const search = {
|
|
832
|
+
maxResults: typeof rawSearch.maxResults === "number" ? rawSearch.maxResults : defaultSearch.maxResults,
|
|
833
|
+
minScore: typeof rawSearch.minScore === "number" ? rawSearch.minScore : defaultSearch.minScore,
|
|
834
|
+
includeContext: typeof rawSearch.includeContext === "boolean" ? rawSearch.includeContext : defaultSearch.includeContext,
|
|
835
|
+
hybridWeight: typeof rawSearch.hybridWeight === "number" ? Math.min(1, Math.max(0, rawSearch.hybridWeight)) : defaultSearch.hybridWeight,
|
|
836
|
+
contextLines: typeof rawSearch.contextLines === "number" ? Math.min(50, Math.max(0, rawSearch.contextLines)) : defaultSearch.contextLines
|
|
837
|
+
};
|
|
838
|
+
const rawDebug = input.debug && typeof input.debug === "object" ? input.debug : {};
|
|
839
|
+
const debug = {
|
|
840
|
+
enabled: typeof rawDebug.enabled === "boolean" ? rawDebug.enabled : defaultDebug.enabled,
|
|
841
|
+
logLevel: isValidLogLevel(rawDebug.logLevel) ? rawDebug.logLevel : defaultDebug.logLevel,
|
|
842
|
+
logSearch: typeof rawDebug.logSearch === "boolean" ? rawDebug.logSearch : defaultDebug.logSearch,
|
|
843
|
+
logEmbedding: typeof rawDebug.logEmbedding === "boolean" ? rawDebug.logEmbedding : defaultDebug.logEmbedding,
|
|
844
|
+
logCache: typeof rawDebug.logCache === "boolean" ? rawDebug.logCache : defaultDebug.logCache,
|
|
845
|
+
logGc: typeof rawDebug.logGc === "boolean" ? rawDebug.logGc : defaultDebug.logGc,
|
|
846
|
+
logBranch: typeof rawDebug.logBranch === "boolean" ? rawDebug.logBranch : defaultDebug.logBranch,
|
|
847
|
+
metrics: typeof rawDebug.metrics === "boolean" ? rawDebug.metrics : defaultDebug.metrics
|
|
848
|
+
};
|
|
849
|
+
let embeddingProvider;
|
|
850
|
+
let embeddingModel = void 0;
|
|
851
|
+
if (isValidProvider(input.embeddingProvider)) {
|
|
852
|
+
embeddingProvider = input.embeddingProvider;
|
|
853
|
+
if (input.embeddingModel) {
|
|
854
|
+
embeddingModel = isValidModel(input.embeddingModel, embeddingProvider) ? input.embeddingModel : DEFAULT_PROVIDER_MODELS[embeddingProvider];
|
|
855
|
+
}
|
|
856
|
+
} else {
|
|
857
|
+
embeddingProvider = "auto";
|
|
858
|
+
}
|
|
859
|
+
return {
|
|
860
|
+
embeddingProvider,
|
|
861
|
+
embeddingModel,
|
|
862
|
+
scope: isValidScope(input.scope) ? input.scope : "project",
|
|
863
|
+
include: isStringArray(input.include) ? input.include : DEFAULT_INCLUDE,
|
|
864
|
+
exclude: isStringArray(input.exclude) ? input.exclude : DEFAULT_EXCLUDE,
|
|
865
|
+
indexing,
|
|
866
|
+
search,
|
|
867
|
+
debug
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
function getDefaultModelForProvider(provider) {
|
|
871
|
+
const models = EMBEDDING_MODELS[provider];
|
|
872
|
+
const providerDefault = DEFAULT_PROVIDER_MODELS[provider];
|
|
873
|
+
return models[providerDefault];
|
|
874
|
+
}
|
|
875
|
+
var availableProviders = Object.keys(EMBEDDING_MODELS);
|
|
876
|
+
|
|
877
|
+
// src/mcp-server.ts
|
|
878
|
+
var import_mcp = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
879
|
+
var import_zod = require("zod");
|
|
880
|
+
|
|
881
|
+
// src/indexer/index.ts
|
|
882
|
+
var import_fs4 = require("fs");
|
|
883
|
+
var path5 = __toESM(require("path"), 1);
|
|
884
|
+
var import_perf_hooks = require("perf_hooks");
|
|
885
|
+
|
|
886
|
+
// node_modules/eventemitter3/index.mjs
|
|
887
|
+
var import_index = __toESM(require_eventemitter3(), 1);
|
|
888
|
+
|
|
889
|
+
// node_modules/p-timeout/index.js
|
|
890
|
+
var TimeoutError = class _TimeoutError extends Error {
|
|
891
|
+
name = "TimeoutError";
|
|
892
|
+
constructor(message, options) {
|
|
893
|
+
super(message, options);
|
|
894
|
+
Error.captureStackTrace?.(this, _TimeoutError);
|
|
895
|
+
}
|
|
896
|
+
};
|
|
897
|
+
var getAbortedReason = (signal) => signal.reason ?? new DOMException("This operation was aborted.", "AbortError");
|
|
898
|
+
function pTimeout(promise, options) {
|
|
899
|
+
const {
|
|
900
|
+
milliseconds,
|
|
901
|
+
fallback,
|
|
902
|
+
message,
|
|
903
|
+
customTimers = { setTimeout, clearTimeout },
|
|
904
|
+
signal
|
|
905
|
+
} = options;
|
|
906
|
+
let timer;
|
|
907
|
+
let abortHandler;
|
|
908
|
+
const wrappedPromise = new Promise((resolve4, reject) => {
|
|
909
|
+
if (typeof milliseconds !== "number" || Math.sign(milliseconds) !== 1) {
|
|
910
|
+
throw new TypeError(`Expected \`milliseconds\` to be a positive number, got \`${milliseconds}\``);
|
|
911
|
+
}
|
|
912
|
+
if (signal?.aborted) {
|
|
913
|
+
reject(getAbortedReason(signal));
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
if (signal) {
|
|
917
|
+
abortHandler = () => {
|
|
918
|
+
reject(getAbortedReason(signal));
|
|
919
|
+
};
|
|
920
|
+
signal.addEventListener("abort", abortHandler, { once: true });
|
|
921
|
+
}
|
|
922
|
+
promise.then(resolve4, reject);
|
|
923
|
+
if (milliseconds === Number.POSITIVE_INFINITY) {
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
const timeoutError = new TimeoutError();
|
|
927
|
+
timer = customTimers.setTimeout.call(void 0, () => {
|
|
928
|
+
if (fallback) {
|
|
929
|
+
try {
|
|
930
|
+
resolve4(fallback());
|
|
931
|
+
} catch (error) {
|
|
932
|
+
reject(error);
|
|
933
|
+
}
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
if (typeof promise.cancel === "function") {
|
|
937
|
+
promise.cancel();
|
|
938
|
+
}
|
|
939
|
+
if (message === false) {
|
|
940
|
+
resolve4();
|
|
941
|
+
} else if (message instanceof Error) {
|
|
942
|
+
reject(message);
|
|
943
|
+
} else {
|
|
944
|
+
timeoutError.message = message ?? `Promise timed out after ${milliseconds} milliseconds`;
|
|
945
|
+
reject(timeoutError);
|
|
946
|
+
}
|
|
947
|
+
}, milliseconds);
|
|
948
|
+
});
|
|
949
|
+
const cancelablePromise = wrappedPromise.finally(() => {
|
|
950
|
+
cancelablePromise.clear();
|
|
951
|
+
if (abortHandler && signal) {
|
|
952
|
+
signal.removeEventListener("abort", abortHandler);
|
|
953
|
+
}
|
|
954
|
+
});
|
|
955
|
+
cancelablePromise.clear = () => {
|
|
956
|
+
customTimers.clearTimeout.call(void 0, timer);
|
|
957
|
+
timer = void 0;
|
|
958
|
+
};
|
|
959
|
+
return cancelablePromise;
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
// node_modules/p-queue/dist/lower-bound.js
|
|
963
|
+
function lowerBound(array, value, comparator) {
|
|
964
|
+
let first = 0;
|
|
965
|
+
let count = array.length;
|
|
966
|
+
while (count > 0) {
|
|
967
|
+
const step = Math.trunc(count / 2);
|
|
968
|
+
let it = first + step;
|
|
969
|
+
if (comparator(array[it], value) <= 0) {
|
|
970
|
+
first = ++it;
|
|
971
|
+
count -= step + 1;
|
|
972
|
+
} else {
|
|
973
|
+
count = step;
|
|
974
|
+
}
|
|
975
|
+
}
|
|
976
|
+
return first;
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
// node_modules/p-queue/dist/priority-queue.js
|
|
980
|
+
var PriorityQueue = class {
|
|
981
|
+
#queue = [];
|
|
982
|
+
enqueue(run, options) {
|
|
983
|
+
const { priority = 0, id } = options ?? {};
|
|
984
|
+
const element = {
|
|
985
|
+
priority,
|
|
986
|
+
id,
|
|
987
|
+
run
|
|
988
|
+
};
|
|
989
|
+
if (this.size === 0 || this.#queue[this.size - 1].priority >= priority) {
|
|
990
|
+
this.#queue.push(element);
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
const index = lowerBound(this.#queue, element, (a, b) => b.priority - a.priority);
|
|
994
|
+
this.#queue.splice(index, 0, element);
|
|
995
|
+
}
|
|
996
|
+
setPriority(id, priority) {
|
|
997
|
+
const index = this.#queue.findIndex((element) => element.id === id);
|
|
998
|
+
if (index === -1) {
|
|
999
|
+
throw new ReferenceError(`No promise function with the id "${id}" exists in the queue.`);
|
|
1000
|
+
}
|
|
1001
|
+
const [item] = this.#queue.splice(index, 1);
|
|
1002
|
+
this.enqueue(item.run, { priority, id });
|
|
1003
|
+
}
|
|
1004
|
+
dequeue() {
|
|
1005
|
+
const item = this.#queue.shift();
|
|
1006
|
+
return item?.run;
|
|
1007
|
+
}
|
|
1008
|
+
filter(options) {
|
|
1009
|
+
return this.#queue.filter((element) => element.priority === options.priority).map((element) => element.run);
|
|
1010
|
+
}
|
|
1011
|
+
get size() {
|
|
1012
|
+
return this.#queue.length;
|
|
1013
|
+
}
|
|
1014
|
+
};
|
|
1015
|
+
|
|
1016
|
+
// node_modules/p-queue/dist/index.js
|
|
1017
|
+
var PQueue = class extends import_index.default {
|
|
1018
|
+
#carryoverIntervalCount;
|
|
1019
|
+
#isIntervalIgnored;
|
|
1020
|
+
#intervalCount = 0;
|
|
1021
|
+
#intervalCap;
|
|
1022
|
+
#rateLimitedInInterval = false;
|
|
1023
|
+
#rateLimitFlushScheduled = false;
|
|
1024
|
+
#interval;
|
|
1025
|
+
#intervalEnd = 0;
|
|
1026
|
+
#lastExecutionTime = 0;
|
|
1027
|
+
#intervalId;
|
|
1028
|
+
#timeoutId;
|
|
1029
|
+
#strict;
|
|
1030
|
+
// Circular buffer implementation for better performance
|
|
1031
|
+
#strictTicks = [];
|
|
1032
|
+
#strictTicksStartIndex = 0;
|
|
1033
|
+
#queue;
|
|
1034
|
+
#queueClass;
|
|
1035
|
+
#pending = 0;
|
|
1036
|
+
// The `!` is needed because of https://github.com/microsoft/TypeScript/issues/32194
|
|
1037
|
+
#concurrency;
|
|
1038
|
+
#isPaused;
|
|
1039
|
+
// Use to assign a unique identifier to a promise function, if not explicitly specified
|
|
1040
|
+
#idAssigner = 1n;
|
|
1041
|
+
// Track currently running tasks for debugging
|
|
1042
|
+
#runningTasks = /* @__PURE__ */ new Map();
|
|
1043
|
+
/**
|
|
1044
|
+
Get or set the default timeout for all tasks. Can be changed at runtime.
|
|
1045
|
+
|
|
1046
|
+
Operations will throw a `TimeoutError` if they don't complete within the specified time.
|
|
1047
|
+
|
|
1048
|
+
The timeout begins when the operation is dequeued and starts execution, not while it's waiting in the queue.
|
|
1049
|
+
|
|
1050
|
+
@example
|
|
1051
|
+
```
|
|
1052
|
+
const queue = new PQueue({timeout: 5000});
|
|
1053
|
+
|
|
1054
|
+
// Change timeout for all future tasks
|
|
1055
|
+
queue.timeout = 10000;
|
|
1056
|
+
```
|
|
1057
|
+
*/
|
|
1058
|
+
timeout;
|
|
1059
|
+
constructor(options) {
|
|
1060
|
+
super();
|
|
1061
|
+
options = {
|
|
1062
|
+
carryoverIntervalCount: false,
|
|
1063
|
+
intervalCap: Number.POSITIVE_INFINITY,
|
|
1064
|
+
interval: 0,
|
|
1065
|
+
concurrency: Number.POSITIVE_INFINITY,
|
|
1066
|
+
autoStart: true,
|
|
1067
|
+
queueClass: PriorityQueue,
|
|
1068
|
+
strict: false,
|
|
1069
|
+
...options
|
|
1070
|
+
};
|
|
1071
|
+
if (!(typeof options.intervalCap === "number" && options.intervalCap >= 1)) {
|
|
1072
|
+
throw new TypeError(`Expected \`intervalCap\` to be a number from 1 and up, got \`${options.intervalCap?.toString() ?? ""}\` (${typeof options.intervalCap})`);
|
|
1073
|
+
}
|
|
1074
|
+
if (options.interval === void 0 || !(Number.isFinite(options.interval) && options.interval >= 0)) {
|
|
1075
|
+
throw new TypeError(`Expected \`interval\` to be a finite number >= 0, got \`${options.interval?.toString() ?? ""}\` (${typeof options.interval})`);
|
|
1076
|
+
}
|
|
1077
|
+
if (options.strict && options.interval === 0) {
|
|
1078
|
+
throw new TypeError("The `strict` option requires a non-zero `interval`");
|
|
1079
|
+
}
|
|
1080
|
+
if (options.strict && options.intervalCap === Number.POSITIVE_INFINITY) {
|
|
1081
|
+
throw new TypeError("The `strict` option requires a finite `intervalCap`");
|
|
1082
|
+
}
|
|
1083
|
+
this.#carryoverIntervalCount = options.carryoverIntervalCount ?? options.carryoverConcurrencyCount ?? false;
|
|
1084
|
+
this.#isIntervalIgnored = options.intervalCap === Number.POSITIVE_INFINITY || options.interval === 0;
|
|
1085
|
+
this.#intervalCap = options.intervalCap;
|
|
1086
|
+
this.#interval = options.interval;
|
|
1087
|
+
this.#strict = options.strict;
|
|
1088
|
+
this.#queue = new options.queueClass();
|
|
1089
|
+
this.#queueClass = options.queueClass;
|
|
1090
|
+
this.concurrency = options.concurrency;
|
|
1091
|
+
if (options.timeout !== void 0 && !(Number.isFinite(options.timeout) && options.timeout > 0)) {
|
|
1092
|
+
throw new TypeError(`Expected \`timeout\` to be a positive finite number, got \`${options.timeout}\` (${typeof options.timeout})`);
|
|
1093
|
+
}
|
|
1094
|
+
this.timeout = options.timeout;
|
|
1095
|
+
this.#isPaused = options.autoStart === false;
|
|
1096
|
+
this.#setupRateLimitTracking();
|
|
1097
|
+
}
|
|
1098
|
+
#cleanupStrictTicks(now) {
|
|
1099
|
+
while (this.#strictTicksStartIndex < this.#strictTicks.length) {
|
|
1100
|
+
const oldestTick = this.#strictTicks[this.#strictTicksStartIndex];
|
|
1101
|
+
if (oldestTick !== void 0 && now - oldestTick >= this.#interval) {
|
|
1102
|
+
this.#strictTicksStartIndex++;
|
|
1103
|
+
} else {
|
|
1104
|
+
break;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
const shouldCompact = this.#strictTicksStartIndex > 100 && this.#strictTicksStartIndex > this.#strictTicks.length / 2 || this.#strictTicksStartIndex === this.#strictTicks.length;
|
|
1108
|
+
if (shouldCompact) {
|
|
1109
|
+
this.#strictTicks = this.#strictTicks.slice(this.#strictTicksStartIndex);
|
|
1110
|
+
this.#strictTicksStartIndex = 0;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
// Helper methods for interval consumption
|
|
1114
|
+
#consumeIntervalSlot(now) {
|
|
1115
|
+
if (this.#strict) {
|
|
1116
|
+
this.#strictTicks.push(now);
|
|
1117
|
+
} else {
|
|
1118
|
+
this.#intervalCount++;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
#rollbackIntervalSlot() {
|
|
1122
|
+
if (this.#strict) {
|
|
1123
|
+
if (this.#strictTicks.length > this.#strictTicksStartIndex) {
|
|
1124
|
+
this.#strictTicks.pop();
|
|
1125
|
+
}
|
|
1126
|
+
} else if (this.#intervalCount > 0) {
|
|
1127
|
+
this.#intervalCount--;
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
#getActiveTicksCount() {
|
|
1131
|
+
return this.#strictTicks.length - this.#strictTicksStartIndex;
|
|
1132
|
+
}
|
|
1133
|
+
get #doesIntervalAllowAnother() {
|
|
1134
|
+
if (this.#isIntervalIgnored) {
|
|
1135
|
+
return true;
|
|
1136
|
+
}
|
|
1137
|
+
if (this.#strict) {
|
|
1138
|
+
return this.#getActiveTicksCount() < this.#intervalCap;
|
|
1139
|
+
}
|
|
1140
|
+
return this.#intervalCount < this.#intervalCap;
|
|
1141
|
+
}
|
|
1142
|
+
get #doesConcurrentAllowAnother() {
|
|
1143
|
+
return this.#pending < this.#concurrency;
|
|
1144
|
+
}
|
|
1145
|
+
#next() {
|
|
1146
|
+
this.#pending--;
|
|
1147
|
+
if (this.#pending === 0) {
|
|
1148
|
+
this.emit("pendingZero");
|
|
1149
|
+
}
|
|
1150
|
+
this.#tryToStartAnother();
|
|
1151
|
+
this.emit("next");
|
|
1152
|
+
}
|
|
1153
|
+
#onResumeInterval() {
|
|
1154
|
+
this.#timeoutId = void 0;
|
|
1155
|
+
this.#onInterval();
|
|
1156
|
+
this.#initializeIntervalIfNeeded();
|
|
1157
|
+
}
|
|
1158
|
+
#isIntervalPausedAt(now) {
|
|
1159
|
+
if (this.#strict) {
|
|
1160
|
+
this.#cleanupStrictTicks(now);
|
|
1161
|
+
const activeTicksCount = this.#getActiveTicksCount();
|
|
1162
|
+
if (activeTicksCount >= this.#intervalCap) {
|
|
1163
|
+
const oldestTick = this.#strictTicks[this.#strictTicksStartIndex];
|
|
1164
|
+
const delay = this.#interval - (now - oldestTick);
|
|
1165
|
+
this.#createIntervalTimeout(delay);
|
|
1166
|
+
return true;
|
|
1167
|
+
}
|
|
1168
|
+
return false;
|
|
1169
|
+
}
|
|
1170
|
+
if (this.#intervalId === void 0) {
|
|
1171
|
+
const delay = this.#intervalEnd - now;
|
|
1172
|
+
if (delay < 0) {
|
|
1173
|
+
if (this.#lastExecutionTime > 0) {
|
|
1174
|
+
const timeSinceLastExecution = now - this.#lastExecutionTime;
|
|
1175
|
+
if (timeSinceLastExecution < this.#interval) {
|
|
1176
|
+
this.#createIntervalTimeout(this.#interval - timeSinceLastExecution);
|
|
1177
|
+
return true;
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
1180
|
+
this.#intervalCount = this.#carryoverIntervalCount ? this.#pending : 0;
|
|
1181
|
+
} else {
|
|
1182
|
+
this.#createIntervalTimeout(delay);
|
|
1183
|
+
return true;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
return false;
|
|
1187
|
+
}
|
|
1188
|
+
#createIntervalTimeout(delay) {
|
|
1189
|
+
if (this.#timeoutId !== void 0) {
|
|
1190
|
+
return;
|
|
1191
|
+
}
|
|
1192
|
+
this.#timeoutId = setTimeout(() => {
|
|
1193
|
+
this.#onResumeInterval();
|
|
1194
|
+
}, delay);
|
|
1195
|
+
}
|
|
1196
|
+
#clearIntervalTimer() {
|
|
1197
|
+
if (this.#intervalId) {
|
|
1198
|
+
clearInterval(this.#intervalId);
|
|
1199
|
+
this.#intervalId = void 0;
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
#clearTimeoutTimer() {
|
|
1203
|
+
if (this.#timeoutId) {
|
|
1204
|
+
clearTimeout(this.#timeoutId);
|
|
1205
|
+
this.#timeoutId = void 0;
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
#tryToStartAnother() {
|
|
1209
|
+
if (this.#queue.size === 0) {
|
|
1210
|
+
this.#clearIntervalTimer();
|
|
1211
|
+
this.emit("empty");
|
|
1212
|
+
if (this.#pending === 0) {
|
|
1213
|
+
this.#clearTimeoutTimer();
|
|
1214
|
+
if (this.#strict && this.#strictTicksStartIndex > 0) {
|
|
1215
|
+
const now = Date.now();
|
|
1216
|
+
this.#cleanupStrictTicks(now);
|
|
1217
|
+
}
|
|
1218
|
+
this.emit("idle");
|
|
1219
|
+
}
|
|
1220
|
+
return false;
|
|
1221
|
+
}
|
|
1222
|
+
let taskStarted = false;
|
|
1223
|
+
if (!this.#isPaused) {
|
|
1224
|
+
const now = Date.now();
|
|
1225
|
+
const canInitializeInterval = !this.#isIntervalPausedAt(now);
|
|
1226
|
+
if (this.#doesIntervalAllowAnother && this.#doesConcurrentAllowAnother) {
|
|
1227
|
+
const job = this.#queue.dequeue();
|
|
1228
|
+
if (!this.#isIntervalIgnored) {
|
|
1229
|
+
this.#consumeIntervalSlot(now);
|
|
1230
|
+
this.#scheduleRateLimitUpdate();
|
|
1231
|
+
}
|
|
1232
|
+
this.emit("active");
|
|
1233
|
+
job();
|
|
1234
|
+
if (canInitializeInterval) {
|
|
1235
|
+
this.#initializeIntervalIfNeeded();
|
|
1236
|
+
}
|
|
1237
|
+
taskStarted = true;
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
return taskStarted;
|
|
1241
|
+
}
|
|
1242
|
+
#initializeIntervalIfNeeded() {
|
|
1243
|
+
if (this.#isIntervalIgnored || this.#intervalId !== void 0) {
|
|
1244
|
+
return;
|
|
1245
|
+
}
|
|
1246
|
+
if (this.#strict) {
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
this.#intervalId = setInterval(() => {
|
|
1250
|
+
this.#onInterval();
|
|
1251
|
+
}, this.#interval);
|
|
1252
|
+
this.#intervalEnd = Date.now() + this.#interval;
|
|
1253
|
+
}
|
|
1254
|
+
#onInterval() {
|
|
1255
|
+
if (!this.#strict) {
|
|
1256
|
+
if (this.#intervalCount === 0 && this.#pending === 0 && this.#intervalId) {
|
|
1257
|
+
this.#clearIntervalTimer();
|
|
1258
|
+
}
|
|
1259
|
+
this.#intervalCount = this.#carryoverIntervalCount ? this.#pending : 0;
|
|
1260
|
+
}
|
|
1261
|
+
this.#processQueue();
|
|
1262
|
+
this.#scheduleRateLimitUpdate();
|
|
1263
|
+
}
|
|
1264
|
+
/**
|
|
1265
|
+
Executes all queued functions until it reaches the limit.
|
|
1266
|
+
*/
|
|
1267
|
+
#processQueue() {
|
|
1268
|
+
while (this.#tryToStartAnother()) {
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
get concurrency() {
|
|
1272
|
+
return this.#concurrency;
|
|
1273
|
+
}
|
|
1274
|
+
set concurrency(newConcurrency) {
|
|
1275
|
+
if (!(typeof newConcurrency === "number" && newConcurrency >= 1)) {
|
|
1276
|
+
throw new TypeError(`Expected \`concurrency\` to be a number from 1 and up, got \`${newConcurrency}\` (${typeof newConcurrency})`);
|
|
1277
|
+
}
|
|
1278
|
+
this.#concurrency = newConcurrency;
|
|
1279
|
+
this.#processQueue();
|
|
1280
|
+
}
|
|
1281
|
+
/**
|
|
1282
|
+
Updates the priority of a promise function by its id, affecting its execution order. Requires a defined concurrency limit to take effect.
|
|
1283
|
+
|
|
1284
|
+
For example, this can be used to prioritize a promise function to run earlier.
|
|
1285
|
+
|
|
1286
|
+
```js
|
|
1287
|
+
import PQueue from 'p-queue';
|
|
1288
|
+
|
|
1289
|
+
const queue = new PQueue({concurrency: 1});
|
|
1290
|
+
|
|
1291
|
+
queue.add(async () => '🦄', {priority: 1});
|
|
1292
|
+
queue.add(async () => '🦀', {priority: 0, id: '🦀'});
|
|
1293
|
+
queue.add(async () => '🦄', {priority: 1});
|
|
1294
|
+
queue.add(async () => '🦄', {priority: 1});
|
|
1295
|
+
|
|
1296
|
+
queue.setPriority('🦀', 2);
|
|
1297
|
+
```
|
|
1298
|
+
|
|
1299
|
+
In this case, the promise function with `id: '🦀'` runs second.
|
|
1300
|
+
|
|
1301
|
+
You can also deprioritize a promise function to delay its execution:
|
|
1302
|
+
|
|
1303
|
+
```js
|
|
1304
|
+
import PQueue from 'p-queue';
|
|
1305
|
+
|
|
1306
|
+
const queue = new PQueue({concurrency: 1});
|
|
1307
|
+
|
|
1308
|
+
queue.add(async () => '🦄', {priority: 1});
|
|
1309
|
+
queue.add(async () => '🦀', {priority: 1, id: '🦀'});
|
|
1310
|
+
queue.add(async () => '🦄');
|
|
1311
|
+
queue.add(async () => '🦄', {priority: 0});
|
|
1312
|
+
|
|
1313
|
+
queue.setPriority('🦀', -1);
|
|
1314
|
+
```
|
|
1315
|
+
Here, the promise function with `id: '🦀'` executes last.
|
|
1316
|
+
*/
|
|
1317
|
+
setPriority(id, priority) {
|
|
1318
|
+
if (typeof priority !== "number" || !Number.isFinite(priority)) {
|
|
1319
|
+
throw new TypeError(`Expected \`priority\` to be a finite number, got \`${priority}\` (${typeof priority})`);
|
|
1320
|
+
}
|
|
1321
|
+
this.#queue.setPriority(id, priority);
|
|
1322
|
+
}
|
|
1323
|
+
async add(function_, options = {}) {
|
|
1324
|
+
options = {
|
|
1325
|
+
timeout: this.timeout,
|
|
1326
|
+
...options,
|
|
1327
|
+
// Assign unique ID if not provided
|
|
1328
|
+
id: options.id ?? (this.#idAssigner++).toString()
|
|
1329
|
+
};
|
|
1330
|
+
return new Promise((resolve4, reject) => {
|
|
1331
|
+
const taskSymbol = /* @__PURE__ */ Symbol(`task-${options.id}`);
|
|
1332
|
+
this.#queue.enqueue(async () => {
|
|
1333
|
+
this.#pending++;
|
|
1334
|
+
this.#runningTasks.set(taskSymbol, {
|
|
1335
|
+
id: options.id,
|
|
1336
|
+
priority: options.priority ?? 0,
|
|
1337
|
+
// Match priority-queue default
|
|
1338
|
+
startTime: Date.now(),
|
|
1339
|
+
timeout: options.timeout
|
|
1340
|
+
});
|
|
1341
|
+
let eventListener;
|
|
1342
|
+
try {
|
|
1343
|
+
try {
|
|
1344
|
+
options.signal?.throwIfAborted();
|
|
1345
|
+
} catch (error) {
|
|
1346
|
+
this.#rollbackIntervalConsumption();
|
|
1347
|
+
this.#runningTasks.delete(taskSymbol);
|
|
1348
|
+
throw error;
|
|
1349
|
+
}
|
|
1350
|
+
this.#lastExecutionTime = Date.now();
|
|
1351
|
+
let operation = function_({ signal: options.signal });
|
|
1352
|
+
if (options.timeout) {
|
|
1353
|
+
operation = pTimeout(Promise.resolve(operation), {
|
|
1354
|
+
milliseconds: options.timeout,
|
|
1355
|
+
message: `Task timed out after ${options.timeout}ms (queue has ${this.#pending} running, ${this.#queue.size} waiting)`
|
|
1356
|
+
});
|
|
1357
|
+
}
|
|
1358
|
+
if (options.signal) {
|
|
1359
|
+
const { signal } = options;
|
|
1360
|
+
operation = Promise.race([operation, new Promise((_resolve, reject2) => {
|
|
1361
|
+
eventListener = () => {
|
|
1362
|
+
reject2(signal.reason);
|
|
1363
|
+
};
|
|
1364
|
+
signal.addEventListener("abort", eventListener, { once: true });
|
|
1365
|
+
})]);
|
|
1366
|
+
}
|
|
1367
|
+
const result = await operation;
|
|
1368
|
+
resolve4(result);
|
|
1369
|
+
this.emit("completed", result);
|
|
1370
|
+
} catch (error) {
|
|
1371
|
+
reject(error);
|
|
1372
|
+
this.emit("error", error);
|
|
1373
|
+
} finally {
|
|
1374
|
+
if (eventListener) {
|
|
1375
|
+
options.signal?.removeEventListener("abort", eventListener);
|
|
1376
|
+
}
|
|
1377
|
+
this.#runningTasks.delete(taskSymbol);
|
|
1378
|
+
queueMicrotask(() => {
|
|
1379
|
+
this.#next();
|
|
1380
|
+
});
|
|
1381
|
+
}
|
|
1382
|
+
}, options);
|
|
1383
|
+
this.emit("add");
|
|
1384
|
+
this.#tryToStartAnother();
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
async addAll(functions, options) {
|
|
1388
|
+
return Promise.all(functions.map(async (function_) => this.add(function_, options)));
|
|
1389
|
+
}
|
|
1390
|
+
/**
|
|
1391
|
+
Start (or resume) executing enqueued tasks within concurrency limit. No need to call this if queue is not paused (via `options.autoStart = false` or by `.pause()` method.)
|
|
1392
|
+
*/
|
|
1393
|
+
start() {
|
|
1394
|
+
if (!this.#isPaused) {
|
|
1395
|
+
return this;
|
|
1396
|
+
}
|
|
1397
|
+
this.#isPaused = false;
|
|
1398
|
+
this.#processQueue();
|
|
1399
|
+
return this;
|
|
1400
|
+
}
|
|
1401
|
+
/**
|
|
1402
|
+
Put queue execution on hold.
|
|
1403
|
+
*/
|
|
1404
|
+
pause() {
|
|
1405
|
+
this.#isPaused = true;
|
|
1406
|
+
}
|
|
1407
|
+
/**
|
|
1408
|
+
Clear the queue.
|
|
1409
|
+
*/
|
|
1410
|
+
clear() {
|
|
1411
|
+
this.#queue = new this.#queueClass();
|
|
1412
|
+
this.#clearIntervalTimer();
|
|
1413
|
+
this.#updateRateLimitState();
|
|
1414
|
+
this.emit("empty");
|
|
1415
|
+
if (this.#pending === 0) {
|
|
1416
|
+
this.#clearTimeoutTimer();
|
|
1417
|
+
this.emit("idle");
|
|
1418
|
+
}
|
|
1419
|
+
this.emit("next");
|
|
1420
|
+
}
|
|
1421
|
+
/**
|
|
1422
|
+
Can be called multiple times. Useful if you for example add additional items at a later time.
|
|
1423
|
+
|
|
1424
|
+
@returns A promise that settles when the queue becomes empty.
|
|
1425
|
+
*/
|
|
1426
|
+
async onEmpty() {
|
|
1427
|
+
if (this.#queue.size === 0) {
|
|
1428
|
+
return;
|
|
1429
|
+
}
|
|
1430
|
+
await this.#onEvent("empty");
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
@returns A promise that settles when the queue size is less than the given limit: `queue.size < limit`.
|
|
1434
|
+
|
|
1435
|
+
If you want to avoid having the queue grow beyond a certain size you can `await queue.onSizeLessThan()` before adding a new item.
|
|
1436
|
+
|
|
1437
|
+
Note that this only limits the number of items waiting to start. There could still be up to `concurrency` jobs already running that this call does not include in its calculation.
|
|
1438
|
+
*/
|
|
1439
|
+
async onSizeLessThan(limit) {
|
|
1440
|
+
if (this.#queue.size < limit) {
|
|
1441
|
+
return;
|
|
1442
|
+
}
|
|
1443
|
+
await this.#onEvent("next", () => this.#queue.size < limit);
|
|
1444
|
+
}
|
|
1445
|
+
/**
|
|
1446
|
+
The difference with `.onEmpty` is that `.onIdle` guarantees that all work from the queue has finished. `.onEmpty` merely signals that the queue is empty, but it could mean that some promises haven't completed yet.
|
|
1447
|
+
|
|
1448
|
+
@returns A promise that settles when the queue becomes empty, and all promises have completed; `queue.size === 0 && queue.pending === 0`.
|
|
1449
|
+
*/
|
|
1450
|
+
async onIdle() {
|
|
1451
|
+
if (this.#pending === 0 && this.#queue.size === 0) {
|
|
1452
|
+
return;
|
|
1453
|
+
}
|
|
1454
|
+
await this.#onEvent("idle");
|
|
1455
|
+
}
|
|
1456
|
+
/**
|
|
1457
|
+
The difference with `.onIdle` is that `.onPendingZero` only waits for currently running tasks to finish, ignoring queued tasks.
|
|
1458
|
+
|
|
1459
|
+
@returns A promise that settles when all currently running tasks have completed; `queue.pending === 0`.
|
|
1460
|
+
*/
|
|
1461
|
+
async onPendingZero() {
|
|
1462
|
+
if (this.#pending === 0) {
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
await this.#onEvent("pendingZero");
|
|
1466
|
+
}
|
|
1467
|
+
/**
|
|
1468
|
+
@returns A promise that settles when the queue becomes rate-limited due to intervalCap.
|
|
1469
|
+
*/
|
|
1470
|
+
async onRateLimit() {
|
|
1471
|
+
if (this.isRateLimited) {
|
|
1472
|
+
return;
|
|
1473
|
+
}
|
|
1474
|
+
await this.#onEvent("rateLimit");
|
|
1475
|
+
}
|
|
1476
|
+
/**
|
|
1477
|
+
@returns A promise that settles when the queue is no longer rate-limited.
|
|
1478
|
+
*/
|
|
1479
|
+
async onRateLimitCleared() {
|
|
1480
|
+
if (!this.isRateLimited) {
|
|
1481
|
+
return;
|
|
1482
|
+
}
|
|
1483
|
+
await this.#onEvent("rateLimitCleared");
|
|
1484
|
+
}
|
|
1485
|
+
/**
|
|
1486
|
+
@returns A promise that rejects when any task in the queue errors.
|
|
1487
|
+
|
|
1488
|
+
Use with `Promise.race([queue.onError(), queue.onIdle()])` to fail fast on the first error while still resolving normally when the queue goes idle.
|
|
1489
|
+
|
|
1490
|
+
Important: The promise returned by `add()` still rejects. You must handle each `add()` promise (for example, `.catch(() => {})`) to avoid unhandled rejections.
|
|
1491
|
+
|
|
1492
|
+
@example
|
|
1493
|
+
```
|
|
1494
|
+
import PQueue from 'p-queue';
|
|
1495
|
+
|
|
1496
|
+
const queue = new PQueue({concurrency: 2});
|
|
1497
|
+
|
|
1498
|
+
queue.add(() => fetchData(1)).catch(() => {});
|
|
1499
|
+
queue.add(() => fetchData(2)).catch(() => {});
|
|
1500
|
+
queue.add(() => fetchData(3)).catch(() => {});
|
|
1501
|
+
|
|
1502
|
+
// Stop processing on first error
|
|
1503
|
+
try {
|
|
1504
|
+
await Promise.race([
|
|
1505
|
+
queue.onError(),
|
|
1506
|
+
queue.onIdle()
|
|
1507
|
+
]);
|
|
1508
|
+
} catch (error) {
|
|
1509
|
+
queue.pause(); // Stop processing remaining tasks
|
|
1510
|
+
console.error('Queue failed:', error);
|
|
1511
|
+
}
|
|
1512
|
+
```
|
|
1513
|
+
*/
|
|
1514
|
+
// eslint-disable-next-line @typescript-eslint/promise-function-async
|
|
1515
|
+
onError() {
|
|
1516
|
+
return new Promise((_resolve, reject) => {
|
|
1517
|
+
const handleError = (error) => {
|
|
1518
|
+
this.off("error", handleError);
|
|
1519
|
+
reject(error);
|
|
1520
|
+
};
|
|
1521
|
+
this.on("error", handleError);
|
|
1522
|
+
});
|
|
1523
|
+
}
|
|
1524
|
+
async #onEvent(event, filter) {
|
|
1525
|
+
return new Promise((resolve4) => {
|
|
1526
|
+
const listener = () => {
|
|
1527
|
+
if (filter && !filter()) {
|
|
1528
|
+
return;
|
|
1529
|
+
}
|
|
1530
|
+
this.off(event, listener);
|
|
1531
|
+
resolve4();
|
|
1532
|
+
};
|
|
1533
|
+
this.on(event, listener);
|
|
1534
|
+
});
|
|
1535
|
+
}
|
|
1536
|
+
/**
|
|
1537
|
+
Size of the queue, the number of queued items waiting to run.
|
|
1538
|
+
*/
|
|
1539
|
+
get size() {
|
|
1540
|
+
return this.#queue.size;
|
|
1541
|
+
}
|
|
1542
|
+
/**
|
|
1543
|
+
Size of the queue, filtered by the given options.
|
|
1544
|
+
|
|
1545
|
+
For example, this can be used to find the number of items remaining in the queue with a specific priority level.
|
|
1546
|
+
*/
|
|
1547
|
+
sizeBy(options) {
|
|
1548
|
+
return this.#queue.filter(options).length;
|
|
1549
|
+
}
|
|
1550
|
+
/**
|
|
1551
|
+
Number of running items (no longer in the queue).
|
|
1552
|
+
*/
|
|
1553
|
+
get pending() {
|
|
1554
|
+
return this.#pending;
|
|
1555
|
+
}
|
|
1556
|
+
/**
|
|
1557
|
+
Whether the queue is currently paused.
|
|
1558
|
+
*/
|
|
1559
|
+
get isPaused() {
|
|
1560
|
+
return this.#isPaused;
|
|
1561
|
+
}
|
|
1562
|
+
#setupRateLimitTracking() {
|
|
1563
|
+
if (this.#isIntervalIgnored) {
|
|
1564
|
+
return;
|
|
1565
|
+
}
|
|
1566
|
+
this.on("add", () => {
|
|
1567
|
+
if (this.#queue.size > 0) {
|
|
1568
|
+
this.#scheduleRateLimitUpdate();
|
|
1569
|
+
}
|
|
1570
|
+
});
|
|
1571
|
+
this.on("next", () => {
|
|
1572
|
+
this.#scheduleRateLimitUpdate();
|
|
1573
|
+
});
|
|
1574
|
+
}
|
|
1575
|
+
#scheduleRateLimitUpdate() {
|
|
1576
|
+
if (this.#isIntervalIgnored || this.#rateLimitFlushScheduled) {
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
this.#rateLimitFlushScheduled = true;
|
|
1580
|
+
queueMicrotask(() => {
|
|
1581
|
+
this.#rateLimitFlushScheduled = false;
|
|
1582
|
+
this.#updateRateLimitState();
|
|
1583
|
+
});
|
|
1584
|
+
}
|
|
1585
|
+
#rollbackIntervalConsumption() {
|
|
1586
|
+
if (this.#isIntervalIgnored) {
|
|
1587
|
+
return;
|
|
1588
|
+
}
|
|
1589
|
+
this.#rollbackIntervalSlot();
|
|
1590
|
+
this.#scheduleRateLimitUpdate();
|
|
1591
|
+
}
|
|
1592
|
+
#updateRateLimitState() {
|
|
1593
|
+
const previous = this.#rateLimitedInInterval;
|
|
1594
|
+
if (this.#isIntervalIgnored || this.#queue.size === 0) {
|
|
1595
|
+
if (previous) {
|
|
1596
|
+
this.#rateLimitedInInterval = false;
|
|
1597
|
+
this.emit("rateLimitCleared");
|
|
1598
|
+
}
|
|
1599
|
+
return;
|
|
1600
|
+
}
|
|
1601
|
+
let count;
|
|
1602
|
+
if (this.#strict) {
|
|
1603
|
+
const now = Date.now();
|
|
1604
|
+
this.#cleanupStrictTicks(now);
|
|
1605
|
+
count = this.#getActiveTicksCount();
|
|
1606
|
+
} else {
|
|
1607
|
+
count = this.#intervalCount;
|
|
1608
|
+
}
|
|
1609
|
+
const shouldBeRateLimited = count >= this.#intervalCap;
|
|
1610
|
+
if (shouldBeRateLimited !== previous) {
|
|
1611
|
+
this.#rateLimitedInInterval = shouldBeRateLimited;
|
|
1612
|
+
this.emit(shouldBeRateLimited ? "rateLimit" : "rateLimitCleared");
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
/**
|
|
1616
|
+
Whether the queue is currently rate-limited due to intervalCap.
|
|
1617
|
+
*/
|
|
1618
|
+
get isRateLimited() {
|
|
1619
|
+
return this.#rateLimitedInInterval;
|
|
1620
|
+
}
|
|
1621
|
+
/**
|
|
1622
|
+
Whether the queue is saturated. Returns `true` when:
|
|
1623
|
+
- All concurrency slots are occupied and tasks are waiting, OR
|
|
1624
|
+
- The queue is rate-limited and tasks are waiting
|
|
1625
|
+
|
|
1626
|
+
Useful for detecting backpressure and potential hanging tasks.
|
|
1627
|
+
|
|
1628
|
+
```js
|
|
1629
|
+
import PQueue from 'p-queue';
|
|
1630
|
+
|
|
1631
|
+
const queue = new PQueue({concurrency: 2});
|
|
1632
|
+
|
|
1633
|
+
// Backpressure handling
|
|
1634
|
+
if (queue.isSaturated) {
|
|
1635
|
+
console.log('Queue is saturated, waiting for capacity...');
|
|
1636
|
+
await queue.onSizeLessThan(queue.concurrency);
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
// Monitoring for stuck tasks
|
|
1640
|
+
setInterval(() => {
|
|
1641
|
+
if (queue.isSaturated) {
|
|
1642
|
+
console.warn(`Queue saturated: ${queue.pending} running, ${queue.size} waiting`);
|
|
1643
|
+
}
|
|
1644
|
+
}, 60000);
|
|
1645
|
+
```
|
|
1646
|
+
*/
|
|
1647
|
+
get isSaturated() {
|
|
1648
|
+
return this.#pending === this.#concurrency && this.#queue.size > 0 || this.isRateLimited && this.#queue.size > 0;
|
|
1649
|
+
}
|
|
1650
|
+
/**
|
|
1651
|
+
The tasks currently being executed. Each task includes its `id`, `priority`, `startTime`, and `timeout` (if set).
|
|
1652
|
+
|
|
1653
|
+
Returns an array of task info objects.
|
|
1654
|
+
|
|
1655
|
+
```js
|
|
1656
|
+
import PQueue from 'p-queue';
|
|
1657
|
+
|
|
1658
|
+
const queue = new PQueue({concurrency: 2});
|
|
1659
|
+
|
|
1660
|
+
// Add tasks with IDs for better debugging
|
|
1661
|
+
queue.add(() => fetchUser(123), {id: 'user-123'});
|
|
1662
|
+
queue.add(() => fetchPosts(456), {id: 'posts-456', priority: 1});
|
|
1663
|
+
|
|
1664
|
+
// Check what's running
|
|
1665
|
+
console.log(queue.runningTasks);
|
|
1666
|
+
// => [{
|
|
1667
|
+
// id: 'user-123',
|
|
1668
|
+
// priority: 0,
|
|
1669
|
+
// startTime: 1759253001716,
|
|
1670
|
+
// timeout: undefined
|
|
1671
|
+
// }, {
|
|
1672
|
+
// id: 'posts-456',
|
|
1673
|
+
// priority: 1,
|
|
1674
|
+
// startTime: 1759253001916,
|
|
1675
|
+
// timeout: undefined
|
|
1676
|
+
// }]
|
|
1677
|
+
```
|
|
1678
|
+
*/
|
|
1679
|
+
get runningTasks() {
|
|
1680
|
+
return [...this.#runningTasks.values()].map((task) => ({ ...task }));
|
|
1681
|
+
}
|
|
1682
|
+
};
|
|
1683
|
+
|
|
1684
|
+
// node_modules/is-network-error/index.js
|
|
1685
|
+
var objectToString = Object.prototype.toString;
|
|
1686
|
+
var isError = (value) => objectToString.call(value) === "[object Error]";
|
|
1687
|
+
var errorMessages = /* @__PURE__ */ new Set([
|
|
1688
|
+
"network error",
|
|
1689
|
+
// Chrome
|
|
1690
|
+
"Failed to fetch",
|
|
1691
|
+
// Chrome
|
|
1692
|
+
"NetworkError when attempting to fetch resource.",
|
|
1693
|
+
// Firefox
|
|
1694
|
+
"The Internet connection appears to be offline.",
|
|
1695
|
+
// Safari 16
|
|
1696
|
+
"Network request failed",
|
|
1697
|
+
// `cross-fetch`
|
|
1698
|
+
"fetch failed",
|
|
1699
|
+
// Undici (Node.js)
|
|
1700
|
+
"terminated",
|
|
1701
|
+
// Undici (Node.js)
|
|
1702
|
+
" A network error occurred.",
|
|
1703
|
+
// Bun (WebKit)
|
|
1704
|
+
"Network connection lost"
|
|
1705
|
+
// Cloudflare Workers (fetch)
|
|
1706
|
+
]);
|
|
1707
|
+
function isNetworkError(error) {
|
|
1708
|
+
const isValid = error && isError(error) && error.name === "TypeError" && typeof error.message === "string";
|
|
1709
|
+
if (!isValid) {
|
|
1710
|
+
return false;
|
|
1711
|
+
}
|
|
1712
|
+
const { message, stack } = error;
|
|
1713
|
+
if (message === "Load failed") {
|
|
1714
|
+
return stack === void 0 || "__sentry_captured__" in error;
|
|
1715
|
+
}
|
|
1716
|
+
if (message.startsWith("error sending request for url")) {
|
|
1717
|
+
return true;
|
|
1718
|
+
}
|
|
1719
|
+
return errorMessages.has(message);
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
// node_modules/p-retry/index.js
|
|
1723
|
+
function validateRetries(retries) {
|
|
1724
|
+
if (typeof retries === "number") {
|
|
1725
|
+
if (retries < 0) {
|
|
1726
|
+
throw new TypeError("Expected `retries` to be a non-negative number.");
|
|
1727
|
+
}
|
|
1728
|
+
if (Number.isNaN(retries)) {
|
|
1729
|
+
throw new TypeError("Expected `retries` to be a valid number or Infinity, got NaN.");
|
|
1730
|
+
}
|
|
1731
|
+
} else if (retries !== void 0) {
|
|
1732
|
+
throw new TypeError("Expected `retries` to be a number or Infinity.");
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
function validateNumberOption(name, value, { min = 0, allowInfinity = false } = {}) {
|
|
1736
|
+
if (value === void 0) {
|
|
1737
|
+
return;
|
|
1738
|
+
}
|
|
1739
|
+
if (typeof value !== "number" || Number.isNaN(value)) {
|
|
1740
|
+
throw new TypeError(`Expected \`${name}\` to be a number${allowInfinity ? " or Infinity" : ""}.`);
|
|
1741
|
+
}
|
|
1742
|
+
if (!allowInfinity && !Number.isFinite(value)) {
|
|
1743
|
+
throw new TypeError(`Expected \`${name}\` to be a finite number.`);
|
|
1744
|
+
}
|
|
1745
|
+
if (value < min) {
|
|
1746
|
+
throw new TypeError(`Expected \`${name}\` to be \u2265 ${min}.`);
|
|
1747
|
+
}
|
|
1748
|
+
}
|
|
1749
|
+
var AbortError = class extends Error {
|
|
1750
|
+
constructor(message) {
|
|
1751
|
+
super();
|
|
1752
|
+
if (message instanceof Error) {
|
|
1753
|
+
this.originalError = message;
|
|
1754
|
+
({ message } = message);
|
|
1755
|
+
} else {
|
|
1756
|
+
this.originalError = new Error(message);
|
|
1757
|
+
this.originalError.stack = this.stack;
|
|
1758
|
+
}
|
|
1759
|
+
this.name = "AbortError";
|
|
1760
|
+
this.message = message;
|
|
1761
|
+
}
|
|
1762
|
+
};
|
|
1763
|
+
function calculateDelay(retriesConsumed, options) {
|
|
1764
|
+
const attempt = Math.max(1, retriesConsumed + 1);
|
|
1765
|
+
const random = options.randomize ? Math.random() + 1 : 1;
|
|
1766
|
+
let timeout = Math.round(random * options.minTimeout * options.factor ** (attempt - 1));
|
|
1767
|
+
timeout = Math.min(timeout, options.maxTimeout);
|
|
1768
|
+
return timeout;
|
|
1769
|
+
}
|
|
1770
|
+
function calculateRemainingTime(start, max) {
|
|
1771
|
+
if (!Number.isFinite(max)) {
|
|
1772
|
+
return max;
|
|
1773
|
+
}
|
|
1774
|
+
return max - (performance.now() - start);
|
|
1775
|
+
}
|
|
1776
|
+
async function onAttemptFailure({ error, attemptNumber, retriesConsumed, startTime, options }) {
|
|
1777
|
+
const normalizedError = error instanceof Error ? error : new TypeError(`Non-error was thrown: "${error}". You should only throw errors.`);
|
|
1778
|
+
if (normalizedError instanceof AbortError) {
|
|
1779
|
+
throw normalizedError.originalError;
|
|
1780
|
+
}
|
|
1781
|
+
const retriesLeft = Number.isFinite(options.retries) ? Math.max(0, options.retries - retriesConsumed) : options.retries;
|
|
1782
|
+
const maxRetryTime = options.maxRetryTime ?? Number.POSITIVE_INFINITY;
|
|
1783
|
+
const context = Object.freeze({
|
|
1784
|
+
error: normalizedError,
|
|
1785
|
+
attemptNumber,
|
|
1786
|
+
retriesLeft,
|
|
1787
|
+
retriesConsumed
|
|
1788
|
+
});
|
|
1789
|
+
await options.onFailedAttempt(context);
|
|
1790
|
+
if (calculateRemainingTime(startTime, maxRetryTime) <= 0) {
|
|
1791
|
+
throw normalizedError;
|
|
1792
|
+
}
|
|
1793
|
+
const consumeRetry = await options.shouldConsumeRetry(context);
|
|
1794
|
+
const remainingTime = calculateRemainingTime(startTime, maxRetryTime);
|
|
1795
|
+
if (remainingTime <= 0 || retriesLeft <= 0) {
|
|
1796
|
+
throw normalizedError;
|
|
1797
|
+
}
|
|
1798
|
+
if (normalizedError instanceof TypeError && !isNetworkError(normalizedError)) {
|
|
1799
|
+
if (consumeRetry) {
|
|
1800
|
+
throw normalizedError;
|
|
1801
|
+
}
|
|
1802
|
+
options.signal?.throwIfAborted();
|
|
1803
|
+
return false;
|
|
1804
|
+
}
|
|
1805
|
+
if (!await options.shouldRetry(context)) {
|
|
1806
|
+
throw normalizedError;
|
|
1807
|
+
}
|
|
1808
|
+
if (!consumeRetry) {
|
|
1809
|
+
options.signal?.throwIfAborted();
|
|
1810
|
+
return false;
|
|
1811
|
+
}
|
|
1812
|
+
const delayTime = calculateDelay(retriesConsumed, options);
|
|
1813
|
+
const finalDelay = Math.min(delayTime, remainingTime);
|
|
1814
|
+
options.signal?.throwIfAborted();
|
|
1815
|
+
if (finalDelay > 0) {
|
|
1816
|
+
await new Promise((resolve4, reject) => {
|
|
1817
|
+
const onAbort = () => {
|
|
1818
|
+
clearTimeout(timeoutToken);
|
|
1819
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
1820
|
+
reject(options.signal.reason);
|
|
1821
|
+
};
|
|
1822
|
+
const timeoutToken = setTimeout(() => {
|
|
1823
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
1824
|
+
resolve4();
|
|
1825
|
+
}, finalDelay);
|
|
1826
|
+
if (options.unref) {
|
|
1827
|
+
timeoutToken.unref?.();
|
|
1828
|
+
}
|
|
1829
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
1830
|
+
});
|
|
1831
|
+
}
|
|
1832
|
+
options.signal?.throwIfAborted();
|
|
1833
|
+
return true;
|
|
1834
|
+
}
|
|
1835
|
+
async function pRetry(input, options = {}) {
|
|
1836
|
+
options = { ...options };
|
|
1837
|
+
validateRetries(options.retries);
|
|
1838
|
+
if (Object.hasOwn(options, "forever")) {
|
|
1839
|
+
throw new Error("The `forever` option is no longer supported. For many use-cases, you can set `retries: Infinity` instead.");
|
|
1840
|
+
}
|
|
1841
|
+
options.retries ??= 10;
|
|
1842
|
+
options.factor ??= 2;
|
|
1843
|
+
options.minTimeout ??= 1e3;
|
|
1844
|
+
options.maxTimeout ??= Number.POSITIVE_INFINITY;
|
|
1845
|
+
options.maxRetryTime ??= Number.POSITIVE_INFINITY;
|
|
1846
|
+
options.randomize ??= false;
|
|
1847
|
+
options.onFailedAttempt ??= () => {
|
|
1848
|
+
};
|
|
1849
|
+
options.shouldRetry ??= () => true;
|
|
1850
|
+
options.shouldConsumeRetry ??= () => true;
|
|
1851
|
+
validateNumberOption("factor", options.factor, { min: 0, allowInfinity: false });
|
|
1852
|
+
validateNumberOption("minTimeout", options.minTimeout, { min: 0, allowInfinity: false });
|
|
1853
|
+
validateNumberOption("maxTimeout", options.maxTimeout, { min: 0, allowInfinity: true });
|
|
1854
|
+
validateNumberOption("maxRetryTime", options.maxRetryTime, { min: 0, allowInfinity: true });
|
|
1855
|
+
if (!(options.factor > 0)) {
|
|
1856
|
+
options.factor = 1;
|
|
1857
|
+
}
|
|
1858
|
+
options.signal?.throwIfAborted();
|
|
1859
|
+
let attemptNumber = 0;
|
|
1860
|
+
let retriesConsumed = 0;
|
|
1861
|
+
const startTime = performance.now();
|
|
1862
|
+
while (Number.isFinite(options.retries) ? retriesConsumed <= options.retries : true) {
|
|
1863
|
+
attemptNumber++;
|
|
1864
|
+
try {
|
|
1865
|
+
options.signal?.throwIfAborted();
|
|
1866
|
+
const result = await input(attemptNumber);
|
|
1867
|
+
options.signal?.throwIfAborted();
|
|
1868
|
+
return result;
|
|
1869
|
+
} catch (error) {
|
|
1870
|
+
if (await onAttemptFailure({
|
|
1871
|
+
error,
|
|
1872
|
+
attemptNumber,
|
|
1873
|
+
retriesConsumed,
|
|
1874
|
+
startTime,
|
|
1875
|
+
options
|
|
1876
|
+
})) {
|
|
1877
|
+
retriesConsumed++;
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
throw new Error("Retry attempts exhausted without throwing an error.");
|
|
1882
|
+
}
|
|
1883
|
+
|
|
1884
|
+
// src/embeddings/detector.ts
|
|
1885
|
+
var import_fs = require("fs");
|
|
1886
|
+
var path = __toESM(require("path"), 1);
|
|
1887
|
+
var os = __toESM(require("os"), 1);
|
|
1888
|
+
function getOpenCodeAuthPath() {
|
|
1889
|
+
return path.join(os.homedir(), ".local", "share", "opencode", "auth.json");
|
|
1890
|
+
}
|
|
1891
|
+
function loadOpenCodeAuth() {
|
|
1892
|
+
const authPath = getOpenCodeAuthPath();
|
|
1893
|
+
try {
|
|
1894
|
+
if ((0, import_fs.existsSync)(authPath)) {
|
|
1895
|
+
return JSON.parse((0, import_fs.readFileSync)(authPath, "utf-8"));
|
|
1896
|
+
}
|
|
1897
|
+
} catch {
|
|
1898
|
+
}
|
|
1899
|
+
return {};
|
|
1900
|
+
}
|
|
1901
|
+
async function detectEmbeddingProvider(preferredProvider, model) {
|
|
1902
|
+
const credentials = await getProviderCredentials(preferredProvider);
|
|
1903
|
+
if (credentials) {
|
|
1904
|
+
if (!model) {
|
|
1905
|
+
return {
|
|
1906
|
+
provider: preferredProvider,
|
|
1907
|
+
credentials,
|
|
1908
|
+
modelInfo: getDefaultModelForProvider(preferredProvider)
|
|
1909
|
+
};
|
|
1910
|
+
}
|
|
1911
|
+
if (!isValidModel(model, preferredProvider)) {
|
|
1912
|
+
throw new Error(
|
|
1913
|
+
`Model '${model}' is not supported by provider '${preferredProvider}'`
|
|
1914
|
+
);
|
|
1915
|
+
}
|
|
1916
|
+
const providerModels = EMBEDDING_MODELS[preferredProvider];
|
|
1917
|
+
return {
|
|
1918
|
+
provider: preferredProvider,
|
|
1919
|
+
credentials,
|
|
1920
|
+
modelInfo: providerModels[model]
|
|
1921
|
+
};
|
|
1922
|
+
}
|
|
1923
|
+
throw new Error(
|
|
1924
|
+
`Preferred provider '${preferredProvider}' is not configured or authenticated`
|
|
1925
|
+
);
|
|
1926
|
+
}
|
|
1927
|
+
async function tryDetectProvider() {
|
|
1928
|
+
for (const provider of availableProviders) {
|
|
1929
|
+
const credentials = await getProviderCredentials(provider);
|
|
1930
|
+
if (credentials) {
|
|
1931
|
+
return {
|
|
1932
|
+
provider,
|
|
1933
|
+
credentials,
|
|
1934
|
+
modelInfo: getDefaultModelForProvider(provider)
|
|
1935
|
+
};
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
throw new Error(
|
|
1939
|
+
`No embedding-capable provider found. Please authenticate with OpenCode using one of: ${availableProviders.join(", ")}.`
|
|
1940
|
+
);
|
|
1941
|
+
}
|
|
1942
|
+
async function getProviderCredentials(provider) {
|
|
1943
|
+
switch (provider) {
|
|
1944
|
+
case "github-copilot":
|
|
1945
|
+
return getGitHubCopilotCredentials();
|
|
1946
|
+
case "openai":
|
|
1947
|
+
return getOpenAICredentials();
|
|
1948
|
+
case "google":
|
|
1949
|
+
return getGoogleCredentials();
|
|
1950
|
+
case "ollama":
|
|
1951
|
+
return getOllamaCredentials();
|
|
1952
|
+
default:
|
|
1953
|
+
return null;
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
function getGitHubCopilotCredentials() {
|
|
1957
|
+
const authData = loadOpenCodeAuth();
|
|
1958
|
+
const copilotAuth = authData["github-copilot"] || authData["github-copilot-enterprise"];
|
|
1959
|
+
if (!copilotAuth || copilotAuth.type !== "oauth") {
|
|
1960
|
+
return null;
|
|
1961
|
+
}
|
|
1962
|
+
const baseUrl = copilotAuth.enterpriseUrl ? `https://copilot-api.${copilotAuth.enterpriseUrl.replace(/^https?:\/\//, "").replace(/\/$/, "")}` : "https://models.github.ai";
|
|
1963
|
+
return {
|
|
1964
|
+
provider: "github-copilot",
|
|
1965
|
+
baseUrl,
|
|
1966
|
+
refreshToken: copilotAuth.refresh,
|
|
1967
|
+
accessToken: copilotAuth.access,
|
|
1968
|
+
tokenExpires: copilotAuth.expires
|
|
1969
|
+
};
|
|
1970
|
+
}
|
|
1971
|
+
function getOpenAICredentials() {
|
|
1972
|
+
const authData = loadOpenCodeAuth();
|
|
1973
|
+
const openaiAuth = authData["openai"];
|
|
1974
|
+
if (openaiAuth?.type === "api") {
|
|
1975
|
+
return {
|
|
1976
|
+
provider: "openai",
|
|
1977
|
+
apiKey: openaiAuth.key,
|
|
1978
|
+
baseUrl: "https://api.openai.com/v1"
|
|
1979
|
+
};
|
|
1980
|
+
}
|
|
1981
|
+
return null;
|
|
1982
|
+
}
|
|
1983
|
+
function getGoogleCredentials() {
|
|
1984
|
+
const authData = loadOpenCodeAuth();
|
|
1985
|
+
const googleAuth = authData["google"] || authData["google-generative-ai"];
|
|
1986
|
+
if (googleAuth?.type === "api") {
|
|
1987
|
+
return {
|
|
1988
|
+
provider: "google",
|
|
1989
|
+
apiKey: googleAuth.key,
|
|
1990
|
+
baseUrl: "https://generativelanguage.googleapis.com/v1beta"
|
|
1991
|
+
};
|
|
1992
|
+
}
|
|
1993
|
+
return null;
|
|
1994
|
+
}
|
|
1995
|
+
async function getOllamaCredentials() {
|
|
1996
|
+
const baseUrl = process.env.OLLAMA_HOST || "http://localhost:11434";
|
|
1997
|
+
try {
|
|
1998
|
+
const controller = new AbortController();
|
|
1999
|
+
const timeoutId = setTimeout(() => controller.abort(), 2e3);
|
|
2000
|
+
const response = await fetch(`${baseUrl}/api/tags`, {
|
|
2001
|
+
signal: controller.signal
|
|
2002
|
+
});
|
|
2003
|
+
clearTimeout(timeoutId);
|
|
2004
|
+
if (response.ok) {
|
|
2005
|
+
const data = await response.json();
|
|
2006
|
+
const hasEmbeddingModel = data.models?.some(
|
|
2007
|
+
(m) => m.name.includes("nomic-embed") || m.name.includes("mxbai-embed") || m.name.includes("all-minilm")
|
|
2008
|
+
);
|
|
2009
|
+
if (hasEmbeddingModel) {
|
|
2010
|
+
return {
|
|
2011
|
+
provider: "ollama",
|
|
2012
|
+
baseUrl
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
2015
|
+
}
|
|
2016
|
+
} catch {
|
|
2017
|
+
return null;
|
|
2018
|
+
}
|
|
2019
|
+
return null;
|
|
2020
|
+
}
|
|
2021
|
+
function getProviderDisplayName(provider) {
|
|
2022
|
+
switch (provider) {
|
|
2023
|
+
case "github-copilot":
|
|
2024
|
+
return "GitHub Copilot";
|
|
2025
|
+
case "openai":
|
|
2026
|
+
return "OpenAI";
|
|
2027
|
+
case "google":
|
|
2028
|
+
return "Google (Gemini)";
|
|
2029
|
+
case "ollama":
|
|
2030
|
+
return "Ollama (Local)";
|
|
2031
|
+
default:
|
|
2032
|
+
return provider;
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
|
|
2036
|
+
// src/embeddings/provider.ts
|
|
2037
|
+
function createEmbeddingProvider(configuredProviderInfo) {
|
|
2038
|
+
switch (configuredProviderInfo.provider) {
|
|
2039
|
+
case "github-copilot":
|
|
2040
|
+
return new GitHubCopilotEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2041
|
+
case "openai":
|
|
2042
|
+
return new OpenAIEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2043
|
+
case "google":
|
|
2044
|
+
return new GoogleEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2045
|
+
case "ollama":
|
|
2046
|
+
return new OllamaEmbeddingProvider(configuredProviderInfo.credentials, configuredProviderInfo.modelInfo);
|
|
2047
|
+
default: {
|
|
2048
|
+
const _exhaustive = configuredProviderInfo;
|
|
2049
|
+
throw new Error(`Unsupported embedding provider: ${_exhaustive.provider}`);
|
|
2050
|
+
}
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
var GitHubCopilotEmbeddingProvider = class {
|
|
2054
|
+
constructor(credentials, modelInfo) {
|
|
2055
|
+
this.credentials = credentials;
|
|
2056
|
+
this.modelInfo = modelInfo;
|
|
2057
|
+
}
|
|
2058
|
+
getToken() {
|
|
2059
|
+
if (!this.credentials.refreshToken) {
|
|
2060
|
+
throw new Error("No OAuth token available for GitHub");
|
|
2061
|
+
}
|
|
2062
|
+
return this.credentials.refreshToken;
|
|
2063
|
+
}
|
|
2064
|
+
async embedQuery(query) {
|
|
2065
|
+
const result = await this.embedBatch([query]);
|
|
2066
|
+
return {
|
|
2067
|
+
embedding: result.embeddings[0],
|
|
2068
|
+
tokensUsed: result.totalTokensUsed
|
|
2069
|
+
};
|
|
2070
|
+
}
|
|
2071
|
+
async embedDocument(document) {
|
|
2072
|
+
const result = await this.embedBatch([document]);
|
|
2073
|
+
return {
|
|
2074
|
+
embedding: result.embeddings[0],
|
|
2075
|
+
tokensUsed: result.totalTokensUsed
|
|
2076
|
+
};
|
|
2077
|
+
}
|
|
2078
|
+
async embedBatch(texts) {
|
|
2079
|
+
const token = this.getToken();
|
|
2080
|
+
const response = await fetch(`${this.credentials.baseUrl}/inference/embeddings`, {
|
|
2081
|
+
method: "POST",
|
|
2082
|
+
headers: {
|
|
2083
|
+
Authorization: `Bearer ${token}`,
|
|
2084
|
+
"Content-Type": "application/json",
|
|
2085
|
+
Accept: "application/vnd.github+json",
|
|
2086
|
+
"X-GitHub-Api-Version": "2022-11-28"
|
|
2087
|
+
},
|
|
2088
|
+
body: JSON.stringify({
|
|
2089
|
+
model: `openai/${this.modelInfo.model}`,
|
|
2090
|
+
input: texts
|
|
2091
|
+
})
|
|
2092
|
+
});
|
|
2093
|
+
if (!response.ok) {
|
|
2094
|
+
const error = await response.text();
|
|
2095
|
+
throw new Error(`GitHub Copilot embedding API error: ${response.status} - ${error}`);
|
|
2096
|
+
}
|
|
2097
|
+
const data = await response.json();
|
|
2098
|
+
return {
|
|
2099
|
+
embeddings: data.data.map((d) => d.embedding),
|
|
2100
|
+
totalTokensUsed: data.usage.total_tokens
|
|
2101
|
+
};
|
|
2102
|
+
}
|
|
2103
|
+
getModelInfo() {
|
|
2104
|
+
return this.modelInfo;
|
|
2105
|
+
}
|
|
2106
|
+
};
|
|
2107
|
+
var OpenAIEmbeddingProvider = class {
|
|
2108
|
+
constructor(credentials, modelInfo) {
|
|
2109
|
+
this.credentials = credentials;
|
|
2110
|
+
this.modelInfo = modelInfo;
|
|
2111
|
+
}
|
|
2112
|
+
async embedQuery(query) {
|
|
2113
|
+
const result = await this.embedBatch([query]);
|
|
2114
|
+
return {
|
|
2115
|
+
embedding: result.embeddings[0],
|
|
2116
|
+
tokensUsed: result.totalTokensUsed
|
|
2117
|
+
};
|
|
2118
|
+
}
|
|
2119
|
+
async embedDocument(document) {
|
|
2120
|
+
const result = await this.embedBatch([document]);
|
|
2121
|
+
return {
|
|
2122
|
+
embedding: result.embeddings[0],
|
|
2123
|
+
tokensUsed: result.totalTokensUsed
|
|
2124
|
+
};
|
|
2125
|
+
}
|
|
2126
|
+
async embedBatch(texts) {
|
|
2127
|
+
const response = await fetch(`${this.credentials.baseUrl}/embeddings`, {
|
|
2128
|
+
method: "POST",
|
|
2129
|
+
headers: {
|
|
2130
|
+
Authorization: `Bearer ${this.credentials.apiKey}`,
|
|
2131
|
+
"Content-Type": "application/json"
|
|
2132
|
+
},
|
|
2133
|
+
body: JSON.stringify({
|
|
2134
|
+
model: this.modelInfo.model,
|
|
2135
|
+
input: texts
|
|
2136
|
+
})
|
|
2137
|
+
});
|
|
2138
|
+
if (!response.ok) {
|
|
2139
|
+
const error = await response.text();
|
|
2140
|
+
throw new Error(`OpenAI embedding API error: ${response.status} - ${error}`);
|
|
2141
|
+
}
|
|
2142
|
+
const data = await response.json();
|
|
2143
|
+
return {
|
|
2144
|
+
embeddings: data.data.map((d) => d.embedding),
|
|
2145
|
+
totalTokensUsed: data.usage.total_tokens
|
|
2146
|
+
};
|
|
2147
|
+
}
|
|
2148
|
+
getModelInfo() {
|
|
2149
|
+
return this.modelInfo;
|
|
2150
|
+
}
|
|
2151
|
+
};
|
|
2152
|
+
var GoogleEmbeddingProvider = class _GoogleEmbeddingProvider {
|
|
2153
|
+
constructor(credentials, modelInfo) {
|
|
2154
|
+
this.credentials = credentials;
|
|
2155
|
+
this.modelInfo = modelInfo;
|
|
2156
|
+
}
|
|
2157
|
+
static BATCH_SIZE = 20;
|
|
2158
|
+
async embedQuery(query) {
|
|
2159
|
+
const taskType = this.modelInfo.taskAble ? "CODE_RETRIEVAL_QUERY" : void 0;
|
|
2160
|
+
const result = await this.embedWithTaskType([query], taskType);
|
|
2161
|
+
return {
|
|
2162
|
+
embedding: result.embeddings[0],
|
|
2163
|
+
tokensUsed: result.totalTokensUsed
|
|
2164
|
+
};
|
|
2165
|
+
}
|
|
2166
|
+
async embedDocument(document) {
|
|
2167
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2168
|
+
const result = await this.embedWithTaskType([document], taskType);
|
|
2169
|
+
return {
|
|
2170
|
+
embedding: result.embeddings[0],
|
|
2171
|
+
tokensUsed: result.totalTokensUsed
|
|
2172
|
+
};
|
|
2173
|
+
}
|
|
2174
|
+
async embedBatch(texts) {
|
|
2175
|
+
const taskType = this.modelInfo.taskAble ? "RETRIEVAL_DOCUMENT" : void 0;
|
|
2176
|
+
return this.embedWithTaskType(texts, taskType);
|
|
2177
|
+
}
|
|
2178
|
+
/**
|
|
2179
|
+
* Embeds texts using the Google embedContent API.
|
|
2180
|
+
* Sends multiple texts as parts in batched requests (up to BATCH_SIZE per call).
|
|
2181
|
+
* When taskType is provided (gemini-embedding-001), includes it in the request
|
|
2182
|
+
* for task-specific embedding optimization.
|
|
2183
|
+
*/
|
|
2184
|
+
async embedWithTaskType(texts, taskType) {
|
|
2185
|
+
const batches = [];
|
|
2186
|
+
for (let i = 0; i < texts.length; i += _GoogleEmbeddingProvider.BATCH_SIZE) {
|
|
2187
|
+
batches.push(texts.slice(i, i + _GoogleEmbeddingProvider.BATCH_SIZE));
|
|
2188
|
+
}
|
|
2189
|
+
const batchResults = await Promise.all(
|
|
2190
|
+
batches.map(async (batch) => {
|
|
2191
|
+
const requests = batch.map((text) => ({
|
|
2192
|
+
model: `models/${this.modelInfo.model}`,
|
|
2193
|
+
content: {
|
|
2194
|
+
parts: [{ text }]
|
|
2195
|
+
},
|
|
2196
|
+
taskType,
|
|
2197
|
+
outputDimensionality: this.modelInfo.dimensions
|
|
2198
|
+
}));
|
|
2199
|
+
const response = await fetch(
|
|
2200
|
+
`${this.credentials.baseUrl}/models/${this.modelInfo.model}:batchEmbedContents?key=${this.credentials.apiKey}`,
|
|
2201
|
+
{
|
|
2202
|
+
method: "POST",
|
|
2203
|
+
headers: {
|
|
2204
|
+
"Content-Type": "application/json"
|
|
2205
|
+
},
|
|
2206
|
+
body: JSON.stringify({ requests })
|
|
2207
|
+
}
|
|
2208
|
+
);
|
|
2209
|
+
if (!response.ok) {
|
|
2210
|
+
const error = await response.text();
|
|
2211
|
+
throw new Error(`Google embedding API error: ${response.status} - ${error}`);
|
|
2212
|
+
}
|
|
2213
|
+
const data = await response.json();
|
|
2214
|
+
return {
|
|
2215
|
+
embeddings: data.embeddings.map((e) => e.values),
|
|
2216
|
+
tokensUsed: batch.reduce((sum, text) => sum + Math.ceil(text.length / 4), 0)
|
|
2217
|
+
};
|
|
2218
|
+
})
|
|
2219
|
+
);
|
|
2220
|
+
return {
|
|
2221
|
+
embeddings: batchResults.flatMap((r) => r.embeddings),
|
|
2222
|
+
totalTokensUsed: batchResults.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2223
|
+
};
|
|
2224
|
+
}
|
|
2225
|
+
getModelInfo() {
|
|
2226
|
+
return this.modelInfo;
|
|
2227
|
+
}
|
|
2228
|
+
};
|
|
2229
|
+
var OllamaEmbeddingProvider = class {
|
|
2230
|
+
constructor(credentials, modelInfo) {
|
|
2231
|
+
this.credentials = credentials;
|
|
2232
|
+
this.modelInfo = modelInfo;
|
|
2233
|
+
}
|
|
2234
|
+
async embedQuery(query) {
|
|
2235
|
+
const result = await this.embedBatch([query]);
|
|
2236
|
+
return {
|
|
2237
|
+
embedding: result.embeddings[0],
|
|
2238
|
+
tokensUsed: result.totalTokensUsed
|
|
2239
|
+
};
|
|
2240
|
+
}
|
|
2241
|
+
async embedDocument(document) {
|
|
2242
|
+
const result = await this.embedBatch([document]);
|
|
2243
|
+
return {
|
|
2244
|
+
embedding: result.embeddings[0],
|
|
2245
|
+
tokensUsed: result.totalTokensUsed
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
async embedBatch(texts) {
|
|
2249
|
+
const results = await Promise.all(
|
|
2250
|
+
texts.map(async (text) => {
|
|
2251
|
+
const response = await fetch(`${this.credentials.baseUrl}/api/embeddings`, {
|
|
2252
|
+
method: "POST",
|
|
2253
|
+
headers: {
|
|
2254
|
+
"Content-Type": "application/json"
|
|
2255
|
+
},
|
|
2256
|
+
body: JSON.stringify({
|
|
2257
|
+
model: this.modelInfo.model,
|
|
2258
|
+
prompt: text
|
|
2259
|
+
})
|
|
2260
|
+
});
|
|
2261
|
+
if (!response.ok) {
|
|
2262
|
+
const error = await response.text();
|
|
2263
|
+
throw new Error(`Ollama embedding API error: ${response.status} - ${error}`);
|
|
2264
|
+
}
|
|
2265
|
+
const data = await response.json();
|
|
2266
|
+
return {
|
|
2267
|
+
embedding: data.embedding,
|
|
2268
|
+
tokensUsed: Math.ceil(text.length / 4)
|
|
2269
|
+
};
|
|
2270
|
+
})
|
|
2271
|
+
);
|
|
2272
|
+
return {
|
|
2273
|
+
embeddings: results.map((r) => r.embedding),
|
|
2274
|
+
totalTokensUsed: results.reduce((sum, r) => sum + r.tokensUsed, 0)
|
|
2275
|
+
};
|
|
2276
|
+
}
|
|
2277
|
+
getModelInfo() {
|
|
2278
|
+
return this.modelInfo;
|
|
2279
|
+
}
|
|
2280
|
+
};
|
|
2281
|
+
|
|
2282
|
+
// src/utils/files.ts
|
|
2283
|
+
var import_ignore = __toESM(require_ignore(), 1);
|
|
2284
|
+
var import_fs2 = require("fs");
|
|
2285
|
+
var path2 = __toESM(require("path"), 1);
|
|
2286
|
+
function createIgnoreFilter(projectRoot) {
|
|
2287
|
+
const ig = (0, import_ignore.default)();
|
|
2288
|
+
const defaultIgnores = [
|
|
2289
|
+
"node_modules",
|
|
2290
|
+
".git",
|
|
2291
|
+
"dist",
|
|
2292
|
+
"build",
|
|
2293
|
+
".next",
|
|
2294
|
+
".nuxt",
|
|
2295
|
+
"coverage",
|
|
2296
|
+
"__pycache__",
|
|
2297
|
+
"target",
|
|
2298
|
+
"vendor",
|
|
2299
|
+
".opencode"
|
|
2300
|
+
];
|
|
2301
|
+
ig.add(defaultIgnores);
|
|
2302
|
+
const gitignorePath = path2.join(projectRoot, ".gitignore");
|
|
2303
|
+
if ((0, import_fs2.existsSync)(gitignorePath)) {
|
|
2304
|
+
const gitignoreContent = (0, import_fs2.readFileSync)(gitignorePath, "utf-8");
|
|
2305
|
+
ig.add(gitignoreContent);
|
|
2306
|
+
}
|
|
2307
|
+
return ig;
|
|
2308
|
+
}
|
|
2309
|
+
function matchGlob(filePath, pattern) {
|
|
2310
|
+
let regexPattern = pattern.replace(/\*\*/g, "<<<DOUBLESTAR>>>").replace(/\*/g, "[^/]*").replace(/<<<DOUBLESTAR>>>/g, ".*").replace(/\?/g, ".").replace(/\{([^}]+)\}/g, (_, p1) => `(${p1.split(",").join("|")})`);
|
|
2311
|
+
if (regexPattern.startsWith(".*/")) {
|
|
2312
|
+
regexPattern = `(.*\\/)?${regexPattern.slice(3)}`;
|
|
2313
|
+
}
|
|
2314
|
+
const regex = new RegExp(`^${regexPattern}$`);
|
|
2315
|
+
return regex.test(filePath);
|
|
2316
|
+
}
|
|
2317
|
+
async function* walkDirectory(dir, projectRoot, includePatterns, excludePatterns, ignoreFilter, maxFileSize, skipped) {
|
|
2318
|
+
const entries = await import_fs2.promises.readdir(dir, { withFileTypes: true });
|
|
2319
|
+
for (const entry of entries) {
|
|
2320
|
+
const fullPath = path2.join(dir, entry.name);
|
|
2321
|
+
const relativePath = path2.relative(projectRoot, fullPath);
|
|
2322
|
+
if (ignoreFilter.ignores(relativePath)) {
|
|
2323
|
+
if (entry.isFile()) {
|
|
2324
|
+
skipped.push({ path: relativePath, reason: "gitignore" });
|
|
2325
|
+
}
|
|
2326
|
+
continue;
|
|
2327
|
+
}
|
|
2328
|
+
if (entry.isDirectory()) {
|
|
2329
|
+
yield* walkDirectory(
|
|
2330
|
+
fullPath,
|
|
2331
|
+
projectRoot,
|
|
2332
|
+
includePatterns,
|
|
2333
|
+
excludePatterns,
|
|
2334
|
+
ignoreFilter,
|
|
2335
|
+
maxFileSize,
|
|
2336
|
+
skipped
|
|
2337
|
+
);
|
|
2338
|
+
} else if (entry.isFile()) {
|
|
2339
|
+
const stat = await import_fs2.promises.stat(fullPath);
|
|
2340
|
+
if (stat.size > maxFileSize) {
|
|
2341
|
+
skipped.push({ path: relativePath, reason: "too_large" });
|
|
2342
|
+
continue;
|
|
2343
|
+
}
|
|
2344
|
+
for (const pattern of excludePatterns) {
|
|
2345
|
+
if (matchGlob(relativePath, pattern)) {
|
|
2346
|
+
skipped.push({ path: relativePath, reason: "excluded" });
|
|
2347
|
+
continue;
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
let matched = false;
|
|
2351
|
+
for (const pattern of includePatterns) {
|
|
2352
|
+
if (matchGlob(relativePath, pattern)) {
|
|
2353
|
+
matched = true;
|
|
2354
|
+
break;
|
|
2355
|
+
}
|
|
2356
|
+
}
|
|
2357
|
+
if (matched) {
|
|
2358
|
+
yield { path: fullPath, size: stat.size };
|
|
2359
|
+
}
|
|
2360
|
+
}
|
|
2361
|
+
}
|
|
2362
|
+
}
|
|
2363
|
+
async function collectFiles(projectRoot, includePatterns, excludePatterns, maxFileSize) {
|
|
2364
|
+
const ignoreFilter = createIgnoreFilter(projectRoot);
|
|
2365
|
+
const files = [];
|
|
2366
|
+
const skipped = [];
|
|
2367
|
+
for await (const file of walkDirectory(
|
|
2368
|
+
projectRoot,
|
|
2369
|
+
projectRoot,
|
|
2370
|
+
includePatterns,
|
|
2371
|
+
excludePatterns,
|
|
2372
|
+
ignoreFilter,
|
|
2373
|
+
maxFileSize,
|
|
2374
|
+
skipped
|
|
2375
|
+
)) {
|
|
2376
|
+
files.push(file);
|
|
2377
|
+
}
|
|
2378
|
+
return { files, skipped };
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
// src/utils/cost.ts
|
|
2382
|
+
function estimateChunksFromFiles(files) {
|
|
2383
|
+
let totalChunks = 0;
|
|
2384
|
+
for (const file of files) {
|
|
2385
|
+
const avgChunkSize = 400;
|
|
2386
|
+
const chunksPerFile = Math.max(1, Math.ceil(file.size / avgChunkSize));
|
|
2387
|
+
totalChunks += chunksPerFile;
|
|
2388
|
+
}
|
|
2389
|
+
return totalChunks;
|
|
2390
|
+
}
|
|
2391
|
+
function estimateCost(estimatedTokens, modelInfo) {
|
|
2392
|
+
return estimatedTokens / 1e6 * modelInfo.costPer1MTokens;
|
|
2393
|
+
}
|
|
2394
|
+
function createCostEstimate(files, provider) {
|
|
2395
|
+
const filesCount = files.length;
|
|
2396
|
+
const totalSizeBytes = files.reduce((sum, f) => sum + f.size, 0);
|
|
2397
|
+
const estimatedChunks = estimateChunksFromFiles(files);
|
|
2398
|
+
const avgTokensPerChunk = 150;
|
|
2399
|
+
const estimatedTokens = estimatedChunks * avgTokensPerChunk;
|
|
2400
|
+
const estimatedCost = estimateCost(estimatedTokens, provider.modelInfo);
|
|
2401
|
+
return {
|
|
2402
|
+
filesCount,
|
|
2403
|
+
totalSizeBytes,
|
|
2404
|
+
estimatedChunks,
|
|
2405
|
+
estimatedTokens,
|
|
2406
|
+
estimatedCost,
|
|
2407
|
+
provider: getProviderDisplayName(provider.provider),
|
|
2408
|
+
model: provider.modelInfo.model,
|
|
2409
|
+
isFree: provider.modelInfo.costPer1MTokens === 0
|
|
2410
|
+
};
|
|
2411
|
+
}
|
|
2412
|
+
function formatCostEstimate(estimate) {
|
|
2413
|
+
const sizeFormatted = formatBytes(estimate.totalSizeBytes);
|
|
2414
|
+
const costFormatted = estimate.isFree ? "Free" : `~$${estimate.estimatedCost.toFixed(4)}`;
|
|
2415
|
+
return `
|
|
2416
|
+
\u250C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
|
|
2417
|
+
\u2502 \u{1F4CA} Indexing Estimate \u2502
|
|
2418
|
+
\u251C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524
|
|
2419
|
+
\u2502 \u2502
|
|
2420
|
+
\u2502 Files to index: ${padRight(estimate.filesCount.toLocaleString() + " files", 40)}\u2502
|
|
2421
|
+
\u2502 Total size: ${padRight(sizeFormatted, 40)}\u2502
|
|
2422
|
+
\u2502 Estimated chunks: ${padRight("~" + estimate.estimatedChunks.toLocaleString() + " chunks", 40)}\u2502
|
|
2423
|
+
\u2502 Estimated tokens: ${padRight("~" + estimate.estimatedTokens.toLocaleString() + " tokens", 40)}\u2502
|
|
2424
|
+
\u2502 \u2502
|
|
2425
|
+
\u2502 Provider: ${padRight(estimate.provider, 52)}\u2502
|
|
2426
|
+
\u2502 Model: ${padRight(estimate.model, 52)}\u2502
|
|
2427
|
+
\u2502 Cost: ${padRight(costFormatted, 52)}\u2502
|
|
2428
|
+
\u2502 \u2502
|
|
2429
|
+
\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
|
|
2430
|
+
`;
|
|
2431
|
+
}
|
|
2432
|
+
function formatBytes(bytes) {
|
|
2433
|
+
if (bytes === 0) return "0 B";
|
|
2434
|
+
const k = 1024;
|
|
2435
|
+
const sizes = ["B", "KB", "MB", "GB"];
|
|
2436
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
2437
|
+
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + " " + sizes[i];
|
|
2438
|
+
}
|
|
2439
|
+
function padRight(str, length) {
|
|
2440
|
+
return str.padEnd(length);
|
|
2441
|
+
}
|
|
2442
|
+
|
|
2443
|
+
// src/utils/logger.ts
|
|
2444
|
+
var LOG_LEVEL_PRIORITY = {
|
|
2445
|
+
error: 0,
|
|
2446
|
+
warn: 1,
|
|
2447
|
+
info: 2,
|
|
2448
|
+
debug: 3
|
|
2449
|
+
};
|
|
2450
|
+
function createEmptyMetrics() {
|
|
2451
|
+
return {
|
|
2452
|
+
filesScanned: 0,
|
|
2453
|
+
filesParsed: 0,
|
|
2454
|
+
parseMs: 0,
|
|
2455
|
+
chunksProcessed: 0,
|
|
2456
|
+
chunksEmbedded: 0,
|
|
2457
|
+
chunksFromCache: 0,
|
|
2458
|
+
chunksRemoved: 0,
|
|
2459
|
+
embeddingApiCalls: 0,
|
|
2460
|
+
embeddingTokensUsed: 0,
|
|
2461
|
+
embeddingErrors: 0,
|
|
2462
|
+
searchCount: 0,
|
|
2463
|
+
searchTotalMs: 0,
|
|
2464
|
+
searchAvgMs: 0,
|
|
2465
|
+
searchLastMs: 0,
|
|
2466
|
+
embeddingCallMs: 0,
|
|
2467
|
+
vectorSearchMs: 0,
|
|
2468
|
+
keywordSearchMs: 0,
|
|
2469
|
+
fusionMs: 0,
|
|
2470
|
+
cacheHits: 0,
|
|
2471
|
+
cacheMisses: 0,
|
|
2472
|
+
queryCacheHits: 0,
|
|
2473
|
+
queryCacheSimilarHits: 0,
|
|
2474
|
+
queryCacheMisses: 0,
|
|
2475
|
+
gcRuns: 0,
|
|
2476
|
+
gcOrphansRemoved: 0,
|
|
2477
|
+
gcChunksRemoved: 0,
|
|
2478
|
+
gcEmbeddingsRemoved: 0
|
|
2479
|
+
};
|
|
2480
|
+
}
|
|
2481
|
+
var Logger = class {
|
|
2482
|
+
config;
|
|
2483
|
+
metrics;
|
|
2484
|
+
logs = [];
|
|
2485
|
+
maxLogs = 1e3;
|
|
2486
|
+
constructor(config) {
|
|
2487
|
+
this.config = config;
|
|
2488
|
+
this.metrics = createEmptyMetrics();
|
|
2489
|
+
}
|
|
2490
|
+
shouldLog(level) {
|
|
2491
|
+
if (!this.config.enabled) return false;
|
|
2492
|
+
return LOG_LEVEL_PRIORITY[level] <= LOG_LEVEL_PRIORITY[this.config.logLevel];
|
|
2493
|
+
}
|
|
2494
|
+
log(level, category, message, data) {
|
|
2495
|
+
if (!this.shouldLog(level)) return;
|
|
2496
|
+
const entry = {
|
|
2497
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2498
|
+
level,
|
|
2499
|
+
category,
|
|
2500
|
+
message,
|
|
2501
|
+
data
|
|
2502
|
+
};
|
|
2503
|
+
this.logs.push(entry);
|
|
2504
|
+
if (this.logs.length > this.maxLogs) {
|
|
2505
|
+
this.logs.shift();
|
|
2506
|
+
}
|
|
2507
|
+
}
|
|
2508
|
+
search(level, message, data) {
|
|
2509
|
+
if (this.config.logSearch) {
|
|
2510
|
+
this.log(level, "search", message, data);
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
embedding(level, message, data) {
|
|
2514
|
+
if (this.config.logEmbedding) {
|
|
2515
|
+
this.log(level, "embedding", message, data);
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
cache(level, message, data) {
|
|
2519
|
+
if (this.config.logCache) {
|
|
2520
|
+
this.log(level, "cache", message, data);
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
gc(level, message, data) {
|
|
2524
|
+
if (this.config.logGc) {
|
|
2525
|
+
this.log(level, "gc", message, data);
|
|
2526
|
+
}
|
|
2527
|
+
}
|
|
2528
|
+
branch(level, message, data) {
|
|
2529
|
+
if (this.config.logBranch) {
|
|
2530
|
+
this.log(level, "branch", message, data);
|
|
2531
|
+
}
|
|
2532
|
+
}
|
|
2533
|
+
info(message, data) {
|
|
2534
|
+
this.log("info", "general", message, data);
|
|
2535
|
+
}
|
|
2536
|
+
warn(message, data) {
|
|
2537
|
+
this.log("warn", "general", message, data);
|
|
2538
|
+
}
|
|
2539
|
+
error(message, data) {
|
|
2540
|
+
this.log("error", "general", message, data);
|
|
2541
|
+
}
|
|
2542
|
+
debug(message, data) {
|
|
2543
|
+
this.log("debug", "general", message, data);
|
|
2544
|
+
}
|
|
2545
|
+
recordIndexingStart() {
|
|
2546
|
+
if (!this.config.metrics) return;
|
|
2547
|
+
this.metrics.indexingStartTime = Date.now();
|
|
2548
|
+
}
|
|
2549
|
+
recordIndexingEnd() {
|
|
2550
|
+
if (!this.config.metrics) return;
|
|
2551
|
+
this.metrics.indexingEndTime = Date.now();
|
|
2552
|
+
}
|
|
2553
|
+
recordFilesScanned(count) {
|
|
2554
|
+
if (!this.config.metrics) return;
|
|
2555
|
+
this.metrics.filesScanned = count;
|
|
2556
|
+
}
|
|
2557
|
+
recordFilesParsed(count) {
|
|
2558
|
+
if (!this.config.metrics) return;
|
|
2559
|
+
this.metrics.filesParsed = count;
|
|
2560
|
+
}
|
|
2561
|
+
recordParseDuration(durationMs) {
|
|
2562
|
+
if (!this.config.metrics) return;
|
|
2563
|
+
this.metrics.parseMs = durationMs;
|
|
2564
|
+
}
|
|
2565
|
+
recordChunksProcessed(count) {
|
|
2566
|
+
if (!this.config.metrics) return;
|
|
2567
|
+
this.metrics.chunksProcessed += count;
|
|
2568
|
+
}
|
|
2569
|
+
recordChunksEmbedded(count) {
|
|
2570
|
+
if (!this.config.metrics) return;
|
|
2571
|
+
this.metrics.chunksEmbedded += count;
|
|
2572
|
+
}
|
|
2573
|
+
recordChunksFromCache(count) {
|
|
2574
|
+
if (!this.config.metrics) return;
|
|
2575
|
+
this.metrics.chunksFromCache += count;
|
|
2576
|
+
}
|
|
2577
|
+
recordChunksRemoved(count) {
|
|
2578
|
+
if (!this.config.metrics) return;
|
|
2579
|
+
this.metrics.chunksRemoved += count;
|
|
2580
|
+
}
|
|
2581
|
+
recordEmbeddingApiCall(tokens) {
|
|
2582
|
+
if (!this.config.metrics) return;
|
|
2583
|
+
this.metrics.embeddingApiCalls++;
|
|
2584
|
+
this.metrics.embeddingTokensUsed += tokens;
|
|
2585
|
+
}
|
|
2586
|
+
recordEmbeddingError() {
|
|
2587
|
+
if (!this.config.metrics) return;
|
|
2588
|
+
this.metrics.embeddingErrors++;
|
|
2589
|
+
}
|
|
2590
|
+
recordSearch(durationMs, breakdown) {
|
|
2591
|
+
if (!this.config.metrics) return;
|
|
2592
|
+
this.metrics.searchCount++;
|
|
2593
|
+
this.metrics.searchTotalMs += durationMs;
|
|
2594
|
+
this.metrics.searchLastMs = durationMs;
|
|
2595
|
+
this.metrics.searchAvgMs = this.metrics.searchTotalMs / this.metrics.searchCount;
|
|
2596
|
+
if (breakdown) {
|
|
2597
|
+
this.metrics.embeddingCallMs = breakdown.embeddingMs;
|
|
2598
|
+
this.metrics.vectorSearchMs = breakdown.vectorMs;
|
|
2599
|
+
this.metrics.keywordSearchMs = breakdown.keywordMs;
|
|
2600
|
+
this.metrics.fusionMs = breakdown.fusionMs;
|
|
2601
|
+
}
|
|
2602
|
+
}
|
|
2603
|
+
recordCacheHit() {
|
|
2604
|
+
if (!this.config.metrics) return;
|
|
2605
|
+
this.metrics.cacheHits++;
|
|
2606
|
+
}
|
|
2607
|
+
recordCacheMiss() {
|
|
2608
|
+
if (!this.config.metrics) return;
|
|
2609
|
+
this.metrics.cacheMisses++;
|
|
2610
|
+
}
|
|
2611
|
+
recordQueryCacheHit() {
|
|
2612
|
+
if (!this.config.metrics) return;
|
|
2613
|
+
this.metrics.queryCacheHits++;
|
|
2614
|
+
}
|
|
2615
|
+
recordQueryCacheSimilarHit() {
|
|
2616
|
+
if (!this.config.metrics) return;
|
|
2617
|
+
this.metrics.queryCacheSimilarHits++;
|
|
2618
|
+
}
|
|
2619
|
+
recordQueryCacheMiss() {
|
|
2620
|
+
if (!this.config.metrics) return;
|
|
2621
|
+
this.metrics.queryCacheMisses++;
|
|
2622
|
+
}
|
|
2623
|
+
recordGc(orphans, chunks, embeddings) {
|
|
2624
|
+
if (!this.config.metrics) return;
|
|
2625
|
+
this.metrics.gcRuns++;
|
|
2626
|
+
this.metrics.gcOrphansRemoved += orphans;
|
|
2627
|
+
this.metrics.gcChunksRemoved += chunks;
|
|
2628
|
+
this.metrics.gcEmbeddingsRemoved += embeddings;
|
|
2629
|
+
}
|
|
2630
|
+
getMetrics() {
|
|
2631
|
+
return { ...this.metrics };
|
|
2632
|
+
}
|
|
2633
|
+
getLogs(limit) {
|
|
2634
|
+
const logs = [...this.logs];
|
|
2635
|
+
if (limit) {
|
|
2636
|
+
return logs.slice(-limit);
|
|
2637
|
+
}
|
|
2638
|
+
return logs;
|
|
2639
|
+
}
|
|
2640
|
+
getLogsByCategory(category, limit) {
|
|
2641
|
+
const filtered = this.logs.filter((l) => l.category === category);
|
|
2642
|
+
if (limit) {
|
|
2643
|
+
return filtered.slice(-limit);
|
|
2644
|
+
}
|
|
2645
|
+
return filtered;
|
|
2646
|
+
}
|
|
2647
|
+
getLogsByLevel(level, limit) {
|
|
2648
|
+
const filtered = this.logs.filter((l) => l.level === level);
|
|
2649
|
+
if (limit) {
|
|
2650
|
+
return filtered.slice(-limit);
|
|
2651
|
+
}
|
|
2652
|
+
return filtered;
|
|
2653
|
+
}
|
|
2654
|
+
resetMetrics() {
|
|
2655
|
+
this.metrics = createEmptyMetrics();
|
|
2656
|
+
}
|
|
2657
|
+
clearLogs() {
|
|
2658
|
+
this.logs = [];
|
|
2659
|
+
}
|
|
2660
|
+
formatMetrics() {
|
|
2661
|
+
const m = this.metrics;
|
|
2662
|
+
const lines = [];
|
|
2663
|
+
lines.push("=== Metrics ===");
|
|
2664
|
+
if (m.indexingStartTime && m.indexingEndTime) {
|
|
2665
|
+
const duration = m.indexingEndTime - m.indexingStartTime;
|
|
2666
|
+
lines.push(`Indexing duration: ${(duration / 1e3).toFixed(2)}s`);
|
|
2667
|
+
}
|
|
2668
|
+
lines.push("");
|
|
2669
|
+
lines.push("Indexing:");
|
|
2670
|
+
lines.push(` Files scanned: ${m.filesScanned}`);
|
|
2671
|
+
lines.push(` Files parsed: ${m.filesParsed}`);
|
|
2672
|
+
lines.push(` Chunks processed: ${m.chunksProcessed}`);
|
|
2673
|
+
lines.push(` Chunks embedded: ${m.chunksEmbedded}`);
|
|
2674
|
+
lines.push(` Chunks from cache: ${m.chunksFromCache}`);
|
|
2675
|
+
lines.push(` Chunks removed: ${m.chunksRemoved}`);
|
|
2676
|
+
lines.push("");
|
|
2677
|
+
lines.push("Embedding API:");
|
|
2678
|
+
lines.push(` API calls: ${m.embeddingApiCalls}`);
|
|
2679
|
+
lines.push(` Tokens used: ${m.embeddingTokensUsed.toLocaleString()}`);
|
|
2680
|
+
lines.push(` Errors: ${m.embeddingErrors}`);
|
|
2681
|
+
if (m.searchCount > 0) {
|
|
2682
|
+
lines.push("");
|
|
2683
|
+
lines.push("Search:");
|
|
2684
|
+
lines.push(` Total searches: ${m.searchCount}`);
|
|
2685
|
+
lines.push(` Average time: ${m.searchAvgMs.toFixed(2)}ms`);
|
|
2686
|
+
lines.push(` Last search: ${m.searchLastMs.toFixed(2)}ms`);
|
|
2687
|
+
if (m.embeddingCallMs > 0) {
|
|
2688
|
+
lines.push(` - Embedding: ${m.embeddingCallMs.toFixed(2)}ms`);
|
|
2689
|
+
lines.push(` - Vector search: ${m.vectorSearchMs.toFixed(2)}ms`);
|
|
2690
|
+
lines.push(` - Keyword search: ${m.keywordSearchMs.toFixed(2)}ms`);
|
|
2691
|
+
lines.push(` - Fusion: ${m.fusionMs.toFixed(2)}ms`);
|
|
2692
|
+
}
|
|
2693
|
+
}
|
|
2694
|
+
const totalCacheOps = m.cacheHits + m.cacheMisses;
|
|
2695
|
+
if (totalCacheOps > 0) {
|
|
2696
|
+
lines.push("");
|
|
2697
|
+
lines.push("Cache:");
|
|
2698
|
+
lines.push(` Hits: ${m.cacheHits}`);
|
|
2699
|
+
lines.push(` Misses: ${m.cacheMisses}`);
|
|
2700
|
+
lines.push(` Hit rate: ${(m.cacheHits / totalCacheOps * 100).toFixed(1)}%`);
|
|
2701
|
+
}
|
|
2702
|
+
if (m.gcRuns > 0) {
|
|
2703
|
+
lines.push("");
|
|
2704
|
+
lines.push("Garbage Collection:");
|
|
2705
|
+
lines.push(` GC runs: ${m.gcRuns}`);
|
|
2706
|
+
lines.push(` Orphans removed: ${m.gcOrphansRemoved}`);
|
|
2707
|
+
lines.push(` Chunks removed: ${m.gcChunksRemoved}`);
|
|
2708
|
+
lines.push(` Embeddings removed: ${m.gcEmbeddingsRemoved}`);
|
|
2709
|
+
}
|
|
2710
|
+
return lines.join("\n");
|
|
2711
|
+
}
|
|
2712
|
+
formatRecentLogs(limit = 20) {
|
|
2713
|
+
const logs = this.getLogs(limit);
|
|
2714
|
+
if (logs.length === 0) {
|
|
2715
|
+
return "No logs recorded.";
|
|
2716
|
+
}
|
|
2717
|
+
return logs.map((l) => {
|
|
2718
|
+
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
2719
|
+
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
2720
|
+
}).join("\n");
|
|
2721
|
+
}
|
|
2722
|
+
isEnabled() {
|
|
2723
|
+
return this.config.enabled;
|
|
2724
|
+
}
|
|
2725
|
+
isMetricsEnabled() {
|
|
2726
|
+
return this.config.enabled && this.config.metrics;
|
|
2727
|
+
}
|
|
2728
|
+
};
|
|
2729
|
+
var globalLogger = null;
|
|
2730
|
+
function initializeLogger(config) {
|
|
2731
|
+
globalLogger = new Logger(config);
|
|
2732
|
+
return globalLogger;
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2735
|
+
// src/native/index.ts
|
|
2736
|
+
var path3 = __toESM(require("path"), 1);
|
|
2737
|
+
var os2 = __toESM(require("os"), 1);
|
|
2738
|
+
var import_url = require("url");
|
|
2739
|
+
var import_meta = {};
|
|
2740
|
+
function getNativeBinding() {
|
|
2741
|
+
const platform2 = os2.platform();
|
|
2742
|
+
const arch2 = os2.arch();
|
|
2743
|
+
let bindingName;
|
|
2744
|
+
if (platform2 === "darwin" && arch2 === "arm64") {
|
|
2745
|
+
bindingName = "codebase-index-native.darwin-arm64.node";
|
|
2746
|
+
} else if (platform2 === "darwin" && arch2 === "x64") {
|
|
2747
|
+
bindingName = "codebase-index-native.darwin-x64.node";
|
|
2748
|
+
} else if (platform2 === "linux" && arch2 === "x64") {
|
|
2749
|
+
bindingName = "codebase-index-native.linux-x64-gnu.node";
|
|
2750
|
+
} else if (platform2 === "linux" && arch2 === "arm64") {
|
|
2751
|
+
bindingName = "codebase-index-native.linux-arm64-gnu.node";
|
|
2752
|
+
} else if (platform2 === "win32" && arch2 === "x64") {
|
|
2753
|
+
bindingName = "codebase-index-native.win32-x64-msvc.node";
|
|
2754
|
+
} else {
|
|
2755
|
+
throw new Error(`Unsupported platform: ${platform2}-${arch2}`);
|
|
2756
|
+
}
|
|
2757
|
+
let currentDir;
|
|
2758
|
+
if (typeof import_meta !== "undefined" && import_meta.url) {
|
|
2759
|
+
currentDir = path3.dirname((0, import_url.fileURLToPath)(import_meta.url));
|
|
2760
|
+
} else if (typeof __dirname !== "undefined") {
|
|
2761
|
+
currentDir = __dirname;
|
|
2762
|
+
} else {
|
|
2763
|
+
currentDir = process.cwd();
|
|
2764
|
+
}
|
|
2765
|
+
const isDevMode = currentDir.includes("/src/native");
|
|
2766
|
+
const packageRoot = isDevMode ? path3.resolve(currentDir, "../..") : path3.resolve(currentDir, "..");
|
|
2767
|
+
const nativePath = path3.join(packageRoot, "native", bindingName);
|
|
2768
|
+
return require(nativePath);
|
|
2769
|
+
}
|
|
2770
|
+
var native = getNativeBinding();
|
|
2771
|
+
function parseFiles(files) {
|
|
2772
|
+
const result = native.parseFiles(files);
|
|
2773
|
+
return result.map((f) => ({
|
|
2774
|
+
path: f.path,
|
|
2775
|
+
chunks: f.chunks.map(mapChunk),
|
|
2776
|
+
hash: f.hash
|
|
2777
|
+
}));
|
|
2778
|
+
}
|
|
2779
|
+
function mapChunk(c) {
|
|
2780
|
+
return {
|
|
2781
|
+
content: c.content,
|
|
2782
|
+
startLine: c.startLine ?? c.start_line,
|
|
2783
|
+
endLine: c.endLine ?? c.end_line,
|
|
2784
|
+
chunkType: c.chunkType ?? c.chunk_type,
|
|
2785
|
+
name: c.name ?? void 0,
|
|
2786
|
+
language: c.language
|
|
2787
|
+
};
|
|
2788
|
+
}
|
|
2789
|
+
function hashContent(content) {
|
|
2790
|
+
return native.hashContent(content);
|
|
2791
|
+
}
|
|
2792
|
+
function hashFile(filePath) {
|
|
2793
|
+
return native.hashFile(filePath);
|
|
2794
|
+
}
|
|
2795
|
+
var VectorStore = class {
|
|
2796
|
+
inner;
|
|
2797
|
+
dimensions;
|
|
2798
|
+
constructor(indexPath, dimensions) {
|
|
2799
|
+
this.inner = new native.VectorStore(indexPath, dimensions);
|
|
2800
|
+
this.dimensions = dimensions;
|
|
2801
|
+
}
|
|
2802
|
+
add(id, vector, metadata) {
|
|
2803
|
+
if (vector.length !== this.dimensions) {
|
|
2804
|
+
throw new Error(
|
|
2805
|
+
`Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}`
|
|
2806
|
+
);
|
|
2807
|
+
}
|
|
2808
|
+
this.inner.add(id, vector, JSON.stringify(metadata));
|
|
2809
|
+
}
|
|
2810
|
+
addBatch(items) {
|
|
2811
|
+
const ids = items.map((i) => i.id);
|
|
2812
|
+
const vectors = items.map((i) => {
|
|
2813
|
+
if (i.vector.length !== this.dimensions) {
|
|
2814
|
+
throw new Error(
|
|
2815
|
+
`Vector dimension mismatch for ${i.id}: expected ${this.dimensions}, got ${i.vector.length}`
|
|
2816
|
+
);
|
|
2817
|
+
}
|
|
2818
|
+
return i.vector;
|
|
2819
|
+
});
|
|
2820
|
+
const metadata = items.map((i) => JSON.stringify(i.metadata));
|
|
2821
|
+
this.inner.addBatch(ids, vectors, metadata);
|
|
2822
|
+
}
|
|
2823
|
+
search(queryVector, limit = 10) {
|
|
2824
|
+
if (queryVector.length !== this.dimensions) {
|
|
2825
|
+
throw new Error(
|
|
2826
|
+
`Query vector dimension mismatch: expected ${this.dimensions}, got ${queryVector.length}`
|
|
2827
|
+
);
|
|
2828
|
+
}
|
|
2829
|
+
const results = this.inner.search(queryVector, limit);
|
|
2830
|
+
return results.map((r) => ({
|
|
2831
|
+
id: r.id,
|
|
2832
|
+
score: r.score,
|
|
2833
|
+
metadata: JSON.parse(r.metadata)
|
|
2834
|
+
}));
|
|
2835
|
+
}
|
|
2836
|
+
remove(id) {
|
|
2837
|
+
return this.inner.remove(id);
|
|
2838
|
+
}
|
|
2839
|
+
save() {
|
|
2840
|
+
this.inner.save();
|
|
2841
|
+
}
|
|
2842
|
+
load() {
|
|
2843
|
+
this.inner.load();
|
|
2844
|
+
}
|
|
2845
|
+
count() {
|
|
2846
|
+
return this.inner.count();
|
|
2847
|
+
}
|
|
2848
|
+
clear() {
|
|
2849
|
+
this.inner.clear();
|
|
2850
|
+
}
|
|
2851
|
+
getDimensions() {
|
|
2852
|
+
return this.dimensions;
|
|
2853
|
+
}
|
|
2854
|
+
getAllKeys() {
|
|
2855
|
+
return this.inner.getAllKeys();
|
|
2856
|
+
}
|
|
2857
|
+
getAllMetadata() {
|
|
2858
|
+
const results = this.inner.getAllMetadata();
|
|
2859
|
+
return results.map((r) => ({
|
|
2860
|
+
key: r.key,
|
|
2861
|
+
metadata: JSON.parse(r.metadata)
|
|
2862
|
+
}));
|
|
2863
|
+
}
|
|
2864
|
+
getMetadata(id) {
|
|
2865
|
+
const result = this.inner.getMetadata(id);
|
|
2866
|
+
if (result === null || result === void 0) {
|
|
2867
|
+
return void 0;
|
|
2868
|
+
}
|
|
2869
|
+
return JSON.parse(result);
|
|
2870
|
+
}
|
|
2871
|
+
getMetadataBatch(ids) {
|
|
2872
|
+
const results = this.inner.getMetadataBatch(ids);
|
|
2873
|
+
const map = /* @__PURE__ */ new Map();
|
|
2874
|
+
for (const { key, metadata } of results) {
|
|
2875
|
+
map.set(key, JSON.parse(metadata));
|
|
2876
|
+
}
|
|
2877
|
+
return map;
|
|
2878
|
+
}
|
|
2879
|
+
};
|
|
2880
|
+
var CHARS_PER_TOKEN = 4;
|
|
2881
|
+
var MAX_BATCH_TOKENS = 7500;
|
|
2882
|
+
var MAX_SINGLE_CHUNK_TOKENS = 2e3;
|
|
2883
|
+
function estimateTokens(text) {
|
|
2884
|
+
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
|
2885
|
+
}
|
|
2886
|
+
function createEmbeddingText(chunk, filePath) {
|
|
2887
|
+
const parts = [];
|
|
2888
|
+
const fileName = filePath.split("/").pop() || filePath;
|
|
2889
|
+
const dirPath = filePath.split("/").slice(-3, -1).join("/");
|
|
2890
|
+
const langDescriptors = {
|
|
2891
|
+
typescript: "TypeScript",
|
|
2892
|
+
javascript: "JavaScript",
|
|
2893
|
+
python: "Python",
|
|
2894
|
+
rust: "Rust",
|
|
2895
|
+
go: "Go",
|
|
2896
|
+
java: "Java"
|
|
2897
|
+
};
|
|
2898
|
+
const typeDescriptors = {
|
|
2899
|
+
function_declaration: "function",
|
|
2900
|
+
function: "function",
|
|
2901
|
+
arrow_function: "arrow function",
|
|
2902
|
+
method_definition: "method",
|
|
2903
|
+
class_declaration: "class",
|
|
2904
|
+
interface_declaration: "interface",
|
|
2905
|
+
type_alias_declaration: "type alias",
|
|
2906
|
+
enum_declaration: "enum",
|
|
2907
|
+
export_statement: "export",
|
|
2908
|
+
lexical_declaration: "variable declaration",
|
|
2909
|
+
function_definition: "function",
|
|
2910
|
+
class_definition: "class",
|
|
2911
|
+
function_item: "function",
|
|
2912
|
+
impl_item: "implementation",
|
|
2913
|
+
struct_item: "struct",
|
|
2914
|
+
enum_item: "enum",
|
|
2915
|
+
trait_item: "trait"
|
|
2916
|
+
};
|
|
2917
|
+
const lang = langDescriptors[chunk.language] || chunk.language;
|
|
2918
|
+
const typeDesc = typeDescriptors[chunk.chunkType] || chunk.chunkType;
|
|
2919
|
+
if (chunk.name) {
|
|
2920
|
+
parts.push(`${lang} ${typeDesc} "${chunk.name}"`);
|
|
2921
|
+
} else {
|
|
2922
|
+
parts.push(`${lang} ${typeDesc}`);
|
|
2923
|
+
}
|
|
2924
|
+
if (dirPath) {
|
|
2925
|
+
parts.push(`in ${dirPath}/${fileName}`);
|
|
2926
|
+
} else {
|
|
2927
|
+
parts.push(`in ${fileName}`);
|
|
2928
|
+
}
|
|
2929
|
+
const semanticHints = extractSemanticHints(chunk.name || "", chunk.content);
|
|
2930
|
+
if (semanticHints.length > 0) {
|
|
2931
|
+
parts.push(`Purpose: ${semanticHints.join(", ")}`);
|
|
2932
|
+
}
|
|
2933
|
+
parts.push("");
|
|
2934
|
+
let content = chunk.content;
|
|
2935
|
+
const headerLength = parts.join("\n").length;
|
|
2936
|
+
const maxContentChars = MAX_SINGLE_CHUNK_TOKENS * CHARS_PER_TOKEN - headerLength;
|
|
2937
|
+
if (content.length > maxContentChars) {
|
|
2938
|
+
content = content.slice(0, maxContentChars) + "\n... [truncated]";
|
|
2939
|
+
}
|
|
2940
|
+
parts.push(content);
|
|
2941
|
+
return parts.join("\n");
|
|
2942
|
+
}
|
|
2943
|
+
function createDynamicBatches(chunks) {
|
|
2944
|
+
const batches = [];
|
|
2945
|
+
let currentBatch = [];
|
|
2946
|
+
let currentTokens = 0;
|
|
2947
|
+
for (const chunk of chunks) {
|
|
2948
|
+
const chunkTokens = estimateTokens(chunk.text);
|
|
2949
|
+
if (currentBatch.length > 0 && currentTokens + chunkTokens > MAX_BATCH_TOKENS) {
|
|
2950
|
+
batches.push(currentBatch);
|
|
2951
|
+
currentBatch = [];
|
|
2952
|
+
currentTokens = 0;
|
|
2953
|
+
}
|
|
2954
|
+
currentBatch.push(chunk);
|
|
2955
|
+
currentTokens += chunkTokens;
|
|
2956
|
+
}
|
|
2957
|
+
if (currentBatch.length > 0) {
|
|
2958
|
+
batches.push(currentBatch);
|
|
2959
|
+
}
|
|
2960
|
+
return batches;
|
|
2961
|
+
}
|
|
2962
|
+
function extractSemanticHints(name, content) {
|
|
2963
|
+
const hints = [];
|
|
2964
|
+
const combined = `${name} ${content}`.toLowerCase();
|
|
2965
|
+
const signature = extractFunctionSignature(content);
|
|
2966
|
+
if (signature) {
|
|
2967
|
+
hints.push(signature);
|
|
2968
|
+
}
|
|
2969
|
+
const patterns = [
|
|
2970
|
+
[/auth|login|logout|signin|signout|credential/i, "authentication"],
|
|
2971
|
+
[/password|hash|bcrypt|argon/i, "password handling"],
|
|
2972
|
+
[/token|jwt|bearer|oauth/i, "token management"],
|
|
2973
|
+
[/user|account|profile|member/i, "user management"],
|
|
2974
|
+
[/permission|role|access|authorize/i, "authorization"],
|
|
2975
|
+
[/validate|verify|check|assert/i, "validation"],
|
|
2976
|
+
[/error|exception|throw|catch/i, "error handling"],
|
|
2977
|
+
[/log|debug|trace|info|warn/i, "logging"],
|
|
2978
|
+
[/cache|memoize|store/i, "caching"],
|
|
2979
|
+
[/fetch|request|response|api|http/i, "HTTP/API"],
|
|
2980
|
+
[/database|db|query|sql|mongo/i, "database"],
|
|
2981
|
+
[/file|read|write|stream|path/i, "file operations"],
|
|
2982
|
+
[/parse|serialize|json|xml/i, "data parsing"],
|
|
2983
|
+
[/encrypt|decrypt|crypto|secret|cipher|cryptographic/i, "encryption/cryptography"],
|
|
2984
|
+
[/test|spec|mock|stub|expect/i, "testing"],
|
|
2985
|
+
[/config|setting|option|env/i, "configuration"],
|
|
2986
|
+
[/route|endpoint|handler|controller|middleware/i, "routing/middleware"],
|
|
2987
|
+
[/render|component|view|template/i, "UI rendering"],
|
|
2988
|
+
[/state|redux|store|dispatch/i, "state management"],
|
|
2989
|
+
[/hook|effect|memo|callback/i, "React hooks"]
|
|
2990
|
+
];
|
|
2991
|
+
for (const [pattern, hint] of patterns) {
|
|
2992
|
+
if (pattern.test(combined) && !hints.includes(hint)) {
|
|
2993
|
+
hints.push(hint);
|
|
2994
|
+
}
|
|
2995
|
+
}
|
|
2996
|
+
return hints.slice(0, 6);
|
|
2997
|
+
}
|
|
2998
|
+
function extractFunctionSignature(content) {
|
|
2999
|
+
const tsJsPatterns = [
|
|
3000
|
+
/(?:export\s+)?(?:async\s+)?function\s+(\w+)\s*(?:<[^>]+>)?\s*\(([^)]*)\)\s*(?::\s*([^{]+))?/,
|
|
3001
|
+
/(?:export\s+)?const\s+(\w+)\s*(?::\s*[^=]+)?\s*=\s*(?:async\s+)?\(([^)]*)\)\s*(?::\s*([^=>{]+))?\s*=>/,
|
|
3002
|
+
/(?:export\s+)?const\s+(\w+)\s*(?::\s*[^=]+)?\s*=\s*(?:async\s+)?function\s*\(([^)]*)\)/
|
|
3003
|
+
];
|
|
3004
|
+
const pyPatterns = [
|
|
3005
|
+
/def\s+(\w+)\s*\(([^)]*)\)\s*(?:->\s*([^:]+))?:/,
|
|
3006
|
+
/async\s+def\s+(\w+)\s*\(([^)]*)\)\s*(?:->\s*([^:]+))?:/
|
|
3007
|
+
];
|
|
3008
|
+
const goPatterns = [
|
|
3009
|
+
/func\s+(?:\([^)]+\)\s+)?(\w+)\s*\(([^)]*)\)\s*(?:\(([^)]+)\)|([^{\n]+))?/
|
|
3010
|
+
];
|
|
3011
|
+
const rustPatterns = [
|
|
3012
|
+
/(?:pub\s+)?(?:async\s+)?fn\s+(\w+)\s*(?:<[^>]+>)?\s*\(([^)]*)\)\s*(?:->\s*([^{]+))?/
|
|
3013
|
+
];
|
|
3014
|
+
for (const pattern of [...tsJsPatterns, ...pyPatterns, ...goPatterns, ...rustPatterns]) {
|
|
3015
|
+
const match = content.match(pattern);
|
|
3016
|
+
if (match) {
|
|
3017
|
+
const funcName = match[1];
|
|
3018
|
+
const params = match[2]?.trim() || "";
|
|
3019
|
+
const returnType = (match[3] || match[4])?.trim();
|
|
3020
|
+
const paramNames = extractParamNames(params);
|
|
3021
|
+
let sig = `${funcName}(${paramNames.join(", ")})`;
|
|
3022
|
+
if (returnType && returnType.length < 50) {
|
|
3023
|
+
sig += ` -> ${returnType.replace(/\s+/g, " ").trim()}`;
|
|
3024
|
+
}
|
|
3025
|
+
if (sig.length < 100) {
|
|
3026
|
+
return sig;
|
|
3027
|
+
}
|
|
3028
|
+
}
|
|
3029
|
+
}
|
|
3030
|
+
return null;
|
|
3031
|
+
}
|
|
3032
|
+
function extractParamNames(params) {
|
|
3033
|
+
if (!params.trim()) return [];
|
|
3034
|
+
const names = [];
|
|
3035
|
+
const parts = params.split(",");
|
|
3036
|
+
for (const part of parts) {
|
|
3037
|
+
const trimmed = part.trim();
|
|
3038
|
+
if (!trimmed) continue;
|
|
3039
|
+
const tsMatch = trimmed.match(/^(\w+)\s*[?:]?/);
|
|
3040
|
+
const pyMatch = trimmed.match(/^(\w+)\s*(?::|=)/);
|
|
3041
|
+
const goMatch = trimmed.match(/^(\w+)\s+\w/);
|
|
3042
|
+
const rustMatch = trimmed.match(/^(\w+)\s*:/);
|
|
3043
|
+
const match = tsMatch || pyMatch || goMatch || rustMatch;
|
|
3044
|
+
if (match && match[1] !== "self" && match[1] !== "this") {
|
|
3045
|
+
names.push(match[1]);
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
return names.slice(0, 5);
|
|
3049
|
+
}
|
|
3050
|
+
function generateChunkId(filePath, chunk) {
|
|
3051
|
+
const hash = hashContent(`${filePath}:${chunk.startLine}:${chunk.endLine}:${chunk.content}`);
|
|
3052
|
+
return `chunk_${hash.slice(0, 16)}`;
|
|
3053
|
+
}
|
|
3054
|
+
function generateChunkHash(chunk) {
|
|
3055
|
+
return hashContent(chunk.content);
|
|
3056
|
+
}
|
|
3057
|
+
var InvertedIndex = class {
|
|
3058
|
+
inner;
|
|
3059
|
+
constructor(indexPath) {
|
|
3060
|
+
this.inner = new native.InvertedIndex(indexPath);
|
|
3061
|
+
}
|
|
3062
|
+
load() {
|
|
3063
|
+
this.inner.load();
|
|
3064
|
+
}
|
|
3065
|
+
save() {
|
|
3066
|
+
this.inner.save();
|
|
3067
|
+
}
|
|
3068
|
+
addChunk(chunkId, content) {
|
|
3069
|
+
this.inner.addChunk(chunkId, content);
|
|
3070
|
+
}
|
|
3071
|
+
removeChunk(chunkId) {
|
|
3072
|
+
return this.inner.removeChunk(chunkId);
|
|
3073
|
+
}
|
|
3074
|
+
search(query, limit) {
|
|
3075
|
+
const results = this.inner.search(query, limit ?? 100);
|
|
3076
|
+
const map = /* @__PURE__ */ new Map();
|
|
3077
|
+
for (const r of results) {
|
|
3078
|
+
map.set(r.chunkId, r.score);
|
|
3079
|
+
}
|
|
3080
|
+
return map;
|
|
3081
|
+
}
|
|
3082
|
+
hasChunk(chunkId) {
|
|
3083
|
+
return this.inner.hasChunk(chunkId);
|
|
3084
|
+
}
|
|
3085
|
+
clear() {
|
|
3086
|
+
this.inner.clear();
|
|
3087
|
+
}
|
|
3088
|
+
getDocumentCount() {
|
|
3089
|
+
return this.inner.documentCount();
|
|
3090
|
+
}
|
|
3091
|
+
};
|
|
3092
|
+
var Database = class {
|
|
3093
|
+
inner;
|
|
3094
|
+
constructor(dbPath) {
|
|
3095
|
+
this.inner = new native.Database(dbPath);
|
|
3096
|
+
}
|
|
3097
|
+
embeddingExists(contentHash) {
|
|
3098
|
+
return this.inner.embeddingExists(contentHash);
|
|
3099
|
+
}
|
|
3100
|
+
getEmbedding(contentHash) {
|
|
3101
|
+
return this.inner.getEmbedding(contentHash) ?? null;
|
|
3102
|
+
}
|
|
3103
|
+
upsertEmbedding(contentHash, embedding, chunkText, model) {
|
|
3104
|
+
this.inner.upsertEmbedding(contentHash, embedding, chunkText, model);
|
|
3105
|
+
}
|
|
3106
|
+
upsertEmbeddingsBatch(items) {
|
|
3107
|
+
if (items.length === 0) return;
|
|
3108
|
+
this.inner.upsertEmbeddingsBatch(items);
|
|
3109
|
+
}
|
|
3110
|
+
getMissingEmbeddings(contentHashes) {
|
|
3111
|
+
return this.inner.getMissingEmbeddings(contentHashes);
|
|
3112
|
+
}
|
|
3113
|
+
upsertChunk(chunk) {
|
|
3114
|
+
this.inner.upsertChunk(chunk);
|
|
3115
|
+
}
|
|
3116
|
+
upsertChunksBatch(chunks) {
|
|
3117
|
+
if (chunks.length === 0) return;
|
|
3118
|
+
this.inner.upsertChunksBatch(chunks);
|
|
3119
|
+
}
|
|
3120
|
+
getChunk(chunkId) {
|
|
3121
|
+
return this.inner.getChunk(chunkId) ?? null;
|
|
3122
|
+
}
|
|
3123
|
+
getChunksByFile(filePath) {
|
|
3124
|
+
return this.inner.getChunksByFile(filePath);
|
|
3125
|
+
}
|
|
3126
|
+
deleteChunksByFile(filePath) {
|
|
3127
|
+
return this.inner.deleteChunksByFile(filePath);
|
|
3128
|
+
}
|
|
3129
|
+
addChunksToBranch(branch, chunkIds) {
|
|
3130
|
+
this.inner.addChunksToBranch(branch, chunkIds);
|
|
3131
|
+
}
|
|
3132
|
+
addChunksToBranchBatch(branch, chunkIds) {
|
|
3133
|
+
if (chunkIds.length === 0) return;
|
|
3134
|
+
this.inner.addChunksToBranchBatch(branch, chunkIds);
|
|
3135
|
+
}
|
|
3136
|
+
clearBranch(branch) {
|
|
3137
|
+
return this.inner.clearBranch(branch);
|
|
3138
|
+
}
|
|
3139
|
+
getBranchChunkIds(branch) {
|
|
3140
|
+
return this.inner.getBranchChunkIds(branch);
|
|
3141
|
+
}
|
|
3142
|
+
getBranchDelta(branch, baseBranch) {
|
|
3143
|
+
return this.inner.getBranchDelta(branch, baseBranch);
|
|
3144
|
+
}
|
|
3145
|
+
chunkExistsOnBranch(branch, chunkId) {
|
|
3146
|
+
return this.inner.chunkExistsOnBranch(branch, chunkId);
|
|
3147
|
+
}
|
|
3148
|
+
getAllBranches() {
|
|
3149
|
+
return this.inner.getAllBranches();
|
|
3150
|
+
}
|
|
3151
|
+
getMetadata(key) {
|
|
3152
|
+
return this.inner.getMetadata(key) ?? null;
|
|
3153
|
+
}
|
|
3154
|
+
setMetadata(key, value) {
|
|
3155
|
+
this.inner.setMetadata(key, value);
|
|
3156
|
+
}
|
|
3157
|
+
deleteMetadata(key) {
|
|
3158
|
+
return this.inner.deleteMetadata(key);
|
|
3159
|
+
}
|
|
3160
|
+
gcOrphanEmbeddings() {
|
|
3161
|
+
return this.inner.gcOrphanEmbeddings();
|
|
3162
|
+
}
|
|
3163
|
+
gcOrphanChunks() {
|
|
3164
|
+
return this.inner.gcOrphanChunks();
|
|
3165
|
+
}
|
|
3166
|
+
getStats() {
|
|
3167
|
+
return this.inner.getStats();
|
|
3168
|
+
}
|
|
3169
|
+
};
|
|
3170
|
+
|
|
3171
|
+
// src/git/index.ts
|
|
3172
|
+
var import_fs3 = require("fs");
|
|
3173
|
+
var path4 = __toESM(require("path"), 1);
|
|
3174
|
+
var import_child_process = require("child_process");
|
|
3175
|
+
function resolveGitDir(repoRoot) {
|
|
3176
|
+
const gitPath = path4.join(repoRoot, ".git");
|
|
3177
|
+
if (!(0, import_fs3.existsSync)(gitPath)) {
|
|
3178
|
+
return null;
|
|
3179
|
+
}
|
|
3180
|
+
try {
|
|
3181
|
+
const stat = (0, import_fs3.statSync)(gitPath);
|
|
3182
|
+
if (stat.isDirectory()) {
|
|
3183
|
+
return gitPath;
|
|
3184
|
+
}
|
|
3185
|
+
if (stat.isFile()) {
|
|
3186
|
+
const content = (0, import_fs3.readFileSync)(gitPath, "utf-8").trim();
|
|
3187
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
3188
|
+
if (match) {
|
|
3189
|
+
const gitdir = match[1];
|
|
3190
|
+
const resolvedPath = path4.isAbsolute(gitdir) ? gitdir : path4.resolve(repoRoot, gitdir);
|
|
3191
|
+
if ((0, import_fs3.existsSync)(resolvedPath)) {
|
|
3192
|
+
return resolvedPath;
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3195
|
+
}
|
|
3196
|
+
} catch {
|
|
3197
|
+
}
|
|
3198
|
+
return null;
|
|
3199
|
+
}
|
|
3200
|
+
function isGitRepo(dir) {
|
|
3201
|
+
return resolveGitDir(dir) !== null;
|
|
3202
|
+
}
|
|
3203
|
+
function getCurrentBranch(repoRoot) {
|
|
3204
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3205
|
+
if (!gitDir) {
|
|
3206
|
+
return null;
|
|
3207
|
+
}
|
|
3208
|
+
const headPath = path4.join(gitDir, "HEAD");
|
|
3209
|
+
if (!(0, import_fs3.existsSync)(headPath)) {
|
|
3210
|
+
return null;
|
|
3211
|
+
}
|
|
3212
|
+
try {
|
|
3213
|
+
const headContent = (0, import_fs3.readFileSync)(headPath, "utf-8").trim();
|
|
3214
|
+
const match = headContent.match(/^ref: refs\/heads\/(.+)$/);
|
|
3215
|
+
if (match) {
|
|
3216
|
+
return match[1];
|
|
3217
|
+
}
|
|
3218
|
+
if (/^[0-9a-f]{40}$/i.test(headContent)) {
|
|
3219
|
+
return headContent.slice(0, 7);
|
|
3220
|
+
}
|
|
3221
|
+
return null;
|
|
3222
|
+
} catch {
|
|
3223
|
+
return null;
|
|
3224
|
+
}
|
|
3225
|
+
}
|
|
3226
|
+
function getBaseBranch(repoRoot) {
|
|
3227
|
+
const gitDir = resolveGitDir(repoRoot);
|
|
3228
|
+
const candidates = ["main", "master", "develop", "trunk"];
|
|
3229
|
+
if (gitDir) {
|
|
3230
|
+
for (const candidate of candidates) {
|
|
3231
|
+
const refPath = path4.join(gitDir, "refs", "heads", candidate);
|
|
3232
|
+
if ((0, import_fs3.existsSync)(refPath)) {
|
|
3233
|
+
return candidate;
|
|
3234
|
+
}
|
|
3235
|
+
const packedRefsPath = path4.join(gitDir, "packed-refs");
|
|
3236
|
+
if ((0, import_fs3.existsSync)(packedRefsPath)) {
|
|
3237
|
+
try {
|
|
3238
|
+
const content = (0, import_fs3.readFileSync)(packedRefsPath, "utf-8");
|
|
3239
|
+
if (content.includes(`refs/heads/${candidate}`)) {
|
|
3240
|
+
return candidate;
|
|
3241
|
+
}
|
|
3242
|
+
} catch {
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
3247
|
+
try {
|
|
3248
|
+
const result = (0, import_child_process.execSync)("git remote show origin", {
|
|
3249
|
+
cwd: repoRoot,
|
|
3250
|
+
encoding: "utf-8",
|
|
3251
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
3252
|
+
});
|
|
3253
|
+
const match = result.match(/HEAD branch: (.+)/);
|
|
3254
|
+
if (match) {
|
|
3255
|
+
return match[1].trim();
|
|
3256
|
+
}
|
|
3257
|
+
} catch {
|
|
3258
|
+
}
|
|
3259
|
+
return getCurrentBranch(repoRoot) ?? "main";
|
|
3260
|
+
}
|
|
3261
|
+
function getBranchOrDefault(repoRoot) {
|
|
3262
|
+
if (!isGitRepo(repoRoot)) {
|
|
3263
|
+
return "default";
|
|
3264
|
+
}
|
|
3265
|
+
return getCurrentBranch(repoRoot) ?? "default";
|
|
3266
|
+
}
|
|
3267
|
+
|
|
3268
|
+
// src/indexer/index.ts
|
|
3269
|
+
function float32ArrayToBuffer(arr) {
|
|
3270
|
+
const float32 = new Float32Array(arr);
|
|
3271
|
+
return Buffer.from(float32.buffer);
|
|
3272
|
+
}
|
|
3273
|
+
function bufferToFloat32Array(buf) {
|
|
3274
|
+
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
3275
|
+
}
|
|
3276
|
+
function getErrorMessage(error) {
|
|
3277
|
+
if (error instanceof Error) {
|
|
3278
|
+
return error.message;
|
|
3279
|
+
}
|
|
3280
|
+
if (typeof error === "string") {
|
|
3281
|
+
return error;
|
|
3282
|
+
}
|
|
3283
|
+
if (error && typeof error === "object" && "message" in error) {
|
|
3284
|
+
return String(error.message);
|
|
3285
|
+
}
|
|
3286
|
+
return String(error);
|
|
3287
|
+
}
|
|
3288
|
+
function isRateLimitError(error) {
|
|
3289
|
+
const message = getErrorMessage(error);
|
|
3290
|
+
return message.includes("429") || message.toLowerCase().includes("rate limit") || message.toLowerCase().includes("too many requests");
|
|
3291
|
+
}
|
|
3292
|
+
var INDEX_METADATA_VERSION = "1";
|
|
3293
|
+
var Indexer = class {
|
|
3294
|
+
config;
|
|
3295
|
+
projectRoot;
|
|
3296
|
+
indexPath;
|
|
3297
|
+
store = null;
|
|
3298
|
+
invertedIndex = null;
|
|
3299
|
+
database = null;
|
|
3300
|
+
provider = null;
|
|
3301
|
+
configuredProviderInfo = null;
|
|
3302
|
+
fileHashCache = /* @__PURE__ */ new Map();
|
|
3303
|
+
fileHashCachePath = "";
|
|
3304
|
+
failedBatchesPath = "";
|
|
3305
|
+
currentBranch = "default";
|
|
3306
|
+
baseBranch = "main";
|
|
3307
|
+
logger;
|
|
3308
|
+
queryEmbeddingCache = /* @__PURE__ */ new Map();
|
|
3309
|
+
maxQueryCacheSize = 100;
|
|
3310
|
+
queryCacheTtlMs = 5 * 60 * 1e3;
|
|
3311
|
+
querySimilarityThreshold = 0.85;
|
|
3312
|
+
indexCompatibility = null;
|
|
3313
|
+
indexingLockPath = "";
|
|
3314
|
+
constructor(projectRoot, config) {
|
|
3315
|
+
this.projectRoot = projectRoot;
|
|
3316
|
+
this.config = config;
|
|
3317
|
+
this.indexPath = this.getIndexPath();
|
|
3318
|
+
this.fileHashCachePath = path5.join(this.indexPath, "file-hashes.json");
|
|
3319
|
+
this.failedBatchesPath = path5.join(this.indexPath, "failed-batches.json");
|
|
3320
|
+
this.indexingLockPath = path5.join(this.indexPath, "indexing.lock");
|
|
3321
|
+
this.logger = initializeLogger(config.debug);
|
|
3322
|
+
}
|
|
3323
|
+
getIndexPath() {
|
|
3324
|
+
if (this.config.scope === "global") {
|
|
3325
|
+
const homeDir = process.env.HOME || process.env.USERPROFILE || "";
|
|
3326
|
+
return path5.join(homeDir, ".opencode", "global-index");
|
|
3327
|
+
}
|
|
3328
|
+
return path5.join(this.projectRoot, ".opencode", "index");
|
|
3329
|
+
}
|
|
3330
|
+
loadFileHashCache() {
|
|
3331
|
+
try {
|
|
3332
|
+
if ((0, import_fs4.existsSync)(this.fileHashCachePath)) {
|
|
3333
|
+
const data = (0, import_fs4.readFileSync)(this.fileHashCachePath, "utf-8");
|
|
3334
|
+
const parsed = JSON.parse(data);
|
|
3335
|
+
this.fileHashCache = new Map(Object.entries(parsed));
|
|
3336
|
+
}
|
|
3337
|
+
} catch {
|
|
3338
|
+
this.fileHashCache = /* @__PURE__ */ new Map();
|
|
3339
|
+
}
|
|
3340
|
+
}
|
|
3341
|
+
saveFileHashCache() {
|
|
3342
|
+
const obj = {};
|
|
3343
|
+
for (const [k, v] of this.fileHashCache) {
|
|
3344
|
+
obj[k] = v;
|
|
3345
|
+
}
|
|
3346
|
+
this.atomicWriteSync(this.fileHashCachePath, JSON.stringify(obj));
|
|
3347
|
+
}
|
|
3348
|
+
atomicWriteSync(targetPath, data) {
|
|
3349
|
+
const tempPath = `${targetPath}.tmp`;
|
|
3350
|
+
(0, import_fs4.writeFileSync)(tempPath, data);
|
|
3351
|
+
(0, import_fs4.renameSync)(tempPath, targetPath);
|
|
3352
|
+
}
|
|
3353
|
+
checkForInterruptedIndexing() {
|
|
3354
|
+
return (0, import_fs4.existsSync)(this.indexingLockPath);
|
|
3355
|
+
}
|
|
3356
|
+
acquireIndexingLock() {
|
|
3357
|
+
const lockData = {
|
|
3358
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
3359
|
+
pid: process.pid
|
|
3360
|
+
};
|
|
3361
|
+
(0, import_fs4.writeFileSync)(this.indexingLockPath, JSON.stringify(lockData));
|
|
3362
|
+
}
|
|
3363
|
+
releaseIndexingLock() {
|
|
3364
|
+
if ((0, import_fs4.existsSync)(this.indexingLockPath)) {
|
|
3365
|
+
(0, import_fs4.unlinkSync)(this.indexingLockPath);
|
|
3366
|
+
}
|
|
3367
|
+
}
|
|
3368
|
+
async recoverFromInterruptedIndexing() {
|
|
3369
|
+
this.logger.warn("Detected interrupted indexing session, recovering...");
|
|
3370
|
+
if ((0, import_fs4.existsSync)(this.fileHashCachePath)) {
|
|
3371
|
+
(0, import_fs4.unlinkSync)(this.fileHashCachePath);
|
|
3372
|
+
}
|
|
3373
|
+
await this.healthCheck();
|
|
3374
|
+
this.releaseIndexingLock();
|
|
3375
|
+
this.logger.info("Recovery complete, next index will re-process all files");
|
|
3376
|
+
}
|
|
3377
|
+
loadFailedBatches() {
|
|
3378
|
+
try {
|
|
3379
|
+
if ((0, import_fs4.existsSync)(this.failedBatchesPath)) {
|
|
3380
|
+
const data = (0, import_fs4.readFileSync)(this.failedBatchesPath, "utf-8");
|
|
3381
|
+
return JSON.parse(data);
|
|
3382
|
+
}
|
|
3383
|
+
} catch {
|
|
3384
|
+
return [];
|
|
3385
|
+
}
|
|
3386
|
+
return [];
|
|
3387
|
+
}
|
|
3388
|
+
saveFailedBatches(batches) {
|
|
3389
|
+
if (batches.length === 0) {
|
|
3390
|
+
if ((0, import_fs4.existsSync)(this.failedBatchesPath)) {
|
|
3391
|
+
import_fs4.promises.unlink(this.failedBatchesPath).catch(() => {
|
|
3392
|
+
});
|
|
3393
|
+
}
|
|
3394
|
+
return;
|
|
3395
|
+
}
|
|
3396
|
+
(0, import_fs4.writeFileSync)(this.failedBatchesPath, JSON.stringify(batches, null, 2));
|
|
3397
|
+
}
|
|
3398
|
+
addFailedBatch(batch, error) {
|
|
3399
|
+
const existing = this.loadFailedBatches();
|
|
3400
|
+
existing.push({
|
|
3401
|
+
chunks: batch,
|
|
3402
|
+
error,
|
|
3403
|
+
attemptCount: 1,
|
|
3404
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString()
|
|
3405
|
+
});
|
|
3406
|
+
this.saveFailedBatches(existing);
|
|
3407
|
+
}
|
|
3408
|
+
getProviderRateLimits(provider) {
|
|
3409
|
+
switch (provider) {
|
|
3410
|
+
case "github-copilot":
|
|
3411
|
+
return { concurrency: 1, intervalMs: 4e3, minRetryMs: 5e3, maxRetryMs: 6e4 };
|
|
3412
|
+
case "openai":
|
|
3413
|
+
return { concurrency: 3, intervalMs: 500, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3414
|
+
case "google":
|
|
3415
|
+
return { concurrency: 5, intervalMs: 200, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3416
|
+
case "ollama":
|
|
3417
|
+
return { concurrency: 5, intervalMs: 0, minRetryMs: 500, maxRetryMs: 5e3 };
|
|
3418
|
+
default:
|
|
3419
|
+
return { concurrency: 3, intervalMs: 1e3, minRetryMs: 1e3, maxRetryMs: 3e4 };
|
|
3420
|
+
}
|
|
3421
|
+
}
|
|
3422
|
+
async initialize() {
|
|
3423
|
+
if (this.config.embeddingProvider === "auto") {
|
|
3424
|
+
this.configuredProviderInfo = await tryDetectProvider();
|
|
3425
|
+
} else {
|
|
3426
|
+
this.configuredProviderInfo = await detectEmbeddingProvider(this.config.embeddingProvider, this.config.embeddingModel);
|
|
3427
|
+
}
|
|
3428
|
+
if (!this.configuredProviderInfo) {
|
|
3429
|
+
throw new Error(
|
|
3430
|
+
"No embedding provider available. Configure GitHub, OpenAI, Google, or Ollama."
|
|
3431
|
+
);
|
|
3432
|
+
}
|
|
3433
|
+
this.logger.info("Initializing indexer", {
|
|
3434
|
+
provider: this.configuredProviderInfo.provider,
|
|
3435
|
+
model: this.configuredProviderInfo.modelInfo.model,
|
|
3436
|
+
scope: this.config.scope
|
|
3437
|
+
});
|
|
3438
|
+
this.provider = createEmbeddingProvider(this.configuredProviderInfo);
|
|
3439
|
+
await import_fs4.promises.mkdir(this.indexPath, { recursive: true });
|
|
3440
|
+
if (this.checkForInterruptedIndexing()) {
|
|
3441
|
+
await this.recoverFromInterruptedIndexing();
|
|
3442
|
+
}
|
|
3443
|
+
const dimensions = this.configuredProviderInfo.modelInfo.dimensions;
|
|
3444
|
+
const storePath = path5.join(this.indexPath, "vectors");
|
|
3445
|
+
this.store = new VectorStore(storePath, dimensions);
|
|
3446
|
+
const indexFilePath = path5.join(this.indexPath, "vectors.usearch");
|
|
3447
|
+
if ((0, import_fs4.existsSync)(indexFilePath)) {
|
|
3448
|
+
this.store.load();
|
|
3449
|
+
}
|
|
3450
|
+
const invertedIndexPath = path5.join(this.indexPath, "inverted-index.json");
|
|
3451
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
3452
|
+
try {
|
|
3453
|
+
this.invertedIndex.load();
|
|
3454
|
+
} catch {
|
|
3455
|
+
if ((0, import_fs4.existsSync)(invertedIndexPath)) {
|
|
3456
|
+
await import_fs4.promises.unlink(invertedIndexPath);
|
|
3457
|
+
}
|
|
3458
|
+
this.invertedIndex = new InvertedIndex(invertedIndexPath);
|
|
3459
|
+
}
|
|
3460
|
+
const dbPath = path5.join(this.indexPath, "codebase.db");
|
|
3461
|
+
const dbIsNew = !(0, import_fs4.existsSync)(dbPath);
|
|
3462
|
+
this.database = new Database(dbPath);
|
|
3463
|
+
if (dbIsNew && this.store.count() > 0) {
|
|
3464
|
+
this.migrateFromLegacyIndex();
|
|
3465
|
+
}
|
|
3466
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3467
|
+
if (!this.indexCompatibility.compatible) {
|
|
3468
|
+
this.logger.warn("Index compatibility issue detected", {
|
|
3469
|
+
reason: this.indexCompatibility.reason,
|
|
3470
|
+
storedMetadata: this.indexCompatibility.storedMetadata,
|
|
3471
|
+
configuredProviderInfo: this.configuredProviderInfo
|
|
3472
|
+
});
|
|
3473
|
+
}
|
|
3474
|
+
if (isGitRepo(this.projectRoot)) {
|
|
3475
|
+
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
3476
|
+
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
3477
|
+
this.logger.branch("info", "Detected git repository", {
|
|
3478
|
+
currentBranch: this.currentBranch,
|
|
3479
|
+
baseBranch: this.baseBranch
|
|
3480
|
+
});
|
|
3481
|
+
} else {
|
|
3482
|
+
this.currentBranch = "default";
|
|
3483
|
+
this.baseBranch = "default";
|
|
3484
|
+
this.logger.branch("debug", "Not a git repository, using default branch");
|
|
3485
|
+
}
|
|
3486
|
+
if (this.config.indexing.autoGc) {
|
|
3487
|
+
await this.maybeRunAutoGc();
|
|
3488
|
+
}
|
|
3489
|
+
}
|
|
3490
|
+
async maybeRunAutoGc() {
|
|
3491
|
+
if (!this.database) return;
|
|
3492
|
+
const lastGcTimestamp = this.database.getMetadata("lastGcTimestamp");
|
|
3493
|
+
const now = Date.now();
|
|
3494
|
+
const intervalMs = this.config.indexing.gcIntervalDays * 24 * 60 * 60 * 1e3;
|
|
3495
|
+
let shouldRunGc = false;
|
|
3496
|
+
if (!lastGcTimestamp) {
|
|
3497
|
+
shouldRunGc = true;
|
|
3498
|
+
} else {
|
|
3499
|
+
const lastGcTime = parseInt(lastGcTimestamp, 10);
|
|
3500
|
+
if (!isNaN(lastGcTime) && now - lastGcTime > intervalMs) {
|
|
3501
|
+
shouldRunGc = true;
|
|
3502
|
+
}
|
|
3503
|
+
}
|
|
3504
|
+
if (shouldRunGc) {
|
|
3505
|
+
await this.healthCheck();
|
|
3506
|
+
this.database.setMetadata("lastGcTimestamp", now.toString());
|
|
3507
|
+
}
|
|
3508
|
+
}
|
|
3509
|
+
async maybeRunOrphanGc() {
|
|
3510
|
+
if (!this.database) return;
|
|
3511
|
+
const stats = this.database.getStats();
|
|
3512
|
+
if (!stats) return;
|
|
3513
|
+
const orphanCount = stats.embeddingCount - stats.chunkCount;
|
|
3514
|
+
if (orphanCount > this.config.indexing.gcOrphanThreshold) {
|
|
3515
|
+
this.database.gcOrphanEmbeddings();
|
|
3516
|
+
this.database.gcOrphanChunks();
|
|
3517
|
+
this.database.setMetadata("lastGcTimestamp", Date.now().toString());
|
|
3518
|
+
}
|
|
3519
|
+
}
|
|
3520
|
+
migrateFromLegacyIndex() {
|
|
3521
|
+
if (!this.store || !this.database) return;
|
|
3522
|
+
const allMetadata = this.store.getAllMetadata();
|
|
3523
|
+
const chunkIds = [];
|
|
3524
|
+
const chunkDataBatch = [];
|
|
3525
|
+
for (const { key, metadata } of allMetadata) {
|
|
3526
|
+
const chunkData = {
|
|
3527
|
+
chunkId: key,
|
|
3528
|
+
contentHash: metadata.hash,
|
|
3529
|
+
filePath: metadata.filePath,
|
|
3530
|
+
startLine: metadata.startLine,
|
|
3531
|
+
endLine: metadata.endLine,
|
|
3532
|
+
nodeType: metadata.chunkType,
|
|
3533
|
+
name: metadata.name,
|
|
3534
|
+
language: metadata.language
|
|
3535
|
+
};
|
|
3536
|
+
chunkDataBatch.push(chunkData);
|
|
3537
|
+
chunkIds.push(key);
|
|
3538
|
+
}
|
|
3539
|
+
if (chunkDataBatch.length > 0) {
|
|
3540
|
+
this.database.upsertChunksBatch(chunkDataBatch);
|
|
3541
|
+
}
|
|
3542
|
+
this.database.addChunksToBranchBatch(this.currentBranch || "default", chunkIds);
|
|
3543
|
+
}
|
|
3544
|
+
loadIndexMetadata() {
|
|
3545
|
+
if (!this.database) return null;
|
|
3546
|
+
const version = this.database.getMetadata("index.version");
|
|
3547
|
+
if (!version) return null;
|
|
3548
|
+
return {
|
|
3549
|
+
indexVersion: version,
|
|
3550
|
+
embeddingProvider: this.database.getMetadata("index.embeddingProvider") ?? "",
|
|
3551
|
+
embeddingModel: this.database.getMetadata("index.embeddingModel") ?? "",
|
|
3552
|
+
embeddingDimensions: parseInt(this.database.getMetadata("index.embeddingDimensions") ?? "0", 10),
|
|
3553
|
+
createdAt: this.database.getMetadata("index.createdAt") ?? "",
|
|
3554
|
+
updatedAt: this.database.getMetadata("index.updatedAt") ?? ""
|
|
3555
|
+
};
|
|
3556
|
+
}
|
|
3557
|
+
saveIndexMetadata(provider) {
|
|
3558
|
+
if (!this.database) return;
|
|
3559
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
3560
|
+
const existingCreatedAt = this.database.getMetadata("index.createdAt");
|
|
3561
|
+
this.database.setMetadata("index.version", INDEX_METADATA_VERSION);
|
|
3562
|
+
this.database.setMetadata("index.embeddingProvider", provider.provider);
|
|
3563
|
+
this.database.setMetadata("index.embeddingModel", provider.modelInfo.model);
|
|
3564
|
+
this.database.setMetadata("index.embeddingDimensions", provider.modelInfo.dimensions.toString());
|
|
3565
|
+
this.database.setMetadata("index.updatedAt", now);
|
|
3566
|
+
if (!existingCreatedAt) {
|
|
3567
|
+
this.database.setMetadata("index.createdAt", now);
|
|
3568
|
+
}
|
|
3569
|
+
}
|
|
3570
|
+
validateIndexCompatibility(provider) {
|
|
3571
|
+
const storedMetadata = this.loadIndexMetadata();
|
|
3572
|
+
if (!storedMetadata) {
|
|
3573
|
+
return { compatible: true };
|
|
3574
|
+
}
|
|
3575
|
+
const currentProvider = provider.provider;
|
|
3576
|
+
const currentModel = provider.modelInfo.model;
|
|
3577
|
+
const currentDimensions = provider.modelInfo.dimensions;
|
|
3578
|
+
if (storedMetadata.embeddingDimensions !== currentDimensions) {
|
|
3579
|
+
return {
|
|
3580
|
+
compatible: false,
|
|
3581
|
+
code: "DIMENSION_MISMATCH" /* DIMENSION_MISMATCH */,
|
|
3582
|
+
reason: `Dimension mismatch: index has ${storedMetadata.embeddingDimensions}D vectors (${storedMetadata.embeddingProvider}/${storedMetadata.embeddingModel}), but current provider uses ${currentDimensions}D (${currentProvider}/${currentModel}). Run index_codebase with force=true to rebuild.`,
|
|
3583
|
+
storedMetadata
|
|
3584
|
+
};
|
|
3585
|
+
}
|
|
3586
|
+
if (storedMetadata.embeddingModel !== currentModel) {
|
|
3587
|
+
return {
|
|
3588
|
+
compatible: false,
|
|
3589
|
+
code: "MODEL_MISMATCH" /* MODEL_MISMATCH */,
|
|
3590
|
+
reason: `Model mismatch: index was built with "${storedMetadata.embeddingModel}", but current model is "${currentModel}". Embeddings are incompatible. Run index_codebase with force=true to rebuild.`,
|
|
3591
|
+
storedMetadata
|
|
3592
|
+
};
|
|
3593
|
+
}
|
|
3594
|
+
if (storedMetadata.embeddingProvider !== currentProvider) {
|
|
3595
|
+
this.logger.warn("Provider changed", {
|
|
3596
|
+
storedProvider: storedMetadata.embeddingProvider,
|
|
3597
|
+
currentProvider
|
|
3598
|
+
});
|
|
3599
|
+
}
|
|
3600
|
+
return {
|
|
3601
|
+
compatible: true,
|
|
3602
|
+
storedMetadata
|
|
3603
|
+
};
|
|
3604
|
+
}
|
|
3605
|
+
checkCompatibility() {
|
|
3606
|
+
if (!this.indexCompatibility) {
|
|
3607
|
+
if (!this.configuredProviderInfo) {
|
|
3608
|
+
throw new Error("No embedding provider info, you must initialize the indexer first.");
|
|
3609
|
+
}
|
|
3610
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
3611
|
+
}
|
|
3612
|
+
return this.indexCompatibility;
|
|
3613
|
+
}
|
|
3614
|
+
async ensureInitialized() {
|
|
3615
|
+
if (!this.store || !this.provider || !this.invertedIndex || !this.configuredProviderInfo || !this.database) {
|
|
3616
|
+
await this.initialize();
|
|
3617
|
+
}
|
|
3618
|
+
return {
|
|
3619
|
+
store: this.store,
|
|
3620
|
+
provider: this.provider,
|
|
3621
|
+
invertedIndex: this.invertedIndex,
|
|
3622
|
+
configuredProviderInfo: this.configuredProviderInfo,
|
|
3623
|
+
database: this.database
|
|
3624
|
+
};
|
|
3625
|
+
}
|
|
3626
|
+
async estimateCost() {
|
|
3627
|
+
const { configuredProviderInfo } = await this.ensureInitialized();
|
|
3628
|
+
const { files } = await collectFiles(
|
|
3629
|
+
this.projectRoot,
|
|
3630
|
+
this.config.include,
|
|
3631
|
+
this.config.exclude,
|
|
3632
|
+
this.config.indexing.maxFileSize
|
|
3633
|
+
);
|
|
3634
|
+
return createCostEstimate(files, configuredProviderInfo);
|
|
3635
|
+
}
|
|
3636
|
+
async index(onProgress) {
|
|
3637
|
+
const { store, provider, invertedIndex, database, configuredProviderInfo } = await this.ensureInitialized();
|
|
3638
|
+
if (!this.indexCompatibility?.compatible) {
|
|
3639
|
+
throw new Error(
|
|
3640
|
+
`${this.indexCompatibility?.reason} Run index_codebase with force=true to rebuild the index.`
|
|
3641
|
+
);
|
|
3642
|
+
}
|
|
3643
|
+
this.acquireIndexingLock();
|
|
3644
|
+
this.logger.recordIndexingStart();
|
|
3645
|
+
this.logger.info("Starting indexing", { projectRoot: this.projectRoot });
|
|
3646
|
+
const startTime = Date.now();
|
|
3647
|
+
const stats = {
|
|
3648
|
+
totalFiles: 0,
|
|
3649
|
+
totalChunks: 0,
|
|
3650
|
+
indexedChunks: 0,
|
|
3651
|
+
failedChunks: 0,
|
|
3652
|
+
tokensUsed: 0,
|
|
3653
|
+
durationMs: 0,
|
|
3654
|
+
existingChunks: 0,
|
|
3655
|
+
removedChunks: 0,
|
|
3656
|
+
skippedFiles: [],
|
|
3657
|
+
parseFailures: []
|
|
3658
|
+
};
|
|
3659
|
+
onProgress?.({
|
|
3660
|
+
phase: "scanning",
|
|
3661
|
+
filesProcessed: 0,
|
|
3662
|
+
totalFiles: 0,
|
|
3663
|
+
chunksProcessed: 0,
|
|
3664
|
+
totalChunks: 0
|
|
3665
|
+
});
|
|
3666
|
+
this.loadFileHashCache();
|
|
3667
|
+
const { files, skipped } = await collectFiles(
|
|
3668
|
+
this.projectRoot,
|
|
3669
|
+
this.config.include,
|
|
3670
|
+
this.config.exclude,
|
|
3671
|
+
this.config.indexing.maxFileSize
|
|
3672
|
+
);
|
|
3673
|
+
stats.totalFiles = files.length;
|
|
3674
|
+
stats.skippedFiles = skipped;
|
|
3675
|
+
this.logger.recordFilesScanned(files.length);
|
|
3676
|
+
this.logger.cache("debug", "Scanning files for changes", {
|
|
3677
|
+
totalFiles: files.length,
|
|
3678
|
+
skippedFiles: skipped.length
|
|
3679
|
+
});
|
|
3680
|
+
const changedFiles = [];
|
|
3681
|
+
const unchangedFilePaths = /* @__PURE__ */ new Set();
|
|
3682
|
+
const currentFileHashes = /* @__PURE__ */ new Map();
|
|
3683
|
+
for (const f of files) {
|
|
3684
|
+
const currentHash = hashFile(f.path);
|
|
3685
|
+
currentFileHashes.set(f.path, currentHash);
|
|
3686
|
+
if (this.fileHashCache.get(f.path) === currentHash) {
|
|
3687
|
+
unchangedFilePaths.add(f.path);
|
|
3688
|
+
this.logger.recordCacheHit();
|
|
3689
|
+
} else {
|
|
3690
|
+
const content = await import_fs4.promises.readFile(f.path, "utf-8");
|
|
3691
|
+
changedFiles.push({ path: f.path, content, hash: currentHash });
|
|
3692
|
+
this.logger.recordCacheMiss();
|
|
3693
|
+
}
|
|
3694
|
+
}
|
|
3695
|
+
this.logger.cache("info", "File hash cache results", {
|
|
3696
|
+
unchanged: unchangedFilePaths.size,
|
|
3697
|
+
changed: changedFiles.length
|
|
3698
|
+
});
|
|
3699
|
+
onProgress?.({
|
|
3700
|
+
phase: "parsing",
|
|
3701
|
+
filesProcessed: 0,
|
|
3702
|
+
totalFiles: files.length,
|
|
3703
|
+
chunksProcessed: 0,
|
|
3704
|
+
totalChunks: 0
|
|
3705
|
+
});
|
|
3706
|
+
const parseStartTime = import_perf_hooks.performance.now();
|
|
3707
|
+
const parsedFiles = parseFiles(changedFiles);
|
|
3708
|
+
const parseMs = import_perf_hooks.performance.now() - parseStartTime;
|
|
3709
|
+
this.logger.recordFilesParsed(parsedFiles.length);
|
|
3710
|
+
this.logger.recordParseDuration(parseMs);
|
|
3711
|
+
this.logger.debug("Parsed changed files", { parsedCount: parsedFiles.length, parseMs: parseMs.toFixed(2) });
|
|
3712
|
+
const existingChunks = /* @__PURE__ */ new Map();
|
|
3713
|
+
const existingChunksByFile = /* @__PURE__ */ new Map();
|
|
3714
|
+
for (const { key, metadata } of store.getAllMetadata()) {
|
|
3715
|
+
existingChunks.set(key, metadata.hash);
|
|
3716
|
+
const fileChunks = existingChunksByFile.get(metadata.filePath) || /* @__PURE__ */ new Set();
|
|
3717
|
+
fileChunks.add(key);
|
|
3718
|
+
existingChunksByFile.set(metadata.filePath, fileChunks);
|
|
3719
|
+
}
|
|
3720
|
+
const currentChunkIds = /* @__PURE__ */ new Set();
|
|
3721
|
+
const currentFilePaths = /* @__PURE__ */ new Set();
|
|
3722
|
+
const pendingChunks = [];
|
|
3723
|
+
for (const filePath of unchangedFilePaths) {
|
|
3724
|
+
currentFilePaths.add(filePath);
|
|
3725
|
+
const fileChunks = existingChunksByFile.get(filePath);
|
|
3726
|
+
if (fileChunks) {
|
|
3727
|
+
for (const chunkId of fileChunks) {
|
|
3728
|
+
currentChunkIds.add(chunkId);
|
|
3729
|
+
}
|
|
3730
|
+
}
|
|
3731
|
+
}
|
|
3732
|
+
const chunkDataBatch = [];
|
|
3733
|
+
for (const parsed of parsedFiles) {
|
|
3734
|
+
currentFilePaths.add(parsed.path);
|
|
3735
|
+
if (parsed.chunks.length === 0) {
|
|
3736
|
+
const relativePath = path5.relative(this.projectRoot, parsed.path);
|
|
3737
|
+
stats.parseFailures.push(relativePath);
|
|
3738
|
+
}
|
|
3739
|
+
let fileChunkCount = 0;
|
|
3740
|
+
for (const chunk of parsed.chunks) {
|
|
3741
|
+
if (fileChunkCount >= this.config.indexing.maxChunksPerFile) {
|
|
3742
|
+
break;
|
|
3743
|
+
}
|
|
3744
|
+
if (this.config.indexing.semanticOnly && chunk.chunkType === "other") {
|
|
3745
|
+
continue;
|
|
3746
|
+
}
|
|
3747
|
+
const id = generateChunkId(parsed.path, chunk);
|
|
3748
|
+
const contentHash = generateChunkHash(chunk);
|
|
3749
|
+
currentChunkIds.add(id);
|
|
3750
|
+
chunkDataBatch.push({
|
|
3751
|
+
chunkId: id,
|
|
3752
|
+
contentHash,
|
|
3753
|
+
filePath: parsed.path,
|
|
3754
|
+
startLine: chunk.startLine,
|
|
3755
|
+
endLine: chunk.endLine,
|
|
3756
|
+
nodeType: chunk.chunkType,
|
|
3757
|
+
name: chunk.name,
|
|
3758
|
+
language: chunk.language
|
|
3759
|
+
});
|
|
3760
|
+
if (existingChunks.get(id) === contentHash) {
|
|
3761
|
+
fileChunkCount++;
|
|
3762
|
+
continue;
|
|
3763
|
+
}
|
|
3764
|
+
const text = createEmbeddingText(chunk, parsed.path);
|
|
3765
|
+
const metadata = {
|
|
3766
|
+
filePath: parsed.path,
|
|
3767
|
+
startLine: chunk.startLine,
|
|
3768
|
+
endLine: chunk.endLine,
|
|
3769
|
+
chunkType: chunk.chunkType,
|
|
3770
|
+
name: chunk.name,
|
|
3771
|
+
language: chunk.language,
|
|
3772
|
+
hash: contentHash
|
|
3773
|
+
};
|
|
3774
|
+
pendingChunks.push({ id, text, content: chunk.content, contentHash, metadata });
|
|
3775
|
+
fileChunkCount++;
|
|
3776
|
+
}
|
|
3777
|
+
}
|
|
3778
|
+
if (chunkDataBatch.length > 0) {
|
|
3779
|
+
database.upsertChunksBatch(chunkDataBatch);
|
|
3780
|
+
}
|
|
3781
|
+
let removedCount = 0;
|
|
3782
|
+
for (const [chunkId] of existingChunks) {
|
|
3783
|
+
if (!currentChunkIds.has(chunkId)) {
|
|
3784
|
+
store.remove(chunkId);
|
|
3785
|
+
invertedIndex.removeChunk(chunkId);
|
|
3786
|
+
removedCount++;
|
|
3787
|
+
}
|
|
3788
|
+
}
|
|
3789
|
+
stats.totalChunks = pendingChunks.length;
|
|
3790
|
+
stats.existingChunks = currentChunkIds.size - pendingChunks.length;
|
|
3791
|
+
stats.removedChunks = removedCount;
|
|
3792
|
+
this.logger.recordChunksProcessed(currentChunkIds.size);
|
|
3793
|
+
this.logger.recordChunksRemoved(removedCount);
|
|
3794
|
+
this.logger.info("Chunk analysis complete", {
|
|
3795
|
+
pending: pendingChunks.length,
|
|
3796
|
+
existing: stats.existingChunks,
|
|
3797
|
+
removed: removedCount
|
|
3798
|
+
});
|
|
3799
|
+
if (pendingChunks.length === 0 && removedCount === 0) {
|
|
3800
|
+
database.clearBranch(this.currentBranch);
|
|
3801
|
+
database.addChunksToBranchBatch(this.currentBranch, Array.from(currentChunkIds));
|
|
3802
|
+
this.fileHashCache = currentFileHashes;
|
|
3803
|
+
this.saveFileHashCache();
|
|
3804
|
+
stats.durationMs = Date.now() - startTime;
|
|
3805
|
+
onProgress?.({
|
|
3806
|
+
phase: "complete",
|
|
3807
|
+
filesProcessed: files.length,
|
|
3808
|
+
totalFiles: files.length,
|
|
3809
|
+
chunksProcessed: 0,
|
|
3810
|
+
totalChunks: 0
|
|
3811
|
+
});
|
|
3812
|
+
this.releaseIndexingLock();
|
|
3813
|
+
return stats;
|
|
3814
|
+
}
|
|
3815
|
+
if (pendingChunks.length === 0) {
|
|
3816
|
+
database.clearBranch(this.currentBranch);
|
|
3817
|
+
database.addChunksToBranchBatch(this.currentBranch, Array.from(currentChunkIds));
|
|
3818
|
+
store.save();
|
|
3819
|
+
invertedIndex.save();
|
|
3820
|
+
this.fileHashCache = currentFileHashes;
|
|
3821
|
+
this.saveFileHashCache();
|
|
3822
|
+
stats.durationMs = Date.now() - startTime;
|
|
3823
|
+
onProgress?.({
|
|
3824
|
+
phase: "complete",
|
|
3825
|
+
filesProcessed: files.length,
|
|
3826
|
+
totalFiles: files.length,
|
|
3827
|
+
chunksProcessed: 0,
|
|
3828
|
+
totalChunks: 0
|
|
3829
|
+
});
|
|
3830
|
+
this.releaseIndexingLock();
|
|
3831
|
+
return stats;
|
|
3832
|
+
}
|
|
3833
|
+
onProgress?.({
|
|
3834
|
+
phase: "embedding",
|
|
3835
|
+
filesProcessed: files.length,
|
|
3836
|
+
totalFiles: files.length,
|
|
3837
|
+
chunksProcessed: 0,
|
|
3838
|
+
totalChunks: pendingChunks.length
|
|
3839
|
+
});
|
|
3840
|
+
const allContentHashes = pendingChunks.map((c) => c.contentHash);
|
|
3841
|
+
const missingHashes = new Set(database.getMissingEmbeddings(allContentHashes));
|
|
3842
|
+
const chunksNeedingEmbedding = pendingChunks.filter((c) => missingHashes.has(c.contentHash));
|
|
3843
|
+
const chunksWithExistingEmbedding = pendingChunks.filter((c) => !missingHashes.has(c.contentHash));
|
|
3844
|
+
this.logger.cache("info", "Embedding cache lookup", {
|
|
3845
|
+
needsEmbedding: chunksNeedingEmbedding.length,
|
|
3846
|
+
fromCache: chunksWithExistingEmbedding.length
|
|
3847
|
+
});
|
|
3848
|
+
this.logger.recordChunksFromCache(chunksWithExistingEmbedding.length);
|
|
3849
|
+
for (const chunk of chunksWithExistingEmbedding) {
|
|
3850
|
+
const embeddingBuffer = database.getEmbedding(chunk.contentHash);
|
|
3851
|
+
if (embeddingBuffer) {
|
|
3852
|
+
const vector = bufferToFloat32Array(embeddingBuffer);
|
|
3853
|
+
store.add(chunk.id, Array.from(vector), chunk.metadata);
|
|
3854
|
+
invertedIndex.removeChunk(chunk.id);
|
|
3855
|
+
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
3856
|
+
stats.indexedChunks++;
|
|
3857
|
+
}
|
|
3858
|
+
}
|
|
3859
|
+
const providerRateLimits = this.getProviderRateLimits(configuredProviderInfo.provider);
|
|
3860
|
+
const queue = new PQueue({
|
|
3861
|
+
concurrency: providerRateLimits.concurrency,
|
|
3862
|
+
interval: providerRateLimits.intervalMs,
|
|
3863
|
+
intervalCap: providerRateLimits.concurrency
|
|
3864
|
+
});
|
|
3865
|
+
const dynamicBatches = createDynamicBatches(chunksNeedingEmbedding);
|
|
3866
|
+
let rateLimitBackoffMs = 0;
|
|
3867
|
+
for (const batch of dynamicBatches) {
|
|
3868
|
+
queue.add(async () => {
|
|
3869
|
+
if (rateLimitBackoffMs > 0) {
|
|
3870
|
+
await new Promise((resolve4) => setTimeout(resolve4, rateLimitBackoffMs));
|
|
3871
|
+
}
|
|
3872
|
+
try {
|
|
3873
|
+
const result = await pRetry(
|
|
3874
|
+
async () => {
|
|
3875
|
+
const texts = batch.map((c) => c.text);
|
|
3876
|
+
return provider.embedBatch(texts);
|
|
3877
|
+
},
|
|
3878
|
+
{
|
|
3879
|
+
retries: this.config.indexing.retries,
|
|
3880
|
+
minTimeout: Math.max(this.config.indexing.retryDelayMs, providerRateLimits.minRetryMs),
|
|
3881
|
+
maxTimeout: providerRateLimits.maxRetryMs,
|
|
3882
|
+
factor: 2,
|
|
3883
|
+
onFailedAttempt: (error) => {
|
|
3884
|
+
const message = getErrorMessage(error);
|
|
3885
|
+
if (isRateLimitError(error)) {
|
|
3886
|
+
rateLimitBackoffMs = Math.min(providerRateLimits.maxRetryMs, (rateLimitBackoffMs || providerRateLimits.minRetryMs) * 2);
|
|
3887
|
+
this.logger.embedding("warn", `Rate limited, backing off`, {
|
|
3888
|
+
attempt: error.attemptNumber,
|
|
3889
|
+
retriesLeft: error.retriesLeft,
|
|
3890
|
+
backoffMs: rateLimitBackoffMs
|
|
3891
|
+
});
|
|
3892
|
+
} else {
|
|
3893
|
+
this.logger.embedding("error", `Embedding batch failed`, {
|
|
3894
|
+
attempt: error.attemptNumber,
|
|
3895
|
+
error: message
|
|
3896
|
+
});
|
|
3897
|
+
}
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
);
|
|
3901
|
+
if (rateLimitBackoffMs > 0) {
|
|
3902
|
+
rateLimitBackoffMs = Math.max(0, rateLimitBackoffMs - 2e3);
|
|
3903
|
+
}
|
|
3904
|
+
const items = batch.map((chunk, idx) => ({
|
|
3905
|
+
id: chunk.id,
|
|
3906
|
+
vector: result.embeddings[idx],
|
|
3907
|
+
metadata: chunk.metadata
|
|
3908
|
+
}));
|
|
3909
|
+
store.addBatch(items);
|
|
3910
|
+
const embeddingBatchItems = batch.map((chunk, i) => ({
|
|
3911
|
+
contentHash: chunk.contentHash,
|
|
3912
|
+
embedding: float32ArrayToBuffer(result.embeddings[i]),
|
|
3913
|
+
chunkText: chunk.text,
|
|
3914
|
+
model: configuredProviderInfo.modelInfo.model
|
|
3915
|
+
}));
|
|
3916
|
+
database.upsertEmbeddingsBatch(embeddingBatchItems);
|
|
3917
|
+
for (const chunk of batch) {
|
|
3918
|
+
invertedIndex.removeChunk(chunk.id);
|
|
3919
|
+
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
3920
|
+
}
|
|
3921
|
+
stats.indexedChunks += batch.length;
|
|
3922
|
+
stats.tokensUsed += result.totalTokensUsed;
|
|
3923
|
+
this.logger.recordChunksEmbedded(batch.length);
|
|
3924
|
+
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
3925
|
+
this.logger.embedding("debug", `Embedded batch`, {
|
|
3926
|
+
batchSize: batch.length,
|
|
3927
|
+
tokens: result.totalTokensUsed
|
|
3928
|
+
});
|
|
3929
|
+
onProgress?.({
|
|
3930
|
+
phase: "embedding",
|
|
3931
|
+
filesProcessed: files.length,
|
|
3932
|
+
totalFiles: files.length,
|
|
3933
|
+
chunksProcessed: stats.indexedChunks,
|
|
3934
|
+
totalChunks: pendingChunks.length
|
|
3935
|
+
});
|
|
3936
|
+
} catch (error) {
|
|
3937
|
+
stats.failedChunks += batch.length;
|
|
3938
|
+
this.addFailedBatch(batch, getErrorMessage(error));
|
|
3939
|
+
this.logger.recordEmbeddingError();
|
|
3940
|
+
this.logger.embedding("error", `Failed to embed batch after retries`, {
|
|
3941
|
+
batchSize: batch.length,
|
|
3942
|
+
error: getErrorMessage(error)
|
|
3943
|
+
});
|
|
3944
|
+
}
|
|
3945
|
+
});
|
|
3946
|
+
}
|
|
3947
|
+
await queue.onIdle();
|
|
3948
|
+
onProgress?.({
|
|
3949
|
+
phase: "storing",
|
|
3950
|
+
filesProcessed: files.length,
|
|
3951
|
+
totalFiles: files.length,
|
|
3952
|
+
chunksProcessed: stats.indexedChunks,
|
|
3953
|
+
totalChunks: pendingChunks.length
|
|
3954
|
+
});
|
|
3955
|
+
database.clearBranch(this.currentBranch);
|
|
3956
|
+
database.addChunksToBranchBatch(this.currentBranch, Array.from(currentChunkIds));
|
|
3957
|
+
store.save();
|
|
3958
|
+
invertedIndex.save();
|
|
3959
|
+
this.fileHashCache = currentFileHashes;
|
|
3960
|
+
this.saveFileHashCache();
|
|
3961
|
+
if (this.config.indexing.autoGc && stats.removedChunks > 0) {
|
|
3962
|
+
await this.maybeRunOrphanGc();
|
|
3963
|
+
}
|
|
3964
|
+
stats.durationMs = Date.now() - startTime;
|
|
3965
|
+
this.saveIndexMetadata(configuredProviderInfo);
|
|
3966
|
+
this.indexCompatibility = { compatible: true };
|
|
3967
|
+
this.logger.recordIndexingEnd();
|
|
3968
|
+
this.logger.info("Indexing complete", {
|
|
3969
|
+
files: stats.totalFiles,
|
|
3970
|
+
indexed: stats.indexedChunks,
|
|
3971
|
+
existing: stats.existingChunks,
|
|
3972
|
+
removed: stats.removedChunks,
|
|
3973
|
+
failed: stats.failedChunks,
|
|
3974
|
+
tokens: stats.tokensUsed,
|
|
3975
|
+
durationMs: stats.durationMs
|
|
3976
|
+
});
|
|
3977
|
+
if (stats.failedChunks > 0) {
|
|
3978
|
+
stats.failedBatchesPath = this.failedBatchesPath;
|
|
3979
|
+
}
|
|
3980
|
+
onProgress?.({
|
|
3981
|
+
phase: "complete",
|
|
3982
|
+
filesProcessed: files.length,
|
|
3983
|
+
totalFiles: files.length,
|
|
3984
|
+
chunksProcessed: stats.indexedChunks,
|
|
3985
|
+
totalChunks: pendingChunks.length
|
|
3986
|
+
});
|
|
3987
|
+
this.releaseIndexingLock();
|
|
3988
|
+
return stats;
|
|
3989
|
+
}
|
|
3990
|
+
async getQueryEmbedding(query, provider) {
|
|
3991
|
+
const now = Date.now();
|
|
3992
|
+
const cached = this.queryEmbeddingCache.get(query);
|
|
3993
|
+
if (cached && now - cached.timestamp < this.queryCacheTtlMs) {
|
|
3994
|
+
this.logger.cache("debug", "Query embedding cache hit (exact)", { query: query.slice(0, 50) });
|
|
3995
|
+
this.logger.recordQueryCacheHit();
|
|
3996
|
+
return cached.embedding;
|
|
3997
|
+
}
|
|
3998
|
+
const similarMatch = this.findSimilarCachedQuery(query, now);
|
|
3999
|
+
if (similarMatch) {
|
|
4000
|
+
this.logger.cache("debug", "Query embedding cache hit (similar)", {
|
|
4001
|
+
query: query.slice(0, 50),
|
|
4002
|
+
similarTo: similarMatch.key.slice(0, 50),
|
|
4003
|
+
similarity: similarMatch.similarity.toFixed(3)
|
|
4004
|
+
});
|
|
4005
|
+
this.logger.recordQueryCacheSimilarHit();
|
|
4006
|
+
return similarMatch.embedding;
|
|
4007
|
+
}
|
|
4008
|
+
this.logger.cache("debug", "Query embedding cache miss", { query: query.slice(0, 50) });
|
|
4009
|
+
this.logger.recordQueryCacheMiss();
|
|
4010
|
+
const { embedding, tokensUsed } = await provider.embedQuery(query);
|
|
4011
|
+
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
4012
|
+
if (this.queryEmbeddingCache.size >= this.maxQueryCacheSize) {
|
|
4013
|
+
const oldestKey = this.queryEmbeddingCache.keys().next().value;
|
|
4014
|
+
if (oldestKey) {
|
|
4015
|
+
this.queryEmbeddingCache.delete(oldestKey);
|
|
4016
|
+
}
|
|
4017
|
+
}
|
|
4018
|
+
this.queryEmbeddingCache.set(query, { embedding, timestamp: now });
|
|
4019
|
+
return embedding;
|
|
4020
|
+
}
|
|
4021
|
+
findSimilarCachedQuery(query, now) {
|
|
4022
|
+
const queryTokens = this.tokenize(query);
|
|
4023
|
+
if (queryTokens.size === 0) return null;
|
|
4024
|
+
let bestMatch = null;
|
|
4025
|
+
for (const [cachedQuery, { embedding, timestamp }] of this.queryEmbeddingCache) {
|
|
4026
|
+
if (now - timestamp >= this.queryCacheTtlMs) continue;
|
|
4027
|
+
const cachedTokens = this.tokenize(cachedQuery);
|
|
4028
|
+
const similarity = this.jaccardSimilarity(queryTokens, cachedTokens);
|
|
4029
|
+
if (similarity >= this.querySimilarityThreshold) {
|
|
4030
|
+
if (!bestMatch || similarity > bestMatch.similarity) {
|
|
4031
|
+
bestMatch = { key: cachedQuery, embedding, similarity };
|
|
4032
|
+
}
|
|
4033
|
+
}
|
|
4034
|
+
}
|
|
4035
|
+
return bestMatch;
|
|
4036
|
+
}
|
|
4037
|
+
tokenize(text) {
|
|
4038
|
+
return new Set(
|
|
4039
|
+
text.toLowerCase().replace(/[^\w\s]/g, " ").split(/\s+/).filter((t) => t.length > 1)
|
|
4040
|
+
);
|
|
4041
|
+
}
|
|
4042
|
+
jaccardSimilarity(a, b) {
|
|
4043
|
+
if (a.size === 0 && b.size === 0) return 1;
|
|
4044
|
+
if (a.size === 0 || b.size === 0) return 0;
|
|
4045
|
+
let intersection = 0;
|
|
4046
|
+
for (const token of a) {
|
|
4047
|
+
if (b.has(token)) intersection++;
|
|
4048
|
+
}
|
|
4049
|
+
const union = a.size + b.size - intersection;
|
|
4050
|
+
return intersection / union;
|
|
4051
|
+
}
|
|
4052
|
+
async search(query, limit, options) {
|
|
4053
|
+
const { store, provider, database } = await this.ensureInitialized();
|
|
4054
|
+
const compatibility = this.checkCompatibility();
|
|
4055
|
+
if (!compatibility.compatible) {
|
|
4056
|
+
throw new Error(
|
|
4057
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} A possible solution is to run index_codebase with force=true to rebuild the index.`
|
|
4058
|
+
);
|
|
4059
|
+
}
|
|
4060
|
+
const searchStartTime = import_perf_hooks.performance.now();
|
|
4061
|
+
if (store.count() === 0) {
|
|
4062
|
+
this.logger.search("debug", "Search on empty index", { query });
|
|
4063
|
+
return [];
|
|
4064
|
+
}
|
|
4065
|
+
const maxResults = limit ?? this.config.search.maxResults;
|
|
4066
|
+
const hybridWeight = options?.hybridWeight ?? this.config.search.hybridWeight;
|
|
4067
|
+
const filterByBranch = options?.filterByBranch ?? true;
|
|
4068
|
+
this.logger.search("debug", "Starting search", {
|
|
4069
|
+
query,
|
|
4070
|
+
maxResults,
|
|
4071
|
+
hybridWeight,
|
|
4072
|
+
filterByBranch
|
|
4073
|
+
});
|
|
4074
|
+
const embeddingStartTime = import_perf_hooks.performance.now();
|
|
4075
|
+
const embedding = await this.getQueryEmbedding(query, provider);
|
|
4076
|
+
const embeddingMs = import_perf_hooks.performance.now() - embeddingStartTime;
|
|
4077
|
+
const vectorStartTime = import_perf_hooks.performance.now();
|
|
4078
|
+
const semanticResults = store.search(embedding, maxResults * 4);
|
|
4079
|
+
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
4080
|
+
const keywordStartTime = import_perf_hooks.performance.now();
|
|
4081
|
+
const keywordResults = await this.keywordSearch(query, maxResults * 4);
|
|
4082
|
+
const keywordMs = import_perf_hooks.performance.now() - keywordStartTime;
|
|
4083
|
+
const fusionStartTime = import_perf_hooks.performance.now();
|
|
4084
|
+
const combined = this.fuseResults(semanticResults, keywordResults, hybridWeight, maxResults * 4);
|
|
4085
|
+
const fusionMs = import_perf_hooks.performance.now() - fusionStartTime;
|
|
4086
|
+
let branchChunkIds = null;
|
|
4087
|
+
if (filterByBranch && this.currentBranch !== "default") {
|
|
4088
|
+
branchChunkIds = new Set(database.getBranchChunkIds(this.currentBranch));
|
|
4089
|
+
}
|
|
4090
|
+
const filtered = combined.filter((r) => {
|
|
4091
|
+
if (r.score < this.config.search.minScore) return false;
|
|
4092
|
+
if (branchChunkIds && !branchChunkIds.has(r.id)) return false;
|
|
4093
|
+
if (options?.fileType) {
|
|
4094
|
+
const ext = r.metadata.filePath.split(".").pop()?.toLowerCase();
|
|
4095
|
+
if (ext !== options.fileType.toLowerCase().replace(/^\./, "")) return false;
|
|
4096
|
+
}
|
|
4097
|
+
if (options?.directory) {
|
|
4098
|
+
const normalizedDir = options.directory.replace(/^\/|\/$/g, "");
|
|
4099
|
+
if (!r.metadata.filePath.includes(`/${normalizedDir}/`) && !r.metadata.filePath.includes(`${normalizedDir}/`)) return false;
|
|
4100
|
+
}
|
|
4101
|
+
if (options?.chunkType) {
|
|
4102
|
+
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
4103
|
+
}
|
|
4104
|
+
return true;
|
|
4105
|
+
}).slice(0, maxResults);
|
|
4106
|
+
const totalSearchMs = import_perf_hooks.performance.now() - searchStartTime;
|
|
4107
|
+
this.logger.recordSearch(totalSearchMs, {
|
|
4108
|
+
embeddingMs,
|
|
4109
|
+
vectorMs,
|
|
4110
|
+
keywordMs,
|
|
4111
|
+
fusionMs
|
|
4112
|
+
});
|
|
4113
|
+
this.logger.search("info", "Search complete", {
|
|
4114
|
+
query,
|
|
4115
|
+
results: filtered.length,
|
|
4116
|
+
totalMs: Math.round(totalSearchMs * 100) / 100,
|
|
4117
|
+
embeddingMs: Math.round(embeddingMs * 100) / 100,
|
|
4118
|
+
vectorMs: Math.round(vectorMs * 100) / 100,
|
|
4119
|
+
keywordMs: Math.round(keywordMs * 100) / 100,
|
|
4120
|
+
fusionMs: Math.round(fusionMs * 100) / 100
|
|
4121
|
+
});
|
|
4122
|
+
const metadataOnly = options?.metadataOnly ?? false;
|
|
4123
|
+
return Promise.all(
|
|
4124
|
+
filtered.map(async (r) => {
|
|
4125
|
+
let content = "";
|
|
4126
|
+
let contextStartLine = r.metadata.startLine;
|
|
4127
|
+
let contextEndLine = r.metadata.endLine;
|
|
4128
|
+
if (!metadataOnly && this.config.search.includeContext) {
|
|
4129
|
+
try {
|
|
4130
|
+
const fileContent = await import_fs4.promises.readFile(
|
|
4131
|
+
r.metadata.filePath,
|
|
4132
|
+
"utf-8"
|
|
4133
|
+
);
|
|
4134
|
+
const lines = fileContent.split("\n");
|
|
4135
|
+
const contextLines = options?.contextLines ?? this.config.search.contextLines;
|
|
4136
|
+
contextStartLine = Math.max(1, r.metadata.startLine - contextLines);
|
|
4137
|
+
contextEndLine = Math.min(lines.length, r.metadata.endLine + contextLines);
|
|
4138
|
+
content = lines.slice(contextStartLine - 1, contextEndLine).join("\n");
|
|
4139
|
+
} catch {
|
|
4140
|
+
content = "[File not accessible]";
|
|
4141
|
+
}
|
|
4142
|
+
}
|
|
4143
|
+
return {
|
|
4144
|
+
filePath: r.metadata.filePath,
|
|
4145
|
+
startLine: contextStartLine,
|
|
4146
|
+
endLine: contextEndLine,
|
|
4147
|
+
content,
|
|
4148
|
+
score: r.score,
|
|
4149
|
+
chunkType: r.metadata.chunkType,
|
|
4150
|
+
name: r.metadata.name
|
|
4151
|
+
};
|
|
4152
|
+
})
|
|
4153
|
+
);
|
|
4154
|
+
}
|
|
4155
|
+
async keywordSearch(query, limit) {
|
|
4156
|
+
const { store, invertedIndex } = await this.ensureInitialized();
|
|
4157
|
+
const scores = invertedIndex.search(query);
|
|
4158
|
+
if (scores.size === 0) {
|
|
4159
|
+
return [];
|
|
4160
|
+
}
|
|
4161
|
+
const chunkIds = Array.from(scores.keys());
|
|
4162
|
+
const metadataMap = store.getMetadataBatch(chunkIds);
|
|
4163
|
+
const results = [];
|
|
4164
|
+
for (const [chunkId, score] of scores) {
|
|
4165
|
+
const metadata = metadataMap.get(chunkId);
|
|
4166
|
+
if (metadata && score > 0) {
|
|
4167
|
+
results.push({ id: chunkId, score, metadata });
|
|
4168
|
+
}
|
|
4169
|
+
}
|
|
4170
|
+
results.sort((a, b) => b.score - a.score);
|
|
4171
|
+
return results.slice(0, limit);
|
|
4172
|
+
}
|
|
4173
|
+
fuseResults(semanticResults, keywordResults, keywordWeight, limit) {
|
|
4174
|
+
const semanticWeight = 1 - keywordWeight;
|
|
4175
|
+
const fusedScores = /* @__PURE__ */ new Map();
|
|
4176
|
+
for (const r of semanticResults) {
|
|
4177
|
+
fusedScores.set(r.id, {
|
|
4178
|
+
score: r.score * semanticWeight,
|
|
4179
|
+
metadata: r.metadata
|
|
4180
|
+
});
|
|
4181
|
+
}
|
|
4182
|
+
for (const r of keywordResults) {
|
|
4183
|
+
const existing = fusedScores.get(r.id);
|
|
4184
|
+
if (existing) {
|
|
4185
|
+
existing.score += r.score * keywordWeight;
|
|
4186
|
+
} else {
|
|
4187
|
+
fusedScores.set(r.id, {
|
|
4188
|
+
score: r.score * keywordWeight,
|
|
4189
|
+
metadata: r.metadata
|
|
4190
|
+
});
|
|
4191
|
+
}
|
|
4192
|
+
}
|
|
4193
|
+
const results = Array.from(fusedScores.entries()).map(([id, data]) => ({
|
|
4194
|
+
id,
|
|
4195
|
+
score: data.score,
|
|
4196
|
+
metadata: data.metadata
|
|
4197
|
+
}));
|
|
4198
|
+
results.sort((a, b) => b.score - a.score);
|
|
4199
|
+
return results.slice(0, limit);
|
|
4200
|
+
}
|
|
4201
|
+
async getStatus() {
|
|
4202
|
+
const { store, configuredProviderInfo } = await this.ensureInitialized();
|
|
4203
|
+
return {
|
|
4204
|
+
indexed: store.count() > 0,
|
|
4205
|
+
vectorCount: store.count(),
|
|
4206
|
+
provider: configuredProviderInfo.provider,
|
|
4207
|
+
model: configuredProviderInfo.modelInfo.model,
|
|
4208
|
+
indexPath: this.indexPath,
|
|
4209
|
+
currentBranch: this.currentBranch,
|
|
4210
|
+
baseBranch: this.baseBranch,
|
|
4211
|
+
compatibility: this.indexCompatibility
|
|
4212
|
+
};
|
|
4213
|
+
}
|
|
4214
|
+
async clearIndex() {
|
|
4215
|
+
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
4216
|
+
store.clear();
|
|
4217
|
+
store.save();
|
|
4218
|
+
invertedIndex.clear();
|
|
4219
|
+
invertedIndex.save();
|
|
4220
|
+
this.fileHashCache.clear();
|
|
4221
|
+
this.saveFileHashCache();
|
|
4222
|
+
database.clearBranch(this.currentBranch);
|
|
4223
|
+
database.deleteMetadata("index.version");
|
|
4224
|
+
database.deleteMetadata("index.embeddingProvider");
|
|
4225
|
+
database.deleteMetadata("index.embeddingModel");
|
|
4226
|
+
database.deleteMetadata("index.embeddingDimensions");
|
|
4227
|
+
database.deleteMetadata("index.createdAt");
|
|
4228
|
+
database.deleteMetadata("index.updatedAt");
|
|
4229
|
+
this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo);
|
|
4230
|
+
}
|
|
4231
|
+
async healthCheck() {
|
|
4232
|
+
const { store, invertedIndex, database } = await this.ensureInitialized();
|
|
4233
|
+
this.logger.gc("info", "Starting health check");
|
|
4234
|
+
const allMetadata = store.getAllMetadata();
|
|
4235
|
+
const filePathsToChunkKeys = /* @__PURE__ */ new Map();
|
|
4236
|
+
for (const { key, metadata } of allMetadata) {
|
|
4237
|
+
const existing = filePathsToChunkKeys.get(metadata.filePath) || [];
|
|
4238
|
+
existing.push(key);
|
|
4239
|
+
filePathsToChunkKeys.set(metadata.filePath, existing);
|
|
4240
|
+
}
|
|
4241
|
+
const removedFilePaths = [];
|
|
4242
|
+
let removedCount = 0;
|
|
4243
|
+
for (const [filePath, chunkKeys] of filePathsToChunkKeys) {
|
|
4244
|
+
if (!(0, import_fs4.existsSync)(filePath)) {
|
|
4245
|
+
for (const key of chunkKeys) {
|
|
4246
|
+
store.remove(key);
|
|
4247
|
+
invertedIndex.removeChunk(key);
|
|
4248
|
+
removedCount++;
|
|
4249
|
+
}
|
|
4250
|
+
database.deleteChunksByFile(filePath);
|
|
4251
|
+
removedFilePaths.push(filePath);
|
|
4252
|
+
}
|
|
4253
|
+
}
|
|
4254
|
+
if (removedCount > 0) {
|
|
4255
|
+
store.save();
|
|
4256
|
+
invertedIndex.save();
|
|
4257
|
+
}
|
|
4258
|
+
const gcOrphanEmbeddings = database.gcOrphanEmbeddings();
|
|
4259
|
+
const gcOrphanChunks = database.gcOrphanChunks();
|
|
4260
|
+
this.logger.recordGc(removedCount, gcOrphanChunks, gcOrphanEmbeddings);
|
|
4261
|
+
this.logger.gc("info", "Health check complete", {
|
|
4262
|
+
removedStale: removedCount,
|
|
4263
|
+
orphanEmbeddings: gcOrphanEmbeddings,
|
|
4264
|
+
orphanChunks: gcOrphanChunks,
|
|
4265
|
+
removedFiles: removedFilePaths.length
|
|
4266
|
+
});
|
|
4267
|
+
return { removed: removedCount, filePaths: removedFilePaths, gcOrphanEmbeddings, gcOrphanChunks };
|
|
4268
|
+
}
|
|
4269
|
+
async retryFailedBatches() {
|
|
4270
|
+
const { store, provider, invertedIndex } = await this.ensureInitialized();
|
|
4271
|
+
const failedBatches = this.loadFailedBatches();
|
|
4272
|
+
if (failedBatches.length === 0) {
|
|
4273
|
+
return { succeeded: 0, failed: 0, remaining: 0 };
|
|
4274
|
+
}
|
|
4275
|
+
let succeeded = 0;
|
|
4276
|
+
let failed = 0;
|
|
4277
|
+
const stillFailing = [];
|
|
4278
|
+
for (const batch of failedBatches) {
|
|
4279
|
+
try {
|
|
4280
|
+
const result = await pRetry(
|
|
4281
|
+
async () => {
|
|
4282
|
+
const texts = batch.chunks.map((c) => c.text);
|
|
4283
|
+
return provider.embedBatch(texts);
|
|
4284
|
+
},
|
|
4285
|
+
{
|
|
4286
|
+
retries: this.config.indexing.retries,
|
|
4287
|
+
minTimeout: this.config.indexing.retryDelayMs
|
|
4288
|
+
}
|
|
4289
|
+
);
|
|
4290
|
+
const items = batch.chunks.map((chunk, idx) => ({
|
|
4291
|
+
id: chunk.id,
|
|
4292
|
+
vector: result.embeddings[idx],
|
|
4293
|
+
metadata: chunk.metadata
|
|
4294
|
+
}));
|
|
4295
|
+
store.addBatch(items);
|
|
4296
|
+
for (const chunk of batch.chunks) {
|
|
4297
|
+
invertedIndex.removeChunk(chunk.id);
|
|
4298
|
+
invertedIndex.addChunk(chunk.id, chunk.content);
|
|
4299
|
+
}
|
|
4300
|
+
this.logger.recordChunksEmbedded(batch.chunks.length);
|
|
4301
|
+
this.logger.recordEmbeddingApiCall(result.totalTokensUsed);
|
|
4302
|
+
succeeded += batch.chunks.length;
|
|
4303
|
+
} catch (error) {
|
|
4304
|
+
failed += batch.chunks.length;
|
|
4305
|
+
this.logger.recordEmbeddingError();
|
|
4306
|
+
stillFailing.push({
|
|
4307
|
+
...batch,
|
|
4308
|
+
attemptCount: batch.attemptCount + 1,
|
|
4309
|
+
lastAttempt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
4310
|
+
error: String(error)
|
|
4311
|
+
});
|
|
4312
|
+
}
|
|
4313
|
+
}
|
|
4314
|
+
this.saveFailedBatches(stillFailing);
|
|
4315
|
+
if (succeeded > 0) {
|
|
4316
|
+
store.save();
|
|
4317
|
+
invertedIndex.save();
|
|
4318
|
+
}
|
|
4319
|
+
return { succeeded, failed, remaining: stillFailing.length };
|
|
4320
|
+
}
|
|
4321
|
+
getFailedBatchesCount() {
|
|
4322
|
+
return this.loadFailedBatches().length;
|
|
4323
|
+
}
|
|
4324
|
+
getCurrentBranch() {
|
|
4325
|
+
return this.currentBranch;
|
|
4326
|
+
}
|
|
4327
|
+
getBaseBranch() {
|
|
4328
|
+
return this.baseBranch;
|
|
4329
|
+
}
|
|
4330
|
+
refreshBranchInfo() {
|
|
4331
|
+
if (isGitRepo(this.projectRoot)) {
|
|
4332
|
+
this.currentBranch = getBranchOrDefault(this.projectRoot);
|
|
4333
|
+
this.baseBranch = getBaseBranch(this.projectRoot);
|
|
4334
|
+
}
|
|
4335
|
+
}
|
|
4336
|
+
async getDatabaseStats() {
|
|
4337
|
+
const { database } = await this.ensureInitialized();
|
|
4338
|
+
return database.getStats();
|
|
4339
|
+
}
|
|
4340
|
+
getLogger() {
|
|
4341
|
+
return this.logger;
|
|
4342
|
+
}
|
|
4343
|
+
async findSimilar(code, limit = this.config.search.maxResults, options) {
|
|
4344
|
+
const { store, provider, database } = await this.ensureInitialized();
|
|
4345
|
+
const compatibility = this.checkCompatibility();
|
|
4346
|
+
if (!compatibility.compatible) {
|
|
4347
|
+
throw new Error(
|
|
4348
|
+
`${compatibility.reason ?? "Index is incompatible with current embedding provider."} Run index_codebase with force=true to rebuild the index.`
|
|
4349
|
+
);
|
|
4350
|
+
}
|
|
4351
|
+
const searchStartTime = import_perf_hooks.performance.now();
|
|
4352
|
+
if (store.count() === 0) {
|
|
4353
|
+
this.logger.search("debug", "Find similar on empty index");
|
|
4354
|
+
return [];
|
|
4355
|
+
}
|
|
4356
|
+
const filterByBranch = options?.filterByBranch ?? true;
|
|
4357
|
+
this.logger.search("debug", "Starting find similar", {
|
|
4358
|
+
codeLength: code.length,
|
|
4359
|
+
limit,
|
|
4360
|
+
filterByBranch
|
|
4361
|
+
});
|
|
4362
|
+
const embeddingStartTime = import_perf_hooks.performance.now();
|
|
4363
|
+
const { embedding, tokensUsed } = await provider.embedDocument(code);
|
|
4364
|
+
const embeddingMs = import_perf_hooks.performance.now() - embeddingStartTime;
|
|
4365
|
+
this.logger.recordEmbeddingApiCall(tokensUsed);
|
|
4366
|
+
const vectorStartTime = import_perf_hooks.performance.now();
|
|
4367
|
+
const semanticResults = store.search(embedding, limit * 2);
|
|
4368
|
+
const vectorMs = import_perf_hooks.performance.now() - vectorStartTime;
|
|
4369
|
+
let branchChunkIds = null;
|
|
4370
|
+
if (filterByBranch && this.currentBranch !== "default") {
|
|
4371
|
+
branchChunkIds = new Set(database.getBranchChunkIds(this.currentBranch));
|
|
4372
|
+
}
|
|
4373
|
+
const filtered = semanticResults.filter((r) => {
|
|
4374
|
+
if (r.score < this.config.search.minScore) return false;
|
|
4375
|
+
if (branchChunkIds && !branchChunkIds.has(r.id)) return false;
|
|
4376
|
+
if (options?.excludeFile) {
|
|
4377
|
+
if (r.metadata.filePath === options.excludeFile) return false;
|
|
4378
|
+
}
|
|
4379
|
+
if (options?.fileType) {
|
|
4380
|
+
const ext = r.metadata.filePath.split(".").pop()?.toLowerCase();
|
|
4381
|
+
if (ext !== options.fileType.toLowerCase().replace(/^\./, "")) return false;
|
|
4382
|
+
}
|
|
4383
|
+
if (options?.directory) {
|
|
4384
|
+
const normalizedDir = options.directory.replace(/^\/|\/$/g, "");
|
|
4385
|
+
if (!r.metadata.filePath.includes(`/${normalizedDir}/`) && !r.metadata.filePath.includes(`${normalizedDir}/`)) return false;
|
|
4386
|
+
}
|
|
4387
|
+
if (options?.chunkType) {
|
|
4388
|
+
if (r.metadata.chunkType !== options.chunkType) return false;
|
|
4389
|
+
}
|
|
4390
|
+
return true;
|
|
4391
|
+
}).slice(0, limit);
|
|
4392
|
+
const totalSearchMs = import_perf_hooks.performance.now() - searchStartTime;
|
|
4393
|
+
this.logger.recordSearch(totalSearchMs, {
|
|
4394
|
+
embeddingMs,
|
|
4395
|
+
vectorMs,
|
|
4396
|
+
keywordMs: 0,
|
|
4397
|
+
fusionMs: 0
|
|
4398
|
+
});
|
|
4399
|
+
this.logger.search("info", "Find similar complete", {
|
|
4400
|
+
codeLength: code.length,
|
|
4401
|
+
results: filtered.length,
|
|
4402
|
+
totalMs: Math.round(totalSearchMs * 100) / 100,
|
|
4403
|
+
embeddingMs: Math.round(embeddingMs * 100) / 100,
|
|
4404
|
+
vectorMs: Math.round(vectorMs * 100) / 100
|
|
4405
|
+
});
|
|
4406
|
+
return Promise.all(
|
|
4407
|
+
filtered.map(async (r) => {
|
|
4408
|
+
let content = "";
|
|
4409
|
+
if (this.config.search.includeContext) {
|
|
4410
|
+
try {
|
|
4411
|
+
const fileContent = await import_fs4.promises.readFile(
|
|
4412
|
+
r.metadata.filePath,
|
|
4413
|
+
"utf-8"
|
|
4414
|
+
);
|
|
4415
|
+
const lines = fileContent.split("\n");
|
|
4416
|
+
content = lines.slice(r.metadata.startLine - 1, r.metadata.endLine).join("\n");
|
|
4417
|
+
} catch {
|
|
4418
|
+
content = "[File not accessible]";
|
|
4419
|
+
}
|
|
4420
|
+
}
|
|
4421
|
+
return {
|
|
4422
|
+
filePath: r.metadata.filePath,
|
|
4423
|
+
startLine: r.metadata.startLine,
|
|
4424
|
+
endLine: r.metadata.endLine,
|
|
4425
|
+
content,
|
|
4426
|
+
score: r.score,
|
|
4427
|
+
chunkType: r.metadata.chunkType,
|
|
4428
|
+
name: r.metadata.name
|
|
4429
|
+
};
|
|
4430
|
+
})
|
|
4431
|
+
);
|
|
4432
|
+
}
|
|
4433
|
+
};
|
|
4434
|
+
|
|
4435
|
+
// src/mcp-server.ts
|
|
4436
|
+
var MAX_CONTENT_LINES = 30;
|
|
4437
|
+
function truncateContent(content) {
|
|
4438
|
+
const lines = content.split("\n");
|
|
4439
|
+
if (lines.length <= MAX_CONTENT_LINES) return content;
|
|
4440
|
+
return lines.slice(0, MAX_CONTENT_LINES).join("\n") + `
|
|
4441
|
+
// ... (${lines.length - MAX_CONTENT_LINES} more lines)`;
|
|
4442
|
+
}
|
|
4443
|
+
function formatIndexStats(stats, verbose = false) {
|
|
4444
|
+
const lines = [];
|
|
4445
|
+
if (stats.indexedChunks === 0 && stats.removedChunks === 0) {
|
|
4446
|
+
lines.push(`Indexed. ${stats.totalFiles} files processed, ${stats.existingChunks} code chunks already up to date.`);
|
|
4447
|
+
} else if (stats.indexedChunks === 0) {
|
|
4448
|
+
lines.push(`Indexed. ${stats.totalFiles} files, removed ${stats.removedChunks} stale chunks, ${stats.existingChunks} chunks remain.`);
|
|
4449
|
+
} else {
|
|
4450
|
+
let main2 = `Indexed. ${stats.totalFiles} files processed, ${stats.indexedChunks} new chunks embedded.`;
|
|
4451
|
+
if (stats.existingChunks > 0) {
|
|
4452
|
+
main2 += ` ${stats.existingChunks} unchanged chunks skipped.`;
|
|
4453
|
+
}
|
|
4454
|
+
lines.push(main2);
|
|
4455
|
+
if (stats.removedChunks > 0) {
|
|
4456
|
+
lines.push(`Removed ${stats.removedChunks} stale chunks.`);
|
|
4457
|
+
}
|
|
4458
|
+
if (stats.failedChunks > 0) {
|
|
4459
|
+
lines.push(`Failed: ${stats.failedChunks} chunks.`);
|
|
4460
|
+
}
|
|
4461
|
+
lines.push(`Tokens: ${stats.tokensUsed.toLocaleString()}, Duration: ${(stats.durationMs / 1e3).toFixed(1)}s`);
|
|
4462
|
+
}
|
|
4463
|
+
if (verbose) {
|
|
4464
|
+
if (stats.skippedFiles.length > 0) {
|
|
4465
|
+
const tooLarge = stats.skippedFiles.filter((f) => f.reason === "too_large");
|
|
4466
|
+
const excluded = stats.skippedFiles.filter((f) => f.reason === "excluded");
|
|
4467
|
+
const gitignored = stats.skippedFiles.filter((f) => f.reason === "gitignore");
|
|
4468
|
+
lines.push("");
|
|
4469
|
+
lines.push(`Skipped files: ${stats.skippedFiles.length}`);
|
|
4470
|
+
if (tooLarge.length > 0) {
|
|
4471
|
+
lines.push(` Too large (${tooLarge.length}): ${tooLarge.slice(0, 5).map((f) => f.path).join(", ")}${tooLarge.length > 5 ? "..." : ""}`);
|
|
4472
|
+
}
|
|
4473
|
+
if (excluded.length > 0) {
|
|
4474
|
+
lines.push(` Excluded (${excluded.length}): ${excluded.slice(0, 5).map((f) => f.path).join(", ")}${excluded.length > 5 ? "..." : ""}`);
|
|
4475
|
+
}
|
|
4476
|
+
if (gitignored.length > 0) {
|
|
4477
|
+
lines.push(` Gitignored (${gitignored.length}): ${gitignored.slice(0, 5).map((f) => f.path).join(", ")}${gitignored.length > 5 ? "..." : ""}`);
|
|
4478
|
+
}
|
|
4479
|
+
}
|
|
4480
|
+
if (stats.parseFailures.length > 0) {
|
|
4481
|
+
lines.push("");
|
|
4482
|
+
lines.push(`Files with no extractable chunks (${stats.parseFailures.length}): ${stats.parseFailures.slice(0, 10).join(", ")}${stats.parseFailures.length > 10 ? "..." : ""}`);
|
|
4483
|
+
}
|
|
4484
|
+
}
|
|
4485
|
+
return lines.join("\n");
|
|
4486
|
+
}
|
|
4487
|
+
function formatStatus(status) {
|
|
4488
|
+
if (!status.indexed) {
|
|
4489
|
+
return "Codebase is not indexed. Run index_codebase to create an index.";
|
|
4490
|
+
}
|
|
4491
|
+
const lines = [
|
|
4492
|
+
`Index status:`,
|
|
4493
|
+
` Indexed chunks: ${status.vectorCount.toLocaleString()}`,
|
|
4494
|
+
` Provider: ${status.provider}`,
|
|
4495
|
+
` Model: ${status.model}`,
|
|
4496
|
+
` Location: ${status.indexPath}`
|
|
4497
|
+
];
|
|
4498
|
+
if (status.currentBranch !== "default") {
|
|
4499
|
+
lines.push(` Current branch: ${status.currentBranch}`);
|
|
4500
|
+
lines.push(` Base branch: ${status.baseBranch}`);
|
|
4501
|
+
}
|
|
4502
|
+
return lines.join("\n");
|
|
4503
|
+
}
|
|
4504
|
+
var CHUNK_TYPE_ENUM = [
|
|
4505
|
+
"function",
|
|
4506
|
+
"class",
|
|
4507
|
+
"method",
|
|
4508
|
+
"interface",
|
|
4509
|
+
"type",
|
|
4510
|
+
"enum",
|
|
4511
|
+
"struct",
|
|
4512
|
+
"impl",
|
|
4513
|
+
"trait",
|
|
4514
|
+
"module",
|
|
4515
|
+
"other"
|
|
4516
|
+
];
|
|
4517
|
+
function createMcpServer(projectRoot, config) {
|
|
4518
|
+
const server = new import_mcp.McpServer({
|
|
4519
|
+
name: "opencode-codebase-index",
|
|
4520
|
+
version: "0.5.0"
|
|
4521
|
+
});
|
|
4522
|
+
const indexer = new Indexer(projectRoot, config);
|
|
4523
|
+
let initialized = false;
|
|
4524
|
+
async function ensureInitialized() {
|
|
4525
|
+
if (!initialized) {
|
|
4526
|
+
await indexer.initialize();
|
|
4527
|
+
initialized = true;
|
|
4528
|
+
}
|
|
4529
|
+
}
|
|
4530
|
+
server.tool(
|
|
4531
|
+
"codebase_search",
|
|
4532
|
+
"Search codebase by MEANING, not keywords. Returns full code content. For just finding WHERE code is (saves ~90% tokens), use codebase_peek instead.",
|
|
4533
|
+
{
|
|
4534
|
+
query: import_zod.z.string().describe("Natural language description of what code you're looking for. Describe behavior, not syntax."),
|
|
4535
|
+
limit: import_zod.z.number().optional().default(5).describe("Maximum number of results to return"),
|
|
4536
|
+
fileType: import_zod.z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
4537
|
+
directory: import_zod.z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
4538
|
+
chunkType: import_zod.z.enum(CHUNK_TYPE_ENUM).optional().describe("Filter by code chunk type"),
|
|
4539
|
+
contextLines: import_zod.z.number().optional().describe("Number of extra lines to include before/after each match (default: 0)")
|
|
4540
|
+
},
|
|
4541
|
+
async (args) => {
|
|
4542
|
+
await ensureInitialized();
|
|
4543
|
+
const results = await indexer.search(args.query, args.limit ?? 5, {
|
|
4544
|
+
fileType: args.fileType,
|
|
4545
|
+
directory: args.directory,
|
|
4546
|
+
chunkType: args.chunkType,
|
|
4547
|
+
contextLines: args.contextLines
|
|
4548
|
+
});
|
|
4549
|
+
if (results.length === 0) {
|
|
4550
|
+
return { content: [{ type: "text", text: "No matching code found. Try a different query or run index_codebase first." }] };
|
|
4551
|
+
}
|
|
4552
|
+
const formatted = results.map((r, idx) => {
|
|
4553
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
4554
|
+
return `${header} (score: ${r.score.toFixed(2)})
|
|
4555
|
+
\`\`\`
|
|
4556
|
+
${truncateContent(r.content)}
|
|
4557
|
+
\`\`\``;
|
|
4558
|
+
});
|
|
4559
|
+
return { content: [{ type: "text", text: `Found ${results.length} results for "${args.query}":
|
|
4560
|
+
|
|
4561
|
+
${formatted.join("\n\n")}` }] };
|
|
4562
|
+
}
|
|
4563
|
+
);
|
|
4564
|
+
server.tool(
|
|
4565
|
+
"codebase_peek",
|
|
4566
|
+
"Quick lookup of code locations by meaning. Returns only metadata (file, line, name, type) WITHOUT code content. Saves ~90% tokens vs codebase_search.",
|
|
4567
|
+
{
|
|
4568
|
+
query: import_zod.z.string().describe("Natural language description of what code you're looking for."),
|
|
4569
|
+
limit: import_zod.z.number().optional().default(10).describe("Maximum number of results to return"),
|
|
4570
|
+
fileType: import_zod.z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
4571
|
+
directory: import_zod.z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
4572
|
+
chunkType: import_zod.z.enum(CHUNK_TYPE_ENUM).optional().describe("Filter by code chunk type")
|
|
4573
|
+
},
|
|
4574
|
+
async (args) => {
|
|
4575
|
+
await ensureInitialized();
|
|
4576
|
+
const results = await indexer.search(args.query, args.limit ?? 10, {
|
|
4577
|
+
fileType: args.fileType,
|
|
4578
|
+
directory: args.directory,
|
|
4579
|
+
chunkType: args.chunkType,
|
|
4580
|
+
metadataOnly: true
|
|
4581
|
+
});
|
|
4582
|
+
if (results.length === 0) {
|
|
4583
|
+
return { content: [{ type: "text", text: "No matching code found. Try a different query or run index_codebase first." }] };
|
|
4584
|
+
}
|
|
4585
|
+
const formatted = results.map((r, idx) => {
|
|
4586
|
+
const location = `${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
4587
|
+
const name = r.name ? `"${r.name}"` : "(anonymous)";
|
|
4588
|
+
return `[${idx + 1}] ${r.chunkType} ${name} at ${location} (score: ${r.score.toFixed(2)})`;
|
|
4589
|
+
});
|
|
4590
|
+
return { content: [{ type: "text", text: `Found ${results.length} locations for "${args.query}":
|
|
4591
|
+
|
|
4592
|
+
${formatted.join("\n")}
|
|
4593
|
+
|
|
4594
|
+
Use Read tool to examine specific files.` }] };
|
|
4595
|
+
}
|
|
4596
|
+
);
|
|
4597
|
+
server.tool(
|
|
4598
|
+
"index_codebase",
|
|
4599
|
+
"Index the codebase for semantic search. Creates vector embeddings of code chunks. Incremental - only re-indexes changed files. Run before first codebase_search.",
|
|
4600
|
+
{
|
|
4601
|
+
force: import_zod.z.boolean().optional().default(false).describe("Force reindex even if already indexed"),
|
|
4602
|
+
estimateOnly: import_zod.z.boolean().optional().default(false).describe("Only show cost estimate without indexing"),
|
|
4603
|
+
verbose: import_zod.z.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures")
|
|
4604
|
+
},
|
|
4605
|
+
async (args) => {
|
|
4606
|
+
await ensureInitialized();
|
|
4607
|
+
if (args.estimateOnly) {
|
|
4608
|
+
const estimate = await indexer.estimateCost();
|
|
4609
|
+
return { content: [{ type: "text", text: formatCostEstimate(estimate) }] };
|
|
4610
|
+
}
|
|
4611
|
+
if (args.force) {
|
|
4612
|
+
await indexer.clearIndex();
|
|
4613
|
+
}
|
|
4614
|
+
const stats = await indexer.index();
|
|
4615
|
+
return { content: [{ type: "text", text: formatIndexStats(stats, args.verbose ?? false) }] };
|
|
4616
|
+
}
|
|
4617
|
+
);
|
|
4618
|
+
server.tool(
|
|
4619
|
+
"index_status",
|
|
4620
|
+
"Check the status of the codebase index. Shows whether the codebase is indexed, how many chunks are stored, and the embedding provider being used.",
|
|
4621
|
+
{},
|
|
4622
|
+
async () => {
|
|
4623
|
+
await ensureInitialized();
|
|
4624
|
+
const status = await indexer.getStatus();
|
|
4625
|
+
return { content: [{ type: "text", text: formatStatus(status) }] };
|
|
4626
|
+
}
|
|
4627
|
+
);
|
|
4628
|
+
server.tool(
|
|
4629
|
+
"index_health_check",
|
|
4630
|
+
"Check index health and remove stale entries from deleted files. Run this to clean up the index after files have been deleted.",
|
|
4631
|
+
{},
|
|
4632
|
+
async () => {
|
|
4633
|
+
await ensureInitialized();
|
|
4634
|
+
const result = await indexer.healthCheck();
|
|
4635
|
+
if (result.removed === 0 && result.gcOrphanEmbeddings === 0 && result.gcOrphanChunks === 0) {
|
|
4636
|
+
return { content: [{ type: "text", text: "Index is healthy. No stale entries found." }] };
|
|
4637
|
+
}
|
|
4638
|
+
const lines = [`Health check complete:`];
|
|
4639
|
+
if (result.removed > 0) {
|
|
4640
|
+
lines.push(` Removed stale entries: ${result.removed}`);
|
|
4641
|
+
}
|
|
4642
|
+
if (result.gcOrphanEmbeddings > 0) {
|
|
4643
|
+
lines.push(` Garbage collected orphan embeddings: ${result.gcOrphanEmbeddings}`);
|
|
4644
|
+
}
|
|
4645
|
+
if (result.gcOrphanChunks > 0) {
|
|
4646
|
+
lines.push(` Garbage collected orphan chunks: ${result.gcOrphanChunks}`);
|
|
4647
|
+
}
|
|
4648
|
+
if (result.filePaths.length > 0) {
|
|
4649
|
+
lines.push(` Cleaned paths: ${result.filePaths.join(", ")}`);
|
|
4650
|
+
}
|
|
4651
|
+
return { content: [{ type: "text", text: lines.join("\n") }] };
|
|
4652
|
+
}
|
|
4653
|
+
);
|
|
4654
|
+
server.tool(
|
|
4655
|
+
"index_metrics",
|
|
4656
|
+
"Get metrics and performance statistics for the codebase index. Requires debug.enabled=true and debug.metrics=true in config.",
|
|
4657
|
+
{},
|
|
4658
|
+
async () => {
|
|
4659
|
+
await ensureInitialized();
|
|
4660
|
+
const logger = indexer.getLogger();
|
|
4661
|
+
if (!logger.isEnabled()) {
|
|
4662
|
+
return { content: [{ type: "text", text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true,\n "metrics": true\n }\n}\n```' }] };
|
|
4663
|
+
}
|
|
4664
|
+
if (!logger.isMetricsEnabled()) {
|
|
4665
|
+
return { content: [{ type: "text", text: 'Metrics collection is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true,\n "metrics": true\n }\n}\n```' }] };
|
|
4666
|
+
}
|
|
4667
|
+
return { content: [{ type: "text", text: logger.formatMetrics() }] };
|
|
4668
|
+
}
|
|
4669
|
+
);
|
|
4670
|
+
server.tool(
|
|
4671
|
+
"index_logs",
|
|
4672
|
+
"Get recent debug logs from the codebase indexer. Requires debug.enabled=true in config.",
|
|
4673
|
+
{
|
|
4674
|
+
limit: import_zod.z.number().optional().default(20).describe("Maximum number of log entries to return"),
|
|
4675
|
+
category: import_zod.z.enum(["search", "embedding", "cache", "gc", "branch", "general"]).optional().describe("Filter by log category"),
|
|
4676
|
+
level: import_zod.z.enum(["error", "warn", "info", "debug"]).optional().describe("Filter by minimum log level")
|
|
4677
|
+
},
|
|
4678
|
+
async (args) => {
|
|
4679
|
+
await ensureInitialized();
|
|
4680
|
+
const logger = indexer.getLogger();
|
|
4681
|
+
if (!logger.isEnabled()) {
|
|
4682
|
+
return { content: [{ type: "text", text: 'Debug mode is disabled. Enable it in your config:\n\n```json\n{\n "debug": {\n "enabled": true\n }\n}\n```' }] };
|
|
4683
|
+
}
|
|
4684
|
+
let logs;
|
|
4685
|
+
if (args.category) {
|
|
4686
|
+
logs = logger.getLogsByCategory(args.category, args.limit);
|
|
4687
|
+
} else if (args.level) {
|
|
4688
|
+
logs = logger.getLogsByLevel(args.level, args.limit);
|
|
4689
|
+
} else {
|
|
4690
|
+
logs = logger.getLogs(args.limit);
|
|
4691
|
+
}
|
|
4692
|
+
if (logs.length === 0) {
|
|
4693
|
+
return { content: [{ type: "text", text: "No logs recorded yet. Logs are captured during indexing and search operations." }] };
|
|
4694
|
+
}
|
|
4695
|
+
const text = logs.map((l) => {
|
|
4696
|
+
const dataStr = l.data ? ` ${JSON.stringify(l.data)}` : "";
|
|
4697
|
+
return `[${l.timestamp}] [${l.level.toUpperCase()}] [${l.category}] ${l.message}${dataStr}`;
|
|
4698
|
+
}).join("\n");
|
|
4699
|
+
return { content: [{ type: "text", text }] };
|
|
4700
|
+
}
|
|
4701
|
+
);
|
|
4702
|
+
server.tool(
|
|
4703
|
+
"find_similar",
|
|
4704
|
+
"Find code similar to a given snippet. Use for duplicate detection, pattern discovery, or refactoring prep.",
|
|
4705
|
+
{
|
|
4706
|
+
code: import_zod.z.string().describe("The code snippet to find similar code for"),
|
|
4707
|
+
limit: import_zod.z.number().optional().default(10).describe("Maximum number of results to return"),
|
|
4708
|
+
fileType: import_zod.z.string().optional().describe("Filter by file extension (e.g., 'ts', 'py', 'rs')"),
|
|
4709
|
+
directory: import_zod.z.string().optional().describe("Filter by directory path (e.g., 'src/utils', 'lib')"),
|
|
4710
|
+
chunkType: import_zod.z.enum(CHUNK_TYPE_ENUM).optional().describe("Filter by code chunk type"),
|
|
4711
|
+
excludeFile: import_zod.z.string().optional().describe("Exclude results from this file path")
|
|
4712
|
+
},
|
|
4713
|
+
async (args) => {
|
|
4714
|
+
await ensureInitialized();
|
|
4715
|
+
const results = await indexer.findSimilar(args.code, args.limit ?? 10, {
|
|
4716
|
+
fileType: args.fileType,
|
|
4717
|
+
directory: args.directory,
|
|
4718
|
+
chunkType: args.chunkType,
|
|
4719
|
+
excludeFile: args.excludeFile
|
|
4720
|
+
});
|
|
4721
|
+
if (results.length === 0) {
|
|
4722
|
+
return { content: [{ type: "text", text: "No similar code found. Try a different snippet or run index_codebase first." }] };
|
|
4723
|
+
}
|
|
4724
|
+
const formatted = results.map((r, idx) => {
|
|
4725
|
+
const header = r.name ? `[${idx + 1}] ${r.chunkType} "${r.name}" in ${r.filePath}:${r.startLine}-${r.endLine}` : `[${idx + 1}] ${r.chunkType} in ${r.filePath}:${r.startLine}-${r.endLine}`;
|
|
4726
|
+
return `${header} (similarity: ${(r.score * 100).toFixed(1)}%)
|
|
4727
|
+
\`\`\`
|
|
4728
|
+
${truncateContent(r.content)}
|
|
4729
|
+
\`\`\``;
|
|
4730
|
+
});
|
|
4731
|
+
return { content: [{ type: "text", text: `Found ${results.length} similar code blocks:
|
|
4732
|
+
|
|
4733
|
+
${formatted.join("\n\n")}` }] };
|
|
4734
|
+
}
|
|
4735
|
+
);
|
|
4736
|
+
server.prompt(
|
|
4737
|
+
"search",
|
|
4738
|
+
"Search codebase by meaning using semantic search",
|
|
4739
|
+
{ query: import_zod.z.string().describe("What to search for in the codebase") },
|
|
4740
|
+
(args) => ({
|
|
4741
|
+
messages: [{
|
|
4742
|
+
role: "user",
|
|
4743
|
+
content: {
|
|
4744
|
+
type: "text",
|
|
4745
|
+
text: `Search the codebase for: "${args.query}"
|
|
4746
|
+
|
|
4747
|
+
Use the codebase_search tool with this query. If you need just locations first, use codebase_peek instead to save tokens.`
|
|
4748
|
+
}
|
|
4749
|
+
}]
|
|
4750
|
+
})
|
|
4751
|
+
);
|
|
4752
|
+
server.prompt(
|
|
4753
|
+
"find",
|
|
4754
|
+
"Find code using hybrid approach (semantic + grep)",
|
|
4755
|
+
{ query: import_zod.z.string().describe("What to find in the codebase") },
|
|
4756
|
+
(args) => ({
|
|
4757
|
+
messages: [{
|
|
4758
|
+
role: "user",
|
|
4759
|
+
content: {
|
|
4760
|
+
type: "text",
|
|
4761
|
+
text: `Find code related to: "${args.query}"
|
|
4762
|
+
|
|
4763
|
+
Use a hybrid approach:
|
|
4764
|
+
1. First use codebase_peek to find semantic matches by meaning
|
|
4765
|
+
2. Then use grep for exact identifier matches
|
|
4766
|
+
3. Combine results for comprehensive coverage`
|
|
4767
|
+
}
|
|
4768
|
+
}]
|
|
4769
|
+
})
|
|
4770
|
+
);
|
|
4771
|
+
server.prompt(
|
|
4772
|
+
"index",
|
|
4773
|
+
"Index the codebase for semantic search",
|
|
4774
|
+
{ options: import_zod.z.string().optional().describe("Options: 'force' to rebuild, 'estimate' to check costs") },
|
|
4775
|
+
(args) => {
|
|
4776
|
+
const opts = args.options?.toLowerCase() ?? "";
|
|
4777
|
+
let instruction = "Use the index_codebase tool to index the codebase for semantic search.";
|
|
4778
|
+
if (opts.includes("force")) {
|
|
4779
|
+
instruction = "Use the index_codebase tool with force=true to rebuild the entire index from scratch.";
|
|
4780
|
+
} else if (opts.includes("estimate")) {
|
|
4781
|
+
instruction = "Use the index_codebase tool with estimateOnly=true to check the cost estimate before indexing.";
|
|
4782
|
+
}
|
|
4783
|
+
return {
|
|
4784
|
+
messages: [{
|
|
4785
|
+
role: "user",
|
|
4786
|
+
content: { type: "text", text: instruction }
|
|
4787
|
+
}]
|
|
4788
|
+
};
|
|
4789
|
+
}
|
|
4790
|
+
);
|
|
4791
|
+
server.prompt(
|
|
4792
|
+
"status",
|
|
4793
|
+
"Check if the codebase is indexed and ready",
|
|
4794
|
+
{},
|
|
4795
|
+
() => ({
|
|
4796
|
+
messages: [{
|
|
4797
|
+
role: "user",
|
|
4798
|
+
content: {
|
|
4799
|
+
type: "text",
|
|
4800
|
+
text: "Use the index_status tool to check if the codebase index is ready and show its current state."
|
|
4801
|
+
}
|
|
4802
|
+
}]
|
|
4803
|
+
})
|
|
4804
|
+
);
|
|
4805
|
+
return server;
|
|
4806
|
+
}
|
|
4807
|
+
|
|
4808
|
+
// src/cli.ts
|
|
4809
|
+
function loadJsonFile(filePath) {
|
|
4810
|
+
try {
|
|
4811
|
+
if ((0, import_fs5.existsSync)(filePath)) {
|
|
4812
|
+
const content = (0, import_fs5.readFileSync)(filePath, "utf-8");
|
|
4813
|
+
return JSON.parse(content);
|
|
4814
|
+
}
|
|
4815
|
+
} catch {
|
|
4816
|
+
}
|
|
4817
|
+
return null;
|
|
4818
|
+
}
|
|
4819
|
+
function loadPluginConfig(projectRoot, configPath) {
|
|
4820
|
+
if (configPath) {
|
|
4821
|
+
const config = loadJsonFile(configPath);
|
|
4822
|
+
if (config) return config;
|
|
4823
|
+
}
|
|
4824
|
+
const projectConfig = loadJsonFile(path6.join(projectRoot, ".opencode", "codebase-index.json"));
|
|
4825
|
+
if (projectConfig) return projectConfig;
|
|
4826
|
+
const globalConfig = loadJsonFile(path6.join(os3.homedir(), ".config", "opencode", "codebase-index.json"));
|
|
4827
|
+
if (globalConfig) return globalConfig;
|
|
4828
|
+
return {};
|
|
4829
|
+
}
|
|
4830
|
+
function parseArgs(argv) {
|
|
4831
|
+
let project = process.cwd();
|
|
4832
|
+
let config;
|
|
4833
|
+
for (let i = 2; i < argv.length; i++) {
|
|
4834
|
+
if (argv[i] === "--project" && argv[i + 1]) {
|
|
4835
|
+
project = path6.resolve(argv[++i]);
|
|
4836
|
+
} else if (argv[i] === "--config" && argv[i + 1]) {
|
|
4837
|
+
config = path6.resolve(argv[++i]);
|
|
4838
|
+
}
|
|
4839
|
+
}
|
|
4840
|
+
return { project, config };
|
|
4841
|
+
}
|
|
4842
|
+
async function main() {
|
|
4843
|
+
const args = parseArgs(process.argv);
|
|
4844
|
+
const rawConfig = loadPluginConfig(args.project, args.config);
|
|
4845
|
+
const config = parseConfig(rawConfig);
|
|
4846
|
+
const server = createMcpServer(args.project, config);
|
|
4847
|
+
const transport = new import_stdio.StdioServerTransport();
|
|
4848
|
+
await server.connect(transport);
|
|
4849
|
+
const shutdown = () => {
|
|
4850
|
+
server.close().catch(() => {
|
|
4851
|
+
});
|
|
4852
|
+
process.exit(0);
|
|
4853
|
+
};
|
|
4854
|
+
process.on("SIGINT", shutdown);
|
|
4855
|
+
process.on("SIGTERM", shutdown);
|
|
4856
|
+
}
|
|
4857
|
+
main().catch((error) => {
|
|
4858
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4859
|
+
console.error(`Fatal: ${message}`);
|
|
4860
|
+
process.exit(1);
|
|
4861
|
+
});
|
|
4862
|
+
//# sourceMappingURL=cli.cjs.map
|