opencode-rag-plugin 1.12.5 → 1.12.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +59 -1
- package/dist/cli.js.map +1 -1
- package/dist/content/doc.d.ts +3 -0
- package/dist/content/doc.js +12 -0
- package/dist/content/doc.js.map +1 -0
- package/dist/content/docx.d.ts +3 -0
- package/dist/content/docx.js +12 -0
- package/dist/content/docx.js.map +1 -0
- package/dist/content/excel.d.ts +3 -0
- package/dist/content/excel.js +12 -0
- package/dist/content/excel.js.map +1 -0
- package/dist/content/image.d.ts +5 -0
- package/dist/content/image.js +24 -0
- package/dist/content/image.js.map +1 -0
- package/dist/content/pdf.d.ts +3 -0
- package/dist/content/pdf.js +12 -0
- package/dist/content/pdf.js.map +1 -0
- package/dist/content/reader.d.ts +18 -0
- package/dist/content/reader.js +104 -0
- package/dist/content/reader.js.map +1 -0
- package/dist/content/types.d.ts +5 -0
- package/dist/content/types.js +2 -0
- package/dist/content/types.js.map +1 -0
- package/dist/core/bootstrap.js +2 -2
- package/dist/core/bootstrap.js.map +1 -1
- package/dist/core/config.d.ts +5 -0
- package/dist/core/config.js +9 -1
- package/dist/core/config.js.map +1 -1
- package/dist/eval/run-token-test.js +2 -2
- package/dist/eval/run-token-test.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/indexer/description-stage.d.ts +14 -0
- package/dist/indexer/description-stage.js +46 -0
- package/dist/indexer/description-stage.js.map +1 -0
- package/dist/indexer/embed-stage.d.ts +8 -0
- package/dist/indexer/embed-stage.js +34 -0
- package/dist/indexer/embed-stage.js.map +1 -0
- package/dist/indexer/metadata.d.ts +1 -0
- package/dist/indexer/metadata.js +99 -0
- package/dist/indexer/metadata.js.map +1 -0
- package/dist/indexer/pipeline.d.ts +22 -0
- package/dist/indexer/pipeline.js +179 -0
- package/dist/indexer/pipeline.js.map +1 -0
- package/dist/indexer/stats.d.ts +29 -0
- package/dist/indexer/stats.js +20 -0
- package/dist/indexer/stats.js.map +1 -0
- package/dist/indexer/watch.d.ts +8 -0
- package/dist/indexer/watch.js +95 -0
- package/dist/indexer/watch.js.map +1 -0
- package/dist/indexer/worker.d.ts +36 -0
- package/dist/indexer/worker.js +151 -0
- package/dist/indexer/worker.js.map +1 -0
- package/dist/indexer.d.ts +7 -54
- package/dist/indexer.js +5 -626
- package/dist/indexer.js.map +1 -1
- package/dist/plugin.d.ts +1 -1
- package/dist/plugin.js +37 -5
- package/dist/plugin.js.map +1 -1
- package/dist/updater.d.ts +19 -0
- package/dist/updater.js +132 -0
- package/dist/updater.js.map +1 -0
- package/dist/vectorstore/factory.d.ts +3 -0
- package/dist/vectorstore/factory.js +13 -0
- package/dist/vectorstore/factory.js.map +1 -0
- package/dist/vectorstore/memory.d.ts +9 -0
- package/dist/vectorstore/memory.js +41 -0
- package/dist/vectorstore/memory.js.map +1 -0
- package/package.json +4 -4
package/dist/indexer.js
CHANGED
|
@@ -1,629 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
import { extractPdfText } from "./chunker/pdf.js";
|
|
6
|
-
import { uuid } from "./chunker/uuid.js";
|
|
7
|
-
import { extractDocxText } from "./chunker/docx.js";
|
|
8
|
-
import { extractDocText } from "./chunker/doc.js";
|
|
9
|
-
import { extractExcelText } from "./chunker/excel.js";
|
|
10
|
-
import { createImageVisionProvider, getMimeType, SUPPORTED_IMAGE_EXTENSIONS } from "./chunker/image.js";
|
|
11
|
-
import { computeFileHash, loadManifest, manifestPathFor, normalizeFilePath, saveManifest, } from "./core/manifest.js";
|
|
12
|
-
import { embedBatch } from "./embedder/factory.js";
|
|
13
|
-
function createLogger(logger) {
|
|
14
|
-
return {
|
|
15
|
-
info: logger?.info ?? (() => { }),
|
|
16
|
-
warn: logger?.warn ?? (() => { }),
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
|
-
const FILE_TYPE_LABELS = {
|
|
20
|
-
".pdf": "pdf",
|
|
21
|
-
".docx": "docx",
|
|
22
|
-
".doc": "doc",
|
|
23
|
-
".xls": "excel",
|
|
24
|
-
".xlsx": "excel",
|
|
25
|
-
".md": "markdown",
|
|
26
|
-
".mdx": "markdown",
|
|
27
|
-
".json": "json",
|
|
28
|
-
".yaml": "yaml",
|
|
29
|
-
".yml": "yaml",
|
|
30
|
-
".toml": "toml",
|
|
31
|
-
".xml": "xml",
|
|
32
|
-
".html": "html",
|
|
33
|
-
".css": "css",
|
|
34
|
-
".csv": "csv",
|
|
35
|
-
".txt": "text",
|
|
36
|
-
".ts": "typescript",
|
|
37
|
-
".tsx": "typescript",
|
|
38
|
-
".js": "javascript",
|
|
39
|
-
".jsx": "javascript",
|
|
40
|
-
".py": "python",
|
|
41
|
-
".java": "java",
|
|
42
|
-
".go": "go",
|
|
43
|
-
".rs": "rust",
|
|
44
|
-
".c": "c",
|
|
45
|
-
".cpp": "cpp",
|
|
46
|
-
".h": "c",
|
|
47
|
-
".hpp": "cpp",
|
|
48
|
-
".cs": "csharp",
|
|
49
|
-
".rb": "ruby",
|
|
50
|
-
".php": "php",
|
|
51
|
-
".swift": "swift",
|
|
52
|
-
".kt": "kotlin",
|
|
53
|
-
".sql": "sql",
|
|
54
|
-
".sh": "bash",
|
|
55
|
-
".bash": "bash",
|
|
56
|
-
".ps1": "powershell",
|
|
57
|
-
".dockerfile": "dockerfile",
|
|
58
|
-
".tex": "latex",
|
|
59
|
-
".razor": "razor",
|
|
60
|
-
".sln": "sln",
|
|
61
|
-
".ini": "ini",
|
|
62
|
-
};
|
|
63
|
-
function classifyContentType(relPath) {
|
|
64
|
-
const lower = relPath.toLowerCase();
|
|
65
|
-
const parts = lower.split("/");
|
|
66
|
-
const basename = parts[parts.length - 1] ?? "";
|
|
67
|
-
if (basename.startsWith("readme"))
|
|
68
|
-
return "readme";
|
|
69
|
-
if (/^(test|tests|__tests__|spec|specs|__spec__)\b/.test(basename) || /\.(test|spec)\.[^.]+$/.test(basename))
|
|
70
|
-
return "test";
|
|
71
|
-
if (parts.some((p) => /^(docs?|documentation|guides?|manual|tutorial|tutorial)s?$/.test(p)))
|
|
72
|
-
return "documentation";
|
|
73
|
-
if (parts.some((p) => /^(config|conf|configuration|settings|env)s?$/.test(p)) || /\.(config|conf)\.[^.]+$/.test(basename))
|
|
74
|
-
return "configuration";
|
|
75
|
-
if (parts.some((p) => /^(src|source|lib|packages?|modules?)$/.test(p)))
|
|
76
|
-
return "source";
|
|
77
|
-
if (parts.some((p) => /^(ci|cd|\.github|\.gitlab|build|deploy|scripts?)$/.test(p)))
|
|
78
|
-
return "build";
|
|
79
|
-
if (parts.some((p) => /^(examples?|samples?|demo|demos?)$/.test(p)))
|
|
80
|
-
return "example";
|
|
81
|
-
return "";
|
|
82
|
-
}
|
|
83
|
-
function extractDocumentTitle(content, filePath) {
|
|
84
|
-
const ext = path.extname(filePath).toLowerCase();
|
|
85
|
-
if (ext === ".md" || ext === ".mdx") {
|
|
86
|
-
const match = content.match(/^#{1,3}\s+(.+)$/m);
|
|
87
|
-
if (match?.[1])
|
|
88
|
-
return match[1].trim().slice(0, 80);
|
|
89
|
-
}
|
|
90
|
-
const basename = path.basename(filePath, ext);
|
|
91
|
-
return basename
|
|
92
|
-
.replace(/[-_]+/g, " ")
|
|
93
|
-
.replace(/\b\w/g, (c) => c.toUpperCase())
|
|
94
|
-
.slice(0, 80);
|
|
95
|
-
}
|
|
96
|
-
function buildFileMetadataHeader(filePath, cwd, content) {
|
|
97
|
-
const ext = path.extname(filePath).toLowerCase();
|
|
98
|
-
const relPath = path.relative(cwd, filePath).replace(/\\/g, "/");
|
|
99
|
-
const fileType = FILE_TYPE_LABELS[ext] ?? ext.slice(1);
|
|
100
|
-
const dirParts = relPath.split("/");
|
|
101
|
-
dirParts.pop();
|
|
102
|
-
const topDir = dirParts[0] ?? "";
|
|
103
|
-
const contentType = classifyContentType(relPath);
|
|
104
|
-
const title = extractDocumentTitle(content, filePath);
|
|
105
|
-
const parts = [];
|
|
106
|
-
if (fileType)
|
|
107
|
-
parts.push(`[${fileType}]`);
|
|
108
|
-
if (topDir)
|
|
109
|
-
parts.push(`[${topDir}]`);
|
|
110
|
-
if (contentType)
|
|
111
|
-
parts.push(`[${contentType}]`);
|
|
112
|
-
if (title)
|
|
113
|
-
parts.push(title);
|
|
114
|
-
return parts.length > 0 ? parts.join(" ") : "";
|
|
115
|
-
}
|
|
116
|
-
export async function walkFiles(dir, extensions, excludeDirs) {
|
|
117
|
-
const results = [];
|
|
118
|
-
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
119
|
-
for (const entry of entries) {
|
|
120
|
-
const fullPath = path.join(dir, entry.name);
|
|
121
|
-
if (entry.isDirectory()) {
|
|
122
|
-
if (excludeDirs.has(entry.name))
|
|
123
|
-
continue;
|
|
124
|
-
if (entry.name.startsWith(".") && !extensions.size)
|
|
125
|
-
continue;
|
|
126
|
-
results.push(...(await walkFiles(fullPath, extensions, excludeDirs)));
|
|
127
|
-
}
|
|
128
|
-
else if (entry.isFile()) {
|
|
129
|
-
const ext = path.extname(entry.name).toLowerCase();
|
|
130
|
-
const basename = entry.name.toLowerCase();
|
|
131
|
-
if (extensions.has(ext) || extensions.has(basename)) {
|
|
132
|
-
results.push(fullPath);
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
return results;
|
|
137
|
-
}
|
|
138
|
-
function isImageFile(fp, extensions) {
|
|
139
|
-
const lower = fp.toLowerCase();
|
|
140
|
-
for (const ext of extensions) {
|
|
141
|
-
if (lower.endsWith(ext.toLowerCase()))
|
|
142
|
-
return true;
|
|
143
|
-
}
|
|
144
|
-
return false;
|
|
145
|
-
}
|
|
146
|
-
function imageExtensionSet(extensions) {
|
|
147
|
-
return new Set(extensions.map((e) => e.toLowerCase()));
|
|
148
|
-
}
|
|
1
|
+
export { createIndexStats } from "./indexer/stats.js";
|
|
2
|
+
export { scanWorkspaceFiles, walkFiles } from "./content/reader.js";
|
|
3
|
+
export { runIndexPass, getIndexStatusSummary, createWatchPassScheduler, createWatchIgnore, } from "./indexer/pipeline.js";
|
|
4
|
+
import { scanWorkspaceFiles } from "./content/reader.js";
|
|
149
5
|
export async function scanWorkspace(cwd, config, logger) {
|
|
150
|
-
|
|
151
|
-
let imageVisionProvider = null;
|
|
152
|
-
const imageCfg = config.imageDescription;
|
|
153
|
-
if (imageCfg?.enabled) {
|
|
154
|
-
for (const ext of SUPPORTED_IMAGE_EXTENSIONS) {
|
|
155
|
-
extensions.add(ext.toLowerCase());
|
|
156
|
-
}
|
|
157
|
-
imageVisionProvider = createImageVisionProvider(imageCfg);
|
|
158
|
-
}
|
|
159
|
-
const files = await walkFiles(cwd, extensions, new Set(config.indexing.excludeDirs));
|
|
160
|
-
const isPdf = (fp) => fp.toLowerCase().endsWith(".pdf");
|
|
161
|
-
const isDocx = (fp) => fp.toLowerCase().endsWith(".docx");
|
|
162
|
-
const isDoc = (fp) => fp.toLowerCase().endsWith(".doc");
|
|
163
|
-
const isExcel = (fp) => { const e = fp.toLowerCase(); return e.endsWith(".xls") || e.endsWith(".xlsx"); };
|
|
164
|
-
const minSize = config.indexing.minFileSizeBytes ?? 0;
|
|
165
|
-
const workspaceFiles = [];
|
|
166
|
-
for (const filePath of files) {
|
|
167
|
-
let content;
|
|
168
|
-
if (isPdf(filePath)) {
|
|
169
|
-
const buffer = await fs.readFile(filePath);
|
|
170
|
-
try {
|
|
171
|
-
content = await extractPdfText(buffer);
|
|
172
|
-
}
|
|
173
|
-
catch (err) {
|
|
174
|
-
content = "";
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
else if (isDocx(filePath)) {
|
|
178
|
-
const buffer = await fs.readFile(filePath);
|
|
179
|
-
try {
|
|
180
|
-
content = await extractDocxText(buffer);
|
|
181
|
-
}
|
|
182
|
-
catch (err) {
|
|
183
|
-
content = "";
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
else if (isDoc(filePath)) {
|
|
187
|
-
const buffer = await fs.readFile(filePath);
|
|
188
|
-
try {
|
|
189
|
-
content = await extractDocText(buffer);
|
|
190
|
-
}
|
|
191
|
-
catch (err) {
|
|
192
|
-
content = "";
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
else if (isExcel(filePath)) {
|
|
196
|
-
const buffer = await fs.readFile(filePath);
|
|
197
|
-
try {
|
|
198
|
-
content = await extractExcelText(buffer);
|
|
199
|
-
}
|
|
200
|
-
catch (err) {
|
|
201
|
-
content = "";
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
else if (imageVisionProvider && isImageFile(filePath, SUPPORTED_IMAGE_EXTENSIONS)) {
|
|
205
|
-
const buffer = await fs.readFile(filePath);
|
|
206
|
-
try {
|
|
207
|
-
const ext = path.extname(filePath).toLowerCase();
|
|
208
|
-
const mimeType = getMimeType(ext);
|
|
209
|
-
const b64 = buffer.toString("base64");
|
|
210
|
-
const prompt = imageCfg.prompt;
|
|
211
|
-
content = await imageVisionProvider.describeImage(b64, mimeType, prompt);
|
|
212
|
-
}
|
|
213
|
-
catch (err) {
|
|
214
|
-
logger?.warn(` ${filePath} (vision failed: ${err.message})`);
|
|
215
|
-
content = "";
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
else {
|
|
219
|
-
content = await fs.readFile(filePath, "utf-8");
|
|
220
|
-
}
|
|
221
|
-
const byteLength = Buffer.byteLength(content, "utf-8");
|
|
222
|
-
workspaceFiles.push({
|
|
223
|
-
filePath,
|
|
224
|
-
normalizedPath: normalizeFilePath(filePath),
|
|
225
|
-
content,
|
|
226
|
-
hash: computeFileHash(content),
|
|
227
|
-
isEmpty: content.trim().length === 0,
|
|
228
|
-
isTooSmall: !content.trim().length === false && byteLength < minSize,
|
|
229
|
-
});
|
|
230
|
-
}
|
|
231
|
-
return workspaceFiles;
|
|
232
|
-
}
|
|
233
|
-
function createIndexStats(totalFiles, manifestStatus) {
|
|
234
|
-
return {
|
|
235
|
-
totalFiles,
|
|
236
|
-
newFiles: 0,
|
|
237
|
-
modifiedFiles: 0,
|
|
238
|
-
unchangedFiles: 0,
|
|
239
|
-
deletedFiles: 0,
|
|
240
|
-
removedFiles: 0,
|
|
241
|
-
skippedEmptyFiles: 0,
|
|
242
|
-
skippedSmallFiles: 0,
|
|
243
|
-
totalChunks: 0,
|
|
244
|
-
finalCount: 0,
|
|
245
|
-
manifestStatus,
|
|
246
|
-
rebuildPerformed: false,
|
|
247
|
-
batchesFlushed: 0,
|
|
248
|
-
};
|
|
249
|
-
}
|
|
250
|
-
export async function runIndexPass(options) {
|
|
251
|
-
const logger = createLogger(options.logger);
|
|
252
|
-
const workspaceFiles = await scanWorkspace(options.cwd, options.config, logger);
|
|
253
|
-
const loadResult = await loadManifest(options.storePath);
|
|
254
|
-
const manifest = loadResult.manifest;
|
|
255
|
-
let manifestStatus = loadResult.status;
|
|
256
|
-
let rebuildPerformed = false;
|
|
257
|
-
const existingCount = await options.store.count();
|
|
258
|
-
if (options.force || (manifestStatus !== "ok" && existingCount > 0)) {
|
|
259
|
-
await options.store.clear();
|
|
260
|
-
options.keywordIndex?.clear();
|
|
261
|
-
for (const key of Object.keys(manifest.files)) {
|
|
262
|
-
delete manifest.files[key];
|
|
263
|
-
}
|
|
264
|
-
manifest.lastIndexedAt = undefined;
|
|
265
|
-
rebuildPerformed = existingCount > 0 || Boolean(options.force);
|
|
266
|
-
if (manifestStatus !== "ok" && existingCount > 0) {
|
|
267
|
-
logger.warn("Manifest missing or corrupt; rebuilding full index.");
|
|
268
|
-
}
|
|
269
|
-
manifestStatus = options.force ? "missing" : manifestStatus;
|
|
270
|
-
}
|
|
271
|
-
const stats = createIndexStats(workspaceFiles.length, manifestStatus);
|
|
272
|
-
stats.rebuildPerformed = rebuildPerformed;
|
|
273
|
-
const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
|
|
274
|
-
for (const indexedPath of Object.keys(manifest.files)) {
|
|
275
|
-
if (!currentPaths.has(indexedPath)) {
|
|
276
|
-
await options.store.deleteByFilePath(indexedPath);
|
|
277
|
-
options.keywordIndex?.removeByFilePath(indexedPath);
|
|
278
|
-
delete manifest.files[indexedPath];
|
|
279
|
-
stats.deletedFiles++;
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
const limit = pLimit(options.config.indexing.concurrency);
|
|
283
|
-
const workerResults = await Promise.all(workspaceFiles.map((file) => limit(async () => {
|
|
284
|
-
const previous = manifest.files[file.normalizedPath];
|
|
285
|
-
const fileLabel = path.relative(options.cwd, file.filePath);
|
|
286
|
-
if (file.isEmpty) {
|
|
287
|
-
if (previous) {
|
|
288
|
-
await options.store.deleteByFilePath(file.normalizedPath);
|
|
289
|
-
options.keywordIndex?.removeByFilePath(file.normalizedPath);
|
|
290
|
-
logger.info(` ${fileLabel} (empty, removed from index)`);
|
|
291
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: true, hadChunks: false };
|
|
292
|
-
}
|
|
293
|
-
logger.info(` ${fileLabel} (empty, skipped)`);
|
|
294
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: false, hadChunks: false };
|
|
295
|
-
}
|
|
296
|
-
if (file.isTooSmall) {
|
|
297
|
-
if (previous) {
|
|
298
|
-
await options.store.deleteByFilePath(file.normalizedPath);
|
|
299
|
-
options.keywordIndex?.removeByFilePath(file.normalizedPath);
|
|
300
|
-
logger.info(` ${fileLabel} (too small, removed from index)`);
|
|
301
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: true, hadChunks: false };
|
|
302
|
-
}
|
|
303
|
-
logger.info(` ${fileLabel} (too small, skipped)`);
|
|
304
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: false, hadChunks: false };
|
|
305
|
-
}
|
|
306
|
-
if (previous && previous.hash === file.hash) {
|
|
307
|
-
logger.info(` ${fileLabel} (unchanged)`);
|
|
308
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false };
|
|
309
|
-
}
|
|
310
|
-
let isModified = false;
|
|
311
|
-
if (previous) {
|
|
312
|
-
await options.store.deleteByFilePath(file.normalizedPath);
|
|
313
|
-
options.keywordIndex?.removeByFilePath(file.normalizedPath);
|
|
314
|
-
isModified = true;
|
|
315
|
-
}
|
|
316
|
-
let chunks;
|
|
317
|
-
// Images: 1 image → 1 chunk (description as content, no paragraph splitting)
|
|
318
|
-
if (isImageFile(file.filePath, SUPPORTED_IMAGE_EXTENSIONS) && file.content.trim().length > 0) {
|
|
319
|
-
const imgExt = path.extname(file.filePath).toLowerCase();
|
|
320
|
-
const imgRelPath = path.relative(options.cwd, file.filePath).replace(/\\/g, "/");
|
|
321
|
-
const metaHeader = `[image] [${imgExt.slice(1)}] [${imgRelPath}]`;
|
|
322
|
-
chunks = [{
|
|
323
|
-
id: uuid(),
|
|
324
|
-
content: metaHeader + " " + file.content,
|
|
325
|
-
metadata: {
|
|
326
|
-
filePath: file.filePath,
|
|
327
|
-
startLine: 1,
|
|
328
|
-
endLine: 1,
|
|
329
|
-
language: "image",
|
|
330
|
-
contentType: "image",
|
|
331
|
-
},
|
|
332
|
-
}];
|
|
333
|
-
}
|
|
334
|
-
else {
|
|
335
|
-
chunks = await chunkFile(file.filePath, file.content, options.config.chunking?.nodeTypes).catch((err) => {
|
|
336
|
-
logger.warn(` ${fileLabel} (chunking failed: ${err.message})`);
|
|
337
|
-
return null;
|
|
338
|
-
});
|
|
339
|
-
}
|
|
340
|
-
if (chunks === null || chunks.length === 0) {
|
|
341
|
-
if (chunks === null) {
|
|
342
|
-
// Chunking failed — keep previous index state if file was indexed before
|
|
343
|
-
if (previous) {
|
|
344
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
|
|
345
|
-
}
|
|
346
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
|
|
347
|
-
}
|
|
348
|
-
logger.info(` ${fileLabel} (0 chunks, removed from index)`);
|
|
349
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
|
|
350
|
-
}
|
|
351
|
-
options.keywordIndex?.addChunks(chunks);
|
|
352
|
-
try {
|
|
353
|
-
const docPrefix = options.config.embedding.documentPrefix ?? "";
|
|
354
|
-
const relPath = path.relative(options.cwd, file.filePath).replace(/\\/g, "/");
|
|
355
|
-
const isImage = isImageFile(file.filePath, SUPPORTED_IMAGE_EXTENSIONS);
|
|
356
|
-
const metaHeader = isImage
|
|
357
|
-
? ""
|
|
358
|
-
: buildFileMetadataHeader(file.filePath, options.cwd, file.content);
|
|
359
|
-
const textToEmbed = [];
|
|
360
|
-
if (options.descriptionProvider) {
|
|
361
|
-
let descriptionMap = null;
|
|
362
|
-
const nonImageChunks = chunks.filter(c => c.metadata.contentType !== "image");
|
|
363
|
-
if (nonImageChunks.length > 1) {
|
|
364
|
-
try {
|
|
365
|
-
descriptionMap = await options.descriptionProvider.generateBatchDescriptions(nonImageChunks);
|
|
366
|
-
}
|
|
367
|
-
catch (err) {
|
|
368
|
-
logger.warn(`Batch description failed, falling back to individual: ${err.message}`);
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
for (const chunk of chunks) {
|
|
372
|
-
if (isImage) {
|
|
373
|
-
chunk.description = chunk.content;
|
|
374
|
-
textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
|
|
375
|
-
}
|
|
376
|
-
else {
|
|
377
|
-
const batchDesc = descriptionMap?.get(chunk.id);
|
|
378
|
-
if (batchDesc && batchDesc.trim().length > 0) {
|
|
379
|
-
chunk.description = batchDesc;
|
|
380
|
-
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
|
|
381
|
-
}
|
|
382
|
-
else {
|
|
383
|
-
try {
|
|
384
|
-
chunk.description = await options.descriptionProvider.generateDescription(chunk);
|
|
385
|
-
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
|
|
386
|
-
}
|
|
387
|
-
catch (err) {
|
|
388
|
-
logger.warn(`Description generation failed for ${chunk.id}, falling back to content: ${err.message}`);
|
|
389
|
-
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.content);
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
}
|
|
394
|
-
}
|
|
395
|
-
else {
|
|
396
|
-
for (const chunk of chunks) {
|
|
397
|
-
if (isImage) {
|
|
398
|
-
chunk.description = chunk.content;
|
|
399
|
-
textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
|
|
400
|
-
}
|
|
401
|
-
else {
|
|
402
|
-
chunk.description = `lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}, ${chunk.metadata.language}`;
|
|
403
|
-
textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
const embeddings = await embedBatch(options.embedder, textToEmbed, options.config.indexing.embedBatchSize, "document");
|
|
408
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
409
|
-
const emb = embeddings[i];
|
|
410
|
-
if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
|
|
411
|
-
chunks[i].embedding = emb;
|
|
412
|
-
}
|
|
413
|
-
else {
|
|
414
|
-
chunks[i].embedding = undefined;
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
const validChunks = chunks.filter((c) => c.embedding && c.embedding.length > 0);
|
|
418
|
-
if (validChunks.length > 0) {
|
|
419
|
-
await options.store.addChunks(validChunks);
|
|
420
|
-
}
|
|
421
|
-
logger.info(` ${fileLabel} (${chunks.length} chunks${isModified ? ", modified" : ", new"})`);
|
|
422
|
-
return {
|
|
423
|
-
normalizedPath: file.normalizedPath,
|
|
424
|
-
hash: file.hash,
|
|
425
|
-
chunkCount: chunks.length,
|
|
426
|
-
fileLabel,
|
|
427
|
-
isNew: !isModified && !previous,
|
|
428
|
-
isModified,
|
|
429
|
-
isUnchanged: false,
|
|
430
|
-
isEmpty: false,
|
|
431
|
-
isTooSmall: false,
|
|
432
|
-
isRemoved: false,
|
|
433
|
-
hadChunks: chunks.length > 0,
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
catch (err) {
|
|
437
|
-
logger.warn(` ${fileLabel} (embed/store failed: ${err.message})`);
|
|
438
|
-
if (previous) {
|
|
439
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
|
|
440
|
-
}
|
|
441
|
-
return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
|
|
442
|
-
}
|
|
443
|
-
})));
|
|
444
|
-
for (const result of workerResults) {
|
|
445
|
-
if (result.isEmpty) {
|
|
446
|
-
stats.skippedEmptyFiles++;
|
|
447
|
-
if (result.isRemoved) {
|
|
448
|
-
delete manifest.files[result.normalizedPath];
|
|
449
|
-
stats.removedFiles++;
|
|
450
|
-
}
|
|
451
|
-
continue;
|
|
452
|
-
}
|
|
453
|
-
if (result.isTooSmall) {
|
|
454
|
-
stats.skippedSmallFiles++;
|
|
455
|
-
if (result.isRemoved) {
|
|
456
|
-
delete manifest.files[result.normalizedPath];
|
|
457
|
-
stats.removedFiles++;
|
|
458
|
-
}
|
|
459
|
-
continue;
|
|
460
|
-
}
|
|
461
|
-
if (result.isUnchanged) {
|
|
462
|
-
stats.unchangedFiles++;
|
|
463
|
-
continue;
|
|
464
|
-
}
|
|
465
|
-
if (result.isRemoved) {
|
|
466
|
-
delete manifest.files[result.normalizedPath];
|
|
467
|
-
stats.removedFiles++;
|
|
468
|
-
continue;
|
|
469
|
-
}
|
|
470
|
-
if (result.isModified) {
|
|
471
|
-
stats.modifiedFiles++;
|
|
472
|
-
}
|
|
473
|
-
else if (result.isNew) {
|
|
474
|
-
stats.newFiles++;
|
|
475
|
-
}
|
|
476
|
-
if (result.chunkCount > 0) {
|
|
477
|
-
manifest.files[result.normalizedPath] = {
|
|
478
|
-
hash: result.hash,
|
|
479
|
-
chunkCount: result.chunkCount,
|
|
480
|
-
indexedAt: Date.now(),
|
|
481
|
-
};
|
|
482
|
-
stats.totalChunks += result.chunkCount;
|
|
483
|
-
stats.batchesFlushed++;
|
|
484
|
-
}
|
|
485
|
-
}
|
|
486
|
-
manifest.lastIndexedAt = Date.now();
|
|
487
|
-
await saveManifest(options.storePath, manifest);
|
|
488
|
-
await options.keywordIndex?.save(options.storePath);
|
|
489
|
-
stats.finalCount = await options.store.count();
|
|
490
|
-
return stats;
|
|
491
|
-
}
|
|
492
|
-
export async function getIndexStatusSummary(cwd, storePath, config, store) {
|
|
493
|
-
const workspaceFiles = await scanWorkspace(cwd, config);
|
|
494
|
-
const loadResult = await loadManifest(storePath);
|
|
495
|
-
const storeCount = await store.count();
|
|
496
|
-
if (loadResult.status !== "ok") {
|
|
497
|
-
return {
|
|
498
|
-
manifestStatus: loadResult.status,
|
|
499
|
-
manifestEntries: 0,
|
|
500
|
-
upToDateFiles: 0,
|
|
501
|
-
pendingFiles: workspaceFiles.length,
|
|
502
|
-
rebuildRequired: storeCount > 0,
|
|
503
|
-
};
|
|
504
|
-
}
|
|
505
|
-
const manifest = loadResult.manifest;
|
|
506
|
-
const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
|
|
507
|
-
let upToDateFiles = 0;
|
|
508
|
-
let pendingFiles = 0;
|
|
509
|
-
for (const file of workspaceFiles) {
|
|
510
|
-
const previous = manifest.files[file.normalizedPath];
|
|
511
|
-
if (file.isEmpty || file.isTooSmall) {
|
|
512
|
-
if (previous)
|
|
513
|
-
pendingFiles++;
|
|
514
|
-
continue;
|
|
515
|
-
}
|
|
516
|
-
if (previous && previous.hash === file.hash) {
|
|
517
|
-
upToDateFiles++;
|
|
518
|
-
}
|
|
519
|
-
else {
|
|
520
|
-
pendingFiles++;
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
for (const indexedPath of Object.keys(manifest.files)) {
|
|
524
|
-
if (!currentPaths.has(indexedPath)) {
|
|
525
|
-
pendingFiles++;
|
|
526
|
-
}
|
|
527
|
-
}
|
|
528
|
-
return {
|
|
529
|
-
manifestStatus: loadResult.status,
|
|
530
|
-
manifestEntries: Object.keys(manifest.files).length,
|
|
531
|
-
upToDateFiles,
|
|
532
|
-
pendingFiles,
|
|
533
|
-
lastIndexedAt: manifest.lastIndexedAt,
|
|
534
|
-
rebuildRequired: false,
|
|
535
|
-
};
|
|
536
|
-
}
|
|
537
|
-
export function createWatchPassScheduler(runPass, onError, debounceMs = 300) {
|
|
538
|
-
let timer = null;
|
|
539
|
-
let running = false;
|
|
540
|
-
let rerunRequested = false;
|
|
541
|
-
let closed = false;
|
|
542
|
-
const waiters = [];
|
|
543
|
-
function resolveWaiters() {
|
|
544
|
-
if (running || timer || rerunRequested)
|
|
545
|
-
return;
|
|
546
|
-
while (waiters.length > 0) {
|
|
547
|
-
waiters.shift()?.();
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
function schedule() {
|
|
551
|
-
if (closed)
|
|
552
|
-
return;
|
|
553
|
-
if (timer)
|
|
554
|
-
clearTimeout(timer);
|
|
555
|
-
timer = setTimeout(() => {
|
|
556
|
-
timer = null;
|
|
557
|
-
void execute();
|
|
558
|
-
}, debounceMs);
|
|
559
|
-
}
|
|
560
|
-
async function execute() {
|
|
561
|
-
if (closed)
|
|
562
|
-
return;
|
|
563
|
-
if (running) {
|
|
564
|
-
rerunRequested = true;
|
|
565
|
-
return;
|
|
566
|
-
}
|
|
567
|
-
running = true;
|
|
568
|
-
try {
|
|
569
|
-
await runPass();
|
|
570
|
-
}
|
|
571
|
-
catch (error) {
|
|
572
|
-
onError(error);
|
|
573
|
-
}
|
|
574
|
-
finally {
|
|
575
|
-
running = false;
|
|
576
|
-
if (rerunRequested) {
|
|
577
|
-
rerunRequested = false;
|
|
578
|
-
schedule();
|
|
579
|
-
}
|
|
580
|
-
else {
|
|
581
|
-
resolveWaiters();
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
return {
|
|
586
|
-
notifyChange() {
|
|
587
|
-
if (closed)
|
|
588
|
-
return;
|
|
589
|
-
if (running) {
|
|
590
|
-
rerunRequested = true;
|
|
591
|
-
return;
|
|
592
|
-
}
|
|
593
|
-
schedule();
|
|
594
|
-
},
|
|
595
|
-
waitForIdle() {
|
|
596
|
-
if (!running && !timer && !rerunRequested) {
|
|
597
|
-
return Promise.resolve();
|
|
598
|
-
}
|
|
599
|
-
return new Promise((resolve) => {
|
|
600
|
-
waiters.push(resolve);
|
|
601
|
-
});
|
|
602
|
-
},
|
|
603
|
-
close() {
|
|
604
|
-
closed = true;
|
|
605
|
-
if (timer) {
|
|
606
|
-
clearTimeout(timer);
|
|
607
|
-
timer = null;
|
|
608
|
-
}
|
|
609
|
-
resolveWaiters();
|
|
610
|
-
},
|
|
611
|
-
};
|
|
612
|
-
}
|
|
613
|
-
export function createWatchIgnore(cwd, config, storePath) {
|
|
614
|
-
const manifestPath = manifestPathFor(storePath);
|
|
615
|
-
const excludeDirs = new Set(config.indexing.excludeDirs);
|
|
616
|
-
return (watchedPath) => {
|
|
617
|
-
const resolved = path.resolve(watchedPath);
|
|
618
|
-
if (resolved.startsWith(storePath))
|
|
619
|
-
return true;
|
|
620
|
-
if (resolved === manifestPath)
|
|
621
|
-
return true;
|
|
622
|
-
const relative = path.relative(cwd, resolved);
|
|
623
|
-
if (!relative || relative.startsWith(".."))
|
|
624
|
-
return false;
|
|
625
|
-
const segments = relative.split(path.sep);
|
|
626
|
-
return segments.some((segment) => excludeDirs.has(segment));
|
|
627
|
-
};
|
|
6
|
+
return scanWorkspaceFiles(cwd, config, logger);
|
|
628
7
|
}
|
|
629
8
|
//# sourceMappingURL=indexer.js.map
|