opencode-rag-plugin 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/ReadMe.md +423 -0
- package/dist/chunker/base.d.ts +10 -0
- package/dist/chunker/base.js +34 -0
- package/dist/chunker/base.js.map +1 -0
- package/dist/chunker/c.d.ts +8 -0
- package/dist/chunker/c.js +16 -0
- package/dist/chunker/c.js.map +1 -0
- package/dist/chunker/cpp.d.ts +8 -0
- package/dist/chunker/cpp.js +17 -0
- package/dist/chunker/cpp.js.map +1 -0
- package/dist/chunker/csharp.d.ts +8 -0
- package/dist/chunker/csharp.js +17 -0
- package/dist/chunker/csharp.js.map +1 -0
- package/dist/chunker/css.d.ts +8 -0
- package/dist/chunker/css.js +14 -0
- package/dist/chunker/css.js.map +1 -0
- package/dist/chunker/factory.d.ts +27 -0
- package/dist/chunker/factory.js +138 -0
- package/dist/chunker/factory.js.map +1 -0
- package/dist/chunker/fallback.d.ts +8 -0
- package/dist/chunker/fallback.js +34 -0
- package/dist/chunker/fallback.js.map +1 -0
- package/dist/chunker/go.d.ts +8 -0
- package/dist/chunker/go.js +13 -0
- package/dist/chunker/go.js.map +1 -0
- package/dist/chunker/grammar.d.ts +12 -0
- package/dist/chunker/grammar.js +43 -0
- package/dist/chunker/grammar.js.map +1 -0
- package/dist/chunker/html.d.ts +8 -0
- package/dist/chunker/html.js +12 -0
- package/dist/chunker/html.js.map +1 -0
- package/dist/chunker/java.d.ts +8 -0
- package/dist/chunker/java.js +14 -0
- package/dist/chunker/java.js.map +1 -0
- package/dist/chunker/javascript.d.ts +8 -0
- package/dist/chunker/javascript.js +15 -0
- package/dist/chunker/javascript.js.map +1 -0
- package/dist/chunker/json.d.ts +8 -0
- package/dist/chunker/json.js +11 -0
- package/dist/chunker/json.js.map +1 -0
- package/dist/chunker/kotlin.d.ts +8 -0
- package/dist/chunker/kotlin.js +15 -0
- package/dist/chunker/kotlin.js.map +1 -0
- package/dist/chunker/loader.d.ts +2 -0
- package/dist/chunker/loader.js +27 -0
- package/dist/chunker/loader.js.map +1 -0
- package/dist/chunker/markdown.d.ts +7 -0
- package/dist/chunker/markdown.js +96 -0
- package/dist/chunker/markdown.js.map +1 -0
- package/dist/chunker/pdf.d.ts +8 -0
- package/dist/chunker/pdf.js +93 -0
- package/dist/chunker/pdf.js.map +1 -0
- package/dist/chunker/python.d.ts +8 -0
- package/dist/chunker/python.js +13 -0
- package/dist/chunker/python.js.map +1 -0
- package/dist/chunker/razor.d.ts +7 -0
- package/dist/chunker/razor.js +85 -0
- package/dist/chunker/razor.js.map +1 -0
- package/dist/chunker/ruby.d.ts +8 -0
- package/dist/chunker/ruby.js +14 -0
- package/dist/chunker/ruby.js.map +1 -0
- package/dist/chunker/rust.d.ts +8 -0
- package/dist/chunker/rust.js +17 -0
- package/dist/chunker/rust.js.map +1 -0
- package/dist/chunker/sln.d.ts +9 -0
- package/dist/chunker/sln.js +65 -0
- package/dist/chunker/sln.js.map +1 -0
- package/dist/chunker/swift.d.ts +8 -0
- package/dist/chunker/swift.js +17 -0
- package/dist/chunker/swift.js.map +1 -0
- package/dist/chunker/tex.d.ts +7 -0
- package/dist/chunker/tex.js +93 -0
- package/dist/chunker/tex.js.map +1 -0
- package/dist/chunker/typescript.d.ts +8 -0
- package/dist/chunker/typescript.js +17 -0
- package/dist/chunker/typescript.js.map +1 -0
- package/dist/chunker/uuid.d.ts +1 -0
- package/dist/chunker/uuid.js +8 -0
- package/dist/chunker/uuid.js.map +1 -0
- package/dist/chunker/xml.d.ts +8 -0
- package/dist/chunker/xml.js +11 -0
- package/dist/chunker/xml.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +291 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/config.d.ts +59 -0
- package/dist/core/config.js +127 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/fileLogger.d.ts +6 -0
- package/dist/core/fileLogger.js +32 -0
- package/dist/core/fileLogger.js.map +1 -0
- package/dist/core/interfaces.d.ts +31 -0
- package/dist/core/interfaces.js +2 -0
- package/dist/core/interfaces.js.map +1 -0
- package/dist/core/manifest.d.ts +21 -0
- package/dist/core/manifest.js +48 -0
- package/dist/core/manifest.js.map +1 -0
- package/dist/embedder/factory.d.ts +4 -0
- package/dist/embedder/factory.js +27 -0
- package/dist/embedder/factory.js.map +1 -0
- package/dist/embedder/http.d.ts +11 -0
- package/dist/embedder/http.js +309 -0
- package/dist/embedder/http.js.map +1 -0
- package/dist/embedder/ollama.d.ts +14 -0
- package/dist/embedder/ollama.js +60 -0
- package/dist/embedder/ollama.js.map +1 -0
- package/dist/embedder/openai.d.ts +12 -0
- package/dist/embedder/openai.js +33 -0
- package/dist/embedder/openai.js.map +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer.d.ts +49 -0
- package/dist/indexer.js +336 -0
- package/dist/indexer.js.map +1 -0
- package/dist/plugin-entry.d.ts +4 -0
- package/dist/plugin-entry.js +5 -0
- package/dist/plugin-entry.js.map +1 -0
- package/dist/plugin.d.ts +22 -0
- package/dist/plugin.js +477 -0
- package/dist/plugin.js.map +1 -0
- package/dist/retriever/retriever.d.ts +5 -0
- package/dist/retriever/retriever.js +14 -0
- package/dist/retriever/retriever.js.map +1 -0
- package/dist/types/opencode-plugin.d.ts +51 -0
- package/dist/vectorstore/lancedb.d.ts +18 -0
- package/dist/vectorstore/lancedb.js +196 -0
- package/dist/vectorstore/lancedb.js.map +1 -0
- package/dist/watcher.d.ts +14 -0
- package/dist/watcher.js +88 -0
- package/dist/watcher.js.map +1 -0
- package/package.json +82 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import * as lancedb from "@lancedb/lancedb";
|
|
2
|
+
import { normalizeFilePath } from "../core/manifest.js";
|
|
3
|
+
const TABLE_NAME = "chunks";
|
|
4
|
+
const VECTOR_COLUMN = "embedding";
|
|
5
|
+
export class LanceDBStore {
|
|
6
|
+
dbPath;
|
|
7
|
+
vectorDimension;
|
|
8
|
+
db = null;
|
|
9
|
+
table = null;
|
|
10
|
+
constructor(dbPath, vectorDimension = 384) {
|
|
11
|
+
this.dbPath = dbPath;
|
|
12
|
+
this.vectorDimension = vectorDimension;
|
|
13
|
+
}
|
|
14
|
+
async getDb() {
|
|
15
|
+
if (!this.db) {
|
|
16
|
+
this.db = await lancedb.connect(this.dbPath);
|
|
17
|
+
}
|
|
18
|
+
return this.db;
|
|
19
|
+
}
|
|
20
|
+
async getTable() {
|
|
21
|
+
if (this.table)
|
|
22
|
+
return this.table;
|
|
23
|
+
const db = await this.getDb();
|
|
24
|
+
const tableNames = await db.tableNames();
|
|
25
|
+
if (tableNames.includes(TABLE_NAME)) {
|
|
26
|
+
this.table = await db.openTable(TABLE_NAME);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
const seedRow = {
|
|
30
|
+
id: "__seed__",
|
|
31
|
+
content: "",
|
|
32
|
+
embedding: new Array(this.vectorDimension).fill(0),
|
|
33
|
+
filePath: "",
|
|
34
|
+
startLine: 0,
|
|
35
|
+
endLine: 0,
|
|
36
|
+
language: "",
|
|
37
|
+
};
|
|
38
|
+
this.table = await db.createTable({
|
|
39
|
+
name: TABLE_NAME,
|
|
40
|
+
data: [seedRow],
|
|
41
|
+
mode: "overwrite",
|
|
42
|
+
});
|
|
43
|
+
await this.table.delete('id = "__seed__"');
|
|
44
|
+
}
|
|
45
|
+
return this.table;
|
|
46
|
+
}
|
|
47
|
+
async addChunks(chunks) {
|
|
48
|
+
if (chunks.length === 0)
|
|
49
|
+
return;
|
|
50
|
+
const table = await this.getTable();
|
|
51
|
+
const rows = chunks
|
|
52
|
+
.filter((c) => c.embedding && c.embedding.length > 0)
|
|
53
|
+
.map((c) => ({
|
|
54
|
+
id: c.id,
|
|
55
|
+
content: c.content,
|
|
56
|
+
embedding: c.embedding,
|
|
57
|
+
filePath: normalizeFilePath(c.metadata.filePath),
|
|
58
|
+
startLine: c.metadata.startLine,
|
|
59
|
+
endLine: c.metadata.endLine,
|
|
60
|
+
language: c.metadata.language,
|
|
61
|
+
}));
|
|
62
|
+
if (rows.length === 0)
|
|
63
|
+
return;
|
|
64
|
+
await table.add(rows);
|
|
65
|
+
}
|
|
66
|
+
async search(embedding, topK) {
|
|
67
|
+
try {
|
|
68
|
+
return await this.searchInternal(embedding, topK);
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
if (this.isCorruptionError(err)) {
|
|
72
|
+
const repaired = await this.tryRepair();
|
|
73
|
+
if (repaired) {
|
|
74
|
+
return this.searchInternal(embedding, topK);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return [];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
async searchInternal(embedding, topK) {
|
|
81
|
+
const db = await this.getDb();
|
|
82
|
+
const tableNames = await db.tableNames();
|
|
83
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
84
|
+
return [];
|
|
85
|
+
const table = await this.getTable();
|
|
86
|
+
const count = await table.countRows();
|
|
87
|
+
if (count === 0)
|
|
88
|
+
return [];
|
|
89
|
+
const results = await table.search(embedding).limit(topK).toArray();
|
|
90
|
+
return results.map((row) => {
|
|
91
|
+
const distance = row._distance ?? 0;
|
|
92
|
+
const score = 1 / (1 + distance);
|
|
93
|
+
return {
|
|
94
|
+
score,
|
|
95
|
+
chunk: {
|
|
96
|
+
id: row.id,
|
|
97
|
+
content: row.content,
|
|
98
|
+
metadata: {
|
|
99
|
+
filePath: row.filePath,
|
|
100
|
+
startLine: row.startLine,
|
|
101
|
+
endLine: row.endLine,
|
|
102
|
+
language: row.language,
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
async count() {
|
|
109
|
+
try {
|
|
110
|
+
const db = await this.getDb();
|
|
111
|
+
const tableNames = await db.tableNames();
|
|
112
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
113
|
+
return 0;
|
|
114
|
+
const table = await this.getTable();
|
|
115
|
+
return await table.countRows();
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return 0;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async clear() {
|
|
122
|
+
try {
|
|
123
|
+
const db = await this.getDb();
|
|
124
|
+
const tableNames = await db.tableNames();
|
|
125
|
+
if (tableNames.includes(TABLE_NAME)) {
|
|
126
|
+
await db.dropTable(TABLE_NAME);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
}
|
|
131
|
+
this.table = null;
|
|
132
|
+
}
|
|
133
|
+
async deleteByFilePath(filePath) {
|
|
134
|
+
const db = await this.getDb();
|
|
135
|
+
const tableNames = await db.tableNames();
|
|
136
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
137
|
+
return;
|
|
138
|
+
const table = await this.getTable();
|
|
139
|
+
const normalizedPath = normalizeFilePath(filePath).replace(/'/g, "''");
|
|
140
|
+
await table.delete(`filePath = '${normalizedPath}'`);
|
|
141
|
+
}
|
|
142
|
+
isCorruptionError(err) {
|
|
143
|
+
if (err instanceof Error) {
|
|
144
|
+
return (err.message.includes("Not found") &&
|
|
145
|
+
err.message.includes(".lance") &&
|
|
146
|
+
err.message.includes("lance error"));
|
|
147
|
+
}
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
async tryRepair() {
|
|
151
|
+
try {
|
|
152
|
+
this.table = null;
|
|
153
|
+
this.db = null;
|
|
154
|
+
const db = await this.getDb();
|
|
155
|
+
const tableNames = await db.tableNames();
|
|
156
|
+
if (!tableNames.includes(TABLE_NAME))
|
|
157
|
+
return false;
|
|
158
|
+
const table = await db.openTable(TABLE_NAME);
|
|
159
|
+
let versions;
|
|
160
|
+
try {
|
|
161
|
+
versions = await table.listVersions();
|
|
162
|
+
}
|
|
163
|
+
catch {
|
|
164
|
+
await db.dropTable(TABLE_NAME).catch(() => { });
|
|
165
|
+
this.table = null;
|
|
166
|
+
return true;
|
|
167
|
+
}
|
|
168
|
+
if (versions.length <= 1) {
|
|
169
|
+
await db.dropTable(TABLE_NAME).catch(() => { });
|
|
170
|
+
this.table = null;
|
|
171
|
+
return true;
|
|
172
|
+
}
|
|
173
|
+
const sorted = [...versions].sort((a, b) => b.version - a.version);
|
|
174
|
+
for (const ver of sorted.slice(1)) {
|
|
175
|
+
try {
|
|
176
|
+
await table.checkout(ver.version);
|
|
177
|
+
const count = await table.countRows();
|
|
178
|
+
await table.restore();
|
|
179
|
+
await table.checkoutLatest();
|
|
180
|
+
this.table = table;
|
|
181
|
+
return true;
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
await db.dropTable(TABLE_NAME).catch(() => { });
|
|
188
|
+
this.table = null;
|
|
189
|
+
return true;
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
return false;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
//# sourceMappingURL=lancedb.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lancedb.js","sourceRoot":"","sources":["../../src/vectorstore/lancedb.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAG5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD,MAAM,UAAU,GAAG,QAAQ,CAAC;AAC5B,MAAM,aAAa,GAAG,WAAW,CAAC;AAYlC,MAAM,OAAO,YAAY;IACf,MAAM,CAAS;IACf,eAAe,CAAS;IACxB,EAAE,GAAsB,IAAI,CAAC;IAC7B,KAAK,GAAiB,IAAI,CAAC;IAEnC,YAAY,MAAc,EAAE,kBAA0B,GAAG;QACvD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/C,CAAC;QACD,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,QAAQ;QACpB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QAElC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QAEzC,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAa;gBACxB,EAAE,EAAE,UAAU;gBACd,OAAO,EAAE,EAAE;gBACX,SAAS,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAClD,QAAQ,EAAE,EAAE;gBACZ,SAAS,EAAE,CAAC;gBACZ,OAAO,EAAE,CAAC;gBACV,QAAQ,EAAE,EAAE;aACb,CAAC;YAEF,IAAI,CAAC,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC;gBAChC,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,CAAC,OAAO,CAAyC;gBACvD,IAAI,EAAE,WAAW;aAClB,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAe;QAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEhC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,IAAI,GAAe,MAAM;aAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;aACpD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACX,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,SAAS,EAAE,CAAC,CAAC,SAAU;YACvB,QAAQ,EAAE,iBAAiB,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAChD,SAAS,EAAE,CAAC,CAAC,QAAQ,CAAC,SAAS;YAC/B,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO;YAC3B,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ;SAC9B,CAAC,CAAC,CAAC;QAEN,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAE9B,MAAM,KAAK,CAAC,GAAG,CAAC,IAA4C,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAmB,EAAE,IAAY;QAC5C,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;QACpD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACxC,IAAI,QAAQ,EAAE,CAAC;oBACb,OAAO,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,SAAmB,EAAE,IAAY;QAC5D,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,OAAO,EAAE,CAAC;QAEhD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACtC,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE3B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpE,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAA4B,EAAE,EAAE;YAClD,MAAM,QAAQ,GAAI,GAAG,CAAC,SAAoB,IAAI,CAAC,CAAC;YAChD,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;YACjC,OAAO;gBACL,KAAK;gBACL,KAAK,EAAE;oBACL,EAAE,EAAE,GAAG,CAAC,EAAY;oBACpB,OAAO,EAAE,GAAG,CAAC,OAAiB;oBAC9B,QAAQ,EAAE;wBACR,QAAQ,EAAE,GAAG,CAAC,QAAkB;wBAChC,SAAS,EAAE,GAAG,CAAC,SAAmB;wBAClC,OAAO,EAAE,GAAG,CAAC,OAAiB;wBAC9B,QAAQ,EAAE,GAAG,CAAC,QAAkB;qBACjC;iBACF;aACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,CAAC,CAAC;YAE/C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,OAAO,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;YACzC,IAAI,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACpC,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;QACT,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,OAAO;QAE7C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACvE,MAAM,KAAK,CAAC,MAAM,CAAC,eAAe,cAAc,GAAG,CAAC,CAAC;IACvD,CAAC;IAEO,iBAAiB,CAAC,GAAY;QACpC,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;YACzB,OAAO,CACL,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACjC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC9B,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CACpC,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YAEf,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,UAAU,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC;gBAAE,OAAO,KAAK,CAAC;YAEnD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAE7C,IAAI,QAAmB,CAAC;YACxB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;gBAClB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;gBAClB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;YAEnE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAClC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;oBACtC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;oBACtB,MAAM,KAAK,CAAC,cAAc,EAAE,CAAC;oBAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;oBACnB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;YACH,CAAC;YAED,MAAM,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { RagConfig } from "./core/config.js";
|
|
2
|
+
import type { EmbeddingProvider, VectorStore } from "./core/interfaces.js";
|
|
3
|
+
export interface BackgroundIndexer {
|
|
4
|
+
close(): Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
export interface CreateBackgroundIndexerOptions {
|
|
7
|
+
cwd: string;
|
|
8
|
+
storePath: string;
|
|
9
|
+
config: RagConfig;
|
|
10
|
+
store: VectorStore;
|
|
11
|
+
embedder: EmbeddingProvider;
|
|
12
|
+
logFilePath: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function createBackgroundIndexer(options: CreateBackgroundIndexerOptions): BackgroundIndexer;
|
package/dist/watcher.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import chokidar from "chokidar";
|
|
2
|
+
import { appendDebugLog } from "./core/fileLogger.js";
|
|
3
|
+
import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "./indexer.js";
|
|
4
|
+
export function createBackgroundIndexer(options) {
|
|
5
|
+
const { cwd, storePath, config, store, embedder, logFilePath } = options;
|
|
6
|
+
// Fire-and-forget initial index pass
|
|
7
|
+
runIndexPass({
|
|
8
|
+
cwd,
|
|
9
|
+
storePath,
|
|
10
|
+
config,
|
|
11
|
+
store,
|
|
12
|
+
embedder,
|
|
13
|
+
logger: {
|
|
14
|
+
//info: (message) => appendDebugLog(logFilePath, { scope: "autoIndex", message }),
|
|
15
|
+
warn: (message) => appendDebugLog(logFilePath, { scope: "autoIndex", message }),
|
|
16
|
+
},
|
|
17
|
+
}).catch((err) => {
|
|
18
|
+
appendDebugLog(logFilePath, {
|
|
19
|
+
scope: "autoIndex",
|
|
20
|
+
message: "Initial index pass failed",
|
|
21
|
+
error: err,
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
const runPass = async () => {
|
|
25
|
+
try {
|
|
26
|
+
await runIndexPass({
|
|
27
|
+
cwd,
|
|
28
|
+
storePath,
|
|
29
|
+
config,
|
|
30
|
+
store,
|
|
31
|
+
embedder,
|
|
32
|
+
logger: {
|
|
33
|
+
//info: (message) => appendDebugLog(logFilePath, { scope: "autoIndex", message }),
|
|
34
|
+
warn: (message) => appendDebugLog(logFilePath, { scope: "autoIndex", message }),
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
catch (err) {
|
|
39
|
+
appendDebugLog(logFilePath, {
|
|
40
|
+
scope: "autoIndex",
|
|
41
|
+
message: "Watch reindex pass failed",
|
|
42
|
+
error: err,
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
const autoIndexCfg = config.openCode.autoIndex ?? { enabled: true, debounceMs: 5000, intervalMs: 300000 };
|
|
47
|
+
const scheduler = createWatchPassScheduler(runPass, (error) => {
|
|
48
|
+
const message = error.message || String(error);
|
|
49
|
+
appendDebugLog(logFilePath, {
|
|
50
|
+
scope: "autoIndex",
|
|
51
|
+
message: `Watch reindex failed: ${message}`,
|
|
52
|
+
error,
|
|
53
|
+
});
|
|
54
|
+
}, autoIndexCfg.debounceMs);
|
|
55
|
+
const watcher = chokidar.watch(cwd, {
|
|
56
|
+
ignored: createWatchIgnore(cwd, config, storePath),
|
|
57
|
+
ignoreInitial: true,
|
|
58
|
+
persistent: true,
|
|
59
|
+
});
|
|
60
|
+
const handleChange = () => scheduler.notifyChange();
|
|
61
|
+
watcher.on("add", handleChange);
|
|
62
|
+
watcher.on("change", handleChange);
|
|
63
|
+
watcher.on("unlink", handleChange);
|
|
64
|
+
watcher.on("unlinkDir", handleChange);
|
|
65
|
+
watcher.on("addDir", handleChange);
|
|
66
|
+
watcher.on("error", (error) => {
|
|
67
|
+
appendDebugLog(logFilePath, {
|
|
68
|
+
scope: "autoIndex",
|
|
69
|
+
message: `Watcher error: ${error.message}`,
|
|
70
|
+
error,
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
const periodicTimer = setInterval(() => {
|
|
74
|
+
scheduler.notifyChange();
|
|
75
|
+
}, autoIndexCfg.intervalMs);
|
|
76
|
+
return {
|
|
77
|
+
async close() {
|
|
78
|
+
clearInterval(periodicTimer);
|
|
79
|
+
scheduler.close();
|
|
80
|
+
await watcher.close();
|
|
81
|
+
appendDebugLog(logFilePath, {
|
|
82
|
+
scope: "autoIndex",
|
|
83
|
+
message: "Background indexer shut down",
|
|
84
|
+
});
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=watcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../src/watcher.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGtD,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,YAAY,GACb,MAAM,cAAc,CAAC;AAetB,MAAM,UAAU,uBAAuB,CAAC,OAAuC;IAC7E,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC;IAEzE,qCAAqC;IACrC,YAAY,CAAC;QACX,GAAG;QACH,SAAS;QACT,MAAM;QACN,KAAK;QACL,QAAQ;QACR,MAAM,EAAE;YACN,kFAAkF;YAClF,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;SAChF;KACF,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACf,cAAc,CAAC,WAAW,EAAE;YAC1B,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,2BAA2B;YACpC,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;QACxC,IAAI,CAAC;YACH,MAAM,YAAY,CAAC;gBACjB,GAAG;gBACH,SAAS;gBACT,MAAM;gBACN,KAAK;gBACL,QAAQ;gBACR,MAAM,EAAE;oBACN,kFAAkF;oBAClF,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,cAAc,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;iBAChF;aACF,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,cAAc,CAAC,WAAW,EAAE;gBAC1B,KAAK,EAAE,WAAW;gBAClB,OAAO,EAAE,2BAA2B;gBACpC,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IAC1G,MAAM,SAAS,GAAG,wBAAwB,CACxC,OAAO,EACP,CAAC,KAAK,EAAE,EAAE;QACR,MAAM,OAAO,GAAI,KAAe,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1D,cAAc,CAAC,WAAW,EAAE;YAC1B,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,yBAAyB,OAAO,EAAE;YAC3C,KAAK;SACN,CAAC,CAAC;IACL,CAAC,EACD,YAAY,CAAC,UAAU,CACxB,CAAC;IAEF,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE;QAClC,OAAO,EAAE,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC;QAClD,aAAa,EAAE,IAAI;QACnB,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;IACpD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAChC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACtC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IACnC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;QAC5B,cAAc,CAAC,WAAW,EAAE;YAC1B,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE,kBAAmB,KAAe,CAAC,OAAO,EAAE;YACrD,KAAK;SACN,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QACrC,SAAS,CAAC,YAAY,EAAE,CAAC;IAC3B,CAAC,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;IAE5B,OAAO;QACL,KAAK,CAAC,KAAK;YACT,aAAa,CAAC,aAAa,CAAC,CAAC;YAC7B,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,cAAc,CAAC,WAAW,EAAE;gBAC1B,KAAK,EAAE,WAAW;gBAClB,OAAO,EAAE,8BAA8B;aACxC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "opencode-rag-plugin",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "OpenCode plugin for local-first RAG-based semantic code search",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/plugin-entry.js",
|
|
7
|
+
"module": "./dist/plugin-entry.js",
|
|
8
|
+
"types": "./dist/plugin-entry.d.ts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/plugin-entry.js",
|
|
12
|
+
"require": "./dist/plugin-entry.js",
|
|
13
|
+
"types": "./dist/plugin-entry.d.ts"
|
|
14
|
+
},
|
|
15
|
+
"./server": {
|
|
16
|
+
"import": "./dist/plugin-entry.js",
|
|
17
|
+
"require": "./dist/plugin-entry.js",
|
|
18
|
+
"types": "./dist/plugin-entry.d.ts"
|
|
19
|
+
},
|
|
20
|
+
"./library": {
|
|
21
|
+
"import": "./dist/index.js",
|
|
22
|
+
"require": "./dist/index.js",
|
|
23
|
+
"types": "./dist/index.d.ts"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
"files": [
|
|
27
|
+
"dist",
|
|
28
|
+
"README.md",
|
|
29
|
+
"LICENSE"
|
|
30
|
+
],
|
|
31
|
+
"bin": {
|
|
32
|
+
"opencode-rag": "./dist/cli.js"
|
|
33
|
+
},
|
|
34
|
+
"scripts": {
|
|
35
|
+
"build": "tsc -p tsconfig.build.json",
|
|
36
|
+
"postbuild": "mkdir -p dist/types && cp src/types/opencode-plugin.d.ts dist/types/",
|
|
37
|
+
"prepublishOnly": "npm run build",
|
|
38
|
+
"typecheck": "tsc --noEmit",
|
|
39
|
+
"test": "node --import tsx --test --test-force-exit \"src/**/*.test.ts\"",
|
|
40
|
+
"cli": "tsx src/cli.ts"
|
|
41
|
+
},
|
|
42
|
+
"keywords": [
|
|
43
|
+
"opencode",
|
|
44
|
+
"rag",
|
|
45
|
+
"code-search",
|
|
46
|
+
"embeddings",
|
|
47
|
+
"vector-search",
|
|
48
|
+
"semantic-search",
|
|
49
|
+
"plugin",
|
|
50
|
+
"ai",
|
|
51
|
+
"code"
|
|
52
|
+
],
|
|
53
|
+
"author": "Christoph Döllinger <christoph.doellinger@med.uni-heidelberg.de>",
|
|
54
|
+
"license": "MIT",
|
|
55
|
+
"repository": {
|
|
56
|
+
"type": "git",
|
|
57
|
+
"url": "git+https://github.com/MrDoe/OpenCodeRAG.git"
|
|
58
|
+
},
|
|
59
|
+
"bugs": {
|
|
60
|
+
"url": "https://github.com/MrDoe/OpenCodeRAG/issues"
|
|
61
|
+
},
|
|
62
|
+
"homepage": "https://github.com/MrDoe/OpenCodeRAG#readme",
|
|
63
|
+
"peerDependencies": {
|
|
64
|
+
"@opencode-ai/plugin": ">=1.0.0"
|
|
65
|
+
},
|
|
66
|
+
"dependencies": {
|
|
67
|
+
"@lancedb/lancedb": "^0.29.0",
|
|
68
|
+
"apache-arrow": "18.1.0",
|
|
69
|
+
"canvas": "^3.2.3",
|
|
70
|
+
"chokidar": "^4.0.3",
|
|
71
|
+
"commander": "^13.1.0",
|
|
72
|
+
"pdfjs-dist": "^5.7.284",
|
|
73
|
+
"tree-sitter-wasm": "^1.0.2",
|
|
74
|
+
"web-tree-sitter": "^0.26.9"
|
|
75
|
+
},
|
|
76
|
+
"devDependencies": {
|
|
77
|
+
"@opencode-ai/plugin": "1.15.5",
|
|
78
|
+
"@types/node": "^22.15.0",
|
|
79
|
+
"tsx": "^4.19.4",
|
|
80
|
+
"typescript": "^5.8.3"
|
|
81
|
+
}
|
|
82
|
+
}
|