opencode-rag-plugin 1.6.2 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/ReadMe.md +41 -59
  2. package/dist/chunker/base.d.ts +1 -0
  3. package/dist/chunker/base.js +26 -0
  4. package/dist/chunker/base.js.map +1 -1
  5. package/dist/chunker/cpp.js +0 -3
  6. package/dist/chunker/cpp.js.map +1 -1
  7. package/dist/chunker/csharp.js +2 -4
  8. package/dist/chunker/csharp.js.map +1 -1
  9. package/dist/chunker/factory.d.ts +1 -1
  10. package/dist/chunker/factory.js +9 -2
  11. package/dist/chunker/factory.js.map +1 -1
  12. package/dist/chunker/go.js +0 -1
  13. package/dist/chunker/go.js.map +1 -1
  14. package/dist/chunker/grammar.js +8 -2
  15. package/dist/chunker/grammar.js.map +1 -1
  16. package/dist/chunker/java.js +0 -1
  17. package/dist/chunker/java.js.map +1 -1
  18. package/dist/chunker/javascript.js +0 -2
  19. package/dist/chunker/javascript.js.map +1 -1
  20. package/dist/chunker/kotlin.js +0 -2
  21. package/dist/chunker/kotlin.js.map +1 -1
  22. package/dist/chunker/python.js +0 -1
  23. package/dist/chunker/python.js.map +1 -1
  24. package/dist/chunker/ruby.js +0 -2
  25. package/dist/chunker/ruby.js.map +1 -1
  26. package/dist/chunker/rust.js +0 -1
  27. package/dist/chunker/rust.js.map +1 -1
  28. package/dist/chunker/swift.js +0 -3
  29. package/dist/chunker/swift.js.map +1 -1
  30. package/dist/chunker/typescript.js +0 -2
  31. package/dist/chunker/typescript.js.map +1 -1
  32. package/dist/cli.js +125 -3
  33. package/dist/cli.js.map +1 -1
  34. package/dist/core/config.d.ts +10 -0
  35. package/dist/core/config.js +16 -0
  36. package/dist/core/config.js.map +1 -1
  37. package/dist/eval/index.d.ts +5 -0
  38. package/dist/eval/index.js +4 -0
  39. package/dist/eval/index.js.map +1 -0
  40. package/dist/eval/session-logger.d.ts +16 -0
  41. package/dist/eval/session-logger.js +124 -0
  42. package/dist/eval/session-logger.js.map +1 -0
  43. package/dist/eval/storage.d.ts +12 -0
  44. package/dist/eval/storage.js +184 -0
  45. package/dist/eval/storage.js.map +1 -0
  46. package/dist/eval/types.d.ts +89 -0
  47. package/dist/eval/types.js +17 -0
  48. package/dist/eval/types.js.map +1 -0
  49. package/dist/indexer.js +131 -91
  50. package/dist/indexer.js.map +1 -1
  51. package/dist/opencode/tools.js +4 -6
  52. package/dist/opencode/tools.js.map +1 -1
  53. package/dist/plugin.js +41 -13
  54. package/dist/plugin.js.map +1 -1
  55. package/dist/retriever/keyword-index.js +2 -0
  56. package/dist/retriever/keyword-index.js.map +1 -1
  57. package/dist/tui.js +310 -0
  58. package/dist/tui.js.map +1 -1
  59. package/dist/vectorstore/lancedb.d.ts +4 -0
  60. package/dist/vectorstore/lancedb.js +14 -0
  61. package/dist/vectorstore/lancedb.js.map +1 -1
  62. package/dist/watcher.js +2 -0
  63. package/dist/watcher.js.map +1 -1
  64. package/dist/web/api.d.ts +4 -0
  65. package/dist/web/api.js +197 -0
  66. package/dist/web/api.js.map +1 -0
  67. package/dist/web/server.d.ts +5 -0
  68. package/dist/web/server.js +41 -0
  69. package/dist/web/server.js.map +1 -0
  70. package/dist/web/static.d.ts +1 -0
  71. package/dist/web/static.js +13 -0
  72. package/dist/web/static.js.map +1 -0
  73. package/dist/web/ui/index.html +1078 -0
  74. package/package.json +7 -2
@@ -0,0 +1 @@
1
+ {"version":3,"file":"session-logger.js","sourceRoot":"","sources":["../../src/eval/session-logger.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAgDlD,MAAM,UAAU,mBAAmB,CAAC,SAAiB;IACnD,OAAO;QACL,OAAO,CAAC,KAAgB;YACtB,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC;gBAC/B,IAAI,CAAC,KAAK;oBAAE,OAAO;gBAEnB,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;oBACnB,KAAK,iBAAiB,CAAC,CAAC,CAAC;wBACvB,MAAM,IAAI,GAAG,KAAK,CAAC,IAA+B,CAAC;wBACnD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW;4BAAE,OAAO;wBAE/C,MAAM,EAAE,GAAiB;4BACvB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;4BACd,KAAK,EAAE,SAAS;4BAChB,SAAS,EAAE,IAAI,CAAC,SAAS;4BACzB,SAAS,EAAE,IAAI,CAAC,EAAE;4BAClB,IAAI,EAAE,WAAW;4BACjB,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,UAAU,EAAE,IAAI,CAAC,UAAU;4BAC3B,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,IAAI,EAAE,IAAI,CAAC,IAAI;4BACf,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,WAAW,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO;4BAC/B,aAAa,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS;4BACnC,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI;yBAC5B,CAAC;wBACF,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBAClC,MAAM;oBACR,CAAC;oBAED,KAAK,sBAAsB,CAAC,CAAC,CAAC;wBAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,IAA4B,CAAC;wBAChD,IAAI,CAAC,IAAI;4BAAE,OAAO;wBAElB,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;4BAChC,MAAM,EAAE,GAAiB;gCACvB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;gCACd,KAAK,EAAE,MAAM;gCACb,SAAS,EAAE,IAAI,CAAC,SAAS;gCACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gCACzB,UAAU,EAAE,IAAI,CAAC,MAAM;gCACvB,QAAQ,EAAE,IAAI,CAAC,IAAI;gCACnB,UAAU,EAAE,IAAI,CAAC,MAAM;6BACxB,CAAC;4BACF,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACpC,CAAC;wBAED,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;4BACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;4BACzB,MAAM,MAAM,GAAG,KAAK,EAAE,MAAoC,CAAC;4BAC3D,MAAM,SAAS,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC;4BACrC,MAAM,OAAO,GAAG,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC;4BACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;4BAE1F,MAAM,EAAE,GAAiB;gCACvB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;gCACd,KAAK,EAAE,MAAM;gCACb,SAAS,EAAE,IAAI,CAAC,SAAS;gCACzB,SAAS,EAAE,IAAI,CAAC,SAAS;gCACzB,IAAI,EAAE,IAAI,CAAC,IAAI;gCACf,UAAU,EAAE,MAAM;gCAClB,aAAa,EAAE,SAAS;gCACxB,WAAW,EAAE,OAAO;gCACpB,cAAc,EAAE,QAAQ;6BACzB,CAAC;4BACF,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBACpC,CAAC;wBACD,MAAM;oBACR,CAAC;oBAED,KAAK,iBAAiB,CAAC,CAAC,CAAC;wBACvB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAmD,CAAC;wBACvE,IAAI,CAAC,IAAI,EAAE,EAAE;4BAAE,OAAO;wBAEtB,MAAM,EAAE,GAAiB;4BACvB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;4BACd,KAAK,EAAE,iBAAiB;4BACxB,SAAS,EAAE,IAAI,CAAC,EAAE;4BAClB,YAAY,EAAE,IAAI,CAAC,KAAK;yBACzB,CAAC;wBACF,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBAClC,MAAM;oBACR,CAAC;oBAED,KAAK,gBAAgB,CAAC,CAAC,CAAC;wBACtB,MAAM,SAAS,GAAG,KAAK,CAAC,SAA+B,CAAC;wBACxD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAuC,CAAC;wBAC7D,IAAI,CAAC,SAAS;4BAAE,OAAO;wBAEvB,MAAM,EAAE,GAAiB;4BACvB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;4BACd,KAAK,EAAE,gBAAgB;4BACvB,SAAS;4BACT,aAAa,EAAE,MAAM,EAAE,IAAI;yBAC5B,CAAC;wBACF,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;wBAClC,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,6CAA6C;YAC/C,CAAC;QACH,CAAC;QAED,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO;YACxC,IAAI,CAAC;gBACH,MAAM,EAAE,GAAiB;oBACvB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;oBACd,KAAK,EAAE,aAAa;oBACpB,SAAS;oBACT,SAAS;oBACT,WAAW,EAAE,OAAO,CAAC,UAAU,GAAG,CAAC;oBACnC,aAAa,EAAE,OAAO,CAAC,UAAU;oBACjC,cAAc,EAAE,OAAO,CAAC,WAAW;oBACnC,gBAAgB,EAAE,OAAO,CAAC,aAAa;oBACvC,WAAW,EAAE,OAAO,CAAC,QAAQ;oBAC7B,kBAAkB,EAAE,OAAO,CAAC,eAAe;iBAC5C,CAAC;gBACF,kBAAkB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACpC,CAAC;YAAC,MAAM,CAAC;gBACP,uCAAuC;YACzC,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,12 @@
1
+ import type { SessionEvent, SessionSummary, ComparisonResult } from "./types.js";
2
+ export declare function appendSessionEvent(storePath: string, event: SessionEvent): void;
3
+ export declare function readSessionEvents(storePath: string, sessionID: string): SessionEvent[];
4
+ export declare function listSessionIDs(storePath: string): string[];
5
+ export declare function deleteSession(storePath: string, sessionID: string): void;
6
+ export declare function computeSummary(events: SessionEvent[]): SessionSummary;
7
+ export declare function listSessions(storePath: string): SessionSummary[];
8
+ export declare function getSession(storePath: string, sessionID: string): {
9
+ events: SessionEvent[];
10
+ summary: SessionSummary;
11
+ } | null;
12
+ export declare function compareSessions(storePath: string, idA: string, idB: string): ComparisonResult | null;
@@ -0,0 +1,184 @@
1
+ import { appendFileSync, readFileSync, mkdirSync, readdirSync, unlinkSync, existsSync } from "node:fs";
2
+ import path from "node:path";
3
+ import { isRagTool } from "./types.js";
4
+ const EVAL_DIR = "eval-sessions";
5
+ function getEvalDir(storePath) {
6
+ return path.join(storePath, EVAL_DIR);
7
+ }
8
+ function getSessionPath(storePath, sessionID) {
9
+ return path.join(getEvalDir(storePath), `${sessionID}.jsonl`);
10
+ }
11
+ export function appendSessionEvent(storePath, event) {
12
+ try {
13
+ const dir = getEvalDir(storePath);
14
+ mkdirSync(dir, { recursive: true });
15
+ const filePath = getSessionPath(storePath, event.sessionID);
16
+ appendFileSync(filePath, `${JSON.stringify(event)}\n`, "utf8");
17
+ }
18
+ catch {
19
+ // Logging must never break the plugin.
20
+ }
21
+ }
22
+ export function readSessionEvents(storePath, sessionID) {
23
+ try {
24
+ const filePath = getSessionPath(storePath, sessionID);
25
+ const content = readFileSync(filePath, "utf8");
26
+ const lines = content.split("\n").filter((l) => l.trim().length > 0);
27
+ return lines.map((line) => JSON.parse(line));
28
+ }
29
+ catch {
30
+ return [];
31
+ }
32
+ }
33
+ export function listSessionIDs(storePath) {
34
+ try {
35
+ const dir = getEvalDir(storePath);
36
+ if (!existsSync(dir))
37
+ return [];
38
+ const files = readdirSync(dir);
39
+ return files
40
+ .filter((f) => f.endsWith(".jsonl"))
41
+ .map((f) => f.slice(0, -6)); // remove .jsonl
42
+ }
43
+ catch {
44
+ return [];
45
+ }
46
+ }
47
+ export function deleteSession(storePath, sessionID) {
48
+ try {
49
+ const filePath = getSessionPath(storePath, sessionID);
50
+ if (existsSync(filePath)) {
51
+ unlinkSync(filePath);
52
+ }
53
+ }
54
+ catch {
55
+ // Ignore errors
56
+ }
57
+ }
58
+ export function computeSummary(events) {
59
+ const summary = {
60
+ sessionID: events[0]?.sessionID ?? "",
61
+ startedAt: events[0]?.ts ?? 0,
62
+ lastEventAt: 0,
63
+ messageCount: 0,
64
+ totalTokens: { input: 0, output: 0, reasoning: 0, cacheRead: 0, cacheWrite: 0 },
65
+ totalCost: 0,
66
+ totalSteps: 0,
67
+ ragContextCount: 0,
68
+ ragToolCalls: 0,
69
+ ragContextTokens: 0,
70
+ toolCallCounts: {},
71
+ models: [],
72
+ };
73
+ const modelSet = new Set();
74
+ const responseTimes = [];
75
+ for (const ev of events) {
76
+ if (ev.ts > summary.lastEventAt) {
77
+ summary.lastEventAt = ev.ts;
78
+ }
79
+ switch (ev.event) {
80
+ case "message":
81
+ if (ev.role === "assistant") {
82
+ summary.messageCount++;
83
+ if (ev.tokens) {
84
+ summary.totalTokens.input += ev.tokens.input;
85
+ summary.totalTokens.output += ev.tokens.output;
86
+ summary.totalTokens.reasoning += ev.tokens.reasoning;
87
+ summary.totalTokens.cacheRead += ev.tokens.cache.read;
88
+ summary.totalTokens.cacheWrite += ev.tokens.cache.write;
89
+ }
90
+ if (ev.cost != null) {
91
+ summary.totalCost += ev.cost;
92
+ }
93
+ if (ev.modelID) {
94
+ modelSet.add(ev.modelID);
95
+ }
96
+ if (ev.timeCreated && ev.timeCompleted && ev.timeCompleted > ev.timeCreated) {
97
+ responseTimes.push(ev.timeCompleted - ev.timeCreated);
98
+ }
99
+ }
100
+ break;
101
+ case "step":
102
+ summary.totalSteps++;
103
+ break;
104
+ case "tool":
105
+ if (ev.tool) {
106
+ summary.toolCallCounts[ev.tool] = (summary.toolCallCounts[ev.tool] ?? 0) + 1;
107
+ if (isRagTool(ev.tool)) {
108
+ summary.ragToolCalls++;
109
+ }
110
+ }
111
+ break;
112
+ case "rag.context":
113
+ if (ev.ragInjected) {
114
+ summary.ragContextCount++;
115
+ }
116
+ if (ev.ragContextTokens) {
117
+ summary.ragContextTokens += ev.ragContextTokens;
118
+ }
119
+ break;
120
+ case "session.created":
121
+ if (ev.sessionTitle) {
122
+ summary.title = ev.sessionTitle;
123
+ }
124
+ break;
125
+ }
126
+ }
127
+ summary.models = [...modelSet];
128
+ if (responseTimes.length > 0) {
129
+ summary.avgResponseTimeMs = Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length);
130
+ }
131
+ return summary;
132
+ }
133
+ export function listSessions(storePath) {
134
+ const ids = listSessionIDs(storePath);
135
+ const summaries = [];
136
+ for (const id of ids) {
137
+ const events = readSessionEvents(storePath, id);
138
+ if (events.length > 0) {
139
+ summaries.push(computeSummary(events));
140
+ }
141
+ }
142
+ summaries.sort((a, b) => b.lastEventAt - a.lastEventAt);
143
+ return summaries;
144
+ }
145
+ export function getSession(storePath, sessionID) {
146
+ const events = readSessionEvents(storePath, sessionID);
147
+ if (events.length === 0)
148
+ return null;
149
+ return { events, summary: computeSummary(events) };
150
+ }
151
+ function avgOrZero(values) {
152
+ if (values.length === 0)
153
+ return 0;
154
+ return Math.round(values.reduce((a, b) => a + b, 0) / values.length);
155
+ }
156
+ export function compareSessions(storePath, idA, idB) {
157
+ const sessionA = getSession(storePath, idA);
158
+ const sessionB = getSession(storePath, idB);
159
+ if (!sessionA || !sessionB)
160
+ return null;
161
+ const a = sessionA.summary;
162
+ const b = sessionB.summary;
163
+ return {
164
+ sessionA: a,
165
+ sessionB: b,
166
+ delta: {
167
+ totalTokens: (b.totalTokens.input + b.totalTokens.output + b.totalTokens.reasoning)
168
+ - (a.totalTokens.input + a.totalTokens.output + a.totalTokens.reasoning),
169
+ inputTokens: b.totalTokens.input - a.totalTokens.input,
170
+ outputTokens: b.totalTokens.output - a.totalTokens.output,
171
+ reasoningTokens: b.totalTokens.reasoning - a.totalTokens.reasoning,
172
+ cacheRead: b.totalTokens.cacheRead - a.totalTokens.cacheRead,
173
+ cacheWrite: b.totalTokens.cacheWrite - a.totalTokens.cacheWrite,
174
+ cost: b.totalCost - a.totalCost,
175
+ messageCount: b.messageCount - a.messageCount,
176
+ totalSteps: b.totalSteps - a.totalSteps,
177
+ ragContextCount: b.ragContextCount - a.ragContextCount,
178
+ ragToolCalls: b.ragToolCalls - a.ragToolCalls,
179
+ ragContextTokens: b.ragContextTokens - a.ragContextTokens,
180
+ avgResponseTimeMs: (b.avgResponseTimeMs ?? 0) - (a.avgResponseTimeMs ?? 0),
181
+ },
182
+ };
183
+ }
184
+ //# sourceMappingURL=storage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"storage.js","sourceRoot":"","sources":["../../src/eval/storage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACvG,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAEvC,MAAM,QAAQ,GAAG,eAAe,CAAC;AAEjC,SAAS,UAAU,CAAC,SAAiB;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB,EAAE,SAAiB;IAC1D,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,GAAG,SAAS,QAAQ,CAAC,CAAC;AAChE,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,KAAmB;IACvE,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;QAClC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QAC5D,cAAc,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,uCAAuC;IACzC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,SAAiB,EAAE,SAAiB;IACpE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAiB,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,SAAiB;IAC9C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC/B,OAAO,KAAK;aACT,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;aACnC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE,SAAiB;IAChE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACtD,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzB,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,gBAAgB;IAClB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAsB;IACnD,MAAM,OAAO,GAAmB;QAC9B,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,IAAI,EAAE;QACrC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC;QAC7B,WAAW,EAAE,CAAC;QACd,YAAY,EAAE,CAAC;QACf,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;QAC/E,SAAS,EAAE,CAAC;QACZ,UAAU,EAAE,CAAC;QACb,eAAe,EAAE,CAAC;QAClB,YAAY,EAAE,CAAC;QACf,gBAAgB,EAAE,CAAC;QACnB,cAAc,EAAE,EAAE;QAClB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAChC,OAAO,CAAC,WAAW,GAAG,EAAE,CAAC,EAAE,CAAC;QAC9B,CAAC;QAED,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;YACjB,KAAK,SAAS;gBACZ,IAAI,EAAE,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;oBAC5B,OAAO,CAAC,YAAY,EAAE,CAAC;oBACvB,IAAI,EAAE,CAAC,MAAM,EAAE,CAAC;wBACd,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;wBAC7C,OAAO,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;wBAC/C,OAAO,CAAC,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC;wBACrD,OAAO,CAAC,WAAW,CAAC,SAAS,IAAI,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;wBACtD,OAAO,CAAC,WAAW,CAAC,UAAU,IAAI,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;oBAC1D,CAAC;oBACD,IAAI,EAAE,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;wBACpB,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,IAAI,CAAC;oBAC/B,CAAC;oBACD,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;wBACf,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;oBAC3B,CAAC;oBACD,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,aAAa,IAAI,EAAE,CAAC,aAAa,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;wBAC5E,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC;oBACxD,CAAC;gBACH,CAAC;gBACD,MAAM;YAER,KAAK,MAAM;gBACT,OAAO,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM;YAER,KAAK,MAAM;gBACT,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;oBACZ,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;oBAC7E,IAAI,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;wBACvB,OAAO,CAAC,YAAY,EAAE,CAAC;oBACzB,CAAC;gBACH,CAAC;gBACD,MAAM;YAER,KAAK,aAAa;gBAChB,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;oBACnB,OAAO,CAAC,eAAe,EAAE,CAAC;gBAC5B,CAAC;gBACD,IAAI,EAAE,CAAC,gBAAgB,EAAE,CAAC;oBACxB,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC,gBAAgB,CAAC;gBAClD,CAAC;gBACD,MAAM;YAER,KAAK,iBAAiB;gBACpB,IAAI,EAAE,CAAC,YAAY,EAAE,CAAC;oBACpB,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC;gBAClC,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;IAE/B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,KAAK,CACpC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM,CAChE,CAAC;IACJ,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,SAAiB;IAC5C,MAAM,GAAG,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IACtC,MAAM,SAAS,GAAqB,EAAE,CAAC;IAEvC,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,SAAS,CAAC,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IAED,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IACxD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,SAAiB,EAAE,SAAiB;IAC7D,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACvD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;AACrD,CAAC;AAED,SAAS,SAAS,CAAC,MAAgB;IACjC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAiB,EAAE,GAAW,EAAE,GAAW;IACzE,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC5C,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAExC,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC;IAC3B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC;IAE3B,OAAO;QACL,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE;YACL,WAAW,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC;kBAC/E,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC;YAC1E,WAAW,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,CAAC,WAAW,CAAC,KAAK;YACtD,YAAY,EAAE,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,MAAM;YACzD,eAAe,EAAE,CAAC,CAAC,WAAW,CAAC,SAAS,GAAG,CAAC,CAAC,WAAW,CAAC,SAAS;YAClE,SAAS,EAAE,CAAC,CAAC,WAAW,CAAC,SAAS,GAAG,CAAC,CAAC,WAAW,CAAC,SAAS;YAC5D,UAAU,EAAE,CAAC,CAAC,WAAW,CAAC,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC,UAAU;YAC/D,IAAI,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS;YAC/B,YAAY,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;YAC7C,UAAU,EAAE,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU;YACvC,eAAe,EAAE,CAAC,CAAC,eAAe,GAAG,CAAC,CAAC,eAAe;YACtD,YAAY,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY;YAC7C,gBAAgB,EAAE,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,gBAAgB;YACzD,iBAAiB,EAAE,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,CAAC;SAC3E;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,89 @@
1
+ /**
2
+ * Evaluation framework types for session logging and comparison.
3
+ *
4
+ * Captures OpenCode session events (messages, tool calls, RAG injections)
5
+ * via the plugin event hook and stores them as JSONL for analysis.
6
+ */
7
+ export interface TokenUsage {
8
+ input: number;
9
+ output: number;
10
+ reasoning: number;
11
+ cache: {
12
+ read: number;
13
+ write: number;
14
+ };
15
+ }
16
+ export interface SessionEvent {
17
+ ts: number;
18
+ event: "message" | "tool" | "rag.context" | "step" | "session.created" | "session.status";
19
+ sessionID: string;
20
+ messageID?: string;
21
+ role?: "user" | "assistant";
22
+ modelID?: string;
23
+ providerID?: string;
24
+ tokens?: TokenUsage;
25
+ cost?: number;
26
+ finish?: string;
27
+ timeCreated?: number;
28
+ timeCompleted?: number;
29
+ errorName?: string;
30
+ tool?: string;
31
+ toolStatus?: "pending" | "running" | "completed" | "error";
32
+ toolTimeStart?: number;
33
+ toolTimeEnd?: number;
34
+ toolDurationMs?: number;
35
+ ragInjected?: boolean;
36
+ ragChunkCount?: number;
37
+ ragUniqueFiles?: number;
38
+ ragContextTokens?: number;
39
+ ragTopScore?: number;
40
+ ragRetrievalTimeMs?: number;
41
+ stepTokens?: TokenUsage;
42
+ stepCost?: number;
43
+ stepReason?: string;
44
+ sessionTitle?: string;
45
+ sessionStatus?: string;
46
+ }
47
+ export interface SessionSummary {
48
+ sessionID: string;
49
+ title?: string;
50
+ startedAt: number;
51
+ lastEventAt: number;
52
+ messageCount: number;
53
+ totalTokens: {
54
+ input: number;
55
+ output: number;
56
+ reasoning: number;
57
+ cacheRead: number;
58
+ cacheWrite: number;
59
+ };
60
+ totalCost: number;
61
+ totalSteps: number;
62
+ ragContextCount: number;
63
+ ragToolCalls: number;
64
+ ragContextTokens: number;
65
+ toolCallCounts: Record<string, number>;
66
+ avgResponseTimeMs?: number;
67
+ models: string[];
68
+ }
69
+ export interface ComparisonResult {
70
+ sessionA: SessionSummary;
71
+ sessionB: SessionSummary;
72
+ delta: {
73
+ totalTokens: number;
74
+ inputTokens: number;
75
+ outputTokens: number;
76
+ reasoningTokens: number;
77
+ cacheRead: number;
78
+ cacheWrite: number;
79
+ cost: number;
80
+ messageCount: number;
81
+ totalSteps: number;
82
+ ragContextCount: number;
83
+ ragToolCalls: number;
84
+ ragContextTokens: number;
85
+ avgResponseTimeMs: number;
86
+ };
87
+ }
88
+ export declare const RAG_TOOL_NAMES: Set<string>;
89
+ export declare function isRagTool(toolName: string): boolean;
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Evaluation framework types for session logging and comparison.
3
+ *
4
+ * Captures OpenCode session events (messages, tool calls, RAG injections)
5
+ * via the plugin event hook and stores them as JSONL for analysis.
6
+ */
7
+ export const RAG_TOOL_NAMES = new Set([
8
+ "opencode-rag-context",
9
+ "search_semantic",
10
+ "get_file_skeleton",
11
+ "find_usages",
12
+ "read",
13
+ ]);
14
+ export function isRagTool(toolName) {
15
+ return RAG_TOOL_NAMES.has(toolName);
16
+ }
17
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAuGH,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IACpC,sBAAsB;IACtB,iBAAiB;IACjB,mBAAmB;IACnB,aAAa;IACb,MAAM;CACP,CAAC,CAAC;AAEH,MAAM,UAAU,SAAS,CAAC,QAAgB;IACxC,OAAO,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;AACtC,CAAC"}
package/dist/indexer.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import fs from "node:fs/promises";
2
2
  import path from "node:path";
3
+ import pLimit from "p-limit";
3
4
  import { chunkFile } from "./chunker/factory.js";
4
5
  import { extractPdfText } from "./chunker/pdf.js";
5
6
  import { extractDocxText } from "./chunker/docx.js";
@@ -7,7 +8,6 @@ import { extractDocText } from "./chunker/doc.js";
7
8
  import { extractExcelText } from "./chunker/excel.js";
8
9
  import { computeFileHash, loadManifest, manifestPathFor, normalizeFilePath, saveManifest, } from "./core/manifest.js";
9
10
  import { embedBatch } from "./embedder/factory.js";
10
- const BATCH_SIZE = 50;
11
11
  function createLogger(logger) {
12
12
  return {
13
13
  info: logger?.info ?? (() => { }),
@@ -93,19 +93,6 @@ function createIndexStats(totalFiles, manifestStatus) {
93
93
  batchesFlushed: 0,
94
94
  };
95
95
  }
96
- function updateManifestEntry(manifest, file, chunkCount) {
97
- manifest.files[file.normalizedPath] = {
98
- hash: file.hash,
99
- chunkCount,
100
- indexedAt: Date.now(),
101
- };
102
- }
103
- async function flushChunkBatch(store, chunkBatch) {
104
- if (chunkBatch.length === 0)
105
- return false;
106
- await store.addChunks(chunkBatch.splice(0, chunkBatch.length));
107
- return true;
108
- }
109
96
  export async function runIndexPass(options) {
110
97
  const logger = createLogger(options.logger);
111
98
  const workspaceFiles = await scanWorkspace(options.cwd, options.config);
@@ -138,119 +125,172 @@ export async function runIndexPass(options) {
138
125
  stats.deletedFiles++;
139
126
  }
140
127
  }
141
- const chunkBatch = [];
142
- for (const file of workspaceFiles) {
128
+ const limit = pLimit(options.config.indexing.concurrency);
129
+ const workerResults = await Promise.all(workspaceFiles.map((file) => limit(async () => {
143
130
  const previous = manifest.files[file.normalizedPath];
131
+ const fileLabel = path.relative(options.cwd, file.filePath);
144
132
  if (file.isEmpty) {
145
- stats.skippedEmptyFiles++;
146
133
  if (previous) {
147
134
  await options.store.deleteByFilePath(file.normalizedPath);
148
135
  options.keywordIndex?.removeByFilePath(file.normalizedPath);
149
- delete manifest.files[file.normalizedPath];
150
- stats.removedFiles++;
151
- logger.info(` ${path.relative(options.cwd, file.filePath)} (empty, removed from index)`);
152
- }
153
- else {
154
- logger.info(` ${path.relative(options.cwd, file.filePath)} (empty, skipped)`);
136
+ logger.info(` ${fileLabel} (empty, removed from index)`);
137
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: true, hadChunks: false };
155
138
  }
156
- continue;
139
+ logger.info(` ${fileLabel} (empty, skipped)`);
140
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: false, hadChunks: false };
157
141
  }
158
142
  if (file.isTooSmall) {
159
- stats.skippedSmallFiles++;
160
143
  if (previous) {
161
144
  await options.store.deleteByFilePath(file.normalizedPath);
162
145
  options.keywordIndex?.removeByFilePath(file.normalizedPath);
163
- delete manifest.files[file.normalizedPath];
164
- stats.removedFiles++;
165
- logger.info(` ${path.relative(options.cwd, file.filePath)} (too small, removed from index)`);
166
- }
167
- else {
168
- logger.info(` ${path.relative(options.cwd, file.filePath)} (too small, skipped)`);
146
+ logger.info(` ${fileLabel} (too small, removed from index)`);
147
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: true, hadChunks: false };
169
148
  }
170
- continue;
149
+ logger.info(` ${fileLabel} (too small, skipped)`);
150
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: false, hadChunks: false };
171
151
  }
172
152
  if (previous && previous.hash === file.hash) {
173
- stats.unchangedFiles++;
174
- logger.info(` ${path.relative(options.cwd, file.filePath)} (unchanged)`);
175
- continue;
153
+ logger.info(` ${fileLabel} (unchanged)`);
154
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false };
176
155
  }
156
+ let isModified = false;
177
157
  if (previous) {
178
158
  await options.store.deleteByFilePath(file.normalizedPath);
179
159
  options.keywordIndex?.removeByFilePath(file.normalizedPath);
180
- stats.modifiedFiles++;
181
- }
182
- else {
183
- stats.newFiles++;
160
+ isModified = true;
184
161
  }
185
- const chunks = await chunkFile(file.filePath, file.content);
186
- if (chunks.length === 0) {
187
- delete manifest.files[file.normalizedPath];
188
- stats.removedFiles++;
189
- logger.info(` ${path.relative(options.cwd, file.filePath)} (0 chunks, removed from index)`);
190
- continue;
162
+ const chunks = await chunkFile(file.filePath, file.content, options.config.chunking?.nodeTypes).catch((err) => {
163
+ logger.warn(` ${fileLabel} (chunking failed: ${err.message})`);
164
+ return null;
165
+ });
166
+ if (chunks === null || chunks.length === 0) {
167
+ if (chunks === null) {
168
+ // Chunking failed — keep previous index state if file was indexed before
169
+ if (previous) {
170
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
171
+ }
172
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
173
+ }
174
+ logger.info(` ${fileLabel} (0 chunks, removed from index)`);
175
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
191
176
  }
192
177
  options.keywordIndex?.addChunks(chunks);
193
- const docPrefix = options.config.embedding.documentPrefix ?? "";
194
- const textToEmbed = [];
195
- if (options.descriptionProvider) {
196
- let descriptionMap = null;
197
- if (chunks.length > 1) {
198
- try {
199
- descriptionMap = await options.descriptionProvider.generateBatchDescriptions(chunks);
178
+ try {
179
+ const docPrefix = options.config.embedding.documentPrefix ?? "";
180
+ const textToEmbed = [];
181
+ if (options.descriptionProvider) {
182
+ let descriptionMap = null;
183
+ if (chunks.length > 1) {
184
+ try {
185
+ descriptionMap = await options.descriptionProvider.generateBatchDescriptions(chunks);
186
+ }
187
+ catch (err) {
188
+ logger.warn(`Batch description failed, falling back to individual: ${err.message}`);
189
+ }
200
190
  }
201
- catch (err) {
202
- logger.warn(`Batch description failed, falling back to individual: ${err.message}`);
191
+ for (const chunk of chunks) {
192
+ const batchDesc = descriptionMap?.get(chunk.id);
193
+ if (batchDesc && batchDesc.trim().length > 0) {
194
+ chunk.description = batchDesc;
195
+ textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
196
+ }
197
+ else {
198
+ try {
199
+ chunk.description = await options.descriptionProvider.generateDescription(chunk);
200
+ textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
201
+ }
202
+ catch (err) {
203
+ logger.warn(`Description generation failed for ${chunk.id}, falling back to content: ${err.message}`);
204
+ textToEmbed.push(docPrefix + chunk.content);
205
+ }
206
+ }
203
207
  }
204
208
  }
205
- for (const chunk of chunks) {
206
- const batchDesc = descriptionMap?.get(chunk.id);
207
- if (batchDesc && batchDesc.trim().length > 0) {
208
- chunk.description = batchDesc;
209
+ else {
210
+ for (const chunk of chunks) {
211
+ const relPath = path.relative(options.cwd, chunk.metadata.filePath).replace(/\\/g, "/");
212
+ chunk.description = `${relPath}, lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}`;
209
213
  textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
210
214
  }
215
+ }
216
+ const embeddings = await embedBatch(options.embedder, textToEmbed, options.config.indexing.embedBatchSize, "document");
217
+ for (let i = 0; i < chunks.length; i++) {
218
+ const emb = embeddings[i];
219
+ if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
220
+ chunks[i].embedding = emb;
221
+ }
211
222
  else {
212
- try {
213
- chunk.description = await options.descriptionProvider.generateDescription(chunk);
214
- textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
215
- }
216
- catch (err) {
217
- logger.warn(`Description generation failed for ${chunk.id}, falling back to content: ${err.message}`);
218
- textToEmbed.push(docPrefix + chunk.content);
219
- }
223
+ chunks[i].embedding = undefined;
220
224
  }
221
225
  }
222
- }
223
- else {
224
- for (const chunk of chunks) {
225
- const relPath = path.relative(options.cwd, chunk.metadata.filePath).replace(/\\/g, "/");
226
- chunk.description = `${relPath}, lines ${chunk.metadata.startLine}-${chunk.metadata.endLine}`;
227
- textToEmbed.push(docPrefix + chunk.description + "\n\n" + chunk.content);
226
+ const validChunks = chunks.filter((c) => c.embedding && c.embedding.length > 0);
227
+ if (validChunks.length > 0) {
228
+ await options.store.addChunks(validChunks);
228
229
  }
230
+ logger.info(` ${fileLabel} (${chunks.length} chunks${isModified ? ", modified" : ", new"})`);
231
+ return {
232
+ normalizedPath: file.normalizedPath,
233
+ hash: file.hash,
234
+ chunkCount: chunks.length,
235
+ fileLabel,
236
+ isNew: !isModified && !previous,
237
+ isModified,
238
+ isUnchanged: false,
239
+ isEmpty: false,
240
+ isTooSmall: false,
241
+ isRemoved: false,
242
+ hadChunks: chunks.length > 0,
243
+ };
229
244
  }
230
- const embeddings = await embedBatch(options.embedder, textToEmbed, undefined, "document");
231
- for (let i = 0; i < chunks.length; i++) {
232
- const emb = embeddings[i];
233
- if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
234
- chunks[i].embedding = emb;
245
+ catch (err) {
246
+ logger.warn(` ${fileLabel} (embed/store failed: ${err.message})`);
247
+ if (previous) {
248
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
235
249
  }
236
- else {
237
- // Guard against malformed provider output so the chunk is skipped
238
- // instead of corrupting the vector store.
239
- chunks[i].embedding = undefined;
250
+ return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
251
+ }
252
+ })));
253
+ for (const result of workerResults) {
254
+ if (result.isEmpty) {
255
+ stats.skippedEmptyFiles++;
256
+ if (result.isRemoved) {
257
+ delete manifest.files[result.normalizedPath];
258
+ stats.removedFiles++;
240
259
  }
260
+ continue;
241
261
  }
242
- chunkBatch.push(...chunks);
243
- stats.totalChunks += chunks.length;
244
- updateManifestEntry(manifest, file, chunks.length);
245
- if (chunkBatch.length >= BATCH_SIZE) {
246
- if (await flushChunkBatch(options.store, chunkBatch)) {
247
- stats.batchesFlushed++;
262
+ if (result.isTooSmall) {
263
+ stats.skippedSmallFiles++;
264
+ if (result.isRemoved) {
265
+ delete manifest.files[result.normalizedPath];
266
+ stats.removedFiles++;
248
267
  }
268
+ continue;
269
+ }
270
+ if (result.isUnchanged) {
271
+ stats.unchangedFiles++;
272
+ continue;
273
+ }
274
+ if (result.isRemoved) {
275
+ delete manifest.files[result.normalizedPath];
276
+ stats.removedFiles++;
277
+ continue;
278
+ }
279
+ if (result.isModified) {
280
+ stats.modifiedFiles++;
281
+ }
282
+ else if (result.isNew) {
283
+ stats.newFiles++;
284
+ }
285
+ if (result.chunkCount > 0) {
286
+ manifest.files[result.normalizedPath] = {
287
+ hash: result.hash,
288
+ chunkCount: result.chunkCount,
289
+ indexedAt: Date.now(),
290
+ };
291
+ stats.totalChunks += result.chunkCount;
292
+ stats.batchesFlushed++;
249
293
  }
250
- logger.info(` ${path.relative(options.cwd, file.filePath)} (${chunks.length} chunks${previous ? ", modified" : ", new"})`);
251
- }
252
- if (await flushChunkBatch(options.store, chunkBatch)) {
253
- stats.batchesFlushed++;
254
294
  }
255
295
  manifest.lastIndexedAt = Date.now();
256
296
  await saveManifest(options.storePath, manifest);