opencode-rag-plugin 1.15.1 → 1.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (240) hide show
  1. package/ReadMe.md +52 -9
  2. package/dist/api.js +44 -24
  3. package/dist/chunker/base.d.ts +34 -3
  4. package/dist/chunker/base.js +70 -21
  5. package/dist/chunker/factory.d.ts +4 -1
  6. package/dist/chunker/factory.js +12 -1
  7. package/dist/chunker/grammar.js +3 -0
  8. package/dist/chunker/image.js +8 -8
  9. package/dist/chunker/pdf.js +11 -13
  10. package/dist/chunker/xml.d.ts +2 -0
  11. package/dist/chunker/xml.js +2 -0
  12. package/dist/cli/commands/index-command.js +0 -4
  13. package/dist/cli/commands/index.d.ts +1 -0
  14. package/dist/cli/commands/index.js +1 -0
  15. package/dist/cli/commands/init-helpers.d.ts +12 -0
  16. package/dist/cli/commands/init-helpers.js +87 -2
  17. package/dist/cli/commands/init.js +20 -2
  18. package/dist/cli/commands/query.js +1 -0
  19. package/dist/cli/commands/setup.d.ts +2 -0
  20. package/dist/cli/commands/setup.js +113 -0
  21. package/dist/cli/commands/status.js +64 -2
  22. package/dist/cli/commands/update.d.ts +4 -6
  23. package/dist/cli/commands/update.js +44 -58
  24. package/dist/cli/index.js +2 -1
  25. package/dist/content/image.js +24 -3
  26. package/dist/content/reader.d.ts +6 -1
  27. package/dist/content/reader.js +49 -4
  28. package/dist/core/config.d.ts +29 -1
  29. package/dist/core/config.js +105 -8
  30. package/dist/core/desc-cache.d.ts +31 -0
  31. package/dist/core/desc-cache.js +124 -0
  32. package/dist/core/interfaces.d.ts +58 -3
  33. package/dist/core/manifest.d.ts +24 -1
  34. package/dist/core/manifest.js +34 -3
  35. package/dist/core/resolve-api-key.js +4 -2
  36. package/dist/core/runtime-overrides.js +12 -8
  37. package/dist/core/setup-runtime.d.ts +23 -0
  38. package/dist/core/setup-runtime.js +231 -0
  39. package/dist/core/version-check.d.ts +85 -0
  40. package/dist/core/version-check.js +160 -0
  41. package/dist/describer/anthropic.d.ts +2 -2
  42. package/dist/describer/anthropic.js +5 -7
  43. package/dist/describer/describer.d.ts +2 -2
  44. package/dist/describer/describer.js +6 -8
  45. package/dist/describer/gemini.d.ts +2 -2
  46. package/dist/describer/gemini.js +5 -7
  47. package/dist/embedder/cohere.d.ts +1 -1
  48. package/dist/embedder/cohere.js +2 -2
  49. package/dist/embedder/factory.d.ts +3 -1
  50. package/dist/embedder/factory.js +8 -2
  51. package/dist/embedder/health.js +5 -1
  52. package/dist/embedder/http.d.ts +1 -1
  53. package/dist/embedder/http.js +74 -43
  54. package/dist/embedder/ollama.d.ts +1 -1
  55. package/dist/embedder/ollama.js +2 -2
  56. package/dist/embedder/openai.d.ts +1 -1
  57. package/dist/embedder/openai.js +2 -2
  58. package/dist/eval/compare-merge.d.ts +10 -0
  59. package/dist/eval/compare-merge.js +537 -0
  60. package/dist/eval/compare-rankings.d.ts +10 -0
  61. package/dist/eval/compare-rankings.js +245 -0
  62. package/dist/eval/dump-descriptions.d.ts +7 -0
  63. package/dist/eval/dump-descriptions.js +58 -0
  64. package/dist/eval/fast-index.d.ts +8 -0
  65. package/dist/eval/fast-index.js +283 -0
  66. package/dist/eval/run-branch-compare.d.ts +7 -0
  67. package/dist/eval/run-branch-compare.js +220 -0
  68. package/dist/eval/run-token-test.js +1 -0
  69. package/dist/eval/test-kw.d.ts +1 -0
  70. package/dist/eval/test-kw.js +22 -0
  71. package/dist/eval/update-descriptions.d.ts +7 -0
  72. package/dist/eval/update-descriptions.js +84 -0
  73. package/dist/index.d.ts +4 -0
  74. package/dist/index.js +3 -0
  75. package/dist/indexer/embed-stage.js +2 -1
  76. package/dist/indexer/git-diff.js +21 -9
  77. package/dist/indexer/pipeline.d.ts +6 -0
  78. package/dist/indexer/pipeline.js +304 -40
  79. package/dist/indexer/watch.js +1 -3
  80. package/dist/indexer/worker.d.ts +15 -2
  81. package/dist/indexer/worker.js +25 -12
  82. package/dist/mcp/handlers.d.ts +9 -0
  83. package/dist/mcp/handlers.js +23 -6
  84. package/dist/mcp/server.js +2 -0
  85. package/dist/opencode/create-read-tool.d.ts +2 -0
  86. package/dist/opencode/create-read-tool.js +8 -2
  87. package/dist/opencode/read-fallback.d.ts +1 -5
  88. package/dist/opencode/read-fallback.js +1 -18
  89. package/dist/opencode/read-format.js +5 -3
  90. package/dist/opencode/tools.js +5 -7
  91. package/dist/plugin.js +223 -71
  92. package/dist/retriever/keyword-index.d.ts +3 -2
  93. package/dist/retriever/keyword-index.js +25 -1
  94. package/dist/retriever/retriever.d.ts +4 -1
  95. package/dist/retriever/retriever.js +34 -56
  96. package/dist/tui.js +18 -1
  97. package/dist/vectorstore/lancedb.d.ts +35 -4
  98. package/dist/vectorstore/lancedb.js +146 -23
  99. package/dist/vectorstore/memory.d.ts +6 -1
  100. package/dist/vectorstore/memory.js +58 -0
  101. package/dist/watcher.d.ts +2 -0
  102. package/dist/watcher.js +5 -1
  103. package/dist/web/api.js +10 -2
  104. package/dist/web/server.js +18 -3
  105. package/package.json +8 -9
  106. package/scripts/postinstall-setup.js +82 -0
  107. package/dist/api.js.map +0 -1
  108. package/dist/chunker/base.js.map +0 -1
  109. package/dist/chunker/bash.js.map +0 -1
  110. package/dist/chunker/c.js.map +0 -1
  111. package/dist/chunker/cpp.js.map +0 -1
  112. package/dist/chunker/csharp.js.map +0 -1
  113. package/dist/chunker/css.js.map +0 -1
  114. package/dist/chunker/doc.js.map +0 -1
  115. package/dist/chunker/dockerfile.js.map +0 -1
  116. package/dist/chunker/docx.js.map +0 -1
  117. package/dist/chunker/excel.js.map +0 -1
  118. package/dist/chunker/factory.js.map +0 -1
  119. package/dist/chunker/fallback.js.map +0 -1
  120. package/dist/chunker/go.js.map +0 -1
  121. package/dist/chunker/grammar.js.map +0 -1
  122. package/dist/chunker/html.js.map +0 -1
  123. package/dist/chunker/image.js.map +0 -1
  124. package/dist/chunker/ini.js.map +0 -1
  125. package/dist/chunker/java.js.map +0 -1
  126. package/dist/chunker/javascript.js.map +0 -1
  127. package/dist/chunker/json.js.map +0 -1
  128. package/dist/chunker/kotlin.js.map +0 -1
  129. package/dist/chunker/loader.js.map +0 -1
  130. package/dist/chunker/markdown.js.map +0 -1
  131. package/dist/chunker/pdf.js.map +0 -1
  132. package/dist/chunker/php.js.map +0 -1
  133. package/dist/chunker/powershell.js.map +0 -1
  134. package/dist/chunker/python.js.map +0 -1
  135. package/dist/chunker/razor.js.map +0 -1
  136. package/dist/chunker/ruby.js.map +0 -1
  137. package/dist/chunker/rust.js.map +0 -1
  138. package/dist/chunker/sln.js.map +0 -1
  139. package/dist/chunker/sql.js.map +0 -1
  140. package/dist/chunker/ssl.js.map +0 -1
  141. package/dist/chunker/swift.js.map +0 -1
  142. package/dist/chunker/tex.js.map +0 -1
  143. package/dist/chunker/toml.js.map +0 -1
  144. package/dist/chunker/typescript.js.map +0 -1
  145. package/dist/chunker/uuid.js.map +0 -1
  146. package/dist/chunker/xml.js.map +0 -1
  147. package/dist/chunker/yaml.js.map +0 -1
  148. package/dist/cli/commands/clear.js.map +0 -1
  149. package/dist/cli/commands/describe-image.js.map +0 -1
  150. package/dist/cli/commands/dump.js.map +0 -1
  151. package/dist/cli/commands/eval.js.map +0 -1
  152. package/dist/cli/commands/index-command.js.map +0 -1
  153. package/dist/cli/commands/index.js.map +0 -1
  154. package/dist/cli/commands/init-helpers.js.map +0 -1
  155. package/dist/cli/commands/init.js.map +0 -1
  156. package/dist/cli/commands/list.js.map +0 -1
  157. package/dist/cli/commands/mcp.js.map +0 -1
  158. package/dist/cli/commands/query.js.map +0 -1
  159. package/dist/cli/commands/show.js.map +0 -1
  160. package/dist/cli/commands/status.js.map +0 -1
  161. package/dist/cli/commands/ui.js.map +0 -1
  162. package/dist/cli/commands/update.js.map +0 -1
  163. package/dist/cli/format.js.map +0 -1
  164. package/dist/cli/helpers.js.map +0 -1
  165. package/dist/cli/index.js.map +0 -1
  166. package/dist/cli/progress.d.ts +0 -42
  167. package/dist/cli/progress.js +0 -137
  168. package/dist/cli/progress.js.map +0 -1
  169. package/dist/cli/types.js.map +0 -1
  170. package/dist/cli.js.map +0 -1
  171. package/dist/content/doc.js.map +0 -1
  172. package/dist/content/docx.js.map +0 -1
  173. package/dist/content/excel.js.map +0 -1
  174. package/dist/content/image.js.map +0 -1
  175. package/dist/content/pdf.js.map +0 -1
  176. package/dist/content/reader.js.map +0 -1
  177. package/dist/content/types.js.map +0 -1
  178. package/dist/core/bootstrap.js.map +0 -1
  179. package/dist/core/config.js.map +0 -1
  180. package/dist/core/doc-progress.js.map +0 -1
  181. package/dist/core/fileLogger.js.map +0 -1
  182. package/dist/core/interfaces.js.map +0 -1
  183. package/dist/core/manifest.js.map +0 -1
  184. package/dist/core/provider-defaults.js.map +0 -1
  185. package/dist/core/rag-injection-flag.js.map +0 -1
  186. package/dist/core/resolve-api-key.js.map +0 -1
  187. package/dist/core/runtime-overrides.js.map +0 -1
  188. package/dist/describer/anthropic.js.map +0 -1
  189. package/dist/describer/describer.js.map +0 -1
  190. package/dist/describer/factory.js.map +0 -1
  191. package/dist/describer/gemini.js.map +0 -1
  192. package/dist/describer/shared.js.map +0 -1
  193. package/dist/embedder/cohere.js.map +0 -1
  194. package/dist/embedder/factory.js.map +0 -1
  195. package/dist/embedder/health.js.map +0 -1
  196. package/dist/embedder/http.js.map +0 -1
  197. package/dist/embedder/ollama.js.map +0 -1
  198. package/dist/embedder/openai.js.map +0 -1
  199. package/dist/eval/index.js.map +0 -1
  200. package/dist/eval/run-token-test.js.map +0 -1
  201. package/dist/eval/session-logger.js.map +0 -1
  202. package/dist/eval/storage.js.map +0 -1
  203. package/dist/eval/token-analysis.js.map +0 -1
  204. package/dist/eval/token-counter.js.map +0 -1
  205. package/dist/eval/types.js.map +0 -1
  206. package/dist/index.js.map +0 -1
  207. package/dist/indexer/description-stage.js.map +0 -1
  208. package/dist/indexer/embed-stage.js.map +0 -1
  209. package/dist/indexer/git-diff.js.map +0 -1
  210. package/dist/indexer/metadata.js.map +0 -1
  211. package/dist/indexer/pipeline.js.map +0 -1
  212. package/dist/indexer/stats.js.map +0 -1
  213. package/dist/indexer/watch.js.map +0 -1
  214. package/dist/indexer/worker.js.map +0 -1
  215. package/dist/indexer.js.map +0 -1
  216. package/dist/mcp/cli.js.map +0 -1
  217. package/dist/mcp/handlers.js.map +0 -1
  218. package/dist/mcp/server.js.map +0 -1
  219. package/dist/opencode/create-read-tool.js.map +0 -1
  220. package/dist/opencode/read-fallback.js.map +0 -1
  221. package/dist/opencode/read-format.js.map +0 -1
  222. package/dist/opencode/read-query.js.map +0 -1
  223. package/dist/opencode/tool-args.js.map +0 -1
  224. package/dist/opencode/tools.js.map +0 -1
  225. package/dist/plugin-entry.js.map +0 -1
  226. package/dist/plugin.js.map +0 -1
  227. package/dist/retriever/context-optimizer.js.map +0 -1
  228. package/dist/retriever/keyword-index.js.map +0 -1
  229. package/dist/retriever/retriever.js.map +0 -1
  230. package/dist/tui.js.map +0 -1
  231. package/dist/updater.d.ts +0 -45
  232. package/dist/updater.js +0 -175
  233. package/dist/updater.js.map +0 -1
  234. package/dist/vectorstore/factory.js.map +0 -1
  235. package/dist/vectorstore/lancedb.js.map +0 -1
  236. package/dist/vectorstore/memory.js.map +0 -1
  237. package/dist/watcher.js.map +0 -1
  238. package/dist/web/api.js.map +0 -1
  239. package/dist/web/server.js.map +0 -1
  240. package/dist/web/static.js.map +0 -1
@@ -0,0 +1,160 @@
1
+ /**
2
+ * @fileoverview Version check and self-update functionality. Checks GitHub
3
+ * releases for new versions and installs the newest version via npm, then
4
+ * re-syncs the OpenCode runtime junctions.
5
+ */
6
+ import { readFileSync } from "node:fs";
7
+ import path from "node:path";
8
+ import { fileURLToPath } from "node:url";
9
+ import { execSync } from "node:child_process";
10
+ import { setupRuntime } from "./setup-runtime.js";
11
+ /** npm package name (must match the `name` field in package.json). */
12
+ const PACKAGE_NAME = "opencode-rag-plugin";
13
+ /** GitHub repo used for release checks (owner/repo). */
14
+ const GITHUB_REPO = "MrDoe/OpenCodeRAG";
15
+ const GITHUB_API_URL = `https://api.github.com/repos/${GITHUB_REPO}/releases/latest`;
16
+ /**
17
+ * Read the current version from the package.json sitting next to this module.
18
+ *
19
+ * Resolves the package root via `import.meta.url` so it works both from the
20
+ * source tree and from the compiled `dist/` output. Returns `"0.0.0"` if the
21
+ * file cannot be read or parsed (best-effort — never throws).
22
+ *
23
+ * @returns The current package version string.
24
+ */
25
+ export function getCurrentVersion() {
26
+ try {
27
+ const pkgPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "package.json");
28
+ const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
29
+ return pkg.version ?? "0.0.0";
30
+ }
31
+ catch {
32
+ return "0.0.0";
33
+ }
34
+ }
35
+ /** Strip a leading 'v' or 'V' prefix from a version tag. */
36
+ function normalizeVersion(tag) {
37
+ return tag.replace(/^v/i, "");
38
+ }
39
+ /**
40
+ * Compare two semver-ish strings.
41
+ * @returns 1 if a > b, -1 if a < b, 0 if equal.
42
+ */
43
+ export function compareVersions(a, b) {
44
+ const pa = a.split(".").map((s) => parseInt(s, 10));
45
+ const pb = b.split(".").map((s) => parseInt(s, 10));
46
+ for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
47
+ const na = pa[i] ?? 0;
48
+ const nb = pb[i] ?? 0;
49
+ if (na > nb)
50
+ return 1;
51
+ if (na < nb)
52
+ return -1;
53
+ }
54
+ return 0;
55
+ }
56
+ /**
57
+ * Check the GitHub releases API for a newer version of OpenCodeRAG.
58
+ *
59
+ * Uses a 5-second timeout; failures (network errors, non-OK responses, missing
60
+ * `tag_name`) are silently caught and reported as "no update available" so the
61
+ * caller never has to handle a rejection.
62
+ *
63
+ * @param currentVersion - The version string to compare against.
64
+ * @returns UpdateInfo indicating whether an update is available.
65
+ */
66
+ export async function checkForUpdate(currentVersion) {
67
+ const controller = new AbortController();
68
+ const timeout = setTimeout(() => controller.abort(), 5_000);
69
+ try {
70
+ const response = await fetch(GITHUB_API_URL, {
71
+ headers: {
72
+ Accept: "application/vnd.github+json",
73
+ "User-Agent": "opencode-rag-updater",
74
+ },
75
+ signal: controller.signal,
76
+ });
77
+ if (!response.ok) {
78
+ return { currentVersion, latestVersion: currentVersion, updateAvailable: false, releaseUrl: "", publishedAt: "" };
79
+ }
80
+ const data = (await response.json());
81
+ const tagName = data.tag_name;
82
+ if (!tagName) {
83
+ return { currentVersion, latestVersion: currentVersion, updateAvailable: false, releaseUrl: "", publishedAt: "" };
84
+ }
85
+ const latestVersion = normalizeVersion(tagName);
86
+ return {
87
+ currentVersion,
88
+ latestVersion,
89
+ updateAvailable: compareVersions(latestVersion, currentVersion) > 0,
90
+ releaseUrl: data.html_url ?? "",
91
+ publishedAt: data.published_at ?? "",
92
+ };
93
+ }
94
+ catch {
95
+ return { currentVersion, latestVersion: currentVersion, updateAvailable: false, releaseUrl: "", publishedAt: "" };
96
+ }
97
+ finally {
98
+ clearTimeout(timeout);
99
+ }
100
+ }
101
+ /**
102
+ * Install the newest published version of OpenCodeRAG.
103
+ *
104
+ * Runs `npm install -g <package>@latest` to refresh the global install, then
105
+ * calls {@link setupRuntime} with `force: true` to re-create the
106
+ * `~/.opencode/node_modules/` junctions so OpenCode picks up the new build on
107
+ * the next restart.
108
+ *
109
+ * @param options - Optional verbosity flag. When `verbose` is true, npm output
110
+ * is streamed to the console; otherwise it is captured silently. The
111
+ * `_execSync` and `_setupRuntime` seams are for testing only.
112
+ * @returns An {@link InstallUpdateResult} with success status, message, and the
113
+ * from/to versions.
114
+ */
115
+ export async function installLatestUpdate(options) {
116
+ const verbose = options?.verbose ?? false;
117
+ const stdio = verbose ? "inherit" : "pipe";
118
+ const run = options?._execSync ?? execSync;
119
+ const syncRuntime = options?._setupRuntime ?? setupRuntime;
120
+ const readVersion = options?._getCurrentVersion ?? getCurrentVersion;
121
+ const fromVersion = readVersion();
122
+ try {
123
+ run(`npm install -g ${PACKAGE_NAME}@latest --no-fund --no-audit`, {
124
+ stdio,
125
+ timeout: 120_000,
126
+ });
127
+ }
128
+ catch (err) {
129
+ return {
130
+ success: false,
131
+ message: `npm install failed: ${err.message}`,
132
+ fromVersion,
133
+ };
134
+ }
135
+ const toVersion = readVersion();
136
+ const result = await syncRuntime({ force: true, silent: !verbose, version: toVersion });
137
+ if (!result.success) {
138
+ return {
139
+ success: false,
140
+ message: `installed v${toVersion} but runtime sync failed: ${result.errors.join("; ")}. Run \`opencode-rag setup\` to retry.`,
141
+ fromVersion,
142
+ toVersion,
143
+ };
144
+ }
145
+ if (compareVersions(toVersion, fromVersion) <= 0) {
146
+ return {
147
+ success: true,
148
+ message: `Already up-to-date (v${toVersion}). Runtime re-synced.`,
149
+ fromVersion,
150
+ toVersion,
151
+ };
152
+ }
153
+ return {
154
+ success: true,
155
+ message: `Updated v${fromVersion} → v${toVersion}. Restart OpenCode to load the new version.`,
156
+ fromVersion,
157
+ toVersion,
158
+ };
159
+ }
160
+ //# sourceMappingURL=version-check.js.map
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * @fileoverview Anthropic Messages API description provider for generating natural-language descriptions of code chunks.
3
3
  */
4
- import type { Chunk, DescriptionProvider, DescriptionLogger } from "../core/interfaces.js";
4
+ import type { BatchDescriptionOptions, Chunk, DescriptionProvider, DescriptionLogger } from "../core/interfaces.js";
5
5
  import type { DescriptionConfig } from "../core/config.js";
6
6
  /**
7
7
  * Description provider that uses Anthropic's Messages API to generate natural-language descriptions of code chunks.
@@ -18,7 +18,7 @@ export declare class AnthropicDescriptionProvider implements DescriptionProvider
18
18
  /** @inheritdoc */
19
19
  generateDescription(chunk: Chunk): Promise<string>;
20
20
  /** @inheritdoc */
21
- generateBatchDescriptions(chunks: Chunk[], logger?: DescriptionLogger): Promise<Map<string, string>>;
21
+ generateBatchDescriptions(chunks: Chunk[], logger?: DescriptionLogger, opts?: BatchDescriptionOptions): Promise<Map<string, string>>;
22
22
  /**
23
23
  * Sends a request to the Anthropic Messages API with retry and exponential backoff.
24
24
  * The system prompt is combined with all user messages into a single message payload, and
@@ -25,11 +25,11 @@ export class AnthropicDescriptionProvider {
25
25
  return this.chatRequest(messages, this.config.timeoutMs ?? 60000);
26
26
  }
27
27
  /** @inheritdoc */
28
- async generateBatchDescriptions(chunks, logger) {
29
- const log = logger ?? { info: (msg) => console.log(msg), warn: (msg) => console.warn(msg), debug: (msg) => console.debug(msg) };
28
+ async generateBatchDescriptions(chunks, logger, opts) {
29
+ const log = logger ?? { info: (msg) => process.stderr.write(`${msg}\n`), warn: (msg) => process.stderr.write(`${msg}\n`), debug: (msg) => process.stderr.write(`${msg}\n`) };
30
30
  const concurrency = this.config.batchConcurrency ?? 3;
31
31
  const total = chunks.length;
32
- log.info(`[describer] Generating descriptions for ${total} chunks (concurrency: ${concurrency})`);
32
+ log.debug(`[describer] Generating descriptions for ${total} chunks (concurrency: ${concurrency})`);
33
33
  const result = new Map();
34
34
  const limit = pLimit(concurrency);
35
35
  let completed = 0;
@@ -45,11 +45,9 @@ export class AnthropicDescriptionProvider {
45
45
  log.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
46
46
  }
47
47
  completed++;
48
- if (completed % 25 === 0 || completed === total) {
49
- log.info(`[describer] Progress: ${completed}/${total}`);
50
- }
48
+ opts?.onProgress?.(chunk, completed, opts.total ?? total);
51
49
  })));
52
- log.info(`[describer] Descriptions generated: ${result.size}/${total}`);
50
+ log.debug(`[describer] Descriptions generated: ${result.size}/${total}`);
53
51
  return result;
54
52
  }
55
53
  /**
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * @fileoverview OpenAI-compatible LLM description provider for generating natural-language descriptions of code chunks.
3
3
  */
4
- import type { Chunk, DescriptionProvider, DescriptionLogger } from "../core/interfaces.js";
4
+ import type { BatchDescriptionOptions, Chunk, DescriptionProvider, DescriptionLogger } from "../core/interfaces.js";
5
5
  import type { DescriptionConfig } from "../core/config.js";
6
6
  /**
7
7
  * Description provider that works with any OpenAI-compatible chat API (including Ollama).
@@ -18,7 +18,7 @@ export declare class LlmDescriptionProvider implements DescriptionProvider {
18
18
  /** @inheritdoc */
19
19
  generateDescription(chunk: Chunk): Promise<string>;
20
20
  /** @inheritdoc */
21
- generateBatchDescriptions(chunks: Chunk[], logger?: DescriptionLogger): Promise<Map<string, string>>;
21
+ generateBatchDescriptions(chunks: Chunk[], logger?: DescriptionLogger, opts?: BatchDescriptionOptions): Promise<Map<string, string>>;
22
22
  /**
23
23
  * Sends a chat completion request to the LLM API with retry and exponential backoff.
24
24
  * For Ollama, uses the `/api/chat` endpoint with streaming disabled; otherwise uses the standard `/v1/chat/completions` endpoint.
@@ -2,7 +2,7 @@ import { postJson } from "../embedder/http.js";
2
2
  import { buildUserMessage, sleep } from "./shared.js";
3
3
  import pLimit from "p-limit";
4
4
  /** HTTP status codes that are safe to retry on. */
5
- const RETRYABLE_STATUSES = new Set([404, 408, 429, 500, 502, 503, 504]);
5
+ const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]);
6
6
  /**
7
7
  * Description provider that works with any OpenAI-compatible chat API (including Ollama).
8
8
  *
@@ -26,11 +26,11 @@ export class LlmDescriptionProvider {
26
26
  return this.chatRequest(messages, this.config.timeoutMs ?? 60000);
27
27
  }
28
28
  /** @inheritdoc */
29
- async generateBatchDescriptions(chunks, logger) {
30
- const log = logger ?? { info: (msg) => process.stderr.write(`${msg}\n`), warn: (msg) => console.warn(msg), debug: (msg) => console.debug(msg) };
29
+ async generateBatchDescriptions(chunks, logger, opts) {
30
+ const log = logger ?? { info: (msg) => process.stderr.write(`${msg}\n`), warn: (msg) => process.stderr.write(`${msg}\n`), debug: (msg) => process.stderr.write(`${msg}\n`) };
31
31
  const concurrency = this.config.batchConcurrency ?? 3;
32
32
  const total = chunks.length;
33
- log.info(`Generating descriptions for ${total} chunks via ${this.config.provider}/${this.config.model} (concurrency: ${concurrency})...`);
33
+ log.debug(`[describer] Generating descriptions for ${total} chunks via ${this.config.provider}/${this.config.model} (concurrency: ${concurrency})`);
34
34
  const result = new Map();
35
35
  const limit = pLimit(concurrency);
36
36
  let completed = 0;
@@ -46,11 +46,9 @@ export class LlmDescriptionProvider {
46
46
  log.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
47
47
  }
48
48
  completed++;
49
- if (completed % 25 === 0 || completed === total) {
50
- log.info(`Descriptions: ${completed}/${total}`);
51
- }
49
+ opts?.onProgress?.(chunk, completed, opts.total ?? total);
52
50
  })));
53
- log.info(`Descriptions: ${result.size}/${total} done.`);
51
+ log.debug(`[describer] Descriptions generated: ${result.size}/${total}`);
54
52
  return result;
55
53
  }
56
54
  /**
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * @fileoverview Google Gemini description provider for generating natural-language descriptions of code chunks.
3
3
  */
4
- import type { Chunk, DescriptionProvider, DescriptionLogger } from "../core/interfaces.js";
4
+ import type { BatchDescriptionOptions, Chunk, DescriptionProvider, DescriptionLogger } from "../core/interfaces.js";
5
5
  import type { DescriptionConfig } from "../core/config.js";
6
6
  /**
7
7
  * Description provider that uses Google Gemini's generateContent API to describe code chunks.
@@ -12,6 +12,6 @@ export declare class GeminiDescriptionProvider implements DescriptionProvider {
12
12
  private readonly config;
13
13
  constructor(config: DescriptionConfig);
14
14
  generateDescription(chunk: Chunk): Promise<string>;
15
- generateBatchDescriptions(chunks: Chunk[], logger?: DescriptionLogger): Promise<Map<string, string>>;
15
+ generateBatchDescriptions(chunks: Chunk[], logger?: DescriptionLogger, opts?: BatchDescriptionOptions): Promise<Map<string, string>>;
16
16
  private chatRequest;
17
17
  }
@@ -21,11 +21,11 @@ export class GeminiDescriptionProvider {
21
21
  ];
22
22
  return this.chatRequest(contents, this.config.timeoutMs ?? 60000);
23
23
  }
24
- async generateBatchDescriptions(chunks, logger) {
25
- const log = logger ?? { info: (msg) => console.log(msg), warn: (msg) => console.warn(msg), debug: (msg) => console.debug(msg) };
24
+ async generateBatchDescriptions(chunks, logger, opts) {
25
+ const log = logger ?? { info: (msg) => process.stderr.write(`${msg}\n`), warn: (msg) => process.stderr.write(`${msg}\n`), debug: (msg) => process.stderr.write(`${msg}\n`) };
26
26
  const concurrency = this.config.batchConcurrency ?? 3;
27
27
  const total = chunks.length;
28
- log.info(`[describer] Generating descriptions for ${total} chunks (concurrency: ${concurrency})`);
28
+ log.debug(`[describer] Generating descriptions for ${total} chunks (concurrency: ${concurrency})`);
29
29
  const result = new Map();
30
30
  const limit = pLimit(concurrency);
31
31
  let completed = 0;
@@ -41,11 +41,9 @@ export class GeminiDescriptionProvider {
41
41
  log.warn(`[describer] Failed to describe chunk ${chunk.id} (${chunk.metadata.filePath}:${chunk.metadata.startLine}): ${err instanceof Error ? err.message : String(err)}`);
42
42
  }
43
43
  completed++;
44
- if (completed % 25 === 0 || completed === total) {
45
- log.info(`[describer] Progress: ${completed}/${total}`);
46
- }
44
+ opts?.onProgress?.(chunk, completed, opts.total ?? total);
47
45
  })));
48
- log.info(`[describer] Descriptions generated: ${result.size}/${total}`);
46
+ log.debug(`[describer] Descriptions generated: ${result.size}/${total}`);
49
47
  return result;
50
48
  }
51
49
  async chatRequest(contents, timeoutMs) {
@@ -9,7 +9,7 @@ import type { ProxyConfig } from "../core/config.js";
9
9
  * @param baseUrl - Cohere API base URL
10
10
  * @param model - Model name to use for embedding
11
11
  * @param apiKey - API key for authentication
12
- * @param timeoutMs - Request timeout in milliseconds (default 30000)
12
+ * @param timeoutMs - Request timeout in milliseconds (default 120000)
13
13
  * @param proxy - Optional proxy configuration
14
14
  */
15
15
  export declare class CohereProvider implements EmbeddingProvider {
@@ -5,7 +5,7 @@ import { postJson } from "./http.js";
5
5
  * @param baseUrl - Cohere API base URL
6
6
  * @param model - Model name to use for embedding
7
7
  * @param apiKey - API key for authentication
8
- * @param timeoutMs - Request timeout in milliseconds (default 30000)
8
+ * @param timeoutMs - Request timeout in milliseconds (default 120000)
9
9
  * @param proxy - Optional proxy configuration
10
10
  */
11
11
  export class CohereProvider {
@@ -15,7 +15,7 @@ export class CohereProvider {
15
15
  apiKey;
16
16
  timeoutMs;
17
17
  proxy;
18
- constructor(baseUrl, model, apiKey, timeoutMs = 30000, proxy) {
18
+ constructor(baseUrl, model, apiKey, timeoutMs = 120000, proxy) {
19
19
  this.baseUrl = baseUrl.replace(/\/+$/, "");
20
20
  this.model = model;
21
21
  this.apiKey = apiKey;
@@ -27,6 +27,8 @@ export declare function createEmbedder(config: RagConfig): EmbeddingProvider;
27
27
  * @param batchSize - Number of texts per batch (default 10)
28
28
  * @param purpose - Optional hint for query vs. document embedding
29
29
  * @param concurrency - Maximum number of concurrent batch requests (default 1)
30
+ * @param onProgress - Optional callback invoked after each batch with the running
31
+ * completed count and total; per-text granularity when `concurrency <= 1`.
30
32
  * @returns A promise resolving to a flat array of embedding vectors (one per input text)
31
33
  */
32
- export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number): Promise<number[][]>;
34
+ export declare function embedBatch(embedder: EmbeddingProvider, texts: string[], batchSize?: number, purpose?: "query" | "document", concurrency?: number, onProgress?: (completed: number, total: number) => void): Promise<number[][]>;
@@ -16,7 +16,7 @@ import pLimit from "p-limit";
16
16
  */
17
17
  export function createEmbedder(config) {
18
18
  const { provider, baseUrl, model, apiKey, proxy, timeoutMs } = config.embedding;
19
- const effectiveTimeoutMs = timeoutMs ?? 30000;
19
+ const effectiveTimeoutMs = timeoutMs ?? 120000;
20
20
  if (provider === "ollama") {
21
21
  return new OllamaProvider(baseUrl, model, apiKey, effectiveTimeoutMs, proxy, config.logging.level);
22
22
  }
@@ -46,9 +46,11 @@ export function createEmbedder(config) {
46
46
  * @param batchSize - Number of texts per batch (default 10)
47
47
  * @param purpose - Optional hint for query vs. document embedding
48
48
  * @param concurrency - Maximum number of concurrent batch requests (default 1)
49
+ * @param onProgress - Optional callback invoked after each batch with the running
50
+ * completed count and total; per-text granularity when `concurrency <= 1`.
49
51
  * @returns A promise resolving to a flat array of embedding vectors (one per input text)
50
52
  */
51
- export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1) {
53
+ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concurrency = 1, onProgress) {
52
54
  if (texts.length === 0)
53
55
  return [];
54
56
  const batches = [];
@@ -60,12 +62,16 @@ export async function embedBatch(embedder, texts, batchSize = 10, purpose, concu
60
62
  for (const batch of batches) {
61
63
  const embeddings = await embedder.embed(batch.texts, purpose);
62
64
  results.push(...embeddings);
65
+ onProgress?.(results.length, texts.length);
63
66
  }
64
67
  return results;
65
68
  }
66
69
  const limit = pLimit(concurrency);
70
+ let completedCount = 0;
67
71
  const batchResults = await Promise.all(batches.map((batch) => limit(async () => {
68
72
  const embeddings = await embedder.embed(batch.texts, purpose);
73
+ completedCount += embeddings.length;
74
+ onProgress?.(completedCount, texts.length);
69
75
  return { index: batch.index, embeddings };
70
76
  })));
71
77
  batchResults.sort((a, b) => a.index - b.index);
@@ -4,7 +4,7 @@ import { postJson } from "./http.js";
4
4
  * Returns one result per configured model (embedding + description + image_description if enabled).
5
5
  */
6
6
  export async function checkProviderHealth(config) {
7
- const timeoutMs = config.embedding.timeoutMs ?? 30000;
7
+ const timeoutMs = config.embedding.timeoutMs ?? 120000;
8
8
  const checks = [
9
9
  checkEmbeddingModel(config, timeoutMs),
10
10
  ];
@@ -328,6 +328,10 @@ export async function pullOllamaModels(models, onProgress) {
328
328
  }
329
329
  }
330
330
  finally {
331
+ try {
332
+ await reader.cancel();
333
+ }
334
+ catch { }
331
335
  reader.releaseLock();
332
336
  }
333
337
  }
@@ -58,4 +58,4 @@ export declare function directRequest(url: URL, body: unknown, headers: Record<s
58
58
  * @param proxy - Optional proxy configuration
59
59
  * @returns A promise resolving to an HttpResponseLike
60
60
  */
61
- export declare function postJson(urlString: string, body: unknown, headers: Record<string, string>, timeoutMs: number, proxy?: ProxyConfig): Promise<HttpResponseLike>;
61
+ export declare function postJson(urlString: string, body: unknown, headers: Record<string, string>, timeoutMs: number, proxy?: ProxyConfig, signal?: AbortSignal): Promise<HttpResponseLike>;
@@ -6,6 +6,10 @@ import tls from "node:tls";
6
6
  const MAX_POOL_SIZE = 4;
7
7
  const IDLE_TIMEOUT_MS = 30000;
8
8
  const connectionPool = new Map();
9
+ // Serialize proxied fetch requests that mutate process.env,
10
+ // preventing races when concurrent requests interleave
11
+ // set/restore of HTTP_PROXY/HTTPS_PROXY.
12
+ let proxyRequestQueue = Promise.resolve(undefined);
9
13
  function poolKey(host, port, isHttps) {
10
14
  return `${isHttps ? "tls" : "tcp"}:${host}:${port}`;
11
15
  }
@@ -43,7 +47,13 @@ function releaseConnection(socket, host, port, isHttps) {
43
47
  pool.splice(idx, 1);
44
48
  socket.destroy();
45
49
  }, IDLE_TIMEOUT_MS);
46
- socket.removeAllListeners();
50
+ // Avoid removeAllListeners — that would strip the error handler
51
+ // and cause an unhandled crash on ECONNRESET/TLS alert.
52
+ socket.removeAllListeners("data");
53
+ socket.removeAllListeners("end");
54
+ socket.removeAllListeners("drain");
55
+ socket.removeAllListeners("error");
56
+ socket.on("error", () => socket.destroy());
47
57
  pool.push({ socket, idleTimer });
48
58
  }
49
59
  /** Destroy all pooled TCP/TLS sockets and clear the connection pool. */
@@ -307,9 +317,9 @@ async function sendRawHttpRequest(url, body, headers, timeoutMs, redirectCount)
307
317
  return;
308
318
  }
309
319
  if (isChunked) {
310
- if (bodyBuffer.length >= 5) {
311
- const tail = bodyBuffer.slice(-7).toString("ascii");
312
- if (tail === "\r\n0\r\n\r\n" || bodyBuffer.toString("ascii") === "0\r\n\r\n") {
320
+ if (bodyBuffer.length >= 7) {
321
+ const ascii = bodyBuffer.toString("ascii");
322
+ if (/0(?:;[^\n]*)?\r\n(?:[^\n]+:[^\n]*\r\n)*\r\n$/.test(ascii)) {
313
323
  settle(() => {
314
324
  releaseOrDestroy();
315
325
  resolve(assembled);
@@ -371,7 +381,15 @@ async function sendRawHttpRequest(url, body, headers, timeoutMs, redirectCount)
371
381
  },
372
382
  };
373
383
  }
374
- return sendRawHttpRequest(new URL(location, url), body, headers, timeoutMs, redirectCount + 1);
384
+ const redirectUrl = new URL(location, url);
385
+ const sameOrigin = url.protocol === redirectUrl.protocol &&
386
+ url.hostname === redirectUrl.hostname &&
387
+ (url.port || (url.protocol === "https:" ? "443" : "80")) ===
388
+ (redirectUrl.port || (redirectUrl.protocol === "https:" ? "443" : "80"));
389
+ const safeHeaders = sameOrigin
390
+ ? headers
391
+ : Object.fromEntries(Object.entries(headers).filter(([key]) => key.toLowerCase() !== "authorization"));
392
+ return sendRawHttpRequest(redirectUrl, body, safeHeaders, timeoutMs, redirectCount + 1);
375
393
  }
376
394
  const text = response.body.toString("utf8");
377
395
  return {
@@ -399,52 +417,65 @@ async function sendRawHttpRequest(url, body, headers, timeoutMs, redirectCount)
399
417
  * @param proxy - Optional proxy configuration
400
418
  * @returns A promise resolving to an HttpResponseLike
401
419
  */
402
- export async function postJson(urlString, body, headers, timeoutMs, proxy) {
420
+ export async function postJson(urlString, body, headers, timeoutMs, proxy, signal) {
403
421
  const url = new URL(urlString);
404
422
  const bypassProxy = isLocalhost(url.hostname) || matchesNoProxy(url.hostname, proxy?.noProxy);
405
423
  if (bypassProxy || !proxy?.url) {
406
424
  return directRequest(url, body, headers, timeoutMs);
407
425
  }
408
- return postJsonViaFetch(urlString, body, headers, timeoutMs, proxy);
426
+ return postJsonViaFetch(urlString, body, headers, timeoutMs, proxy, signal);
409
427
  }
410
428
  /** Send JSON via the global `fetch` API with proxy environment variable overrides. */
411
- async function postJsonViaFetch(urlString, body, headers, timeoutMs, proxy) {
429
+ async function postJsonViaFetch(urlString, body, headers, timeoutMs, proxy, signal) {
412
430
  const authHeader = buildProxyAuthHeader(proxy);
413
431
  const envOverride = applyProxyEnv(proxy);
414
- const savedHttpProxy = process.env.HTTP_PROXY;
415
- const savedHttpsProxy = process.env.HTTPS_PROXY;
416
- try {
417
- if (envOverride) {
418
- process.env.HTTP_PROXY = envOverride.httpProxy;
419
- process.env.HTTPS_PROXY = envOverride.httpsProxy;
420
- }
421
- const requestHeaders = {
422
- "Content-Type": "application/json",
423
- ...headers,
424
- };
425
- if (authHeader) {
426
- requestHeaders["Proxy-Authorization"] = authHeader;
427
- }
428
- const controller = new AbortController();
429
- const timeout = setTimeout(() => controller.abort(), timeoutMs);
430
- try {
431
- const response = await fetch(urlString, {
432
- method: "POST",
433
- headers: requestHeaders,
434
- body: JSON.stringify(body),
435
- signal: controller.signal,
436
- });
437
- return response;
438
- }
439
- finally {
440
- clearTimeout(timeout);
441
- }
442
- }
443
- finally {
444
- if (envOverride) {
445
- process.env.HTTP_PROXY = savedHttpProxy;
446
- process.env.HTTPS_PROXY = savedHttpsProxy;
447
- }
448
- }
432
+ // Serialize fetch requests that mutate process.env to prevent
433
+ // concurrent requests from interleaving set/restore of proxy vars.
434
+ const [savedHttpProxy, savedHttpsProxy] = [process.env.HTTP_PROXY, process.env.HTTPS_PROXY];
435
+ return new Promise((resolve, reject) => {
436
+ proxyRequestQueue = proxyRequestQueue.then(async () => {
437
+ try {
438
+ if (envOverride) {
439
+ process.env.HTTP_PROXY = envOverride.httpProxy;
440
+ process.env.HTTPS_PROXY = envOverride.httpsProxy;
441
+ }
442
+ const requestHeaders = {
443
+ "Content-Type": "application/json",
444
+ ...headers,
445
+ };
446
+ if (authHeader) {
447
+ requestHeaders["Proxy-Authorization"] = authHeader;
448
+ }
449
+ const controller = new AbortController();
450
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
451
+ const onAbort = () => { controller.abort(); clearTimeout(timeout); };
452
+ signal?.addEventListener("abort", onAbort, { once: true });
453
+ try {
454
+ const response = await fetch(urlString, {
455
+ method: "POST",
456
+ headers: requestHeaders,
457
+ body: JSON.stringify(body),
458
+ signal: controller.signal,
459
+ });
460
+ resolve(response);
461
+ }
462
+ finally {
463
+ clearTimeout(timeout);
464
+ signal?.removeEventListener("abort", onAbort);
465
+ if (envOverride) {
466
+ process.env.HTTP_PROXY = savedHttpProxy;
467
+ process.env.HTTPS_PROXY = savedHttpsProxy;
468
+ }
469
+ }
470
+ }
471
+ catch (err) {
472
+ if (envOverride) {
473
+ process.env.HTTP_PROXY = savedHttpProxy;
474
+ process.env.HTTPS_PROXY = savedHttpsProxy;
475
+ }
476
+ reject(err);
477
+ }
478
+ });
479
+ });
449
480
  }
450
481
  //# sourceMappingURL=http.js.map
@@ -9,7 +9,7 @@ import type { ProxyConfig } from "../core/config.js";
9
9
  * @param baseUrl - Ollama server base URL (e.g. http://localhost:11434)
10
10
  * @param model - Model name to use for embedding
11
11
  * @param apiKey - Optional API key for authenticated endpoints
12
- * @param timeoutMs - Request timeout in milliseconds (default 30000)
12
+ * @param timeoutMs - Request timeout in milliseconds (default 120000)
13
13
  * @param proxy - Optional proxy configuration
14
14
  * @param logLevel - Optional logging level for debug output
15
15
  */
@@ -7,7 +7,7 @@ import { appendDebugLog } from "../core/fileLogger.js";
7
7
  * @param baseUrl - Ollama server base URL (e.g. http://localhost:11434)
8
8
  * @param model - Model name to use for embedding
9
9
  * @param apiKey - Optional API key for authenticated endpoints
10
- * @param timeoutMs - Request timeout in milliseconds (default 30000)
10
+ * @param timeoutMs - Request timeout in milliseconds (default 120000)
11
11
  * @param proxy - Optional proxy configuration
12
12
  * @param logLevel - Optional logging level for debug output
13
13
  */
@@ -19,7 +19,7 @@ export class OllamaProvider {
19
19
  timeoutMs;
20
20
  proxy;
21
21
  logLevel;
22
- constructor(baseUrl, model, apiKey, timeoutMs = 30000, proxy, logLevel) {
22
+ constructor(baseUrl, model, apiKey, timeoutMs = 120000, proxy, logLevel) {
23
23
  this.baseUrl = baseUrl.replace(/\/+$/, "");
24
24
  this.model = model;
25
25
  this.apiKey = apiKey;
@@ -9,7 +9,7 @@ import type { ProxyConfig } from "../core/config.js";
9
9
  * @param baseUrl - API base URL (e.g. https://api.openai.com/v1)
10
10
  * @param model - Model name to use for embedding
11
11
  * @param apiKey - API key for authentication
12
- * @param timeoutMs - Request timeout in milliseconds (default 30000)
12
+ * @param timeoutMs - Request timeout in milliseconds (default 120000)
13
13
  * @param proxy - Optional proxy configuration
14
14
  */
15
15
  export declare class OpenAIProvider implements EmbeddingProvider {
@@ -16,7 +16,7 @@ function inferProviderName(baseUrl) {
16
16
  * @param baseUrl - API base URL (e.g. https://api.openai.com/v1)
17
17
  * @param model - Model name to use for embedding
18
18
  * @param apiKey - API key for authentication
19
- * @param timeoutMs - Request timeout in milliseconds (default 30000)
19
+ * @param timeoutMs - Request timeout in milliseconds (default 120000)
20
20
  * @param proxy - Optional proxy configuration
21
21
  */
22
22
  export class OpenAIProvider {
@@ -27,7 +27,7 @@ export class OpenAIProvider {
27
27
  timeoutMs;
28
28
  proxy;
29
29
  provider;
30
- constructor(baseUrl, model, apiKey, timeoutMs = 30000, proxy) {
30
+ constructor(baseUrl, model, apiKey, timeoutMs = 120000, proxy) {
31
31
  this.baseUrl = baseUrl.replace(/\/+$/, "");
32
32
  this.model = model;
33
33
  this.apiKey = apiKey;
@@ -0,0 +1,10 @@
1
+ /**
2
+ * @fileoverview Compare two branch benchmark JSON outputs and produce a side-by-side
3
+ * analysis report (console table + markdown file).
4
+ *
5
+ * Usage: node --import tsx src/eval/compare-merge.ts
6
+ * --main .opencode/rag_db/eval-results/main.json
7
+ * --branch .opencode/rag_db/eval-results/t1-cosine-l2.json
8
+ * --output .opencode/rag_db/eval-results/branch-compare-report.md
9
+ */
10
+ export {};