opencode-rag-plugin 1.15.1 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. package/ReadMe.md +4 -6
  2. package/dist/api.js +44 -24
  3. package/dist/chunker/base.d.ts +34 -3
  4. package/dist/chunker/base.js +70 -21
  5. package/dist/chunker/factory.d.ts +4 -1
  6. package/dist/chunker/factory.js +12 -1
  7. package/dist/chunker/grammar.js +3 -0
  8. package/dist/chunker/image.js +8 -8
  9. package/dist/chunker/pdf.js +11 -13
  10. package/dist/chunker/xml.d.ts +2 -0
  11. package/dist/chunker/xml.js +2 -0
  12. package/dist/cli/commands/index-command.js +0 -4
  13. package/dist/cli/commands/index.d.ts +1 -1
  14. package/dist/cli/commands/index.js +1 -1
  15. package/dist/cli/commands/init-helpers.d.ts +12 -0
  16. package/dist/cli/commands/init-helpers.js +84 -2
  17. package/dist/cli/commands/init.js +20 -2
  18. package/dist/cli/commands/query.js +1 -0
  19. package/dist/cli/commands/setup.d.ts +2 -0
  20. package/dist/cli/commands/setup.js +113 -0
  21. package/dist/cli/commands/status.js +64 -2
  22. package/dist/cli/index.js +2 -2
  23. package/dist/content/image.js +24 -3
  24. package/dist/content/reader.d.ts +6 -1
  25. package/dist/content/reader.js +49 -4
  26. package/dist/core/config.d.ts +19 -0
  27. package/dist/core/config.js +34 -4
  28. package/dist/core/desc-cache.d.ts +31 -0
  29. package/dist/core/desc-cache.js +124 -0
  30. package/dist/core/interfaces.d.ts +58 -3
  31. package/dist/core/manifest.d.ts +24 -1
  32. package/dist/core/manifest.js +34 -3
  33. package/dist/core/resolve-api-key.js +4 -2
  34. package/dist/core/runtime-overrides.js +9 -7
  35. package/dist/core/setup-runtime.d.ts +23 -0
  36. package/dist/core/setup-runtime.js +183 -0
  37. package/dist/core/version-check.d.ts +9 -0
  38. package/dist/core/version-check.js +49 -0
  39. package/dist/describer/anthropic.d.ts +2 -2
  40. package/dist/describer/anthropic.js +5 -7
  41. package/dist/describer/describer.d.ts +2 -2
  42. package/dist/describer/describer.js +6 -8
  43. package/dist/describer/gemini.d.ts +2 -2
  44. package/dist/describer/gemini.js +5 -7
  45. package/dist/embedder/factory.d.ts +3 -1
  46. package/dist/embedder/factory.js +7 -1
  47. package/dist/embedder/health.js +4 -0
  48. package/dist/embedder/http.d.ts +1 -1
  49. package/dist/embedder/http.js +74 -43
  50. package/dist/eval/compare-merge.d.ts +10 -0
  51. package/dist/eval/compare-merge.js +537 -0
  52. package/dist/eval/compare-rankings.d.ts +10 -0
  53. package/dist/eval/compare-rankings.js +245 -0
  54. package/dist/eval/dump-descriptions.d.ts +7 -0
  55. package/dist/eval/dump-descriptions.js +58 -0
  56. package/dist/eval/fast-index.d.ts +8 -0
  57. package/dist/eval/fast-index.js +283 -0
  58. package/dist/eval/run-branch-compare.d.ts +7 -0
  59. package/dist/eval/run-branch-compare.js +220 -0
  60. package/dist/eval/run-token-test.js +1 -0
  61. package/dist/eval/test-kw.d.ts +1 -0
  62. package/dist/eval/test-kw.js +22 -0
  63. package/dist/eval/update-descriptions.d.ts +7 -0
  64. package/dist/eval/update-descriptions.js +84 -0
  65. package/dist/index.d.ts +1 -0
  66. package/dist/index.js +1 -0
  67. package/dist/indexer/embed-stage.js +2 -1
  68. package/dist/indexer/git-diff.js +21 -9
  69. package/dist/indexer/pipeline.d.ts +6 -0
  70. package/dist/indexer/pipeline.js +290 -37
  71. package/dist/indexer/watch.js +1 -3
  72. package/dist/indexer/worker.d.ts +15 -2
  73. package/dist/indexer/worker.js +25 -12
  74. package/dist/mcp/handlers.d.ts +9 -0
  75. package/dist/mcp/handlers.js +23 -6
  76. package/dist/mcp/server.js +2 -0
  77. package/dist/opencode/create-read-tool.d.ts +2 -0
  78. package/dist/opencode/create-read-tool.js +8 -2
  79. package/dist/opencode/read-fallback.d.ts +1 -5
  80. package/dist/opencode/read-fallback.js +1 -18
  81. package/dist/opencode/read-format.js +5 -3
  82. package/dist/opencode/tools.js +5 -7
  83. package/dist/plugin.js +141 -57
  84. package/dist/retriever/keyword-index.d.ts +3 -2
  85. package/dist/retriever/keyword-index.js +25 -1
  86. package/dist/retriever/retriever.d.ts +4 -1
  87. package/dist/retriever/retriever.js +34 -56
  88. package/dist/vectorstore/lancedb.d.ts +35 -4
  89. package/dist/vectorstore/lancedb.js +146 -23
  90. package/dist/vectorstore/memory.d.ts +6 -1
  91. package/dist/vectorstore/memory.js +58 -0
  92. package/dist/watcher.js +3 -0
  93. package/dist/web/api.js +10 -2
  94. package/dist/web/server.js +18 -3
  95. package/package.json +8 -9
  96. package/scripts/postinstall-setup.js +82 -0
  97. package/dist/api.js.map +0 -1
  98. package/dist/chunker/base.js.map +0 -1
  99. package/dist/chunker/bash.js.map +0 -1
  100. package/dist/chunker/c.js.map +0 -1
  101. package/dist/chunker/cpp.js.map +0 -1
  102. package/dist/chunker/csharp.js.map +0 -1
  103. package/dist/chunker/css.js.map +0 -1
  104. package/dist/chunker/doc.js.map +0 -1
  105. package/dist/chunker/dockerfile.js.map +0 -1
  106. package/dist/chunker/docx.js.map +0 -1
  107. package/dist/chunker/excel.js.map +0 -1
  108. package/dist/chunker/factory.js.map +0 -1
  109. package/dist/chunker/fallback.js.map +0 -1
  110. package/dist/chunker/go.js.map +0 -1
  111. package/dist/chunker/grammar.js.map +0 -1
  112. package/dist/chunker/html.js.map +0 -1
  113. package/dist/chunker/image.js.map +0 -1
  114. package/dist/chunker/ini.js.map +0 -1
  115. package/dist/chunker/java.js.map +0 -1
  116. package/dist/chunker/javascript.js.map +0 -1
  117. package/dist/chunker/json.js.map +0 -1
  118. package/dist/chunker/kotlin.js.map +0 -1
  119. package/dist/chunker/loader.js.map +0 -1
  120. package/dist/chunker/markdown.js.map +0 -1
  121. package/dist/chunker/pdf.js.map +0 -1
  122. package/dist/chunker/php.js.map +0 -1
  123. package/dist/chunker/powershell.js.map +0 -1
  124. package/dist/chunker/python.js.map +0 -1
  125. package/dist/chunker/razor.js.map +0 -1
  126. package/dist/chunker/ruby.js.map +0 -1
  127. package/dist/chunker/rust.js.map +0 -1
  128. package/dist/chunker/sln.js.map +0 -1
  129. package/dist/chunker/sql.js.map +0 -1
  130. package/dist/chunker/ssl.js.map +0 -1
  131. package/dist/chunker/swift.js.map +0 -1
  132. package/dist/chunker/tex.js.map +0 -1
  133. package/dist/chunker/toml.js.map +0 -1
  134. package/dist/chunker/typescript.js.map +0 -1
  135. package/dist/chunker/uuid.js.map +0 -1
  136. package/dist/chunker/xml.js.map +0 -1
  137. package/dist/chunker/yaml.js.map +0 -1
  138. package/dist/cli/commands/clear.js.map +0 -1
  139. package/dist/cli/commands/describe-image.js.map +0 -1
  140. package/dist/cli/commands/dump.js.map +0 -1
  141. package/dist/cli/commands/eval.js.map +0 -1
  142. package/dist/cli/commands/index-command.js.map +0 -1
  143. package/dist/cli/commands/index.js.map +0 -1
  144. package/dist/cli/commands/init-helpers.js.map +0 -1
  145. package/dist/cli/commands/init.js.map +0 -1
  146. package/dist/cli/commands/list.js.map +0 -1
  147. package/dist/cli/commands/mcp.js.map +0 -1
  148. package/dist/cli/commands/query.js.map +0 -1
  149. package/dist/cli/commands/show.js.map +0 -1
  150. package/dist/cli/commands/status.js.map +0 -1
  151. package/dist/cli/commands/ui.js.map +0 -1
  152. package/dist/cli/commands/update.d.ts +0 -17
  153. package/dist/cli/commands/update.js +0 -79
  154. package/dist/cli/commands/update.js.map +0 -1
  155. package/dist/cli/format.js.map +0 -1
  156. package/dist/cli/helpers.js.map +0 -1
  157. package/dist/cli/index.js.map +0 -1
  158. package/dist/cli/progress.d.ts +0 -42
  159. package/dist/cli/progress.js +0 -137
  160. package/dist/cli/progress.js.map +0 -1
  161. package/dist/cli/types.js.map +0 -1
  162. package/dist/cli.js.map +0 -1
  163. package/dist/content/doc.js.map +0 -1
  164. package/dist/content/docx.js.map +0 -1
  165. package/dist/content/excel.js.map +0 -1
  166. package/dist/content/image.js.map +0 -1
  167. package/dist/content/pdf.js.map +0 -1
  168. package/dist/content/reader.js.map +0 -1
  169. package/dist/content/types.js.map +0 -1
  170. package/dist/core/bootstrap.js.map +0 -1
  171. package/dist/core/config.js.map +0 -1
  172. package/dist/core/doc-progress.js.map +0 -1
  173. package/dist/core/fileLogger.js.map +0 -1
  174. package/dist/core/interfaces.js.map +0 -1
  175. package/dist/core/manifest.js.map +0 -1
  176. package/dist/core/provider-defaults.js.map +0 -1
  177. package/dist/core/rag-injection-flag.js.map +0 -1
  178. package/dist/core/resolve-api-key.js.map +0 -1
  179. package/dist/core/runtime-overrides.js.map +0 -1
  180. package/dist/describer/anthropic.js.map +0 -1
  181. package/dist/describer/describer.js.map +0 -1
  182. package/dist/describer/factory.js.map +0 -1
  183. package/dist/describer/gemini.js.map +0 -1
  184. package/dist/describer/shared.js.map +0 -1
  185. package/dist/embedder/cohere.js.map +0 -1
  186. package/dist/embedder/factory.js.map +0 -1
  187. package/dist/embedder/health.js.map +0 -1
  188. package/dist/embedder/http.js.map +0 -1
  189. package/dist/embedder/ollama.js.map +0 -1
  190. package/dist/embedder/openai.js.map +0 -1
  191. package/dist/eval/index.js.map +0 -1
  192. package/dist/eval/run-token-test.js.map +0 -1
  193. package/dist/eval/session-logger.js.map +0 -1
  194. package/dist/eval/storage.js.map +0 -1
  195. package/dist/eval/token-analysis.js.map +0 -1
  196. package/dist/eval/token-counter.js.map +0 -1
  197. package/dist/eval/types.js.map +0 -1
  198. package/dist/index.js.map +0 -1
  199. package/dist/indexer/description-stage.js.map +0 -1
  200. package/dist/indexer/embed-stage.js.map +0 -1
  201. package/dist/indexer/git-diff.js.map +0 -1
  202. package/dist/indexer/metadata.js.map +0 -1
  203. package/dist/indexer/pipeline.js.map +0 -1
  204. package/dist/indexer/stats.js.map +0 -1
  205. package/dist/indexer/watch.js.map +0 -1
  206. package/dist/indexer/worker.js.map +0 -1
  207. package/dist/indexer.js.map +0 -1
  208. package/dist/mcp/cli.js.map +0 -1
  209. package/dist/mcp/handlers.js.map +0 -1
  210. package/dist/mcp/server.js.map +0 -1
  211. package/dist/opencode/create-read-tool.js.map +0 -1
  212. package/dist/opencode/read-fallback.js.map +0 -1
  213. package/dist/opencode/read-format.js.map +0 -1
  214. package/dist/opencode/read-query.js.map +0 -1
  215. package/dist/opencode/tool-args.js.map +0 -1
  216. package/dist/opencode/tools.js.map +0 -1
  217. package/dist/plugin-entry.js.map +0 -1
  218. package/dist/plugin.js.map +0 -1
  219. package/dist/retriever/context-optimizer.js.map +0 -1
  220. package/dist/retriever/keyword-index.js.map +0 -1
  221. package/dist/retriever/retriever.js.map +0 -1
  222. package/dist/tui.js.map +0 -1
  223. package/dist/updater.d.ts +0 -45
  224. package/dist/updater.js +0 -175
  225. package/dist/updater.js.map +0 -1
  226. package/dist/vectorstore/factory.js.map +0 -1
  227. package/dist/vectorstore/lancedb.js.map +0 -1
  228. package/dist/vectorstore/memory.js.map +0 -1
  229. package/dist/watcher.js.map +0 -1
  230. package/dist/web/api.js.map +0 -1
  231. package/dist/web/server.js.map +0 -1
  232. package/dist/web/static.js.map +0 -1
package/ReadMe.md CHANGED
@@ -11,12 +11,8 @@ You don't need a dedicated GPU to run smaller embedding LLMs, as these models ca
11
11
  ## Quick Start
12
12
 
13
13
  ```bash
14
- # 1. Clone and install
15
- git clone https://github.com/MrDoe/OpenCodeRAG.git
16
- cd OpenCodeRAG
17
- npm install --legacy-peer-deps
18
- npm run build
19
- ./install.sh
14
+ # 1. Install globally (automatically sets up the OpenCode runtime)
15
+ npm install -g opencode-rag-plugin
20
16
 
21
17
  # 2. Initialize in your project
22
18
  cd /path/to/your/project
@@ -31,6 +27,8 @@ opencode-rag query "authentication middleware"
31
27
 
32
28
  **Prerequisites:** Node.js v22+, [Ollama](https://ollama.ai) (default) or other LLM-hosters (OpenAI-, Google- or Anthropic-compatible).
33
29
 
30
+ > **Contributors / developers:** Clone the repo and use `npm install --legacy-peer-deps; npm run build` — see [Development docs](doc/development.md).
31
+
34
32
  ## Key Features
35
33
 
36
34
  | Feature | Description |
package/dist/api.js CHANGED
@@ -7,6 +7,7 @@ import { retrieve } from "./retriever/retriever.js";
7
7
  import { optimizeContext, DEFAULT_CONTEXT_OPTIMIZATION } from "./retriever/context-optimizer.js";
8
8
  import { runIndexPass } from "./indexer.js";
9
9
  import { scanWorkspaceFiles } from "./content/reader.js";
10
+ import { destroyAllPooledConnections } from "./embedder/http.js";
10
11
  /**
11
12
  * Format a list of search results into a human-readable markdown block.
12
13
  * Each result is rendered as a code block with metadata header.
@@ -41,17 +42,29 @@ export async function search(query, options = {}) {
41
42
  cwd: options.cwd,
42
43
  configPath: options.configPath,
43
44
  });
44
- const topK = options.topK ?? ctx.config.retrieval.topK;
45
- const rawResults = await retrieve(query, ctx.embedder, ctx.store, {
46
- topK,
47
- minScore: options.minScore ?? ctx.config.retrieval.minScore,
48
- keywordIndex: ctx.keywordIndex,
49
- keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
50
- queryPrefix: ctx.config.embedding.queryPrefix,
51
- explain: options.explain,
52
- });
53
- const optCfg = ctx.config.retrieval.contextOptimization ?? DEFAULT_CONTEXT_OPTIMIZATION;
54
- return optimizeContext(rawResults, { topK, config: optCfg });
45
+ try {
46
+ const topK = options.topK ?? ctx.config.retrieval.topK;
47
+ const rawResults = await retrieve(query, ctx.embedder, ctx.store, {
48
+ topK,
49
+ minScore: options.minScore ?? ctx.config.retrieval.minScore,
50
+ keywordIndex: ctx.keywordIndex,
51
+ keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
52
+ hybridEnabled: ctx.config.retrieval.hybridSearch?.enabled,
53
+ queryPrefix: ctx.config.embedding.queryPrefix,
54
+ explain: options.explain,
55
+ filter: {
56
+ pathPatterns: options.pathHints,
57
+ languages: options.languageHints,
58
+ },
59
+ });
60
+ const optCfg = ctx.config.retrieval.contextOptimization ?? DEFAULT_CONTEXT_OPTIMIZATION;
61
+ return optimizeContext(rawResults, { topK, config: optCfg });
62
+ }
63
+ finally {
64
+ await ctx.store.close();
65
+ ctx.keywordIndex.close();
66
+ destroyAllPooledConnections();
67
+ }
55
68
  }
56
69
  /**
57
70
  * Index (or re-index) the workspace files into the vector store.
@@ -67,20 +80,27 @@ export async function indexWorkspace(cwd, options = {}) {
67
80
  cwd: workDir,
68
81
  configPath: options.configPath,
69
82
  });
70
- if (options.onProgress) {
71
- options.onProgress(`Indexing ${workDir}...`);
83
+ try {
84
+ if (options.onProgress) {
85
+ options.onProgress(`Indexing ${workDir}...`);
86
+ }
87
+ const stats = await runIndexPass({
88
+ cwd: workDir,
89
+ storePath: ctx.storePath,
90
+ config: ctx.config,
91
+ store: ctx.store,
92
+ embedder: ctx.embedder,
93
+ force: options.force ?? false,
94
+ keywordIndex: ctx.keywordIndex,
95
+ descriptionProvider: ctx.descriptionProvider,
96
+ });
97
+ return stats;
98
+ }
99
+ finally {
100
+ await ctx.store.close();
101
+ ctx.keywordIndex.close();
102
+ destroyAllPooledConnections();
72
103
  }
73
- const stats = await runIndexPass({
74
- cwd: workDir,
75
- storePath: ctx.storePath,
76
- config: ctx.config,
77
- store: ctx.store,
78
- embedder: ctx.embedder,
79
- force: options.force ?? false,
80
- keywordIndex: ctx.keywordIndex,
81
- descriptionProvider: ctx.descriptionProvider,
82
- });
83
- return stats;
84
104
  }
85
105
  /**
86
106
  * Retrieve and format context chunks for a query in one call.
@@ -5,10 +5,41 @@ export declare abstract class TreeSitterChunker implements Chunker {
5
5
  abstract readonly fileExtensions: string[];
6
6
  abstract readonly grammarName: string;
7
7
  abstract readonly nodeTypes: Set<string>;
8
+ /**
9
+ * Optional path to a custom WebAssembly file for the tree-sitter grammar.
10
+ * If not provided, the default grammar will be loaded.
11
+ * @default undefined
12
+ */
8
13
  readonly wasmFilePath?: string;
9
- private parser;
10
- private parserPromise;
14
+ /**
15
+ * Maximum content length in bytes before chunking is skipped.
16
+ * Tree-sitter can hang on very large files (especially SVGs).
17
+ * Set to 0 or negative for no limit.
18
+ * @default 0 (no limit)
19
+ */
20
+ maxContentBytes: number;
21
+ /**
22
+ * Create a new chunker that only includes nodes of the specified types.
23
+ * @param types - The set of node types to include in the chunking process.
24
+ * @returns A new Chunker instance that filters nodes by the specified types.
25
+ */
11
26
  withNodeTypes(types: Set<string>): Chunker;
12
- private getParser;
27
+ /**
28
+ * Returns a cached Tree-sitter parser for the chunker's language.
29
+ * Parsers are created once per grammar and reused across chunk() calls.
30
+ * @returns A Promise that resolves to a Parser instance.
31
+ */
32
+ private _createParser;
33
+ /**
34
+ * Release all cached parsers for this chunker's language.
35
+ * Call when the chunker is no longer needed.
36
+ */
37
+ dispose(): void;
38
+ /**
39
+ * Chunks the given content into an array of {@link Chunk} objects based on the specified node types.
40
+ * @param filePath - The path of the file being chunked, used for metadata.
41
+ * @param content - The content of the file to be chunked.
42
+ * @returns A Promise that resolves to an array of {@link Chunk} objects.
43
+ */
13
44
  chunk(filePath: string, content: string): Promise<Chunk[]>;
14
45
  }
@@ -4,11 +4,31 @@
4
4
  import { Parser } from "web-tree-sitter";
5
5
  import { loadLanguage, loadLanguageFromPath, walkTree } from "./grammar.js";
6
6
  import { uuid } from "./uuid.js";
7
+ /** Module-level parser pool shared across all TreeSitterChunker instances. */
8
+ const parserPool = new Map();
9
+ function poolKey(grammarName, wasmFilePath) {
10
+ return wasmFilePath ? `${grammarName}::${wasmFilePath}` : grammarName;
11
+ }
7
12
  /** Abstract base for tree-sitter based language chunkers. */
8
13
  export class TreeSitterChunker {
14
+ /**
15
+ * Optional path to a custom WebAssembly file for the tree-sitter grammar.
16
+ * If not provided, the default grammar will be loaded.
17
+ * @default undefined
18
+ */
9
19
  wasmFilePath;
10
- parser = null;
11
- parserPromise;
20
+ /**
21
+ * Maximum content length in bytes before chunking is skipped.
22
+ * Tree-sitter can hang on very large files (especially SVGs).
23
+ * Set to 0 or negative for no limit.
24
+ * @default 0 (no limit)
25
+ */
26
+ maxContentBytes = 0;
27
+ /**
28
+ * Create a new chunker that only includes nodes of the specified types.
29
+ * @param types - The set of node types to include in the chunking process.
30
+ * @returns A new Chunker instance that filters nodes by the specified types.
31
+ */
12
32
  withNodeTypes(types) {
13
33
  const original = this;
14
34
  return {
@@ -17,11 +37,16 @@ export class TreeSitterChunker {
17
37
  async chunk(filePath, content) {
18
38
  if (content.trim().length === 0)
19
39
  return [];
20
- const parser = await original.getParser();
40
+ if (original.maxContentBytes > 0 && Buffer.byteLength(content, "utf-8") > original.maxContentBytes) {
41
+ throw new Error(`File exceeds ${original.maxContentBytes} byte limit for ${original.language} chunker`);
42
+ }
43
+ const parser = await original._createParser();
21
44
  const tree = parser.parse(content);
22
- if (!tree)
45
+ if (!tree) {
23
46
  return [];
47
+ }
24
48
  const nodes = walkTree(tree.rootNode, types, content);
49
+ tree.delete();
25
50
  return nodes.map((node) => ({
26
51
  id: uuid(),
27
52
  content: node.text,
@@ -36,31 +61,55 @@ export class TreeSitterChunker {
36
61
  },
37
62
  };
38
63
  }
39
- getParser() {
40
- if (this.parser)
41
- return Promise.resolve(this.parser);
42
- if (!this.parserPromise) {
43
- this.parserPromise = (async () => {
44
- const lang = this.wasmFilePath
45
- ? await loadLanguageFromPath(this.grammarName, this.wasmFilePath)
46
- : await loadLanguage(this.grammarName);
47
- const parser = new Parser();
48
- parser.setLanguage(lang);
49
- this.parser = parser;
50
- this.parserPromise = undefined;
51
- return parser;
52
- })();
64
+ /**
65
+ * Returns a cached Tree-sitter parser for the chunker's language.
66
+ * Parsers are created once per grammar and reused across chunk() calls.
67
+ * @returns A Promise that resolves to a Parser instance.
68
+ */
69
+ async _createParser() {
70
+ const key = poolKey(this.grammarName, this.wasmFilePath);
71
+ const cached = parserPool.get(key);
72
+ if (cached)
73
+ return cached;
74
+ const lang = this.wasmFilePath
75
+ ? await loadLanguageFromPath(this.grammarName, this.wasmFilePath)
76
+ : await loadLanguage(this.grammarName);
77
+ const parser = new Parser();
78
+ parser.setLanguage(lang);
79
+ parserPool.set(key, parser);
80
+ return parser;
81
+ }
82
+ /**
83
+ * Release all cached parsers for this chunker's language.
84
+ * Call when the chunker is no longer needed.
85
+ */
86
+ dispose() {
87
+ const key = poolKey(this.grammarName, this.wasmFilePath);
88
+ const parser = parserPool.get(key);
89
+ if (parser) {
90
+ parser.delete();
91
+ parserPool.delete(key);
53
92
  }
54
- return this.parserPromise;
55
93
  }
94
+ /**
95
+ * Chunks the given content into an array of {@link Chunk} objects based on the specified node types.
96
+ * @param filePath - The path of the file being chunked, used for metadata.
97
+ * @param content - The content of the file to be chunked.
98
+ * @returns A Promise that resolves to an array of {@link Chunk} objects.
99
+ */
56
100
  async chunk(filePath, content) {
57
101
  if (content.trim().length === 0)
58
102
  return [];
59
- const parser = await this.getParser();
103
+ if (this.maxContentBytes > 0 && Buffer.byteLength(content, "utf-8") > this.maxContentBytes) {
104
+ throw new Error(`File exceeds ${this.maxContentBytes} byte limit for ${this.language} chunker`);
105
+ }
106
+ const parser = await this._createParser();
60
107
  const tree = parser.parse(content);
61
- if (!tree)
108
+ if (!tree) {
62
109
  return [];
110
+ }
63
111
  const nodes = walkTree(tree.rootNode, this.nodeTypes, content);
112
+ tree.delete();
64
113
  return nodes.map((node) => ({
65
114
  id: uuid(),
66
115
  content: node.text,
@@ -59,7 +59,10 @@ export declare function getRegisteredExtensions(): string[];
59
59
  * @param content - The full text content of the file.
60
60
  * @param nodeTypesOverrides - Optional language-specific node type overrides
61
61
  * for tree-sitter based chunkers.
62
+ * @param options - Optional chunking options (maxSvgSizeBytes, etc.).
62
63
  * @returns An array of chunks for the file.
63
64
  */
64
- export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>): Promise<Chunk[]>;
65
+ export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>, options?: {
66
+ maxSvgSizeBytes?: number;
67
+ }): Promise<Chunk[]>;
65
68
  export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
@@ -174,6 +174,15 @@ function splitOversized(chunks, filePath) {
174
174
  }
175
175
  return result;
176
176
  }
177
+ /** Apply config overrides to a chunker before it's used. */
178
+ function applyChunkerConfig(chunker, _filePath, options) {
179
+ if (options?.maxSvgSizeBytes && chunker instanceof TreeSitterChunker) {
180
+ const ext = _filePath.toLowerCase();
181
+ if (ext.endsWith(".svg") || ext.endsWith(".xml") || ext.endsWith(".csproj")) {
182
+ chunker.maxContentBytes = options.maxSvgSizeBytes;
183
+ }
184
+ }
185
+ }
177
186
  /**
178
187
  * Chunk a file by looking up its registered chunker, applying node-type
179
188
  * overrides if provided, and splitting oversized chunks to stay within
@@ -183,9 +192,10 @@ function splitOversized(chunks, filePath) {
183
192
  * @param content - The full text content of the file.
184
193
  * @param nodeTypesOverrides - Optional language-specific node type overrides
185
194
  * for tree-sitter based chunkers.
195
+ * @param options - Optional chunking options (maxSvgSizeBytes, etc.).
186
196
  * @returns An array of chunks for the file.
187
197
  */
188
- export async function chunkFile(filePath, content, nodeTypesOverrides) {
198
+ export async function chunkFile(filePath, content, nodeTypesOverrides, options) {
189
199
  let chunker = getChunker(filePath);
190
200
  if (nodeTypesOverrides && chunker instanceof TreeSitterChunker) {
191
201
  const overrideTypes = nodeTypesOverrides[chunker.language];
@@ -193,6 +203,7 @@ export async function chunkFile(filePath, content, nodeTypesOverrides) {
193
203
  chunker = chunker.withNodeTypes(new Set(overrideTypes));
194
204
  }
195
205
  }
206
+ applyChunkerConfig(chunker, filePath, options);
196
207
  const chunks = await chunker.chunk(filePath, content);
197
208
  if (chunks.length === 0) {
198
209
  return fallbackChunker.chunk(filePath, content);
@@ -63,6 +63,9 @@ export async function initParser() {
63
63
  initPromise = Parser.init().then(() => {
64
64
  initialized = true;
65
65
  initPromise = null;
66
+ }).catch((err) => {
67
+ initPromise = null;
68
+ throw err;
66
69
  });
67
70
  return initPromise;
68
71
  }
@@ -51,7 +51,7 @@ class OllamaImageVisionProvider {
51
51
  this.numCtx = config.numCtx;
52
52
  this.proxy = config.proxy;
53
53
  }
54
- async describeImage(imageBase64, _mimeType, prompt, _abort) {
54
+ async describeImage(imageBase64, _mimeType, prompt, abort) {
55
55
  const body = {
56
56
  model: this.model,
57
57
  messages: [
@@ -67,7 +67,7 @@ class OllamaImageVisionProvider {
67
67
  };
68
68
  let lastError;
69
69
  for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
70
- const response = await postJson(`${this.baseUrl}/chat`, body, {}, this.timeoutMs, this.proxy);
70
+ const response = await postJson(`${this.baseUrl}/chat`, body, {}, this.timeoutMs, this.proxy, abort);
71
71
  if (!response.ok) {
72
72
  const text = await response.text();
73
73
  const error = new Error(`Ollama vision request failed (${response.status}): ${text}`);
@@ -109,7 +109,7 @@ class OpenAIImageVisionProvider {
109
109
  this.timeoutMs = config.timeoutMs;
110
110
  this.proxy = config.proxy;
111
111
  }
112
- async describeImage(imageBase64, mimeType, prompt, _abort) {
112
+ async describeImage(imageBase64, mimeType, prompt, abort) {
113
113
  const url = `${this.baseUrl}${this.baseUrl.endsWith("/v1") ? "" : "/v1"}/chat/completions`;
114
114
  const body = {
115
115
  model: this.model,
@@ -135,7 +135,7 @@ class OpenAIImageVisionProvider {
135
135
  }
136
136
  let lastError;
137
137
  for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
138
- const response = await postJson(url, body, headers, this.timeoutMs, this.proxy);
138
+ const response = await postJson(url, body, headers, this.timeoutMs, this.proxy, abort);
139
139
  if (!response.ok) {
140
140
  const text = await response.text();
141
141
  const error = new Error(`OpenAI vision request failed (${response.status}): ${text}`);
@@ -177,7 +177,7 @@ class AnthropicImageVisionProvider {
177
177
  this.timeoutMs = config.timeoutMs;
178
178
  this.proxy = config.proxy;
179
179
  }
180
- async describeImage(imageBase64, mimeType, prompt, _abort) {
180
+ async describeImage(imageBase64, mimeType, prompt, abort) {
181
181
  const body = {
182
182
  model: this.model,
183
183
  max_tokens: 2048,
@@ -201,7 +201,7 @@ class AnthropicImageVisionProvider {
201
201
  };
202
202
  let lastError;
203
203
  for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
204
- const response = await postJson(`${this.baseUrl}/messages`, body, headers, this.timeoutMs, this.proxy);
204
+ const response = await postJson(`${this.baseUrl}/messages`, body, headers, this.timeoutMs, this.proxy, abort);
205
205
  if (!response.ok) {
206
206
  const text = await response.text();
207
207
  const error = new Error(`Anthropic vision request failed (${response.status}): ${text}`);
@@ -243,7 +243,7 @@ class GeminiImageVisionProvider {
243
243
  this.timeoutMs = config.timeoutMs;
244
244
  this.proxy = config.proxy;
245
245
  }
246
- async describeImage(imageBase64, mimeType, prompt, _abort) {
246
+ async describeImage(imageBase64, mimeType, prompt, abort) {
247
247
  const body = {
248
248
  contents: [
249
249
  {
@@ -268,7 +268,7 @@ class GeminiImageVisionProvider {
268
268
  : `${this.baseUrl}/models/${this.model}:generateContent`;
269
269
  let lastError;
270
270
  for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
271
- const response = await postJson(url, body, headers, this.timeoutMs, this.proxy);
271
+ const response = await postJson(url, body, headers, this.timeoutMs, this.proxy, abort);
272
272
  if (!response.ok) {
273
273
  const text = await response.text();
274
274
  const error = new Error(`Gemini vision request failed (${response.status}): ${text}`);
@@ -4,6 +4,8 @@ import { pathToFileURL } from "node:url";
4
4
  import path from "node:path";
5
5
  const MAX_CHUNK_CHARS = 4000;
6
6
  const MIN_GROUP_CHARS = 300;
7
+ const MAX_PDF_PAGES = 1000;
8
+ const MAX_PDF_SIZE_BYTES = 100 * 1024 * 1024; // 100 MB
7
9
  const PARAGRAPH_SPLIT = /\n\s*\n/;
8
10
  /**
9
11
  * Resolve the file URL to the `standard_fonts` directory shipped with pdfjs-dist.
@@ -18,22 +20,14 @@ function getStandardFontsUrl() {
18
20
  }
19
21
  /**
20
22
  * Create a pdfjs-dist PDF document from a buffer.
21
- * Attempts to load the `canvas` native module for DOMMatrix; falls back to
22
- * `@thednp/dommatrix` polyfill if unavailable.
23
+ * Uses `@thednp/dommatrix` for DOMMatrix polyfill (lighter alternative to native canvas).
23
24
  * @param buffer - Raw buffer of the PDF file.
24
25
  * @returns A promise resolving to a pdfjs-dist PDFDocumentProxy.
25
26
  */
26
27
  async function createPdfDocument(buffer) {
27
- try {
28
- const { DOMMatrix } = await import("canvas");
29
- globalThis.DOMMatrix ??= DOMMatrix;
30
- globalThis.DOMMatrixReadOnly ??= DOMMatrix;
31
- }
32
- catch {
33
- const { default: CSSMatrix } = await import("@thednp/dommatrix");
34
- globalThis.DOMMatrix ??= CSSMatrix;
35
- globalThis.DOMMatrixReadOnly ??= CSSMatrix;
36
- }
28
+ const { default: CSSMatrix } = await import("@thednp/dommatrix");
29
+ globalThis.DOMMatrix ??= CSSMatrix;
30
+ globalThis.DOMMatrixReadOnly ??= CSSMatrix;
37
31
  const { getDocument } = await import("pdfjs-dist/legacy/build/pdf.mjs");
38
32
  const loadingTask = getDocument({
39
33
  data: new Uint8Array(buffer),
@@ -50,9 +44,13 @@ async function createPdfDocument(buffer) {
50
44
  * @returns The extracted text with double-newline page separators.
51
45
  */
52
46
  export async function extractPdfText(buffer) {
47
+ if (buffer.length > MAX_PDF_SIZE_BYTES) {
48
+ throw new Error(`PDF too large: ${(buffer.length / 1024 / 1024).toFixed(1)} MB (max 100 MB)`);
49
+ }
53
50
  const pdf = await createPdfDocument(buffer);
54
51
  const texts = [];
55
- for (let i = 1; i <= pdf.numPages; i++) {
52
+ const pageCount = Math.min(pdf.numPages, MAX_PDF_PAGES);
53
+ for (let i = 1; i <= pageCount; i++) {
56
54
  const page = await pdf.getPage(i);
57
55
  const content = await page.getTextContent();
58
56
  const textItems = content.items.filter((item) => typeof item === "object" && item !== null && "str" in item);
@@ -11,6 +11,8 @@ export declare class XmlChunker extends TreeSitterChunker {
11
11
  readonly fileExtensions: string[];
12
12
  readonly grammarName = "xml";
13
13
  readonly nodeTypes: Set<string>;
14
+ /** Limit to prevent tree-sitter from hanging on large SVG/XML files. */
15
+ maxContentBytes: number;
14
16
  }
15
17
  /** Default singleton instance of {@link XmlChunker}. */
16
18
  export declare const xmlChunker: XmlChunker;
@@ -13,6 +13,8 @@ export class XmlChunker extends TreeSitterChunker {
13
13
  nodeTypes = new Set([
14
14
  "element",
15
15
  ]);
16
+ /** Limit to prevent tree-sitter from hanging on large SVG/XML files. */
17
+ maxContentBytes = 1_048_576;
16
18
  }
17
19
  /** Default singleton instance of {@link XmlChunker}. */
18
20
  export const xmlChunker = new XmlChunker();
@@ -14,7 +14,6 @@ import chokidar from "chokidar";
14
14
  import { appendDebugLog } from "../../core/fileLogger.js";
15
15
  import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "../../indexer.js";
16
16
  import { c, resolveCliContext, cleanupContext, logCliError, logCliInfo, logIndexSummary, formatDuration, } from "../format.js";
17
- import { TerminalProgressTable } from "../progress.js";
18
17
  /**
19
18
  * Build a logger that suppresses console output when watchTriggered is true.
20
19
  */
@@ -98,7 +97,6 @@ export function registerIndexCommand(program) {
98
97
  }
99
98
  }
100
99
  logCliInfo(logFilePath, "index", `${c.label("Scanning:")} ${c.file(cwd)}`);
101
- const progress = new TerminalProgressTable(process.stdout);
102
100
  const runPass = async (watchTriggered = false, abortSignal, filterPaths) => {
103
101
  const passStarted = Date.now();
104
102
  const stats = await runIndexPass({
@@ -109,14 +107,12 @@ export function registerIndexCommand(program) {
109
107
  embedder,
110
108
  keywordIndex,
111
109
  descriptionProvider,
112
- progress,
113
110
  force: !!(options.force && !watchTriggered),
114
111
  abortSignal,
115
112
  dimension,
116
113
  filterPaths,
117
114
  logger: watchAwareLogger(logFilePath, watchTriggered ? "watch" : "index", watchTriggered),
118
115
  });
119
- progress.done();
120
116
  if (!watchTriggered) {
121
117
  logIndexSummary(logFilePath, stats);
122
118
  logCliInfo(logFilePath, "index", `\n${c.success("Indexing complete.")} ${c.num(stats.finalCount)} chunks stored (${formatDuration(Date.now() - passStarted)}).`);
@@ -11,6 +11,6 @@ export { registerDumpCommand } from "./dump.js";
11
11
  export { registerInitCommand } from "./init.js";
12
12
  export { registerUiCommand } from "./ui.js";
13
13
  export { registerMcpCommand } from "./mcp.js";
14
- export { registerUpdateCommand } from "./update.js";
14
+ export { registerSetupCommand } from "./setup.js";
15
15
  export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand } from "./eval.js";
16
16
  export { registerDescribeImageCommand } from "./describe-image.js";
@@ -11,7 +11,7 @@ export { registerDumpCommand } from "./dump.js";
11
11
  export { registerInitCommand } from "./init.js";
12
12
  export { registerUiCommand } from "./ui.js";
13
13
  export { registerMcpCommand } from "./mcp.js";
14
- export { registerUpdateCommand } from "./update.js";
14
+ export { registerSetupCommand } from "./setup.js";
15
15
  export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand } from "./eval.js";
16
16
  export { registerDescribeImageCommand } from "./describe-image.js";
17
17
  //# sourceMappingURL=index.js.map
@@ -74,6 +74,18 @@ export declare function generateSkillFile(): string;
74
74
  * @returns The merged `.gitignore` content with a trailing newline.
75
75
  */
76
76
  export declare function mergeGitignoreContent(existingContent?: string): string;
77
+ /**
78
+ * Merge the OpenCodeRAG directive section into an existing `AGENTS.md`.
79
+ *
80
+ * The directive is wrapped in sentinel markers so re-runs replace the
81
+ * section in place rather than appending duplicates. Existing content
82
+ * outside the markers is always preserved. If no existing content is
83
+ * provided, the section alone is returned as a new file.
84
+ *
85
+ * @param existingContent - The current `AGENTS.md` content, or `undefined` if absent.
86
+ * @returns The merged `AGENTS.md` content with a trailing newline.
87
+ */
88
+ export declare function mergeAgentsMdContent(existingContent?: string): string;
77
89
  /**
78
90
  * Get the runtime directory path (`~/.opencode`).
79
91
  *