opencode-rag-plugin 1.15.0 → 1.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ReadMe.md +6 -7
- package/dist/api.js +44 -24
- package/dist/chunker/base.d.ts +34 -3
- package/dist/chunker/base.js +70 -21
- package/dist/chunker/factory.d.ts +4 -1
- package/dist/chunker/factory.js +12 -1
- package/dist/chunker/grammar.js +3 -0
- package/dist/chunker/image.js +8 -8
- package/dist/chunker/pdf.js +11 -13
- package/dist/chunker/xml.d.ts +2 -0
- package/dist/chunker/xml.js +2 -0
- package/dist/cli/commands/index-command.js +0 -4
- package/dist/cli/commands/index.d.ts +1 -1
- package/dist/cli/commands/index.js +1 -1
- package/dist/cli/commands/init-helpers.d.ts +12 -0
- package/dist/cli/commands/init-helpers.js +84 -2
- package/dist/cli/commands/init.js +20 -2
- package/dist/cli/commands/query.js +1 -0
- package/dist/cli/commands/setup.d.ts +2 -0
- package/dist/cli/commands/setup.js +113 -0
- package/dist/cli/commands/status.js +64 -2
- package/dist/cli/index.js +2 -2
- package/dist/content/image.js +24 -3
- package/dist/content/reader.d.ts +6 -1
- package/dist/content/reader.js +49 -4
- package/dist/core/config.d.ts +19 -0
- package/dist/core/config.js +34 -4
- package/dist/core/desc-cache.d.ts +31 -0
- package/dist/core/desc-cache.js +124 -0
- package/dist/core/interfaces.d.ts +58 -3
- package/dist/core/manifest.d.ts +24 -1
- package/dist/core/manifest.js +34 -3
- package/dist/core/resolve-api-key.js +4 -2
- package/dist/core/runtime-overrides.js +9 -7
- package/dist/core/setup-runtime.d.ts +23 -0
- package/dist/core/setup-runtime.js +183 -0
- package/dist/core/version-check.d.ts +9 -0
- package/dist/core/version-check.js +49 -0
- package/dist/describer/anthropic.d.ts +2 -2
- package/dist/describer/anthropic.js +5 -7
- package/dist/describer/describer.d.ts +2 -2
- package/dist/describer/describer.js +6 -8
- package/dist/describer/gemini.d.ts +2 -2
- package/dist/describer/gemini.js +5 -7
- package/dist/embedder/factory.d.ts +3 -1
- package/dist/embedder/factory.js +7 -1
- package/dist/embedder/health.js +4 -0
- package/dist/embedder/http.d.ts +1 -1
- package/dist/embedder/http.js +74 -43
- package/dist/eval/compare-merge.d.ts +10 -0
- package/dist/eval/compare-merge.js +537 -0
- package/dist/eval/compare-rankings.d.ts +10 -0
- package/dist/eval/compare-rankings.js +245 -0
- package/dist/eval/dump-descriptions.d.ts +7 -0
- package/dist/eval/dump-descriptions.js +58 -0
- package/dist/eval/fast-index.d.ts +8 -0
- package/dist/eval/fast-index.js +283 -0
- package/dist/eval/run-branch-compare.d.ts +7 -0
- package/dist/eval/run-branch-compare.js +220 -0
- package/dist/eval/run-token-test.js +1 -0
- package/dist/eval/test-kw.d.ts +1 -0
- package/dist/eval/test-kw.js +22 -0
- package/dist/eval/update-descriptions.d.ts +7 -0
- package/dist/eval/update-descriptions.js +84 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/indexer/embed-stage.js +2 -1
- package/dist/indexer/git-diff.js +21 -9
- package/dist/indexer/pipeline.d.ts +6 -0
- package/dist/indexer/pipeline.js +290 -37
- package/dist/indexer/watch.js +1 -3
- package/dist/indexer/worker.d.ts +15 -2
- package/dist/indexer/worker.js +25 -12
- package/dist/mcp/handlers.d.ts +9 -0
- package/dist/mcp/handlers.js +23 -6
- package/dist/mcp/server.js +2 -0
- package/dist/opencode/create-read-tool.d.ts +2 -0
- package/dist/opencode/create-read-tool.js +8 -2
- package/dist/opencode/read-fallback.d.ts +1 -5
- package/dist/opencode/read-fallback.js +1 -18
- package/dist/opencode/read-format.js +5 -3
- package/dist/opencode/tools.js +5 -7
- package/dist/plugin.js +196 -81
- package/dist/retriever/keyword-index.d.ts +3 -2
- package/dist/retriever/keyword-index.js +25 -1
- package/dist/retriever/retriever.d.ts +4 -1
- package/dist/retriever/retriever.js +34 -56
- package/dist/vectorstore/lancedb.d.ts +35 -4
- package/dist/vectorstore/lancedb.js +146 -23
- package/dist/vectorstore/memory.d.ts +6 -1
- package/dist/vectorstore/memory.js +58 -0
- package/dist/watcher.js +3 -0
- package/dist/web/api.js +10 -2
- package/dist/web/server.js +18 -3
- package/package.json +8 -9
- package/scripts/postinstall-setup.js +82 -0
- package/dist/api.js.map +0 -1
- package/dist/chunker/base.js.map +0 -1
- package/dist/chunker/bash.js.map +0 -1
- package/dist/chunker/c.js.map +0 -1
- package/dist/chunker/cpp.js.map +0 -1
- package/dist/chunker/csharp.js.map +0 -1
- package/dist/chunker/css.js.map +0 -1
- package/dist/chunker/doc.js.map +0 -1
- package/dist/chunker/dockerfile.js.map +0 -1
- package/dist/chunker/docx.js.map +0 -1
- package/dist/chunker/excel.js.map +0 -1
- package/dist/chunker/factory.js.map +0 -1
- package/dist/chunker/fallback.js.map +0 -1
- package/dist/chunker/go.js.map +0 -1
- package/dist/chunker/grammar.js.map +0 -1
- package/dist/chunker/html.js.map +0 -1
- package/dist/chunker/image.js.map +0 -1
- package/dist/chunker/ini.js.map +0 -1
- package/dist/chunker/java.js.map +0 -1
- package/dist/chunker/javascript.js.map +0 -1
- package/dist/chunker/json.js.map +0 -1
- package/dist/chunker/kotlin.js.map +0 -1
- package/dist/chunker/loader.js.map +0 -1
- package/dist/chunker/markdown.js.map +0 -1
- package/dist/chunker/pdf.js.map +0 -1
- package/dist/chunker/php.js.map +0 -1
- package/dist/chunker/powershell.js.map +0 -1
- package/dist/chunker/python.js.map +0 -1
- package/dist/chunker/razor.js.map +0 -1
- package/dist/chunker/ruby.js.map +0 -1
- package/dist/chunker/rust.js.map +0 -1
- package/dist/chunker/sln.js.map +0 -1
- package/dist/chunker/sql.js.map +0 -1
- package/dist/chunker/ssl.js.map +0 -1
- package/dist/chunker/swift.js.map +0 -1
- package/dist/chunker/tex.js.map +0 -1
- package/dist/chunker/toml.js.map +0 -1
- package/dist/chunker/typescript.js.map +0 -1
- package/dist/chunker/uuid.js.map +0 -1
- package/dist/chunker/xml.js.map +0 -1
- package/dist/chunker/yaml.js.map +0 -1
- package/dist/cli/commands/clear.js.map +0 -1
- package/dist/cli/commands/describe-image.js.map +0 -1
- package/dist/cli/commands/dump.js.map +0 -1
- package/dist/cli/commands/eval.js.map +0 -1
- package/dist/cli/commands/index-command.js.map +0 -1
- package/dist/cli/commands/index.js.map +0 -1
- package/dist/cli/commands/init-helpers.js.map +0 -1
- package/dist/cli/commands/init.js.map +0 -1
- package/dist/cli/commands/list.js.map +0 -1
- package/dist/cli/commands/mcp.js.map +0 -1
- package/dist/cli/commands/query.js.map +0 -1
- package/dist/cli/commands/show.js.map +0 -1
- package/dist/cli/commands/status.js.map +0 -1
- package/dist/cli/commands/ui.js.map +0 -1
- package/dist/cli/commands/update.d.ts +0 -17
- package/dist/cli/commands/update.js +0 -79
- package/dist/cli/commands/update.js.map +0 -1
- package/dist/cli/format.js.map +0 -1
- package/dist/cli/helpers.js.map +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/progress.d.ts +0 -42
- package/dist/cli/progress.js +0 -137
- package/dist/cli/progress.js.map +0 -1
- package/dist/cli/types.js.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/content/doc.js.map +0 -1
- package/dist/content/docx.js.map +0 -1
- package/dist/content/excel.js.map +0 -1
- package/dist/content/image.js.map +0 -1
- package/dist/content/pdf.js.map +0 -1
- package/dist/content/reader.js.map +0 -1
- package/dist/content/types.js.map +0 -1
- package/dist/core/bootstrap.js.map +0 -1
- package/dist/core/config.js.map +0 -1
- package/dist/core/doc-progress.js.map +0 -1
- package/dist/core/fileLogger.js.map +0 -1
- package/dist/core/interfaces.js.map +0 -1
- package/dist/core/manifest.js.map +0 -1
- package/dist/core/provider-defaults.js.map +0 -1
- package/dist/core/rag-injection-flag.js.map +0 -1
- package/dist/core/resolve-api-key.js.map +0 -1
- package/dist/core/runtime-overrides.js.map +0 -1
- package/dist/describer/anthropic.js.map +0 -1
- package/dist/describer/describer.js.map +0 -1
- package/dist/describer/factory.js.map +0 -1
- package/dist/describer/gemini.js.map +0 -1
- package/dist/describer/shared.js.map +0 -1
- package/dist/embedder/cohere.js.map +0 -1
- package/dist/embedder/factory.js.map +0 -1
- package/dist/embedder/health.js.map +0 -1
- package/dist/embedder/http.js.map +0 -1
- package/dist/embedder/ollama.js.map +0 -1
- package/dist/embedder/openai.js.map +0 -1
- package/dist/eval/index.js.map +0 -1
- package/dist/eval/run-token-test.js.map +0 -1
- package/dist/eval/session-logger.js.map +0 -1
- package/dist/eval/storage.js.map +0 -1
- package/dist/eval/token-analysis.js.map +0 -1
- package/dist/eval/token-counter.js.map +0 -1
- package/dist/eval/types.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/indexer/description-stage.js.map +0 -1
- package/dist/indexer/embed-stage.js.map +0 -1
- package/dist/indexer/git-diff.js.map +0 -1
- package/dist/indexer/metadata.js.map +0 -1
- package/dist/indexer/pipeline.js.map +0 -1
- package/dist/indexer/stats.js.map +0 -1
- package/dist/indexer/watch.js.map +0 -1
- package/dist/indexer/worker.js.map +0 -1
- package/dist/indexer.js.map +0 -1
- package/dist/mcp/cli.js.map +0 -1
- package/dist/mcp/handlers.js.map +0 -1
- package/dist/mcp/server.js.map +0 -1
- package/dist/opencode/create-read-tool.js.map +0 -1
- package/dist/opencode/read-fallback.js.map +0 -1
- package/dist/opencode/read-format.js.map +0 -1
- package/dist/opencode/read-query.js.map +0 -1
- package/dist/opencode/tool-args.js.map +0 -1
- package/dist/opencode/tools.js.map +0 -1
- package/dist/plugin-entry.js.map +0 -1
- package/dist/plugin.js.map +0 -1
- package/dist/retriever/context-optimizer.js.map +0 -1
- package/dist/retriever/keyword-index.js.map +0 -1
- package/dist/retriever/retriever.js.map +0 -1
- package/dist/tui.js.map +0 -1
- package/dist/updater.d.ts +0 -45
- package/dist/updater.js +0 -175
- package/dist/updater.js.map +0 -1
- package/dist/vectorstore/factory.js.map +0 -1
- package/dist/vectorstore/lancedb.js.map +0 -1
- package/dist/vectorstore/memory.js.map +0 -1
- package/dist/watcher.js.map +0 -1
- package/dist/web/api.js.map +0 -1
- package/dist/web/server.js.map +0 -1
- package/dist/web/static.js.map +0 -1
package/ReadMe.md
CHANGED
|
@@ -11,12 +11,8 @@ You don't need a dedicated GPU to run smaller embedding LLMs, as these models ca
|
|
|
11
11
|
## Quick Start
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
# 1.
|
|
15
|
-
|
|
16
|
-
cd OpenCodeRAG
|
|
17
|
-
npm install --legacy-peer-deps
|
|
18
|
-
npm run build
|
|
19
|
-
./install.sh
|
|
14
|
+
# 1. Install globally (automatically sets up the OpenCode runtime)
|
|
15
|
+
npm install -g opencode-rag-plugin
|
|
20
16
|
|
|
21
17
|
# 2. Initialize in your project
|
|
22
18
|
cd /path/to/your/project
|
|
@@ -31,6 +27,8 @@ opencode-rag query "authentication middleware"
|
|
|
31
27
|
|
|
32
28
|
**Prerequisites:** Node.js v22+, [Ollama](https://ollama.ai) (default) or other LLM-hosters (OpenAI-, Google- or Anthropic-compatible).
|
|
33
29
|
|
|
30
|
+
> **Contributors / developers:** Clone the repo and use `npm install --legacy-peer-deps; npm run build` — see [Development docs](doc/development.md).
|
|
31
|
+
|
|
34
32
|
## Key Features
|
|
35
33
|
|
|
36
34
|
| Feature | Description |
|
|
@@ -79,11 +77,12 @@ Launch with `opencode-rag ui`. See [Web UI documentation](doc/webui.md) for deta
|
|
|
79
77
|
OpenCodeRAG can index image files (PNG, JPEG, WebP, etc.) by sending them to a vision-capable LLM and storing the generated text descriptions as searchable vector chunks. This makes visual assets discoverable via natural language queries (e.g., "login screen screenshot", "architecture diagram").
|
|
80
78
|
|
|
81
79
|
**Supported providers:** Ollama, OpenAI, Anthropic, Google Gemini compatible providers.
|
|
80
|
+
|
|
82
81
|
**Disabled by default** — enable in `opencode-rag.json` to opt in (recommended for dedicated GPUs).
|
|
83
82
|
|
|
84
83
|
## MCP Server
|
|
85
84
|
|
|
86
|
-
OpenCodeRAG ships a
|
|
85
|
+
OpenCodeRAG ships a CLI-based [MCP (Model Context Protocol)](https://spec.modelcontextprotocol.io/) server that exposes semantic code tools to any MCP-compatible client (Claude Desktop, OpenCode, Cursor, etc.).
|
|
87
86
|
|
|
88
87
|
```bash
|
|
89
88
|
opencode-rag mcp
|
package/dist/api.js
CHANGED
|
@@ -7,6 +7,7 @@ import { retrieve } from "./retriever/retriever.js";
|
|
|
7
7
|
import { optimizeContext, DEFAULT_CONTEXT_OPTIMIZATION } from "./retriever/context-optimizer.js";
|
|
8
8
|
import { runIndexPass } from "./indexer.js";
|
|
9
9
|
import { scanWorkspaceFiles } from "./content/reader.js";
|
|
10
|
+
import { destroyAllPooledConnections } from "./embedder/http.js";
|
|
10
11
|
/**
|
|
11
12
|
* Format a list of search results into a human-readable markdown block.
|
|
12
13
|
* Each result is rendered as a code block with metadata header.
|
|
@@ -41,17 +42,29 @@ export async function search(query, options = {}) {
|
|
|
41
42
|
cwd: options.cwd,
|
|
42
43
|
configPath: options.configPath,
|
|
43
44
|
});
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
45
|
+
try {
|
|
46
|
+
const topK = options.topK ?? ctx.config.retrieval.topK;
|
|
47
|
+
const rawResults = await retrieve(query, ctx.embedder, ctx.store, {
|
|
48
|
+
topK,
|
|
49
|
+
minScore: options.minScore ?? ctx.config.retrieval.minScore,
|
|
50
|
+
keywordIndex: ctx.keywordIndex,
|
|
51
|
+
keywordWeight: options.keywordWeight ?? ctx.config.retrieval.hybridSearch?.keywordWeight ?? 0.4,
|
|
52
|
+
hybridEnabled: ctx.config.retrieval.hybridSearch?.enabled,
|
|
53
|
+
queryPrefix: ctx.config.embedding.queryPrefix,
|
|
54
|
+
explain: options.explain,
|
|
55
|
+
filter: {
|
|
56
|
+
pathPatterns: options.pathHints,
|
|
57
|
+
languages: options.languageHints,
|
|
58
|
+
},
|
|
59
|
+
});
|
|
60
|
+
const optCfg = ctx.config.retrieval.contextOptimization ?? DEFAULT_CONTEXT_OPTIMIZATION;
|
|
61
|
+
return optimizeContext(rawResults, { topK, config: optCfg });
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
await ctx.store.close();
|
|
65
|
+
ctx.keywordIndex.close();
|
|
66
|
+
destroyAllPooledConnections();
|
|
67
|
+
}
|
|
55
68
|
}
|
|
56
69
|
/**
|
|
57
70
|
* Index (or re-index) the workspace files into the vector store.
|
|
@@ -67,20 +80,27 @@ export async function indexWorkspace(cwd, options = {}) {
|
|
|
67
80
|
cwd: workDir,
|
|
68
81
|
configPath: options.configPath,
|
|
69
82
|
});
|
|
70
|
-
|
|
71
|
-
options.onProgress
|
|
83
|
+
try {
|
|
84
|
+
if (options.onProgress) {
|
|
85
|
+
options.onProgress(`Indexing ${workDir}...`);
|
|
86
|
+
}
|
|
87
|
+
const stats = await runIndexPass({
|
|
88
|
+
cwd: workDir,
|
|
89
|
+
storePath: ctx.storePath,
|
|
90
|
+
config: ctx.config,
|
|
91
|
+
store: ctx.store,
|
|
92
|
+
embedder: ctx.embedder,
|
|
93
|
+
force: options.force ?? false,
|
|
94
|
+
keywordIndex: ctx.keywordIndex,
|
|
95
|
+
descriptionProvider: ctx.descriptionProvider,
|
|
96
|
+
});
|
|
97
|
+
return stats;
|
|
98
|
+
}
|
|
99
|
+
finally {
|
|
100
|
+
await ctx.store.close();
|
|
101
|
+
ctx.keywordIndex.close();
|
|
102
|
+
destroyAllPooledConnections();
|
|
72
103
|
}
|
|
73
|
-
const stats = await runIndexPass({
|
|
74
|
-
cwd: workDir,
|
|
75
|
-
storePath: ctx.storePath,
|
|
76
|
-
config: ctx.config,
|
|
77
|
-
store: ctx.store,
|
|
78
|
-
embedder: ctx.embedder,
|
|
79
|
-
force: options.force ?? false,
|
|
80
|
-
keywordIndex: ctx.keywordIndex,
|
|
81
|
-
descriptionProvider: ctx.descriptionProvider,
|
|
82
|
-
});
|
|
83
|
-
return stats;
|
|
84
104
|
}
|
|
85
105
|
/**
|
|
86
106
|
* Retrieve and format context chunks for a query in one call.
|
package/dist/chunker/base.d.ts
CHANGED
|
@@ -5,10 +5,41 @@ export declare abstract class TreeSitterChunker implements Chunker {
|
|
|
5
5
|
abstract readonly fileExtensions: string[];
|
|
6
6
|
abstract readonly grammarName: string;
|
|
7
7
|
abstract readonly nodeTypes: Set<string>;
|
|
8
|
+
/**
|
|
9
|
+
* Optional path to a custom WebAssembly file for the tree-sitter grammar.
|
|
10
|
+
* If not provided, the default grammar will be loaded.
|
|
11
|
+
* @default undefined
|
|
12
|
+
*/
|
|
8
13
|
readonly wasmFilePath?: string;
|
|
9
|
-
|
|
10
|
-
|
|
14
|
+
/**
|
|
15
|
+
* Maximum content length in bytes before chunking is skipped.
|
|
16
|
+
* Tree-sitter can hang on very large files (especially SVGs).
|
|
17
|
+
* Set to 0 or negative for no limit.
|
|
18
|
+
* @default 0 (no limit)
|
|
19
|
+
*/
|
|
20
|
+
maxContentBytes: number;
|
|
21
|
+
/**
|
|
22
|
+
* Create a new chunker that only includes nodes of the specified types.
|
|
23
|
+
* @param types - The set of node types to include in the chunking process.
|
|
24
|
+
* @returns A new Chunker instance that filters nodes by the specified types.
|
|
25
|
+
*/
|
|
11
26
|
withNodeTypes(types: Set<string>): Chunker;
|
|
12
|
-
|
|
27
|
+
/**
|
|
28
|
+
* Returns a cached Tree-sitter parser for the chunker's language.
|
|
29
|
+
* Parsers are created once per grammar and reused across chunk() calls.
|
|
30
|
+
* @returns A Promise that resolves to a Parser instance.
|
|
31
|
+
*/
|
|
32
|
+
private _createParser;
|
|
33
|
+
/**
|
|
34
|
+
* Release all cached parsers for this chunker's language.
|
|
35
|
+
* Call when the chunker is no longer needed.
|
|
36
|
+
*/
|
|
37
|
+
dispose(): void;
|
|
38
|
+
/**
|
|
39
|
+
* Chunks the given content into an array of {@link Chunk} objects based on the specified node types.
|
|
40
|
+
* @param filePath - The path of the file being chunked, used for metadata.
|
|
41
|
+
* @param content - The content of the file to be chunked.
|
|
42
|
+
* @returns A Promise that resolves to an array of {@link Chunk} objects.
|
|
43
|
+
*/
|
|
13
44
|
chunk(filePath: string, content: string): Promise<Chunk[]>;
|
|
14
45
|
}
|
package/dist/chunker/base.js
CHANGED
|
@@ -4,11 +4,31 @@
|
|
|
4
4
|
import { Parser } from "web-tree-sitter";
|
|
5
5
|
import { loadLanguage, loadLanguageFromPath, walkTree } from "./grammar.js";
|
|
6
6
|
import { uuid } from "./uuid.js";
|
|
7
|
+
/** Module-level parser pool shared across all TreeSitterChunker instances. */
|
|
8
|
+
const parserPool = new Map();
|
|
9
|
+
function poolKey(grammarName, wasmFilePath) {
|
|
10
|
+
return wasmFilePath ? `${grammarName}::${wasmFilePath}` : grammarName;
|
|
11
|
+
}
|
|
7
12
|
/** Abstract base for tree-sitter based language chunkers. */
|
|
8
13
|
export class TreeSitterChunker {
|
|
14
|
+
/**
|
|
15
|
+
* Optional path to a custom WebAssembly file for the tree-sitter grammar.
|
|
16
|
+
* If not provided, the default grammar will be loaded.
|
|
17
|
+
* @default undefined
|
|
18
|
+
*/
|
|
9
19
|
wasmFilePath;
|
|
10
|
-
|
|
11
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Maximum content length in bytes before chunking is skipped.
|
|
22
|
+
* Tree-sitter can hang on very large files (especially SVGs).
|
|
23
|
+
* Set to 0 or negative for no limit.
|
|
24
|
+
* @default 0 (no limit)
|
|
25
|
+
*/
|
|
26
|
+
maxContentBytes = 0;
|
|
27
|
+
/**
|
|
28
|
+
* Create a new chunker that only includes nodes of the specified types.
|
|
29
|
+
* @param types - The set of node types to include in the chunking process.
|
|
30
|
+
* @returns A new Chunker instance that filters nodes by the specified types.
|
|
31
|
+
*/
|
|
12
32
|
withNodeTypes(types) {
|
|
13
33
|
const original = this;
|
|
14
34
|
return {
|
|
@@ -17,11 +37,16 @@ export class TreeSitterChunker {
|
|
|
17
37
|
async chunk(filePath, content) {
|
|
18
38
|
if (content.trim().length === 0)
|
|
19
39
|
return [];
|
|
20
|
-
|
|
40
|
+
if (original.maxContentBytes > 0 && Buffer.byteLength(content, "utf-8") > original.maxContentBytes) {
|
|
41
|
+
throw new Error(`File exceeds ${original.maxContentBytes} byte limit for ${original.language} chunker`);
|
|
42
|
+
}
|
|
43
|
+
const parser = await original._createParser();
|
|
21
44
|
const tree = parser.parse(content);
|
|
22
|
-
if (!tree)
|
|
45
|
+
if (!tree) {
|
|
23
46
|
return [];
|
|
47
|
+
}
|
|
24
48
|
const nodes = walkTree(tree.rootNode, types, content);
|
|
49
|
+
tree.delete();
|
|
25
50
|
return nodes.map((node) => ({
|
|
26
51
|
id: uuid(),
|
|
27
52
|
content: node.text,
|
|
@@ -36,31 +61,55 @@ export class TreeSitterChunker {
|
|
|
36
61
|
},
|
|
37
62
|
};
|
|
38
63
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Returns a cached Tree-sitter parser for the chunker's language.
|
|
66
|
+
* Parsers are created once per grammar and reused across chunk() calls.
|
|
67
|
+
* @returns A Promise that resolves to a Parser instance.
|
|
68
|
+
*/
|
|
69
|
+
async _createParser() {
|
|
70
|
+
const key = poolKey(this.grammarName, this.wasmFilePath);
|
|
71
|
+
const cached = parserPool.get(key);
|
|
72
|
+
if (cached)
|
|
73
|
+
return cached;
|
|
74
|
+
const lang = this.wasmFilePath
|
|
75
|
+
? await loadLanguageFromPath(this.grammarName, this.wasmFilePath)
|
|
76
|
+
: await loadLanguage(this.grammarName);
|
|
77
|
+
const parser = new Parser();
|
|
78
|
+
parser.setLanguage(lang);
|
|
79
|
+
parserPool.set(key, parser);
|
|
80
|
+
return parser;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Release all cached parsers for this chunker's language.
|
|
84
|
+
* Call when the chunker is no longer needed.
|
|
85
|
+
*/
|
|
86
|
+
dispose() {
|
|
87
|
+
const key = poolKey(this.grammarName, this.wasmFilePath);
|
|
88
|
+
const parser = parserPool.get(key);
|
|
89
|
+
if (parser) {
|
|
90
|
+
parser.delete();
|
|
91
|
+
parserPool.delete(key);
|
|
53
92
|
}
|
|
54
|
-
return this.parserPromise;
|
|
55
93
|
}
|
|
94
|
+
/**
|
|
95
|
+
* Chunks the given content into an array of {@link Chunk} objects based on the specified node types.
|
|
96
|
+
* @param filePath - The path of the file being chunked, used for metadata.
|
|
97
|
+
* @param content - The content of the file to be chunked.
|
|
98
|
+
* @returns A Promise that resolves to an array of {@link Chunk} objects.
|
|
99
|
+
*/
|
|
56
100
|
async chunk(filePath, content) {
|
|
57
101
|
if (content.trim().length === 0)
|
|
58
102
|
return [];
|
|
59
|
-
|
|
103
|
+
if (this.maxContentBytes > 0 && Buffer.byteLength(content, "utf-8") > this.maxContentBytes) {
|
|
104
|
+
throw new Error(`File exceeds ${this.maxContentBytes} byte limit for ${this.language} chunker`);
|
|
105
|
+
}
|
|
106
|
+
const parser = await this._createParser();
|
|
60
107
|
const tree = parser.parse(content);
|
|
61
|
-
if (!tree)
|
|
108
|
+
if (!tree) {
|
|
62
109
|
return [];
|
|
110
|
+
}
|
|
63
111
|
const nodes = walkTree(tree.rootNode, this.nodeTypes, content);
|
|
112
|
+
tree.delete();
|
|
64
113
|
return nodes.map((node) => ({
|
|
65
114
|
id: uuid(),
|
|
66
115
|
content: node.text,
|
|
@@ -59,7 +59,10 @@ export declare function getRegisteredExtensions(): string[];
|
|
|
59
59
|
* @param content - The full text content of the file.
|
|
60
60
|
* @param nodeTypesOverrides - Optional language-specific node type overrides
|
|
61
61
|
* for tree-sitter based chunkers.
|
|
62
|
+
* @param options - Optional chunking options (maxSvgSizeBytes, etc.).
|
|
62
63
|
* @returns An array of chunks for the file.
|
|
63
64
|
*/
|
|
64
|
-
export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]
|
|
65
|
+
export declare function chunkFile(filePath: string, content: string, nodeTypesOverrides?: Record<string, string[]>, options?: {
|
|
66
|
+
maxSvgSizeBytes?: number;
|
|
67
|
+
}): Promise<Chunk[]>;
|
|
65
68
|
export { typescriptChunker, pythonChunker, javaChunker, goChunker, markdownChunker, cChunker, cppChunker, csharpChunker, javascriptChunker, razorChunker, jsonChunker, htmlChunker, cssChunker, xmlChunker, slnChunker, rustChunker, rubyChunker, kotlinChunker, swiftChunker, bashChunker, phpChunker, powershellChunker, iniChunker, yamlChunker, tomlChunker, dockerfileChunker, sqlChunker, texChunker, pdfChunker, fallbackChunker };
|
package/dist/chunker/factory.js
CHANGED
|
@@ -174,6 +174,15 @@ function splitOversized(chunks, filePath) {
|
|
|
174
174
|
}
|
|
175
175
|
return result;
|
|
176
176
|
}
|
|
177
|
+
/** Apply config overrides to a chunker before it's used. */
|
|
178
|
+
function applyChunkerConfig(chunker, _filePath, options) {
|
|
179
|
+
if (options?.maxSvgSizeBytes && chunker instanceof TreeSitterChunker) {
|
|
180
|
+
const ext = _filePath.toLowerCase();
|
|
181
|
+
if (ext.endsWith(".svg") || ext.endsWith(".xml") || ext.endsWith(".csproj")) {
|
|
182
|
+
chunker.maxContentBytes = options.maxSvgSizeBytes;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
177
186
|
/**
|
|
178
187
|
* Chunk a file by looking up its registered chunker, applying node-type
|
|
179
188
|
* overrides if provided, and splitting oversized chunks to stay within
|
|
@@ -183,9 +192,10 @@ function splitOversized(chunks, filePath) {
|
|
|
183
192
|
* @param content - The full text content of the file.
|
|
184
193
|
* @param nodeTypesOverrides - Optional language-specific node type overrides
|
|
185
194
|
* for tree-sitter based chunkers.
|
|
195
|
+
* @param options - Optional chunking options (maxSvgSizeBytes, etc.).
|
|
186
196
|
* @returns An array of chunks for the file.
|
|
187
197
|
*/
|
|
188
|
-
export async function chunkFile(filePath, content, nodeTypesOverrides) {
|
|
198
|
+
export async function chunkFile(filePath, content, nodeTypesOverrides, options) {
|
|
189
199
|
let chunker = getChunker(filePath);
|
|
190
200
|
if (nodeTypesOverrides && chunker instanceof TreeSitterChunker) {
|
|
191
201
|
const overrideTypes = nodeTypesOverrides[chunker.language];
|
|
@@ -193,6 +203,7 @@ export async function chunkFile(filePath, content, nodeTypesOverrides) {
|
|
|
193
203
|
chunker = chunker.withNodeTypes(new Set(overrideTypes));
|
|
194
204
|
}
|
|
195
205
|
}
|
|
206
|
+
applyChunkerConfig(chunker, filePath, options);
|
|
196
207
|
const chunks = await chunker.chunk(filePath, content);
|
|
197
208
|
if (chunks.length === 0) {
|
|
198
209
|
return fallbackChunker.chunk(filePath, content);
|
package/dist/chunker/grammar.js
CHANGED
package/dist/chunker/image.js
CHANGED
|
@@ -51,7 +51,7 @@ class OllamaImageVisionProvider {
|
|
|
51
51
|
this.numCtx = config.numCtx;
|
|
52
52
|
this.proxy = config.proxy;
|
|
53
53
|
}
|
|
54
|
-
async describeImage(imageBase64, _mimeType, prompt,
|
|
54
|
+
async describeImage(imageBase64, _mimeType, prompt, abort) {
|
|
55
55
|
const body = {
|
|
56
56
|
model: this.model,
|
|
57
57
|
messages: [
|
|
@@ -67,7 +67,7 @@ class OllamaImageVisionProvider {
|
|
|
67
67
|
};
|
|
68
68
|
let lastError;
|
|
69
69
|
for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
|
|
70
|
-
const response = await postJson(`${this.baseUrl}/chat`, body, {}, this.timeoutMs, this.proxy);
|
|
70
|
+
const response = await postJson(`${this.baseUrl}/chat`, body, {}, this.timeoutMs, this.proxy, abort);
|
|
71
71
|
if (!response.ok) {
|
|
72
72
|
const text = await response.text();
|
|
73
73
|
const error = new Error(`Ollama vision request failed (${response.status}): ${text}`);
|
|
@@ -109,7 +109,7 @@ class OpenAIImageVisionProvider {
|
|
|
109
109
|
this.timeoutMs = config.timeoutMs;
|
|
110
110
|
this.proxy = config.proxy;
|
|
111
111
|
}
|
|
112
|
-
async describeImage(imageBase64, mimeType, prompt,
|
|
112
|
+
async describeImage(imageBase64, mimeType, prompt, abort) {
|
|
113
113
|
const url = `${this.baseUrl}${this.baseUrl.endsWith("/v1") ? "" : "/v1"}/chat/completions`;
|
|
114
114
|
const body = {
|
|
115
115
|
model: this.model,
|
|
@@ -135,7 +135,7 @@ class OpenAIImageVisionProvider {
|
|
|
135
135
|
}
|
|
136
136
|
let lastError;
|
|
137
137
|
for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
|
|
138
|
-
const response = await postJson(url, body, headers, this.timeoutMs, this.proxy);
|
|
138
|
+
const response = await postJson(url, body, headers, this.timeoutMs, this.proxy, abort);
|
|
139
139
|
if (!response.ok) {
|
|
140
140
|
const text = await response.text();
|
|
141
141
|
const error = new Error(`OpenAI vision request failed (${response.status}): ${text}`);
|
|
@@ -177,7 +177,7 @@ class AnthropicImageVisionProvider {
|
|
|
177
177
|
this.timeoutMs = config.timeoutMs;
|
|
178
178
|
this.proxy = config.proxy;
|
|
179
179
|
}
|
|
180
|
-
async describeImage(imageBase64, mimeType, prompt,
|
|
180
|
+
async describeImage(imageBase64, mimeType, prompt, abort) {
|
|
181
181
|
const body = {
|
|
182
182
|
model: this.model,
|
|
183
183
|
max_tokens: 2048,
|
|
@@ -201,7 +201,7 @@ class AnthropicImageVisionProvider {
|
|
|
201
201
|
};
|
|
202
202
|
let lastError;
|
|
203
203
|
for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
|
|
204
|
-
const response = await postJson(`${this.baseUrl}/messages`, body, headers, this.timeoutMs, this.proxy);
|
|
204
|
+
const response = await postJson(`${this.baseUrl}/messages`, body, headers, this.timeoutMs, this.proxy, abort);
|
|
205
205
|
if (!response.ok) {
|
|
206
206
|
const text = await response.text();
|
|
207
207
|
const error = new Error(`Anthropic vision request failed (${response.status}): ${text}`);
|
|
@@ -243,7 +243,7 @@ class GeminiImageVisionProvider {
|
|
|
243
243
|
this.timeoutMs = config.timeoutMs;
|
|
244
244
|
this.proxy = config.proxy;
|
|
245
245
|
}
|
|
246
|
-
async describeImage(imageBase64, mimeType, prompt,
|
|
246
|
+
async describeImage(imageBase64, mimeType, prompt, abort) {
|
|
247
247
|
const body = {
|
|
248
248
|
contents: [
|
|
249
249
|
{
|
|
@@ -268,7 +268,7 @@ class GeminiImageVisionProvider {
|
|
|
268
268
|
: `${this.baseUrl}/models/${this.model}:generateContent`;
|
|
269
269
|
let lastError;
|
|
270
270
|
for (let attempt = 0; attempt <= VISION_RETRY_MAX; attempt++) {
|
|
271
|
-
const response = await postJson(url, body, headers, this.timeoutMs, this.proxy);
|
|
271
|
+
const response = await postJson(url, body, headers, this.timeoutMs, this.proxy, abort);
|
|
272
272
|
if (!response.ok) {
|
|
273
273
|
const text = await response.text();
|
|
274
274
|
const error = new Error(`Gemini vision request failed (${response.status}): ${text}`);
|
package/dist/chunker/pdf.js
CHANGED
|
@@ -4,6 +4,8 @@ import { pathToFileURL } from "node:url";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
const MAX_CHUNK_CHARS = 4000;
|
|
6
6
|
const MIN_GROUP_CHARS = 300;
|
|
7
|
+
const MAX_PDF_PAGES = 1000;
|
|
8
|
+
const MAX_PDF_SIZE_BYTES = 100 * 1024 * 1024; // 100 MB
|
|
7
9
|
const PARAGRAPH_SPLIT = /\n\s*\n/;
|
|
8
10
|
/**
|
|
9
11
|
* Resolve the file URL to the `standard_fonts` directory shipped with pdfjs-dist.
|
|
@@ -18,22 +20,14 @@ function getStandardFontsUrl() {
|
|
|
18
20
|
}
|
|
19
21
|
/**
|
|
20
22
|
* Create a pdfjs-dist PDF document from a buffer.
|
|
21
|
-
*
|
|
22
|
-
* `@thednp/dommatrix` polyfill if unavailable.
|
|
23
|
+
* Uses `@thednp/dommatrix` for DOMMatrix polyfill (lighter alternative to native canvas).
|
|
23
24
|
* @param buffer - Raw buffer of the PDF file.
|
|
24
25
|
* @returns A promise resolving to a pdfjs-dist PDFDocumentProxy.
|
|
25
26
|
*/
|
|
26
27
|
async function createPdfDocument(buffer) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
globalThis.DOMMatrixReadOnly ??= DOMMatrix;
|
|
31
|
-
}
|
|
32
|
-
catch {
|
|
33
|
-
const { default: CSSMatrix } = await import("@thednp/dommatrix");
|
|
34
|
-
globalThis.DOMMatrix ??= CSSMatrix;
|
|
35
|
-
globalThis.DOMMatrixReadOnly ??= CSSMatrix;
|
|
36
|
-
}
|
|
28
|
+
const { default: CSSMatrix } = await import("@thednp/dommatrix");
|
|
29
|
+
globalThis.DOMMatrix ??= CSSMatrix;
|
|
30
|
+
globalThis.DOMMatrixReadOnly ??= CSSMatrix;
|
|
37
31
|
const { getDocument } = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
|
38
32
|
const loadingTask = getDocument({
|
|
39
33
|
data: new Uint8Array(buffer),
|
|
@@ -50,9 +44,13 @@ async function createPdfDocument(buffer) {
|
|
|
50
44
|
* @returns The extracted text with double-newline page separators.
|
|
51
45
|
*/
|
|
52
46
|
export async function extractPdfText(buffer) {
|
|
47
|
+
if (buffer.length > MAX_PDF_SIZE_BYTES) {
|
|
48
|
+
throw new Error(`PDF too large: ${(buffer.length / 1024 / 1024).toFixed(1)} MB (max 100 MB)`);
|
|
49
|
+
}
|
|
53
50
|
const pdf = await createPdfDocument(buffer);
|
|
54
51
|
const texts = [];
|
|
55
|
-
|
|
52
|
+
const pageCount = Math.min(pdf.numPages, MAX_PDF_PAGES);
|
|
53
|
+
for (let i = 1; i <= pageCount; i++) {
|
|
56
54
|
const page = await pdf.getPage(i);
|
|
57
55
|
const content = await page.getTextContent();
|
|
58
56
|
const textItems = content.items.filter((item) => typeof item === "object" && item !== null && "str" in item);
|
package/dist/chunker/xml.d.ts
CHANGED
|
@@ -11,6 +11,8 @@ export declare class XmlChunker extends TreeSitterChunker {
|
|
|
11
11
|
readonly fileExtensions: string[];
|
|
12
12
|
readonly grammarName = "xml";
|
|
13
13
|
readonly nodeTypes: Set<string>;
|
|
14
|
+
/** Limit to prevent tree-sitter from hanging on large SVG/XML files. */
|
|
15
|
+
maxContentBytes: number;
|
|
14
16
|
}
|
|
15
17
|
/** Default singleton instance of {@link XmlChunker}. */
|
|
16
18
|
export declare const xmlChunker: XmlChunker;
|
package/dist/chunker/xml.js
CHANGED
|
@@ -13,6 +13,8 @@ export class XmlChunker extends TreeSitterChunker {
|
|
|
13
13
|
nodeTypes = new Set([
|
|
14
14
|
"element",
|
|
15
15
|
]);
|
|
16
|
+
/** Limit to prevent tree-sitter from hanging on large SVG/XML files. */
|
|
17
|
+
maxContentBytes = 1_048_576;
|
|
16
18
|
}
|
|
17
19
|
/** Default singleton instance of {@link XmlChunker}. */
|
|
18
20
|
export const xmlChunker = new XmlChunker();
|
|
@@ -14,7 +14,6 @@ import chokidar from "chokidar";
|
|
|
14
14
|
import { appendDebugLog } from "../../core/fileLogger.js";
|
|
15
15
|
import { createWatchPassScheduler, createWatchIgnore, runIndexPass, } from "../../indexer.js";
|
|
16
16
|
import { c, resolveCliContext, cleanupContext, logCliError, logCliInfo, logIndexSummary, formatDuration, } from "../format.js";
|
|
17
|
-
import { TerminalProgressTable } from "../progress.js";
|
|
18
17
|
/**
|
|
19
18
|
* Build a logger that suppresses console output when watchTriggered is true.
|
|
20
19
|
*/
|
|
@@ -98,7 +97,6 @@ export function registerIndexCommand(program) {
|
|
|
98
97
|
}
|
|
99
98
|
}
|
|
100
99
|
logCliInfo(logFilePath, "index", `${c.label("Scanning:")} ${c.file(cwd)}`);
|
|
101
|
-
const progress = new TerminalProgressTable(process.stdout);
|
|
102
100
|
const runPass = async (watchTriggered = false, abortSignal, filterPaths) => {
|
|
103
101
|
const passStarted = Date.now();
|
|
104
102
|
const stats = await runIndexPass({
|
|
@@ -109,14 +107,12 @@ export function registerIndexCommand(program) {
|
|
|
109
107
|
embedder,
|
|
110
108
|
keywordIndex,
|
|
111
109
|
descriptionProvider,
|
|
112
|
-
progress,
|
|
113
110
|
force: !!(options.force && !watchTriggered),
|
|
114
111
|
abortSignal,
|
|
115
112
|
dimension,
|
|
116
113
|
filterPaths,
|
|
117
114
|
logger: watchAwareLogger(logFilePath, watchTriggered ? "watch" : "index", watchTriggered),
|
|
118
115
|
});
|
|
119
|
-
progress.done();
|
|
120
116
|
if (!watchTriggered) {
|
|
121
117
|
logIndexSummary(logFilePath, stats);
|
|
122
118
|
logCliInfo(logFilePath, "index", `\n${c.success("Indexing complete.")} ${c.num(stats.finalCount)} chunks stored (${formatDuration(Date.now() - passStarted)}).`);
|
|
@@ -11,6 +11,6 @@ export { registerDumpCommand } from "./dump.js";
|
|
|
11
11
|
export { registerInitCommand } from "./init.js";
|
|
12
12
|
export { registerUiCommand } from "./ui.js";
|
|
13
13
|
export { registerMcpCommand } from "./mcp.js";
|
|
14
|
-
export {
|
|
14
|
+
export { registerSetupCommand } from "./setup.js";
|
|
15
15
|
export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand } from "./eval.js";
|
|
16
16
|
export { registerDescribeImageCommand } from "./describe-image.js";
|
|
@@ -11,7 +11,7 @@ export { registerDumpCommand } from "./dump.js";
|
|
|
11
11
|
export { registerInitCommand } from "./init.js";
|
|
12
12
|
export { registerUiCommand } from "./ui.js";
|
|
13
13
|
export { registerMcpCommand } from "./mcp.js";
|
|
14
|
-
export {
|
|
14
|
+
export { registerSetupCommand } from "./setup.js";
|
|
15
15
|
export { registerEvalSessionsCommand, registerEvalAnalyzeCommand, registerEvalCompareCommand } from "./eval.js";
|
|
16
16
|
export { registerDescribeImageCommand } from "./describe-image.js";
|
|
17
17
|
//# sourceMappingURL=index.js.map
|
|
@@ -74,6 +74,18 @@ export declare function generateSkillFile(): string;
|
|
|
74
74
|
* @returns The merged `.gitignore` content with a trailing newline.
|
|
75
75
|
*/
|
|
76
76
|
export declare function mergeGitignoreContent(existingContent?: string): string;
|
|
77
|
+
/**
|
|
78
|
+
* Merge the OpenCodeRAG directive section into an existing `AGENTS.md`.
|
|
79
|
+
*
|
|
80
|
+
* The directive is wrapped in sentinel markers so re-runs replace the
|
|
81
|
+
* section in place rather than appending duplicates. Existing content
|
|
82
|
+
* outside the markers is always preserved. If no existing content is
|
|
83
|
+
* provided, the section alone is returned as a new file.
|
|
84
|
+
*
|
|
85
|
+
* @param existingContent - The current `AGENTS.md` content, or `undefined` if absent.
|
|
86
|
+
* @returns The merged `AGENTS.md` content with a trailing newline.
|
|
87
|
+
*/
|
|
88
|
+
export declare function mergeAgentsMdContent(existingContent?: string): string;
|
|
77
89
|
/**
|
|
78
90
|
* Get the runtime directory path (`~/.opencode`).
|
|
79
91
|
*
|