openclaw-mcp-router 0.2.1 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +9 -0
- package/.github/workflows/release.yml +5 -17
- package/CLAUDE.md +6 -4
- package/README.md +94 -10
- package/openclaw.plugin.json +17 -3
- package/package.json +3 -3
- package/src/chunker.ts +117 -0
- package/src/config.ts +53 -22
- package/src/constants.ts +2 -0
- package/src/embeddings.ts +11 -9
- package/src/index.ts +43 -14
- package/src/indexer.ts +144 -51
- package/src/mcp-client.ts +18 -8
- package/src/mcp-registry.ts +2 -1
- package/src/test/chunker.test.ts +100 -0
- package/src/test/config.test.ts +95 -12
- package/src/test/embeddings.test.ts +17 -17
- package/src/test/indexer.test.ts +361 -3
- package/src/test/tools/mcp-call-tool.test.ts +6 -6
- package/src/test/tools/mcp-search-tool.test.ts +91 -10
- package/src/tools/mcp-call-tool.ts +2 -1
- package/src/tools/mcp-search-tool.ts +18 -2
- package/src/vector-store.ts +23 -10
- package/tsconfig.json +1 -2
|
@@ -5,7 +5,7 @@ on:
|
|
|
5
5
|
types: [published]
|
|
6
6
|
|
|
7
7
|
permissions:
|
|
8
|
-
contents:
|
|
8
|
+
contents: read
|
|
9
9
|
id-token: write # npm provenance (links published package back to this repo+commit)
|
|
10
10
|
|
|
11
11
|
jobs:
|
|
@@ -13,14 +13,10 @@ jobs:
|
|
|
13
13
|
runs-on: ubuntu-latest
|
|
14
14
|
steps:
|
|
15
15
|
- uses: actions/checkout@v4
|
|
16
|
-
with:
|
|
17
|
-
# Use a PAT so the version-bump commit can trigger other workflows if needed.
|
|
18
|
-
# Falls back to GITHUB_TOKEN (works fine, but commits won't trigger further CI).
|
|
19
|
-
token: ${{ secrets.PAT_TOKEN || secrets.GITHUB_TOKEN }}
|
|
20
16
|
|
|
21
17
|
- uses: actions/setup-node@v4
|
|
22
18
|
with:
|
|
23
|
-
node-version:
|
|
19
|
+
node-version: 24
|
|
24
20
|
registry-url: https://registry.npmjs.org
|
|
25
21
|
|
|
26
22
|
- name: Extract version from release tag
|
|
@@ -39,19 +35,11 @@ jobs:
|
|
|
39
35
|
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
|
40
36
|
echo "Resolved version: $VERSION"
|
|
41
37
|
|
|
42
|
-
- name: Bump package.json version
|
|
43
|
-
run: npm version "${{ steps.version.outputs.version }}" --no-git-tag-version
|
|
44
|
-
|
|
45
|
-
- name: Commit version bump
|
|
46
|
-
run: |
|
|
47
|
-
git config user.name "github-actions[bot]"
|
|
48
|
-
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
49
|
-
git add package.json package-lock.json 2>/dev/null || true
|
|
50
|
-
git commit -m "chore: bump version to ${{ steps.version.outputs.version }}" || echo "No changes to commit"
|
|
51
|
-
git push origin HEAD:${{ github.event.release.target_commitish }}
|
|
52
|
-
|
|
53
38
|
- run: npm ci
|
|
54
39
|
|
|
40
|
+
- name: Set package version from tag (publish only, no commit)
|
|
41
|
+
run: npm version "${{ steps.version.outputs.version }}" --no-git-tag-version
|
|
42
|
+
|
|
55
43
|
- name: Publish to npm
|
|
56
44
|
run: npm publish --provenance --access public
|
|
57
45
|
env:
|
package/CLAUDE.md
CHANGED
|
@@ -17,7 +17,7 @@ This is an **OpenClaw plugin** that solves MCP context bloat. Instead of loading
|
|
|
17
17
|
|
|
18
18
|
### Data Flow
|
|
19
19
|
|
|
20
|
-
1. **Startup (indexing):** Plugin connects to each configured MCP server in parallel → lists tools → embeds descriptions via Ollama → stores vectors in local LanceDB
|
|
20
|
+
1. **Startup (indexing):** Plugin connects to each configured MCP server in parallel (with retry/backoff per server) → lists tools → embeds descriptions via Ollama → stores vectors in local LanceDB. An `AbortController` governs the lifecycle — `stop()` cancels all in-flight connections and delays immediately.
|
|
21
21
|
2. **Search:** Agent calls `mcp_search(query)` → query embedded → vector similarity search → returns matching tool cards with schemas
|
|
22
22
|
3. **Execution:** Agent calls `mcp_call(tool_name, params_json)` → registry lookup for owning server → fresh MCP client connection → execute → return result
|
|
23
23
|
|
|
@@ -26,11 +26,11 @@ This is an **OpenClaw plugin** that solves MCP context bloat. Instead of loading
|
|
|
26
26
|
| Module | Role |
|
|
27
27
|
|--------|------|
|
|
28
28
|
| `src/index.ts` | Plugin entry point — registers tools, CLI commands, and startup service with OpenClaw's `OpenClawPluginApi` |
|
|
29
|
-
| `src/config.ts` | Parses plugin YAML config into typed `McpRouterConfig
|
|
29
|
+
| `src/config.ts` | Parses plugin YAML config into typed `McpRouterConfig` (including `IndexerConfig` and per-server `timeout`); validates, applies defaults, expands `${VAR}` and `~/` |
|
|
30
30
|
| `src/embeddings.ts` | `OllamaEmbeddings` — embeds text via Ollama's `/api/embeddings`; SSRF-safe (localhost-only); caches known model dimensions |
|
|
31
31
|
| `src/vector-store.ts` | `McpToolVectorStore` — LanceDB wrapper; lazy init; upsert via delete-then-add; L2 distance → similarity score |
|
|
32
|
-
| `src/indexer.ts` | `runIndexer()` — parallel server indexing with `Promise.allSettled`;
|
|
33
|
-
| `src/mcp-client.ts` | `McpClient` — thin MCP SDK wrapper; supports stdio/sse/http transports |
|
|
32
|
+
| `src/indexer.ts` | `runIndexer()` — parallel server indexing with `Promise.allSettled`; per-server retry with exponential backoff; `AbortSignal` threading for cancellation; `abortableDelay()` helper |
|
|
33
|
+
| `src/mcp-client.ts` | `McpClient` — thin MCP SDK wrapper; supports stdio/sse/http transports; `connect()` accepts optional `{ signal, timeout }` forwarded to SDK |
|
|
34
34
|
| `src/mcp-registry.ts` | `McpRegistry` — in-memory `toolName → serverConfig` map; last-writer-wins on name collisions |
|
|
35
35
|
| `src/tools/mcp-search-tool.ts` | `mcp_search` tool — embeds query, searches vector store, formats tool cards |
|
|
36
36
|
| `src/tools/mcp-call-tool.ts` | `mcp_call` tool — resolves server from registry, opens fresh connection, executes, disconnects |
|
|
@@ -41,6 +41,8 @@ This is an **OpenClaw plugin** that solves MCP context bloat. Instead of loading
|
|
|
41
41
|
- **Optional tool registration.** Both tools use `{ optional: true }` — they only appear in agent context when `tools.alsoAllow` includes them.
|
|
42
42
|
- **Compound tool IDs.** Vector store entries use `"${serverName}::${toolName}"` as stable upsert keys.
|
|
43
43
|
- **Graceful degradation.** Indexer uses `Promise.allSettled` — one failing server doesn't block others. Ollama being unreachable is a warning, not a crash.
|
|
44
|
+
- **Retry with backoff.** Each server gets `maxRetries` attempts with exponential backoff (`initialRetryDelay * 2^(attempt-1)`, capped at `maxRetryDelay`). Per-server `timeout` overrides the global `indexer.connectTimeout`.
|
|
45
|
+
- **AbortSignal lifecycle.** `runIndexer` accepts an optional `AbortSignal`. The plugin entry point manages an `AbortController` — `start()` creates one, `stop()` aborts it. The `reindex` CLI command handles SIGINT. `abortableDelay()` ensures backoff waits are cancelled immediately on abort.
|
|
44
46
|
- **Fresh connections per call.** `mcp_call` opens a new MCP client connection for each invocation (stateless, no pooling).
|
|
45
47
|
|
|
46
48
|
## Testing
|
package/README.md
CHANGED
|
@@ -21,10 +21,17 @@ The agent asks for tools it needs instead of receiving every schema upfront.
|
|
|
21
21
|
openclaw plugins install openclaw-mcp-router
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
+
**Alternative: install from source**
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
git clone https://github.com/lunarmoon26/openclaw-mcp-router.git
|
|
28
|
+
openclaw plugins install ./openclaw-mcp-router
|
|
29
|
+
```
|
|
30
|
+
|
|
24
31
|
Requires [Ollama](https://ollama.ai) running locally with an embedding model:
|
|
25
32
|
|
|
26
33
|
```sh
|
|
27
|
-
ollama pull
|
|
34
|
+
ollama pull embeddinggemma
|
|
28
35
|
ollama serve
|
|
29
36
|
```
|
|
30
37
|
|
|
@@ -39,7 +46,7 @@ tools:
|
|
|
39
46
|
- mcp_call
|
|
40
47
|
|
|
41
48
|
plugins:
|
|
42
|
-
mcp-router:
|
|
49
|
+
openclaw-mcp-router:
|
|
43
50
|
enabled: true
|
|
44
51
|
config:
|
|
45
52
|
servers:
|
|
@@ -52,13 +59,41 @@ plugins:
|
|
|
52
59
|
url: https://api.githubcopilot.com/mcp/
|
|
53
60
|
embedding:
|
|
54
61
|
provider: ollama
|
|
55
|
-
model:
|
|
62
|
+
model: embeddinggemma # or qwen3-embedding:0.6b, all-minilm
|
|
56
63
|
url: http://localhost:11434
|
|
57
64
|
search:
|
|
58
65
|
topK: 5 # tools returned per search (1–20)
|
|
59
66
|
minScore: 0.3 # minimum similarity threshold (0–1)
|
|
60
67
|
```
|
|
61
68
|
|
|
69
|
+
### Important: `tools.alsoAllow` is required
|
|
70
|
+
|
|
71
|
+
The plugin registers `mcp_search` and `mcp_call` as **optional tools** (`optional: true`). This means the gateway loads them, but they are **not exposed to agents** unless explicitly allowlisted.
|
|
72
|
+
|
|
73
|
+
If the plugin is running and `openclaw openclaw-mcp-router stats` shows indexed tools, but your agent can't call `mcp_search` — this is why. Add the allowlist to your config:
|
|
74
|
+
|
|
75
|
+
```yaml
|
|
76
|
+
# Global — all agents get access
|
|
77
|
+
tools:
|
|
78
|
+
alsoAllow:
|
|
79
|
+
- mcp_search
|
|
80
|
+
- mcp_call
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or scope it to specific agents:
|
|
84
|
+
|
|
85
|
+
```yaml
|
|
86
|
+
# Per-agent or under agents.defaults
|
|
87
|
+
agents:
|
|
88
|
+
defaults:
|
|
89
|
+
tools:
|
|
90
|
+
alsoAllow:
|
|
91
|
+
- mcp_search
|
|
92
|
+
- mcp_call
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Restart the gateway after changing the config.
|
|
96
|
+
|
|
62
97
|
## Configuration reference
|
|
63
98
|
|
|
64
99
|
### `servers[]`
|
|
@@ -71,20 +106,54 @@ plugins:
|
|
|
71
106
|
| `args` | stdio only | Arguments array |
|
|
72
107
|
| `env` | no | Extra env vars merged over process.env; supports `${VAR}` expansion |
|
|
73
108
|
| `url` | sse/http only | Server endpoint URL |
|
|
109
|
+
| `timeout` | no | Per-server connect timeout in ms; overrides `indexer.connectTimeout` |
|
|
74
110
|
|
|
75
111
|
### `embedding`
|
|
76
112
|
|
|
77
113
|
| Field | Default | Description |
|
|
78
114
|
|-------|---------|-------------|
|
|
79
115
|
| `provider` | `ollama` | Only Ollama is supported |
|
|
80
|
-
| `model` | `
|
|
116
|
+
| `model` | `embeddinggemma` | Embedding model name |
|
|
81
117
|
| `url` | `http://localhost:11434` | Ollama base URL (must be localhost) |
|
|
82
118
|
|
|
83
119
|
### `vectorDb`
|
|
84
120
|
|
|
85
121
|
| Field | Default | Description |
|
|
86
122
|
|-------|---------|-------------|
|
|
87
|
-
| `path` | `~/.openclaw/mcp-router/lancedb` | LanceDB database directory |
|
|
123
|
+
| `path` | `~/.openclaw/openclaw-mcp-router/lancedb` | LanceDB database directory |
|
|
124
|
+
|
|
125
|
+
### `indexer`
|
|
126
|
+
|
|
127
|
+
Controls retry behavior and timeouts when connecting to MCP servers at startup. Useful for self-hosted servers (e.g. started via `uvx`) that take time to start up.
|
|
128
|
+
|
|
129
|
+
| Field | Default | Description |
|
|
130
|
+
|-------|---------|-------------|
|
|
131
|
+
| `connectTimeout` | `60000` | Per-server default connect timeout in ms |
|
|
132
|
+
| `maxRetries` | `3` | Retry attempts per server (0 = no retry) |
|
|
133
|
+
| `initialRetryDelay` | `2000` | Initial backoff delay in ms |
|
|
134
|
+
| `maxRetryDelay` | `30000` | Max backoff cap in ms |
|
|
135
|
+
| `maxChunkChars` | `500` | Max characters per chunk for long tool descriptions. `0` = disable chunking |
|
|
136
|
+
| `overlapChars` | `100` | Overlap characters between adjacent chunks |
|
|
137
|
+
|
|
138
|
+
Retries use exponential backoff: delays are `initialRetryDelay * 2^(attempt-1)`, capped at `maxRetryDelay`. With defaults, a slow server gets attempts at ~0s, ~2s, ~4s, ~8s before giving up.
|
|
139
|
+
|
|
140
|
+
**Chunking:** When a tool description exceeds `maxChunkChars`, it is split into overlapping chunks at semantic boundaries (paragraphs, lines, sentences). Each chunk is stored as a separate vector, and search results are deduplicated so each tool appears once with its best matching score. Short descriptions (the common case) are unaffected.
|
|
141
|
+
|
|
142
|
+
Example for a slow-starting Python server:
|
|
143
|
+
|
|
144
|
+
```yaml
|
|
145
|
+
plugins:
|
|
146
|
+
openclaw-mcp-router:
|
|
147
|
+
config:
|
|
148
|
+
mcpServers:
|
|
149
|
+
my-python-server:
|
|
150
|
+
command: uvx
|
|
151
|
+
args: ["my-mcp-server"]
|
|
152
|
+
timeout: 120000 # this server needs 2 minutes to start
|
|
153
|
+
indexer:
|
|
154
|
+
maxRetries: 5
|
|
155
|
+
initialRetryDelay: 3000
|
|
156
|
+
```
|
|
88
157
|
|
|
89
158
|
### `search`
|
|
90
159
|
|
|
@@ -97,28 +166,43 @@ plugins:
|
|
|
97
166
|
|
|
98
167
|
```sh
|
|
99
168
|
# Re-index all configured MCP servers
|
|
100
|
-
openclaw mcp-router reindex
|
|
169
|
+
openclaw openclaw-mcp-router reindex
|
|
101
170
|
|
|
102
171
|
# Show indexed tool count
|
|
103
|
-
openclaw mcp-router stats
|
|
172
|
+
openclaw openclaw-mcp-router stats
|
|
104
173
|
```
|
|
105
174
|
|
|
106
175
|
## How it works
|
|
107
176
|
|
|
108
|
-
1. At gateway startup, the plugin connects to each MCP server, lists its tools, embeds each description via Ollama, and stores them in a local LanceDB index.
|
|
177
|
+
1. At gateway startup, the plugin connects to each MCP server in parallel (with retry and configurable timeouts), lists its tools, embeds each description via Ollama, and stores them in a local LanceDB index.
|
|
109
178
|
2. When the agent needs to use an MCP capability, it calls `mcp_search("what I want to do")` to find relevant tools.
|
|
110
179
|
3. The agent then calls `mcp_call("tool_name", '{"param": "value"}')` to execute the chosen tool.
|
|
111
180
|
|
|
181
|
+
Disabling the plugin (`openclaw plugins disable openclaw-mcp-router`) cancels any in-progress indexing immediately. Re-enabling starts fresh.
|
|
182
|
+
|
|
112
183
|
## Supported embedding models
|
|
113
184
|
|
|
114
185
|
| Model | Dims | Notes |
|
|
115
186
|
|-------|------|-------|
|
|
116
|
-
| `
|
|
117
|
-
| `
|
|
187
|
+
| `embeddinggemma` | 768 | Good balance, recommended default |
|
|
188
|
+
| `qwen3-embedding:0.6b` | 1024 | Higher quality, larger footprint |
|
|
118
189
|
| `all-minilm` | 384 | Fast and lightweight |
|
|
119
190
|
|
|
120
191
|
Any Ollama embedding model works — dimensions are detected automatically for unknown models.
|
|
121
192
|
|
|
193
|
+
## Background
|
|
194
|
+
|
|
195
|
+
This plugin is a basic implementation of the [Tool Search Tool](https://www.anthropic.com/engineering/advanced-tool-use) pattern from Anthropic's advanced tool use guide. The core idea: instead of injecting all tool schemas into the system prompt, provide a search tool that dynamically surfaces only relevant tools at runtime. Anthropic's benchmarks showed this improved tool selection accuracy from 49% to 74% (Opus 4) and 79.5% to 88.1% (Opus 4.5).
|
|
196
|
+
|
|
197
|
+
Our approach is intentionally simple — pure vector similarity over tool descriptions. There's plenty of room to improve:
|
|
198
|
+
|
|
199
|
+
- **Hybrid search (BM25 + embedding).** Pure embedding search can miss exact keyword matches. Combining sparse retrieval (BM25/TF-IDF) with dense vectors would improve recall, especially for tools with distinctive names like `git_diff` or `kubectl_apply`.
|
|
200
|
+
- **LLM-based reranking.** After the initial vector search returns candidates, a small LLM could rerank them based on the full query context — catching semantic nuances that cosine similarity misses.
|
|
201
|
+
- **Tool use examples.** Indexing not just descriptions but example invocations (input/output pairs) would let the search match against concrete usage patterns, not just what the tool claims to do.
|
|
202
|
+
- **Programmatic tool calling.** Anthropic's guide describes letting the agent compose tool calls inside code blocks rather than pure JSON — reducing context pollution and enabling multi-step tool pipelines.
|
|
203
|
+
|
|
204
|
+
Contributions welcome if any of these interest you.
|
|
205
|
+
|
|
122
206
|
## License
|
|
123
207
|
|
|
124
208
|
MIT
|
package/openclaw.plugin.json
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"id": "mcp-router",
|
|
2
|
+
"id": "openclaw-mcp-router",
|
|
3
3
|
"name": "MCP Router",
|
|
4
4
|
"description": "Semantic search across MCP tool catalogs. Reduces context bloat from ~77k→8.7k tokens by dynamically surfacing only relevant tools.",
|
|
5
5
|
"configSchema": {
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
"url": { "type": "string" },
|
|
19
19
|
"serverUrl": { "type": "string" },
|
|
20
20
|
"type": { "type": "string", "enum": ["stdio", "sse", "http"] },
|
|
21
|
-
"headers": { "type": "object", "additionalProperties": { "type": "string" } }
|
|
21
|
+
"headers": { "type": "object", "additionalProperties": { "type": "string" } },
|
|
22
|
+
"timeout": { "type": "number", "description": "Per-server connect timeout in ms; overrides indexer.connectTimeout" }
|
|
22
23
|
}
|
|
23
24
|
}
|
|
24
25
|
},
|
|
@@ -39,10 +40,23 @@
|
|
|
39
40
|
"args": { "type": "array", "items": { "type": "string" } },
|
|
40
41
|
"env": { "type": "object", "additionalProperties": { "type": "string" } },
|
|
41
42
|
"url": { "type": "string" },
|
|
42
|
-
"headers": { "type": "object", "additionalProperties": { "type": "string" } }
|
|
43
|
+
"headers": { "type": "object", "additionalProperties": { "type": "string" } },
|
|
44
|
+
"timeout": { "type": "number", "description": "Per-server connect timeout in ms; overrides indexer.connectTimeout" }
|
|
43
45
|
}
|
|
44
46
|
}
|
|
45
47
|
},
|
|
48
|
+
"indexer": {
|
|
49
|
+
"type": "object",
|
|
50
|
+
"description": "Indexer retry and timeout settings",
|
|
51
|
+
"properties": {
|
|
52
|
+
"connectTimeout": { "type": "number", "description": "Per-server default connect timeout in ms (default: 60000)" },
|
|
53
|
+
"maxRetries": { "type": "number", "minimum": 0, "description": "Retry attempts per server, 0 = no retry (default: 3)" },
|
|
54
|
+
"initialRetryDelay": { "type": "number", "description": "Initial backoff delay in ms (default: 2000)" },
|
|
55
|
+
"maxRetryDelay": { "type": "number", "description": "Max backoff cap in ms (default: 30000)" },
|
|
56
|
+
"maxChunkChars": { "type": "number", "minimum": 0, "description": "Max chars per chunk for long tool descriptions. 0 = disable chunking (default: 500)" },
|
|
57
|
+
"overlapChars": { "type": "number", "minimum": 0, "description": "Overlap chars between adjacent chunks (default: 100)" }
|
|
58
|
+
}
|
|
59
|
+
},
|
|
46
60
|
"embedding": {
|
|
47
61
|
"type": "object",
|
|
48
62
|
"properties": {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "openclaw-mcp-router",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.6",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Dynamic MCP tool router for OpenClaw — semantic search over large MCP catalogs to eliminate context bloat",
|
|
6
6
|
"type": "module",
|
|
@@ -23,10 +23,10 @@
|
|
|
23
23
|
"@sinclair/typebox": "^0.34.48"
|
|
24
24
|
},
|
|
25
25
|
"peerDependencies": {
|
|
26
|
-
"openclaw": "^2026.2.
|
|
26
|
+
"openclaw": "^2026.2.23"
|
|
27
27
|
},
|
|
28
28
|
"devDependencies": {
|
|
29
|
-
"openclaw": "^2026.2.
|
|
29
|
+
"openclaw": "^2026.2.23",
|
|
30
30
|
"typescript": "^5.7.0",
|
|
31
31
|
"vitest": "^3.0.0"
|
|
32
32
|
},
|
package/src/chunker.ts
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
export type ChunkConfig = {
|
|
2
|
+
/** Max characters per chunk. 0 = disable chunking. */
|
|
3
|
+
maxChunkChars: number;
|
|
4
|
+
/** Overlap characters between adjacent chunks. */
|
|
5
|
+
overlapChars: number;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export type Chunk = {
|
|
9
|
+
/** 0-based chunk index */
|
|
10
|
+
index: number;
|
|
11
|
+
/** Total chunks for this tool */
|
|
12
|
+
total: number;
|
|
13
|
+
/** Chunk text content */
|
|
14
|
+
text: string;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Split text into overlapping chunks that respect semantic boundaries.
|
|
19
|
+
*
|
|
20
|
+
* Fast path: if the text fits in a single chunk (or chunking is disabled),
|
|
21
|
+
* returns a single chunk with zero overhead.
|
|
22
|
+
*
|
|
23
|
+
* @param text - The full text to chunk (e.g. "tool_name: description")
|
|
24
|
+
* @param toolNamePrefix - Prefixed to chunks with index > 0 so the embedding
|
|
25
|
+
* model always sees the tool name (e.g. "tool_name: ... ")
|
|
26
|
+
* @param config - Chunking parameters
|
|
27
|
+
*/
|
|
28
|
+
export function chunkText(
|
|
29
|
+
text: string,
|
|
30
|
+
toolNamePrefix: string,
|
|
31
|
+
config: ChunkConfig,
|
|
32
|
+
): Chunk[] {
|
|
33
|
+
// Fast path: single chunk
|
|
34
|
+
if (config.maxChunkChars === 0 || text.length <= config.maxChunkChars) {
|
|
35
|
+
return [{ index: 0, total: 1, text }];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const segments = splitSegments(text);
|
|
39
|
+
const chunks = mergeSegments(segments, toolNamePrefix, config);
|
|
40
|
+
const total = chunks.length;
|
|
41
|
+
|
|
42
|
+
return chunks.map((text, index) => ({ index, total, text }));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Split text into segments using a separator hierarchy that preserves
|
|
47
|
+
* semantic boundaries: \n\n → \n → ". " → hard character boundary.
|
|
48
|
+
*/
|
|
49
|
+
function splitSegments(text: string): string[] {
|
|
50
|
+
// Try separators in order of preference
|
|
51
|
+
const separators = ["\n\n", "\n", ". "];
|
|
52
|
+
|
|
53
|
+
for (const sep of separators) {
|
|
54
|
+
if (text.includes(sep)) {
|
|
55
|
+
const parts = text.split(sep);
|
|
56
|
+
// Re-attach separator to end of each part (except last)
|
|
57
|
+
return parts.map((part, i) => (i < parts.length - 1 ? part + sep : part));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// No separators found — return the whole text as one segment
|
|
62
|
+
// (will be hard-split in mergeSegments if needed)
|
|
63
|
+
return [text];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Greedily merge segments into chunks up to maxChunkChars.
|
|
68
|
+
* When a chunk is full, start a new one with overlap from the previous chunk's tail.
|
|
69
|
+
* Chunks with index > 0 get the tool name prefix.
|
|
70
|
+
*/
|
|
71
|
+
function mergeSegments(
|
|
72
|
+
segments: string[],
|
|
73
|
+
toolNamePrefix: string,
|
|
74
|
+
config: ChunkConfig,
|
|
75
|
+
): string[] {
|
|
76
|
+
const { maxChunkChars, overlapChars } = config;
|
|
77
|
+
const continuationPrefix = `${toolNamePrefix}: ... `;
|
|
78
|
+
const chunks: string[] = [];
|
|
79
|
+
|
|
80
|
+
let current = "";
|
|
81
|
+
|
|
82
|
+
for (const segment of segments) {
|
|
83
|
+
// If adding this segment exceeds the limit, finalize current chunk
|
|
84
|
+
if (current.length > 0 && current.length + segment.length > maxChunkChars) {
|
|
85
|
+
chunks.push(current);
|
|
86
|
+
// Start new chunk with overlap from the tail of the previous chunk
|
|
87
|
+
const overlap = overlapChars > 0 ? current.slice(-overlapChars) : "";
|
|
88
|
+
current = continuationPrefix + overlap;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// If a single segment is too long, hard-split it
|
|
92
|
+
if (segment.length > maxChunkChars) {
|
|
93
|
+
// Flush anything accumulated
|
|
94
|
+
if (current.length > 0 && current !== continuationPrefix) {
|
|
95
|
+
// There may be partial content — include it
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
let remaining = current + segment;
|
|
99
|
+
while (remaining.length > maxChunkChars) {
|
|
100
|
+
chunks.push(remaining.slice(0, maxChunkChars));
|
|
101
|
+
const overlap = overlapChars > 0 ? remaining.slice(maxChunkChars - overlapChars, maxChunkChars) : "";
|
|
102
|
+
remaining = continuationPrefix + overlap + remaining.slice(maxChunkChars);
|
|
103
|
+
}
|
|
104
|
+
current = remaining;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
current += segment;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Flush remaining
|
|
112
|
+
if (current.length > 0) {
|
|
113
|
+
chunks.push(current);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return chunks;
|
|
117
|
+
}
|
package/src/config.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { EXTENSION_ID } from "./constants.js";
|
|
4
5
|
import type { EmbeddingConfig, EmbeddingProvider } from "./embeddings.js";
|
|
5
6
|
|
|
6
7
|
export type McpTransportKind = "stdio" | "sse" | "http";
|
|
@@ -18,6 +19,23 @@ export type McpServerConfig = {
|
|
|
18
19
|
url?: string;
|
|
19
20
|
/** sse/http: extra headers; ${VAR} expanded */
|
|
20
21
|
headers?: Record<string, string>;
|
|
22
|
+
/** Per-server connect timeout in ms; overrides indexer.connectTimeout */
|
|
23
|
+
timeout?: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export type IndexerConfig = {
|
|
27
|
+
/** Per-server default connect timeout in ms (default: 60_000) */
|
|
28
|
+
connectTimeout: number;
|
|
29
|
+
/** Retry attempts per server, 0 = no retry (default: 3) */
|
|
30
|
+
maxRetries: number;
|
|
31
|
+
/** Initial backoff delay in ms (default: 2_000) */
|
|
32
|
+
initialRetryDelay: number;
|
|
33
|
+
/** Max backoff cap in ms (default: 30_000) */
|
|
34
|
+
maxRetryDelay: number;
|
|
35
|
+
/** Max characters per chunk for long tool descriptions. 0 = disable chunking. (default: 500) */
|
|
36
|
+
maxChunkChars: number;
|
|
37
|
+
/** Overlap characters between adjacent chunks (default: 100) */
|
|
38
|
+
overlapChars: number;
|
|
21
39
|
};
|
|
22
40
|
|
|
23
41
|
export type McpRouterConfig = {
|
|
@@ -25,6 +43,7 @@ export type McpRouterConfig = {
|
|
|
25
43
|
embedding: EmbeddingConfig;
|
|
26
44
|
vectorDb: { path: string };
|
|
27
45
|
search: { topK: number; minScore: number };
|
|
46
|
+
indexer: IndexerConfig;
|
|
28
47
|
};
|
|
29
48
|
|
|
30
49
|
export type ParseConfigOpts = {
|
|
@@ -53,7 +72,7 @@ function resolveHome(p: string): string {
|
|
|
53
72
|
}
|
|
54
73
|
|
|
55
74
|
const DEFAULT_OLLAMA_BASE_URL = "http://localhost:11434/v1";
|
|
56
|
-
const DEFAULT_EMBEDDING_MODEL = "
|
|
75
|
+
const DEFAULT_EMBEDDING_MODEL = "embeddinggemma";
|
|
57
76
|
const DEFAULT_MCP_SERVERS_FILE = "~/.openclaw/.mcp.json";
|
|
58
77
|
|
|
59
78
|
// ── mcpServers dict parsing ──────────────────────────────────────────────
|
|
@@ -62,7 +81,7 @@ function parseMcpServersDict(dict: Record<string, unknown>): McpServerConfig[] {
|
|
|
62
81
|
const servers: McpServerConfig[] = [];
|
|
63
82
|
for (const [name, raw] of Object.entries(dict)) {
|
|
64
83
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
|
65
|
-
throw new Error(
|
|
84
|
+
throw new Error(`${EXTENSION_ID}: mcpServers["${name}"] must be an object`);
|
|
66
85
|
}
|
|
67
86
|
const sv = raw as Record<string, unknown>;
|
|
68
87
|
|
|
@@ -74,14 +93,14 @@ function parseMcpServersDict(dict: Record<string, unknown>): McpServerConfig[] {
|
|
|
74
93
|
transport = "http";
|
|
75
94
|
} else {
|
|
76
95
|
throw new Error(
|
|
77
|
-
|
|
96
|
+
`${EXTENSION_ID}: mcpServers["${name}"] must have either "command" (stdio) or "url"/"serverUrl" (http)`,
|
|
78
97
|
);
|
|
79
98
|
}
|
|
80
99
|
|
|
81
100
|
// Allow explicit type override (e.g. "sse" for legacy servers)
|
|
82
101
|
if (typeof sv.type === "string") {
|
|
83
102
|
if (!["stdio", "sse", "http"].includes(sv.type)) {
|
|
84
|
-
throw new Error(
|
|
103
|
+
throw new Error(`${EXTENSION_ID}: mcpServers["${name}"].type must be stdio, sse, or http`);
|
|
85
104
|
}
|
|
86
105
|
transport = sv.type as McpTransportKind;
|
|
87
106
|
}
|
|
@@ -98,6 +117,7 @@ function parseMcpServersDict(dict: Record<string, unknown>): McpServerConfig[] {
|
|
|
98
117
|
env: expandEnvRecord(rawEnv),
|
|
99
118
|
url,
|
|
100
119
|
headers: Object.keys(rawHeaders).length > 0 ? expandEnvRecord(rawHeaders) : undefined,
|
|
120
|
+
timeout: typeof sv.timeout === "number" ? sv.timeout : undefined,
|
|
101
121
|
});
|
|
102
122
|
}
|
|
103
123
|
return servers;
|
|
@@ -118,11 +138,11 @@ function loadMcpServersFile(filePath: string, opts?: ParseConfigOpts): McpServer
|
|
|
118
138
|
try {
|
|
119
139
|
parsed = JSON.parse(content);
|
|
120
140
|
} catch {
|
|
121
|
-
throw new Error(
|
|
141
|
+
throw new Error(`${EXTENSION_ID}: failed to parse ${resolved} as JSON`);
|
|
122
142
|
}
|
|
123
143
|
|
|
124
144
|
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
125
|
-
throw new Error(
|
|
145
|
+
throw new Error(`${EXTENSION_ID}: ${resolved} must contain a JSON object`);
|
|
126
146
|
}
|
|
127
147
|
|
|
128
148
|
const obj = parsed as Record<string, unknown>;
|
|
@@ -155,7 +175,7 @@ function resolveEmbeddingConfig(
|
|
|
155
175
|
} else if (provider === "ollama") {
|
|
156
176
|
baseUrl = DEFAULT_OLLAMA_BASE_URL;
|
|
157
177
|
} else {
|
|
158
|
-
throw new Error(
|
|
178
|
+
throw new Error(`${EXTENSION_ID}: embedding.baseUrl is required for provider "${provider}"`);
|
|
159
179
|
}
|
|
160
180
|
|
|
161
181
|
return {
|
|
@@ -212,25 +232,25 @@ function resolveEmbeddingConfig(
|
|
|
212
232
|
function parseLegacyServers(serversRaw: unknown[]): McpServerConfig[] {
|
|
213
233
|
return serversRaw.map((s: unknown, i: number) => {
|
|
214
234
|
if (!s || typeof s !== "object" || Array.isArray(s)) {
|
|
215
|
-
throw new Error(
|
|
235
|
+
throw new Error(`${EXTENSION_ID}: servers[${i}] must be an object`);
|
|
216
236
|
}
|
|
217
237
|
const sv = s as Record<string, unknown>;
|
|
218
238
|
|
|
219
239
|
if (typeof sv.name !== "string" || !sv.name.trim()) {
|
|
220
|
-
throw new Error(
|
|
240
|
+
throw new Error(`${EXTENSION_ID}: servers[${i}].name is required`);
|
|
221
241
|
}
|
|
222
242
|
const transport = sv.transport as McpTransportKind;
|
|
223
243
|
if (!["stdio", "sse", "http"].includes(transport)) {
|
|
224
|
-
throw new Error(
|
|
244
|
+
throw new Error(`${EXTENSION_ID}: servers[${i}].transport must be stdio, sse, or http`);
|
|
225
245
|
}
|
|
226
246
|
|
|
227
247
|
if (transport === "stdio") {
|
|
228
248
|
if (typeof sv.command !== "string" || !sv.command.trim()) {
|
|
229
|
-
throw new Error(
|
|
249
|
+
throw new Error(`${EXTENSION_ID}: servers[${i}] with transport=stdio requires command`);
|
|
230
250
|
}
|
|
231
251
|
} else {
|
|
232
252
|
if (typeof sv.url !== "string" || !sv.url.trim()) {
|
|
233
|
-
throw new Error(
|
|
253
|
+
throw new Error(`${EXTENSION_ID}: servers[${i}] with transport=${transport} requires url`);
|
|
234
254
|
}
|
|
235
255
|
}
|
|
236
256
|
|
|
@@ -245,6 +265,7 @@ function parseLegacyServers(serversRaw: unknown[]): McpServerConfig[] {
|
|
|
245
265
|
env: expandEnvRecord(rawEnv),
|
|
246
266
|
url: typeof sv.url === "string" ? sv.url : undefined,
|
|
247
267
|
headers: Object.keys(rawHeaders).length > 0 ? expandEnvRecord(rawHeaders) : undefined,
|
|
268
|
+
timeout: typeof sv.timeout === "number" ? sv.timeout : undefined,
|
|
248
269
|
};
|
|
249
270
|
});
|
|
250
271
|
}
|
|
@@ -253,10 +274,12 @@ function parseLegacyServers(serversRaw: unknown[]): McpServerConfig[] {
|
|
|
253
274
|
|
|
254
275
|
/** Parse and validate raw plugin config, applying defaults. Throws on invalid input. */
|
|
255
276
|
export function parseConfig(raw: unknown, opts?: ParseConfigOpts): McpRouterConfig {
|
|
256
|
-
|
|
257
|
-
|
|
277
|
+
// Treat null/undefined as empty config — allows auto-loading from default file
|
|
278
|
+
const normalized = raw ?? {};
|
|
279
|
+
if (typeof normalized !== "object" || Array.isArray(normalized)) {
|
|
280
|
+
throw new Error(`${EXTENSION_ID}: config must be an object`);
|
|
258
281
|
}
|
|
259
|
-
const r =
|
|
282
|
+
const r = normalized as Record<string, unknown>;
|
|
260
283
|
|
|
261
284
|
// ── Server resolution priority: mcpServers > mcpServersFile > servers (legacy) ──
|
|
262
285
|
let servers: McpServerConfig[] = [];
|
|
@@ -272,11 +295,8 @@ export function parseConfig(raw: unknown, opts?: ParseConfigOpts): McpRouterConf
|
|
|
272
295
|
servers = loadMcpServersFile(DEFAULT_MCP_SERVERS_FILE, opts);
|
|
273
296
|
}
|
|
274
297
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
"mcp-router: no servers configured. Provide mcpServers, mcpServersFile, or servers[].",
|
|
278
|
-
);
|
|
279
|
-
}
|
|
298
|
+
// Empty servers is valid — user may add servers later.
|
|
299
|
+
// Callers should check servers.length and skip indexing if zero.
|
|
280
300
|
|
|
281
301
|
// ── Embedding config ──
|
|
282
302
|
const embRaw = r.embedding && typeof r.embedding === "object" && !Array.isArray(r.embedding)
|
|
@@ -289,7 +309,7 @@ export function parseConfig(raw: unknown, opts?: ParseConfigOpts): McpRouterConf
|
|
|
289
309
|
const dbPath =
|
|
290
310
|
typeof vdbRaw.path === "string"
|
|
291
311
|
? resolveHome(vdbRaw.path)
|
|
292
|
-
: path.join(os.homedir(), ".openclaw",
|
|
312
|
+
: path.join(os.homedir(), ".openclaw", EXTENSION_ID, "lancedb");
|
|
293
313
|
const vectorDb = { path: dbPath };
|
|
294
314
|
|
|
295
315
|
// ── search defaults ──
|
|
@@ -299,5 +319,16 @@ export function parseConfig(raw: unknown, opts?: ParseConfigOpts): McpRouterConf
|
|
|
299
319
|
minScore: typeof srchRaw.minScore === "number" ? srchRaw.minScore : 0.3,
|
|
300
320
|
};
|
|
301
321
|
|
|
302
|
-
|
|
322
|
+
// ── indexer defaults ──
|
|
323
|
+
const idxRaw = (r.indexer ?? {}) as Record<string, unknown>;
|
|
324
|
+
const indexer: IndexerConfig = {
|
|
325
|
+
connectTimeout: typeof idxRaw.connectTimeout === "number" ? idxRaw.connectTimeout : 60_000,
|
|
326
|
+
maxRetries: typeof idxRaw.maxRetries === "number" ? Math.max(0, idxRaw.maxRetries) : 3,
|
|
327
|
+
initialRetryDelay: typeof idxRaw.initialRetryDelay === "number" ? idxRaw.initialRetryDelay : 2_000,
|
|
328
|
+
maxRetryDelay: typeof idxRaw.maxRetryDelay === "number" ? idxRaw.maxRetryDelay : 30_000,
|
|
329
|
+
maxChunkChars: typeof idxRaw.maxChunkChars === "number" ? Math.max(0, idxRaw.maxChunkChars) : 500,
|
|
330
|
+
overlapChars: typeof idxRaw.overlapChars === "number" ? Math.max(0, idxRaw.overlapChars) : 100,
|
|
331
|
+
};
|
|
332
|
+
|
|
333
|
+
return { servers, embedding, vectorDb, search, indexer };
|
|
303
334
|
}
|
package/src/constants.ts
ADDED