@andrhamm/web-mcp 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/LICENSE +21 -0
- package/README.md +163 -2
- package/dist/src/brave.js +100 -0
- package/dist/src/budget.js +95 -0
- package/dist/src/cache.js +70 -0
- package/dist/src/claude.js +207 -0
- package/dist/src/contract.js +79 -0
- package/dist/src/index.js +62 -0
- package/dist/src/limit.js +41 -0
- package/dist/src/reader.js +187 -0
- package/dist/src/render.js +48 -0
- package/dist/src/research.js +169 -0
- package/dist/src/server.js +72 -0
- package/package.json +64 -4
- package/prompts/page.md +19 -0
- package/prompts/research.md +34 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0 (2026-10-31)
|
|
4
|
+
|
|
5
|
+
- Six tools in three tiers: `web_lookup` and `web_read` (basic), `web_research` and `web_page` (standard), `web_investigate` and `web_search` (advanced).
|
|
6
|
+
- Research runs as one headless Claude Haiku session with WebSearch and WebFetch, with an optional Brave Search head start.
|
|
7
|
+
- Replies are compact markdown sized to `max_tokens`, enforced by per-request schema length limits and a trimming safety net, with an optional `format: "json"`.
|
|
8
|
+
- Flat-file cache for answers and fetched pages; Brave results are held in memory only.
|
|
9
|
+
- stdio and Streamable HTTP transports.
|
|
10
|
+
- Bounded wait queue (`WEB_MCP_QUEUE`, error `busy`); each call's time budget includes waiting.
|
|
11
|
+
- Cancelling a call, or closing its connection, stops its Claude run (error `cancelled`).
|
|
12
|
+
- Brave queries are cut to Brave's 75-word limit; a rejected query no longer switches Brave off.
|
|
13
|
+
- A timeout while a page body downloads reports `timeout`; other body read failures report `fetch_failed`.
|
|
14
|
+
- Pages are decoded with the charset from the Content-Type header or a `<meta>` tag, falling back to UTF-8.
|
|
15
|
+
- A stopped Claude run that ignores SIGTERM is killed after 3 seconds.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Andrew Hammond
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,164 @@
|
|
|
1
|
-
#
|
|
1
|
+
# web-mcp
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A local MCP server that answers web research questions for agents and returns the smallest useful reply: a direct answer, short verbatim quotes, numbered sources and what it could not find, never longer than the token cap the caller sets.
|
|
4
|
+
|
|
5
|
+
It exists so that small local models (and agents in a network sandbox) can use the web without spending their context on search result pages. A Claude Haiku session does the searching, reading and filtering on your own Claude subscription, through the Claude Code command line you are already signed in to. No API keys are needed. A Brave Search API key is optional and gives each research run a head start.
|
|
6
|
+
|
|
7
|
+
## Requirements
|
|
8
|
+
|
|
9
|
+
- Node.js 22 or newer.
|
|
10
|
+
- [Claude Code](https://code.claude.com/docs) installed and signed in with a Claude subscription. The server runs the unmodified `claude` command; it never reads or stores your credentials.
|
|
11
|
+
- Optional: a [Brave Search API](https://brave.com/search/api/) key, in `BRAVE_API_KEY` or in a file named by `BRAVE_API_KEY_FILE`.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
Claude Code:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
claude mcp add --transport stdio web -- npx -y @andrhamm/web-mcp
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
pi coding agent:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pi mcp add web -- npx -y @andrhamm/web-mcp
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
LM Studio and other clients that use an `mcp.json` file:
|
|
28
|
+
|
|
29
|
+
```json
|
|
30
|
+
{
|
|
31
|
+
"mcpServers": {
|
|
32
|
+
"web": { "command": "npx", "args": ["-y", "@andrhamm/web-mcp"], "env": { "BRAVE_API_KEY": "<your key>" } }
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
The server reads `BRAVE_API_KEY` from the environment its client starts it with. A desktop app started outside a shell may not see variables your shell sets, so put the key in the client's `env` block or point `BRAVE_API_KEY_FILE` at a file that holds it.
|
|
38
|
+
|
|
39
|
+
### Sandboxed agents (HTTP)
|
|
40
|
+
|
|
41
|
+
An agent in a sandbox has no Claude sign-in, so run the server on the host and let the sandbox reach it over HTTP:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
npx -y @andrhamm/web-mcp --http 127.0.0.1:7343
|
|
45
|
+
claude mcp add --transport http web http://127.0.0.1:7343/mcp
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
On loopback without a token, the server accepts only localhost `Host` and `Origin` headers. To listen on any other address you must set `WEB_MCP_HTTP_TOKEN`; clients then send `Authorization: Bearer <token>`. One HTTP process also shares one concurrency limit across all its callers.
|
|
49
|
+
|
|
50
|
+
## Tools
|
|
51
|
+
|
|
52
|
+
Tools come in three tiers, so each agent role can be given only the tools it can use well. Allow-list them by name in your client, or start the server with `--tiers basic` (or `basic,standard`) to hide the rest.
|
|
53
|
+
|
|
54
|
+
| Tier | Tool | Inputs | Use |
|
|
55
|
+
|---|---|---|---|
|
|
56
|
+
| basic | `web_lookup` | `question`, `max_tokens` (optional, default 300) | Answer a question from the web. |
|
|
57
|
+
| basic | `web_read` | `url`, `question` (optional), `max_tokens` (optional, default 300) | Say briefly what one page says. |
|
|
58
|
+
| standard | `web_research` | `question`, `goal`, `max_tokens` (default 400), `format` | Research for a stated goal; full reply with findings and quotes. |
|
|
59
|
+
| standard | `web_page` | `url`, `goal`, `max_tokens`, `format` | Read one known page for a goal; quotes checked against the page. |
|
|
60
|
+
| advanced | `web_investigate` | `web_research` inputs plus `depth`, `recency`, `domains`, `cache` | Research with control over effort, freshness and sources. |
|
|
61
|
+
| advanced | `web_search` | `query`, `count`, `recency`, `domains` | Raw Brave results: titles, URLs, snippets, dates. Listed only when a Brave key is set. |
|
|
62
|
+
|
|
63
|
+
`goal` is what the caller will do with the answer. The server keeps only what serves it, so a good goal makes a short reply. It is sent to Claude and to the web, so never put secrets in it.
|
|
64
|
+
|
|
65
|
+
`max_tokens` (50 to 4000) is a ceiling, not a target. Haiku is asked for the shortest reply that serves the goal. If it still runs over, the server drops the least important findings first, then quotes, then gaps, then trailing sentences, and marks the reply `trimmed yes`. Tokens are estimated with an o200k count times 1.3, which tracks Claude's tokenizer on prose and runs low on code.
|
|
66
|
+
|
|
67
|
+
`depth` sets the effort: `quick` allows 2 searches and 3 page reads, `standard` 4 and 6, `deep` 8 and 12. Each call has a time budget (90, 180 or 300 seconds; 90 for page tools) counted from when it arrives, so time spent waiting for a free slot uses it up too.
|
|
68
|
+
|
|
69
|
+
## Replies
|
|
70
|
+
|
|
71
|
+
Replies are compact markdown, because for this kind of content it costs fewer tokens than JSON and every model reads it well:
|
|
72
|
+
|
|
73
|
+
```
|
|
74
|
+
Answer: Send the key in the `X-Subscription-Token` header [1].
|
|
75
|
+
Confidence: high
|
|
76
|
+
|
|
77
|
+
Findings
|
|
78
|
+
- Every request must include the API key in this header. [1] "All requests to the Brave Search API must include an API key in the X-Subscription-Token HTTP header."
|
|
79
|
+
|
|
80
|
+
Sources
|
|
81
|
+
[1] Authentication. https://api-dashboard.search.brave.com/documentation/guides/authentication
|
|
82
|
+
|
|
83
|
+
Gaps
|
|
84
|
+
- Rate limits for the free credit were not checked.
|
|
85
|
+
|
|
86
|
+
meta: claude-haiku-5-5, searches 0, pages 0, brave, verified 1/1, cached no, trimmed no, 7.5 s
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The basic tools return only the answer, the sources and a `Not found:` line. Standard and advanced tools accept `format: "json"` for code that wants to parse the reply; the object matches `src/contract.ts`, with `findings[].source` as a 0-based index into `sources` and `findings[].verified: true` on quotes the server checked. The server does not send MCP structured content, because some clients show that to the model instead of the text.
|
|
90
|
+
|
|
91
|
+
`verified n/m` means n of the m quotes in the reply were checked word for word against page text the server holds: the page itself for `web_read` and `web_page`, and the Brave snippets for research runs. A quote that fails the check is removed and its claim kept, so the other m minus n quotes were not checked: they come from pages Haiku read through WebFetch, whose text the server never sees.
|
|
92
|
+
|
|
93
|
+
Errors come back as tool errors with a code first: `claude_not_signed_in`, `no_backend`, `rate_limited`, `busy`, `timeout`, `cancelled`, `fetch_failed`, `blocked_url`, `bad_output`.
|
|
94
|
+
|
|
95
|
+
## How it works
|
|
96
|
+
|
|
97
|
+
Each research call is one headless `claude -p` run on Haiku with only the WebSearch and WebFetch tools. Haiku splits the question into the parts the goal needs, searches, reads the most promising pages, checks after each read whether every part is backed by a quote, and stops when it is, or when the depth caps are reached. It then fills a JSON structure whose schema carries length limits sized to `max_tokens` (answer length, number of findings, quote and gap length). Claude Code rejects output that breaks them and Haiku rewrites it, which keeps most replies inside the cap without trimming. The server then renders, verifies, trims if still needed, and caches.
|
|
98
|
+
|
|
99
|
+
With a Brave key, the server first runs one Brave web search and gives Haiku the results and snippets as a head start, so simple questions often need no further searching. Each uncached research call spends one Brave request, and each `web_search` call spends one. Page tools spend none. Brave accepts at most 75 words per query, so long questions are cut to fit. If Brave rejects a query, only that call loses its head start; only a key or plan error switches Brave off until restart.
|
|
100
|
+
|
|
101
|
+
Page tools fetch the page directly (public addresses only, 2 MB, 20 seconds), extract the main text with Mozilla Readability, convert it to compact markdown, keep the passages that match the goal, and give those to Haiku with no tools.
|
|
102
|
+
|
|
103
|
+
The Claude run is isolated from your own Claude Code setup: no user or project settings, hooks, plugins, MCP servers or slash commands, an empty working directory, and no saved session. `ANTHROPIC_API_KEY` is removed from its environment so it always uses the subscription and never bills the API.
|
|
104
|
+
|
|
105
|
+
## Cancellation
|
|
106
|
+
|
|
107
|
+
When a caller cancels a call, or drops its connection, the server removes it from the wait queue or stops its Claude run at once, so no subscription usage is spent on an answer nobody will read. Over stdio this works for any client. Over HTTP it works when the client closes the request's connection, as clients on the 2026-07-28 protocol do. Clients on the older 2025 protocol cancel with a separate message that a stateless HTTP server cannot match to the call, so their abandoned runs stop at the time budget instead.
|
|
108
|
+
|
|
109
|
+
## Cache
|
|
110
|
+
|
|
111
|
+
Answers and fetched pages are cached as small JSON files in `WEB_MCP_CACHE_DIR`, or the platform cache directory under `web-mcp` (`~/Library/Caches` on macOS, `XDG_CACHE_HOME` or `~/.cache` on Linux, `LOCALAPPDATA` on Windows).
|
|
112
|
+
|
|
113
|
+
| Entry | Lifetime |
|
|
114
|
+
|---|---|
|
|
115
|
+
| Research answer, recency `day` | 1 hour |
|
|
116
|
+
| `week` | 6 hours |
|
|
117
|
+
| `month` | 1 day |
|
|
118
|
+
| `year` | 3 days |
|
|
119
|
+
| `any` | 7 days |
|
|
120
|
+
| Page text and page answers | 1 day |
|
|
121
|
+
| Brave results | 10 minutes, in memory only |
|
|
122
|
+
|
|
123
|
+
Brave's API terms allow only transient storage of search results, so they are never written to disk. `cache: "refresh"` on `web_investigate` ignores cached answers and pages.
|
|
124
|
+
|
|
125
|
+
## Configuration
|
|
126
|
+
|
|
127
|
+
| Variable | Default | Meaning |
|
|
128
|
+
|---|---|---|
|
|
129
|
+
| `BRAVE_API_KEY`, `BRAVE_API_KEY_FILE` | unset | Brave key, or a file holding it. Enables the head start and `web_search`. |
|
|
130
|
+
| `CLAUDE_BIN` | `claude` | Path to the Claude Code command. |
|
|
131
|
+
| `WEB_MCP_MODEL` | `haiku` | Model alias or id passed to `claude --model`. |
|
|
132
|
+
| `WEB_MCP_CONCURRENCY` | 2 | Claude runs at once; the rest wait in order. |
|
|
133
|
+
| `WEB_MCP_QUEUE` | 8 | Calls allowed to wait for a slot; beyond that a call fails at once with `busy`. |
|
|
134
|
+
| `WEB_MCP_CACHE_DIR` | platform cache dir | Where cached answers and pages live. |
|
|
135
|
+
| `WEB_MCP_HTTP_TOKEN` | unset | Bearer token for `--http`; required off loopback. |
|
|
136
|
+
|
|
137
|
+
Logs go to stderr only.
|
|
138
|
+
|
|
139
|
+
## Use of your Claude subscription
|
|
140
|
+
|
|
141
|
+
Every uncached call runs Claude Code under your own sign-in and counts toward your plan's usage limits. Run this server for your own agents. Do not offer it to other people as a shared Claude backend: Anthropic's terms allow ordinary individual use of Claude Code with your own subscription, not reselling access to it.
|
|
142
|
+
|
|
143
|
+
## Limits
|
|
144
|
+
|
|
145
|
+
- Pages that render their content with JavaScript come back thin; there is no headless browser.
|
|
146
|
+
- Page tools do not consult `robots.txt`.
|
|
147
|
+
- `recency` is exact for the Brave head start and best effort for Claude's own searches.
|
|
148
|
+
- The token estimate is approximate; set `max_tokens` with some margin if a hard limit matters.
|
|
149
|
+
|
|
150
|
+
## Development
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
npm install
|
|
154
|
+
npm test
|
|
155
|
+
npm run lint
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Tests run against a fake `claude` command and need no network or sign-in. To try a local build in Claude Code: `claude mcp add --transport stdio web -- node <clone>/dist/src/index.js`.
|
|
159
|
+
|
|
160
|
+
Releasing: bump `version` in `package.json`, add the changes to `CHANGELOG.md`, commit, then push a matching tag (`git tag v0.2.0 && git push origin v0.2.0`). The publish workflow tests, builds and publishes to npm with provenance.
|
|
161
|
+
|
|
162
|
+
## License
|
|
163
|
+
|
|
164
|
+
MIT
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// Brave Search web results: the head start for research runs and the raw `web_search` tool.
|
|
2
|
+
// Brave's API terms allow only transient storage of results, so they are cached in memory for minutes and never on disk.
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { MemoryCache } from "./cache.js";
|
|
5
|
+
import { ToolError } from "./contract.js";
|
|
6
|
+
export function braveKey(env = process.env) {
|
|
7
|
+
if (env.BRAVE_API_KEY?.trim())
|
|
8
|
+
return env.BRAVE_API_KEY.trim();
|
|
9
|
+
if (env.BRAVE_API_KEY_FILE) {
|
|
10
|
+
try {
|
|
11
|
+
return readFileSync(env.BRAVE_API_KEY_FILE, "utf8").trim() || undefined;
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
console.error("[web-mcp] BRAVE_API_KEY_FILE is set but could not be read; Brave is off");
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
const FRESHNESS = { day: "pd", week: "pw", month: "pm", year: "py" };
|
|
20
|
+
const memo = new MemoryCache(10 * 60_000);
|
|
21
|
+
let disabledReason;
|
|
22
|
+
export const braveAvailable = () => !disabledReason && !!braveKey();
|
|
23
|
+
// Brave answers 422 VALIDATION above 75 words ("Search query must be at most 75 words", checked 2026-10-23).
|
|
24
|
+
const MAX_WORDS = 75;
|
|
25
|
+
/** The question plus site filters, with the question cut so the whole query stays within Brave's word limit. */
|
|
26
|
+
export function braveQuery(query, domains) {
|
|
27
|
+
const inc = domains?.include ?? [];
|
|
28
|
+
const exc = domains?.exclude ?? [];
|
|
29
|
+
const filters = [];
|
|
30
|
+
if (inc.length)
|
|
31
|
+
filters.push(inc.length === 1 ? `site:${inc[0]}` : `(${inc.map((d) => `site:${d}`).join(" OR ")})`);
|
|
32
|
+
for (const d of exc)
|
|
33
|
+
filters.push(`-site:${d}`);
|
|
34
|
+
const filterWords = filters.join(" ").split(/\s+/).filter(Boolean).length;
|
|
35
|
+
const words = query
|
|
36
|
+
.trim()
|
|
37
|
+
.split(/\s+/)
|
|
38
|
+
.slice(0, Math.max(1, MAX_WORDS - filterWords));
|
|
39
|
+
return [words.join(" "), ...filters].join(" ");
|
|
40
|
+
}
|
|
41
|
+
const clean = (s) => s
|
|
42
|
+
.replace(/<[^>]+>/g, "")
|
|
43
|
+
.replace(/'|'/g, "'")
|
|
44
|
+
.replace(/"/g, '"')
|
|
45
|
+
.replace(/&/g, "&")
|
|
46
|
+
.replace(/</g, "<")
|
|
47
|
+
.replace(/>/g, ">")
|
|
48
|
+
.trim();
|
|
49
|
+
export async function braveSearch(query, opts) {
|
|
50
|
+
const key = braveKey();
|
|
51
|
+
if (!key || disabledReason)
|
|
52
|
+
throw new ToolError("no_backend", disabledReason ?? "No Brave API key is configured", "Set BRAVE_API_KEY or BRAVE_API_KEY_FILE");
|
|
53
|
+
const params = new URLSearchParams({ q: braveQuery(query, opts.domains), count: String(opts.count), extra_snippets: "true" });
|
|
54
|
+
const fresh = opts.recency && FRESHNESS[opts.recency];
|
|
55
|
+
if (fresh)
|
|
56
|
+
params.set("freshness", fresh);
|
|
57
|
+
const url = `https://api.search.brave.com/res/v1/web/search?${params}`;
|
|
58
|
+
const hit = memo.get(url);
|
|
59
|
+
if (hit)
|
|
60
|
+
return hit;
|
|
61
|
+
const res = await fetch(url, {
|
|
62
|
+
headers: { accept: "application/json", "x-subscription-token": key },
|
|
63
|
+
signal: AbortSignal.timeout(15_000),
|
|
64
|
+
}).catch((e) => {
|
|
65
|
+
throw new ToolError("fetch_failed", `Brave search failed: ${e.message}`);
|
|
66
|
+
});
|
|
67
|
+
if (res.status === 401 || res.status === 403 || res.status === 422) {
|
|
68
|
+
const err = (await res.json().catch(() => ({})));
|
|
69
|
+
const code = err.error?.code ?? String(res.status);
|
|
70
|
+
// Only a key or subscription problem turns Brave off; a 422 is also how Brave rejects a bad query (code VALIDATION).
|
|
71
|
+
if (res.status !== 422 || /TOKEN|SUBSCRIPTION|AUTH/i.test(code)) {
|
|
72
|
+
disabledReason = `Brave rejected the API key (${code})`;
|
|
73
|
+
console.error(`[web-mcp] ${disabledReason}; Brave is off until restart`);
|
|
74
|
+
throw new ToolError("no_backend", disabledReason, "Check the Brave plan and key");
|
|
75
|
+
}
|
|
76
|
+
const why = err.error?.meta?.errors?.[0]?.msg ?? err.error?.detail ?? code;
|
|
77
|
+
throw new ToolError("fetch_failed", `Brave rejected the query: ${why}`, "Shorten the query or use fewer domain filters");
|
|
78
|
+
}
|
|
79
|
+
if (res.status === 429)
|
|
80
|
+
throw new ToolError("rate_limited", "Brave rate limit or credit exhausted", "Wait a moment, or check the Brave plan's credit");
|
|
81
|
+
if (!res.ok)
|
|
82
|
+
throw new ToolError("fetch_failed", `Brave answered HTTP ${res.status}`);
|
|
83
|
+
const body = (await res.json());
|
|
84
|
+
const hits = (body?.web?.results ?? []).map((r) => ({
|
|
85
|
+
url: String(r.url),
|
|
86
|
+
title: clean(String(r.title ?? r.url)),
|
|
87
|
+
snippets: [r.description, ...(r.extra_snippets ?? [])].filter((s) => typeof s === "string" && s !== "").map(clean),
|
|
88
|
+
...(typeof r.page_age === "string" ? { published: r.page_age.slice(0, 10) } : {}),
|
|
89
|
+
}));
|
|
90
|
+
memo.set(url, hits);
|
|
91
|
+
return hits;
|
|
92
|
+
}
|
|
93
|
+
export function renderHits(hits, maxSnippets = 3) {
|
|
94
|
+
return hits
|
|
95
|
+
.map((h, i) => [
|
|
96
|
+
`[${i + 1}] ${h.title}. ${h.url}${h.published ? ` (${h.published})` : ""}`,
|
|
97
|
+
...h.snippets.slice(0, maxSnippets).map((s) => ` ${s}`),
|
|
98
|
+
].join("\n"))
|
|
99
|
+
.join("\n");
|
|
100
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { countTokens } from "gpt-tokenizer";
|
|
2
|
+
import { renderBody } from "./render.js";
|
|
3
|
+
// shortcut: o200k count times 1.3 approximates Claude's tokenizer on English prose; low on code-heavy text. Recalibrate if callers report overruns.
|
|
4
|
+
const CLAUDE_FACTOR = 1.3;
|
|
5
|
+
export const estimateTokens = (text) => Math.ceil(countTokens(text) * CLAUDE_FACTOR);
|
|
6
|
+
/** Words to ask the model for, leaving room for markup. */
|
|
7
|
+
// Measured 2026-10-22: Haiku lands 15 to 40 percent over the words it is given, and labels plus sources take about 60 tokens.
|
|
8
|
+
export const wordTarget = (maxTokens) => Math.max(20, Math.floor(maxTokens * 0.42));
|
|
9
|
+
/** Findings that fit beside the answer: about one per 150 tokens of cap. */
|
|
10
|
+
export const findingsTarget = (maxTokens) => Math.min(8, Math.floor(maxTokens / 150));
|
|
11
|
+
/** Hard limits for the schema: an answer of about 1.35 characters per token of cap, and the findings target. */
|
|
12
|
+
export const limits = (maxTokens) => ({
|
|
13
|
+
answerChars: Math.max(160, Math.round(maxTokens * 1.35)),
|
|
14
|
+
findings: findingsTarget(maxTokens),
|
|
15
|
+
});
|
|
16
|
+
export const capLine = (maxTokens) => `Output cap: max_tokens ${maxTokens}: about ${wordTarget(maxTokens)} words in total at most, counting answer, findings, quotes and source titles. ` +
|
|
17
|
+
`The answer must fit in ${limits(maxTokens).answerChars} characters. ` +
|
|
18
|
+
`At most ${findingsTarget(maxTokens)} findings${findingsTarget(maxTokens) === 0 ? " (put the facts in the answer)" : ""}; none that repeat the answer without adding a quote.`;
|
|
19
|
+
/** Drop a trailing site name ("Title | Site", "Title - Site") and cap the length: titles repeat on every reply. */
|
|
20
|
+
export function shortTitle(t) {
|
|
21
|
+
const cut = t.replace(/\s+[|\u2013\u2014-]\s+[^|\u2013\u2014-]{1,40}$/, "").trim();
|
|
22
|
+
const s = cut.length >= 8 ? cut : t.trim();
|
|
23
|
+
return s.length > 80 ? `${s.slice(0, 77).trimEnd()}...` : s;
|
|
24
|
+
}
|
|
25
|
+
/** Dedupe sources by URL, drop unreferenced ones, renumber findings and [n] markers. */
|
|
26
|
+
export function compact(r) {
|
|
27
|
+
const byUrl = new Map(); // url -> new 1-based number
|
|
28
|
+
const sources = [];
|
|
29
|
+
const oldToNew = new Map();
|
|
30
|
+
const used = new Set(r.findings.map((f) => f.source));
|
|
31
|
+
for (const m of r.answer.matchAll(/\[(\d+)\]/g))
|
|
32
|
+
used.add(Number(m[1]));
|
|
33
|
+
r.sources.forEach((s, i) => {
|
|
34
|
+
if (!used.has(i + 1))
|
|
35
|
+
return;
|
|
36
|
+
const key = s.url.replace(/\/$/, "");
|
|
37
|
+
let n = byUrl.get(key);
|
|
38
|
+
if (n === undefined) {
|
|
39
|
+
sources.push({ ...s, title: shortTitle(s.title) });
|
|
40
|
+
n = sources.length;
|
|
41
|
+
byUrl.set(key, n);
|
|
42
|
+
}
|
|
43
|
+
oldToNew.set(i + 1, n);
|
|
44
|
+
});
|
|
45
|
+
const findings = r.findings.filter((f) => oldToNew.has(f.source)).map((f) => ({ ...f, source: oldToNew.get(f.source) ?? f.source }));
|
|
46
|
+
const answer = r.answer.replace(/\[(\d+)\]/g, (_all, d) => (oldToNew.has(Number(d)) ? `[${oldToNew.get(Number(d))}]` : ""));
|
|
47
|
+
return { ...r, answer, findings, sources };
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Safety net after the model: drop the least important findings whole (the rest keep their quotes),
|
|
51
|
+
* then the last quote, then gaps, then cut the answer at sentence ends, until the body fits.
|
|
52
|
+
*/
|
|
53
|
+
export function trim(r, maxTokens, format) {
|
|
54
|
+
const fits = (x) => estimateTokens(renderBody(x, format)) <= maxTokens;
|
|
55
|
+
if (fits(r))
|
|
56
|
+
return r;
|
|
57
|
+
let x = { ...r, findings: [...r.findings], meta: { ...r.meta, trimmed: true } };
|
|
58
|
+
while (x.findings.length > 1 && !fits(x))
|
|
59
|
+
x = compact({ ...x, findings: x.findings.slice(0, -1) });
|
|
60
|
+
if (!fits(x) && x.findings[0]?.quote)
|
|
61
|
+
x = { ...x, findings: [{ ...x.findings[0], quote: undefined }] };
|
|
62
|
+
if (!fits(x))
|
|
63
|
+
x = compact({ ...x, findings: [] });
|
|
64
|
+
if (!fits(x))
|
|
65
|
+
x = { ...x, gaps: [] };
|
|
66
|
+
while (!fits(x)) {
|
|
67
|
+
const sentences = x.answer.match(/[^.!?]+[.!?]+\s*/g) ?? [];
|
|
68
|
+
const answer = sentences.length > 1 ? sentences.slice(0, -1).join("").trim() : `${x.answer.slice(0, Math.floor(x.answer.length * 0.8)).trim()}...`;
|
|
69
|
+
if (answer.length < 10)
|
|
70
|
+
break;
|
|
71
|
+
x = compact({ ...x, answer });
|
|
72
|
+
}
|
|
73
|
+
return x;
|
|
74
|
+
}
|
|
75
|
+
// Letters and digits only: quotes survive markdown, smart quotes and spacing differences between the page and the model.
|
|
76
|
+
const norm = (s) => s
|
|
77
|
+
.toLowerCase()
|
|
78
|
+
.replace(/[^\p{L}\p{N}]+/gu, " ")
|
|
79
|
+
.trim();
|
|
80
|
+
/**
|
|
81
|
+
* Check quotes against text the server holds, keyed by URL. Quotes from sources not in `held` are left alone.
|
|
82
|
+
* A quote found there marks its finding `verified`; one not found is dropped and its claim kept.
|
|
83
|
+
*/
|
|
84
|
+
export function verifyQuotes(r, held) {
|
|
85
|
+
const normHeld = new Map([...held].map(([u, t]) => [u.replace(/\/$/, ""), norm(t)]));
|
|
86
|
+
const findings = r.findings.map((f) => {
|
|
87
|
+
if (!f.quote)
|
|
88
|
+
return f;
|
|
89
|
+
const text = normHeld.get(r.sources[f.source - 1]?.url.replace(/\/$/, "") ?? "");
|
|
90
|
+
if (text === undefined)
|
|
91
|
+
return f;
|
|
92
|
+
return text.includes(norm(f.quote)) ? { ...f, verified: true } : { ...f, quote: undefined };
|
|
93
|
+
});
|
|
94
|
+
return { ...r, findings };
|
|
95
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
3
|
+
import { homedir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
export function cacheDir(env = process.env, platform = process.platform) {
|
|
6
|
+
if (env.WEB_MCP_CACHE_DIR)
|
|
7
|
+
return env.WEB_MCP_CACHE_DIR;
|
|
8
|
+
const base = platform === "darwin"
|
|
9
|
+
? join(homedir(), "Library", "Caches")
|
|
10
|
+
: platform === "win32"
|
|
11
|
+
? (env.LOCALAPPDATA ?? join(homedir(), "AppData", "Local"))
|
|
12
|
+
: (env.XDG_CACHE_HOME ?? join(homedir(), ".cache"));
|
|
13
|
+
return join(base, "web-mcp");
|
|
14
|
+
}
|
|
15
|
+
export const HOUR = 3_600_000;
|
|
16
|
+
/**
|
|
17
|
+
* Flat-file TTL cache: one `sha256(key).json` per entry holding `{expires, body}`.
|
|
18
|
+
* shortcut: expired files are ignored, never swept; add a sweep if the directory grows past a few thousand files.
|
|
19
|
+
*/
|
|
20
|
+
export class FileCache {
|
|
21
|
+
dir;
|
|
22
|
+
constructor(dir = cacheDir()) {
|
|
23
|
+
this.dir = dir;
|
|
24
|
+
}
|
|
25
|
+
path(key) {
|
|
26
|
+
return join(this.dir, `${createHash("sha256").update(key).digest("hex")}.json`);
|
|
27
|
+
}
|
|
28
|
+
async get(key, now = Date.now()) {
|
|
29
|
+
try {
|
|
30
|
+
const { expires, body } = JSON.parse(await readFile(this.path(key), "utf8"));
|
|
31
|
+
return expires > now ? body : undefined;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return undefined;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
async set(key, body, ttlMs, now = Date.now()) {
|
|
38
|
+
try {
|
|
39
|
+
await mkdir(this.dir, { recursive: true });
|
|
40
|
+
const p = this.path(key);
|
|
41
|
+
const tmp = `${p}.${process.pid}.tmp`;
|
|
42
|
+
await writeFile(tmp, JSON.stringify({ expires: now + ttlMs, body }));
|
|
43
|
+
await rename(tmp, p);
|
|
44
|
+
}
|
|
45
|
+
catch (e) {
|
|
46
|
+
console.error(`[web-mcp] cache write failed: ${e.message}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** In-memory TTL map for data that must not touch disk (Brave results: its terms allow transient storage only). */
|
|
51
|
+
export class MemoryCache {
|
|
52
|
+
ttlMs;
|
|
53
|
+
m = new Map();
|
|
54
|
+
constructor(ttlMs) {
|
|
55
|
+
this.ttlMs = ttlMs;
|
|
56
|
+
}
|
|
57
|
+
get(key, now = Date.now()) {
|
|
58
|
+
const e = this.m.get(key);
|
|
59
|
+
if (e && e.expires > now)
|
|
60
|
+
return e.value;
|
|
61
|
+
this.m.delete(key);
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
set(key, value, now = Date.now()) {
|
|
65
|
+
if (this.m.size > 500)
|
|
66
|
+
this.m.clear(); // bounded; entries live minutes anyway
|
|
67
|
+
this.m.set(key, { expires: now + this.ttlMs, value });
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
export const normKey = (s) => s.toLowerCase().replace(/\s+/g, " ").trim();
|