@juicesharp/rpiv-web-tools 1.12.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -11
- package/index.ts +1 -1
- package/package.json +13 -4
- package/providers/brave.ts +1 -1
- package/providers/exa.ts +2 -2
- package/providers/factory.ts +3 -0
- package/providers/firecrawl.ts +2 -2
- package/providers/index.ts +10 -0
- package/providers/jina.ts +2 -2
- package/providers/ollama.ts +274 -0
- package/providers/searxng.ts +2 -4
- package/providers/serper.ts +1 -1
- package/providers/tavily.ts +2 -2
- package/providers/types.ts +1 -1
- package/web-tools.ts +7 -7
package/README.md
CHANGED
|
@@ -8,18 +8,31 @@
|
|
|
8
8
|
</a>
|
|
9
9
|
</div>
|
|
10
10
|
|
|
11
|
-
Let the model search the web and read pages. `rpiv-web-tools` adds `web_search` and `web_fetch` tools to [Pi Agent](https://github.com/badlogic/pi-mono) with pluggable providers (Brave, Tavily, Serper, Exa, Jina, Firecrawl, [SearXNG](https://docs.searxng.org/)), plus `/web-
|
|
11
|
+
Let the model search the web and read pages. `rpiv-web-tools` adds `web_search` and `web_fetch` tools to [Pi Agent](https://github.com/badlogic/pi-mono) with pluggable providers (Brave, Tavily, Serper, Exa, Jina, Firecrawl, [SearXNG](https://docs.searxng.org/), [Ollama](https://ollama.com)), plus `/web-tools` for interactive provider selection and API-key setup.
|
|
12
12
|
|
|
13
13
|

|
|
14
14
|
|
|
15
|
+
## Providers
|
|
16
|
+
|
|
17
|
+
Pick one as the active backend; switch any time without losing the others' keys.
|
|
18
|
+
|
|
19
|
+
| Provider | Env var | Signup | Fetch mode | Notes |
|
|
20
|
+
|---|---|---|---|---|
|
|
21
|
+
| Brave | `BRAVE_SEARCH_API_KEY` | [brave.com/search/api](https://brave.com/search/api/) | raw HTTP → htmlToText, `raw: true` available | default |
|
|
22
|
+
| Tavily | `TAVILY_API_KEY` | [tavily.com](https://tavily.com) | native extraction (plain text) | |
|
|
23
|
+
| Serper | `SERPER_API_KEY` | [serper.dev](https://serper.dev) | raw HTTP → htmlToText, `raw: true` available | |
|
|
24
|
+
| Exa | `EXA_API_KEY` | [exa.ai](https://exa.ai) | native extraction (plain text) | |
|
|
25
|
+
| Jina | `JINA_API_KEY` | [jina.ai/reader](https://jina.ai/reader) | native extraction (markdown) | |
|
|
26
|
+
| Firecrawl | `FIRECRAWL_API_KEY` | [firecrawl.dev](https://firecrawl.dev) | native extraction (markdown) | |
|
|
27
|
+
| SearXNG | `SEARXNG_URL` (+ optional `SEARXNG_API_KEY`) | self-hosted | raw HTTP → htmlToText, `raw: true` available | see [§SearXNG](#searxng-self-hosted) |
|
|
28
|
+
| Ollama | `OLLAMA_HOST` / `OLLAMA_API_KEY` | local or [ollama.com](https://ollama.com) | native extraction | see [§Ollama](#ollama-local-or-cloud) |
|
|
29
|
+
|
|
15
30
|
## Features
|
|
16
31
|
|
|
17
|
-
- **
|
|
18
|
-
- **Per-provider fetch strategy** - Brave, Serper, and SearXNG read the URL directly and strip HTML to text; Tavily/Exa/Jina/Firecrawl use their native extraction endpoints (markdown for Jina/Firecrawl, plain text for Tavily/Exa).
|
|
19
|
-
- **Read any URL** - fetch http/https pages with HTML-to-text extraction, or get the raw response with `raw: true` (honoured by Brave/Serper; extraction providers always return their parsed text).
|
|
32
|
+
- **Read any URL** - fetch http/https pages with HTML-to-text extraction, or get the raw response with `raw: true` (honoured by Brave/Serper/SearXNG; extraction providers — Tavily/Exa/Jina/Firecrawl/Ollama — always return their parsed text).
|
|
20
33
|
- **Large-page spillover** - oversized responses truncate inline and spill the full body to a temp file the model can read on demand.
|
|
21
34
|
- **SSRF guard** - refuses loopback, RFC 1918, link-local, and cloud-metadata addresses (`localhost`, `127.0.0.0/8`, `10.0.0.0/8`, `169.254.0.0/16`, `172.16.0.0/12`, `192.168.0.0/16`, `::1`, `fc00::/7`, `fe80::/10`).
|
|
22
|
-
- **Interactive setup** - `/web-
|
|
35
|
+
- **Interactive setup** - `/web-tools` lists providers (active one first, configured ones marked) and writes to `~/.config/rpiv-web-tools/config.json` (chmod 0600); per-provider env vars also work and take precedence over persisted keys.
|
|
23
36
|
|
|
24
37
|
## Install
|
|
25
38
|
|
|
@@ -34,7 +47,7 @@ Then restart your Pi session.
|
|
|
34
47
|
- **`web_search`** - query the active provider's search API and return titled snippets.
|
|
35
48
|
1–10 results per call.
|
|
36
49
|
- **`web_fetch`** - fetch an http/https URL through the active provider's content path
|
|
37
|
-
(raw HTTP+htmlToText for Brave/Serper; native extraction for Tavily/Exa/Jina/Firecrawl),
|
|
50
|
+
(raw HTTP+htmlToText for Brave/Serper/SearXNG; native extraction for Tavily/Exa/Jina/Firecrawl/Ollama),
|
|
38
51
|
truncate large responses with a temp-file spill for the full content.
|
|
39
52
|
|
|
40
53
|
### Schema - `web_search`
|
|
@@ -53,7 +66,7 @@ Returns:
|
|
|
53
66
|
content: [{ type: "text", text: string }], // markdown list of "**title**\n url\n snippet"
|
|
54
67
|
details: {
|
|
55
68
|
query: string,
|
|
56
|
-
backend: "brave" | "tavily" | "serper" | "exa" | "jina" | "firecrawl" | "searxng",
|
|
69
|
+
backend: "brave" | "tavily" | "serper" | "exa" | "jina" | "firecrawl" | "searxng" | "ollama",
|
|
57
70
|
resultCount: number,
|
|
58
71
|
results?: Array<{ title: string, url: string, snippet: string }>,
|
|
59
72
|
}
|
|
@@ -91,7 +104,7 @@ Throws on invalid URL, non-http(s) protocol, private/loopback hostnames (SSRF gu
|
|
|
91
104
|
|
|
92
105
|
## Commands
|
|
93
106
|
|
|
94
|
-
- **`/web-
|
|
107
|
+
- **`/web-tools`** - pick the active provider and set its API key interactively.
|
|
95
108
|
Providers already configured show `(configured)`; the active one is listed first with a `✓`.
|
|
96
109
|
Pressing Enter on an empty input keeps the existing key for the chosen provider while
|
|
97
110
|
persisting the provider switch. Pass `--show` to see all per-provider keys (masked) and env var status.
|
|
@@ -100,11 +113,11 @@ Throws on invalid URL, non-http(s) protocol, private/loopback hostnames (SSRF gu
|
|
|
100
113
|
|
|
101
114
|
First match wins:
|
|
102
115
|
|
|
103
|
-
1. The active provider's environment variable: `BRAVE_SEARCH_API_KEY`, `TAVILY_API_KEY`, `SERPER_API_KEY`, `EXA_API_KEY`, `JINA_API_KEY`, `FIRECRAWL_API_KEY`, or `
|
|
116
|
+
1. The active provider's environment variable: `BRAVE_SEARCH_API_KEY`, `TAVILY_API_KEY`, `SERPER_API_KEY`, `EXA_API_KEY`, `JINA_API_KEY`, `FIRECRAWL_API_KEY`, `SEARXNG_API_KEY`, or `OLLAMA_API_KEY`
|
|
104
117
|
2. `apiKeys.<provider>` field in `~/.config/rpiv-web-tools/config.json`
|
|
105
118
|
3. Legacy `apiKey` field (Brave only — auto-migrated to the new shape on next save)
|
|
106
119
|
|
|
107
|
-
The active provider is `config.provider` (set by `/web-
|
|
120
|
+
The active provider is `config.provider` (set by `/web-tools`); falls back to `brave` if absent.
|
|
108
121
|
|
|
109
122
|
## SearXNG (self-hosted)
|
|
110
123
|
|
|
@@ -116,7 +129,7 @@ export SEARXNG_URL=http://localhost:8080
|
|
|
116
129
|
export SEARXNG_API_KEY=…
|
|
117
130
|
```
|
|
118
131
|
|
|
119
|
-
Resolution order for the URL: `SEARXNG_URL` env var → `baseUrls.searxng` in `~/.config/rpiv-web-tools/config.json` → default `http://localhost:8080`. `/web-
|
|
132
|
+
Resolution order for the URL: `SEARXNG_URL` env var → `baseUrls.searxng` in `~/.config/rpiv-web-tools/config.json` → default `http://localhost:8080`. `/web-tools` prompts for the URL first and the (optional) API key second.
|
|
120
133
|
|
|
121
134
|
Your instance must have `json` enabled in `settings.yml` under `search.formats` — default SearXNG installs ship with JSON disabled and will return `403 Forbidden` otherwise (per the [SearXNG search API docs](https://docs.searxng.org/dev/search_api.html)). The provider surfaces that case with an actionable hint. SearXNG's `web_fetch` reuses the same raw-HTTP + HTML-to-text pipeline as Brave/Serper, so URLs returned by `web_search` can be fetched without any extra setup.
|
|
122
135
|
|
|
@@ -143,6 +156,39 @@ curl -sf 'http://localhost:8080/search?q=hello&format=json' | jq '.results | len
|
|
|
143
156
|
|
|
144
157
|
`403` means JSON is still disabled — re-check `~/.searxng/settings.yml`. Works identically on Docker Desktop or OrbStack. For a throwaway test instance, swap `~/.searxng` for `/tmp/searxng` and drop `--restart unless-stopped`.
|
|
145
158
|
|
|
159
|
+
## Ollama (local or cloud)
|
|
160
|
+
|
|
161
|
+
Ollama provides web search and fetch as built-in capabilities — no third-party API key needed for local usage. For cloud access, an API key is required.
|
|
162
|
+
|
|
163
|
+
### Local Ollama
|
|
164
|
+
|
|
165
|
+
Just run Ollama locally and it works out of the box:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
ollama serve
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
No API key needed. The provider talks to `http://localhost:11434` by default.
|
|
172
|
+
|
|
173
|
+
### Ollama Cloud
|
|
174
|
+
|
|
175
|
+
For cloud access via [Ollama Cloud](https://ollama.com), set the base URL and API key:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
export OLLAMA_HOST=https://ollama.com
|
|
179
|
+
export OLLAMA_API_KEY=your_api_key # generate at https://ollama.com/settings/keys
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Or configure interactively via `/web-tools` — select "Ollama" and enter the URL and key.
|
|
183
|
+
|
|
184
|
+
Resolution order:
|
|
185
|
+
- **Base URL**: `OLLAMA_HOST` env var → `baseUrls.ollama` in config → default `http://localhost:11434`
|
|
186
|
+
- **API key**: `OLLAMA_API_KEY` env var → `apiKeys.ollama` in config (optional for local, required for cloud)
|
|
187
|
+
|
|
188
|
+
The provider automatically uses the correct API paths:
|
|
189
|
+
- **Local** (`localhost`, `127.0.0.1`, `0.0.0.0`): `/api/experimental/web_search` and `/api/experimental/web_fetch`
|
|
190
|
+
- **Cloud** (any other host): `/api/web_search` and `/api/web_fetch`
|
|
191
|
+
|
|
146
192
|
## Executor guidance overrides
|
|
147
193
|
|
|
148
194
|
Override the `promptSnippet` / `promptGuidelines` the model sees for each tool by editing `~/.config/rpiv-web-tools/config.json`. Note the per-tool nesting under `guidance.web_search` / `guidance.web_fetch` — this differs from the flat `guidance` shape used by single-tool siblings (`rpiv-advisor`, `rpiv-todo`, `rpiv-ask-user-question`):
|
package/index.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* rpiv-web-tools — Pi extension
|
|
3
3
|
*
|
|
4
4
|
* Registers the `web_search` and `web_fetch` tools, plus the
|
|
5
|
-
* `/web-
|
|
5
|
+
* `/web-tools` slash command. Body lives in `web-tools.ts`.
|
|
6
6
|
*
|
|
7
7
|
* Config persists at ~/.config/rpiv-web-tools/config.json. Per-provider env
|
|
8
8
|
* vars (e.g. BRAVE_SEARCH_API_KEY, TAVILY_API_KEY) win over the config file.
|
package/package.json
CHANGED
|
@@ -1,21 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juicesharp/rpiv-web-tools",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Pi extension. Web search and fetch for the model with pluggable providers (Brave, Tavily, Serper, Exa, Jina, Firecrawl).",
|
|
3
|
+
"version": "1.14.0",
|
|
4
|
+
"description": "Pi extension. Web search and fetch for the model with pluggable providers (Brave, Tavily, Serper, Exa, Jina, Firecrawl, SearXNG, Ollama).",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
7
7
|
"pi-extension",
|
|
8
8
|
"rpiv",
|
|
9
9
|
"web-search",
|
|
10
|
+
"web-fetch",
|
|
10
11
|
"search",
|
|
11
12
|
"fetch",
|
|
12
13
|
"scrape",
|
|
14
|
+
"web-scraping",
|
|
15
|
+
"ai",
|
|
16
|
+
"llm",
|
|
17
|
+
"agent",
|
|
18
|
+
"local",
|
|
19
|
+
"self-hosted",
|
|
13
20
|
"brave",
|
|
14
21
|
"tavily",
|
|
15
22
|
"serper",
|
|
16
23
|
"exa",
|
|
17
24
|
"jina",
|
|
18
|
-
"firecrawl"
|
|
25
|
+
"firecrawl",
|
|
26
|
+
"searxng",
|
|
27
|
+
"ollama"
|
|
19
28
|
],
|
|
20
29
|
"type": "module",
|
|
21
30
|
"license": "MIT",
|
|
@@ -48,7 +57,7 @@
|
|
|
48
57
|
]
|
|
49
58
|
},
|
|
50
59
|
"dependencies": {
|
|
51
|
-
"@juicesharp/rpiv-config": "^1.
|
|
60
|
+
"@juicesharp/rpiv-config": "^1.14.0"
|
|
52
61
|
},
|
|
53
62
|
"peerDependencies": {
|
|
54
63
|
"@earendil-works/pi-coding-agent": "*",
|
package/providers/brave.ts
CHANGED
|
@@ -30,7 +30,7 @@ export class BraveProvider implements SearchProvider {
|
|
|
30
30
|
|
|
31
31
|
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
32
32
|
if (!this.apiKey) {
|
|
33
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
33
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
const url = new URL(BRAVE_SEARCH_API_URL);
|
package/providers/exa.ts
CHANGED
|
@@ -39,7 +39,7 @@ export class ExaProvider implements SearchProvider {
|
|
|
39
39
|
|
|
40
40
|
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
41
41
|
if (!this.apiKey) {
|
|
42
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
42
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
const res = await fetch(EXA_API_URL, {
|
|
@@ -69,7 +69,7 @@ export class ExaProvider implements SearchProvider {
|
|
|
69
69
|
|
|
70
70
|
async fetch(url: string, _raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
|
|
71
71
|
if (!this.apiKey) {
|
|
72
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
72
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
73
73
|
}
|
|
74
74
|
|
|
75
75
|
const res = await fetch(EXA_CONTENTS_API_URL, {
|
package/providers/factory.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { BraveProvider } from "./brave.js";
|
|
|
2
2
|
import { ExaProvider } from "./exa.js";
|
|
3
3
|
import { FirecrawlProvider } from "./firecrawl.js";
|
|
4
4
|
import { JinaProvider } from "./jina.js";
|
|
5
|
+
import { OllamaProvider } from "./ollama.js";
|
|
5
6
|
import { SearxngProvider } from "./searxng.js";
|
|
6
7
|
import { SerperProvider } from "./serper.js";
|
|
7
8
|
import { TavilyProvider } from "./tavily.js";
|
|
@@ -29,6 +30,8 @@ export function createSearchProvider(name: string, creds: ProviderCredentials):
|
|
|
29
30
|
return new FirecrawlProvider(apiKey);
|
|
30
31
|
case "searxng":
|
|
31
32
|
return new SearxngProvider({ apiKey: creds.apiKey, baseUrl: creds.baseUrl ?? "" });
|
|
33
|
+
case "ollama":
|
|
34
|
+
return new OllamaProvider({ apiKey: creds.apiKey, baseUrl: creds.baseUrl ?? "" });
|
|
32
35
|
default:
|
|
33
36
|
throw new Error(`Unknown search provider: "${name}"`);
|
|
34
37
|
}
|
package/providers/firecrawl.ts
CHANGED
|
@@ -52,7 +52,7 @@ export class FirecrawlProvider implements SearchProvider {
|
|
|
52
52
|
|
|
53
53
|
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
54
54
|
if (!this.apiKey) {
|
|
55
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
55
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
56
56
|
}
|
|
57
57
|
|
|
58
58
|
const res = await fetch(`${FIRECRAWL_API_URL}/search`, {
|
|
@@ -79,7 +79,7 @@ export class FirecrawlProvider implements SearchProvider {
|
|
|
79
79
|
|
|
80
80
|
async fetch(url: string, _raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
|
|
81
81
|
if (!this.apiKey) {
|
|
82
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
82
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
83
83
|
}
|
|
84
84
|
|
|
85
85
|
const res = await fetch(`${FIRECRAWL_API_URL}/scrape`, {
|
package/providers/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { BRAVE_PROVIDER_META } from "./brave.js";
|
|
|
2
2
|
import { EXA_PROVIDER_META } from "./exa.js";
|
|
3
3
|
import { FIRECRAWL_PROVIDER_META } from "./firecrawl.js";
|
|
4
4
|
import { JINA_PROVIDER_META } from "./jina.js";
|
|
5
|
+
import { OLLAMA_PROVIDER_META } from "./ollama.js";
|
|
5
6
|
import { SEARXNG_PROVIDER_META } from "./searxng.js";
|
|
6
7
|
import { SERPER_PROVIDER_META } from "./serper.js";
|
|
7
8
|
import { TAVILY_PROVIDER_META } from "./tavily.js";
|
|
@@ -12,6 +13,14 @@ export { EXA_API_KEY_ENV_VAR, EXA_PROVIDER_META, ExaProvider } from "./exa.js";
|
|
|
12
13
|
export { createSearchProvider, type ProviderCredentials } from "./factory.js";
|
|
13
14
|
export { FIRECRAWL_API_KEY_ENV_VAR, FIRECRAWL_PROVIDER_META, FirecrawlProvider } from "./firecrawl.js";
|
|
14
15
|
export { JINA_API_KEY_ENV_VAR, JINA_PROVIDER_META, JinaProvider } from "./jina.js";
|
|
16
|
+
export {
|
|
17
|
+
configureOllama,
|
|
18
|
+
OLLAMA_API_KEY_ENV_VAR,
|
|
19
|
+
OLLAMA_DEFAULT_URL,
|
|
20
|
+
OLLAMA_HOST_ENV_VAR,
|
|
21
|
+
OLLAMA_PROVIDER_META,
|
|
22
|
+
OllamaProvider,
|
|
23
|
+
} from "./ollama.js";
|
|
15
24
|
export {
|
|
16
25
|
configureSearxng,
|
|
17
26
|
SEARXNG_API_KEY_ENV_VAR,
|
|
@@ -48,4 +57,5 @@ export const PROVIDERS: readonly ProviderMeta[] = [
|
|
|
48
57
|
JINA_PROVIDER_META,
|
|
49
58
|
FIRECRAWL_PROVIDER_META,
|
|
50
59
|
SEARXNG_PROVIDER_META,
|
|
60
|
+
OLLAMA_PROVIDER_META,
|
|
51
61
|
];
|
package/providers/jina.ts
CHANGED
|
@@ -42,7 +42,7 @@ export class JinaProvider implements SearchProvider {
|
|
|
42
42
|
|
|
43
43
|
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
44
44
|
if (!this.apiKey) {
|
|
45
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
45
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
46
46
|
}
|
|
47
47
|
|
|
48
48
|
// Jina s.jina.ai uses the URL path for the query. The `num` query param
|
|
@@ -73,7 +73,7 @@ export class JinaProvider implements SearchProvider {
|
|
|
73
73
|
|
|
74
74
|
async fetch(url: string, _raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
|
|
75
75
|
if (!this.apiKey) {
|
|
76
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
76
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
// No Accept header = Reader returns markdown by default. Setting
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type FetchResponse,
|
|
3
|
+
isCancellation,
|
|
4
|
+
type ProviderConfigChange,
|
|
5
|
+
type ProviderConfigCurrent,
|
|
6
|
+
type ProviderConfigUi,
|
|
7
|
+
type ProviderMeta,
|
|
8
|
+
type SearchProvider,
|
|
9
|
+
type SearchResponse,
|
|
10
|
+
type SearchResult,
|
|
11
|
+
} from "./types.js";
|
|
12
|
+
|
|
13
|
+
export const OLLAMA_API_KEY_ENV_VAR = "OLLAMA_API_KEY";
|
|
14
|
+
export const OLLAMA_HOST_ENV_VAR = "OLLAMA_HOST";
|
|
15
|
+
export const OLLAMA_DEFAULT_URL = "http://localhost:11434";
|
|
16
|
+
|
|
17
|
+
// Ollama API paths — cloud (ollama.com) uses stable /api/... paths,
|
|
18
|
+
// local instances use /api/experimental/... (at least through v0.24).
|
|
19
|
+
const CLOUD_SEARCH_PATH = "/api/web_search";
|
|
20
|
+
const CLOUD_FETCH_PATH = "/api/web_fetch";
|
|
21
|
+
const LOCAL_SEARCH_PATH = "/api/experimental/web_search";
|
|
22
|
+
const LOCAL_FETCH_PATH = "/api/experimental/web_fetch";
|
|
23
|
+
|
|
24
|
+
function isLocalHost(baseUrl: string): boolean {
|
|
25
|
+
try {
|
|
26
|
+
const hostname = new URL(baseUrl).hostname;
|
|
27
|
+
return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "0.0.0.0" || hostname === "[::1]";
|
|
28
|
+
} catch {
|
|
29
|
+
return true; // default to local paths if URL is somehow invalid
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Number of leading + trailing characters preserved when masking an API key
|
|
34
|
+
// in the config prompt. Mirrors API_KEY_MASK_VISIBLE_CHARS in web-tools.ts.
|
|
35
|
+
const MASK_VISIBLE_CHARS = 4;
|
|
36
|
+
|
|
37
|
+
export const OLLAMA_PROVIDER_META: ProviderMeta = {
|
|
38
|
+
name: "ollama",
|
|
39
|
+
label: "Ollama",
|
|
40
|
+
envVar: OLLAMA_API_KEY_ENV_VAR,
|
|
41
|
+
baseUrlEnvVar: OLLAMA_HOST_ENV_VAR,
|
|
42
|
+
defaultBaseUrl: OLLAMA_DEFAULT_URL,
|
|
43
|
+
configure: (ui, current) => configureOllama(ui, current),
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
// ---------------------------------------------------------------------------
|
|
47
|
+
// Vendor response types (file-private)
|
|
48
|
+
// ---------------------------------------------------------------------------
|
|
49
|
+
|
|
50
|
+
interface OllamaRawSearchResult {
|
|
51
|
+
title?: string;
|
|
52
|
+
url?: string;
|
|
53
|
+
content?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface OllamaSearchResponse {
|
|
57
|
+
results?: OllamaRawSearchResult[];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
interface OllamaFetchResponse {
|
|
61
|
+
title?: string;
|
|
62
|
+
content?: string;
|
|
63
|
+
links?: string[];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// Normalization
|
|
68
|
+
// ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
function normalizeOllamaResults(raw: OllamaSearchResponse): SearchResult[] {
|
|
71
|
+
return (raw.results ?? []).map((r) => ({
|
|
72
|
+
title: r.title ?? "",
|
|
73
|
+
url: r.url ?? "",
|
|
74
|
+
snippet: r.content ?? "",
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
// URL helpers
|
|
80
|
+
// ---------------------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
function stripTrailingSlashes(url: string): string {
|
|
83
|
+
return url.replace(/\/+$/, "");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function assertHttpUrl(url: string): void {
|
|
87
|
+
let parsed: URL;
|
|
88
|
+
try {
|
|
89
|
+
parsed = new URL(url);
|
|
90
|
+
} catch {
|
|
91
|
+
throw new Error(`${OLLAMA_HOST_ENV_VAR} is not a valid URL (got: ${url})`);
|
|
92
|
+
}
|
|
93
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
94
|
+
throw new Error(
|
|
95
|
+
`${OLLAMA_HOST_ENV_VAR} must use http:// or https:// (got: ${parsed.protocol.replace(":", "")}://)`,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// ---------------------------------------------------------------------------
|
|
101
|
+
// Network error handling
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
|
|
104
|
+
// Node's fetch wraps ECONNREFUSED in a TypeError. Detect the cause chain
|
|
105
|
+
// and re-throw with an actionable hint.
|
|
106
|
+
function isConnectionRefused(error: unknown): boolean {
|
|
107
|
+
if (error instanceof TypeError) {
|
|
108
|
+
const cause = (error as unknown as { cause?: { code?: string } }).cause;
|
|
109
|
+
return cause?.code === "ECONNREFUSED";
|
|
110
|
+
}
|
|
111
|
+
return false;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function connectionRefusedError(host: string): Error {
|
|
115
|
+
return new Error(`Could not connect to Ollama at ${host}. Make sure Ollama is running (ollama serve).`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ---------------------------------------------------------------------------
|
|
119
|
+
// Provider class
|
|
120
|
+
// ---------------------------------------------------------------------------
|
|
121
|
+
|
|
122
|
+
interface OllamaProviderOptions {
|
|
123
|
+
apiKey?: string;
|
|
124
|
+
baseUrl: string;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export class OllamaProvider implements SearchProvider {
|
|
128
|
+
readonly name = "ollama";
|
|
129
|
+
readonly label = "Ollama";
|
|
130
|
+
readonly envVar = OLLAMA_API_KEY_ENV_VAR;
|
|
131
|
+
|
|
132
|
+
private readonly apiKey?: string;
|
|
133
|
+
private readonly baseUrl: string;
|
|
134
|
+
private readonly local: boolean;
|
|
135
|
+
|
|
136
|
+
constructor(options: OllamaProviderOptions) {
|
|
137
|
+
this.apiKey = options.apiKey?.trim() || undefined;
|
|
138
|
+
const trimmed = stripTrailingSlashes(options.baseUrl?.trim() ?? "");
|
|
139
|
+
if (trimmed) assertHttpUrl(trimmed);
|
|
140
|
+
this.baseUrl = trimmed;
|
|
141
|
+
this.local = isLocalHost(trimmed);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
145
|
+
this.requireBaseUrl();
|
|
146
|
+
const path = this.local ? LOCAL_SEARCH_PATH : CLOUD_SEARCH_PATH;
|
|
147
|
+
try {
|
|
148
|
+
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
149
|
+
method: "POST",
|
|
150
|
+
headers: this.buildHeaders(),
|
|
151
|
+
body: JSON.stringify({ query, max_results: maxResults }),
|
|
152
|
+
signal,
|
|
153
|
+
});
|
|
154
|
+
if (!res.ok) throw await this.formatError("Search", res);
|
|
155
|
+
const raw = (await res.json()) as OllamaSearchResponse;
|
|
156
|
+
return { query, results: normalizeOllamaResults(raw) };
|
|
157
|
+
} catch (error) {
|
|
158
|
+
if (isConnectionRefused(error)) throw connectionRefusedError(this.baseUrl);
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async fetch(url: string, _raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
|
|
164
|
+
this.requireBaseUrl();
|
|
165
|
+
const path = this.local ? LOCAL_FETCH_PATH : CLOUD_FETCH_PATH;
|
|
166
|
+
try {
|
|
167
|
+
const res = await fetch(`${this.baseUrl}${path}`, {
|
|
168
|
+
method: "POST",
|
|
169
|
+
headers: this.buildHeaders(),
|
|
170
|
+
body: JSON.stringify({ url }),
|
|
171
|
+
signal,
|
|
172
|
+
});
|
|
173
|
+
if (!res.ok) throw await this.formatError("Fetch", res);
|
|
174
|
+
const data = (await res.json()) as OllamaFetchResponse;
|
|
175
|
+
if (!data.content) {
|
|
176
|
+
throw new Error(`${this.label} Fetch API error: no content returned for ${url}`);
|
|
177
|
+
}
|
|
178
|
+
return {
|
|
179
|
+
text: data.content,
|
|
180
|
+
title: data.title || undefined,
|
|
181
|
+
contentType: "text/plain",
|
|
182
|
+
};
|
|
183
|
+
} catch (error) {
|
|
184
|
+
if (isConnectionRefused(error)) throw connectionRefusedError(this.baseUrl);
|
|
185
|
+
throw error;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
private requireBaseUrl(): void {
|
|
190
|
+
if (!this.baseUrl) {
|
|
191
|
+
throw new Error(`${OLLAMA_HOST_ENV_VAR} is not set. Run /web-tools to configure, or export the env var.`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
private buildHeaders(): Record<string, string> {
|
|
196
|
+
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
197
|
+
if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
|
|
198
|
+
return headers;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
private async formatError(label: string, res: Response): Promise<Error> {
|
|
202
|
+
const body = await res.text();
|
|
203
|
+
const hint = hintForStatus(res.status);
|
|
204
|
+
return new Error(`${this.label} ${label} API error (${res.status})${hint}: ${body}`);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
// Status hints
|
|
210
|
+
// ---------------------------------------------------------------------------
|
|
211
|
+
|
|
212
|
+
function hintForStatus(status: number): string {
|
|
213
|
+
if (status === 401) {
|
|
214
|
+
return " (run `ollama signin` to authenticate)";
|
|
215
|
+
}
|
|
216
|
+
if (status === 404) {
|
|
217
|
+
return " (the Ollama instance may not support web search; ensure you are running a recent version)";
|
|
218
|
+
}
|
|
219
|
+
return "";
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ---------------------------------------------------------------------------
|
|
223
|
+
// /web-tools helper
|
|
224
|
+
// ---------------------------------------------------------------------------
|
|
225
|
+
|
|
226
|
+
function maskKey(key: string): string {
|
|
227
|
+
const head = key.slice(0, MASK_VISIBLE_CHARS);
|
|
228
|
+
const tail = key.slice(-MASK_VISIBLE_CHARS);
|
|
229
|
+
return `${head}...${tail}`;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
async function promptForBaseUrl(ui: ProviderConfigUi, current: string | undefined): Promise<string | undefined> {
|
|
233
|
+
const existing = current?.trim();
|
|
234
|
+
const input = await ui.input(
|
|
235
|
+
"Ollama base URL",
|
|
236
|
+
existing
|
|
237
|
+
? `Press Enter to keep current (${existing}), or type new URL`
|
|
238
|
+
: `Press Enter for default (${OLLAMA_DEFAULT_URL}), or type instance URL`,
|
|
239
|
+
);
|
|
240
|
+
if (isCancellation(input)) return undefined;
|
|
241
|
+
return input.trim() || existing || OLLAMA_DEFAULT_URL;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function promptForOptionalKey(
|
|
245
|
+
ui: ProviderConfigUi,
|
|
246
|
+
current: string | undefined,
|
|
247
|
+
): Promise<string | null | undefined> {
|
|
248
|
+
const existing = current?.trim() || undefined;
|
|
249
|
+
const input = await ui.input(
|
|
250
|
+
"Ollama API key (optional — for direct cloud access; local Ollama authenticates via `ollama signin`)",
|
|
251
|
+
existing
|
|
252
|
+
? `Press Enter to keep current (${maskKey(existing)}), or type new key`
|
|
253
|
+
: "Press Enter to leave unset, or type a key",
|
|
254
|
+
);
|
|
255
|
+
if (isCancellation(input)) return undefined;
|
|
256
|
+
return input.trim() || existing || null;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* Prompts the user for the Ollama base URL and optional API key.
|
|
261
|
+
* Returns `null` if the user cancels at either prompt.
|
|
262
|
+
*/
|
|
263
|
+
export async function configureOllama(
|
|
264
|
+
ui: ProviderConfigUi,
|
|
265
|
+
current: ProviderConfigCurrent,
|
|
266
|
+
): Promise<ProviderConfigChange | null> {
|
|
267
|
+
const baseUrl = await promptForBaseUrl(ui, current.baseUrl);
|
|
268
|
+
if (baseUrl === undefined) return null;
|
|
269
|
+
|
|
270
|
+
const apiKey = await promptForOptionalKey(ui, current.apiKey);
|
|
271
|
+
if (apiKey === undefined) return null;
|
|
272
|
+
|
|
273
|
+
return { baseUrl, apiKey };
|
|
274
|
+
}
|
package/providers/searxng.ts
CHANGED
|
@@ -144,9 +144,7 @@ export class SearxngProvider implements SearchProvider {
|
|
|
144
144
|
|
|
145
145
|
private requireBaseUrl(): void {
|
|
146
146
|
if (!this.baseUrl) {
|
|
147
|
-
throw new Error(
|
|
148
|
-
`${SEARXNG_URL_ENV_VAR} is not set. Run /web-search-config to configure, or export the env var.`,
|
|
149
|
-
);
|
|
147
|
+
throw new Error(`${SEARXNG_URL_ENV_VAR} is not set. Run /web-tools to configure, or export the env var.`);
|
|
150
148
|
}
|
|
151
149
|
}
|
|
152
150
|
|
|
@@ -176,7 +174,7 @@ export class SearxngProvider implements SearchProvider {
|
|
|
176
174
|
}
|
|
177
175
|
|
|
178
176
|
// ---------------------------------------------------------------------------
|
|
179
|
-
// /web-
|
|
177
|
+
// /web-tools helper — SearXNG branch
|
|
180
178
|
// ---------------------------------------------------------------------------
|
|
181
179
|
// SEARXNG_PROVIDER_META.configure wires configureSearxng() in; the orchestrator
|
|
182
180
|
// dispatches generically through ProviderMeta.configure without naming
|
package/providers/serper.ts
CHANGED
|
@@ -37,7 +37,7 @@ export class SerperProvider implements SearchProvider {
|
|
|
37
37
|
|
|
38
38
|
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
39
39
|
if (!this.apiKey) {
|
|
40
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
40
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
const res = await fetch(SERPER_API_URL, {
|
package/providers/tavily.ts
CHANGED
|
@@ -48,7 +48,7 @@ export class TavilyProvider implements SearchProvider {
|
|
|
48
48
|
|
|
49
49
|
async search(query: string, maxResults: number, signal?: AbortSignal): Promise<SearchResponse> {
|
|
50
50
|
if (!this.apiKey) {
|
|
51
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
51
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
const res = await fetch(TAVILY_API_URL, {
|
|
@@ -73,7 +73,7 @@ export class TavilyProvider implements SearchProvider {
|
|
|
73
73
|
|
|
74
74
|
async fetch(url: string, _raw: boolean, signal?: AbortSignal): Promise<FetchResponse> {
|
|
75
75
|
if (!this.apiKey) {
|
|
76
|
-
throw new Error(`${this.envVar} is not set. Run /web-
|
|
76
|
+
throw new Error(`${this.envVar} is not set. Run /web-tools to configure, or export the env var.`);
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
// Bearer header per current Tavily docs. Existing search() above still
|
package/providers/types.ts
CHANGED
|
@@ -67,7 +67,7 @@ export interface ProviderConfigChange {
|
|
|
67
67
|
// envVar — the API-key env var (omit if the provider has no key)
|
|
68
68
|
// baseUrlEnvVar — the URL env var (set for self-hosted providers)
|
|
69
69
|
// defaultBaseUrl — fallback URL when neither env nor config supplies one
|
|
70
|
-
// configure — interactive setup; if present, /web-
|
|
70
|
+
// configure — interactive setup; if present, /web-tools
|
|
71
71
|
// dispatches here instead of the default single-key prompt
|
|
72
72
|
export interface ProviderMeta {
|
|
73
73
|
name: string;
|
package/web-tools.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* rpiv-web-tools — body
|
|
3
3
|
*
|
|
4
4
|
* Provides `web_search` and `web_fetch` tools backed by configurable search
|
|
5
|
-
* providers (Brave, Tavily, Serper, Exa), plus the `/web-
|
|
5
|
+
* providers (Brave, Tavily, Serper, Exa), plus the `/web-tools`
|
|
6
6
|
* slash command for provider and API key configuration.
|
|
7
7
|
*
|
|
8
8
|
* API key resolution precedence per provider (first wins):
|
|
@@ -49,7 +49,7 @@ const CONFIG_PATH = configPath("rpiv-web-tools");
|
|
|
49
49
|
|
|
50
50
|
const SUPPORTED_HTTP_PROTOCOLS = new Set(["http:", "https:"]);
|
|
51
51
|
|
|
52
|
-
const
|
|
52
|
+
const WEB_TOOLS_COMMAND_NAME = "web-tools";
|
|
53
53
|
const SHOW_FLAG = "--show";
|
|
54
54
|
const UNSET_LABEL = "(not set)";
|
|
55
55
|
|
|
@@ -58,7 +58,7 @@ const DEFAULT_PROVIDER_NAME = "brave";
|
|
|
58
58
|
// Brave is the only provider whose key was historically stored at the top
|
|
59
59
|
// level (config.apiKey) before the per-provider apiKeys map. The legacy
|
|
60
60
|
// field is auto-migrated to apiKeys.brave on the next save by
|
|
61
|
-
// /web-
|
|
61
|
+
// /web-tools (the dispatch deletes apiKey from the saved object).
|
|
62
62
|
const LEGACY_TOP_LEVEL_KEY_PROVIDER = "brave";
|
|
63
63
|
|
|
64
64
|
// ---------------------------------------------------------------------------
|
|
@@ -100,7 +100,7 @@ export const DEFAULT_WEB_SEARCH_GUIDELINES: string[] = [
|
|
|
100
100
|
'Use the current year from "Current date:" in your context when searching for recent information or documentation.',
|
|
101
101
|
'After answering using search results, include a "Sources:" section listing relevant URLs as markdown hyperlinks: [Title](URL). Never skip this.',
|
|
102
102
|
"Domain filtering is supported to include or block specific websites.",
|
|
103
|
-
"If no API key is configured, ask the user to run /web-
|
|
103
|
+
"If no API key is configured, ask the user to run /web-tools before proceeding.",
|
|
104
104
|
];
|
|
105
105
|
|
|
106
106
|
export const DEFAULT_WEB_FETCH_SNIPPET = "Fetch and read content from a specific URL";
|
|
@@ -450,7 +450,7 @@ function renderFetchedContentPreview(content: string, theme: Theme): string {
|
|
|
450
450
|
}
|
|
451
451
|
|
|
452
452
|
// ---------------------------------------------------------------------------
|
|
453
|
-
// /web-
|
|
453
|
+
// /web-tools command
|
|
454
454
|
// ---------------------------------------------------------------------------
|
|
455
455
|
|
|
456
456
|
function formatShowConfigMessage(current: WebToolsConfig): string {
|
|
@@ -485,11 +485,11 @@ function formatShowConfigMessage(current: WebToolsConfig): string {
|
|
|
485
485
|
}
|
|
486
486
|
|
|
487
487
|
export function registerWebSearchConfigCommand(pi: ExtensionAPI): void {
|
|
488
|
-
pi.registerCommand(
|
|
488
|
+
pi.registerCommand(WEB_TOOLS_COMMAND_NAME, {
|
|
489
489
|
description: "Configure the search provider and API key used by web_search",
|
|
490
490
|
handler: async (args, ctx) => {
|
|
491
491
|
if (!ctx.hasUI) {
|
|
492
|
-
ctx.ui?.notify?.(`/${
|
|
492
|
+
ctx.ui?.notify?.(`/${WEB_TOOLS_COMMAND_NAME} requires interactive mode`, "error");
|
|
493
493
|
return;
|
|
494
494
|
}
|
|
495
495
|
|