deepseek-web-retrieval 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -1
- package/README.md +33 -16
- package/dist/cli.js +307 -174
- package/docs/adr/0001-refuse-drift-between-plan-and-apply.md +7 -3
- package/docs/adr/0002-shared-config-base-layer.md +5 -1
- package/docs/adr/0003-need-content-per-call-boolean.md +49 -0
- package/docs/agents/claude-code.md +16 -6
- package/docs/agents/codex.md +14 -8
- package/docs/agents/domain.md +2 -2
- package/docs/agents/issue-tracker.md +5 -5
- package/docs/agents/opencode-v2.md +29 -15
- package/docs/agents/pi.md +41 -12
- package/docs/configuration.md +24 -7
- package/docs/design/config-layering-design.md +10 -5
- package/docs/design/installer-design.md +57 -25
- package/docs/design/search-content-design.md +133 -0
- package/docs/integration.md +19 -16
- package/docs/research/control-matrix.md +39 -34
- package/docs/research/mcp-compatibility.md +3 -1
- package/docs/research/ori-question-sourcing.md +31 -20
- package/docs/research/result-quality-passthrough.md +9 -7
- package/docs/research/search-api-defaults.md +4 -2
- package/docs/research/search-inline-content.md +49 -9
- package/package.json +3 -2
- package/src/core/auth.ts +7 -0
- package/src/core/client.ts +98 -27
- package/src/core/config-summary.ts +3 -0
- package/src/core/constants.ts +30 -2
- package/src/core/descriptions.ts +17 -2
- package/src/core/env.ts +7 -1
- package/src/core/format.ts +35 -5
- package/src/core/options.ts +56 -0
- package/src/core/search.ts +106 -15
- package/src/core/types.ts +29 -0
- package/src/installer/apply.ts +4 -4
- package/src/installer/cli.ts +2 -2
- package/src/installer/config-command.ts +4 -3
- package/src/installer/hosts/claude-code.ts +3 -17
- package/src/installer/hosts/codex.ts +3 -17
- package/src/installer/hosts/index.ts +2 -2
- package/src/installer/hosts/opencode.ts +1 -1
- package/src/installer/hosts/pi.ts +14 -15
- package/src/installer/hosts/shared.ts +23 -0
- package/src/installer/ownership.ts +2 -2
- package/src/installer/paths.ts +2 -2
- package/src/installer/{state.ts → record.ts} +15 -15
- package/src/installer/types.ts +1 -1
- package/src/installer/uninstall.ts +4 -4
- package/src/mcp/serve.ts +1 -2
- package/src/mcp/server.ts +19 -19
- package/src/opencode/plugin.ts +15 -10
- package/src/pi/extension.ts +26 -17
- package/src/mcp/auth.ts +0 -11
|
@@ -17,6 +17,12 @@ source at the repo's locked tag. Secondhand blogs are excluded. Where no primary
|
|
|
17
17
|
a fact — most importantly the size of `content` — that is stated as such: an undocumented
|
|
18
18
|
size is itself a decision-relevant fact.
|
|
19
19
|
|
|
20
|
+
**Short answer:** DeepSeek documents **no size bound** for `need_content` content and no
|
|
21
|
+
per-result budget knob; of the eight architectures surveyed below, only Jina ships
|
|
22
|
+
content-bearing search as the default; and the binding host cap is Pi's 20 KB MCP-path
|
|
23
|
+
truncation — the volume math below puts a safe per-result budget in the ~1k–2k char
|
|
24
|
+
range.
|
|
25
|
+
|
|
20
26
|
## What `need_content=true` provably returns
|
|
21
27
|
|
|
22
28
|
From the official Search & Fetch guide, the parameter and its response field, verbatim:
|
|
@@ -101,15 +107,36 @@ search response without setting any parameter.
|
|
|
101
107
|
|
|
102
108
|
| Vendor | Content inline by default? | Opt-in mechanism | Per-result content budget knob | Separate fetch tool? |
|
|
103
109
|
| --- | --- | --- | --- | --- |
|
|
104
|
-
| **Tavily** | Partial —
|
|
110
|
+
| **Tavily** | Partial — per-result `content` of relevance chunks by default | Full body via `include_raw_content` (default **false**) | Chunk count only (`chunks_per_source` 1–3); no knob limits `raw_content` size in `/search` | Yes — `/extract` (up to 20 URLs) |
|
|
105
111
|
| **Exa** | No — `contents.text` defaults **false** | `contents.text: true` in the **same** `POST /search` body | **`maxCharacters`** (1–1,000,000, no published default) plus `verbosity` `compact`/`standard`/`full` | Yes — `/contents` endpoint by result ID/URL |
|
|
106
112
|
| **You.com** | No — “Omit `extraction` for a plain search that returns snippets only” | `extraction.full_page` (“crawls each result and returns the page content”) | **None for `full_page`** (only `crawl_timeout`); the `highlights` mode has a response-wide `total_max_tokens` budget of **8192** (1,024–8,192) | No dedicated fetch endpoint documented on `/v1/search` |
|
|
107
|
-
| **Perplexity** | No — Search API results are `title`/`url`/`snippet`/`date` only | Token budgets
|
|
108
|
-
| **Anthropic** | Not caller-visible —
|
|
109
|
-
| **Firecrawl** | No — search returns `title`/`description`/`url` (+ `highlights`, default true) | `scrapeOptions` (default `{}`):
|
|
110
|
-
| **Jina** (`s.jina.ai`) | **Yes — the only default-content architecture surveyed
|
|
113
|
+
| **Perplexity** | No — Search API results are `title`/`url`/`snippet`/`date` only | Token budgets on both the Search API and the Agent API `web_search` tool (detail below) | **`max_tokens_per_page`** (min 1, max 1,000,000, no default) | Yes — Agent API “Fetch URL Content” tool (“Fetch full content from known URLs”) |
|
|
114
|
+
| **Anthropic** | Not caller-visible — content is model-only (`encrypted_content`) | None — `response_inclusion` is not a content-size knob | **None** (no caller-set budget) | Yes — `web_fetch` with token-based `max_content_tokens` (no published default) |
|
|
115
|
+
| **Firecrawl** | No — search returns `title`/`description`/`url` (+ `highlights`, default true) | `scrapeOptions` (default `{}`): “Include `scrapeOptions` with `formats: [{“type”: “markdown”}]` to get complete markdown content for each search result” | **None** for content size (`onlyMainContent` default true trims boilerplate, not length) | Collapsed into search via `scrapeOptions`; standalone `/scrape` endpoint also exists |
|
|
116
|
+
| **Jina** (`s.jina.ai`) | **Yes — the only default-content architecture surveyed**; result count is fixed at 5, not a parameter | None needed (content is the product) | **`x-token-budget`** header, no default (parsed only when sent) — a global budget, not per-result | Same product (`r.jina.ai`) serves plain fetch |
|
|
111
117
|
| **DeepSeek (subject of this note)** | No — `need_content` defaults **false** | `need_content: true` | **None documented** (see above) | Yes — Fetch API, Markdown, no documented size cap |
|
|
112
118
|
|
|
119
|
+
Detail for the trimmed rows above:
|
|
120
|
+
|
|
121
|
+
- **Tavily:** the default per-result `content` carries **relevance chunks** —
|
|
122
|
+
`chunks_per_source` **3** chunks of ~500 chars (~1,500 chars/result); `include_raw_content`
|
|
123
|
+
accepts `markdown`/`text`/`true`; the chunk sizes apply to `content`, **not**
|
|
124
|
+
`raw_content`; `/extract` takes up to 20 URLs with `extract_depth` basic/advanced, no
|
|
125
|
+
per-URL cap, ~800-char chunks only when `query` is given.
|
|
126
|
+
- **Perplexity:** `max_tokens` (“Maximum total webpage content tokens to return across
|
|
127
|
+
all search results”) and **`max_tokens_per_page`** (“Maximum webpage content tokens to
|
|
128
|
+
extract from each result page”), **no default published** for either (“the default
|
|
129
|
+
budgets apply”); on the Search API the extracted text surfaces through `snippet`.
|
|
130
|
+
- **Anthropic:** `web_search` results carry `url`/`title`/`page_age` plus
|
|
131
|
+
`encrypted_content` (“Encrypted content that you must pass back in multi-turn
|
|
132
|
+
conversations”) — the model reads content in its context, the caller cannot.
|
|
133
|
+
`response_inclusion` (`”full”` default / `”excluded”`) only controls whether consumed
|
|
134
|
+
search blocks are echoed in the response; dynamic filtering on `web_search_20260209+`
|
|
135
|
+
has Claude write code to filter results before they reach context, but no caller-set
|
|
136
|
+
budget exists.
|
|
137
|
+
- **Jina:** “Reader searches the web, fetches the top 5 results, visits each URL, and
|
|
138
|
+
applies `r.jina.ai` to it.”
|
|
139
|
+
|
|
113
140
|
The split, stated plainly: of the eight architectures surveyed, exactly one (Jina
|
|
114
141
|
`s.jina.ai`) ships content-bearing search as the default, and it pins the result count at 5.
|
|
115
142
|
Every other vendor makes inline content opt-in (Tavily `raw_content`, Exa `contents.text`,
|
|
@@ -119,6 +146,17 @@ exist they are per-result character caps (Exa `maxCharacters`) or per-result tok
|
|
|
119
146
|
(Perplexity `max_tokens_per_page`); Tavily and You.com ship full-content modes with **no**
|
|
120
147
|
per-result size knob at all — the two closest analogues to DeepSeek's `need_content`.
|
|
121
148
|
|
|
149
|
+
One further architecture reading (added 2026-10-24, same sources): **no surveyed vendor
|
|
150
|
+
narrows another parameter's value range when inline content is requested.** Every budget
|
|
151
|
+
knob in the table is static-range, conditional-effect — Exa's `maxCharacters` and
|
|
152
|
+
Perplexity's `max_tokens_per_page` keep their 1–1,000,000 ranges whether or not
|
|
153
|
+
`contents.text`/extraction is engaged, Tavily's `chunks_per_source` and Anthropic's
|
|
154
|
+
`max_content_tokens` are likewise unconditional, and none shrinks `count`/`numResults`
|
|
155
|
+
or any other parameter's domain when content flows (Jina's fixed 5 is its architecture,
|
|
156
|
+
not a conditional rule). The only conditional behavior anywhere in the table is which
|
|
157
|
+
budget *applies*; a content flag that reshaped other parameters' allowed values appears
|
|
158
|
+
in none of the eight.
|
|
159
|
+
|
|
122
160
|
Sources: [Tavily /search](https://docs.tavily.com/documentation/api-reference/endpoint/search) ·
|
|
123
161
|
[Tavily /extract](https://docs.tavily.com/documentation/api-reference/endpoint/extract) ·
|
|
124
162
|
[Exa /search](https://exa.ai/docs/reference/search) ·
|
|
@@ -173,8 +211,9 @@ With `count=10` (the repo's default, and DeepSeek's) and `need_content: true`:
|
|
|
173
211
|
[mcp_types.rs at rust-v0.163.0](https://github.com/openai/codex/blob/rust-v0.163.0/codex-rs/config/src/mcp_types.rs)).
|
|
174
212
|
[control-matrix.md](control-matrix.md) records Codex's 300 s MCP tool timeout — latency,
|
|
175
213
|
not size, is that host's documented ceiling.
|
|
176
|
-
- **The binding constraint is Pi's 20 KB**, roughly
|
|
177
|
-
|
|
214
|
+
- **The binding constraint is Pi's 20 KB**, roughly 5× tighter than Claude Code's
|
|
215
|
+
25k-token default (≈100k English chars by the conversions above): a per-result
|
|
216
|
+
content budget in the **~1k–2k char**
|
|
178
217
|
range keeps count=10 under every published cap; reusing the repo's fetch budget (16,000
|
|
179
218
|
chars) *per search result* would reach 160k chars per response — 8× Pi's cap and ~40k
|
|
180
219
|
tokens on Claude Code.
|
|
@@ -195,8 +234,9 @@ maintainer decides.
|
|
|
195
234
|
- **Today's pipeline would silently drop the content.** `DeepSeekSearchResult`
|
|
196
235
|
([types.ts](../../src/core/types.ts)) has no `content` field, and `processResults`
|
|
197
236
|
([search.ts](../../src/core/search.ts)) maps `SearchResult.content` from
|
|
198
|
-
`summary ?? snippet` only — a `need_content` response's page bodies would be received
|
|
199
|
-
discarded
|
|
237
|
+
`summary ?? snippet` only — a `need_content` response's page bodies would be received
|
|
238
|
+
and discarded: they cost nothing extra (billing is per request), but they are also
|
|
239
|
+
never rendered.
|
|
200
240
|
- **The only per-result clamp in the search path is `maxSummaryChars` (1,200)** — a
|
|
201
241
|
summary/snippet budget. `fetchMaxChars` (16,000) and `formatFetchedContent`'s
|
|
202
242
|
truncation-marker logic apply exclusively to the fetch tool path. There is **no per-result
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "deepseek-web-retrieval",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.10.0",
|
|
5
5
|
"description": "Token-efficient DeepSeek Search and Fetch for coding agents, starting with OpenCode v2.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"license": "MIT",
|
|
@@ -57,7 +57,8 @@
|
|
|
57
57
|
"prepublishOnly": "bun run check:release",
|
|
58
58
|
"test": "bun test",
|
|
59
59
|
"test:package": "node scripts/test-package.mjs",
|
|
60
|
-
"test:e2e:keyless": "bun run test:e2e pi pi-extension installer reinit upgrade pkg capability-off",
|
|
60
|
+
"test:e2e:keyless": "bun run test:e2e pi pi-extension pi-mock opencode-mock installer reinit upgrade pkg capability-off",
|
|
61
|
+
"test:e2e:billed": "bun run test:e2e pi-billed opencode codex claude-code preferred",
|
|
61
62
|
"check:release": "bun run check && bun run test:package && bun run test:e2e:keyless",
|
|
62
63
|
"test:e2e": "bun run test/e2e/run.ts",
|
|
63
64
|
"docs:matrix": "bun run scripts/render-matrix.ts",
|
package/src/core/auth.ts
CHANGED
|
@@ -7,3 +7,10 @@ export function apiKeyFromEnv(env: Readonly<Record<string, string | undefined>>
|
|
|
7
7
|
const key = env[API_KEY_ENV]?.trim();
|
|
8
8
|
return key ? key : undefined;
|
|
9
9
|
}
|
|
10
|
+
|
|
11
|
+
/** Resolves the API key from the environment or fails with shared guidance. */
|
|
12
|
+
export function requireApiKey(env: Readonly<Record<string, string | undefined>>): string {
|
|
13
|
+
const apiKey = apiKeyFromEnv(env);
|
|
14
|
+
if (!apiKey) throw new Error(MISSING_API_KEY_MESSAGE);
|
|
15
|
+
return apiKey;
|
|
16
|
+
}
|
package/src/core/client.ts
CHANGED
|
@@ -1,7 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_API_BASE_URL,
|
|
3
|
+
FETCH_ENDPOINT_PATH,
|
|
4
|
+
MAX_FETCH_CHARS,
|
|
5
|
+
MAX_ORI_QUESTION_CHARS,
|
|
6
|
+
MAX_QUERY_CHARS,
|
|
7
|
+
SEARCH_ENDPOINT_PATH,
|
|
8
|
+
} from "./constants";
|
|
2
9
|
import { assertFetchHostPolicy } from "./domains";
|
|
3
10
|
import { formatFetchedContent, truncate } from "./format";
|
|
4
|
-
import {
|
|
11
|
+
import { composeSearchQuery, processResults } from "./search";
|
|
5
12
|
import { postJson } from "./transport";
|
|
6
13
|
import type {
|
|
7
14
|
DeepSeekSearchResult,
|
|
@@ -9,9 +16,10 @@ import type {
|
|
|
9
16
|
FetchedPage,
|
|
10
17
|
FetchOptions,
|
|
11
18
|
RequestAuth,
|
|
19
|
+
ResolvedOptions,
|
|
12
20
|
RetryOptions,
|
|
13
21
|
SearchOptions,
|
|
14
|
-
|
|
22
|
+
SearchOutcome,
|
|
15
23
|
} from "./types";
|
|
16
24
|
import { parseFetchUrl } from "./url";
|
|
17
25
|
|
|
@@ -22,6 +30,24 @@ function effectiveTimeout(configuredMs: number, perCallMs: number | undefined):
|
|
|
22
30
|
return perCallMs === undefined ? configuredMs : Math.min(configuredMs, perCallMs);
|
|
23
31
|
}
|
|
24
32
|
|
|
33
|
+
/**
|
|
34
|
+
* The per-call budget wins outright, unlike timeouts: a model that just saw a
|
|
35
|
+
* truncation marker is asking for more than the default on purpose, so the
|
|
36
|
+
* configured value stays the default rather than a ceiling.
|
|
37
|
+
*/
|
|
38
|
+
function effectiveMaxChars(configured: number, perCall: number | undefined): number {
|
|
39
|
+
if (perCall === undefined || !Number.isFinite(perCall)) return configured;
|
|
40
|
+
return Math.min(MAX_FETCH_CHARS, Math.max(0, Math.trunc(perCall)));
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Composes a request endpoint from the resolved base URL (already
|
|
45
|
+
* slash-stripped by resolveBaseUrl; absent means the production default).
|
|
46
|
+
*/
|
|
47
|
+
function apiUrl(baseUrl: string | undefined, path: string): string {
|
|
48
|
+
return `${baseUrl ?? DEFAULT_API_BASE_URL}${path}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
25
51
|
/**
|
|
26
52
|
* The wire value for ori_question: absent/whitespace-only means "not
|
|
27
53
|
* provided"; longer values are clamped rather than rejected — the field is
|
|
@@ -49,6 +75,14 @@ export interface RequestOptions {
|
|
|
49
75
|
readonly retry: RetryOptions;
|
|
50
76
|
/** Per-call cap in milliseconds; the shorter of this and the configured timeout wins. */
|
|
51
77
|
readonly timeoutMs?: number;
|
|
78
|
+
/** Per-call fetch budget in characters; overrides the configured cap for this call. */
|
|
79
|
+
readonly maxChars?: number;
|
|
80
|
+
/**
|
|
81
|
+
* The resolved API root; adapters pass `config.baseUrl` so every request
|
|
82
|
+
* follows the `baseUrl` knob. Absent falls back to the production default
|
|
83
|
+
* (unit tests composing requests by hand rely on that).
|
|
84
|
+
*/
|
|
85
|
+
readonly baseUrl?: string;
|
|
52
86
|
}
|
|
53
87
|
|
|
54
88
|
export interface SearchRequest extends RequestOptions {
|
|
@@ -58,37 +92,59 @@ export interface SearchRequest extends RequestOptions {
|
|
|
58
92
|
* ori_question to sharpen intent; adapters fill it from tool input.
|
|
59
93
|
*/
|
|
60
94
|
readonly oriQuestion?: string;
|
|
95
|
+
/**
|
|
96
|
+
* Per-call only. The tool input's need_content flag: true asks for each
|
|
97
|
+
* result's page body, sent as the API's need_content and timed against
|
|
98
|
+
* contentTimeoutMs instead of timeoutMs.
|
|
99
|
+
*/
|
|
100
|
+
readonly needContent?: boolean;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The per-request context every adapter assembles from the resolved config:
|
|
105
|
+
* bound auth, retry policy, and base URL, plus any per-call overrides a
|
|
106
|
+
* tool input supplied.
|
|
107
|
+
*/
|
|
108
|
+
export function requestContext(
|
|
109
|
+
config: Pick<ResolvedOptions, "retry" | "baseUrl">,
|
|
110
|
+
apiKey: string,
|
|
111
|
+
signal: AbortSignal,
|
|
112
|
+
perCall?: { readonly timeoutMs?: number; readonly maxChars?: number },
|
|
113
|
+
): RequestOptions {
|
|
114
|
+
return {
|
|
115
|
+
signal,
|
|
116
|
+
auth: { apiKey },
|
|
117
|
+
retry: config.retry,
|
|
118
|
+
baseUrl: config.baseUrl,
|
|
119
|
+
...(perCall?.timeoutMs === undefined ? {} : { timeoutMs: perCall.timeoutMs }),
|
|
120
|
+
...(perCall?.maxChars === undefined ? {} : { maxChars: perCall.maxChars }),
|
|
121
|
+
};
|
|
61
122
|
}
|
|
62
123
|
|
|
63
|
-
export async function searchWeb(
|
|
64
|
-
query: string,
|
|
65
|
-
options: SearchOptions,
|
|
66
|
-
request: SearchRequest,
|
|
67
|
-
): Promise<SearchResult[]> {
|
|
124
|
+
export async function searchWeb(query: string, options: SearchOptions, request: SearchRequest): Promise<SearchOutcome> {
|
|
68
125
|
const trimmed = query.trim();
|
|
69
126
|
if (!trimmed) throw new Error("DeepSeek search query must not be empty");
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
const builtQuery = buildQuery(trimmed, options);
|
|
75
|
-
const builtQueryLength = Array.from(builtQuery).length;
|
|
76
|
-
if (builtQueryLength > MAX_QUERY_CHARS) {
|
|
127
|
+
const trimmedLength = Array.from(trimmed).length;
|
|
128
|
+
if (trimmedLength > MAX_QUERY_CHARS) {
|
|
129
|
+
// The query rides in the message so the model can see what to shorten.
|
|
77
130
|
throw new Error(
|
|
78
|
-
`DeepSeek search query must be ${MAX_QUERY_CHARS} characters or fewer
|
|
79
|
-
`
|
|
80
|
-
`(built query is ${builtQueryLength} characters)`,
|
|
131
|
+
`DeepSeek search query must be ${MAX_QUERY_CHARS} characters or fewer ` +
|
|
132
|
+
`(got ${trimmedLength}: ${JSON.stringify(truncate(trimmed, 64))})`,
|
|
81
133
|
);
|
|
82
134
|
}
|
|
83
135
|
|
|
84
|
-
const
|
|
136
|
+
const composed = composeSearchQuery(trimmed, options);
|
|
137
|
+
const builtQuery = composed.query;
|
|
138
|
+
|
|
139
|
+
const needContent = request.needContent === true;
|
|
140
|
+
const timeout = effectiveTimeout(needContent ? options.contentTimeoutMs : options.timeoutMs, request.timeoutMs);
|
|
85
141
|
const oriQuestion = oriQuestionValue(request.oriQuestion);
|
|
86
142
|
const payload = await postJson<SearchResponse>(
|
|
87
|
-
|
|
143
|
+
apiUrl(request.baseUrl, SEARCH_ENDPOINT_PATH),
|
|
88
144
|
{
|
|
89
145
|
query: builtQuery,
|
|
90
146
|
count: options.count,
|
|
91
|
-
need_content:
|
|
147
|
+
need_content: needContent,
|
|
92
148
|
...(oriQuestion === undefined ? {} : { ori_question: oriQuestion }),
|
|
93
149
|
...(options.localization === undefined ? {} : { localization: options.localization }),
|
|
94
150
|
},
|
|
@@ -99,7 +155,13 @@ export async function searchWeb(
|
|
|
99
155
|
if (!Array.isArray(payload.results)) {
|
|
100
156
|
throw new Error("DeepSeek Search returned a response without a results array");
|
|
101
157
|
}
|
|
102
|
-
|
|
158
|
+
const results = processResults(payload.results as DeepSeekSearchResult[], options);
|
|
159
|
+
return {
|
|
160
|
+
results,
|
|
161
|
+
inputQuery: trimmed,
|
|
162
|
+
effectiveQuery: builtQuery,
|
|
163
|
+
notes: composed.notes,
|
|
164
|
+
};
|
|
103
165
|
}
|
|
104
166
|
|
|
105
167
|
export async function fetchPage(url: URL, options: FetchOptions, request: RequestOptions): Promise<FetchedPage> {
|
|
@@ -108,10 +170,15 @@ export async function fetchPage(url: URL, options: FetchOptions, request: Reques
|
|
|
108
170
|
const target = parseFetchUrl(url.href);
|
|
109
171
|
assertFetchHostPolicy(target, options);
|
|
110
172
|
const timeout = effectiveTimeout(options.timeoutMs, request.timeoutMs);
|
|
111
|
-
const payload = await postJson<FetchResponse>(
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
173
|
+
const payload = await postJson<FetchResponse>(
|
|
174
|
+
apiUrl(request.baseUrl, FETCH_ENDPOINT_PATH),
|
|
175
|
+
{ url: target.href },
|
|
176
|
+
"DeepSeek Fetch",
|
|
177
|
+
{
|
|
178
|
+
...request,
|
|
179
|
+
timeoutMs: timeout,
|
|
180
|
+
},
|
|
181
|
+
);
|
|
115
182
|
const title = typeof payload.title === "string" ? payload.title.trim() : "";
|
|
116
183
|
const content = typeof payload.content === "string" ? payload.content : "";
|
|
117
184
|
if (!content.trim()) {
|
|
@@ -138,7 +205,11 @@ export async function fetchDocument(
|
|
|
138
205
|
): Promise<FetchedDocument> {
|
|
139
206
|
if (typeof url !== "string" && !(url instanceof URL)) throw new Error('"url" must be a string or URL');
|
|
140
207
|
const page = await fetchPage(url instanceof URL ? url : parseFetchUrl(url), options, request);
|
|
141
|
-
const formatted = formatFetchedContent(
|
|
208
|
+
const formatted = formatFetchedContent(
|
|
209
|
+
page.title,
|
|
210
|
+
page.content,
|
|
211
|
+
effectiveMaxChars(options.maxChars, request.maxChars),
|
|
212
|
+
);
|
|
142
213
|
return {
|
|
143
214
|
url: page.url,
|
|
144
215
|
title: page.title,
|
|
@@ -19,6 +19,8 @@ const VOCABULARY = [
|
|
|
19
19
|
{ key: "language", read: (o: ResolvedOptions) => o.search.language },
|
|
20
20
|
{ key: "searchLocalization", read: (o: ResolvedOptions) => o.search.localization },
|
|
21
21
|
{ key: "maxSummaryChars", read: (o: ResolvedOptions) => o.search.maxSummaryChars },
|
|
22
|
+
{ key: "searchContentMaxChars", read: (o: ResolvedOptions) => o.search.contentMaxChars },
|
|
23
|
+
{ key: "searchContentTimeoutMs", read: (o: ResolvedOptions) => o.search.contentTimeoutMs },
|
|
22
24
|
{ key: "searchTimeoutMs", read: (o: ResolvedOptions) => o.search.timeoutMs },
|
|
23
25
|
{ key: "fetchMaxChars", read: (o: ResolvedOptions) => o.fetch.maxChars },
|
|
24
26
|
{ key: "fetchAllowedDomains", read: (o: ResolvedOptions) => o.fetch.allowedDomains },
|
|
@@ -27,6 +29,7 @@ const VOCABULARY = [
|
|
|
27
29
|
{ key: "fetchBlockPrivateHosts", read: (o: ResolvedOptions) => o.fetch.blockPrivateHosts },
|
|
28
30
|
{ key: "maxRetries", read: (o: ResolvedOptions) => o.retry.maxRetries },
|
|
29
31
|
{ key: "retryBaseDelayMs", read: (o: ResolvedOptions) => o.retry.baseDelayMs },
|
|
32
|
+
{ key: "baseUrl", read: (o: ResolvedOptions) => o.baseUrl },
|
|
30
33
|
] as const;
|
|
31
34
|
|
|
32
35
|
/** The config keys in vocabulary order; the shared config file accepts exactly these. */
|
package/src/core/constants.ts
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* The DeepSeek API root every request is composed from. `baseUrl` overrides it
|
|
3
|
+
* (env `DEEPSEEK_BASE_URL`, shared-file key `baseUrl`) so proxies, enterprise
|
|
4
|
+
* gateways, and mock upstreams can intercept retrieval without touching the
|
|
5
|
+
* adapters; the two endpoints below stay the production default.
|
|
6
|
+
*/
|
|
7
|
+
export const DEFAULT_API_BASE_URL = "https://api.deepseek.com";
|
|
8
|
+
export const SEARCH_ENDPOINT_PATH = "/v1/tools/search";
|
|
9
|
+
export const FETCH_ENDPOINT_PATH = "/v1/tools/fetch";
|
|
10
|
+
|
|
11
|
+
export const SEARCH_ENDPOINT = `${DEFAULT_API_BASE_URL}${SEARCH_ENDPOINT_PATH}`;
|
|
12
|
+
export const FETCH_ENDPOINT = `${DEFAULT_API_BASE_URL}${FETCH_ENDPOINT_PATH}`;
|
|
3
13
|
|
|
4
14
|
/**
|
|
5
15
|
* Search defaults are benchmarked against surveyed AI-native search APIs and
|
|
@@ -17,8 +27,25 @@ export const DEFAULT_MAX_RESULTS = 10;
|
|
|
17
27
|
export const DEFAULT_MIN_RERANK = 0;
|
|
18
28
|
export const DEFAULT_MIN_AUTHORITY = 0;
|
|
19
29
|
export const DEFAULT_MAX_SUMMARY_CHARS = 1200;
|
|
30
|
+
/**
|
|
31
|
+
* The per-result budget a need_content page body is clamped to. 1,500 keeps a
|
|
32
|
+
* default count=10 response (bodies plus metadata ≈ 16 KB) under the tightest
|
|
33
|
+
* host output cap, Pi's 20 KB MCP-path truncation — see
|
|
34
|
+
* docs/research/search-inline-content.md.
|
|
35
|
+
*/
|
|
36
|
+
export const DEFAULT_SEARCH_CONTENT_MAX_CHARS = 1500;
|
|
37
|
+
/**
|
|
38
|
+
* The timeout need_content calls use in place of searchTimeoutMs: DeepSeek
|
|
39
|
+
* fetches every result page server-side before responding, which the 30 s
|
|
40
|
+
* search default does not budget for.
|
|
41
|
+
*/
|
|
42
|
+
export const DEFAULT_SEARCH_CONTENT_TIMEOUT_MS = 60000;
|
|
20
43
|
|
|
21
44
|
export const DEFAULT_FETCH_MAX_CHARS = 16000;
|
|
45
|
+
/** The ceiling for a per-call fetch budget, matching the configured knob's range. */
|
|
46
|
+
export const MAX_FETCH_CHARS = 1_000_000;
|
|
47
|
+
/** The API's documented count range top; per-call counts share it. */
|
|
48
|
+
export const MAX_SEARCH_COUNT = 50;
|
|
22
49
|
export const DEFAULT_TIMEOUT_MS = 30000;
|
|
23
50
|
export const MIN_TIMEOUT_MS = 1000;
|
|
24
51
|
export const MAX_TIMEOUT_MS = 120000;
|
|
@@ -45,6 +72,7 @@ export const MAX_RETRY_DELAY_MS = 10000;
|
|
|
45
72
|
|
|
46
73
|
/** The single source for every DEEPSEEK_* environment variable name. */
|
|
47
74
|
export const API_KEY_ENV = "DEEPSEEK_API_KEY";
|
|
75
|
+
export const BASE_URL_ENV = "DEEPSEEK_BASE_URL";
|
|
48
76
|
export const ENABLE_SEARCH_ENV = "DEEPSEEK_ENABLE_SEARCH";
|
|
49
77
|
export const ENABLE_FETCH_ENV = "DEEPSEEK_ENABLE_FETCH";
|
|
50
78
|
export const SEARCH_LOCALIZATION_ENV = "DEEPSEEK_SEARCH_LOCALIZATION";
|
package/src/core/descriptions.ts
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
export const FETCH_SERVICE_LIMITS =
|
|
7
7
|
"DeepSeek does not render JavaScript and cannot reach private networks; PDFs, images, and other binary files are unsupported.";
|
|
8
8
|
|
|
9
|
-
export const FETCH_OUTPUT_TRUNCATED =
|
|
9
|
+
export const FETCH_OUTPUT_TRUNCATED =
|
|
10
|
+
"Output is truncated to the configured size; the optional maxChars parameter raises or lowers that size for one call.";
|
|
10
11
|
|
|
11
12
|
export const FETCH_MARKDOWN_ONLY = 'Only format "markdown" is supported.';
|
|
12
13
|
|
|
@@ -29,7 +30,8 @@ export const SEARCH_TOOL_DESCRIPTION =
|
|
|
29
30
|
`${SEARCH_RANKED_FILTERED} ` +
|
|
30
31
|
`${SEARCH_EMPTY_IS_NORMAL} ` +
|
|
31
32
|
`${SEARCH_QUERY_OPERATORS} ` +
|
|
32
|
-
"To read
|
|
33
|
+
"To read one full page in depth, pass a result URL to the deepseek_fetch tool; to carry each result's " +
|
|
34
|
+
"page body in this response, set need_content to true.";
|
|
33
35
|
|
|
34
36
|
export const FETCH_TOOL_DESCRIPTION =
|
|
35
37
|
"Fetch one web page as clean Markdown through DeepSeek's Fetch API. " +
|
|
@@ -45,3 +47,16 @@ export const SEARCH_ORI_QUESTION_PARAMETER =
|
|
|
45
47
|
"reformulated query. Optional; 1-128 characters, and longer values are truncated.";
|
|
46
48
|
|
|
47
49
|
export const FETCH_URL_PARAMETER = "Absolute http:// or https:// URL to fetch. Credentials are rejected.";
|
|
50
|
+
|
|
51
|
+
export const FETCH_MAX_CHARS_PARAMETER =
|
|
52
|
+
"Optional per-call character budget for this fetch, 0-1,000,000, title included. Overrides the configured cap " +
|
|
53
|
+
"for this call; a [content truncated] marker is appended when the output is cut.";
|
|
54
|
+
|
|
55
|
+
export const SEARCH_COUNT_PARAMETER =
|
|
56
|
+
"Optional number of results for this call, 1-50. Overrides the configured count and result cap for this call; " +
|
|
57
|
+
"billing is per request, not per result.";
|
|
58
|
+
|
|
59
|
+
export const SEARCH_NEED_CONTENT_PARAMETER =
|
|
60
|
+
"Optional; set true to attach each result's page body as Markdown — an excerpt clamped to a configured per-result " +
|
|
61
|
+
"budget, not a full page — in place of its short summary. Slower per call (DeepSeek fetches the result pages) and " +
|
|
62
|
+
"priced the same per request; for a full page, call deepseek_fetch.";
|
package/src/core/env.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { API_KEY_ENV, ENABLE_FETCH_ENV, ENABLE_SEARCH_ENV, SEARCH_LOCALIZATION_ENV } from "./constants";
|
|
1
|
+
import { API_KEY_ENV, BASE_URL_ENV, ENABLE_FETCH_ENV, ENABLE_SEARCH_ENV, SEARCH_LOCALIZATION_ENV } from "./constants";
|
|
2
2
|
import { normalizeDomain } from "./domains";
|
|
3
3
|
import { resolvePluginOptions } from "./options";
|
|
4
4
|
import type { PluginOptions, ResolvedOptions } from "./types";
|
|
@@ -14,6 +14,8 @@ const NUMBER_ENV = {
|
|
|
14
14
|
minAuthority: "DEEPSEEK_SEARCH_MIN_AUTHORITY",
|
|
15
15
|
maxSummaryChars: "DEEPSEEK_SEARCH_MAX_SUMMARY_CHARS",
|
|
16
16
|
searchTimeoutMs: "DEEPSEEK_SEARCH_TIMEOUT_MS",
|
|
17
|
+
searchContentMaxChars: "DEEPSEEK_SEARCH_CONTENT_MAX_CHARS",
|
|
18
|
+
searchContentTimeoutMs: "DEEPSEEK_SEARCH_CONTENT_TIMEOUT_MS",
|
|
17
19
|
fetchMaxChars: "DEEPSEEK_FETCH_MAX_CHARS",
|
|
18
20
|
fetchTimeoutMs: "DEEPSEEK_FETCH_TIMEOUT_MS",
|
|
19
21
|
maxRetries: "DEEPSEEK_MAX_RETRIES",
|
|
@@ -23,6 +25,7 @@ const NUMBER_ENV = {
|
|
|
23
25
|
const TEXT_ENV = {
|
|
24
26
|
language: "DEEPSEEK_SEARCH_LANGUAGE",
|
|
25
27
|
searchLocalization: SEARCH_LOCALIZATION_ENV,
|
|
28
|
+
baseUrl: BASE_URL_ENV,
|
|
26
29
|
} as const;
|
|
27
30
|
|
|
28
31
|
const DOMAINS_ENV = {
|
|
@@ -96,6 +99,8 @@ export function envOptionsReport(env: Env = process.env): EnvOptionsReport {
|
|
|
96
99
|
putText(options, "searchLocalization", env);
|
|
97
100
|
putNumber(options, "maxSummaryChars", env, warnings);
|
|
98
101
|
putNumber(options, "searchTimeoutMs", env, warnings);
|
|
102
|
+
putNumber(options, "searchContentMaxChars", env, warnings);
|
|
103
|
+
putNumber(options, "searchContentTimeoutMs", env, warnings);
|
|
99
104
|
|
|
100
105
|
putNumber(options, "fetchMaxChars", env, warnings);
|
|
101
106
|
putDomains(options, "fetchAllowedDomains", env, warnings);
|
|
@@ -107,6 +112,7 @@ export function envOptionsReport(env: Env = process.env): EnvOptionsReport {
|
|
|
107
112
|
|
|
108
113
|
putNumber(options, "maxRetries", env, warnings);
|
|
109
114
|
putNumber(options, "retryBaseDelayMs", env, warnings);
|
|
115
|
+
putText(options, "baseUrl", env);
|
|
110
116
|
|
|
111
117
|
const known = new Set(OPTION_ENV_VARS);
|
|
112
118
|
for (const key of Object.keys(env)) {
|
package/src/core/format.ts
CHANGED
|
@@ -85,20 +85,49 @@ export function formatTimeoutError(operation: string, timeoutMs: number): string
|
|
|
85
85
|
return `${operation} timed out after ${timeoutMs} ms`;
|
|
86
86
|
}
|
|
87
87
|
|
|
88
|
+
/**
|
|
89
|
+
* The query context shown to the model above the results: emitted only when
|
|
90
|
+
* the effective query differs from what the model sent (operators applied or
|
|
91
|
+
* degradation happened), so an unfiltered search adds no header noise.
|
|
92
|
+
*/
|
|
93
|
+
export interface SearchQueryContext {
|
|
94
|
+
readonly inputQuery: string;
|
|
95
|
+
readonly effectiveQuery: string;
|
|
96
|
+
readonly notes: readonly string[];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function searchHeader(context: SearchQueryContext | undefined): string | undefined {
|
|
100
|
+
if (!context) return undefined;
|
|
101
|
+
if (context.effectiveQuery === context.inputQuery && context.notes.length === 0) return undefined;
|
|
102
|
+
const header = `Searched for: ${context.effectiveQuery}`;
|
|
103
|
+
return context.notes.length > 0 ? `${header} — ${context.notes.join("; ")}` : header;
|
|
104
|
+
}
|
|
105
|
+
|
|
88
106
|
/**
|
|
89
107
|
* Renders processed search results as Markdown for a text content block.
|
|
90
108
|
* Hosts that ignore structured output read this text.
|
|
91
109
|
*/
|
|
92
|
-
export function formatSearchResults(results: readonly SearchResult[]): string {
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
110
|
+
export function formatSearchResults(results: readonly SearchResult[], context?: SearchQueryContext): string {
|
|
111
|
+
const header = searchHeader(context);
|
|
112
|
+
if (results.length === 0) {
|
|
113
|
+
return header ? `${header}\n\nNo results found.` : "No results found.";
|
|
114
|
+
}
|
|
115
|
+
const body = results
|
|
96
116
|
.map((result, index) => {
|
|
97
117
|
const title = result.title?.trim() || result.url;
|
|
98
118
|
const marker = formatQualityMarker(result);
|
|
99
119
|
const lines = [`${index + 1}. [${escapeBrackets(title)}](${result.url})${marker ? ` ${marker}` : ""}`];
|
|
100
120
|
if (result.content) {
|
|
101
|
-
|
|
121
|
+
// Page bodies keep their Markdown line structure, indented into the
|
|
122
|
+
// list item; digests compact to one line, as summaries always did.
|
|
123
|
+
lines.push(
|
|
124
|
+
result.contentKind === "page"
|
|
125
|
+
? result.content
|
|
126
|
+
.split("\n")
|
|
127
|
+
.map((line) => ` ${line}`)
|
|
128
|
+
.join("\n")
|
|
129
|
+
: ` ${result.content.replaceAll("\n", " ")}`,
|
|
130
|
+
);
|
|
102
131
|
}
|
|
103
132
|
if (result.published !== undefined) {
|
|
104
133
|
lines.push(` Published: ${formatPublished(result.published)}`);
|
|
@@ -106,6 +135,7 @@ export function formatSearchResults(results: readonly SearchResult[]): string {
|
|
|
106
135
|
return lines.join("\n");
|
|
107
136
|
})
|
|
108
137
|
.join("\n\n");
|
|
138
|
+
return header ? `${header}\n\n${body}` : body;
|
|
109
139
|
}
|
|
110
140
|
|
|
111
141
|
/**
|
package/src/core/options.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isIP } from "node:net";
|
|
2
2
|
import {
|
|
3
|
+
DEFAULT_API_BASE_URL,
|
|
3
4
|
DEFAULT_API_KEY_SOURCE,
|
|
4
5
|
DEFAULT_FETCH_MAX_CHARS,
|
|
5
6
|
DEFAULT_MAX_RESULTS,
|
|
@@ -8,10 +9,13 @@ import {
|
|
|
8
9
|
DEFAULT_MIN_AUTHORITY,
|
|
9
10
|
DEFAULT_MIN_RERANK,
|
|
10
11
|
DEFAULT_RETRY_BASE_DELAY_MS,
|
|
12
|
+
DEFAULT_SEARCH_CONTENT_MAX_CHARS,
|
|
13
|
+
DEFAULT_SEARCH_CONTENT_TIMEOUT_MS,
|
|
11
14
|
DEFAULT_SEARCH_COUNT,
|
|
12
15
|
DEFAULT_TIMEOUT_MS,
|
|
13
16
|
MAX_RETRIES,
|
|
14
17
|
MAX_RETRY_DELAY_MS,
|
|
18
|
+
MAX_SEARCH_COUNT,
|
|
15
19
|
MAX_TIMEOUT_MS,
|
|
16
20
|
MIN_TIMEOUT_MS,
|
|
17
21
|
} from "./constants";
|
|
@@ -32,9 +36,22 @@ export function resolvePluginOptions(input?: PluginOptions): ResolvedOptions {
|
|
|
32
36
|
fetch: resolveFetchOptions(input),
|
|
33
37
|
auth: resolveAuthOptions(input),
|
|
34
38
|
retry: resolveRetryOptions(input),
|
|
39
|
+
baseUrl: resolveBaseUrl(input?.baseUrl),
|
|
35
40
|
};
|
|
36
41
|
}
|
|
37
42
|
|
|
43
|
+
/**
|
|
44
|
+
* The both-disabled refusal both server surfaces share: undefined when a tool
|
|
45
|
+
* stays enabled, otherwise one message naming why both are off.
|
|
46
|
+
*/
|
|
47
|
+
export function bothDisabledPrecheck(
|
|
48
|
+
config: Pick<ResolvedOptions, "search" | "fetch">,
|
|
49
|
+
reason: string,
|
|
50
|
+
): string | undefined {
|
|
51
|
+
if (config.search.enabled || config.fetch.enabled) return undefined;
|
|
52
|
+
return `Both tools are disabled (${reason}); at least one must stay enabled.`;
|
|
53
|
+
}
|
|
54
|
+
|
|
38
55
|
export function resolveSearchOptions(input?: PluginOptions): SearchOptions {
|
|
39
56
|
const count = clampInt(input?.count, DEFAULT_SEARCH_COUNT, 1, 50);
|
|
40
57
|
return {
|
|
@@ -52,6 +69,13 @@ export function resolveSearchOptions(input?: PluginOptions): SearchOptions {
|
|
|
52
69
|
language: optionalText(input?.language),
|
|
53
70
|
localization: localizationValue(input?.searchLocalization),
|
|
54
71
|
maxSummaryChars: clampInt(input?.maxSummaryChars, DEFAULT_MAX_SUMMARY_CHARS, 0, 1_000_000),
|
|
72
|
+
contentMaxChars: clampInt(input?.searchContentMaxChars, DEFAULT_SEARCH_CONTENT_MAX_CHARS, 0, 1_000_000),
|
|
73
|
+
contentTimeoutMs: clampInt(
|
|
74
|
+
input?.searchContentTimeoutMs,
|
|
75
|
+
DEFAULT_SEARCH_CONTENT_TIMEOUT_MS,
|
|
76
|
+
MIN_TIMEOUT_MS,
|
|
77
|
+
MAX_TIMEOUT_MS,
|
|
78
|
+
),
|
|
55
79
|
timeoutMs: clampInt(input?.searchTimeoutMs, DEFAULT_TIMEOUT_MS, MIN_TIMEOUT_MS, MAX_TIMEOUT_MS),
|
|
56
80
|
enabled: input?.searchEnabled !== false,
|
|
57
81
|
};
|
|
@@ -82,6 +106,38 @@ export function resolveRetryOptions(input?: PluginOptions): RetryOptions {
|
|
|
82
106
|
};
|
|
83
107
|
}
|
|
84
108
|
|
|
109
|
+
/**
|
|
110
|
+
* Constraint: only an http(s) URL with no fragment survives, trailing slashes
|
|
111
|
+
* stripped so endpoint composition never doubles them. Anything else — a bare
|
|
112
|
+
* hostname, a non-http scheme, an empty string — drops to the production
|
|
113
|
+
* default, the same drop-and-default contract as every other invalid knob.
|
|
114
|
+
*/
|
|
115
|
+
export function resolveBaseUrl(value: unknown): string {
|
|
116
|
+
if (typeof value !== "string") return DEFAULT_API_BASE_URL;
|
|
117
|
+
const trimmed = value.trim();
|
|
118
|
+
if (!trimmed) return DEFAULT_API_BASE_URL;
|
|
119
|
+
let parsed: URL;
|
|
120
|
+
try {
|
|
121
|
+
parsed = new URL(trimmed);
|
|
122
|
+
} catch {
|
|
123
|
+
return DEFAULT_API_BASE_URL;
|
|
124
|
+
}
|
|
125
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return DEFAULT_API_BASE_URL;
|
|
126
|
+
return trimmed.replace(/\/+$/, "");
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Applies a per-call result count to resolved search options. The count the
|
|
131
|
+
* model asked for is what it wants to see, so it sets both the request count
|
|
132
|
+
* and the post-filter cap for that call; the configured knobs stay the
|
|
133
|
+
* default, not a ceiling.
|
|
134
|
+
*/
|
|
135
|
+
export function withPerCallCount(options: SearchOptions, count: number | undefined): SearchOptions {
|
|
136
|
+
if (count === undefined || !Number.isFinite(count)) return options;
|
|
137
|
+
const clamped = Math.min(MAX_SEARCH_COUNT, Math.max(1, Math.trunc(count)));
|
|
138
|
+
return { ...options, count: clamped, maxResults: clamped };
|
|
139
|
+
}
|
|
140
|
+
|
|
85
141
|
/**
|
|
86
142
|
* Resolves the per-call timeout from the plugin cap and the built-in tool's
|
|
87
143
|
* `timeout` field (seconds). The shorter limit wins.
|