@fffangchenghao/dsh-web-search 0.12.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/README.md +206 -43
- package/README.zh-CN.md +206 -43
- package/client.js +16 -0
- package/lib/config.d.ts +9 -0
- package/lib/config.js +14 -0
- package/lib/engines.d.ts +41 -1
- package/lib/engines.js +33 -2
- package/lib/fetch.d.ts +26 -0
- package/lib/fetch.js +187 -8
- package/lib/index.d.ts +6 -0
- package/lib/index.js +23 -1
- package/lib/rank.js +4 -1
- package/lib/read-page.d.ts +136 -0
- package/lib/read-page.js +496 -0
- package/lib/render.d.ts +193 -0
- package/lib/render.js +354 -0
- package/lib/transport.js +19 -0
- package/lib/types.d.ts +6 -0
- package/package.json +1 -1
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import type { FetchProvider } from './fetch.ts';
|
|
2
|
+
import type { FetchFormat } from './markdown.ts';
|
|
3
|
+
import type { RenderMode, RenderReport } from './render.ts';
|
|
4
|
+
import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
|
|
5
|
+
/** The name this tool prefers. */
|
|
6
|
+
export declare const READ_PAGE_TOOL_NAME = "read_page";
|
|
7
|
+
/** The name used when another plugin already registered `read_page`. */
|
|
8
|
+
export declare const READ_PAGE_TOOL_FALLBACK_NAME = "web_read_page";
|
|
9
|
+
/** Smallest chunk a caller may request, matching `fetchMaxLength`'s floor. */
|
|
10
|
+
export declare const READ_PAGE_MIN_LENGTH = 1000;
|
|
11
|
+
/** Largest chunk a caller may request, matching `fetchMaxLength`'s ceiling. */
|
|
12
|
+
export declare const READ_PAGE_MAX_LENGTH = 200000;
|
|
13
|
+
/** Default chunk size when neither the caller nor the config names one. */
|
|
14
|
+
export declare const READ_PAGE_DEFAULT_LENGTH = 20000;
|
|
15
|
+
/**
|
|
16
|
+
* Largest body this tool will hold. A page beyond it is cut once, after
|
|
17
|
+
* conversion, and reported as `truncated`; chunks never pay this cost.
|
|
18
|
+
*/
|
|
19
|
+
export declare const READ_PAGE_INTERNAL_MAX_CHARS = 2000000;
|
|
20
|
+
/**
|
|
21
|
+
* Byte budget handed to the fetch ladder. Four bytes per UTF-8 code point is
|
|
22
|
+
* the worst case, so this is large enough for the full character cap while
|
|
23
|
+
* still bounding what one response can allocate.
|
|
24
|
+
*/
|
|
25
|
+
export declare const READ_PAGE_MAX_BYTES: number;
|
|
26
|
+
/** How many converted pages stay cached per mounted tool. */
|
|
27
|
+
export declare const READ_PAGE_CACHE_ENTRIES = 8;
|
|
28
|
+
/** Internal fetch provider ids, one per output format. */
|
|
29
|
+
export declare const READ_PAGE_PROVIDER_ID = "http-proxy-read-page";
|
|
30
|
+
/** The fields a `read_page` result carries; see the module doc for semantics. */
|
|
31
|
+
export interface ReadPageValue {
|
|
32
|
+
/** The URL exactly as the caller supplied it (trimmed). */
|
|
33
|
+
url: string;
|
|
34
|
+
/** The requested character range, with no hint line appended. */
|
|
35
|
+
content: string;
|
|
36
|
+
/** First character of `content`. */
|
|
37
|
+
offset: number;
|
|
38
|
+
/** Offset to pass on the next call, or `null` when the page is finished. */
|
|
39
|
+
nextOffset: number | null;
|
|
40
|
+
/** Length of the complete converted body, possibly cut by the safety cap. */
|
|
41
|
+
totalLength: number;
|
|
42
|
+
/** True when the source body itself was cut, so `totalLength` is a lower bound. */
|
|
43
|
+
truncated: boolean;
|
|
44
|
+
/** Progress through the document for this call's `maxLength`; `index` is 1-based. */
|
|
45
|
+
chunk: {
|
|
46
|
+
index: number;
|
|
47
|
+
count: number;
|
|
48
|
+
};
|
|
49
|
+
/** Fetch failures, in the same shape `domain_search` and `search_many` use. */
|
|
50
|
+
failures: string[];
|
|
51
|
+
/** Why the optional browser rung did or did not run, when the provider says. */
|
|
52
|
+
render?: RenderReport;
|
|
53
|
+
}
|
|
54
|
+
/** Settings read live for every call, so a Settings commit applies without a remount. */
|
|
55
|
+
export interface ReadPageSettings {
|
|
56
|
+
/** Default chunk size, from `fetchMaxLength`. */
|
|
57
|
+
maxLength: number;
|
|
58
|
+
/** Default output style, from `fetchFormat`. */
|
|
59
|
+
format: FetchFormat;
|
|
60
|
+
/** Per-attempt fetch deadline, from `timeoutMs`. */
|
|
61
|
+
timeoutMs?: number;
|
|
62
|
+
/** Browser-render policy, from `renderJs`; read again by each internal fetch. */
|
|
63
|
+
renderJs?: RenderMode;
|
|
64
|
+
/** Browser-render deadline, from `renderTimeoutMs`. */
|
|
65
|
+
renderTimeoutMs?: number;
|
|
66
|
+
/** Tunnel URL, from `proxy`. */
|
|
67
|
+
proxy?: string;
|
|
68
|
+
/** AnySearch key, when the extraction rung is configured. */
|
|
69
|
+
anysearchApiKey?: string;
|
|
70
|
+
}
|
|
71
|
+
/** Construction seams, so the definition is testable without a network. */
|
|
72
|
+
export interface ReadPageOptions {
|
|
73
|
+
/**
|
|
74
|
+
* Build the internal provider for one output format. The default is
|
|
75
|
+
* `createFetchProvider`, so production always goes through the same ladder as
|
|
76
|
+
* `web_fetch`; tests substitute a recorder.
|
|
77
|
+
*/
|
|
78
|
+
createProvider?: (format: FetchFormat, settings: ReadPageSettings) => FetchProvider;
|
|
79
|
+
/** Registration diagnostics. */
|
|
80
|
+
warn?: (message: string) => void;
|
|
81
|
+
}
|
|
82
|
+
/** A tool definition in the host registry's native shape. */
|
|
83
|
+
export interface ReadPageDefinition {
|
|
84
|
+
name: string;
|
|
85
|
+
description: string;
|
|
86
|
+
parameters: ToolJsonSchema;
|
|
87
|
+
output: {
|
|
88
|
+
schema: ToolJsonSchema;
|
|
89
|
+
render: (args: unknown, value: ReadPageValue) => ToolContent[];
|
|
90
|
+
};
|
|
91
|
+
timeoutMs?: number;
|
|
92
|
+
isConcurrencySafe?: (args: unknown) => boolean;
|
|
93
|
+
execute: (args: unknown, exec: {
|
|
94
|
+
signal?: AbortSignal;
|
|
95
|
+
}) => Promise<ReadPageValue>;
|
|
96
|
+
}
|
|
97
|
+
/** The slice of the tool registry this module needs. */
|
|
98
|
+
export interface ReadPageRegistryLike {
|
|
99
|
+
register(definition: ReadPageDefinition): unknown;
|
|
100
|
+
get?(name: string, scope?: unknown): unknown;
|
|
101
|
+
}
|
|
102
|
+
/** A successful registration plus the exact way to undo it. */
|
|
103
|
+
export interface ReadPageRegistration {
|
|
104
|
+
name: string;
|
|
105
|
+
dispose: () => void;
|
|
106
|
+
}
|
|
107
|
+
/** Format the model-facing result text: the chunk, then exactly one hint line. */
|
|
108
|
+
export declare function formatReadPageValue(value: ReadPageValue): string;
|
|
109
|
+
/**
|
|
110
|
+
* Build the tool definition the host registry accepts.
|
|
111
|
+
* @param name - `read_page`, or the fallback when that name is taken.
|
|
112
|
+
* @param readSettings - read the live config for every call.
|
|
113
|
+
* @param options - test seams and diagnostics.
|
|
114
|
+
* @returns a registry-ready definition.
|
|
115
|
+
*/
|
|
116
|
+
export declare function createReadPageDefinition(name: string, readSettings: () => ReadPageSettings, options?: ReadPageOptions): ReadPageDefinition;
|
|
117
|
+
/**
|
|
118
|
+
* Pick the tool name this deployment can accept.
|
|
119
|
+
* @param tools - the live registry.
|
|
120
|
+
* @returns the preferred or fallback name.
|
|
121
|
+
*/
|
|
122
|
+
export declare function pickReadPageToolName(tools: ReadPageRegistryLike): string;
|
|
123
|
+
/**
|
|
124
|
+
* Register the tool when a registry is available.
|
|
125
|
+
*
|
|
126
|
+
* The failure paths mirror `search_many`: a duplicate name falls back, and a
|
|
127
|
+
* registry rejection is reported and swallowed so the other tools keep working.
|
|
128
|
+
* The returned registration owns the host disposer, so plugin disposal removes
|
|
129
|
+
* exactly the definition this call added.
|
|
130
|
+
*
|
|
131
|
+
* @param tools - the live registry.
|
|
132
|
+
* @param readSettings - live config reader.
|
|
133
|
+
* @param options - test seams and diagnostics.
|
|
134
|
+
* @returns the registered name and its disposer, or `undefined` when skipped.
|
|
135
|
+
*/
|
|
136
|
+
export declare function registerReadPageTool(tools: ReadPageRegistryLike, readSettings: () => ReadPageSettings, options?: ReadPageOptions): ReadPageRegistration | undefined;
|
package/lib/read-page.js
ADDED
|
@@ -0,0 +1,496 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-facing `read_page` tool: true resumable reading of a long page.
|
|
3
|
+
*
|
|
4
|
+
* The host's `web_fetch` request has exactly one field, `url` (see
|
|
5
|
+
* `@deepseek-ai/dsh-web`'s `WebFetchRequest`), so the host itself cannot
|
|
6
|
+
* continue a page from the middle: every call starts at the top and the tail is
|
|
7
|
+
* lost behind the character budget. A plugin, however, owns its own tool
|
|
8
|
+
* registry entry, so it can offer the missing `offset`.
|
|
9
|
+
*
|
|
10
|
+
* The design is two-layered on purpose:
|
|
11
|
+
*
|
|
12
|
+
* 1. one internal `createFetchProvider` fetch obtains the whole converted body,
|
|
13
|
+
* so the direct/tunnel/render/extraction ladder, its proxy handling and its
|
|
14
|
+
* live `renderJs` settings are reused rather than reimplemented;
|
|
15
|
+
* 2. that body is then sliced for the caller. A 2,000,000-character safety cap
|
|
16
|
+
* keeps one hostile page from pinning memory, and it is applied once, after
|
|
17
|
+
* conversion, never to an individual chunk.
|
|
18
|
+
*
|
|
19
|
+
* Successful bodies are cached per URL, output format and fetch-affecting
|
|
20
|
+
* settings, so walking a page chunk by chunk issues exactly one provider fetch
|
|
21
|
+
* (the test suite counts the calls). Failures are not cached. The cache is a
|
|
22
|
+
* bounded eight-entry LRU with no TTL: a long-lived plugin can serve an earlier
|
|
23
|
+
* capture until another page evicts it, which is the price of exact paging.
|
|
24
|
+
*
|
|
25
|
+
* `truncated` describes the *source*: it is true when the safety cap or the
|
|
26
|
+
* transport cut the body, so `totalLength` is then a lower bound. "There is
|
|
27
|
+
* more to read" is signalled by `nextOffset`, not by `truncated`.
|
|
28
|
+
*
|
|
29
|
+
* The end-of-chunk hint is generated here, in `output.render`, and is
|
|
30
|
+
* deliberately **not** part of `content`: concatenating the `content` of every
|
|
31
|
+
* chunk therefore reproduces the complete body character for character.
|
|
32
|
+
*
|
|
33
|
+
* Offsets and lengths are JavaScript string code units, matching `String.length`
|
|
34
|
+
* and the existing fetch budget. A chunk boundary is moved back one code unit
|
|
35
|
+
* when it would split a surrogate pair, so an emoji is never broken across two
|
|
36
|
+
* chunks; `nextOffset` follows that adjusted boundary.
|
|
37
|
+
*
|
|
38
|
+
* @module @fangchenghao/dsh-web-search/read-page
|
|
39
|
+
*/
|
|
40
|
+
import { createFetchProvider } from "./fetch.js";
|
|
41
|
+
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText } from "./markdown.js";
|
|
42
|
+
import { DEFAULT_RENDER_TIMEOUT_MS } from "./render.js";
|
|
43
|
+
/** The name this tool prefers. */
|
|
44
|
+
export const READ_PAGE_TOOL_NAME = 'read_page';
|
|
45
|
+
/** The name used when another plugin already registered `read_page`. */
|
|
46
|
+
export const READ_PAGE_TOOL_FALLBACK_NAME = 'web_read_page';
|
|
47
|
+
/** Smallest chunk a caller may request, matching `fetchMaxLength`'s floor. */
|
|
48
|
+
export const READ_PAGE_MIN_LENGTH = 1_000;
|
|
49
|
+
/** Largest chunk a caller may request, matching `fetchMaxLength`'s ceiling. */
|
|
50
|
+
export const READ_PAGE_MAX_LENGTH = 200_000;
|
|
51
|
+
/** Default chunk size when neither the caller nor the config names one. */
|
|
52
|
+
export const READ_PAGE_DEFAULT_LENGTH = DEFAULT_FETCH_MAX_LENGTH;
|
|
53
|
+
/**
|
|
54
|
+
* Largest body this tool will hold. A page beyond it is cut once, after
|
|
55
|
+
* conversion, and reported as `truncated`; chunks never pay this cost.
|
|
56
|
+
*/
|
|
57
|
+
export const READ_PAGE_INTERNAL_MAX_CHARS = 2_000_000;
|
|
58
|
+
/**
|
|
59
|
+
* Byte budget handed to the fetch ladder. Four bytes per UTF-8 code point is
|
|
60
|
+
* the worst case, so this is large enough for the full character cap while
|
|
61
|
+
* still bounding what one response can allocate.
|
|
62
|
+
*/
|
|
63
|
+
export const READ_PAGE_MAX_BYTES = READ_PAGE_INTERNAL_MAX_CHARS * 4;
|
|
64
|
+
/** How many converted pages stay cached per mounted tool. */
|
|
65
|
+
export const READ_PAGE_CACHE_ENTRIES = 8;
|
|
66
|
+
/** Internal fetch provider ids, one per output format. */
|
|
67
|
+
export const READ_PAGE_PROVIDER_ID = 'http-proxy-read-page';
|
|
68
|
+
/** True for a value usable as a record. */
|
|
69
|
+
function isRecord(value) {
|
|
70
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
71
|
+
}
|
|
72
|
+
/** The parameter schema, in the registry's raw JSON-Schema dialect. */
|
|
73
|
+
function parameterSchema() {
|
|
74
|
+
return {
|
|
75
|
+
type: 'object',
|
|
76
|
+
additionalProperties: false,
|
|
77
|
+
properties: {
|
|
78
|
+
url: {
|
|
79
|
+
type: 'string',
|
|
80
|
+
description: 'Absolute http(s) URL of the page to read.',
|
|
81
|
+
},
|
|
82
|
+
offset: {
|
|
83
|
+
type: 'integer',
|
|
84
|
+
description: 'First character to return, zero-based. Omit or use 0 for the first chunk; then pass the previous response\'s nextOffset.',
|
|
85
|
+
},
|
|
86
|
+
maxLength: {
|
|
87
|
+
type: 'integer',
|
|
88
|
+
description: `Characters to return, ${READ_PAGE_MIN_LENGTH}–${READ_PAGE_MAX_LENGTH}. Omit to use the configured fetchMaxLength.`,
|
|
89
|
+
},
|
|
90
|
+
format: {
|
|
91
|
+
type: 'string',
|
|
92
|
+
enum: ['markdown', 'text'],
|
|
93
|
+
description: 'Output style: markdown keeps headings, lists, tables and links; text keeps only the words. Omit to use the configured fetchFormat.',
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
required: ['url'],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/** The output schema, in the same dialect. */
|
|
100
|
+
function outputSchema() {
|
|
101
|
+
return {
|
|
102
|
+
type: 'object',
|
|
103
|
+
additionalProperties: false,
|
|
104
|
+
properties: {
|
|
105
|
+
url: { type: 'string' },
|
|
106
|
+
content: { type: 'string' },
|
|
107
|
+
offset: { type: 'integer' },
|
|
108
|
+
// A nullable integer is not expressible in the small dialect this module
|
|
109
|
+
// uses, so this field deliberately carries no `type`: its description is
|
|
110
|
+
// the contract.
|
|
111
|
+
nextOffset: { description: 'Offset to pass on the next call, or null when the page is finished. A surrogate-pair boundary can make it one less than offset+maxLength.' },
|
|
112
|
+
totalLength: { type: 'integer' },
|
|
113
|
+
truncated: { type: 'boolean' },
|
|
114
|
+
chunk: {
|
|
115
|
+
type: 'object',
|
|
116
|
+
description: 'Progress for this call\'s maxLength; index is 1-based.',
|
|
117
|
+
additionalProperties: false,
|
|
118
|
+
properties: {
|
|
119
|
+
index: { type: 'integer' },
|
|
120
|
+
count: { type: 'integer' },
|
|
121
|
+
},
|
|
122
|
+
required: ['index', 'count'],
|
|
123
|
+
},
|
|
124
|
+
failures: { type: 'array', items: { type: 'string' } },
|
|
125
|
+
render: {
|
|
126
|
+
type: 'object',
|
|
127
|
+
additionalProperties: false,
|
|
128
|
+
properties: {
|
|
129
|
+
mode: { type: 'string', enum: ['auto', 'always', 'off'] },
|
|
130
|
+
attempted: { type: 'boolean' },
|
|
131
|
+
applied: { type: 'boolean' },
|
|
132
|
+
reason: { type: 'string' },
|
|
133
|
+
},
|
|
134
|
+
required: ['mode', 'attempted', 'applied', 'reason'],
|
|
135
|
+
},
|
|
136
|
+
},
|
|
137
|
+
required: ['url', 'content', 'offset', 'nextOffset', 'totalLength', 'truncated', 'chunk', 'failures'],
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
/** Parse and validate the URL, throwing a message the model can act on. */
|
|
141
|
+
function parseUrl(raw) {
|
|
142
|
+
if (typeof raw !== 'string')
|
|
143
|
+
throw new Error('read_page requires a url string');
|
|
144
|
+
const url = raw.trim();
|
|
145
|
+
if (url.length === 0)
|
|
146
|
+
throw new Error('read_page requires a non-empty url');
|
|
147
|
+
let parsed;
|
|
148
|
+
try {
|
|
149
|
+
parsed = new URL(url);
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
throw new Error(`read_page: not a valid absolute URL: ${JSON.stringify(url)}`);
|
|
153
|
+
}
|
|
154
|
+
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
155
|
+
throw new Error(`read_page: url must use http or https, got ${parsed.protocol}`);
|
|
156
|
+
}
|
|
157
|
+
if (parsed.hostname.length === 0) {
|
|
158
|
+
throw new Error(`read_page: url has no host: ${JSON.stringify(url)}`);
|
|
159
|
+
}
|
|
160
|
+
return url;
|
|
161
|
+
}
|
|
162
|
+
/** Parse the optional zero-based character offset. */
|
|
163
|
+
function parseOffset(raw) {
|
|
164
|
+
if (raw === undefined)
|
|
165
|
+
return 0;
|
|
166
|
+
if (typeof raw !== 'number' || !Number.isSafeInteger(raw) || raw < 0) {
|
|
167
|
+
throw new Error(`read_page offset must be a safe integer >= 0, got ${JSON.stringify(raw)}`);
|
|
168
|
+
}
|
|
169
|
+
return raw;
|
|
170
|
+
}
|
|
171
|
+
/** Normalise the config defaults, never throwing because of a bad config value. */
|
|
172
|
+
function defaultLength(settings) {
|
|
173
|
+
const value = settings.maxLength;
|
|
174
|
+
return Number.isSafeInteger(value) && value >= READ_PAGE_MIN_LENGTH && value <= READ_PAGE_MAX_LENGTH
|
|
175
|
+
? value
|
|
176
|
+
: READ_PAGE_DEFAULT_LENGTH;
|
|
177
|
+
}
|
|
178
|
+
/** Normalise the configured default format. */
|
|
179
|
+
function defaultFormat(settings) {
|
|
180
|
+
return settings.format === 'text' ? 'text' : 'markdown';
|
|
181
|
+
}
|
|
182
|
+
/** Parse the optional chunk size against the documented range. */
|
|
183
|
+
function parseMaxLength(raw, fallback) {
|
|
184
|
+
if (raw === undefined)
|
|
185
|
+
return fallback;
|
|
186
|
+
if (typeof raw !== 'number' || !Number.isSafeInteger(raw)
|
|
187
|
+
|| raw < READ_PAGE_MIN_LENGTH || raw > READ_PAGE_MAX_LENGTH) {
|
|
188
|
+
throw new Error(`read_page maxLength must be an integer between ${READ_PAGE_MIN_LENGTH} and ${READ_PAGE_MAX_LENGTH}, got ${JSON.stringify(raw)}`);
|
|
189
|
+
}
|
|
190
|
+
return raw;
|
|
191
|
+
}
|
|
192
|
+
/** Parse the optional output style. */
|
|
193
|
+
function parseFormat(raw, fallback) {
|
|
194
|
+
if (raw === undefined)
|
|
195
|
+
return fallback;
|
|
196
|
+
if (raw !== 'markdown' && raw !== 'text') {
|
|
197
|
+
throw new Error(`read_page format must be "markdown" or "text", got ${JSON.stringify(raw)}`);
|
|
198
|
+
}
|
|
199
|
+
return raw;
|
|
200
|
+
}
|
|
201
|
+
/** Parse model arguments into a validated shape, throwing on nonsense. */
|
|
202
|
+
function parseArgs(args, settings) {
|
|
203
|
+
if (!isRecord(args))
|
|
204
|
+
throw new Error('read_page requires arguments');
|
|
205
|
+
return {
|
|
206
|
+
url: parseUrl(args['url']),
|
|
207
|
+
offset: parseOffset(args['offset']),
|
|
208
|
+
maxLength: parseMaxLength(args['maxLength'], defaultLength(settings)),
|
|
209
|
+
format: parseFormat(args['format'], defaultFormat(settings)),
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
/** A signature for everything that can change what the internal fetch returns. */
|
|
213
|
+
function fetchSignature(format, settings) {
|
|
214
|
+
return [
|
|
215
|
+
format,
|
|
216
|
+
settings.renderJs ?? 'auto',
|
|
217
|
+
settings.renderTimeoutMs ?? '',
|
|
218
|
+
settings.timeoutMs ?? '',
|
|
219
|
+
settings.proxy ?? '',
|
|
220
|
+
settings.anysearchApiKey ?? '',
|
|
221
|
+
].join('\u0000');
|
|
222
|
+
}
|
|
223
|
+
/** Cut an over-long body once, without leaving half of a surrogate pair. */
|
|
224
|
+
function capBody(content) {
|
|
225
|
+
if (content.length <= READ_PAGE_INTERNAL_MAX_CHARS)
|
|
226
|
+
return { content, truncated: false };
|
|
227
|
+
let cut = content.slice(0, READ_PAGE_INTERNAL_MAX_CHARS);
|
|
228
|
+
const last = cut.charCodeAt(cut.length - 1);
|
|
229
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
230
|
+
cut = cut.slice(0, -1);
|
|
231
|
+
return { content: cut, truncated: true };
|
|
232
|
+
}
|
|
233
|
+
/** Build the production provider: the same ladder, with an effectively unlimited budget. */
|
|
234
|
+
function createDefaultProvider(format, settings, readSettings) {
|
|
235
|
+
const proxy = (settings.proxy ?? '').trim();
|
|
236
|
+
const apiKey = (settings.anysearchApiKey ?? '').trim();
|
|
237
|
+
const options = {
|
|
238
|
+
id: `${READ_PAGE_PROVIDER_ID}-${format}`,
|
|
239
|
+
format,
|
|
240
|
+
// No per-request truncation: `read_page` applies one cap to the whole body
|
|
241
|
+
// and then slices it. The huge budget makes `truncateFetchText` a no-op.
|
|
242
|
+
maxLength: Number.MAX_SAFE_INTEGER,
|
|
243
|
+
maxBytes: READ_PAGE_MAX_BYTES,
|
|
244
|
+
// The renderer policy stays live for the lifetime of this provider.
|
|
245
|
+
renderJs: () => readSettings().renderJs ?? 'auto',
|
|
246
|
+
renderTimeoutMs: () => readSettings().renderTimeoutMs ?? DEFAULT_RENDER_TIMEOUT_MS,
|
|
247
|
+
...(settings.timeoutMs === undefined ? {} : { timeoutMs: settings.timeoutMs }),
|
|
248
|
+
...(proxy.length === 0 ? {} : { proxy }),
|
|
249
|
+
...(apiKey.length === 0 ? {} : { anysearchApiKey: apiKey }),
|
|
250
|
+
};
|
|
251
|
+
return createFetchProvider(options);
|
|
252
|
+
}
|
|
253
|
+
/** Build the value reported for a fetch that failed. */
|
|
254
|
+
function failureValue(url, offset, failure, render) {
|
|
255
|
+
const value = {
|
|
256
|
+
url,
|
|
257
|
+
content: '',
|
|
258
|
+
offset,
|
|
259
|
+
nextOffset: null,
|
|
260
|
+
totalLength: 0,
|
|
261
|
+
truncated: false,
|
|
262
|
+
chunk: { index: 1, count: 1 },
|
|
263
|
+
failures: [failure],
|
|
264
|
+
};
|
|
265
|
+
if (render !== undefined)
|
|
266
|
+
value.render = render;
|
|
267
|
+
return value;
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Build the end-of-chunk hint. This is the tool's own text, kept out of
|
|
271
|
+
* `content` so chunk concatenation stays byte-for-byte exact.
|
|
272
|
+
*/
|
|
273
|
+
function endHint(value) {
|
|
274
|
+
if (value.totalLength === 0)
|
|
275
|
+
return '[the page has no readable text]';
|
|
276
|
+
if (value.nextOffset === null) {
|
|
277
|
+
return value.truncated
|
|
278
|
+
? `[end of page at the ${READ_PAGE_INTERNAL_MAX_CHARS}-character safety cap: ${value.totalLength} characters total]`
|
|
279
|
+
: `[end of page: ${value.totalLength} characters total]`;
|
|
280
|
+
}
|
|
281
|
+
const remaining = value.totalLength - value.nextOffset;
|
|
282
|
+
return value.truncated
|
|
283
|
+
? `[${remaining} characters remain; continue with offset=${value.nextOffset}; the source was truncated at the ${READ_PAGE_INTERNAL_MAX_CHARS}-character safety cap]`
|
|
284
|
+
: `[${remaining} characters remain; continue with offset=${value.nextOffset}]`;
|
|
285
|
+
}
|
|
286
|
+
/** Format the model-facing result text: the chunk, then exactly one hint line. */
|
|
287
|
+
export function formatReadPageValue(value) {
|
|
288
|
+
if (value.failures.length > 0) {
|
|
289
|
+
return [
|
|
290
|
+
`read_page could not read ${value.url}.`,
|
|
291
|
+
...value.failures.map((failure) => `- ${failure}`),
|
|
292
|
+
].join('\n');
|
|
293
|
+
}
|
|
294
|
+
const hint = endHint(value);
|
|
295
|
+
return value.content.length === 0 ? hint : `${value.content}\n\n${hint}`;
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Build the tool definition the host registry accepts.
|
|
299
|
+
* @param name - `read_page`, or the fallback when that name is taken.
|
|
300
|
+
* @param readSettings - read the live config for every call.
|
|
301
|
+
* @param options - test seams and diagnostics.
|
|
302
|
+
* @returns a registry-ready definition.
|
|
303
|
+
*/
|
|
304
|
+
export function createReadPageDefinition(name, readSettings, options = {}) {
|
|
305
|
+
/** Complete bodies, so walking a page chunk by chunk fetches it once. */
|
|
306
|
+
const pages = new Map();
|
|
307
|
+
/** Providers are per format and per fetch-affecting settings. */
|
|
308
|
+
const providers = new Map();
|
|
309
|
+
/** Remember a body, evicting the oldest entry when the small cache is full. */
|
|
310
|
+
const remember = (key, page) => {
|
|
311
|
+
if (pages.size >= READ_PAGE_CACHE_ENTRIES) {
|
|
312
|
+
const oldest = pages.keys().next();
|
|
313
|
+
if (!oldest.done)
|
|
314
|
+
pages.delete(oldest.value);
|
|
315
|
+
}
|
|
316
|
+
pages.set(key, { ...page });
|
|
317
|
+
};
|
|
318
|
+
/** Fetch a page through the internal ladder, or reuse the cached body. */
|
|
319
|
+
const load = async (url, format, settings, signal) => {
|
|
320
|
+
const signature = fetchSignature(format, settings);
|
|
321
|
+
const cacheKey = `${signature}\u0000${url}`;
|
|
322
|
+
const cached = pages.get(cacheKey);
|
|
323
|
+
if (cached !== undefined) {
|
|
324
|
+
// Refresh insertion order: this is a tiny LRU, not a FIFO.
|
|
325
|
+
pages.delete(cacheKey);
|
|
326
|
+
pages.set(cacheKey, cached);
|
|
327
|
+
return { ok: true, page: cached };
|
|
328
|
+
}
|
|
329
|
+
let provider = providers.get(signature);
|
|
330
|
+
if (provider === undefined) {
|
|
331
|
+
provider = options.createProvider !== undefined
|
|
332
|
+
? options.createProvider(format, settings)
|
|
333
|
+
: createDefaultProvider(format, settings, readSettings);
|
|
334
|
+
if (providers.size >= READ_PAGE_CACHE_ENTRIES) {
|
|
335
|
+
const oldest = providers.keys().next();
|
|
336
|
+
if (!oldest.done)
|
|
337
|
+
providers.delete(oldest.value);
|
|
338
|
+
}
|
|
339
|
+
providers.set(signature, provider);
|
|
340
|
+
}
|
|
341
|
+
let outcome;
|
|
342
|
+
try {
|
|
343
|
+
if (provider.available() === false) {
|
|
344
|
+
return { ok: false, failure: `read_page: the fetch provider for ${url} is unavailable` };
|
|
345
|
+
}
|
|
346
|
+
outcome = await provider.fetch({ url }, signal);
|
|
347
|
+
}
|
|
348
|
+
catch (error) {
|
|
349
|
+
// A caller cancellation is not a page failure; let it travel.
|
|
350
|
+
if (signal?.aborted === true)
|
|
351
|
+
throw error;
|
|
352
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
353
|
+
return { ok: false, failure: `read_page: ${reason}` };
|
|
354
|
+
}
|
|
355
|
+
if (outcome.statusCode < 200 || outcome.statusCode >= 300) {
|
|
356
|
+
return {
|
|
357
|
+
ok: false,
|
|
358
|
+
failure: `read_page: HTTP ${outcome.statusCode} for ${url}`,
|
|
359
|
+
...(outcome.render === undefined ? {} : { render: outcome.render }),
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
// The provider normally finalises HTML to text already; the fallback keeps
|
|
363
|
+
// this tool correct for a substituted provider that returns markup.
|
|
364
|
+
const raw = outcome.body.kind === 'text'
|
|
365
|
+
? outcome.body.content
|
|
366
|
+
: renderFetchText(outcome.body.content, {
|
|
367
|
+
format,
|
|
368
|
+
maxLength: Number.MAX_SAFE_INTEGER,
|
|
369
|
+
baseUrl: outcome.url,
|
|
370
|
+
}).content;
|
|
371
|
+
const capped = capBody(raw);
|
|
372
|
+
const page = {
|
|
373
|
+
content: capped.content,
|
|
374
|
+
truncated: outcome.truncated || capped.truncated,
|
|
375
|
+
...(outcome.render === undefined ? {} : { render: outcome.render }),
|
|
376
|
+
};
|
|
377
|
+
remember(cacheKey, page);
|
|
378
|
+
return { ok: true, page };
|
|
379
|
+
};
|
|
380
|
+
return {
|
|
381
|
+
name,
|
|
382
|
+
description: 'Read a long web page in character chunks. Fetches the page once with the same fallback ladder as web_fetch, then returns characters [offset, offset+maxLength) of the converted text. Pass nextOffset back as offset to continue until nextOffset is null. A boundary is moved back one code unit when it would split a surrogate pair, so nextOffset can be one less than offset+maxLength. Use it instead of web_fetch when a page is longer than the fetch budget and you need the rest. A network, HTTP or blocked failure is reported in failures with empty content, never thrown.',
|
|
383
|
+
parameters: parameterSchema(),
|
|
384
|
+
output: {
|
|
385
|
+
schema: outputSchema(),
|
|
386
|
+
render: (_args, value) => [{ type: 'text', text: formatReadPageValue(value) }],
|
|
387
|
+
},
|
|
388
|
+
// A single fetch may try direct, tunnel, a browser and extraction in turn.
|
|
389
|
+
timeoutMs: 60_000,
|
|
390
|
+
isConcurrencySafe: () => true,
|
|
391
|
+
async execute(args, exec) {
|
|
392
|
+
const settings = readSettings();
|
|
393
|
+
const parsed = parseArgs(args, settings);
|
|
394
|
+
const loaded = await load(parsed.url, parsed.format, settings, exec.signal);
|
|
395
|
+
if (!loaded.ok) {
|
|
396
|
+
return failureValue(parsed.url, parsed.offset, loaded.failure, loaded.render);
|
|
397
|
+
}
|
|
398
|
+
const page = loaded.page;
|
|
399
|
+
const totalLength = page.content.length;
|
|
400
|
+
if (parsed.offset > totalLength) {
|
|
401
|
+
throw new Error(`read_page: offset ${parsed.offset} is past the end of ${parsed.url} (totalLength ${totalLength}); use offset 0 to start over or ${totalLength} to read the tail`);
|
|
402
|
+
}
|
|
403
|
+
// Keep surrogate pairs intact at the cut point. `nextOffset` follows the
|
|
404
|
+
// adjusted end, so the concatenated chunks still equal the full body.
|
|
405
|
+
let end = Math.min(totalLength, parsed.offset + parsed.maxLength);
|
|
406
|
+
if (end > parsed.offset && end < totalLength) {
|
|
407
|
+
const last = page.content.charCodeAt(end - 1);
|
|
408
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
409
|
+
end -= 1;
|
|
410
|
+
}
|
|
411
|
+
const content = page.content.slice(parsed.offset, end);
|
|
412
|
+
const nextOffset = end < totalLength ? end : null;
|
|
413
|
+
const count = totalLength === 0 ? 1 : Math.ceil(totalLength / parsed.maxLength);
|
|
414
|
+
const index = totalLength === 0
|
|
415
|
+
? 1
|
|
416
|
+
: Math.min(count, Math.floor(parsed.offset / parsed.maxLength) + 1);
|
|
417
|
+
const value = {
|
|
418
|
+
url: parsed.url,
|
|
419
|
+
content,
|
|
420
|
+
offset: parsed.offset,
|
|
421
|
+
nextOffset,
|
|
422
|
+
totalLength,
|
|
423
|
+
truncated: page.truncated,
|
|
424
|
+
chunk: { index, count },
|
|
425
|
+
failures: [],
|
|
426
|
+
};
|
|
427
|
+
if (page.render !== undefined)
|
|
428
|
+
value.render = page.render;
|
|
429
|
+
return value;
|
|
430
|
+
},
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* Pick the tool name this deployment can accept.
|
|
435
|
+
* @param tools - the live registry.
|
|
436
|
+
* @returns the preferred or fallback name.
|
|
437
|
+
*/
|
|
438
|
+
export function pickReadPageToolName(tools) {
|
|
439
|
+
if (typeof tools.get !== 'function')
|
|
440
|
+
return READ_PAGE_TOOL_NAME;
|
|
441
|
+
try {
|
|
442
|
+
return tools.get(READ_PAGE_TOOL_NAME) === undefined ? READ_PAGE_TOOL_NAME : READ_PAGE_TOOL_FALLBACK_NAME;
|
|
443
|
+
}
|
|
444
|
+
catch {
|
|
445
|
+
return READ_PAGE_TOOL_NAME;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
/** Report a registration failure without throwing out of the plugin. */
|
|
449
|
+
function report(options, error) {
|
|
450
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
451
|
+
try {
|
|
452
|
+
options.warn?.(`read_page registration skipped: ${message}`);
|
|
453
|
+
}
|
|
454
|
+
catch {
|
|
455
|
+
// A diagnostic callback must never break plugin loading.
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
/**
|
|
459
|
+
* Register the tool when a registry is available.
|
|
460
|
+
*
|
|
461
|
+
* The failure paths mirror `search_many`: a duplicate name falls back, and a
|
|
462
|
+
* registry rejection is reported and swallowed so the other tools keep working.
|
|
463
|
+
* The returned registration owns the host disposer, so plugin disposal removes
|
|
464
|
+
* exactly the definition this call added.
|
|
465
|
+
*
|
|
466
|
+
* @param tools - the live registry.
|
|
467
|
+
* @param readSettings - live config reader.
|
|
468
|
+
* @param options - test seams and diagnostics.
|
|
469
|
+
* @returns the registered name and its disposer, or `undefined` when skipped.
|
|
470
|
+
*/
|
|
471
|
+
export function registerReadPageTool(tools, readSettings, options = {}) {
|
|
472
|
+
const name = pickReadPageToolName(tools);
|
|
473
|
+
try {
|
|
474
|
+
const registered = tools.register(createReadPageDefinition(name, readSettings, options));
|
|
475
|
+
const disposeHost = typeof registered === 'function' ? registered : undefined;
|
|
476
|
+
let disposed = false;
|
|
477
|
+
return {
|
|
478
|
+
name,
|
|
479
|
+
dispose: () => {
|
|
480
|
+
if (disposed)
|
|
481
|
+
return;
|
|
482
|
+
disposed = true;
|
|
483
|
+
try {
|
|
484
|
+
disposeHost?.();
|
|
485
|
+
}
|
|
486
|
+
catch {
|
|
487
|
+
// Disposal must stay contained; the plugin is already going away.
|
|
488
|
+
}
|
|
489
|
+
},
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
catch (error) {
|
|
493
|
+
report(options, error);
|
|
494
|
+
return undefined;
|
|
495
|
+
}
|
|
496
|
+
}
|