@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/rank.js CHANGED
@@ -195,6 +195,9 @@ function resolveOptions(options) {
195
195
  freshnessDays: positiveOr(options?.freshnessDays, DEFAULT_FRESHNESS_DAYS),
196
196
  timeRange: options?.timeRange,
197
197
  weights: Array.isArray(options?.weights) ? options.weights : undefined,
198
+ now: typeof options?.now === 'number' && Number.isFinite(options.now)
199
+ ? options.now
200
+ : Date.now(),
198
201
  };
199
202
  }
200
203
  /**
@@ -320,7 +323,7 @@ function fuseEntries(entries, limit, options) {
320
323
  if (budget === 0)
321
324
  return [];
322
325
  const config = resolveOptions(options);
323
- const now = Date.now();
326
+ const now = config.now;
324
327
  // Per-engine lists, deduplicated so one engine gets at most one vote per
325
328
  // document and the earliest row keeps its rank.
326
329
  const lists = [];
@@ -0,0 +1,136 @@
1
+ import type { FetchProvider } from './fetch.ts';
2
+ import type { FetchFormat } from './markdown.ts';
3
+ import type { RenderMode, RenderReport } from './render.ts';
4
+ import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
5
+ /** The name this tool prefers. */
6
+ export declare const READ_PAGE_TOOL_NAME = "read_page";
7
+ /** The name used when another plugin already registered `read_page`. */
8
+ export declare const READ_PAGE_TOOL_FALLBACK_NAME = "web_read_page";
9
+ /** Smallest chunk a caller may request, matching `fetchMaxLength`'s floor. */
10
+ export declare const READ_PAGE_MIN_LENGTH = 1000;
11
+ /** Largest chunk a caller may request, matching `fetchMaxLength`'s ceiling. */
12
+ export declare const READ_PAGE_MAX_LENGTH = 200000;
13
+ /** Default chunk size when neither the caller nor the config names one. */
14
+ export declare const READ_PAGE_DEFAULT_LENGTH = 20000;
15
+ /**
16
+ * Largest body this tool will hold. A page beyond it is cut once, after
17
+ * conversion, and reported as `truncated`; chunks never pay this cost.
18
+ */
19
+ export declare const READ_PAGE_INTERNAL_MAX_CHARS = 2000000;
20
+ /**
21
+ * Byte budget handed to the fetch ladder. Four bytes per UTF-8 code point is
22
+ * the worst case, so this is large enough for the full character cap while
23
+ * still bounding what one response can allocate.
24
+ */
25
+ export declare const READ_PAGE_MAX_BYTES: number;
26
+ /** How many converted pages stay cached per mounted tool. */
27
+ export declare const READ_PAGE_CACHE_ENTRIES = 8;
28
+ /** Internal fetch provider ids, one per output format. */
29
+ export declare const READ_PAGE_PROVIDER_ID = "http-proxy-read-page";
30
+ /** The fields a `read_page` result carries; see the module doc for semantics. */
31
+ export interface ReadPageValue {
32
+ /** The URL exactly as the caller supplied it (trimmed). */
33
+ url: string;
34
+ /** The requested character range, with no hint line appended. */
35
+ content: string;
36
+ /** First character of `content`. */
37
+ offset: number;
38
+ /** Offset to pass on the next call, or `null` when the page is finished. */
39
+ nextOffset: number | null;
40
+ /** Length of the complete converted body, possibly cut by the safety cap. */
41
+ totalLength: number;
42
+ /** True when the source body itself was cut, so `totalLength` is a lower bound. */
43
+ truncated: boolean;
44
+ /** Progress through the document for this call's `maxLength`; `index` is 1-based. */
45
+ chunk: {
46
+ index: number;
47
+ count: number;
48
+ };
49
+ /** Fetch failures, in the same shape `domain_search` and `search_many` use. */
50
+ failures: string[];
51
+ /** Why the optional browser rung did or did not run, when the provider says. */
52
+ render?: RenderReport;
53
+ }
54
+ /** Settings read live for every call, so a Settings commit applies without a remount. */
55
+ export interface ReadPageSettings {
56
+ /** Default chunk size, from `fetchMaxLength`. */
57
+ maxLength: number;
58
+ /** Default output style, from `fetchFormat`. */
59
+ format: FetchFormat;
60
+ /** Per-attempt fetch deadline, from `timeoutMs`. */
61
+ timeoutMs?: number;
62
+ /** Browser-render policy, from `renderJs`; read again by each internal fetch. */
63
+ renderJs?: RenderMode;
64
+ /** Browser-render deadline, from `renderTimeoutMs`. */
65
+ renderTimeoutMs?: number;
66
+ /** Tunnel URL, from `proxy`. */
67
+ proxy?: string;
68
+ /** AnySearch key, when the extraction rung is configured. */
69
+ anysearchApiKey?: string;
70
+ }
71
+ /** Construction seams, so the definition is testable without a network. */
72
+ export interface ReadPageOptions {
73
+ /**
74
+ * Build the internal provider for one output format. The default is
75
+ * `createFetchProvider`, so production always goes through the same ladder as
76
+ * `web_fetch`; tests substitute a recorder.
77
+ */
78
+ createProvider?: (format: FetchFormat, settings: ReadPageSettings) => FetchProvider;
79
+ /** Registration diagnostics. */
80
+ warn?: (message: string) => void;
81
+ }
82
+ /** A tool definition in the host registry's native shape. */
83
+ export interface ReadPageDefinition {
84
+ name: string;
85
+ description: string;
86
+ parameters: ToolJsonSchema;
87
+ output: {
88
+ schema: ToolJsonSchema;
89
+ render: (args: unknown, value: ReadPageValue) => ToolContent[];
90
+ };
91
+ timeoutMs?: number;
92
+ isConcurrencySafe?: (args: unknown) => boolean;
93
+ execute: (args: unknown, exec: {
94
+ signal?: AbortSignal;
95
+ }) => Promise<ReadPageValue>;
96
+ }
97
+ /** The slice of the tool registry this module needs. */
98
+ export interface ReadPageRegistryLike {
99
+ register(definition: ReadPageDefinition): unknown;
100
+ get?(name: string, scope?: unknown): unknown;
101
+ }
102
+ /** A successful registration plus the exact way to undo it. */
103
+ export interface ReadPageRegistration {
104
+ name: string;
105
+ dispose: () => void;
106
+ }
107
+ /** Format the model-facing result text: the chunk, then exactly one hint line. */
108
+ export declare function formatReadPageValue(value: ReadPageValue): string;
109
+ /**
110
+ * Build the tool definition the host registry accepts.
111
+ * @param name - `read_page`, or the fallback when that name is taken.
112
+ * @param readSettings - read the live config for every call.
113
+ * @param options - test seams and diagnostics.
114
+ * @returns a registry-ready definition.
115
+ */
116
+ export declare function createReadPageDefinition(name: string, readSettings: () => ReadPageSettings, options?: ReadPageOptions): ReadPageDefinition;
117
+ /**
118
+ * Pick the tool name this deployment can accept.
119
+ * @param tools - the live registry.
120
+ * @returns the preferred or fallback name.
121
+ */
122
+ export declare function pickReadPageToolName(tools: ReadPageRegistryLike): string;
123
+ /**
124
+ * Register the tool when a registry is available.
125
+ *
126
+ * The failure paths mirror `search_many`: a duplicate name falls back, and a
127
+ * registry rejection is reported and swallowed so the other tools keep working.
128
+ * The returned registration owns the host disposer, so plugin disposal removes
129
+ * exactly the definition this call added.
130
+ *
131
+ * @param tools - the live registry.
132
+ * @param readSettings - live config reader.
133
+ * @param options - test seams and diagnostics.
134
+ * @returns the registered name and its disposer, or `undefined` when skipped.
135
+ */
136
+ export declare function registerReadPageTool(tools: ReadPageRegistryLike, readSettings: () => ReadPageSettings, options?: ReadPageOptions): ReadPageRegistration | undefined;
@@ -0,0 +1,496 @@
1
+ /**
2
+ * The model-facing `read_page` tool: true resumable reading of a long page.
3
+ *
4
+ * The host's `web_fetch` request has exactly one field, `url` (see
5
+ * `@deepseek-ai/dsh-web`'s `WebFetchRequest`), so the host itself cannot
6
+ * continue a page from the middle: every call starts at the top and the tail is
7
+ * lost behind the character budget. A plugin, however, owns its own tool
8
+ * registry entry, so it can offer the missing `offset`.
9
+ *
10
+ * The design is two-layered on purpose:
11
+ *
12
+ * 1. one internal `createFetchProvider` fetch obtains the whole converted body,
13
+ * so the direct/tunnel/render/extraction ladder, its proxy handling and its
14
+ * live `renderJs` settings are reused rather than reimplemented;
15
+ * 2. that body is then sliced for the caller. A 2,000,000-character safety cap
16
+ * keeps one hostile page from pinning memory, and it is applied once, after
17
+ * conversion, never to an individual chunk.
18
+ *
19
+ * Successful bodies are cached per URL, output format and fetch-affecting
20
+ * settings, so walking a page chunk by chunk issues exactly one provider fetch
21
+ * (the test suite counts the calls). Failures are not cached. The cache is a
22
+ * bounded eight-entry LRU with no TTL: a long-lived plugin can serve an earlier
23
+ * capture until another page evicts it, which is the price of exact paging.
24
+ *
25
+ * `truncated` describes the *source*: it is true when the safety cap or the
26
+ * transport cut the body, so `totalLength` is then a lower bound. "There is
27
+ * more to read" is signalled by `nextOffset`, not by `truncated`.
28
+ *
29
+ * The end-of-chunk hint is generated here, in `output.render`, and is
30
+ * deliberately **not** part of `content`: concatenating the `content` of every
31
+ * chunk therefore reproduces the complete body character for character.
32
+ *
33
+ * Offsets and lengths are JavaScript string code units, matching `String.length`
34
+ * and the existing fetch budget. A chunk boundary is moved back one code unit
35
+ * when it would split a surrogate pair, so an emoji is never broken across two
36
+ * chunks; `nextOffset` follows that adjusted boundary.
37
+ *
38
+ * @module @fangchenghao/dsh-web-search/read-page
39
+ */
40
+ import { createFetchProvider } from "./fetch.js";
41
+ import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText } from "./markdown.js";
42
+ import { DEFAULT_RENDER_TIMEOUT_MS } from "./render.js";
43
+ /** The name this tool prefers. */
44
+ export const READ_PAGE_TOOL_NAME = 'read_page';
45
+ /** The name used when another plugin already registered `read_page`. */
46
+ export const READ_PAGE_TOOL_FALLBACK_NAME = 'web_read_page';
47
+ /** Smallest chunk a caller may request, matching `fetchMaxLength`'s floor. */
48
+ export const READ_PAGE_MIN_LENGTH = 1_000;
49
+ /** Largest chunk a caller may request, matching `fetchMaxLength`'s ceiling. */
50
+ export const READ_PAGE_MAX_LENGTH = 200_000;
51
+ /** Default chunk size when neither the caller nor the config names one. */
52
+ export const READ_PAGE_DEFAULT_LENGTH = DEFAULT_FETCH_MAX_LENGTH;
53
+ /**
54
+ * Largest body this tool will hold. A page beyond it is cut once, after
55
+ * conversion, and reported as `truncated`; chunks never pay this cost.
56
+ */
57
+ export const READ_PAGE_INTERNAL_MAX_CHARS = 2_000_000;
58
+ /**
59
+ * Byte budget handed to the fetch ladder. Four bytes per UTF-8 code point is
60
+ * the worst case, so this is large enough for the full character cap while
61
+ * still bounding what one response can allocate.
62
+ */
63
+ export const READ_PAGE_MAX_BYTES = READ_PAGE_INTERNAL_MAX_CHARS * 4;
64
+ /** How many converted pages stay cached per mounted tool. */
65
+ export const READ_PAGE_CACHE_ENTRIES = 8;
66
+ /** Internal fetch provider ids, one per output format. */
67
+ export const READ_PAGE_PROVIDER_ID = 'http-proxy-read-page';
68
+ /** True for a value usable as a record. */
69
+ function isRecord(value) {
70
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
71
+ }
72
+ /** The parameter schema, in the registry's raw JSON-Schema dialect. */
73
+ function parameterSchema() {
74
+ return {
75
+ type: 'object',
76
+ additionalProperties: false,
77
+ properties: {
78
+ url: {
79
+ type: 'string',
80
+ description: 'Absolute http(s) URL of the page to read.',
81
+ },
82
+ offset: {
83
+ type: 'integer',
84
+ description: 'First character to return, zero-based. Omit or use 0 for the first chunk; then pass the previous response\'s nextOffset.',
85
+ },
86
+ maxLength: {
87
+ type: 'integer',
88
+ description: `Characters to return, ${READ_PAGE_MIN_LENGTH}–${READ_PAGE_MAX_LENGTH}. Omit to use the configured fetchMaxLength.`,
89
+ },
90
+ format: {
91
+ type: 'string',
92
+ enum: ['markdown', 'text'],
93
+ description: 'Output style: markdown keeps headings, lists, tables and links; text keeps only the words. Omit to use the configured fetchFormat.',
94
+ },
95
+ },
96
+ required: ['url'],
97
+ };
98
+ }
99
+ /** The output schema, in the same dialect. */
100
+ function outputSchema() {
101
+ return {
102
+ type: 'object',
103
+ additionalProperties: false,
104
+ properties: {
105
+ url: { type: 'string' },
106
+ content: { type: 'string' },
107
+ offset: { type: 'integer' },
108
+ // A nullable integer is not expressible in the small dialect this module
109
+ // uses, so this field deliberately carries no `type`: its description is
110
+ // the contract.
111
+ nextOffset: { description: 'Offset to pass on the next call, or null when the page is finished. A surrogate-pair boundary can make it one less than offset+maxLength.' },
112
+ totalLength: { type: 'integer' },
113
+ truncated: { type: 'boolean' },
114
+ chunk: {
115
+ type: 'object',
116
+ description: 'Progress for this call\'s maxLength; index is 1-based.',
117
+ additionalProperties: false,
118
+ properties: {
119
+ index: { type: 'integer' },
120
+ count: { type: 'integer' },
121
+ },
122
+ required: ['index', 'count'],
123
+ },
124
+ failures: { type: 'array', items: { type: 'string' } },
125
+ render: {
126
+ type: 'object',
127
+ additionalProperties: false,
128
+ properties: {
129
+ mode: { type: 'string', enum: ['auto', 'always', 'off'] },
130
+ attempted: { type: 'boolean' },
131
+ applied: { type: 'boolean' },
132
+ reason: { type: 'string' },
133
+ },
134
+ required: ['mode', 'attempted', 'applied', 'reason'],
135
+ },
136
+ },
137
+ required: ['url', 'content', 'offset', 'nextOffset', 'totalLength', 'truncated', 'chunk', 'failures'],
138
+ };
139
+ }
140
+ /** Parse and validate the URL, throwing a message the model can act on. */
141
+ function parseUrl(raw) {
142
+ if (typeof raw !== 'string')
143
+ throw new Error('read_page requires a url string');
144
+ const url = raw.trim();
145
+ if (url.length === 0)
146
+ throw new Error('read_page requires a non-empty url');
147
+ let parsed;
148
+ try {
149
+ parsed = new URL(url);
150
+ }
151
+ catch {
152
+ throw new Error(`read_page: not a valid absolute URL: ${JSON.stringify(url)}`);
153
+ }
154
+ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
155
+ throw new Error(`read_page: url must use http or https, got ${parsed.protocol}`);
156
+ }
157
+ if (parsed.hostname.length === 0) {
158
+ throw new Error(`read_page: url has no host: ${JSON.stringify(url)}`);
159
+ }
160
+ return url;
161
+ }
162
+ /** Parse the optional zero-based character offset. */
163
+ function parseOffset(raw) {
164
+ if (raw === undefined)
165
+ return 0;
166
+ if (typeof raw !== 'number' || !Number.isSafeInteger(raw) || raw < 0) {
167
+ throw new Error(`read_page offset must be a safe integer >= 0, got ${JSON.stringify(raw)}`);
168
+ }
169
+ return raw;
170
+ }
171
+ /** Normalise the config defaults, never throwing because of a bad config value. */
172
+ function defaultLength(settings) {
173
+ const value = settings.maxLength;
174
+ return Number.isSafeInteger(value) && value >= READ_PAGE_MIN_LENGTH && value <= READ_PAGE_MAX_LENGTH
175
+ ? value
176
+ : READ_PAGE_DEFAULT_LENGTH;
177
+ }
178
+ /** Normalise the configured default format. */
179
+ function defaultFormat(settings) {
180
+ return settings.format === 'text' ? 'text' : 'markdown';
181
+ }
182
+ /** Parse the optional chunk size against the documented range. */
183
+ function parseMaxLength(raw, fallback) {
184
+ if (raw === undefined)
185
+ return fallback;
186
+ if (typeof raw !== 'number' || !Number.isSafeInteger(raw)
187
+ || raw < READ_PAGE_MIN_LENGTH || raw > READ_PAGE_MAX_LENGTH) {
188
+ throw new Error(`read_page maxLength must be an integer between ${READ_PAGE_MIN_LENGTH} and ${READ_PAGE_MAX_LENGTH}, got ${JSON.stringify(raw)}`);
189
+ }
190
+ return raw;
191
+ }
192
+ /** Parse the optional output style. */
193
+ function parseFormat(raw, fallback) {
194
+ if (raw === undefined)
195
+ return fallback;
196
+ if (raw !== 'markdown' && raw !== 'text') {
197
+ throw new Error(`read_page format must be "markdown" or "text", got ${JSON.stringify(raw)}`);
198
+ }
199
+ return raw;
200
+ }
201
+ /** Parse model arguments into a validated shape, throwing on nonsense. */
202
+ function parseArgs(args, settings) {
203
+ if (!isRecord(args))
204
+ throw new Error('read_page requires arguments');
205
+ return {
206
+ url: parseUrl(args['url']),
207
+ offset: parseOffset(args['offset']),
208
+ maxLength: parseMaxLength(args['maxLength'], defaultLength(settings)),
209
+ format: parseFormat(args['format'], defaultFormat(settings)),
210
+ };
211
+ }
212
+ /** A signature for everything that can change what the internal fetch returns. */
213
+ function fetchSignature(format, settings) {
214
+ return [
215
+ format,
216
+ settings.renderJs ?? 'auto',
217
+ settings.renderTimeoutMs ?? '',
218
+ settings.timeoutMs ?? '',
219
+ settings.proxy ?? '',
220
+ settings.anysearchApiKey ?? '',
221
+ ].join('\u0000');
222
+ }
223
+ /** Cut an over-long body once, without leaving half of a surrogate pair. */
224
+ function capBody(content) {
225
+ if (content.length <= READ_PAGE_INTERNAL_MAX_CHARS)
226
+ return { content, truncated: false };
227
+ let cut = content.slice(0, READ_PAGE_INTERNAL_MAX_CHARS);
228
+ const last = cut.charCodeAt(cut.length - 1);
229
+ if (last >= 0xd800 && last <= 0xdbff)
230
+ cut = cut.slice(0, -1);
231
+ return { content: cut, truncated: true };
232
+ }
233
+ /** Build the production provider: the same ladder, with an effectively unlimited budget. */
234
+ function createDefaultProvider(format, settings, readSettings) {
235
+ const proxy = (settings.proxy ?? '').trim();
236
+ const apiKey = (settings.anysearchApiKey ?? '').trim();
237
+ const options = {
238
+ id: `${READ_PAGE_PROVIDER_ID}-${format}`,
239
+ format,
240
+ // No per-request truncation: `read_page` applies one cap to the whole body
241
+ // and then slices it. The huge budget makes `truncateFetchText` a no-op.
242
+ maxLength: Number.MAX_SAFE_INTEGER,
243
+ maxBytes: READ_PAGE_MAX_BYTES,
244
+ // The renderer policy stays live for the lifetime of this provider.
245
+ renderJs: () => readSettings().renderJs ?? 'auto',
246
+ renderTimeoutMs: () => readSettings().renderTimeoutMs ?? DEFAULT_RENDER_TIMEOUT_MS,
247
+ ...(settings.timeoutMs === undefined ? {} : { timeoutMs: settings.timeoutMs }),
248
+ ...(proxy.length === 0 ? {} : { proxy }),
249
+ ...(apiKey.length === 0 ? {} : { anysearchApiKey: apiKey }),
250
+ };
251
+ return createFetchProvider(options);
252
+ }
253
+ /** Build the value reported for a fetch that failed. */
254
+ function failureValue(url, offset, failure, render) {
255
+ const value = {
256
+ url,
257
+ content: '',
258
+ offset,
259
+ nextOffset: null,
260
+ totalLength: 0,
261
+ truncated: false,
262
+ chunk: { index: 1, count: 1 },
263
+ failures: [failure],
264
+ };
265
+ if (render !== undefined)
266
+ value.render = render;
267
+ return value;
268
+ }
269
+ /**
270
+ * Build the end-of-chunk hint. This is the tool's own text, kept out of
271
+ * `content` so chunk concatenation stays byte-for-byte exact.
272
+ */
273
+ function endHint(value) {
274
+ if (value.totalLength === 0)
275
+ return '[the page has no readable text]';
276
+ if (value.nextOffset === null) {
277
+ return value.truncated
278
+ ? `[end of page at the ${READ_PAGE_INTERNAL_MAX_CHARS}-character safety cap: ${value.totalLength} characters total]`
279
+ : `[end of page: ${value.totalLength} characters total]`;
280
+ }
281
+ const remaining = value.totalLength - value.nextOffset;
282
+ return value.truncated
283
+ ? `[${remaining} characters remain; continue with offset=${value.nextOffset}; the source was truncated at the ${READ_PAGE_INTERNAL_MAX_CHARS}-character safety cap]`
284
+ : `[${remaining} characters remain; continue with offset=${value.nextOffset}]`;
285
+ }
286
+ /** Format the model-facing result text: the chunk, then exactly one hint line. */
287
+ export function formatReadPageValue(value) {
288
+ if (value.failures.length > 0) {
289
+ return [
290
+ `read_page could not read ${value.url}.`,
291
+ ...value.failures.map((failure) => `- ${failure}`),
292
+ ].join('\n');
293
+ }
294
+ const hint = endHint(value);
295
+ return value.content.length === 0 ? hint : `${value.content}\n\n${hint}`;
296
+ }
297
+ /**
298
+ * Build the tool definition the host registry accepts.
299
+ * @param name - `read_page`, or the fallback when that name is taken.
300
+ * @param readSettings - read the live config for every call.
301
+ * @param options - test seams and diagnostics.
302
+ * @returns a registry-ready definition.
303
+ */
304
+ export function createReadPageDefinition(name, readSettings, options = {}) {
305
+ /** Complete bodies, so walking a page chunk by chunk fetches it once. */
306
+ const pages = new Map();
307
+ /** Providers are per format and per fetch-affecting settings. */
308
+ const providers = new Map();
309
+ /** Remember a body, evicting the oldest entry when the small cache is full. */
310
+ const remember = (key, page) => {
311
+ if (pages.size >= READ_PAGE_CACHE_ENTRIES) {
312
+ const oldest = pages.keys().next();
313
+ if (!oldest.done)
314
+ pages.delete(oldest.value);
315
+ }
316
+ pages.set(key, { ...page });
317
+ };
318
+ /** Fetch a page through the internal ladder, or reuse the cached body. */
319
+ const load = async (url, format, settings, signal) => {
320
+ const signature = fetchSignature(format, settings);
321
+ const cacheKey = `${signature}\u0000${url}`;
322
+ const cached = pages.get(cacheKey);
323
+ if (cached !== undefined) {
324
+ // Refresh insertion order: this is a tiny LRU, not a FIFO.
325
+ pages.delete(cacheKey);
326
+ pages.set(cacheKey, cached);
327
+ return { ok: true, page: cached };
328
+ }
329
+ let provider = providers.get(signature);
330
+ if (provider === undefined) {
331
+ provider = options.createProvider !== undefined
332
+ ? options.createProvider(format, settings)
333
+ : createDefaultProvider(format, settings, readSettings);
334
+ if (providers.size >= READ_PAGE_CACHE_ENTRIES) {
335
+ const oldest = providers.keys().next();
336
+ if (!oldest.done)
337
+ providers.delete(oldest.value);
338
+ }
339
+ providers.set(signature, provider);
340
+ }
341
+ let outcome;
342
+ try {
343
+ if (provider.available() === false) {
344
+ return { ok: false, failure: `read_page: the fetch provider for ${url} is unavailable` };
345
+ }
346
+ outcome = await provider.fetch({ url }, signal);
347
+ }
348
+ catch (error) {
349
+ // A caller cancellation is not a page failure; let it travel.
350
+ if (signal?.aborted === true)
351
+ throw error;
352
+ const reason = error instanceof Error ? error.message : String(error);
353
+ return { ok: false, failure: `read_page: ${reason}` };
354
+ }
355
+ if (outcome.statusCode < 200 || outcome.statusCode >= 300) {
356
+ return {
357
+ ok: false,
358
+ failure: `read_page: HTTP ${outcome.statusCode} for ${url}`,
359
+ ...(outcome.render === undefined ? {} : { render: outcome.render }),
360
+ };
361
+ }
362
+ // The provider normally finalises HTML to text already; the fallback keeps
363
+ // this tool correct for a substituted provider that returns markup.
364
+ const raw = outcome.body.kind === 'text'
365
+ ? outcome.body.content
366
+ : renderFetchText(outcome.body.content, {
367
+ format,
368
+ maxLength: Number.MAX_SAFE_INTEGER,
369
+ baseUrl: outcome.url,
370
+ }).content;
371
+ const capped = capBody(raw);
372
+ const page = {
373
+ content: capped.content,
374
+ truncated: outcome.truncated || capped.truncated,
375
+ ...(outcome.render === undefined ? {} : { render: outcome.render }),
376
+ };
377
+ remember(cacheKey, page);
378
+ return { ok: true, page };
379
+ };
380
+ return {
381
+ name,
382
+ description: 'Read a long web page in character chunks. Fetches the page once with the same fallback ladder as web_fetch, then returns characters [offset, offset+maxLength) of the converted text. Pass nextOffset back as offset to continue until nextOffset is null. A boundary is moved back one code unit when it would split a surrogate pair, so nextOffset can be one less than offset+maxLength. Use it instead of web_fetch when a page is longer than the fetch budget and you need the rest. A network, HTTP or blocked failure is reported in failures with empty content, never thrown.',
383
+ parameters: parameterSchema(),
384
+ output: {
385
+ schema: outputSchema(),
386
+ render: (_args, value) => [{ type: 'text', text: formatReadPageValue(value) }],
387
+ },
388
+ // A single fetch may try direct, tunnel, a browser and extraction in turn.
389
+ timeoutMs: 60_000,
390
+ isConcurrencySafe: () => true,
391
+ async execute(args, exec) {
392
+ const settings = readSettings();
393
+ const parsed = parseArgs(args, settings);
394
+ const loaded = await load(parsed.url, parsed.format, settings, exec.signal);
395
+ if (!loaded.ok) {
396
+ return failureValue(parsed.url, parsed.offset, loaded.failure, loaded.render);
397
+ }
398
+ const page = loaded.page;
399
+ const totalLength = page.content.length;
400
+ if (parsed.offset > totalLength) {
401
+ throw new Error(`read_page: offset ${parsed.offset} is past the end of ${parsed.url} (totalLength ${totalLength}); use offset 0 to start over or ${totalLength} to read the tail`);
402
+ }
403
+ // Keep surrogate pairs intact at the cut point. `nextOffset` follows the
404
+ // adjusted end, so the concatenated chunks still equal the full body.
405
+ let end = Math.min(totalLength, parsed.offset + parsed.maxLength);
406
+ if (end > parsed.offset && end < totalLength) {
407
+ const last = page.content.charCodeAt(end - 1);
408
+ if (last >= 0xd800 && last <= 0xdbff)
409
+ end -= 1;
410
+ }
411
+ const content = page.content.slice(parsed.offset, end);
412
+ const nextOffset = end < totalLength ? end : null;
413
+ const count = totalLength === 0 ? 1 : Math.ceil(totalLength / parsed.maxLength);
414
+ const index = totalLength === 0
415
+ ? 1
416
+ : Math.min(count, Math.floor(parsed.offset / parsed.maxLength) + 1);
417
+ const value = {
418
+ url: parsed.url,
419
+ content,
420
+ offset: parsed.offset,
421
+ nextOffset,
422
+ totalLength,
423
+ truncated: page.truncated,
424
+ chunk: { index, count },
425
+ failures: [],
426
+ };
427
+ if (page.render !== undefined)
428
+ value.render = page.render;
429
+ return value;
430
+ },
431
+ };
432
+ }
433
+ /**
434
+ * Pick the tool name this deployment can accept.
435
+ * @param tools - the live registry.
436
+ * @returns the preferred or fallback name.
437
+ */
438
+ export function pickReadPageToolName(tools) {
439
+ if (typeof tools.get !== 'function')
440
+ return READ_PAGE_TOOL_NAME;
441
+ try {
442
+ return tools.get(READ_PAGE_TOOL_NAME) === undefined ? READ_PAGE_TOOL_NAME : READ_PAGE_TOOL_FALLBACK_NAME;
443
+ }
444
+ catch {
445
+ return READ_PAGE_TOOL_NAME;
446
+ }
447
+ }
448
+ /** Report a registration failure without throwing out of the plugin. */
449
+ function report(options, error) {
450
+ const message = error instanceof Error ? error.message : String(error);
451
+ try {
452
+ options.warn?.(`read_page registration skipped: ${message}`);
453
+ }
454
+ catch {
455
+ // A diagnostic callback must never break plugin loading.
456
+ }
457
+ }
458
+ /**
459
+ * Register the tool when a registry is available.
460
+ *
461
+ * The failure paths mirror `search_many`: a duplicate name falls back, and a
462
+ * registry rejection is reported and swallowed so the other tools keep working.
463
+ * The returned registration owns the host disposer, so plugin disposal removes
464
+ * exactly the definition this call added.
465
+ *
466
+ * @param tools - the live registry.
467
+ * @param readSettings - live config reader.
468
+ * @param options - test seams and diagnostics.
469
+ * @returns the registered name and its disposer, or `undefined` when skipped.
470
+ */
471
+ export function registerReadPageTool(tools, readSettings, options = {}) {
472
+ const name = pickReadPageToolName(tools);
473
+ try {
474
+ const registered = tools.register(createReadPageDefinition(name, readSettings, options));
475
+ const disposeHost = typeof registered === 'function' ? registered : undefined;
476
+ let disposed = false;
477
+ return {
478
+ name,
479
+ dispose: () => {
480
+ if (disposed)
481
+ return;
482
+ disposed = true;
483
+ try {
484
+ disposeHost?.();
485
+ }
486
+ catch {
487
+ // Disposal must stay contained; the plugin is already going away.
488
+ }
489
+ },
490
+ };
491
+ }
492
+ catch (error) {
493
+ report(options, error);
494
+ return undefined;
495
+ }
496
+ }