@link-assistant/agent 0.26.15 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +49 -7
  2. package/package.json +51 -42
  3. package/src/agent/agent.ts +12 -7
  4. package/src/auth/azure-login.ts +241 -0
  5. package/src/auth/plugins.ts +9 -0
  6. package/src/auth/provider-logins.ts +804 -0
  7. package/src/cli/argv.ts +1 -1
  8. package/src/cli/cmd/auth.ts +4 -0
  9. package/src/cli/cmd/models.ts +68 -13
  10. package/src/cli/cmd/run.ts +7 -1
  11. package/src/cli/continuous-mode.js +38 -12
  12. package/src/cli/event-handler.js +3 -1
  13. package/src/cli/model-config.js +27 -8
  14. package/src/cli/run-options.js +144 -78
  15. package/src/config/config.ts +306 -105
  16. package/src/config/defaults.ts +13 -5
  17. package/src/config/file-config.ts +16 -0
  18. package/src/file/ripgrep.ts +38 -5
  19. package/src/file/time.ts +86 -9
  20. package/src/index.js +42 -33
  21. package/src/mcp/index.ts +370 -106
  22. package/src/permission/index.ts +22 -0
  23. package/src/project/instance.ts +5 -1
  24. package/src/project/project.ts +35 -2
  25. package/src/provider/env-template.ts +115 -0
  26. package/src/provider/extra-providers.ts +347 -0
  27. package/src/provider/free-tier-error.ts +1 -1
  28. package/src/provider/kilo-deprecation.ts +30 -0
  29. package/src/provider/kilo.ts +44 -7
  30. package/src/provider/model-discovery.ts +690 -0
  31. package/src/provider/models.ts +11 -1
  32. package/src/provider/parity-loaders.ts +595 -0
  33. package/src/provider/provider.ts +309 -203
  34. package/src/provider/retry-fetch.ts +5 -0
  35. package/src/provider/transform.ts +168 -17
  36. package/src/session/compaction.ts +18 -2
  37. package/src/session/index.ts +30 -3
  38. package/src/session/message-v2.ts +318 -32
  39. package/src/session/processor.ts +274 -140
  40. package/src/session/prompt.ts +154 -128
  41. package/src/session/retry.ts +26 -13
  42. package/src/session/summary.ts +4 -1
  43. package/src/tool/bash.ts +232 -45
  44. package/src/tool/bash.txt +1 -1
  45. package/src/tool/codesearch.ts +15 -78
  46. package/src/tool/edit.ts +111 -30
  47. package/src/tool/glob.ts +2 -1
  48. package/src/tool/grep.ts +175 -57
  49. package/src/tool/ls.ts +2 -1
  50. package/src/tool/multiedit.ts +3 -1
  51. package/src/tool/read.ts +169 -41
  52. package/src/tool/task.ts +26 -18
  53. package/src/tool/tool.ts +4 -1
  54. package/src/tool/webfetch.ts +68 -56
  55. package/src/tool/webfetch.txt +4 -4
  56. package/src/tool/websearch.ts +20 -75
  57. package/src/tool/websearch.txt +2 -0
  58. package/src/tool/write.ts +23 -6
  59. package/src/util/bom.ts +32 -0
  60. package/src/util/exa-mcp.ts +82 -0
  61. package/src/util/legacy-error-names.ts +39 -0
  62. package/src/util/read-bounded.ts +32 -0
  63. package/src/util/sse-usage-extractor.ts +67 -13
  64. package/src/util/timeout.ts +3 -4
  65. package/src/util/verbose-fetch.ts +40 -1
package/src/tool/read.ts CHANGED
@@ -12,6 +12,13 @@ import { formatBalancedLineWindow, formatColumnWindow } from './text-window';
12
12
 
13
13
  const DEFAULT_READ_LIMIT = 2000;
14
14
  const MAX_LINE_LENGTH = 2000;
15
+ // upstream 1b82511fbd / c1b03b728a (#14009): the output is bounded in bytes
16
+ // too (config.readMaxBytes, 50 KB by default), 2000 long lines would otherwise
17
+ // put megabytes into the context.
18
+
19
+ function formatSize(bytes: number) {
20
+ return bytes % 1024 === 0 ? `${bytes / 1024} KB` : `${bytes} bytes`;
21
+ }
15
22
 
16
23
  export const ReadTool = Tool.define('read', {
17
24
  description: DESCRIPTION,
@@ -36,18 +43,42 @@ export const ReadTool = Tool.define('read', {
36
43
  }),
37
44
  async execute(params, ctx) {
38
45
  let filepath = params.filePath;
39
- if (!path.isAbsolute(filepath)) {
40
- filepath = path.join(Instance.worktree, filepath);
46
+ const relative = !path.isAbsolute(filepath);
47
+ if (relative) {
48
+ // Relative to the session directory, like every other tool, not to the
49
+ // worktree root (upstream 6cf2c3e3db).
50
+ filepath = path.resolve(Instance.directory, filepath);
41
51
  }
42
- const title = path.relative(Instance.worktree, filepath);
43
52
 
44
53
  // No restrictions - unrestricted file read
45
- const file = Bun.file(filepath);
46
- if (!(await file.exists())) {
54
+ let stat = await fs.promises.stat(filepath).catch(() => undefined);
55
+ if (!stat && relative && Instance.directory !== Instance.worktree) {
56
+ // Backward compatibility: relative paths used to resolve against the
57
+ // worktree root. Keep that working when the path does not exist under
58
+ // the session directory.
59
+ const fromWorktree = path.resolve(Instance.worktree, params.filePath);
60
+ const worktreeStat = await fs.promises
61
+ .stat(fromWorktree)
62
+ .catch(() => undefined);
63
+ if (worktreeStat) {
64
+ filepath = fromWorktree;
65
+ stat = worktreeStat;
66
+ }
67
+ }
68
+ if (!stat) {
69
+ const alternate = alternatePath(filepath);
70
+ if (alternate) {
71
+ filepath = alternate;
72
+ stat = await fs.promises.stat(filepath).catch(() => undefined);
73
+ }
74
+ }
75
+ const title = path.relative(Instance.worktree, filepath);
76
+ if (!stat) {
47
77
  const dir = path.dirname(filepath);
48
78
  const base = path.basename(filepath);
49
79
 
50
- const dirEntries = fs.readdirSync(dir);
80
+ // A missing parent directory is still just "File not found".
81
+ const dirEntries = fs.existsSync(dir) ? fs.readdirSync(dir) : [];
51
82
  const suggestions = dirEntries
52
83
  .filter(
53
84
  (entry) =>
@@ -65,6 +96,16 @@ export const ReadTool = Tool.define('read', {
65
96
 
66
97
  throw new Error(`File not found: ${filepath}`);
67
98
  }
99
+ // upstream 823d327401 / ff837fe949 (#33255/#33260)
100
+ if (stat.isDirectory()) {
101
+ throw new Error(
102
+ `Path is a directory, not a file: ${filepath}. Use the ls or glob tool to list it.`
103
+ );
104
+ }
105
+ if (!stat.isFile()) {
106
+ throw new Error(`Path is not a regular file: ${filepath}`);
107
+ }
108
+ const file = Bun.file(filepath);
68
109
 
69
110
  const isImage = isImageFile(filepath);
70
111
  const supportsImages = await (async () => {
@@ -85,9 +126,8 @@ export const ReadTool = Tool.define('read', {
85
126
  `Failed to read image: ${filepath}, model may not be able to read images`
86
127
  );
87
128
  }
129
+ const bytes = await file.bytes();
88
130
  if (verifyImages) {
89
- const bytes = new Uint8Array(await file.arrayBuffer());
90
-
91
131
  // Validate image format matches file extension
92
132
  if (!validateImageFormat(bytes, isImage)) {
93
133
  throw new Error(
@@ -118,54 +158,101 @@ export const ReadTool = Tool.define('read', {
118
158
  messageID: ctx.messageID,
119
159
  type: 'file',
120
160
  mime,
121
- url: `data:${mime};base64,${Buffer.from(await file.bytes()).toString('base64')}`,
161
+ url: `data:${mime};base64,${Buffer.from(bytes).toString('base64')}`,
122
162
  },
123
163
  ],
124
164
  };
125
165
  }
126
166
 
127
- const isBinary = await isBinaryFile(filepath, file);
167
+ const isBinary = await isBinaryFile(filepath, stat.size);
128
168
  if (isBinary) throw new Error(`Cannot read binary file: ${filepath}`);
129
169
 
130
- const limit = normalizeNonNegativeInteger(params.limit, DEFAULT_READ_LIMIT);
170
+ // upstream 683d470fe4 (#51011): never return more than the default limit.
171
+ // --read-max-lines / --read-max-bytes tune both caps; 0 disables a cap.
172
+ const maxLines = config.readMaxLines > 0 ? config.readMaxLines : Infinity;
173
+ const maxBytes = config.readMaxBytes > 0 ? config.readMaxBytes : Infinity;
174
+ const limit = Math.min(
175
+ normalizeNonNegativeInteger(
176
+ params.limit,
177
+ Math.min(DEFAULT_READ_LIMIT, maxLines)
178
+ ),
179
+ maxLines
180
+ );
131
181
  const offset = normalizeNonNegativeInteger(params.offset, 0);
132
182
  const hasExplicitLineRange =
133
183
  params.offset !== undefined || params.limit !== undefined;
134
184
  const hasExplicitColumnRange =
135
185
  params.columnOffset !== undefined || params.columnLimit !== undefined;
136
186
  const columnOffset = normalizeNonNegativeInteger(params.columnOffset, 0);
137
- const columnLimit = normalizeNonNegativeInteger(
138
- params.columnLimit,
139
- MAX_LINE_LENGTH
187
+ const columnLimit = Math.min(
188
+ normalizeNonNegativeInteger(params.columnLimit, MAX_LINE_LENGTH),
189
+ maxBytes
140
190
  );
141
- const lines = await file.text().then((text) => text.split('\n'));
191
+ const lines = splitLines(await file.text());
192
+ // upstream c1b03b728a (#14009)
193
+ // --legacy-tool-output keeps the old empty result past the end.
194
+ if (
195
+ !config.legacyToolOutput &&
196
+ hasExplicitLineRange &&
197
+ lines.length > 0 &&
198
+ offset >= lines.length
199
+ ) {
200
+ throw new Error(
201
+ `Offset ${offset} is out of range for this file (${lines.length} lines)`
202
+ );
203
+ }
142
204
  const selected = selectLines({
143
205
  lines,
144
206
  offset,
145
207
  limit,
146
208
  summarizeMiddle: !hasExplicitLineRange,
147
209
  });
148
- const content = selected.parts.flatMap((part) => {
210
+ const formatLine = ({
211
+ line,
212
+ lineNumber,
213
+ }: {
214
+ line: string;
215
+ lineNumber: number;
216
+ }) => {
217
+ const formattedLine = hasExplicitColumnRange
218
+ ? formatColumnWindow({
219
+ line,
220
+ lineNumber,
221
+ offset: columnOffset,
222
+ limit: columnLimit,
223
+ })
224
+ : formatBalancedLineWindow({
225
+ line,
226
+ lineNumber,
227
+ limit: MAX_LINE_LENGTH,
228
+ });
229
+ return `${lineNumber.toString().padStart(5, '0')}| ${formattedLine}`;
230
+ };
231
+
232
+ const content: string[] = [];
233
+ let bytes = 0;
234
+ let capped: { first: number; last: number } | undefined;
235
+ for (const part of selected.parts) {
149
236
  if (part.type === 'omitted') {
150
- return [`... [omitted lines ${part.start}..${part.end}] ...`];
237
+ content.push(`... [omitted lines ${part.start}..${part.end}] ...`);
238
+ continue;
151
239
  }
152
-
153
- return part.lines.map(({ line, lineNumber }) => {
154
- const formattedLine = hasExplicitColumnRange
155
- ? formatColumnWindow({
156
- line,
157
- lineNumber,
158
- offset: columnOffset,
159
- limit: columnLimit,
160
- })
161
- : formatBalancedLineWindow({
162
- line,
163
- lineNumber,
164
- limit: MAX_LINE_LENGTH,
165
- });
166
- return `${lineNumber.toString().padStart(5, '0')}| ${formattedLine}`;
167
- });
168
- });
240
+ for (const item of part.lines) {
241
+ const formatted = formatLine(item);
242
+ const size = Buffer.byteLength(formatted, 'utf8') + 1;
243
+ // Always show at least one line so the cap cannot return nothing.
244
+ if (bytes + size > maxBytes && content.length > 0) {
245
+ capped = {
246
+ first: part.lines[0].lineNumber,
247
+ last: item.lineNumber - 1,
248
+ };
249
+ break;
250
+ }
251
+ content.push(formatted);
252
+ bytes += size;
253
+ }
254
+ if (capped) break;
255
+ }
169
256
  const preview = content.slice(0, 20).join('\n');
170
257
 
171
258
  let output = '<file>\n';
@@ -175,7 +262,10 @@ export const ReadTool = Tool.define('read', {
175
262
  const lastReadLine = selected.lastLine;
176
263
  const hasMoreLines = hasExplicitLineRange && totalLines > lastReadLine;
177
264
 
178
- if (hasMoreLines) {
265
+ if (capped) {
266
+ // `offset` is 0-based, so the next unread line starts at `last`.
267
+ output += `\n\n(Output capped at ${formatSize(maxBytes)}. Showing lines ${capped.first}-${capped.last}. Use offset=${capped.last} to continue.)`;
268
+ } else if (hasMoreLines) {
179
269
  output += `\n\n(File has more lines. Use 'offset' parameter to read beyond line ${lastReadLine})`;
180
270
  } else {
181
271
  output += `\n\n(End of file - total ${totalLines} lines)`;
@@ -195,6 +285,40 @@ export const ReadTool = Tool.define('read', {
195
285
  },
196
286
  });
197
287
 
288
+ // CRLF files must not leak `\r` into the output, and a trailing newline ends
289
+ // the last line instead of starting an empty one (upstream c1b03b728a,
290
+ // 683d470fe4).
291
+ function splitLines(text: string) {
292
+ const lines = text.split(/\r?\n/);
293
+ if (lines.length > 1 && lines[lines.length - 1] === '') lines.pop();
294
+ return lines;
295
+ }
296
+
297
+ // macOS puts U+202F (narrow no-break space) before AM/PM in screenshot names
298
+ // and models type a plain space; curly apostrophes get straightened the same
299
+ // way. Use the only directory entry that matches after canonicalizing both
300
+ // (upstream d484f070d1 #45807, b47824fb63 #50953).
301
+ function canonicalName(name: string) {
302
+ return name
303
+ .normalize('NFC')
304
+ .replace(/[\u00a0\u202f]/g, ' ')
305
+ .replace(/[\u2018\u2019]/g, "'");
306
+ }
307
+
308
+ function alternatePath(filepath: string): string | undefined {
309
+ const dir = path.dirname(filepath);
310
+ const base = canonicalName(path.basename(filepath));
311
+ let entries: string[];
312
+ try {
313
+ entries = fs.readdirSync(dir);
314
+ } catch {
315
+ return undefined;
316
+ }
317
+ const matches = entries.filter((entry) => canonicalName(entry) === base);
318
+ if (matches.length !== 1) return undefined;
319
+ return path.join(dir, matches[0]);
320
+ }
321
+
198
322
  function normalizeNonNegativeInteger(
199
323
  value: number | undefined,
200
324
  fallback: number
@@ -386,7 +510,7 @@ function validateImageFormat(
386
510
 
387
511
  async function isBinaryFile(
388
512
  filepath: string,
389
- file: Bun.BunFile
513
+ fileSize: number
390
514
  ): Promise<boolean> {
391
515
  const ext = path.extname(filepath).toLowerCase();
392
516
  // binary check for common non-text extensions
@@ -424,14 +548,18 @@ async function isBinaryFile(
424
548
  break;
425
549
  }
426
550
 
427
- const stat = await file.stat();
428
- const fileSize = stat.size;
429
551
  if (fileSize === 0) return false;
430
552
 
553
+ // Sample only the first 4 KB instead of loading the whole file
554
+ // (upstream c1b03b728a #14009).
431
555
  const bufferSize = Math.min(4096, fileSize);
432
- const buffer = await file.arrayBuffer();
433
- if (buffer.byteLength === 0) return false;
434
- const bytes = new Uint8Array(buffer.slice(0, bufferSize));
556
+ const fh = await fs.promises.open(filepath, 'r');
557
+ const sample = Buffer.alloc(bufferSize);
558
+ const { bytesRead } = await fh
559
+ .read(sample, 0, bufferSize, 0)
560
+ .finally(() => fh.close());
561
+ if (bytesRead === 0) return false;
562
+ const bytes = sample.subarray(0, bytesRead);
435
563
 
436
564
  let nonPrintableCount = 0;
437
565
  for (let i = 0; i < bytes.length; i++) {
package/src/tool/task.ts CHANGED
@@ -110,24 +110,32 @@ export const TaskTool = Tool.define('task', async () => {
110
110
  }
111
111
  ctx.abort.addEventListener('abort', cancel);
112
112
  using _ = defer(() => ctx.abort.removeEventListener('abort', cancel));
113
- const promptParts = await SessionPrompt.resolvePromptParts(params.prompt);
114
- const result = await SessionPrompt.prompt({
115
- messageID,
116
- sessionID: session.id,
117
- model: {
118
- modelID: model.modelID,
119
- providerID: model.providerID,
120
- },
121
- agent: agent.name,
122
- tools: {
123
- todowrite: false,
124
- todoread: false,
125
- task: false,
126
- ...agent.tools,
127
- },
128
- parts: promptParts,
129
- });
130
- unsub();
113
+ // Unsubscribe even when the subagent fails, or the subscription lives
114
+ // as long as the instance (upstream 63f5669eb5, #9992).
115
+ let result: Awaited<ReturnType<typeof SessionPrompt.prompt>>;
116
+ try {
117
+ const promptParts = await SessionPrompt.resolvePromptParts(
118
+ params.prompt
119
+ );
120
+ result = await SessionPrompt.prompt({
121
+ messageID,
122
+ sessionID: session.id,
123
+ model: {
124
+ modelID: model.modelID,
125
+ providerID: model.providerID,
126
+ },
127
+ agent: agent.name,
128
+ tools: {
129
+ todowrite: false,
130
+ todoread: false,
131
+ task: false,
132
+ ...agent.tools,
133
+ },
134
+ parts: promptParts,
135
+ });
136
+ } finally {
137
+ unsub();
138
+ }
131
139
  let all;
132
140
  all = await Session.messages({ sessionID: session.id });
133
141
  all = all.filter((x) => x.info.role === 'assistant');
package/src/tool/tool.ts CHANGED
@@ -87,7 +87,10 @@ export namespace Tool {
87
87
  return {
88
88
  id,
89
89
  init: async () => {
90
- const toolInfo = init instanceof Function ? await init() : init;
90
+ // Copy object-defined tools so each init() wraps the original
91
+ // execute once instead of stacking another wrapper on the shared
92
+ // object (upstream 81d3ac3bf0, #16952).
93
+ const toolInfo = init instanceof Function ? await init() : { ...init };
91
94
  const execute = toolInfo.execute;
92
95
  toolInfo.execute = (args, ctx) => {
93
96
  try {
@@ -4,6 +4,8 @@ import TurndownService from 'turndown';
4
4
  import DESCRIPTION from './webfetch.txt';
5
5
  import { createVerboseFetch } from '../util/verbose-fetch';
6
6
  import { Permission } from '../permission';
7
+ import { Branding } from '../branding';
8
+ import { readBounded } from '../util/read-bounded';
7
9
 
8
10
  const verboseFetch = createVerboseFetch(fetch, { caller: 'webfetch' });
9
11
 
@@ -15,10 +17,14 @@ export const WebFetchTool = Tool.define('webfetch', {
15
17
  description: DESCRIPTION,
16
18
  parameters: z.object({
17
19
  url: z.string().describe('The URL to fetch content from'),
20
+ // Optional, not `.default()`: the JSON schema sent to the model would
21
+ // still list a defaulted field as required, and execute receives the raw
22
+ // arguments (upstream fd973d242e #6345, 2d2f587bfa #30215).
18
23
  format: z
19
24
  .enum(['text', 'markdown', 'html'])
25
+ .optional()
20
26
  .describe(
21
- 'The format to return the content in (text, markdown, or html)'
27
+ 'The format to return the content in (text, markdown, or html). Defaults to markdown.'
22
28
  ),
23
29
  timeout: z
24
30
  .number()
@@ -26,6 +32,7 @@ export const WebFetchTool = Tool.define('webfetch', {
26
32
  .optional(),
27
33
  }),
28
34
  async execute(params, ctx) {
35
+ const format = params.format ?? 'markdown';
29
36
  // Validate URL
30
37
  if (
31
38
  !params.url.startsWith('http://') &&
@@ -50,12 +57,15 @@ export const WebFetchTool = Tool.define('webfetch', {
50
57
  MAX_TIMEOUT
51
58
  );
52
59
 
53
- const controller = new AbortController();
54
- const timeoutId = setTimeout(() => controller.abort(), timeout);
60
+ // AbortSignal.timeout keeps no timer alive after a failed fetch, and it
61
+ // also aborts a body that trickles in after the headers (upstream
62
+ // bc1840b196 #21378, 69f1ec22e3 #33259).
63
+ const timeoutSignal = AbortSignal.timeout(timeout);
64
+ const signal = AbortSignal.any([timeoutSignal, ctx.abort]);
55
65
 
56
66
  // Build Accept header based on requested format with q parameters for fallbacks
57
67
  let acceptHeader = '*/*';
58
- switch (params.format) {
68
+ switch (format) {
59
69
  case 'markdown':
60
70
  acceptHeader =
61
71
  'text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1';
@@ -73,40 +83,56 @@ export const WebFetchTool = Tool.define('webfetch', {
73
83
  'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8';
74
84
  }
75
85
 
76
- const response = await verboseFetch(params.url, {
77
- signal: AbortSignal.any([controller.signal, ctx.abort]),
78
- headers: {
79
- 'User-Agent':
80
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
81
- Accept: acceptHeader,
82
- 'Accept-Language': 'en-US,en;q=0.9',
83
- },
84
- });
85
-
86
- clearTimeout(timeoutId);
87
-
88
- if (!response.ok) {
89
- throw new Error(`Request failed with status code: ${response.status}`);
90
- }
91
-
92
- // Check content length
93
- const contentLength = response.headers.get('content-length');
94
- if (contentLength && parseInt(contentLength) > MAX_RESPONSE_SIZE) {
95
- throw new Error('Response too large (exceeds 5MB limit)');
96
- }
97
-
98
- const arrayBuffer = await response.arrayBuffer();
99
- if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
100
- throw new Error('Response too large (exceeds 5MB limit)');
86
+ const headers = {
87
+ 'User-Agent':
88
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36',
89
+ Accept: acceptHeader,
90
+ 'Accept-Language': 'en-US,en;q=0.9',
91
+ };
92
+
93
+ let body: Buffer;
94
+ let response: Response;
95
+ try {
96
+ const initial = await verboseFetch(params.url, { signal, headers });
97
+ // Cloudflare challenges a Chrome UA whose TLS fingerprint is not
98
+ // Chrome's, and lets an honest UA through (upstream b978ca11da #10328).
99
+ if (
100
+ initial.status === 403 &&
101
+ initial.headers.get('cf-mitigated') === 'challenge'
102
+ ) {
103
+ await initial.body?.cancel().catch(() => {});
104
+ response = await verboseFetch(params.url, {
105
+ signal,
106
+ headers: { ...headers, 'User-Agent': Branding.BINARY },
107
+ });
108
+ } else {
109
+ response = initial;
110
+ }
111
+
112
+ if (!response.ok) {
113
+ await response.body?.cancel().catch(() => {});
114
+ throw new Error(`Request failed with status code: ${response.status}`);
115
+ }
116
+
117
+ body = await readBounded(
118
+ response,
119
+ MAX_RESPONSE_SIZE,
120
+ () => new Error('Response too large (exceeds 5MB limit)')
121
+ );
122
+ } catch (error) {
123
+ if (timeoutSignal.aborted && !ctx.abort.aborted) {
124
+ throw new Error(`Request timed out after ${timeout / 1000} seconds`);
125
+ }
126
+ throw error;
101
127
  }
102
128
 
103
- const content = new TextDecoder().decode(arrayBuffer);
129
+ const content = new TextDecoder().decode(body);
104
130
  const contentType = response.headers.get('content-type') || '';
105
131
 
106
132
  const title = `${params.url} (${contentType})`;
107
133
 
108
134
  // Handle content based on requested format and actual content type
109
- switch (params.format) {
135
+ switch (format) {
110
136
  case 'markdown':
111
137
  if (contentType.includes('text/html')) {
112
138
  const markdown = convertHTMLToMarkdown(content);
@@ -156,37 +182,23 @@ export const WebFetchTool = Tool.define('webfetch', {
156
182
 
157
183
  async function extractTextFromHTML(html: string) {
158
184
  let text = '';
159
- let skipContent = false;
185
+ // Count open skipped elements so text after `</script>` and text outside
186
+ // any element is kept (upstream ad6a8a1850, #26309).
187
+ let skipDepth = 0;
160
188
 
161
189
  const rewriter = new HTMLRewriter()
162
190
  .on('script, style, noscript, iframe, object, embed', {
163
- element() {
164
- skipContent = true;
165
- },
166
- text() {
167
- // Skip text content inside these elements
168
- },
169
- })
170
- .on('*', {
171
191
  element(element) {
172
- // Reset skip flag when entering other elements
173
- if (
174
- ![
175
- 'script',
176
- 'style',
177
- 'noscript',
178
- 'iframe',
179
- 'object',
180
- 'embed',
181
- ].includes(element.tagName)
182
- ) {
183
- skipContent = false;
184
- }
192
+ if (!element.canHaveContent) return;
193
+ skipDepth++;
194
+ element.onEndTag(() => {
195
+ skipDepth--;
196
+ });
185
197
  },
198
+ })
199
+ .onDocument({
186
200
  text(input) {
187
- if (!skipContent) {
188
- text += input.text;
189
- }
201
+ if (skipDepth === 0) text += input.text;
190
202
  },
191
203
  })
192
204
  .transform(new Response(html));
@@ -1,14 +1,14 @@
1
1
  - Fetches content from a specified URL
2
- - Takes a URL and a prompt as input
3
- - Fetches the URL content, converts HTML to markdown
4
- - Returns the model's response about the content
2
+ - Takes a URL and optional format as input
3
+ - Fetches the URL content, converts to requested format (markdown by default)
4
+ - Returns the content in the specified format
5
5
  - Use this tool when you need to retrieve and analyze web content
6
6
 
7
7
  Usage notes:
8
8
  - IMPORTANT: if another tool is present that offers better web fetching capabilities, is more targeted to the task, or has fewer restrictions, prefer using that tool instead of this one.
9
9
  - The URL must be a fully-formed valid URL
10
10
  - HTTP URLs will be automatically upgraded to HTTPS
11
- - The prompt should describe what information you want to extract from the page
11
+ - Format options: "markdown" (default), "text", or "html"
12
12
  - This tool is read-only and does not modify any files
13
13
  - Results may be summarized if the content is very large
14
14
  - Includes a self-cleaning 15-minute cache for faster responses when repeatedly accessing the same URL