@link-assistant/agent 0.27.0 → 0.27.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -5
- package/package.json +41 -42
- package/src/cli/argv.ts +1 -1
- package/src/cli/cmd/run.ts +7 -1
- package/src/cli/continuous-mode.js +38 -12
- package/src/cli/event-handler.js +3 -1
- package/src/cli/model-config.js +20 -4
- package/src/cli/run-options.js +144 -78
- package/src/config/config.ts +306 -105
- package/src/config/defaults.ts +13 -5
- package/src/config/file-config.ts +16 -0
- package/src/file/ripgrep.ts +38 -5
- package/src/file/time.ts +86 -9
- package/src/index.js +39 -33
- package/src/mcp/index.ts +370 -106
- package/src/permission/index.ts +22 -0
- package/src/project/instance.ts +5 -1
- package/src/project/project.ts +35 -2
- package/src/provider/free-tier-error.ts +1 -1
- package/src/provider/kilo-deprecation.ts +30 -0
- package/src/provider/kilo.ts +25 -7
- package/src/provider/provider.ts +53 -22
- package/src/provider/retry-fetch.ts +5 -0
- package/src/provider/transform.ts +139 -17
- package/src/session/compaction.ts +18 -2
- package/src/session/index.ts +30 -3
- package/src/session/message-v2.ts +318 -32
- package/src/session/processor.ts +274 -140
- package/src/session/prompt.ts +154 -128
- package/src/session/retry.ts +26 -13
- package/src/session/summary.ts +4 -1
- package/src/tool/bash.ts +232 -45
- package/src/tool/bash.txt +1 -1
- package/src/tool/codesearch.ts +15 -78
- package/src/tool/edit.ts +111 -30
- package/src/tool/glob.ts +2 -1
- package/src/tool/grep.ts +175 -57
- package/src/tool/ls.ts +2 -1
- package/src/tool/multiedit.ts +3 -1
- package/src/tool/read.ts +169 -41
- package/src/tool/task.ts +26 -18
- package/src/tool/tool.ts +4 -1
- package/src/tool/webfetch.ts +68 -56
- package/src/tool/webfetch.txt +4 -4
- package/src/tool/websearch.ts +20 -75
- package/src/tool/websearch.txt +2 -0
- package/src/tool/write.ts +23 -6
- package/src/util/bom.ts +32 -0
- package/src/util/exa-mcp.ts +82 -0
- package/src/util/legacy-error-names.ts +39 -0
- package/src/util/read-bounded.ts +32 -0
- package/src/util/sse-usage-extractor.ts +67 -13
- package/src/util/timeout.ts +3 -4
- package/src/util/verbose-fetch.ts +40 -1
package/src/tool/read.ts
CHANGED
|
@@ -12,6 +12,13 @@ import { formatBalancedLineWindow, formatColumnWindow } from './text-window';
|
|
|
12
12
|
|
|
13
13
|
const DEFAULT_READ_LIMIT = 2000;
|
|
14
14
|
const MAX_LINE_LENGTH = 2000;
|
|
15
|
+
// upstream 1b82511fbd / c1b03b728a (#14009): the output is bounded in bytes
|
|
16
|
+
// too (config.readMaxBytes, 50 KB by default), 2000 long lines would otherwise
|
|
17
|
+
// put megabytes into the context.
|
|
18
|
+
|
|
19
|
+
function formatSize(bytes: number) {
|
|
20
|
+
return bytes % 1024 === 0 ? `${bytes / 1024} KB` : `${bytes} bytes`;
|
|
21
|
+
}
|
|
15
22
|
|
|
16
23
|
export const ReadTool = Tool.define('read', {
|
|
17
24
|
description: DESCRIPTION,
|
|
@@ -36,18 +43,42 @@ export const ReadTool = Tool.define('read', {
|
|
|
36
43
|
}),
|
|
37
44
|
async execute(params, ctx) {
|
|
38
45
|
let filepath = params.filePath;
|
|
39
|
-
|
|
40
|
-
|
|
46
|
+
const relative = !path.isAbsolute(filepath);
|
|
47
|
+
if (relative) {
|
|
48
|
+
// Relative to the session directory, like every other tool, not to the
|
|
49
|
+
// worktree root (upstream 6cf2c3e3db).
|
|
50
|
+
filepath = path.resolve(Instance.directory, filepath);
|
|
41
51
|
}
|
|
42
|
-
const title = path.relative(Instance.worktree, filepath);
|
|
43
52
|
|
|
44
53
|
// No restrictions - unrestricted file read
|
|
45
|
-
|
|
46
|
-
if (!
|
|
54
|
+
let stat = await fs.promises.stat(filepath).catch(() => undefined);
|
|
55
|
+
if (!stat && relative && Instance.directory !== Instance.worktree) {
|
|
56
|
+
// Backward compatibility: relative paths used to resolve against the
|
|
57
|
+
// worktree root. Keep that working when the path does not exist under
|
|
58
|
+
// the session directory.
|
|
59
|
+
const fromWorktree = path.resolve(Instance.worktree, params.filePath);
|
|
60
|
+
const worktreeStat = await fs.promises
|
|
61
|
+
.stat(fromWorktree)
|
|
62
|
+
.catch(() => undefined);
|
|
63
|
+
if (worktreeStat) {
|
|
64
|
+
filepath = fromWorktree;
|
|
65
|
+
stat = worktreeStat;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
if (!stat) {
|
|
69
|
+
const alternate = alternatePath(filepath);
|
|
70
|
+
if (alternate) {
|
|
71
|
+
filepath = alternate;
|
|
72
|
+
stat = await fs.promises.stat(filepath).catch(() => undefined);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const title = path.relative(Instance.worktree, filepath);
|
|
76
|
+
if (!stat) {
|
|
47
77
|
const dir = path.dirname(filepath);
|
|
48
78
|
const base = path.basename(filepath);
|
|
49
79
|
|
|
50
|
-
|
|
80
|
+
// A missing parent directory is still just "File not found".
|
|
81
|
+
const dirEntries = fs.existsSync(dir) ? fs.readdirSync(dir) : [];
|
|
51
82
|
const suggestions = dirEntries
|
|
52
83
|
.filter(
|
|
53
84
|
(entry) =>
|
|
@@ -65,6 +96,16 @@ export const ReadTool = Tool.define('read', {
|
|
|
65
96
|
|
|
66
97
|
throw new Error(`File not found: ${filepath}`);
|
|
67
98
|
}
|
|
99
|
+
// upstream 823d327401 / ff837fe949 (#33255/#33260)
|
|
100
|
+
if (stat.isDirectory()) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`Path is a directory, not a file: ${filepath}. Use the ls or glob tool to list it.`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
if (!stat.isFile()) {
|
|
106
|
+
throw new Error(`Path is not a regular file: ${filepath}`);
|
|
107
|
+
}
|
|
108
|
+
const file = Bun.file(filepath);
|
|
68
109
|
|
|
69
110
|
const isImage = isImageFile(filepath);
|
|
70
111
|
const supportsImages = await (async () => {
|
|
@@ -85,9 +126,8 @@ export const ReadTool = Tool.define('read', {
|
|
|
85
126
|
`Failed to read image: ${filepath}, model may not be able to read images`
|
|
86
127
|
);
|
|
87
128
|
}
|
|
129
|
+
const bytes = await file.bytes();
|
|
88
130
|
if (verifyImages) {
|
|
89
|
-
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
90
|
-
|
|
91
131
|
// Validate image format matches file extension
|
|
92
132
|
if (!validateImageFormat(bytes, isImage)) {
|
|
93
133
|
throw new Error(
|
|
@@ -118,54 +158,101 @@ export const ReadTool = Tool.define('read', {
|
|
|
118
158
|
messageID: ctx.messageID,
|
|
119
159
|
type: 'file',
|
|
120
160
|
mime,
|
|
121
|
-
url: `data:${mime};base64,${Buffer.from(
|
|
161
|
+
url: `data:${mime};base64,${Buffer.from(bytes).toString('base64')}`,
|
|
122
162
|
},
|
|
123
163
|
],
|
|
124
164
|
};
|
|
125
165
|
}
|
|
126
166
|
|
|
127
|
-
const isBinary = await isBinaryFile(filepath,
|
|
167
|
+
const isBinary = await isBinaryFile(filepath, stat.size);
|
|
128
168
|
if (isBinary) throw new Error(`Cannot read binary file: ${filepath}`);
|
|
129
169
|
|
|
130
|
-
|
|
170
|
+
// upstream 683d470fe4 (#51011): never return more than the default limit.
|
|
171
|
+
// --read-max-lines / --read-max-bytes tune both caps; 0 disables a cap.
|
|
172
|
+
const maxLines = config.readMaxLines > 0 ? config.readMaxLines : Infinity;
|
|
173
|
+
const maxBytes = config.readMaxBytes > 0 ? config.readMaxBytes : Infinity;
|
|
174
|
+
const limit = Math.min(
|
|
175
|
+
normalizeNonNegativeInteger(
|
|
176
|
+
params.limit,
|
|
177
|
+
Math.min(DEFAULT_READ_LIMIT, maxLines)
|
|
178
|
+
),
|
|
179
|
+
maxLines
|
|
180
|
+
);
|
|
131
181
|
const offset = normalizeNonNegativeInteger(params.offset, 0);
|
|
132
182
|
const hasExplicitLineRange =
|
|
133
183
|
params.offset !== undefined || params.limit !== undefined;
|
|
134
184
|
const hasExplicitColumnRange =
|
|
135
185
|
params.columnOffset !== undefined || params.columnLimit !== undefined;
|
|
136
186
|
const columnOffset = normalizeNonNegativeInteger(params.columnOffset, 0);
|
|
137
|
-
const columnLimit =
|
|
138
|
-
params.columnLimit,
|
|
139
|
-
|
|
187
|
+
const columnLimit = Math.min(
|
|
188
|
+
normalizeNonNegativeInteger(params.columnLimit, MAX_LINE_LENGTH),
|
|
189
|
+
maxBytes
|
|
140
190
|
);
|
|
141
|
-
const lines = await file.text()
|
|
191
|
+
const lines = splitLines(await file.text());
|
|
192
|
+
// upstream c1b03b728a (#14009)
|
|
193
|
+
// --legacy-tool-output keeps the old empty result past the end.
|
|
194
|
+
if (
|
|
195
|
+
!config.legacyToolOutput &&
|
|
196
|
+
hasExplicitLineRange &&
|
|
197
|
+
lines.length > 0 &&
|
|
198
|
+
offset >= lines.length
|
|
199
|
+
) {
|
|
200
|
+
throw new Error(
|
|
201
|
+
`Offset ${offset} is out of range for this file (${lines.length} lines)`
|
|
202
|
+
);
|
|
203
|
+
}
|
|
142
204
|
const selected = selectLines({
|
|
143
205
|
lines,
|
|
144
206
|
offset,
|
|
145
207
|
limit,
|
|
146
208
|
summarizeMiddle: !hasExplicitLineRange,
|
|
147
209
|
});
|
|
148
|
-
const
|
|
210
|
+
const formatLine = ({
|
|
211
|
+
line,
|
|
212
|
+
lineNumber,
|
|
213
|
+
}: {
|
|
214
|
+
line: string;
|
|
215
|
+
lineNumber: number;
|
|
216
|
+
}) => {
|
|
217
|
+
const formattedLine = hasExplicitColumnRange
|
|
218
|
+
? formatColumnWindow({
|
|
219
|
+
line,
|
|
220
|
+
lineNumber,
|
|
221
|
+
offset: columnOffset,
|
|
222
|
+
limit: columnLimit,
|
|
223
|
+
})
|
|
224
|
+
: formatBalancedLineWindow({
|
|
225
|
+
line,
|
|
226
|
+
lineNumber,
|
|
227
|
+
limit: MAX_LINE_LENGTH,
|
|
228
|
+
});
|
|
229
|
+
return `${lineNumber.toString().padStart(5, '0')}| ${formattedLine}`;
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
const content: string[] = [];
|
|
233
|
+
let bytes = 0;
|
|
234
|
+
let capped: { first: number; last: number } | undefined;
|
|
235
|
+
for (const part of selected.parts) {
|
|
149
236
|
if (part.type === 'omitted') {
|
|
150
|
-
|
|
237
|
+
content.push(`... [omitted lines ${part.start}..${part.end}] ...`);
|
|
238
|
+
continue;
|
|
151
239
|
}
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
});
|
|
240
|
+
for (const item of part.lines) {
|
|
241
|
+
const formatted = formatLine(item);
|
|
242
|
+
const size = Buffer.byteLength(formatted, 'utf8') + 1;
|
|
243
|
+
// Always show at least one line so the cap cannot return nothing.
|
|
244
|
+
if (bytes + size > maxBytes && content.length > 0) {
|
|
245
|
+
capped = {
|
|
246
|
+
first: part.lines[0].lineNumber,
|
|
247
|
+
last: item.lineNumber - 1,
|
|
248
|
+
};
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
content.push(formatted);
|
|
252
|
+
bytes += size;
|
|
253
|
+
}
|
|
254
|
+
if (capped) break;
|
|
255
|
+
}
|
|
169
256
|
const preview = content.slice(0, 20).join('\n');
|
|
170
257
|
|
|
171
258
|
let output = '<file>\n';
|
|
@@ -175,7 +262,10 @@ export const ReadTool = Tool.define('read', {
|
|
|
175
262
|
const lastReadLine = selected.lastLine;
|
|
176
263
|
const hasMoreLines = hasExplicitLineRange && totalLines > lastReadLine;
|
|
177
264
|
|
|
178
|
-
if (
|
|
265
|
+
if (capped) {
|
|
266
|
+
// `offset` is 0-based, so the next unread line starts at `last`.
|
|
267
|
+
output += `\n\n(Output capped at ${formatSize(maxBytes)}. Showing lines ${capped.first}-${capped.last}. Use offset=${capped.last} to continue.)`;
|
|
268
|
+
} else if (hasMoreLines) {
|
|
179
269
|
output += `\n\n(File has more lines. Use 'offset' parameter to read beyond line ${lastReadLine})`;
|
|
180
270
|
} else {
|
|
181
271
|
output += `\n\n(End of file - total ${totalLines} lines)`;
|
|
@@ -195,6 +285,40 @@ export const ReadTool = Tool.define('read', {
|
|
|
195
285
|
},
|
|
196
286
|
});
|
|
197
287
|
|
|
288
|
+
// CRLF files must not leak `\r` into the output, and a trailing newline ends
|
|
289
|
+
// the last line instead of starting an empty one (upstream c1b03b728a,
|
|
290
|
+
// 683d470fe4).
|
|
291
|
+
function splitLines(text: string) {
|
|
292
|
+
const lines = text.split(/\r?\n/);
|
|
293
|
+
if (lines.length > 1 && lines[lines.length - 1] === '') lines.pop();
|
|
294
|
+
return lines;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// macOS puts U+202F (narrow no-break space) before AM/PM in screenshot names
|
|
298
|
+
// and models type a plain space; curly apostrophes get straightened the same
|
|
299
|
+
// way. Use the only directory entry that matches after canonicalizing both
|
|
300
|
+
// (upstream d484f070d1 #45807, b47824fb63 #50953).
|
|
301
|
+
function canonicalName(name: string) {
|
|
302
|
+
return name
|
|
303
|
+
.normalize('NFC')
|
|
304
|
+
.replace(/[\u00a0\u202f]/g, ' ')
|
|
305
|
+
.replace(/[\u2018\u2019]/g, "'");
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function alternatePath(filepath: string): string | undefined {
|
|
309
|
+
const dir = path.dirname(filepath);
|
|
310
|
+
const base = canonicalName(path.basename(filepath));
|
|
311
|
+
let entries: string[];
|
|
312
|
+
try {
|
|
313
|
+
entries = fs.readdirSync(dir);
|
|
314
|
+
} catch {
|
|
315
|
+
return undefined;
|
|
316
|
+
}
|
|
317
|
+
const matches = entries.filter((entry) => canonicalName(entry) === base);
|
|
318
|
+
if (matches.length !== 1) return undefined;
|
|
319
|
+
return path.join(dir, matches[0]);
|
|
320
|
+
}
|
|
321
|
+
|
|
198
322
|
function normalizeNonNegativeInteger(
|
|
199
323
|
value: number | undefined,
|
|
200
324
|
fallback: number
|
|
@@ -386,7 +510,7 @@ function validateImageFormat(
|
|
|
386
510
|
|
|
387
511
|
async function isBinaryFile(
|
|
388
512
|
filepath: string,
|
|
389
|
-
|
|
513
|
+
fileSize: number
|
|
390
514
|
): Promise<boolean> {
|
|
391
515
|
const ext = path.extname(filepath).toLowerCase();
|
|
392
516
|
// binary check for common non-text extensions
|
|
@@ -424,14 +548,18 @@ async function isBinaryFile(
|
|
|
424
548
|
break;
|
|
425
549
|
}
|
|
426
550
|
|
|
427
|
-
const stat = await file.stat();
|
|
428
|
-
const fileSize = stat.size;
|
|
429
551
|
if (fileSize === 0) return false;
|
|
430
552
|
|
|
553
|
+
// Sample only the first 4 KB instead of loading the whole file
|
|
554
|
+
// (upstream c1b03b728a #14009).
|
|
431
555
|
const bufferSize = Math.min(4096, fileSize);
|
|
432
|
-
const
|
|
433
|
-
|
|
434
|
-
const
|
|
556
|
+
const fh = await fs.promises.open(filepath, 'r');
|
|
557
|
+
const sample = Buffer.alloc(bufferSize);
|
|
558
|
+
const { bytesRead } = await fh
|
|
559
|
+
.read(sample, 0, bufferSize, 0)
|
|
560
|
+
.finally(() => fh.close());
|
|
561
|
+
if (bytesRead === 0) return false;
|
|
562
|
+
const bytes = sample.subarray(0, bytesRead);
|
|
435
563
|
|
|
436
564
|
let nonPrintableCount = 0;
|
|
437
565
|
for (let i = 0; i < bytes.length; i++) {
|
package/src/tool/task.ts
CHANGED
|
@@ -110,24 +110,32 @@ export const TaskTool = Tool.define('task', async () => {
|
|
|
110
110
|
}
|
|
111
111
|
ctx.abort.addEventListener('abort', cancel);
|
|
112
112
|
using _ = defer(() => ctx.abort.removeEventListener('abort', cancel));
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
113
|
+
// Unsubscribe even when the subagent fails, or the subscription lives
|
|
114
|
+
// as long as the instance (upstream 63f5669eb5, #9992).
|
|
115
|
+
let result: Awaited<ReturnType<typeof SessionPrompt.prompt>>;
|
|
116
|
+
try {
|
|
117
|
+
const promptParts = await SessionPrompt.resolvePromptParts(
|
|
118
|
+
params.prompt
|
|
119
|
+
);
|
|
120
|
+
result = await SessionPrompt.prompt({
|
|
121
|
+
messageID,
|
|
122
|
+
sessionID: session.id,
|
|
123
|
+
model: {
|
|
124
|
+
modelID: model.modelID,
|
|
125
|
+
providerID: model.providerID,
|
|
126
|
+
},
|
|
127
|
+
agent: agent.name,
|
|
128
|
+
tools: {
|
|
129
|
+
todowrite: false,
|
|
130
|
+
todoread: false,
|
|
131
|
+
task: false,
|
|
132
|
+
...agent.tools,
|
|
133
|
+
},
|
|
134
|
+
parts: promptParts,
|
|
135
|
+
});
|
|
136
|
+
} finally {
|
|
137
|
+
unsub();
|
|
138
|
+
}
|
|
131
139
|
let all;
|
|
132
140
|
all = await Session.messages({ sessionID: session.id });
|
|
133
141
|
all = all.filter((x) => x.info.role === 'assistant');
|
package/src/tool/tool.ts
CHANGED
|
@@ -87,7 +87,10 @@ export namespace Tool {
|
|
|
87
87
|
return {
|
|
88
88
|
id,
|
|
89
89
|
init: async () => {
|
|
90
|
-
|
|
90
|
+
// Copy object-defined tools so each init() wraps the original
|
|
91
|
+
// execute once instead of stacking another wrapper on the shared
|
|
92
|
+
// object (upstream 81d3ac3bf0, #16952).
|
|
93
|
+
const toolInfo = init instanceof Function ? await init() : { ...init };
|
|
91
94
|
const execute = toolInfo.execute;
|
|
92
95
|
toolInfo.execute = (args, ctx) => {
|
|
93
96
|
try {
|
package/src/tool/webfetch.ts
CHANGED
|
@@ -4,6 +4,8 @@ import TurndownService from 'turndown';
|
|
|
4
4
|
import DESCRIPTION from './webfetch.txt';
|
|
5
5
|
import { createVerboseFetch } from '../util/verbose-fetch';
|
|
6
6
|
import { Permission } from '../permission';
|
|
7
|
+
import { Branding } from '../branding';
|
|
8
|
+
import { readBounded } from '../util/read-bounded';
|
|
7
9
|
|
|
8
10
|
const verboseFetch = createVerboseFetch(fetch, { caller: 'webfetch' });
|
|
9
11
|
|
|
@@ -15,10 +17,14 @@ export const WebFetchTool = Tool.define('webfetch', {
|
|
|
15
17
|
description: DESCRIPTION,
|
|
16
18
|
parameters: z.object({
|
|
17
19
|
url: z.string().describe('The URL to fetch content from'),
|
|
20
|
+
// Optional, not `.default()`: the JSON schema sent to the model would
|
|
21
|
+
// still list a defaulted field as required, and execute receives the raw
|
|
22
|
+
// arguments (upstream fd973d242e #6345, 2d2f587bfa #30215).
|
|
18
23
|
format: z
|
|
19
24
|
.enum(['text', 'markdown', 'html'])
|
|
25
|
+
.optional()
|
|
20
26
|
.describe(
|
|
21
|
-
'The format to return the content in (text, markdown, or html)'
|
|
27
|
+
'The format to return the content in (text, markdown, or html). Defaults to markdown.'
|
|
22
28
|
),
|
|
23
29
|
timeout: z
|
|
24
30
|
.number()
|
|
@@ -26,6 +32,7 @@ export const WebFetchTool = Tool.define('webfetch', {
|
|
|
26
32
|
.optional(),
|
|
27
33
|
}),
|
|
28
34
|
async execute(params, ctx) {
|
|
35
|
+
const format = params.format ?? 'markdown';
|
|
29
36
|
// Validate URL
|
|
30
37
|
if (
|
|
31
38
|
!params.url.startsWith('http://') &&
|
|
@@ -50,12 +57,15 @@ export const WebFetchTool = Tool.define('webfetch', {
|
|
|
50
57
|
MAX_TIMEOUT
|
|
51
58
|
);
|
|
52
59
|
|
|
53
|
-
|
|
54
|
-
|
|
60
|
+
// AbortSignal.timeout keeps no timer alive after a failed fetch, and it
|
|
61
|
+
// also aborts a body that trickles in after the headers (upstream
|
|
62
|
+
// bc1840b196 #21378, 69f1ec22e3 #33259).
|
|
63
|
+
const timeoutSignal = AbortSignal.timeout(timeout);
|
|
64
|
+
const signal = AbortSignal.any([timeoutSignal, ctx.abort]);
|
|
55
65
|
|
|
56
66
|
// Build Accept header based on requested format with q parameters for fallbacks
|
|
57
67
|
let acceptHeader = '*/*';
|
|
58
|
-
switch (
|
|
68
|
+
switch (format) {
|
|
59
69
|
case 'markdown':
|
|
60
70
|
acceptHeader =
|
|
61
71
|
'text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1';
|
|
@@ -73,40 +83,56 @@ export const WebFetchTool = Tool.define('webfetch', {
|
|
|
73
83
|
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8';
|
|
74
84
|
}
|
|
75
85
|
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
86
|
+
const headers = {
|
|
87
|
+
'User-Agent':
|
|
88
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36',
|
|
89
|
+
Accept: acceptHeader,
|
|
90
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
let body: Buffer;
|
|
94
|
+
let response: Response;
|
|
95
|
+
try {
|
|
96
|
+
const initial = await verboseFetch(params.url, { signal, headers });
|
|
97
|
+
// Cloudflare challenges a Chrome UA whose TLS fingerprint is not
|
|
98
|
+
// Chrome's, and lets an honest UA through (upstream b978ca11da #10328).
|
|
99
|
+
if (
|
|
100
|
+
initial.status === 403 &&
|
|
101
|
+
initial.headers.get('cf-mitigated') === 'challenge'
|
|
102
|
+
) {
|
|
103
|
+
await initial.body?.cancel().catch(() => {});
|
|
104
|
+
response = await verboseFetch(params.url, {
|
|
105
|
+
signal,
|
|
106
|
+
headers: { ...headers, 'User-Agent': Branding.BINARY },
|
|
107
|
+
});
|
|
108
|
+
} else {
|
|
109
|
+
response = initial;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if (!response.ok) {
|
|
113
|
+
await response.body?.cancel().catch(() => {});
|
|
114
|
+
throw new Error(`Request failed with status code: ${response.status}`);
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
body = await readBounded(
|
|
118
|
+
response,
|
|
119
|
+
MAX_RESPONSE_SIZE,
|
|
120
|
+
() => new Error('Response too large (exceeds 5MB limit)')
|
|
121
|
+
);
|
|
122
|
+
} catch (error) {
|
|
123
|
+
if (timeoutSignal.aborted && !ctx.abort.aborted) {
|
|
124
|
+
throw new Error(`Request timed out after ${timeout / 1000} seconds`);
|
|
125
|
+
}
|
|
126
|
+
throw error;
|
|
101
127
|
}
|
|
102
128
|
|
|
103
|
-
const content = new TextDecoder().decode(
|
|
129
|
+
const content = new TextDecoder().decode(body);
|
|
104
130
|
const contentType = response.headers.get('content-type') || '';
|
|
105
131
|
|
|
106
132
|
const title = `${params.url} (${contentType})`;
|
|
107
133
|
|
|
108
134
|
// Handle content based on requested format and actual content type
|
|
109
|
-
switch (
|
|
135
|
+
switch (format) {
|
|
110
136
|
case 'markdown':
|
|
111
137
|
if (contentType.includes('text/html')) {
|
|
112
138
|
const markdown = convertHTMLToMarkdown(content);
|
|
@@ -156,37 +182,23 @@ export const WebFetchTool = Tool.define('webfetch', {
|
|
|
156
182
|
|
|
157
183
|
async function extractTextFromHTML(html: string) {
|
|
158
184
|
let text = '';
|
|
159
|
-
|
|
185
|
+
// Count open skipped elements so text after `</script>` and text outside
|
|
186
|
+
// any element is kept (upstream ad6a8a1850, #26309).
|
|
187
|
+
let skipDepth = 0;
|
|
160
188
|
|
|
161
189
|
const rewriter = new HTMLRewriter()
|
|
162
190
|
.on('script, style, noscript, iframe, object, embed', {
|
|
163
|
-
element() {
|
|
164
|
-
skipContent = true;
|
|
165
|
-
},
|
|
166
|
-
text() {
|
|
167
|
-
// Skip text content inside these elements
|
|
168
|
-
},
|
|
169
|
-
})
|
|
170
|
-
.on('*', {
|
|
171
191
|
element(element) {
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
'noscript',
|
|
178
|
-
'iframe',
|
|
179
|
-
'object',
|
|
180
|
-
'embed',
|
|
181
|
-
].includes(element.tagName)
|
|
182
|
-
) {
|
|
183
|
-
skipContent = false;
|
|
184
|
-
}
|
|
192
|
+
if (!element.canHaveContent) return;
|
|
193
|
+
skipDepth++;
|
|
194
|
+
element.onEndTag(() => {
|
|
195
|
+
skipDepth--;
|
|
196
|
+
});
|
|
185
197
|
},
|
|
198
|
+
})
|
|
199
|
+
.onDocument({
|
|
186
200
|
text(input) {
|
|
187
|
-
if (
|
|
188
|
-
text += input.text;
|
|
189
|
-
}
|
|
201
|
+
if (skipDepth === 0) text += input.text;
|
|
190
202
|
},
|
|
191
203
|
})
|
|
192
204
|
.transform(new Response(html));
|
package/src/tool/webfetch.txt
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
- Fetches content from a specified URL
|
|
2
|
-
- Takes a URL and
|
|
3
|
-
- Fetches the URL content, converts
|
|
4
|
-
- Returns the
|
|
2
|
+
- Takes a URL and optional format as input
|
|
3
|
+
- Fetches the URL content, converts to requested format (markdown by default)
|
|
4
|
+
- Returns the content in the specified format
|
|
5
5
|
- Use this tool when you need to retrieve and analyze web content
|
|
6
6
|
|
|
7
7
|
Usage notes:
|
|
8
8
|
- IMPORTANT: if another tool is present that offers better web fetching capabilities, is more targeted to the task, or has fewer restrictions, prefer using that tool instead of this one.
|
|
9
9
|
- The URL must be a fully-formed valid URL
|
|
10
10
|
- HTTP URLs will be automatically upgraded to HTTPS
|
|
11
|
-
-
|
|
11
|
+
- Format options: "markdown" (default), "text", or "html"
|
|
12
12
|
- This tool is read-only and does not modify any files
|
|
13
13
|
- Results may be summarized if the content is very large
|
|
14
14
|
- Includes a self-cleaning 15-minute cache for faster responses when repeatedly accessing the same URL
|