@hncode/hncode 0.5.6 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +37 -38
- package/src/agent.js +67 -2
- package/src/image-row.js +111 -0
- package/src/latex.js +1557 -0
- package/src/llm.js +19 -4
- package/src/mermaid/ansi.js +55 -0
- package/src/mermaid/canvas.js +449 -0
- package/src/mermaid/class-style.js +57 -0
- package/src/mermaid/css-colors.js +153 -0
- package/src/mermaid/diagrams/class.js +289 -0
- package/src/mermaid/diagrams/er.js +201 -0
- package/src/mermaid/diagrams/flowchart.js +488 -0
- package/src/mermaid/diagrams/gitgraph.js +205 -0
- package/src/mermaid/diagrams/mindmap.js +101 -0
- package/src/mermaid/diagrams/pie.js +104 -0
- package/src/mermaid/diagrams/sequence.js +282 -0
- package/src/mermaid/diagrams/state.js +292 -0
- package/src/mermaid/diagrams/timeline.js +99 -0
- package/src/mermaid/graph.js +138 -0
- package/src/mermaid/index.js +59 -0
- package/src/mermaid/labels.js +339 -0
- package/src/mermaid/layout-seq.js +225 -0
- package/src/mermaid/layout.js +1454 -0
- package/src/mermaid/registry.js +54 -0
- package/src/mermaid/source-box.js +102 -0
- package/src/mermaid/statements.js +235 -0
- package/src/mermaid/types.js +1 -0
- package/src/mermaid/width.js +29 -0
- package/src/screen-diff.js +512 -0
- package/src/screen-paint.js +128 -0
- package/src/screen.js +277 -0
- package/src/subagent-types.js +136 -147
- package/src/term-caps.js +10 -4
- package/src/tex/font.js +292 -0
- package/src/tex/fonts/LICENSE.txt +11 -0
- package/src/tex/fonts/STIX2Math.ttf +0 -0
- package/src/tex/index.js +293 -0
- package/src/tex/kitty.js +186 -0
- package/src/tex/layout.js +446 -0
- package/src/tex/parse.js +331 -0
- package/src/tex/raster.js +216 -0
- package/src/tex/render.js +167 -0
- package/src/tex/sixel.js +326 -0
- package/src/tex/svg.js +237 -0
- package/src/tool-result.js +2 -2
- package/src/tools/agent-swarm.js +3 -15
- package/src/tools/agent.js +454 -464
- package/src/tools/ask-user-question.js +195 -206
- package/src/tools/bash.js +39 -3
- package/src/tools/edit.js +10 -112
- package/src/tools/fetch-url.js +130 -120
- package/src/tools/memory.js +80 -88
- package/src/tools/read.js +27 -23
- package/src/tools/todo.js +8 -13
- package/src/tools/validate-args.js +78 -0
- package/src/tui.js +1346 -502
package/src/tools/edit.js
CHANGED
|
@@ -28,7 +28,7 @@ import { hashRegion as hashLines } from './read.js';
|
|
|
28
28
|
|
|
29
29
|
export const spec = {
|
|
30
30
|
name: 'Edit',
|
|
31
|
-
description: 'Edit a file.
|
|
31
|
+
description: 'Edit a file. Three shapes: LINE-RANGE (path + start_line + end_line + new_string), SUBSTRING (path + old_string + new_string, unique unless replace_all), RANGED SUBSTRING (the same plus start_line + end_line, which narrows where old_string is searched). \nBOTH BOUNDS ARE INCLUSIVE: start_line=3, end_line=5 replaces lines 3, 4 and 5 — three lines, so new_string must have exactly three lines. A fourth line is a duplicate, not an addition. \nnew_string is VERBATIM: no re-indent, no trim. Every line carries its own leading whitespace, the first one included — copy it from the Read output after the tab that follows each line number. old_string is matched exactly, including its own first-line indentation. \nOmitting new_string is an error. After a multi-line edit, Read the region again: mis-indented code still parses, so a green test is not proof the edit landed where you meant.',
|
|
32
32
|
parameters: {
|
|
33
33
|
type: 'object',
|
|
34
34
|
properties: {
|
|
@@ -37,9 +37,8 @@ export const spec = {
|
|
|
37
37
|
new_string: { type: 'string', description: 'Replacement text, written VERBATIM in BOTH modes — no indentation is added, removed, or re-derived. Write the leading spaces on EVERY line yourself, including the FIRST one, copying them from the Read output after its `N\\t` line-number prefix (Read prints `12\\t return x`, so you send ` return x`). In LINE-RANGE mode it must contain EXACTLY the lines being replaced: a line that sits outside [start_line, end_line] is not deleted, so restating it here leaves a duplicate. In SUBSTRING mode it replaces old_string, which is itself matched exactly including its own indentation.' },
|
|
38
38
|
|
|
39
39
|
replace_all: { type: 'boolean', default: false, description: 'Replace all occurrences of old_string.' },
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
end_line: { type: 'integer', minimum: 1, description: 'PREFERRED MODE. 1-based last line to replace (inclusive; requires start_line). With old_string too, it is the last line to search in.' },
|
|
40
|
+
start_line: { type: 'integer', minimum: 1, description: 'PREFERRED MODE. 1-based first line to replace, INCLUSIVE — the range covers start_line and end_line and every line between them. Requires end_line and new_string. With old_string too, it is the first line to SEARCH for it in.' },
|
|
41
|
+
end_line: { type: 'integer', minimum: 1, description: 'PREFERRED MODE. 1-based last line to replace, INCLUSIVE (requires start_line). start_line=3, end_line=5 replaces lines 3, 4 AND 5 — three lines, so new_string must have three lines. With old_string too, it is the last line to search in.' },
|
|
43
42
|
},
|
|
44
43
|
// Three accepted shapes:
|
|
45
44
|
// line-range start_line + end_line + new_string
|
|
@@ -190,50 +189,7 @@ export const spec = {
|
|
|
190
189
|
// the caller diff the replaced region -> `new_string`.
|
|
191
190
|
const replacedText = lines.slice(start - 1, end).join('\n');
|
|
192
191
|
|
|
193
|
-
|
|
194
|
-
//
|
|
195
|
-
// The recurring mistake: `new_string`'s last line is the same text as the line at `end_line + 1`.
|
|
196
|
-
// That line is OUTSIDE the replaced range, so it is never deleted — the copy inside `new_string`
|
|
197
|
-
// lands directly above it and the file holds the text twice. Every line number below then shifts.
|
|
198
|
-
//
|
|
199
|
-
// REFUSING was the first attempt and it was wrong twice over. It rejected legitimate edits — a
|
|
200
|
-
// one-line `new_string` against a six-line range matched the following line by coincidence and the
|
|
201
|
-
// edit was blocked — and it put the burden on the caller to work out whose line it was.
|
|
202
|
-
//
|
|
203
|
-
// DROPPING THE DUPLICATE is what the caller meant either way. If they restated a line they did not
|
|
204
|
-
// intend to replace, removing the restatement produces exactly the file they described; if they DID
|
|
205
|
-
// want it changed, the range was wrong — but silently producing a duplicate helps no one, so the
|
|
206
|
-
// reply says what was dropped and how to keep it.
|
|
207
|
-
//
|
|
208
|
-
// `allow_dup: true` keeps the duplicate, for the real case where two identical lines are the point.
|
|
209
|
-
// REMOVE ADJACENT DUPLICATES FROM THE RESULT — the file the caller will open.
|
|
210
|
-
//
|
|
211
|
-
// Nothing here tries to work out WHICH line the caller meant; that guessing is what made every
|
|
212
|
-
// previous version leak. The rule is the one a reader would apply: if the edit leaves two identical
|
|
213
|
-
// lines next to each other, one of them is redundant, so it goes.
|
|
214
|
-
//
|
|
215
|
-
// `new_string` is what gets adjusted, because the file's own lines were already there — editing
|
|
216
|
-
// them would delete content the caller never asked to touch.
|
|
217
|
-
//
|
|
218
|
-
// A blank or whitespace-only line between two identical lines counts as adjacent: the blank is
|
|
219
|
-
// almost always the tail of a pasted block, and leaving it strands a gap where the duplicate was.
|
|
220
|
-
//
|
|
221
|
-
// `allow_dup: true` skips all of this and writes exactly what was asked for.
|
|
222
|
-
let insertedFinal = inserted;
|
|
223
|
-
const drops = [];
|
|
224
|
-
if (args.allow_dup !== true) {
|
|
225
|
-
// One pass per candidate line: each removal can expose another pair, so the scan repeats until
|
|
226
|
-
// it finds nothing. Bounded by the block length, so it always terminates.
|
|
227
|
-
for (let guard = 0; guard <= insertedFinal.length; guard++) {
|
|
228
|
-
const hit = findAdjacentDuplicate(insertedFinal) || findSeamDuplicate(lines, insertedFinal, start, end);
|
|
229
|
-
if (!hit) break;
|
|
230
|
-
drops.push(hit);
|
|
231
|
-
insertedFinal = [...insertedFinal.slice(0, hit.index), ...insertedFinal.slice(hit.index + 1)];
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
const seamDrop = drops.length ? drops[0] : null;
|
|
235
|
-
|
|
236
|
-
const out = [...lines.slice(0, start - 1), ...insertedFinal, ...lines.slice(end)];
|
|
192
|
+
const out = [...lines.slice(0, start - 1), ...inserted, ...lines.slice(end)];
|
|
237
193
|
const normUpdated = out.join('\n');
|
|
238
194
|
const updated = eol === '\r\n' ? normUpdated.split('\n').join('\r\n') : normUpdated;
|
|
239
195
|
|
|
@@ -243,7 +199,7 @@ export const spec = {
|
|
|
243
199
|
ensureDir(p);
|
|
244
200
|
fs.writeFileSync(p, updated, 'utf8');
|
|
245
201
|
} catch (e) { return `Error writing ${args.path}: ${e.message}`; }
|
|
246
|
-
refreshReadPool(p, normUpdated, { start, end: start +
|
|
202
|
+
refreshReadPool(p, normUpdated, { start, end: start + inserted.length - 1 }, ctx);
|
|
247
203
|
const removed = end - start + 1;
|
|
248
204
|
// Publish the diff for the UI. The Edit line is drawn from these, so a
|
|
249
205
|
// line-range edit shows the same +/- rows and +N/-N counts as a substring
|
|
@@ -251,28 +207,17 @@ export const spec = {
|
|
|
251
207
|
// the agent loop right after this tool returns.
|
|
252
208
|
//
|
|
253
209
|
// `lines` is the pre-edit content, which is what `start` indexes into.
|
|
254
|
-
// The diff's NEW text is `
|
|
210
|
+
// The diff's NEW text is `inserted` — what the file now holds — not the
|
|
255
211
|
// raw `new_string`: when the seam-drop removed a line, the raw text showed a
|
|
256
212
|
// `+` row that is not in the file and shifted every after-context line
|
|
257
213
|
// number by the dropped count.
|
|
258
|
-
if (ctx) ctx.lastEditDiff = buildEditDiff({ fileLines: lines, oldText: replacedText, newText:
|
|
214
|
+
if (ctx) ctx.lastEditDiff = buildEditDiff({ fileLines: lines, oldText: replacedText, newText: inserted.join('\n'), startLine: start });
|
|
259
215
|
// Say what the seam did, whichever way it went: a silent drop and a silent keep are both
|
|
260
216
|
// indistinguishable from an edit that never had the ambiguity. The dropped line is named
|
|
261
217
|
// with its ACTUAL pair (the two result lines that would have been identical) — it is not
|
|
262
218
|
// always "the last line of new_string against end_line + 1": a block can carry the repeat
|
|
263
219
|
// inside itself, with blank lines between the copies.
|
|
264
|
-
|
|
265
|
-
//
|
|
266
|
-
// It used to read `Dropped a duplicated line: "}" ... Pass allow_dup: true to keep it`, which a
|
|
267
|
-
// caller interpreted as the tool silently deleting a line it had written on purpose — so it stopped
|
|
268
|
-
// trusting the tool and never reached for the flag. What was actually removed is the copy inside
|
|
269
|
-
// new_string; every line already in the file is untouched.
|
|
270
|
-
const seamNote = seamDrop
|
|
271
|
-
? ` One line from new_string was NOT inserted: "${seamDrop.text}" already sits at result line`
|
|
272
|
-
+ ` ${seamDrop.a} (it would have been adjacent to the copy at line ${seamDrop.b}). The file's own`
|
|
273
|
-
+ ' lines were not touched. If that repetition was intended, resend with allow_dup: true.'
|
|
274
|
-
: (args.allow_dup === true ? ' allow_dup: true — the repeated line was kept as asked.' : '');
|
|
275
|
-
return `Edited ${args.path}: replaced lines ${start}-${end} (${removed} line${removed === 1 ? '' : 's'}) with ${insertedFinal.length} line${insertedFinal.length === 1 ? '' : 's'}.${seamNote}`;
|
|
220
|
+
return `Edited ${args.path}: replaced lines ${start}-${end} (${removed} line${removed === 1 ? '' : 's'}) with ${inserted.length} line${inserted.length === 1 ? '' : 's'}.`;
|
|
276
221
|
}
|
|
277
222
|
|
|
278
223
|
// ---- SUBSTRING mode (old_string -> new_string) --------------------------
|
|
@@ -467,54 +412,7 @@ function leadingWs(s) {
|
|
|
467
412
|
return m ? m[0] : '';
|
|
468
413
|
}
|
|
469
414
|
|
|
470
|
-
|
|
471
|
-
* The first repeated line in `inserted`, and where to remove one copy.
|
|
472
|
-
*
|
|
473
|
-
* ONE RULE: a line that repeats the one above it is one too many, so it goes. Nothing here looks at what
|
|
474
|
-
* the line contains — a brace, a comment closer, a string of spaces are all just text, and asking which
|
|
475
|
-
* of them "may legitimately repeat" is a question this tool has no business answering.
|
|
476
|
-
*
|
|
477
|
-
* A blank line between the two does not break the pair: a pasted block usually ends with one, and leaving
|
|
478
|
-
* it behind strands a gap where the duplicate was.
|
|
479
|
-
*
|
|
480
|
-
* Only lines FROM `new_string` are ever removed. The file's own lines were already there, and deleting one
|
|
481
|
-
* would remove content the caller never asked to touch — that is the whole reason to look at `inserted`
|
|
482
|
-
* rather than at the result.
|
|
483
|
-
*/
|
|
484
|
-
function findAdjacentDuplicate(inserted) {
|
|
485
|
-
let prev = null;
|
|
486
|
-
let prevAt = -1;
|
|
487
|
-
for (let i = 0; i < inserted.length; i++) {
|
|
488
|
-
const t = String(inserted[i] == null ? '' : inserted[i]).trim();
|
|
489
|
-
if (!t) continue;
|
|
490
|
-
if (prev !== null && t === prev) return { a: prevAt + 1, b: i + 1, text: t, index: i };
|
|
491
|
-
prev = t;
|
|
492
|
-
prevAt = i;
|
|
493
|
-
}
|
|
494
|
-
return null;
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
/**
|
|
498
|
-
* The pair at the SEAM: the last line of `inserted` against the line at `end_line + 1`, which the edit
|
|
499
|
-
* never replaced.
|
|
500
|
-
*
|
|
501
|
-
* Same rule, same answer — the inserted copy goes, the file's own line stays.
|
|
502
|
-
*/
|
|
503
|
-
function findSeamDuplicate(lines, inserted, start, end) {
|
|
504
|
-
if (!inserted.length) return null;
|
|
505
|
-
const norm = (v) => String(v == null ? '' : v).trim();
|
|
506
|
-
// THE END: new_string's last line against the first line the edit did not replace.
|
|
507
|
-
const last = norm(inserted[inserted.length - 1]);
|
|
508
|
-
if (last && norm(lines[end]) === last) {
|
|
509
|
-
return { a: inserted.length, b: end + 1, text: last, index: inserted.length - 1 };
|
|
510
|
-
}
|
|
511
|
-
// THE START: new_string's first line against the line just above the range.
|
|
512
|
-
const first = norm(inserted[0]);
|
|
513
|
-
if (first && norm(lines[start - 2]) === first) {
|
|
514
|
-
return { a: start - 1, b: start, text: first, index: 0 };
|
|
515
|
-
}
|
|
516
|
-
return null;
|
|
517
|
-
}
|
|
415
|
+
|
|
518
416
|
/**
|
|
519
417
|
* Build the diff payload, including the unchanged lines around the change.
|
|
520
418
|
*
|
|
@@ -679,7 +577,7 @@ function checkStale(p, normContent, normOld, ctx) {
|
|
|
679
577
|
function args_readHint(p, at, span) {
|
|
680
578
|
const from = Math.max(1, at - 3);
|
|
681
579
|
const n = Math.max(span + 6, 10);
|
|
682
|
-
return `Read { path: ${JSON.stringify(p)},
|
|
580
|
+
return `Read { path: ${JSON.stringify(p)}, start_line: ${from}, end_line: ${from + n - 1} }`;
|
|
683
581
|
}
|
|
684
582
|
// ---- helpers for the line-range path ---------------------------------------
|
|
685
583
|
|
package/src/tools/fetch-url.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// FetchURL tool — fetch a web page's main text content (Node built-in fetch, no deps).
|
|
2
2
|
|
|
3
3
|
import os from 'node:os';
|
|
4
|
-
import http from 'node:http';
|
|
5
|
-
import https from 'node:https';
|
|
4
|
+
import http from 'node:http';
|
|
5
|
+
import https from 'node:https';
|
|
6
6
|
|
|
7
7
|
const MAX_BYTES = 200 * 1024; // generous cap for large pages
|
|
8
8
|
|
|
@@ -44,100 +44,100 @@ function rejectUrl(raw) {
|
|
|
44
44
|
return null; // ok
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
/**
|
|
48
|
-
* A browser-ish header set.
|
|
49
|
-
*
|
|
50
|
-
* `Accept` is part of it: a request with a UA but an empty Accept still looks like a script to some
|
|
51
|
-
* servers, and they answer 406 or 429.
|
|
52
|
-
*/
|
|
53
|
-
const HEADERS = {
|
|
54
|
-
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
55
|
-
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5',
|
|
56
|
-
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8',
|
|
57
|
-
};
|
|
58
|
-
|
|
59
|
-
/** How long to wait before retrying a rate-limited request. */
|
|
60
|
-
const RETRY_AFTER_MS = 1500;
|
|
61
|
-
|
|
62
|
-
/**
|
|
63
|
-
* Fetch with the browser headers, retrying a 429 once.
|
|
64
|
-
*
|
|
65
|
-
* One retry after a short pause turns a transient rate limit into a success; a second immediate attempt
|
|
66
|
-
* would only count as another request against the limit.
|
|
67
|
-
*/
|
|
68
|
-
/**
|
|
69
|
-
* Fetch with the browser headers, retrying a 429 once.
|
|
70
|
-
*
|
|
71
|
-
* `insecure` SKIPS CERTIFICATE VERIFICATION, for a host whose chain is broken — an internal server with a
|
|
72
|
-
* self-signed certificate, an expired intermediate — where the alternative is not fetching it at all.
|
|
73
|
-
*
|
|
74
|
-
* `fetch` has no per-request switch for this: the option lives on the dispatcher, and Node does not expose
|
|
75
|
-
* one without the `undici` package this project does not depend on. So that path goes through
|
|
76
|
-
* `node:https` instead. It is the SAME request — same headers, same redirects, same body cap — just built
|
|
77
|
-
* by hand so `rejectUnauthorized: false` can be passed.
|
|
78
|
-
*/
|
|
79
|
-
async function fetchWithRetry(url, signal, insecure = false) {
|
|
80
|
-
if (insecure) return httpsRequest(url, signal, true);
|
|
81
|
-
const first = await fetch(url, { signal, redirect: 'follow', headers: HEADERS });
|
|
82
|
-
if (first.status !== 429) return first;
|
|
83
|
-
await new Promise((r) => setTimeout(r, RETRY_AFTER_MS));
|
|
84
|
-
// The first response body was never read; release it so the connection can be reused.
|
|
85
|
-
try { await first.body?.cancel(); } catch { /* already gone */ }
|
|
86
|
-
return fetch(url, { signal, redirect: 'follow', headers: HEADERS });
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* One GET over `node:https`, returning the shape the caller already handles.
|
|
91
|
-
*
|
|
92
|
-
* Only used for the `insecure` path, where `fetch` cannot express what is needed. The response is
|
|
93
|
-
* buffered up to MAX_BYTES and handed back as an object with `.ok`, `.status`, `.headers.get()` and
|
|
94
|
-
* `.arrayBuffer()`, so the rest of `execute` does not know which transport ran.
|
|
95
|
-
*/
|
|
96
|
-
function httpsRequest(url, signal, insecure) {
|
|
97
|
-
return new Promise((resolve, reject) => {
|
|
98
|
-
let u;
|
|
99
|
-
try { u = new URL(url); } catch (e) { reject(e); return; }
|
|
100
|
-
const mod = u.protocol === 'http:' ? http : https;
|
|
101
|
-
const req = mod.request({
|
|
102
|
-
protocol: u.protocol,
|
|
103
|
-
hostname: u.hostname,
|
|
104
|
-
port: u.port || undefined,
|
|
105
|
-
path: u.pathname + u.search,
|
|
106
|
-
method: 'GET',
|
|
107
|
-
headers: HEADERS,
|
|
108
|
-
// The whole point of this path. Every other request in the tool keeps verification on.
|
|
109
|
-
rejectUnauthorized: !insecure,
|
|
110
|
-
}, (res) => {
|
|
111
|
-
// A redirect is followed rather than reported: `fetch` defaults to `follow`, and a fetch that
|
|
112
|
-
// stopped at a 301 would silently return an empty body for most of the web.
|
|
113
|
-
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
114
|
-
res.resume();
|
|
115
|
-
const next = new URL(res.headers.location, u).href;
|
|
116
|
-
resolve(httpsRequest(next, signal, insecure));
|
|
117
|
-
return;
|
|
118
|
-
}
|
|
119
|
-
const chunks = [];
|
|
120
|
-
let got = 0;
|
|
121
|
-
res.on('data', (c) => { if (got < MAX_BYTES) { chunks.push(c); got += c.length; } });
|
|
122
|
-
res.on('end', () => {
|
|
123
|
-
const body = Buffer.concat(chunks);
|
|
124
|
-
resolve({
|
|
125
|
-
ok: res.statusCode >= 200 && res.statusCode < 300,
|
|
126
|
-
status: res.statusCode,
|
|
127
|
-
statusText: res.statusMessage || '',
|
|
128
|
-
headers: { get: (n) => res.headers[String(n).toLowerCase()] || null },
|
|
129
|
-
arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength),
|
|
130
|
-
body: null,
|
|
131
|
-
});
|
|
132
|
-
});
|
|
133
|
-
res.on('error', reject);
|
|
134
|
-
});
|
|
135
|
-
req.on('error', reject);
|
|
136
|
-
if (signal) signal.addEventListener('abort', () => req.destroy(new Error('interrupted')), { once: true });
|
|
137
|
-
req.end();
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
|
|
47
|
+
/**
|
|
48
|
+
* A browser-ish header set.
|
|
49
|
+
*
|
|
50
|
+
* `Accept` is part of it: a request with a UA but an empty Accept still looks like a script to some
|
|
51
|
+
* servers, and they answer 406 or 429.
|
|
52
|
+
*/
|
|
53
|
+
const HEADERS = {
|
|
54
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
55
|
+
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5',
|
|
56
|
+
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8',
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/** How long to wait before retrying a rate-limited request. */
|
|
60
|
+
const RETRY_AFTER_MS = 1500;
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Fetch with the browser headers, retrying a 429 once.
|
|
64
|
+
*
|
|
65
|
+
* One retry after a short pause turns a transient rate limit into a success; a second immediate attempt
|
|
66
|
+
* would only count as another request against the limit.
|
|
67
|
+
*/
|
|
68
|
+
/**
|
|
69
|
+
* Fetch with the browser headers, retrying a 429 once.
|
|
70
|
+
*
|
|
71
|
+
* `insecure` SKIPS CERTIFICATE VERIFICATION, for a host whose chain is broken — an internal server with a
|
|
72
|
+
* self-signed certificate, an expired intermediate — where the alternative is not fetching it at all.
|
|
73
|
+
*
|
|
74
|
+
* `fetch` has no per-request switch for this: the option lives on the dispatcher, and Node does not expose
|
|
75
|
+
* one without the `undici` package this project does not depend on. So that path goes through
|
|
76
|
+
* `node:https` instead. It is the SAME request — same headers, same redirects, same body cap — just built
|
|
77
|
+
* by hand so `rejectUnauthorized: false` can be passed.
|
|
78
|
+
*/
|
|
79
|
+
async function fetchWithRetry(url, signal, insecure = false) {
|
|
80
|
+
if (insecure) return httpsRequest(url, signal, true);
|
|
81
|
+
const first = await fetch(url, { signal, redirect: 'follow', headers: HEADERS });
|
|
82
|
+
if (first.status !== 429) return first;
|
|
83
|
+
await new Promise((r) => setTimeout(r, RETRY_AFTER_MS));
|
|
84
|
+
// The first response body was never read; release it so the connection can be reused.
|
|
85
|
+
try { await first.body?.cancel(); } catch { /* already gone */ }
|
|
86
|
+
return fetch(url, { signal, redirect: 'follow', headers: HEADERS });
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* One GET over `node:https`, returning the shape the caller already handles.
|
|
91
|
+
*
|
|
92
|
+
* Only used for the `insecure` path, where `fetch` cannot express what is needed. The response is
|
|
93
|
+
* buffered up to MAX_BYTES and handed back as an object with `.ok`, `.status`, `.headers.get()` and
|
|
94
|
+
* `.arrayBuffer()`, so the rest of `execute` does not know which transport ran.
|
|
95
|
+
*/
|
|
96
|
+
function httpsRequest(url, signal, insecure) {
|
|
97
|
+
return new Promise((resolve, reject) => {
|
|
98
|
+
let u;
|
|
99
|
+
try { u = new URL(url); } catch (e) { reject(e); return; }
|
|
100
|
+
const mod = u.protocol === 'http:' ? http : https;
|
|
101
|
+
const req = mod.request({
|
|
102
|
+
protocol: u.protocol,
|
|
103
|
+
hostname: u.hostname,
|
|
104
|
+
port: u.port || undefined,
|
|
105
|
+
path: u.pathname + u.search,
|
|
106
|
+
method: 'GET',
|
|
107
|
+
headers: HEADERS,
|
|
108
|
+
// The whole point of this path. Every other request in the tool keeps verification on.
|
|
109
|
+
rejectUnauthorized: !insecure,
|
|
110
|
+
}, (res) => {
|
|
111
|
+
// A redirect is followed rather than reported: `fetch` defaults to `follow`, and a fetch that
|
|
112
|
+
// stopped at a 301 would silently return an empty body for most of the web.
|
|
113
|
+
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
114
|
+
res.resume();
|
|
115
|
+
const next = new URL(res.headers.location, u).href;
|
|
116
|
+
resolve(httpsRequest(next, signal, insecure));
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const chunks = [];
|
|
120
|
+
let got = 0;
|
|
121
|
+
res.on('data', (c) => { if (got < MAX_BYTES) { chunks.push(c); got += c.length; } });
|
|
122
|
+
res.on('end', () => {
|
|
123
|
+
const body = Buffer.concat(chunks);
|
|
124
|
+
resolve({
|
|
125
|
+
ok: res.statusCode >= 200 && res.statusCode < 300,
|
|
126
|
+
status: res.statusCode,
|
|
127
|
+
statusText: res.statusMessage || '',
|
|
128
|
+
headers: { get: (n) => res.headers[String(n).toLowerCase()] || null },
|
|
129
|
+
arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength),
|
|
130
|
+
body: null,
|
|
131
|
+
});
|
|
132
|
+
});
|
|
133
|
+
res.on('error', reject);
|
|
134
|
+
});
|
|
135
|
+
req.on('error', reject);
|
|
136
|
+
if (signal) signal.addEventListener('abort', () => req.destroy(new Error('interrupted')), { once: true });
|
|
137
|
+
req.end();
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
|
|
141
141
|
export const spec = {
|
|
142
142
|
name: 'FetchURL',
|
|
143
143
|
description: 'Fetch a URL and return its text (main page text, or the full body; a leading note says which). Any http/https URL, including localhost.',
|
|
@@ -145,7 +145,7 @@ export const spec = {
|
|
|
145
145
|
type: 'object',
|
|
146
146
|
properties: {
|
|
147
147
|
url: { type: 'string', description: 'URL to fetch.' },
|
|
148
|
-
insecure: { type: 'boolean', default: false, description: 'Skip TLS certificate verification. For a host with a self-signed or expired certificate whose content is still wanted — the connection is no longer authenticated, so only use it for a source you already trust.' },
|
|
148
|
+
insecure: { type: 'boolean', default: false, description: 'Skip TLS certificate verification. For a host with a self-signed or expired certificate whose content is still wanted — the connection is no longer authenticated, so only use it for a source you already trust.' },
|
|
149
149
|
},
|
|
150
150
|
required: ['url'],
|
|
151
151
|
},
|
|
@@ -154,21 +154,21 @@ export const spec = {
|
|
|
154
154
|
if (rejected) return rejected;
|
|
155
155
|
const signal = (ctx && ctx.signal) || undefined;
|
|
156
156
|
try {
|
|
157
|
-
// A BROWSER USER-AGENT, and it is not cosmetic.
|
|
158
|
-
//
|
|
159
|
-
// Without one Node sends `user-agent: node`, and a great many sites answer that with 403 or 429
|
|
160
|
-
// before serving anything. Measured on a wukongsch.com article: 429 with the default headers,
|
|
161
|
-
// served normally with a browser UA.
|
|
162
|
-
const res = await fetchWithRetry(args.url.trim(), signal, args.insecure === true);
|
|
163
|
-
if (!res.ok) {
|
|
164
|
-
// SAY WHAT THE STATUS MEANS. A bare 429 left the caller re-fetching the same page to see the
|
|
165
|
-
// same number.
|
|
166
|
-
const hint = res.status === 429 ? ' — the site is rate-limiting requests; wait and retry, or find another source.'
|
|
167
|
-
: res.status === 403 ? ' — the site refused the request; it may require a login or block automated fetches.'
|
|
168
|
-
: res.status === 404 ? ' — no such page.'
|
|
169
|
-
: '';
|
|
170
|
-
return `Error: ${res.status} ${res.statusText} from ${args.url}${hint}`;
|
|
171
|
-
}
|
|
157
|
+
// A BROWSER USER-AGENT, and it is not cosmetic.
|
|
158
|
+
//
|
|
159
|
+
// Without one Node sends `user-agent: node`, and a great many sites answer that with 403 or 429
|
|
160
|
+
// before serving anything. Measured on a wukongsch.com article: 429 with the default headers,
|
|
161
|
+
// served normally with a browser UA.
|
|
162
|
+
const res = await fetchWithRetry(args.url.trim(), signal, args.insecure === true);
|
|
163
|
+
if (!res.ok) {
|
|
164
|
+
// SAY WHAT THE STATUS MEANS. A bare 429 left the caller re-fetching the same page to see the
|
|
165
|
+
// same number.
|
|
166
|
+
const hint = res.status === 429 ? ' — the site is rate-limiting requests; wait and retry, or find another source.'
|
|
167
|
+
: res.status === 403 ? ' — the site refused the request; it may require a login or block automated fetches.'
|
|
168
|
+
: res.status === 404 ? ' — no such page.'
|
|
169
|
+
: '';
|
|
170
|
+
return `Error: ${res.status} ${res.statusText} from ${args.url}${hint}`;
|
|
171
|
+
}
|
|
172
172
|
const contentType = (res.headers.get('content-type') || '').toLowerCase();
|
|
173
173
|
const isHtml = contentType.includes('text/html') || contentType.includes('application/xhtml');
|
|
174
174
|
let isTruncated = false;
|
|
@@ -209,13 +209,23 @@ export const spec = {
|
|
|
209
209
|
if (isTruncated) out += `\n\n[... truncated; page larger than ${Math.round(MAX_BYTES / 1024)} KB]`;
|
|
210
210
|
return out;
|
|
211
211
|
} catch (e) {
|
|
212
|
-
// 'fetch failed' IS NOT A REASON. Node puts the real cause — ENOTFOUND, ECONNREFUSED,
|
|
213
|
-
// CERT_HAS_EXPIRED, a TLS handshake failure — on `e.cause`, and only `e.message` was read, so
|
|
214
|
-
// every distinct failure looked identical and told the caller nothing they could act on.
|
|
215
|
-
const cause = e && e.cause ? (e.cause.code || e.cause.message) : '';
|
|
216
|
-
const base = e && e.name === 'AbortError' ? 'interrupted' : (e.message || String(e));
|
|
217
|
-
const msg = cause && !String(base).includes(cause) ? `${base} (${cause})` : base;
|
|
218
|
-
|
|
212
|
+
// 'fetch failed' IS NOT A REASON. Node puts the real cause — ENOTFOUND, ECONNREFUSED,
|
|
213
|
+
// CERT_HAS_EXPIRED, a TLS handshake failure — on `e.cause`, and only `e.message` was read, so
|
|
214
|
+
// every distinct failure looked identical and told the caller nothing they could act on.
|
|
215
|
+
const cause = e && e.cause ? (e.cause.code || e.cause.message) : '';
|
|
216
|
+
const base = e && e.name === 'AbortError' ? 'interrupted' : (e.message || String(e));
|
|
217
|
+
const msg = cause && !String(base).includes(cause) ? `${base} (${cause})` : base;
|
|
218
|
+
// SAY WHAT TO DO ABOUT A BROKEN CHAIN. `UNABLE_TO_GET_ISSUER_CERT_LOCALLY` means the machine cannot
|
|
219
|
+
// reach the issuer — a corporate proxy, a missing intermediate, a self-signed host — and the page
|
|
220
|
+
// is otherwise fine. The caller cannot act on the code alone, so the switch that does work is named.
|
|
221
|
+
// Any code mentioning a certificate is a chain problem the caller can work around.
|
|
222
|
+
const chain = /CERT|SELF_SIGNED|ISSUER|VERIFY_LEAF|UNABLE_TO_GET/i.test(String(cause) + ' ' + String(base));
|
|
223
|
+
const hint = chain
|
|
224
|
+
? ' — the certificate chain could not be verified (a missing intermediate, a proxy, or a self-signed'
|
|
225
|
+
+ ' host). Retry with insecure: true if the content is still wanted; the connection will not be'
|
|
226
|
+
+ ' authenticated.'
|
|
227
|
+
: '';
|
|
228
|
+
return `Error fetching ${args.url}: ${msg}${hint}`;
|
|
219
229
|
}
|
|
220
230
|
},
|
|
221
|
-
};
|
|
231
|
+
};
|