@armadra/agent 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/CHANGELOG.zh-CN.md +31 -0
- package/README.md +5 -0
- package/README.zh-CN.md +4 -0
- package/dist/acp.d.ts +2 -2
- package/dist/acp.js +2 -2
- package/dist/bundle/ama.cjs +1730 -446
- package/dist/cli/compose-agents.js +8 -0
- package/dist/cli/compose.d.ts +3 -0
- package/dist/cli/compose.js +1 -0
- package/dist/drivers/acp/client.d.ts +54 -5
- package/dist/drivers/acp/client.js +96 -9
- package/dist/drivers/acp/types.d.ts +69 -3
- package/dist/drivers/acp/types.js +30 -1
- package/dist/drivers/agents.js +1 -1
- package/dist/drivers/host-runners.d.ts +26 -1
- package/dist/drivers/host-runners.js +147 -0
- package/dist/hooks/matcher.d.ts +2 -2
- package/dist/hooks/matcher.js +3 -3
- package/dist/host/api-impl.d.ts +3 -0
- package/dist/host/api-impl.js +7 -2
- package/dist/host/types.d.ts +37 -1
- package/dist/host/types.js +5 -0
- package/dist/host.d.ts +1 -1
- package/dist/host.js +1 -1
- package/dist/i18n/catalog.d.ts +5 -0
- package/dist/i18n/messages/approval.d.ts +5 -0
- package/dist/i18n/messages/approval.js +5 -0
- package/dist/i18n/messages/drivers.d.ts +2 -0
- package/dist/i18n/messages/drivers.js +2 -0
- package/dist/i18n/messages/interactive-view.d.ts +5 -0
- package/dist/i18n/messages/interactive-view.js +5 -0
- package/dist/i18n/messages/interactive.d.ts +4 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/modes/commands-core.d.ts +7 -0
- package/dist/modes/commands-core.js +23 -5
- package/dist/modes/interactive/approval-dialog.js +17 -4
- package/dist/modes/interactive/commands.d.ts +2 -0
- package/dist/modes/interactive/commands.js +5 -1
- package/dist/modes/interactive/interactive-mode.d.ts +7 -0
- package/dist/modes/interactive/interactive-mode.js +14 -6
- package/dist/modes/interactive/tool-summary.d.ts +2 -0
- package/dist/modes/interactive/tool-summary.js +27 -0
- package/dist/modes/rpc/commands.js +4 -4
- package/dist/permissions/auto-safe.d.ts +3 -0
- package/dist/permissions/auto-safe.js +10 -3
- package/dist/permissions/classifier.js +1 -1
- package/dist/permissions/network-class.d.ts +28 -0
- package/dist/permissions/network-class.js +91 -0
- package/dist/permissions/pipeline.d.ts +1 -1
- package/dist/permissions/pipeline.js +24 -7
- package/dist/permissions/preview.js +3 -0
- package/dist/permissions/readonly-bash.d.ts +2 -2
- package/dist/permissions/readonly-bash.js +2 -2
- package/dist/permissions/rules.d.ts +1 -0
- package/dist/permissions/rules.js +5 -1
- package/dist/tools/presets.d.ts +8 -1
- package/dist/tools/presets.js +12 -3
- package/dist/tools/registry.d.ts +3 -1
- package/dist/tools/registry.js +4 -2
- package/dist/tools/task-ctl.d.ts +9 -0
- package/dist/tools/task-ctl.js +22 -3
- package/dist/tools/types.d.ts +4 -1
- package/dist/tools/web/fetch.d.ts +48 -0
- package/dist/tools/web/fetch.js +307 -0
- package/dist/tools/web/html-to-markdown.d.ts +31 -0
- package/dist/tools/web/html-to-markdown.js +522 -0
- package/dist/tools/web/url-guard.d.ts +59 -0
- package/dist/tools/web/url-guard.js +221 -0
- package/docs/en/guides/permissions.md +33 -11
- package/docs/en/reference/acp.md +7 -3
- package/docs/en/reference/host-api.md +61 -19
- package/docs/en/reference/rpc.md +7 -7
- package/docs/guides/agents.md +18 -1
- package/docs/guides/hooks.md +86 -8
- package/docs/guides/permissions.md +33 -11
- package/docs/guides/sandbox.md +6 -0
- package/docs/reference/acp.md +7 -3
- package/docs/reference/host-api.md +61 -19
- package/docs/reference/rpc.md +7 -7
- package/package.json +1 -1
package/dist/tools/presets.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* | 预设 | 模型直接看到 |
|
|
5
5
|
* | ------------- | ----------------------------------------- |
|
|
6
|
-
* | `default` | read、edit、write、bash、grep、glob
|
|
6
|
+
* | `default` | read、edit、write、bash、grep、glob、web_fetch |
|
|
7
7
|
* | `minimal` | read、edit、write、bash |
|
|
8
8
|
* | `codemode-only` | 只有 codemode(其余工具在脚本里调用) |
|
|
9
9
|
* | `coordinator` | read + 宿主注册的工具 |
|
|
@@ -23,6 +23,8 @@
|
|
|
23
23
|
* `codemode.mode` 显式配置覆盖(缺省配置不写这个键,以后调整映射能惠及老用户)。
|
|
24
24
|
* - `coordinator` 预设即使显式 `on`,脚本里能调用的工具也只限活动集(read + 宿主工具),见
|
|
25
25
|
* `codemodeCallableFilter`:协调者「不写文件、不跑 bash」的约定不能经 codemode 绕过。
|
|
26
|
+
* - (#242)`coordinator` 预设不阻塞:`task` 缺省后台(无视 `subagents.background`,调用参数 `background`
|
|
27
|
+
* 与类型定义的 `background:` 仍优先)、`task_ctl wait` 的超时封顶 5 秒;其它预设不变。
|
|
26
28
|
* - codemode 工具由 B10 经组装根的 `toolFactories` 注册(生效模式为 off 时不注册);不可用
|
|
27
29
|
* (例如 `codemode.requireStrict` 而运行时 Node 不隔离网络)时 codemode-only 预设**回退到 default 并
|
|
28
30
|
* warning**(不报错:零配置用户不应因此起不来),显式 `on` 同样忽略;跟随预设得到的 `on` 静默回退。
|
|
@@ -35,7 +37,7 @@ export const CODEMODE_TOOL = "codemode";
|
|
|
35
37
|
/** 各预设下模型直接看到的内置工具(codemode-only 预设见 `resolvePreset`)。 */
|
|
36
38
|
export const PRESET_TOOLS = Object.freeze({
|
|
37
39
|
// D20:todo 不在 default(W5-Z 复测未过门,见文件头)
|
|
38
|
-
default: ["bash", "edit", "glob", "grep", "read", "write"],
|
|
40
|
+
default: ["bash", "edit", "glob", "grep", "read", "web_fetch", "write"],
|
|
39
41
|
minimal: ["bash", "edit", "read", "write"],
|
|
40
42
|
"codemode-only": [CODEMODE_TOOL],
|
|
41
43
|
coordinator: ["read"],
|
|
@@ -72,7 +74,14 @@ export function effectiveCodemodeMode(config, strict) {
|
|
|
72
74
|
* 不限(`undefined`;`on` 模式下脚本还能调没直接暴露的 ls / task / todo 等)。
|
|
73
75
|
*/
|
|
74
76
|
export function codemodeCallableFilter(config) {
|
|
75
|
-
return
|
|
77
|
+
return isCoordinatorPreset(config) ? "active" : undefined;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* [#242] 是否 `coordinator` 预设:协调者只分派、不阻塞——`task` 缺省后台、`task_ctl wait` 超时封顶
|
|
81
|
+
* (组装根据此绑定,见 cli/compose-agents.ts)。其它预设不受影响。
|
|
82
|
+
*/
|
|
83
|
+
export function isCoordinatorPreset(config) {
|
|
84
|
+
return canonicalPreset(config.tools?.preset) === "coordinator";
|
|
76
85
|
}
|
|
77
86
|
/**
|
|
78
87
|
* `tools.default` 微调:不带前缀的名字整组替换 `base`,再依次应用 `+name` / `-name`。
|
package/dist/tools/registry.d.ts
CHANGED
|
@@ -15,6 +15,7 @@ import type { ToolDefinition, ToolExecutionMode, ToolRegistryApi, ToolSource } f
|
|
|
15
15
|
import { type ReadToolOptions } from "./read.js";
|
|
16
16
|
import { type BashToolOptions } from "./bash.js";
|
|
17
17
|
import { type TaskToolOptions } from "./task.js";
|
|
18
|
+
import { type WebFetchOptions } from "./web/fetch.js";
|
|
18
19
|
/** [W5-C0] 主工具 → 随它出现 / 消失的伴随工具。 */
|
|
19
20
|
export declare const TOOL_COMPANIONS: Readonly<Record<string, readonly string[]>>;
|
|
20
21
|
/** 按 `TOOL_COMPANIONS` 配对:有主工具就带上(已注册的)伴随工具,没有就去掉。 */
|
|
@@ -40,8 +41,9 @@ export interface BuiltinToolOptions {
|
|
|
40
41
|
read?: ReadToolOptions;
|
|
41
42
|
bash?: BashToolOptions;
|
|
42
43
|
task?: TaskToolOptions;
|
|
44
|
+
webFetch?: WebFetchOptions;
|
|
43
45
|
}
|
|
44
|
-
/** 全部内置工具(read / write / edit / bash / grep / glob / ls / todo / task / task_ctl)。 */
|
|
46
|
+
/** 全部内置工具(read / write / edit / bash / grep / glob / ls / todo / task / task_ctl / web_fetch)。 */
|
|
45
47
|
export declare function builtinTools(options?: BuiltinToolOptions): ToolDefinition[];
|
|
46
48
|
/** 新建注册表并登记全部内置工具;`disabled` 来自 config.tools.disabled。 */
|
|
47
49
|
export declare function createToolRegistry(options?: BuiltinToolOptions & {
|
package/dist/tools/registry.js
CHANGED
|
@@ -23,6 +23,7 @@ import { createLsTool } from "./ls.js";
|
|
|
23
23
|
import { createTodoTool } from "./todo.js";
|
|
24
24
|
import { createTaskTool } from "./task.js";
|
|
25
25
|
import { TASK_CTL_TOOL, createTaskCtlTool } from "./task-ctl.js";
|
|
26
|
+
import { createWebFetchTool } from "./web/fetch.js";
|
|
26
27
|
/** [W5-C0] 主工具 → 随它出现 / 消失的伴随工具。 */
|
|
27
28
|
export const TOOL_COMPANIONS = Object.freeze({
|
|
28
29
|
task: [TASK_CTL_TOOL],
|
|
@@ -49,7 +50,7 @@ export function validateToolDefinition(tool) {
|
|
|
49
50
|
if (typeof tool.name !== "string" || !TOOL_NAME_RE.test(tool.name)) {
|
|
50
51
|
throw new AmaError("invalid_arguments", `Invalid tool name "${String(tool.name)}"`);
|
|
51
52
|
}
|
|
52
|
-
if (!["read", "write", "execute", "memory"].includes(tool.permission)) {
|
|
53
|
+
if (!["read", "write", "execute", "memory", "network"].includes(tool.permission)) {
|
|
53
54
|
throw new AmaError("invalid_arguments", `Tool ${tool.name}: invalid permission`);
|
|
54
55
|
}
|
|
55
56
|
if (typeof tool.execute !== "function") {
|
|
@@ -102,7 +103,7 @@ export class ToolRegistry {
|
|
|
102
103
|
this.activeNames = new Set(pairCompanions(names, (n) => this.entries.has(n) && !this.disabled.has(n)));
|
|
103
104
|
}
|
|
104
105
|
}
|
|
105
|
-
/** 全部内置工具(read / write / edit / bash / grep / glob / ls / todo / task / task_ctl)。 */
|
|
106
|
+
/** 全部内置工具(read / write / edit / bash / grep / glob / ls / todo / task / task_ctl / web_fetch)。 */
|
|
106
107
|
export function builtinTools(options = {}) {
|
|
107
108
|
const tools = [
|
|
108
109
|
createReadTool(options.read),
|
|
@@ -115,6 +116,7 @@ export function builtinTools(options = {}) {
|
|
|
115
116
|
createTodoTool(),
|
|
116
117
|
createTaskTool(options.task),
|
|
117
118
|
createTaskCtlTool(),
|
|
119
|
+
createWebFetchTool(options.webFetch),
|
|
118
120
|
];
|
|
119
121
|
return tools;
|
|
120
122
|
}
|
package/dist/tools/task-ctl.d.ts
CHANGED
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
* 与父一致,运行时拒绝;
|
|
8
8
|
* - `wait` / `output` 是轮询类调用(`annotations.pollable`,重复调用检测豁免);
|
|
9
9
|
* - `send` = 续聊放后台:等价于 `task{taskId, prompt, background:true}`,完成后同样收到通知。
|
|
10
|
+
* - [#242] `coordinator` 预设下 `wait` 的超时封顶 `COORDINATOR_WAIT_MAX_MS`(协调者不阻塞,见
|
|
11
|
+
* `bindTaskCtlWaitCap`);其它预设不变。
|
|
10
12
|
* - [W7-B1] `wait` 被转后台(`background()`,TUI `Ctrl+B` / RPC `background_task`)打断时立即返回固定
|
|
11
13
|
* 文案,要求模型不要再等。
|
|
12
14
|
*/
|
|
@@ -14,6 +16,13 @@ import type { ToolDefinition } from "./types.js";
|
|
|
14
16
|
export declare const TASK_CTL_TOOL = "task_ctl";
|
|
15
17
|
export declare const DEFAULT_WAIT_MS = 30000;
|
|
16
18
|
export declare const MAX_WAIT_MS = 600000;
|
|
19
|
+
/** [#242] `coordinator` 预设下 `wait` 的超时上限(毫秒)。 */
|
|
20
|
+
export declare const COORDINATOR_WAIT_MAX_MS = 5000;
|
|
21
|
+
/**
|
|
22
|
+
* [#242] 组装根把 `wait` 的超时上限绑到 `task_ctl` 工具(`coordinator` 预设);没绑定按 `MAX_WAIT_MS`。
|
|
23
|
+
* 工具描述与参数 schema 不变(其它预设逐字节不变);被封顶时在超时回复里说明。
|
|
24
|
+
*/
|
|
25
|
+
export declare function bindTaskCtlWaitCap(tool: ToolDefinition | undefined, capMs: number): void;
|
|
17
26
|
export interface TaskCtlInput {
|
|
18
27
|
action: "list" | "wait" | "stop" | "output" | "send";
|
|
19
28
|
taskId?: string;
|
package/dist/tools/task-ctl.js
CHANGED
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
* 与父一致,运行时拒绝;
|
|
8
8
|
* - `wait` / `output` 是轮询类调用(`annotations.pollable`,重复调用检测豁免);
|
|
9
9
|
* - `send` = 续聊放后台:等价于 `task{taskId, prompt, background:true}`,完成后同样收到通知。
|
|
10
|
+
* - [#242] `coordinator` 预设下 `wait` 的超时封顶 `COORDINATOR_WAIT_MAX_MS`(协调者不阻塞,见
|
|
11
|
+
* `bindTaskCtlWaitCap`);其它预设不变。
|
|
10
12
|
* - [W7-B1] `wait` 被转后台(`background()`,TUI `Ctrl+B` / RPC `background_task`)打断时立即返回固定
|
|
11
13
|
* 文案,要求模型不要再等。
|
|
12
14
|
*/
|
|
@@ -16,6 +18,17 @@ import { nestedTaskRejection } from "./task.js";
|
|
|
16
18
|
export const TASK_CTL_TOOL = "task_ctl";
|
|
17
19
|
export const DEFAULT_WAIT_MS = 30_000;
|
|
18
20
|
export const MAX_WAIT_MS = 600_000;
|
|
21
|
+
/** [#242] `coordinator` 预设下 `wait` 的超时上限(毫秒)。 */
|
|
22
|
+
export const COORDINATOR_WAIT_MAX_MS = 5_000;
|
|
23
|
+
const waitCaps = new WeakMap();
|
|
24
|
+
/**
|
|
25
|
+
* [#242] 组装根把 `wait` 的超时上限绑到 `task_ctl` 工具(`coordinator` 预设);没绑定按 `MAX_WAIT_MS`。
|
|
26
|
+
* 工具描述与参数 schema 不变(其它预设逐字节不变);被封顶时在超时回复里说明。
|
|
27
|
+
*/
|
|
28
|
+
export function bindTaskCtlWaitCap(tool, capMs) {
|
|
29
|
+
if (tool !== undefined)
|
|
30
|
+
waitCaps.set(tool, capMs);
|
|
31
|
+
}
|
|
19
32
|
function fail(message) {
|
|
20
33
|
return { content: message, isError: true };
|
|
21
34
|
}
|
|
@@ -46,7 +59,7 @@ async function send(input, ctx) {
|
|
|
46
59
|
return { content: result.text, isError: result.isError };
|
|
47
60
|
}
|
|
48
61
|
export function createTaskCtlTool() {
|
|
49
|
-
|
|
62
|
+
const tool = {
|
|
50
63
|
name: TASK_CTL_TOOL,
|
|
51
64
|
label: "TaskCtl",
|
|
52
65
|
description: "Manage sub-agent tasks started by task: list them, wait for one, stop it, read its " +
|
|
@@ -98,15 +111,20 @@ export function createTaskCtlTool() {
|
|
|
98
111
|
};
|
|
99
112
|
}
|
|
100
113
|
case "wait": {
|
|
101
|
-
const
|
|
114
|
+
const cap = Math.min(waitCaps.get(tool) ?? MAX_WAIT_MS, MAX_WAIT_MS);
|
|
115
|
+
const wanted = Math.min(Math.max(input.timeoutMs ?? DEFAULT_WAIT_MS, 0), MAX_WAIT_MS);
|
|
116
|
+
const timeout = Math.min(wanted, cap);
|
|
102
117
|
const signal = control.detachSignal(taskId);
|
|
103
118
|
const result = await control.wait(taskId, timeout, { signal });
|
|
104
119
|
if (result === undefined && signal.aborted)
|
|
105
120
|
return { content: waitDetachedText(taskId) };
|
|
106
121
|
if (result === undefined) {
|
|
107
122
|
const turns = control.get(taskId)?.turns ?? 0;
|
|
123
|
+
const capped = wanted > timeout
|
|
124
|
+
? ` Waits are capped at ${Math.round(timeout / 1000)}s here; do not block on it: keep working, a <task-notification> follows when it finishes.`
|
|
125
|
+
: "";
|
|
108
126
|
return {
|
|
109
|
-
content: `Task ${taskId} is still running (${turns} turns so far). Wait again or stop it
|
|
127
|
+
content: `Task ${taskId} is still running (${turns} turns so far). Wait again or stop it.${capped}`,
|
|
110
128
|
};
|
|
111
129
|
}
|
|
112
130
|
return {
|
|
@@ -119,4 +137,5 @@ export function createTaskCtlTool() {
|
|
|
119
137
|
}
|
|
120
138
|
},
|
|
121
139
|
};
|
|
140
|
+
return tool;
|
|
122
141
|
}
|
package/dist/tools/types.d.ts
CHANGED
|
@@ -27,7 +27,10 @@ export type { JsonSchema } from "../ai/types.js";
|
|
|
27
27
|
/**
|
|
28
28
|
* [W6-C0] `memory`:记忆工具专用(`permissions/memory-class.ts`)——`view` 按 read、写命令按 execute 判定。
|
|
29
29
|
*/
|
|
30
|
-
|
|
30
|
+
/**
|
|
31
|
+
* `network`:联网读取(`web_fetch`,`permissions/network-class.ts`)——按 execute 判定,plan 下询问。
|
|
32
|
+
*/
|
|
33
|
+
export type ToolPermission = "read" | "write" | "execute" | "memory" | "network";
|
|
31
34
|
export type ToolExecutionMode = "sequential" | "parallel";
|
|
32
35
|
export interface ToolAnnotations {
|
|
33
36
|
readOnly?: boolean;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `web_fetch` 工具(docs/guides/permissions.md「网络类」):GET 一个公网 http(s) URL,HTML 转 Markdown,
|
|
3
|
+
* JSON 美化,`text/*` 原样,二进制不解析。权限类 `network`(default 询问、`-p` 拒绝,见 network-class.ts)。
|
|
4
|
+
*
|
|
5
|
+
* - 地址:`url-guard.ts` 静态校验 + 解析后全部地址检查 + 连接时钉住已检查的地址;同主机重定向 ≤ 5 跳、
|
|
6
|
+
* 逐跳重检;跨主机不跟,告诉模型目标 URL(下一次调用重新过权限)。
|
|
7
|
+
* - 限制写死:总超时 30 s、首字节 15 s、响应体(解压后)≤ 5 MB;不发 cookie、无自定义头、只有 GET。
|
|
8
|
+
* `agent: false`:不复用连接、不走全局代理 agent(代理会自己解析主机名,钉不住地址)。
|
|
9
|
+
* - 输出首行固定 `Content fetched from <final url> (untrusted):`;正文按 `offset` / `maxChars` 切片,
|
|
10
|
+
* 截断时全文写 `outputs/<toolCallId>.txt` 并在末尾说明续读的 offset。
|
|
11
|
+
* - 测试注入 `lookup` 与 `request`(不出网)。
|
|
12
|
+
*/
|
|
13
|
+
import * as http from "node:http";
|
|
14
|
+
import type { IncomingMessage } from "node:http";
|
|
15
|
+
import type { ToolDefinition } from "../types.js";
|
|
16
|
+
import { type LookupAll } from "./url-guard.js";
|
|
17
|
+
export declare const WEB_FETCH_TOOL = "web_fetch";
|
|
18
|
+
export declare const DEFAULT_FETCH_CHARS = 50000;
|
|
19
|
+
export declare const MAX_FETCH_CHARS = 200000;
|
|
20
|
+
export declare const FETCH_TOTAL_TIMEOUT_MS = 30000;
|
|
21
|
+
export declare const FETCH_FIRST_BYTE_TIMEOUT_MS = 15000;
|
|
22
|
+
export declare const FETCH_MAX_BODY_BYTES: number;
|
|
23
|
+
export declare const FETCH_ACCEPT = "text/markdown, text/html, text/plain, application/json;q=0.9, */*;q=0.1";
|
|
24
|
+
export interface WebFetchInput {
|
|
25
|
+
url: string;
|
|
26
|
+
maxChars?: number;
|
|
27
|
+
offset?: number;
|
|
28
|
+
raw?: boolean;
|
|
29
|
+
}
|
|
30
|
+
export type FetchRequest = (url: URL, options: http.RequestOptions, onResponse: (res: IncomingMessage) => void) => http.ClientRequest;
|
|
31
|
+
export interface WebFetchOptions {
|
|
32
|
+
lookup?: LookupAll;
|
|
33
|
+
request?: FetchRequest;
|
|
34
|
+
totalTimeoutMs?: number;
|
|
35
|
+
firstByteTimeoutMs?: number;
|
|
36
|
+
maxBodyBytes?: number;
|
|
37
|
+
}
|
|
38
|
+
export type FetchFormat = "markdown" | "text" | "json" | "binary";
|
|
39
|
+
export interface WebFetchDetails {
|
|
40
|
+
status: number;
|
|
41
|
+
contentType: string;
|
|
42
|
+
bytes: number;
|
|
43
|
+
truncated: boolean;
|
|
44
|
+
finalUrl: string;
|
|
45
|
+
format?: FetchFormat;
|
|
46
|
+
title?: string;
|
|
47
|
+
}
|
|
48
|
+
export declare function createWebFetchTool(options?: WebFetchOptions): ToolDefinition<WebFetchInput>;
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `web_fetch` 工具(docs/guides/permissions.md「网络类」):GET 一个公网 http(s) URL,HTML 转 Markdown,
|
|
3
|
+
* JSON 美化,`text/*` 原样,二进制不解析。权限类 `network`(default 询问、`-p` 拒绝,见 network-class.ts)。
|
|
4
|
+
*
|
|
5
|
+
* - 地址:`url-guard.ts` 静态校验 + 解析后全部地址检查 + 连接时钉住已检查的地址;同主机重定向 ≤ 5 跳、
|
|
6
|
+
* 逐跳重检;跨主机不跟,告诉模型目标 URL(下一次调用重新过权限)。
|
|
7
|
+
* - 限制写死:总超时 30 s、首字节 15 s、响应体(解压后)≤ 5 MB;不发 cookie、无自定义头、只有 GET。
|
|
8
|
+
* `agent: false`:不复用连接、不走全局代理 agent(代理会自己解析主机名,钉不住地址)。
|
|
9
|
+
* - 输出首行固定 `Content fetched from <final url> (untrusted):`;正文按 `offset` / `maxChars` 切片,
|
|
10
|
+
* 截断时全文写 `outputs/<toolCallId>.txt` 并在末尾说明续读的 offset。
|
|
11
|
+
* - 测试注入 `lookup` 与 `request`(不出网)。
|
|
12
|
+
*/
|
|
13
|
+
import * as http from "node:http";
|
|
14
|
+
import * as https from "node:https";
|
|
15
|
+
import { createBrotliDecompress, createGunzip, createInflate } from "node:zlib";
|
|
16
|
+
import { AMA_VERSION } from "../../version.js";
|
|
17
|
+
import { RESULT_NOTE_RESERVE, writeFullOutput } from "../truncate.js";
|
|
18
|
+
import { htmlToMarkdown, sniffMetaCharset } from "./html-to-markdown.js";
|
|
19
|
+
import { MAX_SAME_HOST_REDIRECTS, checkFetchUrl, pinnedLookup, redirectPolicy, resolvePublicAddresses, } from "./url-guard.js";
|
|
20
|
+
export const WEB_FETCH_TOOL = "web_fetch";
|
|
21
|
+
export const DEFAULT_FETCH_CHARS = 50_000;
|
|
22
|
+
export const MAX_FETCH_CHARS = 200_000;
|
|
23
|
+
export const FETCH_TOTAL_TIMEOUT_MS = 30_000;
|
|
24
|
+
export const FETCH_FIRST_BYTE_TIMEOUT_MS = 15_000;
|
|
25
|
+
export const FETCH_MAX_BODY_BYTES = 5 * 1024 * 1024;
|
|
26
|
+
const ERROR_BODY_CHARS = 2_000;
|
|
27
|
+
/** 地址守卫、超时、重定向等自己产生的错误(说明直接进工具结果)。 */
|
|
28
|
+
class FetchError extends Error {
|
|
29
|
+
}
|
|
30
|
+
export const FETCH_ACCEPT = "text/markdown, text/html, text/plain, application/json;q=0.9, */*;q=0.1";
|
|
31
|
+
const defaultRequest = (url, options, onResponse) => (url.protocol === "https:" ? https : http).request(url, options, onResponse);
|
|
32
|
+
function fail(content, details) {
|
|
33
|
+
return { content, isError: true, ...(details !== undefined ? { details } : {}) };
|
|
34
|
+
}
|
|
35
|
+
/** 读响应体(按 Content-Encoding 解压),解压后超过上限即停。 */
|
|
36
|
+
function readBody(res, limit) {
|
|
37
|
+
const encoding = String(res.headers["content-encoding"] ?? "")
|
|
38
|
+
.trim()
|
|
39
|
+
.toLowerCase();
|
|
40
|
+
let stream = res;
|
|
41
|
+
if (encoding === "gzip" || encoding === "x-gzip")
|
|
42
|
+
stream = res.pipe(createGunzip());
|
|
43
|
+
else if (encoding === "br")
|
|
44
|
+
stream = res.pipe(createBrotliDecompress());
|
|
45
|
+
else if (encoding === "deflate")
|
|
46
|
+
stream = res.pipe(createInflate());
|
|
47
|
+
return new Promise((resolve, reject) => {
|
|
48
|
+
const chunks = [];
|
|
49
|
+
let size = 0;
|
|
50
|
+
let done = false;
|
|
51
|
+
const finish = (cut) => {
|
|
52
|
+
if (done)
|
|
53
|
+
return;
|
|
54
|
+
done = true;
|
|
55
|
+
resolve({ body: Buffer.concat(chunks, Math.min(size, limit)).subarray(0, limit), cut });
|
|
56
|
+
};
|
|
57
|
+
stream.on("data", (chunk) => {
|
|
58
|
+
if (done)
|
|
59
|
+
return;
|
|
60
|
+
chunks.push(chunk);
|
|
61
|
+
size += chunk.length;
|
|
62
|
+
if (size > limit) {
|
|
63
|
+
finish(true);
|
|
64
|
+
res.destroy();
|
|
65
|
+
if (stream !== res)
|
|
66
|
+
stream.destroy();
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
stream.on("end", () => finish(false));
|
|
70
|
+
stream.on("error", (err) => (done ? undefined : ((done = true), reject(err))));
|
|
71
|
+
res.on("error", (err) => (done ? undefined : ((done = true), reject(err))));
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
function decodeText(body, contentType, html) {
|
|
75
|
+
const declared = /charset\s*=\s*"?([\w.:-]+)/i.exec(contentType)?.[1];
|
|
76
|
+
const sniffed = html ? sniffMetaCharset(body.subarray(0, 2048).toString("latin1")) : undefined;
|
|
77
|
+
for (const label of [declared, sniffed, "utf-8"]) {
|
|
78
|
+
if (label === undefined)
|
|
79
|
+
continue;
|
|
80
|
+
try {
|
|
81
|
+
return new TextDecoder(label).decode(body);
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// 不支持的编码:试下一个
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return body.toString("utf8");
|
|
88
|
+
}
|
|
89
|
+
function formatOf(contentType, body) {
|
|
90
|
+
const type = contentType.split(";")[0]?.trim().toLowerCase() ?? "";
|
|
91
|
+
if (type === "text/html" || type === "application/xhtml+xml")
|
|
92
|
+
return "html";
|
|
93
|
+
if (type === "application/json" || type.endsWith("+json"))
|
|
94
|
+
return "json";
|
|
95
|
+
if (type.startsWith("text/") ||
|
|
96
|
+
/^application\/(xml|javascript|ecmascript|x-yaml|yaml|toml)$/.test(type) ||
|
|
97
|
+
type.endsWith("+xml"))
|
|
98
|
+
return "text";
|
|
99
|
+
if (type === "") {
|
|
100
|
+
const head = body.subarray(0, 1024);
|
|
101
|
+
if (head.includes(0))
|
|
102
|
+
return "binary";
|
|
103
|
+
return /^\s*<(!doctype html|html)/i.test(head.toString("latin1")) ? "html" : "text";
|
|
104
|
+
}
|
|
105
|
+
return "binary";
|
|
106
|
+
}
|
|
107
|
+
/** 按格式转成文本:`{ format, title, text }`。 */
|
|
108
|
+
function convert(body, contentType, finalUrl, raw) {
|
|
109
|
+
const kind = formatOf(contentType, body);
|
|
110
|
+
if (kind === "binary") {
|
|
111
|
+
const type = contentType.split(";")[0]?.trim() || "unknown type";
|
|
112
|
+
return {
|
|
113
|
+
format: "binary",
|
|
114
|
+
text: `Binary content (${type}, ${body.length} bytes); not parsed.`,
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
const text = decodeText(body, contentType, kind === "html");
|
|
118
|
+
if (raw)
|
|
119
|
+
return { format: "text", text };
|
|
120
|
+
if (kind === "html") {
|
|
121
|
+
const page = htmlToMarkdown(text, finalUrl);
|
|
122
|
+
return page.title === undefined
|
|
123
|
+
? { format: "markdown", text: page.markdown }
|
|
124
|
+
: { format: "markdown", title: page.title, text: page.markdown };
|
|
125
|
+
}
|
|
126
|
+
if (kind === "json") {
|
|
127
|
+
try {
|
|
128
|
+
return { format: "json", text: JSON.stringify(JSON.parse(text), null, 2) };
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return { format: "text", text };
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return { format: "text", text };
|
|
135
|
+
}
|
|
136
|
+
export function createWebFetchTool(options = {}) {
|
|
137
|
+
const request = options.request ?? defaultRequest;
|
|
138
|
+
const totalMs = options.totalTimeoutMs ?? FETCH_TOTAL_TIMEOUT_MS;
|
|
139
|
+
const firstByteMs = options.firstByteTimeoutMs ?? FETCH_FIRST_BYTE_TIMEOUT_MS;
|
|
140
|
+
const maxBody = options.maxBodyBytes ?? FETCH_MAX_BODY_BYTES;
|
|
141
|
+
/** 一跳:解析 + 检查 + 钉住地址发 GET,等到响应头。 */
|
|
142
|
+
async function get(url, hostname, signal) {
|
|
143
|
+
const resolved = await resolvePublicAddresses(hostname, options.lookup);
|
|
144
|
+
if (!resolved.ok)
|
|
145
|
+
throw new FetchError(resolved.reason);
|
|
146
|
+
return new Promise((resolve, reject) => {
|
|
147
|
+
const timer = setTimeout(() => {
|
|
148
|
+
req.destroy(new FetchError(`no response within ${firstByteMs / 1000}s`));
|
|
149
|
+
}, firstByteMs);
|
|
150
|
+
const req = request(url, {
|
|
151
|
+
method: "GET",
|
|
152
|
+
agent: false,
|
|
153
|
+
lookup: pinnedLookup(resolved.addresses),
|
|
154
|
+
headers: {
|
|
155
|
+
Accept: FETCH_ACCEPT,
|
|
156
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
157
|
+
"User-Agent": `ama/${AMA_VERSION}`,
|
|
158
|
+
},
|
|
159
|
+
signal,
|
|
160
|
+
}, (res) => {
|
|
161
|
+
clearTimeout(timer);
|
|
162
|
+
resolve(res);
|
|
163
|
+
});
|
|
164
|
+
req.on("error", (err) => {
|
|
165
|
+
clearTimeout(timer);
|
|
166
|
+
reject(err);
|
|
167
|
+
});
|
|
168
|
+
req.end();
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
async function fetchFollowing(start, hostname, signal) {
|
|
172
|
+
let url = start;
|
|
173
|
+
let host = hostname;
|
|
174
|
+
for (let hop = 0;; hop++) {
|
|
175
|
+
const res = await get(url, host, signal);
|
|
176
|
+
const status = res.statusCode ?? 0;
|
|
177
|
+
const location = res.headers.location;
|
|
178
|
+
if (status >= 300 && status < 400 && typeof location === "string") {
|
|
179
|
+
res.resume();
|
|
180
|
+
let next;
|
|
181
|
+
try {
|
|
182
|
+
next = new URL(location, url);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
throw new FetchError(`invalid redirect location ${JSON.stringify(location)}`);
|
|
186
|
+
}
|
|
187
|
+
if (redirectPolicy(url, next) === "cross-host")
|
|
188
|
+
return { redirect: next.href, status };
|
|
189
|
+
const checked = checkFetchUrl(next.href);
|
|
190
|
+
if (!checked.ok)
|
|
191
|
+
throw new FetchError(`redirect to ${next.href} refused: ${checked.reason}`);
|
|
192
|
+
if (hop + 1 > MAX_SAME_HOST_REDIRECTS)
|
|
193
|
+
throw new FetchError(`more than ${MAX_SAME_HOST_REDIRECTS} redirects`);
|
|
194
|
+
url = checked.url;
|
|
195
|
+
host = checked.hostname;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
const { body, cut } = await readBody(res, maxBody);
|
|
199
|
+
return { res, finalUrl: url.href, body, bodyCut: cut };
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
async function execute(input, ctx) {
|
|
203
|
+
const checked = checkFetchUrl(input.url);
|
|
204
|
+
if (!checked.ok)
|
|
205
|
+
return fail(`Cannot fetch ${String(input.url)}: ${checked.reason}`);
|
|
206
|
+
const offset = input.offset ?? 0;
|
|
207
|
+
const asked = input.maxChars ?? DEFAULT_FETCH_CHARS;
|
|
208
|
+
if (!Number.isInteger(offset) || offset < 0)
|
|
209
|
+
return fail("offset must be an integer ≥ 0");
|
|
210
|
+
if (!Number.isInteger(asked) || asked < 1)
|
|
211
|
+
return fail("maxChars must be an integer ≥ 1");
|
|
212
|
+
const sessionCap = ctx.maxResultChars === undefined ? Infinity : ctx.maxResultChars - RESULT_NOTE_RESERVE;
|
|
213
|
+
const limit = Math.max(1000, Math.min(asked, MAX_FETCH_CHARS, sessionCap));
|
|
214
|
+
const controller = new AbortController();
|
|
215
|
+
const timer = setTimeout(() => controller.abort(new FetchError(`timed out after ${totalMs / 1000}s`)), totalMs);
|
|
216
|
+
const signal = AbortSignal.any([ctx.signal, controller.signal]);
|
|
217
|
+
let fetched;
|
|
218
|
+
try {
|
|
219
|
+
fetched = await fetchFollowing(checked.url, checked.hostname, signal);
|
|
220
|
+
}
|
|
221
|
+
catch (err) {
|
|
222
|
+
const reason = controller.signal.aborted && controller.signal.reason instanceof FetchError
|
|
223
|
+
? controller.signal.reason.message
|
|
224
|
+
: err instanceof Error
|
|
225
|
+
? err.message
|
|
226
|
+
: String(err);
|
|
227
|
+
return fail(`Cannot fetch ${checked.url.href}: ${reason}`);
|
|
228
|
+
}
|
|
229
|
+
finally {
|
|
230
|
+
clearTimeout(timer);
|
|
231
|
+
}
|
|
232
|
+
if ("redirect" in fetched) {
|
|
233
|
+
return {
|
|
234
|
+
content: `Redirected to ${fetched.redirect}; call web_fetch with it if appropriate.`,
|
|
235
|
+
details: {
|
|
236
|
+
status: fetched.status,
|
|
237
|
+
contentType: "",
|
|
238
|
+
bytes: 0,
|
|
239
|
+
truncated: false,
|
|
240
|
+
finalUrl: fetched.redirect,
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
const { res, finalUrl, body, bodyCut } = fetched;
|
|
245
|
+
const status = res.statusCode ?? 0;
|
|
246
|
+
const contentType = String(res.headers["content-type"] ?? "");
|
|
247
|
+
const page = convert(body, contentType, finalUrl, input.raw === true);
|
|
248
|
+
const base = {
|
|
249
|
+
status,
|
|
250
|
+
contentType,
|
|
251
|
+
bytes: body.length,
|
|
252
|
+
truncated: false,
|
|
253
|
+
finalUrl,
|
|
254
|
+
format: page.format,
|
|
255
|
+
...(page.title !== undefined ? { title: page.title } : {}),
|
|
256
|
+
};
|
|
257
|
+
if (status < 200 || status >= 300) {
|
|
258
|
+
const snippet = page.text.slice(0, ERROR_BODY_CHARS);
|
|
259
|
+
return fail(`HTTP ${status}${res.statusMessage ? ` ${res.statusMessage}` : ""} from ${finalUrl}${snippet.trim() === "" ? "" : `\n\n${snippet}`}`, base);
|
|
260
|
+
}
|
|
261
|
+
const full = page.title === undefined ? page.text : `# ${page.title}\n\n${page.text}`;
|
|
262
|
+
if (offset > 0 && offset >= full.length)
|
|
263
|
+
return fail(`offset ${offset} is past the end of the content (${full.length} chars)`, base);
|
|
264
|
+
const slice = full.slice(offset, offset + limit);
|
|
265
|
+
const end = offset + slice.length;
|
|
266
|
+
const truncated = end < full.length;
|
|
267
|
+
const notes = [];
|
|
268
|
+
if (truncated) {
|
|
269
|
+
let saved = "";
|
|
270
|
+
try {
|
|
271
|
+
saved = `; full text: ${writeFullOutput(ctx.outputDir, `${ctx.toolCallId}.txt`, full)}`;
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
// 落盘失败不影响结果
|
|
275
|
+
}
|
|
276
|
+
notes.push(`[Truncated at ${end} of ${full.length} chars; continue with offset=${end}${saved}]`);
|
|
277
|
+
}
|
|
278
|
+
if (bodyCut)
|
|
279
|
+
notes.push(`[Response body exceeded ${maxBody / (1024 * 1024)} MB; the rest was not downloaded]`);
|
|
280
|
+
const header = `Content fetched from ${finalUrl} (untrusted):`;
|
|
281
|
+
return {
|
|
282
|
+
content: [header, slice, ...notes].join("\n\n"),
|
|
283
|
+
details: { ...base, truncated: truncated || bodyCut },
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
return {
|
|
287
|
+
name: WEB_FETCH_TOOL,
|
|
288
|
+
label: "Fetch",
|
|
289
|
+
description: "Fetch a public http(s) URL as Markdown (HTML) or text, cut at maxChars (default 50000); continue with offset. Cross-host redirects are reported, not followed.",
|
|
290
|
+
parameters: {
|
|
291
|
+
type: "object",
|
|
292
|
+
properties: {
|
|
293
|
+
url: { type: "string" },
|
|
294
|
+
maxChars: { type: "integer" },
|
|
295
|
+
offset: { type: "integer" },
|
|
296
|
+
raw: { type: "boolean", description: "Skip HTML conversion" },
|
|
297
|
+
},
|
|
298
|
+
required: ["url"],
|
|
299
|
+
additionalProperties: false,
|
|
300
|
+
},
|
|
301
|
+
permission: "network",
|
|
302
|
+
executionMode: "parallel",
|
|
303
|
+
annotations: { readOnly: true, openWorld: true },
|
|
304
|
+
promptSnippet: "web_fetch: fetch a web page as text",
|
|
305
|
+
execute,
|
|
306
|
+
};
|
|
307
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTML → Markdown(`web_fetch` 用,零依赖)。宽松的分词 + 轻量树,不做完整 HTML5 解析:
|
|
3
|
+
*
|
|
4
|
+
* - 丢弃 `script` / `style` / `noscript` / `svg` / `template` / `iframe` / `nav` / `footer` / `aside` /
|
|
5
|
+
* `form` / `button` 及其内容,`<!-- -->` 注释与 `<head>`(标题单独取 `<title>`);
|
|
6
|
+
* - 正文优先 `<main>`,其次 `<article>`、`role="main"`,再次 `<body>`;
|
|
7
|
+
* - 保留标题、段落、列表(嵌套缩进)、链接(`[文字](绝对 URL)`,`javascript:` 只留文字)、行内代码与
|
|
8
|
+
* `<pre>` 代码块(`language-xxx` 类名作语言)、引用、简单表格、`<br>` / `<hr>`;图片只留 alt;
|
|
9
|
+
* - 实体:常用命名实体 + 十进制 / 十六进制数字实体;非 `<pre>` 文本折叠空白。
|
|
10
|
+
*
|
|
11
|
+
* 输出只依赖输入,便于黄金测试。
|
|
12
|
+
*/
|
|
13
|
+
export interface HtmlConversion {
|
|
14
|
+
title?: string;
|
|
15
|
+
markdown: string;
|
|
16
|
+
}
|
|
17
|
+
interface Node {
|
|
18
|
+
tag: string;
|
|
19
|
+
attrs: Record<string, string>;
|
|
20
|
+
children: (Node | string)[];
|
|
21
|
+
parent?: Node;
|
|
22
|
+
}
|
|
23
|
+
/** 解码字符实体。 */
|
|
24
|
+
export declare function decodeEntities(text: string): string;
|
|
25
|
+
/** 宽松解析成树。 */
|
|
26
|
+
export declare function parseHtml(html: string): Node;
|
|
27
|
+
/** HTML 全文 → `{ title, markdown }`。`baseUrl` 用来把相对链接变成绝对 URL。 */
|
|
28
|
+
export declare function htmlToMarkdown(html: string, baseUrl?: string): HtmlConversion;
|
|
29
|
+
/** `<meta charset>` / `http-equiv` 里声明的编码(只看前 2 KB)。 */
|
|
30
|
+
export declare function sniffMetaCharset(head: string): string | undefined;
|
|
31
|
+
export {};
|