@epoch-agent/plugin-web 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +20 -51
- package/package.json +3 -2
package/dist/index.js
CHANGED
|
@@ -11,8 +11,8 @@ function htmlToMarkdown(html) {
|
|
|
11
11
|
);
|
|
12
12
|
cleaned = cleaned.replace(/<br\s*\/?>/gi, "\n");
|
|
13
13
|
cleaned = cleaned.replace(/<a[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>/gi, (_, url, text) => {
|
|
14
|
-
const
|
|
15
|
-
return
|
|
14
|
+
const t5 = stripTags(text).trim();
|
|
15
|
+
return t5 ? `[${t5}](${url})` : url;
|
|
16
16
|
});
|
|
17
17
|
cleaned = cleaned.replace(
|
|
18
18
|
/<img[^>]*src="([^"]*)"[^>]*alt="([^"]*)"[^>]*>/gi,
|
|
@@ -53,52 +53,35 @@ function stripTags(html) {
|
|
|
53
53
|
function decodeEntities(text) {
|
|
54
54
|
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/ /g, " ");
|
|
55
55
|
}
|
|
56
|
-
|
|
57
56
|
// src/utils/url-safety.ts
|
|
58
57
|
import { lookup } from "dns/promises";
|
|
59
58
|
import { isIP } from "net";
|
|
60
59
|
var ALLOWED_PROTOCOLS = ["http:", "https:"];
|
|
61
60
|
var BLOCKED_IPV4 = [
|
|
62
61
|
/^0\./,
|
|
63
|
-
// 当前网络 / 未指定
|
|
64
62
|
/^10\./,
|
|
65
|
-
// class A private
|
|
66
63
|
/^127\./,
|
|
67
|
-
// loopback
|
|
68
64
|
/^169\.254\./,
|
|
69
|
-
// link-local(含云元数据 169.254.169.254)
|
|
70
65
|
/^172\.(1[6-9]|2\d|3[01])\./,
|
|
71
|
-
// class B private
|
|
72
66
|
/^192\.0\.0\./,
|
|
73
|
-
// IETF protocol assignments
|
|
74
67
|
/^192\.0\.2\./,
|
|
75
|
-
// TEST-NET-1
|
|
76
68
|
/^192\.168\./,
|
|
77
|
-
// class C private
|
|
78
69
|
/^198\.1[89]\./,
|
|
79
|
-
// benchmark
|
|
80
70
|
/^198\.51\.100\./,
|
|
81
|
-
// TEST-NET-2
|
|
82
71
|
/^203\.0\.113\./,
|
|
83
|
-
// TEST-NET-3
|
|
84
72
|
/^2(2[4-9]|3\d)\./,
|
|
85
|
-
// 224-239 组播
|
|
86
73
|
/^2(4[0-9]|5[0-5])\./,
|
|
87
|
-
// 240-255 保留 / 广播
|
|
88
74
|
/^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./
|
|
89
|
-
// 100.64/10 CGNAT
|
|
90
75
|
];
|
|
91
76
|
var BLOCKED_HOSTNAMES = /* @__PURE__ */ new Set([
|
|
92
77
|
"localhost",
|
|
93
78
|
"localhost.localdomain",
|
|
94
79
|
"ip6-localhost",
|
|
95
80
|
"ip6-loopback",
|
|
96
|
-
// 云元数据端点
|
|
97
81
|
"metadata.google.internal",
|
|
98
82
|
"metadata.goog",
|
|
99
83
|
"metadata",
|
|
100
84
|
"instance-data",
|
|
101
|
-
// 常见的「解析到回环」的公共域名,用来绕过主机名检查
|
|
102
85
|
"localtest.me",
|
|
103
86
|
"lvh.me",
|
|
104
87
|
"vcap.me"
|
|
@@ -168,7 +151,6 @@ async function checkUrlSafetyAsync(url) {
|
|
|
168
151
|
}
|
|
169
152
|
return sync;
|
|
170
153
|
}
|
|
171
|
-
|
|
172
154
|
// src/tools/web-fetch.ts
|
|
173
155
|
var MAX_SIZE = 5 * 1024 * 1024;
|
|
174
156
|
var MAX_REDIRECTS = 5;
|
|
@@ -299,7 +281,6 @@ function extractTitle(html, isHtml) {
|
|
|
299
281
|
const m = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
|
300
282
|
return m && m[1] ? m[1].trim().slice(0, 200) : void 0;
|
|
301
283
|
}
|
|
302
|
-
|
|
303
284
|
// src/search/provider.ts
|
|
304
285
|
var SearchError = class extends Error {
|
|
305
286
|
constructor(message, status) {
|
|
@@ -345,7 +326,6 @@ function toResult(raw, keys) {
|
|
|
345
326
|
...publishedAt ? { publishedAt } : {}
|
|
346
327
|
};
|
|
347
328
|
}
|
|
348
|
-
|
|
349
329
|
// src/tools/web-search.ts
|
|
350
330
|
var DEFAULT_MAX_RESULTS = 5;
|
|
351
331
|
var HARD_MAX_RESULTS = 20;
|
|
@@ -443,8 +423,10 @@ function createWebSearchExec(deps) {
|
|
|
443
423
|
}
|
|
444
424
|
};
|
|
445
425
|
}
|
|
446
|
-
|
|
426
|
+
// src/search/registry.ts
|
|
427
|
+
import { t as t4 } from "@epoch-agent/infra";
|
|
447
428
|
// src/search/brave.ts
|
|
429
|
+
import { t } from "@epoch-agent/infra";
|
|
448
430
|
var ENDPOINT = "https://api.search.brave.com/res/v1/web/search";
|
|
449
431
|
var MAX_RESULTS = 20;
|
|
450
432
|
function createBraveProvider(opts = {}) {
|
|
@@ -452,7 +434,7 @@ function createBraveProvider(opts = {}) {
|
|
|
452
434
|
return {
|
|
453
435
|
name: "brave",
|
|
454
436
|
unavailableReason() {
|
|
455
|
-
return key() ? null : "
|
|
437
|
+
return key() ? null : t("web_search_setup.need_brave_key");
|
|
456
438
|
},
|
|
457
439
|
async search(query, options) {
|
|
458
440
|
const url = new URL(ENDPOINT);
|
|
@@ -507,22 +489,22 @@ function toFreshness(days) {
|
|
|
507
489
|
if (days <= 366) return "py";
|
|
508
490
|
return void 0;
|
|
509
491
|
}
|
|
510
|
-
|
|
511
492
|
// src/search/searxng.ts
|
|
493
|
+
import { t as t2 } from "@epoch-agent/infra";
|
|
512
494
|
function createSearxngProvider(opts) {
|
|
513
495
|
return {
|
|
514
496
|
name: "searxng",
|
|
515
497
|
unavailableReason() {
|
|
516
498
|
if (!opts.baseUrl?.trim()) {
|
|
517
|
-
return "
|
|
499
|
+
return t2("web_search_setup.searxng_need_base_url");
|
|
518
500
|
}
|
|
519
501
|
try {
|
|
520
502
|
const url = new URL(opts.baseUrl);
|
|
521
503
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
522
|
-
return
|
|
504
|
+
return t2("web_search_setup.searxng_bad_protocol", { protocol: url.protocol });
|
|
523
505
|
}
|
|
524
506
|
} catch {
|
|
525
|
-
return
|
|
507
|
+
return t2("web_search_setup.searxng_bad_url", { url: opts.baseUrl });
|
|
526
508
|
}
|
|
527
509
|
return null;
|
|
528
510
|
},
|
|
@@ -583,8 +565,8 @@ function timeRange(days) {
|
|
|
583
565
|
if (days <= 366) return "year";
|
|
584
566
|
return void 0;
|
|
585
567
|
}
|
|
586
|
-
|
|
587
568
|
// src/search/tavily.ts
|
|
569
|
+
import { t as t3 } from "@epoch-agent/infra";
|
|
588
570
|
var ENDPOINT2 = "https://api.tavily.com/search";
|
|
589
571
|
var MAX_RESULTS2 = 20;
|
|
590
572
|
function createTavilyProvider(opts = {}) {
|
|
@@ -592,14 +574,12 @@ function createTavilyProvider(opts = {}) {
|
|
|
592
574
|
return {
|
|
593
575
|
name: "tavily",
|
|
594
576
|
unavailableReason() {
|
|
595
|
-
return key() ? null : "
|
|
577
|
+
return key() ? null : t3("web_search_setup.need_tavily_key");
|
|
596
578
|
},
|
|
597
579
|
async search(query, options) {
|
|
598
580
|
const body = {
|
|
599
581
|
query,
|
|
600
582
|
max_results: Math.min(options.maxResults, MAX_RESULTS2),
|
|
601
|
-
// `basic` 而不是 `advanced`:后者贵一倍,而我们本来就不要正文 ——
|
|
602
|
-
// 模型看完 snippet 自己去 web_fetch
|
|
603
583
|
search_depth: "basic"
|
|
604
584
|
};
|
|
605
585
|
if (options.recencyDays !== void 0 && options.recencyDays > 0) {
|
|
@@ -613,8 +593,6 @@ function createTavilyProvider(opts = {}) {
|
|
|
613
593
|
method: "POST",
|
|
614
594
|
headers: {
|
|
615
595
|
"Content-Type": "application/json",
|
|
616
|
-
// Bearer 而不是 body 里的 api_key 字段:后者是旧写法,
|
|
617
|
-
// 且会让 key 出现在任何记录了请求体的地方
|
|
618
596
|
Authorization: `Bearer ${key()}`
|
|
619
597
|
},
|
|
620
598
|
body: JSON.stringify(body),
|
|
@@ -638,7 +616,6 @@ function createTavilyProvider(opts = {}) {
|
|
|
638
616
|
}
|
|
639
617
|
};
|
|
640
618
|
}
|
|
641
|
-
|
|
642
619
|
// src/search/registry.ts
|
|
643
620
|
var AUTO_ORDER = ["tavily", "brave", "searxng"];
|
|
644
621
|
function createSearchProvider(type, config) {
|
|
@@ -652,22 +629,25 @@ function selectSearchProvider(config) {
|
|
|
652
629
|
if (explicit) {
|
|
653
630
|
const provider = createSearchProvider(explicit, config);
|
|
654
631
|
const reason = provider.unavailableReason();
|
|
655
|
-
return reason ? {
|
|
632
|
+
return reason ? {
|
|
633
|
+
provider: null,
|
|
634
|
+
detail: t4("web_search_setup.explicit_unavailable", { provider: explicit, reason })
|
|
635
|
+
} : { provider, detail: t4("web_search_setup.explicit_ok", { provider: explicit }) };
|
|
656
636
|
}
|
|
657
637
|
const tried = [];
|
|
658
638
|
for (const type of AUTO_ORDER) {
|
|
659
639
|
const provider = createSearchProvider(type, config);
|
|
660
640
|
const reason = provider.unavailableReason();
|
|
661
|
-
if (!reason) return { provider, detail:
|
|
641
|
+
if (!reason) return { provider, detail: t4("web_search_setup.auto", { provider: type }) };
|
|
662
642
|
tried.push(reason);
|
|
663
643
|
}
|
|
664
644
|
return {
|
|
665
645
|
provider: null,
|
|
666
|
-
|
|
667
|
-
|
|
646
|
+
detail: t4("web_search_setup.none", {
|
|
647
|
+
tried: tried.join(t4("web_search_setup.tried_sep"))
|
|
648
|
+
})
|
|
668
649
|
};
|
|
669
650
|
}
|
|
670
|
-
|
|
671
651
|
// src/index.ts
|
|
672
652
|
var webPlugin = {
|
|
673
653
|
name: "@epoch-agent/plugin-web",
|
|
@@ -686,13 +666,8 @@ var webPlugin = {
|
|
|
686
666
|
},
|
|
687
667
|
required: ["url"]
|
|
688
668
|
},
|
|
689
|
-
// `openWorldHint: true` 不只是元数据:core 的 tool-executor 用它决定
|
|
690
|
-
// 要不要把输出包进 `<tool_output untrusted="true">`。抓来的网页内容
|
|
691
|
-
// 作者既不是用户也不是我们,必须标成「数据」而不是「指令」回灌上下文。
|
|
692
669
|
annotations: { readOnlyHint: true, idempotentHint: false, openWorldHint: true },
|
|
693
670
|
operation: "network",
|
|
694
|
-
// 审批目标是 URL —— 五候选里没有 `url`,不给 describeTarget 就只能退回
|
|
695
|
-
// 参数 JSON,「批准抓这个站」下次照样弹窗
|
|
696
671
|
describeTarget: (args) => String(args["url"] ?? ""),
|
|
697
672
|
execute: execWebFetch
|
|
698
673
|
}
|
|
@@ -717,14 +692,8 @@ function createWebSearchTool(provider, opts = {}) {
|
|
|
717
692
|
},
|
|
718
693
|
required: ["query"]
|
|
719
694
|
},
|
|
720
|
-
// `openWorldHint: true` 是这里最重要的一行:core 的 tool-executor 用它决定
|
|
721
|
-
// 要不要把输出包进 `<tool_output untrusted="true">`。搜索结果是攻击者
|
|
722
|
-
// **可以主动投放**的内容(SEO 投毒 + 提示注入),是不可信内容里风险最高的一种。
|
|
723
|
-
// `idempotentHint: false` —— 同一个词今天明天搜出来的东西不一样
|
|
724
695
|
annotations: { readOnlyHint: true, idempotentHint: false, openWorldHint: true },
|
|
725
696
|
operation: "network",
|
|
726
|
-
// 审批目标是查询词而不是参数 JSON:`terminal` 那边批准的是命令,
|
|
727
|
-
// 这边批准的就该是「搜这个词」
|
|
728
697
|
describeTarget: (args) => String(args["query"] ?? ""),
|
|
729
698
|
execute: createWebSearchExec({ provider, ...opts })
|
|
730
699
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@epoch-agent/plugin-web",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Web 抓取插件 — URL 安全校验 + HTML→Markdown + SSRF 防护",
|
|
6
6
|
"repository": {
|
|
@@ -22,7 +22,8 @@
|
|
|
22
22
|
"dist"
|
|
23
23
|
],
|
|
24
24
|
"dependencies": {
|
|
25
|
-
"@epoch-agent/
|
|
25
|
+
"@epoch-agent/infra": "0.3.1",
|
|
26
|
+
"@epoch-agent/protocol": "0.3.1"
|
|
26
27
|
},
|
|
27
28
|
"devDependencies": {
|
|
28
29
|
"@types/node": "^22.0.0",
|