@tachu/extensions 1.0.0-beta.1 → 1.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -66
- package/README.md +83 -708
- package/README_ZH.md +82 -696
- package/dist/backends/file.d.ts +6 -6
- package/dist/backends/file.d.ts.map +1 -1
- package/dist/backends/file.js +27 -11
- package/dist/backends/file.js.map +1 -1
- package/dist/backends/terminal.d.ts +6 -6
- package/dist/backends/terminal.d.ts.map +1 -1
- package/dist/backends/terminal.js +31 -7
- package/dist/backends/terminal.js.map +1 -1
- package/dist/backends/web.d.ts +6 -6
- package/dist/backends/web.d.ts.map +1 -1
- package/dist/backends/web.js +10 -8
- package/dist/backends/web.js.map +1 -1
- package/dist/common/net.d.ts +8 -0
- package/dist/common/net.d.ts.map +1 -1
- package/dist/common/net.js +39 -24
- package/dist/common/net.js.map +1 -1
- package/dist/common/path.d.ts +38 -5
- package/dist/common/path.d.ts.map +1 -1
- package/dist/common/path.js +55 -13
- package/dist/common/path.js.map +1 -1
- package/dist/common/process.js.map +1 -1
- package/dist/index.d.ts +7 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -3
- package/dist/index.js.map +1 -1
- package/dist/mcp/sse-adapter.d.ts +45 -33
- package/dist/mcp/sse-adapter.d.ts.map +1 -1
- package/dist/mcp/sse-adapter.js +59 -34
- package/dist/mcp/sse-adapter.js.map +1 -1
- package/dist/mcp/stdio-adapter.d.ts +45 -33
- package/dist/mcp/stdio-adapter.d.ts.map +1 -1
- package/dist/mcp/stdio-adapter.js +59 -34
- package/dist/mcp/stdio-adapter.js.map +1 -1
- package/dist/memory/fs-memory-system.d.ts +288 -0
- package/dist/memory/fs-memory-system.d.ts.map +1 -0
- package/dist/memory/fs-memory-system.js +482 -0
- package/dist/memory/fs-memory-system.js.map +1 -0
- package/dist/memory/index.d.ts +5 -0
- package/dist/memory/index.d.ts.map +1 -0
- package/dist/memory/index.js +5 -0
- package/dist/memory/index.js.map +1 -0
- package/dist/memory/projection-outbox.d.ts +69 -0
- package/dist/memory/projection-outbox.d.ts.map +1 -0
- package/dist/memory/projection-outbox.js +187 -0
- package/dist/memory/projection-outbox.js.map +1 -0
- package/dist/memory/projection-projector.d.ts +16 -0
- package/dist/memory/projection-projector.d.ts.map +1 -0
- package/dist/memory/projection-projector.js +56 -0
- package/dist/memory/projection-projector.js.map +1 -0
- package/dist/memory/projection-worker.d.ts +28 -0
- package/dist/memory/projection-worker.d.ts.map +1 -0
- package/dist/memory/projection-worker.js +84 -0
- package/dist/memory/projection-worker.js.map +1 -0
- package/dist/observability/jsonl-emitter.d.ts +25 -25
- package/dist/observability/jsonl-emitter.d.ts.map +1 -1
- package/dist/observability/jsonl-emitter.js +25 -25
- package/dist/observability/jsonl-emitter.js.map +1 -1
- package/dist/observability/otel-emitter.d.ts +23 -23
- package/dist/observability/otel-emitter.d.ts.map +1 -1
- package/dist/observability/otel-emitter.js +39 -30
- package/dist/observability/otel-emitter.js.map +1 -1
- package/dist/providers/anthropic.d.ts +51 -32
- package/dist/providers/anthropic.d.ts.map +1 -1
- package/dist/providers/anthropic.js +293 -58
- package/dist/providers/anthropic.js.map +1 -1
- package/dist/providers/gemini.d.ts +115 -0
- package/dist/providers/gemini.d.ts.map +1 -0
- package/dist/providers/gemini.js +901 -0
- package/dist/providers/gemini.js.map +1 -0
- package/dist/providers/index.d.ts +2 -0
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +2 -0
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/mock.d.ts +67 -24
- package/dist/providers/mock.d.ts.map +1 -1
- package/dist/providers/mock.js +122 -41
- package/dist/providers/mock.js.map +1 -1
- package/dist/providers/openai.d.ts +70 -35
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +330 -50
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/qwen.d.ts +146 -0
- package/dist/providers/qwen.d.ts.map +1 -0
- package/dist/providers/qwen.js +672 -0
- package/dist/providers/qwen.js.map +1 -0
- package/dist/safety/default-gate.d.ts +112 -0
- package/dist/safety/default-gate.d.ts.map +1 -0
- package/dist/safety/default-gate.js +188 -0
- package/dist/safety/default-gate.js.map +1 -0
- package/dist/safety/index.d.ts +2 -0
- package/dist/safety/index.d.ts.map +1 -0
- package/dist/safety/index.js +2 -0
- package/dist/safety/index.js.map +1 -0
- package/dist/tools/_shared/web-client.d.ts +18 -0
- package/dist/tools/_shared/web-client.d.ts.map +1 -0
- package/dist/tools/_shared/web-client.js +46 -0
- package/dist/tools/_shared/web-client.js.map +1 -0
- package/dist/tools/apply-patch/executor.d.ts.map +1 -1
- package/dist/tools/apply-patch/executor.js +54 -4
- package/dist/tools/apply-patch/executor.js.map +1 -1
- package/dist/tools/edit-file/executor.d.ts +32 -0
- package/dist/tools/edit-file/executor.d.ts.map +1 -0
- package/dist/tools/edit-file/executor.js +84 -0
- package/dist/tools/edit-file/executor.js.map +1 -0
- package/dist/tools/fetch-url/descriptor.md +6 -0
- package/dist/tools/fetch-url/executor.d.ts +8 -0
- package/dist/tools/fetch-url/executor.d.ts.map +1 -1
- package/dist/tools/fetch-url/executor.js +83 -2
- package/dist/tools/fetch-url/executor.js.map +1 -1
- package/dist/tools/git-blame/executor.d.ts +24 -0
- package/dist/tools/git-blame/executor.d.ts.map +1 -0
- package/dist/tools/git-blame/executor.js +76 -0
- package/dist/tools/git-blame/executor.js.map +1 -0
- package/dist/tools/git-branch/executor.d.ts +22 -0
- package/dist/tools/git-branch/executor.d.ts.map +1 -0
- package/dist/tools/git-branch/executor.js +81 -0
- package/dist/tools/git-branch/executor.js.map +1 -0
- package/dist/tools/git-diff/executor.d.ts +37 -0
- package/dist/tools/git-diff/executor.d.ts.map +1 -0
- package/dist/tools/git-diff/executor.js +156 -0
- package/dist/tools/git-diff/executor.js.map +1 -0
- package/dist/tools/git-log/executor.d.ts +31 -0
- package/dist/tools/git-log/executor.d.ts.map +1 -0
- package/dist/tools/git-log/executor.js +65 -0
- package/dist/tools/git-log/executor.js.map +1 -0
- package/dist/tools/git-show/executor.d.ts +22 -0
- package/dist/tools/git-show/executor.d.ts.map +1 -0
- package/dist/tools/git-show/executor.js +74 -0
- package/dist/tools/git-show/executor.js.map +1 -0
- package/dist/tools/git-status/executor.d.ts +25 -0
- package/dist/tools/git-status/executor.d.ts.map +1 -0
- package/dist/tools/git-status/executor.js +120 -0
- package/dist/tools/git-status/executor.js.map +1 -0
- package/dist/tools/glob/executor.d.ts +18 -0
- package/dist/tools/glob/executor.d.ts.map +1 -0
- package/dist/tools/glob/executor.js +47 -0
- package/dist/tools/glob/executor.js.map +1 -0
- package/dist/tools/index.d.ts +1 -1
- package/dist/tools/index.d.ts.map +1 -1
- package/dist/tools/index.js +459 -4
- package/dist/tools/index.js.map +1 -1
- package/dist/tools/list-dir/executor.d.ts.map +1 -1
- package/dist/tools/list-dir/executor.js +5 -3
- package/dist/tools/list-dir/executor.js.map +1 -1
- package/dist/tools/multi-edit/executor.d.ts +29 -0
- package/dist/tools/multi-edit/executor.d.ts.map +1 -0
- package/dist/tools/multi-edit/executor.js +37 -0
- package/dist/tools/multi-edit/executor.js.map +1 -0
- package/dist/tools/read-file/executor.d.ts +5 -0
- package/dist/tools/read-file/executor.d.ts.map +1 -1
- package/dist/tools/read-file/executor.js +49 -5
- package/dist/tools/read-file/executor.js.map +1 -1
- package/dist/tools/run-shell/executor.d.ts +12 -1
- package/dist/tools/run-shell/executor.d.ts.map +1 -1
- package/dist/tools/run-shell/executor.js +105 -9
- package/dist/tools/run-shell/executor.js.map +1 -1
- package/dist/tools/run-tests/executor.d.ts +28 -0
- package/dist/tools/run-tests/executor.d.ts.map +1 -0
- package/dist/tools/run-tests/executor.js +161 -0
- package/dist/tools/run-tests/executor.js.map +1 -0
- package/dist/tools/run-typecheck/executor.d.ts +25 -0
- package/dist/tools/run-typecheck/executor.d.ts.map +1 -0
- package/dist/tools/run-typecheck/executor.js +83 -0
- package/dist/tools/run-typecheck/executor.js.map +1 -0
- package/dist/tools/search-code/executor.d.ts.map +1 -1
- package/dist/tools/search-code/executor.js +60 -30
- package/dist/tools/search-code/executor.js.map +1 -1
- package/dist/tools/shared.d.ts +26 -0
- package/dist/tools/shared.d.ts.map +1 -1
- package/dist/tools/shared.js +15 -0
- package/dist/tools/shared.js.map +1 -1
- package/dist/tools/todo-read/executor.d.ts +20 -0
- package/dist/tools/todo-read/executor.d.ts.map +1 -0
- package/dist/tools/todo-read/executor.js +26 -0
- package/dist/tools/todo-read/executor.js.map +1 -0
- package/dist/tools/todo-write/executor.d.ts +21 -0
- package/dist/tools/todo-write/executor.d.ts.map +1 -0
- package/dist/tools/todo-write/executor.js +38 -0
- package/dist/tools/todo-write/executor.js.map +1 -0
- package/dist/tools/web-fetch/descriptor.md +198 -0
- package/dist/tools/web-fetch/errors.d.ts +32 -0
- package/dist/tools/web-fetch/errors.d.ts.map +1 -0
- package/dist/tools/web-fetch/errors.js +91 -0
- package/dist/tools/web-fetch/errors.js.map +1 -0
- package/dist/tools/web-fetch/executor.d.ts +10 -0
- package/dist/tools/web-fetch/executor.d.ts.map +1 -0
- package/dist/tools/web-fetch/executor.js +191 -0
- package/dist/tools/web-fetch/executor.js.map +1 -0
- package/dist/tools/web-fetch/index.d.ts +4 -0
- package/dist/tools/web-fetch/index.d.ts.map +1 -0
- package/dist/tools/web-fetch/index.js +3 -0
- package/dist/tools/web-fetch/index.js.map +1 -0
- package/dist/tools/web-fetch/types.d.ts +157 -0
- package/dist/tools/web-fetch/types.d.ts.map +1 -0
- package/dist/tools/web-fetch/types.js +7 -0
- package/dist/tools/web-fetch/types.js.map +1 -0
- package/dist/tools/web-search/descriptor.md +89 -0
- package/dist/tools/web-search/errors.d.ts +33 -0
- package/dist/tools/web-search/errors.d.ts.map +1 -0
- package/dist/tools/web-search/errors.js +45 -0
- package/dist/tools/web-search/errors.js.map +1 -0
- package/dist/tools/web-search/executor.d.ts +10 -0
- package/dist/tools/web-search/executor.d.ts.map +1 -0
- package/dist/tools/web-search/executor.js +185 -0
- package/dist/tools/web-search/executor.js.map +1 -0
- package/dist/tools/web-search/index.d.ts +4 -0
- package/dist/tools/web-search/index.d.ts.map +1 -0
- package/dist/tools/web-search/index.js +3 -0
- package/dist/tools/web-search/index.js.map +1 -0
- package/dist/tools/web-search/types.d.ts +86 -0
- package/dist/tools/web-search/types.d.ts.map +1 -0
- package/dist/tools/web-search/types.js +7 -0
- package/dist/tools/web-search/types.js.map +1 -0
- package/dist/tools/write-file/executor.js +3 -3
- package/dist/tools/write-file/executor.js.map +1 -1
- package/dist/transformers/document-to-text.d.ts +11 -11
- package/dist/transformers/document-to-text.d.ts.map +1 -1
- package/dist/transformers/document-to-text.js +11 -11
- package/dist/transformers/document-to-text.js.map +1 -1
- package/dist/transformers/image-to-text.d.ts +15 -15
- package/dist/transformers/image-to-text.d.ts.map +1 -1
- package/dist/transformers/image-to-text.js +22 -21
- package/dist/transformers/image-to-text.js.map +1 -1
- package/dist/vector/index.d.ts +4 -2
- package/dist/vector/index.d.ts.map +1 -1
- package/dist/vector/index.js +2 -2
- package/dist/vector/index.js.map +1 -1
- package/dist/vector/local-fs-index.d.ts +59 -0
- package/dist/vector/local-fs-index.d.ts.map +1 -0
- package/dist/vector/local-fs-index.js +216 -0
- package/dist/vector/local-fs-index.js.map +1 -0
- package/dist/vector/qdrant.d.ts +11 -52
- package/dist/vector/qdrant.d.ts.map +1 -1
- package/dist/vector/qdrant.js +39 -105
- package/dist/vector/qdrant.js.map +1 -1
- package/package.json +27 -6
- package/dist/vector/local-fs.d.ts +0 -76
- package/dist/vector/local-fs.d.ts.map +0 -1
- package/dist/vector/local-fs.js +0 -153
- package/dist/vector/local-fs.js.map +0 -1
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: tool
|
|
3
|
+
id: web-fetch
|
|
4
|
+
name: web-fetch
|
|
5
|
+
version: "0.1.0"
|
|
6
|
+
category: "web"
|
|
7
|
+
dangerous: true
|
|
8
|
+
description: |
|
|
9
|
+
通过 @tachu/web-fetch-server 远程渲染并结构化抓取 URL:服务端以 Bun.fetch(静态)或 Playwright(浏览器)拉取页面,经 Readability/Turndown 等管线输出标题、正文与可选链接/图片/JSON-LD;`renderMode` 为 auto 时可在静态不足时自动升级到浏览器重试一次。
|
|
10
|
+
sideEffect: readonly
|
|
11
|
+
idempotent: false
|
|
12
|
+
requiresApproval: true
|
|
13
|
+
timeout: 120000
|
|
14
|
+
inputSchema:
|
|
15
|
+
type: object
|
|
16
|
+
properties:
|
|
17
|
+
url:
|
|
18
|
+
type: string
|
|
19
|
+
description: 目标 http(s) URL(协议层唯一必填;将发往服务端并由 SSRF 规则校验)
|
|
20
|
+
renderMode:
|
|
21
|
+
type: string
|
|
22
|
+
enum: [static, browser, auto]
|
|
23
|
+
description: 渲染模式;默认 auto(先 static,不满足条件时浏览器重试一次)
|
|
24
|
+
waitFor:
|
|
25
|
+
description: |
|
|
26
|
+
仅 browser 模式:等待策略。可为 load / domcontentloaded / networkidle,
|
|
27
|
+
或 { "selector": "<css>" } 等待元素可见,或 { "timeMs": <number> } 固定等待。
|
|
28
|
+
oneOf:
|
|
29
|
+
- type: string
|
|
30
|
+
enum: [load, domcontentloaded, networkidle]
|
|
31
|
+
- type: object
|
|
32
|
+
required: [selector]
|
|
33
|
+
properties:
|
|
34
|
+
selector:
|
|
35
|
+
type: string
|
|
36
|
+
- type: object
|
|
37
|
+
required: [timeMs]
|
|
38
|
+
properties:
|
|
39
|
+
timeMs:
|
|
40
|
+
type: number
|
|
41
|
+
waitTimeoutMs:
|
|
42
|
+
type: number
|
|
43
|
+
description: 渲染等待超时(毫秒);默认 15000,硬上限 60000
|
|
44
|
+
scroll:
|
|
45
|
+
description: |
|
|
46
|
+
仅 browser:是否滚动以触发懒加载。false | true |
|
|
47
|
+
{ "steps": number, "delayMs": number }
|
|
48
|
+
oneOf:
|
|
49
|
+
- type: boolean
|
|
50
|
+
- type: object
|
|
51
|
+
required: [steps, delayMs]
|
|
52
|
+
properties:
|
|
53
|
+
steps:
|
|
54
|
+
type: number
|
|
55
|
+
delayMs:
|
|
56
|
+
type: number
|
|
57
|
+
userAgent:
|
|
58
|
+
type: string
|
|
59
|
+
description: 覆盖 User-Agent;请求中可省略或显式传 null 以使用服务端 UA 池
|
|
60
|
+
extraHeaders:
|
|
61
|
+
type: object
|
|
62
|
+
additionalProperties:
|
|
63
|
+
type: string
|
|
64
|
+
description: 额外 HTTP 头;静态模式直传;浏览器模式作为 route 注入
|
|
65
|
+
cookies:
|
|
66
|
+
type: array
|
|
67
|
+
description: 注入 Cookie 列表(仅 browser 模式生效;元素形状由服务端解析)
|
|
68
|
+
items:
|
|
69
|
+
type: object
|
|
70
|
+
additionalProperties: true
|
|
71
|
+
blockResources:
|
|
72
|
+
type: array
|
|
73
|
+
description: |
|
|
74
|
+
仅 browser:资源拦截类型。未传时服务端默认 ["image","font","media"];传 [] 表示不拦截。
|
|
75
|
+
items:
|
|
76
|
+
type: string
|
|
77
|
+
enum: [image, font, media, stylesheet, other]
|
|
78
|
+
stealth:
|
|
79
|
+
type: boolean
|
|
80
|
+
description: 可省略或传 null 继承服务级;传 true/false 为请求级覆盖
|
|
81
|
+
outputFormat:
|
|
82
|
+
type: string
|
|
83
|
+
enum: [markdown, text, html, structured]
|
|
84
|
+
description: 正文形态;markdown 为默认 GFM
|
|
85
|
+
includeLinks:
|
|
86
|
+
type: boolean
|
|
87
|
+
description: 是否在输出中包含 links[]
|
|
88
|
+
includeImages:
|
|
89
|
+
type: boolean
|
|
90
|
+
description: 是否在输出中包含 images[]
|
|
91
|
+
includeStructured:
|
|
92
|
+
type: boolean
|
|
93
|
+
description: 是否包含 JSON-LD structured 字段
|
|
94
|
+
maxBodyChars:
|
|
95
|
+
type: number
|
|
96
|
+
description: body 字符上限;默认 32768,硬上限 524288
|
|
97
|
+
traceId:
|
|
98
|
+
type: string
|
|
99
|
+
description: 可选追踪 ID;省略或 null 时由服务端生成
|
|
100
|
+
required: [url]
|
|
101
|
+
outputSchema:
|
|
102
|
+
type: object
|
|
103
|
+
description: 对应 POST /v1/extract 200 响应(字段随选项省略)
|
|
104
|
+
properties:
|
|
105
|
+
url:
|
|
106
|
+
type: string
|
|
107
|
+
finalUrl:
|
|
108
|
+
type: string
|
|
109
|
+
status:
|
|
110
|
+
type: number
|
|
111
|
+
renderedWith:
|
|
112
|
+
type: string
|
|
113
|
+
enum: [static, browser]
|
|
114
|
+
renderedAtMs:
|
|
115
|
+
type: number
|
|
116
|
+
title:
|
|
117
|
+
type: string
|
|
118
|
+
description:
|
|
119
|
+
type: string
|
|
120
|
+
siteName:
|
|
121
|
+
type: string
|
|
122
|
+
lang:
|
|
123
|
+
type: string
|
|
124
|
+
byline:
|
|
125
|
+
type: string
|
|
126
|
+
publishedTime:
|
|
127
|
+
type: string
|
|
128
|
+
body:
|
|
129
|
+
type: string
|
|
130
|
+
wordCount:
|
|
131
|
+
type: number
|
|
132
|
+
truncated:
|
|
133
|
+
type: boolean
|
|
134
|
+
links:
|
|
135
|
+
type: array
|
|
136
|
+
images:
|
|
137
|
+
type: array
|
|
138
|
+
structured:
|
|
139
|
+
type: object
|
|
140
|
+
additionalProperties: true
|
|
141
|
+
warnings:
|
|
142
|
+
type: array
|
|
143
|
+
items:
|
|
144
|
+
type: string
|
|
145
|
+
traceId:
|
|
146
|
+
type: string
|
|
147
|
+
execute: web-fetch
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## 常用字段说明
|
|
151
|
+
|
|
152
|
+
协议上仅 **`url`** 必填。实际调用中常与其他字段组合使用,例如:
|
|
153
|
+
|
|
154
|
+
- **`renderMode`**:静态站用 `static` 省资源,强 JS 页用 `browser` 或默认 `auto`
|
|
155
|
+
- **`outputFormat`**:需要 Markdown 摘要时用 `markdown`(默认),只要纯文本用 `text`
|
|
156
|
+
- **`waitTimeoutMs` / `waitFor`**:浏览器渲染时的超时与就绪条件
|
|
157
|
+
- **`maxBodyChars`**:控制返回给模型的正文长度,避免上下文爆炸
|
|
158
|
+
- **`includeLinks` / `includeImages` / `includeStructured`**:按需打开结构化附属字段
|
|
159
|
+
|
|
160
|
+
## 输出结构示例(200 成功,节选)
|
|
161
|
+
|
|
162
|
+
```json
|
|
163
|
+
{
|
|
164
|
+
"url": "https://example.com/article",
|
|
165
|
+
"finalUrl": "https://example.com/article",
|
|
166
|
+
"status": 200,
|
|
167
|
+
"renderedWith": "browser",
|
|
168
|
+
"renderedAtMs": 234,
|
|
169
|
+
"title": "示例文章",
|
|
170
|
+
"description": "页面 meta 描述",
|
|
171
|
+
"siteName": "Example",
|
|
172
|
+
"lang": "zh",
|
|
173
|
+
"byline": null,
|
|
174
|
+
"publishedTime": "2026-04-20T00:00:00.000Z",
|
|
175
|
+
"body": "# 示例文章\n\n正文 Markdown …",
|
|
176
|
+
"wordCount": 1234,
|
|
177
|
+
"truncated": false,
|
|
178
|
+
"links": [{ "text": "下一页", "href": "https://example.com/next" }],
|
|
179
|
+
"images": [],
|
|
180
|
+
"structured": null,
|
|
181
|
+
"warnings": [],
|
|
182
|
+
"traceId": "req_01HY..."
|
|
183
|
+
}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## 安全提示
|
|
187
|
+
|
|
188
|
+
1. **客户端须配置 Web Fetch Server 的 endpoint 与鉴权**:工具通过 HTTP 调用服务端;需正确设置服务地址与 `Authorization: Bearer <token>` 所用 token(与部署环境一致)。未配置 token 时服务端仅允许绑定本机回环,运维需按环境变量与网络策略部署。
|
|
189
|
+
2. **URL 会经服务端 SSRF 校验**:私网、元数据地址、黑名单域名等会被拒绝(如 `SSRF_BLOCKED` / `DOMAIN_NOT_ALLOWED`);请勿将本工具用于探测内网。
|
|
190
|
+
3. **robots.txt 与爬取礼仪**:是否遵守 robots 及额外策略由 **服务端实现与配置** 决定,本工具不在客户端重复裁决。
|
|
191
|
+
|
|
192
|
+
## 与 `fetch-url` 的区别
|
|
193
|
+
|
|
194
|
+
| 维度 | `web-fetch` | `fetch-url` |
|
|
195
|
+
| --- | --- | --- |
|
|
196
|
+
| 数据形态 | 结构化正文(标题、Markdown/文本/HTML、可选链接与 JSON-LD) | 原始响应体 + 简单 HTML 清洗 |
|
|
197
|
+
| 依赖 | 需要可用的 `@tachu/web-fetch-server` | 仅需受控直连,无需独立 fetch 服务 |
|
|
198
|
+
| 渲染 | 支持静态与浏览器渲染管线 | 单次 HTTP,无浏览器渲染 |
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { ErrorResponseBody, WebFetchErrorCode } from "./types";
|
|
2
|
+
/**
|
|
3
|
+
* 客户端在解析失败或网络层错误时使用的扩展码(契约 §3 行为要求;不在纯 HTTP 错误体中出现)。
|
|
4
|
+
*/
|
|
5
|
+
export type WebFetchClientOnlyErrorCode = "MALFORMED_RESPONSE" | "NETWORK_ERROR";
|
|
6
|
+
export type WebFetchClientErrorCode = WebFetchErrorCode | WebFetchClientOnlyErrorCode;
|
|
7
|
+
/**
|
|
8
|
+
* `web-fetch` / `web-search` 工具在客户端抛出的统一错误类型。
|
|
9
|
+
*
|
|
10
|
+
* @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §4
|
|
11
|
+
*/
|
|
12
|
+
export declare class WebFetchClientError extends Error {
|
|
13
|
+
readonly code: WebFetchClientErrorCode;
|
|
14
|
+
readonly userMessage: string;
|
|
15
|
+
readonly detail?: unknown | undefined;
|
|
16
|
+
readonly name = "WebFetchClientError";
|
|
17
|
+
constructor(code: WebFetchClientErrorCode, userMessage: string, detail?: unknown | undefined, options?: {
|
|
18
|
+
cause?: unknown;
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* 将服务端错误响应映射为 {@link WebFetchClientError}。
|
|
23
|
+
*
|
|
24
|
+
* @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §4.2
|
|
25
|
+
*/
|
|
26
|
+
export declare function mapServerErrorToClient(httpStatus: number, body: ErrorResponseBody | null, ctx: {
|
|
27
|
+
endpoint: string;
|
|
28
|
+
}): WebFetchClientError;
|
|
29
|
+
export declare function getMalformedResponseError(): WebFetchClientError;
|
|
30
|
+
export declare function getNetworkError(endpoint: string, originalError: unknown): WebFetchClientError;
|
|
31
|
+
export declare function getTimeoutError(timeoutMs: number): WebFetchClientError;
|
|
32
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../src/tools/web-fetch/errors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAEpE;;GAEG;AACH,MAAM,MAAM,2BAA2B,GAAG,oBAAoB,GAAG,eAAe,CAAC;AAEjF,MAAM,MAAM,uBAAuB,GAAG,iBAAiB,GAAG,2BAA2B,CAAC;AAEtF;;;;GAIG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;aAI1B,IAAI,EAAE,uBAAuB;aAC7B,WAAW,EAAE,MAAM;aACnB,MAAM,CAAC,EAAE,OAAO;IALlC,QAAQ,CAAC,IAAI,yBAAyB;gBAGpB,IAAI,EAAE,uBAAuB,EAC7B,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,OAAO,YAAA,EAChC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAIhC;AAkDD;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,iBAAiB,GAAG,IAAI,EAC9B,GAAG,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,GACxB,mBAAmB,CAUrB;AAED,wBAAgB,yBAAyB,IAAI,mBAAmB,CAM/D;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,GAAG,mBAAmB,CAO7F;AAED,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAMtE"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `web-fetch` / `web-search` 工具在客户端抛出的统一错误类型。
|
|
3
|
+
*
|
|
4
|
+
* @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §4
|
|
5
|
+
*/
|
|
6
|
+
export class WebFetchClientError extends Error {
|
|
7
|
+
code;
|
|
8
|
+
userMessage;
|
|
9
|
+
detail;
|
|
10
|
+
name = "WebFetchClientError";
|
|
11
|
+
constructor(code, userMessage, detail, options) {
|
|
12
|
+
super(userMessage, options);
|
|
13
|
+
this.code = code;
|
|
14
|
+
this.userMessage = userMessage;
|
|
15
|
+
this.detail = detail;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/** @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §5 */
|
|
19
|
+
const HTTP_STATUS_TO_CODE = {
|
|
20
|
+
400: "INVALID_REQUEST",
|
|
21
|
+
401: "UNAUTHORIZED",
|
|
22
|
+
403: "FORBIDDEN",
|
|
23
|
+
404: "INVALID_REQUEST",
|
|
24
|
+
408: "REQUEST_TIMEOUT",
|
|
25
|
+
413: "REQUEST_TOO_LARGE",
|
|
26
|
+
422: "RENDER_FAILED",
|
|
27
|
+
429: "RATE_LIMITED",
|
|
28
|
+
500: "INTERNAL_ERROR",
|
|
29
|
+
502: "UPSTREAM_ERROR",
|
|
30
|
+
503: "BROWSER_POOL_EXHAUSTED",
|
|
31
|
+
504: "PROVIDER_TIMEOUT",
|
|
32
|
+
};
|
|
33
|
+
function inferCodeFromStatus(httpStatus) {
|
|
34
|
+
return HTTP_STATUS_TO_CODE[httpStatus] ?? "INTERNAL_ERROR";
|
|
35
|
+
}
|
|
36
|
+
const USER_MESSAGE_ZH = {
|
|
37
|
+
INVALID_REQUEST: "请求格式无效。请检查参数后重试。",
|
|
38
|
+
INVALID_URL: "URL 无效。请使用合法的 http/https 地址。",
|
|
39
|
+
UNAUTHORIZED: "鉴权失败。请检查 Web Fetch 服务的访问令牌配置。",
|
|
40
|
+
FORBIDDEN: "访问被拒绝。请确认令牌与权限配置是否正确。",
|
|
41
|
+
SSRF_BLOCKED: "出于安全策略,无法访问该地址。请仅使用允许的公网 URL。",
|
|
42
|
+
DOMAIN_NOT_ALLOWED: "域名不在允许列表中。请调整白名单或更换 URL。",
|
|
43
|
+
REQUEST_TIMEOUT: "请求处理超时。请稍后重试,或尝试缩短等待时间。",
|
|
44
|
+
REQUEST_TOO_LARGE: "请求体过大。请减小提交内容后重试。",
|
|
45
|
+
RESPONSE_TOO_LARGE: "目标页面响应过大。请尝试降低正文长度上限或更换页面。",
|
|
46
|
+
RENDER_FAILED: "页面渲染失败。可尝试更换渲染模式或稍后重试。",
|
|
47
|
+
RATE_LIMITED: "触发限流。请稍后再试或降低请求频率。",
|
|
48
|
+
INTERNAL_ERROR: "服务端出现内部错误。请稍后重试或联系管理员。",
|
|
49
|
+
UPSTREAM_ERROR: "上游站点返回异常。请稍后重试。",
|
|
50
|
+
BROWSER_POOL_EXHAUSTED: "浏览器资源繁忙。请降低并发或稍后重试。",
|
|
51
|
+
BROWSER_CRASHED: "浏览器进程异常。请立即重试,通常可以恢复。",
|
|
52
|
+
PROVIDER_NOT_CONFIGURED: "搜索提供方未配置。请检查搜索相关环境变量。",
|
|
53
|
+
PROVIDER_UPSTREAM_ERROR: "搜索提供方返回异常。请稍后重试。",
|
|
54
|
+
PROVIDER_TIMEOUT: "搜索提供方调用超时。请稍后重试。",
|
|
55
|
+
TIMEOUT_WEB_FETCH: "网页抓取超时。请缩短正文长度上限或更换 URL 后重试。",
|
|
56
|
+
TIMEOUT_WEB_SEARCH: "搜索超时。请缩短查询或减少抓取条数后重试。",
|
|
57
|
+
WEB_FETCH_SERVER_UNREACHABLE: "渲染服务不可达。请检查网络或服务端状态。",
|
|
58
|
+
WEB_FETCH_ENDPOINT_NOT_CONFIGURED: "未能连接到 Web Fetch 服务。请先启动渲染服务,或通过 TACHU_WEB_FETCH_ENDPOINT 配置远端地址。",
|
|
59
|
+
MALFORMED_RESPONSE: "服务返回了无法解析的响应。请检查服务端版本或稍后重试。",
|
|
60
|
+
NETWORK_ERROR: "网络请求失败。请检查网络连接与服务是否可用。",
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* 将服务端错误响应映射为 {@link WebFetchClientError}。
|
|
64
|
+
*
|
|
65
|
+
* @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §4.2
|
|
66
|
+
*/
|
|
67
|
+
export function mapServerErrorToClient(httpStatus, body, ctx) {
|
|
68
|
+
const code = body?.error.code ?? inferCodeFromStatus(httpStatus);
|
|
69
|
+
const userMessage = USER_MESSAGE_ZH[code] ?? body?.error.message ?? USER_MESSAGE_ZH.INTERNAL_ERROR;
|
|
70
|
+
const detail = {
|
|
71
|
+
...(body?.error.detail ?? {}),
|
|
72
|
+
httpStatus,
|
|
73
|
+
endpoint: ctx.endpoint,
|
|
74
|
+
requestId: body?.error.requestId,
|
|
75
|
+
};
|
|
76
|
+
return new WebFetchClientError(code, userMessage, detail);
|
|
77
|
+
}
|
|
78
|
+
export function getMalformedResponseError() {
|
|
79
|
+
return new WebFetchClientError("MALFORMED_RESPONSE", USER_MESSAGE_ZH.MALFORMED_RESPONSE, undefined);
|
|
80
|
+
}
|
|
81
|
+
export function getNetworkError(endpoint, originalError) {
|
|
82
|
+
const original = originalError instanceof Error ? originalError.message : String(originalError);
|
|
83
|
+
return new WebFetchClientError("NETWORK_ERROR", USER_MESSAGE_ZH.NETWORK_ERROR, {
|
|
84
|
+
endpoint,
|
|
85
|
+
originalError: original,
|
|
86
|
+
}, { cause: originalError instanceof Error ? originalError : undefined });
|
|
87
|
+
}
|
|
88
|
+
export function getTimeoutError(timeoutMs) {
|
|
89
|
+
return new WebFetchClientError("REQUEST_TIMEOUT", USER_MESSAGE_ZH.REQUEST_TIMEOUT, { timeoutMs, phase: "fetch" });
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=errors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.js","sourceRoot":"","sources":["../../../src/tools/web-fetch/errors.ts"],"names":[],"mappings":"AASA;;;;GAIG;AACH,MAAM,OAAO,mBAAoB,SAAQ,KAAK;IAI1B;IACA;IACA;IALT,IAAI,GAAG,qBAAqB,CAAC;IAEtC,YACkB,IAA6B,EAC7B,WAAmB,EACnB,MAAgB,EAChC,OAA6B;QAE7B,KAAK,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QALZ,SAAI,GAAJ,IAAI,CAAyB;QAC7B,gBAAW,GAAX,WAAW,CAAQ;QACnB,WAAM,GAAN,MAAM,CAAU;IAIlC,CAAC;CACF;AAED,sEAAsE;AACtE,MAAM,mBAAmB,GAAsC;IAC7D,GAAG,EAAE,iBAAiB;IACtB,GAAG,EAAE,cAAc;IACnB,GAAG,EAAE,WAAW;IAChB,GAAG,EAAE,iBAAiB;IACtB,GAAG,EAAE,iBAAiB;IACtB,GAAG,EAAE,mBAAmB;IACxB,GAAG,EAAE,eAAe;IACpB,GAAG,EAAE,cAAc;IACnB,GAAG,EAAE,gBAAgB;IACrB,GAAG,EAAE,gBAAgB;IACrB,GAAG,EAAE,wBAAwB;IAC7B,GAAG,EAAE,kBAAkB;CACxB,CAAC;AAEF,SAAS,mBAAmB,CAAC,UAAkB;IAC7C,OAAO,mBAAmB,CAAC,UAAU,CAAC,IAAI,gBAAgB,CAAC;AAC7D,CAAC;AAED,MAAM,eAAe,GAA4C;IAC/D,eAAe,EAAE,kBAAkB;IACnC,WAAW,EAAE,8BAA8B;IAC3C,YAAY,EAAE,+BAA+B;IAC7C,SAAS,EAAE,uBAAuB;IAClC,YAAY,EAAE,+BAA+B;IAC7C,kBAAkB,EAAE,0BAA0B;IAC9C,eAAe,EAAE,yBAAyB;IAC1C,iBAAiB,EAAE,mBAAmB;IACtC,kBAAkB,EAAE,4BAA4B;IAChD,aAAa,EAAE,wBAAwB;IACvC,YAAY,EAAE,oBAAoB;IAClC,cAAc,EAAE,wBAAwB;IACxC,cAAc,EAAE,iBAAiB;IACjC,sBAAsB,EAAE,qBAAqB;IAC7C,eAAe,EAAE,uBAAuB;IACxC,uBAAuB,EAAE,uBAAuB;IAChD,uBAAuB,EAAE,kBAAkB;IAC3C,gBAAgB,EAAE,kBAAkB;IACpC,iBAAiB,EAAE,8BAA8B;IACjD,kBAAkB,EAAE,uBAAuB;IAC3C,4BAA4B,EAAE,sBAAsB;IACpD,iCAAiC,EAC/B,kEAAkE;IACpE,kBAAkB,EAAE,6BAA6B;IACjD,aAAa,EAAE,wBAAwB;CACxC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,UAAkB,EAClB,IAA8B,EAC9B,GAAyB;IAEzB,MAAM,IAAI,GAAG,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,mBAAmB,CAAC,UAAU,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe,CAAC,cAAc,CAAC;IACnG,MAAM,MAAM,GAAG;QACb,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;QAC7B,UAAU;QACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,SAAS;KACjC,CAAC;IACF,OAAO,IAAI,mBAAmB,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,OAAO,IAAI,mBAAmB,CAC5B,oBAAoB,EACpB,eAAe,CAAC,kBAAkB,EAClC,SAAS,CACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,aAAsB;IACtE,MAAM,QAAQ,GACZ,aAAa,YAAY,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IACjF,OAAO,IAAI,mBAAmB,CAAC,eAAe,EAAE,eAAe,CAAC,aAAa,EAAE;QAC7E,QAAQ;QACR,aAAa,EAAE,QAAQ;KACxB,EAAE,EAAE,KAAK,EAAE,aAAa,YAAY,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,SAAiB;IAC/C,OAAO,IAAI,mBAAmB,CAC5B,iBAAiB,EACjB,eAAe,CAAC,eAAe,EAC/B,EAAE,SAAS,EAAE,KAAK,EAAE,OAAgB,EAAE,CACvC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { WebFetchToolInput, WebFetchToolOutput } from "./types";
|
|
2
|
+
import type { ToolExecutionContext } from "../shared";
|
|
3
|
+
/**
|
|
4
|
+
* 调用 Web Fetch Server `/v1/extract`,返回正文与元数据。
|
|
5
|
+
*
|
|
6
|
+
* @see packages/web-fetch-server/src/0003c-web-fetch-config.md §5.1
|
|
7
|
+
* @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §4
|
|
8
|
+
*/
|
|
9
|
+
export declare function executeWebFetch(input: WebFetchToolInput, ctx: ToolExecutionContext): Promise<WebFetchToolOutput>;
|
|
10
|
+
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/tools/web-fetch/executor.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAsD,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AACzH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AAuHtD;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,iBAAiB,EACxB,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,kBAAkB,CAAC,CAmD7B"}
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { getMalformedResponseError, getNetworkError, getTimeoutError, mapServerErrorToClient, } from "./errors";
|
|
2
|
+
import { assertNotAborted } from "../shared";
|
|
3
|
+
let warnedMissingEndpoint = false;
|
|
4
|
+
function getEndpointBase() {
|
|
5
|
+
const raw = process.env.TACHU_WEB_FETCH_ENDPOINT?.trim();
|
|
6
|
+
if (!raw) {
|
|
7
|
+
if (!warnedMissingEndpoint) {
|
|
8
|
+
warnedMissingEndpoint = true;
|
|
9
|
+
console.warn("[@tachu/extensions/web-fetch] TACHU_WEB_FETCH_ENDPOINT is not set; using default http://127.0.0.1:8787");
|
|
10
|
+
}
|
|
11
|
+
return "http://127.0.0.1:8787";
|
|
12
|
+
}
|
|
13
|
+
return raw.replace(/\/$/, "");
|
|
14
|
+
}
|
|
15
|
+
function readClientTimeoutMs(input) {
|
|
16
|
+
if (input.timeoutMs != null)
|
|
17
|
+
return input.timeoutMs;
|
|
18
|
+
const raw = process.env.TACHU_WEB_FETCH_TIMEOUT_MS?.trim();
|
|
19
|
+
if (!raw)
|
|
20
|
+
return 70000;
|
|
21
|
+
const n = Number(raw);
|
|
22
|
+
return Number.isFinite(n) ? n : 70000;
|
|
23
|
+
}
|
|
24
|
+
function toExtractRequest(input) {
|
|
25
|
+
const { timeoutMs: _timeoutMs, ...rest } = input;
|
|
26
|
+
return {
|
|
27
|
+
...rest,
|
|
28
|
+
traceId: null,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function isRecord(value) {
|
|
32
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
33
|
+
}
|
|
34
|
+
function isExtractResponse(value) {
|
|
35
|
+
if (!isRecord(value))
|
|
36
|
+
return false;
|
|
37
|
+
const keys = [
|
|
38
|
+
"url",
|
|
39
|
+
"finalUrl",
|
|
40
|
+
"status",
|
|
41
|
+
"renderedWith",
|
|
42
|
+
"renderedAtMs",
|
|
43
|
+
"body",
|
|
44
|
+
"wordCount",
|
|
45
|
+
"truncated",
|
|
46
|
+
"warnings",
|
|
47
|
+
"traceId",
|
|
48
|
+
];
|
|
49
|
+
for (const k of keys) {
|
|
50
|
+
if (!(k in value))
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
if (typeof value.url !== "string")
|
|
54
|
+
return false;
|
|
55
|
+
if (typeof value.finalUrl !== "string")
|
|
56
|
+
return false;
|
|
57
|
+
if (typeof value.status !== "number")
|
|
58
|
+
return false;
|
|
59
|
+
if (value.renderedWith !== "static" && value.renderedWith !== "browser")
|
|
60
|
+
return false;
|
|
61
|
+
if (typeof value.renderedAtMs !== "number")
|
|
62
|
+
return false;
|
|
63
|
+
if (typeof value.body !== "string")
|
|
64
|
+
return false;
|
|
65
|
+
if (typeof value.wordCount !== "number")
|
|
66
|
+
return false;
|
|
67
|
+
if (typeof value.truncated !== "boolean")
|
|
68
|
+
return false;
|
|
69
|
+
if (!Array.isArray(value.warnings))
|
|
70
|
+
return false;
|
|
71
|
+
if (typeof value.traceId !== "string")
|
|
72
|
+
return false;
|
|
73
|
+
return true;
|
|
74
|
+
}
|
|
75
|
+
function toToolOutput(res) {
|
|
76
|
+
const out = {
|
|
77
|
+
url: res.url,
|
|
78
|
+
finalUrl: res.finalUrl,
|
|
79
|
+
status: res.status,
|
|
80
|
+
renderedWith: res.renderedWith,
|
|
81
|
+
body: res.body,
|
|
82
|
+
wordCount: res.wordCount,
|
|
83
|
+
truncated: res.truncated,
|
|
84
|
+
warnings: res.warnings,
|
|
85
|
+
};
|
|
86
|
+
if (res.title !== undefined)
|
|
87
|
+
out.title = res.title;
|
|
88
|
+
if (res.description !== undefined)
|
|
89
|
+
out.description = res.description;
|
|
90
|
+
if (res.siteName !== undefined)
|
|
91
|
+
out.siteName = res.siteName;
|
|
92
|
+
if (res.lang !== undefined)
|
|
93
|
+
out.lang = res.lang;
|
|
94
|
+
if (res.byline !== undefined)
|
|
95
|
+
out.byline = res.byline;
|
|
96
|
+
if (res.publishedTime !== undefined)
|
|
97
|
+
out.publishedTime = res.publishedTime;
|
|
98
|
+
if (res.links !== undefined) {
|
|
99
|
+
out.links = res.links.map((l) => ({ text: l.text, href: l.href }));
|
|
100
|
+
}
|
|
101
|
+
if (res.images !== undefined) {
|
|
102
|
+
out.images = res.images.map((im) => ({ alt: im.alt, src: im.src }));
|
|
103
|
+
}
|
|
104
|
+
if (res.structured !== undefined)
|
|
105
|
+
out.structured = res.structured;
|
|
106
|
+
return out;
|
|
107
|
+
}
|
|
108
|
+
function tryParseErrorBody(text) {
|
|
109
|
+
try {
|
|
110
|
+
const parsed = JSON.parse(text);
|
|
111
|
+
if (!isRecord(parsed))
|
|
112
|
+
return null;
|
|
113
|
+
const err = parsed.error;
|
|
114
|
+
if (!isRecord(err))
|
|
115
|
+
return null;
|
|
116
|
+
if (typeof err.code !== "string")
|
|
117
|
+
return null;
|
|
118
|
+
if (typeof err.message !== "string")
|
|
119
|
+
return null;
|
|
120
|
+
if (typeof err.requestId !== "string")
|
|
121
|
+
return null;
|
|
122
|
+
const body = {
|
|
123
|
+
error: {
|
|
124
|
+
code: err.code,
|
|
125
|
+
message: err.message,
|
|
126
|
+
requestId: err.requestId,
|
|
127
|
+
...(isRecord(err.detail) ? { detail: err.detail } : {}),
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
return body;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* 调用 Web Fetch Server `/v1/extract`,返回正文与元数据。
|
|
138
|
+
*
|
|
139
|
+
* @see packages/web-fetch-server/src/0003c-web-fetch-config.md §5.1
|
|
140
|
+
* @see packages/web-fetch-server/src/0003d-web-fetch-errors.md §4
|
|
141
|
+
*/
|
|
142
|
+
export async function executeWebFetch(input, ctx) {
|
|
143
|
+
assertNotAborted(ctx.abortSignal);
|
|
144
|
+
const endpoint = getEndpointBase();
|
|
145
|
+
const url = `${endpoint}/v1/extract`;
|
|
146
|
+
const timeoutMs = readClientTimeoutMs(input);
|
|
147
|
+
const signal = AbortSignal.any([AbortSignal.timeout(timeoutMs), ctx.abortSignal]);
|
|
148
|
+
const headers = {
|
|
149
|
+
"Content-Type": "application/json; charset=utf-8",
|
|
150
|
+
};
|
|
151
|
+
const token = process.env.TACHU_WEB_FETCH_TOKEN?.trim();
|
|
152
|
+
if (token) {
|
|
153
|
+
headers.Authorization = `Bearer ${token}`;
|
|
154
|
+
}
|
|
155
|
+
let response;
|
|
156
|
+
try {
|
|
157
|
+
response = await fetch(url, {
|
|
158
|
+
method: "POST",
|
|
159
|
+
headers,
|
|
160
|
+
body: JSON.stringify(toExtractRequest(input)),
|
|
161
|
+
signal,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
catch (e) {
|
|
165
|
+
if (e instanceof Error && e.name === "AbortError") {
|
|
166
|
+
if (ctx.abortSignal.aborted) {
|
|
167
|
+
throw ctx.abortSignal.reason ?? e;
|
|
168
|
+
}
|
|
169
|
+
throw getTimeoutError(timeoutMs);
|
|
170
|
+
}
|
|
171
|
+
throw getNetworkError(endpoint, e);
|
|
172
|
+
}
|
|
173
|
+
if (!response.ok) {
|
|
174
|
+
const text = await response.text();
|
|
175
|
+
const parsed = tryParseErrorBody(text);
|
|
176
|
+
throw mapServerErrorToClient(response.status, parsed, { endpoint });
|
|
177
|
+
}
|
|
178
|
+
const text = await response.text();
|
|
179
|
+
let parsedBody;
|
|
180
|
+
try {
|
|
181
|
+
parsedBody = JSON.parse(text);
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
throw getMalformedResponseError();
|
|
185
|
+
}
|
|
186
|
+
if (!isExtractResponse(parsedBody)) {
|
|
187
|
+
throw getMalformedResponseError();
|
|
188
|
+
}
|
|
189
|
+
return toToolOutput(parsedBody);
|
|
190
|
+
}
|
|
191
|
+
//# sourceMappingURL=executor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"executor.js","sourceRoot":"","sources":["../../../src/tools/web-fetch/executor.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,yBAAyB,EACzB,eAAe,EACf,eAAe,EACf,sBAAsB,GACvB,MAAM,UAAU,CAAC;AAGlB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,IAAI,qBAAqB,GAAG,KAAK,CAAC;AAElC,SAAS,eAAe;IACtB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,IAAI,EAAE,CAAC;IACzD,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC3B,qBAAqB,GAAG,IAAI,CAAC;YAC7B,OAAO,CAAC,IAAI,CACV,wGAAwG,CACzG,CAAC;QACJ,CAAC;QACD,OAAO,uBAAuB,CAAC;IACjC,CAAC;IACD,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAwB;IACnD,IAAI,KAAK,CAAC,SAAS,IAAI,IAAI;QAAE,OAAO,KAAK,CAAC,SAAS,CAAC;IACpD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,EAAE,IAAI,EAAE,CAAC;IAC3D,IAAI,CAAC,GAAG;QAAE,OAAO,KAAK,CAAC;IACvB,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACxC,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAwB;IAChD,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;IACjD,OAAO;QACL,GAAG,IAAI;QACP,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,IAAI,GAAG;QACX,KAAK;QACL,UAAU;QACV,QAAQ;QACR,cAAc;QACd,cAAc;QACd,MAAM;QACN,WAAW;QACX,WAAW;QACX,UAAU;QACV,SAAS;KACD,CAAC;IACX,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;IAClC,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAChD,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACnD,IAAI,KAAK,CAAC,YAAY,KAAK,QAAQ,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACtF,IAAI,OAAO,KAAK,CAAC,YAAY,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACzD,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACjD,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,IAAI,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACvD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IACjD,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACpD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CAAC,GAAoB;IACxC,MAAM,GAAG,GAAuB;QAC9B,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,QAAQ,EAAE,GAAG,CAAC,QAAQ;KACvB,CAAC;IACF,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS;QAAE,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC;IACnD,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS;QAAE,GAAG,CAAC,WAAW,GAAG,GAAG,CAAC,WAAW,CAAC;IACrE,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS;QAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC5D,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS;QAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IAChD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS;QAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;IACtD,IAAI,GAAG,CAAC,aAAa,KAAK,SAAS;QAAE,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC;IAC3E,IAAI,GAAG,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5B,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7B,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;QAAE,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,UAAU,CAAC;IAClE,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;QAChC,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC9C,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACjD,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACnD,MAAM,IAAI,GAAsB;YAC9B,KAAK,EAAE;gBACL,IAAI,EAAE,GAAG,CAAC,IAA0C;gBACpD,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAiC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnF;SACF,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAwB,EACxB,GAAyB;IAEzB,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IAElC,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,GAAG,QAAQ,aAAa,CAAC;IACrC,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IAElF,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,iCAAiC;KAClD,CAAC;IACF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,EAAE,CAAC;IACxD,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,aAAa,GAAG,UAAU,KAAK,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAClD,IAAI,GAAG,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,GAAG,CAAC,WAAW,CAAC,MAAM,IAAI,CAAC,CAAC;YACpC,CAAC;YACD,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnC,CAAC;QACD,MAAM,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,sBAAsB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,IAAI,UAAmB,CAAC;IACxB,IAAI,CAAC;QACH,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,yBAAyB,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;QACnC,MAAM,yBAAyB,EAAE,CAAC;IACpC,CAAC;IACD,OAAO,YAAY,CAAC,UAAU,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { executeWebFetch } from "./executor";
|
|
2
|
+
export { WebFetchClientError, type WebFetchClientErrorCode, type WebFetchClientOnlyErrorCode, } from "./errors";
|
|
3
|
+
export type { CookieInit, ErrorResponseBody, ExtractRequest, ExtractResponse, ImageRef, LinkRef, OutputFormat, RenderMode, ResourceType, ScrollStrategy, WaitStrategy, WebFetchErrorCode, WebFetchToolInput, WebFetchToolOutput, } from "./types";
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/tools/web-fetch/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EACL,mBAAmB,EACnB,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,GACjC,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,cAAc,EACd,eAAe,EACf,QAAQ,EACR,OAAO,EACP,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,SAAS,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/tools/web-fetch/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EACL,mBAAmB,GAGpB,MAAM,UAAU,CAAC"}
|