@raolin2025/claude-code-node 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -3
- package/package.json +1 -1
- package/src/__tests__/web-fetch-guard.test.js +193 -0
- package/src/core/cli.js +79 -2
- package/src/core/config.js +10 -0
- package/src/core/context-window.js +262 -0
- package/src/core/index.js +8 -0
- package/src/core/query-engine.js +15 -0
- package/src/core/token-budget.js +14 -0
- package/src/security/fetch-guard.js +311 -0
- package/src/security/redact.js +76 -0
- package/src/tools/web-fetch-providers.js +140 -0
- package/src/tools/web-fetch.js +114 -60
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 上下文窗口感知 (Context Window) — 自动探测 + 手动指定
|
|
3
|
+
*
|
|
4
|
+
* 解决的问题:TokenBudget.maxTokens 之前是一个"静态配置"(默认 100 万,
|
|
5
|
+
* CLI 兜底 20 万),并不知道所用模型真实上下文窗口。若预算 > 模型真实窗口,
|
|
6
|
+
* 自动压缩永远不会触发,上下文一路涨到模型报错。
|
|
7
|
+
*
|
|
8
|
+
* 本模块提供三层"窗口来源",优先级:
|
|
9
|
+
* 1. manual — 用户通过 /window 手动指定(持久化到 config)
|
|
10
|
+
* 2. probe — 从 API /models 探测到的窗口(本次进程生效,不落盘)
|
|
11
|
+
* 3. table — 内置常见模型上下文表(无法探测时的静态兜底)
|
|
12
|
+
* 4. fallback— 安全默认值(如 64K)
|
|
13
|
+
*
|
|
14
|
+
* 探测结果默认不落盘,手动指定才落盘(方案 A)。
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
// ---- 内置常见模型上下文表(token)----
|
|
18
|
+
// 缺失的厂商/模型会回落到 fallback;不精确属预期,可用 /window 手动纠正。
|
|
19
|
+
export const MODEL_CONTEXT_TABLE = {
|
|
20
|
+
// DeepSeek
|
|
21
|
+
'deepseek-chat': 128_000,
|
|
22
|
+
'deepseek-reasoner': 128_000,
|
|
23
|
+
'deepseek-coder': 128_000,
|
|
24
|
+
'deepseek-v3': 128_000,
|
|
25
|
+
'deepseek-v2': 128_000,
|
|
26
|
+
'deepseek-v2.5': 128_000,
|
|
27
|
+
'deepseek-v4': 128_000,
|
|
28
|
+
// OpenAI
|
|
29
|
+
'gpt-4o': 128_000,
|
|
30
|
+
'gpt-4o-mini': 128_000,
|
|
31
|
+
'gpt-4-turbo': 128_000,
|
|
32
|
+
'gpt-4': 32_768,
|
|
33
|
+
'gpt-3.5-turbo': 16_384,
|
|
34
|
+
'o1': 200_000,
|
|
35
|
+
'o1-mini': 128_000,
|
|
36
|
+
'o3-mini': 200_000,
|
|
37
|
+
// Anthropic (经兼容网关时)
|
|
38
|
+
'claude-3-5-sonnet': 200_000,
|
|
39
|
+
'claude-3-5-haiku': 200_000,
|
|
40
|
+
'claude-sonnet-4': 200_000,
|
|
41
|
+
'claude-haiku-4': 200_000,
|
|
42
|
+
// Qwen (通义千问)
|
|
43
|
+
'qwen-max': 32_768,
|
|
44
|
+
'qwen-plus': 131_072,
|
|
45
|
+
'qwen-turbo': 1_000_000,
|
|
46
|
+
'qwen2.5-72b-instruct': 131_072,
|
|
47
|
+
'qwen2.5-32b-instruct': 131_072,
|
|
48
|
+
'qwen2.5-7b-instruct': 131_072,
|
|
49
|
+
'qwen2.5-14b-instruct': 131_072,
|
|
50
|
+
'qwen2.5-coder-32b-instruct': 131_072,
|
|
51
|
+
// GLM (智谱)
|
|
52
|
+
'glm-4': 128_000,
|
|
53
|
+
'glm-4-plus': 128_000,
|
|
54
|
+
'glm-4-air': 128_000,
|
|
55
|
+
'glm-4-flash': 128_000,
|
|
56
|
+
'glm-4v': 8_192,
|
|
57
|
+
'glm-4.5': 128_000,
|
|
58
|
+
'glm-4.5-air': 128_000,
|
|
59
|
+
// Moonshot (Kimi)
|
|
60
|
+
'moonshot-v1-8k': 8_192,
|
|
61
|
+
'moonshot-v1-32k': 32_768,
|
|
62
|
+
'moonshot-v1-128k': 131_072,
|
|
63
|
+
'moonshot-v1-auto': 131_072,
|
|
64
|
+
'kimi-k2': 128_000,
|
|
65
|
+
// 本地/开源 (Ollama / vLLM 常见)
|
|
66
|
+
'llama3.1': 131_072,
|
|
67
|
+
'llama3': 8_192,
|
|
68
|
+
'llama2': 4_096,
|
|
69
|
+
'mistral': 32_768,
|
|
70
|
+
'mixtral': 32_768,
|
|
71
|
+
'gemma2': 8_192,
|
|
72
|
+
'codellama': 16_384,
|
|
73
|
+
'qwen2.5': 131_072,
|
|
74
|
+
'yi-34b': 4_096,
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// 安全兜底默认值 — 探测不到、表里也没有时使用
|
|
78
|
+
export const FALLBACK_CONTEXT_WINDOW = 64_000
|
|
79
|
+
|
|
80
|
+
// ---- 窗口来源标签 ----
|
|
81
|
+
export const WINDOW_SOURCE = {
|
|
82
|
+
MANUAL: 'manual',
|
|
83
|
+
PROBE: 'probe',
|
|
84
|
+
TABLE: 'table',
|
|
85
|
+
FALLBACK: 'fallback',
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const SOURCE_LABEL = {
|
|
89
|
+
[WINDOW_SOURCE.MANUAL]: '手动指定',
|
|
90
|
+
[WINDOW_SOURCE.PROBE]: 'API 探测',
|
|
91
|
+
[WINDOW_SOURCE.TABLE]: '内置模型表',
|
|
92
|
+
[WINDOW_SOURCE.FALLBACK]: '安全兜底',
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* 从 /models 响应中尝试提取单个模型的上下文窗口
|
|
97
|
+
*
|
|
98
|
+
* 不同服务字段名不一,尽量覆盖常见命名:
|
|
99
|
+
* - OpenAI 兼容 vLLM: context_length / max_model_len / max_context_length
|
|
100
|
+
* - Ollama: (来自 /api/show 的 model_info.llama.context_length)
|
|
101
|
+
*
|
|
102
|
+
* @param {object} modelObj — /models 返回数组中的单个模型对象
|
|
103
|
+
* @returns {number|null} 窗口 token 数;未知返回 null
|
|
104
|
+
*/
|
|
105
|
+
export function extractContextFromModelObj(modelObj) {
|
|
106
|
+
if (!modelObj || typeof modelObj !== 'object') return null
|
|
107
|
+
const keys = [
|
|
108
|
+
'context_length',
|
|
109
|
+
'context_window',
|
|
110
|
+
'max_context_length',
|
|
111
|
+
'max_model_len',
|
|
112
|
+
'max_sequence_length',
|
|
113
|
+
'model_max_length',
|
|
114
|
+
'n_ctx',
|
|
115
|
+
'contextWindow',
|
|
116
|
+
'contextSize',
|
|
117
|
+
'maxContext',
|
|
118
|
+
'maxTokens',
|
|
119
|
+
'context_tokens',
|
|
120
|
+
]
|
|
121
|
+
for (const k of keys) {
|
|
122
|
+
const v = modelObj[k]
|
|
123
|
+
if (typeof v === 'number' && Number.isFinite(v) && v > 0) return v
|
|
124
|
+
}
|
|
125
|
+
// 某些服务把窗口放在 meta / details 子对象
|
|
126
|
+
for (const sub of ['meta', 'details', 'model_info', 'capabilities']) {
|
|
127
|
+
const nested = modelObj[sub]
|
|
128
|
+
if (nested && typeof nested === 'object') {
|
|
129
|
+
const found = extractContextFromModelObj(nested)
|
|
130
|
+
if (found) return found
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// Ollama model_info 专用字段:llama.context_length
|
|
134
|
+
const modelInfo = modelObj.model_info
|
|
135
|
+
if (modelInfo && typeof modelInfo === 'object') {
|
|
136
|
+
const ctxKey = Object.keys(modelInfo).find(k => k.endsWith('.context_length'))
|
|
137
|
+
if (ctxKey && typeof modelInfo[ctxKey] === 'number' && modelInfo[ctxKey] > 0) {
|
|
138
|
+
return modelInfo[ctxKey]
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return null
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* 探测当前模型的上下文窗口
|
|
146
|
+
*
|
|
147
|
+
* 顺序:
|
|
148
|
+
* 1. 若已手动指定(manualWindow > 0)→ 直接返回手动值
|
|
149
|
+
* 2. 尝试从 GET {apiBase}/models 的响应中找匹配模型并提取窗口
|
|
150
|
+
* 3. 查内置表
|
|
151
|
+
* 4. 兜底 fallback
|
|
152
|
+
*
|
|
153
|
+
* @param {object} opts
|
|
154
|
+
* @param {string} opts.model — 模型名
|
|
155
|
+
* @param {string} opts.apiBase — API Base URL
|
|
156
|
+
* @param {string} [opts.apiKey]
|
|
157
|
+
* @param {number} [opts.manualWindow] — 用户手动指定的窗口(0 表示未指定)
|
|
158
|
+
* @param {number} [opts.fetchTimeoutMs] — 探测超时(默认 3000ms)
|
|
159
|
+
* @returns {Promise<{window:number, source:string}>}
|
|
160
|
+
*/
|
|
161
|
+
export async function detectContextWindow({ model, apiBase, apiKey, manualWindow = 0, fetchTimeoutMs = 3000 }) {
|
|
162
|
+
// 1. 手动指定最高优先级
|
|
163
|
+
if (manualWindow > 0) {
|
|
164
|
+
return { window: manualWindow, source: WINDOW_SOURCE.MANUAL }
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// 2. 探测 API
|
|
168
|
+
if (apiBase) {
|
|
169
|
+
try {
|
|
170
|
+
const controller = new AbortController()
|
|
171
|
+
const timer = setTimeout(() => controller.abort(), fetchTimeoutMs)
|
|
172
|
+
const url = apiBase.replace(/\/+$/, '') + '/models'
|
|
173
|
+
const res = await fetch(url, {
|
|
174
|
+
headers: {
|
|
175
|
+
'Content-Type': 'application/json',
|
|
176
|
+
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
|
177
|
+
},
|
|
178
|
+
signal: controller.signal,
|
|
179
|
+
})
|
|
180
|
+
clearTimeout(timer)
|
|
181
|
+
if (res.ok) {
|
|
182
|
+
const data = await res.json()
|
|
183
|
+
const models = data.data || []
|
|
184
|
+
// 优先精确匹配当前 model
|
|
185
|
+
let target = models.find(m => (m.id || '') === model)
|
|
186
|
+
// 其次尝试包含匹配(如 model 是 'qwen2.5' 而列表是 'qwen2.5:14b')
|
|
187
|
+
if (!target) {
|
|
188
|
+
target = models.find(m => (m.id || '').startsWith(model) || model.startsWith(m.id || ''))
|
|
189
|
+
}
|
|
190
|
+
if (target) {
|
|
191
|
+
const win = extractContextFromModelObj(target)
|
|
192
|
+
if (win) return { window: win, source: WINDOW_SOURCE.PROBE }
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
} catch {
|
|
196
|
+
// 探测失败(超时/网络/无权限)→ 继续走表/兜底
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// 3. 内置表
|
|
201
|
+
const normalized = (model || '').toLowerCase()
|
|
202
|
+
if (normalized && MODEL_CONTEXT_TABLE[normalized]) {
|
|
203
|
+
return { window: MODEL_CONTEXT_TABLE[normalized], source: WINDOW_SOURCE.TABLE }
|
|
204
|
+
}
|
|
205
|
+
// 表内做前缀匹配(如 'deepseek-chat:latest'、'qwen2.5-coder:7b')
|
|
206
|
+
for (const key of Object.keys(MODEL_CONTEXT_TABLE)) {
|
|
207
|
+
if (normalized.startsWith(key) || key.startsWith(normalized)) {
|
|
208
|
+
return { window: MODEL_CONTEXT_TABLE[key], source: WINDOW_SOURCE.TABLE }
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// 4. 兜底
|
|
213
|
+
return { window: FALLBACK_CONTEXT_WINDOW, source: WINDOW_SOURCE.FALLBACK }
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* 解析 /window 命令的参数为窗口 token 数
|
|
218
|
+
*
|
|
219
|
+
* 支持:
|
|
220
|
+
* "131072" → 131072
|
|
221
|
+
* "64k" → 65536
|
|
222
|
+
* "128k" → 131072
|
|
223
|
+
* "200k" → 204800
|
|
224
|
+
* "1m" → 1_000_000
|
|
225
|
+
*
|
|
226
|
+
* @param {string} str
|
|
227
|
+
* @returns {number|null} token 数;非法返回 null
|
|
228
|
+
*/
|
|
229
|
+
export function parseWindowArg(str) {
|
|
230
|
+
if (typeof str !== 'string') return null
|
|
231
|
+
const s = str.trim().toLowerCase().replace(/,/g, '')
|
|
232
|
+
if (!s) return null
|
|
233
|
+
const m = s.match(/^(\d+)\s*(k|m|kb|mb)?$/)
|
|
234
|
+
if (!m) return null
|
|
235
|
+
const num = parseInt(m[1], 10)
|
|
236
|
+
const unit = m[2]
|
|
237
|
+
if (!Number.isFinite(num) || num <= 0) return null
|
|
238
|
+
if (unit === 'k' || unit === 'kb') return num * 1000
|
|
239
|
+
if (unit === 'm' || unit === 'mb') return num * 1_000_000
|
|
240
|
+
return num
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* 将 token 数格式化为人类可读(十进制,与 token 常见记法一致:128K = 128,000)
|
|
245
|
+
* @param {number} tokens
|
|
246
|
+
* @returns {string} 如 "128.0K"、"1.00M"
|
|
247
|
+
*/
|
|
248
|
+
export function formatTokens(tokens) {
|
|
249
|
+
if (!Number.isFinite(tokens) || tokens <= 0) return '?'
|
|
250
|
+
if (tokens >= 1_000_000) return (tokens / 1_000_000).toFixed(2) + 'M'
|
|
251
|
+
if (tokens >= 1000) return (tokens / 1000).toFixed(1) + 'K'
|
|
252
|
+
return String(tokens)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* 获取窗口来源的中文标签
|
|
257
|
+
* @param {string} source
|
|
258
|
+
* @returns {string}
|
|
259
|
+
*/
|
|
260
|
+
export function windowSourceLabel(source) {
|
|
261
|
+
return SOURCE_LABEL[source] || source || '未知'
|
|
262
|
+
}
|
package/src/core/index.js
CHANGED
|
@@ -5,6 +5,14 @@ export { UserMessage, AssistantMessage, ToolCall } from "../types/index.js"
|
|
|
5
5
|
*/
|
|
6
6
|
export { QueryEngine, QueryEngineConfig } from './query-engine.js'
|
|
7
7
|
export { TokenBudget, estimateTokens } from './token-budget.js'
|
|
8
|
+
export {
|
|
9
|
+
detectContextWindow,
|
|
10
|
+
parseWindowArg,
|
|
11
|
+
formatTokens,
|
|
12
|
+
windowSourceLabel,
|
|
13
|
+
WINDOW_SOURCE,
|
|
14
|
+
MODEL_CONTEXT_TABLE,
|
|
15
|
+
} from './context-window.js'
|
|
8
16
|
export { SessionManager } from './session.js'
|
|
9
17
|
export { Config } from './config.js'
|
|
10
18
|
export { parseStream, parseNonStreamResponse } from './streaming.js'
|
package/src/core/query-engine.js
CHANGED
|
@@ -47,6 +47,7 @@ export class QueryEngineConfig {
|
|
|
47
47
|
this.onAskUser = options.onAskUser || null // AskUserQuestion 工具回调(宿主按来源分流,避免远程死锁)
|
|
48
48
|
this.readline = options.readline || null // 用于 AskUserQuestion 工具
|
|
49
49
|
this.onDelta = options.onDelta || null // 流式增量回调 {type:'text'|'reasoning', text}(供 VS Code 扩展等 UI 消费)
|
|
50
|
+
this.configStore = options.configStore || null // 配置实例(供工具读取 web.fetch.jinaApiKey 等;可选)
|
|
50
51
|
}
|
|
51
52
|
}
|
|
52
53
|
|
|
@@ -111,6 +112,20 @@ export class QueryEngine {
|
|
|
111
112
|
let finalResponse = ''
|
|
112
113
|
|
|
113
114
|
for (let turn = 0; turn < this.config.maxTurns; turn++) {
|
|
115
|
+
// 发送前硬校验:估算即将发送的消息是否超出窗口,超限则先压缩(最终兜底,防止溢出)
|
|
116
|
+
if (this.tokenBudget) {
|
|
117
|
+
const est = this.tokenBudget.estimateMessages(this.state.messages)
|
|
118
|
+
if (est > this.tokenBudget.maxTokens - this.tokenBudget.reservedForOutput) {
|
|
119
|
+
const { compacted, messages } = autoCompact(this.state.messages, this.tokenBudget, {
|
|
120
|
+
maxTokens: Math.floor(this.tokenBudget.maxTokens * 0.6),
|
|
121
|
+
})
|
|
122
|
+
if (compacted) {
|
|
123
|
+
this.state.messages = messages
|
|
124
|
+
if (this.config.verbose) console.error('[compact] Pre-send hard check: compressed to stay within window')
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
114
129
|
const requestMessages = this._buildRequest(this.state.messages)
|
|
115
130
|
const response = await this._callLLM(requestMessages, this.state.messages)
|
|
116
131
|
|
package/src/core/token-budget.js
CHANGED
|
@@ -24,6 +24,20 @@ export class TokenBudget {
|
|
|
24
24
|
this.used = 0
|
|
25
25
|
this.inputTokens = 0
|
|
26
26
|
this.outputTokens = 0
|
|
27
|
+
// 当前窗口来源:'manual' | 'probe' | 'table' | 'fallback'(见 context-window.js)
|
|
28
|
+
this.windowSource = options.windowSource || 'fallback'
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* 更新上下文窗口上限(运行时生效,立即反映到 usagePercent)
|
|
33
|
+
* @param {number} maxTokens
|
|
34
|
+
* @param {string} [source] — 窗口来源标签
|
|
35
|
+
*/
|
|
36
|
+
setWindow(maxTokens, source) {
|
|
37
|
+
if (Number.isFinite(maxTokens) && maxTokens > 0) {
|
|
38
|
+
this.maxTokens = maxTokens
|
|
39
|
+
}
|
|
40
|
+
if (source) this.windowSource = source
|
|
27
41
|
}
|
|
28
42
|
|
|
29
43
|
/** 可用于上下文的最大 token 数 */
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fetch 安全管道 — 协议白名单 + 连接级 SSRF 防护 + 重定向逐跳校验 + 大小/超时/SSL
|
|
3
|
+
*
|
|
4
|
+
* 移植自 openclaw safe-jina-fetch 设计(DESIGN.md §4),零依赖,仅内置模块。
|
|
5
|
+
*
|
|
6
|
+
* 设计原则:直连优先、兜底不扰。
|
|
7
|
+
* - 直连能拿到的,不多花一跳;
|
|
8
|
+
* - 直连被拦(非 ok / 网络错误 / 超时)才由调用方走 Jina Reader 兜底(见 web-fetch.js)。
|
|
9
|
+
*
|
|
10
|
+
* 相比项目原 ssrf-guard.js 的增强:
|
|
11
|
+
* - 连接级校验:给 http/https.request 注入 lookup 钩子,在 TCP 连接建立时对
|
|
12
|
+
* 全部解析地址逐一校验(防 DNS rebinding 的 TOCTOU 竞态);
|
|
13
|
+
* - 重定向逐跳校验:默认最多 5 跳,每一跳重新走完整协议/SSRF/域名校验。
|
|
14
|
+
*/
|
|
15
|
+
import { lookup as dnsLookup } from 'dns'
|
|
16
|
+
import http from 'http'
|
|
17
|
+
import https from 'https'
|
|
18
|
+
import { isIP } from 'net'
|
|
19
|
+
import { isBlockedAddress } from './ssrf-guard.js'
|
|
20
|
+
|
|
21
|
+
/** 协议白名单 — 仅 http/https,file/ftp/data 等一律拒绝 */
|
|
22
|
+
const ALLOWED_PROTOCOLS = ['http:', 'https:']
|
|
23
|
+
|
|
24
|
+
/** 域名后缀黑名单 */
|
|
25
|
+
const BLOCKED_HOSTNAME_SUFFIXES = ['.local', '.internal', '.localdomain', '.localhost', '.home', '.lan']
|
|
26
|
+
|
|
27
|
+
/** 已知 SSRF 目标主机名(精确匹配,兜底) */
|
|
28
|
+
const BLOCKED_HOSTNAMES = [
|
|
29
|
+
'localhost',
|
|
30
|
+
'localhost.localdomain',
|
|
31
|
+
'ip6-localhost',
|
|
32
|
+
'ip6-loopback',
|
|
33
|
+
'metadata.google.internal',
|
|
34
|
+
'metadata.internal',
|
|
35
|
+
'instance-data',
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
export const DEFAULT_FETCH_OPTIONS = {
|
|
39
|
+
maxBytes: 10 * 1024 * 1024, // 响应大小上限 10MB
|
|
40
|
+
timeoutMs: 30000, // 超时 30s
|
|
41
|
+
maxRedirects: 5, // 重定向最多 5 跳
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* 校验 URL 的协议是否在白名单内
|
|
46
|
+
* @param {string} protocol — 如 'https:'
|
|
47
|
+
* @returns {boolean}
|
|
48
|
+
*/
|
|
49
|
+
export function isAllowedProtocol(protocol) {
|
|
50
|
+
return ALLOWED_PROTOCOLS.includes(protocol)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* 校验主机名是否在内网/SSRF 黑名单
|
|
55
|
+
* @param {string} hostname
|
|
56
|
+
* @returns {{blocked: boolean, reason?: string}}
|
|
57
|
+
*/
|
|
58
|
+
export function checkHostnameBlocked(hostname) {
|
|
59
|
+
const lower = (hostname || '').toLowerCase()
|
|
60
|
+
if (!lower) return { blocked: true, reason: '空主机名' }
|
|
61
|
+
if (BLOCKED_HOSTNAMES.includes(lower)) {
|
|
62
|
+
return { blocked: true, reason: `主机名 ${hostname} 为已知 SSRF 目标` }
|
|
63
|
+
}
|
|
64
|
+
for (const suffix of BLOCKED_HOSTNAME_SUFFIXES) {
|
|
65
|
+
if (lower.endsWith(suffix)) {
|
|
66
|
+
return { blocked: true, reason: `主机名 ${hostname} 为内网域名(后缀 ${suffix})` }
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return { blocked: false }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* 校验单个 IP 地址是否安全(连接级,供 lookup 钩子逐地址调用)
|
|
74
|
+
* @param {string} address
|
|
75
|
+
* @returns {{blocked: boolean, reason?: string}}
|
|
76
|
+
*/
|
|
77
|
+
export function checkAddressBlocked(address) {
|
|
78
|
+
if (isBlockedAddress(address)) {
|
|
79
|
+
return { blocked: true, reason: `地址 ${address} 在私有/保留范围内,可能为 SSRF 目标` }
|
|
80
|
+
}
|
|
81
|
+
return { blocked: false }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* 安全 DNS lookup — 连接时逐地址校验(防 DNS rebinding)
|
|
86
|
+
*
|
|
87
|
+
* 兼容 Node 两种签名:
|
|
88
|
+
* - options.all = true → (err, addresses[])
|
|
89
|
+
* - 否则 → (err, address, family)
|
|
90
|
+
* 若任一解析地址命中私有/保留网段,立即报错阻断连接(而非放行让上层再查)。
|
|
91
|
+
*/
|
|
92
|
+
export function safeLookup(hostname, options, callback) {
|
|
93
|
+
const all = !!(options && options.all)
|
|
94
|
+
|
|
95
|
+
dnsLookup(hostname, { all: true }, (err, addresses) => {
|
|
96
|
+
if (err) {
|
|
97
|
+
// DNS 解析失败 — 交由上层处理(通常导致请求失败)
|
|
98
|
+
if (typeof callback === 'function') {
|
|
99
|
+
if (all) callback(err, [])
|
|
100
|
+
else callback(err, undefined, undefined)
|
|
101
|
+
}
|
|
102
|
+
return
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const list = Array.isArray(addresses) ? addresses : [{ address, family: 4 }]
|
|
106
|
+
|
|
107
|
+
// 逐地址校验 — 任一命中即阻断(拒绝整条连接,防止 rebinding 切到私有地址)
|
|
108
|
+
for (const a of list) {
|
|
109
|
+
const addr = typeof a === 'string' ? a : a.address
|
|
110
|
+
const { blocked, reason } = checkAddressBlocked(addr)
|
|
111
|
+
if (blocked) {
|
|
112
|
+
const e = new Error(`SSRF blocked: ${reason}`)
|
|
113
|
+
e.code = 'SSRF_BLOCKED'
|
|
114
|
+
if (typeof callback === 'function') {
|
|
115
|
+
if (all) callback(e, [])
|
|
116
|
+
else callback(e, undefined, undefined)
|
|
117
|
+
}
|
|
118
|
+
return
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// 全部安全 — 返回解析结果(保持签名兼容)
|
|
123
|
+
if (typeof callback === 'function') {
|
|
124
|
+
if (all) {
|
|
125
|
+
callback(null, list)
|
|
126
|
+
} else {
|
|
127
|
+
const first = list[0]
|
|
128
|
+
callback(null, typeof first === 'string' ? first : first.address, typeof first === 'string' ? 4 : first.family)
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
})
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* 构造带连接级 SSRF 防护的 http/https 客户端
|
|
136
|
+
* 通过注入 lookup 钩子,在 TCP 连接建立时对全部解析地址逐一校验。
|
|
137
|
+
* @returns {{http: import('http').Agent, https: import('https').Agent}}
|
|
138
|
+
*/
|
|
139
|
+
export function createSafeAgents() {
|
|
140
|
+
const agentOptions = { lookup: safeLookup, keepAlive: false }
|
|
141
|
+
return {
|
|
142
|
+
http: new http.Agent(agentOptions),
|
|
143
|
+
https: new https.Agent(agentOptions),
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* 解析并校验一个 URL(协议白名单 + 主机名黑名单)
|
|
149
|
+
* @param {string} url
|
|
150
|
+
* @returns {{ok: true, url: URL} | {ok: false, reason: string}}
|
|
151
|
+
*/
|
|
152
|
+
export function parseAndValidateUrl(url) {
|
|
153
|
+
let parsed
|
|
154
|
+
try {
|
|
155
|
+
parsed = new URL(url)
|
|
156
|
+
} catch {
|
|
157
|
+
return { ok: false, reason: `无效的 URL: ${url}` }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (!isAllowedProtocol(parsed.protocol)) {
|
|
161
|
+
return { ok: false, reason: `不支持的协议:${parsed.protocol}(仅允许 http/https)` }
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const hostBlock = checkHostnameBlocked(parsed.hostname)
|
|
165
|
+
if (hostBlock.blocked) {
|
|
166
|
+
return { ok: false, reason: hostBlock.reason }
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// IP 字面量:直接校验是否在私有/保留网段(Node 对 IP 字面量不走 DNS lookup,
|
|
170
|
+
// 连接级 lookup 钩子不会触发,必须在发起请求前显式校验,否则 SSRF 被绕过)
|
|
171
|
+
if (isIP(parsed.hostname)) {
|
|
172
|
+
const addrBlock = checkAddressBlocked(parsed.hostname)
|
|
173
|
+
if (addrBlock.blocked) {
|
|
174
|
+
return { ok: false, reason: addrBlock.reason }
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
return { ok: true, url: parsed }
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* 读取响应体,限制大小(超限截断并在 warnings 标记)
|
|
183
|
+
* @param {import('http').IncomingMessage} res
|
|
184
|
+
* @param {number} maxBytes
|
|
185
|
+
* @returns {Promise<{body: string, truncated: boolean}>}
|
|
186
|
+
*/
|
|
187
|
+
export async function readBodyLimited(res, maxBytes) {
|
|
188
|
+
const chunks = []
|
|
189
|
+
let total = 0
|
|
190
|
+
let truncated = false
|
|
191
|
+
for await (const chunk of res) {
|
|
192
|
+
total += chunk.length
|
|
193
|
+
if (total > maxBytes) {
|
|
194
|
+
truncated = true
|
|
195
|
+
chunks.push(chunk.slice(0, maxBytes - (total - chunk.length)))
|
|
196
|
+
break
|
|
197
|
+
}
|
|
198
|
+
chunks.push(chunk)
|
|
199
|
+
}
|
|
200
|
+
return { body: Buffer.concat(chunks).toString('utf-8'), truncated }
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* 带安全管道的 HTTP 抓取函数(单次请求,含协议/SSRF/大小/超时/SSL)
|
|
205
|
+
*
|
|
206
|
+
* 注意:本函数只发一次请求,不自动跟随重定向——重定向由 safeFetchWithRedirects
|
|
207
|
+
* 逐跳处理(每跳重新校验)。返回 { status, headers, body, truncated, finalUrl }。
|
|
208
|
+
*
|
|
209
|
+
* @param {string} url
|
|
210
|
+
* @param {object} [options]
|
|
211
|
+
* @param {number} [options.timeoutMs]
|
|
212
|
+
* @param {number} [options.maxBytes]
|
|
213
|
+
* @param {object} [options.headers]
|
|
214
|
+
* @returns {Promise<{ok: boolean, status: number, headers: object, body: string, truncated: boolean, finalUrl: string, error?: string}>}
|
|
215
|
+
*/
|
|
216
|
+
export function safeRequest(url, options = {}) {
|
|
217
|
+
const { timeoutMs = DEFAULT_FETCH_OPTIONS.timeoutMs, maxBytes = DEFAULT_FETCH_OPTIONS.maxBytes, headers = {} } = options
|
|
218
|
+
|
|
219
|
+
const validated = parseAndValidateUrl(url)
|
|
220
|
+
if (!validated.ok) {
|
|
221
|
+
return Promise.resolve({ ok: false, status: 0, headers: {}, body: '', truncated: false, finalUrl: url, error: validated.reason })
|
|
222
|
+
}
|
|
223
|
+
const parsed = validated.url
|
|
224
|
+
|
|
225
|
+
return new Promise((resolve) => {
|
|
226
|
+
const client = parsed.protocol === 'https:' ? https : http
|
|
227
|
+
const req = client.request(
|
|
228
|
+
parsed,
|
|
229
|
+
{
|
|
230
|
+
method: 'GET',
|
|
231
|
+
headers: { 'User-Agent': 'cc-node', 'Accept': 'text/html,application/json,text/plain,*/*', ...headers },
|
|
232
|
+
// 连接级 SSRF 防护(TCP 连接时逐地址校验,防 DNS rebinding)
|
|
233
|
+
lookup: safeLookup,
|
|
234
|
+
// 强制校验证书,不提供跳过选项
|
|
235
|
+
rejectUnauthorized: true,
|
|
236
|
+
},
|
|
237
|
+
async (res) => {
|
|
238
|
+
// 读响应体(限制大小)
|
|
239
|
+
try {
|
|
240
|
+
const { body, truncated } = await readBodyLimited(res, maxBytes)
|
|
241
|
+
resolve({
|
|
242
|
+
ok: res.statusCode >= 200 && res.statusCode < 300,
|
|
243
|
+
status: res.statusCode || 0,
|
|
244
|
+
headers: res.headers || {},
|
|
245
|
+
body,
|
|
246
|
+
truncated,
|
|
247
|
+
finalUrl: res.responseUrl || parsed.href,
|
|
248
|
+
})
|
|
249
|
+
} catch (err) {
|
|
250
|
+
resolve({ ok: false, status: res.statusCode || 0, headers: res.headers || {}, body: '', truncated: false, finalUrl: parsed.href, error: err.message })
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
req.setTimeout(timeoutMs, () => {
|
|
256
|
+
req.destroy(new Error('请求超时'))
|
|
257
|
+
})
|
|
258
|
+
|
|
259
|
+
req.on('error', (err) => {
|
|
260
|
+
resolve({ ok: false, status: 0, headers: {}, body: '', truncated: false, finalUrl: parsed.href, error: err.message })
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
req.end()
|
|
264
|
+
})
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* 带重定向逐跳校验的安全抓取
|
|
269
|
+
* 每一跳都重新执行协议/SSRF/域名校验,防止 302 → 内网 的重定向绕过。
|
|
270
|
+
*
|
|
271
|
+
* @param {string} url
|
|
272
|
+
* @param {object} [options]
|
|
273
|
+
* @param {number} [options.maxRedirects] — 最大跳数(默认 5)
|
|
274
|
+
* @returns {Promise<{ok: boolean, status: number, headers: object, body: string, truncated: boolean, finalUrl: string, redirects: string[], error?: string, warning?: string}>}
|
|
275
|
+
*/
|
|
276
|
+
export async function safeFetchWithRedirects(url, options = {}) {
|
|
277
|
+
const { maxRedirects = DEFAULT_FETCH_OPTIONS.maxRedirects, ...reqOptions } = options
|
|
278
|
+
let currentUrl = url
|
|
279
|
+
const redirects = []
|
|
280
|
+
|
|
281
|
+
for (let hop = 0; hop <= maxRedirects; hop++) {
|
|
282
|
+
const result = await safeRequest(currentUrl, reqOptions)
|
|
283
|
+
|
|
284
|
+
// 3xx 重定向 — 逐跳校验下一目标
|
|
285
|
+
if (result.status >= 300 && result.status < 400 && result.headers.location) {
|
|
286
|
+
// 校验下一跳 URL(相对路径需拼接到当前 URL)
|
|
287
|
+
let nextUrl
|
|
288
|
+
try {
|
|
289
|
+
nextUrl = new URL(result.headers.location, currentUrl).href
|
|
290
|
+
} catch {
|
|
291
|
+
return { ...result, redirects, error: `无效的重定向目标: ${result.headers.location}` }
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// 重定向目标重新走协议/SSRF 校验(防止跳到内网)
|
|
295
|
+
const v = parseAndValidateUrl(nextUrl)
|
|
296
|
+
if (!v.ok) {
|
|
297
|
+
return { ...result, redirects, error: `重定向目标被安全策略阻止: ${v.reason}` }
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
redirects.push(`${result.status} → ${nextUrl}`)
|
|
301
|
+
currentUrl = nextUrl
|
|
302
|
+
continue
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// 非重定向 — 返回最终结果(含重定向链)
|
|
306
|
+
return { ...result, redirects, finalUrl: currentUrl }
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// 超过最大跳数
|
|
310
|
+
return { ok: false, status: 0, headers: {}, body: '', truncated: false, finalUrl: currentUrl, redirects, error: `重定向次数超过上限 (${maxRedirects})` }
|
|
311
|
+
}
|