@v587d/capital-generation 2.8.3 → 2.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -1
- package/README.md +40 -19
- package/chart-ui/README.md +5 -4
- package/chart-ui/client.src.cjs +7 -16
- package/chart-ui/index.js +12 -33
- package/chart-ui/package.json +2 -2
- package/config/client.js +18 -39
- package/config/client.src.cjs +79 -217
- package/config/index.js +22 -72
- package/config/package.json +2 -2
- package/cordis.patch.yml +46 -51
- package/lib/agents/bash-guard.d.ts +9 -55
- package/lib/agents/bash-guard.js +27 -67
- package/lib/agents/preset-roster.d.ts +22 -0
- package/lib/agents/preset-roster.js +87 -0
- package/lib/agents/role-tool-policy.d.ts +46 -0
- package/lib/agents/role-tool-policy.js +205 -0
- package/lib/agents/root-tool-policy.d.ts +28 -124
- package/lib/agents/root-tool-policy.js +164 -237
- package/lib/agents/selected-skills-provider.d.ts +2 -6
- package/lib/agents/selected-skills-provider.js +27 -84
- package/lib/capital-settings.d.ts +27 -0
- package/lib/capital-settings.js +23 -0
- package/lib/chart/artifact-ref.d.ts +13 -8
- package/lib/chart/artifact-ref.js +27 -10
- package/lib/chart/downsample.d.ts +2 -5
- package/lib/chart/downsample.js +3 -8
- package/lib/chart/errors.d.ts +3 -7
- package/lib/chart/errors.js +1 -2
- package/lib/chart/event-context.d.ts +8 -0
- package/lib/chart/event-context.js +39 -0
- package/lib/chart/events.d.ts +21 -78
- package/lib/chart/events.js +43 -189
- package/lib/chart/html.d.ts +2 -5
- package/lib/chart/html.js +8 -15
- package/lib/chart/present-tool.d.ts +28 -0
- package/lib/chart/present-tool.js +128 -0
- package/lib/chart/runtime.d.ts +5 -18
- package/lib/chart/runtime.js +14 -34
- package/lib/chart/series.d.ts +10 -17
- package/lib/chart/series.js +22 -47
- package/lib/chart/source-token.d.ts +5 -13
- package/lib/chart/source-token.js +18 -17
- package/lib/chart/spec.d.ts +6 -12
- package/lib/chart/spec.js +7 -31
- package/lib/chart/tool.d.ts +7 -29
- package/lib/chart/tool.js +15 -25
- package/lib/chart-render-entry.d.ts +10 -0
- package/lib/chart-render-entry.js +14 -0
- package/lib/chart-source-entry.d.ts +13 -0
- package/lib/chart-source-entry.js +20 -0
- package/lib/collect-entry.d.ts +9 -0
- package/lib/collect-entry.js +10 -0
- package/lib/collector-runtime.d.ts +22 -0
- package/lib/collector-runtime.js +97 -0
- package/lib/data-collector/args.d.ts +6 -24
- package/lib/data-collector/args.js +6 -24
- package/lib/data-collector/dataset-tools.d.ts +7 -1
- package/lib/data-collector/dataset-tools.js +27 -56
- package/lib/data-collector/describe.d.ts +8 -33
- package/lib/data-collector/describe.js +17 -70
- package/lib/data-collector/hub.d.ts +27 -85
- package/lib/data-collector/hub.js +19 -85
- package/lib/data-collector/query-time.d.ts +7 -26
- package/lib/data-collector/query-time.js +8 -27
- package/lib/data-collector/store.d.ts +41 -117
- package/lib/data-collector/store.js +32 -111
- package/lib/data-collector/time-axis.d.ts +16 -76
- package/lib/data-collector/time-axis.js +25 -111
- package/lib/data-collector/tools.d.ts +6 -6
- package/lib/data-collector/tools.js +34 -43
- package/lib/dataset-read-entry.d.ts +9 -0
- package/lib/dataset-read-entry.js +7 -0
- package/lib/index.d.ts +7 -117
- package/lib/index.js +43 -422
- package/lib/net/eastmoney-client.d.ts +13 -65
- package/lib/net/eastmoney-client.js +12 -53
- package/lib/net/throttle.d.ts +2 -13
- package/lib/net/throttle.js +3 -14
- package/lib/ocr/client.d.ts +12 -21
- package/lib/ocr/client.js +23 -40
- package/lib/ocr/markdown.d.ts +20 -26
- package/lib/ocr/markdown.js +23 -36
- package/lib/ocr/store.d.ts +14 -47
- package/lib/ocr/store.js +15 -45
- package/lib/ocr-entry.d.ts +9 -0
- package/lib/ocr-entry.js +10 -0
- package/lib/policy-entry.d.ts +17 -0
- package/lib/policy-entry.js +22 -0
- package/lib/present-entry.d.ts +9 -0
- package/lib/present-entry.js +13 -0
- package/lib/preset-claim.d.ts +22 -0
- package/lib/preset-claim.js +24 -0
- package/lib/preset-ids.d.ts +38 -0
- package/lib/preset-ids.js +46 -0
- package/lib/retriever-config.d.ts +37 -0
- package/lib/retriever-config.js +36 -0
- package/lib/retriever-runtime.d.ts +31 -0
- package/lib/retriever-runtime.js +82 -0
- package/lib/root-extras-entry.d.ts +17 -0
- package/lib/root-extras-entry.js +25 -0
- package/lib/runtime-entry.d.ts +15 -0
- package/lib/runtime-entry.js +5 -0
- package/lib/runtime.d.ts +21 -0
- package/lib/runtime.js +33 -0
- package/lib/search-entry.d.ts +9 -0
- package/lib/search-entry.js +9 -0
- package/lib/sources/eastmoney-http.js +20 -68
- package/lib/sources/ftshare-rest.d.ts +6 -34
- package/lib/sources/ftshare-rest.js +84 -292
- package/lib/sources/fuyao-core.d.ts +8 -18
- package/lib/sources/fuyao-core.js +10 -21
- package/lib/sources/fuyao-rest.d.ts +2 -6
- package/lib/sources/fuyao-rest.js +23 -102
- package/lib/sources/security-code.d.ts +6 -12
- package/lib/sources/security-code.js +6 -12
- package/lib/sources/security-identity.d.ts +1 -4
- package/lib/sources/security-identity.js +3 -8
- package/lib/sources/tencent-http.js +14 -28
- package/lib/sources/tencent-public-core.d.ts +8 -60
- package/lib/sources/tencent-public-core.js +12 -77
- package/lib/sources/wind-mcp.d.ts +0 -5
- package/lib/sources/wind-mcp.js +23 -78
- package/lib/sources-entry.d.ts +9 -0
- package/lib/sources-entry.js +9 -0
- package/lib/time/tools.d.ts +8 -31
- package/lib/time/tools.js +30 -66
- package/lib/time-entry.d.ts +9 -0
- package/lib/time-entry.js +7 -0
- package/lib/tool-exec.d.ts +45 -11
- package/lib/tool-exec.js +59 -6
- package/lib/user-customization.d.ts +14 -0
- package/lib/user-customization.js +47 -0
- package/lib/watchlist/tools.d.ts +3 -13
- package/lib/watchlist/tools.js +10 -30
- package/lib/watchlist-entry.d.ts +9 -0
- package/lib/watchlist-entry.js +6 -0
- package/lib/web-retriever/engines.d.ts +6 -11
- package/lib/web-retriever/engines.js +9 -14
- package/lib/web-retriever/html-markdown.d.ts +14 -34
- package/lib/web-retriever/html-markdown.js +39 -75
- package/lib/web-retriever/local-fetch.d.ts +22 -58
- package/lib/web-retriever/local-fetch.js +33 -77
- package/lib/web-retriever/retriever.d.ts +6 -18
- package/lib/web-retriever/retriever.js +7 -7
- package/lib/web-retriever/sources.d.ts +37 -124
- package/lib/web-retriever/sources.js +64 -196
- package/lib/web-retriever/tools.d.ts +43 -27
- package/lib/web-retriever/tools.js +99 -137
- package/lib/web-retriever/wind-client.d.ts +14 -31
- package/lib/web-retriever/wind-client.js +16 -32
- package/locale/zh.json +1 -1
- package/package.json +72 -7
- package/preset/capital-generation/README.md +87 -53
- package/preset/capital-generation/agent.patch.yml +532 -337
- package/preset/capital-generation/skills/capital-chart-protocol/SKILL.md +6 -4
- package/preset/capital-generation/skills/capital-data-profile/SKILL.md +2 -2
- package/preset/capital-generation/skills/capital-data-protocol/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-equity-research/SKILL.md +4 -2
- package/preset/capital-generation/skills/capital-event-impact/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-orchestration/SKILL.md +30 -33
- package/preset/capital-generation/skills/capital-portfolio-review/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-visualization-protocol/SKILL.md +14 -16
- package/preset/capital-generation/skills/capital-web-evidence/SKILL.md +1 -1
- package/watchlist/client.js +15 -148
- package/watchlist/client.src.cjs +148 -662
- package/watchlist/holding-rules.js +11 -38
- package/watchlist/index.js +127 -397
- package/watchlist/package.json +2 -2
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SSRF-gated local HTTP fetcher used after AnySearch extraction fails.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* not pinned into the subsequent HTTP connection. A DNS rebinding window remains
|
|
6
|
-
* between validation and connect. Closing that window would require a custom
|
|
7
|
-
* dispatcher; this module keeps the same explicit trade-off as dsh-search-first.
|
|
2
|
+
* SSRF-gated local HTTP fetcher, used after AnySearch extraction fails.
|
|
3
|
+
* Known limit: DNS resolves per request but the IP is not pinned to the connection,
|
|
4
|
+
* so a rebinding window remains (same trade-off as dsh-search-first).
|
|
8
5
|
*/
|
|
9
|
-
export declare const LOCAL_FETCH_CLIENT_VERSION = "2.
|
|
6
|
+
export declare const LOCAL_FETCH_CLIENT_VERSION = "2.9.1";
|
|
10
7
|
export declare const LOCAL_FETCH_MAX_URL_LENGTH = 2048;
|
|
11
8
|
export type LocalFetchErrorCode = 'ABORTED' | 'TIMEOUT' | 'INVALID_URL' | 'BLOCKED_URL' | 'DNS' | 'REDIRECT' | 'HTTP' | 'TOO_LARGE' | 'UNSUPPORTED_CONTENT_TYPE';
|
|
12
9
|
export declare class LocalFetchError extends Error {
|
|
@@ -35,67 +32,36 @@ export interface LocalFetchOutcome {
|
|
|
35
32
|
/** Return true only for an address safe to use as a public outbound target. */
|
|
36
33
|
export declare function isPublicIp(address: string): boolean;
|
|
37
34
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* `
|
|
41
|
-
*
|
|
42
|
-
* - 非 http(s) 协议:`javascript:` / `data:` 点下去就是执行;
|
|
43
|
-
* - 含空白或控制符:换行能把后半截甩出链接语法,变成正文里的新内容;
|
|
44
|
-
* - 带凭据:`https://sseinfo.com.cn@evil.example/` 显示的是官方域名、跳的是别的站;
|
|
45
|
-
* - 超长串:不是链接该占的体积(条目的输出预算见 `tools.ts` 的 `SOURCE_OUTPUT_BUDGET_CHARS`)。
|
|
46
|
-
* 返回**剥完不可见字符的原串**而不是 `url.toString()`:后者会把非 ASCII 路径按
|
|
47
|
-
* percent-encoding 展开(中文 URL 一字 9 字符),模型引用时白白吃掉预算。
|
|
35
|
+
* Upstream URLs become clickable markdown, so: http(s) only (javascript:/data: execute),
|
|
36
|
+
* no whitespace/control chars (newlines escape the link), no credentials (displayed host
|
|
37
|
+
* != target), length cap. Raw string, not `href`, to avoid percent-encoding non-ASCII;
|
|
38
|
+
* `''` means drop the field, never the item.
|
|
48
39
|
*/
|
|
49
40
|
export declare function safeUrl(value: unknown, maxChars?: number): string;
|
|
50
41
|
/**
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* "把内网 URL 交给第三方去戳"的通道,所以这里复用与本机直连**同一份**主机校验
|
|
55
|
-
* (形态走 `safeUrl` + `validateUrl`,公网性走 `assertAllowedHost`),只是跳过后面的
|
|
56
|
-
* 重定向与体积那几步。SSRF 面因此不新长第二份实现,也不会随两处漂移。
|
|
57
|
-
*
|
|
58
|
-
* 返回**原串**(不是 `url.href`):`href` 会把非 ASCII 路径 percent-encode 展开,
|
|
59
|
-
* 白吃输出预算;`allowPrivate` 在这里恒为 false,不提供开关——关掉的不是我们的风险面。
|
|
42
|
+
* Gate for a URL fetched by someone else (ocr url form): same shape + public-host
|
|
43
|
+
* check, no request. `allowPrivate` is always false — that risk is not ours to waive.
|
|
44
|
+
* Returns the raw string (href would percent-encode non-ASCII and eat budget).
|
|
60
45
|
*/
|
|
61
46
|
export declare function assertPublicUrlTarget(value: unknown, signal: AbortSignal, resolveAddresses?: LocalFetchOptions['resolveAddresses']): Promise<string>;
|
|
62
47
|
/**
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
* 为什么共用而不是各写一份:出口校验(URL 合法性 + DNS 公网 IP 闸门 + 手动重定向重校验 +
|
|
67
|
-
* 大小/超时上限)一旦有第二份实现,两份就会各自漂移,而**安全闸门漂移不会报错**——
|
|
68
|
-
* 本仓在"同一个便利能力接在多个入口、只覆盖接好的那个"上已经踩过两次(AGENTS.md §9.7)。
|
|
48
|
+
* Request descriptor shared by `createLocalFetcher` (GET + markdown for fallback)
|
|
49
|
+
* and the named-source tools, so there is exactly one egress gate.
|
|
69
50
|
*/
|
|
70
51
|
export interface HttpRequest {
|
|
71
52
|
url: string;
|
|
72
53
|
method?: 'GET' | 'POST';
|
|
73
|
-
/**
|
|
54
|
+
/** Extra headers (e.g. Referer); source recipes only, never model input. */
|
|
74
55
|
headers?: Record<string, string>;
|
|
75
|
-
/**
|
|
76
|
-
* 请求体。字符串按 `application/x-www-form-urlencoded` 发(互动易第二步要求 POST 但 body 为空);
|
|
77
|
-
* `FormData` 则**故意不写 content-type**——boundary 必须由实现自己带,写死就是把请求写坏。
|
|
78
|
-
*/
|
|
56
|
+
/** String body -> x-www-form-urlencoded; FormData sets no content-type (impl supplies the boundary). */
|
|
79
57
|
body?: string | FormData;
|
|
80
|
-
/**
|
|
58
|
+
/** Override the accept header for this request. */
|
|
81
59
|
accept?: string;
|
|
82
|
-
/**
|
|
83
|
-
* 正文编码。省略时用响应头声明的 charset,缺省 UTF-8。
|
|
84
|
-
*
|
|
85
|
-
* 为什么需要**工具内部**的显式声明:2026-09-24 实测 `basic.10jqka.com.cn` 声明
|
|
86
|
-
* `content-type: text/html`(**不带 charset**)而正文是 **GBK**——按 UTF-8 解会产出 11,800 个
|
|
87
|
-
* U+FFFD 替换字符,中文全部不可读、正则全部失配(`gbk` 解码后 0 个替换字符)。
|
|
88
|
-
* ⚠️ 这个字段**绝不能暴露成模型入参**:配错编码只会静默产出乱码,属"看起来成功"的坏形态。
|
|
89
|
-
* 与 `headers` 一样,只由来源实现按硬编码知识填写。
|
|
90
|
-
*/
|
|
60
|
+
/** Body encoding, overrides the response charset. Source-internal: a wrong value silently mojibakes. */
|
|
91
61
|
encoding?: string;
|
|
92
62
|
/**
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* 为什么要覆盖而不是把全局值调大:全局 `maxBytes` 是给"抓一个网页正文"定的(512 KB),
|
|
96
|
-
* 网页抓取的响应体积就是风险体积;OCR 结果 JSONL 是**我们自己提交出去再取回来**的解析产物
|
|
97
|
-
* (实测 15 页研报 258 KB),把它挤进网页的额度会让大文档静默 `truncated`——
|
|
98
|
-
* 而截断的 JSONL 是解析失败,不是"少读了几条"。调大全局值等于放宽所有网页出口。
|
|
63
|
+
* Per-request caps: global ones fit a web page, but parsed OCR JSONL can hit 258 KB
|
|
64
|
+
* and a truncated result is a parse failure, not "a few fewer rows".
|
|
99
65
|
*/
|
|
100
66
|
maxBytes?: number;
|
|
101
67
|
maxContentChars?: number;
|
|
@@ -103,7 +69,7 @@ export interface HttpRequest {
|
|
|
103
69
|
export interface HttpOutcome {
|
|
104
70
|
url: string;
|
|
105
71
|
status: number;
|
|
106
|
-
/**
|
|
72
|
+
/** Raw body, no conversion (JSON passes through, HTML stays raw; the tool decides). */
|
|
107
73
|
text: string;
|
|
108
74
|
truncated: boolean;
|
|
109
75
|
}
|
|
@@ -111,10 +77,8 @@ export declare function createHttpRequester(options: LocalFetchOptions): {
|
|
|
111
77
|
request(request: HttpRequest, signal?: AbortSignal): Promise<HttpOutcome>;
|
|
112
78
|
};
|
|
113
79
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
* 出口校验 / 重定向 / 上限 / 取消语义全部委托给 {@link createHttpRequester}——
|
|
117
|
-
* 这里只保留"如何呈现正文"这一层,避免同一套闸门出现第二份实现(见 `HttpRequest` 注释)。
|
|
80
|
+
* Fallback fetcher: GET, then choose presentation (JSON raw, HTML -> markdown).
|
|
81
|
+
* Egress, redirects, limits and cancellation all delegate to `createHttpRequester`.
|
|
118
82
|
*/
|
|
119
83
|
export declare function createLocalFetcher(options: LocalFetchOptions): {
|
|
120
84
|
fetch(url: string, signal?: AbortSignal): Promise<LocalFetchOutcome>;
|
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SSRF-gated local HTTP fetcher used after AnySearch extraction fails.
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* not pinned into the subsequent HTTP connection. A DNS rebinding window remains
|
|
6
|
-
* between validation and connect. Closing that window would require a custom
|
|
7
|
-
* dispatcher; this module keeps the same explicit trade-off as dsh-search-first.
|
|
2
|
+
* SSRF-gated local HTTP fetcher, used after AnySearch extraction fails.
|
|
3
|
+
* Known limit: DNS resolves per request but the IP is not pinned to the connection,
|
|
4
|
+
* so a rebinding window remains (same trade-off as dsh-search-first).
|
|
8
5
|
*/
|
|
9
6
|
import { htmlToMarkdown, stripInvisibleText } from './html-markdown.js';
|
|
10
|
-
export const LOCAL_FETCH_CLIENT_VERSION = '2.
|
|
7
|
+
export const LOCAL_FETCH_CLIENT_VERSION = '2.9.1';
|
|
11
8
|
export const LOCAL_FETCH_MAX_URL_LENGTH = 2048;
|
|
12
9
|
export class LocalFetchError extends Error {
|
|
13
10
|
code;
|
|
@@ -74,8 +71,7 @@ function parseIpv6(address) {
|
|
|
74
71
|
}
|
|
75
72
|
function isPrivateIpv4(address) {
|
|
76
73
|
const [first, second, third] = address;
|
|
77
|
-
// 100.64/10
|
|
78
|
-
// 100.100.100.200 已被 `first===100 && second>=64 && second<=127` 这条挡住。
|
|
74
|
+
// 100.64/10 also covers 100.100.100.200 (cloud metadata endpoint).
|
|
79
75
|
return first === 0
|
|
80
76
|
|| first === 10
|
|
81
77
|
|| first === 127
|
|
@@ -90,7 +86,7 @@ function isPrivateIpv4(address) {
|
|
|
90
86
|
|| (first === 203 && second === 0 && third === 113)
|
|
91
87
|
|| first >= 224;
|
|
92
88
|
}
|
|
93
|
-
/**
|
|
89
|
+
/** Two 16-bit groups -> dotted IPv4 (for 6to4 / Teredo embedded addresses). */
|
|
94
90
|
function groupsToIpv4(high, low) {
|
|
95
91
|
return [high >> 8, high & 0xff, low >> 8, low & 0xff];
|
|
96
92
|
}
|
|
@@ -109,11 +105,11 @@ export function isPublicIp(address) {
|
|
|
109
105
|
const isMappedIpv4 = groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff;
|
|
110
106
|
if (isMappedIpv4)
|
|
111
107
|
return !isPrivateIpv4(groupsToIpv4(groups[6], groups[7]));
|
|
112
|
-
// 6to4
|
|
108
|
+
// 6to4 (2002::/16) embeds IPv4 in groups 2-3; Teredo (2001:0::/32) in groups
|
|
109
|
+
// 7-8 bitwise-negated, plus server IPv4 in 3-4 — decode, or the prefix bypasses
|
|
110
|
+
// the private v4 table.
|
|
113
111
|
if (groups[0] === 0x2002)
|
|
114
112
|
return !isPrivateIpv4(groupsToIpv4(groups[1], groups[2]));
|
|
115
|
-
// Teredo(2001:0000::/32)的客户端 IPv4 在第 7、8 组按位取反存储(服务端 IPv4 在第 3、4 组,
|
|
116
|
-
// 一并校验):解出内嵌 v4 后递归判定,否则 `2001::` 前缀会绕过整张私网 v4 表。
|
|
117
113
|
if (groups[0] === 0x2001 && groups[1] === 0x0000) {
|
|
118
114
|
const client = groupsToIpv4(~groups[6] & 0xffff, ~groups[7] & 0xffff);
|
|
119
115
|
const server = groupsToIpv4(groups[2], groups[3]);
|
|
@@ -155,19 +151,13 @@ function validateUrl(input) {
|
|
|
155
151
|
}
|
|
156
152
|
return url;
|
|
157
153
|
}
|
|
158
|
-
/**
|
|
154
|
+
/** Display form of a link: literal http(s), no whitespace or control chars. */
|
|
159
155
|
const HYPERLINK_PATTERN = /^https?:\/\/[^\s\u0000-\u001F\u007F-\u009F]+$/u;
|
|
160
156
|
/**
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
* `
|
|
164
|
-
*
|
|
165
|
-
* - 非 http(s) 协议:`javascript:` / `data:` 点下去就是执行;
|
|
166
|
-
* - 含空白或控制符:换行能把后半截甩出链接语法,变成正文里的新内容;
|
|
167
|
-
* - 带凭据:`https://sseinfo.com.cn@evil.example/` 显示的是官方域名、跳的是别的站;
|
|
168
|
-
* - 超长串:不是链接该占的体积(条目的输出预算见 `tools.ts` 的 `SOURCE_OUTPUT_BUDGET_CHARS`)。
|
|
169
|
-
* 返回**剥完不可见字符的原串**而不是 `url.toString()`:后者会把非 ASCII 路径按
|
|
170
|
-
* percent-encoding 展开(中文 URL 一字 9 字符),模型引用时白白吃掉预算。
|
|
157
|
+
* Upstream URLs become clickable markdown, so: http(s) only (javascript:/data: execute),
|
|
158
|
+
* no whitespace/control chars (newlines escape the link), no credentials (displayed host
|
|
159
|
+
* != target), length cap. Raw string, not `href`, to avoid percent-encoding non-ASCII;
|
|
160
|
+
* `''` means drop the field, never the item.
|
|
171
161
|
*/
|
|
172
162
|
export function safeUrl(value, maxChars = LOCAL_FETCH_MAX_URL_LENGTH) {
|
|
173
163
|
if (typeof value !== 'string')
|
|
@@ -262,15 +252,9 @@ async function assertAllowedHost(url, options, signal) {
|
|
|
262
252
|
}
|
|
263
253
|
}
|
|
264
254
|
/**
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
* "把内网 URL 交给第三方去戳"的通道,所以这里复用与本机直连**同一份**主机校验
|
|
269
|
-
* (形态走 `safeUrl` + `validateUrl`,公网性走 `assertAllowedHost`),只是跳过后面的
|
|
270
|
-
* 重定向与体积那几步。SSRF 面因此不新长第二份实现,也不会随两处漂移。
|
|
271
|
-
*
|
|
272
|
-
* 返回**原串**(不是 `url.href`):`href` 会把非 ASCII 路径 percent-encode 展开,
|
|
273
|
-
* 白吃输出预算;`allowPrivate` 在这里恒为 false,不提供开关——关掉的不是我们的风险面。
|
|
255
|
+
* Gate for a URL fetched by someone else (ocr url form): same shape + public-host
|
|
256
|
+
* check, no request. `allowPrivate` is always false — that risk is not ours to waive.
|
|
257
|
+
* Returns the raw string (href would percent-encode non-ASCII and eat budget).
|
|
274
258
|
*/
|
|
275
259
|
export async function assertPublicUrlTarget(value, signal, resolveAddresses) {
|
|
276
260
|
const display = safeUrl(value);
|
|
@@ -294,13 +278,8 @@ function isSupportedContentType(mime) {
|
|
|
294
278
|
|| mime.endsWith('+xml');
|
|
295
279
|
}
|
|
296
280
|
/**
|
|
297
|
-
*
|
|
298
|
-
*
|
|
299
|
-
* 为什么需要它:2026-09-24 实测 `sns.sseinfo.com/allcompany.do`(上证e互动公司列表)返回
|
|
300
|
-
* `json/javascript;charset=UTF-8`,正文是标准 JSON。严格按 `application/json` 判定会把
|
|
301
|
-
* 一个**正常的 JSON 接口**判成 `UNSUPPORTED_CONTENT_TYPE`——本仓此前只见过
|
|
302
|
-
* `application/json` / `+json`,所以这条是接入该来源时才暴露的。安全性不受影响:
|
|
303
|
-
* 正文仍要 `JSON.parse` 通过,解析失败照旧报错,只是不再因为 MIME 拼法而误拒。
|
|
281
|
+
* Accept javascript-style MIME spellings: sseinfo returns `json/javascript`.
|
|
282
|
+
* Safe because the body must still pass JSON.parse.
|
|
304
283
|
*/
|
|
305
284
|
function isJsonMime(mime) {
|
|
306
285
|
return mime === 'application/json'
|
|
@@ -310,14 +289,8 @@ function isJsonMime(mime) {
|
|
|
310
289
|
|| mime === 'text/javascript';
|
|
311
290
|
}
|
|
312
291
|
/**
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
* 为什么需要它:实测 `https://www.cls.cn/detail/2188659` 的 GET 响应**不带任何
|
|
316
|
-
* Content-Type**(同一 URL 的 HEAD 却带 `text/html`),而正文是完好的 SSR HTML。
|
|
317
|
-
* 原实现"缺头即拒"会把这类站点误判成 `UNSUPPORTED_CONTENT_TYPE`。
|
|
318
|
-
*
|
|
319
|
-
* 边界(有意保守):只在**缺头**时启用;头存在但不支持的类型(如 `application/pdf`)
|
|
320
|
-
* 仍然照旧拒绝,不放宽既有内容类型闸门。判定只认正文开头的明确标记,认不出就维持失败。
|
|
292
|
+
* Conservative sniff used only when Content-Type is entirely missing (some hosts
|
|
293
|
+
* serve HTML without it); a present-but-unsupported type is still rejected.
|
|
321
294
|
*/
|
|
322
295
|
function sniffMissingContentType(text) {
|
|
323
296
|
const head = text.slice(0, 512).replace(/^\uFEFF/u, '').trimStart().toLowerCase();
|
|
@@ -335,15 +308,11 @@ function createDecoder(charset) {
|
|
|
335
308
|
throw new LocalFetchError('UNSUPPORTED_CONTENT_TYPE', `unsupported charset: ${charset ?? '(default)'}`);
|
|
336
309
|
}
|
|
337
310
|
}
|
|
338
|
-
/**
|
|
311
|
+
/** Error-body excerpt cap: enough to name the cause, not a second content channel. */
|
|
339
312
|
const ERROR_DETAIL_MAX_CHARS = 180;
|
|
340
313
|
/**
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
* 为什么必须带:2026-09-25 实测 PaddleOCR 对某个 PDF 直链回 `HTTP 400` +
|
|
344
|
-
* `{"code":10004,"msg":"文件格式不支持"}`——只报状态码,模型面对 `local fetch returned HTTP 400`
|
|
345
|
-
* 什么也决定不了(换链接?换形态?放弃?)。原因在上游写的正文里,不在我们造的码里。
|
|
346
|
-
* 读失败一律吞掉返回空串:这是给失败**加**信息,不能反过来把原始失败弄丢或改成另一种失败。
|
|
314
|
+
* Surface the upstream reason (e.g. HTTP 400 + `{"code":10004,...}`) in the error;
|
|
315
|
+
* read failures are swallowed so the original failure is never lost.
|
|
347
316
|
*/
|
|
348
317
|
async function errorDetail(response, signal) {
|
|
349
318
|
try {
|
|
@@ -476,15 +445,14 @@ export function createHttpRequester(options) {
|
|
|
476
445
|
throw new LocalFetchError('HTTP', `local fetch returned HTTP ${response.status}${await errorDetail(response, controller.signal)}`);
|
|
477
446
|
}
|
|
478
447
|
const type = contentType(response);
|
|
479
|
-
//
|
|
480
|
-
//
|
|
448
|
+
// Missing Content-Type is deferred to body sniffing; a present unsupported
|
|
449
|
+
// type still fails before reading the body.
|
|
481
450
|
const missingContentType = type.mime === '';
|
|
482
451
|
if (!missingContentType && !isSupportedContentType(type.mime)) {
|
|
483
452
|
throw new LocalFetchError('UNSUPPORTED_CONTENT_TYPE', `unsupported content type: ${type.mime}`);
|
|
484
453
|
}
|
|
485
454
|
const body = await readBody(response,
|
|
486
|
-
//
|
|
487
|
-
// (`basic.10jqka.com.cn`),按 UTF-8 解会整页变成替换字符(见 `HttpRequest.encoding`)。
|
|
455
|
+
// Source-declared encoding wins: some hosts omit charset but serve GBK.
|
|
488
456
|
input.encoding ?? type.charset, input.maxBytes ?? options.maxBytes, input.maxContentChars ?? options.maxContentChars, controller.signal);
|
|
489
457
|
if (missingContentType && sniffMissingContentType(body.text) === undefined) {
|
|
490
458
|
throw new LocalFetchError('UNSUPPORTED_CONTENT_TYPE', 'unsupported content type: (missing)');
|
|
@@ -515,31 +483,19 @@ export function createHttpRequester(options) {
|
|
|
515
483
|
return { request: requestOnce };
|
|
516
484
|
}
|
|
517
485
|
/**
|
|
518
|
-
*
|
|
519
|
-
*
|
|
520
|
-
* 出口校验 / 重定向 / 上限 / 取消语义全部委托给 {@link createHttpRequester}——
|
|
521
|
-
* 这里只保留"如何呈现正文"这一层,避免同一套闸门出现第二份实现(见 `HttpRequest` 注释)。
|
|
486
|
+
* Fallback fetcher: GET, then choose presentation (JSON raw, HTML -> markdown).
|
|
487
|
+
* Egress, redirects, limits and cancellation all delegate to `createHttpRequester`.
|
|
522
488
|
*/
|
|
523
489
|
export function createLocalFetcher(options) {
|
|
524
490
|
const requester = createHttpRequester(options);
|
|
525
491
|
async function fetchLocal(input, callerSignal) {
|
|
526
|
-
// accept
|
|
492
|
+
// The accept header is set centrally by the requester (HTML/text first, JSON last).
|
|
527
493
|
const outcome = await requester.request({ url: input }, callerSignal);
|
|
528
494
|
const head = outcome.text.slice(0, 512).replace(/^\uFEFF/u, '').trimStart().toLowerCase();
|
|
529
495
|
const looksHtml = /^(?:<!doctype|<html|<\?xml|<!--|<[a-z][a-z0-9-]*(?:\s|\/|>))/u.test(head);
|
|
530
|
-
// JSON
|
|
531
|
-
//
|
|
532
|
-
//
|
|
533
|
-
// 数组括号变 `\[ ... \]`(`JSON.parse` 不再成立)、字段名 `content_text` 变
|
|
534
|
-
// `content\_text`(按字段名读取不可靠)、`<script>` 内容被当标签删掉(数据丢失)。
|
|
535
|
-
// 内容类型闸门与 skill 都向模型承诺支持 JSON,此前的行为是"承诺收下、交付却改坏"。
|
|
536
|
-
//
|
|
537
|
-
// ⚠️ 呈现判据**只看正文**,不看响应头:requester 已按内容类型闸门决定"收不收",
|
|
538
|
-
// 而实测存在响应头与正文不一致的站点(`sniffMissingContentType` 的注释记着
|
|
539
|
-
// cls.cn 详情页的实测案例),与其再信一次头,不如直接认正文。
|
|
540
|
-
// JSON 原样返回:用**正文**判据而不是响应头——实测 `json/javascript` 这类 MIME
|
|
541
|
-
// 在严格 `application/json` 判定下不算 JSON(见 `isJsonMime` 的记录),
|
|
542
|
-
// 且 JSON 正文必然以 `{` / `[` 开头,判定是确定的。
|
|
496
|
+
// JSON must pass through raw: the turndown converter escapes brackets, field
|
|
497
|
+
// names and drops script-like content, corrupting it. Judge by the body, not
|
|
498
|
+
// the header (some hosts send mismatched or javascript-ish MIME types).
|
|
543
499
|
const looksJson = head.startsWith('{') || head.startsWith('[');
|
|
544
500
|
if (looksJson || !looksHtml) {
|
|
545
501
|
return {
|
|
@@ -1,28 +1,19 @@
|
|
|
1
1
|
import type { AnySearchClient, AnySearchErrorCode, SearchResult } from './engines.js';
|
|
2
2
|
import type { LocalFetchErrorCode } from './local-fetch.js';
|
|
3
|
-
/**
|
|
3
|
+
/** Which path produced the body; always `'anysearch'` when `ok:false`. */
|
|
4
4
|
export type FetchVia = 'anysearch' | 'local-http';
|
|
5
|
-
/**
|
|
5
|
+
/** The AnySearch failure that triggered fallback. */
|
|
6
6
|
export interface FetchFallbackNote {
|
|
7
7
|
from: 'anysearch';
|
|
8
8
|
code: AnySearchErrorCode;
|
|
9
9
|
error: string;
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
11
|
+
/** Why local fetching failed; only when fallback ran and also failed. */
|
|
12
12
|
export interface FetchLocalErrorNote {
|
|
13
13
|
code: LocalFetchErrorCode;
|
|
14
14
|
error: string;
|
|
15
15
|
}
|
|
16
|
-
/**
|
|
17
|
-
* `web_retriever_fetch` 的回执。
|
|
18
|
-
*
|
|
19
|
-
* - `via`:最终产出这份正文的来源;`ok:false` 时恒为 `'anysearch'`(主路)。
|
|
20
|
-
* - `fallback`:触发回退的那次 AnySearch 失败,永远只描述 AnySearch 侧
|
|
21
|
-
* (回退成功与"回退尝试后本地也失败"两种情形都会出现)。
|
|
22
|
-
* - `local_error`:本地抓取自己失败的原因,只在"回退被尝试且本地也失败"时出现。
|
|
23
|
-
* `fallback.error` 是 AnySearch 的错、`local_error.error` 是本地的错,两者不重复。
|
|
24
|
-
* - `truncated`:正文被截断过(AnySearch 侧截断、本地字节截断、本地字符截断、转换省略,任一为真)。
|
|
25
|
-
*/
|
|
16
|
+
/** `web_retriever_fetch` receipt; `truncated` covers any truncation path. */
|
|
26
17
|
export interface FetchResult {
|
|
27
18
|
url: string;
|
|
28
19
|
ok: boolean;
|
|
@@ -35,10 +26,7 @@ export interface FetchResult {
|
|
|
35
26
|
code?: AnySearchErrorCode;
|
|
36
27
|
local_error?: FetchLocalErrorNote;
|
|
37
28
|
}
|
|
38
|
-
/**
|
|
39
|
-
* 本地抓取器的结构类型。构造函数不传 `localFetch` 即"不启用回退",
|
|
40
|
-
* 由 `src/index.ts` 按配置决定是否传入。
|
|
41
|
-
*/
|
|
29
|
+
/** Structural fetcher; omitting it disables fallback (decided in `src/index.ts`). */
|
|
42
30
|
export interface LocalFetcherLike {
|
|
43
31
|
fetch(url: string, signal?: AbortSignal): Promise<{
|
|
44
32
|
url: string;
|
|
@@ -48,7 +36,7 @@ export interface LocalFetcherLike {
|
|
|
48
36
|
truncated: boolean;
|
|
49
37
|
}>;
|
|
50
38
|
}
|
|
51
|
-
/**
|
|
39
|
+
/** Stateless AnySearch adapter: no cache, queue, turn or corpus index. */
|
|
52
40
|
export declare class WebRetriever {
|
|
53
41
|
private readonly client;
|
|
54
42
|
private readonly localFetch?;
|
|
@@ -3,11 +3,11 @@ import { LocalFetchError } from './local-fetch.js';
|
|
|
3
3
|
function errorText(error) {
|
|
4
4
|
return error instanceof Error ? error.message : String(error);
|
|
5
5
|
}
|
|
6
|
-
/**
|
|
6
|
+
/** Caller cancellation must rethrow, never become a failure envelope. */
|
|
7
7
|
function abortError(signal) {
|
|
8
8
|
return signal.reason ?? new Error('request aborted');
|
|
9
9
|
}
|
|
10
|
-
/**
|
|
10
|
+
/** Stateless AnySearch adapter: no cache, queue, turn or corpus index. */
|
|
11
11
|
export class WebRetriever {
|
|
12
12
|
client;
|
|
13
13
|
localFetch;
|
|
@@ -37,7 +37,7 @@ export class WebRetriever {
|
|
|
37
37
|
...(normalized.truncated ? { truncated: true } : {}),
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
-
// "
|
|
40
|
+
// "Fetched but empty" is equivalent to failure and is fallback-eligible.
|
|
41
41
|
trigger = {
|
|
42
42
|
from: 'anysearch',
|
|
43
43
|
code: 'UPSTREAM',
|
|
@@ -45,18 +45,18 @@ export class WebRetriever {
|
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
47
|
catch (error) {
|
|
48
|
-
//
|
|
48
|
+
// Caller cancel: rethrow, no fallback.
|
|
49
49
|
if (error instanceof AnySearchError && error.code === 'ABORTED')
|
|
50
50
|
throw error;
|
|
51
51
|
const code = error instanceof AnySearchError ? error.code : 'UPSTREAM';
|
|
52
52
|
const message = errorText(error);
|
|
53
|
-
//
|
|
53
|
+
// Invalid URL / unsupported content: local would reject too, fallback is wasted.
|
|
54
54
|
if (!shouldFallbackToLocalFetch(code)) {
|
|
55
55
|
return { url, ok: false, via: 'anysearch', code, error: message };
|
|
56
56
|
}
|
|
57
57
|
trigger = { from: 'anysearch', code, error: message };
|
|
58
58
|
}
|
|
59
|
-
//
|
|
59
|
+
// Hard cancel gate before fallback: never spend another request on an abandoned result.
|
|
60
60
|
if (signal?.aborted)
|
|
61
61
|
throw abortError(signal);
|
|
62
62
|
if (!this.localFetch) {
|
|
@@ -75,7 +75,7 @@ export class WebRetriever {
|
|
|
75
75
|
};
|
|
76
76
|
}
|
|
77
77
|
catch (error) {
|
|
78
|
-
//
|
|
78
|
+
// Cancel raised on the local side also rethrows as-is.
|
|
79
79
|
if (error instanceof LocalFetchError && error.code === 'ABORTED')
|
|
80
80
|
throw error;
|
|
81
81
|
const localMessage = errorText(error);
|