@v587d/capital-generation 2.8.3 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -1
- package/README.md +40 -19
- package/chart-ui/README.md +1 -1
- package/chart-ui/client.src.cjs +7 -16
- package/chart-ui/index.js +12 -33
- package/chart-ui/package.json +2 -2
- package/config/client.js +18 -39
- package/config/client.src.cjs +79 -217
- package/config/index.js +22 -72
- package/config/package.json +2 -2
- package/cordis.patch.yml +46 -51
- package/lib/agents/bash-guard.d.ts +9 -55
- package/lib/agents/bash-guard.js +27 -67
- package/lib/agents/preset-roster.d.ts +22 -0
- package/lib/agents/preset-roster.js +87 -0
- package/lib/agents/role-tool-policy.d.ts +46 -0
- package/lib/agents/role-tool-policy.js +205 -0
- package/lib/agents/root-tool-policy.d.ts +28 -124
- package/lib/agents/root-tool-policy.js +164 -237
- package/lib/agents/selected-skills-provider.d.ts +2 -6
- package/lib/agents/selected-skills-provider.js +27 -84
- package/lib/capital-settings.d.ts +27 -0
- package/lib/capital-settings.js +23 -0
- package/lib/chart/artifact-ref.d.ts +3 -8
- package/lib/chart/artifact-ref.js +3 -8
- package/lib/chart/downsample.d.ts +2 -5
- package/lib/chart/downsample.js +3 -8
- package/lib/chart/errors.d.ts +3 -7
- package/lib/chart/errors.js +1 -2
- package/lib/chart/event-context.d.ts +8 -0
- package/lib/chart/event-context.js +39 -0
- package/lib/chart/events.d.ts +21 -78
- package/lib/chart/events.js +40 -189
- package/lib/chart/html.d.ts +2 -5
- package/lib/chart/html.js +8 -15
- package/lib/chart/runtime.d.ts +5 -18
- package/lib/chart/runtime.js +14 -34
- package/lib/chart/series.d.ts +10 -17
- package/lib/chart/series.js +22 -47
- package/lib/chart/source-token.d.ts +5 -13
- package/lib/chart/source-token.js +18 -17
- package/lib/chart/spec.d.ts +6 -12
- package/lib/chart/spec.js +7 -31
- package/lib/chart/tool.d.ts +6 -29
- package/lib/chart/tool.js +9 -23
- package/lib/chart-render-entry.d.ts +9 -0
- package/lib/chart-render-entry.js +17 -0
- package/lib/chart-source-entry.d.ts +13 -0
- package/lib/chart-source-entry.js +20 -0
- package/lib/collect-entry.d.ts +9 -0
- package/lib/collect-entry.js +10 -0
- package/lib/collector-runtime.d.ts +22 -0
- package/lib/collector-runtime.js +97 -0
- package/lib/data-collector/args.d.ts +6 -24
- package/lib/data-collector/args.js +6 -24
- package/lib/data-collector/dataset-tools.d.ts +7 -1
- package/lib/data-collector/dataset-tools.js +27 -56
- package/lib/data-collector/describe.d.ts +8 -33
- package/lib/data-collector/describe.js +17 -70
- package/lib/data-collector/hub.d.ts +27 -85
- package/lib/data-collector/hub.js +19 -85
- package/lib/data-collector/query-time.d.ts +7 -26
- package/lib/data-collector/query-time.js +8 -27
- package/lib/data-collector/store.d.ts +40 -117
- package/lib/data-collector/store.js +32 -111
- package/lib/data-collector/time-axis.d.ts +16 -76
- package/lib/data-collector/time-axis.js +25 -111
- package/lib/data-collector/tools.d.ts +6 -6
- package/lib/data-collector/tools.js +34 -43
- package/lib/dataset-read-entry.d.ts +9 -0
- package/lib/dataset-read-entry.js +7 -0
- package/lib/index.d.ts +7 -117
- package/lib/index.js +43 -422
- package/lib/net/eastmoney-client.d.ts +13 -65
- package/lib/net/eastmoney-client.js +12 -53
- package/lib/net/throttle.d.ts +2 -13
- package/lib/net/throttle.js +3 -14
- package/lib/ocr/client.d.ts +12 -21
- package/lib/ocr/client.js +23 -40
- package/lib/ocr/markdown.d.ts +20 -26
- package/lib/ocr/markdown.js +23 -36
- package/lib/ocr/store.d.ts +14 -47
- package/lib/ocr/store.js +15 -45
- package/lib/ocr-entry.d.ts +9 -0
- package/lib/ocr-entry.js +10 -0
- package/lib/policy-entry.d.ts +17 -0
- package/lib/policy-entry.js +22 -0
- package/lib/preset-claim.d.ts +22 -0
- package/lib/preset-claim.js +24 -0
- package/lib/preset-ids.d.ts +38 -0
- package/lib/preset-ids.js +46 -0
- package/lib/retriever-config.d.ts +37 -0
- package/lib/retriever-config.js +36 -0
- package/lib/retriever-runtime.d.ts +31 -0
- package/lib/retriever-runtime.js +82 -0
- package/lib/root-extras-entry.d.ts +17 -0
- package/lib/root-extras-entry.js +25 -0
- package/lib/runtime-entry.d.ts +15 -0
- package/lib/runtime-entry.js +5 -0
- package/lib/runtime.d.ts +21 -0
- package/lib/runtime.js +33 -0
- package/lib/search-entry.d.ts +9 -0
- package/lib/search-entry.js +9 -0
- package/lib/sources/eastmoney-http.js +20 -68
- package/lib/sources/ftshare-rest.d.ts +6 -34
- package/lib/sources/ftshare-rest.js +84 -292
- package/lib/sources/fuyao-core.d.ts +8 -18
- package/lib/sources/fuyao-core.js +10 -21
- package/lib/sources/fuyao-rest.d.ts +2 -6
- package/lib/sources/fuyao-rest.js +23 -102
- package/lib/sources/security-code.d.ts +6 -12
- package/lib/sources/security-code.js +6 -12
- package/lib/sources/security-identity.d.ts +1 -4
- package/lib/sources/security-identity.js +3 -8
- package/lib/sources/tencent-http.js +14 -28
- package/lib/sources/tencent-public-core.d.ts +8 -60
- package/lib/sources/tencent-public-core.js +12 -77
- package/lib/sources/wind-mcp.d.ts +0 -5
- package/lib/sources/wind-mcp.js +23 -78
- package/lib/sources-entry.d.ts +9 -0
- package/lib/sources-entry.js +9 -0
- package/lib/time/tools.d.ts +8 -31
- package/lib/time/tools.js +30 -66
- package/lib/time-entry.d.ts +9 -0
- package/lib/time-entry.js +7 -0
- package/lib/tool-exec.d.ts +45 -11
- package/lib/tool-exec.js +59 -6
- package/lib/user-customization.d.ts +14 -0
- package/lib/user-customization.js +47 -0
- package/lib/watchlist/tools.d.ts +3 -13
- package/lib/watchlist/tools.js +10 -30
- package/lib/watchlist-entry.d.ts +9 -0
- package/lib/watchlist-entry.js +6 -0
- package/lib/web-retriever/engines.d.ts +6 -11
- package/lib/web-retriever/engines.js +9 -14
- package/lib/web-retriever/html-markdown.d.ts +14 -34
- package/lib/web-retriever/html-markdown.js +39 -75
- package/lib/web-retriever/local-fetch.d.ts +22 -58
- package/lib/web-retriever/local-fetch.js +33 -77
- package/lib/web-retriever/retriever.d.ts +6 -18
- package/lib/web-retriever/retriever.js +7 -7
- package/lib/web-retriever/sources.d.ts +37 -124
- package/lib/web-retriever/sources.js +64 -196
- package/lib/web-retriever/tools.d.ts +43 -27
- package/lib/web-retriever/tools.js +99 -137
- package/lib/web-retriever/wind-client.d.ts +14 -31
- package/lib/web-retriever/wind-client.js +16 -32
- package/locale/zh.json +1 -1
- package/package.json +67 -7
- package/preset/capital-generation/README.md +87 -53
- package/preset/capital-generation/agent.patch.yml +526 -337
- package/preset/capital-generation/skills/capital-data-profile/SKILL.md +2 -2
- package/preset/capital-generation/skills/capital-data-protocol/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-equity-research/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-event-impact/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-orchestration/SKILL.md +15 -19
- package/preset/capital-generation/skills/capital-portfolio-review/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-visualization-protocol/SKILL.md +9 -12
- package/preset/capital-generation/skills/capital-web-evidence/SKILL.md +1 -1
- package/watchlist/client.js +15 -148
- package/watchlist/client.src.cjs +148 -662
- package/watchlist/holding-rules.js +11 -38
- package/watchlist/index.js +127 -397
- package/watchlist/package.json +2 -2
|
@@ -1,42 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 本仓的东财出口一度是**两份互不知情的实现**:`src/data-collector` 侧经由
|
|
7
|
-
* `src/sources/eastmoney-http.ts` 的裸 `fetch`(无 sleep、无队列),web_retriever 侧是我新加的
|
|
8
|
-
* 第二个客户端。两份独立节流 = 同一个出口 IP 上两个速率,等于没限。
|
|
9
|
-
*
|
|
10
|
-
* 社区实测的东财风控阈值是「每秒 >5 次 / 单 IP 并发 ≥10 / 1 分钟 ≥200 次 / 5 分钟 ≥300 次 →
|
|
11
|
-
* 临时封 IP」,参考实践因此要求 `em_get()` 串行 + 限流。本仓 `DataCollectorHub` 的 FIFO 只保证
|
|
12
|
-
* "同一时刻只跑一个请求",**不含最小间隔**——串行 ≠ 节流,连打十个请求仍是一秒十次。
|
|
13
|
-
*
|
|
14
|
-
* ## 分层(刻意如此,不要合并)
|
|
15
|
-
*
|
|
16
|
-
* - **数据面**按「可复用性」分两处,这是有意的:
|
|
17
|
-
* - 可复用的**数值/分页**端点(行情、财务、资金流)走 `DataCollectorHub`,落成不可变 Dataset ——
|
|
18
|
-
* 那里 Dataset 复用是**收益**;
|
|
19
|
-
* - 流式的**文本**来源(快讯、个股新闻、研报)走 web_retriever 的具名来源工具 ——
|
|
20
|
-
* 那里 Dataset 复用是**语义错误**("最新 20 条快讯"不是可复用快照)。
|
|
21
|
-
* - **网络面**只有这一份:两条数据面都从这里取请求路径与节流器,**共享同一个 process 级实例**。
|
|
22
|
-
* 这正是"切勿一刀切"的落点:切的判据是**可复用性**(数据面),不是 provider,也不是网络面。
|
|
23
|
-
*
|
|
24
|
-
* ## 边界
|
|
25
|
-
*
|
|
26
|
-
* 本模块**不**做 SSRF 出口校验,但**出口校验在哪一份实现里**必须说清楚:
|
|
27
|
-
* - web_retriever 侧的三个 `eastmoney_*` 工具通过 `createEastmoneyClient({ transport })`
|
|
28
|
-
* 注入 `createHttpRequester` 的闸门实现(DNS 公网 IP 校验 + 手动重定向重校验 + 大小/超时上限),
|
|
29
|
-
* 与其余六个来源共用同一份出口治理;
|
|
30
|
-
* - 数据面(Hub 侧能力)用默认 transport(裸 `fetch`):URL 全部是代码内常量 + 白名单参数,
|
|
31
|
-
* 模型不能投喂任意 URL,这是既有行为。
|
|
32
|
-
* 闸门只有 `createHttpRequester` 一份实现;本层只加节流与状态归类,不复制第二套校验
|
|
33
|
-
* (AGENTS.md §9.7)。
|
|
34
|
-
*/
|
|
35
|
-
/**
|
|
36
|
-
* 东财系请求的**共享最小间隔**(毫秒)。
|
|
37
|
-
*
|
|
38
|
-
* 1000ms = 1 次/秒,低于「>5 次/秒」阈值,并同时满足「1 分钟 200 次 / 5 分钟 300 次」这类累计窗口口径。
|
|
2
|
+
* Shared eastmoney network face: one process-wide throttle and one request path for both data
|
|
3
|
+
* planes (Hub datasets and web_retriever text). Eastmoney limits by egress IP, so two throttles
|
|
4
|
+
* equal no throttle. Throttling/status mapping live only here; SSRF checking, when needed, is
|
|
5
|
+
* injected via `transport` and owned by the caller's `createHttpRequester`.
|
|
39
6
|
*/
|
|
7
|
+
/** 1 request/second, below eastmoney's >5/s trigger and within its cumulative windows. */
|
|
40
8
|
export declare const EASTMONEY_MIN_INTERVAL_MS = 1000;
|
|
41
9
|
export interface EastmoneyFetchInit {
|
|
42
10
|
headers?: Record<string, string>;
|
|
@@ -44,42 +12,30 @@ export interface EastmoneyFetchInit {
|
|
|
44
12
|
}
|
|
45
13
|
export interface EastmoneyResponse {
|
|
46
14
|
status: number;
|
|
47
|
-
/** 注入式 transport(闸门适配层)可以不提供 headers。 */
|
|
48
15
|
headers?: Headers;
|
|
49
16
|
text: string;
|
|
50
17
|
}
|
|
51
18
|
export interface EastmoneyClient {
|
|
52
19
|
/**
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
* 刻意**不**解析 JSON、不定义错误码:两个调用方的错误码语义不同(数据面是
|
|
57
|
-
* `eastmoney_http_error` / `eastmoney_rate_limit`,web_retriever 是 `SourceError` 的
|
|
58
|
-
* `UPSTREAM` / `NETWORK`),在这一层统一会覆盖掉调用方自己的契约。
|
|
59
|
-
*/
|
|
20
|
+
* Send and read text: serialization, minimum interval, headers, status classification. No JSON
|
|
21
|
+
* or error codes; callers keep their own semantics.
|
|
22
|
+
*/
|
|
60
23
|
fetchText(url: string, init?: EastmoneyFetchInit): Promise<EastmoneyResponse>;
|
|
61
|
-
/**
|
|
62
|
-
* 同上,但把响应解析成 JSON。
|
|
63
|
-
*
|
|
64
|
-
* 为什么与 `fetchText` 分开而不是让调用方自己 `JSON.parse(text)`:数据面的既有测试用具
|
|
65
|
-
* **只有 `json()` 没有 `text()`** 的假 Response。走 `response.json()` 既保持数据面原有形状,
|
|
66
|
-
* 也让"解析失败"能在这一层被归成同一种错误(调用方仍可自行决定错误码)。
|
|
67
|
-
*/
|
|
24
|
+
/** Separate from fetchText because data-plane test doubles only provide `json()`, and parse failures classify here. */
|
|
68
25
|
fetchJson(url: string, init?: EastmoneyFetchInit): Promise<unknown>;
|
|
69
26
|
}
|
|
70
|
-
|
|
27
|
+
/** Transport error; `status` is set only when upstream returned an HTTP error status. */
|
|
71
28
|
export declare class EastmoneyTransportError extends Error {
|
|
72
29
|
readonly status?: number | undefined;
|
|
73
30
|
constructor(message: string, status?: number | undefined);
|
|
74
31
|
}
|
|
75
|
-
/** 传输器返回的最小响应形状(真实 `Response` 天然满足;闸门适配层按此构造)。 */
|
|
76
32
|
export interface EastmoneyTransportResponse {
|
|
77
33
|
status: number;
|
|
78
34
|
headers?: Headers;
|
|
79
35
|
text(): Promise<string>;
|
|
80
36
|
json(): Promise<unknown>;
|
|
81
37
|
}
|
|
82
|
-
/**
|
|
38
|
+
/** Injectable transport; defaults to bare fetch, web_retriever injects its SSRF-gated implementation. */
|
|
83
39
|
export type EastmoneyTransport = (url: string, init: {
|
|
84
40
|
headers: Record<string, string>;
|
|
85
41
|
signal?: AbortSignal;
|
|
@@ -87,18 +43,10 @@ export type EastmoneyTransport = (url: string, init: {
|
|
|
87
43
|
export declare function createEastmoneyClient(options?: {
|
|
88
44
|
userAgent?: string;
|
|
89
45
|
minIntervalMs?: number;
|
|
90
|
-
/** 注入进程级共享节流器时 `minIntervalMs` 失效(节流只该有一份)。 */
|
|
91
46
|
throttle?: <T>(task: () => Promise<T>) => Promise<T>;
|
|
92
47
|
transport?: EastmoneyTransport;
|
|
93
48
|
}): EastmoneyClient;
|
|
94
|
-
/**
|
|
95
|
-
* **进程级共享节流器**:数据面与 web_retriever 侧可能各持一个客户端实例(transport 不同),
|
|
96
|
-
* 但东财风控按出口 IP 计,节流必须仍只有一份。两侧客户端都从这里取。
|
|
97
|
-
*/
|
|
49
|
+
/** Process-wide throttle: both clients (different transports) hit the same egress IP, so they share one. */
|
|
98
50
|
export declare function sharedEastmoneyThrottle(): <T>(task: () => Promise<T>) => Promise<T>;
|
|
99
|
-
/**
|
|
100
|
-
* **进程级单例**(数据面用,默认裸 fetch transport):所有默认客户端共享同一个节流器,
|
|
101
|
-
* 等于回到"两份独立限流"的老问题的解法。宿主进程内所有东财请求都应经这里或
|
|
102
|
-
* 显式注入 `sharedEastmoneyThrottle()` 的实例。
|
|
103
|
-
*/
|
|
51
|
+
/** Process-wide client for the data plane (bare fetch), sharing the process throttle. */
|
|
104
52
|
export declare function sharedEastmoneyClient(): EastmoneyClient;
|
|
@@ -1,46 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 本仓的东财出口一度是**两份互不知情的实现**:`src/data-collector` 侧经由
|
|
7
|
-
* `src/sources/eastmoney-http.ts` 的裸 `fetch`(无 sleep、无队列),web_retriever 侧是我新加的
|
|
8
|
-
* 第二个客户端。两份独立节流 = 同一个出口 IP 上两个速率,等于没限。
|
|
9
|
-
*
|
|
10
|
-
* 社区实测的东财风控阈值是「每秒 >5 次 / 单 IP 并发 ≥10 / 1 分钟 ≥200 次 / 5 分钟 ≥300 次 →
|
|
11
|
-
* 临时封 IP」,参考实践因此要求 `em_get()` 串行 + 限流。本仓 `DataCollectorHub` 的 FIFO 只保证
|
|
12
|
-
* "同一时刻只跑一个请求",**不含最小间隔**——串行 ≠ 节流,连打十个请求仍是一秒十次。
|
|
13
|
-
*
|
|
14
|
-
* ## 分层(刻意如此,不要合并)
|
|
15
|
-
*
|
|
16
|
-
* - **数据面**按「可复用性」分两处,这是有意的:
|
|
17
|
-
* - 可复用的**数值/分页**端点(行情、财务、资金流)走 `DataCollectorHub`,落成不可变 Dataset ——
|
|
18
|
-
* 那里 Dataset 复用是**收益**;
|
|
19
|
-
* - 流式的**文本**来源(快讯、个股新闻、研报)走 web_retriever 的具名来源工具 ——
|
|
20
|
-
* 那里 Dataset 复用是**语义错误**("最新 20 条快讯"不是可复用快照)。
|
|
21
|
-
* - **网络面**只有这一份:两条数据面都从这里取请求路径与节流器,**共享同一个 process 级实例**。
|
|
22
|
-
* 这正是"切勿一刀切"的落点:切的判据是**可复用性**(数据面),不是 provider,也不是网络面。
|
|
23
|
-
*
|
|
24
|
-
* ## 边界
|
|
25
|
-
*
|
|
26
|
-
* 本模块**不**做 SSRF 出口校验,但**出口校验在哪一份实现里**必须说清楚:
|
|
27
|
-
* - web_retriever 侧的三个 `eastmoney_*` 工具通过 `createEastmoneyClient({ transport })`
|
|
28
|
-
* 注入 `createHttpRequester` 的闸门实现(DNS 公网 IP 校验 + 手动重定向重校验 + 大小/超时上限),
|
|
29
|
-
* 与其余六个来源共用同一份出口治理;
|
|
30
|
-
* - 数据面(Hub 侧能力)用默认 transport(裸 `fetch`):URL 全部是代码内常量 + 白名单参数,
|
|
31
|
-
* 模型不能投喂任意 URL,这是既有行为。
|
|
32
|
-
* 闸门只有 `createHttpRequester` 一份实现;本层只加节流与状态归类,不复制第二套校验
|
|
33
|
-
* (AGENTS.md §9.7)。
|
|
2
|
+
* Shared eastmoney network face: one process-wide throttle and one request path for both data
|
|
3
|
+
* planes (Hub datasets and web_retriever text). Eastmoney limits by egress IP, so two throttles
|
|
4
|
+
* equal no throttle. Throttling/status mapping live only here; SSRF checking, when needed, is
|
|
5
|
+
* injected via `transport` and owned by the caller's `createHttpRequester`.
|
|
34
6
|
*/
|
|
35
7
|
import { createRequestThrottle } from './throttle.js';
|
|
36
|
-
/**
|
|
37
|
-
* 东财系请求的**共享最小间隔**(毫秒)。
|
|
38
|
-
*
|
|
39
|
-
* 1000ms = 1 次/秒,低于「>5 次/秒」阈值,并同时满足「1 分钟 200 次 / 5 分钟 300 次」这类累计窗口口径。
|
|
40
|
-
*/
|
|
8
|
+
/** 1 request/second, below eastmoney's >5/s trigger and within its cumulative windows. */
|
|
41
9
|
export const EASTMONEY_MIN_INTERVAL_MS = 1000;
|
|
42
10
|
const DEFAULT_USER_AGENT = 'Mozilla/5.0';
|
|
43
|
-
|
|
11
|
+
/** Transport error; `status` is set only when upstream returned an HTTP error status. */
|
|
44
12
|
export class EastmoneyTransportError extends Error {
|
|
45
13
|
status;
|
|
46
14
|
constructor(message, status) {
|
|
@@ -53,7 +21,7 @@ export function createEastmoneyClient(options = {}) {
|
|
|
53
21
|
const userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
|
|
54
22
|
const throttle = options.throttle ?? createRequestThrottle(options.minIntervalMs ?? EASTMONEY_MIN_INTERVAL_MS);
|
|
55
23
|
const transport = options.transport ?? createEastmoneyTransport();
|
|
56
|
-
/**
|
|
24
|
+
/** Throttle + send + status check, shared by the text and json paths. */
|
|
57
25
|
const send = (url, init) => throttle(async () => {
|
|
58
26
|
if (init.signal?.aborted) {
|
|
59
27
|
const error = new Error('eastmoney request aborted');
|
|
@@ -68,17 +36,15 @@ export function createEastmoneyClient(options = {}) {
|
|
|
68
36
|
});
|
|
69
37
|
}
|
|
70
38
|
catch (error) {
|
|
71
|
-
//
|
|
39
|
+
// Abort passes through unchanged so callers can tell cancellation from network failure.
|
|
72
40
|
if (error instanceof Error && error.name === 'AbortError')
|
|
73
41
|
throw error;
|
|
74
|
-
//
|
|
75
|
-
// 原样抛出:再包一层会把"取消"降级成网络失败(docs/dev/web-retriever.md §4.2),也丢掉失败原因。
|
|
42
|
+
// Errors that already carry a classification code pass through: wrapping would downgrade cancellation to a network failure.
|
|
76
43
|
if (typeof error?.code === 'string')
|
|
77
44
|
throw error;
|
|
78
45
|
throw new EastmoneyTransportError(`eastmoney request failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
79
46
|
}
|
|
80
|
-
//
|
|
81
|
-
// 用 `ok` 会让它们在 200 上误判成失败(真实的 fetch Response 两者都有)。
|
|
47
|
+
// Check status, not response.ok: some test doubles only expose status/text/json.
|
|
82
48
|
if (response.status >= 400) {
|
|
83
49
|
throw new EastmoneyTransportError(`eastmoney returned HTTP ${response.status}`, response.status);
|
|
84
50
|
}
|
|
@@ -98,20 +64,13 @@ function createEastmoneyTransport() {
|
|
|
98
64
|
return (url, init) => globalThis.fetch(url, init);
|
|
99
65
|
}
|
|
100
66
|
let sharedThrottle;
|
|
101
|
-
/**
|
|
102
|
-
* **进程级共享节流器**:数据面与 web_retriever 侧可能各持一个客户端实例(transport 不同),
|
|
103
|
-
* 但东财风控按出口 IP 计,节流必须仍只有一份。两侧客户端都从这里取。
|
|
104
|
-
*/
|
|
67
|
+
/** Process-wide throttle: both clients (different transports) hit the same egress IP, so they share one. */
|
|
105
68
|
export function sharedEastmoneyThrottle() {
|
|
106
69
|
sharedThrottle ??= createRequestThrottle(EASTMONEY_MIN_INTERVAL_MS);
|
|
107
70
|
return sharedThrottle;
|
|
108
71
|
}
|
|
109
72
|
let shared;
|
|
110
|
-
/**
|
|
111
|
-
* **进程级单例**(数据面用,默认裸 fetch transport):所有默认客户端共享同一个节流器,
|
|
112
|
-
* 等于回到"两份独立限流"的老问题的解法。宿主进程内所有东财请求都应经这里或
|
|
113
|
-
* 显式注入 `sharedEastmoneyThrottle()` 的实例。
|
|
114
|
-
*/
|
|
73
|
+
/** Process-wide client for the data plane (bare fetch), sharing the process throttle. */
|
|
115
74
|
export function sharedEastmoneyClient() {
|
|
116
75
|
shared ??= createEastmoneyClient({ throttle: sharedEastmoneyThrottle() });
|
|
117
76
|
return shared;
|
package/lib/net/throttle.d.ts
CHANGED
|
@@ -1,16 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* ## 为什么是"链"而不是 `await sleep()`
|
|
5
|
-
*
|
|
6
|
-
* 多个工具调用可能**并发**进入(模型在同一条消息里发多个调用时,框架会把它们放进并发池)。
|
|
7
|
-
* 各自 sleep 之后仍会同时发出请求——间隔就白设了。串到一条 Promise 链上,才能保证
|
|
8
|
-
* "上一个请求**发出后**至少 N 毫秒,下一个才发出"。链本身不会因某个任务失败而断掉。
|
|
9
|
-
*
|
|
10
|
-
* ## 为什么是独立的中性模块
|
|
11
|
-
*
|
|
12
|
-
* 它纯属**网络面**关切,不属于任何一条数据面:东财 Hub 侧(`src/sources/eastmoney-http.ts`)
|
|
13
|
-
* 与 web_retriever 侧(`src/web-retriever/sources.ts`)都要用。若把它放在任一侧,另一侧就得
|
|
14
|
-
* 反向依赖对方——那正是"客户端被复制"的前身(见 `src/net/eastmoney-client.ts` 的分层说明)。
|
|
2
|
+
* Serialize requests to one upstream into a chain with a minimum gap. Concurrent callers must not
|
|
3
|
+
* each sleep and then fire together; chaining guarantees the gap after the previous start.
|
|
15
4
|
*/
|
|
16
5
|
export declare function createRequestThrottle(minIntervalMs: number): <T>(task: () => Promise<T>) => Promise<T>;
|
package/lib/net/throttle.js
CHANGED
|
@@ -1,21 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* ## 为什么是"链"而不是 `await sleep()`
|
|
5
|
-
*
|
|
6
|
-
* 多个工具调用可能**并发**进入(模型在同一条消息里发多个调用时,框架会把它们放进并发池)。
|
|
7
|
-
* 各自 sleep 之后仍会同时发出请求——间隔就白设了。串到一条 Promise 链上,才能保证
|
|
8
|
-
* "上一个请求**发出后**至少 N 毫秒,下一个才发出"。链本身不会因某个任务失败而断掉。
|
|
9
|
-
*
|
|
10
|
-
* ## 为什么是独立的中性模块
|
|
11
|
-
*
|
|
12
|
-
* 它纯属**网络面**关切,不属于任何一条数据面:东财 Hub 侧(`src/sources/eastmoney-http.ts`)
|
|
13
|
-
* 与 web_retriever 侧(`src/web-retriever/sources.ts`)都要用。若把它放在任一侧,另一侧就得
|
|
14
|
-
* 反向依赖对方——那正是"客户端被复制"的前身(见 `src/net/eastmoney-client.ts` 的分层说明)。
|
|
2
|
+
* Serialize requests to one upstream into a chain with a minimum gap. Concurrent callers must not
|
|
3
|
+
* each sleep and then fire together; chaining guarantees the gap after the previous start.
|
|
15
4
|
*/
|
|
16
5
|
export function createRequestThrottle(minIntervalMs) {
|
|
17
6
|
if (!Number.isFinite(minIntervalMs) || minIntervalMs < 0) {
|
|
18
|
-
//
|
|
7
|
+
// A misconfigured throttle silently means "no throttle"; upstreams answer with empty data or an IP ban.
|
|
19
8
|
throw new Error(`createRequestThrottle: minIntervalMs must be a non-negative finite number, got ${String(minIntervalMs)}`);
|
|
20
9
|
}
|
|
21
10
|
let chain = Promise.resolve();
|
package/lib/ocr/client.d.ts
CHANGED
|
@@ -1,26 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* PaddleOCR
|
|
2
|
+
* Async PaddleOCR jobs client — the plugin's only OCR egress implementation.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* `POST /api/v2/ocr/jobs` → `{code:0,msg:'Success',data:{jobId}}`
|
|
6
|
-
* → `GET /api/v2/ocr/jobs/{jobId}` 轮询 `data.state`:`pending` → `running`
|
|
7
|
-
* (带 `extractProgress.{totalPages,extractedPages}`)→ `done`(带 `resultUrl.jsonUrl`)
|
|
8
|
-
* → `GET jsonUrl` 拿 **JSONL**(每行一个批次,`result.layoutParsingResults[]` 一项一页)。
|
|
4
|
+
* Flow: POST job → poll state → fetch result JSONL (one layout result per page).
|
|
9
5
|
*
|
|
10
|
-
* ⛔
|
|
11
|
-
* 1.
|
|
12
|
-
*
|
|
13
|
-
* 然后拿 `undefined` 去轮询——所以每个响应都先看 `code`。
|
|
14
|
-
* 2. **`optionalPayload` 的未知键被静默忽略**(塞 `zzzBogusKey` 照样返回 Success)。
|
|
15
|
-
* 公开文档那页讲的其实是同步 `/layout-parsing`,`mergeTables` / `prettifyMarkdown`
|
|
16
|
-
* 在 jobs 上是否生效无法证明,因此**只传官方示例里出现过的三个键**,
|
|
17
|
-
* 不拿没验证过的开关换"看起来能配"的假象。表格噪音靠 `markdown.ts` 自己处理。
|
|
6
|
+
* ⛔ Two response-shape facts determine the code below:
|
|
7
|
+
* 1. Business failures arrive as HTTP 200 with `{ code: nonzero, jobId absent }`, so always check `code`.
|
|
8
|
+
* 2. Unknown `optionalPayload` keys are silently ignored; send only documented example keys.
|
|
18
9
|
*/
|
|
19
10
|
import type { HttpOutcome, HttpRequest } from '../web-retriever/local-fetch.js';
|
|
20
11
|
import { type OcrDocument } from './markdown.js';
|
|
21
12
|
export declare const PADDLE_OCR_DEFAULT_ENDPOINT = "https://paddleocr.aistudio-app.com/api/v2/ocr/jobs";
|
|
22
13
|
export declare const PADDLE_OCR_DEFAULT_MODEL = "PaddleOCR-VL-1.6";
|
|
23
|
-
/**
|
|
14
|
+
/** Upstream PDF page limit; reject earlier than letting the remote job fail. */
|
|
24
15
|
export declare const PADDLE_OCR_MAX_PAGES = 100;
|
|
25
16
|
export type OcrErrorCode = 'NO_CREDENTIAL' | 'UPSTREAM' | 'JOB_FAILED' | 'RESULT_INVALID' | 'TOO_MANY_PAGES' | 'TIMEOUT';
|
|
26
17
|
export declare class OcrError extends Error {
|
|
@@ -31,7 +22,7 @@ export interface OcrProgress {
|
|
|
31
22
|
extracted_pages: number;
|
|
32
23
|
total_pages: number;
|
|
33
24
|
}
|
|
34
|
-
/**
|
|
25
|
+
/** Poll status, narrowed to fields used here. */
|
|
35
26
|
export interface OcrJobState {
|
|
36
27
|
state: string;
|
|
37
28
|
progress?: OcrProgress;
|
|
@@ -39,9 +30,9 @@ export interface OcrJobState {
|
|
|
39
30
|
errorMsg?: string;
|
|
40
31
|
}
|
|
41
32
|
export interface OcrSubmitInput {
|
|
42
|
-
/**
|
|
33
|
+
/** Public PDF URL fetched by the provider; no local bytes are involved. */
|
|
43
34
|
fileUrl?: string;
|
|
44
|
-
/**
|
|
35
|
+
/** Local document bytes uploaded as multipart. */
|
|
45
36
|
file?: {
|
|
46
37
|
bytes: Uint8Array;
|
|
47
38
|
filename: string;
|
|
@@ -53,7 +44,7 @@ export interface OcrWaitOutcome {
|
|
|
53
44
|
status: 'done' | 'pending';
|
|
54
45
|
job_id: string;
|
|
55
46
|
progress?: OcrProgress;
|
|
56
|
-
/**
|
|
47
|
+
/** Present whenever status is done. */
|
|
57
48
|
document?: OcrDocument;
|
|
58
49
|
json_url?: string;
|
|
59
50
|
elapsed_ms: number;
|
|
@@ -66,7 +57,7 @@ export interface PaddleOcrClientOptions {
|
|
|
66
57
|
pollIntervalMs?: number;
|
|
67
58
|
waitBudgetMs?: number;
|
|
68
59
|
resultMaxBytes?: number;
|
|
69
|
-
/**
|
|
60
|
+
/** Injectable sleep keeps polling cancellable. */
|
|
70
61
|
sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
|
|
71
62
|
now?: () => number;
|
|
72
63
|
}
|
|
@@ -75,7 +66,7 @@ export interface PaddleOcrClient {
|
|
|
75
66
|
submit(input: OcrSubmitInput): Promise<string>;
|
|
76
67
|
poll(jobId: string, signal: AbortSignal): Promise<OcrJobState>;
|
|
77
68
|
wait(jobId: string, signal: AbortSignal, progress?: OcrProgress): Promise<OcrWaitOutcome>;
|
|
78
|
-
/**
|
|
69
|
+
/** Submit and poll until terminal, or return pending when the wait budget is exhausted. */
|
|
79
70
|
parse(input: OcrSubmitInput): Promise<OcrWaitOutcome>;
|
|
80
71
|
load(jsonUrl: string, signal: AbortSignal): Promise<OcrDocument>;
|
|
81
72
|
}
|
package/lib/ocr/client.js
CHANGED
|
@@ -1,30 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* PaddleOCR
|
|
2
|
+
* Async PaddleOCR jobs client — the plugin's only OCR egress implementation.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* `POST /api/v2/ocr/jobs` → `{code:0,msg:'Success',data:{jobId}}`
|
|
6
|
-
* → `GET /api/v2/ocr/jobs/{jobId}` 轮询 `data.state`:`pending` → `running`
|
|
7
|
-
* (带 `extractProgress.{totalPages,extractedPages}`)→ `done`(带 `resultUrl.jsonUrl`)
|
|
8
|
-
* → `GET jsonUrl` 拿 **JSONL**(每行一个批次,`result.layoutParsingResults[]` 一项一页)。
|
|
4
|
+
* Flow: POST job → poll state → fetch result JSONL (one layout result per page).
|
|
9
5
|
*
|
|
10
|
-
* ⛔
|
|
11
|
-
* 1.
|
|
12
|
-
*
|
|
13
|
-
* 然后拿 `undefined` 去轮询——所以每个响应都先看 `code`。
|
|
14
|
-
* 2. **`optionalPayload` 的未知键被静默忽略**(塞 `zzzBogusKey` 照样返回 Success)。
|
|
15
|
-
* 公开文档那页讲的其实是同步 `/layout-parsing`,`mergeTables` / `prettifyMarkdown`
|
|
16
|
-
* 在 jobs 上是否生效无法证明,因此**只传官方示例里出现过的三个键**,
|
|
17
|
-
* 不拿没验证过的开关换"看起来能配"的假象。表格噪音靠 `markdown.ts` 自己处理。
|
|
6
|
+
* ⛔ Two response-shape facts determine the code below:
|
|
7
|
+
* 1. Business failures arrive as HTTP 200 with `{ code: nonzero, jobId absent }`, so always check `code`.
|
|
8
|
+
* 2. Unknown `optionalPayload` keys are silently ignored; send only documented example keys.
|
|
18
9
|
*/
|
|
19
10
|
import { buildOcrDocument } from './markdown.js';
|
|
20
11
|
export const PADDLE_OCR_DEFAULT_ENDPOINT = 'https://paddleocr.aistudio-app.com/api/v2/ocr/jobs';
|
|
21
12
|
export const PADDLE_OCR_DEFAULT_MODEL = 'PaddleOCR-VL-1.6';
|
|
22
|
-
/**
|
|
13
|
+
/** Upstream PDF page limit; reject earlier than letting the remote job fail. */
|
|
23
14
|
export const PADDLE_OCR_MAX_PAGES = 100;
|
|
24
15
|
const DEFAULT_POLL_INTERVAL_MS = 5_000;
|
|
25
|
-
/**
|
|
16
|
+
/** Wait budget stays below the tool timeout, leaving time for the response and disk write. */
|
|
26
17
|
const DEFAULT_WAIT_BUDGET_MS = 200_000;
|
|
27
|
-
/**
|
|
18
|
+
/** JSONL response cap; exceeding it fails loudly rather than parsing a partial document. */
|
|
28
19
|
const DEFAULT_RESULT_MAX_BYTES = 8 * 1024 * 1024;
|
|
29
20
|
export class OcrError extends Error {
|
|
30
21
|
code;
|
|
@@ -55,16 +46,14 @@ function sleepAbortable(ms, signal) {
|
|
|
55
46
|
});
|
|
56
47
|
}
|
|
57
48
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
* 状态码只在消息文本里。这里按文本识别鉴权类失败(401/403)**只为了把"该去卡片填 token"
|
|
62
|
-
* 这条最有用的话送到模型面前**;识别不到就退回 `UPSTREAM`,绝不假装知道上游说了什么。
|
|
49
|
+
* Map transport exceptions to OCR codes. `LocalFetchError` carries the HTTP status
|
|
50
|
+
* only in its message, so recognize 401/403 to point at the token card; otherwise
|
|
51
|
+
* fall back to UPSTREAM rather than infer upstream details.
|
|
63
52
|
*/
|
|
64
53
|
function mapTransportFailure(error) {
|
|
65
54
|
const message = error instanceof Error ? error.message : String(error);
|
|
66
55
|
if (/HTTP\s*(401|403)\b/u.test(message)) {
|
|
67
|
-
return new OcrError('NO_CREDENTIAL', `PaddleOCR 拒绝鉴权(${message}):请在 设置 → 插件 →
|
|
56
|
+
return new OcrError('NO_CREDENTIAL', `PaddleOCR 拒绝鉴权(${message}):请在 设置 → 插件 → 自动模式填写 PaddleOCR Token`);
|
|
68
57
|
}
|
|
69
58
|
return new OcrError('UPSTREAM', message);
|
|
70
59
|
}
|
|
@@ -79,11 +68,11 @@ export function createPaddleOcrClient(options) {
|
|
|
79
68
|
async function authorizedHeaders() {
|
|
80
69
|
const token = await options.resolveToken();
|
|
81
70
|
if (!token) {
|
|
82
|
-
throw new OcrError('NO_CREDENTIAL', 'PaddleOCR Token 未配置:在 设置 → 插件 → 插件配置 →
|
|
71
|
+
throw new OcrError('NO_CREDENTIAL', 'PaddleOCR Token 未配置:在 设置 → 插件 → 插件配置 → 自动模式填写(credentials 引用 PADDLE_OCR_TOKEN,环境变量同名亦可)');
|
|
83
72
|
}
|
|
84
73
|
return { authorization: `bearer ${token}` };
|
|
85
74
|
}
|
|
86
|
-
/**
|
|
75
|
+
/** Send and validate `body.code`; HTTP 200 can still be a business error. */
|
|
87
76
|
async function call(request, signal) {
|
|
88
77
|
let outcome;
|
|
89
78
|
try {
|
|
@@ -111,13 +100,13 @@ export function createPaddleOcrClient(options) {
|
|
|
111
100
|
if (code !== undefined && code !== 0) {
|
|
112
101
|
const message = typeof parsed.msg === 'string' ? parsed.msg : '(无 msg)';
|
|
113
102
|
if (code === 401)
|
|
114
|
-
throw new OcrError('NO_CREDENTIAL', `PaddleOCR 未鉴权(code 401 ${message}):请在 设置 → 插件 →
|
|
103
|
+
throw new OcrError('NO_CREDENTIAL', `PaddleOCR 未鉴权(code 401 ${message}):请在 设置 → 插件 → 自动模式填写 Token`);
|
|
115
104
|
throw new OcrError('UPSTREAM', `PaddleOCR 返回业务错误 code=${String(code)}:${message}`);
|
|
116
105
|
}
|
|
117
106
|
return parsed;
|
|
118
107
|
}
|
|
119
108
|
function optionalPayload(charts) {
|
|
120
|
-
//
|
|
109
|
+
// Send only the three documented example keys.
|
|
121
110
|
return { useDocOrientationClassify: false, useDocUnwarping: false, useChartRecognition: charts };
|
|
122
111
|
}
|
|
123
112
|
async function submit(input) {
|
|
@@ -134,15 +123,12 @@ export function createPaddleOcrClient(options) {
|
|
|
134
123
|
const form = new FormData();
|
|
135
124
|
form.append('model', model);
|
|
136
125
|
form.append('optionalPayload', JSON.stringify(payload));
|
|
137
|
-
//
|
|
138
|
-
// ArrayBuffer 背书的视图;字节确实是新分配的 ArrayBuffer,在边界上收敛一次类型。
|
|
126
|
+
// Narrow the buffer-backed view at this runtime boundary; the bytes are freshly allocated.
|
|
139
127
|
form.append('file', new Blob([file.bytes], { type: 'application/octet-stream' }), file.filename);
|
|
140
128
|
extraHeaders = {};
|
|
141
129
|
body = form;
|
|
142
130
|
}
|
|
143
|
-
// ⛔
|
|
144
|
-
// 把作业提交发成一个无鉴权请求(实测 401),而轮询那条路是带着 token 的——
|
|
145
|
-
// 两条出网路径形状不一致,恰是"只测接好的那个入口"能长期潜伏的形态。
|
|
131
|
+
// ⛔ Send both bearer and content-type headers; omitting either makes the two egress paths diverge.
|
|
146
132
|
const parsed = await call({ url: endpoint, method: 'POST', headers: { ...headers, ...extraHeaders }, body }, input.signal);
|
|
147
133
|
const jobId = asRecord(parsed.data)?.jobId;
|
|
148
134
|
if (typeof jobId !== 'string' || jobId.length === 0) {
|
|
@@ -171,10 +157,8 @@ export function createPaddleOcrClient(options) {
|
|
|
171
157
|
return outcome;
|
|
172
158
|
}
|
|
173
159
|
async function load(jsonUrl, signal) {
|
|
174
|
-
// ⛔
|
|
175
|
-
//
|
|
176
|
-
// HTTP 400 `MissingDateHeader`;不带才拿得下来(实测 200 / 308 KB)。而且把 token 发给
|
|
177
|
-
// 第三方存储域本身就是密钥外流——这条路**只**靠签名过的 URL 鉴权。
|
|
160
|
+
// ⛔ Fetch the presigned result URL without our bearer: object storage rejects mixed auth
|
|
161
|
+
// and the token must not leak to a third-party host.
|
|
178
162
|
let outcome;
|
|
179
163
|
try {
|
|
180
164
|
outcome = await options.request({ url: jsonUrl, accept: 'application/json', maxBytes: resultMaxBytes, maxContentChars: resultMaxBytes }, signal);
|
|
@@ -185,7 +169,7 @@ export function createPaddleOcrClient(options) {
|
|
|
185
169
|
throw error;
|
|
186
170
|
throw mapTransportFailure(error);
|
|
187
171
|
}
|
|
188
|
-
//
|
|
172
|
+
// Cap the result JSONL separately; an oversize response is not a usable document.
|
|
189
173
|
if (outcome.truncated) {
|
|
190
174
|
throw new OcrError('RESULT_INVALID', `解析结果超出 ${resultMaxBytes} 字节上限并被截断:整份文档作废,不给半份 markdown`);
|
|
191
175
|
}
|
|
@@ -203,8 +187,7 @@ export function createPaddleOcrClient(options) {
|
|
|
203
187
|
}
|
|
204
188
|
if (!record)
|
|
205
189
|
throw new OcrError('RESULT_INVALID', '结果 JSONL 的某一行不是对象');
|
|
206
|
-
// ⛔
|
|
207
|
-
// 判据只能是 `errorCode !== 0`——把"有 errorMsg"当失败会把每一份成功解析的文档报成失败。
|
|
190
|
+
// ⛔ Successful rows also carry errorMsg; use errorCode !== 0, not field presence.
|
|
208
191
|
const errorCode = record.errorCode;
|
|
209
192
|
const failed = typeof errorCode === 'number' ? errorCode !== 0 : typeof errorCode === 'string' ? errorCode !== '0' && errorCode.length > 0 : false;
|
|
210
193
|
const result = asRecord(record.result);
|
|
@@ -268,7 +251,7 @@ export function createPaddleOcrClient(options) {
|
|
|
268
251
|
const code = error?.code;
|
|
269
252
|
if (code === 'ABORTED' || (error instanceof Error && error.name === 'AbortError'))
|
|
270
253
|
throw error;
|
|
271
|
-
//
|
|
254
|
+
// A sleep failure means job state is unknown; do not relabel it as an upstream error.
|
|
272
255
|
throw new OcrError('RESULT_INVALID', `轮询等待异常(已轮询 ${attempts} 次):${error instanceof Error ? error.message : String(error)}`);
|
|
273
256
|
}
|
|
274
257
|
}
|
package/lib/ocr/markdown.d.ts
CHANGED
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* PaddleOCR
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* 105,639 个字符里 63,232 个(60%)是 `style='...'` 样板,去掉再转成 GFM 管道表只剩 27,156。
|
|
7
|
-
* 本仓的场景恰恰是"大量表格、图形",不处理就等于**预算的六成买的是 HTML 属性**,
|
|
8
|
-
* 而 `prettifyMarkdown` 这类 flag 在上游被静默忽略(未知键也返回 Success),帮不上。
|
|
9
|
-
*
|
|
10
|
-
* 归一化只做**无损排版**:不删正文、不改数字、不做摘要。数字一律照抄上游。
|
|
2
|
+
* PaddleOCR output -> model-readable markdown (the only implementation). Upstream emits
|
|
3
|
+
* tables as inline HTML with per-cell `style=`, so on a measured 15-page report ~60% of
|
|
4
|
+
* characters were style boilerplate; `prettifyMarkdown`-style flags are silently ignored.
|
|
5
|
+
* Normalization is lossless layout only: no text, numbers or tables are dropped.
|
|
11
6
|
*/
|
|
12
|
-
/**
|
|
7
|
+
/** One page as returned upstream (we use markdown text and image count). */
|
|
13
8
|
export interface OcrSourcePage {
|
|
14
9
|
page: number;
|
|
15
10
|
markdown: string;
|
|
@@ -17,23 +12,23 @@ export interface OcrSourcePage {
|
|
|
17
12
|
}
|
|
18
13
|
export interface OcrPageEntry {
|
|
19
14
|
page: number;
|
|
20
|
-
/**
|
|
15
|
+
/** Code-point offset into `document` (inclusive). */
|
|
21
16
|
start: number;
|
|
22
|
-
/**
|
|
17
|
+
/** End offset (exclusive). */
|
|
23
18
|
end: number;
|
|
24
19
|
chars: number;
|
|
25
|
-
/**
|
|
20
|
+
/** First heading line on the page, for the index. */
|
|
26
21
|
heading?: string;
|
|
27
22
|
}
|
|
28
23
|
export interface OcrDocument {
|
|
29
|
-
/**
|
|
24
|
+
/** Pages separated by `<!-- page:N -->`; invisible when rendered, used for page lookup. */
|
|
30
25
|
document: string;
|
|
31
26
|
pages: OcrPageEntry[];
|
|
32
27
|
chars: number;
|
|
33
28
|
images: number;
|
|
34
|
-
/**
|
|
29
|
+
/** Number of HTML tables converted to GFM. */
|
|
35
30
|
tables: number;
|
|
36
|
-
/**
|
|
31
|
+
/** Stripped style characters, so the receipt can prove layout noise, not truncation. */
|
|
37
32
|
styleCharsRemoved: number;
|
|
38
33
|
}
|
|
39
34
|
export interface OcrHit {
|
|
@@ -47,13 +42,13 @@ export interface NormalizedMarkdown {
|
|
|
47
42
|
tables: number;
|
|
48
43
|
styleCharsRemoved: number;
|
|
49
44
|
}
|
|
50
|
-
/**
|
|
45
|
+
/** Strip styles, convert HTML tables to GFM, collapse blank lines; near-identity otherwise. */
|
|
51
46
|
export declare function normalizeOcrMarkdown(source: string): NormalizedMarkdown;
|
|
52
|
-
/**
|
|
47
|
+
/** Join per-page markdown and record page offsets, used by page reads and keyword lookup. */
|
|
53
48
|
export declare function buildOcrDocument(sourcePages: OcrSourcePage[]): OcrDocument;
|
|
54
49
|
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
50
|
+
* `"1-3"` / `"5"` / `"2-4,9"` (1-based, inclusive). Out-of-range pages fail loudly:
|
|
51
|
+
* "page absent" and "not requested" must stay distinguishable in the receipt.
|
|
57
52
|
*/
|
|
58
53
|
export declare function parsePageRange(spec: string, totalPages: number): number[];
|
|
59
54
|
export interface PageExcerpt {
|
|
@@ -62,14 +57,13 @@ export interface PageExcerpt {
|
|
|
62
57
|
heading?: string;
|
|
63
58
|
text: string;
|
|
64
59
|
}
|
|
65
|
-
/**
|
|
60
|
+
/** Slice pages out of the stored text; never re-runs upstream. */
|
|
66
61
|
export declare function selectPages(document: OcrDocument, pages: number[]): PageExcerpt[];
|
|
67
62
|
/**
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* 归一化文本上做,不再出网。
|
|
63
|
+
* Keyword lookup over the persisted normalized text (no egress): the output budget is
|
|
64
|
+
* smaller than a whole report, so callers must be able to jump. Adjacent hits merge into
|
|
65
|
+
* one window, and windows never cross a page boundary — a miscited page is worse than
|
|
66
|
+
* fewer hits — which also keeps `<!-- page:N -->` markers out of excerpts.
|
|
73
67
|
*/
|
|
74
68
|
export declare function locateQuery(document: OcrDocument, query: string, limit: number, windowChars: number): OcrHit[];
|
|
75
69
|
export { PAGE_MARKER_PREFIX };
|