@v587d/capital-generation 2.8.2 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +87 -1
- package/README.md +41 -20
- package/chart-ui/README.md +1 -1
- package/chart-ui/client.src.cjs +7 -16
- package/chart-ui/index.js +12 -33
- package/chart-ui/package.json +2 -2
- package/config/client.js +18 -39
- package/config/client.src.cjs +79 -217
- package/config/index.js +22 -72
- package/config/package.json +2 -2
- package/cordis.patch.yml +46 -51
- package/lib/agents/bash-guard.d.ts +9 -55
- package/lib/agents/bash-guard.js +27 -67
- package/lib/agents/preset-roster.d.ts +22 -0
- package/lib/agents/preset-roster.js +87 -0
- package/lib/agents/role-tool-policy.d.ts +46 -0
- package/lib/agents/role-tool-policy.js +205 -0
- package/lib/agents/root-tool-policy.d.ts +28 -124
- package/lib/agents/root-tool-policy.js +164 -237
- package/lib/agents/selected-skills-provider.d.ts +2 -6
- package/lib/agents/selected-skills-provider.js +27 -84
- package/lib/capital-settings.d.ts +27 -0
- package/lib/capital-settings.js +23 -0
- package/lib/chart/artifact-ref.d.ts +3 -8
- package/lib/chart/artifact-ref.js +3 -8
- package/lib/chart/downsample.d.ts +2 -5
- package/lib/chart/downsample.js +3 -8
- package/lib/chart/errors.d.ts +3 -7
- package/lib/chart/errors.js +1 -2
- package/lib/chart/event-context.d.ts +8 -0
- package/lib/chart/event-context.js +39 -0
- package/lib/chart/events.d.ts +21 -78
- package/lib/chart/events.js +40 -189
- package/lib/chart/html.d.ts +2 -5
- package/lib/chart/html.js +8 -15
- package/lib/chart/runtime.d.ts +5 -18
- package/lib/chart/runtime.js +14 -34
- package/lib/chart/series.d.ts +10 -17
- package/lib/chart/series.js +22 -47
- package/lib/chart/source-token.d.ts +5 -13
- package/lib/chart/source-token.js +18 -17
- package/lib/chart/spec.d.ts +6 -12
- package/lib/chart/spec.js +7 -31
- package/lib/chart/tool.d.ts +6 -29
- package/lib/chart/tool.js +9 -23
- package/lib/chart-render-entry.d.ts +9 -0
- package/lib/chart-render-entry.js +17 -0
- package/lib/chart-source-entry.d.ts +13 -0
- package/lib/chart-source-entry.js +20 -0
- package/lib/collect-entry.d.ts +9 -0
- package/lib/collect-entry.js +10 -0
- package/lib/collector-runtime.d.ts +22 -0
- package/lib/collector-runtime.js +97 -0
- package/lib/data-collector/args.d.ts +6 -24
- package/lib/data-collector/args.js +6 -24
- package/lib/data-collector/dataset-tools.d.ts +7 -1
- package/lib/data-collector/dataset-tools.js +27 -56
- package/lib/data-collector/describe.d.ts +8 -33
- package/lib/data-collector/describe.js +17 -70
- package/lib/data-collector/hub.d.ts +27 -85
- package/lib/data-collector/hub.js +19 -85
- package/lib/data-collector/query-time.d.ts +7 -26
- package/lib/data-collector/query-time.js +8 -27
- package/lib/data-collector/store.d.ts +40 -117
- package/lib/data-collector/store.js +32 -111
- package/lib/data-collector/time-axis.d.ts +16 -76
- package/lib/data-collector/time-axis.js +25 -111
- package/lib/data-collector/tools.d.ts +6 -6
- package/lib/data-collector/tools.js +34 -43
- package/lib/dataset-read-entry.d.ts +9 -0
- package/lib/dataset-read-entry.js +7 -0
- package/lib/index.d.ts +7 -117
- package/lib/index.js +43 -422
- package/lib/net/eastmoney-client.d.ts +13 -65
- package/lib/net/eastmoney-client.js +12 -53
- package/lib/net/throttle.d.ts +2 -13
- package/lib/net/throttle.js +3 -14
- package/lib/ocr/client.d.ts +12 -21
- package/lib/ocr/client.js +23 -40
- package/lib/ocr/markdown.d.ts +20 -26
- package/lib/ocr/markdown.js +23 -36
- package/lib/ocr/store.d.ts +14 -47
- package/lib/ocr/store.js +15 -45
- package/lib/ocr-entry.d.ts +9 -0
- package/lib/ocr-entry.js +10 -0
- package/lib/policy-entry.d.ts +17 -0
- package/lib/policy-entry.js +22 -0
- package/lib/preset-claim.d.ts +22 -0
- package/lib/preset-claim.js +24 -0
- package/lib/preset-ids.d.ts +38 -0
- package/lib/preset-ids.js +46 -0
- package/lib/retriever-config.d.ts +37 -0
- package/lib/retriever-config.js +36 -0
- package/lib/retriever-runtime.d.ts +31 -0
- package/lib/retriever-runtime.js +82 -0
- package/lib/root-extras-entry.d.ts +17 -0
- package/lib/root-extras-entry.js +25 -0
- package/lib/runtime-entry.d.ts +15 -0
- package/lib/runtime-entry.js +5 -0
- package/lib/runtime.d.ts +21 -0
- package/lib/runtime.js +33 -0
- package/lib/search-entry.d.ts +9 -0
- package/lib/search-entry.js +9 -0
- package/lib/sources/eastmoney-http.js +20 -68
- package/lib/sources/ftshare-rest.d.ts +6 -31
- package/lib/sources/ftshare-rest.js +1194 -225
- package/lib/sources/fuyao-core.d.ts +8 -18
- package/lib/sources/fuyao-core.js +10 -21
- package/lib/sources/fuyao-rest.d.ts +2 -6
- package/lib/sources/fuyao-rest.js +23 -102
- package/lib/sources/security-code.d.ts +6 -12
- package/lib/sources/security-code.js +6 -12
- package/lib/sources/security-identity.d.ts +1 -4
- package/lib/sources/security-identity.js +3 -8
- package/lib/sources/tencent-http.js +14 -28
- package/lib/sources/tencent-public-core.d.ts +8 -60
- package/lib/sources/tencent-public-core.js +12 -77
- package/lib/sources/wind-mcp.d.ts +0 -5
- package/lib/sources/wind-mcp.js +23 -78
- package/lib/sources-entry.d.ts +9 -0
- package/lib/sources-entry.js +9 -0
- package/lib/time/tools.d.ts +8 -31
- package/lib/time/tools.js +39 -64
- package/lib/time-entry.d.ts +9 -0
- package/lib/time-entry.js +7 -0
- package/lib/tool-exec.d.ts +45 -11
- package/lib/tool-exec.js +59 -6
- package/lib/user-customization.d.ts +14 -0
- package/lib/user-customization.js +47 -0
- package/lib/watchlist/tools.d.ts +3 -13
- package/lib/watchlist/tools.js +10 -30
- package/lib/watchlist-entry.d.ts +9 -0
- package/lib/watchlist-entry.js +6 -0
- package/lib/web-retriever/engines.d.ts +6 -11
- package/lib/web-retriever/engines.js +9 -14
- package/lib/web-retriever/html-markdown.d.ts +14 -34
- package/lib/web-retriever/html-markdown.js +39 -75
- package/lib/web-retriever/local-fetch.d.ts +22 -58
- package/lib/web-retriever/local-fetch.js +33 -77
- package/lib/web-retriever/retriever.d.ts +6 -18
- package/lib/web-retriever/retriever.js +7 -7
- package/lib/web-retriever/sources.d.ts +37 -124
- package/lib/web-retriever/sources.js +64 -196
- package/lib/web-retriever/tools.d.ts +43 -27
- package/lib/web-retriever/tools.js +99 -137
- package/lib/web-retriever/wind-client.d.ts +14 -31
- package/lib/web-retriever/wind-client.js +16 -32
- package/locale/zh.json +1 -1
- package/package.json +67 -7
- package/preset/capital-generation/README.md +87 -53
- package/preset/capital-generation/agent.patch.yml +526 -337
- package/preset/capital-generation/skills/capital-data-profile/SKILL.md +2 -2
- package/preset/capital-generation/skills/capital-data-protocol/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-equity-research/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-event-impact/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-orchestration/SKILL.md +15 -19
- package/preset/capital-generation/skills/capital-portfolio-review/SKILL.md +1 -1
- package/preset/capital-generation/skills/capital-visualization-protocol/SKILL.md +9 -12
- package/preset/capital-generation/skills/capital-web-evidence/SKILL.md +1 -1
- package/watchlist/client.js +15 -148
- package/watchlist/client.src.cjs +148 -662
- package/watchlist/holding-rules.js +11 -38
- package/watchlist/index.js +127 -397
- package/watchlist/package.json +2 -2
package/lib/watchlist/tools.js
CHANGED
|
@@ -1,22 +1,11 @@
|
|
|
1
1
|
import { dayDistance, readClock } from '../time/tools.js';
|
|
2
2
|
import { stripInvisibleText } from '../web-retriever/html-markdown.js';
|
|
3
3
|
const jsonObject = (properties = {}, required = []) => ({ type: 'object', properties, required, additionalProperties: false });
|
|
4
|
-
/** ToolOutputDefinition.render(args, value):第一参是入参、第二参才是返回值。 */
|
|
5
4
|
function render(_args, value) {
|
|
6
5
|
return [{ type: 'text', text: JSON.stringify(value) }];
|
|
7
6
|
}
|
|
8
7
|
const WATCHLIST_DENIED = '这不是"用户没有自选股":清单读不到,一条都不许猜。';
|
|
9
|
-
/**
|
|
10
|
-
* 一条记录 → 模型看见的那几格。**每个键都声明过**(§9.6:多一个未声明字段同样致命)。
|
|
11
|
-
*
|
|
12
|
-
* 两形态由 `held` 点名:未持仓的那一行**不出现** `weight_pct` / `holding_marked_*`——
|
|
13
|
-
* "根本不是仓位"的时候谈"比例是 null 还是 0"没有意义,而把这三格摊给 30 行会让满载回包
|
|
14
|
-
* 撞破体积预算(下面那条 6144 的闸门实测过)。三态仍然彼此可分:
|
|
15
|
-
* `held:false` / `held:true` 且 `weight_pct:null`(标了没填) / `held:true` 且 `weight_pct:0`(真填 0)。
|
|
16
|
-
*
|
|
17
|
-
* 字段是照着"用户点名的四格 + 判断这句话还能信多久"选的,`added_age_days` 与 `pinned` 都不在这个
|
|
18
|
-
* 问题里:添加日期就是事实本身,而清单**顺序**已经反映了置顶(服务端 `rowsOf` 给的读法)。
|
|
19
|
-
*/
|
|
8
|
+
/** Held rows may carry weight/marked fields; `held:true` with `weight_pct:null` stays distinct from 0. */
|
|
20
9
|
function rowOf(row, today) {
|
|
21
10
|
const stripped = stripInvisibleText(String(row?.name ?? ''));
|
|
22
11
|
const thscode = String(row?.thscode ?? '');
|
|
@@ -24,24 +13,20 @@ function rowOf(row, today) {
|
|
|
24
13
|
const holding = row?.holding;
|
|
25
14
|
const held = holding !== undefined;
|
|
26
15
|
const markedAt = held && typeof holding?.marked_at === 'number' ? String(readClock(holding.marked_at).date) : null;
|
|
27
|
-
//
|
|
28
|
-
// 日期那一格仍给原值,所以异常照样看得见(不靠 clamp 把它藏起来)。
|
|
16
|
+
// Clamp the age at 0 so a clock rollback cannot produce a negative "days ago"; the date itself stays raw.
|
|
29
17
|
const age = (date) => Math.max(0, dayDistance(date, today));
|
|
30
18
|
return {
|
|
31
19
|
thscode,
|
|
32
|
-
//
|
|
33
|
-
// (设计文档 §3.2);剥完为空就退回代码,不放一个空字符串上去。
|
|
20
|
+
// Strip invisible characters so the name the user sees matches what the agent reads; fall back to the code.
|
|
34
21
|
name: stripped.length > 0 ? stripped : thscode,
|
|
35
22
|
asset_type: String(row?.asset_type ?? ''),
|
|
36
23
|
added_date: addedAt,
|
|
37
24
|
held,
|
|
38
|
-
//
|
|
39
|
-
// 把它写成 0% 等于替用户说了一句他没说的话(域 schema 同一口径,见 change_pct 那段)。
|
|
25
|
+
// Empty weight stays null, not 0: writing 0 would put a claim in the user's mouth.
|
|
40
26
|
...(held ? { weight_pct: holding?.weight_pct ?? null } : {}),
|
|
41
27
|
...(markedAt !== null ? { holding_marked_date: markedAt, holding_marked_age_days: age(markedAt) } : {}),
|
|
42
28
|
};
|
|
43
29
|
}
|
|
44
|
-
/** 成功回包。`limit` 只在宿主给了才带(那个数字归 `capital-watchlist` 所有,这里不抄一份)。 */
|
|
45
30
|
export function watchlistReceipt(rows, limit, now) {
|
|
46
31
|
const today = String(readClock(now).date);
|
|
47
32
|
const items = rows.map((row) => rowOf(row, today));
|
|
@@ -84,7 +69,7 @@ export function watchlistToolDefinition(source) {
|
|
|
84
69
|
as_of_date: { type: 'string' },
|
|
85
70
|
items: {
|
|
86
71
|
type: 'array',
|
|
87
|
-
// `required`
|
|
72
|
+
// `required` lists only the five shared fields; held-only fields are keyed by `held`.
|
|
88
73
|
items: jsonObject({
|
|
89
74
|
thscode: { type: 'string' },
|
|
90
75
|
name: { type: 'string' },
|
|
@@ -102,7 +87,7 @@ export function watchlistToolDefinition(source) {
|
|
|
102
87
|
async execute() {
|
|
103
88
|
const service = source();
|
|
104
89
|
if (service === undefined || typeof service.list !== 'function') {
|
|
105
|
-
//
|
|
90
|
+
// Never return an empty list: a missing host row must look different from "user has no watchlist".
|
|
106
91
|
return unavailable('watchlist_unavailable', `自选股服务没有挂载(capital-watchlist 那行没起来)。${WATCHLIST_DENIED}`);
|
|
107
92
|
}
|
|
108
93
|
let result;
|
|
@@ -121,21 +106,16 @@ export function watchlistToolDefinition(source) {
|
|
|
121
106
|
};
|
|
122
107
|
}
|
|
123
108
|
/**
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
* 会话级数据(Dataset / profile / chart)走 workspace,而自选股是**宿主级用户资产**:
|
|
128
|
-
* 本工具不读 `{{cwd}}`、不碰 `~/.dsh/storages/` 那个文件,只调那一路唯一的宿主服务
|
|
129
|
-
* (设计文档 §2.4:不允许另建一份存储)。
|
|
109
|
+
* Register get_watchlist (main agent + all children; the subagent row must deny it). Host-level user
|
|
110
|
+
* data: read it only through the host service, never a second store.
|
|
130
111
|
*/
|
|
131
112
|
export function registerWatchlistTool(ctx) {
|
|
132
113
|
const tools = ctx.get('tools');
|
|
133
114
|
if (tools === undefined)
|
|
134
115
|
return;
|
|
135
116
|
ctx.effect(() => tools.register(watchlistToolDefinition(() => {
|
|
136
|
-
//
|
|
137
|
-
//
|
|
138
|
-
// 钉成"用户的自选股整个进程读不到"(桌面端 2026-09-30 那 61 条能力缺席是同一族形状)。
|
|
117
|
+
// Resolve the service at call time: host-plane row order is not ours, and caching a startup miss
|
|
118
|
+
// would hide the user's watchlist for the whole process.
|
|
139
119
|
try {
|
|
140
120
|
return ctx.get('capitalWatchlist');
|
|
141
121
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `tools/watchlist` row: get_watchlist. User assets live in host storage-domain and only
|
|
3
|
+
* the main Agent mounts this row. Resolve the domain per call: memoizing an unready
|
|
4
|
+
* `storageDomain` as `store_unavailable` disables it for the whole process.
|
|
5
|
+
*/
|
|
6
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
7
|
+
export declare const name = "capital-watchlist-tools";
|
|
8
|
+
export declare const inject: string[];
|
|
9
|
+
export declare function apply(ctx: Context): void;
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* AnySearch
|
|
3
|
-
*
|
|
4
|
-
* web_retriever 目前只支持一个后端,因此不预留多引擎注册表、动态 loader 或
|
|
5
|
-
* provider 目录。未来真正接入第二个后端时,再根据实际差异抽象接口。
|
|
2
|
+
* Minimal AnySearch REST client. No multi-engine registry or provider directory
|
|
3
|
+
* exists until a second backend introduces a real abstraction need.
|
|
6
4
|
*/
|
|
7
5
|
export interface SearchHit {
|
|
8
6
|
url: string;
|
|
@@ -24,9 +22,9 @@ export interface FetchResult {
|
|
|
24
22
|
truncated?: boolean;
|
|
25
23
|
error?: string;
|
|
26
24
|
}
|
|
27
|
-
/**
|
|
25
|
+
/** Structured error codes; spelling is part of the fallback contract. */
|
|
28
26
|
export type AnySearchErrorCode = 'ABORTED' | 'TIMEOUT' | 'NETWORK' | 'AUTH' | 'RATE_LIMIT' | 'HTTP' | 'INVALID_RESPONSE' | 'INVALID_URL' | 'TARGET_BLOCKED' | 'CONTENT_TOO_LARGE' | 'UNSUPPORTED_CONTENT' | 'UPSTREAM';
|
|
29
|
-
/**
|
|
27
|
+
/** Failure with a code; existing message wording remains part of the assertions. */
|
|
30
28
|
export declare class AnySearchError extends Error {
|
|
31
29
|
readonly code: AnySearchErrorCode;
|
|
32
30
|
readonly upstreamCode?: string;
|
|
@@ -36,10 +34,7 @@ export declare class AnySearchError extends Error {
|
|
|
36
34
|
status?: number;
|
|
37
35
|
});
|
|
38
36
|
}
|
|
39
|
-
/**
|
|
40
|
-
* AnySearch 失败后是否值得改走本机直连回退。
|
|
41
|
-
* 只有调用方取消、URL 非法与内容类型不受支持三类不回退;其余都回退。
|
|
42
|
-
*/
|
|
37
|
+
/** Fall back except for cancellation, invalid URL, and unsupported content type. */
|
|
43
38
|
export declare function shouldFallbackToLocalFetch(code: AnySearchErrorCode): boolean;
|
|
44
39
|
export interface AnySearchClient {
|
|
45
40
|
search(request: {
|
|
@@ -57,5 +52,5 @@ export declare function normalizeSearchHit(raw: unknown): SearchHit | null;
|
|
|
57
52
|
export declare function normalizeSearchResponse(query: string, raw: unknown): SearchResult;
|
|
58
53
|
export declare function normalizeFetchResponse(url: string, raw: unknown): FetchResult;
|
|
59
54
|
export declare function envKey(name: string): string | undefined;
|
|
60
|
-
/**
|
|
55
|
+
/** Resolve the API key on every request; do not cache it. */
|
|
61
56
|
export declare function createAnySearchClient(baseURL?: string, resolveApiKey?: () => Promise<string | undefined>): AnySearchClient;
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* AnySearch
|
|
3
|
-
*
|
|
4
|
-
* web_retriever 目前只支持一个后端,因此不预留多引擎注册表、动态 loader 或
|
|
5
|
-
* provider 目录。未来真正接入第二个后端时,再根据实际差异抽象接口。
|
|
2
|
+
* Minimal AnySearch REST client. No multi-engine registry or provider directory
|
|
3
|
+
* exists until a second backend introduces a real abstraction need.
|
|
6
4
|
*/
|
|
7
|
-
/**
|
|
5
|
+
/** Shared invisible-text and link presentation rules with local fetching. */
|
|
8
6
|
import { stripInvisibleText } from './html-markdown.js';
|
|
9
7
|
import { safeUrl } from './local-fetch.js';
|
|
10
|
-
/**
|
|
8
|
+
/** Map upstream error_code values, matching the official fetch-provider webErrorCode table. */
|
|
11
9
|
const ERROR_CODE_MAP = {
|
|
12
10
|
invalid_extract_url: 'INVALID_URL',
|
|
13
11
|
extract_target_blocked: 'TARGET_BLOCKED',
|
|
@@ -15,7 +13,7 @@ const ERROR_CODE_MAP = {
|
|
|
15
13
|
extract_unsupported_content: 'UNSUPPORTED_CONTENT',
|
|
16
14
|
extract_timeout: 'TIMEOUT',
|
|
17
15
|
};
|
|
18
|
-
/**
|
|
16
|
+
/** Failure with a code; existing message wording remains part of the assertions. */
|
|
19
17
|
export class AnySearchError extends Error {
|
|
20
18
|
code;
|
|
21
19
|
upstreamCode;
|
|
@@ -30,10 +28,7 @@ export class AnySearchError extends Error {
|
|
|
30
28
|
this.status = extra.status;
|
|
31
29
|
}
|
|
32
30
|
}
|
|
33
|
-
/**
|
|
34
|
-
* AnySearch 失败后是否值得改走本机直连回退。
|
|
35
|
-
* 只有调用方取消、URL 非法与内容类型不受支持三类不回退;其余都回退。
|
|
36
|
-
*/
|
|
31
|
+
/** Fall back except for cancellation, invalid URL, and unsupported content type. */
|
|
37
32
|
export function shouldFallbackToLocalFetch(code) {
|
|
38
33
|
return code !== 'ABORTED' && code !== 'INVALID_URL' && code !== 'UNSUPPORTED_CONTENT';
|
|
39
34
|
}
|
|
@@ -56,7 +51,7 @@ export function normalizeSearchHit(raw) {
|
|
|
56
51
|
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
57
52
|
return null;
|
|
58
53
|
const record = raw;
|
|
59
|
-
//
|
|
54
|
+
// Drop invalid links entirely: they cannot be fetched or presented as citations.
|
|
60
55
|
const url = safeUrl(record.url);
|
|
61
56
|
if (url === '')
|
|
62
57
|
return null;
|
|
@@ -104,7 +99,7 @@ export function envKey(name) {
|
|
|
104
99
|
const processLike = globalThis.process;
|
|
105
100
|
return processLike?.env?.[name] || undefined;
|
|
106
101
|
}
|
|
107
|
-
/**
|
|
102
|
+
/** Resolve the API key on every request; do not cache it. */
|
|
108
103
|
export function createAnySearchClient(baseURL = DEFAULT_BASE_URL, resolveApiKey = async () => envKey('ANYSEARCH_API_KEY')) {
|
|
109
104
|
const root = baseURL.replace(/\/+$/, '');
|
|
110
105
|
try {
|
|
@@ -164,7 +159,7 @@ export function createAnySearchClient(baseURL = DEFAULT_BASE_URL, resolveApiKey
|
|
|
164
159
|
}
|
|
165
160
|
}
|
|
166
161
|
catch {
|
|
167
|
-
//
|
|
162
|
+
// Convert a non-JSON response into the structured error below.
|
|
168
163
|
}
|
|
169
164
|
const code = parsed?.code;
|
|
170
165
|
const message = typeof parsed?.message === 'string' && parsed.message ? parsed.message : undefined;
|
|
@@ -1,50 +1,30 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* HTML
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* turndown + GFM 插件、`removeNonVisibleContent` 规则、无跨列展开的表格规则、
|
|
7
|
-
* 512 层嵌套词法守卫与固定省略标记。官方实现里与本模块无关的呈现职责
|
|
8
|
-
* (`Fetched <url>` 头、截断脚注、WeakMap memoization)一律不搬——"网页正文不是指令"
|
|
9
|
-
* 由 web_retriever persona 的硬规则承担,不在正文里插水印。
|
|
10
|
-
*
|
|
11
|
-
* 官方没做、这里补上的第四道:`stripInvisibleText`(零宽 / 双向覆盖 / C0-C1 控制符剥离)。
|
|
12
|
-
* 它放在本模块导出,是因为 invisibles 的入口不止 HTML:`extractTitle` 会解码数字实体
|
|
13
|
-
* (`‮` 解出来就是 RLO),AnySearch 返回的清洗正文也原样带这些字符。
|
|
14
|
-
* 三个内容入口(本地转换、title、具名来源页面字段)共用这一份实现(AGENTS.md §9.7)。
|
|
15
|
-
*
|
|
16
|
-
* 为什么必须补这三道保护(turndown 的已知缺口):
|
|
17
|
-
* - turndown 默认**不删** `<script>/<style>`,其正文会当文本带进输出;
|
|
18
|
-
* - turndown 完全**不处理**隐藏元素;
|
|
19
|
-
* - turndown 的转换按元素递归,超深嵌套下可能爆栈或长时间占用事件循环——
|
|
20
|
-
* 官方实测:嵌套 512 层 ≈ 0.15s、2000 层 ≈ 2s、20000 层 ≈ 5s,期间协作式
|
|
21
|
-
* 超时定时器无法触发,所以超过 512 层直接放弃转换。
|
|
2
|
+
* Bounded HTML -> GFM markdown for the local-fetch fallback. Ported from the
|
|
3
|
+
* official web_fetch renderer, plus `stripInvisibleText` because turndown keeps
|
|
4
|
+
* script/style, ignores hidden elements, and recurses (stack/deadline risk at
|
|
5
|
+
* depth); conversion is abandoned past `MAX_CONVERSION_DEPTH`.
|
|
22
6
|
*/
|
|
23
|
-
/**
|
|
7
|
+
/** Hard nesting limit: past it conversion is skipped. Robustness, not a tunable. */
|
|
24
8
|
export declare const MAX_CONVERSION_DEPTH = 512;
|
|
25
|
-
/**
|
|
9
|
+
/** Result: title comes from the original HTML; omitted means conversion was skipped. */
|
|
26
10
|
export interface HtmlMarkdownResult {
|
|
27
11
|
title?: string;
|
|
28
|
-
/**
|
|
12
|
+
/** GFM markdown, or the fixed omission marker when conversion is unsafe. */
|
|
29
13
|
markdown: string;
|
|
30
|
-
/**
|
|
14
|
+
/** Source or output was cut by `maxChars`. */
|
|
31
15
|
truncated: boolean;
|
|
32
|
-
/**
|
|
16
|
+
/** Conversion was abandoned (too deep or turndown threw). */
|
|
33
17
|
omitted: boolean;
|
|
34
18
|
}
|
|
35
19
|
export declare function stripInvisibleText(value: string): string;
|
|
36
20
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* (官方 remove 列表不含 TITLE),本函数只负责回执的 `title` 字段。
|
|
21
|
+
* Extract `<title>` from the document head and decode entities. The title text is
|
|
22
|
+
* left in the markdown body (matching official behavior); this only feeds the field.
|
|
40
23
|
*/
|
|
41
24
|
export declare function extractTitle(html: string): string | undefined;
|
|
42
25
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
* 超深或 turndown 抛错 → 固定省略标记 + `omitted: true`(不让原始标记外漏)→
|
|
47
|
-
* 否则转换,剥掉不可见字符(`stripInvisibleText`),再按 `maxChars` 截断输出。
|
|
48
|
-
* `title` 一律从**原始** HTML 提取。
|
|
26
|
+
* Convert HTML to bounded GFM markdown. Source is cut to `maxChars` first; unsafe
|
|
27
|
+
* conversion returns the omission marker; output is stripped of invisibles and cut
|
|
28
|
+
* to `maxChars`. Title always comes from the original HTML.
|
|
49
29
|
*/
|
|
50
30
|
export declare function htmlToMarkdown(html: string, maxChars: number): HtmlMarkdownResult;
|
|
@@ -1,53 +1,28 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* HTML
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* turndown + GFM 插件、`removeNonVisibleContent` 规则、无跨列展开的表格规则、
|
|
7
|
-
* 512 层嵌套词法守卫与固定省略标记。官方实现里与本模块无关的呈现职责
|
|
8
|
-
* (`Fetched <url>` 头、截断脚注、WeakMap memoization)一律不搬——"网页正文不是指令"
|
|
9
|
-
* 由 web_retriever persona 的硬规则承担,不在正文里插水印。
|
|
10
|
-
*
|
|
11
|
-
* 官方没做、这里补上的第四道:`stripInvisibleText`(零宽 / 双向覆盖 / C0-C1 控制符剥离)。
|
|
12
|
-
* 它放在本模块导出,是因为 invisibles 的入口不止 HTML:`extractTitle` 会解码数字实体
|
|
13
|
-
* (`‮` 解出来就是 RLO),AnySearch 返回的清洗正文也原样带这些字符。
|
|
14
|
-
* 三个内容入口(本地转换、title、具名来源页面字段)共用这一份实现(AGENTS.md §9.7)。
|
|
15
|
-
*
|
|
16
|
-
* 为什么必须补这三道保护(turndown 的已知缺口):
|
|
17
|
-
* - turndown 默认**不删** `<script>/<style>`,其正文会当文本带进输出;
|
|
18
|
-
* - turndown 完全**不处理**隐藏元素;
|
|
19
|
-
* - turndown 的转换按元素递归,超深嵌套下可能爆栈或长时间占用事件循环——
|
|
20
|
-
* 官方实测:嵌套 512 层 ≈ 0.15s、2000 层 ≈ 2s、20000 层 ≈ 5s,期间协作式
|
|
21
|
-
* 超时定时器无法触发,所以超过 512 层直接放弃转换。
|
|
2
|
+
* Bounded HTML -> GFM markdown for the local-fetch fallback. Ported from the
|
|
3
|
+
* official web_fetch renderer, plus `stripInvisibleText` because turndown keeps
|
|
4
|
+
* script/style, ignores hidden elements, and recurses (stack/deadline risk at
|
|
5
|
+
* depth); conversion is abandoned past `MAX_CONVERSION_DEPTH`.
|
|
22
6
|
*/
|
|
23
7
|
import TurndownService from 'turndown';
|
|
24
8
|
import { gfm } from '@joplin/turndown-plugin-gfm';
|
|
25
|
-
/**
|
|
9
|
+
/** Hard nesting limit: past it conversion is skipped. Robustness, not a tunable. */
|
|
26
10
|
export const MAX_CONVERSION_DEPTH = 512;
|
|
27
|
-
//
|
|
28
|
-
// 不可见字符剥离(外部内容共用的出口)
|
|
29
|
-
// ---------------------------------------------------------------------------
|
|
11
|
+
// Invisible character stripping (shared by every external-content entry)
|
|
30
12
|
/**
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* - 双向覆盖 `U+202A–U+202E` / 隔离符 `U+2066–U+2069`:改变一行的**显示顺序**
|
|
36
|
-
* (Trojan Source 一类攻击),回显给用户的就是假的文本;
|
|
37
|
-
* - `U+2028` / `U+2029`:行/段分隔符,渲染上与 `\n` 等价、按行处理时又不算换行。
|
|
38
|
-
*
|
|
39
|
-
* 只删不替换成占位符:这些字符在合法正文里没有承载信息,删除是保序的。
|
|
40
|
-
* 换行 `\n`、回车 `\r`、制表 `\t` 保留(markdown 呈现需要)。
|
|
13
|
+
* Some characters render as nothing yet reach the model and citations: C0/C1
|
|
14
|
+
* controls, soft hyphen/zero-width/BOM (keyword checks miss them), bidi overrides
|
|
15
|
+
* and isolates (Trojan Source display spoofing), U+2028/2029. Deleting preserves
|
|
16
|
+
* order; keep \n, \r and \t for markdown.
|
|
41
17
|
*/
|
|
42
18
|
const INVISIBLE_PATTERN = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F-\u009F\u00AD\u200B-\u200F\u2028\u2029\u202A-\u202E\u2060-\u2064\u2066-\u2069\uFEFF]/gu;
|
|
43
19
|
export function stripInvisibleText(value) {
|
|
44
20
|
return value.replace(INVISIBLE_PATTERN, '');
|
|
45
21
|
}
|
|
46
22
|
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* turndown 按后注册先匹配解析规则,表格规则因此覆盖 gfm 插件自带的跨列展开实现。
|
|
23
|
+
* One module-scope turndown instance (stateless across calls). Rule order matters:
|
|
24
|
+
* gfm, then remove rules, then table rules; table rules must come last to override
|
|
25
|
+
* the plugin's colspan expansion.
|
|
51
26
|
*/
|
|
52
27
|
const turndown = new TurndownService({
|
|
53
28
|
headingStyle: 'atx',
|
|
@@ -55,7 +30,7 @@ const turndown = new TurndownService({
|
|
|
55
30
|
bulletListMarker: '-',
|
|
56
31
|
});
|
|
57
32
|
turndown.use(gfm);
|
|
58
|
-
/**
|
|
33
|
+
/** Match => drop the whole block; script/style/hidden content never reaches output. */
|
|
59
34
|
turndown.addRule('removeNonVisibleContent', {
|
|
60
35
|
filter(node) {
|
|
61
36
|
if (['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'IFRAME', 'OBJECT', 'EMBED'].includes(node.nodeName))
|
|
@@ -79,13 +54,13 @@ turndown.addRule('removeNonVisibleContent', {
|
|
|
79
54
|
return '';
|
|
80
55
|
},
|
|
81
56
|
});
|
|
82
|
-
/**
|
|
57
|
+
/** Render one GFM cell; HTML colspan is deliberately not interpreted. */
|
|
83
58
|
function renderTableCell(content, index) {
|
|
84
59
|
const prefix = index === 0 ? '| ' : ' ';
|
|
85
60
|
const escaped = content.trim().replace(/\n\r/g, '<br>').replace(/\n/g, '<br>').replace(/\|+/g, '\\|').padEnd(3, ' ');
|
|
86
61
|
return `${prefix}${escaped} |`;
|
|
87
62
|
}
|
|
88
|
-
/**
|
|
63
|
+
/** Whether this row is the markdown header row of its table. */
|
|
89
64
|
function isTableHeadingRow(row) {
|
|
90
65
|
const cells = Array.from(row.cells);
|
|
91
66
|
const section = row.parentElement;
|
|
@@ -93,7 +68,7 @@ function isTableHeadingRow(row) {
|
|
|
93
68
|
return (section.nodeName === 'THEAD' || table.rows[0] === row)
|
|
94
69
|
&& cells.every(cell => cell.nodeName === 'TH');
|
|
95
70
|
}
|
|
96
|
-
/**
|
|
71
|
+
/** Map HTML cell alignment to a GFM separator. */
|
|
97
72
|
function tableBorder(cell) {
|
|
98
73
|
const alignment = (cell.getAttribute('align') || cell.style.textAlign || '').toLowerCase();
|
|
99
74
|
if (alignment === 'left')
|
|
@@ -109,8 +84,8 @@ turndown.addRule('tableCellWithoutSpanExpansion', {
|
|
|
109
84
|
replacement(content, node) {
|
|
110
85
|
const cell = node;
|
|
111
86
|
const row = cell.parentNode;
|
|
112
|
-
// GFM
|
|
113
|
-
//
|
|
87
|
+
// GFM cannot express colspan; ignoring it keeps output proportional to source
|
|
88
|
+
// text instead of the numeric attribute (the gfm rule expands empty cells).
|
|
114
89
|
return renderTableCell(content, Array.from(row.childNodes).indexOf(cell));
|
|
115
90
|
},
|
|
116
91
|
});
|
|
@@ -124,21 +99,19 @@ turndown.addRule('tableRowWithoutSpanExpansion', {
|
|
|
124
99
|
return `\n${content}${border.length > 0 ? `\n${border}` : ''}`;
|
|
125
100
|
},
|
|
126
101
|
});
|
|
127
|
-
//
|
|
128
|
-
|
|
129
|
-
// ---------------------------------------------------------------------------
|
|
130
|
-
/** 永不接收闭合标签的元素:不进词法栈。 */
|
|
102
|
+
// Nesting-depth guard (ported from official fetch.ts)
|
|
103
|
+
/** Elements that never take a closing tag: kept out of the lexical stack. */
|
|
131
104
|
const VOID_ELEMENTS = new Set([
|
|
132
105
|
'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input',
|
|
133
106
|
'link', 'meta', 'param', 'source', 'track', 'wbr',
|
|
134
107
|
]);
|
|
135
|
-
/**
|
|
108
|
+
/** Text until the matching close tag is raw text, not markup. */
|
|
136
109
|
const RAW_TEXT_ELEMENTS = new Set(['script', 'style', 'noscript']);
|
|
137
|
-
/**
|
|
110
|
+
/** Whether a char can follow a raw-text close tag name. */
|
|
138
111
|
function isTagBoundary(char) {
|
|
139
112
|
return char === undefined || char === '>' || char === '/' || /\s/.test(char);
|
|
140
113
|
}
|
|
141
|
-
/**
|
|
114
|
+
/** Find a raw-text close tag, ignoring markup-like text inside the body. */
|
|
142
115
|
function findRawTextEnd(lowerHtml, name, from) {
|
|
143
116
|
const prefix = `</${name}`;
|
|
144
117
|
let candidate = lowerHtml.indexOf(prefix, from);
|
|
@@ -148,9 +121,9 @@ function findRawTextEnd(lowerHtml, name, from) {
|
|
|
148
121
|
return candidate;
|
|
149
122
|
}
|
|
150
123
|
/**
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
124
|
+
* Reject HTML whose lexical element stack exceeds the depth limit. Single pass:
|
|
125
|
+
* ignores close tags in comments, skips raw text, respects quoted `>`, only pops a
|
|
126
|
+
* matching top-of-stack, so malformed input over-counts rather than hides nesting.
|
|
154
127
|
*/
|
|
155
128
|
function exceedsConversionDepth(html) {
|
|
156
129
|
const lowerHtml = html.toLowerCase();
|
|
@@ -228,10 +201,8 @@ function exceedsConversionDepth(html) {
|
|
|
228
201
|
}
|
|
229
202
|
return false;
|
|
230
203
|
}
|
|
231
|
-
//
|
|
232
|
-
|
|
233
|
-
// ---------------------------------------------------------------------------
|
|
234
|
-
/** 只在开头固定窗口内找 `<title>`:title 位于 head、靠文档开头,窗口上限避免对超长正文全量扫描。 */
|
|
204
|
+
// Title extraction
|
|
205
|
+
/** Scan only the opening window: `<title>` lives in head, avoid full-document scans. */
|
|
235
206
|
const TITLE_SCAN_CHARS = 2048;
|
|
236
207
|
const TITLE_PATTERN = /<title[^>]*>([^<]*)<\/title>/i;
|
|
237
208
|
const ENTITY_PATTERN = /&(?:amp|lt|gt|quot|nbsp|#[0-9]+|#[xX][0-9a-fA-F]+);/g;
|
|
@@ -242,7 +213,7 @@ const NAMED_ENTITIES = new Map([
|
|
|
242
213
|
['quot', '"'],
|
|
243
214
|
['nbsp', '\u00a0'],
|
|
244
215
|
]);
|
|
245
|
-
/**
|
|
216
|
+
/** Decode a numeric character reference; invalid code point => null (keep text). */
|
|
246
217
|
function decodeNumericEntity(body) {
|
|
247
218
|
const hex = body[1] === 'x' || body[1] === 'X';
|
|
248
219
|
const code = Number.parseInt(body.slice(hex ? 2 : 1), hex ? 16 : 10);
|
|
@@ -250,7 +221,7 @@ function decodeNumericEntity(body) {
|
|
|
250
221
|
return null;
|
|
251
222
|
return String.fromCodePoint(code);
|
|
252
223
|
}
|
|
253
|
-
/**
|
|
224
|
+
/** Single-pass entity decode: `&lt;` becomes `<`, never `<`. */
|
|
254
225
|
function decodeEntities(text) {
|
|
255
226
|
return text.replace(ENTITY_PATTERN, (match) => {
|
|
256
227
|
const body = match.slice(1, -1);
|
|
@@ -261,9 +232,8 @@ function decodeEntities(text) {
|
|
|
261
232
|
});
|
|
262
233
|
}
|
|
263
234
|
/**
|
|
264
|
-
*
|
|
265
|
-
*
|
|
266
|
-
* (官方 remove 列表不含 TITLE),本函数只负责回执的 `title` 字段。
|
|
235
|
+
* Extract `<title>` from the document head and decode entities. The title text is
|
|
236
|
+
* left in the markdown body (matching official behavior); this only feeds the field.
|
|
267
237
|
*/
|
|
268
238
|
export function extractTitle(html) {
|
|
269
239
|
const match = TITLE_PATTERN.exec(html.slice(0, TITLE_SCAN_CHARS));
|
|
@@ -272,10 +242,8 @@ export function extractTitle(html) {
|
|
|
272
242
|
const title = stripInvisibleText(decodeEntities(match[1])).replace(/\s+/g, ' ').trim();
|
|
273
243
|
return title === '' ? undefined : title;
|
|
274
244
|
}
|
|
275
|
-
//
|
|
276
|
-
|
|
277
|
-
// ---------------------------------------------------------------------------
|
|
278
|
-
/** 转换不安全(超深或 turndown 抛错)时返回的固定省略标记。 */
|
|
245
|
+
// Entry point
|
|
246
|
+
/** Fixed marker returned when conversion is unsafe (too deep or turndown threw). */
|
|
279
247
|
const OMITTED_MARKDOWN = '[HTML content omitted: unable to convert safely.]';
|
|
280
248
|
function omittedResult(title, sourceTruncated, maxChars) {
|
|
281
249
|
const outputTruncated = OMITTED_MARKDOWN.length > maxChars;
|
|
@@ -287,12 +255,9 @@ function omittedResult(title, sourceTruncated, maxChars) {
|
|
|
287
255
|
};
|
|
288
256
|
}
|
|
289
257
|
/**
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
-
*
|
|
293
|
-
* 超深或 turndown 抛错 → 固定省略标记 + `omitted: true`(不让原始标记外漏)→
|
|
294
|
-
* 否则转换,剥掉不可见字符(`stripInvisibleText`),再按 `maxChars` 截断输出。
|
|
295
|
-
* `title` 一律从**原始** HTML 提取。
|
|
258
|
+
* Convert HTML to bounded GFM markdown. Source is cut to `maxChars` first; unsafe
|
|
259
|
+
* conversion returns the omission marker; output is stripped of invisibles and cut
|
|
260
|
+
* to `maxChars`. Title always comes from the original HTML.
|
|
296
261
|
*/
|
|
297
262
|
export function htmlToMarkdown(html, maxChars) {
|
|
298
263
|
const title = extractTitle(html);
|
|
@@ -306,8 +271,7 @@ export function htmlToMarkdown(html, maxChars) {
|
|
|
306
271
|
converted = turndown.turndown(content);
|
|
307
272
|
}
|
|
308
273
|
catch {
|
|
309
|
-
//
|
|
310
|
-
// RangeError。转换失败与超深走同一个省略标记,不把源标记带回模型上下文。
|
|
274
|
+
// The lexical guard cannot model every malformed tree; RangeError => same marker.
|
|
311
275
|
return omittedResult(title, sourceTruncated, maxChars);
|
|
312
276
|
}
|
|
313
277
|
const sanitized = stripInvisibleText(converted);
|