candidate-pool 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +245 -0
- package/README.md +397 -0
- package/bin/candidate-pool.js +23 -0
- package/bin/tail-screener.js +22 -0
- package/build_universe.js +231 -0
- package/candidates.example.json +269 -0
- package/db/.env.example +13 -0
- package/db/backfill.js +71 -0
- package/db/clients.js +48 -0
- package/db/d1client.js +69 -0
- package/db/ddl.js +82 -0
- package/db/diff.js +223 -0
- package/db/diff_selftest.js +124 -0
- package/db/enrich_stock_base.js +138 -0
- package/db/migrate.js +177 -0
- package/db/normalize.js +129 -0
- package/db/persist.js +90 -0
- package/db/query_local.js +43 -0
- package/db/schema.sql +225 -0
- package/db/sqlite_client.js +81 -0
- package/db/verify_sync.js +107 -0
- package/db/write_live.js +54 -0
- package/eod/build_industry_map.js +146 -0
- package/eod/data/industry-map.json +1 -0
- package/eod/lib/http.js +188 -0
- package/eod/lib/market.js +345 -0
- package/eod/lib/report.js +329 -0
- package/eod/lib/screen.js +355 -0
- package/eod/lib/sector.js +96 -0
- package/eod/lib/store.js +204 -0
- package/eod/lib/store_d1.js +293 -0
- package/eod/lib/trading.js +252 -0
- package/eod/lib/trading.selftest.js +283 -0
- package/eod/tail.config.js +212 -0
- package/eod/tail_screener.js +373 -0
- package/gen_candidates.js +994 -0
- package/notify.js +236 -0
- package/notify_config.example.json +14 -0
- package/package.json +96 -0
- package/paths.js +127 -0
- package/paths.selftest.js +142 -0
- package/scaffold.js +199 -0
- package/scaffold.selftest.js +153 -0
- package/time.js +17 -0
package/eod/lib/http.js
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/*
|
|
3
|
+
* eod/lib/http.js —— HTTP 抓取层(零第三方依赖,纯 Node 内建 https)
|
|
4
|
+
* ---------------------------------------------------------------------------
|
|
5
|
+
* 从 gen_candidates.js 的 httpGet 提取并泛化,保留了本项目已验证的三条关键经验:
|
|
6
|
+
*
|
|
7
|
+
* 1. **WAF 长退避**:腾讯接口在短时高频请求(尤其盘中)会返回一张反爬跳转页:
|
|
8
|
+
* <!DOCTYPE html><html><head><script>var i=location.href;var v=window.btoa? ...
|
|
9
|
+
* 它是 HTTP 200,只能靠 JSON.parse 报错或嗅探正文发现。必须单独识别并走
|
|
10
|
+
* **秒级长退避**;几百毫秒的短退避拿到的仍是同一张拦截页,等于白重试。
|
|
11
|
+
* 实测:状态码 501 + 跳转页正文,二者都查最稳。
|
|
12
|
+
*
|
|
13
|
+
* 2. **编码**:腾讯报价(qt.gtimg.cn)返回 **GBK**,中文股名直接按 UTF-8 解会乱码;
|
|
14
|
+
* 腾讯 JSON 接口与东财接口返回 **UTF-8**。故解码编码必须显式指定、不可混用。
|
|
15
|
+
* 注意 Node 的 Buffer.toString('gbk') 是**不支持的**(会抛 Unknown encoding),
|
|
16
|
+
* 必须用 TextDecoder('gbk'),且需 try/catch 兜底(旧运行时可能无内置 GBK 表)。
|
|
17
|
+
*
|
|
18
|
+
* 3. **粘性端点**:见 makeSticky()。某条链路被 WAF 封掉后指针前移,
|
|
19
|
+
* 后续标的直接走可用端点,不必每只票都去撞一次已知的墙。
|
|
20
|
+
* ---------------------------------------------------------------------------
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const https = require('https');
|
|
24
|
+
|
|
25
|
+
// 用完整浏览器 UA:残缺 UA(如裸 "Mozilla/5.0")本身就是风控指纹特征。
|
|
26
|
+
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
|
|
27
|
+
|
|
28
|
+
// 命中拦截时的退避梯度(ms):2s → 5s → 12s → 25s
|
|
29
|
+
const WAF_BACKOFF = [2000, 5000, 12000, 25000];
|
|
30
|
+
|
|
31
|
+
function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
|
32
|
+
|
|
33
|
+
/** 按指定编码解码。enc: 'gbk' | 'utf8'。GBK 不可用时降级为 utf8(中文会乱码,但代码/数字仍可用)。 */
|
|
34
|
+
function decode(buf, enc) {
|
|
35
|
+
if (enc === 'gbk') {
|
|
36
|
+
try { return new TextDecoder('gbk').decode(buf); }
|
|
37
|
+
catch (e) { return buf.toString('utf8'); }
|
|
38
|
+
}
|
|
39
|
+
return buf.toString('utf8');
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* 判断响应正文是否是腾讯的反爬跳转页。
|
|
44
|
+
* 只嗅探开头 300 字符,避免把正常的大 JSON 正文误判。
|
|
45
|
+
*/
|
|
46
|
+
function looksLikeWaf(txt) {
|
|
47
|
+
const head = String(txt).slice(0, 300);
|
|
48
|
+
return /<!DOCTYPE html|<html[\s>]/i.test(head) && /window\.btoa|location\.href/i.test(head);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* 通用 GET(带重试 + WAF 长退避)
|
|
53
|
+
* @param {string} url
|
|
54
|
+
* @param {object} opts
|
|
55
|
+
* - enc 'gbk' | 'utf8' 返回文本的解码方式(默认 utf8)
|
|
56
|
+
* - json 是否把返回解析为 JSON(默认 false,返回字符串)
|
|
57
|
+
* - retries 重试次数(默认 4)
|
|
58
|
+
* - timeout 单次超时 ms(默认 15000)
|
|
59
|
+
* - referer Referer 头(默认 https://gu.qq.com/,腾讯行情系;东财用 https://quote.eastmoney.com/)
|
|
60
|
+
* - isWaf 自定义「这条响应算不算被拦截」判定(默认 looksLikeWaf)
|
|
61
|
+
* @returns {Promise<any>} 文本或已解析的 JSON
|
|
62
|
+
*/
|
|
63
|
+
function httpGet(url, opts = {}) {
|
|
64
|
+
const {
|
|
65
|
+
enc = 'utf8', json = false, retries = 4, timeout = 15000,
|
|
66
|
+
referer = 'https://gu.qq.com/', isWaf = null,
|
|
67
|
+
} = opts;
|
|
68
|
+
|
|
69
|
+
return new Promise(async (resolve, reject) => {
|
|
70
|
+
let lastErr = null;
|
|
71
|
+
let wafHits = 0;
|
|
72
|
+
|
|
73
|
+
for (let i = 0; i < retries; i++) {
|
|
74
|
+
if (i > 0) {
|
|
75
|
+
// 被拦截 -> 长退避;普通网络抖动 -> 短退避
|
|
76
|
+
const wait = (lastErr && lastErr.waf)
|
|
77
|
+
? WAF_BACKOFF[Math.min(wafHits - 1, WAF_BACKOFF.length - 1)]
|
|
78
|
+
: 500 * i;
|
|
79
|
+
await sleep(wait);
|
|
80
|
+
}
|
|
81
|
+
try {
|
|
82
|
+
const { status, txt } = await new Promise((res, rej) => {
|
|
83
|
+
const req = https.get(url, {
|
|
84
|
+
timeout,
|
|
85
|
+
headers: {
|
|
86
|
+
'User-Agent': UA,
|
|
87
|
+
Referer: referer,
|
|
88
|
+
Accept: '*/*',
|
|
89
|
+
'Accept-Language': 'zh-CN,zh;q=0.9',
|
|
90
|
+
},
|
|
91
|
+
}, (resp) => {
|
|
92
|
+
const chunks = [];
|
|
93
|
+
resp.on('data', (c) => chunks.push(c));
|
|
94
|
+
resp.on('end', () => res({ status: resp.statusCode, txt: decode(Buffer.concat(chunks), enc) }));
|
|
95
|
+
});
|
|
96
|
+
req.on('error', rej);
|
|
97
|
+
req.on('timeout', () => { req.destroy(); rej(new Error('timeout')); });
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// WAF 拦截:状态码判定 + 正文嗅探,两者都查
|
|
101
|
+
const waf = (isWaf ? isWaf(txt, status) : false) || status === 501 || looksLikeWaf(txt);
|
|
102
|
+
if (waf) {
|
|
103
|
+
wafHits++;
|
|
104
|
+
const e = new Error(`被接口反爬拦截(HTTP ${status})`);
|
|
105
|
+
e.waf = true;
|
|
106
|
+
throw e;
|
|
107
|
+
}
|
|
108
|
+
if (status && status >= 400) throw new Error(`HTTP ${status}`);
|
|
109
|
+
|
|
110
|
+
if (!json) return resolve(txt.trim());
|
|
111
|
+
const body = txt.trim();
|
|
112
|
+
try { return resolve(JSON.parse(body)); }
|
|
113
|
+
catch (e) {
|
|
114
|
+
// 兼容 JSONP 包裹:name({...});
|
|
115
|
+
const m = body.match(/^[a-zA-Z0-9_$.]+\(([\s\S]*)\);?$/);
|
|
116
|
+
if (m) { try { return resolve(JSON.parse(m[1])); } catch (e2) { /* 落到下面报错 */ } }
|
|
117
|
+
throw new Error('JSON 解析失败: ' + body.slice(0, 120));
|
|
118
|
+
}
|
|
119
|
+
} catch (e) {
|
|
120
|
+
lastErr = e;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
reject(lastErr || new Error('httpGet 失败'));
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* 并发受限的 map(本文件自带,避免与 gen_candidates.js 交叉依赖)
|
|
129
|
+
* 注意:不吞异常——返回 { __error } 占位,调用方需自行判定。
|
|
130
|
+
*/
|
|
131
|
+
async function mapLimit(items, limit, fn) {
|
|
132
|
+
const res = new Array(items.length);
|
|
133
|
+
let i = 0;
|
|
134
|
+
const n = Math.max(1, Math.min(limit, items.length));
|
|
135
|
+
const workers = [];
|
|
136
|
+
for (let w = 0; w < n; w++) {
|
|
137
|
+
workers.push((async () => {
|
|
138
|
+
while (i < items.length) {
|
|
139
|
+
const idx = i++;
|
|
140
|
+
try { res[idx] = await fn(items[idx], idx); }
|
|
141
|
+
catch (e) { res[idx] = { __error: String((e && e.message) || e) }; }
|
|
142
|
+
}
|
|
143
|
+
})());
|
|
144
|
+
}
|
|
145
|
+
await Promise.all(workers);
|
|
146
|
+
return res;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* 粘性端点选择器。
|
|
151
|
+
* 场景:同一份数据有多个镜像链路(如日K / 行业映射),其中某条被 WAF 封掉。
|
|
152
|
+
* 用法:
|
|
153
|
+
* const pick = makeSticky(['a', 'b', 'c']);
|
|
154
|
+
* for (;;) { const ep = pick.cur(); try { ...; break; } catch { pick.next(); } }
|
|
155
|
+
* 记住当前可用下标,后续调用直接从可用链路开始。
|
|
156
|
+
*/
|
|
157
|
+
function makeSticky(endpoints) {
|
|
158
|
+
let idx = 0;
|
|
159
|
+
return {
|
|
160
|
+
cur() { return endpoints[idx]; },
|
|
161
|
+
index() { return idx; },
|
|
162
|
+
/** 前移到下一条链路,返回是否还有可用链路 */
|
|
163
|
+
next() { idx = (idx + 1) % endpoints.length; return true; },
|
|
164
|
+
/** 逐条尝试:fn(ep, i) 抛错则换下一条;全部失败抛最后一次错误 */
|
|
165
|
+
async tryAll(fn, onSwitch) {
|
|
166
|
+
let lastErr = null;
|
|
167
|
+
for (let step = 0; step < endpoints.length; step++) {
|
|
168
|
+
const i = (idx + step) % endpoints.length;
|
|
169
|
+
const ep = endpoints[i];
|
|
170
|
+
try {
|
|
171
|
+
const out = await fn(ep, i);
|
|
172
|
+
if (i !== idx) { idx = i; if (onSwitch) onSwitch(ep); }
|
|
173
|
+
return out;
|
|
174
|
+
} catch (e) {
|
|
175
|
+
lastErr = e;
|
|
176
|
+
// 数据层问题(如字段缺失)换链路也没用,直接抛出,避免无意义重试
|
|
177
|
+
if (e && e.fatal) throw e;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
throw lastErr || new Error('全部链路均不可用');
|
|
181
|
+
},
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** 带前导零的代码:toCode('sh', 600519) -> 'sh600519' */
|
|
186
|
+
function toCode(prefix, n) { return prefix + String(n).padStart(6, '0'); }
|
|
187
|
+
|
|
188
|
+
module.exports = { httpGet, mapLimit, makeSticky, decode, looksLikeWaf, toCode, sleep, UA };
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
/*
|
|
3
|
+
* eod/lib/market.js —— 行情抓取(全市场快照 + 分时)
|
|
4
|
+
* ---------------------------------------------------------------------------
|
|
5
|
+
* 数据源:腾讯公开行情接口(与 candidate-pool 主流程同源,零第三方依赖)
|
|
6
|
+
* - 全市场快照: https://qt.gtimg.cn/q=<code>,<code>,... 批量、GBK 编码
|
|
7
|
+
* - 个股分时: https://web.ifzq.gtimg.cn/appstock/app/minute/query?code=...
|
|
8
|
+
*
|
|
9
|
+
* 全市场怎么枚举(实测 2026-09-20):
|
|
10
|
+
* 腾讯没有「列出全部代码」的接口,但**不存在的代码会被响应直接省略**,
|
|
11
|
+
* 所以按标准代码段盲扫即可:~30000 个槽位 / 60 一批 / 6 并发 ≈ 8 秒,完全可接受。
|
|
12
|
+
* (曾实测东财 clist 接口能一次拿到全市场 + 行业字段,但它在十余次请求后即
|
|
13
|
+
* socket hang up 封 IP,**不可作为 14:50 关键路径的数据源**,故仅用于低频刷行业缓存。)
|
|
14
|
+
*
|
|
15
|
+
* 字段布局(腾讯报价,以 ~ 分隔,实测核对过):
|
|
16
|
+
* [1] 名称 [3] 现价 [4] 昨收 [5] 今开 [30] 行情时间戳 YYYYMMDDHHmmss
|
|
17
|
+
* [31] 涨跌额 [32] 涨跌幅% [33] 最高 [34] 最低
|
|
18
|
+
* [36] 成交量(手) [37] 成交额(万) [38] 换手率% [39] 市盈率
|
|
19
|
+
* [44] 流通市值(亿) [45] 总市值(亿) [47] 52周高 [48] 52周低
|
|
20
|
+
* [49] 量比 [51] 当日均价
|
|
21
|
+
* ---------------------------------------------------------------------------
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
const { httpGet, mapLimit, makeSticky, sleep, toCode } = require('./http');
|
|
25
|
+
const { stampToDate } = require('./trading');
|
|
26
|
+
|
|
27
|
+
const QT_QUOTE = 'https://qt.gtimg.cn/q=';
|
|
28
|
+
const IFZQ = 'https://web.ifzq.gtimg.cn/appstock/app';
|
|
29
|
+
|
|
30
|
+
// 模块级日志(默认静默,入口处 setLog 注入)。降级切换必须让人看得见。
|
|
31
|
+
let _log = () => {};
|
|
32
|
+
function setLog(fn) { _log = typeof fn === 'function' ? fn : () => {}; }
|
|
33
|
+
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
// 分时接口的多主机降级链(实测 2026-09-20):
|
|
36
|
+
// web.ifzq.gtimg.cn 被高频请求触发 WAF 后返回 HTTP 501 拦截页,
|
|
37
|
+
// 而同一路径在 ifzq.gtimg.cn 与 proxy.finance.qq.com 上**同时可用**(140ms 正常返回)。
|
|
38
|
+
// 与日K线一样,按链路逐个降级 + 粘性记忆,被封锁后不再反复撞同一堵墙。
|
|
39
|
+
// 注意 retries 压到 2:某台被拦时单台只烧 ~2.5s 就换下一台,
|
|
40
|
+
// 若沿用默认 4 次重试,单台要先烧 19s,486 只票会空跑半小时以上(实测教训)。
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
const MINUTE_ENDPOINTS = [
|
|
43
|
+
{ name: 'web.ifzq/minute', url: (c) => `${IFZQ}/minute/query?code=${c}` },
|
|
44
|
+
{ name: 'ifzq/minute', url: (c) => `https://ifzq.gtimg.cn/appstock/app/minute/query?code=${c}` },
|
|
45
|
+
{ name: 'proxy.finance/minute', url: (c) => `https://proxy.finance.qq.com/ifzqgtimg/appstock/app/minute/query?code=${c}` },
|
|
46
|
+
];
|
|
47
|
+
const _minuteEp = makeSticky(MINUTE_ENDPOINTS);
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* K 线 / 分时成交量单位(沿用 candidate-pool 主流程的实测结论):
|
|
51
|
+
* 主板 / 创业板 / 北交所 -> 「手」(1 手 = 100 股);科创板 sh688/sh689 -> 直接是「股」。
|
|
52
|
+
* 用错会整整差 100 倍,故均价折算必须区分。
|
|
53
|
+
*/
|
|
54
|
+
function volLotSize(code) {
|
|
55
|
+
return /^sh68[89]/.test(String(code)) ? 1 : 100;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** 把配置里的代码段展开成完整代码列表(已剔除排除前缀,如 B 股) */
|
|
59
|
+
function expandCodes(cfg) {
|
|
60
|
+
const skip = cfg.market.excludePrefixes || [];
|
|
61
|
+
const out = [];
|
|
62
|
+
for (const r of cfg.market.ranges) {
|
|
63
|
+
if (!r.enabled) continue;
|
|
64
|
+
for (let i = r.from; i <= r.to; i++) {
|
|
65
|
+
const c = toCode(r.prefix, i);
|
|
66
|
+
if (skip.some((p) => c.startsWith(p))) continue;
|
|
67
|
+
out.push(c);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
return out;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* 解析单条腾讯报价。
|
|
75
|
+
* 无有效价(长期停牌 / 退市 / 代码不存在)返回 null —— 调用方据此整体丢弃,
|
|
76
|
+
* 绝不用「现价 0」这种脏数据往下算(否则会污染涨幅、市值等筛选条件)。
|
|
77
|
+
*/
|
|
78
|
+
function parseQuote(code, inner) {
|
|
79
|
+
const f = String(inner).split('~');
|
|
80
|
+
if (f.length < 52) return null;
|
|
81
|
+
const num = (i) => { const v = parseFloat(f[i]); return Number.isFinite(v) ? v : null; };
|
|
82
|
+
const price = num(3);
|
|
83
|
+
if (price == null || price <= 0) return null;
|
|
84
|
+
|
|
85
|
+
const pyOf = (i) => {
|
|
86
|
+
const v = num(i);
|
|
87
|
+
return v == null ? null : v * 1e8; // 腾讯市值单位为「亿」
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
return {
|
|
91
|
+
code,
|
|
92
|
+
name: (f[1] || '').replace(/\s+/g, ''), // 交易所简称含空格(如「五 粮 液」)
|
|
93
|
+
price,
|
|
94
|
+
prevClose: num(4),
|
|
95
|
+
open: num(5),
|
|
96
|
+
high: num(33),
|
|
97
|
+
low: num(34),
|
|
98
|
+
chg: num(31),
|
|
99
|
+
chgPct: num(32),
|
|
100
|
+
vol: num(36), // 手
|
|
101
|
+
amountWan: num(37), // 万元
|
|
102
|
+
turnover: num(38), // %
|
|
103
|
+
volRatio: num(49), // 量比
|
|
104
|
+
avgPrice: num(51), // 当日均价
|
|
105
|
+
floatCap: pyOf(44), // 元
|
|
106
|
+
totalCap: pyOf(45), // 元
|
|
107
|
+
floatCapYi: num(44), // 亿(筛选直接用这个,避免反复换算)
|
|
108
|
+
totalCapYi: num(45),
|
|
109
|
+
high52: num(47),
|
|
110
|
+
low52: num(48),
|
|
111
|
+
stamp: f[30] || '', // YYYYMMDDHHmmss
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** 抓一批代码(60 只左右),返回解析后的报价数组;代码不存在会被响应省略 */
|
|
116
|
+
async function fetchBatch(codes, cfg) {
|
|
117
|
+
const txt = await httpGet(QT_QUOTE + codes.join(','), {
|
|
118
|
+
enc: 'gbk',
|
|
119
|
+
json: false,
|
|
120
|
+
retries: cfg.market.retries,
|
|
121
|
+
timeout: cfg.market.timeoutMs,
|
|
122
|
+
});
|
|
123
|
+
const out = [];
|
|
124
|
+
for (const seg of txt.split(';')) {
|
|
125
|
+
const m = seg.match(/v_([a-z]{2}\d{6})="([^"]*)"/);
|
|
126
|
+
if (!m) continue;
|
|
127
|
+
const q = parseQuote(m[1], m[2]);
|
|
128
|
+
if (q) out.push(q);
|
|
129
|
+
}
|
|
130
|
+
return out;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* 全市场快照。
|
|
135
|
+
* @returns {{quotes:Array, meta:object}}
|
|
136
|
+
* meta = { slots, batches, failedBatches, valid, marketStamp, marketDate, elapsedMs }
|
|
137
|
+
* marketDate 由**报价时间戳的众数**推出 —— 这是判断「今天到底开没开盘」的依据
|
|
138
|
+
* (比维护节假日日历可靠,且天然覆盖周末 / 法定节假日 / 临时休市)。
|
|
139
|
+
*/
|
|
140
|
+
async function fullMarketSnapshot(cfg, log = () => {}) {
|
|
141
|
+
const codes = expandCodes(cfg);
|
|
142
|
+
const B = cfg.market.batch;
|
|
143
|
+
const batches = [];
|
|
144
|
+
for (let i = 0; i < codes.length; i += B) batches.push(codes.slice(i, i + B));
|
|
145
|
+
|
|
146
|
+
const t0 = Date.now();
|
|
147
|
+
let done = 0;
|
|
148
|
+
const results = await mapLimit(batches, cfg.market.concurrency, async (b) => {
|
|
149
|
+
const r = await fetchBatch(b, cfg);
|
|
150
|
+
done++;
|
|
151
|
+
// 进度写 stderr,避免污染 stdout(stdout 留给 --json 之类的机器输出)
|
|
152
|
+
if (done % 50 === 0) log(` 行情批次 ${done}/${batches.length}`);
|
|
153
|
+
return r;
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
const quotes = [];
|
|
157
|
+
let failedBatches = 0;
|
|
158
|
+
for (const r of results) {
|
|
159
|
+
if (r && r.__error) { failedBatches++; continue; }
|
|
160
|
+
quotes.push(...r);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// 行情时间戳众数 -> 交易日
|
|
164
|
+
const byDate = new Map();
|
|
165
|
+
for (const q of quotes) {
|
|
166
|
+
const d = String(q.stamp || '').slice(0, 8);
|
|
167
|
+
if (d) byDate.set(d, (byDate.get(d) || 0) + 1);
|
|
168
|
+
}
|
|
169
|
+
let marketStamp = '';
|
|
170
|
+
let best = 0;
|
|
171
|
+
for (const [d, n] of byDate) if (n > best) { best = n; marketStamp = d; }
|
|
172
|
+
|
|
173
|
+
return {
|
|
174
|
+
quotes,
|
|
175
|
+
meta: {
|
|
176
|
+
slots: codes.length,
|
|
177
|
+
batches: batches.length,
|
|
178
|
+
failedBatches,
|
|
179
|
+
valid: quotes.length,
|
|
180
|
+
outlierDates: [...byDate.keys()].filter((d) => d !== marketStamp),
|
|
181
|
+
marketStamp,
|
|
182
|
+
marketDate: stampToDate(marketStamp),
|
|
183
|
+
elapsedMs: Date.now() - t0,
|
|
184
|
+
},
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** 个股分时(返回最新一个交易日);rows: [{t:'HHMM', price, cumVol, cumAmt}] */
|
|
189
|
+
async function fetchMinute(code, cfg) {
|
|
190
|
+
const obj = await _minuteEp.tryAll(
|
|
191
|
+
(ep) => httpGet(ep.url(code), {
|
|
192
|
+
json: true,
|
|
193
|
+
retries: 2, // 故意压低:被拦时快速换下一台镜像
|
|
194
|
+
timeout: cfg.market.timeoutMs,
|
|
195
|
+
}),
|
|
196
|
+
(ep) => _log(`[降级] 分时数据源切换为 ${ep.name}`),
|
|
197
|
+
);
|
|
198
|
+
const node = (obj.data && obj.data[code]) || {};
|
|
199
|
+
const arr = (node.data && node.data.data) || [];
|
|
200
|
+
const rows = arr
|
|
201
|
+
.map((line) => {
|
|
202
|
+
const p = String(line).split(/\s+/);
|
|
203
|
+
return { t: p[0], price: +p[1], cumVol: +p[2], cumAmt: +p[3] };
|
|
204
|
+
})
|
|
205
|
+
.filter((r) => r.t && Number.isFinite(r.price) && r.price > 0);
|
|
206
|
+
return { rows, date: (node.data && node.data.date) || null };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** 取分时序列里「时间 ≤ hhmm」的最后一条(hhmm 为 'HHMM' 字符串,零填充可直接字典序比较) */
|
|
210
|
+
function barAtOrBefore(rows, hhmm) {
|
|
211
|
+
let best = null;
|
|
212
|
+
for (const r of rows) {
|
|
213
|
+
if (r.t <= hhmm) best = r;
|
|
214
|
+
else break;
|
|
215
|
+
}
|
|
216
|
+
return best;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** 取分时序列里时间正好等于 hhmm 的一条 */
|
|
220
|
+
function barAt(rows, hhmm) {
|
|
221
|
+
return rows.find((r) => r.t === hhmm) || null;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** 起点 K 线的时间容差(分钟):缺失时最多向前回退这么多,超过仍判数据不足 */
|
|
225
|
+
const SEG_START_LAG_MIN = 3;
|
|
226
|
+
|
|
227
|
+
/** 'HHMM' -> 当日分钟数 */
|
|
228
|
+
function hhmmToMinOfDay(hhmm) {
|
|
229
|
+
const s = String(hhmm);
|
|
230
|
+
return (+s.slice(0, 2)) * 60 + (+s.slice(2, 4));
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* 计算尾盘段指标(FR-04 的核心)。
|
|
235
|
+
*
|
|
236
|
+
* 段区间 segFrom → cut:
|
|
237
|
+
* - cut 通常取 '1450'(固定口径);观察模式取「当前时点」;盘中模式(--intraday)
|
|
238
|
+
* 取「当前已成交的最后一分钟」,且 segFrom 变成「cut 往前 segMinutes 分钟」的滚动窗口。
|
|
239
|
+
* - 起点用 segFrom 那根 K 线的**收盘价**(不是开盘价):尾盘动能衡量的是
|
|
240
|
+
* 「14:30 那一刻之后涨了多少」,用收盘价才能和 cut 端的收盘价同口径。
|
|
241
|
+
*
|
|
242
|
+
* @returns {object|null} 数据不足(起点或终点缺失)返回 null,调用方记为 data_gap
|
|
243
|
+
*/
|
|
244
|
+
function tailMetrics(rows, code, cutHHMM, segFrom) {
|
|
245
|
+
// 起点:优先精确命中 segFrom;缺失时向前回退,但**回退幅度不得超过 SEG_START_LAG_MIN**。
|
|
246
|
+
// 为什么允许回退:分时接口偶有一两分钟无数据,此时把整只票判为 data_gap 是过度惩罚,
|
|
247
|
+
// 而起点差 1–3 分钟对「段涨幅」的影响远小于被整只丢掉。正式口径下 14:30 通常存在,
|
|
248
|
+
// 故行为不变(有则精确命中);盘中模式的滚动起点才真正依赖这条容错。
|
|
249
|
+
let p0 = barAt(rows, segFrom);
|
|
250
|
+
if (!p0) {
|
|
251
|
+
const back = barAtOrBefore(rows, segFrom);
|
|
252
|
+
const lag = back ? hhmmToMinOfDay(segFrom) - hhmmToMinOfDay(back.t) : Infinity;
|
|
253
|
+
if (back && lag >= 0 && lag <= SEG_START_LAG_MIN) p0 = back;
|
|
254
|
+
}
|
|
255
|
+
const p1 = barAtOrBefore(rows, cutHHMM);
|
|
256
|
+
if (!p0 || !p1 || p1.t < p0.t || !(p0.price > 0)) return null;
|
|
257
|
+
|
|
258
|
+
const lot = volLotSize(code);
|
|
259
|
+
const segPct = ((p1.price - p0.price) / p0.price) * 100;
|
|
260
|
+
|
|
261
|
+
// 段内明细:上涨分钟占比 + 最大回撤(判断是「一路走高」还是「尾盘一根拉起来」)
|
|
262
|
+
// 窗口起点用 p0.t(实际命中的那根),而不是请求的 segFrom —— 两者在发生回退时不同。
|
|
263
|
+
const win = rows.filter((r) => r.t >= p0.t && r.t <= cutHHMM);
|
|
264
|
+
let up = 0;
|
|
265
|
+
let down = 0;
|
|
266
|
+
let peak = -Infinity;
|
|
267
|
+
let maxDd = 0;
|
|
268
|
+
let prev = p0.price;
|
|
269
|
+
for (const r of win) {
|
|
270
|
+
if (r.price > prev) up++;
|
|
271
|
+
else if (r.price < prev) down++;
|
|
272
|
+
prev = r.price;
|
|
273
|
+
if (r.price > peak) peak = r.price;
|
|
274
|
+
if (peak > 0) maxDd = Math.max(maxDd, ((peak - r.price) / peak) * 100);
|
|
275
|
+
}
|
|
276
|
+
const dirTotal = up + down;
|
|
277
|
+
|
|
278
|
+
// 均价(VWAP)到 cut 时点:累计成交额 / 累计成交股数
|
|
279
|
+
const avgAtCut = (p1.cumVol > 0 && p1.cumAmt > 0) ? p1.cumAmt / (p1.cumVol * lot) : null;
|
|
280
|
+
const priceVsAvgPct = (avgAtCut && avgAtCut > 0) ? ((p1.price / avgAtCut) - 1) * 100 : null;
|
|
281
|
+
|
|
282
|
+
return {
|
|
283
|
+
segFrom,
|
|
284
|
+
segFromUsed: p0.t, // 实际命中的起点(回退过则与 segFrom 不同,报告/存档用来看真实口径)
|
|
285
|
+
cut: p1.t,
|
|
286
|
+
p0: p0.price,
|
|
287
|
+
p1: p1.price,
|
|
288
|
+
segPct,
|
|
289
|
+
upRatio: dirTotal ? up / dirTotal : null,
|
|
290
|
+
maxDrawdownPct: maxDd,
|
|
291
|
+
avgAtCut,
|
|
292
|
+
priceVsAvgPct,
|
|
293
|
+
bars: win.length,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* 并发抓多只票的分时(只对「初筛通过」的小集合调用)。
|
|
299
|
+
*
|
|
300
|
+
* 自带两道闸门,都来自实测教训:
|
|
301
|
+
* 1. 限速:每个 worker 每次请求后 sleep minuteDelayMs,避免把分时接口打进 WAF;
|
|
302
|
+
* 2. 熔断:失败率超过 minuteMaxFailRate 立即抛错中止。
|
|
303
|
+
* 没有熔断时,被 WAF 拦截的 486 只票每只重试 19 秒,整体空跑半小时以上,
|
|
304
|
+
* 而且**看起来像是在正常工作**——静默慢跑比报错危险得多。
|
|
305
|
+
*/
|
|
306
|
+
async function fetchMinutes(codes, cfg, log = _log) {
|
|
307
|
+
const limit = Math.max(1, Math.min(cfg.market.minuteConcurrency, codes.length));
|
|
308
|
+
const delay = cfg.market.minuteDelayMs || 0;
|
|
309
|
+
const maxFails = Math.max(3, Math.ceil(codes.length * (cfg.market.minuteMaxFailRate || 0.3)));
|
|
310
|
+
|
|
311
|
+
const res = new Map();
|
|
312
|
+
let idx = 0;
|
|
313
|
+
let done = 0;
|
|
314
|
+
let fails = 0;
|
|
315
|
+
let abort = null;
|
|
316
|
+
|
|
317
|
+
const worker = async () => {
|
|
318
|
+
while (idx < codes.length && !abort) {
|
|
319
|
+
const c = codes[idx++];
|
|
320
|
+
try {
|
|
321
|
+
res.set(c, await fetchMinute(c, cfg));
|
|
322
|
+
} catch (e) {
|
|
323
|
+
res.set(c, { error: String((e && e.message) || e) });
|
|
324
|
+
fails++;
|
|
325
|
+
if (fails >= maxFails) {
|
|
326
|
+
abort = new Error(
|
|
327
|
+
`分时接口失败率过高(${fails} 次失败 / 已处理 ${done}),疑似被 WAF 拦截。`
|
|
328
|
+
+ `已中止本次分时抓取,避免在拦截状态下空跑数十分钟。`,
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
done++;
|
|
333
|
+
if (done % 20 === 0) log(` 分时 ${done}/${codes.length}(失败 ${fails})`);
|
|
334
|
+
if (delay) await sleep(delay);
|
|
335
|
+
}
|
|
336
|
+
};
|
|
337
|
+
await Promise.all(Array.from({ length: limit }, worker));
|
|
338
|
+
if (abort) throw abort;
|
|
339
|
+
return res;
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
module.exports = {
|
|
343
|
+
volLotSize, expandCodes, parseQuote, fetchBatch, fullMarketSnapshot,
|
|
344
|
+
fetchMinute, fetchMinutes, tailMetrics, barAt, barAtOrBefore, setLog,
|
|
345
|
+
};
|