@cloud411716/fancy-webnovel 0.3.15 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,104 +0,0 @@
1
- # 平台 × 篇幅 → Scraper 映射表
2
-
3
- ## 平台速查
4
-
5
- | 平台 | 中文 | 默认篇幅 | 支持长篇? | 支持短篇? |
6
- |------|------|---------|-----------|-----------|
7
- | `qidian` | 起点 | 长篇 | ✅ | ❌ |
8
- | `fanqie` | 番茄 | 长篇 | ✅ | ❌ |
9
- | `jinjiang` | 晋江 | 长篇 | ✅ | ❌ |
10
- | `zhihu` | 知乎 | 需选择 | ✅ | ✅ |
11
- | `qimao` | 七猫 | 需选择 | ✅ | ✅ |
12
-
13
- ---
14
-
15
- ## 环境要求
16
-
17
- > ⚠️ **所有 scraper 现在使用 playwright-core 自己管理浏览器,不再依赖 agent-browser 和 Chrome CDP 端口。**
18
- > 每个 scraper 独立 launch/close 浏览器进程,采集完成后自动关闭。
19
- > 如需在采集前留时间登录,可传入 `--login-wait <秒>` 参数。
20
-
21
- ---
22
-
23
- ## Scraper 映射
24
-
25
- ### 起点(qidian)
26
-
27
- | 篇幅 | Scraper | 采集模式 | 示例命令 |
28
- |------|---------|---------|---------|
29
- | 长篇 | qidian-rank-scraper.js | **mobile-SSR 优先**(不需要 Chrome)| `node qidian-rank-scraper.js --type hotsales --outdir {out}` |
30
-
31
- ### 番茄(fanqie)
32
-
33
- | 篇幅 | Scraper | 采集模式 | 示例命令 |
34
- |------|---------|---------|---------|
35
- | 长篇 | fanqie-rank-scraper.cjs | playwright-native | `node fanqie-rank-scraper.cjs --channel all --type all --outdir {out}` |
36
-
37
- ### 晋江(jinjiang)
38
-
39
- | 篇幅 | Scraper | 采集模式 | 示例命令 |
40
- |------|---------|---------|---------|
41
- | 长篇 | jjwxc-rank-scraper.cjs | playwright-native | `node jjwxc-rank-scraper.cjs --type 12 --outdir {out}` |
42
-
43
- ### 知乎(zhihu)
44
-
45
- | 篇幅 | Scraper | 采集模式 | 示例命令 |
46
- |------|---------|---------|---------|
47
- | 长篇 | zhihu-rank-scraper.cjs | playwright-native | `node zhihu-rank-scraper.cjs --length long --outdir {out}` |
48
- | 短篇 | zhihu-rank-scraper.cjs | playwright-native | `node zhihu-rank-scraper.cjs --length short --outdir {out}` |
49
-
50
- 入口 URL:`https://www.zhihu.com/fiore/h5/vip-web`
51
-
52
- - 短篇自动点击 4 个 tab:推荐榜 / 热搜榜 / 热度榜 / 口碑榜
53
- - 长篇自动点击 1 个 tab:长篇榜
54
-
55
- > ⚠️ 知乎**需要登录**才能看完整榜单。先在 Chrome 中手动登录 zhihu.com 再采集。
56
-
57
- ### 七猫(qimao)
58
-
59
- | 篇幅 | Scraper | 采集模式 | 示例命令 |
60
- |------|---------|---------|---------|
61
- | 长篇 | qimao-rank-scraper.cjs | playwright-native | `node qimao-rank-scraper.cjs --type hot --period day --outdir {out}` |
62
- | 短篇 | qimao-rank-scraper.cjs | playwright-native | `node qimao-rank-scraper.cjs --type hot --period day --gender f --outdir {out}` |
63
-
64
- ---
65
-
66
- ## 长短篇判断规则
67
-
68
- **如果用户只写了 `qimao` 或 `zhihu` 而没有写 `long`/`short`**:
69
- - 主会话**必须**用 clarify 弹出:`{平台中文名} 同时支持长篇和短篇,请输入 长篇/短篇`
70
- - 禁止跳过此步骤
71
-
72
- ---
73
-
74
- ## 脚本路径约定
75
-
76
- Scraper 脚本统一放在 `{SKILL_DIR}/references/` 下:
77
-
78
- ```
79
- fancy-scan/
80
- ├── SKILL.md
81
- └── references/
82
- ├── run-scan.py ← Python 编排层
83
- ├── scraper-registry.md ← 本文件
84
- ├── scan-output-format.md
85
- ├── cdp-utils.cjs
86
- ├── qidian-rank-scraper.js
87
- ├── fanqie-rank-scraper.cjs
88
- ├── jjwxc-rank-scraper.cjs
89
- ├── zhihu-rank-scraper.cjs
90
- └── qimao-rank-scraper.cjs
91
- ```
92
-
93
- > ⚠️ 如果 `{SKILL_DIR}/references/` 下没有某个 scraper,run-scan.py 回退到"内置知识"模式,输出占位符报告并标注 `[内置知识-未采集]`。
94
-
95
- ---
96
-
97
- ## 输出文件命名规范
98
-
99
- `{平台}{榜单名称}_{YYYYMMDD}.md`
100
-
101
- 例:
102
- - `起点畅销榜_20260827.md`
103
- - `番茄男频阅读榜_20260827.md`
104
- - `七猫大热榜_20260827.md`
@@ -1,75 +0,0 @@
1
- /**
2
- * 工具函数 — 各平台采集脚本的公共依赖
3
- *
4
- * 提供:
5
- * getArg() — 命令行参数解析
6
- * localDateStamp() — 本地日期戳(YYYYMMDD)
7
- * runCli() — CLI 入口包装(含进程超时和信号处理)
8
- *
9
- * 首次使用注意:
10
- * playwright-core 需要 Chromium 二进制。
11
- * 若报错 "Executable doesn't exist",运行一次以下命令下载:
12
- * node -e "require('playwright-core').chromium.launch().then(b => b.close())"
13
- * Chromium 会缓存到 ~/.cache/ms-playwright/(Linux)或对应平台缓存目录。
14
- */
15
-
16
- /** 解析 --xxx 参数 */
17
- function getArg(args, name) {
18
- const i = args.indexOf(name);
19
- if (i >= 0) return i + 1 < args.length ? args[i + 1] : null;
20
- const prefix = `${name}=`;
21
- const inline = args.find(arg => String(arg).startsWith(prefix));
22
- return inline === undefined ? null : String(inline).slice(prefix.length);
23
- }
24
-
25
- /**
26
- * 输出文件名用的日期戳(YYYYMMDD),一律取本地日历日。
27
- * 不能用 new Date().toISOString().slice(0,10):那是 UTC 日期,比 UTC+8 晚 8 小时。
28
- */
29
- function localDateStamp(date) {
30
- const d = date instanceof Date ? date : new Date();
31
- const y = String(d.getFullYear()).padStart(4, "0");
32
- const m = String(d.getMonth() + 1).padStart(2, "0");
33
- const day = String(d.getDate()).padStart(2, "0");
34
- return `${y}${m}${day}`;
35
- }
36
-
37
- /**
38
- * Run a scraper entrypoint 并将结果转换为机器可读的退出码。
39
- * 返回值支持:
40
- * 整数 — 直接作为 written 计数
41
- * 对象 — { planned, written, failed, partial, partialReasons }
42
- */
43
- function runCli(main, label) {
44
- Promise.resolve()
45
- .then(main)
46
- .then(result => {
47
- const outcome = Number.isInteger(result)
48
- ? { planned: result, written: result, failed: 0, partial: false, partialReasons: [] }
49
- : result;
50
- if (!outcome || !Number.isInteger(outcome.written) || outcome.written < 1) {
51
- throw new Error("no output was written");
52
- }
53
- const failed = Number.isInteger(outcome.failed) ? outcome.failed : 0;
54
- const planned = Number.isInteger(outcome.planned)
55
- ? outcome.planned
56
- : outcome.written + failed;
57
- const reasons = Array.isArray(outcome.partialReasons)
58
- ? outcome.partialReasons.filter(Boolean).map(String)
59
- : [];
60
- if (outcome.partial || failed > 0) {
61
- const details = [`wrote ${outcome.written}/${planned}`];
62
- if (failed > 0) details.push(`failed ${failed}`);
63
- details.push(...reasons);
64
- console.error(`${label} partial: ${details.join("; ")}`);
65
- process.exitCode = 2;
66
- }
67
- })
68
- .catch(error => {
69
- const message = error && error.message ? error.message : String(error);
70
- console.error(`${label} failed: ${message}`);
71
- process.exitCode = 1;
72
- });
73
- }
74
-
75
- module.exports = { getArg, localDateStamp, runCli };
@@ -1,441 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * 番茄小说排行榜采集脚本
4
- *
5
- * 使用 playwright-core 自己管理浏览器生命周期。
6
- * 采集策略:
7
- * 1. 从榜单页 __INITIAL_STATE__ 取书 ID 列表
8
- * 2. 逐本打开详情页,用 innerText() 提取渲染后的真实文字(绕过字体加密)
9
- * 3. 输出 Markdown 格式
10
- *
11
- * 重试机制:每个品类/每个榜单 最多重试3次,每次间隔5秒。
12
- *
13
- * 用法:
14
- * node fanqie-rank-scraper.cjs --channel 1 --type 2
15
- * node fanqie-rank-scraper.cjs --channel all
16
- */
17
-
18
- const fs = require("fs");
19
- const path = require("path");
20
- const { chromium } = require("playwright-core");
21
- const { getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
22
-
23
- const DETAIL_CHUNK = 5; // 每次并行打开的详情页数量
24
- const MAX_RETRIES = 3;
25
- const RETRY_DELAY_MS = 5000;
26
-
27
- function sleep(ms) {
28
- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
29
- }
30
-
31
- /**
32
- * 重试包装器
33
- * @returns {{_failed:true, err, label}|*} 失败时返回 _failed=true 的标记对象
34
- */
35
- async function withRetry(fn, retries = MAX_RETRIES, delayMs = RETRY_DELAY_MS, label = '') {
36
- let lastErr;
37
- for (let attempt = 1; attempt <= retries; attempt++) {
38
- try {
39
- return await fn();
40
- } catch (err) {
41
- lastErr = err;
42
- if (attempt < retries) {
43
- console.log(` ⏳ ${label} 第${attempt}次失败,${delayMs / 1000}s后重试...`);
44
- sleep(delayMs);
45
- }
46
- }
47
- }
48
- return { _failed: true, err: lastErr, label };
49
- }
50
-
51
- // ---------------------------------------------------------------------------
52
- // 页面提取工具
53
- // ---------------------------------------------------------------------------
54
-
55
- /** 连通性自检 */
56
- async function probePage(page) {
57
- return page.evaluate(() => ({
58
- host: window.location.hostname,
59
- hasState: !!(window.__INITIAL_STATE__?.rank || window.__INITIAL_STATE__?.rankData),
60
- }));
61
- }
62
-
63
- /** 从榜单页 __INITIAL_STATE__ 提取品类列表 */
64
- async function extractCategories(page, channel, type) {
65
- return page.evaluate(({ channel, type }) => {
66
- const s = window.__INITIAL_STATE__ || {};
67
- let cats = null;
68
- const cands = [
69
- s.rank?.categories, s.rank?.categoryList,
70
- s.rankData?.categories, s.page?.categories,
71
- ];
72
- for (const c of cands) {
73
- if (Array.isArray(c) && c.length) { cats = c; break; }
74
- }
75
- if (!cats) {
76
- const links = Array.from(document.querySelectorAll(`a[href*="/rank/${channel}_${type}_"]`));
77
- const seen = new Set();
78
- cats = links
79
- .map(a => ({ name: a.innerText.trim() || a.textContent.trim(), href: a.getAttribute('href') }))
80
- .filter(c => c.name && c.href && !seen.has(c.href) && seen.add(c.href));
81
- }
82
- return (cats || []).slice(0, 20);
83
- }, { channel, type });
84
- }
85
-
86
- /** 从榜单页 __INITIAL_STATE__ 提取书籍 ID 列表 */
87
- async function extractBookIds(page) {
88
- // 等 __INITIAL_STATE__ 有书单数据
89
- try {
90
- await page.waitForFunction(
91
- () => {
92
- const s = window.__INITIAL_STATE__ || {};
93
- const cands = [s.rank, s.rankData, s.page];
94
- for (const c of cands) {
95
- if (c && (c.book_list || c.bookList || c.rankList)) return true;
96
- }
97
- return false;
98
- },
99
- { timeout: 15000 }
100
- );
101
- } catch (_) { return []; }
102
-
103
- return page.evaluate(() => {
104
- const s = window.__INITIAL_STATE__ || {};
105
- let list = null;
106
- for (const c of [s.rank, s.rankData, s.page]) {
107
- if (c?.book_list) { list = c.book_list; break; }
108
- if (c?.bookList) { list = c.bookList; break; }
109
- if (c?.rankList) { list = c.rankList; break; }
110
- }
111
- if (!list) {
112
- let found = null;
113
- (function walk(o, d) {
114
- if (found || !o || d > 6) return;
115
- if (Array.isArray(o) && o.length && o[0] && (o[0].bookId || o[0].book_id)) {
116
- found = o; return;
117
- }
118
- if (typeof o === 'object') {
119
- for (const k in o) { try { walk(o[k], d + 1); } catch (_) {} }
120
- }
121
- })(s, 0);
122
- list = found || [];
123
- }
124
- return (list || []).map(b => String(b.bookId || b.book_id || '')).filter(Boolean);
125
- });
126
- }
127
-
128
- /**
129
- * 打开书籍详情页,提取渲染后的干净文字
130
- * @param {object} browser
131
- * @param {string[]} bookIds
132
- * @returns {Promise<object>} { bookId: { title, author, category, desc } }
133
- */
134
- async function fetchBookDetails(browser, bookIds) {
135
- const map = {};
136
- // 复用同一个 context 批量开详情页(比每次新建 context 快)
137
- const context = await browser.newContext();
138
- try {
139
- for (let i = 0; i < bookIds.length; i += DETAIL_CHUNK) {
140
- const chunk = bookIds.slice(i, i + DETAIL_CHUNK);
141
- // 并行打开多个详情页
142
- const pages = await Promise.all(
143
- chunk.map(id => context.newPage())
144
- );
145
- await Promise.all(
146
- pages.map((p, idx) =>
147
- p.goto(`https://fanqienovel.com/page/${chunk[idx]}`, { waitUntil: 'domcontentloaded' })
148
- .catch(() => {})
149
- )
150
- );
151
- // 等待所有页面的主要元素加载
152
- await Promise.all(
153
- pages.map(p => p.waitForTimeout(1500))
154
- );
155
- // 批量提取
156
- const results = await Promise.all(
157
- pages.map((p, idx) =>
158
- p.evaluate(({ id }) => {
159
- const sel = '.info-wrapper .title, .book-title, h1, .book-name';
160
- const titleEl = document.querySelector(sel);
161
- const title = titleEl ? (titleEl.innerText || titleEl.textContent || '').trim() : '';
162
- // 作者可能在多个位置
163
- const authorEl = document.querySelector('.author-name, .author, .writer') ||
164
- document.querySelector(`a[href*="/author/"]`);
165
- const author = authorEl ? (authorEl.innerText || authorEl.textContent || '').replace(/^作者:/, '').trim() : '';
166
- // 题材/分类
167
- const catEl = document.querySelector('.category, .tag, .book-tag');
168
- const category = catEl ? (catEl.innerText || catEl.textContent || '').trim() : '';
169
- // 简介
170
- const descEl = document.querySelector('.description, .book-desc, .intro');
171
- const desc = descEl ? (descEl.innerText || descEl.textContent || '').trim().slice(0, 200) : '';
172
- return { id, title, author, category, desc };
173
- }, { id: chunk[idx] }).catch(() => ({ id: chunk[idx], title: '', author: '', category: '', desc: '' }))
174
- )
175
- );
176
- for (const r of results) {
177
- map[r.id] = r;
178
- }
179
- await Promise.all(pages.map(p => p.close()));
180
- }
181
- } finally {
182
- await context.close();
183
- }
184
- return map;
185
- }
186
-
187
- async function scrollLoad(page, times, interval = 1000) {
188
- for (let i = 0; i < times; i++) {
189
- await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
190
- await page.waitForTimeout(interval);
191
- }
192
- }
193
-
194
- function localDate() {
195
- const d = new Date();
196
- return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
197
- }
198
-
199
- // ---------------------------------------------------------------------------
200
- // 主流程
201
- // ---------------------------------------------------------------------------
202
-
203
- const args = process.argv.slice(2);
204
- const PROBE = getArg(args, '--probe') !== null;
205
- const LOGIN_WAIT = parseInt(getArg(args, '--login-wait') || '0', 10);
206
- const OUTDIR = getArg(args, '--outdir') || '.';
207
- const CHANNEL = getArg(args, '--channel') || '1';
208
- const TYPE = getArg(args, '--type') || '2';
209
- const TOP = parseInt(getArg(args, '--top') || '20', 10);
210
-
211
- function channelLabel(ch) {
212
- return ch === '1' ? '男频' : ch === '0' ? '女频' : ch;
213
- }
214
- function typeLabel(t) {
215
- return t === '2' ? '阅读榜' : t === '1' ? '新书榜' : t;
216
- }
217
-
218
- async function scrapeCategory(browser, page, cat, chLabel) {
219
- return withRetry(async () => {
220
- await page.goto(`https://fanqienovel.com${cat.href}`, { waitUntil: 'domcontentloaded' });
221
- await page.waitForTimeout(2000);
222
- await scrollLoad(page, 2);
223
-
224
- const bookIds = await extractBookIds(page);
225
- if (!bookIds || !bookIds.length) {
226
- throw new Error('extractBookIds 返回空');
227
- }
228
-
229
- // 打开详情页提取渲染后文字
230
- const details = await fetchBookDetails(browser, bookIds.slice(0, TOP));
231
-
232
- const lines = [];
233
- for (const id of bookIds.slice(0, TOP)) {
234
- const info = details[id] || {};
235
- const title = info.title || `(标题待解析 ${id})`;
236
- const author = info.author || '未知';
237
- const category = info.category || cat.name;
238
- const desc = info.desc ? `简介:${info.desc}` : '';
239
- lines.push(`书名:${title}`);
240
- lines.push(`题材:${category}`);
241
- lines.push(`作者:${author}`);
242
- if (desc) lines.push(desc);
243
- lines.push(`作品页:https://fanqienovel.com/page/${id}`);
244
- lines.push('');
245
- }
246
- return lines;
247
- }, MAX_RETRIES, RETRY_DELAY_MS, `品类[${cat.name}]`);
248
- }
249
-
250
- async function scrapeChannel(ch, type) {
251
- const chLabel = channelLabel(ch);
252
- const tyLabel = typeLabel(type);
253
- console.log(`\n→ 采集 ${chLabel}${tyLabel}...`);
254
-
255
- const initCatId = ch === '1' ? '1141' : '1139';
256
- const initUrl = `https://fanqienovel.com/rank/${ch}_${type}_${initCatId}`;
257
-
258
- const browser = await chromium.launch({
259
- headless: true,
260
- args: ['--no-sandbox', '--disable-dev-shm-usage', '--start-maximized'],
261
- });
262
- const context = await browser.newContext();
263
- const page = await context.newPage();
264
-
265
- // CDP 窗口最大化
266
- try {
267
- const cdp = await context.newCDPSession(page);
268
- const { windowId } = await cdp.send('Browser.getWindowForTarget', { target: page.target() });
269
- await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
270
- } catch (_) {}
271
-
272
- let scraped = null;
273
-
274
- try {
275
- // 入口页(带重试)
276
- const homeResult = await withRetry(async () => {
277
- await page.goto(initUrl, { waitUntil: 'domcontentloaded' });
278
- const probe = await probePage(page);
279
- if (!probe.host || probe.host.indexOf('fanqie') === -1) {
280
- throw new Error(`非番茄页面(host=${probe.host})`);
281
- }
282
- return probe;
283
- }, MAX_RETRIES, RETRY_DELAY_MS, '打开入口页');
284
-
285
- if (homeResult._failed) {
286
- console.log(` ✗ 入口页连续失败:${homeResult.err.message}`);
287
- return null;
288
- }
289
-
290
- if (LOGIN_WAIT > 0) {
291
- console.log(` ⏳ 等待 ${LOGIN_WAIT}s 供用户登录...`);
292
- await page.waitForTimeout(LOGIN_WAIT * 1000);
293
- }
294
-
295
- // PROBE 调试模式(保留原有逻辑)
296
- if (PROBE) {
297
- const intercepted = [];
298
- const seen = new Set();
299
- context.on('request', req => {
300
- const url = req.url();
301
- if (seen.has(url) || /\.(css|woff|jpg|png|ico)/.test(url) || !url.includes('fanqie')) return;
302
- seen.add(url);
303
- console.log(`[REQ] ${url}`);
304
- });
305
- page.on('response', async resp => {
306
- const url = resp.url();
307
- if (seen.has(url)) return;
308
- const ct = (resp.headers()['content-type'] || '').toLowerCase();
309
- if (!ct.includes('json') && !ct.includes('text') && !url.includes('api')) return;
310
- try {
311
- const body = await resp.text();
312
- if (!body || body.length < 50) return;
313
- const str = JSON.stringify(JSON.parse(body));
314
- if (str.includes('bookId') || str.includes('book_name')) {
315
- console.log(`[JSON] ${url}`);
316
- intercepted.push({ url, preview: str.slice(0, 300) });
317
- }
318
- } catch (_) {}
319
- });
320
- const cats = await extractCategories(page, ch, type);
321
- if (!cats.length) {
322
- await page.goto(`https://fanqienovel.com/rank/${ch}_${type}_${initCatId}`);
323
- await page.waitForTimeout(3000);
324
- }
325
- console.log(`\n=== 共拦截 ${intercepted.length} 个含书籍数据的接口 ===`);
326
- await browser.close();
327
- return null;
328
- }
329
-
330
- // 提取品类
331
- let categories = await extractCategories(page, ch, type);
332
- if (!categories.length) {
333
- await scrollLoad(page, 2);
334
- sleep(1000);
335
- categories = await extractCategories(page, ch, type);
336
- }
337
- if (!categories.length) {
338
- categories = [{ name: '全部(入口页)', href: `/rank/${ch}_${type}_${initCatId}` }];
339
- } else {
340
- console.log(` 发现 ${categories.length} 个品类`);
341
- }
342
-
343
- const now = new Date().toISOString();
344
- const lines = [
345
- `# 番茄 · ${chLabel}${tyLabel} · 全 ${categories.length} 题材`,
346
- '',
347
- `- 频道参数:channel=${ch},type=${type}`,
348
- `- 抓取时间:${now}`,
349
- '',
350
- '---',
351
- '',
352
- ];
353
-
354
- const failedCategories = [];
355
-
356
- for (let ci = 0; ci < categories.length; ci++) {
357
- const cat = categories[ci];
358
- console.log(` [${ci + 1}/${categories.length}] ${cat.name}`);
359
-
360
- const result = await scrapeCategory(browser, page, cat, chLabel);
361
-
362
- if (result._failed) {
363
- console.log(` ✗ 品类[${cat.name}]连续${MAX_RETRIES}次失败:${result.err.message}`);
364
- failedCategories.push({ name: cat.name, err: result.err.message });
365
- lines.push(`## ${cat.name} — 采集失败(已重试${MAX_RETRIES}次)`, '', '---', '');
366
- continue;
367
- }
368
-
369
- const bookLines = result;
370
- if (!bookLines.length) {
371
- lines.push(`## ${cat.name} — 0 本`, '', '---', '');
372
- continue;
373
- }
374
-
375
- lines.push(`## ${cat.name} — ${bookLines.filter(l => l.startsWith('书名:')).length} 本`, '');
376
- lines.push(...bookLines);
377
- lines.push('---', '');
378
- }
379
-
380
- scraped = lines.join('\n');
381
-
382
- if (failedCategories.length > 0) {
383
- console.log(`\n⚠️ 以下品类采集失败(共 ${failedCategories.length} 项):`);
384
- for (const f of failedCategories) {
385
- console.log(` • ${f.name}:${f.err}`);
386
- }
387
- }
388
-
389
- } finally {
390
- await browser.close();
391
- }
392
-
393
- return scraped;
394
- }
395
-
396
- async function main() {
397
- if (!['0', '1', 'all'].includes(CHANNEL)) throw new Error(`未知 --channel: ${CHANNEL}`);
398
- if (!['1', '2', 'all'].includes(TYPE)) throw new Error(`未知 --type: ${TYPE}`);
399
-
400
- const channels = CHANNEL === 'all' ? ['1', '0'] : [CHANNEL];
401
- const types = TYPE === 'all' ? ['2', '1'] : [TYPE];
402
- let written = 0;
403
- const failedChannels = [];
404
-
405
- for (const ch of channels) {
406
- for (const ty of types) {
407
- try {
408
- const content = await scrapeChannel(ch, ty);
409
- if (!content) {
410
- failedChannels.push({ ch, ty, err: '采集返回空' });
411
- continue;
412
- }
413
- const date = localDateStamp();
414
- const filename = `番茄${channelLabel(ch)}${typeLabel(ty)}_全题材_${date}.md`;
415
- fs.mkdirSync(OUTDIR, { recursive: true });
416
- const filepath = path.join(OUTDIR, filename);
417
- fs.writeFileSync(filepath, content, 'utf-8');
418
- written++;
419
- console.log(` ✓ 已保存: ${filepath}`);
420
- } catch (chErr) {
421
- console.log(` ✗ 频道${channelLabel(ch)}${typeLabel(ty)}采集失败:${chErr.message}`);
422
- failedChannels.push({ ch, ty, err: chErr.message });
423
- }
424
- }
425
- }
426
-
427
- if (failedChannels.length > 0) {
428
- console.log(`\n⚠️ 以下榜单采集失败(共 ${failedChannels.length} 项):`);
429
- for (const f of failedChannels) {
430
- console.log(` • ${channelLabel(f.ch)}${typeLabel(f.ty)}:${f.err}`);
431
- }
432
- }
433
-
434
- return written;
435
- }
436
-
437
- if (require.main === module) {
438
- runCli(main, '番茄采集');
439
- }
440
-
441
- module.exports = {};