@cloud411716/fancy-webnovel 0.2.27 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cloud411716/fancy-webnovel",
3
- "version": "0.2.27",
3
+ "version": "0.3.0",
4
4
  "type": "module",
5
5
  "main": "index.js",
6
6
  "exports": {
@@ -5,101 +5,133 @@ import { fileURLToPath } from 'url';
5
5
  import { dirname, join } from 'path';
6
6
  import { existsSync } from 'fs';
7
7
  import { notify, llmFollowup, spawnScript, startActivity } from '../../infra.js';
8
- import { scan, PLATFORM_TABLE, platformLabel } from './templates.js';
8
+ import { scan, PLATFORM_TABLE, rankOptions } from './templates.js';
9
9
 
10
10
  export const inject = ['commands', 'userQuestions'];
11
11
 
12
12
  const __dirname = dirname(fileURLToPath(import.meta.url));
13
13
  const SCAN_SCRIPTS_DIR = join(__dirname, 'scripts');
14
+ const SCRAPER_TIMEOUT = 30 * 60 * 1000; // 30 分钟
14
15
 
15
16
  export async function apply(ctx) {
16
17
  ctx.commands.register({
17
18
  name: 'fancy-scan',
18
- description: '📊 扫榜分析 ( usage: /fancy-scan [平台] [篇幅] )',
19
+ description: '📊 扫榜分析 自动分析起点、番茄、晋江、七猫榜单并生成报告',
19
20
  handler: async (invocation) => {
20
21
  const session = invocation.agent.session;
21
- const rawInput = invocation.rawInput ? invocation.rawInput.trim() : '';
22
22
 
23
23
  // 获取项目根
24
- let projectRoot = rawInput ? rawInput.split(/\s+/)[0] : '';
25
- if (!projectRoot) projectRoot = process.cwd();
26
-
24
+ const projectRoot = process.cwd();
27
25
  if (!existsSync(join(projectRoot, '.fancy-deployed'))) {
28
26
  notify(session, scan.notify({ type: 'not_initialized' }));
29
27
  return { kind: 'success', text: '' };
30
28
  }
31
29
 
32
- // 解析参数
33
- let platform = '', length = '';
34
- if (rawInput) {
35
- const parts = rawInput.split(/\s+/);
36
- platform = parts[0] || '';
37
- length = parts[1] || '';
38
- const numMatch = platform.match(/^(\d+)$/);
39
- if (numMatch) {
40
- const row = PLATFORM_TABLE.find(r => r[0] === numMatch[1]);
41
- if (row) {
42
- platform = row[1];
43
- if (platform === 'zhihu' || platform === 'qimao') {
44
- length = parts[1] || 'long';
45
- } else {
46
- length = row[2] === '长篇' ? 'long' : (row[2] === '短篇' ? 'short' : 'long');
47
- }
48
- }
49
- }
30
+ // ---------- 第一轮:选择平台 ----------
31
+ notify(session, scan.platformList());
32
+
33
+ const platformResult = await ctx.userQuestions.ask({
34
+ questions: [scan.askPlatform()],
35
+ });
36
+ const answer = platformResult.answers[0]?.custom?.trim();
37
+ if (!answer) {
38
+ notify(session, scan.notify({ type: 'cancelled' }));
39
+ return { kind: 'success', text: '' };
40
+ }
41
+ const numMatch = answer.match(/^(\d+)$/);
42
+ const row = numMatch ? PLATFORM_TABLE.find(r => r[0] === numMatch[1]) : null;
43
+ if (!row) {
44
+ notify(session, scan.notify({ type: 'invalid_choice' }));
45
+ return { kind: 'success', text: '' };
50
46
  }
47
+ const platform = row[1];
51
48
 
52
- // 参数不全 弹窗选择
53
- if (!platform || !length || !PLATFORM_TABLE.find(r => r[1] === platform)) {
54
- notify(session, scan.platformList());
55
-
56
- const choice = await ctx.userQuestions.ask({
57
- questions: [scan.askPlatform()],
58
- });
59
- const answer = choice.answers[0]?.custom?.trim();
60
- if (!answer) {
61
- notify(session, scan.notify({ type: 'cancelled_with_hint' }));
62
- return { kind: 'success', text: '' };
63
- }
64
- const numMatch = answer.match(/^(\d+)$/);
65
- const row = numMatch ? PLATFORM_TABLE.find(r => r[0] === numMatch[1]) : null;
66
- if (!row) {
67
- notify(session, scan.notify({ type: 'invalid_choice' }));
68
- return { kind: 'success', text: '' };
69
- }
70
- platform = row[1];
71
- length = (platform === 'zhihu' || platform === 'qimao')
72
- ? null
73
- : (row[2] === '长篇' ? 'long' : 'short');
49
+ // ---------- 第二轮:选择榜单(多选) ----------
50
+ const rankResult = await ctx.userQuestions.ask({
51
+ questions: [scan.askRankList(platform)],
52
+ });
53
+ const selected = rankResult.answers[0]?.selected ?? [];
54
+ const custom = rankResult.answers[0]?.custom?.trim();
55
+
56
+ // 用户可能直接输入自定义值
57
+ const rawChoices = selected.length > 0 ? selected : (custom ? [custom] : []);
58
+ if (rawChoices.length === 0) {
59
+ notify(session, scan.notify({ type: 'cancelled' }));
60
+ return { kind: 'success', text: '' };
61
+ }
62
+
63
+ // 构建每个榜单的 channel+type 参数
64
+ const platformRanks = rankOptions(platform);
65
+ const isAll = rawChoices.includes('全选') || rawChoices.includes('__all__');
66
+ const rankIds = isAll
67
+ ? platformRanks.filter(r => r.id !== '__all__').map(r => r.id)
68
+ : rawChoices;
69
+
70
+ // rankId 转换成 { channel, type } 并去重
71
+ const rankEntries = rankIds.map(id => parseRankId(platform, id));
72
+ const unique = [];
73
+ const seen = new Set();
74
+ for (const e of rankEntries) {
75
+ const key = `${e.channel}__${e.type}`;
76
+ if (!seen.has(key)) { seen.add(key); unique.push(e); }
74
77
  }
75
78
 
76
- // 知乎/七猫二次选择篇幅
77
- if (platform === 'zhihu' || platform === 'qimao') {
78
- if (!length || !['long', 'short'].includes(length)) {
79
- const lenResult = await ctx.userQuestions.ask({
80
- questions: [scan.askLength(platform)],
81
- });
82
- const lenAnswer = lenResult.answers[0]?.selected?.[0] ?? lenResult.answers[0]?.custom;
83
- if (!lenAnswer || lenAnswer === '取消') {
84
- notify(session, scan.notify({ type: 'cancelled' }));
79
+ // ---------- 串行采集 ----------
80
+ let allFiles = [];
81
+ let failed = 0;
82
+
83
+ for (const rank of unique) {
84
+ const stop = startActivity(session);
85
+ const r = await spawnScript(
86
+ 'node',
87
+ [
88
+ join(SCAN_SCRIPTS_DIR, 'run-scan.js'),
89
+ '--project-root', projectRoot,
90
+ '--platform', platform,
91
+ '--channel', rank.channel,
92
+ '--type', rank.type,
93
+ ],
94
+ { timeout: SCRAPER_TIMEOUT, stop }
95
+ );
96
+
97
+ if (!r.ok) {
98
+ const rawErr = (typeof r.error === 'object' && r.error !== null)
99
+ ? (r.error.message || JSON.stringify(r.error))
100
+ : String(r.error || '');
101
+ if (rawErr.includes("Executable doesn't exist") || rawErr.includes("doesn't exist")) {
102
+ notify(session,
103
+ '⚠️ 未找到 Chromium 浏览器。\n\n安装命令:\n' +
104
+ ' npm install -g playwright@1.48.0 --registry=https://registry.npmmirror.com\n' +
105
+ ' PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ playwright install chromium --with-deps\n\n' +
106
+ '安装完成后重新运行 /fancy-scan 即可。'
107
+ );
85
108
  return { kind: 'success', text: '' };
86
109
  }
87
- length = (lenAnswer === '长篇') ? 'long' : 'short';
110
+ notify(session, scan.notify({ type: 'failed', err: rawErr }));
111
+ failed++;
112
+ continue;
88
113
  }
89
- } else if (!length) {
90
- length = 'long';
91
- }
92
-
93
- // 执行采集,包含 Chromium 缺失处理
94
- const scanResult = await runScanWithChromiumFix(ctx, session, invocation, {
95
- platform, length, projectRoot,
96
- });
97
114
 
98
- if (scanResult === 'retry') {
99
- // Chromium 安装后重试已完成,runScanWithChromiumFix 已处理所有输出
100
- return { kind: 'success', text: '' };
115
+ // 解析采集结果
116
+ let receipt;
117
+ try { receipt = JSON.parse(r.output); } catch (_) {
118
+ notify(session, scan.notify({ type: 'parse_error' }));
119
+ failed++;
120
+ continue;
121
+ }
122
+ const files = receipt.scan_files || [];
123
+ allFiles = allFiles.concat(files);
124
+
125
+ // 逐个文件追加到 topic_decision
126
+ const topicFile = receipt.topic_decision || '';
127
+ if (topicFile) {
128
+ notify(session, scan.notify({ type: 'handover', platform, files, topicFile, projectRoot }));
129
+ const prompt = scan.llmAnalysisPrompt({ platform, date: receipt.date || '', files, topicFile, projectRoot });
130
+ llmFollowup(invocation, prompt);
131
+ }
101
132
  }
102
- if (!scanResult) {
133
+
134
+ if (failed > 0 && allFiles.length === 0) {
103
135
  return { kind: 'success', text: '' };
104
136
  }
105
137
 
@@ -108,59 +140,31 @@ export async function apply(ctx) {
108
140
  });
109
141
  }
110
142
 
111
- /**
112
- * 执行采集,检测 Chromium 缺失则提示用户手动安装。
113
- * 返回 undefined 表示已处理完毕(取消/失败),返回 'retry' 表示重试完成。
114
- */
115
- async function runScanWithChromiumFix(ctx, session, invocation, { platform, length, projectRoot }) {
116
- const runOnce = async (isRetry = false) => {
117
- const stop = startActivity(session);
118
- if (isRetry) notify(session, '⏳ 重新采集...\n');
119
- const r = await spawnScript(
120
- 'node',
121
- [join(SCAN_SCRIPTS_DIR, 'run-scan.js'),
122
- '--project-root', projectRoot, '--platform', platform, '--length', length],
123
- { timeout: 900000, stop }
124
- );
125
- return r;
126
- };
127
-
128
- const handleSuccess = (stdout) => {
129
- let receipt;
130
- try { receipt = JSON.parse(stdout); } catch (_) {
131
- notify(session, scan.notify({ type: 'parse_error' }));
132
- return;
143
+ // ---------------------------------------------------------------------------
144
+ // rankId 转换成 { channel, type } 参数
145
+ // ---------------------------------------------------------------------------
146
+
147
+ function parseRankId(platform, id) {
148
+ switch (platform) {
149
+ case 'qimao': {
150
+ // id 格式: male_hot / female_new
151
+ const [ch, rt] = id.split('_');
152
+ return { channel: ch, type: rt };
133
153
  }
134
- const files = receipt.scan_files || [];
135
- const topicFile = receipt.topic_decision || '';
136
- notify(session, scan.notify({ type: 'handover', platform, length, files, topicFile, projectRoot }));
137
- const prompt = scan.llmAnalysisPrompt({ platform, length, date: receipt.date || '', files, topicFile, projectRoot });
138
- llmFollowup(invocation, prompt);
139
- };
140
-
141
- const scriptResult = await runOnce();
142
-
143
- if (!scriptResult.ok) {
144
- const rawErr = (typeof scriptResult.error === 'object' && scriptResult.error !== null)
145
- ? (scriptResult.error.message || JSON.stringify(scriptResult.error))
146
- : String(scriptResult.error || '未知错误');
147
-
148
- // Chromium 未找到 → 提示用户手动安装
149
- if (rawErr.includes("Executable doesn't exist") || rawErr.includes('Executable doesn')) {
150
- notify(session,
151
- '⚠️ 未找到 Chromium 浏览器,请手动安装后再次运行 /fancy-scan。\n\n' +
152
- '安装命令:\n' +
153
- ' npm install -g playwright@1.48.0 --registry=https://registry.npmmirror.com\n' +
154
- ' PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ playwright install chromium --with-deps\n\n' +
155
- '安装完成后重新运行 /fancy-scan 即可。'
156
- );
157
- return undefined;
154
+ case 'fanqie': {
155
+ // id 格式: 0_1 / 1_2 / __all__
156
+ const [ch, ty] = id.split('_');
157
+ return { channel: ch, type: ty };
158
158
  }
159
-
160
- notify(session, scan.notify({ type: 'failed', err: rawErr }));
161
- return undefined;
159
+ case 'jinjiang': {
160
+ // id 格式: 5 / 7 / 12 等(orderstr)
161
+ return { channel: 'all', type: id };
162
+ }
163
+ case 'qidian': {
164
+ // 起点只有主站一个榜单
165
+ return { channel: 'main', type: 'main' };
166
+ }
167
+ default:
168
+ return { channel: 'all', type: 'all' };
162
169
  }
163
-
164
- handleSuccess(scriptResult.output);
165
- return 'retry';
166
170
  }
@@ -4,12 +4,14 @@
4
4
  *
5
5
  * 职责:
6
6
  * 1. 检查 .fancy-deployed
7
- * 2. 调用平台采集脚本
8
- * 3. 生成扫榜报告
9
- * 4. 生成/追加 topic_decision
7
+ * 2. 调用平台采集脚本(一次一个榜单)
8
+ * 3. 输出 JSON 摘要(供 index.js 解析)
10
9
  *
11
10
  * 调用方式:
12
- * node run-scan.js --project-root <abs> --platform qidian --length long
11
+ * node run-scan.js --project-root <abs> --platform qidian --channel main --type main
12
+ * node run-scan.js --project-root <abs> --platform fanqie --channel 1 --type 2
13
+ * node run-scan.js --project-root <abs> --platform jinjiang --channel all --type 5
14
+ * node run-scan.js --project-root <abs> --platform qimao --channel male --type hot
13
15
  */
14
16
 
15
17
  import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
@@ -17,25 +19,13 @@ import { join, dirname } from 'path';
17
19
  import { fileURLToPath } from 'url';
18
20
 
19
21
  // ---------------------------------------------------------------------------
20
- // 平台配置
22
+ // 平台中文名
21
23
  // ---------------------------------------------------------------------------
22
24
 
23
- const VALID_LENGTHS = ['long', 'short'];
24
- const VALID_PLATFORMS = ['qidian', 'fanqie', 'jinjiang', 'zhihu', 'qimao'];
25
-
26
- const PLATFORM_SUPPORTED_LENGTHS = {
27
- qidian: ['long'],
28
- fanqie: ['long'],
29
- jinjiang: ['long'],
30
- zhihu: ['long', 'short'],
31
- qimao: ['long', 'short'],
32
- };
33
-
34
25
  const PLATFORM_CN = {
35
26
  qidian: '起点',
36
27
  fanqie: '番茄',
37
28
  jinjiang: '晋江',
38
- zhihu: '知乎',
39
29
  qimao: '七猫',
40
30
  };
41
31
 
@@ -57,17 +47,17 @@ function ensureDir(dir) {
57
47
  }
58
48
 
59
49
  // ---------------------------------------------------------------------------
60
- // HTTP 采集 — 起点 mobile SSR
50
+ // 起点:HTTP mobile SSR(自包含,无需 playwright)
61
51
  // ---------------------------------------------------------------------------
62
52
 
63
53
  async function scrapeQidian(outDir) {
64
54
  const MOBILE_BASE = 'https://m.qidian.com';
65
55
  const RANK_TYPES = [
66
- { id: 'hotsales', label: '畅销榜', path: '/rank/hotsales/' },
67
- { id: 'yuepiao', label: '月票榜', path: '/rank/yuepiao/' },
68
- { id: 'signnewbook',label: '签约作者新书榜', path: '/rank/sign/' },
69
- { id: 'pubnewbook', label: '公众作者新书榜', path: '/rank/newbook/' },
70
- { id: 'newauthor', label: '新人作者新书榜', path: '/rank/newauthor/' },
56
+ { id: 'hotsales', label: '畅销榜', path: '/rank/hotsales/' },
57
+ { id: 'yuepiao', label: '月票榜', path: '/rank/yuepiao/' },
58
+ { id: 'signnewbook', label: '签约作者新书榜', path: '/rank/sign/' },
59
+ { id: 'pubnewbook', label: '公众作者新书榜', path: '/rank/newbook/' },
60
+ { id: 'newauthor', label: '新人作者新书榜', path: '/rank/newauthor/' },
71
61
  ];
72
62
 
73
63
  const MOBILE_HEADERS = {
@@ -89,7 +79,6 @@ async function scrapeQidian(outDir) {
89
79
  continue;
90
80
  }
91
81
 
92
- // 提取 pageContext JSON
93
82
  const m = html.match(/<script[^>]+id=["']vite-plugin-ssr_pageContext["'][^>]*>([\s\S]*?)<\/script>/i);
94
83
  if (!m) {
95
84
  console.error(` ⚠ ${rt.label} 未找到 pageContext`);
@@ -99,7 +88,7 @@ async function scrapeQidian(outDir) {
99
88
  let pageContext;
100
89
  try {
101
90
  pageContext = JSON.parse(m[1]);
102
- } catch (e) {
91
+ } catch (_) {
103
92
  console.error(` ⚠ ${rt.label} JSON 解析失败`);
104
93
  continue;
105
94
  }
@@ -111,203 +100,105 @@ async function scrapeQidian(outDir) {
111
100
  }
112
101
 
113
102
  const books = records.map((r, i) => ({
114
- rank: r.rankNum || i + 1,
115
- title: r.bName || r.bookName || '',
116
- url: r.bid ? `${MOBILE_BASE}/book/${r.bid}/` : '',
117
- author: r.bAuth || r.author || '',
118
- genre: [r.cat, r.subCat].filter(Boolean).join('·'),
119
- status: r.status || '',
120
- words: r.cnt || r.wordCount || '',
121
- rankValue: r.rankCnt || '',
122
- totalRecommend: r.totalRecommend || '',
123
- signStatus: r.signStatus || '',
124
- vipStatus: r.vipStatus || '',
103
+ rank: r.rankNum || i + 1,
104
+ title: r.bName || r.bookName || '',
105
+ url: r.bid ? `${MOBILE_BASE}/book/${r.bid}/` : '',
106
+ author: r.bAuth || r.author || '',
107
+ genre: [r.cat, r.subCat].filter(Boolean).join('·'),
108
+ status: r.status || '',
109
+ words: r.cnt || r.wordCount || '',
125
110
  }));
126
111
 
127
- const today = todayStr();
128
- const outFile = join(outDir, `${PLATFORM_CN.qidian}${rt.label}_${today}.md`);
129
- const md = renderQidianMarkdown(rt.label, url, books);
130
- writeFileSync(outFile, md, 'utf-8');
131
- console.error(` ✅ ${rt.label}: ${books.length} 本 → ${outFile}`);
112
+ const outFile = join(outDir, `${PLATFORM_CN.qidian}${rt.label}_${todayStr()}.md`);
113
+ writeFileSync(outFile, renderMarkdown(rt.label, url, books), 'utf-8');
114
+ console.error(` ✅ ${rt.label}: ${books.length} 本`);
132
115
  results.push({ label: rt.label, count: books.length, file: outFile });
133
116
  }
134
117
 
135
118
  return results;
136
119
  }
137
120
 
138
- function renderQidianMarkdown(rankLabel, url, books) {
121
+ function renderMarkdown(rankLabel, url, books) {
139
122
  const now = nowIso();
140
123
  const lines = [
141
124
  `# 起点 · ${rankLabel}`,
142
- '',
125
+ ``,
143
126
  `- 来源:${url}`,
144
127
  `- 抓取方式:mobile-ssr`,
145
128
  `- 抓取时间:${now}`,
146
- '',
129
+ ``,
147
130
  '---',
148
131
  '',
149
132
  ];
150
-
151
133
  for (const b of books) {
152
- lines.push(`书名:${b.title || ''}`);
153
- lines.push(`题材:${b.genre || ''}`);
154
- lines.push(`作者:${b.author || ''}`);
134
+ lines.push(`书名:${b.title}`);
135
+ lines.push(`题材:${b.genre}`);
136
+ lines.push(`作者:${b.author}`);
155
137
  if (b.url) lines.push(`作品页:${b.url}`);
156
138
  lines.push('');
157
139
  }
158
-
159
140
  return lines.join('\n');
160
141
  }
161
142
 
162
143
  // ---------------------------------------------------------------------------
163
- // Browser scraper spawner(调用 scripts/scrapers/ 下的 CJS 脚本)
144
+ // Browser scraper spawner(30 分钟超时)
164
145
  // ---------------------------------------------------------------------------
165
146
 
166
- function spawnScraper(platform, length, outDir) {
147
+ function spawnScraper(script, extraArgs, outDir) {
167
148
  return new Promise(async (resolve) => {
168
149
  const { spawn } = await import('child_process');
169
- const args = scraperArgs(platform, length, outDir);
170
- const scraperScript = join(dirname(fileURLToPath(import.meta.url)), 'scrapers', args.script);
171
- const fullArgs = [...args.extra];
172
-
173
- process.stderr.write(`\n→ 启动采集脚本...`);
174
- const p = spawn('node', [scraperScript, ...fullArgs], {
175
- timeout: 900000,
150
+ const scriptPath = join(dirname(fileURLToPath(import.meta.url)), 'scrapers', script);
151
+ process.stderr.write(`\n→ 启动 ${script}...\n`);
152
+ const p = spawn('node', [scriptPath, ...extraArgs], {
153
+ timeout: 30 * 60 * 1000,
176
154
  stdio: ['ignore', 'pipe', 'inherit'],
177
155
  });
178
156
  let out = '';
179
157
  p.stdout.on('data', d => { out += d.toString(); });
180
158
  p.on('close', (code) => {
181
159
  if (code === 0 && out.trim()) {
182
- resolve({ ok: true, output: out, stderr: '' });
160
+ resolve({ ok: true, output: out });
183
161
  } else {
184
- const errTail = out.trimEnd().slice(-500);
185
- resolve({ ok: false, output: out, error: errTail || `exit ${code}` });
162
+ resolve({ ok: false, output: out, error: out.trimEnd().slice(-500) || `exit ${code}` });
186
163
  }
187
164
  });
188
- p.on('error', (e) => {
189
- resolve({ ok: false, output: '', error: e.message });
190
- });
165
+ p.on('error', e => resolve({ ok: false, output: '', error: e.message }));
191
166
  });
192
167
  }
193
168
 
194
- function scraperArgs(platform, length, outDir) {
195
- switch (platform) {
196
- case 'fanqie':
197
- return {
198
- script: 'fanqie-rank-scraper.cjs',
199
- extra: ['--channel', 'all', '--top', '20', '--outdir', outDir],
200
- };
201
- case 'jinjiang':
202
- // 月榜(orderstr=5) + 新手金榜(orderstr=17),--type all 扫全部
203
- return {
204
- script: 'jjwxc-rank-scraper.cjs',
205
- extra: ['--type', '5,17', '--outdir', outDir],
206
- };
207
- case 'zhihu':
208
- return {
209
- script: 'zhihu-rank-scraper.cjs',
210
- extra: ['--length', length === 'short' ? 'short' : 'long', '--login-wait', '60', '--outdir', outDir],
211
- };
212
- case 'qimao':
213
- return {
214
- script: 'qimao-rank-scraper.cjs',
215
- extra: ['--outdir', outDir],
216
- };
217
- return { script: null, extra: [] };
218
- default:
219
- return { script: null, extra: [] };
220
- }
221
- }
222
-
223
169
  // ---------------------------------------------------------------------------
224
- // 扫榜报告生成
170
+ // 构建 scraper 参数
225
171
  // ---------------------------------------------------------------------------
226
172
 
227
- function generateReport(platform, length, scanFiles, outDir) {
228
- const today = todayStr();
229
- const pcn = PLATFORM_CN[platform] || platform;
230
- const lenStr = length === 'long' ? '长篇' : '短篇';
231
-
232
- // 读取原始数据文件
233
- const books = [];
234
- for (const f of scanFiles) {
235
- if (!existsSync(f)) continue;
236
- const content = readFileSync(f, 'utf-8');
237
- // 简单解析 Markdown 中的 ## #N 书名 格式
238
- const matches = [...content.matchAll(/^## #(\d+) (.+)$/gm)];
239
- for (const m of matches) {
240
- books.push({ rank: parseInt(m[1]), title: m[2].trim() });
173
+ function scraperArgs(platform, channel, type, outDir) {
174
+ switch (platform) {
175
+ case 'fanqie': {
176
+ // channel=1/0, type=2/1, __all__ 展开
177
+ const ch = channel === 'all' ? 'all' : channel;
178
+ const ty = type === 'all' ? 'all' : type;
179
+ const extra = ['--outdir', outDir];
180
+ if (ch !== 'all') extra.push('--channel', ch);
181
+ if (ty !== 'all') extra.push('--type', ty);
182
+ return { script: 'fanqie-rank-scraper.cjs', extra };
241
183
  }
242
- }
243
184
 
244
- const report = [
245
- `# ${pcn}${lenStr}扫榜报告:${today}`,
246
- '',
247
- '## 市场概况',
248
- `- 扫榜时间:${today}`,
249
- `- 核心发现:${books.length > 0 ? `共采集 ${books.length} 本上榜作品` : '(数据采集中)'}`,
250
- '',
251
- '## 题材热度排行',
252
- '- (从原始数据分析提取)',
253
- '',
254
- '## 新题材信号',
255
- '- (从原始数据分析提取)',
256
- '',
257
- '## 关键数据洞察',
258
- `- 字数区间:(待分析)`,
259
- `- 书名特征:(待分析)`,
260
- '',
261
- '## 值得关注的方向',
262
- '1. (待从榜单提取后填入)',
263
- '',
264
- '## 一句话',
265
- '(待分析后填入)',
266
- ].join('\n');
267
-
268
- const reportFile = join(outDir, `${pcn}${lenStr}扫榜报告_${today}.md`);
269
- writeFileSync(reportFile, report, 'utf-8');
270
- console.error(` ✅ 扫榜报告 → ${reportFile}`);
271
- return reportFile;
272
- }
273
-
274
- // ---------------------------------------------------------------------------
275
- // topic_decision 追加
276
- // ---------------------------------------------------------------------------
185
+ case 'jinjiang': {
186
+ // type 是 orderstr id
187
+ const extra = ['--outdir', outDir];
188
+ if (type !== 'all') extra.push('--type', type);
189
+ return { script: 'jjwxc-rank-scraper.cjs', extra };
190
+ }
277
191
 
278
- function appendTopicDecision(platform, length, reportFile, outDir) {
279
- const today = todayStr();
280
- const pcn = PLATFORM_CN[platform] || platform;
281
- const lenStr = length === 'long' ? '长篇' : '短篇';
192
+ case 'qimao': {
193
+ const extra = ['--outdir', outDir];
194
+ if (channel !== 'all') extra.push('--channel', channel);
195
+ if (type !== 'all') extra.push('--type', type);
196
+ return { script: 'qimao-rank-scraper.cjs', extra };
197
+ }
282
198
 
283
- const section = [
284
- `## ${pcn}(${lenStr})推荐选题`,
285
- `- 扫榜日期:${today}`,
286
- `- 数据来源:${reportFile}`,
287
- '',
288
- '### 选题 1:(待从报告提取)',
289
- '- 题材组合:(待填入)',
290
- '- 目标读者:(待填入)',
291
- '- 核心卖点:(待填入)',
292
- '- 能爆的原因:(待填入)',
293
- '- 差异化定位:(待填入)',
294
- '- 可行性:高/中/低 — (待评估)',
295
- '- 失败风险:(待评估)',
296
- '- 验证动作:(待填入)',
297
- '- 篇幅/平台:(待填入)',
298
- ].join('\n');
299
-
300
- const decisionFile = join(outDir, `topic_decision_${today}.md`);
301
- const sep = existsSync(decisionFile) ? '\n\n---\n\n' : '';
302
-
303
- if (existsSync(decisionFile)) {
304
- writeFileSync(decisionFile, readFileSync(decisionFile, 'utf-8') + sep + section + '\n', 'utf-8');
305
- } else {
306
- const header = `# 选题决策:${today}\n\n---\n\n`;
307
- writeFileSync(decisionFile, header + section + '\n', 'utf-8');
199
+ default:
200
+ return { script: null, extra: [] };
308
201
  }
309
- console.error(` ✅ topic_decision 追加 → ${decisionFile}`);
310
- return decisionFile;
311
202
  }
312
203
 
313
204
  // ---------------------------------------------------------------------------
@@ -316,109 +207,65 @@ function appendTopicDecision(platform, length, reportFile, outDir) {
316
207
 
317
208
  async function main() {
318
209
  const args = process.argv.slice(2);
319
- let projectRoot = '', platform = '', length = '';
210
+ let projectRoot = '', platform = '', channel = 'all', type = 'all';
320
211
 
321
212
  for (let i = 0; i < args.length; i++) {
322
213
  if (args[i] === '--project-root') projectRoot = args[i + 1] || '';
323
- if (args[i] === '--platform') platform = args[i + 1] || '';
324
- if (args[i] === '--length') length = args[i + 1] || '';
214
+ if (args[i] === '--platform') platform = args[i + 1] || '';
215
+ if (args[i] === '--channel') channel = args[i + 1] || 'all';
216
+ if (args[i] === '--type') type = args[i + 1] || 'all';
325
217
  }
326
218
 
327
- if (!projectRoot || !platform || !length) {
328
- console.error('缺少必要参数: --project-root --platform --length');
219
+ if (!projectRoot || !platform) {
220
+ console.error('缺少必要参数: --project-root --platform');
329
221
  process.exit(1);
330
222
  }
331
223
 
332
- // 检查 .fancy-deployed
333
224
  if (!existsSync(join(projectRoot, '.fancy-deployed'))) {
334
225
  console.error('项目未初始化:.fancy-deployed 不存在');
335
226
  process.exit(1);
336
227
  }
337
228
 
338
- // 验证平台×篇幅
339
- if (!VALID_PLATFORMS.includes(platform)) {
229
+ const VALID = ['qidian', 'fanqie', 'jinjiang', 'qimao'];
230
+ if (!VALID.includes(platform)) {
340
231
  console.error(`不支持的平台: ${platform}`);
341
232
  process.exit(1);
342
233
  }
343
- if (!VALID_LENGTHS.includes(length)) {
344
- console.error(`不支持的篇幅: ${length}`);
345
- process.exit(1);
346
- }
347
- if (!PLATFORM_SUPPORTED_LENGTHS[platform].includes(length)) {
348
- console.error(`${platform} 不支持 ${length}`);
349
- process.exit(1);
350
- }
351
234
 
352
235
  const today = todayStr();
353
236
  const scanDir = join(projectRoot, '扫榜结果');
354
237
  ensureDir(scanDir);
355
238
 
356
239
  let scanFiles = [];
357
- let scrapeFiles = [];
358
240
 
359
241
  if (platform === 'qidian') {
360
- // 起点:HTTP mobile SSR
361
- console.error('→ 采集 起点(mobile SSR)...');
362
- scrapeFiles = await scrapeQidian(scanDir);
363
- if (!scrapeFiles.length) {
364
- console.error(' ❌ 起点采集失败');
365
- process.exit(1);
366
- }
367
- scanFiles = scrapeFiles.map(f => f.file);
242
+ console.error(`→ 采集 起点(mobile SSR)...`);
243
+ const results = await scrapeQidian(scanDir);
244
+ if (!results.length) { console.error('❌ 起点采集失败'); process.exit(1); }
245
+ scanFiles = results.map(r => r.file);
368
246
  } else {
369
- // 其他平台:spawn scraper 进程
370
- const cfg = scraperArgs(platform, length, scanDir);
371
- if (!cfg.script) {
372
- console.error(` ❌ 平台 ${platform} 暂不支持采集`);
373
- process.exit(1);
374
- }
375
- console.error(`→ 采集 ${PLATFORM_CN[platform]}...`);
376
- const result = await spawnScraper(platform, length, scanDir);
377
- if (!result.ok) {
378
- console.error(` ❌ ${PLATFORM_CN[platform]} 采集失败:${result.error}`);
379
- process.exit(1);
380
- }
381
- // scraper 输出落在 stdout(JSON 摘要),找到它
382
- const lines = result.output.split('\n');
383
- const writtenFiles = [];
384
- for (const line of lines) {
385
- const m = line.match(/已保存:\s*(.+)/);
386
- if (m) writtenFiles.push(m[1].trim());
387
- }
388
- if (!writtenFiles.length) {
389
- console.error(' ❌ 未找到任何输出文件');
390
- process.exit(1);
391
- }
392
- scanFiles.push(...writtenFiles);
393
- }
394
-
395
- // Step 6: 收集原始数据
396
- const rawData = {};
397
- for (const f of scanFiles) {
398
- try {
399
- const name = f.replace(/^.*\//, '').replace(/\.md$/, '');
400
- rawData[name] = { path: f, content: '' };
401
- const { readFileSync } = await import('fs');
402
- rawData[name].content = readFileSync(f, 'utf-8');
403
- } catch (_) {}
247
+ const cfg = scraperArgs(platform, channel, type, scanDir);
248
+ if (!cfg.script) { console.error(`❌ 平台 ${platform} 暂不支持`); process.exit(1); }
249
+ console.error(`→ 采集 ${PLATFORM_CN[platform]}(${channel}/${type})...`);
250
+ const r = await spawnScraper(cfg.script, cfg.extra, scanDir);
251
+ if (!r.ok) { console.error(`❌ 采集失败:${r.error}`); process.exit(1); }
252
+
253
+ // stdout 提取 "已保存: xxx.md"
254
+ const writtenFiles = [...r.output.matchAll(/已保存:\s*(.+)/g)].map(m => m[1].trim());
255
+ if (!writtenFiles.length) { console.error('❌ 未找到输出文件'); process.exit(1); }
256
+ scanFiles = writtenFiles;
404
257
  }
405
258
 
406
259
  const receipt = {
407
260
  ok: true,
408
261
  operation: 'scan',
409
262
  platform,
410
- length,
411
263
  date: today,
412
264
  scan_files: scanFiles,
413
265
  topic_decision: join(scanDir, `topic_decision_${today}.md`),
414
- raw_data: rawData,
415
- summary: `${PLATFORM_CN[platform]}(${length === 'long' ? '长篇' : '短篇'})采集完成,共 ${scanFiles.length} 个文件`,
416
266
  };
417
267
 
418
268
  console.log(JSON.stringify(receipt));
419
269
  }
420
270
 
421
- main().catch(e => {
422
- console.error('Fatal:', e);
423
- process.exit(1);
424
- });
271
+ main().catch(e => { console.error('Fatal:', e); process.exit(1); });
@@ -7,11 +7,10 @@
7
7
  // ---------------------------------------------------------------------------
8
8
 
9
9
  export const PLATFORM_TABLE = [
10
- ['1', 'qidian', '长篇', '起点'],
11
- ['2', 'fanqie', '长篇', '番茄'],
12
- ['3', 'jinjiang', '长篇', '晋江'],
13
- ['4', 'zhihu', '需选择', '知乎'],
14
- ['5', 'qimao', '需选择', '七猫'],
10
+ ['1', 'qidian', '起点'],
11
+ ['2', 'fanqie', '番茄'],
12
+ ['3', 'jinjiang', '晋江'],
13
+ ['4', 'qimao', '七猫'],
15
14
  ];
16
15
 
17
16
  export function platformLabel(platform) {
@@ -19,6 +18,50 @@ export function platformLabel(platform) {
19
18
  return row ? row[3] : platform;
20
19
  }
21
20
 
21
+ // ---------------------------------------------------------------------------
22
+ // 各平台榜单选项({ id, label }[]),供弹窗多选用
23
+ // ---------------------------------------------------------------------------
24
+
25
+ export function rankOptions(platform) {
26
+ switch (platform) {
27
+ case 'qidian':
28
+ // 起点男频大热榜(唯一榜单,无细分)
29
+ return [{ id: 'main', label: '起点中文网主站榜单' }];
30
+
31
+ case 'fanqie':
32
+ return [
33
+ { id: '1_2', label: '男频阅读榜' },
34
+ { id: '1_1', label: '男频新书榜' },
35
+ { id: '0_2', label: '女频阅读榜' },
36
+ { id: '0_1', label: '女频新书榜' },
37
+ { id: '__all__', label: '全选' },
38
+ ];
39
+
40
+ case 'jinjiang':
41
+ return [
42
+ { id: '5', label: '月榜' },
43
+ { id: '7', label: '总分榜' },
44
+ { id: '4', label: '季度榜' },
45
+ { id: '12', label: '收入金榜' },
46
+ { id: '16', label: '完结金榜' },
47
+ { id: '17', label: '新手金榜' },
48
+ { id: '__all__', label: '全选' },
49
+ ];
50
+
51
+ case 'qimao':
52
+ return [
53
+ { id: 'male_hot', label: '男生大热榜' },
54
+ { id: 'male_new', label: '男生新书榜' },
55
+ { id: 'female_hot', label: '女生大热榜' },
56
+ { id: 'female_new', label: '女生新书榜' },
57
+ { id: '__all__', label: '全选' },
58
+ ];
59
+
60
+ default:
61
+ return [];
62
+ }
63
+ }
64
+
22
65
  // ---------------------------------------------------------------------------
23
66
  // 模板
24
67
  // ---------------------------------------------------------------------------
@@ -27,28 +70,24 @@ export const scan = {
27
70
  platformList() {
28
71
  return (
29
72
  '📋 支持的平台:\n' +
30
- PLATFORM_TABLE.map(r => ` ${r[0]}. ${r[3]}(${r[2]})`).join('\n') +
31
- '\n- 知乎和七猫同时支持长篇和短篇'
73
+ PLATFORM_TABLE.map(r => ` ${r[0]}. ${r[3]}`).join('\n')
32
74
  );
33
75
  },
34
76
 
35
77
  askPlatform() {
36
- return { question: '📊 请选择要扫的平台(输入编号 1-5):' };
78
+ return { question: '📊 请选择要扫的平台(输入编号 1-4):' };
37
79
  },
38
80
 
39
- askLength(platform) {
81
+ askRankList(platform) {
82
+ const opts = rankOptions(platform);
40
83
  return {
41
- question: `${platformLabel(platform)} 同时支持长篇和短篇,请选择:`,
84
+ question: `${platformLabel(platform)} 有多个榜单,支持多选。请选择要采集的榜单:`,
42
85
  hideCustomInput: true,
43
- options: [
44
- { label: '长篇' },
45
- { label: '短篇' },
46
- ],
86
+ options: opts.map(o => ({ label: o.label })),
47
87
  };
48
88
  },
49
89
 
50
- notify({ type, platform, length, files, topicFile, projectRoot, err }) {
51
- const lenStr = length === 'long' ? '长篇' : '短篇';
90
+ notify({ type, platform, files, topicFile, projectRoot, err }) {
52
91
  const pLabel = platformLabel(platform);
53
92
 
54
93
  switch (type) {
@@ -56,19 +95,16 @@ export const scan = {
56
95
  return '⚠️ 当前目录尚未初始化。请先运行 /fancy-bootstrap 进行初始化。';
57
96
 
58
97
  case 'invalid_choice':
59
- return '❌ 无效选择,请输入 1-5 的编号。';
98
+ return '❌ 无效选择,请输入 1-4 的编号。';
60
99
 
61
100
  case 'cancelled':
62
- return '🚫 已取消';
63
-
64
- case 'cancelled_with_hint':
65
- return '⚠️ 已取消。请再次调用 /fancy-scan 继续。';
101
+ return '🚫 已取消。';
66
102
 
67
103
  case 'failed':
68
- return '❌ 采集失败:' + err;
104
+ return '❌ 采集失败:' + (err || '');
69
105
 
70
106
  case 'parse_error':
71
- return '❌ 采集结果解析失败';
107
+ return '❌ 采集结果解析失败。';
72
108
 
73
109
  case 'handover':
74
110
  return (
@@ -82,9 +118,8 @@ export const scan = {
82
118
  }
83
119
  },
84
120
 
85
- llmAnalysisPrompt({ platform, length, date, files, topicFile, projectRoot }) {
121
+ llmAnalysisPrompt({ platform, date, files, topicFile, projectRoot }) {
86
122
  const pLabel = platformLabel(platform);
87
- const lenStr = length === 'long' ? '长篇' : '短篇';
88
123
  const fileList = (files || [])
89
124
  .map(f => '- ' + f.replace(projectRoot + '/', ''))
90
125
  .join('\n');
@@ -92,7 +127,7 @@ export const scan = {
92
127
  return [
93
128
  `## 扫榜数据分析任务`,
94
129
  ``,
95
- `平台:${pLabel}(${lenStr})`,
130
+ `平台:${pLabel}`,
96
131
  `时间:${date || ''}`,
97
132
  ``,
98
133
  `已生成的文件:`,
@@ -1,186 +0,0 @@
1
- #!/usr/bin/env node
2
- /**
3
- * 知乎短篇/长篇采集脚本
4
- *
5
- * 使用 playwright-core 自己管理浏览器。
6
- * 入口:https://www.zhihu.com/fiore/h5/vip-web
7
- * 短篇(4 个榜单):推荐榜 / 热搜榜 / 热度榜 / 口碑榜
8
- * 长篇(1 个榜单):长篇榜
9
- *
10
- * 用法:
11
- * node zhihu-rank-scraper.js --length short # 短篇 4 个榜单
12
- * node zhihu-rank-scraper.js --length long # 长篇 1 个榜单
13
- * node zhihu-rank-scraper.js --length short --boards rec,hot # 指定短篇子榜
14
- * node zhihu-rank-scraper.js --login-wait 60 # 等待手动登录
15
- *
16
- * 注意:知乎需要登录态才能看完整榜单。
17
- */
18
-
19
- const fs = require("fs");
20
- const path = require("path");
21
- const { chromium } = require("playwright-core");
22
- const { getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
23
-
24
- const OUTDIR = getArg(process.argv, "--outdir") || "扫榜结果";
25
- const ENTRY_URL = "https://www.zhihu.com/fiore/h5/vip-web";
26
- const LENGTH = getArg(process.argv, "--length") || "short";
27
- const LOGIN_WAIT = parseInt(getArg(process.argv, "--login-wait") || "0", 10);
28
-
29
- // 短篇 4 个榜单
30
- const SHORT_BOARDS = [
31
- { id: "rec", label: "推荐榜", tabText: "推荐榜" },
32
- { id: "hot", label: "热搜榜", tabText: "热搜榜" },
33
- { id: "trending", label: "热度榜", tabText: "热度榜" },
34
- { id: "reputation",label: "口碑榜", tabText: "口碑榜" },
35
- ];
36
-
37
- // 长篇 1 个榜单
38
- const LONG_BOARDS = [
39
- { id: "long", label: "长篇榜", tabText: "长篇榜" },
40
- ];
41
-
42
- const BOARDS = LENGTH === "long" ? LONG_BOARDS : SHORT_BOARDS;
43
-
44
- const customBoards = getArg(process.argv, "--boards");
45
- const targetBoards = customBoards
46
- ? BOARDS.filter(b => customBoards.split(",").map(s => s.trim()).includes(b.id))
47
- : BOARDS;
48
-
49
- // ---------------------------------------------------------------------------
50
- // 工具函数
51
- // ---------------------------------------------------------------------------
52
-
53
- function sleep(ms) {
54
- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
55
- }
56
-
57
- // ---------------------------------------------------------------------------
58
- // 采集
59
- // ---------------------------------------------------------------------------
60
-
61
- async function scrapeBoard(board) {
62
- console.log(` [${board.label}] 正在抓取...`);
63
-
64
- const browser = await chromium.launch({ headless: false, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
65
- const context = await browser.newContext();
66
- const page = await context.newPage();
67
-
68
- // CDP 强制窗口最大化
69
- try {
70
- const cdp = await context.newCDPSession(page);
71
- const { windowId } = await cdp.send('Browser.getWindowForTarget', { target: page.target() });
72
- await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
73
- } catch (_) {}
74
-
75
- try {
76
- await page.goto(ENTRY_URL, { waitUntil: "networkidle" });
77
-
78
- if (LOGIN_WAIT > 0) {
79
- console.log(` ⏳ 等待 ${LOGIN_WAIT}s 供用户登录...`);
80
- await page.waitForTimeout(LOGIN_WAIT * 1000);
81
- }
82
-
83
- const host = page.url();
84
- if (host.indexOf("zhihu") === -1) {
85
- console.error(` ✗ 当前页面非知乎(url=${host}),可能被重定向,已跳过。`);
86
- return null;
87
- }
88
-
89
- // 点击 tab
90
- const clicked = await page.evaluate((target) => {
91
- const candidates = Array.from(document.querySelectorAll("a, button, div, span, li"))
92
- .filter(el => {
93
- const t = (el.innerText || "").trim();
94
- return t === target || t.startsWith(target);
95
- });
96
- if (!candidates.length) return false;
97
- candidates[0].click();
98
- return true;
99
- }, board.tabText);
100
-
101
- if (!clicked) {
102
- console.error(` [${board.label}] 点击 tab 失败`);
103
- return null;
104
- }
105
-
106
- await page.waitForTimeout(2500);
107
-
108
- // 提取故事列表
109
- const stories = await page.evaluate(() => {
110
- const items = [];
111
- const seen = new Set();
112
- const anchors = document.querySelectorAll('a[href*="/p/"], a[href*="/question/"], a[href*="/xen/"]');
113
- anchors.forEach((a, i) => {
114
- const href = a.href || "";
115
- if (!href || seen.has(href)) return;
116
- const text = (a.innerText || "").trim();
117
- if (!text || text.length < 4 || text.length > 200) return;
118
- if (/^(首页|发现|等你来答|登录|注册|更多|查看全部)$/.test(text)) return;
119
- seen.add(href);
120
- items.push({ rank: i + 1, title: text.split("\n")[0].trim(), url: href });
121
- });
122
- return items.slice(0, 100);
123
- });
124
-
125
- if (!stories || stories.length === 0) {
126
- console.warn(` [${board.label}] 未提取到故事`);
127
- return [];
128
- }
129
- return stories;
130
- } finally {
131
- await browser.close();
132
- }
133
- }
134
-
135
- function buildBoardMarkdown(board, stories, date) {
136
- const lines = [
137
- `# 知乎 · ${board.label}`,
138
- "",
139
- `- 来源:${ENTRY_URL}`,
140
- `- 抓取时间:${date}`,
141
- "",
142
- "---",
143
- "",
144
- ];
145
- if (!stories.length) {
146
- lines.push("(未提取到数据,可能页面结构变化或需要登录)", "", "---", "");
147
- } else {
148
- stories.forEach(s => {
149
- lines.push(`书名:${s.title || ""}`);
150
- lines.push(`题材:${LENGTH === "long" ? "长篇" : "短篇"}`);
151
- lines.push(`作者:${s.author || ""}`);
152
- if (s.url) lines.push(`作品页:${s.url}`);
153
- lines.push("");
154
- });
155
- }
156
- return lines.join("\n");
157
- }
158
-
159
- // ---------------------------------------------------------------------------
160
- // 入口
161
- // ---------------------------------------------------------------------------
162
-
163
- async function main() {
164
- const date = localDateStamp();
165
- fs.mkdirSync(OUTDIR, { recursive: true });
166
-
167
- let written = 0;
168
- for (const board of targetBoards) {
169
- const stories = await scrapeBoard(board);
170
- if (stories === null) continue;
171
-
172
- const md = buildBoardMarkdown(board, stories, date);
173
- const filename = `知乎${LENGTH === "long" ? "长篇" : "短篇"}${board.label}_${date}.md`;
174
- const filepath = path.join(OUTDIR, filename);
175
- fs.writeFileSync(filepath, md, "utf-8");
176
- console.log(` ✓ ${filename} (${stories.length} 条)`);
177
- written++;
178
- }
179
- return written;
180
- }
181
-
182
- if (require.main === module) {
183
- runCli(main, "知乎采集");
184
- }
185
-
186
- module.exports = { scrapeBoard, buildBoardMarkdown };