@cloud411716/fancy-webnovel 0.2.27 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -5,101 +5,133 @@ import { fileURLToPath } from 'url';
|
|
|
5
5
|
import { dirname, join } from 'path';
|
|
6
6
|
import { existsSync } from 'fs';
|
|
7
7
|
import { notify, llmFollowup, spawnScript, startActivity } from '../../infra.js';
|
|
8
|
-
import { scan, PLATFORM_TABLE,
|
|
8
|
+
import { scan, PLATFORM_TABLE, rankOptions } from './templates.js';
|
|
9
9
|
|
|
10
10
|
export const inject = ['commands', 'userQuestions'];
|
|
11
11
|
|
|
12
12
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
13
|
const SCAN_SCRIPTS_DIR = join(__dirname, 'scripts');
|
|
14
|
+
const SCRAPER_TIMEOUT = 30 * 60 * 1000; // 30 分钟
|
|
14
15
|
|
|
15
16
|
export async function apply(ctx) {
|
|
16
17
|
ctx.commands.register({
|
|
17
18
|
name: 'fancy-scan',
|
|
18
|
-
description: '📊 扫榜分析
|
|
19
|
+
description: '📊 扫榜分析 — 自动分析起点、番茄、晋江、七猫榜单并生成报告',
|
|
19
20
|
handler: async (invocation) => {
|
|
20
21
|
const session = invocation.agent.session;
|
|
21
|
-
const rawInput = invocation.rawInput ? invocation.rawInput.trim() : '';
|
|
22
22
|
|
|
23
23
|
// 获取项目根
|
|
24
|
-
|
|
25
|
-
if (!projectRoot) projectRoot = process.cwd();
|
|
26
|
-
|
|
24
|
+
const projectRoot = process.cwd();
|
|
27
25
|
if (!existsSync(join(projectRoot, '.fancy-deployed'))) {
|
|
28
26
|
notify(session, scan.notify({ type: 'not_initialized' }));
|
|
29
27
|
return { kind: 'success', text: '' };
|
|
30
28
|
}
|
|
31
29
|
|
|
32
|
-
//
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
49
|
-
}
|
|
30
|
+
// ---------- 第一轮:选择平台 ----------
|
|
31
|
+
notify(session, scan.platformList());
|
|
32
|
+
|
|
33
|
+
const platformResult = await ctx.userQuestions.ask({
|
|
34
|
+
questions: [scan.askPlatform()],
|
|
35
|
+
});
|
|
36
|
+
const answer = platformResult.answers[0]?.custom?.trim();
|
|
37
|
+
if (!answer) {
|
|
38
|
+
notify(session, scan.notify({ type: 'cancelled' }));
|
|
39
|
+
return { kind: 'success', text: '' };
|
|
40
|
+
}
|
|
41
|
+
const numMatch = answer.match(/^(\d+)$/);
|
|
42
|
+
const row = numMatch ? PLATFORM_TABLE.find(r => r[0] === numMatch[1]) : null;
|
|
43
|
+
if (!row) {
|
|
44
|
+
notify(session, scan.notify({ type: 'invalid_choice' }));
|
|
45
|
+
return { kind: 'success', text: '' };
|
|
50
46
|
}
|
|
47
|
+
const platform = row[1];
|
|
51
48
|
|
|
52
|
-
//
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
49
|
+
// ---------- 第二轮:选择榜单(多选) ----------
|
|
50
|
+
const rankResult = await ctx.userQuestions.ask({
|
|
51
|
+
questions: [scan.askRankList(platform)],
|
|
52
|
+
});
|
|
53
|
+
const selected = rankResult.answers[0]?.selected ?? [];
|
|
54
|
+
const custom = rankResult.answers[0]?.custom?.trim();
|
|
55
|
+
|
|
56
|
+
// 用户可能直接输入自定义值
|
|
57
|
+
const rawChoices = selected.length > 0 ? selected : (custom ? [custom] : []);
|
|
58
|
+
if (rawChoices.length === 0) {
|
|
59
|
+
notify(session, scan.notify({ type: 'cancelled' }));
|
|
60
|
+
return { kind: 'success', text: '' };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// 构建每个榜单的 channel+type 参数
|
|
64
|
+
const platformRanks = rankOptions(platform);
|
|
65
|
+
const isAll = rawChoices.includes('全选') || rawChoices.includes('__all__');
|
|
66
|
+
const rankIds = isAll
|
|
67
|
+
? platformRanks.filter(r => r.id !== '__all__').map(r => r.id)
|
|
68
|
+
: rawChoices;
|
|
69
|
+
|
|
70
|
+
// 把 rankId 转换成 { channel, type } 并去重
|
|
71
|
+
const rankEntries = rankIds.map(id => parseRankId(platform, id));
|
|
72
|
+
const unique = [];
|
|
73
|
+
const seen = new Set();
|
|
74
|
+
for (const e of rankEntries) {
|
|
75
|
+
const key = `${e.channel}__${e.type}`;
|
|
76
|
+
if (!seen.has(key)) { seen.add(key); unique.push(e); }
|
|
74
77
|
}
|
|
75
78
|
|
|
76
|
-
//
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
79
|
+
// ---------- 串行采集 ----------
|
|
80
|
+
let allFiles = [];
|
|
81
|
+
let failed = 0;
|
|
82
|
+
|
|
83
|
+
for (const rank of unique) {
|
|
84
|
+
const stop = startActivity(session);
|
|
85
|
+
const r = await spawnScript(
|
|
86
|
+
'node',
|
|
87
|
+
[
|
|
88
|
+
join(SCAN_SCRIPTS_DIR, 'run-scan.js'),
|
|
89
|
+
'--project-root', projectRoot,
|
|
90
|
+
'--platform', platform,
|
|
91
|
+
'--channel', rank.channel,
|
|
92
|
+
'--type', rank.type,
|
|
93
|
+
],
|
|
94
|
+
{ timeout: SCRAPER_TIMEOUT, stop }
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
if (!r.ok) {
|
|
98
|
+
const rawErr = (typeof r.error === 'object' && r.error !== null)
|
|
99
|
+
? (r.error.message || JSON.stringify(r.error))
|
|
100
|
+
: String(r.error || '');
|
|
101
|
+
if (rawErr.includes("Executable doesn't exist") || rawErr.includes("doesn't exist")) {
|
|
102
|
+
notify(session,
|
|
103
|
+
'⚠️ 未找到 Chromium 浏览器。\n\n安装命令:\n' +
|
|
104
|
+
' npm install -g playwright@1.48.0 --registry=https://registry.npmmirror.com\n' +
|
|
105
|
+
' PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ playwright install chromium --with-deps\n\n' +
|
|
106
|
+
'安装完成后重新运行 /fancy-scan 即可。'
|
|
107
|
+
);
|
|
85
108
|
return { kind: 'success', text: '' };
|
|
86
109
|
}
|
|
87
|
-
|
|
110
|
+
notify(session, scan.notify({ type: 'failed', err: rawErr }));
|
|
111
|
+
failed++;
|
|
112
|
+
continue;
|
|
88
113
|
}
|
|
89
|
-
} else if (!length) {
|
|
90
|
-
length = 'long';
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// 执行采集,包含 Chromium 缺失处理
|
|
94
|
-
const scanResult = await runScanWithChromiumFix(ctx, session, invocation, {
|
|
95
|
-
platform, length, projectRoot,
|
|
96
|
-
});
|
|
97
114
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
115
|
+
// 解析采集结果
|
|
116
|
+
let receipt;
|
|
117
|
+
try { receipt = JSON.parse(r.output); } catch (_) {
|
|
118
|
+
notify(session, scan.notify({ type: 'parse_error' }));
|
|
119
|
+
failed++;
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
const files = receipt.scan_files || [];
|
|
123
|
+
allFiles = allFiles.concat(files);
|
|
124
|
+
|
|
125
|
+
// 逐个文件追加到 topic_decision
|
|
126
|
+
const topicFile = receipt.topic_decision || '';
|
|
127
|
+
if (topicFile) {
|
|
128
|
+
notify(session, scan.notify({ type: 'handover', platform, files, topicFile, projectRoot }));
|
|
129
|
+
const prompt = scan.llmAnalysisPrompt({ platform, date: receipt.date || '', files, topicFile, projectRoot });
|
|
130
|
+
llmFollowup(invocation, prompt);
|
|
131
|
+
}
|
|
101
132
|
}
|
|
102
|
-
|
|
133
|
+
|
|
134
|
+
if (failed > 0 && allFiles.length === 0) {
|
|
103
135
|
return { kind: 'success', text: '' };
|
|
104
136
|
}
|
|
105
137
|
|
|
@@ -108,59 +140,31 @@ export async function apply(ctx) {
|
|
|
108
140
|
});
|
|
109
141
|
}
|
|
110
142
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
[join(SCAN_SCRIPTS_DIR, 'run-scan.js'),
|
|
122
|
-
'--project-root', projectRoot, '--platform', platform, '--length', length],
|
|
123
|
-
{ timeout: 900000, stop }
|
|
124
|
-
);
|
|
125
|
-
return r;
|
|
126
|
-
};
|
|
127
|
-
|
|
128
|
-
const handleSuccess = (stdout) => {
|
|
129
|
-
let receipt;
|
|
130
|
-
try { receipt = JSON.parse(stdout); } catch (_) {
|
|
131
|
-
notify(session, scan.notify({ type: 'parse_error' }));
|
|
132
|
-
return;
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
// 将 rankId 转换成 { channel, type } 参数
|
|
145
|
+
// ---------------------------------------------------------------------------
|
|
146
|
+
|
|
147
|
+
function parseRankId(platform, id) {
|
|
148
|
+
switch (platform) {
|
|
149
|
+
case 'qimao': {
|
|
150
|
+
// id 格式: male_hot / female_new
|
|
151
|
+
const [ch, rt] = id.split('_');
|
|
152
|
+
return { channel: ch, type: rt };
|
|
133
153
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
llmFollowup(invocation, prompt);
|
|
139
|
-
};
|
|
140
|
-
|
|
141
|
-
const scriptResult = await runOnce();
|
|
142
|
-
|
|
143
|
-
if (!scriptResult.ok) {
|
|
144
|
-
const rawErr = (typeof scriptResult.error === 'object' && scriptResult.error !== null)
|
|
145
|
-
? (scriptResult.error.message || JSON.stringify(scriptResult.error))
|
|
146
|
-
: String(scriptResult.error || '未知错误');
|
|
147
|
-
|
|
148
|
-
// Chromium 未找到 → 提示用户手动安装
|
|
149
|
-
if (rawErr.includes("Executable doesn't exist") || rawErr.includes('Executable doesn')) {
|
|
150
|
-
notify(session,
|
|
151
|
-
'⚠️ 未找到 Chromium 浏览器,请手动安装后再次运行 /fancy-scan。\n\n' +
|
|
152
|
-
'安装命令:\n' +
|
|
153
|
-
' npm install -g playwright@1.48.0 --registry=https://registry.npmmirror.com\n' +
|
|
154
|
-
' PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ playwright install chromium --with-deps\n\n' +
|
|
155
|
-
'安装完成后重新运行 /fancy-scan 即可。'
|
|
156
|
-
);
|
|
157
|
-
return undefined;
|
|
154
|
+
case 'fanqie': {
|
|
155
|
+
// id 格式: 0_1 / 1_2 / __all__
|
|
156
|
+
const [ch, ty] = id.split('_');
|
|
157
|
+
return { channel: ch, type: ty };
|
|
158
158
|
}
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
159
|
+
case 'jinjiang': {
|
|
160
|
+
// id 格式: 5 / 7 / 12 等(orderstr)
|
|
161
|
+
return { channel: 'all', type: id };
|
|
162
|
+
}
|
|
163
|
+
case 'qidian': {
|
|
164
|
+
// 起点只有主站一个榜单
|
|
165
|
+
return { channel: 'main', type: 'main' };
|
|
166
|
+
}
|
|
167
|
+
default:
|
|
168
|
+
return { channel: 'all', type: 'all' };
|
|
162
169
|
}
|
|
163
|
-
|
|
164
|
-
handleSuccess(scriptResult.output);
|
|
165
|
-
return 'retry';
|
|
166
170
|
}
|
|
@@ -4,12 +4,14 @@
|
|
|
4
4
|
*
|
|
5
5
|
* 职责:
|
|
6
6
|
* 1. 检查 .fancy-deployed
|
|
7
|
-
* 2.
|
|
8
|
-
* 3.
|
|
9
|
-
* 4. 生成/追加 topic_decision
|
|
7
|
+
* 2. 调用平台采集脚本(一次一个榜单)
|
|
8
|
+
* 3. 输出 JSON 摘要(供 index.js 解析)
|
|
10
9
|
*
|
|
11
10
|
* 调用方式:
|
|
12
|
-
* node run-scan.js --project-root <abs> --platform qidian --
|
|
11
|
+
* node run-scan.js --project-root <abs> --platform qidian --channel main --type main
|
|
12
|
+
* node run-scan.js --project-root <abs> --platform fanqie --channel 1 --type 2
|
|
13
|
+
* node run-scan.js --project-root <abs> --platform jinjiang --channel all --type 5
|
|
14
|
+
* node run-scan.js --project-root <abs> --platform qimao --channel male --type hot
|
|
13
15
|
*/
|
|
14
16
|
|
|
15
17
|
import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
|
|
@@ -17,25 +19,13 @@ import { join, dirname } from 'path';
|
|
|
17
19
|
import { fileURLToPath } from 'url';
|
|
18
20
|
|
|
19
21
|
// ---------------------------------------------------------------------------
|
|
20
|
-
//
|
|
22
|
+
// 平台中文名
|
|
21
23
|
// ---------------------------------------------------------------------------
|
|
22
24
|
|
|
23
|
-
const VALID_LENGTHS = ['long', 'short'];
|
|
24
|
-
const VALID_PLATFORMS = ['qidian', 'fanqie', 'jinjiang', 'zhihu', 'qimao'];
|
|
25
|
-
|
|
26
|
-
const PLATFORM_SUPPORTED_LENGTHS = {
|
|
27
|
-
qidian: ['long'],
|
|
28
|
-
fanqie: ['long'],
|
|
29
|
-
jinjiang: ['long'],
|
|
30
|
-
zhihu: ['long', 'short'],
|
|
31
|
-
qimao: ['long', 'short'],
|
|
32
|
-
};
|
|
33
|
-
|
|
34
25
|
const PLATFORM_CN = {
|
|
35
26
|
qidian: '起点',
|
|
36
27
|
fanqie: '番茄',
|
|
37
28
|
jinjiang: '晋江',
|
|
38
|
-
zhihu: '知乎',
|
|
39
29
|
qimao: '七猫',
|
|
40
30
|
};
|
|
41
31
|
|
|
@@ -57,17 +47,17 @@ function ensureDir(dir) {
|
|
|
57
47
|
}
|
|
58
48
|
|
|
59
49
|
// ---------------------------------------------------------------------------
|
|
60
|
-
// HTTP
|
|
50
|
+
// 起点:HTTP mobile SSR(自包含,无需 playwright)
|
|
61
51
|
// ---------------------------------------------------------------------------
|
|
62
52
|
|
|
63
53
|
async function scrapeQidian(outDir) {
|
|
64
54
|
const MOBILE_BASE = 'https://m.qidian.com';
|
|
65
55
|
const RANK_TYPES = [
|
|
66
|
-
{ id: 'hotsales',
|
|
67
|
-
{ id: 'yuepiao',
|
|
68
|
-
{ id: 'signnewbook',label: '签约作者新书榜',
|
|
69
|
-
{ id: 'pubnewbook',
|
|
70
|
-
{ id: 'newauthor',
|
|
56
|
+
{ id: 'hotsales', label: '畅销榜', path: '/rank/hotsales/' },
|
|
57
|
+
{ id: 'yuepiao', label: '月票榜', path: '/rank/yuepiao/' },
|
|
58
|
+
{ id: 'signnewbook', label: '签约作者新书榜', path: '/rank/sign/' },
|
|
59
|
+
{ id: 'pubnewbook', label: '公众作者新书榜', path: '/rank/newbook/' },
|
|
60
|
+
{ id: 'newauthor', label: '新人作者新书榜', path: '/rank/newauthor/' },
|
|
71
61
|
];
|
|
72
62
|
|
|
73
63
|
const MOBILE_HEADERS = {
|
|
@@ -89,7 +79,6 @@ async function scrapeQidian(outDir) {
|
|
|
89
79
|
continue;
|
|
90
80
|
}
|
|
91
81
|
|
|
92
|
-
// 提取 pageContext JSON
|
|
93
82
|
const m = html.match(/<script[^>]+id=["']vite-plugin-ssr_pageContext["'][^>]*>([\s\S]*?)<\/script>/i);
|
|
94
83
|
if (!m) {
|
|
95
84
|
console.error(` ⚠ ${rt.label} 未找到 pageContext`);
|
|
@@ -99,7 +88,7 @@ async function scrapeQidian(outDir) {
|
|
|
99
88
|
let pageContext;
|
|
100
89
|
try {
|
|
101
90
|
pageContext = JSON.parse(m[1]);
|
|
102
|
-
} catch (
|
|
91
|
+
} catch (_) {
|
|
103
92
|
console.error(` ⚠ ${rt.label} JSON 解析失败`);
|
|
104
93
|
continue;
|
|
105
94
|
}
|
|
@@ -111,203 +100,105 @@ async function scrapeQidian(outDir) {
|
|
|
111
100
|
}
|
|
112
101
|
|
|
113
102
|
const books = records.map((r, i) => ({
|
|
114
|
-
rank:
|
|
115
|
-
title:
|
|
116
|
-
url:
|
|
117
|
-
author:
|
|
118
|
-
genre:
|
|
119
|
-
status:
|
|
120
|
-
words:
|
|
121
|
-
rankValue: r.rankCnt || '',
|
|
122
|
-
totalRecommend: r.totalRecommend || '',
|
|
123
|
-
signStatus: r.signStatus || '',
|
|
124
|
-
vipStatus: r.vipStatus || '',
|
|
103
|
+
rank: r.rankNum || i + 1,
|
|
104
|
+
title: r.bName || r.bookName || '',
|
|
105
|
+
url: r.bid ? `${MOBILE_BASE}/book/${r.bid}/` : '',
|
|
106
|
+
author: r.bAuth || r.author || '',
|
|
107
|
+
genre: [r.cat, r.subCat].filter(Boolean).join('·'),
|
|
108
|
+
status: r.status || '',
|
|
109
|
+
words: r.cnt || r.wordCount || '',
|
|
125
110
|
}));
|
|
126
111
|
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
writeFileSync(outFile, md, 'utf-8');
|
|
131
|
-
console.error(` ✅ ${rt.label}: ${books.length} 本 → ${outFile}`);
|
|
112
|
+
const outFile = join(outDir, `${PLATFORM_CN.qidian}${rt.label}_${todayStr()}.md`);
|
|
113
|
+
writeFileSync(outFile, renderMarkdown(rt.label, url, books), 'utf-8');
|
|
114
|
+
console.error(` ✅ ${rt.label}: ${books.length} 本`);
|
|
132
115
|
results.push({ label: rt.label, count: books.length, file: outFile });
|
|
133
116
|
}
|
|
134
117
|
|
|
135
118
|
return results;
|
|
136
119
|
}
|
|
137
120
|
|
|
138
|
-
function
|
|
121
|
+
function renderMarkdown(rankLabel, url, books) {
|
|
139
122
|
const now = nowIso();
|
|
140
123
|
const lines = [
|
|
141
124
|
`# 起点 · ${rankLabel}`,
|
|
142
|
-
|
|
125
|
+
``,
|
|
143
126
|
`- 来源:${url}`,
|
|
144
127
|
`- 抓取方式:mobile-ssr`,
|
|
145
128
|
`- 抓取时间:${now}`,
|
|
146
|
-
|
|
129
|
+
``,
|
|
147
130
|
'---',
|
|
148
131
|
'',
|
|
149
132
|
];
|
|
150
|
-
|
|
151
133
|
for (const b of books) {
|
|
152
|
-
lines.push(`书名:${b.title
|
|
153
|
-
lines.push(`题材:${b.genre
|
|
154
|
-
lines.push(`作者:${b.author
|
|
134
|
+
lines.push(`书名:${b.title}`);
|
|
135
|
+
lines.push(`题材:${b.genre}`);
|
|
136
|
+
lines.push(`作者:${b.author}`);
|
|
155
137
|
if (b.url) lines.push(`作品页:${b.url}`);
|
|
156
138
|
lines.push('');
|
|
157
139
|
}
|
|
158
|
-
|
|
159
140
|
return lines.join('\n');
|
|
160
141
|
}
|
|
161
142
|
|
|
162
143
|
// ---------------------------------------------------------------------------
|
|
163
|
-
// Browser scraper spawner
|
|
144
|
+
// Browser scraper spawner(30 分钟超时)
|
|
164
145
|
// ---------------------------------------------------------------------------
|
|
165
146
|
|
|
166
|
-
function spawnScraper(
|
|
147
|
+
function spawnScraper(script, extraArgs, outDir) {
|
|
167
148
|
return new Promise(async (resolve) => {
|
|
168
149
|
const { spawn } = await import('child_process');
|
|
169
|
-
const
|
|
170
|
-
|
|
171
|
-
const
|
|
172
|
-
|
|
173
|
-
process.stderr.write(`\n→ 启动采集脚本...`);
|
|
174
|
-
const p = spawn('node', [scraperScript, ...fullArgs], {
|
|
175
|
-
timeout: 900000,
|
|
150
|
+
const scriptPath = join(dirname(fileURLToPath(import.meta.url)), 'scrapers', script);
|
|
151
|
+
process.stderr.write(`\n→ 启动 ${script}...\n`);
|
|
152
|
+
const p = spawn('node', [scriptPath, ...extraArgs], {
|
|
153
|
+
timeout: 30 * 60 * 1000,
|
|
176
154
|
stdio: ['ignore', 'pipe', 'inherit'],
|
|
177
155
|
});
|
|
178
156
|
let out = '';
|
|
179
157
|
p.stdout.on('data', d => { out += d.toString(); });
|
|
180
158
|
p.on('close', (code) => {
|
|
181
159
|
if (code === 0 && out.trim()) {
|
|
182
|
-
resolve({ ok: true, output: out
|
|
160
|
+
resolve({ ok: true, output: out });
|
|
183
161
|
} else {
|
|
184
|
-
|
|
185
|
-
resolve({ ok: false, output: out, error: errTail || `exit ${code}` });
|
|
162
|
+
resolve({ ok: false, output: out, error: out.trimEnd().slice(-500) || `exit ${code}` });
|
|
186
163
|
}
|
|
187
164
|
});
|
|
188
|
-
p.on('error',
|
|
189
|
-
resolve({ ok: false, output: '', error: e.message });
|
|
190
|
-
});
|
|
165
|
+
p.on('error', e => resolve({ ok: false, output: '', error: e.message }));
|
|
191
166
|
});
|
|
192
167
|
}
|
|
193
168
|
|
|
194
|
-
function scraperArgs(platform, length, outDir) {
|
|
195
|
-
switch (platform) {
|
|
196
|
-
case 'fanqie':
|
|
197
|
-
return {
|
|
198
|
-
script: 'fanqie-rank-scraper.cjs',
|
|
199
|
-
extra: ['--channel', 'all', '--top', '20', '--outdir', outDir],
|
|
200
|
-
};
|
|
201
|
-
case 'jinjiang':
|
|
202
|
-
// 月榜(orderstr=5) + 新手金榜(orderstr=17),--type all 扫全部
|
|
203
|
-
return {
|
|
204
|
-
script: 'jjwxc-rank-scraper.cjs',
|
|
205
|
-
extra: ['--type', '5,17', '--outdir', outDir],
|
|
206
|
-
};
|
|
207
|
-
case 'zhihu':
|
|
208
|
-
return {
|
|
209
|
-
script: 'zhihu-rank-scraper.cjs',
|
|
210
|
-
extra: ['--length', length === 'short' ? 'short' : 'long', '--login-wait', '60', '--outdir', outDir],
|
|
211
|
-
};
|
|
212
|
-
case 'qimao':
|
|
213
|
-
return {
|
|
214
|
-
script: 'qimao-rank-scraper.cjs',
|
|
215
|
-
extra: ['--outdir', outDir],
|
|
216
|
-
};
|
|
217
|
-
return { script: null, extra: [] };
|
|
218
|
-
default:
|
|
219
|
-
return { script: null, extra: [] };
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
|
|
223
169
|
// ---------------------------------------------------------------------------
|
|
224
|
-
//
|
|
170
|
+
// 构建 scraper 参数
|
|
225
171
|
// ---------------------------------------------------------------------------
|
|
226
172
|
|
|
227
|
-
function
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
// 简单解析 Markdown 中的 ## #N 书名 格式
|
|
238
|
-
const matches = [...content.matchAll(/^## #(\d+) (.+)$/gm)];
|
|
239
|
-
for (const m of matches) {
|
|
240
|
-
books.push({ rank: parseInt(m[1]), title: m[2].trim() });
|
|
173
|
+
function scraperArgs(platform, channel, type, outDir) {
|
|
174
|
+
switch (platform) {
|
|
175
|
+
case 'fanqie': {
|
|
176
|
+
// channel=1/0, type=2/1, __all__ 展开
|
|
177
|
+
const ch = channel === 'all' ? 'all' : channel;
|
|
178
|
+
const ty = type === 'all' ? 'all' : type;
|
|
179
|
+
const extra = ['--outdir', outDir];
|
|
180
|
+
if (ch !== 'all') extra.push('--channel', ch);
|
|
181
|
+
if (ty !== 'all') extra.push('--type', ty);
|
|
182
|
+
return { script: 'fanqie-rank-scraper.cjs', extra };
|
|
241
183
|
}
|
|
242
|
-
}
|
|
243
184
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
'',
|
|
251
|
-
'## 题材热度排行',
|
|
252
|
-
'- (从原始数据分析提取)',
|
|
253
|
-
'',
|
|
254
|
-
'## 新题材信号',
|
|
255
|
-
'- (从原始数据分析提取)',
|
|
256
|
-
'',
|
|
257
|
-
'## 关键数据洞察',
|
|
258
|
-
`- 字数区间:(待分析)`,
|
|
259
|
-
`- 书名特征:(待分析)`,
|
|
260
|
-
'',
|
|
261
|
-
'## 值得关注的方向',
|
|
262
|
-
'1. (待从榜单提取后填入)',
|
|
263
|
-
'',
|
|
264
|
-
'## 一句话',
|
|
265
|
-
'(待分析后填入)',
|
|
266
|
-
].join('\n');
|
|
267
|
-
|
|
268
|
-
const reportFile = join(outDir, `${pcn}${lenStr}扫榜报告_${today}.md`);
|
|
269
|
-
writeFileSync(reportFile, report, 'utf-8');
|
|
270
|
-
console.error(` ✅ 扫榜报告 → ${reportFile}`);
|
|
271
|
-
return reportFile;
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
// ---------------------------------------------------------------------------
|
|
275
|
-
// topic_decision 追加
|
|
276
|
-
// ---------------------------------------------------------------------------
|
|
185
|
+
case 'jinjiang': {
|
|
186
|
+
// type 是 orderstr id
|
|
187
|
+
const extra = ['--outdir', outDir];
|
|
188
|
+
if (type !== 'all') extra.push('--type', type);
|
|
189
|
+
return { script: 'jjwxc-rank-scraper.cjs', extra };
|
|
190
|
+
}
|
|
277
191
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
192
|
+
case 'qimao': {
|
|
193
|
+
const extra = ['--outdir', outDir];
|
|
194
|
+
if (channel !== 'all') extra.push('--channel', channel);
|
|
195
|
+
if (type !== 'all') extra.push('--type', type);
|
|
196
|
+
return { script: 'qimao-rank-scraper.cjs', extra };
|
|
197
|
+
}
|
|
282
198
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
`- 扫榜日期:${today}`,
|
|
286
|
-
`- 数据来源:${reportFile}`,
|
|
287
|
-
'',
|
|
288
|
-
'### 选题 1:(待从报告提取)',
|
|
289
|
-
'- 题材组合:(待填入)',
|
|
290
|
-
'- 目标读者:(待填入)',
|
|
291
|
-
'- 核心卖点:(待填入)',
|
|
292
|
-
'- 能爆的原因:(待填入)',
|
|
293
|
-
'- 差异化定位:(待填入)',
|
|
294
|
-
'- 可行性:高/中/低 — (待评估)',
|
|
295
|
-
'- 失败风险:(待评估)',
|
|
296
|
-
'- 验证动作:(待填入)',
|
|
297
|
-
'- 篇幅/平台:(待填入)',
|
|
298
|
-
].join('\n');
|
|
299
|
-
|
|
300
|
-
const decisionFile = join(outDir, `topic_decision_${today}.md`);
|
|
301
|
-
const sep = existsSync(decisionFile) ? '\n\n---\n\n' : '';
|
|
302
|
-
|
|
303
|
-
if (existsSync(decisionFile)) {
|
|
304
|
-
writeFileSync(decisionFile, readFileSync(decisionFile, 'utf-8') + sep + section + '\n', 'utf-8');
|
|
305
|
-
} else {
|
|
306
|
-
const header = `# 选题决策:${today}\n\n---\n\n`;
|
|
307
|
-
writeFileSync(decisionFile, header + section + '\n', 'utf-8');
|
|
199
|
+
default:
|
|
200
|
+
return { script: null, extra: [] };
|
|
308
201
|
}
|
|
309
|
-
console.error(` ✅ topic_decision 追加 → ${decisionFile}`);
|
|
310
|
-
return decisionFile;
|
|
311
202
|
}
|
|
312
203
|
|
|
313
204
|
// ---------------------------------------------------------------------------
|
|
@@ -316,109 +207,65 @@ function appendTopicDecision(platform, length, reportFile, outDir) {
|
|
|
316
207
|
|
|
317
208
|
async function main() {
|
|
318
209
|
const args = process.argv.slice(2);
|
|
319
|
-
let projectRoot = '', platform = '',
|
|
210
|
+
let projectRoot = '', platform = '', channel = 'all', type = 'all';
|
|
320
211
|
|
|
321
212
|
for (let i = 0; i < args.length; i++) {
|
|
322
213
|
if (args[i] === '--project-root') projectRoot = args[i + 1] || '';
|
|
323
|
-
if (args[i] === '--platform')
|
|
324
|
-
if (args[i] === '--
|
|
214
|
+
if (args[i] === '--platform') platform = args[i + 1] || '';
|
|
215
|
+
if (args[i] === '--channel') channel = args[i + 1] || 'all';
|
|
216
|
+
if (args[i] === '--type') type = args[i + 1] || 'all';
|
|
325
217
|
}
|
|
326
218
|
|
|
327
|
-
if (!projectRoot || !platform
|
|
328
|
-
console.error('缺少必要参数: --project-root --platform
|
|
219
|
+
if (!projectRoot || !platform) {
|
|
220
|
+
console.error('缺少必要参数: --project-root --platform');
|
|
329
221
|
process.exit(1);
|
|
330
222
|
}
|
|
331
223
|
|
|
332
|
-
// 检查 .fancy-deployed
|
|
333
224
|
if (!existsSync(join(projectRoot, '.fancy-deployed'))) {
|
|
334
225
|
console.error('项目未初始化:.fancy-deployed 不存在');
|
|
335
226
|
process.exit(1);
|
|
336
227
|
}
|
|
337
228
|
|
|
338
|
-
|
|
339
|
-
if (!
|
|
229
|
+
const VALID = ['qidian', 'fanqie', 'jinjiang', 'qimao'];
|
|
230
|
+
if (!VALID.includes(platform)) {
|
|
340
231
|
console.error(`不支持的平台: ${platform}`);
|
|
341
232
|
process.exit(1);
|
|
342
233
|
}
|
|
343
|
-
if (!VALID_LENGTHS.includes(length)) {
|
|
344
|
-
console.error(`不支持的篇幅: ${length}`);
|
|
345
|
-
process.exit(1);
|
|
346
|
-
}
|
|
347
|
-
if (!PLATFORM_SUPPORTED_LENGTHS[platform].includes(length)) {
|
|
348
|
-
console.error(`${platform} 不支持 ${length}`);
|
|
349
|
-
process.exit(1);
|
|
350
|
-
}
|
|
351
234
|
|
|
352
235
|
const today = todayStr();
|
|
353
236
|
const scanDir = join(projectRoot, '扫榜结果');
|
|
354
237
|
ensureDir(scanDir);
|
|
355
238
|
|
|
356
239
|
let scanFiles = [];
|
|
357
|
-
let scrapeFiles = [];
|
|
358
240
|
|
|
359
241
|
if (platform === 'qidian') {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
console.error(' ❌ 起点采集失败');
|
|
365
|
-
process.exit(1);
|
|
366
|
-
}
|
|
367
|
-
scanFiles = scrapeFiles.map(f => f.file);
|
|
242
|
+
console.error(`→ 采集 起点(mobile SSR)...`);
|
|
243
|
+
const results = await scrapeQidian(scanDir);
|
|
244
|
+
if (!results.length) { console.error('❌ 起点采集失败'); process.exit(1); }
|
|
245
|
+
scanFiles = results.map(r => r.file);
|
|
368
246
|
} else {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
const
|
|
377
|
-
if (!
|
|
378
|
-
|
|
379
|
-
process.exit(1);
|
|
380
|
-
}
|
|
381
|
-
// scraper 输出落在 stdout(JSON 摘要),找到它
|
|
382
|
-
const lines = result.output.split('\n');
|
|
383
|
-
const writtenFiles = [];
|
|
384
|
-
for (const line of lines) {
|
|
385
|
-
const m = line.match(/已保存:\s*(.+)/);
|
|
386
|
-
if (m) writtenFiles.push(m[1].trim());
|
|
387
|
-
}
|
|
388
|
-
if (!writtenFiles.length) {
|
|
389
|
-
console.error(' ❌ 未找到任何输出文件');
|
|
390
|
-
process.exit(1);
|
|
391
|
-
}
|
|
392
|
-
scanFiles.push(...writtenFiles);
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
// Step 6: 收集原始数据
|
|
396
|
-
const rawData = {};
|
|
397
|
-
for (const f of scanFiles) {
|
|
398
|
-
try {
|
|
399
|
-
const name = f.replace(/^.*\//, '').replace(/\.md$/, '');
|
|
400
|
-
rawData[name] = { path: f, content: '' };
|
|
401
|
-
const { readFileSync } = await import('fs');
|
|
402
|
-
rawData[name].content = readFileSync(f, 'utf-8');
|
|
403
|
-
} catch (_) {}
|
|
247
|
+
const cfg = scraperArgs(platform, channel, type, scanDir);
|
|
248
|
+
if (!cfg.script) { console.error(`❌ 平台 ${platform} 暂不支持`); process.exit(1); }
|
|
249
|
+
console.error(`→ 采集 ${PLATFORM_CN[platform]}(${channel}/${type})...`);
|
|
250
|
+
const r = await spawnScraper(cfg.script, cfg.extra, scanDir);
|
|
251
|
+
if (!r.ok) { console.error(`❌ 采集失败:${r.error}`); process.exit(1); }
|
|
252
|
+
|
|
253
|
+
// 从 stdout 提取 "已保存: xxx.md"
|
|
254
|
+
const writtenFiles = [...r.output.matchAll(/已保存:\s*(.+)/g)].map(m => m[1].trim());
|
|
255
|
+
if (!writtenFiles.length) { console.error('❌ 未找到输出文件'); process.exit(1); }
|
|
256
|
+
scanFiles = writtenFiles;
|
|
404
257
|
}
|
|
405
258
|
|
|
406
259
|
const receipt = {
|
|
407
260
|
ok: true,
|
|
408
261
|
operation: 'scan',
|
|
409
262
|
platform,
|
|
410
|
-
length,
|
|
411
263
|
date: today,
|
|
412
264
|
scan_files: scanFiles,
|
|
413
265
|
topic_decision: join(scanDir, `topic_decision_${today}.md`),
|
|
414
|
-
raw_data: rawData,
|
|
415
|
-
summary: `${PLATFORM_CN[platform]}(${length === 'long' ? '长篇' : '短篇'})采集完成,共 ${scanFiles.length} 个文件`,
|
|
416
266
|
};
|
|
417
267
|
|
|
418
268
|
console.log(JSON.stringify(receipt));
|
|
419
269
|
}
|
|
420
270
|
|
|
421
|
-
main().catch(e => {
|
|
422
|
-
console.error('Fatal:', e);
|
|
423
|
-
process.exit(1);
|
|
424
|
-
});
|
|
271
|
+
main().catch(e => { console.error('Fatal:', e); process.exit(1); });
|
|
@@ -7,11 +7,10 @@
|
|
|
7
7
|
// ---------------------------------------------------------------------------
|
|
8
8
|
|
|
9
9
|
export const PLATFORM_TABLE = [
|
|
10
|
-
['1', 'qidian', '
|
|
11
|
-
['2', 'fanqie', '
|
|
12
|
-
['3', 'jinjiang', '
|
|
13
|
-
['4', '
|
|
14
|
-
['5', 'qimao', '需选择', '七猫'],
|
|
10
|
+
['1', 'qidian', '起点'],
|
|
11
|
+
['2', 'fanqie', '番茄'],
|
|
12
|
+
['3', 'jinjiang', '晋江'],
|
|
13
|
+
['4', 'qimao', '七猫'],
|
|
15
14
|
];
|
|
16
15
|
|
|
17
16
|
export function platformLabel(platform) {
|
|
@@ -19,6 +18,50 @@ export function platformLabel(platform) {
|
|
|
19
18
|
return row ? row[3] : platform;
|
|
20
19
|
}
|
|
21
20
|
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// 各平台榜单选项({ id, label }[]),供弹窗多选用
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
export function rankOptions(platform) {
|
|
26
|
+
switch (platform) {
|
|
27
|
+
case 'qidian':
|
|
28
|
+
// 起点男频大热榜(唯一榜单,无细分)
|
|
29
|
+
return [{ id: 'main', label: '起点中文网主站榜单' }];
|
|
30
|
+
|
|
31
|
+
case 'fanqie':
|
|
32
|
+
return [
|
|
33
|
+
{ id: '1_2', label: '男频阅读榜' },
|
|
34
|
+
{ id: '1_1', label: '男频新书榜' },
|
|
35
|
+
{ id: '0_2', label: '女频阅读榜' },
|
|
36
|
+
{ id: '0_1', label: '女频新书榜' },
|
|
37
|
+
{ id: '__all__', label: '全选' },
|
|
38
|
+
];
|
|
39
|
+
|
|
40
|
+
case 'jinjiang':
|
|
41
|
+
return [
|
|
42
|
+
{ id: '5', label: '月榜' },
|
|
43
|
+
{ id: '7', label: '总分榜' },
|
|
44
|
+
{ id: '4', label: '季度榜' },
|
|
45
|
+
{ id: '12', label: '收入金榜' },
|
|
46
|
+
{ id: '16', label: '完结金榜' },
|
|
47
|
+
{ id: '17', label: '新手金榜' },
|
|
48
|
+
{ id: '__all__', label: '全选' },
|
|
49
|
+
];
|
|
50
|
+
|
|
51
|
+
case 'qimao':
|
|
52
|
+
return [
|
|
53
|
+
{ id: 'male_hot', label: '男生大热榜' },
|
|
54
|
+
{ id: 'male_new', label: '男生新书榜' },
|
|
55
|
+
{ id: 'female_hot', label: '女生大热榜' },
|
|
56
|
+
{ id: 'female_new', label: '女生新书榜' },
|
|
57
|
+
{ id: '__all__', label: '全选' },
|
|
58
|
+
];
|
|
59
|
+
|
|
60
|
+
default:
|
|
61
|
+
return [];
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
22
65
|
// ---------------------------------------------------------------------------
|
|
23
66
|
// 模板
|
|
24
67
|
// ---------------------------------------------------------------------------
|
|
@@ -27,28 +70,24 @@ export const scan = {
|
|
|
27
70
|
platformList() {
|
|
28
71
|
return (
|
|
29
72
|
'📋 支持的平台:\n' +
|
|
30
|
-
PLATFORM_TABLE.map(r => ` ${r[0]}. ${r[3]}
|
|
31
|
-
'\n- 知乎和七猫同时支持长篇和短篇'
|
|
73
|
+
PLATFORM_TABLE.map(r => ` ${r[0]}. ${r[3]}`).join('\n')
|
|
32
74
|
);
|
|
33
75
|
},
|
|
34
76
|
|
|
35
77
|
askPlatform() {
|
|
36
|
-
return { question: '📊 请选择要扫的平台(输入编号 1-
|
|
78
|
+
return { question: '📊 请选择要扫的平台(输入编号 1-4):' };
|
|
37
79
|
},
|
|
38
80
|
|
|
39
|
-
|
|
81
|
+
askRankList(platform) {
|
|
82
|
+
const opts = rankOptions(platform);
|
|
40
83
|
return {
|
|
41
|
-
question: `${platformLabel(platform)}
|
|
84
|
+
question: `${platformLabel(platform)} 有多个榜单,支持多选。请选择要采集的榜单:`,
|
|
42
85
|
hideCustomInput: true,
|
|
43
|
-
options:
|
|
44
|
-
{ label: '长篇' },
|
|
45
|
-
{ label: '短篇' },
|
|
46
|
-
],
|
|
86
|
+
options: opts.map(o => ({ label: o.label })),
|
|
47
87
|
};
|
|
48
88
|
},
|
|
49
89
|
|
|
50
|
-
notify({ type, platform,
|
|
51
|
-
const lenStr = length === 'long' ? '长篇' : '短篇';
|
|
90
|
+
notify({ type, platform, files, topicFile, projectRoot, err }) {
|
|
52
91
|
const pLabel = platformLabel(platform);
|
|
53
92
|
|
|
54
93
|
switch (type) {
|
|
@@ -56,19 +95,16 @@ export const scan = {
|
|
|
56
95
|
return '⚠️ 当前目录尚未初始化。请先运行 /fancy-bootstrap 进行初始化。';
|
|
57
96
|
|
|
58
97
|
case 'invalid_choice':
|
|
59
|
-
return '❌ 无效选择,请输入 1-
|
|
98
|
+
return '❌ 无效选择,请输入 1-4 的编号。';
|
|
60
99
|
|
|
61
100
|
case 'cancelled':
|
|
62
|
-
return '🚫
|
|
63
|
-
|
|
64
|
-
case 'cancelled_with_hint':
|
|
65
|
-
return '⚠️ 已取消。请再次调用 /fancy-scan 继续。';
|
|
101
|
+
return '🚫 已取消。';
|
|
66
102
|
|
|
67
103
|
case 'failed':
|
|
68
|
-
return '❌ 采集失败:' + err;
|
|
104
|
+
return '❌ 采集失败:' + (err || '');
|
|
69
105
|
|
|
70
106
|
case 'parse_error':
|
|
71
|
-
return '❌
|
|
107
|
+
return '❌ 采集结果解析失败。';
|
|
72
108
|
|
|
73
109
|
case 'handover':
|
|
74
110
|
return (
|
|
@@ -82,9 +118,8 @@ export const scan = {
|
|
|
82
118
|
}
|
|
83
119
|
},
|
|
84
120
|
|
|
85
|
-
llmAnalysisPrompt({ platform,
|
|
121
|
+
llmAnalysisPrompt({ platform, date, files, topicFile, projectRoot }) {
|
|
86
122
|
const pLabel = platformLabel(platform);
|
|
87
|
-
const lenStr = length === 'long' ? '长篇' : '短篇';
|
|
88
123
|
const fileList = (files || [])
|
|
89
124
|
.map(f => '- ' + f.replace(projectRoot + '/', ''))
|
|
90
125
|
.join('\n');
|
|
@@ -92,7 +127,7 @@ export const scan = {
|
|
|
92
127
|
return [
|
|
93
128
|
`## 扫榜数据分析任务`,
|
|
94
129
|
``,
|
|
95
|
-
`平台:${pLabel}
|
|
130
|
+
`平台:${pLabel}`,
|
|
96
131
|
`时间:${date || ''}`,
|
|
97
132
|
``,
|
|
98
133
|
`已生成的文件:`,
|
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* 知乎短篇/长篇采集脚本
|
|
4
|
-
*
|
|
5
|
-
* 使用 playwright-core 自己管理浏览器。
|
|
6
|
-
* 入口:https://www.zhihu.com/fiore/h5/vip-web
|
|
7
|
-
* 短篇(4 个榜单):推荐榜 / 热搜榜 / 热度榜 / 口碑榜
|
|
8
|
-
* 长篇(1 个榜单):长篇榜
|
|
9
|
-
*
|
|
10
|
-
* 用法:
|
|
11
|
-
* node zhihu-rank-scraper.js --length short # 短篇 4 个榜单
|
|
12
|
-
* node zhihu-rank-scraper.js --length long # 长篇 1 个榜单
|
|
13
|
-
* node zhihu-rank-scraper.js --length short --boards rec,hot # 指定短篇子榜
|
|
14
|
-
* node zhihu-rank-scraper.js --login-wait 60 # 等待手动登录
|
|
15
|
-
*
|
|
16
|
-
* 注意:知乎需要登录态才能看完整榜单。
|
|
17
|
-
*/
|
|
18
|
-
|
|
19
|
-
const fs = require("fs");
|
|
20
|
-
const path = require("path");
|
|
21
|
-
const { chromium } = require("playwright-core");
|
|
22
|
-
const { getArg, localDateStamp, runCli } = require("./cdp-utils.cjs");
|
|
23
|
-
|
|
24
|
-
const OUTDIR = getArg(process.argv, "--outdir") || "扫榜结果";
|
|
25
|
-
const ENTRY_URL = "https://www.zhihu.com/fiore/h5/vip-web";
|
|
26
|
-
const LENGTH = getArg(process.argv, "--length") || "short";
|
|
27
|
-
const LOGIN_WAIT = parseInt(getArg(process.argv, "--login-wait") || "0", 10);
|
|
28
|
-
|
|
29
|
-
// 短篇 4 个榜单
|
|
30
|
-
const SHORT_BOARDS = [
|
|
31
|
-
{ id: "rec", label: "推荐榜", tabText: "推荐榜" },
|
|
32
|
-
{ id: "hot", label: "热搜榜", tabText: "热搜榜" },
|
|
33
|
-
{ id: "trending", label: "热度榜", tabText: "热度榜" },
|
|
34
|
-
{ id: "reputation",label: "口碑榜", tabText: "口碑榜" },
|
|
35
|
-
];
|
|
36
|
-
|
|
37
|
-
// 长篇 1 个榜单
|
|
38
|
-
const LONG_BOARDS = [
|
|
39
|
-
{ id: "long", label: "长篇榜", tabText: "长篇榜" },
|
|
40
|
-
];
|
|
41
|
-
|
|
42
|
-
const BOARDS = LENGTH === "long" ? LONG_BOARDS : SHORT_BOARDS;
|
|
43
|
-
|
|
44
|
-
const customBoards = getArg(process.argv, "--boards");
|
|
45
|
-
const targetBoards = customBoards
|
|
46
|
-
? BOARDS.filter(b => customBoards.split(",").map(s => s.trim()).includes(b.id))
|
|
47
|
-
: BOARDS;
|
|
48
|
-
|
|
49
|
-
// ---------------------------------------------------------------------------
|
|
50
|
-
// 工具函数
|
|
51
|
-
// ---------------------------------------------------------------------------
|
|
52
|
-
|
|
53
|
-
function sleep(ms) {
|
|
54
|
-
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// ---------------------------------------------------------------------------
|
|
58
|
-
// 采集
|
|
59
|
-
// ---------------------------------------------------------------------------
|
|
60
|
-
|
|
61
|
-
async function scrapeBoard(board) {
|
|
62
|
-
console.log(` [${board.label}] 正在抓取...`);
|
|
63
|
-
|
|
64
|
-
const browser = await chromium.launch({ headless: false, args: ["--no-sandbox", "--disable-dev-shm-usage", "--start-maximized"] });
|
|
65
|
-
const context = await browser.newContext();
|
|
66
|
-
const page = await context.newPage();
|
|
67
|
-
|
|
68
|
-
// CDP 强制窗口最大化
|
|
69
|
-
try {
|
|
70
|
-
const cdp = await context.newCDPSession(page);
|
|
71
|
-
const { windowId } = await cdp.send('Browser.getWindowForTarget', { target: page.target() });
|
|
72
|
-
await cdp.send('Browser.setWindowBounds', { windowId, bounds: { state: 'maximized' } });
|
|
73
|
-
} catch (_) {}
|
|
74
|
-
|
|
75
|
-
try {
|
|
76
|
-
await page.goto(ENTRY_URL, { waitUntil: "networkidle" });
|
|
77
|
-
|
|
78
|
-
if (LOGIN_WAIT > 0) {
|
|
79
|
-
console.log(` ⏳ 等待 ${LOGIN_WAIT}s 供用户登录...`);
|
|
80
|
-
await page.waitForTimeout(LOGIN_WAIT * 1000);
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const host = page.url();
|
|
84
|
-
if (host.indexOf("zhihu") === -1) {
|
|
85
|
-
console.error(` ✗ 当前页面非知乎(url=${host}),可能被重定向,已跳过。`);
|
|
86
|
-
return null;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
// 点击 tab
|
|
90
|
-
const clicked = await page.evaluate((target) => {
|
|
91
|
-
const candidates = Array.from(document.querySelectorAll("a, button, div, span, li"))
|
|
92
|
-
.filter(el => {
|
|
93
|
-
const t = (el.innerText || "").trim();
|
|
94
|
-
return t === target || t.startsWith(target);
|
|
95
|
-
});
|
|
96
|
-
if (!candidates.length) return false;
|
|
97
|
-
candidates[0].click();
|
|
98
|
-
return true;
|
|
99
|
-
}, board.tabText);
|
|
100
|
-
|
|
101
|
-
if (!clicked) {
|
|
102
|
-
console.error(` [${board.label}] 点击 tab 失败`);
|
|
103
|
-
return null;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
await page.waitForTimeout(2500);
|
|
107
|
-
|
|
108
|
-
// 提取故事列表
|
|
109
|
-
const stories = await page.evaluate(() => {
|
|
110
|
-
const items = [];
|
|
111
|
-
const seen = new Set();
|
|
112
|
-
const anchors = document.querySelectorAll('a[href*="/p/"], a[href*="/question/"], a[href*="/xen/"]');
|
|
113
|
-
anchors.forEach((a, i) => {
|
|
114
|
-
const href = a.href || "";
|
|
115
|
-
if (!href || seen.has(href)) return;
|
|
116
|
-
const text = (a.innerText || "").trim();
|
|
117
|
-
if (!text || text.length < 4 || text.length > 200) return;
|
|
118
|
-
if (/^(首页|发现|等你来答|登录|注册|更多|查看全部)$/.test(text)) return;
|
|
119
|
-
seen.add(href);
|
|
120
|
-
items.push({ rank: i + 1, title: text.split("\n")[0].trim(), url: href });
|
|
121
|
-
});
|
|
122
|
-
return items.slice(0, 100);
|
|
123
|
-
});
|
|
124
|
-
|
|
125
|
-
if (!stories || stories.length === 0) {
|
|
126
|
-
console.warn(` [${board.label}] 未提取到故事`);
|
|
127
|
-
return [];
|
|
128
|
-
}
|
|
129
|
-
return stories;
|
|
130
|
-
} finally {
|
|
131
|
-
await browser.close();
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
function buildBoardMarkdown(board, stories, date) {
|
|
136
|
-
const lines = [
|
|
137
|
-
`# 知乎 · ${board.label}`,
|
|
138
|
-
"",
|
|
139
|
-
`- 来源:${ENTRY_URL}`,
|
|
140
|
-
`- 抓取时间:${date}`,
|
|
141
|
-
"",
|
|
142
|
-
"---",
|
|
143
|
-
"",
|
|
144
|
-
];
|
|
145
|
-
if (!stories.length) {
|
|
146
|
-
lines.push("(未提取到数据,可能页面结构变化或需要登录)", "", "---", "");
|
|
147
|
-
} else {
|
|
148
|
-
stories.forEach(s => {
|
|
149
|
-
lines.push(`书名:${s.title || ""}`);
|
|
150
|
-
lines.push(`题材:${LENGTH === "long" ? "长篇" : "短篇"}`);
|
|
151
|
-
lines.push(`作者:${s.author || ""}`);
|
|
152
|
-
if (s.url) lines.push(`作品页:${s.url}`);
|
|
153
|
-
lines.push("");
|
|
154
|
-
});
|
|
155
|
-
}
|
|
156
|
-
return lines.join("\n");
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// ---------------------------------------------------------------------------
|
|
160
|
-
// 入口
|
|
161
|
-
// ---------------------------------------------------------------------------
|
|
162
|
-
|
|
163
|
-
async function main() {
|
|
164
|
-
const date = localDateStamp();
|
|
165
|
-
fs.mkdirSync(OUTDIR, { recursive: true });
|
|
166
|
-
|
|
167
|
-
let written = 0;
|
|
168
|
-
for (const board of targetBoards) {
|
|
169
|
-
const stories = await scrapeBoard(board);
|
|
170
|
-
if (stories === null) continue;
|
|
171
|
-
|
|
172
|
-
const md = buildBoardMarkdown(board, stories, date);
|
|
173
|
-
const filename = `知乎${LENGTH === "long" ? "长篇" : "短篇"}${board.label}_${date}.md`;
|
|
174
|
-
const filepath = path.join(OUTDIR, filename);
|
|
175
|
-
fs.writeFileSync(filepath, md, "utf-8");
|
|
176
|
-
console.log(` ✓ ${filename} (${stories.length} 条)`);
|
|
177
|
-
written++;
|
|
178
|
-
}
|
|
179
|
-
return written;
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
if (require.main === module) {
|
|
183
|
-
runCli(main, "知乎采集");
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
module.exports = { scrapeBoard, buildBoardMarkdown };
|