@cloud411716/fancy-webnovel 0.3.16 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/bootstrap/index.js +138 -0
- package/commands/bootstrap/init-project.js +121 -0
- package/commands/scan/index.js +226 -0
- package/commands/scan/platforms/fanqie.js +148 -0
- package/commands/scan/platforms/jinjiang.js +143 -0
- package/commands/scan/platforms/qidian.js +106 -0
- package/commands/scan/platforms/qimao.js +127 -0
- package/commands/scan/scraper.js +195 -0
- package/cordis.patch.yml +14 -0
- package/events.js +19 -0
- package/index.js +40 -6
- package/infra.js +83 -102
- package/package.json +3 -11
- package/plugins/fancy-bootstrap/index.js +0 -100
- package/plugins/fancy-bootstrap/scripts/init.js +0 -111
- package/plugins/fancy-bootstrap/scripts/path-utils.js +0 -22
- package/plugins/fancy-bootstrap/templates.js +0 -41
- package/plugins/fancy-scan/index.js +0 -195
- package/plugins/fancy-scan/scripts/run-scan.js +0 -295
- package/plugins/fancy-scan/scripts/scan-output-format.md +0 -188
- package/plugins/fancy-scan/scripts/scraper-registry.md +0 -104
- package/plugins/fancy-scan/scripts/scrapers/cdp-utils.cjs +0 -75
- package/plugins/fancy-scan/scripts/scrapers/fanqie-rank-scraper.cjs +0 -441
- package/plugins/fancy-scan/scripts/scrapers/jjwxc-rank-scraper.cjs +0 -406
- package/plugins/fancy-scan/scripts/scrapers/package-lock.json +0 -27
- package/plugins/fancy-scan/scripts/scrapers/qimao-rank-scraper.cjs +0 -461
- package/plugins/fancy-scan/templates.js +0 -158
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* fancy-scan plugin entry
|
|
3
|
-
*/
|
|
4
|
-
import { fileURLToPath } from 'url';
|
|
5
|
-
import { dirname, join } from 'path';
|
|
6
|
-
import { existsSync } from 'fs';
|
|
7
|
-
import { notify, llmFollowup, spawnScript, startActivity } from '../../infra.js';
|
|
8
|
-
import { scan, PLATFORM_TABLE, rankOptions } from './templates.js';
|
|
9
|
-
|
|
10
|
-
export const inject = ['commands', 'userQuestions'];
|
|
11
|
-
|
|
12
|
-
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
13
|
-
const SCAN_SCRIPTS_DIR = join(__dirname, 'scripts');
|
|
14
|
-
const SCRAPER_TIMEOUT = 30 * 60 * 1000; // 30 分钟
|
|
15
|
-
|
|
16
|
-
export async function apply(ctx) {
|
|
17
|
-
ctx.commands.register({
|
|
18
|
-
name: 'fancy-scan',
|
|
19
|
-
description: '📊 扫榜分析 ( usage: /fancy-scan )',
|
|
20
|
-
handler: async (invocation) => {
|
|
21
|
-
const session = invocation.agent.session;
|
|
22
|
-
|
|
23
|
-
// 获取项目根
|
|
24
|
-
const projectRoot = process.cwd();
|
|
25
|
-
if (!existsSync(join(projectRoot, '.fancy-deployed'))) {
|
|
26
|
-
notify(session, scan.notify({ type: 'not_initialized' }));
|
|
27
|
-
return { kind: 'success', text: '' };
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// ---------- 第一轮:选择平台 ----------
|
|
31
|
-
notify(session, scan.platformList());
|
|
32
|
-
|
|
33
|
-
const platformResult = await ctx.userQuestions.ask({
|
|
34
|
-
questions: [scan.askPlatform()],
|
|
35
|
-
});
|
|
36
|
-
const answer = platformResult.answers[0]?.custom?.trim();
|
|
37
|
-
if (!answer) {
|
|
38
|
-
notify(session, scan.notify({ type: 'cancelled' }));
|
|
39
|
-
return { kind: 'success', text: '' };
|
|
40
|
-
}
|
|
41
|
-
const numMatch = answer.match(/^(\d+)$/);
|
|
42
|
-
const row = numMatch ? PLATFORM_TABLE.find(r => r[0] === numMatch[1]) : null;
|
|
43
|
-
if (!row) {
|
|
44
|
-
notify(session, scan.notify({ type: 'invalid_choice' }));
|
|
45
|
-
return { kind: 'success', text: '' };
|
|
46
|
-
}
|
|
47
|
-
const platform = row[1];
|
|
48
|
-
|
|
49
|
-
// ---------- 第二轮:选择榜单(多选) ----------
|
|
50
|
-
const rankResult = await ctx.userQuestions.ask({
|
|
51
|
-
questions: [scan.askRankList(platform)],
|
|
52
|
-
});
|
|
53
|
-
const selected = rankResult.answers[0]?.selected ?? [];
|
|
54
|
-
const custom = rankResult.answers[0]?.custom?.trim();
|
|
55
|
-
|
|
56
|
-
// 用户可能直接输入自定义值
|
|
57
|
-
const rawChoices = selected.length > 0 ? selected : (custom ? [custom] : []);
|
|
58
|
-
if (rawChoices.length === 0) {
|
|
59
|
-
notify(session, scan.notify({ type: 'cancelled' }));
|
|
60
|
-
return { kind: 'success', text: '' };
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
// 构建每个榜单的 channel+type 参数
|
|
64
|
-
const platformRanks = rankOptions(platform);
|
|
65
|
-
const isAll = rawChoices.includes('全选') || rawChoices.includes('__all__');
|
|
66
|
-
const rankIds = isAll
|
|
67
|
-
? platformRanks.filter(r => r.id !== '__all__').map(r => r.id)
|
|
68
|
-
: rawChoices;
|
|
69
|
-
|
|
70
|
-
// 把 rankId 转换成 { channel, type } 并去重
|
|
71
|
-
const rankEntries = rankIds.map(id => parseRankId(platform, resolveId(platform, id)));
|
|
72
|
-
const unique = [];
|
|
73
|
-
const seen = new Set();
|
|
74
|
-
for (const e of rankEntries) {
|
|
75
|
-
const key = `${e.channel}__${e.type}`;
|
|
76
|
-
if (!seen.has(key)) { seen.add(key); unique.push(e); }
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// ---------- 串行采集 ----------
|
|
80
|
-
let allFiles = [];
|
|
81
|
-
let lastReceipt = null;
|
|
82
|
-
const failedRanks = []; // 收集失败项:{ rank, err }
|
|
83
|
-
|
|
84
|
-
for (const rank of unique) {
|
|
85
|
-
const stop = startActivity(session);
|
|
86
|
-
const r = await spawnScript(
|
|
87
|
-
'node',
|
|
88
|
-
[
|
|
89
|
-
join(SCAN_SCRIPTS_DIR, 'run-scan.js'),
|
|
90
|
-
'--project-root', projectRoot,
|
|
91
|
-
'--platform', platform,
|
|
92
|
-
'--channel', rank.channel,
|
|
93
|
-
'--type', rank.type,
|
|
94
|
-
],
|
|
95
|
-
{ timeout: SCRAPER_TIMEOUT, stop }
|
|
96
|
-
);
|
|
97
|
-
|
|
98
|
-
if (!r.ok) {
|
|
99
|
-
const rawErr = (typeof r.error === 'object' && r.error !== null)
|
|
100
|
-
? (r.error.message || JSON.stringify(r.error))
|
|
101
|
-
: String(r.error || '');
|
|
102
|
-
if (rawErr.includes("Executable doesn't exist") || rawErr.includes("doesn't exist")) {
|
|
103
|
-
notify(session,
|
|
104
|
-
'⚠️ 未找到 Chromium 浏览器。\n\n安装命令:\n' +
|
|
105
|
-
' npm install -g playwright@1.48.0 --registry=https://registry.npmmirror.com\n' +
|
|
106
|
-
' PLAYWRIGHT_DOWNLOAD_HOST=https://npmmirror.com/mirrors/playwright/ playwright install chromium --with-deps\n\n' +
|
|
107
|
-
'安装完成后重新运行 /fancy-scan 即可。'
|
|
108
|
-
);
|
|
109
|
-
return { kind: 'success', text: '' };
|
|
110
|
-
}
|
|
111
|
-
failedRanks.push({ rank, err: rawErr });
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
// 解析采集结果
|
|
116
|
-
let receipt;
|
|
117
|
-
try { receipt = JSON.parse(r.output); } catch (_) {
|
|
118
|
-
failedRanks.push({ rank, err: 'JSON 解析失败' });
|
|
119
|
-
continue;
|
|
120
|
-
}
|
|
121
|
-
lastReceipt = receipt;
|
|
122
|
-
allFiles = allFiles.concat(receipt.scan_files || []);
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
// ---------- 全部采集完成后统一输出 ----------
|
|
126
|
-
if (allFiles.length > 0 && lastReceipt) {
|
|
127
|
-
notify(session, scan.notify({
|
|
128
|
-
type: 'handover',
|
|
129
|
-
platform,
|
|
130
|
-
files: allFiles,
|
|
131
|
-
topicFile: lastReceipt.topic_decision || '',
|
|
132
|
-
projectRoot,
|
|
133
|
-
}));
|
|
134
|
-
const prompt = scan.llmAnalysisPrompt({
|
|
135
|
-
platform,
|
|
136
|
-
date: lastReceipt.date || '',
|
|
137
|
-
files: allFiles,
|
|
138
|
-
topicFile: lastReceipt.topic_decision || '',
|
|
139
|
-
projectRoot,
|
|
140
|
-
});
|
|
141
|
-
llmFollowup(invocation, prompt);
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
// 统一展示所有失败项
|
|
145
|
-
if (failedRanks.length > 0) {
|
|
146
|
-
const lines = failedRanks.map(({ rank, err }) => {
|
|
147
|
-
const label = `${rank.channel}/${rank.type}`;
|
|
148
|
-
return `• ${label}:${err}`;
|
|
149
|
-
});
|
|
150
|
-
notify(session, `⚠️ 以下榜单采集失败(共 ${failedRanks.length} 项):\n${lines.join('\n')}`);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
return { kind: 'success', text: '' };
|
|
154
|
-
},
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
// ---------------------------------------------------------------------------
|
|
159
|
-
// 将 rankId 转换成 { channel, type } 参数
|
|
160
|
-
// ---------------------------------------------------------------------------
|
|
161
|
-
|
|
162
|
-
// 将 label 或 id 统一成 id(UI 可能返回 label)
|
|
163
|
-
function resolveId(platform, raw) {
|
|
164
|
-
const opts = rankOptions(platform);
|
|
165
|
-
const byId = opts.find(o => o.id === raw);
|
|
166
|
-
if (byId) return byId.id;
|
|
167
|
-
const byLabel = opts.find(o => o.label === raw);
|
|
168
|
-
if (byLabel) return byLabel.id;
|
|
169
|
-
return raw; // 未知值直接透传
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
function parseRankId(platform, id) {
|
|
173
|
-
switch (platform) {
|
|
174
|
-
case 'qimao': {
|
|
175
|
-
// id 格式: male_hot / female_new
|
|
176
|
-
const [ch, rt] = id.split('_');
|
|
177
|
-
return { channel: ch, type: rt };
|
|
178
|
-
}
|
|
179
|
-
case 'fanqie': {
|
|
180
|
-
// id 格式: 0_1 / 1_2 / __all__
|
|
181
|
-
const [ch, ty] = id.split('_');
|
|
182
|
-
return { channel: ch, type: ty };
|
|
183
|
-
}
|
|
184
|
-
case 'jinjiang': {
|
|
185
|
-
// id 格式: 5 / 7 / 12 等(orderstr)
|
|
186
|
-
return { channel: 'all', type: id };
|
|
187
|
-
}
|
|
188
|
-
case 'qidian': {
|
|
189
|
-
// id: hotsales / yuepiao / signnewbook / pubnewbook / newauthor
|
|
190
|
-
return { channel: id, type: id };
|
|
191
|
-
}
|
|
192
|
-
default:
|
|
193
|
-
return { channel: 'all', type: 'all' };
|
|
194
|
-
}
|
|
195
|
-
}
|
|
@@ -1,295 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
/**
|
|
3
|
-
* fancy-scan run-scan.js — 采集编排层
|
|
4
|
-
*
|
|
5
|
-
* 职责:
|
|
6
|
-
* 1. 检查 .fancy-deployed
|
|
7
|
-
* 2. 调用平台采集脚本(一次一个榜单)
|
|
8
|
-
* 3. 输出 JSON 摘要(供 index.js 解析)
|
|
9
|
-
*
|
|
10
|
-
* 调用方式:
|
|
11
|
-
* node run-scan.js --project-root <abs> --platform qidian --channel main --type main
|
|
12
|
-
* node run-scan.js --project-root <abs> --platform fanqie --channel 1 --type 2
|
|
13
|
-
* node run-scan.js --project-root <abs> --platform jinjiang --channel all --type 5
|
|
14
|
-
* node run-scan.js --project-root <abs> --platform qimao --channel male --type hot
|
|
15
|
-
*/
|
|
16
|
-
|
|
17
|
-
import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
|
|
18
|
-
import { join, dirname } from 'path';
|
|
19
|
-
import { fileURLToPath } from 'url';
|
|
20
|
-
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
// 平台中文名
|
|
23
|
-
// ---------------------------------------------------------------------------
|
|
24
|
-
|
|
25
|
-
const PLATFORM_CN = {
|
|
26
|
-
qidian: '起点',
|
|
27
|
-
fanqie: '番茄',
|
|
28
|
-
jinjiang: '晋江',
|
|
29
|
-
qimao: '七猫',
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
// ---------------------------------------------------------------------------
|
|
33
|
-
// 工具函数
|
|
34
|
-
// ---------------------------------------------------------------------------
|
|
35
|
-
|
|
36
|
-
const MAX_RETRIES = 3;
|
|
37
|
-
const RETRY_DELAY_MS = 5000;
|
|
38
|
-
|
|
39
|
-
function sleep(ms) {
|
|
40
|
-
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async function withRetry(fn, retries = MAX_RETRIES, delayMs = RETRY_DELAY_MS, label = '') {
|
|
44
|
-
let lastErr;
|
|
45
|
-
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
46
|
-
try {
|
|
47
|
-
return await fn();
|
|
48
|
-
} catch (err) {
|
|
49
|
-
lastErr = err;
|
|
50
|
-
if (attempt < retries) {
|
|
51
|
-
console.log(` ⏳ ${label} 第${attempt}次失败,${delayMs / 1000}s后重试...`);
|
|
52
|
-
sleep(delayMs);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
return { _failed: true, err: lastErr, label };
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
function nowIso() {
|
|
60
|
-
return new Date().toISOString();
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
function todayStr() {
|
|
64
|
-
const d = new Date();
|
|
65
|
-
return `${d.getFullYear()}${String(d.getMonth()+1).padStart(2,'0')}${String(d.getDate()).padStart(2,'0')}`;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function ensureDir(dir) {
|
|
69
|
-
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
// ---------------------------------------------------------------------------
|
|
73
|
-
// 起点:HTTP mobile SSR(自包含,无需 playwright)
|
|
74
|
-
// ---------------------------------------------------------------------------
|
|
75
|
-
|
|
76
|
-
async function scrapeQidian(outDir, ranks) {
|
|
77
|
-
const MOBILE_BASE = 'https://m.qidian.com';
|
|
78
|
-
|
|
79
|
-
const MOBILE_HEADERS = {
|
|
80
|
-
'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
|
|
81
|
-
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
82
|
-
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
|
83
|
-
'Accept-Encoding': 'identity',
|
|
84
|
-
};
|
|
85
|
-
|
|
86
|
-
const results = [];
|
|
87
|
-
|
|
88
|
-
for (const rt of ranks) {
|
|
89
|
-
const url = `${MOBILE_BASE}${rt.path}`;
|
|
90
|
-
|
|
91
|
-
const doScrape = async () => {
|
|
92
|
-
const html = await fetch(url, { headers: MOBILE_HEADERS, signal: AbortSignal.timeout(15000) }).then(r => r.text());
|
|
93
|
-
const m = html.match(/<script[^>]+id=["']vite-plugin-ssr_pageContext["'][^>]*>([\s\S]*?)<\/script>/i);
|
|
94
|
-
if (!m) throw new Error(`未找到 pageContext`);
|
|
95
|
-
const pageContext = JSON.parse(m[1]);
|
|
96
|
-
const records = pageContext?.pageContext?.pageProps?.pageData?.records || [];
|
|
97
|
-
if (!records.length) throw new Error(`无书籍数据`);
|
|
98
|
-
return records;
|
|
99
|
-
};
|
|
100
|
-
|
|
101
|
-
const records = await withRetry(doScrape, MAX_RETRIES, RETRY_DELAY_MS, `起点${rt.label}`);
|
|
102
|
-
|
|
103
|
-
if (records._failed) {
|
|
104
|
-
console.error(` ⚠ 起点${rt.label}连续${MAX_RETRIES}次失败:${records.err.message}`);
|
|
105
|
-
continue;
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
const books = records.map((r, i) => ({
|
|
109
|
-
rank: r.rankNum || i + 1,
|
|
110
|
-
title: r.bName || r.bookName || '',
|
|
111
|
-
url: r.bid ? `${MOBILE_BASE}/book/${r.bid}/` : '',
|
|
112
|
-
author: r.bAuth || r.author || '',
|
|
113
|
-
genre: [r.cat, r.subCat].filter(Boolean).join('·'),
|
|
114
|
-
status: r.status || '',
|
|
115
|
-
words: r.cnt || r.wordCount || '',
|
|
116
|
-
}));
|
|
117
|
-
|
|
118
|
-
const outFile = join(outDir, `${PLATFORM_CN.qidian}${rt.label}_${todayStr()}.md`);
|
|
119
|
-
writeFileSync(outFile, renderMarkdown(rt.label, url, books), 'utf-8');
|
|
120
|
-
console.error(` ✅ ${rt.label}: ${books.length} 本`);
|
|
121
|
-
results.push({ label: rt.label, count: books.length, file: outFile });
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
return results;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
function renderMarkdown(rankLabel, url, books) {
|
|
128
|
-
const now = nowIso();
|
|
129
|
-
const lines = [
|
|
130
|
-
`# 起点 · ${rankLabel}`,
|
|
131
|
-
``,
|
|
132
|
-
`- 来源:${url}`,
|
|
133
|
-
`- 抓取方式:mobile-ssr`,
|
|
134
|
-
`- 抓取时间:${now}`,
|
|
135
|
-
``,
|
|
136
|
-
'---',
|
|
137
|
-
'',
|
|
138
|
-
];
|
|
139
|
-
for (const b of books) {
|
|
140
|
-
lines.push(`书名:${b.title}`);
|
|
141
|
-
lines.push(`题材:${b.genre}`);
|
|
142
|
-
lines.push(`作者:${b.author}`);
|
|
143
|
-
if (b.url) lines.push(`作品页:${b.url}`);
|
|
144
|
-
lines.push('');
|
|
145
|
-
}
|
|
146
|
-
return lines.join('\n');
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// ---------------------------------------------------------------------------
|
|
150
|
-
// Browser scraper spawner(30 分钟超时)
|
|
151
|
-
// ---------------------------------------------------------------------------
|
|
152
|
-
|
|
153
|
-
function spawnScraper(script, extraArgs, outDir) {
|
|
154
|
-
return new Promise(async (resolve) => {
|
|
155
|
-
const { spawn } = await import('child_process');
|
|
156
|
-
const scriptPath = join(dirname(fileURLToPath(import.meta.url)), 'scrapers', script);
|
|
157
|
-
process.stderr.write(`\n→ 启动 ${script}...\n`);
|
|
158
|
-
const p = spawn('node', [scriptPath, ...extraArgs], {
|
|
159
|
-
timeout: 30 * 60 * 1000,
|
|
160
|
-
stdio: ['ignore', 'pipe', 'inherit'],
|
|
161
|
-
});
|
|
162
|
-
let out = '';
|
|
163
|
-
p.stdout.on('data', d => { out += d.toString(); });
|
|
164
|
-
p.on('close', (code) => {
|
|
165
|
-
const outTrimmed = out.trim();
|
|
166
|
-
// scraper 内部吞掉了很多错误(返回 null 但 exit 0),通过 stdout 里的失败标识判断
|
|
167
|
-
const softFail = outTrimmed.includes('已跳过') ||
|
|
168
|
-
outTrimmed.includes('采集失败') ||
|
|
169
|
-
outTrimmed.includes('✗') ||
|
|
170
|
-
outTrimmed.includes('failed');
|
|
171
|
-
if (code === 0 && !softFail && outTrimmed) {
|
|
172
|
-
resolve({ ok: true, output: out });
|
|
173
|
-
} else {
|
|
174
|
-
// 把 stdout 末 500 字符当错误摘要
|
|
175
|
-
resolve({ ok: false, output: out, error: outTrimmed.slice(-500) || `exit ${code}` });
|
|
176
|
-
}
|
|
177
|
-
});
|
|
178
|
-
p.on('error', e => resolve({ ok: false, output: '', error: e.message }));
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
// ---------------------------------------------------------------------------
|
|
183
|
-
// 构建 scraper 参数
|
|
184
|
-
// ---------------------------------------------------------------------------
|
|
185
|
-
|
|
186
|
-
function scraperArgs(platform, channel, type, outDir) {
|
|
187
|
-
switch (platform) {
|
|
188
|
-
case 'fanqie': {
|
|
189
|
-
// channel=1/0, type=2/1, __all__ 展开
|
|
190
|
-
const ch = channel === 'all' ? 'all' : channel;
|
|
191
|
-
const ty = type === 'all' ? 'all' : type;
|
|
192
|
-
const extra = ['--outdir', outDir];
|
|
193
|
-
if (ch !== 'all') extra.push('--channel', ch);
|
|
194
|
-
if (ty !== 'all') extra.push('--type', ty);
|
|
195
|
-
return { script: 'fanqie-rank-scraper.cjs', extra };
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
case 'jinjiang': {
|
|
199
|
-
// type 是 orderstr id
|
|
200
|
-
const extra = ['--outdir', outDir];
|
|
201
|
-
if (type !== 'all') extra.push('--type', type);
|
|
202
|
-
return { script: 'jjwxc-rank-scraper.cjs', extra };
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
case 'qimao': {
|
|
206
|
-
const extra = ['--outdir', outDir];
|
|
207
|
-
if (channel !== 'all') extra.push('--channel', channel);
|
|
208
|
-
if (type !== 'all') extra.push('--type', type);
|
|
209
|
-
return { script: 'qimao-rank-scraper.cjs', extra };
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
default:
|
|
213
|
-
return { script: null, extra: [] };
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
// ---------------------------------------------------------------------------
|
|
218
|
-
// 主流程
|
|
219
|
-
// ---------------------------------------------------------------------------
|
|
220
|
-
|
|
221
|
-
async function main() {
|
|
222
|
-
const args = process.argv.slice(2);
|
|
223
|
-
let projectRoot = '', platform = '', channel = 'all', type = 'all';
|
|
224
|
-
|
|
225
|
-
for (let i = 0; i < args.length; i++) {
|
|
226
|
-
if (args[i] === '--project-root') projectRoot = args[i + 1] || '';
|
|
227
|
-
if (args[i] === '--platform') platform = args[i + 1] || '';
|
|
228
|
-
if (args[i] === '--channel') channel = args[i + 1] || 'all';
|
|
229
|
-
if (args[i] === '--type') type = args[i + 1] || 'all';
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
if (!projectRoot || !platform) {
|
|
233
|
-
console.error('缺少必要参数: --project-root --platform');
|
|
234
|
-
process.exit(1);
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
if (!existsSync(join(projectRoot, '.fancy-deployed'))) {
|
|
238
|
-
console.error('项目未初始化:.fancy-deployed 不存在');
|
|
239
|
-
process.exit(1);
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const VALID = ['qidian', 'fanqie', 'jinjiang', 'qimao'];
|
|
243
|
-
if (!VALID.includes(platform)) {
|
|
244
|
-
console.error(`不支持的平台: ${platform}`);
|
|
245
|
-
process.exit(1);
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
const today = todayStr();
|
|
249
|
-
const scanDir = join(projectRoot, '扫榜结果');
|
|
250
|
-
ensureDir(scanDir);
|
|
251
|
-
|
|
252
|
-
let scanFiles = [];
|
|
253
|
-
|
|
254
|
-
if (platform === 'qidian') {
|
|
255
|
-
const allRanks = [
|
|
256
|
-
{ id: 'hotsales', label: '畅销榜', path: '/rank/hotsales/' },
|
|
257
|
-
{ id: 'yuepiao', label: '月票榜', path: '/rank/yuepiao/' },
|
|
258
|
-
{ id: 'signnewbook', label: '签约作者新书榜', path: '/rank/sign/' },
|
|
259
|
-
{ id: 'pubnewbook', label: '公众作者新书榜', path: '/rank/newbook/' },
|
|
260
|
-
{ id: 'newauthor', label: '新人作者新书榜', path: '/rank/newauthor/' },
|
|
261
|
-
];
|
|
262
|
-
const ranks = channel === 'all'
|
|
263
|
-
? allRanks
|
|
264
|
-
: allRanks.filter(r => r.id === channel);
|
|
265
|
-
if (!ranks.length) { console.error('❌ 未知起点榜单: ' + channel); process.exit(1); }
|
|
266
|
-
console.error(`→ 采集 起点(${channel === 'all' ? '全部' : ranks[0].label})...`);
|
|
267
|
-
const results = await scrapeQidian(scanDir, ranks);
|
|
268
|
-
if (!results.length) { console.error('❌ 起点采集失败'); process.exit(1); }
|
|
269
|
-
scanFiles = results.map(r => r.file);
|
|
270
|
-
} else {
|
|
271
|
-
const cfg = scraperArgs(platform, channel, type, scanDir);
|
|
272
|
-
if (!cfg.script) { console.error(`❌ 平台 ${platform} 暂不支持`); process.exit(1); }
|
|
273
|
-
console.error(`→ 采集 ${PLATFORM_CN[platform]}(${channel}/${type})...`);
|
|
274
|
-
const r = await spawnScraper(cfg.script, cfg.extra, scanDir);
|
|
275
|
-
if (!r.ok) { console.error(`❌ 采集失败:${r.error}`); process.exit(1); }
|
|
276
|
-
|
|
277
|
-
// 从 stdout 提取 "已保存: xxx.md"
|
|
278
|
-
const writtenFiles = [...r.output.matchAll(/已保存:\s*(.+)/g)].map(m => m[1].trim());
|
|
279
|
-
if (!writtenFiles.length) { console.error('❌ 未找到输出文件'); process.exit(1); }
|
|
280
|
-
scanFiles = writtenFiles;
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
const receipt = {
|
|
284
|
-
ok: true,
|
|
285
|
-
operation: 'scan',
|
|
286
|
-
platform,
|
|
287
|
-
date: today,
|
|
288
|
-
scan_files: scanFiles,
|
|
289
|
-
topic_decision: join(scanDir, `topic_decision_${today}.md`),
|
|
290
|
-
};
|
|
291
|
-
|
|
292
|
-
console.log(JSON.stringify(receipt));
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
main().catch(e => { console.error('Fatal:', e); process.exit(1); });
|
|
@@ -1,188 +0,0 @@
|
|
|
1
|
-
# 扫榜数据采集格式规范
|
|
2
|
-
定义起点/番茄/七猫/晋江的采集字段、输出模板和清洗规则。
|
|
3
|
-
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
## 字段规范(所有平台通用)
|
|
7
|
-
|
|
8
|
-
每本书单独输出以下四个字段,不得合并为一行:
|
|
9
|
-
|
|
10
|
-
```
|
|
11
|
-
书名:{书名}
|
|
12
|
-
题材:{题材}
|
|
13
|
-
作者:{作者}
|
|
14
|
-
作品页:{URL}
|
|
15
|
-
```
|
|
16
|
-
|
|
17
|
-
---
|
|
18
|
-
|
|
19
|
-
## 番茄小说
|
|
20
|
-
|
|
21
|
-
### 题材 cat_id
|
|
22
|
-
|
|
23
|
-
男频19个:西方奇幻(1141) / 东方仙侠(1140) / 科幻末世(8) / 都市日常(261) / 都市修真(124) / 都市高武(1014) / 历史古代(273) / 战神赘婿(27) / 都市种田(263) / 传统玄幻(258) / 历史脑洞(272) / 悬疑脑洞(539) / 都市脑洞(262) / 玄幻脑洞(257) / 悬疑灵异(751) / 抗战谍战(504) / 游戏体育(746) / 动漫衍生(718) / 男频衍生(1016)
|
|
24
|
-
|
|
25
|
-
女频18个:古风世情(1139) / 科幻末世(8) / 游戏体育(746) / 女频衍生(1015) / 玄幻言情(248) / 种田(23) / 年代(79) / 现言脑洞(267) / 宫斗宅斗(246) / 悬疑脑洞(539) / 古言脑洞(253) / 快穿(24) / 青春甜宠(749) / 星光璀璨(745) / 女频悬疑(747) / 职场婚恋(750) / 豪门总裁(748) / 民国言情(1017)
|
|
26
|
-
|
|
27
|
-
### 输出模板
|
|
28
|
-
|
|
29
|
-
```markdown
|
|
30
|
-
# 番茄 · {频道}{榜单名} · 全 {N} 题材
|
|
31
|
-
- 频道参数:channel={0女频/1男频},type={1新书榜/2阅读榜}
|
|
32
|
-
- 抓取时间:{ISO 8601}
|
|
33
|
-
- 条目数:{N}
|
|
34
|
-
|
|
35
|
-
---
|
|
36
|
-
|
|
37
|
-
## {题材名称}
|
|
38
|
-
|
|
39
|
-
书名:{书名}
|
|
40
|
-
题材:{题材}
|
|
41
|
-
作者:{作者}
|
|
42
|
-
作品页:{URL}
|
|
43
|
-
|
|
44
|
-
书名:{书名}
|
|
45
|
-
题材:{题材}
|
|
46
|
-
作者:{作者}
|
|
47
|
-
作品页:{URL}
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
> 番茄字体反爬:详情页 HTML 里有明文 bookName/author,列表页 innerText 被自定义字体混淆需详情页解码。
|
|
51
|
-
|
|
52
|
-
---
|
|
53
|
-
|
|
54
|
-
## 晋江文学城
|
|
55
|
-
|
|
56
|
-
### 榜单 URL
|
|
57
|
-
|
|
58
|
-
`jjwxc.net/topten.php?orderstr={榜单ID}&t={频道ID}`
|
|
59
|
-
|
|
60
|
-
| 榜单 | orderstr | 页面布局 |
|
|
61
|
-
|------|----------|----------|
|
|
62
|
-
| 月榜 | 5 | 表格型(布局A) |
|
|
63
|
-
| 总分榜 | 7 | 表格型(布局A) |
|
|
64
|
-
| 季度榜 | 4 | 表格型(布局A) |
|
|
65
|
-
| 收入金榜 | 12 | 交替行型(布局B) |
|
|
66
|
-
| 完结金榜 | 16 | 交替行型(布局B) |
|
|
67
|
-
| 勤奋指数榜 | 15 | 表格型(布局A) |
|
|
68
|
-
| 新手金榜 | 17 | 交替行型(布局B) |
|
|
69
|
-
|
|
70
|
-
### 输出模板
|
|
71
|
-
|
|
72
|
-
```markdown
|
|
73
|
-
# 晋江 · {榜单名}
|
|
74
|
-
- 来源:{topten URL}
|
|
75
|
-
- 抓取时间:{ISO 8601}
|
|
76
|
-
- 条目数:{N}
|
|
77
|
-
|
|
78
|
-
---
|
|
79
|
-
|
|
80
|
-
书名:{书名}
|
|
81
|
-
题材:{频道名}
|
|
82
|
-
作者:{作者}
|
|
83
|
-
作品页:https://www.jjwxc.net/onebook.php?novelid={id}
|
|
84
|
-
|
|
85
|
-
书名:{书名}
|
|
86
|
-
题材:{频道名}
|
|
87
|
-
作者:{作者}
|
|
88
|
-
作品页:https://www.jjwxc.net/onebook.php?novelid={id}
|
|
89
|
-
```
|
|
90
|
-
|
|
91
|
-
> 编码:详情页必须 `fetch+arrayBuffer+TextDecoder('gb18030')` 解码。
|
|
92
|
-
|
|
93
|
-
---
|
|
94
|
-
|
|
95
|
-
## 七猫小说
|
|
96
|
-
|
|
97
|
-
### 榜单
|
|
98
|
-
|
|
99
|
-
入口:`https://www.qimao.com/paihang/{boy|girl}/{hot|new|over|collect|update}/{date|month}/`
|
|
100
|
-
|
|
101
|
-
男频大热榜日榜:`https://www.qimao.com/paihang/boy/hot/date/`
|
|
102
|
-
|
|
103
|
-
### 输出模板
|
|
104
|
-
|
|
105
|
-
```markdown
|
|
106
|
-
# 七猫 · {男/女}频 · {榜单名称}
|
|
107
|
-
- 来源:qimao.com/paihang
|
|
108
|
-
- 抓取时间:{ISO 8601}
|
|
109
|
-
- 条目数:{N}
|
|
110
|
-
|
|
111
|
-
---
|
|
112
|
-
|
|
113
|
-
书名:{书名}
|
|
114
|
-
题材:{题材}
|
|
115
|
-
作者:{作者}
|
|
116
|
-
作品页:{URL}
|
|
117
|
-
|
|
118
|
-
书名:{书名}
|
|
119
|
-
题材:{题材}
|
|
120
|
-
作者:{作者}
|
|
121
|
-
作品页:{URL}
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
---
|
|
125
|
-
|
|
126
|
-
## 起点中文网
|
|
127
|
-
|
|
128
|
-
### 榜单
|
|
129
|
-
|
|
130
|
-
见 SKILL.md「起点采集目标」表。优先使用移动端 SSR pageContext JSON,规避 PC 站风控。
|
|
131
|
-
|
|
132
|
-
### 输出模板
|
|
133
|
-
|
|
134
|
-
```markdown
|
|
135
|
-
# 起点 · {榜单名称}
|
|
136
|
-
- 来源:{榜单URL}
|
|
137
|
-
- 抓取时间:{ISO 8601}
|
|
138
|
-
- 条目数:{N}
|
|
139
|
-
|
|
140
|
-
---
|
|
141
|
-
|
|
142
|
-
书名:{书名}
|
|
143
|
-
题材:{题材}
|
|
144
|
-
作者:{作者}
|
|
145
|
-
作品页:{URL}
|
|
146
|
-
|
|
147
|
-
书名:{书名}
|
|
148
|
-
题材:{题材}
|
|
149
|
-
作者:{作者}
|
|
150
|
-
作品页:{URL}
|
|
151
|
-
```
|
|
152
|
-
|
|
153
|
-
---
|
|
154
|
-
|
|
155
|
-
## 知乎
|
|
156
|
-
|
|
157
|
-
### 榜单
|
|
158
|
-
|
|
159
|
-
入口:https://www.zhihu.com/fiore/h5/vip-web
|
|
160
|
-
短篇4榜:推荐榜 / 热搜榜 / 热度榜 / 口碑榜
|
|
161
|
-
长篇1榜:长篇榜
|
|
162
|
-
|
|
163
|
-
### 输出模板
|
|
164
|
-
|
|
165
|
-
```markdown
|
|
166
|
-
# 知乎 · {短篇/长篇} · {榜单名}
|
|
167
|
-
- 来源:https://www.zhihu.com/fiore/h5/vip-web
|
|
168
|
-
- 抓取时间:{ISO 8601}
|
|
169
|
-
- 条目数:{N}
|
|
170
|
-
|
|
171
|
-
---
|
|
172
|
-
|
|
173
|
-
书名:{书名}
|
|
174
|
-
题材:{题材}
|
|
175
|
-
作者:{作者}
|
|
176
|
-
作品页:{URL}
|
|
177
|
-
|
|
178
|
-
书名:{书名}
|
|
179
|
-
题材:{题材}
|
|
180
|
-
作者:{作者}
|
|
181
|
-
作品页:{URL}
|
|
182
|
-
```
|
|
183
|
-
|
|
184
|
-
---
|
|
185
|
-
|
|
186
|
-
## 数据清洗
|
|
187
|
-
|
|
188
|
-
最低采集量:主流平台15条,小平台10条。低于底线标 `[数据稀疏]`。
|