@cloud411716/fancy-webnovel 0.2.24 → 0.2.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json
CHANGED
|
@@ -96,7 +96,9 @@
|
|
|
96
96
|
|
|
97
97
|
### 榜单
|
|
98
98
|
|
|
99
|
-
|
|
99
|
+
入口:`https://www.qimao.com/paihang/{boy|girl}/{hot|new|over|collect|update}/{date|month}/`
|
|
100
|
+
|
|
101
|
+
男频大热榜日榜:`https://www.qimao.com/paihang/boy/hot/date/`
|
|
100
102
|
|
|
101
103
|
### 输出模板
|
|
102
104
|
|
|
@@ -6,12 +6,11 @@
|
|
|
6
6
|
* 采集策略:tab 切换男生榜/女生榜和榜单类型,滚动加载后从页面文本解析结构化数据。
|
|
7
7
|
*
|
|
8
8
|
* 用法:
|
|
9
|
-
* node qimao-rank-scraper.js
|
|
10
|
-
* node qimao-rank-scraper.js --channel male --type hot
|
|
11
|
-
* node qimao-rank-scraper.js --channel
|
|
12
|
-
* node qimao-rank-scraper.js --channel
|
|
13
|
-
* node qimao-rank-scraper.js --
|
|
14
|
-
* node qimao-rank-scraper.js --login-wait 60 # 等待手动登录
|
|
9
|
+
* node qimao-rank-scraper.js # 默认:扫全部(4个榜单)
|
|
10
|
+
* node qimao-rank-scraper.js --channel male --type hot # 男生大热榜(默认日榜)
|
|
11
|
+
* node qimao-rank-scraper.js --channel female --type new # 女生新书榜(默认日榜)
|
|
12
|
+
* node qimao-rank-scraper.js --channel all --type all # 扫全部(含月榜)
|
|
13
|
+
* node qimao-rank-scraper.js --login-wait 60 # 等待手动登录
|
|
15
14
|
*/
|
|
16
15
|
|
|
17
16
|
const fs = require("fs");
|
|
@@ -281,6 +280,7 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
|
|
|
281
280
|
|
|
282
281
|
// 直接从 DOM 提取 /shuku/{id}/ 链接补到每本书上(不依赖 XHR 匹配)
|
|
283
282
|
await page.evaluate(() => {
|
|
283
|
+
if (window.__qimaoUrlMap) return;
|
|
284
284
|
var urlMap = {};
|
|
285
285
|
document.querySelectorAll('a[href*="/shuku/"]').forEach(a => {
|
|
286
286
|
var m = a.href.match(/\/shuku\/(\d+)\//);
|
|
@@ -291,10 +291,10 @@ async function scrapeRank(channelId, rankTypeId, periodId) {
|
|
|
291
291
|
urlMap[title] = 'https://www.qimao.com/shuku/' + id + '/';
|
|
292
292
|
}
|
|
293
293
|
});
|
|
294
|
-
|
|
295
|
-
window.__qimaoUrlMap = urlMap;
|
|
294
|
+
window.__qimaoUrlMap = JSON.stringify(urlMap);
|
|
296
295
|
});
|
|
297
|
-
const
|
|
296
|
+
const rawUrlMap = await page.evaluate(() => window.__qimaoUrlMap || '{}');
|
|
297
|
+
const urlMap = JSON.parse(rawUrlMap);
|
|
298
298
|
const norm = s => (s || '').replace(/\s+/g, '');
|
|
299
299
|
for (const b of books) {
|
|
300
300
|
if (!b.url) {
|
|
@@ -364,7 +364,8 @@ function buildTargets(channel, rankType, period) {
|
|
|
364
364
|
const targets = [];
|
|
365
365
|
for (const channelId of channels) {
|
|
366
366
|
for (const rankTypeId of rankTypes) {
|
|
367
|
-
|
|
367
|
+
// hot 和 new 都用 day period(用户只扫大热榜日榜和新书榜日榜)
|
|
368
|
+
if (rankTypeId === "hot" || rankTypeId === "new") {
|
|
368
369
|
const periods = period === "all" ? PERIODS.map(p => p.id) : [period];
|
|
369
370
|
for (const periodId of periods) {
|
|
370
371
|
targets.push({ channel: channelId, rankType: rankTypeId, period: periodId });
|
|
@@ -390,8 +391,8 @@ function outputFilename(channelId, rankTypeId, periodId, date) {
|
|
|
390
391
|
|
|
391
392
|
const args = process.argv.slice(2);
|
|
392
393
|
const OUTDIR = getArg(args, "--outdir") || ".";
|
|
393
|
-
const CHANNEL = getArg(args, "--channel") || "
|
|
394
|
-
const RANKTYPE = getArg(args, "--type") || "
|
|
394
|
+
const CHANNEL = getArg(args, "--channel") || "all";
|
|
395
|
+
const RANKTYPE = getArg(args, "--type") || "all";
|
|
395
396
|
const PERIOD = getArg(args, "--period") || "day";
|
|
396
397
|
const LOGIN_WAIT = parseInt(getArg(args, "--login-wait") || "0", 10);
|
|
397
398
|
|