@duyquangnvx/webnovel-downloader 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +352 -266
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +21 -2
- package/dist/index.d.ts +21 -2
- package/dist/index.js +344 -265
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -448,7 +448,8 @@ __export(index_exports, {
|
|
|
448
448
|
builtinAdapters: () => builtinAdapters,
|
|
449
449
|
createDownloader: () => createDownloader,
|
|
450
450
|
createLogger: () => createLogger,
|
|
451
|
-
downloader: () => downloader
|
|
451
|
+
downloader: () => downloader,
|
|
452
|
+
parseNovelHeader: () => parseNovelHeader
|
|
452
453
|
});
|
|
453
454
|
module.exports = __toCommonJS(index_exports);
|
|
454
455
|
|
|
@@ -472,6 +473,9 @@ var silentLogger = createLogger({ level: "silent" });
|
|
|
472
473
|
// src/index.ts
|
|
473
474
|
init_errors();
|
|
474
475
|
|
|
476
|
+
// src/core/downloader.ts
|
|
477
|
+
var import_undici2 = require("undici");
|
|
478
|
+
|
|
475
479
|
// src/core/registry.ts
|
|
476
480
|
init_errors();
|
|
477
481
|
var AdapterRegistry = class {
|
|
@@ -550,8 +554,12 @@ init_errors();
|
|
|
550
554
|
var DEFAULT_TIMEOUT_MS = 3e4;
|
|
551
555
|
var UndiciHttpClient = class {
|
|
552
556
|
#logger;
|
|
557
|
+
#dispatcher;
|
|
558
|
+
#ownsDispatcher;
|
|
553
559
|
constructor(opts) {
|
|
554
560
|
this.#logger = opts.logger;
|
|
561
|
+
this.#ownsDispatcher = opts.dispatcher === void 0;
|
|
562
|
+
this.#dispatcher = opts.dispatcher ?? new import_undici.Agent();
|
|
555
563
|
}
|
|
556
564
|
async get(url, opts) {
|
|
557
565
|
const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
@@ -560,6 +568,7 @@ var UndiciHttpClient = class {
|
|
|
560
568
|
try {
|
|
561
569
|
res = await (0, import_undici.request)(url, {
|
|
562
570
|
method: "GET",
|
|
571
|
+
dispatcher: this.#dispatcher,
|
|
563
572
|
...opts?.headers !== void 0 ? { headers: opts.headers } : {},
|
|
564
573
|
...opts?.signal !== void 0 ? { signal: opts.signal } : {},
|
|
565
574
|
headersTimeout: half,
|
|
@@ -584,10 +593,15 @@ var UndiciHttpClient = class {
|
|
|
584
593
|
if (cause instanceof import_undici.errors.HeadersTimeoutError || cause instanceof import_undici.errors.BodyTimeoutError) {
|
|
585
594
|
throw new TimeoutError(url, { cause });
|
|
586
595
|
}
|
|
587
|
-
|
|
588
|
-
|
|
596
|
+
const detail = cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause);
|
|
597
|
+
this.#logger.warn({ url, cause }, "undici network error");
|
|
598
|
+
throw new HttpError(0, url, `Network error (${detail})`, { cause });
|
|
589
599
|
}
|
|
590
600
|
}
|
|
601
|
+
/** Closes the dispatcher this client created. A no-op when one was injected — the caller owns that. */
|
|
602
|
+
async dispose() {
|
|
603
|
+
if (this.#ownsDispatcher) await this.#dispatcher.close();
|
|
604
|
+
}
|
|
591
605
|
};
|
|
592
606
|
function normalizeHeaders(h) {
|
|
593
607
|
const out = {};
|
|
@@ -719,9 +733,13 @@ var RateLimiter = class {
|
|
|
719
733
|
#buckets = /* @__PURE__ */ new Map();
|
|
720
734
|
#pauses = /* @__PURE__ */ new Map();
|
|
721
735
|
constructor(opts = DEFAULT_RATE_LIMIT) {
|
|
736
|
+
const burst = opts.burst ?? Math.max(1, opts.requestsPerSecond);
|
|
737
|
+
if (burst < 1) {
|
|
738
|
+
throw new RangeError(`RateLimiter: burst must be >= 1 (got ${burst})`);
|
|
739
|
+
}
|
|
722
740
|
this.#opts = {
|
|
723
741
|
requestsPerSecond: opts.requestsPerSecond,
|
|
724
|
-
burst
|
|
742
|
+
burst
|
|
725
743
|
};
|
|
726
744
|
}
|
|
727
745
|
async acquire(host, signal) {
|
|
@@ -881,6 +899,31 @@ function clamp(n, lo, hi) {
|
|
|
881
899
|
// src/http/index.ts
|
|
882
900
|
init_cookie_jar();
|
|
883
901
|
|
|
902
|
+
// src/http/stack-builder.ts
|
|
903
|
+
var HttpStack = class _HttpStack {
|
|
904
|
+
#client;
|
|
905
|
+
constructor(client) {
|
|
906
|
+
this.#client = client;
|
|
907
|
+
}
|
|
908
|
+
static from(leaf) {
|
|
909
|
+
return new _HttpStack(leaf);
|
|
910
|
+
}
|
|
911
|
+
rotateUserAgent(uas, defaultHeaders, cookieJar) {
|
|
912
|
+
return new _HttpStack(
|
|
913
|
+
new UserAgentRotatingHttpClient(this.#client, uas, defaultHeaders, cookieJar)
|
|
914
|
+
);
|
|
915
|
+
}
|
|
916
|
+
rateLimit(limiter, onEvent) {
|
|
917
|
+
return new _HttpStack(new RateLimitedHttpClient(this.#client, limiter, onEvent));
|
|
918
|
+
}
|
|
919
|
+
retry(opts, logger, limiter, onEvent) {
|
|
920
|
+
return new _HttpStack(new RetryingHttpClient(this.#client, opts, logger, limiter, onEvent));
|
|
921
|
+
}
|
|
922
|
+
build() {
|
|
923
|
+
return this.#client;
|
|
924
|
+
}
|
|
925
|
+
};
|
|
926
|
+
|
|
884
927
|
// src/http/tiered.ts
|
|
885
928
|
init_cf_challenge();
|
|
886
929
|
var FALLBACK_UA = DEFAULT_UA_POOL[0] ?? "Mozilla/5.0";
|
|
@@ -941,22 +984,12 @@ function normalizeTransport(t) {
|
|
|
941
984
|
function buildHttpClient(opts = {}) {
|
|
942
985
|
const logger = opts.logger ?? silentLogger;
|
|
943
986
|
const cookieJar = opts.cookieJar ?? new CookieJar();
|
|
944
|
-
const leaf = opts.undiciOverride ?? new UndiciHttpClient({
|
|
945
|
-
const ua = new UserAgentRotatingHttpClient(
|
|
946
|
-
leaf,
|
|
947
|
-
opts.userAgents ?? new UserAgents(),
|
|
948
|
-
opts.defaultHeaders,
|
|
949
|
-
cookieJar
|
|
950
|
-
);
|
|
951
|
-
const limiter = opts.rateLimiter ?? new RateLimiter(opts.rateLimit ?? DEFAULT_RATE_LIMIT);
|
|
952
|
-
const limited = new RateLimitedHttpClient(ua, limiter, opts.onEvent);
|
|
953
|
-
const httpBranch = new RetryingHttpClient(
|
|
954
|
-
limited,
|
|
955
|
-
normalizeRetry(opts.retry),
|
|
987
|
+
const leaf = opts.undiciOverride ?? new UndiciHttpClient({
|
|
956
988
|
logger,
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
);
|
|
989
|
+
...opts.dispatcher !== void 0 ? { dispatcher: opts.dispatcher } : {}
|
|
990
|
+
});
|
|
991
|
+
const limiter = opts.rateLimiter ?? new RateLimiter(opts.rateLimit ?? DEFAULT_RATE_LIMIT);
|
|
992
|
+
const httpBranch = HttpStack.from(leaf).rotateUserAgent(opts.userAgents ?? new UserAgents(), opts.defaultHeaders, cookieJar).rateLimit(limiter, opts.onEvent).retry(normalizeRetry(opts.retry), logger, limiter, opts.onEvent).build();
|
|
960
993
|
const { mode } = normalizeTransport(opts.transport);
|
|
961
994
|
if (mode === "http-only") {
|
|
962
995
|
return httpBranch;
|
|
@@ -1411,7 +1444,6 @@ function toResponse(entry) {
|
|
|
1411
1444
|
|
|
1412
1445
|
// src/core/download-run.ts
|
|
1413
1446
|
var import_p_queue = __toESM(require("p-queue"), 1);
|
|
1414
|
-
var import_node_path6 = require("path");
|
|
1415
1447
|
|
|
1416
1448
|
// src/core/debouncer.ts
|
|
1417
1449
|
var Debouncer = class {
|
|
@@ -1474,30 +1506,47 @@ init_primitives();
|
|
|
1474
1506
|
|
|
1475
1507
|
// src/pipeline/fetch-toc.ts
|
|
1476
1508
|
init_errors();
|
|
1509
|
+
|
|
1510
|
+
// src/pipeline/walk-toc.ts
|
|
1511
|
+
init_errors();
|
|
1477
1512
|
var TOC_PROGRESS_EMIT_EVERY = 50;
|
|
1478
|
-
async function
|
|
1513
|
+
async function walkToc(source, opts) {
|
|
1514
|
+
const from = opts.range?.from;
|
|
1515
|
+
const to = opts.range?.to;
|
|
1479
1516
|
let count = 0;
|
|
1480
|
-
|
|
1481
|
-
const
|
|
1482
|
-
|
|
1483
|
-
for await (const raw of adapter.fetchChapterList(url, ctx)) {
|
|
1484
|
-
if (ctx.signal.aborted) throw new CancelledError({ cause: ctx.signal.reason });
|
|
1485
|
-
const parsed = ChapterRefSchema.safeParse(raw);
|
|
1486
|
-
if (!parsed.success) {
|
|
1487
|
-
throw new ParseError(`Invalid ChapterRef from adapter`, { cause: parsed.error, url });
|
|
1488
|
-
}
|
|
1489
|
-
const ref = parsed.data;
|
|
1517
|
+
if (opts.signal.aborted) throw new CancelledError({ cause: opts.signal.reason });
|
|
1518
|
+
for await (const ref of source) {
|
|
1519
|
+
if (opts.signal.aborted) throw new CancelledError({ cause: opts.signal.reason });
|
|
1490
1520
|
if (to !== void 0 && ref.index > to) break;
|
|
1491
1521
|
if (from !== void 0 && ref.index < from) continue;
|
|
1492
|
-
await onRef(ref);
|
|
1522
|
+
await opts.onRef(ref);
|
|
1493
1523
|
count++;
|
|
1494
|
-
if (count % TOC_PROGRESS_EMIT_EVERY === 0) bus.emit({ type: "toc:progress", discovered: count });
|
|
1524
|
+
if (count % TOC_PROGRESS_EMIT_EVERY === 0) opts.bus.emit({ type: "toc:progress", discovered: count });
|
|
1495
1525
|
}
|
|
1496
|
-
bus.emit({ type: "toc:progress", discovered: count });
|
|
1497
|
-
bus.emit({ type: "toc:complete", total: count });
|
|
1526
|
+
opts.bus.emit({ type: "toc:progress", discovered: count });
|
|
1527
|
+
opts.bus.emit({ type: "toc:complete", total: count });
|
|
1498
1528
|
return count;
|
|
1499
1529
|
}
|
|
1500
1530
|
|
|
1531
|
+
// src/pipeline/fetch-toc.ts
|
|
1532
|
+
async function fetchTocStage(adapter, url, ctx, bus, onRef, range) {
|
|
1533
|
+
async function* validated() {
|
|
1534
|
+
for await (const raw of adapter.fetchChapterList(url, ctx)) {
|
|
1535
|
+
const parsed = ChapterRefSchema.safeParse(raw);
|
|
1536
|
+
if (!parsed.success) {
|
|
1537
|
+
throw new ParseError(`Invalid ChapterRef from adapter`, { cause: parsed.error, url });
|
|
1538
|
+
}
|
|
1539
|
+
yield parsed.data;
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
return walkToc(validated(), {
|
|
1543
|
+
...range !== void 0 ? { range } : {},
|
|
1544
|
+
bus,
|
|
1545
|
+
signal: ctx.signal,
|
|
1546
|
+
onRef
|
|
1547
|
+
});
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1501
1550
|
// src/pipeline/fetch-chapters.ts
|
|
1502
1551
|
init_errors();
|
|
1503
1552
|
async function fetchChapterTask(adapter, ref, ctx, bus) {
|
|
@@ -1525,130 +1574,15 @@ function assembleNovelData(metadata, results) {
|
|
|
1525
1574
|
// src/core/download-run.ts
|
|
1526
1575
|
init_errors();
|
|
1527
1576
|
|
|
1528
|
-
// src/storage/resume-
|
|
1529
|
-
var import_promises3 = require("fs/promises");
|
|
1577
|
+
// src/storage/resume-token.ts
|
|
1530
1578
|
var import_node_path3 = require("path");
|
|
1531
|
-
init_errors();
|
|
1532
|
-
|
|
1533
|
-
// src/storage/resume-state.ts
|
|
1534
1579
|
var import_zod4 = require("zod");
|
|
1535
|
-
init_primitives();
|
|
1536
|
-
var ResumeFailureRecordSchema = import_zod4.z.object({
|
|
1537
|
-
index: ChapterIndexSchema,
|
|
1538
|
-
lastError: import_zod4.z.object({ code: import_zod4.z.string(), message: import_zod4.z.string() }).readonly()
|
|
1539
|
-
}).readonly();
|
|
1540
|
-
var ResumeStateSchema = import_zod4.z.object({
|
|
1541
|
-
schemaVersion: import_zod4.z.literal(1),
|
|
1542
|
-
url: UrlSchema,
|
|
1543
|
-
adapterId: AdapterIdSchema,
|
|
1544
|
-
metadata: NovelMetadataSchema.nullable(),
|
|
1545
|
-
toc: import_zod4.z.array(ChapterRefSchema).readonly().nullable(),
|
|
1546
|
-
completed: import_zod4.z.array(ChapterIndexSchema).readonly(),
|
|
1547
|
-
failed: import_zod4.z.array(ResumeFailureRecordSchema).readonly(),
|
|
1548
|
-
startedAt: IsoDateSchema,
|
|
1549
|
-
updatedAt: IsoDateSchema
|
|
1550
|
-
}).readonly();
|
|
1551
|
-
|
|
1552
|
-
// src/storage/resume-store.ts
|
|
1553
|
-
var FilesystemResumeStore = class {
|
|
1554
|
-
async load(path) {
|
|
1555
|
-
let text;
|
|
1556
|
-
try {
|
|
1557
|
-
text = await (0, import_promises3.readFile)(path, "utf8");
|
|
1558
|
-
} catch (err) {
|
|
1559
|
-
if (isEnoent(err)) return null;
|
|
1560
|
-
throw err;
|
|
1561
|
-
}
|
|
1562
|
-
let raw;
|
|
1563
|
-
try {
|
|
1564
|
-
raw = JSON.parse(text);
|
|
1565
|
-
} catch (cause) {
|
|
1566
|
-
throw new ParseError("Resume state file is corrupt; delete it to start over", { cause, path });
|
|
1567
|
-
}
|
|
1568
|
-
const parsed = ResumeStateSchema.safeParse(raw);
|
|
1569
|
-
if (!parsed.success) {
|
|
1570
|
-
throw new ParseError("Resume state file is corrupt; delete it to start over", {
|
|
1571
|
-
cause: parsed.error,
|
|
1572
|
-
path
|
|
1573
|
-
});
|
|
1574
|
-
}
|
|
1575
|
-
return parsed.data;
|
|
1576
|
-
}
|
|
1577
|
-
async save(path, state) {
|
|
1578
|
-
await atomicWriteJson(path, state);
|
|
1579
|
-
}
|
|
1580
|
-
async delete(path) {
|
|
1581
|
-
try {
|
|
1582
|
-
await (0, import_promises3.unlink)(path);
|
|
1583
|
-
} catch (err) {
|
|
1584
|
-
if (!isEnoent(err)) throw err;
|
|
1585
|
-
}
|
|
1586
|
-
}
|
|
1587
|
-
};
|
|
1588
|
-
|
|
1589
|
-
// src/storage/chapter-store.ts
|
|
1590
|
-
var import_promises4 = require("fs/promises");
|
|
1591
|
-
var import_node_path4 = require("path");
|
|
1592
|
-
var FilesystemChapterStore = class {
|
|
1593
|
-
#logger;
|
|
1594
|
-
#dir;
|
|
1595
|
-
constructor(dir, opts = {}) {
|
|
1596
|
-
this.#dir = (0, import_node_path4.resolve)(dir);
|
|
1597
|
-
this.#logger = opts.logger ?? silentLogger;
|
|
1598
|
-
}
|
|
1599
|
-
async save(chapter) {
|
|
1600
|
-
await atomicWriteJson((0, import_node_path4.join)(this.#dir, `${Number(chapter.index)}.json`), chapter);
|
|
1601
|
-
}
|
|
1602
|
-
async loadAll() {
|
|
1603
|
-
let entries;
|
|
1604
|
-
try {
|
|
1605
|
-
entries = await (0, import_promises4.readdir)(this.#dir);
|
|
1606
|
-
} catch (err) {
|
|
1607
|
-
if (isEnoent(err)) return [];
|
|
1608
|
-
throw err;
|
|
1609
|
-
}
|
|
1610
|
-
const chapters = [];
|
|
1611
|
-
for (const name of entries) {
|
|
1612
|
-
if (!/^\d+\.json$/.test(name)) continue;
|
|
1613
|
-
const path = (0, import_node_path4.join)(this.#dir, name);
|
|
1614
|
-
let text;
|
|
1615
|
-
try {
|
|
1616
|
-
text = await (0, import_promises4.readFile)(path, "utf8");
|
|
1617
|
-
} catch (err) {
|
|
1618
|
-
this.#logger.warn({ path, err }, "chapter-store: failed to read file, skipping");
|
|
1619
|
-
continue;
|
|
1620
|
-
}
|
|
1621
|
-
let raw;
|
|
1622
|
-
try {
|
|
1623
|
-
raw = JSON.parse(text);
|
|
1624
|
-
} catch (err) {
|
|
1625
|
-
this.#logger.warn({ path, err }, "chapter-store: invalid JSON, skipping");
|
|
1626
|
-
continue;
|
|
1627
|
-
}
|
|
1628
|
-
const parsed = ChapterSchema.safeParse(raw);
|
|
1629
|
-
if (!parsed.success) {
|
|
1630
|
-
this.#logger.warn(
|
|
1631
|
-
{ path, issues: parsed.error.issues },
|
|
1632
|
-
"chapter-store: schema mismatch, skipping"
|
|
1633
|
-
);
|
|
1634
|
-
continue;
|
|
1635
|
-
}
|
|
1636
|
-
chapters.push(parsed.data);
|
|
1637
|
-
}
|
|
1638
|
-
chapters.sort((a, b) => Number(a.index) - Number(b.index));
|
|
1639
|
-
return chapters;
|
|
1640
|
-
}
|
|
1641
|
-
};
|
|
1642
|
-
|
|
1643
|
-
// src/storage/resume-token.ts
|
|
1644
|
-
var import_node_path5 = require("path");
|
|
1645
|
-
var import_zod5 = require("zod");
|
|
1646
1580
|
init_errors();
|
|
1647
1581
|
init_primitives();
|
|
1648
|
-
var TokenPayloadSchema =
|
|
1649
|
-
v:
|
|
1650
|
-
stateFile:
|
|
1651
|
-
urlHash:
|
|
1582
|
+
var TokenPayloadSchema = import_zod4.z.object({
|
|
1583
|
+
v: import_zod4.z.literal(1),
|
|
1584
|
+
stateFile: import_zod4.z.string().min(1),
|
|
1585
|
+
urlHash: import_zod4.z.string().min(1)
|
|
1652
1586
|
});
|
|
1653
1587
|
function encodeResumeToken(stateFile, url) {
|
|
1654
1588
|
const payload = { v: 1, stateFile, urlHash: sha1(url) };
|
|
@@ -1669,9 +1603,9 @@ function decodeResumeToken(token) {
|
|
|
1669
1603
|
return { stateFile: parsed.data.stateFile, urlHash: parsed.data.urlHash };
|
|
1670
1604
|
}
|
|
1671
1605
|
function assertWithinCacheDir(stateFile) {
|
|
1672
|
-
const root = (0,
|
|
1673
|
-
const rel = (0,
|
|
1674
|
-
if (rel === "" || rel.startsWith("..") || (0,
|
|
1606
|
+
const root = (0, import_node_path3.resolve)((0, import_node_path3.join)(userCacheDir(), "webnovel-downloader"));
|
|
1607
|
+
const rel = (0, import_node_path3.relative)(root, (0, import_node_path3.resolve)(stateFile));
|
|
1608
|
+
if (rel === "" || rel.startsWith("..") || (0, import_node_path3.isAbsolute)(rel)) {
|
|
1675
1609
|
throw new ParseError("Resume token state path escapes the cache directory");
|
|
1676
1610
|
}
|
|
1677
1611
|
}
|
|
@@ -1687,25 +1621,6 @@ function verifyResumeToken(token, url) {
|
|
|
1687
1621
|
// src/core/download-run.ts
|
|
1688
1622
|
var QUEUE_BACKPRESSURE_RATIO = 4;
|
|
1689
1623
|
var STATE_PERSIST_DEBOUNCE_MS = 500;
|
|
1690
|
-
var TOC_PROGRESS_EMIT_EVERY2 = 50;
|
|
1691
|
-
function resolveResumeConfig(opt, url, logger) {
|
|
1692
|
-
if (opt === void 0) return null;
|
|
1693
|
-
const make = (stateFile) => ({
|
|
1694
|
-
stateFile,
|
|
1695
|
-
store: new FilesystemResumeStore(),
|
|
1696
|
-
chapterStore: new FilesystemChapterStore(`${stateFile}.chapters`, { logger })
|
|
1697
|
-
});
|
|
1698
|
-
if (opt === true) {
|
|
1699
|
-
const stateFile = (0, import_node_path6.join)(userCacheDir(), "webnovel-downloader", "state", `${sha1(url)}.json`);
|
|
1700
|
-
return make(stateFile);
|
|
1701
|
-
}
|
|
1702
|
-
if ("stateFile" in opt) return make(opt.stateFile);
|
|
1703
|
-
if ("token" in opt) {
|
|
1704
|
-
const { stateFile } = verifyResumeToken(opt.token, url);
|
|
1705
|
-
return make(stateFile);
|
|
1706
|
-
}
|
|
1707
|
-
return null;
|
|
1708
|
-
}
|
|
1709
1624
|
function freshState(url, adapterId) {
|
|
1710
1625
|
const now = nowIso();
|
|
1711
1626
|
return {
|
|
@@ -1727,14 +1642,13 @@ var DownloadRun = class {
|
|
|
1727
1642
|
#adapter;
|
|
1728
1643
|
#validated;
|
|
1729
1644
|
#normalizedRange;
|
|
1730
|
-
#
|
|
1645
|
+
#resumeStorage;
|
|
1731
1646
|
#concurrency;
|
|
1732
1647
|
#signal;
|
|
1733
1648
|
#ctx;
|
|
1734
1649
|
#bus;
|
|
1735
1650
|
#logger;
|
|
1736
1651
|
#state = null;
|
|
1737
|
-
#metadata;
|
|
1738
1652
|
#successes = [];
|
|
1739
1653
|
#failures = [];
|
|
1740
1654
|
#completedSet = /* @__PURE__ */ new Set();
|
|
@@ -1748,7 +1662,7 @@ var DownloadRun = class {
|
|
|
1748
1662
|
this.#adapter = args.adapter;
|
|
1749
1663
|
this.#validated = args.validated;
|
|
1750
1664
|
this.#normalizedRange = args.normalizedRange;
|
|
1751
|
-
this.#
|
|
1665
|
+
this.#resumeStorage = args.resumeStorage;
|
|
1752
1666
|
this.#concurrency = args.concurrency;
|
|
1753
1667
|
this.#signal = args.signal;
|
|
1754
1668
|
this.#ctx = args.ctx;
|
|
@@ -1764,9 +1678,10 @@ var DownloadRun = class {
|
|
|
1764
1678
|
async execute() {
|
|
1765
1679
|
const initErr = await this.#initResume();
|
|
1766
1680
|
if (initErr) return initErr;
|
|
1767
|
-
const
|
|
1768
|
-
if (
|
|
1769
|
-
const
|
|
1681
|
+
const resolved = await this.#resolveMetadata();
|
|
1682
|
+
if ("error" in resolved) return resolved.error;
|
|
1683
|
+
const meta = resolved.meta;
|
|
1684
|
+
const ctxWithMeta = this.#startQueue(meta);
|
|
1770
1685
|
const onAbort = () => {
|
|
1771
1686
|
this.#aborted = true;
|
|
1772
1687
|
this.#queue.clear();
|
|
@@ -1776,16 +1691,16 @@ var DownloadRun = class {
|
|
|
1776
1691
|
const stage2Error = await this.#runTocStage(ctxWithMeta);
|
|
1777
1692
|
await this.#drainQueueOrAbort();
|
|
1778
1693
|
await this.#flushQuietly();
|
|
1779
|
-
return this.#finalize(stage2Error);
|
|
1694
|
+
return this.#finalize(meta, stage2Error);
|
|
1780
1695
|
} finally {
|
|
1781
1696
|
this.#signal.removeEventListener("abort", onAbort);
|
|
1782
1697
|
}
|
|
1783
1698
|
}
|
|
1784
1699
|
async #initResume() {
|
|
1785
|
-
const resumeCfg = this.#
|
|
1700
|
+
const resumeCfg = this.#resumeStorage;
|
|
1786
1701
|
if (resumeCfg) {
|
|
1787
1702
|
try {
|
|
1788
|
-
let state = await resumeCfg.
|
|
1703
|
+
let state = await resumeCfg.resume.load(resumeCfg.stateFile);
|
|
1789
1704
|
if (state) {
|
|
1790
1705
|
if (state.url !== this.#validated) {
|
|
1791
1706
|
throw new ParseError("Stale resume state: URL mismatch");
|
|
@@ -1802,7 +1717,7 @@ var DownloadRun = class {
|
|
|
1802
1717
|
}
|
|
1803
1718
|
}
|
|
1804
1719
|
if (resumeCfg && this.#state) {
|
|
1805
|
-
const prior = await resumeCfg.
|
|
1720
|
+
const prior = await resumeCfg.chapters.loadAll();
|
|
1806
1721
|
this.#successes.push(...prior);
|
|
1807
1722
|
const merged = new Set(this.#state.completed);
|
|
1808
1723
|
for (const c of prior) merged.add(c.index);
|
|
@@ -1814,32 +1729,30 @@ var DownloadRun = class {
|
|
|
1814
1729
|
return null;
|
|
1815
1730
|
}
|
|
1816
1731
|
async #persistState() {
|
|
1817
|
-
const resumeCfg = this.#
|
|
1732
|
+
const resumeCfg = this.#resumeStorage;
|
|
1818
1733
|
if (!resumeCfg || !this.#state) return;
|
|
1819
1734
|
const next = { ...this.#state, updatedAt: nowIso() };
|
|
1820
1735
|
this.#state = next;
|
|
1821
|
-
await resumeCfg.
|
|
1736
|
+
await resumeCfg.resume.save(resumeCfg.stateFile, next);
|
|
1822
1737
|
}
|
|
1823
1738
|
async #resolveMetadata() {
|
|
1824
1739
|
try {
|
|
1740
|
+
let meta;
|
|
1825
1741
|
if (this.#state?.metadata) {
|
|
1826
|
-
|
|
1742
|
+
meta = this.#state.metadata;
|
|
1827
1743
|
} else {
|
|
1828
|
-
|
|
1744
|
+
meta = await fetchMetadataStage(this.#adapter, this.#validated, this.#ctx, this.#bus);
|
|
1829
1745
|
if (this.#state) {
|
|
1830
|
-
this.#state = { ...this.#state, metadata:
|
|
1746
|
+
this.#state = { ...this.#state, metadata: meta, updatedAt: nowIso() };
|
|
1831
1747
|
await this.#persistState();
|
|
1832
1748
|
}
|
|
1833
1749
|
}
|
|
1750
|
+
return { meta };
|
|
1834
1751
|
} catch (err) {
|
|
1835
|
-
return {
|
|
1836
|
-
status: "error",
|
|
1837
|
-
error: normalizeToEnvelopeError(err)
|
|
1838
|
-
};
|
|
1752
|
+
return { error: { status: "error", error: normalizeToEnvelopeError(err) } };
|
|
1839
1753
|
}
|
|
1840
|
-
return null;
|
|
1841
1754
|
}
|
|
1842
|
-
#startQueue() {
|
|
1755
|
+
#startQueue(meta) {
|
|
1843
1756
|
const state = this.#state;
|
|
1844
1757
|
const completedSet = this.#completedSet;
|
|
1845
1758
|
const completedList = this.#completedList;
|
|
@@ -1851,8 +1764,7 @@ var DownloadRun = class {
|
|
|
1851
1764
|
if (!completedSet.has(f.index)) failedMap.set(f.index, f);
|
|
1852
1765
|
}
|
|
1853
1766
|
}
|
|
1854
|
-
|
|
1855
|
-
return { ...this.#ctx, novelMetadata: this.#metadata };
|
|
1767
|
+
return { ...this.#ctx, novelMetadata: meta };
|
|
1856
1768
|
}
|
|
1857
1769
|
async #enqueueRef(ref, ctxWithMeta) {
|
|
1858
1770
|
if (this.#aborted || this.#signal.aborted) return;
|
|
@@ -1872,7 +1784,7 @@ var DownloadRun = class {
|
|
|
1872
1784
|
try {
|
|
1873
1785
|
const chapter = await fetchChapterTask(this.#adapter, ref, ctxWithMeta, this.#bus);
|
|
1874
1786
|
if (this.#aborted) return;
|
|
1875
|
-
if (this.#
|
|
1787
|
+
if (this.#resumeStorage) await this.#resumeStorage.chapters.save(chapter);
|
|
1876
1788
|
this.#successes.push(chapter);
|
|
1877
1789
|
this.#completedList.push(ref.index);
|
|
1878
1790
|
this.#completedSet.add(ref.index);
|
|
@@ -1889,7 +1801,7 @@ var DownloadRun = class {
|
|
|
1889
1801
|
this.#bus.emit({
|
|
1890
1802
|
type: "progress",
|
|
1891
1803
|
completed: this.#successes.length,
|
|
1892
|
-
total:
|
|
1804
|
+
total: ctxWithMeta.novelMetadata?.totalChapters ?? this.#enqueuedSet.size
|
|
1893
1805
|
});
|
|
1894
1806
|
} catch (cause) {
|
|
1895
1807
|
if (cause instanceof CancelledError) return;
|
|
@@ -1910,20 +1822,16 @@ var DownloadRun = class {
|
|
|
1910
1822
|
let stage2Error;
|
|
1911
1823
|
try {
|
|
1912
1824
|
if (this.#state?.toc) {
|
|
1913
|
-
const
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
for (const ref of this.#state.toc) {
|
|
1917
|
-
if (to !== void 0 && ref.index > to) break;
|
|
1918
|
-
if (from !== void 0 && ref.index < from) continue;
|
|
1919
|
-
await this.#enqueueRef(ref, ctxWithMeta);
|
|
1920
|
-
discovered++;
|
|
1921
|
-
if (discovered % TOC_PROGRESS_EMIT_EVERY2 === 0) {
|
|
1922
|
-
this.#bus.emit({ type: "toc:progress", discovered });
|
|
1923
|
-
}
|
|
1825
|
+
const toc = this.#state.toc;
|
|
1826
|
+
async function* fromState() {
|
|
1827
|
+
for (const ref of toc) yield ref;
|
|
1924
1828
|
}
|
|
1925
|
-
|
|
1926
|
-
|
|
1829
|
+
await walkToc(fromState(), {
|
|
1830
|
+
...this.#normalizedRange !== void 0 ? { range: this.#normalizedRange } : {},
|
|
1831
|
+
bus: this.#bus,
|
|
1832
|
+
signal: this.#signal,
|
|
1833
|
+
onRef: (ref) => this.#enqueueRef(ref, ctxWithMeta)
|
|
1834
|
+
});
|
|
1927
1835
|
} else {
|
|
1928
1836
|
const tocBuffer = [];
|
|
1929
1837
|
await fetchTocStage(
|
|
@@ -1998,10 +1906,8 @@ var DownloadRun = class {
|
|
|
1998
1906
|
this.#logger.warn({ err, url: this.#validated }, "resume state persist failed (non-fatal)");
|
|
1999
1907
|
}
|
|
2000
1908
|
}
|
|
2001
|
-
#finalize(stage2Error) {
|
|
1909
|
+
#finalize(metadata, stage2Error) {
|
|
2002
1910
|
if (stage2Error) return { status: "error", error: stage2Error };
|
|
2003
|
-
if (!this.#metadata) throw new Error("invariant: metadata must be set before #finalize");
|
|
2004
|
-
const metadata = this.#metadata;
|
|
2005
1911
|
let data;
|
|
2006
1912
|
try {
|
|
2007
1913
|
data = assembleNovelData(metadata, this.#successes);
|
|
@@ -2012,8 +1918,8 @@ var DownloadRun = class {
|
|
|
2012
1918
|
};
|
|
2013
1919
|
}
|
|
2014
1920
|
if (this.#failures.length > 0) {
|
|
2015
|
-
if (this.#
|
|
2016
|
-
const token = encodeResumeToken(this.#
|
|
1921
|
+
if (this.#resumeStorage) {
|
|
1922
|
+
const token = encodeResumeToken(this.#resumeStorage.stateFile, this.#validated);
|
|
2017
1923
|
return { status: "partial", resumable: true, data, failures: this.#failures, resumeToken: token };
|
|
2018
1924
|
}
|
|
2019
1925
|
return { status: "partial", resumable: false, data, failures: this.#failures };
|
|
@@ -2029,6 +1935,185 @@ function normalizeChapterRange(range) {
|
|
|
2029
1935
|
return out;
|
|
2030
1936
|
}
|
|
2031
1937
|
|
|
1938
|
+
// src/storage/resume-storage.ts
|
|
1939
|
+
var import_node_path6 = require("path");
|
|
1940
|
+
|
|
1941
|
+
// src/storage/resume-store.ts
|
|
1942
|
+
var import_promises3 = require("fs/promises");
|
|
1943
|
+
var import_node_path4 = require("path");
|
|
1944
|
+
init_errors();
|
|
1945
|
+
|
|
1946
|
+
// src/storage/resume-state.ts
|
|
1947
|
+
var import_zod5 = require("zod");
|
|
1948
|
+
init_primitives();
|
|
1949
|
+
var ResumeFailureRecordSchema = import_zod5.z.object({
|
|
1950
|
+
index: ChapterIndexSchema,
|
|
1951
|
+
lastError: import_zod5.z.object({ code: import_zod5.z.string(), message: import_zod5.z.string() }).readonly()
|
|
1952
|
+
}).readonly();
|
|
1953
|
+
var ResumeStateSchema = import_zod5.z.object({
|
|
1954
|
+
schemaVersion: import_zod5.z.literal(1),
|
|
1955
|
+
url: UrlSchema,
|
|
1956
|
+
adapterId: AdapterIdSchema,
|
|
1957
|
+
metadata: NovelMetadataSchema.nullable(),
|
|
1958
|
+
toc: import_zod5.z.array(ChapterRefSchema).readonly().nullable(),
|
|
1959
|
+
completed: import_zod5.z.array(ChapterIndexSchema).readonly(),
|
|
1960
|
+
failed: import_zod5.z.array(ResumeFailureRecordSchema).readonly(),
|
|
1961
|
+
startedAt: IsoDateSchema,
|
|
1962
|
+
updatedAt: IsoDateSchema
|
|
1963
|
+
}).readonly();
|
|
1964
|
+
|
|
1965
|
+
// src/storage/resume-store.ts
|
|
1966
|
+
var MemoryResumeStore = class {
|
|
1967
|
+
#map = /* @__PURE__ */ new Map();
|
|
1968
|
+
async load(path) {
|
|
1969
|
+
return this.#map.get((0, import_node_path4.resolve)(path)) ?? null;
|
|
1970
|
+
}
|
|
1971
|
+
async save(path, state) {
|
|
1972
|
+
this.#map.set((0, import_node_path4.resolve)(path), state);
|
|
1973
|
+
}
|
|
1974
|
+
async delete(path) {
|
|
1975
|
+
this.#map.delete((0, import_node_path4.resolve)(path));
|
|
1976
|
+
}
|
|
1977
|
+
};
|
|
1978
|
+
var FilesystemResumeStore = class {
|
|
1979
|
+
async load(path) {
|
|
1980
|
+
let text;
|
|
1981
|
+
try {
|
|
1982
|
+
text = await (0, import_promises3.readFile)(path, "utf8");
|
|
1983
|
+
} catch (err) {
|
|
1984
|
+
if (isEnoent(err)) return null;
|
|
1985
|
+
throw err;
|
|
1986
|
+
}
|
|
1987
|
+
let raw;
|
|
1988
|
+
try {
|
|
1989
|
+
raw = JSON.parse(text);
|
|
1990
|
+
} catch (cause) {
|
|
1991
|
+
throw new ParseError("Resume state file is corrupt; delete it to start over", { cause, path });
|
|
1992
|
+
}
|
|
1993
|
+
const parsed = ResumeStateSchema.safeParse(raw);
|
|
1994
|
+
if (!parsed.success) {
|
|
1995
|
+
throw new ParseError("Resume state file is corrupt; delete it to start over", {
|
|
1996
|
+
cause: parsed.error,
|
|
1997
|
+
path
|
|
1998
|
+
});
|
|
1999
|
+
}
|
|
2000
|
+
return parsed.data;
|
|
2001
|
+
}
|
|
2002
|
+
async save(path, state) {
|
|
2003
|
+
await atomicWriteJson(path, state);
|
|
2004
|
+
}
|
|
2005
|
+
async delete(path) {
|
|
2006
|
+
try {
|
|
2007
|
+
await (0, import_promises3.unlink)(path);
|
|
2008
|
+
} catch (err) {
|
|
2009
|
+
if (!isEnoent(err)) throw err;
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
};
|
|
2013
|
+
|
|
2014
|
+
// src/storage/chapter-store.ts
|
|
2015
|
+
var import_promises4 = require("fs/promises");
|
|
2016
|
+
var import_node_path5 = require("path");
|
|
2017
|
+
var MemoryChapterStore = class {
|
|
2018
|
+
#map = /* @__PURE__ */ new Map();
|
|
2019
|
+
async save(chapter) {
|
|
2020
|
+
this.#map.set(Number(chapter.index), chapter);
|
|
2021
|
+
}
|
|
2022
|
+
async loadAll() {
|
|
2023
|
+
return [...this.#map.values()].sort((a, b) => Number(a.index) - Number(b.index));
|
|
2024
|
+
}
|
|
2025
|
+
};
|
|
2026
|
+
var FilesystemChapterStore = class {
|
|
2027
|
+
#logger;
|
|
2028
|
+
#dir;
|
|
2029
|
+
constructor(dir, opts = {}) {
|
|
2030
|
+
this.#dir = (0, import_node_path5.resolve)(dir);
|
|
2031
|
+
this.#logger = opts.logger ?? silentLogger;
|
|
2032
|
+
}
|
|
2033
|
+
async save(chapter) {
|
|
2034
|
+
await atomicWriteJson((0, import_node_path5.join)(this.#dir, `${Number(chapter.index)}.json`), chapter);
|
|
2035
|
+
}
|
|
2036
|
+
async loadAll() {
|
|
2037
|
+
let entries;
|
|
2038
|
+
try {
|
|
2039
|
+
entries = await (0, import_promises4.readdir)(this.#dir);
|
|
2040
|
+
} catch (err) {
|
|
2041
|
+
if (isEnoent(err)) return [];
|
|
2042
|
+
throw err;
|
|
2043
|
+
}
|
|
2044
|
+
const chapters = [];
|
|
2045
|
+
for (const name of entries) {
|
|
2046
|
+
if (!/^\d+\.json$/.test(name)) continue;
|
|
2047
|
+
const path = (0, import_node_path5.join)(this.#dir, name);
|
|
2048
|
+
let text;
|
|
2049
|
+
try {
|
|
2050
|
+
text = await (0, import_promises4.readFile)(path, "utf8");
|
|
2051
|
+
} catch (err) {
|
|
2052
|
+
this.#logger.warn({ path, err }, "chapter-store: failed to read file, skipping");
|
|
2053
|
+
continue;
|
|
2054
|
+
}
|
|
2055
|
+
let raw;
|
|
2056
|
+
try {
|
|
2057
|
+
raw = JSON.parse(text);
|
|
2058
|
+
} catch (err) {
|
|
2059
|
+
this.#logger.warn({ path, err }, "chapter-store: invalid JSON, skipping");
|
|
2060
|
+
continue;
|
|
2061
|
+
}
|
|
2062
|
+
const parsed = ChapterSchema.safeParse(raw);
|
|
2063
|
+
if (!parsed.success) {
|
|
2064
|
+
this.#logger.warn(
|
|
2065
|
+
{ path, issues: parsed.error.issues },
|
|
2066
|
+
"chapter-store: schema mismatch, skipping"
|
|
2067
|
+
);
|
|
2068
|
+
continue;
|
|
2069
|
+
}
|
|
2070
|
+
chapters.push(parsed.data);
|
|
2071
|
+
}
|
|
2072
|
+
chapters.sort((a, b) => Number(a.index) - Number(b.index));
|
|
2073
|
+
return chapters;
|
|
2074
|
+
}
|
|
2075
|
+
};
|
|
2076
|
+
|
|
2077
|
+
// src/storage/resume-storage.ts
|
|
2078
|
+
var ResumeStorage = class _ResumeStorage {
|
|
2079
|
+
resume;
|
|
2080
|
+
chapters;
|
|
2081
|
+
stateFile;
|
|
2082
|
+
constructor(resume, chapters, stateFile) {
|
|
2083
|
+
this.resume = resume;
|
|
2084
|
+
this.chapters = chapters;
|
|
2085
|
+
this.stateFile = stateFile;
|
|
2086
|
+
}
|
|
2087
|
+
static forDefault(url, logger) {
|
|
2088
|
+
const stateFile = (0, import_node_path6.join)(userCacheDir(), "webnovel-downloader", "state", `${sha1(url)}.json`);
|
|
2089
|
+
return _ResumeStorage.#filesystem(stateFile, logger);
|
|
2090
|
+
}
|
|
2091
|
+
static forStateFile(stateFile, logger) {
|
|
2092
|
+
return _ResumeStorage.#filesystem(stateFile, logger);
|
|
2093
|
+
}
|
|
2094
|
+
static forToken(token, url, logger) {
|
|
2095
|
+
const { stateFile } = verifyResumeToken(token, url);
|
|
2096
|
+
return _ResumeStorage.#filesystem(stateFile, logger);
|
|
2097
|
+
}
|
|
2098
|
+
static memory(stateFile) {
|
|
2099
|
+
return new _ResumeStorage(new MemoryResumeStore(), new MemoryChapterStore(), stateFile);
|
|
2100
|
+
}
|
|
2101
|
+
static #filesystem(stateFile, logger) {
|
|
2102
|
+
return new _ResumeStorage(
|
|
2103
|
+
new FilesystemResumeStore(),
|
|
2104
|
+
new FilesystemChapterStore(`${stateFile}.chapters`, { logger }),
|
|
2105
|
+
stateFile
|
|
2106
|
+
);
|
|
2107
|
+
}
|
|
2108
|
+
};
|
|
2109
|
+
function resolveResumeStorage(opt, url, logger) {
|
|
2110
|
+
if (opt === void 0) return null;
|
|
2111
|
+
if (opt === true) return ResumeStorage.forDefault(url, logger);
|
|
2112
|
+
if ("stateFile" in opt) return ResumeStorage.forStateFile(opt.stateFile, logger);
|
|
2113
|
+
if ("token" in opt) return ResumeStorage.forToken(opt.token, url, logger);
|
|
2114
|
+
return null;
|
|
2115
|
+
}
|
|
2116
|
+
|
|
2032
2117
|
// src/core/downloader.ts
|
|
2033
2118
|
var DEFAULT_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
2034
2119
|
var DEFAULT_CONCURRENCY = 4;
|
|
@@ -2039,6 +2124,7 @@ var Downloader = class {
|
|
|
2039
2124
|
#transport;
|
|
2040
2125
|
#cookieJar;
|
|
2041
2126
|
#browserPool;
|
|
2127
|
+
#httpDispatcher;
|
|
2042
2128
|
#transportListeners = /* @__PURE__ */ new Set();
|
|
2043
2129
|
/**
|
|
2044
2130
|
* Build a downloader from a set of adapters. `rateLimit`, `retry`, and
|
|
@@ -2073,10 +2159,12 @@ var Downloader = class {
|
|
|
2073
2159
|
await handle.release();
|
|
2074
2160
|
}
|
|
2075
2161
|
} : void 0;
|
|
2162
|
+
this.#httpDispatcher = opts.http === void 0 && opts.undiciOverride === void 0 ? new import_undici2.Agent() : void 0;
|
|
2076
2163
|
this.#http = opts.http ?? buildHttpClient({
|
|
2077
2164
|
logger: this.#logger,
|
|
2078
2165
|
transport: this.#transport,
|
|
2079
2166
|
cookieJar: this.#cookieJar,
|
|
2167
|
+
...this.#httpDispatcher !== void 0 ? { dispatcher: this.#httpDispatcher } : {},
|
|
2080
2168
|
...opts.rateLimit !== void 0 ? { rateLimit: opts.rateLimit } : {},
|
|
2081
2169
|
...opts.retry !== void 0 ? { retry: opts.retry } : {},
|
|
2082
2170
|
onEvent: (e) => {
|
|
@@ -2107,10 +2195,11 @@ var Downloader = class {
|
|
|
2107
2195
|
...onCookieCapture !== void 0 ? { onCookieCapture } : {}
|
|
2108
2196
|
});
|
|
2109
2197
|
}
|
|
2110
|
-
/** Release held resources (closes the browser pool, if any). Safe to call once when done. */
|
|
2198
|
+
/** Release held resources (closes the browser pool and HTTP dispatcher, if any). Safe to call once when done. */
|
|
2111
2199
|
async dispose() {
|
|
2112
2200
|
const pool = this.#browserPool;
|
|
2113
2201
|
if (pool) await pool.dispose();
|
|
2202
|
+
if (this.#httpDispatcher) await this.#httpDispatcher.close();
|
|
2114
2203
|
}
|
|
2115
2204
|
/** True when a built-in/registered adapter can handle this URL. */
|
|
2116
2205
|
canHandle(url) {
|
|
@@ -2200,12 +2289,12 @@ var Downloader = class {
|
|
|
2200
2289
|
};
|
|
2201
2290
|
try {
|
|
2202
2291
|
this.#transportListeners.add(transportListener);
|
|
2203
|
-
const
|
|
2292
|
+
const resumeStorage = resolveResumeStorage(options?.resume, validated, this.#logger);
|
|
2204
2293
|
const run = new DownloadRun({
|
|
2205
2294
|
adapter,
|
|
2206
2295
|
validated,
|
|
2207
2296
|
normalizedRange,
|
|
2208
|
-
|
|
2297
|
+
resumeStorage,
|
|
2209
2298
|
concurrency,
|
|
2210
2299
|
signal,
|
|
2211
2300
|
ctx,
|
|
@@ -2307,13 +2396,6 @@ function normalizeText(html) {
|
|
|
2307
2396
|
});
|
|
2308
2397
|
return text.replace(new RegExp(LINE, "g"), " ").replace(new RegExp(PARA, "g"), "\n\n").split(/\n{2,}/).map((p) => p.replace(/\s+/g, " ").trim()).filter(Boolean).join("\n\n").trim();
|
|
2309
2398
|
}
|
|
2310
|
-
function pickText($, selectors) {
|
|
2311
|
-
for (const sel of selectors) {
|
|
2312
|
-
const t = $(sel).first().text().trim();
|
|
2313
|
-
if (t) return t;
|
|
2314
|
-
}
|
|
2315
|
-
return "";
|
|
2316
|
-
}
|
|
2317
2399
|
function wordCount(text) {
|
|
2318
2400
|
return text.split(/\s+/).filter(Boolean).length;
|
|
2319
2401
|
}
|
|
@@ -2385,6 +2467,9 @@ function refsFromAnchors($, opts) {
|
|
|
2385
2467
|
return refs;
|
|
2386
2468
|
}
|
|
2387
2469
|
|
|
2470
|
+
// src/adapters/shared/header.ts
|
|
2471
|
+
init_errors();
|
|
2472
|
+
|
|
2388
2473
|
// src/adapters/shared/cover.ts
|
|
2389
2474
|
function extractCoverUrl($, selector, baseUrl) {
|
|
2390
2475
|
const el = $(selector).first();
|
|
@@ -2392,6 +2477,36 @@ function extractCoverUrl($, selector, baseUrl) {
|
|
|
2392
2477
|
return raw ? resolveUrl(baseUrl, raw) : void 0;
|
|
2393
2478
|
}
|
|
2394
2479
|
|
|
2480
|
+
// src/adapters/shared/header.ts
|
|
2481
|
+
function parseNovelHeader($, selectors, sourceUrl) {
|
|
2482
|
+
const title = $(selectors.title).first().text().trim();
|
|
2483
|
+
if (!title) throw new ParseError("Missing title on novel page", { url: sourceUrl });
|
|
2484
|
+
const author = $(selectors.author).first().text().trim();
|
|
2485
|
+
const descSelectors = typeof selectors.description === "string" ? [selectors.description] : selectors.description;
|
|
2486
|
+
let description = "";
|
|
2487
|
+
for (const sel of descSelectors) {
|
|
2488
|
+
const t = $(sel).first().text().trim();
|
|
2489
|
+
if (t) {
|
|
2490
|
+
description = t;
|
|
2491
|
+
break;
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
description = description.replace(/\s+/g, " ").trim();
|
|
2495
|
+
const genres = [];
|
|
2496
|
+
$(selectors.genres).each((_, el) => {
|
|
2497
|
+
const t = $(el).text().trim();
|
|
2498
|
+
if (t && !genres.includes(t)) genres.push(t);
|
|
2499
|
+
});
|
|
2500
|
+
const coverUrl = extractCoverUrl($, selectors.cover, sourceUrl);
|
|
2501
|
+
return {
|
|
2502
|
+
title,
|
|
2503
|
+
author,
|
|
2504
|
+
description,
|
|
2505
|
+
genres,
|
|
2506
|
+
...coverUrl !== void 0 ? { coverUrl } : {}
|
|
2507
|
+
};
|
|
2508
|
+
}
|
|
2509
|
+
|
|
2395
2510
|
// src/adapters/shared/content.ts
|
|
2396
2511
|
init_errors();
|
|
2397
2512
|
function stripLeadingTitle(content, title) {
|
|
@@ -2432,31 +2547,16 @@ var TRUYENFULL_SELECTORS = {
|
|
|
2432
2547
|
|
|
2433
2548
|
// src/adapters/truyenfull/parser.ts
|
|
2434
2549
|
function parseNovelPage($, sourceUrl) {
|
|
2435
|
-
const
|
|
2436
|
-
if (!title) {
|
|
2437
|
-
throw new ParseError("Missing title on novel page", { url: sourceUrl });
|
|
2438
|
-
}
|
|
2439
|
-
const author = $(TRUYENFULL_SELECTORS.metadata.author).first().text().trim();
|
|
2440
|
-
const description = $(TRUYENFULL_SELECTORS.metadata.description).text().replace(/\s+/g, " ").trim();
|
|
2441
|
-
const genres = [];
|
|
2442
|
-
$(TRUYENFULL_SELECTORS.metadata.genres).each((_, el) => {
|
|
2443
|
-
const t = $(el).text().trim();
|
|
2444
|
-
if (t) genres.push(t);
|
|
2445
|
-
});
|
|
2446
|
-
const coverUrl = extractCoverUrl($, TRUYENFULL_SELECTORS.metadata.cover, sourceUrl);
|
|
2550
|
+
const header = parseNovelHeader($, TRUYENFULL_SELECTORS.metadata, sourceUrl);
|
|
2447
2551
|
let status = "unknown";
|
|
2448
2552
|
if ($(TRUYENFULL_SELECTORS.metadata.statusFull).length > 0) status = "completed";
|
|
2449
2553
|
else if ($(TRUYENFULL_SELECTORS.metadata.statusOngoing).length > 0) status = "ongoing";
|
|
2450
2554
|
return {
|
|
2555
|
+
...header,
|
|
2451
2556
|
sourceUrl,
|
|
2452
2557
|
sourceSite: unsafeBrandAdapterId("truyenfull"),
|
|
2453
|
-
title,
|
|
2454
|
-
author,
|
|
2455
|
-
description,
|
|
2456
|
-
genres,
|
|
2457
2558
|
status,
|
|
2458
|
-
fetchedAt: /* @__PURE__ */ new Date()
|
|
2459
|
-
...coverUrl !== void 0 ? { coverUrl } : {}
|
|
2559
|
+
fetchedAt: /* @__PURE__ */ new Date()
|
|
2460
2560
|
};
|
|
2461
2561
|
}
|
|
2462
2562
|
function parseTocPage($, startIndex, baseUrl) {
|
|
@@ -2622,32 +2722,17 @@ var METRUYENCHU_SELECTORS = {
|
|
|
2622
2722
|
// src/adapters/metruyenchu/parser.ts
|
|
2623
2723
|
var SOURCE_SITE = unsafeBrandAdapterId("metruyenchu-com-vn");
|
|
2624
2724
|
function parseNovelPage2($, sourceUrl) {
|
|
2625
|
-
const
|
|
2626
|
-
if (!title) {
|
|
2627
|
-
throw new ParseError("Missing title on novel page", { url: sourceUrl });
|
|
2628
|
-
}
|
|
2629
|
-
const author = $(METRUYENCHU_SELECTORS.metadata.author).first().text().trim();
|
|
2630
|
-
const description = pickText($, METRUYENCHU_SELECTORS.metadata.description).replace(/\s+/g, " ").trim();
|
|
2631
|
-
const genres = [];
|
|
2632
|
-
$(METRUYENCHU_SELECTORS.metadata.genres).each((_, el) => {
|
|
2633
|
-
const t = $(el).text().trim();
|
|
2634
|
-
if (t && !genres.includes(t)) genres.push(t);
|
|
2635
|
-
});
|
|
2636
|
-
const coverUrl = extractCoverUrl($, METRUYENCHU_SELECTORS.metadata.cover, sourceUrl);
|
|
2725
|
+
const header = parseNovelHeader($, METRUYENCHU_SELECTORS.metadata, sourceUrl);
|
|
2637
2726
|
let status = "unknown";
|
|
2638
2727
|
if ($(METRUYENCHU_SELECTORS.metadata.statusFull).length > 0) status = "completed";
|
|
2639
2728
|
else if (METRUYENCHU_SELECTORS.metadata.statusOngoing.some((sel) => $(sel).length > 0)) status = "ongoing";
|
|
2640
2729
|
const totalChapters = extractTotalChapters($);
|
|
2641
2730
|
return {
|
|
2731
|
+
...header,
|
|
2642
2732
|
sourceUrl,
|
|
2643
2733
|
sourceSite: SOURCE_SITE,
|
|
2644
|
-
title,
|
|
2645
|
-
author,
|
|
2646
|
-
description,
|
|
2647
|
-
genres,
|
|
2648
2734
|
status,
|
|
2649
2735
|
fetchedAt: /* @__PURE__ */ new Date(),
|
|
2650
|
-
...coverUrl ? { coverUrl } : {},
|
|
2651
2736
|
...totalChapters !== void 0 ? { totalChapters } : {}
|
|
2652
2737
|
};
|
|
2653
2738
|
}
|
|
@@ -3078,6 +3163,7 @@ var downloader = createDownloader();
|
|
|
3078
3163
|
builtinAdapters,
|
|
3079
3164
|
createDownloader,
|
|
3080
3165
|
createLogger,
|
|
3081
|
-
downloader
|
|
3166
|
+
downloader,
|
|
3167
|
+
parseNovelHeader
|
|
3082
3168
|
});
|
|
3083
3169
|
//# sourceMappingURL=index.cjs.map
|