@duyquangnvx/webnovel-downloader 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -448,7 +448,8 @@ __export(index_exports, {
448
448
  builtinAdapters: () => builtinAdapters,
449
449
  createDownloader: () => createDownloader,
450
450
  createLogger: () => createLogger,
451
- downloader: () => downloader
451
+ downloader: () => downloader,
452
+ parseNovelHeader: () => parseNovelHeader
452
453
  });
453
454
  module.exports = __toCommonJS(index_exports);
454
455
 
@@ -472,6 +473,9 @@ var silentLogger = createLogger({ level: "silent" });
472
473
  // src/index.ts
473
474
  init_errors();
474
475
 
476
+ // src/core/downloader.ts
477
+ var import_undici2 = require("undici");
478
+
475
479
  // src/core/registry.ts
476
480
  init_errors();
477
481
  var AdapterRegistry = class {
@@ -550,8 +554,12 @@ init_errors();
550
554
  var DEFAULT_TIMEOUT_MS = 3e4;
551
555
  var UndiciHttpClient = class {
552
556
  #logger;
557
+ #dispatcher;
558
+ #ownsDispatcher;
553
559
  constructor(opts) {
554
560
  this.#logger = opts.logger;
561
+ this.#ownsDispatcher = opts.dispatcher === void 0;
562
+ this.#dispatcher = opts.dispatcher ?? new import_undici.Agent();
555
563
  }
556
564
  async get(url, opts) {
557
565
  const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
@@ -560,6 +568,7 @@ var UndiciHttpClient = class {
560
568
  try {
561
569
  res = await (0, import_undici.request)(url, {
562
570
  method: "GET",
571
+ dispatcher: this.#dispatcher,
563
572
  ...opts?.headers !== void 0 ? { headers: opts.headers } : {},
564
573
  ...opts?.signal !== void 0 ? { signal: opts.signal } : {},
565
574
  headersTimeout: half,
@@ -584,10 +593,15 @@ var UndiciHttpClient = class {
584
593
  if (cause instanceof import_undici.errors.HeadersTimeoutError || cause instanceof import_undici.errors.BodyTimeoutError) {
585
594
  throw new TimeoutError(url, { cause });
586
595
  }
587
- this.#logger.debug({ url, cause }, "undici network error");
588
- throw new HttpError(0, url, "Network error", { cause });
596
+ const detail = cause instanceof Error ? `${cause.name}: ${cause.message}` : String(cause);
597
+ this.#logger.warn({ url, cause }, "undici network error");
598
+ throw new HttpError(0, url, `Network error (${detail})`, { cause });
589
599
  }
590
600
  }
601
+ /** Closes the dispatcher this client created. A no-op when one was injected — the caller owns that. */
602
+ async dispose() {
603
+ if (this.#ownsDispatcher) await this.#dispatcher.close();
604
+ }
591
605
  };
592
606
  function normalizeHeaders(h) {
593
607
  const out = {};
@@ -719,9 +733,13 @@ var RateLimiter = class {
719
733
  #buckets = /* @__PURE__ */ new Map();
720
734
  #pauses = /* @__PURE__ */ new Map();
721
735
  constructor(opts = DEFAULT_RATE_LIMIT) {
736
+ const burst = opts.burst ?? Math.max(1, opts.requestsPerSecond);
737
+ if (burst < 1) {
738
+ throw new RangeError(`RateLimiter: burst must be >= 1 (got ${burst})`);
739
+ }
722
740
  this.#opts = {
723
741
  requestsPerSecond: opts.requestsPerSecond,
724
- burst: opts.burst ?? opts.requestsPerSecond
742
+ burst
725
743
  };
726
744
  }
727
745
  async acquire(host, signal) {
@@ -881,6 +899,31 @@ function clamp(n, lo, hi) {
881
899
  // src/http/index.ts
882
900
  init_cookie_jar();
883
901
 
902
+ // src/http/stack-builder.ts
903
+ var HttpStack = class _HttpStack {
904
+ #client;
905
+ constructor(client) {
906
+ this.#client = client;
907
+ }
908
+ static from(leaf) {
909
+ return new _HttpStack(leaf);
910
+ }
911
+ rotateUserAgent(uas, defaultHeaders, cookieJar) {
912
+ return new _HttpStack(
913
+ new UserAgentRotatingHttpClient(this.#client, uas, defaultHeaders, cookieJar)
914
+ );
915
+ }
916
+ rateLimit(limiter, onEvent) {
917
+ return new _HttpStack(new RateLimitedHttpClient(this.#client, limiter, onEvent));
918
+ }
919
+ retry(opts, logger, limiter, onEvent) {
920
+ return new _HttpStack(new RetryingHttpClient(this.#client, opts, logger, limiter, onEvent));
921
+ }
922
+ build() {
923
+ return this.#client;
924
+ }
925
+ };
926
+
884
927
  // src/http/tiered.ts
885
928
  init_cf_challenge();
886
929
  var FALLBACK_UA = DEFAULT_UA_POOL[0] ?? "Mozilla/5.0";
@@ -941,22 +984,12 @@ function normalizeTransport(t) {
941
984
  function buildHttpClient(opts = {}) {
942
985
  const logger = opts.logger ?? silentLogger;
943
986
  const cookieJar = opts.cookieJar ?? new CookieJar();
944
- const leaf = opts.undiciOverride ?? new UndiciHttpClient({ logger });
945
- const ua = new UserAgentRotatingHttpClient(
946
- leaf,
947
- opts.userAgents ?? new UserAgents(),
948
- opts.defaultHeaders,
949
- cookieJar
950
- );
951
- const limiter = opts.rateLimiter ?? new RateLimiter(opts.rateLimit ?? DEFAULT_RATE_LIMIT);
952
- const limited = new RateLimitedHttpClient(ua, limiter, opts.onEvent);
953
- const httpBranch = new RetryingHttpClient(
954
- limited,
955
- normalizeRetry(opts.retry),
987
+ const leaf = opts.undiciOverride ?? new UndiciHttpClient({
956
988
  logger,
957
- limiter,
958
- opts.onEvent
959
- );
989
+ ...opts.dispatcher !== void 0 ? { dispatcher: opts.dispatcher } : {}
990
+ });
991
+ const limiter = opts.rateLimiter ?? new RateLimiter(opts.rateLimit ?? DEFAULT_RATE_LIMIT);
992
+ const httpBranch = HttpStack.from(leaf).rotateUserAgent(opts.userAgents ?? new UserAgents(), opts.defaultHeaders, cookieJar).rateLimit(limiter, opts.onEvent).retry(normalizeRetry(opts.retry), logger, limiter, opts.onEvent).build();
960
993
  const { mode } = normalizeTransport(opts.transport);
961
994
  if (mode === "http-only") {
962
995
  return httpBranch;
@@ -1411,7 +1444,6 @@ function toResponse(entry) {
1411
1444
 
1412
1445
  // src/core/download-run.ts
1413
1446
  var import_p_queue = __toESM(require("p-queue"), 1);
1414
- var import_node_path6 = require("path");
1415
1447
 
1416
1448
  // src/core/debouncer.ts
1417
1449
  var Debouncer = class {
@@ -1474,30 +1506,47 @@ init_primitives();
1474
1506
 
1475
1507
  // src/pipeline/fetch-toc.ts
1476
1508
  init_errors();
1509
+
1510
+ // src/pipeline/walk-toc.ts
1511
+ init_errors();
1477
1512
  var TOC_PROGRESS_EMIT_EVERY = 50;
1478
- async function fetchTocStage(adapter, url, ctx, bus, onRef, range) {
1513
+ async function walkToc(source, opts) {
1514
+ const from = opts.range?.from;
1515
+ const to = opts.range?.to;
1479
1516
  let count = 0;
1480
- const from = range?.from;
1481
- const to = range?.to;
1482
- if (ctx.signal.aborted) throw new CancelledError({ cause: ctx.signal.reason });
1483
- for await (const raw of adapter.fetchChapterList(url, ctx)) {
1484
- if (ctx.signal.aborted) throw new CancelledError({ cause: ctx.signal.reason });
1485
- const parsed = ChapterRefSchema.safeParse(raw);
1486
- if (!parsed.success) {
1487
- throw new ParseError(`Invalid ChapterRef from adapter`, { cause: parsed.error, url });
1488
- }
1489
- const ref = parsed.data;
1517
+ if (opts.signal.aborted) throw new CancelledError({ cause: opts.signal.reason });
1518
+ for await (const ref of source) {
1519
+ if (opts.signal.aborted) throw new CancelledError({ cause: opts.signal.reason });
1490
1520
  if (to !== void 0 && ref.index > to) break;
1491
1521
  if (from !== void 0 && ref.index < from) continue;
1492
- await onRef(ref);
1522
+ await opts.onRef(ref);
1493
1523
  count++;
1494
- if (count % TOC_PROGRESS_EMIT_EVERY === 0) bus.emit({ type: "toc:progress", discovered: count });
1524
+ if (count % TOC_PROGRESS_EMIT_EVERY === 0) opts.bus.emit({ type: "toc:progress", discovered: count });
1495
1525
  }
1496
- bus.emit({ type: "toc:progress", discovered: count });
1497
- bus.emit({ type: "toc:complete", total: count });
1526
+ opts.bus.emit({ type: "toc:progress", discovered: count });
1527
+ opts.bus.emit({ type: "toc:complete", total: count });
1498
1528
  return count;
1499
1529
  }
1500
1530
 
1531
+ // src/pipeline/fetch-toc.ts
1532
+ async function fetchTocStage(adapter, url, ctx, bus, onRef, range) {
1533
+ async function* validated() {
1534
+ for await (const raw of adapter.fetchChapterList(url, ctx)) {
1535
+ const parsed = ChapterRefSchema.safeParse(raw);
1536
+ if (!parsed.success) {
1537
+ throw new ParseError(`Invalid ChapterRef from adapter`, { cause: parsed.error, url });
1538
+ }
1539
+ yield parsed.data;
1540
+ }
1541
+ }
1542
+ return walkToc(validated(), {
1543
+ ...range !== void 0 ? { range } : {},
1544
+ bus,
1545
+ signal: ctx.signal,
1546
+ onRef
1547
+ });
1548
+ }
1549
+
1501
1550
  // src/pipeline/fetch-chapters.ts
1502
1551
  init_errors();
1503
1552
  async function fetchChapterTask(adapter, ref, ctx, bus) {
@@ -1525,130 +1574,15 @@ function assembleNovelData(metadata, results) {
1525
1574
  // src/core/download-run.ts
1526
1575
  init_errors();
1527
1576
 
1528
- // src/storage/resume-store.ts
1529
- var import_promises3 = require("fs/promises");
1577
+ // src/storage/resume-token.ts
1530
1578
  var import_node_path3 = require("path");
1531
- init_errors();
1532
-
1533
- // src/storage/resume-state.ts
1534
1579
  var import_zod4 = require("zod");
1535
- init_primitives();
1536
- var ResumeFailureRecordSchema = import_zod4.z.object({
1537
- index: ChapterIndexSchema,
1538
- lastError: import_zod4.z.object({ code: import_zod4.z.string(), message: import_zod4.z.string() }).readonly()
1539
- }).readonly();
1540
- var ResumeStateSchema = import_zod4.z.object({
1541
- schemaVersion: import_zod4.z.literal(1),
1542
- url: UrlSchema,
1543
- adapterId: AdapterIdSchema,
1544
- metadata: NovelMetadataSchema.nullable(),
1545
- toc: import_zod4.z.array(ChapterRefSchema).readonly().nullable(),
1546
- completed: import_zod4.z.array(ChapterIndexSchema).readonly(),
1547
- failed: import_zod4.z.array(ResumeFailureRecordSchema).readonly(),
1548
- startedAt: IsoDateSchema,
1549
- updatedAt: IsoDateSchema
1550
- }).readonly();
1551
-
1552
- // src/storage/resume-store.ts
1553
- var FilesystemResumeStore = class {
1554
- async load(path) {
1555
- let text;
1556
- try {
1557
- text = await (0, import_promises3.readFile)(path, "utf8");
1558
- } catch (err) {
1559
- if (isEnoent(err)) return null;
1560
- throw err;
1561
- }
1562
- let raw;
1563
- try {
1564
- raw = JSON.parse(text);
1565
- } catch (cause) {
1566
- throw new ParseError("Resume state file is corrupt; delete it to start over", { cause, path });
1567
- }
1568
- const parsed = ResumeStateSchema.safeParse(raw);
1569
- if (!parsed.success) {
1570
- throw new ParseError("Resume state file is corrupt; delete it to start over", {
1571
- cause: parsed.error,
1572
- path
1573
- });
1574
- }
1575
- return parsed.data;
1576
- }
1577
- async save(path, state) {
1578
- await atomicWriteJson(path, state);
1579
- }
1580
- async delete(path) {
1581
- try {
1582
- await (0, import_promises3.unlink)(path);
1583
- } catch (err) {
1584
- if (!isEnoent(err)) throw err;
1585
- }
1586
- }
1587
- };
1588
-
1589
- // src/storage/chapter-store.ts
1590
- var import_promises4 = require("fs/promises");
1591
- var import_node_path4 = require("path");
1592
- var FilesystemChapterStore = class {
1593
- #logger;
1594
- #dir;
1595
- constructor(dir, opts = {}) {
1596
- this.#dir = (0, import_node_path4.resolve)(dir);
1597
- this.#logger = opts.logger ?? silentLogger;
1598
- }
1599
- async save(chapter) {
1600
- await atomicWriteJson((0, import_node_path4.join)(this.#dir, `${Number(chapter.index)}.json`), chapter);
1601
- }
1602
- async loadAll() {
1603
- let entries;
1604
- try {
1605
- entries = await (0, import_promises4.readdir)(this.#dir);
1606
- } catch (err) {
1607
- if (isEnoent(err)) return [];
1608
- throw err;
1609
- }
1610
- const chapters = [];
1611
- for (const name of entries) {
1612
- if (!/^\d+\.json$/.test(name)) continue;
1613
- const path = (0, import_node_path4.join)(this.#dir, name);
1614
- let text;
1615
- try {
1616
- text = await (0, import_promises4.readFile)(path, "utf8");
1617
- } catch (err) {
1618
- this.#logger.warn({ path, err }, "chapter-store: failed to read file, skipping");
1619
- continue;
1620
- }
1621
- let raw;
1622
- try {
1623
- raw = JSON.parse(text);
1624
- } catch (err) {
1625
- this.#logger.warn({ path, err }, "chapter-store: invalid JSON, skipping");
1626
- continue;
1627
- }
1628
- const parsed = ChapterSchema.safeParse(raw);
1629
- if (!parsed.success) {
1630
- this.#logger.warn(
1631
- { path, issues: parsed.error.issues },
1632
- "chapter-store: schema mismatch, skipping"
1633
- );
1634
- continue;
1635
- }
1636
- chapters.push(parsed.data);
1637
- }
1638
- chapters.sort((a, b) => Number(a.index) - Number(b.index));
1639
- return chapters;
1640
- }
1641
- };
1642
-
1643
- // src/storage/resume-token.ts
1644
- var import_node_path5 = require("path");
1645
- var import_zod5 = require("zod");
1646
1580
  init_errors();
1647
1581
  init_primitives();
1648
- var TokenPayloadSchema = import_zod5.z.object({
1649
- v: import_zod5.z.literal(1),
1650
- stateFile: import_zod5.z.string().min(1),
1651
- urlHash: import_zod5.z.string().min(1)
1582
+ var TokenPayloadSchema = import_zod4.z.object({
1583
+ v: import_zod4.z.literal(1),
1584
+ stateFile: import_zod4.z.string().min(1),
1585
+ urlHash: import_zod4.z.string().min(1)
1652
1586
  });
1653
1587
  function encodeResumeToken(stateFile, url) {
1654
1588
  const payload = { v: 1, stateFile, urlHash: sha1(url) };
@@ -1669,9 +1603,9 @@ function decodeResumeToken(token) {
1669
1603
  return { stateFile: parsed.data.stateFile, urlHash: parsed.data.urlHash };
1670
1604
  }
1671
1605
  function assertWithinCacheDir(stateFile) {
1672
- const root = (0, import_node_path5.resolve)((0, import_node_path5.join)(userCacheDir(), "webnovel-downloader"));
1673
- const rel = (0, import_node_path5.relative)(root, (0, import_node_path5.resolve)(stateFile));
1674
- if (rel === "" || rel.startsWith("..") || (0, import_node_path5.isAbsolute)(rel)) {
1606
+ const root = (0, import_node_path3.resolve)((0, import_node_path3.join)(userCacheDir(), "webnovel-downloader"));
1607
+ const rel = (0, import_node_path3.relative)(root, (0, import_node_path3.resolve)(stateFile));
1608
+ if (rel === "" || rel.startsWith("..") || (0, import_node_path3.isAbsolute)(rel)) {
1675
1609
  throw new ParseError("Resume token state path escapes the cache directory");
1676
1610
  }
1677
1611
  }
@@ -1687,25 +1621,6 @@ function verifyResumeToken(token, url) {
1687
1621
  // src/core/download-run.ts
1688
1622
  var QUEUE_BACKPRESSURE_RATIO = 4;
1689
1623
  var STATE_PERSIST_DEBOUNCE_MS = 500;
1690
- var TOC_PROGRESS_EMIT_EVERY2 = 50;
1691
- function resolveResumeConfig(opt, url, logger) {
1692
- if (opt === void 0) return null;
1693
- const make = (stateFile) => ({
1694
- stateFile,
1695
- store: new FilesystemResumeStore(),
1696
- chapterStore: new FilesystemChapterStore(`${stateFile}.chapters`, { logger })
1697
- });
1698
- if (opt === true) {
1699
- const stateFile = (0, import_node_path6.join)(userCacheDir(), "webnovel-downloader", "state", `${sha1(url)}.json`);
1700
- return make(stateFile);
1701
- }
1702
- if ("stateFile" in opt) return make(opt.stateFile);
1703
- if ("token" in opt) {
1704
- const { stateFile } = verifyResumeToken(opt.token, url);
1705
- return make(stateFile);
1706
- }
1707
- return null;
1708
- }
1709
1624
  function freshState(url, adapterId) {
1710
1625
  const now = nowIso();
1711
1626
  return {
@@ -1727,14 +1642,13 @@ var DownloadRun = class {
1727
1642
  #adapter;
1728
1643
  #validated;
1729
1644
  #normalizedRange;
1730
- #resumeCfg;
1645
+ #resumeStorage;
1731
1646
  #concurrency;
1732
1647
  #signal;
1733
1648
  #ctx;
1734
1649
  #bus;
1735
1650
  #logger;
1736
1651
  #state = null;
1737
- #metadata;
1738
1652
  #successes = [];
1739
1653
  #failures = [];
1740
1654
  #completedSet = /* @__PURE__ */ new Set();
@@ -1748,7 +1662,7 @@ var DownloadRun = class {
1748
1662
  this.#adapter = args.adapter;
1749
1663
  this.#validated = args.validated;
1750
1664
  this.#normalizedRange = args.normalizedRange;
1751
- this.#resumeCfg = args.resumeCfg;
1665
+ this.#resumeStorage = args.resumeStorage;
1752
1666
  this.#concurrency = args.concurrency;
1753
1667
  this.#signal = args.signal;
1754
1668
  this.#ctx = args.ctx;
@@ -1764,9 +1678,10 @@ var DownloadRun = class {
1764
1678
  async execute() {
1765
1679
  const initErr = await this.#initResume();
1766
1680
  if (initErr) return initErr;
1767
- const metaErr = await this.#resolveMetadata();
1768
- if (metaErr) return metaErr;
1769
- const ctxWithMeta = this.#startQueue();
1681
+ const resolved = await this.#resolveMetadata();
1682
+ if ("error" in resolved) return resolved.error;
1683
+ const meta = resolved.meta;
1684
+ const ctxWithMeta = this.#startQueue(meta);
1770
1685
  const onAbort = () => {
1771
1686
  this.#aborted = true;
1772
1687
  this.#queue.clear();
@@ -1776,16 +1691,16 @@ var DownloadRun = class {
1776
1691
  const stage2Error = await this.#runTocStage(ctxWithMeta);
1777
1692
  await this.#drainQueueOrAbort();
1778
1693
  await this.#flushQuietly();
1779
- return this.#finalize(stage2Error);
1694
+ return this.#finalize(meta, stage2Error);
1780
1695
  } finally {
1781
1696
  this.#signal.removeEventListener("abort", onAbort);
1782
1697
  }
1783
1698
  }
1784
1699
  async #initResume() {
1785
- const resumeCfg = this.#resumeCfg;
1700
+ const resumeCfg = this.#resumeStorage;
1786
1701
  if (resumeCfg) {
1787
1702
  try {
1788
- let state = await resumeCfg.store.load(resumeCfg.stateFile);
1703
+ let state = await resumeCfg.resume.load(resumeCfg.stateFile);
1789
1704
  if (state) {
1790
1705
  if (state.url !== this.#validated) {
1791
1706
  throw new ParseError("Stale resume state: URL mismatch");
@@ -1802,7 +1717,7 @@ var DownloadRun = class {
1802
1717
  }
1803
1718
  }
1804
1719
  if (resumeCfg && this.#state) {
1805
- const prior = await resumeCfg.chapterStore.loadAll();
1720
+ const prior = await resumeCfg.chapters.loadAll();
1806
1721
  this.#successes.push(...prior);
1807
1722
  const merged = new Set(this.#state.completed);
1808
1723
  for (const c of prior) merged.add(c.index);
@@ -1814,32 +1729,30 @@ var DownloadRun = class {
1814
1729
  return null;
1815
1730
  }
1816
1731
  async #persistState() {
1817
- const resumeCfg = this.#resumeCfg;
1732
+ const resumeCfg = this.#resumeStorage;
1818
1733
  if (!resumeCfg || !this.#state) return;
1819
1734
  const next = { ...this.#state, updatedAt: nowIso() };
1820
1735
  this.#state = next;
1821
- await resumeCfg.store.save(resumeCfg.stateFile, next);
1736
+ await resumeCfg.resume.save(resumeCfg.stateFile, next);
1822
1737
  }
1823
1738
  async #resolveMetadata() {
1824
1739
  try {
1740
+ let meta;
1825
1741
  if (this.#state?.metadata) {
1826
- this.#metadata = this.#state.metadata;
1742
+ meta = this.#state.metadata;
1827
1743
  } else {
1828
- this.#metadata = await fetchMetadataStage(this.#adapter, this.#validated, this.#ctx, this.#bus);
1744
+ meta = await fetchMetadataStage(this.#adapter, this.#validated, this.#ctx, this.#bus);
1829
1745
  if (this.#state) {
1830
- this.#state = { ...this.#state, metadata: this.#metadata, updatedAt: nowIso() };
1746
+ this.#state = { ...this.#state, metadata: meta, updatedAt: nowIso() };
1831
1747
  await this.#persistState();
1832
1748
  }
1833
1749
  }
1750
+ return { meta };
1834
1751
  } catch (err) {
1835
- return {
1836
- status: "error",
1837
- error: normalizeToEnvelopeError(err)
1838
- };
1752
+ return { error: { status: "error", error: normalizeToEnvelopeError(err) } };
1839
1753
  }
1840
- return null;
1841
1754
  }
1842
- #startQueue() {
1755
+ #startQueue(meta) {
1843
1756
  const state = this.#state;
1844
1757
  const completedSet = this.#completedSet;
1845
1758
  const completedList = this.#completedList;
@@ -1851,8 +1764,7 @@ var DownloadRun = class {
1851
1764
  if (!completedSet.has(f.index)) failedMap.set(f.index, f);
1852
1765
  }
1853
1766
  }
1854
- if (!this.#metadata) throw new Error("invariant: metadata must be set before #startQueue");
1855
- return { ...this.#ctx, novelMetadata: this.#metadata };
1767
+ return { ...this.#ctx, novelMetadata: meta };
1856
1768
  }
1857
1769
  async #enqueueRef(ref, ctxWithMeta) {
1858
1770
  if (this.#aborted || this.#signal.aborted) return;
@@ -1872,7 +1784,7 @@ var DownloadRun = class {
1872
1784
  try {
1873
1785
  const chapter = await fetchChapterTask(this.#adapter, ref, ctxWithMeta, this.#bus);
1874
1786
  if (this.#aborted) return;
1875
- if (this.#resumeCfg) await this.#resumeCfg.chapterStore.save(chapter);
1787
+ if (this.#resumeStorage) await this.#resumeStorage.chapters.save(chapter);
1876
1788
  this.#successes.push(chapter);
1877
1789
  this.#completedList.push(ref.index);
1878
1790
  this.#completedSet.add(ref.index);
@@ -1889,7 +1801,7 @@ var DownloadRun = class {
1889
1801
  this.#bus.emit({
1890
1802
  type: "progress",
1891
1803
  completed: this.#successes.length,
1892
- total: this.#metadata?.totalChapters ?? this.#enqueuedSet.size
1804
+ total: ctxWithMeta.novelMetadata?.totalChapters ?? this.#enqueuedSet.size
1893
1805
  });
1894
1806
  } catch (cause) {
1895
1807
  if (cause instanceof CancelledError) return;
@@ -1910,20 +1822,16 @@ var DownloadRun = class {
1910
1822
  let stage2Error;
1911
1823
  try {
1912
1824
  if (this.#state?.toc) {
1913
- const from = this.#normalizedRange?.from;
1914
- const to = this.#normalizedRange?.to;
1915
- let discovered = 0;
1916
- for (const ref of this.#state.toc) {
1917
- if (to !== void 0 && ref.index > to) break;
1918
- if (from !== void 0 && ref.index < from) continue;
1919
- await this.#enqueueRef(ref, ctxWithMeta);
1920
- discovered++;
1921
- if (discovered % TOC_PROGRESS_EMIT_EVERY2 === 0) {
1922
- this.#bus.emit({ type: "toc:progress", discovered });
1923
- }
1825
+ const toc = this.#state.toc;
1826
+ async function* fromState() {
1827
+ for (const ref of toc) yield ref;
1924
1828
  }
1925
- this.#bus.emit({ type: "toc:progress", discovered });
1926
- this.#bus.emit({ type: "toc:complete", total: discovered });
1829
+ await walkToc(fromState(), {
1830
+ ...this.#normalizedRange !== void 0 ? { range: this.#normalizedRange } : {},
1831
+ bus: this.#bus,
1832
+ signal: this.#signal,
1833
+ onRef: (ref) => this.#enqueueRef(ref, ctxWithMeta)
1834
+ });
1927
1835
  } else {
1928
1836
  const tocBuffer = [];
1929
1837
  await fetchTocStage(
@@ -1998,10 +1906,8 @@ var DownloadRun = class {
1998
1906
  this.#logger.warn({ err, url: this.#validated }, "resume state persist failed (non-fatal)");
1999
1907
  }
2000
1908
  }
2001
- #finalize(stage2Error) {
1909
+ #finalize(metadata, stage2Error) {
2002
1910
  if (stage2Error) return { status: "error", error: stage2Error };
2003
- if (!this.#metadata) throw new Error("invariant: metadata must be set before #finalize");
2004
- const metadata = this.#metadata;
2005
1911
  let data;
2006
1912
  try {
2007
1913
  data = assembleNovelData(metadata, this.#successes);
@@ -2012,8 +1918,8 @@ var DownloadRun = class {
2012
1918
  };
2013
1919
  }
2014
1920
  if (this.#failures.length > 0) {
2015
- if (this.#resumeCfg) {
2016
- const token = encodeResumeToken(this.#resumeCfg.stateFile, this.#validated);
1921
+ if (this.#resumeStorage) {
1922
+ const token = encodeResumeToken(this.#resumeStorage.stateFile, this.#validated);
2017
1923
  return { status: "partial", resumable: true, data, failures: this.#failures, resumeToken: token };
2018
1924
  }
2019
1925
  return { status: "partial", resumable: false, data, failures: this.#failures };
@@ -2029,6 +1935,185 @@ function normalizeChapterRange(range) {
2029
1935
  return out;
2030
1936
  }
2031
1937
 
1938
+ // src/storage/resume-storage.ts
1939
+ var import_node_path6 = require("path");
1940
+
1941
+ // src/storage/resume-store.ts
1942
+ var import_promises3 = require("fs/promises");
1943
+ var import_node_path4 = require("path");
1944
+ init_errors();
1945
+
1946
+ // src/storage/resume-state.ts
1947
+ var import_zod5 = require("zod");
1948
+ init_primitives();
1949
+ var ResumeFailureRecordSchema = import_zod5.z.object({
1950
+ index: ChapterIndexSchema,
1951
+ lastError: import_zod5.z.object({ code: import_zod5.z.string(), message: import_zod5.z.string() }).readonly()
1952
+ }).readonly();
1953
+ var ResumeStateSchema = import_zod5.z.object({
1954
+ schemaVersion: import_zod5.z.literal(1),
1955
+ url: UrlSchema,
1956
+ adapterId: AdapterIdSchema,
1957
+ metadata: NovelMetadataSchema.nullable(),
1958
+ toc: import_zod5.z.array(ChapterRefSchema).readonly().nullable(),
1959
+ completed: import_zod5.z.array(ChapterIndexSchema).readonly(),
1960
+ failed: import_zod5.z.array(ResumeFailureRecordSchema).readonly(),
1961
+ startedAt: IsoDateSchema,
1962
+ updatedAt: IsoDateSchema
1963
+ }).readonly();
1964
+
1965
+ // src/storage/resume-store.ts
1966
+ var MemoryResumeStore = class {
1967
+ #map = /* @__PURE__ */ new Map();
1968
+ async load(path) {
1969
+ return this.#map.get((0, import_node_path4.resolve)(path)) ?? null;
1970
+ }
1971
+ async save(path, state) {
1972
+ this.#map.set((0, import_node_path4.resolve)(path), state);
1973
+ }
1974
+ async delete(path) {
1975
+ this.#map.delete((0, import_node_path4.resolve)(path));
1976
+ }
1977
+ };
1978
+ var FilesystemResumeStore = class {
1979
+ async load(path) {
1980
+ let text;
1981
+ try {
1982
+ text = await (0, import_promises3.readFile)(path, "utf8");
1983
+ } catch (err) {
1984
+ if (isEnoent(err)) return null;
1985
+ throw err;
1986
+ }
1987
+ let raw;
1988
+ try {
1989
+ raw = JSON.parse(text);
1990
+ } catch (cause) {
1991
+ throw new ParseError("Resume state file is corrupt; delete it to start over", { cause, path });
1992
+ }
1993
+ const parsed = ResumeStateSchema.safeParse(raw);
1994
+ if (!parsed.success) {
1995
+ throw new ParseError("Resume state file is corrupt; delete it to start over", {
1996
+ cause: parsed.error,
1997
+ path
1998
+ });
1999
+ }
2000
+ return parsed.data;
2001
+ }
2002
+ async save(path, state) {
2003
+ await atomicWriteJson(path, state);
2004
+ }
2005
+ async delete(path) {
2006
+ try {
2007
+ await (0, import_promises3.unlink)(path);
2008
+ } catch (err) {
2009
+ if (!isEnoent(err)) throw err;
2010
+ }
2011
+ }
2012
+ };
2013
+
2014
+ // src/storage/chapter-store.ts
2015
+ var import_promises4 = require("fs/promises");
2016
+ var import_node_path5 = require("path");
2017
+ var MemoryChapterStore = class {
2018
+ #map = /* @__PURE__ */ new Map();
2019
+ async save(chapter) {
2020
+ this.#map.set(Number(chapter.index), chapter);
2021
+ }
2022
+ async loadAll() {
2023
+ return [...this.#map.values()].sort((a, b) => Number(a.index) - Number(b.index));
2024
+ }
2025
+ };
2026
+ var FilesystemChapterStore = class {
2027
+ #logger;
2028
+ #dir;
2029
+ constructor(dir, opts = {}) {
2030
+ this.#dir = (0, import_node_path5.resolve)(dir);
2031
+ this.#logger = opts.logger ?? silentLogger;
2032
+ }
2033
+ async save(chapter) {
2034
+ await atomicWriteJson((0, import_node_path5.join)(this.#dir, `${Number(chapter.index)}.json`), chapter);
2035
+ }
2036
+ async loadAll() {
2037
+ let entries;
2038
+ try {
2039
+ entries = await (0, import_promises4.readdir)(this.#dir);
2040
+ } catch (err) {
2041
+ if (isEnoent(err)) return [];
2042
+ throw err;
2043
+ }
2044
+ const chapters = [];
2045
+ for (const name of entries) {
2046
+ if (!/^\d+\.json$/.test(name)) continue;
2047
+ const path = (0, import_node_path5.join)(this.#dir, name);
2048
+ let text;
2049
+ try {
2050
+ text = await (0, import_promises4.readFile)(path, "utf8");
2051
+ } catch (err) {
2052
+ this.#logger.warn({ path, err }, "chapter-store: failed to read file, skipping");
2053
+ continue;
2054
+ }
2055
+ let raw;
2056
+ try {
2057
+ raw = JSON.parse(text);
2058
+ } catch (err) {
2059
+ this.#logger.warn({ path, err }, "chapter-store: invalid JSON, skipping");
2060
+ continue;
2061
+ }
2062
+ const parsed = ChapterSchema.safeParse(raw);
2063
+ if (!parsed.success) {
2064
+ this.#logger.warn(
2065
+ { path, issues: parsed.error.issues },
2066
+ "chapter-store: schema mismatch, skipping"
2067
+ );
2068
+ continue;
2069
+ }
2070
+ chapters.push(parsed.data);
2071
+ }
2072
+ chapters.sort((a, b) => Number(a.index) - Number(b.index));
2073
+ return chapters;
2074
+ }
2075
+ };
2076
+
2077
+ // src/storage/resume-storage.ts
2078
+ var ResumeStorage = class _ResumeStorage {
2079
+ resume;
2080
+ chapters;
2081
+ stateFile;
2082
+ constructor(resume, chapters, stateFile) {
2083
+ this.resume = resume;
2084
+ this.chapters = chapters;
2085
+ this.stateFile = stateFile;
2086
+ }
2087
+ static forDefault(url, logger) {
2088
+ const stateFile = (0, import_node_path6.join)(userCacheDir(), "webnovel-downloader", "state", `${sha1(url)}.json`);
2089
+ return _ResumeStorage.#filesystem(stateFile, logger);
2090
+ }
2091
+ static forStateFile(stateFile, logger) {
2092
+ return _ResumeStorage.#filesystem(stateFile, logger);
2093
+ }
2094
+ static forToken(token, url, logger) {
2095
+ const { stateFile } = verifyResumeToken(token, url);
2096
+ return _ResumeStorage.#filesystem(stateFile, logger);
2097
+ }
2098
+ static memory(stateFile) {
2099
+ return new _ResumeStorage(new MemoryResumeStore(), new MemoryChapterStore(), stateFile);
2100
+ }
2101
+ static #filesystem(stateFile, logger) {
2102
+ return new _ResumeStorage(
2103
+ new FilesystemResumeStore(),
2104
+ new FilesystemChapterStore(`${stateFile}.chapters`, { logger }),
2105
+ stateFile
2106
+ );
2107
+ }
2108
+ };
2109
+ function resolveResumeStorage(opt, url, logger) {
2110
+ if (opt === void 0) return null;
2111
+ if (opt === true) return ResumeStorage.forDefault(url, logger);
2112
+ if ("stateFile" in opt) return ResumeStorage.forStateFile(opt.stateFile, logger);
2113
+ if ("token" in opt) return ResumeStorage.forToken(opt.token, url, logger);
2114
+ return null;
2115
+ }
2116
+
2032
2117
  // src/core/downloader.ts
2033
2118
  var DEFAULT_CACHE_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1e3;
2034
2119
  var DEFAULT_CONCURRENCY = 4;
@@ -2039,6 +2124,7 @@ var Downloader = class {
2039
2124
  #transport;
2040
2125
  #cookieJar;
2041
2126
  #browserPool;
2127
+ #httpDispatcher;
2042
2128
  #transportListeners = /* @__PURE__ */ new Set();
2043
2129
  /**
2044
2130
  * Build a downloader from a set of adapters. `rateLimit`, `retry`, and
@@ -2073,10 +2159,12 @@ var Downloader = class {
2073
2159
  await handle.release();
2074
2160
  }
2075
2161
  } : void 0;
2162
+ this.#httpDispatcher = opts.http === void 0 && opts.undiciOverride === void 0 ? new import_undici2.Agent() : void 0;
2076
2163
  this.#http = opts.http ?? buildHttpClient({
2077
2164
  logger: this.#logger,
2078
2165
  transport: this.#transport,
2079
2166
  cookieJar: this.#cookieJar,
2167
+ ...this.#httpDispatcher !== void 0 ? { dispatcher: this.#httpDispatcher } : {},
2080
2168
  ...opts.rateLimit !== void 0 ? { rateLimit: opts.rateLimit } : {},
2081
2169
  ...opts.retry !== void 0 ? { retry: opts.retry } : {},
2082
2170
  onEvent: (e) => {
@@ -2107,10 +2195,11 @@ var Downloader = class {
2107
2195
  ...onCookieCapture !== void 0 ? { onCookieCapture } : {}
2108
2196
  });
2109
2197
  }
2110
- /** Release held resources (closes the browser pool, if any). Safe to call once when done. */
2198
+ /** Release held resources (closes the browser pool and HTTP dispatcher, if any). Safe to call once when done. */
2111
2199
  async dispose() {
2112
2200
  const pool = this.#browserPool;
2113
2201
  if (pool) await pool.dispose();
2202
+ if (this.#httpDispatcher) await this.#httpDispatcher.close();
2114
2203
  }
2115
2204
  /** True when a built-in/registered adapter can handle this URL. */
2116
2205
  canHandle(url) {
@@ -2200,12 +2289,12 @@ var Downloader = class {
2200
2289
  };
2201
2290
  try {
2202
2291
  this.#transportListeners.add(transportListener);
2203
- const resumeCfg = resolveResumeConfig(options?.resume, validated, this.#logger);
2292
+ const resumeStorage = resolveResumeStorage(options?.resume, validated, this.#logger);
2204
2293
  const run = new DownloadRun({
2205
2294
  adapter,
2206
2295
  validated,
2207
2296
  normalizedRange,
2208
- resumeCfg,
2297
+ resumeStorage,
2209
2298
  concurrency,
2210
2299
  signal,
2211
2300
  ctx,
@@ -2307,13 +2396,6 @@ function normalizeText(html) {
2307
2396
  });
2308
2397
  return text.replace(new RegExp(LINE, "g"), " ").replace(new RegExp(PARA, "g"), "\n\n").split(/\n{2,}/).map((p) => p.replace(/\s+/g, " ").trim()).filter(Boolean).join("\n\n").trim();
2309
2398
  }
2310
- function pickText($, selectors) {
2311
- for (const sel of selectors) {
2312
- const t = $(sel).first().text().trim();
2313
- if (t) return t;
2314
- }
2315
- return "";
2316
- }
2317
2399
  function wordCount(text) {
2318
2400
  return text.split(/\s+/).filter(Boolean).length;
2319
2401
  }
@@ -2385,6 +2467,9 @@ function refsFromAnchors($, opts) {
2385
2467
  return refs;
2386
2468
  }
2387
2469
 
2470
+ // src/adapters/shared/header.ts
2471
+ init_errors();
2472
+
2388
2473
  // src/adapters/shared/cover.ts
2389
2474
  function extractCoverUrl($, selector, baseUrl) {
2390
2475
  const el = $(selector).first();
@@ -2392,6 +2477,36 @@ function extractCoverUrl($, selector, baseUrl) {
2392
2477
  return raw ? resolveUrl(baseUrl, raw) : void 0;
2393
2478
  }
2394
2479
 
2480
+ // src/adapters/shared/header.ts
2481
+ function parseNovelHeader($, selectors, sourceUrl) {
2482
+ const title = $(selectors.title).first().text().trim();
2483
+ if (!title) throw new ParseError("Missing title on novel page", { url: sourceUrl });
2484
+ const author = $(selectors.author).first().text().trim();
2485
+ const descSelectors = typeof selectors.description === "string" ? [selectors.description] : selectors.description;
2486
+ let description = "";
2487
+ for (const sel of descSelectors) {
2488
+ const t = $(sel).first().text().trim();
2489
+ if (t) {
2490
+ description = t;
2491
+ break;
2492
+ }
2493
+ }
2494
+ description = description.replace(/\s+/g, " ").trim();
2495
+ const genres = [];
2496
+ $(selectors.genres).each((_, el) => {
2497
+ const t = $(el).text().trim();
2498
+ if (t && !genres.includes(t)) genres.push(t);
2499
+ });
2500
+ const coverUrl = extractCoverUrl($, selectors.cover, sourceUrl);
2501
+ return {
2502
+ title,
2503
+ author,
2504
+ description,
2505
+ genres,
2506
+ ...coverUrl !== void 0 ? { coverUrl } : {}
2507
+ };
2508
+ }
2509
+
2395
2510
  // src/adapters/shared/content.ts
2396
2511
  init_errors();
2397
2512
  function stripLeadingTitle(content, title) {
@@ -2432,31 +2547,16 @@ var TRUYENFULL_SELECTORS = {
2432
2547
 
2433
2548
  // src/adapters/truyenfull/parser.ts
2434
2549
  function parseNovelPage($, sourceUrl) {
2435
- const title = $(TRUYENFULL_SELECTORS.metadata.title).first().text().trim();
2436
- if (!title) {
2437
- throw new ParseError("Missing title on novel page", { url: sourceUrl });
2438
- }
2439
- const author = $(TRUYENFULL_SELECTORS.metadata.author).first().text().trim();
2440
- const description = $(TRUYENFULL_SELECTORS.metadata.description).text().replace(/\s+/g, " ").trim();
2441
- const genres = [];
2442
- $(TRUYENFULL_SELECTORS.metadata.genres).each((_, el) => {
2443
- const t = $(el).text().trim();
2444
- if (t) genres.push(t);
2445
- });
2446
- const coverUrl = extractCoverUrl($, TRUYENFULL_SELECTORS.metadata.cover, sourceUrl);
2550
+ const header = parseNovelHeader($, TRUYENFULL_SELECTORS.metadata, sourceUrl);
2447
2551
  let status = "unknown";
2448
2552
  if ($(TRUYENFULL_SELECTORS.metadata.statusFull).length > 0) status = "completed";
2449
2553
  else if ($(TRUYENFULL_SELECTORS.metadata.statusOngoing).length > 0) status = "ongoing";
2450
2554
  return {
2555
+ ...header,
2451
2556
  sourceUrl,
2452
2557
  sourceSite: unsafeBrandAdapterId("truyenfull"),
2453
- title,
2454
- author,
2455
- description,
2456
- genres,
2457
2558
  status,
2458
- fetchedAt: /* @__PURE__ */ new Date(),
2459
- ...coverUrl !== void 0 ? { coverUrl } : {}
2559
+ fetchedAt: /* @__PURE__ */ new Date()
2460
2560
  };
2461
2561
  }
2462
2562
  function parseTocPage($, startIndex, baseUrl) {
@@ -2622,32 +2722,17 @@ var METRUYENCHU_SELECTORS = {
2622
2722
  // src/adapters/metruyenchu/parser.ts
2623
2723
  var SOURCE_SITE = unsafeBrandAdapterId("metruyenchu-com-vn");
2624
2724
  function parseNovelPage2($, sourceUrl) {
2625
- const title = $(METRUYENCHU_SELECTORS.metadata.title).first().text().trim();
2626
- if (!title) {
2627
- throw new ParseError("Missing title on novel page", { url: sourceUrl });
2628
- }
2629
- const author = $(METRUYENCHU_SELECTORS.metadata.author).first().text().trim();
2630
- const description = pickText($, METRUYENCHU_SELECTORS.metadata.description).replace(/\s+/g, " ").trim();
2631
- const genres = [];
2632
- $(METRUYENCHU_SELECTORS.metadata.genres).each((_, el) => {
2633
- const t = $(el).text().trim();
2634
- if (t && !genres.includes(t)) genres.push(t);
2635
- });
2636
- const coverUrl = extractCoverUrl($, METRUYENCHU_SELECTORS.metadata.cover, sourceUrl);
2725
+ const header = parseNovelHeader($, METRUYENCHU_SELECTORS.metadata, sourceUrl);
2637
2726
  let status = "unknown";
2638
2727
  if ($(METRUYENCHU_SELECTORS.metadata.statusFull).length > 0) status = "completed";
2639
2728
  else if (METRUYENCHU_SELECTORS.metadata.statusOngoing.some((sel) => $(sel).length > 0)) status = "ongoing";
2640
2729
  const totalChapters = extractTotalChapters($);
2641
2730
  return {
2731
+ ...header,
2642
2732
  sourceUrl,
2643
2733
  sourceSite: SOURCE_SITE,
2644
- title,
2645
- author,
2646
- description,
2647
- genres,
2648
2734
  status,
2649
2735
  fetchedAt: /* @__PURE__ */ new Date(),
2650
- ...coverUrl ? { coverUrl } : {},
2651
2736
  ...totalChapters !== void 0 ? { totalChapters } : {}
2652
2737
  };
2653
2738
  }
@@ -3078,6 +3163,7 @@ var downloader = createDownloader();
3078
3163
  builtinAdapters,
3079
3164
  createDownloader,
3080
3165
  createLogger,
3081
- downloader
3166
+ downloader,
3167
+ parseNovelHeader
3082
3168
  });
3083
3169
  //# sourceMappingURL=index.cjs.map