novel-full-scraper 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Emmanuel Zvinoera
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,32 @@
+# novel-full-scraper
+
+Interactive CLI to scrape web novels from novelfull.com and export them as EPUB files.
+
+## Installation
+
+​```bash
+npm install -g novel-full-scraper
+​```
+
+## Usage
+
+​```bash
+novel-scrape
+​```
+
+Follow the interactive prompts to search for a novel, choose a chapter range,
+and save the result as an EPUB.
+
+## Requirements
+
+- Node.js 18 or later
+
+## Disclaimer
+
+For personal and educational use only. Please respect novelfull.com's Terms of
+Service and support the original authors by purchasing official releases when
+available.
+
+## License
+
+MIT

package/bin/cli.js

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+import { main } from '../src/index.js';
+
+// Handle --help and --version without loading the whole app
+const args = process.argv.slice(2);
+
+if (args.includes('--version') || args.includes('-v')) {
+    const { readFileSync } = await import('node:fs');
+    const { fileURLToPath } = await import('node:url');
+    const { dirname, join } = await import('node:path');
+    const __dirname = dirname(fileURLToPath(import.meta.url));
+    const pkg = JSON.parse(
+        readFileSync(join(__dirname, '..', 'package.json'), 'utf8')
+    );
+    console.log(pkg.version);
+    process.exit(0);
+}
+
+if (args.includes('--help') || args.includes('-h')) {
+    console.log(`
+  novel-scrape — Interactive CLI to scrape web novels into EPUB files.
+
+  Usage:
+    novel-scrape              Launch interactive prompts
+
+  Options:
+    -h, --help                Show this help message
+    -v, --version             Show version number
+
+  Example:
+    $ novel-scrape
+
+  Disclaimer:
+    For personal/educational use only. Please respect novelfull.com's
+    Terms of Service and support the original authors of the novels.
+`);
+    process.exit(0);
+}
+
+// Handle Ctrl+C cleanly
+process.on('SIGINT', () => {
+    console.log('\nInterrupted. Exiting...');
+    process.exit(130);
+});
+
+main().catch((err) => {
+    console.error('\n❌ Unexpected error:', err.message);
+    process.exit(1);
+});

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "novel-full-scraper",
+  "version": "1.0.0",
+  "description": "Interactive CLI to scrape web novels from novelfull.com and export them as EPUB files.",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "novel-scrape": "./bin/cli.js"
+  },
+  "files": [
+    "bin",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "start": "node bin/cli.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "cli",
+    "epub",
+    "scraper",
+    "novel",
+    "web-novel",
+    "novelfull",
+    "puppeteer",
+    "ebook"
+  ],
+  "author": "Your Name <your.email@example.com>",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/YOUR_USERNAME/novel-full-scraper.git"
+  },
+  "bugs": {
+    "url": "https://github.com/YOUR_USERNAME/novel-full-scraper/issues"
+  },
+  "homepage": "https://github.com/YOUR_USERNAME/novel-full-scraper#readme",
+  "dependencies": {
+    "@clack/prompts": "^1.8.1",
+    "epub-gen-memory": "^1.1.2",
+    "picocolors": "^1.1.1",
+    "puppeteer": "^25.11.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  }
+}

package/src/epub.js

@@ -0,0 +1,54 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import pkg from 'epub-gen-memory';
+
+// epub-gen-memory ships as CJS; handle both interop shapes.
+const epub = pkg.default || pkg;
+
+/**
+ * Generate an EPUB file from an array of chapters.
+ *
+ * @param {Array<{title: string, content: string}>} chapters
+ * @param {object} meta
+ * @param {string} meta.fileName    Output file name (no extension)
+ * @param {string} meta.bookTitle
+ * @param {string} meta.filePath    Absolute output directory
+ * @param {string} [meta.author]
+ * @returns {Promise<string>}       Absolute path to the written file
+ */
+export async function createAndSaveEpub(chapters, meta) {
+    const {
+        fileName,
+        bookTitle,
+        filePath,
+        author = 'Unknown',
+        publisher = 'novel-full-scraper',
+    } = meta;
+
+    if (!chapters?.length) {
+        throw new Error('No chapters provided to createAndSaveEpub.');
+    }
+
+    const options = {
+        title: bookTitle,
+        author,
+        publisher,
+        // intentionally no `cover` — empty string trips the library
+    };
+
+    const chaptersData = chapters.map((ch) => ({
+        title: ch.title || 'Untitled Chapter',
+        content: ch.content ? ch.content.toString() : '<p>No content available.</p>',
+    }));
+
+    const epubBuffer = await epub(options, chaptersData);
+
+    if (!fs.existsSync(filePath)) {
+        fs.mkdirSync(filePath, { recursive: true });
+    }
+
+    const finalPath = path.join(filePath, `${fileName}.epub`);
+    fs.writeFileSync(finalPath, epubBuffer);
+
+    return finalPath;
+}

package/src/index.js

@@ -0,0 +1,177 @@
+import path from 'node:path';
+import fs from 'node:fs';
+import { setTimeout as sleep } from 'node:timers/promises';
+import * as p from '@clack/prompts';
+import color from 'picocolors';
+
+import {
+    launchBrowser,
+    getSearchResults,
+    getChapterLinks,
+    scrapeChaptersBatched,
+} from './scraper.js';
+import { createAndSaveEpub } from './epub.js';
+
+export async function main() {
+    console.clear();
+    await sleep(500);
+
+    p.intro(`${color.bgCyan(color.black(' novel-full-scraper '))}`);
+
+    const browser = await launchBrowser();
+    let links = [];
+
+    try {
+        const project = await p.group(
+            {
+                search: () =>
+                    p.text({
+                        message: 'What novel do you want to search?',
+                        placeholder: 'Lord of the Mysteries',
+                        validate: (v) =>
+                            v.trim().length < 2
+                                ? 'Please enter at least 2 characters.'
+                                : undefined,
+                    }),
+
+                novel: async ({ results }) => {
+                    const s = p.spinner();
+                    s.start(`Searching for "${results.search}"...`);
+                    const options = await getSearchResults(results.search, browser);
+                    s.stop(`Found ${options.length} match(es).`);
+
+                    if (options.length === 0) {
+                        p.cancel('No novels found. Try a different keyword.');
+                        process.exit(0);
+                    }
+
+                    return p.select({ message: 'Choose a novel', options });
+                },
+
+                start: async ({ results }) => {
+                    const s = p.spinner();
+                    s.start(`Fetching chapter list...`);
+                    links = await getChapterLinks(results.novel, browser);
+                    s.stop(`Found ${links.length} chapters.`);
+
+                    if (links.length === 0) {
+                        p.cancel('No chapters found for this novel.');
+                        process.exit(0);
+                    }
+
+                    return p.text({
+                        message: 'Start chapter index',
+                        placeholder: '1',
+                        validate(value) {
+                            const num = parseInt(value, 10);
+                            if (isNaN(num) || num < 1 || num > links.length) {
+                                return `Enter a number between 1 and ${links.length}`;
+                            }
+                        },
+                    });
+                },
+
+                end: ({ results }) => {
+                    const startNum = parseInt(results.start, 10);
+                    return p.text({
+                        message: 'End chapter index',
+                        placeholder: `${links.length}`,
+                        validate(value) {
+                            const num = parseInt(value, 10);
+                            if (isNaN(num) || num < startNum || num > links.length) {
+                                return `Enter a number between ${startNum} and ${links.length}`;
+                            }
+                        },
+                    });
+                },
+
+                filepath: () =>
+                    p.text({
+                        message: 'Where to save the file (absolute path)',
+                        placeholder:
+                            process.platform === 'win32'
+                                ? 'C:\\Users\\Name\\Documents\\Books'
+                                : '/Users/Name/Documents/Books',
+                        validate(value) {
+                            const cleaned = value.trim();
+                            if (!cleaned) return 'Path cannot be empty.';
+                            if (!path.isAbsolute(cleaned))
+                                return 'Path must be absolute.';
+                            if (
+                                process.platform !== 'win32' &&
+                                /[<>:"|?*]/.test(cleaned)
+                            ) {
+                                return 'Path contains invalid characters.';
+                            }
+                        },
+                    }),
+
+                filename: () =>
+                    p.text({
+                        message: 'File name (no extension)',
+                        placeholder: 'lord_of_the_mysteries',
+                        validate(value) {
+                            const cleaned = value.trim();
+                            if (cleaned.length <= 3)
+                                return 'File name must be more than 3 characters.';
+                            if (/[\\/:*?"<>|]/.test(cleaned))
+                                return 'File name contains invalid characters.';
+                        },
+                    }),
+            },
+            {
+                onCancel: () => {
+                    p.cancel('Operation cancelled.');
+                    browser.close();
+                    process.exit(0);
+                },
+            }
+        );
+
+        // ── Scrape ──
+        const startIdx = parseInt(project.start, 10) - 1;
+        const endIdx = parseInt(project.end, 10);
+        const targetLinks = links.slice(startIdx, endIdx);
+
+        const scrapeSpinner = p.spinner();
+        scrapeSpinner.start(`Scraping ${targetLinks.length} chapters...`);
+
+        const chapters = await scrapeChaptersBatched(targetLinks, browser, {
+            batchSize: 5,
+            delayMs: 1000,
+            onBatch: (from, to, total) => {
+                scrapeSpinner.message(
+                    `Scraping chapters ${from + 1}–${to} of ${total}...`
+                );
+            },
+        });
+
+        scrapeSpinner.stop(`Compiled ${chapters.length} chapters.`);
+
+        // ── Build EPUB ──
+        const targetDirectory = project.filepath.trim();
+        if (!fs.existsSync(targetDirectory)) {
+            fs.mkdirSync(targetDirectory, { recursive: true });
+        }
+
+        const epubSpinner = p.spinner();
+        epubSpinner.start('Generating EPUB file...');
+
+        try {
+            const finalPath = await createAndSaveEpub(chapters, {
+                fileName: project.filename,
+                bookTitle: project.novel,
+                filePath: targetDirectory,
+            });
+            epubSpinner.stop(`📦 EPUB saved to: ${finalPath}`);
+        } catch (err) {
+            epubSpinner.stop('❌ Failed to create EPUB.');
+            p.cancel(`Error building book: ${err.message}`);
+            process.exit(1);
+        }
+
+        p.outro(`${color.green('Done!')} Thanks for using novel-full-scraper.`);
+    } finally {
+        await browser.close();
+    }
+}

package/src/scraper.js

@@ -0,0 +1,151 @@
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+puppeteer.use(StealthPlugin());
+
+const BASE_URL = 'https://novelfull.com';
+
+/**
+ * Launch a single reusable browser. Callers must close it.
+ */
+export async function launchBrowser() {
+    return puppeteer.launch({ headless: true });
+}
+
+/**
+ * Search for novels by keyword. Returns array of { label, value }
+ * ready to feed to @clack/prompts' select().
+ */
+export async function getSearchResults(novel, browser) {
+    const page = await browser.newPage();
+    try {
+        const query = novel.replace(/ /g, '+');
+        const url = `${BASE_URL}/search?keyword=${query}`;
+
+        await page.goto(url, { waitUntil: 'domcontentloaded' });
+        await page.waitForSelector('.list-truyen .row', { timeout: 5000 });
+
+        return await page.evaluate(() => {
+            const linkElements = document.querySelectorAll('h3.truyen-title a');
+            return Array.from(linkElements).map((book) => ({
+                label: book.textContent.trim(),
+                value: book.getAttribute('href'),
+            }));
+        });
+    } finally {
+        await page.close();
+    }
+}
+
+/**
+ * Scrape every chapter link across all paginated pages of a novel.
+ */
+export async function getChapterLinks(relativePath, browser) {
+    const page = await browser.newPage();
+    try {
+        const url = `${BASE_URL}${relativePath}`;
+        await page.goto(url, { waitUntil: 'domcontentloaded' });
+        await page.waitForSelector('.l-chapter', { timeout: 5000 });
+
+        const latest = await page.evaluate(() => {
+            const latestChapter = document.querySelector('.l-chapters li a');
+            const num = latestChapter.getAttribute('title').split(' ')[1];
+            return parseInt(num, 10);
+        });
+
+        const CHAPTERS_PER_PAGE = 50;
+        const maxPages = Math.ceil(latest / CHAPTERS_PER_PAGE);
+        const allChapterLinks = [];
+
+        for (let currentPage = 1; currentPage <= maxPages; currentPage++) {
+            const pageUrl = `${BASE_URL}${relativePath}?page=${currentPage}`;
+            await page.goto(pageUrl, { waitUntil: 'domcontentloaded' });
+            await page.waitForSelector('.l-chapter', { timeout: 5000 });
+
+            const chapterLinks = await page.evaluate(() => {
+                const chapters = document.querySelectorAll('ul.list-chapter li a');
+                return Array.from(chapters).map((c) => c.getAttribute('href'));
+            });
+
+            allChapterLinks.push(...chapterLinks);
+        }
+
+        return allChapterLinks;
+    } finally {
+        await page.close();
+    }
+}
+
+/**
+ * Fetch the HTML content of a single chapter. Returns { title, content }.
+ */
+export async function getChapterContents(relativeUrl, browser) {
+    const page = await browser.newPage();
+    try {
+        await page.goto(`${BASE_URL}${relativeUrl}`, {
+            waitUntil: 'domcontentloaded',
+        });
+
+        const chapterContents = await page.evaluate(() => {
+            const container = document.querySelector('.chapter-c');
+            if (!container) return '';
+
+            // Strip ad blocks
+            container
+                .querySelectorAll('div[align="center"], script, ins')
+                .forEach((el) => el.remove());
+
+            return container.innerHTML;
+        });
+
+        const filename = relativeUrl.split('/').pop().replace('.html', '');
+        const parsedTitle = filename
+            .split('-')
+            .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+            .join(' ');
+
+        return {
+            title: parsedTitle,
+            content: chapterContents || '<p>No content available.</p>',
+        };
+    } finally {
+        await page.close();
+    }
+}
+
+/**
+ * Scrape a range of chapters in batches with delay between batches.
+ */
+export async function scrapeChaptersBatched(
+    links,
+    browser,
+    { batchSize = 5, delayMs = 1000, onBatch = () => {} } = {}
+) {
+    const results = [];
+
+    for (let i = 0; i < links.length; i += batchSize) {
+        const batch = links.slice(i, i + batchSize);
+        onBatch(i, Math.min(i + batchSize, links.length), links.length);
+
+        const batchResults = await Promise.all(
+            batch.map(async (link) => {
+                try {
+                    return await getChapterContents(link, browser);
+                } catch (err) {
+                    return {
+                        title: `Failed: ${link}`,
+                        content: `<p>Failed to fetch: ${err.message}</p>`,
+                    };
+                }
+            })
+        );
+
+        results.push(...batchResults);
+
+        if (i + batchSize < links.length) {
+            await new Promise((r) => setTimeout(r, delayMs));
+        }
+    }
+
+    return results;
+}