node-pdf2img 0.1.7 → 0.1.9

package/src/core/output-handler.js

@@ -0,0 +1,179 @@
+/**
+ * 输出处理模块
+ * 
+ * 负责将渲染结果保存到文件或上传到 COS
+ */
+
+import fs from 'fs';
+import path from 'path';
+import pLimit from 'p-limit';
+import { createLogger } from '../utils/logger.js';
+import { getExtension, getMimeType } from './config.js';
+
+const logger = createLogger('OutputHandler');
+
+/**
+ * 默认并发限制
+ */
+export const DEFAULT_CONCURRENCY = {
+    FILE_IO: 10,      // 文件写入并发数
+    COS_UPLOAD: 4,    // COS 上传并发数（降低以减少 EPIPE 错误）
+};
+
+/**
+ * 延迟函数
+ */
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * 保存单个页面到文件
+ */
+async function savePageToFile(page, outputDir, prefix, ext) {
+    if (!page.success || !page.buffer) {
+        return { ...page, outputPath: null };
+    }
+
+    try {
+        const filename = `${prefix}_${page.pageNum}.${ext}`;
+        const outputPath = path.join(outputDir, filename);
+        await fs.promises.writeFile(outputPath, page.buffer);
+
+        return {
+            pageNum: page.pageNum,
+            width: page.width,
+            height: page.height,
+            success: true,
+            outputPath,
+            size: page.buffer.length,
+        };
+    } catch (err) {
+        return {
+            pageNum: page.pageNum,
+            width: page.width,
+            height: page.height,
+            success: false,
+            error: `File save failed: ${err.message}`,
+            outputPath: null,
+        };
+    }
+}
+
+/**
+ * 保存渲染结果到文件
+ * 
+ * @param {Array} pages - 渲染结果数组
+ * @param {string} outputDir - 输出目录
+ * @param {string} prefix - 文件名前缀
+ * @param {string} format - 输出格式
+ * @param {number} concurrency - 并发数
+ * @returns {Promise<Array>} 保存结果
+ */
+export async function saveToFiles(pages, outputDir, prefix = 'page', format = 'webp', concurrency = DEFAULT_CONCURRENCY.FILE_IO) {
+    await fs.promises.mkdir(outputDir, { recursive: true });
+
+    const ext = getExtension(format);
+    const limit = pLimit(concurrency);
+
+    const results = await Promise.all(
+        pages.map(page => limit(() => savePageToFile(page, outputDir, prefix, ext)))
+    );
+
+    return results.sort((a, b) => a.pageNum - b.pageNum);
+}
+
+/**
+ * 上传单个页面到 COS（带重试）
+ */
+async function uploadPageToCos(page, cos, cosConfig, keyPrefix, ext, mimeType, maxRetries = 3) {
+    if (!page.success || !page.buffer) {
+        return { ...page, cosKey: null };
+    }
+
+    const key = `${keyPrefix}/page_${page.pageNum}.${ext}`;
+    let lastError;
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            await new Promise((resolve, reject) => {
+                cos.putObject({
+                    Bucket: cosConfig.bucket,
+                    Region: cosConfig.region,
+                    Key: key,
+                    Body: page.buffer,
+                    ContentType: mimeType,
+                }, (err) => {
+                    if (err) reject(err);
+                    else resolve();
+                });
+            });
+
+            return {
+                pageNum: page.pageNum,
+                width: page.width,
+                height: page.height,
+                success: true,
+                cosKey: key,
+                size: page.buffer.length,
+            };
+        } catch (err) {
+            lastError = err;
+            const isRetryable = err.code === 'EPIPE' || 
+                               err.code === 'ECONNRESET' || 
+                               err.code === 'ETIMEDOUT' ||
+                               err.code === 'ECONNREFUSED' ||
+                               (err.statusCode && err.statusCode >= 500);
+            
+            if (isRetryable && attempt < maxRetries) {
+                // 指数退避：1s, 2s, 4s...
+                const delay = Math.pow(2, attempt - 1) * 1000;
+                logger.debug(`Page ${page.pageNum} upload failed (${err.code || err.message}), retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`);
+                await sleep(delay);
+            } else if (!isRetryable) {
+                break;
+            }
+        }
+    }
+
+    return {
+        pageNum: page.pageNum,
+        width: page.width,
+        height: page.height,
+        success: false,
+        error: `Upload failed after ${maxRetries} attempts: ${lastError.message}`,
+        cosKey: null,
+    };
+}
+
+/**
+ * 上传渲染结果到 COS
+ * 
+ * @param {Array} pages - 渲染结果数组
+ * @param {Object} cosConfig - COS 配置
+ * @param {string} keyPrefix - COS key 前缀
+ * @param {string} format - 输出格式
+ * @param {number} concurrency - 并发数
+ * @returns {Promise<Array>} 上传结果
+ */
+export async function uploadToCos(pages, cosConfig, keyPrefix, format = 'webp', concurrency = DEFAULT_CONCURRENCY.COS_UPLOAD) {
+    const COS = (await import('cos-nodejs-sdk-v5')).default;
+
+    const cos = new COS({
+        SecretId: cosConfig.secretId,
+        SecretKey: cosConfig.secretKey,
+        Protocol: cosConfig.protocol || 'https:',
+        ServiceDomain: cosConfig.serviceDomain,
+        Domain: cosConfig.domain,
+    });
+
+    const ext = getExtension(format);
+    const mimeType = getMimeType(format);
+    const limit = pLimit(concurrency);
+
+    const results = await Promise.all(
+        pages.map(page => limit(() => uploadPageToCos(page, cos, cosConfig, keyPrefix, ext, mimeType)))
+    );
+
+    return results.sort((a, b) => a.pageNum - b.pageNum);
+}

package/src/core/renderer.js

@@ -0,0 +1,289 @@
+/**
+ * PDF 渲染模块
+ * 
+ * 负责 PDF 页面的渲染逻辑，支持本地文件、Buffer 和 URL 输入
+ * 支持两种渲染器：
+ * - pdfium: PDFium 原生渲染器（默认，高性能）
+ * - pdfjs: PDF.js 渲染器（纯 JavaScript，无需原生依赖）
+ */
+
+import fs from 'fs';
+import { createLogger } from '../utils/logger.js';
+import { RendererType, DEFAULT_RENDERER } from './config.js';
+import * as nativeRenderer from '../renderers/native.js';
+import * as pdfjsRenderer from '../renderers/pdfjs.js';
+import { getThreadPool, getThreadCount } from './thread-pool.js';
+import { getRemoteFileSize, downloadToTempFile } from './downloader.js';
+
+const logger = createLogger('Renderer');
+
+/**
+ * 输入类型枚举
+ */
+export const InputType = {
+    FILE: 'file',
+    URL: 'url',
+    BUFFER: 'buffer',
+};
+
+/**
+ * 流式渲染阈值（小于此值使用下载模式）
+ */
+const STREAM_THRESHOLD = 2 * 1024 * 1024; // 2MB
+
+/**
+ * 检测输入类型
+ * 
+ * @param {string|Buffer} input - 输入
+ * @returns {string} 输入类型
+ */
+export function detectInputType(input) {
+    if (Buffer.isBuffer(input)) {
+        return InputType.BUFFER;
+    }
+    if (typeof input === 'string') {
+        if (input.startsWith('http://') || input.startsWith('https://')) {
+            return InputType.URL;
+        }
+        return InputType.FILE;
+    }
+    throw new Error('Invalid input: must be a file path, URL, or Buffer');
+}
+
+/**
+ * 获取当前使用的渲染器类型
+ * 
+ * @param {Object} options - 选项
+ * @returns {string} 渲染器类型
+ */
+export function getRendererType(options = {}) {
+    const renderer = options.renderer || DEFAULT_RENDERER;
+    
+    // 验证渲染器类型
+    if (renderer === RendererType.PDFJS) {
+        return RendererType.PDFJS;
+    }
+    
+    // 默认使用 pdfium，如果不可用则回退到 pdfjs
+    if (!nativeRenderer.isNativeAvailable()) {
+        logger.warn('PDFium 渲染器不可用，回退到 PDF.js');
+        return RendererType.PDFJS;
+    }
+    
+    return RendererType.PDFIUM;
+}
+
+/**
+ * 使用指定渲染器渲染 PDF 页面
+ * 
+ * @param {string|Buffer} input - 输入
+ * @param {string} inputType - 输入类型
+ * @param {number[]} pages - 页码数组
+ * @param {Object} options - 选项
+ * @returns {Promise<Object>} 渲染结果
+ */
+export async function renderPages(input, inputType, pages, options) {
+    const startTime = Date.now();
+    const rendererType = getRendererType(options);
+    
+    logger.debug(`Using renderer: ${rendererType}`);
+    
+    if (rendererType === RendererType.PDFJS) {
+        return renderPagesWithPdfjs(input, inputType, pages, options, startTime);
+    }
+    
+    // 使用 pdfium 渲染器
+    if (inputType === InputType.URL) {
+        return renderPagesFromUrl(input, pages, options, startTime);
+    }
+    
+    return renderPagesFromLocal(input, inputType, pages, options, startTime);
+}
+
+/**
+ * 使用 PDF.js 渲染器渲染 PDF 页面
+ */
+async function renderPagesWithPdfjs(input, inputType, pages, options, startTime) {
+    let result;
+    
+    if (inputType === InputType.URL) {
+        result = await pdfjsRenderer.renderFromUrl(input, pages, options);
+    } else if (inputType === InputType.BUFFER) {
+        result = await pdfjsRenderer.renderFromBuffer(input, pages, options);
+    } else {
+        result = await pdfjsRenderer.renderFromFile(input, pages, options);
+    }
+    
+    if (!result.success) {
+        throw new Error(result.error || 'PDF.js 渲染失败');
+    }
+    
+    return {
+        success: true,
+        numPages: result.numPages,
+        pages: result.pages,
+        totalTime: Date.now() - startTime,
+        renderTime: result.renderTime || result.pages.reduce((sum, p) => sum + (p.renderTime || 0), 0),
+        encodeTime: result.pages.reduce((sum, p) => sum + (p.encodeTime || 0), 0),
+        renderer: RendererType.PDFJS,
+        streamStats: result.streamStats,
+    };
+}
+
+/**
+ * 从 URL 渲染 PDF 页面（流式）
+ * 
+ * 使用 HTTP Range 请求按需获取数据，避免完整下载
+ */
+async function renderPagesFromUrl(url, pages, options, startTime) {
+    // 获取文件大小
+    const fileSize = await getRemoteFileSize(url);
+    
+    // 小文件直接下载后渲染，避免多次 Range 请求开销
+    if (fileSize < STREAM_THRESHOLD) {
+        logger.debug(`Remote file size: ${(fileSize / 1024 / 1024).toFixed(2)}MB (< 2MB), using download mode`);
+        return renderPagesWithDownload(url, pages, options, startTime);
+    }
+
+    logger.debug(`Remote file size: ${(fileSize / 1024 / 1024).toFixed(2)}MB, using stream rendering`);
+
+    try {
+        // 使用流式渲染
+        const result = await nativeRenderer.renderFromStream(url, fileSize, pages, options);
+
+        return {
+            success: true,
+            numPages: result.numPages,
+            pages: result.pages,
+            totalTime: Date.now() - startTime,
+            renderTime: result.pages.reduce((sum, p) => sum + (p.renderTime || 0), 0),
+            encodeTime: result.pages.reduce((sum, p) => sum + (p.encodeTime || 0), 0),
+            streamStats: result.streamStats,
+            renderer: RendererType.PDFIUM,
+        };
+    } catch (err) {
+        // 流式渲染失败，回退到下载后渲染
+        logger.warn(`Stream rendering failed: ${err.message}, falling back to download`);
+        return renderPagesWithDownload(url, pages, options, startTime);
+    }
+}
+
+/**
+ * 下载后渲染（回退方案）
+ */
+async function renderPagesWithDownload(url, pages, options, startTime) {
+    const tempFile = await downloadToTempFile(url);
+    const threadCount = getThreadCount();
+    
+    try {
+        const numPages = nativeRenderer.getPageCountFromFile(tempFile);
+        
+        // 确定目标页码
+        let targetPages;
+        if (pages.length === 0) {
+            targetPages = Array.from({ length: numPages }, (_, i) => i + 1);
+        } else {
+            targetPages = pages.filter(p => p >= 1 && p <= numPages);
+        }
+
+        logger.debug(`Rendering ${targetPages.length} pages using thread pool (${threadCount} workers)`);
+
+        const pool = getThreadPool();
+        
+        const tasks = targetPages.map(pageNum => {
+            return pool.run({
+                pageNum,
+                options,
+                filePath: tempFile,
+            });
+        });
+
+        const results = await Promise.all(tasks);
+        results.sort((a, b) => a.pageNum - b.pageNum);
+
+        return {
+            success: true,
+            numPages,
+            pages: results,
+            totalTime: Date.now() - startTime,
+            renderTime: results.reduce((sum, p) => sum + (p.renderTime || 0), 0),
+            encodeTime: results.reduce((sum, p) => sum + (p.encodeTime || 0), 0),
+            renderer: RendererType.PDFIUM,
+        };
+    } finally {
+        try {
+            await fs.promises.unlink(tempFile);
+        } catch {}
+    }
+}
+
+/**
+ * 从本地文件或 Buffer 渲染 PDF 页面
+ */
+async function renderPagesFromLocal(input, inputType, pages, options, startTime) {
+    let filePath = null;
+    let pdfBuffer = null;
+    let numPages;
+    const threadCount = getThreadCount();
+
+    // 准备输入
+    if (inputType === InputType.FILE) {
+        try {
+            await fs.promises.access(input, fs.constants.R_OK);
+        } catch {
+            throw new Error(`File not found or not readable: ${input}`);
+        }
+        filePath = input;
+        numPages = nativeRenderer.getPageCountFromFile(filePath);
+    } else if (inputType === InputType.BUFFER) {
+        pdfBuffer = Buffer.isBuffer(input) ? input : Buffer.from(input);
+        numPages = nativeRenderer.getPageCount(pdfBuffer);
+    }
+
+    // 确定目标页码
+    let targetPages;
+    if (pages.length === 0) {
+        targetPages = Array.from({ length: numPages }, (_, i) => i + 1);
+    } else {
+        targetPages = pages.filter(p => p >= 1 && p <= numPages);
+    }
+
+    logger.debug(`Rendering ${targetPages.length} pages using thread pool (${threadCount} workers)`);
+
+    // 获取线程池
+    const pool = getThreadPool();
+
+    // 为每一页创建任务并提交到线程池
+    const tasks = targetPages.map(pageNum => {
+        const task = {
+            pageNum,
+            options,
+        };
+        
+        if (filePath) {
+            task.filePath = filePath;
+        } else if (pdfBuffer) {
+            // 注意：Buffer 会被序列化传递给工作线程
+            // 对于大文件，建议先保存到临时文件再传递路径
+            task.pdfBuffer = pdfBuffer;
+        }
+        
+        // 提交任务到线程池
+        return pool.run(task);
+    });
+
+    // 等待所有页面的并行处理完成
+    const results = await Promise.all(tasks);
+
+    results.sort((a, b) => a.pageNum - b.pageNum);
+
+    return {
+        success: true,
+        numPages,
+        pages: results,
+        totalTime: Date.now() - startTime,
+        renderTime: results.reduce((sum, p) => sum + (p.renderTime || 0), 0),
+        encodeTime: results.reduce((sum, p) => sum + (p.encodeTime || 0), 0),
+        renderer: RendererType.PDFIUM,
+    };
+}

package/src/core/thread-pool.js

@@ -0,0 +1,78 @@
+/**
+ * 线程池管理模块
+ * 
+ * 使用 Piscina 管理工作线程池，用于 CPU 密集型任务
+ */
+
+import os from 'os';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import Piscina from 'piscina';
+import { createLogger } from '../utils/logger.js';
+
+const logger = createLogger('ThreadPool');
+
+// 获取 worker.js 的路径
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const workerPath = path.resolve(__dirname, '../worker.js');
+
+// 线程数默认为 CPU 核心数，可通过环境变量调整
+const threadCount = parseInt(process.env.PDF2IMG_THREAD_COUNT, 10) || os.cpus().length;
+
+let piscina = null;
+
+/**
+ * 获取或创建线程池实例（懒加载）
+ */
+export function getThreadPool() {
+    if (!piscina) {
+        piscina = new Piscina({
+            filename: workerPath,
+            maxThreads: threadCount,
+            idleTimeout: 30000, // 空闲 30 秒后销毁线程
+        });
+        logger.info(`Thread pool initialized with ${threadCount} workers`);
+    }
+    return piscina;
+}
+
+/**
+ * 获取线程数
+ */
+export function getThreadCount() {
+    return threadCount;
+}
+
+/**
+ * 获取线程池统计信息
+ */
+export function getThreadPoolStats() {
+    if (!piscina) {
+        return {
+            initialized: false,
+            workers: threadCount,
+        };
+    }
+    return {
+        initialized: true,
+        workers: threadCount,
+        completed: piscina.completed,
+        waitTime: piscina.waitTime,
+        runTime: piscina.runTime,
+        utilization: piscina.utilization,
+    };
+}
+
+/**
+ * 销毁线程池
+ * 
+ * 在应用关闭时调用，释放工作线程资源
+ */
+export async function destroyThreadPool() {
+    if (piscina) {
+        await piscina.destroy();
+        piscina = null;
+        logger.info('Thread pool destroyed');
+    }
+}