foliko 1.1.13 → 1.1.15

package/src/utils/download.js

@@ -0,0 +1,596 @@
+const https = require('https');
+const http = require('http');
+const fs = require('fs');
+const fsPromises = require('fs/promises');
+const path = require('path');
+const crypto = require('crypto');
+const { fileTypeFromFile, fileTypeFromBuffer } = require('file-type');
+const { downloadAndDecryptMedia } = require('@chnak/weixin-bot');
+class FileDownloader {
+  constructor(options = {}) {
+    this.timeout = options.timeout || 30000;
+    this.retries = options.retries || 3;
+    this.headers = options.headers || {};
+    this.baseDir = options.baseDir || './downloads'; // 基础下载目录
+  }
+
+  /**
+   * 根据文件类型获取保存目录
+   */
+  _getTypeDir(mimeType, extension) {
+    const typeMap = {
+      // 图片
+      'image/jpeg': 'images',
+      'image/jpg': 'images',
+      'image/png': 'images',
+      'image/gif': 'images',
+      'image/webp': 'images',
+      'image/svg+xml': 'images',
+
+      // 视频
+      'video/mp4': 'videos',
+      'video/webm': 'videos',
+      'video/avi': 'videos',
+      'video/mov': 'videos',
+      'video/mkv': 'videos',
+      'video/x-msvideo': 'videos',
+      'video/quicktime': 'videos',
+
+      // 音频
+      'audio/mpeg': 'audio',
+      'audio/mp3': 'audio',
+      'audio/wav': 'audio',
+      'audio/ogg': 'audio',
+      'audio/m4a': 'audio',
+      'audio/flac': 'audio',
+      'audio/aac': 'audio',
+
+      // 文档
+      'application/pdf': 'documents',
+      'application/msword': 'documents',
+      'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'documents',
+      'application/vnd.ms-excel': 'documents',
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'documents',
+      'application/vnd.ms-powerpoint': 'documents',
+      'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'documents',
+      'text/plain': 'documents',
+      'text/markdown': 'documents',
+      'text/csv': 'documents',
+
+      // 压缩包
+      'application/zip': 'archives',
+      'application/x-rar-compressed': 'archives',
+      'application/x-7z-compressed': 'archives',
+      'application/x-tar': 'archives',
+      'application/gzip': 'archives',
+      'application/x-bzip2': 'archives',
+
+      // 可执行文件
+      'application/x-msdownload': 'executables',
+      'application/x-sh': 'executables',
+      'application/x-executable': 'executables',
+
+      // 代码文件
+      'text/html': 'code',
+      'text/css': 'code',
+      'text/javascript': 'code',
+      'application/javascript': 'code',
+      'application/json': 'code',
+      'text/xml': 'code',
+      'application/xml': 'code',
+      'text/x-python': 'code',
+      'text/x-java': 'code',
+      'text/x-c': 'code',
+      'text/x-c++': 'code',
+    };
+
+    // 先尝试根据 MIME 类型判断
+    if (mimeType && typeMap[mimeType]) {
+      return typeMap[mimeType];
+    }
+
+    // 如果 MIME 类型未匹配，根据扩展名判断
+    const extMap = {
+      // 图片
+      '.jpg': 'images',
+      '.jpeg': 'images',
+      '.png': 'images',
+      '.gif': 'images',
+      '.webp': 'images',
+      '.svg': 'images',
+      '.bmp': 'images',
+      '.ico': 'images',
+
+      // 视频
+      '.mp4': 'videos',
+      '.webm': 'videos',
+      '.avi': 'videos',
+      '.mov': 'videos',
+      '.mkv': 'videos',
+      '.flv': 'videos',
+      '.wmv': 'videos',
+      '.m4v': 'videos',
+      '.3gp': 'videos',
+
+      // 音频
+      '.mp3': 'audio',
+      '.wav': 'audio',
+      '.ogg': 'audio',
+      '.m4a': 'audio',
+      '.flac': 'audio',
+      '.aac': 'audio',
+      '.wma': 'audio',
+      '.opus': 'audio',
+      '.silk': 'audio',
+
+      // 文档
+      '.pdf': 'documents',
+      '.doc': 'documents',
+      '.docx': 'documents',
+      '.xls': 'documents',
+      '.xlsx': 'documents',
+      '.ppt': 'documents',
+      '.pptx': 'documents',
+      '.txt': 'documents',
+      '.md': 'documents',
+      '.csv': 'documents',
+      '.rtf': 'documents',
+      '.odt': 'documents',
+
+      // 压缩包
+      '.zip': 'archives',
+      '.rar': 'archives',
+      '.7z': 'archives',
+      '.tar': 'archives',
+      '.gz': 'archives',
+      '.bz2': 'archives',
+      '.xz': 'archives',
+      '.tgz': 'archives',
+
+      // 可执行文件
+      '.exe': 'executables',
+      '.msi': 'executables',
+      '.sh': 'executables',
+      '.bat': 'executables',
+      '.cmd': 'executables',
+      '.app': 'executables',
+      '.deb': 'executables',
+      '.rpm': 'executables',
+      '.dmg': 'executables',
+
+      // 代码
+      '.html': 'code',
+      '.htm': 'code',
+      '.css': 'code',
+      '.js': 'code',
+      '.json': 'code',
+      '.xml': 'code',
+      '.yaml': 'code',
+      '.yml': 'code',
+      '.py': 'code',
+      '.java': 'code',
+      '.c': 'code',
+      '.cpp': 'code',
+      '.h': 'code',
+      '.php': 'code',
+      '.rb': 'code',
+      '.go': 'code',
+      '.rs': 'code',
+      '.ts': 'code',
+      '.jsx': 'code',
+      '.tsx': 'code',
+      '.vue': 'code',
+      '.sql': 'code',
+      '.sh': 'code',
+    };
+
+    if (extension && extMap[extension.toLowerCase()]) {
+      return extMap[extension.toLowerCase()];
+    }
+
+    // 默认目录
+    return 'others';
+  }
+
+  /**
+   * 从 URL 或 Content-Type 提取文件信息
+   */
+  _extractFileInfo(url, contentType, fileExt) {
+    const urlObj = new URL(url);
+    let filename = path.basename(urlObj.pathname);
+    // 如果没有文件名，生成一个
+    if (!filename || !filename.includes('.')) {
+      const timestamp = Date.now();
+      let ext = fileExt || this._getExtensionFromMime(contentType);
+
+      // Content-Type 为空时，尝试从 URL 参数推断
+      if (!ext) {
+        const taskid = urlObj.searchParams.get('taskid');
+        if (taskid) {
+          ext = path.extname(taskid) || '';
+        }
+      }
+      // 仍然没有扩展名，默认 .jpg（微信图片通常是 jpeg）
+      if (!ext) {
+        ext = '.jpg';
+      }
+
+      filename = `file_${timestamp}.${ext}`;
+    }
+
+    // 清理文件名中的特殊字符
+    filename = this._sanitizeFilename(filename);
+
+    const extension = path.extname(filename);
+
+    return { filename, extension };
+  }
+
+  /**
+   * 根据 MIME 类型获取扩展名
+   */
+  _getExtensionFromMime(mimeType) {
+    if (!mimeType) return '';
+
+    const mimeMap = {
+      'image/jpeg': '.jpg',
+      'image/png': '.png',
+      'image/gif': '.gif',
+      'image/webp': '.webp',
+      'image/svg+xml': '.svg',
+      'video/mp4': '.mp4',
+      'video/webm': '.webm',
+      'video/quicktime': '.mov',
+      'audio/mpeg': '.mp3',
+      'audio/wav': '.wav',
+      'audio/ogg': '.ogg',
+      'audio/silk': '.silk',
+      'application/pdf': '.pdf',
+      'application/zip': '.zip',
+      'text/html': '.html',
+      'text/plain': '.txt',
+      'application/json': '.json',
+    };
+
+    return mimeMap[mimeType] || '';
+  }
+
+  /**
+   * 清理文件名中的非法字符
+   */
+  _sanitizeFilename(filename) {
+    return filename
+      .replace(/[<>:"/\\|?*]/g, '_') // Windows 非法字符
+      .replace(/\s+/g, '_') // 空格转下划线
+      .replace(/_+/g, '_') // 多个下划线合并
+      .substring(0, 255); // 限制长度
+  }
+
+  /**
+   * 生成唯一文件名（避免重名）
+   */
+  async _getUniqueFilePath(dir, filename) {
+    const ext = path.extname(filename);
+    const baseName = path.basename(filename, ext);
+    let filePath = path.join(dir, filename);
+    let counter = 1;
+
+    while (true) {
+      try {
+        await fsPromises.access(filePath);
+        // 文件存在，添加序号
+        filePath = path.join(dir, `${baseName}_${counter}${ext}`);
+        counter++;
+      } catch {
+        // 文件不存在，可以使用
+        break;
+      }
+    }
+
+    return filePath;
+  }
+
+  /**
+   * 下载文件（自动分类保存）
+   */
+  async download(url, options = {}) {
+    const {
+      customFilename,
+      customDir,
+      fileExt,
+      aes_key,
+      onProgress,
+      retries = this.retries,
+      skipExisting = true,
+    } = options;
+
+    let lastError;
+
+    for (let i = 0; i < retries; i++) {
+      try {
+        return await this._doDownload(url, {
+          customFilename,
+          customDir,
+          fileExt,
+          aes_key,
+          onProgress,
+          skipExisting,
+        });
+      } catch (error) {
+        lastError = error;
+        console.log(`下载失败，第 ${i + 1}/${retries} 次重试...`);
+        await this._sleep(1000 * (i + 1));
+      }
+    }
+
+    throw lastError;
+  }
+
+  async fromWeXin(url, aesKey, { customFilename, customDir, fileExt } = {}) {
+    try {
+      // 1. 下载并解密
+      const buffer = await downloadAndDecryptMedia({ url, aesKey });
+
+      // 2. 检测文件类型
+      let ext, mime;
+      if (fileExt) {
+        // 如果手动指定了扩展名
+        ext = fileExt.startsWith('.') ? fileExt.slice(1) : fileExt;
+        mime = null;
+      } else {
+        const fileType = await fileTypeFromBuffer(buffer);
+        ext = fileType?.ext || 'bin';
+        mime = fileType?.mime || 'application/octet-stream';
+      }
+
+      // 3. 确定目标目录
+      let targetDir;
+      if (customDir) {
+        targetDir = customDir;
+      } else {
+        const typeDir = this._getTypeDir(mime, `.${ext}`);
+        targetDir = path.join(this.baseDir, typeDir);
+      }
+
+      // 4. 创建目录
+      await fsPromises.mkdir(targetDir, { recursive: true });
+
+      // 5. 生成文件名
+      let filename;
+      if (customFilename) {
+        // 确保自定义文件名有正确的扩展名
+        filename = customFilename.includes('.') ? customFilename : `${customFilename}.${ext}`;
+      } else {
+        // 自动生成
+        const { filename: autoFilename } = this._extractFileInfo(url, mime, ext);
+        filename = autoFilename;
+      }
+
+      // 6. 获取唯一文件路径（避免覆盖）
+      const destPath = await this._getUniqueFilePath(targetDir, filename);
+
+      // 7. 安全写入文件
+      const tempPath = `${destPath}.tmp`;
+      await fsPromises.writeFile(tempPath, buffer);
+      await fsPromises.rename(tempPath, destPath);
+
+      // 8. 返回结果
+      return {
+        filename: path.basename(destPath),
+        path: destPath,
+        dir: targetDir,
+        size: buffer.length,
+        ext: ext,
+        mime: mime,
+      };
+    } catch (err) {
+      throw new Error(`微信媒体下载失败: ${err.message}`);
+    }
+  }
+
+  /**
+   * 实际下载逻辑
+   */
+  async _doDownload(
+    url,
+    { customFilename, customDir, fileExt, onProgress, skipExisting, aes_key }
+  ) {
+    return new Promise((resolve, reject) => {
+      const protocol = url.startsWith('https') ? https : http;
+
+      const requestOptions = {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+          Accept: '*/*',
+          'Accept-Encoding': 'gzip, deflate, br',
+          ...this.headers,
+        },
+        timeout: this.timeout,
+      };
+
+      const req = protocol.get(url, requestOptions, async (response) => {
+        try {
+          // 处理重定向
+          if ([301, 302, 307, 308].includes(response.statusCode)) {
+            const redirectUrl = response.headers.location;
+            return this._doDownload(redirectUrl, {
+              customFilename,
+              customDir,
+              fileExt,
+              onProgress,
+              aes_key,
+              skipExisting,
+            })
+              .then(resolve)
+              .catch(reject);
+          }
+
+          if (response.statusCode !== 200) {
+            reject(new Error(`HTTP ${response.statusCode}: ${response.statusMessage}`));
+            return;
+          }
+
+          // 获取 Content-Type
+          const contentType = response.headers['content-type'] || '';
+          console.log(`[Download] URL: ${url}`);
+          console.log(`[Download] Content-Type: ${contentType}`);
+
+          // 提取文件信息
+          const { filename: autoFilename, extension } = this._extractFileInfo(
+            url,
+            contentType,
+            fileExt
+          );
+          const filename = customFilename || autoFilename;
+
+          // 确定保存目录
+          let targetDir;
+          if (customDir) {
+            targetDir = customDir;
+          } else {
+            const typeDir = this._getTypeDir(contentType, extension);
+            targetDir = path.join(this.baseDir, typeDir);
+          }
+
+          // 创建目录
+          await fsPromises.mkdir(targetDir, { recursive: true });
+
+          // 获取唯一文件路径
+          let destPath = path.join(targetDir, filename);
+          if (skipExisting) {
+            destPath = await this._getUniqueFilePath(targetDir, filename);
+          }
+
+          const tempPath = `${destPath}.tmp`;
+          const totalSize = parseInt(response.headers['content-length'], 10);
+          let downloadedSize = 0;
+
+          // 创建写入流
+          const fileStream = fs.createWriteStream(tempPath);
+
+          // 监听进度
+          response.on('data', (chunk) => {
+            downloadedSize += chunk.length;
+            if (onProgress) {
+              onProgress({
+                downloaded: downloadedSize,
+                total: totalSize,
+                percent: totalSize ? ((downloadedSize / totalSize) * 100).toFixed(2) : '未知',
+                filename,
+                contentType,
+                category: path.basename(targetDir),
+              });
+            }
+          });
+
+          req.on('error', (err) => {
+            console.log('下载请求错误:', err.message);
+            reject(err);
+          });
+
+          // 使用 pipe 下载
+          if (aes_key) {
+            // aes_key 是 base64 编码的
+            const keyBuffer = Buffer.from(aes_key, 'base64');
+            response.pipe(crypto.createDecipheriv('aes-256-ecb', keyBuffer, null)).pipe(fileStream);
+          } else {
+            response.pipe(fileStream);
+          }
+          await new Promise((resolve, reject) => {
+            fileStream.on('finish', resolve);
+            fileStream.on('error', reject);
+          });
+
+          // 如果没有 content-type，检测文件魔数来修正扩展名
+          let finalPath = destPath;
+          const detectedExt = fileExt || this._getExtensionFromMime(contentType);
+          if (!detectedExt) {
+            try {
+              const fileType = await fileTypeFromFile(tempPath);
+              if (fileType) {
+                const ext = `.${fileType.ext}`;
+                const dir = path.dirname(destPath);
+                const base = path.basename(destPath, path.extname(destPath));
+                const newFilename = `${base}${ext}`;
+                finalPath = path.join(dir, newFilename);
+
+                // 如果文件名不同，删除旧的重命名
+                if (newFilename !== path.basename(destPath)) {
+                  await fsPromises.unlink(destPath).catch(() => {});
+                  await fsPromises.rename(tempPath, finalPath);
+                }
+              } else {
+                await fsPromises.rename(tempPath, destPath);
+              }
+            } catch (e) {
+              // 魔数检测失败，直接用原文件名
+              await fsPromises.rename(tempPath, destPath);
+            }
+          } else {
+            await fsPromises.rename(tempPath, destPath);
+          }
+
+          // 返回下载信息
+          resolve({
+            url,
+            path: finalPath,
+            filename: path.basename(finalPath),
+            size: downloadedSize,
+            contentType,
+            category: path.basename(path.dirname(finalPath)),
+            downloadTime: new Date().toISOString(),
+          });
+        } catch (error) {
+          reject(error);
+        }
+      });
+
+      req.on('timeout', () => {
+        req.destroy();
+        reject(new Error('请求超时'));
+      });
+
+      req.on('error', reject);
+    });
+  }
+
+  /**
+   * 批量下载
+   */
+  async downloadBatch(urls, options = {}) {
+    const results = {
+      success: [],
+      failed: [],
+      total: urls.length,
+    };
+
+    const { concurrency = 3, ...downloadOptions } = options;
+
+    // 控制并发数
+    for (let i = 0; i < urls.length; i += concurrency) {
+      const batch = urls.slice(i, i + concurrency);
+      const promises = batch.map(async (url) => {
+        try {
+          const result = await this.download(url, downloadOptions);
+          results.success.push(result);
+          return result;
+        } catch (error) {
+          results.failed.push({ url, error: error.message });
+          return null;
+        }
+      });
+
+      await Promise.all(promises);
+    }
+
+    return results;
+  }
+
+  _sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+module.exports = {
+  FileDownloader,
+  downloader: new FileDownloader({ retries: 3 }),
+};