@nest-omni/core 4.1.3-2 → 4.1.3-20

package/file-upload/processors/csv.processor.js

@@ -0,0 +1,391 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var CsvProcessor_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CsvProcessor = void 0;
+const common_1 = require("@nestjs/common");
+const utils_1 = require("../utils");
+const column_decorator_1 = require("../decorators/column.decorator");
+const class_transformer_1 = require("class-transformer");
+const class_validator_1 = require("class-validator");
+const fs = require("fs");
+/**
+ * CSV 文件处理器
+ * 解析 CSV 文件并转换为 JSON 格式，支持装饰器映射
+ */
+let CsvProcessor = CsvProcessor_1 = class CsvProcessor {
+    constructor() {
+        this.name = 'csv';
+        this.supportedTypes = ['text/csv', 'application/csv'];
+        this.logger = new common_1.Logger(CsvProcessor_1.name);
+    }
+    /**
+     * 验证是否为 CSV 文件
+     */
+    validate(buffer) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const content = buffer.toString('utf8', 0, Math.min(1024, buffer.length));
+            // 简单的 CSV 检测：查找逗号分隔的值
+            const lines = content.split('\n').slice(0, 5);
+            return lines.some((line) => line.includes(','));
+        });
+    }
+    /**
+     * 处理CSV文件（流式处理优化版）
+     */
+    process(file, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const filePath = file.path;
+            if (!filePath) {
+                throw new Error('CSV file requires path');
+            }
+            if (!fs.existsSync(filePath)) {
+                throw new Error('CSV file not found');
+            }
+            // 动态加载 fast-csv
+            const csv = yield utils_1.DynamicImportUtil.importFastCsv();
+            if (!csv) {
+                throw new Error('Fast CSV is not installed. Please install it using: npm install fast-csv');
+            }
+            return new Promise((resolve, reject) => {
+                const processedData = [];
+                const errors = [];
+                let headers = [];
+                let rowCount = 0;
+                let columnCount = 0;
+                // 设置默认最大行数（防止无限读取）
+                const maxRows = (options === null || options === void 0 ? void 0 : options.maxRows) || 100000; // 默认 10 万行
+                const saveRawRecords = (options === null || options === void 0 ? void 0 : options.maxRows) ? options.maxRows <= 10000 : false; // 只在小数据集保存原始数据
+                const rawRecords = saveRawRecords ? [] : undefined;
+                const readOptions = {
+                    encoding: (options === null || options === void 0 ? void 0 : options.encoding) || 'utf8',
+                    headers: (options === null || options === void 0 ? void 0 : options.headers) !== false, // 默认有表头
+                    skipLines: (options === null || options === void 0 ? void 0 : options.skipLines) || 0,
+                    skipEmpty: (options === null || options === void 0 ? void 0 : options.skipEmpty) !== false,
+                    strict: (options === null || options === void 0 ? void 0 : options.strict) !== false,
+                    trim: (options === null || options === void 0 ? void 0 : options.trim) !== false,
+                };
+                // 只添加已定义的选项，避免 undefined
+                if ((options === null || options === void 0 ? void 0 : options.delimiter) !== undefined) {
+                    readOptions.delimiter = options.delimiter;
+                }
+                if ((options === null || options === void 0 ? void 0 : options.quote) !== undefined) {
+                    readOptions.quote = options.quote;
+                }
+                if ((options === null || options === void 0 ? void 0 : options.escape) !== undefined) {
+                    readOptions.escape = options.escape;
+                }
+                // 创建文件读取流（流式处理，不将整个文件加载到内存）
+                const fileStream = fs.createReadStream(filePath, {
+                    encoding: ((options === null || options === void 0 ? void 0 : options.encoding) || 'utf8'),
+                    highWaterMark: 64 * 1024, // 每次读取 64KB
+                });
+                // 预编译装饰器映射（提升性能）
+                let mapping = null;
+                let mappedColumns = [];
+                let allProperties = [];
+                if ((options === null || options === void 0 ? void 0 : options.useDecorators) && (options === null || options === void 0 ? void 0 : options.dto)) {
+                    const columnMapping = (0, column_decorator_1.getCsvColumnMapping)(options.dto);
+                    // 转换为 Map<string, string> 以保持兼容性
+                    mapping = new Map();
+                    for (const [key, value] of columnMapping.entries()) {
+                        mapping.set(String(key), value);
+                    }
+                    mappedColumns = Array.from(mapping.keys());
+                    allProperties = Object.getOwnPropertyNames(options.dto.prototype);
+                }
+                // 创建解析流
+                const parser = csv
+                    .parse(readOptions)
+                    .on('data', (row) => {
+                    // 实时检查行数限制
+                    if (rowCount >= maxRows) {
+                        fileStream.destroy(); // 立即停止读取
+                        parser.destroy();
+                        reject(new Error(`Exceeded maximum rows limit: ${maxRows}`));
+                        return;
+                    }
+                    rowCount++;
+                    // 第一次获取列数
+                    if (columnCount === 0 && row) {
+                        columnCount = Array.isArray(row)
+                            ? row.length
+                            : Object.keys(row).length;
+                    }
+                    // 保存原始数据（仅在小数据集）
+                    if (saveRawRecords && rawRecords) {
+                        rawRecords.push(row);
+                    }
+                    // 处理数据
+                    try {
+                        if ((options === null || options === void 0 ? void 0 : options.useDecorators) && (options === null || options === void 0 ? void 0 : options.dto) && mapping) {
+                            // 使用装饰器映射和验证
+                            const processedRow = this.processRowWithDecorators(row, mapping, mappedColumns, allProperties, options);
+                            if (processedRow.error) {
+                                errors.push({
+                                    row: rowCount + ((options === null || options === void 0 ? void 0 : options.skipLines) || 0),
+                                    errors: processedRow.error,
+                                    data: row,
+                                });
+                            }
+                            else if (processedRow.data) {
+                                processedData.push(processedRow.data);
+                            }
+                        }
+                        else {
+                            // 不使用装饰器，直接保存
+                            processedData.push(row);
+                        }
+                    }
+                    catch (error) {
+                        this.logger.warn(`Error processing row ${rowCount}: ${error.message}`);
+                        errors.push({
+                            row: rowCount + ((options === null || options === void 0 ? void 0 : options.skipLines) || 0),
+                            errors: [error.message],
+                            data: row,
+                        });
+                    }
+                })
+                    .on('headers', (headerList) => {
+                    if (headerList && headerList.length > 0) {
+                        headers = headerList;
+                    }
+                })
+                    .on('error', (error) => {
+                    this.logger.error('CSV parsing error:', error);
+                    fileStream.destroy();
+                    reject(error);
+                })
+                    .on('end', () => {
+                    const result = {
+                        success: errors.length === 0,
+                        data: processedData,
+                        rawRecords,
+                        headers,
+                        errors: errors.length > 0 ? errors : undefined,
+                        rowCount: processedData.length,
+                        columnCount,
+                        successCount: processedData.length,
+                        errorCount: errors.length,
+                    };
+                    // 标记是否已配置保存到元数据（由拦截器处理）
+                    if ((options === null || options === void 0 ? void 0 : options.saveToMetadata) !== false) {
+                        result.savedToMetadata = true;
+                        result.metadataKey = (options === null || options === void 0 ? void 0 : options.metadataKey) || 'csv_parsed_data';
+                        // 准备要保存的元数据（处理器决定内容）
+                        // 对于大数据集，不保存完整数据，只保存统计信息
+                        if (processedData.length > 1000) {
+                            result.metadata = {
+                                rowCount: processedData.length,
+                                columnCount,
+                                headers,
+                                successCount: processedData.length,
+                                errorCount: errors.length,
+                                sample: processedData.slice(0, 10), // 只保存前 10 行作为样本
+                            };
+                        }
+                        else {
+                            result.metadata = {
+                                data: processedData,
+                                headers,
+                                rowCount: processedData.length,
+                                columnCount,
+                                successCount: processedData.length,
+                                errorCount: errors.length,
+                            };
+                        }
+                        this.logger.log(`CSV data will be saved to metadata by interceptor: rows=${processedData.length}`);
+                    }
+                    resolve(result);
+                });
+                // 连接流：文件流 -> 解析器
+                fileStream
+                    .pipe(parser)
+                    .on('error', (error) => {
+                    this.logger.error('File stream error:', error);
+                    fileStream.destroy();
+                    parser.destroy();
+                    reject(error);
+                });
+            });
+        });
+    }
+    /**
+     * 使用装饰器处理单行数据
+     */
+    processRowWithDecorators(row, mapping, mappedColumns, allProperties, options) {
+        const obj = {};
+        // 映射列
+        for (const columnIdentifierStr of mappedColumns) {
+            const propertyKey = mapping.get(columnIdentifierStr);
+            if (!propertyKey)
+                continue;
+            let value;
+            // 尝试解析为数字索引
+            const columnIdentifier = columnIdentifierStr.match(/^\d+$/)
+                ? parseInt(columnIdentifierStr, 10)
+                : columnIdentifierStr;
+            // 处理不同类型的列标识
+            if (typeof columnIdentifier === 'number') {
+                // 列序号（0基索引）
+                if (Array.isArray(row)) {
+                    // 如果row是数组，直接使用索引
+                    value = row[columnIdentifier];
+                }
+                else {
+                    // 如果row是对象，无法通过数字索引获取
+                    value = undefined;
+                }
+            }
+            else if (typeof columnIdentifier === 'string') {
+                // 列名
+                if (row.hasOwnProperty(columnIdentifier)) {
+                    value = row[columnIdentifier];
+                }
+                else {
+                    value = undefined;
+                }
+            }
+            // 处理值
+            if (value !== undefined) {
+                const transformedValue = (options === null || options === void 0 ? void 0 : options.trim) !== false && typeof value === 'string'
+                    ? value.trim()
+                    : value;
+                obj[propertyKey] = transformedValue;
+            }
+        }
+        // 处理其他属性（未在装饰器中定义的）
+        for (const property of allProperties) {
+            if (!Array.from(mapping.values()).includes(property) && row.hasOwnProperty(property)) {
+                let value = row[property];
+                if ((options === null || options === void 0 ? void 0 : options.trim) && typeof value === 'string') {
+                    value = value.trim();
+                }
+                obj[property] = value;
+            }
+        }
+        // 创建DTO实例并验证
+        const instance = (0, class_transformer_1.plainToInstance)(options.dto, obj);
+        // 使用class-validator验证
+        const validationErrors = (0, class_validator_1.validateSync)(instance);
+        if (validationErrors.length > 0) {
+            return { error: validationErrors };
+        }
+        return { data: instance };
+    }
+    /**
+     * 获取处理器信息
+     */
+    getInfo() {
+        return {
+            name: this.name,
+            version: '1.0.0',
+            description: 'CSV file processor for parsing comma-separated values',
+            supportedTypes: this.supportedTypes,
+            supportedExtensions: ['.csv'],
+            capabilities: {
+                read: true,
+                write: false,
+                transform: true,
+                batch: true,
+                stream: true,
+                realtime: false,
+                parallel: false,
+                compress: false,
+                encrypt: false,
+            },
+            performance: {
+                maxFileSize: 50 * 1024 * 1024, // 50MB
+                avgProcessTime: 100, // 100ms per MB
+                memoryUsage: 10, // 10MB
+            },
+            configOptions: {
+                delimiter: {
+                    type: 'string',
+                    description: 'Field delimiter',
+                    default: ',',
+                },
+                headers: {
+                    type: 'boolean',
+                    description: 'First row contains headers',
+                    default: true,
+                },
+                skipLines: {
+                    type: 'number',
+                    description: 'Number of lines to skip',
+                    default: 0,
+                },
+                encoding: {
+                    type: 'string',
+                    description: 'File encoding',
+                    default: 'utf8',
+                },
+                maxRows: {
+                    type: 'number',
+                    description: 'Maximum rows to parse',
+                },
+                trim: {
+                    type: 'boolean',
+                    description: 'Trim field values',
+                    default: true,
+                },
+                strict: {
+                    type: 'boolean',
+                    description: 'Strict column count validation',
+                    default: false,
+                },
+                useDecorators: {
+                    type: 'boolean',
+                    description: 'Use decorators for column mapping',
+                    default: false,
+                },
+                dto: {
+                    type: 'string',
+                    description: 'DTO class for decorator mapping',
+                },
+            },
+        };
+    }
+    /**
+     * 健康检查
+     */
+    healthCheck() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                // 尝试导入 fast-csv
+                const csv = yield utils_1.DynamicImportUtil.importFastCsv();
+                return !!csv;
+            }
+            catch (_a) {
+                // 备用解析器始终可用
+                return true;
+            }
+        });
+    }
+    /**
+     * 清理资源
+     */
+    cleanup() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // 无需清理资源
+        });
+    }
+};
+exports.CsvProcessor = CsvProcessor;
+exports.CsvProcessor = CsvProcessor = CsvProcessor_1 = __decorate([
+    (0, common_1.Injectable)()
+], CsvProcessor);

package/file-upload/processors/excel.processor.d.ts

@@ -0,0 +1,130 @@
+import { FileEntity } from '../interfaces/file-entity.interface';
+import { IFileProcessor, ProcessorInfo } from '../interfaces/file-processor.interface';
+import { ProcessOptions, ProcessResult } from '../interfaces/processor-options.interface';
+import { ValidationError } from 'class-validator';
+/**
+ * Excel 处理器配置
+ */
+export interface ExcelProcessorOptions extends ProcessOptions {
+    /** 工作表索引 */
+    sheetIndex?: number;
+    /** 工作表名称 */
+    sheetName?: string;
+    /** 读取范围 (如: A1:C10) */
+    range?: string;
+    /** 起始行 */
+    startRow?: number;
+    /** 最大行数 */
+    maxRows?: number;
+    /** 最大列数 */
+    maxColumns?: number;
+    /** 是否包含表头 */
+    headers?: boolean;
+    /** 是否读取所有工作表 */
+    allSheets?: boolean;
+    /** 日期格式 */
+    dateFormat?: string;
+    /** 是否跳过空行 */
+    skipEmptyRows?: boolean;
+    /** 是否转换数字为字符串 */
+    rawNumbers?: boolean;
+    /** DTO类（用于装饰器映射） */
+    dto?: new () => any;
+    /** 是否使用装饰器映射 */
+    useDecorators?: boolean;
+    /** 是否修剪字段值 */
+    trim?: boolean;
+    /** 严格模式（字段数必须一致） */
+    strict?: boolean;
+    /** 是否将解析结果存入 file_metadata */
+    saveToMetadata?: boolean;
+    /** 元数据存储键名 */
+    metadataKey?: string;
+}
+/**
+ * Excel 处理结果
+ */
+export interface ExcelProcessResult extends ProcessResult {
+    /** Excel 数据 */
+    data: any[] | Record<string, any[]>;
+    /** 当前工作表名称 */
+    sheetName?: string;
+    /** 表头 */
+    headers?: string[];
+    /** 行数 */
+    rowCount: number;
+    /** 列数 */
+    columnCount: number;
+    /** 工作表总数 */
+    totalSheets?: number;
+    /** 工作表名称列表 */
+    sheetNames?: string[];
+    /** 错误信息 */
+    errors?: {
+        row: number;
+        column?: string;
+        errors: ValidationError[];
+        data: any;
+    }[];
+    /** 成功的行数 */
+    successCount?: number;
+    /** 失败的行数 */
+    errorCount?: number;
+    /** 原始数据（用于调试） */
+    rawRecords?: any[];
+    /** 是否已保存到元数据 */
+    savedToMetadata?: boolean;
+    /** 元数据存储键名 */
+    metadataKey?: string;
+}
+/**
+ * Excel 文件处理器
+ * 使用 ExcelJS 库处理上传的 Excel 文件
+ */
+export declare class ExcelProcessor implements IFileProcessor {
+    readonly name = "excel";
+    readonly supportedTypes: string[];
+    private readonly logger;
+    /**
+     * 验证是否为 Excel 文件
+     */
+    validate(buffer: Buffer): Promise<boolean>;
+    process(file: FileEntity, options?: ExcelProcessorOptions): Promise<ExcelProcessResult>;
+    /**
+     * 获取处理器信息
+     */
+    getInfo(): ProcessorInfo;
+    /**
+     * 健康检查
+     */
+    healthCheck(): Promise<boolean>;
+    /**
+     * 清理资源
+     */
+    cleanup(): Promise<void>;
+    /**
+     * 流式解析工作表（逐行处理，避免一次性加载所有数据）
+     */
+    private parseWorksheetStreaming;
+    /**
+     * 获取单元格值（处理各种类型）
+     */
+    private getCellValue;
+    /**
+     * 使用装饰器处理单行数据
+     */
+    private processRowWithDecorators;
+    /**
+     * 解析工作表（保留原方法作为备用）
+     * @deprecated 使用 parseWorksheetStreaming 代替
+     */
+    private parseWorksheet;
+    /**
+     * 获取列字母 (A, B, C..., AA, AB...)
+     */
+    private getColumnLetter;
+    /**
+     * 获取表头键
+     */
+    private getHeaderKey;
+}