@nest-omni/core 4.1.3-2 → 4.1.3-22

package/file-upload/processors/excel.processor.js

@@ -0,0 +1,547 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var ExcelProcessor_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExcelProcessor = void 0;
+const common_1 = require("@nestjs/common");
+const utils_1 = require("../utils");
+const column_decorator_1 = require("../decorators/column.decorator");
+const class_transformer_1 = require("class-transformer");
+const class_validator_1 = require("class-validator");
+const fs = require("fs");
+/**
+ * Excel 文件处理器
+ * 使用 ExcelJS 库处理上传的 Excel 文件
+ */
+let ExcelProcessor = ExcelProcessor_1 = class ExcelProcessor {
+    constructor() {
+        this.name = 'excel';
+        this.supportedTypes = [
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // .xlsx
+            'application/vnd.ms-excel', // .xls
+        ];
+        this.logger = new common_1.Logger(ExcelProcessor_1.name);
+    }
+    /**
+     * 验证是否为 Excel 文件
+     */
+    validate(buffer) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // 检查文件头
+            const header = buffer.slice(0, 8);
+            // XLSX 文件以 ZIP 头开始
+            if (header[0] === 0x50 && header[1] === 0x4b) {
+                return true;
+            }
+            // XLS 文件以特定签名开始
+            if (buffer[0] === 0xd0 &&
+                buffer[1] === 0xcf &&
+                buffer[2] === 0x11 &&
+                buffer[3] === 0xe0) {
+                return true;
+            }
+            return false;
+        });
+    }
+    process(file, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const filePath = file.path;
+            if (!filePath) {
+                throw new Error('Excel file requires path');
+            }
+            if (!fs.existsSync(filePath)) {
+                throw new Error('Excel file not found');
+            }
+            const excel = yield utils_1.DynamicImportUtil.importExcelJs();
+            if (!excel) {
+                throw new Error('ExcelJS is not installed. Please install it using: npm install exceljs');
+            }
+            try {
+                // 使用流式读取（不将整个文件加载到内存）
+                const workbook = new excel.Workbook();
+                const stream = fs.createReadStream(filePath);
+                yield workbook.xlsx.read(stream);
+                if (!workbook || workbook.worksheets.length === 0) {
+                    throw new Error('No sheets found in Excel file');
+                }
+                const sheetNames = workbook.worksheets.map((ws) => ws.name);
+                const totalSheets = workbook.worksheets.length;
+                // 获取目标工作表
+                let worksheet;
+                let sheetName;
+                let sheetIndex;
+                if (options === null || options === void 0 ? void 0 : options.sheetName) {
+                    worksheet = workbook.getWorksheet(options.sheetName);
+                    sheetName = options.sheetName;
+                    sheetIndex = sheetNames.indexOf(sheetName);
+                }
+                else if ((options === null || options === void 0 ? void 0 : options.sheetIndex) !== undefined) {
+                    worksheet = workbook.getWorksheet(options.sheetIndex + 1);
+                    sheetName = worksheet === null || worksheet === void 0 ? void 0 : worksheet.name;
+                    sheetIndex = options.sheetIndex;
+                }
+                else {
+                    worksheet = workbook.getWorksheet(1);
+                    sheetName = worksheet === null || worksheet === void 0 ? void 0 : worksheet.name;
+                    sheetIndex = 0;
+                }
+                if (!worksheet) {
+                    throw new Error(`Sheet not found: ${sheetName || 'index ' + sheetIndex}`);
+                }
+                // 流式解析工作表数据（逐行处理）
+                const parseResult = yield this.parseWorksheetStreaming(worksheet, options);
+                // 处理装饰器验证
+                let processedData = [];
+                const errors = [];
+                let successCount = 0;
+                let errorCount = 0;
+                // 设置默认最大行数
+                const maxRows = (options === null || options === void 0 ? void 0 : options.maxRows) || 100000;
+                const saveRawRecords = parseResult.data.length <= 10000;
+                const rawRecords = saveRawRecords ? [...parseResult.data] : [];
+                if ((options === null || options === void 0 ? void 0 : options.useDecorators) && (options === null || options === void 0 ? void 0 : options.dto)) {
+                    const columnMapping = (0, column_decorator_1.getExcelColumnMapping)(options.dto);
+                    // 转换为 Map<string, string> 以保持兼容性
+                    const mapping = new Map();
+                    const mappedColumns = [];
+                    for (const [key, value] of columnMapping.entries()) {
+                        const keyStr = String(key);
+                        mapping.set(keyStr, value);
+                        mappedColumns.push(keyStr);
+                    }
+                    const allProperties = Object.getOwnPropertyNames(options.dto.prototype);
+                    for (let i = 0; i < parseResult.data.length && i < maxRows; i++) {
+                        const row = parseResult.data[i];
+                        const processedRow = this.processRowWithDecorators(row, mapping, mappedColumns, allProperties, options);
+                        if (processedRow.error) {
+                            errors.push({
+                                row: i + ((options === null || options === void 0 ? void 0 : options.startRow) || 1),
+                                errors: processedRow.error,
+                                data: row,
+                            });
+                            errorCount++;
+                        }
+                        else if (processedRow.data) {
+                            processedData.push(processedRow.data);
+                            successCount++;
+                        }
+                    }
+                }
+                else {
+                    // 不使用装饰器，直接返回原始数据
+                    processedData = parseResult.data.slice(0, maxRows);
+                    successCount = processedData.length;
+                }
+                // 获取表头
+                let headers;
+                if ((options === null || options === void 0 ? void 0 : options.headers) && processedData.length > 0) {
+                    headers = Object.keys(processedData[0]);
+                }
+                // 计算行列数
+                const rowCount = processedData.length;
+                const columnCount = processedData.length > 0 ? Object.keys(processedData[0]).length : 0;
+                const result = {
+                    success: errors.length === 0,
+                    data: processedData,
+                    sheetName,
+                    headers,
+                    rowCount,
+                    columnCount,
+                    totalSheets,
+                    sheetNames,
+                    rawRecords: saveRawRecords ? rawRecords : undefined,
+                    errors: errors.length > 0 ? errors : undefined,
+                    successCount,
+                    errorCount,
+                };
+                // 标记是否已配置保存到元数据（由拦截器处理）
+                if ((options === null || options === void 0 ? void 0 : options.saveToMetadata) !== false) {
+                    result.savedToMetadata = true;
+                    result.metadataKey = (options === null || options === void 0 ? void 0 : options.metadataKey) || 'excel_parsed_data';
+                    // 对于大数据集，不保存完整数据
+                    if (processedData.length > 1000) {
+                        result.metadata = {
+                            rowCount,
+                            columnCount,
+                            headers,
+                            sheetName,
+                            totalSheets,
+                            sheetNames,
+                            successCount,
+                            errorCount,
+                            sample: processedData.slice(0, 10), // 只保存前 10 行
+                        };
+                    }
+                    else {
+                        result.metadata = {
+                            data: processedData,
+                            headers,
+                            rowCount,
+                            columnCount,
+                            sheetName,
+                            totalSheets,
+                            sheetNames,
+                            successCount,
+                            errorCount,
+                        };
+                    }
+                    this.logger.log(`Excel data will be saved to metadata by interceptor: rows=${rowCount}`);
+                }
+                return result;
+            }
+            catch (error) {
+                this.logger.error('Excel processing error:', error);
+                throw new Error(`Failed to process Excel file: ${error.message}`);
+            }
+        });
+    }
+    /**
+     * 获取处理器信息
+     */
+    getInfo() {
+        return {
+            name: this.name,
+            version: '1.0.0',
+            description: 'Excel file processor for .xlsx and .xls files',
+            supportedTypes: this.supportedTypes,
+            supportedExtensions: ['.xlsx', '.xls'],
+            capabilities: {
+                read: true,
+                write: false,
+                transform: true,
+                batch: true,
+                stream: false,
+                realtime: false,
+                parallel: false,
+                compress: false,
+                encrypt: false,
+            },
+            performance: {
+                maxFileSize: 100 * 1024 * 1024, // 100MB
+                avgProcessTime: 500, // 500ms per MB
+                memoryUsage: 50, // 50MB、
+            },
+            configOptions: {
+                sheetIndex: {
+                    type: 'number',
+                    description: 'Worksheet index (0-based)',
+                    default: 0,
+                },
+                sheetName: {
+                    type: 'string',
+                    description: 'Worksheet name',
+                },
+                headers: {
+                    type: 'boolean',
+                    description: 'Use first row as headers',
+                    default: true,
+                },
+                startRow: {
+                    type: 'number',
+                    description: 'Start reading from row',
+                    default: 1,
+                },
+                maxRows: {
+                    type: 'number',
+                    description: 'Maximum rows to read',
+                },
+                maxColumns: {
+                    type: 'number',
+                    description: 'Maximum columns to read',
+                },
+                skipEmptyRows: {
+                    type: 'boolean',
+                    description: 'Skip empty rows',
+                    default: true,
+                },
+                useDecorators: {
+                    type: 'boolean',
+                    description: 'Use decorators for column mapping',
+                    default: false,
+                },
+                dto: {
+                    type: 'string',
+                    description: 'DTO class for decorator mapping',
+                },
+                trim: {
+                    type: 'boolean',
+                    description: 'Trim field values',
+                    default: true,
+                },
+                strict: {
+                    type: 'boolean',
+                    description: 'Strict column count validation',
+                    default: false,
+                },
+            },
+        };
+    }
+    /**
+     * 健康检查
+     */
+    healthCheck() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const excel = yield utils_1.DynamicImportUtil.importExcelJs();
+                return !!excel;
+            }
+            catch (_a) {
+                return false;
+            }
+        });
+    }
+    /**
+     * 清理资源
+     */
+    cleanup() {
+        return __awaiter(this, void 0, void 0, function* () {
+            // ExcelJS 会自动清理
+        });
+    }
+    /**
+     * 流式解析工作表（逐行处理，避免一次性加载所有数据）
+     */
+    parseWorksheetStreaming(worksheet, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const data = [];
+            const startRow = (options === null || options === void 0 ? void 0 : options.startRow) || 1;
+            const maxRows = (options === null || options === void 0 ? void 0 : options.maxRows) || 100000;
+            const skipEmpty = (options === null || options === void 0 ? void 0 : options.skipEmptyRows) !== false;
+            let currentRow = 0;
+            let headers = null;
+            // 逐行处理
+            worksheet.eachRow((row, rowNumber) => {
+                // 跳过起始行之前的行
+                if (rowNumber < startRow) {
+                    return;
+                }
+                // 检查最大行数（实时限制）
+                if (currentRow >= maxRows) {
+                    return false; // 停止迭代
+                }
+                const rowData = {};
+                let hasData = false;
+                // 第一行作为表头（如果配置）
+                if ((options === null || options === void 0 ? void 0 : options.headers) !== false && rowNumber === startRow) {
+                    headers = {};
+                    row.eachCell((cell, colNumber) => {
+                        headers[colNumber] = this.getCellValue(cell);
+                    });
+                    return; // 跳过表头行
+                }
+                // 处理数据行
+                row.eachCell((cell, colNumber) => {
+                    // 检查最大列数
+                    if ((options === null || options === void 0 ? void 0 : options.maxColumns) && colNumber > options.maxColumns) {
+                        return;
+                    }
+                    const value = this.getCellValue(cell);
+                    if (value !== null && value !== undefined && value !== '') {
+                        hasData = true;
+                    }
+                    // 使用表头或列字母作为 key
+                    const key = headers && headers[colNumber]
+                        ? headers[colNumber]
+                        : this.getColumnLetter(colNumber);
+                    rowData[key] = value;
+                });
+                // 跳过空行
+                if (skipEmpty && !hasData) {
+                    return;
+                }
+                data.push(rowData);
+                currentRow++;
+            });
+            return { data };
+        });
+    }
+    /**
+     * 获取单元格值（处理各种类型）
+     */
+    getCellValue(cell) {
+        if (!cell || !cell.value) {
+            return null;
+        }
+        let value = cell.value;
+        // 处理不同类型的值
+        if (value && typeof value === 'object') {
+            if (value.richText) {
+                // 富文本
+                value = value.richText.map((t) => t.text).join('');
+            }
+            else if (value.formula) {
+                // 公式
+                value = value.result || value.formula;
+            }
+            else if (value instanceof Date) {
+                // 日期
+                value = value.toISOString();
+            }
+        }
+        return value;
+    }
+    /**
+     * 使用装饰器处理单行数据
+     */
+    processRowWithDecorators(row, mapping, mappedColumns, allProperties, options) {
+        const obj = {};
+        // 映射列
+        for (const columnIdentifierStr of mappedColumns) {
+            const propertyKey = mapping.get(columnIdentifierStr);
+            if (!propertyKey)
+                continue;
+            let value;
+            // 尝试解析为数字索引
+            const columnIdentifier = columnIdentifierStr.match(/^\d+$/)
+                ? parseInt(columnIdentifierStr, 10)
+                : columnIdentifierStr;
+            // 处理不同类型的列标识
+            if (typeof columnIdentifier === 'number') {
+                // 列序号（0基索引）- 转换为Excel列名
+                const columnLetter = this.getColumnLetter(columnIdentifier + 1);
+                if (row.hasOwnProperty(columnLetter)) {
+                    value = row[columnLetter];
+                }
+                else {
+                    value = undefined;
+                }
+            }
+            else if (typeof columnIdentifier === 'string') {
+                // 列名或Excel列名
+                if (row.hasOwnProperty(columnIdentifier)) {
+                    value = row[columnIdentifier];
+                }
+                else {
+                    value = undefined;
+                }
+            }
+            // 处理值
+            if (value !== undefined) {
+                // 处理日期类型
+                if (value instanceof Date) {
+                    value = value.toISOString();
+                }
+                // 使用class-transformer进行转换
+                const transformedValue = (options === null || options === void 0 ? void 0 : options.trim) !== false && typeof value === 'string'
+                    ? value.trim()
+                    : value;
+                obj[propertyKey] = transformedValue;
+            }
+        }
+        // 处理其他属性（未在装饰器中定义的）
+        for (const property of allProperties) {
+            if (!Array.from(mapping.values()).includes(property) && row.hasOwnProperty(property)) {
+                let value = row[property];
+                // 基本的类型转换
+                if ((options === null || options === void 0 ? void 0 : options.trim) && typeof value === 'string') {
+                    value = value.trim();
+                }
+                obj[property] = value;
+            }
+        }
+        // 创建DTO实例并验证
+        const instance = (0, class_transformer_1.plainToInstance)(options.dto, obj);
+        // 使用class-validator验证
+        const validationErrors = (0, class_validator_1.validateSync)(instance);
+        if (validationErrors.length > 0) {
+            return { error: validationErrors };
+        }
+        return { data: instance };
+    }
+    /**
+     * 解析工作表（保留原方法作为备用）
+     * @deprecated 使用 parseWorksheetStreaming 代替
+     */
+    parseWorksheet(worksheet, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const data = [];
+            const startRow = ((options === null || options === void 0 ? void 0 : options.startRow) || 1) - 1;
+            const maxRows = options === null || options === void 0 ? void 0 : options.maxRows;
+            const skipEmpty = (options === null || options === void 0 ? void 0 : options.skipEmptyRows) !== false;
+            worksheet.eachRow((row, rowNumber) => {
+                // 跳过起始行之前的行
+                if (rowNumber <= startRow)
+                    return;
+                // 检查最大行数
+                if (maxRows && data.length >= maxRows)
+                    return false; // 停止迭代
+                const rowData = {};
+                let hasData = false;
+                row.eachCell((cell, colNumber) => {
+                    // 检查最大列数
+                    if ((options === null || options === void 0 ? void 0 : options.maxColumns) && colNumber > options.maxColumns)
+                        return;
+                    let value = cell.value;
+                    // 处理不同类型的值
+                    if (value && typeof value === 'object') {
+                        if (value.type === 'Date') {
+                            value = value.toISOString();
+                        }
+                        else if (value.richText) {
+                            value = value.richText.map((t) => t.text).join('');
+                        }
+                        else if (value.formula) {
+                            value = value.result || value.formula;
+                        }
+                    }
+                    if (value !== null && value !== undefined && value !== '') {
+                        hasData = true;
+                    }
+                    // 如果是第一行且有headers选项，使用列字母作为key
+                    if ((options === null || options === void 0 ? void 0 : options.headers) && rowNumber === 1) {
+                        const columnLetter = this.getColumnLetter(colNumber);
+                        rowData[columnLetter] = value;
+                    }
+                    else {
+                        const key = (options === null || options === void 0 ? void 0 : options.headers) && rowNumber > 1
+                            ? this.getHeaderKey(worksheet, colNumber)
+                            : this.getColumnLetter(colNumber);
+                        rowData[key] = value;
+                    }
+                });
+                // 跳过空行
+                if (skipEmpty && !hasData)
+                    return;
+                data.push(rowData);
+            });
+            return data;
+        });
+    }
+    /**
+     * 获取列字母 (A, B, C..., AA, AB...)
+     */
+    getColumnLetter(colNumber) {
+        let letters = '';
+        while (colNumber > 0) {
+            colNumber--;
+            letters = String.fromCharCode(65 + (colNumber % 26)) + letters;
+            colNumber = Math.floor(colNumber / 26);
+        }
+        return letters;
+    }
+    /**
+     * 获取表头键
+     */
+    getHeaderKey(worksheet, colNumber) {
+        const headerCell = worksheet.getCell(1, colNumber);
+        return headerCell.value || this.getColumnLetter(colNumber);
+    }
+};
+exports.ExcelProcessor = ExcelProcessor;
+exports.ExcelProcessor = ExcelProcessor = ExcelProcessor_1 = __decorate([
+    (0, common_1.Injectable)()
+], ExcelProcessor);