monsqlize 1.0.1 → 1.0.5

package/lib/mongodb/queries/find-page.js

@@ -11,6 +11,7 @@ const { decodeCursor } = require('../../common/cursor');
 const { validateLimitAfterBefore, assertCursorSortCompatible } = require('../../common/validation');
 const { makePageResult } = require('../../common/page-result');
 const { normalizeSort } = require('../../common/normalize');
+const { convertObjectIdStrings } = require('../../utils/objectid-converter');
 
 // —— Count 队列支持（高并发控制）——
 let countQueue = null;
@@ -213,6 +214,17 @@ function createFindPage(ctx) {
 
     return async function findPage(options = {}) {
         const startTime = Date.now(); // 记录开始时间
+
+        // ✅ v1.3.0: 自动转换 ObjectId 字符串
+        if (options.query) {
+            options.query = convertObjectIdStrings(options.query, 'filter', 0, new WeakSet(), {
+                logger: ctx.logger,
+                excludeFields: ctx.autoConvertConfig?.excludeFields,
+                customFieldPatterns: ctx.autoConvertConfig?.customFieldPatterns,
+                maxDepth: ctx.autoConvertConfig?.maxDepth
+            });
+        }
+
         const MAX_LIMIT = defaults?.findPageMaxLimit ?? 500;
         // 基础校验：limit + after/before 互斥
         validateLimitAfterBefore(options, { maxLimit: MAX_LIMIT });

package/lib/mongodb/queries/find.js

@@ -5,6 +5,7 @@
 
 const { normalizeProjection, normalizeSort } = require('../../common/normalize');
 const { FindChain } = require('./chain');
+const { convertObjectIdStrings } = require('../../utils/objectid-converter');
 
 /**
  * 创建 find 查询操作
@@ -51,12 +52,20 @@ function createFindOps(context) {
          * @returns {Promise<Array>|ReadableStream|FindChain} 记录数组或可读流（当 stream: true 时）；当 explain=true 时返回执行计划；默认返回 FindChain 实例支持链式调用
          */
         find: (query = {}, options = {}) => {
+            // ✅ v1.3.0: 自动转换 ObjectId 字符串
+            const convertedQuery = convertObjectIdStrings(query, 'query', 0, new WeakSet(), {
+                logger: context.logger,
+                excludeFields: context.autoConvertConfig?.excludeFields,
+                customFieldPatterns: context.autoConvertConfig?.customFieldPatterns,
+                maxDepth: context.autoConvertConfig?.maxDepth
+            });
+
             // 如果没有提供 options 或 options 为空对象，返回 FindChain 以支持完整的链式调用
             const hasOptions = options && Object.keys(options).length > 0;
 
             if (!hasOptions) {
                 // 返回 FindChain 实例，支持 .limit().skip().sort() 等链式调用
-                return new FindChain(context, query, {});
+                return new FindChain(context, convertedQuery, {});
             }
 
             // 如果提供了 options，执行原有逻辑（向后兼容）
@@ -81,13 +90,13 @@ function createFindOps(context) {
             // 如果启用 explain，直接返回执行计划（不缓存）
             if (explain) {
                 const verbosity = typeof explain === 'string' ? explain : 'queryPlanner';
-                const cursor = collection.find(query, driverOpts);
+                const cursor = collection.find(convertedQuery, driverOpts);
                 return cursor.explain(verbosity);
             }
 
             // 如果启用流式返回，直接返回 MongoDB 游标流
             if (stream) {
-                const cursor = collection.find(query, driverOpts);
+                const cursor = collection.find(convertedQuery, driverOpts);
                 const readableStream = cursor.stream();
 
                 // 添加慢查询日志支持
@@ -127,13 +136,13 @@ function createFindOps(context) {
             // 执行查询的 Promise
             const resultPromise = run(
                 'find',
-                { query, ...options },
-                async () => collection.find(query, driverOpts).toArray()
+                { query: convertedQuery, ...options },
+                async () => collection.find(convertedQuery, driverOpts).toArray()
             );
 
             // 添加 explain 方法支持链式调用（与原生 MongoDB 一致）
             resultPromise.explain = async (verbosity = 'queryPlanner') => {
-                const cursor = collection.find(query, driverOpts);
+                const cursor = collection.find(convertedQuery, driverOpts);
                 return cursor.explain(verbosity);
             };
 

package/lib/mongodb/queries/watch.js

@@ -488,6 +488,15 @@ function createWatchOps(context) {
                 throw new Error('pipeline must be an array');
             }
 
+            // ✅ v1.3.0: 自动转换 ObjectId 字符串
+            const { convertAggregationPipeline } = require('../../utils/objectid-converter');
+            const convertedPipeline = convertAggregationPipeline(pipeline, 0, {
+                logger: context.logger,
+                excludeFields: context.autoConvertConfig?.excludeFields,
+                customFieldPatterns: context.autoConvertConfig?.customFieldPatterns,
+                maxDepth: context.autoConvertConfig?.maxDepth || 5
+            });
+
             // 构建 MongoDB watch 选项
             const watchOptions = {
                 fullDocument: options.fullDocument || 'updateLookup',
@@ -501,13 +510,13 @@ function createWatchOps(context) {
             delete watchOptions.autoInvalidateCache;
 
             // 创建 MongoDB ChangeStream
-            const changeStream = context.collection.watch(pipeline, watchOptions);
+            const changeStream = context.collection.watch(convertedPipeline, watchOptions);
 
             // 包装为 ChangeStreamWrapper
             const wrapper = new ChangeStreamWrapper(
                 changeStream,
                 context.collection,
-                pipeline,
+                convertedPipeline,
                 options,
                 context
             );

package/lib/mongodb/writes/common/batch-retry.js

@@ -0,0 +1,64 @@
+/**
+ * 批量操作重试机制
+ * 供 insertBatch、deleteBatch、updateBatch 复用
+ */
+
+/**
+ * 执行批次操作（带重试）
+ * @param {Function} fn - 要执行的批次操作函数
+ * @param {Object} retryContext - 重试上下文
+ * @param {string} retryContext.onError - 错误策略: 'stop'/'skip'/'collect'/'retry'
+ * @param {number} retryContext.retryAttempts - 最大重试次数
+ * @param {number} retryContext.retryDelay - 重试延迟（毫秒）
+ * @param {Function} retryContext.onRetry - 重试回调
+ * @param {number} retryContext.batchIndex - 当前批次索引
+ * @returns {Promise<Object>} { success, result, attempts }
+ */
+async function executeBatchWithRetry(fn, retryContext) {
+    const {
+        onError = 'stop',
+        retryAttempts = 3,
+        retryDelay = 1000,
+        onRetry,
+        batchIndex = 0
+    } = retryContext;
+
+    let lastError = null;
+    let attempts = 0;
+    const maxAttempts = onError === 'retry' ? retryAttempts + 1 : 1;
+
+    while (attempts < maxAttempts) {
+        try {
+            const result = await fn();
+            return { success: true, result, attempts };
+        } catch (error) {
+            lastError = error;
+            attempts++;
+
+            // 还有重试机会
+            if (attempts < maxAttempts) {
+                // 触发重试回调
+                if (onRetry) {
+                    onRetry({
+                        batchIndex,
+                        attempt: attempts,
+                        maxAttempts: retryAttempts,
+                        error,
+                        nextRetryDelay: retryDelay
+                    });
+                }
+
+                // 延迟后重试
+                if (retryDelay > 0) {
+                    await new Promise(resolve => setTimeout(resolve, retryDelay));
+                }
+            }
+        }
+    }
+
+    // 所有重试都失败
+    throw lastError;
+}
+
+module.exports = { executeBatchWithRetry };
+

package/lib/mongodb/writes/delete-batch.js

@@ -0,0 +1,322 @@
+/**
+ * deleteBatch 操作实现
+ * 分批删除大量匹配的文档（基于流式查询）
+ */
+
+const { createError, ErrorCodes } = require('../../errors');
+const CacheFactory = require('../../cache');
+const { isInTransaction, getTransactionFromSession } = require('../common/transaction-aware');
+const { convertObjectIdStrings } = require('../../utils/objectid-converter');
+const { executeBatchWithRetry } = require('./common/batch-retry');
+
+/**
+ * 创建 deleteBatch 操作
+ * @param {Object} context - 模块上下文
+ * @returns {Object} 包含 deleteBatch 方法的对象
+ */
+function createDeleteBatchOps(context) {
+    const { db, cache, logger, defaults, collection, effectiveDbName: databaseName, instanceId } = context;
+
+    const collectionName = collection.collectionName;
+    const nativeCollection = collection;
+
+    /**
+     * 分批删除大量匹配的文档
+     * @param {Object} filter - 筛选条件（必需）
+     * @param {Object} [options] - 操作选项
+     * @param {number} [options.batchSize=1000] - 每批删除的文档数量
+     * @param {boolean} [options.estimateProgress=true] - 是否预先 count 获取总数
+     * @param {Function} [options.onProgress] - 进度回调函数 (progress) => {}
+     * @param {string} [options.onError='stop'] - 错误处理策略: 'stop'/'skip'/'collect'/'retry'
+     * @param {number} [options.retryAttempts=3] - 失败批次最大重试次数
+     * @param {number} [options.retryDelay=1000] - 重试延迟时间（毫秒）
+     * @param {Function} [options.onRetry] - 重试回调函数 (retryInfo) => {}
+     * @param {Object} [options.writeConcern] - 写关注选项
+     * @param {string} [options.comment] - 操作注释（用于日志追踪）
+     * @returns {Promise<Object>} 删除结果
+     *
+     * @example
+     * const result = await collection('logs').deleteBatch(
+     *     { createdAt: { $lt: new Date('2024-01-01') } },
+     *     {
+     *         batchSize: 1000,
+     *         estimateProgress: true,
+     *         onProgress: (p) => console.log(`进度: ${p.percentage}%`)
+     *     }
+     * );
+     */
+    const deleteBatch = async function deleteBatch(filter, options = {}) {
+        const startTime = Date.now();
+
+        // 1. 参数验证
+        if (!filter || typeof filter !== 'object' || Array.isArray(filter)) {
+            throw createError(
+                ErrorCodes.INVALID_ARGUMENT,
+                'filter 必须是对象类型',
+                [{ field: 'filter', type: 'object.required', message: 'filter 是必需参数且必须是对象' }]
+            );
+        }
+
+        // 自动转换 ObjectId 字符串
+        const convertedFilter = convertObjectIdStrings(filter, 'filter', 0, new WeakSet(), {
+            logger: context.logger,
+            excludeFields: context.autoConvertConfig?.excludeFields,
+            customFieldPatterns: context.autoConvertConfig?.customFieldPatterns,
+            maxDepth: context.autoConvertConfig?.maxDepth
+        });
+
+        // 解构选项
+        const {
+            batchSize = 1000,
+            estimateProgress = true,
+            onProgress,
+            onError = 'stop',
+            retryAttempts = 3,
+            retryDelay = 1000,
+            onRetry,
+            writeConcern,
+            comment
+        } = options;
+
+        // 2. 预先 count（可选）
+        const totalCount = estimateProgress
+            ? await nativeCollection.countDocuments(convertedFilter)
+            : null;
+
+        // 3. 初始化结果
+        const result = {
+            acknowledged: true,
+            totalCount,
+            deletedCount: 0,
+            batchCount: 0,
+            errors: [],
+            retries: []
+        };
+
+        // 4. 🔴 关键：直接调用 context 的 find 流式方法
+        const stream = await context.find(convertedFilter, {
+            stream: true,
+            batchSize,
+            limit: 0,  // 🔴 重要：禁用默认的 limit，否则只会查询默认的 10 条
+            comment: comment ? `${comment}:deleteBatch` : 'deleteBatch'
+        });
+
+        let batch = [];
+
+        return new Promise((resolve, reject) => {
+            // 5. 监听数据事件
+            stream.on('data', async (doc) => {
+                batch.push(doc._id);
+
+                // 达到批次大小，执行删除
+                if (batch.length >= batchSize) {
+                    stream.pause();
+
+                    try {
+                        // 🔴 带重试的批量删除
+                        const batchResult = await executeBatchWithRetry(
+                            () => nativeCollection.deleteMany(
+                                { _id: { $in: batch } },
+                                { writeConcern }
+                            ),
+                            { onError, retryAttempts, retryDelay, onRetry, batchIndex: result.batchCount }
+                        );
+
+                        result.deletedCount += batchResult.result.deletedCount;
+                        result.batchCount++;
+
+                        // 记录重试信息
+                        if (batchResult.attempts > 0) {
+                            result.retries.push({
+                                batchIndex: result.batchCount - 1,
+                                attempts: batchResult.attempts,
+                                success: true
+                            });
+                        }
+
+                        batch = [];
+
+                        // 🔴 进度回调
+                        if (onProgress) {
+                            onProgress({
+                                currentBatch: result.batchCount,
+                                totalBatches: totalCount ? Math.ceil(totalCount / batchSize) : null,
+                                deleted: result.deletedCount,
+                                total: totalCount,
+                                percentage: totalCount ? Math.round((result.deletedCount / totalCount) * 100) : null,
+                                errors: result.errors.length,
+                                retries: result.retries.length
+                            });
+                        }
+
+                        stream.resume();
+
+                    } catch (error) {
+                        // 🔴 错误处理
+                        result.errors.push({
+                            batchIndex: result.batchCount,
+                            batchSize: batch.length,
+                            error: error,
+                            message: error.message
+                        });
+
+                        if (onError === 'stop') {
+                            stream.destroy();
+                            reject(createError(
+                                ErrorCodes.WRITE_ERROR,
+                                `deleteBatch 操作失败: ${error.message}`,
+                                null,
+                                error
+                            ));
+                            return;
+                        }
+
+                        // skip 或 collect：清空批次，继续
+                        batch = [];
+                        result.batchCount++;
+                        stream.resume();
+                    }
+                }
+            });
+
+            // 6. 监听流结束事件
+            stream.on('end', async () => {
+                // 处理剩余的批次
+                if (batch.length > 0) {
+                    try {
+                        const batchResult = await executeBatchWithRetry(
+                            () => nativeCollection.deleteMany(
+                                { _id: { $in: batch } },
+                                { writeConcern }
+                            ),
+                            { onError, retryAttempts, retryDelay, onRetry, batchIndex: result.batchCount }
+                        );
+
+                        result.deletedCount += batchResult.result.deletedCount;
+                        result.batchCount++;
+
+                        if (batchResult.attempts > 0) {
+                            result.retries.push({
+                                batchIndex: result.batchCount - 1,
+                                attempts: batchResult.attempts,
+                                success: true
+                            });
+                        }
+
+                        // 最后一批的进度回调
+                        if (onProgress) {
+                            onProgress({
+                                currentBatch: result.batchCount,
+                                deleted: result.deletedCount,
+                                total: totalCount,
+                                percentage: totalCount ? 100 : null,
+                                errors: result.errors.length,
+                                retries: result.retries.length
+                            });
+                        }
+
+                    } catch (error) {
+                        result.errors.push({
+                            batchIndex: result.batchCount,
+                            batchSize: batch.length,
+                            error: error,
+                            message: error.message
+                        });
+                    }
+                }
+
+                // 7. 最终缓存失效
+                if (cache && result.deletedCount > 0) {
+                    try {
+                        const ns = {
+                            iid: instanceId,
+                            type: 'mongodb',
+                            db: databaseName,
+                            collection: collectionName
+                        };
+                        const pattern = CacheFactory.buildNamespacePattern(ns);
+
+                        // 检查是否在事务中
+                        if (isInTransaction(options)) {
+                            const tx = getTransactionFromSession(options.session);
+                            if (tx && typeof tx.recordInvalidation === 'function') {
+                                await tx.recordInvalidation(pattern, {
+                                    operation: 'write',
+                                    query: filter || {},
+                                    collection: collectionName
+                                });
+                                logger.debug(`[deleteBatch] 事务中失效缓存: ${ns.db}.${ns.collection}`);
+                            } else {
+                                const deleted = await cache.delPattern(pattern);
+                                if (deleted > 0) {
+                                    logger.debug(`[deleteBatch] 自动失效缓存: ${ns.db}.${ns.collection}, 删除 ${deleted} 个缓存键`);
+                                }
+                            }
+                        } else {
+                            const deleted = await cache.delPattern(pattern);
+                            if (deleted > 0) {
+                                logger.debug(`[deleteBatch] 自动失效缓存: ${ns.db}.${ns.collection}, 删除 ${deleted} 个缓存键`);
+                            }
+                        }
+                    } catch (cacheErr) {
+                        logger.warn(`[deleteBatch] 缓存失效失败: ${cacheErr.message}`);
+                    }
+                }
+
+                // 8. 慢查询日志
+                const duration = Date.now() - startTime;
+                const slowQueryMs = defaults.slowQueryMs || 1000;
+
+                if (duration > slowQueryMs) {
+                    logger.warn(`[deleteBatch] 慢操作警告`, {
+                        ns: `${databaseName}.${collectionName}`,
+                        duration,
+                        threshold: slowQueryMs,
+                        totalCount,
+                        deletedCount: result.deletedCount,
+                        batchCount: result.batchCount,
+                        errorCount: result.errors.length,
+                        retryCount: result.retries.length,
+                        comment
+                    });
+                } else {
+                    logger.debug(`[deleteBatch] 操作完成`, {
+                        ns: `${databaseName}.${collectionName}`,
+                        duration,
+                        deletedCount: result.deletedCount,
+                        batchCount: result.batchCount
+                    });
+                }
+
+                resolve(result);
+            });
+
+            // 9. 监听流错误事件
+            stream.on('error', (error) => {
+                logger.error(`[deleteBatch] 流式查询错误`, {
+                    ns: `${databaseName}.${collectionName}`,
+                    error: error.message,
+                    code: error.code
+                });
+
+                result.errors.push({
+                    batchIndex: result.batchCount,
+                    error: error,
+                    message: `流式查询错误: ${error.message}`
+                });
+
+                reject(createError(
+                    ErrorCodes.WRITE_ERROR,
+                    `deleteBatch 流式查询失败: ${error.message}`,
+                    null,
+                    error
+                ));
+            });
+        });
+    };
+
+    return { deleteBatch };
+}
+
+module.exports = { createDeleteBatchOps };
+

package/lib/mongodb/writes/delete-many.js

@@ -3,9 +3,10 @@
  * 删除所有匹配的文档
  */
 
-const { createError, ErrorCodes } = require("../../errors");
-const CacheFactory = require("../../cache");
-const { isInTransaction, getTransactionFromSession } = require("../common/transaction-aware");
+const { createError, ErrorCodes } = require('../../errors');
+const CacheFactory = require('../../cache');
+const { isInTransaction, getTransactionFromSession } = require('../common/transaction-aware');
+const { convertObjectIdStrings } = require('../../utils/objectid-converter');
 
 /**
  * 创建 deleteMany 操作
@@ -61,36 +62,44 @@ function createDeleteManyOps(context) {
         const startTime = Date.now();
 
         // 1. 参数验证
-        if (!filter || typeof filter !== "object" || Array.isArray(filter)) {
+        if (!filter || typeof filter !== 'object' || Array.isArray(filter)) {
             throw createError(
                 ErrorCodes.INVALID_ARGUMENT,
-                "filter 必须是对象类型",
-                [{ field: "filter", type: "object.required", message: "filter 是必需参数且必须是对象" }]
+                'filter 必须是对象类型',
+                [{ field: 'filter', type: 'object.required', message: 'filter 是必需参数且必须是对象' }]
             );
         }
 
+        // ✅ v1.3.0: 自动转换 ObjectId 字符串
+        const convertedFilter = convertObjectIdStrings(filter, 'filter', 0, new WeakSet(), {
+            logger: context.logger,
+            excludeFields: context.autoConvertConfig?.excludeFields,
+            customFieldPatterns: context.autoConvertConfig?.customFieldPatterns,
+            maxDepth: context.autoConvertConfig?.maxDepth
+        });
+
         // 2. 警告：空 filter 会删除所有文档
-        if (Object.keys(filter).length === 0) {
-            logger.warn(`[deleteMany] 警告: 空 filter 将删除集合中的所有文档`, {
+        if (Object.keys(convertedFilter).length === 0) {
+            logger.warn('[deleteMany] 警告: 空 filter 将删除集合中的所有文档', {
                 ns: `${databaseName}.${collectionName}`,
                 comment: options.comment
             });
         }
 
         // 3. 构建操作上下文
-        const operation = "deleteMany";
+        const operation = 'deleteMany';
         const ns = `${databaseName}.${collectionName}`;
 
         try {
             // 4. 执行删除操作
-            const result = await nativeCollection.deleteMany(filter, options);
+            const result = await nativeCollection.deleteMany(convertedFilter, options);
 
             // 5. 自动失效缓存（如果有文档被删除）
             if (cache && result.deletedCount > 0) {
                 try {
                     const ns = {
                         iid: instanceId,
-                        type: "mongodb",
+                        type: 'mongodb',
                         db: databaseName,
                         collection: collectionName
                     };

package/lib/mongodb/writes/delete-one.js

@@ -3,9 +3,10 @@
  * 删除单个匹配的文档
  */
 
-const { createError, ErrorCodes } = require("../../errors");
-const CacheFactory = require("../../cache");
-const { isInTransaction, getTransactionFromSession } = require("../common/transaction-aware");
+const { createError, ErrorCodes } = require('../../errors');
+const CacheFactory = require('../../cache');
+const { isInTransaction, getTransactionFromSession } = require('../common/transaction-aware');
+const { convertObjectIdStrings } = require('../../utils/objectid-converter');
 
 /**
  * 创建 deleteOne 操作
@@ -61,28 +62,36 @@ function createDeleteOneOps(context) {
         const startTime = Date.now();
 
         // 1. 参数验证
-        if (!filter || typeof filter !== "object" || Array.isArray(filter)) {
+        if (!filter || typeof filter !== 'object' || Array.isArray(filter)) {
             throw createError(
                 ErrorCodes.INVALID_ARGUMENT,
-                "filter 必须是对象类型",
-                [{ field: "filter", type: "object.required", message: "filter 是必需参数且必须是对象" }]
+                'filter 必须是对象类型',
+                [{ field: 'filter', type: 'object.required', message: 'filter 是必需参数且必须是对象' }]
             );
         }
 
+        // ✅ v1.3.0: 自动转换 ObjectId 字符串
+        const convertedFilter = convertObjectIdStrings(filter, 'filter', 0, new WeakSet(), {
+            logger: context.logger,
+            excludeFields: context.autoConvertConfig?.excludeFields,
+            customFieldPatterns: context.autoConvertConfig?.customFieldPatterns,
+            maxDepth: context.autoConvertConfig?.maxDepth
+        });
+
         // 2. 构建操作上下文
-        const operation = "deleteOne";
+        const operation = 'deleteOne';
         const ns = `${databaseName}.${collectionName}`;
 
         try {
             // 3. 执行删除操作
-            const result = await nativeCollection.deleteOne(filter, options);
+            const result = await nativeCollection.deleteOne(convertedFilter, options);
 
             // 4. 自动失效缓存（如果有文档被删除）
             if (cache && result.deletedCount > 0) {
                 try {
                     const ns = {
                         iid: instanceId,
-                        type: "mongodb",
+                        type: 'mongodb',
                         db: databaseName,
                         collection: collectionName
                     };