monsqlize 1.0.1 → 1.0.5

package/lib/slow-query-log/base-storage.js

@@ -0,0 +1,69 @@
+/**
+ * 慢查询日志存储接口
+ * 所有存储适配器必须实现此接口
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+class ISlowQueryLogStorage {
+  /**
+   * 初始化存储（创建集合/表、索引等）
+   * @returns {Promise<void>}
+   */
+  async initialize() {
+    throw new Error('ISlowQueryLogStorage.initialize() must be implemented');
+  }
+
+  /**
+   * 保存单条慢查询日志
+   * @param {Object} log - 慢查询日志对象
+   * @param {string} log.db - 数据库名
+   * @param {string} log.collection - 集合名
+   * @param {string} log.operation - 操作类型
+   * @param {Object} log.queryShape - 查询模式（已脱敏）
+   * @param {number} log.executionTimeMs - 执行时间（毫秒）
+   * @param {Date} log.timestamp - 时间戳
+   * @param {Object} [log.metadata] - 扩展元数据
+   * @returns {Promise<void>}
+   */
+  async save(log) {
+    throw new Error('ISlowQueryLogStorage.save() must be implemented');
+  }
+
+  /**
+   * 批量保存慢查询日志
+   * @param {Object[]} logs - 慢查询日志数组
+   * @returns {Promise<void>}
+   */
+  async saveBatch(logs) {
+    throw new Error('ISlowQueryLogStorage.saveBatch() must be implemented');
+  }
+
+  /**
+   * 查询慢查询日志
+   * @param {Object} filter - 查询条件
+   * @param {string} [filter.db] - 数据库名
+   * @param {string} [filter.collection] - 集合名
+   * @param {string} [filter.operation] - 操作类型
+   * @param {Object} options - 查询选项
+   * @param {Object} [options.sort] - 排序规则
+   * @param {number} [options.limit] - 限制数量
+   * @param {number} [options.skip] - 跳过数量
+   * @returns {Promise<Object[]>}
+   */
+  async query(filter, options) {
+    throw new Error('ISlowQueryLogStorage.query() must be implemented');
+  }
+
+  /**
+   * 关闭连接
+   * @returns {Promise<void>}
+   */
+  async close() {
+    throw new Error('ISlowQueryLogStorage.close() must be implemented');
+  }
+}
+
+module.exports = { ISlowQueryLogStorage };
+

package/lib/slow-query-log/batch-queue.js

@@ -0,0 +1,96 @@
+/**
+ * 批量队列管理器
+ * 用于批量写入慢查询日志，优化性能
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+class BatchQueue {
+  /**
+   * 创建批量队列
+   * @param {Object} storage - 存储适配器实例
+   * @param {Object} options - 队列配置
+   * @param {number} [options.batchSize=10] - 批量大小
+   * @param {number} [options.flushInterval=5000] - 刷新间隔（毫秒）
+   * @param {number} [options.maxBufferSize=100] - 最大缓冲区大小
+   * @param {Object} [logger] - 日志记录器
+   */
+  constructor(storage, options = {}, logger) {
+    this.storage = storage;
+    this.buffer = [];
+    this.batchSize = options.batchSize || 10;
+    this.flushInterval = options.flushInterval || 5000;
+    this.maxBufferSize = options.maxBufferSize || 100;
+    this.timer = null;
+    this.flushing = false;
+    this.logger = logger || console;
+  }
+
+  /**
+   * 添加日志到队列
+   * @param {Object} log - 慢查询日志对象
+   * @returns {Promise<void>}
+   */
+  async add(log) {
+    this.buffer.push(log);
+
+    // 防止内存溢出：达到最大缓冲区，立即刷新
+    if (this.buffer.length >= this.maxBufferSize) {
+      await this.flush();
+      return;
+    }
+
+    // 达到批量大小，立即刷新
+    if (this.buffer.length >= this.batchSize) {
+      await this.flush();
+    } else if (!this.timer) {
+      // 启动定时器（防止数据积压）
+      this.timer = setTimeout(() => this.flush(), this.flushInterval);
+    }
+  }
+
+  /**
+   * 刷新队列（批量写入）
+   * @returns {Promise<void>}
+   */
+  async flush() {
+    // 防止并发刷新
+    if (this.flushing || this.buffer.length === 0) {
+      return;
+    }
+
+    this.flushing = true;
+    const logs = this.buffer.splice(0);  // 清空缓冲区
+    clearTimeout(this.timer);
+    this.timer = null;
+
+    try {
+      // 批量写入
+      await this.storage.saveBatch(logs);
+      if (this.logger.debug) {
+        this.logger.debug(`[SlowQueryLog] Batch flushed: ${logs.length} logs`);
+      }
+    } catch (err) {
+      // ⚠️ 失败不阻塞主流程，仅记录错误
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to save slow query logs batch:', err);
+      }
+    } finally {
+      this.flushing = false;
+    }
+  }
+
+  /**
+   * 关闭队列（确保数据不丢失）
+   * @returns {Promise<void>}
+   */
+  async close() {
+    clearTimeout(this.timer);
+    this.timer = null;
+    await this.flush();  // 最后刷新一次
+  }
+}
+
+module.exports = { BatchQueue };
+

package/lib/slow-query-log/config-manager.js

@@ -0,0 +1,195 @@
+/**
+ * 慢查询日志配置管理器
+ * 负责配置合并、验证和默认值处理
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+// 默认配置
+const DEFAULT_CONFIG = {
+  enabled: false,
+
+  storage: {
+    type: null,  // null = 自动推断
+    useBusinessConnection: true,
+    uri: null,
+
+    mongodb: {
+      database: 'admin',
+      collection: 'slow_query_logs',
+      ttl: 7 * 24 * 3600,  // 7天
+      ttlField: 'lastSeen'
+    }
+  },
+
+  deduplication: {
+    enabled: true,
+    strategy: 'aggregate',
+    keepRecentExecutions: 0
+  },
+
+  batch: {
+    enabled: true,
+    size: 10,
+    interval: 5000,
+    maxBufferSize: 100
+  },
+
+  filter: {
+    excludeDatabases: [],
+    excludeCollections: [],
+    excludeOperations: [],
+    minExecutionTimeMs: 0
+  },
+
+  advanced: {
+    autoCreateIndexes: true,
+    validateConnection: true,
+    errorHandling: 'log',  // log | throw | silent
+    debug: false
+  }
+};
+
+class SlowQueryLogConfigManager {
+  /**
+   * 合并用户配置与默认配置
+   * @param {*} userConfig - 用户配置（可以是boolean或object）
+   * @param {string} businessType - 业务库类型
+   * @returns {Object} 合并后的完整配置
+   */
+  static mergeConfig(userConfig, businessType) {
+    // 场景1：未配置（默认禁用）
+    if (userConfig === undefined || userConfig === null) {
+      return { ...deepClone(DEFAULT_CONFIG), enabled: false };
+    }
+
+    // 场景2：boolean快捷配置
+    if (typeof userConfig === 'boolean') {
+      const config = deepClone(DEFAULT_CONFIG);
+      config.enabled = userConfig;
+
+      // 自动推断storage.type
+      if (userConfig && businessType) {
+        config.storage.type = businessType;
+      }
+
+      return config;
+    }
+
+    // 场景3：对象配置（深度合并）
+    if (typeof userConfig === 'object') {
+      const merged = deepMerge(deepClone(DEFAULT_CONFIG), userConfig);
+
+      // 智能推断：如果提供了storage配置，自动启用
+      if (userConfig.storage && merged.enabled === false) {
+        merged.enabled = true;
+      }
+
+      // 自动推断storage.type
+      if (merged.storage.type === null && businessType) {
+        if (merged.storage.useBusinessConnection) {
+          merged.storage.type = businessType;
+        } else {
+          merged.storage.type = 'mongodb';  // 独立连接默认MongoDB
+        }
+      }
+
+      return merged;
+    }
+
+    throw new Error('Invalid slowQueryLog config type. Expected boolean or object');
+  }
+
+  /**
+   * 验证配置合法性
+   * @param {Object} config - 配置对象
+   * @param {string} businessType - 业务库类型
+   * @throws {Error} 配置错误时抛出异常
+   */
+  static validate(config, businessType) {
+    const { storage } = config;
+
+    // 验证storage.type
+    const validTypes = ['mongodb', 'postgresql', 'mysql', 'file', 'custom'];
+    if (storage.type && !validTypes.includes(storage.type)) {
+      throw new Error(
+        `Invalid storage.type: ${storage.type}. ` +
+        `Valid types are: ${validTypes.join(', ')}`
+      );
+    }
+
+    // 验证复用连接的类型一致性
+    if (storage.useBusinessConnection === true) {
+      if (storage.type && storage.type !== businessType) {
+        throw new Error(
+          `Cannot use business connection when storage type (${storage.type}) ` +
+          `differs from business type (${businessType}). ` +
+          `Set useBusinessConnection=false and provide storage.uri`
+        );
+      }
+    } else {
+      // 验证独立连接的uri
+      if (!storage.uri) {
+        throw new Error(
+          'storage.uri is required when useBusinessConnection=false'
+        );
+      }
+    }
+
+    // 验证deduplication.strategy
+    const validStrategies = ['aggregate', 'none'];
+    if (!validStrategies.includes(config.deduplication.strategy)) {
+      throw new Error(
+        `Invalid deduplication.strategy: ${config.deduplication.strategy}. ` +
+        `Valid strategies are: ${validStrategies.join(', ')}`
+      );
+    }
+
+    // 验证TTL
+    if (storage.mongodb.ttl < 0) {
+      throw new Error('storage.mongodb.ttl must be positive');
+    }
+
+    // 验证batch配置
+    if (config.batch.size < 1 || config.batch.size > 1000) {
+      throw new Error('batch.size must be between 1 and 1000');
+    }
+
+    if (config.batch.interval < 100) {
+      throw new Error('batch.interval must be >= 100ms');
+    }
+
+    return true;
+  }
+}
+
+/**
+ * 深度克隆对象
+ * @param {Object} obj - 源对象
+ * @returns {Object} 克隆后的对象
+ */
+function deepClone(obj) {
+  return JSON.parse(JSON.stringify(obj));
+}
+
+/**
+ * 深度合并对象
+ * @param {Object} target - 目标对象
+ * @param {Object} source - 源对象
+ * @returns {Object} 合并后的对象
+ */
+function deepMerge(target, source) {
+  for (const key in source) {
+    if (source[key] && typeof source[key] === 'object' && !Array.isArray(source[key])) {
+      target[key] = target[key] || {};
+      deepMerge(target[key], source[key]);
+    } else {
+      target[key] = source[key];
+    }
+  }
+  return target;
+}
+
+module.exports = { SlowQueryLogConfigManager, DEFAULT_CONFIG };
+

package/lib/slow-query-log/index.js

@@ -0,0 +1,237 @@
+/**
+ * 慢查询日志模块导出
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+const { ISlowQueryLogStorage } = require('./base-storage');
+const { MongoDBSlowQueryLogStorage } = require('./mongodb-storage');
+const { BatchQueue } = require('./batch-queue');
+const { generateQueryHash } = require('./query-hash');
+const { SlowQueryLogConfigManager, DEFAULT_CONFIG } = require('./config-manager');
+
+/**
+ * 慢查询日志管理器
+ * 统一管理存储、队列、配置
+ */
+class SlowQueryLogManager {
+  /**
+   * 创建慢查询日志管理器
+   * @param {Object} userConfig - 用户配置
+   * @param {Object} [businessClient] - 业务数据库客户端（复用连接时）
+   * @param {string} businessType - 业务数据库类型
+   * @param {Object} logger - 日志记录器
+   */
+  constructor(userConfig, businessClient, businessType, logger) {
+    this.logger = logger || console;
+
+    // 合并配置
+    this.config = SlowQueryLogConfigManager.mergeConfig(userConfig, businessType);
+
+    // 验证配置
+    SlowQueryLogConfigManager.validate(this.config, businessType);
+
+    // 初始化存储适配器
+    this.storage = this.createStorage(businessClient);
+
+    // 初始化批量队列
+    if (this.config.batch.enabled) {
+      this.queue = new BatchQueue(this.storage, this.config.batch, this.logger);
+    } else {
+      this.queue = null;
+    }
+
+    this.initialized = false;
+  }
+
+  /**
+   * 创建存储适配器
+   * @param {Object} businessClient - 业务数据库客户端
+   * @returns {ISlowQueryLogStorage}
+   */
+  createStorage(businessClient) {
+    const { storage } = this.config;
+
+    // 根据存储类型创建适配器
+    switch (storage.type) {
+      case 'mongodb': {
+        const client = storage.useBusinessConnection ? businessClient : null;
+        return new MongoDBSlowQueryLogStorage(
+          storage.mongodb,
+          client,
+          this.logger
+        );
+      }
+
+      case 'postgresql':
+      case 'mysql':
+      case 'file':
+      case 'custom':
+        throw new Error(
+          `Storage type '${storage.type}' is not yet implemented. ` +
+          `Currently only 'mongodb' is supported in v1.3`
+        );
+
+      default:
+        throw new Error(`Unknown storage type: ${storage.type}`);
+    }
+  }
+
+  /**
+   * 初始化（延迟初始化）
+   * @returns {Promise<void>}
+   */
+  async initialize() {
+    if (this.initialized) {
+      return;
+    }
+
+    try {
+      await this.storage.initialize();
+      this.initialized = true;
+
+      if (this.logger.info) {
+        this.logger.info('[SlowQueryLog] Manager initialized successfully');
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to initialize manager:', err);
+      }
+
+      // 根据错误处理策略决定是否抛出异常
+      if (this.config.advanced.errorHandling === 'throw') {
+        throw err;
+      }
+    }
+  }
+
+  /**
+   * 保存慢查询日志
+   * @param {Object} log - 慢查询日志对象
+   * @returns {Promise<void>}
+   */
+  async save(log) {
+    // 延迟初始化
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    // 过滤检查
+    if (this.shouldFilter(log)) {
+      return;
+    }
+
+    try {
+      if (this.queue) {
+        // 批量模式：添加到队列
+        await this.queue.add(log);
+      } else {
+        // 实时模式：直接保存
+        await this.storage.save(log);
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to save log:', err);
+      }
+
+      // 保存失败不抛异常（不影响主流程）
+      if (this.config.advanced.errorHandling === 'throw') {
+        throw err;
+      }
+    }
+  }
+
+  /**
+   * 检查是否应该过滤该日志
+   * @param {Object} log - 慢查询日志对象
+   * @returns {boolean}
+   */
+  shouldFilter(log) {
+    const { filter } = this.config;
+
+    // 检查数据库过滤
+    if (filter.excludeDatabases.length > 0) {
+      if (filter.excludeDatabases.includes(log.db)) {
+        return true;
+      }
+    }
+
+    // 检查集合过滤
+    if (filter.excludeCollections.length > 0) {
+      const coll = log.collection || log.coll;
+      if (filter.excludeCollections.includes(coll)) {
+        return true;
+      }
+    }
+
+    // 检查操作类型过滤
+    if (filter.excludeOperations.length > 0) {
+      const op = log.operation || log.op;
+      if (filter.excludeOperations.includes(op)) {
+        return true;
+      }
+    }
+
+    // 检查最小执行时间
+    if (filter.minExecutionTimeMs > 0) {
+      const ms = log.executionTimeMs || log.ms || 0;
+      if (ms < filter.minExecutionTimeMs) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * 查询慢查询日志
+   * @param {Object} filter - 查询条件
+   * @param {Object} options - 查询选项
+   * @returns {Promise<Object[]>}
+   */
+  async query(filter, options) {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    return this.storage.query(filter, options);
+  }
+
+  /**
+   * 关闭管理器
+   * @returns {Promise<void>}
+   */
+  async close() {
+    try {
+      // 关闭队列（确保数据不丢失）
+      if (this.queue) {
+        await this.queue.close();
+      }
+
+      // 关闭存储
+      await this.storage.close();
+
+      this.initialized = false;
+
+      if (this.logger.info) {
+        this.logger.info('[SlowQueryLog] Manager closed');
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to close manager:', err);
+      }
+    }
+  }
+}
+
+module.exports = {
+  SlowQueryLogManager,
+  SlowQueryLogConfigManager,
+  MongoDBSlowQueryLogStorage,
+  BatchQueue,
+  generateQueryHash,
+  DEFAULT_CONFIG,
+  ISlowQueryLogStorage
+};
+