monsqlize 1.0.0 → 1.0.2

package/lib/slow-query-log/index.js

@@ -0,0 +1,237 @@
+/**
+ * 慢查询日志模块导出
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+const { ISlowQueryLogStorage } = require('./base-storage');
+const { MongoDBSlowQueryLogStorage } = require('./mongodb-storage');
+const { BatchQueue } = require('./batch-queue');
+const { generateQueryHash } = require('./query-hash');
+const { SlowQueryLogConfigManager, DEFAULT_CONFIG } = require('./config-manager');
+
+/**
+ * 慢查询日志管理器
+ * 统一管理存储、队列、配置
+ */
+class SlowQueryLogManager {
+  /**
+   * 创建慢查询日志管理器
+   * @param {Object} userConfig - 用户配置
+   * @param {Object} [businessClient] - 业务数据库客户端（复用连接时）
+   * @param {string} businessType - 业务数据库类型
+   * @param {Object} logger - 日志记录器
+   */
+  constructor(userConfig, businessClient, businessType, logger) {
+    this.logger = logger || console;
+
+    // 合并配置
+    this.config = SlowQueryLogConfigManager.mergeConfig(userConfig, businessType);
+
+    // 验证配置
+    SlowQueryLogConfigManager.validate(this.config, businessType);
+
+    // 初始化存储适配器
+    this.storage = this.createStorage(businessClient);
+
+    // 初始化批量队列
+    if (this.config.batch.enabled) {
+      this.queue = new BatchQueue(this.storage, this.config.batch, this.logger);
+    } else {
+      this.queue = null;
+    }
+
+    this.initialized = false;
+  }
+
+  /**
+   * 创建存储适配器
+   * @param {Object} businessClient - 业务数据库客户端
+   * @returns {ISlowQueryLogStorage}
+   */
+  createStorage(businessClient) {
+    const { storage } = this.config;
+
+    // 根据存储类型创建适配器
+    switch (storage.type) {
+      case 'mongodb': {
+        const client = storage.useBusinessConnection ? businessClient : null;
+        return new MongoDBSlowQueryLogStorage(
+          storage.mongodb,
+          client,
+          this.logger
+        );
+      }
+
+      case 'postgresql':
+      case 'mysql':
+      case 'file':
+      case 'custom':
+        throw new Error(
+          `Storage type '${storage.type}' is not yet implemented. ` +
+          `Currently only 'mongodb' is supported in v1.3`
+        );
+
+      default:
+        throw new Error(`Unknown storage type: ${storage.type}`);
+    }
+  }
+
+  /**
+   * 初始化（延迟初始化）
+   * @returns {Promise<void>}
+   */
+  async initialize() {
+    if (this.initialized) {
+      return;
+    }
+
+    try {
+      await this.storage.initialize();
+      this.initialized = true;
+
+      if (this.logger.info) {
+        this.logger.info('[SlowQueryLog] Manager initialized successfully');
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to initialize manager:', err);
+      }
+
+      // 根据错误处理策略决定是否抛出异常
+      if (this.config.advanced.errorHandling === 'throw') {
+        throw err;
+      }
+    }
+  }
+
+  /**
+   * 保存慢查询日志
+   * @param {Object} log - 慢查询日志对象
+   * @returns {Promise<void>}
+   */
+  async save(log) {
+    // 延迟初始化
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    // 过滤检查
+    if (this.shouldFilter(log)) {
+      return;
+    }
+
+    try {
+      if (this.queue) {
+        // 批量模式：添加到队列
+        await this.queue.add(log);
+      } else {
+        // 实时模式：直接保存
+        await this.storage.save(log);
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to save log:', err);
+      }
+
+      // 保存失败不抛异常（不影响主流程）
+      if (this.config.advanced.errorHandling === 'throw') {
+        throw err;
+      }
+    }
+  }
+
+  /**
+   * 检查是否应该过滤该日志
+   * @param {Object} log - 慢查询日志对象
+   * @returns {boolean}
+   */
+  shouldFilter(log) {
+    const { filter } = this.config;
+
+    // 检查数据库过滤
+    if (filter.excludeDatabases.length > 0) {
+      if (filter.excludeDatabases.includes(log.db)) {
+        return true;
+      }
+    }
+
+    // 检查集合过滤
+    if (filter.excludeCollections.length > 0) {
+      const coll = log.collection || log.coll;
+      if (filter.excludeCollections.includes(coll)) {
+        return true;
+      }
+    }
+
+    // 检查操作类型过滤
+    if (filter.excludeOperations.length > 0) {
+      const op = log.operation || log.op;
+      if (filter.excludeOperations.includes(op)) {
+        return true;
+      }
+    }
+
+    // 检查最小执行时间
+    if (filter.minExecutionTimeMs > 0) {
+      const ms = log.executionTimeMs || log.ms || 0;
+      if (ms < filter.minExecutionTimeMs) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * 查询慢查询日志
+   * @param {Object} filter - 查询条件
+   * @param {Object} options - 查询选项
+   * @returns {Promise<Object[]>}
+   */
+  async query(filter, options) {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    return this.storage.query(filter, options);
+  }
+
+  /**
+   * 关闭管理器
+   * @returns {Promise<void>}
+   */
+  async close() {
+    try {
+      // 关闭队列（确保数据不丢失）
+      if (this.queue) {
+        await this.queue.close();
+      }
+
+      // 关闭存储
+      await this.storage.close();
+
+      this.initialized = false;
+
+      if (this.logger.info) {
+        this.logger.info('[SlowQueryLog] Manager closed');
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to close manager:', err);
+      }
+    }
+  }
+}
+
+module.exports = {
+  SlowQueryLogManager,
+  SlowQueryLogConfigManager,
+  MongoDBSlowQueryLogStorage,
+  BatchQueue,
+  generateQueryHash,
+  DEFAULT_CONFIG,
+  ISlowQueryLogStorage
+};
+

package/lib/slow-query-log/mongodb-storage.js

@@ -0,0 +1,323 @@
+/**
+ * MongoDB慢查询日志存储实现
+ * 实现方案B（更新记录去重）
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+const { ISlowQueryLogStorage } = require('./base-storage');
+const { generateQueryHash } = require('./query-hash');
+
+class MongoDBSlowQueryLogStorage extends ISlowQueryLogStorage {
+  /**
+   * 创建MongoDB存储实例
+   * @param {Object} config - 存储配置
+   * @param {string} [config.uri] - MongoDB连接URI（独立连接时）
+   * @param {string} [config.database='admin'] - 存储数据库
+   * @param {string} [config.collection='slow_query_logs'] - 存储集合
+   * @param {number} [config.ttl=604800] - TTL过期时间（秒）
+   * @param {string} [config.ttlField='lastSeen'] - TTL字段
+   * @param {Object} [client] - MongoDB客户端（复用连接时）
+   * @param {Object} [logger] - 日志记录器
+   */
+  constructor(config, client, logger) {
+    super();
+    this.config = config;
+    this.client = client;  // 可能是复用的业务连接
+    this.ownClient = null;  // 独立连接（如果创建）
+    this.collection = null;
+    this.logger = logger || console;
+    this.initialized = false;
+  }
+
+  /**
+   * 初始化存储（连接+创建索引）
+   * @returns {Promise<void>}
+   */
+  async initialize() {
+    if (this.initialized) {
+      return;
+    }
+
+    try {
+      // 如果没有传入client，创建独立连接
+      if (!this.client) {
+        if (!this.config.uri) {
+          throw new Error('MongoDB URI is required when client is not provided');
+        }
+
+        const { MongoClient } = require('mongodb');
+        this.ownClient = await MongoClient.connect(this.config.uri);
+        this.client = this.ownClient;
+
+        if (this.logger.info) {
+          this.logger.info(`[SlowQueryLog] Connected to MongoDB: ${this.config.uri}`);
+        }
+      }
+
+      // 获取数据库和集合
+      const dbName = this.config.database || 'admin';
+      const collName = this.config.collection || 'slow_query_logs';
+
+      this.db = this.client.db(dbName);
+      this.collection = this.db.collection(collName);
+
+      // 创建索引
+      await this.setupIndexes();
+
+      this.initialized = true;
+
+      if (this.logger.info) {
+        this.logger.info(
+          `[SlowQueryLog] Initialized MongoDB storage: ${dbName}.${collName}`
+        );
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to initialize MongoDB storage:', err);
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * 创建索引
+   * @returns {Promise<void>}
+   */
+  async setupIndexes() {
+    try {
+      // 索引1：queryHash唯一索引（去重关键）
+      await this.collection.createIndex(
+        { queryHash: 1 },
+        { unique: true, name: 'idx_queryHash_unique' }
+      );
+
+      // 索引2：lastSeen TTL索引（自动过期）
+      const ttl = this.config.ttl || 7 * 24 * 3600;
+      const ttlField = this.config.ttlField || 'lastSeen';
+
+      await this.collection.createIndex(
+        { [ttlField]: 1 },
+        {
+          name: 'idx_lastSeen_ttl',
+          expireAfterSeconds: ttl
+        }
+      );
+
+      // 索引3：按集合查询（辅助索引）
+      await this.collection.createIndex(
+        { db: 1, collection: 1 },
+        { name: 'idx_db_collection' }
+      );
+
+      // 索引4：按执行次数查询（找高频慢查询）
+      await this.collection.createIndex(
+        { count: -1 },
+        { name: 'idx_count_desc' }
+      );
+
+      if (this.logger.debug) {
+        this.logger.debug('[SlowQueryLog] MongoDB indexes created successfully');
+      }
+    } catch (err) {
+      // 索引创建失败不应该阻塞功能
+      if (this.logger.warn) {
+        this.logger.warn('[SlowQueryLog] Failed to create some indexes:', err.message);
+      }
+    }
+  }
+
+  /**
+   * 保存单条慢查询日志（upsert）
+   * @param {Object} log - 慢查询日志对象
+   * @returns {Promise<void>}
+   */
+  async save(log) {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    const queryHash = generateQueryHash(log);
+    await this.upsert(queryHash, log);
+  }
+
+  /**
+   * 批量保存慢查询日志（bulkWrite + upsert）
+   * @param {Object[]} logs - 慢查询日志数组
+   * @returns {Promise<void>}
+   */
+  async saveBatch(logs) {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    if (!logs || logs.length === 0) {
+      return;
+    }
+
+    try {
+      // 构建bulkWrite操作数组
+      const operations = logs.map(log => {
+        const queryHash = generateQueryHash(log);
+        const timestamp = log.timestamp || new Date();
+        const executionTimeMs = log.executionTimeMs || log.ms || 0;
+
+        return {
+          updateOne: {
+            filter: { queryHash },
+            update: {
+              $set: {
+                lastSeen: timestamp,
+                lastExecution: {
+                  executionTimeMs,
+                  timestamp,
+                  metadata: log.metadata || {}
+                }
+              },
+              $inc: {
+                count: 1,
+                totalTimeMs: executionTimeMs
+              },
+              $min: { minTimeMs: executionTimeMs },
+              $max: { maxTimeMs: executionTimeMs },
+              $setOnInsert: {
+                queryHash,
+                db: log.db,
+                collection: log.collection || log.coll,
+                operation: log.operation || log.op,
+                queryShape: log.queryShape || {},
+                type: log.type || 'mongodb',
+                firstSeen: timestamp
+              }
+            },
+            upsert: true
+          }
+        };
+      });
+
+      // 批量写入（ordered=false 允许部分失败）
+      const result = await this.collection.bulkWrite(operations, { ordered: false });
+
+      if (this.logger.debug) {
+        this.logger.debug(
+          `[SlowQueryLog] Batch saved: ${result.upsertedCount} inserted, ` +
+          `${result.modifiedCount} updated`
+        );
+      }
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to save batch:', err);
+      }
+      throw err;
+    }
+  }
+
+  /**
+   * 单条upsert操作
+   * @param {string} queryHash - 查询Hash
+   * @param {Object} log - 慢查询日志对象
+   * @returns {Promise<void>}
+   */
+  async upsert(queryHash, log) {
+    const timestamp = log.timestamp || new Date();
+    const executionTimeMs = log.executionTimeMs || log.ms || 0;
+
+    await this.collection.updateOne(
+      { queryHash },
+      {
+        $set: {
+          lastSeen: timestamp,
+          lastExecution: {
+            executionTimeMs,
+            timestamp,
+            metadata: log.metadata || {}
+          }
+        },
+        $inc: {
+          count: 1,
+          totalTimeMs: executionTimeMs
+        },
+        $min: { minTimeMs: executionTimeMs },
+        $max: { maxTimeMs: executionTimeMs },
+        $setOnInsert: {
+          queryHash,
+          db: log.db,
+          collection: log.collection || log.coll,
+          operation: log.operation || log.op,
+          queryShape: log.queryShape || {},
+          type: log.type || 'mongodb',
+          firstSeen: timestamp
+        }
+      },
+      { upsert: true }
+    );
+  }
+
+  /**
+   * 查询慢查询日志
+   * @param {Object} filter - 查询条件
+   * @param {Object} options - 查询选项
+   * @returns {Promise<Object[]>}
+   */
+  async query(filter = {}, options = {}) {
+    if (!this.initialized) {
+      await this.initialize();
+    }
+
+    try {
+      const cursor = this.collection.find(filter);
+
+      if (options.sort) {
+        cursor.sort(options.sort);
+      }
+      if (options.limit) {
+        cursor.limit(options.limit);
+      }
+      if (options.skip) {
+        cursor.skip(options.skip);
+      }
+
+      const results = await cursor.toArray();
+
+      // 计算avgTimeMs（动态计算）
+      return results.map(doc => ({
+        ...doc,
+        avgTimeMs: doc.count > 0 ? Math.round(doc.totalTimeMs / doc.count) : 0
+      }));
+    } catch (err) {
+      if (this.logger.error) {
+        this.logger.error('[SlowQueryLog] Failed to query:', err);
+      }
+      return [];
+    }
+  }
+
+  /**
+   * 关闭连接
+   * @returns {Promise<void>}
+   */
+  async close() {
+    // 只关闭自己创建的连接，不关闭复用的连接
+    if (this.ownClient) {
+      try {
+        await this.ownClient.close();
+        if (this.logger.info) {
+          this.logger.info('[SlowQueryLog] MongoDB connection closed');
+        }
+      } catch (err) {
+        if (this.logger.error) {
+          this.logger.error('[SlowQueryLog] Failed to close MongoDB connection:', err);
+        }
+      }
+      this.ownClient = null;
+    }
+
+    this.client = null;
+    this.collection = null;
+    this.initialized = false;
+  }
+}
+
+module.exports = { MongoDBSlowQueryLogStorage };
+

package/lib/slow-query-log/query-hash.js

@@ -0,0 +1,38 @@
+/**
+ * 慢查询Hash生成工具
+ * 用于生成queryHash（去重标识）
+ *
+ * @version 1.3.0
+ * @since 2025-12-22
+ */
+
+const crypto = require('crypto');
+
+/**
+ * 生成慢查询的唯一Hash标识
+ * @param {Object} log - 慢查询日志对象
+ * @param {string} log.db - 数据库名
+ * @param {string} log.collection - 集合名
+ * @param {string} log.operation - 操作类型
+ * @param {Object} log.queryShape - 查询模式（已脱敏）
+ * @returns {string} 16位Hash字符串
+ */
+function generateQueryHash(log) {
+  // 构建唯一键（不包含executionTimeMs、timestamp等动态字段）
+  const key = JSON.stringify({
+    db: log.db || '',
+    collection: log.collection || log.coll || '',
+    operation: log.operation || log.op || '',
+    queryShape: log.queryShape || {}
+  });
+
+  // 生成SHA256 Hash，取前16位
+  return crypto
+    .createHash('sha256')
+    .update(key)
+    .digest('hex')
+    .substring(0, 16);
+}
+
+module.exports = { generateQueryHash };
+