@kne/fastify-mq 0.1.0

package/index.js

@@ -0,0 +1,126 @@
+const fp = require('fastify-plugin');
+const path = require('node:path');
+const { registry, sampler } = require('./libs/utils/metrics');
+
+module.exports = fp(async (fastify, options) => {
+  options = Object.assign(
+    {},
+    {
+      dbTableNamePrefix: 't_',
+      name: 'mq',
+      prefix: '/mq',
+      defaultMaxRetries: 3,
+      pollLimit: 10,
+      pollInterval: 5000,
+      lockTimeout: 30000,
+      lockRecoveryInterval: 30000,
+      retryBaseDelay: 1000,
+      retryMaxDelay: 60000,
+      metricsSampleInterval: 10000,
+      metricsMaxSamples: 360,
+      adapter: 'pg',
+      kafka: null,
+      topicPrefix: '',
+      kafkaGroupId: undefined,
+      getAuthenticate: type => {
+        switch (type) {
+          case 'dlq:manage':
+          case 'message':
+          case 'dlq':
+          case 'trace':
+          case 'dashboard':
+          default:
+            return [];
+        }
+      },
+      getMessageModel: () => {
+        return fastify[options.name]?.models?.message;
+      }
+    },
+    options
+  );
+
+  options._subscribers = {};
+  options._consumerTimer = null;
+  options._consumerRunning = false;
+  options._consumerRunningPromise = null;
+  options._lockRecoveryTimer = null;
+  options._adapter = null;
+
+  fastify.register(require('@kne/fastify-namespace'), {
+    options,
+    name: options.name,
+    modules: [
+      ['controllers', path.resolve(__dirname, './libs/controllers')],
+      [
+        'models',
+        await fastify.sequelize.addModels(path.resolve(__dirname, './libs/models'), {
+          prefix: options.dbTableNamePrefix,
+          getMessageModel: options.getMessageModel
+        })
+      ],
+      ['services', path.resolve(__dirname, './libs/services')]
+    ]
+  });
+
+  // Metrics endpoint
+  if (registry) {
+    fastify.get(`${options.prefix}/metrics`, async (request, reply) => {
+      reply.type('text/plain');
+      return registry.metrics();
+    });
+  }
+
+  // Start sampler
+  if (sampler) {
+    sampler.start();
+  }
+
+  // 启动锁定恢复定时器
+  fastify.addHook('onReady', () => {
+    const { services } = fastify[options.name];
+    if (services?.queue?.recoverLocked) {
+      options._lockRecoveryTimer = setInterval(async () => {
+        try {
+          const result = await services.queue.recoverLocked();
+          if (result.recovered > 0) {
+            console.warn(`[fastify-mq] Recovered ${result.recovered} timed-out locked messages`);
+          }
+        } catch (e) {
+          console.error('[fastify-mq] Lock recovery error:', e.message);
+        }
+      }, options.lockRecoveryInterval);
+      options._lockRecoveryTimer.unref();
+    }
+  });
+
+  // 优雅关闭
+  fastify.addHook('onClose', async () => {
+    const { services } = fastify[options.name];
+
+    // 停止消费者（优雅等待当前消费周期完成）
+    if (services?.queue?.stopConsumer) {
+      await services.queue.stopConsumer();
+    } else {
+      if (options._consumerTimer) {
+        clearInterval(options._consumerTimer);
+        options._consumerTimer = null;
+      }
+      options._consumerRunning = false;
+    }
+
+    // 停止锁定恢复定时器
+    if (options._lockRecoveryTimer) {
+      clearInterval(options._lockRecoveryTimer);
+      options._lockRecoveryTimer = null;
+    }
+
+    // 停止采样器
+    if (sampler) sampler.stop();
+
+    // 断开适配器连接
+    if (options._adapter?.disconnect) {
+      await options._adapter.disconnect();
+    }
+  });
+});

package/libs/constants.js

@@ -0,0 +1,18 @@
+const MESSAGE_STATUS = {
+  PENDING: 'PENDING',
+  PROCESSING: 'PROCESSING',
+  COMPLETED: 'COMPLETED',
+  FAILED: 'FAILED'
+};
+
+const TRACE_EVENTS = {
+  PUBLISHED: 'PUBLISHED',
+  PROCESSING: 'PROCESSING',
+  COMPLETED: 'COMPLETED',
+  FAILED: 'FAILED',
+  MOVED_TO_DLQ: 'MOVED_TO_DLQ',
+  REPLAYED: 'REPLAYED',
+  LOCK_RECOVERED: 'LOCK_RECOVERED'
+};
+
+module.exports = { MESSAGE_STATUS, TRACE_EVENTS };

package/libs/controllers/dashboard.js

@@ -0,0 +1,28 @@
+const fp = require('fastify-plugin');
+
+module.exports = fp(async (fastify, options) => {
+  const { services } = fastify[options.name];
+
+  fastify.get(
+    `${options.prefix}/dashboard`,
+    {
+      onRequest: options.getAuthenticate('dashboard'),
+      schema: {
+        summary: 'Dashboard 数据接口',
+        query: {
+          type: 'object',
+          properties: {
+            window: { type: 'number', default: 300000, description: 'Rate计算窗口(ms), 默认5分钟' },
+            step: { type: 'number', default: 60000, description: '时序数据步长(ms), 默认1分钟' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.dashboard.getData({
+        windowMs: request.query.window || 300000,
+        stepMs: request.query.step || 60000
+      });
+    }
+  );
+});

package/libs/controllers/dead-letter.js

@@ -0,0 +1,143 @@
+const fp = require('fastify-plugin');
+
+module.exports = fp(async (fastify, options) => {
+  const { services } = fastify[options.name];
+
+  fastify.get(
+    `${options.prefix}/dlq/list`,
+    {
+      onRequest: options.getAuthenticate('dlq'),
+      schema: {
+        summary: '死信列表',
+        query: {
+          type: 'object',
+          properties: {
+            topic: { type: 'string', description: '按主题筛选' },
+            replayed: { type: 'boolean', description: '按重放状态筛选' },
+            perPage: { type: 'number', default: 20 },
+            currentPage: { type: 'number', default: 1 }
+          }
+        }
+      }
+    },
+    async request => {
+      const { topic, replayed, perPage, currentPage } = request.query;
+      const filter = {};
+      if (topic) filter.topic = topic;
+      if (replayed !== undefined) filter.replayed = replayed;
+      return services.deadLetter.list({ filter, perPage, currentPage });
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/dlq/replay`,
+    {
+      onRequest: options.getAuthenticate('dlq:manage'),
+      schema: {
+        summary: '重放死信(批量)',
+        body: {
+          type: 'object',
+          properties: {
+            id: { type: 'string', description: '死信ID(单条重放)' },
+            ids: { type: 'array', items: { type: 'string' }, description: '死信ID数组(批量重放)' }
+          }
+        }
+      }
+    },
+    async request => {
+      const { id, ids } = request.body;
+      if (ids && ids.length > 0) {
+        return services.deadLetter.batchReplay({ ids });
+      }
+      return services.deadLetter.replay({ id });
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/dlq/replay/:id`,
+    {
+      onRequest: options.getAuthenticate('dlq:manage'),
+      schema: {
+        summary: '重放死信(单条)',
+        params: {
+          type: 'object',
+          required: ['id'],
+          properties: {
+            id: { type: 'string', description: '死信ID' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.deadLetter.replay({ id: request.params.id });
+    }
+  );
+
+  fastify.get(
+    `${options.prefix}/trace/detail`,
+    {
+      onRequest: options.getAuthenticate('trace'),
+      schema: {
+        summary: '查询消息轨迹(query参数)',
+        query: {
+          type: 'object',
+          required: ['traceId'],
+          properties: {
+            traceId: { type: 'string', description: '追踪ID' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.trace.get(request.query);
+    }
+  );
+
+  fastify.get(
+    `${options.prefix}/trace/:traceId`,
+    {
+      onRequest: options.getAuthenticate('trace'),
+      schema: {
+        summary: '查询消息轨迹(路径参数)',
+        params: {
+          type: 'object',
+          required: ['traceId'],
+          properties: {
+            traceId: { type: 'string', description: '追踪ID' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.trace.get({ traceId: request.params.traceId });
+    }
+  );
+
+  fastify.get(
+    `${options.prefix}/trace/list`,
+    {
+      onRequest: options.getAuthenticate('trace'),
+      schema: {
+        summary: '消息轨迹列表',
+        query: {
+          type: 'object',
+          properties: {
+            topic: { type: 'string', description: '按主题筛选' },
+            messageId: { type: 'string', description: '按消息ID筛选' },
+            event: { type: 'string', description: '按事件类型筛选' },
+            perPage: { type: 'number', default: 20 },
+            currentPage: { type: 'number', default: 1 }
+          }
+        }
+      }
+    },
+    async request => {
+      const { topic, messageId, event, perPage, currentPage } = request.query;
+      const filter = {};
+      if (topic) filter.topic = topic;
+      if (messageId) filter.messageId = messageId;
+      if (event) filter.event = event;
+      return services.trace.list({ filter, perPage, currentPage });
+    }
+  );
+});

package/libs/controllers/message.js

@@ -0,0 +1,161 @@
+const fp = require('fastify-plugin');
+
+module.exports = fp(async (fastify, options) => {
+  const { services } = fastify[options.name];
+
+  fastify.post(
+    `${options.prefix}/message/publish`,
+    {
+      onRequest: options.getAuthenticate('message'),
+      schema: {
+        summary: '发布消息',
+        body: {
+          type: 'object',
+          required: ['topic', 'payload'],
+          properties: {
+            topic: { type: 'string', description: '消息主题' },
+            payload: { type: 'object', description: '消息内容' },
+            priority: { type: 'number', default: 0, description: '优先级' },
+            executeAt: { type: 'string', format: 'date-time', description: '定时执行时间' },
+            maxRetries: { type: 'number', description: '最大重试次数' },
+            traceId: { type: 'string', description: '追踪ID' },
+            meta: { type: 'object', description: '扩展元数据' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.message.publish(request.body);
+    }
+  );
+
+  fastify.get(
+    `${options.prefix}/message/poll`,
+    {
+      onRequest: options.getAuthenticate('message'),
+      schema: {
+        summary: '拉取消息',
+        query: {
+          type: 'object',
+          required: ['topic'],
+          properties: {
+            topic: { type: 'string', description: '消息主题' },
+            limit: { type: 'number', default: 10, description: '拉取数量' },
+            lockTimeout: { type: 'number', description: '锁定超时时间(ms)' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.message.poll(request.query);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/message/complete`,
+    {
+      onRequest: options.getAuthenticate('message'),
+      schema: {
+        summary: '确认消息完成',
+        body: {
+          type: 'object',
+          required: ['id'],
+          properties: {
+            id: { type: 'string', description: '消息ID' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.message.complete(request.body);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/message/fail`,
+    {
+      onRequest: options.getAuthenticate('message'),
+      schema: {
+        summary: '标记消息失败',
+        body: {
+          type: 'object',
+          required: ['id'],
+          properties: {
+            id: { type: 'string', description: '消息ID' },
+            error: { type: 'string', description: '错误信息' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.message.fail(request.body);
+    }
+  );
+
+  fastify.get(
+    `${options.prefix}/message/list`,
+    {
+      onRequest: options.getAuthenticate('message'),
+      schema: {
+        summary: '消息列表',
+        query: {
+          type: 'object',
+          properties: {
+            topic: { type: 'string', description: '按主题筛选' },
+            status: { type: 'string', description: '按状态筛选' },
+            traceId: { type: 'string', description: '按追踪ID筛选' },
+            perPage: { type: 'number', default: 20 },
+            currentPage: { type: 'number', default: 1 }
+          }
+        }
+      }
+    },
+    async request => {
+      const { topic, status, traceId, perPage, currentPage } = request.query;
+      const filter = {};
+      if (topic) filter.topic = topic;
+      if (status) filter.status = status;
+      if (traceId) filter.traceId = traceId;
+      return services.message.list({ filter, perPage, currentPage });
+    }
+  );
+
+  fastify.get(
+    `${options.prefix}/queue/depth`,
+    {
+      onRequest: options.getAuthenticate('message'),
+      schema: {
+        summary: '队列深度',
+        query: {
+          type: 'object',
+          properties: {
+            topic: { type: 'string', description: '消息主题(可选,不传则查询全部)' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.queue.getDepth(request.query);
+    }
+  );
+
+  fastify.post(
+    `${options.prefix}/queue/cleanup`,
+    {
+      onRequest: options.getAuthenticate('dlq:manage'),
+      schema: {
+        summary: '清理已完成的消息',
+        body: {
+          type: 'object',
+          properties: {
+            status: { type: 'string', default: 'COMPLETED', description: '要清理的消息状态' },
+            olderThan: { type: 'string', format: 'date-time', description: '清理此时间之前更新的消息' }
+          }
+        }
+      }
+    },
+    async request => {
+      return services.queue.cleanup(request.body);
+    }
+  );
+});

package/libs/models/dead-letter.js

@@ -0,0 +1,44 @@
+module.exports = ({ DataTypes, options }) => {
+  return {
+    name: 'deadLetter',
+    model: {
+      originalId: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        comment: '原始消息ID'
+      },
+      topic: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        comment: '消息主题'
+      },
+      payload: {
+        type: DataTypes.JSONB,
+        allowNull: false,
+        comment: '消息内容'
+      },
+      errorMessage: {
+        type: DataTypes.TEXT,
+        comment: '错误信息'
+      },
+      replayed: {
+        type: DataTypes.BOOLEAN,
+        allowNull: false,
+        defaultValue: false,
+        comment: '是否已重放'
+      },
+      replayedAt: {
+        type: DataTypes.DATE,
+        comment: '重放时间'
+      },
+      options: {
+        type: DataTypes.JSONB,
+        comment: '扩展字段'
+      }
+    },
+    options: {
+      comment: 'MQ死信表',
+      indexes: [{ fields: ['original_id'] }, { fields: ['topic'] }, { fields: ['replayed'] }, { fields: ['created_at'] }]
+    }
+  };
+};

package/libs/models/message-trace.js

@@ -0,0 +1,44 @@
+module.exports = ({ DataTypes, options }) => {
+  return {
+    name: 'messageTrace',
+    model: {
+      traceId: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        comment: '追踪ID'
+      },
+      topic: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        comment: '消息主题'
+      },
+      event: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        comment: '事件类型: PUBLISHED/PROCESSING/COMPLETED/FAILED/MOVED_TO_DLQ/REPLAYED'
+      },
+      messageId: {
+        type: DataTypes.STRING,
+        comment: '关联消息ID'
+      },
+      detail: {
+        type: DataTypes.JSONB,
+        comment: '事件详情'
+      },
+      options: {
+        type: DataTypes.JSONB,
+        comment: '扩展字段'
+      }
+    },
+    associate: ({ messageTrace }) => {
+      messageTrace.belongsTo(options.getMessageModel(), {
+        foreignKey: 'messageId',
+        as: 'message'
+      });
+    },
+    options: {
+      comment: 'MQ消息轨迹表',
+      indexes: [{ fields: ['trace_id', 'created_at'] }, { fields: ['message_id'] }, { fields: ['topic'] }]
+    }
+  };
+};

package/libs/models/message.js

@@ -0,0 +1,83 @@
+const { MESSAGE_STATUS } = require('../constants');
+
+module.exports = ({ DataTypes, options }) => {
+  return {
+    model: {
+      topic: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        comment: '消息主题'
+      },
+      payload: {
+        type: DataTypes.JSONB,
+        allowNull: false,
+        comment: '消息内容'
+      },
+      status: {
+        type: DataTypes.STRING,
+        allowNull: false,
+        defaultValue: MESSAGE_STATUS.PENDING,
+        comment: `消息状态: ${Object.values(MESSAGE_STATUS).join('/')}`
+      },
+      retryCount: {
+        type: DataTypes.INTEGER,
+        allowNull: false,
+        defaultValue: 0,
+        comment: '已重试次数'
+      },
+      maxRetries: {
+        type: DataTypes.INTEGER,
+        allowNull: false,
+        defaultValue: 3,
+        comment: '最大重试次数'
+      },
+      priority: {
+        type: DataTypes.INTEGER,
+        allowNull: false,
+        defaultValue: 0,
+        comment: '优先级, 数值越大优先级越高'
+      },
+      executeAt: {
+        type: DataTypes.DATE,
+        comment: '定时执行时间'
+      },
+      nextRetryAt: {
+        type: DataTypes.DATE,
+        comment: '下次重试时间'
+      },
+      consumerId: {
+        type: DataTypes.STRING,
+        comment: '当前消费者标识'
+      },
+      lockedAt: {
+        type: DataTypes.DATE,
+        comment: '锁定时间'
+      },
+      traceId: {
+        type: DataTypes.STRING,
+        comment: '追踪ID'
+      },
+      options: {
+        type: DataTypes.JSONB,
+        comment: '扩展字段'
+      }
+    },
+    associate: ({ message, messageTrace }) => {
+      message.hasMany(messageTrace, {
+        foreignKey: 'messageId',
+        as: 'traces',
+        onDelete: 'CASCADE'
+      });
+    },
+    options: {
+      comment: 'MQ消息表',
+      indexes: [
+        { fields: ['topic', 'status', 'priority', 'created_at'] },
+        { fields: ['topic', 'status', 'execute_at', 'next_retry_at', 'priority', 'created_at'] },
+        { fields: ['trace_id'] },
+        { fields: ['consumer_id'] },
+        { fields: ['status', 'next_retry_at'] }
+      ]
+    }
+  };
+};