@abtnode/blocklet-services 1.17.4-beta-20251201-225048-b1682a09 → 1.17.4-beta-20251202-122551-267b614d

package/api/routes/user.js

@@ -661,6 +661,8 @@ const notificationConfigSetSchema = Joi.object({
       Joi.object({
         url: Joi.string().uri(),
         type: Joi.string().valid('slack', 'api'),
+        enabled: Joi.boolean().optional().default(true),
+        consecutiveFailures: Joi.number().optional().default(0),
       })
     )
     .optional(),

package/api/services/notification/queue.js

@@ -17,9 +17,11 @@ const {
   NOTIFICATION_SEND_STATUS,
   NOTIFICATION_SEND_FAILED_REASON,
 } = require('@abtnode/constant');
+const sleep = require('@abtnode/util/lib/sleep');
 const { nanoid } = require('@blocklet/meta/lib/util');
 const get = require('lodash/get');
 const uniqBy = require('lodash/uniqBy');
+const { getQueueConcurrencyByMem } = require('@abtnode/core/lib/util');
 const { getBlockletInfo } = require('../../cache');
 const { updateNotificationSendStatus } = require('../../socket/channel/did');
 const eventHub =
@@ -29,6 +31,8 @@ const logger = require('../../libs/logger')('blocklet-services:notification-queu
 
 const emailSchema = Joi.string().email().required();
 
+const concurrency = getQueueConcurrencyByMem();
+
 /**
  *
  * 校验是否是有效的 passthrough 消息
@@ -58,10 +62,14 @@ const createNotificationQueue = (name, options, handler) => {
       maxRetries: 3,
       retryDelay: 10 * 1000,
       maxTimeout: 60 * 1000, // throw timeout error after 1 minutes
+      concurrency,
       ...(options ?? {}),
     },
     onJob: async (job) => {
       await handler(job);
+      if (options.delay) {
+        await sleep(options.delay * 1000);
+      }
     },
   });
 };
@@ -73,7 +81,15 @@ const init = ({ node, notificationService }) => {
   const getServerWebhooks = async () => {
     const webhookList = (await webhookState.list()) ?? [];
     return webhookList.flatMap((item) =>
-      item.params.filter((param) => param.name === 'url').map((param) => ({ type: item.type, url: param.value }))
+      item.params
+        .filter((param) => param.name === 'url' && param.value)
+        .map((param) => ({
+          id: item.id,
+          type: item.type,
+          url: param.value,
+          enabled: param.enabled ?? true,
+          consecutiveFailures: param.consecutiveFailures || 0,
+        }))
     );
   };
 
@@ -85,6 +101,7 @@ const init = ({ node, notificationService }) => {
     {
       maxRetries: 1,
       retryDelay: 0,
+      enableScheduledJob: true,
     },
     async (job) => {
       try {
@@ -104,6 +121,7 @@ const init = ({ node, notificationService }) => {
         });
       } catch (error) {
         logger.error('Failed to send to app', { notificationId: job.notification.id, receiver: job.receiver, error });
+        throw error;
       }
     }
   );
@@ -111,28 +129,42 @@ const init = ({ node, notificationService }) => {
   /**
    * Push Kit 推送队列
    */
-  const pushKitPushQueue = createNotificationQueue('send-notification-push', {}, async (job) => {
-    try {
-      const { notification, receiver, sender } = job;
+  const pushKitPushQueue = createNotificationQueue(
+    'send-notification-push',
+    {
+      enableScheduledJob: true,
+    },
+    async (job) => {
+      try {
+        const { notification, receiver, sender } = job;
 
-      if (!receiver) {
-        throw new Error('Invalid receiver');
-      }
+        if (!receiver) {
+          throw new Error('Invalid receiver');
+        }
 
-      await notificationService.sendToPush.exec({
-        sender,
-        receiver,
-        notification,
-        pushOnly: job.pushOnly,
-      });
-    } catch (error) {
-      if (error.logLevel === 'debug') {
-        logger.debug('Failed to send to push', { notificationId: job.notification.id, receiver: job.receiver, error });
-      } else {
-        logger.error('Failed to send to push', { notificationId: job.notification.id, receiver: job.receiver, error });
+        await notificationService.sendToPush.exec({
+          sender,
+          receiver,
+          notification,
+          pushOnly: job.pushOnly,
+        });
+      } catch (error) {
+        if (error.logLevel === 'debug') {
+          logger.debug('Failed to send to push', {
+            notificationId: job.notification.id,
+            receiver: job.receiver,
+            error,
+          });
+        } else {
+          logger.error('Failed to send to push', {
+            notificationId: job.notification.id,
+            receiver: job.receiver,
+            error,
+          });
+        }
       }
     }
-  });
+  );
 
   /**
    * email 推送队列
@@ -184,7 +216,6 @@ const init = ({ node, notificationService }) => {
             : null;
 
         logger.info('Start send to email', {
-          email,
           notificationId: job.notificationId,
         });
         await notificationService.sendToMail.exec({
@@ -206,7 +237,6 @@ const init = ({ node, notificationService }) => {
           options,
         });
         logger.info('End send to email', {
-          email,
           notificationId: job.notificationId,
         });
       } catch (error) {
@@ -225,6 +255,7 @@ const init = ({ node, notificationService }) => {
             error,
           });
         }
+        throw error;
       }
     }
   );
@@ -285,6 +316,7 @@ const init = ({ node, notificationService }) => {
           receivers: job.input.receivers.join(','),
           error,
         });
+        throw error;
       }
     }
   );
@@ -381,7 +413,7 @@ const init = ({ node, notificationService }) => {
             options,
           },
         },
-        delay: 5,
+        delay: 8,
       });
     } else {
       // eslint-disable-next-line no-lonely-if
@@ -403,8 +435,40 @@ const init = ({ node, notificationService }) => {
     }
   };
 
+  /**
+   * 批量更新 webhook 发送失败状态（并行处理）
+   * @param {Array} webhooks - webhook 列表
+   * @param {string} failedReason - 失败原因
+   * @param {object} params - 公共参数
+   * @returns {Promise<void>}
+   */
+  const batchUpdateWebhookFailedStatus = (webhooks, failedReason, params) => {
+    const updatePromises = webhooks.map((webhook) => {
+      const { url, type } = webhook;
+      const webhookParams = {
+        [url]: {
+          type,
+          sendAt: new Date(),
+          status: NOTIFICATION_SEND_STATUS.FAILED,
+          failedReason,
+        },
+      };
+
+      return updateNotificationSendStatus({
+        ...params,
+        channel: NOTIFICATION_SEND_CHANNEL.WEBHOOK,
+        status: NOTIFICATION_SEND_STATUS.FAILED,
+        webhookParams,
+      }).catch((err) => {
+        logger.debug('update webhook failed status error', { err, url });
+      });
+    });
+
+    return Promise.all(updatePromises);
+  };
+
   const insertToWebhookPushQueue = async (props, nodeInfo, isResend) => {
-    const { channels, notification, sender, userInfo, teamDid } = props;
+    const { channels, notification, sender, userInfo, teamDid, pushOnly } = props;
 
     const receiverDid = userInfo.did;
 
@@ -412,6 +476,7 @@ const init = ({ node, notificationService }) => {
       node,
       teamDid: teamDid ?? nodeInfo.did,
       notificationId: notification.id,
+      receivers: [receiverDid],
     };
 
     const isServer = teamDid === nodeInfo.did;
@@ -423,20 +488,34 @@ const init = ({ node, notificationService }) => {
 
     const webhookList = uniqBy(webhooks, 'url');
 
-    // 如果类型不一致，则不进行推送
+    // 分类 webhook：有效、禁用、类型不匹配
     const validWebhookList = [];
+    const invalidWebhooks = []; // { webhook, reason }
+
     for (const webhook of webhookList) {
-      const { url, type } = webhook;
-      if (type === 'api' && isUrl(url)) {
+      const { url, type, enabled = true } = webhook;
+
+      // 检查是否禁用
+      if (!enabled) {
+        invalidWebhooks.push({ webhook, reason: 'Current webhook is disabled' });
+      } else if (type === 'api' && isUrl(url)) {
+        // api 类型且 URL 有效
         validWebhookList.push(webhook);
       } else if (type === 'slack' && isUrl(url) && isSlackWebhookUrl(url)) {
+        // slack 类型且 URL 有效
         validWebhookList.push(webhook);
+      } else {
+        // 类型不支持或 URL 无效
+        invalidWebhooks.push({ webhook, reason: 'Webhook type is not supported or URL is invalid' });
       }
     }
 
-    if (validWebhookList.length > 0 && channels.includes(NOTIFICATION_SEND_CHANNEL.WEBHOOK)) {
-      for (const webhook of webhookList) {
-        const { url, type } = webhook;
+    const isWebhookChannelEnabled = channels.includes(NOTIFICATION_SEND_CHANNEL.WEBHOOK);
+
+    // 如果有有效的 webhook 且 channel 启用，推送到队列
+    if (validWebhookList.length > 0 && isWebhookChannelEnabled) {
+      for (const webhook of validWebhookList) {
+        const { url } = webhook;
         logger.info('Insert to webhook push queue', {
           notificationId: notification.id,
           receiver: receiverDid,
@@ -452,45 +531,50 @@ const init = ({ node, notificationService }) => {
               receivers: [receiverDid],
               isResend,
               pushOnly: props.pushOnly && !isResend,
-              webhook: {
-                url,
-                type,
-              },
+              webhook,
             },
           },
           delay: 5,
         });
       }
-    } else {
-      // eslint-disable-next-line no-lonely-if
-      if (webhookList.length > 0 && !isResend) {
-        const webhookParams = {};
+    }
 
-        let reason = '';
-        if (validWebhookList.length > 0) {
-          reason = NOTIFICATION_SEND_FAILED_REASON.CHANNEL_DISABLED;
-        } else {
-          reason = 'Webhook type is not supported';
+    // 如果不是重发，需要更新失败状态（并行处理）
+    if (!isResend && !pushOnly) {
+      const updatePromises = [];
+
+      // 处理无效的 webhook（按失败原因分组并行更新）
+      if (invalidWebhooks.length > 0) {
+        // 按失败原因分组
+        const groupedByReason = {};
+        for (const { webhook, reason } of invalidWebhooks) {
+          if (!groupedByReason[reason]) {
+            groupedByReason[reason] = [];
+          }
+          groupedByReason[reason].push(webhook);
         }
 
-        for (const { url, type } of webhookList) {
-          webhookParams[url] = {
-            type,
-            sendAt: new Date(),
-            status: NOTIFICATION_SEND_STATUS.FAILED,
-            failedReason: reason,
-          };
-
-          // eslint-disable-next-line no-await-in-loop
-          await updateNotificationSendStatus({
-            ...commonParams,
-            receivers: [receiverDid],
-            channel: NOTIFICATION_SEND_CHANNEL.WEBHOOK,
-            status: NOTIFICATION_SEND_STATUS.FAILED,
-            webhookParams: { [url]: webhookParams[url] },
-          });
+        // 每组并行更新
+        for (const [reason, webhookGroup] of Object.entries(groupedByReason)) {
+          updatePromises.push(batchUpdateWebhookFailedStatus(webhookGroup, reason, commonParams));
         }
       }
+
+      // 处理有效但 channel 禁用的 webhook
+      if (validWebhookList.length > 0 && !isWebhookChannelEnabled) {
+        updatePromises.push(
+          batchUpdateWebhookFailedStatus(
+            validWebhookList,
+            NOTIFICATION_SEND_FAILED_REASON.CHANNEL_DISABLED,
+            commonParams
+          )
+        );
+      }
+
+      // 等待所有更新完成（不阻塞主流程）
+      Promise.all(updatePromises).catch((err) => {
+        logger.error('batch update webhook failed status error', { err });
+      });
     }
   };
 
@@ -516,12 +600,15 @@ const init = ({ node, notificationService }) => {
         receiver: receiverDid,
       });
       pushKitPushQueue.push({
-        notification,
-        sender,
-        options,
-        receiver: receiverDid,
-        isResend,
-        pushOnly: props.pushOnly && !isResend,
+        job: {
+          notification,
+          sender,
+          options,
+          receiver: receiverDid,
+          isResend,
+          pushOnly: props.pushOnly && !isResend,
+        },
+        delay: 10,
       });
     } else {
       // eslint-disable-next-line no-lonely-if
@@ -557,13 +644,16 @@ const init = ({ node, notificationService }) => {
         receiver: receiverDid,
       });
       walletPushQueue.push({
-        notification,
-        sender,
-        options,
-        receiver: receiverDid,
-        isResend,
-        pushOnly: props.pushOnly && !isResend,
-        source,
+        job: {
+          notification,
+          sender,
+          options,
+          receiver: receiverDid,
+          isResend,
+          pushOnly: props.pushOnly && !isResend,
+          source,
+        },
+        delay: 5,
       });
     } else {
       // 如果是重发的消息，只需要更新推送状态，不需要更新 channel状态
@@ -628,7 +718,7 @@ const init = ({ node, notificationService }) => {
    * }
    */
   const queue = createNotificationQueue('notification-receivers', {}, async (job) => {
-    const { teamDid, channels, receiver, sender, notification, nodeInfo, userInfo, ...rest } = job;
+    const { teamDid, channels, receiver, sender, notification, nodeInfo, ...rest } = job;
 
     logger.info('notification start insert to queue', {
       teamDid,
@@ -636,6 +726,17 @@ const init = ({ node, notificationService }) => {
       receiver,
     });
     try {
+      const selection = {
+        did: 1,
+        fullName: 1,
+        email: 1,
+        extra: 1,
+      };
+      const userInfo = await node.getUser({
+        teamDid,
+        user: { did: receiver },
+        options: { enableConnectedAccount: true, selection, includePassports: false, includeConnectedAccounts: false },
+      });
       if (!userInfo) {
         throw new Error(`Invalid receiver user: ${receiver}`);
       }
@@ -654,26 +755,22 @@ const init = ({ node, notificationService }) => {
         });
         // websocket 通知
         const receiverDid = userInfo.did;
-        try {
-          websocketQueue.push({
-            input: {
-              notification: {
-                ...notification,
-                entityType: rest.entityType,
-                entityId: rest.entityId,
-                componentDid: rest.componentDid,
-                source: rest.source,
-                createdAt: rest.createdAt,
-                ...(rest.actorInfo ? { actorInfo: rest.actorInfo } : {}),
-              },
-              receiver: receiverDid,
-              teamDid,
-              isServices: rest.isServices,
+        websocketQueue.push({
+          input: {
+            notification: {
+              ...notification,
+              entityType: rest.entityType,
+              entityId: rest.entityId,
+              componentDid: rest.componentDid,
+              source: rest.source,
+              createdAt: rest.createdAt,
+              ...(rest.actorInfo ? { actorInfo: rest.actorInfo } : {}),
             },
-          });
-        } catch (error) {
-          logger.error('Failed to insert to websocket push queue', { error });
-        }
+            receiver: receiverDid,
+            teamDid,
+            isServices: rest.isServices,
+          },
+        });
       }
 
       notification.type = notification.type || 'notification';
@@ -727,7 +824,6 @@ const init = ({ node, notificationService }) => {
       logger.info('notification start insert to queue:', {
         teamDid: data?.teamDid,
         notificationId: data?.notification?.id,
-        receivers: data?.receivers?.join(','),
       });
       if (isInstanceWorker()) {
         return;
@@ -753,13 +849,7 @@ const init = ({ node, notificationService }) => {
           type: 'server',
         });
 
-      const users = await node.getNotificationReceivers({
-        teamDid,
-        userDids: receivers,
-        includeConnectedAccounts: true,
-      });
-
-      if (users.length === 0) {
+      if (receivers.length === 0) {
         throw new Error('No users found');
       }
       // 如果 notification 没有 id，则生成一个, wallet 要基于这唯一个ID进行处理
@@ -771,8 +861,7 @@ const init = ({ node, notificationService }) => {
         notification.id = `NOTIF-${nanoid()}`;
       }
 
-      users.forEach((userInfo) => {
-        const receiverDid = userInfo.did;
+      receivers.forEach((receiverDid) => {
         queue.push({
           ...rest,
           notification,
@@ -781,7 +870,6 @@ const init = ({ node, notificationService }) => {
           receiver: receiverDid,
           sender,
           nodeInfo,
-          userInfo,
         });
       });
     } catch (error) {

package/api/socket/channel/did.js

@@ -164,40 +164,38 @@ const sendToUserDid = async ({ sender, receiver: rawDid, notification, options,
 
   const isServer = !teamDid || teamDid === nodeInfo.did;
 
-  try {
-    const result = await Promise.all(
-      notifications.map((_notification) => {
-        // 如果类型不存在或者是 notification 类型是才进行保存数据库，其他类型只需要通知到用户即可
-        // eg: type = 'passthrough', 'hi', 'connect', 'feed', 默认只需要通知 wallet
-        const pushOnly = _notification.type && _notification.type.toLowerCase() !== NOTIFICATION_TYPES.NOTIFICATION;
-
-        // 通知渠道的确定有两种方式 1. 用户传入的 channels 2. 根据 notification 类型确定
-        // 如果是 hi connect, 只需要通知 wallet
-        // passthrough 类型，用于 discuss kit 的 chat channel
-        // 1. push kit: native 通知快速跳转到 chat channel
-        // 2. app: 钱包消息，可以查看到接收到消息
-
-        return node.createNotification({
-          teamDid,
-          ...(pushOnly ? { notification: _notification } : { ..._notification }),
-          receiver: rawDid,
-          componentDid: sender.componentDid,
-          // 这两个字段用于 socket 通知
-          entityType: _notification.entityType || 'blocklet',
-          entityId: _notification.entityId || sender.appDid,
-          source: isServer ? 'system' : _notification.source || 'component',
-          channels: CHANNEL_MAP[_notification.type] || channels,
-          sender,
-          options: { ...rest },
-          pushOnly,
-        });
-      })
-    );
-    return result;
-  } catch (error) {
-    logger.error('Failed to create notification', { error });
-    return undefined;
-  }
+  // 开始执行 service 消息的推送，不需要同步执行
+  Promise.all(
+    notifications.map((_notification) => {
+      // 如果类型不存在或者是 notification 类型是才进行保存数据库，其他类型只需要通知到用户即可
+      // eg: type = 'passthrough', 'hi', 'connect', 'feed', 默认只需要通知 wallet
+      const pushOnly = _notification.type && _notification.type.toLowerCase() !== NOTIFICATION_TYPES.NOTIFICATION;
+
+      // 通知渠道的确定有两种方式 1. 用户传入的 channels 2. 根据 notification 类型确定
+      // 如果是 hi connect, 只需要通知 wallet
+      // passthrough 类型，用于 discuss kit 的 chat channel
+      // 1. push kit: native 通知快速跳转到 chat channel
+      // 2. app: 钱包消息，可以查看到接收到消息
+
+      return node.createNotification({
+        teamDid,
+        ...(pushOnly ? { notification: _notification } : { ..._notification }),
+        receiver: rawDid,
+        componentDid: sender.componentDid,
+        // 这两个字段用于 socket 通知
+        entityType: _notification.entityType || 'blocklet',
+        entityId: _notification.entityId || sender.appDid,
+        source: isServer ? 'system' : _notification.source || 'component',
+        channels: CHANNEL_MAP[_notification.type] || channels,
+        sender,
+        options: { ...rest },
+        pushOnly,
+      });
+    })
+  ).catch((error) => {
+    logger.error('Failed to send notification', { error });
+  });
+  return true;
 };
 
 // server send notification to app
@@ -352,6 +350,36 @@ const onAuthenticate = async ({ channel, did, payload }) => {
   }
 };
 
+/**
+ * 安全地更新 webhook 状态，不影响主流程
+ * @param {object} params - 更新参数
+ * @param {object} params.node - node 实例
+ * @param {object} params.webhook - webhook 对象
+ * @param {string} params.teamDid - team DID
+ * @param {boolean} params.isService - 是否为服务
+ * @param {Array<string>} params.receivers - 接收者列表
+ * @param {number|undefined} params.consecutiveFailures - 连续失败次数，0=重置，undefined=自动+1
+ */
+const safeUpdateWebhookState = async ({ node, webhook, teamDid, isService, receivers, consecutiveFailures }) => {
+  try {
+    await node.updateWebHookState({
+      webhook,
+      teamDid,
+      isService,
+      userDids: receivers,
+      consecutiveFailures,
+    });
+  } catch (error) {
+    logger.debug('Failed to update webhook state', {
+      error,
+      teamDid,
+      receivers,
+      webhook: webhook.url,
+      consecutiveFailures,
+    });
+  }
+};
+
 const sendToWebhook = async ({ sender, receiver, notification, node, pushOnly }) => {
   const teamDid = sender.appDid;
   const { id, appInfo, ...rest } = notification;
@@ -371,6 +399,7 @@ const sendToWebhook = async ({ sender, receiver, notification, node, pushOnly })
 
   const senderInfo = await ensureSenderApp({ sender, node, nodeInfo });
 
+  const isService = teamDid && teamDid !== nodeInfo.did;
   // 发送消息前要添加 sender 内容。否则邮件中没有 blocklet 的信息
   const notifications = parseNotification({ id, ...rest }, senderInfo);
   const webhookSenderMap = new Map();
@@ -405,6 +434,15 @@ const sendToWebhook = async ({ sender, receiver, notification, node, pushOnly })
           webhookParams,
         });
       }
+      // 发送成功，重置失败次数
+      await safeUpdateWebhookState({
+        node,
+        webhook,
+        teamDid,
+        isService,
+        receivers,
+        consecutiveFailures: 0,
+      });
       return res;
     } catch (error) {
       // 使用 debug 级别记录详细执行信息，避免与上层业务日志重复
@@ -424,6 +462,14 @@ const sendToWebhook = async ({ sender, receiver, notification, node, pushOnly })
           webhookParams,
         });
       }
+      // 发送失败，不传入 consecutiveFailures，让底层自动 +1
+      await safeUpdateWebhookState({
+        node,
+        webhook,
+        teamDid,
+        isService,
+        receivers,
+      });
       const err = new Error('Failed to send webhook');
       err.details = error;
       throw err;

package/api/util/user-util.js

@@ -315,7 +315,12 @@ const validateWebhooks = async ({ webhooks, user }) => {
   }
 
   // 去重：webhooks 包含全量的数据，去重时会把历史数据存在的重复数据清理掉
-  const uniqueWebhooks = uniqBy(webhooks, 'url');
+  const uniqueWebhooks = uniqBy(webhooks, 'url').map((webhook) => {
+    return {
+      ...webhook,
+      consecutiveFailures: webhook.enabled ? webhook.consecutiveFailures : 0,
+    };
+  });
 
   return { value: uniqueWebhooks, error: null };
 };