@nocobase/server 1.8.7 → 1.8.8

package/lib/event-queue.d.ts

@@ -15,7 +15,7 @@ export type QueueCallbackOptions = {
     retried?: number;
     signal?: AbortSignal;
 };
-export type QueueCallback = (message: any, options: QueueCallbackOptions) => Promise<void> | void;
+export type QueueCallback = (message: any, options: QueueCallbackOptions) => Promise<void>;
 export type QueueEventOptions = {
     /**
      * @experimental
@@ -53,9 +53,9 @@ export declare class MemoryEventQueueAdapter implements IEventQueueAdapter {
         content: any;
         options?: QueueMessageOptions;
     }[]>;
-    get processing(): Promise<void[]>;
+    get processing(): Promise<Promise<void>[][]>;
     private get storagePath();
-    listen: (channel: string) => Promise<void>;
+    listen: (channel: string) => void;
     constructor(options: {
         appName: string;
     });
@@ -69,7 +69,7 @@ export declare class MemoryEventQueueAdapter implements IEventQueueAdapter {
     unsubscribe(channel: string): void;
     publish(channel: string, content: any, options?: QueueMessageOptions): void;
     consume(channel: string, once?: boolean): Promise<void>;
-    read(channel: string): Promise<void>;
+    read(channel: string, n: number): Promise<void>[];
     process(channel: string, { id, message }: {
         id: any;
         message: any;

package/lib/event-queue.js

@@ -73,26 +73,37 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
   get storagePath() {
     return import_path.default.resolve(process.cwd(), "storage", "apps", this.options.appName, "event-queue.json");
   }
-  listen = /* @__PURE__ */ __name(async (channel) => {
+  listen = /* @__PURE__ */ __name((channel) => {
     if (!this.connected) {
       return;
     }
-    if (this.reading.has(channel)) {
-      console.debug(`memory queue (${channel}) is already reading, waiting last reading to end...`);
-      await this.reading.get(channel);
-    }
     const event = this.events.get(channel);
     if (!event) {
       console.warn(`memory queue (${channel}) not found, skipping...`);
       return;
     }
     if (!event.idle()) {
-      console.debug(`memory queue (${channel}) is not idle, skipping...`);
       return;
     }
-    const reading = this.read(channel);
+    const reading = this.reading.get(channel) || [];
+    const count = (event.concurrency || QUEUE_DEFAULT_CONCURRENCY) - reading.length;
+    if (count <= 0) {
+      console.debug(
+        `memory queue (${channel}) is already reading as max concurrency (${reading.length}), waiting last reading to end...`
+      );
+      return;
+    }
+    console.debug(`reading more from queue (${channel}), count: ${count}`);
+    this.read(channel, count).forEach((promise) => {
+      reading.push(promise);
+      promise.finally(() => {
+        const index = reading.indexOf(promise);
+        if (index > -1) {
+          reading.splice(index, 1);
+        }
+      });
+    });
     this.reading.set(channel, reading);
-    await reading;
   }, "listen");
   isConnected() {
     return this.connected;
@@ -149,6 +160,9 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
     });
   }
   async close() {
+    if (!this.connected) {
+      return;
+    }
     this.connected = false;
     if (this.processing) {
       console.info("memory queue waiting for processing job...");
@@ -200,7 +214,7 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
       const interval = event.interval || QUEUE_DEFAULT_INTERVAL;
       const queue = this.queues.get(channel);
       if (event.idle() && (queue == null ? void 0 : queue.length)) {
-        await this.listen(channel);
+        this.listen(channel);
       }
       if (once) {
         break;
@@ -208,47 +222,41 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
       await (0, import_utils.sleep)(interval);
     }
   }
-  async read(channel) {
-    const event = this.events.get(channel);
-    if (!event) {
-      this.reading.delete(channel);
-      return;
-    }
+  read(channel, n) {
     const queue = this.queues.get(channel);
-    if (queue == null ? void 0 : queue.length) {
-      const messages = queue.slice(0, event.concurrency || QUEUE_DEFAULT_CONCURRENCY);
-      console.debug(`memory queue (${channel}) read ${messages.length} messages`, messages);
-      queue.splice(0, messages.length);
-      const batch = messages.map(({ id, ...message }) => this.process(channel, { id, message }));
-      await Promise.all(batch);
+    if (!(queue == null ? void 0 : queue.length)) {
+      return [];
     }
-    this.reading.delete(channel);
+    const messages = queue.slice(0, n);
+    console.debug(`memory queue (${channel}) read ${messages.length} messages`, messages);
+    queue.splice(0, messages.length);
+    const batch = messages.map(({ id, ...message }) => this.process(channel, { id, message }));
+    return batch;
   }
   async process(channel, { id, message }) {
     const event = this.events.get(channel);
     const { content, options: { timeout = QUEUE_DEFAULT_ACK_TIMEOUT, maxRetries = 0, retried = 0 } = {} } = message;
-    try {
-      console.debug(`memory queue (${channel}) processing message (${id})...`, content);
-      await event.process(content, {
-        id,
-        retried,
-        signal: AbortSignal.timeout(timeout)
-      });
+    console.debug(`memory queue (${channel}) processing message (${id})...`, content);
+    return (async () => event.process(content, {
+      id,
+      retried,
+      signal: AbortSignal.timeout(timeout)
+    }))().then(() => {
       console.debug(`memory queue (${channel}) consumed message (${id})`);
-    } catch (ex) {
+    }).catch((ex) => {
       if (maxRetries > 0 && retried < maxRetries) {
         const currentRetry = retried + 1;
         console.warn(
           `memory queue (${channel}) consum message (${id}) failed, retrying (${currentRetry} / ${maxRetries})...`,
           ex
         );
-        setImmediate(() => {
+        setTimeout(() => {
           this.publish(channel, content, { timeout, maxRetries, retried: currentRetry, timestamp: Date.now() });
-        });
+        }, 500);
       } else {
         console.error(ex);
       }
-    }
+    });
   }
 };
 __name(_MemoryEventQueueAdapter, "MemoryEventQueueAdapter");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/server",
-  "version": "1.8.7",
+  "version": "1.8.8",
   "main": "lib/index.js",
   "types": "./lib/index.d.ts",
   "license": "AGPL-3.0",
@@ -10,19 +10,19 @@
     "@koa/cors": "^5.0.0",
     "@koa/multer": "^3.1.0",
     "@koa/router": "^13.1.0",
-    "@nocobase/acl": "1.8.7",
-    "@nocobase/actions": "1.8.7",
-    "@nocobase/auth": "1.8.7",
-    "@nocobase/cache": "1.8.7",
-    "@nocobase/data-source-manager": "1.8.7",
-    "@nocobase/database": "1.8.7",
-    "@nocobase/evaluators": "1.8.7",
-    "@nocobase/lock-manager": "1.8.7",
-    "@nocobase/logger": "1.8.7",
-    "@nocobase/resourcer": "1.8.7",
-    "@nocobase/sdk": "1.8.7",
-    "@nocobase/telemetry": "1.8.7",
-    "@nocobase/utils": "1.8.7",
+    "@nocobase/acl": "1.8.8",
+    "@nocobase/actions": "1.8.8",
+    "@nocobase/auth": "1.8.8",
+    "@nocobase/cache": "1.8.8",
+    "@nocobase/data-source-manager": "1.8.8",
+    "@nocobase/database": "1.8.8",
+    "@nocobase/evaluators": "1.8.8",
+    "@nocobase/lock-manager": "1.8.8",
+    "@nocobase/logger": "1.8.8",
+    "@nocobase/resourcer": "1.8.8",
+    "@nocobase/sdk": "1.8.8",
+    "@nocobase/telemetry": "1.8.8",
+    "@nocobase/utils": "1.8.8",
     "@types/decompress": "4.2.7",
     "@types/ini": "^1.3.31",
     "@types/koa-send": "^4.1.3",
@@ -57,5 +57,5 @@
     "@types/serve-handler": "^6.1.1",
     "@types/ws": "^8.5.5"
   },
-  "gitHead": "31229ac95e4fa3edf998d2a02fc982ade14ac09e"
+  "gitHead": "33a6e5154a9b6b2ffe1a7919b91f23304e97c5c5"
 }