@nocobase/server 1.9.0-beta.4 → 1.9.0-beta.6

package/lib/application.js

@@ -279,6 +279,9 @@ const _Application = class _Application extends import_koa.default {
     if (!WORKER_MODE) {
       return true;
     }
+    if (WORKER_MODE === "-") {
+      return false;
+    }
     const topics = WORKER_MODE.trim().split(",");
     if (key) {
       if (WORKER_MODE === "*") {

package/lib/audit-manager/index.js

@@ -252,6 +252,9 @@ const _AuditManager = class _AuditManager {
   async output(ctx, reqId, metadata) {
     var _a;
     try {
+      if (!ctx.action) {
+        return;
+      }
       const { resourceName, actionName } = ctx.action;
       const action = this.getAction(actionName, resourceName);
       if (!action) {

package/lib/event-queue.d.ts

@@ -15,7 +15,7 @@ export type QueueCallbackOptions = {
     retried?: number;
     signal?: AbortSignal;
 };
-export type QueueCallback = (message: any, options: QueueCallbackOptions) => Promise<void> | void;
+export type QueueCallback = (message: any, options: QueueCallbackOptions) => Promise<void>;
 export type QueueEventOptions = {
     /**
      * @experimental
@@ -53,9 +53,9 @@ export declare class MemoryEventQueueAdapter implements IEventQueueAdapter {
         content: any;
         options?: QueueMessageOptions;
     }[]>;
-    get processing(): Promise<void[]>;
+    get processing(): Promise<Promise<void>[][]>;
     private get storagePath();
-    listen: (channel: string) => Promise<void>;
+    listen: (channel: string) => void;
     constructor(options: {
         appName: string;
     });
@@ -69,7 +69,7 @@ export declare class MemoryEventQueueAdapter implements IEventQueueAdapter {
     unsubscribe(channel: string): void;
     publish(channel: string, content: any, options?: QueueMessageOptions): void;
     consume(channel: string, once?: boolean): Promise<void>;
-    read(channel: string): Promise<void>;
+    read(channel: string, n: number): Promise<void>[];
     process(channel: string, { id, message }: {
         id: any;
         message: any;

package/lib/event-queue.js

@@ -73,26 +73,37 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
   get storagePath() {
     return import_path.default.resolve(process.cwd(), "storage", "apps", this.options.appName, "event-queue.json");
   }
-  listen = /* @__PURE__ */ __name(async (channel) => {
+  listen = /* @__PURE__ */ __name((channel) => {
     if (!this.connected) {
       return;
     }
-    if (this.reading.has(channel)) {
-      console.debug(`memory queue (${channel}) is already reading, waiting last reading to end...`);
-      await this.reading.get(channel);
-    }
     const event = this.events.get(channel);
     if (!event) {
       console.warn(`memory queue (${channel}) not found, skipping...`);
       return;
     }
     if (!event.idle()) {
-      console.debug(`memory queue (${channel}) is not idle, skipping...`);
       return;
     }
-    const reading = this.read(channel);
+    const reading = this.reading.get(channel) || [];
+    const count = (event.concurrency || QUEUE_DEFAULT_CONCURRENCY) - reading.length;
+    if (count <= 0) {
+      console.debug(
+        `memory queue (${channel}) is already reading as max concurrency (${reading.length}), waiting last reading to end...`
+      );
+      return;
+    }
+    console.debug(`reading more from queue (${channel}), count: ${count}`);
+    this.read(channel, count).forEach((promise) => {
+      reading.push(promise);
+      promise.finally(() => {
+        const index = reading.indexOf(promise);
+        if (index > -1) {
+          reading.splice(index, 1);
+        }
+      });
+    });
     this.reading.set(channel, reading);
-    await reading;
   }, "listen");
   isConnected() {
     return this.connected;
@@ -149,6 +160,9 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
     });
   }
   async close() {
+    if (!this.connected) {
+      return;
+    }
     this.connected = false;
     if (this.processing) {
       console.info("memory queue waiting for processing job...");
@@ -181,6 +195,7 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
   publish(channel, content, options = { timestamp: Date.now() }) {
     const event = this.events.get(channel);
     if (!event) {
+      console.debug(`memory queue (${channel}) not subscribed, skip`);
       return;
     }
     if (!this.queues.get(channel)) {
@@ -200,7 +215,7 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
       const interval = event.interval || QUEUE_DEFAULT_INTERVAL;
       const queue = this.queues.get(channel);
       if (event.idle() && (queue == null ? void 0 : queue.length)) {
-        await this.listen(channel);
+        this.listen(channel);
       }
       if (once) {
         break;
@@ -208,47 +223,41 @@ const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
       await (0, import_utils.sleep)(interval);
     }
   }
-  async read(channel) {
-    const event = this.events.get(channel);
-    if (!event) {
-      this.reading.delete(channel);
-      return;
-    }
+  read(channel, n) {
     const queue = this.queues.get(channel);
-    if (queue == null ? void 0 : queue.length) {
-      const messages = queue.slice(0, event.concurrency || QUEUE_DEFAULT_CONCURRENCY);
-      console.debug(`memory queue (${channel}) read ${messages.length} messages`, messages);
-      queue.splice(0, messages.length);
-      const batch = messages.map(({ id, ...message }) => this.process(channel, { id, message }));
-      await Promise.all(batch);
-    }
-    this.reading.delete(channel);
+    if (!(queue == null ? void 0 : queue.length)) {
+      return [];
+    }
+    const messages = queue.slice(0, n);
+    console.debug(`memory queue (${channel}) read ${messages.length} messages`, messages);
+    queue.splice(0, messages.length);
+    const batch = messages.map(({ id, ...message }) => this.process(channel, { id, message }));
+    return batch;
   }
   async process(channel, { id, message }) {
     const event = this.events.get(channel);
     const { content, options: { timeout = QUEUE_DEFAULT_ACK_TIMEOUT, maxRetries = 0, retried = 0 } = {} } = message;
-    try {
-      console.debug(`memory queue (${channel}) processing message (${id})...`, content);
-      await event.process(content, {
-        id,
-        retried,
-        signal: AbortSignal.timeout(timeout)
-      });
+    console.debug(`memory queue (${channel}) processing message (${id})...`, content);
+    return (async () => event.process(content, {
+      id,
+      retried,
+      signal: AbortSignal.timeout(timeout)
+    }))().then(() => {
       console.debug(`memory queue (${channel}) consumed message (${id})`);
-    } catch (ex) {
+    }).catch((ex) => {
       if (maxRetries > 0 && retried < maxRetries) {
         const currentRetry = retried + 1;
         console.warn(
           `memory queue (${channel}) consum message (${id}) failed, retrying (${currentRetry} / ${maxRetries})...`,
           ex
         );
-        setImmediate(() => {
+        setTimeout(() => {
           this.publish(channel, content, { timeout, maxRetries, retried: currentRetry, timestamp: Date.now() });
-        });
+        }, 500);
       } else {
         console.error(ex);
       }
-    }
+    });
   }
 };
 __name(_MemoryEventQueueAdapter, "MemoryEventQueueAdapter");
@@ -257,14 +266,16 @@ const _EventQueue = class _EventQueue {
   constructor(app, options = {}) {
     this.app = app;
     this.options = options;
-    this.setAdapter(new MemoryEventQueueAdapter({ appName: this.app.name }));
-    app.on("afterStart", async () => {
-      await this.connect();
-    });
-    app.on("beforeStop", async () => {
-      app.logger.info("[queue] gracefully shutting down...");
-      await this.close();
-    });
+    if (app.serving()) {
+      this.setAdapter(new MemoryEventQueueAdapter({ appName: this.app.name }));
+      app.on("afterStart", async () => {
+        await this.connect();
+      });
+      app.on("beforeStop", async () => {
+        app.logger.info("[queue] gracefully shutting down...");
+        await this.close();
+      });
+    }
   }
   adapter;
   events = /* @__PURE__ */ new Map();
@@ -288,7 +299,12 @@ const _EventQueue = class _EventQueue {
     if (!this.adapter) {
       throw new Error("no adapter set, cannot connect");
     }
+    if (!this.app.serving()) {
+      this.app.logger.warn("app is not serving, will not connect to event queue");
+      return;
+    }
     await this.adapter.connect();
+    this.app.logger.debug(`connected to adapter, using memory? ${this.adapter instanceof MemoryEventQueueAdapter}`);
     for (const [channel, event] of this.events.entries()) {
       this.adapter.subscribe(this.getFullChannel(channel), event);
     }
@@ -329,7 +345,7 @@ const _EventQueue = class _EventQueue {
       throw new Error("event queue not connected, cannot publish");
     }
     const c = this.getFullChannel(channel);
-    this.app.logger.debug("event queue publishing:", { channel: c, message });
+    this.app.logger.debug(`event queue publishing to channel(${c})`, { message });
     await this.adapter.publish(c, message, {
       timeout: QUEUE_DEFAULT_ACK_TIMEOUT,
       ...options,

package/lib/gateway/ws-server.js

@@ -172,6 +172,20 @@ const _WSServer = class _WSServer extends import_events.default {
         message
       );
     });
+    app.on("ws:sendToUser", ({ userId, message }) => {
+      this.sendToAppUser(app.name, userId, message);
+      app.logger.trace(`[broadcasting message] ws:sendToUser for user ${userId}`, { message });
+      app.pubSubManager.publish(
+        "ws:sendToUser",
+        {
+          userId,
+          message
+        },
+        {
+          skipSelf: true
+        }
+      );
+    });
     app.on("ws:sendToClient", ({ clientId, message }) => {
       this.sendToClient(clientId, message);
     });
@@ -184,6 +198,12 @@ const _WSServer = class _WSServer extends import_events.default {
     app.on("ws:authorized", ({ clientId, userId }) => {
       this.sendToClient(clientId, { type: "authorized" });
     });
+    app.on("afterLoad", () => {
+      app.pubSubManager.subscribe("ws:sendToUser", ({ userId, message }) => {
+        app.logger.debug(`[receive broadcasting message] ws:sendToUser for user ${userId}`, { message });
+        this.sendToAppUser(app.name, userId, message);
+      });
+    });
   }
   addNewConnection(ws, request) {
     const id = (0, import_nanoid.nanoid)();

package/lib/helper.d.ts

@@ -16,3 +16,4 @@ export declare const createAppProxy: (app: Application) => Application<import(".
 export declare const getCommandFullName: (command: Command) => string;
 export declare const tsxRerunning: () => Promise<void>;
 export declare const enablePerfHooks: (app: Application) => void;
+export declare function getBodyLimit(): string;

package/lib/helper.js

@@ -41,6 +41,7 @@ __export(helper_exports, {
   createI18n: () => createI18n,
   createResourcer: () => createResourcer,
   enablePerfHooks: () => enablePerfHooks,
+  getBodyLimit: () => getBodyLimit,
   getCommandFullName: () => getCommandFullName,
   registerMiddlewares: () => registerMiddlewares,
   tsxRerunning: () => tsxRerunning
@@ -99,7 +100,7 @@ function registerMiddlewares(app, options) {
     }
   );
   if (options.bodyParser !== false) {
-    const bodyLimit = "10mb";
+    const bodyLimit = getBodyLimit();
     app.use(
       (0, import_koa_bodyparser.default)({
         jsonLimit: bodyLimit,
@@ -188,12 +189,17 @@ const enablePerfHooks = /* @__PURE__ */ __name((app) => {
   });
   app.acl.allow("perf", "*", "public");
 }, "enablePerfHooks");
+function getBodyLimit() {
+  return process.env.REQUEST_BODY_LIMIT || "10mb";
+}
+__name(getBodyLimit, "getBodyLimit");
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   createAppProxy,
   createI18n,
   createResourcer,
   enablePerfHooks,
+  getBodyLimit,
   getCommandFullName,
   registerMiddlewares,
   tsxRerunning

package/lib/middlewares/data-template.js

@@ -46,7 +46,7 @@ const traverseHasMany = /* @__PURE__ */ __name((arr, { collection, exclude = [],
   if (!arr) {
     return arr;
   }
-  return arr.map((item) => traverseJSON(item, { collection, exclude, include }));
+  return arr.map((item) => traverseJSON(item, { collection, exclude, include, isHasManyField: true }));
 }, "traverseHasMany");
 const traverseBelongsToMany = /* @__PURE__ */ __name((arr, { collection, exclude = [], through }) => {
   if (!arr) {
@@ -107,7 +107,10 @@ const traverseJSON = /* @__PURE__ */ __name((data, options) => {
     if (field.options.isForeignKey) {
       continue;
     }
-    if (["sort", "password", "sequence"].includes(field.type)) {
+    if (!options.isHasManyField && ["sort"].includes(field.type)) {
+      continue;
+    }
+    if (["password", "sequence"].includes(field.type)) {
       continue;
     }
     if (field.type === "hasOne") {

package/lib/pub-sub-manager/pub-sub-manager.d.ts

@@ -11,11 +11,12 @@ import { HandlerManager } from './handler-manager';
 import { PubSubCallback, type IPubSubAdapter, type PubSubManagerOptions, type PubSubManagerPublishOptions, type PubSubManagerSubscribeOptions } from './types';
 export declare const createPubSubManager: (app: Application, options: PubSubManagerOptions) => PubSubManager;
 export declare class PubSubManager {
+    protected app: Application;
     protected options: PubSubManagerOptions;
     protected publisherId: string;
     protected adapter: IPubSubAdapter;
     protected handlerManager: HandlerManager;
-    constructor(options?: PubSubManagerOptions);
+    constructor(app: Application, options?: PubSubManagerOptions);
     get channelPrefix(): string;
     setAdapter(adapter: IPubSubAdapter): void;
     isConnected(): Promise<boolean>;
@@ -23,5 +24,5 @@ export declare class PubSubManager {
     close(): Promise<any>;
     subscribe(channel: string, callback: PubSubCallback, options?: PubSubManagerSubscribeOptions): Promise<void>;
     unsubscribe(channel: string, callback: PubSubCallback): Promise<any>;
-    publish(channel: string, message: any, options?: PubSubManagerPublishOptions): Promise<any>;
+    publish(channel: string, message: any, options?: PubSubManagerPublishOptions): Promise<void>;
 }

package/lib/pub-sub-manager/pub-sub-manager.js

@@ -34,17 +34,20 @@ module.exports = __toCommonJS(pub_sub_manager_exports);
 var import_utils = require("@nocobase/utils");
 var import_handler_manager = require("./handler-manager");
 const createPubSubManager = /* @__PURE__ */ __name((app, options) => {
-  const pubSubManager = new PubSubManager(options);
-  app.on("afterStart", async () => {
-    await pubSubManager.connect();
-  });
-  app.on("afterStop", async () => {
-    await pubSubManager.close();
-  });
+  const pubSubManager = new PubSubManager(app, options);
+  if (app.serving()) {
+    app.on("afterStart", async () => {
+      await pubSubManager.connect();
+    });
+    app.on("afterStop", async () => {
+      await pubSubManager.close();
+    });
+  }
   return pubSubManager;
 }, "createPubSubManager");
 const _PubSubManager = class _PubSubManager {
-  constructor(options = {}) {
+  constructor(app, options = {}) {
+    this.app = app;
     this.options = options;
     this.publisherId = (0, import_utils.uid)();
     this.handlerManager = new import_handler_manager.HandlerManager(this.publisherId);
@@ -69,8 +72,13 @@ const _PubSubManager = class _PubSubManager {
     if (!this.adapter) {
       return;
     }
+    if (!this.app.serving()) {
+      this.app.logger.warn("app is not serving, will not connect to event queue");
+      return;
+    }
     await this.adapter.connect();
     await this.handlerManager.each(async (channel, headler) => {
+      this.app.logger.debug(`[PubSubManager] subscribe ${channel} added before connected`);
       await this.adapter.subscribe(`${this.channelPrefix}${channel}`, headler);
     });
   }
@@ -84,6 +92,7 @@ const _PubSubManager = class _PubSubManager {
     await this.unsubscribe(channel, callback);
     const handler = this.handlerManager.set(channel, callback, options);
     if (await this.isConnected()) {
+      this.app.logger.debug(`[PubSubManager] subscribe ${channel} added after connected`);
       await this.adapter.subscribe(`${this.channelPrefix}${channel}`, handler);
     }
   }
@@ -97,6 +106,9 @@ const _PubSubManager = class _PubSubManager {
   async publish(channel, message, options) {
     var _a;
     if (!((_a = this.adapter) == null ? void 0 : _a.isConnected())) {
+      this.app.logger.warn(
+        `[PubSubManager] adapter is not exist or not connected, cannot publish message to channel ${channel}`
+      );
       return;
     }
     const wrappedMessage = JSON.stringify({
@@ -104,7 +116,8 @@ const _PubSubManager = class _PubSubManager {
       ...options,
       message
     });
-    return this.adapter.publish(`${this.channelPrefix}${channel}`, wrappedMessage);
+    await this.adapter.publish(`${this.channelPrefix}${channel}`, wrappedMessage);
+    this.app.logger.trace(`[PubSubManager] published message to channel ${channel}`);
   }
 };
 __name(_PubSubManager, "PubSubManager");

package/lib/sync-message-manager.d.ts

@@ -16,7 +16,7 @@ export declare class SyncMessageManager {
     protected pubSubManager: PubSubManager;
     constructor(app: Application, options?: any);
     get debounce(): any;
-    publish(channel: string, message: any, options?: PubSubManagerPublishOptions & Transactionable): Promise<any>;
+    publish(channel: string, message: any, options?: PubSubManagerPublishOptions & Transactionable): Promise<unknown>;
     subscribe(channel: string, callback: PubSubCallback): Promise<void>;
     unsubscribe(channel: string, callback: PubSubCallback): Promise<any>;
     sync(): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/server",
-  "version": "1.9.0-beta.4",
+  "version": "1.9.0-beta.6",
   "main": "lib/index.js",
   "types": "./lib/index.d.ts",
   "license": "AGPL-3.0",
@@ -10,19 +10,19 @@
     "@koa/cors": "^5.0.0",
     "@koa/multer": "^3.1.0",
     "@koa/router": "^13.1.0",
-    "@nocobase/acl": "1.9.0-beta.4",
-    "@nocobase/actions": "1.9.0-beta.4",
-    "@nocobase/auth": "1.9.0-beta.4",
-    "@nocobase/cache": "1.9.0-beta.4",
-    "@nocobase/data-source-manager": "1.9.0-beta.4",
-    "@nocobase/database": "1.9.0-beta.4",
-    "@nocobase/evaluators": "1.9.0-beta.4",
-    "@nocobase/lock-manager": "1.9.0-beta.4",
-    "@nocobase/logger": "1.9.0-beta.4",
-    "@nocobase/resourcer": "1.9.0-beta.4",
-    "@nocobase/sdk": "1.9.0-beta.4",
-    "@nocobase/telemetry": "1.9.0-beta.4",
-    "@nocobase/utils": "1.9.0-beta.4",
+    "@nocobase/acl": "1.9.0-beta.6",
+    "@nocobase/actions": "1.9.0-beta.6",
+    "@nocobase/auth": "1.9.0-beta.6",
+    "@nocobase/cache": "1.9.0-beta.6",
+    "@nocobase/data-source-manager": "1.9.0-beta.6",
+    "@nocobase/database": "1.9.0-beta.6",
+    "@nocobase/evaluators": "1.9.0-beta.6",
+    "@nocobase/lock-manager": "1.9.0-beta.6",
+    "@nocobase/logger": "1.9.0-beta.6",
+    "@nocobase/resourcer": "1.9.0-beta.6",
+    "@nocobase/sdk": "1.9.0-beta.6",
+    "@nocobase/telemetry": "1.9.0-beta.6",
+    "@nocobase/utils": "1.9.0-beta.6",
     "@types/decompress": "4.2.7",
     "@types/ini": "^1.3.31",
     "@types/koa-send": "^4.1.3",
@@ -57,5 +57,5 @@
     "@types/serve-handler": "^6.1.1",
     "@types/ws": "^8.5.5"
   },
-  "gitHead": "0276aa12d87f82c73d62a9712134fe363e2c772a"
+  "gitHead": "ada5e359a91135cb9baf605c04c5c380f067a046"
 }