@nocobase/server 1.8.0-beta.12 → 1.8.0-beta.13

package/lib/application.d.ts

@@ -35,6 +35,7 @@ import AesEncryptor from './aes-encryptor';
 import { AuditManager } from './audit-manager';
 import { Environment } from './environment';
 import { ServiceContainer } from './service-container';
+import { EventQueue, EventQueueOptions } from './event-queue';
 export type PluginType = string | typeof Plugin;
 export type PluginConfiguration = PluginType | [PluginType, any];
 export interface ResourceManagerOptions {
@@ -55,6 +56,7 @@ export interface AppTelemetryOptions extends TelemetryOptions {
     enabled?: boolean;
 }
 export interface ApplicationOptions {
+    instanceId?: string;
     database?: IDatabaseOptions | Database;
     cacheManager?: CacheManagerOptions;
     /**
@@ -82,6 +84,7 @@ export interface ApplicationOptions {
     authManager?: AuthManagerOptions;
     auditManager?: AuditManager;
     lockManager?: LockManagerOptions;
+    eventQueue?: EventQueueOptions;
     /**
      * @internal
      */
@@ -128,6 +131,7 @@ export type MaintainingCommandStatus = {
 };
 export declare class Application<StateT = DefaultState, ContextT = DefaultContext> extends Koa implements AsyncEmitter {
     options: ApplicationOptions;
+    readonly instanceId: string;
     /**
      * @internal
      */
@@ -174,6 +178,7 @@ export declare class Application<StateT = DefaultState, ContextT = DefaultContex
     private _actionCommand;
     container: ServiceContainer;
     lockManager: LockManager;
+    eventQueue: EventQueue;
     constructor(options: ApplicationOptions);
     private static staticCommands;
     static addCommand(callback: (app: Application) => void): void;

package/lib/application.js

@@ -56,6 +56,7 @@ var import_glob = __toESM(require("glob"));
 var import_koa = __toESM(require("koa"));
 var import_koa_compose = __toESM(require("koa-compose"));
 var import_lodash = __toESM(require("lodash"));
+var import_nanoid = require("nanoid");
 var import_path = __toESM(require("path"));
 var import_semver = __toESM(require("semver"));
 var import_acl = require("./acl");
@@ -81,10 +82,12 @@ var import_aes_encryptor = __toESM(require("./aes-encryptor"));
 var import_audit_manager = require("./audit-manager");
 var import_environment = require("./environment");
 var import_service_container = require("./service-container");
+var import_event_queue = require("./event-queue");
 const _Application = class _Application extends import_koa.default {
   constructor(options) {
     super();
     this.options = options;
+    this.instanceId = options.instanceId || (0, import_nanoid.nanoid)();
     this.context.reqId = (0, import_crypto.randomUUID)();
     this.rawOptions = this.name == "main" ? import_lodash.default.cloneDeep(options) : {};
     this.init();
@@ -92,6 +95,7 @@ const _Application = class _Application extends import_koa.default {
       this._appSupervisor.addApp(this);
     }
   }
+  instanceId;
   /**
    * @internal
    */
@@ -131,6 +135,7 @@ const _Application = class _Application extends import_koa.default {
   _actionCommand;
   container = new import_service_container.ServiceContainer();
   lockManager;
+  eventQueue;
   static addCommand(callback) {
     this.staticCommands.push(callback);
   }
@@ -855,6 +860,7 @@ const _Application = class _Application extends import_koa.default {
     this._i18n = (0, import_helper.createI18n)(options);
     this.pubSubManager = (0, import_pub_sub_manager.createPubSubManager)(this, options.pubSubManager);
     this.syncMessageManager = new import_sync_message_manager.SyncMessageManager(this, options.syncMessageManager);
+    this.eventQueue = new import_event_queue.EventQueue(this, options.eventQueue);
     this.lockManager = new import_lock_manager.LockManager({
       defaultAdapter: process.env.LOCK_ADAPTER_DEFAULT,
       ...options.lockManager

package/lib/event-queue.d.ts

@@ -0,0 +1,89 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import Application from './application';
+export type QueueCallbackOptions = {
+    id?: string;
+    retried?: number;
+    signal?: AbortSignal;
+};
+export type QueueCallback = (message: any, options: QueueCallbackOptions) => Promise<void> | void;
+export type QueueEventOptions = {
+    interval?: number;
+    concurrency?: number;
+    idle(): boolean;
+    process: QueueCallback;
+};
+export type QueueMessageOptions = {
+    timeout?: number;
+    maxRetries?: number;
+    retried?: number;
+    timestamp?: number;
+};
+export interface IEventQueueAdapter {
+    isConnected(): boolean;
+    connect(): Promise<void> | void;
+    close(): Promise<void> | void;
+    subscribe(channel: string, event: QueueEventOptions): void;
+    unsubscribe(channel: string): void;
+    publish(channel: string, message: any, options: QueueMessageOptions): Promise<void> | void;
+}
+export interface EventQueueOptions {
+    channelPrefix?: string;
+}
+export declare const QUEUE_DEFAULT_INTERVAL = 1000;
+export declare const QUEUE_DEFAULT_CONCURRENCY = 1;
+export declare const QUEUE_DEFAULT_ACK_TIMEOUT = 15000;
+export declare class MemoryEventQueueAdapter implements IEventQueueAdapter {
+    private options;
+    private connected;
+    private emitter;
+    private reading;
+    private events;
+    protected queues: Map<string, {
+        id: string;
+        content: any;
+        options?: QueueMessageOptions;
+    }[]>;
+    get processing(): Promise<void[]>;
+    get storagePath(): string;
+    listen: (channel: string) => Promise<void>;
+    constructor(options: {
+        appName: string;
+    });
+    isConnected(): boolean;
+    private loadFromStorage;
+    private saveToStorage;
+    connect(): Promise<void>;
+    close(): Promise<void>;
+    subscribe(channel: string, options: QueueEventOptions): void;
+    unsubscribe(channel: string): void;
+    publish(channel: string, content: any, options?: QueueMessageOptions): void;
+    read(channel: string): Promise<void>;
+    process(channel: any, { id, message }: {
+        id: any;
+        message: any;
+    }): Promise<void>;
+}
+export declare class EventQueue {
+    protected app: Application;
+    protected options: EventQueueOptions;
+    protected adapter: IEventQueueAdapter;
+    protected events: Map<string, QueueEventOptions>;
+    get channelPrefix(): string;
+    constructor(app: Application, options?: EventQueueOptions);
+    getFullChannel(channel: string): string;
+    setAdapter<A extends IEventQueueAdapter>(adapter: A): void;
+    isConnected(): boolean;
+    connect(): Promise<void>;
+    close(): Promise<void>;
+    subscribe(channel: string, options: QueueEventOptions): void;
+    unsubscribe(channel: string): void;
+    publish(channel: string, message: any, options?: QueueMessageOptions): Promise<void>;
+}
+export default EventQueue;

package/lib/event-queue.js

@@ -0,0 +1,328 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __name = (target, value) => __defProp(target, "name", { value, configurable: true });
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var event_queue_exports = {};
+__export(event_queue_exports, {
+  EventQueue: () => EventQueue,
+  MemoryEventQueueAdapter: () => MemoryEventQueueAdapter,
+  QUEUE_DEFAULT_ACK_TIMEOUT: () => QUEUE_DEFAULT_ACK_TIMEOUT,
+  QUEUE_DEFAULT_CONCURRENCY: () => QUEUE_DEFAULT_CONCURRENCY,
+  QUEUE_DEFAULT_INTERVAL: () => QUEUE_DEFAULT_INTERVAL,
+  default: () => event_queue_default
+});
+module.exports = __toCommonJS(event_queue_exports);
+var import_crypto = require("crypto");
+var import_events = require("events");
+var import_path = __toESM(require("path"));
+var import_promises = __toESM(require("fs/promises"));
+const QUEUE_DEFAULT_INTERVAL = 1e3;
+const QUEUE_DEFAULT_CONCURRENCY = 1;
+const QUEUE_DEFAULT_ACK_TIMEOUT = 15e3;
+const _MemoryEventQueueAdapter = class _MemoryEventQueueAdapter {
+  constructor(options) {
+    this.options = options;
+    this.emitter.setMaxListeners(0);
+  }
+  connected = false;
+  emitter = new import_events.EventEmitter();
+  reading = /* @__PURE__ */ new Map();
+  events = /* @__PURE__ */ new Map();
+  queues = /* @__PURE__ */ new Map();
+  get processing() {
+    const processing = Array.from(this.reading.values());
+    if (processing.length > 0) {
+      return Promise.all(processing);
+    }
+    return null;
+  }
+  get storagePath() {
+    return import_path.default.resolve(process.cwd(), "storage", "apps", this.options.appName, "event-queue.json");
+  }
+  listen = /* @__PURE__ */ __name(async (channel) => {
+    if (!this.connected) {
+      return;
+    }
+    if (this.reading.has(channel)) {
+      console.debug(`memory queue (${channel}) is already reading, waiting last reading to end...`);
+      await this.reading.get(channel);
+    }
+    const event = this.events.get(channel);
+    if (!event) {
+      console.warn(`memory queue (${channel}) not found, skipping...`);
+      return;
+    }
+    if (!event.idle()) {
+      console.debug(`memory queue (${channel}) is not idle, skipping...`);
+      return;
+    }
+    const reading = this.read(channel);
+    this.reading.set(channel, reading);
+    await reading;
+  }, "listen");
+  isConnected() {
+    return this.connected;
+  }
+  async loadFromStorage() {
+    let queues = {};
+    let exists = false;
+    try {
+      await import_promises.default.stat(this.storagePath);
+      exists = true;
+    } catch (ex) {
+      console.info(`memory queue storage file not found, skip`);
+    }
+    if (exists) {
+      try {
+        const queueJson = await import_promises.default.readFile(this.storagePath);
+        queues = JSON.parse(queueJson.toString());
+        console.debug("memory queue loaded from storage", queues);
+        await import_promises.default.unlink(this.storagePath);
+      } catch (ex) {
+        console.error("failed to load queue from storage", ex);
+      }
+    }
+    this.queues = new Map(Object.entries(queues));
+  }
+  async saveToStorage() {
+    const queues = Array.from(this.queues.entries()).reduce((acc, [channel, queue]) => {
+      if (queue == null ? void 0 : queue.length) {
+        acc[channel] = queue;
+      }
+      return acc;
+    }, {});
+    if (Object.keys(queues).length) {
+      await import_promises.default.mkdir(import_path.default.dirname(this.storagePath), { recursive: true });
+      await import_promises.default.writeFile(this.storagePath, JSON.stringify(queues));
+      console.debug("memory queue saved to storage", queues);
+    } else {
+      console.debug("memory queue empty, no need to save to storage");
+    }
+  }
+  async connect() {
+    if (this.connected) {
+      return;
+    }
+    await this.loadFromStorage();
+    this.connected = true;
+    setImmediate(() => {
+      for (const channel of this.queues.keys()) {
+        const queue = this.queues.get(channel);
+        if (!(queue == null ? void 0 : queue.length)) {
+          continue;
+        }
+        this.emitter.emit(channel, channel);
+      }
+    });
+  }
+  async close() {
+    this.connected = false;
+    if (this.processing) {
+      console.info("memory queue waiting for processing job...");
+      await this.processing;
+      console.info("memory queue job cleaned");
+    }
+    console.log("memory queue gracefully shutting down...");
+    await this.saveToStorage();
+  }
+  subscribe(channel, options) {
+    if (this.events.has(channel)) {
+      return;
+    }
+    this.events.set(channel, options);
+    if (!this.queues.has(channel)) {
+      this.queues.set(channel, []);
+    }
+    this.emitter.on(channel, this.listen);
+  }
+  unsubscribe(channel) {
+    if (!this.events.has(channel)) {
+      return;
+    }
+    this.events.delete(channel);
+  }
+  publish(channel, content, options = { timestamp: Date.now() }) {
+    const event = this.events.get(channel);
+    if (!event) {
+      return;
+    }
+    if (!this.queues.get(channel)) {
+      this.queues.set(channel, []);
+    }
+    const queue = this.queues.get(channel);
+    queue.push({ id: (0, import_crypto.randomUUID)(), content, options });
+    console.debug(`memory queue (${channel}) published message`, content);
+    setImmediate(() => {
+      this.emitter.emit(channel, channel);
+    });
+  }
+  async read(channel) {
+    const event = this.events.get(channel);
+    if (!event) {
+      this.reading.delete(channel);
+      return;
+    }
+    const queue = this.queues.get(channel);
+    while (queue == null ? void 0 : queue.length) {
+      const messages = queue.slice(0, event.concurrency || QUEUE_DEFAULT_CONCURRENCY);
+      console.debug(`memory queue (${channel}) read ${messages.length} messages`, messages);
+      queue.splice(0, messages.length);
+      const batch = messages.map(({ id, ...message }) => this.process(channel, { id, message }));
+      await Promise.all(batch);
+    }
+    this.reading.delete(channel);
+  }
+  async process(channel, { id, message }) {
+    const event = this.events.get(channel);
+    const { content, options: { timeout = QUEUE_DEFAULT_ACK_TIMEOUT, maxRetries = 0, retried = 0 } = {} } = message;
+    try {
+      console.debug(`memory queue (${channel}) processing message (${id})...`, content);
+      await event.process(content, {
+        id,
+        retried,
+        signal: AbortSignal.timeout(timeout)
+      });
+      console.debug(`memory queue (${channel}) consumed message (${id})`);
+    } catch (ex) {
+      if (maxRetries > 0 && retried < maxRetries) {
+        const currentRetry = retried + 1;
+        console.warn(
+          `memory queue (${channel}) consum message (${id}) failed, retrying (${currentRetry} / ${maxRetries})...`,
+          ex
+        );
+        setImmediate(() => {
+          this.publish(channel, content, { timeout, maxRetries, retried: currentRetry, timestamp: Date.now() });
+        });
+      } else {
+        console.error(ex);
+      }
+    }
+  }
+};
+__name(_MemoryEventQueueAdapter, "MemoryEventQueueAdapter");
+let MemoryEventQueueAdapter = _MemoryEventQueueAdapter;
+const _EventQueue = class _EventQueue {
+  constructor(app, options = {}) {
+    this.app = app;
+    this.options = options;
+    this.events = /* @__PURE__ */ new Map();
+    this.setAdapter(new MemoryEventQueueAdapter({ appName: this.app.name }));
+    app.on("afterStart", async () => {
+      await this.connect();
+    });
+    app.on("beforeStop", async () => {
+      app.logger.info("[queue] gracefully shuting down...");
+      await this.close();
+    });
+  }
+  adapter;
+  events = /* @__PURE__ */ new Map();
+  get channelPrefix() {
+    var _a;
+    return (_a = this.options) == null ? void 0 : _a.channelPrefix;
+  }
+  getFullChannel(channel) {
+    return [this.app.name, this.channelPrefix, channel].filter(Boolean).join(".");
+  }
+  setAdapter(adapter) {
+    this.adapter = adapter;
+  }
+  isConnected() {
+    if (!this.adapter) {
+      return false;
+    }
+    return this.adapter.isConnected();
+  }
+  async connect() {
+    if (!this.adapter) {
+      throw new Error("no adapter set, cannot connect");
+    }
+    await this.adapter.connect();
+    for (const [channel, event] of this.events.entries()) {
+      this.adapter.subscribe(this.getFullChannel(channel), event);
+    }
+  }
+  async close() {
+    if (!this.adapter) {
+      return;
+    }
+    await this.adapter.close();
+    for (const channel of this.events.keys()) {
+      this.adapter.unsubscribe(this.getFullChannel(channel));
+    }
+  }
+  subscribe(channel, options) {
+    if (this.events.has(channel)) {
+      this.app.logger.warn(`event queue already subscribed on channel "${channel}", new subscription will be ignored`);
+      return;
+    }
+    this.events.set(channel, options);
+    if (this.isConnected()) {
+      this.adapter.subscribe(this.getFullChannel(channel), options);
+    }
+  }
+  unsubscribe(channel) {
+    if (!this.events.has(channel)) {
+      return;
+    }
+    this.events.delete(channel);
+    if (this.isConnected()) {
+      this.adapter.unsubscribe(this.getFullChannel(channel));
+    }
+  }
+  async publish(channel, message, options = {}) {
+    if (!this.adapter) {
+      throw new Error("no adapter set, cannot publish");
+    }
+    if (!this.isConnected()) {
+      throw new Error("event queue not connected, cannot publish");
+    }
+    const c = this.getFullChannel(channel);
+    this.app.logger.debug("event queue publishing:", { channel: c, message });
+    await this.adapter.publish(c, message, { timeout: QUEUE_DEFAULT_ACK_TIMEOUT, ...options, timestamp: Date.now() });
+  }
+};
+__name(_EventQueue, "EventQueue");
+let EventQueue = _EventQueue;
+var event_queue_default = EventQueue;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  EventQueue,
+  MemoryEventQueueAdapter,
+  QUEUE_DEFAULT_ACK_TIMEOUT,
+  QUEUE_DEFAULT_CONCURRENCY,
+  QUEUE_DEFAULT_INTERVAL
+});

package/lib/index.d.ts

@@ -17,6 +17,7 @@ export * from './migration';
 export * from './plugin';
 export * from './plugin-manager';
 export * from './pub-sub-manager';
+export * from './event-queue';
 export declare const OFFICIAL_PLUGIN_PREFIX = "@nocobase/plugin-";
 export { appendToBuiltInPlugins, findAllPlugins, findBuiltInPlugins, findLocalPlugins, packageNameTrim, } from './plugin-manager/findPackageNames';
 export { runPluginStaticImports } from './run-plugin-static-imports';

package/lib/index.js

@@ -59,6 +59,7 @@ __reExport(src_exports, require("./migration"), module.exports);
 __reExport(src_exports, require("./plugin"), module.exports);
 __reExport(src_exports, require("./plugin-manager"), module.exports);
 __reExport(src_exports, require("./pub-sub-manager"), module.exports);
+__reExport(src_exports, require("./event-queue"), module.exports);
 var import_findPackageNames = require("./plugin-manager/findPackageNames");
 var import_run_plugin_static_imports = require("./run-plugin-static-imports");
 const OFFICIAL_PLUGIN_PREFIX = "@nocobase/plugin-";
@@ -80,5 +81,6 @@ const OFFICIAL_PLUGIN_PREFIX = "@nocobase/plugin-";
   ...require("./migration"),
   ...require("./plugin"),
   ...require("./plugin-manager"),
-  ...require("./pub-sub-manager")
+  ...require("./pub-sub-manager"),
+  ...require("./event-queue")
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/server",
-  "version": "1.8.0-beta.12",
+  "version": "1.8.0-beta.13",
   "main": "lib/index.js",
   "types": "./lib/index.d.ts",
   "license": "AGPL-3.0",
@@ -10,19 +10,19 @@
     "@koa/cors": "^5.0.0",
     "@koa/multer": "^3.1.0",
     "@koa/router": "^13.1.0",
-    "@nocobase/acl": "1.8.0-beta.12",
-    "@nocobase/actions": "1.8.0-beta.12",
-    "@nocobase/auth": "1.8.0-beta.12",
-    "@nocobase/cache": "1.8.0-beta.12",
-    "@nocobase/data-source-manager": "1.8.0-beta.12",
-    "@nocobase/database": "1.8.0-beta.12",
-    "@nocobase/evaluators": "1.8.0-beta.12",
-    "@nocobase/lock-manager": "1.8.0-beta.12",
-    "@nocobase/logger": "1.8.0-beta.12",
-    "@nocobase/resourcer": "1.8.0-beta.12",
-    "@nocobase/sdk": "1.8.0-beta.12",
-    "@nocobase/telemetry": "1.8.0-beta.12",
-    "@nocobase/utils": "1.8.0-beta.12",
+    "@nocobase/acl": "1.8.0-beta.13",
+    "@nocobase/actions": "1.8.0-beta.13",
+    "@nocobase/auth": "1.8.0-beta.13",
+    "@nocobase/cache": "1.8.0-beta.13",
+    "@nocobase/data-source-manager": "1.8.0-beta.13",
+    "@nocobase/database": "1.8.0-beta.13",
+    "@nocobase/evaluators": "1.8.0-beta.13",
+    "@nocobase/lock-manager": "1.8.0-beta.13",
+    "@nocobase/logger": "1.8.0-beta.13",
+    "@nocobase/resourcer": "1.8.0-beta.13",
+    "@nocobase/sdk": "1.8.0-beta.13",
+    "@nocobase/telemetry": "1.8.0-beta.13",
+    "@nocobase/utils": "1.8.0-beta.13",
     "@types/decompress": "4.2.7",
     "@types/ini": "^1.3.31",
     "@types/koa-send": "^4.1.3",
@@ -57,5 +57,5 @@
     "@types/serve-handler": "^6.1.1",
     "@types/ws": "^8.5.5"
   },
-  "gitHead": "b244f9b23ce35bd75baf7b357c6d53d0fbddf3be"
+  "gitHead": "9ccd306ff667a54317ba07a000fc5aa00685d5e6"
 }