@okf/ootils 1.5.5 → 1.5.6

package/dist/node.d.mts

@@ -516,6 +516,42 @@ declare class WorkerManager {
     }[];
 }
 
+declare class ProducerManager {
+    static instance: null;
+    static initialize(producerClasses?: any[]): any;
+    static addJob(producerClassName: any, jobName: any, jobData: any, opts?: {}): Promise<any>;
+    static addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    static getStatus(): Promise<any>;
+    static shutdown(): Promise<void>;
+    constructor(producerClasses?: any[]);
+    producerClasses: any[] | undefined;
+    activeProducers: Map<any, any> | undefined;
+    /**
+     * Initialize all producers as singletons
+     */
+    initializeAllProducers(): any[];
+    /**
+     * Get a specific producer instance
+     */
+    getProducer(producerClassName: any): any;
+    /**
+     * Add a job using a specific producer
+     */
+    addJob(producerClassName: any, jobData: any, opts?: {}): Promise<any>;
+    /**
+     * Add bulk jobs using a specific producer
+     */
+    addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    /**
+     * Get status of all producers
+     */
+    getStatus(): Promise<any[]>;
+    /**
+     * Gracefully shutdown all producers
+     */
+    shutdown(): Promise<void>;
+}
+
 declare class BaseProducer {
     constructor(config: any);
     config: any;
@@ -593,4 +629,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, ProducerManager, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.d.ts

@@ -516,6 +516,42 @@ declare class WorkerManager {
     }[];
 }
 
+declare class ProducerManager {
+    static instance: null;
+    static initialize(producerClasses?: any[]): any;
+    static addJob(producerClassName: any, jobName: any, jobData: any, opts?: {}): Promise<any>;
+    static addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    static getStatus(): Promise<any>;
+    static shutdown(): Promise<void>;
+    constructor(producerClasses?: any[]);
+    producerClasses: any[] | undefined;
+    activeProducers: Map<any, any> | undefined;
+    /**
+     * Initialize all producers as singletons
+     */
+    initializeAllProducers(): any[];
+    /**
+     * Get a specific producer instance
+     */
+    getProducer(producerClassName: any): any;
+    /**
+     * Add a job using a specific producer
+     */
+    addJob(producerClassName: any, jobData: any, opts?: {}): Promise<any>;
+    /**
+     * Add bulk jobs using a specific producer
+     */
+    addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    /**
+     * Get status of all producers
+     */
+    getStatus(): Promise<any[]>;
+    /**
+     * Gracefully shutdown all producers
+     */
+    shutdown(): Promise<void>;
+}
+
 declare class BaseProducer {
     constructor(config: any);
     config: any;
@@ -593,4 +629,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, ProducerManager, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.js

@@ -779,6 +779,145 @@ var require_WorkerManager = __commonJS({
   }
 });
 
+// src/bullmq/ProducerManager.js
+var require_ProducerManager = __commonJS({
+  "src/bullmq/ProducerManager.js"(exports2, module2) {
+    "use strict";
+    var _ProducerManager = class _ProducerManager {
+      constructor(producerClasses = []) {
+        if (_ProducerManager.instance) {
+          return _ProducerManager.instance;
+        }
+        this.producerClasses = producerClasses;
+        this.activeProducers = /* @__PURE__ */ new Map();
+        _ProducerManager.instance = this;
+      }
+      // Static methods that delegate to singleton instance
+      static initialize(producerClasses = []) {
+        if (!this.instance) {
+          this.instance = new _ProducerManager(producerClasses);
+        }
+        return this.instance.initializeAllProducers();
+      }
+      static async addJob(producerClassName, jobName, jobData, opts = {}) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addJob(producerClassName, jobName, jobData, opts);
+      }
+      static async addBulkJobs(producerClassName, jobs) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addBulkJobs(producerClassName, jobs);
+      }
+      static async getStatus() {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.getStatus();
+      }
+      static async shutdown() {
+        if (this.instance) {
+          await this.instance.shutdown();
+          this.instance = null;
+        }
+      }
+      /**
+       * Initialize all producers as singletons
+       */
+      initializeAllProducers() {
+        if (this.producerClasses.length === 0) {
+          console.log("No producers provided to initialize");
+          return [];
+        }
+        console.log("\u{1F680} Initializing all producers...");
+        this.producerClasses.forEach((ProducerClass) => {
+          try {
+            const producerInstance = new ProducerClass();
+            this.activeProducers.set(ProducerClass.name, {
+              class: ProducerClass,
+              instance: producerInstance
+            });
+            console.log(`\u2705 Initialized ${ProducerClass.name}`);
+          } catch (error) {
+            console.error(`\u274C Failed to initialize ${ProducerClass.name}:`, error);
+          }
+        });
+        console.log(`\u{1F389} Successfully initialized ${this.activeProducers.size} producers`);
+        return Array.from(this.activeProducers.values());
+      }
+      /**
+       * Get a specific producer instance
+       */
+      getProducer(producerClassName) {
+        const producer = this.activeProducers.get(producerClassName);
+        if (!producer) {
+          throw new Error(`Producer ${producerClassName} not found. Make sure it's initialized.`);
+        }
+        return producer.instance;
+      }
+      /**
+       * Add a job using a specific producer
+       */
+      async addJob(producerClassName, jobData, opts = {}) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addJobWithName(jobData, opts);
+      }
+      /**
+       * Add bulk jobs using a specific producer
+       */
+      async addBulkJobs(producerClassName, jobs) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addBulkJobs(jobs);
+      }
+      /**
+       * Get status of all producers
+       */
+      async getStatus() {
+        const statusPromises = Array.from(this.activeProducers.entries()).map(
+          async ([name, { instance }]) => {
+            try {
+              const status = await instance.getStatus();
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: !!instance.queue,
+                ...status
+              };
+            } catch (error) {
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: false,
+                error: error.message
+              };
+            }
+          }
+        );
+        return await Promise.all(statusPromises);
+      }
+      /**
+       * Gracefully shutdown all producers
+       */
+      async shutdown() {
+        console.log("\u{1F6D1} Stopping all producers...");
+        const stopPromises = Array.from(this.activeProducers.entries()).map(
+          ([name, { instance }]) => instance.stop().catch(
+            (err) => console.error(`\u274C Error stopping ${name}:`, err)
+          )
+        );
+        await Promise.all(stopPromises);
+        this.activeProducers.clear();
+        console.log("\u2705 All producers stopped");
+      }
+    };
+    __publicField(_ProducerManager, "instance", null);
+    var ProducerManager2 = _ProducerManager;
+    module2.exports = { ProducerManager: ProducerManager2 };
+  }
+});
+
 // src/bullmq/BaseProducer.js
 var require_BaseProducer = __commonJS({
   "src/bullmq/BaseProducer.js"(exports2, module2) {
@@ -861,7 +1000,8 @@ var require_BaseProducer = __commonJS({
       async stop() {
         if (this.queue) {
           await this.queue.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
+          await this.queue.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} queue closed & disconnected`);
         }
       }
     };
@@ -921,7 +1061,8 @@ var require_BaseWorker = __commonJS({
       async stop() {
         if (this.worker) {
           await this.worker.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
+          await this.worker.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} stopped & disconnected`);
         }
       }
     };
@@ -956,7 +1097,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           concurrency: 1,
           // Process jobs one at a time to avoid race conditions
           limiter: {
-            max: 10,
+            max: 5,
             // Max 10 jobs per...
             duration: 6e4
             // ...60 seconds (rate limiting)
@@ -1028,7 +1169,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         workerConfig: {
           concurrency: 1,
           limiter: {
-            max: 5,
+            max: 15,
             // Max 50 jobs per...
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
@@ -1074,8 +1215,8 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         workerConfig: {
           concurrency: 1,
           limiter: {
-            max: 5,
-            // Max 5 jobs per...
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
           }
@@ -1097,8 +1238,8 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         workerConfig: {
           concurrency: 5,
           limiter: {
-            max: 5,
-            // Max 5 jobs per...
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
           }
@@ -1153,6 +1294,7 @@ __export(node_exports, {
   GET_GLOBAL_BULLMQ_CONFIG: () => import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG,
   MongoConnector: () => import_MongoConnector2.MongoConnector,
   PlatformConfigsSchema: () => PlatformConfigs_default,
+  ProducerManager: () => import_ProducerManager.ProducerManager,
   TplSchema: () => Tpl_default,
   WorkerManager: () => import_WorkerManager.WorkerManager,
   connectToRedis: () => connectToRedis,
@@ -1594,6 +1736,7 @@ var getAIConfigs = async ({
 // src/node.ts
 init_models();
 var import_WorkerManager = __toESM(require_WorkerManager());
+var import_ProducerManager = __toESM(require_ProducerManager());
 var import_BaseProducer = __toESM(require_BaseProducer());
 var import_BaseWorker = __toESM(require_BaseWorker());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
@@ -1607,6 +1750,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   GET_GLOBAL_BULLMQ_CONFIG,
   MongoConnector,
   PlatformConfigsSchema,
+  ProducerManager,
   TplSchema,
   WorkerManager,
   connectToRedis,

package/dist/node.mjs

@@ -784,6 +784,145 @@ var require_WorkerManager = __commonJS({
   }
 });
 
+// src/bullmq/ProducerManager.js
+var require_ProducerManager = __commonJS({
+  "src/bullmq/ProducerManager.js"(exports, module) {
+    "use strict";
+    var _ProducerManager = class _ProducerManager {
+      constructor(producerClasses = []) {
+        if (_ProducerManager.instance) {
+          return _ProducerManager.instance;
+        }
+        this.producerClasses = producerClasses;
+        this.activeProducers = /* @__PURE__ */ new Map();
+        _ProducerManager.instance = this;
+      }
+      // Static methods that delegate to singleton instance
+      static initialize(producerClasses = []) {
+        if (!this.instance) {
+          this.instance = new _ProducerManager(producerClasses);
+        }
+        return this.instance.initializeAllProducers();
+      }
+      static async addJob(producerClassName, jobName, jobData, opts = {}) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addJob(producerClassName, jobName, jobData, opts);
+      }
+      static async addBulkJobs(producerClassName, jobs) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addBulkJobs(producerClassName, jobs);
+      }
+      static async getStatus() {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.getStatus();
+      }
+      static async shutdown() {
+        if (this.instance) {
+          await this.instance.shutdown();
+          this.instance = null;
+        }
+      }
+      /**
+       * Initialize all producers as singletons
+       */
+      initializeAllProducers() {
+        if (this.producerClasses.length === 0) {
+          console.log("No producers provided to initialize");
+          return [];
+        }
+        console.log("\u{1F680} Initializing all producers...");
+        this.producerClasses.forEach((ProducerClass) => {
+          try {
+            const producerInstance = new ProducerClass();
+            this.activeProducers.set(ProducerClass.name, {
+              class: ProducerClass,
+              instance: producerInstance
+            });
+            console.log(`\u2705 Initialized ${ProducerClass.name}`);
+          } catch (error) {
+            console.error(`\u274C Failed to initialize ${ProducerClass.name}:`, error);
+          }
+        });
+        console.log(`\u{1F389} Successfully initialized ${this.activeProducers.size} producers`);
+        return Array.from(this.activeProducers.values());
+      }
+      /**
+       * Get a specific producer instance
+       */
+      getProducer(producerClassName) {
+        const producer = this.activeProducers.get(producerClassName);
+        if (!producer) {
+          throw new Error(`Producer ${producerClassName} not found. Make sure it's initialized.`);
+        }
+        return producer.instance;
+      }
+      /**
+       * Add a job using a specific producer
+       */
+      async addJob(producerClassName, jobData, opts = {}) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addJobWithName(jobData, opts);
+      }
+      /**
+       * Add bulk jobs using a specific producer
+       */
+      async addBulkJobs(producerClassName, jobs) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addBulkJobs(jobs);
+      }
+      /**
+       * Get status of all producers
+       */
+      async getStatus() {
+        const statusPromises = Array.from(this.activeProducers.entries()).map(
+          async ([name, { instance }]) => {
+            try {
+              const status = await instance.getStatus();
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: !!instance.queue,
+                ...status
+              };
+            } catch (error) {
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: false,
+                error: error.message
+              };
+            }
+          }
+        );
+        return await Promise.all(statusPromises);
+      }
+      /**
+       * Gracefully shutdown all producers
+       */
+      async shutdown() {
+        console.log("\u{1F6D1} Stopping all producers...");
+        const stopPromises = Array.from(this.activeProducers.entries()).map(
+          ([name, { instance }]) => instance.stop().catch(
+            (err) => console.error(`\u274C Error stopping ${name}:`, err)
+          )
+        );
+        await Promise.all(stopPromises);
+        this.activeProducers.clear();
+        console.log("\u2705 All producers stopped");
+      }
+    };
+    __publicField(_ProducerManager, "instance", null);
+    var ProducerManager2 = _ProducerManager;
+    module.exports = { ProducerManager: ProducerManager2 };
+  }
+});
+
 // src/bullmq/BaseProducer.js
 var require_BaseProducer = __commonJS({
   "src/bullmq/BaseProducer.js"(exports, module) {
@@ -866,7 +1005,8 @@ var require_BaseProducer = __commonJS({
       async stop() {
         if (this.queue) {
           await this.queue.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
+          await this.queue.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} queue closed & disconnected`);
         }
       }
     };
@@ -926,7 +1066,8 @@ var require_BaseWorker = __commonJS({
       async stop() {
         if (this.worker) {
           await this.worker.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
+          await this.worker.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} stopped & disconnected`);
         }
       }
     };
@@ -961,7 +1102,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           concurrency: 1,
           // Process jobs one at a time to avoid race conditions
           limiter: {
-            max: 10,
+            max: 5,
             // Max 10 jobs per...
             duration: 6e4
             // ...60 seconds (rate limiting)
@@ -1033,7 +1174,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         workerConfig: {
           concurrency: 1,
           limiter: {
-            max: 5,
+            max: 15,
             // Max 50 jobs per...
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
@@ -1079,8 +1220,8 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         workerConfig: {
           concurrency: 1,
           limiter: {
-            max: 5,
-            // Max 5 jobs per...
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
           }
@@ -1102,8 +1243,8 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         workerConfig: {
           concurrency: 5,
           limiter: {
-            max: 5,
-            // Max 5 jobs per...
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
           }
@@ -1564,6 +1705,7 @@ var getAIConfigs = async ({
 // src/node.ts
 init_models();
 var import_WorkerManager = __toESM(require_WorkerManager());
+var import_ProducerManager = __toESM(require_ProducerManager());
 var import_BaseProducer = __toESM(require_BaseProducer());
 var import_BaseWorker = __toESM(require_BaseWorker());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
@@ -1572,6 +1714,7 @@ var export_BaseWorker = import_BaseWorker.BaseWorker;
 var export_ElasticSearchConnector = import_ElasticSearchConnector.ElasticSearchConnector;
 var export_GET_GLOBAL_BULLMQ_CONFIG = import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG;
 var export_MongoConnector = import_MongoConnector2.MongoConnector;
+var export_ProducerManager = import_ProducerManager.ProducerManager;
 var export_WorkerManager = import_WorkerManager.WorkerManager;
 var export_getAIChatModelByTenant = import_getModelByTenant.getAIChatModelByTenant;
 var export_getAnnotationsModelByTenant = import_getModelByTenant.getAnnotationsModelByTenant;
@@ -1587,6 +1730,7 @@ export {
   export_GET_GLOBAL_BULLMQ_CONFIG as GET_GLOBAL_BULLMQ_CONFIG,
   export_MongoConnector as MongoConnector,
   PlatformConfigs_default as PlatformConfigsSchema,
+  export_ProducerManager as ProducerManager,
   Tpl_default as TplSchema,
   export_WorkerManager as WorkerManager,
   connectToRedis,

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.5.5",
+  "version": "1.5.6",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",