@okf/ootils 1.5.4 → 1.5.6

package/dist/node.d.mts

@@ -1,3 +1,4 @@
+import { Client } from '@elastic/elasticsearch';
 import mongoose, { Connection, Document, Types } from 'mongoose';
 import IORedis from 'ioredis';
 import * as bullmq from 'bullmq';
@@ -187,6 +188,26 @@ declare namespace MongoConnector {
     let instance: any;
 }
 
+declare class ElasticSearchConnector {
+    static getInstance(): any;
+    static getClient(): any;
+    constructor(options?: {});
+    client: Client | null;
+    cloudId: any;
+    apiKey: any;
+    /**
+     * Initialize Elasticsearch client
+     */
+    connect(): Client;
+    /**
+     * Close the Elasticsearch client
+     */
+    close(): Promise<void>;
+}
+declare namespace ElasticSearchConnector {
+    let instance: any;
+}
+
 /**
  * Get database connection by tenant
  * @param tenant - Tenant identifier for the database
@@ -495,6 +516,42 @@ declare class WorkerManager {
     }[];
 }
 
+declare class ProducerManager {
+    static instance: null;
+    static initialize(producerClasses?: any[]): any;
+    static addJob(producerClassName: any, jobName: any, jobData: any, opts?: {}): Promise<any>;
+    static addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    static getStatus(): Promise<any>;
+    static shutdown(): Promise<void>;
+    constructor(producerClasses?: any[]);
+    producerClasses: any[] | undefined;
+    activeProducers: Map<any, any> | undefined;
+    /**
+     * Initialize all producers as singletons
+     */
+    initializeAllProducers(): any[];
+    /**
+     * Get a specific producer instance
+     */
+    getProducer(producerClassName: any): any;
+    /**
+     * Add a job using a specific producer
+     */
+    addJob(producerClassName: any, jobData: any, opts?: {}): Promise<any>;
+    /**
+     * Add bulk jobs using a specific producer
+     */
+    addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    /**
+     * Get status of all producers
+     */
+    getStatus(): Promise<any[]>;
+    /**
+     * Gracefully shutdown all producers
+     */
+    shutdown(): Promise<void>;
+}
+
 declare class BaseProducer {
     constructor(config: any);
     config: any;
@@ -572,4 +629,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, ProducerManager, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.d.ts

@@ -1,3 +1,4 @@
+import { Client } from '@elastic/elasticsearch';
 import mongoose, { Connection, Document, Types } from 'mongoose';
 import IORedis from 'ioredis';
 import * as bullmq from 'bullmq';
@@ -187,6 +188,26 @@ declare namespace MongoConnector {
     let instance: any;
 }
 
+declare class ElasticSearchConnector {
+    static getInstance(): any;
+    static getClient(): any;
+    constructor(options?: {});
+    client: Client | null;
+    cloudId: any;
+    apiKey: any;
+    /**
+     * Initialize Elasticsearch client
+     */
+    connect(): Client;
+    /**
+     * Close the Elasticsearch client
+     */
+    close(): Promise<void>;
+}
+declare namespace ElasticSearchConnector {
+    let instance: any;
+}
+
 /**
  * Get database connection by tenant
  * @param tenant - Tenant identifier for the database
@@ -495,6 +516,42 @@ declare class WorkerManager {
     }[];
 }
 
+declare class ProducerManager {
+    static instance: null;
+    static initialize(producerClasses?: any[]): any;
+    static addJob(producerClassName: any, jobName: any, jobData: any, opts?: {}): Promise<any>;
+    static addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    static getStatus(): Promise<any>;
+    static shutdown(): Promise<void>;
+    constructor(producerClasses?: any[]);
+    producerClasses: any[] | undefined;
+    activeProducers: Map<any, any> | undefined;
+    /**
+     * Initialize all producers as singletons
+     */
+    initializeAllProducers(): any[];
+    /**
+     * Get a specific producer instance
+     */
+    getProducer(producerClassName: any): any;
+    /**
+     * Add a job using a specific producer
+     */
+    addJob(producerClassName: any, jobData: any, opts?: {}): Promise<any>;
+    /**
+     * Add bulk jobs using a specific producer
+     */
+    addBulkJobs(producerClassName: any, jobs: any): Promise<any>;
+    /**
+     * Get status of all producers
+     */
+    getStatus(): Promise<any[]>;
+    /**
+     * Gracefully shutdown all producers
+     */
+    shutdown(): Promise<void>;
+}
+
 declare class BaseProducer {
     constructor(config: any);
     config: any;
@@ -572,4 +629,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, ProducerManager, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.js

@@ -518,6 +518,71 @@ var require_MongoConnector = __commonJS({
   }
 });
 
+// src/ElasticSearchConnector.js
+var require_ElasticSearchConnector = __commonJS({
+  "src/ElasticSearchConnector.js"(exports2, module2) {
+    "use strict";
+    var { Client } = require("@elastic/elasticsearch");
+    var ElasticSearchConnector2 = class _ElasticSearchConnector {
+      constructor(options = {}) {
+        this.client = null;
+        this.cloudId = options.cloudId;
+        this.apiKey = options.apiKey;
+        _ElasticSearchConnector.instance = this;
+        if (!this.cloudId) {
+          throw new Error("Cloud ID must be provided in constructor options or ELASTIC_CLOUD_ID environment variable");
+        }
+        if (!this.apiKey) {
+          throw new Error("API Key must be provided in constructor options or ELASTIC_API_KEY environment variable");
+        }
+      }
+      /**
+       * Initialize Elasticsearch client
+       */
+      connect() {
+        this.client = new Client({
+          cloud: {
+            id: this.cloudId
+          },
+          auth: {
+            apiKey: this.apiKey
+          }
+        });
+        console.log("\u2705 Elasticsearch client initialized");
+        return this.client;
+      }
+      // Static method to get the instance
+      static getInstance() {
+        if (!_ElasticSearchConnector.instance) {
+          throw new Error("ElasticSearchConnector not initialized");
+        }
+        return _ElasticSearchConnector.instance;
+      }
+      // Static method to get the client
+      static getClient() {
+        if (!_ElasticSearchConnector.instance || !_ElasticSearchConnector.instance.client) {
+          throw new Error("ElasticSearchConnector not initialized or client not connected");
+        }
+        return _ElasticSearchConnector.instance.client;
+      }
+      /**
+       * Close the Elasticsearch client
+       */
+      async close() {
+        if (this.client) {
+          await this.client.close();
+          console.log("\u2705 Elasticsearch client closed");
+          this.client = null;
+        }
+      }
+    };
+    ElasticSearchConnector2.instance = null;
+    module2.exports = {
+      ElasticSearchConnector: ElasticSearchConnector2
+    };
+  }
+});
+
 // src/db/getDbByTenant.js
 var getDbByTenant_exports = {};
 __export(getDbByTenant_exports, {
@@ -714,6 +779,145 @@ var require_WorkerManager = __commonJS({
   }
 });
 
+// src/bullmq/ProducerManager.js
+var require_ProducerManager = __commonJS({
+  "src/bullmq/ProducerManager.js"(exports2, module2) {
+    "use strict";
+    var _ProducerManager = class _ProducerManager {
+      constructor(producerClasses = []) {
+        if (_ProducerManager.instance) {
+          return _ProducerManager.instance;
+        }
+        this.producerClasses = producerClasses;
+        this.activeProducers = /* @__PURE__ */ new Map();
+        _ProducerManager.instance = this;
+      }
+      // Static methods that delegate to singleton instance
+      static initialize(producerClasses = []) {
+        if (!this.instance) {
+          this.instance = new _ProducerManager(producerClasses);
+        }
+        return this.instance.initializeAllProducers();
+      }
+      static async addJob(producerClassName, jobName, jobData, opts = {}) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addJob(producerClassName, jobName, jobData, opts);
+      }
+      static async addBulkJobs(producerClassName, jobs) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addBulkJobs(producerClassName, jobs);
+      }
+      static async getStatus() {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.getStatus();
+      }
+      static async shutdown() {
+        if (this.instance) {
+          await this.instance.shutdown();
+          this.instance = null;
+        }
+      }
+      /**
+       * Initialize all producers as singletons
+       */
+      initializeAllProducers() {
+        if (this.producerClasses.length === 0) {
+          console.log("No producers provided to initialize");
+          return [];
+        }
+        console.log("\u{1F680} Initializing all producers...");
+        this.producerClasses.forEach((ProducerClass) => {
+          try {
+            const producerInstance = new ProducerClass();
+            this.activeProducers.set(ProducerClass.name, {
+              class: ProducerClass,
+              instance: producerInstance
+            });
+            console.log(`\u2705 Initialized ${ProducerClass.name}`);
+          } catch (error) {
+            console.error(`\u274C Failed to initialize ${ProducerClass.name}:`, error);
+          }
+        });
+        console.log(`\u{1F389} Successfully initialized ${this.activeProducers.size} producers`);
+        return Array.from(this.activeProducers.values());
+      }
+      /**
+       * Get a specific producer instance
+       */
+      getProducer(producerClassName) {
+        const producer = this.activeProducers.get(producerClassName);
+        if (!producer) {
+          throw new Error(`Producer ${producerClassName} not found. Make sure it's initialized.`);
+        }
+        return producer.instance;
+      }
+      /**
+       * Add a job using a specific producer
+       */
+      async addJob(producerClassName, jobData, opts = {}) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addJobWithName(jobData, opts);
+      }
+      /**
+       * Add bulk jobs using a specific producer
+       */
+      async addBulkJobs(producerClassName, jobs) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addBulkJobs(jobs);
+      }
+      /**
+       * Get status of all producers
+       */
+      async getStatus() {
+        const statusPromises = Array.from(this.activeProducers.entries()).map(
+          async ([name, { instance }]) => {
+            try {
+              const status = await instance.getStatus();
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: !!instance.queue,
+                ...status
+              };
+            } catch (error) {
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: false,
+                error: error.message
+              };
+            }
+          }
+        );
+        return await Promise.all(statusPromises);
+      }
+      /**
+       * Gracefully shutdown all producers
+       */
+      async shutdown() {
+        console.log("\u{1F6D1} Stopping all producers...");
+        const stopPromises = Array.from(this.activeProducers.entries()).map(
+          ([name, { instance }]) => instance.stop().catch(
+            (err) => console.error(`\u274C Error stopping ${name}:`, err)
+          )
+        );
+        await Promise.all(stopPromises);
+        this.activeProducers.clear();
+        console.log("\u2705 All producers stopped");
+      }
+    };
+    __publicField(_ProducerManager, "instance", null);
+    var ProducerManager2 = _ProducerManager;
+    module2.exports = { ProducerManager: ProducerManager2 };
+  }
+});
+
 // src/bullmq/BaseProducer.js
 var require_BaseProducer = __commonJS({
   "src/bullmq/BaseProducer.js"(exports2, module2) {
@@ -796,7 +1000,8 @@ var require_BaseProducer = __commonJS({
       async stop() {
         if (this.queue) {
           await this.queue.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
+          await this.queue.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} queue closed & disconnected`);
         }
       }
     };
@@ -856,7 +1061,8 @@ var require_BaseWorker = __commonJS({
       async stop() {
         if (this.worker) {
           await this.worker.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
+          await this.worker.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} stopped & disconnected`);
         }
       }
     };
@@ -891,7 +1097,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           concurrency: 1,
           // Process jobs one at a time to avoid race conditions
           limiter: {
-            max: 10,
+            max: 5,
             // Max 10 jobs per...
             duration: 6e4
             // ...60 seconds (rate limiting)
@@ -962,9 +1168,8 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         },
         workerConfig: {
           concurrency: 1,
-          // Process 10 jobs at once for chunk processing
           limiter: {
-            max: 5,
+            max: 15,
             // Max 50 jobs per...
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
@@ -1008,7 +1213,36 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1
+          concurrency: 1,
+          limiter: {
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
+        }
+      },
+      ANNOS_ELASTIC_SYNC_QUEUE: {
+        id: "annos-elastic-sync-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            attempts: 3,
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            removeOnComplete: 50,
+            removeOnFail: 100
+          }
+        },
+        workerConfig: {
+          concurrency: 5,
+          limiter: {
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
         }
       }
     };
@@ -1056,9 +1290,11 @@ __export(node_exports, {
   AnnotationSchema: () => Annotations_default,
   BaseProducer: () => import_BaseProducer.BaseProducer,
   BaseWorker: () => import_BaseWorker.BaseWorker,
+  ElasticSearchConnector: () => import_ElasticSearchConnector.ElasticSearchConnector,
   GET_GLOBAL_BULLMQ_CONFIG: () => import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG,
   MongoConnector: () => import_MongoConnector2.MongoConnector,
   PlatformConfigsSchema: () => PlatformConfigs_default,
+  ProducerManager: () => import_ProducerManager.ProducerManager,
   TplSchema: () => Tpl_default,
   WorkerManager: () => import_WorkerManager.WorkerManager,
   connectToRedis: () => connectToRedis,
@@ -1360,6 +1596,7 @@ var _extractBlocksFromSomeBuilders = ({
 
 // src/node.ts
 var import_MongoConnector2 = __toESM(require_MongoConnector());
+var import_ElasticSearchConnector = __toESM(require_ElasticSearchConnector());
 init_getDbByTenant();
 
 // src/db/getGlobalConfig.ts
@@ -1499,6 +1736,7 @@ var getAIConfigs = async ({
 // src/node.ts
 init_models();
 var import_WorkerManager = __toESM(require_WorkerManager());
+var import_ProducerManager = __toESM(require_ProducerManager());
 var import_BaseProducer = __toESM(require_BaseProducer());
 var import_BaseWorker = __toESM(require_BaseWorker());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
@@ -1508,9 +1746,11 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   AnnotationSchema,
   BaseProducer,
   BaseWorker,
+  ElasticSearchConnector,
   GET_GLOBAL_BULLMQ_CONFIG,
   MongoConnector,
   PlatformConfigsSchema,
+  ProducerManager,
   TplSchema,
   WorkerManager,
   connectToRedis,

package/dist/node.mjs

@@ -523,6 +523,71 @@ var require_MongoConnector = __commonJS({
   }
 });
 
+// src/ElasticSearchConnector.js
+var require_ElasticSearchConnector = __commonJS({
+  "src/ElasticSearchConnector.js"(exports, module) {
+    "use strict";
+    var { Client } = __require("@elastic/elasticsearch");
+    var ElasticSearchConnector2 = class _ElasticSearchConnector {
+      constructor(options = {}) {
+        this.client = null;
+        this.cloudId = options.cloudId;
+        this.apiKey = options.apiKey;
+        _ElasticSearchConnector.instance = this;
+        if (!this.cloudId) {
+          throw new Error("Cloud ID must be provided in constructor options or ELASTIC_CLOUD_ID environment variable");
+        }
+        if (!this.apiKey) {
+          throw new Error("API Key must be provided in constructor options or ELASTIC_API_KEY environment variable");
+        }
+      }
+      /**
+       * Initialize Elasticsearch client
+       */
+      connect() {
+        this.client = new Client({
+          cloud: {
+            id: this.cloudId
+          },
+          auth: {
+            apiKey: this.apiKey
+          }
+        });
+        console.log("\u2705 Elasticsearch client initialized");
+        return this.client;
+      }
+      // Static method to get the instance
+      static getInstance() {
+        if (!_ElasticSearchConnector.instance) {
+          throw new Error("ElasticSearchConnector not initialized");
+        }
+        return _ElasticSearchConnector.instance;
+      }
+      // Static method to get the client
+      static getClient() {
+        if (!_ElasticSearchConnector.instance || !_ElasticSearchConnector.instance.client) {
+          throw new Error("ElasticSearchConnector not initialized or client not connected");
+        }
+        return _ElasticSearchConnector.instance.client;
+      }
+      /**
+       * Close the Elasticsearch client
+       */
+      async close() {
+        if (this.client) {
+          await this.client.close();
+          console.log("\u2705 Elasticsearch client closed");
+          this.client = null;
+        }
+      }
+    };
+    ElasticSearchConnector2.instance = null;
+    module.exports = {
+      ElasticSearchConnector: ElasticSearchConnector2
+    };
+  }
+});
+
 // src/db/getDbByTenant.js
 var getDbByTenant_exports = {};
 __export(getDbByTenant_exports, {
@@ -719,6 +784,145 @@ var require_WorkerManager = __commonJS({
   }
 });
 
+// src/bullmq/ProducerManager.js
+var require_ProducerManager = __commonJS({
+  "src/bullmq/ProducerManager.js"(exports, module) {
+    "use strict";
+    var _ProducerManager = class _ProducerManager {
+      constructor(producerClasses = []) {
+        if (_ProducerManager.instance) {
+          return _ProducerManager.instance;
+        }
+        this.producerClasses = producerClasses;
+        this.activeProducers = /* @__PURE__ */ new Map();
+        _ProducerManager.instance = this;
+      }
+      // Static methods that delegate to singleton instance
+      static initialize(producerClasses = []) {
+        if (!this.instance) {
+          this.instance = new _ProducerManager(producerClasses);
+        }
+        return this.instance.initializeAllProducers();
+      }
+      static async addJob(producerClassName, jobName, jobData, opts = {}) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addJob(producerClassName, jobName, jobData, opts);
+      }
+      static async addBulkJobs(producerClassName, jobs) {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.addBulkJobs(producerClassName, jobs);
+      }
+      static async getStatus() {
+        if (!this.instance) {
+          throw new Error("ProducerManager not initialized. Call ProducerManager.initialize() first.");
+        }
+        return await this.instance.getStatus();
+      }
+      static async shutdown() {
+        if (this.instance) {
+          await this.instance.shutdown();
+          this.instance = null;
+        }
+      }
+      /**
+       * Initialize all producers as singletons
+       */
+      initializeAllProducers() {
+        if (this.producerClasses.length === 0) {
+          console.log("No producers provided to initialize");
+          return [];
+        }
+        console.log("\u{1F680} Initializing all producers...");
+        this.producerClasses.forEach((ProducerClass) => {
+          try {
+            const producerInstance = new ProducerClass();
+            this.activeProducers.set(ProducerClass.name, {
+              class: ProducerClass,
+              instance: producerInstance
+            });
+            console.log(`\u2705 Initialized ${ProducerClass.name}`);
+          } catch (error) {
+            console.error(`\u274C Failed to initialize ${ProducerClass.name}:`, error);
+          }
+        });
+        console.log(`\u{1F389} Successfully initialized ${this.activeProducers.size} producers`);
+        return Array.from(this.activeProducers.values());
+      }
+      /**
+       * Get a specific producer instance
+       */
+      getProducer(producerClassName) {
+        const producer = this.activeProducers.get(producerClassName);
+        if (!producer) {
+          throw new Error(`Producer ${producerClassName} not found. Make sure it's initialized.`);
+        }
+        return producer.instance;
+      }
+      /**
+       * Add a job using a specific producer
+       */
+      async addJob(producerClassName, jobData, opts = {}) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addJobWithName(jobData, opts);
+      }
+      /**
+       * Add bulk jobs using a specific producer
+       */
+      async addBulkJobs(producerClassName, jobs) {
+        const producer = this.getProducer(producerClassName);
+        return await producer.addBulkJobs(jobs);
+      }
+      /**
+       * Get status of all producers
+       */
+      async getStatus() {
+        const statusPromises = Array.from(this.activeProducers.entries()).map(
+          async ([name, { instance }]) => {
+            try {
+              const status = await instance.getStatus();
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: !!instance.queue,
+                ...status
+              };
+            } catch (error) {
+              return {
+                name,
+                queueId: instance.config?.id || "unknown",
+                isActive: false,
+                error: error.message
+              };
+            }
+          }
+        );
+        return await Promise.all(statusPromises);
+      }
+      /**
+       * Gracefully shutdown all producers
+       */
+      async shutdown() {
+        console.log("\u{1F6D1} Stopping all producers...");
+        const stopPromises = Array.from(this.activeProducers.entries()).map(
+          ([name, { instance }]) => instance.stop().catch(
+            (err) => console.error(`\u274C Error stopping ${name}:`, err)
+          )
+        );
+        await Promise.all(stopPromises);
+        this.activeProducers.clear();
+        console.log("\u2705 All producers stopped");
+      }
+    };
+    __publicField(_ProducerManager, "instance", null);
+    var ProducerManager2 = _ProducerManager;
+    module.exports = { ProducerManager: ProducerManager2 };
+  }
+});
+
 // src/bullmq/BaseProducer.js
 var require_BaseProducer = __commonJS({
   "src/bullmq/BaseProducer.js"(exports, module) {
@@ -801,7 +1005,8 @@ var require_BaseProducer = __commonJS({
       async stop() {
         if (this.queue) {
           await this.queue.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
+          await this.queue.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} queue closed & disconnected`);
         }
       }
     };
@@ -861,7 +1066,8 @@ var require_BaseWorker = __commonJS({
       async stop() {
         if (this.worker) {
           await this.worker.close();
-          console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
+          await this.worker.disconnect();
+          console.log(`\u{1F6D1} ${this.constructor.name} stopped & disconnected`);
         }
       }
     };
@@ -896,7 +1102,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           concurrency: 1,
           // Process jobs one at a time to avoid race conditions
           limiter: {
-            max: 10,
+            max: 5,
             // Max 10 jobs per...
             duration: 6e4
             // ...60 seconds (rate limiting)
@@ -967,9 +1173,8 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         },
         workerConfig: {
           concurrency: 1,
-          // Process 10 jobs at once for chunk processing
           limiter: {
-            max: 5,
+            max: 15,
             // Max 50 jobs per...
             duration: 6e4
             // ...60 seconds (higher throughput for chunking)
@@ -1013,7 +1218,36 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1
+          concurrency: 1,
+          limiter: {
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
+        }
+      },
+      ANNOS_ELASTIC_SYNC_QUEUE: {
+        id: "annos-elastic-sync-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            attempts: 3,
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            removeOnComplete: 50,
+            removeOnFail: 100
+          }
+        },
+        workerConfig: {
+          concurrency: 5,
+          limiter: {
+            max: 15,
+            // (lets always keep this same as content enhance & embed since it comes immediately after)
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
         }
       }
     };
@@ -1331,6 +1565,7 @@ var _extractBlocksFromSomeBuilders = ({
 
 // src/node.ts
 var import_MongoConnector2 = __toESM(require_MongoConnector());
+var import_ElasticSearchConnector = __toESM(require_ElasticSearchConnector());
 init_getDbByTenant();
 
 // src/db/getGlobalConfig.ts
@@ -1470,13 +1705,16 @@ var getAIConfigs = async ({
 // src/node.ts
 init_models();
 var import_WorkerManager = __toESM(require_WorkerManager());
+var import_ProducerManager = __toESM(require_ProducerManager());
 var import_BaseProducer = __toESM(require_BaseProducer());
 var import_BaseWorker = __toESM(require_BaseWorker());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
 var export_BaseProducer = import_BaseProducer.BaseProducer;
 var export_BaseWorker = import_BaseWorker.BaseWorker;
+var export_ElasticSearchConnector = import_ElasticSearchConnector.ElasticSearchConnector;
 var export_GET_GLOBAL_BULLMQ_CONFIG = import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG;
 var export_MongoConnector = import_MongoConnector2.MongoConnector;
+var export_ProducerManager = import_ProducerManager.ProducerManager;
 var export_WorkerManager = import_WorkerManager.WorkerManager;
 var export_getAIChatModelByTenant = import_getModelByTenant.getAIChatModelByTenant;
 var export_getAnnotationsModelByTenant = import_getModelByTenant.getAnnotationsModelByTenant;
@@ -1488,9 +1726,11 @@ export {
   Annotations_default as AnnotationSchema,
   export_BaseProducer as BaseProducer,
   export_BaseWorker as BaseWorker,
+  export_ElasticSearchConnector as ElasticSearchConnector,
   export_GET_GLOBAL_BULLMQ_CONFIG as GET_GLOBAL_BULLMQ_CONFIG,
   export_MongoConnector as MongoConnector,
   PlatformConfigs_default as PlatformConfigsSchema,
+  export_ProducerManager as ProducerManager,
   Tpl_default as TplSchema,
   export_WorkerManager as WorkerManager,
   connectToRedis,

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.5.4",
+  "version": "1.5.6",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -74,6 +74,7 @@
     "typescript": "^5.8.2"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^8.2.1",
     "bullmq": "^5.58.2",
     "ioredis": "^5.6.1",
     "mongodb": "^6.13.0",