@okf/ootils 1.3.7 → 1.3.9

package/dist/node.d.mts

@@ -1,5 +1,8 @@
 import mongoose, { Connection, Document, Schema, Model, Types } from 'mongoose';
 import IORedis from 'ioredis';
+import * as bullmq from 'bullmq';
+import { Queue } from 'bullmq/dist/esm/classes/queue';
+import { Worker } from 'bullmq/dist/esm/classes/worker';
 
 declare const deleteVal: (data: any, valuePath: string) => any;
 
@@ -250,6 +253,11 @@ declare const getAnnotationsModelByTenant: ({ tenant, env, mongodb, dbConfigs, m
 }> & {
     __v: number;
 }, any>;
+declare const getChunksModelByTenant: ({ tenant, env, mongodb, dbConfigs, modelName }: GetModelShortParams) => Model<Document<unknown, any, any, Record<string, any>>, {}, {}, {}, Document<unknown, {}, Document<unknown, any, any, Record<string, any>>, {}> & Document<unknown, any, any, Record<string, any>> & Required<{
+    _id: unknown;
+}> & {
+    __v: number;
+}, any>;
 declare const getPlatformConfigsModelByTenant: ({ tenant, env, mongodb, dbConfigs }: GetModelShortParams) => Model<Document<unknown, any, any, Record<string, any>>, {}, {}, {}, Document<unknown, {}, Document<unknown, any, any, Record<string, any>>, {}> & Document<unknown, any, any, Record<string, any>> & Required<{
     _id: unknown;
 }> & {
@@ -496,4 +504,94 @@ declare const TplSchema: mongoose.Schema<ITplDocument, mongoose.Model<ITplDocume
     __v: number;
 }>;
 
-export { AIChatSchema, AnnotationSchema, PlatformConfigsSchema, TplSchema, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, multiConnectToMongoDB, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+declare class WorkerManager {
+    constructor(workers?: any[]);
+    workers: any[];
+    activeWorkers: any[];
+    startAllWorkers(): any[];
+    shutdown(): Promise<void>;
+    getStatus(): {
+        name: any;
+        queueId: any;
+        isActive: boolean;
+    }[];
+}
+
+declare class BaseProducer {
+    constructor(config: any);
+    config: any;
+    queue: Queue<any, any, string, any, any, string>;
+    execute(params: any): Promise<void>;
+    addBulkJobs(jobs: any): Promise<bullmq.Job<any, any, string>[]>;
+    addJob(name: any, data: any, opts?: {}): Promise<bullmq.Job<any, any, string>>;
+    getStatus(): Promise<{
+        waiting: number;
+        active: number;
+        completed: number;
+        failed: number;
+        total: number;
+    }>;
+    getJobDetails(limit?: number): Promise<{
+        waiting: {
+            id: any;
+            batchIndex: any;
+            totalBatches: any;
+            rawRecordsCount: any;
+        }[];
+        active: {
+            id: any;
+            batchIndex: any;
+            totalBatches: any;
+            progress: any;
+            rawRecordsCount: any;
+        }[];
+        completed: {
+            id: any;
+            batchIndex: any;
+            totalProcessed: any;
+            summary: any;
+        }[];
+        failed: {
+            id: any;
+            batchIndex: any;
+            error: any;
+            attempts: any;
+        }[];
+    }>;
+    stop(): Promise<void>;
+}
+
+declare class BaseWorker {
+    constructor(config: any);
+    config: any;
+    worker: Worker<any, void, string> | null;
+    execute(job: any): Promise<void>;
+    start(): Worker<any, void, string>;
+    setupEventHandlers(): void;
+    onCompleted(job: any, returnValue: any): void;
+    onFailed(job: any, err: any): void;
+    onProgress(job: any, progress: any): void;
+    onError(error: any): void;
+    stop(): Promise<void>;
+}
+
+/**
+ * Creates BullMQ configuration with explicit dependencies
+ * @param {Object} params Configuration parameters
+ * @param {string} params.env Environment (staging, production, etc.)
+ * @param {Object} params.redisCredentials Redis connection details
+ * @param {string} params.redisCredentials.REDIS_HOST Redis host
+ * @param {number} params.redisCredentials.REDIS_PORT Redis port
+ * @param {string} params.redisCredentials.REDIS_PASSWORD Redis password
+ * @returns {Object} Complete BullMQ configuration
+ */
+declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
+    env: string;
+    redisCredentials: {
+        REDIS_HOST: string;
+        REDIS_PORT: number;
+        REDIS_PASSWORD: string;
+    };
+}): Object;
+
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, GET_GLOBAL_BULLMQ_CONFIG, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getChunksModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, multiConnectToMongoDB, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.d.ts

@@ -1,5 +1,8 @@
 import mongoose, { Connection, Document, Schema, Model, Types } from 'mongoose';
 import IORedis from 'ioredis';
+import * as bullmq from 'bullmq';
+import { Queue } from 'bullmq/dist/esm/classes/queue';
+import { Worker } from 'bullmq/dist/esm/classes/worker';
 
 declare const deleteVal: (data: any, valuePath: string) => any;
 
@@ -250,6 +253,11 @@ declare const getAnnotationsModelByTenant: ({ tenant, env, mongodb, dbConfigs, m
 }> & {
     __v: number;
 }, any>;
+declare const getChunksModelByTenant: ({ tenant, env, mongodb, dbConfigs, modelName }: GetModelShortParams) => Model<Document<unknown, any, any, Record<string, any>>, {}, {}, {}, Document<unknown, {}, Document<unknown, any, any, Record<string, any>>, {}> & Document<unknown, any, any, Record<string, any>> & Required<{
+    _id: unknown;
+}> & {
+    __v: number;
+}, any>;
 declare const getPlatformConfigsModelByTenant: ({ tenant, env, mongodb, dbConfigs }: GetModelShortParams) => Model<Document<unknown, any, any, Record<string, any>>, {}, {}, {}, Document<unknown, {}, Document<unknown, any, any, Record<string, any>>, {}> & Document<unknown, any, any, Record<string, any>> & Required<{
     _id: unknown;
 }> & {
@@ -496,4 +504,94 @@ declare const TplSchema: mongoose.Schema<ITplDocument, mongoose.Model<ITplDocume
     __v: number;
 }>;
 
-export { AIChatSchema, AnnotationSchema, PlatformConfigsSchema, TplSchema, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, multiConnectToMongoDB, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+declare class WorkerManager {
+    constructor(workers?: any[]);
+    workers: any[];
+    activeWorkers: any[];
+    startAllWorkers(): any[];
+    shutdown(): Promise<void>;
+    getStatus(): {
+        name: any;
+        queueId: any;
+        isActive: boolean;
+    }[];
+}
+
+declare class BaseProducer {
+    constructor(config: any);
+    config: any;
+    queue: Queue<any, any, string, any, any, string>;
+    execute(params: any): Promise<void>;
+    addBulkJobs(jobs: any): Promise<bullmq.Job<any, any, string>[]>;
+    addJob(name: any, data: any, opts?: {}): Promise<bullmq.Job<any, any, string>>;
+    getStatus(): Promise<{
+        waiting: number;
+        active: number;
+        completed: number;
+        failed: number;
+        total: number;
+    }>;
+    getJobDetails(limit?: number): Promise<{
+        waiting: {
+            id: any;
+            batchIndex: any;
+            totalBatches: any;
+            rawRecordsCount: any;
+        }[];
+        active: {
+            id: any;
+            batchIndex: any;
+            totalBatches: any;
+            progress: any;
+            rawRecordsCount: any;
+        }[];
+        completed: {
+            id: any;
+            batchIndex: any;
+            totalProcessed: any;
+            summary: any;
+        }[];
+        failed: {
+            id: any;
+            batchIndex: any;
+            error: any;
+            attempts: any;
+        }[];
+    }>;
+    stop(): Promise<void>;
+}
+
+declare class BaseWorker {
+    constructor(config: any);
+    config: any;
+    worker: Worker<any, void, string> | null;
+    execute(job: any): Promise<void>;
+    start(): Worker<any, void, string>;
+    setupEventHandlers(): void;
+    onCompleted(job: any, returnValue: any): void;
+    onFailed(job: any, err: any): void;
+    onProgress(job: any, progress: any): void;
+    onError(error: any): void;
+    stop(): Promise<void>;
+}
+
+/**
+ * Creates BullMQ configuration with explicit dependencies
+ * @param {Object} params Configuration parameters
+ * @param {string} params.env Environment (staging, production, etc.)
+ * @param {Object} params.redisCredentials Redis connection details
+ * @param {string} params.redisCredentials.REDIS_HOST Redis host
+ * @param {number} params.redisCredentials.REDIS_PORT Redis port
+ * @param {string} params.redisCredentials.REDIS_PASSWORD Redis password
+ * @returns {Object} Complete BullMQ configuration
+ */
+declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
+    env: string;
+    redisCredentials: {
+        REDIS_HOST: string;
+        REDIS_PORT: number;
+        REDIS_PASSWORD: string;
+    };
+}): Object;
+
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, GET_GLOBAL_BULLMQ_CONFIG, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getChunksModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, multiConnectToMongoDB, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.js

@@ -5,6 +5,12 @@ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
 var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __commonJS = (cb, mod) => function __require() {
+  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+};
 var __export = (target, all) => {
   for (var name in all)
     __defProp(target, name, { get: all[name], enumerable: true });
@@ -27,13 +33,318 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
+// src/bullmq/WorkerManager.js
+var require_WorkerManager = __commonJS({
+  "src/bullmq/WorkerManager.js"(exports2, module2) {
+    "use strict";
+    var WorkerManager2 = class {
+      constructor(workers = []) {
+        this.workers = workers;
+        this.activeWorkers = [];
+      }
+      startAllWorkers() {
+        if (this.workers.length === 0) {
+          console.log("No workers provided to start");
+          return [];
+        }
+        console.log("\u{1F680} Starting all workers...");
+        this.workers.forEach((WorkerClass) => {
+          try {
+            const workerInstance = new WorkerClass();
+            workerInstance.start();
+            this.activeWorkers.push({
+              name: WorkerClass.name,
+              instance: workerInstance
+            });
+            console.log(`\u2705 Started ${WorkerClass.name}`);
+          } catch (error) {
+            console.error(`\u274C Failed to start ${WorkerClass.name}:`, error);
+          }
+        });
+        console.log(`\u{1F389} Successfully started ${this.activeWorkers.length} workers`);
+        return this.activeWorkers;
+      }
+      async shutdown() {
+        console.log("\u{1F6D1} Stopping all workers...");
+        const stopPromises = this.activeWorkers.map(
+          ({ name, instance }) => instance.stop().catch(
+            (err) => console.error(`\u274C Error stopping ${name}:`, err)
+          )
+        );
+        await Promise.all(stopPromises);
+        this.activeWorkers = [];
+        console.log("\u2705 All workers stopped");
+      }
+      getStatus() {
+        return this.activeWorkers.map(({ name, instance }) => ({
+          name,
+          queueId: instance.config?.id || "unknown",
+          isActive: !!instance.worker
+        }));
+      }
+    };
+    module2.exports = { WorkerManager: WorkerManager2 };
+  }
+});
+
+// src/bullmq/BaseProducer.js
+var require_BaseProducer = __commonJS({
+  "src/bullmq/BaseProducer.js"(exports2, module2) {
+    "use strict";
+    var { Queue } = require("bullmq");
+    var BaseProducer2 = class {
+      constructor(config) {
+        this.config = config;
+        this.queue = new Queue(config.id, config.queueConfig);
+      }
+      async execute(params) {
+        throw new Error("execute() method must be implemented by subclass");
+      }
+      async addBulkJobs(jobs) {
+        try {
+          const results = await this.queue.addBulk(jobs);
+          console.log(`\u2705 ${this.constructor.name}: Successfully queued ${results.length} jobs`);
+          return results;
+        } catch (error) {
+          console.error(`\u274C ${this.constructor.name}: Failed to queue jobs:`, error);
+          throw error;
+        }
+      }
+      async addJob(name, data, opts = {}) {
+        try {
+          const job = await this.queue.add(name, data, opts);
+          console.log(`\u2705 ${this.constructor.name}: Successfully queued job ${job.id}`);
+          return job;
+        } catch (error) {
+          console.error(`\u274C ${this.constructor.name}: Failed to queue job:`, error);
+          throw error;
+        }
+      }
+      async getStatus() {
+        const waiting = await this.queue.getWaiting();
+        const active = await this.queue.getActive();
+        const completed = await this.queue.getCompleted();
+        const failed = await this.queue.getFailed();
+        return {
+          waiting: waiting.length,
+          active: active.length,
+          completed: completed.length,
+          failed: failed.length,
+          total: waiting.length + active.length + completed.length + failed.length
+        };
+      }
+      async getJobDetails(limit = 10) {
+        const waiting = await this.queue.getWaiting(0, limit - 1);
+        const active = await this.queue.getActive(0, limit - 1);
+        const completed = await this.queue.getCompleted(0, limit - 1);
+        const failed = await this.queue.getFailed(0, limit - 1);
+        return {
+          waiting: waiting.map((job) => ({
+            id: job.id,
+            batchIndex: job.data?.batchIndex,
+            totalBatches: job.data?.totalBatches,
+            rawRecordsCount: job.data?.rawBatchData?.length
+          })),
+          active: active.map((job) => ({
+            id: job.id,
+            batchIndex: job.data?.batchIndex,
+            totalBatches: job.data?.totalBatches,
+            progress: job.progress,
+            rawRecordsCount: job.data?.rawBatchData?.length
+          })),
+          completed: completed.map((job) => ({
+            id: job.id,
+            batchIndex: job.returnvalue?.batchIndex,
+            totalProcessed: job.returnvalue?.totalItemsProcessed,
+            summary: job.returnvalue?.summary
+          })),
+          failed: failed.map((job) => ({
+            id: job.id,
+            batchIndex: job.data?.batchIndex,
+            error: job.failedReason,
+            attempts: job.attemptsMade
+          }))
+        };
+      }
+      async stop() {
+        if (this.queue) {
+          await this.queue.close();
+          console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
+        }
+      }
+    };
+    module2.exports = { BaseProducer: BaseProducer2 };
+  }
+});
+
+// src/bullmq/BaseWorker.js
+var require_BaseWorker = __commonJS({
+  "src/bullmq/BaseWorker.js"(exports2, module2) {
+    "use strict";
+    var { Worker } = require("bullmq");
+    var BaseWorker2 = class {
+      constructor(config) {
+        this.config = config;
+        this.worker = null;
+      }
+      async execute(job) {
+        throw new Error("execute() method must be implemented by subclass");
+      }
+      start() {
+        this.worker = new Worker(
+          this.config.id,
+          this.execute.bind(this),
+          this.config.workerConfig
+        );
+        this.setupEventHandlers();
+        console.log(`\u{1F680} ${this.constructor.name} started for queue: ${this.config.id}`);
+        return this.worker;
+      }
+      setupEventHandlers() {
+        this.worker.on("completed", (job, returnValue) => {
+          this.onCompleted(job, returnValue);
+        });
+        this.worker.on("failed", (job, err) => {
+          this.onFailed(job, err);
+        });
+        this.worker.on("progress", (job, progress) => {
+          this.onProgress(job, progress);
+        });
+        this.worker.on("error", (error) => {
+          this.onError(error);
+        });
+      }
+      onCompleted(job, returnValue) {
+        console.log(`\u2705 Job ${job.id} completed in ${this.constructor.name}`);
+      }
+      onFailed(job, err) {
+        console.error(`\u{1F4A5} Job ${job.id} failed in ${this.constructor.name}:`, err.message);
+      }
+      onProgress(job, progress) {
+        console.log(`\u{1F4CA} Job ${job.id} progress in ${this.constructor.name}: ${progress}%`);
+      }
+      onError(error) {
+        console.error(`\u274C Worker error in ${this.constructor.name}:`, error);
+      }
+      async stop() {
+        if (this.worker) {
+          await this.worker.close();
+          console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
+        }
+      }
+    };
+    module2.exports = { BaseWorker: BaseWorker2 };
+  }
+});
+
+// src/bullmq/GLOBAL_BULLMQ_CONFIG.js
+var GLOBAL_BULLMQ_CONFIG_exports = {};
+__export(GLOBAL_BULLMQ_CONFIG_exports, {
+  BASE_BULLMQ_CONFIG: () => BASE_BULLMQ_CONFIG
+});
+var BASE_BULLMQ_CONFIG;
+var init_GLOBAL_BULLMQ_CONFIG = __esm({
+  "src/bullmq/GLOBAL_BULLMQ_CONFIG.js"() {
+    "use strict";
+    BASE_BULLMQ_CONFIG = {
+      PLUGIN__MAD_USERS_SYNC_QUEUE: {
+        id: "plugin--mad-users-sync-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            attempts: 3,
+            removeOnComplete: 100,
+            removeOnFail: 50
+          }
+        },
+        workerConfig: {
+          concurrency: 1,
+          // Process jobs one at a time to avoid race conditions
+          limiter: {
+            max: 10,
+            // Max 10 jobs per...
+            duration: 6e4
+            // ...60 seconds (rate limiting)
+          }
+        }
+      },
+      // Chunk Processing Queue
+      CHUNK_PROCESSING_QUEUE: {
+        id: "chunk-processing-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            attempts: 3,
+            removeOnComplete: 100,
+            removeOnFail: 50
+          }
+        },
+        workerConfig: {
+          concurrency: 10,
+          // Process 10 jobs at once for chunk processing
+          limiter: {
+            max: 5,
+            // Max 50 jobs per...
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
+        }
+      }
+    };
+  }
+});
+
+// src/bullmq/GET_GLOBAL_BULLMQ_CONFIG.js
+var require_GET_GLOBAL_BULLMQ_CONFIG = __commonJS({
+  "src/bullmq/GET_GLOBAL_BULLMQ_CONFIG.js"(exports2, module2) {
+    "use strict";
+    var { default: Redis } = require("ioredis");
+    var { BASE_BULLMQ_CONFIG: BASE_BULLMQ_CONFIG2 } = (init_GLOBAL_BULLMQ_CONFIG(), __toCommonJS(GLOBAL_BULLMQ_CONFIG_exports));
+    function GET_GLOBAL_BULLMQ_CONFIG2({ env, redisCredentials }) {
+      const redisConnectionForBullMQ = new Redis({
+        host: redisCredentials.REDIS_HOST,
+        port: redisCredentials.REDIS_PORT,
+        password: redisCredentials.REDIS_PASSWORD,
+        maxRetriesPerRequest: null
+      });
+      return Object.keys(BASE_BULLMQ_CONFIG2).reduce((acc, key) => ({
+        ...acc,
+        [key]: {
+          ...BASE_BULLMQ_CONFIG2[key],
+          id: BASE_BULLMQ_CONFIG2[key].id + `_${env}`,
+          // suffix env to queue to keep it unique for each environment
+          queueConfig: {
+            connection: redisConnectionForBullMQ,
+            ...BASE_BULLMQ_CONFIG2[key].queueConfig
+          },
+          workerConfig: {
+            connection: redisConnectionForBullMQ,
+            ...BASE_BULLMQ_CONFIG2[key].workerConfig
+          }
+        }
+      }), {});
+    }
+    module2.exports = { GET_GLOBAL_BULLMQ_CONFIG: GET_GLOBAL_BULLMQ_CONFIG2 };
+  }
+});
+
 // src/node.ts
 var node_exports = {};
 __export(node_exports, {
   AIChatSchema: () => AIChat_default,
   AnnotationSchema: () => Annotations_default,
+  BaseProducer: () => import_BaseProducer.BaseProducer,
+  BaseWorker: () => import_BaseWorker.BaseWorker,
+  GET_GLOBAL_BULLMQ_CONFIG: () => import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG,
   PlatformConfigsSchema: () => PlatformConfigs_default,
   TplSchema: () => Tpl_default,
+  WorkerManager: () => import_WorkerManager.WorkerManager,
   connectToRedis: () => connectToRedis,
   deleteVal: () => deleteVal,
   extractAllBlocksFromTpl: () => extractAllBlocksFromTpl,
@@ -41,6 +352,7 @@ __export(node_exports, {
   getAIChatModelByTenant: () => getAIChatModelByTenant,
   getAIConfigs: () => getAIConfigs,
   getAnnotationsModelByTenant: () => getAnnotationsModelByTenant,
+  getChunksModelByTenant: () => getChunksModelByTenant,
   getDbByTenant: () => getDbByTenant,
   getModelByTenant: () => getModelByTenant,
   getPlatformConfigsModelByTenant: () => getPlatformConfigsModelByTenant,
@@ -762,6 +1074,12 @@ var getAnnotationsModelByTenant = ({ tenant, env, mongodb, dbConfigs, modelName
   schema: Annotations_default,
   env
 });
+var getChunksModelByTenant = ({ tenant, env, mongodb, dbConfigs, modelName }) => getModelByTenant({
+  tenant,
+  modelName: modelName || "chunks",
+  schema: Annotations_default,
+  env
+});
 var getPlatformConfigsModelByTenant = ({ tenant, env, mongodb, dbConfigs }) => getModelByTenant({
   tenant,
   modelName: "platformConfigs",
@@ -887,12 +1205,22 @@ var getAIConfigs = async ({
     }
   });
 };
+
+// src/node.ts
+var import_WorkerManager = __toESM(require_WorkerManager());
+var import_BaseProducer = __toESM(require_BaseProducer());
+var import_BaseWorker = __toESM(require_BaseWorker());
+var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AIChatSchema,
   AnnotationSchema,
+  BaseProducer,
+  BaseWorker,
+  GET_GLOBAL_BULLMQ_CONFIG,
   PlatformConfigsSchema,
   TplSchema,
+  WorkerManager,
   connectToRedis,
   deleteVal,
   extractAllBlocksFromTpl,
@@ -900,6 +1228,7 @@ var getAIConfigs = async ({
   getAIChatModelByTenant,
   getAIConfigs,
   getAnnotationsModelByTenant,
+  getChunksModelByTenant,
   getDbByTenant,
   getModelByTenant,
   getPlatformConfigsModelByTenant,

package/dist/node.mjs

@@ -1,3 +1,344 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
+  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
+}) : x)(function(x) {
+  if (typeof require !== "undefined") return require.apply(this, arguments);
+  throw Error('Dynamic require of "' + x + '" is not supported');
+});
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __commonJS = (cb, mod) => function __require2() {
+  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/bullmq/WorkerManager.js
+var require_WorkerManager = __commonJS({
+  "src/bullmq/WorkerManager.js"(exports, module) {
+    "use strict";
+    var WorkerManager2 = class {
+      constructor(workers = []) {
+        this.workers = workers;
+        this.activeWorkers = [];
+      }
+      startAllWorkers() {
+        if (this.workers.length === 0) {
+          console.log("No workers provided to start");
+          return [];
+        }
+        console.log("\u{1F680} Starting all workers...");
+        this.workers.forEach((WorkerClass) => {
+          try {
+            const workerInstance = new WorkerClass();
+            workerInstance.start();
+            this.activeWorkers.push({
+              name: WorkerClass.name,
+              instance: workerInstance
+            });
+            console.log(`\u2705 Started ${WorkerClass.name}`);
+          } catch (error) {
+            console.error(`\u274C Failed to start ${WorkerClass.name}:`, error);
+          }
+        });
+        console.log(`\u{1F389} Successfully started ${this.activeWorkers.length} workers`);
+        return this.activeWorkers;
+      }
+      async shutdown() {
+        console.log("\u{1F6D1} Stopping all workers...");
+        const stopPromises = this.activeWorkers.map(
+          ({ name, instance }) => instance.stop().catch(
+            (err) => console.error(`\u274C Error stopping ${name}:`, err)
+          )
+        );
+        await Promise.all(stopPromises);
+        this.activeWorkers = [];
+        console.log("\u2705 All workers stopped");
+      }
+      getStatus() {
+        return this.activeWorkers.map(({ name, instance }) => ({
+          name,
+          queueId: instance.config?.id || "unknown",
+          isActive: !!instance.worker
+        }));
+      }
+    };
+    module.exports = { WorkerManager: WorkerManager2 };
+  }
+});
+
+// src/bullmq/BaseProducer.js
+var require_BaseProducer = __commonJS({
+  "src/bullmq/BaseProducer.js"(exports, module) {
+    "use strict";
+    var { Queue } = __require("bullmq");
+    var BaseProducer2 = class {
+      constructor(config) {
+        this.config = config;
+        this.queue = new Queue(config.id, config.queueConfig);
+      }
+      async execute(params) {
+        throw new Error("execute() method must be implemented by subclass");
+      }
+      async addBulkJobs(jobs) {
+        try {
+          const results = await this.queue.addBulk(jobs);
+          console.log(`\u2705 ${this.constructor.name}: Successfully queued ${results.length} jobs`);
+          return results;
+        } catch (error) {
+          console.error(`\u274C ${this.constructor.name}: Failed to queue jobs:`, error);
+          throw error;
+        }
+      }
+      async addJob(name, data, opts = {}) {
+        try {
+          const job = await this.queue.add(name, data, opts);
+          console.log(`\u2705 ${this.constructor.name}: Successfully queued job ${job.id}`);
+          return job;
+        } catch (error) {
+          console.error(`\u274C ${this.constructor.name}: Failed to queue job:`, error);
+          throw error;
+        }
+      }
+      async getStatus() {
+        const waiting = await this.queue.getWaiting();
+        const active = await this.queue.getActive();
+        const completed = await this.queue.getCompleted();
+        const failed = await this.queue.getFailed();
+        return {
+          waiting: waiting.length,
+          active: active.length,
+          completed: completed.length,
+          failed: failed.length,
+          total: waiting.length + active.length + completed.length + failed.length
+        };
+      }
+      async getJobDetails(limit = 10) {
+        const waiting = await this.queue.getWaiting(0, limit - 1);
+        const active = await this.queue.getActive(0, limit - 1);
+        const completed = await this.queue.getCompleted(0, limit - 1);
+        const failed = await this.queue.getFailed(0, limit - 1);
+        return {
+          waiting: waiting.map((job) => ({
+            id: job.id,
+            batchIndex: job.data?.batchIndex,
+            totalBatches: job.data?.totalBatches,
+            rawRecordsCount: job.data?.rawBatchData?.length
+          })),
+          active: active.map((job) => ({
+            id: job.id,
+            batchIndex: job.data?.batchIndex,
+            totalBatches: job.data?.totalBatches,
+            progress: job.progress,
+            rawRecordsCount: job.data?.rawBatchData?.length
+          })),
+          completed: completed.map((job) => ({
+            id: job.id,
+            batchIndex: job.returnvalue?.batchIndex,
+            totalProcessed: job.returnvalue?.totalItemsProcessed,
+            summary: job.returnvalue?.summary
+          })),
+          failed: failed.map((job) => ({
+            id: job.id,
+            batchIndex: job.data?.batchIndex,
+            error: job.failedReason,
+            attempts: job.attemptsMade
+          }))
+        };
+      }
+      async stop() {
+        if (this.queue) {
+          await this.queue.close();
+          console.log(`\u{1F6D1} ${this.constructor.name} queue closed`);
+        }
+      }
+    };
+    module.exports = { BaseProducer: BaseProducer2 };
+  }
+});
+
+// src/bullmq/BaseWorker.js
+var require_BaseWorker = __commonJS({
+  "src/bullmq/BaseWorker.js"(exports, module) {
+    "use strict";
+    var { Worker } = __require("bullmq");
+    var BaseWorker2 = class {
+      constructor(config) {
+        this.config = config;
+        this.worker = null;
+      }
+      async execute(job) {
+        throw new Error("execute() method must be implemented by subclass");
+      }
+      start() {
+        this.worker = new Worker(
+          this.config.id,
+          this.execute.bind(this),
+          this.config.workerConfig
+        );
+        this.setupEventHandlers();
+        console.log(`\u{1F680} ${this.constructor.name} started for queue: ${this.config.id}`);
+        return this.worker;
+      }
+      setupEventHandlers() {
+        this.worker.on("completed", (job, returnValue) => {
+          this.onCompleted(job, returnValue);
+        });
+        this.worker.on("failed", (job, err) => {
+          this.onFailed(job, err);
+        });
+        this.worker.on("progress", (job, progress) => {
+          this.onProgress(job, progress);
+        });
+        this.worker.on("error", (error) => {
+          this.onError(error);
+        });
+      }
+      onCompleted(job, returnValue) {
+        console.log(`\u2705 Job ${job.id} completed in ${this.constructor.name}`);
+      }
+      onFailed(job, err) {
+        console.error(`\u{1F4A5} Job ${job.id} failed in ${this.constructor.name}:`, err.message);
+      }
+      onProgress(job, progress) {
+        console.log(`\u{1F4CA} Job ${job.id} progress in ${this.constructor.name}: ${progress}%`);
+      }
+      onError(error) {
+        console.error(`\u274C Worker error in ${this.constructor.name}:`, error);
+      }
+      async stop() {
+        if (this.worker) {
+          await this.worker.close();
+          console.log(`\u{1F6D1} ${this.constructor.name} stopped`);
+        }
+      }
+    };
+    module.exports = { BaseWorker: BaseWorker2 };
+  }
+});
+
+// src/bullmq/GLOBAL_BULLMQ_CONFIG.js
+var GLOBAL_BULLMQ_CONFIG_exports = {};
+__export(GLOBAL_BULLMQ_CONFIG_exports, {
+  BASE_BULLMQ_CONFIG: () => BASE_BULLMQ_CONFIG
+});
+var BASE_BULLMQ_CONFIG;
+var init_GLOBAL_BULLMQ_CONFIG = __esm({
+  "src/bullmq/GLOBAL_BULLMQ_CONFIG.js"() {
+    "use strict";
+    BASE_BULLMQ_CONFIG = {
+      PLUGIN__MAD_USERS_SYNC_QUEUE: {
+        id: "plugin--mad-users-sync-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            attempts: 3,
+            removeOnComplete: 100,
+            removeOnFail: 50
+          }
+        },
+        workerConfig: {
+          concurrency: 1,
+          // Process jobs one at a time to avoid race conditions
+          limiter: {
+            max: 10,
+            // Max 10 jobs per...
+            duration: 6e4
+            // ...60 seconds (rate limiting)
+          }
+        }
+      },
+      // Chunk Processing Queue
+      CHUNK_PROCESSING_QUEUE: {
+        id: "chunk-processing-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            attempts: 3,
+            removeOnComplete: 100,
+            removeOnFail: 50
+          }
+        },
+        workerConfig: {
+          concurrency: 10,
+          // Process 10 jobs at once for chunk processing
+          limiter: {
+            max: 5,
+            // Max 50 jobs per...
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
+        }
+      }
+    };
+  }
+});
+
+// src/bullmq/GET_GLOBAL_BULLMQ_CONFIG.js
+var require_GET_GLOBAL_BULLMQ_CONFIG = __commonJS({
+  "src/bullmq/GET_GLOBAL_BULLMQ_CONFIG.js"(exports, module) {
+    "use strict";
+    var { default: Redis } = __require("ioredis");
+    var { BASE_BULLMQ_CONFIG: BASE_BULLMQ_CONFIG2 } = (init_GLOBAL_BULLMQ_CONFIG(), __toCommonJS(GLOBAL_BULLMQ_CONFIG_exports));
+    function GET_GLOBAL_BULLMQ_CONFIG2({ env, redisCredentials }) {
+      const redisConnectionForBullMQ = new Redis({
+        host: redisCredentials.REDIS_HOST,
+        port: redisCredentials.REDIS_PORT,
+        password: redisCredentials.REDIS_PASSWORD,
+        maxRetriesPerRequest: null
+      });
+      return Object.keys(BASE_BULLMQ_CONFIG2).reduce((acc, key) => ({
+        ...acc,
+        [key]: {
+          ...BASE_BULLMQ_CONFIG2[key],
+          id: BASE_BULLMQ_CONFIG2[key].id + `_${env}`,
+          // suffix env to queue to keep it unique for each environment
+          queueConfig: {
+            connection: redisConnectionForBullMQ,
+            ...BASE_BULLMQ_CONFIG2[key].queueConfig
+          },
+          workerConfig: {
+            connection: redisConnectionForBullMQ,
+            ...BASE_BULLMQ_CONFIG2[key].workerConfig
+          }
+        }
+      }), {});
+    }
+    module.exports = { GET_GLOBAL_BULLMQ_CONFIG: GET_GLOBAL_BULLMQ_CONFIG2 };
+  }
+});
+
 // src/utils/getterSetterDeleter/utils/set_deleteVal.ts
 var set_deleteVal = (action, data, valuePath, value) => {
   if (valuePath === void 0) return;
@@ -703,6 +1044,12 @@ var getAnnotationsModelByTenant = ({ tenant, env, mongodb, dbConfigs, modelName
   schema: Annotations_default,
   env
 });
+var getChunksModelByTenant = ({ tenant, env, mongodb, dbConfigs, modelName }) => getModelByTenant({
+  tenant,
+  modelName: modelName || "chunks",
+  schema: Annotations_default,
+  env
+});
 var getPlatformConfigsModelByTenant = ({ tenant, env, mongodb, dbConfigs }) => getModelByTenant({
   tenant,
   modelName: "platformConfigs",
@@ -828,11 +1175,25 @@ var getAIConfigs = async ({
     }
   });
 };
+
+// src/node.ts
+var import_WorkerManager = __toESM(require_WorkerManager());
+var import_BaseProducer = __toESM(require_BaseProducer());
+var import_BaseWorker = __toESM(require_BaseWorker());
+var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
+var export_BaseProducer = import_BaseProducer.BaseProducer;
+var export_BaseWorker = import_BaseWorker.BaseWorker;
+var export_GET_GLOBAL_BULLMQ_CONFIG = import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG;
+var export_WorkerManager = import_WorkerManager.WorkerManager;
 export {
   AIChat_default as AIChatSchema,
   Annotations_default as AnnotationSchema,
+  export_BaseProducer as BaseProducer,
+  export_BaseWorker as BaseWorker,
+  export_GET_GLOBAL_BULLMQ_CONFIG as GET_GLOBAL_BULLMQ_CONFIG,
   PlatformConfigs_default as PlatformConfigsSchema,
   Tpl_default as TplSchema,
+  export_WorkerManager as WorkerManager,
   connectToRedis,
   deleteVal,
   extractAllBlocksFromTpl,
@@ -840,6 +1201,7 @@ export {
   getAIChatModelByTenant,
   getAIConfigs,
   getAnnotationsModelByTenant,
+  getChunksModelByTenant,
   getDbByTenant,
   getModelByTenant,
   getPlatformConfigsModelByTenant,

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.3.7",
+  "version": "1.3.9",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -74,6 +74,7 @@
     "typescript": "^5.8.2"
   },
   "dependencies": {
+    "bullmq": "^5.58.2",
     "ioredis": "^5.6.1",
     "mongoose": "^8.15.1"
   }