@okf/ootils 1.5.3 → 1.5.5

package/dist/node.d.mts

@@ -1,3 +1,4 @@
+import { Client } from '@elastic/elasticsearch';
 import mongoose, { Connection, Document, Types } from 'mongoose';
 import IORedis from 'ioredis';
 import * as bullmq from 'bullmq';
@@ -187,6 +188,26 @@ declare namespace MongoConnector {
     let instance: any;
 }
 
+declare class ElasticSearchConnector {
+    static getInstance(): any;
+    static getClient(): any;
+    constructor(options?: {});
+    client: Client | null;
+    cloudId: any;
+    apiKey: any;
+    /**
+     * Initialize Elasticsearch client
+     */
+    connect(): Client;
+    /**
+     * Close the Elasticsearch client
+     */
+    close(): Promise<void>;
+}
+declare namespace ElasticSearchConnector {
+    let instance: any;
+}
+
 /**
  * Get database connection by tenant
  * @param tenant - Tenant identifier for the database
@@ -322,7 +343,7 @@ interface IAnnotation extends Document {
     meta: IMeta;
     main?: any;
     annotations?: IAnnotations;
-    embeddings?: any[];
+    embeddings?: number[] | undefined;
     contentEnhancedText?: string;
     clusterId?: string;
     kp_date_published?: Date;
@@ -572,4 +593,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.d.ts

@@ -1,3 +1,4 @@
+import { Client } from '@elastic/elasticsearch';
 import mongoose, { Connection, Document, Types } from 'mongoose';
 import IORedis from 'ioredis';
 import * as bullmq from 'bullmq';
@@ -187,6 +188,26 @@ declare namespace MongoConnector {
     let instance: any;
 }
 
+declare class ElasticSearchConnector {
+    static getInstance(): any;
+    static getClient(): any;
+    constructor(options?: {});
+    client: Client | null;
+    cloudId: any;
+    apiKey: any;
+    /**
+     * Initialize Elasticsearch client
+     */
+    connect(): Client;
+    /**
+     * Close the Elasticsearch client
+     */
+    close(): Promise<void>;
+}
+declare namespace ElasticSearchConnector {
+    let instance: any;
+}
+
 /**
  * Get database connection by tenant
  * @param tenant - Tenant identifier for the database
@@ -322,7 +343,7 @@ interface IAnnotation extends Document {
     meta: IMeta;
     main?: any;
     annotations?: IAnnotations;
-    embeddings?: any[];
+    embeddings?: number[] | undefined;
     contentEnhancedText?: string;
     clusterId?: string;
     kp_date_published?: Date;
@@ -572,4 +593,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };
+export { AIChatSchema, AnnotationSchema, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, MongoConnector, PlatformConfigsSchema, TplSchema, WorkerManager, connectToRedis, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAIConfigs, getAnnotationsModelByTenant, getDbByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getRedisClient, getTpl, getTplModelByTenant, getVal, initializeGlobalConfig, _recursExtractBlocks as recursivelyExtractBlocks, setVal, toArray, updateGlobalConfig };

package/dist/node.js

@@ -108,7 +108,10 @@ var init_Annotations = __esm({
             name: String
           }
         },
-        embeddings: Array,
+        embeddings: {
+          type: [Number],
+          default: void 0
+        },
         contentEnhancedText: String,
         // // Optional chunk related fields
         // chunk: {
@@ -515,6 +518,71 @@ var require_MongoConnector = __commonJS({
   }
 });
 
+// src/ElasticSearchConnector.js
+var require_ElasticSearchConnector = __commonJS({
+  "src/ElasticSearchConnector.js"(exports2, module2) {
+    "use strict";
+    var { Client } = require("@elastic/elasticsearch");
+    var ElasticSearchConnector2 = class _ElasticSearchConnector {
+      constructor(options = {}) {
+        this.client = null;
+        this.cloudId = options.cloudId;
+        this.apiKey = options.apiKey;
+        _ElasticSearchConnector.instance = this;
+        if (!this.cloudId) {
+          throw new Error("Cloud ID must be provided in constructor options or ELASTIC_CLOUD_ID environment variable");
+        }
+        if (!this.apiKey) {
+          throw new Error("API Key must be provided in constructor options or ELASTIC_API_KEY environment variable");
+        }
+      }
+      /**
+       * Initialize Elasticsearch client
+       */
+      connect() {
+        this.client = new Client({
+          cloud: {
+            id: this.cloudId
+          },
+          auth: {
+            apiKey: this.apiKey
+          }
+        });
+        console.log("\u2705 Elasticsearch client initialized");
+        return this.client;
+      }
+      // Static method to get the instance
+      static getInstance() {
+        if (!_ElasticSearchConnector.instance) {
+          throw new Error("ElasticSearchConnector not initialized");
+        }
+        return _ElasticSearchConnector.instance;
+      }
+      // Static method to get the client
+      static getClient() {
+        if (!_ElasticSearchConnector.instance || !_ElasticSearchConnector.instance.client) {
+          throw new Error("ElasticSearchConnector not initialized or client not connected");
+        }
+        return _ElasticSearchConnector.instance.client;
+      }
+      /**
+       * Close the Elasticsearch client
+       */
+      async close() {
+        if (this.client) {
+          await this.client.close();
+          console.log("\u2705 Elasticsearch client closed");
+          this.client = null;
+        }
+      }
+    };
+    ElasticSearchConnector2.instance = null;
+    module2.exports = {
+      ElasticSearchConnector: ElasticSearchConnector2
+    };
+  }
+});
+
 // src/db/getDbByTenant.js
 var getDbByTenant_exports = {};
 __export(getDbByTenant_exports, {
@@ -959,7 +1027,6 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         },
         workerConfig: {
           concurrency: 1,
-          // Process 10 jobs at once for chunk processing
           limiter: {
             max: 5,
             // Max 50 jobs per...
@@ -1005,7 +1072,36 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1
+          concurrency: 1,
+          limiter: {
+            max: 5,
+            // Max 5 jobs per...
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
+        }
+      },
+      ANNOS_ELASTIC_SYNC_QUEUE: {
+        id: "annos-elastic-sync-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            attempts: 3,
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            removeOnComplete: 50,
+            removeOnFail: 100
+          }
+        },
+        workerConfig: {
+          concurrency: 5,
+          limiter: {
+            max: 5,
+            // Max 5 jobs per...
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
         }
       }
     };
@@ -1053,6 +1149,7 @@ __export(node_exports, {
   AnnotationSchema: () => Annotations_default,
   BaseProducer: () => import_BaseProducer.BaseProducer,
   BaseWorker: () => import_BaseWorker.BaseWorker,
+  ElasticSearchConnector: () => import_ElasticSearchConnector.ElasticSearchConnector,
   GET_GLOBAL_BULLMQ_CONFIG: () => import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG,
   MongoConnector: () => import_MongoConnector2.MongoConnector,
   PlatformConfigsSchema: () => PlatformConfigs_default,
@@ -1357,6 +1454,7 @@ var _extractBlocksFromSomeBuilders = ({
 
 // src/node.ts
 var import_MongoConnector2 = __toESM(require_MongoConnector());
+var import_ElasticSearchConnector = __toESM(require_ElasticSearchConnector());
 init_getDbByTenant();
 
 // src/db/getGlobalConfig.ts
@@ -1505,6 +1603,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   AnnotationSchema,
   BaseProducer,
   BaseWorker,
+  ElasticSearchConnector,
   GET_GLOBAL_BULLMQ_CONFIG,
   MongoConnector,
   PlatformConfigsSchema,

package/dist/node.mjs

@@ -113,7 +113,10 @@ var init_Annotations = __esm({
             name: String
           }
         },
-        embeddings: Array,
+        embeddings: {
+          type: [Number],
+          default: void 0
+        },
         contentEnhancedText: String,
         // // Optional chunk related fields
         // chunk: {
@@ -520,6 +523,71 @@ var require_MongoConnector = __commonJS({
   }
 });
 
+// src/ElasticSearchConnector.js
+var require_ElasticSearchConnector = __commonJS({
+  "src/ElasticSearchConnector.js"(exports, module) {
+    "use strict";
+    var { Client } = __require("@elastic/elasticsearch");
+    var ElasticSearchConnector2 = class _ElasticSearchConnector {
+      constructor(options = {}) {
+        this.client = null;
+        this.cloudId = options.cloudId;
+        this.apiKey = options.apiKey;
+        _ElasticSearchConnector.instance = this;
+        if (!this.cloudId) {
+          throw new Error("Cloud ID must be provided in constructor options or ELASTIC_CLOUD_ID environment variable");
+        }
+        if (!this.apiKey) {
+          throw new Error("API Key must be provided in constructor options or ELASTIC_API_KEY environment variable");
+        }
+      }
+      /**
+       * Initialize Elasticsearch client
+       */
+      connect() {
+        this.client = new Client({
+          cloud: {
+            id: this.cloudId
+          },
+          auth: {
+            apiKey: this.apiKey
+          }
+        });
+        console.log("\u2705 Elasticsearch client initialized");
+        return this.client;
+      }
+      // Static method to get the instance
+      static getInstance() {
+        if (!_ElasticSearchConnector.instance) {
+          throw new Error("ElasticSearchConnector not initialized");
+        }
+        return _ElasticSearchConnector.instance;
+      }
+      // Static method to get the client
+      static getClient() {
+        if (!_ElasticSearchConnector.instance || !_ElasticSearchConnector.instance.client) {
+          throw new Error("ElasticSearchConnector not initialized or client not connected");
+        }
+        return _ElasticSearchConnector.instance.client;
+      }
+      /**
+       * Close the Elasticsearch client
+       */
+      async close() {
+        if (this.client) {
+          await this.client.close();
+          console.log("\u2705 Elasticsearch client closed");
+          this.client = null;
+        }
+      }
+    };
+    ElasticSearchConnector2.instance = null;
+    module.exports = {
+      ElasticSearchConnector: ElasticSearchConnector2
+    };
+  }
+});
+
 // src/db/getDbByTenant.js
 var getDbByTenant_exports = {};
 __export(getDbByTenant_exports, {
@@ -964,7 +1032,6 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         },
         workerConfig: {
           concurrency: 1,
-          // Process 10 jobs at once for chunk processing
           limiter: {
             max: 5,
             // Max 50 jobs per...
@@ -1010,7 +1077,36 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1
+          concurrency: 1,
+          limiter: {
+            max: 5,
+            // Max 5 jobs per...
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
+        }
+      },
+      ANNOS_ELASTIC_SYNC_QUEUE: {
+        id: "annos-elastic-sync-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            attempts: 3,
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            removeOnComplete: 50,
+            removeOnFail: 100
+          }
+        },
+        workerConfig: {
+          concurrency: 5,
+          limiter: {
+            max: 5,
+            // Max 5 jobs per...
+            duration: 6e4
+            // ...60 seconds (higher throughput for chunking)
+          }
         }
       }
     };
@@ -1328,6 +1424,7 @@ var _extractBlocksFromSomeBuilders = ({
 
 // src/node.ts
 var import_MongoConnector2 = __toESM(require_MongoConnector());
+var import_ElasticSearchConnector = __toESM(require_ElasticSearchConnector());
 init_getDbByTenant();
 
 // src/db/getGlobalConfig.ts
@@ -1472,6 +1569,7 @@ var import_BaseWorker = __toESM(require_BaseWorker());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
 var export_BaseProducer = import_BaseProducer.BaseProducer;
 var export_BaseWorker = import_BaseWorker.BaseWorker;
+var export_ElasticSearchConnector = import_ElasticSearchConnector.ElasticSearchConnector;
 var export_GET_GLOBAL_BULLMQ_CONFIG = import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG;
 var export_MongoConnector = import_MongoConnector2.MongoConnector;
 var export_WorkerManager = import_WorkerManager.WorkerManager;
@@ -1485,6 +1583,7 @@ export {
   Annotations_default as AnnotationSchema,
   export_BaseProducer as BaseProducer,
   export_BaseWorker as BaseWorker,
+  export_ElasticSearchConnector as ElasticSearchConnector,
   export_GET_GLOBAL_BULLMQ_CONFIG as GET_GLOBAL_BULLMQ_CONFIG,
   export_MongoConnector as MongoConnector,
   PlatformConfigs_default as PlatformConfigsSchema,

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.5.3",
+  "version": "1.5.5",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -74,6 +74,7 @@
     "typescript": "^5.8.2"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^8.2.1",
     "bullmq": "^5.58.2",
     "ioredis": "^5.6.1",
     "mongodb": "^6.13.0",