@okf/ootils 1.26.1 → 1.26.3

package/dist/node.js

@@ -218,6 +218,34 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         }
       },
+      SENTIMENT_AND_NER_QUEUE: {
+        id: "sentiment-and-ner-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            attempts: 3,
+            backoff: {
+              type: "exponential",
+              delay: 2e3
+            },
+            removeOnComplete: 30,
+            removeOnFail: 100
+          },
+          streams: {
+            events: {
+              maxLen: 10
+            }
+          }
+        },
+        workerConfig: {
+          concurrency: 1,
+          lockDuration: 9e4,
+          maxStalledCount: 3,
+          limiter: {
+            max: 100,
+            duration: 6e4
+          }
+        }
+      },
       AI_AUTO_ANNOTATE_QUEUE: {
         id: "ai-auto-annotate-queue",
         queueConfig: {
@@ -874,31 +902,54 @@ var require_ElasticSearchConnector = __commonJS({
     "use strict";
     var { Client } = require("@elastic/elasticsearch");
     var ElasticSearchConnector2 = class _ElasticSearchConnector {
+      /**
+       * @param {Object} options
+       *
+       * Multi-env mode (preferred):
+       *   { env: 'dev', configs: { dev: { cloudId, apiKey }, staging: { cloudId, apiKey }, prod: { cloudId, apiKey } } }
+       *
+       * Legacy single-env mode (backwards compat):
+       *   { cloudId: '...', apiKey: '...' }
+       */
       constructor(options = {}) {
+        this.env = options.env || null;
+        this.clients = {};
         this.client = null;
-        this.cloudId = options.cloudId;
-        this.apiKey = options.apiKey;
-        _ElasticSearchConnector.instance = this;
-        if (!this.cloudId) {
-          throw new Error("Cloud ID must be provided in constructor options or ELASTIC_CLOUD_ID environment variable");
-        }
-        if (!this.apiKey) {
-          throw new Error("API Key must be provided in constructor options or ELASTIC_API_KEY environment variable");
+        if (options.configs) {
+          this._configs = options.configs;
+        } else if (options.cloudId && options.apiKey) {
+          this._legacyConfig = { cloudId: options.cloudId, apiKey: options.apiKey };
+        } else {
+          throw new Error("ElasticSearchConnector: provide either { configs } for multi-env or { cloudId, apiKey } for single-env");
         }
+        _ElasticSearchConnector.instance = this;
       }
       /**
-       * Initialize Elasticsearch client
+       * Initialize Elasticsearch client(s).
+       * Multi-env: connects to all configured envs.
+       * Legacy: connects a single client.
        */
       connect() {
-        this.client = new Client({
-          cloud: {
-            id: this.cloudId
-          },
-          auth: {
-            apiKey: this.apiKey
+        if (this._configs) {
+          for (const [env, conf] of Object.entries(this._configs)) {
+            if (!conf.cloudId || !conf.apiKey) {
+              console.log(`[ES] Skipping env "${env}" \u2014 missing cloudId or apiKey`);
+              continue;
+            }
+            this.clients[env] = new Client({
+              cloud: { id: conf.cloudId },
+              auth: { apiKey: conf.apiKey }
+            });
           }
-        });
-        console.log("\u2705 Elasticsearch client initialized");
+          this.client = this.clients[this.env] || null;
+          console.log(`\u2705 Elasticsearch clients initialized for envs: ${Object.keys(this.clients).join(", ")}`);
+        } else {
+          this.client = new Client({
+            cloud: { id: this._legacyConfig.cloudId },
+            auth: { apiKey: this._legacyConfig.apiKey }
+          });
+          console.log("\u2705 Elasticsearch client initialized");
+        }
         return this.client;
       }
       // Static method to get the instance
@@ -908,22 +959,43 @@ var require_ElasticSearchConnector = __commonJS({
         }
         return _ElasticSearchConnector.instance;
       }
-      // Static method to get the client
-      static getClient() {
-        if (!_ElasticSearchConnector.instance || !_ElasticSearchConnector.instance.client) {
-          throw new Error("ElasticSearchConnector not initialized or client not connected");
+      /**
+       * Get an Elasticsearch client.
+       * @param {string} [env] - Optional env to get a specific env's client.
+       *                         If omitted, returns the current env's client (backwards compat).
+       */
+      static getClient(env) {
+        const inst = _ElasticSearchConnector.instance;
+        if (!inst) {
+          throw new Error("ElasticSearchConnector not initialized");
         }
-        return _ElasticSearchConnector.instance.client;
+        if (env && inst.clients[env]) {
+          return inst.clients[env];
+        }
+        if (inst.client) {
+          return inst.client;
+        }
+        throw new Error("ElasticSearchConnector not initialized or client not connected");
+      }
+      static getEnv() {
+        return _ElasticSearchConnector.instance?.env;
       }
       /**
-       * Close the Elasticsearch client
+       * Close all Elasticsearch clients.
        */
       async close() {
-        if (this.client) {
+        for (const [env, client] of Object.entries(this.clients)) {
+          try {
+            await client.close();
+          } catch (_) {
+          }
+        }
+        if (this.client && !Object.values(this.clients).includes(this.client)) {
           await this.client.close();
-          console.log("\u2705 Elasticsearch client closed");
-          this.client = null;
         }
+        this.clients = {};
+        this.client = null;
+        console.log("\u2705 Elasticsearch client(s) closed");
       }
     };
     ElasticSearchConnector2.instance = null;
@@ -1066,11 +1138,55 @@ var init_GeneratedTopics = __esm({
   }
 });
 
+// src/models/GeneratedEntities.ts
+var import_mongoose6, GeneratedEntitiesSchema, GeneratedEntities_default;
+var init_GeneratedEntities = __esm({
+  "src/models/GeneratedEntities.ts"() {
+    "use strict";
+    import_mongoose6 = require("mongoose");
+    GeneratedEntitiesSchema = new import_mongoose6.Schema(
+      {
+        canonicalName: {
+          type: String,
+          trim: true,
+          required: true
+        },
+        type: {
+          type: String,
+          required: true,
+          trim: true
+        },
+        embeddings: {
+          type: [Number],
+          required: true
+        },
+        aliases: {
+          type: [String],
+          default: []
+        }
+      },
+      { timestamps: true, collection: "SYSTEM_generatedEntities" }
+    );
+    GeneratedEntitiesSchema.index(
+      {
+        canonicalName: 1,
+        type: 1
+      },
+      {
+        unique: true,
+        name: "unique_entity_per_type"
+      }
+    );
+    GeneratedEntities_default = GeneratedEntitiesSchema;
+  }
+});
+
 // src/models/index.ts
 var models_exports = {};
 __export(models_exports, {
   AIChatSchema: () => AIChat_default,
   AnnotationSchema: () => Annotations_default,
+  GeneratedEntitiesSchema: () => GeneratedEntities_default,
   GeneratedTopicsSchema: () => GeneratedTopics_default,
   PlatformConfigsSchema: () => PlatformConfigs_default,
   TplSchema: () => Tpl_default
@@ -1083,6 +1199,7 @@ var init_models = __esm({
     init_PlatformConfigs();
     init_Tpl();
     init_GeneratedTopics();
+    init_GeneratedEntities();
   }
 });
 
@@ -1091,11 +1208,11 @@ var getDbByTenant_exports = {};
 __export(getDbByTenant_exports, {
   getDbByTenant: () => getDbByTenant
 });
-var import_mongoose6, import_MongoConnector, getDbByTenant;
+var import_mongoose7, import_MongoConnector, getDbByTenant;
 var init_getDbByTenant = __esm({
   "src/db/getDbByTenant.js"() {
     "use strict";
-    import_mongoose6 = require("mongoose");
+    import_mongoose7 = require("mongoose");
     import_MongoConnector = __toESM(require_MongoConnector());
     getDbByTenant = ({
       tenant,
@@ -1123,7 +1240,7 @@ var require_getModelByTenant = __commonJS({
   "src/db/getModelByTenant.js"(exports2, module2) {
     "use strict";
     var { getDbByTenant: getDbByTenant2 } = (init_getDbByTenant(), __toCommonJS(getDbByTenant_exports));
-    var { AIChatSchema: AIChatSchema2, AnnotationSchema: AnnotationSchema2, PlatformConfigsSchema: PlatformConfigsSchema2, TplSchema: TplSchema2, GeneratedTopicsSchema: GeneratedTopicsSchema2 } = (init_models(), __toCommonJS(models_exports));
+    var { AIChatSchema: AIChatSchema2, AnnotationSchema: AnnotationSchema2, PlatformConfigsSchema: PlatformConfigsSchema2, TplSchema: TplSchema2, GeneratedTopicsSchema: GeneratedTopicsSchema2, GeneratedEntitiesSchema: GeneratedEntitiesSchema2 } = (init_models(), __toCommonJS(models_exports));
     var getModelByTenant3 = ({ tenant, modelName, schema, env }) => {
       if (!tenant) {
         throw new Error("tenant id has not been provided");
@@ -1164,13 +1281,20 @@ var require_getModelByTenant = __commonJS({
       schema: GeneratedTopicsSchema2,
       env
     });
+    var getGeneratedEntitiesModelByTenant2 = ({ tenant, env }) => getModelByTenant3({
+      tenant,
+      modelName: "generatedEntities",
+      schema: GeneratedEntitiesSchema2,
+      env
+    });
     module2.exports = {
       getModelByTenant: getModelByTenant3,
       getAIChatModelByTenant: getAIChatModelByTenant2,
       getAnnotationsModelByTenant: getAnnotationsModelByTenant2,
       getPlatformConfigsModelByTenant: getPlatformConfigsModelByTenant2,
       getTplModelByTenant: getTplModelByTenant2,
-      getGeneratedTopicsModelByTenant: getGeneratedTopicsModelByTenant2
+      getGeneratedTopicsModelByTenant: getGeneratedTopicsModelByTenant2,
+      getGeneratedEntitiesModelByTenant: getGeneratedEntitiesModelByTenant2
     };
   }
 });
@@ -1235,6 +1359,36 @@ var init_SecretManagerConnector = __esm({
         const configs = _SecretManagerConnector.getSecret("envSpecificConfigs");
         return env ? configs[env] : configs;
       }
+      /**
+       * Get Redis configs for all envs, structured for RedisCacheConnector multi-env mode.
+       * @returns {{ dev: { host, port, password }, staging: { host, port, password }, prod: { host, port, password } }}
+       */
+      static getRedisConfigsForAllEnvs() {
+        const all = _SecretManagerConnector.getEnvSpecificConfigs();
+        const result = {};
+        for (const env of Object.keys(all)) {
+          const c = all[env];
+          if (c?.REDIS_HOST) {
+            result[env] = { host: c.REDIS_HOST, port: c.REDIS_PORT, password: c.REDIS_PASSWORD };
+          }
+        }
+        return result;
+      }
+      /**
+       * Get Elasticsearch configs for all envs, structured for ElasticSearchConnector multi-env mode.
+       * @returns {{ dev: { cloudId, apiKey }, staging: { cloudId, apiKey }, prod: { cloudId, apiKey } }}
+       */
+      static getElasticConfigsForAllEnvs() {
+        const all = _SecretManagerConnector.getEnvSpecificConfigs();
+        const result = {};
+        for (const env of Object.keys(all)) {
+          const c = all[env];
+          if (c?.ELASTIC_CLOUD_ID) {
+            result[env] = { cloudId: c.ELASTIC_CLOUD_ID, apiKey: c.ELASTIC_API_KEY };
+          }
+        }
+        return result;
+      }
       static async addSecretVersion(secretName, data) {
         const instance = _SecretManagerConnector.instance;
         const payload = Buffer.from(JSON.stringify(data, null, 2));
@@ -1762,6 +1916,7 @@ __export(node_exports, {
   ElasticSearchConnector: () => import_ElasticSearchConnector.ElasticSearchConnector,
   FILTER_IDS: () => FILTER_IDS,
   GET_GLOBAL_BULLMQ_CONFIG: () => import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG,
+  GeneratedEntitiesSchema: () => GeneratedEntities_default,
   GeneratedTopicsSchema: () => GeneratedTopics_default,
   MongoConnector: () => import_MongoConnector3.MongoConnector,
   PlatformConfigsSchema: () => PlatformConfigs_default,
@@ -1787,6 +1942,7 @@ __export(node_exports, {
   getAnnotationsModelByTenant: () => import_getModelByTenant2.getAnnotationsModelByTenant,
   getDbByTenant: () => getDbByTenant,
   getFilterKeyForBlock: () => getFilterKeyForBlock,
+  getGeneratedEntitiesModelByTenant: () => import_getModelByTenant2.getGeneratedEntitiesModelByTenant,
   getGeneratedTopicsModelByTenant: () => import_getModelByTenant2.getGeneratedTopicsModelByTenant,
   getModelByTenant: () => import_getModelByTenant2.getModelByTenant,
   getPlatformConfigsModelByTenant: () => import_getModelByTenant2.getPlatformConfigsModelByTenant,
@@ -3324,24 +3480,56 @@ var BASE_SETTINGS_FOR_CONFIGS_CACHE = [
   }
 ];
 var RedisCacheConnector = class _RedisCacheConnector {
+  /**
+   * @param {Object} options
+   *
+   * Multi-env mode (preferred):
+   *   { env: 'dev', configs: { dev: { host, port, password }, staging: { host, port, password }, prod: { host, port, password } } }
+   *
+   * Legacy single-env mode (backwards compat):
+   *   { host: '...', port: 6379, password: '...', env: 'dev' }
+   */
   constructor(options = {}) {
     /**
-     * Initialize Redis client
+     * Initialize Redis client(s).
+     * Multi-env: connects to all configured envs.
+     * Legacy: connects a single client.
      */
     __publicField(this, "connect", async () => {
       try {
-        this.client = new import_ioredis.default({
-          host: this.host,
-          port: this.port,
-          password: this.password
-        });
-        this.client.on("connect", () => {
-          console.log("\u2705 Connected to our redis cache instance!");
-        });
-        await new Promise((resolve, reject) => {
-          this.client.on("ready", resolve);
-          this.client.on("error", reject);
-        });
+        if (this._configs) {
+          for (const [env, conf] of Object.entries(this._configs)) {
+            if (!conf.host || !conf.port || !conf.password) {
+              console.log(`[Redis] Skipping env "${env}" \u2014 missing config`);
+              continue;
+            }
+            const redisClient = new import_ioredis.default({
+              host: conf.host,
+              port: conf.port,
+              password: conf.password
+            });
+            await new Promise((resolve, reject) => {
+              redisClient.on("ready", resolve);
+              redisClient.on("error", reject);
+            });
+            this.clients[env] = redisClient;
+          }
+          this.client = this.clients[this.env] || null;
+          console.log(`\u2705 Redis clients initialized for envs: ${Object.keys(this.clients).join(", ")}`);
+        } else {
+          this.client = new import_ioredis.default({
+            host: this._legacyConfig.host,
+            port: this._legacyConfig.port,
+            password: this._legacyConfig.password
+          });
+          this.client.on("connect", () => {
+            console.log("\u2705 Connected to our redis cache instance!");
+          });
+          await new Promise((resolve, reject) => {
+            this.client.on("ready", resolve);
+            this.client.on("error", reject);
+          });
+        }
         return this.client;
       } catch (err) {
         console.error(err.message);
@@ -3353,24 +3541,20 @@ var RedisCacheConnector = class _RedisCacheConnector {
         "RedisCacheConnector instance already exists. Use RedisCacheConnector.getInstance() instead."
       );
     }
-    this.client = null;
-    this.host = options.host;
     this.env = options.env;
-    this.port = options.port;
-    this.password = options.password;
-    _RedisCacheConnector.instance = this;
-    if (!this.host) {
-      throw new Error("Host must be provided in constructor options");
-    }
-    if (!this.port) {
-      throw new Error("Port must be provided in constructor options");
-    }
-    if (!this.password) {
-      throw new Error("Password must be provided in constructor options");
+    this.clients = {};
+    this.client = null;
+    if (options.configs) {
+      this._configs = options.configs;
+    } else if (options.host && options.port && options.password) {
+      this._legacyConfig = { host: options.host, port: options.port, password: options.password };
+    } else {
+      throw new Error("RedisCacheConnector: provide either { configs } for multi-env or { host, port, password } for single-env");
     }
     if (!this.env) {
       throw new Error("Env must be provided in constructor options");
     }
+    _RedisCacheConnector.instance = this;
   }
   static getEnv() {
     if (!_RedisCacheConnector.instance) {
@@ -3385,14 +3569,23 @@ var RedisCacheConnector = class _RedisCacheConnector {
     }
     return _RedisCacheConnector.instance;
   }
-  // Static method to get the client
-  static getClient() {
-    if (!_RedisCacheConnector.instance || !_RedisCacheConnector.instance.client) {
-      throw new Error(
-        "RedisCacheConnector not initialized or client not connected"
-      );
+  /**
+   * Get a Redis client.
+   * @param {string} [env] - Optional env to get a specific env's client.
+   *                         If omitted, returns the current env's client (backwards compat).
+   */
+  static getClient(env) {
+    const inst = _RedisCacheConnector.instance;
+    if (!inst) {
+      throw new Error("RedisCacheConnector not initialized or client not connected");
+    }
+    if (env && inst.clients[env]) {
+      return inst.clients[env];
     }
-    return _RedisCacheConnector.instance.client;
+    if (inst.client) {
+      return inst.client;
+    }
+    throw new Error("RedisCacheConnector not initialized or client not connected");
   }
   /**
    * Find documents in Redis cache
@@ -3406,8 +3599,8 @@ var RedisCacheConnector = class _RedisCacheConnector {
    */
   static async findConfigsInCache({ tenant, modelName, type, env }) {
     try {
-      const client = this.getClient();
       const environment = env || this.getEnv();
+      const client = this.getClient(environment);
       const keyPrefix = `${environment}:${tenant}:${modelName}:`;
       const keys = await client.keys(`${keyPrefix}${type || "*"}`);
       if (keys.length === 0) {
@@ -3472,12 +3665,6 @@ var RedisCacheConnector = class _RedisCacheConnector {
   /**
    * Generate Redis key for config caching
    * @private
-   * @param {Object} params - Parameters for key generation
-   * @param {string} params.tenant - The tenant identifier
-   * @param {'platformConfigs'|'tpl'} params.modelName - The model/collection name
-   * @param {string} params.type - The document type
-   * @param {string} [params.env] - Environment (defaults to current env)
-   * @returns {string} The Redis key
    */
   static _getRedisKeyForConfig({ tenant, modelName, type, env }) {
     const environment = env || this.getEnv();
@@ -3495,12 +3682,13 @@ var RedisCacheConnector = class _RedisCacheConnector {
    * @see {platformConfigTypes} for valid platformConfigs type values
    */
   static async setOneConfigInCache({ tenant, modelName, type, val, env }) {
-    const client = this.getClient();
+    const environment = env || this.getEnv();
+    const client = this.getClient(environment);
     const cacheKey = this._getRedisKeyForConfig({
       tenant,
       modelName,
       type,
-      env
+      env: environment
     });
     await client.set(cacheKey, JSON.stringify(val));
   }
@@ -3526,12 +3714,13 @@ var RedisCacheConnector = class _RedisCacheConnector {
     newVal,
     env
   }) {
-    const client = this.getClient();
+    const environment = env || this.getEnv();
+    const client = this.getClient(environment);
     const cacheKey = this._getRedisKeyForConfig({
       tenant,
       modelName,
       type,
-      env
+      env: environment
     });
     const delResult = await client.del(cacheKey);
     let setResult;
@@ -3556,8 +3745,8 @@ var RedisCacheConnector = class _RedisCacheConnector {
    */
   static async loadTplsAndPlatformConfigsToCache({ tenant }) {
     try {
-      const client = this.getClient();
       const environment = this.getEnv();
+      const client = this.getClient(environment);
       if (!tenant) throw new Error("No tenant/s defined to recache");
       console.log(`Loading configs and templates into cache...`);
       const tenantToClusterMapping = import_MongoConnector2.MongoConnector.getTenantToClusterMapping();
@@ -3607,14 +3796,21 @@ var RedisCacheConnector = class _RedisCacheConnector {
     }
   }
   /**
-   * Close the Redis client
+   * Close all Redis clients.
    */
   async close() {
-    if (this.client) {
+    for (const [env, client] of Object.entries(this.clients)) {
+      try {
+        await client.quit();
+      } catch (_) {
+      }
+    }
+    if (this.client && !Object.values(this.clients).includes(this.client)) {
       await this.client.quit();
-      console.log("\u2705 Redis client closed");
-      this.client = null;
     }
+    this.clients = {};
+    this.client = null;
+    console.log("\u2705 Redis client closed");
   }
 };
 RedisCacheConnector.instance = null;
@@ -3643,6 +3839,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   ElasticSearchConnector,
   FILTER_IDS,
   GET_GLOBAL_BULLMQ_CONFIG,
+  GeneratedEntitiesSchema,
   GeneratedTopicsSchema,
   MongoConnector,
   PlatformConfigsSchema,
@@ -3668,6 +3865,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   getAnnotationsModelByTenant,
   getDbByTenant,
   getFilterKeyForBlock,
+  getGeneratedEntitiesModelByTenant,
   getGeneratedTopicsModelByTenant,
   getModelByTenant,
   getPlatformConfigsModelByTenant,