@okf/ootils 1.18.3 → 1.20.0

package/dist/browser.d.mts

@@ -723,4 +723,30 @@ interface GetPlatformContextContentParams {
 }
 declare const getPlatformContextContent: ({ platformConfigs_ai, }: GetPlatformContextContentParams) => string;
 
-export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+interface Author {
+    id: string | null;
+    name: string;
+}
+interface MergeParams {
+    prevObj: any;
+    newLexValue: any;
+    thisBlockValuePath: string;
+    author?: Author;
+}
+/**
+ * Merges annoData from a Lexical field into the consolidated annotations.tags structure.
+ *
+ * This function:
+ * 1. Removes all fragments from prevObj that belong to the given valuePath
+ * 2. Injects new fragments from annoData into the consolidated structure
+ * 3. Removes tags that have no fragments left
+ *
+ * @param prevObj - The existing consolidated annotations.tags object
+ * @param newLexValue - The new Lexical field value containing annoData
+ * @param thisBlockValuePath - The valuePath of the Lexical field being processed
+ * @param author - Optional author object. Defaults to { id: null, name: "AI" } for backend usage
+ * @returns The updated consolidated annotations.tags object
+ */
+declare const mergeAnnoDataIntoAnnotationsTags: ({ prevObj, newLexValue, thisBlockValuePath, author, }: MergeParams) => any;
+
+export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, mergeAnnoDataIntoAnnotationsTags, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };

package/dist/browser.d.ts

@@ -723,4 +723,30 @@ interface GetPlatformContextContentParams {
 }
 declare const getPlatformContextContent: ({ platformConfigs_ai, }: GetPlatformContextContentParams) => string;
 
-export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+interface Author {
+    id: string | null;
+    name: string;
+}
+interface MergeParams {
+    prevObj: any;
+    newLexValue: any;
+    thisBlockValuePath: string;
+    author?: Author;
+}
+/**
+ * Merges annoData from a Lexical field into the consolidated annotations.tags structure.
+ *
+ * This function:
+ * 1. Removes all fragments from prevObj that belong to the given valuePath
+ * 2. Injects new fragments from annoData into the consolidated structure
+ * 3. Removes tags that have no fragments left
+ *
+ * @param prevObj - The existing consolidated annotations.tags object
+ * @param newLexValue - The new Lexical field value containing annoData
+ * @param thisBlockValuePath - The valuePath of the Lexical field being processed
+ * @param author - Optional author object. Defaults to { id: null, name: "AI" } for backend usage
+ * @returns The updated consolidated annotations.tags object
+ */
+declare const mergeAnnoDataIntoAnnotationsTags: ({ prevObj, newLexValue, thisBlockValuePath, author, }: MergeParams) => any;
+
+export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, mergeAnnoDataIntoAnnotationsTags, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };

package/dist/browser.js

@@ -27,6 +27,7 @@ __export(browser_exports, {
   getPlatformContextContent: () => getPlatformContextContent,
   getRollupPossibilities: () => getRollupPossibilities,
   getVal: () => getVal,
+  mergeAnnoDataIntoAnnotationsTags: () => mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks: () => _recursExtractBlocks,
   segrigateDocs: () => segrigateDocs,
   setVal: () => setVal,
@@ -765,6 +766,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   }
   return "";
 };
+
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+var import_lodash = require("lodash");
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = (0, import_lodash.cloneDeep)(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BASE_BULLMQ_CONFIG,
@@ -774,6 +859,7 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   getPlatformContextContent,
   getRollupPossibilities,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/dist/browser.mjs

@@ -729,6 +729,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   }
   return "";
 };
+
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+import { cloneDeep } from "lodash";
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = cloneDeep(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
 export {
   BASE_BULLMQ_CONFIG,
   deleteVal,
@@ -737,6 +821,7 @@ export {
   getPlatformContextContent,
   getRollupPossibilities,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   _recursExtractBlocks as recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/dist/node.d.mts

@@ -731,6 +731,32 @@ interface GetPlatformContextContentParams {
 }
 declare const getPlatformContextContent: ({ platformConfigs_ai, }: GetPlatformContextContentParams) => string;
 
+interface Author {
+    id: string | null;
+    name: string;
+}
+interface MergeParams {
+    prevObj: any;
+    newLexValue: any;
+    thisBlockValuePath: string;
+    author?: Author;
+}
+/**
+ * Merges annoData from a Lexical field into the consolidated annotations.tags structure.
+ *
+ * This function:
+ * 1. Removes all fragments from prevObj that belong to the given valuePath
+ * 2. Injects new fragments from annoData into the consolidated structure
+ * 3. Removes tags that have no fragments left
+ *
+ * @param prevObj - The existing consolidated annotations.tags object
+ * @param newLexValue - The new Lexical field value containing annoData
+ * @param thisBlockValuePath - The valuePath of the Lexical field being processed
+ * @param author - Optional author object. Defaults to { id: null, name: "AI" } for backend usage
+ * @returns The updated consolidated annotations.tags object
+ */
+declare const mergeAnnoDataIntoAnnotationsTags: ({ prevObj, newLexValue, thisBlockValuePath, author, }: MergeParams) => any;
+
 declare class MongoConnector {
     static getInstance(): any;
     static getClusterConnections(): any;
@@ -1277,6 +1303,19 @@ declare class BaseWorker {
     stop(): Promise<void>;
 }
 
+/**
+ * Producer for Elasticsearch sync jobs for chunks
+ */
+declare class ChunksElasticSyncProducer extends BaseProducer {
+    constructor(options?: {});
+    /**
+     * Add a single sync job
+     * @param {Object} jobData - Job data containing docIds, tenant, operationType
+     * @returns {Promise<Object>} Job creation result
+     */
+    addJobWithName(jobData: Object): Promise<Object>;
+}
+
 /**
  * Creates BullMQ configuration with explicit dependencies
  * @param {Object} params Configuration parameters
@@ -1296,4 +1335,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BASE_BULLMQ_CONFIG, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, GeneratedTopicsSchema, MongoConnector, PlatformConfigsSchema, ProducerManager, RedisCacheConnector, TplSchema, WorkerManager, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAnnotationsModelByTenant, getDbByTenant, getGeneratedTopicsModelByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getPlatformContextContent, getRollupPossibilities, getTplModelByTenant, getVal, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+export { AIChatSchema, AnnotationSchema, BASE_BULLMQ_CONFIG, BaseProducer, BaseWorker, ChunksElasticSyncProducer, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, GeneratedTopicsSchema, MongoConnector, PlatformConfigsSchema, ProducerManager, RedisCacheConnector, TplSchema, WorkerManager, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAnnotationsModelByTenant, getDbByTenant, getGeneratedTopicsModelByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getPlatformContextContent, getRollupPossibilities, getTplModelByTenant, getVal, mergeAnnoDataIntoAnnotationsTags, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };

package/dist/node.d.ts

@@ -731,6 +731,32 @@ interface GetPlatformContextContentParams {
 }
 declare const getPlatformContextContent: ({ platformConfigs_ai, }: GetPlatformContextContentParams) => string;
 
+interface Author {
+    id: string | null;
+    name: string;
+}
+interface MergeParams {
+    prevObj: any;
+    newLexValue: any;
+    thisBlockValuePath: string;
+    author?: Author;
+}
+/**
+ * Merges annoData from a Lexical field into the consolidated annotations.tags structure.
+ *
+ * This function:
+ * 1. Removes all fragments from prevObj that belong to the given valuePath
+ * 2. Injects new fragments from annoData into the consolidated structure
+ * 3. Removes tags that have no fragments left
+ *
+ * @param prevObj - The existing consolidated annotations.tags object
+ * @param newLexValue - The new Lexical field value containing annoData
+ * @param thisBlockValuePath - The valuePath of the Lexical field being processed
+ * @param author - Optional author object. Defaults to { id: null, name: "AI" } for backend usage
+ * @returns The updated consolidated annotations.tags object
+ */
+declare const mergeAnnoDataIntoAnnotationsTags: ({ prevObj, newLexValue, thisBlockValuePath, author, }: MergeParams) => any;
+
 declare class MongoConnector {
     static getInstance(): any;
     static getClusterConnections(): any;
@@ -1277,6 +1303,19 @@ declare class BaseWorker {
     stop(): Promise<void>;
 }
 
+/**
+ * Producer for Elasticsearch sync jobs for chunks
+ */
+declare class ChunksElasticSyncProducer extends BaseProducer {
+    constructor(options?: {});
+    /**
+     * Add a single sync job
+     * @param {Object} jobData - Job data containing docIds, tenant, operationType
+     * @returns {Promise<Object>} Job creation result
+     */
+    addJobWithName(jobData: Object): Promise<Object>;
+}
+
 /**
  * Creates BullMQ configuration with explicit dependencies
  * @param {Object} params Configuration parameters
@@ -1296,4 +1335,4 @@ declare function GET_GLOBAL_BULLMQ_CONFIG({ env, redisCredentials }: {
     };
 }): Object;
 
-export { AIChatSchema, AnnotationSchema, BASE_BULLMQ_CONFIG, BaseProducer, BaseWorker, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, GeneratedTopicsSchema, MongoConnector, PlatformConfigsSchema, ProducerManager, RedisCacheConnector, TplSchema, WorkerManager, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAnnotationsModelByTenant, getDbByTenant, getGeneratedTopicsModelByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getPlatformContextContent, getRollupPossibilities, getTplModelByTenant, getVal, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+export { AIChatSchema, AnnotationSchema, BASE_BULLMQ_CONFIG, BaseProducer, BaseWorker, ChunksElasticSyncProducer, ElasticSearchConnector, GET_GLOBAL_BULLMQ_CONFIG, GeneratedTopicsSchema, MongoConnector, PlatformConfigsSchema, ProducerManager, RedisCacheConnector, TplSchema, WorkerManager, deleteVal, extractAllBlocksFromTpl, genTagId, getAIChatModelByTenant, getAnnotationsModelByTenant, getDbByTenant, getGeneratedTopicsModelByTenant, getModelByTenant, getPlatformConfigsModelByTenant, getPlatformContextContent, getRollupPossibilities, getTplModelByTenant, getVal, mergeAnnoDataIntoAnnotationsTags, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };

package/dist/node.js

@@ -1552,6 +1552,48 @@ var require_GET_GLOBAL_BULLMQ_CONFIG = __commonJS({
   }
 });
 
+// src/bullmq/ChunksElasticSyncProducer.js
+var require_ChunksElasticSyncProducer = __commonJS({
+  "src/bullmq/ChunksElasticSyncProducer.js"(exports2, module2) {
+    "use strict";
+    var { BaseProducer: BaseProducer2 } = require_BaseProducer();
+    var { GET_GLOBAL_BULLMQ_CONFIG: GET_GLOBAL_BULLMQ_CONFIG2 } = require_GET_GLOBAL_BULLMQ_CONFIG();
+    var ChunksElasticSyncProducer2 = class extends BaseProducer2 {
+      // Preserve class name through minification for ProducerManager lookup
+      static get name() {
+        return "ChunksElasticSyncProducer";
+      }
+      // options - Configuration options (optional, defaults to process.env)
+      constructor(options = {}) {
+        const env = options.env || process.env.ENV;
+        const redisCredentials = options.redisCredentials || {
+          REDIS_HOST: process.env.REDIS_HOST,
+          REDIS_PORT: process.env.REDIS_PORT,
+          REDIS_PASSWORD: process.env.REDIS_PASSWORD
+        };
+        const GLOBAL_BULLMQ_CONFIG = GET_GLOBAL_BULLMQ_CONFIG2({
+          env,
+          redisCredentials
+        });
+        super(GLOBAL_BULLMQ_CONFIG.CHUNKS_ELASTIC_SYNC_QUEUE);
+      }
+      /**
+       * Add a single sync job
+       * @param {Object} jobData - Job data containing docIds, tenant, operationType
+       * @returns {Promise<Object>} Job creation result
+       */
+      async addJobWithName(jobData) {
+        const job = await this.addJob("processChunksElasticSync", jobData);
+        console.log(`\u{1F4C8} Added process chunks elastic sync job: ${job.id}`);
+        return { jobId: job.id };
+      }
+    };
+    module2.exports = {
+      ChunksElasticSyncProducer: ChunksElasticSyncProducer2
+    };
+  }
+});
+
 // src/node.ts
 var node_exports = {};
 __export(node_exports, {
@@ -1560,6 +1602,7 @@ __export(node_exports, {
   BASE_BULLMQ_CONFIG: () => BASE_BULLMQ_CONFIG,
   BaseProducer: () => import_BaseProducer.BaseProducer,
   BaseWorker: () => import_BaseWorker.BaseWorker,
+  ChunksElasticSyncProducer: () => import_ChunksElasticSyncProducer.ChunksElasticSyncProducer,
   ElasticSearchConnector: () => import_ElasticSearchConnector.ElasticSearchConnector,
   GET_GLOBAL_BULLMQ_CONFIG: () => import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG,
   GeneratedTopicsSchema: () => GeneratedTopics_default,
@@ -1582,6 +1625,7 @@ __export(node_exports, {
   getRollupPossibilities: () => getRollupPossibilities,
   getTplModelByTenant: () => import_getModelByTenant2.getTplModelByTenant,
   getVal: () => getVal,
+  mergeAnnoDataIntoAnnotationsTags: () => mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks: () => _recursExtractBlocks,
   segrigateDocs: () => segrigateDocs,
   setVal: () => setVal,
@@ -2021,6 +2065,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   return "";
 };
 
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+var import_lodash = require("lodash");
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = (0, import_lodash.cloneDeep)(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
+
 // src/node.ts
 var import_MongoConnector2 = __toESM(require_MongoConnector());
 var import_ElasticSearchConnector = __toESM(require_ElasticSearchConnector());
@@ -2353,6 +2481,7 @@ var import_WorkerManager = __toESM(require_WorkerManager());
 var import_ProducerManager = __toESM(require_ProducerManager());
 var import_BaseProducer = __toESM(require_BaseProducer());
 var import_BaseWorker = __toESM(require_BaseWorker());
+var import_ChunksElasticSyncProducer = __toESM(require_ChunksElasticSyncProducer());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
@@ -2361,6 +2490,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   BASE_BULLMQ_CONFIG,
   BaseProducer,
   BaseWorker,
+  ChunksElasticSyncProducer,
   ElasticSearchConnector,
   GET_GLOBAL_BULLMQ_CONFIG,
   GeneratedTopicsSchema,
@@ -2383,6 +2513,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   getRollupPossibilities,
   getTplModelByTenant,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/dist/node.mjs

@@ -1557,6 +1557,48 @@ var require_GET_GLOBAL_BULLMQ_CONFIG = __commonJS({
   }
 });
 
+// src/bullmq/ChunksElasticSyncProducer.js
+var require_ChunksElasticSyncProducer = __commonJS({
+  "src/bullmq/ChunksElasticSyncProducer.js"(exports, module) {
+    "use strict";
+    var { BaseProducer: BaseProducer2 } = require_BaseProducer();
+    var { GET_GLOBAL_BULLMQ_CONFIG: GET_GLOBAL_BULLMQ_CONFIG2 } = require_GET_GLOBAL_BULLMQ_CONFIG();
+    var ChunksElasticSyncProducer2 = class extends BaseProducer2 {
+      // Preserve class name through minification for ProducerManager lookup
+      static get name() {
+        return "ChunksElasticSyncProducer";
+      }
+      // options - Configuration options (optional, defaults to process.env)
+      constructor(options = {}) {
+        const env = options.env || process.env.ENV;
+        const redisCredentials = options.redisCredentials || {
+          REDIS_HOST: process.env.REDIS_HOST,
+          REDIS_PORT: process.env.REDIS_PORT,
+          REDIS_PASSWORD: process.env.REDIS_PASSWORD
+        };
+        const GLOBAL_BULLMQ_CONFIG = GET_GLOBAL_BULLMQ_CONFIG2({
+          env,
+          redisCredentials
+        });
+        super(GLOBAL_BULLMQ_CONFIG.CHUNKS_ELASTIC_SYNC_QUEUE);
+      }
+      /**
+       * Add a single sync job
+       * @param {Object} jobData - Job data containing docIds, tenant, operationType
+       * @returns {Promise<Object>} Job creation result
+       */
+      async addJobWithName(jobData) {
+        const job = await this.addJob("processChunksElasticSync", jobData);
+        console.log(`\u{1F4C8} Added process chunks elastic sync job: ${job.id}`);
+        return { jobId: job.id };
+      }
+    };
+    module.exports = {
+      ChunksElasticSyncProducer: ChunksElasticSyncProducer2
+    };
+  }
+});
+
 // src/utils/getterSetterDeleter/utils/set_deleteVal.ts
 var set_deleteVal = (action, data, valuePath, value) => {
   if (valuePath === void 0) return;
@@ -1989,6 +2031,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   return "";
 };
 
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+import { cloneDeep } from "lodash";
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = cloneDeep(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
+
 // src/node.ts
 var import_MongoConnector2 = __toESM(require_MongoConnector());
 var import_ElasticSearchConnector = __toESM(require_ElasticSearchConnector());
@@ -2321,9 +2447,11 @@ var import_WorkerManager = __toESM(require_WorkerManager());
 var import_ProducerManager = __toESM(require_ProducerManager());
 var import_BaseProducer = __toESM(require_BaseProducer());
 var import_BaseWorker = __toESM(require_BaseWorker());
+var import_ChunksElasticSyncProducer = __toESM(require_ChunksElasticSyncProducer());
 var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG());
 var export_BaseProducer = import_BaseProducer.BaseProducer;
 var export_BaseWorker = import_BaseWorker.BaseWorker;
+var export_ChunksElasticSyncProducer = import_ChunksElasticSyncProducer.ChunksElasticSyncProducer;
 var export_ElasticSearchConnector = import_ElasticSearchConnector.ElasticSearchConnector;
 var export_GET_GLOBAL_BULLMQ_CONFIG = import_GET_GLOBAL_BULLMQ_CONFIG.GET_GLOBAL_BULLMQ_CONFIG;
 var export_MongoConnector = import_MongoConnector2.MongoConnector;
@@ -2341,6 +2469,7 @@ export {
   BASE_BULLMQ_CONFIG,
   export_BaseProducer as BaseProducer,
   export_BaseWorker as BaseWorker,
+  export_ChunksElasticSyncProducer as ChunksElasticSyncProducer,
   export_ElasticSearchConnector as ElasticSearchConnector,
   export_GET_GLOBAL_BULLMQ_CONFIG as GET_GLOBAL_BULLMQ_CONFIG,
   GeneratedTopics_default as GeneratedTopicsSchema,
@@ -2363,6 +2492,7 @@ export {
   getRollupPossibilities,
   export_getTplModelByTenant as getTplModelByTenant,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   _recursExtractBlocks as recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/dist/universal.d.mts

@@ -723,4 +723,30 @@ interface GetPlatformContextContentParams {
 }
 declare const getPlatformContextContent: ({ platformConfigs_ai, }: GetPlatformContextContentParams) => string;
 
-export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+interface Author {
+    id: string | null;
+    name: string;
+}
+interface MergeParams {
+    prevObj: any;
+    newLexValue: any;
+    thisBlockValuePath: string;
+    author?: Author;
+}
+/**
+ * Merges annoData from a Lexical field into the consolidated annotations.tags structure.
+ *
+ * This function:
+ * 1. Removes all fragments from prevObj that belong to the given valuePath
+ * 2. Injects new fragments from annoData into the consolidated structure
+ * 3. Removes tags that have no fragments left
+ *
+ * @param prevObj - The existing consolidated annotations.tags object
+ * @param newLexValue - The new Lexical field value containing annoData
+ * @param thisBlockValuePath - The valuePath of the Lexical field being processed
+ * @param author - Optional author object. Defaults to { id: null, name: "AI" } for backend usage
+ * @returns The updated consolidated annotations.tags object
+ */
+declare const mergeAnnoDataIntoAnnotationsTags: ({ prevObj, newLexValue, thisBlockValuePath, author, }: MergeParams) => any;
+
+export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, mergeAnnoDataIntoAnnotationsTags, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };

package/dist/universal.d.ts

@@ -723,4 +723,30 @@ interface GetPlatformContextContentParams {
 }
 declare const getPlatformContextContent: ({ platformConfigs_ai, }: GetPlatformContextContentParams) => string;
 
-export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+interface Author {
+    id: string | null;
+    name: string;
+}
+interface MergeParams {
+    prevObj: any;
+    newLexValue: any;
+    thisBlockValuePath: string;
+    author?: Author;
+}
+/**
+ * Merges annoData from a Lexical field into the consolidated annotations.tags structure.
+ *
+ * This function:
+ * 1. Removes all fragments from prevObj that belong to the given valuePath
+ * 2. Injects new fragments from annoData into the consolidated structure
+ * 3. Removes tags that have no fragments left
+ *
+ * @param prevObj - The existing consolidated annotations.tags object
+ * @param newLexValue - The new Lexical field value containing annoData
+ * @param thisBlockValuePath - The valuePath of the Lexical field being processed
+ * @param author - Optional author object. Defaults to { id: null, name: "AI" } for backend usage
+ * @returns The updated consolidated annotations.tags object
+ */
+declare const mergeAnnoDataIntoAnnotationsTags: ({ prevObj, newLexValue, thisBlockValuePath, author, }: MergeParams) => any;
+
+export { BASE_BULLMQ_CONFIG, deleteVal, extractAllBlocksFromTpl, genTagId, getPlatformContextContent, getRollupPossibilities, getVal, mergeAnnoDataIntoAnnotationsTags, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };

package/dist/universal.js

@@ -27,6 +27,7 @@ __export(universal_exports, {
   getPlatformContextContent: () => getPlatformContextContent,
   getRollupPossibilities: () => getRollupPossibilities,
   getVal: () => getVal,
+  mergeAnnoDataIntoAnnotationsTags: () => mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks: () => _recursExtractBlocks,
   segrigateDocs: () => segrigateDocs,
   setVal: () => setVal,
@@ -765,6 +766,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   }
   return "";
 };
+
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+var import_lodash = require("lodash");
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = (0, import_lodash.cloneDeep)(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BASE_BULLMQ_CONFIG,
@@ -774,6 +859,7 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   getPlatformContextContent,
   getRollupPossibilities,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/dist/universal.mjs

@@ -729,6 +729,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   }
   return "";
 };
+
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+import { cloneDeep } from "lodash";
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = cloneDeep(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
 export {
   BASE_BULLMQ_CONFIG,
   deleteVal,
@@ -737,6 +821,7 @@ export {
   getPlatformContextContent,
   getRollupPossibilities,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   _recursExtractBlocks as recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.18.3",
+  "version": "1.20.0",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -65,6 +65,7 @@
     "@semantic-release/npm": "^13.1.3",
     "@semantic-release/release-notes-generator": "^14.0.3",
     "@types/ioredis": "^4.28.10",
+    "@types/lodash": "^4.17.23",
     "@types/mongoose": "^5.11.97",
     "@types/node": "^22.13.10",
     "commitizen": "^4.3.1",
@@ -77,6 +78,7 @@
     "@elastic/elasticsearch": "^8.2.1",
     "bullmq": "^5.58.2",
     "ioredis": "^5.6.1",
+    "lodash": "^4.17.23",
     "mongodb": "^6.13.0",
     "mongoose": "^8.15.1"
   }