@okf/ootils 1.18.2 → 1.19.0

package/dist/universal.js

@@ -27,6 +27,7 @@ __export(universal_exports, {
   getPlatformContextContent: () => getPlatformContextContent,
   getRollupPossibilities: () => getRollupPossibilities,
   getVal: () => getVal,
+  mergeAnnoDataIntoAnnotationsTags: () => mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks: () => _recursExtractBlocks,
   segrigateDocs: () => segrigateDocs,
   setVal: () => setVal,
@@ -614,6 +615,31 @@ var BASE_BULLMQ_CONFIG = {
       }
     }
   },
+  AI_AUTO_ANNOTATE_QUEUE: {
+    id: "ai-auto-annotate-queue",
+    queueConfig: {
+      defaultJobOptions: {
+        attempts: 3,
+        backoff: {
+          type: "exponential",
+          delay: 5e3
+        },
+        removeOnComplete: 30,
+        removeOnFail: 100
+      },
+      streams: {
+        events: {
+          maxLen: 10
+        }
+      }
+    },
+    workerConfig: {
+      concurrency: 50,
+      lockDuration: 3e5,
+      // 5 minutes lock duration since annotation can be slow
+      maxStalledCount: 3
+    }
+  },
   ANNOS_ELASTIC_SYNC_QUEUE: {
     id: "annos-elastic-sync-queue",
     queueConfig: {
@@ -740,6 +766,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   }
   return "";
 };
+
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+var import_lodash = require("lodash");
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = (0, import_lodash.cloneDeep)(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   BASE_BULLMQ_CONFIG,
@@ -749,6 +859,7 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   getPlatformContextContent,
   getRollupPossibilities,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/dist/universal.mjs

@@ -578,6 +578,31 @@ var BASE_BULLMQ_CONFIG = {
       }
     }
   },
+  AI_AUTO_ANNOTATE_QUEUE: {
+    id: "ai-auto-annotate-queue",
+    queueConfig: {
+      defaultJobOptions: {
+        attempts: 3,
+        backoff: {
+          type: "exponential",
+          delay: 5e3
+        },
+        removeOnComplete: 30,
+        removeOnFail: 100
+      },
+      streams: {
+        events: {
+          maxLen: 10
+        }
+      }
+    },
+    workerConfig: {
+      concurrency: 50,
+      lockDuration: 3e5,
+      // 5 minutes lock duration since annotation can be slow
+      maxStalledCount: 3
+    }
+  },
   ANNOS_ELASTIC_SYNC_QUEUE: {
     id: "annos-elastic-sync-queue",
     queueConfig: {
@@ -704,6 +729,90 @@ ${v?.markdownText || ""}` : v?.markdownText || "";
   }
   return "";
 };
+
+// src/utils/mergeAnnoDataIntoAnnotationsTags.ts
+import { cloneDeep } from "lodash";
+var mergeAnnoDataIntoAnnotationsTags = ({
+  prevObj,
+  newLexValue,
+  thisBlockValuePath,
+  author = { id: null, name: "AI" }
+}) => {
+  let newConsolidated = cloneDeep(prevObj) || {};
+  const { annoData = {} } = newLexValue || {};
+  const existingKeys = /* @__PURE__ */ new Map();
+  Object.keys(newConsolidated).map((tagType) => {
+    if (newConsolidated[tagType].data?.length > 0) {
+      newConsolidated[tagType].data.forEach((tag, tagIdx) => {
+        if (tag.fragments?.length > 0) {
+          newConsolidated[tagType].data[tagIdx].fragments = newConsolidated[tagType].data[tagIdx].fragments.filter((frag) => {
+            existingKeys.set(frag.annoDataId, frag?.author ?? "");
+            if (frag.extractedFromValuePath !== thisBlockValuePath) {
+              return true;
+            } else {
+              return false;
+            }
+          });
+        }
+      });
+    }
+  });
+  Object.keys(annoData).map((annoDataId) => {
+    const { tags, fragment = {}, ...restAnnoData } = annoData[annoDataId];
+    const tagTypes = Object.keys(tags || {});
+    const buildFragment = () => ({
+      ...fragment,
+      ...restAnnoData,
+      annoDataId,
+      // Fallback to provided author if author not set (for backwards compatibility)
+      author: restAnnoData.author || existingKeys.get(annoDataId) || author,
+      extractedFromValuePath: thisBlockValuePath
+    });
+    if (tagTypes.length > 0) {
+      tagTypes.forEach((tagType) => {
+        const value = tags[tagType];
+        if (newConsolidated[tagType]?.data?.length > 0) {
+          value.data.forEach((d) => {
+            const foundTagIdx = newConsolidated[tagType].data.findIndex(
+              (dd) => dd._id === d._id
+            );
+            if (foundTagIdx !== -1) {
+              const fragsAry = newConsolidated[tagType].data[foundTagIdx].fragments || [];
+              if (fragsAry.length === 0) {
+                newConsolidated[tagType].data[foundTagIdx].fragments = [
+                  buildFragment()
+                ];
+              } else {
+                newConsolidated[tagType].data[foundTagIdx].fragments.push(
+                  buildFragment()
+                );
+              }
+            } else {
+              newConsolidated[tagType].data.push({
+                ...d,
+                fragments: [buildFragment()]
+              });
+            }
+          });
+        } else {
+          newConsolidated[tagType] = {
+            collectionId: value.collectionId,
+            data: value.data.map((d) => ({
+              ...d,
+              fragments: [buildFragment()]
+            }))
+          };
+        }
+      });
+    }
+  });
+  Object.keys(newConsolidated).forEach((tagType) => {
+    newConsolidated[tagType].data = newConsolidated[tagType].data.filter(
+      (d) => d.fragments?.length > 0
+    );
+  });
+  return newConsolidated;
+};
 export {
   BASE_BULLMQ_CONFIG,
   deleteVal,
@@ -712,6 +821,7 @@ export {
   getPlatformContextContent,
   getRollupPossibilities,
   getVal,
+  mergeAnnoDataIntoAnnotationsTags,
   _recursExtractBlocks as recursivelyExtractBlocks,
   segrigateDocs,
   setVal,

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.18.2",
+  "version": "1.19.0",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -65,6 +65,7 @@
     "@semantic-release/npm": "^13.1.3",
     "@semantic-release/release-notes-generator": "^14.0.3",
     "@types/ioredis": "^4.28.10",
+    "@types/lodash": "^4.17.23",
     "@types/mongoose": "^5.11.97",
     "@types/node": "^22.13.10",
     "commitizen": "^4.3.1",
@@ -77,6 +78,7 @@
     "@elastic/elasticsearch": "^8.2.1",
     "bullmq": "^5.58.2",
     "ioredis": "^5.6.1",
+    "lodash": "^4.17.23",
     "mongodb": "^6.13.0",
     "mongoose": "^8.15.1"
   }