@okf/ootils 1.37.1 → 1.39.0

package/dist/node.js

@@ -247,6 +247,15 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         }
       },
       AI_AUTO_ANNOTATE_QUEUE: {
+        // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
+        // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
+        // the parent only after all children settle. Parent job's worker
+        // collects children's suggestions, runs ONE doc-level inductive batch
+        // dedup, applies merged annotations to editor state, writes the doc.
+        //
+        // The progress tracker aggregator (calculateSegByAutoAnnotateIdJobs in
+        // okf-sub) filters by this queue's id, so parents = 1 job per doc
+        // matches the existing FE progress counting semantics 1:1.
         id: "ai-auto-annotate-queue",
         queueConfig: {
           defaultJobOptions: {
@@ -265,14 +274,70 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          // Parallel-safe: createTag/createSubTheme catch E11000 on the unique tagId
-          // index and re-fetch the existing doc, so concurrent jobs proposing the
-          // same tag converge to one. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js.
-          // Starting at 10; real ceiling is the LLM provider's rate limit.
-          concurrency: 15,
+          // Parallel-safe across docs: per-doc isolation on annotations.tags
+          // rollup (each parent processes a different doc); tenant-level Redis
+          // lock in resolveInductiveBatch serializes inductive dedup across
+          // concurrent parents; createTag/createSubTheme catch E11000 on the
+          // unique tagId index. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js
+          // + ai/dualAgentAnnotation/tools/resolveInductiveBatch.js.
+          concurrency: 5,
           lockDuration: 3e5,
-          // 5 minutes lock duration since annotation can be slow
-          maxStalledCount: 3
+          // 5 min — generous ceiling so heartbeat stutters don't trip stalled detection
+          // With attempts:1 a stalled job has no retry left anyway, but keeping
+          // stalled-count low ensures BullMQ doesn't mis-pickup a job whose
+          // worker is just CPU-busy.
+          maxStalledCount: 1
+        }
+      },
+      AI_AUTO_ANNOTATE_CHUNK_QUEUE: {
+        // CHILDREN queue in the per-doc Flow. One job per (doc, field, chunk).
+        // Each job calls okf-be /internal/getChunkSuggestions for its one chunk
+        // — a small fast call well under App Engine's 60s ceiling regardless
+        // of how large the source doc is. Failure of a single chunk does NOT
+        // kill the parent (children opt in via failParentOnFailure: false) —
+        // the parent's worker handles partial-success aggregation, mirroring
+        // the Promise.allSettled semantic at the BullMQ level.
+        //
+        // The progress tracker IGNORES this queue (the aggregator filters on
+        // AI_AUTO_ANNOTATE_QUEUE.id), so chunk granularity is available in
+        // job state for any future drill-in UI without disrupting today's
+        // doc-level progress UI.
+        id: "ai-auto-annotate-chunk-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            // One try, no retries. A retry would re-fire the dual-agent LLM
+            // chain (annotator + moderator + attribute classifier) for the
+            // same chunk — expensive, and if the first try hit an OpenAI
+            // 429/timeout the retry will likely hit the same condition.
+            // Failed chunks surface to the parent via getFailedChildrenValues
+            // and become a partial-success on the field (the rest still apply).
+            attempts: 1,
+            backoff: {
+              type: "exponential",
+              delay: 5e3
+            },
+            removeOnComplete: 100,
+            // chunks fan out wider; keep a few more
+            removeOnFail: 300
+          },
+          streams: {
+            events: {
+              maxLen: 10
+            }
+          }
+        },
+        workerConfig: {
+          // OpenAI gpt-4.1-mini TPM ceiling is 10M tokens/min. Each chunk
+          // does ~4 LLM calls in series totalling ~22K tokens.
+          //   5 chunks in flight × 4 calls/chunk × 60s / 15s-per-call
+          //     ≈ 80 calls/min × 6.5K tokens
+          //     ≈ 520K tokens/min sustained → ~5% of TPM ceiling.
+          // Comfortable headroom for variance (long-tail chunks, bigger
+          // taxonomies, mixed-tenant batches).
+          concurrency: 5,
+          lockDuration: 18e4,
+          // 3 min — a single chunk's dual-agent run fits well under
+          maxStalledCount: 1
         }
       },
       ANNOS_ELASTIC_SYNC_QUEUE: {
@@ -2124,6 +2189,7 @@ __export(node_exports, {
   autoGenFilterConfigsFromTpl: () => autoGenFilterConfigsFromTpl,
   blockRegistry: () => blockRegistry,
   buildFilterConfigurations: () => buildFilterConfigurations,
+  collectMarkIdsInOrder: () => collectMarkIdsInOrder,
   compareAndGroupBlocks: () => compareAndGroupBlocks,
   deleteVal: () => deleteVal,
   extractAllBlocksFromTpl: () => extractAllBlocksFromTpl,
@@ -2789,6 +2855,31 @@ var isTplAnnotationEnabled = (tpl) => {
   return blockRegistry.getAnnotationEnabledBlocks(allBlocks).length > 0;
 };
 
+// src/utils/collectMarkIdsInOrder.ts
+var collectMarkIdsInOrder = (editorState) => {
+  const ordered = [];
+  const seen = /* @__PURE__ */ new Set();
+  const walk = (node) => {
+    if (Array.isArray(node)) {
+      for (const n of node) walk(n);
+      return;
+    }
+    if (node && typeof node === "object") {
+      if (node.type === "mark" && Array.isArray(node.ids)) {
+        for (const id of node.ids) {
+          if (!seen.has(id)) {
+            seen.add(id);
+            ordered.push(id);
+          }
+        }
+      }
+      for (const key in node) walk(node[key]);
+    }
+  };
+  walk(editorState);
+  return ordered;
+};
+
 // src/utils/getRollupPossibilities.ts
 var MAX_DEPTH_ROLLUP_POSSIBILITIES = 10;
 function getRollupPossibilities({
@@ -4772,6 +4863,7 @@ var import_GET_GLOBAL_BULLMQ_CONFIG = __toESM(require_GET_GLOBAL_BULLMQ_CONFIG()
   autoGenFilterConfigsFromTpl,
   blockRegistry,
   buildFilterConfigurations,
+  collectMarkIdsInOrder,
   compareAndGroupBlocks,
   deleteVal,
   extractAllBlocksFromTpl,

package/dist/node.mjs

@@ -252,6 +252,15 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         }
       },
       AI_AUTO_ANNOTATE_QUEUE: {
+        // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
+        // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
+        // the parent only after all children settle. Parent job's worker
+        // collects children's suggestions, runs ONE doc-level inductive batch
+        // dedup, applies merged annotations to editor state, writes the doc.
+        //
+        // The progress tracker aggregator (calculateSegByAutoAnnotateIdJobs in
+        // okf-sub) filters by this queue's id, so parents = 1 job per doc
+        // matches the existing FE progress counting semantics 1:1.
         id: "ai-auto-annotate-queue",
         queueConfig: {
           defaultJobOptions: {
@@ -270,14 +279,70 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          // Parallel-safe: createTag/createSubTheme catch E11000 on the unique tagId
-          // index and re-fetch the existing doc, so concurrent jobs proposing the
-          // same tag converge to one. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js.
-          // Starting at 10; real ceiling is the LLM provider's rate limit.
-          concurrency: 15,
+          // Parallel-safe across docs: per-doc isolation on annotations.tags
+          // rollup (each parent processes a different doc); tenant-level Redis
+          // lock in resolveInductiveBatch serializes inductive dedup across
+          // concurrent parents; createTag/createSubTheme catch E11000 on the
+          // unique tagId index. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js
+          // + ai/dualAgentAnnotation/tools/resolveInductiveBatch.js.
+          concurrency: 5,
           lockDuration: 3e5,
-          // 5 minutes lock duration since annotation can be slow
-          maxStalledCount: 3
+          // 5 min — generous ceiling so heartbeat stutters don't trip stalled detection
+          // With attempts:1 a stalled job has no retry left anyway, but keeping
+          // stalled-count low ensures BullMQ doesn't mis-pickup a job whose
+          // worker is just CPU-busy.
+          maxStalledCount: 1
+        }
+      },
+      AI_AUTO_ANNOTATE_CHUNK_QUEUE: {
+        // CHILDREN queue in the per-doc Flow. One job per (doc, field, chunk).
+        // Each job calls okf-be /internal/getChunkSuggestions for its one chunk
+        // — a small fast call well under App Engine's 60s ceiling regardless
+        // of how large the source doc is. Failure of a single chunk does NOT
+        // kill the parent (children opt in via failParentOnFailure: false) —
+        // the parent's worker handles partial-success aggregation, mirroring
+        // the Promise.allSettled semantic at the BullMQ level.
+        //
+        // The progress tracker IGNORES this queue (the aggregator filters on
+        // AI_AUTO_ANNOTATE_QUEUE.id), so chunk granularity is available in
+        // job state for any future drill-in UI without disrupting today's
+        // doc-level progress UI.
+        id: "ai-auto-annotate-chunk-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            // One try, no retries. A retry would re-fire the dual-agent LLM
+            // chain (annotator + moderator + attribute classifier) for the
+            // same chunk — expensive, and if the first try hit an OpenAI
+            // 429/timeout the retry will likely hit the same condition.
+            // Failed chunks surface to the parent via getFailedChildrenValues
+            // and become a partial-success on the field (the rest still apply).
+            attempts: 1,
+            backoff: {
+              type: "exponential",
+              delay: 5e3
+            },
+            removeOnComplete: 100,
+            // chunks fan out wider; keep a few more
+            removeOnFail: 300
+          },
+          streams: {
+            events: {
+              maxLen: 10
+            }
+          }
+        },
+        workerConfig: {
+          // OpenAI gpt-4.1-mini TPM ceiling is 10M tokens/min. Each chunk
+          // does ~4 LLM calls in series totalling ~22K tokens.
+          //   5 chunks in flight × 4 calls/chunk × 60s / 15s-per-call
+          //     ≈ 80 calls/min × 6.5K tokens
+          //     ≈ 520K tokens/min sustained → ~5% of TPM ceiling.
+          // Comfortable headroom for variance (long-tail chunks, bigger
+          // taxonomies, mixed-tenant batches).
+          concurrency: 5,
+          lockDuration: 18e4,
+          // 3 min — a single chunk's dual-agent run fits well under
+          maxStalledCount: 1
         }
       },
       ANNOS_ELASTIC_SYNC_QUEUE: {
@@ -2719,6 +2784,31 @@ var isTplAnnotationEnabled = (tpl) => {
   return blockRegistry.getAnnotationEnabledBlocks(allBlocks).length > 0;
 };
 
+// src/utils/collectMarkIdsInOrder.ts
+var collectMarkIdsInOrder = (editorState) => {
+  const ordered = [];
+  const seen = /* @__PURE__ */ new Set();
+  const walk = (node) => {
+    if (Array.isArray(node)) {
+      for (const n of node) walk(n);
+      return;
+    }
+    if (node && typeof node === "object") {
+      if (node.type === "mark" && Array.isArray(node.ids)) {
+        for (const id of node.ids) {
+          if (!seen.has(id)) {
+            seen.add(id);
+            ordered.push(id);
+          }
+        }
+      }
+      for (const key in node) walk(node[key]);
+    }
+  };
+  walk(editorState);
+  return ordered;
+};
+
 // src/utils/getRollupPossibilities.ts
 var MAX_DEPTH_ROLLUP_POSSIBILITIES = 10;
 function getRollupPossibilities({
@@ -4717,6 +4807,7 @@ export {
   autoGenFilterConfigsFromTpl,
   blockRegistry,
   buildFilterConfigurations,
+  collectMarkIdsInOrder,
   compareAndGroupBlocks,
   deleteVal,
   extractAllBlocksFromTpl,

package/dist/universal.d.mts

@@ -222,6 +222,29 @@ declare const _recursExtractBlocks: ({ data, cb, sectionStack, blockPathPrefix }
  */
 declare const isTplAnnotationEnabled: (tpl: any) => boolean;
 
+/**
+ * Collect annotation (mark-node) ids from a serialized Lexical editorState tree,
+ * in document reading order, deduped on FIRST occurrence.
+ *
+ * Lexical stores annotation highlights as MarkNodes: `{ type: "mark", ids: [...],
+ * children: [...] }`. Overlapping annotations split the text into contiguous
+ * segments, each a mark whose `ids` lists every annotation covering that segment —
+ * so a single annotation can appear across multiple marks, and a single mark can
+ * carry multiple ids. A depth-first walk visits segments left-to-right (document
+ * order); taking each id's first appearance yields the position where that
+ * annotation's highlight first starts. Two annotations that start at the exact
+ * same character share their first segment, so their relative order falls to that
+ * segment's `ids` array order — an inherent tie with no positional answer.
+ *
+ * Pass the editorState root (the `{ root: { children } }` tree). It walks generic
+ * JSON, so any subtree works too — but never pass an object that also nests
+ * unrelated mark-bearing states (e.g. a Lexical value's sibling `annoData`, whose
+ * per-anno `fragment` mini-states would inject out-of-order marks).
+ *
+ * Callers that only need the unordered SET of ids can wrap the result in `new Set(...)`.
+ */
+declare const collectMarkIdsInOrder: (editorState: any) => string[];
+
 /**
  * Calculates all possible rollup relationships for a tag type
  *
@@ -597,7 +620,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -637,7 +660,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -677,7 +700,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -717,7 +740,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -757,7 +780,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -797,7 +820,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -837,7 +860,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -877,7 +900,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
@@ -917,11 +940,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -929,8 +954,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -957,13 +980,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_17: string;
         export { id_17 as id };
         export namespace queueConfig_17 {
             export namespace defaultJobOptions_17 {
-                let attempts_17: number;
-                export { attempts_17 as attempts };
                 export namespace backoff_17 {
                     let type_17: string;
                     export { type_17 as type };
@@ -971,8 +992,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_17 as delay };
                 }
                 export { backoff_17 as backoff };
-                let delay_18: number;
-                export { delay_18 as delay };
+                let attempts_17: number;
+                export { attempts_17 as attempts };
                 let removeOnComplete_17: number;
                 export { removeOnComplete_17 as removeOnComplete };
                 let removeOnFail_17: number;
@@ -999,6 +1020,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_17 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_18: string;
+        export { id_18 as id };
+        export namespace queueConfig_18 {
+            export namespace defaultJobOptions_18 {
+                let attempts_18: number;
+                export { attempts_18 as attempts };
+                export namespace backoff_18 {
+                    let type_18: string;
+                    export { type_18 as type };
+                    let delay_18: number;
+                    export { delay_18 as delay };
+                }
+                export { backoff_18 as backoff };
+                let delay_19: number;
+                export { delay_19 as delay };
+                let removeOnComplete_18: number;
+                export { removeOnComplete_18 as removeOnComplete };
+                let removeOnFail_18: number;
+                export { removeOnFail_18 as removeOnFail };
+            }
+            export { defaultJobOptions_18 as defaultJobOptions };
+            export namespace streams_18 {
+                export namespace events_18 {
+                    let maxLen_18: number;
+                    export { maxLen_18 as maxLen };
+                }
+                export { events_18 as events };
+            }
+            export { streams_18 as streams };
+        }
+        export { queueConfig_18 as queueConfig };
+        export namespace workerConfig_18 {
+            let concurrency_18: number;
+            export { concurrency_18 as concurrency };
+            let lockDuration_18: number;
+            export { lockDuration_18 as lockDuration };
+            let maxStalledCount_18: number;
+            export { maxStalledCount_18 as maxStalledCount };
+        }
+        export { workerConfig_18 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {
@@ -1937,4 +2000,4 @@ declare class BlockRegistry {
 /** Singleton instance — the one registry shared across the app. */
 declare const blockRegistry: BlockRegistry;
 
-export { BASE_BULLMQ_CONFIG, type BlockCapabilities, type BlockDef, BlockRegistry, CHUNKING_PRESETS, ELASTIC_MAPPING_PRESETS, FILTER_IDS, MONGO_SCHEMA_PRESETS, TEMP_removeDuplicateFilters, UI_CONTENT, _self_managed_buildAnnoHierarchyConfig, _self_managed_buildDocHierarchyConfig, _self_managed_getFixedAnnoRollupBlocks, _self_managed_getFixedAnnoTagBlock, autoGenFilterConfigsFromTpl, blockRegistry, buildFilterConfigurations, compareAndGroupBlocks, deleteVal, extractAllBlocksFromTpl, extractAndOrganizeBlocks, genCleanCamelCaseId, genTagId, generateFilterKey, getAnnoFilterBucketKey, getFilterKeyForBlock, getPlatformContextContent, getRollupPossibilities, getRoutePathToContentTypeLanding, getRoutePathToEditContent, getRoutePathToModerateContent, getRoutePathToMyContent, getRoutePathToPublishedContent, getRoutePathToReviewDashboard, getRoutePathToTCI, getRoutePathToTagCategoryLanding, getVal, isTplAnnotationEnabled, mergeAnnoDataIntoAnnotationsTags, parseSpecialConfigSyntax, plainTextToLexical, processAuthorAndCommonFilters, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };
+export { BASE_BULLMQ_CONFIG, type BlockCapabilities, type BlockDef, BlockRegistry, CHUNKING_PRESETS, ELASTIC_MAPPING_PRESETS, FILTER_IDS, MONGO_SCHEMA_PRESETS, TEMP_removeDuplicateFilters, UI_CONTENT, _self_managed_buildAnnoHierarchyConfig, _self_managed_buildDocHierarchyConfig, _self_managed_getFixedAnnoRollupBlocks, _self_managed_getFixedAnnoTagBlock, autoGenFilterConfigsFromTpl, blockRegistry, buildFilterConfigurations, collectMarkIdsInOrder, compareAndGroupBlocks, deleteVal, extractAllBlocksFromTpl, extractAndOrganizeBlocks, genCleanCamelCaseId, genTagId, generateFilterKey, getAnnoFilterBucketKey, getFilterKeyForBlock, getPlatformContextContent, getRollupPossibilities, getRoutePathToContentTypeLanding, getRoutePathToEditContent, getRoutePathToModerateContent, getRoutePathToMyContent, getRoutePathToPublishedContent, getRoutePathToReviewDashboard, getRoutePathToTCI, getRoutePathToTagCategoryLanding, getVal, isTplAnnotationEnabled, mergeAnnoDataIntoAnnotationsTags, parseSpecialConfigSyntax, plainTextToLexical, processAuthorAndCommonFilters, _recursExtractBlocks as recursivelyExtractBlocks, segrigateDocs, setVal, toArray };