@okf/ootils 1.37.1 → 1.38.0

package/dist/universal.d.mts

@@ -597,7 +597,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -637,7 +637,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -677,7 +677,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -717,7 +717,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -757,7 +757,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -797,7 +797,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -837,7 +837,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -877,7 +877,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
@@ -917,11 +917,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -929,8 +931,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -957,13 +957,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_17: string;
         export { id_17 as id };
         export namespace queueConfig_17 {
             export namespace defaultJobOptions_17 {
-                let attempts_17: number;
-                export { attempts_17 as attempts };
                 export namespace backoff_17 {
                     let type_17: string;
                     export { type_17 as type };
@@ -971,8 +969,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_17 as delay };
                 }
                 export { backoff_17 as backoff };
-                let delay_18: number;
-                export { delay_18 as delay };
+                let attempts_17: number;
+                export { attempts_17 as attempts };
                 let removeOnComplete_17: number;
                 export { removeOnComplete_17 as removeOnComplete };
                 let removeOnFail_17: number;
@@ -999,6 +997,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_17 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_18: string;
+        export { id_18 as id };
+        export namespace queueConfig_18 {
+            export namespace defaultJobOptions_18 {
+                let attempts_18: number;
+                export { attempts_18 as attempts };
+                export namespace backoff_18 {
+                    let type_18: string;
+                    export { type_18 as type };
+                    let delay_18: number;
+                    export { delay_18 as delay };
+                }
+                export { backoff_18 as backoff };
+                let delay_19: number;
+                export { delay_19 as delay };
+                let removeOnComplete_18: number;
+                export { removeOnComplete_18 as removeOnComplete };
+                let removeOnFail_18: number;
+                export { removeOnFail_18 as removeOnFail };
+            }
+            export { defaultJobOptions_18 as defaultJobOptions };
+            export namespace streams_18 {
+                export namespace events_18 {
+                    let maxLen_18: number;
+                    export { maxLen_18 as maxLen };
+                }
+                export { events_18 as events };
+            }
+            export { streams_18 as streams };
+        }
+        export { queueConfig_18 as queueConfig };
+        export namespace workerConfig_18 {
+            let concurrency_18: number;
+            export { concurrency_18 as concurrency };
+            let lockDuration_18: number;
+            export { lockDuration_18 as lockDuration };
+            let maxStalledCount_18: number;
+            export { maxStalledCount_18 as maxStalledCount };
+        }
+        export { workerConfig_18 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {

package/dist/universal.d.ts

@@ -597,7 +597,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -637,7 +637,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -677,7 +677,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -717,7 +717,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -757,7 +757,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -797,7 +797,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -837,7 +837,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -877,7 +877,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
@@ -917,11 +917,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -929,8 +931,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -957,13 +957,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_17: string;
         export { id_17 as id };
         export namespace queueConfig_17 {
             export namespace defaultJobOptions_17 {
-                let attempts_17: number;
-                export { attempts_17 as attempts };
                 export namespace backoff_17 {
                     let type_17: string;
                     export { type_17 as type };
@@ -971,8 +969,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_17 as delay };
                 }
                 export { backoff_17 as backoff };
-                let delay_18: number;
-                export { delay_18 as delay };
+                let attempts_17: number;
+                export { attempts_17 as attempts };
                 let removeOnComplete_17: number;
                 export { removeOnComplete_17 as removeOnComplete };
                 let removeOnFail_17: number;
@@ -999,6 +997,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_17 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_18: string;
+        export { id_18 as id };
+        export namespace queueConfig_18 {
+            export namespace defaultJobOptions_18 {
+                let attempts_18: number;
+                export { attempts_18 as attempts };
+                export namespace backoff_18 {
+                    let type_18: string;
+                    export { type_18 as type };
+                    let delay_18: number;
+                    export { delay_18 as delay };
+                }
+                export { backoff_18 as backoff };
+                let delay_19: number;
+                export { delay_19 as delay };
+                let removeOnComplete_18: number;
+                export { removeOnComplete_18 as removeOnComplete };
+                let removeOnFail_18: number;
+                export { removeOnFail_18 as removeOnFail };
+            }
+            export { defaultJobOptions_18 as defaultJobOptions };
+            export namespace streams_18 {
+                export namespace events_18 {
+                    let maxLen_18: number;
+                    export { maxLen_18 as maxLen };
+                }
+                export { events_18 as events };
+            }
+            export { streams_18 as streams };
+        }
+        export { queueConfig_18 as queueConfig };
+        export namespace workerConfig_18 {
+            let concurrency_18: number;
+            export { concurrency_18 as concurrency };
+            let lockDuration_18: number;
+            export { lockDuration_18 as lockDuration };
+            let maxStalledCount_18: number;
+            export { maxStalledCount_18 as maxStalledCount };
+        }
+        export { workerConfig_18 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {

package/dist/universal.js

@@ -990,6 +990,15 @@ var BASE_BULLMQ_CONFIG = {
     }
   },
   AI_AUTO_ANNOTATE_QUEUE: {
+    // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
+    // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
+    // the parent only after all children settle. Parent job's worker
+    // collects children's suggestions, runs ONE doc-level inductive batch
+    // dedup, applies merged annotations to editor state, writes the doc.
+    //
+    // The progress tracker aggregator (calculateSegByAutoAnnotateIdJobs in
+    // okf-sub) filters by this queue's id, so parents = 1 job per doc
+    // matches the existing FE progress counting semantics 1:1.
     id: "ai-auto-annotate-queue",
     queueConfig: {
       defaultJobOptions: {
@@ -1008,14 +1017,70 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      // Parallel-safe: createTag/createSubTheme catch E11000 on the unique tagId
-      // index and re-fetch the existing doc, so concurrent jobs proposing the
-      // same tag converge to one. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js.
-      // Starting at 10; real ceiling is the LLM provider's rate limit.
-      concurrency: 15,
+      // Parallel-safe across docs: per-doc isolation on annotations.tags
+      // rollup (each parent processes a different doc); tenant-level Redis
+      // lock in resolveInductiveBatch serializes inductive dedup across
+      // concurrent parents; createTag/createSubTheme catch E11000 on the
+      // unique tagId index. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js
+      // + ai/dualAgentAnnotation/tools/resolveInductiveBatch.js.
+      concurrency: 5,
       lockDuration: 3e5,
-      // 5 minutes lock duration since annotation can be slow
-      maxStalledCount: 3
+      // 5 min — generous ceiling so heartbeat stutters don't trip stalled detection
+      // With attempts:1 a stalled job has no retry left anyway, but keeping
+      // stalled-count low ensures BullMQ doesn't mis-pickup a job whose
+      // worker is just CPU-busy.
+      maxStalledCount: 1
+    }
+  },
+  AI_AUTO_ANNOTATE_CHUNK_QUEUE: {
+    // CHILDREN queue in the per-doc Flow. One job per (doc, field, chunk).
+    // Each job calls okf-be /internal/getChunkSuggestions for its one chunk
+    // — a small fast call well under App Engine's 60s ceiling regardless
+    // of how large the source doc is. Failure of a single chunk does NOT
+    // kill the parent (children opt in via failParentOnFailure: false) —
+    // the parent's worker handles partial-success aggregation, mirroring
+    // the Promise.allSettled semantic at the BullMQ level.
+    //
+    // The progress tracker IGNORES this queue (the aggregator filters on
+    // AI_AUTO_ANNOTATE_QUEUE.id), so chunk granularity is available in
+    // job state for any future drill-in UI without disrupting today's
+    // doc-level progress UI.
+    id: "ai-auto-annotate-chunk-queue",
+    queueConfig: {
+      defaultJobOptions: {
+        // One try, no retries. A retry would re-fire the dual-agent LLM
+        // chain (annotator + moderator + attribute classifier) for the
+        // same chunk — expensive, and if the first try hit an OpenAI
+        // 429/timeout the retry will likely hit the same condition.
+        // Failed chunks surface to the parent via getFailedChildrenValues
+        // and become a partial-success on the field (the rest still apply).
+        attempts: 1,
+        backoff: {
+          type: "exponential",
+          delay: 5e3
+        },
+        removeOnComplete: 100,
+        // chunks fan out wider; keep a few more
+        removeOnFail: 300
+      },
+      streams: {
+        events: {
+          maxLen: 10
+        }
+      }
+    },
+    workerConfig: {
+      // OpenAI gpt-4.1-mini TPM ceiling is 10M tokens/min. Each chunk
+      // does ~4 LLM calls in series totalling ~22K tokens.
+      //   5 chunks in flight × 4 calls/chunk × 60s / 15s-per-call
+      //     ≈ 80 calls/min × 6.5K tokens
+      //     ≈ 520K tokens/min sustained → ~5% of TPM ceiling.
+      // Comfortable headroom for variance (long-tail chunks, bigger
+      // taxonomies, mixed-tenant batches).
+      concurrency: 5,
+      lockDuration: 18e4,
+      // 3 min — a single chunk's dual-agent run fits well under
+      maxStalledCount: 1
     }
   },
   ANNOS_ELASTIC_SYNC_QUEUE: {

package/dist/universal.mjs

@@ -921,6 +921,15 @@ var BASE_BULLMQ_CONFIG = {
     }
   },
   AI_AUTO_ANNOTATE_QUEUE: {
+    // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
+    // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
+    // the parent only after all children settle. Parent job's worker
+    // collects children's suggestions, runs ONE doc-level inductive batch
+    // dedup, applies merged annotations to editor state, writes the doc.
+    //
+    // The progress tracker aggregator (calculateSegByAutoAnnotateIdJobs in
+    // okf-sub) filters by this queue's id, so parents = 1 job per doc
+    // matches the existing FE progress counting semantics 1:1.
     id: "ai-auto-annotate-queue",
     queueConfig: {
       defaultJobOptions: {
@@ -939,14 +948,70 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      // Parallel-safe: createTag/createSubTheme catch E11000 on the unique tagId
-      // index and re-fetch the existing doc, so concurrent jobs proposing the
-      // same tag converge to one. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js.
-      // Starting at 10; real ceiling is the LLM provider's rate limit.
-      concurrency: 15,
+      // Parallel-safe across docs: per-doc isolation on annotations.tags
+      // rollup (each parent processes a different doc); tenant-level Redis
+      // lock in resolveInductiveBatch serializes inductive dedup across
+      // concurrent parents; createTag/createSubTheme catch E11000 on the
+      // unique tagId index. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js
+      // + ai/dualAgentAnnotation/tools/resolveInductiveBatch.js.
+      concurrency: 5,
       lockDuration: 3e5,
-      // 5 minutes lock duration since annotation can be slow
-      maxStalledCount: 3
+      // 5 min — generous ceiling so heartbeat stutters don't trip stalled detection
+      // With attempts:1 a stalled job has no retry left anyway, but keeping
+      // stalled-count low ensures BullMQ doesn't mis-pickup a job whose
+      // worker is just CPU-busy.
+      maxStalledCount: 1
+    }
+  },
+  AI_AUTO_ANNOTATE_CHUNK_QUEUE: {
+    // CHILDREN queue in the per-doc Flow. One job per (doc, field, chunk).
+    // Each job calls okf-be /internal/getChunkSuggestions for its one chunk
+    // — a small fast call well under App Engine's 60s ceiling regardless
+    // of how large the source doc is. Failure of a single chunk does NOT
+    // kill the parent (children opt in via failParentOnFailure: false) —
+    // the parent's worker handles partial-success aggregation, mirroring
+    // the Promise.allSettled semantic at the BullMQ level.
+    //
+    // The progress tracker IGNORES this queue (the aggregator filters on
+    // AI_AUTO_ANNOTATE_QUEUE.id), so chunk granularity is available in
+    // job state for any future drill-in UI without disrupting today's
+    // doc-level progress UI.
+    id: "ai-auto-annotate-chunk-queue",
+    queueConfig: {
+      defaultJobOptions: {
+        // One try, no retries. A retry would re-fire the dual-agent LLM
+        // chain (annotator + moderator + attribute classifier) for the
+        // same chunk — expensive, and if the first try hit an OpenAI
+        // 429/timeout the retry will likely hit the same condition.
+        // Failed chunks surface to the parent via getFailedChildrenValues
+        // and become a partial-success on the field (the rest still apply).
+        attempts: 1,
+        backoff: {
+          type: "exponential",
+          delay: 5e3
+        },
+        removeOnComplete: 100,
+        // chunks fan out wider; keep a few more
+        removeOnFail: 300
+      },
+      streams: {
+        events: {
+          maxLen: 10
+        }
+      }
+    },
+    workerConfig: {
+      // OpenAI gpt-4.1-mini TPM ceiling is 10M tokens/min. Each chunk
+      // does ~4 LLM calls in series totalling ~22K tokens.
+      //   5 chunks in flight × 4 calls/chunk × 60s / 15s-per-call
+      //     ≈ 80 calls/min × 6.5K tokens
+      //     ≈ 520K tokens/min sustained → ~5% of TPM ceiling.
+      // Comfortable headroom for variance (long-tail chunks, bigger
+      // taxonomies, mixed-tenant batches).
+      concurrency: 5,
+      lockDuration: 18e4,
+      // 3 min — a single chunk's dual-agent run fits well under
+      maxStalledCount: 1
     }
   },
   ANNOS_ELASTIC_SYNC_QUEUE: {

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.37.1",
+  "version": "1.38.0",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",