@okf/ootils 1.37.1 → 1.38.0

package/dist/node.d.mts

@@ -604,7 +604,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -644,7 +644,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -684,7 +684,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -724,7 +724,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -764,7 +764,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -804,7 +804,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -844,7 +844,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -884,7 +884,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
@@ -924,11 +924,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -936,8 +938,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -964,13 +964,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_17: string;
         export { id_17 as id };
         export namespace queueConfig_17 {
             export namespace defaultJobOptions_17 {
-                let attempts_17: number;
-                export { attempts_17 as attempts };
                 export namespace backoff_17 {
                     let type_17: string;
                     export { type_17 as type };
@@ -978,8 +976,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_17 as delay };
                 }
                 export { backoff_17 as backoff };
-                let delay_18: number;
-                export { delay_18 as delay };
+                let attempts_17: number;
+                export { attempts_17 as attempts };
                 let removeOnComplete_17: number;
                 export { removeOnComplete_17 as removeOnComplete };
                 let removeOnFail_17: number;
@@ -1006,6 +1004,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_17 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_18: string;
+        export { id_18 as id };
+        export namespace queueConfig_18 {
+            export namespace defaultJobOptions_18 {
+                let attempts_18: number;
+                export { attempts_18 as attempts };
+                export namespace backoff_18 {
+                    let type_18: string;
+                    export { type_18 as type };
+                    let delay_18: number;
+                    export { delay_18 as delay };
+                }
+                export { backoff_18 as backoff };
+                let delay_19: number;
+                export { delay_19 as delay };
+                let removeOnComplete_18: number;
+                export { removeOnComplete_18 as removeOnComplete };
+                let removeOnFail_18: number;
+                export { removeOnFail_18 as removeOnFail };
+            }
+            export { defaultJobOptions_18 as defaultJobOptions };
+            export namespace streams_18 {
+                export namespace events_18 {
+                    let maxLen_18: number;
+                    export { maxLen_18 as maxLen };
+                }
+                export { events_18 as events };
+            }
+            export { streams_18 as streams };
+        }
+        export { queueConfig_18 as queueConfig };
+        export namespace workerConfig_18 {
+            let concurrency_18: number;
+            export { concurrency_18 as concurrency };
+            let lockDuration_18: number;
+            export { lockDuration_18 as lockDuration };
+            let maxStalledCount_18: number;
+            export { maxStalledCount_18 as maxStalledCount };
+        }
+        export { workerConfig_18 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {

package/dist/node.d.ts

@@ -604,7 +604,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -644,7 +644,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -684,7 +684,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -724,7 +724,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -764,7 +764,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -804,7 +804,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -844,7 +844,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -884,7 +884,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
@@ -924,11 +924,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -936,8 +938,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -964,13 +964,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_17: string;
         export { id_17 as id };
         export namespace queueConfig_17 {
             export namespace defaultJobOptions_17 {
-                let attempts_17: number;
-                export { attempts_17 as attempts };
                 export namespace backoff_17 {
                     let type_17: string;
                     export { type_17 as type };
@@ -978,8 +976,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_17 as delay };
                 }
                 export { backoff_17 as backoff };
-                let delay_18: number;
-                export { delay_18 as delay };
+                let attempts_17: number;
+                export { attempts_17 as attempts };
                 let removeOnComplete_17: number;
                 export { removeOnComplete_17 as removeOnComplete };
                 let removeOnFail_17: number;
@@ -1006,6 +1004,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_17 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_18: string;
+        export { id_18 as id };
+        export namespace queueConfig_18 {
+            export namespace defaultJobOptions_18 {
+                let attempts_18: number;
+                export { attempts_18 as attempts };
+                export namespace backoff_18 {
+                    let type_18: string;
+                    export { type_18 as type };
+                    let delay_18: number;
+                    export { delay_18 as delay };
+                }
+                export { backoff_18 as backoff };
+                let delay_19: number;
+                export { delay_19 as delay };
+                let removeOnComplete_18: number;
+                export { removeOnComplete_18 as removeOnComplete };
+                let removeOnFail_18: number;
+                export { removeOnFail_18 as removeOnFail };
+            }
+            export { defaultJobOptions_18 as defaultJobOptions };
+            export namespace streams_18 {
+                export namespace events_18 {
+                    let maxLen_18: number;
+                    export { maxLen_18 as maxLen };
+                }
+                export { events_18 as events };
+            }
+            export { streams_18 as streams };
+        }
+        export { queueConfig_18 as queueConfig };
+        export namespace workerConfig_18 {
+            let concurrency_18: number;
+            export { concurrency_18 as concurrency };
+            let lockDuration_18: number;
+            export { lockDuration_18 as lockDuration };
+            let maxStalledCount_18: number;
+            export { maxStalledCount_18 as maxStalledCount };
+        }
+        export { workerConfig_18 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {

package/dist/node.js

@@ -247,6 +247,15 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         }
       },
       AI_AUTO_ANNOTATE_QUEUE: {
+        // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
+        // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
+        // the parent only after all children settle. Parent job's worker
+        // collects children's suggestions, runs ONE doc-level inductive batch
+        // dedup, applies merged annotations to editor state, writes the doc.
+        //
+        // The progress tracker aggregator (calculateSegByAutoAnnotateIdJobs in
+        // okf-sub) filters by this queue's id, so parents = 1 job per doc
+        // matches the existing FE progress counting semantics 1:1.
         id: "ai-auto-annotate-queue",
         queueConfig: {
           defaultJobOptions: {
@@ -265,14 +274,70 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          // Parallel-safe: createTag/createSubTheme catch E11000 on the unique tagId
-          // index and re-fetch the existing doc, so concurrent jobs proposing the
-          // same tag converge to one. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js.
-          // Starting at 10; real ceiling is the LLM provider's rate limit.
-          concurrency: 15,
+          // Parallel-safe across docs: per-doc isolation on annotations.tags
+          // rollup (each parent processes a different doc); tenant-level Redis
+          // lock in resolveInductiveBatch serializes inductive dedup across
+          // concurrent parents; createTag/createSubTheme catch E11000 on the
+          // unique tagId index. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js
+          // + ai/dualAgentAnnotation/tools/resolveInductiveBatch.js.
+          concurrency: 5,
           lockDuration: 3e5,
-          // 5 minutes lock duration since annotation can be slow
-          maxStalledCount: 3
+          // 5 min — generous ceiling so heartbeat stutters don't trip stalled detection
+          // With attempts:1 a stalled job has no retry left anyway, but keeping
+          // stalled-count low ensures BullMQ doesn't mis-pickup a job whose
+          // worker is just CPU-busy.
+          maxStalledCount: 1
+        }
+      },
+      AI_AUTO_ANNOTATE_CHUNK_QUEUE: {
+        // CHILDREN queue in the per-doc Flow. One job per (doc, field, chunk).
+        // Each job calls okf-be /internal/getChunkSuggestions for its one chunk
+        // — a small fast call well under App Engine's 60s ceiling regardless
+        // of how large the source doc is. Failure of a single chunk does NOT
+        // kill the parent (children opt in via failParentOnFailure: false) —
+        // the parent's worker handles partial-success aggregation, mirroring
+        // the Promise.allSettled semantic at the BullMQ level.
+        //
+        // The progress tracker IGNORES this queue (the aggregator filters on
+        // AI_AUTO_ANNOTATE_QUEUE.id), so chunk granularity is available in
+        // job state for any future drill-in UI without disrupting today's
+        // doc-level progress UI.
+        id: "ai-auto-annotate-chunk-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            // One try, no retries. A retry would re-fire the dual-agent LLM
+            // chain (annotator + moderator + attribute classifier) for the
+            // same chunk — expensive, and if the first try hit an OpenAI
+            // 429/timeout the retry will likely hit the same condition.
+            // Failed chunks surface to the parent via getFailedChildrenValues
+            // and become a partial-success on the field (the rest still apply).
+            attempts: 1,
+            backoff: {
+              type: "exponential",
+              delay: 5e3
+            },
+            removeOnComplete: 100,
+            // chunks fan out wider; keep a few more
+            removeOnFail: 300
+          },
+          streams: {
+            events: {
+              maxLen: 10
+            }
+          }
+        },
+        workerConfig: {
+          // OpenAI gpt-4.1-mini TPM ceiling is 10M tokens/min. Each chunk
+          // does ~4 LLM calls in series totalling ~22K tokens.
+          //   5 chunks in flight × 4 calls/chunk × 60s / 15s-per-call
+          //     ≈ 80 calls/min × 6.5K tokens
+          //     ≈ 520K tokens/min sustained → ~5% of TPM ceiling.
+          // Comfortable headroom for variance (long-tail chunks, bigger
+          // taxonomies, mixed-tenant batches).
+          concurrency: 5,
+          lockDuration: 18e4,
+          // 3 min — a single chunk's dual-agent run fits well under
+          maxStalledCount: 1
         }
       },
       ANNOS_ELASTIC_SYNC_QUEUE: {

package/dist/node.mjs

@@ -252,6 +252,15 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
         }
       },
       AI_AUTO_ANNOTATE_QUEUE: {
+        // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
+        // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
+        // the parent only after all children settle. Parent job's worker
+        // collects children's suggestions, runs ONE doc-level inductive batch
+        // dedup, applies merged annotations to editor state, writes the doc.
+        //
+        // The progress tracker aggregator (calculateSegByAutoAnnotateIdJobs in
+        // okf-sub) filters by this queue's id, so parents = 1 job per doc
+        // matches the existing FE progress counting semantics 1:1.
         id: "ai-auto-annotate-queue",
         queueConfig: {
           defaultJobOptions: {
@@ -270,14 +279,70 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          // Parallel-safe: createTag/createSubTheme catch E11000 on the unique tagId
-          // index and re-fetch the existing doc, so concurrent jobs proposing the
-          // same tag converge to one. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js.
-          // Starting at 10; real ceiling is the LLM provider's rate limit.
-          concurrency: 15,
+          // Parallel-safe across docs: per-doc isolation on annotations.tags
+          // rollup (each parent processes a different doc); tenant-level Redis
+          // lock in resolveInductiveBatch serializes inductive dedup across
+          // concurrent parents; createTag/createSubTheme catch E11000 on the
+          // unique tagId index. See ai/dualAgentAnnotation/tools/createTaxonomyItem.js
+          // + ai/dualAgentAnnotation/tools/resolveInductiveBatch.js.
+          concurrency: 5,
           lockDuration: 3e5,
-          // 5 minutes lock duration since annotation can be slow
-          maxStalledCount: 3
+          // 5 min — generous ceiling so heartbeat stutters don't trip stalled detection
+          // With attempts:1 a stalled job has no retry left anyway, but keeping
+          // stalled-count low ensures BullMQ doesn't mis-pickup a job whose
+          // worker is just CPU-busy.
+          maxStalledCount: 1
+        }
+      },
+      AI_AUTO_ANNOTATE_CHUNK_QUEUE: {
+        // CHILDREN queue in the per-doc Flow. One job per (doc, field, chunk).
+        // Each job calls okf-be /internal/getChunkSuggestions for its one chunk
+        // — a small fast call well under App Engine's 60s ceiling regardless
+        // of how large the source doc is. Failure of a single chunk does NOT
+        // kill the parent (children opt in via failParentOnFailure: false) —
+        // the parent's worker handles partial-success aggregation, mirroring
+        // the Promise.allSettled semantic at the BullMQ level.
+        //
+        // The progress tracker IGNORES this queue (the aggregator filters on
+        // AI_AUTO_ANNOTATE_QUEUE.id), so chunk granularity is available in
+        // job state for any future drill-in UI without disrupting today's
+        // doc-level progress UI.
+        id: "ai-auto-annotate-chunk-queue",
+        queueConfig: {
+          defaultJobOptions: {
+            // One try, no retries. A retry would re-fire the dual-agent LLM
+            // chain (annotator + moderator + attribute classifier) for the
+            // same chunk — expensive, and if the first try hit an OpenAI
+            // 429/timeout the retry will likely hit the same condition.
+            // Failed chunks surface to the parent via getFailedChildrenValues
+            // and become a partial-success on the field (the rest still apply).
+            attempts: 1,
+            backoff: {
+              type: "exponential",
+              delay: 5e3
+            },
+            removeOnComplete: 100,
+            // chunks fan out wider; keep a few more
+            removeOnFail: 300
+          },
+          streams: {
+            events: {
+              maxLen: 10
+            }
+          }
+        },
+        workerConfig: {
+          // OpenAI gpt-4.1-mini TPM ceiling is 10M tokens/min. Each chunk
+          // does ~4 LLM calls in series totalling ~22K tokens.
+          //   5 chunks in flight × 4 calls/chunk × 60s / 15s-per-call
+          //     ≈ 80 calls/min × 6.5K tokens
+          //     ≈ 520K tokens/min sustained → ~5% of TPM ceiling.
+          // Comfortable headroom for variance (long-tail chunks, bigger
+          // taxonomies, mixed-tenant batches).
+          concurrency: 5,
+          lockDuration: 18e4,
+          // 3 min — a single chunk's dual-agent run fits well under
+          maxStalledCount: 1
         }
       },
       ANNOS_ELASTIC_SYNC_QUEUE: {