@okf/ootils 1.41.0 → 1.41.2

package/dist/universal.d.mts

@@ -487,7 +487,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_4 as workerConfig };
     }
-    namespace AI_AUTO_ANNOTATE_QUEUE {
+    namespace AI_AUTO_ANNOTATE_PREP_QUEUE {
         let id_5: string;
         export { id_5 as id };
         export namespace queueConfig_5 {
@@ -527,7 +527,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_5 as workerConfig };
     }
-    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
+    namespace AI_AUTO_ANNOTATE_QUEUE {
         let id_6: string;
         export { id_6 as id };
         export namespace queueConfig_6 {
@@ -567,7 +567,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_6 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_7: string;
         export { id_7 as id };
         export namespace queueConfig_7 {
@@ -607,7 +607,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -647,7 +647,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -687,7 +687,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -727,7 +727,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -767,7 +767,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -807,7 +807,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -847,7 +847,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -887,11 +887,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
             export namespace defaultJobOptions_15 {
+                let attempts_15: number;
+                export { attempts_15 as attempts };
                 export namespace backoff_15 {
                     let type_15: string;
                     export { type_15 as type };
@@ -899,8 +901,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_15 as delay };
                 }
                 export { backoff_15 as backoff };
-                let attempts_15: number;
-                export { attempts_15 as attempts };
                 let removeOnComplete_15: number;
                 export { removeOnComplete_15 as removeOnComplete };
                 let removeOnFail_15: number;
@@ -927,13 +927,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -941,8 +939,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let delay_17: number;
-                export { delay_17 as delay };
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -969,6 +967,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_17: string;
+        export { id_17 as id };
+        export namespace queueConfig_17 {
+            export namespace defaultJobOptions_17 {
+                let attempts_17: number;
+                export { attempts_17 as attempts };
+                export namespace backoff_17 {
+                    let type_17: string;
+                    export { type_17 as type };
+                    let delay_17: number;
+                    export { delay_17 as delay };
+                }
+                export { backoff_17 as backoff };
+                let delay_18: number;
+                export { delay_18 as delay };
+                let removeOnComplete_17: number;
+                export { removeOnComplete_17 as removeOnComplete };
+                let removeOnFail_17: number;
+                export { removeOnFail_17 as removeOnFail };
+            }
+            export { defaultJobOptions_17 as defaultJobOptions };
+            export namespace streams_17 {
+                export namespace events_17 {
+                    let maxLen_17: number;
+                    export { maxLen_17 as maxLen };
+                }
+                export { events_17 as events };
+            }
+            export { streams_17 as streams };
+        }
+        export { queueConfig_17 as queueConfig };
+        export namespace workerConfig_17 {
+            let concurrency_17: number;
+            export { concurrency_17 as concurrency };
+            let lockDuration_17: number;
+            export { lockDuration_17 as lockDuration };
+            let maxStalledCount_17: number;
+            export { maxStalledCount_17 as maxStalledCount };
+        }
+        export { workerConfig_17 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {

package/dist/universal.d.ts

@@ -487,7 +487,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_4 as workerConfig };
     }
-    namespace AI_AUTO_ANNOTATE_QUEUE {
+    namespace AI_AUTO_ANNOTATE_PREP_QUEUE {
         let id_5: string;
         export { id_5 as id };
         export namespace queueConfig_5 {
@@ -527,7 +527,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_5 as workerConfig };
     }
-    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
+    namespace AI_AUTO_ANNOTATE_QUEUE {
         let id_6: string;
         export { id_6 as id };
         export namespace queueConfig_6 {
@@ -567,7 +567,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_6 as workerConfig };
     }
-    namespace ANNOS_ELASTIC_SYNC_QUEUE {
+    namespace AI_AUTO_ANNOTATE_CHUNK_QUEUE {
         let id_7: string;
         export { id_7 as id };
         export namespace queueConfig_7 {
@@ -607,7 +607,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_7 as workerConfig };
     }
-    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
+    namespace ANNOS_ELASTIC_SYNC_QUEUE {
         let id_8: string;
         export { id_8 as id };
         export namespace queueConfig_8 {
@@ -647,7 +647,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_8 as workerConfig };
     }
-    namespace USERS_ELASTIC_SYNC_QUEUE {
+    namespace CHUNKS_ELASTIC_SYNC_QUEUE {
         let id_9: string;
         export { id_9 as id };
         export namespace queueConfig_9 {
@@ -687,7 +687,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_9 as workerConfig };
     }
-    namespace CONTENT_ELASTIC_SYNC_QUEUE {
+    namespace USERS_ELASTIC_SYNC_QUEUE {
         let id_10: string;
         export { id_10 as id };
         export namespace queueConfig_10 {
@@ -727,7 +727,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_10 as workerConfig };
     }
-    namespace SARVAM_TRANSCRIPTION_QUEUE {
+    namespace CONTENT_ELASTIC_SYNC_QUEUE {
         let id_11: string;
         export { id_11 as id };
         export namespace queueConfig_11 {
@@ -767,7 +767,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_11 as workerConfig };
     }
-    namespace INTERCOM_CONVERSATIONS_QUEUE {
+    namespace SARVAM_TRANSCRIPTION_QUEUE {
         let id_12: string;
         export { id_12 as id };
         export namespace queueConfig_12 {
@@ -807,7 +807,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_12 as workerConfig };
     }
-    namespace TAG_SYNC_PLAN_QUEUE {
+    namespace INTERCOM_CONVERSATIONS_QUEUE {
         let id_13: string;
         export { id_13 as id };
         export namespace queueConfig_13 {
@@ -847,7 +847,7 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_13 as workerConfig };
     }
-    namespace TAG_SYNC_BATCH_QUEUE {
+    namespace TAG_SYNC_PLAN_QUEUE {
         let id_14: string;
         export { id_14 as id };
         export namespace queueConfig_14 {
@@ -887,11 +887,13 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_14 as workerConfig };
     }
-    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
+    namespace TAG_SYNC_BATCH_QUEUE {
         let id_15: string;
         export { id_15 as id };
         export namespace queueConfig_15 {
             export namespace defaultJobOptions_15 {
+                let attempts_15: number;
+                export { attempts_15 as attempts };
                 export namespace backoff_15 {
                     let type_15: string;
                     export { type_15 as type };
@@ -899,8 +901,6 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_15 as delay };
                 }
                 export { backoff_15 as backoff };
-                let attempts_15: number;
-                export { attempts_15 as attempts };
                 let removeOnComplete_15: number;
                 export { removeOnComplete_15 as removeOnComplete };
                 let removeOnFail_15: number;
@@ -927,13 +927,11 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_15 as workerConfig };
     }
-    namespace REINDEX_QUEUE {
+    namespace SCHEDULED_EXTERNAL_SYNC_QUEUE {
         let id_16: string;
         export { id_16 as id };
         export namespace queueConfig_16 {
             export namespace defaultJobOptions_16 {
-                let attempts_16: number;
-                export { attempts_16 as attempts };
                 export namespace backoff_16 {
                     let type_16: string;
                     export { type_16 as type };
@@ -941,8 +939,8 @@ declare namespace BASE_BULLMQ_CONFIG {
                     export { delay_16 as delay };
                 }
                 export { backoff_16 as backoff };
-                let delay_17: number;
-                export { delay_17 as delay };
+                let attempts_16: number;
+                export { attempts_16 as attempts };
                 let removeOnComplete_16: number;
                 export { removeOnComplete_16 as removeOnComplete };
                 let removeOnFail_16: number;
@@ -969,6 +967,48 @@ declare namespace BASE_BULLMQ_CONFIG {
         }
         export { workerConfig_16 as workerConfig };
     }
+    namespace REINDEX_QUEUE {
+        let id_17: string;
+        export { id_17 as id };
+        export namespace queueConfig_17 {
+            export namespace defaultJobOptions_17 {
+                let attempts_17: number;
+                export { attempts_17 as attempts };
+                export namespace backoff_17 {
+                    let type_17: string;
+                    export { type_17 as type };
+                    let delay_17: number;
+                    export { delay_17 as delay };
+                }
+                export { backoff_17 as backoff };
+                let delay_18: number;
+                export { delay_18 as delay };
+                let removeOnComplete_17: number;
+                export { removeOnComplete_17 as removeOnComplete };
+                let removeOnFail_17: number;
+                export { removeOnFail_17 as removeOnFail };
+            }
+            export { defaultJobOptions_17 as defaultJobOptions };
+            export namespace streams_17 {
+                export namespace events_17 {
+                    let maxLen_17: number;
+                    export { maxLen_17 as maxLen };
+                }
+                export { events_17 as events };
+            }
+            export { streams_17 as streams };
+        }
+        export { queueConfig_17 as queueConfig };
+        export namespace workerConfig_17 {
+            let concurrency_17: number;
+            export { concurrency_17 as concurrency };
+            let lockDuration_17: number;
+            export { lockDuration_17 as lockDuration };
+            let maxStalledCount_17: number;
+            export { maxStalledCount_17 as maxStalledCount };
+        }
+        export { workerConfig_17 as workerConfig };
+    }
 }
 
 interface PlatformContextContentItem {

package/dist/universal.js

@@ -966,6 +966,58 @@ var BASE_BULLMQ_CONFIG = {
       }
     }
   },
+  AI_AUTO_ANNOTATE_PREP_QUEUE: {
+    // PREP queue — the front of the per-doc Flow pipeline. The FE-facing
+    // /AIAutoAnnotate handler no longer fetches + summarizes + chunks +
+    // enqueues each doc inline (that serial, LLM-heavy loop blocked the HTTP
+    // request and risked App Engine clipping large batches mid-loop, silently
+    // dropping every doc past the cutoff). Instead the handler computes the
+    // batch-level prereqs ONCE (7th criterion + platform-derived categories)
+    // and enqueues one PREP job per doc here, then returns fast.
+    //
+    // Each prep job's worker (okf-sub) calls okf-be /internal/prepDoc, which
+    // does the per-doc work: fetch → summarizeDocument (warms the doc-level
+    // summary cache) → chunk fields → enqueue that doc's Flow (parent on
+    // AI_AUTO_ANNOTATE_QUEUE + one child per chunk on the CHUNK queue). So the
+    // Flow parents now get created rolling, as each prep job completes, rather
+    // than all up-front in the request. A prep failure isolates to its one doc
+    // (that doc is skipped); sibling prep jobs keep going.
+    id: "ai-auto-annotate-prep-queue",
+    queueConfig: {
+      defaultJobOptions: {
+        // attempts:1 — a retry would re-summarize (cache-HIT, cheap) but then
+        // re-enqueue the doc's Flow, double-annotating the doc. The rest of
+        // the per-doc Flow pipeline is attempts:1 for the same fail-loud
+        // reason (a retry just re-fires expensive LLM work).
+        attempts: 1,
+        backoff: {
+          type: "exponential",
+          delay: 5e3
+        },
+        removeOnComplete: 30,
+        removeOnFail: 100
+      },
+      streams: {
+        events: {
+          maxLen: 10
+        }
+      }
+    },
+    workerConfig: {
+      // Each prep job = one Mongo doc fetch + ONE summarize LLM call + a
+      // deterministic chunk + a Flow enqueue. It's lighter than a chunk job
+      // (which does ~4 LLM calls), so we can run more in parallel — 10 keeps
+      // prep comfortably AHEAD of the downstream chunk queue (concurrency 5),
+      // which is the real throughput cap. Going much higher than that buys
+      // little: prep would just drain faster into a chunk queue that still
+      // processes 5 at a time, while adding concurrent okf-be /internal/prepDoc
+      // load + concurrent summarize calls against the shared OpenAI TPM budget.
+      concurrency: 10,
+      lockDuration: 3e5,
+      // 5 min — generous ceiling over the 180s summarizer timeout
+      maxStalledCount: 1
+    }
+  },
   AI_AUTO_ANNOTATE_QUEUE: {
     // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
     // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
@@ -1769,6 +1821,7 @@ var buildFilterConfigurations = ({ groups, type, selectedTpls, allTpls, isRollup
             scope: section.filterScope
           };
         };
+        const isSelfTerminalTag = element.rollupPath.length === 1 && !isLastTagProfile && !isTerminalValuePath;
         return {
           filterId: `ROLLUP_${section.filterScope}_${element.rollupPath.join(
             "_"
@@ -1782,9 +1835,14 @@ var buildFilterConfigurations = ({ groups, type, selectedTpls, allTpls, isRollup
             scope: section.filterScope
           }),
           contentType: element.rollupPath[0],
-          path: `tags.${element.rollupPath[1]}`,
+          path: `tags.${isSelfTerminalTag ? element.rollupPath[0] : element.rollupPath[1]}`,
           source: getSourceConfig(),
-          target: {
+          target: isSelfTerminalTag ? {
+            filterType: "tagType",
+            tagType: element.rollupPath[0],
+            rollupPath: element.rollupPath,
+            ...isAnno && { isAnno: true }
+          } : {
             filterType: "nestedRollupTagType",
             rollupPath: element.rollupPath,
             ...isAnno && { isAnno: true }

package/dist/universal.mjs

@@ -896,6 +896,58 @@ var BASE_BULLMQ_CONFIG = {
       }
     }
   },
+  AI_AUTO_ANNOTATE_PREP_QUEUE: {
+    // PREP queue — the front of the per-doc Flow pipeline. The FE-facing
+    // /AIAutoAnnotate handler no longer fetches + summarizes + chunks +
+    // enqueues each doc inline (that serial, LLM-heavy loop blocked the HTTP
+    // request and risked App Engine clipping large batches mid-loop, silently
+    // dropping every doc past the cutoff). Instead the handler computes the
+    // batch-level prereqs ONCE (7th criterion + platform-derived categories)
+    // and enqueues one PREP job per doc here, then returns fast.
+    //
+    // Each prep job's worker (okf-sub) calls okf-be /internal/prepDoc, which
+    // does the per-doc work: fetch → summarizeDocument (warms the doc-level
+    // summary cache) → chunk fields → enqueue that doc's Flow (parent on
+    // AI_AUTO_ANNOTATE_QUEUE + one child per chunk on the CHUNK queue). So the
+    // Flow parents now get created rolling, as each prep job completes, rather
+    // than all up-front in the request. A prep failure isolates to its one doc
+    // (that doc is skipped); sibling prep jobs keep going.
+    id: "ai-auto-annotate-prep-queue",
+    queueConfig: {
+      defaultJobOptions: {
+        // attempts:1 — a retry would re-summarize (cache-HIT, cheap) but then
+        // re-enqueue the doc's Flow, double-annotating the doc. The rest of
+        // the per-doc Flow pipeline is attempts:1 for the same fail-loud
+        // reason (a retry just re-fires expensive LLM work).
+        attempts: 1,
+        backoff: {
+          type: "exponential",
+          delay: 5e3
+        },
+        removeOnComplete: 30,
+        removeOnFail: 100
+      },
+      streams: {
+        events: {
+          maxLen: 10
+        }
+      }
+    },
+    workerConfig: {
+      // Each prep job = one Mongo doc fetch + ONE summarize LLM call + a
+      // deterministic chunk + a Flow enqueue. It's lighter than a chunk job
+      // (which does ~4 LLM calls), so we can run more in parallel — 10 keeps
+      // prep comfortably AHEAD of the downstream chunk queue (concurrency 5),
+      // which is the real throughput cap. Going much higher than that buys
+      // little: prep would just drain faster into a chunk queue that still
+      // processes 5 at a time, while adding concurrent okf-be /internal/prepDoc
+      // load + concurrent summarize calls against the shared OpenAI TPM budget.
+      concurrency: 10,
+      lockDuration: 3e5,
+      // 5 min — generous ceiling over the 180s summarizer timeout
+      maxStalledCount: 1
+    }
+  },
   AI_AUTO_ANNOTATE_QUEUE: {
     // PARENT queue in the per-doc Flow. One job per doc; child jobs (one
     // per chunk) live on AI_AUTO_ANNOTATE_CHUNK_QUEUE. BullMQ Flow fires
@@ -1699,6 +1751,7 @@ var buildFilterConfigurations = ({ groups, type, selectedTpls, allTpls, isRollup
             scope: section.filterScope
           };
         };
+        const isSelfTerminalTag = element.rollupPath.length === 1 && !isLastTagProfile && !isTerminalValuePath;
         return {
           filterId: `ROLLUP_${section.filterScope}_${element.rollupPath.join(
             "_"
@@ -1712,9 +1765,14 @@ var buildFilterConfigurations = ({ groups, type, selectedTpls, allTpls, isRollup
             scope: section.filterScope
           }),
           contentType: element.rollupPath[0],
-          path: `tags.${element.rollupPath[1]}`,
+          path: `tags.${isSelfTerminalTag ? element.rollupPath[0] : element.rollupPath[1]}`,
           source: getSourceConfig(),
-          target: {
+          target: isSelfTerminalTag ? {
+            filterType: "tagType",
+            tagType: element.rollupPath[0],
+            rollupPath: element.rollupPath,
+            ...isAnno && { isAnno: true }
+          } : {
             filterType: "nestedRollupTagType",
             rollupPath: element.rollupPath,
             ...isAnno && { isAnno: true }

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.41.0",
+  "version": "1.41.2",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",