@okf/ootils 1.9.0 → 1.10.0

package/dist/universal.d.mts

@@ -29,6 +29,48 @@ interface RichTextValue {
     [key: string]: any;
 }
 type TagNameInput = string | number | RichTextValue | null | undefined;
+/**
+ * Generate a normalized tag ID from a tag name
+ *
+ * This function performs aggressive normalization to ensure that similar-looking
+ * tag names produce the same ID, preventing duplicates and collision errors.
+ *
+ * @param tagName - The tag name (string, number, or rich text object)
+ * @returns A normalized tag ID string
+ *
+ * @example
+ * ```typescript
+ * genTagId('Post-COVID')     // → 'post_covid'
+ * genTagId('Post—COVID')     // → 'post_covid' (em-dash)
+ * genTagId('Post–COVID')     // → 'post_covid' (en-dash)
+ * genTagId('café')           // → 'cafe'
+ * genTagId('CAFÉ')           // → 'cafe'
+ * genTagId('Café')           // → 'cafe'
+ * genTagId('C++')            // → 'cplusplus'
+ * genTagId('New  York')      // → 'new_york' (double space)
+ * genTagId('José Müller')    // → 'jose_muller'
+ * ```
+ *
+ * Normalization rules (applied in order):
+ * 1. Convert rich text to plain text
+ * 2. Normalize Unicode to NFD (Canonical Decomposition)
+ * 3. Remove all diacritical marks (accents, umlauts, etc.)
+ * 4. Trim whitespace from both ends
+ * 5. Convert to lowercase
+ * 6. Replace all non-alphanumeric characters with underscore
+ * 7. Replace '+' with 'plus'
+ * 8. Remove leading/trailing underscores
+ *
+ * Character variants that normalize to the same ID:
+ * - Dashes: hyphen (-), en-dash (–), em-dash (—), minus (−)
+ * - Whitespace: space, multiple spaces, tab, newline, non-breaking space
+ * - Quotes: straight ("), curly left ("), curly right ("), backtick (`)
+ * - Apostrophes: straight ('), curly (')
+ * - Accents: café/cafe, José/Jose, Müller/Muller, naïve/naive
+ * - Case: TestTag/testtag/TESTTAG
+ *
+ * @see TAG_NORMALIZATION_GUIDE.md in okf-be for comprehensive documentation
+ */
 declare const genTagId: (tagName: TagNameInput) => string;
 
 /**
@@ -277,13 +319,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_1 {
             let concurrency_1: number;
             export { concurrency_1 as concurrency };
-            export namespace limiter_1 {
-                let max_1: number;
-                export { max_1 as max };
-                let duration_1: number;
-                export { duration_1 as duration };
-            }
-            export { limiter_1 as limiter };
         }
         export { workerConfig_1 as workerConfig };
     }
@@ -320,13 +355,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_2 {
             let concurrency_2: number;
             export { concurrency_2 as concurrency };
-            export namespace limiter_2 {
-                let max_2: number;
-                export { max_2 as max };
-                let duration_2: number;
-                export { duration_2 as duration };
-            }
-            export { limiter_2 as limiter };
         }
         export { workerConfig_2 as workerConfig };
     }
@@ -363,13 +391,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_3 {
             let concurrency_3: number;
             export { concurrency_3 as concurrency };
-            export namespace limiter_3 {
-                let max_3: number;
-                export { max_3 as max };
-                let duration_3: number;
-                export { duration_3 as duration };
-            }
-            export { limiter_3 as limiter };
         }
         export { workerConfig_3 as workerConfig };
     }
@@ -406,13 +427,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_4 {
             let concurrency_4: number;
             export { concurrency_4 as concurrency };
-            export namespace limiter_4 {
-                let max_4: number;
-                export { max_4 as max };
-                let duration_4: number;
-                export { duration_4 as duration };
-            }
-            export { limiter_4 as limiter };
         }
         export { workerConfig_4 as workerConfig };
     }
@@ -449,13 +463,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_5 {
             let concurrency_5: number;
             export { concurrency_5 as concurrency };
-            export namespace limiter_5 {
-                let max_5: number;
-                export { max_5 as max };
-                let duration_5: number;
-                export { duration_5 as duration };
-            }
-            export { limiter_5 as limiter };
         }
         export { workerConfig_5 as workerConfig };
     }
@@ -492,13 +499,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_6 {
             let concurrency_6: number;
             export { concurrency_6 as concurrency };
-            export namespace limiter_6 {
-                let max_6: number;
-                export { max_6 as max };
-                let duration_6: number;
-                export { duration_6 as duration };
-            }
-            export { limiter_6 as limiter };
         }
         export { workerConfig_6 as workerConfig };
     }
@@ -535,13 +535,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_7 {
             let concurrency_7: number;
             export { concurrency_7 as concurrency };
-            export namespace limiter_7 {
-                let max_7: number;
-                export { max_7 as max };
-                let duration_7: number;
-                export { duration_7 as duration };
-            }
-            export { limiter_7 as limiter };
         }
         export { workerConfig_7 as workerConfig };
     }
@@ -578,13 +571,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_8 {
             let concurrency_8: number;
             export { concurrency_8 as concurrency };
-            export namespace limiter_8 {
-                let max_8: number;
-                export { max_8 as max };
-                let duration_8: number;
-                export { duration_8 as duration };
-            }
-            export { limiter_8 as limiter };
         }
         export { workerConfig_8 as workerConfig };
     }

package/dist/universal.d.ts

@@ -29,6 +29,48 @@ interface RichTextValue {
     [key: string]: any;
 }
 type TagNameInput = string | number | RichTextValue | null | undefined;
+/**
+ * Generate a normalized tag ID from a tag name
+ *
+ * This function performs aggressive normalization to ensure that similar-looking
+ * tag names produce the same ID, preventing duplicates and collision errors.
+ *
+ * @param tagName - The tag name (string, number, or rich text object)
+ * @returns A normalized tag ID string
+ *
+ * @example
+ * ```typescript
+ * genTagId('Post-COVID')     // → 'post_covid'
+ * genTagId('Post—COVID')     // → 'post_covid' (em-dash)
+ * genTagId('Post–COVID')     // → 'post_covid' (en-dash)
+ * genTagId('café')           // → 'cafe'
+ * genTagId('CAFÉ')           // → 'cafe'
+ * genTagId('Café')           // → 'cafe'
+ * genTagId('C++')            // → 'cplusplus'
+ * genTagId('New  York')      // → 'new_york' (double space)
+ * genTagId('José Müller')    // → 'jose_muller'
+ * ```
+ *
+ * Normalization rules (applied in order):
+ * 1. Convert rich text to plain text
+ * 2. Normalize Unicode to NFD (Canonical Decomposition)
+ * 3. Remove all diacritical marks (accents, umlauts, etc.)
+ * 4. Trim whitespace from both ends
+ * 5. Convert to lowercase
+ * 6. Replace all non-alphanumeric characters with underscore
+ * 7. Replace '+' with 'plus'
+ * 8. Remove leading/trailing underscores
+ *
+ * Character variants that normalize to the same ID:
+ * - Dashes: hyphen (-), en-dash (–), em-dash (—), minus (−)
+ * - Whitespace: space, multiple spaces, tab, newline, non-breaking space
+ * - Quotes: straight ("), curly left ("), curly right ("), backtick (`)
+ * - Apostrophes: straight ('), curly (')
+ * - Accents: café/cafe, José/Jose, Müller/Muller, naïve/naive
+ * - Case: TestTag/testtag/TESTTAG
+ *
+ * @see TAG_NORMALIZATION_GUIDE.md in okf-be for comprehensive documentation
+ */
 declare const genTagId: (tagName: TagNameInput) => string;
 
 /**
@@ -277,13 +319,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_1 {
             let concurrency_1: number;
             export { concurrency_1 as concurrency };
-            export namespace limiter_1 {
-                let max_1: number;
-                export { max_1 as max };
-                let duration_1: number;
-                export { duration_1 as duration };
-            }
-            export { limiter_1 as limiter };
         }
         export { workerConfig_1 as workerConfig };
     }
@@ -320,13 +355,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_2 {
             let concurrency_2: number;
             export { concurrency_2 as concurrency };
-            export namespace limiter_2 {
-                let max_2: number;
-                export { max_2 as max };
-                let duration_2: number;
-                export { duration_2 as duration };
-            }
-            export { limiter_2 as limiter };
         }
         export { workerConfig_2 as workerConfig };
     }
@@ -363,13 +391,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_3 {
             let concurrency_3: number;
             export { concurrency_3 as concurrency };
-            export namespace limiter_3 {
-                let max_3: number;
-                export { max_3 as max };
-                let duration_3: number;
-                export { duration_3 as duration };
-            }
-            export { limiter_3 as limiter };
         }
         export { workerConfig_3 as workerConfig };
     }
@@ -406,13 +427,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_4 {
             let concurrency_4: number;
             export { concurrency_4 as concurrency };
-            export namespace limiter_4 {
-                let max_4: number;
-                export { max_4 as max };
-                let duration_4: number;
-                export { duration_4 as duration };
-            }
-            export { limiter_4 as limiter };
         }
         export { workerConfig_4 as workerConfig };
     }
@@ -449,13 +463,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_5 {
             let concurrency_5: number;
             export { concurrency_5 as concurrency };
-            export namespace limiter_5 {
-                let max_5: number;
-                export { max_5 as max };
-                let duration_5: number;
-                export { duration_5 as duration };
-            }
-            export { limiter_5 as limiter };
         }
         export { workerConfig_5 as workerConfig };
     }
@@ -492,13 +499,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_6 {
             let concurrency_6: number;
             export { concurrency_6 as concurrency };
-            export namespace limiter_6 {
-                let max_6: number;
-                export { max_6 as max };
-                let duration_6: number;
-                export { duration_6 as duration };
-            }
-            export { limiter_6 as limiter };
         }
         export { workerConfig_6 as workerConfig };
     }
@@ -535,13 +535,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_7 {
             let concurrency_7: number;
             export { concurrency_7 as concurrency };
-            export namespace limiter_7 {
-                let max_7: number;
-                export { max_7 as max };
-                let duration_7: number;
-                export { duration_7 as duration };
-            }
-            export { limiter_7 as limiter };
         }
         export { workerConfig_7 as workerConfig };
     }
@@ -578,13 +571,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_8 {
             let concurrency_8: number;
             export { concurrency_8 as concurrency };
-            export namespace limiter_8 {
-                let max_8: number;
-                export { max_8 as max };
-                let duration_8: number;
-                export { duration_8 as duration };
-            }
-            export { limiter_8 as limiter };
         }
         export { workerConfig_8 as workerConfig };
     }

package/dist/universal.js

@@ -184,6 +184,8 @@ var convertFromRichText = (value) => {
 };
 var genTagId = (tagName) => {
   let toReturn = convertFromRichText(tagName);
+  toReturn = toReturn.normalize("NFD");
+  toReturn = toReturn.replace(/\p{Mn}/gu, "");
   const regex = /[^\p{L}\p{N}\+]+/gui;
   toReturn = toReturn.trim().toLowerCase().replace(regex, "_");
   toReturn = toReturn.replace(/\+/g, "plus");
@@ -475,14 +477,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 10,
-      // Process 10 jobs at once for chunk processing
-      limiter: {
-        max: 200,
-        // Max 5 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 50
     }
   },
   CREATE_ANNOS_QUEUE: {
@@ -505,14 +500,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 10,
-      // Process 10 jobs at once for chunk processing
-      limiter: {
-        max: 100,
-        // Max 50 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 50
     }
   },
   CONTENT_ENHANCE_AND_EMBED_QUEUE: {
@@ -535,13 +523,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 1,
-      limiter: {
-        max: 200,
-        // Max 50 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 1
     }
   },
   DIRECT_DATA_IMPORT_QUEUE: {
@@ -564,14 +546,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 1,
-      // Cannot mess with this else duplicate options in tpl, maybe even duplicate tags
-      limiter: {
-        max: 20,
-        // Max 5 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 1
     }
   },
   AI_CATEGORIZE_QUEUE: {
@@ -594,13 +569,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 1,
-      limiter: {
-        max: 100,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 1
     }
   },
   ANNOS_ELASTIC_SYNC_QUEUE: {
@@ -623,13 +592,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 5,
-      limiter: {
-        max: 100,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 20
     }
   },
   CHUNKS_ELASTIC_SYNC_QUEUE: {
@@ -652,13 +615,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 5,
-      limiter: {
-        max: 200,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 20
     }
   },
   CONTENT_ELASTIC_SYNC_QUEUE: {
@@ -681,13 +638,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 5,
-      limiter: {
-        max: 200,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 20
     }
   },
   REINDEX_QUEUE: {

package/dist/universal.mjs

@@ -148,6 +148,8 @@ var convertFromRichText = (value) => {
 };
 var genTagId = (tagName) => {
   let toReturn = convertFromRichText(tagName);
+  toReturn = toReturn.normalize("NFD");
+  toReturn = toReturn.replace(/\p{Mn}/gu, "");
   const regex = /[^\p{L}\p{N}\+]+/gui;
   toReturn = toReturn.trim().toLowerCase().replace(regex, "_");
   toReturn = toReturn.replace(/\+/g, "plus");
@@ -439,14 +441,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 10,
-      // Process 10 jobs at once for chunk processing
-      limiter: {
-        max: 200,
-        // Max 5 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 50
     }
   },
   CREATE_ANNOS_QUEUE: {
@@ -469,14 +464,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 10,
-      // Process 10 jobs at once for chunk processing
-      limiter: {
-        max: 100,
-        // Max 50 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 50
     }
   },
   CONTENT_ENHANCE_AND_EMBED_QUEUE: {
@@ -499,13 +487,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 1,
-      limiter: {
-        max: 200,
-        // Max 50 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 1
     }
   },
   DIRECT_DATA_IMPORT_QUEUE: {
@@ -528,14 +510,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 1,
-      // Cannot mess with this else duplicate options in tpl, maybe even duplicate tags
-      limiter: {
-        max: 20,
-        // Max 5 jobs per...
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 1
     }
   },
   AI_CATEGORIZE_QUEUE: {
@@ -558,13 +533,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 1,
-      limiter: {
-        max: 100,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 1
     }
   },
   ANNOS_ELASTIC_SYNC_QUEUE: {
@@ -587,13 +556,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 5,
-      limiter: {
-        max: 100,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 20
     }
   },
   CHUNKS_ELASTIC_SYNC_QUEUE: {
@@ -616,13 +579,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 5,
-      limiter: {
-        max: 200,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 20
     }
   },
   CONTENT_ELASTIC_SYNC_QUEUE: {
@@ -645,13 +602,7 @@ var BASE_BULLMQ_CONFIG = {
       }
     },
     workerConfig: {
-      concurrency: 5,
-      limiter: {
-        max: 200,
-        // (lets always keep this same as content enhance & embed since it comes immediately after)
-        duration: 6e4
-        // ...60 seconds (higher throughput for chunking)
-      }
+      concurrency: 20
     }
   },
   REINDEX_QUEUE: {

package/package.json

@@ -3,7 +3,7 @@
   "publishConfig": {
     "access": "public"
   },
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Utility functions for both browser and Node.js",
   "main": "dist/index.js",
   "module": "dist/index.mjs",