@okf/ootils 1.9.0 → 1.10.0

package/dist/node.d.mts

@@ -36,6 +36,48 @@ interface RichTextValue {
     [key: string]: any;
 }
 type TagNameInput = string | number | RichTextValue | null | undefined;
+/**
+ * Generate a normalized tag ID from a tag name
+ *
+ * This function performs aggressive normalization to ensure that similar-looking
+ * tag names produce the same ID, preventing duplicates and collision errors.
+ *
+ * @param tagName - The tag name (string, number, or rich text object)
+ * @returns A normalized tag ID string
+ *
+ * @example
+ * ```typescript
+ * genTagId('Post-COVID')     // → 'post_covid'
+ * genTagId('Post—COVID')     // → 'post_covid' (em-dash)
+ * genTagId('Post–COVID')     // → 'post_covid' (en-dash)
+ * genTagId('café')           // → 'cafe'
+ * genTagId('CAFÉ')           // → 'cafe'
+ * genTagId('Café')           // → 'cafe'
+ * genTagId('C++')            // → 'cplusplus'
+ * genTagId('New  York')      // → 'new_york' (double space)
+ * genTagId('José Müller')    // → 'jose_muller'
+ * ```
+ *
+ * Normalization rules (applied in order):
+ * 1. Convert rich text to plain text
+ * 2. Normalize Unicode to NFD (Canonical Decomposition)
+ * 3. Remove all diacritical marks (accents, umlauts, etc.)
+ * 4. Trim whitespace from both ends
+ * 5. Convert to lowercase
+ * 6. Replace all non-alphanumeric characters with underscore
+ * 7. Replace '+' with 'plus'
+ * 8. Remove leading/trailing underscores
+ *
+ * Character variants that normalize to the same ID:
+ * - Dashes: hyphen (-), en-dash (–), em-dash (—), minus (−)
+ * - Whitespace: space, multiple spaces, tab, newline, non-breaking space
+ * - Quotes: straight ("), curly left ("), curly right ("), backtick (`)
+ * - Apostrophes: straight ('), curly (')
+ * - Accents: café/cafe, José/Jose, Müller/Muller, naïve/naive
+ * - Case: TestTag/testtag/TESTTAG
+ *
+ * @see TAG_NORMALIZATION_GUIDE.md in okf-be for comprehensive documentation
+ */
 declare const genTagId: (tagName: TagNameInput) => string;
 
 /**
@@ -284,13 +326,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_1 {
             let concurrency_1: number;
             export { concurrency_1 as concurrency };
-            export namespace limiter_1 {
-                let max_1: number;
-                export { max_1 as max };
-                let duration_1: number;
-                export { duration_1 as duration };
-            }
-            export { limiter_1 as limiter };
         }
         export { workerConfig_1 as workerConfig };
     }
@@ -327,13 +362,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_2 {
             let concurrency_2: number;
             export { concurrency_2 as concurrency };
-            export namespace limiter_2 {
-                let max_2: number;
-                export { max_2 as max };
-                let duration_2: number;
-                export { duration_2 as duration };
-            }
-            export { limiter_2 as limiter };
         }
         export { workerConfig_2 as workerConfig };
     }
@@ -370,13 +398,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_3 {
             let concurrency_3: number;
             export { concurrency_3 as concurrency };
-            export namespace limiter_3 {
-                let max_3: number;
-                export { max_3 as max };
-                let duration_3: number;
-                export { duration_3 as duration };
-            }
-            export { limiter_3 as limiter };
         }
         export { workerConfig_3 as workerConfig };
     }
@@ -413,13 +434,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_4 {
             let concurrency_4: number;
             export { concurrency_4 as concurrency };
-            export namespace limiter_4 {
-                let max_4: number;
-                export { max_4 as max };
-                let duration_4: number;
-                export { duration_4 as duration };
-            }
-            export { limiter_4 as limiter };
         }
         export { workerConfig_4 as workerConfig };
     }
@@ -456,13 +470,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_5 {
             let concurrency_5: number;
             export { concurrency_5 as concurrency };
-            export namespace limiter_5 {
-                let max_5: number;
-                export { max_5 as max };
-                let duration_5: number;
-                export { duration_5 as duration };
-            }
-            export { limiter_5 as limiter };
         }
         export { workerConfig_5 as workerConfig };
     }
@@ -499,13 +506,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_6 {
             let concurrency_6: number;
             export { concurrency_6 as concurrency };
-            export namespace limiter_6 {
-                let max_6: number;
-                export { max_6 as max };
-                let duration_6: number;
-                export { duration_6 as duration };
-            }
-            export { limiter_6 as limiter };
         }
         export { workerConfig_6 as workerConfig };
     }
@@ -542,13 +542,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_7 {
             let concurrency_7: number;
             export { concurrency_7 as concurrency };
-            export namespace limiter_7 {
-                let max_7: number;
-                export { max_7 as max };
-                let duration_7: number;
-                export { duration_7 as duration };
-            }
-            export { limiter_7 as limiter };
         }
         export { workerConfig_7 as workerConfig };
     }
@@ -585,13 +578,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_8 {
             let concurrency_8: number;
             export { concurrency_8 as concurrency };
-            export namespace limiter_8 {
-                let max_8: number;
-                export { max_8 as max };
-                let duration_8: number;
-                export { duration_8 as duration };
-            }
-            export { limiter_8 as limiter };
         }
         export { workerConfig_8 as workerConfig };
     }

package/dist/node.d.ts

@@ -36,6 +36,48 @@ interface RichTextValue {
     [key: string]: any;
 }
 type TagNameInput = string | number | RichTextValue | null | undefined;
+/**
+ * Generate a normalized tag ID from a tag name
+ *
+ * This function performs aggressive normalization to ensure that similar-looking
+ * tag names produce the same ID, preventing duplicates and collision errors.
+ *
+ * @param tagName - The tag name (string, number, or rich text object)
+ * @returns A normalized tag ID string
+ *
+ * @example
+ * ```typescript
+ * genTagId('Post-COVID')     // → 'post_covid'
+ * genTagId('Post—COVID')     // → 'post_covid' (em-dash)
+ * genTagId('Post–COVID')     // → 'post_covid' (en-dash)
+ * genTagId('café')           // → 'cafe'
+ * genTagId('CAFÉ')           // → 'cafe'
+ * genTagId('Café')           // → 'cafe'
+ * genTagId('C++')            // → 'cplusplus'
+ * genTagId('New  York')      // → 'new_york' (double space)
+ * genTagId('José Müller')    // → 'jose_muller'
+ * ```
+ *
+ * Normalization rules (applied in order):
+ * 1. Convert rich text to plain text
+ * 2. Normalize Unicode to NFD (Canonical Decomposition)
+ * 3. Remove all diacritical marks (accents, umlauts, etc.)
+ * 4. Trim whitespace from both ends
+ * 5. Convert to lowercase
+ * 6. Replace all non-alphanumeric characters with underscore
+ * 7. Replace '+' with 'plus'
+ * 8. Remove leading/trailing underscores
+ *
+ * Character variants that normalize to the same ID:
+ * - Dashes: hyphen (-), en-dash (–), em-dash (—), minus (−)
+ * - Whitespace: space, multiple spaces, tab, newline, non-breaking space
+ * - Quotes: straight ("), curly left ("), curly right ("), backtick (`)
+ * - Apostrophes: straight ('), curly (')
+ * - Accents: café/cafe, José/Jose, Müller/Muller, naïve/naive
+ * - Case: TestTag/testtag/TESTTAG
+ *
+ * @see TAG_NORMALIZATION_GUIDE.md in okf-be for comprehensive documentation
+ */
 declare const genTagId: (tagName: TagNameInput) => string;
 
 /**
@@ -284,13 +326,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_1 {
             let concurrency_1: number;
             export { concurrency_1 as concurrency };
-            export namespace limiter_1 {
-                let max_1: number;
-                export { max_1 as max };
-                let duration_1: number;
-                export { duration_1 as duration };
-            }
-            export { limiter_1 as limiter };
         }
         export { workerConfig_1 as workerConfig };
     }
@@ -327,13 +362,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_2 {
             let concurrency_2: number;
             export { concurrency_2 as concurrency };
-            export namespace limiter_2 {
-                let max_2: number;
-                export { max_2 as max };
-                let duration_2: number;
-                export { duration_2 as duration };
-            }
-            export { limiter_2 as limiter };
         }
         export { workerConfig_2 as workerConfig };
     }
@@ -370,13 +398,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_3 {
             let concurrency_3: number;
             export { concurrency_3 as concurrency };
-            export namespace limiter_3 {
-                let max_3: number;
-                export { max_3 as max };
-                let duration_3: number;
-                export { duration_3 as duration };
-            }
-            export { limiter_3 as limiter };
         }
         export { workerConfig_3 as workerConfig };
     }
@@ -413,13 +434,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_4 {
             let concurrency_4: number;
             export { concurrency_4 as concurrency };
-            export namespace limiter_4 {
-                let max_4: number;
-                export { max_4 as max };
-                let duration_4: number;
-                export { duration_4 as duration };
-            }
-            export { limiter_4 as limiter };
         }
         export { workerConfig_4 as workerConfig };
     }
@@ -456,13 +470,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_5 {
             let concurrency_5: number;
             export { concurrency_5 as concurrency };
-            export namespace limiter_5 {
-                let max_5: number;
-                export { max_5 as max };
-                let duration_5: number;
-                export { duration_5 as duration };
-            }
-            export { limiter_5 as limiter };
         }
         export { workerConfig_5 as workerConfig };
     }
@@ -499,13 +506,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_6 {
             let concurrency_6: number;
             export { concurrency_6 as concurrency };
-            export namespace limiter_6 {
-                let max_6: number;
-                export { max_6 as max };
-                let duration_6: number;
-                export { duration_6 as duration };
-            }
-            export { limiter_6 as limiter };
         }
         export { workerConfig_6 as workerConfig };
     }
@@ -542,13 +542,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_7 {
             let concurrency_7: number;
             export { concurrency_7 as concurrency };
-            export namespace limiter_7 {
-                let max_7: number;
-                export { max_7 as max };
-                let duration_7: number;
-                export { duration_7 as duration };
-            }
-            export { limiter_7 as limiter };
         }
         export { workerConfig_7 as workerConfig };
     }
@@ -585,13 +578,6 @@ declare namespace BASE_BULLMQ_CONFIG {
         export namespace workerConfig_8 {
             let concurrency_8: number;
             export { concurrency_8 as concurrency };
-            export namespace limiter_8 {
-                let max_8: number;
-                export { max_8 as max };
-                let duration_8: number;
-                export { duration_8 as duration };
-            }
-            export { limiter_8 as limiter };
         }
         export { workerConfig_8 as workerConfig };
     }

package/dist/node.js

@@ -96,14 +96,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 10,
-          // Process 10 jobs at once for chunk processing
-          limiter: {
-            max: 200,
-            // Max 5 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 50
         }
       },
       CREATE_ANNOS_QUEUE: {
@@ -126,14 +119,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 10,
-          // Process 10 jobs at once for chunk processing
-          limiter: {
-            max: 100,
-            // Max 50 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 50
         }
       },
       CONTENT_ENHANCE_AND_EMBED_QUEUE: {
@@ -156,13 +142,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1,
-          limiter: {
-            max: 200,
-            // Max 50 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 1
         }
       },
       DIRECT_DATA_IMPORT_QUEUE: {
@@ -185,14 +165,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1,
-          // Cannot mess with this else duplicate options in tpl, maybe even duplicate tags
-          limiter: {
-            max: 20,
-            // Max 5 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 1
         }
       },
       AI_CATEGORIZE_QUEUE: {
@@ -215,13 +188,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1,
-          limiter: {
-            max: 100,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 1
         }
       },
       ANNOS_ELASTIC_SYNC_QUEUE: {
@@ -244,13 +211,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 5,
-          limiter: {
-            max: 100,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 20
         }
       },
       CHUNKS_ELASTIC_SYNC_QUEUE: {
@@ -273,13 +234,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 5,
-          limiter: {
-            max: 200,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 20
         }
       },
       CONTENT_ELASTIC_SYNC_QUEUE: {
@@ -302,13 +257,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 5,
-          limiter: {
-            max: 200,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 20
         }
       },
       REINDEX_QUEUE: {
@@ -1613,6 +1562,8 @@ var convertFromRichText = (value) => {
 };
 var genTagId = (tagName) => {
   let toReturn = convertFromRichText(tagName);
+  toReturn = toReturn.normalize("NFD");
+  toReturn = toReturn.replace(/\p{Mn}/gu, "");
   const regex = /[^\p{L}\p{N}\+]+/gui;
   toReturn = toReturn.trim().toLowerCase().replace(regex, "_");
   toReturn = toReturn.replace(/\+/g, "plus");

package/dist/node.mjs

@@ -101,14 +101,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 10,
-          // Process 10 jobs at once for chunk processing
-          limiter: {
-            max: 200,
-            // Max 5 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 50
         }
       },
       CREATE_ANNOS_QUEUE: {
@@ -131,14 +124,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 10,
-          // Process 10 jobs at once for chunk processing
-          limiter: {
-            max: 100,
-            // Max 50 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 50
         }
       },
       CONTENT_ENHANCE_AND_EMBED_QUEUE: {
@@ -161,13 +147,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1,
-          limiter: {
-            max: 200,
-            // Max 50 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 1
         }
       },
       DIRECT_DATA_IMPORT_QUEUE: {
@@ -190,14 +170,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1,
-          // Cannot mess with this else duplicate options in tpl, maybe even duplicate tags
-          limiter: {
-            max: 20,
-            // Max 5 jobs per...
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 1
         }
       },
       AI_CATEGORIZE_QUEUE: {
@@ -220,13 +193,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 1,
-          limiter: {
-            max: 100,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 1
         }
       },
       ANNOS_ELASTIC_SYNC_QUEUE: {
@@ -249,13 +216,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 5,
-          limiter: {
-            max: 100,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 20
         }
       },
       CHUNKS_ELASTIC_SYNC_QUEUE: {
@@ -278,13 +239,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 5,
-          limiter: {
-            max: 200,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 20
         }
       },
       CONTENT_ELASTIC_SYNC_QUEUE: {
@@ -307,13 +262,7 @@ var init_GLOBAL_BULLMQ_CONFIG = __esm({
           }
         },
         workerConfig: {
-          concurrency: 5,
-          limiter: {
-            max: 200,
-            // (lets always keep this same as content enhance & embed since it comes immediately after)
-            duration: 6e4
-            // ...60 seconds (higher throughput for chunking)
-          }
+          concurrency: 20
         }
       },
       REINDEX_QUEUE: {
@@ -1583,6 +1532,8 @@ var convertFromRichText = (value) => {
 };
 var genTagId = (tagName) => {
   let toReturn = convertFromRichText(tagName);
+  toReturn = toReturn.normalize("NFD");
+  toReturn = toReturn.replace(/\p{Mn}/gu, "");
   const regex = /[^\p{L}\p{N}\+]+/gui;
   toReturn = toReturn.trim().toLowerCase().replace(regex, "_");
   toReturn = toReturn.replace(/\+/g, "plus");