iraqi-dialect-bad-words 1.2.0 → 1.2.1

package/README.md

@@ -131,10 +131,13 @@ Matching is done per whole word, after normalizing both the text and the word li
 | Stretched letters | `عاااارة` |
 | Tatweel | `عـــارة` |
 | ة / ه, ى / ي, أ / إ / آ / ا | `عاره` |
+| Persian/Kurdish keyboard letters ی ک ە | `عیر`, `کس` |
+| Invisible characters hidden inside a word (zero-width, direction marks, soft hyphen) | `عا‌رة` |
 | Attached prefixes و ف ب ل ك ال | `والعارة`, `للعارة` |
 | Pronoun suffixes (with `stripSuffixes`) | `طيزهم`, `وطيزكم` |
 | Punctuation next to the word | `عارة!` `عارة،` |
-| Arabizi (Latin letters and numbers), including spelling variants | `ga7ba` = `qahba` = `gahba`, `kosomak` = `kusumak`, `3ayri` = `ayri` |
+| Digits glued to or inside an Arabic word | `عارة3`, `عا٣رة` |
+| Arabizi (Latin letters and numbers, Western or Arabic-Indic), including spelling variants | `ga7ba` = `ga٧ba` = `qahba` = `gahba`, `kosomak` = `kusumak`, `3ayri` = `ayri` |
 
 Because matching is whole-word, a clean word that merely contains a bad word is not flagged.
 

package/dist/index.d.mts

@@ -56,8 +56,10 @@ declare class IraqiBadWordsFilter {
     constructor(options?: string[] | IraqiBadWordsFilterOptions);
     /**
      * Normalizes Arabic text so spelling variants compare equal:
-     * removes diacritics (Tashkeel) and tatweel, unifies Alef/Teh Marbuta/Alef Maksura
-     * forms, and collapses stretched letters (عاااارة -> عارة).
+     * removes diacritics (Tashkeel), tatweel and invisible characters, unifies
+     * Alef/Teh Marbuta/Alef Maksura forms and the Persian/Kurdish keyboard letters
+     * ی ک ە, turns Arabic-Indic digits into 0-9, and collapses stretched letters
+     * (عاااارة -> عارة).
      */
     private normalizeArabic;
     /**
@@ -80,8 +82,13 @@ declare class IraqiBadWordsFilter {
      * A match cleared by a safe stem is returned with `suppressedBy` set.
      */
     private matchWord;
-    /** True when the word is profane, i.e. matched and not cleared by a safe stem. */
-    private isProfaneWord;
+    /**
+     * Finds every match in the text, including ones cleared by a safe stem.
+     * A token with invisible characters is matched whole first (عا‌رة -> عارة);
+     * if that finds nothing profane, each visible piece is tried on its own, so an
+     * invisible character between two words can't hide a bad word either.
+     */
+    private findMatches;
     /**
      * Checks if a given text contains profane words.
      */

package/dist/index.d.ts

@@ -56,8 +56,10 @@ declare class IraqiBadWordsFilter {
     constructor(options?: string[] | IraqiBadWordsFilterOptions);
     /**
      * Normalizes Arabic text so spelling variants compare equal:
-     * removes diacritics (Tashkeel) and tatweel, unifies Alef/Teh Marbuta/Alef Maksura
-     * forms, and collapses stretched letters (عاااارة -> عارة).
+     * removes diacritics (Tashkeel), tatweel and invisible characters, unifies
+     * Alef/Teh Marbuta/Alef Maksura forms and the Persian/Kurdish keyboard letters
+     * ی ک ە, turns Arabic-Indic digits into 0-9, and collapses stretched letters
+     * (عاااارة -> عارة).
      */
     private normalizeArabic;
     /**
@@ -80,8 +82,13 @@ declare class IraqiBadWordsFilter {
      * A match cleared by a safe stem is returned with `suppressedBy` set.
      */
     private matchWord;
-    /** True when the word is profane, i.e. matched and not cleared by a safe stem. */
-    private isProfaneWord;
+    /**
+     * Finds every match in the text, including ones cleared by a safe stem.
+     * A token with invisible characters is matched whole first (عا‌رة -> عارة);
+     * if that finds nothing profane, each visible piece is tried on its own, so an
+     * invisible character between two words can't hide a bad word either.
+     */
+    private findMatches;
     /**
      * Checks if a given text contains profane words.
      */

package/dist/index.js

@@ -239,7 +239,14 @@ var safe_stems_default = [
 ];
 
 // src/index.ts
-var WORD_PATTERN = /[A-Za-z0-9_À-ɏ؀-؋؍-ؚ؜-؞ؠ-٩ٮ-ۓە-ۿݐ-ݿࢠ-ࣿﭐ-﷿ﹰ-ﻼ]+/g;
+var INVISIBLE = "\\u00AD\\u061C\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF";
+var INVISIBLE_CHARS = new RegExp(`[${INVISIBLE}]`, "g");
+var HAS_INVISIBLE = new RegExp(`[${INVISIBLE}]`);
+var VISIBLE_RUN = new RegExp(`[^${INVISIBLE}]+`, "g");
+var WORD_PATTERN = new RegExp(
+  `[A-Za-z0-9_\xC0-\u024F\u0600-\u060B\u060D-\u061A\u061C-\u061E\u0620-\u0669\u066E-\u06D3\u06D5-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFC${INVISIBLE}]+`,
+  "g"
+);
 var CONJUNCTIONS = ["", "\u0648", "\u0641"];
 var PREPOSITIONS = ["", "\u0628", "\u0644", "\u0643"];
 var ARTICLES = ["", "\u0627\u0644"];
@@ -256,11 +263,13 @@ var IraqiBadWordsFilter = class {
   }
   /**
    * Normalizes Arabic text so spelling variants compare equal:
-   * removes diacritics (Tashkeel) and tatweel, unifies Alef/Teh Marbuta/Alef Maksura
-   * forms, and collapses stretched letters (عاااارة -> عارة).
+   * removes diacritics (Tashkeel), tatweel and invisible characters, unifies
+   * Alef/Teh Marbuta/Alef Maksura forms and the Persian/Kurdish keyboard letters
+   * ی ک ە, turns Arabic-Indic digits into 0-9, and collapses stretched letters
+   * (عاااارة -> عارة).
    */
   normalizeArabic(text) {
-    return text.replace(/[ً-ٰٟـ]/g, "").replace(/[أإآٱ]/g, "\u0627").replace(/ة/g, "\u0647").replace(/ى/g, "\u064A").replace(/(.)\1+/gu, "$1").toLowerCase();
+    return text.replace(/[\u064B-\u065F\u0670\u0640]/g, "").replace(INVISIBLE_CHARS, "").replace(/[أإآٱ]/g, "\u0627").replace(/[ةەہۀ]/g, "\u0647").replace(/[ىی]/g, "\u064A").replace(/[کڪ]/g, "\u0643").replace(/[٠-٩]/g, (digit) => String(digit.charCodeAt(0) - 1632)).replace(/[۰-۹]/g, (digit) => String(digit.charCodeAt(0) - 1776)).replace(/(.)\1+/gu, "$1").toLowerCase();
   }
   /**
    * Folds Arabizi (Arabic written in Latin letters and digits) so spelling
@@ -274,7 +283,8 @@ var IraqiBadWordsFilter = class {
   }
   /** Normalizes a word or token for comparison. */
   normalize(text) {
-    return this.foldArabizi(this.normalizeArabic(text));
+    const word = this.normalizeArabic(text);
+    return /[\u0600-\uFEFF]/.test(word) ? word.replace(/[0-9]/g, "").replace(/(.)\1+/gu, "$1") : this.foldArabizi(word);
   }
   /**
    * Adds words from one list. A word already present keeps its first source.
@@ -333,20 +343,43 @@ var IraqiBadWordsFilter = class {
     }
     return result;
   }
-  /** True when the word is profane, i.e. matched and not cleared by a safe stem. */
-  isProfaneWord(token) {
-    const match = this.matchWord(token);
-    return match !== null && match.suppressedBy === void 0;
+  /**
+   * Finds every match in the text, including ones cleared by a safe stem.
+   * A token with invisible characters is matched whole first (عا‌رة -> عارة);
+   * if that finds nothing profane, each visible piece is tried on its own, so an
+   * invisible character between two words can't hide a bad word either.
+   */
+  *findMatches(text) {
+    const pattern = new RegExp(WORD_PATTERN.source, "g");
+    let found;
+    while (found = pattern.exec(text)) {
+      const token = found[0];
+      const start = found.index;
+      const whole = this.matchWord(token);
+      const wholeMatch = whole && { token, start, end: start + token.length, ...whole };
+      if (whole && whole.suppressedBy === void 0 || !HAS_INVISIBLE.test(token)) {
+        if (wholeMatch) yield wholeMatch;
+        continue;
+      }
+      const pieces = [];
+      const piecePattern = new RegExp(VISIBLE_RUN.source, "g");
+      let piece;
+      while (piece = piecePattern.exec(token)) {
+        const match = this.matchWord(piece[0]);
+        const pieceStart = start + piece.index;
+        if (match) pieces.push({ token: piece[0], start: pieceStart, end: pieceStart + piece[0].length, ...match });
+      }
+      if (pieces.length) yield* pieces;
+      else if (wholeMatch) yield wholeMatch;
+    }
   }
   /**
    * Checks if a given text contains profane words.
    */
   isProfane(text) {
     if (!text) return false;
-    for (const token of text.match(WORD_PATTERN) ?? []) {
-      if (this.isProfaneWord(token)) {
-        return true;
-      }
+    for (const match of this.findMatches(text)) {
+      if (match.suppressedBy === void 0) return true;
     }
     return false;
   }
@@ -357,14 +390,8 @@ var IraqiBadWordsFilter = class {
   inspect(text, options = {}) {
     if (!text) return [];
     const matches = [];
-    const pattern = new RegExp(WORD_PATTERN.source, "g");
-    let found;
-    while (found = pattern.exec(text)) {
-      const token = found[0];
-      const match = this.matchWord(token);
-      if (match && (match.suppressedBy === void 0 || options.includeSuppressed)) {
-        matches.push({ token, start: found.index, end: found.index + token.length, ...match });
-      }
+    for (const match of this.findMatches(text)) {
+      if (match.suppressedBy === void 0 || options.includeSuppressed) matches.push(match);
     }
     return matches;
   }

package/dist/index.mjs

@@ -212,7 +212,14 @@ var safe_stems_default = [
 ];
 
 // src/index.ts
-var WORD_PATTERN = /[A-Za-z0-9_À-ɏ؀-؋؍-ؚ؜-؞ؠ-٩ٮ-ۓە-ۿݐ-ݿࢠ-ࣿﭐ-﷿ﹰ-ﻼ]+/g;
+var INVISIBLE = "\\u00AD\\u061C\\u200B-\\u200F\\u202A-\\u202E\\u2060-\\u2064\\u2066-\\u2069\\uFEFF";
+var INVISIBLE_CHARS = new RegExp(`[${INVISIBLE}]`, "g");
+var HAS_INVISIBLE = new RegExp(`[${INVISIBLE}]`);
+var VISIBLE_RUN = new RegExp(`[^${INVISIBLE}]+`, "g");
+var WORD_PATTERN = new RegExp(
+  `[A-Za-z0-9_\xC0-\u024F\u0600-\u060B\u060D-\u061A\u061C-\u061E\u0620-\u0669\u066E-\u06D3\u06D5-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFC${INVISIBLE}]+`,
+  "g"
+);
 var CONJUNCTIONS = ["", "\u0648", "\u0641"];
 var PREPOSITIONS = ["", "\u0628", "\u0644", "\u0643"];
 var ARTICLES = ["", "\u0627\u0644"];
@@ -229,11 +236,13 @@ var IraqiBadWordsFilter = class {
   }
   /**
    * Normalizes Arabic text so spelling variants compare equal:
-   * removes diacritics (Tashkeel) and tatweel, unifies Alef/Teh Marbuta/Alef Maksura
-   * forms, and collapses stretched letters (عاااارة -> عارة).
+   * removes diacritics (Tashkeel), tatweel and invisible characters, unifies
+   * Alef/Teh Marbuta/Alef Maksura forms and the Persian/Kurdish keyboard letters
+   * ی ک ە, turns Arabic-Indic digits into 0-9, and collapses stretched letters
+   * (عاااارة -> عارة).
    */
   normalizeArabic(text) {
-    return text.replace(/[ً-ٰٟـ]/g, "").replace(/[أإآٱ]/g, "\u0627").replace(/ة/g, "\u0647").replace(/ى/g, "\u064A").replace(/(.)\1+/gu, "$1").toLowerCase();
+    return text.replace(/[\u064B-\u065F\u0670\u0640]/g, "").replace(INVISIBLE_CHARS, "").replace(/[أإآٱ]/g, "\u0627").replace(/[ةەہۀ]/g, "\u0647").replace(/[ىی]/g, "\u064A").replace(/[کڪ]/g, "\u0643").replace(/[٠-٩]/g, (digit) => String(digit.charCodeAt(0) - 1632)).replace(/[۰-۹]/g, (digit) => String(digit.charCodeAt(0) - 1776)).replace(/(.)\1+/gu, "$1").toLowerCase();
   }
   /**
    * Folds Arabizi (Arabic written in Latin letters and digits) so spelling
@@ -247,7 +256,8 @@ var IraqiBadWordsFilter = class {
   }
   /** Normalizes a word or token for comparison. */
   normalize(text) {
-    return this.foldArabizi(this.normalizeArabic(text));
+    const word = this.normalizeArabic(text);
+    return /[\u0600-\uFEFF]/.test(word) ? word.replace(/[0-9]/g, "").replace(/(.)\1+/gu, "$1") : this.foldArabizi(word);
   }
   /**
    * Adds words from one list. A word already present keeps its first source.
@@ -306,20 +316,43 @@ var IraqiBadWordsFilter = class {
     }
     return result;
   }
-  /** True when the word is profane, i.e. matched and not cleared by a safe stem. */
-  isProfaneWord(token) {
-    const match = this.matchWord(token);
-    return match !== null && match.suppressedBy === void 0;
+  /**
+   * Finds every match in the text, including ones cleared by a safe stem.
+   * A token with invisible characters is matched whole first (عا‌رة -> عارة);
+   * if that finds nothing profane, each visible piece is tried on its own, so an
+   * invisible character between two words can't hide a bad word either.
+   */
+  *findMatches(text) {
+    const pattern = new RegExp(WORD_PATTERN.source, "g");
+    let found;
+    while (found = pattern.exec(text)) {
+      const token = found[0];
+      const start = found.index;
+      const whole = this.matchWord(token);
+      const wholeMatch = whole && { token, start, end: start + token.length, ...whole };
+      if (whole && whole.suppressedBy === void 0 || !HAS_INVISIBLE.test(token)) {
+        if (wholeMatch) yield wholeMatch;
+        continue;
+      }
+      const pieces = [];
+      const piecePattern = new RegExp(VISIBLE_RUN.source, "g");
+      let piece;
+      while (piece = piecePattern.exec(token)) {
+        const match = this.matchWord(piece[0]);
+        const pieceStart = start + piece.index;
+        if (match) pieces.push({ token: piece[0], start: pieceStart, end: pieceStart + piece[0].length, ...match });
+      }
+      if (pieces.length) yield* pieces;
+      else if (wholeMatch) yield wholeMatch;
+    }
   }
   /**
    * Checks if a given text contains profane words.
    */
   isProfane(text) {
     if (!text) return false;
-    for (const token of text.match(WORD_PATTERN) ?? []) {
-      if (this.isProfaneWord(token)) {
-        return true;
-      }
+    for (const match of this.findMatches(text)) {
+      if (match.suppressedBy === void 0) return true;
     }
     return false;
   }
@@ -330,14 +363,8 @@ var IraqiBadWordsFilter = class {
   inspect(text, options = {}) {
     if (!text) return [];
     const matches = [];
-    const pattern = new RegExp(WORD_PATTERN.source, "g");
-    let found;
-    while (found = pattern.exec(text)) {
-      const token = found[0];
-      const match = this.matchWord(token);
-      if (match && (match.suppressedBy === void 0 || options.includeSuppressed)) {
-        matches.push({ token, start: found.index, end: found.index + token.length, ...match });
-      }
+    for (const match of this.findMatches(text)) {
+      if (match.suppressedBy === void 0 || options.includeSuppressed) matches.push(match);
     }
     return matches;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "iraqi-dialect-bad-words",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "description": "An Iraqi dialect profanity filter for React, React Native Expo, and Node.js",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",