@sideid/id-profanity-filter 1.9.6 → 1.10.2

package/src/core/matcher.ts

@@ -5,24 +5,34 @@ import {
   FilterOptions,
 } from "../types";
 
-import { wordObjects, getWordsByFilter } from "../constants/wordList";
-import {
-  normalizeText,
-  escapeRegExp,
-  containsAnyWord,
-  detectSplitWords,
-} from "../utils/stringUtils";
-import {
-  createWordRegex,
-  addLeetSpeakVariations,
-  addIndonesianVariations,
-  addSplitVariations,
-} from "../utils/regexUtils";
+import { wordObjects } from "../constants/wordList";
+import { normalizeText } from "../utils/stringUtils";
+import { createWordRegex } from "../utils/regexUtils";
 import {
   findPossibleProfanityBySimiliarity,
-  stringSimilarity,
+  findProfanityByLevenshteinDistance,
 } from "../utils/similarityUtils";
 import { DEFAULT_OPTIONS } from "../config/options";
+import { AhoCorasick } from "../utils/ahoCorasick";
+
+const globalAhoCorasick = new AhoCorasick();
+let ahoCorasickInitialized = false;
+
+function initializeAhoCorasick(words: string[]) {
+  if (ahoCorasickInitialized) return;
+
+  for (const word of words) {
+    globalAhoCorasick.addPattern(word);
+  }
+
+  globalAhoCorasick.build();
+  ahoCorasickInitialized = true;
+}
+
+interface FindProfanityFunction {
+  (text: string, options?: FilterOptions): string[];
+  lastActualMatches?: Map<string, string[]>;
+}
 
 export function findProfanity(
   text: string,
@@ -40,53 +50,69 @@ export function findProfanity(
     detectSimilarity = false,
     similarityThreshold = 0.8,
     detectSplit = false,
+    useLevenshtein = false,
+    maxLevenshteinDistance = 2,
   } = { ...DEFAULT_OPTIONS, ...options };
 
   const normalizedText = normalizeText(text);
 
-  let wordsToCheck: string[] = wordList.length > 0 ? wordList : [];
+  let baseWordsToCheck: string[] = wordList.length > 0 ? wordList : [];
 
-  if (wordsToCheck.length === 0) {
-    if (categories || regions || severityThreshold > 0) {
-      wordsToCheck = wordObjects
-        .filter((word) => {
-          const matchCategory = categories
-            ? categories.includes(word.category)
-            : true;
-          const matchRegion = regions ? regions.includes(word.region) : true;
-          const matchSeverity = word.severity >= severityThreshold;
-          return matchCategory && matchRegion && matchSeverity;
-        })
-        .map((word) => word.word);
-    } else {
-      wordsToCheck = wordObjects.map((word) => word.word);
-    }
+  if (baseWordsToCheck.length === 0) {
+    const filteredWords = wordObjects.filter((word) => {
+      const matchCategory = categories
+        ? categories.includes(word.category)
+        : true;
+      const matchRegion = regions ? regions.includes(word.region) : true;
+      const matchSeverity = word.severity >= severityThreshold;
+      return matchCategory && matchRegion && matchSeverity;
+    });
+
+    baseWordsToCheck = filteredWords.map((word) => word.word);
   }
 
-  wordsToCheck = wordsToCheck.filter(
-    (word) => !whitelist.includes(word.toLocaleLowerCase()),
-  );
+  const aliasMap = new Map<string, string>();
+  wordObjects.forEach((wordObj) => {
+    if (wordObj.aliases && wordObj.aliases.length > 0) {
+      const matchCategory = categories
+        ? categories.includes(wordObj.category)
+        : true;
+      const matchRegion = regions ? regions.includes(wordObj.region) : true;
+      const matchSeverity = wordObj.severity >= severityThreshold;
+
+      if (matchCategory && matchRegion && matchSeverity) {
+        wordObj.aliases.forEach((alias) => {
+          aliasMap.set(alias.toLowerCase(), wordObj.word.toLowerCase());
+        });
+      }
+    }
+  });
+
+  const wordsToCheck = [
+    ...baseWordsToCheck,
+    ...Array.from(aliasMap.keys()),
+  ].filter((word) => !whitelist.includes(word.toLowerCase()));
 
   if (wordsToCheck.length === 0) {
     return [];
   }
 
   const matches = new Set<string>();
+  const actualMatches = new Map<string, string[]>();
 
-  wordsToCheck.forEach((word) => {
-    const regex = createWordRegex(word, {
-      wholeWord: !checkSubstring,
-      caseSensitive: false,
-      leetSpeak: false,
-      detectSplit: false,
-      indonesianVariation: false,
-    });
+  initializeAhoCorasick(wordsToCheck);
 
-    let match;
-    while ((match = regex.exec(normalizedText)) !== null) {
-      matches.add(word.toLowerCase());
+  const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
+  for (const match of basicMatches) {
+    const originalWord =
+      aliasMap.get(match.toLowerCase()) || match.toLowerCase();
+    matches.add(originalWord);
+
+    if (!actualMatches.has(originalWord)) {
+      actualMatches.set(originalWord, []);
     }
-  });
+    actualMatches.get(originalWord)?.push(match);
+  }
 
   if (detectLeetSpeak) {
     wordsToCheck.forEach((word) => {
@@ -100,7 +126,14 @@ export function findProfanity(
 
       let match;
       while ((match = leetRegex.exec(text)) !== null) {
-        matches.add(word.toLowerCase());
+        const originalWord =
+          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(match[0]);
       }
     });
   }
@@ -117,41 +150,85 @@ export function findProfanity(
 
       let match;
       while ((match = variantRegex.exec(text)) !== null) {
-        matches.add(word.toLowerCase());
+        const originalWord =
+          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(match[0]);
       }
     });
   }
 
   if (detectSplit) {
-    if (detectSplitWords(text, wordsToCheck)) {
-      wordsToCheck.forEach((word) => {
-        const splitRegex = createWordRegex(word, {
-          wholeWord: false,
-          caseSensitive: false,
-          leetSpeak: false,
-          detectSplit: true,
-          indonesianVariation: false,
-        });
+    wordsToCheck.forEach((word) => {
+      const splitRegex = createWordRegex(word, {
+        wholeWord: false,
+        caseSensitive: false,
+        leetSpeak: false,
+        detectSplit: true,
+        indonesianVariation: false,
+      });
 
-        if (splitRegex.test(text)) {
-          matches.add(word.toLowerCase());
+      let match;
+      while ((match = splitRegex.exec(text)) !== null) {
+        const originalWord =
+          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
         }
-      });
-    }
+        actualMatches.get(originalWord)?.push(match[0]);
+      }
+    });
   }
 
   if (detectSimilarity) {
-    const possibleProfanity = findPossibleProfanityBySimiliarity(
-      text,
-      wordsToCheck,
-      similarityThreshold,
-    );
-
-    possibleProfanity.forEach((item) => {
-      matches.add(item.original.toLowerCase());
-    });
+    if (useLevenshtein) {
+      const possibleProfanity = findProfanityByLevenshteinDistance(
+        text,
+        wordsToCheck,
+        similarityThreshold,
+        maxLevenshteinDistance,
+      );
+
+      possibleProfanity.forEach((item) => {
+        const originalWord =
+          aliasMap.get(item.original.toLowerCase()) ||
+          item.original.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(item.word);
+      });
+    } else {
+      const possibleProfanity = findPossibleProfanityBySimiliarity(
+        text,
+        wordsToCheck,
+        similarityThreshold,
+      );
+
+      possibleProfanity.forEach((item) => {
+        matches.add(item.original.toLowerCase());
+
+        const originalWord =
+          aliasMap.get(item.original.toLowerCase()) ||
+          item.original.toLowerCase();
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(item.word);
+      });
+    }
   }
 
+  (findProfanity as FindProfanityFunction).lastActualMatches = actualMatches;
+
   return Array.from(matches);
 }
 

package/src/index.ts

@@ -21,7 +21,6 @@ import {
   CATEGORY_PRESETS,
   REGION_PRESETS,
   getPresetOptions,
-  makeRandomGrawlixString,
 } from "./config/options";
 
 export class IDProfanityFilter {
@@ -161,10 +160,30 @@ export class IDProfanityFilter {
   /**
    * Mengaktifkan deteksi berdasarkan kesamaan
    * @param threshold Threshold kesamaan (0-1)
+   * @param useLevenshtein Gunakan algoritma Levenshtein untuk deteksi
+   * @param maxLevenshteinDistance Jarak maksimal Levenshtein (default: 2)
    */
-  enableSimilarityDetection(threshold: number = 0.8) {
+  enableSimilarityDetection(
+    threshold: number = 0.8,
+    useLevenshtein: boolean = false,
+    maxLevenshteinDistance: number = 2,
+  ) {
+    this.options.detectSimilarity = true;
+    this.options.similarityThreshold = threshold;
+    this.options.useLevenshtein = useLevenshtein;
+    this.options.maxLevenshteinDistance = maxLevenshteinDistance;
+  }
+
+  /**
+   * Mengaktifkan deteksi berbasis Levenshtein distance
+   * @param threshold Threshold kesamaan (0-1)
+   * @param maxDistance Jarak maksimal Levenshtein (default: 2)
+   */
+  enableLevenshteinDetection(threshold: number = 0.8, maxDistance: number = 2) {
     this.options.detectSimilarity = true;
+    this.options.useLevenshtein = true;
     this.options.similarityThreshold = threshold;
+    this.options.maxLevenshteinDistance = maxDistance;
   }
 }
 

package/src/types/index.ts

@@ -50,9 +50,11 @@ export interface FilterOptions {
   useRandomGrawlix?: boolean;
   keepFirstAndLast?: boolean;
   indonesianVariation?: boolean;
-  detectSimilarity?: boolean;
-  similarityThreshold?: number;
+  detectSimilarity?: boolean; // Enable/disable Levenshtein distance matching
+  similarityThreshold?: number; // Threshold for Levenshtein distance similarity (0-1)
   detectSplit?: boolean;
+  useLevenshtein?: boolean; // New option specifically for Levenshtein algorithm
+  maxLevenshteinDistance?: number; // Maximum allowed Levenshtein distance
 }
 
 export interface FilterResult {

package/src/utils/ahoCorasick.ts

@@ -0,0 +1,179 @@
+/**
+ * Implementasi algoritma Aho-Corasick untuk pencocokan string
+ * Algoritma ini memungkinkan pencarian banyak pola dalam satu kali pembacaan teks
+ */
+
+interface AhoCorasickNode {
+  children: Map<string, AhoCorasickNode>;
+  fail: AhoCorasickNode | null;
+  output: Set<string>;
+  depth: number;
+  char?: string;
+}
+
+export class AhoCorasick {
+  private root: AhoCorasickNode;
+  private built: boolean = false;
+
+  constructor() {
+    this.root = {
+      children: new Map(),
+      fail: null,
+      output: new Set(),
+      depth: 0,
+    };
+  }
+
+  /**
+   * Menambahkan pola ke dalam trie
+   * @param pattern Pola yang akan ditambahkan
+   */
+  addPattern(pattern: string): void {
+    if (this.built) {
+      throw new Error("Cannot add patterns after the automaton is built");
+    }
+
+    let node = this.root;
+    const normalizedPattern = pattern.toLowerCase();
+
+    for (let i = 0; i < normalizedPattern.length; i++) {
+      const char = normalizedPattern[i];
+
+      if (!node.children.has(char)) {
+        node.children.set(char, {
+          children: new Map(),
+          fail: null,
+          output: new Set(),
+          depth: node.depth + 1,
+          char,
+        });
+      }
+
+      node = node.children.get(char)!;
+    }
+
+    node.output.add(normalizedPattern);
+  }
+
+  /**
+   * Membangun fungsi failure
+   */
+  build(): void {
+    if (this.built) return;
+
+    const queue: AhoCorasickNode[] = [];
+
+    // Set fail pointer for depth 1 nodes to root
+    for (const child of this.root.children.values()) {
+      child.fail = this.root;
+      queue.push(child);
+    }
+
+    // BFS to build failure links
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+
+      for (const [char, child] of current.children.entries()) {
+        queue.push(child);
+
+        let failNode = current.fail;
+
+        // Find the longest proper suffix that is also a prefix
+        while (failNode !== null && !failNode.children.has(char)) {
+          failNode = failNode.fail;
+        }
+
+        if (failNode === null) {
+          child.fail = this.root;
+        } else {
+          child.fail = failNode.children.get(char)!;
+
+          // Add outputs from the fail state to this node
+          for (const output of child.fail.output) {
+            child.output.add(output);
+          }
+        }
+      }
+    }
+
+    this.built = true;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks
+   * @param text Teks yang akan dicari
+   * @returns Map pola yang ditemukan dengan jumlah kemunculannya
+   */
+  search(text: string): Map<string, number> {
+    if (!this.built) {
+      this.build();
+    }
+
+    const matches = new Map<string, number>();
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      // Follow failure links until we find a matching transition or reach root
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      // Try to follow the transition
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      // Check for any matches at this node
+      for (const match of node.output) {
+        matches.set(match, (matches.get(match) || 0) + 1);
+      }
+    }
+
+    return matches;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks dan mengembalikan hanya pola unik
+   * @param text Teks yang akan dicari
+   * @returns Set pola yang ditemukan
+   */
+  searchUnique(text: string): Set<string> {
+    const matches = this.search(text);
+    return new Set(matches.keys());
+  }
+
+  /**
+   * Mengecek apakah teks mengandung setidaknya satu pola
+   * @param text Teks yang akan dicari
+   * @returns Boolean apakah pola ditemukan
+   */
+  containsAny(text: string): boolean {
+    if (!this.built) {
+      this.build();
+    }
+
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      if (node.output.size > 0) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}

package/src/utils/regexUtils.ts

@@ -72,7 +72,6 @@ export function addLeetSpeakVariations(pattern: string): string {
     z: ["z", "2"],
   };
 
-  // Ganti tiap karakter dengan variasinya dalam grup character class
   return pattern
     .split("")
     .map((char) => {