@sideid/id-profanity-filter 1.10.6 → 1.11.5

package/src/core/analyzer.ts

@@ -12,7 +12,10 @@ import {
   calculateSeverity,
 } from "./matcher";
 import { splitIntoSentences } from "../utils/stringUtils";
-import { findPossibleProfanityBySimiliarity } from "../utils/similarityUtils";
+import {
+  findPossibleProfanityBySimiliarity,
+  findProfanityByLevenshteinDistance,
+} from "../utils/similarityUtils";
 import { createContextRegex } from "../utils/regexUtils";
 import { DEFAULT_OPTIONS } from "../config/options";
 
@@ -54,12 +57,30 @@ export function analyze(
   }> = [];
 
   if (mergedOptions.detectSimilarity) {
-    const wordList = matchDetails.map((word) => word.word);
-    similarWords = findPossibleProfanityBySimiliarity(
-      text,
-      wordList,
-      mergedOptions.similarityThreshold || 0.8,
-    );
+    if (matchDetails.length > 0) {
+      const wordList = matchDetails.map((word) => word.word);
+
+      if (mergedOptions.useLevenshtein) {
+        const levenshteinResults = findProfanityByLevenshteinDistance(
+          text,
+          wordList,
+          mergedOptions.similarityThreshold || 0.8,
+          mergedOptions.maxLevenshteinDistance || 2,
+        );
+
+        similarWords = levenshteinResults.map((item) => ({
+          word: item.word,
+          original: item.original,
+          similarity: item.similarity,
+        }));
+      } else {
+        similarWords = findPossibleProfanityBySimiliarity(
+          text,
+          wordList,
+          mergedOptions.similarityThreshold || 0.8,
+        );
+      }
+    }
   }
 
   return {

package/src/core/matcher.ts

@@ -13,6 +13,21 @@ import {
   findProfanityByLevenshteinDistance,
 } from "../utils/similarityUtils";
 import { DEFAULT_OPTIONS } from "../config/options";
+import { AhoCorasick } from "../utils/ahoCorasick";
+
+const globalAhoCorasick = new AhoCorasick();
+let ahoCorasickInitialized = false;
+
+function initializeAhoCorasick(words: string[]) {
+  if (ahoCorasickInitialized) return;
+
+  for (const word of words) {
+    globalAhoCorasick.addPattern(word);
+  }
+
+  globalAhoCorasick.build();
+  ahoCorasickInitialized = true;
+}
 
 interface FindProfanityFunction {
   (text: string, options?: FilterOptions): string[];
@@ -85,27 +100,19 @@ export function findProfanity(
   const matches = new Set<string>();
   const actualMatches = new Map<string, string[]>();
 
-  wordsToCheck.forEach((word) => {
-    const regex = createWordRegex(word, {
-      wholeWord: !checkSubstring,
-      caseSensitive: false,
-      leetSpeak: false,
-      detectSplit: false,
-      indonesianVariation: false,
-    });
+  initializeAhoCorasick(wordsToCheck);
 
-    let match;
-    while ((match = regex.exec(normalizedText)) !== null) {
-      const originalWord =
-        aliasMap.get(word.toLowerCase()) || word.toLowerCase();
-      matches.add(originalWord);
+  const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
+  for (const match of basicMatches) {
+    const originalWord =
+      aliasMap.get(match.toLowerCase()) || match.toLowerCase();
+    matches.add(originalWord);
 
-      if (!actualMatches.has(originalWord)) {
-        actualMatches.set(originalWord, []);
-      }
-      actualMatches.get(originalWord)?.push(match[0]);
+    if (!actualMatches.has(originalWord)) {
+      actualMatches.set(originalWord, []);
     }
-  });
+    actualMatches.get(originalWord)?.push(match);
+  }
 
   if (detectLeetSpeak) {
     wordsToCheck.forEach((word) => {

package/src/index.ts

@@ -1,27 +1,27 @@
-export * from "./types";
-export * from "./core/matcher";
-export * from "./core/filter";
-export * from "./core/analyzer";
-export * from "./utils/stringUtils";
-export * from "./utils/regexUtils";
-export * from "./utils/similarityUtils";
-export * from "./config/options";
-
-import { filter, isProfane } from "./core/filter";
+export * from './types';
+export * from './core/matcher';
+export * from './core/filter';
+export * from './core/analyzer';
+export * from './utils/stringUtils';
+export * from './utils/regexUtils';
+export * from './utils/similarityUtils';
+export * from './config/options';
+
+import { filter, isProfane } from './core/filter';
 import {
   analyze,
   batchAnalyze,
   analyzeBySentence,
   analyzeWithContext,
-} from "./core/analyzer";
-import { FilterOptions, FilterResult, AnalysisResult } from "./types";
+} from './core/analyzer';
+import { FilterOptions, FilterResult, AnalysisResult } from './types';
 import {
   DEFAULT_OPTIONS,
   FILTER_PRESETS,
   CATEGORY_PRESETS,
   REGION_PRESETS,
   getPresetOptions,
-} from "./config/options";
+} from './config/options';
 
 export class IDProfanityFilter {
   private options: FilterOptions;
@@ -187,8 +187,6 @@ export class IDProfanityFilter {
   }
 }
 
-export default IDProfanityFilter;
-
 export const idFilter = {
   filter: (text: string, options?: FilterOptions) =>
     filter(text, { ...DEFAULT_OPTIONS, ...options }),
@@ -205,3 +203,5 @@ export const idFilter = {
     region: REGION_PRESETS,
   },
 };
+
+export default IDProfanityFilter;

package/src/utils/ahoCorasick.ts

@@ -0,0 +1,179 @@
+/**
+ * Implementasi algoritma Aho-Corasick untuk pencocokan string
+ * Algoritma ini memungkinkan pencarian banyak pola dalam satu kali pembacaan teks
+ */
+
+interface AhoCorasickNode {
+  children: Map<string, AhoCorasickNode>;
+  fail: AhoCorasickNode | null;
+  output: Set<string>;
+  depth: number;
+  char?: string;
+}
+
+export class AhoCorasick {
+  private root: AhoCorasickNode;
+  private built: boolean = false;
+
+  constructor() {
+    this.root = {
+      children: new Map(),
+      fail: null,
+      output: new Set(),
+      depth: 0,
+    };
+  }
+
+  /**
+   * Menambahkan pola ke dalam trie
+   * @param pattern Pola yang akan ditambahkan
+   */
+  addPattern(pattern: string): void {
+    if (this.built) {
+      throw new Error("Cannot add patterns after the automaton is built");
+    }
+
+    let node = this.root;
+    const normalizedPattern = pattern.toLowerCase();
+
+    for (let i = 0; i < normalizedPattern.length; i++) {
+      const char = normalizedPattern[i];
+
+      if (!node.children.has(char)) {
+        node.children.set(char, {
+          children: new Map(),
+          fail: null,
+          output: new Set(),
+          depth: node.depth + 1,
+          char,
+        });
+      }
+
+      node = node.children.get(char)!;
+    }
+
+    node.output.add(normalizedPattern);
+  }
+
+  /**
+   * Membangun fungsi failure
+   */
+  build(): void {
+    if (this.built) return;
+
+    const queue: AhoCorasickNode[] = [];
+
+    // Set fail pointer for depth 1 nodes to root
+    for (const child of this.root.children.values()) {
+      child.fail = this.root;
+      queue.push(child);
+    }
+
+    // BFS to build failure links
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+
+      for (const [char, child] of current.children.entries()) {
+        queue.push(child);
+
+        let failNode = current.fail;
+
+        // Find the longest proper suffix that is also a prefix
+        while (failNode !== null && !failNode.children.has(char)) {
+          failNode = failNode.fail;
+        }
+
+        if (failNode === null) {
+          child.fail = this.root;
+        } else {
+          child.fail = failNode.children.get(char)!;
+
+          // Add outputs from the fail state to this node
+          for (const output of child.fail.output) {
+            child.output.add(output);
+          }
+        }
+      }
+    }
+
+    this.built = true;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks
+   * @param text Teks yang akan dicari
+   * @returns Map pola yang ditemukan dengan jumlah kemunculannya
+   */
+  search(text: string): Map<string, number> {
+    if (!this.built) {
+      this.build();
+    }
+
+    const matches = new Map<string, number>();
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      // Follow failure links until we find a matching transition or reach root
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      // Try to follow the transition
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      // Check for any matches at this node
+      for (const match of node.output) {
+        matches.set(match, (matches.get(match) || 0) + 1);
+      }
+    }
+
+    return matches;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks dan mengembalikan hanya pola unik
+   * @param text Teks yang akan dicari
+   * @returns Set pola yang ditemukan
+   */
+  searchUnique(text: string): Set<string> {
+    const matches = this.search(text);
+    return new Set(matches.keys());
+  }
+
+  /**
+   * Mengecek apakah teks mengandung setidaknya satu pola
+   * @param text Teks yang akan dicari
+   * @returns Boolean apakah pola ditemukan
+   */
+  containsAny(text: string): boolean {
+    if (!this.built) {
+      this.build();
+    }
+
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      if (node.output.size > 0) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}

package/src/utils/similarityUtils.ts

@@ -198,12 +198,22 @@ export function clusterSimilarWords(
 
 /**
  * Cari kata-kata kotor yang mungkin dari teks menggunakan kesamaan string
+ * dengan optimasi untuk mengurangi kompleksitas
  *
  * @param text Teks yang akan diperiksa
  * @param profanityWords Daftar kata kotor
  * @param threshold Batas minimum kesamaan (default: 0.8)
  * @returns Array kata yang mungkin merupakan kata kotor
  */
+/**
+ * Cari kata-kata kotor yang mungkin dari teks berdasarkan kemiripan string
+ * dengan optimasi biar prosesnya nggak terlalu berat
+ *
+ * @param text Teks yang mau dicek
+ * @param profanityWords Daftar kata-kata kotor/kasar
+ * @param threshold Batas minimal kemiripan (default: 0.8)
+ * @returns Array kata yang kemungkinan kata kotor/kasar
+ */
 export function findPossibleProfanityBySimiliarity(
   text: string,
   profanityWords: string[],
@@ -212,23 +222,74 @@ export function findPossibleProfanityBySimiliarity(
   const result: Array<{ word: string; original: string; similarity: number }> =
     [];
 
+  // Optimasi dengan bikin map kata-kata kotor berdasarkan huruf pertama
+  const profanityMap = new Map<string, string[]>();
+
+  // Kelompokkan kata-kata kotor berdasarkan huruf pertama biar pencariannya lebih cepat
+  for (const word of profanityWords) {
+    if (word.length < 1) continue;
+
+    const firstChar = word[0].toLowerCase();
+    if (!profanityMap.has(firstChar)) {
+      profanityMap.set(firstChar, []);
+    }
+    profanityMap.get(firstChar)!.push(word);
+  }
+
   const words = text.toLowerCase().split(/\s+/);
 
   for (const word of words) {
     if (word.length < 3) continue;
 
-    for (const profanity of profanityWords) {
+    // Cuma bandingin dengan kata-kata kotor yang huruf pertamanya sama
+    // atau yang perbedaan panjangnya masih masuk akal
+    const firstChar = word[0];
+    const candidateWords = profanityMap.get(firstChar) || [];
+
+    // Cek juga huruf-huruf yang berdekatan (buat antisipasi typo di huruf pertama)
+    // Ini opsional tapi bikin deteksinya lebih bagus
+    const charCode = firstChar.charCodeAt(0);
+    const prevChar = String.fromCharCode(charCode - 1);
+    const nextChar = String.fromCharCode(charCode + 1);
+
+    const adjacentCandidates = [
+      ...(profanityMap.get(prevChar) || []),
+      ...(profanityMap.get(nextChar) || []),
+    ];
+
+    // Gabungin kandidat-kandidatnya, prioritasin yang huruf pertamanya sama persis
+    const allCandidates = [...candidateWords, ...adjacentCandidates];
+
+    // Filter kandidat berdasarkan perbedaan panjang sebelum ngitung kemiripannya
+    const lengthFilteredCandidates = allCandidates.filter(
+      (candidate) => Math.abs(candidate.length - word.length) <= 2,
+    );
+
+    // Cari yang paling cocok
+    let bestMatch: {
+      word: string;
+      original: string;
+      similarity: number;
+    } | null = null;
+
+    for (const profanity of lengthFilteredCandidates) {
       const similarity = stringSimilarity(word, profanity);
 
-      if (similarity >= threshold) {
-        result.push({
+      if (
+        similarity >= threshold &&
+        (!bestMatch || similarity > bestMatch.similarity)
+      ) {
+        bestMatch = {
           word,
           original: profanity,
           similarity,
-        });
-        break;
+        };
       }
     }
+
+    if (bestMatch) {
+      result.push(bestMatch);
+    }
   }
 
   return result;
@@ -261,30 +322,106 @@ export function findProfanityByLevenshteinDistance(
     distance: number;
   }> = [];
 
+  // map kata-kata kotor dikelompokkan sesuai panjangnya
+  const profanityByLength = new Map<number, string[]>();
+
+  // Kelompokkin kata-kata kotor berdasarkan panjangnya biar pencarian lebih cepat
+  for (const word of profanityWords) {
+    const length = word.length;
+    if (!profanityByLength.has(length)) {
+      profanityByLength.set(length, []);
+    }
+    profanityByLength.get(length)!.push(word);
+  }
+
   const words = text.toLowerCase().split(/\s+/);
 
   for (const word of words) {
     if (word.length < 3) continue;
 
-    for (const profanity of profanityWords) {
-      if (Math.abs(word.length - profanity.length) > maxDistance) continue;
-
-      const distance = levenshteinDistance(word, profanity);
-      if (distance <= maxDistance) {
-        const similarity = stringSimilarity(word, profanity);
-
-        if (similarity >= threshold) {
-          result.push({
-            word,
-            original: profanity,
-            similarity,
-            distance,
-          });
-          break;
+    let bestMatch: {
+      word: string;
+      original: string;
+      similarity: number;
+      distance: number;
+    } | null = null;
+
+    for (
+      let len = Math.max(3, word.length - maxDistance);
+      len <= word.length + maxDistance;
+      len++
+    ) {
+      const candidates = profanityByLength.get(len) || [];
+
+      for (const profanity of candidates) {
+        if (!isCharacterCountSimilar(word, profanity, maxDistance)) {
+          continue;
+        }
+
+        const distance = levenshteinDistance(word, profanity);
+
+        if (distance <= maxDistance) {
+          const similarity =
+            1 - distance / Math.max(word.length, profanity.length);
+
+          if (
+            similarity >= threshold &&
+            (!bestMatch || similarity > bestMatch.similarity)
+          ) {
+            bestMatch = {
+              word,
+              original: profanity,
+              similarity,
+              distance,
+            };
+
+            if (distance === 0 || similarity > 0.95) {
+              break;
+            }
+          }
         }
       }
     }
+
+    if (bestMatch) {
+      result.push(bestMatch);
+    }
   }
 
   return result;
 }
+
+/**
+ * Helper function to efficiently check if character counts between two strings
+ * are similar enough to warrant a full Levenshtein calculation
+ */
+function isCharacterCountSimilar(
+  str1: string,
+  str2: string,
+  maxDifference: number,
+): boolean {
+  const charCount1: Record<string, number> = {};
+  const charCount2: Record<string, number> = {};
+
+  for (const char of str1) {
+    charCount1[char] = (charCount1[char] || 0) + 1;
+  }
+
+  for (const char of str2) {
+    charCount2[char] = (charCount2[char] || 0) + 1;
+  }
+
+  let diffCount = 0;
+
+  for (const char in charCount1) {
+    diffCount += Math.abs((charCount1[char] || 0) - (charCount2[char] || 0));
+  }
+
+  for (const char in charCount2) {
+    if (!charCount1[char]) {
+      diffCount += charCount2[char];
+    }
+  }
+
+  return diffCount <= maxDifference * 2;
+}