@sideid/id-profanity-filter 1.9.5 → 1.10.2

package/src/utils/ahoCorasick.ts

@@ -0,0 +1,179 @@
+/**
+ * Implementasi algoritma Aho-Corasick untuk pencocokan string
+ * Algoritma ini memungkinkan pencarian banyak pola dalam satu kali pembacaan teks
+ */
+
+interface AhoCorasickNode {
+  children: Map<string, AhoCorasickNode>;
+  fail: AhoCorasickNode | null;
+  output: Set<string>;
+  depth: number;
+  char?: string;
+}
+
+export class AhoCorasick {
+  private root: AhoCorasickNode;
+  private built: boolean = false;
+
+  constructor() {
+    this.root = {
+      children: new Map(),
+      fail: null,
+      output: new Set(),
+      depth: 0,
+    };
+  }
+
+  /**
+   * Menambahkan pola ke dalam trie
+   * @param pattern Pola yang akan ditambahkan
+   */
+  addPattern(pattern: string): void {
+    if (this.built) {
+      throw new Error("Cannot add patterns after the automaton is built");
+    }
+
+    let node = this.root;
+    const normalizedPattern = pattern.toLowerCase();
+
+    for (let i = 0; i < normalizedPattern.length; i++) {
+      const char = normalizedPattern[i];
+
+      if (!node.children.has(char)) {
+        node.children.set(char, {
+          children: new Map(),
+          fail: null,
+          output: new Set(),
+          depth: node.depth + 1,
+          char,
+        });
+      }
+
+      node = node.children.get(char)!;
+    }
+
+    node.output.add(normalizedPattern);
+  }
+
+  /**
+   * Membangun fungsi failure
+   */
+  build(): void {
+    if (this.built) return;
+
+    const queue: AhoCorasickNode[] = [];
+
+    // Set fail pointer for depth 1 nodes to root
+    for (const child of this.root.children.values()) {
+      child.fail = this.root;
+      queue.push(child);
+    }
+
+    // BFS to build failure links
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+
+      for (const [char, child] of current.children.entries()) {
+        queue.push(child);
+
+        let failNode = current.fail;
+
+        // Find the longest proper suffix that is also a prefix
+        while (failNode !== null && !failNode.children.has(char)) {
+          failNode = failNode.fail;
+        }
+
+        if (failNode === null) {
+          child.fail = this.root;
+        } else {
+          child.fail = failNode.children.get(char)!;
+
+          // Add outputs from the fail state to this node
+          for (const output of child.fail.output) {
+            child.output.add(output);
+          }
+        }
+      }
+    }
+
+    this.built = true;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks
+   * @param text Teks yang akan dicari
+   * @returns Map pola yang ditemukan dengan jumlah kemunculannya
+   */
+  search(text: string): Map<string, number> {
+    if (!this.built) {
+      this.build();
+    }
+
+    const matches = new Map<string, number>();
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      // Follow failure links until we find a matching transition or reach root
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      // Try to follow the transition
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      // Check for any matches at this node
+      for (const match of node.output) {
+        matches.set(match, (matches.get(match) || 0) + 1);
+      }
+    }
+
+    return matches;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks dan mengembalikan hanya pola unik
+   * @param text Teks yang akan dicari
+   * @returns Set pola yang ditemukan
+   */
+  searchUnique(text: string): Set<string> {
+    const matches = this.search(text);
+    return new Set(matches.keys());
+  }
+
+  /**
+   * Mengecek apakah teks mengandung setidaknya satu pola
+   * @param text Teks yang akan dicari
+   * @returns Boolean apakah pola ditemukan
+   */
+  containsAny(text: string): boolean {
+    if (!this.built) {
+      this.build();
+    }
+
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      if (node.output.size > 0) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}

package/src/utils/regexUtils.ts

@@ -72,7 +72,6 @@ export function addLeetSpeakVariations(pattern: string): string {
     z: ["z", "2"],
   };
 
-  // Ganti tiap karakter dengan variasinya dalam grup character class
   return pattern
     .split("")
     .map((char) => {

package/src/utils/similarityUtils.ts

@@ -91,6 +91,48 @@ export function findMostSimilar(
   return mostSimilar;
 }
 
+/**
+ * Mencari string yang paling mirip dari array menggunakan Levenshtein distance
+ *
+ * @param target String target
+ * @param candidates Array string kandidat
+ * @param threshold Minimum kesamaan yang diterima (0-1)
+ * @param maxDistance Jarak Levenshtein maksimal yang diterima (default: 3)
+ * @returns String yang paling mirip atau null jika tidak ada yang di atas threshold
+ */
+export function findMostSimilarWithLevenshtein(
+  target: string,
+  candidates: string[],
+  threshold: number = 0.7,
+  maxDistance: number = 3,
+): string | null {
+  if (!candidates.length) return null;
+
+  let maxSimilarity = 0;
+  let minDistance = Infinity;
+  let mostSimilar: string | null = null;
+
+  for (const candidate of candidates) {
+    if (Math.abs(target.length - candidate.length) > maxDistance) continue;
+
+    const distance = levenshteinDistance(target, candidate);
+    const similarity = stringSimilarity(target, candidate);
+
+    if (
+      (similarity > maxSimilarity && similarity >= threshold) ||
+      (similarity >= threshold && distance < minDistance)
+    ) {
+      maxSimilarity = similarity;
+      minDistance = distance;
+      mostSimilar = candidate;
+
+      if (distance <= 1 || similarity > 0.95) break;
+    }
+  }
+
+  return mostSimilar;
+}
+
 /**
  * Cek apakah string mungkin merupakan variasi dari kata kotor
  * menggunakan kesamaan string
@@ -156,12 +198,22 @@ export function clusterSimilarWords(
 
 /**
  * Cari kata-kata kotor yang mungkin dari teks menggunakan kesamaan string
+ * dengan optimasi untuk mengurangi kompleksitas
  *
  * @param text Teks yang akan diperiksa
  * @param profanityWords Daftar kata kotor
  * @param threshold Batas minimum kesamaan (default: 0.8)
  * @returns Array kata yang mungkin merupakan kata kotor
  */
+/**
+ * Cari kata-kata kotor yang mungkin dari teks berdasarkan kemiripan string
+ * dengan optimasi biar prosesnya nggak terlalu berat
+ *
+ * @param text Teks yang mau dicek
+ * @param profanityWords Daftar kata-kata kotor/kasar
+ * @param threshold Batas minimal kemiripan (default: 0.8)
+ * @returns Array kata yang kemungkinan kata kotor/kasar
+ */
 export function findPossibleProfanityBySimiliarity(
   text: string,
   profanityWords: string[],
@@ -170,26 +222,206 @@ export function findPossibleProfanityBySimiliarity(
   const result: Array<{ word: string; original: string; similarity: number }> =
     [];
 
-  // Pisahkan teks menjadi kata-kata
+  // Optimasi dengan bikin map kata-kata kotor berdasarkan huruf pertama
+  const profanityMap = new Map<string, string[]>();
+
+  // Kelompokkan kata-kata kotor berdasarkan huruf pertama biar pencariannya lebih cepat
+  for (const word of profanityWords) {
+    if (word.length < 1) continue;
+
+    const firstChar = word[0].toLowerCase();
+    if (!profanityMap.has(firstChar)) {
+      profanityMap.set(firstChar, []);
+    }
+    profanityMap.get(firstChar)!.push(word);
+  }
+
   const words = text.toLowerCase().split(/\s+/);
 
   for (const word of words) {
-    // Lewati kata-kata yang terlalu pendek
     if (word.length < 3) continue;
 
-    for (const profanity of profanityWords) {
+    // Cuma bandingin dengan kata-kata kotor yang huruf pertamanya sama
+    // atau yang perbedaan panjangnya masih masuk akal
+    const firstChar = word[0];
+    const candidateWords = profanityMap.get(firstChar) || [];
+
+    // Cek juga huruf-huruf yang berdekatan (buat antisipasi typo di huruf pertama)
+    // Ini opsional tapi bikin deteksinya lebih bagus
+    const charCode = firstChar.charCodeAt(0);
+    const prevChar = String.fromCharCode(charCode - 1);
+    const nextChar = String.fromCharCode(charCode + 1);
+
+    const adjacentCandidates = [
+      ...(profanityMap.get(prevChar) || []),
+      ...(profanityMap.get(nextChar) || []),
+    ];
+
+    // Gabungin kandidat-kandidatnya, prioritasin yang huruf pertamanya sama persis
+    const allCandidates = [...candidateWords, ...adjacentCandidates];
+
+    // Filter kandidat berdasarkan perbedaan panjang sebelum ngitung kemiripannya
+    const lengthFilteredCandidates = allCandidates.filter(
+      (candidate) => Math.abs(candidate.length - word.length) <= 2,
+    );
+
+    // Cari yang paling cocok
+    let bestMatch: {
+      word: string;
+      original: string;
+      similarity: number;
+    } | null = null;
+
+    for (const profanity of lengthFilteredCandidates) {
       const similarity = stringSimilarity(word, profanity);
 
-      if (similarity >= threshold) {
-        result.push({
+      if (
+        similarity >= threshold &&
+        (!bestMatch || similarity > bestMatch.similarity)
+      ) {
+        bestMatch = {
           word,
           original: profanity,
           similarity,
-        });
-        break;
+        };
       }
     }
+
+    if (bestMatch) {
+      result.push(bestMatch);
+    }
   }
 
   return result;
 }
+
+/**
+ * Cari kata-kata kotor yang mungkin dari teks menggunakan Levenshtein distance
+ *
+ * @param text Teks yang akan diperiksa
+ * @param profanityWords Daftar kata kotor
+ * @param threshold Batas minimum kesamaan (default: 0.8)
+ * @param maxDistance Jarak Levenshtein maksimal (default: 2)
+ * @returns Array kata yang mungkin merupakan kata kotor
+ */
+export function findProfanityByLevenshteinDistance(
+  text: string,
+  profanityWords: string[],
+  threshold: number = 0.8,
+  maxDistance: number = 2,
+): Array<{
+  word: string;
+  original: string;
+  similarity: number;
+  distance: number;
+}> {
+  const result: Array<{
+    word: string;
+    original: string;
+    similarity: number;
+    distance: number;
+  }> = [];
+
+  // map kata-kata kotor dikelompokkan sesuai panjangnya
+  const profanityByLength = new Map<number, string[]>();
+
+  // Kelompokkin kata-kata kotor berdasarkan panjangnya biar pencarian lebih cepat
+  for (const word of profanityWords) {
+    const length = word.length;
+    if (!profanityByLength.has(length)) {
+      profanityByLength.set(length, []);
+    }
+    profanityByLength.get(length)!.push(word);
+  }
+
+  const words = text.toLowerCase().split(/\s+/);
+
+  for (const word of words) {
+    if (word.length < 3) continue;
+
+    let bestMatch: {
+      word: string;
+      original: string;
+      similarity: number;
+      distance: number;
+    } | null = null;
+
+    for (
+      let len = Math.max(3, word.length - maxDistance);
+      len <= word.length + maxDistance;
+      len++
+    ) {
+      const candidates = profanityByLength.get(len) || [];
+
+      for (const profanity of candidates) {
+        if (!isCharacterCountSimilar(word, profanity, maxDistance)) {
+          continue;
+        }
+
+        const distance = levenshteinDistance(word, profanity);
+
+        if (distance <= maxDistance) {
+          const similarity =
+            1 - distance / Math.max(word.length, profanity.length);
+
+          if (
+            similarity >= threshold &&
+            (!bestMatch || similarity > bestMatch.similarity)
+          ) {
+            bestMatch = {
+              word,
+              original: profanity,
+              similarity,
+              distance,
+            };
+
+            if (distance === 0 || similarity > 0.95) {
+              break;
+            }
+          }
+        }
+      }
+    }
+
+    if (bestMatch) {
+      result.push(bestMatch);
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Helper function to efficiently check if character counts between two strings
+ * are similar enough to warrant a full Levenshtein calculation
+ */
+function isCharacterCountSimilar(
+  str1: string,
+  str2: string,
+  maxDifference: number,
+): boolean {
+  const charCount1: Record<string, number> = {};
+  const charCount2: Record<string, number> = {};
+
+  for (const char of str1) {
+    charCount1[char] = (charCount1[char] || 0) + 1;
+  }
+
+  for (const char of str2) {
+    charCount2[char] = (charCount2[char] || 0) + 1;
+  }
+
+  let diffCount = 0;
+
+  for (const char in charCount1) {
+    diffCount += Math.abs((charCount1[char] || 0) - (charCount2[char] || 0));
+  }
+
+  for (const char in charCount2) {
+    if (!charCount1[char]) {
+      diffCount += charCount2[char];
+    }
+  }
+
+  return diffCount <= maxDifference * 2;
+}