@sideid/id-profanity-filter 1.10.6 → 1.11.0

package/src/core/analyzer.ts

@@ -12,7 +12,10 @@ import {
   calculateSeverity,
 } from "./matcher";
 import { splitIntoSentences } from "../utils/stringUtils";
-import { findPossibleProfanityBySimiliarity } from "../utils/similarityUtils";
+import {
+  findPossibleProfanityBySimiliarity,
+  findProfanityByLevenshteinDistance,
+} from "../utils/similarityUtils";
 import { createContextRegex } from "../utils/regexUtils";
 import { DEFAULT_OPTIONS } from "../config/options";
 
@@ -54,12 +57,30 @@ export function analyze(
   }> = [];
 
   if (mergedOptions.detectSimilarity) {
-    const wordList = matchDetails.map((word) => word.word);
-    similarWords = findPossibleProfanityBySimiliarity(
-      text,
-      wordList,
-      mergedOptions.similarityThreshold || 0.8,
-    );
+    if (matchDetails.length > 0) {
+      const wordList = matchDetails.map((word) => word.word);
+
+      if (mergedOptions.useLevenshtein) {
+        const levenshteinResults = findProfanityByLevenshteinDistance(
+          text,
+          wordList,
+          mergedOptions.similarityThreshold || 0.8,
+          mergedOptions.maxLevenshteinDistance || 2,
+        );
+
+        similarWords = levenshteinResults.map((item) => ({
+          word: item.word,
+          original: item.original,
+          similarity: item.similarity,
+        }));
+      } else {
+        similarWords = findPossibleProfanityBySimiliarity(
+          text,
+          wordList,
+          mergedOptions.similarityThreshold || 0.8,
+        );
+      }
+    }
   }
 
   return {

package/src/core/matcher.ts

@@ -13,6 +13,21 @@ import {
   findProfanityByLevenshteinDistance,
 } from "../utils/similarityUtils";
 import { DEFAULT_OPTIONS } from "../config/options";
+import { AhoCorasick } from "../utils/ahoCorasick";
+
+const globalAhoCorasick = new AhoCorasick();
+let ahoCorasickInitialized = false;
+
+function initializeAhoCorasick(words: string[]) {
+  if (ahoCorasickInitialized) return;
+
+  for (const word of words) {
+    globalAhoCorasick.addPattern(word);
+  }
+
+  globalAhoCorasick.build();
+  ahoCorasickInitialized = true;
+}
 
 interface FindProfanityFunction {
   (text: string, options?: FilterOptions): string[];
@@ -85,27 +100,19 @@ export function findProfanity(
   const matches = new Set<string>();
   const actualMatches = new Map<string, string[]>();
 
-  wordsToCheck.forEach((word) => {
-    const regex = createWordRegex(word, {
-      wholeWord: !checkSubstring,
-      caseSensitive: false,
-      leetSpeak: false,
-      detectSplit: false,
-      indonesianVariation: false,
-    });
+  initializeAhoCorasick(wordsToCheck);
 
-    let match;
-    while ((match = regex.exec(normalizedText)) !== null) {
-      const originalWord =
-        aliasMap.get(word.toLowerCase()) || word.toLowerCase();
-      matches.add(originalWord);
+  const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
+  for (const match of basicMatches) {
+    const originalWord =
+      aliasMap.get(match.toLowerCase()) || match.toLowerCase();
+    matches.add(originalWord);
 
-      if (!actualMatches.has(originalWord)) {
-        actualMatches.set(originalWord, []);
-      }
-      actualMatches.get(originalWord)?.push(match[0]);
+    if (!actualMatches.has(originalWord)) {
+      actualMatches.set(originalWord, []);
     }
-  });
+    actualMatches.get(originalWord)?.push(match);
+  }
 
   if (detectLeetSpeak) {
     wordsToCheck.forEach((word) => {

package/src/utils/ahoCorasick.ts

@@ -0,0 +1,179 @@
+/**
+ * Implementasi algoritma Aho-Corasick untuk pencocokan string
+ * Algoritma ini memungkinkan pencarian banyak pola dalam satu kali pembacaan teks
+ */
+
+interface AhoCorasickNode {
+  children: Map<string, AhoCorasickNode>;
+  fail: AhoCorasickNode | null;
+  output: Set<string>;
+  depth: number;
+  char?: string;
+}
+
+export class AhoCorasick {
+  private root: AhoCorasickNode;
+  private built: boolean = false;
+
+  constructor() {
+    this.root = {
+      children: new Map(),
+      fail: null,
+      output: new Set(),
+      depth: 0,
+    };
+  }
+
+  /**
+   * Menambahkan pola ke dalam trie
+   * @param pattern Pola yang akan ditambahkan
+   */
+  addPattern(pattern: string): void {
+    if (this.built) {
+      throw new Error("Cannot add patterns after the automaton is built");
+    }
+
+    let node = this.root;
+    const normalizedPattern = pattern.toLowerCase();
+
+    for (let i = 0; i < normalizedPattern.length; i++) {
+      const char = normalizedPattern[i];
+
+      if (!node.children.has(char)) {
+        node.children.set(char, {
+          children: new Map(),
+          fail: null,
+          output: new Set(),
+          depth: node.depth + 1,
+          char,
+        });
+      }
+
+      node = node.children.get(char)!;
+    }
+
+    node.output.add(normalizedPattern);
+  }
+
+  /**
+   * Membangun fungsi failure
+   */
+  build(): void {
+    if (this.built) return;
+
+    const queue: AhoCorasickNode[] = [];
+
+    // Set fail pointer for depth 1 nodes to root
+    for (const child of this.root.children.values()) {
+      child.fail = this.root;
+      queue.push(child);
+    }
+
+    // BFS to build failure links
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+
+      for (const [char, child] of current.children.entries()) {
+        queue.push(child);
+
+        let failNode = current.fail;
+
+        // Find the longest proper suffix that is also a prefix
+        while (failNode !== null && !failNode.children.has(char)) {
+          failNode = failNode.fail;
+        }
+
+        if (failNode === null) {
+          child.fail = this.root;
+        } else {
+          child.fail = failNode.children.get(char)!;
+
+          // Add outputs from the fail state to this node
+          for (const output of child.fail.output) {
+            child.output.add(output);
+          }
+        }
+      }
+    }
+
+    this.built = true;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks
+   * @param text Teks yang akan dicari
+   * @returns Map pola yang ditemukan dengan jumlah kemunculannya
+   */
+  search(text: string): Map<string, number> {
+    if (!this.built) {
+      this.build();
+    }
+
+    const matches = new Map<string, number>();
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      // Follow failure links until we find a matching transition or reach root
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      // Try to follow the transition
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      // Check for any matches at this node
+      for (const match of node.output) {
+        matches.set(match, (matches.get(match) || 0) + 1);
+      }
+    }
+
+    return matches;
+  }
+
+  /**
+   * Mencari semua kemunculan pola dalam teks dan mengembalikan hanya pola unik
+   * @param text Teks yang akan dicari
+   * @returns Set pola yang ditemukan
+   */
+  searchUnique(text: string): Set<string> {
+    const matches = this.search(text);
+    return new Set(matches.keys());
+  }
+
+  /**
+   * Mengecek apakah teks mengandung setidaknya satu pola
+   * @param text Teks yang akan dicari
+   * @returns Boolean apakah pola ditemukan
+   */
+  containsAny(text: string): boolean {
+    if (!this.built) {
+      this.build();
+    }
+
+    const normalizedText = text.toLowerCase();
+    let node = this.root;
+
+    for (let i = 0; i < normalizedText.length; i++) {
+      const char = normalizedText[i];
+
+      while (node !== this.root && !node.children.has(char)) {
+        node = node.fail!;
+      }
+
+      if (node.children.has(char)) {
+        node = node.children.get(char)!;
+      }
+
+      if (node.output.size > 0) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+}

package/src/utils/similarityUtils.ts

@@ -198,12 +198,22 @@ export function clusterSimilarWords(
 
 /**
  * Cari kata-kata kotor yang mungkin dari teks menggunakan kesamaan string
+ * dengan optimasi untuk mengurangi kompleksitas
  *
  * @param text Teks yang akan diperiksa
  * @param profanityWords Daftar kata kotor
  * @param threshold Batas minimum kesamaan (default: 0.8)
  * @returns Array kata yang mungkin merupakan kata kotor
  */
+/**
+ * Cari kata-kata kotor yang mungkin dari teks berdasarkan kemiripan string
+ * dengan optimasi biar prosesnya nggak terlalu berat
+ *
+ * @param text Teks yang mau dicek
+ * @param profanityWords Daftar kata-kata kotor/kasar
+ * @param threshold Batas minimal kemiripan (default: 0.8)
+ * @returns Array kata yang kemungkinan kata kotor/kasar
+ */
 export function findPossibleProfanityBySimiliarity(
   text: string,
   profanityWords: string[],
@@ -212,23 +222,74 @@ export function findPossibleProfanityBySimiliarity(
   const result: Array<{ word: string; original: string; similarity: number }> =
     [];
 
+  // Optimasi dengan bikin map kata-kata kotor berdasarkan huruf pertama
+  const profanityMap = new Map<string, string[]>();
+
+  // Kelompokkan kata-kata kotor berdasarkan huruf pertama biar pencariannya lebih cepat
+  for (const word of profanityWords) {
+    if (word.length < 1) continue;
+
+    const firstChar = word[0].toLowerCase();
+    if (!profanityMap.has(firstChar)) {
+      profanityMap.set(firstChar, []);
+    }
+    profanityMap.get(firstChar)!.push(word);
+  }
+
   const words = text.toLowerCase().split(/\s+/);
 
   for (const word of words) {
     if (word.length < 3) continue;
 
-    for (const profanity of profanityWords) {
+    // Cuma bandingin dengan kata-kata kotor yang huruf pertamanya sama
+    // atau yang perbedaan panjangnya masih masuk akal
+    const firstChar = word[0];
+    const candidateWords = profanityMap.get(firstChar) || [];
+
+    // Cek juga huruf-huruf yang berdekatan (buat antisipasi typo di huruf pertama)
+    // Ini opsional tapi bikin deteksinya lebih bagus
+    const charCode = firstChar.charCodeAt(0);
+    const prevChar = String.fromCharCode(charCode - 1);
+    const nextChar = String.fromCharCode(charCode + 1);
+
+    const adjacentCandidates = [
+      ...(profanityMap.get(prevChar) || []),
+      ...(profanityMap.get(nextChar) || []),
+    ];
+
+    // Gabungin kandidat-kandidatnya, prioritasin yang huruf pertamanya sama persis
+    const allCandidates = [...candidateWords, ...adjacentCandidates];
+
+    // Filter kandidat berdasarkan perbedaan panjang sebelum ngitung kemiripannya
+    const lengthFilteredCandidates = allCandidates.filter(
+      (candidate) => Math.abs(candidate.length - word.length) <= 2,
+    );
+
+    // Cari yang paling cocok
+    let bestMatch: {
+      word: string;
+      original: string;
+      similarity: number;
+    } | null = null;
+
+    for (const profanity of lengthFilteredCandidates) {
       const similarity = stringSimilarity(word, profanity);
 
-      if (similarity >= threshold) {
-        result.push({
+      if (
+        similarity >= threshold &&
+        (!bestMatch || similarity > bestMatch.similarity)
+      ) {
+        bestMatch = {
           word,
           original: profanity,
           similarity,
-        });
-        break;
+        };
       }
     }
+
+    if (bestMatch) {
+      result.push(bestMatch);
+    }
   }
 
   return result;
@@ -261,30 +322,106 @@ export function findProfanityByLevenshteinDistance(
     distance: number;
   }> = [];
 
+  // map kata-kata kotor dikelompokkan sesuai panjangnya
+  const profanityByLength = new Map<number, string[]>();
+
+  // Kelompokkin kata-kata kotor berdasarkan panjangnya biar pencarian lebih cepat
+  for (const word of profanityWords) {
+    const length = word.length;
+    if (!profanityByLength.has(length)) {
+      profanityByLength.set(length, []);
+    }
+    profanityByLength.get(length)!.push(word);
+  }
+
   const words = text.toLowerCase().split(/\s+/);
 
   for (const word of words) {
     if (word.length < 3) continue;
 
-    for (const profanity of profanityWords) {
-      if (Math.abs(word.length - profanity.length) > maxDistance) continue;
-
-      const distance = levenshteinDistance(word, profanity);
-      if (distance <= maxDistance) {
-        const similarity = stringSimilarity(word, profanity);
-
-        if (similarity >= threshold) {
-          result.push({
-            word,
-            original: profanity,
-            similarity,
-            distance,
-          });
-          break;
+    let bestMatch: {
+      word: string;
+      original: string;
+      similarity: number;
+      distance: number;
+    } | null = null;
+
+    for (
+      let len = Math.max(3, word.length - maxDistance);
+      len <= word.length + maxDistance;
+      len++
+    ) {
+      const candidates = profanityByLength.get(len) || [];
+
+      for (const profanity of candidates) {
+        if (!isCharacterCountSimilar(word, profanity, maxDistance)) {
+          continue;
+        }
+
+        const distance = levenshteinDistance(word, profanity);
+
+        if (distance <= maxDistance) {
+          const similarity =
+            1 - distance / Math.max(word.length, profanity.length);
+
+          if (
+            similarity >= threshold &&
+            (!bestMatch || similarity > bestMatch.similarity)
+          ) {
+            bestMatch = {
+              word,
+              original: profanity,
+              similarity,
+              distance,
+            };
+
+            if (distance === 0 || similarity > 0.95) {
+              break;
+            }
+          }
         }
       }
     }
+
+    if (bestMatch) {
+      result.push(bestMatch);
+    }
   }
 
   return result;
 }
+
+/**
+ * Helper function to efficiently check if character counts between two strings
+ * are similar enough to warrant a full Levenshtein calculation
+ */
+function isCharacterCountSimilar(
+  str1: string,
+  str2: string,
+  maxDifference: number,
+): boolean {
+  const charCount1: Record<string, number> = {};
+  const charCount2: Record<string, number> = {};
+
+  for (const char of str1) {
+    charCount1[char] = (charCount1[char] || 0) + 1;
+  }
+
+  for (const char of str2) {
+    charCount2[char] = (charCount2[char] || 0) + 1;
+  }
+
+  let diffCount = 0;
+
+  for (const char in charCount1) {
+    diffCount += Math.abs((charCount1[char] || 0) - (charCount2[char] || 0));
+  }
+
+  for (const char in charCount2) {
+    if (!charCount1[char]) {
+      diffCount += charCount2[char];
+    }
+  }
+
+  return diffCount <= maxDifference * 2;
+}

package/test.js

@@ -1,184 +0,0 @@
-// full-example.js
-const { IDProfanityFilter, idFilter } = require('./dist');
-
-/**
- * Contoh Penggunaan Lengkap ID-Profanity-Filter
- * ============================================
- * File ini mencakup semua contoh penggunaan utama library.
- */
-
-console.log('============ CONTOH PENGGUNAAN DASAR ============\n');
-
-const filter = new IDProfanityFilter();
-
-const teks =
-  'Dasar kontol kontol kntl babi asu ngentod kamu, jangan banyak bacot! perek';
-
-// ===== 1. Cek apakah teks mengandung kata kotor =====
-const hasProfanity = filter.isProfane(teks);
-console.log('1. Apakah teks mengandung kata kotor?', hasProfanity);
-
-// ===== 2. Filter kata kotor (mengganti dengan sensor) =====
-const hasil = filter.filter(teks);
-console.log('\n2. Hasil filter:');
-console.log('- Teks asli:', teks);
-console.log('- Teks tersensor:', hasil.filtered);
-console.log('- Jumlah kata yang disensor:', hasil.censored);
-console.log('- Detail penggantian:', hasil.replacements);
-
-// ===== 3. Analisis konten =====
-const analisis = filter.analyze(teks);
-console.log('\n3. Hasil analisis:');
-console.log('- Mengandung kata kotor:', analisis.hasProfanity);
-console.log('- Kata kotor yang ditemukan:', analisis.matches);
-console.log('- Kategori kata kotor:', analisis.categories);
-console.log('- Daerah asal kata kotor:', analisis.regions);
-console.log('- Skor keparahan:', analisis.severityScore.toFixed(2));
-
-// console.log('\n============ PENGGUNAAN PRESET ============\n');
-
-// // ===== 4. Menggunakan preset filter =====
-// console.log('4. Menggunakan preset filter:');
-
-// // Preset strict
-// filter.usePreset('strict');
-// console.log('- Preset strict:', filter.filter(teks).filtered);
-
-// // Preset childSafe
-// filter.usePreset('childSafe');
-// console.log('- Preset childSafe:', filter.filter(teks).filtered);
-
-// // Preset light
-// filter.usePreset('light');
-// console.log('- Preset light:', filter.filter(teks).filtered);
-
-console.log('\n============ KUSTOMISASI FILTER ============\n');
-
-// ===== 5. Kustomisasi filter =====
-console.log('5. Kustomisasi filter:');
-
-filter.setOptions({
-  replaceWith: '#', // Menggunakan # sebagai karakter pengganti
-  fullWordCensor: false, // Hanya menyensor sebagian kata
-  keepFirstAndLast: true, // Menyimpan huruf pertama dan terakhir
-});
-
-console.log('- Filter dengan opsi kustom:', filter.filter(teks).filtered);
-
-// Mendeteksi variasi penulisan
-filter.setOptions({
-  detectLeetSpeak: true,
-  indonesianVariation: true,
-  detectSplit: true,
-  detectSimilarity: true,
-  useLevenshtein: true,
-  similarityThreshold: 0.85,
-  maxLevenshteinDistance: 2,
-});
-
-const teks2 =
-  'Dasar b4b1 kamu, j4nc0k! Ngent0d! k-o-n-t-o-l! k o n t o l k-0nt0l! anjiing kontool kwontol';
-console.log('- Teks asli dengan variasi:', teks2);
-console.log('- Hasil filter variasi:', filter.filter(teks2).filtered);
-
-// ===== 6. Menggunakan whitelist =====
-console.log('\n6. Menggunakan whitelist:');
-
-filter.addToWhitelist('anjing');
-const tekstBinatang =
-  'Anjing itu hewan peliharaan yang setia, tidak seperti bajingan itu';
-
-console.log('- Teks dengan "anjing" dalam konteks binatang:', tekstBinatang);
-console.log(
-  '- Hasil filter dengan whitelist:',
-  filter.filter(tekstBinatang).filtered,
-);
-
-// ===== 7. Menggunakan daftar kata kustom =====
-console.log('\n7. Menggunakan daftar kata kustom:');
-
-const customBadWords = ['jelek', 'buruk', 'sampah', 'payah'];
-
-// Reset filter dan gunakan daftar kustom
-filter.setOptions({
-  wordList: customBadWords,
-  replaceWith: '*',
-});
-
-const teksKustom = 'Film ini jelek dan payah sekali!';
-console.log('- Teks asli:', teksKustom);
-console.log('- Hasil filter kustom:', filter.filter(teksKustom).filtered);
-
-console.log('\n============ ANALISIS LANJUTAN ============\n');
-
-// ===== 8. Analisis per kalimat =====
-console.log('8. Analisis per kalimat:');
-
-filter.setOptions({}); // Reset ke default
-const kalimat =
-  'Saya sangat suka filmnya. Tapi pemainnya seperti anjing, aktingnya buruk.';
-console.log('- Kalimat:', kalimat);
-
-const kalimatAnalisis = filter.analyzeBySentence(kalimat);
-console.log('- Hasil analisis per kalimat:');
-kalimatAnalisis.forEach((hasil, index) => {
-  console.log(`  Kalimat ${index + 1}: "${hasil.sentence}"`);
-  console.log(`  Mengandung kata kotor: ${hasil.hasProfanity}`);
-  if (hasil.hasProfanity) {
-    console.log(`  Kata kotor: ${hasil.matches.join(', ')}`);
-  }
-});
-
-// ===== 9. Analisis batch untuk komentar =====
-console.log('\n9. Analisis batch:');
-
-const komentar = [
-  'Film ini sangat bagus, ceritanya menarik sekali!',
-  'Dasar goblok, sialan kamu!',
-  'Anjing emang filmnya, sampah banget.',
-];
-
-console.log('- Batch teks:');
-komentar.forEach((k, i) => console.log(`  ${i + 1}. "${k}"`));
-
-const hasilBatch = filter.batchAnalyze(komentar);
-console.log('- Hasil analisis batch:');
-console.log(`  - Total teks: ${hasilBatch.totalTexts}`);
-console.log(`  - Teks mengandung kata kotor: ${hasilBatch.profaneTexts}`);
-console.log(`  - Teks bersih: ${hasilBatch.cleanTexts}`);
-console.log(`  - Kategori teratas: ${hasilBatch.topCategories.join(', ')}`);
-console.log(`  - Daerah teratas: ${hasilBatch.topRegions.join(', ')}`);
-console.log('  - Kata kotor terbanyak:');
-hasilBatch.mostFrequentWords.forEach((word) => {
-  console.log(`    * "${word.word}": ${word.count} kali`);
-});
-
-// ===== 10. Filter berdasarkan kategori dan daerah =====
-console.log('\n10. Filter berdasarkan kategori dan daerah:');
-
-// Filter untuk kata dari daerah Jawa saja
-const filterJawa = new IDProfanityFilter({
-  regions: ['jawa'], // Hanya filter kata dari Jawa
-});
-
-const contohTeksJawa = 'Dasar jancok, asu, bangsat, bacot!';
-console.log('- Teks asli:', contohTeksJawa);
-console.log(
-  '- Filter khusus kata Jawa:',
-  filterJawa.filter(contohTeksJawa).filtered,
-);
-
-// Filter untuk kategori sexual saja
-const filterSexual = new IDProfanityFilter({
-  categories: ['sexual'], // Hanya filter kata kategori seksual
-});
-
-const contohTeksSexual =
-  'Kata kotor seperti anjing dan goblok tidak disensor, tapi bokep disensor.';
-console.log('- Teks asli:', contohTeksSexual);
-console.log(
-  '- Filter khusus kategori sexual:',
-  filterSexual.filter(contohTeksSexual).filtered,
-);
-
-console.log('\n============ SELESAI ============');