@sideid/id-profanity-filter 1.9.5 → 1.10.4

package/src/core/matcher.ts

@@ -3,26 +3,21 @@ import {
   ProfanityCategory,
   Region,
   FilterOptions,
-} from "../types";
+} from '../types';
 
-import { wordObjects, getWordsByFilter } from "../constants/wordList";
-import {
-  normalizeText,
-  escapeRegExp,
-  containsAnyWord,
-  detectSplitWords,
-} from "../utils/stringUtils";
-import {
-  createWordRegex,
-  addLeetSpeakVariations,
-  addIndonesianVariations,
-  addSplitVariations,
-} from "../utils/regexUtils";
+import { wordObjects } from '../constants/wordList';
+import { normalizeText } from '../utils/stringUtils';
+import { createWordRegex } from '../utils/regexUtils';
 import {
   findPossibleProfanityBySimiliarity,
-  stringSimilarity,
-} from "../utils/similarityUtils";
-import { DEFAULT_OPTIONS } from "../config/options";
+  findProfanityByLevenshteinDistance,
+} from '../utils/similarityUtils';
+import { DEFAULT_OPTIONS } from '../config/options';
+
+interface FindProfanityFunction {
+  (text: string, options?: FilterOptions): string[];
+  lastActualMatches?: Map<string, string[]>;
+}
 
 export function findProfanity(
   text: string,
@@ -40,38 +35,55 @@ export function findProfanity(
     detectSimilarity = false,
     similarityThreshold = 0.8,
     detectSplit = false,
+    useLevenshtein = false,
+    maxLevenshteinDistance = 2,
   } = { ...DEFAULT_OPTIONS, ...options };
 
   const normalizedText = normalizeText(text);
 
-  let wordsToCheck: string[] = wordList.length > 0 ? wordList : [];
+  let baseWordsToCheck: string[] = wordList.length > 0 ? wordList : [];
 
-  if (wordsToCheck.length === 0) {
-    if (categories || regions || severityThreshold > 0) {
-      wordsToCheck = wordObjects
-        .filter((word) => {
-          const matchCategory = categories
-            ? categories.includes(word.category)
-            : true;
-          const matchRegion = regions ? regions.includes(word.region) : true;
-          const matchSeverity = word.severity >= severityThreshold;
-          return matchCategory && matchRegion && matchSeverity;
-        })
-        .map((word) => word.word);
-    } else {
-      wordsToCheck = wordObjects.map((word) => word.word);
-    }
+  if (baseWordsToCheck.length === 0) {
+    const filteredWords = wordObjects.filter((word) => {
+      const matchCategory = categories
+        ? categories.includes(word.category)
+        : true;
+      const matchRegion = regions ? regions.includes(word.region) : true;
+      const matchSeverity = word.severity >= severityThreshold;
+      return matchCategory && matchRegion && matchSeverity;
+    });
+
+    baseWordsToCheck = filteredWords.map((word) => word.word);
   }
 
-  wordsToCheck = wordsToCheck.filter(
-    (word) => !whitelist.includes(word.toLocaleLowerCase()),
-  );
+  const aliasMap = new Map<string, string>();
+  wordObjects.forEach((wordObj) => {
+    if (wordObj.aliases && wordObj.aliases.length > 0) {
+      const matchCategory = categories
+        ? categories.includes(wordObj.category)
+        : true;
+      const matchRegion = regions ? regions.includes(wordObj.region) : true;
+      const matchSeverity = wordObj.severity >= severityThreshold;
+
+      if (matchCategory && matchRegion && matchSeverity) {
+        wordObj.aliases.forEach((alias) => {
+          aliasMap.set(alias.toLowerCase(), wordObj.word.toLowerCase());
+        });
+      }
+    }
+  });
+
+  const wordsToCheck = [
+    ...baseWordsToCheck,
+    ...Array.from(aliasMap.keys()),
+  ].filter((word) => !whitelist.includes(word.toLowerCase()));
 
   if (wordsToCheck.length === 0) {
     return [];
   }
 
   const matches = new Set<string>();
+  const actualMatches = new Map<string, string[]>();
 
   wordsToCheck.forEach((word) => {
     const regex = createWordRegex(word, {
@@ -84,7 +96,14 @@ export function findProfanity(
 
     let match;
     while ((match = regex.exec(normalizedText)) !== null) {
-      matches.add(word.toLowerCase());
+      const originalWord =
+        aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+      matches.add(originalWord);
+
+      if (!actualMatches.has(originalWord)) {
+        actualMatches.set(originalWord, []);
+      }
+      actualMatches.get(originalWord)?.push(match[0]);
     }
   });
 
@@ -100,7 +119,14 @@ export function findProfanity(
 
       let match;
       while ((match = leetRegex.exec(text)) !== null) {
-        matches.add(word.toLowerCase());
+        const originalWord =
+          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(match[0]);
       }
     });
   }
@@ -117,41 +143,85 @@ export function findProfanity(
 
       let match;
       while ((match = variantRegex.exec(text)) !== null) {
-        matches.add(word.toLowerCase());
+        const originalWord =
+          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(match[0]);
       }
     });
   }
 
   if (detectSplit) {
-    if (detectSplitWords(text, wordsToCheck)) {
-      wordsToCheck.forEach((word) => {
-        const splitRegex = createWordRegex(word, {
-          wholeWord: false,
-          caseSensitive: false,
-          leetSpeak: false,
-          detectSplit: true,
-          indonesianVariation: false,
-        });
+    wordsToCheck.forEach((word) => {
+      const splitRegex = createWordRegex(word, {
+        wholeWord: false,
+        caseSensitive: false,
+        leetSpeak: false,
+        detectSplit: true,
+        indonesianVariation: false,
+      });
 
-        if (splitRegex.test(text)) {
-          matches.add(word.toLowerCase());
+      let match;
+      while ((match = splitRegex.exec(text)) !== null) {
+        const originalWord =
+          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
         }
-      });
-    }
+        actualMatches.get(originalWord)?.push(match[0]);
+      }
+    });
   }
 
   if (detectSimilarity) {
-    const possibleProfanity = findPossibleProfanityBySimiliarity(
-      text,
-      wordsToCheck,
-      similarityThreshold,
-    );
-
-    possibleProfanity.forEach((item) => {
-      matches.add(item.original.toLowerCase());
-    });
+    if (useLevenshtein) {
+      const possibleProfanity = findProfanityByLevenshteinDistance(
+        text,
+        wordsToCheck,
+        similarityThreshold,
+        maxLevenshteinDistance,
+      );
+
+      possibleProfanity.forEach((item) => {
+        const originalWord =
+          aliasMap.get(item.original.toLowerCase()) ||
+          item.original.toLowerCase();
+        matches.add(originalWord);
+
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(item.word);
+      });
+    } else {
+      const possibleProfanity = findPossibleProfanityBySimiliarity(
+        text,
+        wordsToCheck,
+        similarityThreshold,
+      );
+
+      possibleProfanity.forEach((item) => {
+        matches.add(item.original.toLowerCase());
+
+        const originalWord =
+          aliasMap.get(item.original.toLowerCase()) ||
+          item.original.toLowerCase();
+        if (!actualMatches.has(originalWord)) {
+          actualMatches.set(originalWord, []);
+        }
+        actualMatches.get(originalWord)?.push(item.word);
+      });
+    }
   }
 
+  (findProfanity as FindProfanityFunction).lastActualMatches = actualMatches;
+
   return Array.from(matches);
 }
 

package/src/index.ts

@@ -1,28 +1,27 @@
-export * from "./types";
-export * from "./core/matcher";
-export * from "./core/filter";
-export * from "./core/analyzer";
-export * from "./utils/stringUtils";
-export * from "./utils/regexUtils";
-export * from "./utils/similarityUtils";
-export * from "./config/options";
-
-import { filter, isProfane } from "./core/filter";
+export * from './types';
+export * from './core/matcher';
+export * from './core/filter';
+export * from './core/analyzer';
+export * from './utils/stringUtils';
+export * from './utils/regexUtils';
+export * from './utils/similarityUtils';
+export * from './config/options';
+
+import { filter, isProfane } from './core/filter';
 import {
   analyze,
   batchAnalyze,
   analyzeBySentence,
   analyzeWithContext,
-} from "./core/analyzer";
-import { FilterOptions, FilterResult, AnalysisResult } from "./types";
+} from './core/analyzer';
+import { FilterOptions, FilterResult, AnalysisResult } from './types';
 import {
   DEFAULT_OPTIONS,
   FILTER_PRESETS,
   CATEGORY_PRESETS,
   REGION_PRESETS,
   getPresetOptions,
-  makeRandomGrawlixString,
-} from "./config/options";
+} from './config/options';
 
 export class IDProfanityFilter {
   private options: FilterOptions;
@@ -161,10 +160,30 @@ export class IDProfanityFilter {
   /**
    * Mengaktifkan deteksi berdasarkan kesamaan
    * @param threshold Threshold kesamaan (0-1)
+   * @param useLevenshtein Gunakan algoritma Levenshtein untuk deteksi
+   * @param maxLevenshteinDistance Jarak maksimal Levenshtein (default: 2)
    */
-  enableSimilarityDetection(threshold: number = 0.8) {
+  enableSimilarityDetection(
+    threshold: number = 0.8,
+    useLevenshtein: boolean = false,
+    maxLevenshteinDistance: number = 2,
+  ) {
+    this.options.detectSimilarity = true;
+    this.options.similarityThreshold = threshold;
+    this.options.useLevenshtein = useLevenshtein;
+    this.options.maxLevenshteinDistance = maxLevenshteinDistance;
+  }
+
+  /**
+   * Mengaktifkan deteksi berbasis Levenshtein distance
+   * @param threshold Threshold kesamaan (0-1)
+   * @param maxDistance Jarak maksimal Levenshtein (default: 2)
+   */
+  enableLevenshteinDetection(threshold: number = 0.8, maxDistance: number = 2) {
     this.options.detectSimilarity = true;
+    this.options.useLevenshtein = true;
     this.options.similarityThreshold = threshold;
+    this.options.maxLevenshteinDistance = maxDistance;
   }
 }
 

package/src/types/index.ts

@@ -50,9 +50,11 @@ export interface FilterOptions {
   useRandomGrawlix?: boolean;
   keepFirstAndLast?: boolean;
   indonesianVariation?: boolean;
-  detectSimilarity?: boolean;
-  similarityThreshold?: number;
+  detectSimilarity?: boolean; // Enable/disable Levenshtein distance matching
+  similarityThreshold?: number; // Threshold for Levenshtein distance similarity (0-1)
   detectSplit?: boolean;
+  useLevenshtein?: boolean; // New option specifically for Levenshtein algorithm
+  maxLevenshteinDistance?: number; // Maximum allowed Levenshtein distance
 }
 
 export interface FilterResult {

package/src/utils/regexUtils.ts

@@ -72,7 +72,6 @@ export function addLeetSpeakVariations(pattern: string): string {
     z: ["z", "2"],
   };
 
-  // Ganti tiap karakter dengan variasinya dalam grup character class
   return pattern
     .split("")
     .map((char) => {

package/src/utils/similarityUtils.ts

@@ -91,6 +91,48 @@ export function findMostSimilar(
   return mostSimilar;
 }
 
+/**
+ * Mencari string yang paling mirip dari array menggunakan Levenshtein distance
+ *
+ * @param target String target
+ * @param candidates Array string kandidat
+ * @param threshold Minimum kesamaan yang diterima (0-1)
+ * @param maxDistance Jarak Levenshtein maksimal yang diterima (default: 3)
+ * @returns String yang paling mirip atau null jika tidak ada yang di atas threshold
+ */
+export function findMostSimilarWithLevenshtein(
+  target: string,
+  candidates: string[],
+  threshold: number = 0.7,
+  maxDistance: number = 3,
+): string | null {
+  if (!candidates.length) return null;
+
+  let maxSimilarity = 0;
+  let minDistance = Infinity;
+  let mostSimilar: string | null = null;
+
+  for (const candidate of candidates) {
+    if (Math.abs(target.length - candidate.length) > maxDistance) continue;
+
+    const distance = levenshteinDistance(target, candidate);
+    const similarity = stringSimilarity(target, candidate);
+
+    if (
+      (similarity > maxSimilarity && similarity >= threshold) ||
+      (similarity >= threshold && distance < minDistance)
+    ) {
+      maxSimilarity = similarity;
+      minDistance = distance;
+      mostSimilar = candidate;
+
+      if (distance <= 1 || similarity > 0.95) break;
+    }
+  }
+
+  return mostSimilar;
+}
+
 /**
  * Cek apakah string mungkin merupakan variasi dari kata kotor
  * menggunakan kesamaan string
@@ -170,11 +212,9 @@ export function findPossibleProfanityBySimiliarity(
   const result: Array<{ word: string; original: string; similarity: number }> =
     [];
 
-  // Pisahkan teks menjadi kata-kata
   const words = text.toLowerCase().split(/\s+/);
 
   for (const word of words) {
-    // Lewati kata-kata yang terlalu pendek
     if (word.length < 3) continue;
 
     for (const profanity of profanityWords) {
@@ -193,3 +233,58 @@ export function findPossibleProfanityBySimiliarity(
 
   return result;
 }
+
+/**
+ * Cari kata-kata kotor yang mungkin dari teks menggunakan Levenshtein distance
+ *
+ * @param text Teks yang akan diperiksa
+ * @param profanityWords Daftar kata kotor
+ * @param threshold Batas minimum kesamaan (default: 0.8)
+ * @param maxDistance Jarak Levenshtein maksimal (default: 2)
+ * @returns Array kata yang mungkin merupakan kata kotor
+ */
+export function findProfanityByLevenshteinDistance(
+  text: string,
+  profanityWords: string[],
+  threshold: number = 0.8,
+  maxDistance: number = 2,
+): Array<{
+  word: string;
+  original: string;
+  similarity: number;
+  distance: number;
+}> {
+  const result: Array<{
+    word: string;
+    original: string;
+    similarity: number;
+    distance: number;
+  }> = [];
+
+  const words = text.toLowerCase().split(/\s+/);
+
+  for (const word of words) {
+    if (word.length < 3) continue;
+
+    for (const profanity of profanityWords) {
+      if (Math.abs(word.length - profanity.length) > maxDistance) continue;
+
+      const distance = levenshteinDistance(word, profanity);
+      if (distance <= maxDistance) {
+        const similarity = stringSimilarity(word, profanity);
+
+        if (similarity >= threshold) {
+          result.push({
+            word,
+            original: profanity,
+            similarity,
+            distance,
+          });
+          break;
+        }
+      }
+    }
+  }
+
+  return result;
+}

package/test.js

@@ -0,0 +1,184 @@
+// full-example.js
+const { IDProfanityFilter, idFilter } = require('./dist');
+
+/**
+ * Contoh Penggunaan Lengkap ID-Profanity-Filter
+ * ============================================
+ * File ini mencakup semua contoh penggunaan utama library.
+ */
+
+console.log('============ CONTOH PENGGUNAAN DASAR ============\n');
+
+const filter = new IDProfanityFilter();
+
+const teks =
+  'Dasar kontol kontol kntl babi asu ngentod kamu, jangan banyak bacot! perek';
+
+// ===== 1. Cek apakah teks mengandung kata kotor =====
+const hasProfanity = filter.isProfane(teks);
+console.log('1. Apakah teks mengandung kata kotor?', hasProfanity);
+
+// ===== 2. Filter kata kotor (mengganti dengan sensor) =====
+const hasil = filter.filter(teks);
+console.log('\n2. Hasil filter:');
+console.log('- Teks asli:', teks);
+console.log('- Teks tersensor:', hasil.filtered);
+console.log('- Jumlah kata yang disensor:', hasil.censored);
+console.log('- Detail penggantian:', hasil.replacements);
+
+// ===== 3. Analisis konten =====
+const analisis = filter.analyze(teks);
+console.log('\n3. Hasil analisis:');
+console.log('- Mengandung kata kotor:', analisis.hasProfanity);
+console.log('- Kata kotor yang ditemukan:', analisis.matches);
+console.log('- Kategori kata kotor:', analisis.categories);
+console.log('- Daerah asal kata kotor:', analisis.regions);
+console.log('- Skor keparahan:', analisis.severityScore.toFixed(2));
+
+// console.log('\n============ PENGGUNAAN PRESET ============\n');
+
+// // ===== 4. Menggunakan preset filter =====
+// console.log('4. Menggunakan preset filter:');
+
+// // Preset strict
+// filter.usePreset('strict');
+// console.log('- Preset strict:', filter.filter(teks).filtered);
+
+// // Preset childSafe
+// filter.usePreset('childSafe');
+// console.log('- Preset childSafe:', filter.filter(teks).filtered);
+
+// // Preset light
+// filter.usePreset('light');
+// console.log('- Preset light:', filter.filter(teks).filtered);
+
+console.log('\n============ KUSTOMISASI FILTER ============\n');
+
+// ===== 5. Kustomisasi filter =====
+console.log('5. Kustomisasi filter:');
+
+filter.setOptions({
+  replaceWith: '#', // Menggunakan # sebagai karakter pengganti
+  fullWordCensor: false, // Hanya menyensor sebagian kata
+  keepFirstAndLast: true, // Menyimpan huruf pertama dan terakhir
+});
+
+console.log('- Filter dengan opsi kustom:', filter.filter(teks).filtered);
+
+// Mendeteksi variasi penulisan
+filter.setOptions({
+  detectLeetSpeak: true,
+  indonesianVariation: true,
+  detectSplit: true,
+  detectSimilarity: true,
+  useLevenshtein: true,
+  similarityThreshold: 0.85,
+  maxLevenshteinDistance: 2,
+});
+
+const teks2 =
+  'Dasar b4b1 kamu, j4nc0k! Ngent0d! k-o-n-t-o-l! k o n t o l k-0nt0l! anjiing kontool kwontol';
+console.log('- Teks asli dengan variasi:', teks2);
+console.log('- Hasil filter variasi:', filter.filter(teks2).filtered);
+
+// ===== 6. Menggunakan whitelist =====
+console.log('\n6. Menggunakan whitelist:');
+
+filter.addToWhitelist('anjing');
+const tekstBinatang =
+  'Anjing itu hewan peliharaan yang setia, tidak seperti bajingan itu';
+
+console.log('- Teks dengan "anjing" dalam konteks binatang:', tekstBinatang);
+console.log(
+  '- Hasil filter dengan whitelist:',
+  filter.filter(tekstBinatang).filtered,
+);
+
+// ===== 7. Menggunakan daftar kata kustom =====
+console.log('\n7. Menggunakan daftar kata kustom:');
+
+const customBadWords = ['jelek', 'buruk', 'sampah', 'payah'];
+
+// Reset filter dan gunakan daftar kustom
+filter.setOptions({
+  wordList: customBadWords,
+  replaceWith: '*',
+});
+
+const teksKustom = 'Film ini jelek dan payah sekali!';
+console.log('- Teks asli:', teksKustom);
+console.log('- Hasil filter kustom:', filter.filter(teksKustom).filtered);
+
+console.log('\n============ ANALISIS LANJUTAN ============\n');
+
+// ===== 8. Analisis per kalimat =====
+console.log('8. Analisis per kalimat:');
+
+filter.setOptions({}); // Reset ke default
+const kalimat =
+  'Saya sangat suka filmnya. Tapi pemainnya seperti anjing, aktingnya buruk.';
+console.log('- Kalimat:', kalimat);
+
+const kalimatAnalisis = filter.analyzeBySentence(kalimat);
+console.log('- Hasil analisis per kalimat:');
+kalimatAnalisis.forEach((hasil, index) => {
+  console.log(`  Kalimat ${index + 1}: "${hasil.sentence}"`);
+  console.log(`  Mengandung kata kotor: ${hasil.hasProfanity}`);
+  if (hasil.hasProfanity) {
+    console.log(`  Kata kotor: ${hasil.matches.join(', ')}`);
+  }
+});
+
+// ===== 9. Analisis batch untuk komentar =====
+console.log('\n9. Analisis batch:');
+
+const komentar = [
+  'Film ini sangat bagus, ceritanya menarik sekali!',
+  'Dasar goblok, sialan kamu!',
+  'Anjing emang filmnya, sampah banget.',
+];
+
+console.log('- Batch teks:');
+komentar.forEach((k, i) => console.log(`  ${i + 1}. "${k}"`));
+
+const hasilBatch = filter.batchAnalyze(komentar);
+console.log('- Hasil analisis batch:');
+console.log(`  - Total teks: ${hasilBatch.totalTexts}`);
+console.log(`  - Teks mengandung kata kotor: ${hasilBatch.profaneTexts}`);
+console.log(`  - Teks bersih: ${hasilBatch.cleanTexts}`);
+console.log(`  - Kategori teratas: ${hasilBatch.topCategories.join(', ')}`);
+console.log(`  - Daerah teratas: ${hasilBatch.topRegions.join(', ')}`);
+console.log('  - Kata kotor terbanyak:');
+hasilBatch.mostFrequentWords.forEach((word) => {
+  console.log(`    * "${word.word}": ${word.count} kali`);
+});
+
+// ===== 10. Filter berdasarkan kategori dan daerah =====
+console.log('\n10. Filter berdasarkan kategori dan daerah:');
+
+// Filter untuk kata dari daerah Jawa saja
+const filterJawa = new IDProfanityFilter({
+  regions: ['jawa'], // Hanya filter kata dari Jawa
+});
+
+const contohTeksJawa = 'Dasar jancok, asu, bangsat, bacot!';
+console.log('- Teks asli:', contohTeksJawa);
+console.log(
+  '- Filter khusus kata Jawa:',
+  filterJawa.filter(contohTeksJawa).filtered,
+);
+
+// Filter untuk kategori sexual saja
+const filterSexual = new IDProfanityFilter({
+  categories: ['sexual'], // Hanya filter kata kategori seksual
+});
+
+const contohTeksSexual =
+  'Kata kotor seperti anjing dan goblok tidak disensor, tapi bokep disensor.';
+console.log('- Teks asli:', contohTeksSexual);
+console.log(
+  '- Filter khusus kategori sexual:',
+  filterSexual.filter(contohTeksSexual).filtered,
+);
+
+console.log('\n============ SELESAI ============');

package/src/constants/categories/index.ts

@@ -1,31 +0,0 @@
-// import { sexual } from './sexual';
-// import { insult } from './insult';
-// import { profanity } from './profanity';
-// import { slur } from './slur';
-// import { drugs } from './drugs';
-// import { disgusting } from './disgusting';
-// import { blasphemy } from './blasphemy';
-
-// export const categories = {
-//   sexual, // Kata-kata berbau seksual
-//   insult, // Kata-kata penghinaan
-//   profanity, // Umpatan umum
-//   slur, // Perkataan merendahkan berdasarkan identitas
-//   drugs, // Terkait narkoba
-//   disgusting, // Kata-kata menjijikkan
-//   blasphemy, // Penistaan agama
-// };
-
-// export { sexual, insult, profanity, slur, drugs, disgusting, blasphemy };
-
-// export const allCategoryWords = [
-//   ...sexual,
-//   ...insult,
-//   ...profanity,
-//   ...slur,
-//   ...drugs,
-//   ...disgusting,
-//   ...blasphemy,
-// ];
-
-// export default allCategoryWords;