@sideid/id-profanity-filter 1.9.6 → 1.10.2

package/src/constants/regions/jawa.ts

@@ -14,7 +14,7 @@ export const jawa: ProfanityWord[] = [
     word: "jancok",
     category: "sexual",
     region: "jawa",
-    severity: 0.8,
+    severity: 0.9,
     aliases: ["jancuk", "jncok", "jancuk", "jncuk", "dancok", "dancuk"],
     description: "Kata umpatan kasar dalam Bahasa Jawa",
     context: "Umpatan kasar yang umum digunakan di Jawa Timur",
@@ -52,7 +52,7 @@ export const jawa: ProfanityWord[] = [
     word: "mbokne ancok",
     category: "insult",
     region: "jawa",
-    severity: 0.8,
+    severity: 0.9,
     aliases: ["mbokne", "mbokneancok"],
     description: "Umpatan yang menyinggung ibu seseorang",
     context: "Umpatan kasar yang menyinggung orangtua orang lain",
@@ -61,7 +61,7 @@ export const jawa: ProfanityWord[] = [
     word: "pekok",
     category: "insult",
     region: "jawa",
-    severity: 0.6,
+    severity: 0.7,
     aliases: ["pekak", "pekilk"],
     description: "Kata hinaan yang menunjukkan kebodohan",
     context: "Hinaan untuk menyebut orang yang dianggap sangat bodoh",
@@ -95,6 +95,260 @@ export const jawa: ProfanityWord[] = [
     context:
       "Umpatan ringan untuk menanggapi sesuatu yang dianggap tidak benar",
   },
+  {
+    word: "kontol",
+    category: "sexual",
+    region: "jawa",
+    severity: 0.8,
+    aliases: ["kntl", "kontl"],
+    description: "Mengacu ke alat kelamin laki-laki",
+    context: "Kata vulgar yang sering digunakan sebagai umpatan kasar",
+  },
+  {
+    word: "tempek",
+    category: "sexual",
+    region: "jawa",
+    severity: 0.8,
+    aliases: ["mpek", "torok", "tempk"],
+    description: "Mengacu pada alat kelamin perempuan",
+    context: "Kata vulgar yang digunakan sebagai umpatan atau hinaan",
+  },
+  {
+    word: "silit",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["selet", "tilis"],
+    description: "Mengacu pada bagian dubur atau anus",
+    context: "Kata kasar yang digunakan sebagai hinaan",
+  },
+  {
+    word: "mbahmu",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["mbahmu kiper", "mbah mu"],
+    description: "Hinaan yang menyinggung nenek/kakek seseorang",
+    context: "Umpatan ringan untuk menanggapi sesuatu yang tidak masuk akal",
+  },
+  {
+    word: "makmu",
+    category: "insult",
+    region: "jawa",
+    severity: 0.7,
+    aliases: ["mak mu", "mamamu"],
+    description: "Hinaan yang menyinggung ibu seseorang",
+    context: "Umpatan yang dianggap kasar karena menyinggung orang tua",
+  },
+  {
+    word: "bajingan",
+    category: "insult",
+    region: "jawa",
+    severity: 0.7,
+    aliases: [
+      "bajilak",
+      "bajhingan",
+      "bajingak",
+      "bajingseng",
+      "bajindul",
+      "bajigur",
+      "jingan",
+    ],
+    description: "Sebutan untuk orang yang dianggap jahat atau tidak bermoral",
+    context: "Umpatan untuk mengekspresikan kemarahan atau kekesalan",
+  },
+  {
+    word: "cocote",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["cocot", "bacot", "nyocot"],
+    description: "Mengacu pada mulut dengan konotasi negatif",
+    context: "Umpatan untuk menyuruh seseorang berhenti berbicara",
+  },
+  {
+    word: "ngentot",
+    category: "sexual",
+    region: "jawa",
+    severity: 0.9,
+    aliases: ["kentu", "kentot", "iclik", "ngtt", "iclk"],
+    description: "Mengacu pada aktivitas seksual",
+    context: "Kata vulgar yang digunakan sebagai umpatan kasar",
+  },
+  {
+    word: "edan",
+    category: "insult",
+    region: "jawa",
+    severity: 0.5,
+    aliases: ["gendeng", "gila", "gendheng", "sarap"],
+    description: "Secara harfiah berarti gila atau tidak waras",
+    context: "Umpatan untuk menyebut seseorang yang dianggap tidak masuk akal",
+  },
+  {
+    word: "dapuranmu",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["raimu", "rai mu"],
+    description: "Secara harfiah mengacu pada wajah atau rupa seseorang",
+    context: "Umpatan untuk menghina penampilan atau wajah seseorang",
+  },
+  {
+    word: "damput",
+    category: "insult",
+    region: "jawa",
+    severity: 0.7,
+    aliases: ["diamput"],
+    description: "Variasi bentuk umpatan dengan makna serupa dengan jancok",
+    context: "Umpatan kasar untuk mengekspresikan kemarahan",
+  },
+  {
+    word: "mbathang",
+    category: "insult",
+    region: "jawa",
+    severity: 0.7,
+    aliases: ["mbatang"],
+    description: "Secara harfiah berarti bangkai",
+    context: "Umpatan kasar untuk menghina seseorang",
+  },
+  {
+    word: "ndlogok",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["ndelodok", "ndlodok"],
+    description:
+      "Mengacu pada tindakan yang dianggap bodoh atau tidak masuk akal",
+    context: "Hinaan untuk mengkritik tindakan seseorang",
+  },
+  {
+    word: "nggateli",
+    category: "insult",
+    region: "jawa",
+    severity: 0.5,
+    aliases: ["gateli", "gathel"],
+    description: "Secara harfiah berarti gatal atau menyebalkan",
+    context: "Ungkapan untuk menunjukkan kekesalan terhadap perilaku seseorang",
+  },
+  {
+    word: "perek",
+    category: "sexual",
+    region: "jawa",
+    severity: 0.8,
+    aliases: ["lonthe", "pelacur"],
+    description: "Istilah merendahkan untuk pekerja seks komersial",
+    context: "Kata kasar untuk menghina wanita",
+  },
+  {
+    word: "picek",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["pcek", "buta"],
+    description: "Secara harfiah berarti buta atau tidak bisa melihat",
+    context: "Hinaan untuk orang yang dianggap tidak bisa melihat kenyataan",
+  },
+  {
+    word: "untumu",
+    category: "insult",
+    region: "jawa",
+    severity: 0.5,
+    aliases: ["gigimu", "untu mu"],
+    description: "Secara harfiah berarti gigimu",
+    context: "Umpatan ringan untuk menanggapi sesuatu yang tidak disetujui",
+  },
+  {
+    word: "goblog",
+    category: "insult",
+    region: "jawa",
+    severity: 0.7,
+    aliases: ["ghoblog", "goblok", "gobhlok", "pekok"],
+    description: "Kata hinaan yang menunjukkan kebodohan ekstrem",
+    context: "Hinaan untuk menyebut seseorang yang dianggap sangat bodoh",
+  },
+  {
+    word: "tolol",
+    category: "insult",
+    region: "jawa",
+    severity: 0.7,
+    aliases: ["tholol", "tlol"],
+    description: "Kata hinaan yang menunjukkan kebodohan",
+    context: "Hinaan untuk menyebut seseorang yang dianggap bodoh",
+  },
+  {
+    word: "budheg",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["budeg", "bdeg"],
+    description: "Secara harfiah berarti tuli atau tidak bisa mendengar",
+    context: "Hinaan untuk orang yang dianggap tidak mau mendengarkan",
+  },
+  {
+    word: "jiangkrik",
+    category: "insult",
+    region: "jawa",
+    severity: 0.4,
+    aliases: ["jiangkrek", "jangkrik"],
+    description: "Secara harfiah berarti jangkrik, digunakan sebagai eufemisme",
+    context: "Umpatan ringan sebagai pengganti kata kasar yang lebih vulgar",
+  },
+  {
+    word: "diamput",
+    category: "insult",
+    region: "jawa",
+    severity: 0.8,
+    aliases: ["damput", "djamput"],
+    description: "Bentuk umpatan kasar dengan makna serupa jancok",
+    context: "Kata kasar untuk mengekspresikan kemarahan",
+  },
+  {
+    word: "celeng",
+    category: "insult",
+    region: "jawa",
+    severity: 0.6,
+    aliases: ["cleng", "babi hutan"],
+    description: "Secara harfiah berarti babi hutan",
+    context: "Hinaan untuk orang yang dianggap jorok atau rakus",
+  },
+  {
+    word: "kampret",
+    category: "insult",
+    region: "jawa",
+    severity: 0.5,
+    aliases: ["kmpret", "kmprt"],
+    description: "Secara harfiah berarti kelelawar kecil",
+    context: "Umpatan ringan untuk mengekspresikan kekesalan",
+  },
+  {
+    word: "ndeso",
+    category: "insult",
+    region: "jawa",
+    severity: 0.4,
+    aliases: ["ndesa", "deso"],
+    description: "Secara harfiah berarti dari desa atau kampungan",
+    context:
+      "Hinaan untuk orang yang dianggap kurang modern atau berpendidikan",
+  },
+  {
+    word: "kere",
+    category: "insult",
+    region: "jawa",
+    severity: 0.5,
+    aliases: ["miskin", "mlarat"],
+    description: "Secara harfiah berarti miskin atau tidak punya uang",
+    context: "Hinaan untuk status ekonomi seseorang yang dianggap rendah",
+  },
+  {
+    word: "itil",
+    category: "sexual",
+    region: "jawa",
+    severity: 0.9,
+    aliases: ["itl", "itul"],
+    description:
+      "Kata vulgar yang mengacu pada bagian dari alat kelamin perempuan",
+    context: "Kata vulgar yang merujuk pada anatomi seksual",
+  },
 ];
 
 export const jawaWords = jawa.map((item) => item.word);

package/src/constants/wordList.ts

@@ -1,12 +1,11 @@
 import { ProfanityWord } from "../types";
-
-import { sexual, sexualWords } from "./categories/sexual";
-import { insult, insultWords } from "./categories/insult";
-// import { profanity, profanityWords } from './categories/profanity';
-// import { slur, slurWords } from './categories/slur';
-// import { drugs, drugsWords } from './categories/drugs';
-// import { disgusting, disgustingWords } from './categories/disgusting';
-// import { blasphemy, blasphemyWords } from './categories/blasphemy';
+import { sexualWords } from "./categories/sexual";
+import { insultWords } from "./categories/insult";
+// import { profanityWords } from './categories/profanity';
+// import { slurWords } from './categories/slur';
+// import { drugsWords } from './categories/drugs';
+// import { disgustingWords } from './categories/disgusting';
+// import { blasphemyWords } from './categories/blasphemy';
 
 import { general, generalWords } from "./regions/general";
 import { jawa, jawaWords } from "./regions/jawa";
@@ -96,6 +95,14 @@ export const severeWords: string[] = wordObjects
  * @returns Array dari kata kotor
  */
 export function getWordsByFilter(category?: string, region?: string): string[] {
+  if (category && !region && category in wordCategories) {
+    return wordCategories[category as keyof typeof wordCategories];
+  }
+
+  if (region && !category && region in wordRegions) {
+    return wordRegions[region as keyof typeof wordRegions];
+  }
+
   return wordObjects
     .filter((word) => {
       const matchCategory = category ? word.category === category : true;

package/src/core/analyzer.ts

@@ -1,7 +1,6 @@
 import {
   FilterOptions,
   AnalysisResult,
-  ProfanityWord,
   ProfanityCategory,
   Region,
 } from "../types";
@@ -12,13 +11,11 @@ import {
   findRegions,
   calculateSeverity,
 } from "./matcher";
+import { splitIntoSentences } from "../utils/stringUtils";
 import {
-  normalizeText,
-  escapeRegExp,
-  splitIntoSentences,
-  getContextAroundIndex,
-} from "../utils/stringUtils";
-import { findPossibleProfanityBySimiliarity } from "../utils/similarityUtils";
+  findPossibleProfanityBySimiliarity,
+  findProfanityByLevenshteinDistance,
+} from "../utils/similarityUtils";
 import { createContextRegex } from "../utils/regexUtils";
 import { DEFAULT_OPTIONS } from "../config/options";
 
@@ -60,12 +57,30 @@ export function analyze(
   }> = [];
 
   if (mergedOptions.detectSimilarity) {
-    const wordList = matchDetails.map((word) => word.word);
-    similarWords = findPossibleProfanityBySimiliarity(
-      text,
-      wordList,
-      mergedOptions.similarityThreshold || 0.8,
-    );
+    if (matchDetails.length > 0) {
+      const wordList = matchDetails.map((word) => word.word);
+
+      if (mergedOptions.useLevenshtein) {
+        const levenshteinResults = findProfanityByLevenshteinDistance(
+          text,
+          wordList,
+          mergedOptions.similarityThreshold || 0.8,
+          mergedOptions.maxLevenshteinDistance || 2,
+        );
+
+        similarWords = levenshteinResults.map((item) => ({
+          word: item.word,
+          original: item.original,
+          similarity: item.similarity,
+        }));
+      } else {
+        similarWords = findPossibleProfanityBySimiliarity(
+          text,
+          wordList,
+          mergedOptions.similarityThreshold || 0.8,
+        );
+      }
+    }
   }
 
   return {

package/src/core/filter.ts

@@ -1,12 +1,13 @@
 import { FilterOptions, FilterResult, ProfanityWord } from "../types";
 import { findProfanity, findProfanityWithMetadata } from "./matcher";
-import { censorWord, escapeRegExp, normalizeText } from "../utils/stringUtils";
+import { censorWord, escapeRegExp } from "../utils/stringUtils";
 import { createWordRegex } from "../utils/regexUtils";
-import {
-  DEFAULT_OPTIONS,
-  makeRandomGrawlixString,
-  getRandomGrawlix,
-} from "../config/options";
+import { DEFAULT_OPTIONS, makeRandomGrawlixString } from "../config/options";
+
+interface FindProfanityFunction {
+  (text: string, options?: FilterOptions): string[];
+  lastActualMatches?: Map<string, string[]>;
+}
 
 /**
  * Menyensor kata kotor dalam teks
@@ -28,6 +29,11 @@ export function filter(
     useRandomGrawlix = false,
     keepFirstAndLast = false,
     indonesianVariation = false,
+    detectSplit = false,
+    detectSimilarity = false,
+    useLevenshtein = false,
+    maxLevenshteinDistance = 2,
+    similarityThreshold = 0.8,
   } = { ...DEFAULT_OPTIONS, ...options };
 
   const matches = findProfanity(text, {
@@ -36,8 +42,16 @@ export function filter(
     whitelist,
     checkSubstring,
     indonesianVariation,
+    detectSplit,
+    detectSimilarity,
+    useLevenshtein,
+    maxLevenshteinDistance,
+    similarityThreshold,
   });
 
+  const actualMatches: Map<string, string[]> =
+    (findProfanity as FindProfanityFunction).lastActualMatches || new Map();
+
   const matchDetails = findProfanityWithMetadata(text, options);
 
   if (matches.length === 0) {
@@ -66,49 +80,176 @@ export function filter(
           )),
     );
 
-    const regex = createWordRegex(word, {
-      wholeWord: true,
-      caseSensitive: false,
-      leetSpeak: false,
-      detectSplit: false,
-      indonesianVariation: false,
-    });
+    const variants = actualMatches.get(word.toLowerCase()) || [];
+    variants.push(word);
+
+    const uniqueVariants = [...new Set(variants)];
 
-    let match;
-    const textToSearch = filteredText;
+    uniqueVariants.forEach((variant) => {
+      const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, "gi");
 
-    regex.lastIndex = 0;
+      let match;
+      while ((match = regex.exec(filteredText)) !== null) {
+        const originalWord = match[0];
 
-    while ((match = regex.exec(textToSearch)) !== null) {
-      const originalWord = match[0];
+        if (whitelist.includes(originalWord.toLowerCase())) continue;
 
-      if (whitelist.includes(originalWord.toLowerCase())) continue;
+        let censoredWord;
+        if (useRandomGrawlix) {
+          censoredWord = makeRandomGrawlixString(originalWord.length);
+        } else {
+          censoredWord = censorWord(
+            originalWord,
+            replaceWith,
+            !fullWordCensor && keepFirstAndLast,
+          );
+        }
 
-      let censoredWord;
-      if (useRandomGrawlix) {
-        censoredWord = makeRandomGrawlixString(originalWord.length);
-      } else {
-        censoredWord = censorWord(
-          originalWord,
-          replaceWith,
-          !fullWordCensor && keepFirstAndLast,
+        replacements.push({
+          original: originalWord,
+          censored: censoredWord,
+          metadata,
+        });
+
+        filteredText = filteredText.replace(
+          new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"),
+          censoredWord,
         );
       }
+    });
 
-      replacements.push({
-        original: originalWord,
-        censored: censoredWord,
-        metadata,
-      });
+    if (detectSplit || detectLeetSpeak) {
+      if (detectLeetSpeak) {
+        const leetRegex = createWordRegex(word, {
+          wholeWord: true,
+          caseSensitive: false,
+          leetSpeak: true,
+          detectSplit: false,
+          indonesianVariation: false,
+        });
 
-      const replaceRegex = new RegExp(
-        `\\b${escapeRegExp(originalWord)}\\b`,
-        "g",
-      );
-      filteredText = filteredText.replace(replaceRegex, censoredWord);
+        let match;
+        while ((match = leetRegex.exec(filteredText)) !== null) {
+          const originalWord = match[0];
+
+          if (whitelist.includes(originalWord.toLowerCase())) continue;
+
+          let censoredWord;
+          if (useRandomGrawlix) {
+            censoredWord = makeRandomGrawlixString(originalWord.length);
+          } else {
+            censoredWord = censorWord(
+              originalWord,
+              replaceWith,
+              !fullWordCensor && keepFirstAndLast,
+            );
+          }
+
+          replacements.push({
+            original: originalWord,
+            censored: censoredWord,
+            metadata,
+          });
+
+          filteredText = filteredText.replace(
+            new RegExp(escapeRegExp(originalWord), "g"),
+            censoredWord,
+          );
+        }
+      }
+
+      if (detectSplit) {
+        const splitRegex = createWordRegex(word, {
+          wholeWord: false,
+          caseSensitive: false,
+          leetSpeak: false,
+          detectSplit: true,
+          indonesianVariation: false,
+        });
+
+        let match;
+        while ((match = splitRegex.exec(filteredText)) !== null) {
+          const originalWord = match[0];
+
+          if (whitelist.includes(originalWord.toLowerCase())) continue;
+
+          let censoredWord;
+          if (useRandomGrawlix) {
+            censoredWord = makeRandomGrawlixString(originalWord.length);
+          } else {
+            censoredWord = censorWord(
+              originalWord,
+              replaceWith,
+              !fullWordCensor && keepFirstAndLast,
+            );
+          }
+
+          replacements.push({
+            original: originalWord,
+            censored: censoredWord,
+            metadata,
+          });
+
+          filteredText = filteredText.replace(
+            new RegExp(escapeRegExp(originalWord), "g"),
+            censoredWord,
+          );
+        }
+      }
     }
   });
 
+  if (detectSimilarity && useLevenshtein) {
+    matches.forEach((word) => {
+      const metadata = matchDetails.find(
+        (m) =>
+          m.word.toLowerCase() === word.toLowerCase() ||
+          (m.aliases &&
+            m.aliases.some(
+              (alias) => alias.toLowerCase() === word.toLowerCase(),
+            )),
+      );
+
+      const variants = actualMatches.get(word.toLowerCase()) || [];
+
+      variants.forEach((variant) => {
+        const exactVariantRegex = new RegExp(
+          `\\b${escapeRegExp(variant)}\\b`,
+          "gi",
+        );
+
+        let match;
+        while ((match = exactVariantRegex.exec(filteredText)) !== null) {
+          const originalWord = match[0];
+
+          if (whitelist.includes(originalWord.toLowerCase())) continue;
+
+          let censoredWord;
+          if (useRandomGrawlix) {
+            censoredWord = makeRandomGrawlixString(originalWord.length);
+          } else {
+            censoredWord = censorWord(
+              originalWord,
+              replaceWith,
+              !fullWordCensor && keepFirstAndLast,
+            );
+          }
+
+          replacements.push({
+            original: originalWord,
+            censored: censoredWord,
+            metadata,
+          });
+
+          filteredText = filteredText.replace(
+            new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"),
+            censoredWord,
+          );
+        }
+      });
+    });
+  }
+
   return {
     filtered: filteredText,
     censored: replacements.length,