@sideid/id-profanity-filter 1.11.7 → 1.11.8

package/src/constants/wordList.ts

@@ -1,22 +1,22 @@
-import { ProfanityWord } from "../types";
-import { sexualWords } from "./categories/sexual";
-import { insultWords } from "./categories/insult";
-// import { profanityWords } from './categories/profanity';
-// import { slurWords } from './categories/slur';
-// import { drugsWords } from './categories/drugs';
-// import { disgustingWords } from './categories/disgusting';
-// import { blasphemyWords } from './categories/blasphemy';
+import { ProfanityWord } from '../types';
+import { sexualWords } from './categories/sexual';
+import { insultWords } from './categories/insult';
+import { profanityWords } from './categories/profanity';
+import { slurWords } from './categories/slur';
+import { drugsWords } from './categories/drugs';
+import { disgustingWords } from './categories/disgusting';
+import { blasphemyWords } from './categories/blasphemy';
 
-import { general, generalWords } from "./regions/general";
-import { jawa, jawaWords } from "./regions/jawa";
-import { sunda, sundaWords } from "./regions/sunda";
-import { betawi, betawiWords } from "./regions/betawi";
-import { batak, batakWords } from "./regions/batak";
-// import { minang, minangWords } from './regions/minang';
-// import { bali, baliWords } from './regions/bali';
-// import { madura, maduraWords } from './regions/madura';
+import { general, generalWords } from './regions/general';
+import { jawa, jawaWords } from './regions/jawa';
+import { sunda, sundaWords } from './regions/sunda';
+import { betawi, betawiWords } from './regions/betawi';
+import { batak, batakWords } from './regions/batak';
+import { minang, minangWords } from './regions/minang';
+import { bali, baliWords } from './regions/bali';
+import { madura, maduraWords } from './regions/madura';
 // import { bugis, bugisWords } from './regions/bugis';
-// import { aceh, acehWords } from './regions/aceh';
+import { aceh, acehWords } from './regions/aceh';
 // import { ambon, ambonWords } from './regions/ambon';
 // import { papua, papuaWords } from './regions/papua';
 // import { manado, manadoWords } from './regions/manado';
@@ -29,11 +29,11 @@ import { batak, batakWords } from "./regions/batak";
 export const wordCategories = {
   sexual: sexualWords,
   insult: insultWords,
-  // profanity: profanityWords,
-  // slur: slurWords,
-  // drugs: drugsWords,
-  // disgusting: disgustingWords,
-  // blasphemy: blasphemyWords,
+  profanity: profanityWords,
+  slur: slurWords,
+  drugs: drugsWords,
+  disgusting: disgustingWords,
+  blasphemy: blasphemyWords,
 };
 
 export const wordRegions = {
@@ -42,11 +42,11 @@ export const wordRegions = {
   sunda: sundaWords,
   betawi: betawiWords,
   batak: batakWords,
-  // minang: minangWords,
-  // bali: baliWords,
-  // madura: maduraWords,
+  minang: minangWords,
+  bali: baliWords,
+  madura: maduraWords,
   // bugis: bugisWords,
-  // aceh: acehWords,
+  aceh: acehWords,
   // ambon: ambonWords,
   // papua: papuaWords,
   // manado: manadoWords,
@@ -63,11 +63,11 @@ export const wordObjects: ProfanityWord[] = [
   ...sunda,
   ...betawi,
   ...batak,
-  // ...minang,
-  // ...bali,
-  // ...madura,
+  ...minang,
+  ...bali,
+  ...madura,
   // ...bugis,
-  // ...aceh,
+  ...aceh,
   // ...ambon,
   // ...papua,
   // ...manado,
@@ -122,10 +122,7 @@ export function getWordMetadata(word: string): ProfanityWord | undefined {
   return wordObjects.find(
     (item) =>
       item.word.toLowerCase() === word.toLowerCase() ||
-      (item.aliases &&
-        item.aliases.some(
-          (alias) => alias.toLowerCase() === word.toLowerCase(),
-        )),
+      (item.aliases && item.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase()))
   );
 }
 

package/src/core/analyzer.ts

@@ -1,23 +1,18 @@
-import {
-  FilterOptions,
-  AnalysisResult,
-  ProfanityCategory,
-  Region,
-} from "../types";
+import { FilterOptions, AnalysisResult, ProfanityCategory, Region } from '../types';
 import {
   findProfanity,
   findProfanityWithMetadata,
   findCategories,
   findRegions,
   calculateSeverity,
-} from "./matcher";
-import { splitIntoSentences } from "../utils/stringUtils";
+} from './matcher';
+import { splitIntoSentences } from '../utils/stringUtils';
 import {
   findPossibleProfanityBySimiliarity,
   findProfanityByLevenshteinDistance,
-} from "../utils/similarityUtils";
-import { createContextRegex } from "../utils/regexUtils";
-import { DEFAULT_OPTIONS } from "../config/options";
+} from '../utils/similarityUtils';
+import { createContextRegex } from '../utils/regexUtils';
+import { DEFAULT_OPTIONS } from '../config/options';
 
 /**
  * Menganalisis teks untuk kata kotor
@@ -26,10 +21,7 @@ import { DEFAULT_OPTIONS } from "../config/options";
  * @param options Opsi untuk analisis
  * @return AnalysisResult dengan hasil analisis
  */
-export function analyze(
-  text: string,
-  options: FilterOptions = {},
-): AnalysisResult {
+export function analyze(text: string, options: FilterOptions = {}): AnalysisResult {
   const mergedOptions = { ...DEFAULT_OPTIONS, ...options };
 
   const matches = findProfanity(text, mergedOptions);
@@ -65,7 +57,7 @@ export function analyze(
           text,
           wordList,
           mergedOptions.similarityThreshold || 0.8,
-          mergedOptions.maxLevenshteinDistance || 2,
+          mergedOptions.maxLevenshteinDistance || 2
         );
 
         similarWords = levenshteinResults.map((item) => ({
@@ -77,7 +69,7 @@ export function analyze(
         similarWords = findPossibleProfanityBySimiliarity(
           text,
           wordList,
-          mergedOptions.similarityThreshold || 0.8,
+          mergedOptions.similarityThreshold || 0.8
         );
       }
     }
@@ -103,7 +95,7 @@ export function analyze(
  */
 export function batchAnalyze(
   texts: string[],
-  options: FilterOptions = {},
+  options: FilterOptions = {}
 ): {
   totalTexts: number;
   profaneTexts: number;
@@ -116,10 +108,7 @@ export function batchAnalyze(
   const mergedOptions = { ...DEFAULT_OPTIONS, ...options };
   const results = texts.map((text) => analyze(text, mergedOptions));
   const profaneTexts = results.filter((result) => result.hasProfanity).length;
-  const totalSeverity = results.reduce(
-    (sum, result) => sum + result.severityScore,
-    0,
-  );
+  const totalSeverity = results.reduce((sum, result) => sum + result.severityScore, 0);
   const averageSeverity = profaneTexts > 0 ? totalSeverity / profaneTexts : 0;
   const allCategories = results.flatMap((result) => result.categories);
   const categoryCount: Record<string, number> = {};
@@ -175,7 +164,7 @@ export function batchAnalyze(
  */
 export function analyzeBySentence(
   text: string,
-  options: FilterOptions = {},
+  options: FilterOptions = {}
 ): Array<AnalysisResult & { sentence: string }> {
   const sentences = splitIntoSentences(text);
   const mergedOptions = { ...DEFAULT_OPTIONS, ...options };
@@ -201,7 +190,7 @@ export function analyzeBySentence(
 export function analyzeWithContext(
   text: string,
   contextWindowSize: number = 5,
-  options: FilterOptions = {},
+  options: FilterOptions = {}
 ): Array<{
   word: string;
   context: string;
@@ -221,9 +210,9 @@ export function analyzeWithContext(
 
     let match;
     while ((match = regex.exec(text)) !== null) {
-      const beforeContext = match[1] || "";
+      const beforeContext = match[1] || '';
       const wordMatch = match[2];
-      const afterContext = match[3] || "";
+      const afterContext = match[3] || '';
 
       result.push({
         word: wordMatch,

package/src/core/filter.ts

@@ -1,8 +1,8 @@
-import { FilterOptions, FilterResult, ProfanityWord } from "../types";
-import { findProfanity, findProfanityWithMetadata } from "./matcher";
-import { censorWord, escapeRegExp } from "../utils/stringUtils";
-import { createWordRegex } from "../utils/regexUtils";
-import { DEFAULT_OPTIONS, makeRandomGrawlixString } from "../config/options";
+import { FilterOptions, FilterResult, ProfanityWord } from '../types';
+import { findProfanity, findProfanityWithMetadata } from './matcher';
+import { censorWord, escapeRegExp } from '../utils/stringUtils';
+import { createWordRegex } from '../utils/regexUtils';
+import { DEFAULT_OPTIONS, makeRandomGrawlixString } from '../config/options';
 
 interface FindProfanityFunction {
   (text: string, options?: FilterOptions): string[];
@@ -16,12 +16,9 @@ interface FindProfanityFunction {
  * @param options Opsi untuk filter
  * @returns FilterResult dengan hasil filter
  */
-export function filter(
-  text: string,
-  options: FilterOptions = {},
-): FilterResult {
+export function filter(text: string, options: FilterOptions = {}): FilterResult {
   const {
-    replaceWith = "*",
+    replaceWith = '*',
     fullWordCensor = true,
     detectLeetSpeak = true,
     whitelist = [],
@@ -74,10 +71,7 @@ export function filter(
     const metadata = matchDetails.find(
       (m) =>
         m.word.toLowerCase() === word.toLowerCase() ||
-        (m.aliases &&
-          m.aliases.some(
-            (alias) => alias.toLowerCase() === word.toLowerCase(),
-          )),
+        (m.aliases && m.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase()))
     );
 
     const variants = actualMatches.get(word.toLowerCase()) || [];
@@ -86,7 +80,7 @@ export function filter(
     const uniqueVariants = [...new Set(variants)];
 
     uniqueVariants.forEach((variant) => {
-      const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, "gi");
+      const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, 'gi');
 
       let match;
       while ((match = regex.exec(filteredText)) !== null) {
@@ -98,11 +92,7 @@ export function filter(
         if (useRandomGrawlix) {
           censoredWord = makeRandomGrawlixString(originalWord.length);
         } else {
-          censoredWord = censorWord(
-            originalWord,
-            replaceWith,
-            !fullWordCensor && keepFirstAndLast,
-          );
+          censoredWord = censorWord(originalWord, replaceWith, !fullWordCensor && keepFirstAndLast);
         }
 
         replacements.push({
@@ -112,8 +102,8 @@ export function filter(
         });
 
         filteredText = filteredText.replace(
-          new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"),
-          censoredWord,
+          new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, 'g'),
+          censoredWord
         );
       }
     });
@@ -141,7 +131,7 @@ export function filter(
             censoredWord = censorWord(
               originalWord,
               replaceWith,
-              !fullWordCensor && keepFirstAndLast,
+              !fullWordCensor && keepFirstAndLast
             );
           }
 
@@ -152,8 +142,8 @@ export function filter(
           });
 
           filteredText = filteredText.replace(
-            new RegExp(escapeRegExp(originalWord), "g"),
-            censoredWord,
+            new RegExp(escapeRegExp(originalWord), 'g'),
+            censoredWord
           );
         }
       }
@@ -180,7 +170,7 @@ export function filter(
             censoredWord = censorWord(
               originalWord,
               replaceWith,
-              !fullWordCensor && keepFirstAndLast,
+              !fullWordCensor && keepFirstAndLast
             );
           }
 
@@ -191,8 +181,8 @@ export function filter(
           });
 
           filteredText = filteredText.replace(
-            new RegExp(escapeRegExp(originalWord), "g"),
-            censoredWord,
+            new RegExp(escapeRegExp(originalWord), 'g'),
+            censoredWord
           );
         }
       }
@@ -204,19 +194,13 @@ export function filter(
       const metadata = matchDetails.find(
         (m) =>
           m.word.toLowerCase() === word.toLowerCase() ||
-          (m.aliases &&
-            m.aliases.some(
-              (alias) => alias.toLowerCase() === word.toLowerCase(),
-            )),
+          (m.aliases && m.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase()))
       );
 
       const variants = actualMatches.get(word.toLowerCase()) || [];
 
       variants.forEach((variant) => {
-        const exactVariantRegex = new RegExp(
-          `\\b${escapeRegExp(variant)}\\b`,
-          "gi",
-        );
+        const exactVariantRegex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, 'gi');
 
         let match;
         while ((match = exactVariantRegex.exec(filteredText)) !== null) {
@@ -231,7 +215,7 @@ export function filter(
             censoredWord = censorWord(
               originalWord,
               replaceWith,
-              !fullWordCensor && keepFirstAndLast,
+              !fullWordCensor && keepFirstAndLast
             );
           }
 
@@ -242,8 +226,8 @@ export function filter(
           });
 
           filteredText = filteredText.replace(
-            new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, "g"),
-            censoredWord,
+            new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, 'g'),
+            censoredWord
           );
         }
       });

package/src/core/matcher.ts

@@ -1,19 +1,14 @@
-import {
-  ProfanityWord,
-  ProfanityCategory,
-  Region,
-  FilterOptions,
-} from "../types";
-
-import { wordObjects } from "../constants/wordList";
-import { normalizeText } from "../utils/stringUtils";
-import { createWordRegex } from "../utils/regexUtils";
+import { ProfanityWord, ProfanityCategory, Region, FilterOptions } from '../types';
+
+import { wordObjects } from '../constants/wordList';
+import { normalizeText } from '../utils/stringUtils';
+import { createWordRegex } from '../utils/regexUtils';
 import {
   findPossibleProfanityBySimiliarity,
   findProfanityByLevenshteinDistance,
-} from "../utils/similarityUtils";
-import { DEFAULT_OPTIONS } from "../config/options";
-import { AhoCorasick } from "../utils/ahoCorasick";
+} from '../utils/similarityUtils';
+import { DEFAULT_OPTIONS } from '../config/options';
+import { AhoCorasick } from '../utils/ahoCorasick';
 
 const globalAhoCorasick = new AhoCorasick();
 let ahoCorasickInitialized = false;
@@ -34,10 +29,7 @@ interface FindProfanityFunction {
   lastActualMatches?: Map<string, string[]>;
 }
 
-export function findProfanity(
-  text: string,
-  options: FilterOptions = {},
-): string[] {
+export function findProfanity(text: string, options: FilterOptions = {}): string[] {
   const {
     wordList = [],
     detectLeetSpeak = true,
@@ -60,9 +52,7 @@ export function findProfanity(
 
   if (baseWordsToCheck.length === 0) {
     const filteredWords = wordObjects.filter((word) => {
-      const matchCategory = categories
-        ? categories.includes(word.category)
-        : true;
+      const matchCategory = categories ? categories.includes(word.category) : true;
       const matchRegion = regions ? regions.includes(word.region) : true;
       const matchSeverity = word.severity >= severityThreshold;
       return matchCategory && matchRegion && matchSeverity;
@@ -74,9 +64,7 @@ export function findProfanity(
   const aliasMap = new Map<string, string>();
   wordObjects.forEach((wordObj) => {
     if (wordObj.aliases && wordObj.aliases.length > 0) {
-      const matchCategory = categories
-        ? categories.includes(wordObj.category)
-        : true;
+      const matchCategory = categories ? categories.includes(wordObj.category) : true;
       const matchRegion = regions ? regions.includes(wordObj.region) : true;
       const matchSeverity = wordObj.severity >= severityThreshold;
 
@@ -88,10 +76,9 @@ export function findProfanity(
     }
   });
 
-  const wordsToCheck = [
-    ...baseWordsToCheck,
-    ...Array.from(aliasMap.keys()),
-  ].filter((word) => !whitelist.includes(word.toLowerCase()));
+  const wordsToCheck = [...baseWordsToCheck, ...Array.from(aliasMap.keys())].filter(
+    (word) => !whitelist.includes(word.toLowerCase())
+  );
 
   if (wordsToCheck.length === 0) {
     return [];
@@ -104,8 +91,7 @@ export function findProfanity(
 
   const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
   for (const match of basicMatches) {
-    const originalWord =
-      aliasMap.get(match.toLowerCase()) || match.toLowerCase();
+    const originalWord = aliasMap.get(match.toLowerCase()) || match.toLowerCase();
     matches.add(originalWord);
 
     if (!actualMatches.has(originalWord)) {
@@ -126,8 +112,7 @@ export function findProfanity(
 
       let match;
       while ((match = leetRegex.exec(text)) !== null) {
-        const originalWord =
-          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -150,8 +135,7 @@ export function findProfanity(
 
       let match;
       while ((match = variantRegex.exec(text)) !== null) {
-        const originalWord =
-          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -174,8 +158,7 @@ export function findProfanity(
 
       let match;
       while ((match = splitRegex.exec(text)) !== null) {
-        const originalWord =
-          aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+        const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -192,13 +175,12 @@ export function findProfanity(
         text,
         wordsToCheck,
         similarityThreshold,
-        maxLevenshteinDistance,
+        maxLevenshteinDistance
       );
 
       possibleProfanity.forEach((item) => {
         const originalWord =
-          aliasMap.get(item.original.toLowerCase()) ||
-          item.original.toLowerCase();
+          aliasMap.get(item.original.toLowerCase()) || item.original.toLowerCase();
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -210,15 +192,14 @@ export function findProfanity(
       const possibleProfanity = findPossibleProfanityBySimiliarity(
         text,
         wordsToCheck,
-        similarityThreshold,
+        similarityThreshold
       );
 
       possibleProfanity.forEach((item) => {
         matches.add(item.original.toLowerCase());
 
         const originalWord =
-          aliasMap.get(item.original.toLowerCase()) ||
-          item.original.toLowerCase();
+          aliasMap.get(item.original.toLowerCase()) || item.original.toLowerCase();
         if (!actualMatches.has(originalWord)) {
           actualMatches.set(originalWord, []);
         }
@@ -241,7 +222,7 @@ export function findProfanity(
  */
 export function findProfanityWithMetadata(
   text: string,
-  options: FilterOptions = {},
+  options: FilterOptions = {}
 ): ProfanityWord[] {
   const matches = findProfanity(text, options);
   if (matches.length === 0) {
@@ -253,10 +234,7 @@ export function findProfanityWithMetadata(
       const wordObject = wordObjects.find(
         (obj) =>
           obj.word.toLowerCase() === word.toLowerCase() ||
-          (obj.aliases &&
-            obj.aliases.some(
-              (alias) => alias.toLowerCase() === word.toLowerCase(),
-            )),
+          (obj.aliases && obj.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase()))
       );
 
       return wordObject;
@@ -270,9 +248,7 @@ export function findProfanityWithMetadata(
  * @param matchDetails Hasil pencarian dari fingProfanityWithMetadata()
  * @return Array kategori unik
  */
-export function findCategories(
-  matchDetails: ProfanityWord[],
-): ProfanityCategory[] {
+export function findCategories(matchDetails: ProfanityWord[]): ProfanityCategory[] {
   const categories = new Set<ProfanityCategory>();
 
   matchDetails.forEach((word) => {

package/src/index.ts

@@ -8,12 +8,7 @@ export * from './utils/similarityUtils';
 export * from './config/options';
 
 import { filter, isProfane } from './core/filter';
-import {
-  analyze,
-  batchAnalyze,
-  analyzeBySentence,
-  analyzeWithContext,
-} from './core/analyzer';
+import { analyze, batchAnalyze, analyzeBySentence, analyzeWithContext } from './core/analyzer';
 import { FilterOptions, FilterResult, AnalysisResult } from './types';
 import {
   DEFAULT_OPTIONS,
@@ -105,10 +100,7 @@ export class IDProfanityFilter {
    * @param presetName Nama preset yang akan digunakan
    * @param additionalOptions Opsi tambahan untuk override
    */
-  usePreset(
-    presetName: string,
-    additionalOptions: Partial<FilterOptions> = {},
-  ) {
+  usePreset(presetName: string, additionalOptions: Partial<FilterOptions> = {}) {
     this.options = getPresetOptions(presetName, additionalOptions);
   }
 
@@ -125,10 +117,7 @@ export class IDProfanityFilter {
    * @param word Kata yang akan diabaikan
    */
   addToWhitelist(word: string) {
-    this.options.whitelist = [
-      ...(this.options.whitelist || []),
-      word.toLowerCase(),
-    ];
+    this.options.whitelist = [...(this.options.whitelist || []), word.toLowerCase()];
   }
 
   /**
@@ -139,7 +128,7 @@ export class IDProfanityFilter {
     if (!this.options.whitelist) return;
 
     this.options.whitelist = this.options.whitelist.filter(
-      (w) => w.toLowerCase() !== word.toLowerCase(),
+      (w) => w.toLowerCase() !== word.toLowerCase()
     );
   }
 
@@ -166,7 +155,7 @@ export class IDProfanityFilter {
   enableSimilarityDetection(
     threshold: number = 0.8,
     useLevenshtein: boolean = false,
-    maxLevenshteinDistance: number = 2,
+    maxLevenshteinDistance: number = 2
   ) {
     this.options.detectSimilarity = true;
     this.options.similarityThreshold = threshold;

package/src/types/index.ts

@@ -1,31 +1,32 @@
 export type ProfanityCategory =
-  | "sexual"
-  | "insult"
-  | "profanity"
-  | "slur"
-  | "drugs"
-  | "disgusting"
-  | "blasphemy";
+  | 'sexual'
+  | 'insult'
+  | 'profanity'
+  | 'slur'
+  | 'drugs'
+  | 'disgusting'
+  | 'blasphemy';
 
 export type Region =
-  | "general"
-  | "jawa"
-  | "sunda"
-  | "betawi"
-  | "batak"
-  | "minang"
-  | "bali"
-  | "madura"
-  | "bugis"
-  | "aceh"
-  | "ambon"
-  | "papua"
-  | "manado"
-  | "banjar"
-  | "palembang"
-  | "lampung"
-  | "ntt"
-  | "ntb";
+  | 'general'
+  | 'jawa'
+  | 'sunda'
+  | 'betawi'
+  | 'batak'
+  | 'minang'
+  | 'bali'
+  | 'madura'
+  | 'bugis'
+  | 'aceh'
+  | 'ambon'
+  | 'papua'
+  | 'manado'
+  | 'banjar'
+  | 'palembang'
+  | 'lampung'
+  | 'ntt'
+  | 'mandailing'
+  | 'ntb';
 
 export interface ProfanityWord {
   word: string;

package/src/utils/ahoCorasick.ts

@@ -30,7 +30,7 @@ export class AhoCorasick {
    */
   addPattern(pattern: string): void {
     if (this.built) {
-      throw new Error("Cannot add patterns after the automaton is built");
+      throw new Error('Cannot add patterns after the automaton is built');
     }
 
     let node = this.root;