@sideid/id-profanity-filter 1.11.11 → 1.11.13

package/src/core/matcher.ts

@@ -29,6 +29,14 @@ interface FindProfanityFunction {
   lastActualMatches?: Map<string, string[]>;
 }
 
+function getWordMetadata(word: string): ProfanityWord | undefined {
+  return wordObjects.find(
+    (obj) =>
+      obj.word.toLowerCase() === word.toLowerCase() ||
+      (obj.aliases && obj.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase()))
+  );
+}
+
 export function findProfanity(text: string, options: FilterOptions = {}): string[] {
   const {
     wordList = [],
@@ -46,6 +54,8 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
     maxLevenshteinDistance = 2,
   } = { ...DEFAULT_OPTIONS, ...options };
 
+  const normalizedWhitelist = whitelist.map((w) => w.toLowerCase());
+
   const normalizedText = normalizeText(text);
 
   let baseWordsToCheck: string[] = wordList.length > 0 ? wordList : [];
@@ -63,21 +73,25 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
 
   const aliasMap = new Map<string, string>();
   wordObjects.forEach((wordObj) => {
-    if (wordObj.aliases && wordObj.aliases.length > 0) {
-      const matchCategory = categories ? categories.includes(wordObj.category) : true;
-      const matchRegion = regions ? regions.includes(wordObj.region) : true;
-      const matchSeverity = wordObj.severity >= severityThreshold;
-
-      if (matchCategory && matchRegion && matchSeverity) {
-        wordObj.aliases.forEach((alias) => {
-          aliasMap.set(alias.toLowerCase(), wordObj.word.toLowerCase());
-        });
-      }
+    const matchCategory = categories ? categories.includes(wordObj.category) : true;
+    const matchRegion = regions ? regions.includes(wordObj.region) : true;
+    const matchSeverity = wordObj.severity >= severityThreshold;
+
+    if (
+      matchCategory &&
+      matchRegion &&
+      matchSeverity &&
+      wordObj.aliases &&
+      wordObj.aliases.length > 0
+    ) {
+      wordObj.aliases.forEach((alias) => {
+        aliasMap.set(alias.toLowerCase(), wordObj.word.toLowerCase());
+      });
     }
   });
 
   const wordsToCheck = [...baseWordsToCheck, ...Array.from(aliasMap.keys())].filter(
-    (word) => !whitelist.includes(word.toLowerCase())
+    (word) => !normalizedWhitelist.includes(word.toLowerCase())
   );
 
   if (wordsToCheck.length === 0) {
@@ -87,11 +101,29 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
   const matches = new Set<string>();
   const actualMatches = new Map<string, string[]>();
 
+  const passesFilters = (word: string): boolean => {
+    if (normalizedWhitelist.includes(word.toLowerCase())) return false;
+
+    const metadata = getWordMetadata(word);
+    if (!metadata) return false;
+
+    const matchCategory = categories ? categories.includes(metadata.category) : true;
+    const matchRegion = regions ? regions.includes(metadata.region) : true;
+    const matchSeverity = metadata.severity >= severityThreshold;
+
+    return matchCategory && matchRegion && matchSeverity;
+  };
+
   initializeAhoCorasick(wordsToCheck);
 
   const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
   for (const match of basicMatches) {
+    if (normalizedWhitelist.includes(match.toLowerCase())) continue;
+
     const originalWord = aliasMap.get(match.toLowerCase()) || match.toLowerCase();
+
+    if (!passesFilters(originalWord)) continue;
+
     matches.add(originalWord);
 
     if (!actualMatches.has(originalWord)) {
@@ -112,7 +144,13 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
 
       let match;
       while ((match = leetRegex.exec(text)) !== null) {
+        const matchedText = match[0];
+        if (normalizedWhitelist.includes(matchedText.toLowerCase())) continue;
+
         const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+
+        if (!passesFilters(originalWord)) continue;
+
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -135,7 +173,13 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
 
       let match;
       while ((match = variantRegex.exec(text)) !== null) {
+        const matchedText = match[0];
+        if (normalizedWhitelist.includes(matchedText.toLowerCase())) continue;
+
         const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+
+        if (!passesFilters(originalWord)) continue;
+
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -158,7 +202,13 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
 
       let match;
       while ((match = splitRegex.exec(text)) !== null) {
+        const matchedText = match[0];
+        if (normalizedWhitelist.includes(matchedText.toLowerCase())) continue;
+
         const originalWord = aliasMap.get(word.toLowerCase()) || word.toLowerCase();
+
+        if (!passesFilters(originalWord)) continue;
+
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -179,8 +229,12 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
       );
 
       possibleProfanity.forEach((item) => {
+        if (normalizedWhitelist.includes(item.word.toLowerCase())) return;
+
         const originalWord =
           aliasMap.get(item.original.toLowerCase()) || item.original.toLowerCase();
+        if (!passesFilters(originalWord)) return;
+
         matches.add(originalWord);
 
         if (!actualMatches.has(originalWord)) {
@@ -196,10 +250,14 @@ export function findProfanity(text: string, options: FilterOptions = {}): string
       );
 
       possibleProfanity.forEach((item) => {
-        matches.add(item.original.toLowerCase());
+        if (normalizedWhitelist.includes(item.word.toLowerCase())) return;
 
         const originalWord =
           aliasMap.get(item.original.toLowerCase()) || item.original.toLowerCase();
+        if (!passesFilters(originalWord)) return;
+
+        matches.add(originalWord);
+
         if (!actualMatches.has(originalWord)) {
           actualMatches.set(originalWord, []);
         }
@@ -306,7 +364,5 @@ export function calculateSeverity(matchDetails: ProfanityWord[]): number {
 
   const severityAvg = severitySum / matchDetails.length;
 
-  // Gabungkan jumlah kata dan keparahan rata-rata
-  // 70% keparahan kata + 30% faktor jumlah
   return 0.7 * severityAvg + 0.3 * countFactor;
 }

package/src/utils/regexUtils.ts

@@ -58,14 +58,14 @@ export function addLeetSpeakVariations(pattern: string): string {
   const leetMap: Record<string, string[]> = {
     a: ['a', '4', '@'],
     b: ['b', '8', '6'],
-    c: ['c', '(', '{', '<'],
+    c: ['c', '\\(', '\\{', '<'],
     e: ['e', '3'],
     g: ['g', '6', '9'],
-    i: ['i', '1', '!', '|'],
-    l: ['l', '1', '|'],
+    i: ['i', '1', '!', '\\|'],
+    l: ['l', '1', '\\|'],
     o: ['o', '0'],
-    s: ['s', '5', '$'],
-    t: ['t', '7', '+'],
+    s: ['s', '5', '\\$'],
+    t: ['t', '7', '\\+'],
     z: ['z', '2'],
   };
 
@@ -91,8 +91,23 @@ export function addLeetSpeakVariations(pattern: string): string {
  * @returns Pola regex dengan kemungkinan split
  */
 export function addSplitVariations(pattern: string): string {
-  // Tambahkan kemungkinan spasi atau karakter penghubung di antara setiap huruf
-  return pattern.split('').join('[\\s\\-._*+]?');
+  // Instead of joining character by character with a separator pattern,
+  // we'll create a simpler version that matches the pattern with optional separators
+
+  // Convert each character to a pattern that allows optional separators before it
+  // except for the first character
+  let result = '';
+  const chars = pattern.split('');
+
+  for (let i = 0; i < chars.length; i++) {
+    if (i > 0) {
+      // Add optional separator before each character except the first
+      result += '[\\s\\-._*+]?';
+    }
+    result += chars[i];
+  }
+
+  return result;
 }
 
 /**
@@ -103,10 +118,24 @@ export function addSplitVariations(pattern: string): string {
  * @returns Objek RegExp
  */
 export function createEvasionRegex(word: string): RegExp {
-  // Tambahkan kemungkinan spasi atau karakter penghubung di antara setiap huruf
-  const pattern = addSplitVariations(escapeRegExp(word));
+  // Escape karakter khusus regex
+  const escaped = escapeRegExp(word);
 
-  return new RegExp(pattern, 'gi');
+  // Create a regex pattern that allows any separator between characters
+  let result = '';
+  const chars = escaped.split('');
+
+  for (let i = 0; i < chars.length; i++) {
+    // Add the character
+    result += chars[i];
+
+    // Add optional separator after each character except the last
+    if (i < chars.length - 1) {
+      result += '[\\s\\-._*+]?';
+    }
+  }
+
+  return new RegExp(result, 'gi');
 }
 
 /**
@@ -120,7 +149,7 @@ export function addIndonesianVariations(pattern: string): string {
   const variationMap: Record<string, string[]> = {
     c: ['c', 'k'], // contoh: becok/bekok
     k: ['k', 'c', 'q'], // contoh: kacau/qacau
-    j: ['j', 'dj'], // contoh: jualan/djualan (ejaan lama)
+    j: ['j', 'd'], // contoh: jualan/dualan (ejaan lama)
     y: ['y', 'j'], // contoh: ya/ja
     u: ['u', 'oe'], // contoh: untuk/oentoek (ejaan lama)
     f: ['f', 'p', 'v'], // contoh: kafir/kapir
@@ -128,20 +157,21 @@ export function addIndonesianVariations(pattern: string): string {
     x: ['x', 'ks'], // contoh: taxi/taksi
   };
 
-  // Ganti tiap karakter dengan variasinya
-  return pattern
-    .split('')
-    .map((char) => {
-      const lowerChar = char.toLowerCase();
-      const variations = variationMap[lowerChar];
-
-      if (variations && variations.length > 1) {
-        return `[${variations.join('')}]`;
-      }
+  // Go through each character in the pattern and replace with variations
+  let result = '';
+  for (const char of pattern) {
+    const lowerChar = char.toLowerCase();
+    const variations = variationMap[lowerChar];
+
+    if (variations && variations.length > 1) {
+      // Create a character class with all variations
+      result += `[${variations.join('')}]`;
+    } else {
+      result += char;
+    }
+  }
 
-      return char;
-    })
-    .join('');
+  return result;
 }
 
 /**
@@ -151,7 +181,8 @@ export function addIndonesianVariations(pattern: string): string {
  * @returns Objek RegExp
  */
 export function createIndonesianVariationRegex(word: string): RegExp {
-  const pattern = addIndonesianVariations(escapeRegExp(word));
+  const escapedWord = escapeRegExp(word);
+  const pattern = addIndonesianVariations(escapedWord);
   return new RegExp(`\\b${pattern}\\b`, 'gi');
 }
 
@@ -179,18 +210,37 @@ export function createContextRegex(word: string, contextSize: number = 3): RegEx
  */
 export function createWordFormRegex(word: string): RegExp {
   // Implementasi sederhana untuk mencocokkan berbagai imbuhan
-  // Ini bisa dikembangkan lebih lanjut untuk mencocokkan bentukan kata yang lebih kompleks
-  const prefixes = ['', 'me', 'pe', 'ber', 'di', 'ter', 'se'];
-  const suffixes = ['', 'kan', 'an', 'i', 'nya'];
+  const escapedWord = escapeRegExp(word);
+
+  // Common Indonesian prefixes and suffixes
+  const prefixes = [
+    '',
+    'me',
+    'pe',
+    'ber',
+    'di',
+    'ter',
+    'se',
+    'ke',
+    'mem',
+    'pem',
+    'bel',
+    'peng',
+    'meng',
+  ];
+  const suffixes = ['', 'kan', 'an', 'i', 'nya', 'lah', 'kah'];
 
   const patterns = [];
 
-  // Kombinasikan prefix dan suffix
   for (const prefix of prefixes) {
     for (const suffix of suffixes) {
-      patterns.push(`\\b${prefix}${escapeRegExp(word)}${suffix}\\b`);
+      patterns.push(`\\b${prefix}${escapedWord}${suffix}\\b`);
     }
   }
 
+  if (/^[aiueo]/.test(word)) {
+    patterns.push(`\\bng${escapedWord}\\b`);
+  }
+
   return new RegExp(patterns.join('|'), 'gi');
 }

package/test/analyzer.test.ts

@@ -0,0 +1,89 @@
+import { analyze, batchAnalyze, analyzeBySentence, analyzeWithContext } from '../src/core/analyzer';
+
+describe('Analyzer Core', () => {
+  describe('analyze function', () => {
+    test('should correctly analyze clean text', () => {
+      const result = analyze('Ini adalah kalimat yang sopan');
+      expect(result.hasProfanity).toBe(false);
+      expect(result.matches).toEqual([]);
+      expect(result.severityScore).toBe(0);
+    });
+
+    test('should correctly analyze text with profanity', () => {
+      const result = analyze('Dasar anjing!');
+      expect(result.hasProfanity).toBe(true);
+      expect(result.matches).toContain('anjing');
+      expect(result.severityScore).toBeGreaterThan(0);
+    });
+
+    test('should correctly analyze text with multiple profanities', () => {
+      const result = analyze('Dasar anjing dan babi!');
+      expect(result.hasProfanity).toBe(true);
+      expect(result.matches).toContain('anjing');
+      expect(result.matches).toContain('babi');
+      expect(result.matches.length).toBe(2);
+    });
+
+    test('should provide region information', () => {
+      // Assuming "anjing" is categorized as 'general' region
+      const result = analyze('anjing');
+      expect(result.regions).toContain('general');
+    });
+
+    test('should provide category information', () => {
+      // Assuming "anjing" is categorized as 'profanity'
+      const result = analyze('anjing');
+      expect(result.categories).toContain('profanity');
+    });
+
+    test('should detect similar words when option enabled', () => {
+      const result = analyze('anjiing', { detectSimilarity: true });
+      expect(result.hasProfanity).toBe(true);
+      expect(result.similarWords).toBeDefined();
+      expect(result.similarWords!.length).toBeGreaterThan(0);
+    });
+
+    test('should respect whitelist', () => {
+      const result = analyze('anjing', { whitelist: ['anjing'] });
+      expect(result.hasProfanity).toBe(false);
+      expect(result.matches).toEqual([]);
+    });
+  });
+
+  describe('batchAnalyze function', () => {
+    test('should analyze multiple texts', () => {
+      const texts = ['Ini kalimat bersih', 'Dasar anjing kamu!', 'Babi itu jorok'];
+
+      const result = batchAnalyze(texts);
+      expect(result.totalTexts).toBe(3);
+      expect(result.profaneTexts).toBe(2);
+      expect(result.cleanTexts).toBe(1);
+      expect(result.mostFrequentWords.length).toBeGreaterThan(0);
+    });
+  });
+
+  describe('analyzeBySentence function', () => {
+    test('should analyze text by sentence', () => {
+      const text =
+        'Kalimat pertama bersih. Kalimat kedua anjing kotor. Kalimat ketiga juga bersih.';
+      const results = analyzeBySentence(text);
+
+      expect(results.length).toBe(3);
+      expect(results[0].hasProfanity).toBe(false);
+      expect(results[1].hasProfanity).toBe(true);
+      expect(results[1].matches).toContain('anjing');
+      expect(results[2].hasProfanity).toBe(false);
+    });
+  });
+
+  describe('analyzeWithContext function', () => {
+    test('should provide context around profanity', () => {
+      const text = 'Saya sangat kesal dengan perilaku anjing tersebut karena sangat mengganggu';
+      const results = analyzeWithContext(text, 3);
+
+      expect(results.length).toBe(1);
+      expect(results[0].word).toBe('anjing');
+      expect(results[0].context).toContain('perilaku anjing tersebut');
+    });
+  });
+});

package/test/filter.test.ts

@@ -0,0 +1,80 @@
+import { filter, isProfane } from '../src/core/filter';
+
+describe('Filter Core', () => {
+  describe('filter function', () => {
+    test('should filter profanity with default settings', () => {
+      const result = filter('Dasar anjing kamu!');
+      expect(result.filtered).toBe('Dasar ****** kamu!');
+      expect(result.censored).toBe(1);
+    });
+
+    test('should not filter clean text', () => {
+      const cleanText = 'Ini adalah kalimat yang sopan';
+      const result = filter(cleanText);
+      expect(result.filtered).toBe(cleanText);
+      expect(result.censored).toBe(0);
+      expect(result.replacements).toEqual([]);
+    });
+
+    test('should filter multiple profanities', () => {
+      const text = 'Dasar anjing dan babi!';
+      const result = filter(text);
+      expect(result.filtered).not.toBe(text);
+      expect(result.censored).toBe(2);
+      expect(result.replacements.length).toBe(2);
+    });
+
+    test('should respect custom replacement character', () => {
+      const result = filter('Dasar anjing!', { replaceWith: '#' });
+      expect(result.filtered).toBe('Dasar ######!');
+    });
+
+    test('should handle keepFirstAndLast option', () => {
+      const result = filter('anjing', {
+        keepFirstAndLast: true,
+        fullWordCensor: false,
+      });
+      expect(result.filtered).toBe('a****g');
+    });
+
+    // test('should handle random grawlix option', () => {
+    //   const result = filter('anjing', { useRandomGrawlix: true });
+    //   expect(result.filtered).not.toBe('anjing');
+    //   expect(result.filtered.length).toBe(6);
+    // });
+
+    test('should use whitelist', () => {
+      const text = 'Anjing adalah binatang peliharaan yang setia';
+
+      // Without whitelist
+      const withoutWhitelist = filter(text);
+      expect(withoutWhitelist.censored).toBe(1);
+
+      // With whitelist
+      const withWhitelist = filter(text, { whitelist: ['anjing'] });
+      expect(withWhitelist.censored).toBe(0);
+      expect(withWhitelist.filtered).toBe(text);
+    });
+  });
+
+  describe('isProfane function', () => {
+    test('should detect profanity', () => {
+      expect(isProfane('anjing')).toBe(true);
+      expect(isProfane('babi')).toBe(true);
+      expect(isProfane('kucing')).toBe(false);
+    });
+
+    test('should respect whitelist', () => {
+      expect(isProfane('anjing', { whitelist: ['anjing'] })).toBe(false);
+    });
+
+    test('should detect based on threshold', () => {
+      // Assuming "kontol" has a higher severity than "babi"
+      expect(isProfane('kontol', { severityThreshold: 0.8 })).toBe(true);
+
+      // This might fail if the severity isn't configured as expected
+      // Just an example of how the test would work
+      expect(isProfane('babi', { severityThreshold: 0.7 })).toBe(false);
+    });
+  });
+});

package/test/jest.setup.ts

@@ -0,0 +1,9 @@
+// Any global setup code can go here
+
+// For example, if you need to mock some global objects:
+// global.fetch = jest.fn();
+
+// Or silence console logs during tests
+// global.console.log = jest.fn();
+// global.console.warn = jest.fn();
+// global.console.error = jest.fn();

package/test/matcher.test.ts

@@ -0,0 +1,120 @@
+import {
+  findProfanity,
+  findProfanityWithMetadata,
+  findCategories,
+  findRegions,
+  calculateSeverity,
+} from '../src/core/matcher';
+
+describe('Matcher Core', () => {
+  describe('findProfanity function', () => {
+    test('should find profanity in text', () => {
+      const results = findProfanity('Dasar anjing!');
+      expect(results).toContain('anjing');
+      expect(results.length).toBe(1);
+    });
+
+    test('should find multiple profanities', () => {
+      const results = findProfanity('Dasar anjing dan babi!');
+      expect(results).toContain('anjing');
+      expect(results).toContain('babi');
+      expect(results.length).toBe(2);
+    });
+
+    test('should detect leetspeak variations', () => {
+      const results = findProfanity('Dasar 4nj1ng!', { detectLeetSpeak: true });
+      expect(results).toContain('anjing');
+    });
+
+    test('should detect Indonesian variations', () => {
+      const results = findProfanity('Kamu itu anjic!', { indonesianVariation: true });
+      expect(results).toContain('anjing');
+    });
+
+    test('should detect split words', () => {
+      const results = findProfanity('a-n-j-i-n-g', { detectSplit: true });
+      expect(results).toContain('anjing');
+    });
+
+    test('should respect whitelist', () => {
+      const results = findProfanity('anjing', { whitelist: ['anjing'] });
+      expect(results).toEqual([]);
+    });
+
+    test('should filter by category', () => {
+      // Assuming "kontol" is sexual and "anjing" is profanity
+      const resultsWithSexual = findProfanity('memek dan anjing', {
+        categories: ['sexual'],
+      });
+      expect(resultsWithSexual).toContain('memek');
+      expect(resultsWithSexual).not.toContain('anjing');
+
+      const resultsWithProfanity = findProfanity('kontol dan anjing', {
+        categories: ['profanity'],
+      });
+      expect(resultsWithProfanity).toContain('anjing');
+      expect(resultsWithProfanity).not.toContain('kontol');
+    });
+
+    test('should filter by region', () => {
+      // Assuming "jancok" is from jawa region and "anjing" is general
+      const resultsWithJawa = findProfanity('jancok dan anjing', { regions: ['jawa'] });
+      expect(resultsWithJawa).toContain('jancok');
+      expect(resultsWithJawa).not.toContain('anjing');
+
+      const resultsWithGeneral = findProfanity('jancok dan anjing', { regions: ['general'] });
+      expect(resultsWithGeneral).toContain('anjing');
+      expect(resultsWithGeneral).not.toContain('jancok');
+    });
+  });
+
+  describe('findProfanityWithMetadata function', () => {
+    test('should return metadata for profanity', () => {
+      const results = findProfanityWithMetadata('anjing');
+      expect(results.length).toBe(1);
+      expect(results[0].word).toBe('anjing');
+      expect(results[0].category).toBeDefined();
+      expect(results[0].region).toBeDefined();
+      expect(results[0].severity).toBeDefined();
+    });
+  });
+
+  describe('findCategories function', () => {
+    test('should extract unique categories', () => {
+      const metadata = findProfanityWithMetadata('anjing dan kontol');
+      const categories = findCategories(metadata);
+
+      // Assuming "anjing" is profanity and "kontol" is sexual
+      expect(categories.includes('profanity')).toBe(true);
+      expect(categories.includes('sexual')).toBe(true);
+      expect(categories.length).toBe(2); // Should be unique
+    });
+  });
+
+  describe('findRegions function', () => {
+    test('should extract unique regions', () => {
+      const metadata = findProfanityWithMetadata('anjing dan jancok');
+      const regions = findRegions(metadata);
+
+      // Assuming "anjing" is general and "jancok" is jawa
+      expect(regions.includes('general')).toBe(true);
+      expect(regions.includes('jawa')).toBe(true);
+      expect(regions.length).toBe(2); // Should be unique
+    });
+  });
+
+  describe('calculateSeverity function', () => {
+    test('should calculate severity score', () => {
+      const metadata = findProfanityWithMetadata('anjing');
+      const severity = calculateSeverity(metadata);
+
+      expect(severity).toBeGreaterThanOrEqual(0);
+      expect(severity).toBeLessThanOrEqual(1);
+    });
+
+    test('should return 0 for empty metadata', () => {
+      const severity = calculateSeverity([]);
+      expect(severity).toBe(0);
+    });
+  });
+});