@sideid/id-profanity-filter 1.11.12 → 1.11.13

package/test/profanity-filter.test.ts

@@ -0,0 +1,130 @@
+import { IDProfanityFilter } from '../src/index';
+
+describe('IDProfanityFilter', () => {
+  let filter: IDProfanityFilter;
+
+  beforeEach(() => {
+    filter = new IDProfanityFilter();
+  });
+
+  describe('basic functionality', () => {
+    test('should detect profanity', () => {
+      expect(filter.isProfane('Ini kalimat tanpa kata kotor')).toBe(false);
+      expect(filter.isProfane('Dasar anjing kamu!')).toBe(true);
+    });
+
+    test('should filter profanity', () => {
+      const result = filter.filter('Dasar anjing kamu!');
+      expect(result.filtered).not.toEqual('Dasar anjing kamu!');
+      expect(result.filtered).toContain('*****');
+      expect(result.censored).toBe(1);
+    });
+
+    test('should analyze profanity', () => {
+      const result = filter.analyze('Kamu ini babi dan anjing!');
+      expect(result.hasProfanity).toBe(true);
+      expect(result.matches.length).toBe(2);
+      expect(result.severityScore).toBeGreaterThan(0);
+    });
+  });
+
+  describe('options configuration', () => {
+    test('should apply options when filtering', () => {
+      filter.setOptions({ replaceWith: '#' });
+      const result = filter.filter('Dasar anjing kamu!');
+      expect(result.filtered).toContain('#####');
+      expect(result.filtered).not.toContain('*****');
+    });
+
+    test('should apply keep first and last option', () => {
+      filter.setOptions({ replaceWith: '*', fullWordCensor: false, keepFirstAndLast: true });
+      const result = filter.filter('Dasar anjing kamu!');
+      expect(result.filtered).toContain('Dasar a****g kamu!');
+    });
+
+    test('should apply random grawlix option', () => {
+      filter.setOptions({ useRandomGrawlix: true });
+      const result = filter.filter('Dasar anjing kamu!');
+      // Should replace with random characters from the grawlix set
+      expect(result.filtered).not.toContain('anjing');
+      expect(result.filtered).not.toEqual('Dasar anjing kamu!');
+    });
+  });
+
+  describe('preset functionality', () => {
+    test('should apply presets correctly', () => {
+      filter.usePreset('strict');
+      expect(filter.filter('Dasar anjing!').censored).toBe(1);
+
+      filter.usePreset('childSafe');
+      const result = filter.filter('Kamu ini babi dan anjing!');
+      expect(result.censored).toBe(2);
+    });
+  });
+
+  describe('whitelist functionality', () => {
+    test('should respect whitelist', () => {
+      filter.addToWhitelist('anjing');
+      expect(filter.isProfane('anjing')).toBe(false);
+
+      filter.removeFromWhitelist('anjing');
+      expect(filter.isProfane('anjing')).toBe(true);
+    });
+  });
+
+  describe('advanced detection', () => {
+    test('should detect leet speak variations', () => {
+      filter.setOptions({ detectLeetSpeak: true });
+      expect(filter.isProfane('4nj1ng')).toBe(true);
+    });
+
+    test('should detect split words', () => {
+      filter.enableSplitWordDetection();
+      expect(filter.isProfane('a-n-j-i-n-g')).toBe(true);
+    });
+
+    test('should detect similar words with levenshtein', () => {
+      filter.enableLevenshteinDetection(0.8, 2);
+      expect(filter.isProfane('anjiing')).toBe(true); // Extra 'i'
+    });
+  });
+
+  describe('batch analysis', () => {
+    test('should analyze multiple strings correctly', () => {
+      const texts = [
+        'Ini kalimat bersih tanpa kata kotor',
+        'Dasar anjing kamu!',
+        'Saya sangat suka kucing dan anjing peliharaan',
+      ];
+
+      const result = filter.batchAnalyze(texts);
+      expect(result.totalTexts).toBe(3);
+      expect(result.profaneTexts).toBe(2);
+      expect(result.cleanTexts).toBe(1);
+    });
+  });
+
+  describe('sentence analysis', () => {
+    test('should analyze by sentence correctly', () => {
+      const text =
+        'Kalimat pertama bersih. Kalimat kedua anjing kotor. Kalimat ketiga juga bersih.';
+      const results = filter.analyzeBySentence(text);
+
+      expect(results.length).toBe(3);
+      expect(results[0].hasProfanity).toBe(false);
+      expect(results[1].hasProfanity).toBe(true);
+      expect(results[2].hasProfanity).toBe(false);
+    });
+  });
+
+  describe('context analysis', () => {
+    test('should provide context around profanity', () => {
+      const text = 'Saya sangat kesal dengan perilaku anjing tersebut karena sangat mengganggu';
+      const results = filter.analyzeWithContext(text, 3);
+
+      expect(results.length).toBe(1);
+      expect(results[0].word).toBe('anjing');
+      expect(results[0].context).toContain('kesal dengan perilaku anjing tersebut karena');
+    });
+  });
+});

package/test/utils/ahoCorasick.test.ts

@@ -0,0 +1,106 @@
+import { AhoCorasick } from '../../src/utils/ahoCorasick';
+
+describe('AhoCorasick', () => {
+  let ac: AhoCorasick;
+
+  beforeEach(() => {
+    ac = new AhoCorasick();
+  });
+
+  describe('basic functionality', () => {
+    test('should build trie and search patterns', () => {
+      ac.addPattern('anjing');
+      ac.addPattern('babi');
+      ac.build();
+
+      const matches = ac.search('Dasar anjing dan babi!');
+
+      expect(matches.size).toBe(2);
+      expect(matches.get('anjing')).toBe(1);
+      expect(matches.get('babi')).toBe(1);
+    });
+
+    test('should return unique matches', () => {
+      ac.addPattern('anjing');
+      ac.addPattern('babi');
+      ac.build();
+
+      const matches = ac.searchUnique('Dasar anjing dan babi dan anjing!');
+
+      expect(matches.size).toBe(2);
+      expect(matches.has('anjing')).toBe(true);
+      expect(matches.has('babi')).toBe(true);
+    });
+
+    test('should detect presence of patterns', () => {
+      ac.addPattern('anjing');
+      ac.addPattern('babi');
+      ac.build();
+
+      expect(ac.containsAny('Dasar anjing!')).toBe(true);
+      expect(ac.containsAny('Dasar kucing!')).toBe(false);
+    });
+
+    test('should count multiple occurrences', () => {
+      ac.addPattern('anjing');
+      ac.build();
+
+      const matches = ac.search('anjing dan anjing lagi anjing');
+
+      expect(matches.get('anjing')).toBe(3);
+    });
+  });
+
+  describe('edge cases', () => {
+    test('should handle empty search text', () => {
+      ac.addPattern('anjing');
+      ac.build();
+
+      const matches = ac.search('');
+      expect(matches.size).toBe(0);
+    });
+
+    test('should handle overlapping patterns', () => {
+      ac.addPattern('an');
+      ac.addPattern('anjing');
+      ac.build();
+
+      const matches = ac.search('anjing');
+
+      expect(matches.size).toBe(2);
+      expect(matches.has('an')).toBe(true);
+      expect(matches.has('anjing')).toBe(true);
+    });
+
+    test('should throw error when trying to add pattern after build', () => {
+      ac.build();
+      expect(() => ac.addPattern('test')).toThrow();
+    });
+  });
+
+  describe('failure function', () => {
+    test('should handle prefix matches', () => {
+      ac.addPattern('anjing');
+      ac.addPattern('ing');
+      ac.build();
+
+      const matches = ac.search('anjing');
+
+      expect(matches.size).toBe(2);
+      expect(matches.has('anjing')).toBe(true);
+      expect(matches.has('ing')).toBe(true);
+    });
+
+    test('should handle suffix matches', () => {
+      ac.addPattern('an');
+      ac.addPattern('anjing');
+      ac.build();
+
+      const matches = ac.search('anjing');
+
+      expect(matches.size).toBe(2);
+      expect(matches.has('an')).toBe(true);
+      expect(matches.has('anjing')).toBe(true);
+    });
+  });
+});

package/test/utils/regexUtils.test.ts

@@ -0,0 +1,111 @@
+import {
+  createWordRegex,
+  addLeetSpeakVariations,
+  addSplitVariations,
+  createEvasionRegex,
+  addIndonesianVariations,
+  createIndonesianVariationRegex,
+  createContextRegex,
+  createWordFormRegex,
+} from '../../src/utils/regexUtils';
+
+describe('Regex Utils', () => {
+  describe('createWordRegex function', () => {
+    test('should create basic word regex', () => {
+      const regex = createWordRegex('test');
+      expect(regex.test('test')).toBe(true);
+      expect(regex.test('testing')).toBe(false);
+      expect(regex.test('a test')).toBe(true);
+    });
+
+    test('should handle case sensitivity', () => {
+      const regex = createWordRegex('test', { caseSensitive: true });
+      expect(regex.test('test')).toBe(true);
+      expect(regex.test('TEST')).toBe(false);
+    });
+
+    test('should handle leetspeak option', () => {
+      const regex = createWordRegex('test', { leetSpeak: true });
+      expect(regex.test('t3st')).toBe(true);
+    });
+
+    test('should handle Indonesian variation option', () => {
+      const regex = createWordRegex('kafir', { indonesianVariation: true });
+      expect(regex.test('kapir')).toBe(true); 
+    });
+
+    test('should not use word boundary when wholeWord is false', () => {
+      const regex = createWordRegex('test', { wholeWord: false });
+      expect(regex.test('testing')).toBe(true);
+    });
+  });
+
+  describe('addLeetSpeakVariations function', () => {
+    test('should add leetspeak variations', () => {
+      const pattern = addLeetSpeakVariations('test');
+      const regex = new RegExp(pattern);
+
+      expect(regex.test('t3st')).toBe(true);
+      expect(regex.test('te5t')).toBe(true);
+      expect(regex.test('t35t')).toBe(true);
+    });
+  });
+
+  describe('addSplitVariations function', () => {
+    test('should add split variations', () => {
+      const pattern = addSplitVariations('test');
+      const regex = new RegExp(pattern);
+
+      expect(regex.test('t-e-s-t')).toBe(true);
+      expect(regex.test('t e s t')).toBe(true);
+      expect(regex.test('t.e.s.t')).toBe(true);
+    });
+  });
+
+  // describe('createEvasionRegex function', () => {
+  //   test('should create regex to detect evasion attempts', () => {
+  //     const regex = createEvasionRegex('test');
+
+  //     expect(regex.test('t-e-s-t')).toBe(true);
+  //     expect(regex.test('t e s t')).toBe(true);
+  //     expect(regex.test('t.e.s.t')).toBe(true);
+  //     expect(regex.test('t*e*s*t')).toBe(true);
+  //   });
+  // });
+
+  describe('addIndonesianVariations function', () => {
+    test('should add Indonesian spelling variations', () => {
+      const pattern = addIndonesianVariations('cacing');
+      const regex = new RegExp(pattern);
+
+      expect(regex.test('kacing')).toBe(true); // c/k variation
+    });
+  });
+
+  describe('createContextRegex function', () => {
+    test('should create regex that captures context', () => {
+      const regex = createContextRegex('test', 2);
+      const text = 'This is a test of the context';
+      const match = regex.exec(text);
+
+      expect(match).not.toBeNull();
+      if (match) {
+        expect(match[1]).toBe('is a '); // Before context
+        expect(match[2]).toBe('test'); // The word
+        expect(match[3]).toBe(' of the'); // After context
+      }
+    });
+  });
+
+  // describe('createWordFormRegex function', () => {
+  //   test('should create regex for different word forms', () => {
+  //     const regex = createWordFormRegex('ajar');
+
+  //     expect(regex.test('ajar')).toBe(true);
+  //     expect(regex.test('belajar')).toBe(true);
+  //     expect(regex.test('mengajar')).toBe(true);
+  //     expect(regex.test('pengajaran')).toBe(true);
+  //     expect(regex.test('ajari')).toBe(true);
+  //   });
+  // });
+});

package/test/utils/similarityUtils.test.ts

@@ -0,0 +1,139 @@
+import {
+  levenshteinDistance,
+  stringSimilarity,
+  findMostSimilar,
+  findMostSimilarWithLevenshtein,
+  isPossibleProfanityVariation,
+  findPossibleProfanityBySimiliarity,
+  findProfanityByLevenshteinDistance,
+} from '../../src/utils/similarityUtils';
+
+describe('Similarity Utils', () => {
+  describe('levenshteinDistance function', () => {
+    test('should calculate correct distance', () => {
+      expect(levenshteinDistance('kitten', 'sitting')).toBe(3);
+      expect(levenshteinDistance('test', 'test')).toBe(0);
+      expect(levenshteinDistance('book', 'back')).toBe(2);
+    });
+
+    test('should handle empty strings', () => {
+      expect(levenshteinDistance('', '')).toBe(0);
+      expect(levenshteinDistance('test', '')).toBe(4);
+      expect(levenshteinDistance('', 'test')).toBe(4);
+    });
+  });
+
+  describe('stringSimilarity function', () => {
+    test('should calculate correct similarity', () => {
+      expect(stringSimilarity('test', 'test')).toBe(1); // identical
+      expect(stringSimilarity('test', 'tent')).toBe(0.75); // 1 change out of 4
+      expect(stringSimilarity('book', 'back')).toBe(0.5); // 2 changes out of 4
+    });
+
+    test('should handle empty strings', () => {
+      expect(stringSimilarity('', '')).toBe(1); // both empty = identical
+      expect(stringSimilarity('test', '')).toBe(0);
+      expect(stringSimilarity('', 'test')).toBe(0);
+    });
+  });
+
+  describe('findMostSimilar function', () => {
+    test('should find most similar string', () => {
+      const candidates = ['apple', 'banana', 'orange', 'appl'];
+      expect(findMostSimilar('aple', candidates, 0.7)).toBe('apple');
+    });
+
+    test('should return null when no match above threshold', () => {
+      const candidates = ['apple', 'banana', 'orange'];
+      expect(findMostSimilar('xyz', candidates, 0.7)).toBeNull();
+    });
+
+    test('should handle empty candidates', () => {
+      expect(findMostSimilar('test', [], 0.7)).toBeNull();
+    });
+  });
+
+  describe('findMostSimilarWithLevenshtein function', () => {
+    test('should find most similar string using Levenshtein', () => {
+      const candidates = ['anjing', 'babi', 'kontol'];
+      expect(findMostSimilarWithLevenshtein('anjiing', candidates, 0.7, 2)).toBe('anjing');
+    });
+
+    test('should respect max distance limit', () => {
+      const candidates = ['anjing', 'anjiang'];
+      // With default max distance (3), it should match
+      expect(findMostSimilarWithLevenshtein('anjiiing', candidates, 0.7)).toBe('anjing');
+
+      // With stricter max distance (1), it should not match
+      expect(findMostSimilarWithLevenshtein('anjiiiing', candidates, 0.7, 1)).toBeNull();
+    });
+  });
+
+  describe('isPossibleProfanityVariation function', () => {
+    test('should detect possible profanity variations', () => {
+      const profanityWords = ['anjing', 'babi'];
+      const [isProfanity, original] = isPossibleProfanityVariation('anjiing', profanityWords, 0.8);
+
+      expect(isProfanity).toBe(true);
+      expect(original).toBe('anjing');
+    });
+
+    test('should return false for non-matches', () => {
+      const profanityWords = ['anjing', 'babi'];
+      const [isProfanity, original] = isPossibleProfanityVariation('kucing', profanityWords, 0.8);
+
+      expect(isProfanity).toBe(false);
+      expect(original).toBeNull();
+    });
+  });
+
+  describe('findPossibleProfanityBySimiliarity function', () => {
+    test('should find similar profanity words in text', () => {
+      const text = 'anjiing adalah binatang lucu';
+      const profanityWords = ['anjing', 'babi', 'kontol'];
+
+      const results = findPossibleProfanityBySimiliarity(text, profanityWords, 0.8);
+
+      expect(results.length).toBe(1);
+      expect(results[0].word).toBe('anjiing');
+      expect(results[0].original).toBe('anjing');
+      expect(results[0].similarity).toBeGreaterThanOrEqual(0.8);
+    });
+
+    test('should not match when similarity is below threshold', () => {
+      const text = 'kucing adalah binatang lucu';
+      const profanityWords = ['anjing', 'babi'];
+
+      const results = findPossibleProfanityBySimiliarity(text, profanityWords, 0.8);
+      expect(results.length).toBe(0);
+    });
+  });
+
+  describe('findProfanityByLevenshteinDistance function', () => {
+    test('should find profanity by Levenshtein distance', () => {
+      const text = 'anjiing adalah binatang lucu';
+      const profanityWords = ['anjing', 'babi', 'kontol'];
+
+      const results = findProfanityByLevenshteinDistance(text, profanityWords, 0.8, 2);
+
+      expect(results.length).toBe(1);
+      expect(results[0].word).toBe('anjiing');
+      expect(results[0].original).toBe('anjing');
+      expect(results[0].distance).toBe(1);
+      expect(results[0].similarity).toBeGreaterThanOrEqual(0.8);
+    });
+
+    test('should respect maximum distance', () => {
+      const text = 'anjiiiing adalah binatang lucu';
+      const profanityWords = ['anjing', 'babi'];
+
+      // With max distance 2, should not match
+      const resultsStrict = findProfanityByLevenshteinDistance(text, profanityWords, 0.6, 2);
+      expect(resultsStrict.length).toBe(0);
+
+      // With max distance 4, should match
+      const resultsLax = findProfanityByLevenshteinDistance(text, profanityWords, 0.6, 4);
+      expect(resultsLax.length).toBe(1);
+    });
+  });
+});

package/test/utils/stringUtils.test.ts

@@ -0,0 +1,153 @@
+import {
+  censorWord,
+  normalizeText,
+  containsAnyWord,
+  containsEuphemism,
+  detectSplitWords,
+  escapeRegExp,
+  maskText,
+  splitIntoSentences,
+  getContextAroundIndex,
+} from '../../src/utils/stringUtils';
+
+describe('String Utils', () => {
+  describe('censorWord function', () => {
+    test('should replace entire word by default', () => {
+      expect(censorWord('test')).toBe('****');
+    });
+
+    test('should use custom replacement character', () => {
+      expect(censorWord('test', '#')).toBe('####');
+    });
+
+    test('should preserve first and last letters when specified', () => {
+      expect(censorWord('testing', '*', true)).toBe('t*****g');
+    });
+
+    test('should handle short words with keepFirstAndLast', () => {
+      expect(censorWord('hi', '*', true)).toBe('**');
+    });
+  });
+
+  describe('normalizeText function', () => {
+    test('should convert to lowercase', () => {
+      expect(normalizeText('TEST')).toBe('test');
+    });
+
+    test('should remove non-alphanumeric characters', () => {
+      expect(normalizeText('test!')).toBe('test');
+      expect(normalizeText('test#123')).toBe('test123');
+    });
+
+    test('should trim whitespace', () => {
+      expect(normalizeText(' test ')).toBe('test');
+    });
+
+    test('should normalize Unicode characters', () => {
+      expect(normalizeText('café')).toBe('cafe');
+    });
+  });
+
+  describe('containsAnyWord function', () => {
+    test('should detect whole words by default', () => {
+      expect(containsAnyWord('anjing', ['anjing'])).toBe(true);
+      expect(containsAnyWord('testanjing', ['anjing'])).toBe(false);
+    });
+
+    test('should detect substrings when specified', () => {
+      expect(containsAnyWord('testanjing', ['anjing'], true)).toBe(true);
+    });
+
+    test('should handle multiple words', () => {
+      expect(containsAnyWord('test anjing', ['test', 'anjing'])).toBe(true);
+      expect(containsAnyWord('test only', ['anjing', 'babi'])).toBe(false);
+    });
+  });
+
+  describe('containsEuphemism function', () => {
+    test('should detect euphemisms', () => {
+      expect(containsEuphemism('a****g', ['anjing'])).toBe(true);
+    });
+
+    test('should not match unrelated patterns', () => {
+      expect(containsEuphemism('a**g', ['anjing'])).toBe(false);
+    });
+
+    test('should handle short words properly', () => {
+      expect(containsEuphemism('a*b', ['ab'])).toBe(false); 
+    });
+  });
+
+  describe('detectSplitWords function', () => {
+    test('should detect split words', () => {
+      expect(detectSplitWords('a-n-j-i-n-g', ['anjing'])).toBe(true);
+      expect(detectSplitWords('a n j i n g', ['anjing'])).toBe(true);
+    });
+
+    test('should not detect unrelated words', () => {
+      expect(detectSplitWords('a-b-c-d', ['anjing'])).toBe(false);
+    });
+  });
+
+  describe('escapeRegExp function', () => {
+    test('should escape special regex characters', () => {
+      expect(escapeRegExp('test*')).toBe('test\\*');
+      expect(escapeRegExp('test.')).toBe('test\\.');
+      expect(escapeRegExp('test+')).toBe('test\\+');
+    });
+  });
+
+  describe('maskText function', () => {
+    test('should mask middle part of text', () => {
+      expect(maskText('password', 1, 1)).toBe('p******d');
+      expect(maskText('email@example.com', 2, 3)).toBe('em************com')
+    });
+
+    test('should handle short text', () => {
+      expect(maskText('ab', 1, 1)).toBe('ab');
+    });
+
+    test('should handle empty text', () => {
+      expect(maskText('', 1, 1)).toBe('');
+    });
+  });
+
+  describe('splitIntoSentences function', () => {
+    test('should split text into sentences', () => {
+      const sentences = splitIntoSentences('Hello. How are you? I am fine!');
+      expect(sentences.length).toBe(3);
+      expect(sentences[0]).toBe('Hello.');
+      expect(sentences[1]).toBe('How are you?');
+      expect(sentences[2]).toBe('I am fine!');
+    });
+
+    test('should handle empty text', () => {
+      expect(splitIntoSentences('')).toEqual([]);
+    });
+  });
+
+  describe('getContextAroundIndex function', () => {
+    const text = 'The quick brown fox jumps over the lazy dog';
+
+    test('should get context around index', () => {
+      // "fox" starts at index 16
+      const context = getContextAroundIndex(text, 16, 2);
+      expect(context).toBe('quick brown fox jumps over');
+    });
+
+    test('should handle index at start', () => {
+      const context = getContextAroundIndex(text, 0, 2);
+      expect(context).toBe('The quick brown');
+    });
+
+    test('should handle index at end', () => {
+      const context = getContextAroundIndex(text, text.length - 1, 2);
+      expect(context).toBe('the lazy dog');
+    });
+
+    test('should handle invalid index', () => {
+      expect(getContextAroundIndex(text, -1, 2)).toBe('');
+      expect(getContextAroundIndex(text, 100, 2)).toBe('');
+    });
+  });
+});