@sideid/id-profanity-filter 1.12.0 → 1.13.0

package/dist/index.js

@@ -8,7 +8,7 @@ const general = [
         category: 'profanity',
         region: 'general',
         severity: 0.7,
-        aliases: ['anjay', 'anjir', 'anying', 'njing', 'anj', 'anjg', 'ajg'],
+        aliases: ['anjay', 'anjir', 'anying', 'njing', 'anj', 'anjg', 'ajg', 'anjic', 'anjink'],
         description: 'Mengacu pada hewan anjing, digunakan sebagai umpatan',
         context: 'Umpatan umum untuk menunjukkan kemarahan atau ketidaksetujuan',
     },
@@ -116,7 +116,7 @@ const general = [
         category: 'insult',
         region: 'general',
         severity: 0.5,
-        aliases: ['sialn', 'sl'],
+        aliases: ['sialn'],
         description: 'Kata yang mengacu pada orang yang membawa sial',
         context: 'Hinaan untuk menyebut orang yang dianggap membawa sial',
     },
@@ -179,7 +179,7 @@ const general = [
         category: 'sexual',
         region: 'general',
         severity: 0.9,
-        aliases: ['ngew', 'we'],
+        aliases: ['ngew'],
         description: 'Istilah kasar untuk aktivitas seksual',
         context: 'Kata vulgar yang merujuk pada aktivitas seksual',
     },
@@ -1611,7 +1611,7 @@ const sexual = [
         category: 'sexual',
         region: 'general',
         severity: 0.9,
-        aliases: ['ngew', 'we'],
+        aliases: ['ngew'],
         description: 'Istilah kasar untuk aktivitas seksual',
         context: 'Kata vulgar yang merujuk pada aktivitas seksual',
     },
@@ -2143,16 +2143,7 @@ const wordObjects = [
     ...minang,
     ...bali,
     ...madura,
-    // ...bugis,
     ...aceh,
-    // ...ambon,
-    // ...papua,
-    // ...manado,
-    // ...banjar,
-    // ...palembang,
-    // ...lampung,
-    // ...ntt,
-    // ...ntb,
 ];
 wordObjects.map((item) => item.word);
 /**
@@ -2468,8 +2459,7 @@ function addIndonesianVariations(pattern) {
         const lowerChar = char.toLowerCase();
         const variations = variationMap[lowerChar];
         if (variations && variations.length > 1) {
-            // Create a character class with all variations
-            result += `[${variations.join('')}]`;
+            result += `(?:${variations.join('|')})`;
         }
         else {
             result += char;
@@ -3105,6 +3095,36 @@ class AhoCorasick {
         const matches = this.search(text);
         return new Set(matches.keys());
     }
+    /**
+     * Mencari semua kemunculan pola beserta posisi indeks awal dan akhirnya
+     * @param text Teks yang akan dicari
+     * @returns Array objek berisi pattern, start, dan end
+     */
+    searchWithPositions(text) {
+        if (!this.built) {
+            this.build();
+        }
+        const results = [];
+        const normalizedText = text.toLowerCase();
+        let node = this.root;
+        for (let i = 0; i < normalizedText.length; i++) {
+            const char = normalizedText[i];
+            while (node !== this.root && !node.children.has(char)) {
+                node = node.fail;
+            }
+            if (node.children.has(char)) {
+                node = node.children.get(char);
+            }
+            for (const match of node.output) {
+                results.push({
+                    pattern: match,
+                    start: i + 1 - match.length,
+                    end: i + 1,
+                });
+            }
+        }
+        return results;
+    }
     /**
      * Mengecek apakah teks mengandung setidaknya satu pola
      * @param text Teks yang akan dicari
@@ -3132,16 +3152,22 @@ class AhoCorasick {
     }
 }
 
-const globalAhoCorasick = new AhoCorasick();
-let ahoCorasickInitialized = false;
-function initializeAhoCorasick(words) {
-    if (ahoCorasickInitialized)
-        return;
-    for (const word of words) {
-        globalAhoCorasick.addPattern(word);
+let defaultAhoCorasick = null;
+function getDefaultAhoCorasick() {
+    if (!defaultAhoCorasick) {
+        const ac = new AhoCorasick();
+        for (const wordObj of wordObjects) {
+            ac.addPattern(wordObj.word);
+            if (wordObj.aliases) {
+                for (const alias of wordObj.aliases) {
+                    ac.addPattern(alias);
+                }
+            }
+        }
+        ac.build();
+        defaultAhoCorasick = ac;
     }
-    globalAhoCorasick.build();
-    ahoCorasickInitialized = true;
+    return defaultAhoCorasick;
 }
 function getWordMetadata(word) {
     return wordObjects.find((obj) => obj.word.toLowerCase() === word.toLowerCase() ||
@@ -3149,10 +3175,15 @@ function getWordMetadata(word) {
 }
 function findProfanity(text, options = {}) {
     const { wordList = [], detectLeetSpeak = true, checkSubstring = false, whitelist = [], categories, regions, severityThreshold = 0, indonesianVariation = false, detectSimilarity = false, similarityThreshold = 0.8, detectSplit = false, useLevenshtein = false, maxLevenshteinDistance = 2, } = { ...DEFAULT_OPTIONS, ...options };
+    const hasCustomWordList = Boolean(wordList && wordList.length > 0);
     const normalizedWhitelist = whitelist.map((w) => w.toLowerCase());
     const normalizedText = normalizeText(text);
-    let baseWordsToCheck = wordList.length > 0 ? wordList : [];
-    if (baseWordsToCheck.length === 0) {
+    let baseWordsToCheck = [];
+    const aliasMap = new Map();
+    if (hasCustomWordList) {
+        baseWordsToCheck = wordList;
+    }
+    else {
         const filteredWords = wordObjects.filter((word) => {
             const matchCategory = categories ? categories.includes(word.category) : true;
             const matchRegion = regions ? regions.includes(word.region) : true;
@@ -3160,22 +3191,14 @@ function findProfanity(text, options = {}) {
             return matchCategory && matchRegion && matchSeverity;
         });
         baseWordsToCheck = filteredWords.map((word) => word.word);
+        filteredWords.forEach((wordObj) => {
+            if (wordObj.aliases && wordObj.aliases.length > 0) {
+                wordObj.aliases.forEach((alias) => {
+                    aliasMap.set(alias.toLowerCase(), wordObj.word.toLowerCase());
+                });
+            }
+        });
     }
-    const aliasMap = new Map();
-    wordObjects.forEach((wordObj) => {
-        const matchCategory = categories ? categories.includes(wordObj.category) : true;
-        const matchRegion = regions ? regions.includes(wordObj.region) : true;
-        const matchSeverity = wordObj.severity >= severityThreshold;
-        if (matchCategory &&
-            matchRegion &&
-            matchSeverity &&
-            wordObj.aliases &&
-            wordObj.aliases.length > 0) {
-            wordObj.aliases.forEach((alias) => {
-                aliasMap.set(alias.toLowerCase(), wordObj.word.toLowerCase());
-            });
-        }
-    });
     const wordsToCheck = [...baseWordsToCheck, ...Array.from(aliasMap.keys())].filter((word) => !normalizedWhitelist.includes(word.toLowerCase()));
     if (wordsToCheck.length === 0) {
         return [];
@@ -3185,6 +3208,8 @@ function findProfanity(text, options = {}) {
     const passesFilters = (word) => {
         if (normalizedWhitelist.includes(word.toLowerCase()))
             return false;
+        if (hasCustomWordList)
+            return true;
         const metadata = getWordMetadata(word);
         if (!metadata)
             return false;
@@ -3193,9 +3218,27 @@ function findProfanity(text, options = {}) {
         const matchSeverity = metadata.severity >= severityThreshold;
         return matchCategory && matchRegion && matchSeverity;
     };
-    initializeAhoCorasick(wordsToCheck);
-    const basicMatches = globalAhoCorasick.searchUnique(normalizedText);
-    for (const match of basicMatches) {
+    let ac;
+    if (hasCustomWordList) {
+        ac = new AhoCorasick();
+        for (const w of wordsToCheck) {
+            ac.addPattern(w);
+        }
+        ac.build();
+    }
+    else {
+        ac = getDefaultAhoCorasick();
+    }
+    const occurrences = ac.searchWithPositions(normalizedText);
+    for (const occ of occurrences) {
+        const { pattern: match, start, end } = occ;
+        if (!checkSubstring) {
+            const isWordStart = start === 0 || !/[a-z0-9_]/i.test(normalizedText[start - 1]);
+            const isWordEnd = end === normalizedText.length || !/[a-z0-9_]/i.test(normalizedText[end]);
+            if (!isWordStart || !isWordEnd) {
+                continue;
+            }
+        }
         if (normalizedWhitelist.includes(match.toLowerCase()))
             continue;
         const originalWord = aliasMap.get(match.toLowerCase()) || match.toLowerCase();
@@ -3320,8 +3363,8 @@ function findProfanity(text, options = {}) {
 /**
  * Mencari kata kotor lengkap dengan metadata
  *
- * @param text Teks yang akan diperika
- * @param options Opsi utnuk pencarian kata kotor
+ * @param text Teks yang akan diperiksa
+ * @param options Opsi untuk pencarian kata kotor
  * @return Array dari objek kata kotor yang ditemukan
  */
 function findProfanityWithMetadata(text, options = {}) {
@@ -3333,14 +3376,22 @@ function findProfanityWithMetadata(text, options = {}) {
         .map((word) => {
         const wordObject = wordObjects.find((obj) => obj.word.toLowerCase() === word.toLowerCase() ||
             (obj.aliases && obj.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase())));
-        return wordObject;
+        if (wordObject) {
+            return wordObject;
+        }
+        return {
+            word,
+            category: 'profanity',
+            region: 'general',
+            severity: 0.5,
+        };
     })
         .filter((word) => word !== undefined);
 }
 /**
- * Mencari kategory kata kotor yang ada dalam teks
+ * Mencari kategori kata kotor yang ada dalam teks
  *
- * @param matchDetails Hasil pencarian dari fingProfanityWithMetadata()
+ * @param matchDetails Hasil pencarian dari findProfanityWithMetadata()
  * @return Array kategori unik
  */
 function findCategories(matchDetails) {
@@ -3353,7 +3404,7 @@ function findCategories(matchDetails) {
 /**
  * Mencari region kata kotor yang ada dalam teks
  *
- * @param matchDetails Hasil pencarian dari fingProfanityWithMetadata()
+ * @param matchDetails Hasil pencarian dari findProfanityWithMetadata()
  * @return Array region unik
  */
 function findRegions(matchDetails) {
@@ -3373,7 +3424,7 @@ function calculateSeverity(matchDetails) {
     if (matchDetails.length === 0) {
         return 0;
     }
-    const countFactor = Math.min(matchDetails.length / 10, 1); // Maksimal 10 kata
+    const countFactor = Math.min(matchDetails.length / 10, 1);
     const categoryWeights = {
         sexual: 0.9,
         blasphemy: 0.9,
@@ -3401,21 +3452,9 @@ function calculateSeverity(matchDetails) {
  * @returns FilterResult dengan hasil filter
  */
 function filter(text, options = {}) {
-    const { replaceWith = '*', fullWordCensor = true, detectLeetSpeak = true, whitelist = [], checkSubstring = false, useRandomGrawlix = false, keepFirstAndLast = false, indonesianVariation = false, detectSplit = false, detectSimilarity = false, useLevenshtein = false, maxLevenshteinDistance = 2, similarityThreshold = 0.8, } = { ...DEFAULT_OPTIONS, ...options };
-    const matches = findProfanity(text, {
-        ...options,
-        detectLeetSpeak,
-        whitelist,
-        checkSubstring,
-        indonesianVariation,
-        detectSplit,
-        detectSimilarity,
-        useLevenshtein,
-        maxLevenshteinDistance,
-        similarityThreshold,
-    });
-    const actualMatches = findProfanity.lastActualMatches || new Map();
-    const matchDetails = findProfanityWithMetadata(text, options);
+    const mergedOptions = { ...DEFAULT_OPTIONS, ...options };
+    const { replaceWith = '*', fullWordCensor = true, detectLeetSpeak = true, whitelist = [], checkSubstring = false, useRandomGrawlix = false, keepFirstAndLast = false, detectSplit = false, } = mergedOptions;
+    const matches = findProfanity(text, mergedOptions);
     if (matches.length === 0) {
         return {
             filtered: text,
@@ -3423,123 +3462,62 @@ function filter(text, options = {}) {
             replacements: [],
         };
     }
+    const actualMatches = findProfanity.lastActualMatches || new Map();
+    const matchDetails = findProfanityWithMetadata(text, mergedOptions);
+    const normalizedWhitelist = whitelist.map((w) => w.toLowerCase());
     let filteredText = text;
     const replacements = [];
-    matches.forEach((word) => {
+    const getCensoredWord = (originalWord) => {
+        if (useRandomGrawlix) {
+            return makeRandomGrawlixString(originalWord.length);
+        }
+        return censorWord(originalWord, replaceWith, !fullWordCensor && keepFirstAndLast);
+    };
+    const applyReplacement = (pattern, metadata) => {
+        filteredText = filteredText.replace(pattern, (matchedStr) => {
+            if (normalizedWhitelist.includes(matchedStr.toLowerCase())) {
+                return matchedStr;
+            }
+            const censored = getCensoredWord(matchedStr);
+            replacements.push({
+                original: matchedStr,
+                censored,
+                metadata,
+            });
+            return censored;
+        });
+    };
+    for (const word of matches) {
         const metadata = matchDetails.find((m) => m.word.toLowerCase() === word.toLowerCase() ||
             (m.aliases && m.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase())));
         const variants = actualMatches.get(word.toLowerCase()) || [];
-        variants.push(word);
-        const uniqueVariants = [...new Set(variants)];
-        uniqueVariants.forEach((variant) => {
-            const regex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, 'gi');
-            let match;
-            while ((match = regex.exec(filteredText)) !== null) {
-                const originalWord = match[0];
-                if (whitelist.includes(originalWord.toLowerCase()))
-                    continue;
-                let censoredWord;
-                if (useRandomGrawlix) {
-                    censoredWord = makeRandomGrawlixString(originalWord.length);
-                }
-                else {
-                    censoredWord = censorWord(originalWord, replaceWith, !fullWordCensor && keepFirstAndLast);
-                }
-                replacements.push({
-                    original: originalWord,
-                    censored: censoredWord,
-                    metadata,
-                });
-                filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, 'g'), censoredWord);
-            }
-        });
-        if (detectSplit || detectLeetSpeak) {
-            if (detectLeetSpeak) {
-                const leetRegex = createWordRegex(word, {
-                    wholeWord: true,
-                    caseSensitive: false,
-                    leetSpeak: true,
-                    detectSplit: false,
-                    indonesianVariation: false,
-                });
-                let match;
-                while ((match = leetRegex.exec(filteredText)) !== null) {
-                    const originalWord = match[0];
-                    if (whitelist.includes(originalWord.toLowerCase()))
-                        continue;
-                    let censoredWord;
-                    if (useRandomGrawlix) {
-                        censoredWord = makeRandomGrawlixString(originalWord.length);
-                    }
-                    else {
-                        censoredWord = censorWord(originalWord, replaceWith, !fullWordCensor && keepFirstAndLast);
-                    }
-                    replacements.push({
-                        original: originalWord,
-                        censored: censoredWord,
-                        metadata,
-                    });
-                    filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord), 'g'), censoredWord);
-                }
-            }
-            if (detectSplit) {
-                const splitRegex = createWordRegex(word, {
-                    wholeWord: false,
-                    caseSensitive: false,
-                    leetSpeak: false,
-                    detectSplit: true,
-                    indonesianVariation: false,
-                });
-                let match;
-                while ((match = splitRegex.exec(filteredText)) !== null) {
-                    const originalWord = match[0];
-                    if (whitelist.includes(originalWord.toLowerCase()))
-                        continue;
-                    let censoredWord;
-                    if (useRandomGrawlix) {
-                        censoredWord = makeRandomGrawlixString(originalWord.length);
-                    }
-                    else {
-                        censoredWord = censorWord(originalWord, replaceWith, !fullWordCensor && keepFirstAndLast);
-                    }
-                    replacements.push({
-                        original: originalWord,
-                        censored: censoredWord,
-                        metadata,
-                    });
-                    filteredText = filteredText.replace(new RegExp(escapeRegExp(originalWord), 'g'), censoredWord);
-                }
-            }
+        const allVariants = [...new Set([...variants, word])].sort((a, b) => b.length - a.length);
+        for (const variant of allVariants) {
+            const boundaryPattern = checkSubstring
+                ? escapeRegExp(variant)
+                : `\\b${escapeRegExp(variant)}\\b`;
+            applyReplacement(new RegExp(boundaryPattern, 'gi'), metadata);
         }
-    });
-    if (detectSimilarity && useLevenshtein) {
-        matches.forEach((word) => {
-            const metadata = matchDetails.find((m) => m.word.toLowerCase() === word.toLowerCase() ||
-                (m.aliases && m.aliases.some((alias) => alias.toLowerCase() === word.toLowerCase())));
-            const variants = actualMatches.get(word.toLowerCase()) || [];
-            variants.forEach((variant) => {
-                const exactVariantRegex = new RegExp(`\\b${escapeRegExp(variant)}\\b`, 'gi');
-                let match;
-                while ((match = exactVariantRegex.exec(filteredText)) !== null) {
-                    const originalWord = match[0];
-                    if (whitelist.includes(originalWord.toLowerCase()))
-                        continue;
-                    let censoredWord;
-                    if (useRandomGrawlix) {
-                        censoredWord = makeRandomGrawlixString(originalWord.length);
-                    }
-                    else {
-                        censoredWord = censorWord(originalWord, replaceWith, !fullWordCensor && keepFirstAndLast);
-                    }
-                    replacements.push({
-                        original: originalWord,
-                        censored: censoredWord,
-                        metadata,
-                    });
-                    filteredText = filteredText.replace(new RegExp(`\\b${escapeRegExp(originalWord)}\\b`, 'g'), censoredWord);
-                }
+        if (detectLeetSpeak) {
+            const leetRegex = createWordRegex(word, {
+                wholeWord: !checkSubstring,
+                caseSensitive: false,
+                leetSpeak: true,
+                detectSplit: false,
+                indonesianVariation: false,
             });
-        });
+            applyReplacement(leetRegex, metadata);
+        }
+        if (detectSplit) {
+            const splitRegex = createWordRegex(word, {
+                wholeWord: false,
+                caseSensitive: false,
+                leetSpeak: false,
+                detectSplit: true,
+                indonesianVariation: false,
+            });
+            applyReplacement(splitRegex, metadata);
+        }
     }
     return {
         filtered: filteredText,
@@ -3781,6 +3759,13 @@ class IDProfanityFilter {
             ...options,
         };
     }
+    /**
+     * Mengatur ulang opsi filter ke default
+     * @param options Opsi baru untuk override default
+     */
+    resetOptions(options = {}) {
+        this.options = { ...DEFAULT_OPTIONS, ...options };
+    }
     /**
      * Menggunakan preset yang telah ditentukan
      * @param presetName Nama preset yang akan digunakan