glin-profanity 3.1.5 → 3.2.2

package/dist/ml/index.js

@@ -810,7 +810,7 @@ var danish_default = {
 // ../../shared/dictionaries/dutch.json
 var dutch_default = {
   words: [
-    "aardappels afgieteng",
+    "aardappels afgieten",
     "achter het raam zitten",
     "afberen",
     "aflebberen",
@@ -827,7 +827,7 @@ var dutch_default = {
     "bagger schijten",
     "balen",
     "bedonderen",
-    "befborstelg",
+    "befborstel",
     "beffen",
     "bekken",
     "belazeren",
@@ -836,11 +836,11 @@ var dutch_default = {
     "beurt",
     "boemelen",
     "boerelul",
-    "boerenpummelg",
+    "boerenpummel",
     "bokkelul",
     "botergeil",
     "broekhoesten",
-    "brugpieperg",
+    "brugpieper",
     "buffelen",
     "buiten de pot piesen",
     "da's kloten van de bok",
@@ -848,13 +848,13 @@ var dutch_default = {
     "de hoer spelen",
     "de hond uitlaten",
     "de koffer induiken",
-    "delg",
+    "del",
     "de pijp aan maarten geven",
     "de pijp uitgaan",
     "dombo",
-    "draaikontg",
+    "draaikont",
     "driehoog achter wonen",
-    "drolg",
+    "drol",
     "drooggeiler",
     "droogkloot",
     "een beurt geven",
@@ -874,7 +874,7 @@ var dutch_default = {
     "godverdomme",
     "graftak",
     "gras maaien",
-    "gratenkutg",
+    "gratenkut",
     "greppeldel",
     "griet",
     "hoempert",
@@ -887,7 +887,7 @@ var dutch_default = {
     "huisdealer",
     "johny",
     "kanen",
-    "kettingzeugg",
+    "kettingzeug",
     "klaarkomen",
     "klerebeer",
     "klojo",
@@ -897,22 +897,22 @@ var dutch_default = {
     "klootzak",
     "kloten",
     "knor",
-    "kontg",
+    "kont",
     "kontneuken",
     "krentekakker",
     "kut",
     "kuttelikkertje",
-    "kwakkieg",
+    "kwakkie",
     "liefdesgrot",
     "lul",
     "lul-de-behanger",
     "lulhannes",
     "lummel",
     "mafketel",
-    "matennaaierg",
+    "matennaaier",
     "matje",
     "mof",
-    "mutsg",
+    "muts",
     "naaien",
     "naakt",
     "neuken",
@@ -932,9 +932,9 @@ var dutch_default = {
     "paal",
     "paardelul",
     "palen",
-    "penozeg",
+    "penoze",
     "piesen",
-    "pijpbekkieg",
+    "pijpbekkie",
     "pijpen",
     "pik",
     "pleurislaaier",
@@ -949,7 +949,7 @@ var dutch_default = {
     "reet",
     "reetridder",
     "reet trappen, voor zijn",
-    "remsporeng",
+    "remsporen",
     "reutelen",
     "rothoer",
     "rotzak",
@@ -962,25 +962,25 @@ var dutch_default = {
     "schuinsmarcheerder",
     "shit",
     "slempen",
-    "sletg",
+    "slet",
     "sletterig",
     "slik mijn zaad",
-    "snolg",
+    "snol",
     "spuiten",
     "standje",
-    "standje-69g",
+    "standje-69",
     "stoephoer",
     "stootje",
-    "strontg",
-    "sufferdg",
+    "stront",
+    "sufferd",
     "tapijtnek",
-    "teefg",
+    "teef",
     "temeier",
     "teringlijer",
     "toeter",
-    "tongzoeng",
-    "triootjeg",
-    "trottoir prostitu\xE9e",
+    "tongzoen",
+    "triootje",
+    "trottoir prostituee",
     "trottoirteef",
     "vergallen",
     "verkloten",
@@ -1053,6 +1053,8 @@ var english_default = {
     "2 girls 1 cup",
     "2g1c",
     "a$$",
+    "@ss",
+    "4ss",
     "acrotomophilia",
     "alabama hot pocket",
     "alaskan pipeline",
@@ -1192,6 +1194,10 @@ var english_default = {
     "eunuch",
     "f*ck",
     "f@ck",
+    "f4ck",
+    "fvck",
+    "phuck",
+    "fuk",
     "faggot",
     "fecal",
     "felch",
@@ -1373,6 +1379,9 @@ var english_default = {
     "shemale",
     "shibari",
     "shit",
+    "sh1t",
+    "$hit",
+    "$h!t",
     "shitblimp",
     "shithead",
     "shitshow",
@@ -2476,7 +2485,7 @@ var italian_default = {
     "di merda",
     "ditalino",
     "duro",
-    "fare una\u0160",
+    "fare una sega",
     "fava",
     "femminuccia",
     "fica",
@@ -2724,7 +2733,6 @@ var japanese_default = {
     "\u7389\u8210\u3081",
     "\u7DCA\u7E1B",
     "\u8FD1\u89AA\u76F8\u59E6",
-    "\u5ACC\u3044",
     "\u5F8C\u80CC\u4F4D",
     "\u5408\u610F\u306E\u6027\u4EA4",
     "\u62F7\u554F",
@@ -2737,7 +2745,6 @@ var japanese_default = {
     "\u5C04\u7CBE",
     "\u624B\u30B3\u30AD",
     "\u7363\u59E6",
-    "\u5973\u306E\u5B50",
     "\u5973\u738B\u69D8",
     "\u5973\u5B50\u9AD8\u751F",
     "\u5973\u88C5",
@@ -2814,7 +2821,6 @@ var turkish_default = {
     "ak",
     "akp",
     "al a\u011Fz\u0131na",
-    "allah",
     "allahs\u0131z",
     "am",
     "am biti",
@@ -2909,7 +2915,6 @@ var turkish_default = {
     "am\u0131n\u0131 s",
     "am\u0131s\u0131na",
     "am\u0131s\u0131n\u0131",
-    "ana",
     "anaaann",
     "anal",
     "analarn",
@@ -3041,8 +3046,6 @@ var turkish_default = {
     "cikar",
     "cim",
     "cm",
-    "coca cola",
-    "cola",
     "dalaks\u0131z",
     "dallama",
     "daltassak",
@@ -3840,7 +3843,7 @@ var turkish_default = {
 // ../../shared/dictionaries/spanish.json
 var spanish_default = {
   words: [
-    "sesinato",
+    "asesinato",
     "asno",
     "bastardo",
     "Bollera",
@@ -4140,6 +4143,34 @@ var GAMING_POSITIVE = /* @__PURE__ */ new Set([
   "move",
   "combo"
 ]);
+var GAMING_ACCEPTABLE_WORDS = /* @__PURE__ */ new Set([
+  "kill",
+  "killer",
+  "killed",
+  "killing",
+  "shoot",
+  "shot",
+  "shooting",
+  "die",
+  "dying",
+  "died",
+  "dead",
+  "death",
+  "badass",
+  "sick",
+  "insane",
+  "crazy",
+  "mad",
+  "beast",
+  "savage",
+  "suck",
+  "sucks",
+  "wtf",
+  "omg",
+  "hell",
+  "damn",
+  "crap"
+]);
 var POSITIVE_PHRASES = /* @__PURE__ */ new Map([
   ["the bomb", 0.9],
   // "this movie is the bomb"
@@ -4172,7 +4203,9 @@ var ContextAnalyzer = class {
   constructor(config) {
     this.contextWindow = config.contextWindow;
     this.language = config.language;
-    this.domainWhitelists = new Set(config.domainWhitelists || []);
+    this.domainWhitelists = new Set(
+      (config.domainWhitelists || []).map((word) => word.toLowerCase())
+    );
   }
   /**
    * Analyzes the context around a profanity match to determine if it should be flagged
@@ -4209,10 +4242,9 @@ var ContextAnalyzer = class {
       isWhitelisted: false
     };
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   checkPhraseContext(contextText, matchWord) {
     for (const [phrase, score] of POSITIVE_PHRASES.entries()) {
-      if (contextText.includes(phrase)) {
+      if (phrase.includes(matchWord) && contextText.includes(phrase)) {
         return {
           contextScore: score,
           reason: `Positive phrase detected: "${phrase}"`,
@@ -4231,21 +4263,29 @@ var ContextAnalyzer = class {
     }
     return null;
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   isDomainWhitelisted(contextWords, matchWord) {
+    const normalizedMatchWord = matchWord.toLowerCase();
     for (const word of contextWords) {
-      if (this.domainWhitelists.has(word) || GAMING_POSITIVE.has(word)) {
+      if (this.domainWhitelists.has(word)) {
         return true;
       }
+      if (GAMING_POSITIVE.has(word)) {
+        if (GAMING_ACCEPTABLE_WORDS.has(normalizedMatchWord)) {
+          return true;
+        }
+      }
     }
     return false;
   }
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   generateReason(score, contextWords) {
+    const foundPositive = Array.from(new Set(contextWords.filter((word) => POSITIVE_INDICATORS.has(word))));
+    const foundNegative = Array.from(new Set(contextWords.filter((word) => NEGATIVE_INDICATORS.has(word))));
     if (score >= 0.7) {
-      return "Positive context detected - likely not profanity";
+      const details = foundPositive.length > 0 ? ` (found: ${foundPositive.join(", ")})` : "";
+      return `Positive context detected${details} - likely not profanity`;
     } else if (score <= 0.3) {
-      return "Negative context detected - likely profanity";
+      const details = foundNegative.length > 0 ? ` (found: ${foundNegative.join(", ")})` : "";
+      return `Negative context detected${details} - likely profanity`;
     } else {
       return "Neutral context - uncertain classification";
     }
@@ -4303,7 +4343,7 @@ var ContextAnalyzer = class {
    * Updates the domain whitelist for this analyzer instance
    */
   updateDomainWhitelist(newWhitelist) {
-    this.domainWhitelists = new Set(newWhitelist);
+    this.domainWhitelists = new Set(newWhitelist.map((word) => word.toLowerCase()));
   }
   /**
    * Adds words to the domain whitelist
@@ -4462,6 +4502,10 @@ var HOMOGLYPHS = {
   // Cyrillic small e
   "\u0415": "E",
   // Cyrillic capital E
+  "\u043A": "k",
+  // Cyrillic small ka
+  "\u041A": "K",
+  // Cyrillic capital Ka
   "\u043E": "o",
   // Cyrillic small o
   "\u041E": "O",
@@ -4474,9 +4518,9 @@ var HOMOGLYPHS = {
   // Cyrillic small es
   "\u0421": "C",
   // Cyrillic capital Es
-  "\u0443": "y",
-  // Cyrillic small u
-  "\u0423": "Y",
+  "\u0443": "u",
+  // Cyrillic small u (map to u, not y)
+  "\u0423": "U",
   // Cyrillic capital U
   "\u0445": "x",
   // Cyrillic small ha
@@ -4494,6 +4538,11 @@ var HOMOGLYPHS = {
   // Cyrillic small dze
   "\u0405": "S",
   // Cyrillic capital Dze
+  // Currency and special symbols that look like letters
+  "\xA2": "c",
+  // Cent sign
+  "\u0192": "f",
+  // Latin small f with hook (florin)
   // Greek homoglyphs
   "\u03B1": "a",
   // Greek small alpha
@@ -4840,6 +4889,7 @@ var Filter = class {
     this.cacheResults = config?.cacheResults ?? false;
     this.maxCacheSize = config?.maxCacheSize ?? 1e3;
     this.cache = /* @__PURE__ */ new Map();
+    this.regexCache = /* @__PURE__ */ new Map();
     let words = [];
     if (config?.allLanguages) {
       for (const lang in dictionary_default) {
@@ -4869,9 +4919,10 @@ var Filter = class {
    * Applies Unicode normalization, leetspeak detection, and obfuscation handling.
    *
    * @param text - The input text to normalize
+   * @param aggressive - If true, collapses to single chars (for repeated char detection)
    * @returns The normalized text
    */
-  normalizeText(text) {
+  normalizeText(text, aggressive = false) {
     let normalized = text;
     if (this.normalizeUnicodeEnabled) {
       normalized = normalizeUnicode(normalized);
@@ -4880,6 +4931,8 @@ var Filter = class {
       normalized = normalizeLeetspeak(normalized, {
         level: this.leetspeakLevel,
         collapseRepeated: true,
+        // Keep double letters like "ss" for normal check, collapse all for aggressive
+        maxRepeated: aggressive ? 1 : 2,
         removeSpacedChars: true
       });
     }
@@ -4910,6 +4963,7 @@ var Filter = class {
    */
   clearCache() {
     this.cache.clear();
+    this.regexCache.clear();
   }
   /**
    * Gets the current cache size.
@@ -4991,10 +5045,17 @@ var Filter = class {
     return this.cache.get(key);
   }
   getRegex(word) {
+    if (this.regexCache.has(word)) {
+      const regex2 = this.regexCache.get(word);
+      regex2.lastIndex = 0;
+      return regex2;
+    }
     const flags = this.caseSensitive ? "g" : "gi";
     const escapedWord = word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
     const boundary = this.wordBoundaries ? "\\b" : "";
-    return new RegExp(`${boundary}${escapedWord}${boundary}`, flags);
+    const regex = new RegExp(`${boundary}${escapedWord}${boundary}`, flags);
+    this.regexCache.set(word, regex);
+    return regex;
   }
   isFuzzyToleranceMatch(word, text) {
     const simplifiedText = text.toLowerCase().replace(/[^a-z]/g, "");
@@ -5012,11 +5073,12 @@ var Filter = class {
     return score >= this.fuzzyToleranceLevel;
   }
   evaluateSeverity(word, text) {
-    if (this.wordBoundaries && this.getRegex(word).test(text)) {
+    if (this.getRegex(word).test(text)) {
       return 1 /* EXACT */;
     }
-    if (this.getRegex(word).test(text)) return 1 /* EXACT */;
-    if (this.isFuzzyToleranceMatch(word, text)) return 2 /* FUZZY */;
+    if (!this.wordBoundaries && this.isFuzzyToleranceMatch(word, text)) {
+      return 2 /* FUZZY */;
+    }
     return void 0;
   }
   /**
@@ -5036,9 +5098,20 @@ var Filter = class {
    * ```
    */
   isProfane(value) {
-    const input = this.normalizeText(value);
+    const originalInput = value;
+    const normalizedInput = this.normalizeText(value);
+    const aggressiveInput = this.normalizeText(value, true);
     for (const word of this.words.keys()) {
-      if (!this.ignoreWords.has(word.toLowerCase()) && this.evaluateSeverity(word, input) !== void 0) {
+      if (this.ignoreWords.has(word.toLowerCase())) {
+        continue;
+      }
+      if (this.evaluateSeverity(word, originalInput) !== void 0) {
+        return true;
+      }
+      if (this.evaluateSeverity(word, normalizedInput) !== void 0) {
+        return true;
+      }
+      if (this.evaluateSeverity(word, aggressiveInput) !== void 0) {
         return true;
       }
     }
@@ -5079,23 +5152,45 @@ var Filter = class {
       return cachedResult;
     }
     if (!this.enableContextAware) {
-      let input2 = this.normalizeText(text);
-      input2 = input2.toLowerCase();
+      const originalInput = text.toLowerCase();
+      const normalizedInput = this.normalizeText(text).toLowerCase();
+      const aggressiveInput = this.normalizeText(text, true).toLowerCase();
       const profaneWords2 = [];
       const severityMap2 = {};
       for (const dictWord of this.words.keys()) {
         if (this.ignoreWords.has(dictWord.toLowerCase())) continue;
-        const severity = this.evaluateSeverity(dictWord, input2);
+        let severity = this.evaluateSeverity(dictWord, originalInput);
         if (severity !== void 0) {
           const regex = this.getRegex(dictWord);
           let match;
-          while ((match = regex.exec(input2)) !== null) {
+          while ((match = regex.exec(originalInput)) !== null) {
             profaneWords2.push(match[0]);
             if (severityMap2[match[0]] === void 0) {
               severityMap2[match[0]] = severity;
             }
           }
         }
+        severity = this.evaluateSeverity(dictWord, normalizedInput);
+        if (severity !== void 0) {
+          const regex = this.getRegex(dictWord);
+          while ((regex.exec(normalizedInput)) !== null) {
+            if (!profaneWords2.includes(dictWord)) {
+              profaneWords2.push(dictWord);
+              if (severityMap2[dictWord] === void 0) {
+                severityMap2[dictWord] = severity;
+              }
+            }
+          }
+        }
+        severity = this.evaluateSeverity(dictWord, aggressiveInput);
+        if (severity !== void 0) {
+          if (!profaneWords2.includes(dictWord)) {
+            profaneWords2.push(dictWord);
+            if (severityMap2[dictWord] === void 0) {
+              severityMap2[dictWord] = severity;
+            }
+          }
+        }
       }
       let processedText2 = text;
       if (this.replaceWith && profaneWords2.length > 0) {
@@ -5481,4 +5576,206 @@ var HybridFilter = class {
   }
 };
 
-export { HybridFilter, ToxicityDetector };
+// src/ml/transformers.ts
+var RECOMMENDED_MODELS = {
+  /** High accuracy English model (97.5%) - 67M params */
+  pardonmyai: "tarekziade/pardonmyai",
+  /** Smaller version for constrained environments */
+  pardonmyaiTiny: "tarekziade/pardonmyai-tiny",
+  /** Multilingual toxicity detection (7 languages) */
+  toxicBert: "unitary/toxic-bert",
+  /** Offensive speech detector (DeBERTa-based) */
+  offensiveSpeech: "KoalaAI/OffensiveSpeechDetector"
+};
+var MODEL_PROFANE_LABELS = {
+  "tarekziade/pardonmyai": "profane",
+  "tarekziade/pardonmyai-tiny": "profane",
+  "unitary/toxic-bert": "toxic",
+  "KoalaAI/OffensiveSpeechDetector": "LABEL_1",
+  // Offensive
+  default: "LABEL_1"
+};
+async function getTransformers() {
+  try {
+    const transformers = await import('@xenova/transformers');
+    return transformers;
+  } catch {
+    throw new Error(
+      "Transformers.js is required for ML features. Install it with: npm install @xenova/transformers"
+    );
+  }
+}
+async function createMLChecker(config = {}) {
+  const {
+    model = RECOMMENDED_MODELS.pardonmyai,
+    threshold = 0.5,
+    profaneLabel = MODEL_PROFANE_LABELS[model] || MODEL_PROFANE_LABELS.default,
+    quantized = true,
+    device = "cpu"
+  } = config;
+  const transformers = await getTransformers();
+  const classifier = await transformers.pipeline("text-classification", model, {
+    quantized,
+    device
+  });
+  return {
+    /**
+     * Check a single text for profanity
+     */
+    async check(text) {
+      const startTime = Date.now();
+      const output = await classifier(text);
+      const processingTimeMs = Date.now() - startTime;
+      const profaneScore = output.find((o) => o.label === profaneLabel)?.score || 0;
+      const containsProfanity = profaneScore >= threshold;
+      return {
+        containsProfanity,
+        confidence: profaneScore,
+        rawOutput: output,
+        processingTimeMs
+      };
+    },
+    /**
+     * Check multiple texts
+     */
+    async checkBatch(texts) {
+      return Promise.all(texts.map((text) => this.check(text)));
+    },
+    /**
+     * Get the profanity score for text (0-1)
+     */
+    async getScore(text) {
+      const result = await this.check(text);
+      return result.confidence;
+    },
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+      return { model, threshold, profaneLabel, quantized, device };
+    },
+    /**
+     * Dispose of the model (free memory)
+     */
+    dispose() {
+    }
+  };
+}
+async function createHybridChecker(config = {}) {
+  const {
+    model = RECOMMENDED_MODELS.pardonmyai,
+    threshold = 0.5,
+    profaneLabel,
+    quantized = true,
+    device = "cpu",
+    filterConfig = {},
+    mlThreshold = 0.3,
+    dictionaryWeight = 0.6,
+    mlWeight = 0.4
+  } = config;
+  const filter = new Filter({
+    languages: filterConfig.languages || ["english"],
+    detectLeetspeak: filterConfig.detectLeetspeak ?? true,
+    normalizeUnicode: filterConfig.normalizeUnicode ?? true,
+    severityLevels: true,
+    cacheResults: true,
+    ...filterConfig
+  });
+  let mlChecker = null;
+  async function getMLChecker() {
+    if (!mlChecker) {
+      mlChecker = await createMLChecker({
+        model,
+        threshold,
+        profaneLabel,
+        quantized,
+        device
+      });
+    }
+    return mlChecker;
+  }
+  return {
+    /**
+     * Check text using hybrid approach
+     */
+    async check(text) {
+      const startTime = Date.now();
+      const dictionaryResult = filter.checkProfanity(text);
+      if (dictionaryResult.containsProfanity) {
+        return {
+          containsProfanity: true,
+          confidence: 1,
+          dictionaryResult,
+          usedML: false,
+          profaneWords: dictionaryResult.profaneWords,
+          processingTimeMs: Date.now() - startTime
+        };
+      }
+      const ml = await getMLChecker();
+      const mlResult = await ml.check(text);
+      const dictionaryScore = dictionaryResult.containsProfanity ? 1 : 0;
+      const combinedScore = dictionaryScore * dictionaryWeight + mlResult.confidence * mlWeight;
+      const containsProfanity = combinedScore >= mlThreshold;
+      return {
+        containsProfanity,
+        confidence: combinedScore,
+        dictionaryResult,
+        mlResult,
+        usedML: true,
+        profaneWords: dictionaryResult.profaneWords,
+        processingTimeMs: Date.now() - startTime
+      };
+    },
+    /**
+     * Check multiple texts
+     */
+    async checkBatch(texts) {
+      return Promise.all(texts.map((text) => this.check(text)));
+    },
+    /**
+     * Dictionary-only check (fast, no ML)
+     */
+    checkFast(text) {
+      return filter.checkProfanity(text);
+    },
+    /**
+     * ML-only check (slower, more accurate)
+     */
+    async checkML(text) {
+      const ml = await getMLChecker();
+      return ml.check(text);
+    },
+    /**
+     * Get the underlying filter
+     */
+    getFilter() {
+      return filter;
+    },
+    /**
+     * Dispose of resources
+     */
+    async dispose() {
+      if (mlChecker) {
+        mlChecker.dispose();
+        mlChecker = null;
+      }
+    }
+  };
+}
+async function isTransformersAvailable() {
+  try {
+    await getTransformers();
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function preloadModel(model = RECOMMENDED_MODELS.pardonmyai, options = {}) {
+  const { quantized = true } = options;
+  const transformers = await getTransformers();
+  await transformers.pipeline("text-classification", model, {
+    quantized
+  });
+}
+
+export { HybridFilter, RECOMMENDED_MODELS, ToxicityDetector, createHybridChecker, createMLChecker, isTransformersAvailable, preloadModel };