glin-profanity 3.2.0 → 3.2.2

package/dist/ml/index.cjs

@@ -5578,5 +5578,212 @@ var HybridFilter = class {
   }
 };
 
+// src/ml/transformers.ts
+var RECOMMENDED_MODELS = {
+  /** High accuracy English model (97.5%) - 67M params */
+  pardonmyai: "tarekziade/pardonmyai",
+  /** Smaller version for constrained environments */
+  pardonmyaiTiny: "tarekziade/pardonmyai-tiny",
+  /** Multilingual toxicity detection (7 languages) */
+  toxicBert: "unitary/toxic-bert",
+  /** Offensive speech detector (DeBERTa-based) */
+  offensiveSpeech: "KoalaAI/OffensiveSpeechDetector"
+};
+var MODEL_PROFANE_LABELS = {
+  "tarekziade/pardonmyai": "profane",
+  "tarekziade/pardonmyai-tiny": "profane",
+  "unitary/toxic-bert": "toxic",
+  "KoalaAI/OffensiveSpeechDetector": "LABEL_1",
+  // Offensive
+  default: "LABEL_1"
+};
+async function getTransformers() {
+  try {
+    const transformers = await import('@xenova/transformers');
+    return transformers;
+  } catch {
+    throw new Error(
+      "Transformers.js is required for ML features. Install it with: npm install @xenova/transformers"
+    );
+  }
+}
+async function createMLChecker(config = {}) {
+  const {
+    model = RECOMMENDED_MODELS.pardonmyai,
+    threshold = 0.5,
+    profaneLabel = MODEL_PROFANE_LABELS[model] || MODEL_PROFANE_LABELS.default,
+    quantized = true,
+    device = "cpu"
+  } = config;
+  const transformers = await getTransformers();
+  const classifier = await transformers.pipeline("text-classification", model, {
+    quantized,
+    device
+  });
+  return {
+    /**
+     * Check a single text for profanity
+     */
+    async check(text) {
+      const startTime = Date.now();
+      const output = await classifier(text);
+      const processingTimeMs = Date.now() - startTime;
+      const profaneScore = output.find((o) => o.label === profaneLabel)?.score || 0;
+      const containsProfanity = profaneScore >= threshold;
+      return {
+        containsProfanity,
+        confidence: profaneScore,
+        rawOutput: output,
+        processingTimeMs
+      };
+    },
+    /**
+     * Check multiple texts
+     */
+    async checkBatch(texts) {
+      return Promise.all(texts.map((text) => this.check(text)));
+    },
+    /**
+     * Get the profanity score for text (0-1)
+     */
+    async getScore(text) {
+      const result = await this.check(text);
+      return result.confidence;
+    },
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+      return { model, threshold, profaneLabel, quantized, device };
+    },
+    /**
+     * Dispose of the model (free memory)
+     */
+    dispose() {
+    }
+  };
+}
+async function createHybridChecker(config = {}) {
+  const {
+    model = RECOMMENDED_MODELS.pardonmyai,
+    threshold = 0.5,
+    profaneLabel,
+    quantized = true,
+    device = "cpu",
+    filterConfig = {},
+    mlThreshold = 0.3,
+    dictionaryWeight = 0.6,
+    mlWeight = 0.4
+  } = config;
+  const filter = new Filter({
+    languages: filterConfig.languages || ["english"],
+    detectLeetspeak: filterConfig.detectLeetspeak ?? true,
+    normalizeUnicode: filterConfig.normalizeUnicode ?? true,
+    severityLevels: true,
+    cacheResults: true,
+    ...filterConfig
+  });
+  let mlChecker = null;
+  async function getMLChecker() {
+    if (!mlChecker) {
+      mlChecker = await createMLChecker({
+        model,
+        threshold,
+        profaneLabel,
+        quantized,
+        device
+      });
+    }
+    return mlChecker;
+  }
+  return {
+    /**
+     * Check text using hybrid approach
+     */
+    async check(text) {
+      const startTime = Date.now();
+      const dictionaryResult = filter.checkProfanity(text);
+      if (dictionaryResult.containsProfanity) {
+        return {
+          containsProfanity: true,
+          confidence: 1,
+          dictionaryResult,
+          usedML: false,
+          profaneWords: dictionaryResult.profaneWords,
+          processingTimeMs: Date.now() - startTime
+        };
+      }
+      const ml = await getMLChecker();
+      const mlResult = await ml.check(text);
+      const dictionaryScore = dictionaryResult.containsProfanity ? 1 : 0;
+      const combinedScore = dictionaryScore * dictionaryWeight + mlResult.confidence * mlWeight;
+      const containsProfanity = combinedScore >= mlThreshold;
+      return {
+        containsProfanity,
+        confidence: combinedScore,
+        dictionaryResult,
+        mlResult,
+        usedML: true,
+        profaneWords: dictionaryResult.profaneWords,
+        processingTimeMs: Date.now() - startTime
+      };
+    },
+    /**
+     * Check multiple texts
+     */
+    async checkBatch(texts) {
+      return Promise.all(texts.map((text) => this.check(text)));
+    },
+    /**
+     * Dictionary-only check (fast, no ML)
+     */
+    checkFast(text) {
+      return filter.checkProfanity(text);
+    },
+    /**
+     * ML-only check (slower, more accurate)
+     */
+    async checkML(text) {
+      const ml = await getMLChecker();
+      return ml.check(text);
+    },
+    /**
+     * Get the underlying filter
+     */
+    getFilter() {
+      return filter;
+    },
+    /**
+     * Dispose of resources
+     */
+    async dispose() {
+      if (mlChecker) {
+        mlChecker.dispose();
+        mlChecker = null;
+      }
+    }
+  };
+}
+async function isTransformersAvailable() {
+  try {
+    await getTransformers();
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function preloadModel(model = RECOMMENDED_MODELS.pardonmyai, options = {}) {
+  const { quantized = true } = options;
+  const transformers = await getTransformers();
+  await transformers.pipeline("text-classification", model, {
+    quantized
+  });
+}
+
 exports.HybridFilter = HybridFilter;
+exports.RECOMMENDED_MODELS = RECOMMENDED_MODELS;
 exports.ToxicityDetector = ToxicityDetector;
+exports.createHybridChecker = createHybridChecker;
+exports.createMLChecker = createMLChecker;
+exports.isTransformersAvailable = isTransformersAvailable;
+exports.preloadModel = preloadModel;

package/dist/ml/index.d.cts

@@ -1,5 +1,8 @@
-import { T as ToxicityLabel, f as MLDetectorConfig, e as MLAnalysisResult, F as FilterConfig, C as CheckProfanityResult, H as HybridAnalysisResult, a as Filter } from '../types-Dj5vaoch.cjs';
-export { d as ToxicityPrediction } from '../types-Dj5vaoch.cjs';
+import { T as ToxicityLabel, M as MLDetectorConfig, a as MLAnalysisResult, H as HybridAnalysisResult } from '../types-Ct_ueYqw.cjs';
+export { b as ToxicityPrediction } from '../types-Ct_ueYqw.cjs';
+import { F as Filter } from '../Filter-D34Wsmrj.cjs';
+import { F as FilterConfig, C as CheckProfanityResult } from '../types-B9c_ik4k.cjs';
+export { HybridCheckResult, HybridCheckerConfig, MLCheckResult, MLCheckerConfig, RECOMMENDED_MODELS, createHybridChecker, createMLChecker, isTransformersAvailable, preloadModel } from './transformers.cjs';
 
 /**
  * ML-based toxicity detection using TensorFlow.js.

package/dist/ml/index.d.ts

@@ -1,5 +1,8 @@
-import { T as ToxicityLabel, f as MLDetectorConfig, e as MLAnalysisResult, F as FilterConfig, C as CheckProfanityResult, H as HybridAnalysisResult, a as Filter } from '../types-Dj5vaoch.js';
-export { d as ToxicityPrediction } from '../types-Dj5vaoch.js';
+import { T as ToxicityLabel, M as MLDetectorConfig, a as MLAnalysisResult, H as HybridAnalysisResult } from '../types-Ct_ueYqw.js';
+export { b as ToxicityPrediction } from '../types-Ct_ueYqw.js';
+import { F as Filter } from '../Filter-BGcyIAvO.js';
+import { F as FilterConfig, C as CheckProfanityResult } from '../types-B9c_ik4k.js';
+export { HybridCheckResult, HybridCheckerConfig, MLCheckResult, MLCheckerConfig, RECOMMENDED_MODELS, createHybridChecker, createMLChecker, isTransformersAvailable, preloadModel } from './transformers.js';
 
 /**
  * ML-based toxicity detection using TensorFlow.js.

package/dist/ml/index.js

@@ -5576,4 +5576,206 @@ var HybridFilter = class {
   }
 };
 
-export { HybridFilter, ToxicityDetector };
+// src/ml/transformers.ts
+var RECOMMENDED_MODELS = {
+  /** High accuracy English model (97.5%) - 67M params */
+  pardonmyai: "tarekziade/pardonmyai",
+  /** Smaller version for constrained environments */
+  pardonmyaiTiny: "tarekziade/pardonmyai-tiny",
+  /** Multilingual toxicity detection (7 languages) */
+  toxicBert: "unitary/toxic-bert",
+  /** Offensive speech detector (DeBERTa-based) */
+  offensiveSpeech: "KoalaAI/OffensiveSpeechDetector"
+};
+var MODEL_PROFANE_LABELS = {
+  "tarekziade/pardonmyai": "profane",
+  "tarekziade/pardonmyai-tiny": "profane",
+  "unitary/toxic-bert": "toxic",
+  "KoalaAI/OffensiveSpeechDetector": "LABEL_1",
+  // Offensive
+  default: "LABEL_1"
+};
+async function getTransformers() {
+  try {
+    const transformers = await import('@xenova/transformers');
+    return transformers;
+  } catch {
+    throw new Error(
+      "Transformers.js is required for ML features. Install it with: npm install @xenova/transformers"
+    );
+  }
+}
+async function createMLChecker(config = {}) {
+  const {
+    model = RECOMMENDED_MODELS.pardonmyai,
+    threshold = 0.5,
+    profaneLabel = MODEL_PROFANE_LABELS[model] || MODEL_PROFANE_LABELS.default,
+    quantized = true,
+    device = "cpu"
+  } = config;
+  const transformers = await getTransformers();
+  const classifier = await transformers.pipeline("text-classification", model, {
+    quantized,
+    device
+  });
+  return {
+    /**
+     * Check a single text for profanity
+     */
+    async check(text) {
+      const startTime = Date.now();
+      const output = await classifier(text);
+      const processingTimeMs = Date.now() - startTime;
+      const profaneScore = output.find((o) => o.label === profaneLabel)?.score || 0;
+      const containsProfanity = profaneScore >= threshold;
+      return {
+        containsProfanity,
+        confidence: profaneScore,
+        rawOutput: output,
+        processingTimeMs
+      };
+    },
+    /**
+     * Check multiple texts
+     */
+    async checkBatch(texts) {
+      return Promise.all(texts.map((text) => this.check(text)));
+    },
+    /**
+     * Get the profanity score for text (0-1)
+     */
+    async getScore(text) {
+      const result = await this.check(text);
+      return result.confidence;
+    },
+    /**
+     * Get current configuration
+     */
+    getConfig() {
+      return { model, threshold, profaneLabel, quantized, device };
+    },
+    /**
+     * Dispose of the model (free memory)
+     */
+    dispose() {
+    }
+  };
+}
+async function createHybridChecker(config = {}) {
+  const {
+    model = RECOMMENDED_MODELS.pardonmyai,
+    threshold = 0.5,
+    profaneLabel,
+    quantized = true,
+    device = "cpu",
+    filterConfig = {},
+    mlThreshold = 0.3,
+    dictionaryWeight = 0.6,
+    mlWeight = 0.4
+  } = config;
+  const filter = new Filter({
+    languages: filterConfig.languages || ["english"],
+    detectLeetspeak: filterConfig.detectLeetspeak ?? true,
+    normalizeUnicode: filterConfig.normalizeUnicode ?? true,
+    severityLevels: true,
+    cacheResults: true,
+    ...filterConfig
+  });
+  let mlChecker = null;
+  async function getMLChecker() {
+    if (!mlChecker) {
+      mlChecker = await createMLChecker({
+        model,
+        threshold,
+        profaneLabel,
+        quantized,
+        device
+      });
+    }
+    return mlChecker;
+  }
+  return {
+    /**
+     * Check text using hybrid approach
+     */
+    async check(text) {
+      const startTime = Date.now();
+      const dictionaryResult = filter.checkProfanity(text);
+      if (dictionaryResult.containsProfanity) {
+        return {
+          containsProfanity: true,
+          confidence: 1,
+          dictionaryResult,
+          usedML: false,
+          profaneWords: dictionaryResult.profaneWords,
+          processingTimeMs: Date.now() - startTime
+        };
+      }
+      const ml = await getMLChecker();
+      const mlResult = await ml.check(text);
+      const dictionaryScore = dictionaryResult.containsProfanity ? 1 : 0;
+      const combinedScore = dictionaryScore * dictionaryWeight + mlResult.confidence * mlWeight;
+      const containsProfanity = combinedScore >= mlThreshold;
+      return {
+        containsProfanity,
+        confidence: combinedScore,
+        dictionaryResult,
+        mlResult,
+        usedML: true,
+        profaneWords: dictionaryResult.profaneWords,
+        processingTimeMs: Date.now() - startTime
+      };
+    },
+    /**
+     * Check multiple texts
+     */
+    async checkBatch(texts) {
+      return Promise.all(texts.map((text) => this.check(text)));
+    },
+    /**
+     * Dictionary-only check (fast, no ML)
+     */
+    checkFast(text) {
+      return filter.checkProfanity(text);
+    },
+    /**
+     * ML-only check (slower, more accurate)
+     */
+    async checkML(text) {
+      const ml = await getMLChecker();
+      return ml.check(text);
+    },
+    /**
+     * Get the underlying filter
+     */
+    getFilter() {
+      return filter;
+    },
+    /**
+     * Dispose of resources
+     */
+    async dispose() {
+      if (mlChecker) {
+        mlChecker.dispose();
+        mlChecker = null;
+      }
+    }
+  };
+}
+async function isTransformersAvailable() {
+  try {
+    await getTransformers();
+    return true;
+  } catch {
+    return false;
+  }
+}
+async function preloadModel(model = RECOMMENDED_MODELS.pardonmyai, options = {}) {
+  const { quantized = true } = options;
+  const transformers = await getTransformers();
+  await transformers.pipeline("text-classification", model, {
+    quantized
+  });
+}
+
+export { HybridFilter, RECOMMENDED_MODELS, ToxicityDetector, createHybridChecker, createMLChecker, isTransformersAvailable, preloadModel };