@shankarkharel/profanity-core 1.0.0

package/dist/index.d.mts

@@ -0,0 +1,79 @@
+type Severity = 1 | 2 | 3 | 4 | 5;
+type MatchKind = "word" | "phrase";
+interface TermEntry {
+    term: string;
+    severity: Severity;
+    category?: string[];
+    match?: MatchKind;
+    variants?: string[];
+}
+interface NormalizerStep {
+    name: string;
+    run: (input: string) => string;
+}
+interface LanguagePack {
+    code: string;
+    version: string;
+    terms: TermEntry[];
+    allowlist?: string[];
+    normalizers?: NormalizerStep[];
+}
+interface MatchDetail {
+    pack: string;
+    term: string;
+    severity: Severity;
+    category: string[];
+    index: number;
+}
+interface AnalyzeResult {
+    profane: boolean;
+    score: number;
+    maxSeverity: Severity | 0;
+    matches: MatchDetail[];
+}
+interface EngineOptions {
+    severityThreshold?: Severity;
+    enabledLanguages?: string[];
+    extraTerms?: TermEntry[];
+    extraAllowlist?: string[];
+    enableRepeatCollapse?: boolean;
+    maxTextLength?: number;
+}
+interface CensorOptions {
+    censorChar?: string;
+    preserveFirstLast?: boolean;
+    replaceWith?: string;
+}
+interface CensorOptions {
+    censorChar?: string;
+    replaceWith?: string;
+    preserveFirstLast?: boolean;
+    preservePrefix?: number;
+    preserveSuffix?: number;
+}
+
+declare class ProfanityEngine {
+    private packs;
+    private opts;
+    constructor(packs: LanguagePack[], options?: EngineOptions);
+    analyze(input: string): AnalyzeResult;
+    isProfane(text: string): boolean;
+    censor(text: string, options?: {
+        censorChar?: string;
+        replaceWith?: string;
+        preserveFirstLast?: boolean;
+        preservePrefix?: number;
+        preserveSuffix?: number;
+    }): string;
+    private expandTerms;
+    private normalizeForPack;
+}
+
+declare const nfkc: NormalizerStep;
+declare const lower: NormalizerStep;
+declare const collapseWhitespace: NormalizerStep;
+declare const stripPunctuation: NormalizerStep;
+declare function collapseRepeats(maxRepeats?: number): NormalizerStep;
+declare const leetspeak: NormalizerStep;
+
+export { type AnalyzeResult, type CensorOptions, type EngineOptions, type LanguagePack, type MatchDetail, type MatchKind, type NormalizerStep, ProfanityEngine, type Severity, type TermEntry, collapseRepeats, collapseWhitespace, leetspeak, lower, nfkc, stripPunctuation };

package/dist/index.d.ts

@@ -0,0 +1,79 @@
+type Severity = 1 | 2 | 3 | 4 | 5;
+type MatchKind = "word" | "phrase";
+interface TermEntry {
+    term: string;
+    severity: Severity;
+    category?: string[];
+    match?: MatchKind;
+    variants?: string[];
+}
+interface NormalizerStep {
+    name: string;
+    run: (input: string) => string;
+}
+interface LanguagePack {
+    code: string;
+    version: string;
+    terms: TermEntry[];
+    allowlist?: string[];
+    normalizers?: NormalizerStep[];
+}
+interface MatchDetail {
+    pack: string;
+    term: string;
+    severity: Severity;
+    category: string[];
+    index: number;
+}
+interface AnalyzeResult {
+    profane: boolean;
+    score: number;
+    maxSeverity: Severity | 0;
+    matches: MatchDetail[];
+}
+interface EngineOptions {
+    severityThreshold?: Severity;
+    enabledLanguages?: string[];
+    extraTerms?: TermEntry[];
+    extraAllowlist?: string[];
+    enableRepeatCollapse?: boolean;
+    maxTextLength?: number;
+}
+interface CensorOptions {
+    censorChar?: string;
+    preserveFirstLast?: boolean;
+    replaceWith?: string;
+}
+interface CensorOptions {
+    censorChar?: string;
+    replaceWith?: string;
+    preserveFirstLast?: boolean;
+    preservePrefix?: number;
+    preserveSuffix?: number;
+}
+
+declare class ProfanityEngine {
+    private packs;
+    private opts;
+    constructor(packs: LanguagePack[], options?: EngineOptions);
+    analyze(input: string): AnalyzeResult;
+    isProfane(text: string): boolean;
+    censor(text: string, options?: {
+        censorChar?: string;
+        replaceWith?: string;
+        preserveFirstLast?: boolean;
+        preservePrefix?: number;
+        preserveSuffix?: number;
+    }): string;
+    private expandTerms;
+    private normalizeForPack;
+}
+
+declare const nfkc: NormalizerStep;
+declare const lower: NormalizerStep;
+declare const collapseWhitespace: NormalizerStep;
+declare const stripPunctuation: NormalizerStep;
+declare function collapseRepeats(maxRepeats?: number): NormalizerStep;
+declare const leetspeak: NormalizerStep;
+
+export { type AnalyzeResult, type CensorOptions, type EngineOptions, type LanguagePack, type MatchDetail, type MatchKind, type NormalizerStep, ProfanityEngine, type Severity, type TermEntry, collapseRepeats, collapseWhitespace, leetspeak, lower, nfkc, stripPunctuation };

package/dist/index.js

@@ -0,0 +1,227 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  ProfanityEngine: () => ProfanityEngine,
+  collapseRepeats: () => collapseRepeats,
+  collapseWhitespace: () => collapseWhitespace,
+  leetspeak: () => leetspeak,
+  lower: () => lower,
+  nfkc: () => nfkc,
+  stripPunctuation: () => stripPunctuation
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/tokenize.ts
+function tokenizeWords(text) {
+  const tokens = [];
+  let i = 0;
+  while (i < text.length) {
+    while (i < text.length && text[i] === " ") i++;
+    const start = i;
+    while (i < text.length && text[i] !== " ") i++;
+    const end = i;
+    if (end > start) {
+      tokens.push({ value: text.slice(start, end), start, end });
+    }
+  }
+  return tokens;
+}
+
+// src/normalizers.ts
+var nfkc = {
+  name: "nfkc",
+  run: (s) => s.normalize("NFKC")
+};
+var lower = {
+  name: "lower",
+  run: (s) => s.toLowerCase()
+};
+var collapseWhitespace = {
+  name: "collapseWhitespace",
+  run: (s) => s.replace(/\s+/g, " ").trim()
+};
+var stripPunctuation = {
+  name: "stripPunctuation",
+  run: (s) => s.replace(/[^\p{L}\p{N}\u0900-\u097F ]+/gu, " ")
+};
+function collapseRepeats(maxRepeats = 2) {
+  return {
+    name: "collapseRepeats",
+    run: (s) => s.replace(/(.)\1{2,}/g, (_m, ch) => String(ch).repeat(maxRepeats))
+  };
+}
+var leetspeak = {
+  name: "leetspeak",
+  run: (s) => s.replace(/[@]/g, "a").replace(/[!]/g, "i").replace(/[0]/g, "o").replace(/[1|]/g, "i").replace(/[$]/g, "s")
+};
+
+// src/engine.ts
+function uniq(arr) {
+  return Array.from(new Set(arr));
+}
+function clampText(text, max) {
+  if (text.length <= max) return text;
+  return text.slice(0, max);
+}
+var ProfanityEngine = class {
+  constructor(packs, options) {
+    this.packs = [];
+    this.packs = packs;
+    this.opts = {
+      severityThreshold: options?.severityThreshold ?? 1,
+      enabledLanguages: options?.enabledLanguages ?? [],
+      extraTerms: options?.extraTerms ?? [],
+      extraAllowlist: options?.extraAllowlist ?? [],
+      enableRepeatCollapse: options?.enableRepeatCollapse ?? true,
+      maxTextLength: options?.maxTextLength ?? 2e4
+    };
+  }
+  analyze(input) {
+    const text = clampText(input ?? "", this.opts.maxTextLength);
+    const enabled = this.opts.enabledLanguages.length ? this.packs.filter((p) => this.opts.enabledLanguages.includes(p.code)) : this.packs;
+    const matches = [];
+    for (const pack of enabled) {
+      const normalized = this.normalizeForPack(text, pack);
+      const allow = new Set(uniq([...pack.allowlist ?? [], ...this.opts.extraAllowlist]).map((x) => this.normalizeForPack(x, pack)));
+      const allTerms = [...pack.terms, ...this.opts.extraTerms];
+      const expanded = this.expandTerms(allTerms);
+      const tokens = tokenizeWords(normalized);
+      const wordIndex = /* @__PURE__ */ new Map();
+      for (const t of expanded.filter((x) => (x.match ?? "word") === "word")) {
+        const key = this.normalizeForPack(t.term, pack);
+        if (!wordIndex.has(key)) wordIndex.set(key, []);
+        wordIndex.get(key).push(t);
+      }
+      for (const tok of tokens) {
+        if (allow.has(tok.value)) continue;
+        const hit = wordIndex.get(tok.value);
+        if (hit) {
+          for (const h of hit) {
+            if (h.severity < this.opts.severityThreshold) continue;
+            matches.push({
+              pack: pack.code,
+              term: h.term,
+              severity: h.severity,
+              category: h.category ?? [],
+              index: tok.start
+            });
+          }
+        }
+      }
+      const phrases = expanded.filter((x) => (x.match ?? "word") === "phrase");
+      for (const ph of phrases) {
+        const needle = this.normalizeForPack(ph.term, pack);
+        if (!needle || allow.has(needle)) continue;
+        let idx = normalized.indexOf(needle);
+        while (idx !== -1) {
+          if (ph.severity >= this.opts.severityThreshold) {
+            matches.push({
+              pack: pack.code,
+              term: ph.term,
+              severity: ph.severity,
+              category: ph.category ?? [],
+              index: idx
+            });
+          }
+          idx = normalized.indexOf(needle, idx + needle.length);
+        }
+      }
+    }
+    const maxSeverity = matches.length ? matches.reduce((m, x) => x.severity > m ? x.severity : m, 0) : 0;
+    const score = matches.length ? Math.min(100, matches.length * 15 + maxSeverity * 10) : 0;
+    return {
+      profane: matches.length > 0,
+      score,
+      maxSeverity,
+      matches
+    };
+  }
+  isProfane(text) {
+    return this.analyze(text).profane;
+  }
+  censor(text, options) {
+    const result = this.analyze(text);
+    if (!result.profane) return text;
+    const censorChar = options?.censorChar ?? "*";
+    const replaceWith = options?.replaceWith;
+    const preservePrefix = options?.preservePrefix ?? (options?.preserveFirstLast ? 1 : 0);
+    const preserveSuffix = options?.preserveSuffix ?? (options?.preserveFirstLast ? 1 : 0);
+    const makeMasked = (term) => {
+      if (replaceWith) return replaceWith;
+      const len = term.length;
+      const pre = Math.max(0, Math.min(preservePrefix, len));
+      const suf = Math.max(0, Math.min(preserveSuffix, len - pre));
+      let mid = Math.max(0, len - pre - suf);
+      if (len >= 2 && mid === 0) {
+        const safePre = Math.max(0, Math.min(pre, len - 1));
+        return term.slice(0, safePre) + censorChar + term.slice(safePre + 1);
+      }
+      return term.slice(0, pre) + censorChar.repeat(mid) + term.slice(len - suf);
+    };
+    let out = text;
+    for (const m of result.matches) {
+      const term = m.term;
+      if (!term) continue;
+      const replacement = makeMasked(term);
+      const re = new RegExp(escapeRegExp(term), "gi");
+      out = out.replace(re, replacement);
+    }
+    return out;
+  }
+  expandTerms(terms) {
+    const out = [];
+    for (const t of terms) {
+      out.push(t);
+      for (const v of t.variants ?? []) {
+        out.push({ ...t, term: v });
+      }
+    }
+    return out;
+  }
+  normalizeForPack(text, pack) {
+    const steps = [
+      nfkc,
+      lower,
+      ...pack.code === "en" ? [leetspeak] : [],
+      ...this.opts.enableRepeatCollapse ? [collapseRepeats(2)] : [],
+      stripPunctuation,
+      collapseWhitespace,
+      ...pack.normalizers ?? []
+    ];
+    let s = text;
+    for (const step of steps) s = step.run(s);
+    return s;
+  }
+};
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ProfanityEngine,
+  collapseRepeats,
+  collapseWhitespace,
+  leetspeak,
+  lower,
+  nfkc,
+  stripPunctuation
+});

package/dist/index.mjs

@@ -0,0 +1,194 @@
+// src/tokenize.ts
+function tokenizeWords(text) {
+  const tokens = [];
+  let i = 0;
+  while (i < text.length) {
+    while (i < text.length && text[i] === " ") i++;
+    const start = i;
+    while (i < text.length && text[i] !== " ") i++;
+    const end = i;
+    if (end > start) {
+      tokens.push({ value: text.slice(start, end), start, end });
+    }
+  }
+  return tokens;
+}
+
+// src/normalizers.ts
+var nfkc = {
+  name: "nfkc",
+  run: (s) => s.normalize("NFKC")
+};
+var lower = {
+  name: "lower",
+  run: (s) => s.toLowerCase()
+};
+var collapseWhitespace = {
+  name: "collapseWhitespace",
+  run: (s) => s.replace(/\s+/g, " ").trim()
+};
+var stripPunctuation = {
+  name: "stripPunctuation",
+  run: (s) => s.replace(/[^\p{L}\p{N}\u0900-\u097F ]+/gu, " ")
+};
+function collapseRepeats(maxRepeats = 2) {
+  return {
+    name: "collapseRepeats",
+    run: (s) => s.replace(/(.)\1{2,}/g, (_m, ch) => String(ch).repeat(maxRepeats))
+  };
+}
+var leetspeak = {
+  name: "leetspeak",
+  run: (s) => s.replace(/[@]/g, "a").replace(/[!]/g, "i").replace(/[0]/g, "o").replace(/[1|]/g, "i").replace(/[$]/g, "s")
+};
+
+// src/engine.ts
+function uniq(arr) {
+  return Array.from(new Set(arr));
+}
+function clampText(text, max) {
+  if (text.length <= max) return text;
+  return text.slice(0, max);
+}
+var ProfanityEngine = class {
+  constructor(packs, options) {
+    this.packs = [];
+    this.packs = packs;
+    this.opts = {
+      severityThreshold: options?.severityThreshold ?? 1,
+      enabledLanguages: options?.enabledLanguages ?? [],
+      extraTerms: options?.extraTerms ?? [],
+      extraAllowlist: options?.extraAllowlist ?? [],
+      enableRepeatCollapse: options?.enableRepeatCollapse ?? true,
+      maxTextLength: options?.maxTextLength ?? 2e4
+    };
+  }
+  analyze(input) {
+    const text = clampText(input ?? "", this.opts.maxTextLength);
+    const enabled = this.opts.enabledLanguages.length ? this.packs.filter((p) => this.opts.enabledLanguages.includes(p.code)) : this.packs;
+    const matches = [];
+    for (const pack of enabled) {
+      const normalized = this.normalizeForPack(text, pack);
+      const allow = new Set(uniq([...pack.allowlist ?? [], ...this.opts.extraAllowlist]).map((x) => this.normalizeForPack(x, pack)));
+      const allTerms = [...pack.terms, ...this.opts.extraTerms];
+      const expanded = this.expandTerms(allTerms);
+      const tokens = tokenizeWords(normalized);
+      const wordIndex = /* @__PURE__ */ new Map();
+      for (const t of expanded.filter((x) => (x.match ?? "word") === "word")) {
+        const key = this.normalizeForPack(t.term, pack);
+        if (!wordIndex.has(key)) wordIndex.set(key, []);
+        wordIndex.get(key).push(t);
+      }
+      for (const tok of tokens) {
+        if (allow.has(tok.value)) continue;
+        const hit = wordIndex.get(tok.value);
+        if (hit) {
+          for (const h of hit) {
+            if (h.severity < this.opts.severityThreshold) continue;
+            matches.push({
+              pack: pack.code,
+              term: h.term,
+              severity: h.severity,
+              category: h.category ?? [],
+              index: tok.start
+            });
+          }
+        }
+      }
+      const phrases = expanded.filter((x) => (x.match ?? "word") === "phrase");
+      for (const ph of phrases) {
+        const needle = this.normalizeForPack(ph.term, pack);
+        if (!needle || allow.has(needle)) continue;
+        let idx = normalized.indexOf(needle);
+        while (idx !== -1) {
+          if (ph.severity >= this.opts.severityThreshold) {
+            matches.push({
+              pack: pack.code,
+              term: ph.term,
+              severity: ph.severity,
+              category: ph.category ?? [],
+              index: idx
+            });
+          }
+          idx = normalized.indexOf(needle, idx + needle.length);
+        }
+      }
+    }
+    const maxSeverity = matches.length ? matches.reduce((m, x) => x.severity > m ? x.severity : m, 0) : 0;
+    const score = matches.length ? Math.min(100, matches.length * 15 + maxSeverity * 10) : 0;
+    return {
+      profane: matches.length > 0,
+      score,
+      maxSeverity,
+      matches
+    };
+  }
+  isProfane(text) {
+    return this.analyze(text).profane;
+  }
+  censor(text, options) {
+    const result = this.analyze(text);
+    if (!result.profane) return text;
+    const censorChar = options?.censorChar ?? "*";
+    const replaceWith = options?.replaceWith;
+    const preservePrefix = options?.preservePrefix ?? (options?.preserveFirstLast ? 1 : 0);
+    const preserveSuffix = options?.preserveSuffix ?? (options?.preserveFirstLast ? 1 : 0);
+    const makeMasked = (term) => {
+      if (replaceWith) return replaceWith;
+      const len = term.length;
+      const pre = Math.max(0, Math.min(preservePrefix, len));
+      const suf = Math.max(0, Math.min(preserveSuffix, len - pre));
+      let mid = Math.max(0, len - pre - suf);
+      if (len >= 2 && mid === 0) {
+        const safePre = Math.max(0, Math.min(pre, len - 1));
+        return term.slice(0, safePre) + censorChar + term.slice(safePre + 1);
+      }
+      return term.slice(0, pre) + censorChar.repeat(mid) + term.slice(len - suf);
+    };
+    let out = text;
+    for (const m of result.matches) {
+      const term = m.term;
+      if (!term) continue;
+      const replacement = makeMasked(term);
+      const re = new RegExp(escapeRegExp(term), "gi");
+      out = out.replace(re, replacement);
+    }
+    return out;
+  }
+  expandTerms(terms) {
+    const out = [];
+    for (const t of terms) {
+      out.push(t);
+      for (const v of t.variants ?? []) {
+        out.push({ ...t, term: v });
+      }
+    }
+    return out;
+  }
+  normalizeForPack(text, pack) {
+    const steps = [
+      nfkc,
+      lower,
+      ...pack.code === "en" ? [leetspeak] : [],
+      ...this.opts.enableRepeatCollapse ? [collapseRepeats(2)] : [],
+      stripPunctuation,
+      collapseWhitespace,
+      ...pack.normalizers ?? []
+    ];
+    let s = text;
+    for (const step of steps) s = step.run(s);
+    return s;
+  }
+};
+function escapeRegExp(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+export {
+  ProfanityEngine,
+  collapseRepeats,
+  collapseWhitespace,
+  leetspeak,
+  lower,
+  nfkc,
+  stripPunctuation
+};

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@shankarkharel/profanity-core",
+  "version": "1.0.0",
+  "description": "Core profanity moderation engine (language pack based).",
+  "main": "dist/index.cjs",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format cjs,esm --dts --clean",
+    "test": "yarn build && node --test dist",
+    "lint": "echo \"(add eslint later)\""
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {},
+  "license": "MIT"
+}