bekindprofanityfilter 0.0.1

package/dist/algos/bloom-filter.js

@@ -0,0 +1,208 @@
+/**
+ * Bloom Filter implementation for efficient set membership testing
+ */
+export class BloomFilter {
+    constructor(expectedItems, falsePositiveRate = 0.01) {
+        this.itemCount = 0;
+        // Calculate optimal size and hash count
+        this.size = this.calculateOptimalSize(expectedItems, falsePositiveRate);
+        this.hashCount = this.calculateOptimalHashCount(this.size, expectedItems);
+        this.bitArray = new Uint8Array(Math.ceil(this.size / 8));
+    }
+    /**
+     * Calculate optimal bit array size
+     */
+    calculateOptimalSize(n, p) {
+        return Math.ceil((-n * Math.log(p)) / Math.log(2) ** 2);
+    }
+    /**
+     * Calculate optimal number of hash functions
+     */
+    calculateOptimalHashCount(m, n) {
+        return Math.ceil((m / n) * Math.log(2));
+    }
+    /**
+     * Hash function 1 (FNV-1a variant)
+     */
+    hash1(item) {
+        let hash = 2166136261;
+        for (let i = 0; i < item.length; i++) {
+            hash ^= item.charCodeAt(i);
+            hash *= 16777619;
+        }
+        return Math.abs(hash) % this.size;
+    }
+    /**
+     * Hash function 2 (djb2 variant)
+     */
+    hash2(item) {
+        let hash = 5381;
+        for (let i = 0; i < item.length; i++) {
+            hash = (hash << 5) + hash + item.charCodeAt(i);
+        }
+        return Math.abs(hash) % this.size;
+    }
+    /**
+     * Generate k hash values for an item using double hashing
+     */
+    getHashes(item) {
+        const hash1 = this.hash1(item);
+        const hash2 = this.hash2(item);
+        const hashes = [];
+        for (let i = 0; i < this.hashCount; i++) {
+            const hash = (hash1 + i * hash2) % this.size;
+            hashes.push(Math.abs(hash));
+        }
+        return hashes;
+    }
+    /**
+     * Set a bit in the bit array
+     */
+    setBit(index) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        this.bitArray[byteIndex] |= 1 << bitIndex;
+    }
+    /**
+     * Get a bit from the bit array
+     */
+    getBit(index) {
+        const byteIndex = Math.floor(index / 8);
+        const bitIndex = index % 8;
+        return (this.bitArray[byteIndex] & (1 << bitIndex)) !== 0;
+    }
+    /**
+     * Add an item to the bloom filter
+     */
+    add(item) {
+        const hashes = this.getHashes(item);
+        for (const hash of hashes) {
+            this.setBit(hash);
+        }
+        this.itemCount++;
+    }
+    /**
+     * Add multiple items to the bloom filter
+     */
+    addAll(items) {
+        for (const item of items) {
+            this.add(item);
+        }
+    }
+    /**
+     * Test if an item might be in the set
+     */
+    mightContain(item) {
+        const hashes = this.getHashes(item);
+        for (const hash of hashes) {
+            if (!this.getBit(hash)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Test multiple items at once
+     */
+    mightContainAny(items) {
+        return items.some((item) => this.mightContain(item));
+    }
+    /**
+     * Filter items that might be in the set
+     */
+    filter(items) {
+        return items.filter((item) => this.mightContain(item));
+    }
+    /**
+     * Clear the bloom filter
+     */
+    clear() {
+        this.bitArray.fill(0);
+        this.itemCount = 0;
+    }
+    /**
+     * Get current false positive probability
+     */
+    getCurrentFalsePositiveRate() {
+        const ratio = this.itemCount / this.size;
+        return Math.pow(1 - Math.exp(-this.hashCount * ratio), this.hashCount);
+    }
+    /**
+     * Get bloom filter statistics
+     */
+    getStats() {
+        let bitsSet = 0;
+        for (let i = 0; i < this.size; i++) {
+            if (this.getBit(i)) {
+                bitsSet++;
+            }
+        }
+        const loadFactor = bitsSet / this.size;
+        const estimatedFalsePositiveRate = Math.pow(loadFactor, this.hashCount);
+        return {
+            size: this.size,
+            hashCount: this.hashCount,
+            itemCount: this.itemCount,
+            bitsSet,
+            loadFactor,
+            estimatedFalsePositiveRate,
+        };
+    }
+    /**
+     * Serialize bloom filter to JSON
+     */
+    toJSON() {
+        return {
+            size: this.size,
+            hashCount: this.hashCount,
+            itemCount: this.itemCount,
+            bitArray: Array.from(this.bitArray),
+        };
+    }
+    /**
+     * Deserialize bloom filter from JSON
+     */
+    static fromJSON(data) {
+        const filter = Object.create(BloomFilter.prototype);
+        filter.size = data.size;
+        filter.hashCount = data.hashCount;
+        filter.itemCount = data.itemCount;
+        filter.bitArray = new Uint8Array(data.bitArray);
+        return filter;
+    }
+    /**
+     * Union operation with another bloom filter
+     */
+    union(other) {
+        if (this.size !== other.size || this.hashCount !== other.hashCount) {
+            throw new Error("Bloom filters must have same size and hash count for union operation");
+        }
+        const result = new BloomFilter(1, 0.01);
+        result.size = this.size;
+        result.hashCount = this.hashCount;
+        result.bitArray = new Uint8Array(this.bitArray.length);
+        result.itemCount = this.itemCount + other.itemCount;
+        for (let i = 0; i < this.bitArray.length; i++) {
+            result.bitArray[i] = this.bitArray[i] | other.bitArray[i];
+        }
+        return result;
+    }
+    /**
+     * Intersection operation with another bloom filter
+     */
+    intersect(other) {
+        if (this.size !== other.size || this.hashCount !== other.hashCount) {
+            throw new Error("Bloom filters must have same size and hash count for intersection operation");
+        }
+        const result = new BloomFilter(1, 0.01);
+        result.size = this.size;
+        result.hashCount = this.hashCount;
+        result.bitArray = new Uint8Array(this.bitArray.length);
+        result.itemCount = Math.min(this.itemCount, other.itemCount);
+        for (let i = 0; i < this.bitArray.length; i++) {
+            result.bitArray[i] = this.bitArray[i] & other.bitArray[i];
+        }
+        return result;
+    }
+}
+//# sourceMappingURL=bloom-filter.js.map

package/dist/algos/bloom-filter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bloom-filter.js","sourceRoot":"","sources":["../../src/algos/bloom-filter.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,OAAO,WAAW;IAMtB,YAAY,aAAqB,EAAE,oBAA4B,IAAI;QAF3D,cAAS,GAAW,CAAC,CAAC;QAG5B,wCAAwC;QACxC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC;QACxE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC1E,IAAI,CAAC,QAAQ,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,CAAS,EAAE,CAAS;QAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACK,yBAAyB,CAAC,CAAS,EAAE,CAAS;QACpD,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,IAAY;QACxB,IAAI,IAAI,GAAG,UAAU,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,IAAI,QAAQ,CAAC;SAClB;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,IAAY;QACxB,IAAI,IAAI,GAAG,IAAI,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACpC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;SAChD;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IACpC,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,IAAY;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAa,EAAE,CAAC;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;SAC7B;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,KAAa;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;IAC5C,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,KAAa;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,GAAG,CAAC,IAAY;QACd,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SACnB;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAe;QACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;SAChB;IACH,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,IAAY;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;gBACtB,OAAO,KAAK,CAAC;aACd;SACF;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,KAAe;QAC7B,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAe;QACpB,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,2BAA2B;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC;QACzC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACzE,CAAC;IAED;;OAEG;IACH,QAAQ;QAQN,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YAClC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;gBAClB,OAAO,EAAE,CAAC;aACX;SACF;QAED,MAAM,UAAU,GAAG,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC;QACvC,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QAExE,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO;YACP,UAAU;YACV,0BAA0B;SAC3B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM;QAMJ,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;SACpC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,QAAQ,CAAC,IAKf;QACC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QACpD,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAClC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAClC,MAAM,CAAC,QAAQ,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAkB;QACtB,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,EAAE;YAClE,MAAM,IAAI,KAAK,CACb,sEAAsE,CACvE,CAAC;SACH;QAED,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAClC,MAAM,CAAC,QAAQ,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;QAEpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;SAC3D;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,KAAkB;QAC1B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,EAAE;YAClE,MAAM,IAAI,KAAK,CACb,6EAA6E,CAC9E,CAAC;SACH;QAED,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACxC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACxB,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAClC,MAAM,CAAC,QAAQ,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;QAE7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;SAC3D;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/algos/context-patterns.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Universal context patterns for multi-language profanity detection
+ */
+export interface UniversalContextPattern {
+    type: "negation" | "possessive" | "compound" | "proper_noun" | "article" | "quotation" | "medical" | "anatomical" | "sexual_verb_before" | "sexual_verb_after" | "compound_slur" | "insult_construction" | "direct_address" | "pejorative_adj";
+    pattern: RegExp;
+    weight: number;
+    delta: number;
+    languages: string[];
+    description: string;
+    examples: string[];
+}
+export interface ContextRule {
+    pattern: RegExp;
+    action: "reduce_score" | "increase_score" | "whitelist" | "flag";
+    weight: number;
+    delta: number;
+    priority: number;
+}
+/**
+ * Universal context patterns that work across multiple languages
+ */
+export declare const UNIVERSAL_CONTEXT_PATTERNS: UniversalContextPattern[];
+/**
+ * Language-specific context patterns
+ */
+export declare const LANGUAGE_SPECIFIC_PATTERNS: Record<string, UniversalContextPattern[]>;
+/**
+ * Word-specific context patterns for disambiguating ambiguous profane words.
+ * Keyed by the lowercase profane word.
+ */
+export declare const WORD_SPECIFIC_PATTERNS: Record<string, UniversalContextPattern[]>;
+/**
+ * Context rule generator
+ */
+export declare class ContextPatternMatcher {
+    private patterns;
+    private languagePatterns;
+    constructor(languages?: string[]);
+    /**
+     * Generate context rules for a specific word
+     */
+    generateRules(word: string, languages?: string[]): ContextRule[];
+    /**
+     * Get priority for pattern type (reducers before boosters)
+     */
+    private getPriority;
+    /**
+     * Escape regex special characters
+     */
+    private escapeRegex;
+    /**
+     * Add custom pattern
+     */
+    addPattern(pattern: UniversalContextPattern): void;
+    /**
+     * Add language-specific pattern
+     */
+    addLanguagePattern(language: string, pattern: UniversalContextPattern): void;
+    /**
+     * Get all patterns for debugging
+     */
+    getAllPatterns(): {
+        universal: UniversalContextPattern[];
+        languageSpecific: Map<string, UniversalContextPattern[]>;
+    };
+}
+/**
+ * Context analyzer for scoring matches
+ */
+export declare class ContextAnalyzer {
+    private patternMatcher;
+    private contextWindow;
+    constructor(languages?: string[]);
+    /**
+     * Analyze context around a potential profanity match (legacy score-based model)
+     */
+    analyzeContext(text: string, matchStart: number, matchEnd: number, word: string): {
+        score: number;
+        confidence: "high" | "medium" | "low";
+        appliedRules: Array<{
+            rule: ContextRule;
+            matched: boolean;
+        }>;
+        context: string;
+    };
+    /**
+     * Calculate the certainty delta for a word based on surrounding context.
+     * Positive delta = booster (more likely profane).
+     * Negative delta = reducer (more likely innocent).
+     * Returns the sum of all matching pattern deltas.
+     */
+    getCertaintyDelta(text: string, matchStart: number, matchEnd: number, word: string): number;
+    /**
+     * Set context window size
+     */
+    setContextWindow(size: number): void;
+    /**
+     * Add custom pattern to the analyzer
+     */
+    addCustomPattern(pattern: UniversalContextPattern): void;
+}