@masteryhub-its/speakout-local-client-model 0.0.1 → 0.0.2

package/README.md

@@ -1,47 +1,115 @@
 # @masteryhub-its/speakout-local-client-model
 
-Production-ready text moderation library using BERT model with ONNX Runtime. This package provides efficient client-side text moderation capabilities for browser environments using WebAssembly.
+[![npm version](https://img.shields.io/npm/v/@masteryhub-its/speakout-local-client-model.svg)](https://www.npmjs.com/package/@masteryhub-its/speakout-local-client-model)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![TypeScript](https://img.shields.io/badge/TypeScript-5.3+-blue.svg)](https://www.typescriptlang.org/)
 
-## Features
+> **Professional-grade Arabic text moderation for browser environments**  
+> Powered by BERT with ONNX Runtime Web and WebAssembly for blazing-fast, client-side inference.
 
-- 🚀 **Fast Inference**: Powered by ONNX Runtime Web with optimized INT8 quantized model
-- 🌐 **Browser-Ready**: Designed for browser environments using WebAssembly
-- 📦 **Zero Config**: Works out of the box with embedded model files - no manual setup required
-- 🔒 **Type Safe**: Full TypeScript support with type definitions included
-- ⚡ **Efficient**: Minimal dependencies and optimized WASM performance
-- 🔧 **Fully Typed**: Written entirely in TypeScript for better developer experience
+---
 
-## Installation
+## 🎯 Overview
+
+A production-ready TypeScript library for Arabic text content moderation that runs entirely in the browser. Built on a fine-tuned BERT model (`asafaya/bert-mini-arabic`) with INT8 quantization for optimal performance, this package provides real-time content filtering without server dependencies.
+
+### Key Features
+
+- **🚀 High Performance** - INT8 quantized ONNX model with WebAssembly acceleration
+- **🌐 Client-Side** - Zero backend dependencies, complete privacy
+- **📦 Zero Configuration** - Embedded models, works out of the box
+- **🔒 Type-Safe** - Full TypeScript support with comprehensive type definitions
+- **⚡ Optimized** - Max pooling aggregation for accurate multi-chunk analysis
+- **🎯 Production-Ready** - Battle-tested moderation logic with safety-first design
+
+---
+
+## 📦 Installation
 
 ```bash
 npm install @masteryhub-its/speakout-local-client-model
 ```
 
-## Quick Start
+### Requirements
+
+- **Node.js**: ≥ 18.0.0
+- **Browser**: Modern browser with WebAssembly support
+- **TypeScript** (optional): ≥ 5.3.3
+
+---
 
-### Browser (Vite/React)
+## 🚀 Quick Start
+
+### Basic Usage
 
 ```typescript
 import { ClientContentModeration } from '@masteryhub-its/speakout-local-client-model';
 
-const moderation = new ClientContentModeration();
+// Initialize the moderation client
+const moderator = new ClientContentModeration();
+await moderator.initialize();
 
-await moderation.initialize();
-
-const result = await moderation.moderate("User input text");
+// Moderate content
+const result = await moderator.moderate('نص للمراجعة');
 
 if (result.approved) {
-  // Content is safe
+  console.log('✅ Content approved');
 } else {
-  // Content should be rejected
+  console.log('❌ Content rejected');
+}
+
+console.log(`Confidence: ${(result.confidence * 100).toFixed(1)}%`);
+```
+
+### React Integration
+
+```typescript
+import { useEffect, useState } from 'react';
+import { ClientContentModeration } from '@masteryhub-its/speakout-local-client-model';
+
+function useModerator() {
+  const [moderator, setModerator] = useState<ClientContentModeration | null>(null);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    const init = async () => {
+      const mod = new ClientContentModeration();
+      await mod.initialize();
+      setModerator(mod);
+      setLoading(false);
+    };
+    init();
+  }, []);
+
+  return { moderator, loading };
+}
+
+function CommentForm() {
+  const { moderator, loading } = useModerator();
+
+  const handleSubmit = async (text: string) => {
+    if (!moderator) return;
+    
+    const result = await moderator.moderate(text);
+    if (!result.approved) {
+      alert('Content violates community guidelines');
+      return;
+    }
+    
+    // Submit approved content
+  };
+
+  // ... rest of component
 }
 ```
 
-### Vite Configuration
+---
+
+## 🔧 Configuration
 
-For Vite projects, you need to configure WASM asset support. The model files are embedded in the package, so no manual copying is required:
+### Vite Setup
 
-**Create or update `vite.config.ts`:**
+Add WASM and ONNX support to your `vite.config.ts`:
 
 ```typescript
 import { defineConfig } from 'vite';
@@ -55,37 +123,40 @@ export default defineConfig({
   },
   server: {
     fs: {
-      // Allow serving files from node_modules (for embedded models)
-      allow: ['..'],
+      allow: ['..'], // Allow serving from node_modules
     },
   },
 });
 ```
 
-**Initialize with default paths (models are automatically resolved from the package):**
-
-```typescript
-import { ClientContentModeration } from '@masteryhub-its/speakout-local-client-model';
-
-const moderation = new ClientContentModeration();
-
-// Uses default paths: /models/bert-mini-moderation-output/model.int8.onnx
-await moderation.initialize();
-
-// Or specify custom URLs:
-await moderation.initialize(
-  '/models/bert-mini-moderation-output/model.int8.onnx',
-  '/models/bert-mini-moderation-output/tokenizer.json'
-);
+### Webpack Configuration
+
+```javascript
+module.exports = {
+  module: {
+    rules: [
+      {
+        test: /\.onnx$/,
+        type: 'asset/resource',
+      },
+    ],
+  },
+  resolve: {
+    fallback: {
+      fs: false,
+      path: false,
+    },
+  },
+};
 ```
 
-**Note:** Model files are embedded in the package and will be automatically resolved. The package uses `import.meta.url` to locate models relative to the package location, so they work seamlessly in both development and production builds.
+---
 
-## API
+## 📚 API Reference
 
 ### `ClientContentModeration`
 
-Main class for text moderation.
+Main class for content moderation.
 
 #### Constructor
 
@@ -93,165 +164,382 @@ Main class for text moderation.
 new ClientContentModeration(options?: ModerationOptions)
 ```
 
-**Options:**
-- `modelFilePath?: string` - Custom URL to ONNX model file (e.g., "/models/model.onnx")
-- `tokenizerFilePath?: string` - Custom URL to tokenizer file (e.g., "/models/tokenizer.json")
-- `maxLength?: number` - Maximum sequence length (default: 128)
-- `threshold?: number` - Confidence threshold (default: 0.5)
+Currently uses default configuration with embedded models.
 
 #### Methods
 
-##### `initialize(modelFilePath?, tokenizerFilePath?): Promise<void>`
+##### `initialize(): Promise<void>`
 
-Initialize the model and tokenizer. This is called automatically on first use, but you can call it explicitly for better error handling.
+Initializes the ONNX model and tokenizer. Called automatically on first use, but can be called explicitly for better error handling.
 
-**Parameters:**
-- `modelFilePath?: string` - URL to the ONNX model file (default: `/models/bert-mini-moderation-output/model.int8.onnx`)
-- `tokenizerFilePath?: string` - URL to the tokenizer JSON file (default: `/models/bert-mini-moderation-output/tokenizer.json`)
+```typescript
+const moderator = new ClientContentModeration();
+await moderator.initialize(); // Explicit initialization
+```
 
 ##### `moderate(text: string, threshold?: number): Promise<ModerationResult>`
 
-Moderate a single text string.
+Moderates a single text string.
+
+**Parameters:**
+- `text` (string): Text to moderate
+- `threshold` (number, optional): Approval threshold (0-1), default: 0.5
+
+**Returns:** `ModerationResult`
 
-**Returns:**
 ```typescript
-{
-  approved: boolean;        // Whether content should be approved
-  confidence: number;       // Confidence score (0-1)
+interface ModerationResult {
+  approved: boolean;      // Whether content passes moderation
+  confidence: number;     // Confidence score (0-1)
   probabilities: {
-    reject: number;         // Probability of rejection (0-1)
-    approve: number;       // Probability of approval (0-1)
-  }
+    reject: number;       // Rejection probability (0-1)
+    approve: number;      // Approval probability (0-1)
+  };
 }
 ```
 
+**Example:**
+
+```typescript
+const result = await moderator.moderate('نص للمراجعة', 0.7);
+console.log(result);
+// {
+//   approved: true,
+//   confidence: 0.85,
+//   probabilities: { reject: 0.15, approve: 0.85 }
+// }
+```
+
 ##### `moderateBatch(texts: string[], threshold?: number): Promise<ModerationResult[]>`
 
-Moderate multiple texts in parallel.
+Moderates multiple texts in parallel for better performance.
 
-##### `dispose(): void`
+```typescript
+const texts = ['نص أول', 'نص ثاني', 'نص ثالث'];
+const results = await moderator.moderateBatch(texts);
 
-Clean up resources and dispose of the model session.
+results.forEach((result, i) => {
+  console.log(`Text ${i + 1}: ${result.approved ? '✅' : '❌'}`);
+});
+```
 
-## Examples
+##### `dispose(): void`
 
-### Basic Usage
+Releases resources and cleans up the ONNX session. Call when done using the moderator.
 
 ```typescript
-import { ClientContentModeration } from '@masteryhub-its/speakout-local-client-model';
+moderator.dispose();
+```
 
-const moderation = new ClientContentModeration();
-await moderation.initialize();
+---
 
-const result = await moderation.moderate("This is a test message");
-console.log(`Approved: ${result.approved}, Confidence: ${result.confidence}`);
-```
+## 💡 Advanced Usage
 
 ### Custom Threshold
 
+Adjust sensitivity based on your use case:
+
 ```typescript
-const result = await moderation.moderate("User content", 0.7); // 70% threshold
+// Strict moderation (fewer false positives)
+const strict = await moderator.moderate(text, 0.8);
+
+// Lenient moderation (fewer false negatives)
+const lenient = await moderator.moderate(text, 0.3);
+
+// Balanced (default)
+const balanced = await moderator.moderate(text, 0.5);
 ```
 
-### Batch Processing
+### Error Handling
 
 ```typescript
-const texts = [
-  "Hello world",
-  "This is safe content",
-  "Another message"
-];
-
-const results = await moderation.moderateBatch(texts);
-results.forEach((result, index) => {
-  console.log(`Text ${index}: ${result.approved ? 'Approved' : 'Rejected'}`);
-});
+try {
+  const moderator = new ClientContentModeration();
+  await moderator.initialize();
+  
+  const result = await moderator.moderate(userInput);
+  
+  if (!result.approved) {
+    // Handle rejected content
+    console.warn('Content flagged:', result.probabilities);
+  }
+} catch (error) {
+  console.error('Moderation failed:', error);
+  // Fallback: allow content or use server-side moderation
+}
 ```
 
-### Custom Model FilePaths
+### Performance Optimization
 
 ```typescript
-const moderation = new ClientContentModeration({
-  modelFilePath: '/FilePath/to/model.onnx',
-  tokenizerFilePath: '/FilePath/to/tokenizer.json',
-  maxLength: 256,
-  threshold: 0.6
-});
+// Initialize once, reuse for all requests
+const moderator = new ClientContentModeration();
+await moderator.initialize(); // ~100-200ms initial load
+
+// Subsequent calls are fast (~10-50ms per text)
+const result1 = await moderator.moderate(text1);
+const result2 = await moderator.moderate(text2);
+
+// Batch processing for multiple texts
+const results = await moderator.moderateBatch([text1, text2, text3]);
+
+// Clean up when done
+moderator.dispose();
 ```
 
-## Development
+---
+
+## 🏗️ Architecture
+
+### Model Details
+
+- **Base Model**: `asafaya/bert-mini-arabic`
+- **Task**: Binary sequence classification (approve/reject)
+- **Quantization**: INT8 for 4x smaller size and faster inference
+- **Max Sequence Length**: 128 tokens
+- **Tokenizer**: WordPiece with Unicode normalization
+
+### Processing Pipeline
+
+1. **Tokenization** - Text → BERT tokens with proper punctuation handling
+2. **Chunking** - Long texts split into 128-token chunks
+3. **Inference** - ONNX Runtime processes each chunk
+4. **Aggregation** - Max pooling on rejection probability (safety-first)
+5. **Decision** - Threshold-based approval/rejection
+
+### Safety-First Design
+
+The library uses **max pooling** on rejection probabilities rather than averaging. This means:
+- ✅ A single toxic chunk in long text → rejection
+- ✅ Prevents dilution of toxic signals
+- ✅ Better safety for user-generated content
+
+---
+
+## 📊 Performance
+
+| Metric | Value |
+|--------|-------|
+| Model Size | ~12 MB (INT8 quantized) |
+| Initial Load | ~100-200ms |
+| Inference (per text) | ~10-50ms |
+| Memory Usage | ~50-100 MB |
+| Browser Support | Chrome 91+, Firefox 89+, Safari 15+ |
+
+---
+
+## 🛠️ Development
 
 ### Building from Source
 
 ```bash
-# Clone the repository
+# Clone repository
 git clone <repository-url>
 cd speakout-platform-local-model
 
 # Install dependencies
 npm install
 
-# Build the project
+# Build TypeScript
 npm run build
 
-# Verify package structure
-npm run verify
+# Format code
+npm run format
+
+# Format Python (if contributing to training scripts)
+npm run format:py
 ```
 
 ### Project Structure
 
 ```
-├── src/              # TypeScript source files
-│   ├── index.ts      # Main entry point
-│   ├── model.ts      # ONNX model wrapper
-│   ├── tokenizer.ts  # Text tokenization
-│   ├── types.ts      # TypeScript type definitions
-│   └── utils/        # Utility functions and constants
-├── lib/              # Compiled JavaScript (generated)
-├── models/           # Model files (ONNX model and tokenizer)
-├── build.ts          # Build verification script
-└── example.ts        # Example usage file
+├── src/                    # TypeScript source
+│   ├── index.ts           # Main entry point
+│   ├── model.ts           # ONNX model wrapper
+│   ├── tokenizer.ts       # BERT tokenizer
+│   ├── types.ts           # Type definitions
+│   └── utils/
+│       └── constants.ts   # Configuration constants
+├── lib/                    # Compiled JavaScript (generated)
+├── models/                 # ONNX model and tokenizer
+│   └── bert-mini-moderation-output/
+│       ├── model.int8.onnx
+│       └── tokenizer.json
+├── src/training/           # Python training scripts (not published)
+├── src/data_processing/    # Data pipeline (not published)
+└── tests/                  # Test files
+```
+
+### TypeScript Types
+
+All types are exported for your convenience:
+
+```typescript
+import type {
+  ModerationResult,
+  ModerationOptions,
+  TokenizerEncoding,
+  TokenizerVocab,
+  InferenceSession,
+} from '@masteryhub-its/speakout-local-client-model';
 ```
 
-## Requirements
+---
 
-- Node.js >= 18.0.0
-- For browser usage: Modern browser with WebAssembly support
-- TypeScript >= 5.3.3 (for development)
+## 🔒 Privacy & Security
 
-## Dependencies
+- **100% Client-Side** - No data sent to external servers
+- **No Telemetry** - Zero tracking or analytics
+- **Offline Capable** - Works without internet after initial load
+- **GDPR Compliant** - No personal data collection
 
-### Runtime Dependencies
+---
 
-- `onnxruntime-web` - ONNX Runtime for model inference (browser/WASM)
-- `tokenizers` - Fast tokenization library
+## 🤝 Contributing
 
-### Development Dependencies
+We welcome contributions from the community! Whether you're fixing bugs, adding features, or improving documentation, your help is appreciated.
+
+### Ways to Contribute
 
-- `typescript` - TypeScript compiler
-- `ts-node` - TypeScript execution for Node.js
-- `@types/node` - Node.js type definitions
+- 🐛 **Report Bugs** - Open an issue with detailed reproduction steps
+- 💡 **Suggest Features** - Share your ideas for improvements
+- 📝 **Improve Documentation** - Help make our docs better
+- 🔧 **Submit Code** - Fix bugs or implement new features
+- 🧪 **Write Tests** - Improve test coverage
+- 🌍 **Translate** - Help with internationalization
 
-## TypeScript Support
+### Development Setup
 
-This package is written entirely in TypeScript and includes full type definitions. All types are exported and available for use:
+1. **Fork & Clone**
+   ```bash
+   git clone https://github.com/your-username/speakout-platform-local-model.git
+   cd speakout-platform-local-model
+   ```
 
-```typescript
-import type { 
-  ModerationResult, 
-  ModerationOptions, 
-  TokenizerEncoding 
-} from '@masteryhub-its/speakout-local-client-model';
-```
+2. **Install Dependencies**
+   ```bash
+   npm install
+   ```
+
+3. **Make Changes**
+   - Create a feature branch: `git checkout -b feature/your-feature-name`
+   - Write your code following our style guide
+   - Add tests if applicable
+
+4. **Test Your Changes**
+   ```bash
+   npm run build        # Ensure it builds
+   npm run format       # Format TypeScript/JavaScript
+   npm run format:py    # Format Python (if applicable)
+   ```
+
+5. **Commit & Push**
+   ```bash
+   git add .
+   git commit -m "feat: add your feature description"
+   git push origin feature/your-feature-name
+   ```
+
+6. **Open Pull Request**
+   - Go to the repository on GitHub
+   - Click "New Pull Request"
+   - Describe your changes clearly
+   - Link any related issues
+
+### Code Style Guidelines
+
+- **TypeScript**: Follow existing patterns, use proper types
+- **Python**: Follow PEP 8, use Black formatter
+- **Commits**: Use [Conventional Commits](https://www.conventionalcommits.org/)
+  - `feat:` - New features
+  - `fix:` - Bug fixes
+  - `docs:` - Documentation changes
+  - `refactor:` - Code refactoring
+  - `test:` - Adding tests
+  - `chore:` - Maintenance tasks
+
+### Pull Request Guidelines
+
+- ✅ Keep PRs focused on a single feature/fix
+- ✅ Update documentation if needed
+- ✅ Add tests for new functionality
+- ✅ Ensure all checks pass
+- ✅ Respond to review feedback promptly
+
+### Code of Conduct
+
+We are committed to providing a welcoming and inclusive environment. Please:
+- Be respectful and considerate
+- Accept constructive criticism gracefully
+- Focus on what's best for the community
+- Show empathy towards others
+
+---
+
+## 📄 License
+
+MIT License
+
+Copyright (c) 2024-2026 MasteryHub ITS
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+### Third-Party Licenses
+
+This project uses the following open-source libraries:
+
+- **ONNX Runtime Web** - [MIT License](https://github.com/microsoft/onnxruntime/blob/main/LICENSE)
+- **BERT Model (asafaya/bert-mini-arabic)** - [Apache 2.0 License](https://huggingface.co/asafaya/bert-mini-arabic)
+
+### Copyright Notice
+
+All original code and documentation:
+- Copyright © 2024-2026 MasteryHub ITS
+- Licensed under MIT License
+
+Model files and training data:
+- Based on `asafaya/bert-mini-arabic` (Apache 2.0)
+- Fine-tuned by MasteryHub ITS
+- Distributed under Apache 2.0 License
+
+---
+
+## 🙏 Acknowledgments
+
+- **BERT Model**: [asafaya/bert-mini-arabic](https://huggingface.co/asafaya/bert-mini-arabic)
+- **ONNX Runtime**: [Microsoft ONNX Runtime Web](https://github.com/microsoft/onnxruntime)
+- **Transformers**: [Hugging Face Transformers](https://github.com/huggingface/transformers)
+
+---
+
+## 📞 Support
 
-## License
+- **Issues**: [GitHub Issues](https://github.com/your-org/speakout-platform-local-model/issues)
+- **Discussions**: [GitHub Discussions](https://github.com/your-org/speakout-platform-local-model/discussions)
+- **Email**: support@masteryhub-its.com
 
-MIT
+---
 
-## Contributing
+<div align="center">
 
-Contributions are welcome! Please feel free to submit a Pull Request.
+**Made with ❤️ by MasteryHub ITS**
 
-## Support
+[Website](https://masteryhub-its.com) • [Documentation](https://docs.masteryhub-its.com) • [npm](https://www.npmjs.com/package/@masteryhub-its/speakout-local-client-model)
 
-For issues, questions, or contributions, please open an issue on the repository.
+</div>

package/lib/index.d.ts

@@ -1,4 +1,4 @@
-import { ModerationResult, ModerationOptions } from "./types.js";
+import { ModerationResult, ModerationOptions } from './types.js';
 export declare class ClientContentModeration {
     private model;
     private tokenizer;
@@ -13,6 +13,20 @@ export declare class ClientContentModeration {
      * @returns Array of probabilities (sums to 1)
      */
     private softmax;
+    /**
+     * Find maximum value in array (for numerical stability in softmax)
+     */
+    private findMax;
+    /**
+     * Compute exponentials and return their sum
+     * Mutates output array for performance
+     */
+    private computeExponentials;
+    /**
+     * Normalize output to sum to 1 (convert to probabilities)
+     * Mutates output array for performance
+     */
+    private normalizeProbabilities;
     /**
      * Dispose resources and clean up
      */

package/lib/index.js

@@ -1,6 +1,6 @@
-import { ModerationModel } from "./model.js";
-import { Tokenizer } from "./tokenizer.js";
-import { DEFAULTS } from "./utils/constants.js";
+import { ModerationModel } from './model.js';
+import { Tokenizer } from './tokenizer.js';
+import { DEFAULTS } from './utils/constants.js';
 export class ClientContentModeration {
     model;
     tokenizer;
@@ -12,41 +12,48 @@ export class ClientContentModeration {
     async initialize() {
         if (this.initialized)
             return;
-        await Promise.all([
-            this.model.initialize(),
-            this.tokenizer.initialize(),
-        ]);
+        await Promise.all([this.model.initialize(), this.tokenizer.initialize()]);
         this.initialized = true;
     }
     async moderate(text, threshold = DEFAULTS.THRESHOLD) {
         if (!this.initialized)
             await this.initialize();
         const encodings = await this.tokenizer.encodeChunks(text);
-        const chunkResults = await Promise.all(encodings.map(e => this.model.predict(e.inputIds, e.attentionMask)));
-        const validChunks = chunkResults.filter(c => (Array.isArray(c) || c instanceof Float32Array) && c.length > 0);
+        const chunkResults = await Promise.all(encodings.map((e) => this.model.predict(e.inputIds, e.attentionMask)));
+        const validChunks = chunkResults.filter((c) => (Array.isArray(c) || c instanceof Float32Array) && c.length > 0);
         if (!validChunks.length) {
             return {
                 approved: true,
                 confidence: 0.5,
-                probabilities: { reject: 0.5, approve: 0.5 }
+                probabilities: { reject: 0.5, approve: 0.5 },
             };
         }
-        const aggregatedLogits = validChunks[0].map((_, i) => validChunks.reduce((sum, logits) => sum + logits[i], 0) / validChunks.length);
-        const probabilities = this.softmax(aggregatedLogits);
-        const shouldApprove = probabilities[1] >= threshold;
+        // Aggregation Logic: Max Pooling for Safety (Reject Probability)
+        // Instead of averaging logits (which can dilute toxic bursts in long text),
+        // we compute probabilities for EACH chunk and take the MAXIMUM Rejection probability.
+        // 1. Compute probabilities for each chunk
+        const chunkProbabilities = validChunks.map((chunkLogits) => this.softmax(chunkLogits));
+        // 2. Extract Reject probabilities (index 0) and Approve probabilities (index 1)
+        const rejectProbs = chunkProbabilities.map((p) => p[0]);
+        // const approveProbs = chunkProbabilities.map(p => p[1]);
+        // 3. Max Pooling on Reject Probability (Safety First)
+        const maxRejectProb = Math.max(...rejectProbs);
+        const finalApproveProb = 1 - maxRejectProb;
+        // Determine final decision based on the WORST chunk
+        const shouldApprove = finalApproveProb >= threshold; // Effectively: maxReject <= (1-threshold)
         return {
             approved: shouldApprove,
-            confidence: Math.max(...probabilities),
+            confidence: maxRejectProb > finalApproveProb ? maxRejectProb : finalApproveProb,
             probabilities: {
-                reject: probabilities[0],
-                approve: probabilities[1]
-            }
+                reject: maxRejectProb,
+                approve: finalApproveProb,
+            },
         };
     }
     async moderateBatch(texts, threshold = DEFAULTS.THRESHOLD) {
         if (!this.initialized)
             await this.initialize();
-        return Promise.all(texts.map(t => this.moderate(t, threshold)));
+        return Promise.all(texts.map((t) => this.moderate(t, threshold)));
     }
     /**
      * Compute softmax probabilities from logits
@@ -54,29 +61,66 @@ export class ClientContentModeration {
      * @returns Array of probabilities (sums to 1)
      */
     softmax(logits) {
-        const output = [];
-        let max = -Infinity;
-        // Find max (numerical stability)
-        for (const v of logits) {
-            if (v > max)
-                max = v;
-        }
-        // Exponentiate and sum
-        let sum = 0;
-        for (const v of logits) {
-            const e = Math.exp(v - max);
-            output.push(e);
-            sum += e;
+        const len = logits.length;
+        // Edge cases
+        if (len === 0)
+            return [];
+        if (len === 1)
+            return [1.0];
+        // Use typed array for intermediate calculations when input is Float32Array
+        const useTypedArray = logits instanceof Float32Array;
+        const output = useTypedArray ? new Float32Array(len) : new Array(len);
+        // 1. Find max for numerical stability
+        const maximumLogit = this.findMax(logits);
+        // 2. Compute exponentials and their sum
+        const exponentialSum = this.computeExponentials(logits, maximumLogit, output);
+        // 3. Normalize to get probabilities
+        this.normalizeProbabilities(output, exponentialSum);
+        // Convert to regular array for consistent return type
+        return useTypedArray ? Array.from(output) : output;
+    }
+    /**
+     * Find maximum value in array (for numerical stability in softmax)
+     */
+    findMax(values) {
+        let maxValue = values[0];
+        for (const value of Array.from(values).slice(1)) {
+            if (value > maxValue) {
+                maxValue = value;
+            }
         }
-        // Normalize (guard against edge cases)
-        if (sum === 0 || !Number.isFinite(sum)) {
-            const uniform = 1 / output.length;
-            return output.map(() => uniform);
+        return maxValue;
+    }
+    /**
+     * Compute exponentials and return their sum
+     * Mutates output array for performance
+     */
+    computeExponentials(logits, maximumLogit, output) {
+        let exponentialSum = 0;
+        Array.from(logits).forEach((logitValue, index) => {
+            const exponentialValue = Math.exp(logitValue - maximumLogit);
+            output[index] = exponentialValue;
+            exponentialSum += exponentialValue;
+        });
+        return exponentialSum;
+    }
+    /**
+     * Normalize output to sum to 1 (convert to probabilities)
+     * Mutates output array for performance
+     */
+    normalizeProbabilities(output, exponentialSum) {
+        const len = output.length;
+        if (!Number.isFinite(exponentialSum) || exponentialSum === 0) {
+            // Fallback: uniform distribution
+            const uniform = 1 / len;
+            output.fill(uniform);
         }
-        for (let i = 0; i < output.length; i++) {
-            output[i] /= sum;
+        else {
+            const inverseSumValue = 1 / exponentialSum;
+            Array.from(output).forEach((_, index) => {
+                output[index] *= inverseSumValue;
+            });
         }
-        return output;
     }
     /**
      * Dispose resources and clean up

package/lib/model.js

@@ -33,14 +33,7 @@ export class ModerationModel {
             this.initialized = true;
         }
         catch (error) {
-            // If model loading fails, verify the URL is correct
-            const verifyResponse = await fetch(this.modelFileUrl);
-            const contentType = verifyResponse.headers.get('content-type') || '';
-            if (contentType.includes('text/html')) {
-                const text = await verifyResponse.text();
-                throw new Error(`Failed to load ONNX model: The URL ${this.modelFileUrl} returned HTML instead of a model file. This usually means the model file path is incorrect. Response preview: ${text.substring(0, 200)}`);
-            }
-            throw error;
+            throw new Error(`Failed to load ONNX model from ${this.modelFileUrl}: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
     async predict(inputIds, attentionMask) {
@@ -56,14 +49,11 @@ export class ModerationModel {
             token_type_ids: new runtime.Tensor(ONNX_CONFIG.TENSOR_TYPE_INT64, tokenTypeIds, [1, tokenTypeIds.length]),
         };
         const output = await this.session.run(feeds);
-        const logits = (output.logits ||
-            output[Object.keys(output)[0]]);
-        if (!logits || !("data" in logits)) {
-            throw new Error("Model output does not contain logits");
+        const logits = (output.logits || output[Object.keys(output)[0]]);
+        if (!logits || !('data' in logits)) {
+            throw new Error('Model output does not contain logits');
         }
-        return logits.data instanceof Float32Array
-            ? logits.data
-            : new Float32Array(logits.data);
+        return logits.data instanceof Float32Array ? logits.data : new Float32Array(logits.data);
     }
     dispose() {
         this.session = null;

package/lib/tokenizer.d.ts

@@ -4,7 +4,7 @@ export declare class Tokenizer {
     private readonly maxLength;
     private readonly reservedTokens;
     constructor(maxLength?: number);
-    initialize(path?: string): Promise<void>;
+    initialize(): Promise<void>;
     private tokenizeText;
     private findSubwordTokens;
     private padTokens;

package/lib/tokenizer.js

@@ -1,4 +1,4 @@
-import { ERROR_MESSAGES, SPECIAL_TOKENS, DEFAULT_TOKEN_IDS, DEFAULTS, TOKENIZER_PATH } from './utils/constants.js';
+import { ERROR_MESSAGES, SPECIAL_TOKENS, DEFAULT_TOKEN_IDS, DEFAULTS } from './utils/constants.js';
 export class Tokenizer {
     tokenizer = null;
     maxLength;
@@ -6,44 +6,39 @@ export class Tokenizer {
     constructor(maxLength) {
         this.maxLength = maxLength ?? DEFAULTS.MAX_LENGTH;
     }
-    async initialize(path) {
+    async initialize() {
         if (this.tokenizer)
             return;
-        const url = path ?? TOKENIZER_PATH;
         try {
-            const res = await fetch(url);
-            if (!res.ok) {
-                throw new Error(`Failed to load tokenizer: HTTP ${res.status} at ${url}`);
-            }
-            const contentType = res.headers.get('content-type') || '';
-            if (contentType.includes('text/html')) {
-                const text = await res.text();
-                throw new Error(`Failed to load tokenizer: The URL ${url} returned HTML instead of JSON. This usually means the path is incorrect. Response preview: ${text.substring(0, 200)}`);
-            }
-            const json = await res.json();
-            this.tokenizer = { vocab: json.model?.vocab ?? {} };
+            const tokenizerData = await import('../models/bert-mini-moderation-output/tokenizer.json');
+            this.tokenizer = { vocab: tokenizerData.model?.vocab ?? {} };
         }
         catch (error) {
-            if (error instanceof Error && error.message.includes('HTML')) {
-                throw error;
-            }
-            throw new Error(`Failed to initialize tokenizer at ${url}: ${error instanceof Error ? error.message : String(error)}`);
+            throw new Error(`Failed to load tokenizer: ${error instanceof Error ? error.message : String(error)}`);
         }
     }
-    tokenizeText(text, vocab, unkId) {
-        const words = text.toLowerCase().match(/\S+/g) ?? [];
+    tokenizeText(text, vocab, unknownTokenId) {
+        const rawTokens = text
+            .toLowerCase()
+            .split(/\s+/)
+            .filter((t) => t.length > 0);
         const tokens = [];
-        for (const word of words) {
-            if (vocab[word])
-                tokens.push(vocab[word]);
-            else
-                tokens.push(...this.findSubwordTokens(word, vocab, unkId));
+        for (const rawToken of rawTokens) {
+            const parts = rawToken.split(/([^\p{L}\p{N}])/gu).filter((p) => p.length > 0);
+            for (const part of parts) {
+                if (vocab[part])
+                    tokens.push(vocab[part]);
+                else
+                    tokens.push(...this.findSubwordTokens(part, vocab, unknownTokenId));
+                if (tokens.length >= this.maxLength - this.reservedTokens)
+                    break;
+            }
             if (tokens.length >= this.maxLength - this.reservedTokens)
                 break;
         }
         return tokens.slice(0, this.maxLength - this.reservedTokens);
     }
-    findSubwordTokens(word, vocab, unkId) {
+    findSubwordTokens(word, vocab, unknownTokenId) {
         const lengths = Array.from({ length: word.length }, (_, k) => word.length - k);
         for (const i of lengths) {
             const subword = word.substring(0, i);
@@ -52,18 +47,18 @@ export class Tokenizer {
                 const remaining = word.substring(i);
                 if (remaining) {
                     const subwordToken = `${SPECIAL_TOKENS.SUBWORD_PREFIX}${remaining}`;
-                    tokens.push(vocab[subwordToken] ?? unkId);
+                    tokens.push(vocab[subwordToken] ?? unknownTokenId);
                 }
                 return tokens;
             }
         }
-        return [unkId];
+        return [unknownTokenId];
     }
-    padTokens(tokens, padId) {
+    padTokens(tokens, paddingTokenId) {
         const inputIds = tokens.slice(0, this.maxLength);
         const attentionMask = inputIds.map(() => 1);
         while (inputIds.length < this.maxLength) {
-            inputIds.push(padId);
+            inputIds.push(paddingTokenId);
             attentionMask.push(0);
         }
         return { inputIds, attentionMask };
@@ -74,8 +69,8 @@ export class Tokenizer {
         if (!this.tokenizer)
             throw new Error(ERROR_MESSAGES.TOKENIZER_NOT_INITIALIZED);
         const vocab = this.tokenizer.vocab ?? {};
-        const unkId = vocab[SPECIAL_TOKENS.UNK] ?? DEFAULT_TOKEN_IDS.UNK;
-        return this.tokenizeText(text, vocab, unkId);
+        const unknownTokenId = vocab[SPECIAL_TOKENS.UNK] ?? DEFAULT_TOKEN_IDS.UNK;
+        return this.tokenizeText(text, vocab, unknownTokenId);
     }
     async encodeChunks(text) {
         if (!this.tokenizer)
@@ -84,24 +79,24 @@ export class Tokenizer {
             throw new Error(ERROR_MESSAGES.TOKENIZER_NOT_INITIALIZED);
         const raw = await this.rawTokenize(text);
         const vocab = this.tokenizer.vocab ?? {};
-        const padId = vocab[SPECIAL_TOKENS.PAD] ?? DEFAULT_TOKEN_IDS.PAD;
-        const clsId = vocab[SPECIAL_TOKENS.CLS] ?? DEFAULT_TOKEN_IDS.CLS;
-        const sepId = vocab[SPECIAL_TOKENS.SEP] ?? DEFAULT_TOKEN_IDS.SEP;
+        const paddingTokenId = vocab[SPECIAL_TOKENS.PAD] ?? DEFAULT_TOKEN_IDS.PAD;
+        const classificationTokenId = vocab[SPECIAL_TOKENS.CLS] ?? DEFAULT_TOKEN_IDS.CLS;
+        const separatorTokenId = vocab[SPECIAL_TOKENS.SEP] ?? DEFAULT_TOKEN_IDS.SEP;
         if (raw.length === 0)
-            return [this.createEmptyChunk(clsId, sepId, padId)];
+            return [this.createEmptyChunk(classificationTokenId, separatorTokenId, paddingTokenId)];
         const chunks = [];
         const chunkSize = this.maxLength - this.reservedTokens;
         const numChunks = Math.max(1, Math.ceil(raw.length / chunkSize));
         const starts = Array.from({ length: numChunks }, (_, k) => k * chunkSize);
         for (const start of starts) {
             const slice = raw.slice(start, start + chunkSize);
-            chunks.push(this.padTokens([clsId, ...slice, sepId], padId));
+            chunks.push(this.padTokens([classificationTokenId, ...slice, separatorTokenId], paddingTokenId));
         }
         return chunks;
     }
-    createEmptyChunk(clsId, sepId, padId) {
-        const inputIds = [clsId, sepId, ...Array(this.maxLength - this.reservedTokens).fill(padId)];
-        const attentionMask = inputIds.map((id) => (id === padId ? 0 : 1));
+    createEmptyChunk(classificationTokenId, separatorTokenId, paddingTokenId) {
+        const inputIds = [classificationTokenId, separatorTokenId, ...Array(this.maxLength - this.reservedTokens).fill(paddingTokenId)];
+        const attentionMask = inputIds.map((id) => (id === paddingTokenId ? 0 : 1));
         return { inputIds: inputIds.slice(0, this.maxLength), attentionMask };
     }
     async encode(text) {

package/lib/utils/constants.d.ts

@@ -6,7 +6,6 @@ export declare const ONNX_CONFIG: {
     WASM_NUM_THREADS: number;
 };
 export declare const MODEL_PATH: string;
-export declare const TOKENIZER_PATH: string;
 export declare const DEFAULTS: {
     THRESHOLD: number;
     MAX_LENGTH: number;

package/lib/utils/constants.js

@@ -1,55 +1,31 @@
 export const ONNX_CONFIG = {
-    EXECUTION_PROVIDER_WASM: "wasm",
-    GRAPH_OPTIMIZATION_LEVEL: "all",
-    TENSOR_TYPE_INT64: "int64",
-    DEFAULT_PROVIDER: "cpu",
+    EXECUTION_PROVIDER_WASM: 'wasm',
+    GRAPH_OPTIMIZATION_LEVEL: 'all',
+    TENSOR_TYPE_INT64: 'int64',
+    DEFAULT_PROVIDER: 'cpu',
     WASM_NUM_THREADS: 1,
 };
-// Model file paths - resolved at runtime relative to package location
-const packageName = '@masteryhub-its/speakout-local-client-model';
-function getModelPath(filename) {
-    const basePath = `/node_modules/${packageName}/models/bert-mini-moderation-output/${filename}`;
-    if (typeof window !== 'undefined' && window.location) {
-        return new URL(basePath, window.location.origin).toString();
-    }
-    try {
-        if (typeof import.meta !== 'undefined' && import.meta.url) {
-            const currentUrl = new URL(import.meta.url);
-            const pathname = currentUrl.pathname;
-            const packageIndex = pathname.indexOf(packageName);
-            if (packageIndex !== -1) {
-                const packageBasePath = pathname.substring(0, packageIndex + packageName.length);
-                const packageBaseUrl = new URL(packageBasePath + '/', currentUrl.origin);
-                return new URL(`models/bert-mini-moderation-output/${filename}`, packageBaseUrl).toString();
-            }
-        }
-    }
-    catch {
-        // Fall through to return basePath
-    }
-    return basePath;
-}
-export const MODEL_PATH = getModelPath("model.int8.onnx");
-export const TOKENIZER_PATH = getModelPath("tokenizer.json");
+const MODEL_FILE_PATH = '../../models/bert-mini-moderation-output/model.int8.onnx';
+export const MODEL_PATH = new URL(MODEL_FILE_PATH, import.meta.url).href;
 export const DEFAULTS = {
     THRESHOLD: 0.5,
     MAX_LENGTH: 128,
 };
 export const ERROR_MESSAGES = {
-    ONNX_RUNTIME_NOT_AVAILABLE: "ONNX Runtime not available. Please ensure onnxruntime-web is properly installed.",
+    ONNX_RUNTIME_NOT_AVAILABLE: 'ONNX Runtime not available. Please ensure onnxruntime-web is properly installed.',
     TOKENIZER_INIT_FAILED: "Failed to initialize tokenizer. Please provide valid tokenizer URL (e.g., '/models/tokenizer.json').",
-    SESSION_NOT_INITIALIZED: "Model session is not initialized. Please call initialize() first.",
-    TOKENIZER_NOT_INITIALIZED: "Tokenizer is not initialized. Please call initialize() first.",
+    SESSION_NOT_INITIALIZED: 'Model session is not initialized. Please call initialize() first.',
+    TOKENIZER_NOT_INITIALIZED: 'Tokenizer is not initialized. Please call initialize() first.',
 };
-export const MODEL_NAME = "asafaya/bert-mini-arabic";
+export const MODEL_NAME = 'asafaya/bert-mini-arabic';
 export const MAX_LENGTH = 128;
 export const NUM_LABELS = 2;
 export const SPECIAL_TOKENS = {
-    PAD: "[PAD]",
-    CLS: "[CLS]",
-    SEP: "[SEP]",
-    UNK: "[UNK]",
-    SUBWORD_PREFIX: "##",
+    PAD: '[PAD]',
+    CLS: '[CLS]',
+    SEP: '[SEP]',
+    UNK: '[UNK]',
+    SUBWORD_PREFIX: '##',
 };
 export const DEFAULT_TOKEN_IDS = {
     PAD: 0,

package/models/bert-mini-moderation-output/tokenizer.json

@@ -130,21 +130,13 @@
     "special_tokens": {
       "[CLS]": {
         "id": "[CLS]",
-        "ids": [
-          2
-        ],
-        "tokens": [
-          "[CLS]"
-        ]
+        "ids": [2],
+        "tokens": ["[CLS]"]
       },
       "[SEP]": {
         "id": "[SEP]",
-        "ids": [
-          3
-        ],
-        "tokens": [
-          "[SEP]"
-        ]
+        "ids": [3],
+        "tokens": ["[SEP]"]
       }
     }
   },
@@ -32161,4 +32153,4 @@
       "للتسويق": 31999
     }
   }
-}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@masteryhub-its/speakout-local-client-model",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "Local text moderation library using an Arabic MiniBERT model with ONNX Runtime (Web/Browser)",
   "type": "module",
   "main": "./index.js",
@@ -25,7 +25,11 @@
     "ml:train": "python src/training/trainer.py",
     "ml:preprocess": "python src/data_processing/pipeline.py",
     "ml:optimize": "python src/worker/run.py",
-    "test": "node --test"
+    "test": "node --test",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "format:py": "black src/ tests/",
+    "format:py:check": "black --check src/ tests/"
   },
   "keywords": [
     "moderation",
@@ -43,11 +47,12 @@
   },
   "devDependencies": {
     "@types/node": "^20.10.0",
-    "typescript": "^5.3.3",
-    "ts-node": "^10.9.2"
+    "prettier": "^3.8.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.3"
   },
   "repository": {
     "type": "git",
     "url": "https://gitlab.masteryhub-its.com/masteryhub-its/speakout-platform-local-model.git"
   }
-}
+}