client-llm-preprocessor 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,1319 @@
+import * as webllm from '@mlc-ai/web-llm';
+
+/**
+ * Advanced Internal LLM Logging System
+ * 
+ * Captures detailed insights into LLM processing:
+ * - Token-by-token generation (if streaming available)
+ * - Prompt construction steps
+ * - Intermediate processing states
+ * - Validation steps
+ * - Performance metrics
+ */
+
+class InternalLogger {
+  constructor(options = {}) {
+    this.enabled = options.enabled !== false; // Default: enabled
+    this.verbose = options.verbose || false;
+    this.logLevel = options.logLevel || 'info'; // 'debug', 'info', 'warn', 'error'
+    this.logs = [];
+    this.maxLogs = options.maxLogs || 1000;
+    this.onLogCallback = options.onLogCallback || null;
+  }
+
+  /**
+   * Enable or disable logging
+   */
+  setEnabled(enabled) {
+    this.enabled = enabled;
+  }
+
+  /**
+   * Set verbosity level
+   */
+  setVerbose(verbose) {
+    this.verbose = verbose;
+  }
+
+  /**
+   * Log an event with metadata
+   */
+  log(level, category, message, data = {}) {
+    if (!this.enabled) return;
+
+    const logEntry = {
+      timestamp: new Date().toISOString(),
+      level,
+      category,
+      message,
+      data,
+      stackTrace: this.verbose ? new Error().stack : undefined
+    };
+
+    // Store log
+    this.logs.push(logEntry);
+    if (this.logs.length > this.maxLogs) {
+      this.logs.shift(); // Remove oldest
+    }
+
+    // Console output based on level
+    if (this.shouldLog(level)) {
+      const prefix = `[${level.toUpperCase()}] [${category}]`;
+      console.log(`${prefix} ${message}`, data);
+    }
+
+    // Callback for external handlers
+    if (this.onLogCallback) {
+      this.onLogCallback(logEntry);
+    }
+  }
+
+  /**
+   * Check if log level should be output
+   */
+  shouldLog(level) {
+    const levels = { debug: 0, info: 1, warn: 2, error: 3 };
+    return levels[level] >= levels[this.logLevel];
+  }
+
+  /**
+   * Log prompt construction
+   */
+  logPromptConstruction(operation, originalPrompt, finalPrompt, options = {}) {
+    this.log('debug', 'PROMPT', 'Constructing prompt', {
+      operation,
+      originalLength: originalPrompt.length,
+      finalLength: finalPrompt.length,
+      options,
+      promptPreview: finalPrompt.substring(0, 200) + '...'
+    });
+  }
+
+  /**
+   * Log token generation (if streaming available)
+   */
+  logTokenGeneration(token, cumulativeText, tokenIndex) {
+    if (this.verbose) {
+      this.log('debug', 'TOKEN', `Generated token ${tokenIndex}`, {
+        token,
+        cumulativeLength: cumulativeText.length,
+        tokenIndex
+      });
+    }
+  }
+
+  /**
+   * Log LLM inference start
+   */
+  logInferenceStart(prompt, options) {
+    this.log('info', 'INFERENCE', 'Starting LLM inference', {
+      promptLength: prompt.length,
+      temperature: options.temperature,
+      maxTokens: options.maxTokens,
+      promptPreview: prompt.substring(0, 100) + '...'
+    });
+  }
+
+  /**
+   * Log LLM inference completion
+   */
+  logInferenceComplete(response, duration, tokenCount) {
+    this.log('info', 'INFERENCE', 'LLM inference completed', {
+      responseLength: response.length,
+      duration: `${duration}ms`,
+      estimatedTokens: tokenCount,
+      responsePreview: response.substring(0, 200) + '...'
+    });
+  }
+
+  /**
+   * Log validation step
+   */
+  logValidation(step, input, output, isValid, error = null) {
+    this.log(isValid ? 'info' : 'warn', 'VALIDATION', `Validation: ${step}`, {
+      inputPreview: typeof input === 'string' ? input.substring(0, 100) : input,
+      outputPreview: typeof output === 'string' ? output.substring(0, 100) : output,
+      isValid,
+      error: error?.message
+    });
+  }
+
+
+  /**
+   * Log pipeline step
+   */
+  logPipelineStep(stepIndex, stepName, input, output, duration) {
+    this.log('info', 'PIPELINE', `Pipeline step ${stepIndex + 1}: ${stepName}`, {
+      inputLength: typeof input === 'string' ? input.length : 'N/A',
+      outputLength: typeof output === 'string' ? output.length : 'N/A',
+      duration: `${duration}ms`,
+      inputPreview: typeof input === 'string' ? input.substring(0, 50) : input,
+      outputPreview: typeof output === 'string' ? output.substring(0, 50) : output
+    });
+  }
+
+  /**
+   * Log error with context
+   */
+  logError(operation, error, context = {}) {
+    this.log('error', 'ERROR', `Error in ${operation}`, {
+      error: error.message,
+      stack: error.stack,
+      context
+    });
+  }
+
+  /**
+   * Log performance metrics
+   */
+  logPerformance(operation, metrics) {
+    this.log('info', 'PERFORMANCE', `Performance: ${operation}`, metrics);
+  }
+
+  /**
+   * Get all logs
+   */
+  getLogs(filter = {}) {
+    let filtered = [...this.logs];
+
+    if (filter.level) {
+      filtered = filtered.filter(log => log.level === filter.level);
+    }
+
+    if (filter.category) {
+      filtered = filtered.filter(log => log.category === filter.category);
+    }
+
+    if (filter.since) {
+      const sinceTime = new Date(filter.since).getTime();
+      filtered = filtered.filter(log => new Date(log.timestamp).getTime() >= sinceTime);
+    }
+
+    return filtered;
+  }
+
+  /**
+   * Get logs as formatted string
+   */
+  getLogsAsString(filter = {}) {
+    const logs = this.getLogs(filter);
+    return logs.map(log => {
+      return `[${log.timestamp}] [${log.level}] [${log.category}] ${log.message}`;
+    }).join('\n');
+  }
+
+  /**
+   * Clear logs
+   */
+  clear() {
+    this.logs = [];
+  }
+
+  /**
+   * Export logs as JSON
+   */
+  exportLogs() {
+    return JSON.stringify(this.logs, null, 2);
+  }
+
+  /**
+   * Get summary statistics
+   */
+  getStats() {
+    const stats = {
+      totalLogs: this.logs.length,
+      byLevel: {},
+      byCategory: {},
+      errors: 0,
+      warnings: 0,
+      timeRange: {
+        start: this.logs[0]?.timestamp,
+        end: this.logs[this.logs.length - 1]?.timestamp
+      }
+    };
+
+    this.logs.forEach(log => {
+      stats.byLevel[log.level] = (stats.byLevel[log.level] || 0) + 1;
+      stats.byCategory[log.category] = (stats.byCategory[log.category] || 0) + 1;
+      if (log.level === 'error') stats.errors++;
+      if (log.level === 'warn') stats.warnings++;
+    });
+
+    return stats;
+  }
+}
+
+// Singleton instance
+let defaultLogger = null;
+
+/**
+ * Get or create default logger
+ */
+function getLogger(options = {}) {
+  if (!defaultLogger) {
+    defaultLogger = new InternalLogger(options);
+  }
+  return defaultLogger;
+}
+
+/**
+ * WebLLM Engine Wrapper
+ * Handles model loading and inference with detailed internal logging
+ */
+class LLMEngine {
+  constructor(options = {}) {
+    this.engine = null;
+    this.model = null;
+    this.logger = options.logger || getLogger(options.loggerOptions);
+    this.streamingEnabled = options.streaming !== false; // Try streaming by default
+  }
+
+  /**
+   * Load a WebLLM model
+   * @param {string} model - Model name (default: "Llama-3.2-1B-Instruct-q4f16_1")
+   * @returns {Promise<void>}
+   */
+  async loadModel(model = "Llama-3.2-1B-Instruct-q4f16_1-MLC") {
+    if (this.engine && this.model === model) {
+      this.logger.log('info', 'MODEL', 'Model already loaded, skipping');
+      return;
+    }
+
+    const startTime = Date.now();
+    this.logger.log('info', 'MODEL', `Loading model: ${model}`, { model });
+
+    try {
+      this.engine = await webllm.CreateMLCEngine(model, {
+        initProgressCallback: (report) => {
+          if (report.progress) {
+            const progress = (report.progress * 100).toFixed(1);
+            this.logger.log('info', 'MODEL', `Loading progress: ${progress}%`, {
+              progress: parseFloat(progress),
+              report
+            });
+          }
+        },
+      });
+
+      this.model = model;
+      const loadTime = Date.now() - startTime;
+      this.logger.log('info', 'MODEL', 'Model loaded successfully', {
+        model,
+        loadTime: `${loadTime}ms`
+      });
+    } catch (error) {
+      this.logger.logError('loadModel', error, { model });
+      throw new Error(`Failed to load model: ${error.message}`);
+    }
+  }
+
+  /**
+   * Run inference with the loaded model
+   * Captures detailed internal state including token-by-token generation
+   * @param {string} prompt - The prompt to send to the model
+   * @param {Object} options - Generation options
+   * @returns {Promise<string>}
+   */
+  async run(prompt, options = {}) {
+    if (!this.engine) {
+      throw new Error("Model not loaded. Call loadModel() first.");
+    }
+
+    const {
+      temperature = 0.7,
+      maxTokens = 512,
+      stopSequences = [],
+      stream = this.streamingEnabled, // Try streaming for token-by-token logging
+    } = options;
+
+    const startTime = Date.now();
+    this.logger.logInferenceStart(prompt, { temperature, maxTokens, stopSequences, stream });
+
+    try {
+      let fullResponse = '';
+      let tokenCount = 0;
+      const tokens = [];
+
+      // Try streaming first for token-by-token visibility
+      if (stream && this.engine.chat?.completions?.createStream) {
+        this.logger.log('info', 'INFERENCE', 'Using streaming mode for token-by-token logging');
+
+        try {
+          const stream = await this.engine.chat.completions.createStream({
+            messages: [{ role: "user", content: prompt }],
+            temperature,
+            max_tokens: maxTokens,
+            stop: stopSequences.length > 0 ? stopSequences : undefined,
+          });
+
+          // Capture each token as it's generated
+          for await (const chunk of stream) {
+            const delta = chunk.choices?.[0]?.delta?.content || '';
+            if (delta) {
+              fullResponse += delta;
+              tokenCount++;
+              tokens.push(delta);
+
+              // Log each token (if verbose)
+              this.logger.logTokenGeneration(delta, fullResponse, tokenCount);
+            }
+          }
+
+          this.logger.log('info', 'INFERENCE', 'Streaming completed', {
+            totalTokens: tokenCount,
+            responseLength: fullResponse.length
+          });
+        } catch (streamError) {
+          // Fallback to non-streaming if streaming fails
+          this.logger.log('warn', 'INFERENCE', 'Streaming failed, falling back to non-streaming', {
+            error: streamError.message
+          });
+          return await this.runNonStreaming(prompt, options, startTime);
+        }
+      } else {
+        // Non-streaming mode
+        return await this.runNonStreaming(prompt, options, startTime);
+      }
+
+      const duration = Date.now() - startTime;
+      this.logger.logInferenceComplete(fullResponse, duration, tokenCount);
+
+      // Log token sequence for analysis
+      this.logger.log('debug', 'INFERENCE', 'Token sequence captured', {
+        tokenCount,
+        tokens: tokens.slice(0, 20), // First 20 tokens
+        fullSequenceLength: tokens.length
+      });
+
+      return fullResponse;
+    } catch (error) {
+      const duration = Date.now() - startTime;
+      this.logger.logError('run', error, {
+        promptLength: prompt.length,
+        duration: `${duration}ms`,
+        options
+      });
+      throw new Error(`Inference failed: ${error.message}`);
+    }
+  }
+
+  /**
+   * Non-streaming inference (fallback)
+   * @private
+   */
+  async runNonStreaming(prompt, options, startTime) {
+    const {
+      temperature = 0.7,
+      maxTokens = 512,
+      stopSequences = [],
+    } = options;
+
+    this.logger.log('info', 'INFERENCE', 'Using non-streaming mode');
+
+    const response = await this.engine.chat.completions.create({
+      messages: [{ role: "user", content: prompt }],
+      temperature,
+      max_tokens: maxTokens,
+      stop: stopSequences.length > 0 ? stopSequences : undefined,
+    });
+
+    const result = response.choices[0].message.content;
+    const duration = Date.now() - (startTime || Date.now());
+    const estimatedTokens = Math.ceil(result.length / 4); // Rough estimate: ~4 chars per token
+
+    this.logger.logInferenceComplete(result, duration, estimatedTokens);
+
+    return result;
+  }
+
+  /**
+   * Check if model is loaded
+   * @returns {boolean}
+   */
+  isLoaded() {
+    return this.engine !== null;
+  }
+
+  /**
+   * Get the logger instance
+   * @returns {InternalLogger}
+   */
+  getLogger() {
+    return this.logger;
+  }
+
+  /**
+   * Enable/disable streaming for token-by-token logging
+   */
+  setStreaming(enabled) {
+    this.streamingEnabled = enabled;
+    this.logger.log('info', 'ENGINE', `Streaming ${enabled ? 'enabled' : 'disabled'}`);
+  }
+}
+
+/**
+ * Non-LLM cleaning using rules and regex
+ * Fast, deterministic, works without model
+ * All options are opt-in (default: false) - user chooses what to remove
+ */
+
+/**
+ * Clean text using rule-based approach (no LLM)
+ * @param {string} text - Text to clean
+ * @param {Object} options - Cleaning options (all optional, default: false)
+ * @param {boolean} options.removeHtml - Remove HTML tags (default: false)
+ * @param {boolean} options.removeUrls - Remove URLs (default: false)
+ * @param {boolean} options.removeExtraWhitespace - Remove extra whitespace (default: false)
+ * @param {boolean} options.removeLineBreaks - Remove line breaks (default: false)
+ * @param {boolean} options.removeSpecialChars - Remove special characters (default: false)
+ * @param {boolean} options.decodeHtmlEntities - Decode HTML entities like &amp; (default: false)
+ * @returns {string}
+ */
+function cleanWithRules(text, options = {}) {
+  const {
+    removeHtml = false,
+    removeUrls = false,
+    removeExtraWhitespace = false,
+    removeLineBreaks = false,
+    removeSpecialChars = false,
+    decodeHtmlEntities = false,
+  } = options;
+
+  let cleaned = text;
+
+  // Decode HTML entities (if requested)
+  if (decodeHtmlEntities) {
+    cleaned = cleaned
+      .replace(/&nbsp;/g, ' ')
+      .replace(/&amp;/g, '&')
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&quot;/g, '"')
+      .replace(/&#39;/g, "'")
+      .replace(/&#x27;/g, "'")
+      .replace(/&#x2F;/g, '/');
+  }
+
+  // Remove HTML tags (if requested)
+  if (removeHtml) {
+    cleaned = cleaned.replace(/<[^>]+>/g, '');
+  }
+
+  // Remove URLs (if requested)
+  if (removeUrls) {
+    cleaned = cleaned.replace(/https?:\/\/[^\s]+/g, '');
+  }
+
+  // Remove line breaks (if requested)
+  if (removeLineBreaks) {
+    cleaned = cleaned.replace(/[\r\n]+/g, ' ');
+  }
+
+  // Remove extra whitespace (if requested)
+  if (removeExtraWhitespace) {
+    // Replace multiple spaces with single space
+    cleaned = cleaned.replace(/[ \t]+/g, ' ');
+    // Remove leading/trailing whitespace from each line
+    if (!removeLineBreaks) {
+      cleaned = cleaned.split('\n').map(line => line.trim()).join('\n');
+    }
+    // Remove multiple newlines (if line breaks not removed)
+    if (!removeLineBreaks) {
+      cleaned = cleaned.replace(/\n{3,}/g, '\n\n');
+    }
+    // Trim overall
+    cleaned = cleaned.trim();
+  }
+
+  // Remove special characters (if requested)
+  if (removeSpecialChars) {
+    // Keep alphanumeric, spaces, and basic punctuation
+    cleaned = cleaned.replace(/[^\w\s.,!?;:()\-'"]/g, '');
+  }
+
+  return cleaned;
+}
+
+/**
+ * Clean text by removing noise, HTML, and irrelevant content
+ * Uses LLM if available, falls back to rule-based cleaning if not
+ * All options are opt-in (default: false) - user chooses what to remove
+ * @param {LLMEngine|null} engine - The LLM engine instance (can be null)
+ * @param {string} text - Text to clean
+ * @param {Object} options - Cleaning options (all optional, default: false)
+ * @param {boolean} options.removeHtml - Remove HTML tags (default: false)
+ * @param {boolean} options.removeUrls - Remove URLs (default: false)
+ * @param {boolean} options.removeExtraWhitespace - Remove extra whitespace (default: false)
+ * @param {boolean} options.removeLineBreaks - Remove line breaks (default: false)
+ * @param {boolean} options.removeSpecialChars - Remove special characters (default: false)
+ * @param {boolean} options.decodeHtmlEntities - Decode HTML entities like &amp; (default: false)
+ * @param {string} options.customInstructions - Additional cleaning instructions (requires LLM)
+ * @param {boolean} options.useLLM - Force LLM usage if model is loaded (default: auto-detect)
+ * @returns {Promise<string>|string}
+ */
+async function clean(engine, text, options = {}) {
+  const {
+    removeHtml = false,
+    removeUrls = false,
+    removeExtraWhitespace = false,
+    removeLineBreaks = false,
+    removeSpecialChars = false,
+    decodeHtmlEntities = false,
+    customInstructions = "",
+    useLLM = null, // null = auto-detect, true = force LLM, false = force rules
+  } = options;
+
+  // Check if we should use LLM
+  const shouldUseLLM = useLLM !== false && 
+                       engine !== null && 
+                       engine.isLoaded() && 
+                       (useLLM === true || customInstructions !== "");
+
+  if (!shouldUseLLM) {
+    // Use fast rule-based cleaning (no LLM needed)
+    const logger = engine?.getLogger();
+    if (logger) {
+      logger.log('info', 'CLEAN', 'Using rule-based cleaning (no LLM)', {
+        reason: !engine ? 'No engine' : !engine.isLoaded() ? 'Model not loaded' : 'useLLM=false',
+        options: { removeHtml, removeUrls, removeExtraWhitespace, removeLineBreaks, removeSpecialChars, decodeHtmlEntities }
+      });
+    }
+    
+    return cleanWithRules(text, {
+      removeHtml,
+      removeUrls,
+      removeExtraWhitespace,
+      removeLineBreaks,
+      removeSpecialChars,
+      decodeHtmlEntities
+    });
+  }
+
+  // Use LLM for semantic cleaning (especially if customInstructions provided)
+  const logger = engine.getLogger();
+  
+  // Build prompt based on user's selections
+  const cleaningSteps = [];
+  
+  if (removeHtml) cleaningSteps.push('HTML tags');
+  if (removeUrls) cleaningSteps.push('URLs');
+  if (removeExtraWhitespace) cleaningSteps.push('extra whitespace');
+  if (removeLineBreaks) cleaningSteps.push('line breaks');
+  if (removeSpecialChars) cleaningSteps.push('special characters');
+  if (decodeHtmlEntities) cleaningSteps.push('decode HTML entities');
+  
+  let originalPrompt = `Clean the following text`;
+  let prompt = originalPrompt;
+  
+  if (cleaningSteps.length > 0) {
+    prompt += ` by removing: ${cleaningSteps.join(', ')}`;
+  } else if (!customInstructions) {
+    // If no options selected and no custom instructions, just return text
+    return text;
+  }
+  
+  // Add instruction to preserve meaning
+  prompt += `. IMPORTANT: Do NOT modify the meaning or remove important information. Only remove what was requested.`;
+  
+  if (customInstructions) {
+    prompt += ` Also: ${customInstructions}`;
+  }
+  
+  prompt += `:\n\n${text}`;
+
+  logger.logPromptConstruction('clean', originalPrompt, prompt, options);
+  logger.log('info', 'CLEAN', 'Using LLM-based cleaning');
+
+  const result = await engine.run(prompt, { temperature: 0.3 });
+  const cleaned = result.trim();
+
+  logger.log('info', 'CLEAN', 'LLM cleaning completed', {
+    originalLength: text.length,
+    finalLength: cleaned.length
+  });
+
+  return cleaned;
+}
+
+/**
+ * Chunk text into smaller pieces
+ * @param {string} text - Text to chunk
+ * @param {Object} options - Chunking options
+ * @returns {string[]}
+ */
+function chunk(text, options = {}) {
+  const {
+    size = 500, // Character count per chunk
+    overlap = 0, // Overlap between chunks (in characters)
+    strategy = "character", // "character", "sentence", "word"
+  } = options;
+
+  if (!text || text.length === 0) {
+    return [];
+  }
+
+  const chunks = [];
+
+  if (strategy === "character") {
+    for (let i = 0; i < text.length; i += size - overlap) {
+      chunks.push(text.slice(i, i + size));
+    }
+  } else if (strategy === "sentence") {
+    const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
+    let currentChunk = "";
+    
+    for (const sentence of sentences) {
+      if (currentChunk.length + sentence.length > size && currentChunk) {
+        chunks.push(currentChunk.trim());
+        currentChunk = sentence;
+      } else {
+        currentChunk += sentence;
+      }
+    }
+    
+    if (currentChunk.trim()) {
+      chunks.push(currentChunk.trim());
+    }
+  } else if (strategy === "word") {
+    const words = text.split(/\s+/);
+    let currentChunk = [];
+    let currentSize = 0;
+    
+    for (const word of words) {
+      if (currentSize + word.length > size && currentChunk.length > 0) {
+        chunks.push(currentChunk.join(" "));
+        currentChunk = [word];
+        currentSize = word.length;
+      } else {
+        currentChunk.push(word);
+        currentSize += word.length + 1; // +1 for space
+      }
+    }
+    
+    if (currentChunk.length > 0) {
+      chunks.push(currentChunk.join(" "));
+    }
+  }
+
+  return chunks.filter(ch => ch.length > 0);
+}
+
+/**
+ * Rule-based validation utilities
+ * Prevents hallucinations by validating LLM output against source text
+ */
+
+/**
+ * Validate JSON structure and parse safely
+ */
+function validateJSON(text, expectedFields = []) {
+  try {
+    // Strip markdown code blocks if present (e.g., ```json ... ```)
+    const jsonMatch = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
+    const cleanText = jsonMatch ? jsonMatch[1] : text;
+    const parsed = JSON.parse(cleanText);
+
+    // If fields specified, check they exist
+    if (expectedFields.length > 0) {
+      const missingFields = expectedFields.filter(field => !(field in parsed));
+      if (missingFields.length > 0) {
+        return {
+          isValid: false,
+          error: `Missing required fields: ${missingFields.join(', ')}`,
+          data: parsed
+        };
+      }
+    }
+
+    return {
+      isValid: true,
+      data: parsed
+    };
+  } catch (error) {
+    return {
+      isValid: false,
+      error: `Invalid JSON: ${error.message}`,
+      data: null
+    };
+  }
+}
+
+/**
+ * Format-specific validators
+ */
+const validators = {
+  email: (value) => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value),
+  phone: (value) => /^[\d\s\-\+\(\)]+$/.test(value) && value.replace(/\D/g, '').length >= 7,
+  url: (value) => /^https?:\/\/.+/.test(value),
+};
+
+/**
+ * Verify extracted data exists in source text
+ * Prevents hallucination by checking if extracted values appear in original text
+ * Now uses exact matching for structured fields and format validation
+ */
+function verifyExtraction(extracted, sourceText, fields = []) {
+  const sourceLower = sourceText.toLowerCase();
+  const issues = [];
+
+  if (typeof extracted === 'object' && extracted !== null) {
+    // Check each field
+    for (const [key, value] of Object.entries(extracted)) {
+      if (fields.length > 0 && !fields.includes(key)) {
+        continue; // Skip fields not in expected list
+      }
+
+      if (value && typeof value === 'string' && value.trim().length > 0) {
+        const valueLower = value.toLowerCase();
+
+        // First, try exact substring match (case-insensitive)
+        let foundInSource = sourceLower.includes(valueLower);
+
+        // If not found, try format-specific validation
+        if (!foundInSource) {
+          // Check if it matches expected format
+          const fieldType = key.toLowerCase();
+          if (validators[fieldType]) {
+            if (!validators[fieldType](value)) {
+              issues.push({
+                field: key,
+                value,
+                reason: `Invalid ${fieldType} format`
+              });
+              continue;
+            }
+          }
+
+          // For non-exact matches, try word-level matching with stricter threshold
+          const words = valueLower.split(/\s+/).filter(w => w.length > 3);
+          const matchedWords = words.filter(word => sourceLower.includes(word));
+          const matchRatio = words.length > 0 ? matchedWords.length / words.length : 0;
+
+          // Require at least 80% of words to match (stricter than before)
+          foundInSource = matchRatio >= 0.8;
+        }
+
+        if (!foundInSource) {
+          issues.push({
+            field: key,
+            value,
+            reason: 'Value not found in source text (possible hallucination)'
+          });
+        }
+      }
+    }
+  }
+
+  return {
+    isValid: issues.length === 0,
+    issues,
+    extracted
+  };
+}
+
+/**
+ * Clean and normalize extracted values
+ */
+function normalizeExtracted(extracted) {
+  if (typeof extracted !== 'object' || extracted === null) {
+    return extracted;
+  }
+
+  const normalized = {};
+  for (const [key, value] of Object.entries(extracted)) {
+    if (typeof value === 'string') {
+      // Remove common LLM artifacts
+      normalized[key] = value
+        .trim()
+        .replace(/^["']|["']$/g, '') // Remove quotes
+        .replace(/^[-•*]\s*/, '') // Remove list markers
+        .trim();
+    } else {
+      normalized[key] = value;
+    }
+  }
+
+  return normalized;
+}
+
+/**
+ * Validate extraction result with multiple checks
+ */
+function validateExtraction(llmOutput, sourceText, options = {}) {
+  const {
+    format = 'json',
+    fields = [],
+    strict = true, // If true, reject if validation fails
+  } = options;
+
+  let parsed = llmOutput.trim();
+
+  // Step 1: Parse JSON if needed
+  if (format === 'json') {
+    const jsonResult = validateJSON(parsed, fields);
+    if (!jsonResult.isValid) {
+      return {
+        isValid: false,
+        error: jsonResult.error,
+        raw: llmOutput,
+        validated: null
+      };
+    }
+    parsed = jsonResult.data;
+  }
+
+  // Step 2: Normalize
+  const normalized = normalizeExtracted(parsed);
+
+  // Step 3: Verify against source
+  const verification = verifyExtraction(normalized, sourceText, fields);
+
+  // Step 4: Return result
+  if (strict && !verification.isValid) {
+    return {
+      isValid: false,
+      error: 'Extraction validation failed',
+      issues: verification.issues,
+      raw: llmOutput,
+      validated: null
+    };
+  }
+
+  return {
+    isValid: true,
+    raw: llmOutput,
+    validated: normalized,
+    warnings: verification.issues // Include warnings even if not strict
+  };
+}
+
+/**
+ * Extract specific information from text
+ * Uses rule-based validation to prevent hallucinations
+ * @param {LLMEngine} engine - The LLM engine instance
+ * @param {string} text - Text to extract from
+ * @param {Object} options - Extraction options
+ * @returns {Promise<string>}
+ */
+async function extract(engine, text, options = {}) {
+  const logger = engine.getLogger();
+  
+  const {
+    what = "key information", // What to extract
+    format = "text", // "text", "json", "list"
+    fields = [], // Specific fields to extract (for JSON)
+    validate = true, // Enable rule-based validation
+    strict = false, // If true, throw error on validation failure
+  } = options;
+
+  // Log prompt construction
+  const originalPrompt = `Extract ${what} from the following text`;
+  let prompt = originalPrompt;
+
+  if (format === "json") {
+    if (fields.length > 0) {
+      prompt += ` in JSON format with these fields: ${fields.join(", ")}`;
+    } else {
+      prompt += ` in JSON format`;
+    }
+  } else if (format === "list") {
+    prompt += ` as a list`;
+  }
+
+  prompt += `:\n\n${text}`;
+
+  logger.logPromptConstruction('extract', originalPrompt, prompt, options);
+
+  // Run LLM extraction
+  const llmResult = await engine.run(prompt, { temperature: 0.3 });
+  const rawResult = llmResult.trim();
+
+  logger.log('info', 'EXTRACT', 'LLM extraction completed', {
+    format,
+    fields,
+    resultLength: rawResult.length
+  });
+
+  // Apply validation if enabled and format is JSON
+  if (validate && format === "json") {
+    logger.log('info', 'VALIDATION', 'Starting rule-based validation');
+    
+    const validation = validateExtraction(rawResult, text, {
+      format,
+      fields,
+      strict
+    });
+
+    logger.logValidation('extract', text, validation.validated || rawResult, validation.isValid, 
+      validation.error ? new Error(validation.error) : null);
+
+    if (!validation.isValid) {
+      if (strict) {
+        throw new Error(`Extraction validation failed: ${validation.error}`);
+      } else {
+        logger.log('warn', 'VALIDATION', 'Validation failed but continuing (non-strict mode)', {
+          error: validation.error,
+          issues: validation.issues
+        });
+        // Return raw result with warning
+        return rawResult;
+      }
+    }
+
+    // Return validated result
+    if (validation.validated) {
+      logger.log('info', 'VALIDATION', 'Validation passed, returning validated data');
+      return JSON.stringify(validation.validated, null, 2);
+    }
+  }
+
+  return rawResult;
+}
+
+/**
+ * Client-Side LLM Preprocessor
+ * 
+ * A flexible SDK for preprocessing text using local LLM models in the browser.
+ * Supports cleaning, extraction, and custom prompts.
+ */
+class Preprocessor {
+  constructor(options = {}) {
+    this.engine = new LLMEngine(options);
+    this.isModelLoaded = false;
+    this.logger = this.engine.getLogger();
+  }
+
+  /**
+   * Load the WebLLM model
+   * @param {string} model - Model name (default: "Llama-3.2-1B-Instruct-q4f16_1-MLC")
+   * @returns {Promise<void>}
+   */
+  async loadModel(model) {
+    await this.engine.loadModel(model);
+    this.isModelLoaded = true;
+    this.logger.log('info', 'PREPROCESSOR', 'Model loaded and ready');
+  }
+
+  /**
+   * Check if WebGPU is supported in the current environment
+   * @returns {Promise<boolean>}
+   */
+  async checkWebGPU() {
+    if (typeof navigator === 'undefined' || !navigator.gpu) {
+      return false;
+    }
+    try {
+      const adapter = await navigator.gpu.requestAdapter();
+      return !!adapter;
+    } catch (e) {
+      return false;
+    }
+  }
+
+  /**
+   * Get the logger instance for accessing internal logs
+   * @returns {InternalLogger}
+   */
+  getLogger() {
+    return this.logger;
+  }
+
+  /**
+   * Enable/disable internal logging
+   */
+  setLogging(enabled, verbose = false) {
+    this.logger.setEnabled(enabled);
+    this.logger.setVerbose(verbose);
+    this.logger.log('info', 'PREPROCESSOR', `Logging ${enabled ? 'enabled' : 'disabled'}`, { verbose });
+  }
+
+  /**
+   * Ensure model is loaded
+   * @private
+   */
+  _ensureLoaded() {
+    if (!this.isModelLoaded && !this.engine.isLoaded()) {
+      throw new Error(
+        "Model not loaded. Call loadModel() first before using preprocessing functions."
+      );
+    }
+  }
+
+  /**
+   * Clean text
+   * Works with or without LLM model loaded
+   * Uses rule-based cleaning if model not loaded, LLM if available
+   * All options are opt-in (default: false) - user chooses what to remove
+   * @param {string} text - Text to clean
+   * @param {Object} options - Cleaning options (all optional, default: false)
+   * @param {boolean} options.removeHtml - Remove HTML tags (default: false)
+   * @param {boolean} options.removeUrls - Remove URLs (default: false)
+   * @param {boolean} options.removeExtraWhitespace - Remove extra whitespace (default: false)
+   * @param {boolean} options.removeLineBreaks - Remove line breaks (default: false)
+   * @param {boolean} options.removeSpecialChars - Remove special characters (default: false)
+   * @param {boolean} options.decodeHtmlEntities - Decode HTML entities like &amp; (default: false)
+   * @param {string} options.customInstructions - Additional cleaning instructions (requires LLM)
+   * @param {boolean} options.useLLM - Force LLM usage (requires model loaded)
+   * @returns {Promise<string>|string}
+   * 
+   * @example
+   * // No options - returns text as-is
+   * await p.clean(text);
+   * 
+   * // User chooses what to remove
+   * await p.clean(text, { removeHtml: true, removeExtraWhitespace: true });
+   * 
+   * // Use LLM for semantic cleaning
+   * await p.clean(text, { useLLM: true, customInstructions: "Remove all dates" });
+   */
+  async clean(text, options = {}) {
+    // Don't require model - can work without it
+    // Only require if explicitly using LLM or custom instructions
+    if (options.useLLM === true || options.customInstructions) {
+      this._ensureLoaded();
+    }
+
+    return await clean(this.engine, text, options);
+  }
+
+  /**
+   * Extract information from text
+   * @param {string} text - Text to extract from
+   * @param {Object} options - Extraction options
+   * @returns {Promise<string>}
+   */
+  async extract(text, options = {}) {
+    this._ensureLoaded();
+    return await extract(this.engine, text, options);
+  }
+
+  /**
+   * Chunk text into smaller pieces (non-LLM, fast operation)
+   * Works immediately, no model needed
+   * @param {string} text - Text to chunk
+   * @param {Object} options - Chunking options
+   * @returns {string[]}
+   */
+  chunk(text, options = {}) {
+    // No model check needed - chunk is pure string operation
+    return chunk(text, options);
+  }
+
+  /**
+   * Run a custom prompt on text
+   * @param {string} text - Input text
+   * @param {string|Object} instruction - Custom instruction or config object
+   * @param {Object} options - Generation options
+   * @returns {Promise<string>}
+   */
+  async prompt(text, instruction, options = {}) {
+    this._ensureLoaded();
+
+    let promptText;
+    let genOptions = { ...options };
+
+    if (typeof instruction === "string") {
+      promptText = `${instruction}\n\n${text}`;
+    } else if (typeof instruction === "object") {
+      // Advanced prompt configuration
+      const { instruction: inst, format, temperature, maxTokens } = instruction;
+
+      promptText = inst;
+      if (format) {
+        if (typeof format === "object") {
+          promptText += `\n\nReturn the result in JSON format with these fields: ${JSON.stringify(format)}`;
+        } else {
+          promptText += `\n\nFormat: ${format}`;
+        }
+      }
+      promptText += `\n\n${text}`;
+
+      if (temperature !== undefined) genOptions.temperature = temperature;
+      if (maxTokens !== undefined) genOptions.maxTokens = maxTokens;
+    } else {
+      throw new Error("Instruction must be a string or object");
+    }
+
+    return await this.engine.run(promptText, genOptions);
+  }
+
+  /**
+   * Enforce correct pipeline ordering
+   * Always ensures: clean → extract (if both present)
+   * @private
+   */
+  _enforcePipelineOrder(pipeline) {
+    const ordered = [...pipeline];
+    const cleanIndex = ordered.findIndex(step =>
+      step === "clean" || (typeof step === "object" && step.clean !== undefined)
+    );
+    const extractIndex = ordered.findIndex(step =>
+      step === "extract" || (typeof step === "object" && step.extract !== undefined)
+    );
+
+    // If both clean and extract exist, ensure clean comes first
+    if (cleanIndex !== -1 && extractIndex !== -1 && cleanIndex > extractIndex) {
+      this.logger.log('warn', 'PIPELINE', 'Reordering pipeline: clean must come before extract', {
+        originalOrder: ordered.map(s => typeof s === 'string' ? s : Object.keys(s)[0]),
+        reordered: true
+      });
+
+      // Move clean before extract
+      const cleanStep = ordered.splice(cleanIndex, 1)[0];
+      const newExtractIndex = ordered.findIndex(step =>
+        step === "extract" || (typeof step === "object" && step.extract !== undefined)
+      );
+      ordered.splice(newExtractIndex, 0, cleanStep);
+    }
+
+    return ordered;
+  }
+
+  /**
+   * Process text with multiple operations in a pipeline
+   * Automatically enforces correct ordering (clean → extract)
+   * @param {string} text - Input text
+   * @param {Array} pipeline - Array of operations to apply
+   * @returns {Promise<string|string[]>}
+   * 
+   * @example
+   * await p.pipeline(text, [
+   *   "extract",  // Will be reordered to run after clean
+   *   "clean",
+   *   { prompt: "Rewrite in pirate style" }
+   * ]);
+   */
+  async pipeline(text, pipeline) {
+    this._ensureLoaded();
+
+    if (!Array.isArray(pipeline) || pipeline.length === 0) {
+      throw new Error("Pipeline must be a non-empty array");
+    }
+
+    // Enforce correct ordering
+    const orderedPipeline = this._enforcePipelineOrder(pipeline);
+
+    this.logger.log('info', 'PIPELINE', 'Starting pipeline execution', {
+      stepCount: orderedPipeline.length,
+      steps: orderedPipeline.map(s => typeof s === 'string' ? s : Object.keys(s)[0])
+    });
+
+    let result = text;
+    const startTime = Date.now();
+
+    for (let i = 0; i < orderedPipeline.length; i++) {
+      const step = orderedPipeline[i];
+      const stepStartTime = Date.now();
+      const stepName = typeof step === 'string' ? step : Object.keys(step)[0] || 'unknown';
+
+      try {
+        if (typeof step === "string") {
+          // Built-in operation name
+          switch (step) {
+            case "clean":
+              result = await this.clean(result);
+              break;
+            case "extract":
+              result = await this.extract(result);
+              break;
+            case "chunk":
+              result = this.chunk(result);
+              break;
+            default:
+              throw new Error(`Unknown operation: ${step}`);
+          }
+        } else if (typeof step === "object") {
+          // Custom operation with options
+          if (step.prompt) {
+            result = await this.prompt(result, step.prompt, step.options || {});
+          } else if (step.clean) {
+            result = await this.clean(result, step.clean);
+          } else if (step.extract) {
+            result = await this.extract(result, step.extract);
+          } else if (step.chunk) {
+            result = this.chunk(result, step.chunk);
+          } else {
+            throw new Error(`Unknown operation object: ${JSON.stringify(step)}`);
+          }
+        } else {
+          throw new Error(`Invalid pipeline step: ${step}`);
+        }
+
+        // If chunking was applied, result is now an array
+        if (Array.isArray(result)) {
+          this.logger.log('info', 'PIPELINE', 'Chunking applied, stopping pipeline', {
+            chunks: result.length
+          });
+          break; // Can't process arrays further
+        }
+
+        const stepDuration = Date.now() - stepStartTime;
+        this.logger.logPipelineStep(i, stepName, text, result, stepDuration);
+      } catch (error) {
+        this.logger.logError(`pipeline step ${i + 1} (${stepName})`, error, {
+          step,
+          inputLength: typeof text === 'string' ? text.length : 'N/A'
+        });
+        throw error;
+      }
+    }
+
+    const totalDuration = Date.now() - startTime;
+    this.logger.logPerformance('pipeline', {
+      totalSteps: orderedPipeline.length,
+      duration: `${totalDuration}ms`,
+      averageStepTime: `${(totalDuration / orderedPipeline.length).toFixed(2)}ms`
+    });
+
+    return result;
+  }
+
+  /**
+   * Process text with a simple configuration object
+   * @param {string} text - Input text
+   * @param {Object} config - Processing configuration
+   * @returns {Promise<string>}
+   * 
+   * @example
+   * await p.process(text, {
+   *   clean: true,
+   *   extract: { format: "json", fields: ["name", "email"] },
+   *   customPrompt: "Convert to bullet points"
+   * });
+   */
+  async process(text, config = {}) {
+    this._ensureLoaded();
+
+    let result = text;
+
+    // Apply operations in order
+    if (config.clean) {
+      result = await this.clean(
+        result,
+        typeof config.clean === "object" ? config.clean : {}
+      );
+    }
+
+    if (config.extract) {
+      result = await this.extract(
+        result,
+        typeof config.extract === "object" ? config.extract : {}
+      );
+    }
+
+    if (config.customPrompt) {
+      result = await this.prompt(result, config.customPrompt, config.promptOptions || {});
+    }
+
+    if (config.chunk) {
+      result = this.chunk(
+        result,
+        typeof config.chunk === "object" ? config.chunk : {}
+      );
+    }
+
+    return result;
+  }
+}
+
+export { LLMEngine, Preprocessor, chunk, clean, cleanWithRules, extract };