ragdoll 0.1.10 → 0.1.12

data/app/services/ragdoll/audio_to_text_service.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Ragdoll
+  class AudioToTextService
+    class TranscriptionError < StandardError; end
+
+    DEFAULT_OPTIONS = {
+      model: "whisper-1",
+      provider: :openai,
+      temperature: 0.0,
+      language: nil # Auto-detect
+    }.freeze
+
+    def self.transcribe(file_path, **options)
+      new(**options).transcribe(file_path)
+    end
+
+    def initialize(**options)
+      @options = DEFAULT_OPTIONS.merge(options)
+      configure_transcription_service
+    end
+
+    def transcribe(file_path)
+      return "" unless File.exist?(file_path)
+      return "" unless audio_file?(file_path)
+
+      begin
+        # Use RubyLLM for transcription
+        # Note: This is a placeholder implementation
+        # Real implementation would depend on the transcription service available
+
+        if transcription_available?
+          perform_transcription(file_path)
+        else
+          generate_fallback_transcript(file_path)
+        end
+      rescue StandardError => e
+        puts "Warning: Audio transcription failed (#{e.message}), using fallback"
+        generate_fallback_transcript(file_path)
+      end
+    end
+
+    def supported_formats
+      %w[.mp3 .wav .m4a .flac .ogg .aac .wma .mp4 .mov .avi .webm]
+    end
+
+    private
+
+    def configure_transcription_service
+      # Configure transcription service based on provider
+      case @options[:provider]
+      when :openai
+        configure_openai_transcription
+      when :azure
+        configure_azure_transcription
+      when :google
+        configure_google_transcription
+      when :whisper_local
+        configure_local_whisper
+      else
+        puts "Warning: Unsupported transcription provider: #{@options[:provider]}"
+      end
+    end
+
+    def configure_openai_transcription
+      # OpenAI Whisper API configuration
+      @api_key = ENV["OPENAI_API_KEY"]
+      @endpoint = "https://api.openai.com/v1/audio/transcriptions"
+    end
+
+    def configure_azure_transcription
+      # Azure Speech Services configuration
+      @api_key = ENV["AZURE_SPEECH_KEY"]
+      @region = ENV["AZURE_SPEECH_REGION"]
+    end
+
+    def configure_google_transcription
+      # Google Cloud Speech-to-Text configuration
+      @api_key = ENV["GOOGLE_CLOUD_API_KEY"]
+      @project_id = ENV["GOOGLE_CLOUD_PROJECT_ID"]
+    end
+
+    def configure_local_whisper
+      # Local Whisper installation configuration
+      @whisper_command = `which whisper`.strip
+    end
+
+    def transcription_available?
+      case @options[:provider]
+      when :openai
+        !@api_key.nil? && !@api_key.empty?
+      when :azure
+        !@api_key.nil? && !@api_key.empty? && !@region.nil?
+      when :google
+        !@api_key.nil? && !@api_key.empty?
+      when :whisper_local
+        !@whisper_command.empty? && File.executable?(@whisper_command)
+      else
+        false
+      end
+    end
+
+    def perform_transcription(file_path)
+      case @options[:provider]
+      when :openai
+        transcribe_with_openai(file_path)
+      when :azure
+        transcribe_with_azure(file_path)
+      when :google
+        transcribe_with_google(file_path)
+      when :whisper_local
+        transcribe_with_local_whisper(file_path)
+      else
+        raise TranscriptionError, "Unsupported transcription provider"
+      end
+    end
+
+    def transcribe_with_openai(file_path)
+      # Placeholder for OpenAI Whisper API implementation
+      # This would use HTTP requests to OpenAI's API
+      # For now, return a placeholder
+      generate_fallback_transcript(file_path)
+    end
+
+    def transcribe_with_azure(file_path)
+      # Placeholder for Azure Speech Services implementation
+      generate_fallback_transcript(file_path)
+    end
+
+    def transcribe_with_google(file_path)
+      # Placeholder for Google Cloud Speech-to-Text implementation
+      generate_fallback_transcript(file_path)
+    end
+
+    def transcribe_with_local_whisper(file_path)
+      # Use local Whisper installation
+      output_file = "#{file_path}.txt"
+
+      begin
+        # Run whisper command
+        command = "#{@whisper_command} \"#{file_path}\" --output_format txt --output_dir \"#{File.dirname(file_path)}\""
+        command += " --language #{@options[:language]}" if @options[:language]
+        command += " --temperature #{@options[:temperature]}"
+
+        result = `#{command} 2>&1`
+
+        if $?.success? && File.exist?(output_file)
+          transcript = File.read(output_file)
+          File.delete(output_file) # Cleanup
+          transcript.strip
+        else
+          raise TranscriptionError, "Whisper command failed: #{result}"
+        end
+      rescue StandardError => e
+        raise TranscriptionError, "Local Whisper transcription failed: #{e.message}"
+      end
+    end
+
+    def audio_file?(file_path)
+      extension = File.extname(file_path).downcase
+      supported_formats.include?(extension)
+    end
+
+    def generate_fallback_transcript(file_path)
+      filename = File.basename(file_path)
+      duration = estimate_duration(file_path)
+
+      if duration
+        "[Audio file: #{filename} (#{format_duration(duration)})]"
+      else
+        "[Audio file: #{filename}]"
+      end
+    end
+
+    def estimate_duration(file_path)
+      # Try to get duration using file size estimation
+      # This is very rough and not accurate
+      begin
+        file_size = File.size(file_path)
+        # Rough estimation: 1MB per minute for compressed audio
+        estimated_minutes = file_size / (1024 * 1024)
+        estimated_minutes > 0 ? estimated_minutes : nil
+      rescue StandardError
+        nil
+      end
+    end
+
+    def format_duration(minutes)
+      if minutes < 60
+        "#{minutes.round}m"
+      else
+        hours = minutes / 60
+        remaining_minutes = minutes % 60
+        "#{hours.round}h #{remaining_minutes.round}m"
+      end
+    end
+  end
+end

data/app/services/ragdoll/document_converter.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class DocumentConverter
+    class ConversionError < StandardError; end
+
+    def self.convert_to_text(file_path, document_type = nil, **options)
+      new(**options).convert_to_text(file_path, document_type)
+    end
+
+    def initialize(**options)
+      @options = options
+    end
+
+    def convert_to_text(file_path, document_type = nil)
+      return "" unless File.exist?(file_path)
+
+      document_type ||= determine_document_type(file_path)
+
+      begin
+        case document_type
+        when "text", "markdown", "html", "pdf", "docx", "csv", "json", "xml", "yaml"
+          convert_text_based_document(file_path, document_type)
+        when "image"
+          convert_image_to_text(file_path)
+        when "audio"
+          convert_audio_to_text(file_path)
+        when "video"
+          convert_video_to_text(file_path)
+        else
+          convert_unknown_document(file_path)
+        end
+      rescue StandardError => e
+        puts "Warning: Document conversion failed for #{file_path}: #{e.message}"
+        generate_fallback_text(file_path, document_type)
+      end
+    end
+
+    def determine_document_type(file_path)
+      extension = File.extname(file_path).downcase
+
+      case extension
+      when ".pdf" then "pdf"
+      when ".docx" then "docx"
+      when ".txt" then "text"
+      when ".md", ".markdown" then "markdown"
+      when ".html", ".htm" then "html"
+      when ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".webp", ".svg", ".ico", ".tiff", ".tif"
+        "image"
+      when ".mp3", ".wav", ".m4a", ".flac", ".ogg", ".aac", ".wma"
+        "audio"
+      when ".mp4", ".mov", ".avi", ".webm", ".mkv"
+        "video"
+      when ".csv" then "csv"
+      when ".json" then "json"
+      when ".xml" then "xml"
+      when ".yml", ".yaml" then "yaml"
+      else
+        "text"  # Default to text for unknown extensions
+      end
+    end
+
+    def supported_formats
+      {
+        text: %w[.txt .md .markdown .html .htm .csv .json .xml .yml .yaml],
+        documents: %w[.pdf .docx],
+        images: %w[.jpg .jpeg .png .gif .bmp .webp .svg .ico .tiff .tif],
+        audio: %w[.mp3 .wav .m4a .flac .ogg .aac .wma],
+        video: %w[.mp4 .mov .avi .webm .mkv]
+      }
+    end
+
+    private
+
+    def convert_text_based_document(file_path, document_type)
+      service = Ragdoll::TextExtractionService.new(file_path, document_type)
+      service.extract
+    end
+
+    def convert_image_to_text(file_path)
+      service = Ragdoll::ImageToTextService.new(@options)
+      service.convert(file_path)
+    end
+
+    def convert_audio_to_text(file_path)
+      service = Ragdoll::AudioToTextService.new(@options)
+      service.transcribe(file_path)
+    end
+
+    def convert_video_to_text(file_path)
+      # For video files, we'll extract audio and transcribe it
+      # This is a simplified approach - in production you might want to:
+      # 1. Extract keyframes as images and describe them
+      # 2. Extract audio track and transcribe it
+      # 3. Combine both approaches
+
+      begin
+        # Try to extract basic metadata
+        video_info = extract_video_metadata(file_path)
+        audio_text = attempt_video_audio_extraction(file_path)
+
+        if audio_text && !audio_text.empty?
+          if video_info
+            "Video content: #{video_info}\n\nAudio transcript: #{audio_text}"
+          else
+            "Video with audio transcript: #{audio_text}"
+          end
+        else
+          video_info || generate_fallback_text(file_path, "video")
+        end
+      rescue StandardError => e
+        puts "Warning: Video conversion failed: #{e.message}"
+        generate_fallback_text(file_path, "video")
+      end
+    end
+
+    def convert_unknown_document(file_path)
+      # Try to read as text first
+      begin
+        content = File.read(file_path, encoding: "UTF-8")
+        return content if looks_like_text?(content)
+      rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
+        # Try with different encoding
+        begin
+          content = File.read(file_path, encoding: "ISO-8859-1")
+                        .encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
+          return content if looks_like_text?(content)
+        rescue StandardError
+          # Fall through to binary handling
+        end
+      rescue StandardError
+        # Fall through to fallback
+      end
+
+      # If not readable as text, generate metadata-based description
+      generate_fallback_text(file_path, "unknown")
+    end
+
+    def extract_video_metadata(file_path)
+      # Basic video metadata extraction
+      # In production, you might use ffmpeg or similar tools
+      file_size = File.size(file_path)
+      filename = File.basename(file_path, File.extname(file_path))
+
+      # Extract meaningful parts from filename
+      descriptive_parts = filename
+                         .gsub(/[-_]+/, ' ')
+                         .gsub(/([a-z])([A-Z])/, '\1 \2')
+                         .split(' ')
+                         .reject { |part| part.match?(/^\d+$/) }
+                         .map(&:capitalize)
+
+      if descriptive_parts.any?
+        "Video: #{descriptive_parts.join(' ')} (#{format_file_size(file_size)})"
+      else
+        "Video file: #{File.basename(file_path)} (#{format_file_size(file_size)})"
+      end
+    end
+
+    def attempt_video_audio_extraction(file_path)
+      # Placeholder for video audio extraction
+      # In production, you would:
+      # 1. Use ffmpeg to extract audio track
+      # 2. Save to temporary audio file
+      # 3. Transcribe the audio file
+      # 4. Clean up temporary file
+
+      # For now, return nil to indicate no audio extraction
+      nil
+    end
+
+    def looks_like_text?(content)
+      # Simple heuristic to determine if content is text
+      return false if content.empty?
+
+      # Check for reasonable ratio of printable characters
+      printable_chars = content.count(" -~")
+      total_chars = content.length
+
+      printable_ratio = printable_chars.to_f / total_chars
+      printable_ratio > 0.8 && total_chars > 0
+    end
+
+    def generate_fallback_text(file_path, document_type)
+      filename = File.basename(file_path)
+      file_size = File.size(file_path)
+
+      case document_type
+      when "image"
+        "Image file: #{filename} (#{format_file_size(file_size)})"
+      when "audio"
+        "Audio file: #{filename} (#{format_file_size(file_size)})"
+      when "video"
+        "Video file: #{filename} (#{format_file_size(file_size)})"
+      else
+        "Document: #{filename} (#{format_file_size(file_size)})"
+      end
+    end
+
+    def format_file_size(size)
+      units = %w[B KB MB GB TB]
+      unit_index = 0
+
+      while size >= 1024 && unit_index < units.length - 1
+        size /= 1024.0
+        unit_index += 1
+      end
+
+      if unit_index == 0
+        "#{size} #{units[unit_index]}"
+      else
+        "#{size.round(1)} #{units[unit_index]}"
+      end
+    end
+  end
+end

data/app/services/ragdoll/document_management.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
+require "securerandom"
+
 module Ragdoll
   class DocumentManagement
     class << self
-      def add_document(location, content, metadata = {})
+      def add_document(location, content, metadata = {}, force: false)
         # Ensure location is an absolute path if it's a file path
         absolute_location = location.start_with?("http") || location.start_with?("ftp") ? location : File.expand_path(location)
 
@@ -14,17 +16,21 @@ module Ragdoll
                              Time.current
                            end
 
-        # Check if document already exists with same location and file_modified_at
-        existing_document = Ragdoll::Document.find_by(
-          location: absolute_location,
-          file_modified_at: file_modified_at
-        )
+        # Skip duplicate detection if force is true
+        unless force
+          existing_document = find_duplicate_document(absolute_location, content, metadata, file_modified_at)
+          return existing_document.id.to_s if existing_document
+        end
 
-        # Return existing document ID if found (skip duplicate)
-        return existing_document.id.to_s if existing_document
+        # Modify location if force is used to avoid unique constraint violation
+        final_location = if force
+                           "#{absolute_location}#forced_#{Time.current.to_i}_#{SecureRandom.hex(4)}"
+                         else
+                           absolute_location
+                         end
 
         document = Ragdoll::Document.create!(
-          location: absolute_location,
+          location: final_location,
           title: metadata[:title] || metadata["title"] || extract_title_from_location(location),
           document_type: metadata[:document_type] || metadata["document_type"] || "text",
           metadata: metadata.is_a?(Hash) ? metadata : {},
@@ -100,6 +106,108 @@ module Ragdoll
 
       private
 
+      def find_duplicate_document(location, content, metadata, file_modified_at)
+        # Primary check: exact location match (simple duplicate detection)
+        existing = Ragdoll::Document.find_by(location: location)
+        return existing if existing
+
+        # Secondary check: exact location and file modification time (for files)
+        existing_with_time = Ragdoll::Document.find_by(
+          location: location,
+          file_modified_at: file_modified_at
+        )
+        return existing_with_time if existing_with_time
+
+        # Enhanced duplicate detection for file-based documents
+        if File.exist?(location) && !location.start_with?("http")
+          file_size = File.size(location)
+          content_hash = calculate_file_hash(location)
+          
+          # Check for documents with same file hash (most reliable)
+          potential_duplicates = Ragdoll::Document.where("metadata->>'file_hash' = ?", content_hash)
+          return potential_duplicates.first if potential_duplicates.any?
+          
+          # Check for documents with same file size and similar metadata
+          same_size_docs = Ragdoll::Document.where("metadata->>'file_size' = ?", file_size.to_s)
+          same_size_docs.each do |doc|
+            return doc if documents_are_duplicates?(doc, location, content, metadata, file_size, content_hash)
+          end
+        end
+
+        # For non-file documents (URLs, etc), check content-based duplicates
+        unless File.exist?(location)
+          return find_content_based_duplicate(content, metadata)
+        end
+
+        nil
+      end
+
+      def documents_are_duplicates?(existing_doc, location, content, metadata, file_size, content_hash)
+        # Compare multiple factors to determine if documents are duplicates
+        
+        # Check filename similarity (basename without extension)
+        existing_basename = File.basename(existing_doc.location, File.extname(existing_doc.location))
+        new_basename = File.basename(location, File.extname(location))
+        return false unless existing_basename == new_basename
+        
+        # Check content length similarity (within 5% tolerance)
+        if content.present? && existing_doc.content.present?
+          content_length_diff = (content.length - existing_doc.content.length).abs
+          max_length = [content.length, existing_doc.content.length].max
+          return false if max_length > 0 && (content_length_diff.to_f / max_length) > 0.05
+        end
+        
+        # Check key metadata fields
+        existing_metadata = existing_doc.metadata || {}
+        new_metadata = metadata || {}
+        
+        # Compare file type/document type
+        return false if existing_doc.document_type != (new_metadata[:document_type] || new_metadata["document_type"] || "text")
+        
+        # Compare title if available
+        existing_title = existing_metadata["title"] || existing_doc.title
+        new_title = new_metadata[:title] || new_metadata["title"] || extract_title_from_location(location)
+        return false if existing_title && new_title && existing_title != new_title
+        
+        # If we reach here, documents are likely duplicates
+        true
+      end
+
+      def find_content_based_duplicate(content, metadata)
+        return nil unless content.present?
+        
+        content_hash = calculate_content_hash(content)
+        title = metadata[:title] || metadata["title"]
+        
+        # Look for documents with same content hash
+        Ragdoll::Document.where("metadata->>'content_hash' = ?", content_hash).first ||
+        # Look for documents with same title and similar content length (within 5% tolerance)
+        (title ? find_by_title_and_content_similarity(title, content) : nil)
+      end
+
+      def find_by_title_and_content_similarity(title, content)
+        content_length = content.length
+        tolerance = content_length * 0.05
+        
+        Ragdoll::Document.where(title: title).find do |doc|
+          doc.content.present? && 
+          (doc.content.length - content_length).abs <= tolerance
+        end
+      end
+
+      def calculate_file_hash(file_path)
+        require 'digest'
+        Digest::SHA256.file(file_path).hexdigest
+      rescue StandardError => e
+        Rails.logger.warn "Failed to calculate file hash for #{file_path}: #{e.message}" if defined?(Rails)
+        nil
+      end
+
+      def calculate_content_hash(content)
+        require 'digest'
+        Digest::SHA256.hexdigest(content)
+      end
+
       def extract_title_from_location(location)
         File.basename(location, File.extname(location))
       end