ragdoll 0.0.2

data/lib/ragdoll/core/models/audio_content.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+require "active_record"
+require_relative "content"
+
+# == Schema Information
+#
+# Table name: ragdoll_contents (STI)
+#
+#  id                                                            :bigint           not null, primary key
+#  type(Type of content - TextContent, ImageContent, AudioContent) :string         not null
+#  document_id(Reference to parent document)                     :bigint           not null
+#  embedding_model(Embedding model to use for this content)      :string           not null
+#  content(Text content or description of the file)              :text
+#  data(Raw data from file)                                      :text
+#  metadata(Additional metadata about the file's raw data)       :json             default({})
+#  duration(Duration of audio in seconds - for audio content)    :float
+#  sample_rate(Audio sample rate in Hz - for audio content)      :integer
+#  created_at(Standard creation and update timestamps)           :datetime         not null
+#  updated_at(Standard creation and update timestamps)           :datetime         not null
+#
+# Indexes
+#
+#  index_ragdoll_contents_on_document_id        (document_id)
+#  index_ragdoll_contents_on_embedding_model    (embedding_model)
+#  index_ragdoll_contents_on_type               (type)
+#  index_ragdoll_contents_on_fulltext_search    (to_tsvector('english'::regconfig, COALESCE(content, ''::text))) USING gin
+#
+# Foreign Keys
+#
+#  fk_rails_...  (document_id => ragdoll_documents.id)
+#
+
+module Ragdoll
+  module Core
+    module Models
+      class AudioContent < Content
+        validate :audio_data_or_transcript_present
+        validates :duration, numericality: { greater_than: 0 }, allow_nil: true
+        validates :sample_rate, numericality: { greater_than: 0 }, allow_nil: true
+
+        scope :recent, -> { order(created_at: :desc) }
+        scope :with_audio, -> { where.not(data: [nil, ""]) }
+        scope :with_transcripts, -> { where.not(content: [nil, ""]) }
+        scope :by_duration, lambda { |min_duration, max_duration = nil|
+          scope = where("duration >= ?", min_duration)
+          scope = scope.where("duration <= ?", max_duration) if max_duration
+          scope
+        }
+
+
+        # Audio content accessors - content field stores transcript for embedding
+        def transcript
+          content
+        end
+
+        def transcript=(value)
+          self.content = value
+        end
+
+        # Audio file data accessor
+        def audio_data
+          data
+        end
+
+        def audio_data=(value)
+          self.data = value
+        end
+
+        # Audio file technical properties (stored in content metadata - raw file data)
+        def audio_attached?
+          data.present?
+        end
+
+        def audio_size
+          metadata.dig('file_size') || 0
+        end
+
+        def audio_size=(value)
+          self.metadata = metadata.merge('file_size' => value)
+        end
+
+        def audio_content_type
+          metadata.dig('content_type')
+        end
+
+        def audio_content_type=(value)
+          self.metadata = metadata.merge('content_type' => value)
+        end
+
+        def audio_filename
+          metadata.dig('filename')
+        end
+
+        def audio_filename=(value)
+          self.metadata = metadata.merge('filename' => value)
+        end
+
+        # Audio format and technical details
+        def codec
+          metadata.dig('codec')
+        end
+
+        def codec=(value)
+          self.metadata = metadata.merge('codec' => value)
+        end
+
+        def bitrate
+          metadata.dig('bitrate')
+        end
+
+        def bitrate=(value)
+          self.metadata = metadata.merge('bitrate' => value)
+        end
+
+        def channels
+          metadata.dig('channels')
+        end
+
+        def channels=(value)
+          self.metadata = metadata.merge('channels' => value)
+        end
+
+        def duration_formatted
+          return "Unknown" unless duration
+
+          minutes = (duration / 60).floor
+          seconds = (duration % 60).round
+          "#{minutes}:#{seconds.to_s.rjust(2, '0')}"
+        end
+
+        # Override content for embedding to use transcript
+        def content_for_embedding
+          transcript.presence || "Audio content without transcript"
+        end
+
+        def generate_embeddings!
+          return unless should_generate_embeddings?
+
+          embedding_content = content_for_embedding
+          return if embedding_content.blank?
+
+          # Generate embeddings using the base class method
+          super
+        end
+
+        # Override should_generate_embeddings to check for transcript
+        def should_generate_embeddings?
+          content_for_embedding.present? && embeddings.empty?
+        end
+
+        def self.stats
+          {
+            total_audio_contents: count,
+            by_model: group(:embedding_model).count,
+            total_embeddings: joins(:embeddings).count,
+            with_audio: with_audio.count,
+            with_transcripts: with_transcripts.count,
+            total_duration: sum(:duration),
+            average_duration: average(:duration),
+            average_audio_size: joins(:audio_attachment).average("active_storage_blobs.byte_size")
+          }
+        end
+
+        private
+
+        def audio_data_or_transcript_present
+          return if audio_attached? || transcript.present?
+
+          errors.add(:base, "Must have either audio data or transcript")
+        end
+      end
+    end
+  end
+end

data/lib/ragdoll/core/models/content.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+# == Schema Information
+#
+# Table name: ragdoll_contents
+#
+#  id              :bigint           not null, primary key
+#  type            :string           not null
+#  document_id     :bigint           not null
+#  embedding_model :string           not null
+#  content         :text
+#  data            :text
+#  metadata        :json             default({})
+#  duration        :float
+#  sample_rate     :integer
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#
+# Indexes
+#
+#  index_ragdoll_contents_on_embedding_model      (embedding_model)
+#  index_ragdoll_contents_on_type                 (type)
+#  index_ragdoll_contents_on_fulltext_search      (to_tsvector('english'::regconfig, COALESCE(content, ''::text))) USING gin
+#
+# Foreign Keys
+#
+#  fk_rails_...  (document_id => ragdoll_documents.id)
+#
+
+module Ragdoll
+  module Core
+    module Models
+      class Content < ActiveRecord::Base
+        self.table_name = "ragdoll_contents"
+
+        belongs_to :document,
+                   class_name: "Ragdoll::Core::Models::Document",
+                   foreign_key: "document_id"
+
+        has_many :embeddings,
+                 class_name: "Ragdoll::Core::Models::Embedding",
+                 as: :embeddable,
+                 dependent: :destroy
+
+        validates :type, presence: true
+        validates :embedding_model, presence: true
+        validates :document_id, presence: true
+
+        # JSON columns are handled natively by PostgreSQL
+        
+        scope :by_type, ->(content_type) { where(type: content_type) }
+        scope :with_embeddings, -> { joins(:embeddings).distinct }
+        scope :without_embeddings, -> { left_joins(:embeddings).where(embeddings: { id: nil }) }
+
+        # Generate embeddings for this content
+        def generate_embeddings!
+          return unless should_generate_embeddings?
+
+          embedding_content = content_for_embedding
+          return if embedding_content.blank?
+
+          # Clear existing embeddings
+          embeddings.destroy_all
+
+          # Use TextChunker to split content into chunks
+          chunks = Ragdoll::Core::TextChunker.chunk(embedding_content)
+
+          # Generate embeddings for each chunk
+          embedding_service = Ragdoll::Core::EmbeddingService.new
+          
+          chunks.each_with_index do |chunk_text, index|
+            begin
+              vector = embedding_service.generate_embedding(chunk_text)
+              
+              embeddings.create!(
+                content: chunk_text,
+                embedding_vector: vector,
+                chunk_index: index
+              )
+            rescue StandardError => e
+              puts "Failed to generate embedding for chunk #{index}: #{e.message}"
+            end
+          end
+
+          update!(metadata: metadata.merge("embeddings_generated_at" => Time.current))
+        end
+
+        # Content to use for embedding generation (overridden by subclasses)
+        def content_for_embedding
+          content
+        end
+
+        # Whether this content should generate embeddings
+        def should_generate_embeddings?
+          content_for_embedding.present? && embeddings.empty?
+        end
+
+        # Statistics
+        def word_count
+          return 0 unless content.present?
+          content.split(/\s+/).length
+        end
+
+        def character_count
+          content&.length || 0
+        end
+
+        def embedding_count
+          embeddings.count
+        end
+
+        # Search within this content type
+        def self.search_content(query, **options)
+          return none if query.blank?
+
+          where(
+            "to_tsvector('english', COALESCE(content, '')) @@ plainto_tsquery('english', ?)",
+            query
+          ).limit(options[:limit] || 20)
+        end
+      end
+    end
+  end
+end