rubyllm-observ 0.5.0

data/app/models/observ/dataset_run_item.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetRunItem < ApplicationRecord
+    self.table_name = "observ_dataset_run_items"
+
+    belongs_to :dataset_run, class_name: "Observ::DatasetRun", inverse_of: :run_items
+    belongs_to :dataset_item, class_name: "Observ::DatasetItem", inverse_of: :run_items
+    belongs_to :trace, class_name: "Observ::Trace", optional: true
+    belongs_to :observation, class_name: "Observ::Observation", optional: true
+    has_many :scores, class_name: "Observ::Score",
+             foreign_key: :dataset_run_item_id, dependent: :destroy, inverse_of: :dataset_run_item
+
+    validates :dataset_run_id, uniqueness: { scope: :dataset_item_id }
+
+    # Status scopes
+    scope :succeeded, -> { where.not(trace_id: nil).where(error: nil) }
+    scope :failed, -> { where.not(error: nil) }
+    scope :pending, -> { where(trace_id: nil, error: nil) }
+
+    # Status helpers
+    def succeeded?
+      trace_id.present? && error.blank?
+    end
+
+    def failed?
+      error.present?
+    end
+
+    def pending?
+      trace_id.nil? && error.nil?
+    end
+
+    def status
+      return :failed if failed?
+      return :succeeded if succeeded?
+      :pending
+    end
+
+    # Access helpers
+    def input
+      dataset_item.input
+    end
+
+    def expected_output
+      dataset_item.expected_output
+    end
+
+    def actual_output
+      trace&.output
+    end
+
+    # Comparison helpers
+    def output_matches?
+      return nil if expected_output.blank? || actual_output.blank?
+
+      normalize_for_comparison(expected_output) == normalize_for_comparison(actual_output)
+    end
+
+    # Metrics from trace
+    def cost
+      trace&.total_cost
+    end
+
+    def tokens
+      trace&.total_tokens
+    end
+
+    def duration_ms
+      trace&.duration_ms
+    end
+
+    # Score helpers
+    def score_for(name, source: nil)
+      scope = scores.where(name: name)
+      scope = scope.where(source: source) if source
+      scope.order(created_at: :desc).first
+    end
+
+    def scored?
+      scores.any?
+    end
+
+    def passing_scores_count
+      scores.where("value >= 0.5").count
+    end
+
+    def failing_scores_count
+      scores.where("value < 0.5").count
+    end
+
+    private
+
+    # Normalize output for comparison by parsing JSON strings into comparable structures
+    def normalize_for_comparison(output)
+      case output
+      when Hash
+        output.deep_symbolize_keys
+      when String
+        begin
+          parsed = JSON.parse(output)
+          parsed.is_a?(Hash) ? parsed.deep_symbolize_keys : parsed
+        rescue JSON::ParserError
+          output.strip
+        end
+      else
+        output
+      end
+    end
+  end
+end

data/app/models/observ/generation.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Observ
+  class Generation < Observation
+    def set_input(input, messages: nil)
+      update!(
+        input: input.is_a?(String) ? input : input.to_json,
+        messages: messages || self.messages
+      )
+    end
+
+    def set_messages(messages)
+      update!(messages: messages)
+    end
+
+    def set_tools(tools, tool_choice: nil)
+      update!(tools: tools, tool_choice: tool_choice)
+    end
+
+    def finalize(output:, usage: {}, cost_usd: 0.0, status_message: nil, finish_reason: nil,
+                 completion_start_time: nil, provider_metadata: {}, messages: nil, raw_response: nil)
+      merged_usage = (self.usage || {}).merge(usage.stringify_keys)
+      merged_provider_metadata = (self.provider_metadata || {}).merge(provider_metadata.stringify_keys)
+
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        usage: merged_usage,
+        cost_usd: cost_usd,
+        finish_reason: finish_reason,
+        completion_start_time: completion_start_time,
+        provider_metadata: merged_provider_metadata,
+        messages: messages || self.messages,
+        raw_response: raw_response,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    def time_to_first_token_ms
+      return nil unless completion_start_time && start_time
+      ((completion_start_time - start_time) * 1000).round(2)
+    end
+
+    def total_tokens
+      usage&.dig("total_tokens") || 0
+    end
+
+    def input_tokens
+      usage&.dig("input_tokens") || 0
+    end
+
+    def output_tokens
+      usage&.dig("output_tokens") || 0
+    end
+  end
+end

data/app/models/observ/null_prompt.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Observ
+  # Null Object pattern for Prompt
+  # Used when a prompt is not found, providing a fallback with the same interface
+  class NullPrompt
+    attr_reader :name, :prompt, :config
+
+    def initialize(name:, fallback_text:)
+      @name = name
+      @prompt = fallback_text
+      @config = {}
+    end
+
+    # Returns nil to indicate this is not a real prompt
+    def version
+      nil
+    end
+
+    # Returns the fallback text as-is (no variable compilation)
+    def compile(variables = {})
+      @prompt
+    end
+
+    # Null prompts are always in a "fallback" state
+    def state
+      "fallback"
+    end
+
+    def draft?
+      false
+    end
+
+    def production?
+      false
+    end
+
+    def archived?
+      false
+    end
+
+    def persisted?
+      false
+    end
+
+    def id
+      nil
+    end
+
+    # For logging/debugging
+    def to_s
+      "NullPrompt(#{name})"
+    end
+
+    def inspect
+      "#<Observ::NullPrompt name: #{name.inspect}, fallback: #{prompt[0..50].inspect}...>"
+    end
+  end
+end

data/app/models/observ/observation.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Observ
+  class Observation < ApplicationRecord
+    self.table_name = "observ_observations"
+    self.inheritance_column = :type
+
+    belongs_to :trace, class_name: "Observ::Trace", foreign_key: :observ_trace_id, inverse_of: :observations
+
+    validates :observation_id, presence: true, uniqueness: true
+    validates :start_time, presence: true
+    validates :type, presence: true, inclusion: { in: %w[Observ::Generation Observ::Span] }
+
+    after_save :update_trace_metrics, if: :saved_change_to_cost_or_usage?
+
+    def finalize(status_message: nil)
+      update!(
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    def duration_ms
+      return nil unless end_time
+      ((end_time - start_time) * 1000).round(2)
+    end
+
+    private
+
+    def saved_change_to_cost_or_usage?
+      saved_change_to_cost_usd? || saved_change_to_usage?
+    end
+
+    def update_trace_metrics
+      trace&.update_aggregated_metrics
+    end
+  end
+end

data/app/models/observ/prompt.rb

@@ -0,0 +1,315 @@
+# frozen_string_literal: true
+
+require "mustache"
+
+module Observ
+  class Prompt < ApplicationRecord
+    include AASM
+
+    self.table_name = "observ_prompts"
+
+    # ============================================
+    # VALIDATIONS
+    # ============================================
+    validates :name, presence: true
+    validates :prompt, presence: true
+    validates :version, presence: true, numericality: { only_integer: true, greater_than: 0 }
+    validates :state, presence: true, inclusion: { in: %w[draft production archived] }
+
+    # Only ONE production version per prompt name
+    validate :only_one_production_per_name, if: :production?
+
+    # Validate config format
+    validate :validate_config_format
+
+    # Prevent editing immutable prompts
+    before_update :ensure_editable!, if: :content_changed?
+
+    # Ensure config is always a Hash, not a String
+    before_validation :normalize_config
+
+    # ============================================
+    # SCOPES
+    # ============================================
+    scope :by_name, ->(name) { where(name: name) }
+    scope :latest_version, -> { order(version: :desc).limit(1) }
+
+    # ============================================
+    # AASM STATE MACHINE
+    # ============================================
+    aasm column: :state, after_commit: true do
+      state :draft, initial: true
+      state :production
+      state :archived
+
+      event :promote do
+        transitions from: :draft, to: :production, after: :demote_other_production_versions
+      end
+
+      event :demote do
+        transitions from: :production, to: :archived
+      end
+
+      event :restore do
+        transitions from: :archived, to: :production, after: :demote_other_production_versions
+      end
+
+      # Invalidate cache after any state transition
+      after_all_transitions :invalidate_cache_after_transition
+    end
+
+    # ============================================
+    # CALLBACKS
+    # ============================================
+
+    # Invalidate cache after updates or deletion
+    after_save :invalidate_cache_if_changed
+    after_destroy :invalidate_cache_on_destroy
+
+    # ============================================
+    # CLASS METHODS
+    # ============================================
+
+    # Fetch prompt by name, state, or version
+    def self.fetch(name:, version: nil, state: :production, fallback: nil)
+      state ||= Observ.config.prompt_default_state
+      cache_key = cache_key_for(name: name, version: version, state: state)
+      cache_ttl = Observ.config.prompt_cache_ttl
+
+      Rails.cache.fetch(cache_key, expires_in: cache_ttl) do
+        fetch_from_database(name: name, version: version, state: state, fallback: fallback)
+      end
+    end
+
+    def self.fetch_from_database(name:, version:, state:, fallback:)
+      query = where(name: name)
+
+      prompt = if version.present?
+        query.find_by(version: version)
+      else
+        query.public_send(state).first
+      end
+
+      return prompt if prompt
+      return fallback if fallback
+      raise PromptNotFoundError, "Prompt '#{name}' not found"
+    end
+
+    def self.cache_key_for(name:, version:, state:)
+      "observ:prompt:#{name}:#{version || state}"
+    end
+
+    def self.clear_cache(name:)
+      # Clear all cache keys for this prompt
+      [ :draft, :production, :archived ].each do |state|
+        Rails.cache.delete(cache_key_for(name: name, version: nil, state: state))
+      end
+    end
+
+    # Create new version (auto-increment)
+    def self.create_version(name:, prompt:, config: {}, commit_message: nil, created_by: nil, promote_to_production: false)
+      latest_version = where(name: name).maximum(:version) || 0
+      new_version = latest_version + 1
+
+      new_prompt = create!(
+        name: name,
+        prompt: prompt,
+        version: new_version,
+        config: config,
+        commit_message: commit_message,
+        created_by: created_by,
+        state: :draft
+      )
+
+      new_prompt.promote! if promote_to_production
+      new_prompt
+    end
+
+    # ============================================
+    # INSTANCE METHODS
+    # ============================================
+
+    # Compile prompt with Mustache templating
+    # Supports: variables {{name}}, loops {{#items}}...{{/items}},
+    # conditionals {{#flag}}...{{/flag}}, inverted sections {{^items}}...{{/items}}
+    def compile(variables = {})
+      Mustache.render(prompt, variables)
+    end
+
+    # Compile with validation (raises if missing top-level variables)
+    # Note: Variables inside sections (loops) are validated at render time by Mustache
+    def compile_with_validation(variables = {})
+      # Extract top-level variables (outside of sections)
+      # This is a simplified approach - we strip section content and check remaining vars
+      template_without_sections = strip_sections(prompt)
+
+      # Matches: {{name}}, {{user.name}} but not {{#section}}, {{/section}}, {{^section}}, {{!comment}}, {{>partial}}, {{{raw}}}
+      required_vars = template_without_sections.scan(/\{\{([^#\^\/!>\{\s][^}\s]*)\}\}/).flatten.uniq
+
+      # Check which variables are missing (convert all keys to strings for comparison)
+      provided_keys = variables.keys.map(&:to_s)
+      missing_vars = required_vars.reject do |var|
+        # Handle dot notation (e.g., "user.name" - check if "user" key exists)
+        root_key = var.split(".").first
+        provided_keys.include?(var) || provided_keys.include?(root_key)
+      end
+
+      if missing_vars.any?
+        raise VariableSubstitutionError, "Missing variables: #{missing_vars.join(', ')}"
+      end
+
+      compile(variables)
+    end
+
+    # Extract top-level variables from template (for validation purposes)
+    def required_variables
+      template_without_sections = strip_sections(prompt)
+      template_without_sections.scan(/\{\{([^#\^\/!>\{\s][^}\s]*)\}\}/).flatten.uniq
+    end
+
+    # Immutability checks
+    def editable?
+      draft?
+    end
+
+    def immutable?
+      production? || archived?
+    end
+
+    def can_delete?
+      draft? || archived?
+    end
+
+    # Clone to new draft version
+    def clone_to_draft
+      self.class.create_version(
+        name: name,
+        prompt: prompt,
+        config: config,
+        commit_message: "Cloned from v#{version} (#{state})",
+        created_by: nil
+      )
+    end
+
+    # Version navigation
+    def previous_version
+      self.class.where(name: name).where("version < ?", version).order(version: :desc).first
+    end
+
+    def next_version
+      self.class.where(name: name).where("version > ?", version).order(version: :asc).first
+    end
+
+    def latest_version
+      self.class.where(name: name).order(version: :desc).first
+    end
+
+    # Export
+    def to_json_export
+      as_json(except: [ :id, :created_at, :updated_at ])
+    end
+
+    def to_yaml_export
+      to_json_export.to_yaml
+    end
+
+    private
+
+    # ============================================
+    # VALIDATIONS
+    # ============================================
+
+    def only_one_production_per_name
+      existing_production = self.class.where(name: name, state: :production).where.not(id: id).exists?
+      if existing_production
+        errors.add(:state, "Only one production version allowed per prompt name")
+      end
+    end
+
+    def validate_config_format
+      return if config.blank?
+
+      validator = Observ::PromptConfigValidator.new(config)
+      unless validator.valid?
+        validator.errors.each do |error|
+          errors.add(:config, error)
+        end
+      end
+    end
+
+    def ensure_editable!
+      if immutable?
+        errors.add(:base, "Cannot edit #{state} prompt. Clone to draft first.")
+        raise ActiveRecord::RecordInvalid, self
+      end
+    end
+
+    def content_changed?
+      prompt_changed? || config_changed?
+    end
+
+    # ============================================
+    # CALLBACKS
+    # ============================================
+
+    def demote_other_production_versions
+      self.class.where(name: name, state: :production).where.not(id: id).update_all(state: :archived)
+    end
+
+    def invalidate_cache_after_transition
+      Observ::PromptManager.invalidate_cache(name: name)
+      Rails.logger.info("Cache invalidated after state transition for #{name} v#{version}")
+    end
+
+    def invalidate_cache_if_changed
+      return unless saved_change_to_prompt? || saved_change_to_config? || saved_change_to_state?
+
+      Observ::PromptManager.invalidate_cache(name: name)
+    end
+
+    def invalidate_cache_on_destroy
+      Observ::PromptManager.invalidate_cache(name: name)
+    end
+
+    def clear_prompt_cache
+      self.class.clear_cache(name: name)
+    end
+
+    def normalize_config
+      return if config.nil?
+
+      # If config is a String, parse it to a Hash
+      if config.is_a?(String)
+        self.config = begin
+          JSON.parse(config)
+        rescue JSON::ParserError
+          {} # Default to empty hash if parsing fails
+        end
+      end
+
+      # Ensure it's a Hash (could be other types in edge cases)
+      self.config = {} unless config.is_a?(Hash)
+    end
+
+    # Strip section content from template for top-level variable extraction
+    # Removes content between {{#section}}...{{/section}} and {{^section}}...{{/section}}
+    def strip_sections(template)
+      # Recursively strip nested sections
+      result = template.dup
+
+      # Match sections: {{#name}}...{{/name}} or {{^name}}...{{/name}}
+      # Use non-greedy matching and handle nesting by repeating until stable
+      loop do
+        previous = result
+        result = result.gsub(/\{\{[#\^](\w+)\}\}.*?\{\{\/\1\}\}/m, "")
+        break if result == previous
+      end
+
+      result
+    end
+  end
+
+  # Custom exceptions
+  class PromptNotFoundError < StandardError; end
+  class VariableSubstitutionError < StandardError; end
+end

data/app/models/observ/score.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Observ
+  class Score < ApplicationRecord
+    self.table_name = "observ_scores"
+
+    belongs_to :dataset_run_item, class_name: "Observ::DatasetRunItem", inverse_of: :scores
+    belongs_to :trace, class_name: "Observ::Trace"
+    belongs_to :observation, class_name: "Observ::Observation", optional: true
+
+    enum :data_type, { numeric: 0, boolean: 1, categorical: 2 }
+    enum :source, { programmatic: 0, manual: 1, llm_judge: 2 }
+
+    validates :name, presence: true
+    validates :value, presence: true, numericality: true
+    validates :dataset_run_item_id, uniqueness: { scope: [ :name, :source ], message: "already has a score with this name and source" }
+
+    # Delegations for convenience
+    delegate :dataset_run, to: :dataset_run_item
+    delegate :dataset_item, to: :dataset_run_item
+
+    # Boolean helpers
+    def passed?
+      value >= 0.5
+    end
+
+    def failed?
+      !passed?
+    end
+
+    # Display helpers
+    def display_value
+      case data_type
+      when "boolean"
+        passed? ? "Pass" : "Fail"
+      when "categorical"
+        string_value.presence || value.to_s
+      else
+        value.round(2).to_s
+      end
+    end
+
+    def badge_class
+      if boolean?
+        passed? ? "observ-badge--success" : "observ-badge--danger"
+      else
+        value >= 0.7 ? "observ-badge--success" : (value >= 0.4 ? "observ-badge--warning" : "observ-badge--danger")
+      end
+    end
+  end
+end