rubyllm-observ 0.5.0

data/app/services/observ/evaluator_runner_service.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Observ
+  class EvaluatorRunnerService
+    BUILT_IN_EVALUATORS = {
+      "exact_match" => Evaluators::ExactMatchEvaluator,
+      "contains" => Evaluators::ContainsEvaluator,
+      "json_structure" => Evaluators::JsonStructureEvaluator
+    }.freeze
+
+    attr_reader :dataset_run, :evaluator_configs
+
+    def initialize(dataset_run, evaluator_configs: nil)
+      @dataset_run = dataset_run
+      @evaluator_configs = evaluator_configs || default_evaluator_configs
+    end
+
+    def call
+      return if evaluator_configs.blank?
+
+      dataset_run.run_items.includes(:dataset_item, :trace).find_each do |run_item|
+        next unless run_item.succeeded?
+
+        evaluate_item(run_item)
+      end
+
+      dataset_run
+    end
+
+    def evaluate_item(run_item)
+      evaluator_configs.each do |config|
+        evaluator = build_evaluator(config)
+        next unless evaluator
+
+        evaluator.call(run_item)
+      rescue StandardError => e
+        Rails.logger.error("Evaluator #{config['type']} failed for run_item #{run_item.id}: #{e.message}")
+      end
+    end
+
+    private
+
+    def default_evaluator_configs
+      # Default to exact_match if no config specified
+      [ { "type" => "exact_match" } ]
+    end
+
+    def build_evaluator(config)
+      type = config["type"]
+      evaluator_class = BUILT_IN_EVALUATORS[type]
+
+      return nil unless evaluator_class
+
+      options = config.except("type").symbolize_keys
+      evaluator_class.new(**options)
+    end
+  end
+end

data/app/services/observ/evaluators/base_evaluator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Observ
+  module Evaluators
+    class BaseEvaluator
+      attr_reader :name, :options
+
+      def initialize(name: nil, **options)
+        @name = name || default_name
+        @options = options
+      end
+
+      # Override in subclasses
+      def evaluate(run_item)
+        raise NotImplementedError, "Subclasses must implement #evaluate"
+      end
+
+      # Creates and persists a score for the run item
+      def call(run_item)
+        return nil unless run_item.trace.present?
+
+        value = evaluate(run_item)
+        return nil if value.nil?
+
+        create_or_update_score(run_item, value)
+      end
+
+      protected
+
+      def default_name
+        self.class.name.demodulize.underscore.sub(/_evaluator$/, "")
+      end
+
+      def data_type
+        :numeric
+      end
+
+      def create_or_update_score(run_item, value)
+        score = run_item.scores.find_or_initialize_by(name: name, source: :programmatic)
+        score.assign_attributes(
+          trace: run_item.trace,
+          value: value,
+          data_type: data_type,
+          comment: options[:comment]
+        )
+        score.save!
+        score
+      end
+    end
+  end
+end

data/app/services/observ/evaluators/contains_evaluator.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Observ
+  module Evaluators
+    class ContainsEvaluator < BaseEvaluator
+      def evaluate(run_item)
+        keywords = options[:keywords] || extract_keywords_from_expected(run_item)
+        return nil if keywords.blank?
+
+        output = normalize_output(run_item.actual_output)
+        return 0.0 if output.blank?
+
+        matched = keywords.count { |kw| output.downcase.include?(kw.downcase) }
+        matched.to_f / keywords.size
+      end
+
+      protected
+
+      def default_name
+        "contains"
+      end
+
+      private
+
+      def extract_keywords_from_expected(run_item)
+        expected = run_item.expected_output
+        return [] if expected.blank?
+
+        case expected
+        when Hash
+          expected["keywords"] || expected[:keywords] || []
+        when Array
+          expected
+        when String
+          [ expected ]
+        else
+          []
+        end
+      end
+
+      def normalize_output(output)
+        case output
+        when Hash
+          output.to_json
+        when String
+          output
+        else
+          output.to_s
+        end
+      end
+    end
+  end
+end

data/app/services/observ/evaluators/exact_match_evaluator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Observ
+  module Evaluators
+    class ExactMatchEvaluator < BaseEvaluator
+      def evaluate(run_item)
+        return nil if run_item.expected_output.blank?
+
+        run_item.output_matches? ? 1.0 : 0.0
+      end
+
+      protected
+
+      def data_type
+        :boolean
+      end
+
+      def default_name
+        "exact_match"
+      end
+    end
+  end
+end

data/app/services/observ/evaluators/json_structure_evaluator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Observ
+  module Evaluators
+    class JsonStructureEvaluator < BaseEvaluator
+      def evaluate(run_item)
+        required_keys = options[:required_keys] || extract_keys_from_expected(run_item)
+        return nil if required_keys.blank?
+
+        output = parse_output(run_item.actual_output)
+        return 0.0 if output.nil?
+
+        present_keys = required_keys.count { |key| output.key?(key.to_s) || output.key?(key.to_sym) }
+        present_keys.to_f / required_keys.size
+      end
+
+      protected
+
+      def default_name
+        "json_structure"
+      end
+
+      private
+
+      def extract_keys_from_expected(run_item)
+        expected = run_item.expected_output
+        return [] unless expected.is_a?(Hash)
+
+        expected.keys.map(&:to_s)
+      end
+
+      def parse_output(output)
+        case output
+        when Hash
+          output
+        when String
+          JSON.parse(output) rescue nil
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/app/services/observ/prompt_manager/cache_statistics.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Observ
+  class PromptManager
+    # Concern for cache statistics tracking and reporting.
+    # Handles hit/miss tracking and statistics aggregation.
+    module CacheStatistics
+      # ============================================
+      # CACHE STATISTICS
+      # ============================================
+
+      # Get cache statistics for a prompt
+      # @param name [String] The prompt name
+      # @return [Hash] Statistics hash with :name, :hits, :misses, :total, :hit_rate
+      def cache_stats(name)
+        hits_key = "#{Observ.config.prompt_cache_namespace}:stats:#{name}:hits"
+        misses_key = "#{Observ.config.prompt_cache_namespace}:stats:#{name}:misses"
+
+        hits = Rails.cache.read(hits_key) || 0
+        misses = Rails.cache.read(misses_key) || 0
+        total = hits + misses
+        hit_rate = total > 0 ? (hits.to_f / total * 100).round(2) : 0
+
+        {
+          name: name,
+          hits: hits,
+          misses: misses,
+          total: total,
+          hit_rate: hit_rate
+        }
+      end
+
+      # Clear all cache statistics
+      # @return [Boolean] true if successful
+      def clear_stats
+        Observ::Prompt.distinct.pluck(:name).each do |name|
+          hits_key = "#{Observ.config.prompt_cache_namespace}:stats:#{name}:hits"
+          misses_key = "#{Observ.config.prompt_cache_namespace}:stats:#{name}:misses"
+
+          Rails.cache.delete(hits_key)
+          Rails.cache.delete(misses_key)
+        end
+
+        Rails.logger.info("Cache statistics cleared")
+        true
+      end
+
+      private
+
+      # ============================================
+      # PRIVATE TRACKING METHODS
+      # ============================================
+
+      # Track a cache hit
+      # @param name [String] The prompt name
+      # @param state [Symbol] The prompt state
+      # @param version [Integer, nil] The prompt version
+      def track_cache_hit(name, state, version)
+        cache_stats_key = "#{Observ.config.prompt_cache_namespace}:stats:#{name}:hits"
+
+        # Use Rails cache for atomic operation
+        current_value = Rails.cache.read(cache_stats_key) || 0
+        Rails.cache.write(cache_stats_key, current_value + 1, expires_in: 1.day)
+      rescue => e
+        Rails.logger.error("Failed to track cache hit: #{e.message}")
+      end
+
+      # Track a cache miss
+      # @param name [String] The prompt name
+      # @param state [Symbol] The prompt state
+      # @param version [Integer, nil] The prompt version
+      def track_cache_miss(name, state, version)
+        cache_stats_key = "#{Observ.config.prompt_cache_namespace}:stats:#{name}:misses"
+
+        current_value = Rails.cache.read(cache_stats_key) || 0
+        Rails.cache.write(cache_stats_key, current_value + 1, expires_in: 1.day)
+      rescue => e
+        Rails.logger.error("Failed to track cache miss: #{e.message}")
+      end
+    end
+  end
+end

data/app/services/observ/prompt_manager/caching.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module Observ
+  class PromptManager
+    # Concern for prompt caching operations including cache key generation,
+    # fetching with cache, invalidation, and cache warming.
+    module Caching
+      # ============================================
+      # CACHE KEY MANAGEMENT
+      # ============================================
+
+      # Enhanced cache key strategy
+      # @param name [String] The prompt name
+      # @param state [Symbol, nil] The prompt state (:draft, :production, :archived)
+      # @param version [Integer, nil] The prompt version number
+      # @return [String] The cache key for the prompt
+      def cache_key(name:, state: nil, version: nil)
+        namespace = Observ.config.prompt_cache_namespace
+
+        if version
+          "#{namespace}:#{name}:version:#{version}"
+        elsif state
+          "#{namespace}:#{name}:state:#{state}"
+        else
+          "#{namespace}:#{name}:production"
+        end
+      end
+
+      # ============================================
+      # PROMPT FETCHING WITH ADVANCED CACHING
+      # ============================================
+
+      # Fetch single prompt with advanced caching
+      # @param name [String] The prompt name
+      # @param state [Symbol] The prompt state (default: :production)
+      # @param version [Integer, nil] Specific version to fetch
+      # @param fallback [String, nil] Fallback text if prompt not found
+      # @return [Observ::Prompt, Observ::NullPrompt] The fetched prompt or NullPrompt with fallback
+      def fetch(name:, state: :production, version: nil, fallback: nil)
+        return fetch_from_db(name: name, state: state, version: version, fallback: fallback) unless caching_enabled?
+
+        cache_key_value = cache_key(name: name, state: state, version: version)
+
+        # Check if value exists in cache
+        cache_hit = Rails.cache.exist?(cache_key_value)
+
+        result = Rails.cache.fetch(cache_key_value, expires_in: Observ.config.prompt_cache_ttl) do
+          fetch_from_db(name: name, state: state, version: version, fallback: fallback).tap do |prompt|
+            # Only track cache miss for real prompts (not NullPrompt)
+            if Observ.config.prompt_cache_monitoring_enabled && prompt && !prompt.is_a?(NullPrompt)
+              track_cache_miss(name, state, version)
+            end
+          end
+        end
+
+        # Only track hit if it was actually in cache and is a real prompt
+        if cache_hit && result && !result.is_a?(NullPrompt) && Observ.config.prompt_cache_monitoring_enabled
+          track_cache_hit(name, state, version)
+        end
+
+        result
+      rescue => e
+        Rails.logger.error("Cache fetch failed for #{name}: #{e.message}")
+        fetch_from_db(name: name, state: state, version: version, fallback: fallback)
+      end
+
+      # Fetch multiple prompts at once
+      # @param names [Array<String>] The prompt names to fetch
+      # @param state [Symbol] The prompt state (default: :production)
+      # @return [Hash] Hash of prompt names to prompt objects
+      def fetch_all(names:, state: :production)
+        Prompt.where(name: names, state: state).index_by(&:name)
+      end
+
+      # ============================================
+      # CACHE INVALIDATION
+      # ============================================
+
+      # Invalidate cache for a prompt
+      # @param name [String] The prompt name
+      # @param version [Integer, nil] Specific version to invalidate (nil = all states)
+      # @return [Boolean] true if successful
+      def invalidate_cache(name:, version: nil)
+        keys = if version
+          [ cache_key(name: name, version: version) ]
+        else
+          # Invalidate all state-based keys for this prompt
+          [ :draft, :production, :archived ].map { |state| cache_key(name: name, state: state) }
+        end
+
+        keys.each { |key| Rails.cache.delete(key) }
+        Rails.logger.info("Cache invalidated for #{name}#{version ? " v#{version}" : ""}")
+
+        true
+      end
+
+      # ============================================
+      # CACHE WARMING
+      # ============================================
+
+      # Warm cache for critical prompts
+      # @param prompt_names [Array<String>, nil] Specific prompts to warm (nil = all critical)
+      # @return [Hash] Hash with :success and :failed arrays
+      def warm_cache(prompt_names = nil)
+        names = prompt_names || critical_prompt_names
+
+        results = { success: [], failed: [] }
+
+        names.each do |name|
+          begin
+            # Fetch production version to warm cache
+            fetch(name: name, state: :production)
+            results[:success] << name
+          rescue => e
+            results[:failed] << { name: name, error: e.message }
+            Rails.logger.error("Failed to warm cache for #{name}: #{e.message}")
+          end
+        end
+
+        Rails.logger.info("Cache warming completed: #{results[:success].count} success, #{results[:failed].count} failed")
+        results
+      end
+
+      # Get list of critical prompts (prompts used by agents)
+      # @return [Array<String>] Array of prompt names
+      def critical_prompt_names
+        return Observ.config.prompt_cache_critical_prompts if Observ.config.prompt_cache_critical_prompts.any?
+
+        # Auto-discover from production prompts
+        Observ::Prompt.where(state: :production).distinct.pluck(:name)
+      end
+
+      private
+
+      # ============================================
+      # PRIVATE HELPER METHODS
+      # ============================================
+
+      # Check if caching is enabled
+      # @return [Boolean]
+      def caching_enabled?
+        Observ.config.prompt_cache_ttl.present? && Observ.config.prompt_cache_ttl > 0
+      end
+
+      # Fetch prompt from database
+      # @param name [String] The prompt name
+      # @param state [Symbol] The prompt state
+      # @param version [Integer, nil] Specific version to fetch
+      # @param fallback [String, nil] Fallback text if not found
+      # @return [Observ::Prompt, Observ::NullPrompt] The prompt or NullPrompt
+      # @raise [PromptNotFoundError] If prompt not found and no fallback provided
+      def fetch_from_db(name:, state:, version:, fallback:)
+        query = Observ::Prompt.where(name: name)
+
+        prompt = if version.present?
+          query.find_by(version: version)
+        else
+          query.public_send(state).first
+        end
+
+        return prompt if prompt
+        return NullPrompt.new(name: name, fallback_text: fallback) if fallback
+        raise PromptNotFoundError, "Prompt '#{name}' not found"
+      end
+    end
+  end
+end

data/app/services/observ/prompt_manager/comparison.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Observ
+  class PromptManager
+    # Concern for comparing different versions of prompts.
+    # Provides diff functionality to highlight changes between versions.
+    module Comparison
+      # ============================================
+      # VERSION COMPARISON
+      # ============================================
+
+      # Compare two versions of a prompt
+      # @param name [String] The prompt name
+      # @param version_a [Integer] First version number
+      # @param version_b [Integer] Second version number
+      # @return [Hash] Hash with :from, :to, and :diff keys
+      def compare_versions(name:, version_a:, version_b:)
+        prompt_a = Prompt.find_by!(name: name, version: version_a)
+        prompt_b = Prompt.find_by!(name: name, version: version_b)
+
+        {
+          from: prompt_a,
+          to: prompt_b,
+          diff: calculate_diff(prompt_a.prompt, prompt_b.prompt)
+        }
+      end
+
+      private
+
+      # ============================================
+      # PRIVATE DIFF CALCULATION
+      # ============================================
+
+      # Calculate diff between two text strings
+      # @param text_a [String] First text
+      # @param text_b [String] Second text
+      # @return [Hash] Hash with :added_lines, :removed_lines, and :changed keys
+      def calculate_diff(text_a, text_b)
+        # Simple line-by-line diff
+        # In production, consider using 'diff-lcs' gem for better diffs
+        {
+          added_lines: text_b.lines - text_a.lines,
+          removed_lines: text_a.lines - text_b.lines,
+          changed: text_a != text_b
+        }
+      end
+    end
+  end
+end

data/app/services/observ/prompt_manager/version_management.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Observ
+  class PromptManager
+    # Concern for prompt version management operations including creation,
+    # state transitions (promote, demote, restore), and version queries.
+    module VersionManagement
+      # ============================================
+      # PROMPT CREATION
+      # ============================================
+
+      # Create new version of a prompt
+      # @param name [String] The prompt name
+      # @param prompt [String] The prompt content
+      # @param config [Hash] Configuration options (default: {})
+      # @param commit_message [String, nil] Optional commit message
+      # @param created_by [String, nil] Optional creator identifier
+      # @param promote_to_production [Boolean] Whether to promote immediately (default: false)
+      # @return [Observ::Prompt] The newly created prompt
+      def create(name:, prompt:, config: {}, commit_message: nil, created_by: nil, promote_to_production: false)
+        Prompt.create_version(
+          name: name,
+          prompt: prompt,
+          config: config,
+          commit_message: commit_message,
+          created_by: created_by,
+          promote_to_production: promote_to_production
+        )
+      end
+
+      # ============================================
+      # VERSION QUERIES
+      # ============================================
+
+      # Get all versions for a prompt
+      # @param name [String] The prompt name
+      # @return [ActiveRecord::Relation] Versions ordered by version number descending
+      def versions(name:)
+        Prompt.where(name: name).order(version: :desc)
+      end
+
+      # ============================================
+      # STATE TRANSITIONS
+      # ============================================
+
+      # Rollback to specific version (restore archived to production)
+      # @param name [String] The prompt name
+      # @param to_version [Integer] The version number to rollback to
+      # @return [Observ::Prompt] The rolled back prompt
+      # @raise [StateTransitionError] If trying to rollback to a draft version
+      def rollback(name:, to_version:)
+        prompt = Prompt.find_by!(name: name, version: to_version)
+
+        if prompt.archived?
+          prompt.restore!
+          prompt
+        elsif prompt.production?
+          # Already production, nothing to do
+          prompt
+        else
+          raise StateTransitionError, "Cannot rollback to draft version"
+        end
+      end
+
+      # Promote specific version to production
+      # @param name [String] The prompt name
+      # @param version [Integer] The version number to promote
+      # @return [Observ::Prompt] The promoted prompt
+      def promote(name:, version:)
+        prompt = Prompt.find_by!(name: name, version: version)
+        prompt.promote! if prompt.draft?
+        prompt
+      end
+
+      # Demote production to archived
+      # @param name [String] The prompt name
+      # @param version [Integer] The version number to demote
+      # @return [Observ::Prompt] The demoted prompt
+      def demote(name:, version:)
+        prompt = Prompt.find_by!(name: name, version: version)
+        prompt.demote! if prompt.production?
+        prompt
+      end
+
+      # Restore archived to production
+      # @param name [String] The prompt name
+      # @param version [Integer] The version number to restore
+      # @return [Observ::Prompt] The restored prompt
+      def restore(name:, version:)
+        prompt = Prompt.find_by!(name: name, version: version)
+        prompt.restore! if prompt.archived?
+        prompt
+      end
+    end
+  end
+end

data/app/services/observ/prompt_manager.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Observ
+  # PromptManager provides a high-level interface for managing prompts
+  # with advanced caching, versioning, and comparison capabilities.
+  #
+  # This service is organized using concerns:
+  # - Caching: Cache operations, invalidation, and warming
+  # - CacheStatistics: Hit/miss tracking and reporting
+  # - VersionManagement: CRUD and state transition operations
+  # - Comparison: Version comparison and diffing
+  #
+  # @example Fetching a prompt with caching
+  #   prompt = PromptManager.fetch(name: 'my-prompt', state: :production)
+  #
+  # @example Creating and promoting a new version
+  #   prompt = PromptManager.create(
+  #     name: 'my-prompt',
+  #     prompt: 'Hello {{name}}',
+  #     config: { model: 'gpt-4o' },
+  #     promote_to_production: true
+  #   )
+  #
+  # @example Cache management
+  #   PromptManager.warm_cache(['prompt1', 'prompt2'])
+  #   PromptManager.invalidate_cache(name: 'my-prompt')
+  #   stats = PromptManager.cache_stats('my-prompt')
+  #
+  class PromptManager
+    # Extend with concerns for clean separation of responsibilities
+    extend Caching
+    extend CacheStatistics
+    extend VersionManagement
+    extend Comparison
+  end
+
+  # Custom exceptions
+  class StateTransitionError < StandardError; end
+  class PromptNotFoundError < StandardError; end
+end