rubyllm-observ 0.5.1 → 0.6.1

data/app/helpers/observ/application_helper.rb

@@ -5,5 +5,6 @@ module Observ
     include Observ::ChatsHelper
     include Observ::PaginationHelper
     include Observ::DatasetsHelper
+    include Observ::ReviewsHelper
   end
 end

data/app/helpers/observ/reviews_helper.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Observ
+  module ReviewsHelper
+    # Returns the appropriate badge class for a priority level
+    def priority_badge_class(priority)
+      case priority.to_s
+      when "critical"
+        "observ-badge--danger"
+      when "high"
+        "observ-badge--warning"
+      else
+        "observ-badge--secondary"
+      end
+    end
+
+    # Returns the appropriate badge class for a review status
+    def review_status_badge_class(status)
+      case status.to_s
+      when "pending"
+        "observ-badge--default"
+      when "in_progress"
+        "observ-badge--info"
+      when "completed"
+        "observ-badge--success"
+      when "skipped"
+        "observ-badge--secondary"
+      else
+        "observ-badge--default"
+      end
+    end
+  end
+end

data/app/models/concerns/observ/json_queryable.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Observ
+  # Provides database-agnostic JSON querying capabilities.
+  #
+  # This concern allows models with JSON/JSONB columns to query nested
+  # JSON fields in a way that works across different database adapters
+  # (PostgreSQL, SQLite, MySQL, etc.)
+  #
+  # Usage:
+  #   class Session < ApplicationRecord
+  #     include Observ::JsonQueryable
+  #   end
+  #
+  #   # Query by JSON field
+  #   Session.where_json(:metadata, :agent_type, "MyAgent")
+  #   Session.where_json(:metadata, "nested.path", "value")
+  #
+  module JsonQueryable
+    extend ActiveSupport::Concern
+
+    class_methods do
+      # Query records where a JSON column's nested field equals a value.
+      #
+      # @param column [Symbol] The JSON column name
+      # @param path [String, Symbol] The JSON path (e.g., "agent_type" or "nested.path")
+      # @param value [String, Integer, Boolean] The value to match
+      # @return [ActiveRecord::Relation]
+      #
+      # @example Simple path
+      #   Session.where_json(:metadata, :agent_type, "MyAgent")
+      #
+      # @example Nested path
+      #   Session.where_json(:metadata, "config.mode", "production")
+      #
+      def where_json(column, path, value)
+        json_query = JsonQuery.new(connection, table_name, column, path)
+        where(json_query.to_sql, value)
+      end
+
+      # Query records where a JSON column's nested field is not null.
+      #
+      # @param column [Symbol] The JSON column name
+      # @param path [String, Symbol] The JSON path
+      # @return [ActiveRecord::Relation]
+      #
+      def where_json_present(column, path)
+        json_query = JsonQuery.new(connection, table_name, column, path)
+        where("#{json_query.extract_sql} IS NOT NULL")
+      end
+
+      # Pluck values from a JSON column's nested field.
+      #
+      # @param column [Symbol] The JSON column name
+      # @param path [String, Symbol] The JSON path
+      # @return [Array]
+      #
+      def pluck_json(column, path)
+        json_query = JsonQuery.new(connection, table_name, column, path)
+        pluck(Arel.sql(json_query.extract_sql))
+      end
+    end
+
+    # Internal class that generates database-specific SQL for JSON queries.
+    class JsonQuery
+      attr_reader :connection, :table_name, :column, :path
+
+      def initialize(connection, table_name, column, path)
+        @connection = connection
+        @table_name = table_name
+        @column = column.to_s
+        @path = path.to_s
+      end
+
+      # Returns SQL fragment for WHERE clause comparison (with placeholder)
+      def to_sql
+        "#{extract_sql} = ?"
+      end
+
+      # Returns SQL fragment for extracting the JSON value
+      def extract_sql
+        case adapter_name
+        when /postgresql/i
+          postgresql_extract
+        when /sqlite/i
+          sqlite_extract
+        when /mysql|mariadb/i
+          mysql_extract
+        else
+          # Fallback for unknown adapters - try PostgreSQL syntax
+          postgresql_extract
+        end
+      end
+
+      private
+
+      def adapter_name
+        connection.adapter_name
+      end
+
+      def quoted_column
+        "#{quoted_table}.#{connection.quote_column_name(column)}"
+      end
+
+      def quoted_table
+        connection.quote_table_name(table_name)
+      end
+
+      # PostgreSQL: Uses ->> operator for text extraction
+      # For nested paths like "a.b.c", chains -> operators: col->'a'->'b'->>'c'
+      def postgresql_extract
+        parts = path.split(".")
+
+        if parts.size == 1
+          "#{quoted_column}->>'#{parts.first}'"
+        else
+          # Chain -> operators for intermediate keys, ->> for final key
+          intermediate = parts[0..-2].map { |p| "->#{connection.quote(p)}" }.join
+          "#{quoted_column}#{intermediate}->>#{connection.quote(parts.last)}"
+        end
+      end
+
+      # SQLite: Uses json_extract function
+      # Path format: $.key or $.nested.path
+      def sqlite_extract
+        json_path = "$." + path.gsub(".", ".")
+        "json_extract(#{quoted_column}, '#{json_path}')"
+      end
+
+      # MySQL/MariaDB: Uses JSON_EXTRACT or ->> operator (MySQL 5.7.13+)
+      # Path format: $.key or $.nested.path
+      def mysql_extract
+        json_path = "$." + path.gsub(".", ".")
+        "JSON_UNQUOTE(JSON_EXTRACT(#{quoted_column}, '#{json_path}'))"
+      end
+    end
+  end
+end

data/app/models/concerns/observ/reviewable.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Observ
+  module Reviewable
+    extend ActiveSupport::Concern
+
+    included do
+      has_one :review_item, as: :reviewable, class_name: "Observ::ReviewItem", dependent: :destroy
+    end
+
+    # Enqueue this item for review, or return existing review_item if already queued
+    def enqueue_for_review!(reason:, priority: :normal, details: {})
+      review_item || create_review_item!(
+        reason: reason.to_s,
+        reason_details: details,
+        priority: priority,
+        status: :pending
+      )
+    end
+
+    # Returns the review status or 'not_queued' if not in queue
+    def review_status
+      review_item&.status || "not_queued"
+    end
+
+    # Returns true if review has been completed
+    def reviewed?
+      review_item&.completed?
+    end
+
+    # Returns true if review is pending or in progress
+    def pending_review?
+      review_item&.pending? || review_item&.in_progress?
+    end
+
+    # Returns true if this item is in the review queue
+    def in_review_queue?
+      review_item.present?
+    end
+  end
+end

data/app/models/concerns/observ/scoreable.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Observ
+  module Scoreable
+    extend ActiveSupport::Concern
+
+    included do
+      has_many :scores, as: :scoreable, class_name: "Observ::Score", dependent: :destroy
+    end
+
+    # Find a score by name, optionally filtered by source
+    # Returns the most recent score if multiple exist
+    def score_for(name, source: nil)
+      scope = scores.where(name: name)
+      scope = scope.where(source: source) if source
+      scope.order(created_at: :desc).first
+    end
+
+    # Returns true if any scores exist for this record
+    def scored?
+      scores.exists?
+    end
+
+    # Returns the manual score (name="manual", source="manual")
+    def manual_score
+      score_for("manual", source: :manual)
+    end
+
+    # Returns a hash of score names to their average values
+    def score_summary
+      scores.group(:name).average(:value).transform_values { |v| v.round(4) }
+    end
+  end
+end

data/app/models/observ/dataset_run_item.rb

@@ -2,14 +2,14 @@
 
 module Observ
   class DatasetRunItem < ApplicationRecord
+    include Observ::Scoreable
+
     self.table_name = "observ_dataset_run_items"
 
     belongs_to :dataset_run, class_name: "Observ::DatasetRun", inverse_of: :run_items
     belongs_to :dataset_item, class_name: "Observ::DatasetItem", inverse_of: :run_items
     belongs_to :trace, class_name: "Observ::Trace", optional: true
     belongs_to :observation, class_name: "Observ::Observation", optional: true
-    has_many :scores, class_name: "Observ::Score",
-             foreign_key: :dataset_run_item_id, dependent: :destroy, inverse_of: :dataset_run_item
 
     validates :dataset_run_id, uniqueness: { scope: :dataset_item_id }
 
@@ -70,17 +70,7 @@ module Observ
       trace&.duration_ms
     end
 
-    # Score helpers
-    def score_for(name, source: nil)
-      scope = scores.where(name: name)
-      scope = scope.where(source: source) if source
-      scope.order(created_at: :desc).first
-    end
-
-    def scored?
-      scores.any?
-    end
-
+    # DatasetRunItem-specific score helpers
     def passing_scores_count
       scores.where("value >= 0.5").count
     end

data/app/models/observ/review_item.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Observ
+  class ReviewItem < ApplicationRecord
+    self.table_name = "observ_review_items"
+
+    belongs_to :reviewable, polymorphic: true
+
+    enum :status, { pending: 0, in_progress: 1, completed: 2, skipped: 3 }
+    enum :priority, { normal: 0, high: 1, critical: 2 }
+
+    validates :reviewable, presence: true
+    validates :reviewable_id, uniqueness: { scope: :reviewable_type }
+
+    scope :actionable, -> { where(status: [ :pending, :in_progress ]) }
+    scope :by_priority, -> { order(priority: :desc, created_at: :asc) }
+    scope :sessions, -> { where(reviewable_type: "Observ::Session") }
+    scope :traces, -> { where(reviewable_type: "Observ::Trace") }
+
+    def complete!(by: nil)
+      update!(status: :completed, completed_at: Time.current, completed_by: by)
+    end
+
+    def skip!(by: nil)
+      update!(status: :skipped, completed_at: Time.current, completed_by: by)
+    end
+
+    def start_review!
+      update!(status: :in_progress) if pending?
+    end
+
+    def priority_badge_class
+      case priority
+      when "critical" then "observ-badge--danger"
+      when "high" then "observ-badge--warning"
+      else "observ-badge--secondary"
+      end
+    end
+
+    def reason_display
+      reason&.titleize&.gsub("_", " ") || "Manual"
+    end
+
+    def reviewable_type_display
+      reviewable_type.demodulize
+    end
+  end
+end

data/app/models/observ/score.rb

@@ -4,8 +4,7 @@ module Observ
   class Score < ApplicationRecord
     self.table_name = "observ_scores"
 
-    belongs_to :dataset_run_item, class_name: "Observ::DatasetRunItem", inverse_of: :scores
-    belongs_to :trace, class_name: "Observ::Trace"
+    belongs_to :scoreable, polymorphic: true
     belongs_to :observation, class_name: "Observ::Observation", optional: true
 
     enum :data_type, { numeric: 0, boolean: 1, categorical: 2 }
@@ -13,11 +12,44 @@ module Observ
 
     validates :name, presence: true
     validates :value, presence: true, numericality: true
-    validates :dataset_run_item_id, uniqueness: { scope: [ :name, :source ], message: "already has a score with this name and source" }
+    validates :scoreable_id, uniqueness: {
+      scope: [ :scoreable_type, :name, :source ],
+      message: "already has a score with this name and source"
+    }
 
-    # Delegations for convenience
-    delegate :dataset_run, to: :dataset_run_item
-    delegate :dataset_item, to: :dataset_run_item
+    # Scopes
+    scope :for_sessions, -> { where(scoreable_type: "Observ::Session") }
+    scope :for_traces, -> { where(scoreable_type: "Observ::Trace") }
+    scope :for_dataset_run_items, -> { where(scoreable_type: "Observ::DatasetRunItem") }
+
+    # Convenience accessors for polymorphic parent
+    def dataset_run_item
+      scoreable if scoreable_type == "Observ::DatasetRunItem"
+    end
+
+    def trace
+      case scoreable_type
+      when "Observ::Trace" then scoreable
+      when "Observ::DatasetRunItem" then scoreable.trace
+      end
+    end
+
+    def session
+      case scoreable_type
+      when "Observ::Session" then scoreable
+      when "Observ::Trace" then scoreable.observ_session
+      when "Observ::DatasetRunItem" then scoreable.trace&.observ_session
+      end
+    end
+
+    # Delegations for backward compatibility with dataset scoring
+    def dataset_run
+      dataset_run_item&.dataset_run
+    end
+
+    def dataset_item
+      dataset_run_item&.dataset_item
+    end
 
     # Boolean helpers
     def passed?

data/app/models/observ/session.rb

@@ -2,9 +2,13 @@
 
 module Observ
   class Session < ApplicationRecord
+    include Observ::Scoreable
+    include Observ::Reviewable
+    include Observ::JsonQueryable
+
     self.table_name = "observ_sessions"
 
-  has_many :traces, class_name: "Observ::Trace",
+    has_many :traces, class_name: "Observ::Trace",
            foreign_key: :observ_session_id, dependent: :destroy, inverse_of: :observ_session
   has_many :annotations, as: :annotatable, dependent: :destroy
 

data/app/models/observ/trace.rb

@@ -2,6 +2,9 @@
 
 module Observ
   class Trace < ApplicationRecord
+    include Observ::Scoreable
+    include Observ::Reviewable
+
     self.table_name = "observ_traces"
 
     belongs_to :observ_session, class_name: "Observ::Session", inverse_of: :traces

data/app/services/observ/evaluators/base_evaluator.rb

@@ -38,7 +38,6 @@ module Observ
       def create_or_update_score(run_item, value)
         score = run_item.scores.find_or_initialize_by(name: name, source: :programmatic)
         score.assign_attributes(
-          trace: run_item.trace,
           value: value,
           data_type: data_type,
           comment: options[:comment]

data/app/services/observ/guardrail_service.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Observ
+  class GuardrailService
+    class << self
+      def evaluate_trace(trace)
+        return if trace.in_review_queue?
+
+        trace_rules.each do |rule|
+          next unless rule[:condition].call(trace)
+
+          trace.enqueue_for_review!(
+            reason: rule[:name].to_s,
+            priority: rule[:priority],
+            details: rule[:details]&.call(trace) || {}
+          )
+          return # One reason is enough
+        end
+      end
+
+      def evaluate_session(session)
+        return if session.in_review_queue?
+
+        session_rules.each do |rule|
+          next unless rule[:condition].call(session)
+
+          session.enqueue_for_review!(
+            reason: rule[:name].to_s,
+            priority: rule[:priority],
+            details: rule[:details]&.call(session) || {}
+          )
+          return
+        end
+      end
+
+      def evaluate_all_recent(since: 1.hour.ago)
+        Observ::Trace.where(created_at: since..).find_each do |trace|
+          evaluate_trace(trace)
+        end
+
+        Observ::Session.where(created_at: since..).find_each do |session|
+          evaluate_session(session)
+        end
+      end
+
+      def random_sample(scope:, percentage: 5)
+        items = scope.where(created_at: 1.day.ago..)
+                     .left_joins(:review_item)
+                     .where(observ_review_items: { id: nil })
+
+        sample_size = [ (items.count * percentage / 100.0).ceil, 1 ].max
+
+        items.order("RANDOM()").limit(sample_size).find_each do |item|
+          item.enqueue_for_review!(reason: "random_sample", priority: :normal)
+        end
+      end
+
+      private
+
+      def trace_rules
+        [
+          {
+            name: :error_detected,
+            priority: :critical,
+            condition: ->(t) { t.metadata&.dig("error").present? },
+            details: ->(t) { { error: t.metadata["error"] } }
+          },
+          {
+            name: :high_cost,
+            priority: :high,
+            condition: ->(t) { t.total_cost.present? && t.total_cost > thresholds[:trace_cost] },
+            details: ->(t) { { cost: t.total_cost.to_f, threshold: thresholds[:trace_cost] } }
+          },
+          {
+            name: :high_latency,
+            priority: :normal,
+            condition: ->(t) { t.duration_ms.present? && t.duration_ms > thresholds[:latency_ms] },
+            details: ->(t) { { latency_ms: t.duration_ms, threshold: thresholds[:latency_ms] } }
+          },
+          {
+            name: :no_output,
+            priority: :high,
+            condition: ->(t) { t.output.blank? && t.end_time.present? }
+          },
+          {
+            name: :high_token_count,
+            priority: :normal,
+            condition: ->(t) { t.total_tokens.present? && t.total_tokens > thresholds[:tokens] },
+            details: ->(t) { { tokens: t.total_tokens, threshold: thresholds[:tokens] } }
+          }
+        ]
+      end
+
+      def session_rules
+        [
+          {
+            name: :high_cost,
+            priority: :high,
+            condition: ->(s) { s.total_cost.present? && s.total_cost > thresholds[:session_cost] },
+            details: ->(s) { { cost: s.total_cost.to_f, threshold: thresholds[:session_cost] } }
+          },
+          # {
+          #   name: :short_session,
+          #   priority: :normal,
+          #   condition: ->(s) { s.total_traces_count == 1 && s.end_time.present? },
+          #   details: ->(s) { { trace_count: s.total_traces_count } }
+          # },
+          {
+            name: :many_traces,
+            priority: :normal,
+            condition: ->(s) { s.total_traces_count.present? && s.total_traces_count > thresholds[:max_traces] },
+            details: ->(s) { { trace_count: s.total_traces_count, threshold: thresholds[:max_traces] } }
+          }
+        ]
+      end
+
+      def thresholds
+        @thresholds ||= {
+          trace_cost: 0.10,
+          session_cost: 0.50,
+          latency_ms: 30_000,
+          tokens: 10_000,
+          max_traces: 20
+        }
+      end
+    end
+  end
+end

data/app/views/kaminari/_first_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="first">
+<span class="observ-pagination__first">
   <%= link_to_unless current_page.first?, t('views.pagination.first').html_safe, url, remote: remote %>
 </span>

data/app/views/kaminari/_gap.html.erb

@@ -5,4 +5,4 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="page gap"><%= t('views.pagination.truncate').html_safe %></span>
+<span class="observ-pagination__page observ-pagination__page--gap"><%= t('views.pagination.truncate').html_safe %></span>

data/app/views/kaminari/_last_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="last">
+<span class="observ-pagination__last">
   <%= link_to_unless current_page.last?, t('views.pagination.last').html_safe, url, remote: remote %>
 </span>

data/app/views/kaminari/_next_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="next">
+<span class="observ-pagination__next">
   <%= link_to_unless current_page.last?, t('views.pagination.next').html_safe, url, rel: 'next', remote: remote %>
 </span>

data/app/views/kaminari/_page.html.erb

@@ -7,6 +7,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="page<%= ' current' if page.current? %>">
+<span class="observ-pagination__page<%= ' observ-pagination__page--current' if page.current? %>">
   <%= link_to_unless page.current?, page, url, {remote: remote, rel: page.rel} %>
 </span>

data/app/views/kaminari/_paginator.html.erb

@@ -7,7 +7,7 @@
     paginator:     the paginator that renders the pagination tags inside
 -%>
 <%= paginator.render do -%>
-  <nav class="pagination" role="navigation" aria-label="pager">
+  <nav class="observ-pagination" role="navigation" aria-label="pager">
     <%= first_page_tag unless current_page.first? %>
     <%= prev_page_tag unless current_page.first? %>
     <% each_page do |page| -%>

data/app/views/kaminari/_prev_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="prev">
+<span class="observ-pagination__prev">
   <%= link_to_unless current_page.first?, t('views.pagination.previous').html_safe, url, rel: 'prev', remote: remote %>
 </span>

data/app/views/kaminari/observ/_first_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="first">
+<span class="observ-pagination__first">
   <%= link_to_unless current_page.first?, t('views.pagination.first').html_safe, url, remote: remote %>
 </span>

data/app/views/kaminari/observ/_gap.html.erb

@@ -5,4 +5,4 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="page gap"><%= t('views.pagination.truncate').html_safe %></span>
+<span class="observ-pagination__page observ-pagination__page--gap"><%= t('views.pagination.truncate').html_safe %></span>

data/app/views/kaminari/observ/_last_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="last">
+<span class="observ-pagination__last">
   <%= link_to_unless current_page.last?, t('views.pagination.last').html_safe, url, remote: remote %>
 </span>

data/app/views/kaminari/observ/_next_page.html.erb

@@ -6,6 +6,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="next">
+<span class="observ-pagination__next">
   <%= link_to_unless current_page.last?, t('views.pagination.next').html_safe, url, rel: 'next', remote: remote %>
 </span>

data/app/views/kaminari/observ/_page.html.erb

@@ -7,6 +7,6 @@
     per_page:      number of items to fetch per page
     remote:        data-remote
 -%>
-<span class="page<%= ' current' if page.current? %>">
+<span class="observ-pagination__page<%= ' observ-pagination__page--current' if page.current? %>">
   <%= link_to_unless page.current?, page, url, {remote: remote, rel: page.rel} %>
 </span>