rubyllm-observ 0.5.0

data/app/controllers/observ/dashboard_controller.rb

@@ -0,0 +1,159 @@
+module Observ
+  class DashboardController < ApplicationController
+    def index
+      @time_period = params[:period] || "24h"
+      @metrics = calculate_dashboard_metrics
+      @recent_sessions = Observ::Session.order(start_time: :desc).limit(10)
+      @cost_by_model = calculate_cost_by_model
+      @token_usage_over_time = calculate_token_usage_over_time
+      @metrics_by_agent = calculate_metrics_by_agent
+    end
+
+    def metrics
+      render json: calculate_dashboard_metrics
+    end
+
+    def cost_analysis
+      render json: {
+        by_model: calculate_cost_by_model,
+        over_time: calculate_cost_over_time
+      }
+    end
+
+    private
+
+    def calculate_dashboard_metrics
+      time_range = time_range_from_period(@time_period)
+      sessions = Observ::Session.where("start_time >= ?", time_range)
+
+      period_duration = Time.current - time_range
+      previous_time_range = time_range - period_duration
+      previous_sessions = Observ::Session.where("start_time >= ? AND start_time < ?", previous_time_range, time_range)
+
+      current_metrics = {
+        total_sessions: sessions.count,
+        total_traces: Observ::Trace.joins(:observ_session).where("observ_sessions.start_time >= ?", time_range).count,
+        total_llm_calls: Observ::Generation.joins(trace: :observ_session).where("observ_sessions.start_time >= ?", time_range).count,
+        total_tokens: Observ::Trace.joins(:observ_session).where("observ_sessions.start_time >= ?", time_range).sum(:total_tokens),
+        total_cost: Observ::Trace.joins(:observ_session).where("observ_sessions.start_time >= ?", time_range).sum(:total_cost).to_f,
+        avg_latency_ms: calculate_average_llm_latency(time_range),
+        success_rate: calculate_success_rate(time_range),
+        avg_cost_per_call: calculate_avg_cost_per_call(sessions)
+      }
+
+      previous_metrics = {
+        total_sessions: previous_sessions.count,
+        total_tokens: Observ::Trace.joins(:observ_session).where("observ_sessions.start_time >= ? AND observ_sessions.start_time < ?", previous_time_range, time_range).sum(:total_tokens),
+        total_cost: Observ::Trace.joins(:observ_session).where("observ_sessions.start_time >= ? AND observ_sessions.start_time < ?", previous_time_range, time_range).sum(:total_cost).to_f
+      }
+
+      current_metrics.merge(
+        trends: calculate_trends(current_metrics, previous_metrics)
+      )
+    end
+
+    def calculate_cost_by_model
+      Observ::Generation
+        .where("created_at >= ?", time_range_from_period(@time_period))
+        .group(:model)
+        .sum(:cost_usd)
+        .transform_values(&:to_f)
+    end
+
+    def calculate_token_usage_over_time
+      Observ::Session
+        .where("start_time >= ?", time_range_from_period(@time_period))
+        .group("DATE(start_time)")
+        .sum(:total_tokens)
+    end
+
+    def calculate_cost_over_time
+      Observ::Session
+        .where("start_time >= ?", time_range_from_period(@time_period))
+        .group("DATE(start_time)")
+        .sum(:total_cost)
+        .transform_values(&:to_f)
+    end
+
+    def calculate_average_llm_latency(time_range)
+      generations = Observ::Generation
+        .joins(trace: :observ_session)
+        .where("observ_sessions.start_time >= ?", time_range)
+        .where.not(end_time: nil)
+
+      return 0 if generations.empty?
+
+      total_duration = generations.sum { |g| ((g.end_time - g.start_time) * 1000).round(2) }
+      (total_duration / generations.count).round(0)
+    end
+
+    def calculate_success_rate(time_range)
+      total = Observ::Generation
+        .joins(trace: :observ_session)
+        .where("observ_sessions.start_time >= ?", time_range)
+        .count
+
+      return 100.0 if total.zero?
+
+      failed = Observ::Generation
+        .joins(trace: :observ_session)
+        .where("observ_sessions.start_time >= ?", time_range)
+        .where.not(status_message: nil)
+        .count
+
+      (((total - failed).to_f / total) * 100).round(1)
+    end
+
+    def calculate_avg_cost_per_call(sessions)
+      time_range = time_range_from_period(@time_period)
+      total_cost = Observ::Trace.joins(:observ_session).where("observ_sessions.start_time >= ?", time_range).sum(:total_cost).to_f
+      total_calls = Observ::Generation.joins(trace: :observ_session).where("observ_sessions.start_time >= ?", time_range).count
+
+      return 0.0 if total_calls.zero?
+
+      (total_cost / total_calls).round(6)
+    end
+
+    def calculate_trends(current, previous)
+      {
+        sessions: calculate_percentage_change(current[:total_sessions], previous[:total_sessions]),
+        tokens: calculate_percentage_change(current[:total_tokens], previous[:total_tokens]),
+        cost: calculate_percentage_change(current[:total_cost], previous[:total_cost])
+      }
+    end
+
+    def calculate_percentage_change(current, previous)
+      return 0 if previous.zero?
+      (((current - previous).to_f / previous) * 100).round(1)
+    end
+
+    def calculate_metrics_by_agent
+      time_range = time_range_from_period(@time_period)
+      sessions = Observ::Session.where("start_time >= ?", time_range)
+
+      sessions.group_by { |s| s.metadata&.dig("agent_type") || "Unknown" }.map do |agent_type, agent_sessions|
+        session_ids = agent_sessions.map(&:id)
+        traces = Observ::Trace.where(observ_session_id: session_ids)
+        generations = Observ::Generation.joins(:trace).where(observ_traces: { observ_session_id: session_ids })
+
+        {
+          agent_type: agent_type,
+          sessions: agent_sessions.count,
+          traces: traces.count,
+          llm_calls: generations.count,
+          tokens: traces.sum(:total_tokens),
+          cost: traces.sum(:total_cost).to_f
+        }
+      end.sort_by { |m| -m[:cost] }
+    end
+
+    def time_range_from_period(period)
+      case period
+      when "24h" then 24.hours.ago
+      when "7d" then 7.days.ago
+      when "30d" then 30.days.ago
+      else 100.years.ago
+      end
+    end
+  end
+end

data/app/controllers/observ/dataset_items_controller.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetItemsController < ApplicationController
+    before_action :set_dataset
+    before_action :set_item, only: [ :edit, :update, :destroy ]
+
+    def index
+      @items = @dataset.items.order(created_at: :desc)
+
+      if params[:status].present?
+        @items = @items.where(status: params[:status])
+      end
+
+      @items = @items.page(params[:page]).per(Observ.config.pagination_per_page)
+    end
+
+    def new
+      @item = @dataset.items.build
+    end
+
+    def create
+      @item = @dataset.items.build(item_params)
+
+      if @item.save
+        redirect_to dataset_path(@dataset, tab: "items"),
+          notice: "Item added to dataset successfully."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+    end
+
+    def update
+      if @item.update(item_params)
+        redirect_to dataset_path(@dataset, tab: "items"),
+          notice: "Item updated successfully."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      @item.destroy
+      redirect_to dataset_path(@dataset, tab: "items"),
+        notice: "Item removed from dataset."
+    end
+
+    private
+
+    def set_dataset
+      @dataset = Observ::Dataset.find(params[:dataset_id])
+    end
+
+    def set_item
+      @item = @dataset.items.find(params[:id])
+    end
+
+    def item_params
+      permitted = params.require(:observ_dataset_item).permit(:status, :expected_output_text)
+
+      # Handle input as JSON text
+      if params[:observ_dataset_item][:input_text].present?
+        permitted[:input] = parse_json_field(params[:observ_dataset_item][:input_text])
+      end
+
+      # Handle expected_output as JSON text
+      if params[:observ_dataset_item][:expected_output_text].present?
+        permitted[:expected_output] = parse_json_field(params[:observ_dataset_item][:expected_output_text])
+      end
+
+      permitted.except(:expected_output_text)
+    end
+
+    def parse_json_field(text)
+      return text if text.blank?
+      JSON.parse(text)
+    rescue JSON::ParserError
+      # If it's not valid JSON, treat it as a plain string
+      text
+    end
+  end
+end

data/app/controllers/observ/dataset_run_items_controller.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetRunItemsController < ApplicationController
+    before_action :set_dataset
+    before_action :set_run
+    before_action :set_run_item
+
+    def details_drawer
+      # The drawer template will use @run_item
+    end
+
+    def score_drawer
+      # Renders the score drawer partial
+    end
+
+    def score
+      value = params[:value].to_i == 1 ? 1.0 : 0.0
+
+      score = @run_item.scores.find_or_initialize_by(name: "manual", source: :manual)
+      score.assign_attributes(
+        trace: @run_item.trace,
+        value: value,
+        data_type: :boolean,
+        comment: params[:comment],
+        created_by: params[:created_by]
+      )
+
+      if score.save
+        respond_to do |format|
+          format.turbo_stream do
+            render turbo_stream: [
+              turbo_stream.replace("run-item-#{@run_item.id}-scores",
+                partial: "observ/dataset_run_items/scores_cell",
+                locals: { run_item: @run_item }),
+              turbo_stream.update("drawer-content",
+                partial: "observ/dataset_run_items/score_close_drawer")
+            ]
+          end
+          format.html do
+            if params[:review_mode].present?
+              redirect_to review_dataset_run_path(@dataset, @run),
+                notice: "Score saved!"
+            else
+              redirect_to dataset_run_path(@dataset, @run)
+            end
+          end
+        end
+      else
+        respond_to do |format|
+          format.turbo_stream do
+            render turbo_stream: turbo_stream.replace(
+              "drawer-content",
+              partial: "observ/dataset_run_items/score_drawer",
+              locals: { run_item: @run_item, error: score.errors.full_messages.join(", ") }
+            )
+          end
+          format.html do
+            if params[:review_mode].present?
+              redirect_to review_dataset_run_path(@dataset, @run),
+                alert: "Failed to save score: #{score.errors.full_messages.join(', ')}"
+            else
+              redirect_to dataset_run_path(@dataset, @run), alert: "Failed to save score."
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def set_dataset
+      @dataset = Observ::Dataset.find(params[:dataset_id])
+    end
+
+    def set_run
+      @run = @dataset.runs.find(params[:run_id])
+    end
+
+    def set_run_item
+      @run_item = @run.run_items.find(params[:id])
+    end
+  end
+end

data/app/controllers/observ/dataset_runs_controller.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetRunsController < ApplicationController
+    before_action :set_dataset
+    before_action :set_run, only: [ :show, :destroy, :run_evaluators, :review ]
+
+    def index
+      @runs = @dataset.runs.order(created_at: :desc)
+
+      if params[:status].present?
+        @runs = @runs.where(status: params[:status])
+      end
+
+      @runs = @runs.page(params[:page]).per(Observ.config.pagination_per_page)
+    end
+
+    def show
+      @run_items = @run.run_items
+        .includes(:dataset_item, :trace)
+        .order(created_at: :asc)
+        .page(params[:page])
+        .per(Observ.config.pagination_per_page)
+    end
+
+    def new
+      @run = @dataset.runs.build
+    end
+
+    def create
+      @run = @dataset.runs.build(run_params)
+
+      if @run.save
+        # Initialize run items for all active dataset items
+        @run.initialize_run_items!
+
+        # Queue the run for async execution
+        Observ::DatasetRunnerJob.perform_later(@run.id)
+
+        redirect_to dataset_run_path(@dataset, @run),
+          notice: "Run '#{@run.name}' created with #{@run.total_items} items. Execution will begin shortly."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      name = @run.name
+      @run.destroy
+      redirect_to dataset_path(@dataset, tab: "runs"),
+        notice: "Run '#{name}' deleted successfully."
+    end
+
+    def run_evaluators
+      evaluator_configs = @dataset.metadata&.dig("evaluators") || [ { "type" => "exact_match" } ]
+      Observ::EvaluatorRunnerService.new(@run, evaluator_configs: evaluator_configs).call
+
+      redirect_to dataset_run_path(@dataset, @run),
+        notice: "Evaluators completed. #{@run.items_with_scores_count} items scored."
+    end
+
+    def review
+      @run_item = next_item_to_review(@run)
+
+      if @run_item.nil?
+        redirect_to dataset_run_path(@dataset, @run),
+          notice: "All items have been reviewed!"
+        return
+      end
+
+      @progress = review_progress(@run)
+      @existing_manual = @run_item.score_for("manual", source: :manual)
+    end
+
+    private
+
+    def next_item_to_review(run, after_item: nil)
+      items = run.run_items.succeeded.includes(:dataset_item, :scores).order(:id)
+
+      if after_item
+        items = items.where("id > ?", after_item.id)
+      end
+
+      # Find first item without a manual score
+      items.find { |item| item.score_for("manual", source: :manual).nil? } ||
+        # If all scored after current, wrap around to find any unscored
+        (after_item ? run.run_items.succeeded.includes(:dataset_item, :scores).order(:id).find { |item| item.score_for("manual", source: :manual).nil? } : nil)
+    end
+
+    def review_progress(run)
+      succeeded_items = run.run_items.succeeded
+      total = succeeded_items.count
+      scored = succeeded_items.joins(:scores).where(observ_scores: { name: "manual", source: :manual }).distinct.count
+      { scored: scored, total: total }
+    end
+
+    def set_dataset
+      @dataset = Observ::Dataset.find(params[:dataset_id])
+    end
+
+    def set_run
+      @run = @dataset.runs.find(params[:id])
+    end
+
+    def run_params
+      # form_with generates param key based on model class name without module prefix
+      params.require(:dataset_run).permit(:name, :description)
+    end
+  end
+end

data/app/controllers/observ/datasets_controller.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Observ
+  class DatasetsController < ApplicationController
+    before_action :set_dataset, only: [ :show, :edit, :update, :destroy ]
+
+    def index
+      @datasets = Observ::Dataset.order(updated_at: :desc)
+
+      if params[:search].present?
+        @datasets = @datasets.where("name LIKE ?", "%#{params[:search]}%")
+      end
+
+      @datasets = @datasets.page(params[:page]).per(Observ.config.pagination_per_page)
+    end
+
+    def show
+      @items = @dataset.items.order(created_at: :desc).page(params[:items_page]).per(10)
+      @runs = @dataset.runs.order(created_at: :desc).page(params[:runs_page]).per(10)
+      @active_tab = params[:tab] || "items"
+    end
+
+    def new
+      @dataset = Observ::Dataset.new
+      @agents = available_agents
+    end
+
+    def create
+      @dataset = Observ::Dataset.new(dataset_params)
+
+      if @dataset.save
+        redirect_to dataset_path(@dataset), notice: "Dataset '#{@dataset.name}' created successfully."
+      else
+        @agents = available_agents
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+      @agents = available_agents
+    end
+
+    def update
+      if @dataset.update(dataset_params)
+        redirect_to dataset_path(@dataset), notice: "Dataset '#{@dataset.name}' updated successfully."
+      else
+        @agents = available_agents
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      name = @dataset.name
+      @dataset.destroy
+      redirect_to datasets_path, notice: "Dataset '#{name}' deleted successfully."
+    end
+
+    private
+
+    def set_dataset
+      @dataset = Observ::Dataset.find(params[:id])
+    end
+
+    def dataset_params
+      params.require(:observ_dataset).permit(:name, :description, :agent_class)
+    end
+
+    def available_agents
+      Observ::AgentProvider.all_agents.map do |agent|
+        [ agent.display_name, agent.name ]
+      end
+    end
+  end
+end

data/app/controllers/observ/messages_controller.rb

@@ -0,0 +1,26 @@
+module Observ
+  class MessagesController < ApplicationController
+    before_action :set_chat
+
+    def create
+      return unless content.present?
+
+      ChatResponseJob.perform_later(@chat.id, content)
+
+      respond_to do |format|
+        format.turbo_stream
+        format.html { redirect_to chat_path(@chat) }
+      end
+    end
+
+    private
+
+    def set_chat
+      @chat = Chat.find(params[:chat_id])
+    end
+
+    def content
+      params[:message][:content]
+    end
+  end
+end

data/app/controllers/observ/observations_controller.rb

@@ -0,0 +1,59 @@
+module Observ
+  class ObservationsController < ApplicationController
+    def index
+      @observations = Observ::Observation
+        .includes(:trace)
+        .order(start_time: :desc)
+        .page(params[:page])
+        .per(Observ.config.pagination_per_page)
+
+      apply_filters if params[:filter].present?
+    end
+
+    def show
+      @observation = Observ::Observation.includes(:trace).find(params[:id])
+
+      if @observation.is_a?(Observ::Generation)
+        render :show_generation
+      else
+        render :show_span
+      end
+    end
+
+    def generations
+      @observations = Observ::Generation
+        .includes(:trace)
+        .order(start_time: :desc)
+        .page(params[:page])
+        .per(Observ.config.pagination_per_page)
+
+      render :index
+    end
+
+    def spans
+      @observations = Observ::Span
+        .includes(:trace)
+        .order(start_time: :desc)
+        .page(params[:page])
+        .per(Observ.config.pagination_per_page)
+
+      render :index
+    end
+
+    private
+
+    def apply_filters
+      @observations = @observations.where(type: params[:filter][:type]) if params[:filter][:type].present?
+      @observations = @observations.where(name: params[:filter][:name]) if params[:filter][:name].present?
+      @observations = @observations.where(model: params[:filter][:model]) if params[:filter][:model].present?
+
+      if params[:filter][:start_date].present?
+        @observations = @observations.where("start_time >= ?", params[:filter][:start_date])
+      end
+
+      if params[:filter][:end_date].present?
+        @observations = @observations.where("start_time <= ?", params[:filter][:end_date])
+      end
+    end
+  end
+end