rubyllm-observ 0.6.6 → 0.6.7

data/app/assets/stylesheets/observ/_chat.scss

@@ -129,6 +129,149 @@
           color: inherit;
         }
       }
+
+      // Markdown lists
+      ul,
+      ol {
+        margin: 0.5rem 0;
+        padding-left: 1.5rem;
+
+        li {
+          margin-bottom: 0.25rem;
+
+          &:last-child {
+            margin-bottom: 0;
+          }
+        }
+      }
+
+      ul {
+        list-style-type: disc;
+
+        ul {
+          list-style-type: circle;
+
+          ul {
+            list-style-type: square;
+          }
+        }
+      }
+
+      ol {
+        list-style-type: decimal;
+      }
+
+      // Markdown blockquotes
+      blockquote {
+        margin: 0.5rem 0;
+        padding: 0.5rem 1rem;
+        border-left: 3px solid $observ-border-strong;
+        background-color: $observ-bg-elevated;
+        color: $observ-text-secondary;
+        font-style: italic;
+
+        p {
+          margin: 0;
+        }
+      }
+
+      // Markdown headers
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6 {
+        margin: 1rem 0 0.5rem 0;
+        font-weight: 600;
+        line-height: 1.3;
+        color: $observ-text-primary;
+
+        &:first-child {
+          margin-top: 0;
+        }
+      }
+
+      h1 {
+        font-size: 1.5em;
+      }
+
+      h2 {
+        font-size: 1.3em;
+      }
+
+      h3 {
+        font-size: 1.15em;
+      }
+
+      h4,
+      h5,
+      h6 {
+        font-size: 1em;
+      }
+
+      // Markdown tables
+      table {
+        width: 100%;
+        border-collapse: collapse;
+        margin: 0.5rem 0;
+        font-size: 0.9em;
+
+        th,
+        td {
+          padding: 0.5rem;
+          border: 1px solid $observ-border-color;
+          text-align: left;
+        }
+
+        th {
+          background-color: $observ-bg-elevated;
+          font-weight: 600;
+        }
+
+        tr:nth-child(even) {
+          background-color: rgba($observ-bg-elevated, 0.5);
+        }
+      }
+
+      // Markdown horizontal rule
+      hr {
+        margin: 1rem 0;
+        border: none;
+        border-top: 1px solid $observ-border-color;
+      }
+
+      // Markdown strikethrough
+      del {
+        text-decoration: line-through;
+        color: $observ-text-muted;
+      }
+
+      // Markdown links
+      a {
+        color: $observ-primary;
+        text-decoration: none;
+
+        &:hover {
+          text-decoration: underline;
+        }
+      }
+
+      // Markdown highlight (==text==)
+      mark {
+        background-color: rgba($observ-warning, 0.3);
+        padding: 0.1rem 0.2rem;
+        border-radius: $observ-border-radius-sm;
+      }
+
+      // Strong and emphasis
+      strong {
+        font-weight: 600;
+      }
+
+      em {
+        font-style: italic;
+      }
     }
 
     &__tool-calls {
@@ -160,6 +303,62 @@
     }
   }
 
+  // ==========================================================================
+  // TYPING INDICATOR (AI thinking)
+  // ==========================================================================
+  .observ-typing-indicator {
+    display: flex;
+    align-items: center;
+    gap: $observ-spacing-sm;
+    padding: $observ-spacing-md $observ-spacing-lg;
+    background-color: $observ-bg-surface;
+    border-left: 3px solid $observ-success;
+    border-radius: $observ-border-radius-sm;
+    
+    &__dots {
+      display: flex;
+      align-items: center;
+      gap: 4px;
+      
+      span {
+        width: 8px;
+        height: 8px;
+        background-color: $observ-success;
+        border-radius: 50%;
+        animation: observ-typing-bounce 1.4s ease-in-out infinite;
+        
+        &:nth-child(1) {
+          animation-delay: 0s;
+        }
+        
+        &:nth-child(2) {
+          animation-delay: 0.2s;
+        }
+        
+        &:nth-child(3) {
+          animation-delay: 0.4s;
+        }
+      }
+    }
+    
+    &__text {
+      font-size: $observ-font-size-sm;
+      color: $observ-text-secondary;
+      font-style: italic;
+    }
+  }
+  
+  @keyframes observ-typing-bounce {
+    0%, 60%, 100% {
+      transform: translateY(0);
+      opacity: 0.4;
+    }
+    30% {
+      transform: translateY(-6px);
+      opacity: 1;
+    }
+  }
+
   // ==========================================================================
   // MESSAGE FORM (chat input)
   // ==========================================================================

data/app/assets/stylesheets/observ/_config_editor.scss

@@ -0,0 +1,119 @@
+@import 'variables';
+@import 'namespace';
+
+@include observ-scoped {
+  .observ-config-editor {
+    // Main container
+    &__fieldset {
+      border: 1px solid $observ-border-color;
+      border-radius: $observ-border-radius;
+      padding: $observ-spacing-lg;
+      margin: 0 0 $observ-spacing-lg 0;
+      background-color: $observ-bg-surface;
+    }
+
+    &__legend {
+      font-size: $observ-font-size-sm;
+      font-weight: 600;
+      color: $observ-text-secondary;
+      text-transform: uppercase;
+      letter-spacing: 0.05em;
+      padding: 0 $observ-spacing-sm;
+    }
+
+    &__row {
+      margin-bottom: $observ-spacing-lg;
+
+      &:last-child {
+        margin-bottom: 0;
+      }
+    }
+
+    &__input-group {
+      display: flex;
+      align-items: center;
+      gap: $observ-spacing-sm;
+    }
+
+    &__number-input {
+      max-width: 150px;
+    }
+
+    &__range-hint {
+      font-size: $observ-font-size-xs;
+      color: $observ-text-muted;
+    }
+
+    // Advanced section (collapsible)
+    &__advanced {
+      border: 1px solid $observ-border-color;
+      border-radius: $observ-border-radius;
+      background-color: $observ-bg-surface;
+
+      &[open] {
+        .observ-config-editor__advanced-summary {
+          border-bottom: 1px solid $observ-border-subtle;
+        }
+      }
+    }
+
+    &__advanced-summary {
+      padding: $observ-spacing-md $observ-spacing-lg;
+      cursor: pointer;
+      font-weight: 500;
+      color: $observ-text-secondary;
+      transition: $observ-transition;
+      list-style: none;
+
+      &::-webkit-details-marker {
+        display: none;
+      }
+
+      &::before {
+        content: '▸ ';
+        display: inline-block;
+        transition: transform 0.2s ease;
+      }
+
+      &:hover {
+        color: $observ-text-primary;
+        background-color: $observ-bg-hover;
+      }
+    }
+
+    &__advanced[open] &__advanced-summary::before {
+      transform: rotate(90deg);
+    }
+
+    &__advanced-content {
+      padding: $observ-spacing-lg;
+    }
+
+    // Validation status
+    &__status {
+      margin-top: $observ-spacing-sm;
+      padding: $observ-spacing-sm $observ-spacing-md;
+      border-radius: $observ-border-radius-sm;
+      font-size: $observ-font-size-sm;
+      font-family: $observ-font-family-mono;
+
+      &--success {
+        background-color: rgba($observ-success, 0.1);
+        color: lighten($observ-success, 15%);
+      }
+
+      &--error {
+        background-color: rgba($observ-danger, 0.1);
+        color: lighten($observ-danger, 15%);
+      }
+
+      &--valid {
+        color: lighten($observ-success, 15%);
+      }
+
+      &--invalid {
+        color: lighten($observ-danger, 15%);
+      }
+    }
+  }
+}

data/app/assets/stylesheets/observ/application.scss

@@ -12,6 +12,7 @@
 @import 'drawer';
 @import 'annotations';
 @import 'prompts';
+@import 'config_editor';
 @import 'datasets';
 @import 'filters';
 @import 'json_viewer';

data/app/controllers/observ/dataset_items_controller.rb

@@ -26,7 +26,7 @@ module Observ
         redirect_to dataset_path(@dataset, tab: "items"),
           notice: "Item added to dataset successfully."
       else
-        render :new, status: :unprocessable_entity
+        render :new, status: :unprocessable_content
       end
     end
 
@@ -38,7 +38,7 @@ module Observ
         redirect_to dataset_path(@dataset, tab: "items"),
           notice: "Item updated successfully."
       else
-        render :edit, status: :unprocessable_entity
+        render :edit, status: :unprocessable_content
       end
     end
 

data/app/controllers/observ/dataset_runs_controller.rb

@@ -40,7 +40,7 @@ module Observ
         redirect_to dataset_run_path(@dataset, @run),
           notice: "Run '#{@run.name}' created with #{@run.total_items} items. Execution will begin shortly."
       else
-        render :new, status: :unprocessable_entity
+        render :new, status: :unprocessable_content
       end
     end
 

data/app/controllers/observ/datasets_controller.rb

@@ -32,7 +32,7 @@ module Observ
         redirect_to dataset_path(@dataset), notice: "Dataset '#{@dataset.name}' created successfully."
       else
         @agents = available_agents
-        render :new, status: :unprocessable_entity
+        render :new, status: :unprocessable_content
       end
     end
 
@@ -45,7 +45,7 @@ module Observ
         redirect_to dataset_path(@dataset), notice: "Dataset '#{@dataset.name}' updated successfully."
       else
         @agents = available_agents
-        render :edit, status: :unprocessable_entity
+        render :edit, status: :unprocessable_content
       end
     end
 

data/app/controllers/observ/messages_controller.rb

@@ -5,7 +5,11 @@ module Observ
     def create
       return unless content.present?
 
-      ChatResponseJob.perform_later(@chat.id, content)
+      # Create user message synchronously so it appears immediately
+      @message = @chat.messages.create!(role: :user, content: content)
+
+      # Enqueue job to get assistant response (will broadcast when complete)
+      ChatResponseJob.perform_later(@chat.id, @message.id)
 
       respond_to do |format|
         format.turbo_stream

data/app/controllers/observ/prompts_controller.rb

@@ -103,7 +103,13 @@ module Observ
       # Parse config JSON string before updating
       update_params = prompt_params.except(:name, :version, :promote_to_production)
       if update_params[:config].present?
-        update_params[:config] = parse_config(update_params[:config])
+        parsed = parse_config(update_params[:config])
+        if parsed.nil?
+          # JSON parsing failed, re-render form with error
+          render :edit, status: :unprocessable_content
+          return
+        end
+        update_params[:config] = parsed
       end
 
       if @prompt.update(update_params)
@@ -180,8 +186,10 @@ module Observ
     def parse_config(config_string)
       return {} if config_string.blank?
       JSON.parse(config_string)
-    rescue JSON::ParserError
-      {}
+    rescue JSON::ParserError => e
+      # Add error to flash for display
+      flash.now[:alert] = "Invalid JSON configuration: #{e.message}"
+      nil
     end
 
     def current_user_identifier

data/app/controllers/observ/scores_controller.rb

@@ -26,7 +26,7 @@ module Observ
         end
       else
         respond_to do |format|
-          format.turbo_stream { render :create_error, status: :unprocessable_entity }
+          format.turbo_stream { render :create_error, status: :unprocessable_content }
           format.html { redirect_back(fallback_location: root_path, alert: "Failed to save score.") }
         end
       end

data/app/controllers/observ/traces_controller.rb

@@ -60,7 +60,7 @@ module Observ
       else
         @datasets = Observ::Dataset.order(:name)
         flash.now[:alert] = "Failed to add trace to dataset: #{@item.errors.full_messages.join(', ')}"
-        render :add_to_dataset_drawer, status: :unprocessable_entity
+        render :add_to_dataset_drawer, status: :unprocessable_content
       end
     end
 

data/app/helpers/observ/application_helper.rb

@@ -6,5 +6,6 @@ module Observ
     include Observ::PaginationHelper
     include Observ::DatasetsHelper
     include Observ::ReviewsHelper
+    include Observ::MarkdownHelper
   end
 end

data/app/helpers/observ/markdown_helper.rb

@@ -0,0 +1,29 @@
+require "redcarpet"
+
+module Observ
+  module MarkdownHelper
+    def render_markdown(content)
+      return "" if content.blank?
+
+      markdown_renderer.render(content).html_safe
+    end
+
+    private
+
+    def markdown_renderer
+      @markdown_renderer ||= Redcarpet::Markdown.new(
+        Redcarpet::Render::HTML.new(
+          hard_wrap: true,
+          link_attributes: { target: "_blank", rel: "noopener noreferrer" }
+        ),
+        autolink: true,
+        fenced_code_blocks: true,
+        tables: true,
+        strikethrough: true,
+        no_intra_emphasis: true,
+        highlight: true,
+        quote: true
+      )
+    end
+  end
+end

data/app/helpers/observ/prompts_helper.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Observ
+  module PromptsHelper
+    # Returns model options grouped by provider for use with grouped_collection_select
+    # or manually building optgroups
+    def chat_model_options_grouped
+      return [] unless defined?(RubyLLM) && RubyLLM.respond_to?(:models)
+
+      RubyLLM.models.chat_models
+        .group_by(&:provider)
+        .sort_by { |provider, _| provider }
+        .map do |provider, models|
+          [
+            provider.titleize,
+            models.sort_by(&:display_name).map { |m| [ m.display_name, m.id ] }
+          ]
+        end
+    rescue StandardError => e
+      Rails.logger.warn "[Observ] Failed to load RubyLLM models: #{e.message}"
+      []
+    end
+
+    # Extract config value with fallback
+    def config_value(prompt, key, default = nil)
+      config = prompt_config_hash(prompt)
+      return default unless config.is_a?(Hash)
+
+      config[key.to_s] || config[key.to_sym] || default
+    end
+
+    private
+
+    # Extract config hash from prompt or form object
+    def prompt_config_hash(prompt)
+      return {} unless prompt
+
+      # Handle PromptForm which has config as string
+      if prompt.respond_to?(:parsed_config)
+        prompt.parsed_config
+      elsif prompt.respond_to?(:config)
+        prompt.config
+      else
+        {}
+      end
+    end
+  end
+end

data/app/jobs/observ/moderation_guardrail_job.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Observ
+  class ModerationGuardrailJob < ApplicationJob
+    queue_as :moderation
+
+    # Retry configuration
+    retry_on StandardError, wait: :polynomially_longer, attempts: 3
+    discard_on ActiveRecord::RecordNotFound
+
+    # Process a single trace or session
+    #
+    # @param trace_id [Integer] ID of the trace to moderate
+    # @param session_id [Integer] ID of the session to moderate
+    # @param options [Hash] Options for moderation
+    # @option options [Boolean] :moderate_input Whether to moderate input (default: true)
+    # @option options [Boolean] :moderate_output Whether to moderate output (default: true)
+    # @option options [Boolean] :aggregate Whether to moderate aggregated session content
+    def perform(trace_id: nil, session_id: nil, **options)
+      if trace_id
+        moderate_trace(trace_id, options)
+      elsif session_id
+        moderate_session(session_id, options)
+      else
+        Rails.logger.warn "[ModerationGuardrailJob] No trace_id or session_id provided"
+      end
+    end
+
+    # Class method to enqueue moderation for traces matching criteria
+    #
+    # @param scope [ActiveRecord::Relation] Scope of traces to moderate
+    # @param sample_percentage [Integer] Percentage of traces to sample (1-100)
+    def self.enqueue_for_scope(scope, sample_percentage: 100)
+      traces = scope.left_joins(:review_item)
+                    .where(observ_review_items: { id: nil })
+
+      if sample_percentage < 100
+        sample_size = (traces.count * sample_percentage / 100.0).ceil
+        traces = traces.order("RANDOM()").limit(sample_size)
+      end
+
+      traces.find_each do |trace|
+        perform_later(trace_id: trace.id)
+      end
+    end
+
+    # Enqueue moderation for user-facing sessions only
+    #
+    # @param since [Time] Only process sessions created after this time
+    def self.enqueue_user_facing(since: 1.hour.ago)
+      Observ::Session
+        .where(created_at: since..)
+        .where("metadata->>'user_facing' = ?", "true")
+        .find_each do |session|
+          perform_later(session_id: session.id)
+        end
+    end
+
+    # Enqueue moderation for specific agent types
+    #
+    # @param agent_types [Array<String>] Agent types to moderate
+    # @param since [Time] Only process sessions created after this time
+    def self.enqueue_for_agent_types(agent_types, since: 1.hour.ago)
+      Observ::Session
+        .where(created_at: since..)
+        .where("metadata->>'agent_type' IN (?)", agent_types)
+        .find_each do |session|
+          perform_later(session_id: session.id)
+        end
+    end
+
+    private
+
+    def moderate_trace(trace_id, options)
+      trace = Observ::Trace.find(trace_id)
+
+      service = ModerationGuardrailService.new
+      result = service.evaluate_trace(
+        trace,
+        moderate_input: options.fetch(:moderate_input, true),
+        moderate_output: options.fetch(:moderate_output, true)
+      )
+
+      log_result("Trace #{trace_id}", result)
+    end
+
+    def moderate_session(session_id, options)
+      session = Observ::Session.find(session_id)
+
+      service = ModerationGuardrailService.new
+
+      if options[:aggregate]
+        # Moderate aggregated session content
+        result = service.evaluate_session_content(session)
+        log_result("Session #{session_id} (aggregated)", result)
+      else
+        # Moderate each trace individually
+        results = service.evaluate_session(session)
+        flagged_count = results.count(&:flagged?)
+        Rails.logger.info "[ModerationGuardrailJob] Session #{session_id}: #{flagged_count}/#{results.size} traces flagged"
+      end
+    end
+
+    def log_result(identifier, result)
+      case result.action
+      when :flagged
+        Rails.logger.info "[ModerationGuardrailJob] #{identifier} flagged (#{result.priority}): #{result.details[:flagged_categories]}"
+      when :skipped
+        Rails.logger.debug "[ModerationGuardrailJob] #{identifier} skipped: #{result.reason}"
+      when :passed
+        Rails.logger.debug "[ModerationGuardrailJob] #{identifier} passed moderation"
+      end
+    end
+  end
+end

data/app/models/observ/embedding.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Observ
+  class Embedding < Observation
+    # Set input texts for the embedding call
+    def set_input(texts)
+      update!(
+        input: texts.is_a?(Array) ? texts.to_json : texts
+      )
+    end
+
+    def finalize(output:, usage: {}, cost_usd: 0.0, status_message: nil)
+      merged_usage = (self.usage || {}).merge(usage.stringify_keys)
+
+      update!(
+        output: output.is_a?(String) ? output : output.to_json,
+        usage: merged_usage,
+        cost_usd: cost_usd,
+        end_time: Time.current,
+        status_message: status_message
+      )
+    end
+
+    # Embedding-specific helpers
+    def input_tokens
+      usage&.dig("input_tokens") || 0
+    end
+
+    def total_tokens
+      input_tokens # Embeddings only have input tokens
+    end
+
+    def batch_size
+      metadata&.dig("batch_size") || 1
+    end
+
+    def dimensions
+      metadata&.dig("dimensions")
+    end
+
+    def vectors_count
+      metadata&.dig("vectors_count") || 1
+    end
+  end
+end