rubyllm-observ 0.6.6 → 0.6.8

data/app/services/observ/transcription_instrumenter.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Observ
+  class TranscriptionInstrumenter
+    attr_reader :session, :context
+
+    def initialize(session, context: {})
+      @session = session
+      @context = context
+      @original_transcribe_method = nil
+      @instrumented = false
+    end
+
+    def instrument!
+      return if @instrumented
+
+      wrap_transcribe_method
+      @instrumented = true
+
+      Rails.logger.info "[Observability] Instrumented RubyLLM.transcribe for session #{session.session_id}"
+    end
+
+    def uninstrument!
+      return unless @instrumented
+      return unless @original_transcribe_method
+
+      RubyLLM.define_singleton_method(:transcribe, @original_transcribe_method)
+      @instrumented = false
+
+      Rails.logger.info "[Observability] Uninstrumented RubyLLM.transcribe"
+    end
+
+    private
+
+    def wrap_transcribe_method
+      return if @original_transcribe_method
+
+      @original_transcribe_method = RubyLLM.method(:transcribe)
+      instrumenter = self
+
+      RubyLLM.define_singleton_method(:transcribe) do |*args, **kwargs|
+        instrumenter.send(:handle_transcribe_call, args, kwargs)
+      end
+    end
+
+    def handle_transcribe_call(args, kwargs)
+      audio_path = args[0]
+      model_id = kwargs[:model] || default_transcription_model
+      language = kwargs[:language]
+
+      trace = session.create_trace(
+        name: "transcription",
+        input: { audio_path: audio_path.to_s },
+        metadata: @context.merge(
+          model: model_id,
+          language: language
+        ).compact
+      )
+
+      transcription_obs = trace.create_transcription(
+        name: "transcribe",
+        model: model_id,
+        metadata: {
+          language: language,
+          has_diarization: kwargs[:speaker_names].present?
+        }.compact
+      )
+
+      result = @original_transcribe_method.call(*args, **kwargs)
+
+      finalize_transcription(transcription_obs, result)
+      trace.finalize(
+        output: format_output(result),
+        metadata: extract_trace_metadata(result)
+      )
+
+      result
+    rescue StandardError => e
+      handle_error(e, trace, transcription_obs)
+      raise
+    end
+
+    def finalize_transcription(transcription_obs, result)
+      cost = calculate_cost(result)
+
+      transcription_obs.finalize(
+        output: format_output(result),
+        usage: {},
+        cost_usd: cost
+      )
+
+      transcription_obs.update!(
+        input: result.text&.truncate(1000),
+        metadata: transcription_obs.metadata.merge(
+          audio_duration_s: result.duration,
+          language: result.respond_to?(:language) ? result.language : nil,
+          segments_count: result.segments&.count || 0,
+          speakers_count: extract_speakers_count(result),
+          has_diarization: has_diarization?(result)
+        ).compact
+      )
+    end
+
+    def calculate_cost(result)
+      model_id = result.model
+      return 0.0 unless model_id
+
+      model_info = RubyLLM.models.find(model_id)
+      return 0.0 unless model_info
+
+      duration_minutes = (result.duration || 0) / 60.0
+
+      # Transcription models typically use per-minute pricing
+      if model_info.respond_to?(:audio_price_per_minute) && model_info.audio_price_per_minute
+        (duration_minutes * model_info.audio_price_per_minute).round(6)
+      elsif model_info.respond_to?(:input_price_per_million) && model_info.input_price_per_million
+        # Fallback: some models might use token-based pricing
+        # Estimate ~150 tokens per minute of audio
+        estimated_tokens = duration_minutes * 150
+        (estimated_tokens * model_info.input_price_per_million / 1_000_000.0).round(6)
+      else
+        0.0
+      end
+    rescue StandardError => e
+      Rails.logger.warn "[Observability] Failed to calculate transcription cost: #{e.message}"
+      0.0
+    end
+
+    def extract_speakers_count(result)
+      return nil unless has_diarization?(result)
+      return nil unless result.segments
+
+      result.segments.map { |s| s.respond_to?(:speaker) ? s.speaker : nil }.compact.uniq.count
+    end
+
+    def has_diarization?(result)
+      return false unless result.segments&.any?
+
+      result.segments.first.respond_to?(:speaker)
+    end
+
+    def format_output(result)
+      {
+        model: result.model,
+        text_length: result.text&.length || 0,
+        duration_s: result.duration,
+        segments_count: result.segments&.count || 0
+      }.compact
+    end
+
+    def extract_trace_metadata(result)
+      {
+        audio_duration_s: result.duration,
+        language: result.respond_to?(:language) ? result.language : nil
+      }.compact
+    end
+
+    def default_transcription_model
+      if RubyLLM.config.respond_to?(:default_transcription_model)
+        RubyLLM.config.default_transcription_model
+      else
+        "whisper-1"
+      end
+    end
+
+    def handle_error(error, trace, transcription_obs)
+      return unless trace
+
+      error_span = trace.create_span(
+        name: "error",
+        metadata: {
+          error_type: error.class.name,
+          level: "ERROR"
+        },
+        input: {
+          error_message: error.message,
+          backtrace: error.backtrace&.first(10)
+        }.to_json
+      )
+      error_span.finalize(output: { error_captured: true }.to_json)
+
+      transcription_obs&.update(status_message: "FAILED") rescue nil
+
+      Rails.logger.error "[Observability] Transcription error captured: #{error.class.name} - #{error.message}"
+    end
+  end
+end

data/app/validators/observ/prompt_config_validator.rb

@@ -77,7 +77,7 @@ module Observ
           @errors << "#{key} must be a string"
         end
       when :boolean
-        unless [ true, false ].include?(value)
+        unless [true, false].include?(value)
           @errors << "#{key} must be a boolean"
         end
       when :array
@@ -133,7 +133,7 @@ module Observ
     end
 
     def validate_unknown_keys
-      schema_keys = schema.keys.map { |k| [ k.to_s, k.to_sym ] }.flatten
+      schema_keys = schema.keys.map { |k| [k.to_s, k.to_sym] }.flatten
       config_keys = config.keys
 
       unknown_keys = config_keys - schema_keys
@@ -153,11 +153,11 @@ module Observ
 
     def value_with_key(key)
       if config.key?(key.to_s)
-        [ config[key.to_s], key.to_s ]
+        [config[key.to_s], key.to_s]
       elsif config.key?(key.to_sym)
-        [ config[key.to_sym], key.to_sym ]
+        [config[key.to_sym], key.to_sym]
       else
-        [ nil, nil ]
+        [nil, nil]
       end
     end
 

data/app/views/observ/chats/show.html.erb

@@ -19,6 +19,15 @@
         <%= render message %>
       <% end %>
     </div>
+    
+    <div id="typing-indicator" class="observ-typing-indicator" style="display: none;">
+      <div class="observ-typing-indicator__dots">
+        <span></span>
+        <span></span>
+        <span></span>
+      </div>
+      <span class="observ-typing-indicator__text">AI is thinking...</span>
+    </div>
 
     <div class="observ-form-separator">
       <%= render "observ/messages/form", chat: @chat, message: @message %>

data/app/views/observ/messages/_message.html.erb

@@ -5,7 +5,7 @@
   </div>
   
   <div id="message_<%= message.id %>_content" class="observ-chat-message__content">
-    <%= simple_format(message.content) %>
+    <%= render_markdown(message.content) %>
   </div>
   
   <% if message.tool_call? %>

data/app/views/observ/messages/create.turbo_stream.erb

@@ -1,7 +1,5 @@
 <%= turbo_stream.append "messages" do %>
-  <% @chat.messages.last(2).each do |message| %>
-    <%= render message %>
-  <% end %>
+  <%= render @message %>
 <% end %>
 
 <%= turbo_stream.replace "new_message" do %>

data/app/views/observ/prompts/_config_editor.html.erb

@@ -0,0 +1,115 @@
+<%# 
+  Config Editor Partial
+  
+  Provides a hybrid interface for editing prompt configuration:
+  - Structured fields for common settings (model, temperature, max_tokens)
+  - Collapsible advanced section for raw JSON editing
+  
+  Local variables:
+    - prompt: The Observ::Prompt record or PromptForm being edited
+    - f: The form builder
+%>
+
+<% config = prompt_config_hash(prompt) %>
+<% config_json = config.present? ? JSON.pretty_generate(config) : "" %>
+
+<div class="observ-config-editor" 
+     data-controller="observ--config-editor"
+     data-observ--config-editor-known-keys-value='["model", "temperature", "max_tokens"]'>
+  
+  <!-- Common Settings -->
+  <fieldset class="observ-config-editor__fieldset">
+    <legend class="observ-config-editor__legend">Model Settings</legend>
+    
+    <!-- Model Select -->
+    <div class="observ-config-editor__row">
+      <label class="observ-form__label" for="config_model">Model</label>
+      <select id="config_model" 
+              class="observ-form__select"
+              data-observ--config-editor-target="model"
+              data-action="change->observ--config-editor#syncToJson">
+        <option value="">-- Select a model (optional) --</option>
+        <% chat_model_options_grouped.each do |provider, models| %>
+          <optgroup label="<%= provider %>">
+            <% models.each do |display_name, model_id| %>
+              <option value="<%= model_id %>" <%= 'selected' if config_value(prompt, :model) == model_id %>>
+                <%= display_name %>
+              </option>
+            <% end %>
+          </optgroup>
+        <% end %>
+      </select>
+      <p class="observ-form__hint">The LLM model to use for this prompt</p>
+    </div>
+    
+    <!-- Temperature -->
+    <div class="observ-config-editor__row">
+      <label class="observ-form__label" for="config_temperature">Temperature</label>
+      <div class="observ-config-editor__input-group">
+        <input type="number" 
+               id="config_temperature"
+               class="observ-form__input observ-config-editor__number-input"
+               min="0" 
+               max="2" 
+               step="0.1"
+               placeholder="0.7"
+               value="<%= config_value(prompt, :temperature) %>"
+               data-observ--config-editor-target="temperature"
+               data-action="input->observ--config-editor#syncToJson">
+        <span class="observ-config-editor__range-hint">0.0 - 2.0</span>
+      </div>
+      <p class="observ-form__hint">
+        Controls randomness: 0.0 = deterministic, 1.0 = balanced, 2.0 = creative
+      </p>
+    </div>
+    
+    <!-- Max Tokens -->
+    <div class="observ-config-editor__row">
+      <label class="observ-form__label" for="config_max_tokens">Max Tokens</label>
+      <div class="observ-config-editor__input-group">
+        <input type="number" 
+               id="config_max_tokens"
+               class="observ-form__input observ-config-editor__number-input"
+               min="1" 
+               max="128000"
+               placeholder="2000"
+               value="<%= config_value(prompt, :max_tokens) %>"
+               data-observ--config-editor-target="maxTokens"
+               data-action="input->observ--config-editor#syncToJson">
+      </div>
+      <p class="observ-form__hint">
+        Maximum number of tokens in the response
+      </p>
+    </div>
+  </fieldset>
+  
+  <!-- Advanced JSON Section (Collapsible) -->
+  <details class="observ-config-editor__advanced">
+    <summary class="observ-config-editor__advanced-summary">
+      Advanced Configuration (JSON)
+    </summary>
+    
+    <div class="observ-config-editor__advanced-content">
+      <p class="observ-form__hint">
+        Edit the raw JSON configuration. Changes here will sync with the fields above.
+        You can add custom parameters not available in the structured fields.
+      </p>
+      
+      <textarea class="observ-form__textarea observ-form__textarea--code"
+                rows="8"
+                placeholder='{"model": "gpt-4o", "temperature": 0.7, "max_tokens": 2000}'
+                data-observ--config-editor-target="jsonInput"
+                data-action="input->observ--config-editor#syncFromJson"><%= config_json %></textarea>
+      
+      <!-- Validation Status -->
+      <div class="observ-config-editor__status" 
+           data-observ--config-editor-target="status">
+      </div>
+    </div>
+  </details>
+  
+  <!-- Hidden field for form submission -->
+  <%= f.hidden_field :config, 
+        value: config_json,
+        data: { "observ--config-editor-target": "hiddenField" } %>
+</div>

data/app/views/observ/prompts/_form.html.erb

@@ -66,19 +66,8 @@
         </div>
       </div>
 
-      <!-- Configuration (JSON) -->
-      <div class="observ-form__group">
-        <%= f.label :config, "Configuration (JSON)", class: "observ-form__label" %>
-        <%= f.text_area :config,
-          value: prompt.config.present? ? JSON.pretty_generate(prompt.config) : "",
-          rows: 8,
-          placeholder: '{\n  "model": "gpt-4o",\n  "temperature": 0.7,\n  "max_tokens": 2000\n}',
-          class: "observ-form__textarea observ-form__textarea--code",
-          data: { controller: "json-editor" } %>
-        <p class="observ-form__hint">
-          Optional JSON configuration for model parameters and metadata
-        </p>
-      </div>
+      <!-- Configuration -->
+      <%= render 'observ/prompts/config_editor', prompt: prompt, f: f %>
 
       <!-- Actions -->
       <div class="observ-form__actions observ-form__actions--between">

data/app/views/observ/prompts/_new_form.html.erb

@@ -60,18 +60,8 @@
         </div>
       </div>
 
-      <!-- Configuration (JSON) -->
-      <div class="observ-form__group">
-        <%= f.label :config, "Configuration (JSON)", class: "observ-form__label" %>
-        <%= f.text_area :config,
-          rows: 8,
-          placeholder: '{\n  "model": "gpt-4o",\n  "temperature": 0.7,\n  "max_tokens": 2000\n}',
-          class: "observ-form__textarea observ-form__textarea--code",
-          data: { controller: "json-editor" } %>
-        <p class="observ-form__hint">
-          Optional JSON configuration for model parameters and metadata
-        </p>
-      </div>
+      <!-- Configuration -->
+      <%= render 'observ/prompts/config_editor', prompt: form, f: f %>
 
       <!-- Hidden field for from_version -->
       <%= f.hidden_field :from_version %>

data/config/routes.rb

@@ -7,22 +7,22 @@ Observ::Engine.routes.draw do
 
   # Chat routes - only available if Chat model exists in host app
   if defined?(::Chat) && ::Chat.respond_to?(:acts_as_chat)
-    resources :chats, only: [ :index, :new, :create, :show ] do
-      resources :messages, only: [ :create ]
+    resources :chats, only: [:index, :new, :create, :show] do
+      resources :messages, only: [:create]
     end
   end
 
-  resources :sessions, only: [ :index, :show ] do
+  resources :sessions, only: [:index, :show] do
     member do
       get :metrics
       get :drawer_test
       get :annotations_drawer
     end
-    resources :annotations, only: [ :index, :create, :destroy ]
-    resources :scores, only: [ :create, :destroy ]
+    resources :annotations, only: [:index, :create, :destroy]
+    resources :scores, only: [:create, :destroy]
   end
 
-  resources :traces, only: [ :index, :show ] do
+  resources :traces, only: [:index, :show] do
     collection do
       get :search
     end
@@ -32,11 +32,11 @@ Observ::Engine.routes.draw do
       get :add_to_dataset_drawer
       post :add_to_dataset
     end
-    resources :annotations, only: [ :index, :create, :destroy ]
-    resources :scores, only: [ :create, :destroy ]
+    resources :annotations, only: [:index, :create, :destroy]
+    resources :scores, only: [:create, :destroy]
   end
 
-  resources :observations, only: [ :index, :show ] do
+  resources :observations, only: [:index, :show] do
     collection do
       get :generations
       get :spans
@@ -47,7 +47,7 @@ Observ::Engine.routes.draw do
   get "annotations/traces", to: "annotations#traces_index", as: :traces_annotations
   get "annotations/export", to: "annotations#export", as: :export_annotations
 
-  resources :reviews, only: [ :index, :show ], controller: "review_queue" do
+  resources :reviews, only: [:index, :show], controller: "review_queue" do
     collection do
       get :sessions
       get :traces
@@ -65,7 +65,7 @@ Observ::Engine.routes.draw do
       get :compare               # Compare versions
     end
 
-    resources :versions, only: [ :show ], controller: "prompt_versions" do
+    resources :versions, only: [:show], controller: "prompt_versions" do
       member do
         post :promote    # draft -> production
         post :demote     # production -> archived
@@ -76,8 +76,8 @@ Observ::Engine.routes.draw do
   end
 
   resources :datasets do
-    resources :items, controller: "dataset_items", except: [ :show ]
-    resources :runs, controller: "dataset_runs", only: [ :index, :show, :new, :create, :destroy ] do
+    resources :items, controller: "dataset_items", except: [:show]
+    resources :runs, controller: "dataset_runs", only: [:index, :show, :new, :create, :destroy] do
       member do
         post :run_evaluators
         get :review

data/db/migrate/005_create_observ_prompts.rb

@@ -12,10 +12,10 @@ class CreateObservPrompts < ActiveRecord::Migration[7.0]
       t.timestamps
 
       # Composite unique index for name + version
-      t.index [ :name, :version ], unique: true
+      t.index [:name, :version], unique: true
 
       # Index for state queries (e.g., find all production prompts)
-      t.index [ :name, :state ]
+      t.index [:name, :state]
     end
   end
 end

data/db/migrate/011_create_observ_dataset_items.rb

@@ -11,7 +11,7 @@ class CreateObservDatasetItems < ActiveRecord::Migration[7.0]
       t.references :source_trace, foreign_key: { to_table: :observ_traces }
       t.timestamps
 
-      t.index [ :dataset_id, :status ]
+      t.index [:dataset_id, :status]
     end
   end
 end

data/db/migrate/012_create_observ_dataset_runs.rb

@@ -15,8 +15,8 @@ class CreateObservDatasetRuns < ActiveRecord::Migration[7.0]
       t.integer :total_tokens, default: 0
       t.timestamps
 
-      t.index [ :dataset_id, :name ], unique: true
-      t.index [ :dataset_id, :status ]
+      t.index [:dataset_id, :name], unique: true
+      t.index [:dataset_id, :status]
     end
   end
 end

data/db/migrate/013_create_observ_dataset_run_items.rb

@@ -10,7 +10,7 @@ class CreateObservDatasetRunItems < ActiveRecord::Migration[7.0]
       t.text :error
       t.timestamps
 
-      t.index [ :dataset_run_id, :dataset_item_id ], unique: true, name: "idx_run_items_on_run_and_item"
+      t.index [:dataset_run_id, :dataset_item_id], unique: true, name: "idx_run_items_on_run_and_item"
     end
   end
 end

data/db/migrate/014_create_observ_scores.rb

@@ -18,8 +18,8 @@ class CreateObservScores < ActiveRecord::Migration[7.0]
 
       t.timestamps
 
-      t.index [ :dataset_run_item_id, :name, :source ], unique: true, name: "idx_scores_on_run_item_name_source"
-      t.index [ :trace_id, :name ]
+      t.index [:dataset_run_item_id, :name, :source], unique: true, name: "idx_scores_on_run_item_name_source"
+      t.index [:trace_id, :name]
       t.index :name
     end
   end

data/db/migrate/015_refactor_scores_to_polymorphic.rb

@@ -19,8 +19,8 @@ class RefactorScoresToPolymorphic < ActiveRecord::Migration[7.0]
     remove_column :observ_scores, :trace_id, :bigint
 
     # Add new indexes
-    add_index :observ_scores, [ :scoreable_type, :scoreable_id ]
-    add_index :observ_scores, [ :scoreable_type, :scoreable_id, :name, :source ],
+    add_index :observ_scores, [:scoreable_type, :scoreable_id]
+    add_index :observ_scores, [:scoreable_type, :scoreable_id, :name, :source],
               unique: true,
               name: "idx_scores_unique_on_scoreable_name_source"
   end

data/db/migrate/016_create_observ_review_items.rb

@@ -17,8 +17,8 @@ class CreateObservReviewItems < ActiveRecord::Migration[7.0]
 
       t.timestamps
 
-      t.index [ :reviewable_type, :reviewable_id ], unique: true, name: "idx_review_items_on_reviewable"
-      t.index [ :status, :priority, :created_at ], name: "idx_review_items_on_status_priority_created"
+      t.index [:reviewable_type, :reviewable_id], unique: true, name: "idx_review_items_on_reviewable"
+      t.index [:status, :priority, :created_at], name: "idx_review_items_on_status_priority_created"
       t.index :status
     end
   end

data/lib/generators/observ/install_chat/templates/jobs/chat_response_job.rb.tt

@@ -1,8 +1,11 @@
 class ChatResponseJob < ApplicationJob
   retry_on RubyLLM::BadRequestError, wait: 2.seconds, attempts: 1
 
-  def perform(chat_id, content)
+  # @param chat_id [Integer] The chat ID
+  # @param user_message_id [Integer] The ID of the user message (already created by the controller)
+  def perform(chat_id, user_message_id)
     chat = Chat.find(chat_id)
+    user_message = chat.messages.find(user_message_id)
 
     # Observability is automatically enabled via after_find callback
     # All LLM calls, tool calls, and metrics are tracked automatically
@@ -12,7 +15,10 @@ class ChatResponseJob < ApplicationJob
     begin
       # Model parameters (temperature, max_tokens, etc.) are automatically configured
       # via the initialize_agent callback when the chat is created
-      chat.ask(content) do |chunk|
+      #
+      # Use complete instead of ask to avoid creating a duplicate user message.
+      # The user message was already created by the controller for immediate UI feedback.
+      chat.complete do |chunk|
         if chunk.content && !chunk.content.blank?
           message = chat.messages.last
           message.broadcast_append_chunk(chunk.content)
@@ -23,7 +29,7 @@ class ChatResponseJob < ApplicationJob
 
       error_message = chat.messages.create!(
         role: :assistant,
-        content: "I apologize, but I encountered an error while processing your request. This might be due to a tool call issue. Please try rephrasing your question or try again."
+        content: "**Error:** #{e.message}"
       )
 
       error_message.broadcast_replace_to(

data/lib/observ/engine.rb

@@ -8,6 +8,13 @@ module Observ
       g.factory_bot dir: "spec/factories"
     end
 
+    # Make helpers available to host app for Turbo broadcasts
+    initializer "observ.helpers" do
+      ActiveSupport.on_load(:action_controller_base) do
+        helper Observ::MarkdownHelper
+      end
+    end
+
     # Make concerns available to host app
     initializer "observ.load_concerns" do
       config.to_prepare do

data/lib/observ/version.rb

@@ -1,3 +1,3 @@
 module Observ
-  VERSION = "0.6.6"
+  VERSION = "0.6.8"
 end

data/lib/tasks/observ_tasks.rake

@@ -11,7 +11,7 @@ namespace :observ do
     rails observ:sync_assets[app/javascript/stylesheets/observ]
     rails observ:sync_assets[app/assets/stylesheets/observ,app/javascript/controllers/observ]
   "
-  task :sync_assets, [ :styles_dest, :js_dest ] => :environment do |t, args|
+  task :sync_assets, [:styles_dest, :js_dest] => :environment do |t, args|
     require "observ/asset_installer"
 
     # Get the observ gem root (this task is in observ/lib/tasks)
@@ -46,7 +46,7 @@ namespace :observ do
     rails observ:install_assets[app/javascript/stylesheets/observ]
     rails observ:install_assets[app/assets/stylesheets/observ,app/javascript/controllers/observ]
   "
-  task :install_assets, [ :styles_dest, :js_dest ] => :environment do |t, args|
+  task :install_assets, [:styles_dest, :js_dest] => :environment do |t, args|
     require "observ/asset_installer"
 
     # Get the observ gem root