rubyllm-observ 0.6.6 → 0.6.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9515276911fc3bbb7170e6a9e5f67e58ae7e4285a5c929ebdd469b13d334fa3c
-  data.tar.gz: 5efd608bda38651dc6e606f9216fd423f0f69b655f7264fe7af51a337e37f9d6
+  metadata.gz: 446467563c1aa1279887817be0e1ac27b2cb735780366f6f14e1e07b593e4941
+  data.tar.gz: d9cc8c06d4f4d9872a3977ddb7c2823c4065b129adf1df4021f1a320d8669651
 SHA512:
-  metadata.gz: 39e136a3e9ddb512b5231fbd25b7d588550f8476101ac7c9e8f3e278fbe68f0692836bd15f0f45aebcfe82be5d4d74de0195249cd9a1be099088c69ee1c5926c
-  data.tar.gz: 369289406fc87cfc150b86c7d32bdaafdb52fc5a42836068bd21b16745141d0baaa4313ea98fd0630d33ecfe486b460c39a7f42d1b85a4b6416fb79c2069b99d
+  metadata.gz: d05759ba80f8441218eb7cac5d686c025febdb8d8794380893327d710c2fb24a521f8270ca90a54b0de40073ae782f41247e923281da6cf9a3febf03ca7b6aa9
+  data.tar.gz: 929f994a3578971668c158bc1d341e2feddf8c9213576b3c860e809d9fadfd12fc6caffdc2a4957a01db70a9a51f8bcc7c656452bd4db03d5181b3e18d2d63df

data/README.md

@@ -482,6 +482,324 @@ trace.update(
 )
 ```
 
+### Embedding Instrumentation
+
+Observ can track `RubyLLM.embed` calls for observability. This is useful for RAG applications, semantic search, and any workflow using embeddings.
+
+**Basic usage:**
+
+```ruby
+# Create a session and instrument embeddings
+session = Observ::Session.create!(user_id: "rag_service")
+session.instrument_embedding(context: { operation: "semantic_search" })
+
+# All RubyLLM.embed calls are now tracked
+embedding = RubyLLM.embed("Ruby is a programmer's best friend")
+
+# Batch embeddings are also tracked
+embeddings = RubyLLM.embed(["text 1", "text 2", "text 3"])
+
+session.finalize
+```
+
+**Using with ObservableService concern:**
+
+```ruby
+class SemanticSearchService
+  include Observ::Concerns::ObservableService
+
+  def initialize(observability_session: nil)
+    initialize_observability(
+      observability_session,
+      service_name: "semantic_search",
+      metadata: { version: "1.0" }
+    )
+  end
+
+  def search(query, documents)
+    with_observability do |session|
+      # Instrument embedding calls
+      instrument_embedding(context: { operation: "search" })
+
+      # Generate query embedding
+      query_embedding = RubyLLM.embed(query)
+
+      # Generate document embeddings
+      doc_embeddings = RubyLLM.embed(documents)
+
+      # Perform similarity search...
+      find_similar(query_embedding, doc_embeddings)
+    end
+  end
+end
+```
+
+**What gets tracked:**
+
+| Metric | Description |
+|--------|-------------|
+| `model` | The embedding model used (e.g., `text-embedding-3-small`) |
+| `input_tokens` | Number of tokens in the input text(s) |
+| `cost_usd` | Calculated cost based on model pricing |
+| `dimensions` | Vector dimensions (e.g., 1536) |
+| `batch_size` | Number of texts embedded in a single call |
+| `vectors_count` | Number of vectors returned |
+| `latency` | Time taken for the embedding call |
+
+**Viewing embedding data:**
+
+Embedding observations appear in the trace view at `/observ/sessions`. Each embedding call creates:
+- A **trace** with input/output summary
+- An **embedding observation** with detailed metrics
+
+**Cost aggregation:**
+
+Embedding costs are automatically aggregated into trace and session totals, alongside generation costs. This gives you a complete picture of your LLM spending.
+
+```ruby
+session.session_metrics
+# => {
+#   total_cost: 0.0125,      # Includes both chat and embedding costs
+#   total_tokens: 1500,      # Includes embedding input tokens
+#   ...
+# }
+```
+
+### Image Generation Instrumentation
+
+Observ can track `RubyLLM.paint` calls for image generation observability. This is useful for tracking DALL-E, GPT-Image, Imagen, and other image generation models.
+
+**Basic usage:**
+
+```ruby
+# Create a session and instrument image generation
+session = Observ::Session.create!(user_id: "image_service")
+session.instrument_image_generation(context: { operation: "product_image" })
+
+# All RubyLLM.paint calls are now tracked
+image = RubyLLM.paint("A modern logo for a tech startup")
+
+# With options
+image = RubyLLM.paint(
+  "A panoramic mountain landscape",
+  model: "gpt-image-1",
+  size: "1792x1024"
+)
+
+session.finalize
+```
+
+**Using with ObservableService concern:**
+
+```ruby
+class ImageGenerationService
+  include Observ::Concerns::ObservableService
+
+  def initialize(observability_session: nil)
+    initialize_observability(
+      observability_session,
+      service_name: "image_generation",
+      metadata: { version: "1.0" }
+    )
+  end
+
+  def generate_product_image(prompt)
+    with_observability do |session|
+      instrument_image_generation(context: { operation: "product_image" })
+      RubyLLM.paint(prompt, model: "dall-e-3", size: "1024x1024")
+    end
+  end
+end
+```
+
+**What gets tracked:**
+
+| Metric | Description |
+|--------|-------------|
+| `model` | The image model used (e.g., `dall-e-3`, `gpt-image-1`) |
+| `prompt` | The original prompt text |
+| `revised_prompt` | The model's revised/enhanced prompt (if available) |
+| `size` | Image dimensions (e.g., `1024x1024`, `1792x1024`) |
+| `cost_usd` | Generation cost |
+| `latency_ms` | Time to generate in milliseconds |
+| `output_format` | `url` or `base64` |
+| `mime_type` | Image MIME type (e.g., `image/png`) |
+
+### Transcription Instrumentation
+
+Observ can track `RubyLLM.transcribe` calls for audio-to-text transcription observability. This supports Whisper, GPT-4o transcription models, and other audio transcription providers.
+
+**Basic usage:**
+
+```ruby
+# Create a session and instrument transcription
+session = Observ::Session.create!(user_id: "transcription_service")
+session.instrument_transcription(context: { operation: "meeting_notes" })
+
+# All RubyLLM.transcribe calls are now tracked
+transcript = RubyLLM.transcribe("meeting.wav")
+
+# With options
+transcript = RubyLLM.transcribe(
+  "interview.mp3",
+  model: "gpt-4o-transcribe",
+  language: "es"
+)
+
+# With speaker diarization
+transcript = RubyLLM.transcribe(
+  "team-meeting.wav",
+  model: "gpt-4o-transcribe",
+  speaker_names: ["Alice", "Bob"]
+)
+
+session.finalize
+```
+
+**Using with ObservableService concern:**
+
+```ruby
+class MeetingNotesService
+  include Observ::Concerns::ObservableService
+
+  def initialize(observability_session: nil)
+    initialize_observability(
+      observability_session,
+      service_name: "meeting_notes",
+      metadata: { version: "1.0" }
+    )
+  end
+
+  def transcribe_meeting(audio_path)
+    with_observability do |session|
+      instrument_transcription(context: { operation: "meeting_notes" })
+      RubyLLM.transcribe(audio_path, model: "whisper-1")
+    end
+  end
+end
+```
+
+**What gets tracked:**
+
+| Metric | Description |
+|--------|-------------|
+| `model` | The transcription model (e.g., `whisper-1`, `gpt-4o-transcribe`) |
+| `audio_duration_s` | Length of audio in seconds |
+| `language` | Detected or specified language (ISO 639-1) |
+| `segments_count` | Number of transcript segments |
+| `speakers_count` | Number of speakers (for diarization) |
+| `has_diarization` | Whether speaker diarization was used |
+| `cost_usd` | Transcription cost (based on audio duration) |
+| `latency_ms` | Processing time in milliseconds |
+
+### Content Moderation Instrumentation
+
+Observ can track `RubyLLM.moderate` calls for content moderation observability. This is useful for safety filtering and content policy enforcement.
+
+**Basic usage:**
+
+```ruby
+# Create a session and instrument moderation
+session = Observ::Session.create!(user_id: "content_filter")
+session.instrument_moderation(context: { operation: "user_input_check" })
+
+# All RubyLLM.moderate calls are now tracked
+result = RubyLLM.moderate(user_input)
+
+if result.flagged?
+  # Handle flagged content
+  puts "Content flagged for: #{result.flagged_categories.join(', ')}"
+end
+
+session.finalize
+```
+
+**Using with ObservableService concern:**
+
+```ruby
+class ContentModerationService
+  include Observ::Concerns::ObservableService
+
+  def initialize(observability_session: nil)
+    initialize_observability(
+      observability_session,
+      service_name: "content_moderation",
+      metadata: { version: "1.0" }
+    )
+  end
+
+  def check_content(text)
+    with_observability do |session|
+      instrument_moderation(context: { operation: "user_content_check" })
+      result = RubyLLM.moderate(text)
+      
+      {
+        safe: !result.flagged?,
+        flagged_categories: result.flagged_categories,
+        highest_risk: result.flagged_categories.first
+      }
+    end
+  end
+end
+```
+
+**What gets tracked:**
+
+| Metric | Description |
+|--------|-------------|
+| `model` | The moderation model (e.g., `omni-moderation-latest`) |
+| `flagged` | Whether content was flagged |
+| `categories` | Hash of category boolean flags |
+| `category_scores` | Hash of category confidence scores (0.0-1.0) |
+| `flagged_categories` | List of categories that triggered flagging |
+| `latency_ms` | Processing time in milliseconds |
+
+**Moderation categories tracked:**
+
+- `sexual` - Sexually explicit content
+- `hate` - Hate speech based on identity
+- `harassment` - Harassing or threatening content
+- `self-harm` - Self-harm promotion
+- `violence` - Violence promotion
+- `violence/graphic` - Graphic violent content
+
+**Note:** Moderation is typically free (cost_usd = 0.0), but all calls are tracked for observability and audit purposes.
+
+### Combined Instrumentation
+
+You can instrument multiple RubyLLM methods in the same session:
+
+```ruby
+session = Observ::Session.create!(user_id: "multimodal_service")
+
+# Instrument all methods you'll use
+session.instrument_embedding(context: { operation: "search" })
+session.instrument_image_generation(context: { operation: "illustration" })
+session.instrument_transcription(context: { operation: "audio_input" })
+session.instrument_moderation(context: { operation: "safety_check" })
+
+# Now all calls are tracked
+embedding = RubyLLM.embed("search query")
+image = RubyLLM.paint("generate an illustration")
+transcript = RubyLLM.transcribe("audio.wav")
+moderation = RubyLLM.moderate(user_input)
+
+session.finalize
+```
+
+**Cost aggregation across all types:**
+
+All observation types are automatically aggregated into trace and session totals:
+
+```ruby
+session.session_metrics
+# => {
+#   total_cost: 0.0825,      # Includes chat, embedding, image, and transcription costs
+#   total_tokens: 1500,      # Includes generation and embedding tokens
+#   ...
+# }
+```
+
 ### Prompt Management
 
 Fetch prompts in your code:
@@ -653,7 +971,7 @@ bundle exec rspec
 Observ uses:
 - **Isolated namespace**: All classes under `Observ::` module
 - **Engine pattern**: Mountable Rails engine for easy integration
-- **STI for observations**: `Observ::Generation` and `Observ::Span` inherit from `Observ::Observation`
+- **STI for observations**: `Observ::Generation`, `Observ::Span`, `Observ::Embedding`, `Observ::ImageGeneration`, `Observ::Transcription`, and `Observ::Moderation` inherit from `Observ::Observation`
 - **AASM for state machine**: Prompt lifecycle management
 - **Kaminari for pagination**: Session and trace listings
 - **Stimulus controllers**: Interactive UI components

data/app/assets/javascripts/observ/controllers/config_editor_controller.js

@@ -0,0 +1,178 @@
+import { Controller } from "@hotwired/stimulus"
+
+/**
+ * Config Editor Controller
+ * 
+ * Handles bi-directional sync between structured form fields and JSON textarea.
+ * Provides real-time JSON validation with visual feedback.
+ * 
+ * Targets:
+ *   - model: Select dropdown for model selection
+ *   - temperature: Number input for temperature
+ *   - maxTokens: Number input for max_tokens
+ *   - jsonInput: Textarea for advanced JSON editing
+ *   - status: Element to display validation status
+ *   - hiddenField: Hidden input that submits the final JSON
+ * 
+ * Values:
+ *   - knownKeys: Array of keys managed by structured fields (default: ["model", "temperature", "max_tokens"])
+ */
+export default class extends Controller {
+  static targets = ["model", "temperature", "maxTokens", "jsonInput", "status", "hiddenField"]
+  static values = {
+    knownKeys: { type: Array, default: ["model", "temperature", "max_tokens"] }
+  }
+
+  connect() {
+    console.log('[Observ] config-editor controller connected')
+    this.syncFromHiddenField()
+  }
+
+  // Called when any structured field changes
+  syncToJson() {
+    const config = this.buildConfigFromFields()
+    const jsonString = Object.keys(config).length > 0 
+      ? JSON.stringify(config, null, 2) 
+      : ""
+    
+    if (this.hasJsonInputTarget) {
+      this.jsonInputTarget.value = jsonString
+    }
+    this.updateHiddenField(config)
+    this.showValidStatus()
+  }
+
+  // Called when JSON textarea changes
+  syncFromJson() {
+    const jsonString = this.jsonInputTarget.value.trim()
+    
+    if (jsonString === "") {
+      this.clearStructuredFields()
+      this.updateHiddenField({})
+      this.showValidStatus()
+      return
+    }
+
+    try {
+      const config = JSON.parse(jsonString)
+      this.populateStructuredFields(config)
+      this.updateHiddenField(config)
+      this.showValidStatus()
+    } catch (e) {
+      this.showInvalidStatus(e.message)
+    }
+  }
+
+  // Build config object from structured fields
+  buildConfigFromFields() {
+    const config = this.getExtraJsonKeys()
+
+    if (this.hasModelTarget && this.modelTarget.value) {
+      config.model = this.modelTarget.value
+    }
+    if (this.hasTemperatureTarget && this.temperatureTarget.value !== "") {
+      config.temperature = parseFloat(this.temperatureTarget.value)
+    }
+    if (this.hasMaxTokensTarget && this.maxTokensTarget.value !== "") {
+      config.max_tokens = parseInt(this.maxTokensTarget.value, 10)
+    }
+
+    return config
+  }
+
+  // Get any extra keys from JSON that aren't in structured fields
+  getExtraJsonKeys() {
+    if (!this.hasJsonInputTarget || this.jsonInputTarget.value.trim() === "") {
+      return {}
+    }
+
+    try {
+      const fullConfig = JSON.parse(this.jsonInputTarget.value)
+      const extra = {}
+      
+      Object.keys(fullConfig).forEach(key => {
+        if (!this.knownKeysValue.includes(key)) {
+          extra[key] = fullConfig[key]
+        }
+      })
+      
+      return extra
+    } catch {
+      return {}
+    }
+  }
+
+  // Populate structured fields from config object
+  populateStructuredFields(config) {
+    if (this.hasModelTarget && config.model !== undefined) {
+      this.modelTarget.value = config.model
+    }
+    if (this.hasTemperatureTarget && config.temperature !== undefined) {
+      this.temperatureTarget.value = config.temperature
+    }
+    if (this.hasMaxTokensTarget && config.max_tokens !== undefined) {
+      this.maxTokensTarget.value = config.max_tokens
+    }
+  }
+
+  // Clear all structured fields
+  clearStructuredFields() {
+    if (this.hasModelTarget) this.modelTarget.value = ""
+    if (this.hasTemperatureTarget) this.temperatureTarget.value = ""
+    if (this.hasMaxTokensTarget) this.maxTokensTarget.value = ""
+  }
+
+  // Load initial state from hidden field
+  syncFromHiddenField() {
+    if (!this.hasHiddenFieldTarget) return
+
+    const jsonString = this.hiddenFieldTarget.value.trim()
+    if (jsonString === "") return
+
+    try {
+      const config = JSON.parse(jsonString)
+      this.populateStructuredFields(config)
+      
+      if (this.hasJsonInputTarget) {
+        this.jsonInputTarget.value = JSON.stringify(config, null, 2)
+      }
+      this.showValidStatus()
+    } catch {
+      // If initial value is invalid, just show it in the textarea
+      if (this.hasJsonInputTarget) {
+        this.jsonInputTarget.value = jsonString
+      }
+      this.showInvalidStatus("Initial config is invalid JSON")
+    }
+  }
+
+  // Update the hidden field that gets submitted
+  updateHiddenField(config) {
+    if (!this.hasHiddenFieldTarget) return
+    this.hiddenFieldTarget.value = Object.keys(config).length > 0 
+      ? JSON.stringify(config) 
+      : ""
+  }
+
+  // Show valid status
+  showValidStatus() {
+    if (!this.hasStatusTarget) return
+    this.statusTarget.innerHTML = '<span class="observ-config-editor__status--valid">&#10003; Valid JSON</span>'
+    this.statusTarget.classList.remove("observ-config-editor__status--error")
+    this.statusTarget.classList.add("observ-config-editor__status--success")
+  }
+
+  // Show invalid status with error message
+  showInvalidStatus(message) {
+    if (!this.hasStatusTarget) return
+    this.statusTarget.innerHTML = `<span class="observ-config-editor__status--invalid">&#10007; Invalid JSON: ${this.escapeHtml(message)}</span>`
+    this.statusTarget.classList.remove("observ-config-editor__status--success")
+    this.statusTarget.classList.add("observ-config-editor__status--error")
+  }
+
+  escapeHtml(text) {
+    const div = document.createElement('div')
+    div.textContent = text
+    return div.innerHTML
+  }
+}

data/app/assets/javascripts/observ/controllers/index.js

@@ -0,0 +1,29 @@
+// Auto-generated index file for Observ Stimulus controllers
+// Register all Observ controllers with the observ-- prefix
+import { application } from "../application"
+
+import AutoscrollController from "./autoscroll_controller.js"
+import ChatFormController from "./chat_form_controller.js"
+import ConfigEditorController from "./config_editor_controller.js"
+import CopyController from "./copy_controller.js"
+import DashboardController from "./dashboard_controller.js"
+import DrawerController from "./drawer_controller.js"
+import ExpandableController from "./expandable_controller.js"
+import FilterController from "./filter_controller.js"
+import JsonViewerController from "./json_viewer_controller.js"
+import MessageFormController from "./message_form_controller.js"
+import PromptVariablesController from "./prompt_variables_controller.js"
+import TextSelectController from "./text_select_controller.js"
+
+application.register("observ--autoscroll", AutoscrollController)
+application.register("observ--chat-form", ChatFormController)
+application.register("observ--config-editor", ConfigEditorController)
+application.register("observ--copy", CopyController)
+application.register("observ--dashboard", DashboardController)
+application.register("observ--drawer", DrawerController)
+application.register("observ--expandable", ExpandableController)
+application.register("observ--filter", FilterController)
+application.register("observ--json-viewer", JsonViewerController)
+application.register("observ--message-form", MessageFormController)
+application.register("observ--prompt-variables", PromptVariablesController)
+application.register("observ--text-select", TextSelectController)

data/app/assets/javascripts/observ/controllers/message_form_controller.js

@@ -38,6 +38,12 @@ export default class extends Controller {
     }
     
     this.submitTarget.style.display = this.loadingValue ? 'none' : 'inline-flex'
+    
+    // Show/hide typing indicator in messages area
+    const typingIndicator = document.getElementById('typing-indicator')
+    if (typingIndicator) {
+      typingIndicator.style.display = this.loadingValue ? 'flex' : 'none'
+    }
   }
 
   handleTurboSubmit(event) {
@@ -48,11 +54,27 @@ export default class extends Controller {
 
   handleTurboRender(event) {
     const streamElement = event.target
-    if (streamElement.target === 'new_message' || streamElement.target === 'messages') {
+    
+    // Only reset loading when the form is replaced (after user message submitted)
+    // The typing indicator will be hidden when assistant message arrives
+    if (streamElement.target === 'new_message') {
+      // Form was replaced, re-enable submit but keep typing indicator showing
       setTimeout(() => {
-        this.loadingValue = false
         this.toggleSubmit()
       }, 100)
     }
+    
+    // Hide typing indicator when an assistant message is appended
+    if (streamElement.target === 'messages') {
+      const action = streamElement.getAttribute('action')
+      // Check if this is an append (new message) containing assistant content
+      if (action === 'append') {
+        const content = streamElement.innerHTML
+        // Check if it's an assistant message (has the assistant class)
+        if (content.includes('observ-chat-message--assistant')) {
+          this.loadingValue = false
+        }
+      }
+    }
   }
 }