rubyllm-observ 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60de7719d3c816c05410d89f7919d32605e6d2d5bb32de977a10987034f35727
-  data.tar.gz: 5739e9eb838a39c82fa3b87c5183e4c5dc7df34ae46d5e93878d25ffe53833c2
+  metadata.gz: '0585d05a3b7dcf8789fafd4e6b2fb9a7a938b80427caf2051f1de0b4b312e6e2'
+  data.tar.gz: f11909cce646368528328c3db82a469983c02e1b157c100cdb633ec54ffb97ce
 SHA512:
-  metadata.gz: '03790e8e17c76e8f68f337e3d336b0228fda23aaf3c5b9aececf9c6d3123eeed64f9d8dda0bc6810aed1918b14b8ef4b54989ed1a799aaeac3560459ad2b7b4e'
-  data.tar.gz: 48286a5f8a4b311bb443b68fba587206e9940d4c6c9c7a1b9b1c881941d8551769c3f9d4271a3b47a5de2656d9deffa5c1811d19f04a84d8c908e11773348bc9
+  metadata.gz: ac12fb93b47733c0e63adcf4959a050758a9131e71dfa54c1043e84e344c2ba392d9f6f4dd28a1eebff690eb1c51b3ff6707c5f303b43c7618e7477ebbba63eb
+  data.tar.gz: c2d9d34e3f2e72e2382d3ca9f29f28e871a0897f3149e35b6531b43de21b46756b60dcd0d1a831660528c856613f2bbbe16aa5e0891b602167a1ebb2b14c7050

data/README.md

@@ -30,7 +30,7 @@ For LLM observability without the chat UI:
 **1. Add to Gemfile:**
 
 ```ruby
-gem "observ"
+gem "rubyllm-observ"
 ```
 
 **2. Install:**
@@ -61,7 +61,7 @@ For full observability + interactive agent testing UI:
 **1. Add to Gemfile:**
 
 ```ruby
-gem "observ"
+gem "rubyllm-observ"
 gem "ruby_llm"  # Required for chat feature
 ```
 
@@ -89,7 +89,7 @@ rails db:migrate
 - Observ enhancements on RubyLLM infrastructure
 - Example agents and tools
 
-See **[Chat Installation Guide](docs/CHAT_INSTALLATION.md)** for detailed setup.
+See **Creating Agents and Services** in `docs/creating-agents-and-services.md` for detailed setup.
 
 ---
 
@@ -345,7 +345,7 @@ end
    - All chat interactions appear in `/observ/sessions`
    - Full observability of tokens, costs, and tool calls
 
-See **[Chat Installation Guide](docs/CHAT_INSTALLATION.md)** for complete documentation.
+See `docs/creating-agents-and-services.md` for complete documentation on creating agents.
 
 ### Phase Tracking (Optional Chat Feature)
 
@@ -528,6 +528,54 @@ session.annotations.create(
 # Visit /observ/annotations/export in browser
 ```
 
+### Datasets & Evaluators
+
+Observ includes a dataset and evaluator system for testing LLM outputs against predefined inputs and scoring results.
+
+**Creating a dataset:**
+
+```ruby
+# Create a dataset
+dataset = Observ::Dataset.create!(
+  name: "Article Recommendations Test Set",
+  description: "Test cases for article recommendation system"
+)
+
+# Add test items
+dataset.items.create!(
+  input: { user_query: "Recommend articles for someone feeling anxious" },
+  expected_output: { recommended_articles: ["art_001", "art_003"] }
+)
+```
+
+**Running evaluations:**
+
+```ruby
+# Create a dataset run
+run = dataset.runs.create!(
+  name: "GPT-4 baseline",
+  model_name: "gpt-4",
+  status: :pending
+)
+
+# Run items are created when executing your LLM against the dataset
+# Each run item links a dataset item to a trace
+
+# Score outputs with built-in evaluators
+Observ::Evaluators::ExactMatchEvaluator.new.evaluate(run_item)
+Observ::Evaluators::ContainsEvaluator.new(keywords: ["anxiety"]).evaluate(run_item)
+```
+
+**Built-in evaluators:**
+- `ExactMatchEvaluator` - Exact string match against expected output
+- `ContainsEvaluator` - Check if output contains specific keywords
+- `JsonStructureEvaluator` - Validate JSON structure
+- `LlmJudgeEvaluator` - Use an LLM to score output quality
+
+Visit `/observ/datasets` to manage datasets and view run results in the UI.
+
+See `docs/dataset_and_evaluator_feature.md` for complete documentation.
+
 ## Asset Management
 
 Observ provides several tools for managing assets in your Rails application:
@@ -622,7 +670,7 @@ Observ uses:
 ### Chat Feature (Optional Add-on)
 - **RubyLLM**: Required for chat/agent testing feature
 - Installed with `rails generate observ:install:chat`
-- See [Chat Installation Guide](docs/CHAT_INSTALLATION.md)
+- See `docs/creating-agents-and-services.md` for agent documentation
 
 ## Testing
 
@@ -775,4 +823,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Version
 
-Current version: 0.1.0
+Current version: 0.6.0

data/app/assets/stylesheets/observ/_annotations.scss

@@ -1,127 +1,138 @@
-.observ-drawer__section {
-  padding: 1.5rem;
-  border-bottom: 1px solid var(--observ-border-color, #e5e7eb);
+@import 'variables';
+@import 'namespace';
 
-  &:last-child {
-    border-bottom: none;
-  }
-}
-
-.observ-drawer__section-title {
-  font-size: 1rem;
-  font-weight: 600;
-  color: var(--observ-text-primary, #111827);
-  margin: 0 0 1rem 0;
-}
+@include observ-scoped {
+  .observ-drawer__section {
+    padding: $observ-spacing-lg;
+    border-bottom: 1px solid $observ-border-color;
 
-.observ-annotation-form {
-  .observ-form-field {
-    margin-bottom: 1rem;
+    &:last-child {
+      border-bottom: none;
+    }
   }
 
-  .observ-form-label {
-    display: block;
-    font-size: 0.875rem;
-    font-weight: 500;
-    color: var(--observ-text-primary, #111827);
-    margin-bottom: 0.5rem;
+  .observ-drawer__section-title {
+    font-size: $observ-font-size-base;
+    font-weight: 600;
+    color: $observ-text-primary;
+    margin: 0 0 $observ-spacing-md 0;
   }
 
-  .observ-form-textarea {
-    width: 100%;
-    padding: 0.75rem;
-    border: 1px solid var(--observ-border-color, #d1d5db);
-    border-radius: 0.375rem;
-    font-size: 0.875rem;
-    font-family: inherit;
-    resize: vertical;
-    transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;
-
-    &:focus {
-      outline: none;
-      border-color: var(--observ-primary-color, #3b82f6);
-      box-shadow: 0 0 0 3px rgba(59, 130, 246, 0.1);
+  .observ-annotation-form {
+    .observ-form-field {
+      margin-bottom: $observ-spacing-md;
+    }
+
+    .observ-form-label {
+      display: block;
+      font-size: $observ-font-size-sm;
+      font-weight: 500;
+      color: $observ-text-primary;
+      margin-bottom: $observ-spacing-xs;
+    }
+
+    .observ-form-textarea {
+      width: 100%;
+      padding: $observ-spacing-sm $observ-spacing-md;
+      border: 1px solid $observ-border-color;
+      border-radius: $observ-border-radius;
+      font-size: $observ-font-size-sm;
+      font-family: inherit;
+      background-color: $observ-bg-elevated;
+      color: $observ-text-primary;
+      resize: vertical;
+      transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;
+
+      &::placeholder {
+        color: $observ-text-muted;
+      }
+
+      &:focus {
+        outline: none;
+        border-color: $observ-primary;
+        box-shadow: 0 0 0 3px rgba($observ-primary, 0.1);
+      }
+    }
+
+    .observ-form-error {
+      display: block;
+      font-size: $observ-font-size-xs;
+      color: $observ-danger;
+      margin-top: $observ-spacing-xs;
     }
-  }
 
-  .observ-form-error {
-    display: block;
-    font-size: 0.75rem;
-    color: var(--observ-error-color, #ef4444);
-    margin-top: 0.25rem;
+    .observ-form-actions {
+      display: flex;
+      justify-content: flex-end;
+    }
   }
 
-  .observ-form-actions {
+  .observ-annotations-list {
     display: flex;
-    justify-content: flex-end;
+    flex-direction: column;
+    gap: $observ-spacing-md;
   }
-}
-
-.observ-annotations-list {
-  display: flex;
-  flex-direction: column;
-  gap: 1rem;
-}
 
-.observ-annotation {
-  padding: 1rem;
-  background: var(--observ-surface-color, #f9fafb);
-  border: 1px solid var(--observ-border-color, #e5e7eb);
-  border-radius: 0.5rem;
-}
+  .observ-annotation {
+    padding: $observ-spacing-md;
+    background: $observ-bg-elevated;
+    border: 1px solid $observ-border-color;
+    border-radius: $observ-border-radius;
+  }
 
-.observ-annotation__header {
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
-  margin-bottom: 0.75rem;
-  padding-bottom: 0.5rem;
-  border-bottom: 1px solid var(--observ-border-color, #e5e7eb);
-}
+  .observ-annotation__header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: $observ-spacing-sm;
+    padding-bottom: $observ-spacing-xs;
+    border-bottom: 1px solid $observ-border-subtle;
+  }
 
-.observ-annotation__time {
-  font-size: 0.75rem;
-  color: var(--observ-text-secondary, #6b7280);
-}
+  .observ-annotation__time {
+    font-size: $observ-font-size-xs;
+    color: $observ-text-muted;
+  }
 
-.observ-annotation__delete {
-  background: none;
-  border: none;
-  color: var(--observ-error-color, #ef4444);
-  font-size: 0.75rem;
-  cursor: pointer;
-  padding: 0.25rem 0.5rem;
-  border-radius: 0.25rem;
-  transition: background-color 0.15s ease-in-out;
-
-  &:hover {
-    background-color: rgba(239, 68, 68, 0.1);
+  .observ-annotation__delete {
+    background: none;
+    border: none;
+    color: $observ-danger;
+    font-size: $observ-font-size-xs;
+    cursor: pointer;
+    padding: $observ-spacing-xs $observ-spacing-sm;
+    border-radius: $observ-border-radius-sm;
+    transition: background-color 0.15s ease-in-out;
+
+    &:hover {
+      background-color: rgba($observ-danger, 0.1);
+    }
   }
-}
 
-.observ-annotation__content {
-  font-size: 0.875rem;
-  color: var(--observ-text-primary, #111827);
-  line-height: 1.5;
-  white-space: pre-wrap;
-  word-break: break-word;
+  .observ-annotation__content {
+    font-size: $observ-font-size-sm;
+    color: $observ-text-primary;
+    line-height: 1.5;
+    white-space: pre-wrap;
+    word-break: break-word;
 
-  p {
-    margin: 0 0 0.5rem 0;
-    
-    &:last-child {
-      margin-bottom: 0;
+    p {
+      margin: 0 0 $observ-spacing-xs 0;
+
+      &:last-child {
+        margin-bottom: 0;
+      }
     }
   }
-}
 
-.observ-empty-state {
-  padding: 2rem;
-  text-align: center;
-}
+  .observ-empty-state {
+    padding: $observ-spacing-xl;
+    text-align: center;
+  }
 
-.observ-empty-state__text {
-  font-size: 0.875rem;
-  color: var(--observ-text-secondary, #6b7280);
-  margin: 0;
+  .observ-empty-state__text {
+    font-size: $observ-font-size-sm;
+    color: $observ-text-muted;
+    margin: 0;
+  }
 }

data/app/assets/stylesheets/observ/_card.scss

@@ -1,52 +1,61 @@
 @import 'variables';
-
-.observ-card {
-  background-color: $observ-white;
-  border-radius: $observ-border-radius;
-  box-shadow: $observ-shadow;
-  overflow: hidden;
-
-  &__header {
-    padding: $observ-spacing-lg;
-    border-bottom: 1px solid $observ-gray-200;
-    display: flex;
-    align-items: center;
-    justify-content: space-between;
-  }
-
-  &__title {
-    font-size: $observ-font-size-xl;
-    font-weight: 600;
-    margin: 0;
-    color: $observ-gray-900;
-  }
-
-  &__actions {
-    display: flex;
-    gap: $observ-spacing-sm;
-  }
-
-  &__body {
-    padding: $observ-spacing-lg;
-  }
-
-  &__footer {
-    padding: $observ-spacing-lg;
-    border-top: 1px solid $observ-gray-200;
-    background-color: $observ-gray-50;
-  }
-
-  &__empty {
-    text-align: center;
-    padding: $observ-spacing-2xl;
-    color: $observ-gray-500;
-  }
-
-  &--span-2 {
-    grid-column: span 2;
-  }
-
-  &--highlighted {
-    border: 2px solid $observ-primary;
+@import 'namespace';
+
+@include observ-scoped {
+  .observ-card {
+    background-color: $observ-bg-surface;
+    border: 1px solid $observ-border-color;
+    border-radius: $observ-border-radius;
+    overflow: hidden;
+    transition: all 0.2s ease-in-out;
+
+    &:hover {
+      border-color: $observ-border-strong;
+    }
+
+    &__header {
+      padding: $observ-spacing-lg;
+      border-bottom: 1px solid $observ-border-subtle;
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+    }
+
+    &__title {
+      font-size: $observ-font-size-xl;
+      font-weight: 600;
+      margin: 0;
+      color: $observ-text-primary;
+      letter-spacing: -0.01em;
+    }
+
+    &__actions {
+      display: flex;
+      gap: $observ-spacing-sm;
+    }
+
+    &__body {
+      padding: $observ-spacing-lg;
+    }
+
+    &__footer {
+      padding: $observ-spacing-lg;
+      border-top: 1px solid $observ-border-subtle;
+      background-color: rgba($observ-black, 0.1);
+    }
+
+    &__empty {
+      text-align: center;
+      padding: $observ-spacing-2xl;
+      color: $observ-text-muted;
+    }
+
+    &--span-2 {
+      grid-column: span 2;
+    }
+
+    &--highlighted {
+      border: 2px solid $observ-primary;
+    }
   }
 }