rubyllm-observ 0.5.1 → 0.6.0

data/app/views/observ/review_queue/index.html.erb

@@ -0,0 +1,49 @@
+<div class="observ-page">
+  <div class="observ-page__header">
+    <h1 class="observ-page__title">Review Queue</h1>
+    <div class="observ-page__actions">
+      <%= link_to "Stats", stats_reviews_path, class: "observ-button observ-button--secondary" %>
+    </div>
+  </div>
+
+  <div class="observ-tabs">
+    <%= link_to "All", reviews_path, class: "observ-tab #{'observ-tab--active' if @filter == :all}" %>
+    <%= link_to "Sessions", sessions_reviews_path, class: "observ-tab #{'observ-tab--active' if @filter == :sessions}" %>
+    <%= link_to "Traces", traces_reviews_path, class: "observ-tab #{'observ-tab--active' if @filter == :traces}" %>
+  </div>
+
+  <%= render "stats", stats: @stats %>
+
+  <% if @review_items.any? %>
+    <div class="observ-card">
+      <div class="observ-card__body">
+        <table class="observ-table">
+          <thead class="observ-table__header">
+            <tr class="observ-table__row">
+              <th class="observ-table__cell">Priority</th>
+              <th class="observ-table__cell">Type</th>
+              <th class="observ-table__cell">Reason</th>
+              <th class="observ-table__cell">Status</th>
+              <th class="observ-table__cell">Created</th>
+              <th class="observ-table__cell"></th>
+            </tr>
+          </thead>
+          <tbody>
+            <% @review_items.each do |review_item| %>
+              <%= render "item", review_item: review_item %>
+            <% end %>
+          </tbody>
+        </table>
+      </div>
+    </div>
+
+    <%= observ_pagination(@review_items) %>
+  <% else %>
+    <div class="observ-empty-state">
+      <p class="observ-empty-state__text">No items to review</p>
+      <p class="observ-empty-state__subtext">
+        Items will appear here when they are flagged by guardrails or added manually.
+      </p>
+    </div>
+  <% end %>
+</div>

data/app/views/observ/review_queue/show.html.erb

@@ -0,0 +1,76 @@
+<div class="observ-page">
+  <div class="observ-page__header">
+    <div class="observ-page__breadcrumb">
+      <%= link_to "Review Queue", reviews_path %> /
+      <span>Review Item</span>
+    </div>
+    <h1 class="observ-page__title">
+      Review <%= @review_item.reviewable_type_display %>
+      <span class="observ-badge <%= @review_item.priority_badge_class %>">
+        <%= @review_item.priority.capitalize %>
+      </span>
+    </h1>
+    <% if @queue_position %>
+      <p class="observ-text--muted">
+        Reviewing <%= @queue_position[:current] %> of <%= @queue_position[:total] %>
+      </p>
+    <% end %>
+  </div>
+
+  <div class="observ-review-layout">
+    <div class="observ-review-main">
+      <div class="observ-alert observ-alert--warning">
+        <strong>Flagged: <%= @review_item.reason_display %></strong>
+        <% if @review_item.reason_details.present? %>
+          <% @review_item.reason_details.each do |key, value| %>
+            <br><%= key.to_s.humanize %>: <%= value %>
+          <% end %>
+        <% end %>
+      </div>
+
+      <% if @reviewable.is_a?(Observ::Trace) %>
+        <%= render "observ/traces/details", trace: @reviewable, open_in_new_tab: true %>
+
+        <%= render "observ/traces/input", trace: @reviewable %>
+
+        <%= render "observ/traces/output", trace: @reviewable %>
+
+        <%= render "observ/traces/metadata", trace: @reviewable %>
+
+        <%= render "observ/traces/observations", observations: @observations, trace: @reviewable, show_total: true, total_count: @reviewable.observations.count %>
+      <% elsif @reviewable.is_a?(Observ::Session) %>
+        <%= render "observ/sessions/metrics", session_metrics: @session_metrics, session: @reviewable %>
+
+        <%= render "observ/sessions/metadata", session: @reviewable %>
+
+        <%= render "observ/sessions/traces", traces: @traces, session: @reviewable, show_total: true, total_count: @session_metrics[:total_traces], open_in_new_tab: true %>
+
+        <%= render "observ/sessions/chat", chat: @chat, limit: 10, hide_actions: true %>
+      <% end %>
+    </div>
+
+    <div class="observ-review-sidebar">
+      <div class="observ-card">
+        <div class="observ-card__header">
+          <h2 class="observ-card__title">Score This Item</h2>
+        </div>
+        <div class="observ-card__body">
+          <%= render "observ/scores/form", scoreable: @reviewable %>
+        </div>
+      </div>
+
+      <div class="observ-review-actions">
+        <%= form_with url: complete_review_path(@review_item), method: :post, local: true do |f| %>
+          <input type="hidden" name="next" value="true">
+          <%= f.submit "Save & Next", class: "observ-button observ-button--primary observ-button--block" %>
+        <% end %>
+
+        <%= form_with url: skip_review_path(@review_item), method: :post, local: true do |f| %>
+          <%= f.submit "Skip", class: "observ-button observ-button--secondary observ-button--block" %>
+        <% end %>
+
+        <%= link_to "Back to Queue", reviews_path, class: "observ-button observ-button--link observ-button--block" %>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/observ/review_queue/stats.html.erb

@@ -0,0 +1,100 @@
+<div class="observ-page">
+  <div class="observ-page__header">
+    <h1 class="observ-page__title">Review Queue Statistics</h1>
+    <div class="observ-page__actions">
+      <%= link_to "Back to Queue", reviews_path, class: "observ-button observ-button--secondary" %>
+    </div>
+  </div>
+
+  <div class="observ-page__content">
+    <div class="observ-grid observ-grid--4">
+      <div class="observ-card">
+        <div class="observ-card__body observ-text--center">
+          <div class="observ-stat__value observ-stat__value--large"><%= @stats[:total_pending] %></div>
+          <div class="observ-stat__label">Total Pending</div>
+        </div>
+      </div>
+
+      <div class="observ-card">
+        <div class="observ-card__body observ-text--center">
+          <div class="observ-stat__value observ-stat__value--large"><%= @stats[:total_completed] %></div>
+          <div class="observ-stat__label">Total Completed</div>
+        </div>
+      </div>
+
+      <div class="observ-card">
+        <div class="observ-card__body observ-text--center">
+          <div class="observ-stat__value observ-stat__value--large"><%= @stats[:completed_today] %></div>
+          <div class="observ-stat__label">Completed Today</div>
+        </div>
+      </div>
+
+      <div class="observ-card">
+        <div class="observ-card__body observ-text--center">
+          <div class="observ-stat__value observ-stat__value--large"><%= @stats[:completed_this_week] %></div>
+          <div class="observ-stat__label">This Week</div>
+        </div>
+      </div>
+    </div>
+
+    <div class="observ-grid observ-grid--2">
+      <div class="observ-card">
+        <div class="observ-card__header">
+          <h3 class="observ-card__title">Pass Rate</h3>
+        </div>
+        <div class="observ-card__body observ-text--center">
+          <% if @stats[:pass_rate] %>
+            <div class="observ-stat__value observ-stat__value--large <%= @stats[:pass_rate] >= 70 ? 'observ-text--success' : (@stats[:pass_rate] >= 40 ? 'observ-text--warning' : 'observ-text--danger') %>">
+              <%= @stats[:pass_rate] %>%
+            </div>
+            <div class="observ-stat__label">of reviewed items passed</div>
+          <% else %>
+            <div class="observ-text--muted">No completed reviews yet</div>
+          <% end %>
+        </div>
+      </div>
+
+      <div class="observ-card">
+        <div class="observ-card__header">
+          <h3 class="observ-card__title">By Priority</h3>
+        </div>
+        <div class="observ-card__body">
+          <% if @stats[:by_priority].any? %>
+            <div class="observ-stat-list">
+              <% @stats[:by_priority].each do |priority, count| %>
+                <div class="observ-stat-list__item">
+                  <span class="observ-badge <%= priority_badge_class(priority) %>">
+                    <%= priority.to_s.titleize %>
+                  </span>
+                  <span class="observ-stat-list__value"><%= count %></span>
+                </div>
+              <% end %>
+            </div>
+          <% else %>
+            <div class="observ-text--muted">No data</div>
+          <% end %>
+        </div>
+      </div>
+    </div>
+
+    <div class="observ-card">
+      <div class="observ-card__header">
+        <h3 class="observ-card__title">By Reason</h3>
+      </div>
+      <div class="observ-card__body">
+        <% if @stats[:by_reason].any? %>
+          <div class="observ-stat-list observ-stat-list--spacious">
+            <% @stats[:by_reason].sort_by { |_, count| -count }.each do |reason, count| %>
+              <div class="observ-stat-list__item">
+                <span class="observ-stat-list__label"><%= reason&.titleize&.gsub("_", " ") || "Manual" %></span>
+                <span class="observ-stat-list__value"><%= count %></span>
+              </div>
+            <% end %>
+          </div>
+        <% else %>
+          <div class="observ-text--muted">No data</div>
+        <% end %>
+      </div>
+    </div>
+  </div>
+</div>

data/app/views/observ/scores/_form.html.erb

@@ -0,0 +1,39 @@
+<%# Score form partial for scoring sessions and traces %>
+<%# Usage: render "observ/scores/form", scoreable: @trace %>
+
+<% score_url = scoreable.is_a?(Observ::Session) ? session_scores_path(scoreable) : trace_scores_path(scoreable) %>
+<% existing = scoreable.manual_score %>
+
+<%= form_with(
+  url: score_url,
+  id: "score-form",
+  class: "observ-score-form",
+  data: { turbo_frame: "_top" }
+) do |form| %>
+  <div class="observ-form-field">
+    <label class="observ-form-label">Score this <%= scoreable.class.name.demodulize.downcase %></label>
+    <div class="observ-score-buttons">
+      <label class="observ-score-button <%= 'observ-score-button--selected' if existing&.passed? %>">
+        <input type="radio" name="value" value="1" <%= "checked" if existing&.passed? %>>
+        <span class="observ-score-icon observ-score-icon--pass">&#10003;</span>
+        Good
+      </label>
+      <label class="observ-score-button <%= 'observ-score-button--selected' if existing&.failed? %>">
+        <input type="radio" name="value" value="0" <%= "checked" if existing&.failed? %>>
+        <span class="observ-score-icon observ-score-icon--fail">&#10005;</span>
+        Bad
+      </label>
+    </div>
+    <input type="hidden" name="data_type" value="boolean">
+    <input type="hidden" name="name" value="manual">
+  </div>
+
+  <div class="observ-form-field">
+    <label class="observ-form-label" for="score_comment">Comment (optional)</label>
+    <textarea name="comment" id="score_comment" class="observ-form-textarea" rows="2" placeholder="Add a comment..."><%= existing&.comment %></textarea>
+  </div>
+
+  <div class="observ-form-actions">
+    <%= form.submit "Save Score", class: "observ-button observ-button--primary" %>
+  </div>
+<% end %>

data/app/views/observ/scores/create.turbo_stream.erb

@@ -0,0 +1,10 @@
+<%= turbo_stream.replace "score-form" do %>
+  <%= render "observ/scores/form", scoreable: @scoreable %>
+<% end %>
+
+<%= turbo_stream.replace "score-status" do %>
+  <div id="score-status" class="observ-score-status observ-score-status--saved">
+    <span class="observ-score-status__icon">&#10003;</span>
+    Score saved
+  </div>
+<% end %>

data/app/views/observ/sessions/_chat.html.erb

@@ -0,0 +1,59 @@
+<%
+  # Local variables:
+  # - chat: the chat object
+  # - limit: optional, number of messages to show (default: all)
+%>
+
+<% if chat.present? %>
+  <section class="observ-card">
+    <header class="observ-card__header">
+      <h2 class="observ-card__title">Chat Conversation</h2>
+      <% unless local_assigns[:hide_actions] %>
+        <div class="observ-card__actions">
+          <%= link_to "View Chat", chat_path(chat), class: "observ-button observ-button--sm" %>
+        </div>
+      <% end %>
+    </header>
+    <div class="observ-card__body">
+      <% if chat.messages.any? %>
+        <% messages = local_assigns[:limit] ? chat.messages.order(created_at: :asc).limit(limit) : chat.messages.order(created_at: :asc) %>
+        <div class="observ-chat-messages">
+          <% messages.each do |message| %>
+            <div class="observ-chat-message observ-chat-message--<%= message.role %>">
+              <div class="observ-chat-message__header">
+                <span class="observ-chat-message__role"><%= message.role.capitalize %></span>
+                <span class="observ-chat-message__time"><%= observ_relative_time(message.created_at) %></span>
+                <% if message.traces.exists? %>
+                  <%= link_to "View Trace", trace_path(message.traces.first), class: "observ-chat-message__trace-link" %>
+                <% end %>
+                <% if message.input_tokens || message.output_tokens %>
+                  <span class="observ-chat-message__tokens">
+                    <% if message.input_tokens %>
+                      <span>In: <%= format_tokens(message.input_tokens) %></span>
+                    <% end %>
+                    <% if message.output_tokens %>
+                      <span>Out: <%= format_tokens(message.output_tokens) %></span>
+                    <% end %>
+                  </span>
+                <% end %>
+              </div>
+              <div class="observ-chat-message__content">
+                <%= simple_format(message.content) %>
+              </div>
+            </div>
+          <% end %>
+        </div>
+        <% if local_assigns[:limit] && chat.messages.count > limit %>
+          <p class="observ-text--muted observ-review-preview-notice">
+            Showing first <%= limit %> messages. 
+            <%= link_to "View all #{chat.messages.count} messages", chat_path(chat), class: "observ-link", target: "_blank" %>
+          </p>
+        <% end %>
+      <% else %>
+        <div class="observ-card__empty">
+          <p>No messages found in this chat.</p>
+        </div>
+      <% end %>
+    </div>
+  </section>
+<% end %>

data/app/views/observ/sessions/_metadata.html.erb

@@ -0,0 +1,17 @@
+<% if session.metadata.present? && session.metadata.any? %>
+  <section class="observ-card">
+    <header class="observ-card__header">
+      <h2 class="observ-card__title">Metadata</h2>
+    </header>
+    <div class="observ-card__body">
+      <dl class="observ-definition-list">
+        <% session.metadata.each do |key, value| %>
+          <div class="observ-definition-list__item">
+            <dt class="observ-definition-list__term"><%= key %></dt>
+            <dd class="observ-definition-list__definition"><%= value %></dd>
+          </div>
+        <% end %>
+      </dl>
+    </div>
+  </section>
+<% end %>

data/app/views/observ/sessions/_metrics.html.erb

@@ -0,0 +1,81 @@
+<section class="observ-metrics-grid observ-metrics-grid--4col">
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Traces</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value"><%= session_metrics[:total_traces] %></p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">LLM Calls</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value"><%= session_metrics[:total_llm_calls] %></p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Total Tokens</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value"><%= format_tokens(session_metrics[:total_tokens]) %></p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card observ-metric-card--highlighted">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Total Cost</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value"><%= format_currency(session_metrics[:total_cost]) %></p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Duration</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value">
+        <% if session_metrics[:duration_s] %>
+          <%= format_duration_s(session_metrics[:duration_s]) %>
+        <% else %>
+          <span class="observ-text--muted">In Progress</span>
+        <% end %>
+      </p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Avg Latency</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value"><%= format_duration_ms(session_metrics[:average_llm_latency_ms]) %></p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Total LLM Time</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value"><%= format_duration_ms(session_metrics[:total_llm_duration_ms]) %></p>
+    </div>
+  </div>
+
+  <div class="observ-metric-card">
+    <div class="observ-metric-card__header">
+      <h3 class="observ-metric-card__label">Started</h3>
+    </div>
+    <div class="observ-metric-card__body">
+      <p class="observ-metric-card__value observ-metric-card__value--small">
+        <%= observ_relative_time(session.start_time) %>
+      </p>
+    </div>
+  </div>
+</section>

data/app/views/observ/sessions/_traces.html.erb

@@ -0,0 +1,92 @@
+<%
+  # Local variables:
+  # - traces: collection of traces to display
+  # - session: the session object
+  # - limit: optional, if set will show "X of Y" notice
+%>
+
+<section class="observ-card">
+  <header class="observ-card__header">
+    <h2 class="observ-card__title">
+      Traces (<%= traces.count %>)
+      <% if local_assigns[:show_total] && local_assigns[:total_count] && total_count > traces.count %>
+        of <%= total_count %>
+      <% end %>
+    </h2>
+  </header>
+  <div class="observ-card__body">
+    <% if traces.any? %>
+      <table class="observ-table">
+        <thead class="observ-table__header">
+          <tr class="observ-table__row">
+            <th class="observ-table__cell">Trace ID</th>
+            <th class="observ-table__cell">Name</th>
+            <th class="observ-table__cell">Model(s)</th>
+            <th class="observ-table__cell">Start Time</th>
+            <th class="observ-table__cell">Duration</th>
+            <th class="observ-table__cell observ-table__cell--numeric">Observations</th>
+            <th class="observ-table__cell observ-table__cell--numeric">Tokens</th>
+            <th class="observ-table__cell observ-table__cell--numeric">Cost</th>
+            <th class="observ-table__cell">Status</th>
+            <th class="observ-table__cell"></th>
+          </tr>
+        </thead>
+        <tbody>
+          <% traces.each do |trace| %>
+            <tr class="observ-table__row">
+              <td class="observ-table__cell">
+                <code class="observ-code observ-code--inline"><%= truncate_id(trace.trace_id, 12) %></code>
+              </td>
+              <td class="observ-table__cell">
+                <%= trace.name || "—" %>
+              </td>
+              <td class="observ-table__cell">
+                <% models = trace.models_used %>
+                <% if models.any? %>
+                  <%= models.join(", ") %>
+                <% else %>
+                  <span class="observ-text--muted">—</span>
+                <% end %>
+              </td>
+              <td class="observ-table__cell">
+                <%= observ_timestamp(trace.start_time) %>
+              </td>
+              <td class="observ-table__cell">
+                <% if trace.duration_ms %>
+                  <%= format_duration_ms(trace.duration_ms) %>
+                <% else %>
+                  <span class="observ-text--muted">—</span>
+                <% end %>
+              </td>
+              <td class="observ-table__cell observ-table__cell--numeric">
+                <%= trace.observations.count %>
+              </td>
+              <td class="observ-table__cell observ-table__cell--numeric">
+                <%= format_tokens(trace.total_tokens) %>
+              </td>
+              <td class="observ-table__cell observ-table__cell--numeric">
+                <%= format_currency(trace.total_cost) %>
+              </td>
+              <td class="observ-table__cell">
+                <%= observ_status_badge(observ_trace_status(trace)) %>
+              </td>
+              <td class="observ-table__cell observ-table__cell--actions">
+                <%= link_to "View", trace_path(trace), class: "observ-button observ-button--sm", target: local_assigns[:open_in_new_tab] ? "_blank" : nil %>
+              </td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+      <% if local_assigns[:show_total] && local_assigns[:total_count] && total_count > traces.count %>
+        <p class="observ-text--muted observ-review-preview-notice">
+          Showing first <%= traces.count %> traces. 
+          <%= link_to "View all #{total_count} traces", session_path(session), class: "observ-link", target: "_blank" %>
+        </p>
+      <% end %>
+    <% else %>
+      <div class="observ-card__empty">
+        <p>No traces found for this session.</p>
+      </div>
+    <% end %>
+  </div>
+</section>

data/app/views/observ/sessions/annotations_drawer.turbo_stream.erb

@@ -1,9 +1,16 @@
 <%= turbo_stream.update "drawer-header-title" do %>
-  Session Annotations
+  Scores & Annotations
 <% end %>
 
 <%= turbo_stream.update "drawer-content" do %>
   <div class="observ-drawer__section">
+    <h3 class="observ-drawer__section-title">Score</h3>
+    <div id="score-status"></div>
+    <%= render "observ/scores/form", scoreable: @session %>
+  </div>
+
+  <div class="observ-drawer__section">
+    <h3 class="observ-drawer__section-title">Add Annotation</h3>
     <%= render "observ/annotations/form", annotatable: @session, annotation: @annotation %>
   </div>
 

data/app/views/observ/sessions/index.html.erb

@@ -7,6 +7,63 @@
 <% end %>
 
 <div class="observ-container">
+  <!-- Filters -->
+  <section class="observ-card observ-filters">
+    <div class="observ-card__body">
+      <%= form_with url: sessions_path, method: :get, class: "observ-filters__form" do |f| %>
+        <div class="observ-filters__field observ-filters__field--md">
+          <%= f.label "filter[agent_type]", "Agent", class: "observ-filters__label" %>
+          <%= f.select "filter[agent_type]",
+            options_for_select(
+              [["All Agents", ""]] + @agent_types.map { |t| [t, t] },
+              params.dig(:filter, :agent_type)
+            ),
+            {},
+            class: "observ-filters__select" %>
+        </div>
+
+        <div class="observ-filters__field observ-filters__field--flex">
+          <%= f.label "filter[user_id]", "User ID", class: "observ-filters__label" %>
+          <%= f.text_field "filter[user_id]",
+            value: params.dig(:filter, :user_id),
+            placeholder: "Filter by user ID...",
+            class: "observ-filters__input" %>
+        </div>
+
+        <div class="observ-filters__field observ-filters__field--md">
+          <%= f.label "filter[start_date]", "From", class: "observ-filters__label" %>
+          <%= f.date_field "filter[start_date]",
+            value: params.dig(:filter, :start_date),
+            class: "observ-filters__input" %>
+        </div>
+
+        <div class="observ-filters__field observ-filters__field--md">
+          <%= f.label "filter[end_date]", "To", class: "observ-filters__label" %>
+          <%= f.date_field "filter[end_date]",
+            value: params.dig(:filter, :end_date),
+            class: "observ-filters__input" %>
+        </div>
+
+        <div class="observ-filters__field observ-filters__field--md">
+          <%= f.label "filter[status]", "Status", class: "observ-filters__label" %>
+          <%= f.select "filter[status]",
+            options_for_select(
+              [["All", ""], ["Completed", "completed"], ["In Progress", "in_progress"]],
+              params.dig(:filter, :status)
+            ),
+            {},
+            class: "observ-filters__select" %>
+        </div>
+
+        <div class="observ-filters__actions">
+          <%= f.submit "Filter", class: "observ-button observ-button--secondary" %>
+          <%= link_to "Clear", sessions_path, class: "observ-button" %>
+        </div>
+      <% end %>
+    </div>
+  </section>
+
+  <!-- Sessions Table -->
   <section class="observ-card">
     <div class="observ-card__body">
       <% if @sessions.any? %>
@@ -81,11 +138,10 @@
       <% else %>
         <div class="observ-card__empty">
           <p>No sessions found.</p>
-      </div>
-    <% end %>
+        </div>
+      <% end %>
+    </div>
   </section>
 
   <%= observ_pagination(@sessions) %>
-</div>
-  </section>
 </div>