tracebook 0.1.0

data/Rakefile

@@ -0,0 +1,21 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+  t.verbose = false
+end
+
+task :prepare_test_db do
+  ENV["RAILS_ENV"] = "test"
+  Rake::Task["app:db:prepare"].invoke
+end
+
+task default: [ :prepare_test_db, :test ]

data/app/assets/javascripts/tracebook/application.js

@@ -0,0 +1,88 @@
+(function() {
+  const STIMULUS_SRC = "https://unpkg.com/@hotwired/stimulus/dist/stimulus.umd.js";
+  let booted = false;
+
+  function warnStimulusMissing() {
+    if (booted) return;
+
+    console.warn("TraceBook: Stimulus failed to load; keyboard shortcuts disabled.");
+    if (document.body) {
+      const banner = document.createElement("div");
+      banner.className = "tb-alert";
+      banner.textContent = "Keyboard navigation is unavailable because Stimulus failed to load.";
+      document.body.prepend(banner);
+    }
+  }
+
+  function startApplication() {
+    if (booted || !window.Stimulus) {
+      warnStimulusMissing();
+      return;
+    }
+
+    booted = true;
+    const application = window.Stimulus.Application.start();
+
+    class KeyboardController extends window.Stimulus.Controller {
+      static get targets() {
+        return ["table", "row", "checkbox", "toggleAll", "reviewState"];
+      }
+
+      connect() {
+        this.index = 0;
+        this.updateSelection();
+        this.element.addEventListener("keydown", this.handleKeydown.bind(this));
+      }
+
+      handleKeydown(event) {
+        if (["j", "J"].includes(event.key)) {
+          event.preventDefault();
+          this.index = Math.min(this.rowTargets.length - 1, this.index + 1);
+          this.updateSelection();
+        }
+        if (["k", "K"].includes(event.key)) {
+          event.preventDefault();
+          this.index = Math.max(0, this.index - 1);
+          this.updateSelection();
+        }
+        if ([" ", "Enter"].includes(event.key)) {
+          event.preventDefault();
+          const checkbox = this.checkboxTargets[this.index];
+          if (checkbox) {
+            checkbox.checked = !checkbox.checked;
+          }
+        }
+      }
+
+      updateSelection() {
+        this.rowTargets.forEach((row, idx) => {
+          row.classList.toggle("tb-selected", idx === this.index);
+        });
+      }
+    }
+
+    class JsonViewerController extends window.Stimulus.Controller {
+      static get targets() {
+        return ["content"];
+      }
+
+      toggle() {
+        this.element.classList.toggle("tb-collapsed");
+      }
+    }
+
+    application.register("keyboard", KeyboardController);
+    application.register("json-viewer", JsonViewerController);
+  }
+
+  if (window.Stimulus) {
+    startApplication();
+  } else {
+    const script = document.createElement("script");
+    script.src = STIMULUS_SRC;
+    script.async = true;
+    script.onload = startApplication;
+    script.onerror = warnStimulusMissing;
+    document.head.appendChild(script);
+  }
+})();

data/app/assets/stylesheets/tracebook/application.css

@@ -0,0 +1,173 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */
+
+:root {
+  color-scheme: light dark;
+  --tb-bg: #f9f9fb;
+  --tb-border: #d0d7de;
+  --tb-text: #202124;
+  --tb-accent: #0b5fff;
+  --tb-muted: #4a4d52;
+  font-family: "Inter", system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+
+body {
+  background: var(--tb-bg);
+  color: var(--tb-text);
+  margin: 0;
+  padding: 1.5rem;
+}
+
+.tb-container {
+  max-width: 1200px;
+  margin: 0 auto;
+  display: flex;
+  flex-direction: column;
+  gap: 1.5rem;
+}
+
+.tb-header h1 {
+  margin: 0;
+}
+
+.tb-kpis {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
+  gap: 1rem;
+  margin-top: 1rem;
+}
+
+.tb-kpi {
+  padding: 0.75rem;
+  border: 1px solid var(--tb-border);
+  border-radius: 0.75rem;
+  background: white;
+  display: flex;
+  flex-direction: column;
+  gap: 0.5rem;
+}
+
+.tb-kpi span {
+  font-size: 0.85rem;
+  color: var(--tb-muted);
+}
+
+.tb-kpi strong {
+  font-size: 1.25rem;
+}
+
+.tb-filter-form {
+  display: flex;
+  flex-direction: column;
+  gap: 1rem;
+  padding: 1rem;
+  border: 1px solid var(--tb-border);
+  border-radius: 0.75rem;
+  background: white;
+}
+
+.tb-filter-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+  gap: 1rem;
+}
+
+.tb-filter-actions {
+  display: flex;
+  gap: 0.75rem;
+  align-items: center;
+}
+
+.tb-link {
+  color: var(--tb-accent);
+  text-decoration: none;
+}
+
+.tb-table-wrapper {
+  border: 1px solid var(--tb-border);
+  border-radius: 0.75rem;
+  background: white;
+  padding: 1rem;
+}
+
+.tb-table {
+  width: 100%;
+  border-collapse: collapse;
+}
+
+.tb-table th,
+.tb-table td {
+  padding: 0.75rem;
+  border-bottom: 1px solid var(--tb-border);
+  text-align: left;
+}
+
+.tb-table tbody tr:hover {
+  background: rgba(11, 95, 255, 0.08);
+}
+
+.tb-selected {
+  outline: 2px solid var(--tb-accent);
+}
+
+.tb-collapsed {
+  max-height: 200px;
+  overflow: hidden;
+}
+
+.tb-alert {
+  background: #fffbdd;
+  border: 1px solid #f7d070;
+  color: #5c4400;
+  padding: 0.75rem 1rem;
+  border-radius: 0.5rem;
+  margin-bottom: 1rem;
+  font-size: 0.9rem;
+}
+
+.tb-section {
+  border: 1px solid var(--tb-border);
+  border-radius: 0.75rem;
+  background: white;
+  padding: 1rem;
+}
+
+.tb-meta-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
+  gap: 1rem;
+  margin-top: 1rem;
+}
+
+.tb-meta-grid span {
+  display: block;
+  font-size: 0.8rem;
+  color: var(--tb-muted);
+}
+
+.tb-review-form {
+  display: flex;
+  gap: 1rem;
+  align-items: flex-end;
+}
+
+pre {
+  background: #0d1117;
+  color: #e6edf3;
+  padding: 1rem;
+  border-radius: 0.5rem;
+  overflow-x: auto;
+}

data/app/controllers/tracebook/application_controller.rb

@@ -0,0 +1,4 @@
+module Tracebook
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/controllers/tracebook/exports_controller.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Tracebook
+  class ExportsController < ApplicationController
+    def create
+      blob = ExportJob.perform_now(format: params.fetch(:format, :csv), filters: export_filters)
+      redirect_to export_path(blob.signed_id), notice: "Export ready"
+    end
+
+    def show
+      blob = ActiveStorage::Blob.find_signed(params[:id])
+      send_data blob.download, filename: blob.filename.to_s, type: blob.content_type
+    rescue ActiveSupport::MessageVerifier::InvalidSignature
+      head :not_found
+    end
+
+    private
+
+    def export_filters
+      params.fetch(:filters, {}).permit(:provider, :model, :project, :status, :review_state, :from, :to)
+    end
+  end
+end
+
+TraceBook = Tracebook unless defined?(TraceBook)

data/app/controllers/tracebook/interactions_controller.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Tracebook
+  class InteractionsController < ApplicationController
+    before_action :set_interaction, only: [ :show, :review ]
+    helper InteractionsHelper
+
+    PER_PAGE = 100
+
+    def index
+      @filters = filter_params
+      scope = Interaction.filtered(@filters)
+      @kpis = kpis_for(scope)
+      @interactions = scope.order(created_at: :desc).limit(PER_PAGE)
+      @providers = Interaction.distinct.order(:provider).pluck(:provider)
+      @models = Interaction.distinct.order(:model).pluck(:model)
+      @projects = Interaction.distinct.order(:project).pluck(:project).compact
+    end
+
+    def show
+    end
+
+    def review
+      state = params.require(:review_state).to_s
+      unless Interaction.review_states.key?(state)
+        redirect_to interaction_path(@interaction), alert: "Invalid review state: #{state}"
+        return
+      end
+
+      if @interaction.update(review_state: state)
+        redirect_to interaction_path(@interaction), notice: "Review updated"
+      else
+        render :show, status: :unprocessable_entity
+      end
+    end
+
+    def bulk_review
+      ids = Array(params[:interaction_ids])
+      state = params.require(:review_state).to_s
+      unless Interaction.review_states.key?(state)
+        redirect_to interactions_path, alert: "Invalid review state: #{state}"
+        return
+      end
+
+      Interaction.where(id: ids).update_all(review_state: Interaction.review_states.fetch(state))
+      redirect_to interactions_path, notice: "Updated #{ids.size} interactions"
+    end
+
+    private
+
+    def set_interaction
+      @interaction = Interaction.find(params[:id])
+    end
+
+    def filter_params
+      params.fetch(:filters, {}).permit(:provider, :model, :project, :status, :review_state, :tag, :from, :to)
+    end
+
+    def kpis_for(scope)
+      {
+        total: scope.count,
+        success: scope.status_success.count,
+        cost_cents: scope.sum(:cost_total_cents),
+        input_tokens: scope.sum(:input_tokens),
+        output_tokens: scope.sum(:output_tokens)
+      }
+    end
+  end
+end
+
+TraceBook = Tracebook unless defined?(TraceBook)

data/app/helpers/tracebook/application_helper.rb

@@ -0,0 +1,4 @@
+module Tracebook
+  module ApplicationHelper
+  end
+end

data/app/helpers/tracebook/interactions_helper.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Tracebook
+  module InteractionsHelper
+    def payload_for(interaction, type)
+      inline = interaction.public_send("#{type}_payload")
+      return inline unless inline.nil? || (inline.respond_to?(:empty?) && inline.empty?)
+
+      blob = interaction.public_send("#{type}_payload_blob")
+      return nil unless blob
+
+      raw = blob.download
+      JSON.parse(raw)
+    rescue JSON::ParserError
+      raw
+    end
+
+    def formatted_payload(payload, fallback_text = nil)
+      case payload
+      when Hash, Array
+        JSON.pretty_generate(payload)
+      when String
+        payload
+      when nil
+        fallback_text.to_s
+      else
+        JSON.pretty_generate(payload.as_json)
+      end
+    rescue JSON::GeneratorError, TypeError
+      fallback_text ? fallback_text.to_s : payload.to_s
+    end
+  end
+end

data/app/jobs/tracebook/application_job.rb

@@ -0,0 +1,5 @@
+module Tracebook
+  class ApplicationJob < ActiveJob::Base
+    queue_as :tracebook
+  end
+end

data/app/jobs/tracebook/daily_rollups_job.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Tracebook
+  # Background job for aggregating daily metrics.
+  #
+  # Summarizes interactions by date/provider/model/project into {RollupDaily}
+  # records for analytics and cost reporting. Should be scheduled nightly for
+  # each active provider/model combination.
+  #
+  # ## Aggregated Metrics
+  # - Total interaction count
+  # - Success/error counts
+  # - Input/output token sums
+  # - Total cost in cents
+  #
+  # @example Schedule with Sidekiq Cron
+  #   Sidekiq::Cron::Job.create(
+  #     name: "TraceBook OpenAI rollups",
+  #     cron: "0 2 * * *",
+  #     class: "Tracebook::DailyRollupsJob",
+  #     kwargs: { date: Date.yesterday, provider: "openai", model: nil, project: nil }
+  #   )
+  #
+  # @example Run manually for specific date/model
+  #   DailyRollupsJob.perform_now(
+  #     date: Date.yesterday,
+  #     provider: "openai",
+  #     model: "gpt-4o",
+  #     project: "support"
+  #   )
+  #
+  # @see RollupDaily
+  class DailyRollupsJob < ApplicationJob
+    # Aggregates metrics for a specific date/provider/model/project.
+    #
+    # Creates or updates a {RollupDaily} record with summarized statistics.
+    #
+    # @param date [Date] Date to aggregate (usually Date.yesterday)
+    # @param provider [String] Provider name (e.g., "openai")
+    # @param model [String, nil] Model identifier (nil for all models)
+    # @param project [String, nil] Project name (nil for all projects)
+    #
+    # @return [void]
+    #
+    # @raise [ActiveRecord::RecordInvalid] if rollup fails validation
+    def perform(date:, provider:, model:, project: nil)
+      scope = Interaction.where(provider: provider, model: model)
+      scope = scope.where(project: project) if project
+      scope = scope.where(created_at: date.beginning_of_day..date.end_of_day)
+
+      counts = normalize_status_counts(scope.group(:status).count)
+      tokens = scope.pluck(Arel.sql("COALESCE(input_tokens, 0)"), Arel.sql("COALESCE(output_tokens, 0)"))
+      costs = scope.pluck(Arel.sql("COALESCE(cost_total_cents, 0)"))
+
+      input_sum = tokens.sum { |(input, _)| input.to_i }
+      output_sum = tokens.sum { |(_, output)| output.to_i }
+      cost_sum = costs.sum(&:to_i)
+
+      rollup = RollupDaily.find_or_initialize_by(date: date, provider: provider, model: model, project: project)
+      rollup.interactions_count = scope.count
+      rollup.success_count = counts.fetch("success", 0)
+      rollup.error_count = counts.fetch("error", 0)
+      rollup.input_tokens_sum = input_sum
+      rollup.output_tokens_sum = output_sum
+      rollup.cost_cents_sum = cost_sum
+      rollup.currency = determine_currency(scope) || rollup.currency
+      rollup.save!
+    end
+
+    private
+
+    def normalize_status_counts(counts)
+      counts.each_with_object(Hash.new(0)) do |(raw_key, count), normalized|
+        status_name = status_name_for(raw_key)
+        next unless status_name
+
+        normalized[status_name] += count
+      end
+    end
+
+    def status_name_for(raw_key)
+      key_string = raw_key.to_s
+      return key_string if Interaction.statuses.key?(key_string)
+
+      integer_string?(key_string) ? Interaction.statuses.invert[key_string.to_i] : nil
+    end
+
+    def integer_string?(value)
+      value.match?(/\A-?\d+\z/)
+    end
+
+    def determine_currency(scope)
+      scope.pick(:currency)
+    rescue NoMethodError
+      scope.first&.currency
+    end
+  end
+end
+
+TraceBook = Tracebook unless defined?(TraceBook)

data/app/jobs/tracebook/export_job.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Tracebook
+  # Background job for exporting interactions to CSV or NDJSON.
+  #
+  # Streams filtered interactions to an export file stored in ActiveStorage.
+  # Supports filtering by provider, model, project, date range, tags, etc.
+  #
+  # ## Supported Formats
+  # - **CSV** - Comma-separated values with headers
+  # - **NDJSON** - Newline-delimited JSON (one interaction per line)
+  #
+  # ## Exported Fields
+  # - timestamp, project, provider, model, status
+  # - input_tokens, output_tokens, cost_total_cents
+  # - tags (pipe-separated)
+  # - request_payload, response_payload (JSON)
+  # - metadata
+  #
+  # @example Enqueue export job
+  #   blob = ExportJob.perform_now(
+  #     format: :csv,
+  #     filters: {
+  #       provider: "openai",
+  #       from: 30.days.ago,
+  #       to: Date.current
+  #     }
+  #   )
+  #   download_url = Rails.application.routes.url_helpers.rails_blob_url(blob)
+  #
+  # @example Export specific project
+  #   ExportJob.perform_later(
+  #     format: :ndjson,
+  #     filters: { project: "support", review_state: "approved" }
+  #   )
+  #
+  # @see Interaction.filtered
+  class ExportJob < ApplicationJob
+    # Exports filtered interactions to specified format.
+    #
+    # @param format [Symbol, String] Export format (:csv or :ndjson)
+    # @param filters [Hash] Filters to apply (see {Interaction.filtered})
+    #
+    # @option filters [String] :provider Provider name
+    # @option filters [String] :model Model identifier
+    # @option filters [String] :project Project name
+    # @option filters [Symbol, String] :status Status filter
+    # @option filters [Symbol, String] :review_state Review state filter
+    # @option filters [String] :tag Tag to filter by
+    # @option filters [Date, String] :from Start date
+    # @option filters [Date, String] :to End date
+    #
+    # @return [ActiveStorage::Blob] The created export blob
+    #
+    # @raise [ArgumentError] if format is not supported
+    def perform(format:, filters: {})
+      interactions = Interaction.filtered(filters).order(:created_at)
+      data = export_as(interactions, format.to_sym)
+      create_blob(data, format)
+    end
+
+    private
+
+    def export_as(interactions, format)
+      case format
+      when :csv
+        csv_for(interactions)
+      when :ndjson
+        ndjson_for(interactions)
+      else
+        raise ArgumentError, "Unsupported export format: #{format}"
+      end
+    end
+
+    def csv_for(interactions)
+      CSV.generate do |csv|
+        csv << csv_headers
+        interactions.find_each do |interaction|
+          csv << serialize_interaction(interaction).values_at(*csv_headers)
+        end
+      end
+    end
+
+    def ndjson_for(interactions)
+      Enumerator.new do |yielder|
+        interactions.find_each do |interaction|
+          yielder << serialize_interaction(interaction).to_json
+        end
+      end.to_a.join("\n")
+    end
+
+    def serialize_interaction(interaction)
+      payload = {
+        "timestamp" => interaction.created_at.iso8601,
+        "project" => interaction.project,
+        "provider" => interaction.provider,
+        "model" => interaction.model,
+        "status" => interaction.status,
+        "input_tokens" => interaction.input_tokens,
+        "output_tokens" => interaction.output_tokens,
+        "cost_total_cents" => interaction.cost_total_cents,
+        "tags" => Array(interaction.tags).join("|"),
+        "request_payload" => load_payload(interaction, :request),
+        "response_payload" => load_payload(interaction, :response)
+      }
+
+      payload.merge("metadata" => interaction.metadata)
+    end
+
+    def load_payload(interaction, type)
+      store = interaction.public_send("#{type}_payload_store")
+      if store == "active_storage"
+        blob = interaction.public_send("#{type}_payload_blob")
+        return JSON.parse(blob.download) if blob
+      end
+
+      interaction.public_send("#{type}_payload")
+    rescue JSON::ParserError
+      interaction.public_send("#{type}_payload")
+    end
+
+    def create_blob(data, format)
+      ActiveStorage::Blob.create_and_upload!(
+        io: StringIO.new(data),
+        filename: "tracebook-export-#{Time.current.to_i}.#{format}",
+        content_type: content_type_for(format)
+      )
+    end
+
+    def content_type_for(format)
+      case format.to_sym
+      when :csv
+        "text/csv"
+      when :ndjson
+        "application/x-ndjson"
+      else
+        "application/octet-stream"
+      end
+    end
+
+    def csv_headers
+      @csv_headers ||= [
+        "timestamp",
+        "project",
+        "provider",
+        "model",
+        "status",
+        "input_tokens",
+        "output_tokens",
+        "cost_total_cents",
+        "tags",
+        "request_payload",
+        "response_payload",
+        "metadata"
+      ]
+    end
+  end
+end
+
+TraceBook = Tracebook unless defined?(TraceBook)