llm_cost_tracker 0.1.4 → 0.2.0.alpha2

data/app/controllers/llm_cost_tracker/application_controller.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class ApplicationController < ActionController::Base
+    layout "llm_cost_tracker/application"
+
+    before_action :ensure_llm_api_calls_table
+
+    rescue_from ActiveRecord::ConnectionNotEstablished, with: :render_database_error
+    rescue_from ActiveRecord::RecordNotFound, with: :render_not_found
+    rescue_from ActiveRecord::StatementInvalid, with: :render_database_error
+    rescue_from LlmCostTracker::InvalidFilterError, with: :render_invalid_filter
+
+    private
+
+    def ensure_llm_api_calls_table
+      return if llm_api_calls_table_available?
+
+      render template: "llm_cost_tracker/shared/setup_required"
+    end
+
+    def llm_api_calls_table_available?
+      LlmCostTracker::LlmApiCall.table_exists?
+    rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::StatementInvalid
+      false
+    end
+
+    def render_database_error(error)
+      @error = error
+      render "llm_cost_tracker/errors/database", status: :internal_server_error
+    end
+
+    def render_invalid_filter(error)
+      @error_message = error.message
+      render "llm_cost_tracker/errors/invalid_filter", status: :bad_request
+    end
+
+    def render_not_found
+      render "llm_cost_tracker/errors/not_found", status: :not_found
+    end
+  end
+end

data/app/controllers/llm_cost_tracker/calls_controller.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module LlmCostTracker
+  class CallsController < ApplicationController
+    SORT_ORDERS = {
+      "expensive"       => "total_cost DESC NULLS LAST, tracked_at DESC",
+      "input"           => "input_tokens DESC, tracked_at DESC",
+      "output"          => "output_tokens DESC, tracked_at DESC",
+      "slow"            => "latency_ms DESC NULLS LAST, tracked_at DESC",
+      "unknown_pricing" => "tracked_at DESC, id DESC"
+    }.freeze
+
+    CSV_EXPORT_LIMIT = 10_000
+
+    def index
+      @sort = params[:sort].to_s
+      scope = Dashboard::Filter.call(params: params)
+      scope = scope.unknown_pricing if @sort == "unknown_pricing"
+      ordered_scope = scope.order(Arel.sql(calls_order(@sort)))
+      @latency_available = LlmApiCall.latency_column?
+
+      respond_to do |format|
+        format.html do
+          @page = Pagination.call(params)
+          @calls_count = scope.count
+          @calls = ordered_scope.limit(@page.limit).offset(@page.offset).to_a
+        end
+        format.csv do
+          send_data render_csv(ordered_scope.limit(CSV_EXPORT_LIMIT)),
+                    type: "text/csv",
+                    disposition: %(attachment; filename="llm_calls_#{Time.now.utc.strftime('%Y%m%d_%H%M%S')}.csv")
+        end
+      end
+    end
+
+    def show
+      @call = LlmApiCall.find(params[:id])
+      @tags = @call.parsed_tags
+      @metadata_available = @call.has_attribute?("metadata")
+      @metadata = @call.read_attribute("metadata") if @metadata_available
+      @latency_available = LlmApiCall.latency_column?
+    end
+
+    private
+
+    def calls_order(sort)
+      SORT_ORDERS[sort] || "tracked_at DESC, id DESC"
+    end
+
+    def render_csv(relation)
+      latency = LlmApiCall.latency_column?
+      CSV.generate do |csv|
+        headers = %w[tracked_at provider model input_tokens output_tokens total_tokens total_cost]
+        headers << "latency_ms" if latency
+        headers << "tags"
+        csv << headers
+
+        relation.each do |call|
+          row = [
+            call.tracked_at&.utc&.iso8601,
+            call.provider,
+            call.model,
+            call.input_tokens,
+            call.output_tokens,
+            call.total_tokens,
+            call.total_cost
+          ]
+          row << call.latency_ms if latency
+          row << call.parsed_tags.to_json
+          csv << row
+        end
+      end
+    end
+  end
+end

data/app/controllers/llm_cost_tracker/dashboard_controller.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class DashboardController < ApplicationController
+    def index
+      @from_date, @to_date = overview_range
+      scope = Dashboard::Filter.call(params: overview_filter_params)
+      previous_scope = Dashboard::Filter.call(params: previous_filter_params)
+
+      @stats = Dashboard::OverviewStats.call(scope: scope, previous_scope: previous_scope)
+      @time_series = Dashboard::TimeSeries.call(scope: scope, from: @from_date, to: @to_date)
+      @top_models = Dashboard::TopModels.call(scope: scope, limit: 5)
+      @providers = Dashboard::ProviderBreakdown.call(scope: scope)
+    end
+
+    private
+
+    def overview_range
+      to_date = parsed_date(params[:to]) || Date.current
+      from_date = parsed_date(params[:from]) || (to_date - 29)
+      [from_date, to_date]
+    end
+
+    def previous_range
+      span_days = (@to_date - @from_date).to_i + 1
+      prev_to = @from_date - 1
+      prev_from = prev_to - (span_days - 1)
+      [prev_from, prev_to]
+    end
+
+    def overview_filter_params
+      params.to_unsafe_h.merge(
+        "from" => @from_date.iso8601,
+        "to" => @to_date.iso8601
+      )
+    end
+
+    def previous_filter_params
+      prev_from, prev_to = previous_range
+      params.to_unsafe_h.merge(
+        "from" => prev_from.iso8601,
+        "to" => prev_to.iso8601
+      )
+    end
+
+    def parsed_date(value)
+      return nil if value.to_s.strip.empty?
+
+      Date.iso8601(value.to_s)
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/app/controllers/llm_cost_tracker/data_quality_controller.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class DataQualityController < ApplicationController
+    def index
+      scope = Dashboard::Filter.call(params: params)
+      @stats = Dashboard::DataQuality.call(scope: scope)
+    end
+  end
+end

data/app/controllers/llm_cost_tracker/models_controller.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class ModelsController < ApplicationController
+    def index
+      scope = Dashboard::Filter.call(params: params)
+      @sort = params[:sort].to_s
+      @rows = Dashboard::TopModels.call(scope: scope, limit: nil, sort: @sort)
+      @latency_available = LlmApiCall.latency_column?
+    end
+  end
+end

data/app/controllers/llm_cost_tracker/tags_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class TagsController < ApplicationController
+    def index
+      scope = Dashboard::Filter.call(params: params)
+      @rows = Dashboard::TagKeyExplorer.call(scope: scope)
+    end
+
+    def show
+      @tag_key = params[:key]
+      scope = Dashboard::Filter.call(params: params)
+      @rows = Dashboard::TagBreakdown.call(scope: scope, key: @tag_key)
+      @total_calls = @rows.sum(&:calls)
+
+      tagged_rows = @rows.reject { |r| r.value == "(untagged)" }
+      @tagged_calls = tagged_rows.sum(&:calls)
+      @distinct_values = tagged_rows.size
+    end
+  end
+end

data/app/helpers/llm_cost_tracker/application_helper.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module ApplicationHelper
+    def coverage_percent(numerator, denominator)
+      return 0.0 unless denominator.to_i.positive?
+
+      (numerator.to_f / denominator) * 100.0
+    end
+
+    def money(value)
+      value = value.to_f
+      precision = value.abs < 0.01 && value != 0.0 ? 6 : 2
+
+      "$#{format("%.#{precision}f", value)}"
+    end
+
+    def optional_money(value)
+      value.nil? ? "n/a" : money(value)
+    end
+
+    def optional_number(value)
+      value.nil? ? "n/a" : number(value)
+    end
+
+    def number(value)
+      number_with_delimiter(value.to_i)
+    end
+
+    def format_tokens(value)
+      number(value)
+    end
+
+    def format_date(value)
+      value.respond_to?(:strftime) ? value.strftime("%Y-%m-%d %H:%M") : value.to_s
+    end
+
+    def pricing_status(call)
+      call.total_cost.nil? ? "Unknown pricing" : "Estimated"
+    end
+
+    def percent(value)
+      "#{format('%.1f', value.to_f)}%"
+    end
+
+    def delta_badge(delta_percent, mode: :cost)
+      return { text: "vs. prior: n/a", css_class: "lct-delta lct-delta-neutral" } if delta_percent.nil?
+
+      rounded = delta_percent.round(1)
+      return { text: "= vs. prior", css_class: "lct-delta lct-delta-neutral" } if rounded.zero?
+
+      sign = rounded.positive? ? "+" : ""
+      text = "#{sign}#{format('%.1f', rounded)}% vs. prior"
+      css_class = if mode == :neutral
+                    "lct-delta lct-delta-neutral"
+                  elsif rounded.positive?
+                    "lct-delta lct-delta-up"
+                  else
+                    "lct-delta lct-delta-down"
+                  end
+
+      { text: text, css_class: css_class }
+    end
+
+    def bar_width(value, max)
+      max = max.to_f
+      return "0%" unless max.positive?
+
+      "#{[(value.to_f / max) * 100.0, 100.0].min.round(2)}%"
+    end
+
+    def safe_json(value)
+      parsed = value.is_a?(String) ? JSON.parse(value) : value
+      JSON.pretty_generate(parsed || {})
+    rescue JSON::ParserError, TypeError
+      value.to_s
+    end
+
+    def tags_summary(tags, limit: 3)
+      tags = normalized_tags(tags)
+      return "(untagged)" if tags.empty?
+
+      summary = tags.first(limit).map { |key, value| "#{key}=#{tag_value_summary(value)}" }
+      summary << "+#{tags.size - limit}" if tags.size > limit
+      summary.join(", ")
+    end
+
+    def current_query(overrides = {})
+      request.query_parameters.symbolize_keys.merge(overrides)
+    end
+
+    private
+
+    def normalized_tags(tags)
+      return tags.transform_keys(&:to_s) if tags.is_a?(Hash)
+
+      JSON.parse(tags || "{}")
+    rescue JSON::ParserError, TypeError
+      {}
+    end
+
+    def tag_value_summary(value)
+      case value
+      when Hash, Array
+        JSON.generate(value)
+      else
+        value.to_s
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/data_quality.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module Dashboard
+    DataQualityStats = Data.define(
+      :total_calls,
+      :unknown_pricing_count,
+      :untagged_calls_count,
+      :missing_latency_count,
+      :latency_column_present,
+      :unknown_pricing_by_model
+    )
+
+    # Computes data quality metrics: coverage of cost, tags, and latency.
+    class DataQuality
+      class << self
+        def call(scope: LlmCostTracker::LlmApiCall.all)
+          total = scope.count
+          latency_present = LlmCostTracker::LlmApiCall.latency_column?
+
+          DataQualityStats.new(
+            total_calls: total,
+            unknown_pricing_count: scope.unknown_pricing.count,
+            untagged_calls_count: total - scope.with_json_tags.count,
+            missing_latency_count: latency_present ? scope.where(latency_ms: nil).count : nil,
+            latency_column_present: latency_present,
+            unknown_pricing_by_model: scope.unknown_pricing
+                                      .group(:model)
+                                      .order(Arel.sql("COUNT(*) DESC"))
+                                      .count
+                                      .first(10)
+                                      .to_h
+          )
+        end
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/filter.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "date"
+
+module LlmCostTracker
+  module Dashboard
+    # Parses dashboard params into an ActiveRecord relation.
+    #
+    # Invalid dates are ignored, pagination is handled elsewhere, and invalid
+    # tag keys raise InvalidFilterError so controllers can fail closed with HTTP 400.
+    class Filter
+      class << self
+        def call(scope: LlmCostTracker::LlmApiCall.all, params: {})
+          new(scope: scope, params: params).relation
+        end
+      end
+
+      def initialize(scope:, params:)
+        @scope = scope
+        @params = normalize_params(params)
+      end
+
+      def relation
+        filtered_scope = scope
+        filtered_scope = apply_date_filters(filtered_scope)
+        filtered_scope = apply_exact_filter(filtered_scope, :provider)
+        filtered_scope = apply_exact_filter(filtered_scope, :model)
+        apply_tag_filters(filtered_scope)
+      end
+
+      private
+
+      attr_reader :scope, :params
+
+      def normalize_params(params)
+        return {}.with_indifferent_access if params.nil?
+
+        raw = params.respond_to?(:to_unsafe_h) ? params.to_unsafe_h : params.to_h
+        raw.with_indifferent_access
+      rescue NoMethodError
+        {}.with_indifferent_access
+      end
+
+      def apply_date_filters(relation)
+        from = parse_date(:from)&.beginning_of_day
+        to = parse_date(:to)&.end_of_day
+
+        relation = relation.where(tracked_at: from..) if from
+        relation = relation.where(tracked_at: ..to) if to
+        relation
+      end
+
+      def apply_exact_filter(relation, key)
+        value = string_param(key)
+        return relation if value.nil?
+
+        relation.where(key => value)
+      end
+
+      def apply_tag_filters(relation)
+        tags = tag_params
+        return relation if tags.empty?
+
+        relation.by_tags(tags)
+      end
+
+      def tag_params
+        tags = hash_param(:tag)
+        tag_key = string_param(:tag_key)
+        tag_value = string_param(:tag_value)
+        tags = tags.merge(tag_key => tag_value) if tag_key && tag_value
+
+        tags.each_with_object({}) do |(key, value), normalized|
+          value = normalized_string(value)
+          next if value.nil?
+
+          normalized[LlmCostTracker::TagKey.validate!(key, error_class: LlmCostTracker::InvalidFilterError)] = value
+        end
+      end
+
+      def hash_param(key)
+        raw = params[key]
+        raw = raw.to_unsafe_h if raw.respond_to?(:to_unsafe_h)
+        raw = raw.to_h if raw.respond_to?(:to_h)
+        raw.is_a?(Hash) ? raw : {}
+      end
+
+      def parse_date(key)
+        value = string_param(key)
+        return nil if value.nil?
+
+        Date.iso8601(value)
+      rescue ArgumentError
+        nil
+      end
+
+      def string_param(key)
+        normalized_string(params[key])
+      end
+
+      def normalized_string(value)
+        return nil if value.nil?
+
+        value = value.to_s.strip
+        value.empty? ? nil : value
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/overview_stats.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module Dashboard
+    OverviewStatsData = Data.define(
+      :total_cost,
+      :total_calls,
+      :average_cost_per_call,
+      :average_latency_ms,
+      :unknown_pricing_count,
+      :previous_total_cost,
+      :previous_total_calls,
+      :cost_delta_percent,
+      :calls_delta_percent,
+      :monthly_budget_status
+    )
+
+    class OverviewStats
+      class << self
+        def call(scope: LlmCostTracker::LlmApiCall.all, previous_scope: nil)
+          current = aggregate(scope)
+          total_calls = current.calls_count.to_i
+          total_cost = current.total_cost_sum.to_f
+
+          previous = previous_scope && aggregate(previous_scope)
+          prev_cost = previous&.total_cost_sum.to_f
+          prev_calls = previous&.calls_count.to_i
+
+          OverviewStatsData.new(
+            total_cost: total_cost,
+            total_calls: total_calls,
+            average_cost_per_call: total_calls.positive? ? total_cost / total_calls : 0.0,
+            average_latency_ms: latency_value(current, scope),
+            unknown_pricing_count: current.unknown_pricing_count.to_i,
+            previous_total_cost: previous ? prev_cost : nil,
+            previous_total_calls: previous ? prev_calls : nil,
+            cost_delta_percent: previous ? delta_percent(total_cost, prev_cost) : nil,
+            calls_delta_percent: previous ? delta_percent(total_calls, prev_calls) : nil,
+            monthly_budget_status: budget_status
+          )
+        end
+
+        private
+
+        def aggregate(scope)
+          scope.select(aggregate_selects(scope)).take
+        end
+
+        def aggregate_selects(scope)
+          selects = [
+            "COUNT(*) AS calls_count",
+            "COALESCE(SUM(total_cost), 0) AS total_cost_sum",
+            "SUM(CASE WHEN total_cost IS NULL THEN 1 ELSE 0 END) AS unknown_pricing_count"
+          ]
+          selects << "AVG(latency_ms) AS average_latency" if scope.klass.latency_column?
+          selects.join(", ")
+        end
+
+        def latency_value(row, scope)
+          return nil unless scope.klass.latency_column?
+
+          row.average_latency&.to_f
+        end
+
+        def delta_percent(current, previous)
+          current = current.to_f
+          previous = previous.to_f
+          return nil if previous.zero?
+
+          ((current - previous) / previous) * 100.0
+        end
+
+        def budget_status
+          budget = LlmCostTracker.configuration.monthly_budget
+          return nil unless budget
+
+          spent = LlmCostTracker::LlmApiCall.this_month.total_cost
+          {
+            budget: budget.to_f,
+            spent: spent,
+            percent_used: budget.to_f.positive? ? (spent / budget.to_f) * 100.0 : 0.0
+          }
+        end
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module Dashboard
+    ProviderRow = Data.define(:provider, :calls, :total_cost, :share_percent)
+
+    # Aggregates cost and call counts per provider for a given scope.
+    # Sorted by total cost descending; providers with zero cost fall to the bottom
+    # but are still returned so users can see calls without pricing.
+    class ProviderBreakdown
+      def self.call(scope: LlmCostTracker::LlmApiCall.all)
+        new(scope: scope).rows
+      end
+
+      def initialize(scope:)
+        @scope = scope
+      end
+
+      def rows
+        grouped = scope
+                  .group(:provider)
+                  .select("provider, COUNT(*) AS calls_count, COALESCE(SUM(total_cost), 0) AS total_cost_sum")
+                  .order(Arel.sql("total_cost_sum DESC, calls_count DESC"))
+                  .to_a
+
+        total_cost = grouped.sum { |row| row.total_cost_sum.to_f }
+
+        grouped.map do |row|
+          cost = row.total_cost_sum.to_f
+          ProviderRow.new(
+            provider: row.provider,
+            calls: row.calls_count.to_i,
+            total_cost: cost,
+            share_percent: total_cost.positive? ? (cost / total_cost) * 100.0 : 0.0
+          )
+        end
+      end
+
+      private
+
+      attr_reader :scope
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module Dashboard
+    TagBreakdownRow = Data.define(
+      :value,
+      :calls,
+      :total_cost,
+      :average_cost_per_call
+    )
+
+    # Aggregates calls grouped by the distinct values of a single tag key.
+    # Invalid keys raise InvalidFilterError so controllers can return HTTP 400.
+    class TagBreakdown
+      class << self
+        def call(key:, scope: LlmCostTracker::LlmApiCall.all)
+          new(scope: scope, key: key).rows
+        end
+      end
+
+      def initialize(scope:, key:)
+        @scope = scope
+        @key = LlmCostTracker::TagKey.validate!(key, error_class: LlmCostTracker::InvalidFilterError)
+      end
+
+      def rows
+        costs = scope.cost_by_tag(key)
+        counts = counts_by_tag
+
+        costs.map do |value, total_cost|
+          calls = counts[value].to_i
+          total_cost = total_cost.to_f
+
+          TagBreakdownRow.new(
+            value: value,
+            calls: calls,
+            total_cost: total_cost,
+            average_cost_per_call: calls.positive? ? total_cost / calls : 0.0
+          )
+        end
+      end
+
+      private
+
+      attr_reader :scope, :key
+
+      def counts_by_tag
+        scope.group_by_tag(key).count.each_with_object(Hash.new(0)) do |(raw, count), hash|
+          hash[label(raw)] += count.to_i
+        end
+      end
+
+      def label(value)
+        value.nil? || value == "" ? "(untagged)" : value.to_s
+      end
+    end
+  end
+end