llm_cost_tracker 0.1.4 → 0.2.0.alpha2

data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module Dashboard
+    TagKeyRow = Data.define(:key, :calls_count, :distinct_values)
+
+    class TagKeyExplorer
+      RUBY_FALLBACK_LIMIT = 50_000
+
+      class << self
+        def call(scope: LlmCostTracker::LlmApiCall.all)
+          new(scope: scope).rows
+        end
+      end
+
+      def initialize(scope:)
+        @scope = scope
+        @connection = LlmCostTracker::LlmApiCall.connection
+      end
+
+      def rows
+        sql = build_sql
+        return ruby_fallback_rows if sql.nil?
+
+        results = @connection.select_all(sql).to_a
+        results.map do |row|
+          TagKeyRow.new(
+            key: row["key"].to_s,
+            calls_count: row["calls_count"].to_i,
+            distinct_values: row["distinct_values"].to_i
+          )
+        end
+      rescue StandardError => e
+        LlmCostTracker::Logging.warn("Tag key discovery failed (#{connection.adapter_name}): #{e.class}: #{e.message}")
+        []
+      end
+
+      private
+
+      attr_reader :scope, :connection
+
+      def subquery
+        scope.to_sql
+      end
+
+      def build_sql
+        case connection.adapter_name
+        when /postgres/i then postgresql_sql
+        when /mysql/i    then nil
+        else                  sqlite_sql
+        end
+      end
+
+      def ruby_fallback_rows
+        calls_counter = Hash.new(0)
+        values_per_key = Hash.new { |h, k| h[k] = Set.new }
+
+        scope.limit(RUBY_FALLBACK_LIMIT).pluck(:tags).each do |raw|
+          tags = parse_tags(raw)
+          next if tags.empty?
+
+          tags.each do |key, value|
+            calls_counter[key] += 1
+            values_per_key[key] << value.to_s
+          end
+        end
+
+        calls_counter
+          .sort_by { |_, count| -count }
+          .map do |key, count|
+            TagKeyRow.new(key: key.to_s, calls_count: count, distinct_values: values_per_key[key].size)
+          end
+      rescue StandardError => e
+        LlmCostTracker::Logging.warn("Tag key Ruby fallback failed: #{e.class}: #{e.message}")
+        []
+      end
+
+      def parse_tags(raw)
+        case raw
+        when Hash then raw
+        when String
+          return {} if raw.strip.empty?
+
+          parsed = JSON.parse(raw)
+          parsed.is_a?(Hash) ? parsed : {}
+        else
+          {}
+        end
+      rescue JSON::ParserError
+        {}
+      end
+
+      def postgresql_sql
+        <<~SQL.squish
+          SELECT key,
+                 COUNT(*) AS calls_count,
+                 COUNT(DISTINCT (sub.tags::jsonb)->>key) AS distinct_values
+          FROM (#{subquery}) AS sub,
+               jsonb_object_keys(sub.tags::jsonb) AS key
+          WHERE sub.tags IS NOT NULL
+            AND sub.tags::jsonb <> '{}'::jsonb
+          GROUP BY key
+          ORDER BY calls_count DESC
+        SQL
+      end
+
+      def sqlite_sql
+        <<~SQL.squish
+          SELECT je.key AS key,
+                 COUNT(*) AS calls_count,
+                 COUNT(DISTINCT je.value) AS distinct_values
+          FROM (#{subquery}) AS sub,
+               json_each(sub.tags) AS je
+          WHERE sub.tags IS NOT NULL
+            AND sub.tags != '{}'
+            AND sub.tags != ''
+          GROUP BY je.key
+          ORDER BY calls_count DESC
+        SQL
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/time_series.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "date"
+
+module LlmCostTracker
+  module Dashboard
+    class TimeSeries
+      DEFAULT_DAYS = 30
+
+      class << self
+        def call(scope: LlmCostTracker::LlmApiCall.all, from: nil, to: Date.current)
+          new(scope: scope, from: from, to: to).points
+        end
+      end
+
+      def initialize(scope:, from:, to:)
+        @scope = scope
+        @to = to.to_date
+        @from = from ? from.to_date : (@to - (DEFAULT_DAYS - 1))
+      end
+
+      def points
+        costs = scoped_costs
+
+        (from..to).map do |date|
+          label = date.iso8601
+          { label: label, cost: costs.fetch(label, 0.0) }
+        end
+      end
+
+      private
+
+      attr_reader :scope, :from, :to
+
+      def scoped_costs
+        scope
+          .where(tracked_at: from.beginning_of_day..to.end_of_day)
+          .group_by_period(:day)
+          .sum(:total_cost)
+          .transform_values(&:to_f)
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/dashboard/top_models.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module Dashboard
+    TopModel = Data.define(
+      :provider,
+      :model,
+      :calls,
+      :total_cost,
+      :average_cost_per_call,
+      :input_tokens,
+      :output_tokens,
+      :average_latency_ms
+    )
+
+    class TopModels
+      DEFAULT_LIMIT = 5
+      SORT_OPTIONS = {
+        "cost"     => "total_cost_sum DESC",
+        "calls"    => "calls_count DESC",
+        "avg_cost" => "total_cost_sum / calls_count DESC",
+        "latency"  => "average_latency DESC NULLS LAST"
+      }.freeze
+      DEFAULT_SORT = "cost"
+
+      class << self
+        def call(scope: LlmCostTracker::LlmApiCall.all, limit: DEFAULT_LIMIT, sort: DEFAULT_SORT)
+          new(scope: scope, limit: limit, sort: sort).rows
+        end
+      end
+
+      def initialize(scope:, limit:, sort: DEFAULT_SORT)
+        @scope = scope
+        @limit = limit
+        @sort = SORT_OPTIONS.key?(sort.to_s) ? sort.to_s : DEFAULT_SORT
+      end
+
+      def rows
+        grouped_rows.map do |row|
+          calls = row.calls_count.to_i
+          total_cost = row.total_cost_sum.to_f
+
+          TopModel.new(
+            provider: row.provider,
+            model: row.model,
+            calls: calls,
+            total_cost: total_cost,
+            average_cost_per_call: calls.positive? ? total_cost / calls : 0.0,
+            input_tokens: row.input_tokens_sum.to_i,
+            output_tokens: row.output_tokens_sum.to_i,
+            average_latency_ms: average_latency(row)
+          )
+        end
+      end
+
+      private
+
+      attr_reader :scope, :limit, :sort
+
+      def grouped_rows
+        order_sql = sort == "latency" && !scope.klass.latency_column? ? "total_cost_sum DESC" : SORT_OPTIONS[sort]
+        scope
+          .group(:provider, :model)
+          .select(selects)
+          .order(Arel.sql(order_sql))
+          .then { |r| limit ? r.limit(limit) : r }
+      end
+
+      def selects
+        columns = [
+          "provider",
+          "model",
+          "COUNT(*) AS calls_count",
+          "COALESCE(SUM(total_cost), 0) AS total_cost_sum",
+          "COALESCE(SUM(input_tokens), 0) AS input_tokens_sum",
+          "COALESCE(SUM(output_tokens), 0) AS output_tokens_sum"
+        ]
+        columns << "AVG(latency_ms) AS average_latency" if scope.klass.latency_column?
+        columns.join(", ")
+      end
+
+      def average_latency(row)
+        return nil unless scope.klass.latency_column?
+
+        row.average_latency&.to_f
+      end
+    end
+  end
+end

data/app/services/llm_cost_tracker/pagination.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class Pagination
+    DEFAULT_PER = 50
+    MAX_PER = 200
+    MIN_PAGE = 1
+
+    attr_reader :page, :per
+
+    def self.call(params)
+      params = normalize_params(params)
+      new(
+        page: integer_param(params, :page, default: MIN_PAGE, min: MIN_PAGE),
+        per: integer_param(params, :per, default: DEFAULT_PER, min: 1, max: MAX_PER)
+      )
+    end
+
+    def self.normalize_params(params)
+      return {}.with_indifferent_access if params.nil?
+
+      raw = params.respond_to?(:to_unsafe_h) ? params.to_unsafe_h : params.to_h
+      raw.with_indifferent_access
+    end
+    private_class_method :normalize_params
+
+    def self.integer_param(params, key, default:, min:, max: nil)
+      value = Integer(params[key], 10)
+      value = [value, min].max
+      value = [value, max].min if max
+      value
+    rescue ArgumentError, TypeError
+      default
+    end
+    private_class_method :integer_param
+
+    def initialize(page:, per:)
+      @page = page
+      @per = per
+      freeze
+    end
+
+    def limit
+      per
+    end
+
+    def offset
+      (page - 1) * per
+    end
+
+    def prev_page?
+      page > MIN_PAGE
+    end
+
+    def next_page?(total_count)
+      offset + per < total_count.to_i
+    end
+  end
+end

data/app/views/layouts/llm_cost_tracker/application.html.erb

@@ -0,0 +1,342 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>LLM Costs</title>
+    <style>
+      :root {
+        --lct-bg: #f7f8fa;
+        --lct-panel: #ffffff;
+        --lct-border: #d9dee7;
+        --lct-text: #1f2937;
+        --lct-muted: #64748b;
+        --lct-accent: #0f766e;
+        --lct-accent-soft: #ccfbf1;
+        --lct-danger: #b91c1c;
+      }
+
+      body {
+        margin: 0;
+        background: var(--lct-bg);
+        color: var(--lct-text);
+        font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+      }
+
+      .lct-shell {
+        max-width: 1120px;
+        margin: 0 auto;
+        padding: 24px;
+      }
+
+      .lct-header {
+        display: flex;
+        align-items: center;
+        justify-content: space-between;
+        gap: 24px;
+        margin-bottom: 24px;
+      }
+
+      .lct-title {
+        margin: 0;
+        font-size: 24px;
+        line-height: 1.2;
+      }
+
+      .lct-nav {
+        display: flex;
+        gap: 12px;
+        flex-wrap: wrap;
+      }
+
+      .lct-nav a {
+        color: var(--lct-muted);
+        text-decoration: none;
+        font-size: 14px;
+      }
+
+      .lct-nav a:hover {
+        color: var(--lct-accent);
+      }
+
+      .lct-nav .lct-active {
+        color: var(--lct-accent);
+        font-weight: 700;
+      }
+
+      .lct-panel {
+        background: var(--lct-panel);
+        border: 1px solid var(--lct-border);
+        border-radius: 8px;
+        padding: 24px;
+        margin-bottom: 20px;
+      }
+
+      .lct-muted {
+        color: var(--lct-muted);
+      }
+
+      .lct-code {
+        background: #eef2f7;
+        border-radius: 6px;
+        padding: 2px 6px;
+        font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace;
+        font-size: 0.95em;
+      }
+
+      .lct-grid {
+        display: grid;
+        gap: 16px;
+      }
+
+      .lct-stats {
+        grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+      }
+
+      .lct-two-col {
+        grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
+      }
+
+      .lct-stat {
+        background: var(--lct-panel);
+        border: 1px solid var(--lct-border);
+        border-radius: 8px;
+        padding: 18px;
+      }
+
+      .lct-stat-label {
+        color: var(--lct-muted);
+        font-size: 13px;
+        margin: 0 0 8px;
+      }
+
+      .lct-stat-value {
+        font-size: 24px;
+        font-weight: 700;
+        margin: 0;
+      }
+
+      .lct-section-title {
+        margin: 0 0 16px;
+        font-size: 18px;
+      }
+
+      .lct-table {
+        border-collapse: collapse;
+        width: 100%;
+        font-size: 14px;
+      }
+
+      .lct-stat-sub {
+        color: var(--lct-muted);
+        font-size: 12px;
+        margin: 4px 0 0;
+      }
+
+      .lct-delta {
+        font-size: 12px;
+        margin: 4px 0 0;
+      }
+
+      .lct-delta-neutral { color: var(--lct-muted); }
+      .lct-delta-up      { color: #b91c1c; }
+      .lct-delta-down    { color: #047857; }
+
+      .lct-table-wrap {
+        overflow-x: auto;
+      }
+
+      .lct-table th,
+      .lct-table td {
+        border-bottom: 1px solid var(--lct-border);
+        padding: 10px 8px;
+        text-align: left;
+        vertical-align: middle;
+      }
+
+      .lct-table th {
+        color: var(--lct-muted);
+        font-weight: 600;
+        font-size: 12px;
+        text-transform: uppercase;
+      }
+
+      .lct-table td:last-child,
+      .lct-table th:last-child {
+        text-align: right;
+      }
+
+      .lct-calls-table td:last-child,
+      .lct-calls-table th:last-child {
+        text-align: left;
+      }
+
+      .lct-bar-track {
+        background: #e5e7eb;
+        border-radius: 999px;
+        height: 8px;
+        overflow: hidden;
+        min-width: 96px;
+      }
+
+      .lct-bar-fill {
+        background: var(--lct-accent);
+        height: 100%;
+      }
+
+      .lct-budget-fill {
+        background: var(--lct-danger);
+      }
+
+      .lct-empty {
+        padding: 32px;
+        text-align: center;
+      }
+
+      .lct-filters {
+        display: grid;
+        gap: 12px;
+        grid-template-columns: repeat(6, minmax(0, 1fr));
+      }
+
+      .lct-field {
+        display: flex;
+        flex-direction: column;
+        gap: 6px;
+      }
+
+      .lct-field label {
+        color: var(--lct-muted);
+        font-size: 12px;
+        font-weight: 600;
+        text-transform: uppercase;
+      }
+
+      .lct-field input,
+      .lct-field select {
+        border: 1px solid var(--lct-border);
+        border-radius: 6px;
+        color: var(--lct-text);
+        font: inherit;
+        padding: 8px 10px;
+        background: var(--lct-panel);
+      }
+
+      .lct-button-row {
+        align-items: end;
+        display: flex;
+        gap: 8px;
+      }
+
+      .lct-button {
+        background: var(--lct-accent);
+        border: 1px solid var(--lct-accent);
+        border-radius: 6px;
+        color: #ffffff;
+        display: inline-block;
+        font: inherit;
+        padding: 8px 12px;
+        text-decoration: none;
+      }
+
+      .lct-button-secondary {
+        background: var(--lct-panel);
+        color: var(--lct-accent);
+      }
+
+      .lct-pagination {
+        align-items: center;
+        display: flex;
+        justify-content: space-between;
+        gap: 12px;
+        margin-top: 16px;
+      }
+
+      .lct-tags {
+        display: inline-block;
+        max-width: 280px;
+        overflow: hidden;
+        text-overflow: ellipsis;
+        white-space: nowrap;
+      }
+
+      .lct-nowrap {
+        white-space: nowrap;
+      }
+
+      .lct-detail-grid {
+        display: grid;
+        gap: 16px;
+        grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
+      }
+
+      .lct-dl {
+        display: grid;
+        gap: 12px;
+        grid-template-columns: minmax(120px, 0.4fr) minmax(0, 1fr);
+        margin: 0;
+      }
+
+      .lct-dl dt {
+        color: var(--lct-muted);
+        font-size: 12px;
+        font-weight: 600;
+        text-transform: uppercase;
+      }
+
+      .lct-dl dd {
+        margin: 0;
+      }
+
+      .lct-pre {
+        background: #0f172a;
+        border-radius: 8px;
+        color: #e5e7eb;
+        margin: 0;
+        overflow: auto;
+        padding: 16px;
+      }
+
+      @media (max-width: 800px) {
+        .lct-header {
+          align-items: flex-start;
+          flex-direction: column;
+        }
+
+        .lct-stats,
+        .lct-two-col {
+          grid-template-columns: 1fr;
+        }
+
+        .lct-filters {
+          grid-template-columns: 1fr;
+        }
+
+        .lct-pagination {
+          align-items: flex-start;
+          flex-direction: column;
+        }
+
+        .lct-detail-grid {
+          grid-template-columns: 1fr;
+        }
+      }
+    </style>
+  </head>
+  <body>
+    <main class="lct-shell">
+      <header class="lct-header">
+        <h1 class="lct-title">LLM Costs</h1>
+        <nav class="lct-nav" aria-label="Dashboard">
+          <%= link_to "Overview", root_path, class: ("lct-active" if request.path.delete_suffix("/") == root_path.delete_suffix("/")) %>
+          <%= link_to "Models", models_path, class: ("lct-active" if request.path.start_with?(models_path)) %>
+          <%= link_to "Calls", calls_path, class: ("lct-active" if request.path.start_with?(calls_path)) %>
+          <%= link_to "Tags", tags_path, class: ("lct-active" if request.path.start_with?(tags_path)) %>
+          <%= link_to "Data Quality", data_quality_path, class: ("lct-active" if request.path.start_with?(data_quality_path)) %>
+        </nav>
+      </header>
+
+      <%= yield %>
+    </main>
+  </body>
+</html>