llm_cost_tracker 0.14.1 → 0.15.0

data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb

@@ -1,5 +1,3 @@
-<% extra_hidden = local_assigns.fetch(:extra_hidden, {}) %>
-<% extra_except = local_assigns.fetch(:extra_except, []) %>
 <% active = params[:model].presence %>
 <details class="lct-filter-pop" name="lct-filter">
   <summary class="lct-filter-pill <%= 'lct-active' if active %>">
@@ -9,8 +7,7 @@
     <svg class="lct-chev" width="10" height="6" viewBox="0 0 10 6" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="M1 1l4 4 4-4"/></svg>
   </summary>
   <%= form_with url: path, method: :get, local: true, html: { class: "lct-filter-pop-body" } do %>
-    <% extra_hidden.each do |k, v| %><%= hidden_field_tag k, v %><% end %>
-    <%= hidden_query_fields(current_query.except(:model, :page, :per, *extra_except)) %>
+    <%= hidden_query_fields(current_query.except(:model, :page, :per)) %>
     <div class="lct-filter-pop-field">
       <label for="lct-filter-model">Model</label>
       <%= select_tag :model, options_for_select(filter_options_for(:model), active), include_blank: "All models", id: "lct-filter-model" %>

data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb

@@ -1,5 +1,3 @@
-<% extra_hidden = local_assigns.fetch(:extra_hidden, {}) %>
-<% extra_except = local_assigns.fetch(:extra_except, []) %>
 <% active = params[:provider].presence %>
 <details class="lct-filter-pop" name="lct-filter">
   <summary class="lct-filter-pill <%= 'lct-active' if active %>">
@@ -9,8 +7,7 @@
     <svg class="lct-chev" width="10" height="6" viewBox="0 0 10 6" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="M1 1l4 4 4-4"/></svg>
   </summary>
   <%= form_with url: path, method: :get, local: true, html: { class: "lct-filter-pop-body" } do %>
-    <% extra_hidden.each do |k, v| %><%= hidden_field_tag k, v %><% end %>
-    <%= hidden_query_fields(current_query.except(:provider, :page, :per, *extra_except)) %>
+    <%= hidden_query_fields(current_query.except(:provider, :page, :per)) %>
     <div class="lct-filter-pop-field">
       <label for="lct-filter-provider">Provider</label>
       <%= select_tag :provider, options_for_select(filter_options_for(:provider), active), include_blank: "All providers", id: "lct-filter-provider" %>

data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb

@@ -1,7 +1,5 @@
-<% extra_hidden = local_assigns.fetch(:extra_hidden, {}) %>
-<% extra_except = local_assigns.fetch(:extra_except, []) %>
 <% active = params[:stream].presence %>
-<% display_value = case active when "yes" then "Streaming" when "no" then "Non-streaming" else "All" end %>
+<% display_value = LlmCostTracker::Dashboard::Filter::STREAM_FILTER_LABELS.fetch(active, "All") %>
 <details class="lct-filter-pop" name="lct-filter">
   <summary class="lct-filter-pill <%= 'lct-active' if active %>">
     <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><polyline points="22 12 18 12 15 21 9 3 6 12 2 12"/></svg>
@@ -10,8 +8,7 @@
     <svg class="lct-chev" width="10" height="6" viewBox="0 0 10 6" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round"><path d="M1 1l4 4 4-4"/></svg>
   </summary>
   <%= form_with url: path, method: :get, local: true, html: { class: "lct-filter-pop-body" } do %>
-    <% extra_hidden.each do |k, v| %><%= hidden_field_tag k, v %><% end %>
-    <%= hidden_query_fields(current_query.except(:stream, :page, :per, *extra_except)) %>
+    <%= hidden_query_fields(current_query.except(:stream, :page, :per)) %>
     <div class="lct-filter-pop-field">
       <label for="lct-filter-stream">Stream</label>
       <%= select_tag :stream, options_for_select(LlmCostTracker::Dashboard::Filter::STREAM_FILTER_OPTIONS, active), include_blank: "All calls", id: "lct-filter-stream" %>

data/app/views/llm_cost_tracker/tags/show.html.erb

@@ -1,13 +1,13 @@
 <% form_url = tag_path(params[:key]) %>
 
-<% if @value.present? %>
+<% if @breakdown.nil? %>
   <p class="lct-breadcrumb-back"><%= link_to "← All values for #{params[:key]}", tag_path(params[:key], current_query.except(:tag_value)) %></p>
   <h2 class="lct-page-title">Tag: <code class="lct-code-id"><%= params[:key] %></code> = <code class="lct-code-id"><%= @value %></code></h2>
 
   <div class="lct-filter-row">
-    <%= render "llm_cost_tracker/shared/filter_pill_date", path: form_url, extra_hidden: { tag_value: @value }, extra_except: [:tag_value] %>
-    <%= render "llm_cost_tracker/shared/filter_pill_provider", path: form_url, extra_hidden: { tag_value: @value }, extra_except: [:tag_value] %>
-    <%= render "llm_cost_tracker/shared/filter_pill_model", path: form_url, extra_hidden: { tag_value: @value }, extra_except: [:tag_value] %>
+    <%= render "llm_cost_tracker/shared/filter_pill_date", path: form_url %>
+    <%= render "llm_cost_tracker/shared/filter_pill_provider", path: form_url %>
+    <%= render "llm_cost_tracker/shared/filter_pill_model", path: form_url %>
 
     <% if params[:provider].present? || params[:model].present? %>
       <%= link_to "× Clear filters", tag_path(params[:key], current_query(provider: nil, model: nil, page: nil)), class: "lct-filter-clear" %>
@@ -35,7 +35,7 @@
       </div>
       <div class="lct-stat">
         <div class="lct-stat-head"><p class="lct-stat-label">Avg cost / call</p></div>
-        <p class="lct-stat-value"><%= money(@value_calls.positive? ? @value_total_cost / @value_calls : 0) %></p>
+        <p class="lct-stat-value"><%= money(@value_total_cost / @value_calls) %></p>
       </div>
     </div>
 
@@ -44,7 +44,7 @@
         <h2 class="lct-panel-title">Spend over time</h2>
         <span class="lct-panel-meta"><%= link_to "View calls →", calls_path(calls_query_for_tag(key: @key, value: @value)) %></span>
       </div>
-      <%= render "llm_cost_tracker/shared/spend_chart", series: @value_points, comparison_series: [] %>
+      <%= render "llm_cost_tracker/shared/spend_chart", series: @value_points %>
     </section>
   <% end %>
 

data/lib/llm_cost_tracker/budget/estimate.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+require_relative "../pricing/estimator"
+
+module LlmCostTracker
+  module Budget
+    Estimate = Data.define(:total, :largest) do
+      def self.for(provider:, model:, request:)
+        costs = request_costs(provider, model, request)
+        total = costs.sum(BigDecimal("0"))
+        new(total: total, largest: costs.max || total)
+      end
+
+      def self.request_costs(provider, model, request)
+        batch = request["requests"] if request && !model
+        return [cost(provider, model, request)] unless batch.is_a?(Array)
+
+        batch.map do |entry|
+          params = entry["params"] if entry.is_a?(Hash)
+          params.is_a?(Hash) ? cost(provider, params["model"], params) : 0
+        end
+      end
+
+      def self.cost(provider, model, request)
+        return BigDecimal("0") unless provider && model && request
+
+        Pricing::Estimator.call(provider: provider, model: model, request: request) || BigDecimal("0")
+      end
+
+      private_class_method :request_costs, :cost
+    end
+  end
+end

data/lib/llm_cost_tracker/budget/global.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+require_relative "limit"
+require_relative "../ledger"
+
+module LlmCostTracker
+  module Budget
+    module Global
+      PERIODS = { monthly: :month, daily: :day }.freeze
+
+      class << self
+        def enforce!(estimate, time:)
+          budgets = LlmCostTracker.configuration.budgets
+          per_call = budgets.per_call
+          largest = estimate.largest
+          Limit.global(:per_call, per_call).block!(largest) if per_call && largest.positive? && largest >= per_call
+
+          windows = { monthly: budgets.monthly, daily: budgets.daily }
+          each_exceeded(windows, time: time, estimate: estimate.total) { |limit, total| limit.block!(total) }
+        end
+
+        def post_spend_errors(event, behavior_override)
+          budgets = LlmCostTracker.configuration.budgets
+          errors = [per_call_error(event, budgets.per_call, behavior_override)]
+          each_exceeded({ daily: budgets.daily, monthly: budgets.monthly }, time: event.tracked_at) do |limit, total|
+            errors << limit.handle_exceeded(total: total,
+                                            previous_total: total - event.total_cost,
+                                            last_event: event,
+                                            behavior_override: behavior_override)
+          end
+          errors
+        end
+
+        def notify_repriced(changes, time:)
+          budgets = LlmCostTracker.configuration.budgets
+          each_exceeded({ daily: budgets.daily, monthly: budgets.monthly }, time: time) do |limit, total|
+            limit.handle_exceeded(total: total,
+                                  previous_total: total - limit.amount_in_window(changes, time),
+                                  behavior_override: :notify)
+          end
+        end
+
+        private
+
+        def per_call_error(event, per_call, behavior_override)
+          total = event.total_cost
+          return unless per_call && total >= per_call
+
+          Limit.global(:per_call, per_call).handle_exceeded(
+            total: total, previous_total: nil, last_event: event, behavior_override: behavior_override
+          )
+        end
+
+        def each_exceeded(windows, time:, estimate: BigDecimal("0"))
+          windows = windows.compact
+          return if windows.empty?
+
+          totals = Ledger::Period::Totals.call(windows.keys.map { |type| PERIODS.fetch(type) }, time: time)
+          windows.each do |type, budget|
+            total = totals.fetch(PERIODS.fetch(type)) + estimate
+            yield Limit.global(type, budget), total if total >= budget
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/budget/limit.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+module LlmCostTracker
+  module Budget
+    Limit = Data.define(:budget_type, :budget, :scope, :behavior, :on_exceeded) do
+      def self.global(budget_type, budget)
+        new(budget_type: budget_type, budget: budget, scope: nil, behavior: nil, on_exceeded: nil)
+      end
+
+      def over?(total)
+        budget_type == :calls ? total > budget : total >= budget
+      end
+
+      def charge(cost)
+        budget_type == :calls ? 1 : cost
+      end
+
+      def counts?(event)
+        budget_type == :calls || !event.total_cost.nil?
+      end
+
+      def amount_in_window(changes, time)
+        start = PerTag.window_start(budget_type, time)
+        changes.sum(BigDecimal("0")) { |change| start.nil? || change.tracked_at >= start ? change.total_cost : 0 }
+      end
+
+      def block!(total)
+        raise BudgetExceededError.new(
+          budget_type: budget_type, total: total, budget: budget, stage: :pre_send, scope: scope
+        )
+      end
+
+      def handle_exceeded(total:, previous_total:, last_event: nil, behavior_override: nil)
+        budgets = LlmCostTracker.configuration.budgets
+        raising = %i[raise block_requests].include?(behavior_override || behavior || budgets.exceeded_behavior)
+        callback = on_exceeded || budgets.on_exceeded
+        payload = {
+          budget_type: budget_type,
+          total: total,
+          budget: budget,
+          last_event: last_event,
+          stage: :post_spend,
+          scope: scope
+        }
+        callback.call(payload) if callback && (previous_total.nil? || !over?(previous_total))
+        BudgetExceededError.new(**payload) if raising
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/budget/per_tag.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "limit"
+require_relative "../timing"
 require_relative "../ledger/isolation"
 require_relative "../ledger/schema/adapter"
 require_relative "../ledger/tags/encoding"
@@ -14,7 +16,15 @@ module LlmCostTracker
       SLOW_READ_SECONDS = 0.1
       DEFAULT_BACKFILL_BATCH = 5_000
 
-      Rule = Data.define(:key, :value, :windows, :behavior, :on_exceeded)
+      Rule = Data.define(:key, :value, :windows, :behavior, :on_exceeded) do
+        def limit(window)
+          Limit.new(budget_type: window,
+                    budget: windows.fetch(window),
+                    scope: { key: key, value: value },
+                    behavior: behavior,
+                    on_exceeded: on_exceeded)
+        end
+      end
 
       class << self
         def configured
@@ -30,7 +40,7 @@ module LlmCostTracker
         end
 
         def blocking?
-          active? && configured.each_value.any? { |entry| behavior_for(entry) == :block_requests }
+          configured.each_value.any? { |entry| behavior_for(entry) == :block_requests } && active?
         end
 
         def rules_for(tags, blocking_only: false)
@@ -38,13 +48,13 @@ module LlmCostTracker
 
           normalized = (tags || {}).to_h.transform_keys(&:to_s)
           configured.filter_map do |key, entry|
-            raw = normalized[key]
-            next if raw.nil?
+            value = Ledger::Tags::Encoding.encode(normalized[key])
+            next if value.empty?
             next if blocking_only && behavior_for(entry) != :block_requests
 
             Rule.new(
               key: key,
-              value: Ledger::Tags::Encoding.encode(raw),
+              value: value,
               windows: entry[:windows],
               behavior: behavior_for(entry),
               on_exceeded: on_exceeded_for(entry)
@@ -59,33 +69,36 @@ module LlmCostTracker
         end
 
         def spend(key, value, window, time:)
-          spend_by_value(key, [value], window, window_start(window, time)).fetch(value, 0).to_d
+          spend_by_value(key, [value], window, window_start(window, time)).fetch(value, [0]).first
         end
 
-        def spend_by_value(key, values, window, bucket)
-          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        def spend_by_value(key, values, window, bucket, upto = nil)
+          started_at = Timing.now_monotonic
+          aggregates = sum_columns(window, upto)
           totals = Ledger::Isolation.guard(LlmCostTracker::CallTag) do
-            LlmCostTracker::CallTag
-              .where(key: key, value: values, TIME_COLUMN => window_range(window, bucket))
-              .group(:value).sum(COST_COLUMN)
+            pluck_by_exact_value(spend_rows(key, values, window, bucket), values, aggregates).to_h do |value, *sums|
+              [value.dup.force_encoding(Encoding::UTF_8), sums.map { |sum| window == :calls ? sum.to_i : sum.to_d }]
+            end
           end
-          warn_slow_read(key, window, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at)
+          warn_slow_read(key, window, Timing.now_monotonic - started_at)
           totals
         end
 
         def window_start(window, time)
-          time.to_time.utc.public_send(WINDOW_STARTS.fetch(window))
+          WINDOW_STARTS[window]&.then { |start| time.to_time.utc.public_send(start) }
         end
 
         def backfill(batch_size: DEFAULT_BACKFILL_BATCH)
           return 0 unless columns?
 
           filled = 0
+          after = 0
           loop do
-            copied = copy_next_batch(batch_size)
-            break if copied.zero?
+            ids = unfilled_ids(after, batch_size)
+            break if ids.empty?
 
-            filled += copied
+            filled += LlmCostTracker::CallTag.lease_connection.update(copy_sql(ids))
+            after = ids.last
           end
           filled
         end
@@ -97,22 +110,42 @@ module LlmCostTracker
 
         private
 
+        def sum_columns(window, upto)
+          measure = window == :calls ? "1" : COST_COLUMN
+          total = "SUM(#{measure})"
+          upto_sum = "SUM(CASE WHEN #{TIME_COLUMN} <= ? THEN #{measure} ELSE 0 END)"
+          upto_total = upto ? LlmCostTracker::CallTag.sanitize_sql_array([upto_sum, upto]) : total
+          [Arel.sql(total), Arel.sql(upto_total)]
+        end
+
+        def spend_rows(key, values, window, bucket)
+          rows = LlmCostTracker::CallTag.where(key: key, value: values)
+          bucket ? rows.where(TIME_COLUMN => window_range(window, bucket)) : rows.where.not(TIME_COLUMN => nil)
+        end
+
+        def pluck_by_exact_value(rows, values, aggregates)
+          column = :value
+          if Ledger::Schema::Adapter.mysql?(LlmCostTracker::CallTag.lease_connection)
+            column = Arel.sql("CAST(value AS BINARY)")
+            rows = rows.where(column.in(values))
+          end
+          rows.group(column).pluck(column, *aggregates)
+        end
+
         def window_range(window, bucket)
           bucket...bucket.public_send(WINDOW_NEXTS.fetch(window))
         end
 
-        def copy_next_batch(batch_size)
-          ids = LlmCostTracker::CallTag.where(TIME_COLUMN => nil).limit(batch_size).pluck(:id)
-          return 0 if ids.empty?
-
-          LlmCostTracker::CallTag.connection.update(copy_sql(ids))
+        def unfilled_ids(after, batch_size)
+          LlmCostTracker::CallTag.where(TIME_COLUMN => nil).where(id: (after + 1)..)
+                                 .order(:id).limit(batch_size).pluck(:id)
         end
 
         def copy_sql(ids)
           tags = LlmCostTracker::CallTag.quoted_table_name
           calls = LlmCostTracker::Call.quoted_table_name
           list = ids.join(",")
-          return <<~SQL.squish unless Ledger::Schema::Adapter.postgresql?(LlmCostTracker::CallTag.connection)
+          return <<~SQL.squish unless Ledger::Schema::Adapter.postgresql?(LlmCostTracker::CallTag.lease_connection)
             UPDATE #{tags} t JOIN #{calls} c ON c.id = t.llm_cost_tracker_call_id
                SET t.#{COST_COLUMN} = c.#{COST_COLUMN}, t.#{TIME_COLUMN} = c.#{TIME_COLUMN}
              WHERE t.id IN (#{list})
@@ -140,10 +173,15 @@ module LlmCostTracker
           @slow_keys ||= Set.new
           return unless @slow_keys.add?(key)
 
+          reason = if WINDOW_STARTS.key?(window)
+                     "A tag with few distinct values covers most of the ledger, so its budget check cannot use " \
+                       "an index effectively. Budget high-cardinality tags such as a tenant or user id."
+                   else
+                     "A #{window} limit sums every call the value has recorded, so it suits short-lived values " \
+                       "such as a run."
+                   end
           Logging.warn(
-            "config.budgets.per_tag[#{key.inspect}] #{window} read took #{(seconds * 1000).round} ms. " \
-            "A tag with few distinct values covers most of the ledger, so its budget check cannot use " \
-            "an index effectively. Budget high-cardinality tags such as a tenant or user id."
+            "config.budgets.per_tag[#{key.inspect}] #{window} read took #{(seconds * 1000).round} ms. #{reason}"
           )
         end
 

data/lib/llm_cost_tracker/budget/tagged.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "per_tag"
+require_relative "../tags/context"
+
+module LlmCostTracker
+  module Budget
+    module Tagged
+      class << self
+        def enforce!(tags, estimated_cost, time:, blocking_only:)
+          PerTag.rules_for(tags || Tags::Context.tags, blocking_only: blocking_only).each do |rule|
+            rule.windows.each_key do |window|
+              limit = rule.limit(window)
+              total = PerTag.spend(rule.key, rule.value, window, time: time) + limit.charge(estimated_cost)
+              limit.block!(total) if limit.over?(total)
+            end
+          end
+        end
+
+        def post_spend_errors(events, behavior_override)
+          window_buckets(scored_rules(events, behavior_override)).flat_map do |(key, window, bucket), recorded_by_rule|
+            upto = recorded_by_rule.values.flatten.map(&:tracked_at).max unless Ingestion.async?
+            totals = PerTag.spend_by_value(key, recorded_by_rule.keys.map(&:value), window, bucket, upto)
+            recorded_by_rule.filter_map do |rule, recorded|
+              total, total_through_batch = totals.fetch(rule.value, [0, 0])
+              bucket_error(rule.limit(window), recorded, total, total_through_batch, behavior_override)
+            end
+          end
+        end
+
+        def notify_repriced(changes, time:)
+          PerTag.rules_for_events(changes).each do |rule, repriced|
+            rule.windows.except(:calls).each_key do |window|
+              limit = rule.limit(window)
+              amount = limit.amount_in_window(repriced, time)
+              total = PerTag.spend(rule.key, rule.value, window, time: time) if rule.on_exceeded && amount.positive?
+              next unless total && limit.over?(total)
+
+              limit.handle_exceeded(total: total, previous_total: total - amount, behavior_override: :notify)
+            end
+          end
+        end
+
+        private
+
+        def scored_rules(events, behavior_override)
+          by_rule = PerTag.rules_for_events(events)
+          behavior_override == :notify ? by_rule.reject { |rule, _| rule.on_exceeded.nil? } : by_rule
+        end
+
+        def bucket_error(limit, recorded, total, total_through_batch, behavior_override)
+          return unless limit.over?(total)
+
+          reference = limit.over?(total_through_batch) ? total_through_batch : total
+          limit.handle_exceeded(total: total,
+                                previous_total: reference - recorded.sum { |event| limit.charge(event.total_cost) },
+                                last_event: recorded.last,
+                                behavior_override: behavior_override)
+        end
+
+        def window_buckets(by_rule)
+          by_rule.each_with_object({}) do |(rule, events), grouped|
+            rule.windows.each_key do |window|
+              limit = rule.limit(window)
+              events.select { |event| limit.counts?(event) }
+                    .group_by { |event| PerTag.window_start(window, event.tracked_at) }
+                    .each { |bucket, bucket_events| (grouped[[rule.key, window, bucket]] ||= {})[rule] = bucket_events }
+            end
+          end
+        end
+      end
+    end
+  end
+end