llm_cost_tracker 0.7.3 → 0.9.0

data/lib/llm_cost_tracker/pricing/sync.rb

@@ -8,7 +8,6 @@ require "rubygems"
 require_relative "registry"
 require_relative "sync/fetcher"
 require_relative "sync/registry_diff"
-require_relative "sync/registry_loader"
 require_relative "sync/registry_writer"
 
 module LlmCostTracker
@@ -39,7 +38,7 @@ module LlmCostTracker
 
         def refresh(path: DEFAULT_OUTPUT_PATH, url: DEFAULT_REMOTE_URL, preview: false, fetcher: Fetcher.new,
                     today: Date.today)
-          current = RegistryLoader.new.call(path: path, seed_path: Registry::DEFAULT_PRICES_PATH)
+          current = load_registry(path)
           response = fetcher.get(url, etag: current.dig("metadata", "source_version"))
 
           if response.not_modified
@@ -55,7 +54,10 @@ module LlmCostTracker
           end
 
           remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
-          RegistryWriter.new.call(path: path, registry: remote) unless preview
+          unless preview
+            RegistryWriter.new.call(path: path, registry: remote)
+            invalidate_pricing_caches!
+          end
           refresh_result(
             path: path,
             url: url,
@@ -67,8 +69,14 @@ module LlmCostTracker
           )
         end
 
+        def invalidate_pricing_caches!
+          Pricing::Lookup.reset!
+          Pricing::Registry.reset!
+          Pricing::ServiceCharges.reset!
+        end
+
         def check(path: DEFAULT_OUTPUT_PATH, url: DEFAULT_REMOTE_URL, fetcher: Fetcher.new, today: Date.today)
-          current = RegistryLoader.new.call(path: path, seed_path: Registry::DEFAULT_PRICES_PATH)
+          current = load_registry(path)
           response = fetcher.get(url, etag: current.dig("metadata", "source_version"))
 
           if response.not_modified
@@ -82,7 +90,7 @@ module LlmCostTracker
           end
 
           remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
-          changes = RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {}))
+          changes = registry_changes(current, remote)
 
           CheckResult.new(
             path: path,
@@ -118,10 +126,15 @@ module LlmCostTracker
             raise Error, "remote pricing snapshot requires llm_cost_tracker >= #{min_gem_version}"
           end
 
-          models = registry.fetch("models", {})
-          Registry.normalize_price_table(models)
+          raw_models = registry.fetch("models", {})
+          models = Registry.normalize_price_table(raw_models).each_with_object({}) do |(model, prices), normalized|
+            model_metadata = (raw_models[model] || {}).slice(*Registry::METADATA_KEYS)
+            normalized[model] = model_metadata.merge(prices.to_h { |key, value| [key.name, value] })
+          end
+          service_charges = registry["service_charges"]
+          ServiceCharges.rates_from_registry(registry, context: "remote pricing snapshot") if service_charges
 
-          registry.merge(
+          normalized = {
             "metadata" => metadata.merge(
               "schema_version" => schema_version,
               "updated_at" => metadata["updated_at"] || today.iso8601,
@@ -129,11 +142,21 @@ module LlmCostTracker
               "source_version" => response.source_version
             ),
             "models" => models
-          )
+          }
+          normalized["service_charges"] = service_charges if service_charges.present?
+          normalized
         rescue ArgumentError, TypeError => e
           raise Error, "Unable to load remote pricing snapshot: #{e.message}"
         end
 
+        def load_registry(path)
+          return {} unless File.exist?(path)
+
+          YAML.safe_load_file(path, aliases: false) || {}
+        rescue Psych::Exception, ArgumentError, TypeError => e
+          raise Error, "Unable to load pricing registry #{path.inspect}: #{e.message}"
+        end
+
         def parse_registry(body)
           registry = JSON.parse(body.to_s)
           raise Error, "remote pricing snapshot must be a JSON object" unless registry.is_a?(Hash)
@@ -148,11 +171,37 @@ module LlmCostTracker
             path: path,
             source_url: url,
             source_version: response.source_version,
-            changes: RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {})),
+            changes: registry_changes(current, remote),
             written: written,
             not_modified: not_modified
           )
         end
+
+        def registry_changes(current, remote)
+          model_changes = RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {}))
+          charge_changes = service_charges_diff(
+            current.fetch("service_charges", {}),
+            remote.fetch("service_charges", {})
+          )
+          return model_changes if charge_changes.empty?
+
+          model_changes.merge("service_charges" => charge_changes)
+        end
+
+        def service_charges_diff(current, remote)
+          (current.keys | remote.keys).sort.each_with_object({}) do |provider, changes|
+            current_rates = (current[provider] || {}).transform_keys(&:to_s)
+            remote_rates = (remote[provider] || {}).transform_keys(&:to_s)
+            (current_rates.keys | remote_rates.keys).sort.each_with_object(changes) do |component, _|
+              from = current_rates[component]
+              to = remote_rates[component]
+              next if from == to
+
+              changes[provider] ||= {}
+              changes[provider][component] = { "from" => from, "to" => to }
+            end
+          end
+        end
       end
     end
   end

data/lib/llm_cost_tracker/pricing/sync_change_printer.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module Pricing
+    module SyncChangePrinter
+      class << self
+        def call(changes, output: $stdout)
+          service_changes = changes["service_charges"]
+          model_changes = changes.except("service_charges")
+
+          output.puts "  changed models: #{model_changes.size}"
+          model_changes.each do |model, fields|
+            output.puts "    - #{model}"
+            fields.each do |field, values|
+              output.puts "      #{field}: #{values['from'].inspect} -> #{values['to'].inspect}"
+            end
+          end
+
+          return if service_changes.nil? || service_changes.empty?
+
+          output.puts "  changed service charges: #{service_changes.values.sum(&:size)}"
+          service_changes.each do |provider, components|
+            components.each do |component, values|
+              output.puts "    - #{provider}.#{component}: " \
+                          "#{values['from'].inspect} -> #{values['to'].inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/pricing.rb

@@ -1,75 +1,267 @@
 # frozen_string_literal: true
 
-require "active_support/core_ext/hash/keys"
 require "active_support/core_ext/object/blank"
+require "time"
 
-require_relative "pricing/components"
+require_relative "version"
+require_relative "token_usage"
+require_relative "billing/components"
+require_relative "pricing/mode"
 require_relative "pricing/registry"
 require_relative "pricing/lookup"
 require_relative "pricing/effective_prices"
 require_relative "pricing/explainer"
+require_relative "pricing/service_charges"
 
 module LlmCostTracker
-  module Pricing
-    STANDARD_MODE_VALUES = %w[auto default standard standard_only].freeze
-    private_constant :STANDARD_MODE_VALUES
+  module Pricing # rubocop:disable Metrics/ModuleLength
+    extend ServiceCharges
+
+    STANDARD_MODE_VALUES = %i[auto default standard standard_only].freeze
+    RATE_DENOMINATOR_TOKENS = 1_000_000
+    private_constant :STANDARD_MODE_VALUES, :RATE_DENOMINATOR_TOKENS
 
     class << self
       def normalize_mode(value)
-        mode = value.to_s.strip.presence
+        return nil if value.nil?
+
+        mode = normalize_string_mode(value.to_s)
         return nil unless mode
 
-        mode = mode.tr("-", "_")
         STANDARD_MODE_VALUES.include?(mode) ? nil : mode
       end
 
-      def cost_for(provider:, model:, token_usage:, pricing_mode: nil)
-        prices = lookup(provider: provider, model: model)
-        return nil unless prices
+      def cost_for(provider:, model:, tokens:, pricing_mode: nil)
+        calculation = calculation_for(
+          provider: provider,
+          model: model,
+          tokens: tokens,
+          pricing_mode: pricing_mode
+        )
+        return nil unless calculation
 
-        costs = calculate_costs(token_usage, prices, pricing_mode: pricing_mode)
-        return nil unless costs
+        cost_from(calculation)
+      end
 
-        values = COMPONENTS.to_h do |component|
-          [component.cost_key, costs.fetch(component.price_key).round(8)]
-        end
+      def calculate(provider:, model:, tokens:, line_items:, pricing_mode: nil)
+        calculation = calculation_for(
+          provider: provider,
+          model: model,
+          tokens: tokens,
+          pricing_mode: pricing_mode
+        )
+        cost_data = calculation && cost_from(calculation)
+        snapshot = calculation && snapshot_from(calculation)
+        priced = apply_calculation_to_line_items(line_items, calculation,
+                                                 provider: provider, pricing_mode: pricing_mode)
+        [cost_data, snapshot, priced]
+      end
 
-        values.merge(total_cost: costs.values.sum.round(8))
+      def price_line_items(provider:, model:, line_items:, pricing_mode: nil)
+        token_usage = TokenUsage.build_from_tokens(token_attributes_from(line_items))
+        calculation = calculation_for(provider: provider, model: model, tokens: token_usage, pricing_mode: pricing_mode)
+        snapshot = calculation && snapshot_from(calculation)
+        priced = apply_calculation_to_line_items(line_items, calculation,
+                                                 provider: provider, pricing_mode: pricing_mode)
+        [priced, snapshot]
       end
 
-      def lookup(provider:, model:)
-        Lookup.call(provider: provider, model: model)&.prices
+      def snapshot_for(provider:, model:, tokens:, pricing_mode: nil)
+        calculation = calculation_for(
+          provider: provider,
+          model: model,
+          tokens: tokens,
+          pricing_mode: pricing_mode
+        )
+        return nil unless calculation
+
+        snapshot_from(calculation)
       end
 
-      def explain(provider:, model:, token_usage:, pricing_mode: nil)
+      def explain(provider:, model:, tokens:, pricing_mode: nil)
         Explainer.call(
           provider: provider,
           model: model,
-          token_usage: token_usage,
+          tokens: tokens,
           pricing_mode: pricing_mode
         )
       end
 
       def stored_cost_attributes(attributes)
-        attributes.to_h.symbolize_keys.slice(*COST_KEYS).compact
+        value = attributes.to_h[:total_cost]
+        value ? { total_cost: value } : {}
       end
 
       private
 
-      def calculate_costs(usage, prices, pricing_mode:)
-        effective = EffectivePrices.call(usage: usage, prices: prices, pricing_mode: pricing_mode)
-        return nil if effective.value?(nil)
+      def normalize_string_mode(value)
+        normalized = value.strip
+        return nil if normalized.empty?
+
+        normalized.downcase.tr("-", "_").to_sym
+      end
+
+      def cost_from(calculation)
+        costs = calculation[:costs]
+        values = Billing::Components::TOKEN_PRICED.each_with_object({}) do |component, result|
+          cost = costs[component.key]
+          result[component.cost_key] = cost.round(8) unless cost.nil?
+        end
+        values[:total_cost] = costs.values.compact.sum(BigDecimal("0")).round(8)
+        values
+      end
+
+      def snapshot_from(calculation)
+        match = calculation[:match]
+        effective = calculation[:effective]
+        token_usage = calculation[:token_usage]
+        rates = Billing::Components::TOKEN_PRICED.each_with_object({}) do |component, values|
+          quantity = token_usage.public_send(component.token_key)
+          price = effective[component.key]
+          next if quantity.zero? || price.nil?
+
+          values[component.key] = {
+            amount: price,
+            quantity: RATE_DENOMINATOR_TOKENS
+          }
+        end
+
+        {
+          schema_version: 1,
+          source: match.source,
+          source_key: match.key,
+          source_version: source_version_for(match.source),
+          matched_by: match.matched_by,
+          currency: "USD",
+          rates: rates
+        }
+      end
+
+      def calculation_for(provider:, model:, tokens:, pricing_mode:)
+        match = Lookup.call(provider: provider, model: model)
+        return nil unless match
+
+        token_usage = TokenUsage.build_from_tokens(tokens)
+        mode = normalize_mode(pricing_mode)
+        effective = EffectivePrices.call(usage: token_usage, prices: match.prices, pricing_mode: mode)
+        return nil unless any_billable_priced?(token_usage, effective)
+
+        { match: match, effective: effective, token_usage: token_usage, costs: costs_for(token_usage, effective) }
+      end
+
+      def any_billable_priced?(token_usage, effective)
+        billable = Billing::Components::TOKEN_PRICED.select { |c| token_usage.public_send(c.token_key).positive? }
+        billable.empty? || billable.any? { |c| effective[c.key] }
+      end
+
+      def costs_for(usage, effective)
+        Billing::Components::TOKEN_PRICED.to_h do |component|
+          tokens = usage.public_send(component.token_key)
+          [component.key, token_cost(tokens, effective[component.key])]
+        end
+      end
+
+      def apply_calculation_to_line_items(line_items, calculation, provider:, pricing_mode:)
+        line_items.map do |line_item|
+          next price_token_line_item(line_item, calculation) if line_item.unit == :token
+
+          price_service_charge_line_item(line_item,
+                                         provider: provider,
+                                         calculation: calculation,
+                                         pricing_mode: pricing_mode)
+        end
+      end
+
+      def token_attributes_from(line_items)
+        line_items.each_with_object({}) do |line_item, totals|
+          next unless line_item.unit == :token
+
+          component = component_for_line_item(line_item)
+          next unless component
+
+          totals[component.key] = (totals[component.key] || 0) + line_item.quantity.to_i
+        end
+      end
+
+      def price_token_line_item(line_item, calculation)
+        component = component_for_line_item(line_item)
+        return line_item unless component
+        return line_item.with(cost_status: Billing::CostStatus::UNKNOWN) unless calculation
+
+        effective_price = calculation[:effective][component.key]
+        return line_item.with(cost_status: Billing::CostStatus::UNKNOWN) if effective_price.nil?
+
+        cost = (line_item.quantity * BigDecimal(effective_price.to_s)) / RATE_DENOMINATOR_TOKENS
+        match = calculation[:match]
+        line_item.with(
+          rate_amount: BigDecimal(effective_price.to_s),
+          rate_quantity: BigDecimal(RATE_DENOMINATOR_TOKENS),
+          cost: cost,
+          cost_status: cost.zero? ? Billing::CostStatus::FREE : Billing::CostStatus::COMPLETE,
+          price_key: component.key,
+          price_source: match.source,
+          price_source_version: source_version_for(match.source)
+        )
+      end
+
+      def price_service_charge_line_item(line_item, provider:, calculation:, pricing_mode:)
+        return line_item if line_item.priced?
+        return line_item unless line_item.billable?
+
+        rate = model_rate_for(line_item, calculation) ||
+               charge_rate(provider: provider, component: line_item.kind, pricing_mode: pricing_mode)
+        return line_item unless rate
+
+        line_item.apply_rate(rate)
+      end
+
+      def model_rate_for(line_item, calculation)
+        return nil unless calculation
+
+        match = calculation[:match]
+        amount = match.prices[line_item.kind] || match.prices[line_item.kind.to_s]
+        return nil unless amount.is_a?(Numeric)
+
+        component = Billing::Components::BY_KEY[line_item.kind]
+        {
+          amount: BigDecimal(amount.to_s),
+          quantity: BigDecimal(Billing::RATE_BASIS_QUANTITIES.fetch(component.rate_basis).to_s),
+          currency: "USD",
+          source: match.source,
+          source_key: "#{match.key}.#{line_item.kind}",
+          source_version: source_version_for(match.source)
+        }
+      end
+
+      def component_for_line_item(line_item)
+        Billing::Components::REGISTRY.find do |component|
+          component.kind == line_item.kind &&
+            component.direction == line_item.direction &&
+            component.modality == line_item.modality &&
+            component.cache_state == line_item.cache_state &&
+            component.unit == line_item.unit
+        end
+      end
 
-        usage.price_quantities.to_h do |key, tokens|
-          [key, token_cost(tokens, effective.fetch(key))]
+      def source_version_for(source)
+        case source
+        when :bundled
+          LlmCostTracker::VERSION
+        when :prices_file
+          path = LlmCostTracker.configuration.prices_file
+          path ? File.mtime(path).utc.iso8601 : nil
+        when :pricing_overrides
+          "configuration"
         end
+      rescue Errno::ENOENT
+        nil
       end
 
       def token_cost(tokens, per_million_price)
-        return 0.0 if tokens.to_i.zero?
+        return BigDecimal("0") if tokens.zero?
         return nil if per_million_price.nil?
 
-        (tokens.to_f / 1_000_000) * per_million_price
+        (BigDecimal(tokens.to_s) * BigDecimal(per_million_price.to_s)) / RATE_DENOMINATOR_TOKENS
       end
     end
   end

data/lib/llm_cost_tracker/railtie.rb

@@ -9,16 +9,14 @@ module LlmCostTracker
     end
 
     generators do
-      require_relative "generators/llm_cost_tracker/add_ingestion_generator"
-      require_relative "generators/llm_cost_tracker/add_period_totals_generator"
-      require_relative "generators/llm_cost_tracker/add_latency_ms_generator"
-      require_relative "generators/llm_cost_tracker/add_provider_response_id_generator"
-      require_relative "generators/llm_cost_tracker/add_streaming_generator"
-      require_relative "generators/llm_cost_tracker/add_token_usage_generator"
       require_relative "generators/llm_cost_tracker/install_generator"
       require_relative "generators/llm_cost_tracker/prices_generator"
-      require_relative "generators/llm_cost_tracker/upgrade_cost_precision_generator"
-      require_relative "generators/llm_cost_tracker/upgrade_tags_to_jsonb_generator"
+      require_relative "generators/llm_cost_tracker/call_rollups_generator"
+      require_relative "generators/llm_cost_tracker/durable_ingestion_generator"
+      require_relative "generators/llm_cost_tracker/reconciliation_generator"
+      require_relative "generators/llm_cost_tracker/upgrade_call_rollups_provider_generator"
+      require_relative "generators/llm_cost_tracker/upgrade_image_tokens_generator"
+      require_relative "generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator"
     end
 
     rake_tasks do

data/lib/llm_cost_tracker/reconcile_tasks.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "reconciliation"
+
+module LlmCostTracker
+  module ReconcileTasks
+    SOURCE_PARSERS = {
+      "openai" => Reconciliation::Sources::OpenaiUsage,
+      "anthropic" => Reconciliation::Sources::AnthropicUsage
+    }.freeze
+    GENERIC_SOURCES = %w[csv].freeze
+
+    module_function
+
+    def run_import(env: ENV, output: $stdout, error_output: $stderr)
+      result = import_from_env(env: env)
+      output.puts "llm_cost_tracker: imported #{result.total_imported} rows " \
+                  "(inserted=#{result.inserted}, updated=#{result.updated}, skipped=#{result.skipped})"
+      result.errors.each { |error| error_output.puts "  error: #{error}" }
+      raise "llm_cost_tracker: reconcile import had errors" unless result.success?
+
+      result
+    end
+
+    def run_diff(env: ENV, output: $stdout)
+      diff = diff_from_env(env: env)
+      print_diff(diff, output: output)
+      diff
+    end
+
+    def import_from_env(env: ENV)
+      source = required_env(env, "SOURCE")
+      input_path = required_env(env, "INPUT")
+      raise ArgumentError, "INPUT file not found: #{input_path}" unless File.exist?(input_path)
+
+      payload = JSON.parse(File.read(input_path))
+      rows = parse_rows(source: source, payload: payload)
+      Reconciliation.import(source: source.to_sym, rows: rows, provider: env["PROVIDER"])
+    end
+
+    def diff_from_env(env: ENV)
+      source = required_env(env, "SOURCE")
+      period_start = Date.parse(required_env(env, "PERIOD_START"))
+      period_end = Date.parse(required_env(env, "PERIOD_END"))
+      Reconciliation.diff(source: source.to_sym, period_start: period_start, period_end: period_end,
+                          provider: env["PROVIDER"],
+                          drilldown_limit: parse_drilldown_limit(env["DRILLDOWN_LIMIT"]))
+    end
+
+    def parse_drilldown_limit(value)
+      return Reconciliation::Diff::DEFAULT_DRILLDOWN_LIMIT if value.nil? || value.to_s.empty?
+      return nil if value.to_s.downcase == "all"
+
+      Integer(value)
+    end
+
+    def print_diff(diff, output: $stdout)
+      output.puts "llm_cost_tracker: reconciliation diff for #{diff.source} " \
+                  "#{diff.period_start}..#{diff.period_end}"
+      output.puts "  provider_total: #{diff.provider_total.to_s('F')} #{diff.currency}"
+      output.puts "  local_total:    #{diff.local_total.to_s('F')} #{diff.currency} " \
+                  "(from #{diff.local_total_source})"
+      output.puts "  delta:          #{diff.delta_amount.to_s('F')} (#{diff.delta_percent || 'n/a'}%)"
+      print_unmatched_provider_rows(diff, output)
+      print_unmatched_local_calls(diff, output)
+      print_non_cost_rows(diff, output)
+    end
+
+    def parse_rows(source:, payload:)
+      parser = SOURCE_PARSERS[source.to_s]
+      return parser.parse(payload) if parser
+      return Array(payload["rows"]) if GENERIC_SOURCES.include?(source.to_s)
+
+      known = (SOURCE_PARSERS.keys + GENERIC_SOURCES).join(", ")
+      raise ArgumentError, "unknown SOURCE #{source.inspect}; known sources: #{known}"
+    end
+
+    def required_env(env, key)
+      value = env[key].to_s.strip
+      raise ArgumentError, "missing #{key}" if value.empty?
+
+      value
+    end
+
+    def print_unmatched_provider_rows(diff, output)
+      return if diff.unmatched_provider_rows.empty?
+
+      output.puts "  unmatched provider rows#{truncation_suffix(diff.unmatched_provider_rows.size,
+                                                                diff.unmatched_provider_rows_total)}:"
+      diff.unmatched_provider_rows.each do |row|
+        output.puts "    #{row[:external_id]} (#{row[:match_basis]}): " \
+                    "#{format_amount(row[:billed_amount])} #{format_attribution(row[:attribution])}"
+      end
+    end
+
+    def print_unmatched_local_calls(diff, output)
+      return if diff.unmatched_local_calls.empty?
+
+      output.puts "  unmatched local calls#{truncation_suffix(diff.unmatched_local_calls.size,
+                                                              diff.unmatched_local_calls_total)}:"
+      diff.unmatched_local_calls.each do |row|
+        output.puts "    #{row[:count]} calls / #{row[:total_cost].to_s('F')} " \
+                    "#{format_attribution(row[:attribution])}"
+      end
+    end
+
+    def print_non_cost_rows(diff, output)
+      return if diff.non_cost_rows.empty?
+
+      output.puts "  non-cost evidence#{truncation_suffix(diff.non_cost_rows.size,
+                                                          diff.non_cost_rows_total)}:"
+      diff.non_cost_rows.each do |row|
+        output.puts "    [#{row[:row_type]}/#{row[:meter]}] #{format_amount(row[:billed_amount])} " \
+                    "#{format_attribution(row[:attribution])}"
+      end
+    end
+
+    def truncation_suffix(shown, total)
+      return "" if shown >= total
+
+      " (showing #{shown} of #{total} — pass DRILLDOWN_LIMIT=all to see every row)"
+    end
+
+    def format_amount(value)
+      value.nil? ? "n/a" : value.to_s("F")
+    end
+
+    def format_attribution(attribution)
+      LlmCostTracker::Masking.format_attribution(attribution, separator: ",")
+    end
+  end
+end