llm_cost_tracker 0.2.0 → 0.3.1

data/lib/llm_cost_tracker/price_sync/model_catalog.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class ModelCatalog
+      OPENROUTER_PROVIDER_PREFIXES = {
+        openai: %w[openai],
+        anthropic: %w[anthropic],
+        gemini: %w[google]
+      }.freeze
+      LITELLM_PROVIDER_PREFIXES = {
+        openai: [nil, "openai"],
+        anthropic: [nil, "anthropic"],
+        gemini: [nil, "gemini"]
+      }.freeze
+      ALIASES = {
+        "gpt-4o-2024-05-13" => "gpt-4o"
+      }.freeze
+
+      class << self
+        def resolve_from_litellm(our_model, payload)
+          litellm_candidates(our_model).find { |candidate| payload.key?(candidate) }
+        end
+
+        def resolve_from_openrouter(our_model, index)
+          openrouter_candidates(our_model).find { |candidate| index.key?(candidate) }
+        end
+
+        def guess_provider(our_model)
+          case our_model.to_s
+          when /\A(?:gpt-|o1|o3|o4|chatgpt|text-embedding)/
+            :openai
+          when /\Aclaude-/
+            :anthropic
+          when /\Agemini-/
+            :gemini
+          end
+        end
+
+        private
+
+        def litellm_candidates(our_model)
+          provider = guess_provider(our_model)
+          prefixes = LITELLM_PROVIDER_PREFIXES.fetch(provider, [nil])
+
+          model_variants(our_model).flat_map do |variant|
+            prefixes.map { |prefix| prefix ? "#{prefix}/#{variant}" : variant }
+          end.uniq
+        end
+
+        def openrouter_candidates(our_model)
+          provider = guess_provider(our_model)
+          prefixes = OPENROUTER_PROVIDER_PREFIXES.fetch(provider, [])
+
+          model_variants(our_model).flat_map do |variant|
+            prefixes.map { |prefix| "#{prefix}/#{variant}" }
+          end.uniq
+        end
+
+        def model_variants(our_model)
+          model = our_model.to_s
+          canonical = ALIASES.fetch(model, model)
+
+          [model, canonical].flat_map do |variant|
+            [variant, anthropic_version_variant(variant)]
+          end.compact.uniq
+        end
+
+        def anthropic_version_variant(model)
+          return nil unless guess_provider(model) == :anthropic
+
+          model.gsub(/(?<=\d)-(?=\d)/, ".")
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/raw_price.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    RawPrice = Data.define(
+      :model,
+      :provider,
+      :input,
+      :output,
+      :cached_input,
+      :cache_read_input,
+      :cache_creation_input,
+      :source,
+      :source_version,
+      :fetched_at
+    )
+
+    class RawPrice
+      PRICE_FIELDS = %w[input output cached_input cache_read_input cache_creation_input].freeze
+
+      def to_registry_entry(today:)
+        {
+          "input" => input,
+          "output" => output,
+          "cached_input" => cached_input,
+          "cache_read_input" => cache_read_input,
+          "cache_creation_input" => cache_creation_input,
+          "_source" => source.to_s,
+          "_source_version" => source_version,
+          "_fetched_at" => fetched_at || today.iso8601
+        }.compact
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/refresh_plan_builder.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class RefreshPlanBuilder
+      def initialize(sources:, loader: RegistryLoader.new)
+        @sources = sources
+        @loader = loader
+      end
+
+      def call(path:, seed_path:, fetcher:, today:)
+        path = path.to_s
+        registry = loader.call(path: path, seed_path: seed_path)
+        current_models = registry.fetch("models", {})
+        source_results, failed_sources = fetch_all(current_models, fetcher)
+        merged, discrepancies = Merger.new.merge(source_results)
+        validated = Validator.new.validate_batch(merged, existing_registry: current_models)
+        updated_models = apply_changes(current_models, validated.accepted, today)
+
+        PriceSync::RefreshPlan.new(
+          path: path,
+          registry: registry,
+          updated_registry: registry.merge(
+            "metadata" => updated_metadata(
+              registry["metadata"],
+              today,
+              refresh_succeeded: source_results.any? { |_source, result| result.prices.any? },
+              source_results: source_results
+            ),
+            "models" => updated_models
+          ),
+          accepted: validated.accepted,
+          changes: price_changes(current_models, updated_models),
+          orphaned_models: compute_orphaned(current_models, merged.keys),
+          failed_sources: failed_sources,
+          discrepancies: discrepancies,
+          rejected: validated.rejected,
+          flagged: validated.flagged,
+          sources_used: source_usage(source_results),
+          source_results: source_results
+        )
+      end
+
+      private
+
+      attr_reader :sources, :loader
+
+      def fetch_all(current_models, fetcher)
+        results = {}
+        failures = {}
+
+        sources.each do |source|
+          results[source.name.to_sym] = source.fetch(current_models: current_models, fetcher: fetcher)
+        rescue Error => e
+          failures[source.name.to_sym] = e.message
+        end
+
+        [results, failures]
+      end
+
+      def apply_changes(current_models, accepted, today)
+        merged = seed_models(current_models)
+
+        accepted.each do |model, price|
+          next if manual_model?(merged[model])
+
+          merged[model] = registry_entry_for(merged[model], price, today)
+        end
+
+        merged.sort.to_h
+      end
+
+      def compute_orphaned(current_models, merged_models)
+        seed_models(current_models).keys.reject do |model|
+          manual_model?(current_models[model]) || merged_models.include?(model)
+        end.sort
+      end
+
+      def seed_models(current_models)
+        normalize_models(current_models).transform_values do |entry|
+          next entry if entry.key?("_source")
+
+          entry.merge("_source" => "seed")
+        end
+      end
+
+      def normalize_models(models)
+        normalize_hash(models).each_with_object({}) do |(model, entry), normalized|
+          normalized[model.to_s] = normalize_hash(entry)
+        end
+      end
+
+      def normalize_hash(hash)
+        return {} if hash.nil?
+        raise ArgumentError, "price sync entries must be hashes" unless hash.is_a?(Hash)
+
+        hash.each_with_object({}) do |(key, value), normalized|
+          normalized[key.to_s] = value
+        end
+      end
+
+      def manual_model?(entry)
+        normalize_hash(entry)["_source"] == "manual"
+      end
+
+      def registry_entry_for(existing_entry, price, today)
+        normalize_hash(existing_entry)
+          .except(*PriceRegistry::PRICE_KEYS)
+          .merge(price.to_registry_entry(today: today))
+      end
+
+      def updated_metadata(existing, today, refresh_succeeded:, source_results:)
+        metadata = normalize_hash(existing)
+        metadata["currency"] ||= "USD"
+        metadata["unit"] ||= "1M tokens"
+        return metadata unless refresh_succeeded
+
+        metadata["updated_at"] = today.iso8601
+        metadata["source_urls"] = source_urls(source_results)
+        metadata
+      end
+
+      def source_usage(source_results)
+        source_results.transform_values do |result|
+          PriceSync::SourceUsage.new(prices_count: result.prices.size, source_version: result.source_version)
+        end
+      end
+
+      def price_changes(current_models, updated_models)
+        current_models = normalize_models(current_models)
+        updated_models = normalize_models(updated_models)
+
+        (current_models.keys | updated_models.keys).sort.each_with_object({}) do |model, changes|
+          fields = price_field_changes(current_models[model], updated_models[model])
+          changes[model] = fields if fields.any?
+        end
+      end
+
+      def price_field_changes(current_entry, updated_entry)
+        current_price = comparable_price(current_entry)
+        updated_price = comparable_price(updated_entry)
+
+        (current_price.keys | updated_price.keys).sort.each_with_object({}) do |field, changes|
+          from = current_price[field]
+          to = updated_price[field]
+          next if from == to
+
+          changes[field] = { "from" => from, "to" => to }
+        end
+      end
+
+      def comparable_price(entry)
+        normalize_hash(entry).slice(*PriceRegistry::PRICE_KEYS)
+      end
+
+      def source_urls(source_results)
+        names = source_results.keys.map(&:to_sym)
+        sources.select { |source| names.include?(source.name.to_sym) }.map(&:url)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/registry_loader.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+
+module LlmCostTracker
+  module PriceSync
+    class RegistryLoader
+      YAML_EXTENSIONS = %w[.yml .yaml].freeze
+
+      def call(path:, seed_path:)
+        source_path = File.exist?(path.to_s) ? path.to_s : seed_path.to_s
+        normalize_registry(load_registry_file(source_path))
+      rescue Errno::ENOENT, JSON::ParserError, Psych::Exception, ArgumentError, TypeError => e
+        raise Error, "Unable to load pricing registry #{source_path.inspect}: #{e.message}"
+      end
+
+      private
+
+      def load_registry_file(path)
+        contents = File.read(path)
+        registry = yaml_file?(path) ? (YAML.safe_load(contents, aliases: false) || {}) : JSON.parse(contents)
+        raise ArgumentError, "pricing registry must be a hash" unless registry.is_a?(Hash)
+
+        registry
+      end
+
+      def normalize_registry(registry)
+        {
+          "metadata" => normalize_hash(registry.fetch("metadata", {}), label: "pricing metadata"),
+          "models" => normalize_models(registry.fetch("models", {}))
+        }
+      end
+
+      def normalize_models(models)
+        normalize_hash(models, label: "pricing models").each_with_object({}) do |(model, entry), normalized|
+          normalized[model.to_s] = normalize_hash(entry, label: "pricing model entry")
+        end
+      end
+
+      def normalize_hash(hash, label:)
+        return {} if hash.nil?
+        raise ArgumentError, "#{label} must be a hash" unless hash.is_a?(Hash)
+
+        hash.each_with_object({}) do |(key, value), normalized|
+          normalized[key.to_s] = value
+        end
+      end
+
+      def yaml_file?(path)
+        YAML_EXTENSIONS.include?(File.extname(path).downcase)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/registry_writer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "yaml"
+
+module LlmCostTracker
+  module PriceSync
+    class RegistryWriter
+      YAML_EXTENSIONS = %w[.yml .yaml].freeze
+
+      def call(path:, registry:)
+        FileUtils.mkdir_p(File.dirname(path))
+        payload = yaml_file?(path) ? YAML.dump(registry) : "#{JSON.pretty_generate(registry)}\n"
+        File.write(path, payload)
+      end
+
+      private
+
+      def yaml_file?(path)
+        YAML_EXTENSIONS.include?(File.extname(path).downcase)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/source.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Source
+      def fetch(current_models:, fetcher:)
+        raise NotImplementedError
+      end
+
+      def name
+        self.class.name.split("::").last.downcase.to_sym
+      end
+
+      def priority
+        100
+      end
+
+      def url
+        raise NotImplementedError
+      end
+
+      private
+
+      def response_version(response)
+        response.source_version
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/source_result.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    SourceResult = Data.define(:prices, :missing_models, :source_version)
+  end
+end

data/lib/llm_cost_tracker/price_sync/sources/litellm.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module PriceSync
+    module Sources
+      class Litellm < Source
+        PER_TOKEN_TO_PER_MILLION = 1_000_000
+        SUPPORTED_MODES = %w[chat completion embedding responses].freeze
+        SUPPORTED_PROVIDERS = %w[openai anthropic gemini text-completion-openai].freeze
+        URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+
+        def priority
+          10
+        end
+
+        def url
+          URL
+        end
+
+        def fetch(current_models:, fetcher:)
+          response = fetcher.get(url)
+          payload = JSON.parse(response.body.to_s)
+
+          prices = []
+          missing_models = []
+
+          current_models.each_key do |our_model|
+            entry_id = ModelCatalog.resolve_from_litellm(our_model, payload)
+            entry = entry_id && payload[entry_id]
+
+            if entry && supported_entry?(entry)
+              prices << build_raw_price(our_model, entry, response)
+            else
+              missing_models << our_model
+            end
+          end
+
+          SourceResult.new(
+            prices: prices,
+            missing_models: missing_models.sort,
+            source_version: response_version(response)
+          )
+        rescue JSON::ParserError => e
+          raise Error, "Unable to parse #{url}: #{e.message}"
+        end
+
+        private
+
+        def supported_entry?(entry)
+          SUPPORTED_PROVIDERS.include?(entry["litellm_provider"]) &&
+            SUPPORTED_MODES.include?(entry["mode"]) &&
+            entry.key?("input_cost_per_token") &&
+            entry.key?("output_cost_per_token")
+        end
+
+        def build_raw_price(model, entry, response)
+          provider = normalize_provider(entry["litellm_provider"])
+          cache_read = price_per_million(entry["cache_read_input_token_cost"])
+          cache_write = price_per_million(entry["cache_creation_input_token_cost"])
+
+          RawPrice.new(
+            model: model,
+            provider: provider,
+            input: price_per_million(entry["input_cost_per_token"]),
+            output: price_per_million(entry["output_cost_per_token"]),
+            cached_input: provider == "anthropic" ? nil : cache_read,
+            cache_read_input: provider == "anthropic" ? cache_read : nil,
+            cache_creation_input: provider == "anthropic" ? cache_write : nil,
+            source: name,
+            source_version: response_version(response),
+            fetched_at: response.fetched_at
+          )
+        end
+
+        def normalize_provider(provider)
+          return "openai" if provider == "text-completion-openai"
+
+          provider
+        end
+
+        def price_per_million(value)
+          return nil if value.nil?
+
+          value.to_f * PER_TOKEN_TO_PER_MILLION
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/sources/open_router.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module PriceSync
+    module Sources
+      class OpenRouter < Source
+        PER_TOKEN_TO_PER_MILLION = 1_000_000
+        SUPPORTED_PREFIXES = %w[openai anthropic google].freeze
+        URL = "https://openrouter.ai/api/v1/models"
+
+        def priority
+          20
+        end
+
+        def url
+          URL
+        end
+
+        def fetch(current_models:, fetcher:)
+          response = fetcher.get(url)
+          payload = JSON.parse(response.body.to_s)
+          index = payload.fetch("data", []).to_h { |entry| [entry["id"], entry] }
+
+          prices = []
+          missing_models = []
+
+          current_models.each_key do |our_model|
+            entry_id = ModelCatalog.resolve_from_openrouter(our_model, index)
+            entry = entry_id && index[entry_id]
+
+            if entry && supported_entry?(entry)
+              prices << build_raw_price(our_model, entry, response)
+            else
+              missing_models << our_model
+            end
+          end
+
+          SourceResult.new(
+            prices: prices,
+            missing_models: missing_models.sort,
+            source_version: response_version(response)
+          )
+        rescue JSON::ParserError => e
+          raise Error, "Unable to parse #{url}: #{e.message}"
+        end
+
+        private
+
+        def supported_entry?(entry)
+          pricing = entry["pricing"] || {}
+          provider = entry["id"].to_s.split("/").first
+
+          SUPPORTED_PREFIXES.include?(provider) &&
+            pricing["prompt"] &&
+            pricing["completion"]
+        end
+
+        def build_raw_price(model, entry, response)
+          pricing = entry.fetch("pricing", {})
+          provider = normalize_provider(entry.fetch("id").split("/").first)
+          cache_read = price_per_million(pricing["input_cache_read"])
+          cache_write = price_per_million(pricing["input_cache_write"])
+
+          RawPrice.new(
+            model: model,
+            provider: provider,
+            input: price_per_million(pricing["prompt"]),
+            output: price_per_million(pricing["completion"]),
+            cached_input: provider == "anthropic" ? nil : cache_read,
+            cache_read_input: provider == "anthropic" ? cache_read : nil,
+            cache_creation_input: provider == "anthropic" ? cache_write : nil,
+            source: name,
+            source_version: response_version(response),
+            fetched_at: response.fetched_at
+          )
+        end
+
+        def normalize_provider(provider)
+          return "gemini" if provider == "google"
+
+          provider
+        end
+
+        def price_per_million(value)
+          return nil if value.nil?
+
+          value.to_f * PER_TOKEN_TO_PER_MILLION
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/validator.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Validator
+      Result = Data.define(:accepted, :rejected, :flagged)
+      Issue = Data.define(:model, :reason, :old_price, :new_price)
+
+      MAX_INPUT_PER_MILLION = 100.0
+      MAX_OUTPUT_PER_MILLION = 500.0
+      MAX_RELATIVE_CHANGE = 3.0
+
+      def validate_batch(merged_prices, existing_registry:)
+        merged_prices.each_with_object(Result.new(accepted: {}, rejected: [], flagged: [])) do |(model, price), result|
+          old_price = normalize_entry(existing_registry[model])
+          status, reason = validate(new_price: price, old_price: old_price)
+
+          case status
+          when :rejected
+            result.rejected << Issue.new(model: model, reason: reason, old_price: old_price, new_price: price)
+          when :flagged
+            result.flagged << Issue.new(model: model, reason: reason, old_price: old_price, new_price: price)
+            result.accepted[model] = price
+          else
+            result.accepted[model] = price
+          end
+        end
+      end
+
+      private
+
+      def validate(new_price:, old_price:)
+        overrides = Array(normalize_entry(old_price)["_validator_override"])
+
+        return [:rejected, "input > $#{MAX_INPUT_PER_MILLION}/1M"] if new_price.input > MAX_INPUT_PER_MILLION
+        return [:rejected, "output > $#{MAX_OUTPUT_PER_MILLION}/1M"] if new_price.output > MAX_OUTPUT_PER_MILLION
+        return [:ok, nil] if overrides.include?("skip_relative_change")
+
+        if old_price.any? && changed_too_much?(old_price, new_price)
+          return [:flagged, "price changed >#{MAX_RELATIVE_CHANGE}x"]
+        end
+
+        [:ok, nil]
+      end
+
+      def changed_too_much?(old_price, new_price)
+        %i[input output].any? do |field|
+          old_value = old_price[field.to_s].to_f
+          next false if old_value.zero?
+
+          new_value = new_price.public_send(field).to_f
+          next false if new_value.zero?
+
+          ratio = [new_value / old_value, old_value / new_value].max
+          ratio > MAX_RELATIVE_CHANGE
+        end
+      end
+
+      def normalize_entry(entry)
+        (entry || {}).each_with_object({}) do |(key, value), normalized|
+          normalized[key.to_s] = value
+        end
+      end
+    end
+  end
+end