llm_cost_tracker 0.2.0.alpha2 → 0.3.0

data/lib/llm_cost_tracker/price_sync/fetcher.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "digest"
+require "net/http"
+require "time"
+require "uri"
+
+module LlmCostTracker
+  module PriceSync
+    class Fetcher
+      Response = Data.define(:body, :etag, :last_modified, :not_modified, :fetched_at) do
+        def source_version
+          etag || last_modified || Digest::SHA256.hexdigest(body.to_s)
+        end
+      end
+
+      USER_AGENT = "llm_cost_tracker price sync"
+      MAX_REDIRECTS = 5
+      OPEN_TIMEOUT = 5
+      READ_TIMEOUT = 10
+      WRITE_TIMEOUT = 10
+
+      def get(url, etag: nil, redirects: 0)
+        raise Error, "Too many redirects while fetching #{url}" if redirects > MAX_REDIRECTS
+
+        uri = URI.parse(url)
+        request = Net::HTTP::Get.new(uri)
+        request["User-Agent"] = USER_AGENT
+        request["If-None-Match"] = etag if etag
+
+        response = Net::HTTP.start(
+          uri.host,
+          uri.port,
+          use_ssl: uri.scheme == "https",
+          open_timeout: OPEN_TIMEOUT,
+          read_timeout: READ_TIMEOUT,
+          write_timeout: WRITE_TIMEOUT
+        ) do |http|
+          http.request(request)
+        end
+
+        case response
+        when Net::HTTPSuccess
+          build_response(response, not_modified: false)
+        when Net::HTTPNotModified
+          build_response(response, body: nil, not_modified: true)
+        when Net::HTTPRedirection
+          location = response["location"]
+          raise Error, "Redirect without location while fetching #{url}" if location.nil? || location.empty?
+
+          get(URI.join(url, location).to_s, etag: etag, redirects: redirects + 1)
+        else
+          raise Error, "Unable to fetch #{url}: HTTP #{response.code}"
+        end
+      rescue SocketError, SystemCallError, Timeout::Error => e
+        raise Error, "Unable to fetch #{url}: #{e.class}: #{e.message}"
+      end
+
+      private
+
+      def build_response(response, not_modified:, body: response.body)
+        Response.new(
+          body: body,
+          etag: response["etag"],
+          last_modified: response["last-modified"],
+          not_modified: not_modified,
+          fetched_at: Time.now.utc.iso8601
+        )
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/merger.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Merger
+      Discrepancy = Data.define(:model, :field, :values)
+
+      PRIORITY_ORDER = %i[litellm openrouter].freeze
+      SUPPLEMENTAL_FIELDS = %i[cached_input cache_read_input cache_creation_input].freeze
+
+      def merge(results_by_source)
+        prices = collect_prices(results_by_source)
+        discrepancies = []
+
+        merged = prices.group_by(&:model).sort.to_h.transform_values do |candidates|
+          sorted = sort_candidates(candidates)
+          discrepancies.concat(detect_discrepancies(sorted))
+          fill_missing_fields(sorted.first, sorted.drop(1))
+        end
+
+        [merged, discrepancies]
+      end
+
+      private
+
+      def collect_prices(results_by_source)
+        results_by_source.flat_map do |source_name, result|
+          result.prices.map do |price|
+            price.with(source: source_name)
+          end
+        end
+      end
+
+      def sort_candidates(candidates)
+        candidates.sort_by do |price|
+          PRIORITY_ORDER.index(price.source.to_sym) || PRIORITY_ORDER.length
+        end
+      end
+
+      def fill_missing_fields(primary, fallbacks)
+        SUPPLEMENTAL_FIELDS.reduce(primary) do |current, field|
+          next current if current.public_send(field)
+
+          fallback = fallbacks.find { |candidate| candidate.public_send(field) }
+          fallback ? current.with(field => fallback.public_send(field)) : current
+        end
+      end
+
+      def detect_discrepancies(candidates)
+        return [] if candidates.length < 2
+
+        RawPrice::PRICE_FIELDS.filter_map do |field|
+          values = candidates.each_with_object({}) do |price, collected|
+            value = price.public_send(field)
+            collected[price.source] = value unless value.nil?
+          end
+          next if values.size < 2
+          next unless discrepant?(values.values)
+
+          Discrepancy.new(model: candidates.first.model, field: field, values: values)
+        end
+      end
+
+      def discrepant?(values)
+        min, max = values.minmax
+        return max != min if min.to_f.zero?
+
+        ((max - min).abs / min.to_f) >= 0.05
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/model_catalog.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class ModelCatalog
+      OPENROUTER_PROVIDER_PREFIXES = {
+        openai: %w[openai],
+        anthropic: %w[anthropic],
+        gemini: %w[google]
+      }.freeze
+      LITELLM_PROVIDER_PREFIXES = {
+        openai: [nil, "openai"],
+        anthropic: [nil, "anthropic"],
+        gemini: [nil, "gemini"]
+      }.freeze
+      ALIASES = {
+        "gpt-4o-2024-05-13" => "gpt-4o"
+      }.freeze
+
+      class << self
+        def resolve_from_litellm(our_model, payload)
+          litellm_candidates(our_model).find { |candidate| payload.key?(candidate) }
+        end
+
+        def resolve_from_openrouter(our_model, index)
+          openrouter_candidates(our_model).find { |candidate| index.key?(candidate) }
+        end
+
+        def guess_provider(our_model)
+          case our_model.to_s
+          when /\A(?:gpt-|o1|o3|o4|chatgpt|text-embedding)/
+            :openai
+          when /\Aclaude-/
+            :anthropic
+          when /\Agemini-/
+            :gemini
+          end
+        end
+
+        private
+
+        def litellm_candidates(our_model)
+          provider = guess_provider(our_model)
+          prefixes = LITELLM_PROVIDER_PREFIXES.fetch(provider, [nil])
+
+          model_variants(our_model).flat_map do |variant|
+            prefixes.map { |prefix| prefix ? "#{prefix}/#{variant}" : variant }
+          end.uniq
+        end
+
+        def openrouter_candidates(our_model)
+          provider = guess_provider(our_model)
+          prefixes = OPENROUTER_PROVIDER_PREFIXES.fetch(provider, [])
+
+          model_variants(our_model).flat_map do |variant|
+            prefixes.map { |prefix| "#{prefix}/#{variant}" }
+          end.uniq
+        end
+
+        def model_variants(our_model)
+          model = our_model.to_s
+          canonical = ALIASES.fetch(model, model)
+
+          [model, canonical].flat_map do |variant|
+            [variant, anthropic_version_variant(variant)]
+          end.compact.uniq
+        end
+
+        def anthropic_version_variant(model)
+          return nil unless guess_provider(model) == :anthropic
+
+          model.gsub(/(?<=\d)-(?=\d)/, ".")
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/raw_price.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    RawPrice = Data.define(
+      :model,
+      :provider,
+      :input,
+      :output,
+      :cached_input,
+      :cache_read_input,
+      :cache_creation_input,
+      :source,
+      :source_version,
+      :fetched_at
+    )
+
+    class RawPrice
+      PRICE_FIELDS = %w[input output cached_input cache_read_input cache_creation_input].freeze
+
+      def to_registry_entry(today:)
+        {
+          "input" => input,
+          "output" => output,
+          "cached_input" => cached_input,
+          "cache_read_input" => cache_read_input,
+          "cache_creation_input" => cache_creation_input,
+          "_source" => source.to_s,
+          "_source_version" => source_version,
+          "_fetched_at" => fetched_at || today.iso8601
+        }.compact
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/source.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Source
+      def fetch(current_models:, fetcher:)
+        raise NotImplementedError
+      end
+
+      def name
+        self.class.name.split("::").last.downcase.to_sym
+      end
+
+      def priority
+        100
+      end
+
+      def url
+        raise NotImplementedError
+      end
+
+      private
+
+      def response_version(response)
+        response.source_version
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/source_result.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    SourceResult = Data.define(:prices, :missing_models, :source_version)
+  end
+end

data/lib/llm_cost_tracker/price_sync/sources/litellm.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module PriceSync
+    module Sources
+      class Litellm < Source
+        PER_TOKEN_TO_PER_MILLION = 1_000_000
+        SUPPORTED_MODES = %w[chat completion embedding responses].freeze
+        SUPPORTED_PROVIDERS = %w[openai anthropic gemini text-completion-openai].freeze
+        URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+
+        def priority
+          10
+        end
+
+        def url
+          URL
+        end
+
+        def fetch(current_models:, fetcher:)
+          response = fetcher.get(url)
+          payload = JSON.parse(response.body.to_s)
+
+          prices = []
+          missing_models = []
+
+          current_models.each_key do |our_model|
+            entry_id = ModelCatalog.resolve_from_litellm(our_model, payload)
+            entry = entry_id && payload[entry_id]
+
+            if entry && supported_entry?(entry)
+              prices << build_raw_price(our_model, entry, response)
+            else
+              missing_models << our_model
+            end
+          end
+
+          SourceResult.new(
+            prices: prices,
+            missing_models: missing_models.sort,
+            source_version: response_version(response)
+          )
+        rescue JSON::ParserError => e
+          raise Error, "Unable to parse #{url}: #{e.message}"
+        end
+
+        private
+
+        def supported_entry?(entry)
+          SUPPORTED_PROVIDERS.include?(entry["litellm_provider"]) &&
+            SUPPORTED_MODES.include?(entry["mode"]) &&
+            entry.key?("input_cost_per_token") &&
+            entry.key?("output_cost_per_token")
+        end
+
+        def build_raw_price(model, entry, response)
+          provider = normalize_provider(entry["litellm_provider"])
+          cache_read = price_per_million(entry["cache_read_input_token_cost"])
+          cache_write = price_per_million(entry["cache_creation_input_token_cost"])
+
+          RawPrice.new(
+            model: model,
+            provider: provider,
+            input: price_per_million(entry["input_cost_per_token"]),
+            output: price_per_million(entry["output_cost_per_token"]),
+            cached_input: provider == "anthropic" ? nil : cache_read,
+            cache_read_input: provider == "anthropic" ? cache_read : nil,
+            cache_creation_input: provider == "anthropic" ? cache_write : nil,
+            source: name,
+            source_version: response_version(response),
+            fetched_at: response.fetched_at
+          )
+        end
+
+        def normalize_provider(provider)
+          return "openai" if provider == "text-completion-openai"
+
+          provider
+        end
+
+        def price_per_million(value)
+          return nil if value.nil?
+
+          value.to_f * PER_TOKEN_TO_PER_MILLION
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/sources/open_router.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module PriceSync
+    module Sources
+      class OpenRouter < Source
+        PER_TOKEN_TO_PER_MILLION = 1_000_000
+        SUPPORTED_PREFIXES = %w[openai anthropic google].freeze
+        URL = "https://openrouter.ai/api/v1/models"
+
+        def priority
+          20
+        end
+
+        def url
+          URL
+        end
+
+        def fetch(current_models:, fetcher:)
+          response = fetcher.get(url)
+          payload = JSON.parse(response.body.to_s)
+          index = payload.fetch("data", []).to_h { |entry| [entry["id"], entry] }
+
+          prices = []
+          missing_models = []
+
+          current_models.each_key do |our_model|
+            entry_id = ModelCatalog.resolve_from_openrouter(our_model, index)
+            entry = entry_id && index[entry_id]
+
+            if entry && supported_entry?(entry)
+              prices << build_raw_price(our_model, entry, response)
+            else
+              missing_models << our_model
+            end
+          end
+
+          SourceResult.new(
+            prices: prices,
+            missing_models: missing_models.sort,
+            source_version: response_version(response)
+          )
+        rescue JSON::ParserError => e
+          raise Error, "Unable to parse #{url}: #{e.message}"
+        end
+
+        private
+
+        def supported_entry?(entry)
+          pricing = entry["pricing"] || {}
+          provider = entry["id"].to_s.split("/").first
+
+          SUPPORTED_PREFIXES.include?(provider) &&
+            pricing["prompt"] &&
+            pricing["completion"]
+        end
+
+        def build_raw_price(model, entry, response)
+          pricing = entry.fetch("pricing", {})
+          provider = normalize_provider(entry.fetch("id").split("/").first)
+          cache_read = price_per_million(pricing["input_cache_read"])
+          cache_write = price_per_million(pricing["input_cache_write"])
+
+          RawPrice.new(
+            model: model,
+            provider: provider,
+            input: price_per_million(pricing["prompt"]),
+            output: price_per_million(pricing["completion"]),
+            cached_input: provider == "anthropic" ? nil : cache_read,
+            cache_read_input: provider == "anthropic" ? cache_read : nil,
+            cache_creation_input: provider == "anthropic" ? cache_write : nil,
+            source: name,
+            source_version: response_version(response),
+            fetched_at: response.fetched_at
+          )
+        end
+
+        def normalize_provider(provider)
+          return "gemini" if provider == "google"
+
+          provider
+        end
+
+        def price_per_million(value)
+          return nil if value.nil?
+
+          value.to_f * PER_TOKEN_TO_PER_MILLION
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/validator.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Validator
+      Result = Data.define(:accepted, :rejected, :flagged)
+      Issue = Data.define(:model, :reason, :old_price, :new_price)
+
+      MAX_INPUT_PER_MILLION = 100.0
+      MAX_OUTPUT_PER_MILLION = 500.0
+      MAX_RELATIVE_CHANGE = 3.0
+
+      def validate_batch(merged_prices, existing_registry:)
+        merged_prices.each_with_object(Result.new(accepted: {}, rejected: [], flagged: [])) do |(model, price), result|
+          old_price = normalize_entry(existing_registry[model])
+          status, reason = validate(new_price: price, old_price: old_price)
+
+          case status
+          when :rejected
+            result.rejected << Issue.new(model: model, reason: reason, old_price: old_price, new_price: price)
+          when :flagged
+            result.flagged << Issue.new(model: model, reason: reason, old_price: old_price, new_price: price)
+            result.accepted[model] = price
+          else
+            result.accepted[model] = price
+          end
+        end
+      end
+
+      private
+
+      def validate(new_price:, old_price:)
+        overrides = Array(normalize_entry(old_price)["_validator_override"])
+
+        return [:rejected, "input > $#{MAX_INPUT_PER_MILLION}/1M"] if new_price.input > MAX_INPUT_PER_MILLION
+        return [:rejected, "output > $#{MAX_OUTPUT_PER_MILLION}/1M"] if new_price.output > MAX_OUTPUT_PER_MILLION
+        return [:ok, nil] if overrides.include?("skip_relative_change")
+
+        if old_price.any? && changed_too_much?(old_price, new_price)
+          return [:flagged, "price changed >#{MAX_RELATIVE_CHANGE}x"]
+        end
+
+        [:ok, nil]
+      end
+
+      def changed_too_much?(old_price, new_price)
+        %i[input output].any? do |field|
+          old_value = old_price[field.to_s].to_f
+          next false if old_value.zero?
+
+          new_value = new_price.public_send(field).to_f
+          next false if new_value.zero?
+
+          ratio = [new_value / old_value, old_value / new_value].max
+          ratio > MAX_RELATIVE_CHANGE
+        end
+      end
+
+      def normalize_entry(entry)
+        (entry || {}).each_with_object({}) do |(key, value), normalized|
+          normalized[key.to_s] = value
+        end
+      end
+    end
+  end
+end