llm_cost_tracker 0.2.0 → 0.3.0

data/lib/llm_cost_tracker/parsers/openai_compatible.rb

@@ -19,10 +19,18 @@ module LlmCostTracker
         false
       end
 
+      def provider_names
+        ["openai_compatible", *configured_providers.each_value.map(&:to_s)].uniq.freeze
+      end
+
       def parse(request_url, request_body, response_status, response_body)
         parse_openai_usage(request_url, request_body, response_status, response_body)
       end
 
+      def parse_stream(request_url, request_body, response_status, events)
+        parse_openai_stream_usage(request_url, request_body, response_status, events)
+      end
+
       private
 
       def provider_for(request_url)

data/lib/llm_cost_tracker/parsers/openai_usage.rb

@@ -20,10 +20,64 @@ module LlmCostTracker
           input_tokens: (usage["prompt_tokens"] || usage["input_tokens"]).to_i,
           output_tokens: (usage["completion_tokens"] || usage["output_tokens"]).to_i,
           total_tokens: usage["total_tokens"].to_i,
-          cached_input_tokens: cached_input_tokens(usage)
+          cached_input_tokens: cached_input_tokens(usage),
+          usage_source: :response
         )
       end
 
+      def parse_openai_stream_usage(request_url, request_body, response_status, events)
+        return nil unless response_status == 200
+
+        request = safe_json_parse(request_body)
+        model = detect_stream_model(events) || request["model"]
+        usage = detect_stream_usage(events)
+
+        if usage
+          ParsedUsage.build(
+            provider: provider_for(request_url),
+            model: model,
+            input_tokens: (usage["prompt_tokens"] || usage["input_tokens"]).to_i,
+            output_tokens: (usage["completion_tokens"] || usage["output_tokens"]).to_i,
+            total_tokens: usage["total_tokens"].to_i,
+            cached_input_tokens: cached_input_tokens(usage),
+            stream: true,
+            usage_source: :stream_final
+          )
+        else
+          ParsedUsage.build(
+            provider: provider_for(request_url),
+            model: model,
+            input_tokens: 0,
+            output_tokens: 0,
+            total_tokens: 0,
+            stream: true,
+            usage_source: :unknown
+          )
+        end
+      end
+
+      def detect_stream_usage(events)
+        events.reverse_each do |event|
+          data = event[:data]
+          next unless data.is_a?(Hash)
+
+          usage = data["usage"]
+          return usage if usage.is_a?(Hash) && !usage.empty?
+        end
+        nil
+      end
+
+      def detect_stream_model(events)
+        events.each do |event|
+          data = event[:data]
+          next unless data.is_a?(Hash)
+
+          model = data["model"]
+          return model if model && !model.to_s.empty?
+        end
+        nil
+      end
+
       def cached_input_tokens(usage)
         details = usage["prompt_tokens_details"] || usage["input_tokens_details"] || {}
         details["cached_tokens"]

data/lib/llm_cost_tracker/parsers/registry.rb

@@ -1,23 +1,35 @@
 # frozen_string_literal: true
 
+require "monitor"
+
 module LlmCostTracker
   module Parsers
     class Registry
+      MUTEX = Monitor.new
+
       class << self
         def parsers
-          @parsers ||= default_parsers
+          @parsers || MUTEX.synchronize { @parsers ||= default_parsers.freeze }
         end
 
         def register(parser)
-          parsers.unshift(parser)
+          MUTEX.synchronize do
+            current = @parsers || default_parsers.freeze
+            @parsers = ([parser] + current).freeze
+          end
         end
 
         def find_for(url)
           parsers.find { |parser| parser.match?(url) }
         end
 
+        def find_for_provider(provider)
+          provider_name = provider.to_s
+          parsers.find { |parser| parser.provider_names.include?(provider_name) }
+        end
+
         def reset!
-          @parsers = nil
+          MUTEX.synchronize { @parsers = nil }
         end
 
         private

data/lib/llm_cost_tracker/parsers/sse.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module Parsers
+    module SSE
+      DONE_MARKER = "[DONE]"
+
+      class << self
+        def parse(body)
+          return [] if body.nil? || body.empty?
+
+          return parse_json_array(body) if probably_json_array?(body)
+
+          parse_event_stream(body)
+        end
+
+        private
+
+        def parse_event_stream(body)
+          events = []
+          current_event = nil
+          data_lines = []
+
+          body.each_line do |raw|
+            line = raw.chomp
+
+            if line.empty?
+              events << finalize_event(current_event, data_lines) if data_lines.any?
+              current_event = nil
+              data_lines = []
+              next
+            end
+
+            next if line.start_with?(":")
+
+            field, _, value = line.partition(":")
+            value = value[1..] if value.start_with?(" ")
+
+            case field
+            when "event" then current_event = value
+            when "data"  then data_lines << value
+            end
+          end
+
+          events << finalize_event(current_event, data_lines) if data_lines.any?
+          events.compact
+        end
+
+        def parse_json_array(body)
+          parsed = JSON.parse(body)
+          return [] unless parsed.is_a?(Array)
+
+          parsed.map { |entry| { event: nil, data: entry } }
+        rescue JSON::ParserError
+          []
+        end
+
+        def finalize_event(event_name, data_lines)
+          payload = data_lines.join("\n")
+          return nil if payload == DONE_MARKER
+
+          { event: event_name, data: decode_data(payload) }
+        end
+
+        def decode_data(payload)
+          return payload if payload.empty?
+
+          JSON.parse(payload)
+        rescue JSON::ParserError
+          payload
+        end
+
+        def probably_json_array?(body)
+          body.lstrip.start_with?("[")
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_registry.rb

@@ -11,7 +11,7 @@ module LlmCostTracker
     DEFAULT_PRICES_PATH = File.expand_path("prices.json", __dir__)
     EMPTY_PRICES = {}.freeze
     PRICE_KEYS = %w[input cached_input output cache_read_input cache_creation_input].freeze
-    METADATA_KEYS = %w[_source _updated _notes].freeze
+    METADATA_KEYS = %w[_source _source_version _fetched_at _updated _notes _validator_override].freeze
     MUTEX = Monitor.new
 
     class << self

data/lib/llm_cost_tracker/price_sync/fetcher.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "digest"
+require "net/http"
+require "time"
+require "uri"
+
+module LlmCostTracker
+  module PriceSync
+    class Fetcher
+      Response = Data.define(:body, :etag, :last_modified, :not_modified, :fetched_at) do
+        def source_version
+          etag || last_modified || Digest::SHA256.hexdigest(body.to_s)
+        end
+      end
+
+      USER_AGENT = "llm_cost_tracker price sync"
+      MAX_REDIRECTS = 5
+      OPEN_TIMEOUT = 5
+      READ_TIMEOUT = 10
+      WRITE_TIMEOUT = 10
+
+      def get(url, etag: nil, redirects: 0)
+        raise Error, "Too many redirects while fetching #{url}" if redirects > MAX_REDIRECTS
+
+        uri = URI.parse(url)
+        request = Net::HTTP::Get.new(uri)
+        request["User-Agent"] = USER_AGENT
+        request["If-None-Match"] = etag if etag
+
+        response = Net::HTTP.start(
+          uri.host,
+          uri.port,
+          use_ssl: uri.scheme == "https",
+          open_timeout: OPEN_TIMEOUT,
+          read_timeout: READ_TIMEOUT,
+          write_timeout: WRITE_TIMEOUT
+        ) do |http|
+          http.request(request)
+        end
+
+        case response
+        when Net::HTTPSuccess
+          build_response(response, not_modified: false)
+        when Net::HTTPNotModified
+          build_response(response, body: nil, not_modified: true)
+        when Net::HTTPRedirection
+          location = response["location"]
+          raise Error, "Redirect without location while fetching #{url}" if location.nil? || location.empty?
+
+          get(URI.join(url, location).to_s, etag: etag, redirects: redirects + 1)
+        else
+          raise Error, "Unable to fetch #{url}: HTTP #{response.code}"
+        end
+      rescue SocketError, SystemCallError, Timeout::Error => e
+        raise Error, "Unable to fetch #{url}: #{e.class}: #{e.message}"
+      end
+
+      private
+
+      def build_response(response, not_modified:, body: response.body)
+        Response.new(
+          body: body,
+          etag: response["etag"],
+          last_modified: response["last-modified"],
+          not_modified: not_modified,
+          fetched_at: Time.now.utc.iso8601
+        )
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/merger.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Merger
+      Discrepancy = Data.define(:model, :field, :values)
+
+      PRIORITY_ORDER = %i[litellm openrouter].freeze
+      SUPPLEMENTAL_FIELDS = %i[cached_input cache_read_input cache_creation_input].freeze
+
+      def merge(results_by_source)
+        prices = collect_prices(results_by_source)
+        discrepancies = []
+
+        merged = prices.group_by(&:model).sort.to_h.transform_values do |candidates|
+          sorted = sort_candidates(candidates)
+          discrepancies.concat(detect_discrepancies(sorted))
+          fill_missing_fields(sorted.first, sorted.drop(1))
+        end
+
+        [merged, discrepancies]
+      end
+
+      private
+
+      def collect_prices(results_by_source)
+        results_by_source.flat_map do |source_name, result|
+          result.prices.map do |price|
+            price.with(source: source_name)
+          end
+        end
+      end
+
+      def sort_candidates(candidates)
+        candidates.sort_by do |price|
+          PRIORITY_ORDER.index(price.source.to_sym) || PRIORITY_ORDER.length
+        end
+      end
+
+      def fill_missing_fields(primary, fallbacks)
+        SUPPLEMENTAL_FIELDS.reduce(primary) do |current, field|
+          next current if current.public_send(field)
+
+          fallback = fallbacks.find { |candidate| candidate.public_send(field) }
+          fallback ? current.with(field => fallback.public_send(field)) : current
+        end
+      end
+
+      def detect_discrepancies(candidates)
+        return [] if candidates.length < 2
+
+        RawPrice::PRICE_FIELDS.filter_map do |field|
+          values = candidates.each_with_object({}) do |price, collected|
+            value = price.public_send(field)
+            collected[price.source] = value unless value.nil?
+          end
+          next if values.size < 2
+          next unless discrepant?(values.values)
+
+          Discrepancy.new(model: candidates.first.model, field: field, values: values)
+        end
+      end
+
+      def discrepant?(values)
+        min, max = values.minmax
+        return max != min if min.to_f.zero?
+
+        ((max - min).abs / min.to_f) >= 0.05
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/model_catalog.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class ModelCatalog
+      OPENROUTER_PROVIDER_PREFIXES = {
+        openai: %w[openai],
+        anthropic: %w[anthropic],
+        gemini: %w[google]
+      }.freeze
+      LITELLM_PROVIDER_PREFIXES = {
+        openai: [nil, "openai"],
+        anthropic: [nil, "anthropic"],
+        gemini: [nil, "gemini"]
+      }.freeze
+      ALIASES = {
+        "gpt-4o-2024-05-13" => "gpt-4o"
+      }.freeze
+
+      class << self
+        def resolve_from_litellm(our_model, payload)
+          litellm_candidates(our_model).find { |candidate| payload.key?(candidate) }
+        end
+
+        def resolve_from_openrouter(our_model, index)
+          openrouter_candidates(our_model).find { |candidate| index.key?(candidate) }
+        end
+
+        def guess_provider(our_model)
+          case our_model.to_s
+          when /\A(?:gpt-|o1|o3|o4|chatgpt|text-embedding)/
+            :openai
+          when /\Aclaude-/
+            :anthropic
+          when /\Agemini-/
+            :gemini
+          end
+        end
+
+        private
+
+        def litellm_candidates(our_model)
+          provider = guess_provider(our_model)
+          prefixes = LITELLM_PROVIDER_PREFIXES.fetch(provider, [nil])
+
+          model_variants(our_model).flat_map do |variant|
+            prefixes.map { |prefix| prefix ? "#{prefix}/#{variant}" : variant }
+          end.uniq
+        end
+
+        def openrouter_candidates(our_model)
+          provider = guess_provider(our_model)
+          prefixes = OPENROUTER_PROVIDER_PREFIXES.fetch(provider, [])
+
+          model_variants(our_model).flat_map do |variant|
+            prefixes.map { |prefix| "#{prefix}/#{variant}" }
+          end.uniq
+        end
+
+        def model_variants(our_model)
+          model = our_model.to_s
+          canonical = ALIASES.fetch(model, model)
+
+          [model, canonical].flat_map do |variant|
+            [variant, anthropic_version_variant(variant)]
+          end.compact.uniq
+        end
+
+        def anthropic_version_variant(model)
+          return nil unless guess_provider(model) == :anthropic
+
+          model.gsub(/(?<=\d)-(?=\d)/, ".")
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/raw_price.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    RawPrice = Data.define(
+      :model,
+      :provider,
+      :input,
+      :output,
+      :cached_input,
+      :cache_read_input,
+      :cache_creation_input,
+      :source,
+      :source_version,
+      :fetched_at
+    )
+
+    class RawPrice
+      PRICE_FIELDS = %w[input output cached_input cache_read_input cache_creation_input].freeze
+
+      def to_registry_entry(today:)
+        {
+          "input" => input,
+          "output" => output,
+          "cached_input" => cached_input,
+          "cache_read_input" => cache_read_input,
+          "cache_creation_input" => cache_creation_input,
+          "_source" => source.to_s,
+          "_source_version" => source_version,
+          "_fetched_at" => fetched_at || today.iso8601
+        }.compact
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/source.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class Source
+      def fetch(current_models:, fetcher:)
+        raise NotImplementedError
+      end
+
+      def name
+        self.class.name.split("::").last.downcase.to_sym
+      end
+
+      def priority
+        100
+      end
+
+      def url
+        raise NotImplementedError
+      end
+
+      private
+
+      def response_version(response)
+        response.source_version
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/source_result.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    SourceResult = Data.define(:prices, :missing_models, :source_version)
+  end
+end

data/lib/llm_cost_tracker/price_sync/sources/litellm.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module PriceSync
+    module Sources
+      class Litellm < Source
+        PER_TOKEN_TO_PER_MILLION = 1_000_000
+        SUPPORTED_MODES = %w[chat completion embedding responses].freeze
+        SUPPORTED_PROVIDERS = %w[openai anthropic gemini text-completion-openai].freeze
+        URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
+
+        def priority
+          10
+        end
+
+        def url
+          URL
+        end
+
+        def fetch(current_models:, fetcher:)
+          response = fetcher.get(url)
+          payload = JSON.parse(response.body.to_s)
+
+          prices = []
+          missing_models = []
+
+          current_models.each_key do |our_model|
+            entry_id = ModelCatalog.resolve_from_litellm(our_model, payload)
+            entry = entry_id && payload[entry_id]
+
+            if entry && supported_entry?(entry)
+              prices << build_raw_price(our_model, entry, response)
+            else
+              missing_models << our_model
+            end
+          end
+
+          SourceResult.new(
+            prices: prices,
+            missing_models: missing_models.sort,
+            source_version: response_version(response)
+          )
+        rescue JSON::ParserError => e
+          raise Error, "Unable to parse #{url}: #{e.message}"
+        end
+
+        private
+
+        def supported_entry?(entry)
+          SUPPORTED_PROVIDERS.include?(entry["litellm_provider"]) &&
+            SUPPORTED_MODES.include?(entry["mode"]) &&
+            entry.key?("input_cost_per_token") &&
+            entry.key?("output_cost_per_token")
+        end
+
+        def build_raw_price(model, entry, response)
+          provider = normalize_provider(entry["litellm_provider"])
+          cache_read = price_per_million(entry["cache_read_input_token_cost"])
+          cache_write = price_per_million(entry["cache_creation_input_token_cost"])
+
+          RawPrice.new(
+            model: model,
+            provider: provider,
+            input: price_per_million(entry["input_cost_per_token"]),
+            output: price_per_million(entry["output_cost_per_token"]),
+            cached_input: provider == "anthropic" ? nil : cache_read,
+            cache_read_input: provider == "anthropic" ? cache_read : nil,
+            cache_creation_input: provider == "anthropic" ? cache_write : nil,
+            source: name,
+            source_version: response_version(response),
+            fetched_at: response.fetched_at
+          )
+        end
+
+        def normalize_provider(provider)
+          return "openai" if provider == "text-completion-openai"
+
+          provider
+        end
+
+        def price_per_million(value)
+          return nil if value.nil?
+
+          value.to_f * PER_TOKEN_TO_PER_MILLION
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/sources/open_router.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LlmCostTracker
+  module PriceSync
+    module Sources
+      class OpenRouter < Source
+        PER_TOKEN_TO_PER_MILLION = 1_000_000
+        SUPPORTED_PREFIXES = %w[openai anthropic google].freeze
+        URL = "https://openrouter.ai/api/v1/models"
+
+        def priority
+          20
+        end
+
+        def url
+          URL
+        end
+
+        def fetch(current_models:, fetcher:)
+          response = fetcher.get(url)
+          payload = JSON.parse(response.body.to_s)
+          index = payload.fetch("data", []).to_h { |entry| [entry["id"], entry] }
+
+          prices = []
+          missing_models = []
+
+          current_models.each_key do |our_model|
+            entry_id = ModelCatalog.resolve_from_openrouter(our_model, index)
+            entry = entry_id && index[entry_id]
+
+            if entry && supported_entry?(entry)
+              prices << build_raw_price(our_model, entry, response)
+            else
+              missing_models << our_model
+            end
+          end
+
+          SourceResult.new(
+            prices: prices,
+            missing_models: missing_models.sort,
+            source_version: response_version(response)
+          )
+        rescue JSON::ParserError => e
+          raise Error, "Unable to parse #{url}: #{e.message}"
+        end
+
+        private
+
+        def supported_entry?(entry)
+          pricing = entry["pricing"] || {}
+          provider = entry["id"].to_s.split("/").first
+
+          SUPPORTED_PREFIXES.include?(provider) &&
+            pricing["prompt"] &&
+            pricing["completion"]
+        end
+
+        def build_raw_price(model, entry, response)
+          pricing = entry.fetch("pricing", {})
+          provider = normalize_provider(entry.fetch("id").split("/").first)
+          cache_read = price_per_million(pricing["input_cache_read"])
+          cache_write = price_per_million(pricing["input_cache_write"])
+
+          RawPrice.new(
+            model: model,
+            provider: provider,
+            input: price_per_million(pricing["prompt"]),
+            output: price_per_million(pricing["completion"]),
+            cached_input: provider == "anthropic" ? nil : cache_read,
+            cache_read_input: provider == "anthropic" ? cache_read : nil,
+            cache_creation_input: provider == "anthropic" ? cache_write : nil,
+            source: name,
+            source_version: response_version(response),
+            fetched_at: response.fetched_at
+          )
+        end
+
+        def normalize_provider(provider)
+          return "gemini" if provider == "google"
+
+          provider
+        end
+
+        def price_per_million(value)
+          return nil if value.nil?
+
+          value.to_f * PER_TOKEN_TO_PER_MILLION
+        end
+      end
+    end
+  end
+end