llm_cost_tracker 0.3.0 → 0.3.2

data/lib/llm_cost_tracker/parsers/gemini.rb

@@ -35,7 +35,12 @@ module LlmCostTracker
         usage    = response["usageMetadata"]
         return nil unless usage
 
-        build_parsed_usage(request_url, usage, usage_source: :response)
+        build_parsed_usage(
+          request_url,
+          usage,
+          usage_source: :response,
+          provider_response_id: response["responseId"]
+        )
       end
 
       def parse_stream(request_url, _request_body, response_status, events)
@@ -45,10 +50,17 @@ module LlmCostTracker
         model = extract_model_from_url(request_url)
 
         if usage
-          build_parsed_usage(request_url, usage, stream: true, usage_source: :stream_final)
+          build_parsed_usage(
+            request_url,
+            usage,
+            stream: true,
+            usage_source: :stream_final,
+            provider_response_id: stream_response_id(events)
+          )
         else
           ParsedUsage.build(
             provider: "gemini",
+            provider_response_id: stream_response_id(events),
             model: model,
             input_tokens: 0,
             output_tokens: 0,
@@ -61,7 +73,7 @@ module LlmCostTracker
 
       private
 
-      def build_parsed_usage(request_url, usage, usage_source:, stream: false)
+      def build_parsed_usage(request_url, usage, usage_source:, stream: false, provider_response_id: nil)
         ParsedUsage.build(
           provider: "gemini",
           model: extract_model_from_url(request_url),
@@ -70,7 +82,8 @@ module LlmCostTracker
           total_tokens: usage["totalTokenCount"].to_i,
           cached_input_tokens: usage["cachedContentTokenCount"],
           stream: stream,
-          usage_source: usage_source
+          usage_source: usage_source,
+          provider_response_id: provider_response_id
         )
       end
 
@@ -90,6 +103,17 @@ module LlmCostTracker
         usage["candidatesTokenCount"].to_i + usage["thoughtsTokenCount"].to_i
       end
 
+      def stream_response_id(events)
+        events.each do |event|
+          data = event[:data]
+          next unless data.is_a?(Hash)
+
+          id = data["responseId"]
+          return id if id && !id.to_s.empty?
+        end
+        nil
+      end
+
       def streaming_url?(request_url)
         URI.parse(request_url.to_s).path.match?(STREAM_PATH_PATTERN)
       rescue URI::InvalidURIError

data/lib/llm_cost_tracker/parsers/openai_compatible.rb

@@ -20,7 +20,10 @@ module LlmCostTracker
       end
 
       def provider_names
-        ["openai_compatible", *configured_providers.each_value.map(&:to_s)].uniq.freeze
+        [
+          "openai_compatible",
+          *LlmCostTracker.configuration.openai_compatible_providers.each_value.map(&:to_s)
+        ].uniq.freeze
       end
 
       def parse(request_url, request_body, response_status, response_body)
@@ -41,11 +44,7 @@ module LlmCostTracker
       end
 
       def provider_for_host(host)
-        configured_providers[host.to_s.downcase]&.to_s
-      end
-
-      def configured_providers
-        LlmCostTracker.configuration.openai_compatible_providers
+        LlmCostTracker.configuration.openai_compatible_providers[host.to_s.downcase]&.to_s
       end
 
       def tracked_path?(path)

data/lib/llm_cost_tracker/parsers/openai_usage.rb

@@ -16,6 +16,7 @@ module LlmCostTracker
 
         ParsedUsage.build(
           provider: provider_for(request_url),
+          provider_response_id: response["id"],
           model: response["model"] || request["model"],
           input_tokens: (usage["prompt_tokens"] || usage["input_tokens"]).to_i,
           output_tokens: (usage["completion_tokens"] || usage["output_tokens"]).to_i,
@@ -35,6 +36,7 @@ module LlmCostTracker
         if usage
           ParsedUsage.build(
             provider: provider_for(request_url),
+            provider_response_id: detect_stream_response_id(events),
             model: model,
             input_tokens: (usage["prompt_tokens"] || usage["input_tokens"]).to_i,
             output_tokens: (usage["completion_tokens"] || usage["output_tokens"]).to_i,
@@ -46,6 +48,7 @@ module LlmCostTracker
         else
           ParsedUsage.build(
             provider: provider_for(request_url),
+            provider_response_id: detect_stream_response_id(events),
             model: model,
             input_tokens: 0,
             output_tokens: 0,
@@ -78,6 +81,17 @@ module LlmCostTracker
         nil
       end
 
+      def detect_stream_response_id(events)
+        events.each do |event|
+          data = event[:data]
+          next unless data.is_a?(Hash)
+
+          id = data["id"] || data.dig("response", "id")
+          return id if id && !id.to_s.empty?
+        end
+        nil
+      end
+
       def cached_input_tokens(usage)
         details = usage["prompt_tokens_details"] || usage["input_tokens_details"] || {}
         details["cached_tokens"]

data/lib/llm_cost_tracker/price_registry.rb

@@ -26,9 +26,7 @@ module LlmCostTracker
       end
 
       def normalize_price_table(table)
-        (table || {}).each_with_object({}) do |(model, price), normalized|
-          normalized[model.to_s] = normalize_price_entry(price)
-        end
+        normalize_price_entries(table, context: "price table")
       end
 
       def file_prices(path)
@@ -47,7 +45,7 @@ module LlmCostTracker
           @file_prices_cache = { key: cache_key, value: value }.freeze
           value
         end
-      rescue Errno::ENOENT, JSON::ParserError, Psych::Exception, ArgumentError, TypeError, NoMethodError => e
+      rescue Errno::ENOENT, JSON::ParserError, Psych::Exception, ArgumentError, TypeError => e
         raise Error, "Unable to load prices_file #{path.inspect}: #{e.message}"
       end
 
@@ -60,15 +58,23 @@ module LlmCostTracker
       end
 
       def normalize_price_entry(price)
-        (price || {}).each_with_object({}) do |(key, value), normalized|
+        price.each_with_object({}) do |(key, value), normalized|
           key = key.to_s
           normalized[key.to_sym] = Float(value) if PRICE_KEYS.include?(key)
         end
       end
 
       def normalize_file_prices(table, path:)
-        (table || {}).each_with_object({}) do |(model, price), normalized|
-          warn_unknown_keys(model, price, path)
+        normalize_price_entries(table, context: path)
+      end
+
+      def normalize_price_entries(table, context:)
+        table = {} if table.nil?
+        raise ArgumentError, "#{context} must be a hash of models" unless table.is_a?(Hash)
+
+        table.each_with_object({}) do |(model, price), normalized|
+          price = validate_price_entry(price, model: model, context: context)
+          warn_unknown_keys(model, price, context)
           normalized[model.to_s] = normalize_price_entry(price)
         end
       end
@@ -95,8 +101,17 @@ module LlmCostTracker
       end
 
       def price_file_models(registry)
+        raise ArgumentError, "prices_file must be a hash" unless registry.is_a?(Hash)
+
         registry.fetch("models", registry)
       end
+
+      def validate_price_entry(price, model:, context:)
+        return {} if price.nil?
+        return price if price.is_a?(Hash)
+
+        raise ArgumentError, "price entry for #{model.inspect} in #{context} must be a hash"
+      end
     end
   end
 end

data/lib/llm_cost_tracker/price_sync/refresh_plan_builder.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    class RefreshPlanBuilder
+      def initialize(sources:, loader: RegistryLoader.new)
+        @sources = sources
+        @loader = loader
+      end
+
+      def call(path:, seed_path:, fetcher:, today:)
+        path = path.to_s
+        registry = loader.call(path: path, seed_path: seed_path)
+        current_models = registry.fetch("models", {})
+        source_results, failed_sources = fetch_all(current_models, fetcher)
+        merged, discrepancies = Merger.new.merge(source_results)
+        validated = Validator.new.validate_batch(merged, existing_registry: current_models)
+        updated_models = apply_changes(current_models, validated.accepted, today)
+
+        PriceSync::RefreshPlan.new(
+          path: path,
+          registry: registry,
+          updated_registry: registry.merge(
+            "metadata" => updated_metadata(
+              registry["metadata"],
+              today,
+              refresh_succeeded: source_results.any? { |_source, result| result.prices.any? },
+              source_results: source_results
+            ),
+            "models" => updated_models
+          ),
+          accepted: validated.accepted,
+          changes: price_changes(current_models, updated_models),
+          orphaned_models: compute_orphaned(current_models, merged.keys),
+          failed_sources: failed_sources,
+          discrepancies: discrepancies,
+          rejected: validated.rejected,
+          flagged: validated.flagged,
+          sources_used: source_usage(source_results),
+          source_results: source_results
+        )
+      end
+
+      private
+
+      attr_reader :sources, :loader
+
+      def fetch_all(current_models, fetcher)
+        results = {}
+        failures = {}
+
+        sources.each do |source|
+          results[source.name.to_sym] = source.fetch(current_models: current_models, fetcher: fetcher)
+        rescue Error => e
+          failures[source.name.to_sym] = e.message
+        end
+
+        [results, failures]
+      end
+
+      def apply_changes(current_models, accepted, today)
+        merged = seed_models(current_models)
+
+        accepted.each do |model, price|
+          next if manual_model?(merged[model])
+
+          merged[model] = registry_entry_for(merged[model], price, today)
+        end
+
+        merged.sort.to_h
+      end
+
+      def compute_orphaned(current_models, merged_models)
+        seed_models(current_models).keys.reject do |model|
+          manual_model?(current_models[model]) || merged_models.include?(model)
+        end.sort
+      end
+
+      def seed_models(current_models)
+        normalize_models(current_models).transform_values do |entry|
+          next entry if entry.key?("_source")
+
+          entry.merge("_source" => "seed")
+        end
+      end
+
+      def normalize_models(models)
+        normalize_hash(models).each_with_object({}) do |(model, entry), normalized|
+          normalized[model.to_s] = normalize_hash(entry)
+        end
+      end
+
+      def normalize_hash(hash)
+        return {} if hash.nil?
+        raise ArgumentError, "price sync entries must be hashes" unless hash.is_a?(Hash)
+
+        hash.each_with_object({}) do |(key, value), normalized|
+          normalized[key.to_s] = value
+        end
+      end
+
+      def manual_model?(entry)
+        normalize_hash(entry)["_source"] == "manual"
+      end
+
+      def registry_entry_for(existing_entry, price, today)
+        normalize_hash(existing_entry)
+          .except(*PriceRegistry::PRICE_KEYS)
+          .merge(price.to_registry_entry(today: today))
+      end
+
+      def updated_metadata(existing, today, refresh_succeeded:, source_results:)
+        metadata = normalize_hash(existing)
+        metadata["currency"] ||= "USD"
+        metadata["unit"] ||= "1M tokens"
+        return metadata unless refresh_succeeded
+
+        metadata["updated_at"] = today.iso8601
+        metadata["source_urls"] = source_urls(source_results)
+        metadata
+      end
+
+      def source_usage(source_results)
+        source_results.transform_values do |result|
+          PriceSync::SourceUsage.new(prices_count: result.prices.size, source_version: result.source_version)
+        end
+      end
+
+      def price_changes(current_models, updated_models)
+        current_models = normalize_models(current_models)
+        updated_models = normalize_models(updated_models)
+
+        (current_models.keys | updated_models.keys).sort.each_with_object({}) do |model, changes|
+          fields = price_field_changes(current_models[model], updated_models[model])
+          changes[model] = fields if fields.any?
+        end
+      end
+
+      def price_field_changes(current_entry, updated_entry)
+        current_price = comparable_price(current_entry)
+        updated_price = comparable_price(updated_entry)
+
+        (current_price.keys | updated_price.keys).sort.each_with_object({}) do |field, changes|
+          from = current_price[field]
+          to = updated_price[field]
+          next if from == to
+
+          changes[field] = { "from" => from, "to" => to }
+        end
+      end
+
+      def comparable_price(entry)
+        normalize_hash(entry).slice(*PriceRegistry::PRICE_KEYS)
+      end
+
+      def source_urls(source_results)
+        names = source_results.keys.map(&:to_sym)
+        sources.select { |source| names.include?(source.name.to_sym) }.map(&:url)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/registry_loader.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+
+module LlmCostTracker
+  module PriceSync
+    class RegistryLoader
+      YAML_EXTENSIONS = %w[.yml .yaml].freeze
+
+      def call(path:, seed_path:)
+        source_path = File.exist?(path.to_s) ? path.to_s : seed_path.to_s
+        normalize_registry(load_registry_file(source_path))
+      rescue Errno::ENOENT, JSON::ParserError, Psych::Exception, ArgumentError, TypeError => e
+        raise Error, "Unable to load pricing registry #{source_path.inspect}: #{e.message}"
+      end
+
+      private
+
+      def load_registry_file(path)
+        contents = File.read(path)
+        registry = yaml_file?(path) ? (YAML.safe_load(contents, aliases: false) || {}) : JSON.parse(contents)
+        raise ArgumentError, "pricing registry must be a hash" unless registry.is_a?(Hash)
+
+        registry
+      end
+
+      def normalize_registry(registry)
+        {
+          "metadata" => normalize_hash(registry.fetch("metadata", {}), label: "pricing metadata"),
+          "models" => normalize_models(registry.fetch("models", {}))
+        }
+      end
+
+      def normalize_models(models)
+        normalize_hash(models, label: "pricing models").each_with_object({}) do |(model, entry), normalized|
+          normalized[model.to_s] = normalize_hash(entry, label: "pricing model entry")
+        end
+      end
+
+      def normalize_hash(hash, label:)
+        return {} if hash.nil?
+        raise ArgumentError, "#{label} must be a hash" unless hash.is_a?(Hash)
+
+        hash.each_with_object({}) do |(key, value), normalized|
+          normalized[key.to_s] = value
+        end
+      end
+
+      def yaml_file?(path)
+        YAML_EXTENSIONS.include?(File.extname(path).downcase)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/registry_writer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "yaml"
+
+module LlmCostTracker
+  module PriceSync
+    class RegistryWriter
+      YAML_EXTENSIONS = %w[.yml .yaml].freeze
+
+      def call(path:, registry:)
+        FileUtils.mkdir_p(File.dirname(path))
+        payload = yaml_file?(path) ? YAML.dump(registry) : "#{JSON.pretty_generate(registry)}\n"
+        File.write(path, payload)
+      end
+
+      private
+
+      def yaml_file?(path)
+        YAML_EXTENSIONS.include?(File.extname(path).downcase)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 require "date"
-require "fileutils"
-require "json"
-require "yaml"
 
 require_relative "price_sync/fetcher"
 require_relative "price_sync/raw_price"
 require_relative "price_sync/source"
 require_relative "price_sync/source_result"
+require_relative "price_sync/registry_loader"
+require_relative "price_sync/registry_writer"
+require_relative "price_sync/refresh_plan_builder"
 require_relative "price_sync/model_catalog"
 require_relative "price_sync/merger"
 require_relative "price_sync/validator"
@@ -16,10 +16,8 @@ require_relative "price_sync/sources/litellm"
 require_relative "price_sync/sources/open_router"
 
 module LlmCostTracker
-  # rubocop:disable Metrics/ModuleLength, Metrics/ClassLength
   module PriceSync
     DEFAULT_OUTPUT_PATH = PriceRegistry::DEFAULT_PRICES_PATH
-    YAML_EXTENSIONS = %w[.yml .yaml].freeze
 
     SourceUsage = Data.define(:prices_count, :source_version)
     SyncResult = Data.define(
@@ -71,11 +69,16 @@ module LlmCostTracker
     class << self
       def sync(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, preview: false, strict: false,
                fetcher: Fetcher.new, today: Date.today)
-        plan = build_refresh_plan(path: path, seed_path: seed_path, fetcher: fetcher, today: today)
+        plan = RefreshPlanBuilder.new(sources: sources).call(
+          path: path,
+          seed_path: seed_path,
+          fetcher: fetcher,
+          today: today
+        )
         raise Error, strict_failure_message(plan) if strict_sync_failure?(plan, strict: strict)
 
         written = !preview && plan.refresh_succeeded?
-        write_registry(plan.path, plan.updated_registry) if written
+        RegistryWriter.new.call(path: plan.path, registry: plan.updated_registry) if written
 
         SyncResult.new(
           path: plan.path,
@@ -92,7 +95,12 @@ module LlmCostTracker
       end
 
       def check(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, fetcher: Fetcher.new, today: Date.today)
-        plan = build_refresh_plan(path: path, seed_path: seed_path, fetcher: fetcher, today: today)
+        plan = RefreshPlanBuilder.new(sources: sources).call(
+          path: path,
+          seed_path: seed_path,
+          fetcher: fetcher,
+          today: today
+        )
 
         CheckResult.new(
           path: plan.path,
@@ -113,166 +121,6 @@ module LlmCostTracker
         [Sources::Litellm.new, Sources::OpenRouter.new]
       end
 
-      def build_refresh_plan(path:, seed_path:, fetcher:, today:)
-        path = path.to_s
-        registry = load_registry(path, seed_path: seed_path)
-        current_models = registry.fetch("models", {})
-        source_results, failed_sources = fetch_all(current_models, fetcher)
-        merged, discrepancies = Merger.new.merge(source_results)
-        validated = Validator.new.validate_batch(merged, existing_registry: current_models)
-        updated_models = apply_changes(current_models, validated.accepted, today)
-        refresh_succeeded = source_results.any? { |_source, result| result.prices.any? }
-
-        RefreshPlan.new(
-          path: path,
-          registry: registry,
-          updated_registry: registry.merge(
-            "metadata" => updated_metadata(
-              registry["metadata"],
-              today,
-              refresh_succeeded: refresh_succeeded,
-              source_results: source_results
-            ),
-            "models" => updated_models
-          ),
-          accepted: validated.accepted,
-          changes: price_changes(current_models, updated_models),
-          orphaned_models: compute_orphaned(current_models, merged.keys),
-          failed_sources: failed_sources,
-          discrepancies: discrepancies,
-          rejected: validated.rejected,
-          flagged: validated.flagged,
-          sources_used: source_usage(source_results),
-          source_results: source_results
-        )
-      end
-
-      def fetch_all(current_models, fetcher)
-        results = {}
-        failures = {}
-
-        sources.each do |source|
-          results[source.name.to_sym] = source.fetch(current_models: current_models, fetcher: fetcher)
-        rescue Error => e
-          failures[source.name.to_sym] = e.message
-        end
-
-        [results, failures]
-      end
-
-      def apply_changes(current_models, accepted, today)
-        merged = seed_models(current_models)
-
-        accepted.each do |model, price|
-          next if manual_model?(merged[model])
-
-          merged[model] = registry_entry_for(merged[model], price, today)
-        end
-
-        merged.sort.to_h
-      end
-
-      def compute_orphaned(current_models, merged_models)
-        seed_models(current_models).keys.reject do |model|
-          manual_model?(current_models[model]) || merged_models.include?(model)
-        end.sort
-      end
-
-      def load_registry(path, seed_path:)
-        source_path = File.exist?(path) ? path : seed_path.to_s
-        normalize_registry(load_registry_file(source_path))
-      rescue Errno::ENOENT, JSON::ParserError, Psych::Exception, ArgumentError, TypeError, NoMethodError => e
-        raise Error, "Unable to load pricing registry #{source_path.inspect}: #{e.message}"
-      end
-
-      def load_registry_file(path)
-        contents = File.read(path)
-        return YAML.safe_load(contents, aliases: false) || {} if yaml_file?(path)
-
-        JSON.parse(contents)
-      end
-
-      def normalize_registry(registry)
-        {
-          "metadata" => normalize_hash(registry.fetch("metadata", {})),
-          "models" => normalize_models(registry.fetch("models", {}))
-        }
-      end
-
-      def normalize_models(models)
-        (models || {}).each_with_object({}) do |(model, entry), normalized|
-          normalized[model.to_s] = normalize_hash(entry)
-        end
-      end
-
-      def normalize_hash(hash)
-        (hash || {}).each_with_object({}) do |(key, value), normalized|
-          normalized[key.to_s] = value
-        end
-      end
-
-      def seed_models(current_models)
-        normalize_models(current_models).transform_values do |entry|
-          next entry if entry.key?("_source")
-
-          entry.merge("_source" => "seed")
-        end
-      end
-
-      def manual_model?(entry)
-        normalize_hash(entry)["_source"] == "manual"
-      end
-
-      def registry_entry_for(existing_entry, price, today)
-        normalize_hash(existing_entry)
-          .except(*PriceRegistry::PRICE_KEYS)
-          .merge(price.to_registry_entry(today: today))
-      end
-
-      def updated_metadata(existing, today, refresh_succeeded:, source_results:)
-        metadata = normalize_hash(existing)
-        metadata["currency"] ||= "USD"
-        metadata["unit"] ||= "1M tokens"
-        return metadata unless refresh_succeeded
-
-        metadata["updated_at"] = today.iso8601
-        metadata["source_urls"] = source_urls(source_results)
-        metadata
-      end
-
-      def source_usage(source_results)
-        source_results.transform_values do |result|
-          SourceUsage.new(prices_count: result.prices.size, source_version: result.source_version)
-        end
-      end
-
-      def price_changes(current_models, updated_models)
-        current_models = normalize_models(current_models)
-        updated_models = normalize_models(updated_models)
-
-        (current_models.keys | updated_models.keys).sort.each_with_object({}) do |model, changes|
-          fields = price_field_changes(current_models[model], updated_models[model])
-          changes[model] = fields if fields.any?
-        end
-      end
-
-      def price_field_changes(current_entry, updated_entry)
-        current_price = comparable_price(current_entry)
-        updated_price = comparable_price(updated_entry)
-
-        (current_price.keys | updated_price.keys).sort.each_with_object({}) do |field, changes|
-          from = current_price[field]
-          to = updated_price[field]
-          next if from == to
-
-          changes[field] = { "from" => from, "to" => to }
-        end
-      end
-
-      def comparable_price(entry)
-        normalize_hash(entry).slice(*PriceRegistry::PRICE_KEYS)
-      end
-
       def strict_sync_failure?(plan, strict:)
         strict && (plan.failed_sources.any? || plan.rejected.any?)
       end
@@ -289,22 +137,6 @@ module LlmCostTracker
         end
         "Price sync failed in strict mode: #{messages.join('; ')}"
       end
-
-      def source_urls(source_results)
-        names = source_results.keys.map(&:to_sym)
-        sources.select { |source| names.include?(source.name.to_sym) }.map(&:url)
-      end
-
-      def write_registry(path, registry)
-        FileUtils.mkdir_p(File.dirname(path))
-        payload = yaml_file?(path) ? YAML.dump(registry) : "#{JSON.pretty_generate(registry)}\n"
-        File.write(path, payload)
-      end
-
-      def yaml_file?(path)
-        YAML_EXTENSIONS.include?(File.extname(path).downcase)
-      end
     end
   end
-  # rubocop:enable Metrics/ModuleLength, Metrics/ClassLength
 end