llm_cost_tracker 0.4.0 → 0.5.0

data/lib/llm_cost_tracker/parsers/anthropic.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uri"
-
 require_relative "base"
 
 module LlmCostTracker
@@ -10,10 +8,7 @@ module LlmCostTracker
       HOSTS = %w[api.anthropic.com].freeze
 
       def match?(url)
-        uri = URI.parse(url.to_s)
-        HOSTS.include?(uri.host.to_s.downcase) && uri.path.include?("/v1/messages")
-      rescue URI::InvalidURIError
-        false
+        match_uri?(url, hosts: HOSTS, path_includes: "/v1/messages")
       end
 
       def provider_names
@@ -52,25 +47,25 @@ module LlmCostTracker
         usage = stream_usage(events)
         response_id = stream_response_id(events)
 
-        usage ? build_stream_result(model, usage, response_id) : build_unknown_stream_result(model, response_id)
+        if usage
+          build_stream_result(model, usage, response_id)
+        else
+          build_unknown_stream_usage(
+            provider: "anthropic",
+            model: model,
+            provider_response_id: response_id
+          )
+        end
       end
 
       private
 
       def stream_usage(events)
-        start_usage = nil
-        latest_delta = nil
-
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
-          case data["type"]
-          when "message_start"
-            start_usage = data.dig("message", "usage")
-          when "message_delta"
-            latest_delta = data["usage"] if data["usage"].is_a?(Hash)
-          end
+        start_usage = find_event_value(events, reverse: true) do |data|
+          data.dig("message", "usage") if data["type"] == "message_start"
+        end
+        latest_delta = find_event_value(events, reverse: true) do |data|
+          data["usage"] if data["type"] == "message_delta" && data["usage"].is_a?(Hash)
         end
 
         return nil unless start_usage || latest_delta
@@ -81,25 +76,11 @@ module LlmCostTracker
       end
 
       def stream_model(events)
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
-          model = data.dig("message", "model")
-          return model if model && !model.empty?
-        end
-        nil
+        find_event_value(events) { |data| data.dig("message", "model") }
       end
 
       def stream_response_id(events)
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
-          id = data.dig("message", "id") || data["id"]
-          return id if id && !id.to_s.empty?
-        end
-        nil
+        find_event_value(events) { |data| data.dig("message", "id") || data["id"] }
       end
 
       def build_stream_result(model, usage, response_id)
@@ -121,19 +102,6 @@ module LlmCostTracker
           usage_source: :stream_final
         )
       end
-
-      def build_unknown_stream_result(model, response_id)
-        ParsedUsage.build(
-          provider: "anthropic",
-          provider_response_id: response_id,
-          model: model,
-          input_tokens: 0,
-          output_tokens: 0,
-          total_tokens: 0,
-          stream: true,
-          usage_source: :unknown
-        )
-      end
     end
   end
 end

data/lib/llm_cost_tracker/parsers/base.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "json"
+require "uri"
 
 module LlmCostTracker
   module Parsers
@@ -40,6 +41,85 @@ module LlmCostTracker
       rescue JSON::ParserError
         {}
       end
+
+      def uri_matches?(url)
+        uri = parsed_uri(url)
+        uri ? yield(uri) : false
+      end
+
+      def match_uri?(url, hosts: nil, exact_paths: nil, path_includes: nil, path_suffixes: nil, path_pattern: nil)
+        uri_matches?(url) do |uri|
+          host_match = hosts.nil? || host_matches?(uri, hosts)
+          path_match = path_matches?(
+            uri,
+            exact_paths: exact_paths,
+            path_includes: path_includes,
+            path_suffixes: path_suffixes,
+            path_pattern: path_pattern
+          )
+          extra_match = block_given? ? yield(uri) : true
+
+          host_match && path_match && extra_match ? true : false
+        end
+      end
+
+      def parsed_uri(url)
+        URI.parse(url.to_s)
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def host_matches?(uri, hosts)
+        hosts.include?(uri.host.to_s.downcase)
+      end
+
+      def path_matches?(uri, exact_paths: nil, path_includes: nil, path_suffixes: nil, path_pattern: nil)
+        path = uri.path.to_s
+        matches = true
+
+        matches &&= exact_paths.include?(path) if exact_paths
+        matches &&= Array(path_includes).all? { |fragment| path.include?(fragment) } if path_includes
+        matches &&= path.match?(path_pattern) if path_pattern
+
+        matches &&= path_suffixes.any? { |suffix| path == suffix || path.end_with?(suffix) } if path_suffixes
+
+        matches
+      end
+
+      def each_event_data(events, reverse: false)
+        enumerator = reverse ? events.reverse_each : events.each
+
+        enumerator.each do |event|
+          data = event[:data]
+          yield data if data.is_a?(Hash)
+        end
+      end
+
+      def find_event_value(events, reverse: false)
+        each_event_data(events, reverse:) do |data|
+          value = yield(data)
+          return value if event_value_present?(value)
+        end
+
+        nil
+      end
+
+      def build_unknown_stream_usage(provider:, model:, provider_response_id:)
+        ParsedUsage.build(
+          provider: provider,
+          provider_response_id: provider_response_id,
+          model: model || ParsedUsage::UNKNOWN_MODEL,
+          input_tokens: 0,
+          output_tokens: 0,
+          total_tokens: 0,
+          stream: true,
+          usage_source: :unknown
+        )
+      end
+
+      def event_value_present?(value)
+        !value.nil? && (!value.respond_to?(:empty?) || !value.empty?)
+      end
     end
   end
 end

data/lib/llm_cost_tracker/parsers/gemini.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uri"
-
 require_relative "base"
 
 module LlmCostTracker
@@ -12,10 +10,7 @@ module LlmCostTracker
       STREAM_PATH_PATTERN  = /:streamGenerateContent\z/
 
       def match?(url)
-        uri = URI.parse(url.to_s)
-        HOSTS.include?(uri.host.to_s.downcase) && uri.path.match?(TRACKED_PATH_PATTERN)
-      rescue URI::InvalidURIError
-        false
+        match_uri?(url, hosts: HOSTS, path_pattern: TRACKED_PATH_PATTERN)
       end
 
       def provider_names
@@ -48,6 +43,7 @@ module LlmCostTracker
 
         usage = merged_stream_usage(events)
         model = extract_model_from_url(request_url)
+        response_id = stream_response_id(events)
 
         if usage
           build_parsed_usage(
@@ -55,18 +51,13 @@ module LlmCostTracker
             usage,
             stream: true,
             usage_source: :stream_final,
-            provider_response_id: stream_response_id(events)
+            provider_response_id: response_id
           )
         else
-          ParsedUsage.build(
+          build_unknown_stream_usage(
             provider: "gemini",
-            provider_response_id: stream_response_id(events),
             model: model,
-            input_tokens: 0,
-            output_tokens: 0,
-            total_tokens: 0,
-            stream: true,
-            usage_source: :unknown
+            provider_response_id: response_id
           )
         end
       end
@@ -91,15 +82,10 @@ module LlmCostTracker
       end
 
       def merged_stream_usage(events)
-        latest = nil
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
+        find_event_value(events, reverse: true) do |data|
           meta = data["usageMetadata"]
-          latest = meta if meta.is_a?(Hash)
+          meta if meta.is_a?(Hash)
         end
-        latest
       end
 
       def output_tokens(usage)
@@ -107,28 +93,19 @@ module LlmCostTracker
       end
 
       def stream_response_id(events)
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
-          id = data["responseId"]
-          return id if id && !id.to_s.empty?
-        end
-        nil
+        find_event_value(events) { |data| data["responseId"] }
       end
 
       def streaming_url?(request_url)
-        URI.parse(request_url.to_s).path.match?(STREAM_PATH_PATTERN)
-      rescue URI::InvalidURIError
-        false
+        match_uri?(request_url, path_pattern: STREAM_PATH_PATTERN)
       end
 
       def extract_model_from_url(url)
-        uri = URI.parse(url.to_s)
+        uri = parsed_uri(url)
+        return nil unless uri
+
         match = uri.path.match(%r{/models/([^/:]+)})
         match && match[1]
-      rescue URI::InvalidURIError
-        nil
       end
     end
   end

data/lib/llm_cost_tracker/parsers/openai.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uri"
-
 require_relative "base"
 require_relative "openai_usage"
 
@@ -14,10 +12,7 @@ module LlmCostTracker
       TRACKED_PATHS = %w[/v1/chat/completions /v1/completions /v1/embeddings /v1/responses].freeze
 
       def match?(url)
-        uri = URI.parse(url.to_s)
-        HOSTS.include?(uri.host.to_s.downcase) && TRACKED_PATHS.include?(uri.path)
-      rescue URI::InvalidURIError
-        false
+        match_uri?(url, hosts: HOSTS, exact_paths: TRACKED_PATHS)
       end
 
       def provider_names

data/lib/llm_cost_tracker/parsers/openai_compatible.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "uri"
-
 require_relative "base"
 require_relative "openai_usage"
 
@@ -13,10 +11,7 @@ module LlmCostTracker
       TRACKED_PATH_SUFFIXES = %w[/chat/completions /completions /embeddings /responses].freeze
 
       def match?(url)
-        uri = URI.parse(url.to_s)
-        !provider_for_host(uri.host).nil? && tracked_path?(uri.path)
-      rescue URI::InvalidURIError
-        false
+        match_uri?(url, path_suffixes: TRACKED_PATH_SUFFIXES) { |uri| provider_for_uri(uri) }
       end
 
       def provider_names
@@ -37,18 +32,14 @@ module LlmCostTracker
       private
 
       def provider_for(request_url)
-        uri = URI.parse(request_url.to_s)
-        provider_for_host(uri.host) || "openai_compatible"
-      rescue URI::InvalidURIError
-        "openai_compatible"
+        uri = parsed_uri(request_url)
+        provider_for_uri(uri) || "openai_compatible"
       end
 
-      def provider_for_host(host)
-        LlmCostTracker.configuration.openai_compatible_providers[host.to_s.downcase]&.to_s
-      end
+      def provider_for_uri(uri)
+        return nil unless uri
 
-      def tracked_path?(path)
-        TRACKED_PATH_SUFFIXES.any? { |suffix| path == suffix || path.end_with?(suffix) }
+        LlmCostTracker.configuration.openai_compatible_providers[uri.host.to_s.downcase]&.to_s
       end
     end
   end

data/lib/llm_cost_tracker/parsers/openai_usage.rb

@@ -34,12 +34,13 @@ module LlmCostTracker
         request = safe_json_parse(request_body)
         model = detect_stream_model(events) || request["model"]
         usage = detect_stream_usage(events)
+        response_id = detect_stream_response_id(events)
 
         if usage
           cache_read = cache_read_input_tokens(usage)
           ParsedUsage.build(
             provider: provider_for(request_url),
-            provider_response_id: detect_stream_response_id(events),
+            provider_response_id: response_id,
             model: model,
             input_tokens: regular_input_tokens(usage, cache_read),
             output_tokens: (usage["completion_tokens"] || usage["output_tokens"]).to_i,
@@ -50,50 +51,27 @@ module LlmCostTracker
             usage_source: :stream_final
           )
         else
-          ParsedUsage.build(
+          build_unknown_stream_usage(
             provider: provider_for(request_url),
-            provider_response_id: detect_stream_response_id(events),
             model: model,
-            input_tokens: 0,
-            output_tokens: 0,
-            total_tokens: 0,
-            stream: true,
-            usage_source: :unknown
+            provider_response_id: response_id
           )
         end
       end
 
       def detect_stream_usage(events)
-        events.reverse_each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
+        find_event_value(events, reverse: true) do |data|
           usage = data["usage"]
-          return usage if usage.is_a?(Hash) && !usage.empty?
+          usage if usage.is_a?(Hash)
         end
-        nil
       end
 
       def detect_stream_model(events)
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
-          model = data["model"]
-          return model if model && !model.to_s.empty?
-        end
-        nil
+        find_event_value(events) { |data| data["model"] || data.dig("response", "model") }
       end
 
       def detect_stream_response_id(events)
-        events.each do |event|
-          data = event[:data]
-          next unless data.is_a?(Hash)
-
-          id = data["id"] || data.dig("response", "id")
-          return id if id && !id.to_s.empty?
-        end
-        nil
+        find_event_value(events) { |data| data["id"] || data.dig("response", "id") }
       end
 
       def regular_input_tokens(usage, cache_read)

data/lib/llm_cost_tracker/parsers/registry.rb

@@ -13,10 +13,14 @@ module LlmCostTracker
         end
 
         def register(parser)
+          parser = coerce_parser(parser)
+
           MUTEX.synchronize do
             current = @parsers || default_parsers.freeze
             @parsers = ([parser] + current).freeze
           end
+
+          parser
         end
 
         def find_for(url)
@@ -24,8 +28,8 @@ module LlmCostTracker
         end
 
         def find_for_provider(provider)
-          provider_name = provider.to_s
-          parsers.find { |parser| parser.provider_names.include?(provider_name) }
+          provider_name = provider.to_s.downcase
+          parsers.find { |parser| provider_names_for(parser).include?(provider_name) }
         end
 
         def reset!
@@ -34,6 +38,17 @@ module LlmCostTracker
 
         private
 
+        def coerce_parser(parser)
+          return parser.new if parser.is_a?(Class) && parser <= Base
+          return parser if parser.is_a?(Base)
+
+          raise ArgumentError, "parser must be a LlmCostTracker::Parsers::Base instance or class"
+        end
+
+        def provider_names_for(parser)
+          Array(parser.provider_names).map { |name| name.to_s.downcase }
+        end
+
         def default_parsers
           [Openai.new, OpenaiCompatible.new, Anthropic.new, Gemini.new]
         end

data/lib/llm_cost_tracker/price_freshness.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "date"
+
+module LlmCostTracker
+  module PriceFreshness
+    STALE_AFTER_DAYS = 30
+
+    class << self
+      def call(metadata, today: Date.today)
+        updated_at = metadata["updated_at"] || metadata[:updated_at]
+        return missing unless updated_at
+
+        date = Date.iso8601(updated_at.to_s)
+        age_days = (today - date).to_i
+        return stale(updated_at) if age_days > STALE_AFTER_DAYS
+
+        [:ok, "updated_at=#{updated_at}"]
+      rescue Date::Error
+        [:warn, "metadata.updated_at=#{updated_at.inspect} is invalid; run bin/rails llm_cost_tracker:prices:sync"]
+      end
+
+      private
+
+      def missing
+        [:warn, "metadata.updated_at missing; run bin/rails llm_cost_tracker:prices:sync"]
+      end
+
+      def stale(updated_at)
+        [
+          :warn,
+          "updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days; " \
+          "run bin/rails llm_cost_tracker:prices:sync"
+        ]
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_registry.rb

@@ -25,6 +25,20 @@ module LlmCostTracker
         @metadata ||= MUTEX.synchronize { @metadata || raw_registry.fetch("metadata", {}).freeze }
       end
 
+      def file_metadata(path)
+        return {} unless path
+
+        registry = load_price_file(path.to_s)
+        raise ArgumentError, "prices_file must be a hash" unless registry.is_a?(Hash)
+
+        metadata = registry.fetch("metadata", {})
+        raise ArgumentError, "prices_file metadata must be a hash" unless metadata.is_a?(Hash)
+
+        metadata
+      rescue Errno::ENOENT, JSON::ParserError, Psych::Exception, ArgumentError, TypeError => e
+        raise Error, "Unable to load prices_file #{path.inspect}: #{e.message}"
+      end
+
       def normalize_price_table(table)
         normalize_price_entries(table, context: "price table")
       end

data/lib/llm_cost_tracker/price_sync/fetcher.rb

@@ -2,6 +2,7 @@
 
 require "digest"
 require "net/http"
+require "openssl"
 require "time"
 require "uri"
 
@@ -52,7 +53,7 @@ module LlmCostTracker
         else
           raise Error, "Unable to fetch #{url}: HTTP #{response.code}"
         end
-      rescue SocketError, SystemCallError, Timeout::Error => e
+      rescue OpenSSL::SSL::SSLError, SocketError, SystemCallError, Timeout::Error => e
         raise Error, "Unable to fetch #{url}: #{e.class}: #{e.message}"
       end
 

data/lib/llm_cost_tracker/price_sync/refresh_plan_builder.rb

@@ -31,7 +31,7 @@ module LlmCostTracker
           ),
           accepted: validated.accepted,
           changes: price_changes(current_models, updated_models),
-          orphaned_models: compute_orphaned(current_models, merged.keys),
+          orphaned_models: compute_orphaned(current_models, merged.keys, source_results),
           failed_sources: failed_sources,
           discrepancies: discrepancies,
           rejected: validated.rejected,
@@ -70,7 +70,9 @@ module LlmCostTracker
         merged.sort.to_h
       end
 
-      def compute_orphaned(current_models, merged_models)
+      def compute_orphaned(current_models, merged_models, source_results)
+        return [] if source_results.empty?
+
         seed_models(current_models).keys.reject do |model|
           manual_model?(current_models[model]) || merged_models.include?(model)
         end.sort

data/lib/llm_cost_tracker/price_sync.rb

@@ -67,6 +67,16 @@ module LlmCostTracker
     end
 
     class << self
+      def configured_output_path(env: ENV, config: LlmCostTracker.configuration)
+        output = env["OUTPUT"].to_s.strip
+        return output unless output.empty?
+
+        prices_file = config.prices_file
+        return prices_file.to_s if prices_file
+
+        nil
+      end
+
       def sync(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, preview: false, strict: false,
                fetcher: Fetcher.new, today: Date.today)
         plan = RefreshPlanBuilder.new(sources: sources).call(