llm_cost_tracker 0.5.0 → 0.5.1

data/lib/llm_cost_tracker/doctor.rb

@@ -160,7 +160,7 @@ module LlmCostTracker
     def feature_generators(columns) = columns.map { |column| FEATURE_COLUMNS.fetch(column) }.uniq
 
     def builtin_prices_updated_at
-      LlmCostTracker::Pricing.metadata.fetch("updated_at", "unknown")
+      LlmCostTracker::PriceRegistry.metadata.fetch("updated_at", "unknown")
     end
   end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb

@@ -33,7 +33,7 @@ LlmCostTracker.configure do |config|
 <% if options[:prices] -%>
 
   # Local JSON/YAML pricing file generated by --prices. Keep it in source control
-  # and refresh it with bin/rails llm_cost_tracker:prices:sync.
+  # and refresh it with bin/rails llm_cost_tracker:prices:refresh.
   config.prices_file = Rails.root.join("config/llm_cost_tracker_prices.yml")
 <% end -%>
 

data/lib/llm_cost_tracker/middleware/faraday.rb

@@ -78,8 +78,8 @@ module LlmCostTracker
         unless response_body
           Logging.warn(
             "Unable to read response body for #{RequestUrl.label(request_url)}; " \
-            "streaming responses are captured automatically for OpenAI/Anthropic/Gemini " \
-            "or via LlmCostTracker.track_stream for custom clients."
+            "known streaming responses are captured automatically, or via LlmCostTracker.track_stream " \
+            "for custom clients."
           )
           return nil
         end

data/lib/llm_cost_tracker/price_freshness.rb

@@ -17,20 +17,20 @@ module LlmCostTracker
 
         [:ok, "updated_at=#{updated_at}"]
       rescue Date::Error
-        [:warn, "metadata.updated_at=#{updated_at.inspect} is invalid; run bin/rails llm_cost_tracker:prices:sync"]
+        [:warn, "metadata.updated_at=#{updated_at.inspect} is invalid; run bin/rails llm_cost_tracker:prices:refresh"]
       end
 
       private
 
       def missing
-        [:warn, "metadata.updated_at missing; run bin/rails llm_cost_tracker:prices:sync"]
+        [:warn, "metadata.updated_at missing; run bin/rails llm_cost_tracker:prices:refresh"]
       end
 
       def stale(updated_at)
         [
           :warn,
           "updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days; " \
-          "run bin/rails llm_cost_tracker:prices:sync"
+          "run bin/rails llm_cost_tracker:prices:refresh"
         ]
       end
     end

data/lib/llm_cost_tracker/price_sync/fetcher.rb

@@ -15,7 +15,7 @@ module LlmCostTracker
         end
       end
 
-      USER_AGENT = "llm_cost_tracker price sync"
+      USER_AGENT = "llm_cost_tracker price refresh"
       MAX_REDIRECTS = 5
       OPEN_TIMEOUT = 5
       READ_TIMEOUT = 10
@@ -25,6 +25,8 @@ module LlmCostTracker
         raise Error, "Too many redirects while fetching #{url}" if redirects > MAX_REDIRECTS
 
         uri = URI.parse(url)
+        raise Error, "Pricing snapshot URL must use http or https" unless %w[http https].include?(uri.scheme)
+
         request = Net::HTTP::Get.new(uri)
         request["User-Agent"] = USER_AGENT
         request["If-None-Match"] = etag if etag

data/lib/llm_cost_tracker/price_sync/registry_diff.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    module RegistryDiff
+      class << self
+        def call(current_models, updated_models)
+          current_models = normalize_models(current_models)
+          updated_models = normalize_models(updated_models)
+
+          (current_models.keys | updated_models.keys).sort.each_with_object({}) do |model, changes|
+            fields = price_field_changes(current_models[model], updated_models[model])
+            changes[model] = fields if fields.any?
+          end
+        end
+
+        private
+
+        def price_field_changes(current_entry, updated_entry)
+          current_price = comparable_price(current_entry)
+          updated_price = comparable_price(updated_entry)
+
+          (current_price.keys | updated_price.keys).sort.each_with_object({}) do |field, changes|
+            from = current_price[field]
+            to = updated_price[field]
+            next if from == to
+
+            changes[field] = { "from" => from, "to" => to }
+          end
+        end
+
+        def comparable_price(entry)
+          normalize_hash(entry).slice(*PriceRegistry::PRICE_KEYS)
+        end
+
+        def normalize_models(models)
+          normalize_hash(models).transform_values { |entry| normalize_hash(entry) }
+        end
+
+        def normalize_hash(hash)
+          return {} if hash.nil?
+          raise Error, "pricing entries must be hashes" unless hash.is_a?(Hash)
+
+          hash.each_with_object({}) do |(key, value), normalized|
+            normalized[key.to_s] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/registry_writer.rb

@@ -12,7 +12,11 @@ module LlmCostTracker
       def call(path:, registry:)
         FileUtils.mkdir_p(File.dirname(path))
         payload = yaml_file?(path) ? YAML.dump(registry) : "#{JSON.pretty_generate(registry)}\n"
-        File.write(path, payload)
+        temp_path = "#{path}.tmp-#{Process.pid}-#{Thread.current.object_id}"
+        File.write(temp_path, payload)
+        File.rename(temp_path, path)
+      ensure
+        FileUtils.rm_f(temp_path) if temp_path && File.exist?(temp_path)
       end
 
       private

data/lib/llm_cost_tracker/price_sync.rb

@@ -1,70 +1,23 @@
 # frozen_string_literal: true
 
 require "date"
+require "json"
+require "rubygems"
 
 require_relative "price_sync/fetcher"
-require_relative "price_sync/raw_price"
-require_relative "price_sync/source"
-require_relative "price_sync/source_result"
+require_relative "price_sync/registry_diff"
 require_relative "price_sync/registry_loader"
 require_relative "price_sync/registry_writer"
-require_relative "price_sync/refresh_plan_builder"
-require_relative "price_sync/model_catalog"
-require_relative "price_sync/merger"
-require_relative "price_sync/validator"
-require_relative "price_sync/sources/litellm"
-require_relative "price_sync/sources/open_router"
 
 module LlmCostTracker
   module PriceSync
-    DEFAULT_OUTPUT_PATH = PriceRegistry::DEFAULT_PRICES_PATH
-
-    SourceUsage = Data.define(:prices_count, :source_version)
-    SyncResult = Data.define(
-      :path,
-      :updated_models,
-      :changes,
-      :orphaned_models,
-      :failed_sources,
-      :discrepancies,
-      :rejected,
-      :flagged,
-      :sources_used,
-      :written
-    )
-    CheckResult = Data.define(
-      :path,
-      :changes,
-      :orphaned_models,
-      :failed_sources,
-      :discrepancies,
-      :rejected,
-      :flagged,
-      :sources_used,
-      :up_to_date
-    )
-    RefreshPlan = Data.define(
-      :path,
-      :registry,
-      :updated_registry,
-      :accepted,
-      :changes,
-      :orphaned_models,
-      :failed_sources,
-      :discrepancies,
-      :rejected,
-      :flagged,
-      :sources_used,
-      :source_results
-    ) do
-      def refresh_succeeded?
-        source_results.any? { |_source, result| result.prices.any? }
-      end
+    DEFAULT_OUTPUT_PATH = "config/llm_cost_tracker_prices.yml"
+    DEFAULT_REMOTE_URL =
+      "https://raw.githubusercontent.com/sergey-homenko/llm_cost_tracker/main/lib/llm_cost_tracker/prices.json"
+    SUPPORTED_SCHEMA_VERSION = 1
 
-      def up_to_date?
-        changes.empty? && failed_sources.empty? && rejected.empty?
-      end
-    end
+    RefreshResult = Data.define(:path, :source_url, :source_version, :changes, :written, :not_modified)
+    CheckResult = Data.define(:path, :source_url, :source_version, :changes, :up_to_date)
 
     class << self
       def configured_output_path(env: ENV, config: LlmCostTracker.configuration)
@@ -74,78 +27,117 @@ module LlmCostTracker
         prices_file = config.prices_file
         return prices_file.to_s if prices_file
 
-        nil
+        default_output_path
       end
 
-      def sync(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, preview: false, strict: false,
-               fetcher: Fetcher.new, today: Date.today)
-        plan = RefreshPlanBuilder.new(sources: sources).call(
-          path: path,
-          seed_path: seed_path,
-          fetcher: fetcher,
-          today: today
-        )
-        raise Error, strict_failure_message(plan) if strict_sync_failure?(plan, strict: strict)
-
-        written = !preview && plan.refresh_succeeded?
-        RegistryWriter.new.call(path: plan.path, registry: plan.updated_registry) if written
-
-        SyncResult.new(
-          path: plan.path,
-          updated_models: plan.changes.keys.sort,
-          changes: plan.changes,
-          orphaned_models: plan.orphaned_models,
-          failed_sources: plan.failed_sources,
-          discrepancies: plan.discrepancies,
-          rejected: plan.rejected,
-          flagged: plan.flagged,
-          sources_used: plan.sources_used,
-          written: written
-        )
+      def configured_remote_url(env: ENV)
+        url = env["URL"].to_s.strip
+        url.empty? ? DEFAULT_REMOTE_URL : url
       end
 
-      def check(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, fetcher: Fetcher.new, today: Date.today)
-        plan = RefreshPlanBuilder.new(sources: sources).call(
-          path: path,
-          seed_path: seed_path,
-          fetcher: fetcher,
-          today: today
-        )
+      def refresh(path: DEFAULT_OUTPUT_PATH, url: DEFAULT_REMOTE_URL, preview: false, fetcher: Fetcher.new,
+                  today: Date.today)
+        current = load_current_registry(path)
+        response = fetcher.get(url, etag: current.dig("metadata", "source_version"))
+
+        if response.not_modified
+          return refresh_result(path, url, response, current, current, written: false, not_modified: true)
+        end
+
+        remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
+        RegistryWriter.new.call(path: path, registry: remote) unless preview
+        refresh_result(path, url, response, current, remote, written: !preview, not_modified: false)
+      end
+
+      def check(path: DEFAULT_OUTPUT_PATH, url: DEFAULT_REMOTE_URL, fetcher: Fetcher.new, today: Date.today)
+        current = load_current_registry(path)
+        response = fetcher.get(url, etag: current.dig("metadata", "source_version"))
+
+        if response.not_modified
+          return CheckResult.new(
+            path: path,
+            source_url: url,
+            source_version: response.source_version,
+            changes: {},
+            up_to_date: true
+          )
+        end
+
+        remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
+        changes = RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {}))
 
         CheckResult.new(
-          path: plan.path,
-          changes: plan.changes,
-          orphaned_models: plan.orphaned_models,
-          failed_sources: plan.failed_sources,
-          discrepancies: plan.discrepancies,
-          rejected: plan.rejected,
-          flagged: plan.flagged,
-          sources_used: plan.sources_used,
-          up_to_date: plan.up_to_date?
+          path: path,
+          source_url: url,
+          source_version: response.source_version,
+          changes: changes,
+          up_to_date: changes.empty?
         )
       end
 
       private
 
-      def sources
-        [Sources::Litellm.new, Sources::OpenRouter.new]
+      def default_output_path
+        if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+          Rails.root.join(DEFAULT_OUTPUT_PATH).to_s
+        else
+          DEFAULT_OUTPUT_PATH
+        end
       end
 
-      def strict_sync_failure?(plan, strict:)
-        strict && (plan.failed_sources.any? || plan.rejected.any?)
+      def load_current_registry(path)
+        RegistryLoader.new.call(path: path, seed_path: PriceRegistry::DEFAULT_PRICES_PATH)
       end
 
-      def strict_failure_message(plan)
-        messages = []
-        if plan.failed_sources.any?
-          details = plan.failed_sources.map { |source, message| "#{source}: #{message}" }.join(", ")
-          messages << "source failures: #{details}"
+      def normalize_remote_registry(body, url:, response:, today:)
+        registry = parse_registry(body)
+        metadata = registry.fetch("metadata", {})
+        raise Error, "remote pricing metadata must be a hash" unless metadata.is_a?(Hash)
+
+        schema_version = Integer(metadata.fetch("schema_version", 1))
+        if schema_version > SUPPORTED_SCHEMA_VERSION
+          raise Error, "remote pricing schema_version=#{schema_version} requires a newer llm_cost_tracker"
         end
-        if plan.rejected.any?
-          details = plan.rejected.map { |issue| "#{issue.model} (#{issue.reason})" }.join(", ")
-          messages << "validator rejections: #{details}"
+
+        min_gem_version = metadata["min_gem_version"]
+        if min_gem_version && Gem::Version.new(min_gem_version) > Gem::Version.new(LlmCostTracker::VERSION)
+          raise Error, "remote pricing snapshot requires llm_cost_tracker >= #{min_gem_version}"
         end
-        "Price sync failed in strict mode: #{messages.join('; ')}"
+
+        models = registry.fetch("models", {})
+        PriceRegistry.normalize_price_table(models)
+
+        registry.merge(
+          "metadata" => metadata.merge(
+            "schema_version" => schema_version,
+            "updated_at" => metadata["updated_at"] || today.iso8601,
+            "source_url" => url,
+            "source_version" => response.source_version
+          ),
+          "models" => models
+        )
+      rescue ArgumentError, TypeError => e
+        raise Error, "Unable to load remote pricing snapshot: #{e.message}"
+      end
+
+      def parse_registry(body)
+        registry = JSON.parse(body.to_s)
+        raise Error, "remote pricing snapshot must be a JSON object" unless registry.is_a?(Hash)
+
+        registry
+      rescue JSON::ParserError => e
+        raise Error, "Unable to parse remote pricing snapshot: #{e.message}"
+      end
+
+      def refresh_result(path, url, response, current, remote, written:, not_modified:)
+        RefreshResult.new(
+          path: path,
+          source_url: url,
+          source_version: response.source_version,
+          changes: RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {})),
+          written: written,
+          not_modified: not_modified
+        )
       end
     end
   end