llm_cost_tracker 0.5.0 → 0.5.2

data/lib/llm_cost_tracker/price_sync/registry_diff.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module PriceSync
+    module RegistryDiff
+      class << self
+        def call(current_models, updated_models)
+          current_models = normalize_models(current_models)
+          updated_models = normalize_models(updated_models)
+
+          (current_models.keys | updated_models.keys).sort.each_with_object({}) do |model, changes|
+            fields = price_field_changes(current_models[model], updated_models[model])
+            changes[model] = fields if fields.any?
+          end
+        end
+
+        private
+
+        def price_field_changes(current_entry, updated_entry)
+          current_price = comparable_price(current_entry)
+          updated_price = comparable_price(updated_entry)
+
+          (current_price.keys | updated_price.keys).sort.each_with_object({}) do |field, changes|
+            from = current_price[field]
+            to = updated_price[field]
+            next if from == to
+
+            changes[field] = { "from" => from, "to" => to }
+          end
+        end
+
+        def comparable_price(entry)
+          normalize_hash(entry).slice(*PriceRegistry::PRICE_KEYS)
+        end
+
+        def normalize_models(models)
+          normalize_hash(models).transform_values { |entry| normalize_hash(entry) }
+        end
+
+        def normalize_hash(hash)
+          return {} if hash.nil?
+          raise Error, "pricing entries must be hashes" unless hash.is_a?(Hash)
+
+          hash.each_with_object({}) do |(key, value), normalized|
+            normalized[key.to_s] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/price_sync/registry_loader.rb

@@ -3,6 +3,8 @@
 require "json"
 require "yaml"
 
+require_relative "../price_registry"
+
 module LlmCostTracker
   module PriceSync
     class RegistryLoader
@@ -18,6 +20,10 @@ module LlmCostTracker
       private
 
       def load_registry_file(path)
+        if File.size(path) > PriceRegistry::MAX_FILE_BYTES
+          raise ArgumentError, "pricing registry exceeds #{PriceRegistry::MAX_FILE_BYTES} bytes"
+        end
+
         contents = File.read(path)
         registry = yaml_file?(path) ? (YAML.safe_load(contents, aliases: false) || {}) : JSON.parse(contents)
         raise ArgumentError, "pricing registry must be a hash" unless registry.is_a?(Hash)

data/lib/llm_cost_tracker/price_sync/registry_writer.rb

@@ -12,7 +12,11 @@ module LlmCostTracker
       def call(path:, registry:)
         FileUtils.mkdir_p(File.dirname(path))
         payload = yaml_file?(path) ? YAML.dump(registry) : "#{JSON.pretty_generate(registry)}\n"
-        File.write(path, payload)
+        temp_path = "#{path}.tmp-#{Process.pid}-#{Thread.current.object_id}"
+        File.write(temp_path, payload)
+        File.rename(temp_path, path)
+      ensure
+        FileUtils.rm_f(temp_path) if temp_path && File.exist?(temp_path)
       end
 
       private

data/lib/llm_cost_tracker/price_sync.rb

@@ -1,70 +1,23 @@
 # frozen_string_literal: true
 
 require "date"
+require "json"
+require "rubygems"
 
 require_relative "price_sync/fetcher"
-require_relative "price_sync/raw_price"
-require_relative "price_sync/source"
-require_relative "price_sync/source_result"
+require_relative "price_sync/registry_diff"
 require_relative "price_sync/registry_loader"
 require_relative "price_sync/registry_writer"
-require_relative "price_sync/refresh_plan_builder"
-require_relative "price_sync/model_catalog"
-require_relative "price_sync/merger"
-require_relative "price_sync/validator"
-require_relative "price_sync/sources/litellm"
-require_relative "price_sync/sources/open_router"
 
 module LlmCostTracker
   module PriceSync
-    DEFAULT_OUTPUT_PATH = PriceRegistry::DEFAULT_PRICES_PATH
-
-    SourceUsage = Data.define(:prices_count, :source_version)
-    SyncResult = Data.define(
-      :path,
-      :updated_models,
-      :changes,
-      :orphaned_models,
-      :failed_sources,
-      :discrepancies,
-      :rejected,
-      :flagged,
-      :sources_used,
-      :written
-    )
-    CheckResult = Data.define(
-      :path,
-      :changes,
-      :orphaned_models,
-      :failed_sources,
-      :discrepancies,
-      :rejected,
-      :flagged,
-      :sources_used,
-      :up_to_date
-    )
-    RefreshPlan = Data.define(
-      :path,
-      :registry,
-      :updated_registry,
-      :accepted,
-      :changes,
-      :orphaned_models,
-      :failed_sources,
-      :discrepancies,
-      :rejected,
-      :flagged,
-      :sources_used,
-      :source_results
-    ) do
-      def refresh_succeeded?
-        source_results.any? { |_source, result| result.prices.any? }
-      end
+    DEFAULT_OUTPUT_PATH = "config/llm_cost_tracker_prices.yml"
+    DEFAULT_REMOTE_URL =
+      "https://raw.githubusercontent.com/sergey-homenko/llm_cost_tracker/main/lib/llm_cost_tracker/prices.json"
+    SUPPORTED_SCHEMA_VERSION = 1
 
-      def up_to_date?
-        changes.empty? && failed_sources.empty? && rejected.empty?
-      end
-    end
+    RefreshResult = Data.define(:path, :source_url, :source_version, :changes, :written, :not_modified)
+    CheckResult = Data.define(:path, :source_url, :source_version, :changes, :up_to_date)
 
     class << self
       def configured_output_path(env: ENV, config: LlmCostTracker.configuration)
@@ -74,78 +27,117 @@ module LlmCostTracker
         prices_file = config.prices_file
         return prices_file.to_s if prices_file
 
-        nil
+        default_output_path
       end
 
-      def sync(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, preview: false, strict: false,
-               fetcher: Fetcher.new, today: Date.today)
-        plan = RefreshPlanBuilder.new(sources: sources).call(
-          path: path,
-          seed_path: seed_path,
-          fetcher: fetcher,
-          today: today
-        )
-        raise Error, strict_failure_message(plan) if strict_sync_failure?(plan, strict: strict)
-
-        written = !preview && plan.refresh_succeeded?
-        RegistryWriter.new.call(path: plan.path, registry: plan.updated_registry) if written
-
-        SyncResult.new(
-          path: plan.path,
-          updated_models: plan.changes.keys.sort,
-          changes: plan.changes,
-          orphaned_models: plan.orphaned_models,
-          failed_sources: plan.failed_sources,
-          discrepancies: plan.discrepancies,
-          rejected: plan.rejected,
-          flagged: plan.flagged,
-          sources_used: plan.sources_used,
-          written: written
-        )
+      def configured_remote_url(env: ENV)
+        url = env["URL"].to_s.strip
+        url.empty? ? DEFAULT_REMOTE_URL : url
       end
 
-      def check(path: DEFAULT_OUTPUT_PATH, seed_path: DEFAULT_OUTPUT_PATH, fetcher: Fetcher.new, today: Date.today)
-        plan = RefreshPlanBuilder.new(sources: sources).call(
-          path: path,
-          seed_path: seed_path,
-          fetcher: fetcher,
-          today: today
-        )
+      def refresh(path: DEFAULT_OUTPUT_PATH, url: DEFAULT_REMOTE_URL, preview: false, fetcher: Fetcher.new,
+                  today: Date.today)
+        current = load_current_registry(path)
+        response = fetcher.get(url, etag: current.dig("metadata", "source_version"))
+
+        if response.not_modified
+          return refresh_result(path, url, response, current, current, written: false, not_modified: true)
+        end
+
+        remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
+        RegistryWriter.new.call(path: path, registry: remote) unless preview
+        refresh_result(path, url, response, current, remote, written: !preview, not_modified: false)
+      end
+
+      def check(path: DEFAULT_OUTPUT_PATH, url: DEFAULT_REMOTE_URL, fetcher: Fetcher.new, today: Date.today)
+        current = load_current_registry(path)
+        response = fetcher.get(url, etag: current.dig("metadata", "source_version"))
+
+        if response.not_modified
+          return CheckResult.new(
+            path: path,
+            source_url: url,
+            source_version: response.source_version,
+            changes: {},
+            up_to_date: true
+          )
+        end
+
+        remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
+        changes = RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {}))
 
         CheckResult.new(
-          path: plan.path,
-          changes: plan.changes,
-          orphaned_models: plan.orphaned_models,
-          failed_sources: plan.failed_sources,
-          discrepancies: plan.discrepancies,
-          rejected: plan.rejected,
-          flagged: plan.flagged,
-          sources_used: plan.sources_used,
-          up_to_date: plan.up_to_date?
+          path: path,
+          source_url: url,
+          source_version: response.source_version,
+          changes: changes,
+          up_to_date: changes.empty?
         )
       end
 
       private
 
-      def sources
-        [Sources::Litellm.new, Sources::OpenRouter.new]
+      def default_output_path
+        if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+          Rails.root.join(DEFAULT_OUTPUT_PATH).to_s
+        else
+          DEFAULT_OUTPUT_PATH
+        end
       end
 
-      def strict_sync_failure?(plan, strict:)
-        strict && (plan.failed_sources.any? || plan.rejected.any?)
+      def load_current_registry(path)
+        RegistryLoader.new.call(path: path, seed_path: PriceRegistry::DEFAULT_PRICES_PATH)
       end
 
-      def strict_failure_message(plan)
-        messages = []
-        if plan.failed_sources.any?
-          details = plan.failed_sources.map { |source, message| "#{source}: #{message}" }.join(", ")
-          messages << "source failures: #{details}"
+      def normalize_remote_registry(body, url:, response:, today:)
+        registry = parse_registry(body)
+        metadata = registry.fetch("metadata", {})
+        raise Error, "remote pricing metadata must be a hash" unless metadata.is_a?(Hash)
+
+        schema_version = Integer(metadata.fetch("schema_version", 1))
+        if schema_version > SUPPORTED_SCHEMA_VERSION
+          raise Error, "remote pricing schema_version=#{schema_version} requires a newer llm_cost_tracker"
         end
-        if plan.rejected.any?
-          details = plan.rejected.map { |issue| "#{issue.model} (#{issue.reason})" }.join(", ")
-          messages << "validator rejections: #{details}"
+
+        min_gem_version = metadata["min_gem_version"]
+        if min_gem_version && Gem::Version.new(min_gem_version) > Gem::Version.new(LlmCostTracker::VERSION)
+          raise Error, "remote pricing snapshot requires llm_cost_tracker >= #{min_gem_version}"
         end
-        "Price sync failed in strict mode: #{messages.join('; ')}"
+
+        models = registry.fetch("models", {})
+        PriceRegistry.normalize_price_table(models)
+
+        registry.merge(
+          "metadata" => metadata.merge(
+            "schema_version" => schema_version,
+            "updated_at" => metadata["updated_at"] || today.iso8601,
+            "source_url" => url,
+            "source_version" => response.source_version
+          ),
+          "models" => models
+        )
+      rescue ArgumentError, TypeError => e
+        raise Error, "Unable to load remote pricing snapshot: #{e.message}"
+      end
+
+      def parse_registry(body)
+        registry = JSON.parse(body.to_s)
+        raise Error, "remote pricing snapshot must be a JSON object" unless registry.is_a?(Hash)
+
+        registry
+      rescue JSON::ParserError => e
+        raise Error, "Unable to parse remote pricing snapshot: #{e.message}"
+      end
+
+      def refresh_result(path, url, response, current, remote, written:, not_modified:)
+        RefreshResult.new(
+          path: path,
+          source_url: url,
+          source_version: response.source_version,
+          changes: RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {})),
+          written: written,
+          not_modified: not_modified
+        )
       end
     end
   end