llm_cost_tracker 0.8.0 → 0.10.0

data/lib/llm_cost_tracker/report/data.rb

@@ -2,6 +2,7 @@
 
 require "active_support/core_ext/integer/time"
 
+require_relative "../billing/cost_status"
 require_relative "../ledger"
 
 module LlmCostTracker
@@ -57,7 +58,9 @@ module LlmCostTracker
             "COALESCE(SUM(total_cost), 0) AS total_cost, " \
             "COUNT(*) AS requests_count, " \
             "AVG(latency_ms) AS average_latency_ms, " \
-            "COALESCE(SUM(CASE WHEN total_cost IS NULL THEN 1 ELSE 0 END), 0) AS unknown_pricing_count"
+            "COALESCE(SUM(CASE WHEN total_cost IS NULL " \
+            "OR cost_status IN ('#{Billing::CostStatus::UNKNOWN}', '#{Billing::CostStatus::PARTIAL}') " \
+            "THEN 1 ELSE 0 END), 0) AS unknown_pricing_count"
           )
           .take
       end

data/lib/llm_cost_tracker/report.rb

@@ -15,10 +15,6 @@ module LlmCostTracker
         )
 
         Formatter.new(report_data).to_s
-      rescue LoadError => e
-        "Unable to build LLM cost report: ActiveRecord storage is unavailable (#{e.message})"
-      rescue StandardError => e
-        "Unable to build LLM cost report: #{e.class}: #{e.message}"
       end
     end
   end

data/lib/llm_cost_tracker/retention.rb

@@ -21,6 +21,25 @@ module LlmCostTracker
         deleted
       end
 
+      def prune_invoice_imports(older_than:, now: Time.now.utc)
+        cutoff = resolve_cutoff(older_than, now)
+        require_relative "ledger"
+        return 0 unless LlmCostTracker::ProviderInvoiceImport.table_exists?
+
+        LlmCostTracker::ProviderInvoiceImport
+          .where(state: %w[completed failed])
+          .where(finished_at: ...cutoff)
+          .delete_all
+      end
+
+      def prune_inbox(older_than:, now: Time.now.utc)
+        cutoff = resolve_cutoff(older_than, now)
+        require_relative "ingestion"
+        return 0 unless LlmCostTracker::Ingestion::InboxEntry.table_exists?
+
+        LlmCostTracker::Ingestion::InboxEntry.where(tracked_at: ...cutoff).delete_all
+      end
+
       private
 
       def resolve_cutoff(older_than, now)
@@ -50,18 +69,24 @@ module LlmCostTracker
 
       def prune_batch(cutoff, batch_size)
         LlmCostTracker::Call.transaction do
-          rows = pluck_prunable(cutoff, batch_size)
+          cache_rollups = LlmCostTracker.configuration.cache_rollups
+          rows = pluck_prunable(cutoff, batch_size, with_rollup_columns: cache_rollups)
           next 0 if rows.empty?
 
-          deleted = LlmCostTracker::Call.where(id: rows.map(&:first)).delete_all
-          LlmCostTracker::Ledger::Rollups.decrement!(rows) if deleted.positive?
+          ids = cache_rollups ? rows.map(&:first) : rows
+          deleted = LlmCostTracker::Call.where(id: ids).delete_all
+          LlmCostTracker::Ledger::Rollups.decrement!(rows) if cache_rollups && deleted.positive?
           deleted
         end
       end
 
-      def pluck_prunable(cutoff, batch_size)
-        LlmCostTracker::Call.where(tracked_at: ...cutoff).order(:id).limit(batch_size).lock
-                            .pluck(:id, :tracked_at, :total_cost, :pricing_snapshot)
+      def pluck_prunable(cutoff, batch_size, with_rollup_columns:)
+        relation = LlmCostTracker::Call.where(tracked_at: ...cutoff).order(:id).limit(batch_size).lock
+        if with_rollup_columns
+          relation.pluck(:id, :tracked_at, :total_cost, :pricing_snapshot, :provider)
+        else
+          relation.pluck(:id)
+        end
       end
     end
   end

data/lib/llm_cost_tracker/tags/context.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "active_support/core_ext/object/deep_dup"
 require "active_support/isolated_execution_state"
 
 module LlmCostTracker
@@ -11,7 +10,7 @@ module LlmCostTracker
       class << self
         def with(tags)
           stack = ActiveSupport::IsolatedExecutionState[KEY] || []
-          ActiveSupport::IsolatedExecutionState[KEY] = stack + [(tags || {}).deep_dup.to_h]
+          ActiveSupport::IsolatedExecutionState[KEY] = stack + [Sanitizer.call((tags || {}).to_h)]
           yield
         ensure
           ActiveSupport::IsolatedExecutionState[KEY] = stack
@@ -19,9 +18,9 @@ module LlmCostTracker
 
         def tags
           default_tags = LlmCostTracker.configuration.default_tags
-          default_tags = default_tags.call.deep_dup if default_tags.respond_to?(:call)
+          default_tags = default_tags.call if default_tags.respond_to?(:call)
 
-          default_tags.to_h.merge(*Array(ActiveSupport::IsolatedExecutionState[KEY]))
+          Sanitizer.call(default_tags.to_h).merge(*Array(ActiveSupport::IsolatedExecutionState[KEY]))
         end
 
         def clear!

data/lib/llm_cost_tracker/tags/sanitizer.rb

@@ -1,32 +1,99 @@
 # frozen_string_literal: true
 
 require "active_support/core_ext/string/inflections"
-require "json"
 
 module LlmCostTracker
   module Tags
     module Sanitizer
       REDACTED_VALUE = "[REDACTED]"
 
+      SECRET_VALUE_PATTERNS = [
+        /\Ask-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}\z/,
+        /\AAKIA[0-9A-Z]{16}\z/,
+        /\Agh[opsur]_[A-Za-z0-9]{16,}\z/,
+        /\Agithub_pat_[A-Za-z0-9_]{20,}\z/,
+        /\Aeyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\z/i,
+        /\Abearer\s+[A-Za-z0-9_.-]{20,}\z/i,
+        /\Axox[abprs]-[A-Za-z0-9-]{10,}\z/,
+        /\A(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\z/,
+        /\AAIza[0-9A-Za-z_-]{35}\z/
+      ].freeze
+      private_constant :SECRET_VALUE_PATTERNS
+
       class << self
         def call(tags, config: LlmCostTracker.configuration)
           tags = (tags || {}).to_h
-          redacted = Array(config.redacted_tag_keys).map { |key| normalized_key(key) }
+          redacted = config.normalized_redacted_tag_keys
           limit = [config.max_tag_value_bytesize.to_i, 0].max
-          tags.first([config.max_tag_count.to_i, 0].max).each_with_object({}) do |(key, value), sanitized|
+          max_count = [config.max_tag_count.to_i, 0].max
+          tags.to_a.last(max_count).each_with_object({}) do |(key, value), sanitized|
             sanitized[key] = sanitized_value(key, value, redacted, limit)
           end
         end
 
+        def cap(tags, config: LlmCostTracker.configuration)
+          tags = (tags || {}).to_h
+          max_count = [config.max_tag_count.to_i, 0].max
+          return tags if tags.size <= max_count
+
+          tags.to_a.last(max_count).to_h
+        end
+
+        def normalized_key(key)
+          key.to_s.underscore.gsub(/[^a-z0-9]+/, "_").delete_prefix("_").delete_suffix("_")
+        end
+
         private
 
         def sanitized_value(key, value, redacted, limit)
           return REDACTED_VALUE if redacted_key?(key, redacted)
 
-          string = value_string(value)
-          return value if string.bytesize <= limit
+          scrubbed = scrub_secrets(value)
+          return REDACTED_VALUE if scrubbed.equal?(REDACTED_SENTINEL)
+
+          scalar_truncate(scrubbed, limit)
+        end
+
+        REDACTED_SENTINEL = Object.new.freeze
+        private_constant :REDACTED_SENTINEL
+
+        def scalar_truncate(value, limit)
+          case value
+          when Hash
+            value.transform_values { |nested| scalar_truncate(nested, limit) }
+          when Array
+            value.map { |nested| scalar_truncate(nested, limit) }
+          else
+            return value if value == REDACTED_VALUE
+
+            string = value.to_s
+            return value if string.bytesize <= limit
+
+            string.byteslice(0, limit).encode("UTF-8", invalid: :replace, undef: :replace)
+          end
+        end
 
-          string.byteslice(0, limit).encode("UTF-8", invalid: :replace, undef: :replace)
+        def scrub_secrets(value)
+          case value
+          when Hash
+            value.each_with_object({}) do |(key, nested), out|
+              scrubbed = scrub_secrets(nested)
+              out[key] = scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
+            end
+          when Array
+            value.map do |nested|
+              scrubbed = scrub_secrets(nested)
+              scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
+            end
+          else
+            secret_shaped?(value.to_s) ? REDACTED_SENTINEL : value
+          end
+        end
+
+        def secret_shaped?(string)
+          return false if string.bytesize < 16
+
+          SECRET_VALUE_PATTERNS.any? { |pattern| pattern.match?(string) }
         end
 
         def redacted_key?(key, redacted)
@@ -36,27 +103,12 @@ module LlmCostTracker
           redacted.any? { |candidate| redacted_key_component?(normalized, candidate) }
         end
 
-        def normalized_key(key)
-          key.to_s.underscore.gsub(/[^a-z0-9]+/, "_").delete_prefix("_").delete_suffix("_")
-        end
-
         def redacted_key_component?(key, candidate)
           key == candidate ||
             key.start_with?("#{candidate}_") ||
             key.end_with?("_#{candidate}") ||
             key.include?("_#{candidate}_")
         end
-
-        def value_string(value)
-          case value
-          when Hash, Array
-            JSON.generate(value)
-          else
-            value.to_s
-          end
-        rescue JSON::GeneratorError, TypeError
-          value.to_s
-        end
       end
     end
   end

data/lib/llm_cost_tracker/token_usage.rb

@@ -14,11 +14,17 @@ module LlmCostTracker
     :cache_write_input_tokens,
     :cache_write_extended_input_tokens,
     :audio_input_tokens,
+    :image_input_tokens,
     :output_tokens,
     :audio_output_tokens,
+    :image_output_tokens,
     :total_tokens,
     :hidden_output_tokens
   ) do
+    def priced_quantities
+      Billing::Components::TOKEN_PRICED.to_h { |component| [component.key, public_send(component.token_key)] }
+    end
+
     def self.build_from_tokens(tokens)
       return tokens if tokens.is_a?(self)
       raise ArgumentError, "tokens must be a Hash, got #{tokens.class}" unless tokens.respond_to?(:to_h)
@@ -53,6 +59,7 @@ module LlmCostTracker
     def self.build(input_tokens:, output_tokens:, cache_read_input_tokens: 0,
                    cache_write_input_tokens: 0, cache_write_extended_input_tokens: 0,
                    audio_input_tokens: 0, audio_output_tokens: 0,
+                   image_input_tokens: 0, image_output_tokens: 0,
                    total_tokens: nil, hidden_output_tokens: 0)
       input = non_negative_int(input_tokens)
       output = non_negative_int(output_tokens)
@@ -61,9 +68,12 @@ module LlmCostTracker
       cache_write_extended = non_negative_int(cache_write_extended_input_tokens)
       audio_input = non_negative_int(audio_input_tokens)
       audio_output = non_negative_int(audio_output_tokens)
+      image_input = non_negative_int(image_input_tokens)
+      image_output = non_negative_int(image_output_tokens)
       hidden_output = non_negative_int(hidden_output_tokens)
-      calculated_total = input + cache_read + cache_write + cache_write_extended + audio_input + output + audio_output
-      total = total_tokens.nil? ? calculated_total : [non_negative_int(total_tokens), calculated_total].max
+      calculated_total = input + cache_read + cache_write + cache_write_extended +
+                         audio_input + image_input + output + audio_output + image_output
+      total = total_tokens ? [non_negative_int(total_tokens), calculated_total].max : calculated_total
 
       new(
         input_tokens: input,
@@ -71,8 +81,10 @@ module LlmCostTracker
         cache_write_input_tokens: cache_write,
         cache_write_extended_input_tokens: cache_write_extended,
         audio_input_tokens: audio_input,
+        image_input_tokens: image_input,
         output_tokens: output,
         audio_output_tokens: audio_output,
+        image_output_tokens: image_output,
         total_tokens: total,
         hidden_output_tokens: hidden_output
       )

data/lib/llm_cost_tracker/tracker.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 require "active_support/core_ext/object/blank"
-require "bigdecimal"
 require "securerandom"
 
 require_relative "ingestion"
 require_relative "ledger"
+require_relative "logging"
 require_relative "pricing"
 require_relative "billing/cost_status"
 
@@ -14,38 +14,48 @@ module LlmCostTracker
     EVENT_NAME = "llm_request.llm_cost_tracker"
 
     class << self
-      def enforce_budget!
+      def enforce_budget!(provider: nil, model: nil, request: nil)
         return unless LlmCostTracker.configuration.enabled
 
-        Budget.enforce!
+        Budget.enforce!(provider: provider, model: model, request: request)
       end
 
-      def record(capture:, latency_ms: nil, pricing_mode: nil, metadata: {}, context_tags: nil)
+      def record(event:, latency_ms: nil, pricing_mode: nil, metadata: {}, context_tags: nil)
         return unless LlmCostTracker.configuration.enabled
 
-        pricing_mode = Pricing.normalize_mode(pricing_mode) || capture.pricing_mode
-        cost_data, pricing_snapshot = Pricing.cost_and_snapshot_for(
-          provider: capture.provider,
-          model: capture.model,
-          tokens: capture.token_usage,
+        pricing_mode = Pricing.normalize_mode(pricing_mode) || event.pricing_mode
+        cost_data, pricing_snapshot, priced_line_items = Pricing.calculate(
+          provider: event.provider,
+          model: event.model,
+          tokens: event.token_usage,
+          line_items: event.line_items,
           pricing_mode: pricing_mode
         )
 
-        Pricing::Unknown.handle!(capture.model) if cost_data.nil? && capture.token_usage.total_tokens.positive?
+        if cost_data.nil? && event.token_usage.total_tokens.positive? && priced_line_items.none?(&:priced?)
+          Pricing::Unknown.process(event.model)
+        end
 
         event = build_event(
-          capture: capture,
+          event: event,
           pricing_mode: pricing_mode,
           cost_data: cost_data,
           pricing_snapshot: pricing_snapshot,
+          line_items: priced_line_items,
           metadata: metadata,
           latency_ms: latency_ms,
           context_tags: context_tags
         )
 
-        ActiveSupport::Notifications.instrument(EVENT_NAME, event.to_h)
+        if Ingestion.async?
+          Ingestion::Inbox.save(event)
+          Ingestion::Worker.ensure_started
+        else
+          Ledger::Store.insert(event, skip_existence_check: true)
+        end
 
-        Ingestion::Inbox.save(event)
+        yield if block_given?
+        notify_subscribers(event)
         Budget.check!(event)
 
         event
@@ -53,56 +63,44 @@ module LlmCostTracker
 
       private
 
-      def token_pricing_partial?(token_usage:, cost_data:)
-        return false unless cost_data
+      def notify_subscribers(event)
+        return unless ActiveSupport::Notifications.notifier.listening?(EVENT_NAME)
 
-        Billing::Components::TOKEN_PRICED.any? do |component|
-          token_usage.public_send(component.token_key).positive? && cost_data[component.cost_key].nil?
-        end
+        ActiveSupport::Notifications.instrument(EVENT_NAME, event.to_h)
+      rescue StandardError => e
+        Logging.warn("Subscriber raised on #{EVENT_NAME}: #{e.class}: #{e.message}")
       end
 
-      # rubocop:disable Metrics/MethodLength
-      def build_event(capture:, pricing_mode:, cost_data:, pricing_snapshot:, metadata:, latency_ms:, context_tags:)
+      def build_event(event:, pricing_mode:, cost_data:, pricing_snapshot:, line_items:,
+                      metadata:, latency_ms:, context_tags:)
         context_tags = (context_tags || LlmCostTracker::Tags::Context.tags).to_h
-        line_items, = Pricing.price_line_items(
-          provider: capture.provider,
-          model: capture.model,
-          line_items: capture.line_items,
-          pricing_mode: pricing_mode
-        )
-        cost = cost_with_service_lines(cost_data, line_items)
+        cost = Pricing.combine_with_service_lines(cost_data, line_items)
         cost_status = Billing::CostStatus.call(
-          token_usage: capture.token_usage,
-          usage_source: capture.usage_source,
+          token_usage: event.token_usage,
+          usage_source: event.usage_source,
           token_cost: cost_data,
-          token_pricing_partial: token_pricing_partial?(token_usage: capture.token_usage, cost_data: cost_data),
+          token_pricing_partial: Pricing.token_pricing_partial?(event.token_usage, cost_data),
           service_line_items: line_items.reject(&:token?),
           total_cost: cost&.fetch(:total_cost, nil)
         )
 
-        Event.new(
+        event.with(
           event_id: SecureRandom.uuid,
-          provider: capture.provider,
-          model: capture.model,
-          token_usage: capture.token_usage,
           pricing_mode: pricing_mode,
           cost: cost,
-          tags: LlmCostTracker::Tags::Sanitizer.call(context_tags.merge(metadata.to_h)).freeze,
+          tags: build_tags(context_tags: context_tags, metadata: metadata),
           latency_ms: finite_latency_ms(latency_ms),
-          stream: capture.stream,
-          usage_source: capture.usage_source,
-          provider_response_id: capture.provider_response_id,
-          provider_project_id: capture.provider_project_id,
-          provider_api_key_id: capture.provider_api_key_id,
-          provider_workspace_id: capture.provider_workspace_id,
-          batch: capture.batch,
           tracked_at: Time.now.utc,
           cost_status: cost_status,
           pricing_snapshot: pricing_snapshot,
           line_items: line_items
         )
       end
-      # rubocop:enable Metrics/MethodLength
+
+      def build_tags(context_tags:, metadata:)
+        sanitized_metadata = LlmCostTracker::Tags::Sanitizer.call(metadata.to_h)
+        LlmCostTracker::Tags::Sanitizer.cap(context_tags.merge(sanitized_metadata)).freeze
+      end
 
       def finite_latency_ms(latency_ms)
         return nil if latency_ms.nil?
@@ -111,18 +109,6 @@ module LlmCostTracker
       rescue ArgumentError, TypeError, FloatDomainError
         nil
       end
-
-      def cost_with_service_lines(cost_data, line_items)
-        service_lines = line_items.reject(&:token?)
-        return cost_data if service_lines.empty?
-        return cost_data if service_lines.none?(&:priced?)
-
-        service_total = service_lines.sum(BigDecimal("0"), &:cost_value)
-        cost = cost_data ? cost_data.dup : {}
-        base_total = BigDecimal(cost.fetch(:total_cost, 0).to_s)
-        cost[:total_cost] = (base_total + service_total).round(8).to_f
-        cost
-      end
     end
   end
 end

data/lib/llm_cost_tracker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LlmCostTracker
-  VERSION = "0.8.0"
+  VERSION = "0.10.0"
 end

data/lib/llm_cost_tracker.rb

@@ -16,21 +16,13 @@ require_relative "llm_cost_tracker/logging"
 require_relative "llm_cost_tracker/tags/key"
 require_relative "llm_cost_tracker/tags/context"
 require_relative "llm_cost_tracker/tags/sanitizer"
+require_relative "llm_cost_tracker/masking"
 require_relative "llm_cost_tracker/token_usage"
 require_relative "llm_cost_tracker/billing/components"
 require_relative "llm_cost_tracker/billing/line_item"
 require_relative "llm_cost_tracker/billing/cost_status"
 require_relative "llm_cost_tracker/event"
 require_relative "llm_cost_tracker/pricing"
-require_relative "llm_cost_tracker/usage_capture"
-require_relative "llm_cost_tracker/pricing/sync"
-require_relative "llm_cost_tracker/parsers/base"
-require_relative "llm_cost_tracker/parsers/openai_usage"
-require_relative "llm_cost_tracker/parsers/openai"
-require_relative "llm_cost_tracker/parsers/openai_compatible"
-require_relative "llm_cost_tracker/parsers/anthropic"
-require_relative "llm_cost_tracker/parsers/gemini"
-require_relative "llm_cost_tracker/parsers/sse"
 require_relative "llm_cost_tracker/parsers"
 require_relative "llm_cost_tracker/middleware/faraday"
 require_relative "llm_cost_tracker/integrations"
@@ -39,12 +31,19 @@ require_relative "llm_cost_tracker/pricing/unknown"
 require_relative "llm_cost_tracker/ledger"
 require_relative "llm_cost_tracker/ingestion"
 require_relative "llm_cost_tracker/tracker"
-require_relative "llm_cost_tracker/retention"
-require_relative "llm_cost_tracker/report"
-require_relative "llm_cost_tracker/doctor"
-require_relative "llm_cost_tracker/doctor/capture_verifier"
 
 module LlmCostTracker
+  autoload :Engine,         "llm_cost_tracker/engine"
+  autoload :Reconciliation, "llm_cost_tracker/reconciliation"
+  autoload :ReconcileTasks, "llm_cost_tracker/reconcile_tasks"
+  autoload :Doctor,         "llm_cost_tracker/doctor"
+  autoload :Report,         "llm_cost_tracker/report"
+  autoload :Retention,      "llm_cost_tracker/retention"
+
+  module Pricing
+    autoload :Sync, "llm_cost_tracker/pricing/sync"
+  end
+
   @configuration = Configuration.new
 
   class << self
@@ -54,6 +53,10 @@ module LlmCostTracker
       "llm_cost_tracker_"
     end
 
+    def reconciliation_enabled?
+      configuration.reconciliation_enabled
+    end
+
     def configure
       config = configuration
       raise Error, "LlmCostTracker is already configured" if config.finalized?
@@ -69,6 +72,7 @@ module LlmCostTracker
 
     def reset_configuration!
       Ingestion::Worker.shutdown!(drain: false)
+      Ingestion::Pool.reset!
       @configuration = Configuration.new
       Pricing::Lookup.reset!
       Pricing::Registry.reset!
@@ -76,6 +80,7 @@ module LlmCostTracker
       Pricing::Unknown.reset!
       Ingestion::Worker.reset!
       Tags::Context.clear!
+      Dashboard::SetupState.reset!
     end
 
     def with_tags(tags = nil, **kwargs, &)
@@ -89,7 +94,7 @@ module LlmCostTracker
       Tracker.enforce_budget! if enforce_budget
 
       Tracker.record(
-        capture: UsageCapture.build(
+        event: Event.build(
           provider: provider,
           model: model,
           token_usage: TokenUsage.build_from_tokens(tokens),

data/lib/tasks/llm_cost_tracker.rake

@@ -4,14 +4,21 @@ require "fileutils"
 require "rails/generators"
 
 require_relative "../llm_cost_tracker/generators/llm_cost_tracker/install_generator"
-require_relative "../llm_cost_tracker/pricing/sync_change_printer"
+require_relative "../llm_cost_tracker/pricing/sync/change_printer"
 
 # rubocop:disable Metrics/BlockLength
 namespace :llm_cost_tracker do
   desc "Install LLM Cost Tracker with dashboard and prices, migrate, and run doctor"
   task :setup do
-    Rails::Generators.invoke("llm_cost_tracker:install", %w[--dashboard --prices])
-    Rake::Task["db:migrate"].invoke
+    Rails::Generators.invoke("llm_cost_tracker:install", %w[--dashboard --prices --skip])
+    begin
+      Rake::Task["db:migrate"].invoke
+    rescue ActiveRecord::NoDatabaseError, ActiveRecord::ConnectionNotEstablished => e
+      abort(
+        "llm_cost_tracker: database is not reachable (#{e.class}). " \
+        "Start your database, run 'rails db:create db:migrate', then re-run 'rails llm_cost_tracker:setup'."
+      )
+    end
     Rake::Task["llm_cost_tracker:doctor"].invoke
   end
 
@@ -41,12 +48,26 @@ namespace :llm_cost_tracker do
     puts LlmCostTracker::Report.generate(days: days)
   end
 
+  desc "Recompute total_cost for calls with unknown pricing using the current price registry. " \
+       "Use BATCH_SIZE=N to tune."
+  task backfill_unknown_pricing: :environment do
+    require_relative "../llm_cost_tracker/pricing/backfill"
+    batch_size = (ENV["BATCH_SIZE"] || LlmCostTracker::Pricing::Backfill::DEFAULT_BATCH_SIZE).to_i
+    result = LlmCostTracker::Pricing::Backfill.call(batch_size: batch_size)
+    puts "llm_cost_tracker: examined #{result.examined} calls, recomputed #{result.recomputed}, " \
+         "still unknown #{result.still_unknown}"
+  end
+
   desc "Delete llm_cost_tracker_calls older than DAYS (default: 90). Use BATCH_SIZE=N to tune."
   task prune: :environment do
     days = (ENV["DAYS"] || 90).to_i
     batch_size = (ENV["BATCH_SIZE"] || LlmCostTracker::Retention::DEFAULT_BATCH_SIZE).to_i
     deleted = LlmCostTracker::Retention.prune(older_than: days, batch_size: batch_size)
     puts "llm_cost_tracker: pruned #{deleted} calls older than #{days} days"
+    inbox_pruned = LlmCostTracker::Retention.prune_inbox(older_than: days)
+    puts "llm_cost_tracker: pruned #{inbox_pruned} inbox entries older than #{days} days"
+    invoice_imports_pruned = LlmCostTracker::Retention.prune_invoice_imports(older_than: days)
+    puts "llm_cost_tracker: pruned #{invoice_imports_pruned} provider invoice imports older than #{days} days"
   end
 
   namespace :prices do
@@ -109,11 +130,27 @@ namespace :llm_cost_tracker do
       abort("llm_cost_tracker: price is incomplete or unknown") unless explanation.complete?
     end
   end
+
+  namespace :reconcile do
+    desc "Import provider invoice rows from a JSON INPUT file. " \
+         "Use SOURCE=openai INPUT=path/to/file.json. Pass PROVIDER=openai for unmapped sources (csv, ...)."
+    task(:import) { reconcile_run(:run_import) }
+
+    desc "Print a reconciliation diff. " \
+         "Use SOURCE=openai PERIOD_START=YYYY-MM-DD PERIOD_END=YYYY-MM-DD. PROVIDER=openai for unmapped sources."
+    task(:diff) { reconcile_run(:run_diff) }
+  end
 end
 # rubocop:enable Metrics/BlockLength
 
+def reconcile_run(method)
+  Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
+  require_relative "../llm_cost_tracker"
+  LlmCostTracker::ReconcileTasks.public_send(method, env: ENV)
+end
+
 def print_changes(changes)
-  LlmCostTracker::Pricing::SyncChangePrinter.call(changes)
+  LlmCostTracker::Pricing::Sync::ChangePrinter.call(changes)
 end
 
 def price_refresh_output_path