llm_cost_tracker 0.7.3 → 0.9.0

data/lib/llm_cost_tracker/doctor/ingestion_check.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "check"
+require_relative "probe"
 require_relative "../ingestion"
 
 module LlmCostTracker
@@ -9,66 +10,83 @@ module LlmCostTracker
       PENDING_AGE_WARNING_SECONDS = 60
 
       def call
-        return unless table_exists?("llm_api_calls")
+        return unless Probe.table_exists?("llm_cost_tracker_calls")
+        return inline_check unless LlmCostTracker::Ingestion.durable?
 
         missing = missing_parts
         if missing.empty?
-          quarantined = quarantined_count
+          inbox = inbox_snapshot
+          quarantined = inbox.try(:quarantined_count).to_i
           if quarantined.positive?
-            return Check.new(:warn, "durable ingestion", "#{quarantined} inbox events quarantined after retries")
+            return Check.new(:warn, "durable ingestion", "#{quarantined} inbox entries quarantined after retries")
           end
 
-          pending = pending_snapshot
-          pending_count = pending.try(:pending_count).to_i
-          oldest_pending_at = pending.try(:oldest_created_at)&.to_time&.utc
+          pending_count = inbox.try(:pending_count).to_i
+          oldest_pending_at = inbox.try(:oldest_pending_at)&.to_time&.utc
           pending_age = oldest_pending_at && (Time.now.utc - oldest_pending_at)
           if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
             return Check.new(
               :warn,
               "durable ingestion",
-              "#{pending_count} inbox events pending; oldest pending age #{pending_age.round}s"
+              "#{pending_count} inbox entries pending; oldest pending age #{pending_age.round}s"
             )
           end
 
-          return Check.new(:ok, "durable ingestion", "inbox and ingestor lease tables available")
+          return Check.new(:ok, "durable ingestion", "inbox and ingestion lease tables available")
         end
 
         Check.new(
           :error,
           "durable ingestion",
-          "missing #{missing.join(', ')}; run bin/rails generate llm_cost_tracker:add_ingestion && bin/rails db:migrate"
+          "missing #{missing.join(', ')}; see docs/upgrading.md for the recovery steps"
         )
       end
 
       private
 
-      def missing_parts
-        [
-          table_exists?("llm_cost_tracker_inbox_events") ? nil : "llm_cost_tracker_inbox_events",
-          table_exists?("llm_cost_tracker_ingestor_leases") ? nil : "llm_cost_tracker_ingestor_leases"
-        ].compact
-      end
+      def inline_check
+        leftovers = inline_leftover_tables
+        if leftovers.empty?
+          return Check.new(
+            :ok,
+            "inline ingestion",
+            "durable_ingestion=false; events write directly to the ledger"
+          )
+        end
 
-      def table_exists?(name)
-        LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
-      rescue StandardError
-        false
+        Check.new(
+          :warn,
+          "inline ingestion",
+          "durable_ingestion=false but found unused durable ingestion tables: #{leftovers.join(', ')}. " \
+          "Set config.durable_ingestion = true to keep the durable inbox path or drop the tables."
+        )
       end
 
-      def quarantined_count
-        return 0 unless table_exists?("llm_cost_tracker_inbox_events")
+      def inline_leftover_tables
+        [
+          LlmCostTracker::Ingestion::InboxEntry.table_name,
+          LlmCostTracker::Ingestion::Lease.table_name
+        ].select { |table| Probe.table_exists?(table) }
+      end
 
-        LlmCostTracker::Ingestion::Event
-          .where("attempts >= ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
-          .count
-      rescue StandardError
-        0
+      def missing_parts
+        [
+          LlmCostTracker::Ingestion::InboxEntry.table_name,
+          LlmCostTracker::Ingestion::Lease.table_name
+        ].reject { |table| Probe.table_exists?(table) }
       end
 
-      def pending_snapshot
-        LlmCostTracker::Ingestion::Event
-          .where("attempts < ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
-          .select("COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at")
+      def inbox_snapshot
+        max_attempts = LlmCostTracker::Ingestion::InboxEntry::MAX_ATTEMPTS_BEFORE_QUARANTINE
+        LlmCostTracker::Ingestion::InboxEntry
+          .select(
+            "COALESCE(SUM(CASE WHEN attempts >= #{max_attempts} " \
+            "THEN 1 ELSE 0 END), 0) AS quarantined_count, " \
+            "COALESCE(SUM(CASE WHEN attempts < #{max_attempts} " \
+            "THEN 1 ELSE 0 END), 0) AS pending_count, " \
+            "MIN(CASE WHEN attempts < #{max_attempts} " \
+            "THEN created_at ELSE NULL END) AS oldest_pending_at"
+          )
           .take
       rescue StandardError
         nil

data/lib/llm_cost_tracker/doctor/invoice_reconciliation_check.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+require_relative "check"
+require_relative "probe"
+require_relative "../ledger/schema/adapter"
+
+module LlmCostTracker
+  class Doctor
+    class InvoiceReconciliationCheck
+      def call
+        return unless LlmCostTracker.reconciliation_enabled?
+        return unless Probe.table_exists?("llm_cost_tracker_provider_invoices")
+        return if no_imports?
+
+        scopes = imported_scopes
+        return Check.new(:ok, "invoice reconciliation", "no provider invoices imported yet") if scopes.empty?
+
+        non_canonical = non_canonical_currency_check
+        checks = scopes.map { |scope| check_scope_safely(scope) }
+        checks << non_canonical if non_canonical
+        checks
+      rescue StandardError => e
+        Check.new(:error, "invoice reconciliation", e.message)
+      end
+
+      private
+
+      def no_imports?
+        LlmCostTracker::ProviderInvoice.none?
+      end
+
+      def non_canonical_currency_check
+        legacy = LlmCostTracker::ProviderInvoice.where("currency <> UPPER(currency)").count
+        return nil if legacy.zero?
+
+        Check.new(
+          :warn,
+          "invoice reconciliation: currency canonicalisation",
+          "#{legacy} provider invoice row(s) stored with non-uppercase currency. Diff queries " \
+          "are case-sensitive — run " \
+          "`UPDATE llm_cost_tracker_provider_invoices SET currency = UPPER(currency);` to backfill."
+        )
+      end
+
+      def threshold
+        Reconciliation::DEFAULT_THRESHOLD_PERCENT
+      end
+
+      def imported_scopes
+        connection = LlmCostTracker::ProviderInvoice.connection
+        provider_expr =
+          if Ledger::Schema::Adapter.postgresql?(connection)
+            Arel.sql("metadata->>'provider'")
+          else
+            Arel.sql("JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.provider'))")
+          end
+        LlmCostTracker::ProviderInvoice
+          .group(:source, provider_expr, :currency)
+          .order(:source, :currency)
+          .pluck(:source, provider_expr, :currency)
+          .map { |source, provider, currency| { source: source, provider: provider, currency: currency.upcase } }
+      end
+
+      def scope_label(scope)
+        "#{scope[:source]}/#{scope[:provider]}/#{scope[:currency]}"
+      end
+
+      def check_scope_safely(scope)
+        check_scope(scope)
+      rescue ArgumentError => e
+        Check.new(:warn, "invoice reconciliation: #{scope_label(scope)}", e.message)
+      end
+
+      def check_scope(scope)
+        window = latest_window_for(scope)
+        return stale_check(scope) if window.nil?
+
+        diff = run_diff(scope, window)
+        return ok_check(scope, window, diff) if diff.aligned?(threshold_percent: threshold)
+
+        warn_check(scope, window, diff)
+      end
+
+      def scope_relation(scope)
+        relation = LlmCostTracker::ProviderInvoice
+                   .where(source: scope[:source], currency: scope[:currency])
+        provider = scope[:provider]
+        return relation if provider.nil? || provider.to_s.empty?
+
+        connection = LlmCostTracker::ProviderInvoice.connection
+        if Ledger::Schema::Adapter.postgresql?(connection)
+          relation.where("metadata->>'provider' = ?", provider)
+        else
+          relation.where("JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.provider')) = ?", provider)
+        end
+      end
+
+      def latest_window_for(scope)
+        latest = scope_relation(scope)
+                 .select(:period_start, :period_end)
+                 .order(period_end: :desc, period_start: :desc)
+                 .limit(1)
+                 .first
+        return nil unless latest
+        return nil if (Time.now.utc.to_date - latest.period_end).to_i > Reconciliation::INVOICE_FRESHNESS_DAYS
+
+        latest
+      end
+
+      def run_diff(scope, window)
+        Reconciliation.diff(
+          source: scope[:source],
+          provider: scope[:provider],
+          currency: scope[:currency],
+          period_start: window.period_start,
+          period_end: window.period_end
+        )
+      end
+
+      def stale_check(scope)
+        latest = scope_relation(scope).maximum(:period_end)
+        return scope_unreachable_check(scope) if latest.nil?
+
+        days = (Time.now.utc.to_date - latest).to_i
+        Check.new(
+          :warn,
+          "invoice reconciliation: #{scope_label(scope)}",
+          "no invoice imported in #{days} days (threshold #{Reconciliation::INVOICE_FRESHNESS_DAYS} days); " \
+          "run reconciliation import"
+        )
+      end
+
+      def scope_unreachable_check(scope)
+        Check.new(
+          :warn,
+          "invoice reconciliation: #{scope_label(scope)}",
+          "scope grouped invoices but the filter (likely currency case mismatch) matches zero rows; " \
+          "the currency-canonicalisation check below points at the backfill SQL"
+        )
+      end
+
+      def ok_check(scope, window, diff)
+        Check.new(
+          :ok,
+          "invoice reconciliation: #{scope_label(scope)}",
+          "#{window.period_start}..#{window.period_end} aligned " \
+          "(local=#{diff.local_total.to_s('F')}, provider=#{diff.provider_total.to_s('F')})"
+        )
+      end
+
+      def warn_check(scope, window, diff)
+        Check.new(
+          :warn,
+          "invoice reconciliation: #{scope_label(scope)}",
+          "#{window.period_start}..#{window.period_end} drift " \
+          "delta=#{diff.delta_amount.to_s('F')} (#{diff.delta_percent}%) " \
+          "exceeds #{threshold}% threshold"
+        )
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "check"
+require_relative "probe"
+require_relative "../ledger"
+
+module LlmCostTracker
+  class Doctor
+    class LegacyAuditCheck
+      WARNING_PERCENT = 10
+
+      def call
+        return unless Probe.table_exists?("llm_cost_tracker_calls")
+        return unless LlmCostTracker::Call.column_names.include?("pricing_snapshot")
+
+        counts = LlmCostTracker::Call
+                 .select(
+                   "COUNT(*) AS total_count, " \
+                   "COALESCE(SUM(CASE WHEN pricing_snapshot IS NULL THEN 1 ELSE 0 END), 0) AS missing_count"
+                 )
+                 .take
+        total = counts.total_count.to_i
+        return if total.zero?
+
+        missing = counts.missing_count.to_i
+        return unless (missing * 100) > (total * WARNING_PERCENT)
+
+        message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
+                  "stored totals remain stable but applied rates cannot be audited"
+        Check.new(:warn, "pricing snapshot audit", message)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "check"
+require_relative "probe"
+require_relative "../ledger"
+
+module LlmCostTracker
+  class Doctor
+    class LegacyBillingStatusCheck
+      def call
+        return unless Probe.table_exists?("llm_cost_tracker_calls")
+        return unless LlmCostTracker::Call.column_names.include?("cost_status")
+
+        return unless LlmCostTracker::Call.where(cost_status: nil).exists?
+
+        Check.new(:warn, "cost status", "legacy rows without cost_status remain; new rows will populate it")
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/price_check.rb

@@ -8,7 +8,7 @@ module LlmCostTracker
   class Doctor
     class PriceCheck
       STALE_AFTER_DAYS = 30
-      REFRESH_COMMAND = "run bin/rails llm_cost_tracker:prices:refresh"
+      REFRESH_COMMAND = "refresh the source-controlled prices file with bin/rails llm_cost_tracker:prices:refresh"
 
       def call
         path = LlmCostTracker.configuration.prices_file
@@ -48,7 +48,7 @@ module LlmCostTracker
         Check.new(
           :warn,
           "prices",
-          "using bundled prices updated_at=#{updated_at}; configure prices_file for production"
+          "using bundled prices updated_at=#{updated_at}; commit a prices_file for production releases"
         )
       end
 

data/lib/llm_cost_tracker/doctor/pricing_snapshot_drift_check.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+
+require_relative "check"
+require_relative "probe"
+
+module LlmCostTracker
+  class Doctor
+    class PricingSnapshotDriftCheck
+      SAMPLE_SIZE = 200
+      EPSILON = BigDecimal("0.00000001")
+
+      def call
+        return unless Probe.table_exists?("llm_cost_tracker_calls")
+        return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
+
+        sampled_ids = LlmCostTracker::Call
+                      .where.not(pricing_snapshot: nil)
+                      .where(cost_status: %w[complete free])
+                      .order(id: :desc)
+                      .limit(SAMPLE_SIZE)
+                      .pluck(:id)
+        return Check.new(:ok, "pricing snapshot drift", "no snapshotted calls to inspect") if sampled_ids.empty?
+
+        calls_by_id = LlmCostTracker::Call.where(id: sampled_ids).index_by(&:id)
+        line_items_by_call = LlmCostTracker::CallLineItem
+                             .where(llm_cost_tracker_call_id: sampled_ids, unit: "token")
+                             .group_by(&:llm_cost_tracker_call_id)
+
+        drifted = sampled_ids.flat_map do |id|
+          call = calls_by_id[id]
+          rates = rates_for(call.pricing_snapshot)
+          next [] if rates.nil? || rates.empty?
+
+          (line_items_by_call[id] || []).filter_map { |item| drift_message_for(item, rates, call_id: id) }
+        end
+
+        return ok_check(sampled_ids.size) if drifted.empty?
+
+        Check.new(
+          :warn,
+          "pricing snapshot drift",
+          "line item cost diverges from pricing_snapshot rate in #{drifted.size} cases across " \
+          "#{sampled_ids.size} sampled calls: #{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
+        )
+      end
+
+      private
+
+      def ok_check(sample_size)
+        Check.new(:ok, "pricing snapshot drift",
+                  "line item costs match pricing_snapshot rates in #{sample_size} sampled calls")
+      end
+
+      def rates_for(snapshot)
+        rates = snapshot.is_a?(Hash) ? (snapshot["rates"] || snapshot[:rates]) : nil
+        rates.is_a?(Hash) ? rates : nil
+      end
+
+      def drift_message_for(line_item, rates, call_id:)
+        return nil unless line_item.price_key
+
+        rate = rates[line_item.price_key.to_s] || rates[line_item.price_key.to_sym]
+        return nil unless rate.is_a?(Hash)
+
+        rate_amount = decimal(rate["amount"] || rate[:amount])
+        rate_quantity = decimal(rate["quantity"] || rate[:quantity])
+        return nil if rate_amount.nil? || rate_quantity.nil? || rate_quantity.zero?
+
+        expected = (decimal(line_item.quantity) * rate_amount) / rate_quantity
+        actual = decimal(line_item.cost) || BigDecimal("0")
+        return nil if (expected - actual).abs <= EPSILON
+
+        "##{call_id}.#{line_item.price_key}: expected=#{expected.round(8).to_s('F')} stored=#{actual.to_s('F')}"
+      end
+
+      def decimal(value)
+        return nil if value.nil?
+
+        BigDecimal(value.to_s)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/probe.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "../ledger"
+
+module LlmCostTracker
+  class Doctor
+    module Probe
+      module_function
+
+      def table_exists?(name)
+        LlmCostTracker::Call.connection.data_source_exists?(name)
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/schema_check.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative "check"
+require_relative "probe"
+require_relative "../ledger"
+
+module LlmCostTracker
+  class Doctor
+    class SchemaCheck
+      def initialize(name:, schema:, table:, optional: false, install_command: "llm_cost_tracker:install")
+        @name = name
+        @schema = schema
+        @table = table
+        @optional = optional
+        @install_command = install_command
+      end
+
+      def call
+        return unless Probe.table_exists?("llm_cost_tracker_calls")
+        return if @optional && !Probe.table_exists?(@table)
+
+        errors = @schema.current_schema_errors
+        return Check.new(:ok, @name, "#{@table} exists") if errors.empty?
+
+        Check.new(
+          :error,
+          @name,
+          "current schema required; #{errors.join('; ')}; " \
+          "run bin/rails generate #{@install_command} && bin/rails db:migrate"
+        )
+      end
+    end
+  end
+end