llm_cost_tracker 0.8.0 → 0.10.0

data/lib/llm_cost_tracker/doctor.rb

@@ -13,6 +13,9 @@ require_relative "doctor/pricing_snapshot_drift_check"
 
 module LlmCostTracker
   class Doctor
+    autoload :InvoiceReconciliationCheck, "llm_cost_tracker/doctor/invoice_reconciliation_check"
+    autoload :CaptureVerifier,            "llm_cost_tracker/doctor/capture_verifier"
+
     class << self
       def call
         new.checks
@@ -33,7 +36,7 @@ module LlmCostTracker
       [
         configuration_check,
         capture_check,
-        *integration_checks,
+        *LlmCostTracker::Integrations.checks,
         active_record_check,
         table_check,
         column_check,
@@ -41,10 +44,10 @@ module LlmCostTracker
                         table: "llm_cost_tracker_call_line_items").call,
         SchemaCheck.new(name: "call tags", schema: Ledger::Schema::CallTags,
                         table: "llm_cost_tracker_call_tags").call,
-        SchemaCheck.new(name: "provider invoices", schema: Ledger::Schema::ProviderInvoices,
-                        table: "llm_cost_tracker_provider_invoices").call,
+        *reconciliation_schema_checks,
         CostDriftCheck.new.call,
         PricingSnapshotDriftCheck.new.call,
+        *reconciliation_invoice_check,
         LegacyBillingStatusCheck.new.call,
         LegacyAuditCheck.new.call,
         call_rollups_check,
@@ -56,6 +59,22 @@ module LlmCostTracker
 
     private
 
+    def reconciliation_schema_checks
+      return [] unless LlmCostTracker.reconciliation_enabled?
+
+      Reconciliation::SCHEMA_TABLES.map do |schema, table|
+        SchemaCheck.new(name: table.delete_prefix("llm_cost_tracker_").tr("_", " "),
+                        schema: schema, table: table,
+                        optional: false, install_command: "llm_cost_tracker:reconciliation").call
+      end.compact
+    end
+
+    def reconciliation_invoice_check
+      return [] unless LlmCostTracker.reconciliation_enabled?
+
+      Array(InvoiceReconciliationCheck.new.call)
+    end
+
     def configuration_check
       config = LlmCostTracker.configuration
       Check.new(:ok, "configuration", "active_record ledger enabled=#{config.enabled}")
@@ -82,12 +101,6 @@ module LlmCostTracker
       )
     end
 
-    def integration_checks
-      LlmCostTracker::Integrations.checks.map do |check|
-        Check.new(check.status, check.name.to_s, check.message)
-      end
-    end
-
     def active_record_check
       return Check.new(:ok, "active_record", "available") if active_record_available?
 
@@ -120,9 +133,10 @@ module LlmCostTracker
 
     def call_rollups_check
       return unless llm_cost_tracker_calls_table?
+      return live_rollups_check unless LlmCostTracker.configuration.cache_rollups
 
       errors = LlmCostTracker::Ledger::Schema::CallRollups.current_schema_errors
-      return Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists") if errors.empty?
+      return rollups_drift_check if errors.empty?
 
       Check.new(
         :error,
@@ -131,6 +145,52 @@ module LlmCostTracker
       )
     end
 
+    ROLLUPS_DRIFT_TOLERANCE_PERCENT = 1.0
+    private_constant :ROLLUPS_DRIFT_TOLERANCE_PERCENT
+
+    def rollups_drift_check
+      drift_window = Time.now.utc.beginning_of_day
+      calls_total = LlmCostTracker::Call
+                    .where(tracked_at: drift_window..)
+                    .where.not(total_cost: nil)
+                    .sum(:total_cost)
+      rollup_total = LlmCostTracker::CallRollup
+                     .where(period: "day", period_start: drift_window.to_date)
+                     .sum(:total_cost)
+      return Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists") if calls_total.zero?
+
+      drift_percent = ((calls_total - rollup_total).abs * 100.0 / calls_total)
+      if drift_percent > ROLLUPS_DRIFT_TOLERANCE_PERCENT
+        return Check.new(
+          :warn, "call rollups",
+          "rollups drift detected: today's calls SUM=#{calls_total} vs rollups SUM=#{rollup_total} " \
+          "(#{drift_percent.round(2)}% > #{ROLLUPS_DRIFT_TOLERANCE_PERCENT}% threshold). " \
+          "Cached budget reads may understate spend until a rebuild."
+        )
+      end
+
+      Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists")
+    rescue StandardError => e
+      Check.new(:warn, "call rollups", "rollups drift check failed: #{e.class}: #{e.message}")
+    end
+
+    def live_rollups_check
+      if Probe.table_exists?("llm_cost_tracker_call_rollups")
+        Check.new(
+          :warn,
+          "call rollups",
+          "cache_rollups=false but llm_cost_tracker_call_rollups exists. " \
+          "Set config.cache_rollups = true to keep budget reads on the rollups fast path or drop the table."
+        )
+      else
+        Check.new(
+          :ok,
+          "call rollups",
+          "cache_rollups=false; budget reads aggregate from llm_cost_tracker_calls directly"
+        )
+      end
+    end
+
     def calls_check
       return unless llm_cost_tracker_calls_table?
 
@@ -140,16 +200,14 @@ module LlmCostTracker
       count = snapshot.tracked_call_count.to_i
       return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
 
-      latest_at = snapshot.latest_tracked_at
-      latest_at = latest_at.to_time if latest_at.respond_to?(:to_time)
-      latest = latest_at&.utc&.iso8601
+      latest = snapshot.latest_tracked_at.to_time.utc.iso8601
       Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
     end
 
     def active_record_available?
       LlmCostTracker::Call.connection
       true
-    rescue LoadError, StandardError
+    rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
       false
     end
 

data/lib/llm_cost_tracker/engine.rb

@@ -8,5 +8,13 @@ require "rack/files"
 module LlmCostTracker
   class Engine < ::Rails::Engine
     isolate_namespace LlmCostTracker
+
+    initializer "llm_cost_tracker.filter_parameters" do |app|
+      app.config.filter_parameters += %i[tag tag_value]
+    end
+
+    initializer "llm_cost_tracker.dashboard_setup_state" do |app|
+      app.reloader.to_prepare { LlmCostTracker::Dashboard::SetupState.reset! }
+    end
   end
 end

data/lib/llm_cost_tracker/errors.rb

@@ -6,13 +6,14 @@ module LlmCostTracker
   class InvalidFilterError < Error; end
 
   class BudgetExceededError < Error
-    attr_reader :total, :budget, :budget_type, :last_event
+    attr_reader :total, :budget, :budget_type, :last_event, :stage
 
-    def initialize(budget:, budget_type:, total:, last_event: nil)
+    def initialize(budget:, budget_type:, total:, last_event: nil, stage: :post_spend)
       @total = total
       @budget = budget
       @budget_type = budget_type
       @last_event = last_event
+      @stage = stage
 
       super(
         "LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded: " \

data/lib/llm_cost_tracker/event.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "pricing"
+require_relative "billing/line_item"
+
 module LlmCostTracker
   Event = Data.define(
     :event_id,
@@ -22,6 +25,46 @@ module LlmCostTracker
     :pricing_snapshot,
     :line_items
   ) do
+    def self.batch_from_pricing_mode?(pricing_mode)
+      pricing_mode.to_s.split("_").include?("batch")
+    end
+
+    def self.build(**attributes)
+      pricing_mode = Pricing.normalize_mode(attributes[:pricing_mode])
+      token_usage = attributes.fetch(:token_usage)
+      batch = attributes[:batch].nil? ? batch_from_pricing_mode?(pricing_mode) : attributes[:batch]
+      line_items = attributes[:line_items] || resolve_line_items(attributes[:service_line_items], token_usage)
+
+      new(
+        event_id: attributes[:event_id],
+        provider: attributes.fetch(:provider).to_s,
+        model: attributes.fetch(:model).to_s.strip.presence || Event::UNKNOWN_MODEL,
+        token_usage: token_usage,
+        pricing_mode: pricing_mode,
+        cost: attributes[:cost],
+        tags: attributes[:tags],
+        latency_ms: attributes[:latency_ms],
+        stream: attributes[:stream] || false,
+        usage_source: attributes[:usage_source],
+        provider_response_id: attributes[:provider_response_id].to_s.strip.presence,
+        provider_project_id: attributes[:provider_project_id].to_s.strip.presence,
+        provider_api_key_id: attributes[:provider_api_key_id].to_s.strip.presence,
+        provider_workspace_id: attributes[:provider_workspace_id].to_s.strip.presence,
+        batch: batch,
+        tracked_at: attributes[:tracked_at],
+        cost_status: attributes[:cost_status],
+        pricing_snapshot: attributes[:pricing_snapshot],
+        line_items: line_items
+      )
+    end
+
+    def self.resolve_line_items(service_items, token_usage)
+      service_line_items = Array(service_items).map do |item|
+        item.is_a?(Billing::LineItem) ? item : Billing::LineItem.build(item)
+      end
+      Billing::LineItem.from_token_usage(token_usage) + service_line_items
+    end
+
     def total_cost
       cost&.fetch(:total_cost, nil)
     end
@@ -29,10 +72,14 @@ module LlmCostTracker
     def to_h
       super.merge(
         token_usage: token_usage.to_h,
-        cost: cost&.to_h,
+        cost: cost && cost.to_h.transform_values { |v| v.is_a?(BigDecimal) ? v.to_f : v },
         tags: tags ? tags.to_h : {},
         line_items: (line_items || []).map(&:to_h)
       )
     end
   end
+
+  class Event
+    UNKNOWN_MODEL = "unknown"
+  end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module LlmCostTracker
+  module Generators
+    class AsyncIngestionGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates the async ingestion tables (llm_cost_tracker_ingestion_inbox_entries + _leases). " \
+           "Required when config.ingestion = :async."
+
+      def create_migration_file
+        migration_template(
+          "create_llm_cost_tracker_async_ingestion.rb.erb",
+          "db/migrate/create_llm_cost_tracker_async_ingestion.rb"
+        )
+      end
+
+      def warn_about_config_flag
+        say(<<~MSG, :yellow)
+          After migrating, set the following in config/initializers/llm_cost_tracker.rb:
+
+            LlmCostTracker.configure do |config|
+              config.ingestion = :async
+            end
+
+          Without it the async inbox tables stay unused and Tracker keeps writing
+          inline. The doctor check warns about unused async ingestion tables.
+        MSG
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module LlmCostTracker
+  module Generators
+    class CallRollupsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates the optional llm_cost_tracker_call_rollups table for fast budget reads. " \
+           "Required when config.cache_rollups = true."
+
+      def create_migration_file
+        migration_template(
+          "create_llm_cost_tracker_call_rollups.rb.erb",
+          "db/migrate/create_llm_cost_tracker_call_rollups.rb"
+        )
+      end
+
+      def warn_about_config_flag
+        say(<<~MSG, :yellow)
+          After migrating, set the following in config/initializers/llm_cost_tracker.rb:
+
+            LlmCostTracker.configure do |config|
+              config.cache_rollups = true
+            end
+
+          Without it Tracker keeps reading budget totals as live SUM aggregates over
+          llm_cost_tracker_calls. The doctor check warns about an unused rollups table.
+        MSG
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb

@@ -26,24 +26,33 @@ module LlmCostTracker
       end
 
       def create_initializer
-        template(
-          "initializer.rb.erb",
-          "config/initializers/llm_cost_tracker.rb"
-        )
+        destination = "config/initializers/llm_cost_tracker.rb"
+        return if File.exist?(File.join(destination_root, destination))
+
+        template("initializer.rb.erb", destination)
       end
 
       def create_prices_file
         return unless options[:prices]
 
-        invoke "llm_cost_tracker:prices"
+        require_relative "prices_generator"
+        invoke LlmCostTracker::Generators::PricesGenerator
       end
 
       def mount_engine
         return unless options[:dashboard]
 
-        add_engine_require
-        route %(mount LlmCostTracker::Engine => "/llm-costs")
-        say "Mount /llm-costs behind your app's admin auth before deploying.", :yellow
+        say(<<~MSG, :yellow)
+          The LLM Cost Tracker dashboard ships without authentication.
+          Mount it in config/routes.rb behind your app's admin auth, e.g.:
+
+            authenticate :admin do
+              mount LlmCostTracker::Engine => "/llm-costs"
+            end
+
+          The generator does NOT add a route automatically — leaving the dashboard
+          unauthenticated would expose spend, tags, and provider IDs to anyone.
+        MSG
       end
 
       private
@@ -51,24 +60,6 @@ module LlmCostTracker
       def migration_version
         "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
       end
-
-      def add_engine_require
-        return unless File.exist?("config/application.rb")
-
-        contents = File.read("config/application.rb")
-        return if contents.include?(%(require "llm_cost_tracker/engine"))
-
-        unless contents.include?(%(require "rails/all"\n))
-          prepend_to_file("config/application.rb", %(require "llm_cost_tracker/engine"\n))
-          return
-        end
-
-        inject_into_file(
-          "config/application.rb",
-          %(require "llm_cost_tracker/engine"\n),
-          after: %(require "rails/all"\n)
-        )
-      end
     end
   end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/reconciliation_generator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module LlmCostTracker
+  module Generators
+    class ReconciliationGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates the optional invoice reconciliation tables. Requires provider admin/org-level API keys."
+
+      def create_migration_file
+        migration_template(
+          "create_llm_cost_tracker_reconciliation.rb.erb",
+          "db/migrate/create_llm_cost_tracker_reconciliation.rb"
+        )
+      end
+
+      def warn_about_admin_keys
+        say "Reconciliation requires admin/org-level API keys (OpenAI sk-admin-..., Anthropic admin keys, " \
+            "GCP billing.viewer service accounts). Do NOT use the runtime inference key.", :yellow
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb

@@ -0,0 +1,29 @@
+class CreateLlmCostTrackerAsyncIngestion < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :llm_cost_tracker_ingestion_inbox_entries do |t|
+      t.string   :event_id, null: false
+      t.decimal  :total_cost, precision: 20, scale: 8
+      t.datetime :tracked_at, null: false
+      t.text     :payload, null: false, limit: 16.megabytes
+      t.datetime :locked_at
+      t.string   :locked_by
+      t.integer  :attempts, null: false, default: 0
+      t.text     :last_error, limit: 16.megabytes
+
+      t.timestamps
+    end
+
+    create_table :llm_cost_tracker_ingestion_leases do |t|
+      t.string   :name, null: false
+      t.string   :locked_by
+      t.datetime :locked_until
+
+      t.timestamps
+    end
+
+    add_index :llm_cost_tracker_ingestion_inbox_entries, :event_id, unique: true
+    add_index :llm_cost_tracker_ingestion_inbox_entries, [:tracked_at, :attempts]
+    add_index :llm_cost_tracker_ingestion_inbox_entries, [:locked_at, :id]
+    add_index :llm_cost_tracker_ingestion_leases, :name, unique: true
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_call_rollups.rb.erb

@@ -0,0 +1,15 @@
+class CreateLlmCostTrackerCallRollups < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :llm_cost_tracker_call_rollups do |t|
+      t.string :period, null: false
+      t.date :period_start, null: false
+      t.string :currency, null: false, default: "USD"
+      t.string :provider, null: false, default: ""
+      t.decimal :total_cost, precision: 20, scale: 8, null: false, default: 0
+
+      t.timestamps
+    end
+
+    add_index :llm_cost_tracker_call_rollups, [:period, :period_start, :currency, :provider], unique: true
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb

@@ -34,36 +34,6 @@ class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%= migration_version %
       t.timestamps
     end
 
-    create_table :llm_cost_tracker_call_rollups do |t|
-      t.string :period, null: false
-      t.date :period_start, null: false
-      t.string :currency, null: false, default: "USD"
-      t.decimal :total_cost, precision: 20, scale: 8, null: false, default: 0
-
-      t.timestamps
-    end
-
-    create_table :llm_cost_tracker_ingestion_inbox_entries do |t|
-      t.string   :event_id, null: false
-      t.decimal  :total_cost, precision: 20, scale: 8
-      t.datetime :tracked_at, null: false
-      t.text     :payload, null: false
-      t.datetime :locked_at
-      t.string   :locked_by
-      t.integer  :attempts, null: false, default: 0
-      t.text     :last_error
-
-      t.timestamps
-    end
-
-    create_table :llm_cost_tracker_ingestion_leases do |t|
-      t.string   :name, null: false
-      t.string   :locked_by
-      t.datetime :locked_until
-
-      t.timestamps
-    end
-
     create_table :llm_cost_tracker_call_line_items do |t|
       t.references :llm_cost_tracker_call,
                    null: false,
@@ -107,25 +77,6 @@ class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%= migration_version %
       t.text :value, null: false
     end
 
-    create_table :llm_cost_tracker_provider_invoices do |t|
-      t.string :source, null: false
-      t.date :period_start, null: false
-      t.date :period_end, null: false
-      t.string :external_id, null: false
-      t.decimal :billed_amount, precision: 20, scale: 8
-      t.string :currency, null: false, default: "USD"
-      if postgresql?
-        t.jsonb :metadata, null: false, default: {}
-      elsif mysql?
-        t.json :metadata, null: false
-      else
-        raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
-      end
-      t.datetime :imported_at, null: false
-
-      t.timestamps
-    end
-
     add_index :llm_cost_tracker_calls, :event_id, unique: true
     add_index :llm_cost_tracker_calls, :tracked_at
     add_index :llm_cost_tracker_calls, [:provider, :tracked_at]
@@ -133,16 +84,12 @@ class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%= migration_version %
     add_index :llm_cost_tracker_calls, :cost_status
     add_index :llm_cost_tracker_calls, :provider_response_id
     add_index :llm_cost_tracker_call_line_items, [:llm_cost_tracker_call_id, :position]
-    add_index :llm_cost_tracker_call_line_items, :kind
     add_index :llm_cost_tracker_call_tags, :llm_cost_tracker_call_id
-    add_index :llm_cost_tracker_call_tags, :key
-    add_index :llm_cost_tracker_call_rollups, [:period, :period_start, :currency], unique: true
-    add_index :llm_cost_tracker_ingestion_inbox_entries, :event_id, unique: true
-    add_index :llm_cost_tracker_ingestion_inbox_entries, [:tracked_at, :attempts]
-    add_index :llm_cost_tracker_ingestion_inbox_entries, [:locked_at, :id]
-    add_index :llm_cost_tracker_ingestion_leases, :name, unique: true
-    add_index :llm_cost_tracker_provider_invoices, :external_id, unique: true
-    add_index :llm_cost_tracker_provider_invoices, [:source, :period_start]
+    if postgresql?
+      add_index :llm_cost_tracker_call_tags, [:key, :value]
+    elsif mysql?
+      add_index :llm_cost_tracker_call_tags, [:key, :value], length: { value: 191 }
+    end
   end
 
   private

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb

@@ -0,0 +1,60 @@
+require "llm_cost_tracker/ledger/schema/adapter"
+
+class CreateLlmCostTrackerReconciliation < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :llm_cost_tracker_provider_invoices, if_not_exists: true do |t|
+      t.string :source, null: false
+      t.date :period_start, null: false
+      t.date :period_end, null: false
+      t.string :external_id, null: false
+      t.decimal :billed_amount, precision: 20, scale: 8
+      t.string :currency, null: false, default: "USD"
+      if postgresql?
+        t.jsonb :metadata, null: false, default: {}
+      elsif mysql?
+        t.json :metadata, null: false
+      else
+        raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
+      end
+      t.datetime :imported_at, null: false
+
+      t.timestamps
+    end
+
+    create_table :llm_cost_tracker_provider_invoice_imports, if_not_exists: true do |t|
+      t.string :source, null: false
+      t.string :provider, null: false, default: ""
+      t.string :cursor
+      t.date :window_start
+      t.date :window_end
+      t.string :state, null: false
+      t.text :last_error
+      t.integer :rows_imported, null: false, default: 0
+      t.datetime :started_at, null: false
+      t.datetime :finished_at
+
+      t.timestamps
+    end
+
+    add_index :llm_cost_tracker_provider_invoices, :external_id, unique: true,
+              if_not_exists: true
+    add_index :llm_cost_tracker_provider_invoices, %i[source currency period_start],
+              if_not_exists: true
+    if postgresql?
+      add_index :llm_cost_tracker_provider_invoices, :metadata, using: :gin,
+                if_not_exists: true
+    end
+    add_index :llm_cost_tracker_provider_invoice_imports, %i[source provider started_at],
+              if_not_exists: true
+  end
+
+  private
+
+  def postgresql?
+    LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
+  end
+
+  def mysql?
+    LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
+  end
+end