llm_cost_tracker 0.5.2 → 0.6.0

data/docs/upgrading.md

@@ -0,0 +1,47 @@
+# Upgrading
+
+LLM Cost Tracker is still moving quickly, so upgrades should be explicit:
+inspect the changelog, run doctor, and apply only the generators your schema is
+missing.
+
+The version-by-version upgrade guide is moving here from the README.
+
+## Canonical Sources
+
+Until this page is expanded, use:
+
+- [Changelog](../CHANGELOG.md)
+- [Quickstart](../README.md#quickstart)
+- [Operations](operations.md)
+
+## Schema Generators
+
+Existing installs can add newer optional columns through focused generators:
+
+```bash
+bin/rails generate llm_cost_tracker:add_period_totals
+bin/rails generate llm_cost_tracker:add_ingestion
+bin/rails generate llm_cost_tracker:add_streaming
+bin/rails generate llm_cost_tracker:add_provider_response_id
+bin/rails generate llm_cost_tracker:add_usage_breakdown
+bin/rails generate llm_cost_tracker:upgrade_tags_to_jsonb
+bin/rails generate llm_cost_tracker:upgrade_cost_precision
+bin/rails generate llm_cost_tracker:add_latency_ms
+bin/rails db:migrate
+bin/rails llm_cost_tracker:doctor
+```
+
+On PostgreSQL, `upgrade_tags_to_jsonb` rewrites `llm_api_calls`. For large
+tables, run it during a maintenance window or replace it with a two-phase
+backfill.
+
+## Upgrade Habit
+
+Run:
+
+```bash
+bin/rails llm_cost_tracker:doctor
+```
+
+Doctor tells you which optional columns and production-hardening pieces are still
+missing.

data/lib/llm_cost_tracker/active_record_adapter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module ActiveRecordAdapter
+    MYSQL_ADAPTERS = %w[
+      ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter
+      ActiveRecord::ConnectionAdapters::Mysql2Adapter
+      ActiveRecord::ConnectionAdapters::TrilogyAdapter
+    ].freeze
+    POSTGRESQL_ADAPTERS = %w[
+      ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+    ].freeze
+    SQLITE_ADAPTERS = %w[
+      ActiveRecord::ConnectionAdapters::SQLite3Adapter
+    ].freeze
+    MYSQL_PATTERN = /mysql|trilogy|mariadb/i
+    POSTGRESQL_PATTERN = /postgres/i
+    SQLITE_PATTERN = /sqlite/i
+
+    class << self
+      def mysql?(value) = adapter_instance?(value, MYSQL_ADAPTERS) || adapter_name(value).match?(MYSQL_PATTERN)
+
+      def postgresql?(value)
+        adapter_instance?(value, POSTGRESQL_ADAPTERS) || adapter_name(value).match?(POSTGRESQL_PATTERN)
+      end
+
+      def sqlite?(value) = adapter_instance?(value, SQLITE_ADAPTERS) || adapter_name(value).match?(SQLITE_PATTERN)
+
+      private
+
+      def adapter_instance?(value, class_names)
+        class_names.any? do |class_name|
+          adapter_class = constantize(class_name)
+          adapter_class && value.is_a?(adapter_class)
+        end
+      end
+
+      def constantize(name)
+        name.split("::").reduce(Object) { |namespace, part| namespace.const_get(part, false) }
+      rescue NameError
+        nil
+      end
+
+      def adapter_name(value)
+        value.respond_to?(:adapter_name) ? value.adapter_name.to_s : value.to_s
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/capture_verifier.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "storage/dispatcher"
+
+module LlmCostTracker
+  class CaptureVerifier
+    Check = Data.define(:status, :name, :message)
+
+    class << self
+      def call = new.checks
+
+      def report(checks = call)
+        (["LLM Cost Tracker capture verification"] + checks.map { |check| format_check(check) }).join("\n")
+      end
+
+      def healthy?(checks = call)
+        checks.none? { |check| check.status == :error }
+      end
+
+      private
+
+      def format_check(check)
+        "[#{check.status}] #{check.name}: #{check.message}"
+      end
+    end
+
+    def checks
+      [
+        enabled_check,
+        *integration_checks,
+        *storage_checks
+      ].compact
+    end
+
+    private
+
+    def enabled_check
+      return Check.new(:ok, "tracking", "enabled") if LlmCostTracker.configuration.enabled
+
+      Check.new(:error, "tracking", "disabled; set config.enabled = true before verifying capture")
+    end
+
+    def integration_checks
+      enabled = LlmCostTracker.configuration.instrumented_integrations
+      if enabled.empty?
+        return [
+          Check.new(:ok, "sdk integrations", "none enabled; Faraday middleware and manual capture remain available")
+        ]
+      end
+
+      LlmCostTracker::Integrations.checks.map do |check|
+        Check.new(check.status, "sdk integration #{check.name}", check.message)
+      end
+    end
+
+    def storage_checks
+      backend = LlmCostTracker::Storage::Registry.fetch(LlmCostTracker.configuration.storage_backend)
+      unless backend.respond_to?(:verify)
+        return [
+          Check.new(:warn, "storage", "#{LlmCostTracker.configuration.storage_backend} backend has no verifier")
+        ]
+      end
+
+      backend.verify.map do |check|
+        Check.new(check.status, check.name, check.message)
+      end
+    rescue LlmCostTracker::Error => e
+      [Check.new(:error, "storage", e.message)]
+    end
+  end
+end

data/lib/llm_cost_tracker/configuration/instrumentation.rb

@@ -31,7 +31,7 @@ module LlmCostTracker
     end
 
     def available_instrumentation_names
-      Integrations::Registry::INTEGRATIONS.keys
+      Integrations::Registry.names
     end
   end
 end

data/lib/llm_cost_tracker/configuration/storage_backend.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module ConfigurationStorageBackend
+    def storage_backend=(value)
+      ensure_shared_configuration_mutable!
+      @storage_backend = normalize_storage_backend(value)
+    end
+
+    private
+
+    def normalize_storage_backend(value)
+      value = :log if value.nil?
+      value = value.to_sym
+      return value if self.class::STORAGE_BACKENDS.include?(value)
+      return value if defined?(Storage::Registry) && Storage::Registry.registered?(value)
+
+      names = if defined?(Storage::Registry)
+                Storage::Registry.names
+              else
+                self.class::STORAGE_BACKENDS
+              end
+      raise Error, "Unknown storage_backend: #{value.inspect}. Use one of: #{names.join(', ')}"
+    end
+  end
+end

data/lib/llm_cost_tracker/configuration.rb

@@ -4,10 +4,12 @@ require_relative "errors"
 require_relative "tag_key"
 require_relative "value_helpers"
 require_relative "configuration/instrumentation"
+require_relative "configuration/storage_backend"
 
 module LlmCostTracker
   class Configuration
     include ConfigurationInstrumentation
+    include ConfigurationStorageBackend
 
     OPENAI_COMPATIBLE_PROVIDERS = { "openrouter.ai" => "openrouter", "api.deepseek.com" => "deepseek" }.freeze
 
@@ -18,7 +20,6 @@ module LlmCostTracker
     SHARED_SCALAR_ATTRIBUTES = %i[enabled custom_storage on_budget_exceeded monthly_budget daily_budget per_call_budget
                                   log_level prices_file max_tag_count max_tag_value_bytesize].freeze
     SHARED_ENUM_ATTRIBUTES = {
-      storage_backend: [STORAGE_BACKENDS, :log],
       budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
       storage_error_behavior: [STORAGE_ERROR_BEHAVIORS, :warn],
       unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]

data/lib/llm_cost_tracker/doctor/capture_check.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  class Doctor
+    class CaptureCheck
+      def self.call(check_class)
+        new(check_class).call
+      end
+
+      def initialize(check_class)
+        @check_class = check_class
+      end
+
+      def call
+        config = LlmCostTracker.configuration
+        return disabled_check unless config.enabled
+        return integrations_check(config.instrumented_integrations) if config.instrumented_integrations.any?
+
+        check(:ok, "no SDK integrations enabled; Faraday middleware and manual capture remain available")
+      end
+
+      private
+
+      attr_reader :check_class
+
+      def disabled_check
+        check(:warn, "tracking is disabled; set config.enabled = true to record calls")
+      end
+
+      def integrations_check(integrations)
+        check(:ok, "SDK integrations enabled: #{integrations.join(', ')}")
+      end
+
+      def check(status, message)
+        check_class.new(status, "capture", message)
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/ingestion_check.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "time"
+
+module LlmCostTracker
+  class Doctor
+    class IngestionCheck
+      PENDING_AGE_WARNING_SECONDS = 60
+
+      def self.call(check_class)
+        new(check_class).call
+      end
+
+      def initialize(check_class)
+        @check_class = check_class
+      end
+
+      def call
+        return unless active_record_storage? && llm_api_calls_table?
+
+        missing = missing_parts
+        if missing.empty?
+          quarantined = quarantined_count
+          if quarantined.positive?
+            return check_class.new(:warn, "durable ingestion", "#{quarantined} inbox events quarantined after retries")
+          end
+
+          pending = pending_snapshot
+          if stale_pending?(pending)
+            return check_class.new(
+              :warn,
+              "durable ingestion",
+              "#{pending.fetch(:count)} inbox events pending; oldest pending age #{pending_age(pending).round}s"
+            )
+          end
+
+          return check_class.new(:ok, "durable ingestion", "inbox and ingestor lease tables available")
+        end
+
+        check_class.new(
+          :warn,
+          "durable ingestion",
+          "missing #{missing.join(', ')}; run bin/rails generate llm_cost_tracker:add_ingestion && bin/rails db:migrate"
+        )
+      end
+
+      private
+
+      attr_reader :check_class
+
+      def missing_parts
+        [
+          column_names("llm_api_calls").include?("event_id") ? nil : "llm_api_calls.event_id",
+          table_exists?("llm_cost_tracker_inbox_events") ? nil : "llm_cost_tracker_inbox_events",
+          table_exists?("llm_cost_tracker_ingestor_leases") ? nil : "llm_cost_tracker_ingestor_leases"
+        ].compact
+      end
+
+      def active_record_storage? = LlmCostTracker.configuration.storage_backend == :active_record
+
+      def llm_api_calls_table? = table_exists?("llm_api_calls")
+
+      def table_exists?(name)
+        LlmCostTracker::LlmApiCall.connection.data_source_exists?(name)
+      rescue StandardError
+        false
+      end
+
+      def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
+
+      def quarantined_count
+        return 0 unless table_exists?("llm_cost_tracker_inbox_events")
+
+        LlmCostTracker::LlmApiCall.connection.select_value(quarantined_sql).to_i
+      rescue StandardError
+        0
+      end
+
+      def quarantined_sql
+        table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
+        "SELECT COUNT(*) FROM #{table} WHERE attempts >= #{max_attempts}"
+      end
+
+      def pending_snapshot
+        row = LlmCostTracker::LlmApiCall.connection.select_one(pending_sql) || {}
+        {
+          count: row.fetch("pending_count").to_i,
+          oldest_at: row["oldest_created_at"] && Time.parse(row.fetch("oldest_created_at").to_s).utc
+        }
+      rescue StandardError
+        { count: 0, oldest_at: nil }
+      end
+
+      def pending_sql
+        table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
+        "SELECT COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at " \
+          "FROM #{table} WHERE attempts < #{max_attempts}"
+      end
+
+      def stale_pending?(pending)
+        pending.fetch(:count).positive? &&
+          pending.fetch(:oldest_at) &&
+          pending_age(pending) >= PENDING_AGE_WARNING_SECONDS
+      end
+
+      def pending_age(pending) = Time.now.utc - pending.fetch(:oldest_at)
+
+      def max_attempts
+        if defined?(LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS)
+          LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS
+        else
+          5
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "price_freshness"
+require_relative "doctor/capture_check"
+require_relative "doctor/ingestion_check"
 
 module LlmCostTracker
   class Doctor
@@ -38,11 +40,13 @@ module LlmCostTracker
     def checks
       [
         configuration_check,
+        capture_check,
         *integration_checks,
         active_record_check,
         table_check,
         column_check,
         period_totals_check,
+        IngestionCheck.call(Check),
         prices_check,
         calls_check
       ].compact
@@ -51,9 +55,12 @@ module LlmCostTracker
     private
 
     def configuration_check
-      Check.new(:ok, "configuration", "storage_backend=#{LlmCostTracker.configuration.storage_backend.inspect}")
+      config = LlmCostTracker.configuration
+      Check.new(:ok, "configuration", "storage_backend=#{config.storage_backend.inspect}, enabled=#{config.enabled}")
     end
 
+    def capture_check = CaptureCheck.call(Check)
+
     def integration_checks
       LlmCostTracker::Integrations.checks.map do |check|
         Check.new(check.status, check.name.to_s, check.message)

data/lib/llm_cost_tracker/event.rb

@@ -2,6 +2,7 @@
 
 module LlmCostTracker
   Event = Data.define(
+    :event_id,
     :provider,
     :model,
     :input_tokens,

data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_ingestion_generator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module LlmCostTracker
+  module Generators
+    class AddIngestionGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates a migration to add durable ActiveRecord ingestion"
+
+      def create_migration_file
+        migration_template(
+          "add_ingestion_to_llm_cost_tracker.rb.erb",
+          "db/migrate/add_ingestion_to_llm_cost_tracker.rb"
+        )
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_ingestion_to_llm_cost_tracker.rb.erb

@@ -0,0 +1,33 @@
+class AddIngestionToLlmCostTracker < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column :llm_api_calls, :event_id, :string unless column_exists?(:llm_api_calls, :event_id)
+    add_index :llm_api_calls, :event_id, unique: true if column_exists?(:llm_api_calls, :event_id) &&
+                                                         !index_exists?(:llm_api_calls, :event_id)
+
+    create_table :llm_cost_tracker_inbox_events do |t|
+      t.string   :event_id, null: false
+      t.decimal  :total_cost, precision: 20, scale: 8
+      t.datetime :tracked_at, null: false
+      t.text     :payload, null: false
+      t.datetime :locked_at
+      t.string   :locked_by
+      t.integer  :attempts, null: false, default: 0
+      t.text     :last_error
+
+      t.timestamps
+    end unless table_exists?(:llm_cost_tracker_inbox_events)
+
+    create_table :llm_cost_tracker_ingestor_leases do |t|
+      t.string   :name, null: false
+      t.string   :locked_by
+      t.datetime :locked_until
+
+      t.timestamps
+    end unless table_exists?(:llm_cost_tracker_ingestor_leases)
+
+    add_index :llm_cost_tracker_inbox_events, :event_id, unique: true unless index_exists?(:llm_cost_tracker_inbox_events, :event_id)
+    add_index :llm_cost_tracker_inbox_events, :tracked_at unless index_exists?(:llm_cost_tracker_inbox_events, :tracked_at)
+    add_index :llm_cost_tracker_inbox_events, [:locked_at, :id] unless index_exists?(:llm_cost_tracker_inbox_events, [:locked_at, :id])
+    add_index :llm_cost_tracker_ingestor_leases, :name, unique: true unless index_exists?(:llm_cost_tracker_ingestor_leases, :name)
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb

@@ -1,3 +1,5 @@
+require "llm_cost_tracker/active_record_adapter"
+
 class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_version %>
   def up
     create_table :llm_cost_tracker_period_totals do |t|
@@ -73,10 +75,9 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
   end
 
   def day_bucket_sql
-    case connection.adapter_name
-    when /postgres/i
+    if postgresql?
       "DATE_TRUNC('day', tracked_at)::date"
-    when /mysql/i
+    elsif mysql?
       "DATE(tracked_at)"
     else
       "date(tracked_at)"
@@ -84,13 +85,20 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
   end
 
   def month_bucket_sql
-    case connection.adapter_name
-    when /postgres/i
+    if postgresql?
       "DATE_TRUNC('month', tracked_at)::date"
-    when /mysql/i
+    elsif mysql?
       "DATE_FORMAT(tracked_at, '%Y-%m-01')"
     else
       "strftime('%Y-%m-01', tracked_at)"
     end
   end
+
+  def postgresql?
+    LlmCostTracker::ActiveRecordAdapter.postgresql?(connection)
+  end
+
+  def mysql?
+    LlmCostTracker::ActiveRecordAdapter.mysql?(connection)
+  end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_streaming_to_llm_api_calls.rb.erb

@@ -2,23 +2,19 @@ class AddStreamingToLlmApiCalls < ActiveRecord::Migration<%= migration_version %
   def up
     unless column_exists?(:llm_api_calls, :stream)
       add_column :llm_api_calls, :stream, :boolean, null: false, default: false
-      add_index  :llm_api_calls, :stream
     end
 
     unless column_exists?(:llm_api_calls, :usage_source)
       add_column :llm_api_calls, :usage_source, :string
-      add_index  :llm_api_calls, :usage_source
     end
   end
 
   def down
     if column_exists?(:llm_api_calls, :usage_source)
-      remove_index  :llm_api_calls, :usage_source if index_exists?(:llm_api_calls, :usage_source)
       remove_column :llm_api_calls, :usage_source
     end
 
     if column_exists?(:llm_api_calls, :stream)
-      remove_index  :llm_api_calls, :stream if index_exists?(:llm_api_calls, :stream)
       remove_column :llm_api_calls, :stream
     end
   end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb

@@ -1,6 +1,9 @@
+require "llm_cost_tracker/active_record_adapter"
+
 class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :llm_api_calls do |t|
+      t.string  :event_id,      null: false
       t.string  :provider,      null: false
       t.string  :model,         null: false
       t.integer :input_tokens,  null: false, default: 0
@@ -37,19 +40,43 @@ class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
       t.timestamps
     end
 
+    create_table :llm_cost_tracker_inbox_events do |t|
+      t.string   :event_id, null: false
+      t.decimal  :total_cost, precision: 20, scale: 8
+      t.datetime :tracked_at, null: false
+      t.text     :payload, null: false
+      t.datetime :locked_at
+      t.string   :locked_by
+      t.integer  :attempts, null: false, default: 0
+      t.text     :last_error
+
+      t.timestamps
+    end
+
+    create_table :llm_cost_tracker_ingestor_leases do |t|
+      t.string   :name, null: false
+      t.string   :locked_by
+      t.datetime :locked_until
+
+      t.timestamps
+    end
+
+    add_index :llm_api_calls, :event_id, unique: true
     add_index :llm_api_calls, :tracked_at
     add_index :llm_api_calls, [:provider, :tracked_at]
     add_index :llm_api_calls, [:model, :tracked_at]
-    add_index :llm_api_calls, :stream
-    add_index :llm_api_calls, :usage_source
     add_index :llm_api_calls, :provider_response_id
     add_index :llm_api_calls, :tags, using: :gin if postgresql?
     add_index :llm_cost_tracker_period_totals, [:period, :period_start], unique: true
+    add_index :llm_cost_tracker_inbox_events, :event_id, unique: true
+    add_index :llm_cost_tracker_inbox_events, :tracked_at
+    add_index :llm_cost_tracker_inbox_events, [:locked_at, :id]
+    add_index :llm_cost_tracker_ingestor_leases, :name, unique: true
   end
 
   private
 
   def postgresql?
-    connection.adapter_name.downcase.include?("postgres")
+    LlmCostTracker::ActiveRecordAdapter.postgresql?(connection)
   end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb

@@ -4,7 +4,7 @@ LlmCostTracker.configure do |config|
   # Set to false to temporarily disable tracking without removing middleware.
   config.enabled = true
 
-  # :active_record stores events in llm_api_calls for dashboards, reports, and shared budgets.
+  # :active_record captures events into a durable inbox and ingests them into llm_api_calls.
   # Other options: :log for local logging, :custom for your own storage callable.
   config.storage_backend = :active_record
 

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb

@@ -1,3 +1,5 @@
+require "llm_cost_tracker/active_record_adapter"
+
 class UpgradeLlmApiCallTagsToJsonb < ActiveRecord::Migration<%= migration_version %>
   def up
     unless postgresql?
@@ -32,7 +34,7 @@ class UpgradeLlmApiCallTagsToJsonb < ActiveRecord::Migration<%= migration_versio
   private
 
   def postgresql?
-    connection.adapter_name.downcase.include?("postgres")
+    LlmCostTracker::ActiveRecordAdapter.postgresql?(connection)
   end
 
   def tags_jsonb?

data/lib/llm_cost_tracker/inbox_event.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module LlmCostTracker
+  class InboxEvent < ActiveRecord::Base
+    self.table_name = "llm_cost_tracker_inbox_events"
+  end
+end

data/lib/llm_cost_tracker/ingestor_lease.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require "active_record"
+
+module LlmCostTracker
+  class IngestorLease < ActiveRecord::Base
+    self.table_name = "llm_cost_tracker_ingestor_leases"
+  end
+end