llm_cost_tracker 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3fe81fdf7b12f977d7dfc4aa86f629e8b5ad6e099100bfbfe1b18507db17fff
-  data.tar.gz: dffe0c0ebeb30fa111273b654141ee06e1275426846796b1749435429da3414f
+  metadata.gz: fa3f705baf280c2c2239b2dab3522fe7db9b60e26060b00fc08dcc039117da83
+  data.tar.gz: ea34bdad7cb0d7c9fb3233b40b609cea1361ded833ad130c4a7a7ce559b34758
 SHA512:
-  metadata.gz: 4849f0d0b09d640ed3a902bbc19a975e576b0539d2c71eac3165199eed196ecd88f249ff1d571c43b2686d59466dde26a0467f984c2cbb7804a5f771e184df31
-  data.tar.gz: 8aa17efc9dd68b1489cdc69351a67fe2398c35720750cf1440f215bceeef0392c9f3af1d41ebd444ddcd9876c2c1e612ea50d6403fb7f650735fc9d9161a2a85
+  metadata.gz: 03c55866b522b36b0728f73fd0ae0075e0a42faa6f80c429865f120d2c597e0b0996665b6a82528c566a0f8554d7636dfd03d9ab600441dafd5a4f0233d3f56b
+  data.tar.gz: af01c4554912d80276bf54ed97aa379c2eaed791fa357ca135aa008fdcbdd41e365c236ac74c9ceb1982c0fbbf2538bc569df1690acb4b5758895dd48c4497b5

data/CHANGELOG.md

@@ -4,6 +4,36 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning: [S
 
 ## [Unreleased]
 
+## [0.6.0] - 2026-04-29
+
+### Added
+
+- Durable ActiveRecord ingestion through `llm_cost_tracker_inbox_events` and `llm_cost_tracker_ingestor_leases`.
+- `llm_cost_tracker:add_ingestion` generator for upgrading existing ActiveRecord installs.
+- `LlmCostTracker.flush!` and `LlmCostTracker.shutdown!` for draining or stopping durable ingestion.
+- Doctor diagnostics for missing durable ingestion schema, stale pending inbox rows, and quarantined inbox rows.
+- PostgreSQL and MySQL smoke checks for ActiveRecord durable ingestion.
+
+### Changed
+
+- Fresh ActiveRecord installs now include durable ingestion tables, event IDs, and production indexes.
+- ActiveRecord budget totals now read stored period rollups plus pending inbox totals while durable ingestion is enabled.
+- ActiveRecord writes now use a durable inbox before batching ledger inserts and period rollup updates when ingestion tables are present.
+- Pricing lookup now caches normalized runtime price tables and model matches by configuration generation.
+- Stream capture now estimates buffered event size without serializing every captured event.
+- CSV export now selects only exported columns instead of loading full ActiveRecord objects.
+
+### Fixed
+
+- ActiveRecord rollups no longer double-count retried events when duplicate event IDs race across workers.
+- Invalid inbox rows are retried and quarantined without blocking healthy rows behind them.
+- Idle ingestors no longer acquire the leader lease while the inbox is empty.
+- ActiveRecord inbox writes now fail honestly when a separate connection is unavailable inside a caller transaction.
+- Ingestor shutdown/reset no longer lets an old sleeping thread resume as a second local ingestor.
+- `flush!` now returns `false` instead of raising when its timeout expires during ingestion.
+- ActiveRecord adapter family detection now works through known adapter class ancestry with an adapter-name fallback.
+- CSV export now emits `{}` for invalid stored tag payloads.
+
 ## [0.5.3] - 2026-04-28
 
 ### Added

data/app/controllers/llm_cost_tracker/calls_controller.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "csv"
+require "json"
 
 module LlmCostTracker
   class CallsController < ApplicationController
@@ -54,32 +55,45 @@ module LlmCostTracker
     end
 
     def render_csv(relation)
-      latency = LlmApiCall.latency_column?
+      fields = csv_fields
       CSV.generate do |csv|
-        headers = %w[tracked_at provider model input_tokens output_tokens total_tokens total_cost]
-        headers << "latency_ms" if latency
-        headers << "provider_response_id" if LlmApiCall.provider_response_id_column?
-        headers << "tags"
-        csv << headers
-
-        relation.each do |call|
-          row = [
-            call.tracked_at&.utc&.iso8601,
-            csv_safe(call.provider),
-            csv_safe(call.model),
-            call.input_tokens,
-            call.output_tokens,
-            call.total_tokens,
-            call.total_cost
-          ]
-          row << call.latency_ms if latency
-          row << csv_safe(call.provider_response_id) if LlmApiCall.provider_response_id_column?
-          row << csv_safe(call.parsed_tags.to_json)
-          csv << row
+        csv << fields.map(&:to_s)
+
+        relation.pluck(*fields).each do |values|
+          csv << fields.zip(values).map { |field, value| csv_value(field, value) }
         end
       end
     end
 
+    def csv_fields
+      fields = %i[tracked_at provider model input_tokens output_tokens total_tokens total_cost]
+      fields << :latency_ms if LlmApiCall.latency_column?
+      fields << :provider_response_id if LlmApiCall.provider_response_id_column?
+      fields << :tags
+      fields
+    end
+
+    def csv_value(field, value)
+      case field
+      when :tracked_at
+        value&.utc&.iso8601
+      when :provider, :model, :provider_response_id
+        csv_safe(value)
+      when :tags
+        csv_safe(csv_tags(value))
+      else
+        value
+      end
+    end
+
+    def csv_tags(value)
+      return value.transform_keys(&:to_s).to_json if value.is_a?(Hash)
+
+      JSON.parse(value || "{}").to_json
+    rescue JSON::ParserError
+      "{}"
+    end
+
     def csv_safe(value)
       return value if value.nil?
 

data/app/services/llm_cost_tracker/dashboard/overview_stats.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "llm_cost_tracker/storage/active_record_store"
+
 module LlmCostTracker
   module Dashboard
     OverviewStatsData = Data.define(
@@ -77,7 +79,7 @@ module LlmCostTracker
           now = Time.now.utc
           month_start = now.beginning_of_month
           month_end = now.end_of_month
-          spent = LlmCostTracker::LlmApiCall.this_month.total_cost
+          spent = LlmCostTracker::Storage::ActiveRecordStore.monthly_total(time: now)
           elapsed_seconds = now - month_start
           total_seconds = month_end - month_start
           projected_spent = if spent.zero? || !elapsed_seconds.positive?

data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb

@@ -42,11 +42,10 @@ module LlmCostTracker
       end
 
       def build_sql
-        case connection.adapter_name
-        when /postgres/i then postgresql_sql
-        when /mysql/i    then mysql_sql
-        else                  sqlite_sql
-        end
+        return postgresql_sql if ActiveRecordAdapter.postgresql?(connection)
+        return mysql_sql if ActiveRecordAdapter.mysql?(connection)
+
+        sqlite_sql
       end
 
       def mysql_sql

data/docs/architecture.md

@@ -18,7 +18,7 @@ Pricing logic should prefer generic mechanisms over provider branches. Use provi
 
 Tags remain the extension point for app-specific attribution such as tenant, user, feature, trace, job, workflow, or agent session. Do not promote those dimensions into first-class columns unless the ledger itself needs them for provider-agnostic billing behavior.
 
-Hot-path guardrails must not aggregate over the growing call ledger. ActiveRecord period budgets should read maintained rows in `llm_cost_tracker_period_totals`; dashboard analytics may run grouped queries because they are user-initiated reporting paths.
+Hot-path guardrails must not aggregate over the growing call ledger. ActiveRecord period budgets should read maintained rows in `llm_cost_tracker_period_totals`; dashboard analytics may run grouped queries because they are user-initiated reporting paths. Do not add dashboard-only aggregate tables until bounded indexed reads from `llm_api_calls` are no longer enough for the supported date range.
 
 ## Technical Docs
 

data/docs/configuration.md

@@ -21,7 +21,7 @@ Until this page is expanded, use:
 
 This page is scoped to:
 
-- `storage_backend`: `:log`, `:active_record`, and `:custom`
+- `storage_backend`: `:log`, `:active_record`, and `:custom`; ActiveRecord capture uses a durable inbox when the ingestion migration is present
 - `default_tags`: static tags and per-request callable tags
 - `instrument`: RubyLLM and official SDK integrations
 - `prices_file` and `pricing_overrides`

data/docs/technical/data-flow.md

@@ -41,11 +41,14 @@ This is the normal path from an application LLM call to stored ledger data.
 
 ## ActiveRecord Storage
 
-1. `Storage::ActiveRecordStore.save` converts tags for JSON or text storage.
-2. Optional fields are written only when their columns exist.
-3. The call row and period rollup updates happen in one transaction.
-4. `ActiveRecordRollups.increment!` updates daily and monthly totals atomically.
-5. Budget reads use period totals when available.
+1. `Storage::ActiveRecordInbox.save` writes a compact durable event row when the ingestion tables are present.
+2. `Storage::ActiveRecordIngestor` claims retryable inbox rows through a database lease and writes batches into `llm_api_calls`.
+3. `Storage::ActiveRecordStore.insert_many` converts tags for JSON or text storage and writes optional fields only when their columns exist.
+4. The call rows, period rollup updates, and inbox deletes happen in one transaction.
+5. `ActiveRecordRollups.increment_many!` updates daily and monthly totals only for rows inserted by the batch.
+6. Budget reads use period totals plus pending inbox totals when available.
+
+The inbox write is the durability boundary. Ledger freshness is eventually consistent unless the caller explicitly waits with `LlmCostTracker.flush!`.
 
 ## Dashboard Reads
 

data/docs/technical/operational-notes.md

@@ -34,10 +34,28 @@ Price update tasks are operational tooling. They can fetch the maintained LLM Co
 
 ## Budget Reads
 
-Monthly and daily budgets should read `llm_cost_tracker_period_totals` when the table exists. Falling back to summing `llm_api_calls` is an upgrade compatibility path, not the preferred production path.
+Monthly and daily budgets should read `llm_cost_tracker_period_totals` when the table exists and add pending `llm_cost_tracker_inbox_events` totals while durable ingestion is enabled. Falling back to summing `llm_api_calls` is an upgrade compatibility path, not the preferred production path.
+
+The stored period total and pending inbox total should be read in one database statement so request-time budget checks do not undercount during the inbox-to-ledger handoff.
 
 Per-call budgets are checked from the current event only.
 
+## Durable Ingestion
+
+Inbox writes inside an open caller transaction need a separate database connection to survive caller rollbacks. If the pool cannot provide one, storage should fail honestly through `storage_error_behavior` instead of writing into the caller transaction and pretending the event is durable.
+
+The ingestor is database-leased and database-polled, with an opportunistic local wake after a successful inbox insert. The wake only reduces freshness latency in the process that wrote the row; correctness still comes from the shared database lease, retryable row locks, and adaptive polling across Puma, Sidekiq, Unicorn, deploy restarts, and multi-process hosts.
+
+Freshness and durability are separate concerns. If the writing process exits before its local ingestor drains the row, another process can pick it up on a later poll; budget reads include pending inbox totals and operators can call `LlmCostTracker.flush!` when they need the ledger drained before continuing.
+
+The ingestor should check for claimable rows before acquiring the leader lease. Empty queues should not create steady lease-table writes across an idle fleet.
+
+Batch size is a conservative internal constant. Do not expose it as a configuration knob until production measurements show that a supported workload needs tuning; more knobs make installations harder to reason about.
+
+Ingestors should claim only retryable rows. Rows that keep failing after the retry cap stay in `llm_cost_tracker_inbox_events` with `last_error` for operator inspection and must not block healthy rows behind them.
+
+Process shutdown should stop the local ingestor thread without forcing every exiting process to drain the shared inbox. Operators can call `LlmCostTracker.flush!` when they intentionally want to wait for the durable inbox to drain.
+
 ## Retention
 
 Retention may delete old `llm_api_calls`. Period rollups are the durable budget aggregate. Any migration or refactor that changes rollups must preserve the meaning of retained totals or clearly document a breaking change.
@@ -60,6 +78,8 @@ Dashboard queries can aggregate because they are user-initiated. They should sti
 
 Avoid loading ledger rows into Ruby just to count, sum, group, or sort.
 
+The dashboard is not the center of the storage design. Prefer bounded ranges, existing ledger indexes, pagination, and database-side aggregates over new dashboard-specific tables. Add a summary table only when a measured supported dashboard query cannot be made acceptable with the existing ledger and period totals.
+
 ## Streaming
 
 Streaming capture must keep the host app's stream behavior intact.

data/docs/upgrading.md

@@ -20,6 +20,7 @@ Existing installs can add newer optional columns through focused generators:
 
 ```bash
 bin/rails generate llm_cost_tracker:add_period_totals
+bin/rails generate llm_cost_tracker:add_ingestion
 bin/rails generate llm_cost_tracker:add_streaming
 bin/rails generate llm_cost_tracker:add_provider_response_id
 bin/rails generate llm_cost_tracker:add_usage_breakdown

data/lib/llm_cost_tracker/active_record_adapter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module LlmCostTracker
+  module ActiveRecordAdapter
+    MYSQL_ADAPTERS = %w[
+      ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter
+      ActiveRecord::ConnectionAdapters::Mysql2Adapter
+      ActiveRecord::ConnectionAdapters::TrilogyAdapter
+    ].freeze
+    POSTGRESQL_ADAPTERS = %w[
+      ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+    ].freeze
+    SQLITE_ADAPTERS = %w[
+      ActiveRecord::ConnectionAdapters::SQLite3Adapter
+    ].freeze
+    MYSQL_PATTERN = /mysql|trilogy|mariadb/i
+    POSTGRESQL_PATTERN = /postgres/i
+    SQLITE_PATTERN = /sqlite/i
+
+    class << self
+      def mysql?(value) = adapter_instance?(value, MYSQL_ADAPTERS) || adapter_name(value).match?(MYSQL_PATTERN)
+
+      def postgresql?(value)
+        adapter_instance?(value, POSTGRESQL_ADAPTERS) || adapter_name(value).match?(POSTGRESQL_PATTERN)
+      end
+
+      def sqlite?(value) = adapter_instance?(value, SQLITE_ADAPTERS) || adapter_name(value).match?(SQLITE_PATTERN)
+
+      private
+
+      def adapter_instance?(value, class_names)
+        class_names.any? do |class_name|
+          adapter_class = constantize(class_name)
+          adapter_class && value.is_a?(adapter_class)
+        end
+      end
+
+      def constantize(name)
+        name.split("::").reduce(Object) { |namespace, part| namespace.const_get(part, false) }
+      rescue NameError
+        nil
+      end
+
+      def adapter_name(value)
+        value.respond_to?(:adapter_name) ? value.adapter_name.to_s : value.to_s
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor/ingestion_check.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "time"
+
+module LlmCostTracker
+  class Doctor
+    class IngestionCheck
+      PENDING_AGE_WARNING_SECONDS = 60
+
+      def self.call(check_class)
+        new(check_class).call
+      end
+
+      def initialize(check_class)
+        @check_class = check_class
+      end
+
+      def call
+        return unless active_record_storage? && llm_api_calls_table?
+
+        missing = missing_parts
+        if missing.empty?
+          quarantined = quarantined_count
+          if quarantined.positive?
+            return check_class.new(:warn, "durable ingestion", "#{quarantined} inbox events quarantined after retries")
+          end
+
+          pending = pending_snapshot
+          if stale_pending?(pending)
+            return check_class.new(
+              :warn,
+              "durable ingestion",
+              "#{pending.fetch(:count)} inbox events pending; oldest pending age #{pending_age(pending).round}s"
+            )
+          end
+
+          return check_class.new(:ok, "durable ingestion", "inbox and ingestor lease tables available")
+        end
+
+        check_class.new(
+          :warn,
+          "durable ingestion",
+          "missing #{missing.join(', ')}; run bin/rails generate llm_cost_tracker:add_ingestion && bin/rails db:migrate"
+        )
+      end
+
+      private
+
+      attr_reader :check_class
+
+      def missing_parts
+        [
+          column_names("llm_api_calls").include?("event_id") ? nil : "llm_api_calls.event_id",
+          table_exists?("llm_cost_tracker_inbox_events") ? nil : "llm_cost_tracker_inbox_events",
+          table_exists?("llm_cost_tracker_ingestor_leases") ? nil : "llm_cost_tracker_ingestor_leases"
+        ].compact
+      end
+
+      def active_record_storage? = LlmCostTracker.configuration.storage_backend == :active_record
+
+      def llm_api_calls_table? = table_exists?("llm_api_calls")
+
+      def table_exists?(name)
+        LlmCostTracker::LlmApiCall.connection.data_source_exists?(name)
+      rescue StandardError
+        false
+      end
+
+      def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
+
+      def quarantined_count
+        return 0 unless table_exists?("llm_cost_tracker_inbox_events")
+
+        LlmCostTracker::LlmApiCall.connection.select_value(quarantined_sql).to_i
+      rescue StandardError
+        0
+      end
+
+      def quarantined_sql
+        table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
+        "SELECT COUNT(*) FROM #{table} WHERE attempts >= #{max_attempts}"
+      end
+
+      def pending_snapshot
+        row = LlmCostTracker::LlmApiCall.connection.select_one(pending_sql) || {}
+        {
+          count: row.fetch("pending_count").to_i,
+          oldest_at: row["oldest_created_at"] && Time.parse(row.fetch("oldest_created_at").to_s).utc
+        }
+      rescue StandardError
+        { count: 0, oldest_at: nil }
+      end
+
+      def pending_sql
+        table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
+        "SELECT COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at " \
+          "FROM #{table} WHERE attempts < #{max_attempts}"
+      end
+
+      def stale_pending?(pending)
+        pending.fetch(:count).positive? &&
+          pending.fetch(:oldest_at) &&
+          pending_age(pending) >= PENDING_AGE_WARNING_SECONDS
+      end
+
+      def pending_age(pending) = Time.now.utc - pending.fetch(:oldest_at)
+
+      def max_attempts
+        if defined?(LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS)
+          LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS
+        else
+          5
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/doctor.rb

@@ -2,6 +2,7 @@
 
 require_relative "price_freshness"
 require_relative "doctor/capture_check"
+require_relative "doctor/ingestion_check"
 
 module LlmCostTracker
   class Doctor
@@ -45,6 +46,7 @@ module LlmCostTracker
         table_check,
         column_check,
         period_totals_check,
+        IngestionCheck.call(Check),
         prices_check,
         calls_check
       ].compact

data/lib/llm_cost_tracker/event.rb

@@ -2,6 +2,7 @@
 
 module LlmCostTracker
   Event = Data.define(
+    :event_id,
     :provider,
     :model,
     :input_tokens,

data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_ingestion_generator.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module LlmCostTracker
+  module Generators
+    class AddIngestionGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Creates a migration to add durable ActiveRecord ingestion"
+
+      def create_migration_file
+        migration_template(
+          "add_ingestion_to_llm_cost_tracker.rb.erb",
+          "db/migrate/add_ingestion_to_llm_cost_tracker.rb"
+        )
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_ingestion_to_llm_cost_tracker.rb.erb

@@ -0,0 +1,33 @@
+class AddIngestionToLlmCostTracker < ActiveRecord::Migration<%= migration_version %>
+  def change
+    add_column :llm_api_calls, :event_id, :string unless column_exists?(:llm_api_calls, :event_id)
+    add_index :llm_api_calls, :event_id, unique: true if column_exists?(:llm_api_calls, :event_id) &&
+                                                         !index_exists?(:llm_api_calls, :event_id)
+
+    create_table :llm_cost_tracker_inbox_events do |t|
+      t.string   :event_id, null: false
+      t.decimal  :total_cost, precision: 20, scale: 8
+      t.datetime :tracked_at, null: false
+      t.text     :payload, null: false
+      t.datetime :locked_at
+      t.string   :locked_by
+      t.integer  :attempts, null: false, default: 0
+      t.text     :last_error
+
+      t.timestamps
+    end unless table_exists?(:llm_cost_tracker_inbox_events)
+
+    create_table :llm_cost_tracker_ingestor_leases do |t|
+      t.string   :name, null: false
+      t.string   :locked_by
+      t.datetime :locked_until
+
+      t.timestamps
+    end unless table_exists?(:llm_cost_tracker_ingestor_leases)
+
+    add_index :llm_cost_tracker_inbox_events, :event_id, unique: true unless index_exists?(:llm_cost_tracker_inbox_events, :event_id)
+    add_index :llm_cost_tracker_inbox_events, :tracked_at unless index_exists?(:llm_cost_tracker_inbox_events, :tracked_at)
+    add_index :llm_cost_tracker_inbox_events, [:locked_at, :id] unless index_exists?(:llm_cost_tracker_inbox_events, [:locked_at, :id])
+    add_index :llm_cost_tracker_ingestor_leases, :name, unique: true unless index_exists?(:llm_cost_tracker_ingestor_leases, :name)
+  end
+end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb

@@ -1,3 +1,5 @@
+require "llm_cost_tracker/active_record_adapter"
+
 class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_version %>
   def up
     create_table :llm_cost_tracker_period_totals do |t|
@@ -73,10 +75,9 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
   end
 
   def day_bucket_sql
-    case connection.adapter_name
-    when /postgres/i
+    if postgresql?
       "DATE_TRUNC('day', tracked_at)::date"
-    when /mysql/i
+    elsif mysql?
       "DATE(tracked_at)"
     else
       "date(tracked_at)"
@@ -84,13 +85,20 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
   end
 
   def month_bucket_sql
-    case connection.adapter_name
-    when /postgres/i
+    if postgresql?
       "DATE_TRUNC('month', tracked_at)::date"
-    when /mysql/i
+    elsif mysql?
       "DATE_FORMAT(tracked_at, '%Y-%m-01')"
     else
       "strftime('%Y-%m-01', tracked_at)"
     end
   end
+
+  def postgresql?
+    LlmCostTracker::ActiveRecordAdapter.postgresql?(connection)
+  end
+
+  def mysql?
+    LlmCostTracker::ActiveRecordAdapter.mysql?(connection)
+  end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_streaming_to_llm_api_calls.rb.erb

@@ -2,23 +2,19 @@ class AddStreamingToLlmApiCalls < ActiveRecord::Migration<%= migration_version %
   def up
     unless column_exists?(:llm_api_calls, :stream)
       add_column :llm_api_calls, :stream, :boolean, null: false, default: false
-      add_index  :llm_api_calls, :stream
     end
 
     unless column_exists?(:llm_api_calls, :usage_source)
       add_column :llm_api_calls, :usage_source, :string
-      add_index  :llm_api_calls, :usage_source
     end
   end
 
   def down
     if column_exists?(:llm_api_calls, :usage_source)
-      remove_index  :llm_api_calls, :usage_source if index_exists?(:llm_api_calls, :usage_source)
       remove_column :llm_api_calls, :usage_source
     end
 
     if column_exists?(:llm_api_calls, :stream)
-      remove_index  :llm_api_calls, :stream if index_exists?(:llm_api_calls, :stream)
       remove_column :llm_api_calls, :stream
     end
   end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb

@@ -1,6 +1,9 @@
+require "llm_cost_tracker/active_record_adapter"
+
 class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
   def change
     create_table :llm_api_calls do |t|
+      t.string  :event_id,      null: false
       t.string  :provider,      null: false
       t.string  :model,         null: false
       t.integer :input_tokens,  null: false, default: 0
@@ -37,19 +40,43 @@ class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
       t.timestamps
     end
 
+    create_table :llm_cost_tracker_inbox_events do |t|
+      t.string   :event_id, null: false
+      t.decimal  :total_cost, precision: 20, scale: 8
+      t.datetime :tracked_at, null: false
+      t.text     :payload, null: false
+      t.datetime :locked_at
+      t.string   :locked_by
+      t.integer  :attempts, null: false, default: 0
+      t.text     :last_error
+
+      t.timestamps
+    end
+
+    create_table :llm_cost_tracker_ingestor_leases do |t|
+      t.string   :name, null: false
+      t.string   :locked_by
+      t.datetime :locked_until
+
+      t.timestamps
+    end
+
+    add_index :llm_api_calls, :event_id, unique: true
     add_index :llm_api_calls, :tracked_at
     add_index :llm_api_calls, [:provider, :tracked_at]
     add_index :llm_api_calls, [:model, :tracked_at]
-    add_index :llm_api_calls, :stream
-    add_index :llm_api_calls, :usage_source
     add_index :llm_api_calls, :provider_response_id
     add_index :llm_api_calls, :tags, using: :gin if postgresql?
     add_index :llm_cost_tracker_period_totals, [:period, :period_start], unique: true
+    add_index :llm_cost_tracker_inbox_events, :event_id, unique: true
+    add_index :llm_cost_tracker_inbox_events, :tracked_at
+    add_index :llm_cost_tracker_inbox_events, [:locked_at, :id]
+    add_index :llm_cost_tracker_ingestor_leases, :name, unique: true
   end
 
   private
 
   def postgresql?
-    connection.adapter_name.downcase.include?("postgres")
+    LlmCostTracker::ActiveRecordAdapter.postgresql?(connection)
   end
 end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb

@@ -4,7 +4,7 @@ LlmCostTracker.configure do |config|
   # Set to false to temporarily disable tracking without removing middleware.
   config.enabled = true
 
-  # :active_record stores events in llm_api_calls for dashboards, reports, and shared budgets.
+  # :active_record captures events into a durable inbox and ingests them into llm_api_calls.
   # Other options: :log for local logging, :custom for your own storage callable.
   config.storage_backend = :active_record
 

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb

@@ -1,3 +1,5 @@
+require "llm_cost_tracker/active_record_adapter"
+
 class UpgradeLlmApiCallTagsToJsonb < ActiveRecord::Migration<%= migration_version %>
   def up
     unless postgresql?
@@ -32,7 +34,7 @@ class UpgradeLlmApiCallTagsToJsonb < ActiveRecord::Migration<%= migration_versio
   private
 
   def postgresql?
-    connection.adapter_name.downcase.include?("postgres")
+    LlmCostTracker::ActiveRecordAdapter.postgresql?(connection)
   end
 
   def tags_jsonb?