llm_cost_tracker 0.11.0 → 0.12.0

data/lib/llm_cost_tracker/billing/components.rb

@@ -1,95 +0,0 @@
-# frozen_string_literal: true
-
-require "psych"
-
-require_relative "../errors"
-
-module LlmCostTracker
-  module Billing
-    RATE_BASES = %i[
-      per_million_tokens
-      per_million_characters
-      per_request
-      per_1k_requests
-      per_session
-      per_hour
-      per_gb_day
-      per_image
-    ].freeze
-
-    RATE_BASIS_QUANTITIES = {
-      per_million_tokens: 1_000_000,
-      per_million_characters: 1_000_000,
-      per_request: 1,
-      per_1k_requests: 1_000,
-      per_session: 1,
-      per_hour: 1,
-      per_gb_day: 1,
-      per_image: 1
-    }.freeze
-
-    DEFAULT_RATE_BASIS_BY_UNIT = {
-      token: :per_million_tokens,
-      character: :per_million_characters,
-      request: :per_request,
-      session: :per_session,
-      hour: :per_hour,
-      image: :per_image
-    }.freeze
-
-    module Components
-      Component = Data.define(
-        :key,
-        :kind,
-        :direction,
-        :modality,
-        :cache_state,
-        :unit,
-        :category,
-        :token_key,
-        :cost_key,
-        :rate_basis
-      )
-
-      REQUIRED_FIELDS = %i[key kind direction modality cache_state unit category].freeze
-      DEFINITIONS_PATH = File.expand_path("components.yml", __dir__)
-
-      def self.load_registry
-        Psych.safe_load_file(DEFINITIONS_PATH, permitted_classes: [], symbolize_names: true)
-             .map { |attributes| build(attributes) }
-             .freeze
-      end
-
-      def self.build(attributes)
-        missing = REQUIRED_FIELDS - attributes.keys
-        raise Error, "components.yml entry missing #{missing.join(', ')}: #{attributes.inspect}" if missing.any?
-
-        unit = attributes.fetch(:unit).to_sym
-        rate_basis = attributes[:rate_basis]&.to_sym || Billing::DEFAULT_RATE_BASIS_BY_UNIT[unit]
-        if rate_basis.nil?
-          raise Error, "components.yml entry needs rate_basis for unit #{unit.inspect}: #{attributes.inspect}"
-        end
-        unless Billing::RATE_BASES.include?(rate_basis)
-          raise Error, "components.yml entry has unknown rate_basis #{rate_basis.inspect}: #{attributes.inspect}"
-        end
-
-        Component.new(
-          key: attributes.fetch(:key).to_sym,
-          kind: attributes.fetch(:kind).to_sym,
-          direction: attributes.fetch(:direction).to_sym,
-          modality: attributes.fetch(:modality).to_sym,
-          cache_state: attributes.fetch(:cache_state).to_sym,
-          unit: unit,
-          category: attributes.fetch(:category).to_sym,
-          token_key: attributes[:token_key]&.to_sym,
-          cost_key: attributes[:cost_key]&.to_sym,
-          rate_basis: rate_basis
-        )
-      end
-
-      REGISTRY = load_registry
-      BY_KEY = REGISTRY.to_h { |component| [component.key, component] }.freeze
-      TOKEN_PRICED = REGISTRY.select { |component| component.token_key && component.cost_key }.freeze
-    end
-  end
-end

data/lib/llm_cost_tracker/capture/stream.rb

@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module LlmCostTracker
-  module Capture
-    module Stream
-      LIMIT_BYTES = 1_048_576
-    end
-  end
-end

data/lib/llm_cost_tracker/doctor/capture_verifier.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "check"
-require_relative "../ingestion"
-
-module LlmCostTracker
-  class Doctor
-    class CaptureVerifier
-      class << self
-        def call
-          new.checks
-        end
-
-        def report(checks = call)
-          (["LLM Cost Tracker capture verification"] + checks.map do |check|
-            "[#{check.status}] #{check.name}: #{check.message}"
-          end).join("\n")
-        end
-
-        def healthy?(checks = call)
-          checks.none? { |check| check.status == :error }
-        end
-      end
-
-      def checks
-        [
-          enabled_check,
-          *integration_checks,
-          *storage_checks
-        ].compact
-      end
-
-      private
-
-      def enabled_check
-        return Check.new(:ok, "tracking", "enabled") if LlmCostTracker.configuration.enabled
-
-        Check.new(:error, "tracking", "disabled; set config.enabled = true before verifying capture")
-      end
-
-      def integration_checks
-        enabled = LlmCostTracker.configuration.instrumented_integrations
-        if enabled.empty?
-          return [
-            Check.new(:ok, "sdk integrations", "none enabled; Faraday middleware and manual capture remain available")
-          ]
-        end
-
-        LlmCostTracker::Integrations.checks.map do |check|
-          check.with(name: "sdk integration #{check.name}")
-        end
-      end
-
-      def storage_checks
-        LlmCostTracker::Ingestion.verify
-      rescue LlmCostTracker::Error => e
-        [Check.new(:error, "storage", e.message)]
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/doctor/check.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module LlmCostTracker
-  class Doctor
-    Check = Data.define(:status, :name, :message)
-  end
-end

data/lib/llm_cost_tracker/doctor/cost_drift_check.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require "bigdecimal"
-
-require_relative "check"
-require_relative "probe"
-require_relative "../ledger/rollups"
-
-module LlmCostTracker
-  class Doctor
-    class CostDriftCheck
-      SAMPLE_SIZE = 200
-      EPSILON = BigDecimal("0.00000001")
-
-      def call
-        return unless Probe.table_exists?("llm_cost_tracker_calls")
-        return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
-
-        sampled = LlmCostTracker::Call
-                  .where.not(total_cost: nil)
-                  .where(cost_status: %w[complete free partial])
-                  .order(id: :desc)
-                  .limit(SAMPLE_SIZE)
-                  .pluck(:id, :total_cost, :cost_status)
-        return Check.new(:ok, "cost drift", "no priced calls to inspect") if sampled.empty?
-
-        line_item_totals = LlmCostTracker::CallLineItem
-                           .where(llm_cost_tracker_call_id: sampled.map(&:first))
-                           .where(currency: Ledger::Rollups::DEFAULT_CURRENCY)
-                           .group(:llm_cost_tracker_call_id)
-                           .sum(:cost)
-
-        drifted = sampled.filter_map do |id, total_cost, cost_status|
-          line_total = line_item_totals[id] || BigDecimal("0")
-          header = BigDecimal(total_cost.to_s)
-          next if cost_status == "partial" && header >= line_total
-          next if (header - line_total).abs <= EPSILON
-
-          "##{id}: header=#{header.to_s('F')} line_items=#{line_total.to_s('F')}"
-        end
-
-        if drifted.empty?
-          return Check.new(:ok, "cost drift",
-                           "header total_cost matches line items in #{sampled.size} sampled calls")
-        end
-
-        Check.new(
-          :warn,
-          "cost drift",
-          "header total_cost diverges from line items in #{drifted.size}/#{sampled.size} sampled calls: " \
-          "#{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
-        )
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/doctor/invoice_reconciliation_check.rb

@@ -1,164 +0,0 @@
-# frozen_string_literal: true
-
-require "bigdecimal"
-
-require_relative "check"
-require_relative "probe"
-require_relative "../ledger/schema/adapter"
-
-module LlmCostTracker
-  class Doctor
-    class InvoiceReconciliationCheck
-      def call
-        return unless LlmCostTracker.reconciliation_enabled?
-        return unless Probe.table_exists?("llm_cost_tracker_provider_invoices")
-        return if no_imports?
-
-        scopes = imported_scopes
-        return Check.new(:ok, "invoice reconciliation", "no provider invoices imported yet") if scopes.empty?
-
-        non_canonical = non_canonical_currency_check
-        checks = scopes.map { |scope| check_scope_safely(scope) }
-        checks << non_canonical if non_canonical
-        checks
-      rescue StandardError => e
-        Check.new(:error, "invoice reconciliation", e.message)
-      end
-
-      private
-
-      def no_imports?
-        LlmCostTracker::ProviderInvoice.none?
-      end
-
-      def non_canonical_currency_check
-        legacy = LlmCostTracker::ProviderInvoice.where("currency <> UPPER(currency)").count
-        return nil if legacy.zero?
-
-        Check.new(
-          :warn,
-          "invoice reconciliation: currency canonicalisation",
-          "#{legacy} provider invoice row(s) stored with non-uppercase currency. Diff queries " \
-          "are case-sensitive — run " \
-          "`UPDATE llm_cost_tracker_provider_invoices SET currency = UPPER(currency);` to backfill."
-        )
-      end
-
-      def threshold
-        Reconciliation::DEFAULT_THRESHOLD_PERCENT
-      end
-
-      def imported_scopes
-        connection = LlmCostTracker::ProviderInvoice.connection
-        provider_expr =
-          if Ledger::Schema::Adapter.postgresql?(connection)
-            Arel.sql("metadata->>'provider'")
-          else
-            Arel.sql("JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.provider'))")
-          end
-        LlmCostTracker::ProviderInvoice
-          .group(:source, provider_expr, :currency)
-          .order(:source, :currency)
-          .pluck(:source, provider_expr, :currency)
-          .map { |source, provider, currency| { source: source, provider: provider, currency: currency.upcase } }
-      end
-
-      def scope_label(scope)
-        "#{scope[:source]}/#{scope[:provider]}/#{scope[:currency]}"
-      end
-
-      def check_scope_safely(scope)
-        check_scope(scope)
-      rescue ArgumentError => e
-        Check.new(:warn, "invoice reconciliation: #{scope_label(scope)}", e.message)
-      end
-
-      def check_scope(scope)
-        window = latest_window_for(scope)
-        return stale_check(scope) if window.nil?
-
-        diff = run_diff(scope, window)
-        return ok_check(scope, window, diff) if diff.aligned?(threshold_percent: threshold)
-
-        warn_check(scope, window, diff)
-      end
-
-      def scope_relation(scope)
-        relation = LlmCostTracker::ProviderInvoice
-                   .where(source: scope[:source], currency: scope[:currency])
-        provider = scope[:provider]
-        return relation if provider.nil? || provider.to_s.empty?
-
-        connection = LlmCostTracker::ProviderInvoice.connection
-        if Ledger::Schema::Adapter.postgresql?(connection)
-          relation.where("metadata->>'provider' = ?", provider)
-        else
-          relation.where("JSON_UNQUOTE(JSON_EXTRACT(metadata, '$.provider')) = ?", provider)
-        end
-      end
-
-      def latest_window_for(scope)
-        latest = scope_relation(scope)
-                 .select(:period_start, :period_end)
-                 .order(period_end: :desc, period_start: :desc)
-                 .limit(1)
-                 .first
-        return nil unless latest
-        return nil if (Time.now.utc.to_date - latest.period_end).to_i > Reconciliation::INVOICE_FRESHNESS_DAYS
-
-        latest
-      end
-
-      def run_diff(scope, window)
-        Reconciliation.diff(
-          source: scope[:source],
-          provider: scope[:provider],
-          currency: scope[:currency],
-          period_start: window.period_start,
-          period_end: window.period_end
-        )
-      end
-
-      def stale_check(scope)
-        latest = scope_relation(scope).maximum(:period_end)
-        return scope_unreachable_check(scope) if latest.nil?
-
-        days = (Time.now.utc.to_date - latest).to_i
-        Check.new(
-          :warn,
-          "invoice reconciliation: #{scope_label(scope)}",
-          "no invoice imported in #{days} days (threshold #{Reconciliation::INVOICE_FRESHNESS_DAYS} days); " \
-          "run reconciliation import"
-        )
-      end
-
-      def scope_unreachable_check(scope)
-        Check.new(
-          :warn,
-          "invoice reconciliation: #{scope_label(scope)}",
-          "scope grouped invoices but the filter (likely currency case mismatch) matches zero rows; " \
-          "the currency-canonicalisation check below points at the backfill SQL"
-        )
-      end
-
-      def ok_check(scope, window, diff)
-        Check.new(
-          :ok,
-          "invoice reconciliation: #{scope_label(scope)}",
-          "#{window.period_start}..#{window.period_end} aligned " \
-          "(local=#{diff.local_total.to_s('F')}, provider=#{diff.provider_total.to_s('F')})"
-        )
-      end
-
-      def warn_check(scope, window, diff)
-        Check.new(
-          :warn,
-          "invoice reconciliation: #{scope_label(scope)}",
-          "#{window.period_start}..#{window.period_end} drift " \
-          "delta=#{diff.delta_amount.to_s('F')} (#{diff.delta_percent}%) " \
-          "exceeds #{threshold}% threshold"
-        )
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "check"
-require_relative "probe"
-require_relative "../ledger"
-
-module LlmCostTracker
-  class Doctor
-    class LegacyAuditCheck
-      WARNING_PERCENT = 10
-
-      def call
-        return unless Probe.table_exists?("llm_cost_tracker_calls")
-        return unless LlmCostTracker::Call.column_names.include?("pricing_snapshot")
-
-        counts = LlmCostTracker::Call
-                 .select(
-                   "COUNT(*) AS total_count, " \
-                   "COALESCE(SUM(CASE WHEN pricing_snapshot IS NULL THEN 1 ELSE 0 END), 0) AS missing_count"
-                 )
-                 .take
-        total = counts.total_count.to_i
-        return if total.zero?
-
-        missing = counts.missing_count.to_i
-        return unless (missing * 100) > (total * WARNING_PERCENT)
-
-        message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
-                  "stored totals remain stable but applied rates cannot be audited"
-        Check.new(:warn, "pricing snapshot audit", message)
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "check"
-require_relative "probe"
-require_relative "../ledger"
-
-module LlmCostTracker
-  class Doctor
-    class LegacyBillingStatusCheck
-      def call
-        return unless Probe.table_exists?("llm_cost_tracker_calls")
-        return unless LlmCostTracker::Call.column_names.include?("cost_status")
-
-        return unless LlmCostTracker::Call.where(cost_status: nil).exists?
-
-        Check.new(:warn, "cost status", "legacy rows without cost_status remain; new rows will populate it")
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/doctor/pricing_snapshot_drift_check.rb

@@ -1,85 +0,0 @@
-# frozen_string_literal: true
-
-require "bigdecimal"
-
-require_relative "check"
-require_relative "probe"
-
-module LlmCostTracker
-  class Doctor
-    class PricingSnapshotDriftCheck
-      SAMPLE_SIZE = 200
-      EPSILON = BigDecimal("0.00000001")
-
-      def call
-        return unless Probe.table_exists?("llm_cost_tracker_calls")
-        return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
-
-        sampled_ids = LlmCostTracker::Call
-                      .where.not(pricing_snapshot: nil)
-                      .where(cost_status: %w[complete free])
-                      .order(id: :desc)
-                      .limit(SAMPLE_SIZE)
-                      .pluck(:id)
-        return Check.new(:ok, "pricing snapshot drift", "no snapshotted calls to inspect") if sampled_ids.empty?
-
-        calls_by_id = LlmCostTracker::Call.where(id: sampled_ids).index_by(&:id)
-        line_items_by_call = LlmCostTracker::CallLineItem
-                             .where(llm_cost_tracker_call_id: sampled_ids, unit: "token")
-                             .group_by(&:llm_cost_tracker_call_id)
-
-        drifted = sampled_ids.flat_map do |id|
-          call = calls_by_id[id]
-          rates = rates_for(call.pricing_snapshot)
-          next [] if rates.nil? || rates.empty?
-
-          (line_items_by_call[id] || []).filter_map { |item| drift_message_for(item, rates, call_id: id) }
-        end
-
-        return ok_check(sampled_ids.size) if drifted.empty?
-
-        Check.new(
-          :warn,
-          "pricing snapshot drift",
-          "line item cost diverges from pricing_snapshot rate in #{drifted.size} cases across " \
-          "#{sampled_ids.size} sampled calls: #{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
-        )
-      end
-
-      private
-
-      def ok_check(sample_size)
-        Check.new(:ok, "pricing snapshot drift",
-                  "line item costs match pricing_snapshot rates in #{sample_size} sampled calls")
-      end
-
-      def rates_for(snapshot)
-        rates = snapshot.is_a?(Hash) ? (snapshot["rates"] || snapshot[:rates]) : nil
-        rates.is_a?(Hash) ? rates : nil
-      end
-
-      def drift_message_for(line_item, rates, call_id:)
-        return nil unless line_item.price_key
-
-        rate = rates[line_item.price_key.to_s] || rates[line_item.price_key.to_sym]
-        return nil unless rate.is_a?(Hash)
-
-        rate_amount = decimal(rate["amount"] || rate[:amount])
-        rate_quantity = decimal(rate["quantity"] || rate[:quantity])
-        return nil if rate_amount.nil? || rate_quantity.nil? || rate_quantity.zero?
-
-        expected = (decimal(line_item.quantity) * rate_amount) / rate_quantity
-        actual = decimal(line_item.cost) || BigDecimal("0")
-        return nil if (expected - actual).abs <= EPSILON
-
-        "##{call_id}.#{line_item.price_key}: expected=#{expected.round(8).to_s('F')} stored=#{actual.to_s('F')}"
-      end
-
-      def decimal(value)
-        return nil if value.nil?
-
-        BigDecimal(value.to_s)
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/reconciliation_generator.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require "rails/generators"
-require "rails/generators/active_record"
-
-module LlmCostTracker
-  module Generators
-    class ReconciliationGenerator < Rails::Generators::Base
-      include ActiveRecord::Generators::Migration
-
-      source_root File.expand_path("templates", __dir__)
-
-      desc "Creates the optional invoice reconciliation tables. Requires provider admin/org-level API keys."
-
-      def create_migration_file
-        migration_template(
-          "create_llm_cost_tracker_reconciliation.rb.erb",
-          "db/migrate/create_llm_cost_tracker_reconciliation.rb"
-        )
-      end
-
-      def warn_about_admin_keys
-        say "Reconciliation requires admin/org-level API keys (OpenAI sk-admin-..., Anthropic admin keys, " \
-            "GCP billing.viewer service accounts). Do NOT use the runtime inference key.", :yellow
-      end
-
-      private
-
-      def migration_version
-        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb

@@ -1,60 +0,0 @@
-require "llm_cost_tracker/ledger/schema/adapter"
-
-class CreateLlmCostTrackerReconciliation < ActiveRecord::Migration<%= migration_version %>
-  def change
-    create_table :llm_cost_tracker_provider_invoices, if_not_exists: true do |t|
-      t.string :source, null: false
-      t.date :period_start, null: false
-      t.date :period_end, null: false
-      t.string :external_id, null: false
-      t.decimal :billed_amount, precision: 20, scale: 8
-      t.string :currency, null: false, default: "USD"
-      if postgresql?
-        t.jsonb :metadata, null: false, default: {}
-      elsif mysql?
-        t.json :metadata, null: false
-      else
-        raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
-      end
-      t.datetime :imported_at, null: false
-
-      t.timestamps
-    end
-
-    create_table :llm_cost_tracker_provider_invoice_imports, if_not_exists: true do |t|
-      t.string :source, null: false
-      t.string :provider, null: false, default: ""
-      t.string :cursor
-      t.date :window_start
-      t.date :window_end
-      t.string :state, null: false
-      t.text :last_error
-      t.integer :rows_imported, null: false, default: 0
-      t.datetime :started_at, null: false
-      t.datetime :finished_at
-
-      t.timestamps
-    end
-
-    add_index :llm_cost_tracker_provider_invoices, :external_id, unique: true,
-              if_not_exists: true
-    add_index :llm_cost_tracker_provider_invoices, %i[source currency period_start],
-              if_not_exists: true
-    if postgresql?
-      add_index :llm_cost_tracker_provider_invoices, :metadata, using: :gin,
-                if_not_exists: true
-    end
-    add_index :llm_cost_tracker_provider_invoice_imports, %i[source provider started_at],
-              if_not_exists: true
-  end
-
-  private
-
-  def postgresql?
-    LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
-  end
-
-  def mysql?
-    LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
-  end
-end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoice_imports_provider.rb.erb

@@ -1,36 +0,0 @@
-require "llm_cost_tracker/ledger/schema/adapter"
-
-class UpgradeLlmCostTrackerProviderInvoiceImportsProvider < ActiveRecord::Migration<%= migration_version %>
-  disable_ddl_transaction!
-
-  TABLE = :llm_cost_tracker_provider_invoice_imports
-  OLD_INDEX = %i[source started_at].freeze
-  NEW_INDEX = %i[source provider started_at].freeze
-
-  def up
-    return unless table_exists?(TABLE)
-
-    add_column TABLE, :provider, :string, null: false, default: "" unless column_exists?(TABLE, :provider)
-    if postgresql?
-      remove_index TABLE, column: OLD_INDEX, algorithm: :concurrently, if_exists: true
-      add_index TABLE, NEW_INDEX, algorithm: :concurrently, if_not_exists: true
-    else
-      remove_index TABLE, column: OLD_INDEX, if_exists: true
-      add_index TABLE, NEW_INDEX, if_not_exists: true
-    end
-  end
-
-  def down
-    return unless table_exists?(TABLE)
-
-    remove_index TABLE, column: NEW_INDEX, if_exists: true
-    add_index TABLE, OLD_INDEX, if_not_exists: true
-    remove_column TABLE, :provider if column_exists?(TABLE, :provider)
-  end
-
-  private
-
-  def postgresql?
-    LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
-  end
-end