llm_cost_tracker 0.8.0 → 0.9.0

data/lib/llm_cost_tracker/reconciliation/importer.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+require "date"
+require "json"
+
+require_relative "import_result"
+require_relative "../ledger/rollups"
+
+module LlmCostTracker
+  module Reconciliation
+    class Importer
+      REQUIRED_FIELDS = %i[external_id period_start period_end].freeze
+      FORGIVING_METADATA_SOURCES = %i[csv].to_set.freeze
+      ENVELOPE_KEYS = %w[row_type meter authority match_basis].freeze
+
+      def initialize(source:, imported_at:, provider:, window: nil, strict_metadata: nil, cursor: nil)
+        @source = source.to_s
+        @provider = provider.to_s
+        @imported_at = imported_at
+        @window = coerce_window(window)
+        @cursor = cursor
+        @strict_metadata = strict_metadata.nil? ? !FORGIVING_METADATA_SOURCES.include?(source.to_sym) : strict_metadata
+        raise ArgumentError, "source must be present" if @source.empty?
+        raise ArgumentError, "provider must be present" if @provider.empty?
+      end
+
+      def call(rows)
+        import_record = nil
+        ensure_reconciliation_installed!
+        return ImportResult.empty if skippable?(rows)
+
+        import_record = open_import_record
+        result = perform_import(rows)
+        complete_import_record(import_record, result)
+        result.with(import_id: import_record&.id)
+      rescue StandardError => e
+        fail_import_record(import_record, e)
+        raise
+      end
+
+      private
+
+      attr_reader :source, :provider, :imported_at, :window, :cursor, :strict_metadata
+
+      def skippable?(rows)
+        (rows.nil? || rows.empty?) && cursor.nil?
+      end
+
+      def ensure_reconciliation_installed!
+        return if ProviderInvoice.table_exists?
+
+        raise Error,
+              "llm_cost_tracker_provider_invoices table is missing; " \
+              "run `rails generate llm_cost_tracker:reconciliation && rails db:migrate`"
+      end
+
+      def perform_import(rows)
+        return ImportResult.empty if rows.nil? || rows.empty?
+
+        normalized, errors = normalize_rows(rows)
+        if normalized.empty?
+          return ImportResult.new(inserted: 0, updated: 0, skipped: rows.size, errors: errors,
+                                  import_id: nil)
+        end
+
+        existing = existing_external_ids(normalized.map { |row| row[:external_id] })
+        rows_payload = normalized.map { |row| persistable_attributes(row) }
+        upsert_options = { record_timestamps: true }
+        upsert_options[:unique_by] = :external_id if ProviderInvoice.connection.supports_insert_conflict_target?
+        ProviderInvoice.upsert_all(rows_payload, **upsert_options)
+
+        inserted = normalized.count { |row| !existing.include?(row[:external_id]) }
+        updated = normalized.size - inserted
+        ImportResult.new(inserted: inserted, updated: updated, skipped: rows.size - normalized.size,
+                         errors: errors, import_id: nil)
+      end
+
+      def open_import_record
+        return nil unless tracking_table_present?
+
+        ProviderInvoiceImport.create!(
+          source: source,
+          cursor: cursor,
+          window_start: window&.first,
+          window_end: window&.last,
+          state: ProviderInvoiceImport::STATE_RUNNING,
+          started_at: Time.now.utc
+        )
+      end
+
+      def complete_import_record(record, result)
+        return unless record
+
+        terminal_state = result.success? ? ProviderInvoiceImport::STATE_COMPLETED : ProviderInvoiceImport::STATE_FAILED
+        record.update!(
+          state: terminal_state,
+          rows_imported: result.total_imported,
+          finished_at: Time.now.utc,
+          last_error: result.errors.first
+        )
+      end
+
+      def fail_import_record(record, error)
+        return unless record
+
+        record.update!(
+          state: ProviderInvoiceImport::STATE_FAILED,
+          last_error: "#{error.class}: #{error.message}",
+          finished_at: Time.now.utc
+        )
+      end
+
+      def tracking_table_present?
+        @tracking_table_present = ProviderInvoiceImport.table_exists? unless defined?(@tracking_table_present)
+        @tracking_table_present
+      end
+
+      def normalize_rows(rows)
+        errors = []
+        normalized = rows.each_with_index.filter_map do |row, index|
+          attrs = symbolize(row)
+          missing = REQUIRED_FIELDS - attrs.keys
+          if missing.any?
+            errors << "row #{index}: missing #{missing.join(', ')}"
+            next
+          end
+          period_start = parse_date(attrs[:period_start])
+          period_end = parse_date(attrs[:period_end])
+          next unless within_window?(period_start, period_end)
+
+          attrs.merge(
+            external_id: namespaced_external_id(attrs[:external_id]),
+            period_start: period_start,
+            period_end: period_end,
+            metadata: parse_metadata(attrs[:metadata])
+          )
+        rescue ArgumentError => e
+          errors << "row #{index}: #{e.message}"
+          nil
+        end
+        [normalized, errors]
+      end
+
+      def within_window?(period_start, period_end)
+        return true if window.nil?
+
+        period_start <= window.last && period_end >= window.first
+      end
+
+      def coerce_window(window)
+        return nil if window.nil?
+        raise ArgumentError, "window must be a Range of dates" unless window.is_a?(Range)
+
+        Range.new(parse_date(window.first), parse_date(window.last))
+      end
+
+      def existing_external_ids(external_ids)
+        ProviderInvoice.where(external_id: external_ids).pluck(:external_id).to_set
+      end
+
+      def persistable_attributes(row)
+        billed_amount = row[:billed_amount] && BigDecimal(row[:billed_amount].to_s)
+        {
+          source: source,
+          external_id: row[:external_id],
+          period_start: row[:period_start],
+          period_end: row[:period_end],
+          billed_amount: billed_amount,
+          currency: (row[:currency] || Ledger::Rollups::DEFAULT_CURRENCY).to_s.upcase,
+          metadata: stamp_metadata(row[:metadata]),
+          imported_at: imported_at || Time.now.utc
+        }
+      end
+
+      BASIS_DIMENSIONS_BY_PRIORITY = [
+        %w[project provider_project_id],
+        %w[api_key provider_api_key_id],
+        %w[workspace provider_workspace_id],
+        %w[model model]
+      ].freeze
+      private_constant :BASIS_DIMENSIONS_BY_PRIORITY
+
+      def stamp_metadata(metadata)
+        merged = metadata_with_provider(metadata)
+        metadata_with_match_basis(merged)
+      end
+
+      def metadata_with_provider(metadata)
+        return { "provider" => provider } if metadata.nil? || metadata.empty?
+
+        existing = metadata["provider"] || metadata[:provider]
+        return metadata if existing.is_a?(String) && !existing.empty?
+
+        metadata.merge("provider" => provider)
+      end
+
+      def metadata_with_match_basis(metadata)
+        existing = metadata["match_basis"] || metadata[:match_basis]
+        return metadata if existing.is_a?(String) && !existing.empty?
+
+        inferred = BASIS_DIMENSIONS_BY_PRIORITY.find { |_basis, key| metadata[key] || metadata[key.to_sym] }
+        return metadata.merge("match_basis" => "period_only") if inferred.nil?
+
+        metadata.merge("match_basis" => inferred.first)
+      end
+
+      def namespaced_external_id(external_id)
+        raw = external_id.to_s
+        scope = source == provider ? source : "#{source}/#{provider}"
+        prefix = "#{scope}:"
+        raw.start_with?(prefix) ? raw : "#{prefix}#{raw}"
+      end
+
+      def symbolize(row)
+        return row if row.is_a?(Hash) && row.keys.all?(Symbol)
+
+        row.to_h.transform_keys { |key| key.to_s.to_sym }
+      end
+
+      def parse_date(value)
+        return value if value.is_a?(Date)
+
+        Date.parse(value.to_s)
+      end
+
+      def parse_metadata(metadata)
+        parsed = parse_metadata_payload(metadata)
+        validate_envelope!(parsed) if strict_metadata
+        parsed
+      end
+
+      def parse_metadata_payload(metadata)
+        return {} if metadata.nil?
+        return metadata if metadata.is_a?(Hash)
+
+        JSON.parse(metadata.to_s)
+      rescue JSON::ParserError => e
+        raise ArgumentError, "invalid metadata JSON: #{e.message}" if strict_metadata
+
+        {}
+      end
+
+      def validate_envelope!(metadata)
+        keys = metadata.keys.map(&:to_s)
+        missing = ENVELOPE_KEYS - keys
+        return if missing.empty?
+
+        raise ArgumentError, "metadata missing envelope keys: #{missing.join(', ')}"
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "bigdecimal"
+require "json"
+require "time"
+
+require_relative "fingerprint"
+
+module LlmCostTracker
+  module Reconciliation
+    module Sources
+      module AnthropicUsage
+        FINGERPRINT_KEYS = %i[
+          starting_at ending_at model workspace_id
+          service_tier context_window cost_type token_type description
+          inference_geo
+        ].freeze
+        ROW_TYPE_COST = "cost"
+        AUTHORITY_COST_API = "cost_api"
+        DEFAULT_METER = "tokens"
+        DATA_RESIDENCY_GEOS = %w[us].freeze
+        private_constant :DATA_RESIDENCY_GEOS
+
+        module_function
+
+        def parse(response, authority: AUTHORITY_COST_API, row_type: ROW_TYPE_COST)
+          payload = coerce_hash(response)
+          buckets = Array(payload[:data])
+          buckets.flat_map do |bucket|
+            rows_for_bucket(bucket, authority: authority, row_type: row_type)
+          end.compact
+        end
+
+        def rows_for_bucket(bucket, authority:, row_type:)
+          bucket = symbolize(bucket)
+          starting_at = bucket[:starting_at]
+          ending_at = bucket[:ending_at]
+          return [] unless starting_at && ending_at
+
+          period_start = parse_date(starting_at)
+          period_end = end_inclusive_date(ending_at)
+
+          Array(bucket[:results]).filter_map do |raw|
+            row_for_result(raw,
+                           period_start: period_start, period_end: period_end,
+                           starting_at: starting_at, ending_at: ending_at,
+                           authority: authority, row_type: row_type)
+          end
+        rescue ArgumentError
+          []
+        end
+
+        def row_for_result(raw, period_start:, period_end:, starting_at:, ending_at:, authority:, row_type:)
+          result = symbolize(raw)
+          raw_amount = result[:amount]
+          return nil if raw_amount.nil?
+
+          fingerprint = fingerprint_for(result, starting_at: starting_at, ending_at: ending_at)
+          {
+            external_id: "cost-#{fingerprint}",
+            period_start: period_start,
+            period_end: period_end,
+            billed_amount: dollars_from_cents(raw_amount),
+            currency: (result[:currency] || "USD").to_s.upcase,
+            metadata: metadata_for(result, authority: authority, row_type: row_type)
+          }
+        end
+
+        def dollars_from_cents(amount)
+          (BigDecimal(amount.to_s) / 100).to_s("F")
+        end
+
+        def metadata_for(result, authority:, row_type:)
+          {
+            "row_type" => row_type,
+            "meter" => meter_for(result),
+            "authority" => authority,
+            "match_basis" => match_basis_for(result),
+            "model" => result[:model],
+            "pricing_mode" => pricing_mode_for(result),
+            "context_window" => result[:context_window],
+            "cost_type" => result[:cost_type],
+            "description" => result[:description],
+            "token_type" => result[:token_type],
+            "inference_geo" => result[:inference_geo],
+            "provider_workspace_id" => result[:workspace_id]
+          }.compact
+        end
+
+        def meter_for(result)
+          case result[:cost_type].to_s
+          when "web_search" then "web_search"
+          when "code_execution" then "code_execution_hour"
+          when "session_usage" then "session_usage"
+          when "tokens" then token_meter(result[:token_type].to_s)
+          else DEFAULT_METER
+          end
+        end
+
+        def token_meter(token_type)
+          return "cache_read_input_tokens" if token_type.include?("cache_read")
+          return "cache_creation_input_tokens" if token_type.include?("cache_creation")
+          return "input_tokens" if token_type.include?("input")
+          return "output_tokens" if token_type.include?("output")
+
+          DEFAULT_METER
+        end
+
+        def pricing_mode_for(result)
+          modes = []
+          modes << "batch" if result[:service_tier].to_s.downcase == "batch"
+          modes << "data_residency" if DATA_RESIDENCY_GEOS.include?(result[:inference_geo].to_s.downcase)
+          modes.empty? ? nil : modes.uniq.join("_")
+        end
+
+        def match_basis_for(result)
+          return "workspace" if result[:workspace_id]
+          return "model" if result[:model]
+
+          "period_only"
+        end
+
+        def fingerprint_for(result, starting_at:, ending_at:)
+          attributes = result.merge(starting_at: normalized_epoch(starting_at),
+                                    ending_at: normalized_epoch(ending_at))
+          Fingerprint.compute(FINGERPRINT_KEYS, attributes)
+        end
+
+        def normalized_epoch(value)
+          return value.to_i if value.is_a?(Numeric)
+
+          Time.parse(value.to_s).utc.to_i
+        rescue ArgumentError
+          value.to_s
+        end
+
+        def parse_date(value)
+          return value if value.is_a?(Date)
+          return Time.at(value).utc.to_date if value.is_a?(Numeric)
+
+          Time.parse(value.to_s).utc.to_date
+        end
+
+        def end_inclusive_date(value)
+          time = case value
+                 when Numeric then Time.at(value).utc
+                 when Date then value.to_time.utc
+                 else Time.parse(value.to_s).utc
+                 end
+          (time - 1).utc.to_date
+        end
+
+        def coerce_hash(response)
+          return {} if response.nil?
+          return symbolize(response) if response.is_a?(Hash)
+
+          parsed = JSON.parse(response.to_s)
+          raise ArgumentError, "Anthropic Usage payload must be a JSON object" unless parsed.is_a?(Hash)
+
+          symbolize(parsed)
+        rescue JSON::ParserError => e
+          raise ArgumentError, "Unable to parse Anthropic Usage payload: #{e.message}"
+        end
+
+        def symbolize(hash)
+          hash.to_h.transform_keys { |key| key.to_s.to_sym }
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/reconciliation/sources/fingerprint.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module LlmCostTracker
+  module Reconciliation
+    module Sources
+      module Fingerprint
+        DIGEST_LENGTH = 16
+
+        module_function
+
+        def compute(keys, attributes)
+          source_string = keys.map { |key| attributes[key].to_s }.join("|")
+          Digest::SHA256.hexdigest(source_string)[0, DIGEST_LENGTH]
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+require_relative "fingerprint"
+
+module LlmCostTracker
+  module Reconciliation
+    module Sources
+      module OpenaiUsage
+        FINGERPRINT_KEYS = %i[start_time end_time line_item model project_id api_key_id organization_id].freeze
+        ROW_TYPE_COST = "cost"
+        AUTHORITY_COST_API = "cost_api"
+        DEFAULT_METER = "tokens"
+
+        module_function
+
+        def parse(response, authority: AUTHORITY_COST_API, row_type: ROW_TYPE_COST)
+          payload = coerce_hash(response)
+          buckets = Array(payload[:data])
+          buckets.flat_map do |bucket|
+            rows_for_bucket(bucket, authority: authority, row_type: row_type)
+          end.compact
+        end
+
+        def rows_for_bucket(bucket, authority:, row_type:)
+          bucket = symbolize(bucket)
+          start_time = bucket[:start_time]
+          end_time = bucket[:end_time]
+          return [] unless start_time && end_time
+
+          period_start = epoch_to_date(start_time)
+          period_end = end_inclusive_date(end_time)
+
+          Array(bucket[:results]).filter_map do |raw|
+            row_for_result(raw,
+                           period_start: period_start, period_end: period_end,
+                           start_time: start_time, end_time: end_time,
+                           authority: authority, row_type: row_type)
+          end
+        rescue ArgumentError
+          []
+        end
+
+        def row_for_result(raw, period_start:, period_end:, start_time:, end_time:, authority:, row_type:)
+          result = symbolize(raw)
+          amount = symbolize(result[:amount] || {})
+          billed_amount = amount[:value]
+          return nil if billed_amount.nil?
+
+          fingerprint = fingerprint_for(result, start_time: start_time, end_time: end_time)
+          {
+            external_id: "cost-#{fingerprint}",
+            period_start: period_start,
+            period_end: period_end,
+            billed_amount: billed_amount,
+            currency: (amount[:currency] || "USD").to_s.upcase,
+            metadata: metadata_for(result, authority: authority, row_type: row_type)
+          }
+        end
+
+        def metadata_for(result, authority:, row_type:)
+          {
+            "row_type" => row_type,
+            "meter" => meter_for(result),
+            "authority" => authority,
+            "match_basis" => match_basis_for(result),
+            "line_item" => result[:line_item],
+            "model" => result[:model],
+            "provider_project_id" => result[:project_id],
+            "provider_api_key_id" => result[:api_key_id],
+            "provider_workspace_id" => result[:organization_id]
+          }.compact
+        end
+
+        def meter_for(result)
+          line_item = result[:line_item].to_s.downcase
+          case line_item
+          when /web search/, /search content/ then "web_search"
+          when /file search/ then "file_search_storage"
+          when /code interpreter/, /container/ then "container_session"
+          else DEFAULT_METER
+          end
+        end
+
+        def match_basis_for(result)
+          return "project" if result[:project_id]
+          return "api_key" if result[:api_key_id]
+          return "model" if result[:model]
+
+          "period_only"
+        end
+
+        def fingerprint_for(result, start_time:, end_time:)
+          attributes = result.merge(start_time: normalized_epoch(start_time),
+                                    end_time: normalized_epoch(end_time))
+          Fingerprint.compute(FINGERPRINT_KEYS, attributes)
+        end
+
+        def normalized_epoch(value)
+          return value.to_i if value.is_a?(Numeric)
+
+          Time.parse(value.to_s).utc.to_i
+        rescue ArgumentError
+          value.to_s
+        end
+
+        def epoch_to_date(value)
+          return Time.at(Integer(value)).utc.to_date if value.is_a?(Numeric) || value.to_s.match?(/\A\d+\z/)
+
+          Time.parse(value.to_s).utc.to_date
+        end
+
+        def end_inclusive_date(value)
+          time = if value.is_a?(Numeric) || value.to_s.match?(/\A\d+\z/)
+                   Time.at(Integer(value)).utc
+                 else
+                   Time.parse(value.to_s).utc
+                 end
+          (time - 1).utc.to_date
+        end
+
+        def coerce_hash(response)
+          return {} if response.nil?
+          return symbolize(response) if response.is_a?(Hash)
+
+          parsed = JSON.parse(response.to_s)
+          raise ArgumentError, "OpenAI Costs payload must be a JSON object" unless parsed.is_a?(Hash)
+
+          symbolize(parsed)
+        rescue JSON::ParserError => e
+          raise ArgumentError, "Unable to parse OpenAI Costs payload: #{e.message}"
+        end
+
+        def symbolize(hash)
+          hash.to_h.transform_keys { |key| key.to_s.to_sym }
+        end
+      end
+    end
+  end
+end

data/lib/llm_cost_tracker/reconciliation.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "ledger/schema/provider_invoices"
+require_relative "ledger/schema/provider_invoice_imports"
+require_relative "reconciliation/import_result"
+require_relative "reconciliation/importer"
+require_relative "reconciliation/diff_result"
+require_relative "reconciliation/diff"
+require_relative "reconciliation/sources/fingerprint"
+require_relative "reconciliation/sources/openai_usage"
+require_relative "reconciliation/sources/anthropic_usage"
+
+module LlmCostTracker
+  module Reconciliation
+    SUPPORTED_SOURCES = %i[openai anthropic gemini csv].freeze
+    DEFAULT_THRESHOLD_PERCENT = 5.0
+    INVOICE_FRESHNESS_DAYS = 14
+    SOURCE_TO_PROVIDER = {
+      "openai" => "openai",
+      "openai_usage" => "openai",
+      "anthropic" => "anthropic",
+      "anthropic_usage" => "anthropic",
+      "gemini" => "gemini"
+    }.freeze
+
+    SCHEMA_TABLES = {
+      Ledger::Schema::ProviderInvoices => "llm_cost_tracker_provider_invoices",
+      Ledger::Schema::ProviderInvoiceImports => "llm_cost_tracker_provider_invoice_imports"
+    }.freeze
+
+    class << self
+      def import(source:, rows:, provider: nil, imported_at: nil, window: nil,
+                 strict_metadata: nil, cursor: nil)
+        ensure_enabled!
+        ensure_source_present!(source)
+        Importer.new(
+          source: source,
+          provider: resolve_provider(source: source, provider: provider),
+          imported_at: imported_at,
+          window: window,
+          strict_metadata: strict_metadata,
+          cursor: cursor
+        ).call(rows)
+      end
+
+      def diff(source:, period_start:, period_end:, provider: nil, scope: {}, currency: nil,
+               drilldown_limit: Diff::DEFAULT_DRILLDOWN_LIMIT)
+        ensure_enabled!
+        ensure_source_present!(source)
+        Diff.new(
+          source: source,
+          provider: resolve_provider(source: source, provider: provider),
+          period_start: period_start,
+          period_end: period_end,
+          scope: scope,
+          currency: currency,
+          drilldown_limit: drilldown_limit
+        ).call
+      end
+
+      def ensure_source_present!(source)
+        return unless source.to_s.empty?
+
+        raise ArgumentError, "source must be present"
+      end
+
+      def resolve_provider(source:, provider:)
+        return provider.to_s if provider
+
+        mapped = SOURCE_TO_PROVIDER[source.to_s]
+        return mapped if mapped
+
+        recorded = recorded_provider_for(source)
+        return recorded if recorded
+
+        known = SOURCE_TO_PROVIDER.keys.join(", ")
+        raise ArgumentError,
+              "provider: must be specified for reconciliation source #{source.inspect}; " \
+              "sources with a default provider mapping: #{known}"
+      end
+
+      def recorded_provider_for(source)
+        return nil unless LlmCostTracker::ProviderInvoice.table_exists?
+
+        metadata = LlmCostTracker::ProviderInvoice
+                   .where(source: source.to_s)
+                   .order(imported_at: :desc)
+                   .limit(1)
+                   .pick(:metadata)
+        value = metadata_provider_value(metadata)
+        value if value.is_a?(String) && !value.empty?
+      end
+
+      def metadata_provider_value(metadata)
+        case metadata
+        when Hash then metadata["provider"]
+        when String
+          parsed = JSON.parse(metadata) rescue nil # rubocop:disable Style/RescueModifier
+          parsed.is_a?(Hash) ? parsed["provider"] : nil
+        end
+      end
+
+      def enabled?
+        LlmCostTracker.configuration.reconciliation_enabled
+      end
+
+      def ensure_enabled!
+        return if enabled?
+
+        raise Error,
+              "reconciliation is disabled; set `config.reconciliation_enabled = true` in your initializer " \
+              "(requires admin/org-level provider API keys; see docs/upgrading.md)"
+      end
+    end
+  end
+end