llm_cost_tracker 0.10.0 → 0.12.0

data/lib/llm_cost_tracker/reconciliation/importer.rb

@@ -1,254 +0,0 @@
-# frozen_string_literal: true
-
-require "bigdecimal"
-require "date"
-require "json"
-
-require_relative "import_result"
-require_relative "../ledger/rollups"
-
-module LlmCostTracker
-  module Reconciliation
-    class Importer
-      REQUIRED_FIELDS = %i[external_id period_start period_end].freeze
-      FORGIVING_METADATA_SOURCES = %i[csv].to_set.freeze
-      ENVELOPE_KEYS = %w[row_type meter authority match_basis].freeze
-
-      def initialize(source:, imported_at:, provider:, window: nil, strict_metadata: nil, cursor: nil)
-        @source = source.to_s
-        @provider = provider.to_s
-        @imported_at = imported_at
-        @window = coerce_window(window)
-        @cursor = cursor
-        @strict_metadata = strict_metadata.nil? ? !FORGIVING_METADATA_SOURCES.include?(source.to_sym) : strict_metadata
-        raise ArgumentError, "source must be present" if @source.empty?
-        raise ArgumentError, "provider must be present" if @provider.empty?
-      end
-
-      def call(rows)
-        import_record = nil
-        ensure_reconciliation_installed!
-        return ImportResult.empty if skippable?(rows)
-
-        import_record = open_import_record
-        result = perform_import(rows)
-        complete_import_record(import_record, result)
-        result.with(import_id: import_record&.id)
-      rescue StandardError => e
-        fail_import_record(import_record, e)
-        raise
-      end
-
-      private
-
-      attr_reader :source, :provider, :imported_at, :window, :cursor, :strict_metadata
-
-      def skippable?(rows)
-        (rows.nil? || rows.empty?) && cursor.nil?
-      end
-
-      def ensure_reconciliation_installed!
-        return if ProviderInvoice.table_exists?
-
-        raise Error,
-              "llm_cost_tracker_provider_invoices table is missing; " \
-              "run `rails generate llm_cost_tracker:reconciliation && rails db:migrate`"
-      end
-
-      def perform_import(rows)
-        return ImportResult.empty if rows.nil? || rows.empty?
-
-        normalized, errors = normalize_rows(rows)
-        if normalized.empty?
-          return ImportResult.new(inserted: 0, updated: 0, skipped: rows.size, errors: errors,
-                                  import_id: nil)
-        end
-
-        existing = existing_external_ids(normalized.map { |row| row[:external_id] })
-        rows_payload = normalized.map { |row| persistable_attributes(row) }
-        upsert_options = { record_timestamps: true }
-        upsert_options[:unique_by] = :external_id if ProviderInvoice.connection.supports_insert_conflict_target?
-        ProviderInvoice.upsert_all(rows_payload, **upsert_options)
-
-        inserted = normalized.count { |row| !existing.include?(row[:external_id]) }
-        updated = normalized.size - inserted
-        ImportResult.new(inserted: inserted, updated: updated, skipped: rows.size - normalized.size,
-                         errors: errors, import_id: nil)
-      end
-
-      def open_import_record
-        return nil unless tracking_table_present?
-
-        ProviderInvoiceImport.create!(
-          source: source,
-          provider: provider,
-          cursor: cursor,
-          window_start: window&.first,
-          window_end: window&.last,
-          state: ProviderInvoiceImport::STATE_RUNNING,
-          started_at: Time.now.utc
-        )
-      end
-
-      def complete_import_record(record, result)
-        return unless record
-
-        terminal_state = result.success? ? ProviderInvoiceImport::STATE_COMPLETED : ProviderInvoiceImport::STATE_FAILED
-        record.update!(
-          state: terminal_state,
-          rows_imported: result.total_imported,
-          finished_at: Time.now.utc,
-          last_error: result.errors.first
-        )
-      end
-
-      def fail_import_record(record, error)
-        return unless record
-
-        record.update!(
-          state: ProviderInvoiceImport::STATE_FAILED,
-          last_error: "#{error.class}: #{error.message}",
-          finished_at: Time.now.utc
-        )
-      end
-
-      def tracking_table_present?
-        @tracking_table_present = ProviderInvoiceImport.table_exists? unless defined?(@tracking_table_present)
-        @tracking_table_present
-      end
-
-      def normalize_rows(rows)
-        errors = []
-        normalized = rows.each_with_index.filter_map do |row, index|
-          attrs = symbolize(row)
-          missing = REQUIRED_FIELDS - attrs.keys
-          if missing.any?
-            errors << "row #{index}: missing #{missing.join(', ')}"
-            next
-          end
-          period_start = parse_date(attrs[:period_start])
-          period_end = parse_date(attrs[:period_end])
-          next unless within_window?(period_start, period_end)
-
-          attrs.merge(
-            external_id: namespaced_external_id(attrs[:external_id]),
-            period_start: period_start,
-            period_end: period_end,
-            metadata: parse_metadata(attrs[:metadata])
-          )
-        rescue ArgumentError => e
-          errors << "row #{index}: #{e.message}"
-          nil
-        end
-        [normalized, errors]
-      end
-
-      def within_window?(period_start, period_end)
-        return true if window.nil?
-
-        period_start <= window.last && period_end >= window.first
-      end
-
-      def coerce_window(window)
-        return nil if window.nil?
-        raise ArgumentError, "window must be a Range of dates" unless window.is_a?(Range)
-
-        Range.new(parse_date(window.first), parse_date(window.last))
-      end
-
-      def existing_external_ids(external_ids)
-        ProviderInvoice.where(external_id: external_ids).pluck(:external_id).to_set
-      end
-
-      def persistable_attributes(row)
-        billed_amount = row[:billed_amount] && BigDecimal(row[:billed_amount].to_s)
-        {
-          source: source,
-          external_id: row[:external_id],
-          period_start: row[:period_start],
-          period_end: row[:period_end],
-          billed_amount: billed_amount,
-          currency: (row[:currency] || Ledger::Rollups::DEFAULT_CURRENCY).to_s.upcase,
-          metadata: stamp_metadata(row[:metadata]),
-          imported_at: imported_at || Time.now.utc
-        }
-      end
-
-      BASIS_DIMENSIONS_BY_PRIORITY = [
-        %w[project provider_project_id],
-        %w[api_key provider_api_key_id],
-        %w[workspace provider_workspace_id],
-        %w[model model]
-      ].freeze
-      private_constant :BASIS_DIMENSIONS_BY_PRIORITY
-
-      def stamp_metadata(metadata)
-        merged = metadata_with_provider(metadata)
-        metadata_with_match_basis(merged)
-      end
-
-      def metadata_with_provider(metadata)
-        return { "provider" => provider } if metadata.nil? || metadata.empty?
-
-        existing = metadata["provider"] || metadata[:provider]
-        return metadata if existing.is_a?(String) && !existing.empty?
-
-        metadata.merge("provider" => provider)
-      end
-
-      def metadata_with_match_basis(metadata)
-        existing = metadata["match_basis"] || metadata[:match_basis]
-        return metadata if existing.is_a?(String) && !existing.empty?
-
-        inferred = BASIS_DIMENSIONS_BY_PRIORITY.find { |_basis, key| metadata[key] || metadata[key.to_sym] }
-        return metadata.merge("match_basis" => "period_only") if inferred.nil?
-
-        metadata.merge("match_basis" => inferred.first)
-      end
-
-      def namespaced_external_id(external_id)
-        raw = external_id.to_s
-        scope = source == provider ? source : "#{source}/#{provider}"
-        prefix = "#{scope}:"
-        raw.start_with?(prefix) ? raw : "#{prefix}#{raw}"
-      end
-
-      def symbolize(row)
-        return row if row.is_a?(Hash) && row.keys.all?(Symbol)
-
-        row.to_h.transform_keys { |key| key.to_s.to_sym }
-      end
-
-      def parse_date(value)
-        return value if value.is_a?(Date)
-
-        Date.parse(value.to_s)
-      end
-
-      def parse_metadata(metadata)
-        parsed = parse_metadata_payload(metadata)
-        validate_envelope!(parsed) if strict_metadata
-        parsed
-      end
-
-      def parse_metadata_payload(metadata)
-        return {} if metadata.nil?
-        return metadata if metadata.is_a?(Hash)
-
-        JSON.parse(metadata.to_s)
-      rescue JSON::ParserError => e
-        raise ArgumentError, "invalid metadata JSON: #{e.message}" if strict_metadata
-
-        {}
-      end
-
-      def validate_envelope!(metadata)
-        keys = metadata.keys.map(&:to_s)
-        missing = ENVELOPE_KEYS - keys
-        return if missing.empty?
-
-        raise ArgumentError, "metadata missing envelope keys: #{missing.join(', ')}"
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb

@@ -1,172 +0,0 @@
-# frozen_string_literal: true
-
-require "bigdecimal"
-require "json"
-require "time"
-
-require_relative "fingerprint"
-require_relative "../../providers/anthropic/tier_classification"
-
-module LlmCostTracker
-  module Reconciliation
-    module Sources
-      module AnthropicUsage
-        FINGERPRINT_KEYS = %i[
-          starting_at ending_at model workspace_id
-          service_tier context_window cost_type token_type description
-          inference_geo
-        ].freeze
-        ROW_TYPE_COST = "cost"
-        AUTHORITY_COST_API = "cost_api"
-        DEFAULT_METER = "tokens"
-
-        module_function
-
-        def parse(response, authority: AUTHORITY_COST_API, row_type: ROW_TYPE_COST)
-          payload = coerce_hash(response)
-          buckets = Array(payload[:data])
-          buckets.flat_map do |bucket|
-            rows_for_bucket(bucket, authority: authority, row_type: row_type)
-          end.compact
-        end
-
-        def rows_for_bucket(bucket, authority:, row_type:)
-          bucket = symbolize(bucket)
-          starting_at = bucket[:starting_at]
-          ending_at = bucket[:ending_at]
-          return [] unless starting_at && ending_at
-
-          period_start = parse_date(starting_at)
-          period_end = end_inclusive_date(ending_at)
-
-          Array(bucket[:results]).filter_map do |raw|
-            row_for_result(raw,
-                           period_start: period_start, period_end: period_end,
-                           starting_at: starting_at, ending_at: ending_at,
-                           authority: authority, row_type: row_type)
-          end
-        rescue ArgumentError
-          []
-        end
-
-        def row_for_result(raw, period_start:, period_end:, starting_at:, ending_at:, authority:, row_type:)
-          result = symbolize(raw)
-          raw_amount = result[:amount]
-          return nil if raw_amount.nil?
-
-          fingerprint = fingerprint_for(result, starting_at: starting_at, ending_at: ending_at)
-          {
-            external_id: "cost-#{fingerprint}",
-            period_start: period_start,
-            period_end: period_end,
-            billed_amount: dollars_from_cents(raw_amount),
-            currency: (result[:currency] || "USD").to_s.upcase,
-            metadata: metadata_for(result, authority: authority, row_type: row_type)
-          }
-        end
-
-        def dollars_from_cents(amount)
-          (BigDecimal(amount.to_s) / 100).to_s("F")
-        end
-
-        def metadata_for(result, authority:, row_type:)
-          {
-            "row_type" => row_type,
-            "meter" => meter_for(result),
-            "authority" => authority,
-            "match_basis" => match_basis_for(result),
-            "model" => result[:model],
-            "pricing_mode" => pricing_mode_for(result),
-            "context_window" => result[:context_window],
-            "cost_type" => result[:cost_type],
-            "description" => result[:description],
-            "token_type" => result[:token_type],
-            "inference_geo" => result[:inference_geo],
-            "provider_workspace_id" => result[:workspace_id]
-          }.compact
-        end
-
-        def meter_for(result)
-          case result[:cost_type].to_s
-          when "web_search" then "web_search"
-          when "code_execution" then "code_execution_hour"
-          when "session_usage" then "session_usage"
-          when "tokens" then token_meter(result[:token_type].to_s)
-          else DEFAULT_METER
-          end
-        end
-
-        def token_meter(token_type)
-          return "cache_read_input_tokens" if token_type.include?("cache_read")
-          return "cache_creation_input_tokens" if token_type.include?("cache_creation")
-          return "input_tokens" if token_type.include?("input")
-          return "output_tokens" if token_type.include?("output")
-
-          DEFAULT_METER
-        end
-
-        def pricing_mode_for(result)
-          modes = []
-          modes << "batch" if result[:service_tier].to_s.downcase == "batch"
-          if LlmCostTracker::Providers::Anthropic::TierClassification.data_residency_geo?(result[:inference_geo])
-            modes << "data_residency"
-          end
-          modes.empty? ? nil : modes.uniq.join("_")
-        end
-
-        def match_basis_for(result)
-          return "workspace" if result[:workspace_id]
-          return "model" if result[:model]
-
-          "period_only"
-        end
-
-        def fingerprint_for(result, starting_at:, ending_at:)
-          attributes = result.merge(starting_at: normalized_epoch(starting_at),
-                                    ending_at: normalized_epoch(ending_at))
-          Fingerprint.compute(FINGERPRINT_KEYS, attributes)
-        end
-
-        def normalized_epoch(value)
-          return value.to_i if value.is_a?(Numeric)
-
-          Time.parse(value.to_s).utc.to_i
-        rescue ArgumentError
-          value.to_s
-        end
-
-        def parse_date(value)
-          return value if value.is_a?(Date)
-          return Time.at(value).utc.to_date if value.is_a?(Numeric)
-
-          Time.parse(value.to_s).utc.to_date
-        end
-
-        def end_inclusive_date(value)
-          time = case value
-                 when Numeric then Time.at(value).utc
-                 when Date then value.to_time.utc
-                 else Time.parse(value.to_s).utc
-                 end
-          (time - 1).utc.to_date
-        end
-
-        def coerce_hash(response)
-          return {} if response.nil?
-          return symbolize(response) if response.is_a?(Hash)
-
-          parsed = JSON.parse(response.to_s)
-          raise ArgumentError, "Anthropic Usage payload must be a JSON object" unless parsed.is_a?(Hash)
-
-          symbolize(parsed)
-        rescue JSON::ParserError => e
-          raise ArgumentError, "Unable to parse Anthropic Usage payload: #{e.message}"
-        end
-
-        def symbolize(hash)
-          hash.to_h.transform_keys { |key| key.to_s.to_sym }
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/reconciliation/sources/fingerprint.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require "digest"
-
-module LlmCostTracker
-  module Reconciliation
-    module Sources
-      module Fingerprint
-        DIGEST_LENGTH = 16
-
-        module_function
-
-        def compute(keys, attributes)
-          source_string = keys.map { |key| attributes[key].to_s }.join("|")
-          Digest::SHA256.hexdigest(source_string)[0, DIGEST_LENGTH]
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb

@@ -1,142 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "time"
-
-require_relative "fingerprint"
-
-module LlmCostTracker
-  module Reconciliation
-    module Sources
-      module OpenaiUsage
-        FINGERPRINT_KEYS = %i[start_time end_time line_item model project_id api_key_id organization_id].freeze
-        ROW_TYPE_COST = "cost"
-        AUTHORITY_COST_API = "cost_api"
-        DEFAULT_METER = "tokens"
-
-        module_function
-
-        def parse(response, authority: AUTHORITY_COST_API, row_type: ROW_TYPE_COST)
-          payload = coerce_hash(response)
-          buckets = Array(payload[:data])
-          buckets.flat_map do |bucket|
-            rows_for_bucket(bucket, authority: authority, row_type: row_type)
-          end.compact
-        end
-
-        def rows_for_bucket(bucket, authority:, row_type:)
-          bucket = symbolize(bucket)
-          start_time = bucket[:start_time]
-          end_time = bucket[:end_time]
-          return [] unless start_time && end_time
-
-          period_start = epoch_to_date(start_time)
-          period_end = end_inclusive_date(end_time)
-
-          Array(bucket[:results]).filter_map do |raw|
-            row_for_result(raw,
-                           period_start: period_start, period_end: period_end,
-                           start_time: start_time, end_time: end_time,
-                           authority: authority, row_type: row_type)
-          end
-        rescue ArgumentError
-          []
-        end
-
-        def row_for_result(raw, period_start:, period_end:, start_time:, end_time:, authority:, row_type:)
-          result = symbolize(raw)
-          amount = symbolize(result[:amount] || {})
-          billed_amount = amount[:value]
-          return nil if billed_amount.nil?
-
-          fingerprint = fingerprint_for(result, start_time: start_time, end_time: end_time)
-          {
-            external_id: "cost-#{fingerprint}",
-            period_start: period_start,
-            period_end: period_end,
-            billed_amount: billed_amount,
-            currency: (amount[:currency] || "USD").to_s.upcase,
-            metadata: metadata_for(result, authority: authority, row_type: row_type)
-          }
-        end
-
-        def metadata_for(result, authority:, row_type:)
-          {
-            "row_type" => row_type,
-            "meter" => meter_for(result),
-            "authority" => authority,
-            "match_basis" => match_basis_for(result),
-            "line_item" => result[:line_item],
-            "model" => result[:model],
-            "provider_project_id" => result[:project_id],
-            "provider_api_key_id" => result[:api_key_id],
-            "provider_workspace_id" => result[:organization_id]
-          }.compact
-        end
-
-        def meter_for(result)
-          line_item = result[:line_item].to_s.downcase
-          case line_item
-          when /web search/, /search content/ then "web_search"
-          when /file search/ then "file_search_storage"
-          when /code interpreter/, /container/ then "container_session"
-          else DEFAULT_METER
-          end
-        end
-
-        def match_basis_for(result)
-          return "project" if result[:project_id]
-          return "api_key" if result[:api_key_id]
-          return "model" if result[:model]
-
-          "period_only"
-        end
-
-        def fingerprint_for(result, start_time:, end_time:)
-          attributes = result.merge(start_time: normalized_epoch(start_time),
-                                    end_time: normalized_epoch(end_time))
-          Fingerprint.compute(FINGERPRINT_KEYS, attributes)
-        end
-
-        def normalized_epoch(value)
-          return value.to_i if value.is_a?(Numeric)
-
-          Time.parse(value.to_s).utc.to_i
-        rescue ArgumentError
-          value.to_s
-        end
-
-        def epoch_to_date(value)
-          return Time.at(Integer(value)).utc.to_date if value.is_a?(Numeric) || value.to_s.match?(/\A\d+\z/)
-
-          Time.parse(value.to_s).utc.to_date
-        end
-
-        def end_inclusive_date(value)
-          time = if value.is_a?(Numeric) || value.to_s.match?(/\A\d+\z/)
-                   Time.at(Integer(value)).utc
-                 else
-                   Time.parse(value.to_s).utc
-                 end
-          (time - 1).utc.to_date
-        end
-
-        def coerce_hash(response)
-          return {} if response.nil?
-          return symbolize(response) if response.is_a?(Hash)
-
-          parsed = JSON.parse(response.to_s)
-          raise ArgumentError, "OpenAI Costs payload must be a JSON object" unless parsed.is_a?(Hash)
-
-          symbolize(parsed)
-        rescue JSON::ParserError => e
-          raise ArgumentError, "Unable to parse OpenAI Costs payload: #{e.message}"
-        end
-
-        def symbolize(hash)
-          hash.to_h.transform_keys { |key| key.to_s.to_sym }
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/reconciliation.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-require_relative "ledger/schema/provider_invoices"
-require_relative "ledger/schema/provider_invoice_imports"
-require_relative "reconciliation/import_result"
-require_relative "reconciliation/importer"
-require_relative "reconciliation/diff_result"
-require_relative "reconciliation/diff"
-require_relative "reconciliation/sources/fingerprint"
-require_relative "reconciliation/sources/openai_usage"
-require_relative "reconciliation/sources/anthropic_usage"
-
-module LlmCostTracker
-  module Reconciliation
-    SUPPORTED_SOURCES = %i[openai anthropic gemini csv].freeze
-    DEFAULT_THRESHOLD_PERCENT = 5.0
-    INVOICE_FRESHNESS_DAYS = 14
-    SOURCE_TO_PROVIDER = {
-      "openai" => "openai",
-      "openai_usage" => "openai",
-      "anthropic" => "anthropic",
-      "anthropic_usage" => "anthropic",
-      "gemini" => "gemini"
-    }.freeze
-
-    SCHEMA_TABLES = {
-      Ledger::Schema::ProviderInvoices => "llm_cost_tracker_provider_invoices",
-      Ledger::Schema::ProviderInvoiceImports => "llm_cost_tracker_provider_invoice_imports"
-    }.freeze
-
-    class << self
-      def import(source:, rows:, provider: nil, imported_at: nil, window: nil,
-                 strict_metadata: nil, cursor: nil)
-        ensure_enabled!
-        ensure_source_present!(source)
-        Importer.new(
-          source: source,
-          provider: resolve_provider(source: source, provider: provider),
-          imported_at: imported_at,
-          window: window,
-          strict_metadata: strict_metadata,
-          cursor: cursor
-        ).call(rows)
-      end
-
-      def diff(source:, period_start:, period_end:, provider: nil, scope: {}, currency: nil,
-               drilldown_limit: Diff::DEFAULT_DRILLDOWN_LIMIT)
-        ensure_enabled!
-        ensure_source_present!(source)
-        Diff.new(
-          source: source,
-          provider: resolve_provider(source: source, provider: provider),
-          period_start: period_start,
-          period_end: period_end,
-          scope: scope,
-          currency: currency,
-          drilldown_limit: drilldown_limit
-        ).call
-      end
-
-      def ensure_source_present!(source)
-        return unless source.to_s.empty?
-
-        raise ArgumentError, "source must be present"
-      end
-
-      def resolve_provider(source:, provider:)
-        return provider.to_s if provider
-
-        mapped = SOURCE_TO_PROVIDER[source.to_s]
-        return mapped if mapped
-
-        recorded = recorded_provider_for(source)
-        return recorded if recorded
-
-        known = SOURCE_TO_PROVIDER.keys.join(", ")
-        raise ArgumentError,
-              "provider: must be specified for reconciliation source #{source.inspect}; " \
-              "sources with a default provider mapping: #{known}"
-      end
-
-      def recorded_provider_for(source)
-        return nil unless LlmCostTracker::ProviderInvoice.table_exists?
-
-        metadata = LlmCostTracker::ProviderInvoice
-                   .where(source: source.to_s)
-                   .order(imported_at: :desc)
-                   .limit(1)
-                   .pick(:metadata)
-        value = metadata_provider_value(metadata)
-        value if value.is_a?(String) && !value.empty?
-      end
-
-      def metadata_provider_value(metadata)
-        case metadata
-        when Hash then metadata["provider"]
-        when String
-          parsed = JSON.parse(metadata) rescue nil # rubocop:disable Style/RescueModifier
-          parsed.is_a?(Hash) ? parsed["provider"] : nil
-        end
-      end
-
-      def enabled?
-        LlmCostTracker.configuration.reconciliation_enabled
-      end
-
-      def ensure_enabled!
-        return if enabled?
-
-        raise Error,
-              "reconciliation is disabled; set `config.reconciliation_enabled = true` in your initializer " \
-              "(requires admin/org-level provider API keys; see docs/upgrading.md)"
-      end
-    end
-  end
-end