llm_cost_tracker 0.11.0 → 0.12.0

data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoices_metadata_index.rb.erb

@@ -1,27 +0,0 @@
-require "llm_cost_tracker/ledger/schema/adapter"
-
-class UpgradeLlmCostTrackerProviderInvoicesMetadataIndex < ActiveRecord::Migration<%= migration_version %>
-  disable_ddl_transaction!
-
-  TABLE = :llm_cost_tracker_provider_invoices
-
-  def up
-    return unless postgresql?
-    return unless table_exists?(TABLE)
-
-    add_index TABLE, :metadata, using: :gin, algorithm: :concurrently, if_not_exists: true
-  end
-
-  def down
-    return unless postgresql?
-    return unless table_exists?(TABLE)
-
-    remove_index TABLE, column: :metadata, if_exists: true
-  end
-
-  private
-
-  def postgresql?
-    LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
-  end
-end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoice_imports_provider_generator.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require "rails/generators"
-require "rails/generators/active_record"
-
-module LlmCostTracker
-  module Generators
-    class UpgradeProviderInvoiceImportsProviderGenerator < Rails::Generators::Base
-      include ActiveRecord::Generators::Migration
-
-      source_root File.expand_path("templates", __dir__)
-
-      desc "Adds a provider column to llm_cost_tracker_provider_invoice_imports and a " \
-           "(source, provider, started_at) index so resume_cursor_for and " \
-           "last_completed_window_for can isolate per-provider state on shared sources (e.g. csv)."
-
-      def create_migration_file
-        migration_template(
-          "upgrade_provider_invoice_imports_provider.rb.erb",
-          "db/migrate/upgrade_llm_cost_tracker_provider_invoice_imports_provider.rb"
-        )
-      end
-
-      private
-
-      def migration_version
-        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoices_metadata_index_generator.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-require "rails/generators"
-require "rails/generators/active_record"
-
-module LlmCostTracker
-  module Generators
-    class UpgradeProviderInvoicesMetadataIndexGenerator < Rails::Generators::Base
-      include ActiveRecord::Generators::Migration
-
-      source_root File.expand_path("templates", __dir__)
-
-      desc "Adds a GIN index on llm_cost_tracker_provider_invoices.metadata for PostgreSQL " \
-           "so Reconciliation::Diff queries that filter on metadata->>'provider' / 'row_type' / " \
-           "'match_basis' hit an index instead of a sequential scan. No-op on MySQL."
-
-      def create_migration_file
-        migration_template(
-          "upgrade_provider_invoices_metadata_index.rb.erb",
-          "db/migrate/upgrade_llm_cost_tracker_provider_invoices_metadata_index.rb"
-        )
-      end
-
-      private
-
-      def migration_version
-        "[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb

@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../schema/adapter"
-
-module LlmCostTracker
-  module Ledger
-    class Rollups
-      class UpsertSql
-        def self.call
-          new.call
-        end
-
-        def call
-          return Arel.sql(mysql_sql) if Ledger::Schema::Adapter.mysql?(connection)
-          return Arel.sql(postgres_sql) if Ledger::Schema::Adapter.postgresql?(connection)
-
-          Ledger::Schema::Adapter.ensure_supported!(connection)
-        end
-
-        private
-
-        def postgres_sql
-          total_cost = connection.quote_column_name("total_cost")
-          updated_at = connection.quote_column_name("updated_at")
-
-          "#{total_cost} = #{LlmCostTracker::CallRollup.quoted_table_name}.#{total_cost} + excluded.#{total_cost}, " \
-            "#{updated_at} = excluded.#{updated_at}"
-        end
-
-        def mysql_sql
-          "total_cost = total_cost + VALUES(total_cost), updated_at = VALUES(updated_at)"
-        end
-
-        def connection
-          LlmCostTracker::CallRollup.connection
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "adapter"
-
-module LlmCostTracker
-  module Ledger
-    module Schema
-      module IngestionInboxEntries
-        REQUIRED_COLUMNS = %w[
-          event_id
-          total_cost
-          tracked_at
-          payload
-          locked_at
-          locked_by
-          attempts
-          last_error
-          created_at
-          updated_at
-        ].freeze
-
-        UNIQUE_COLUMNS = %i[event_id].freeze
-
-        class << self
-          def current_schema_errors
-            connection = LlmCostTracker::Ingestion::InboxEntry.connection
-            Adapter.ensure_supported!(connection)
-            table_name = LlmCostTracker::Ingestion::InboxEntry.table_name
-            return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
-
-            columns = LlmCostTracker::Ingestion::InboxEntry.columns_hash
-            cache = @schema_capabilities
-            return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
-
-            errors = compute_errors(connection, table_name, columns)
-            @schema_capabilities = { columns: columns, errors: errors }
-            errors
-          end
-
-          private
-
-          def compute_errors(connection, table_name, columns)
-            errors = []
-            missing = REQUIRED_COLUMNS - columns.keys
-            errors << "missing columns: #{missing.join(', ')}" if missing.any?
-            errors << "missing unique index: event_id" unless event_id_unique_index?(connection, table_name)
-            errors
-          end
-
-          def event_id_unique_index?(connection, table_name)
-            connection.index_exists?(table_name, UNIQUE_COLUMNS, unique: true)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "adapter"
-
-module LlmCostTracker
-  module Ledger
-    module Schema
-      module IngestionLeases
-        REQUIRED_COLUMNS = %w[
-          name
-          locked_by
-          locked_until
-          created_at
-          updated_at
-        ].freeze
-
-        UNIQUE_COLUMNS = %i[name].freeze
-
-        class << self
-          def current_schema_errors
-            connection = LlmCostTracker::Ingestion::Lease.connection
-            Adapter.ensure_supported!(connection)
-            table_name = LlmCostTracker::Ingestion::Lease.table_name
-            return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
-
-            columns = LlmCostTracker::Ingestion::Lease.columns_hash
-            cache = @schema_capabilities
-            return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
-
-            errors = compute_errors(connection, table_name, columns)
-            @schema_capabilities = { columns: columns, errors: errors }
-            errors
-          end
-
-          private
-
-          def compute_errors(connection, table_name, columns)
-            errors = []
-            missing = REQUIRED_COLUMNS - columns.keys
-            errors << "missing columns: #{missing.join(', ')}" if missing.any?
-            errors << "missing unique index: name" unless name_unique_index?(connection, table_name)
-            errors
-          end
-
-          def name_unique_index?(connection, table_name)
-            connection.index_exists?(table_name, UNIQUE_COLUMNS, unique: true)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb

@@ -1,56 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "adapter"
-
-module LlmCostTracker
-  module Ledger
-    module Schema
-      module ProviderInvoiceImports
-        REQUIRED_COLUMNS = %w[
-          source provider cursor window_start window_end state last_error
-          rows_imported started_at finished_at
-        ].freeze
-        SOURCE_PROVIDER_STARTED_AT_INDEX = %i[source provider started_at].freeze
-
-        class << self
-          def current_schema_errors
-            connection = LlmCostTracker::Call.connection
-            Adapter.ensure_supported!(connection)
-            table_name = LlmCostTracker::ProviderInvoiceImport.table_name
-            return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
-
-            columns = LlmCostTracker::ProviderInvoiceImport.columns_hash
-            cache = @schema_capabilities
-            return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
-
-            errors = compute_errors(connection, table_name, columns)
-            @schema_capabilities = { columns: columns, errors: errors }
-            errors
-          end
-
-          private
-
-          def compute_errors(connection, table_name, columns)
-            errors = []
-            errors.concat(column_errors(columns))
-            errors.concat(index_errors(connection, table_name))
-            errors
-          end
-
-          def column_errors(columns)
-            missing = REQUIRED_COLUMNS - columns.keys
-            return [] if missing.empty?
-
-            ["missing columns: #{missing.join(', ')}"]
-          end
-
-          def index_errors(connection, table_name)
-            return [] if connection.index_exists?(table_name, SOURCE_PROVIDER_STARTED_AT_INDEX)
-
-            ["missing index: source, provider, started_at"]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "adapter"
-
-module LlmCostTracker
-  module Ledger
-    module Schema
-      module ProviderInvoices
-        REQUIRED_COLUMNS = %w[
-          source period_start period_end external_id billed_amount currency metadata imported_at
-        ].freeze
-        UNIQUE_INDEX_COLUMNS = %i[external_id].freeze
-        SOURCE_PERIOD_INDEX_COLUMNS = %i[source currency period_start].freeze
-
-        class << self
-          def current_schema_errors
-            connection = LlmCostTracker::Call.connection
-            Adapter.ensure_supported!(connection)
-            table_name = LlmCostTracker::ProviderInvoice.table_name
-            return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
-
-            columns = LlmCostTracker::ProviderInvoice.columns_hash
-            cache = @schema_capabilities
-            return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
-
-            errors = compute_errors(connection, table_name, columns)
-            @schema_capabilities = { columns: columns, errors: errors }
-            errors
-          end
-
-          private
-
-          def compute_errors(connection, table_name, columns)
-            errors = []
-            errors.concat(column_errors(columns))
-            errors.concat(Adapter.json_column_errors(columns["metadata"], connection, "metadata"))
-            errors.concat(index_errors(connection, table_name))
-            errors
-          end
-
-          def column_errors(columns)
-            missing = REQUIRED_COLUMNS - columns.keys
-            return [] if missing.empty?
-
-            ["missing columns: #{missing.join(', ')}"]
-          end
-
-          def index_errors(connection, table_name)
-            errors = []
-            unless connection.index_exists?(table_name, UNIQUE_INDEX_COLUMNS, unique: true)
-              errors << "missing unique index: external_id"
-            end
-            unless connection.index_exists?(table_name, SOURCE_PERIOD_INDEX_COLUMNS)
-              errors << "missing index: source, currency, period_start"
-            end
-            if Adapter.postgresql?(connection) && !gin_metadata_index?(connection, table_name)
-              errors << "missing GIN index on metadata " \
-                        "(run bin/rails generate llm_cost_tracker:upgrade_provider_invoices_metadata_index)"
-            end
-            errors
-          end
-
-          def gin_metadata_index?(connection, table_name)
-            connection.indexes(table_name).any? do |index|
-              index.columns == ["metadata"] && index.using.to_s == "gin"
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/masking.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module LlmCostTracker
-  module Masking
-    SENSITIVE_KEYS = %i[
-      provider_api_key_id provider_workspace_id provider_organization_id provider_project_id
-    ].to_set.freeze
-    MASK_TAIL_LENGTH = 4
-
-    module_function
-
-    def mask_value(key, value)
-      string = value.to_s
-      return string unless SENSITIVE_KEYS.include?(key.to_sym)
-      return string if string.length <= MASK_TAIL_LENGTH
-
-      "***#{string[-MASK_TAIL_LENGTH, MASK_TAIL_LENGTH]}"
-    end
-
-    def format_attribution(attribution, separator: ", ")
-      return "" if attribution.nil? || attribution.empty?
-
-      attribution.map { |key, value| "#{key}=#{mask_value(key, value)}" }.join(separator)
-    end
-
-    def mask_hash(hash)
-      return hash unless hash.is_a?(Hash)
-
-      hash.each_with_object({}) do |(key, value), masked|
-        masked[key] = case value
-                      when Hash then mask_hash(value)
-                      when Array then value.map { |entry| entry.is_a?(Hash) ? mask_hash(entry) : entry }
-                      else
-                        mask_value(key, value)
-                      end
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/parsers/anthropic.rb

@@ -1,176 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base"
-require_relative "../providers/anthropic/server_tools"
-require_relative "../providers/anthropic/tier_classification"
-
-module LlmCostTracker
-  module Parsers
-    class Anthropic < Base
-      HOSTS = %w[api.anthropic.com].freeze
-
-      class << self
-        def match?(url)
-          match_uri?(url, hosts: HOSTS, path_includes: "/v1/messages")
-        end
-
-        def provider_names
-          %w[anthropic]
-        end
-      end
-
-      def parse(request_body:, response_status:, response_body:, **)
-        return nil unless response_status == 200
-
-        response = safe_json_parse(response_body)
-        usage    = response["usage"]
-        return nil unless usage
-
-        request = safe_json_parse(request_body)
-        cache_read = usage["cache_read_input_tokens"].to_i
-
-        Event.build(
-          provider: "anthropic",
-          provider_response_id: response["id"],
-          pricing_mode: pricing_mode(request: request, usage: usage),
-          model: response["model"] || request["model"],
-          token_usage: token_usage(usage: usage, cache_read: cache_read),
-          usage_source: :response,
-          service_line_items: service_line_items(usage)
-        )
-      end
-
-      def parse_stream(response_status:, request_body: nil, events: [], **)
-        return nil unless response_status == 200
-
-        request = safe_json_parse(request_body)
-        model = find_event_value(events) { |data| data.dig("message", "model") } || request["model"]
-        usage = stream_usage(events)
-        response_id = find_event_value(events) { |data| data.dig("message", "id") || data["id"] }
-
-        if usage
-          build_stream_result(
-            model: model,
-            usage: usage,
-            response_id: response_id,
-            pricing_mode: pricing_mode(request: request, usage: usage)
-          )
-        else
-          build_unknown_stream_usage(
-            provider: "anthropic",
-            model: model,
-            provider_response_id: response_id,
-            pricing_mode: pricing_mode(request: request, usage: usage)
-          )
-        end
-      end
-
-      def provider_for(_request_url)
-        "anthropic"
-      end
-
-      private
-
-      def stream_usage(events)
-        latest_delta = find_event_value(events, reverse: true) do |data|
-          data["usage"] if data["type"] == "message_delta" && data["usage"].is_a?(Hash)
-        end
-        return nil unless latest_delta
-
-        start_usage = find_event_value(events, reverse: true) do |data|
-          data.dig("message", "usage") if data["type"] == "message_start"
-        end
-
-        (start_usage || {}).merge(latest_delta) do |_key, start_val, delta_val|
-          delta_val || start_val
-        end
-      end
-
-      def build_stream_result(model:, usage:, response_id:, pricing_mode:)
-        cache_read = usage["cache_read_input_tokens"].to_i
-
-        Event.build(
-          provider: "anthropic",
-          provider_response_id: response_id,
-          pricing_mode: pricing_mode,
-          model: model,
-          token_usage: token_usage(usage: usage, cache_read: cache_read),
-          stream: true,
-          usage_source: :stream_final,
-          service_line_items: service_line_items(usage)
-        )
-      end
-
-      def service_line_items(usage)
-        server_tool_use = usage["server_tool_use"]
-        return [] unless server_tool_use.is_a?(Hash)
-
-        Providers::Anthropic::ServerTools::LINE_ITEMS.filter_map do |component_key, count_key|
-          quantity = server_tool_use[count_key.to_s].to_i
-          next if quantity.zero?
-
-          Billing::LineItem.build(
-            component_key: component_key,
-            quantity: quantity,
-            cost_status: Billing::CostStatus::UNKNOWN,
-            pricing_basis: :provider_usage,
-            provider_field: "usage.server_tool_use.#{count_key}"
-          )
-        end
-      end
-
-      def token_usage(usage:, cache_read:)
-        input = usage["input_tokens"].to_i
-        output = usage["output_tokens"].to_i
-        cache_creation = usage["cache_creation"]
-        if cache_creation.is_a?(Hash)
-          cache_write = cache_creation["ephemeral_5m_input_tokens"].to_i
-          cache_write_extended = cache_creation["ephemeral_1h_input_tokens"].to_i
-        else
-          warn_unexpected_cache_creation(cache_creation, usage)
-          cache_write = usage["cache_creation_input_tokens"].to_i
-          cache_write_extended = 0
-        end
-        hidden_output = (
-          usage["thinking_tokens"] || usage["thinking_output_tokens"] ||
-            usage.dig("output_tokens_details", "reasoning_tokens")
-        ).to_i
-
-        TokenUsage.build(
-          input_tokens: input,
-          output_tokens: output,
-          total_tokens: input + output + cache_read + cache_write + cache_write_extended,
-          cache_read_input_tokens: cache_read,
-          cache_write_input_tokens: cache_write,
-          cache_write_extended_input_tokens: cache_write_extended,
-          hidden_output_tokens: hidden_output
-        )
-      end
-
-      def warn_unexpected_cache_creation(cache_creation, usage)
-        return if cache_creation.nil? || usage.key?("cache_creation_input_tokens")
-
-        Logging.warn("Anthropic usage.cache_creation has unexpected shape: #{cache_creation.class}")
-      end
-
-      def pricing_mode(request:, usage:)
-        modes = []
-        speed = usage&.fetch("speed", nil) || request["speed"]
-        service_tier = usage&.fetch("service_tier", nil) || request["service_tier"]
-        service_tier = nil if Providers::Anthropic::TierClassification.standard_equivalent_tier?(service_tier)
-
-        modes << Pricing::Mode.normalize(speed)
-        modes << Pricing::Mode.normalize(service_tier)
-        geo = inference_geo(request: request, usage: usage).downcase
-        modes << "data_residency" if Providers::Anthropic::TierClassification.data_residency_geo?(geo)
-
-        modes = modes.compact.uniq
-        modes.empty? ? nil : modes.join("_")
-      end
-
-      def inference_geo(request:, usage:)
-        (usage&.fetch("inference_geo", nil) || request["inference_geo"]).to_s
-      end
-    end
-  end
-end

data/lib/llm_cost_tracker/parsers/azure.rb

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "base"
-require_relative "openai_usage"
-require_relative "../providers/azure/hosts"
-
-module LlmCostTracker
-  module Parsers
-    class Azure < Base
-      include OpenaiUsage
-
-      TRACKED_ENDPOINTS = %w[
-        chat/completions completions embeddings moderations responses
-        audio/transcriptions audio/translations audio/speech
-        images/generations images/edits images/variations
-      ].freeze
-
-      PATH_PATTERN = %r{\A/openai/(?:deployments/[^/]+|v1)/(?:#{TRACKED_ENDPOINTS.join('|')})\z}
-
-      class << self
-        def match?(url)
-          uri_matches?(url) do |uri|
-            LlmCostTracker::Providers::Azure::Hosts.openai?(uri.host) && uri.path.to_s.match?(PATH_PATTERN)
-          end
-        end
-
-        def provider_names
-          %w[azure_openai]
-        end
-      end
-
-      def provider_for(_request_url)
-        "azure_openai"
-      end
-
-      def model_for(request_url, request_parsed)
-        body_model = super
-        return body_model if body_model
-
-        uri = parsed_uri(request_url)
-        match = uri&.path&.match(%r{/openai/deployments/([^/]+)/})
-        match && match[1]
-      end
-    end
-  end
-end