llm_cost_tracker 0.14.1 → 0.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +82 -0
- data/README.md +16 -5
- data/app/assets/llm_cost_tracker/application.css +16 -0
- data/app/controllers/llm_cost_tracker/calls_controller.rb +1 -3
- data/app/controllers/llm_cost_tracker/models_controller.rb +2 -4
- data/app/controllers/llm_cost_tracker/tags_controller.rb +5 -4
- data/app/helpers/llm_cost_tracker/application_helper.rb +4 -25
- data/app/helpers/llm_cost_tracker/chart_helper.rb +6 -10
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +8 -28
- data/app/models/llm_cost_tracker/call.rb +10 -14
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +4 -15
- data/app/services/llm_cost_tracker/dashboard/filter.rb +5 -9
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +2 -5
- data/app/services/llm_cost_tracker/dashboard/pricing_overview.rb +2 -1
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +2 -3
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +1 -0
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +3 -5
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +1 -2
- data/app/views/llm_cost_tracker/calls/show.html.erb +8 -22
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +1 -1
- data/app/views/llm_cost_tracker/shared/_bar.html.erb +1 -3
- data/app/views/llm_cost_tracker/shared/_filter_pill_date.html.erb +1 -4
- data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb +1 -4
- data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb +1 -4
- data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +1 -4
- data/app/views/llm_cost_tracker/tags/show.html.erb +5 -5
- data/lib/llm_cost_tracker/budget/per_tag.rb +15 -9
- data/lib/llm_cost_tracker/budget.rb +44 -55
- data/lib/llm_cost_tracker/capture/event_window.rb +3 -2
- data/lib/llm_cost_tracker/capture/sdk_payload.rb +5 -1
- data/lib/llm_cost_tracker/capture/stream_collector.rb +25 -26
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +9 -23
- data/lib/llm_cost_tracker/capture_verifier.rb +1 -7
- data/lib/llm_cost_tracker/charges/line_item.rb +5 -1
- data/lib/llm_cost_tracker/configuration/budgets.rb +1 -1
- data/lib/llm_cost_tracker/configuration/pricing.rb +1 -1
- data/lib/llm_cost_tracker/configuration.rb +5 -9
- data/lib/llm_cost_tracker/doctor/price_check.rb +13 -3
- data/lib/llm_cost_tracker/doctor/schema_check.rb +1 -2
- data/lib/llm_cost_tracker/doctor.rb +19 -2
- data/lib/llm_cost_tracker/engine.rb +0 -1
- data/lib/llm_cost_tracker/event.rb +8 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +5 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_call_rollups.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +11 -9
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_indexes_generator.rb +3 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_per_tag_budgets_generator.rb +3 -8
- data/lib/llm_cost_tracker/ingestion/inbox.rb +2 -8
- data/lib/llm_cost_tracker/ingestion.rb +3 -1
- data/lib/llm_cost_tracker/integrations/anthropic.rb +63 -10
- data/lib/llm_cost_tracker/integrations/base.rb +39 -18
- data/lib/llm_cost_tracker/integrations/openai/batch_capture.rb +16 -13
- data/lib/llm_cost_tracker/integrations/openai/patches.rb +8 -0
- data/lib/llm_cost_tracker/integrations/openai.rb +42 -59
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +241 -76
- data/lib/llm_cost_tracker/integrations.rb +1 -20
- data/lib/llm_cost_tracker/ledger/isolation.rb +0 -7
- data/lib/llm_cost_tracker/ledger/period/totals.rb +14 -10
- data/lib/llm_cost_tracker/ledger/rollups.rb +14 -14
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +14 -3
- data/lib/llm_cost_tracker/ledger/schema/base.rb +5 -4
- data/lib/llm_cost_tracker/ledger/store.rb +2 -2
- data/lib/llm_cost_tracker/ledger/tags/breakdown.rb +1 -5
- data/lib/llm_cost_tracker/ledger/tags/encoding.rb +3 -12
- data/lib/llm_cost_tracker/ledger/tags/query.rb +0 -2
- data/lib/llm_cost_tracker/middleware/faraday.rb +36 -19
- data/lib/llm_cost_tracker/parsers.rb +7 -40
- data/lib/llm_cost_tracker/prices.json +2237 -154
- data/lib/llm_cost_tracker/pricing/backfill.rb +46 -13
- data/lib/llm_cost_tracker/pricing/calculation.rb +118 -75
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +11 -10
- data/lib/llm_cost_tracker/pricing/estimator.rb +5 -2
- data/lib/llm_cost_tracker/pricing/matcher.rb +36 -22
- data/lib/llm_cost_tracker/pricing/mode.rb +2 -13
- data/lib/llm_cost_tracker/pricing/price_key.rb +5 -3
- data/lib/llm_cost_tracker/pricing/rate.rb +1 -0
- data/lib/llm_cost_tracker/pricing/registry.rb +5 -16
- data/lib/llm_cost_tracker/pricing/service_rates.rb +1 -8
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +13 -9
- data/lib/llm_cost_tracker/pricing/sync.rb +22 -71
- data/lib/llm_cost_tracker/providers/anthropic/parser.rb +33 -3
- data/lib/llm_cost_tracker/providers/anthropic/response_parser.rb +11 -5
- data/lib/llm_cost_tracker/providers/anthropic/usage_extractor.rb +81 -12
- data/lib/llm_cost_tracker/providers/azure/parser.rb +3 -7
- data/lib/llm_cost_tracker/providers/gemini/parser.rb +172 -48
- data/lib/llm_cost_tracker/providers/gemini/usage_extractor.rb +13 -22
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +1 -1
- data/lib/llm_cost_tracker/providers/openai/parser.rb +11 -13
- data/lib/llm_cost_tracker/providers/openai/response_parser.rb +96 -26
- data/lib/llm_cost_tracker/providers/openai/service_charges.rb +57 -40
- data/lib/llm_cost_tracker/providers/openai/usage_extractor.rb +14 -1
- data/lib/llm_cost_tracker/providers/openai_compatible/parser.rb +3 -1
- data/lib/llm_cost_tracker/report/data.rb +1 -1
- data/lib/llm_cost_tracker/retention.rb +1 -3
- data/lib/llm_cost_tracker/tracker.rb +17 -9
- data/lib/llm_cost_tracker/usage/catalog.rb +13 -4
- data/lib/llm_cost_tracker/usage/dimension.rb +1 -1
- data/lib/llm_cost_tracker/usage/dimensions.yml +61 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/tasks/llm_cost_tracker.rake +24 -21
- data/llm_cost_tracker.gemspec +63 -0
- metadata +8 -7
|
@@ -27,39 +27,19 @@ module LlmCostTracker
|
|
|
27
27
|
line_items_from_output(output_items, request: request, model: model)
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
CHAT_COMPLETIONS_ANNOTATION_PROVIDER_FIELD = "choices.message.annotations.url_citation"
|
|
31
|
-
CHAT_COMPLETIONS_SEARCH_MODEL_PROVIDER_FIELD = "request.model"
|
|
32
|
-
|
|
33
30
|
def chat_completions_web_search_items(response, model: nil)
|
|
34
|
-
return [] unless response["choices"]
|
|
35
|
-
|
|
36
|
-
provider_field = chat_completions_search_provider_field(response["choices"], model)
|
|
37
|
-
return [] unless provider_field
|
|
31
|
+
return [] unless response["choices"] && chat_completions_search_model?(model)
|
|
38
32
|
|
|
39
33
|
[{ "type" => "web_search_call", "id" => response["id"], "action" => { "type" => "search" },
|
|
40
|
-
"provider_field" =>
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
def chat_completions_search_provider_field(choices, model)
|
|
44
|
-
return CHAT_COMPLETIONS_ANNOTATION_PROVIDER_FIELD if chat_completions_used_web_search?(choices)
|
|
45
|
-
return CHAT_COMPLETIONS_SEARCH_MODEL_PROVIDER_FIELD if chat_completions_search_model?(model)
|
|
46
|
-
|
|
47
|
-
nil
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def chat_completions_used_web_search?(choices)
|
|
51
|
-
Array(choices).any? do |choice|
|
|
52
|
-
Array(choice.dig("message", "annotations")).any? do |annotation|
|
|
53
|
-
annotation.is_a?(Hash) && annotation["type"].to_s == "url_citation"
|
|
54
|
-
end
|
|
55
|
-
end
|
|
34
|
+
"provider_field" => "request.model" }]
|
|
56
35
|
end
|
|
57
36
|
|
|
58
37
|
def billable?(item)
|
|
59
38
|
return false unless item.is_a?(Hash)
|
|
60
39
|
|
|
61
|
-
dimension =
|
|
40
|
+
dimension = item_dimension(item)
|
|
62
41
|
return false unless dimension
|
|
42
|
+
return item["status"] == "completed" if dimension == "image_generation_call"
|
|
63
43
|
return true unless dimension == "web_search_request"
|
|
64
44
|
|
|
65
45
|
action_type = item.dig("action", "type")
|
|
@@ -69,11 +49,11 @@ module LlmCostTracker
|
|
|
69
49
|
def store_output_item(output_items, item)
|
|
70
50
|
return unless item.is_a?(Hash)
|
|
71
51
|
|
|
72
|
-
dimension =
|
|
52
|
+
dimension = item_dimension(item)
|
|
73
53
|
return unless dimension
|
|
74
54
|
|
|
75
|
-
key = if dimension == "container_session" && item
|
|
76
|
-
"#{dimension}:#{item
|
|
55
|
+
key = if dimension == "container_session" && container_id(item)
|
|
56
|
+
"#{dimension}:#{container_id(item)}"
|
|
77
57
|
else
|
|
78
58
|
item["id"] || "#{item['type']}:#{output_items.length}"
|
|
79
59
|
end
|
|
@@ -87,7 +67,7 @@ module LlmCostTracker
|
|
|
87
67
|
return nil unless dimension_key
|
|
88
68
|
|
|
89
69
|
provider_item_id = if dimension_key == "container_session"
|
|
90
|
-
item
|
|
70
|
+
container_id(item) || item["id"]
|
|
91
71
|
else
|
|
92
72
|
item["id"]
|
|
93
73
|
end
|
|
@@ -103,13 +83,28 @@ module LlmCostTracker
|
|
|
103
83
|
end
|
|
104
84
|
|
|
105
85
|
def dimension_key_for(item, request:, model:)
|
|
106
|
-
dimension =
|
|
86
|
+
dimension = item_dimension(item)
|
|
107
87
|
return dimension unless dimension == "web_search_request"
|
|
108
88
|
return dimension unless web_search_preview_used?(request) || chat_completions_search_model?(model)
|
|
109
89
|
|
|
110
90
|
reasoning_model?(model) ? "web_search_preview_request_reasoning" : "web_search_preview_request_non_reasoning"
|
|
111
91
|
end
|
|
112
92
|
|
|
93
|
+
def item_dimension(item)
|
|
94
|
+
return "container_session" if item["type"] == "shell_call" && shell_container_id(item)
|
|
95
|
+
|
|
96
|
+
output_dimension(item["type"])
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def container_id(item)
|
|
100
|
+
item["container_id"] || shell_container_id(item)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def shell_container_id(item)
|
|
104
|
+
environment = item["environment"]
|
|
105
|
+
environment["container_id"] if environment.is_a?(Hash) && environment["type"] == "container_reference"
|
|
106
|
+
end
|
|
107
|
+
|
|
113
108
|
def output_dimension(type)
|
|
114
109
|
key = RESPONSE_OUTPUT_RENAMES[type] || type
|
|
115
110
|
dimension = Usage::Catalog[key]
|
|
@@ -144,19 +139,10 @@ module LlmCostTracker
|
|
|
144
139
|
{
|
|
145
140
|
status: item["status"],
|
|
146
141
|
action_type: item.dig("action", "type"),
|
|
147
|
-
container_id: item
|
|
142
|
+
container_id: container_id(item)
|
|
148
143
|
}.compact
|
|
149
144
|
end
|
|
150
145
|
|
|
151
|
-
def openai_stream_service_line_items(events, request: nil, model: nil)
|
|
152
|
-
output_items = []
|
|
153
|
-
each_event_data(events) do |data|
|
|
154
|
-
output_items.concat(Array(data.dig("response", "output")))
|
|
155
|
-
output_items << data["item"] if data["item"]
|
|
156
|
-
end
|
|
157
|
-
line_items_from_output(output_items, request: request, model: model)
|
|
158
|
-
end
|
|
159
|
-
|
|
160
146
|
def transcription_line_items(usage)
|
|
161
147
|
return [] unless usage
|
|
162
148
|
|
|
@@ -168,13 +154,44 @@ module LlmCostTracker
|
|
|
168
154
|
|
|
169
155
|
[Charges::LineItem.build(
|
|
170
156
|
dimension_key: "transcription_minute",
|
|
171
|
-
quantity: (seconds / 60
|
|
157
|
+
quantity: BigDecimal(seconds.to_s) / 60,
|
|
172
158
|
cost_status: Charges::CostStatus::UNKNOWN,
|
|
173
159
|
pricing_basis: "provider_usage",
|
|
174
160
|
provider_field: "usage.seconds",
|
|
175
161
|
details: { seconds: seconds }
|
|
176
162
|
)]
|
|
177
163
|
end
|
|
164
|
+
|
|
165
|
+
def speech_line_items(request)
|
|
166
|
+
input = request["input"]
|
|
167
|
+
return [] unless input.is_a?(String) && ModelFamilies.character_billed_tts?(request["model"])
|
|
168
|
+
|
|
169
|
+
[Charges::LineItem.build(
|
|
170
|
+
dimension_key: "text_to_speech_character",
|
|
171
|
+
quantity: input.length,
|
|
172
|
+
cost_status: Charges::CostStatus::UNKNOWN,
|
|
173
|
+
pricing_basis: "provider_usage",
|
|
174
|
+
provider_field: "request.input"
|
|
175
|
+
)]
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# OpenRouter bills BYOK inference to the user's own provider key and puts only its fee in usage.cost.
|
|
179
|
+
def billed_line_items(usage)
|
|
180
|
+
amounts = [usage[:cost]]
|
|
181
|
+
amounts << usage.dig(:cost_details, :upstream_inference_cost) if usage[:is_byok]
|
|
182
|
+
return [] unless amounts.all?(Numeric)
|
|
183
|
+
|
|
184
|
+
amount = amounts.sum { |value| BigDecimal(value.to_s) }
|
|
185
|
+
[Charges::LineItem.build(
|
|
186
|
+
dimension_key: "billed_request",
|
|
187
|
+
quantity: 1,
|
|
188
|
+
rate_amount: amount,
|
|
189
|
+
cost: amount,
|
|
190
|
+
pricing_basis: "provider_usage",
|
|
191
|
+
price_source: "provider_response",
|
|
192
|
+
provider_field: "usage.cost"
|
|
193
|
+
)]
|
|
194
|
+
end
|
|
178
195
|
end
|
|
179
196
|
end
|
|
180
197
|
end
|
|
@@ -8,9 +8,15 @@ module LlmCostTracker
|
|
|
8
8
|
module UsageExtractor
|
|
9
9
|
INPUT_DETAIL_KEYS = %i[input_tokens_details input_token_details prompt_tokens_details].freeze
|
|
10
10
|
OUTPUT_DETAIL_KEYS = %i[output_tokens_details output_token_details completion_tokens_details].freeze
|
|
11
|
+
CACHED_MODALITY_DIMENSIONS = {
|
|
12
|
+
audio_tokens: "audio_cache_read_input", image_tokens: "image_cache_read_input"
|
|
13
|
+
}.freeze
|
|
14
|
+
|
|
11
15
|
def self.token_usage(usage, model: nil)
|
|
12
16
|
input_tokens = (usage[:input_tokens] || usage[:prompt_tokens]).to_i
|
|
13
17
|
output_tokens = (usage[:output_tokens] || usage[:completion_tokens]).to_i
|
|
18
|
+
reasoning = hidden_output_tokens(usage)
|
|
19
|
+
output_tokens += reasoning if usage[:total_tokens].to_i == input_tokens + output_tokens + reasoning
|
|
14
20
|
cache_read = cache_read_input_tokens(usage)
|
|
15
21
|
cache_write = cache_write_input_tokens(usage)
|
|
16
22
|
uncached_input = [input_tokens - cache_read - cache_write, 0].max
|
|
@@ -35,10 +41,17 @@ module LlmCostTracker
|
|
|
35
41
|
audio_output_tokens: audio_output,
|
|
36
42
|
image_input_tokens: image_input,
|
|
37
43
|
image_output_tokens: image_output,
|
|
38
|
-
hidden_output_tokens:
|
|
44
|
+
hidden_output_tokens: reasoning
|
|
39
45
|
)
|
|
40
46
|
end
|
|
41
47
|
|
|
48
|
+
def self.cache_read_line_items(usage)
|
|
49
|
+
CACHED_MODALITY_DIMENSIONS.filter_map do |detail_key, dimension_key|
|
|
50
|
+
quantity = detail(usage, INPUT_DETAIL_KEYS, :cached_tokens_details, detail_key)
|
|
51
|
+
Charges::LineItem.build(dimension_key: dimension_key, quantity: quantity) if quantity.positive?
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
42
55
|
def self.split_output(output_tokens:,
|
|
43
56
|
image_output_details:,
|
|
44
57
|
text_output_details:,
|
|
@@ -10,7 +10,9 @@ module LlmCostTracker
|
|
|
10
10
|
|
|
11
11
|
class << self
|
|
12
12
|
def match?(url)
|
|
13
|
-
|
|
13
|
+
uri_matches?(url) do |uri|
|
|
14
|
+
TRACKED_PATH_SUFFIXES.any? { |suffix| uri.path.to_s.end_with?(suffix) } && !provider_for_uri(uri).nil?
|
|
15
|
+
end
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
def provider_names
|
|
@@ -34,7 +34,7 @@ module LlmCostTracker
|
|
|
34
34
|
end
|
|
35
35
|
from = now - days.days
|
|
36
36
|
scope = LlmCostTracker::Call.where(tracked_at: from..now)
|
|
37
|
-
tag_breakdowns ||= LlmCostTracker.configuration.tags.report_breakdown_keys
|
|
37
|
+
tag_breakdowns ||= LlmCostTracker.configuration.tags.report_breakdown_keys
|
|
38
38
|
aggregate = totals(scope)
|
|
39
39
|
|
|
40
40
|
new(
|
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Retention
|
|
5
5
|
DEFAULT_BATCH_SIZE = 5_000
|
|
6
|
-
ROLLUP_COLUMNS = %i[tracked_at total_cost pricing_snapshot provider].freeze
|
|
7
|
-
private_constant :ROLLUP_COLUMNS
|
|
8
6
|
|
|
9
7
|
class << self
|
|
10
8
|
def prune(older_than:, batch_size: DEFAULT_BATCH_SIZE, now: Time.now.utc)
|
|
@@ -84,7 +82,7 @@ module LlmCostTracker
|
|
|
84
82
|
def prunable_rows(cutoff, batch_size)
|
|
85
83
|
relation = LlmCostTracker::Call.where(tracked_at: ...cutoff).order(:id).limit(batch_size).lock
|
|
86
84
|
columns = [:id]
|
|
87
|
-
columns +=
|
|
85
|
+
columns += LlmCostTracker::Ledger::Rollups::SOURCE_COLUMNS if LlmCostTracker::Ledger::Rollups.cache_active?
|
|
88
86
|
relation.select(*columns).to_a
|
|
89
87
|
end
|
|
90
88
|
end
|
|
@@ -27,9 +27,6 @@ module LlmCostTracker
|
|
|
27
27
|
|
|
28
28
|
tags = build_tags(context_tags: context_tags, metadata: metadata)
|
|
29
29
|
|
|
30
|
-
unpriced = calculation.token_cost.nil? && event.token_usage.total_tokens.positive? &&
|
|
31
|
-
calculation.priced_line_items.none?(&:priced?)
|
|
32
|
-
|
|
33
30
|
event = build_event(event: event, calculation: calculation, tags: tags, latency_ms: latency_ms)
|
|
34
31
|
|
|
35
32
|
if Ingestion.async?
|
|
@@ -41,22 +38,33 @@ module LlmCostTracker
|
|
|
41
38
|
|
|
42
39
|
yield if block_given?
|
|
43
40
|
notify_subscribers(event)
|
|
44
|
-
Pricing::Unknown.process(event.model, pricing_mode: calculation.mode) if unpriced
|
|
45
41
|
behavior_override = :raise if enforce_budget
|
|
46
|
-
|
|
47
|
-
|
|
42
|
+
begin
|
|
43
|
+
signal_unpriced(event, calculation)
|
|
44
|
+
ensure
|
|
45
|
+
Budget.check!(event, behavior_override: behavior_override)
|
|
46
|
+
end
|
|
48
47
|
|
|
49
48
|
event
|
|
50
49
|
end
|
|
51
50
|
|
|
52
51
|
def build_tags(context_tags:, metadata:)
|
|
53
52
|
resolved = (context_tags || LlmCostTracker::Tags::Context.tags).to_h
|
|
54
|
-
|
|
55
|
-
LlmCostTracker::Tags::Sanitizer.cap(
|
|
53
|
+
merged = resolved.merge(LlmCostTracker::Tags::Sanitizer.call(metadata.to_h))
|
|
54
|
+
LlmCostTracker::Tags::Sanitizer.cap(merged.to_a.reverse.uniq { |key, _| key.to_s }.reverse.to_h).freeze
|
|
56
55
|
end
|
|
57
56
|
|
|
58
57
|
private
|
|
59
58
|
|
|
59
|
+
def signal_unpriced(event, calculation)
|
|
60
|
+
iterations, lines = calculation.priced_line_items.partition { |line| line.kind == "model_iteration" }
|
|
61
|
+
models = iterations.select(&:unpriced?).map { |line| line.details[:model] }
|
|
62
|
+
if calculation.token_cost.nil? && event.token_usage.total_tokens.positive? && lines.none?(&:priced?)
|
|
63
|
+
models.unshift(event.model)
|
|
64
|
+
end
|
|
65
|
+
models.each { |model| Pricing::Unknown.process(model, pricing_mode: calculation.mode) }
|
|
66
|
+
end
|
|
67
|
+
|
|
60
68
|
def notify_subscribers(event)
|
|
61
69
|
return unless ActiveSupport::Notifications.notifier.listening?(EVENT_NAME)
|
|
62
70
|
|
|
@@ -67,7 +75,7 @@ module LlmCostTracker
|
|
|
67
75
|
|
|
68
76
|
def build_event(event:, calculation:, tags:, latency_ms:)
|
|
69
77
|
event.with(
|
|
70
|
-
event_id: SecureRandom.uuid,
|
|
78
|
+
event_id: event.event_id || SecureRandom.uuid,
|
|
71
79
|
pricing_mode: calculation.mode,
|
|
72
80
|
cost: calculation.cost,
|
|
73
81
|
tags: tags,
|
|
@@ -27,7 +27,7 @@ module LlmCostTracker
|
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
def token_priced
|
|
30
|
-
@token_priced ||= all.select
|
|
30
|
+
@token_priced ||= all.select { |dimension| dimension.token_key && dimension.parent.nil? }.freeze
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def find_by(kind:, direction:, modality:, cache_state:, unit:)
|
|
@@ -35,8 +35,17 @@ module LlmCostTracker
|
|
|
35
35
|
end
|
|
36
36
|
|
|
37
37
|
def token_priced_for(kind:, direction:, cache_state:)
|
|
38
|
-
|
|
39
|
-
|
|
38
|
+
dimension = all.find do |candidate|
|
|
39
|
+
candidate.token? && candidate.kind == kind && candidate.direction == direction &&
|
|
40
|
+
candidate.cache_state == cache_state
|
|
41
|
+
end
|
|
42
|
+
dimension&.parent ? fetch(dimension.parent) : dimension
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def costs_by_component(rows)
|
|
46
|
+
rows.each_with_object({}) do |(kind, direction, cache_state, cost), totals|
|
|
47
|
+
component = token_priced_for(kind: kind, direction: direction, cache_state: cache_state)
|
|
48
|
+
totals[component.key] = totals.fetch(component.key, 0) + cost if component && cost
|
|
40
49
|
end
|
|
41
50
|
end
|
|
42
51
|
|
|
@@ -60,7 +69,7 @@ module LlmCostTracker
|
|
|
60
69
|
|
|
61
70
|
def build(attributes)
|
|
62
71
|
rate_basis = attributes[:rate_basis] || DEFAULT_RATE_BASIS_BY_UNIT.fetch(attributes.fetch(:unit))
|
|
63
|
-
Dimension.new(**attributes, rate_basis: rate_basis)
|
|
72
|
+
Dimension.new(parent: nil, **attributes, rate_basis: rate_basis)
|
|
64
73
|
end
|
|
65
74
|
end
|
|
66
75
|
end
|
|
@@ -61,6 +61,30 @@
|
|
|
61
61
|
cache_state: none
|
|
62
62
|
unit: token
|
|
63
63
|
|
|
64
|
+
- key: audio_cache_read_input
|
|
65
|
+
kind: audio_token
|
|
66
|
+
direction: input
|
|
67
|
+
modality: audio
|
|
68
|
+
cache_state: read
|
|
69
|
+
unit: token
|
|
70
|
+
parent: cache_read_input
|
|
71
|
+
|
|
72
|
+
- key: image_cache_read_input
|
|
73
|
+
kind: image_token
|
|
74
|
+
direction: input
|
|
75
|
+
modality: image
|
|
76
|
+
cache_state: read
|
|
77
|
+
unit: token
|
|
78
|
+
parent: cache_read_input
|
|
79
|
+
|
|
80
|
+
- key: video_input
|
|
81
|
+
kind: video_token
|
|
82
|
+
direction: input
|
|
83
|
+
modality: video
|
|
84
|
+
cache_state: none
|
|
85
|
+
unit: token
|
|
86
|
+
parent: input
|
|
87
|
+
|
|
64
88
|
- key: web_search_request
|
|
65
89
|
kind: web_search_request
|
|
66
90
|
direction: neither
|
|
@@ -101,6 +125,13 @@
|
|
|
101
125
|
unit: request
|
|
102
126
|
rate_basis: per_1k_requests
|
|
103
127
|
|
|
128
|
+
- key: image_generation_call
|
|
129
|
+
kind: image_generation_call
|
|
130
|
+
direction: output
|
|
131
|
+
modality: image
|
|
132
|
+
cache_state: none
|
|
133
|
+
unit: request
|
|
134
|
+
|
|
104
135
|
- key: container_session
|
|
105
136
|
kind: container_session
|
|
106
137
|
direction: neither
|
|
@@ -125,6 +156,22 @@
|
|
|
125
156
|
unit: request
|
|
126
157
|
rate_basis: per_1k_requests
|
|
127
158
|
|
|
159
|
+
- key: maps_grounding_request
|
|
160
|
+
kind: maps_grounding_request
|
|
161
|
+
direction: neither
|
|
162
|
+
modality: text
|
|
163
|
+
cache_state: none
|
|
164
|
+
unit: request
|
|
165
|
+
rate_basis: per_1k_requests
|
|
166
|
+
|
|
167
|
+
- key: cache_storage_token_hour
|
|
168
|
+
kind: cache_storage_token_hour
|
|
169
|
+
direction: neither
|
|
170
|
+
modality: none
|
|
171
|
+
cache_state: none
|
|
172
|
+
unit: token_hour
|
|
173
|
+
rate_basis: per_million_token_hours
|
|
174
|
+
|
|
128
175
|
- key: text_to_speech_character
|
|
129
176
|
kind: text_to_speech_character
|
|
130
177
|
direction: output
|
|
@@ -140,3 +187,17 @@
|
|
|
140
187
|
cache_state: none
|
|
141
188
|
unit: minute
|
|
142
189
|
rate_basis: per_minute
|
|
190
|
+
|
|
191
|
+
- key: billed_request
|
|
192
|
+
kind: billed_request
|
|
193
|
+
direction: neither
|
|
194
|
+
modality: none
|
|
195
|
+
cache_state: none
|
|
196
|
+
unit: request
|
|
197
|
+
|
|
198
|
+
- key: model_iteration
|
|
199
|
+
kind: model_iteration
|
|
200
|
+
direction: neither
|
|
201
|
+
modality: none
|
|
202
|
+
cache_state: none
|
|
203
|
+
unit: request
|
|
@@ -16,7 +16,7 @@ end
|
|
|
16
16
|
|
|
17
17
|
# rubocop:disable-next Metrics/BlockLength
|
|
18
18
|
namespace :llm_cost_tracker do
|
|
19
|
-
desc "Install LLM Cost Tracker with
|
|
19
|
+
desc "Install LLM Cost Tracker with a local prices file, migrate, run doctor, and print dashboard mount steps"
|
|
20
20
|
task :setup do
|
|
21
21
|
Rails::Generators.invoke("llm_cost_tracker:install", %w[--dashboard --prices --skip])
|
|
22
22
|
begin
|
|
@@ -54,7 +54,7 @@ namespace :llm_cost_tracker do
|
|
|
54
54
|
puts LlmCostTracker::Report.generate(days: days)
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
-
desc "Recompute total_cost for calls with unknown pricing using the current price registry. " \
|
|
57
|
+
desc "Recompute total_cost for calls with unknown or partial pricing using the current price registry. " \
|
|
58
58
|
"Use BATCH_SIZE=N to tune."
|
|
59
59
|
task backfill_unknown_pricing: :environment do
|
|
60
60
|
require_relative "../llm_cost_tracker/pricing/backfill"
|
|
@@ -64,7 +64,22 @@ namespace :llm_cost_tracker do
|
|
|
64
64
|
"still unknown #{result.still_unknown}"
|
|
65
65
|
end
|
|
66
66
|
|
|
67
|
-
desc "
|
|
67
|
+
desc "Reprice calls tracked from FROM (and before TO, if set) at the current price registry, including " \
|
|
68
|
+
"calls already priced; rollups and per-tag costs move by the difference. Use BATCH_SIZE=N to tune."
|
|
69
|
+
task reprice: :environment do
|
|
70
|
+
require_relative "../llm_cost_tracker/pricing/backfill"
|
|
71
|
+
from = Time.zone.parse(ENV["FROM"].to_s)
|
|
72
|
+
abort("llm_cost_tracker: set FROM to the first day to reprice, e.g. FROM=2026-09-01") unless from
|
|
73
|
+
to_text = ENV["TO"].presence
|
|
74
|
+
to = to_text && Time.zone.parse(to_text)
|
|
75
|
+
abort("llm_cost_tracker: TO=#{to_text} is not a date, e.g. TO=2026-10-01") if to_text && !to
|
|
76
|
+
batch_size = (ENV["BATCH_SIZE"] || LlmCostTracker::Pricing::Backfill::DEFAULT_BATCH_SIZE).to_i
|
|
77
|
+
scope = LlmCostTracker::Pricing::Backfill.reprice_scope(from...to)
|
|
78
|
+
result = LlmCostTracker::Pricing::Backfill.call(scope: scope, batch_size: batch_size, reprice: true)
|
|
79
|
+
puts "llm_cost_tracker: examined #{result.examined} calls, repriced #{result.recomputed}"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
desc "Delete llm_cost_tracker_calls and ingestion inbox rows older than DAYS (default: 90). Use BATCH_SIZE=N to tune."
|
|
68
83
|
task prune: :environment do
|
|
69
84
|
days = (ENV["DAYS"] || 90).to_i
|
|
70
85
|
batch_size = (ENV["BATCH_SIZE"] || LlmCostTracker::Retention::DEFAULT_BATCH_SIZE).to_i
|
|
@@ -98,32 +113,20 @@ namespace :llm_cost_tracker do
|
|
|
98
113
|
namespace :prices do
|
|
99
114
|
desc(
|
|
100
115
|
"Refresh the configured pricing file from the maintained LLM Cost Tracker price snapshot. " \
|
|
101
|
-
"
|
|
102
|
-
"or OUTPUT=path/to/file.json."
|
|
116
|
+
"Review changes first with llm_cost_tracker:prices:check. Use FORCE=1 to accept suspicious price changes, " \
|
|
117
|
+
"URL=... to override the source, or OUTPUT=path/to/file.json."
|
|
103
118
|
)
|
|
104
119
|
task :refresh do
|
|
105
120
|
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
106
121
|
require_relative "../llm_cost_tracker"
|
|
107
122
|
|
|
123
|
+
abort("llm_cost_tracker: PREVIEW is not supported; run llm_cost_tracker:prices:check") if ENV.key?("PREVIEW")
|
|
124
|
+
|
|
108
125
|
output_path = LlmCostTrackerTasks.price_refresh_output_path
|
|
109
126
|
source_url = LlmCostTracker::Pricing::Sync.configured_remote_url
|
|
110
|
-
|
|
111
|
-
result = LlmCostTracker::Pricing::Sync.refresh(
|
|
112
|
-
path: output_path,
|
|
113
|
-
url: source_url,
|
|
114
|
-
preview: preview,
|
|
115
|
-
force: ENV["FORCE"] == "1"
|
|
116
|
-
)
|
|
117
|
-
|
|
118
|
-
action = if preview
|
|
119
|
-
"previewed"
|
|
120
|
-
elsif result.written
|
|
121
|
-
"refreshed"
|
|
122
|
-
else
|
|
123
|
-
"kept"
|
|
124
|
-
end
|
|
127
|
+
result = LlmCostTracker::Pricing::Sync.refresh(path: output_path, url: source_url, force: ENV["FORCE"] == "1")
|
|
125
128
|
|
|
126
|
-
puts "llm_cost_tracker: #{
|
|
129
|
+
puts "llm_cost_tracker: #{result.written ? 'refreshed' : 'kept'} pricing file #{result.path}"
|
|
127
130
|
puts " source: #{result.source_url}"
|
|
128
131
|
puts " version: #{result.source_version.inspect}" if result.source_version
|
|
129
132
|
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "lib/llm_cost_tracker/version"
|
|
4
|
+
|
|
5
|
+
Gem::Specification.new do |spec|
|
|
6
|
+
spec.name = "llm_cost_tracker"
|
|
7
|
+
spec.version = LlmCostTracker::VERSION
|
|
8
|
+
spec.authors = ["Sergii Khomenko"]
|
|
9
|
+
spec.email = ["sergey@mm.st"]
|
|
10
|
+
|
|
11
|
+
spec.summary = "LLM API cost tracking for Rails applications"
|
|
12
|
+
spec.description = "Logs every call your Rails app makes to OpenAI, Anthropic, Gemini, RubyLLM, " \
|
|
13
|
+
"or an OpenAI-compatible API: tokens, cost, latency, tags. Calls go straight " \
|
|
14
|
+
"to the provider — no proxy. Includes price sync, budget guardrails, and a " \
|
|
15
|
+
"mountable dashboard."
|
|
16
|
+
spec.homepage = "https://github.com/sergey-homenko/llm_cost_tracker"
|
|
17
|
+
spec.license = "MIT"
|
|
18
|
+
|
|
19
|
+
spec.required_ruby_version = ">= 3.3.0"
|
|
20
|
+
|
|
21
|
+
spec.metadata["bug_tracker_uri"] = "#{spec.homepage}/issues"
|
|
22
|
+
spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
|
|
23
|
+
spec.metadata["source_code_uri"] = spec.homepage
|
|
24
|
+
spec.metadata["documentation_uri"] = "#{spec.homepage}#readme"
|
|
25
|
+
spec.metadata["rubygems_mfa_required"] = "true"
|
|
26
|
+
|
|
27
|
+
spec.files = Dir.chdir(__dir__) do
|
|
28
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
|
29
|
+
(File.expand_path(f) == __FILE__) ||
|
|
30
|
+
f.start_with?("bin/",
|
|
31
|
+
"test/",
|
|
32
|
+
"spec/",
|
|
33
|
+
"scripts/",
|
|
34
|
+
"docs/",
|
|
35
|
+
".git",
|
|
36
|
+
".github",
|
|
37
|
+
"gemfiles/",
|
|
38
|
+
".rubocop",
|
|
39
|
+
"Gemfile") ||
|
|
40
|
+
f == "codecov.yml"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
spec.require_paths = ["lib"]
|
|
45
|
+
|
|
46
|
+
spec.add_dependency "activerecord", ">= 8.0", "< 9.0"
|
|
47
|
+
spec.add_dependency "activesupport", ">= 8.0", "< 9.0"
|
|
48
|
+
spec.add_dependency "csv", "~> 3.0"
|
|
49
|
+
spec.add_dependency "faraday", ">= 2.0", "< 3.0"
|
|
50
|
+
spec.add_dependency "railties", ">= 8.0", "< 9.0"
|
|
51
|
+
|
|
52
|
+
spec.add_development_dependency "anthropic", "~> 1.61"
|
|
53
|
+
spec.add_development_dependency "nokogiri", "~> 1.19"
|
|
54
|
+
spec.add_development_dependency "openai", "~> 0.80"
|
|
55
|
+
spec.add_development_dependency "pg", "~> 1.6"
|
|
56
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
|
57
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
|
58
|
+
spec.add_development_dependency "rubocop", "~> 1.0"
|
|
59
|
+
spec.add_development_dependency "ruby_llm", ">= 1.16", "< 3"
|
|
60
|
+
spec.add_development_dependency "simplecov", "~> 0.22"
|
|
61
|
+
spec.add_development_dependency "simplecov-lcov", "~> 0.8"
|
|
62
|
+
spec.add_development_dependency "webmock", "~> 3.0"
|
|
63
|
+
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm_cost_tracker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.14.
|
|
4
|
+
version: 0.14.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sergii Khomenko
|
|
@@ -15,7 +15,7 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '
|
|
18
|
+
version: '8.0'
|
|
19
19
|
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
21
|
version: '9.0'
|
|
@@ -25,7 +25,7 @@ dependencies:
|
|
|
25
25
|
requirements:
|
|
26
26
|
- - ">="
|
|
27
27
|
- !ruby/object:Gem::Version
|
|
28
|
-
version: '
|
|
28
|
+
version: '8.0'
|
|
29
29
|
- - "<"
|
|
30
30
|
- !ruby/object:Gem::Version
|
|
31
31
|
version: '9.0'
|
|
@@ -35,7 +35,7 @@ dependencies:
|
|
|
35
35
|
requirements:
|
|
36
36
|
- - ">="
|
|
37
37
|
- !ruby/object:Gem::Version
|
|
38
|
-
version: '
|
|
38
|
+
version: '8.0'
|
|
39
39
|
- - "<"
|
|
40
40
|
- !ruby/object:Gem::Version
|
|
41
41
|
version: '9.0'
|
|
@@ -45,7 +45,7 @@ dependencies:
|
|
|
45
45
|
requirements:
|
|
46
46
|
- - ">="
|
|
47
47
|
- !ruby/object:Gem::Version
|
|
48
|
-
version: '
|
|
48
|
+
version: '8.0'
|
|
49
49
|
- - "<"
|
|
50
50
|
- !ruby/object:Gem::Version
|
|
51
51
|
version: '9.0'
|
|
@@ -89,7 +89,7 @@ dependencies:
|
|
|
89
89
|
requirements:
|
|
90
90
|
- - ">="
|
|
91
91
|
- !ruby/object:Gem::Version
|
|
92
|
-
version: '
|
|
92
|
+
version: '8.0'
|
|
93
93
|
- - "<"
|
|
94
94
|
- !ruby/object:Gem::Version
|
|
95
95
|
version: '9.0'
|
|
@@ -99,7 +99,7 @@ dependencies:
|
|
|
99
99
|
requirements:
|
|
100
100
|
- - ">="
|
|
101
101
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '
|
|
102
|
+
version: '8.0'
|
|
103
103
|
- - "<"
|
|
104
104
|
- !ruby/object:Gem::Version
|
|
105
105
|
version: '9.0'
|
|
@@ -482,6 +482,7 @@ files:
|
|
|
482
482
|
- lib/llm_cost_tracker/usage/token_usage.rb
|
|
483
483
|
- lib/llm_cost_tracker/version.rb
|
|
484
484
|
- lib/tasks/llm_cost_tracker.rake
|
|
485
|
+
- llm_cost_tracker.gemspec
|
|
485
486
|
homepage: https://github.com/sergey-homenko/llm_cost_tracker
|
|
486
487
|
licenses:
|
|
487
488
|
- MIT
|