llm_cost_tracker 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +116 -0
- data/LICENSE.txt +28 -0
- data/README.md +33 -41
- data/app/controllers/llm_cost_tracker/application_controller.rb +7 -1
- data/app/controllers/llm_cost_tracker/calls_controller.rb +12 -94
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +11 -12
- data/app/controllers/llm_cost_tracker/pricing_controller.rb +15 -4
- data/app/controllers/llm_cost_tracker/tags_controller.rb +21 -14
- data/app/helpers/llm_cost_tracker/application_helper.rb +22 -31
- data/app/helpers/llm_cost_tracker/chart_helper.rb +57 -108
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +30 -17
- data/app/helpers/llm_cost_tracker/sortable_table_helper.rb +18 -25
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +30 -0
- data/app/models/llm_cost_tracker/call.rb +6 -4
- data/app/models/llm_cost_tracker/call_rollup.rb +6 -6
- data/app/services/llm_cost_tracker/dashboard/call_order.rb +36 -0
- data/app/services/llm_cost_tracker/dashboard/calls_export.rb +68 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality/stats_query.rb +77 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality/summary.rb +47 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +66 -210
- data/app/services/llm_cost_tracker/dashboard/filter.rb +2 -1
- data/app/services/llm_cost_tracker/dashboard/monthly_budget.rb +42 -35
- data/app/services/llm_cost_tracker/dashboard/percent.rb +12 -0
- data/app/services/llm_cost_tracker/dashboard/sort.rb +9 -0
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +19 -17
- data/app/services/llm_cost_tracker/dashboard/spend_chart.rb +50 -0
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +7 -8
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +1 -4
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +3 -4
- data/app/views/layouts/llm_cost_tracker/application.html.erb +6 -6
- data/app/views/llm_cost_tracker/calls/index.html.erb +2 -1
- data/app/views/llm_cost_tracker/calls/show.html.erb +3 -14
- data/app/views/llm_cost_tracker/errors/query.html.erb +6 -0
- data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +1 -1
- data/lib/llm_cost_tracker/budget/estimate.rb +35 -0
- data/lib/llm_cost_tracker/budget/global.rb +69 -0
- data/lib/llm_cost_tracker/budget/limit.rb +52 -0
- data/lib/llm_cost_tracker/budget/per_tag.rb +59 -27
- data/lib/llm_cost_tracker/budget/tagged.rb +74 -0
- data/lib/llm_cost_tracker/budget.rb +19 -157
- data/lib/llm_cost_tracker/capture/event_window.rb +16 -13
- data/lib/llm_cost_tracker/capture/sdk_payload.rb +22 -17
- data/lib/llm_cost_tracker/capture/sse.rb +7 -5
- data/lib/llm_cost_tracker/capture/stream_collector.rb +89 -96
- data/lib/llm_cost_tracker/capture/stream_tap.rb +2 -2
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +32 -37
- data/lib/llm_cost_tracker/capture_verifier.rb +2 -3
- data/lib/llm_cost_tracker/charges/cost_status.rb +23 -19
- data/lib/llm_cost_tracker/charges/line_item.rb +31 -33
- data/lib/llm_cost_tracker/configuration/budgets.rb +49 -18
- data/lib/llm_cost_tracker/configuration/capture.rb +7 -1
- data/lib/llm_cost_tracker/configuration/mutability.rb +3 -3
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +20 -2
- data/lib/llm_cost_tracker/doctor/price_check.rb +24 -35
- data/lib/llm_cost_tracker/doctor/probe.rb +1 -1
- data/lib/llm_cost_tracker/doctor.rb +23 -20
- data/lib/llm_cost_tracker/engine.rb +8 -0
- data/lib/llm_cost_tracker/errors.rb +2 -4
- data/lib/llm_cost_tracker/event.rb +22 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +4 -2
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +6 -4
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +1 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +1 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb +2 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb +1 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +33 -19
- data/lib/llm_cost_tracker/ingestion/inbox.rb +20 -11
- data/lib/llm_cost_tracker/ingestion/worker.rb +58 -37
- data/lib/llm_cost_tracker/ingestion.rb +20 -35
- data/lib/llm_cost_tracker/integrations/anthropic/batch_results_capture.rb +40 -0
- data/lib/llm_cost_tracker/integrations/anthropic/patches.rb +56 -0
- data/lib/llm_cost_tracker/integrations/anthropic.rb +59 -120
- data/lib/llm_cost_tracker/integrations/base/patch_target.rb +35 -0
- data/lib/llm_cost_tracker/integrations/base.rb +65 -81
- data/lib/llm_cost_tracker/integrations/openai/batch_capture.rb +20 -19
- data/lib/llm_cost_tracker/integrations/openai/patches.rb +51 -40
- data/lib/llm_cost_tracker/integrations/openai/websocket_capture.rb +89 -0
- data/lib/llm_cost_tracker/integrations/openai.rb +114 -83
- data/lib/llm_cost_tracker/integrations/ruby_llm/v1/patches.rb +67 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v1/reply.rb +155 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v1.rb +154 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v2/attempt/token_counts.rb +159 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v2/attempt.rb +175 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v2/frame.rb +75 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v2/recorder.rb +70 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v2/seams.rb +86 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm/v2.rb +161 -0
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +7 -408
- data/lib/llm_cost_tracker/ledger/isolation.rb +15 -2
- data/lib/llm_cost_tracker/ledger/period/totals.rb +1 -1
- data/lib/llm_cost_tracker/ledger/rollups.rb +24 -8
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +23 -6
- data/lib/llm_cost_tracker/ledger/schema/base.rb +9 -5
- data/lib/llm_cost_tracker/ledger/store.rb +17 -35
- data/lib/llm_cost_tracker/ledger/tags/breakdown.rb +2 -2
- data/lib/llm_cost_tracker/middleware/faraday/body.rb +57 -0
- data/lib/llm_cost_tracker/middleware/faraday/exchange.rb +153 -0
- data/lib/llm_cost_tracker/middleware/faraday/response_reader.rb +75 -0
- data/lib/llm_cost_tracker/middleware/faraday/stream_tee.rb +47 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +2 -301
- data/lib/llm_cost_tracker/parsers.rb +12 -0
- data/lib/llm_cost_tracker/prices.json +1179 -208
- data/lib/llm_cost_tracker/pricing/backfill.rb +59 -29
- data/lib/llm_cost_tracker/pricing/calculation/iterations.rb +49 -0
- data/lib/llm_cost_tracker/pricing/calculation/quantities.rb +40 -0
- data/lib/llm_cost_tracker/pricing/calculation/snapshot.rb +51 -0
- data/lib/llm_cost_tracker/pricing/calculation/totals.rb +59 -0
- data/lib/llm_cost_tracker/pricing/calculation.rb +82 -173
- data/lib/llm_cost_tracker/pricing/matcher.rb +31 -14
- data/lib/llm_cost_tracker/pricing/mode.rb +6 -19
- data/lib/llm_cost_tracker/pricing/off_peak.rb +50 -0
- data/lib/llm_cost_tracker/pricing/rate.rb +1 -0
- data/lib/llm_cost_tracker/pricing/registry/normalizer.rb +110 -0
- data/lib/llm_cost_tracker/pricing/registry.rb +38 -111
- data/lib/llm_cost_tracker/pricing/sync/change_printer.rb +40 -20
- data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +35 -34
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +29 -18
- data/lib/llm_cost_tracker/pricing/sync/remote_snapshot.rb +78 -0
- data/lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb +2 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +7 -53
- data/lib/llm_cost_tracker/pricing/unknown.rb +1 -1
- data/lib/llm_cost_tracker/providers/anthropic/parser.rb +30 -25
- data/lib/llm_cost_tracker/providers/anthropic/response_parser.rb +5 -3
- data/lib/llm_cost_tracker/providers/anthropic/usage_extractor.rb +122 -101
- data/lib/llm_cost_tracker/providers/azure/hosts.rb +1 -1
- data/lib/llm_cost_tracker/providers/azure/parser.rb +3 -6
- data/lib/llm_cost_tracker/providers/gemini/grounding.rb +87 -0
- data/lib/llm_cost_tracker/providers/gemini/parser.rb +120 -183
- data/lib/llm_cost_tracker/providers/gemini/usage_extractor.rb +66 -47
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +11 -0
- data/lib/llm_cost_tracker/providers/openai/model_families.rb +1 -1
- data/lib/llm_cost_tracker/providers/openai/output_item.rb +84 -0
- data/lib/llm_cost_tracker/providers/openai/parser.rb +2 -1
- data/lib/llm_cost_tracker/providers/openai/response_parser.rb +81 -149
- data/lib/llm_cost_tracker/providers/openai/service_charges.rb +133 -156
- data/lib/llm_cost_tracker/providers/openai/stream_parser.rb +127 -0
- data/lib/llm_cost_tracker/providers/openai/usage_extractor.rb +72 -62
- data/lib/llm_cost_tracker/providers/openai_compatible/parser.rb +15 -4
- data/lib/llm_cost_tracker/providers.rb +3 -0
- data/lib/llm_cost_tracker/redaction.rb +13 -0
- data/lib/llm_cost_tracker/report/data.rb +25 -28
- data/lib/llm_cost_tracker/tags/context.rb +22 -7
- data/lib/llm_cost_tracker/tracker.rb +32 -26
- data/lib/llm_cost_tracker/usage/dimensions.yml +24 -0
- data/lib/llm_cost_tracker/usage/token_usage.rb +8 -4
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +8 -4
- data/lib/tasks/llm_cost_tracker.rake +35 -5
- data/llm_cost_tracker.gemspec +6 -5
- metadata +58 -5
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_support/core_ext/object/try"
|
|
4
|
-
|
|
5
3
|
module LlmCostTracker
|
|
6
4
|
module Capture
|
|
7
5
|
class StreamTracker
|
|
@@ -20,10 +18,7 @@ module LlmCostTracker
|
|
|
20
18
|
|
|
21
19
|
iterator = @stream.instance_variable_get(:@iterator)
|
|
22
20
|
if iterator.respond_to?(:each)
|
|
23
|
-
|
|
24
|
-
Enumerator.new do |yielder|
|
|
25
|
-
each_from(iterator) { |event| yielder << event }
|
|
26
|
-
end)
|
|
21
|
+
wrap_iterator(iterator)
|
|
27
22
|
elsif @stream.respond_to?(:each)
|
|
28
23
|
wrap_each
|
|
29
24
|
else
|
|
@@ -41,30 +36,40 @@ module LlmCostTracker
|
|
|
41
36
|
|
|
42
37
|
private
|
|
43
38
|
|
|
39
|
+
def wrap_iterator(iterator)
|
|
40
|
+
relayed = Enumerator.new { |yielder| relay(iterator.method(:each)) { |event| yielder << event } }
|
|
41
|
+
@stream.instance_variable_set(:@iterator, relayed)
|
|
42
|
+
close_through(iterator) if @stream.respond_to?(:close)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def close_through(iterator)
|
|
46
|
+
close = @stream.method(:close)
|
|
47
|
+
finish = method(:finish!)
|
|
48
|
+
@stream.define_singleton_method(:close) do
|
|
49
|
+
@iterator = iterator
|
|
50
|
+
close.call
|
|
51
|
+
ensure
|
|
52
|
+
finish.call(errored: false)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
44
56
|
def wrap_each
|
|
45
|
-
tracker = self
|
|
46
57
|
original_each = @stream.method(:each)
|
|
58
|
+
relayed_each = ->(&block) { relay(original_each, &block) }
|
|
47
59
|
@stream.define_singleton_method(:each) do |&block|
|
|
48
60
|
next enum_for(:each) unless block
|
|
49
61
|
|
|
50
|
-
|
|
62
|
+
relayed_each.call(&block)
|
|
51
63
|
end
|
|
52
64
|
end
|
|
53
65
|
|
|
54
|
-
def
|
|
66
|
+
def relay(source)
|
|
55
67
|
errored = false
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
yield event
|
|
60
|
-
end
|
|
61
|
-
else
|
|
62
|
-
iterable.call do |event|
|
|
63
|
-
capture(event)
|
|
64
|
-
yield event
|
|
65
|
-
end
|
|
68
|
+
source.call do |event|
|
|
69
|
+
capture(event)
|
|
70
|
+
yield event
|
|
66
71
|
end
|
|
67
|
-
rescue Exception # rubocop:disable Lint/RescueException
|
|
72
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
68
73
|
errored = true
|
|
69
74
|
raise
|
|
70
75
|
ensure
|
|
@@ -72,31 +77,21 @@ module LlmCostTracker
|
|
|
72
77
|
end
|
|
73
78
|
|
|
74
79
|
def capture(event)
|
|
75
|
-
|
|
80
|
+
data = event.to_h if event.respond_to?(:to_h)
|
|
81
|
+
type = event.type if event.respond_to?(:type)
|
|
82
|
+
@collector.event(data || {}, type: type&.to_s)
|
|
76
83
|
rescue StandardError => e
|
|
77
84
|
warn_capture_failure(e)
|
|
78
85
|
end
|
|
79
86
|
|
|
80
87
|
def warn_capture_failure(error)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
@capture_failed = true
|
|
85
|
-
true
|
|
86
|
-
end
|
|
87
|
-
return unless should_warn
|
|
88
|
-
|
|
89
|
-
Logging.warn("stream integration failed to capture event: #{error.class}: #{error.message}")
|
|
88
|
+
first_failure = @mutex.synchronize { !@capture_failed && (@capture_failed = true) }
|
|
89
|
+
Logging.warn("stream integration failed to capture event: #{error.class}: #{error.message}") if first_failure
|
|
90
90
|
end
|
|
91
91
|
|
|
92
92
|
def finish!(errored:)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
@finished = true
|
|
97
|
-
true
|
|
98
|
-
end
|
|
99
|
-
return unless should_finish && @active.call
|
|
93
|
+
claimed = @mutex.synchronize { !@finished && (@finished = true) }
|
|
94
|
+
return unless claimed && @active.call
|
|
100
95
|
|
|
101
96
|
begin
|
|
102
97
|
@finish.call(errored)
|
|
@@ -11,9 +11,8 @@ module LlmCostTracker
|
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def report(checks = call)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
end).join("\n")
|
|
14
|
+
lines = checks.map { |check| "[#{check.status}] #{check.name}: #{check.message}" }
|
|
15
|
+
["LLM Cost Tracker capture verification", *lines].join("\n")
|
|
17
16
|
end
|
|
18
17
|
|
|
19
18
|
def healthy?(checks = call)
|
|
@@ -11,12 +11,18 @@ module LlmCostTracker
|
|
|
11
11
|
UNKNOWN = "unknown"
|
|
12
12
|
INCOMPLETE = [UNKNOWN, PARTIAL].freeze
|
|
13
13
|
|
|
14
|
+
TokenCharge = Data.define(:cost, :partial) do
|
|
15
|
+
def priced? = !cost.nil?
|
|
16
|
+
|
|
17
|
+
def unpriced? = cost.nil? || partial
|
|
18
|
+
end
|
|
19
|
+
private_constant :TokenCharge
|
|
20
|
+
|
|
14
21
|
def self.unknown_pricing_sql(total_cost: "total_cost", cost_status: "cost_status")
|
|
15
|
-
statuses = INCOMPLETE.map { |status| ActiveRecord::Base.
|
|
22
|
+
statuses = INCOMPLETE.map { |status| ActiveRecord::Base.lease_connection.quote(status) }.join(", ")
|
|
16
23
|
"#{total_cost} IS NULL OR #{cost_status} IN (#{statuses})"
|
|
17
24
|
end
|
|
18
25
|
|
|
19
|
-
# rubocop:disable-next Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
20
26
|
def self.call(token_usage:,
|
|
21
27
|
usage_source:,
|
|
22
28
|
token_cost:,
|
|
@@ -25,28 +31,26 @@ module LlmCostTracker
|
|
|
25
31
|
token_pricing_partial: false)
|
|
26
32
|
return UNKNOWN if usage_source == Usage::Source::UNKNOWN
|
|
27
33
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
service_priced = false
|
|
31
|
-
service_unpriced = false
|
|
32
|
-
service_line_items.each do |line_item|
|
|
33
|
-
next unless line_item.billable?
|
|
34
|
+
charges = billable_charges(token_usage, token_cost, token_pricing_partial, service_line_items)
|
|
35
|
+
return for_cost(total_cost) if charges.none?(&:unpriced?)
|
|
34
36
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
charges.any?(&:priced?) ? PARTIAL : UNKNOWN
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.for_cost(cost)
|
|
41
|
+
return UNKNOWN if cost.nil?
|
|
40
42
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
return UNKNOWN if unpriced && !priced
|
|
44
|
-
return PARTIAL if unpriced
|
|
43
|
+
cost.zero? ? FREE : COMPLETE
|
|
44
|
+
end
|
|
45
45
|
|
|
46
|
-
|
|
46
|
+
def self.billable_charges(token_usage, token_cost, token_pricing_partial, service_line_items)
|
|
47
|
+
charges = service_line_items.select(&:billable?)
|
|
48
|
+
return charges unless token_usage.total_tokens.to_i.positive? ||
|
|
49
|
+
token_usage.priced_quantities.any? { |_key, quantity| quantity.positive? }
|
|
47
50
|
|
|
48
|
-
|
|
51
|
+
charges + [TokenCharge.new(cost: token_cost, partial: token_pricing_partial)]
|
|
49
52
|
end
|
|
53
|
+
private_class_method :billable_charges
|
|
50
54
|
end
|
|
51
55
|
end
|
|
52
56
|
end
|
|
@@ -30,29 +30,17 @@ module LlmCostTracker
|
|
|
30
30
|
)
|
|
31
31
|
|
|
32
32
|
class LineItem
|
|
33
|
+
DIMENSION_FIELDS = %i[kind direction modality cache_state unit].freeze
|
|
34
|
+
DIMENSION_DEFAULTS = { cache_state: "none" }.freeze
|
|
35
|
+
AMOUNT_DEFAULTS = {
|
|
36
|
+
quantity: BigDecimal("0"), rate_amount: nil, rate_quantity: BigDecimal("1"), cost: nil
|
|
37
|
+
}.freeze
|
|
38
|
+
SOURCE_FIELDS = %i[pricing_basis price_key price_source].freeze
|
|
39
|
+
private_constant :DIMENSION_FIELDS, :DIMENSION_DEFAULTS, :AMOUNT_DEFAULTS, :SOURCE_FIELDS
|
|
40
|
+
|
|
33
41
|
def self.build(attributes)
|
|
34
42
|
attributes = attributes.to_h
|
|
35
|
-
|
|
36
|
-
new(
|
|
37
|
-
kind: attributes[:kind]&.to_s || dimension&.kind,
|
|
38
|
-
direction: attributes[:direction]&.to_s || dimension&.direction,
|
|
39
|
-
modality: attributes[:modality]&.to_s || dimension&.modality,
|
|
40
|
-
cache_state: attributes[:cache_state]&.to_s || dimension&.cache_state || "none",
|
|
41
|
-
quantity: decimal_or_nil(attributes[:quantity]) || BigDecimal("0"),
|
|
42
|
-
unit: attributes[:unit]&.to_s || dimension&.unit,
|
|
43
|
-
rate_amount: decimal_or_nil(attributes[:rate_amount]),
|
|
44
|
-
rate_quantity: decimal_or_nil(attributes[:rate_quantity]) || BigDecimal("1"),
|
|
45
|
-
cost: decimal_or_nil(attributes[:cost]),
|
|
46
|
-
currency: canonical_currency(attributes[:currency]),
|
|
47
|
-
cost_status: cost_status_for(attributes),
|
|
48
|
-
pricing_basis: attributes[:pricing_basis]&.to_s,
|
|
49
|
-
price_key: attributes[:price_key]&.to_s,
|
|
50
|
-
price_source: attributes[:price_source]&.to_s,
|
|
51
|
-
price_source_version: attributes[:price_source_version],
|
|
52
|
-
provider_field: attributes[:provider_field],
|
|
53
|
-
provider_item_id: attributes[:provider_item_id],
|
|
54
|
-
details: attributes[:details] || {}
|
|
55
|
-
)
|
|
43
|
+
new(**dimension_fields(attributes), **amount_fields(attributes), **source_fields(attributes))
|
|
56
44
|
end
|
|
57
45
|
|
|
58
46
|
def self.from_token_usage(token_usage)
|
|
@@ -77,14 +65,28 @@ module LlmCostTracker
|
|
|
77
65
|
end
|
|
78
66
|
end
|
|
79
67
|
|
|
80
|
-
def self.
|
|
81
|
-
|
|
82
|
-
|
|
68
|
+
def self.dimension_fields(attributes)
|
|
69
|
+
dimension = dimension_for(attributes).to_h
|
|
70
|
+
DIMENSION_FIELDS.to_h do |field|
|
|
71
|
+
[field, attributes[field]&.to_s || dimension[field] || DIMENSION_DEFAULTS[field]]
|
|
72
|
+
end
|
|
73
|
+
end
|
|
83
74
|
|
|
84
|
-
|
|
85
|
-
|
|
75
|
+
def self.amount_fields(attributes)
|
|
76
|
+
amounts = AMOUNT_DEFAULTS.to_h { |field, default| [field, decimal_or_nil(attributes[field]) || default] }
|
|
77
|
+
amounts.merge(
|
|
78
|
+
currency: (attributes[:currency] || LlmCostTracker::DEFAULT_CURRENCY).to_s.upcase,
|
|
79
|
+
cost_status: (attributes[:cost_status] || CostStatus.for_cost(amounts[:cost])).to_s
|
|
80
|
+
)
|
|
81
|
+
end
|
|
86
82
|
|
|
87
|
-
|
|
83
|
+
def self.source_fields(attributes)
|
|
84
|
+
SOURCE_FIELDS.to_h { |field| [field, attributes[field]&.to_s] }.merge(
|
|
85
|
+
price_source_version: attributes[:price_source_version],
|
|
86
|
+
provider_field: attributes[:provider_field],
|
|
87
|
+
provider_item_id: attributes[:provider_item_id],
|
|
88
|
+
details: attributes[:details] || {}
|
|
89
|
+
)
|
|
88
90
|
end
|
|
89
91
|
|
|
90
92
|
def self.dimension_for(attributes)
|
|
@@ -100,11 +102,7 @@ module LlmCostTracker
|
|
|
100
102
|
BigDecimal(value.to_s)
|
|
101
103
|
end
|
|
102
104
|
|
|
103
|
-
|
|
104
|
-
(value || LlmCostTracker::DEFAULT_CURRENCY).to_s.upcase
|
|
105
|
-
end
|
|
106
|
-
|
|
107
|
-
private_class_method :cost_status_for, :dimension_for, :decimal_or_nil, :canonical_currency
|
|
105
|
+
private_class_method :dimension_fields, :amount_fields, :source_fields, :dimension_for, :decimal_or_nil
|
|
108
106
|
|
|
109
107
|
def billable?
|
|
110
108
|
quantity.positive?
|
|
@@ -139,7 +137,7 @@ module LlmCostTracker
|
|
|
139
137
|
rate_quantity: rate.quantity,
|
|
140
138
|
cost: applied_cost,
|
|
141
139
|
currency: rate.currency.upcase,
|
|
142
|
-
cost_status: applied_cost
|
|
140
|
+
cost_status: CostStatus.for_cost(applied_cost),
|
|
143
141
|
price_key: rate.source_key,
|
|
144
142
|
price_source: rate.source,
|
|
145
143
|
price_source_version: rate.source_version
|
|
@@ -9,14 +9,28 @@ module LlmCostTracker
|
|
|
9
9
|
class Budgets < Section
|
|
10
10
|
EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
|
|
11
11
|
TOTALS_SOURCES = %i[ledger cache].freeze
|
|
12
|
-
PER_TAG_WINDOWS = %i[daily weekly monthly].freeze
|
|
12
|
+
PER_TAG_WINDOWS = %i[daily weekly monthly total calls].freeze
|
|
13
13
|
PER_TAG_OPTIONS = %i[behavior on_exceeded].freeze
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
LIMITS = %i[monthly daily per_call].freeze
|
|
16
|
+
|
|
16
17
|
enum_attribute :exceeded_behavior, allowed: EXCEEDED_BEHAVIORS, default: :notify
|
|
17
18
|
enum_attribute :totals_source, allowed: TOTALS_SOURCES, default: :ledger
|
|
18
19
|
|
|
19
|
-
attr_reader :per_tag
|
|
20
|
+
attr_reader(*LIMITS, :on_exceeded, :per_tag)
|
|
21
|
+
|
|
22
|
+
LIMITS.each do |name|
|
|
23
|
+
define_method(:"#{name}=") do |value|
|
|
24
|
+
ensure_mutable!
|
|
25
|
+
amount = validated_amount("budgets.#{name}", value, minimum: 0) unless value.nil?
|
|
26
|
+
instance_variable_set(:"@#{name}", value.is_a?(Numeric) ? value : amount)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def on_exceeded=(value)
|
|
31
|
+
ensure_mutable!
|
|
32
|
+
@on_exceeded = validated_callback("budgets.on_exceeded", value)
|
|
33
|
+
end
|
|
20
34
|
|
|
21
35
|
def initialize(owner)
|
|
22
36
|
super
|
|
@@ -48,19 +62,32 @@ module LlmCostTracker
|
|
|
48
62
|
raise Error, "budgets.per_tag[#{key.inspect}] must be a hash" unless entry.is_a?(Hash)
|
|
49
63
|
|
|
50
64
|
normalized = entry.to_h.transform_keys(&:to_sym)
|
|
51
|
-
|
|
65
|
+
{
|
|
66
|
+
windows: validated_windows(key, normalized.except(*PER_TAG_OPTIONS)),
|
|
67
|
+
behavior: validated_behavior(key, normalized[:behavior]),
|
|
68
|
+
on_exceeded: validated_callback("budgets.per_tag[#{key.inspect}][:on_exceeded]", normalized[:on_exceeded])
|
|
69
|
+
}
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def validated_amount(name, value, minimum: nil)
|
|
73
|
+
number = Float(value, exception: false)
|
|
74
|
+
return BigDecimal(value.to_s) if number && (minimum ? number >= minimum : number.positive?)
|
|
75
|
+
|
|
76
|
+
raise Error, "#{name} must be a #{minimum ? 'non-negative' : 'positive'} number, got #{value.inspect}"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def validated_callback(name, value)
|
|
80
|
+
return value if value.nil? || value.respond_to?(:call)
|
|
81
|
+
|
|
82
|
+
raise Error, "#{name} must respond to call, got #{value.inspect}"
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def validated_windows(key, windows)
|
|
52
86
|
if windows.empty?
|
|
53
|
-
raise Error,
|
|
54
|
-
"budgets.per_tag[#{key.inspect}] needs at least one of: #{PER_TAG_WINDOWS.join(', ')}"
|
|
87
|
+
raise Error, "budgets.per_tag[#{key.inspect}] needs at least one of: #{PER_TAG_WINDOWS.join(', ')}"
|
|
55
88
|
end
|
|
56
89
|
|
|
57
|
-
{
|
|
58
|
-
windows: windows.to_h do |window, limit|
|
|
59
|
-
[validated_window(key, window), validated_limit(key, window, limit)]
|
|
60
|
-
end,
|
|
61
|
-
behavior: validated_behavior(key, options[:behavior]),
|
|
62
|
-
on_exceeded: options[:on_exceeded]
|
|
63
|
-
}
|
|
90
|
+
windows.to_h { |window, limit| [validated_window(key, window), validated_limit(key, window, limit)] }
|
|
64
91
|
end
|
|
65
92
|
|
|
66
93
|
def validated_behavior(key, behavior)
|
|
@@ -81,12 +108,16 @@ module LlmCostTracker
|
|
|
81
108
|
end
|
|
82
109
|
|
|
83
110
|
def validated_limit(key, window, limit)
|
|
84
|
-
|
|
85
|
-
return BigDecimal(limit.to_s) if numeric&.positive?
|
|
111
|
+
return validated_calls(key, limit) if window == :calls
|
|
86
112
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
113
|
+
validated_amount("budgets.per_tag[#{key.inspect}][#{window.inspect}]", limit)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def validated_calls(key, limit)
|
|
117
|
+
count = Integer(limit.to_s, exception: false)
|
|
118
|
+
return count if count&.positive?
|
|
119
|
+
|
|
120
|
+
raise Error, "budgets.per_tag[#{key.inspect}][:calls] must be a positive integer, got #{limit.inspect}"
|
|
90
121
|
end
|
|
91
122
|
end
|
|
92
123
|
end
|
|
@@ -8,7 +8,13 @@ module LlmCostTracker
|
|
|
8
8
|
OPENAI_COMPATIBLE_PROVIDERS = {
|
|
9
9
|
"openrouter.ai" => "openrouter",
|
|
10
10
|
"api.deepseek.com" => "deepseek",
|
|
11
|
-
"api.groq.com" => "groq"
|
|
11
|
+
"api.groq.com" => "groq",
|
|
12
|
+
"api.x.ai" => "xai",
|
|
13
|
+
"us.api.x.ai" => "xai",
|
|
14
|
+
"api.mistral.ai" => "mistral",
|
|
15
|
+
"api.eu.mistral.ai" => "mistral",
|
|
16
|
+
"api.us.mistral.ai" => "mistral",
|
|
17
|
+
"api.perplexity.ai" => "perplexity"
|
|
12
18
|
}.freeze
|
|
13
19
|
|
|
14
20
|
attributes :request_stream_usage
|
|
@@ -18,12 +18,12 @@ module LlmCostTracker
|
|
|
18
18
|
deep_freeze(key)
|
|
19
19
|
deep_freeze(nested_value)
|
|
20
20
|
end
|
|
21
|
-
value.
|
|
21
|
+
value.freeze
|
|
22
22
|
when Array, Set
|
|
23
23
|
value.each { |nested_value| deep_freeze(nested_value) }
|
|
24
|
-
value.
|
|
24
|
+
value.freeze
|
|
25
25
|
when String
|
|
26
|
-
value.
|
|
26
|
+
value.freeze
|
|
27
27
|
else
|
|
28
28
|
value
|
|
29
29
|
end
|
|
@@ -24,7 +24,17 @@ module LlmCostTracker
|
|
|
24
24
|
private
|
|
25
25
|
|
|
26
26
|
def async_ok
|
|
27
|
-
|
|
27
|
+
quarantined = LlmCostTracker::Ingestion::InboxEntry.quarantined.count
|
|
28
|
+
if quarantined.zero?
|
|
29
|
+
return Check.new(:ok, "async ingestion", "inbox and ingestion lease tables are on the current schema")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
Check.new(
|
|
33
|
+
:warn,
|
|
34
|
+
"async ingestion",
|
|
35
|
+
"#{quarantined} inbox row(s) are quarantined, so their spend is not in the ledger; " \
|
|
36
|
+
"requeue them as docs/operations.md shows once the cause is fixed"
|
|
37
|
+
)
|
|
28
38
|
end
|
|
29
39
|
|
|
30
40
|
def inline_check
|
|
@@ -38,11 +48,19 @@ module LlmCostTracker
|
|
|
38
48
|
Check.new(
|
|
39
49
|
:warn,
|
|
40
50
|
"inline ingestion",
|
|
41
|
-
"config.ingestion.mode = :inline but found unused async ingestion tables: #{leftovers.join(', ')}
|
|
51
|
+
"config.ingestion.mode = :inline but found unused async ingestion tables: #{leftovers.join(', ')}" \
|
|
52
|
+
"#{undrained_note(leftovers)}. " \
|
|
42
53
|
"Set config.ingestion.mode = :async to keep the inbox path or drop the tables."
|
|
43
54
|
)
|
|
44
55
|
end
|
|
45
56
|
|
|
57
|
+
def undrained_note(leftovers)
|
|
58
|
+
return unless leftovers.include?(LlmCostTracker::Ingestion::InboxEntry.table_name)
|
|
59
|
+
|
|
60
|
+
rows = LlmCostTracker::Ingestion::InboxEntry.count
|
|
61
|
+
"; #{rows} inbox row(s) never reached the ledger, so drain them in :async mode first" if rows.positive?
|
|
62
|
+
end
|
|
63
|
+
|
|
46
64
|
def inline_leftover_tables
|
|
47
65
|
async_tables.select { |table| Probe.table_exists?(table) }
|
|
48
66
|
end
|
|
@@ -15,38 +15,9 @@ module LlmCostTracker
|
|
|
15
15
|
return bundled_check unless path
|
|
16
16
|
return Check.new(:error, "prices", "#{path} does not exist; #{REFRESH_COMMAND}") unless File.exist?(path)
|
|
17
17
|
|
|
18
|
-
count =
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
return configured_check(:warn, path, count, "metadata.updated_at missing; #{REFRESH_COMMAND}") unless updated_at
|
|
22
|
-
|
|
23
|
-
file_date = Date.iso8601(updated_at.to_s)
|
|
24
|
-
if (Date.today - file_date).to_i > STALE_AFTER_DAYS
|
|
25
|
-
return configured_check(
|
|
26
|
-
:warn,
|
|
27
|
-
path,
|
|
28
|
-
count,
|
|
29
|
-
"updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days; #{REFRESH_COMMAND}"
|
|
30
|
-
)
|
|
31
|
-
end
|
|
32
|
-
bundled_at = LlmCostTracker::Pricing::Registry.metadata["updated_at"]
|
|
33
|
-
if bundled_at && file_date < Date.iso8601(bundled_at.to_s)
|
|
34
|
-
return configured_check(
|
|
35
|
-
:warn,
|
|
36
|
-
path,
|
|
37
|
-
count,
|
|
38
|
-
"updated_at=#{updated_at} is older than the bundled prices (#{bundled_at}); #{REFRESH_COMMAND}"
|
|
39
|
-
)
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
configured_check(:ok, path, count, "updated_at=#{updated_at}")
|
|
43
|
-
rescue Date::Error
|
|
44
|
-
configured_check(
|
|
45
|
-
:warn,
|
|
46
|
-
path,
|
|
47
|
-
count,
|
|
48
|
-
"metadata.updated_at=#{updated_at.inspect} is invalid; #{REFRESH_COMMAND}"
|
|
49
|
-
)
|
|
18
|
+
count = Pricing::Registry.file_prices(path).size
|
|
19
|
+
status, freshness = freshness(Pricing::Registry.file_metadata(path))
|
|
20
|
+
Check.new(status, "prices", "loaded #{count} models from #{path}; #{freshness}")
|
|
50
21
|
rescue LlmCostTracker::Error => e
|
|
51
22
|
Check.new(:error, "prices", e.message)
|
|
52
23
|
end
|
|
@@ -54,7 +25,7 @@ module LlmCostTracker
|
|
|
54
25
|
private
|
|
55
26
|
|
|
56
27
|
def bundled_check
|
|
57
|
-
updated_at =
|
|
28
|
+
updated_at = Pricing::Registry.metadata.fetch("updated_at", "unknown")
|
|
58
29
|
Check.new(
|
|
59
30
|
:warn,
|
|
60
31
|
"prices",
|
|
@@ -63,8 +34,26 @@ module LlmCostTracker
|
|
|
63
34
|
)
|
|
64
35
|
end
|
|
65
36
|
|
|
66
|
-
def
|
|
67
|
-
|
|
37
|
+
def freshness(metadata)
|
|
38
|
+
updated_at = metadata["updated_at"] || metadata[:updated_at]
|
|
39
|
+
return [:warn, "metadata.updated_at missing; #{REFRESH_COMMAND}"] unless updated_at
|
|
40
|
+
|
|
41
|
+
reason = staleness(updated_at)
|
|
42
|
+
reason ? [:warn, "#{reason}; #{REFRESH_COMMAND}"] : [:ok, "updated_at=#{updated_at}"]
|
|
43
|
+
rescue Date::Error
|
|
44
|
+
[:warn, "metadata.updated_at=#{updated_at.inspect} is invalid; #{REFRESH_COMMAND}"]
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def staleness(updated_at)
|
|
48
|
+
file_date = Date.iso8601(updated_at.to_s)
|
|
49
|
+
if (Date.today - file_date).to_i > STALE_AFTER_DAYS
|
|
50
|
+
return "updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
bundled_at = Pricing::Registry.metadata["updated_at"]
|
|
54
|
+
return unless bundled_at && file_date < Date.iso8601(bundled_at.to_s)
|
|
55
|
+
|
|
56
|
+
"updated_at=#{updated_at} is older than the bundled prices (#{bundled_at})"
|
|
68
57
|
end
|
|
69
58
|
end
|
|
70
59
|
end
|
|
@@ -6,7 +6,7 @@ module LlmCostTracker
|
|
|
6
6
|
class Doctor
|
|
7
7
|
module Probe
|
|
8
8
|
def self.table_exists?(name)
|
|
9
|
-
LlmCostTracker::Call.
|
|
9
|
+
LlmCostTracker::Call.lease_connection.data_source_exists?(name)
|
|
10
10
|
rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError,
|
|
11
11
|
ActiveRecord::ConnectionFailed, ActiveRecord::StatementInvalid
|
|
12
12
|
false
|
|
@@ -37,19 +37,12 @@ module LlmCostTracker
|
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def report(checks = call, color: $stdout.tty?)
|
|
40
|
-
name_width = checks.map { |
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
lines << bold(section, color)
|
|
45
|
-
members.each do |check|
|
|
46
|
-
status = paint_status("[#{STATUS_GLYPHS.fetch(check.status, check.status)}]", check.status, color)
|
|
47
|
-
lines << " #{status} #{"#{check.name}:".ljust(name_width + 1)} #{check.message}"
|
|
48
|
-
end
|
|
49
|
-
lines << ""
|
|
40
|
+
name_width = checks.map { |check| check.name.length }.max.to_i
|
|
41
|
+
sections = checks.group_by { |check| SECTION_FOR_CHECK.fetch(check.name, "Setup") }.slice(*SECTIONS)
|
|
42
|
+
blocks = sections.map do |section, members|
|
|
43
|
+
[bold(section, color), *members.map { |check| check_line(check, name_width, color) }].join("\n")
|
|
50
44
|
end
|
|
51
|
-
|
|
52
|
-
lines.join("\n")
|
|
45
|
+
[bold("LLM Cost Tracker doctor", color), *blocks].join("\n\n")
|
|
53
46
|
end
|
|
54
47
|
|
|
55
48
|
def healthy?(checks = call)
|
|
@@ -58,13 +51,9 @@ module LlmCostTracker
|
|
|
58
51
|
|
|
59
52
|
private
|
|
60
53
|
|
|
61
|
-
def
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
next if members.empty?
|
|
65
|
-
|
|
66
|
-
yield section, members
|
|
67
|
-
end
|
|
54
|
+
def check_line(check, name_width, color)
|
|
55
|
+
status = paint_status("[#{STATUS_GLYPHS.fetch(check.status, check.status)}]", check.status, color)
|
|
56
|
+
" #{status} #{"#{check.name}:".ljust(name_width + 1)} #{check.message}"
|
|
68
57
|
end
|
|
69
58
|
|
|
70
59
|
def paint_status(text, status, color)
|
|
@@ -89,6 +78,7 @@ module LlmCostTracker
|
|
|
89
78
|
table_check,
|
|
90
79
|
column_check,
|
|
91
80
|
*dependent_core_schema_checks,
|
|
81
|
+
per_tag_budget_check,
|
|
92
82
|
call_rollups_check,
|
|
93
83
|
IngestionCheck.new.call,
|
|
94
84
|
PriceCheck.new.call,
|
|
@@ -106,6 +96,19 @@ module LlmCostTracker
|
|
|
106
96
|
end
|
|
107
97
|
end
|
|
108
98
|
|
|
99
|
+
def per_tag_budget_check
|
|
100
|
+
return if LlmCostTracker.configuration.budgets.per_tag.empty?
|
|
101
|
+
return unless Probe.table_exists?("llm_cost_tracker_call_tags")
|
|
102
|
+
return if Budget::PerTag.columns?
|
|
103
|
+
|
|
104
|
+
Check.new(
|
|
105
|
+
:error,
|
|
106
|
+
"per-tag budgets",
|
|
107
|
+
"budgets.per_tag is set but llm_cost_tracker_call_tags has no cost columns, so no per-tag budget is " \
|
|
108
|
+
"enforced; run bin/rails generate llm_cost_tracker:upgrade_per_tag_budgets and migrate"
|
|
109
|
+
)
|
|
110
|
+
end
|
|
111
|
+
|
|
109
112
|
def configuration_check
|
|
110
113
|
config = LlmCostTracker.configuration
|
|
111
114
|
Check.new(:ok, "configuration", "active_record ledger enabled=#{config.enabled}")
|
|
@@ -224,7 +227,7 @@ module LlmCostTracker
|
|
|
224
227
|
end
|
|
225
228
|
|
|
226
229
|
def active_record_available?
|
|
227
|
-
LlmCostTracker::Call.
|
|
230
|
+
LlmCostTracker::Call.lease_connection
|
|
228
231
|
true
|
|
229
232
|
rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
|
|
230
233
|
false
|
|
@@ -8,6 +8,14 @@ module LlmCostTracker
|
|
|
8
8
|
class Engine < ::Rails::Engine
|
|
9
9
|
isolate_namespace LlmCostTracker
|
|
10
10
|
|
|
11
|
+
initializer "llm_cost_tracker.host_inflections" do |app|
|
|
12
|
+
Rails.autoloaders.each { |autoloader| autoloader.inflector.inflect("llm_cost_tracker" => "LlmCostTracker") }
|
|
13
|
+
app.reloader.to_prepare do
|
|
14
|
+
host_name = "llm_cost_tracker".camelize
|
|
15
|
+
Object.const_set(host_name, LlmCostTracker) unless Object.const_defined?(host_name)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
11
19
|
initializer "llm_cost_tracker.deprecator" do |app|
|
|
12
20
|
app.deprecators[:llm_cost_tracker] = LlmCostTracker.deprecator
|
|
13
21
|
end
|
|
@@ -27,10 +27,8 @@ module LlmCostTracker
|
|
|
27
27
|
@stage = stage
|
|
28
28
|
@scope = scope
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
"$#{format('%.6f', @total)} / $#{format('%.6f', budget)}"
|
|
33
|
-
)
|
|
30
|
+
amounts = budget_type == :calls ? "#{total} / #{budget}" : format("$%<total>.6f / $%<budget>.6f", total:, budget:)
|
|
31
|
+
super("LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded#{scope_suffix}: #{amounts}")
|
|
34
32
|
end
|
|
35
33
|
|
|
36
34
|
private
|