llm_cost_tracker 0.14.0 → 0.14.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +128 -0
- data/README.md +16 -5
- data/app/assets/llm_cost_tracker/application.css +16 -0
- data/app/controllers/llm_cost_tracker/application_controller.rb +16 -3
- data/app/controllers/llm_cost_tracker/calls_controller.rb +1 -3
- data/app/controllers/llm_cost_tracker/models_controller.rb +2 -4
- data/app/controllers/llm_cost_tracker/pricing_controller.rb +2 -2
- data/app/controllers/llm_cost_tracker/tags_controller.rb +9 -7
- data/app/helpers/llm_cost_tracker/application_helper.rb +5 -26
- data/app/helpers/llm_cost_tracker/chart_helper.rb +6 -10
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +5 -0
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +8 -28
- data/app/models/llm_cost_tracker/call.rb +10 -14
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +4 -15
- data/app/services/llm_cost_tracker/dashboard/filter.rb +29 -30
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +2 -5
- data/app/services/llm_cost_tracker/dashboard/params.rb +10 -0
- data/app/services/llm_cost_tracker/dashboard/pricing_overview.rb +2 -1
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +2 -3
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +1 -0
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +3 -5
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +1 -2
- data/app/views/llm_cost_tracker/calls/show.html.erb +8 -22
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +1 -1
- data/app/views/llm_cost_tracker/shared/_bar.html.erb +1 -3
- data/app/views/llm_cost_tracker/shared/_filter_pill_date.html.erb +1 -4
- data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb +1 -4
- data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb +1 -4
- data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +1 -4
- data/app/views/llm_cost_tracker/tags/show.html.erb +8 -5
- data/config/routes.rb +6 -1
- data/lib/llm_cost_tracker/budget/per_tag.rb +21 -11
- data/lib/llm_cost_tracker/budget.rb +44 -55
- data/lib/llm_cost_tracker/capture/event_window.rb +100 -0
- data/lib/llm_cost_tracker/capture/sdk_payload.rb +5 -1
- data/lib/llm_cost_tracker/capture/sse.rb +108 -32
- data/lib/llm_cost_tracker/capture/stream_collector.rb +41 -81
- data/lib/llm_cost_tracker/capture/stream_tap.rb +57 -0
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +14 -58
- data/lib/llm_cost_tracker/capture_verifier.rb +1 -7
- data/lib/llm_cost_tracker/charges/line_item.rb +5 -1
- data/lib/llm_cost_tracker/configuration/budgets.rb +1 -1
- data/lib/llm_cost_tracker/configuration/pricing.rb +1 -1
- data/lib/llm_cost_tracker/configuration.rb +5 -9
- data/lib/llm_cost_tracker/doctor/price_check.rb +14 -3
- data/lib/llm_cost_tracker/doctor/schema_check.rb +1 -2
- data/lib/llm_cost_tracker/doctor.rb +19 -2
- data/lib/llm_cost_tracker/engine.rb +0 -1
- data/lib/llm_cost_tracker/errors.rb +13 -0
- data/lib/llm_cost_tracker/event.rb +8 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +5 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_call_rollups.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +14 -12
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb +0 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_indexes_generator.rb +3 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_per_tag_budgets_generator.rb +3 -8
- data/lib/llm_cost_tracker/ingestion/batch.rb +37 -8
- data/lib/llm_cost_tracker/ingestion/inbox.rb +2 -8
- data/lib/llm_cost_tracker/ingestion.rb +3 -1
- data/lib/llm_cost_tracker/integrations/anthropic.rb +70 -11
- data/lib/llm_cost_tracker/integrations/base.rb +56 -19
- data/lib/llm_cost_tracker/integrations/openai/batch_capture.rb +21 -13
- data/lib/llm_cost_tracker/integrations/openai/patches.rb +8 -0
- data/lib/llm_cost_tracker/integrations/openai.rb +42 -59
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +275 -75
- data/lib/llm_cost_tracker/integrations.rb +1 -20
- data/lib/llm_cost_tracker/ledger/isolation.rb +23 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +16 -11
- data/lib/llm_cost_tracker/ledger/rollups.rb +25 -17
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +14 -3
- data/lib/llm_cost_tracker/ledger/schema/base.rb +5 -4
- data/lib/llm_cost_tracker/ledger/storable.rb +16 -0
- data/lib/llm_cost_tracker/ledger/store.rb +14 -13
- data/lib/llm_cost_tracker/ledger/tags/breakdown.rb +1 -5
- data/lib/llm_cost_tracker/ledger/tags/encoding.rb +3 -12
- data/lib/llm_cost_tracker/ledger/tags/query.rb +0 -2
- data/lib/llm_cost_tracker/ledger.rb +1 -0
- data/lib/llm_cost_tracker/logging.rb +3 -1
- data/lib/llm_cost_tracker/middleware/faraday.rb +67 -61
- data/lib/llm_cost_tracker/parsers.rb +12 -41
- data/lib/llm_cost_tracker/prices.json +3071 -249
- data/lib/llm_cost_tracker/pricing/backfill.rb +46 -13
- data/lib/llm_cost_tracker/pricing/calculation.rb +118 -75
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +11 -10
- data/lib/llm_cost_tracker/pricing/estimator.rb +5 -2
- data/lib/llm_cost_tracker/pricing/matcher.rb +36 -22
- data/lib/llm_cost_tracker/pricing/mode.rb +2 -13
- data/lib/llm_cost_tracker/pricing/price_key.rb +5 -3
- data/lib/llm_cost_tracker/pricing/rate.rb +1 -0
- data/lib/llm_cost_tracker/pricing/registry.rb +17 -17
- data/lib/llm_cost_tracker/pricing/service_rates.rb +1 -8
- data/lib/llm_cost_tracker/pricing/sync/change_printer.rb +6 -1
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +13 -9
- data/lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb +47 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +38 -70
- data/lib/llm_cost_tracker/providers/anthropic/parser.rb +33 -3
- data/lib/llm_cost_tracker/providers/anthropic/response_parser.rb +11 -5
- data/lib/llm_cost_tracker/providers/anthropic/usage_extractor.rb +81 -12
- data/lib/llm_cost_tracker/providers/azure/parser.rb +22 -7
- data/lib/llm_cost_tracker/providers/gemini/parser.rb +175 -47
- data/lib/llm_cost_tracker/providers/gemini/usage_extractor.rb +13 -22
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +1 -1
- data/lib/llm_cost_tracker/providers/openai/parser.rb +11 -13
- data/lib/llm_cost_tracker/providers/openai/response_parser.rb +101 -27
- data/lib/llm_cost_tracker/providers/openai/service_charges.rb +57 -40
- data/lib/llm_cost_tracker/providers/openai/usage_extractor.rb +24 -6
- data/lib/llm_cost_tracker/providers/openai_compatible/parser.rb +7 -1
- data/lib/llm_cost_tracker/redaction.rb +32 -0
- data/lib/llm_cost_tracker/report/data.rb +1 -1
- data/lib/llm_cost_tracker/retention.rb +1 -3
- data/lib/llm_cost_tracker/tags/sanitizer.rb +10 -36
- data/lib/llm_cost_tracker/tracker.rb +17 -10
- data/lib/llm_cost_tracker/usage/catalog.rb +13 -4
- data/lib/llm_cost_tracker/usage/dimension.rb +1 -1
- data/lib/llm_cost_tracker/usage/dimensions.yml +61 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +4 -2
- data/lib/tasks/llm_cost_tracker.rake +26 -21
- data/llm_cost_tracker.gemspec +63 -0
- metadata +23 -10
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "event_window"
|
|
4
|
+
require_relative "sse"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Capture
|
|
8
|
+
class StreamTap
|
|
9
|
+
MAX_PENDING_BYTES = 16 * 1_048_576
|
|
10
|
+
|
|
11
|
+
def initialize(notable: nil)
|
|
12
|
+
@window = EventWindow.new(notable: notable)
|
|
13
|
+
@reader = SSE::Reader.new { |event| @window.push(event[:data], type: event[:event]) }
|
|
14
|
+
@received = false
|
|
15
|
+
@failed = false
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def <<(chunk)
|
|
19
|
+
return if @failed
|
|
20
|
+
|
|
21
|
+
@received = true
|
|
22
|
+
@reader << chunk
|
|
23
|
+
fail! if @reader.pending_bytesize > MAX_PENDING_BYTES
|
|
24
|
+
rescue StandardError
|
|
25
|
+
fail!
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def received?
|
|
29
|
+
@received
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def failed?
|
|
33
|
+
@failed
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def overflowed?
|
|
37
|
+
@window.overflowed?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def events
|
|
41
|
+
@reader&.finish
|
|
42
|
+
@failed ? [] : @window.events
|
|
43
|
+
rescue StandardError
|
|
44
|
+
fail!
|
|
45
|
+
[]
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def fail!
|
|
51
|
+
@failed = true
|
|
52
|
+
@reader = nil
|
|
53
|
+
@window = EventWindow.new
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_support/core_ext/object/deep_dup"
|
|
4
3
|
require "active_support/core_ext/object/try"
|
|
5
4
|
|
|
6
|
-
require_relative "sdk_payload"
|
|
7
|
-
|
|
8
5
|
module LlmCostTracker
|
|
9
6
|
module Capture
|
|
10
7
|
class StreamTracker
|
|
@@ -13,8 +10,7 @@ module LlmCostTracker
|
|
|
13
10
|
@collector = collector
|
|
14
11
|
@active = active
|
|
15
12
|
@finish = finish || proc { |errored| collector.finish!(errored: errored) }
|
|
16
|
-
@
|
|
17
|
-
@attempted_ref = [false]
|
|
13
|
+
@finished = false
|
|
18
14
|
@capture_failed = false
|
|
19
15
|
@mutex = Mutex.new
|
|
20
16
|
end
|
|
@@ -22,30 +18,21 @@ module LlmCostTracker
|
|
|
22
18
|
def wrap
|
|
23
19
|
return @stream unless @stream
|
|
24
20
|
|
|
25
|
-
|
|
26
|
-
if
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
end)
|
|
33
|
-
iterator_wrapped = true
|
|
34
|
-
end
|
|
35
|
-
end
|
|
36
|
-
each_wrapped = false
|
|
37
|
-
if !iterator_wrapped && @stream.respond_to?(:each)
|
|
21
|
+
iterator = @stream.instance_variable_get(:@iterator)
|
|
22
|
+
if iterator.respond_to?(:each)
|
|
23
|
+
@stream.instance_variable_set(:@iterator,
|
|
24
|
+
Enumerator.new do |yielder|
|
|
25
|
+
each_from(iterator) { |event| yielder << event }
|
|
26
|
+
end)
|
|
27
|
+
elsif @stream.respond_to?(:each)
|
|
38
28
|
wrap_each
|
|
39
|
-
|
|
40
|
-
end
|
|
41
|
-
unless iterator_wrapped || each_wrapped
|
|
29
|
+
else
|
|
42
30
|
Logging.warn(
|
|
43
31
|
"stream integration found no wrappable iterator on #{@stream.class} " \
|
|
44
32
|
"(missing both `@iterator` ivar and `#each`); usage will not be captured"
|
|
45
33
|
)
|
|
46
34
|
end
|
|
47
35
|
|
|
48
|
-
register_orphan_finalizer
|
|
49
36
|
@stream
|
|
50
37
|
rescue StandardError => e
|
|
51
38
|
Logging.warn("stream integration failed to install wrapper: #{e.class}: #{e.message}")
|
|
@@ -77,7 +64,7 @@ module LlmCostTracker
|
|
|
77
64
|
yield event
|
|
78
65
|
end
|
|
79
66
|
end
|
|
80
|
-
rescue
|
|
67
|
+
rescue Exception # rubocop:disable Lint/RescueException -- mark Interrupt/shutdown too, so it is not masked
|
|
81
68
|
errored = true
|
|
82
69
|
raise
|
|
83
70
|
ensure
|
|
@@ -85,10 +72,7 @@ module LlmCostTracker
|
|
|
85
72
|
end
|
|
86
73
|
|
|
87
74
|
def capture(event)
|
|
88
|
-
|
|
89
|
-
payload = SdkPayload.normalize(raw_payload)
|
|
90
|
-
type = event.try(:type) || payload["type"]
|
|
91
|
-
@collector.event(payload, type: type&.to_s)
|
|
75
|
+
@collector.event(event.try(:to_h) || {}, type: event.try(:type)&.to_s)
|
|
92
76
|
rescue StandardError => e
|
|
93
77
|
warn_capture_failure(e)
|
|
94
78
|
end
|
|
@@ -107,10 +91,9 @@ module LlmCostTracker
|
|
|
107
91
|
|
|
108
92
|
def finish!(errored:)
|
|
109
93
|
should_finish = @mutex.synchronize do
|
|
110
|
-
|
|
111
|
-
next false if @finished_ref[0]
|
|
94
|
+
next false if @finished
|
|
112
95
|
|
|
113
|
-
@
|
|
96
|
+
@finished = true
|
|
114
97
|
true
|
|
115
98
|
end
|
|
116
99
|
return unless should_finish && @active.call
|
|
@@ -118,37 +101,10 @@ module LlmCostTracker
|
|
|
118
101
|
begin
|
|
119
102
|
@finish.call(errored)
|
|
120
103
|
rescue StandardError
|
|
121
|
-
@mutex.synchronize { @
|
|
104
|
+
@mutex.synchronize { @finished = false }
|
|
122
105
|
raise
|
|
123
106
|
end
|
|
124
107
|
end
|
|
125
|
-
|
|
126
|
-
def register_orphan_finalizer
|
|
127
|
-
finished_ref = @finished_ref
|
|
128
|
-
attempted_ref = @attempted_ref
|
|
129
|
-
finish_proc = @finish
|
|
130
|
-
active_proc = @active
|
|
131
|
-
mutex = @mutex
|
|
132
|
-
finalizer = lambda do |_object_id|
|
|
133
|
-
should_finish = mutex.synchronize do
|
|
134
|
-
next false if finished_ref[0] || attempted_ref[0]
|
|
135
|
-
|
|
136
|
-
finished_ref[0] = true
|
|
137
|
-
attempted_ref[0] = true
|
|
138
|
-
true
|
|
139
|
-
end
|
|
140
|
-
next unless should_finish && active_proc.call
|
|
141
|
-
|
|
142
|
-
begin
|
|
143
|
-
Rails.application.executor.wrap { finish_proc.call(false) }
|
|
144
|
-
rescue StandardError
|
|
145
|
-
nil
|
|
146
|
-
end
|
|
147
|
-
end
|
|
148
|
-
ObjectSpace.define_finalizer(@stream, finalizer)
|
|
149
|
-
rescue TypeError, ArgumentError
|
|
150
|
-
nil
|
|
151
|
-
end
|
|
152
108
|
end
|
|
153
109
|
end
|
|
154
110
|
end
|
|
@@ -25,7 +25,7 @@ module LlmCostTracker
|
|
|
25
25
|
[
|
|
26
26
|
enabled_check,
|
|
27
27
|
*integration_checks,
|
|
28
|
-
*
|
|
28
|
+
*LlmCostTracker::Ingestion.verify
|
|
29
29
|
].compact
|
|
30
30
|
end
|
|
31
31
|
|
|
@@ -49,11 +49,5 @@ module LlmCostTracker
|
|
|
49
49
|
check.with(name: "sdk integration #{check.name}")
|
|
50
50
|
end
|
|
51
51
|
end
|
|
52
|
-
|
|
53
|
-
def storage_checks
|
|
54
|
-
LlmCostTracker::Ingestion.verify
|
|
55
|
-
rescue LlmCostTracker::Error => e
|
|
56
|
-
[Check.new(:error, "storage", e.message)]
|
|
57
|
-
end
|
|
58
52
|
end
|
|
59
53
|
end
|
|
@@ -58,7 +58,11 @@ module LlmCostTracker
|
|
|
58
58
|
def self.from_token_usage(token_usage)
|
|
59
59
|
return [] unless token_usage
|
|
60
60
|
|
|
61
|
-
token_usage.priced_quantities
|
|
61
|
+
from_quantities(token_usage.priced_quantities)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def self.from_quantities(quantities)
|
|
65
|
+
quantities.filter_map do |key, quantity|
|
|
62
66
|
next unless quantity.positive?
|
|
63
67
|
|
|
64
68
|
dimension = Usage::Catalog.fetch(key)
|
|
@@ -18,8 +18,6 @@ module LlmCostTracker
|
|
|
18
18
|
budgets: Budgets, capture: Capture, ingestion: Ingestion, pricing: Pricing, tags: Tags
|
|
19
19
|
}.freeze
|
|
20
20
|
|
|
21
|
-
SCALAR_ATTRIBUTES = %i[enabled].freeze
|
|
22
|
-
|
|
23
21
|
DEPRECATED_OPTIONS = {
|
|
24
22
|
monthly_budget: { to: %i[budgets monthly] },
|
|
25
23
|
daily_budget: { to: %i[budgets daily] },
|
|
@@ -46,7 +44,7 @@ module LlmCostTracker
|
|
|
46
44
|
log_level: { to: nil, note: "LlmCostTracker logs through Rails.logger, which owns the level" }
|
|
47
45
|
}.freeze
|
|
48
46
|
|
|
49
|
-
attr_reader(
|
|
47
|
+
attr_reader(:enabled, *SECTIONS.keys, :instrumented_integrations)
|
|
50
48
|
|
|
51
49
|
def initialize
|
|
52
50
|
SECTIONS.each { |name, klass| instance_variable_set(:"@#{name}", klass.new(self)) }
|
|
@@ -67,11 +65,9 @@ module LlmCostTracker
|
|
|
67
65
|
@instrumented_integrations.include?(name)
|
|
68
66
|
end
|
|
69
67
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
instance_variable_set(:"@#{name}", value)
|
|
74
|
-
end
|
|
68
|
+
def enabled=(value)
|
|
69
|
+
ensure_mutable!
|
|
70
|
+
@enabled = value
|
|
75
71
|
end
|
|
76
72
|
|
|
77
73
|
DEPRECATED_OPTIONS.each do |old_name, spec|
|
|
@@ -101,7 +97,7 @@ module LlmCostTracker
|
|
|
101
97
|
|
|
102
98
|
def finalize!
|
|
103
99
|
SECTIONS.each_key { |name| public_send(name).finalize! }
|
|
104
|
-
@instrumented_integrations = deep_freeze(@instrumented_integrations
|
|
100
|
+
@instrumented_integrations = deep_freeze(@instrumented_integrations)
|
|
105
101
|
@finalized = true
|
|
106
102
|
end
|
|
107
103
|
|
|
@@ -13,14 +13,15 @@ module LlmCostTracker
|
|
|
13
13
|
def call
|
|
14
14
|
path = LlmCostTracker.configuration.pricing.file
|
|
15
15
|
return bundled_check unless path
|
|
16
|
+
return Check.new(:error, "prices", "#{path} does not exist; #{REFRESH_COMMAND}") unless File.exist?(path)
|
|
16
17
|
|
|
17
18
|
count = LlmCostTracker::Pricing::Registry.file_prices(path).size
|
|
18
19
|
metadata = LlmCostTracker::Pricing::Registry.file_metadata(path)
|
|
19
20
|
updated_at = metadata["updated_at"] || metadata[:updated_at]
|
|
20
21
|
return configured_check(:warn, path, count, "metadata.updated_at missing; #{REFRESH_COMMAND}") unless updated_at
|
|
21
22
|
|
|
22
|
-
|
|
23
|
-
if
|
|
23
|
+
file_date = Date.iso8601(updated_at.to_s)
|
|
24
|
+
if (Date.today - file_date).to_i > STALE_AFTER_DAYS
|
|
24
25
|
return configured_check(
|
|
25
26
|
:warn,
|
|
26
27
|
path,
|
|
@@ -28,6 +29,15 @@ module LlmCostTracker
|
|
|
28
29
|
"updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days; #{REFRESH_COMMAND}"
|
|
29
30
|
)
|
|
30
31
|
end
|
|
32
|
+
bundled_at = LlmCostTracker::Pricing::Registry.metadata["updated_at"]
|
|
33
|
+
if bundled_at && file_date < Date.iso8601(bundled_at.to_s)
|
|
34
|
+
return configured_check(
|
|
35
|
+
:warn,
|
|
36
|
+
path,
|
|
37
|
+
count,
|
|
38
|
+
"updated_at=#{updated_at} is older than the bundled prices (#{bundled_at}); #{REFRESH_COMMAND}"
|
|
39
|
+
)
|
|
40
|
+
end
|
|
31
41
|
|
|
32
42
|
configured_check(:ok, path, count, "updated_at=#{updated_at}")
|
|
33
43
|
rescue Date::Error
|
|
@@ -48,7 +58,8 @@ module LlmCostTracker
|
|
|
48
58
|
Check.new(
|
|
49
59
|
:warn,
|
|
50
60
|
"prices",
|
|
51
|
-
"using bundled prices updated_at=#{updated_at};
|
|
61
|
+
"using bundled prices updated_at=#{updated_at}; " \
|
|
62
|
+
"commit a prices_file (config.pricing.file) for production releases"
|
|
52
63
|
)
|
|
53
64
|
end
|
|
54
65
|
|
|
@@ -22,8 +22,7 @@ module LlmCostTracker
|
|
|
22
22
|
Check.new(
|
|
23
23
|
:error,
|
|
24
24
|
@name,
|
|
25
|
-
"current schema required; #{errors.join('; ')}; "
|
|
26
|
-
"run bin/rails generate llm_cost_tracker:install && bin/rails db:migrate"
|
|
25
|
+
"current schema required; #{errors.join('; ')}; see docs/upgrading.md"
|
|
27
26
|
)
|
|
28
27
|
end
|
|
29
28
|
end
|
|
@@ -167,7 +167,7 @@ module LlmCostTracker
|
|
|
167
167
|
return live_rollups_check unless LlmCostTracker.configuration.budgets.totals_source == :cache
|
|
168
168
|
|
|
169
169
|
errors = LlmCostTracker::Ledger::Schema::CallRollups.current_schema_errors
|
|
170
|
-
return
|
|
170
|
+
return rollups_drift_check if errors.empty?
|
|
171
171
|
|
|
172
172
|
Check.new(
|
|
173
173
|
:error,
|
|
@@ -176,13 +176,30 @@ module LlmCostTracker
|
|
|
176
176
|
)
|
|
177
177
|
end
|
|
178
178
|
|
|
179
|
+
def rollups_drift_check
|
|
180
|
+
month_start = LlmCostTracker::Ledger::Period.range_start(:month, Time.now)
|
|
181
|
+
today = LlmCostTracker::Ledger::Period.range_start(:day, Time.now)
|
|
182
|
+
rolled = LlmCostTracker::CallRollup.where(period: "day", period_start: month_start.to_date...today.to_date)
|
|
183
|
+
.sum(:total_cost)
|
|
184
|
+
recorded = LlmCostTracker::Call.where(tracked_at: month_start...today).sum(:total_cost)
|
|
185
|
+
return Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists") if rolled == recorded
|
|
186
|
+
|
|
187
|
+
Check.new(
|
|
188
|
+
:warn,
|
|
189
|
+
"call rollups",
|
|
190
|
+
"this month's day rollups before today total #{rolled.to_d.to_s('F')} but the calls total " \
|
|
191
|
+
"#{recorded.to_d.to_s('F')}, and monthly budgets read the rollups; " \
|
|
192
|
+
"run bin/rails llm_cost_tracker:rebuild_rollups"
|
|
193
|
+
)
|
|
194
|
+
end
|
|
195
|
+
|
|
179
196
|
def live_rollups_check
|
|
180
197
|
if Probe.table_exists?("llm_cost_tracker_call_rollups")
|
|
181
198
|
Check.new(
|
|
182
199
|
:warn,
|
|
183
200
|
"call rollups",
|
|
184
201
|
"budgets.totals_source=:ledger but llm_cost_tracker_call_rollups exists. " \
|
|
185
|
-
"Set config.budgets.totals_source = :cache to
|
|
202
|
+
"Set config.budgets.totals_source = :cache to use it or drop the table."
|
|
186
203
|
)
|
|
187
204
|
else
|
|
188
205
|
Check.new(
|
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "redaction"
|
|
4
|
+
|
|
3
5
|
module LlmCostTracker
|
|
4
6
|
class Error < StandardError; end
|
|
5
7
|
|
|
6
8
|
class InvalidFilterError < Error; end
|
|
7
9
|
|
|
10
|
+
class TransactionAbortedError < Error
|
|
11
|
+
def initialize(error)
|
|
12
|
+
super(
|
|
13
|
+
"The database rolled back the whole surrounding transaction while recording LLM usage " \
|
|
14
|
+
"(#{error.class}: #{Redaction.text(error.message)}); the caller's transaction no longer exists"
|
|
15
|
+
)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
8
19
|
class BudgetExceededError < Error
|
|
9
20
|
attr_reader :total, :budget, :budget_type, :last_event, :stage, :scope
|
|
10
21
|
|
|
@@ -40,4 +51,6 @@ module LlmCostTracker
|
|
|
40
51
|
super("No pricing configured for LLM model: #{model.inspect}")
|
|
41
52
|
end
|
|
42
53
|
end
|
|
54
|
+
|
|
55
|
+
CALLER_ERRORS = [BudgetExceededError, UnknownPricingError, TransactionAbortedError].freeze
|
|
43
56
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "active_support/core_ext/digest/uuid"
|
|
4
|
+
|
|
3
5
|
require_relative "pricing/mode"
|
|
4
6
|
|
|
5
7
|
module LlmCostTracker
|
|
@@ -53,6 +55,12 @@ module LlmCostTracker
|
|
|
53
55
|
Pricing::Mode.tokenize(pricing_mode.to_s).include?("batch")
|
|
54
56
|
end
|
|
55
57
|
|
|
58
|
+
def keyed_by_response_id
|
|
59
|
+
return self unless provider_response_id
|
|
60
|
+
|
|
61
|
+
with(event_id: Digest::UUID.uuid_v5(Digest::UUID::OID_NAMESPACE, "#{provider}/#{provider_response_id}"))
|
|
62
|
+
end
|
|
63
|
+
|
|
56
64
|
def self.resolve_line_items(service_items)
|
|
57
65
|
Array(service_items).map do |item|
|
|
58
66
|
item.is_a?(Charges::LineItem) ? item : Charges::LineItem.build(item)
|
|
@@ -10,7 +10,7 @@ module LlmCostTracker
|
|
|
10
10
|
|
|
11
11
|
source_root File.expand_path("templates", __dir__)
|
|
12
12
|
|
|
13
|
-
desc "Creates the optional llm_cost_tracker_call_rollups table
|
|
13
|
+
desc "Creates the optional llm_cost_tracker_call_rollups table of running budget totals. " \
|
|
14
14
|
"Required when config.budgets.totals_source = :cache."
|
|
15
15
|
|
|
16
16
|
def create_migration_file
|
|
@@ -28,16 +28,13 @@ module LlmCostTracker
|
|
|
28
28
|
config.budgets.totals_source = :cache
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
Once that is deployed, run bin/rails llm_cost_tracker:rebuild_rollups. Until then
|
|
32
|
+
calls recorded before the switch are missing from monthly budget totals.
|
|
33
|
+
|
|
34
|
+
Without the setting Tracker keeps reading budget totals as live SUM aggregates over
|
|
32
35
|
llm_cost_tracker_calls. The doctor check warns about an unused rollups table.
|
|
33
36
|
MSG
|
|
34
37
|
end
|
|
35
|
-
|
|
36
|
-
private
|
|
37
|
-
|
|
38
|
-
def migration_version
|
|
39
|
-
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
40
|
-
end
|
|
41
38
|
end
|
|
42
39
|
end
|
|
43
40
|
end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
class CreateLlmCostTrackerAsyncIngestion < ActiveRecord::Migration<%=
|
|
1
|
+
class CreateLlmCostTrackerAsyncIngestion < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
2
2
|
def change
|
|
3
3
|
create_table :llm_cost_tracker_ingestion_inbox_entries do |t|
|
|
4
4
|
t.string :event_id, null: false
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
class CreateLlmCostTrackerCallRollups < ActiveRecord::Migration<%=
|
|
1
|
+
class CreateLlmCostTrackerCallRollups < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
2
2
|
def change
|
|
3
3
|
create_table :llm_cost_tracker_call_rollups do |t|
|
|
4
4
|
t.string :period, null: false
|
data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
require "llm_cost_tracker/charges/cost_status"
|
|
2
2
|
require "llm_cost_tracker/ledger/schema/adapter"
|
|
3
3
|
|
|
4
|
-
class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%=
|
|
4
|
+
class CreateLlmCostTrackerCalls < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
5
5
|
def change
|
|
6
6
|
create_table :llm_cost_tracker_calls do |t|
|
|
7
7
|
t.string :event_id, null: false
|
|
@@ -34,8 +34,8 @@ LlmCostTracker.configure do |config|
|
|
|
34
34
|
# "my-custom-model" => { input: 1.00, output: 2.00 }
|
|
35
35
|
# }
|
|
36
36
|
# :warn (default) records token usage with nil cost when a model has no rate.
|
|
37
|
-
# Use :raise to
|
|
38
|
-
# component rate missing lands as partial and does not trigger it.
|
|
37
|
+
# Use :raise to record and then raise for models that have no rate at all; a model
|
|
38
|
+
# priced with one component rate missing lands as partial and does not trigger it.
|
|
39
39
|
config.pricing.unknown_model_behavior = :warn
|
|
40
40
|
|
|
41
41
|
# Budget guardrails — cumulative monthly/daily and per-call ceilings in USD,
|
|
@@ -50,8 +50,9 @@ LlmCostTracker.configure do |config|
|
|
|
50
50
|
|
|
51
51
|
# One budget applied to every distinct value of a tag: each tenant gets its own
|
|
52
52
|
# 1000 a month, not 1000 shared between them. Declare as many tags as you need; a rule
|
|
53
|
-
# can set its own behavior and callback instead of following the global ones.
|
|
54
|
-
# cost columns
|
|
53
|
+
# can set its own behavior and callback instead of following the global ones. Installs created
|
|
54
|
+
# before v0.14 add the cost columns with `bin/rails generate llm_cost_tracker:upgrade_per_tag_budgets`
|
|
55
|
+
# and `bin/rails db:migrate`, then run `bin/rails llm_cost_tracker:backfill_tag_costs` so earlier calls count.
|
|
55
56
|
# Budget high-cardinality tags only — a tag with a handful of values covers most of the
|
|
56
57
|
# ledger, so its check cannot use an index and every call pays for a near-full scan.
|
|
57
58
|
# config.budgets.per_tag = {
|
|
@@ -63,9 +64,9 @@ LlmCostTracker.configure do |config|
|
|
|
63
64
|
# }
|
|
64
65
|
|
|
65
66
|
# Where budget checks read period spend from. :ledger sums llm_cost_tracker_calls on
|
|
66
|
-
# every check; :cache
|
|
67
|
-
#
|
|
68
|
-
#
|
|
67
|
+
# every check; :cache reads earlier days from llm_cost_tracker_call_rollups and sums
|
|
68
|
+
# only today's calls. Run `bin/rails generate llm_cost_tracker:call_rollups`, deploy
|
|
69
|
+
# this setting, then run `bin/rails llm_cost_tracker:rebuild_rollups`.
|
|
69
70
|
# config.budgets.totals_source = :cache
|
|
70
71
|
|
|
71
72
|
# Ingestion path — :inline (default) writes events synchronously from the request
|
|
@@ -74,18 +75,19 @@ LlmCostTracker.configure do |config|
|
|
|
74
75
|
# inbox/leases tables created by `bin/rails generate llm_cost_tracker:async_ingestion`.
|
|
75
76
|
# Synchronous inbox writes use a dedicated ActiveRecord pool (defaults to 2 connections)
|
|
76
77
|
# so they don't compete with request threads for the default pool when a tracked call
|
|
77
|
-
# happens inside an open caller transaction.
|
|
78
|
-
#
|
|
78
|
+
# happens inside an open caller transaction. The pool is per process: bump ingestion.pool_size
|
|
79
|
+
# if concurrent tracked calls in one process (e.g. Puma threads) outgrow that.
|
|
79
80
|
# config.ingestion.mode = :async
|
|
80
81
|
# config.ingestion.pool_size = 5
|
|
81
82
|
|
|
82
|
-
# Register OpenAI-compatible gateway hosts
|
|
83
|
-
#
|
|
83
|
+
# Register OpenAI-compatible gateway hosts beyond the built-in OpenRouter, DeepSeek and Groq;
|
|
84
|
+
# the Faraday middleware captures only listed hosts. Its calls, and openai gem calls whose
|
|
85
|
+
# base_url is a listed host, are recorded under the provider name given here.
|
|
84
86
|
# config.capture.openai_compatible_providers["llm.my-company.com"] = "internal_gateway"
|
|
85
87
|
|
|
86
88
|
# Streaming calls only report token usage when the request asks for it. Leave this
|
|
87
89
|
# on and the Faraday middleware adds stream_options: { include_usage: true } to
|
|
88
|
-
#
|
|
90
|
+
# streaming chat-completions requests on hosts known to accept it.
|
|
89
91
|
# config.capture.request_stream_usage = false
|
|
90
92
|
|
|
91
93
|
end
|
data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
|
-
class UpgradeLlmCostTrackerCallRollupsProvider < ActiveRecord::Migration<%=
|
|
3
|
+
class UpgradeLlmCostTrackerCallRollupsProvider < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
4
4
|
disable_ddl_transaction!
|
|
5
5
|
|
|
6
6
|
TABLE = :llm_cost_tracker_call_rollups
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
|
-
class UpgradeLlmCostTrackerCallTagsKeyValueIndex < ActiveRecord::Migration<%=
|
|
3
|
+
class UpgradeLlmCostTrackerCallTagsKeyValueIndex < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
4
4
|
disable_ddl_transaction!
|
|
5
5
|
|
|
6
6
|
TABLE = :llm_cost_tracker_call_tags
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
class UpgradeLlmCostTrackerImageTokens < ActiveRecord::Migration<%=
|
|
1
|
+
class UpgradeLlmCostTrackerImageTokens < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
2
2
|
TABLE = :llm_cost_tracker_calls
|
|
3
3
|
COLUMNS = %i[image_input_tokens image_output_tokens].freeze
|
|
4
4
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
|
-
class UpgradeLlmCostTrackerIndexes < ActiveRecord::Migration<%=
|
|
3
|
+
class UpgradeLlmCostTrackerIndexes < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
4
4
|
disable_ddl_transaction!
|
|
5
5
|
|
|
6
6
|
CALLS = :llm_cost_tracker_calls
|
data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
|
-
class UpgradeLlmCostTrackerPerTagBudgets < ActiveRecord::Migration<%=
|
|
3
|
+
class UpgradeLlmCostTrackerPerTagBudgets < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
|
|
4
4
|
disable_ddl_transaction!
|
|
5
5
|
|
|
6
6
|
TABLE = :llm_cost_tracker_call_tags
|