llm_cost_tracker 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -1
- data/README.md +2 -1
- data/app/controllers/llm_cost_tracker/application_controller.rb +1 -1
- data/app/controllers/llm_cost_tracker/calls_controller.rb +16 -4
- data/app/helpers/llm_cost_tracker/application_helper.rb +1 -1
- data/app/models/llm_cost_tracker/provider_invoice_import.rb +9 -4
- data/app/services/llm_cost_tracker/dashboard/setup_state.rb +110 -0
- data/app/views/llm_cost_tracker/calls/show.html.erb +1 -1
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +1 -1
- data/lib/llm_cost_tracker/billing/cost_status.rb +21 -25
- data/lib/llm_cost_tracker/billing/line_item.rb +15 -49
- data/lib/llm_cost_tracker/budget.rb +28 -6
- data/lib/llm_cost_tracker/capture/stream_collector.rb +35 -29
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +1 -1
- data/lib/llm_cost_tracker/configuration.rb +31 -28
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +1 -1
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +8 -8
- data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor.rb +6 -17
- data/lib/llm_cost_tracker/engine.rb +1 -2
- data/lib/llm_cost_tracker/errors.rb +3 -2
- data/lib/llm_cost_tracker/event.rb +47 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/{durable_ingestion_generator.rb → async_ingestion_generator.rb} +8 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +4 -23
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/{create_llm_cost_tracker_durable_ingestion.rb.erb → create_llm_cost_tracker_async_ingestion.rb.erb} +3 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb +6 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +14 -7
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +23 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +5 -5
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoice_imports_provider.rb.erb +32 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoices_metadata_index.rb.erb +25 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +0 -9
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoice_imports_provider_generator.rb +31 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoices_metadata_index_generator.rb +31 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +5 -2
- data/lib/llm_cost_tracker/ingestion/inbox.rb +3 -24
- data/lib/llm_cost_tracker/ingestion/pool.rb +44 -0
- data/lib/llm_cost_tracker/ingestion/worker.rb +22 -36
- data/lib/llm_cost_tracker/ingestion.rb +8 -9
- data/lib/llm_cost_tracker/integrations/anthropic.rb +28 -42
- data/lib/llm_cost_tracker/integrations/base.rb +14 -11
- data/lib/llm_cost_tracker/integrations/openai.rb +93 -66
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +18 -20
- data/lib/llm_cost_tracker/integrations.rb +14 -13
- data/lib/llm_cost_tracker/ledger/period/totals.rb +5 -3
- data/lib/llm_cost_tracker/ledger/rollups.rb +4 -13
- data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +11 -0
- data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +13 -3
- data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +11 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +0 -4
- data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb +13 -3
- data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb +13 -3
- data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb +19 -9
- data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +26 -11
- data/lib/llm_cost_tracker/ledger/store.rb +21 -18
- data/lib/llm_cost_tracker/ledger/tags/query.rb +0 -1
- data/lib/llm_cost_tracker/logging.rb +0 -4
- data/lib/llm_cost_tracker/middleware/faraday.rb +44 -16
- data/lib/llm_cost_tracker/parsers/anthropic.rb +21 -28
- data/lib/llm_cost_tracker/parsers/azure.rb +46 -0
- data/lib/llm_cost_tracker/parsers/base.rb +53 -47
- data/lib/llm_cost_tracker/parsers/gemini.rb +20 -22
- data/lib/llm_cost_tracker/parsers/openai.rb +8 -40
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +26 -43
- data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +45 -16
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +16 -20
- data/lib/llm_cost_tracker/parsers.rb +31 -4
- data/lib/llm_cost_tracker/prices.json +567 -579
- data/lib/llm_cost_tracker/pricing/backfill.rb +140 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +2 -4
- data/lib/llm_cost_tracker/pricing/estimator.rb +33 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +4 -1
- data/lib/llm_cost_tracker/pricing/lookup.rb +37 -2
- data/lib/llm_cost_tracker/pricing/registry.rb +0 -7
- data/lib/llm_cost_tracker/pricing/service_charges.rb +5 -9
- data/lib/llm_cost_tracker/pricing/{sync_change_printer.rb → sync/change_printer.rb} +3 -3
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +14 -2
- data/lib/llm_cost_tracker/pricing/sync.rb +1 -9
- data/lib/llm_cost_tracker/pricing/unknown.rb +5 -2
- data/lib/llm_cost_tracker/pricing.rb +72 -27
- data/lib/llm_cost_tracker/providers/anthropic/tier_classification.rb +22 -0
- data/lib/llm_cost_tracker/providers/azure/hosts.rb +17 -0
- data/lib/llm_cost_tracker/providers/gemini/model_families.rb +17 -0
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +35 -0
- data/lib/llm_cost_tracker/providers/openai/model_families.rb +51 -0
- data/lib/llm_cost_tracker/railtie.rb +3 -1
- data/lib/llm_cost_tracker/reconciliation/diff.rb +26 -45
- data/lib/llm_cost_tracker/reconciliation/diff_result.rb +0 -4
- data/lib/llm_cost_tracker/reconciliation/importer.rb +1 -0
- data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb +4 -3
- data/lib/llm_cost_tracker/report.rb +0 -4
- data/lib/llm_cost_tracker/retention.rb +20 -8
- data/lib/llm_cost_tracker/tags/sanitizer.rb +13 -17
- data/lib/llm_cost_tracker/token_usage.rb +4 -0
- data/lib/llm_cost_tracker/tracker.rb +33 -74
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +11 -15
- data/lib/tasks/llm_cost_tracker.rake +16 -2
- metadata +18 -7
- data/lib/llm_cost_tracker/dashboard_setup_state.rb +0 -109
- data/lib/llm_cost_tracker/ingestion/inline.rb +0 -22
- data/lib/llm_cost_tracker/usage_capture.rb +0 -58
|
@@ -69,7 +69,6 @@ module LlmCostTracker
|
|
|
69
69
|
ensure_open!
|
|
70
70
|
capture_event(data, type: type) unless data.nil?
|
|
71
71
|
end
|
|
72
|
-
self
|
|
73
72
|
end
|
|
74
73
|
|
|
75
74
|
def usage(input_tokens:, output_tokens:, **extra)
|
|
@@ -87,7 +86,6 @@ module LlmCostTracker
|
|
|
87
86
|
output_tokens: output_tokens
|
|
88
87
|
)
|
|
89
88
|
end
|
|
90
|
-
self
|
|
91
89
|
end
|
|
92
90
|
|
|
93
91
|
def finish!(errored: false)
|
|
@@ -124,17 +122,17 @@ module LlmCostTracker
|
|
|
124
122
|
def record_snapshot(snapshot, errored:)
|
|
125
123
|
save_succeeded = false
|
|
126
124
|
begin
|
|
127
|
-
|
|
128
|
-
provider_response_id =
|
|
129
|
-
|
|
125
|
+
event = build_event(snapshot)
|
|
126
|
+
provider_response_id = event.provider_response_id || snapshot[:provider_response_id]
|
|
127
|
+
event = event.with(provider_response_id: provider_response_id)
|
|
130
128
|
|
|
131
129
|
Tracker.record(
|
|
132
|
-
|
|
130
|
+
event: event,
|
|
133
131
|
latency_ms: snapshot[:latency_ms] || LlmCostTracker::Timing.elapsed_ms(@started_at),
|
|
134
|
-
pricing_mode:
|
|
132
|
+
pricing_mode: merge_pricing_modes(event.pricing_mode, snapshot[:pricing_mode]),
|
|
135
133
|
metadata: (errored ? { stream_errored: true } : {}).merge(snapshot[:metadata]),
|
|
136
134
|
context_tags: snapshot[:context_tags]
|
|
137
|
-
) {
|
|
135
|
+
) { save_succeeded = true }
|
|
138
136
|
ensure
|
|
139
137
|
@mutex.synchronize do
|
|
140
138
|
@finished = save_succeeded
|
|
@@ -144,17 +142,12 @@ module LlmCostTracker
|
|
|
144
142
|
end
|
|
145
143
|
|
|
146
144
|
HOST_DERIVED_MODE_TOKENS = %i[data_residency].freeze
|
|
147
|
-
|
|
148
|
-
private_constant :HOST_DERIVED_MODE_TOKENS, :STANDARD_LIKE_MODE_TOKENS
|
|
149
|
-
|
|
150
|
-
def pricing_mode_for(capture:, snapshot:)
|
|
151
|
-
merge_pricing_modes(capture.pricing_mode, snapshot[:pricing_mode])
|
|
152
|
-
end
|
|
145
|
+
private_constant :HOST_DERIVED_MODE_TOKENS
|
|
153
146
|
|
|
154
147
|
def merge_pricing_modes(provider_mode, request_mode)
|
|
155
148
|
return Pricing.normalize_mode(request_mode) if provider_mode.to_s.strip.empty?
|
|
156
149
|
|
|
157
|
-
provider_tokens = Pricing::Mode.tokenize(provider_mode) -
|
|
150
|
+
provider_tokens = Pricing::Mode.tokenize(provider_mode) - Pricing::STANDARD_MODE_VALUES
|
|
158
151
|
request_host_tokens = Pricing::Mode.tokenize(request_mode || "") & HOST_DERIVED_MODE_TOKENS
|
|
159
152
|
combined = provider_tokens | request_host_tokens
|
|
160
153
|
return nil if combined.empty?
|
|
@@ -163,7 +156,7 @@ module LlmCostTracker
|
|
|
163
156
|
end
|
|
164
157
|
|
|
165
158
|
def capture_dimensions(pricing_mode)
|
|
166
|
-
batch = @batch.nil? ?
|
|
159
|
+
batch = @batch.nil? ? Event.batch_from_pricing_mode?(pricing_mode).presence : @batch
|
|
167
160
|
{
|
|
168
161
|
provider_project_id: @provider_project_id.to_s.strip.presence,
|
|
169
162
|
provider_api_key_id: @provider_api_key_id.to_s.strip.presence,
|
|
@@ -178,18 +171,18 @@ module LlmCostTracker
|
|
|
178
171
|
raise FrozenError, "can't modify finished LlmCostTracker::Capture::StreamCollector"
|
|
179
172
|
end
|
|
180
173
|
|
|
181
|
-
def
|
|
174
|
+
def build_event(snapshot)
|
|
182
175
|
return build_from_explicit_usage(snapshot) if snapshot[:explicit_usage]
|
|
183
176
|
return build_unknown_usage(snapshot) if snapshot[:overflowed]
|
|
184
177
|
|
|
185
|
-
|
|
178
|
+
event = Parsers.find_for_provider(@provider)&.parse_stream(
|
|
186
179
|
response_status: 200,
|
|
187
180
|
events: snapshot[:events],
|
|
188
181
|
request_body: request_body_for(snapshot[:request])
|
|
189
182
|
)
|
|
190
|
-
if
|
|
191
|
-
model = present_model(
|
|
192
|
-
return
|
|
183
|
+
if event
|
|
184
|
+
model = present_model(event.model) || present_model(snapshot[:model]) || Event::UNKNOWN_MODEL
|
|
185
|
+
return event.with(provider: @provider, model: model, **snapshot.fetch(:capture_dimensions))
|
|
193
186
|
end
|
|
194
187
|
|
|
195
188
|
build_unknown_usage(snapshot)
|
|
@@ -207,15 +200,15 @@ module LlmCostTracker
|
|
|
207
200
|
return nil if value.nil?
|
|
208
201
|
|
|
209
202
|
string = value.to_s.presence
|
|
210
|
-
return nil if string.nil? || string ==
|
|
203
|
+
return nil if string.nil? || string == Event::UNKNOWN_MODEL
|
|
211
204
|
|
|
212
205
|
string
|
|
213
206
|
end
|
|
214
207
|
|
|
215
208
|
def build_from_explicit_usage(snapshot)
|
|
216
|
-
|
|
209
|
+
Event.build(
|
|
217
210
|
provider: @provider,
|
|
218
|
-
model: snapshot[:model] ||
|
|
211
|
+
model: snapshot[:model] || Event::UNKNOWN_MODEL,
|
|
219
212
|
token_usage: snapshot[:explicit_usage],
|
|
220
213
|
stream: true,
|
|
221
214
|
usage_source: :manual,
|
|
@@ -225,9 +218,9 @@ module LlmCostTracker
|
|
|
225
218
|
end
|
|
226
219
|
|
|
227
220
|
def build_unknown_usage(snapshot)
|
|
228
|
-
|
|
221
|
+
Event.build(
|
|
229
222
|
provider: @provider,
|
|
230
|
-
model: snapshot[:model] ||
|
|
223
|
+
model: snapshot[:model] || Event::UNKNOWN_MODEL,
|
|
231
224
|
token_usage: TokenUsage.build(input_tokens: 0, output_tokens: 0, total_tokens: 0),
|
|
232
225
|
stream: true,
|
|
233
226
|
usage_source: :unknown,
|
|
@@ -244,14 +237,14 @@ module LlmCostTracker
|
|
|
244
237
|
|
|
245
238
|
def capture_event(data, type:)
|
|
246
239
|
event = { event: type, data: strip_heavy_payload(data) }
|
|
247
|
-
size =
|
|
240
|
+
size = approximate_bytesize(event)
|
|
248
241
|
if @captured_bytes + size <= Capture::Stream::LIMIT_BYTES
|
|
249
|
-
@events << event
|
|
242
|
+
@events << event
|
|
250
243
|
@captured_bytes += size
|
|
251
244
|
else
|
|
252
245
|
@overflowed = true
|
|
253
246
|
end
|
|
254
|
-
rescue
|
|
247
|
+
rescue TypeError, SystemStackError
|
|
255
248
|
@overflowed = true
|
|
256
249
|
end
|
|
257
250
|
|
|
@@ -271,6 +264,19 @@ module LlmCostTracker
|
|
|
271
264
|
value
|
|
272
265
|
end
|
|
273
266
|
end
|
|
267
|
+
|
|
268
|
+
def approximate_bytesize(value)
|
|
269
|
+
case value
|
|
270
|
+
when Hash
|
|
271
|
+
value.sum { |key, nested| approximate_bytesize(key) + approximate_bytesize(nested) + 4 }
|
|
272
|
+
when Array
|
|
273
|
+
value.sum { |nested| approximate_bytesize(nested) + 2 }
|
|
274
|
+
when Numeric, true, false, nil
|
|
275
|
+
8
|
|
276
|
+
else
|
|
277
|
+
value.to_s.bytesize + 2
|
|
278
|
+
end
|
|
279
|
+
end
|
|
274
280
|
end
|
|
275
281
|
end
|
|
276
282
|
end
|
|
@@ -14,18 +14,21 @@ module LlmCostTracker
|
|
|
14
14
|
|
|
15
15
|
BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
|
|
16
16
|
UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
INGESTION_MODES = %i[inline async].freeze
|
|
18
|
+
SCALAR_ATTRIBUTES = %i[enabled default_tags on_budget_exceeded monthly_budget daily_budget per_call_budget
|
|
19
|
+
log_level prices_file max_tag_count max_tag_value_bytesize
|
|
20
|
+
ingestion_pool_size].freeze
|
|
21
|
+
ENUM_ATTRIBUTES = {
|
|
20
22
|
budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
|
|
21
|
-
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
|
|
23
|
+
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn],
|
|
24
|
+
ingestion: [INGESTION_MODES, :inline]
|
|
22
25
|
}.freeze
|
|
23
26
|
DEFAULT_REDACTED_TAG_KEYS = %w[api_key access_token authorization credential password refresh_token secret].freeze
|
|
24
27
|
|
|
25
28
|
attr_reader(
|
|
26
|
-
*
|
|
29
|
+
*SCALAR_ATTRIBUTES,
|
|
27
30
|
:budget_exceeded_behavior,
|
|
28
|
-
:
|
|
31
|
+
:ingestion,
|
|
29
32
|
:instrumented_integrations,
|
|
30
33
|
:pricing_overrides,
|
|
31
34
|
:report_tag_breakdowns,
|
|
@@ -51,6 +54,7 @@ module LlmCostTracker
|
|
|
51
54
|
@prices_file = nil
|
|
52
55
|
@max_tag_count = 50
|
|
53
56
|
@max_tag_value_bytesize = 1024
|
|
57
|
+
@ingestion_pool_size = nil
|
|
54
58
|
self.pricing_overrides = {}
|
|
55
59
|
@instrumented_integrations = Set.new
|
|
56
60
|
@report_tag_breakdowns = []
|
|
@@ -59,33 +63,28 @@ module LlmCostTracker
|
|
|
59
63
|
@reconciliation_importers = {}
|
|
60
64
|
@reconciliation_enabled = false
|
|
61
65
|
@auto_enable_stream_usage = true
|
|
62
|
-
|
|
66
|
+
self.ingestion = :inline
|
|
63
67
|
@cache_rollups = false
|
|
64
68
|
@finalized = false
|
|
65
69
|
end
|
|
66
70
|
|
|
67
|
-
def durable_ingestion=(value)
|
|
68
|
-
ensure_shared_configuration_mutable!
|
|
69
|
-
@durable_ingestion = value
|
|
70
|
-
end
|
|
71
|
-
|
|
72
71
|
def cache_rollups=(value)
|
|
73
|
-
|
|
72
|
+
ensure_mutable!
|
|
74
73
|
@cache_rollups = value
|
|
75
74
|
end
|
|
76
75
|
|
|
77
76
|
def reconciliation_enabled=(value)
|
|
78
|
-
|
|
77
|
+
ensure_mutable!
|
|
79
78
|
@reconciliation_enabled = value
|
|
80
79
|
end
|
|
81
80
|
|
|
82
81
|
def auto_enable_stream_usage=(value)
|
|
83
|
-
|
|
82
|
+
ensure_mutable!
|
|
84
83
|
@auto_enable_stream_usage = value
|
|
85
84
|
end
|
|
86
85
|
|
|
87
86
|
def reconciliation_importers=(importers)
|
|
88
|
-
|
|
87
|
+
ensure_mutable!
|
|
89
88
|
raise Error, RECONCILIATION_DISABLED_MESSAGE unless @reconciliation_enabled
|
|
90
89
|
|
|
91
90
|
@reconciliation_importers = (importers || {}).to_h do |source, importer|
|
|
@@ -96,7 +95,7 @@ module LlmCostTracker
|
|
|
96
95
|
end
|
|
97
96
|
|
|
98
97
|
def register_reconciliation_importer(source, &block)
|
|
99
|
-
|
|
98
|
+
ensure_mutable!
|
|
100
99
|
raise Error, RECONCILIATION_DISABLED_MESSAGE unless @reconciliation_enabled
|
|
101
100
|
raise Error, "register_reconciliation_importer requires a block" unless block
|
|
102
101
|
|
|
@@ -107,29 +106,29 @@ module LlmCostTracker
|
|
|
107
106
|
private_constant :RECONCILIATION_DISABLED_MESSAGE
|
|
108
107
|
|
|
109
108
|
def openai_compatible_providers=(providers)
|
|
110
|
-
|
|
109
|
+
ensure_mutable!
|
|
111
110
|
@openai_compatible_providers = normalize_openai_compatible_providers(providers)
|
|
112
111
|
end
|
|
113
112
|
|
|
114
113
|
def pricing_overrides=(value)
|
|
115
|
-
|
|
114
|
+
ensure_mutable!
|
|
116
115
|
@pricing_overrides = Pricing::Registry.normalize_price_table(value || {})
|
|
117
116
|
rescue ArgumentError => e
|
|
118
117
|
raise Error, "invalid pricing_overrides: #{e.message}"
|
|
119
118
|
end
|
|
120
119
|
|
|
121
120
|
def report_tag_breakdowns=(value)
|
|
122
|
-
|
|
121
|
+
ensure_mutable!
|
|
123
122
|
@report_tag_breakdowns = Array(value).map { |key| Tags::Key.validate!(key, error_class: Error) }
|
|
124
123
|
end
|
|
125
124
|
|
|
126
125
|
def redacted_tag_keys=(value)
|
|
127
|
-
|
|
126
|
+
ensure_mutable!
|
|
128
127
|
@redacted_tag_keys = Array(value).map(&:to_s)
|
|
129
128
|
end
|
|
130
129
|
|
|
131
130
|
def instrument(*names)
|
|
132
|
-
|
|
131
|
+
ensure_mutable!
|
|
133
132
|
@instrumented_integrations.merge(normalize_instrumentation_names(names))
|
|
134
133
|
end
|
|
135
134
|
|
|
@@ -137,16 +136,16 @@ module LlmCostTracker
|
|
|
137
136
|
@instrumented_integrations.include?(name)
|
|
138
137
|
end
|
|
139
138
|
|
|
140
|
-
|
|
139
|
+
SCALAR_ATTRIBUTES.each do |name|
|
|
141
140
|
define_method("#{name}=") do |value|
|
|
142
|
-
|
|
141
|
+
ensure_mutable!
|
|
143
142
|
instance_variable_set(:"@#{name}", value)
|
|
144
143
|
end
|
|
145
144
|
end
|
|
146
145
|
|
|
147
|
-
|
|
146
|
+
ENUM_ATTRIBUTES.each do |name, (allowed, default)|
|
|
148
147
|
define_method("#{name}=") do |value|
|
|
149
|
-
|
|
148
|
+
ensure_mutable!
|
|
150
149
|
instance_variable_set(:"@#{name}", normalize_enum(name, value, allowed, default: default))
|
|
151
150
|
end
|
|
152
151
|
end
|
|
@@ -161,7 +160,11 @@ module LlmCostTracker
|
|
|
161
160
|
normalize_openai_compatible_providers(@openai_compatible_providers)
|
|
162
161
|
)
|
|
163
162
|
@finalized = true
|
|
164
|
-
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def normalized_redacted_tag_keys
|
|
166
|
+
@normalized_redacted_tag_keys ||=
|
|
167
|
+
Array(@redacted_tag_keys).map { |key| Tags::Sanitizer.normalized_key(key) }.freeze
|
|
165
168
|
end
|
|
166
169
|
|
|
167
170
|
def finalized?
|
|
@@ -196,7 +199,7 @@ module LlmCostTracker
|
|
|
196
199
|
names
|
|
197
200
|
end
|
|
198
201
|
|
|
199
|
-
def
|
|
202
|
+
def ensure_mutable!
|
|
200
203
|
return unless finalized?
|
|
201
204
|
|
|
202
205
|
raise FrozenError, "can't modify frozen LlmCostTracker::Configuration"
|
|
@@ -11,14 +11,14 @@ module LlmCostTracker
|
|
|
11
11
|
|
|
12
12
|
def call
|
|
13
13
|
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
14
|
-
return inline_check unless LlmCostTracker::Ingestion.
|
|
14
|
+
return inline_check unless LlmCostTracker::Ingestion.async?
|
|
15
15
|
|
|
16
16
|
missing = missing_parts
|
|
17
17
|
if missing.empty?
|
|
18
18
|
inbox = inbox_snapshot
|
|
19
19
|
quarantined = inbox.try(:quarantined_count).to_i
|
|
20
20
|
if quarantined.positive?
|
|
21
|
-
return Check.new(:warn, "
|
|
21
|
+
return Check.new(:warn, "async ingestion", "#{quarantined} inbox entries quarantined after retries")
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
pending_count = inbox.try(:pending_count).to_i
|
|
@@ -27,17 +27,17 @@ module LlmCostTracker
|
|
|
27
27
|
if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
|
|
28
28
|
return Check.new(
|
|
29
29
|
:warn,
|
|
30
|
-
"
|
|
30
|
+
"async ingestion",
|
|
31
31
|
"#{pending_count} inbox entries pending; oldest pending age #{pending_age.round}s"
|
|
32
32
|
)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
return Check.new(:ok, "
|
|
35
|
+
return Check.new(:ok, "async ingestion", "inbox and ingestion lease tables available")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
Check.new(
|
|
39
39
|
:error,
|
|
40
|
-
"
|
|
40
|
+
"async ingestion",
|
|
41
41
|
"missing #{missing.join(', ')}; see docs/upgrading.md for the recovery steps"
|
|
42
42
|
)
|
|
43
43
|
end
|
|
@@ -50,15 +50,15 @@ module LlmCostTracker
|
|
|
50
50
|
return Check.new(
|
|
51
51
|
:ok,
|
|
52
52
|
"inline ingestion",
|
|
53
|
-
"
|
|
53
|
+
"config.ingestion = :inline; events write directly to the ledger"
|
|
54
54
|
)
|
|
55
55
|
end
|
|
56
56
|
|
|
57
57
|
Check.new(
|
|
58
58
|
:warn,
|
|
59
59
|
"inline ingestion",
|
|
60
|
-
"
|
|
61
|
-
"Set config.
|
|
60
|
+
"config.ingestion = :inline but found unused async ingestion tables: #{leftovers.join(', ')}. " \
|
|
61
|
+
"Set config.ingestion = :async to keep the inbox path or drop the tables."
|
|
62
62
|
)
|
|
63
63
|
end
|
|
64
64
|
|
|
@@ -28,8 +28,6 @@ module LlmCostTracker
|
|
|
28
28
|
message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
|
|
29
29
|
"stored totals remain stable but applied rates cannot be audited"
|
|
30
30
|
Check.new(:warn, "pricing snapshot audit", message)
|
|
31
|
-
rescue StandardError
|
|
32
|
-
nil
|
|
33
31
|
end
|
|
34
32
|
end
|
|
35
33
|
end
|
|
@@ -14,6 +14,7 @@ require_relative "doctor/pricing_snapshot_drift_check"
|
|
|
14
14
|
module LlmCostTracker
|
|
15
15
|
class Doctor
|
|
16
16
|
autoload :InvoiceReconciliationCheck, "llm_cost_tracker/doctor/invoice_reconciliation_check"
|
|
17
|
+
autoload :CaptureVerifier, "llm_cost_tracker/doctor/capture_verifier"
|
|
17
18
|
|
|
18
19
|
class << self
|
|
19
20
|
def call
|
|
@@ -35,7 +36,7 @@ module LlmCostTracker
|
|
|
35
36
|
[
|
|
36
37
|
configuration_check,
|
|
37
38
|
capture_check,
|
|
38
|
-
*
|
|
39
|
+
*LlmCostTracker::Integrations.checks,
|
|
39
40
|
active_record_check,
|
|
40
41
|
table_check,
|
|
41
42
|
column_check,
|
|
@@ -61,17 +62,13 @@ module LlmCostTracker
|
|
|
61
62
|
def reconciliation_schema_checks
|
|
62
63
|
return [] unless LlmCostTracker.reconciliation_enabled?
|
|
63
64
|
|
|
64
|
-
LlmCostTracker.const_get(:Reconciliation) # autoload reconciliation + its ledger schemas
|
|
65
65
|
Reconciliation::SCHEMA_TABLES.map do |schema, table|
|
|
66
|
-
SchemaCheck.new(name:
|
|
66
|
+
SchemaCheck.new(name: table.delete_prefix("llm_cost_tracker_").tr("_", " "),
|
|
67
|
+
schema: schema, table: table,
|
|
67
68
|
optional: false, install_command: "llm_cost_tracker:reconciliation").call
|
|
68
69
|
end.compact
|
|
69
70
|
end
|
|
70
71
|
|
|
71
|
-
def humanize_table(table)
|
|
72
|
-
table.delete_prefix("llm_cost_tracker_").tr("_", " ")
|
|
73
|
-
end
|
|
74
|
-
|
|
75
72
|
def reconciliation_invoice_check
|
|
76
73
|
return [] unless LlmCostTracker.reconciliation_enabled?
|
|
77
74
|
|
|
@@ -104,12 +101,6 @@ module LlmCostTracker
|
|
|
104
101
|
)
|
|
105
102
|
end
|
|
106
103
|
|
|
107
|
-
def integration_checks
|
|
108
|
-
LlmCostTracker::Integrations.checks.map do |check|
|
|
109
|
-
Check.new(check.status, check.name.to_s, check.message)
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
|
|
113
104
|
def active_record_check
|
|
114
105
|
return Check.new(:ok, "active_record", "available") if active_record_available?
|
|
115
106
|
|
|
@@ -209,16 +200,14 @@ module LlmCostTracker
|
|
|
209
200
|
count = snapshot.tracked_call_count.to_i
|
|
210
201
|
return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
|
|
211
202
|
|
|
212
|
-
|
|
213
|
-
latest_at = latest_at.to_time if latest_at.respond_to?(:to_time)
|
|
214
|
-
latest = latest_at&.utc&.iso8601
|
|
203
|
+
latest = snapshot.latest_tracked_at.to_time.utc.iso8601
|
|
215
204
|
Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
|
|
216
205
|
end
|
|
217
206
|
|
|
218
207
|
def active_record_available?
|
|
219
208
|
LlmCostTracker::Call.connection
|
|
220
209
|
true
|
|
221
|
-
rescue
|
|
210
|
+
rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
|
|
222
211
|
false
|
|
223
212
|
end
|
|
224
213
|
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
require "rails"
|
|
4
4
|
require_relative "../llm_cost_tracker"
|
|
5
5
|
require_relative "assets"
|
|
6
|
-
require_relative "dashboard_setup_state"
|
|
7
6
|
require "rack/files"
|
|
8
7
|
|
|
9
8
|
module LlmCostTracker
|
|
@@ -15,7 +14,7 @@ module LlmCostTracker
|
|
|
15
14
|
end
|
|
16
15
|
|
|
17
16
|
initializer "llm_cost_tracker.dashboard_setup_state" do |app|
|
|
18
|
-
app.reloader.to_prepare { LlmCostTracker::
|
|
17
|
+
app.reloader.to_prepare { LlmCostTracker::Dashboard::SetupState.reset! }
|
|
19
18
|
end
|
|
20
19
|
end
|
|
21
20
|
end
|
|
@@ -6,13 +6,14 @@ module LlmCostTracker
|
|
|
6
6
|
class InvalidFilterError < Error; end
|
|
7
7
|
|
|
8
8
|
class BudgetExceededError < Error
|
|
9
|
-
attr_reader :total, :budget, :budget_type, :last_event
|
|
9
|
+
attr_reader :total, :budget, :budget_type, :last_event, :stage
|
|
10
10
|
|
|
11
|
-
def initialize(budget:, budget_type:, total:, last_event: nil)
|
|
11
|
+
def initialize(budget:, budget_type:, total:, last_event: nil, stage: :post_spend)
|
|
12
12
|
@total = total
|
|
13
13
|
@budget = budget
|
|
14
14
|
@budget_type = budget_type
|
|
15
15
|
@last_event = last_event
|
|
16
|
+
@stage = stage
|
|
16
17
|
|
|
17
18
|
super(
|
|
18
19
|
"LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded: " \
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "pricing"
|
|
4
|
+
require_relative "billing/line_item"
|
|
5
|
+
|
|
3
6
|
module LlmCostTracker
|
|
4
7
|
Event = Data.define(
|
|
5
8
|
:event_id,
|
|
@@ -22,6 +25,46 @@ module LlmCostTracker
|
|
|
22
25
|
:pricing_snapshot,
|
|
23
26
|
:line_items
|
|
24
27
|
) do
|
|
28
|
+
def self.batch_from_pricing_mode?(pricing_mode)
|
|
29
|
+
pricing_mode.to_s.split("_").include?("batch")
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def self.build(**attributes)
|
|
33
|
+
pricing_mode = Pricing.normalize_mode(attributes[:pricing_mode])
|
|
34
|
+
token_usage = attributes.fetch(:token_usage)
|
|
35
|
+
batch = attributes[:batch].nil? ? batch_from_pricing_mode?(pricing_mode) : attributes[:batch]
|
|
36
|
+
line_items = attributes[:line_items] || resolve_line_items(attributes[:service_line_items], token_usage)
|
|
37
|
+
|
|
38
|
+
new(
|
|
39
|
+
event_id: attributes[:event_id],
|
|
40
|
+
provider: attributes.fetch(:provider).to_s,
|
|
41
|
+
model: attributes.fetch(:model).to_s.strip.presence || Event::UNKNOWN_MODEL,
|
|
42
|
+
token_usage: token_usage,
|
|
43
|
+
pricing_mode: pricing_mode,
|
|
44
|
+
cost: attributes[:cost],
|
|
45
|
+
tags: attributes[:tags],
|
|
46
|
+
latency_ms: attributes[:latency_ms],
|
|
47
|
+
stream: attributes[:stream] || false,
|
|
48
|
+
usage_source: attributes[:usage_source],
|
|
49
|
+
provider_response_id: attributes[:provider_response_id].to_s.strip.presence,
|
|
50
|
+
provider_project_id: attributes[:provider_project_id].to_s.strip.presence,
|
|
51
|
+
provider_api_key_id: attributes[:provider_api_key_id].to_s.strip.presence,
|
|
52
|
+
provider_workspace_id: attributes[:provider_workspace_id].to_s.strip.presence,
|
|
53
|
+
batch: batch,
|
|
54
|
+
tracked_at: attributes[:tracked_at],
|
|
55
|
+
cost_status: attributes[:cost_status],
|
|
56
|
+
pricing_snapshot: attributes[:pricing_snapshot],
|
|
57
|
+
line_items: line_items
|
|
58
|
+
)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def self.resolve_line_items(service_items, token_usage)
|
|
62
|
+
service_line_items = Array(service_items).map do |item|
|
|
63
|
+
item.is_a?(Billing::LineItem) ? item : Billing::LineItem.build(item)
|
|
64
|
+
end
|
|
65
|
+
Billing::LineItem.from_token_usage(token_usage) + service_line_items
|
|
66
|
+
end
|
|
67
|
+
|
|
25
68
|
def total_cost
|
|
26
69
|
cost&.fetch(:total_cost, nil)
|
|
27
70
|
end
|
|
@@ -35,4 +78,8 @@ module LlmCostTracker
|
|
|
35
78
|
)
|
|
36
79
|
end
|
|
37
80
|
end
|
|
81
|
+
|
|
82
|
+
class Event
|
|
83
|
+
UNKNOWN_MODEL = "unknown"
|
|
84
|
+
end
|
|
38
85
|
end
|
|
@@ -5,18 +5,18 @@ require "rails/generators/active_record"
|
|
|
5
5
|
|
|
6
6
|
module LlmCostTracker
|
|
7
7
|
module Generators
|
|
8
|
-
class
|
|
8
|
+
class AsyncIngestionGenerator < Rails::Generators::Base
|
|
9
9
|
include ActiveRecord::Generators::Migration
|
|
10
10
|
|
|
11
11
|
source_root File.expand_path("templates", __dir__)
|
|
12
12
|
|
|
13
|
-
desc "Creates the
|
|
14
|
-
"Required when config.
|
|
13
|
+
desc "Creates the async ingestion tables (llm_cost_tracker_ingestion_inbox_entries + _leases). " \
|
|
14
|
+
"Required when config.ingestion = :async."
|
|
15
15
|
|
|
16
16
|
def create_migration_file
|
|
17
17
|
migration_template(
|
|
18
|
-
"
|
|
19
|
-
"db/migrate/
|
|
18
|
+
"create_llm_cost_tracker_async_ingestion.rb.erb",
|
|
19
|
+
"db/migrate/create_llm_cost_tracker_async_ingestion.rb"
|
|
20
20
|
)
|
|
21
21
|
end
|
|
22
22
|
|
|
@@ -25,11 +25,11 @@ module LlmCostTracker
|
|
|
25
25
|
After migrating, set the following in config/initializers/llm_cost_tracker.rb:
|
|
26
26
|
|
|
27
27
|
LlmCostTracker.configure do |config|
|
|
28
|
-
config.
|
|
28
|
+
config.ingestion = :async
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
-
Without it the
|
|
32
|
-
inline. The doctor check warns about unused
|
|
31
|
+
Without it the async inbox tables stay unused and Tracker keeps writing
|
|
32
|
+
inline. The doctor check warns about unused async ingestion tables.
|
|
33
33
|
MSG
|
|
34
34
|
end
|
|
35
35
|
|
|
@@ -26,10 +26,10 @@ module LlmCostTracker
|
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def create_initializer
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
)
|
|
29
|
+
destination = "config/initializers/llm_cost_tracker.rb"
|
|
30
|
+
return if File.exist?(File.join(destination_root, destination))
|
|
31
|
+
|
|
32
|
+
template("initializer.rb.erb", destination)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
35
|
def create_prices_file
|
|
@@ -42,7 +42,6 @@ module LlmCostTracker
|
|
|
42
42
|
def mount_engine
|
|
43
43
|
return unless options[:dashboard]
|
|
44
44
|
|
|
45
|
-
add_engine_require
|
|
46
45
|
say(<<~MSG, :yellow)
|
|
47
46
|
The LLM Cost Tracker dashboard ships without authentication.
|
|
48
47
|
Mount it in config/routes.rb behind your app's admin auth, e.g.:
|
|
@@ -61,24 +60,6 @@ module LlmCostTracker
|
|
|
61
60
|
def migration_version
|
|
62
61
|
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
63
62
|
end
|
|
64
|
-
|
|
65
|
-
def add_engine_require
|
|
66
|
-
return unless File.exist?("config/application.rb")
|
|
67
|
-
|
|
68
|
-
contents = File.read("config/application.rb")
|
|
69
|
-
return if contents.include?(%(require "llm_cost_tracker/engine"))
|
|
70
|
-
|
|
71
|
-
unless contents.include?(%(require "rails/all"\n))
|
|
72
|
-
prepend_to_file("config/application.rb", %(require "llm_cost_tracker/engine"\n))
|
|
73
|
-
return
|
|
74
|
-
end
|
|
75
|
-
|
|
76
|
-
inject_into_file(
|
|
77
|
-
"config/application.rb",
|
|
78
|
-
%(require "llm_cost_tracker/engine"\n),
|
|
79
|
-
after: %(require "rails/all"\n)
|
|
80
|
-
)
|
|
81
|
-
end
|
|
82
63
|
end
|
|
83
64
|
end
|
|
84
65
|
end
|