llm_cost_tracker 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +55 -0
- data/README.md +6 -2
- data/app/assets/llm_cost_tracker/application.css +782 -801
- data/app/controllers/llm_cost_tracker/application_controller.rb +15 -3
- data/app/controllers/llm_cost_tracker/calls_controller.rb +39 -20
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +0 -3
- data/app/controllers/llm_cost_tracker/models_controller.rb +3 -1
- data/app/controllers/llm_cost_tracker/pricing_controller.rb +16 -0
- data/app/controllers/llm_cost_tracker/reconciliation_controller.rb +13 -19
- data/app/controllers/llm_cost_tracker/tags_controller.rb +3 -1
- data/app/helpers/llm_cost_tracker/application_helper.rb +16 -4
- data/app/helpers/llm_cost_tracker/chart_helper.rb +22 -6
- data/app/helpers/llm_cost_tracker/sortable_table_helper.rb +41 -0
- data/app/models/llm_cost_tracker/provider_invoice_import.rb +9 -4
- data/app/services/llm_cost_tracker/dashboard/pricing_overview.rb +95 -0
- data/app/services/llm_cost_tracker/dashboard/setup_state.rb +104 -0
- data/app/services/llm_cost_tracker/dashboard/sort.rb +9 -0
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +19 -5
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +34 -19
- data/app/views/layouts/llm_cost_tracker/application.html.erb +80 -17
- data/app/views/llm_cost_tracker/calls/index.html.erb +69 -90
- data/app/views/llm_cost_tracker/calls/show.html.erb +119 -120
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +119 -158
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +109 -108
- data/app/views/llm_cost_tracker/errors/database.html.erb +2 -2
- data/app/views/llm_cost_tracker/models/index.html.erb +39 -59
- data/app/views/llm_cost_tracker/pricing/index.html.erb +93 -0
- data/app/views/llm_cost_tracker/reconciliation/index.html.erb +49 -58
- data/app/views/llm_cost_tracker/shared/_filter_pill_date.html.erb +19 -0
- data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb +22 -0
- data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb +22 -0
- data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +23 -0
- data/app/views/llm_cost_tracker/shared/_spend_chart.html.erb +3 -13
- data/app/views/llm_cost_tracker/shared/_tag_chips.html.erb +1 -1
- data/app/views/llm_cost_tracker/shared/setup_required.html.erb +16 -15
- data/app/views/llm_cost_tracker/tags/index.html.erb +27 -32
- data/app/views/llm_cost_tracker/tags/show.html.erb +83 -102
- data/config/routes.rb +1 -0
- data/lib/llm_cost_tracker/billing/cost_status.rb +21 -25
- data/lib/llm_cost_tracker/billing/line_item.rb +15 -49
- data/lib/llm_cost_tracker/budget.rb +29 -8
- data/lib/llm_cost_tracker/{parsers → capture}/sse.rb +1 -1
- data/lib/llm_cost_tracker/capture/stream_collector.rb +34 -42
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +2 -6
- data/lib/llm_cost_tracker/configuration.rb +30 -44
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +1 -1
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +8 -8
- data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor.rb +80 -25
- data/lib/llm_cost_tracker/engine.rb +1 -2
- data/lib/llm_cost_tracker/errors.rb +3 -2
- data/lib/llm_cost_tracker/event.rb +47 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/{durable_ingestion_generator.rb → async_ingestion_generator.rb} +8 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +4 -23
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/{create_llm_cost_tracker_durable_ingestion.rb.erb → create_llm_cost_tracker_async_ingestion.rb.erb} +3 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb +6 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +14 -7
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +27 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +5 -5
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoice_imports_provider.rb.erb +36 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoices_metadata_index.rb.erb +27 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +0 -9
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoice_imports_provider_generator.rb +31 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoices_metadata_index_generator.rb +31 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +5 -2
- data/lib/llm_cost_tracker/ingestion/inbox.rb +4 -25
- data/lib/llm_cost_tracker/ingestion/pool.rb +44 -0
- data/lib/llm_cost_tracker/ingestion/worker.rb +22 -36
- data/lib/llm_cost_tracker/ingestion.rb +8 -9
- data/lib/llm_cost_tracker/integrations/anthropic.rb +46 -68
- data/lib/llm_cost_tracker/integrations/base.rb +14 -11
- data/lib/llm_cost_tracker/integrations/openai.rb +104 -131
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +27 -73
- data/lib/llm_cost_tracker/integrations.rb +14 -13
- data/lib/llm_cost_tracker/ledger/period/totals.rb +5 -3
- data/lib/llm_cost_tracker/ledger/rollups.rb +4 -13
- data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +11 -0
- data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +13 -3
- data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +11 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +0 -4
- data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb +13 -3
- data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb +13 -3
- data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb +19 -9
- data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +26 -11
- data/lib/llm_cost_tracker/ledger/store.rb +21 -18
- data/lib/llm_cost_tracker/ledger/tags/query.rb +0 -1
- data/lib/llm_cost_tracker/ledger.rb +13 -0
- data/lib/llm_cost_tracker/logging.rb +0 -4
- data/lib/llm_cost_tracker/middleware/faraday.rb +46 -17
- data/lib/llm_cost_tracker/parsers/anthropic.rb +35 -59
- data/lib/llm_cost_tracker/parsers/azure.rb +46 -0
- data/lib/llm_cost_tracker/parsers/base.rb +53 -47
- data/lib/llm_cost_tracker/parsers/gemini.rb +23 -27
- data/lib/llm_cost_tracker/parsers/openai.rb +8 -40
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +26 -49
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +19 -23
- data/lib/llm_cost_tracker/parsers.rb +29 -4
- data/lib/llm_cost_tracker/prices.json +567 -579
- data/lib/llm_cost_tracker/pricing/backfill.rb +140 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +2 -4
- data/lib/llm_cost_tracker/pricing/estimator.rb +33 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +5 -2
- data/lib/llm_cost_tracker/pricing/lookup.rb +37 -2
- data/lib/llm_cost_tracker/pricing/mode.rb +34 -4
- data/lib/llm_cost_tracker/pricing/registry.rb +0 -7
- data/lib/llm_cost_tracker/pricing/service_charges.rb +6 -10
- data/lib/llm_cost_tracker/pricing/{sync_change_printer.rb → sync/change_printer.rb} +3 -3
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +14 -2
- data/lib/llm_cost_tracker/pricing/sync.rb +1 -9
- data/lib/llm_cost_tracker/pricing/unknown.rb +5 -2
- data/lib/llm_cost_tracker/pricing.rb +71 -43
- data/lib/llm_cost_tracker/providers/anthropic/server_tools.rb +15 -0
- data/lib/llm_cost_tracker/providers/anthropic/tier_classification.rb +22 -0
- data/lib/llm_cost_tracker/providers/azure/hosts.rb +17 -0
- data/lib/llm_cost_tracker/providers/gemini/model_families.rb +17 -0
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +35 -0
- data/lib/llm_cost_tracker/providers/openai/model_families.rb +51 -0
- data/lib/llm_cost_tracker/providers/openai/service_charges.rb +157 -0
- data/lib/llm_cost_tracker/railtie.rb +3 -5
- data/lib/llm_cost_tracker/reconcile_tasks.rb +18 -21
- data/lib/llm_cost_tracker/reconciliation/diff.rb +26 -45
- data/lib/llm_cost_tracker/reconciliation/diff_result.rb +0 -4
- data/lib/llm_cost_tracker/reconciliation/importer.rb +3 -7
- data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb +10 -33
- data/lib/llm_cost_tracker/reconciliation/sources/coercion.rb +40 -0
- data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb +7 -31
- data/lib/llm_cost_tracker/report/formatter.rb +32 -19
- data/lib/llm_cost_tracker/report.rb +0 -4
- data/lib/llm_cost_tracker/retention.rb +20 -8
- data/lib/llm_cost_tracker/tags/sanitizer.rb +13 -17
- data/lib/llm_cost_tracker/token_usage.rb +4 -0
- data/lib/llm_cost_tracker/tracker.rb +33 -74
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +11 -15
- data/lib/tasks/llm_cost_tracker.rake +16 -2
- metadata +31 -12
- data/app/views/llm_cost_tracker/shared/_active_filters.html.erb +0 -16
- data/app/views/llm_cost_tracker/shared/_filters.html.erb +0 -66
- data/app/views/llm_cost_tracker/shared/_sort.html.erb +0 -13
- data/lib/llm_cost_tracker/dashboard_setup_state.rb +0 -109
- data/lib/llm_cost_tracker/ingestion/inline.rb +0 -22
- data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +0 -126
- data/lib/llm_cost_tracker/usage_capture.rb +0 -58
|
@@ -69,7 +69,6 @@ module LlmCostTracker
|
|
|
69
69
|
ensure_open!
|
|
70
70
|
capture_event(data, type: type) unless data.nil?
|
|
71
71
|
end
|
|
72
|
-
self
|
|
73
72
|
end
|
|
74
73
|
|
|
75
74
|
def usage(input_tokens:, output_tokens:, **extra)
|
|
@@ -87,7 +86,6 @@ module LlmCostTracker
|
|
|
87
86
|
output_tokens: output_tokens
|
|
88
87
|
)
|
|
89
88
|
end
|
|
90
|
-
self
|
|
91
89
|
end
|
|
92
90
|
|
|
93
91
|
def finish!(errored: false)
|
|
@@ -104,7 +102,7 @@ module LlmCostTracker
|
|
|
104
102
|
return nil if @finished || @recording
|
|
105
103
|
|
|
106
104
|
@recording = true
|
|
107
|
-
pricing_mode = Pricing.
|
|
105
|
+
pricing_mode = Pricing::Mode.normalize(@pricing_mode)
|
|
108
106
|
{
|
|
109
107
|
events: @events.dup,
|
|
110
108
|
overflowed: @overflowed,
|
|
@@ -124,17 +122,17 @@ module LlmCostTracker
|
|
|
124
122
|
def record_snapshot(snapshot, errored:)
|
|
125
123
|
save_succeeded = false
|
|
126
124
|
begin
|
|
127
|
-
|
|
128
|
-
provider_response_id =
|
|
129
|
-
|
|
125
|
+
event = build_event(snapshot)
|
|
126
|
+
provider_response_id = event.provider_response_id || snapshot[:provider_response_id]
|
|
127
|
+
event = event.with(provider_response_id: provider_response_id)
|
|
130
128
|
|
|
131
129
|
Tracker.record(
|
|
132
|
-
|
|
130
|
+
event: event,
|
|
133
131
|
latency_ms: snapshot[:latency_ms] || LlmCostTracker::Timing.elapsed_ms(@started_at),
|
|
134
|
-
pricing_mode:
|
|
132
|
+
pricing_mode: Pricing::Mode.merge(event.pricing_mode, snapshot[:pricing_mode]),
|
|
135
133
|
metadata: (errored ? { stream_errored: true } : {}).merge(snapshot[:metadata]),
|
|
136
134
|
context_tags: snapshot[:context_tags]
|
|
137
|
-
) {
|
|
135
|
+
) { save_succeeded = true }
|
|
138
136
|
ensure
|
|
139
137
|
@mutex.synchronize do
|
|
140
138
|
@finished = save_succeeded
|
|
@@ -143,27 +141,8 @@ module LlmCostTracker
|
|
|
143
141
|
end
|
|
144
142
|
end
|
|
145
143
|
|
|
146
|
-
HOST_DERIVED_MODE_TOKENS = %i[data_residency].freeze
|
|
147
|
-
STANDARD_LIKE_MODE_TOKENS = %i[standard standard_only auto default].freeze
|
|
148
|
-
private_constant :HOST_DERIVED_MODE_TOKENS, :STANDARD_LIKE_MODE_TOKENS
|
|
149
|
-
|
|
150
|
-
def pricing_mode_for(capture:, snapshot:)
|
|
151
|
-
merge_pricing_modes(capture.pricing_mode, snapshot[:pricing_mode])
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
def merge_pricing_modes(provider_mode, request_mode)
|
|
155
|
-
return Pricing.normalize_mode(request_mode) if provider_mode.to_s.strip.empty?
|
|
156
|
-
|
|
157
|
-
provider_tokens = Pricing::Mode.tokenize(provider_mode) - STANDARD_LIKE_MODE_TOKENS
|
|
158
|
-
request_host_tokens = Pricing::Mode.tokenize(request_mode || "") & HOST_DERIVED_MODE_TOKENS
|
|
159
|
-
combined = provider_tokens | request_host_tokens
|
|
160
|
-
return nil if combined.empty?
|
|
161
|
-
|
|
162
|
-
Pricing.normalize_mode(combined.join("_"))
|
|
163
|
-
end
|
|
164
|
-
|
|
165
144
|
def capture_dimensions(pricing_mode)
|
|
166
|
-
batch = @batch.nil? ?
|
|
145
|
+
batch = @batch.nil? ? Event.batch_from_pricing_mode?(pricing_mode).presence : @batch
|
|
167
146
|
{
|
|
168
147
|
provider_project_id: @provider_project_id.to_s.strip.presence,
|
|
169
148
|
provider_api_key_id: @provider_api_key_id.to_s.strip.presence,
|
|
@@ -178,18 +157,18 @@ module LlmCostTracker
|
|
|
178
157
|
raise FrozenError, "can't modify finished LlmCostTracker::Capture::StreamCollector"
|
|
179
158
|
end
|
|
180
159
|
|
|
181
|
-
def
|
|
160
|
+
def build_event(snapshot)
|
|
182
161
|
return build_from_explicit_usage(snapshot) if snapshot[:explicit_usage]
|
|
183
162
|
return build_unknown_usage(snapshot) if snapshot[:overflowed]
|
|
184
163
|
|
|
185
|
-
|
|
164
|
+
event = Parsers.find_for_provider(@provider)&.parse_stream(
|
|
186
165
|
response_status: 200,
|
|
187
166
|
events: snapshot[:events],
|
|
188
167
|
request_body: request_body_for(snapshot[:request])
|
|
189
168
|
)
|
|
190
|
-
if
|
|
191
|
-
model = present_model(
|
|
192
|
-
return
|
|
169
|
+
if event
|
|
170
|
+
model = present_model(event.model) || present_model(snapshot[:model]) || Event::UNKNOWN_MODEL
|
|
171
|
+
return event.with(provider: @provider, model: model, **snapshot.fetch(:capture_dimensions))
|
|
193
172
|
end
|
|
194
173
|
|
|
195
174
|
build_unknown_usage(snapshot)
|
|
@@ -207,15 +186,15 @@ module LlmCostTracker
|
|
|
207
186
|
return nil if value.nil?
|
|
208
187
|
|
|
209
188
|
string = value.to_s.presence
|
|
210
|
-
return nil if string.nil? || string ==
|
|
189
|
+
return nil if string.nil? || string == Event::UNKNOWN_MODEL
|
|
211
190
|
|
|
212
191
|
string
|
|
213
192
|
end
|
|
214
193
|
|
|
215
194
|
def build_from_explicit_usage(snapshot)
|
|
216
|
-
|
|
195
|
+
Event.build(
|
|
217
196
|
provider: @provider,
|
|
218
|
-
model: snapshot[:model] ||
|
|
197
|
+
model: snapshot[:model] || Event::UNKNOWN_MODEL,
|
|
219
198
|
token_usage: snapshot[:explicit_usage],
|
|
220
199
|
stream: true,
|
|
221
200
|
usage_source: :manual,
|
|
@@ -225,9 +204,9 @@ module LlmCostTracker
|
|
|
225
204
|
end
|
|
226
205
|
|
|
227
206
|
def build_unknown_usage(snapshot)
|
|
228
|
-
|
|
207
|
+
Event.build(
|
|
229
208
|
provider: @provider,
|
|
230
|
-
model: snapshot[:model] ||
|
|
209
|
+
model: snapshot[:model] || Event::UNKNOWN_MODEL,
|
|
231
210
|
token_usage: TokenUsage.build(input_tokens: 0, output_tokens: 0, total_tokens: 0),
|
|
232
211
|
stream: true,
|
|
233
212
|
usage_source: :unknown,
|
|
@@ -244,14 +223,14 @@ module LlmCostTracker
|
|
|
244
223
|
|
|
245
224
|
def capture_event(data, type:)
|
|
246
225
|
event = { event: type, data: strip_heavy_payload(data) }
|
|
247
|
-
size =
|
|
226
|
+
size = approximate_bytesize(event)
|
|
248
227
|
if @captured_bytes + size <= Capture::Stream::LIMIT_BYTES
|
|
249
|
-
@events << event
|
|
228
|
+
@events << event
|
|
250
229
|
@captured_bytes += size
|
|
251
230
|
else
|
|
252
231
|
@overflowed = true
|
|
253
232
|
end
|
|
254
|
-
rescue
|
|
233
|
+
rescue TypeError, SystemStackError
|
|
255
234
|
@overflowed = true
|
|
256
235
|
end
|
|
257
236
|
|
|
@@ -271,6 +250,19 @@ module LlmCostTracker
|
|
|
271
250
|
value
|
|
272
251
|
end
|
|
273
252
|
end
|
|
253
|
+
|
|
254
|
+
def approximate_bytesize(value)
|
|
255
|
+
case value
|
|
256
|
+
when Hash
|
|
257
|
+
value.sum { |key, nested| approximate_bytesize(key) + approximate_bytesize(nested) + 4 }
|
|
258
|
+
when Array
|
|
259
|
+
value.sum { |nested| approximate_bytesize(nested) + 2 }
|
|
260
|
+
when Numeric, true, false, nil
|
|
261
|
+
8
|
|
262
|
+
else
|
|
263
|
+
value.to_s.bytesize + 2
|
|
264
|
+
end
|
|
265
|
+
end
|
|
274
266
|
end
|
|
275
267
|
end
|
|
276
268
|
end
|
|
@@ -74,11 +74,7 @@ module LlmCostTracker
|
|
|
74
74
|
end
|
|
75
75
|
|
|
76
76
|
def capture(event)
|
|
77
|
-
raw_payload = event.try(:deep_to_h) || event.try(:to_h)
|
|
78
|
-
raw_payload ||= %i[type id model usage response message].each_with_object({}) do |key, attributes|
|
|
79
|
-
value = event.try(key)
|
|
80
|
-
attributes[key] = value unless value.nil?
|
|
81
|
-
end
|
|
77
|
+
raw_payload = event.try(:deep_to_h) || event.try(:to_h) || {}
|
|
82
78
|
payload = normalize(raw_payload)
|
|
83
79
|
type = event.try(:type) || payload["type"]
|
|
84
80
|
@collector.event(payload, type: type&.to_s)
|
|
@@ -155,7 +151,7 @@ module LlmCostTracker
|
|
|
155
151
|
next unless should_finish && active_proc.call
|
|
156
152
|
|
|
157
153
|
begin
|
|
158
|
-
finish_proc.call(false)
|
|
154
|
+
Rails.application.executor.wrap { finish_proc.call(false) }
|
|
159
155
|
rescue StandardError
|
|
160
156
|
nil
|
|
161
157
|
end
|
|
@@ -14,28 +14,29 @@ module LlmCostTracker
|
|
|
14
14
|
|
|
15
15
|
BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
|
|
16
16
|
UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
INGESTION_MODES = %i[inline async].freeze
|
|
18
|
+
SCALAR_ATTRIBUTES = %i[enabled default_tags on_budget_exceeded monthly_budget daily_budget per_call_budget
|
|
19
|
+
log_level prices_file max_tag_count max_tag_value_bytesize
|
|
20
|
+
ingestion_pool_size auto_enable_stream_usage cache_rollups
|
|
21
|
+
reconciliation_enabled].freeze
|
|
22
|
+
ENUM_ATTRIBUTES = {
|
|
20
23
|
budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
|
|
21
|
-
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
|
|
24
|
+
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn],
|
|
25
|
+
ingestion: [INGESTION_MODES, :inline]
|
|
22
26
|
}.freeze
|
|
23
27
|
DEFAULT_REDACTED_TAG_KEYS = %w[api_key access_token authorization credential password refresh_token secret].freeze
|
|
24
28
|
|
|
25
29
|
attr_reader(
|
|
26
|
-
*
|
|
30
|
+
*SCALAR_ATTRIBUTES,
|
|
27
31
|
:budget_exceeded_behavior,
|
|
28
|
-
:
|
|
32
|
+
:ingestion,
|
|
29
33
|
:instrumented_integrations,
|
|
30
34
|
:pricing_overrides,
|
|
31
35
|
:report_tag_breakdowns,
|
|
32
36
|
:redacted_tag_keys,
|
|
33
37
|
:unknown_pricing_behavior,
|
|
34
38
|
:openai_compatible_providers,
|
|
35
|
-
:reconciliation_importers
|
|
36
|
-
:reconciliation_enabled,
|
|
37
|
-
:auto_enable_stream_usage,
|
|
38
|
-
:cache_rollups
|
|
39
|
+
:reconciliation_importers
|
|
39
40
|
)
|
|
40
41
|
|
|
41
42
|
def initialize
|
|
@@ -51,6 +52,7 @@ module LlmCostTracker
|
|
|
51
52
|
@prices_file = nil
|
|
52
53
|
@max_tag_count = 50
|
|
53
54
|
@max_tag_value_bytesize = 1024
|
|
55
|
+
@ingestion_pool_size = nil
|
|
54
56
|
self.pricing_overrides = {}
|
|
55
57
|
@instrumented_integrations = Set.new
|
|
56
58
|
@report_tag_breakdowns = []
|
|
@@ -59,33 +61,13 @@ module LlmCostTracker
|
|
|
59
61
|
@reconciliation_importers = {}
|
|
60
62
|
@reconciliation_enabled = false
|
|
61
63
|
@auto_enable_stream_usage = true
|
|
62
|
-
|
|
64
|
+
self.ingestion = :inline
|
|
63
65
|
@cache_rollups = false
|
|
64
66
|
@finalized = false
|
|
65
67
|
end
|
|
66
68
|
|
|
67
|
-
def durable_ingestion=(value)
|
|
68
|
-
ensure_shared_configuration_mutable!
|
|
69
|
-
@durable_ingestion = value
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
def cache_rollups=(value)
|
|
73
|
-
ensure_shared_configuration_mutable!
|
|
74
|
-
@cache_rollups = value
|
|
75
|
-
end
|
|
76
|
-
|
|
77
|
-
def reconciliation_enabled=(value)
|
|
78
|
-
ensure_shared_configuration_mutable!
|
|
79
|
-
@reconciliation_enabled = value
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
def auto_enable_stream_usage=(value)
|
|
83
|
-
ensure_shared_configuration_mutable!
|
|
84
|
-
@auto_enable_stream_usage = value
|
|
85
|
-
end
|
|
86
|
-
|
|
87
69
|
def reconciliation_importers=(importers)
|
|
88
|
-
|
|
70
|
+
ensure_mutable!
|
|
89
71
|
raise Error, RECONCILIATION_DISABLED_MESSAGE unless @reconciliation_enabled
|
|
90
72
|
|
|
91
73
|
@reconciliation_importers = (importers || {}).to_h do |source, importer|
|
|
@@ -96,7 +78,7 @@ module LlmCostTracker
|
|
|
96
78
|
end
|
|
97
79
|
|
|
98
80
|
def register_reconciliation_importer(source, &block)
|
|
99
|
-
|
|
81
|
+
ensure_mutable!
|
|
100
82
|
raise Error, RECONCILIATION_DISABLED_MESSAGE unless @reconciliation_enabled
|
|
101
83
|
raise Error, "register_reconciliation_importer requires a block" unless block
|
|
102
84
|
|
|
@@ -107,29 +89,29 @@ module LlmCostTracker
|
|
|
107
89
|
private_constant :RECONCILIATION_DISABLED_MESSAGE
|
|
108
90
|
|
|
109
91
|
def openai_compatible_providers=(providers)
|
|
110
|
-
|
|
92
|
+
ensure_mutable!
|
|
111
93
|
@openai_compatible_providers = normalize_openai_compatible_providers(providers)
|
|
112
94
|
end
|
|
113
95
|
|
|
114
96
|
def pricing_overrides=(value)
|
|
115
|
-
|
|
97
|
+
ensure_mutable!
|
|
116
98
|
@pricing_overrides = Pricing::Registry.normalize_price_table(value || {})
|
|
117
99
|
rescue ArgumentError => e
|
|
118
100
|
raise Error, "invalid pricing_overrides: #{e.message}"
|
|
119
101
|
end
|
|
120
102
|
|
|
121
103
|
def report_tag_breakdowns=(value)
|
|
122
|
-
|
|
104
|
+
ensure_mutable!
|
|
123
105
|
@report_tag_breakdowns = Array(value).map { |key| Tags::Key.validate!(key, error_class: Error) }
|
|
124
106
|
end
|
|
125
107
|
|
|
126
108
|
def redacted_tag_keys=(value)
|
|
127
|
-
|
|
109
|
+
ensure_mutable!
|
|
128
110
|
@redacted_tag_keys = Array(value).map(&:to_s)
|
|
129
111
|
end
|
|
130
112
|
|
|
131
113
|
def instrument(*names)
|
|
132
|
-
|
|
114
|
+
ensure_mutable!
|
|
133
115
|
@instrumented_integrations.merge(normalize_instrumentation_names(names))
|
|
134
116
|
end
|
|
135
117
|
|
|
@@ -137,16 +119,16 @@ module LlmCostTracker
|
|
|
137
119
|
@instrumented_integrations.include?(name)
|
|
138
120
|
end
|
|
139
121
|
|
|
140
|
-
|
|
122
|
+
SCALAR_ATTRIBUTES.each do |name|
|
|
141
123
|
define_method("#{name}=") do |value|
|
|
142
|
-
|
|
124
|
+
ensure_mutable!
|
|
143
125
|
instance_variable_set(:"@#{name}", value)
|
|
144
126
|
end
|
|
145
127
|
end
|
|
146
128
|
|
|
147
|
-
|
|
129
|
+
ENUM_ATTRIBUTES.each do |name, (allowed, default)|
|
|
148
130
|
define_method("#{name}=") do |value|
|
|
149
|
-
|
|
131
|
+
ensure_mutable!
|
|
150
132
|
instance_variable_set(:"@#{name}", normalize_enum(name, value, allowed, default: default))
|
|
151
133
|
end
|
|
152
134
|
end
|
|
@@ -161,7 +143,11 @@ module LlmCostTracker
|
|
|
161
143
|
normalize_openai_compatible_providers(@openai_compatible_providers)
|
|
162
144
|
)
|
|
163
145
|
@finalized = true
|
|
164
|
-
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def normalized_redacted_tag_keys
|
|
149
|
+
@normalized_redacted_tag_keys ||=
|
|
150
|
+
Array(@redacted_tag_keys).map { |key| Tags::Sanitizer.normalized_key(key) }.freeze
|
|
165
151
|
end
|
|
166
152
|
|
|
167
153
|
def finalized?
|
|
@@ -196,7 +182,7 @@ module LlmCostTracker
|
|
|
196
182
|
names
|
|
197
183
|
end
|
|
198
184
|
|
|
199
|
-
def
|
|
185
|
+
def ensure_mutable!
|
|
200
186
|
return unless finalized?
|
|
201
187
|
|
|
202
188
|
raise FrozenError, "can't modify frozen LlmCostTracker::Configuration"
|
|
@@ -11,14 +11,14 @@ module LlmCostTracker
|
|
|
11
11
|
|
|
12
12
|
def call
|
|
13
13
|
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
14
|
-
return inline_check unless LlmCostTracker::Ingestion.
|
|
14
|
+
return inline_check unless LlmCostTracker::Ingestion.async?
|
|
15
15
|
|
|
16
16
|
missing = missing_parts
|
|
17
17
|
if missing.empty?
|
|
18
18
|
inbox = inbox_snapshot
|
|
19
19
|
quarantined = inbox.try(:quarantined_count).to_i
|
|
20
20
|
if quarantined.positive?
|
|
21
|
-
return Check.new(:warn, "
|
|
21
|
+
return Check.new(:warn, "async ingestion", "#{quarantined} inbox entries quarantined after retries")
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
pending_count = inbox.try(:pending_count).to_i
|
|
@@ -27,17 +27,17 @@ module LlmCostTracker
|
|
|
27
27
|
if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
|
|
28
28
|
return Check.new(
|
|
29
29
|
:warn,
|
|
30
|
-
"
|
|
30
|
+
"async ingestion",
|
|
31
31
|
"#{pending_count} inbox entries pending; oldest pending age #{pending_age.round}s"
|
|
32
32
|
)
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
return Check.new(:ok, "
|
|
35
|
+
return Check.new(:ok, "async ingestion", "inbox and ingestion lease tables available")
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
Check.new(
|
|
39
39
|
:error,
|
|
40
|
-
"
|
|
40
|
+
"async ingestion",
|
|
41
41
|
"missing #{missing.join(', ')}; see docs/upgrading.md for the recovery steps"
|
|
42
42
|
)
|
|
43
43
|
end
|
|
@@ -50,15 +50,15 @@ module LlmCostTracker
|
|
|
50
50
|
return Check.new(
|
|
51
51
|
:ok,
|
|
52
52
|
"inline ingestion",
|
|
53
|
-
"
|
|
53
|
+
"config.ingestion = :inline; events write directly to the ledger"
|
|
54
54
|
)
|
|
55
55
|
end
|
|
56
56
|
|
|
57
57
|
Check.new(
|
|
58
58
|
:warn,
|
|
59
59
|
"inline ingestion",
|
|
60
|
-
"
|
|
61
|
-
"Set config.
|
|
60
|
+
"config.ingestion = :inline but found unused async ingestion tables: #{leftovers.join(', ')}. " \
|
|
61
|
+
"Set config.ingestion = :async to keep the inbox path or drop the tables."
|
|
62
62
|
)
|
|
63
63
|
end
|
|
64
64
|
|
|
@@ -28,8 +28,6 @@ module LlmCostTracker
|
|
|
28
28
|
message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
|
|
29
29
|
"stored totals remain stable but applied rates cannot be audited"
|
|
30
30
|
Check.new(:warn, "pricing snapshot audit", message)
|
|
31
|
-
rescue StandardError
|
|
32
|
-
nil
|
|
33
31
|
end
|
|
34
32
|
end
|
|
35
33
|
end
|
|
@@ -14,35 +14,95 @@ require_relative "doctor/pricing_snapshot_drift_check"
|
|
|
14
14
|
module LlmCostTracker
|
|
15
15
|
class Doctor
|
|
16
16
|
autoload :InvoiceReconciliationCheck, "llm_cost_tracker/doctor/invoice_reconciliation_check"
|
|
17
|
+
autoload :CaptureVerifier, "llm_cost_tracker/doctor/capture_verifier"
|
|
18
|
+
|
|
19
|
+
STATUS_GLYPHS = { ok: "✓", warn: "!", error: "x" }.freeze
|
|
20
|
+
STATUS_COLORS = { ok: 32, warn: 33, error: 31 }.freeze
|
|
21
|
+
|
|
22
|
+
SECTIONS = ["Setup", "Schema", "Data integrity", "Operations"].freeze
|
|
23
|
+
|
|
24
|
+
SECTION_FOR_CHECK = {
|
|
25
|
+
"configuration" => "Setup",
|
|
26
|
+
"capture" => "Setup",
|
|
27
|
+
"active_record" => "Schema",
|
|
28
|
+
"llm_cost_tracker_calls" => "Schema",
|
|
29
|
+
"llm_cost_tracker_calls columns" => "Schema",
|
|
30
|
+
"call line items" => "Schema",
|
|
31
|
+
"call tags" => "Schema",
|
|
32
|
+
"provider invoices" => "Schema",
|
|
33
|
+
"provider invoice imports" => "Schema",
|
|
34
|
+
"cost drift" => "Data integrity",
|
|
35
|
+
"pricing snapshot drift" => "Data integrity",
|
|
36
|
+
"pricing snapshot audit" => "Data integrity",
|
|
37
|
+
"cost status" => "Data integrity",
|
|
38
|
+
"invoice reconciliation" => "Data integrity",
|
|
39
|
+
"call rollups" => "Operations",
|
|
40
|
+
"inline ingestion" => "Operations",
|
|
41
|
+
"async ingestion" => "Operations",
|
|
42
|
+
"prices" => "Operations",
|
|
43
|
+
"tracked calls" => "Operations"
|
|
44
|
+
}.freeze
|
|
45
|
+
|
|
46
|
+
private_constant :STATUS_GLYPHS, :STATUS_COLORS, :SECTIONS, :SECTION_FOR_CHECK
|
|
17
47
|
|
|
18
48
|
class << self
|
|
19
49
|
def call
|
|
20
50
|
new.checks
|
|
21
51
|
end
|
|
22
52
|
|
|
23
|
-
def report(checks = call)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
53
|
+
def report(checks = call, color: $stdout.tty?)
|
|
54
|
+
name_width = checks.map { |c| c.name.length }.max.to_i
|
|
55
|
+
|
|
56
|
+
lines = [bold("LLM Cost Tracker doctor", color), ""]
|
|
57
|
+
each_section(checks) do |section, members|
|
|
58
|
+
lines << bold(section, color)
|
|
59
|
+
members.each do |check|
|
|
60
|
+
status = paint_status("[#{STATUS_GLYPHS.fetch(check.status, check.status)}]", check.status, color)
|
|
61
|
+
lines << " #{status} #{"#{check.name}:".ljust(name_width + 1)} #{check.message}"
|
|
62
|
+
end
|
|
63
|
+
lines << ""
|
|
64
|
+
end
|
|
65
|
+
lines.pop if lines.last == ""
|
|
66
|
+
lines.join("\n")
|
|
27
67
|
end
|
|
28
68
|
|
|
29
69
|
def healthy?(checks = call)
|
|
30
70
|
checks.none? { |check| check.status == :error }
|
|
31
71
|
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def each_section(checks)
|
|
76
|
+
SECTIONS.each do |section|
|
|
77
|
+
members = checks.select { |c| (SECTION_FOR_CHECK[c.name] || "Setup") == section }
|
|
78
|
+
next if members.empty?
|
|
79
|
+
|
|
80
|
+
yield section, members
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def paint_status(text, status, color)
|
|
85
|
+
return text unless color && STATUS_COLORS.key?(status)
|
|
86
|
+
|
|
87
|
+
"\e[#{STATUS_COLORS[status]}m#{text}\e[0m"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def bold(text, color)
|
|
91
|
+
return text unless color
|
|
92
|
+
|
|
93
|
+
"\e[1m#{text}\e[0m"
|
|
94
|
+
end
|
|
32
95
|
end
|
|
33
96
|
|
|
34
97
|
def checks
|
|
35
98
|
[
|
|
36
99
|
configuration_check,
|
|
37
100
|
capture_check,
|
|
38
|
-
*
|
|
101
|
+
*LlmCostTracker::Integrations.checks,
|
|
39
102
|
active_record_check,
|
|
40
103
|
table_check,
|
|
41
104
|
column_check,
|
|
42
|
-
|
|
43
|
-
table: "llm_cost_tracker_call_line_items").call,
|
|
44
|
-
SchemaCheck.new(name: "call tags", schema: Ledger::Schema::CallTags,
|
|
45
|
-
table: "llm_cost_tracker_call_tags").call,
|
|
105
|
+
*dependent_core_schema_checks,
|
|
46
106
|
*reconciliation_schema_checks,
|
|
47
107
|
CostDriftCheck.new.call,
|
|
48
108
|
PricingSnapshotDriftCheck.new.call,
|
|
@@ -58,20 +118,23 @@ module LlmCostTracker
|
|
|
58
118
|
|
|
59
119
|
private
|
|
60
120
|
|
|
121
|
+
def dependent_core_schema_checks
|
|
122
|
+
Ledger::Schema::CORE_SCHEMAS.reject { |schema, _| schema == Ledger::Schema::Calls }.map do |schema, table|
|
|
123
|
+
SchemaCheck.new(name: table.delete_prefix("llm_cost_tracker_").tr("_", " "),
|
|
124
|
+
schema: schema, table: table).call
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
61
128
|
def reconciliation_schema_checks
|
|
62
129
|
return [] unless LlmCostTracker.reconciliation_enabled?
|
|
63
130
|
|
|
64
|
-
LlmCostTracker.const_get(:Reconciliation) # autoload reconciliation + its ledger schemas
|
|
65
131
|
Reconciliation::SCHEMA_TABLES.map do |schema, table|
|
|
66
|
-
SchemaCheck.new(name:
|
|
132
|
+
SchemaCheck.new(name: table.delete_prefix("llm_cost_tracker_").tr("_", " "),
|
|
133
|
+
schema: schema, table: table,
|
|
67
134
|
optional: false, install_command: "llm_cost_tracker:reconciliation").call
|
|
68
135
|
end.compact
|
|
69
136
|
end
|
|
70
137
|
|
|
71
|
-
def humanize_table(table)
|
|
72
|
-
table.delete_prefix("llm_cost_tracker_").tr("_", " ")
|
|
73
|
-
end
|
|
74
|
-
|
|
75
138
|
def reconciliation_invoice_check
|
|
76
139
|
return [] unless LlmCostTracker.reconciliation_enabled?
|
|
77
140
|
|
|
@@ -104,12 +167,6 @@ module LlmCostTracker
|
|
|
104
167
|
)
|
|
105
168
|
end
|
|
106
169
|
|
|
107
|
-
def integration_checks
|
|
108
|
-
LlmCostTracker::Integrations.checks.map do |check|
|
|
109
|
-
Check.new(check.status, check.name.to_s, check.message)
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
|
|
113
170
|
def active_record_check
|
|
114
171
|
return Check.new(:ok, "active_record", "available") if active_record_available?
|
|
115
172
|
|
|
@@ -209,16 +266,14 @@ module LlmCostTracker
|
|
|
209
266
|
count = snapshot.tracked_call_count.to_i
|
|
210
267
|
return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
|
|
211
268
|
|
|
212
|
-
|
|
213
|
-
latest_at = latest_at.to_time if latest_at.respond_to?(:to_time)
|
|
214
|
-
latest = latest_at&.utc&.iso8601
|
|
269
|
+
latest = snapshot.latest_tracked_at.to_time.utc.iso8601
|
|
215
270
|
Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
|
|
216
271
|
end
|
|
217
272
|
|
|
218
273
|
def active_record_available?
|
|
219
274
|
LlmCostTracker::Call.connection
|
|
220
275
|
true
|
|
221
|
-
rescue
|
|
276
|
+
rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
|
|
222
277
|
false
|
|
223
278
|
end
|
|
224
279
|
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
require "rails"
|
|
4
4
|
require_relative "../llm_cost_tracker"
|
|
5
5
|
require_relative "assets"
|
|
6
|
-
require_relative "dashboard_setup_state"
|
|
7
6
|
require "rack/files"
|
|
8
7
|
|
|
9
8
|
module LlmCostTracker
|
|
@@ -15,7 +14,7 @@ module LlmCostTracker
|
|
|
15
14
|
end
|
|
16
15
|
|
|
17
16
|
initializer "llm_cost_tracker.dashboard_setup_state" do |app|
|
|
18
|
-
app.reloader.to_prepare { LlmCostTracker::
|
|
17
|
+
app.reloader.to_prepare { LlmCostTracker::Dashboard::SetupState.reset! }
|
|
19
18
|
end
|
|
20
19
|
end
|
|
21
20
|
end
|
|
@@ -6,13 +6,14 @@ module LlmCostTracker
|
|
|
6
6
|
class InvalidFilterError < Error; end
|
|
7
7
|
|
|
8
8
|
class BudgetExceededError < Error
|
|
9
|
-
attr_reader :total, :budget, :budget_type, :last_event
|
|
9
|
+
attr_reader :total, :budget, :budget_type, :last_event, :stage
|
|
10
10
|
|
|
11
|
-
def initialize(budget:, budget_type:, total:, last_event: nil)
|
|
11
|
+
def initialize(budget:, budget_type:, total:, last_event: nil, stage: :post_spend)
|
|
12
12
|
@total = total
|
|
13
13
|
@budget = budget
|
|
14
14
|
@budget_type = budget_type
|
|
15
15
|
@last_event = last_event
|
|
16
|
+
@stage = stage
|
|
16
17
|
|
|
17
18
|
super(
|
|
18
19
|
"LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded: " \
|