llm_cost_tracker 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +55 -0
  3. data/README.md +6 -2
  4. data/app/assets/llm_cost_tracker/application.css +782 -801
  5. data/app/controllers/llm_cost_tracker/application_controller.rb +15 -3
  6. data/app/controllers/llm_cost_tracker/calls_controller.rb +39 -20
  7. data/app/controllers/llm_cost_tracker/dashboard_controller.rb +0 -3
  8. data/app/controllers/llm_cost_tracker/models_controller.rb +3 -1
  9. data/app/controllers/llm_cost_tracker/pricing_controller.rb +16 -0
  10. data/app/controllers/llm_cost_tracker/reconciliation_controller.rb +13 -19
  11. data/app/controllers/llm_cost_tracker/tags_controller.rb +3 -1
  12. data/app/helpers/llm_cost_tracker/application_helper.rb +16 -4
  13. data/app/helpers/llm_cost_tracker/chart_helper.rb +22 -6
  14. data/app/helpers/llm_cost_tracker/sortable_table_helper.rb +41 -0
  15. data/app/models/llm_cost_tracker/provider_invoice_import.rb +9 -4
  16. data/app/services/llm_cost_tracker/dashboard/pricing_overview.rb +95 -0
  17. data/app/services/llm_cost_tracker/dashboard/setup_state.rb +104 -0
  18. data/app/services/llm_cost_tracker/dashboard/sort.rb +9 -0
  19. data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +19 -5
  20. data/app/services/llm_cost_tracker/dashboard/top_models.rb +34 -19
  21. data/app/views/layouts/llm_cost_tracker/application.html.erb +80 -17
  22. data/app/views/llm_cost_tracker/calls/index.html.erb +69 -90
  23. data/app/views/llm_cost_tracker/calls/show.html.erb +119 -120
  24. data/app/views/llm_cost_tracker/dashboard/index.html.erb +119 -158
  25. data/app/views/llm_cost_tracker/data_quality/index.html.erb +109 -108
  26. data/app/views/llm_cost_tracker/errors/database.html.erb +2 -2
  27. data/app/views/llm_cost_tracker/models/index.html.erb +39 -59
  28. data/app/views/llm_cost_tracker/pricing/index.html.erb +93 -0
  29. data/app/views/llm_cost_tracker/reconciliation/index.html.erb +49 -58
  30. data/app/views/llm_cost_tracker/shared/_filter_pill_date.html.erb +19 -0
  31. data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb +22 -0
  32. data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb +22 -0
  33. data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +23 -0
  34. data/app/views/llm_cost_tracker/shared/_spend_chart.html.erb +3 -13
  35. data/app/views/llm_cost_tracker/shared/_tag_chips.html.erb +1 -1
  36. data/app/views/llm_cost_tracker/shared/setup_required.html.erb +16 -15
  37. data/app/views/llm_cost_tracker/tags/index.html.erb +27 -32
  38. data/app/views/llm_cost_tracker/tags/show.html.erb +83 -102
  39. data/config/routes.rb +1 -0
  40. data/lib/llm_cost_tracker/billing/cost_status.rb +21 -25
  41. data/lib/llm_cost_tracker/billing/line_item.rb +15 -49
  42. data/lib/llm_cost_tracker/budget.rb +29 -8
  43. data/lib/llm_cost_tracker/{parsers → capture}/sse.rb +1 -1
  44. data/lib/llm_cost_tracker/capture/stream_collector.rb +34 -42
  45. data/lib/llm_cost_tracker/capture/stream_tracker.rb +2 -6
  46. data/lib/llm_cost_tracker/configuration.rb +30 -44
  47. data/lib/llm_cost_tracker/doctor/capture_verifier.rb +1 -1
  48. data/lib/llm_cost_tracker/doctor/ingestion_check.rb +8 -8
  49. data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +0 -2
  50. data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +0 -2
  51. data/lib/llm_cost_tracker/doctor.rb +80 -25
  52. data/lib/llm_cost_tracker/engine.rb +1 -2
  53. data/lib/llm_cost_tracker/errors.rb +3 -2
  54. data/lib/llm_cost_tracker/event.rb +47 -0
  55. data/lib/llm_cost_tracker/generators/llm_cost_tracker/{durable_ingestion_generator.rb → async_ingestion_generator.rb} +8 -8
  56. data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +4 -23
  57. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/{create_llm_cost_tracker_durable_ingestion.rb.erb → create_llm_cost_tracker_async_ingestion.rb.erb} +3 -3
  58. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb +6 -1
  59. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +14 -7
  60. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +27 -8
  61. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +5 -5
  62. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoice_imports_provider.rb.erb +36 -0
  63. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoices_metadata_index.rb.erb +27 -0
  64. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +0 -9
  65. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoice_imports_provider_generator.rb +31 -0
  66. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoices_metadata_index_generator.rb +31 -0
  67. data/lib/llm_cost_tracker/ingestion/batch.rb +5 -2
  68. data/lib/llm_cost_tracker/ingestion/inbox.rb +4 -25
  69. data/lib/llm_cost_tracker/ingestion/pool.rb +44 -0
  70. data/lib/llm_cost_tracker/ingestion/worker.rb +22 -36
  71. data/lib/llm_cost_tracker/ingestion.rb +8 -9
  72. data/lib/llm_cost_tracker/integrations/anthropic.rb +46 -68
  73. data/lib/llm_cost_tracker/integrations/base.rb +14 -11
  74. data/lib/llm_cost_tracker/integrations/openai.rb +104 -131
  75. data/lib/llm_cost_tracker/integrations/ruby_llm.rb +27 -73
  76. data/lib/llm_cost_tracker/integrations.rb +14 -13
  77. data/lib/llm_cost_tracker/ledger/period/totals.rb +5 -3
  78. data/lib/llm_cost_tracker/ledger/rollups.rb +4 -13
  79. data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +11 -0
  80. data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +13 -3
  81. data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +11 -0
  82. data/lib/llm_cost_tracker/ledger/schema/calls.rb +0 -4
  83. data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb +13 -3
  84. data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb +13 -3
  85. data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb +19 -9
  86. data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +26 -11
  87. data/lib/llm_cost_tracker/ledger/store.rb +21 -18
  88. data/lib/llm_cost_tracker/ledger/tags/query.rb +0 -1
  89. data/lib/llm_cost_tracker/ledger.rb +13 -0
  90. data/lib/llm_cost_tracker/logging.rb +0 -4
  91. data/lib/llm_cost_tracker/middleware/faraday.rb +46 -17
  92. data/lib/llm_cost_tracker/parsers/anthropic.rb +35 -59
  93. data/lib/llm_cost_tracker/parsers/azure.rb +46 -0
  94. data/lib/llm_cost_tracker/parsers/base.rb +53 -47
  95. data/lib/llm_cost_tracker/parsers/gemini.rb +23 -27
  96. data/lib/llm_cost_tracker/parsers/openai.rb +8 -40
  97. data/lib/llm_cost_tracker/parsers/openai_compatible.rb +26 -49
  98. data/lib/llm_cost_tracker/parsers/openai_usage.rb +19 -23
  99. data/lib/llm_cost_tracker/parsers.rb +29 -4
  100. data/lib/llm_cost_tracker/prices.json +567 -579
  101. data/lib/llm_cost_tracker/pricing/backfill.rb +140 -0
  102. data/lib/llm_cost_tracker/pricing/effective_prices.rb +2 -4
  103. data/lib/llm_cost_tracker/pricing/estimator.rb +33 -0
  104. data/lib/llm_cost_tracker/pricing/explainer.rb +5 -2
  105. data/lib/llm_cost_tracker/pricing/lookup.rb +37 -2
  106. data/lib/llm_cost_tracker/pricing/mode.rb +34 -4
  107. data/lib/llm_cost_tracker/pricing/registry.rb +0 -7
  108. data/lib/llm_cost_tracker/pricing/service_charges.rb +6 -10
  109. data/lib/llm_cost_tracker/pricing/{sync_change_printer.rb → sync/change_printer.rb} +3 -3
  110. data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +14 -2
  111. data/lib/llm_cost_tracker/pricing/sync.rb +1 -9
  112. data/lib/llm_cost_tracker/pricing/unknown.rb +5 -2
  113. data/lib/llm_cost_tracker/pricing.rb +71 -43
  114. data/lib/llm_cost_tracker/providers/anthropic/server_tools.rb +15 -0
  115. data/lib/llm_cost_tracker/providers/anthropic/tier_classification.rb +22 -0
  116. data/lib/llm_cost_tracker/providers/azure/hosts.rb +17 -0
  117. data/lib/llm_cost_tracker/providers/gemini/model_families.rb +17 -0
  118. data/lib/llm_cost_tracker/providers/openai/hosts.rb +35 -0
  119. data/lib/llm_cost_tracker/providers/openai/model_families.rb +51 -0
  120. data/lib/llm_cost_tracker/providers/openai/service_charges.rb +157 -0
  121. data/lib/llm_cost_tracker/railtie.rb +3 -5
  122. data/lib/llm_cost_tracker/reconcile_tasks.rb +18 -21
  123. data/lib/llm_cost_tracker/reconciliation/diff.rb +26 -45
  124. data/lib/llm_cost_tracker/reconciliation/diff_result.rb +0 -4
  125. data/lib/llm_cost_tracker/reconciliation/importer.rb +3 -7
  126. data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb +10 -33
  127. data/lib/llm_cost_tracker/reconciliation/sources/coercion.rb +40 -0
  128. data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb +7 -31
  129. data/lib/llm_cost_tracker/report/formatter.rb +32 -19
  130. data/lib/llm_cost_tracker/report.rb +0 -4
  131. data/lib/llm_cost_tracker/retention.rb +20 -8
  132. data/lib/llm_cost_tracker/tags/sanitizer.rb +13 -17
  133. data/lib/llm_cost_tracker/token_usage.rb +4 -0
  134. data/lib/llm_cost_tracker/tracker.rb +33 -74
  135. data/lib/llm_cost_tracker/version.rb +1 -1
  136. data/lib/llm_cost_tracker.rb +11 -15
  137. data/lib/tasks/llm_cost_tracker.rake +16 -2
  138. metadata +31 -12
  139. data/app/views/llm_cost_tracker/shared/_active_filters.html.erb +0 -16
  140. data/app/views/llm_cost_tracker/shared/_filters.html.erb +0 -66
  141. data/app/views/llm_cost_tracker/shared/_sort.html.erb +0 -13
  142. data/lib/llm_cost_tracker/dashboard_setup_state.rb +0 -109
  143. data/lib/llm_cost_tracker/ingestion/inline.rb +0 -22
  144. data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +0 -126
  145. data/lib/llm_cost_tracker/usage_capture.rb +0 -58
@@ -69,7 +69,6 @@ module LlmCostTracker
69
69
  ensure_open!
70
70
  capture_event(data, type: type) unless data.nil?
71
71
  end
72
- self
73
72
  end
74
73
 
75
74
  def usage(input_tokens:, output_tokens:, **extra)
@@ -87,7 +86,6 @@ module LlmCostTracker
87
86
  output_tokens: output_tokens
88
87
  )
89
88
  end
90
- self
91
89
  end
92
90
 
93
91
  def finish!(errored: false)
@@ -104,7 +102,7 @@ module LlmCostTracker
104
102
  return nil if @finished || @recording
105
103
 
106
104
  @recording = true
107
- pricing_mode = Pricing.normalize_mode(@pricing_mode)
105
+ pricing_mode = Pricing::Mode.normalize(@pricing_mode)
108
106
  {
109
107
  events: @events.dup,
110
108
  overflowed: @overflowed,
@@ -124,17 +122,17 @@ module LlmCostTracker
124
122
  def record_snapshot(snapshot, errored:)
125
123
  save_succeeded = false
126
124
  begin
127
- capture = build_usage_capture(snapshot)
128
- provider_response_id = capture.provider_response_id || snapshot[:provider_response_id]
129
- capture = capture.with(provider_response_id: provider_response_id)
125
+ event = build_event(snapshot)
126
+ provider_response_id = event.provider_response_id || snapshot[:provider_response_id]
127
+ event = event.with(provider_response_id: provider_response_id)
130
128
 
131
129
  Tracker.record(
132
- capture: capture,
130
+ event: event,
133
131
  latency_ms: snapshot[:latency_ms] || LlmCostTracker::Timing.elapsed_ms(@started_at),
134
- pricing_mode: pricing_mode_for(capture: capture, snapshot: snapshot),
132
+ pricing_mode: Pricing::Mode.merge(event.pricing_mode, snapshot[:pricing_mode]),
135
133
  metadata: (errored ? { stream_errored: true } : {}).merge(snapshot[:metadata]),
136
134
  context_tags: snapshot[:context_tags]
137
- ) { |stage| save_succeeded = true if stage == :after_save }
135
+ ) { save_succeeded = true }
138
136
  ensure
139
137
  @mutex.synchronize do
140
138
  @finished = save_succeeded
@@ -143,27 +141,8 @@ module LlmCostTracker
143
141
  end
144
142
  end
145
143
 
146
- HOST_DERIVED_MODE_TOKENS = %i[data_residency].freeze
147
- STANDARD_LIKE_MODE_TOKENS = %i[standard standard_only auto default].freeze
148
- private_constant :HOST_DERIVED_MODE_TOKENS, :STANDARD_LIKE_MODE_TOKENS
149
-
150
- def pricing_mode_for(capture:, snapshot:)
151
- merge_pricing_modes(capture.pricing_mode, snapshot[:pricing_mode])
152
- end
153
-
154
- def merge_pricing_modes(provider_mode, request_mode)
155
- return Pricing.normalize_mode(request_mode) if provider_mode.to_s.strip.empty?
156
-
157
- provider_tokens = Pricing::Mode.tokenize(provider_mode) - STANDARD_LIKE_MODE_TOKENS
158
- request_host_tokens = Pricing::Mode.tokenize(request_mode || "") & HOST_DERIVED_MODE_TOKENS
159
- combined = provider_tokens | request_host_tokens
160
- return nil if combined.empty?
161
-
162
- Pricing.normalize_mode(combined.join("_"))
163
- end
164
-
165
144
  def capture_dimensions(pricing_mode)
166
- batch = @batch.nil? ? UsageCapture.batch_from_pricing_mode?(pricing_mode).presence : @batch
145
+ batch = @batch.nil? ? Event.batch_from_pricing_mode?(pricing_mode).presence : @batch
167
146
  {
168
147
  provider_project_id: @provider_project_id.to_s.strip.presence,
169
148
  provider_api_key_id: @provider_api_key_id.to_s.strip.presence,
@@ -178,18 +157,18 @@ module LlmCostTracker
178
157
  raise FrozenError, "can't modify finished LlmCostTracker::Capture::StreamCollector"
179
158
  end
180
159
 
181
- def build_usage_capture(snapshot)
160
+ def build_event(snapshot)
182
161
  return build_from_explicit_usage(snapshot) if snapshot[:explicit_usage]
183
162
  return build_unknown_usage(snapshot) if snapshot[:overflowed]
184
163
 
185
- capture = Parsers.find_for_provider(@provider)&.parse_stream(
164
+ event = Parsers.find_for_provider(@provider)&.parse_stream(
186
165
  response_status: 200,
187
166
  events: snapshot[:events],
188
167
  request_body: request_body_for(snapshot[:request])
189
168
  )
190
- if capture
191
- model = present_model(capture.model) || present_model(snapshot[:model]) || UsageCapture::UNKNOWN_MODEL
192
- return capture.with(provider: @provider, model: model, **snapshot.fetch(:capture_dimensions))
169
+ if event
170
+ model = present_model(event.model) || present_model(snapshot[:model]) || Event::UNKNOWN_MODEL
171
+ return event.with(provider: @provider, model: model, **snapshot.fetch(:capture_dimensions))
193
172
  end
194
173
 
195
174
  build_unknown_usage(snapshot)
@@ -207,15 +186,15 @@ module LlmCostTracker
207
186
  return nil if value.nil?
208
187
 
209
188
  string = value.to_s.presence
210
- return nil if string.nil? || string == "unknown"
189
+ return nil if string.nil? || string == Event::UNKNOWN_MODEL
211
190
 
212
191
  string
213
192
  end
214
193
 
215
194
  def build_from_explicit_usage(snapshot)
216
- UsageCapture.build(
195
+ Event.build(
217
196
  provider: @provider,
218
- model: snapshot[:model] || UsageCapture::UNKNOWN_MODEL,
197
+ model: snapshot[:model] || Event::UNKNOWN_MODEL,
219
198
  token_usage: snapshot[:explicit_usage],
220
199
  stream: true,
221
200
  usage_source: :manual,
@@ -225,9 +204,9 @@ module LlmCostTracker
225
204
  end
226
205
 
227
206
  def build_unknown_usage(snapshot)
228
- UsageCapture.build(
207
+ Event.build(
229
208
  provider: @provider,
230
- model: snapshot[:model] || UsageCapture::UNKNOWN_MODEL,
209
+ model: snapshot[:model] || Event::UNKNOWN_MODEL,
231
210
  token_usage: TokenUsage.build(input_tokens: 0, output_tokens: 0, total_tokens: 0),
232
211
  stream: true,
233
212
  usage_source: :unknown,
@@ -244,14 +223,14 @@ module LlmCostTracker
244
223
 
245
224
  def capture_event(data, type:)
246
225
  event = { event: type, data: strip_heavy_payload(data) }
247
- size = JSON.generate(event).bytesize
226
+ size = approximate_bytesize(event)
248
227
  if @captured_bytes + size <= Capture::Stream::LIMIT_BYTES
249
- @events << event.deep_dup
228
+ @events << event
250
229
  @captured_bytes += size
251
230
  else
252
231
  @overflowed = true
253
232
  end
254
- rescue JSON::JSONError, TypeError, SystemStackError
233
+ rescue TypeError, SystemStackError
255
234
  @overflowed = true
256
235
  end
257
236
 
@@ -271,6 +250,19 @@ module LlmCostTracker
271
250
  value
272
251
  end
273
252
  end
253
+
254
+ def approximate_bytesize(value)
255
+ case value
256
+ when Hash
257
+ value.sum { |key, nested| approximate_bytesize(key) + approximate_bytesize(nested) + 4 }
258
+ when Array
259
+ value.sum { |nested| approximate_bytesize(nested) + 2 }
260
+ when Numeric, true, false, nil
261
+ 8
262
+ else
263
+ value.to_s.bytesize + 2
264
+ end
265
+ end
274
266
  end
275
267
  end
276
268
  end
@@ -74,11 +74,7 @@ module LlmCostTracker
74
74
  end
75
75
 
76
76
  def capture(event)
77
- raw_payload = event.try(:deep_to_h) || event.try(:to_h)
78
- raw_payload ||= %i[type id model usage response message].each_with_object({}) do |key, attributes|
79
- value = event.try(key)
80
- attributes[key] = value unless value.nil?
81
- end
77
+ raw_payload = event.try(:deep_to_h) || event.try(:to_h) || {}
82
78
  payload = normalize(raw_payload)
83
79
  type = event.try(:type) || payload["type"]
84
80
  @collector.event(payload, type: type&.to_s)
@@ -155,7 +151,7 @@ module LlmCostTracker
155
151
  next unless should_finish && active_proc.call
156
152
 
157
153
  begin
158
- finish_proc.call(false)
154
+ Rails.application.executor.wrap { finish_proc.call(false) }
159
155
  rescue StandardError
160
156
  nil
161
157
  end
@@ -14,28 +14,29 @@ module LlmCostTracker
14
14
 
15
15
  BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
16
16
  UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
17
- SHARED_SCALAR_ATTRIBUTES = %i[enabled default_tags on_budget_exceeded monthly_budget daily_budget per_call_budget
18
- log_level prices_file max_tag_count max_tag_value_bytesize].freeze
19
- SHARED_ENUM_ATTRIBUTES = {
17
+ INGESTION_MODES = %i[inline async].freeze
18
+ SCALAR_ATTRIBUTES = %i[enabled default_tags on_budget_exceeded monthly_budget daily_budget per_call_budget
19
+ log_level prices_file max_tag_count max_tag_value_bytesize
20
+ ingestion_pool_size auto_enable_stream_usage cache_rollups
21
+ reconciliation_enabled].freeze
22
+ ENUM_ATTRIBUTES = {
20
23
  budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
21
- unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
24
+ unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn],
25
+ ingestion: [INGESTION_MODES, :inline]
22
26
  }.freeze
23
27
  DEFAULT_REDACTED_TAG_KEYS = %w[api_key access_token authorization credential password refresh_token secret].freeze
24
28
 
25
29
  attr_reader(
26
- *SHARED_SCALAR_ATTRIBUTES,
30
+ *SCALAR_ATTRIBUTES,
27
31
  :budget_exceeded_behavior,
28
- :durable_ingestion,
32
+ :ingestion,
29
33
  :instrumented_integrations,
30
34
  :pricing_overrides,
31
35
  :report_tag_breakdowns,
32
36
  :redacted_tag_keys,
33
37
  :unknown_pricing_behavior,
34
38
  :openai_compatible_providers,
35
- :reconciliation_importers,
36
- :reconciliation_enabled,
37
- :auto_enable_stream_usage,
38
- :cache_rollups
39
+ :reconciliation_importers
39
40
  )
40
41
 
41
42
  def initialize
@@ -51,6 +52,7 @@ module LlmCostTracker
51
52
  @prices_file = nil
52
53
  @max_tag_count = 50
53
54
  @max_tag_value_bytesize = 1024
55
+ @ingestion_pool_size = nil
54
56
  self.pricing_overrides = {}
55
57
  @instrumented_integrations = Set.new
56
58
  @report_tag_breakdowns = []
@@ -59,33 +61,13 @@ module LlmCostTracker
59
61
  @reconciliation_importers = {}
60
62
  @reconciliation_enabled = false
61
63
  @auto_enable_stream_usage = true
62
- @durable_ingestion = false
64
+ self.ingestion = :inline
63
65
  @cache_rollups = false
64
66
  @finalized = false
65
67
  end
66
68
 
67
- def durable_ingestion=(value)
68
- ensure_shared_configuration_mutable!
69
- @durable_ingestion = value
70
- end
71
-
72
- def cache_rollups=(value)
73
- ensure_shared_configuration_mutable!
74
- @cache_rollups = value
75
- end
76
-
77
- def reconciliation_enabled=(value)
78
- ensure_shared_configuration_mutable!
79
- @reconciliation_enabled = value
80
- end
81
-
82
- def auto_enable_stream_usage=(value)
83
- ensure_shared_configuration_mutable!
84
- @auto_enable_stream_usage = value
85
- end
86
-
87
69
  def reconciliation_importers=(importers)
88
- ensure_shared_configuration_mutable!
70
+ ensure_mutable!
89
71
  raise Error, RECONCILIATION_DISABLED_MESSAGE unless @reconciliation_enabled
90
72
 
91
73
  @reconciliation_importers = (importers || {}).to_h do |source, importer|
@@ -96,7 +78,7 @@ module LlmCostTracker
96
78
  end
97
79
 
98
80
  def register_reconciliation_importer(source, &block)
99
- ensure_shared_configuration_mutable!
81
+ ensure_mutable!
100
82
  raise Error, RECONCILIATION_DISABLED_MESSAGE unless @reconciliation_enabled
101
83
  raise Error, "register_reconciliation_importer requires a block" unless block
102
84
 
@@ -107,29 +89,29 @@ module LlmCostTracker
107
89
  private_constant :RECONCILIATION_DISABLED_MESSAGE
108
90
 
109
91
  def openai_compatible_providers=(providers)
110
- ensure_shared_configuration_mutable!
92
+ ensure_mutable!
111
93
  @openai_compatible_providers = normalize_openai_compatible_providers(providers)
112
94
  end
113
95
 
114
96
  def pricing_overrides=(value)
115
- ensure_shared_configuration_mutable!
97
+ ensure_mutable!
116
98
  @pricing_overrides = Pricing::Registry.normalize_price_table(value || {})
117
99
  rescue ArgumentError => e
118
100
  raise Error, "invalid pricing_overrides: #{e.message}"
119
101
  end
120
102
 
121
103
  def report_tag_breakdowns=(value)
122
- ensure_shared_configuration_mutable!
104
+ ensure_mutable!
123
105
  @report_tag_breakdowns = Array(value).map { |key| Tags::Key.validate!(key, error_class: Error) }
124
106
  end
125
107
 
126
108
  def redacted_tag_keys=(value)
127
- ensure_shared_configuration_mutable!
109
+ ensure_mutable!
128
110
  @redacted_tag_keys = Array(value).map(&:to_s)
129
111
  end
130
112
 
131
113
  def instrument(*names)
132
- ensure_shared_configuration_mutable!
114
+ ensure_mutable!
133
115
  @instrumented_integrations.merge(normalize_instrumentation_names(names))
134
116
  end
135
117
 
@@ -137,16 +119,16 @@ module LlmCostTracker
137
119
  @instrumented_integrations.include?(name)
138
120
  end
139
121
 
140
- SHARED_SCALAR_ATTRIBUTES.each do |name|
122
+ SCALAR_ATTRIBUTES.each do |name|
141
123
  define_method("#{name}=") do |value|
142
- ensure_shared_configuration_mutable!
124
+ ensure_mutable!
143
125
  instance_variable_set(:"@#{name}", value)
144
126
  end
145
127
  end
146
128
 
147
- SHARED_ENUM_ATTRIBUTES.each do |name, (allowed, default)|
129
+ ENUM_ATTRIBUTES.each do |name, (allowed, default)|
148
130
  define_method("#{name}=") do |value|
149
- ensure_shared_configuration_mutable!
131
+ ensure_mutable!
150
132
  instance_variable_set(:"@#{name}", normalize_enum(name, value, allowed, default: default))
151
133
  end
152
134
  end
@@ -161,7 +143,11 @@ module LlmCostTracker
161
143
  normalize_openai_compatible_providers(@openai_compatible_providers)
162
144
  )
163
145
  @finalized = true
164
- self
146
+ end
147
+
148
+ def normalized_redacted_tag_keys
149
+ @normalized_redacted_tag_keys ||=
150
+ Array(@redacted_tag_keys).map { |key| Tags::Sanitizer.normalized_key(key) }.freeze
165
151
  end
166
152
 
167
153
  def finalized?
@@ -196,7 +182,7 @@ module LlmCostTracker
196
182
  names
197
183
  end
198
184
 
199
- def ensure_shared_configuration_mutable!
185
+ def ensure_mutable!
200
186
  return unless finalized?
201
187
 
202
188
  raise FrozenError, "can't modify frozen LlmCostTracker::Configuration"
@@ -47,7 +47,7 @@ module LlmCostTracker
47
47
  end
48
48
 
49
49
  LlmCostTracker::Integrations.checks.map do |check|
50
- Check.new(check.status, "sdk integration #{check.name}", check.message)
50
+ check.with(name: "sdk integration #{check.name}")
51
51
  end
52
52
  end
53
53
 
@@ -11,14 +11,14 @@ module LlmCostTracker
11
11
 
12
12
  def call
13
13
  return unless Probe.table_exists?("llm_cost_tracker_calls")
14
- return inline_check unless LlmCostTracker::Ingestion.durable?
14
+ return inline_check unless LlmCostTracker::Ingestion.async?
15
15
 
16
16
  missing = missing_parts
17
17
  if missing.empty?
18
18
  inbox = inbox_snapshot
19
19
  quarantined = inbox.try(:quarantined_count).to_i
20
20
  if quarantined.positive?
21
- return Check.new(:warn, "durable ingestion", "#{quarantined} inbox entries quarantined after retries")
21
+ return Check.new(:warn, "async ingestion", "#{quarantined} inbox entries quarantined after retries")
22
22
  end
23
23
 
24
24
  pending_count = inbox.try(:pending_count).to_i
@@ -27,17 +27,17 @@ module LlmCostTracker
27
27
  if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
28
28
  return Check.new(
29
29
  :warn,
30
- "durable ingestion",
30
+ "async ingestion",
31
31
  "#{pending_count} inbox entries pending; oldest pending age #{pending_age.round}s"
32
32
  )
33
33
  end
34
34
 
35
- return Check.new(:ok, "durable ingestion", "inbox and ingestion lease tables available")
35
+ return Check.new(:ok, "async ingestion", "inbox and ingestion lease tables available")
36
36
  end
37
37
 
38
38
  Check.new(
39
39
  :error,
40
- "durable ingestion",
40
+ "async ingestion",
41
41
  "missing #{missing.join(', ')}; see docs/upgrading.md for the recovery steps"
42
42
  )
43
43
  end
@@ -50,15 +50,15 @@ module LlmCostTracker
50
50
  return Check.new(
51
51
  :ok,
52
52
  "inline ingestion",
53
- "durable_ingestion=false; events write directly to the ledger"
53
+ "config.ingestion = :inline; events write directly to the ledger"
54
54
  )
55
55
  end
56
56
 
57
57
  Check.new(
58
58
  :warn,
59
59
  "inline ingestion",
60
- "durable_ingestion=false but found unused durable ingestion tables: #{leftovers.join(', ')}. " \
61
- "Set config.durable_ingestion = true to keep the durable inbox path or drop the tables."
60
+ "config.ingestion = :inline but found unused async ingestion tables: #{leftovers.join(', ')}. " \
61
+ "Set config.ingestion = :async to keep the inbox path or drop the tables."
62
62
  )
63
63
  end
64
64
 
@@ -28,8 +28,6 @@ module LlmCostTracker
28
28
  message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
29
29
  "stored totals remain stable but applied rates cannot be audited"
30
30
  Check.new(:warn, "pricing snapshot audit", message)
31
- rescue StandardError
32
- nil
33
31
  end
34
32
  end
35
33
  end
@@ -14,8 +14,6 @@ module LlmCostTracker
14
14
  return unless LlmCostTracker::Call.where(cost_status: nil).exists?
15
15
 
16
16
  Check.new(:warn, "cost status", "legacy rows without cost_status remain; new rows will populate it")
17
- rescue StandardError
18
- nil
19
17
  end
20
18
  end
21
19
  end
@@ -14,35 +14,95 @@ require_relative "doctor/pricing_snapshot_drift_check"
14
14
  module LlmCostTracker
15
15
  class Doctor
16
16
  autoload :InvoiceReconciliationCheck, "llm_cost_tracker/doctor/invoice_reconciliation_check"
17
+ autoload :CaptureVerifier, "llm_cost_tracker/doctor/capture_verifier"
18
+
19
+ STATUS_GLYPHS = { ok: "✓", warn: "!", error: "x" }.freeze
20
+ STATUS_COLORS = { ok: 32, warn: 33, error: 31 }.freeze
21
+
22
+ SECTIONS = ["Setup", "Schema", "Data integrity", "Operations"].freeze
23
+
24
+ SECTION_FOR_CHECK = {
25
+ "configuration" => "Setup",
26
+ "capture" => "Setup",
27
+ "active_record" => "Schema",
28
+ "llm_cost_tracker_calls" => "Schema",
29
+ "llm_cost_tracker_calls columns" => "Schema",
30
+ "call line items" => "Schema",
31
+ "call tags" => "Schema",
32
+ "provider invoices" => "Schema",
33
+ "provider invoice imports" => "Schema",
34
+ "cost drift" => "Data integrity",
35
+ "pricing snapshot drift" => "Data integrity",
36
+ "pricing snapshot audit" => "Data integrity",
37
+ "cost status" => "Data integrity",
38
+ "invoice reconciliation" => "Data integrity",
39
+ "call rollups" => "Operations",
40
+ "inline ingestion" => "Operations",
41
+ "async ingestion" => "Operations",
42
+ "prices" => "Operations",
43
+ "tracked calls" => "Operations"
44
+ }.freeze
45
+
46
+ private_constant :STATUS_GLYPHS, :STATUS_COLORS, :SECTIONS, :SECTION_FOR_CHECK
17
47
 
18
48
  class << self
19
49
  def call
20
50
  new.checks
21
51
  end
22
52
 
23
- def report(checks = call)
24
- (["LLM Cost Tracker doctor"] + checks.map do |check|
25
- "[#{check.status}] #{check.name}: #{check.message}"
26
- end).join("\n")
53
+ def report(checks = call, color: $stdout.tty?)
54
+ name_width = checks.map { |c| c.name.length }.max.to_i
55
+
56
+ lines = [bold("LLM Cost Tracker doctor", color), ""]
57
+ each_section(checks) do |section, members|
58
+ lines << bold(section, color)
59
+ members.each do |check|
60
+ status = paint_status("[#{STATUS_GLYPHS.fetch(check.status, check.status)}]", check.status, color)
61
+ lines << " #{status} #{"#{check.name}:".ljust(name_width + 1)} #{check.message}"
62
+ end
63
+ lines << ""
64
+ end
65
+ lines.pop if lines.last == ""
66
+ lines.join("\n")
27
67
  end
28
68
 
29
69
  def healthy?(checks = call)
30
70
  checks.none? { |check| check.status == :error }
31
71
  end
72
+
73
+ private
74
+
75
+ def each_section(checks)
76
+ SECTIONS.each do |section|
77
+ members = checks.select { |c| (SECTION_FOR_CHECK[c.name] || "Setup") == section }
78
+ next if members.empty?
79
+
80
+ yield section, members
81
+ end
82
+ end
83
+
84
+ def paint_status(text, status, color)
85
+ return text unless color && STATUS_COLORS.key?(status)
86
+
87
+ "\e[#{STATUS_COLORS[status]}m#{text}\e[0m"
88
+ end
89
+
90
+ def bold(text, color)
91
+ return text unless color
92
+
93
+ "\e[1m#{text}\e[0m"
94
+ end
32
95
  end
33
96
 
34
97
  def checks
35
98
  [
36
99
  configuration_check,
37
100
  capture_check,
38
- *integration_checks,
101
+ *LlmCostTracker::Integrations.checks,
39
102
  active_record_check,
40
103
  table_check,
41
104
  column_check,
42
- SchemaCheck.new(name: "call line items", schema: Ledger::Schema::CallLineItems,
43
- table: "llm_cost_tracker_call_line_items").call,
44
- SchemaCheck.new(name: "call tags", schema: Ledger::Schema::CallTags,
45
- table: "llm_cost_tracker_call_tags").call,
105
+ *dependent_core_schema_checks,
46
106
  *reconciliation_schema_checks,
47
107
  CostDriftCheck.new.call,
48
108
  PricingSnapshotDriftCheck.new.call,
@@ -58,20 +118,23 @@ module LlmCostTracker
58
118
 
59
119
  private
60
120
 
121
+ def dependent_core_schema_checks
122
+ Ledger::Schema::CORE_SCHEMAS.reject { |schema, _| schema == Ledger::Schema::Calls }.map do |schema, table|
123
+ SchemaCheck.new(name: table.delete_prefix("llm_cost_tracker_").tr("_", " "),
124
+ schema: schema, table: table).call
125
+ end
126
+ end
127
+
61
128
  def reconciliation_schema_checks
62
129
  return [] unless LlmCostTracker.reconciliation_enabled?
63
130
 
64
- LlmCostTracker.const_get(:Reconciliation) # autoload reconciliation + its ledger schemas
65
131
  Reconciliation::SCHEMA_TABLES.map do |schema, table|
66
- SchemaCheck.new(name: humanize_table(table), schema: schema, table: table,
132
+ SchemaCheck.new(name: table.delete_prefix("llm_cost_tracker_").tr("_", " "),
133
+ schema: schema, table: table,
67
134
  optional: false, install_command: "llm_cost_tracker:reconciliation").call
68
135
  end.compact
69
136
  end
70
137
 
71
- def humanize_table(table)
72
- table.delete_prefix("llm_cost_tracker_").tr("_", " ")
73
- end
74
-
75
138
  def reconciliation_invoice_check
76
139
  return [] unless LlmCostTracker.reconciliation_enabled?
77
140
 
@@ -104,12 +167,6 @@ module LlmCostTracker
104
167
  )
105
168
  end
106
169
 
107
- def integration_checks
108
- LlmCostTracker::Integrations.checks.map do |check|
109
- Check.new(check.status, check.name.to_s, check.message)
110
- end
111
- end
112
-
113
170
  def active_record_check
114
171
  return Check.new(:ok, "active_record", "available") if active_record_available?
115
172
 
@@ -209,16 +266,14 @@ module LlmCostTracker
209
266
  count = snapshot.tracked_call_count.to_i
210
267
  return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
211
268
 
212
- latest_at = snapshot.latest_tracked_at
213
- latest_at = latest_at.to_time if latest_at.respond_to?(:to_time)
214
- latest = latest_at&.utc&.iso8601
269
+ latest = snapshot.latest_tracked_at.to_time.utc.iso8601
215
270
  Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
216
271
  end
217
272
 
218
273
  def active_record_available?
219
274
  LlmCostTracker::Call.connection
220
275
  true
221
- rescue LoadError, StandardError
276
+ rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::NoDatabaseError
222
277
  false
223
278
  end
224
279
 
@@ -3,7 +3,6 @@
3
3
  require "rails"
4
4
  require_relative "../llm_cost_tracker"
5
5
  require_relative "assets"
6
- require_relative "dashboard_setup_state"
7
6
  require "rack/files"
8
7
 
9
8
  module LlmCostTracker
@@ -15,7 +14,7 @@ module LlmCostTracker
15
14
  end
16
15
 
17
16
  initializer "llm_cost_tracker.dashboard_setup_state" do |app|
18
- app.reloader.to_prepare { LlmCostTracker::DashboardSetupState.reset! }
17
+ app.reloader.to_prepare { LlmCostTracker::Dashboard::SetupState.reset! }
19
18
  end
20
19
  end
21
20
  end
@@ -6,13 +6,14 @@ module LlmCostTracker
6
6
  class InvalidFilterError < Error; end
7
7
 
8
8
  class BudgetExceededError < Error
9
- attr_reader :total, :budget, :budget_type, :last_event
9
+ attr_reader :total, :budget, :budget_type, :last_event, :stage
10
10
 
11
- def initialize(budget:, budget_type:, total:, last_event: nil)
11
+ def initialize(budget:, budget_type:, total:, last_event: nil, stage: :post_spend)
12
12
  @total = total
13
13
  @budget = budget
14
14
  @budget_type = budget_type
15
15
  @last_event = last_event
16
+ @stage = stage
16
17
 
17
18
  super(
18
19
  "LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded: " \