llm_cost_tracker 0.7.3 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -0
  3. data/CHANGELOG.md +66 -1
  4. data/README.md +58 -225
  5. data/app/assets/llm_cost_tracker/application.css +218 -41
  6. data/app/controllers/llm_cost_tracker/application_controller.rb +30 -17
  7. data/app/controllers/llm_cost_tracker/assets_controller.rb +11 -1
  8. data/app/controllers/llm_cost_tracker/calls_controller.rb +19 -14
  9. data/app/controllers/llm_cost_tracker/data_quality_controller.rb +10 -2
  10. data/app/helpers/llm_cost_tracker/application_helper.rb +11 -24
  11. data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +3 -21
  12. data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +4 -4
  13. data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +1 -1
  14. data/app/helpers/llm_cost_tracker/token_usage_helper.rb +20 -7
  15. data/app/models/llm_cost_tracker/call.rb +169 -0
  16. data/app/models/llm_cost_tracker/call_line_item.rb +22 -0
  17. data/app/models/llm_cost_tracker/call_rollup.rb +9 -0
  18. data/app/models/llm_cost_tracker/call_tag.rb +16 -0
  19. data/app/models/llm_cost_tracker/ingestion/inbox_entry.rb +13 -0
  20. data/app/models/llm_cost_tracker/ingestion/lease.rb +1 -1
  21. data/app/models/llm_cost_tracker/provider_invoice.rb +9 -0
  22. data/app/services/llm_cost_tracker/dashboard/data_quality.rb +121 -30
  23. data/app/services/llm_cost_tracker/dashboard/date_range.rb +1 -1
  24. data/app/services/llm_cost_tracker/dashboard/filter.rb +2 -2
  25. data/app/services/llm_cost_tracker/dashboard/overview_stats.rb +74 -21
  26. data/app/services/llm_cost_tracker/dashboard/pagination.rb +6 -4
  27. data/app/services/llm_cost_tracker/dashboard/params.rb +8 -2
  28. data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +1 -1
  29. data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +4 -3
  30. data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +42 -9
  31. data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +14 -37
  32. data/app/services/llm_cost_tracker/dashboard/time_series.rb +1 -1
  33. data/app/services/llm_cost_tracker/dashboard/top_models.rb +1 -1
  34. data/app/views/llm_cost_tracker/calls/index.html.erb +33 -75
  35. data/app/views/llm_cost_tracker/calls/show.html.erb +62 -7
  36. data/app/views/llm_cost_tracker/dashboard/index.html.erb +9 -50
  37. data/app/views/llm_cost_tracker/data_quality/index.html.erb +103 -126
  38. data/app/views/llm_cost_tracker/errors/database.html.erb +1 -1
  39. data/app/views/llm_cost_tracker/models/index.html.erb +18 -50
  40. data/app/views/llm_cost_tracker/shared/_filters.html.erb +63 -0
  41. data/app/views/llm_cost_tracker/shared/_sort.html.erb +13 -0
  42. data/app/views/llm_cost_tracker/shared/setup_required.html.erb +1 -1
  43. data/app/views/llm_cost_tracker/tags/index.html.erb +3 -34
  44. data/app/views/llm_cost_tracker/tags/show.html.erb +5 -37
  45. data/lib/llm_cost_tracker/billing/components.rb +53 -0
  46. data/lib/llm_cost_tracker/billing/components.yml +117 -0
  47. data/lib/llm_cost_tracker/billing/cost_status.rb +45 -0
  48. data/lib/llm_cost_tracker/billing/line_item.rb +189 -0
  49. data/lib/llm_cost_tracker/budget.rb +23 -35
  50. data/lib/llm_cost_tracker/capture/stream_collector.rb +47 -33
  51. data/lib/llm_cost_tracker/configuration.rb +36 -19
  52. data/lib/llm_cost_tracker/doctor/cost_drift_check.rb +54 -0
  53. data/lib/llm_cost_tracker/doctor/ingestion_check.rb +24 -32
  54. data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +36 -0
  55. data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +22 -0
  56. data/lib/llm_cost_tracker/doctor/price_check.rb +2 -2
  57. data/lib/llm_cost_tracker/doctor/pricing_snapshot_drift_check.rb +85 -0
  58. data/lib/llm_cost_tracker/doctor/probe.rb +17 -0
  59. data/lib/llm_cost_tracker/doctor/schema_check.rb +31 -0
  60. data/lib/llm_cost_tracker/doctor.rb +43 -45
  61. data/lib/llm_cost_tracker/errors.rb +5 -19
  62. data/lib/llm_cost_tracker/event.rb +10 -2
  63. data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +4 -2
  64. data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +2 -6
  65. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +157 -0
  66. data/lib/llm_cost_tracker/ingestion/batch.rb +11 -12
  67. data/lib/llm_cost_tracker/ingestion/inbox.rb +39 -23
  68. data/lib/llm_cost_tracker/ingestion/worker.rb +14 -5
  69. data/lib/llm_cost_tracker/ingestion.rb +28 -22
  70. data/lib/llm_cost_tracker/integrations/anthropic.rb +45 -38
  71. data/lib/llm_cost_tracker/integrations/base.rb +36 -29
  72. data/lib/llm_cost_tracker/integrations/openai.rb +85 -40
  73. data/lib/llm_cost_tracker/integrations/ruby_llm.rb +5 -5
  74. data/lib/llm_cost_tracker/integrations.rb +2 -2
  75. data/lib/llm_cost_tracker/ledger/period/totals.rb +12 -9
  76. data/lib/llm_cost_tracker/ledger/period.rb +5 -5
  77. data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb +2 -2
  78. data/lib/llm_cost_tracker/ledger/rollups.rb +76 -25
  79. data/lib/llm_cost_tracker/ledger/schema/adapter.rb +18 -0
  80. data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +50 -0
  81. data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +37 -0
  82. data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +26 -0
  83. data/lib/llm_cost_tracker/ledger/schema/calls.rb +34 -23
  84. data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +57 -0
  85. data/lib/llm_cost_tracker/ledger/store.rb +96 -13
  86. data/lib/llm_cost_tracker/ledger/tags/query.rb +4 -10
  87. data/lib/llm_cost_tracker/ledger/tags/sql.rb +27 -15
  88. data/lib/llm_cost_tracker/ledger.rb +4 -2
  89. data/lib/llm_cost_tracker/logging.rb +2 -5
  90. data/lib/llm_cost_tracker/middleware/faraday.rb +7 -6
  91. data/lib/llm_cost_tracker/parsers/anthropic.rb +52 -7
  92. data/lib/llm_cost_tracker/parsers/base.rb +8 -3
  93. data/lib/llm_cost_tracker/parsers/gemini.rb +101 -15
  94. data/lib/llm_cost_tracker/parsers/openai_compatible.rb +10 -2
  95. data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +87 -0
  96. data/lib/llm_cost_tracker/parsers/openai_usage.rb +48 -21
  97. data/lib/llm_cost_tracker/parsers/sse.rb +1 -1
  98. data/lib/llm_cost_tracker/parsers.rb +1 -1
  99. data/lib/llm_cost_tracker/prices.json +105 -20
  100. data/lib/llm_cost_tracker/pricing/effective_prices.rb +57 -19
  101. data/lib/llm_cost_tracker/pricing/explainer.rb +4 -5
  102. data/lib/llm_cost_tracker/pricing/lookup.rb +38 -34
  103. data/lib/llm_cost_tracker/pricing/registry.rb +65 -45
  104. data/lib/llm_cost_tracker/pricing/service_charges.rb +204 -0
  105. data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +26 -17
  106. data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +6 -15
  107. data/lib/llm_cost_tracker/pricing/sync.rb +57 -10
  108. data/lib/llm_cost_tracker/pricing/sync_change_printer.rb +32 -0
  109. data/lib/llm_cost_tracker/pricing.rb +190 -26
  110. data/lib/llm_cost_tracker/railtie.rb +0 -8
  111. data/lib/llm_cost_tracker/report/data.rb +16 -8
  112. data/lib/llm_cost_tracker/report.rb +0 -4
  113. data/lib/llm_cost_tracker/retention.rb +8 -8
  114. data/lib/llm_cost_tracker/tags/context.rb +2 -4
  115. data/lib/llm_cost_tracker/tags/key.rb +4 -0
  116. data/lib/llm_cost_tracker/tags/sanitizer.rb +12 -17
  117. data/lib/llm_cost_tracker/timing.rb +15 -0
  118. data/lib/llm_cost_tracker/token_usage.rb +56 -42
  119. data/lib/llm_cost_tracker/tracker.rb +67 -24
  120. data/lib/llm_cost_tracker/usage_capture.rb +29 -8
  121. data/lib/llm_cost_tracker/version.rb +1 -1
  122. data/lib/llm_cost_tracker.rb +36 -35
  123. data/lib/tasks/llm_cost_tracker.rake +22 -17
  124. metadata +36 -41
  125. data/app/models/llm_cost_tracker/ingestion/event.rb +0 -13
  126. data/app/models/llm_cost_tracker/ledger/call.rb +0 -45
  127. data/app/models/llm_cost_tracker/ledger/call_metrics.rb +0 -66
  128. data/app/models/llm_cost_tracker/ledger/period/grouping.rb +0 -71
  129. data/app/models/llm_cost_tracker/ledger/period/total.rb +0 -13
  130. data/app/models/llm_cost_tracker/ledger/tags/accessors.rb +0 -19
  131. data/lib/llm_cost_tracker/configuration/instrumentation.rb +0 -33
  132. data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_ingestion_generator.rb +0 -29
  133. data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_latency_ms_generator.rb +0 -29
  134. data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_period_totals_generator.rb +0 -29
  135. data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_provider_response_id_generator.rb +0 -29
  136. data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_streaming_generator.rb +0 -29
  137. data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_token_usage_generator.rb +0 -42
  138. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_ingestion_to_llm_cost_tracker.rb.erb +0 -33
  139. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_latency_ms_to_llm_api_calls.rb.erb +0 -9
  140. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +0 -104
  141. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_provider_response_id_to_llm_api_calls.rb.erb +0 -15
  142. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_streaming_to_llm_api_calls.rb.erb +0 -21
  143. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_token_usage_to_llm_api_calls.rb.erb +0 -22
  144. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +0 -83
  145. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_cost_precision.rb.erb +0 -26
  146. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb +0 -44
  147. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_cost_precision_generator.rb +0 -29
  148. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_tags_to_jsonb_generator.rb +0 -29
  149. data/lib/llm_cost_tracker/ledger/rollups/batch.rb +0 -43
  150. data/lib/llm_cost_tracker/ledger/schema/period_totals.rb +0 -32
  151. data/lib/llm_cost_tracker/pricing/components.rb +0 -37
  152. data/lib/llm_cost_tracker/pricing/sync/registry_loader.rb +0 -63
@@ -1,13 +1,11 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "errors"
4
+ require_relative "pricing/registry"
4
5
  require_relative "tags/key"
5
- require_relative "configuration/instrumentation"
6
6
 
7
7
  module LlmCostTracker
8
8
  class Configuration
9
- include ConfigurationInstrumentation
10
-
11
9
  OPENAI_COMPATIBLE_PROVIDERS = {
12
10
  "openrouter.ai" => "openrouter",
13
11
  "api.deepseek.com" => "deepseek",
@@ -16,8 +14,8 @@ module LlmCostTracker
16
14
 
17
15
  BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
18
16
  UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
19
- SHARED_SCALAR_ATTRIBUTES = %i[enabled on_budget_exceeded monthly_budget daily_budget per_call_budget log_level
20
- prices_file max_tag_count max_tag_value_bytesize].freeze
17
+ SHARED_SCALAR_ATTRIBUTES = %i[enabled default_tags on_budget_exceeded monthly_budget daily_budget per_call_budget
18
+ log_level prices_file max_tag_count max_tag_value_bytesize].freeze
21
19
  SHARED_ENUM_ATTRIBUTES = {
22
20
  budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
23
21
  unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
@@ -27,9 +25,8 @@ module LlmCostTracker
27
25
  attr_reader(
28
26
  *SHARED_SCALAR_ATTRIBUTES,
29
27
  :budget_exceeded_behavior,
30
- :default_tags,
31
- :pricing_overrides,
32
28
  :instrumented_integrations,
29
+ :pricing_overrides,
33
30
  :report_tag_breakdowns,
34
31
  :redacted_tag_keys,
35
32
  :unknown_pricing_behavior,
@@ -49,19 +46,14 @@ module LlmCostTracker
49
46
  @prices_file = nil
50
47
  @max_tag_count = 50
51
48
  @max_tag_value_bytesize = 1024
52
- @pricing_overrides = {}
53
- @instrumented_integrations = []
49
+ self.pricing_overrides = {}
50
+ @instrumented_integrations = Set.new
54
51
  @report_tag_breakdowns = []
55
52
  @redacted_tag_keys = DEFAULT_REDACTED_TAG_KEYS.dup
56
53
  self.openai_compatible_providers = OPENAI_COMPATIBLE_PROVIDERS
57
54
  @finalized = false
58
55
  end
59
56
 
60
- def default_tags=(value)
61
- ensure_shared_configuration_mutable!
62
- @default_tags = value
63
- end
64
-
65
57
  def openai_compatible_providers=(providers)
66
58
  ensure_shared_configuration_mutable!
67
59
  @openai_compatible_providers = normalize_openai_compatible_providers(providers)
@@ -69,7 +61,9 @@ module LlmCostTracker
69
61
 
70
62
  def pricing_overrides=(value)
71
63
  ensure_shared_configuration_mutable!
72
- @pricing_overrides = value
64
+ @pricing_overrides = Pricing::Registry.normalize_price_table(value || {})
65
+ rescue ArgumentError => e
66
+ raise Error, "invalid pricing_overrides: #{e.message}"
73
67
  end
74
68
 
75
69
  def report_tag_breakdowns=(value)
@@ -82,6 +76,15 @@ module LlmCostTracker
82
76
  @redacted_tag_keys = Array(value).map(&:to_s)
83
77
  end
84
78
 
79
+ def instrument(*names)
80
+ ensure_shared_configuration_mutable!
81
+ @instrumented_integrations.merge(normalize_instrumentation_names(names))
82
+ end
83
+
84
+ def instrumented?(name)
85
+ @instrumented_integrations.include?(name)
86
+ end
87
+
85
88
  SHARED_SCALAR_ATTRIBUTES.each do |name|
86
89
  define_method("#{name}=") do |value|
87
90
  ensure_shared_configuration_mutable!
@@ -99,10 +102,12 @@ module LlmCostTracker
99
102
  def finalize!
100
103
  @default_tags = deep_freeze(@default_tags || {})
101
104
  @pricing_overrides = deep_freeze(@pricing_overrides || {})
102
- @instrumented_integrations = deep_freeze(@instrumented_integrations || [])
105
+ @instrumented_integrations = deep_freeze(@instrumented_integrations || Set.new)
103
106
  @report_tag_breakdowns = deep_freeze(Array(@report_tag_breakdowns))
104
107
  @redacted_tag_keys = deep_freeze(Array(@redacted_tag_keys))
105
- @openai_compatible_providers = deep_freeze(@openai_compatible_providers || {})
108
+ @openai_compatible_providers = deep_freeze(
109
+ normalize_openai_compatible_providers(@openai_compatible_providers)
110
+ )
106
111
  @finalized = true
107
112
  self
108
113
  end
@@ -115,7 +120,6 @@ module LlmCostTracker
115
120
 
116
121
  def normalize_enum(name, value, allowed, default:)
117
122
  value = default if value.nil?
118
- value = value.to_sym
119
123
  return value if allowed.include?(value)
120
124
 
121
125
  raise Error, "Unknown #{name}: #{value.inspect}. Use one of: #{allowed.join(', ')}"
@@ -127,6 +131,19 @@ module LlmCostTracker
127
131
  end
128
132
  end
129
133
 
134
+ def normalize_instrumentation_names(names)
135
+ names = names.flatten
136
+ integrations = Integrations.names
137
+ return integrations if names == [:all]
138
+
139
+ names.each do |name|
140
+ next if integrations.include?(name)
141
+
142
+ raise Error, "Unknown integration: #{name.inspect}. Use one of: #{integrations.join(', ')}"
143
+ end
144
+ names
145
+ end
146
+
130
147
  def ensure_shared_configuration_mutable!
131
148
  return unless finalized?
132
149
 
@@ -141,7 +158,7 @@ module LlmCostTracker
141
158
  deep_freeze(nested_value)
142
159
  end
143
160
  value.frozen? ? value : value.freeze
144
- when Array
161
+ when Array, Set
145
162
  value.each { |nested_value| deep_freeze(nested_value) }
146
163
  value.frozen? ? value : value.freeze
147
164
  when String
@@ -0,0 +1,54 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bigdecimal"
4
+
5
+ require_relative "check"
6
+ require_relative "probe"
7
+
8
+ module LlmCostTracker
9
+ class Doctor
10
+ class CostDriftCheck
11
+ SAMPLE_SIZE = 200
12
+ EPSILON = BigDecimal("0.00000001")
13
+
14
+ def call
15
+ return unless Probe.table_exists?("llm_cost_tracker_calls")
16
+ return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
17
+
18
+ sampled = LlmCostTracker::Call
19
+ .where.not(total_cost: nil)
20
+ .where(cost_status: %w[complete free partial])
21
+ .order(id: :desc)
22
+ .limit(SAMPLE_SIZE)
23
+ .pluck(:id, :total_cost, :cost_status)
24
+ return Check.new(:ok, "cost drift", "no priced calls to inspect") if sampled.empty?
25
+
26
+ line_item_totals = LlmCostTracker::CallLineItem
27
+ .where(llm_cost_tracker_call_id: sampled.map(&:first))
28
+ .group(:llm_cost_tracker_call_id)
29
+ .sum(:cost)
30
+
31
+ drifted = sampled.filter_map do |id, total_cost, cost_status|
32
+ line_total = line_item_totals[id] || BigDecimal("0")
33
+ header = BigDecimal(total_cost.to_s)
34
+ next if cost_status == "partial" && header >= line_total
35
+ next if (header - line_total).abs <= EPSILON
36
+
37
+ "##{id}: header=#{header.to_s('F')} line_items=#{line_total.to_s('F')}"
38
+ end
39
+
40
+ if drifted.empty?
41
+ return Check.new(:ok, "cost drift",
42
+ "header total_cost matches line items in #{sampled.size} sampled calls")
43
+ end
44
+
45
+ Check.new(
46
+ :warn,
47
+ "cost drift",
48
+ "header total_cost diverges from line items in #{drifted.size}/#{sampled.size} sampled calls: " \
49
+ "#{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
50
+ )
51
+ end
52
+ end
53
+ end
54
+ end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "check"
4
+ require_relative "probe"
4
5
  require_relative "../ingestion"
5
6
 
6
7
  module LlmCostTracker
@@ -9,34 +10,34 @@ module LlmCostTracker
9
10
  PENDING_AGE_WARNING_SECONDS = 60
10
11
 
11
12
  def call
12
- return unless table_exists?("llm_api_calls")
13
+ return unless Probe.table_exists?("llm_cost_tracker_calls")
13
14
 
14
15
  missing = missing_parts
15
16
  if missing.empty?
16
- quarantined = quarantined_count
17
+ inbox = inbox_snapshot
18
+ quarantined = inbox.try(:quarantined_count).to_i
17
19
  if quarantined.positive?
18
- return Check.new(:warn, "durable ingestion", "#{quarantined} inbox events quarantined after retries")
20
+ return Check.new(:warn, "durable ingestion", "#{quarantined} inbox entries quarantined after retries")
19
21
  end
20
22
 
21
- pending = pending_snapshot
22
- pending_count = pending.try(:pending_count).to_i
23
- oldest_pending_at = pending.try(:oldest_created_at)&.to_time&.utc
23
+ pending_count = inbox.try(:pending_count).to_i
24
+ oldest_pending_at = inbox.try(:oldest_pending_at)&.to_time&.utc
24
25
  pending_age = oldest_pending_at && (Time.now.utc - oldest_pending_at)
25
26
  if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
26
27
  return Check.new(
27
28
  :warn,
28
29
  "durable ingestion",
29
- "#{pending_count} inbox events pending; oldest pending age #{pending_age.round}s"
30
+ "#{pending_count} inbox entries pending; oldest pending age #{pending_age.round}s"
30
31
  )
31
32
  end
32
33
 
33
- return Check.new(:ok, "durable ingestion", "inbox and ingestor lease tables available")
34
+ return Check.new(:ok, "durable ingestion", "inbox and ingestion lease tables available")
34
35
  end
35
36
 
36
37
  Check.new(
37
38
  :error,
38
39
  "durable ingestion",
39
- "missing #{missing.join(', ')}; run bin/rails generate llm_cost_tracker:add_ingestion && bin/rails db:migrate"
40
+ "missing #{missing.join(', ')}; see docs/upgrading.md for the recovery steps"
40
41
  )
41
42
  end
42
43
 
@@ -44,31 +45,22 @@ module LlmCostTracker
44
45
 
45
46
  def missing_parts
46
47
  [
47
- table_exists?("llm_cost_tracker_inbox_events") ? nil : "llm_cost_tracker_inbox_events",
48
- table_exists?("llm_cost_tracker_ingestor_leases") ? nil : "llm_cost_tracker_ingestor_leases"
49
- ].compact
48
+ LlmCostTracker::Ingestion::InboxEntry.table_name,
49
+ LlmCostTracker::Ingestion::Lease.table_name
50
+ ].reject { |table| Probe.table_exists?(table) }
50
51
  end
51
52
 
52
- def table_exists?(name)
53
- LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
54
- rescue StandardError
55
- false
56
- end
57
-
58
- def quarantined_count
59
- return 0 unless table_exists?("llm_cost_tracker_inbox_events")
60
-
61
- LlmCostTracker::Ingestion::Event
62
- .where("attempts >= ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
63
- .count
64
- rescue StandardError
65
- 0
66
- end
67
-
68
- def pending_snapshot
69
- LlmCostTracker::Ingestion::Event
70
- .where("attempts < ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
71
- .select("COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at")
53
+ def inbox_snapshot
54
+ max_attempts = LlmCostTracker::Ingestion::InboxEntry::MAX_ATTEMPTS_BEFORE_QUARANTINE
55
+ LlmCostTracker::Ingestion::InboxEntry
56
+ .select(
57
+ "COALESCE(SUM(CASE WHEN attempts >= #{max_attempts} " \
58
+ "THEN 1 ELSE 0 END), 0) AS quarantined_count, " \
59
+ "COALESCE(SUM(CASE WHEN attempts < #{max_attempts} " \
60
+ "THEN 1 ELSE 0 END), 0) AS pending_count, " \
61
+ "MIN(CASE WHEN attempts < #{max_attempts} " \
62
+ "THEN created_at ELSE NULL END) AS oldest_pending_at"
63
+ )
72
64
  .take
73
65
  rescue StandardError
74
66
  nil
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "check"
4
+ require_relative "probe"
5
+ require_relative "../ledger"
6
+
7
+ module LlmCostTracker
8
+ class Doctor
9
+ class LegacyAuditCheck
10
+ WARNING_PERCENT = 10
11
+
12
+ def call
13
+ return unless Probe.table_exists?("llm_cost_tracker_calls")
14
+ return unless LlmCostTracker::Call.column_names.include?("pricing_snapshot")
15
+
16
+ counts = LlmCostTracker::Call
17
+ .select(
18
+ "COUNT(*) AS total_count, " \
19
+ "COALESCE(SUM(CASE WHEN pricing_snapshot IS NULL THEN 1 ELSE 0 END), 0) AS missing_count"
20
+ )
21
+ .take
22
+ total = counts.total_count.to_i
23
+ return if total.zero?
24
+
25
+ missing = counts.missing_count.to_i
26
+ return unless (missing * 100) > (total * WARNING_PERCENT)
27
+
28
+ message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
29
+ "stored totals remain stable but applied rates cannot be audited"
30
+ Check.new(:warn, "pricing snapshot audit", message)
31
+ rescue StandardError
32
+ nil
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "check"
4
+ require_relative "probe"
5
+ require_relative "../ledger"
6
+
7
+ module LlmCostTracker
8
+ class Doctor
9
+ class LegacyBillingStatusCheck
10
+ def call
11
+ return unless Probe.table_exists?("llm_cost_tracker_calls")
12
+ return unless LlmCostTracker::Call.column_names.include?("cost_status")
13
+
14
+ return unless LlmCostTracker::Call.where(cost_status: nil).exists?
15
+
16
+ Check.new(:warn, "cost status", "legacy rows without cost_status remain; new rows will populate it")
17
+ rescue StandardError
18
+ nil
19
+ end
20
+ end
21
+ end
22
+ end
@@ -8,7 +8,7 @@ module LlmCostTracker
8
8
  class Doctor
9
9
  class PriceCheck
10
10
  STALE_AFTER_DAYS = 30
11
- REFRESH_COMMAND = "run bin/rails llm_cost_tracker:prices:refresh"
11
+ REFRESH_COMMAND = "refresh the source-controlled prices file with bin/rails llm_cost_tracker:prices:refresh"
12
12
 
13
13
  def call
14
14
  path = LlmCostTracker.configuration.prices_file
@@ -48,7 +48,7 @@ module LlmCostTracker
48
48
  Check.new(
49
49
  :warn,
50
50
  "prices",
51
- "using bundled prices updated_at=#{updated_at}; configure prices_file for production"
51
+ "using bundled prices updated_at=#{updated_at}; commit a prices_file for production releases"
52
52
  )
53
53
  end
54
54
 
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bigdecimal"
4
+
5
+ require_relative "check"
6
+ require_relative "probe"
7
+
8
+ module LlmCostTracker
9
+ class Doctor
10
+ class PricingSnapshotDriftCheck
11
+ SAMPLE_SIZE = 200
12
+ EPSILON = BigDecimal("0.00000001")
13
+
14
+ def call
15
+ return unless Probe.table_exists?("llm_cost_tracker_calls")
16
+ return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
17
+
18
+ sampled_ids = LlmCostTracker::Call
19
+ .where.not(pricing_snapshot: nil)
20
+ .where(cost_status: %w[complete free])
21
+ .order(id: :desc)
22
+ .limit(SAMPLE_SIZE)
23
+ .pluck(:id)
24
+ return Check.new(:ok, "pricing snapshot drift", "no snapshotted calls to inspect") if sampled_ids.empty?
25
+
26
+ calls_by_id = LlmCostTracker::Call.where(id: sampled_ids).index_by(&:id)
27
+ line_items_by_call = LlmCostTracker::CallLineItem
28
+ .where(llm_cost_tracker_call_id: sampled_ids, unit: "token")
29
+ .group_by(&:llm_cost_tracker_call_id)
30
+
31
+ drifted = sampled_ids.flat_map do |id|
32
+ call = calls_by_id[id]
33
+ rates = rates_for(call.pricing_snapshot)
34
+ next [] if rates.nil? || rates.empty?
35
+
36
+ (line_items_by_call[id] || []).filter_map { |item| drift_message_for(item, rates, call_id: id) }
37
+ end
38
+
39
+ return ok_check(sampled_ids.size) if drifted.empty?
40
+
41
+ Check.new(
42
+ :warn,
43
+ "pricing snapshot drift",
44
+ "line item cost diverges from pricing_snapshot rate in #{drifted.size} cases across " \
45
+ "#{sampled_ids.size} sampled calls: #{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
46
+ )
47
+ end
48
+
49
+ private
50
+
51
+ def ok_check(sample_size)
52
+ Check.new(:ok, "pricing snapshot drift",
53
+ "line item costs match pricing_snapshot rates in #{sample_size} sampled calls")
54
+ end
55
+
56
+ def rates_for(snapshot)
57
+ rates = snapshot.is_a?(Hash) ? (snapshot["rates"] || snapshot[:rates]) : nil
58
+ rates.is_a?(Hash) ? rates : nil
59
+ end
60
+
61
+ def drift_message_for(line_item, rates, call_id:)
62
+ return nil unless line_item.price_key
63
+
64
+ rate = rates[line_item.price_key.to_s] || rates[line_item.price_key.to_sym]
65
+ return nil unless rate.is_a?(Hash)
66
+
67
+ rate_amount = decimal(rate["amount"] || rate[:amount])
68
+ rate_quantity = decimal(rate["quantity"] || rate[:quantity])
69
+ return nil if rate_amount.nil? || rate_quantity.nil? || rate_quantity.zero?
70
+
71
+ expected = (decimal(line_item.quantity) * rate_amount) / rate_quantity
72
+ actual = decimal(line_item.cost) || BigDecimal("0")
73
+ return nil if (expected - actual).abs <= EPSILON
74
+
75
+ "##{call_id}.#{line_item.price_key}: expected=#{expected.round(8).to_s('F')} stored=#{actual.to_s('F')}"
76
+ end
77
+
78
+ def decimal(value)
79
+ return nil if value.nil?
80
+
81
+ BigDecimal(value.to_s)
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../ledger"
4
+
5
+ module LlmCostTracker
6
+ class Doctor
7
+ module Probe
8
+ module_function
9
+
10
+ def table_exists?(name)
11
+ LlmCostTracker::Call.connection.data_source_exists?(name)
12
+ rescue StandardError
13
+ false
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "check"
4
+ require_relative "probe"
5
+ require_relative "../ledger"
6
+
7
+ module LlmCostTracker
8
+ class Doctor
9
+ class SchemaCheck
10
+ def initialize(name:, schema:, table:)
11
+ @name = name
12
+ @schema = schema
13
+ @table = table
14
+ end
15
+
16
+ def call
17
+ return unless Probe.table_exists?("llm_cost_tracker_calls")
18
+
19
+ errors = @schema.current_schema_errors
20
+ return Check.new(:ok, @name, "#{@table} exists") if errors.empty?
21
+
22
+ Check.new(
23
+ :error,
24
+ @name,
25
+ "current schema required; #{errors.join('; ')}; " \
26
+ "run bin/rails generate llm_cost_tracker:install && bin/rails db:migrate"
27
+ )
28
+ end
29
+ end
30
+ end
31
+ end
@@ -2,24 +2,17 @@
2
2
 
3
3
  require_relative "ledger"
4
4
  require_relative "doctor/check"
5
+ require_relative "doctor/probe"
5
6
  require_relative "doctor/ingestion_check"
7
+ require_relative "doctor/legacy_audit_check"
8
+ require_relative "doctor/legacy_billing_status_check"
6
9
  require_relative "doctor/price_check"
7
- require_relative "generators/llm_cost_tracker/add_token_usage_generator"
10
+ require_relative "doctor/schema_check"
11
+ require_relative "doctor/cost_drift_check"
12
+ require_relative "doctor/pricing_snapshot_drift_check"
8
13
 
9
14
  module LlmCostTracker
10
15
  class Doctor
11
- COLUMN_GENERATORS = {
12
- "event_id" => "bin/rails generate llm_cost_tracker:add_ingestion",
13
- "latency_ms" => "bin/rails generate llm_cost_tracker:add_latency_ms",
14
- "stream" => "bin/rails generate llm_cost_tracker:add_streaming",
15
- "usage_source" => "bin/rails generate llm_cost_tracker:add_streaming",
16
- "provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id"
17
- }.merge(
18
- Generators::AddTokenUsageGenerator::COLUMN_NAMES.to_h do |column|
19
- [column, "bin/rails generate llm_cost_tracker:add_token_usage"]
20
- end
21
- ).freeze
22
-
23
16
  class << self
24
17
  def call
25
18
  new.checks
@@ -44,7 +37,17 @@ module LlmCostTracker
44
37
  active_record_check,
45
38
  table_check,
46
39
  column_check,
47
- period_totals_check,
40
+ SchemaCheck.new(name: "call line items", schema: Ledger::Schema::CallLineItems,
41
+ table: "llm_cost_tracker_call_line_items").call,
42
+ SchemaCheck.new(name: "call tags", schema: Ledger::Schema::CallTags,
43
+ table: "llm_cost_tracker_call_tags").call,
44
+ SchemaCheck.new(name: "provider invoices", schema: Ledger::Schema::ProviderInvoices,
45
+ table: "llm_cost_tracker_provider_invoices").call,
46
+ CostDriftCheck.new.call,
47
+ PricingSnapshotDriftCheck.new.call,
48
+ LegacyBillingStatusCheck.new.call,
49
+ LegacyAuditCheck.new.call,
50
+ call_rollups_check,
48
51
  IngestionCheck.new.call,
49
52
  PriceCheck.new.call,
50
53
  calls_check
@@ -68,7 +71,7 @@ module LlmCostTracker
68
71
  return Check.new(
69
72
  :ok,
70
73
  "capture",
71
- "SDK integrations enabled: #{config.instrumented_integrations.join(', ')}"
74
+ "SDK integrations enabled: #{config.instrumented_integrations.to_a.join(', ')}"
72
75
  )
73
76
  end
74
77
 
@@ -93,68 +96,63 @@ module LlmCostTracker
93
96
 
94
97
  def table_check
95
98
  return unless active_record_available?
96
- return Check.new(:ok, "llm_api_calls", "table exists") if llm_api_calls_table?
99
+ return Check.new(:ok, "llm_cost_tracker_calls", "table exists") if llm_cost_tracker_calls_table?
97
100
 
98
101
  Check.new(
99
102
  :error,
100
- "llm_api_calls",
103
+ "llm_cost_tracker_calls",
101
104
  "missing; run bin/rails generate llm_cost_tracker:install && bin/rails db:migrate"
102
105
  )
103
106
  end
104
107
 
105
108
  def column_check
106
- return unless llm_api_calls_table?
109
+ return unless llm_cost_tracker_calls_table?
107
110
 
108
111
  errors = LlmCostTracker::Ledger::Schema::Calls.current_schema_errors
109
- return Check.new(:ok, "llm_api_calls columns", "current") if errors.empty?
110
-
111
- missing = LlmCostTracker::Ledger::Schema::Calls.missing_current_schema_columns
112
- generators = missing.filter_map { |column| COLUMN_GENERATORS[column] }.uniq
113
- message = "current schema required; #{errors.join('; ')}"
114
- message = "#{message}; run #{generators.join(' && ')} && bin/rails db:migrate" if generators.any?
112
+ return Check.new(:ok, "llm_cost_tracker_calls columns", "current") if errors.empty?
115
113
 
116
- Check.new(:error, "llm_api_calls columns", message)
114
+ Check.new(
115
+ :error,
116
+ "llm_cost_tracker_calls columns",
117
+ "schema mismatch: #{errors.join('; ')}; see docs/upgrading.md"
118
+ )
117
119
  end
118
120
 
119
- def period_totals_check
120
- return unless llm_api_calls_table?
121
+ def call_rollups_check
122
+ return unless llm_cost_tracker_calls_table?
121
123
 
122
- errors = LlmCostTracker::Ledger::Schema::PeriodTotals.current_schema_errors
123
- return Check.new(:ok, "period totals", "llm_cost_tracker_period_totals exists") if errors.empty?
124
+ errors = LlmCostTracker::Ledger::Schema::CallRollups.current_schema_errors
125
+ return Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists") if errors.empty?
124
126
 
125
127
  Check.new(
126
128
  :error,
127
- "period totals",
128
- "current schema required; #{errors.join('; ')}; " \
129
- "run bin/rails generate llm_cost_tracker:add_period_totals && bin/rails db:migrate"
129
+ "call rollups",
130
+ "schema mismatch: #{errors.join('; ')}; see docs/upgrading.md"
130
131
  )
131
132
  end
132
133
 
133
134
  def calls_check
134
- return unless llm_api_calls_table?
135
+ return unless llm_cost_tracker_calls_table?
135
136
 
136
- count = LlmCostTracker::Ledger::Call.count
137
+ snapshot = LlmCostTracker::Call
138
+ .select("COUNT(*) AS tracked_call_count, MAX(tracked_at) AS latest_tracked_at")
139
+ .take
140
+ count = snapshot.tracked_call_count.to_i
137
141
  return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
138
142
 
139
- latest = LlmCostTracker::Ledger::Call.maximum(:tracked_at)&.utc&.iso8601
143
+ latest_at = snapshot.latest_tracked_at
144
+ latest_at = latest_at.to_time if latest_at.respond_to?(:to_time)
145
+ latest = latest_at&.utc&.iso8601
140
146
  Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
141
147
  end
142
148
 
143
149
  def active_record_available?
144
- LlmCostTracker::Ledger::Call.connection
150
+ LlmCostTracker::Call.connection
145
151
  true
146
152
  rescue LoadError, StandardError
147
153
  false
148
154
  end
149
155
 
150
- def llm_api_calls_table?
151
- active_record_available? && table_exists?("llm_api_calls")
152
- end
153
-
154
- def table_exists?(name)
155
- LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
156
- rescue StandardError
157
- false
158
- end
156
+ def llm_cost_tracker_calls_table? = active_record_available? && Probe.table_exists?("llm_cost_tracker_calls")
159
157
  end
160
158
  end