llm_cost_tracker 0.7.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/CHANGELOG.md +66 -1
- data/README.md +58 -225
- data/app/assets/llm_cost_tracker/application.css +218 -41
- data/app/controllers/llm_cost_tracker/application_controller.rb +30 -17
- data/app/controllers/llm_cost_tracker/assets_controller.rb +11 -1
- data/app/controllers/llm_cost_tracker/calls_controller.rb +19 -14
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +10 -2
- data/app/helpers/llm_cost_tracker/application_helper.rb +11 -24
- data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +3 -21
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +4 -4
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +20 -7
- data/app/models/llm_cost_tracker/call.rb +169 -0
- data/app/models/llm_cost_tracker/call_line_item.rb +22 -0
- data/app/models/llm_cost_tracker/call_rollup.rb +9 -0
- data/app/models/llm_cost_tracker/call_tag.rb +16 -0
- data/app/models/llm_cost_tracker/ingestion/inbox_entry.rb +13 -0
- data/app/models/llm_cost_tracker/ingestion/lease.rb +1 -1
- data/app/models/llm_cost_tracker/provider_invoice.rb +9 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +121 -30
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/filter.rb +2 -2
- data/app/services/llm_cost_tracker/dashboard/overview_stats.rb +74 -21
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +6 -4
- data/app/services/llm_cost_tracker/dashboard/params.rb +8 -2
- data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +4 -3
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +42 -9
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +14 -37
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +1 -1
- data/app/views/llm_cost_tracker/calls/index.html.erb +33 -75
- data/app/views/llm_cost_tracker/calls/show.html.erb +62 -7
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +9 -50
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +103 -126
- data/app/views/llm_cost_tracker/errors/database.html.erb +1 -1
- data/app/views/llm_cost_tracker/models/index.html.erb +18 -50
- data/app/views/llm_cost_tracker/shared/_filters.html.erb +63 -0
- data/app/views/llm_cost_tracker/shared/_sort.html.erb +13 -0
- data/app/views/llm_cost_tracker/shared/setup_required.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/index.html.erb +3 -34
- data/app/views/llm_cost_tracker/tags/show.html.erb +5 -37
- data/lib/llm_cost_tracker/billing/components.rb +53 -0
- data/lib/llm_cost_tracker/billing/components.yml +117 -0
- data/lib/llm_cost_tracker/billing/cost_status.rb +45 -0
- data/lib/llm_cost_tracker/billing/line_item.rb +189 -0
- data/lib/llm_cost_tracker/budget.rb +23 -35
- data/lib/llm_cost_tracker/capture/stream_collector.rb +47 -33
- data/lib/llm_cost_tracker/configuration.rb +36 -19
- data/lib/llm_cost_tracker/doctor/cost_drift_check.rb +54 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +24 -32
- data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +36 -0
- data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +22 -0
- data/lib/llm_cost_tracker/doctor/price_check.rb +2 -2
- data/lib/llm_cost_tracker/doctor/pricing_snapshot_drift_check.rb +85 -0
- data/lib/llm_cost_tracker/doctor/probe.rb +17 -0
- data/lib/llm_cost_tracker/doctor/schema_check.rb +31 -0
- data/lib/llm_cost_tracker/doctor.rb +43 -45
- data/lib/llm_cost_tracker/errors.rb +5 -19
- data/lib/llm_cost_tracker/event.rb +10 -2
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +4 -2
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +2 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +157 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +11 -12
- data/lib/llm_cost_tracker/ingestion/inbox.rb +39 -23
- data/lib/llm_cost_tracker/ingestion/worker.rb +14 -5
- data/lib/llm_cost_tracker/ingestion.rb +28 -22
- data/lib/llm_cost_tracker/integrations/anthropic.rb +45 -38
- data/lib/llm_cost_tracker/integrations/base.rb +36 -29
- data/lib/llm_cost_tracker/integrations/openai.rb +85 -40
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +5 -5
- data/lib/llm_cost_tracker/integrations.rb +2 -2
- data/lib/llm_cost_tracker/ledger/period/totals.rb +12 -9
- data/lib/llm_cost_tracker/ledger/period.rb +5 -5
- data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb +2 -2
- data/lib/llm_cost_tracker/ledger/rollups.rb +76 -25
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +18 -0
- data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +50 -0
- data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +37 -0
- data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +26 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +34 -23
- data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +57 -0
- data/lib/llm_cost_tracker/ledger/store.rb +96 -13
- data/lib/llm_cost_tracker/ledger/tags/query.rb +4 -10
- data/lib/llm_cost_tracker/ledger/tags/sql.rb +27 -15
- data/lib/llm_cost_tracker/ledger.rb +4 -2
- data/lib/llm_cost_tracker/logging.rb +2 -5
- data/lib/llm_cost_tracker/middleware/faraday.rb +7 -6
- data/lib/llm_cost_tracker/parsers/anthropic.rb +52 -7
- data/lib/llm_cost_tracker/parsers/base.rb +8 -3
- data/lib/llm_cost_tracker/parsers/gemini.rb +101 -15
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +10 -2
- data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +87 -0
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +48 -21
- data/lib/llm_cost_tracker/parsers/sse.rb +1 -1
- data/lib/llm_cost_tracker/parsers.rb +1 -1
- data/lib/llm_cost_tracker/prices.json +105 -20
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +57 -19
- data/lib/llm_cost_tracker/pricing/explainer.rb +4 -5
- data/lib/llm_cost_tracker/pricing/lookup.rb +38 -34
- data/lib/llm_cost_tracker/pricing/registry.rb +65 -45
- data/lib/llm_cost_tracker/pricing/service_charges.rb +204 -0
- data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +26 -17
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +6 -15
- data/lib/llm_cost_tracker/pricing/sync.rb +57 -10
- data/lib/llm_cost_tracker/pricing/sync_change_printer.rb +32 -0
- data/lib/llm_cost_tracker/pricing.rb +190 -26
- data/lib/llm_cost_tracker/railtie.rb +0 -8
- data/lib/llm_cost_tracker/report/data.rb +16 -8
- data/lib/llm_cost_tracker/report.rb +0 -4
- data/lib/llm_cost_tracker/retention.rb +8 -8
- data/lib/llm_cost_tracker/tags/context.rb +2 -4
- data/lib/llm_cost_tracker/tags/key.rb +4 -0
- data/lib/llm_cost_tracker/tags/sanitizer.rb +12 -17
- data/lib/llm_cost_tracker/timing.rb +15 -0
- data/lib/llm_cost_tracker/token_usage.rb +56 -42
- data/lib/llm_cost_tracker/tracker.rb +67 -24
- data/lib/llm_cost_tracker/usage_capture.rb +29 -8
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +36 -35
- data/lib/tasks/llm_cost_tracker.rake +22 -17
- metadata +36 -41
- data/app/models/llm_cost_tracker/ingestion/event.rb +0 -13
- data/app/models/llm_cost_tracker/ledger/call.rb +0 -45
- data/app/models/llm_cost_tracker/ledger/call_metrics.rb +0 -66
- data/app/models/llm_cost_tracker/ledger/period/grouping.rb +0 -71
- data/app/models/llm_cost_tracker/ledger/period/total.rb +0 -13
- data/app/models/llm_cost_tracker/ledger/tags/accessors.rb +0 -19
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +0 -33
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_ingestion_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_latency_ms_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_period_totals_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_provider_response_id_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_streaming_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_token_usage_generator.rb +0 -42
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_ingestion_to_llm_cost_tracker.rb.erb +0 -33
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_latency_ms_to_llm_api_calls.rb.erb +0 -9
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +0 -104
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_provider_response_id_to_llm_api_calls.rb.erb +0 -15
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_streaming_to_llm_api_calls.rb.erb +0 -21
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_token_usage_to_llm_api_calls.rb.erb +0 -22
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +0 -83
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_cost_precision.rb.erb +0 -26
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb +0 -44
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_cost_precision_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_tags_to_jsonb_generator.rb +0 -29
- data/lib/llm_cost_tracker/ledger/rollups/batch.rb +0 -43
- data/lib/llm_cost_tracker/ledger/schema/period_totals.rb +0 -32
- data/lib/llm_cost_tracker/pricing/components.rb +0 -37
- data/lib/llm_cost_tracker/pricing/sync/registry_loader.rb +0 -63
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "errors"
|
|
4
|
+
require_relative "pricing/registry"
|
|
4
5
|
require_relative "tags/key"
|
|
5
|
-
require_relative "configuration/instrumentation"
|
|
6
6
|
|
|
7
7
|
module LlmCostTracker
|
|
8
8
|
class Configuration
|
|
9
|
-
include ConfigurationInstrumentation
|
|
10
|
-
|
|
11
9
|
OPENAI_COMPATIBLE_PROVIDERS = {
|
|
12
10
|
"openrouter.ai" => "openrouter",
|
|
13
11
|
"api.deepseek.com" => "deepseek",
|
|
@@ -16,8 +14,8 @@ module LlmCostTracker
|
|
|
16
14
|
|
|
17
15
|
BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
|
|
18
16
|
UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
|
|
19
|
-
SHARED_SCALAR_ATTRIBUTES = %i[enabled on_budget_exceeded monthly_budget daily_budget per_call_budget
|
|
20
|
-
prices_file max_tag_count max_tag_value_bytesize].freeze
|
|
17
|
+
SHARED_SCALAR_ATTRIBUTES = %i[enabled default_tags on_budget_exceeded monthly_budget daily_budget per_call_budget
|
|
18
|
+
log_level prices_file max_tag_count max_tag_value_bytesize].freeze
|
|
21
19
|
SHARED_ENUM_ATTRIBUTES = {
|
|
22
20
|
budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
|
|
23
21
|
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
|
|
@@ -27,9 +25,8 @@ module LlmCostTracker
|
|
|
27
25
|
attr_reader(
|
|
28
26
|
*SHARED_SCALAR_ATTRIBUTES,
|
|
29
27
|
:budget_exceeded_behavior,
|
|
30
|
-
:default_tags,
|
|
31
|
-
:pricing_overrides,
|
|
32
28
|
:instrumented_integrations,
|
|
29
|
+
:pricing_overrides,
|
|
33
30
|
:report_tag_breakdowns,
|
|
34
31
|
:redacted_tag_keys,
|
|
35
32
|
:unknown_pricing_behavior,
|
|
@@ -49,19 +46,14 @@ module LlmCostTracker
|
|
|
49
46
|
@prices_file = nil
|
|
50
47
|
@max_tag_count = 50
|
|
51
48
|
@max_tag_value_bytesize = 1024
|
|
52
|
-
|
|
53
|
-
@instrumented_integrations =
|
|
49
|
+
self.pricing_overrides = {}
|
|
50
|
+
@instrumented_integrations = Set.new
|
|
54
51
|
@report_tag_breakdowns = []
|
|
55
52
|
@redacted_tag_keys = DEFAULT_REDACTED_TAG_KEYS.dup
|
|
56
53
|
self.openai_compatible_providers = OPENAI_COMPATIBLE_PROVIDERS
|
|
57
54
|
@finalized = false
|
|
58
55
|
end
|
|
59
56
|
|
|
60
|
-
def default_tags=(value)
|
|
61
|
-
ensure_shared_configuration_mutable!
|
|
62
|
-
@default_tags = value
|
|
63
|
-
end
|
|
64
|
-
|
|
65
57
|
def openai_compatible_providers=(providers)
|
|
66
58
|
ensure_shared_configuration_mutable!
|
|
67
59
|
@openai_compatible_providers = normalize_openai_compatible_providers(providers)
|
|
@@ -69,7 +61,9 @@ module LlmCostTracker
|
|
|
69
61
|
|
|
70
62
|
def pricing_overrides=(value)
|
|
71
63
|
ensure_shared_configuration_mutable!
|
|
72
|
-
@pricing_overrides = value
|
|
64
|
+
@pricing_overrides = Pricing::Registry.normalize_price_table(value || {})
|
|
65
|
+
rescue ArgumentError => e
|
|
66
|
+
raise Error, "invalid pricing_overrides: #{e.message}"
|
|
73
67
|
end
|
|
74
68
|
|
|
75
69
|
def report_tag_breakdowns=(value)
|
|
@@ -82,6 +76,15 @@ module LlmCostTracker
|
|
|
82
76
|
@redacted_tag_keys = Array(value).map(&:to_s)
|
|
83
77
|
end
|
|
84
78
|
|
|
79
|
+
def instrument(*names)
|
|
80
|
+
ensure_shared_configuration_mutable!
|
|
81
|
+
@instrumented_integrations.merge(normalize_instrumentation_names(names))
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def instrumented?(name)
|
|
85
|
+
@instrumented_integrations.include?(name)
|
|
86
|
+
end
|
|
87
|
+
|
|
85
88
|
SHARED_SCALAR_ATTRIBUTES.each do |name|
|
|
86
89
|
define_method("#{name}=") do |value|
|
|
87
90
|
ensure_shared_configuration_mutable!
|
|
@@ -99,10 +102,12 @@ module LlmCostTracker
|
|
|
99
102
|
def finalize!
|
|
100
103
|
@default_tags = deep_freeze(@default_tags || {})
|
|
101
104
|
@pricing_overrides = deep_freeze(@pricing_overrides || {})
|
|
102
|
-
@instrumented_integrations = deep_freeze(@instrumented_integrations ||
|
|
105
|
+
@instrumented_integrations = deep_freeze(@instrumented_integrations || Set.new)
|
|
103
106
|
@report_tag_breakdowns = deep_freeze(Array(@report_tag_breakdowns))
|
|
104
107
|
@redacted_tag_keys = deep_freeze(Array(@redacted_tag_keys))
|
|
105
|
-
@openai_compatible_providers = deep_freeze(
|
|
108
|
+
@openai_compatible_providers = deep_freeze(
|
|
109
|
+
normalize_openai_compatible_providers(@openai_compatible_providers)
|
|
110
|
+
)
|
|
106
111
|
@finalized = true
|
|
107
112
|
self
|
|
108
113
|
end
|
|
@@ -115,7 +120,6 @@ module LlmCostTracker
|
|
|
115
120
|
|
|
116
121
|
def normalize_enum(name, value, allowed, default:)
|
|
117
122
|
value = default if value.nil?
|
|
118
|
-
value = value.to_sym
|
|
119
123
|
return value if allowed.include?(value)
|
|
120
124
|
|
|
121
125
|
raise Error, "Unknown #{name}: #{value.inspect}. Use one of: #{allowed.join(', ')}"
|
|
@@ -127,6 +131,19 @@ module LlmCostTracker
|
|
|
127
131
|
end
|
|
128
132
|
end
|
|
129
133
|
|
|
134
|
+
def normalize_instrumentation_names(names)
|
|
135
|
+
names = names.flatten
|
|
136
|
+
integrations = Integrations.names
|
|
137
|
+
return integrations if names == [:all]
|
|
138
|
+
|
|
139
|
+
names.each do |name|
|
|
140
|
+
next if integrations.include?(name)
|
|
141
|
+
|
|
142
|
+
raise Error, "Unknown integration: #{name.inspect}. Use one of: #{integrations.join(', ')}"
|
|
143
|
+
end
|
|
144
|
+
names
|
|
145
|
+
end
|
|
146
|
+
|
|
130
147
|
def ensure_shared_configuration_mutable!
|
|
131
148
|
return unless finalized?
|
|
132
149
|
|
|
@@ -141,7 +158,7 @@ module LlmCostTracker
|
|
|
141
158
|
deep_freeze(nested_value)
|
|
142
159
|
end
|
|
143
160
|
value.frozen? ? value : value.freeze
|
|
144
|
-
when Array
|
|
161
|
+
when Array, Set
|
|
145
162
|
value.each { |nested_value| deep_freeze(nested_value) }
|
|
146
163
|
value.frozen? ? value : value.freeze
|
|
147
164
|
when String
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
|
|
5
|
+
require_relative "check"
|
|
6
|
+
require_relative "probe"
|
|
7
|
+
|
|
8
|
+
module LlmCostTracker
|
|
9
|
+
class Doctor
|
|
10
|
+
class CostDriftCheck
|
|
11
|
+
SAMPLE_SIZE = 200
|
|
12
|
+
EPSILON = BigDecimal("0.00000001")
|
|
13
|
+
|
|
14
|
+
def call
|
|
15
|
+
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
16
|
+
return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
|
|
17
|
+
|
|
18
|
+
sampled = LlmCostTracker::Call
|
|
19
|
+
.where.not(total_cost: nil)
|
|
20
|
+
.where(cost_status: %w[complete free partial])
|
|
21
|
+
.order(id: :desc)
|
|
22
|
+
.limit(SAMPLE_SIZE)
|
|
23
|
+
.pluck(:id, :total_cost, :cost_status)
|
|
24
|
+
return Check.new(:ok, "cost drift", "no priced calls to inspect") if sampled.empty?
|
|
25
|
+
|
|
26
|
+
line_item_totals = LlmCostTracker::CallLineItem
|
|
27
|
+
.where(llm_cost_tracker_call_id: sampled.map(&:first))
|
|
28
|
+
.group(:llm_cost_tracker_call_id)
|
|
29
|
+
.sum(:cost)
|
|
30
|
+
|
|
31
|
+
drifted = sampled.filter_map do |id, total_cost, cost_status|
|
|
32
|
+
line_total = line_item_totals[id] || BigDecimal("0")
|
|
33
|
+
header = BigDecimal(total_cost.to_s)
|
|
34
|
+
next if cost_status == "partial" && header >= line_total
|
|
35
|
+
next if (header - line_total).abs <= EPSILON
|
|
36
|
+
|
|
37
|
+
"##{id}: header=#{header.to_s('F')} line_items=#{line_total.to_s('F')}"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
if drifted.empty?
|
|
41
|
+
return Check.new(:ok, "cost drift",
|
|
42
|
+
"header total_cost matches line items in #{sampled.size} sampled calls")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
Check.new(
|
|
46
|
+
:warn,
|
|
47
|
+
"cost drift",
|
|
48
|
+
"header total_cost diverges from line items in #{drifted.size}/#{sampled.size} sampled calls: " \
|
|
49
|
+
"#{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "check"
|
|
4
|
+
require_relative "probe"
|
|
4
5
|
require_relative "../ingestion"
|
|
5
6
|
|
|
6
7
|
module LlmCostTracker
|
|
@@ -9,34 +10,34 @@ module LlmCostTracker
|
|
|
9
10
|
PENDING_AGE_WARNING_SECONDS = 60
|
|
10
11
|
|
|
11
12
|
def call
|
|
12
|
-
return unless table_exists?("
|
|
13
|
+
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
13
14
|
|
|
14
15
|
missing = missing_parts
|
|
15
16
|
if missing.empty?
|
|
16
|
-
|
|
17
|
+
inbox = inbox_snapshot
|
|
18
|
+
quarantined = inbox.try(:quarantined_count).to_i
|
|
17
19
|
if quarantined.positive?
|
|
18
|
-
return Check.new(:warn, "durable ingestion", "#{quarantined} inbox
|
|
20
|
+
return Check.new(:warn, "durable ingestion", "#{quarantined} inbox entries quarantined after retries")
|
|
19
21
|
end
|
|
20
22
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
oldest_pending_at = pending.try(:oldest_created_at)&.to_time&.utc
|
|
23
|
+
pending_count = inbox.try(:pending_count).to_i
|
|
24
|
+
oldest_pending_at = inbox.try(:oldest_pending_at)&.to_time&.utc
|
|
24
25
|
pending_age = oldest_pending_at && (Time.now.utc - oldest_pending_at)
|
|
25
26
|
if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
|
|
26
27
|
return Check.new(
|
|
27
28
|
:warn,
|
|
28
29
|
"durable ingestion",
|
|
29
|
-
"#{pending_count} inbox
|
|
30
|
+
"#{pending_count} inbox entries pending; oldest pending age #{pending_age.round}s"
|
|
30
31
|
)
|
|
31
32
|
end
|
|
32
33
|
|
|
33
|
-
return Check.new(:ok, "durable ingestion", "inbox and
|
|
34
|
+
return Check.new(:ok, "durable ingestion", "inbox and ingestion lease tables available")
|
|
34
35
|
end
|
|
35
36
|
|
|
36
37
|
Check.new(
|
|
37
38
|
:error,
|
|
38
39
|
"durable ingestion",
|
|
39
|
-
"missing #{missing.join(', ')};
|
|
40
|
+
"missing #{missing.join(', ')}; see docs/upgrading.md for the recovery steps"
|
|
40
41
|
)
|
|
41
42
|
end
|
|
42
43
|
|
|
@@ -44,31 +45,22 @@ module LlmCostTracker
|
|
|
44
45
|
|
|
45
46
|
def missing_parts
|
|
46
47
|
[
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
].
|
|
48
|
+
LlmCostTracker::Ingestion::InboxEntry.table_name,
|
|
49
|
+
LlmCostTracker::Ingestion::Lease.table_name
|
|
50
|
+
].reject { |table| Probe.table_exists?(table) }
|
|
50
51
|
end
|
|
51
52
|
|
|
52
|
-
def
|
|
53
|
-
LlmCostTracker::
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
.count
|
|
64
|
-
rescue StandardError
|
|
65
|
-
0
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
def pending_snapshot
|
|
69
|
-
LlmCostTracker::Ingestion::Event
|
|
70
|
-
.where("attempts < ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
|
|
71
|
-
.select("COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at")
|
|
53
|
+
def inbox_snapshot
|
|
54
|
+
max_attempts = LlmCostTracker::Ingestion::InboxEntry::MAX_ATTEMPTS_BEFORE_QUARANTINE
|
|
55
|
+
LlmCostTracker::Ingestion::InboxEntry
|
|
56
|
+
.select(
|
|
57
|
+
"COALESCE(SUM(CASE WHEN attempts >= #{max_attempts} " \
|
|
58
|
+
"THEN 1 ELSE 0 END), 0) AS quarantined_count, " \
|
|
59
|
+
"COALESCE(SUM(CASE WHEN attempts < #{max_attempts} " \
|
|
60
|
+
"THEN 1 ELSE 0 END), 0) AS pending_count, " \
|
|
61
|
+
"MIN(CASE WHEN attempts < #{max_attempts} " \
|
|
62
|
+
"THEN created_at ELSE NULL END) AS oldest_pending_at"
|
|
63
|
+
)
|
|
72
64
|
.take
|
|
73
65
|
rescue StandardError
|
|
74
66
|
nil
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "check"
|
|
4
|
+
require_relative "probe"
|
|
5
|
+
require_relative "../ledger"
|
|
6
|
+
|
|
7
|
+
module LlmCostTracker
|
|
8
|
+
class Doctor
|
|
9
|
+
class LegacyAuditCheck
|
|
10
|
+
WARNING_PERCENT = 10
|
|
11
|
+
|
|
12
|
+
def call
|
|
13
|
+
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
14
|
+
return unless LlmCostTracker::Call.column_names.include?("pricing_snapshot")
|
|
15
|
+
|
|
16
|
+
counts = LlmCostTracker::Call
|
|
17
|
+
.select(
|
|
18
|
+
"COUNT(*) AS total_count, " \
|
|
19
|
+
"COALESCE(SUM(CASE WHEN pricing_snapshot IS NULL THEN 1 ELSE 0 END), 0) AS missing_count"
|
|
20
|
+
)
|
|
21
|
+
.take
|
|
22
|
+
total = counts.total_count.to_i
|
|
23
|
+
return if total.zero?
|
|
24
|
+
|
|
25
|
+
missing = counts.missing_count.to_i
|
|
26
|
+
return unless (missing * 100) > (total * WARNING_PERCENT)
|
|
27
|
+
|
|
28
|
+
message = "#{missing}/#{total} tracked calls lack pricing_snapshot; " \
|
|
29
|
+
"stored totals remain stable but applied rates cannot be audited"
|
|
30
|
+
Check.new(:warn, "pricing snapshot audit", message)
|
|
31
|
+
rescue StandardError
|
|
32
|
+
nil
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "check"
|
|
4
|
+
require_relative "probe"
|
|
5
|
+
require_relative "../ledger"
|
|
6
|
+
|
|
7
|
+
module LlmCostTracker
|
|
8
|
+
class Doctor
|
|
9
|
+
class LegacyBillingStatusCheck
|
|
10
|
+
def call
|
|
11
|
+
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
12
|
+
return unless LlmCostTracker::Call.column_names.include?("cost_status")
|
|
13
|
+
|
|
14
|
+
return unless LlmCostTracker::Call.where(cost_status: nil).exists?
|
|
15
|
+
|
|
16
|
+
Check.new(:warn, "cost status", "legacy rows without cost_status remain; new rows will populate it")
|
|
17
|
+
rescue StandardError
|
|
18
|
+
nil
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -8,7 +8,7 @@ module LlmCostTracker
|
|
|
8
8
|
class Doctor
|
|
9
9
|
class PriceCheck
|
|
10
10
|
STALE_AFTER_DAYS = 30
|
|
11
|
-
REFRESH_COMMAND = "
|
|
11
|
+
REFRESH_COMMAND = "refresh the source-controlled prices file with bin/rails llm_cost_tracker:prices:refresh"
|
|
12
12
|
|
|
13
13
|
def call
|
|
14
14
|
path = LlmCostTracker.configuration.prices_file
|
|
@@ -48,7 +48,7 @@ module LlmCostTracker
|
|
|
48
48
|
Check.new(
|
|
49
49
|
:warn,
|
|
50
50
|
"prices",
|
|
51
|
-
"using bundled prices updated_at=#{updated_at};
|
|
51
|
+
"using bundled prices updated_at=#{updated_at}; commit a prices_file for production releases"
|
|
52
52
|
)
|
|
53
53
|
end
|
|
54
54
|
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
|
|
5
|
+
require_relative "check"
|
|
6
|
+
require_relative "probe"
|
|
7
|
+
|
|
8
|
+
module LlmCostTracker
|
|
9
|
+
class Doctor
|
|
10
|
+
class PricingSnapshotDriftCheck
|
|
11
|
+
SAMPLE_SIZE = 200
|
|
12
|
+
EPSILON = BigDecimal("0.00000001")
|
|
13
|
+
|
|
14
|
+
def call
|
|
15
|
+
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
16
|
+
return unless Probe.table_exists?("llm_cost_tracker_call_line_items")
|
|
17
|
+
|
|
18
|
+
sampled_ids = LlmCostTracker::Call
|
|
19
|
+
.where.not(pricing_snapshot: nil)
|
|
20
|
+
.where(cost_status: %w[complete free])
|
|
21
|
+
.order(id: :desc)
|
|
22
|
+
.limit(SAMPLE_SIZE)
|
|
23
|
+
.pluck(:id)
|
|
24
|
+
return Check.new(:ok, "pricing snapshot drift", "no snapshotted calls to inspect") if sampled_ids.empty?
|
|
25
|
+
|
|
26
|
+
calls_by_id = LlmCostTracker::Call.where(id: sampled_ids).index_by(&:id)
|
|
27
|
+
line_items_by_call = LlmCostTracker::CallLineItem
|
|
28
|
+
.where(llm_cost_tracker_call_id: sampled_ids, unit: "token")
|
|
29
|
+
.group_by(&:llm_cost_tracker_call_id)
|
|
30
|
+
|
|
31
|
+
drifted = sampled_ids.flat_map do |id|
|
|
32
|
+
call = calls_by_id[id]
|
|
33
|
+
rates = rates_for(call.pricing_snapshot)
|
|
34
|
+
next [] if rates.nil? || rates.empty?
|
|
35
|
+
|
|
36
|
+
(line_items_by_call[id] || []).filter_map { |item| drift_message_for(item, rates, call_id: id) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
return ok_check(sampled_ids.size) if drifted.empty?
|
|
40
|
+
|
|
41
|
+
Check.new(
|
|
42
|
+
:warn,
|
|
43
|
+
"pricing snapshot drift",
|
|
44
|
+
"line item cost diverges from pricing_snapshot rate in #{drifted.size} cases across " \
|
|
45
|
+
"#{sampled_ids.size} sampled calls: #{drifted.first(5).join('; ')}#{'; ...' if drifted.size > 5}"
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def ok_check(sample_size)
|
|
52
|
+
Check.new(:ok, "pricing snapshot drift",
|
|
53
|
+
"line item costs match pricing_snapshot rates in #{sample_size} sampled calls")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def rates_for(snapshot)
|
|
57
|
+
rates = snapshot.is_a?(Hash) ? (snapshot["rates"] || snapshot[:rates]) : nil
|
|
58
|
+
rates.is_a?(Hash) ? rates : nil
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def drift_message_for(line_item, rates, call_id:)
|
|
62
|
+
return nil unless line_item.price_key
|
|
63
|
+
|
|
64
|
+
rate = rates[line_item.price_key.to_s] || rates[line_item.price_key.to_sym]
|
|
65
|
+
return nil unless rate.is_a?(Hash)
|
|
66
|
+
|
|
67
|
+
rate_amount = decimal(rate["amount"] || rate[:amount])
|
|
68
|
+
rate_quantity = decimal(rate["quantity"] || rate[:quantity])
|
|
69
|
+
return nil if rate_amount.nil? || rate_quantity.nil? || rate_quantity.zero?
|
|
70
|
+
|
|
71
|
+
expected = (decimal(line_item.quantity) * rate_amount) / rate_quantity
|
|
72
|
+
actual = decimal(line_item.cost) || BigDecimal("0")
|
|
73
|
+
return nil if (expected - actual).abs <= EPSILON
|
|
74
|
+
|
|
75
|
+
"##{call_id}.#{line_item.price_key}: expected=#{expected.round(8).to_s('F')} stored=#{actual.to_s('F')}"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def decimal(value)
|
|
79
|
+
return nil if value.nil?
|
|
80
|
+
|
|
81
|
+
BigDecimal(value.to_s)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../ledger"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
class Doctor
|
|
7
|
+
module Probe
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def table_exists?(name)
|
|
11
|
+
LlmCostTracker::Call.connection.data_source_exists?(name)
|
|
12
|
+
rescue StandardError
|
|
13
|
+
false
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "check"
|
|
4
|
+
require_relative "probe"
|
|
5
|
+
require_relative "../ledger"
|
|
6
|
+
|
|
7
|
+
module LlmCostTracker
|
|
8
|
+
class Doctor
|
|
9
|
+
class SchemaCheck
|
|
10
|
+
def initialize(name:, schema:, table:)
|
|
11
|
+
@name = name
|
|
12
|
+
@schema = schema
|
|
13
|
+
@table = table
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call
|
|
17
|
+
return unless Probe.table_exists?("llm_cost_tracker_calls")
|
|
18
|
+
|
|
19
|
+
errors = @schema.current_schema_errors
|
|
20
|
+
return Check.new(:ok, @name, "#{@table} exists") if errors.empty?
|
|
21
|
+
|
|
22
|
+
Check.new(
|
|
23
|
+
:error,
|
|
24
|
+
@name,
|
|
25
|
+
"current schema required; #{errors.join('; ')}; " \
|
|
26
|
+
"run bin/rails generate llm_cost_tracker:install && bin/rails db:migrate"
|
|
27
|
+
)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -2,24 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "ledger"
|
|
4
4
|
require_relative "doctor/check"
|
|
5
|
+
require_relative "doctor/probe"
|
|
5
6
|
require_relative "doctor/ingestion_check"
|
|
7
|
+
require_relative "doctor/legacy_audit_check"
|
|
8
|
+
require_relative "doctor/legacy_billing_status_check"
|
|
6
9
|
require_relative "doctor/price_check"
|
|
7
|
-
require_relative "
|
|
10
|
+
require_relative "doctor/schema_check"
|
|
11
|
+
require_relative "doctor/cost_drift_check"
|
|
12
|
+
require_relative "doctor/pricing_snapshot_drift_check"
|
|
8
13
|
|
|
9
14
|
module LlmCostTracker
|
|
10
15
|
class Doctor
|
|
11
|
-
COLUMN_GENERATORS = {
|
|
12
|
-
"event_id" => "bin/rails generate llm_cost_tracker:add_ingestion",
|
|
13
|
-
"latency_ms" => "bin/rails generate llm_cost_tracker:add_latency_ms",
|
|
14
|
-
"stream" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
15
|
-
"usage_source" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
16
|
-
"provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id"
|
|
17
|
-
}.merge(
|
|
18
|
-
Generators::AddTokenUsageGenerator::COLUMN_NAMES.to_h do |column|
|
|
19
|
-
[column, "bin/rails generate llm_cost_tracker:add_token_usage"]
|
|
20
|
-
end
|
|
21
|
-
).freeze
|
|
22
|
-
|
|
23
16
|
class << self
|
|
24
17
|
def call
|
|
25
18
|
new.checks
|
|
@@ -44,7 +37,17 @@ module LlmCostTracker
|
|
|
44
37
|
active_record_check,
|
|
45
38
|
table_check,
|
|
46
39
|
column_check,
|
|
47
|
-
|
|
40
|
+
SchemaCheck.new(name: "call line items", schema: Ledger::Schema::CallLineItems,
|
|
41
|
+
table: "llm_cost_tracker_call_line_items").call,
|
|
42
|
+
SchemaCheck.new(name: "call tags", schema: Ledger::Schema::CallTags,
|
|
43
|
+
table: "llm_cost_tracker_call_tags").call,
|
|
44
|
+
SchemaCheck.new(name: "provider invoices", schema: Ledger::Schema::ProviderInvoices,
|
|
45
|
+
table: "llm_cost_tracker_provider_invoices").call,
|
|
46
|
+
CostDriftCheck.new.call,
|
|
47
|
+
PricingSnapshotDriftCheck.new.call,
|
|
48
|
+
LegacyBillingStatusCheck.new.call,
|
|
49
|
+
LegacyAuditCheck.new.call,
|
|
50
|
+
call_rollups_check,
|
|
48
51
|
IngestionCheck.new.call,
|
|
49
52
|
PriceCheck.new.call,
|
|
50
53
|
calls_check
|
|
@@ -68,7 +71,7 @@ module LlmCostTracker
|
|
|
68
71
|
return Check.new(
|
|
69
72
|
:ok,
|
|
70
73
|
"capture",
|
|
71
|
-
"SDK integrations enabled: #{config.instrumented_integrations.join(', ')}"
|
|
74
|
+
"SDK integrations enabled: #{config.instrumented_integrations.to_a.join(', ')}"
|
|
72
75
|
)
|
|
73
76
|
end
|
|
74
77
|
|
|
@@ -93,68 +96,63 @@ module LlmCostTracker
|
|
|
93
96
|
|
|
94
97
|
def table_check
|
|
95
98
|
return unless active_record_available?
|
|
96
|
-
return Check.new(:ok, "
|
|
99
|
+
return Check.new(:ok, "llm_cost_tracker_calls", "table exists") if llm_cost_tracker_calls_table?
|
|
97
100
|
|
|
98
101
|
Check.new(
|
|
99
102
|
:error,
|
|
100
|
-
"
|
|
103
|
+
"llm_cost_tracker_calls",
|
|
101
104
|
"missing; run bin/rails generate llm_cost_tracker:install && bin/rails db:migrate"
|
|
102
105
|
)
|
|
103
106
|
end
|
|
104
107
|
|
|
105
108
|
def column_check
|
|
106
|
-
return unless
|
|
109
|
+
return unless llm_cost_tracker_calls_table?
|
|
107
110
|
|
|
108
111
|
errors = LlmCostTracker::Ledger::Schema::Calls.current_schema_errors
|
|
109
|
-
return Check.new(:ok, "
|
|
110
|
-
|
|
111
|
-
missing = LlmCostTracker::Ledger::Schema::Calls.missing_current_schema_columns
|
|
112
|
-
generators = missing.filter_map { |column| COLUMN_GENERATORS[column] }.uniq
|
|
113
|
-
message = "current schema required; #{errors.join('; ')}"
|
|
114
|
-
message = "#{message}; run #{generators.join(' && ')} && bin/rails db:migrate" if generators.any?
|
|
112
|
+
return Check.new(:ok, "llm_cost_tracker_calls columns", "current") if errors.empty?
|
|
115
113
|
|
|
116
|
-
Check.new(
|
|
114
|
+
Check.new(
|
|
115
|
+
:error,
|
|
116
|
+
"llm_cost_tracker_calls columns",
|
|
117
|
+
"schema mismatch: #{errors.join('; ')}; see docs/upgrading.md"
|
|
118
|
+
)
|
|
117
119
|
end
|
|
118
120
|
|
|
119
|
-
def
|
|
120
|
-
return unless
|
|
121
|
+
def call_rollups_check
|
|
122
|
+
return unless llm_cost_tracker_calls_table?
|
|
121
123
|
|
|
122
|
-
errors = LlmCostTracker::Ledger::Schema::
|
|
123
|
-
return Check.new(:ok, "
|
|
124
|
+
errors = LlmCostTracker::Ledger::Schema::CallRollups.current_schema_errors
|
|
125
|
+
return Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists") if errors.empty?
|
|
124
126
|
|
|
125
127
|
Check.new(
|
|
126
128
|
:error,
|
|
127
|
-
"
|
|
128
|
-
"
|
|
129
|
-
"run bin/rails generate llm_cost_tracker:add_period_totals && bin/rails db:migrate"
|
|
129
|
+
"call rollups",
|
|
130
|
+
"schema mismatch: #{errors.join('; ')}; see docs/upgrading.md"
|
|
130
131
|
)
|
|
131
132
|
end
|
|
132
133
|
|
|
133
134
|
def calls_check
|
|
134
|
-
return unless
|
|
135
|
+
return unless llm_cost_tracker_calls_table?
|
|
135
136
|
|
|
136
|
-
|
|
137
|
+
snapshot = LlmCostTracker::Call
|
|
138
|
+
.select("COUNT(*) AS tracked_call_count, MAX(tracked_at) AS latest_tracked_at")
|
|
139
|
+
.take
|
|
140
|
+
count = snapshot.tracked_call_count.to_i
|
|
137
141
|
return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
|
|
138
142
|
|
|
139
|
-
|
|
143
|
+
latest_at = snapshot.latest_tracked_at
|
|
144
|
+
latest_at = latest_at.to_time if latest_at.respond_to?(:to_time)
|
|
145
|
+
latest = latest_at&.utc&.iso8601
|
|
140
146
|
Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
|
|
141
147
|
end
|
|
142
148
|
|
|
143
149
|
def active_record_available?
|
|
144
|
-
LlmCostTracker::
|
|
150
|
+
LlmCostTracker::Call.connection
|
|
145
151
|
true
|
|
146
152
|
rescue LoadError, StandardError
|
|
147
153
|
false
|
|
148
154
|
end
|
|
149
155
|
|
|
150
|
-
def
|
|
151
|
-
active_record_available? && table_exists?("llm_api_calls")
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
def table_exists?(name)
|
|
155
|
-
LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
|
|
156
|
-
rescue StandardError
|
|
157
|
-
false
|
|
158
|
-
end
|
|
156
|
+
def llm_cost_tracker_calls_table? = active_record_available? && Probe.table_exists?("llm_cost_tracker_calls")
|
|
159
157
|
end
|
|
160
158
|
end
|