llm_cost_tracker 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/README.md +21 -16
- data/app/assets/llm_cost_tracker/application.css +3 -0
- data/app/controllers/llm_cost_tracker/application_controller.rb +22 -4
- data/app/controllers/llm_cost_tracker/calls_controller.rb +6 -11
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +5 -1
- data/app/controllers/llm_cost_tracker/models_controller.rb +0 -1
- data/app/controllers/llm_cost_tracker/tags_controller.rb +1 -8
- data/app/helpers/llm_cost_tracker/application_helper.rb +2 -1
- data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +1 -2
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +10 -27
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +58 -0
- data/app/models/llm_cost_tracker/ingestion/event.rb +13 -0
- data/app/models/llm_cost_tracker/ingestion/lease.rb +11 -0
- data/app/models/llm_cost_tracker/ledger/call.rb +45 -0
- data/app/models/llm_cost_tracker/ledger/call_metrics.rb +66 -0
- data/app/models/llm_cost_tracker/ledger/period/grouping.rb +71 -0
- data/app/models/llm_cost_tracker/ledger/period/total.rb +13 -0
- data/app/models/llm_cost_tracker/ledger/tags/accessors.rb +19 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +111 -94
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +2 -2
- data/app/services/llm_cost_tracker/dashboard/filter.rb +7 -18
- data/app/services/llm_cost_tracker/dashboard/overview_stats.rb +58 -67
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +59 -0
- data/app/services/llm_cost_tracker/dashboard/params.rb +26 -0
- data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +18 -20
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +4 -13
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +28 -61
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +8 -21
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +12 -47
- data/app/views/llm_cost_tracker/calls/index.html.erb +12 -18
- data/app/views/llm_cost_tracker/calls/show.html.erb +30 -32
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +17 -19
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +108 -135
- data/app/views/llm_cost_tracker/models/index.html.erb +8 -9
- data/app/views/llm_cost_tracker/shared/setup_required.html.erb +13 -2
- data/app/views/llm_cost_tracker/tags/show.html.erb +20 -20
- data/lib/llm_cost_tracker/budget.rb +8 -20
- data/lib/llm_cost_tracker/capture/stream.rb +9 -0
- data/lib/llm_cost_tracker/capture/stream_collector.rb +189 -0
- data/lib/llm_cost_tracker/{integrations → capture}/stream_tracker.rb +41 -73
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +3 -7
- data/lib/llm_cost_tracker/configuration.rb +33 -36
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +61 -0
- data/lib/llm_cost_tracker/doctor/check.rb +7 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +22 -59
- data/lib/llm_cost_tracker/doctor/price_check.rb +60 -0
- data/lib/llm_cost_tracker/doctor.rb +63 -71
- data/lib/llm_cost_tracker/errors.rb +4 -15
- data/lib/llm_cost_tracker/event.rb +6 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_token_usage_generator.rb +42 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +2 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +7 -7
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +3 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_token_usage_to_llm_api_calls.rb.erb +22 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +9 -14
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +0 -4
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_cost_precision.rb.erb +12 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb +2 -2
- data/lib/llm_cost_tracker/{storage/active_record_inbox_batch.rb → ingestion/batch.rb} +21 -20
- data/lib/llm_cost_tracker/ingestion/inbox.rb +105 -0
- data/lib/llm_cost_tracker/{storage/active_record_ingestor_lease.rb → ingestion/lease_claim.rb} +5 -7
- data/lib/llm_cost_tracker/{storage/active_record_ingestor.rb → ingestion/worker.rb} +38 -48
- data/lib/llm_cost_tracker/ingestion.rb +129 -0
- data/lib/llm_cost_tracker/integrations/anthropic.rb +66 -31
- data/lib/llm_cost_tracker/integrations/base.rb +73 -34
- data/lib/llm_cost_tracker/integrations/openai.rb +43 -37
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +40 -30
- data/lib/llm_cost_tracker/integrations.rb +43 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +66 -0
- data/lib/llm_cost_tracker/{storage/active_record_periods.rb → ledger/period.rb} +2 -2
- data/lib/llm_cost_tracker/ledger/rollups/batch.rb +43 -0
- data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb +46 -0
- data/lib/llm_cost_tracker/ledger/rollups.rb +87 -0
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +51 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +101 -0
- data/lib/llm_cost_tracker/ledger/schema/period_totals.rb +32 -0
- data/lib/llm_cost_tracker/ledger/store.rb +60 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +29 -0
- data/lib/llm_cost_tracker/ledger/tags/sql.rb +33 -0
- data/lib/llm_cost_tracker/ledger.rb +13 -0
- data/lib/llm_cost_tracker/logging.rb +3 -6
- data/lib/llm_cost_tracker/middleware/faraday.rb +88 -46
- data/lib/llm_cost_tracker/parsers/anthropic.rb +62 -29
- data/lib/llm_cost_tracker/parsers/base.rb +12 -21
- data/lib/llm_cost_tracker/parsers/gemini.rb +50 -25
- data/lib/llm_cost_tracker/parsers/openai.rb +27 -5
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +14 -4
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +58 -25
- data/lib/llm_cost_tracker/parsers/sse.rb +4 -7
- data/lib/llm_cost_tracker/parsers.rb +20 -0
- data/lib/llm_cost_tracker/prices.json +361 -36
- data/lib/llm_cost_tracker/pricing/components.rb +37 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +46 -50
- data/lib/llm_cost_tracker/pricing/explainer.rb +25 -30
- data/lib/llm_cost_tracker/pricing/lookup.rb +67 -46
- data/lib/llm_cost_tracker/pricing/registry.rb +156 -0
- data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +107 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +53 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_loader.rb +63 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +31 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +159 -0
- data/lib/llm_cost_tracker/pricing/unknown.rb +46 -0
- data/lib/llm_cost_tracker/pricing.rb +33 -32
- data/lib/llm_cost_tracker/railtie.rb +7 -8
- data/lib/llm_cost_tracker/report/data.rb +72 -0
- data/lib/llm_cost_tracker/report/formatter.rb +69 -0
- data/lib/llm_cost_tracker/report.rb +8 -8
- data/lib/llm_cost_tracker/retention.rb +27 -10
- data/lib/llm_cost_tracker/tags/context.rb +35 -0
- data/lib/llm_cost_tracker/tags/key.rb +18 -0
- data/lib/llm_cost_tracker/tags/sanitizer.rb +68 -0
- data/lib/llm_cost_tracker/token_usage.rb +67 -0
- data/lib/llm_cost_tracker/tracker.rb +39 -69
- data/lib/llm_cost_tracker/usage_capture.rb +37 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +56 -78
- data/lib/tasks/llm_cost_tracker.rake +18 -13
- metadata +54 -58
- data/app/services/llm_cost_tracker/dashboard/data_quality_aggregate.rb +0 -81
- data/app/services/llm_cost_tracker/pagination.rb +0 -57
- data/lib/llm_cost_tracker/active_record_adapter.rb +0 -53
- data/lib/llm_cost_tracker/capture_verifier.rb +0 -64
- data/lib/llm_cost_tracker/cost.rb +0 -12
- data/lib/llm_cost_tracker/doctor/capture_check.rb +0 -39
- data/lib/llm_cost_tracker/event_metadata.rb +0 -52
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_usage_breakdown_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_usage_breakdown_to_llm_api_calls.rb.erb +0 -29
- data/lib/llm_cost_tracker/inbox_event.rb +0 -9
- data/lib/llm_cost_tracker/ingestor_lease.rb +0 -9
- data/lib/llm_cost_tracker/integrations/object_reader.rb +0 -56
- data/lib/llm_cost_tracker/integrations/registry.rb +0 -71
- data/lib/llm_cost_tracker/llm_api_call.rb +0 -60
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +0 -63
- data/lib/llm_cost_tracker/parameter_hash.rb +0 -33
- data/lib/llm_cost_tracker/parsed_usage.rb +0 -72
- data/lib/llm_cost_tracker/parsers/registry.rb +0 -58
- data/lib/llm_cost_tracker/period_grouping.rb +0 -67
- data/lib/llm_cost_tracker/period_total.rb +0 -9
- data/lib/llm_cost_tracker/price_freshness.rb +0 -38
- data/lib/llm_cost_tracker/price_registry.rb +0 -144
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +0 -104
- data/lib/llm_cost_tracker/price_sync/registry_diff.rb +0 -51
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +0 -61
- data/lib/llm_cost_tracker/price_sync/registry_writer.rb +0 -29
- data/lib/llm_cost_tracker/price_sync.rb +0 -144
- data/lib/llm_cost_tracker/report_data.rb +0 -94
- data/lib/llm_cost_tracker/report_formatter.rb +0 -67
- data/lib/llm_cost_tracker/request_url.rb +0 -20
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +0 -167
- data/lib/llm_cost_tracker/storage/active_record_connection_cleanup.rb +0 -13
- data/lib/llm_cost_tracker/storage/active_record_inbox.rb +0 -160
- data/lib/llm_cost_tracker/storage/active_record_period_totals.rb +0 -84
- data/lib/llm_cost_tracker/storage/active_record_rollup_batch.rb +0 -41
- data/lib/llm_cost_tracker/storage/active_record_rollup_upsert_sql.rb +0 -42
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +0 -146
- data/lib/llm_cost_tracker/storage/active_record_store.rb +0 -145
- data/lib/llm_cost_tracker/storage/writer.rb +0 -35
- data/lib/llm_cost_tracker/stream_capture.rb +0 -7
- data/lib/llm_cost_tracker/stream_collector.rb +0 -199
- data/lib/llm_cost_tracker/tag_accessors.rb +0 -15
- data/lib/llm_cost_tracker/tag_context.rb +0 -52
- data/lib/llm_cost_tracker/tag_key.rb +0 -16
- data/lib/llm_cost_tracker/tag_query.rb +0 -43
- data/lib/llm_cost_tracker/tag_sanitizer.rb +0 -81
- data/lib/llm_cost_tracker/tag_sql.rb +0 -34
- data/lib/llm_cost_tracker/tags_column.rb +0 -105
- data/lib/llm_cost_tracker/unknown_pricing.rb +0 -54
- data/lib/llm_cost_tracker/usage_breakdown.rb +0 -30
- data/lib/llm_cost_tracker/value_helpers.rb +0 -40
|
@@ -1,44 +1,40 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
require_relative "check"
|
|
4
|
+
require_relative "../ingestion"
|
|
4
5
|
|
|
5
6
|
module LlmCostTracker
|
|
6
7
|
class Doctor
|
|
7
8
|
class IngestionCheck
|
|
8
9
|
PENDING_AGE_WARNING_SECONDS = 60
|
|
9
10
|
|
|
10
|
-
def self.call(check_class)
|
|
11
|
-
new(check_class).call
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def initialize(check_class)
|
|
15
|
-
@check_class = check_class
|
|
16
|
-
end
|
|
17
|
-
|
|
18
11
|
def call
|
|
19
|
-
return unless
|
|
12
|
+
return unless table_exists?("llm_api_calls")
|
|
20
13
|
|
|
21
14
|
missing = missing_parts
|
|
22
15
|
if missing.empty?
|
|
23
16
|
quarantined = quarantined_count
|
|
24
17
|
if quarantined.positive?
|
|
25
|
-
return
|
|
18
|
+
return Check.new(:warn, "durable ingestion", "#{quarantined} inbox events quarantined after retries")
|
|
26
19
|
end
|
|
27
20
|
|
|
28
21
|
pending = pending_snapshot
|
|
29
|
-
|
|
30
|
-
|
|
22
|
+
pending_count = pending.try(:pending_count).to_i
|
|
23
|
+
oldest_pending_at = pending.try(:oldest_created_at)&.to_time&.utc
|
|
24
|
+
pending_age = oldest_pending_at && (Time.now.utc - oldest_pending_at)
|
|
25
|
+
if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
|
|
26
|
+
return Check.new(
|
|
31
27
|
:warn,
|
|
32
28
|
"durable ingestion",
|
|
33
|
-
"#{
|
|
29
|
+
"#{pending_count} inbox events pending; oldest pending age #{pending_age.round}s"
|
|
34
30
|
)
|
|
35
31
|
end
|
|
36
32
|
|
|
37
|
-
return
|
|
33
|
+
return Check.new(:ok, "durable ingestion", "inbox and ingestor lease tables available")
|
|
38
34
|
end
|
|
39
35
|
|
|
40
|
-
|
|
41
|
-
:
|
|
36
|
+
Check.new(
|
|
37
|
+
:error,
|
|
42
38
|
"durable ingestion",
|
|
43
39
|
"missing #{missing.join(', ')}; run bin/rails generate llm_cost_tracker:add_ingestion && bin/rails db:migrate"
|
|
44
40
|
)
|
|
@@ -46,69 +42,36 @@ module LlmCostTracker
|
|
|
46
42
|
|
|
47
43
|
private
|
|
48
44
|
|
|
49
|
-
attr_reader :check_class
|
|
50
|
-
|
|
51
45
|
def missing_parts
|
|
52
46
|
[
|
|
53
|
-
column_names("llm_api_calls").include?("event_id") ? nil : "llm_api_calls.event_id",
|
|
54
47
|
table_exists?("llm_cost_tracker_inbox_events") ? nil : "llm_cost_tracker_inbox_events",
|
|
55
48
|
table_exists?("llm_cost_tracker_ingestor_leases") ? nil : "llm_cost_tracker_ingestor_leases"
|
|
56
49
|
].compact
|
|
57
50
|
end
|
|
58
51
|
|
|
59
|
-
def llm_api_calls_table? = table_exists?("llm_api_calls")
|
|
60
|
-
|
|
61
52
|
def table_exists?(name)
|
|
62
|
-
LlmCostTracker::
|
|
53
|
+
LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
|
|
63
54
|
rescue StandardError
|
|
64
55
|
false
|
|
65
56
|
end
|
|
66
57
|
|
|
67
|
-
def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
|
|
68
|
-
|
|
69
58
|
def quarantined_count
|
|
70
59
|
return 0 unless table_exists?("llm_cost_tracker_inbox_events")
|
|
71
60
|
|
|
72
|
-
LlmCostTracker::
|
|
61
|
+
LlmCostTracker::Ingestion::Event
|
|
62
|
+
.where("attempts >= ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
|
|
63
|
+
.count
|
|
73
64
|
rescue StandardError
|
|
74
65
|
0
|
|
75
66
|
end
|
|
76
67
|
|
|
77
|
-
def quarantined_sql
|
|
78
|
-
table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
|
|
79
|
-
"SELECT COUNT(*) FROM #{table} WHERE attempts >= #{max_attempts}"
|
|
80
|
-
end
|
|
81
|
-
|
|
82
68
|
def pending_snapshot
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
}
|
|
69
|
+
LlmCostTracker::Ingestion::Event
|
|
70
|
+
.where("attempts < ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
|
|
71
|
+
.select("COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at")
|
|
72
|
+
.take
|
|
88
73
|
rescue StandardError
|
|
89
|
-
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
def pending_sql
|
|
93
|
-
table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
|
|
94
|
-
"SELECT COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at " \
|
|
95
|
-
"FROM #{table} WHERE attempts < #{max_attempts}"
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
def stale_pending?(pending)
|
|
99
|
-
pending.fetch(:count).positive? &&
|
|
100
|
-
pending.fetch(:oldest_at) &&
|
|
101
|
-
pending_age(pending) >= PENDING_AGE_WARNING_SECONDS
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
def pending_age(pending) = Time.now.utc - pending.fetch(:oldest_at)
|
|
105
|
-
|
|
106
|
-
def max_attempts
|
|
107
|
-
if defined?(LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS)
|
|
108
|
-
LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS
|
|
109
|
-
else
|
|
110
|
-
5
|
|
111
|
-
end
|
|
74
|
+
nil
|
|
112
75
|
end
|
|
113
76
|
end
|
|
114
77
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
|
|
5
|
+
require_relative "check"
|
|
6
|
+
|
|
7
|
+
module LlmCostTracker
|
|
8
|
+
class Doctor
|
|
9
|
+
class PriceCheck
|
|
10
|
+
STALE_AFTER_DAYS = 30
|
|
11
|
+
REFRESH_COMMAND = "run bin/rails llm_cost_tracker:prices:refresh"
|
|
12
|
+
|
|
13
|
+
def call
|
|
14
|
+
path = LlmCostTracker.configuration.prices_file
|
|
15
|
+
return bundled_check unless path
|
|
16
|
+
|
|
17
|
+
count = LlmCostTracker::Pricing::Registry.file_prices(path).size
|
|
18
|
+
metadata = LlmCostTracker::Pricing::Registry.file_metadata(path)
|
|
19
|
+
updated_at = metadata["updated_at"] || metadata[:updated_at]
|
|
20
|
+
return configured_check(:warn, path, count, "metadata.updated_at missing; #{REFRESH_COMMAND}") unless updated_at
|
|
21
|
+
|
|
22
|
+
age_days = (Date.today - Date.iso8601(updated_at.to_s)).to_i
|
|
23
|
+
if age_days > STALE_AFTER_DAYS
|
|
24
|
+
return configured_check(
|
|
25
|
+
:warn,
|
|
26
|
+
path,
|
|
27
|
+
count,
|
|
28
|
+
"updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days; #{REFRESH_COMMAND}"
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
configured_check(:ok, path, count, "updated_at=#{updated_at}")
|
|
33
|
+
rescue Date::Error
|
|
34
|
+
configured_check(
|
|
35
|
+
:warn,
|
|
36
|
+
path,
|
|
37
|
+
count,
|
|
38
|
+
"metadata.updated_at=#{updated_at.inspect} is invalid; #{REFRESH_COMMAND}"
|
|
39
|
+
)
|
|
40
|
+
rescue LlmCostTracker::Error => e
|
|
41
|
+
Check.new(:error, "prices", e.message)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def bundled_check
|
|
47
|
+
updated_at = LlmCostTracker::Pricing::Registry.metadata.fetch("updated_at", "unknown")
|
|
48
|
+
Check.new(
|
|
49
|
+
:warn,
|
|
50
|
+
"prices",
|
|
51
|
+
"using bundled prices updated_at=#{updated_at}; configure prices_file for production"
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def configured_check(status, path, count, freshness)
|
|
56
|
+
Check.new(status, "prices", "loaded #{count} models from #{path}; #{freshness}")
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -1,53 +1,52 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative "
|
|
4
|
-
require_relative "doctor/
|
|
3
|
+
require_relative "ledger"
|
|
4
|
+
require_relative "doctor/check"
|
|
5
5
|
require_relative "doctor/ingestion_check"
|
|
6
|
+
require_relative "doctor/price_check"
|
|
7
|
+
require_relative "generators/llm_cost_tracker/add_token_usage_generator"
|
|
6
8
|
|
|
7
9
|
module LlmCostTracker
|
|
8
10
|
class Doctor
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
FEATURE_COLUMNS = {
|
|
11
|
+
COLUMN_GENERATORS = {
|
|
12
|
+
"event_id" => "bin/rails generate llm_cost_tracker:add_ingestion",
|
|
12
13
|
"latency_ms" => "bin/rails generate llm_cost_tracker:add_latency_ms",
|
|
13
14
|
"stream" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
14
15
|
"usage_source" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
15
|
-
"provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id"
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
"provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id"
|
|
17
|
+
}.merge(
|
|
18
|
+
Generators::AddTokenUsageGenerator::COLUMN_NAMES.to_h do |column|
|
|
19
|
+
[column, "bin/rails generate llm_cost_tracker:add_token_usage"]
|
|
20
|
+
end
|
|
21
|
+
).freeze
|
|
21
22
|
|
|
22
23
|
class << self
|
|
23
|
-
def call
|
|
24
|
+
def call
|
|
25
|
+
new.checks
|
|
26
|
+
end
|
|
24
27
|
|
|
25
28
|
def report(checks = call)
|
|
26
|
-
(["LLM Cost Tracker doctor"] + checks.map
|
|
29
|
+
(["LLM Cost Tracker doctor"] + checks.map do |check|
|
|
30
|
+
"[#{check.status}] #{check.name}: #{check.message}"
|
|
31
|
+
end).join("\n")
|
|
27
32
|
end
|
|
28
33
|
|
|
29
34
|
def healthy?(checks = call)
|
|
30
35
|
checks.none? { |check| check.status == :error }
|
|
31
36
|
end
|
|
32
|
-
|
|
33
|
-
private
|
|
34
|
-
|
|
35
|
-
def format_check(check)
|
|
36
|
-
"[#{check.status}] #{check.name}: #{check.message}"
|
|
37
|
-
end
|
|
38
37
|
end
|
|
39
38
|
|
|
40
39
|
def checks
|
|
41
40
|
[
|
|
42
41
|
configuration_check,
|
|
43
|
-
|
|
42
|
+
capture_check,
|
|
44
43
|
*integration_checks,
|
|
45
44
|
active_record_check,
|
|
46
45
|
table_check,
|
|
47
46
|
column_check,
|
|
48
47
|
period_totals_check,
|
|
49
|
-
IngestionCheck.call
|
|
50
|
-
|
|
48
|
+
IngestionCheck.new.call,
|
|
49
|
+
PriceCheck.new.call,
|
|
51
50
|
calls_check
|
|
52
51
|
].compact
|
|
53
52
|
end
|
|
@@ -59,8 +58,29 @@ module LlmCostTracker
|
|
|
59
58
|
Check.new(:ok, "configuration", "active_record ledger enabled=#{config.enabled}")
|
|
60
59
|
end
|
|
61
60
|
|
|
61
|
+
def capture_check
|
|
62
|
+
config = LlmCostTracker.configuration
|
|
63
|
+
unless config.enabled
|
|
64
|
+
return Check.new(:warn, "capture", "tracking is disabled; set config.enabled = true to record calls")
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
if config.instrumented_integrations.any?
|
|
68
|
+
return Check.new(
|
|
69
|
+
:ok,
|
|
70
|
+
"capture",
|
|
71
|
+
"SDK integrations enabled: #{config.instrumented_integrations.join(', ')}"
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
Check.new(
|
|
76
|
+
:ok,
|
|
77
|
+
"capture",
|
|
78
|
+
"no SDK integrations enabled; Faraday middleware and manual capture remain available"
|
|
79
|
+
)
|
|
80
|
+
end
|
|
81
|
+
|
|
62
82
|
def integration_checks
|
|
63
|
-
LlmCostTracker::Integrations
|
|
83
|
+
LlmCostTracker::Integrations.checks.map do |check|
|
|
64
84
|
Check.new(check.status, check.name.to_s, check.message)
|
|
65
85
|
end
|
|
66
86
|
end
|
|
@@ -85,63 +105,43 @@ module LlmCostTracker
|
|
|
85
105
|
def column_check
|
|
86
106
|
return unless llm_api_calls_table?
|
|
87
107
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
missing_features = FEATURE_COLUMNS.keys - columns
|
|
91
|
-
if missing_core.any?
|
|
92
|
-
return Check.new(:error, "llm_api_calls columns", "missing core columns: #{missing_core.join(', ')}")
|
|
93
|
-
end
|
|
94
|
-
if missing_features.any?
|
|
95
|
-
return Check.new(
|
|
96
|
-
:warn,
|
|
97
|
-
"llm_api_calls columns",
|
|
98
|
-
"missing optional columns; run #{feature_generators(missing_features).join(' && ')}"
|
|
99
|
-
)
|
|
100
|
-
end
|
|
108
|
+
errors = LlmCostTracker::Ledger::Schema::Calls.current_schema_errors
|
|
109
|
+
return Check.new(:ok, "llm_api_calls columns", "current") if errors.empty?
|
|
101
110
|
|
|
102
|
-
|
|
111
|
+
missing = LlmCostTracker::Ledger::Schema::Calls.missing_current_schema_columns
|
|
112
|
+
generators = missing.filter_map { |column| COLUMN_GENERATORS[column] }.uniq
|
|
113
|
+
message = "current schema required; #{errors.join('; ')}"
|
|
114
|
+
message = "#{message}; run #{generators.join(' && ')} && bin/rails db:migrate" if generators.any?
|
|
115
|
+
|
|
116
|
+
Check.new(:error, "llm_api_calls columns", message)
|
|
103
117
|
end
|
|
104
118
|
|
|
105
119
|
def period_totals_check
|
|
106
120
|
return unless llm_api_calls_table?
|
|
107
|
-
if table_exists?("llm_cost_tracker_period_totals")
|
|
108
|
-
return Check.new(:ok, "period totals", "llm_cost_tracker_period_totals exists")
|
|
109
|
-
end
|
|
110
121
|
|
|
111
|
-
|
|
112
|
-
|
|
122
|
+
errors = LlmCostTracker::Ledger::Schema::PeriodTotals.current_schema_errors
|
|
123
|
+
return Check.new(:ok, "period totals", "llm_cost_tracker_period_totals exists") if errors.empty?
|
|
113
124
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
"using bundled prices updated_at=#{builtin_prices_updated_at}; configure prices_file for production"
|
|
121
|
-
)
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
count = LlmCostTracker::PriceRegistry.file_prices(path).size
|
|
125
|
-
metadata = LlmCostTracker::PriceRegistry.file_metadata(path)
|
|
126
|
-
status, freshness = LlmCostTracker::PriceFreshness.call(metadata)
|
|
127
|
-
Check.new(status, "prices", "loaded #{count} models from #{path}; #{freshness}")
|
|
128
|
-
rescue LlmCostTracker::Error => e
|
|
129
|
-
Check.new(:error, "prices", e.message)
|
|
125
|
+
Check.new(
|
|
126
|
+
:error,
|
|
127
|
+
"period totals",
|
|
128
|
+
"current schema required; #{errors.join('; ')}; " \
|
|
129
|
+
"run bin/rails generate llm_cost_tracker:add_period_totals && bin/rails db:migrate"
|
|
130
|
+
)
|
|
130
131
|
end
|
|
131
132
|
|
|
132
133
|
def calls_check
|
|
133
134
|
return unless llm_api_calls_table?
|
|
134
135
|
|
|
135
|
-
count = LlmCostTracker::
|
|
136
|
+
count = LlmCostTracker::Ledger::Call.count
|
|
136
137
|
return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
|
|
137
138
|
|
|
138
|
-
latest = LlmCostTracker::
|
|
139
|
+
latest = LlmCostTracker::Ledger::Call.maximum(:tracked_at)&.utc&.iso8601
|
|
139
140
|
Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
|
|
140
141
|
end
|
|
141
142
|
|
|
142
143
|
def active_record_available?
|
|
143
|
-
|
|
144
|
-
LlmCostTracker::LlmApiCall.connection
|
|
144
|
+
LlmCostTracker::Ledger::Call.connection
|
|
145
145
|
true
|
|
146
146
|
rescue LoadError, StandardError
|
|
147
147
|
false
|
|
@@ -152,17 +152,9 @@ module LlmCostTracker
|
|
|
152
152
|
end
|
|
153
153
|
|
|
154
154
|
def table_exists?(name)
|
|
155
|
-
LlmCostTracker::
|
|
155
|
+
LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
|
|
156
156
|
rescue StandardError
|
|
157
157
|
false
|
|
158
158
|
end
|
|
159
|
-
|
|
160
|
-
def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
|
|
161
|
-
|
|
162
|
-
def feature_generators(columns) = columns.map { |column| FEATURE_COLUMNS.fetch(column) }.uniq
|
|
163
|
-
|
|
164
|
-
def builtin_prices_updated_at
|
|
165
|
-
LlmCostTracker::PriceRegistry.metadata.fetch("updated_at", "unknown")
|
|
166
|
-
end
|
|
167
159
|
end
|
|
168
160
|
end
|
|
@@ -18,7 +18,10 @@ module LlmCostTracker
|
|
|
18
18
|
@budget_type = budget_type || inferred_budget_type
|
|
19
19
|
@last_event = last_event
|
|
20
20
|
|
|
21
|
-
super(
|
|
21
|
+
super(
|
|
22
|
+
"LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded: " \
|
|
23
|
+
"$#{format('%.6f', @total)} / $#{format('%.6f', budget)}"
|
|
24
|
+
)
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
private
|
|
@@ -30,10 +33,6 @@ module LlmCostTracker
|
|
|
30
33
|
|
|
31
34
|
:unknown
|
|
32
35
|
end
|
|
33
|
-
|
|
34
|
-
def budget_label
|
|
35
|
-
budget_type.to_s.tr("_", "-")
|
|
36
|
-
end
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
class UnknownPricingError < Error
|
|
@@ -45,14 +44,4 @@ module LlmCostTracker
|
|
|
45
44
|
super("No pricing configured for LLM model: #{model.inspect}")
|
|
46
45
|
end
|
|
47
46
|
end
|
|
48
|
-
|
|
49
|
-
class StorageError < Error
|
|
50
|
-
attr_reader :original_error
|
|
51
|
-
|
|
52
|
-
def initialize(original_error)
|
|
53
|
-
@original_error = original_error
|
|
54
|
-
|
|
55
|
-
super("Failed to store LLM cost event: #{original_error.class}: #{original_error.message}")
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
47
|
end
|
|
@@ -5,12 +5,7 @@ module LlmCostTracker
|
|
|
5
5
|
:event_id,
|
|
6
6
|
:provider,
|
|
7
7
|
:model,
|
|
8
|
-
:
|
|
9
|
-
:output_tokens,
|
|
10
|
-
:total_tokens,
|
|
11
|
-
:cache_read_input_tokens,
|
|
12
|
-
:cache_write_input_tokens,
|
|
13
|
-
:hidden_output_tokens,
|
|
8
|
+
:token_usage,
|
|
14
9
|
:pricing_mode,
|
|
15
10
|
:cost,
|
|
16
11
|
:tags,
|
|
@@ -20,8 +15,13 @@ module LlmCostTracker
|
|
|
20
15
|
:provider_response_id,
|
|
21
16
|
:tracked_at
|
|
22
17
|
) do
|
|
18
|
+
def total_cost
|
|
19
|
+
cost&.fetch(:total_cost, nil)
|
|
20
|
+
end
|
|
21
|
+
|
|
23
22
|
def to_h
|
|
24
23
|
super.merge(
|
|
24
|
+
token_usage: token_usage.to_h,
|
|
25
25
|
cost: cost&.to_h,
|
|
26
26
|
tags: tags ? tags.to_h : {}
|
|
27
27
|
)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Generators
|
|
8
|
+
class AddTokenUsageGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
TOKEN_COLUMNS = %w[
|
|
12
|
+
cache_read_input_tokens
|
|
13
|
+
cache_write_input_tokens
|
|
14
|
+
cache_write_1h_input_tokens
|
|
15
|
+
hidden_output_tokens
|
|
16
|
+
].freeze
|
|
17
|
+
COST_COLUMNS = %w[
|
|
18
|
+
cache_read_input_cost
|
|
19
|
+
cache_write_input_cost
|
|
20
|
+
cache_write_1h_input_cost
|
|
21
|
+
].freeze
|
|
22
|
+
COLUMN_NAMES = (TOKEN_COLUMNS + COST_COLUMNS + %w[pricing_mode]).freeze
|
|
23
|
+
|
|
24
|
+
source_root File.expand_path("templates", __dir__)
|
|
25
|
+
|
|
26
|
+
desc "Creates a migration to add token usage and token cost columns to llm_api_calls"
|
|
27
|
+
|
|
28
|
+
def create_migration_file
|
|
29
|
+
migration_template(
|
|
30
|
+
"add_token_usage_to_llm_api_calls.rb.erb",
|
|
31
|
+
"db/migrate/add_token_usage_to_llm_api_calls.rb"
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def migration_version
|
|
38
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "rails/generators"
|
|
4
4
|
|
|
5
|
-
require_relative "../../
|
|
6
|
-
require_relative "../../
|
|
7
|
-
require_relative "../../
|
|
5
|
+
require_relative "../../pricing/registry"
|
|
6
|
+
require_relative "../../pricing/sync/registry_loader"
|
|
7
|
+
require_relative "../../pricing/sync/registry_writer"
|
|
8
8
|
|
|
9
9
|
module LlmCostTracker
|
|
10
10
|
module Generators
|
|
@@ -12,11 +12,11 @@ module LlmCostTracker
|
|
|
12
12
|
desc "Creates a local LLM Cost Tracker price snapshot"
|
|
13
13
|
|
|
14
14
|
def create_prices_file
|
|
15
|
-
registry = LlmCostTracker::
|
|
16
|
-
path: LlmCostTracker::
|
|
17
|
-
seed_path: LlmCostTracker::
|
|
15
|
+
registry = LlmCostTracker::Pricing::Sync::RegistryLoader.new.call(
|
|
16
|
+
path: LlmCostTracker::Pricing::Registry::DEFAULT_PRICES_PATH,
|
|
17
|
+
seed_path: LlmCostTracker::Pricing::Registry::DEFAULT_PRICES_PATH
|
|
18
18
|
)
|
|
19
|
-
LlmCostTracker::
|
|
19
|
+
LlmCostTracker::Pricing::Sync::RegistryWriter.new.call(
|
|
20
20
|
path: File.join(destination_root, "config/llm_cost_tracker_prices.yml"),
|
|
21
21
|
registry: registry
|
|
22
22
|
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require "llm_cost_tracker/
|
|
1
|
+
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
3
|
class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_version %>
|
|
4
4
|
def up
|
|
@@ -95,10 +95,10 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
|
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
def postgresql?
|
|
98
|
-
LlmCostTracker::
|
|
98
|
+
LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
def mysql?
|
|
102
|
-
LlmCostTracker::
|
|
102
|
+
LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
|
|
103
103
|
end
|
|
104
104
|
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class AddTokenUsageToLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def up
|
|
3
|
+
<% LlmCostTracker::Generators::AddTokenUsageGenerator::TOKEN_COLUMNS.each do |column| -%>
|
|
4
|
+
unless column_exists?(:llm_api_calls, :<%= column %>)
|
|
5
|
+
add_column :llm_api_calls, :<%= column %>, :integer, null: false, default: 0
|
|
6
|
+
end
|
|
7
|
+
<% end -%>
|
|
8
|
+
<% LlmCostTracker::Generators::AddTokenUsageGenerator::COST_COLUMNS.each do |column| -%>
|
|
9
|
+
unless column_exists?(:llm_api_calls, :<%= column %>)
|
|
10
|
+
add_column :llm_api_calls, :<%= column %>, :decimal, precision: 20, scale: 8
|
|
11
|
+
end
|
|
12
|
+
<% end -%>
|
|
13
|
+
add_column :llm_api_calls, :pricing_mode, :string unless column_exists?(:llm_api_calls, :pricing_mode)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def down
|
|
17
|
+
remove_column :llm_api_calls, :pricing_mode if column_exists?(:llm_api_calls, :pricing_mode)
|
|
18
|
+
<% (LlmCostTracker::Generators::AddTokenUsageGenerator::COST_COLUMNS + LlmCostTracker::Generators::AddTokenUsageGenerator::TOKEN_COLUMNS).reverse.each do |column| -%>
|
|
19
|
+
remove_column :llm_api_calls, :<%= column %> if column_exists?(:llm_api_calls, :<%= column %>)
|
|
20
|
+
<% end -%>
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require "llm_cost_tracker/
|
|
1
|
+
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
3
|
class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
|
|
4
4
|
def change
|
|
@@ -6,17 +6,12 @@ class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
|
|
|
6
6
|
t.string :event_id, null: false
|
|
7
7
|
t.string :provider, null: false
|
|
8
8
|
t.string :model, null: false
|
|
9
|
-
|
|
10
|
-
t.integer
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
t.
|
|
14
|
-
|
|
15
|
-
t.decimal :input_cost, precision: 20, scale: 8
|
|
16
|
-
t.decimal :cache_read_input_cost, precision: 20, scale: 8
|
|
17
|
-
t.decimal :cache_write_input_cost, precision: 20, scale: 8
|
|
18
|
-
t.decimal :output_cost, precision: 20, scale: 8
|
|
19
|
-
t.decimal :total_cost, precision: 20, scale: 8
|
|
9
|
+
<% LlmCostTracker::TokenUsage::STORED_KEYS.each do |column| -%>
|
|
10
|
+
t.integer :<%= column %>, null: false, default: 0
|
|
11
|
+
<% end -%>
|
|
12
|
+
<% LlmCostTracker::Pricing::COST_KEYS.each do |column| -%>
|
|
13
|
+
t.decimal :<%= column %>, precision: 20, scale: 8
|
|
14
|
+
<% end -%>
|
|
20
15
|
t.integer :latency_ms
|
|
21
16
|
t.boolean :stream, null: false, default: false
|
|
22
17
|
t.string :usage_source
|
|
@@ -79,10 +74,10 @@ class CreateLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
|
|
|
79
74
|
private
|
|
80
75
|
|
|
81
76
|
def postgresql?
|
|
82
|
-
LlmCostTracker::
|
|
77
|
+
LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
|
|
83
78
|
end
|
|
84
79
|
|
|
85
80
|
def mysql?
|
|
86
|
-
LlmCostTracker::
|
|
81
|
+
LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
|
|
87
82
|
end
|
|
88
83
|
end
|
|
@@ -22,10 +22,6 @@ LlmCostTracker.configure do |config|
|
|
|
22
22
|
# :block_requests preflights monthly/daily budgets before supported requests.
|
|
23
23
|
config.budget_exceeded_behavior = :notify
|
|
24
24
|
|
|
25
|
-
# Storage failures are non-fatal by default so LLM responses can still return.
|
|
26
|
-
# Use :raise if failed ledger writes should fail the request/job.
|
|
27
|
-
config.storage_error_behavior = :warn
|
|
28
|
-
|
|
29
25
|
# Unknown pricing records token usage with nil cost by default. Use :raise if
|
|
30
26
|
# every model must have known pricing before it can be used.
|
|
31
27
|
config.unknown_pricing_behavior = :warn
|