llm_cost_tracker 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/README.md +13 -12
- data/app/assets/llm_cost_tracker/application.css +3 -0
- data/app/controllers/llm_cost_tracker/application_controller.rb +22 -4
- data/app/controllers/llm_cost_tracker/calls_controller.rb +6 -11
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +5 -1
- data/app/controllers/llm_cost_tracker/models_controller.rb +0 -1
- data/app/controllers/llm_cost_tracker/tags_controller.rb +1 -8
- data/app/helpers/llm_cost_tracker/application_helper.rb +2 -1
- data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +1 -2
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +10 -27
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +58 -0
- data/app/models/llm_cost_tracker/ingestion/event.rb +13 -0
- data/app/models/llm_cost_tracker/ingestion/lease.rb +11 -0
- data/app/models/llm_cost_tracker/ledger/call.rb +45 -0
- data/app/models/llm_cost_tracker/ledger/call_metrics.rb +66 -0
- data/app/models/llm_cost_tracker/ledger/period/grouping.rb +71 -0
- data/app/models/llm_cost_tracker/ledger/period/total.rb +13 -0
- data/app/models/llm_cost_tracker/ledger/tags/accessors.rb +19 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +111 -94
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +2 -2
- data/app/services/llm_cost_tracker/dashboard/filter.rb +7 -18
- data/app/services/llm_cost_tracker/dashboard/overview_stats.rb +58 -67
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +59 -0
- data/app/services/llm_cost_tracker/dashboard/params.rb +26 -0
- data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +18 -20
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +4 -13
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +28 -61
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +8 -37
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +12 -47
- data/app/views/llm_cost_tracker/calls/index.html.erb +12 -18
- data/app/views/llm_cost_tracker/calls/show.html.erb +30 -32
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +17 -19
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +108 -135
- data/app/views/llm_cost_tracker/models/index.html.erb +8 -9
- data/app/views/llm_cost_tracker/shared/setup_required.html.erb +13 -2
- data/app/views/llm_cost_tracker/tags/show.html.erb +20 -20
- data/config/routes.rb +1 -1
- data/lib/llm_cost_tracker/assets.rb +0 -6
- data/lib/llm_cost_tracker/budget.rb +10 -24
- data/lib/llm_cost_tracker/capture/stream.rb +9 -0
- data/lib/llm_cost_tracker/capture/stream_collector.rb +182 -0
- data/lib/llm_cost_tracker/{integrations → capture}/stream_tracker.rb +40 -72
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +3 -7
- data/lib/llm_cost_tracker/configuration.rb +30 -45
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +61 -0
- data/lib/llm_cost_tracker/doctor/check.rb +7 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +22 -61
- data/lib/llm_cost_tracker/doctor/price_check.rb +60 -0
- data/lib/llm_cost_tracker/doctor.rb +66 -79
- data/lib/llm_cost_tracker/engine.rb +0 -3
- data/lib/llm_cost_tracker/errors.rb +4 -15
- data/lib/llm_cost_tracker/event.rb +6 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_token_usage_generator.rb +42 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +2 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +7 -7
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +5 -5
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_token_usage_to_llm_api_calls.rb.erb +22 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +15 -14
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +1 -21
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_cost_precision.rb.erb +12 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb +2 -2
- data/lib/llm_cost_tracker/{storage/active_record_inbox_batch.rb → ingestion/batch.rb} +21 -20
- data/lib/llm_cost_tracker/ingestion/inbox.rb +105 -0
- data/lib/llm_cost_tracker/{storage/active_record_ingestor_lease.rb → ingestion/lease_claim.rb} +5 -7
- data/lib/llm_cost_tracker/{storage/active_record_ingestor.rb → ingestion/worker.rb} +38 -48
- data/lib/llm_cost_tracker/ingestion.rb +129 -0
- data/lib/llm_cost_tracker/integrations/anthropic.rb +52 -34
- data/lib/llm_cost_tracker/integrations/base.rb +73 -34
- data/lib/llm_cost_tracker/integrations/openai.rb +45 -39
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +40 -30
- data/lib/llm_cost_tracker/integrations.rb +43 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +66 -0
- data/lib/llm_cost_tracker/{storage/active_record_periods.rb → ledger/period.rb} +2 -2
- data/lib/llm_cost_tracker/ledger/rollups/batch.rb +43 -0
- data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb +46 -0
- data/lib/llm_cost_tracker/ledger/rollups.rb +87 -0
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +51 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +101 -0
- data/lib/llm_cost_tracker/ledger/schema/period_totals.rb +32 -0
- data/lib/llm_cost_tracker/ledger/store.rb +60 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +29 -0
- data/lib/llm_cost_tracker/ledger/tags/sql.rb +33 -0
- data/lib/llm_cost_tracker/ledger.rb +13 -0
- data/lib/llm_cost_tracker/logging.rb +3 -6
- data/lib/llm_cost_tracker/middleware/faraday.rb +35 -36
- data/lib/llm_cost_tracker/parsers/anthropic.rb +38 -27
- data/lib/llm_cost_tracker/parsers/base.rb +10 -19
- data/lib/llm_cost_tracker/parsers/gemini.rb +15 -16
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +24 -19
- data/lib/llm_cost_tracker/parsers/sse.rb +4 -7
- data/lib/llm_cost_tracker/parsers.rb +20 -0
- data/lib/llm_cost_tracker/prices.json +52 -11
- data/lib/llm_cost_tracker/pricing/components.rb +37 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +40 -50
- data/lib/llm_cost_tracker/pricing/explainer.rb +12 -23
- data/lib/llm_cost_tracker/pricing/lookup.rb +24 -25
- data/lib/llm_cost_tracker/pricing/registry.rb +156 -0
- data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +107 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +53 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_loader.rb +63 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +31 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +143 -0
- data/lib/llm_cost_tracker/pricing/unknown.rb +46 -0
- data/lib/llm_cost_tracker/pricing.rb +33 -32
- data/lib/llm_cost_tracker/railtie.rb +7 -10
- data/lib/llm_cost_tracker/report/data.rb +72 -0
- data/lib/llm_cost_tracker/report/formatter.rb +69 -0
- data/lib/llm_cost_tracker/report.rb +8 -10
- data/lib/llm_cost_tracker/retention.rb +27 -10
- data/lib/llm_cost_tracker/tags/context.rb +35 -0
- data/lib/llm_cost_tracker/tags/key.rb +18 -0
- data/lib/llm_cost_tracker/tags/sanitizer.rb +68 -0
- data/lib/llm_cost_tracker/token_usage.rb +67 -0
- data/lib/llm_cost_tracker/tracker.rb +38 -70
- data/lib/llm_cost_tracker/usage_capture.rb +37 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +56 -90
- data/lib/tasks/llm_cost_tracker.rake +18 -13
- metadata +85 -99
- data/app/services/llm_cost_tracker/dashboard/data_quality_aggregate.rb +0 -81
- data/app/services/llm_cost_tracker/pagination.rb +0 -57
- data/lib/llm_cost_tracker/active_record_adapter.rb +0 -49
- data/lib/llm_cost_tracker/capture_verifier.rb +0 -71
- data/lib/llm_cost_tracker/configuration/storage_backend.rb +0 -26
- data/lib/llm_cost_tracker/cost.rb +0 -12
- data/lib/llm_cost_tracker/doctor/capture_check.rb +0 -39
- data/lib/llm_cost_tracker/engine_compatibility.rb +0 -15
- data/lib/llm_cost_tracker/event_metadata.rb +0 -52
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_usage_breakdown_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_usage_breakdown_to_llm_api_calls.rb.erb +0 -29
- data/lib/llm_cost_tracker/inbox_event.rb +0 -9
- data/lib/llm_cost_tracker/ingestor_lease.rb +0 -9
- data/lib/llm_cost_tracker/integrations/object_reader.rb +0 -56
- data/lib/llm_cost_tracker/integrations/registry.rb +0 -73
- data/lib/llm_cost_tracker/llm_api_call.rb +0 -60
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +0 -63
- data/lib/llm_cost_tracker/parameter_hash.rb +0 -33
- data/lib/llm_cost_tracker/parsed_usage.rb +0 -72
- data/lib/llm_cost_tracker/parsers/registry.rb +0 -58
- data/lib/llm_cost_tracker/period_grouping.rb +0 -69
- data/lib/llm_cost_tracker/period_total.rb +0 -9
- data/lib/llm_cost_tracker/price_freshness.rb +0 -38
- data/lib/llm_cost_tracker/price_registry.rb +0 -144
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +0 -104
- data/lib/llm_cost_tracker/price_sync/registry_diff.rb +0 -51
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +0 -61
- data/lib/llm_cost_tracker/price_sync/registry_writer.rb +0 -29
- data/lib/llm_cost_tracker/price_sync.rb +0 -144
- data/lib/llm_cost_tracker/report_data.rb +0 -94
- data/lib/llm_cost_tracker/report_formatter.rb +0 -67
- data/lib/llm_cost_tracker/request_url.rb +0 -20
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +0 -166
- data/lib/llm_cost_tracker/storage/active_record_connection_cleanup.rb +0 -13
- data/lib/llm_cost_tracker/storage/active_record_inbox.rb +0 -165
- data/lib/llm_cost_tracker/storage/active_record_period_totals.rb +0 -84
- data/lib/llm_cost_tracker/storage/active_record_rollup_batch.rb +0 -41
- data/lib/llm_cost_tracker/storage/active_record_rollup_upsert_sql.rb +0 -42
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +0 -146
- data/lib/llm_cost_tracker/storage/active_record_store.rb +0 -145
- data/lib/llm_cost_tracker/storage/custom_backend.rb +0 -32
- data/lib/llm_cost_tracker/storage/dispatcher.rb +0 -45
- data/lib/llm_cost_tracker/storage/log_backend.rb +0 -38
- data/lib/llm_cost_tracker/storage/registry.rb +0 -63
- data/lib/llm_cost_tracker/stream_capture.rb +0 -7
- data/lib/llm_cost_tracker/stream_collector.rb +0 -199
- data/lib/llm_cost_tracker/tag_accessors.rb +0 -15
- data/lib/llm_cost_tracker/tag_context.rb +0 -52
- data/lib/llm_cost_tracker/tag_key.rb +0 -16
- data/lib/llm_cost_tracker/tag_query.rb +0 -43
- data/lib/llm_cost_tracker/tag_sanitizer.rb +0 -81
- data/lib/llm_cost_tracker/tag_sql.rb +0 -34
- data/lib/llm_cost_tracker/tags_column.rb +0 -103
- data/lib/llm_cost_tracker/unknown_pricing.rb +0 -54
- data/lib/llm_cost_tracker/usage_breakdown.rb +0 -30
- data/lib/llm_cost_tracker/value_helpers.rb +0 -40
|
@@ -1,44 +1,40 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
require_relative "check"
|
|
4
|
+
require_relative "../ingestion"
|
|
4
5
|
|
|
5
6
|
module LlmCostTracker
|
|
6
7
|
class Doctor
|
|
7
8
|
class IngestionCheck
|
|
8
9
|
PENDING_AGE_WARNING_SECONDS = 60
|
|
9
10
|
|
|
10
|
-
def self.call(check_class)
|
|
11
|
-
new(check_class).call
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def initialize(check_class)
|
|
15
|
-
@check_class = check_class
|
|
16
|
-
end
|
|
17
|
-
|
|
18
11
|
def call
|
|
19
|
-
return unless
|
|
12
|
+
return unless table_exists?("llm_api_calls")
|
|
20
13
|
|
|
21
14
|
missing = missing_parts
|
|
22
15
|
if missing.empty?
|
|
23
16
|
quarantined = quarantined_count
|
|
24
17
|
if quarantined.positive?
|
|
25
|
-
return
|
|
18
|
+
return Check.new(:warn, "durable ingestion", "#{quarantined} inbox events quarantined after retries")
|
|
26
19
|
end
|
|
27
20
|
|
|
28
21
|
pending = pending_snapshot
|
|
29
|
-
|
|
30
|
-
|
|
22
|
+
pending_count = pending.try(:pending_count).to_i
|
|
23
|
+
oldest_pending_at = pending.try(:oldest_created_at)&.to_time&.utc
|
|
24
|
+
pending_age = oldest_pending_at && (Time.now.utc - oldest_pending_at)
|
|
25
|
+
if pending_count.positive? && pending_age && pending_age >= PENDING_AGE_WARNING_SECONDS
|
|
26
|
+
return Check.new(
|
|
31
27
|
:warn,
|
|
32
28
|
"durable ingestion",
|
|
33
|
-
"#{
|
|
29
|
+
"#{pending_count} inbox events pending; oldest pending age #{pending_age.round}s"
|
|
34
30
|
)
|
|
35
31
|
end
|
|
36
32
|
|
|
37
|
-
return
|
|
33
|
+
return Check.new(:ok, "durable ingestion", "inbox and ingestor lease tables available")
|
|
38
34
|
end
|
|
39
35
|
|
|
40
|
-
|
|
41
|
-
:
|
|
36
|
+
Check.new(
|
|
37
|
+
:error,
|
|
42
38
|
"durable ingestion",
|
|
43
39
|
"missing #{missing.join(', ')}; run bin/rails generate llm_cost_tracker:add_ingestion && bin/rails db:migrate"
|
|
44
40
|
)
|
|
@@ -46,71 +42,36 @@ module LlmCostTracker
|
|
|
46
42
|
|
|
47
43
|
private
|
|
48
44
|
|
|
49
|
-
attr_reader :check_class
|
|
50
|
-
|
|
51
45
|
def missing_parts
|
|
52
46
|
[
|
|
53
|
-
column_names("llm_api_calls").include?("event_id") ? nil : "llm_api_calls.event_id",
|
|
54
47
|
table_exists?("llm_cost_tracker_inbox_events") ? nil : "llm_cost_tracker_inbox_events",
|
|
55
48
|
table_exists?("llm_cost_tracker_ingestor_leases") ? nil : "llm_cost_tracker_ingestor_leases"
|
|
56
49
|
].compact
|
|
57
50
|
end
|
|
58
51
|
|
|
59
|
-
def active_record_storage? = LlmCostTracker.configuration.storage_backend == :active_record
|
|
60
|
-
|
|
61
|
-
def llm_api_calls_table? = table_exists?("llm_api_calls")
|
|
62
|
-
|
|
63
52
|
def table_exists?(name)
|
|
64
|
-
LlmCostTracker::
|
|
53
|
+
LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
|
|
65
54
|
rescue StandardError
|
|
66
55
|
false
|
|
67
56
|
end
|
|
68
57
|
|
|
69
|
-
def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
|
|
70
|
-
|
|
71
58
|
def quarantined_count
|
|
72
59
|
return 0 unless table_exists?("llm_cost_tracker_inbox_events")
|
|
73
60
|
|
|
74
|
-
LlmCostTracker::
|
|
61
|
+
LlmCostTracker::Ingestion::Event
|
|
62
|
+
.where("attempts >= ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
|
|
63
|
+
.count
|
|
75
64
|
rescue StandardError
|
|
76
65
|
0
|
|
77
66
|
end
|
|
78
67
|
|
|
79
|
-
def quarantined_sql
|
|
80
|
-
table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
|
|
81
|
-
"SELECT COUNT(*) FROM #{table} WHERE attempts >= #{max_attempts}"
|
|
82
|
-
end
|
|
83
|
-
|
|
84
68
|
def pending_snapshot
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
}
|
|
69
|
+
LlmCostTracker::Ingestion::Event
|
|
70
|
+
.where("attempts < ?", LlmCostTracker::Ingestion::Event::MAX_ATTEMPTS)
|
|
71
|
+
.select("COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at")
|
|
72
|
+
.take
|
|
90
73
|
rescue StandardError
|
|
91
|
-
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
def pending_sql
|
|
95
|
-
table = LlmCostTracker::LlmApiCall.connection.quote_table_name("llm_cost_tracker_inbox_events")
|
|
96
|
-
"SELECT COUNT(*) AS pending_count, MIN(created_at) AS oldest_created_at " \
|
|
97
|
-
"FROM #{table} WHERE attempts < #{max_attempts}"
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
def stale_pending?(pending)
|
|
101
|
-
pending.fetch(:count).positive? &&
|
|
102
|
-
pending.fetch(:oldest_at) &&
|
|
103
|
-
pending_age(pending) >= PENDING_AGE_WARNING_SECONDS
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
def pending_age(pending) = Time.now.utc - pending.fetch(:oldest_at)
|
|
107
|
-
|
|
108
|
-
def max_attempts
|
|
109
|
-
if defined?(LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS)
|
|
110
|
-
LlmCostTracker::Storage::ActiveRecordInbox::MAX_ATTEMPTS
|
|
111
|
-
else
|
|
112
|
-
5
|
|
113
|
-
end
|
|
74
|
+
nil
|
|
114
75
|
end
|
|
115
76
|
end
|
|
116
77
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
|
|
5
|
+
require_relative "check"
|
|
6
|
+
|
|
7
|
+
module LlmCostTracker
|
|
8
|
+
class Doctor
|
|
9
|
+
class PriceCheck
|
|
10
|
+
STALE_AFTER_DAYS = 30
|
|
11
|
+
REFRESH_COMMAND = "run bin/rails llm_cost_tracker:prices:refresh"
|
|
12
|
+
|
|
13
|
+
def call
|
|
14
|
+
path = LlmCostTracker.configuration.prices_file
|
|
15
|
+
return bundled_check unless path
|
|
16
|
+
|
|
17
|
+
count = LlmCostTracker::Pricing::Registry.file_prices(path).size
|
|
18
|
+
metadata = LlmCostTracker::Pricing::Registry.file_metadata(path)
|
|
19
|
+
updated_at = metadata["updated_at"] || metadata[:updated_at]
|
|
20
|
+
return configured_check(:warn, path, count, "metadata.updated_at missing; #{REFRESH_COMMAND}") unless updated_at
|
|
21
|
+
|
|
22
|
+
age_days = (Date.today - Date.iso8601(updated_at.to_s)).to_i
|
|
23
|
+
if age_days > STALE_AFTER_DAYS
|
|
24
|
+
return configured_check(
|
|
25
|
+
:warn,
|
|
26
|
+
path,
|
|
27
|
+
count,
|
|
28
|
+
"updated_at=#{updated_at} is older than #{STALE_AFTER_DAYS} days; #{REFRESH_COMMAND}"
|
|
29
|
+
)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
configured_check(:ok, path, count, "updated_at=#{updated_at}")
|
|
33
|
+
rescue Date::Error
|
|
34
|
+
configured_check(
|
|
35
|
+
:warn,
|
|
36
|
+
path,
|
|
37
|
+
count,
|
|
38
|
+
"metadata.updated_at=#{updated_at.inspect} is invalid; #{REFRESH_COMMAND}"
|
|
39
|
+
)
|
|
40
|
+
rescue LlmCostTracker::Error => e
|
|
41
|
+
Check.new(:error, "prices", e.message)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def bundled_check
|
|
47
|
+
updated_at = LlmCostTracker::Pricing::Registry.metadata.fetch("updated_at", "unknown")
|
|
48
|
+
Check.new(
|
|
49
|
+
:warn,
|
|
50
|
+
"prices",
|
|
51
|
+
"using bundled prices updated_at=#{updated_at}; configure prices_file for production"
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def configured_check(status, path, count, freshness)
|
|
56
|
+
Check.new(status, "prices", "loaded #{count} models from #{path}; #{freshness}")
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -1,40 +1,39 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative "
|
|
4
|
-
require_relative "doctor/
|
|
3
|
+
require_relative "ledger"
|
|
4
|
+
require_relative "doctor/check"
|
|
5
5
|
require_relative "doctor/ingestion_check"
|
|
6
|
+
require_relative "doctor/price_check"
|
|
7
|
+
require_relative "generators/llm_cost_tracker/add_token_usage_generator"
|
|
6
8
|
|
|
7
9
|
module LlmCostTracker
|
|
8
10
|
class Doctor
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
FEATURE_COLUMNS = {
|
|
11
|
+
COLUMN_GENERATORS = {
|
|
12
|
+
"event_id" => "bin/rails generate llm_cost_tracker:add_ingestion",
|
|
12
13
|
"latency_ms" => "bin/rails generate llm_cost_tracker:add_latency_ms",
|
|
13
14
|
"stream" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
14
15
|
"usage_source" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
15
|
-
"provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id"
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
"provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id"
|
|
17
|
+
}.merge(
|
|
18
|
+
Generators::AddTokenUsageGenerator::COLUMN_NAMES.to_h do |column|
|
|
19
|
+
[column, "bin/rails generate llm_cost_tracker:add_token_usage"]
|
|
20
|
+
end
|
|
21
|
+
).freeze
|
|
21
22
|
|
|
22
23
|
class << self
|
|
23
|
-
def call
|
|
24
|
+
def call
|
|
25
|
+
new.checks
|
|
26
|
+
end
|
|
24
27
|
|
|
25
28
|
def report(checks = call)
|
|
26
|
-
(["LLM Cost Tracker doctor"] + checks.map
|
|
29
|
+
(["LLM Cost Tracker doctor"] + checks.map do |check|
|
|
30
|
+
"[#{check.status}] #{check.name}: #{check.message}"
|
|
31
|
+
end).join("\n")
|
|
27
32
|
end
|
|
28
33
|
|
|
29
34
|
def healthy?(checks = call)
|
|
30
35
|
checks.none? { |check| check.status == :error }
|
|
31
36
|
end
|
|
32
|
-
|
|
33
|
-
private
|
|
34
|
-
|
|
35
|
-
def format_check(check)
|
|
36
|
-
"[#{check.status}] #{check.name}: #{check.message}"
|
|
37
|
-
end
|
|
38
37
|
end
|
|
39
38
|
|
|
40
39
|
def checks
|
|
@@ -46,8 +45,8 @@ module LlmCostTracker
|
|
|
46
45
|
table_check,
|
|
47
46
|
column_check,
|
|
48
47
|
period_totals_check,
|
|
49
|
-
IngestionCheck.call
|
|
50
|
-
|
|
48
|
+
IngestionCheck.new.call,
|
|
49
|
+
PriceCheck.new.call,
|
|
51
50
|
calls_check
|
|
52
51
|
].compact
|
|
53
52
|
end
|
|
@@ -56,10 +55,29 @@ module LlmCostTracker
|
|
|
56
55
|
|
|
57
56
|
def configuration_check
|
|
58
57
|
config = LlmCostTracker.configuration
|
|
59
|
-
Check.new(:ok, "configuration", "
|
|
58
|
+
Check.new(:ok, "configuration", "active_record ledger enabled=#{config.enabled}")
|
|
60
59
|
end
|
|
61
60
|
|
|
62
|
-
def capture_check
|
|
61
|
+
def capture_check
|
|
62
|
+
config = LlmCostTracker.configuration
|
|
63
|
+
unless config.enabled
|
|
64
|
+
return Check.new(:warn, "capture", "tracking is disabled; set config.enabled = true to record calls")
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
if config.instrumented_integrations.any?
|
|
68
|
+
return Check.new(
|
|
69
|
+
:ok,
|
|
70
|
+
"capture",
|
|
71
|
+
"SDK integrations enabled: #{config.instrumented_integrations.join(', ')}"
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
Check.new(
|
|
76
|
+
:ok,
|
|
77
|
+
"capture",
|
|
78
|
+
"no SDK integrations enabled; Faraday middleware and manual capture remain available"
|
|
79
|
+
)
|
|
80
|
+
end
|
|
63
81
|
|
|
64
82
|
def integration_checks
|
|
65
83
|
LlmCostTracker::Integrations.checks.map do |check|
|
|
@@ -68,14 +86,13 @@ module LlmCostTracker
|
|
|
68
86
|
end
|
|
69
87
|
|
|
70
88
|
def active_record_check
|
|
71
|
-
return Check.new(:ok, "storage", "ActiveRecord storage is disabled") unless active_record_storage?
|
|
72
89
|
return Check.new(:ok, "active_record", "available") if active_record_available?
|
|
73
90
|
|
|
74
|
-
Check.new(:error, "active_record", "unavailable
|
|
91
|
+
Check.new(:error, "active_record", "unavailable")
|
|
75
92
|
end
|
|
76
93
|
|
|
77
94
|
def table_check
|
|
78
|
-
return unless
|
|
95
|
+
return unless active_record_available?
|
|
79
96
|
return Check.new(:ok, "llm_api_calls", "table exists") if llm_api_calls_table?
|
|
80
97
|
|
|
81
98
|
Check.new(
|
|
@@ -86,67 +103,45 @@ module LlmCostTracker
|
|
|
86
103
|
end
|
|
87
104
|
|
|
88
105
|
def column_check
|
|
89
|
-
return unless
|
|
106
|
+
return unless llm_api_calls_table?
|
|
90
107
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if
|
|
98
|
-
return Check.new(
|
|
99
|
-
:warn,
|
|
100
|
-
"llm_api_calls columns",
|
|
101
|
-
"missing optional columns; run #{feature_generators(missing_features).join(' && ')}"
|
|
102
|
-
)
|
|
103
|
-
end
|
|
108
|
+
errors = LlmCostTracker::Ledger::Schema::Calls.current_schema_errors
|
|
109
|
+
return Check.new(:ok, "llm_api_calls columns", "current") if errors.empty?
|
|
110
|
+
|
|
111
|
+
missing = LlmCostTracker::Ledger::Schema::Calls.missing_current_schema_columns
|
|
112
|
+
generators = missing.filter_map { |column| COLUMN_GENERATORS[column] }.uniq
|
|
113
|
+
message = "current schema required; #{errors.join('; ')}"
|
|
114
|
+
message = "#{message}; run #{generators.join(' && ')} && bin/rails db:migrate" if generators.any?
|
|
104
115
|
|
|
105
|
-
Check.new(:
|
|
116
|
+
Check.new(:error, "llm_api_calls columns", message)
|
|
106
117
|
end
|
|
107
118
|
|
|
108
119
|
def period_totals_check
|
|
109
|
-
return unless
|
|
110
|
-
if table_exists?("llm_cost_tracker_period_totals")
|
|
111
|
-
return Check.new(:ok, "period totals", "llm_cost_tracker_period_totals exists")
|
|
112
|
-
end
|
|
120
|
+
return unless llm_api_calls_table?
|
|
113
121
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
def prices_check
|
|
118
|
-
path = LlmCostTracker.configuration.prices_file
|
|
119
|
-
unless path
|
|
120
|
-
return Check.new(
|
|
121
|
-
:warn,
|
|
122
|
-
"prices",
|
|
123
|
-
"using bundled prices updated_at=#{builtin_prices_updated_at}; configure prices_file for production"
|
|
124
|
-
)
|
|
125
|
-
end
|
|
122
|
+
errors = LlmCostTracker::Ledger::Schema::PeriodTotals.current_schema_errors
|
|
123
|
+
return Check.new(:ok, "period totals", "llm_cost_tracker_period_totals exists") if errors.empty?
|
|
126
124
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
125
|
+
Check.new(
|
|
126
|
+
:error,
|
|
127
|
+
"period totals",
|
|
128
|
+
"current schema required; #{errors.join('; ')}; " \
|
|
129
|
+
"run bin/rails generate llm_cost_tracker:add_period_totals && bin/rails db:migrate"
|
|
130
|
+
)
|
|
133
131
|
end
|
|
134
132
|
|
|
135
133
|
def calls_check
|
|
136
|
-
return unless
|
|
134
|
+
return unless llm_api_calls_table?
|
|
137
135
|
|
|
138
|
-
count = LlmCostTracker::
|
|
136
|
+
count = LlmCostTracker::Ledger::Call.count
|
|
139
137
|
return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
|
|
140
138
|
|
|
141
|
-
latest = LlmCostTracker::
|
|
139
|
+
latest = LlmCostTracker::Ledger::Call.maximum(:tracked_at)&.utc&.iso8601
|
|
142
140
|
Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
|
|
143
141
|
end
|
|
144
142
|
|
|
145
|
-
def active_record_storage? = LlmCostTracker.configuration.storage_backend == :active_record
|
|
146
|
-
|
|
147
143
|
def active_record_available?
|
|
148
|
-
|
|
149
|
-
LlmCostTracker::LlmApiCall.connection
|
|
144
|
+
LlmCostTracker::Ledger::Call.connection
|
|
150
145
|
true
|
|
151
146
|
rescue LoadError, StandardError
|
|
152
147
|
false
|
|
@@ -157,17 +152,9 @@ module LlmCostTracker
|
|
|
157
152
|
end
|
|
158
153
|
|
|
159
154
|
def table_exists?(name)
|
|
160
|
-
LlmCostTracker::
|
|
155
|
+
LlmCostTracker::Ledger::Call.connection.data_source_exists?(name)
|
|
161
156
|
rescue StandardError
|
|
162
157
|
false
|
|
163
158
|
end
|
|
164
|
-
|
|
165
|
-
def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
|
|
166
|
-
|
|
167
|
-
def feature_generators(columns) = columns.map { |column| FEATURE_COLUMNS.fetch(column) }.uniq
|
|
168
|
-
|
|
169
|
-
def builtin_prices_updated_at
|
|
170
|
-
LlmCostTracker::PriceRegistry.metadata.fetch("updated_at", "unknown")
|
|
171
|
-
end
|
|
172
159
|
end
|
|
173
160
|
end
|
|
@@ -2,12 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "rails"
|
|
4
4
|
require_relative "../llm_cost_tracker"
|
|
5
|
-
require_relative "engine_compatibility"
|
|
6
5
|
require_relative "assets"
|
|
7
6
|
require "rack/files"
|
|
8
7
|
|
|
9
|
-
LlmCostTracker::EngineCompatibility.check_rails_version!(Rails.version)
|
|
10
|
-
|
|
11
8
|
module LlmCostTracker
|
|
12
9
|
class Engine < ::Rails::Engine
|
|
13
10
|
isolate_namespace LlmCostTracker
|
|
@@ -18,7 +18,10 @@ module LlmCostTracker
|
|
|
18
18
|
@budget_type = budget_type || inferred_budget_type
|
|
19
19
|
@last_event = last_event
|
|
20
20
|
|
|
21
|
-
super(
|
|
21
|
+
super(
|
|
22
|
+
"LLM #{@budget_type.to_s.tr('_', '-')} budget exceeded: " \
|
|
23
|
+
"$#{format('%.6f', @total)} / $#{format('%.6f', budget)}"
|
|
24
|
+
)
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
private
|
|
@@ -30,10 +33,6 @@ module LlmCostTracker
|
|
|
30
33
|
|
|
31
34
|
:unknown
|
|
32
35
|
end
|
|
33
|
-
|
|
34
|
-
def budget_label
|
|
35
|
-
budget_type.to_s.tr("_", "-")
|
|
36
|
-
end
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
class UnknownPricingError < Error
|
|
@@ -45,14 +44,4 @@ module LlmCostTracker
|
|
|
45
44
|
super("No pricing configured for LLM model: #{model.inspect}")
|
|
46
45
|
end
|
|
47
46
|
end
|
|
48
|
-
|
|
49
|
-
class StorageError < Error
|
|
50
|
-
attr_reader :original_error
|
|
51
|
-
|
|
52
|
-
def initialize(original_error)
|
|
53
|
-
@original_error = original_error
|
|
54
|
-
|
|
55
|
-
super("Failed to store LLM cost event: #{original_error.class}: #{original_error.message}")
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
47
|
end
|
|
@@ -5,12 +5,7 @@ module LlmCostTracker
|
|
|
5
5
|
:event_id,
|
|
6
6
|
:provider,
|
|
7
7
|
:model,
|
|
8
|
-
:
|
|
9
|
-
:output_tokens,
|
|
10
|
-
:total_tokens,
|
|
11
|
-
:cache_read_input_tokens,
|
|
12
|
-
:cache_write_input_tokens,
|
|
13
|
-
:hidden_output_tokens,
|
|
8
|
+
:token_usage,
|
|
14
9
|
:pricing_mode,
|
|
15
10
|
:cost,
|
|
16
11
|
:tags,
|
|
@@ -20,8 +15,13 @@ module LlmCostTracker
|
|
|
20
15
|
:provider_response_id,
|
|
21
16
|
:tracked_at
|
|
22
17
|
) do
|
|
18
|
+
def total_cost
|
|
19
|
+
cost&.fetch(:total_cost, nil)
|
|
20
|
+
end
|
|
21
|
+
|
|
23
22
|
def to_h
|
|
24
23
|
super.merge(
|
|
24
|
+
token_usage: token_usage.to_h,
|
|
25
25
|
cost: cost&.to_h,
|
|
26
26
|
tags: tags ? tags.to_h : {}
|
|
27
27
|
)
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Generators
|
|
8
|
+
class AddTokenUsageGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
TOKEN_COLUMNS = %w[
|
|
12
|
+
cache_read_input_tokens
|
|
13
|
+
cache_write_input_tokens
|
|
14
|
+
cache_write_1h_input_tokens
|
|
15
|
+
hidden_output_tokens
|
|
16
|
+
].freeze
|
|
17
|
+
COST_COLUMNS = %w[
|
|
18
|
+
cache_read_input_cost
|
|
19
|
+
cache_write_input_cost
|
|
20
|
+
cache_write_1h_input_cost
|
|
21
|
+
].freeze
|
|
22
|
+
COLUMN_NAMES = (TOKEN_COLUMNS + COST_COLUMNS + %w[pricing_mode]).freeze
|
|
23
|
+
|
|
24
|
+
source_root File.expand_path("templates", __dir__)
|
|
25
|
+
|
|
26
|
+
desc "Creates a migration to add token usage and token cost columns to llm_api_calls"
|
|
27
|
+
|
|
28
|
+
def create_migration_file
|
|
29
|
+
migration_template(
|
|
30
|
+
"add_token_usage_to_llm_api_calls.rb.erb",
|
|
31
|
+
"db/migrate/add_token_usage_to_llm_api_calls.rb"
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def migration_version
|
|
38
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "rails/generators"
|
|
4
4
|
|
|
5
|
-
require_relative "../../
|
|
6
|
-
require_relative "../../
|
|
7
|
-
require_relative "../../
|
|
5
|
+
require_relative "../../pricing/registry"
|
|
6
|
+
require_relative "../../pricing/sync/registry_loader"
|
|
7
|
+
require_relative "../../pricing/sync/registry_writer"
|
|
8
8
|
|
|
9
9
|
module LlmCostTracker
|
|
10
10
|
module Generators
|
|
@@ -12,11 +12,11 @@ module LlmCostTracker
|
|
|
12
12
|
desc "Creates a local LLM Cost Tracker price snapshot"
|
|
13
13
|
|
|
14
14
|
def create_prices_file
|
|
15
|
-
registry = LlmCostTracker::
|
|
16
|
-
path: LlmCostTracker::
|
|
17
|
-
seed_path: LlmCostTracker::
|
|
15
|
+
registry = LlmCostTracker::Pricing::Sync::RegistryLoader.new.call(
|
|
16
|
+
path: LlmCostTracker::Pricing::Registry::DEFAULT_PRICES_PATH,
|
|
17
|
+
seed_path: LlmCostTracker::Pricing::Registry::DEFAULT_PRICES_PATH
|
|
18
18
|
)
|
|
19
|
-
LlmCostTracker::
|
|
19
|
+
LlmCostTracker::Pricing::Sync::RegistryWriter.new.call(
|
|
20
20
|
path: File.join(destination_root, "config/llm_cost_tracker_prices.yml"),
|
|
21
21
|
registry: registry
|
|
22
22
|
)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require "llm_cost_tracker/
|
|
1
|
+
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
2
|
|
|
3
3
|
class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_version %>
|
|
4
4
|
def up
|
|
@@ -80,7 +80,7 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
|
|
|
80
80
|
elsif mysql?
|
|
81
81
|
"DATE(tracked_at)"
|
|
82
82
|
else
|
|
83
|
-
"
|
|
83
|
+
raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
|
|
84
84
|
end
|
|
85
85
|
end
|
|
86
86
|
|
|
@@ -90,15 +90,15 @@ class AddPeriodTotalsToLlmCostTracker < ActiveRecord::Migration<%= migration_ver
|
|
|
90
90
|
elsif mysql?
|
|
91
91
|
"DATE_FORMAT(tracked_at, '%Y-%m-01')"
|
|
92
92
|
else
|
|
93
|
-
"
|
|
93
|
+
raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
|
|
94
94
|
end
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
def postgresql?
|
|
98
|
-
LlmCostTracker::
|
|
98
|
+
LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
def mysql?
|
|
102
|
-
LlmCostTracker::
|
|
102
|
+
LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
|
|
103
103
|
end
|
|
104
104
|
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
class AddTokenUsageToLlmApiCalls < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def up
|
|
3
|
+
<% LlmCostTracker::Generators::AddTokenUsageGenerator::TOKEN_COLUMNS.each do |column| -%>
|
|
4
|
+
unless column_exists?(:llm_api_calls, :<%= column %>)
|
|
5
|
+
add_column :llm_api_calls, :<%= column %>, :integer, null: false, default: 0
|
|
6
|
+
end
|
|
7
|
+
<% end -%>
|
|
8
|
+
<% LlmCostTracker::Generators::AddTokenUsageGenerator::COST_COLUMNS.each do |column| -%>
|
|
9
|
+
unless column_exists?(:llm_api_calls, :<%= column %>)
|
|
10
|
+
add_column :llm_api_calls, :<%= column %>, :decimal, precision: 20, scale: 8
|
|
11
|
+
end
|
|
12
|
+
<% end -%>
|
|
13
|
+
add_column :llm_api_calls, :pricing_mode, :string unless column_exists?(:llm_api_calls, :pricing_mode)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def down
|
|
17
|
+
remove_column :llm_api_calls, :pricing_mode if column_exists?(:llm_api_calls, :pricing_mode)
|
|
18
|
+
<% (LlmCostTracker::Generators::AddTokenUsageGenerator::COST_COLUMNS + LlmCostTracker::Generators::AddTokenUsageGenerator::TOKEN_COLUMNS).reverse.each do |column| -%>
|
|
19
|
+
remove_column :llm_api_calls, :<%= column %> if column_exists?(:llm_api_calls, :<%= column %>)
|
|
20
|
+
<% end -%>
|
|
21
|
+
end
|
|
22
|
+
end
|