llm_cost_tracker 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -0
- data/README.md +195 -109
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +46 -55
- data/app/services/llm_cost_tracker/dashboard/data_quality_aggregate.rb +81 -0
- data/lib/llm_cost_tracker/budget.rb +34 -37
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +37 -0
- data/lib/llm_cost_tracker/configuration.rb +10 -5
- data/lib/llm_cost_tracker/doctor.rb +166 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +33 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +12 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +38 -8
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +1 -2
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +53 -21
- data/lib/llm_cost_tracker/integrations/anthropic.rb +75 -0
- data/lib/llm_cost_tracker/integrations/base.rb +72 -0
- data/lib/llm_cost_tracker/integrations/object_reader.rb +56 -0
- data/lib/llm_cost_tracker/integrations/openai.rb +95 -0
- data/lib/llm_cost_tracker/integrations/registry.rb +41 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +4 -3
- data/lib/llm_cost_tracker/parsed_usage.rb +8 -1
- data/lib/llm_cost_tracker/parsers/anthropic.rb +17 -49
- data/lib/llm_cost_tracker/parsers/base.rb +80 -0
- data/lib/llm_cost_tracker/parsers/gemini.rb +12 -35
- data/lib/llm_cost_tracker/parsers/openai.rb +1 -6
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +6 -15
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +8 -30
- data/lib/llm_cost_tracker/parsers/registry.rb +17 -2
- data/lib/llm_cost_tracker/price_freshness.rb +38 -0
- data/lib/llm_cost_tracker/price_registry.rb +14 -0
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +2 -1
- data/lib/llm_cost_tracker/price_sync/refresh_plan_builder.rb +4 -2
- data/lib/llm_cost_tracker/price_sync.rb +10 -0
- data/lib/llm_cost_tracker/prices.json +394 -41
- data/lib/llm_cost_tracker/pricing.rb +8 -1
- data/lib/llm_cost_tracker/request_url.rb +20 -0
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +47 -27
- data/lib/llm_cost_tracker/storage/active_record_store.rb +4 -0
- data/lib/llm_cost_tracker/stream_collector.rb +3 -3
- data/lib/llm_cost_tracker/tag_context.rb +52 -0
- data/lib/llm_cost_tracker/tags_column.rb +62 -24
- data/lib/llm_cost_tracker/tracker.rb +5 -2
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +14 -4
- data/lib/tasks/llm_cost_tracker.rake +21 -3
- metadata +13 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/llm_cost_tracker_prices.yml.erb +0 -51
|
@@ -29,84 +29,85 @@ module LlmCostTracker
|
|
|
29
29
|
class DataQuality
|
|
30
30
|
class << self
|
|
31
31
|
def call(scope: LlmCostTracker::LlmApiCall.all)
|
|
32
|
-
|
|
32
|
+
model = scope.klass
|
|
33
|
+
aggregates = DataQualityAggregate.call(scope: scope)
|
|
34
|
+
total = aggregates.fetch(:total_calls).to_i
|
|
33
35
|
|
|
34
36
|
DataQualityStats.new(
|
|
35
37
|
total_calls: total,
|
|
36
|
-
unknown_pricing_count:
|
|
37
|
-
untagged_calls_count: total -
|
|
38
|
-
**latency_stats(
|
|
39
|
-
**stream_stats(
|
|
40
|
-
**provider_response_id_stats(
|
|
41
|
-
**usage_stats(
|
|
38
|
+
unknown_pricing_count: aggregates.fetch(:unknown_pricing_count).to_i,
|
|
39
|
+
untagged_calls_count: total - aggregates.fetch(:tagged_calls_count).to_i,
|
|
40
|
+
**latency_stats(aggregates, model:),
|
|
41
|
+
**stream_stats(aggregates, model:),
|
|
42
|
+
**provider_response_id_stats(aggregates, model:),
|
|
43
|
+
**usage_stats(aggregates, model:),
|
|
42
44
|
unknown_pricing_by_model: unknown_pricing_by_model(scope)
|
|
43
45
|
)
|
|
44
46
|
end
|
|
45
47
|
|
|
46
48
|
private
|
|
47
49
|
|
|
48
|
-
def latency_stats(
|
|
49
|
-
latency_present =
|
|
50
|
+
def latency_stats(aggregates, model:)
|
|
51
|
+
latency_present = model.latency_column?
|
|
50
52
|
|
|
51
53
|
{
|
|
52
|
-
missing_latency_count: latency_present ?
|
|
54
|
+
missing_latency_count: latency_present ? aggregates.fetch(:missing_latency_count).to_i : nil,
|
|
53
55
|
latency_column_present: latency_present
|
|
54
56
|
}
|
|
55
57
|
end
|
|
56
58
|
|
|
57
|
-
def stream_stats(
|
|
58
|
-
stream_present =
|
|
59
|
+
def stream_stats(aggregates, model:)
|
|
60
|
+
stream_present = model.stream_column?
|
|
61
|
+
usage_source_present = model.usage_source_column?
|
|
62
|
+
streaming_missing_usage_count = nil
|
|
63
|
+
if stream_present && usage_source_present
|
|
64
|
+
streaming_missing_usage_count = aggregates.fetch(:streaming_missing_usage_count).to_i
|
|
65
|
+
end
|
|
59
66
|
|
|
60
67
|
{
|
|
61
|
-
streaming_count: stream_present ?
|
|
62
|
-
streaming_missing_usage_count: streaming_missing_usage_count
|
|
68
|
+
streaming_count: stream_present ? aggregates.fetch(:streaming_count).to_i : nil,
|
|
69
|
+
streaming_missing_usage_count: streaming_missing_usage_count,
|
|
63
70
|
stream_column_present: stream_present
|
|
64
71
|
}
|
|
65
72
|
end
|
|
66
73
|
|
|
67
|
-
def provider_response_id_stats(
|
|
68
|
-
column_present =
|
|
74
|
+
def provider_response_id_stats(aggregates, model:)
|
|
75
|
+
column_present = model.provider_response_id_column?
|
|
76
|
+
missing_provider_response_id_count = nil
|
|
77
|
+
if column_present
|
|
78
|
+
missing_provider_response_id_count = aggregates.fetch(:missing_provider_response_id_count).to_i
|
|
79
|
+
end
|
|
69
80
|
|
|
70
81
|
{
|
|
71
|
-
missing_provider_response_id_count:
|
|
82
|
+
missing_provider_response_id_count: missing_provider_response_id_count,
|
|
72
83
|
provider_response_id_column_present: column_present
|
|
73
84
|
}
|
|
74
85
|
end
|
|
75
86
|
|
|
76
|
-
def usage_stats(
|
|
77
|
-
usage_breakdown_present =
|
|
78
|
-
usage_breakdown_cost_present =
|
|
79
|
-
|
|
87
|
+
def usage_stats(aggregates, model:)
|
|
88
|
+
usage_breakdown_present = model.usage_breakdown_columns?
|
|
89
|
+
usage_breakdown_cost_present = model.usage_breakdown_cost_columns?
|
|
90
|
+
cache_read_input_cost = nil
|
|
91
|
+
cache_write_input_cost = nil
|
|
92
|
+
if usage_breakdown_cost_present
|
|
93
|
+
cache_read_input_cost = decimal_sum(aggregates.fetch(:cache_read_input_cost))
|
|
94
|
+
cache_write_input_cost = decimal_sum(aggregates.fetch(:cache_write_input_cost))
|
|
95
|
+
end
|
|
80
96
|
|
|
81
97
|
{
|
|
82
98
|
usage_breakdown_column_present: usage_breakdown_present,
|
|
83
|
-
input_tokens:
|
|
84
|
-
cache_read_input_tokens: usage_breakdown_present ?
|
|
85
|
-
cache_write_input_tokens: usage_breakdown_present ?
|
|
86
|
-
output_tokens:
|
|
87
|
-
hidden_output_tokens: usage_breakdown_present ?
|
|
88
|
-
input_cost: decimal_sum(
|
|
89
|
-
cache_read_input_cost:
|
|
90
|
-
cache_write_input_cost:
|
|
91
|
-
output_cost: decimal_sum(
|
|
99
|
+
input_tokens: aggregates.fetch(:input_tokens).to_i,
|
|
100
|
+
cache_read_input_tokens: usage_breakdown_present ? aggregates.fetch(:cache_read_input_tokens).to_i : nil,
|
|
101
|
+
cache_write_input_tokens: usage_breakdown_present ? aggregates.fetch(:cache_write_input_tokens).to_i : nil,
|
|
102
|
+
output_tokens: aggregates.fetch(:output_tokens).to_i,
|
|
103
|
+
hidden_output_tokens: usage_breakdown_present ? aggregates.fetch(:hidden_output_tokens).to_i : nil,
|
|
104
|
+
input_cost: decimal_sum(aggregates.fetch(:input_cost)),
|
|
105
|
+
cache_read_input_cost: cache_read_input_cost,
|
|
106
|
+
cache_write_input_cost: cache_write_input_cost,
|
|
107
|
+
output_cost: decimal_sum(aggregates.fetch(:output_cost))
|
|
92
108
|
}
|
|
93
109
|
end
|
|
94
110
|
|
|
95
|
-
def usage_sum_columns(usage_breakdown_present, usage_breakdown_cost_present)
|
|
96
|
-
columns = %i[input_tokens output_tokens input_cost output_cost]
|
|
97
|
-
if usage_breakdown_present
|
|
98
|
-
columns += %i[cache_read_input_tokens cache_write_input_tokens hidden_output_tokens]
|
|
99
|
-
end
|
|
100
|
-
columns += %i[cache_read_input_cost cache_write_input_cost] if usage_breakdown_cost_present
|
|
101
|
-
columns
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
def streaming_missing_usage_count(scope, stream_present)
|
|
105
|
-
return unless stream_present && LlmCostTracker::LlmApiCall.usage_source_column?
|
|
106
|
-
|
|
107
|
-
scope.streaming_missing_usage.count
|
|
108
|
-
end
|
|
109
|
-
|
|
110
111
|
def unknown_pricing_by_model(scope)
|
|
111
112
|
scope.unknown_pricing
|
|
112
113
|
.group(:model)
|
|
@@ -116,16 +117,6 @@ module LlmCostTracker
|
|
|
116
117
|
.to_h
|
|
117
118
|
end
|
|
118
119
|
|
|
119
|
-
def sum_columns(scope, columns)
|
|
120
|
-
values = scope.unscope(:order).pick(*columns.map { |column| sum_expression(scope, column) })
|
|
121
|
-
|
|
122
|
-
columns.zip(values).to_h
|
|
123
|
-
end
|
|
124
|
-
|
|
125
|
-
def sum_expression(scope, column)
|
|
126
|
-
Arel.sql("COALESCE(SUM(#{scope.connection.quote_column_name(column)}), 0)")
|
|
127
|
-
end
|
|
128
|
-
|
|
129
120
|
def decimal_sum(value)
|
|
130
121
|
value.to_f.round(8)
|
|
131
122
|
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module Dashboard
|
|
5
|
+
class DataQualityAggregate
|
|
6
|
+
class << self
|
|
7
|
+
def call(scope:)
|
|
8
|
+
model = scope.klass
|
|
9
|
+
expressions = aggregate_expressions(scope, model:)
|
|
10
|
+
values = Array(scope.unscope(:order).pick(*expressions.values))
|
|
11
|
+
|
|
12
|
+
expressions.keys.zip(values).to_h
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def aggregate_expressions(scope, model:)
|
|
18
|
+
usage_breakdown_present = model.usage_breakdown_columns?
|
|
19
|
+
usage_breakdown_cost_present = model.usage_breakdown_cost_columns?
|
|
20
|
+
|
|
21
|
+
expressions = {
|
|
22
|
+
total_calls: Arel.sql("COUNT(*)"),
|
|
23
|
+
unknown_pricing_count: conditional_count_expression("total_cost IS NULL"),
|
|
24
|
+
tagged_calls_count: tagged_calls_expression(model)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if model.latency_column?
|
|
28
|
+
expressions[:missing_latency_count] = conditional_count_expression("latency_ms IS NULL")
|
|
29
|
+
end
|
|
30
|
+
expressions[:streaming_count] = conditional_count_expression("stream") if model.stream_column?
|
|
31
|
+
if model.stream_column? && model.usage_source_column?
|
|
32
|
+
expressions[:streaming_missing_usage_count] =
|
|
33
|
+
conditional_count_expression("stream AND (usage_source = 'unknown' OR usage_source IS NULL)")
|
|
34
|
+
end
|
|
35
|
+
if model.provider_response_id_column?
|
|
36
|
+
expressions[:missing_provider_response_id_count] =
|
|
37
|
+
conditional_count_expression("provider_response_id IS NULL OR provider_response_id = ''")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
usage_sum_columns(usage_breakdown_present, usage_breakdown_cost_present).each do |column|
|
|
41
|
+
expressions[column] = sum_expression(scope, column)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
expressions
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def usage_sum_columns(usage_breakdown_present, usage_breakdown_cost_present)
|
|
48
|
+
columns = %i[input_tokens output_tokens input_cost output_cost]
|
|
49
|
+
if usage_breakdown_present
|
|
50
|
+
columns += %i[cache_read_input_tokens cache_write_input_tokens hidden_output_tokens]
|
|
51
|
+
end
|
|
52
|
+
columns += %i[cache_read_input_cost cache_write_input_cost] if usage_breakdown_cost_present
|
|
53
|
+
columns
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def conditional_count_expression(predicate)
|
|
57
|
+
Arel.sql("COALESCE(SUM(CASE WHEN #{predicate} THEN 1 ELSE 0 END), 0)")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def tagged_calls_expression(model)
|
|
61
|
+
table = model.quoted_table_name
|
|
62
|
+
column = "#{table}.#{model.connection.quote_column_name('tags')}"
|
|
63
|
+
|
|
64
|
+
Arel.sql(case
|
|
65
|
+
when model.tags_jsonb_column?
|
|
66
|
+
"COALESCE(SUM(CASE WHEN #{column} <> '{}'::jsonb THEN 1 ELSE 0 END), 0)"
|
|
67
|
+
when model.tags_mysql_json_column?
|
|
68
|
+
"COALESCE(SUM(CASE WHEN JSON_LENGTH(#{column}) > 0 THEN 1 ELSE 0 END), 0)"
|
|
69
|
+
else
|
|
70
|
+
"COALESCE(SUM(CASE WHEN #{column} IS NOT NULL AND #{column} <> '' " \
|
|
71
|
+
"AND #{column} <> '{}' THEN 1 ELSE 0 END), 0)"
|
|
72
|
+
end)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def sum_expression(scope, column)
|
|
76
|
+
Arel.sql("COALESCE(SUM(#{scope.connection.quote_column_name(column)}), 0)")
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -10,8 +10,16 @@ module LlmCostTracker
|
|
|
10
10
|
return unless config.budget_exceeded_behavior == :block_requests
|
|
11
11
|
return unless config.active_record?
|
|
12
12
|
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
budgets = enforce_period_budgets(config)
|
|
14
|
+
return if budgets.empty?
|
|
15
|
+
|
|
16
|
+
totals = active_record_totals(budgets.keys, time: Time.now.utc)
|
|
17
|
+
|
|
18
|
+
budgets.each do |period, budget|
|
|
19
|
+
total = totals.fetch(period)
|
|
20
|
+
|
|
21
|
+
handle_exceeded(budget_type: period, total: total, budget: budget) if total >= budget
|
|
22
|
+
end
|
|
15
23
|
end
|
|
16
24
|
|
|
17
25
|
def check!(event)
|
|
@@ -19,21 +27,18 @@ module LlmCostTracker
|
|
|
19
27
|
return unless event.cost
|
|
20
28
|
|
|
21
29
|
check_per_call_budget(event, config)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
end
|
|
30
|
+
budgets = check_period_budgets(config)
|
|
31
|
+
totals = totals_for_check(event, config, budgets)
|
|
25
32
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def enforce_period_budget(period, budget)
|
|
29
|
-
return unless budget
|
|
33
|
+
budgets.each do |period, budget|
|
|
34
|
+
total = totals.fetch(period)
|
|
30
35
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
handle_exceeded(budget_type: period, total: total, budget: budget)
|
|
36
|
+
handle_exceeded(budget_type: period, total: total, budget: budget, last_event: event) if total >= budget
|
|
37
|
+
end
|
|
35
38
|
end
|
|
36
39
|
|
|
40
|
+
private
|
|
41
|
+
|
|
37
42
|
def check_per_call_budget(event, config)
|
|
38
43
|
budget = config.per_call_budget
|
|
39
44
|
return unless budget
|
|
@@ -44,40 +49,32 @@ module LlmCostTracker
|
|
|
44
49
|
handle_exceeded(budget_type: :per_call, total: call_cost, budget: budget, last_event: event)
|
|
45
50
|
end
|
|
46
51
|
|
|
47
|
-
def
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
else
|
|
53
|
-
event.cost.total_cost
|
|
54
|
-
end
|
|
55
|
-
return unless total >= budget
|
|
56
|
-
|
|
57
|
-
handle_exceeded(budget_type: period, total: total, budget: budget, last_event: event)
|
|
52
|
+
def enforce_period_budgets(config)
|
|
53
|
+
{
|
|
54
|
+
monthly: config.monthly_budget,
|
|
55
|
+
daily: config.daily_budget
|
|
56
|
+
}.compact
|
|
58
57
|
end
|
|
59
58
|
|
|
60
|
-
def
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
59
|
+
def check_period_budgets(config)
|
|
60
|
+
{
|
|
61
|
+
daily: config.daily_budget,
|
|
62
|
+
monthly: config.monthly_budget
|
|
63
|
+
}.compact
|
|
65
64
|
end
|
|
66
65
|
|
|
67
|
-
def
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
def totals_for_check(event, config, budgets)
|
|
67
|
+
return {} if budgets.empty?
|
|
68
|
+
return active_record_totals(budgets.keys, time: event.tracked_at) if config.active_record?
|
|
70
69
|
|
|
71
|
-
|
|
72
|
-
rescue LoadError => e
|
|
73
|
-
raise Error, "ActiveRecord storage requires the active_record gem: #{e.message}"
|
|
70
|
+
budgets.to_h { |period, _budget| [period, event.cost.total_cost] }
|
|
74
71
|
end
|
|
75
72
|
|
|
76
|
-
def
|
|
73
|
+
def active_record_totals(periods, time:)
|
|
77
74
|
require_relative "llm_api_call" unless defined?(LlmCostTracker::LlmApiCall)
|
|
78
75
|
require_relative "storage/active_record_store" unless defined?(LlmCostTracker::Storage::ActiveRecordStore)
|
|
79
76
|
|
|
80
|
-
LlmCostTracker::Storage::ActiveRecordStore.
|
|
77
|
+
LlmCostTracker::Storage::ActiveRecordStore.period_totals(periods, time: time)
|
|
81
78
|
rescue LoadError => e
|
|
82
79
|
raise Error, "ActiveRecord storage requires the active_record gem: #{e.message}"
|
|
83
80
|
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module ConfigurationInstrumentation
|
|
5
|
+
def instrument(*names)
|
|
6
|
+
ensure_shared_configuration_mutable!
|
|
7
|
+
@instrumented_integrations = (@instrumented_integrations + normalize_instrumentation_names(names)).uniq
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def instrumented?(name)
|
|
11
|
+
@instrumented_integrations.include?(name.to_sym)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def normalize_instrumentation_names(names)
|
|
17
|
+
names.flatten.flat_map do |name|
|
|
18
|
+
key = name.to_sym
|
|
19
|
+
next available_instrumentation_names if key == :all
|
|
20
|
+
|
|
21
|
+
validate_instrumentation_name!(key)
|
|
22
|
+
key
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def validate_instrumentation_name!(name)
|
|
27
|
+
return if available_instrumentation_names.include?(name)
|
|
28
|
+
|
|
29
|
+
raise Error, "Unknown integration: #{name.inspect}. " \
|
|
30
|
+
"Use one of: #{available_instrumentation_names.join(', ')}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def available_instrumentation_names
|
|
34
|
+
Integrations::Registry::INTEGRATIONS.keys
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -2,9 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "errors"
|
|
4
4
|
require_relative "value_helpers"
|
|
5
|
+
require_relative "configuration/instrumentation"
|
|
5
6
|
|
|
6
7
|
module LlmCostTracker
|
|
7
8
|
class Configuration
|
|
9
|
+
include ConfigurationInstrumentation
|
|
10
|
+
|
|
8
11
|
OPENAI_COMPATIBLE_PROVIDERS = {
|
|
9
12
|
"openrouter.ai" => "openrouter",
|
|
10
13
|
"api.deepseek.com" => "deepseek"
|
|
@@ -36,6 +39,7 @@ module LlmCostTracker
|
|
|
36
39
|
:budget_exceeded_behavior,
|
|
37
40
|
:default_tags,
|
|
38
41
|
:pricing_overrides,
|
|
42
|
+
:instrumented_integrations,
|
|
39
43
|
:report_tag_breakdowns,
|
|
40
44
|
:storage_backend,
|
|
41
45
|
:storage_error_behavior,
|
|
@@ -58,6 +62,7 @@ module LlmCostTracker
|
|
|
58
62
|
@log_level = :info
|
|
59
63
|
@prices_file = nil
|
|
60
64
|
@pricing_overrides = {}
|
|
65
|
+
@instrumented_integrations = []
|
|
61
66
|
@report_tag_breakdowns = []
|
|
62
67
|
self.openai_compatible_providers = OPENAI_COMPATIBLE_PROVIDERS
|
|
63
68
|
@finalized = false
|
|
@@ -97,13 +102,10 @@ module LlmCostTracker
|
|
|
97
102
|
end
|
|
98
103
|
end
|
|
99
104
|
|
|
100
|
-
def normalize_openai_compatible_providers!
|
|
101
|
-
self.openai_compatible_providers = openai_compatible_providers
|
|
102
|
-
end
|
|
103
|
-
|
|
104
105
|
def finalize!
|
|
105
106
|
@default_tags = ValueHelpers.deep_freeze(@default_tags || {})
|
|
106
107
|
@pricing_overrides = ValueHelpers.deep_freeze(@pricing_overrides || {})
|
|
108
|
+
@instrumented_integrations = ValueHelpers.deep_freeze(@instrumented_integrations || [])
|
|
107
109
|
@report_tag_breakdowns = ValueHelpers.deep_freeze(Array(@report_tag_breakdowns))
|
|
108
110
|
@openai_compatible_providers = ValueHelpers.deep_freeze(@openai_compatible_providers || {})
|
|
109
111
|
@finalized = true
|
|
@@ -116,6 +118,10 @@ module LlmCostTracker
|
|
|
116
118
|
copy = dup
|
|
117
119
|
copy.instance_variable_set(:@default_tags, ValueHelpers.deep_dup(@default_tags || {}))
|
|
118
120
|
copy.instance_variable_set(:@pricing_overrides, ValueHelpers.deep_dup(@pricing_overrides || {}))
|
|
121
|
+
copy.instance_variable_set(
|
|
122
|
+
:@instrumented_integrations,
|
|
123
|
+
ValueHelpers.deep_dup(@instrumented_integrations || [])
|
|
124
|
+
)
|
|
119
125
|
copy.instance_variable_set(:@report_tag_breakdowns, ValueHelpers.deep_dup(@report_tag_breakdowns || []))
|
|
120
126
|
copy.instance_variable_set(
|
|
121
127
|
:@openai_compatible_providers,
|
|
@@ -126,7 +132,6 @@ module LlmCostTracker
|
|
|
126
132
|
end
|
|
127
133
|
|
|
128
134
|
def active_record? = storage_backend == :active_record
|
|
129
|
-
def log? = storage_backend == :log
|
|
130
135
|
|
|
131
136
|
private
|
|
132
137
|
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "price_freshness"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
class Doctor
|
|
7
|
+
Check = Data.define(:status, :name, :message)
|
|
8
|
+
CORE_COLUMNS = %w[provider model input_tokens output_tokens total_tokens total_cost tags tracked_at].freeze
|
|
9
|
+
FEATURE_COLUMNS = {
|
|
10
|
+
"latency_ms" => "bin/rails generate llm_cost_tracker:add_latency_ms",
|
|
11
|
+
"stream" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
12
|
+
"usage_source" => "bin/rails generate llm_cost_tracker:add_streaming",
|
|
13
|
+
"provider_response_id" => "bin/rails generate llm_cost_tracker:add_provider_response_id",
|
|
14
|
+
"cache_read_input_tokens" => "bin/rails generate llm_cost_tracker:add_usage_breakdown",
|
|
15
|
+
"cache_write_input_tokens" => "bin/rails generate llm_cost_tracker:add_usage_breakdown",
|
|
16
|
+
"hidden_output_tokens" => "bin/rails generate llm_cost_tracker:add_usage_breakdown",
|
|
17
|
+
"pricing_mode" => "bin/rails generate llm_cost_tracker:add_usage_breakdown"
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
def call = new.checks
|
|
22
|
+
|
|
23
|
+
def report(checks = call)
|
|
24
|
+
(["LLM Cost Tracker doctor"] + checks.map { |check| format_check(check) }).join("\n")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def healthy?(checks = call)
|
|
28
|
+
checks.none? { |check| check.status == :error }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def format_check(check)
|
|
34
|
+
"[#{check.status}] #{check.name}: #{check.message}"
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def checks
|
|
39
|
+
[
|
|
40
|
+
configuration_check,
|
|
41
|
+
*integration_checks,
|
|
42
|
+
active_record_check,
|
|
43
|
+
table_check,
|
|
44
|
+
column_check,
|
|
45
|
+
period_totals_check,
|
|
46
|
+
prices_check,
|
|
47
|
+
calls_check
|
|
48
|
+
].compact
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def configuration_check
|
|
54
|
+
Check.new(:ok, "configuration", "storage_backend=#{LlmCostTracker.configuration.storage_backend.inspect}")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def integration_checks
|
|
58
|
+
LlmCostTracker::Integrations.checks.map do |check|
|
|
59
|
+
Check.new(check.status, check.name.to_s, check.message)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def active_record_check
|
|
64
|
+
return Check.new(:ok, "storage", "ActiveRecord storage is disabled") unless active_record_storage?
|
|
65
|
+
return Check.new(:ok, "active_record", "available") if active_record_available?
|
|
66
|
+
|
|
67
|
+
Check.new(:error, "active_record", "unavailable; add ActiveRecord/Rails or change storage_backend")
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def table_check
|
|
71
|
+
return unless active_record_storage? && active_record_available?
|
|
72
|
+
return Check.new(:ok, "llm_api_calls", "table exists") if llm_api_calls_table?
|
|
73
|
+
|
|
74
|
+
Check.new(
|
|
75
|
+
:error,
|
|
76
|
+
"llm_api_calls",
|
|
77
|
+
"missing; run bin/rails generate llm_cost_tracker:install && bin/rails db:migrate"
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def column_check
|
|
82
|
+
return unless active_record_storage? && llm_api_calls_table?
|
|
83
|
+
|
|
84
|
+
columns = column_names("llm_api_calls")
|
|
85
|
+
missing_core = CORE_COLUMNS - columns
|
|
86
|
+
missing_features = FEATURE_COLUMNS.keys - columns
|
|
87
|
+
if missing_core.any?
|
|
88
|
+
return Check.new(:error, "llm_api_calls columns", "missing core columns: #{missing_core.join(', ')}")
|
|
89
|
+
end
|
|
90
|
+
if missing_features.any?
|
|
91
|
+
return Check.new(
|
|
92
|
+
:warn,
|
|
93
|
+
"llm_api_calls columns",
|
|
94
|
+
"missing optional columns; run #{feature_generators(missing_features).join(' && ')}"
|
|
95
|
+
)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
Check.new(:ok, "llm_api_calls columns", "current")
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def period_totals_check
|
|
102
|
+
return unless active_record_storage? && llm_api_calls_table?
|
|
103
|
+
if table_exists?("llm_cost_tracker_period_totals")
|
|
104
|
+
return Check.new(:ok, "period totals", "llm_cost_tracker_period_totals exists")
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
Check.new(:warn, "period totals", "missing; budget preflight falls back to llm_api_calls sums")
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def prices_check
|
|
111
|
+
path = LlmCostTracker.configuration.prices_file
|
|
112
|
+
unless path
|
|
113
|
+
return Check.new(
|
|
114
|
+
:warn,
|
|
115
|
+
"prices",
|
|
116
|
+
"using bundled prices updated_at=#{builtin_prices_updated_at}; configure prices_file for production"
|
|
117
|
+
)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
count = LlmCostTracker::PriceRegistry.file_prices(path).size
|
|
121
|
+
metadata = LlmCostTracker::PriceRegistry.file_metadata(path)
|
|
122
|
+
status, freshness = LlmCostTracker::PriceFreshness.call(metadata)
|
|
123
|
+
Check.new(status, "prices", "loaded #{count} models from #{path}; #{freshness}")
|
|
124
|
+
rescue LlmCostTracker::Error => e
|
|
125
|
+
Check.new(:error, "prices", e.message)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def calls_check
|
|
129
|
+
return unless active_record_storage? && llm_api_calls_table?
|
|
130
|
+
|
|
131
|
+
count = LlmCostTracker::LlmApiCall.count
|
|
132
|
+
return Check.new(:warn, "tracked calls", "none recorded yet") if count.zero?
|
|
133
|
+
|
|
134
|
+
latest = LlmCostTracker::LlmApiCall.maximum(:tracked_at)&.utc&.iso8601
|
|
135
|
+
Check.new(:ok, "tracked calls", "#{count} recorded; latest #{latest}")
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def active_record_storage? = LlmCostTracker.configuration.storage_backend == :active_record
|
|
139
|
+
|
|
140
|
+
def active_record_available?
|
|
141
|
+
require_relative "llm_api_call" unless defined?(LlmCostTracker::LlmApiCall)
|
|
142
|
+
LlmCostTracker::LlmApiCall.connection
|
|
143
|
+
true
|
|
144
|
+
rescue LoadError, StandardError
|
|
145
|
+
false
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def llm_api_calls_table?
|
|
149
|
+
active_record_available? && table_exists?("llm_api_calls")
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def table_exists?(name)
|
|
153
|
+
LlmCostTracker::LlmApiCall.connection.data_source_exists?(name)
|
|
154
|
+
rescue StandardError
|
|
155
|
+
false
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def column_names(table) = LlmCostTracker::LlmApiCall.connection.columns(table).map(&:name)
|
|
159
|
+
|
|
160
|
+
def feature_generators(columns) = columns.map { |column| FEATURE_COLUMNS.fetch(column) }.uniq
|
|
161
|
+
|
|
162
|
+
def builtin_prices_updated_at
|
|
163
|
+
LlmCostTracker::Pricing.metadata.fetch("updated_at", "unknown")
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|
|
@@ -11,6 +11,8 @@ module LlmCostTracker
|
|
|
11
11
|
source_root File.expand_path("templates", __dir__)
|
|
12
12
|
|
|
13
13
|
desc "Creates the LlmCostTracker migration and initializer"
|
|
14
|
+
class_option :dashboard, type: :boolean, default: false
|
|
15
|
+
class_option :prices, type: :boolean, default: false
|
|
14
16
|
|
|
15
17
|
def create_migration_file
|
|
16
18
|
migration_template(
|
|
@@ -26,11 +28,42 @@ module LlmCostTracker
|
|
|
26
28
|
)
|
|
27
29
|
end
|
|
28
30
|
|
|
31
|
+
def create_prices_file
|
|
32
|
+
return unless options[:prices]
|
|
33
|
+
|
|
34
|
+
invoke "llm_cost_tracker:prices"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def mount_engine
|
|
38
|
+
return unless options[:dashboard]
|
|
39
|
+
|
|
40
|
+
add_engine_require
|
|
41
|
+
route %(mount LlmCostTracker::Engine => "/llm-costs")
|
|
42
|
+
end
|
|
43
|
+
|
|
29
44
|
private
|
|
30
45
|
|
|
31
46
|
def migration_version
|
|
32
47
|
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
33
48
|
end
|
|
49
|
+
|
|
50
|
+
def add_engine_require
|
|
51
|
+
return unless File.exist?("config/application.rb")
|
|
52
|
+
|
|
53
|
+
contents = File.read("config/application.rb")
|
|
54
|
+
return if contents.include?(%(require "llm_cost_tracker/engine"))
|
|
55
|
+
|
|
56
|
+
unless contents.include?(%(require "rails/all"\n))
|
|
57
|
+
prepend_to_file("config/application.rb", %(require "llm_cost_tracker/engine"\n))
|
|
58
|
+
return
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
inject_into_file(
|
|
62
|
+
"config/application.rb",
|
|
63
|
+
%(require "llm_cost_tracker/engine"\n),
|
|
64
|
+
after: %(require "rails/all"\n)
|
|
65
|
+
)
|
|
66
|
+
end
|
|
34
67
|
end
|
|
35
68
|
end
|
|
36
69
|
end
|