llm_cost_tracker 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/README.md +21 -16
- data/app/assets/llm_cost_tracker/application.css +3 -0
- data/app/controllers/llm_cost_tracker/application_controller.rb +22 -4
- data/app/controllers/llm_cost_tracker/calls_controller.rb +6 -11
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +5 -1
- data/app/controllers/llm_cost_tracker/models_controller.rb +0 -1
- data/app/controllers/llm_cost_tracker/tags_controller.rb +1 -8
- data/app/helpers/llm_cost_tracker/application_helper.rb +2 -1
- data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +1 -2
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +10 -27
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +58 -0
- data/app/models/llm_cost_tracker/ingestion/event.rb +13 -0
- data/app/models/llm_cost_tracker/ingestion/lease.rb +11 -0
- data/app/models/llm_cost_tracker/ledger/call.rb +45 -0
- data/app/models/llm_cost_tracker/ledger/call_metrics.rb +66 -0
- data/app/models/llm_cost_tracker/ledger/period/grouping.rb +71 -0
- data/app/models/llm_cost_tracker/ledger/period/total.rb +13 -0
- data/app/models/llm_cost_tracker/ledger/tags/accessors.rb +19 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +111 -94
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +2 -2
- data/app/services/llm_cost_tracker/dashboard/filter.rb +7 -18
- data/app/services/llm_cost_tracker/dashboard/overview_stats.rb +58 -67
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +59 -0
- data/app/services/llm_cost_tracker/dashboard/params.rb +26 -0
- data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +18 -20
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +4 -13
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +28 -61
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +8 -21
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +12 -47
- data/app/views/llm_cost_tracker/calls/index.html.erb +12 -18
- data/app/views/llm_cost_tracker/calls/show.html.erb +30 -32
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +17 -19
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +108 -135
- data/app/views/llm_cost_tracker/models/index.html.erb +8 -9
- data/app/views/llm_cost_tracker/shared/setup_required.html.erb +13 -2
- data/app/views/llm_cost_tracker/tags/show.html.erb +20 -20
- data/lib/llm_cost_tracker/budget.rb +8 -20
- data/lib/llm_cost_tracker/capture/stream.rb +9 -0
- data/lib/llm_cost_tracker/capture/stream_collector.rb +189 -0
- data/lib/llm_cost_tracker/{integrations → capture}/stream_tracker.rb +41 -73
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +3 -7
- data/lib/llm_cost_tracker/configuration.rb +33 -36
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +61 -0
- data/lib/llm_cost_tracker/doctor/check.rb +7 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +22 -59
- data/lib/llm_cost_tracker/doctor/price_check.rb +60 -0
- data/lib/llm_cost_tracker/doctor.rb +63 -71
- data/lib/llm_cost_tracker/errors.rb +4 -15
- data/lib/llm_cost_tracker/event.rb +6 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_token_usage_generator.rb +42 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +2 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +7 -7
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +3 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_token_usage_to_llm_api_calls.rb.erb +22 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +9 -14
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +0 -4
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_cost_precision.rb.erb +12 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb +2 -2
- data/lib/llm_cost_tracker/{storage/active_record_inbox_batch.rb → ingestion/batch.rb} +21 -20
- data/lib/llm_cost_tracker/ingestion/inbox.rb +105 -0
- data/lib/llm_cost_tracker/{storage/active_record_ingestor_lease.rb → ingestion/lease_claim.rb} +5 -7
- data/lib/llm_cost_tracker/{storage/active_record_ingestor.rb → ingestion/worker.rb} +38 -48
- data/lib/llm_cost_tracker/ingestion.rb +129 -0
- data/lib/llm_cost_tracker/integrations/anthropic.rb +66 -31
- data/lib/llm_cost_tracker/integrations/base.rb +73 -34
- data/lib/llm_cost_tracker/integrations/openai.rb +43 -37
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +40 -30
- data/lib/llm_cost_tracker/integrations.rb +43 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +66 -0
- data/lib/llm_cost_tracker/{storage/active_record_periods.rb → ledger/period.rb} +2 -2
- data/lib/llm_cost_tracker/ledger/rollups/batch.rb +43 -0
- data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb +46 -0
- data/lib/llm_cost_tracker/ledger/rollups.rb +87 -0
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +51 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +101 -0
- data/lib/llm_cost_tracker/ledger/schema/period_totals.rb +32 -0
- data/lib/llm_cost_tracker/ledger/store.rb +60 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +29 -0
- data/lib/llm_cost_tracker/ledger/tags/sql.rb +33 -0
- data/lib/llm_cost_tracker/ledger.rb +13 -0
- data/lib/llm_cost_tracker/logging.rb +3 -6
- data/lib/llm_cost_tracker/middleware/faraday.rb +88 -46
- data/lib/llm_cost_tracker/parsers/anthropic.rb +62 -29
- data/lib/llm_cost_tracker/parsers/base.rb +12 -21
- data/lib/llm_cost_tracker/parsers/gemini.rb +50 -25
- data/lib/llm_cost_tracker/parsers/openai.rb +27 -5
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +14 -4
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +58 -25
- data/lib/llm_cost_tracker/parsers/sse.rb +4 -7
- data/lib/llm_cost_tracker/parsers.rb +20 -0
- data/lib/llm_cost_tracker/prices.json +361 -36
- data/lib/llm_cost_tracker/pricing/components.rb +37 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +46 -50
- data/lib/llm_cost_tracker/pricing/explainer.rb +25 -30
- data/lib/llm_cost_tracker/pricing/lookup.rb +67 -46
- data/lib/llm_cost_tracker/pricing/registry.rb +156 -0
- data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +107 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +53 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_loader.rb +63 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +31 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +159 -0
- data/lib/llm_cost_tracker/pricing/unknown.rb +46 -0
- data/lib/llm_cost_tracker/pricing.rb +33 -32
- data/lib/llm_cost_tracker/railtie.rb +7 -8
- data/lib/llm_cost_tracker/report/data.rb +72 -0
- data/lib/llm_cost_tracker/report/formatter.rb +69 -0
- data/lib/llm_cost_tracker/report.rb +8 -8
- data/lib/llm_cost_tracker/retention.rb +27 -10
- data/lib/llm_cost_tracker/tags/context.rb +35 -0
- data/lib/llm_cost_tracker/tags/key.rb +18 -0
- data/lib/llm_cost_tracker/tags/sanitizer.rb +68 -0
- data/lib/llm_cost_tracker/token_usage.rb +67 -0
- data/lib/llm_cost_tracker/tracker.rb +39 -69
- data/lib/llm_cost_tracker/usage_capture.rb +37 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +56 -78
- data/lib/tasks/llm_cost_tracker.rake +18 -13
- metadata +54 -58
- data/app/services/llm_cost_tracker/dashboard/data_quality_aggregate.rb +0 -81
- data/app/services/llm_cost_tracker/pagination.rb +0 -57
- data/lib/llm_cost_tracker/active_record_adapter.rb +0 -53
- data/lib/llm_cost_tracker/capture_verifier.rb +0 -64
- data/lib/llm_cost_tracker/cost.rb +0 -12
- data/lib/llm_cost_tracker/doctor/capture_check.rb +0 -39
- data/lib/llm_cost_tracker/event_metadata.rb +0 -52
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_usage_breakdown_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_usage_breakdown_to_llm_api_calls.rb.erb +0 -29
- data/lib/llm_cost_tracker/inbox_event.rb +0 -9
- data/lib/llm_cost_tracker/ingestor_lease.rb +0 -9
- data/lib/llm_cost_tracker/integrations/object_reader.rb +0 -56
- data/lib/llm_cost_tracker/integrations/registry.rb +0 -71
- data/lib/llm_cost_tracker/llm_api_call.rb +0 -60
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +0 -63
- data/lib/llm_cost_tracker/parameter_hash.rb +0 -33
- data/lib/llm_cost_tracker/parsed_usage.rb +0 -72
- data/lib/llm_cost_tracker/parsers/registry.rb +0 -58
- data/lib/llm_cost_tracker/period_grouping.rb +0 -67
- data/lib/llm_cost_tracker/period_total.rb +0 -9
- data/lib/llm_cost_tracker/price_freshness.rb +0 -38
- data/lib/llm_cost_tracker/price_registry.rb +0 -144
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +0 -104
- data/lib/llm_cost_tracker/price_sync/registry_diff.rb +0 -51
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +0 -61
- data/lib/llm_cost_tracker/price_sync/registry_writer.rb +0 -29
- data/lib/llm_cost_tracker/price_sync.rb +0 -144
- data/lib/llm_cost_tracker/report_data.rb +0 -94
- data/lib/llm_cost_tracker/report_formatter.rb +0 -67
- data/lib/llm_cost_tracker/request_url.rb +0 -20
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +0 -167
- data/lib/llm_cost_tracker/storage/active_record_connection_cleanup.rb +0 -13
- data/lib/llm_cost_tracker/storage/active_record_inbox.rb +0 -160
- data/lib/llm_cost_tracker/storage/active_record_period_totals.rb +0 -84
- data/lib/llm_cost_tracker/storage/active_record_rollup_batch.rb +0 -41
- data/lib/llm_cost_tracker/storage/active_record_rollup_upsert_sql.rb +0 -42
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +0 -146
- data/lib/llm_cost_tracker/storage/active_record_store.rb +0 -145
- data/lib/llm_cost_tracker/storage/writer.rb +0 -35
- data/lib/llm_cost_tracker/stream_capture.rb +0 -7
- data/lib/llm_cost_tracker/stream_collector.rb +0 -199
- data/lib/llm_cost_tracker/tag_accessors.rb +0 -15
- data/lib/llm_cost_tracker/tag_context.rb +0 -52
- data/lib/llm_cost_tracker/tag_key.rb +0 -16
- data/lib/llm_cost_tracker/tag_query.rb +0 -43
- data/lib/llm_cost_tracker/tag_sanitizer.rb +0 -81
- data/lib/llm_cost_tracker/tag_sql.rb +0 -34
- data/lib/llm_cost_tracker/tags_column.rb +0 -105
- data/lib/llm_cost_tracker/unknown_pricing.rb +0 -54
- data/lib/llm_cost_tracker/usage_breakdown.rb +0 -30
- data/lib/llm_cost_tracker/value_helpers.rb +0 -40
|
@@ -2,10 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Dashboard
|
|
5
|
-
ProviderRow = Data.define(:provider, :calls, :total_cost, :share_percent)
|
|
6
|
-
|
|
7
5
|
class ProviderBreakdown
|
|
8
|
-
def self.call(scope: LlmCostTracker::
|
|
6
|
+
def self.call(scope: LlmCostTracker::Ledger::Call.all)
|
|
9
7
|
new(scope: scope).rows
|
|
10
8
|
end
|
|
11
9
|
|
|
@@ -14,28 +12,28 @@ module LlmCostTracker
|
|
|
14
12
|
end
|
|
15
13
|
|
|
16
14
|
def rows
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
.to_a
|
|
22
|
-
|
|
23
|
-
total_cost = grouped.sum { |row| row.total_cost_sum.to_f }
|
|
24
|
-
|
|
25
|
-
grouped.map do |row|
|
|
26
|
-
cost = row.total_cost_sum.to_f
|
|
27
|
-
ProviderRow.new(
|
|
28
|
-
provider: row.provider,
|
|
29
|
-
calls: row.calls_count.to_i,
|
|
30
|
-
total_cost: cost,
|
|
31
|
-
share_percent: total_cost.positive? ? (cost / total_cost) * 100.0 : 0.0
|
|
32
|
-
)
|
|
33
|
-
end
|
|
15
|
+
scope
|
|
16
|
+
.group(:provider)
|
|
17
|
+
.select(selects)
|
|
18
|
+
.order(Arel.sql("total_cost DESC, calls DESC"))
|
|
34
19
|
end
|
|
35
20
|
|
|
36
21
|
private
|
|
37
22
|
|
|
38
23
|
attr_reader :scope
|
|
24
|
+
|
|
25
|
+
def selects
|
|
26
|
+
<<~SQL.squish
|
|
27
|
+
provider,
|
|
28
|
+
COUNT(*) AS calls,
|
|
29
|
+
COALESCE(SUM(total_cost), 0) AS total_cost,
|
|
30
|
+
CASE
|
|
31
|
+
WHEN SUM(COALESCE(SUM(total_cost), 0)) OVER () > 0
|
|
32
|
+
THEN COALESCE(SUM(total_cost), 0) / SUM(COALESCE(SUM(total_cost), 0)) OVER () * 100.0
|
|
33
|
+
ELSE 0
|
|
34
|
+
END AS share_percent
|
|
35
|
+
SQL
|
|
36
|
+
end
|
|
39
37
|
end
|
|
40
38
|
end
|
|
41
39
|
end
|
|
@@ -2,20 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Dashboard
|
|
5
|
-
SpendAnomalyData = Data.define(
|
|
6
|
-
:provider,
|
|
7
|
-
:model,
|
|
8
|
-
:day,
|
|
9
|
-
:latest_spend,
|
|
10
|
-
:baseline_mean,
|
|
11
|
-
:ratio
|
|
12
|
-
)
|
|
13
|
-
|
|
14
5
|
class SpendAnomaly
|
|
15
6
|
WINDOW_DAYS = 7
|
|
16
7
|
|
|
17
8
|
class << self
|
|
18
|
-
def call(from:, to:, scope: LlmCostTracker::
|
|
9
|
+
def call(from:, to:, scope: LlmCostTracker::Ledger::Call.all)
|
|
19
10
|
new(scope: scope, from: from, to: to).alert
|
|
20
11
|
end
|
|
21
12
|
end
|
|
@@ -29,7 +20,7 @@ module LlmCostTracker
|
|
|
29
20
|
def alert
|
|
30
21
|
return nil if from > (to - WINDOW_DAYS)
|
|
31
22
|
|
|
32
|
-
alerts.max_by { |item| [item.ratio || 0.0, item.latest_spend] }
|
|
23
|
+
alerts.max_by { |item| [item.fetch(:ratio) || 0.0, item.fetch(:latest_spend)] }
|
|
33
24
|
end
|
|
34
25
|
|
|
35
26
|
private
|
|
@@ -47,14 +38,14 @@ module LlmCostTracker
|
|
|
47
38
|
threshold = mean + (2 * Math.sqrt(variance))
|
|
48
39
|
next unless latest_spend > threshold
|
|
49
40
|
|
|
50
|
-
rows <<
|
|
41
|
+
rows << {
|
|
51
42
|
provider: provider,
|
|
52
43
|
model: model,
|
|
53
44
|
day: to,
|
|
54
45
|
latest_spend: latest_spend,
|
|
55
46
|
baseline_mean: mean,
|
|
56
47
|
ratio: mean.positive? ? (latest_spend / mean) : nil
|
|
57
|
-
|
|
48
|
+
}
|
|
58
49
|
end
|
|
59
50
|
end
|
|
60
51
|
|
|
@@ -2,86 +2,58 @@
|
|
|
2
2
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Dashboard
|
|
5
|
-
TagBreakdownRow = Data.define(
|
|
6
|
-
:value,
|
|
7
|
-
:calls,
|
|
8
|
-
:total_cost,
|
|
9
|
-
:average_cost_per_call
|
|
10
|
-
)
|
|
11
|
-
|
|
12
|
-
TagBreakdownResult = Data.define(
|
|
13
|
-
:rows,
|
|
14
|
-
:total_calls,
|
|
15
|
-
:tagged_calls,
|
|
16
|
-
:distinct_values,
|
|
17
|
-
:limit
|
|
18
|
-
) do
|
|
19
|
-
def limited? = distinct_values > rows.size
|
|
20
|
-
end
|
|
21
|
-
|
|
22
5
|
class TagBreakdown
|
|
23
6
|
DEFAULT_LIMIT = 100
|
|
24
7
|
|
|
25
8
|
class << self
|
|
26
|
-
def call(key:, scope: LlmCostTracker::
|
|
27
|
-
new(scope: scope, key: key, limit: limit)
|
|
9
|
+
def call(key:, scope: LlmCostTracker::Ledger::Call.all, limit: DEFAULT_LIMIT)
|
|
10
|
+
new(scope: scope, key: key, limit: limit)
|
|
28
11
|
end
|
|
29
12
|
end
|
|
30
13
|
|
|
14
|
+
attr_reader :limit
|
|
15
|
+
|
|
31
16
|
def initialize(scope:, key:, limit:)
|
|
32
17
|
@scope = scope
|
|
33
|
-
@key = LlmCostTracker::
|
|
34
|
-
|
|
35
|
-
@
|
|
18
|
+
@key = LlmCostTracker::Tags::Key.validate!(key, error_class: LlmCostTracker::InvalidFilterError)
|
|
19
|
+
limit = limit.to_i
|
|
20
|
+
@limit = limit.positive? ? [limit, DEFAULT_LIMIT].min : DEFAULT_LIMIT
|
|
36
21
|
end
|
|
37
22
|
|
|
38
|
-
def
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
TagBreakdownResult.new(
|
|
42
|
-
rows: rows,
|
|
43
|
-
total_calls: counts.fetch(:total_calls),
|
|
44
|
-
tagged_calls: counts.fetch(:tagged_calls),
|
|
45
|
-
distinct_values: counts.fetch(:distinct_values),
|
|
46
|
-
limit: limit
|
|
47
|
-
)
|
|
23
|
+
def rows
|
|
24
|
+
@rows ||= scope.klass.find_by_sql(rows_sql)
|
|
48
25
|
end
|
|
49
26
|
|
|
50
|
-
|
|
27
|
+
def total_calls
|
|
28
|
+
summary_counts.total_calls.to_i
|
|
29
|
+
end
|
|
51
30
|
|
|
52
|
-
|
|
31
|
+
def tagged_calls
|
|
32
|
+
summary_counts.tagged_calls.to_i
|
|
33
|
+
end
|
|
53
34
|
|
|
54
|
-
def
|
|
55
|
-
|
|
56
|
-
calls = row["calls_count"].to_i
|
|
57
|
-
total_cost = row["total_cost_sum"].to_f
|
|
58
|
-
TagBreakdownRow.new(
|
|
59
|
-
value: LlmCostTracker::LlmApiCall.tag_value_label(row["tag_value"]),
|
|
60
|
-
calls: calls,
|
|
61
|
-
total_cost: total_cost,
|
|
62
|
-
average_cost_per_call: calls.positive? ? total_cost / calls : 0.0
|
|
63
|
-
)
|
|
64
|
-
end
|
|
35
|
+
def distinct_values
|
|
36
|
+
summary_counts.distinct_values.to_i
|
|
65
37
|
end
|
|
66
38
|
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
attr_reader :scope, :key
|
|
42
|
+
|
|
67
43
|
def summary_counts
|
|
68
|
-
|
|
69
|
-
{
|
|
70
|
-
total_calls: row["total_calls"].to_i,
|
|
71
|
-
tagged_calls: row["tagged_calls"].to_i,
|
|
72
|
-
distinct_values: row["distinct_values"].to_i
|
|
73
|
-
}
|
|
44
|
+
@summary_counts ||= scope.klass.find_by_sql(summary_sql).first
|
|
74
45
|
end
|
|
75
46
|
|
|
76
47
|
def rows_sql
|
|
77
48
|
<<~SQL.squish
|
|
78
|
-
SELECT #{tag_expression} AS
|
|
79
|
-
COUNT(*) AS
|
|
80
|
-
COALESCE(SUM(sub.total_cost), 0) AS
|
|
49
|
+
SELECT #{tag_expression} AS value,
|
|
50
|
+
COUNT(*) AS calls,
|
|
51
|
+
COALESCE(SUM(sub.total_cost), 0) AS total_cost,
|
|
52
|
+
COALESCE(SUM(sub.total_cost), 0) / NULLIF(COUNT(*), 0) AS average_cost_per_call
|
|
81
53
|
FROM (#{scope.to_sql}) AS sub
|
|
82
54
|
WHERE #{tag_present_predicate}
|
|
83
55
|
GROUP BY #{tag_expression}
|
|
84
|
-
ORDER BY
|
|
56
|
+
ORDER BY total_cost DESC, calls DESC, value ASC
|
|
85
57
|
LIMIT #{limit}
|
|
86
58
|
SQL
|
|
87
59
|
end
|
|
@@ -100,12 +72,7 @@ module LlmCostTracker
|
|
|
100
72
|
end
|
|
101
73
|
|
|
102
74
|
def tag_expression
|
|
103
|
-
@tag_expression ||= LlmCostTracker::
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
def normalized_limit(value)
|
|
107
|
-
value = value.to_i
|
|
108
|
-
value.positive? ? [value, DEFAULT_LIMIT].min : DEFAULT_LIMIT
|
|
75
|
+
@tag_expression ||= LlmCostTracker::Ledger::Call.tag_value_expression(key, table_name: "sub")
|
|
109
76
|
end
|
|
110
77
|
end
|
|
111
78
|
end
|
|
@@ -2,32 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Dashboard
|
|
5
|
-
TagKeyRow = Data.define(:key, :calls_count, :distinct_values)
|
|
6
|
-
|
|
7
5
|
class TagKeyExplorer
|
|
8
6
|
DEFAULT_LIMIT = 100
|
|
9
7
|
|
|
10
8
|
class << self
|
|
11
|
-
def call(scope: LlmCostTracker::
|
|
9
|
+
def call(scope: LlmCostTracker::Ledger::Call.all, limit: DEFAULT_LIMIT)
|
|
12
10
|
new(scope: scope, limit: limit).rows
|
|
13
11
|
end
|
|
14
12
|
end
|
|
15
13
|
|
|
16
14
|
def initialize(scope:, limit:)
|
|
17
15
|
@scope = scope
|
|
18
|
-
@connection = LlmCostTracker::
|
|
19
|
-
|
|
16
|
+
@connection = LlmCostTracker::Ledger::Call.connection
|
|
17
|
+
limit = limit.to_i
|
|
18
|
+
@limit = limit.positive? ? [limit, DEFAULT_LIMIT].min : DEFAULT_LIMIT
|
|
20
19
|
end
|
|
21
20
|
|
|
22
21
|
def rows
|
|
23
|
-
|
|
24
|
-
results.map do |row|
|
|
25
|
-
TagKeyRow.new(
|
|
26
|
-
key: row["key"].to_s,
|
|
27
|
-
calls_count: row["calls_count"].to_i,
|
|
28
|
-
distinct_values: row["distinct_values"].to_i
|
|
29
|
-
)
|
|
30
|
-
end
|
|
22
|
+
scope.klass.find_by_sql(build_sql)
|
|
31
23
|
rescue StandardError => e
|
|
32
24
|
LlmCostTracker::Logging.warn("Tag key discovery failed (#{connection.adapter_name}): #{e.class}: #{e.message}")
|
|
33
25
|
[]
|
|
@@ -42,10 +34,10 @@ module LlmCostTracker
|
|
|
42
34
|
end
|
|
43
35
|
|
|
44
36
|
def build_sql
|
|
45
|
-
return postgresql_sql if
|
|
46
|
-
return mysql_sql if
|
|
37
|
+
return postgresql_sql if Ledger::Schema::Adapter.postgresql?(connection)
|
|
38
|
+
return mysql_sql if Ledger::Schema::Adapter.mysql?(connection)
|
|
47
39
|
|
|
48
|
-
|
|
40
|
+
Ledger::Schema::Adapter.ensure_supported!(connection)
|
|
49
41
|
end
|
|
50
42
|
|
|
51
43
|
def mysql_sql
|
|
@@ -82,11 +74,6 @@ module LlmCostTracker
|
|
|
82
74
|
LIMIT #{limit}
|
|
83
75
|
SQL
|
|
84
76
|
end
|
|
85
|
-
|
|
86
|
-
def normalized_limit(value)
|
|
87
|
-
value = value.to_i
|
|
88
|
-
value.positive? ? [value, DEFAULT_LIMIT].min : DEFAULT_LIMIT
|
|
89
|
-
end
|
|
90
77
|
end
|
|
91
78
|
end
|
|
92
79
|
end
|
|
@@ -8,7 +8,7 @@ module LlmCostTracker
|
|
|
8
8
|
DEFAULT_DAYS = 30
|
|
9
9
|
|
|
10
10
|
class << self
|
|
11
|
-
def call(scope: LlmCostTracker::
|
|
11
|
+
def call(scope: LlmCostTracker::Ledger::Call.all, from: nil, to: Date.current)
|
|
12
12
|
new(scope: scope, from: from, to: to).points
|
|
13
13
|
end
|
|
14
14
|
end
|
|
@@ -2,24 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Dashboard
|
|
5
|
-
TopModel = Data.define(
|
|
6
|
-
:provider,
|
|
7
|
-
:model,
|
|
8
|
-
:calls,
|
|
9
|
-
:total_cost,
|
|
10
|
-
:average_cost_per_call,
|
|
11
|
-
:input_tokens,
|
|
12
|
-
:output_tokens,
|
|
13
|
-
:average_latency_ms
|
|
14
|
-
)
|
|
15
|
-
|
|
16
5
|
class TopModels
|
|
17
6
|
DEFAULT_LIMIT = 5
|
|
18
7
|
SORT_OPTIONS = %w[cost calls avg_cost latency].freeze
|
|
19
8
|
DEFAULT_SORT = "cost"
|
|
20
9
|
|
|
21
10
|
class << self
|
|
22
|
-
def call(scope: LlmCostTracker::
|
|
11
|
+
def call(scope: LlmCostTracker::Ledger::Call.all, limit: DEFAULT_LIMIT, sort: DEFAULT_SORT)
|
|
23
12
|
new(scope: scope, limit: limit, sort: sort).rows
|
|
24
13
|
end
|
|
25
14
|
end
|
|
@@ -31,28 +20,6 @@ module LlmCostTracker
|
|
|
31
20
|
end
|
|
32
21
|
|
|
33
22
|
def rows
|
|
34
|
-
grouped_rows.map do |row|
|
|
35
|
-
calls = row.calls_count.to_i
|
|
36
|
-
total_cost = row.total_cost_sum.to_f
|
|
37
|
-
|
|
38
|
-
TopModel.new(
|
|
39
|
-
provider: row.provider,
|
|
40
|
-
model: row.model,
|
|
41
|
-
calls: calls,
|
|
42
|
-
total_cost: total_cost,
|
|
43
|
-
average_cost_per_call: calls.positive? ? total_cost / calls : 0.0,
|
|
44
|
-
input_tokens: row.input_tokens_sum.to_i,
|
|
45
|
-
output_tokens: row.output_tokens_sum.to_i,
|
|
46
|
-
average_latency_ms: average_latency(row)
|
|
47
|
-
)
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
private
|
|
52
|
-
|
|
53
|
-
attr_reader :scope, :limit, :sort
|
|
54
|
-
|
|
55
|
-
def grouped_rows
|
|
56
23
|
scope
|
|
57
24
|
.group(:provider, :model)
|
|
58
25
|
.select(selects)
|
|
@@ -60,6 +27,10 @@ module LlmCostTracker
|
|
|
60
27
|
.then { |r| limit ? r.limit(limit) : r }
|
|
61
28
|
end
|
|
62
29
|
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
attr_reader :scope, :limit, :sort
|
|
33
|
+
|
|
63
34
|
def order_sql
|
|
64
35
|
case sort
|
|
65
36
|
when "calls"
|
|
@@ -67,8 +38,6 @@ module LlmCostTracker
|
|
|
67
38
|
when "avg_cost"
|
|
68
39
|
"COALESCE(SUM(total_cost), 0) / NULLIF(COUNT(*), 0) DESC"
|
|
69
40
|
when "latency"
|
|
70
|
-
return "COALESCE(SUM(total_cost), 0) DESC" unless scope.klass.latency_column?
|
|
71
|
-
|
|
72
41
|
"CASE WHEN AVG(latency_ms) IS NULL THEN 1 ELSE 0 END ASC, AVG(latency_ms) DESC"
|
|
73
42
|
else
|
|
74
43
|
"COALESCE(SUM(total_cost), 0) DESC"
|
|
@@ -79,20 +48,16 @@ module LlmCostTracker
|
|
|
79
48
|
columns = [
|
|
80
49
|
"provider",
|
|
81
50
|
"model",
|
|
82
|
-
"COUNT(*) AS
|
|
83
|
-
"COALESCE(SUM(total_cost), 0) AS
|
|
84
|
-
"COALESCE(SUM(
|
|
85
|
-
"COALESCE(SUM(
|
|
51
|
+
"COUNT(*) AS calls",
|
|
52
|
+
"COALESCE(SUM(total_cost), 0) AS total_cost",
|
|
53
|
+
"COALESCE(SUM(total_cost), 0) / NULLIF(COUNT(*), 0) AS average_cost_per_call",
|
|
54
|
+
"COALESCE(SUM(total_tokens), 0) AS total_tokens",
|
|
55
|
+
"COALESCE(SUM(input_tokens), 0) AS input_tokens",
|
|
56
|
+
"COALESCE(SUM(output_tokens), 0) AS output_tokens",
|
|
57
|
+
"AVG(latency_ms) AS average_latency_ms"
|
|
86
58
|
]
|
|
87
|
-
columns << "AVG(latency_ms) AS average_latency" if scope.klass.latency_column?
|
|
88
59
|
columns.join(", ")
|
|
89
60
|
end
|
|
90
|
-
|
|
91
|
-
def average_latency(row)
|
|
92
|
-
return nil unless scope.klass.latency_column?
|
|
93
|
-
|
|
94
|
-
row.average_latency&.to_f
|
|
95
|
-
end
|
|
96
61
|
end
|
|
97
62
|
end
|
|
98
63
|
end
|
|
@@ -34,15 +34,13 @@
|
|
|
34
34
|
id: "lct-model" %>
|
|
35
35
|
</div>
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
<
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
</div>
|
|
45
|
-
<% end %>
|
|
37
|
+
<div class="lct-field">
|
|
38
|
+
<label for="lct-stream">Stream</label>
|
|
39
|
+
<%= select_tag :stream,
|
|
40
|
+
options_for_select(LlmCostTracker::DashboardFilterHelper::STREAM_FILTER_OPTIONS, params[:stream]),
|
|
41
|
+
include_blank: "All calls",
|
|
42
|
+
id: "lct-stream" %>
|
|
43
|
+
</div>
|
|
46
44
|
|
|
47
45
|
<div class="lct-field">
|
|
48
46
|
<label for="lct-sort">Sort</label>
|
|
@@ -51,9 +49,9 @@
|
|
|
51
49
|
[["Recent first", ""],
|
|
52
50
|
["Most expensive", "expensive"],
|
|
53
51
|
["Largest input", "input"],
|
|
54
|
-
["Largest output", "output"]
|
|
55
|
-
|
|
56
|
-
|
|
52
|
+
["Largest output", "output"],
|
|
53
|
+
["Slowest", "slow"],
|
|
54
|
+
["Unknown pricing only", "unknown_pricing"]],
|
|
57
55
|
@sort
|
|
58
56
|
),
|
|
59
57
|
id: "lct-sort" %>
|
|
@@ -104,9 +102,7 @@
|
|
|
104
102
|
<th class="lct-num">Output</th>
|
|
105
103
|
<th class="lct-num">Total</th>
|
|
106
104
|
<th class="lct-num">Cost</th>
|
|
107
|
-
|
|
108
|
-
<th class="lct-num">Latency</th>
|
|
109
|
-
<% end %>
|
|
105
|
+
<th class="lct-num">Latency</th>
|
|
110
106
|
<th>Tags</th>
|
|
111
107
|
<th></th>
|
|
112
108
|
</tr>
|
|
@@ -121,9 +117,7 @@
|
|
|
121
117
|
<td class="lct-num"><%= format_tokens(call.output_tokens) %></td>
|
|
122
118
|
<td class="lct-num"><%= format_tokens(call.total_tokens) %></td>
|
|
123
119
|
<td class="lct-num<%= ' lct-num-muted' if call.total_cost.nil? %>"><%= optional_money(call.total_cost) %></td>
|
|
124
|
-
|
|
125
|
-
<td class="lct-num<%= ' lct-num-muted' if call.latency_ms.nil? %>"><%= call.latency_ms ? "#{number(call.latency_ms)}ms" : "n/a" %></td>
|
|
126
|
-
<% end %>
|
|
120
|
+
<td class="lct-num<%= ' lct-num-muted' if call.latency_ms.nil? %>"><%= call.latency_ms ? "#{number(call.latency_ms)}ms" : "n/a" %></td>
|
|
127
121
|
<td><%= render "llm_cost_tracker/shared/tag_chips", tags: call.parsed_tags %></td>
|
|
128
122
|
<td><%= link_to "Details", call_path(call), class: "lct-button lct-button-secondary lct-button-compact" %></td>
|
|
129
123
|
</tr>
|
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
<%
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
]
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
1
|
+
<% priced_components = token_usage_stack_components %>
|
|
2
|
+
<% token_segments = priced_components.map do |component|
|
|
3
|
+
token_key = component.fetch(:token_key)
|
|
4
|
+
value = @call.has_attribute?(token_key) ? @call[token_key] : 0
|
|
5
|
+
{ label: component.fetch(:label), value: value, formatted_value: format_tokens(value), css_class: component.fetch(:css_class) }
|
|
6
|
+
end %>
|
|
7
|
+
<% cost_segments = [] %>
|
|
8
|
+
<% unless @call.total_cost.nil? %>
|
|
9
|
+
<% cost_segments = priced_components.map do |component|
|
|
10
|
+
cost_key = component.fetch(:cost_key)
|
|
11
|
+
value = @call.has_attribute?(cost_key) ? @call[cost_key] : nil
|
|
12
|
+
{ label: component.fetch(:label), value: value, formatted_value: optional_money(value), css_class: component.fetch(:css_class) }
|
|
13
|
+
end %>
|
|
14
|
+
<% end %>
|
|
9
15
|
|
|
10
16
|
<section class="lct-panel">
|
|
11
17
|
<p class="lct-muted"><%= link_to "Back to calls", calls_path %></p>
|
|
@@ -34,12 +40,10 @@
|
|
|
34
40
|
<span class="lct-call-summary-label">Total tokens</span>
|
|
35
41
|
<strong><%= format_tokens(@call.total_tokens) %></strong>
|
|
36
42
|
</div>
|
|
37
|
-
|
|
38
|
-
<
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
</div>
|
|
42
|
-
<% end %>
|
|
43
|
+
<div class="lct-call-summary-item">
|
|
44
|
+
<span class="lct-call-summary-label">Latency</span>
|
|
45
|
+
<strong><%= @call.latency_ms ? "#{optional_number(@call.latency_ms)}ms" : "n/a" %></strong>
|
|
46
|
+
</div>
|
|
43
47
|
</div>
|
|
44
48
|
</div>
|
|
45
49
|
|
|
@@ -73,10 +77,8 @@
|
|
|
73
77
|
<dt>Pricing Status</dt>
|
|
74
78
|
<dd><%= pricing_status(@call) %></dd>
|
|
75
79
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
<dd><%= @call.provider_response_id.presence || "n/a" %></dd>
|
|
79
|
-
<% end %>
|
|
80
|
+
<dt>Provider Response ID</dt>
|
|
81
|
+
<dd><%= @call.provider_response_id.presence || "n/a" %></dd>
|
|
80
82
|
|
|
81
83
|
<% if @call.has_attribute?("created_at") %>
|
|
82
84
|
<dt>Created At</dt>
|
|
@@ -90,28 +92,24 @@
|
|
|
90
92
|
</dl>
|
|
91
93
|
|
|
92
94
|
<dl class="lct-dl">
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
<dd><%= format_tokens(@call.output_tokens) %></dd>
|
|
95
|
+
<% priced_components.each do |component| %>
|
|
96
|
+
<dt><%= component.fetch(:label).titleize %> Tokens</dt>
|
|
97
|
+
<dd><%= format_tokens(@call[component.fetch(:token_key)]) %></dd>
|
|
98
|
+
<% end %>
|
|
98
99
|
|
|
99
100
|
<dt>Total Tokens</dt>
|
|
100
101
|
<dd><%= format_tokens(@call.total_tokens) %></dd>
|
|
101
102
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
<dd><%= optional_money(@call.output_cost) %></dd>
|
|
103
|
+
<% priced_components.each do |component| %>
|
|
104
|
+
<dt><%= component.fetch(:label).titleize %> Cost</dt>
|
|
105
|
+
<dd><%= optional_money(@call[component.fetch(:cost_key)]) %></dd>
|
|
106
|
+
<% end %>
|
|
107
107
|
|
|
108
108
|
<dt>Total Cost</dt>
|
|
109
109
|
<dd><%= optional_money(@call.total_cost) %></dd>
|
|
110
110
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
<dd><%= @call.latency_ms ? "#{optional_number(@call.latency_ms)}ms" : "n/a" %></dd>
|
|
114
|
-
<% end %>
|
|
111
|
+
<dt>Latency</dt>
|
|
112
|
+
<dd><%= @call.latency_ms ? "#{optional_number(@call.latency_ms)}ms" : "n/a" %></dd>
|
|
115
113
|
</dl>
|
|
116
114
|
</div>
|
|
117
115
|
</section>
|
|
@@ -29,15 +29,13 @@
|
|
|
29
29
|
id: "lct-overview-model" %>
|
|
30
30
|
</div>
|
|
31
31
|
|
|
32
|
-
|
|
33
|
-
<
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
</div>
|
|
40
|
-
<% end %>
|
|
32
|
+
<div class="lct-field">
|
|
33
|
+
<label for="lct-overview-stream">Stream</label>
|
|
34
|
+
<%= select_tag :stream,
|
|
35
|
+
options_for_select(LlmCostTracker::DashboardFilterHelper::STREAM_FILTER_OPTIONS, params[:stream]),
|
|
36
|
+
include_blank: "All calls",
|
|
37
|
+
id: "lct-overview-stream" %>
|
|
38
|
+
</div>
|
|
41
39
|
|
|
42
40
|
<div class="lct-filter-actions">
|
|
43
41
|
<button class="lct-button" type="submit">Apply</button>
|
|
@@ -49,7 +47,7 @@
|
|
|
49
47
|
<%= render "llm_cost_tracker/shared/active_filters", chips: active_tag_filters, clear_path: root_path %>
|
|
50
48
|
</section>
|
|
51
49
|
|
|
52
|
-
<% if @stats.total_calls.zero? %>
|
|
50
|
+
<% if @stats.total_calls.to_i.zero? %>
|
|
53
51
|
<section class="lct-panel lct-empty">
|
|
54
52
|
<h2 class="lct-state-title">No LLM calls yet</h2>
|
|
55
53
|
<p class="lct-state-copy">Tracked requests will appear here after your application records its first LLM API call.</p>
|
|
@@ -58,11 +56,11 @@
|
|
|
58
56
|
</div>
|
|
59
57
|
</section>
|
|
60
58
|
<% else %>
|
|
61
|
-
<% if @stats.unknown_pricing_count.positive? %>
|
|
59
|
+
<% if @stats.unknown_pricing_count.to_i.positive? %>
|
|
62
60
|
<aside class="lct-banner lct-banner-warning" role="status">
|
|
63
61
|
<div class="lct-banner-body">
|
|
64
62
|
<p class="lct-banner-title">
|
|
65
|
-
<%= number(@stats.unknown_pricing_count) %> call<%= "s" unless @stats.unknown_pricing_count == 1 %> missing pricing
|
|
63
|
+
<%= number(@stats.unknown_pricing_count) %> call<%= "s" unless @stats.unknown_pricing_count.to_i == 1 %> missing pricing
|
|
66
64
|
<span class="lct-banner-muted">· <%= percent(coverage_percent(@stats.unknown_pricing_count, @stats.total_calls)) %> of the slice</span>
|
|
67
65
|
</p>
|
|
68
66
|
<p class="lct-banner-copy">Totals undercount until every model has a known price. Update <code class="lct-code">pricing_overrides</code> or <code class="lct-code">prices_file</code> to fix.</p>
|
|
@@ -76,18 +74,18 @@
|
|
|
76
74
|
<div class="lct-banner-body">
|
|
77
75
|
<p class="lct-banner-title">
|
|
78
76
|
Spend anomaly detected
|
|
79
|
-
<span class="lct-banner-muted">· <code class="lct-code"><%= @spend_anomaly.model %></code> on <%= @spend_anomaly.day.strftime("%b %-d") %></span>
|
|
77
|
+
<span class="lct-banner-muted">· <code class="lct-code"><%= @spend_anomaly.fetch(:model) %></code> on <%= @spend_anomaly.fetch(:day).strftime("%b %-d") %></span>
|
|
80
78
|
</p>
|
|
81
79
|
<p class="lct-banner-copy">
|
|
82
|
-
<% if @spend_anomaly.ratio %>
|
|
83
|
-
<%= number_with_precision(@spend_anomaly.ratio, precision: 1) %>× its prior 7-day average in this slice
|
|
80
|
+
<% if @spend_anomaly.fetch(:ratio) %>
|
|
81
|
+
<%= number_with_precision(@spend_anomaly.fetch(:ratio), precision: 1) %>× its prior 7-day average in this slice
|
|
84
82
|
<% else %>
|
|
85
|
-
<%= money(@spend_anomaly.latest_spend) %> after seven quiet days in this slice
|
|
83
|
+
<%= money(@spend_anomaly.fetch(:latest_spend)) %> after seven quiet days in this slice
|
|
86
84
|
<% end %>
|
|
87
85
|
</p>
|
|
88
86
|
</div>
|
|
89
87
|
<%= link_to "Review calls →",
|
|
90
|
-
calls_path(current_query(provider: @spend_anomaly.provider, model: @spend_anomaly.model, from: @to_date.iso8601, to: @to_date.iso8601, page: nil, per: nil, format: nil)),
|
|
88
|
+
calls_path(current_query(provider: @spend_anomaly.fetch(:provider), model: @spend_anomaly.fetch(:model), from: @to_date.iso8601, to: @to_date.iso8601, page: nil, per: nil, format: nil)),
|
|
91
89
|
class: "lct-button lct-button-secondary" %>
|
|
92
90
|
</aside>
|
|
93
91
|
<% end %>
|
|
@@ -125,8 +123,8 @@
|
|
|
125
123
|
|
|
126
124
|
</div>
|
|
127
125
|
|
|
128
|
-
<% if @
|
|
129
|
-
<% budget = @
|
|
126
|
+
<% if @monthly_budget_status %>
|
|
127
|
+
<% budget = @monthly_budget_status %>
|
|
130
128
|
<% fill_mod = budget_fill_modifier(budget[:percent_used]) %>
|
|
131
129
|
<% projected_marker = [[budget[:projected_percent_used].to_f, 0.0].max, 100.0].min %>
|
|
132
130
|
<% projected_delta = budget[:projected_delta].to_f %>
|