llm_cost_tracker 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/README.md +21 -16
- data/app/assets/llm_cost_tracker/application.css +3 -0
- data/app/controllers/llm_cost_tracker/application_controller.rb +22 -4
- data/app/controllers/llm_cost_tracker/calls_controller.rb +6 -11
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +5 -1
- data/app/controllers/llm_cost_tracker/models_controller.rb +0 -1
- data/app/controllers/llm_cost_tracker/tags_controller.rb +1 -8
- data/app/helpers/llm_cost_tracker/application_helper.rb +2 -1
- data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +1 -2
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +10 -27
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +58 -0
- data/app/models/llm_cost_tracker/ingestion/event.rb +13 -0
- data/app/models/llm_cost_tracker/ingestion/lease.rb +11 -0
- data/app/models/llm_cost_tracker/ledger/call.rb +45 -0
- data/app/models/llm_cost_tracker/ledger/call_metrics.rb +66 -0
- data/app/models/llm_cost_tracker/ledger/period/grouping.rb +71 -0
- data/app/models/llm_cost_tracker/ledger/period/total.rb +13 -0
- data/app/models/llm_cost_tracker/ledger/tags/accessors.rb +19 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +111 -94
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +2 -2
- data/app/services/llm_cost_tracker/dashboard/filter.rb +7 -18
- data/app/services/llm_cost_tracker/dashboard/overview_stats.rb +58 -67
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +59 -0
- data/app/services/llm_cost_tracker/dashboard/params.rb +26 -0
- data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +18 -20
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +4 -13
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +28 -61
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +8 -21
- data/app/services/llm_cost_tracker/dashboard/time_series.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/top_models.rb +12 -47
- data/app/views/llm_cost_tracker/calls/index.html.erb +12 -18
- data/app/views/llm_cost_tracker/calls/show.html.erb +30 -32
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +17 -19
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +108 -135
- data/app/views/llm_cost_tracker/models/index.html.erb +8 -9
- data/app/views/llm_cost_tracker/shared/setup_required.html.erb +13 -2
- data/app/views/llm_cost_tracker/tags/show.html.erb +20 -20
- data/lib/llm_cost_tracker/budget.rb +8 -20
- data/lib/llm_cost_tracker/capture/stream.rb +9 -0
- data/lib/llm_cost_tracker/capture/stream_collector.rb +189 -0
- data/lib/llm_cost_tracker/{integrations → capture}/stream_tracker.rb +41 -73
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +3 -7
- data/lib/llm_cost_tracker/configuration.rb +33 -36
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +61 -0
- data/lib/llm_cost_tracker/doctor/check.rb +7 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +22 -59
- data/lib/llm_cost_tracker/doctor/price_check.rb +60 -0
- data/lib/llm_cost_tracker/doctor.rb +63 -71
- data/lib/llm_cost_tracker/errors.rb +4 -15
- data/lib/llm_cost_tracker/event.rb +6 -6
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_token_usage_generator.rb +42 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +2 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/prices_generator.rb +7 -7
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_period_totals_to_llm_cost_tracker.rb.erb +3 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_token_usage_to_llm_api_calls.rb.erb +22 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +9 -14
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +0 -4
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_cost_precision.rb.erb +12 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_llm_api_call_tags_to_jsonb.rb.erb +2 -2
- data/lib/llm_cost_tracker/{storage/active_record_inbox_batch.rb → ingestion/batch.rb} +21 -20
- data/lib/llm_cost_tracker/ingestion/inbox.rb +105 -0
- data/lib/llm_cost_tracker/{storage/active_record_ingestor_lease.rb → ingestion/lease_claim.rb} +5 -7
- data/lib/llm_cost_tracker/{storage/active_record_ingestor.rb → ingestion/worker.rb} +38 -48
- data/lib/llm_cost_tracker/ingestion.rb +129 -0
- data/lib/llm_cost_tracker/integrations/anthropic.rb +66 -31
- data/lib/llm_cost_tracker/integrations/base.rb +73 -34
- data/lib/llm_cost_tracker/integrations/openai.rb +43 -37
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +40 -30
- data/lib/llm_cost_tracker/integrations.rb +43 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +66 -0
- data/lib/llm_cost_tracker/{storage/active_record_periods.rb → ledger/period.rb} +2 -2
- data/lib/llm_cost_tracker/ledger/rollups/batch.rb +43 -0
- data/lib/llm_cost_tracker/ledger/rollups/upsert_sql.rb +46 -0
- data/lib/llm_cost_tracker/ledger/rollups.rb +87 -0
- data/lib/llm_cost_tracker/ledger/schema/adapter.rb +51 -0
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +101 -0
- data/lib/llm_cost_tracker/ledger/schema/period_totals.rb +32 -0
- data/lib/llm_cost_tracker/ledger/store.rb +60 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +29 -0
- data/lib/llm_cost_tracker/ledger/tags/sql.rb +33 -0
- data/lib/llm_cost_tracker/ledger.rb +13 -0
- data/lib/llm_cost_tracker/logging.rb +3 -6
- data/lib/llm_cost_tracker/middleware/faraday.rb +88 -46
- data/lib/llm_cost_tracker/parsers/anthropic.rb +62 -29
- data/lib/llm_cost_tracker/parsers/base.rb +12 -21
- data/lib/llm_cost_tracker/parsers/gemini.rb +50 -25
- data/lib/llm_cost_tracker/parsers/openai.rb +27 -5
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +14 -4
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +58 -25
- data/lib/llm_cost_tracker/parsers/sse.rb +4 -7
- data/lib/llm_cost_tracker/parsers.rb +20 -0
- data/lib/llm_cost_tracker/prices.json +361 -36
- data/lib/llm_cost_tracker/pricing/components.rb +37 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +46 -50
- data/lib/llm_cost_tracker/pricing/explainer.rb +25 -30
- data/lib/llm_cost_tracker/pricing/lookup.rb +67 -46
- data/lib/llm_cost_tracker/pricing/registry.rb +156 -0
- data/lib/llm_cost_tracker/pricing/sync/fetcher.rb +107 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_diff.rb +53 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_loader.rb +63 -0
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +31 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +159 -0
- data/lib/llm_cost_tracker/pricing/unknown.rb +46 -0
- data/lib/llm_cost_tracker/pricing.rb +33 -32
- data/lib/llm_cost_tracker/railtie.rb +7 -8
- data/lib/llm_cost_tracker/report/data.rb +72 -0
- data/lib/llm_cost_tracker/report/formatter.rb +69 -0
- data/lib/llm_cost_tracker/report.rb +8 -8
- data/lib/llm_cost_tracker/retention.rb +27 -10
- data/lib/llm_cost_tracker/tags/context.rb +35 -0
- data/lib/llm_cost_tracker/tags/key.rb +18 -0
- data/lib/llm_cost_tracker/tags/sanitizer.rb +68 -0
- data/lib/llm_cost_tracker/token_usage.rb +67 -0
- data/lib/llm_cost_tracker/tracker.rb +39 -69
- data/lib/llm_cost_tracker/usage_capture.rb +37 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +56 -78
- data/lib/tasks/llm_cost_tracker.rake +18 -13
- metadata +54 -58
- data/app/services/llm_cost_tracker/dashboard/data_quality_aggregate.rb +0 -81
- data/app/services/llm_cost_tracker/pagination.rb +0 -57
- data/lib/llm_cost_tracker/active_record_adapter.rb +0 -53
- data/lib/llm_cost_tracker/capture_verifier.rb +0 -64
- data/lib/llm_cost_tracker/cost.rb +0 -12
- data/lib/llm_cost_tracker/doctor/capture_check.rb +0 -39
- data/lib/llm_cost_tracker/event_metadata.rb +0 -52
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_usage_breakdown_generator.rb +0 -29
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_usage_breakdown_to_llm_api_calls.rb.erb +0 -29
- data/lib/llm_cost_tracker/inbox_event.rb +0 -9
- data/lib/llm_cost_tracker/ingestor_lease.rb +0 -9
- data/lib/llm_cost_tracker/integrations/object_reader.rb +0 -56
- data/lib/llm_cost_tracker/integrations/registry.rb +0 -71
- data/lib/llm_cost_tracker/llm_api_call.rb +0 -60
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +0 -63
- data/lib/llm_cost_tracker/parameter_hash.rb +0 -33
- data/lib/llm_cost_tracker/parsed_usage.rb +0 -72
- data/lib/llm_cost_tracker/parsers/registry.rb +0 -58
- data/lib/llm_cost_tracker/period_grouping.rb +0 -67
- data/lib/llm_cost_tracker/period_total.rb +0 -9
- data/lib/llm_cost_tracker/price_freshness.rb +0 -38
- data/lib/llm_cost_tracker/price_registry.rb +0 -144
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +0 -104
- data/lib/llm_cost_tracker/price_sync/registry_diff.rb +0 -51
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +0 -61
- data/lib/llm_cost_tracker/price_sync/registry_writer.rb +0 -29
- data/lib/llm_cost_tracker/price_sync.rb +0 -144
- data/lib/llm_cost_tracker/report_data.rb +0 -94
- data/lib/llm_cost_tracker/report_formatter.rb +0 -67
- data/lib/llm_cost_tracker/request_url.rb +0 -20
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +0 -167
- data/lib/llm_cost_tracker/storage/active_record_connection_cleanup.rb +0 -13
- data/lib/llm_cost_tracker/storage/active_record_inbox.rb +0 -160
- data/lib/llm_cost_tracker/storage/active_record_period_totals.rb +0 -84
- data/lib/llm_cost_tracker/storage/active_record_rollup_batch.rb +0 -41
- data/lib/llm_cost_tracker/storage/active_record_rollup_upsert_sql.rb +0 -42
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +0 -146
- data/lib/llm_cost_tracker/storage/active_record_store.rb +0 -145
- data/lib/llm_cost_tracker/storage/writer.rb +0 -35
- data/lib/llm_cost_tracker/stream_capture.rb +0 -7
- data/lib/llm_cost_tracker/stream_collector.rb +0 -199
- data/lib/llm_cost_tracker/tag_accessors.rb +0 -15
- data/lib/llm_cost_tracker/tag_context.rb +0 -52
- data/lib/llm_cost_tracker/tag_key.rb +0 -16
- data/lib/llm_cost_tracker/tag_query.rb +0 -43
- data/lib/llm_cost_tracker/tag_sanitizer.rb +0 -81
- data/lib/llm_cost_tracker/tag_sql.rb +0 -34
- data/lib/llm_cost_tracker/tags_column.rb +0 -105
- data/lib/llm_cost_tracker/unknown_pricing.rb +0 -54
- data/lib/llm_cost_tracker/usage_breakdown.rb +0 -30
- data/lib/llm_cost_tracker/value_helpers.rb +0 -40
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
<% total = @stats.total_calls %>
|
|
2
|
-
<%
|
|
3
|
-
<%
|
|
4
|
-
<%
|
|
5
|
-
<%
|
|
6
|
-
<%
|
|
1
|
+
<% total = @stats.total_calls.to_i %>
|
|
2
|
+
<% unknown_pricing_count = @stats.unknown_pricing_count.to_i %>
|
|
3
|
+
<% untagged_calls_count = @stats.untagged_calls_count.to_i %>
|
|
4
|
+
<% missing_latency_count = @stats.missing_latency_count&.to_i %>
|
|
5
|
+
<% streaming_count = @stats.streaming_count&.to_i %>
|
|
6
|
+
<% streaming_missing_usage = @stats.streaming_missing_usage_count&.to_i %>
|
|
7
|
+
<% missing_provider_response_id_count = @stats.missing_provider_response_id_count&.to_i %>
|
|
8
|
+
<% calls_with_provider_response_id = total - missing_provider_response_id_count %>
|
|
7
9
|
|
|
8
10
|
<section class="lct-panel lct-toolbar">
|
|
9
11
|
<div class="lct-toolbar-head">
|
|
@@ -38,15 +40,13 @@
|
|
|
38
40
|
id: "lct-quality-model" %>
|
|
39
41
|
</div>
|
|
40
42
|
|
|
41
|
-
|
|
42
|
-
<
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
</div>
|
|
49
|
-
<% end %>
|
|
43
|
+
<div class="lct-field">
|
|
44
|
+
<label for="lct-quality-stream">Stream</label>
|
|
45
|
+
<%= select_tag :stream,
|
|
46
|
+
options_for_select(LlmCostTracker::DashboardFilterHelper::STREAM_FILTER_OPTIONS, params[:stream]),
|
|
47
|
+
include_blank: "All calls",
|
|
48
|
+
id: "lct-quality-stream" %>
|
|
49
|
+
</div>
|
|
50
50
|
|
|
51
51
|
<div class="lct-filter-actions">
|
|
52
52
|
<button class="lct-button" type="submit">Apply</button>
|
|
@@ -79,53 +79,47 @@
|
|
|
79
79
|
<div class="lct-stat-grid">
|
|
80
80
|
<article class="lct-stat">
|
|
81
81
|
<p class="lct-stat-label">Unknown pricing</p>
|
|
82
|
-
<p class="lct-stat-value"><%= number(
|
|
83
|
-
<p class="lct-stat-sub"><%= percent(coverage_percent(
|
|
82
|
+
<p class="lct-stat-value"><%= number(unknown_pricing_count) %></p>
|
|
83
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(unknown_pricing_count, total)) %> of calls</p>
|
|
84
84
|
</article>
|
|
85
85
|
|
|
86
86
|
<article class="lct-stat">
|
|
87
87
|
<p class="lct-stat-label">Calls without tags</p>
|
|
88
|
-
<p class="lct-stat-value"><%= number(
|
|
89
|
-
<p class="lct-stat-sub"><%= percent(coverage_percent(
|
|
88
|
+
<p class="lct-stat-value"><%= number(untagged_calls_count) %></p>
|
|
89
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(untagged_calls_count, total)) %> of calls</p>
|
|
90
90
|
</article>
|
|
91
91
|
|
|
92
|
-
|
|
93
|
-
<
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
</article>
|
|
98
|
-
<% end %>
|
|
99
|
-
|
|
100
|
-
<% if @stats.stream_column_present %>
|
|
101
|
-
<article class="lct-stat">
|
|
102
|
-
<p class="lct-stat-label">Streaming calls</p>
|
|
103
|
-
<p class="lct-stat-value"><%= number(streaming_count) %></p>
|
|
104
|
-
<p class="lct-stat-sub"><%= percent(coverage_percent(streaming_count, total)) %> of calls</p>
|
|
105
|
-
</article>
|
|
92
|
+
<article class="lct-stat">
|
|
93
|
+
<p class="lct-stat-label">Missing latency</p>
|
|
94
|
+
<p class="lct-stat-value"><%= number(missing_latency_count) %></p>
|
|
95
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(missing_latency_count, total)) %> of calls</p>
|
|
96
|
+
</article>
|
|
106
97
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
</article>
|
|
113
|
-
<% end %>
|
|
114
|
-
<% end %>
|
|
98
|
+
<article class="lct-stat">
|
|
99
|
+
<p class="lct-stat-label">Streaming calls</p>
|
|
100
|
+
<p class="lct-stat-value"><%= number(streaming_count) %></p>
|
|
101
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(streaming_count, total)) %> of calls</p>
|
|
102
|
+
</article>
|
|
115
103
|
|
|
116
|
-
<% if
|
|
104
|
+
<% if streaming_count.positive? %>
|
|
117
105
|
<article class="lct-stat">
|
|
118
|
-
<p class="lct-stat-label">
|
|
119
|
-
<p class="lct-stat-value"><%= number(
|
|
120
|
-
<p class="lct-stat-sub"><%= percent(coverage_percent(
|
|
106
|
+
<p class="lct-stat-label">Streams without usage</p>
|
|
107
|
+
<p class="lct-stat-value"><%= number(streaming_missing_usage) %></p>
|
|
108
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(streaming_missing_usage, streaming_count)) %> of streams</p>
|
|
121
109
|
</article>
|
|
122
110
|
<% end %>
|
|
123
111
|
|
|
124
|
-
|
|
112
|
+
<article class="lct-stat">
|
|
113
|
+
<p class="lct-stat-label">Calls with provider response ID</p>
|
|
114
|
+
<p class="lct-stat-value"><%= number(calls_with_provider_response_id) %></p>
|
|
115
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(calls_with_provider_response_id, total)) %> of calls</p>
|
|
116
|
+
</article>
|
|
117
|
+
|
|
118
|
+
<% if @hidden_output_summary %>
|
|
125
119
|
<article class="lct-stat">
|
|
126
120
|
<p class="lct-stat-label">Hidden output share</p>
|
|
127
|
-
<p class="lct-stat-value"><%= percent(
|
|
128
|
-
<p class="lct-stat-sub"><%= number(@
|
|
121
|
+
<p class="lct-stat-value"><%= percent(@hidden_output_summary.fetch(:share_percent)) %></p>
|
|
122
|
+
<p class="lct-stat-sub"><%= number(@hidden_output_summary.fetch(:hidden_output_tokens)) %> of <%= number(@hidden_output_summary.fetch(:output_tokens)) %> output tokens</p>
|
|
129
123
|
</article>
|
|
130
124
|
<% end %>
|
|
131
125
|
</div>
|
|
@@ -151,33 +145,31 @@
|
|
|
151
145
|
</tr>
|
|
152
146
|
</thead>
|
|
153
147
|
<tbody>
|
|
154
|
-
<% cost_coverage = coverage_percent(total -
|
|
148
|
+
<% cost_coverage = coverage_percent(total - unknown_pricing_count, total) %>
|
|
155
149
|
<tr>
|
|
156
150
|
<td>Cost (pricing known)</td>
|
|
157
151
|
<td class="lct-num"><%= percent(cost_coverage) %></td>
|
|
158
|
-
<td class="lct-num"><%= number(total -
|
|
152
|
+
<td class="lct-num"><%= number(total - unknown_pricing_count) %></td>
|
|
159
153
|
<td><%= render "llm_cost_tracker/shared/bar", value: cost_coverage, max: 100.0 %></td>
|
|
160
154
|
</tr>
|
|
161
155
|
|
|
162
|
-
<% tag_coverage = coverage_percent(total -
|
|
156
|
+
<% tag_coverage = coverage_percent(total - untagged_calls_count, total) %>
|
|
163
157
|
<tr>
|
|
164
158
|
<td>Tags (at least one tag)</td>
|
|
165
159
|
<td class="lct-num"><%= percent(tag_coverage) %></td>
|
|
166
|
-
<td class="lct-num"><%= number(total -
|
|
160
|
+
<td class="lct-num"><%= number(total - untagged_calls_count) %></td>
|
|
167
161
|
<td><%= render "llm_cost_tracker/shared/bar", value: tag_coverage, max: 100.0 %></td>
|
|
168
162
|
</tr>
|
|
169
163
|
|
|
170
|
-
<%
|
|
171
|
-
|
|
172
|
-
<
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
</tr>
|
|
178
|
-
<% end %>
|
|
164
|
+
<% latency_coverage = coverage_percent(total - missing_latency_count, total) %>
|
|
165
|
+
<tr>
|
|
166
|
+
<td>Latency</td>
|
|
167
|
+
<td class="lct-num"><%= percent(latency_coverage) %></td>
|
|
168
|
+
<td class="lct-num"><%= number(total - missing_latency_count) %></td>
|
|
169
|
+
<td><%= render "llm_cost_tracker/shared/bar", value: latency_coverage, max: 100.0 %></td>
|
|
170
|
+
</tr>
|
|
179
171
|
|
|
180
|
-
<% if
|
|
172
|
+
<% if streaming_count.to_i.positive? %>
|
|
181
173
|
<% stream_coverage = coverage_percent(streaming_count - streaming_missing_usage, streaming_count) %>
|
|
182
174
|
<tr>
|
|
183
175
|
<td>Streaming usage captured</td>
|
|
@@ -187,15 +179,13 @@
|
|
|
187
179
|
</tr>
|
|
188
180
|
<% end %>
|
|
189
181
|
|
|
190
|
-
<%
|
|
191
|
-
|
|
192
|
-
<
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
</tr>
|
|
198
|
-
<% end %>
|
|
182
|
+
<% provider_response_id_coverage = coverage_percent(calls_with_provider_response_id, total) %>
|
|
183
|
+
<tr>
|
|
184
|
+
<td>Provider response ID</td>
|
|
185
|
+
<td class="lct-num"><%= percent(provider_response_id_coverage) %></td>
|
|
186
|
+
<td class="lct-num"><%= number(calls_with_provider_response_id) %></td>
|
|
187
|
+
<td><%= render "llm_cost_tracker/shared/bar", value: provider_response_id_coverage, max: 100.0 %></td>
|
|
188
|
+
</tr>
|
|
199
189
|
</tbody>
|
|
200
190
|
</table>
|
|
201
191
|
</section>
|
|
@@ -227,21 +217,19 @@
|
|
|
227
217
|
<td>Attribution by tenant, user, or feature becomes less useful.</td>
|
|
228
218
|
<td>Pass <code class="lct-code">tags:</code> from middleware using request context.</td>
|
|
229
219
|
</tr>
|
|
230
|
-
|
|
231
|
-
<
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
<% end %>
|
|
237
|
-
<% if @stats.stream_column_present && streaming_missing_usage.to_i.positive? %>
|
|
220
|
+
<tr>
|
|
221
|
+
<td>Missing latency</td>
|
|
222
|
+
<td>Slow requests become harder to isolate on the calls page.</td>
|
|
223
|
+
<td>Make sure latency capture is enabled on every tracked request.</td>
|
|
224
|
+
</tr>
|
|
225
|
+
<% if streaming_missing_usage.to_i.positive? %>
|
|
238
226
|
<tr>
|
|
239
227
|
<td>Streams without usage</td>
|
|
240
228
|
<td>Token totals undercount when streaming responses drop the final usage event.</td>
|
|
241
229
|
<td>Send OpenAI requests with <code class="lct-code">stream_options: { include_usage: true }</code>, or wrap custom clients with <code class="lct-code">LlmCostTracker.track_stream</code>.</td>
|
|
242
230
|
</tr>
|
|
243
231
|
<% end %>
|
|
244
|
-
<% if
|
|
232
|
+
<% if missing_provider_response_id_count.to_i.positive? %>
|
|
245
233
|
<tr>
|
|
246
234
|
<td>Missing provider response IDs</td>
|
|
247
235
|
<td>Proof of provider-issued responses is weaker when calls cannot be tied back to provider objects.</td>
|
|
@@ -253,62 +241,47 @@
|
|
|
253
241
|
</section>
|
|
254
242
|
</section>
|
|
255
243
|
|
|
256
|
-
|
|
257
|
-
<
|
|
258
|
-
<div
|
|
259
|
-
<
|
|
260
|
-
<h2 class="lct-section-title">Usage breakdown</h2>
|
|
261
|
-
</div>
|
|
244
|
+
<section class="lct-panel">
|
|
245
|
+
<div class="lct-section-head">
|
|
246
|
+
<div>
|
|
247
|
+
<h2 class="lct-section-title">Token usage</h2>
|
|
262
248
|
</div>
|
|
249
|
+
</div>
|
|
263
250
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
<td class="lct-num"><%= money(@stats.input_cost) %></td>
|
|
280
|
-
</tr>
|
|
281
|
-
<tr>
|
|
282
|
-
<td>Cache read input</td>
|
|
283
|
-
<td class="lct-num"><%= number(@stats.cache_read_input_tokens) %></td>
|
|
284
|
-
<td class="lct-num"><%= percent(coverage_percent(@stats.cache_read_input_tokens, billable_tokens)) %></td>
|
|
285
|
-
<td class="lct-num<%= ' lct-num-muted' if @stats.cache_read_input_cost.nil? %>"><%= optional_money(@stats.cache_read_input_cost) %></td>
|
|
286
|
-
</tr>
|
|
287
|
-
<tr>
|
|
288
|
-
<td>Cache write input</td>
|
|
289
|
-
<td class="lct-num"><%= number(@stats.cache_write_input_tokens) %></td>
|
|
290
|
-
<td class="lct-num"><%= percent(coverage_percent(@stats.cache_write_input_tokens, billable_tokens)) %></td>
|
|
291
|
-
<td class="lct-num<%= ' lct-num-muted' if @stats.cache_write_input_cost.nil? %>"><%= optional_money(@stats.cache_write_input_cost) %></td>
|
|
292
|
-
</tr>
|
|
293
|
-
<tr>
|
|
294
|
-
<td>Output</td>
|
|
295
|
-
<td class="lct-num"><%= number(@stats.output_tokens) %></td>
|
|
296
|
-
<td class="lct-num"><%= percent(coverage_percent(@stats.output_tokens, billable_tokens)) %></td>
|
|
297
|
-
<td class="lct-num"><%= money(@stats.output_cost) %></td>
|
|
298
|
-
</tr>
|
|
251
|
+
<div class="lct-table-wrap">
|
|
252
|
+
<table class="lct-table lct-table-compact">
|
|
253
|
+
<thead>
|
|
254
|
+
<tr>
|
|
255
|
+
<th>Bucket</th>
|
|
256
|
+
<th class="lct-num">Tokens</th>
|
|
257
|
+
<th class="lct-num">Share</th>
|
|
258
|
+
<th class="lct-num">Cost</th>
|
|
259
|
+
</tr>
|
|
260
|
+
</thead>
|
|
261
|
+
<tbody>
|
|
262
|
+
<% @usage_rows.each do |row| %>
|
|
263
|
+
<% token_key = row.fetch(:token_key) %>
|
|
264
|
+
<% cost_key = row.fetch(:cost_key) %>
|
|
265
|
+
<% cost_value = row.fetch(:cost_value) %>
|
|
299
266
|
<tr>
|
|
300
|
-
<td
|
|
301
|
-
<td class="lct-num"><%= number(
|
|
302
|
-
|
|
303
|
-
|
|
267
|
+
<td><%= token_usage_quality_label(token_key) %></td>
|
|
268
|
+
<td class="lct-num"><%= number(row.fetch(:token_value)) %></td>
|
|
269
|
+
<% if row.fetch(:share_basis) == :output %>
|
|
270
|
+
<td class="lct-num"><%= percent(row.fetch(:share_percent)) %> of output</td>
|
|
271
|
+
<% else %>
|
|
272
|
+
<td class="lct-num"><%= percent(row.fetch(:share_percent)) %></td>
|
|
273
|
+
<% end %>
|
|
274
|
+
<td class="lct-num<%= ' lct-num-muted' if cost_key.nil? || cost_value.nil? %>">
|
|
275
|
+
<%= cost_key ? optional_money(cost_value) : "n/a" %>
|
|
276
|
+
</td>
|
|
304
277
|
</tr>
|
|
305
|
-
|
|
306
|
-
</
|
|
307
|
-
</
|
|
308
|
-
</
|
|
309
|
-
|
|
278
|
+
<% end %>
|
|
279
|
+
</tbody>
|
|
280
|
+
</table>
|
|
281
|
+
</div>
|
|
282
|
+
</section>
|
|
310
283
|
|
|
311
|
-
<% unless @
|
|
284
|
+
<% unless @unknown_pricing_by_model.empty? %>
|
|
312
285
|
<section class="lct-panel">
|
|
313
286
|
<div class="lct-section-head">
|
|
314
287
|
<div>
|
|
@@ -328,11 +301,11 @@
|
|
|
328
301
|
</tr>
|
|
329
302
|
</thead>
|
|
330
303
|
<tbody>
|
|
331
|
-
<% @
|
|
304
|
+
<% @unknown_pricing_by_model.each do |row| %>
|
|
332
305
|
<tr>
|
|
333
|
-
<td><code class="lct-code"><%= model %></code></td>
|
|
334
|
-
<td class="lct-num"><%= number(
|
|
335
|
-
<td class="lct-num"><%= percent(total.positive? ? (
|
|
306
|
+
<td><code class="lct-code"><%= row.model %></code></td>
|
|
307
|
+
<td class="lct-num"><%= number(row.calls) %></td>
|
|
308
|
+
<td class="lct-num"><%= percent(total.positive? ? (row.calls.to_f / total) * 100.0 : 0.0) %></td>
|
|
336
309
|
</tr>
|
|
337
310
|
<% end %>
|
|
338
311
|
</tbody>
|
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
options_for_select(
|
|
38
38
|
[["Total spend", "cost"],
|
|
39
39
|
["Call volume", "calls"],
|
|
40
|
-
["Avg cost / call", "avg_cost"]
|
|
41
|
-
|
|
40
|
+
["Avg cost / call", "avg_cost"],
|
|
41
|
+
["Avg latency", "latency"]],
|
|
42
42
|
@sort.presence || "cost"
|
|
43
43
|
),
|
|
44
44
|
id: "lct-models-sort" %>
|
|
@@ -71,13 +71,12 @@
|
|
|
71
71
|
<th>Provider</th>
|
|
72
72
|
<th>Model</th>
|
|
73
73
|
<th class="lct-num">Calls</th>
|
|
74
|
-
<th class="lct-num">
|
|
74
|
+
<th class="lct-num">Total tokens</th>
|
|
75
|
+
<th class="lct-num">Regular input</th>
|
|
75
76
|
<th class="lct-num">Output</th>
|
|
76
77
|
<th class="lct-num">Total cost</th>
|
|
77
78
|
<th class="lct-num">Avg cost / call</th>
|
|
78
|
-
|
|
79
|
-
<th class="lct-num">Avg latency</th>
|
|
80
|
-
<% end %>
|
|
79
|
+
<th class="lct-num">Avg latency</th>
|
|
81
80
|
<th></th>
|
|
82
81
|
</tr>
|
|
83
82
|
</thead>
|
|
@@ -87,13 +86,13 @@
|
|
|
87
86
|
<td><%= row.provider %></td>
|
|
88
87
|
<td><code class="lct-code"><%= row.model %></code></td>
|
|
89
88
|
<td class="lct-num"><%= number(row.calls) %></td>
|
|
89
|
+
<td class="lct-num"><%= format_tokens(row.total_tokens) %></td>
|
|
90
90
|
<td class="lct-num"><%= format_tokens(row.input_tokens) %></td>
|
|
91
91
|
<td class="lct-num"><%= format_tokens(row.output_tokens) %></td>
|
|
92
92
|
<td class="lct-num"><%= money(row.total_cost) %></td>
|
|
93
93
|
<td class="lct-num"><%= money(row.average_cost_per_call) %></td>
|
|
94
|
-
<%
|
|
95
|
-
|
|
96
|
-
<% end %>
|
|
94
|
+
<% average_latency_ms = row.average_latency_ms %>
|
|
95
|
+
<td class="lct-num<%= ' lct-num-muted' if average_latency_ms.nil? %>"><%= average_latency_ms ? "#{number(average_latency_ms.round)}ms" : "n/a" %></td>
|
|
97
96
|
<td><%= link_to "Calls", calls_path(calls_query_for_model(provider: row.provider, model: row.model)), class: "lct-button lct-button-secondary lct-button-compact" %></td>
|
|
98
97
|
</tr>
|
|
99
98
|
<% end %>
|
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
<section class="lct-panel lct-empty">
|
|
2
2
|
<h2 class="lct-state-title">Setup required</h2>
|
|
3
3
|
<p class="lct-state-copy">
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
<%= @setup_message || "The llm_api_calls table is not available yet." %>
|
|
5
|
+
<% if @setup_details.present? %>
|
|
6
|
+
Run <span class="lct-code">bin/rails llm_cost_tracker:doctor</span>, apply the listed migrations, and migrate your database.
|
|
7
|
+
<% else %>
|
|
8
|
+
Run <span class="lct-code">rails generate llm_cost_tracker:install</span> and migrate your database.
|
|
9
|
+
<% end %>
|
|
6
10
|
</p>
|
|
11
|
+
<% if @setup_details.present? %>
|
|
12
|
+
<ul class="lct-state-copy">
|
|
13
|
+
<% @setup_details.each do |detail| %>
|
|
14
|
+
<li><code class="lct-code"><%= detail %></code></li>
|
|
15
|
+
<% end %>
|
|
16
|
+
</ul>
|
|
17
|
+
<% end %>
|
|
7
18
|
</section>
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
<% share_base = @tagged_calls.positive? ? @tagged_calls.to_f : 1.0 %>
|
|
1
|
+
<% share_base = @breakdown.tagged_calls.positive? ? @breakdown.tagged_calls.to_f : 1.0 %>
|
|
2
2
|
|
|
3
3
|
<section class="lct-panel lct-toolbar">
|
|
4
4
|
<div class="lct-toolbar-head">
|
|
5
5
|
<div>
|
|
6
6
|
<p class="lct-muted"><%= link_to "← All tag keys", tags_path(current_query) %></p>
|
|
7
|
-
<h2 class="lct-section-title">Tag: <code class="lct-code"><%=
|
|
7
|
+
<h2 class="lct-section-title">Tag: <code class="lct-code"><%= params[:key] %></code></h2>
|
|
8
8
|
</div>
|
|
9
9
|
</div>
|
|
10
10
|
|
|
11
|
-
<form class="lct-filters" action="<%= tag_path(
|
|
11
|
+
<form class="lct-filters" action="<%= tag_path(params[:key]) %>" method="get">
|
|
12
12
|
<div class="lct-filter-row lct-filter-row-basic">
|
|
13
13
|
<div class="lct-field">
|
|
14
14
|
<label for="lct-tag-show-from">From</label>
|
|
@@ -38,49 +38,49 @@
|
|
|
38
38
|
|
|
39
39
|
<div class="lct-filter-actions">
|
|
40
40
|
<button class="lct-button" type="submit">Apply</button>
|
|
41
|
-
<%= link_to("Reset", tag_path(
|
|
41
|
+
<%= link_to("Reset", tag_path(params[:key]), class: "lct-button lct-button-secondary") if any_filter_applied? %>
|
|
42
42
|
</div>
|
|
43
43
|
</div>
|
|
44
44
|
</form>
|
|
45
45
|
|
|
46
|
-
<%= render "llm_cost_tracker/shared/active_filters", chips: active_tag_filters, clear_path: tag_path(
|
|
46
|
+
<%= render "llm_cost_tracker/shared/active_filters", chips: active_tag_filters, clear_path: tag_path(params[:key]) %>
|
|
47
47
|
|
|
48
48
|
<p class="lct-summary-row">
|
|
49
|
-
<span><strong><%= number(@tagged_calls) %></strong> tagged calls</span>
|
|
50
|
-
<span><strong><%= percent(coverage_percent(@tagged_calls, @total_calls)) %></strong> coverage</span>
|
|
51
|
-
<span><strong><%= number(@distinct_values) %></strong> distinct values</span>
|
|
49
|
+
<span><strong><%= number(@breakdown.tagged_calls) %></strong> tagged calls</span>
|
|
50
|
+
<span><strong><%= percent(coverage_percent(@breakdown.tagged_calls, @breakdown.total_calls)) %></strong> coverage</span>
|
|
51
|
+
<span><strong><%= number(@breakdown.distinct_values) %></strong> distinct values</span>
|
|
52
52
|
</p>
|
|
53
53
|
|
|
54
|
-
<% if @
|
|
55
|
-
<p class="lct-toolbar-note">Showing top <%= number(@
|
|
54
|
+
<% if @breakdown.distinct_values > @breakdown.rows.size %>
|
|
55
|
+
<p class="lct-toolbar-note">Showing top <%= number(@breakdown.limit) %> values by spend.</p>
|
|
56
56
|
<% end %>
|
|
57
57
|
</section>
|
|
58
58
|
|
|
59
|
-
<% if @rows.empty? %>
|
|
59
|
+
<% if @breakdown.rows.empty? %>
|
|
60
60
|
<section class="lct-panel lct-empty">
|
|
61
|
-
<h2 class="lct-state-title">No calls tagged with <%=
|
|
61
|
+
<h2 class="lct-state-title">No calls tagged with <%= params[:key] %></h2>
|
|
62
62
|
<p class="lct-state-copy">Values for this key will appear here once matching calls carry the tag in the current slice.</p>
|
|
63
63
|
<div class="lct-state-actions">
|
|
64
|
-
<%= link_to "Clear filters", tag_path(
|
|
64
|
+
<%= link_to "Clear filters", tag_path(params[:key]), class: "lct-button lct-button-secondary" %>
|
|
65
65
|
</div>
|
|
66
66
|
</section>
|
|
67
67
|
<% else %>
|
|
68
68
|
<section class="lct-stat-grid lct-stat-grid-spaced">
|
|
69
69
|
<article class="lct-stat">
|
|
70
70
|
<p class="lct-stat-label">Tagged calls</p>
|
|
71
|
-
<p class="lct-stat-value"><%= number(@tagged_calls) %></p>
|
|
72
|
-
<p class="lct-stat-copy">Rows that include <code class="lct-code"><%=
|
|
71
|
+
<p class="lct-stat-value"><%= number(@breakdown.tagged_calls) %></p>
|
|
72
|
+
<p class="lct-stat-copy">Rows that include <code class="lct-code"><%= params[:key] %></code></p>
|
|
73
73
|
</article>
|
|
74
74
|
|
|
75
75
|
<article class="lct-stat">
|
|
76
76
|
<p class="lct-stat-label">Coverage</p>
|
|
77
|
-
<p class="lct-stat-value"><%= percent(coverage_percent(@tagged_calls, @total_calls)) %></p>
|
|
78
|
-
<p class="lct-stat-copy"><%= number(@total_calls) %> total calls in this slice</p>
|
|
77
|
+
<p class="lct-stat-value"><%= percent(coverage_percent(@breakdown.tagged_calls, @breakdown.total_calls)) %></p>
|
|
78
|
+
<p class="lct-stat-copy"><%= number(@breakdown.total_calls) %> total calls in this slice</p>
|
|
79
79
|
</article>
|
|
80
80
|
|
|
81
81
|
<article class="lct-stat">
|
|
82
82
|
<p class="lct-stat-label">Distinct values</p>
|
|
83
|
-
<p class="lct-stat-value"><%= number(@distinct_values) %></p>
|
|
83
|
+
<p class="lct-stat-value"><%= number(@breakdown.distinct_values) %></p>
|
|
84
84
|
<p class="lct-stat-copy">Unique values currently visible</p>
|
|
85
85
|
</article>
|
|
86
86
|
</section>
|
|
@@ -99,7 +99,7 @@
|
|
|
99
99
|
</tr>
|
|
100
100
|
</thead>
|
|
101
101
|
<tbody>
|
|
102
|
-
<% @rows.each do |row| %>
|
|
102
|
+
<% @breakdown.rows.each do |row| %>
|
|
103
103
|
<tr>
|
|
104
104
|
<td><code class="lct-code"><%= row.value %></code></td>
|
|
105
105
|
<td class="lct-num"><%= number(row.calls) %></td>
|
|
@@ -110,7 +110,7 @@
|
|
|
110
110
|
<% if row.value == "(untagged)" %>
|
|
111
111
|
<span class="lct-muted">n/a</span>
|
|
112
112
|
<% else %>
|
|
113
|
-
<%= link_to "Calls", calls_path(calls_query_for_tag(key:
|
|
113
|
+
<%= link_to "Calls", calls_path(calls_query_for_tag(key: params[:key], value: row.value)), class: "lct-button lct-button-secondary lct-button-compact" %>
|
|
114
114
|
<% end %>
|
|
115
115
|
</td>
|
|
116
116
|
</tr>
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "logging"
|
|
4
|
+
require_relative "ledger"
|
|
4
5
|
|
|
5
6
|
module LlmCostTracker
|
|
6
7
|
class Budget
|
|
@@ -12,7 +13,7 @@ module LlmCostTracker
|
|
|
12
13
|
budgets = enforce_period_budgets(config)
|
|
13
14
|
return if budgets.empty?
|
|
14
15
|
|
|
15
|
-
totals =
|
|
16
|
+
totals = LlmCostTracker::Ledger::Period::Totals.call(budgets.keys, time: Time.now.utc)
|
|
16
17
|
|
|
17
18
|
budgets.each do |period, budget|
|
|
18
19
|
total = totals.fetch(period)
|
|
@@ -23,7 +24,7 @@ module LlmCostTracker
|
|
|
23
24
|
|
|
24
25
|
def check!(event)
|
|
25
26
|
config = LlmCostTracker.configuration
|
|
26
|
-
return unless event.
|
|
27
|
+
return unless event.total_cost
|
|
27
28
|
|
|
28
29
|
check_per_call_budget(event, config)
|
|
29
30
|
budgets = check_period_budgets(config)
|
|
@@ -42,7 +43,7 @@ module LlmCostTracker
|
|
|
42
43
|
budget = config.per_call_budget
|
|
43
44
|
return unless budget
|
|
44
45
|
|
|
45
|
-
call_cost = event.
|
|
46
|
+
call_cost = event.total_cost
|
|
46
47
|
return unless call_cost >= budget
|
|
47
48
|
|
|
48
49
|
handle_exceeded(budget_type: :per_call, total: call_cost, budget: budget, last_event: event)
|
|
@@ -65,16 +66,7 @@ module LlmCostTracker
|
|
|
65
66
|
def totals_for_check(event, budgets)
|
|
66
67
|
return {} if budgets.empty?
|
|
67
68
|
|
|
68
|
-
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
def active_record_totals(periods, time:)
|
|
72
|
-
require_relative "llm_api_call" unless defined?(LlmCostTracker::LlmApiCall)
|
|
73
|
-
require_relative "storage/active_record_store" unless defined?(LlmCostTracker::Storage::ActiveRecordStore)
|
|
74
|
-
|
|
75
|
-
LlmCostTracker::Storage::ActiveRecordStore.period_totals(periods, time: time)
|
|
76
|
-
rescue LoadError => e
|
|
77
|
-
raise Error, "ActiveRecord storage requires the active_record gem: #{e.message}"
|
|
69
|
+
LlmCostTracker::Ledger::Period::Totals.call(budgets.keys, time: event.tracked_at)
|
|
78
70
|
end
|
|
79
71
|
|
|
80
72
|
def handle_exceeded(budget_type:, total:, budget:, last_event: nil)
|
|
@@ -89,7 +81,7 @@ module LlmCostTracker
|
|
|
89
81
|
if notify_exceeded?(config, budget_type: budget_type, total: total, budget: budget, last_event: last_event)
|
|
90
82
|
config.on_budget_exceeded&.call(payload)
|
|
91
83
|
end
|
|
92
|
-
raise BudgetExceededError.new(**payload) if
|
|
84
|
+
raise BudgetExceededError.new(**payload) if %i[raise block_requests].include?(config.budget_exceeded_behavior)
|
|
93
85
|
end
|
|
94
86
|
|
|
95
87
|
def budget_payload(budget_type:, total:, budget:, last_event:)
|
|
@@ -108,14 +100,10 @@ module LlmCostTracker
|
|
|
108
100
|
def notify_exceeded?(config, budget_type:, total:, budget:, last_event:)
|
|
109
101
|
return false unless config.on_budget_exceeded
|
|
110
102
|
return true unless config.budget_exceeded_behavior == :notify
|
|
111
|
-
return true unless last_event&.
|
|
103
|
+
return true unless last_event&.total_cost
|
|
112
104
|
return true if budget_type == :per_call
|
|
113
105
|
|
|
114
|
-
total - last_event.
|
|
115
|
-
end
|
|
116
|
-
|
|
117
|
-
def raise_on_exceeded?(config)
|
|
118
|
-
%i[raise block_requests].include?(config.budget_exceeded_behavior)
|
|
106
|
+
total - last_event.total_cost < budget
|
|
119
107
|
end
|
|
120
108
|
end
|
|
121
109
|
end
|