llm_cost_tracker 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/README.md +124 -68
- data/Rakefile +2 -0
- data/app/assets/llm_cost_tracker/application.css +1 -4
- data/app/controllers/llm_cost_tracker/assets_controller.rb +1 -2
- data/app/controllers/llm_cost_tracker/calls_controller.rb +9 -13
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +8 -19
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +1 -2
- data/app/controllers/llm_cost_tracker/models_controller.rb +5 -2
- data/app/controllers/llm_cost_tracker/tags_controller.rb +2 -4
- data/app/helpers/llm_cost_tracker/dashboard_filter_helper.rb +6 -1
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +1 -7
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +5 -9
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +16 -1
- data/app/services/llm_cost_tracker/dashboard/filter.rb +26 -24
- data/app/services/llm_cost_tracker/dashboard/provider_breakdown.rb +0 -3
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +0 -2
- data/app/services/llm_cost_tracker/pagination.rb +1 -9
- data/app/views/layouts/llm_cost_tracker/application.html.erb +1 -16
- data/app/views/llm_cost_tracker/calls/index.html.erb +23 -13
- data/app/views/llm_cost_tracker/calls/show.html.erb +8 -3
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +11 -1
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +78 -10
- data/app/views/llm_cost_tracker/models/index.html.erb +10 -9
- data/app/views/llm_cost_tracker/shared/_spend_chart.html.erb +0 -1
- data/app/views/llm_cost_tracker/shared/_tag_chips.html.erb +0 -1
- data/app/views/llm_cost_tracker/tags/index.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +1 -1
- data/lib/llm_cost_tracker/assets.rb +6 -11
- data/lib/llm_cost_tracker/configuration.rb +78 -43
- data/lib/llm_cost_tracker/event.rb +3 -0
- data/lib/llm_cost_tracker/event_metadata.rb +1 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_provider_response_id_generator.rb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/add_streaming_generator.rb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_provider_response_id_to_llm_api_calls.rb.erb +15 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/add_streaming_to_llm_api_calls.rb.erb +25 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_api_calls.rb.erb +6 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/llm_cost_tracker_prices.yml.erb +8 -1
- data/lib/llm_cost_tracker/llm_api_call.rb +14 -2
- data/lib/llm_cost_tracker/middleware/faraday.rb +58 -9
- data/lib/llm_cost_tracker/parameter_hash.rb +33 -0
- data/lib/llm_cost_tracker/parsed_usage.rb +18 -3
- data/lib/llm_cost_tracker/parsers/anthropic.rb +98 -1
- data/lib/llm_cost_tracker/parsers/base.rb +17 -5
- data/lib/llm_cost_tracker/parsers/gemini.rb +83 -6
- data/lib/llm_cost_tracker/parsers/openai.rb +8 -0
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +12 -5
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +69 -1
- data/lib/llm_cost_tracker/parsers/registry.rb +15 -3
- data/lib/llm_cost_tracker/parsers/sse.rb +81 -0
- data/lib/llm_cost_tracker/price_registry.rb +23 -8
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +72 -0
- data/lib/llm_cost_tracker/price_sync/merger.rb +72 -0
- data/lib/llm_cost_tracker/price_sync/model_catalog.rb +77 -0
- data/lib/llm_cost_tracker/price_sync/raw_price.rb +35 -0
- data/lib/llm_cost_tracker/price_sync/refresh_plan_builder.rb +162 -0
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +55 -0
- data/lib/llm_cost_tracker/price_sync/registry_writer.rb +25 -0
- data/lib/llm_cost_tracker/price_sync/source.rb +29 -0
- data/lib/llm_cost_tracker/price_sync/source_result.rb +7 -0
- data/lib/llm_cost_tracker/price_sync/sources/litellm.rb +91 -0
- data/lib/llm_cost_tracker/price_sync/sources/open_router.rb +94 -0
- data/lib/llm_cost_tracker/price_sync/validator.rb +66 -0
- data/lib/llm_cost_tracker/price_sync.rb +142 -0
- data/lib/llm_cost_tracker/pricing.rb +0 -11
- data/lib/llm_cost_tracker/railtie.rb +0 -1
- data/lib/llm_cost_tracker/report.rb +0 -5
- data/lib/llm_cost_tracker/storage/active_record_store.rb +10 -9
- data/lib/llm_cost_tracker/stream_collector.rb +162 -0
- data/lib/llm_cost_tracker/tags_column.rb +12 -0
- data/lib/llm_cost_tracker/tracker.rb +23 -12
- data/lib/llm_cost_tracker/value_helpers.rb +40 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +48 -35
- data/lib/tasks/llm_cost_tracker.rake +116 -0
- data/llm_cost_tracker.gemspec +8 -6
- metadata +30 -8
|
@@ -8,15 +8,21 @@ module LlmCostTracker
|
|
|
8
8
|
:untagged_calls_count,
|
|
9
9
|
:missing_latency_count,
|
|
10
10
|
:latency_column_present,
|
|
11
|
+
:streaming_count,
|
|
12
|
+
:streaming_missing_usage_count,
|
|
13
|
+
:stream_column_present,
|
|
14
|
+
:missing_provider_response_id_count,
|
|
15
|
+
:provider_response_id_column_present,
|
|
11
16
|
:unknown_pricing_by_model
|
|
12
17
|
)
|
|
13
18
|
|
|
14
|
-
# Computes data quality metrics: coverage of cost, tags, and latency.
|
|
15
19
|
class DataQuality
|
|
16
20
|
class << self
|
|
17
21
|
def call(scope: LlmCostTracker::LlmApiCall.all)
|
|
18
22
|
total = scope.count
|
|
19
23
|
latency_present = LlmCostTracker::LlmApiCall.latency_column?
|
|
24
|
+
stream_present = LlmCostTracker::LlmApiCall.stream_column?
|
|
25
|
+
provider_response_id_present = LlmCostTracker::LlmApiCall.provider_response_id_column?
|
|
20
26
|
|
|
21
27
|
DataQualityStats.new(
|
|
22
28
|
total_calls: total,
|
|
@@ -24,6 +30,15 @@ module LlmCostTracker
|
|
|
24
30
|
untagged_calls_count: total - scope.with_json_tags.count,
|
|
25
31
|
missing_latency_count: latency_present ? scope.where(latency_ms: nil).count : nil,
|
|
26
32
|
latency_column_present: latency_present,
|
|
33
|
+
streaming_count: stream_present ? scope.streaming.count : nil,
|
|
34
|
+
streaming_missing_usage_count: if stream_present && LlmCostTracker::LlmApiCall.usage_source_column?
|
|
35
|
+
scope.streaming_missing_usage.count
|
|
36
|
+
end,
|
|
37
|
+
stream_column_present: stream_present,
|
|
38
|
+
missing_provider_response_id_count: (
|
|
39
|
+
provider_response_id_present ? scope.missing_provider_response_id.count : nil
|
|
40
|
+
),
|
|
41
|
+
provider_response_id_column_present: provider_response_id_present,
|
|
27
42
|
unknown_pricing_by_model: scope.unknown_pricing
|
|
28
43
|
.group(:model)
|
|
29
44
|
.order(Arel.sql("COUNT(*) DESC"))
|
|
@@ -4,10 +4,6 @@ require "date"
|
|
|
4
4
|
|
|
5
5
|
module LlmCostTracker
|
|
6
6
|
module Dashboard
|
|
7
|
-
# Parses dashboard params into an ActiveRecord relation.
|
|
8
|
-
#
|
|
9
|
-
# Invalid dates are ignored, pagination is handled elsewhere, and invalid
|
|
10
|
-
# tag keys raise InvalidFilterError so controllers can fail closed with HTTP 400.
|
|
11
7
|
class Filter
|
|
12
8
|
class << self
|
|
13
9
|
def call(scope: LlmCostTracker::LlmApiCall.all, params: {})
|
|
@@ -17,7 +13,7 @@ module LlmCostTracker
|
|
|
17
13
|
|
|
18
14
|
def initialize(scope:, params:)
|
|
19
15
|
@scope = scope
|
|
20
|
-
@params =
|
|
16
|
+
@params = LlmCostTracker::ParameterHash.with_indifferent_access(params)
|
|
21
17
|
end
|
|
22
18
|
|
|
23
19
|
def relation
|
|
@@ -25,6 +21,8 @@ module LlmCostTracker
|
|
|
25
21
|
filtered_scope = apply_date_filters(filtered_scope)
|
|
26
22
|
filtered_scope = apply_exact_filter(filtered_scope, :provider)
|
|
27
23
|
filtered_scope = apply_exact_filter(filtered_scope, :model)
|
|
24
|
+
filtered_scope = apply_stream_filter(filtered_scope)
|
|
25
|
+
filtered_scope = apply_usage_source_filter(filtered_scope)
|
|
28
26
|
apply_tag_filters(filtered_scope)
|
|
29
27
|
end
|
|
30
28
|
|
|
@@ -32,15 +30,6 @@ module LlmCostTracker
|
|
|
32
30
|
|
|
33
31
|
attr_reader :scope, :params
|
|
34
32
|
|
|
35
|
-
def normalize_params(params)
|
|
36
|
-
return {}.with_indifferent_access if params.nil?
|
|
37
|
-
|
|
38
|
-
raw = params.respond_to?(:to_unsafe_h) ? params.to_unsafe_h : params.to_h
|
|
39
|
-
raw.with_indifferent_access
|
|
40
|
-
rescue NoMethodError
|
|
41
|
-
{}.with_indifferent_access
|
|
42
|
-
end
|
|
43
|
-
|
|
44
33
|
def apply_date_filters(relation)
|
|
45
34
|
from = parse_date(:from)&.beginning_of_day
|
|
46
35
|
to = parse_date(:to)&.end_of_day
|
|
@@ -51,7 +40,7 @@ module LlmCostTracker
|
|
|
51
40
|
end
|
|
52
41
|
|
|
53
42
|
def apply_exact_filter(relation, key)
|
|
54
|
-
value =
|
|
43
|
+
value = normalized_string(params[key])
|
|
55
44
|
return relation if value.nil?
|
|
56
45
|
|
|
57
46
|
relation.where(key => value)
|
|
@@ -64,6 +53,26 @@ module LlmCostTracker
|
|
|
64
53
|
relation.by_tags(tags)
|
|
65
54
|
end
|
|
66
55
|
|
|
56
|
+
def apply_stream_filter(relation)
|
|
57
|
+
value = normalized_string(params[:stream])
|
|
58
|
+
return relation if value.nil?
|
|
59
|
+
return relation unless relation.klass.stream_column?
|
|
60
|
+
|
|
61
|
+
case value.downcase
|
|
62
|
+
when "yes", "true", "1" then relation.where(stream: true)
|
|
63
|
+
when "no", "false", "0" then relation.where(stream: [false, nil])
|
|
64
|
+
else relation
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def apply_usage_source_filter(relation)
|
|
69
|
+
value = normalized_string(params[:usage_source])
|
|
70
|
+
return relation if value.nil?
|
|
71
|
+
return relation unless relation.klass.usage_source_column?
|
|
72
|
+
|
|
73
|
+
relation.where(usage_source: value)
|
|
74
|
+
end
|
|
75
|
+
|
|
67
76
|
def tag_params
|
|
68
77
|
tags = hash_param(:tag)
|
|
69
78
|
|
|
@@ -76,14 +85,11 @@ module LlmCostTracker
|
|
|
76
85
|
end
|
|
77
86
|
|
|
78
87
|
def hash_param(key)
|
|
79
|
-
|
|
80
|
-
raw = raw.to_unsafe_h if raw.respond_to?(:to_unsafe_h)
|
|
81
|
-
raw = raw.to_h if raw.respond_to?(:to_h)
|
|
82
|
-
raw.is_a?(Hash) ? raw : {}
|
|
88
|
+
LlmCostTracker::ParameterHash.to_hash(params[key])
|
|
83
89
|
end
|
|
84
90
|
|
|
85
91
|
def parse_date(key)
|
|
86
|
-
value =
|
|
92
|
+
value = normalized_string(params[key])
|
|
87
93
|
return nil if value.nil?
|
|
88
94
|
|
|
89
95
|
Date.iso8601(value)
|
|
@@ -91,10 +97,6 @@ module LlmCostTracker
|
|
|
91
97
|
nil
|
|
92
98
|
end
|
|
93
99
|
|
|
94
|
-
def string_param(key)
|
|
95
|
-
normalized_string(params[key])
|
|
96
|
-
end
|
|
97
|
-
|
|
98
100
|
def normalized_string(value)
|
|
99
101
|
return nil if value.nil?
|
|
100
102
|
|
|
@@ -4,9 +4,6 @@ module LlmCostTracker
|
|
|
4
4
|
module Dashboard
|
|
5
5
|
ProviderRow = Data.define(:provider, :calls, :total_cost, :share_percent)
|
|
6
6
|
|
|
7
|
-
# Aggregates cost and call counts per provider for a given scope.
|
|
8
|
-
# Sorted by total cost descending; providers with zero cost fall to the bottom
|
|
9
|
-
# but are still returned so users can see calls without pricing.
|
|
10
7
|
class ProviderBreakdown
|
|
11
8
|
def self.call(scope: LlmCostTracker::LlmApiCall.all)
|
|
12
9
|
new(scope: scope).rows
|
|
@@ -9,8 +9,6 @@ module LlmCostTracker
|
|
|
9
9
|
:average_cost_per_call
|
|
10
10
|
)
|
|
11
11
|
|
|
12
|
-
# Aggregates calls grouped by the distinct values of a single tag key.
|
|
13
|
-
# Invalid keys raise InvalidFilterError so controllers can return HTTP 400.
|
|
14
12
|
class TagBreakdown
|
|
15
13
|
class << self
|
|
16
14
|
def call(key:, scope: LlmCostTracker::LlmApiCall.all)
|
|
@@ -9,21 +9,13 @@ module LlmCostTracker
|
|
|
9
9
|
attr_reader :page, :per
|
|
10
10
|
|
|
11
11
|
def self.call(params)
|
|
12
|
-
params =
|
|
12
|
+
params = LlmCostTracker::ParameterHash.with_indifferent_access(params)
|
|
13
13
|
new(
|
|
14
14
|
page: integer_param(params, :page, default: MIN_PAGE, min: MIN_PAGE),
|
|
15
15
|
per: integer_param(params, :per, default: DEFAULT_PER, min: 1, max: MAX_PER)
|
|
16
16
|
)
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
def self.normalize_params(params)
|
|
20
|
-
return {}.with_indifferent_access if params.nil?
|
|
21
|
-
|
|
22
|
-
raw = params.respond_to?(:to_unsafe_h) ? params.to_unsafe_h : params.to_h
|
|
23
|
-
raw.with_indifferent_access
|
|
24
|
-
end
|
|
25
|
-
private_class_method :normalize_params
|
|
26
|
-
|
|
27
19
|
def self.integer_param(params, key, default:, min:, max: nil)
|
|
28
20
|
value = Integer(params[key], 10)
|
|
29
21
|
value = [value, min].max
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
<title>LLM Cost Tracker</title>
|
|
7
7
|
<%= stylesheet_link_tag stylesheet_path %>
|
|
8
8
|
</head>
|
|
9
|
-
<body>
|
|
9
|
+
<body class="lct-body">
|
|
10
10
|
<div class="lct-app">
|
|
11
11
|
<main class="lct-shell">
|
|
12
12
|
<header class="lct-header">
|
|
@@ -25,20 +25,5 @@
|
|
|
25
25
|
<%= yield %>
|
|
26
26
|
</main>
|
|
27
27
|
</div>
|
|
28
|
-
<script>
|
|
29
|
-
document.addEventListener("keydown", function(event) {
|
|
30
|
-
if (event.key !== "/" || event.defaultPrevented || event.metaKey || event.ctrlKey || event.altKey) return;
|
|
31
|
-
|
|
32
|
-
var target = event.target;
|
|
33
|
-
if (target && (target.isContentEditable || /^(INPUT|TEXTAREA|SELECT)$/.test(target.tagName))) return;
|
|
34
|
-
|
|
35
|
-
var input = document.querySelector("[data-lct-filter-input]");
|
|
36
|
-
if (!input) return;
|
|
37
|
-
|
|
38
|
-
event.preventDefault();
|
|
39
|
-
input.focus();
|
|
40
|
-
if (typeof input.select === "function") input.select();
|
|
41
|
-
});
|
|
42
|
-
</script>
|
|
43
28
|
</body>
|
|
44
29
|
</html>
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
<div class="lct-filter-row lct-filter-row-with-sort">
|
|
11
11
|
<div class="lct-field">
|
|
12
12
|
<label for="lct-from">From</label>
|
|
13
|
-
<input id="lct-from"
|
|
13
|
+
<input id="lct-from" type="date" name="from" value="<%= params[:from] %>">
|
|
14
14
|
</div>
|
|
15
15
|
|
|
16
16
|
<div class="lct-field">
|
|
@@ -34,18 +34,29 @@
|
|
|
34
34
|
id: "lct-model" %>
|
|
35
35
|
</div>
|
|
36
36
|
|
|
37
|
+
<% if LlmCostTracker::LlmApiCall.stream_column? %>
|
|
38
|
+
<div class="lct-field">
|
|
39
|
+
<label for="lct-stream">Stream</label>
|
|
40
|
+
<%= select_tag :stream,
|
|
41
|
+
options_for_select(LlmCostTracker::DashboardFilterHelper::STREAM_FILTER_OPTIONS, params[:stream]),
|
|
42
|
+
include_blank: "All calls",
|
|
43
|
+
id: "lct-stream" %>
|
|
44
|
+
</div>
|
|
45
|
+
<% end %>
|
|
46
|
+
|
|
37
47
|
<div class="lct-field">
|
|
38
48
|
<label for="lct-sort">Sort</label>
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
+
<%= select_tag :sort,
|
|
50
|
+
options_for_select(
|
|
51
|
+
[["Recent first", ""],
|
|
52
|
+
["Most expensive", "expensive"],
|
|
53
|
+
["Largest input", "input"],
|
|
54
|
+
["Largest output", "output"]] +
|
|
55
|
+
(@latency_available ? [["Slowest", "slow"]] : []) +
|
|
56
|
+
[["Unknown pricing only", "unknown_pricing"]],
|
|
57
|
+
@sort
|
|
58
|
+
),
|
|
59
|
+
id: "lct-sort" %>
|
|
49
60
|
</div>
|
|
50
61
|
|
|
51
62
|
<div class="lct-filter-actions">
|
|
@@ -102,7 +113,6 @@
|
|
|
102
113
|
</thead>
|
|
103
114
|
<tbody>
|
|
104
115
|
<% @calls.each do |call| %>
|
|
105
|
-
<% tags = call.parsed_tags %>
|
|
106
116
|
<tr>
|
|
107
117
|
<td class="lct-nowrap"><%= format_date(call.tracked_at) %></td>
|
|
108
118
|
<td><%= call.provider %></td>
|
|
@@ -114,7 +124,7 @@
|
|
|
114
124
|
<% if @latency_available %>
|
|
115
125
|
<td class="lct-num<%= ' lct-num-muted' if call.latency_ms.nil? %>"><%= call.latency_ms ? "#{number(call.latency_ms)}ms" : "n/a" %></td>
|
|
116
126
|
<% end %>
|
|
117
|
-
<td><%= render "llm_cost_tracker/shared/tag_chips", tags:
|
|
127
|
+
<td><%= render "llm_cost_tracker/shared/tag_chips", tags: call.parsed_tags %></td>
|
|
118
128
|
<td><%= link_to "Details", call_path(call), class: "lct-button lct-button-secondary lct-button-compact" %></td>
|
|
119
129
|
</tr>
|
|
120
130
|
<% end %>
|
|
@@ -73,6 +73,11 @@
|
|
|
73
73
|
<dt>Pricing Status</dt>
|
|
74
74
|
<dd><%= pricing_status(@call) %></dd>
|
|
75
75
|
|
|
76
|
+
<% if LlmCostTracker::LlmApiCall.provider_response_id_column? %>
|
|
77
|
+
<dt>Provider Response ID</dt>
|
|
78
|
+
<dd><%= @call.provider_response_id.presence || "n/a" %></dd>
|
|
79
|
+
<% end %>
|
|
80
|
+
|
|
76
81
|
<% if @call.has_attribute?("created_at") %>
|
|
77
82
|
<dt>Created At</dt>
|
|
78
83
|
<dd><%= format_date(@call.created_at) %></dd>
|
|
@@ -113,12 +118,12 @@
|
|
|
113
118
|
|
|
114
119
|
<section class="lct-panel">
|
|
115
120
|
<h2 class="lct-section-title">Tags</h2>
|
|
116
|
-
<pre class="lct-pre"><%= safe_json(@
|
|
121
|
+
<pre class="lct-pre"><%= safe_json(@call.parsed_tags) %></pre>
|
|
117
122
|
</section>
|
|
118
123
|
|
|
119
|
-
<% if @
|
|
124
|
+
<% if @call.has_attribute?("metadata") %>
|
|
120
125
|
<section class="lct-panel">
|
|
121
126
|
<h2 class="lct-section-title">Metadata</h2>
|
|
122
|
-
<pre class="lct-pre"><%= safe_json(@metadata) %></pre>
|
|
127
|
+
<pre class="lct-pre"><%= safe_json(@call.read_attribute("metadata")) %></pre>
|
|
123
128
|
</section>
|
|
124
129
|
<% end %>
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
<div class="lct-filter-row lct-filter-row-basic">
|
|
6
6
|
<div class="lct-field">
|
|
7
7
|
<label for="lct-overview-from">From</label>
|
|
8
|
-
<input id="lct-overview-from"
|
|
8
|
+
<input id="lct-overview-from" type="date" name="from" value="<%= params[:from] || @from_date.iso8601 %>">
|
|
9
9
|
</div>
|
|
10
10
|
|
|
11
11
|
<div class="lct-field">
|
|
@@ -29,6 +29,16 @@
|
|
|
29
29
|
id: "lct-overview-model" %>
|
|
30
30
|
</div>
|
|
31
31
|
|
|
32
|
+
<% if LlmCostTracker::LlmApiCall.stream_column? %>
|
|
33
|
+
<div class="lct-field">
|
|
34
|
+
<label for="lct-overview-stream">Stream</label>
|
|
35
|
+
<%= select_tag :stream,
|
|
36
|
+
options_for_select(LlmCostTracker::DashboardFilterHelper::STREAM_FILTER_OPTIONS, params[:stream]),
|
|
37
|
+
include_blank: "All calls",
|
|
38
|
+
id: "lct-overview-stream" %>
|
|
39
|
+
</div>
|
|
40
|
+
<% end %>
|
|
41
|
+
|
|
32
42
|
<div class="lct-filter-actions">
|
|
33
43
|
<button class="lct-button" type="submit">Apply</button>
|
|
34
44
|
<%= link_to("Reset", root_path, class: "lct-button lct-button-secondary") if any_filter_applied? %>
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<% total = @stats.total_calls %>
|
|
2
|
-
<%
|
|
3
|
-
<%
|
|
4
|
-
<%
|
|
2
|
+
<% streaming_count = @stats.streaming_count %>
|
|
3
|
+
<% streaming_missing_usage = @stats.streaming_missing_usage_count %>
|
|
4
|
+
<% calls_with_provider_response_id = @stats.provider_response_id_column_present ? total - @stats.missing_provider_response_id_count : nil %>
|
|
5
5
|
|
|
6
6
|
<section class="lct-panel lct-toolbar">
|
|
7
7
|
<div class="lct-toolbar-head">
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
<div class="lct-filter-row lct-filter-row-basic">
|
|
13
13
|
<div class="lct-field">
|
|
14
14
|
<label for="lct-quality-from">From</label>
|
|
15
|
-
<input id="lct-quality-from"
|
|
15
|
+
<input id="lct-quality-from" type="date" name="from" value="<%= params[:from] %>">
|
|
16
16
|
</div>
|
|
17
17
|
|
|
18
18
|
<div class="lct-field">
|
|
@@ -36,6 +36,16 @@
|
|
|
36
36
|
id: "lct-quality-model" %>
|
|
37
37
|
</div>
|
|
38
38
|
|
|
39
|
+
<% if LlmCostTracker::LlmApiCall.stream_column? %>
|
|
40
|
+
<div class="lct-field">
|
|
41
|
+
<label for="lct-quality-stream">Stream</label>
|
|
42
|
+
<%= select_tag :stream,
|
|
43
|
+
options_for_select(LlmCostTracker::DashboardFilterHelper::STREAM_FILTER_OPTIONS, params[:stream]),
|
|
44
|
+
include_blank: "All calls",
|
|
45
|
+
id: "lct-quality-stream" %>
|
|
46
|
+
</div>
|
|
47
|
+
<% end %>
|
|
48
|
+
|
|
39
49
|
<div class="lct-filter-actions">
|
|
40
50
|
<button class="lct-button" type="submit">Apply</button>
|
|
41
51
|
<%= link_to("Reset", data_quality_path, class: "lct-button lct-button-secondary") if any_filter_applied? %>
|
|
@@ -84,6 +94,30 @@
|
|
|
84
94
|
<p class="lct-stat-sub"><%= percent(coverage_percent(@stats.missing_latency_count, total)) %> of calls</p>
|
|
85
95
|
</article>
|
|
86
96
|
<% end %>
|
|
97
|
+
|
|
98
|
+
<% if @stats.stream_column_present %>
|
|
99
|
+
<article class="lct-stat">
|
|
100
|
+
<p class="lct-stat-label">Streaming calls</p>
|
|
101
|
+
<p class="lct-stat-value"><%= number(streaming_count) %></p>
|
|
102
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(streaming_count, total)) %> of calls</p>
|
|
103
|
+
</article>
|
|
104
|
+
|
|
105
|
+
<% if streaming_missing_usage && streaming_count.positive? %>
|
|
106
|
+
<article class="lct-stat">
|
|
107
|
+
<p class="lct-stat-label">Streams without usage</p>
|
|
108
|
+
<p class="lct-stat-value"><%= number(streaming_missing_usage) %></p>
|
|
109
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(streaming_missing_usage, streaming_count)) %> of streams</p>
|
|
110
|
+
</article>
|
|
111
|
+
<% end %>
|
|
112
|
+
<% end %>
|
|
113
|
+
|
|
114
|
+
<% if @stats.provider_response_id_column_present %>
|
|
115
|
+
<article class="lct-stat">
|
|
116
|
+
<p class="lct-stat-label">Calls with provider response ID</p>
|
|
117
|
+
<p class="lct-stat-value"><%= number(calls_with_provider_response_id) %></p>
|
|
118
|
+
<p class="lct-stat-sub"><%= percent(coverage_percent(calls_with_provider_response_id, total)) %> of calls</p>
|
|
119
|
+
</article>
|
|
120
|
+
<% end %>
|
|
87
121
|
</div>
|
|
88
122
|
</div>
|
|
89
123
|
</section>
|
|
@@ -107,31 +141,51 @@
|
|
|
107
141
|
</tr>
|
|
108
142
|
</thead>
|
|
109
143
|
<tbody>
|
|
110
|
-
<% cost_coverage = coverage_percent(
|
|
144
|
+
<% cost_coverage = coverage_percent(total - @stats.unknown_pricing_count, total) %>
|
|
111
145
|
<tr>
|
|
112
146
|
<td>Cost (pricing known)</td>
|
|
113
147
|
<td class="lct-num"><%= percent(cost_coverage) %></td>
|
|
114
|
-
<td class="lct-num"><%= number(
|
|
148
|
+
<td class="lct-num"><%= number(total - @stats.unknown_pricing_count) %></td>
|
|
115
149
|
<td><%= render "llm_cost_tracker/shared/bar", value: cost_coverage, max: 100.0 %></td>
|
|
116
150
|
</tr>
|
|
117
151
|
|
|
118
|
-
<% tag_coverage = coverage_percent(
|
|
152
|
+
<% tag_coverage = coverage_percent(total - @stats.untagged_calls_count, total) %>
|
|
119
153
|
<tr>
|
|
120
154
|
<td>Tags (at least one tag)</td>
|
|
121
155
|
<td class="lct-num"><%= percent(tag_coverage) %></td>
|
|
122
|
-
<td class="lct-num"><%= number(
|
|
156
|
+
<td class="lct-num"><%= number(total - @stats.untagged_calls_count) %></td>
|
|
123
157
|
<td><%= render "llm_cost_tracker/shared/bar", value: tag_coverage, max: 100.0 %></td>
|
|
124
158
|
</tr>
|
|
125
159
|
|
|
126
160
|
<% if @stats.latency_column_present %>
|
|
127
|
-
<% latency_coverage = coverage_percent(
|
|
161
|
+
<% latency_coverage = coverage_percent(total - @stats.missing_latency_count, total) %>
|
|
128
162
|
<tr>
|
|
129
163
|
<td>Latency</td>
|
|
130
164
|
<td class="lct-num"><%= percent(latency_coverage) %></td>
|
|
131
|
-
<td class="lct-num"><%= number(
|
|
165
|
+
<td class="lct-num"><%= number(total - @stats.missing_latency_count) %></td>
|
|
132
166
|
<td><%= render "llm_cost_tracker/shared/bar", value: latency_coverage, max: 100.0 %></td>
|
|
133
167
|
</tr>
|
|
134
168
|
<% end %>
|
|
169
|
+
|
|
170
|
+
<% if @stats.stream_column_present && streaming_count.to_i.positive? && streaming_missing_usage %>
|
|
171
|
+
<% stream_coverage = coverage_percent(streaming_count - streaming_missing_usage, streaming_count) %>
|
|
172
|
+
<tr>
|
|
173
|
+
<td>Streaming usage captured</td>
|
|
174
|
+
<td class="lct-num"><%= percent(stream_coverage) %></td>
|
|
175
|
+
<td class="lct-num"><%= number(streaming_count - streaming_missing_usage) %> / <%= number(streaming_count) %></td>
|
|
176
|
+
<td><%= render "llm_cost_tracker/shared/bar", value: stream_coverage, max: 100.0 %></td>
|
|
177
|
+
</tr>
|
|
178
|
+
<% end %>
|
|
179
|
+
|
|
180
|
+
<% if @stats.provider_response_id_column_present %>
|
|
181
|
+
<% provider_response_id_coverage = coverage_percent(calls_with_provider_response_id, total) %>
|
|
182
|
+
<tr>
|
|
183
|
+
<td>Provider response ID</td>
|
|
184
|
+
<td class="lct-num"><%= percent(provider_response_id_coverage) %></td>
|
|
185
|
+
<td class="lct-num"><%= number(calls_with_provider_response_id) %></td>
|
|
186
|
+
<td><%= render "llm_cost_tracker/shared/bar", value: provider_response_id_coverage, max: 100.0 %></td>
|
|
187
|
+
</tr>
|
|
188
|
+
<% end %>
|
|
135
189
|
</tbody>
|
|
136
190
|
</table>
|
|
137
191
|
</section>
|
|
@@ -170,6 +224,20 @@
|
|
|
170
224
|
<td>Make sure latency capture is enabled on every tracked request.</td>
|
|
171
225
|
</tr>
|
|
172
226
|
<% end %>
|
|
227
|
+
<% if @stats.stream_column_present && streaming_missing_usage.to_i.positive? %>
|
|
228
|
+
<tr>
|
|
229
|
+
<td>Streams without usage</td>
|
|
230
|
+
<td>Token totals undercount when streaming responses drop the final usage event.</td>
|
|
231
|
+
<td>Send OpenAI requests with <code class="lct-code">stream_options: { include_usage: true }</code>, or wrap custom clients with <code class="lct-code">LlmCostTracker.track_stream</code>.</td>
|
|
232
|
+
</tr>
|
|
233
|
+
<% end %>
|
|
234
|
+
<% if @stats.provider_response_id_column_present && @stats.missing_provider_response_id_count.to_i.positive? %>
|
|
235
|
+
<tr>
|
|
236
|
+
<td>Missing provider response IDs</td>
|
|
237
|
+
<td>Proof of provider-issued responses is weaker when calls cannot be tied back to provider objects.</td>
|
|
238
|
+
<td>Upgrade to the latest parser coverage and pass <code class="lct-code">provider_response_id:</code> for custom clients when the provider exposes one.</td>
|
|
239
|
+
</tr>
|
|
240
|
+
<% end %>
|
|
173
241
|
</tbody>
|
|
174
242
|
</table>
|
|
175
243
|
</section>
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
<div class="lct-filter-row lct-filter-row-with-sort">
|
|
8
8
|
<div class="lct-field">
|
|
9
9
|
<label for="lct-models-from">From</label>
|
|
10
|
-
<input id="lct-models-from"
|
|
10
|
+
<input id="lct-models-from" type="date" name="from" value="<%= params[:from] %>">
|
|
11
11
|
</div>
|
|
12
12
|
|
|
13
13
|
<div class="lct-field">
|
|
@@ -33,14 +33,15 @@
|
|
|
33
33
|
|
|
34
34
|
<div class="lct-field">
|
|
35
35
|
<label for="lct-models-sort">Sort</label>
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
36
|
+
<%= select_tag :sort,
|
|
37
|
+
options_for_select(
|
|
38
|
+
[["Total spend", "cost"],
|
|
39
|
+
["Call volume", "calls"],
|
|
40
|
+
["Avg cost / call", "avg_cost"]] +
|
|
41
|
+
(@latency_available ? [["Avg latency", "latency"]] : []),
|
|
42
|
+
@sort.presence || "cost"
|
|
43
|
+
),
|
|
44
|
+
id: "lct-models-sort" %>
|
|
44
45
|
</div>
|
|
45
46
|
|
|
46
47
|
<div class="lct-filter-actions">
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
<div class="lct-filter-row lct-filter-row-basic">
|
|
8
8
|
<div class="lct-field">
|
|
9
9
|
<label for="lct-tags-from">From</label>
|
|
10
|
-
<input id="lct-tags-from"
|
|
10
|
+
<input id="lct-tags-from" type="date" name="from" value="<%= params[:from] %>">
|
|
11
11
|
</div>
|
|
12
12
|
|
|
13
13
|
<div class="lct-field">
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
<div class="lct-filter-row lct-filter-row-basic">
|
|
13
13
|
<div class="lct-field">
|
|
14
14
|
<label for="lct-tag-show-from">From</label>
|
|
15
|
-
<input id="lct-tag-show-from"
|
|
15
|
+
<input id="lct-tag-show-from" type="date" name="from" value="<%= params[:from] %>">
|
|
16
16
|
</div>
|
|
17
17
|
|
|
18
18
|
<div class="lct-field">
|
|
@@ -6,19 +6,14 @@ module LlmCostTracker
|
|
|
6
6
|
module Assets
|
|
7
7
|
ROOT = File.expand_path("../../app/assets/llm_cost_tracker", __dir__)
|
|
8
8
|
STYLESHEET = "application.css"
|
|
9
|
+
STYLESHEET_PATH = File.join(ROOT, STYLESHEET).freeze
|
|
10
|
+
STYLESHEET_FINGERPRINT = Digest::SHA256.file(STYLESHEET_PATH).hexdigest[0, 12].freeze
|
|
11
|
+
STYLESHEET_FILENAME = "application-#{STYLESHEET_FINGERPRINT}.css".freeze
|
|
9
12
|
|
|
10
13
|
class << self
|
|
11
|
-
def root
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
def stylesheet_fingerprint
|
|
16
|
-
@stylesheet_fingerprint ||= Digest::SHA256.file(File.join(ROOT, STYLESHEET)).hexdigest[0, 12]
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def stylesheet_filename
|
|
20
|
-
"application-#{stylesheet_fingerprint}.css"
|
|
21
|
-
end
|
|
14
|
+
def root = ROOT
|
|
15
|
+
def stylesheet_fingerprint = STYLESHEET_FINGERPRINT
|
|
16
|
+
def stylesheet_filename = STYLESHEET_FILENAME
|
|
22
17
|
end
|
|
23
18
|
end
|
|
24
19
|
end
|