llm_cost_tracker 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -0
- data/README.md +11 -7
- data/app/controllers/llm_cost_tracker/calls_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +3 -15
- data/app/controllers/llm_cost_tracker/tags_controller.rb +7 -6
- data/app/helpers/llm_cost_tracker/application_helper.rb +21 -6
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +3 -1
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +42 -0
- data/app/services/llm_cost_tracker/dashboard/filter.rb +6 -8
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +6 -5
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +74 -18
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +15 -4
- data/app/views/llm_cost_tracker/shared/_tag_chips.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +4 -0
- data/lib/llm_cost_tracker/configuration.rb +22 -16
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +1 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +7 -1
- data/lib/llm_cost_tracker/integrations/anthropic.rb +12 -3
- data/lib/llm_cost_tracker/integrations/base.rb +77 -6
- data/lib/llm_cost_tracker/integrations/object_reader.rb +1 -1
- data/lib/llm_cost_tracker/integrations/openai.rb +14 -5
- data/lib/llm_cost_tracker/integrations/registry.rb +3 -1
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +171 -0
- data/lib/llm_cost_tracker/llm_api_call.rb +10 -9
- data/lib/llm_cost_tracker/middleware/faraday.rb +8 -4
- data/lib/llm_cost_tracker/parsers/gemini.rb +8 -1
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +11 -2
- data/lib/llm_cost_tracker/price_registry.rb +3 -0
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +41 -12
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +6 -0
- data/lib/llm_cost_tracker/report.rb +8 -1
- data/lib/llm_cost_tracker/report_data.rb +25 -9
- data/lib/llm_cost_tracker/retention.rb +30 -7
- data/lib/llm_cost_tracker/stream_capture.rb +7 -0
- data/lib/llm_cost_tracker/stream_collector.rb +25 -1
- data/lib/llm_cost_tracker/tag_sanitizer.rb +81 -0
- data/lib/llm_cost_tracker/tracker.rb +6 -2
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +1 -0
- metadata +9 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9d515abad24d5659f797d5ea0426d6111dc41baf78809181128546e29ac566db
|
|
4
|
+
data.tar.gz: 90da1dda161661423f6034f28d7bb359e06c39f48b9f4872cb0eba9a123f8948
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: aa7af4b5caca984cbcb4c8a7a267600251962e1879de8f3bff32d8a3f6e8d22888ba4a26036de09c36345566e3c309a2868012b66b5dcc970e52bcb6b5e444be
|
|
7
|
+
data.tar.gz: 932f2bb680690ed043eb719debddd58ac85c7f1cbbd0ec9c30c95f5595d71a5f70516b3d8d19ca65929856c4ade902a813da34b4de2053f553aa916eece66865
|
data/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,33 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning: [S
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.5.2] - 2026-04-27
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- RubyLLM SDK integration for chat, embedding, and transcription calls.
|
|
12
|
+
- Tag guardrails for redacted tag keys, maximum tag count, and maximum tag value byte size.
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- SDK integrations now validate minimum versions and method contracts before installing wrappers.
|
|
17
|
+
- `config.instrument :all` now includes RubyLLM.
|
|
18
|
+
- Dashboard date filters now reject one-sided, reversed, and over-366-day ranges.
|
|
19
|
+
- Dashboard provider/model/tag option lists and tag value breakdowns now cap rendered rows.
|
|
20
|
+
- Reports now cap rendered breakdown groups while keeping complete structured report data available.
|
|
21
|
+
- Stream capture now enforces a shared 1 MiB buffer cap and records unknown usage on overflow.
|
|
22
|
+
- Price refresh, price scrape, and local price registry reads now enforce response or file size caps.
|
|
23
|
+
- Retention pruning now rejects non-positive batch sizes and invalid cutoffs before deleting rows.
|
|
24
|
+
- The install generator now warns to mount the dashboard behind host-app admin authentication.
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- OpenAI SDK integration now separates cached input tokens from regular input tokens.
|
|
29
|
+
- OpenAI and Gemini parsers now compute total tokens when provider responses omit totals.
|
|
30
|
+
- CSV export now prefixes formula-like values even when they have leading whitespace.
|
|
31
|
+
- Tag chips now truncate oversized values and tooltips.
|
|
32
|
+
- Report tag breakdown keys are validated at configuration time.
|
|
33
|
+
|
|
7
34
|
## [0.5.1] - 2026-04-27
|
|
8
35
|
|
|
9
36
|
### Changed
|
data/README.md
CHANGED
|
@@ -20,7 +20,7 @@ Add to your Gemfile alongside whatever LLM client you already use:
|
|
|
20
20
|
|
|
21
21
|
```ruby
|
|
22
22
|
gem "llm_cost_tracker"
|
|
23
|
-
gem "openai" # or "anthropic", or your existing client
|
|
23
|
+
gem "openai" # or "anthropic", "ruby_llm", or your existing client
|
|
24
24
|
```
|
|
25
25
|
|
|
26
26
|
Install, migrate, verify:
|
|
@@ -55,7 +55,7 @@ Visit `/llm-costs` for the dashboard. **Mount it behind your app's auth before d
|
|
|
55
55
|
## What you get
|
|
56
56
|
|
|
57
57
|
- Local ActiveRecord ledger of every call: provider, model, token breakdown, cost, latency, tags, response IDs
|
|
58
|
-
- Auto-capture for the official `openai` and `anthropic` Ruby SDKs, plus Faraday middleware for `ruby-openai`, the Gemini REST API, and any client you can inject middleware into
|
|
58
|
+
- Auto-capture for RubyLLM and the official `openai` and `anthropic` Ruby SDKs, plus Faraday middleware for `ruby-openai`, the Gemini REST API, and any client you can inject middleware into
|
|
59
59
|
- Server-rendered dashboard (plain ERB, zero JavaScript) with overview, models, calls, tags, CSV export, and a data-quality page
|
|
60
60
|
- Local pricing snapshots refreshed daily from the official provider pricing pages, applied with `bin/rails llm_cost_tracker:prices:refresh`
|
|
61
61
|
- Monthly / daily / per-call budget guardrails with notify, raise, or block-requests behaviour
|
|
@@ -73,13 +73,13 @@ Visit `/llm-costs` for the dashboard. **Mount it behind your app's auth before d
|
|
|
73
73
|
|
|
74
74
|
Three paths, in order of preference. Use the first one that fits your stack.
|
|
75
75
|
|
|
76
|
-
### 1.
|
|
76
|
+
### 1. SDK integrations
|
|
77
77
|
|
|
78
|
-
Drop-in for the official `openai` and `anthropic` gems. `config.instrument` patches
|
|
78
|
+
Drop-in for RubyLLM and the official `openai` and `anthropic` gems. `config.instrument` patches tested SDK methods so you don't change a single call site:
|
|
79
79
|
|
|
80
80
|
```ruby
|
|
81
81
|
LlmCostTracker.configure do |config|
|
|
82
|
-
config.instrument :openai # or :anthropic
|
|
82
|
+
config.instrument :openai # or :anthropic / :ruby_llm
|
|
83
83
|
end
|
|
84
84
|
|
|
85
85
|
LlmCostTracker.with_tags(feature: "support_chat") do
|
|
@@ -93,7 +93,9 @@ end
|
|
|
93
93
|
|
|
94
94
|
Captures usage, model, latency, response ID, cache tokens, and reasoning tokens whenever the SDK exposes them. Provider SDKs are not added as gem dependencies — you install whichever you actually use.
|
|
95
95
|
|
|
96
|
-
|
|
96
|
+
Enabled integrations are checked at boot: the client gem must be loaded, meet the minimum supported version, and expose the expected classes and methods. If the contract check fails, boot raises instead of silently missing spend.
|
|
97
|
+
|
|
98
|
+
This patches **only** RubyLLM and the official Ruby SDKs. `ruby-openai` (alexrudall) and any custom client go through Faraday middleware below.
|
|
97
99
|
|
|
98
100
|
### 2. Faraday middleware
|
|
99
101
|
|
|
@@ -227,6 +229,8 @@ Auth is your job. Examples for basic auth and Devise: [`docs/dashboard.md`](docs
|
|
|
227
229
|
| Other OpenAI-compatible hosts | Configurable | Register the host via `config.openai_compatible_providers` |
|
|
228
230
|
| Anything else | Configurable | Custom parser — see [`docs/extending.md`](docs/extending.md) |
|
|
229
231
|
|
|
232
|
+
RubyLLM chat, embedding, and transcription calls are captured through RubyLLM's provider layer when `config.instrument :ruby_llm` is enabled.
|
|
233
|
+
|
|
230
234
|
Endpoints covered end-to-end: OpenAI Chat Completions / Responses / Completions / Embeddings, Anthropic Messages, Gemini `generateContent` and `streamGenerateContent`, plus their OpenAI-compatible equivalents. Streaming is captured for Faraday paths whenever the provider emits final-usage events.
|
|
231
235
|
|
|
232
236
|
## Privacy
|
|
@@ -256,7 +260,7 @@ is still brief.
|
|
|
256
260
|
## Known limitations
|
|
257
261
|
|
|
258
262
|
- `:block_requests` is best-effort under concurrency, not a transactional cap.
|
|
259
|
-
- Official SDK integrations cover non-streaming calls; streaming via
|
|
263
|
+
- Official OpenAI and Anthropic SDK integrations cover non-streaming calls; streaming via those SDKs falls back to Faraday middleware or `track_stream`.
|
|
260
264
|
- Streaming usage capture relies on the provider emitting a final-usage event. Missing events are stored with `usage_source: "unknown"` so they appear on the data-quality page rather than vanishing.
|
|
261
265
|
- `provider_response_id` is stored only when the provider exposes a stable ID. Gemini is best-effort and varies by endpoint.
|
|
262
266
|
- Cache write TTL variants on Anthropic (1h vs 5min writes) are not modeled separately yet.
|
|
@@ -84,7 +84,8 @@ module LlmCostTracker
|
|
|
84
84
|
return value if value.nil?
|
|
85
85
|
|
|
86
86
|
string = value.to_s
|
|
87
|
-
|
|
87
|
+
stripped = string.lstrip
|
|
88
|
+
CSV_FORMULA_PREFIXES.include?(stripped[0]) ? "'#{string}" : string
|
|
88
89
|
end
|
|
89
90
|
end
|
|
90
91
|
end
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
class DashboardController < ApplicationController
|
|
5
5
|
def index
|
|
6
|
-
|
|
6
|
+
range = Dashboard::DateRange.call(params: params)
|
|
7
|
+
@from_date = range.from
|
|
8
|
+
@to_date = range.to
|
|
7
9
|
prev_from, prev_to = previous_range
|
|
8
10
|
filter_params = LlmCostTracker::ParameterHash.to_hash(params)
|
|
9
11
|
scope = Dashboard::Filter.call(
|
|
@@ -23,25 +25,11 @@ module LlmCostTracker
|
|
|
23
25
|
|
|
24
26
|
private
|
|
25
27
|
|
|
26
|
-
def overview_range
|
|
27
|
-
to_date = parsed_date(params[:to]) || Date.current
|
|
28
|
-
from_date = parsed_date(params[:from]) || (to_date - 29)
|
|
29
|
-
[from_date, to_date]
|
|
30
|
-
end
|
|
31
|
-
|
|
32
28
|
def previous_range
|
|
33
29
|
span_days = (@to_date - @from_date).to_i + 1
|
|
34
30
|
prev_to = @from_date - 1
|
|
35
31
|
prev_from = prev_to - (span_days - 1)
|
|
36
32
|
[prev_from, prev_to]
|
|
37
33
|
end
|
|
38
|
-
|
|
39
|
-
def parsed_date(value)
|
|
40
|
-
return nil if value.to_s.strip.empty?
|
|
41
|
-
|
|
42
|
-
Date.iso8601(value.to_s)
|
|
43
|
-
rescue ArgumentError
|
|
44
|
-
nil
|
|
45
|
-
end
|
|
46
34
|
end
|
|
47
35
|
end
|
|
@@ -8,12 +8,13 @@ module LlmCostTracker
|
|
|
8
8
|
|
|
9
9
|
def show
|
|
10
10
|
@tag_key = params[:key]
|
|
11
|
-
|
|
12
|
-
@
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
@
|
|
16
|
-
@
|
|
11
|
+
breakdown = Dashboard::TagBreakdown.call(scope: Dashboard::Filter.call(params: params), key: @tag_key)
|
|
12
|
+
@rows = breakdown.rows
|
|
13
|
+
@total_calls = breakdown.total_calls
|
|
14
|
+
@tagged_calls = breakdown.tagged_calls
|
|
15
|
+
@distinct_values = breakdown.distinct_values
|
|
16
|
+
@tag_value_limit = breakdown.limit
|
|
17
|
+
@tag_values_limited = breakdown.limited?
|
|
17
18
|
end
|
|
18
19
|
end
|
|
19
20
|
end
|
|
@@ -4,6 +4,9 @@ require "json"
|
|
|
4
4
|
|
|
5
5
|
module LlmCostTracker
|
|
6
6
|
module ApplicationHelper
|
|
7
|
+
TAG_VALUE_SUMMARY_BYTES = 80
|
|
8
|
+
TAG_TOOLTIP_BYTES = 512
|
|
9
|
+
|
|
7
10
|
include DashboardFilterHelper
|
|
8
11
|
include DashboardFilterOptionsHelper
|
|
9
12
|
include DashboardQueryHelper
|
|
@@ -116,6 +119,10 @@ module LlmCostTracker
|
|
|
116
119
|
visible
|
|
117
120
|
end
|
|
118
121
|
|
|
122
|
+
def tag_chips_title(tags)
|
|
123
|
+
truncate_text(safe_json(tags), TAG_TOOLTIP_BYTES)
|
|
124
|
+
end
|
|
125
|
+
|
|
119
126
|
def budget_fill_modifier(percent)
|
|
120
127
|
percent = percent.to_f
|
|
121
128
|
return "lct-budget-fill--over" if percent >= 100.0
|
|
@@ -143,12 +150,20 @@ module LlmCostTracker
|
|
|
143
150
|
end
|
|
144
151
|
|
|
145
152
|
def tag_value_summary(value)
|
|
146
|
-
case value
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
153
|
+
string = case value
|
|
154
|
+
when Hash, Array
|
|
155
|
+
JSON.generate(value)
|
|
156
|
+
else
|
|
157
|
+
value.to_s
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
truncate_text(string, TAG_VALUE_SUMMARY_BYTES)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def truncate_text(string, limit)
|
|
164
|
+
return string if string.bytesize <= limit
|
|
165
|
+
|
|
166
|
+
"#{string.byteslice(0, limit).to_s.encode('UTF-8', invalid: :replace, undef: :replace)}..."
|
|
152
167
|
end
|
|
153
168
|
end
|
|
154
169
|
end
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module DashboardFilterOptionsHelper
|
|
5
|
+
MAX_FILTER_OPTIONS = 100
|
|
6
|
+
|
|
5
7
|
def provider_filter_options(filter_params: params)
|
|
6
8
|
filter_options_for(:provider, filter_params: filter_params)
|
|
7
9
|
end
|
|
@@ -19,7 +21,7 @@ module LlmCostTracker
|
|
|
19
21
|
)
|
|
20
22
|
values = LlmCostTracker::Dashboard::Filter.call(params: scope_params)
|
|
21
23
|
.where.not(column => [nil, ""])
|
|
22
|
-
.distinct.order(column).pluck(column)
|
|
24
|
+
.distinct.order(column).limit(MAX_FILTER_OPTIONS).pluck(column)
|
|
23
25
|
current = source[column.to_s].presence || source[column].presence
|
|
24
26
|
values.unshift(current) if current && !values.include?(current)
|
|
25
27
|
values
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module Dashboard
|
|
5
|
+
class DateRange
|
|
6
|
+
DEFAULT_DAYS = 30
|
|
7
|
+
MAX_DAYS = 366
|
|
8
|
+
|
|
9
|
+
attr_reader :from, :to
|
|
10
|
+
|
|
11
|
+
def self.call(params:, today: Date.current)
|
|
12
|
+
new(params: params, today: today)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.parse(params, key)
|
|
16
|
+
value = LlmCostTracker::ParameterHash.with_indifferent_access(params)[key].to_s.strip
|
|
17
|
+
return nil if value.empty?
|
|
18
|
+
|
|
19
|
+
Date.iso8601(value)
|
|
20
|
+
rescue ArgumentError
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.validate!(from:, to:)
|
|
25
|
+
return if from.nil? && to.nil?
|
|
26
|
+
|
|
27
|
+
raise InvalidFilterError, "from and to dates must be provided together" if from.nil? || to.nil?
|
|
28
|
+
raise InvalidFilterError, "from date must be on or before to date" if from > to
|
|
29
|
+
return if ((to - from).to_i + 1) <= MAX_DAYS
|
|
30
|
+
|
|
31
|
+
raise InvalidFilterError, "date range cannot exceed #{MAX_DAYS} days"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def initialize(params:, today:)
|
|
35
|
+
@to = self.class.parse(params, :to) || today
|
|
36
|
+
@from = self.class.parse(params, :from) || (@to - (DEFAULT_DAYS - 1))
|
|
37
|
+
self.class.validate!(from: @from, to: @to)
|
|
38
|
+
freeze
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -31,9 +31,12 @@ module LlmCostTracker
|
|
|
31
31
|
attr_reader :scope, :params
|
|
32
32
|
|
|
33
33
|
def apply_date_filters(relation)
|
|
34
|
-
|
|
35
|
-
|
|
34
|
+
from_date = parse_date(:from)
|
|
35
|
+
to_date = parse_date(:to)
|
|
36
|
+
Dashboard::DateRange.validate!(from: from_date, to: to_date)
|
|
36
37
|
|
|
38
|
+
from = from_date&.beginning_of_day
|
|
39
|
+
to = to_date&.end_of_day
|
|
37
40
|
relation = relation.where(tracked_at: from..) if from
|
|
38
41
|
relation = relation.where(tracked_at: ..to) if to
|
|
39
42
|
relation
|
|
@@ -89,12 +92,7 @@ module LlmCostTracker
|
|
|
89
92
|
end
|
|
90
93
|
|
|
91
94
|
def parse_date(key)
|
|
92
|
-
|
|
93
|
-
return nil if value.nil?
|
|
94
|
-
|
|
95
|
-
Date.iso8601(value)
|
|
96
|
-
rescue ArgumentError
|
|
97
|
-
nil
|
|
95
|
+
Dashboard::DateRange.parse(params, key)
|
|
98
96
|
end
|
|
99
97
|
|
|
100
98
|
def normalized_string(value)
|
|
@@ -65,11 +65,12 @@ module LlmCostTracker
|
|
|
65
65
|
|
|
66
66
|
scope
|
|
67
67
|
.where(tracked_at: window)
|
|
68
|
-
.
|
|
69
|
-
.
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
68
|
+
.where.not(total_cost: nil)
|
|
69
|
+
.group(:provider, :model)
|
|
70
|
+
.group_by_period(:day)
|
|
71
|
+
.sum(:total_cost)
|
|
72
|
+
.each do |(provider, model, day), total_cost|
|
|
73
|
+
grouped[[provider, model]][Date.iso8601(day.to_s)] += total_cost.to_f
|
|
73
74
|
end
|
|
74
75
|
|
|
75
76
|
grouped
|
|
@@ -9,28 +9,54 @@ module LlmCostTracker
|
|
|
9
9
|
:average_cost_per_call
|
|
10
10
|
)
|
|
11
11
|
|
|
12
|
+
TagBreakdownResult = Data.define(
|
|
13
|
+
:rows,
|
|
14
|
+
:total_calls,
|
|
15
|
+
:tagged_calls,
|
|
16
|
+
:distinct_values,
|
|
17
|
+
:limit
|
|
18
|
+
) do
|
|
19
|
+
def limited? = distinct_values > rows.size
|
|
20
|
+
end
|
|
21
|
+
|
|
12
22
|
class TagBreakdown
|
|
23
|
+
DEFAULT_LIMIT = 100
|
|
24
|
+
|
|
13
25
|
class << self
|
|
14
|
-
def call(key:, scope: LlmCostTracker::LlmApiCall.all)
|
|
15
|
-
new(scope: scope, key: key).
|
|
26
|
+
def call(key:, scope: LlmCostTracker::LlmApiCall.all, limit: DEFAULT_LIMIT)
|
|
27
|
+
new(scope: scope, key: key, limit: limit).result
|
|
16
28
|
end
|
|
17
29
|
end
|
|
18
30
|
|
|
19
|
-
def initialize(scope:, key:)
|
|
31
|
+
def initialize(scope:, key:, limit:)
|
|
20
32
|
@scope = scope
|
|
21
33
|
@key = LlmCostTracker::TagKey.validate!(key, error_class: LlmCostTracker::InvalidFilterError)
|
|
34
|
+
@limit = normalized_limit(limit)
|
|
35
|
+
@connection = LlmCostTracker::LlmApiCall.connection
|
|
22
36
|
end
|
|
23
37
|
|
|
24
|
-
def
|
|
25
|
-
|
|
26
|
-
counts = counts_by_tag
|
|
38
|
+
def result
|
|
39
|
+
counts = summary_counts
|
|
27
40
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
41
|
+
TagBreakdownResult.new(
|
|
42
|
+
rows: rows,
|
|
43
|
+
total_calls: counts.fetch(:total_calls),
|
|
44
|
+
tagged_calls: counts.fetch(:tagged_calls),
|
|
45
|
+
distinct_values: counts.fetch(:distinct_values),
|
|
46
|
+
limit: limit
|
|
47
|
+
)
|
|
48
|
+
end
|
|
31
49
|
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
attr_reader :scope, :key, :limit, :connection
|
|
53
|
+
|
|
54
|
+
def rows
|
|
55
|
+
connection.select_all(rows_sql).map do |row|
|
|
56
|
+
calls = row["calls_count"].to_i
|
|
57
|
+
total_cost = row["total_cost_sum"].to_f
|
|
32
58
|
TagBreakdownRow.new(
|
|
33
|
-
value:
|
|
59
|
+
value: LlmCostTracker::LlmApiCall.tag_value_label(row["tag_value"]),
|
|
34
60
|
calls: calls,
|
|
35
61
|
total_cost: total_cost,
|
|
36
62
|
average_cost_per_call: calls.positive? ? total_cost / calls : 0.0
|
|
@@ -38,18 +64,48 @@ module LlmCostTracker
|
|
|
38
64
|
end
|
|
39
65
|
end
|
|
40
66
|
|
|
41
|
-
|
|
67
|
+
def summary_counts
|
|
68
|
+
row = connection.select_one(summary_sql) || {}
|
|
69
|
+
{
|
|
70
|
+
total_calls: row["total_calls"].to_i,
|
|
71
|
+
tagged_calls: row["tagged_calls"].to_i,
|
|
72
|
+
distinct_values: row["distinct_values"].to_i
|
|
73
|
+
}
|
|
74
|
+
end
|
|
42
75
|
|
|
43
|
-
|
|
76
|
+
def rows_sql
|
|
77
|
+
<<~SQL.squish
|
|
78
|
+
SELECT #{tag_expression} AS tag_value,
|
|
79
|
+
COUNT(*) AS calls_count,
|
|
80
|
+
COALESCE(SUM(sub.total_cost), 0) AS total_cost_sum
|
|
81
|
+
FROM (#{scope.to_sql}) AS sub
|
|
82
|
+
WHERE #{tag_present_predicate}
|
|
83
|
+
GROUP BY #{tag_expression}
|
|
84
|
+
ORDER BY total_cost_sum DESC, calls_count DESC, tag_value ASC
|
|
85
|
+
LIMIT #{limit}
|
|
86
|
+
SQL
|
|
87
|
+
end
|
|
44
88
|
|
|
45
|
-
def
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
89
|
+
def summary_sql
|
|
90
|
+
<<~SQL.squish
|
|
91
|
+
SELECT COUNT(*) AS total_calls,
|
|
92
|
+
COALESCE(SUM(CASE WHEN #{tag_present_predicate} THEN 1 ELSE 0 END), 0) AS tagged_calls,
|
|
93
|
+
COUNT(DISTINCT CASE WHEN #{tag_present_predicate} THEN #{tag_expression} END) AS distinct_values
|
|
94
|
+
FROM (#{scope.to_sql}) AS sub
|
|
95
|
+
SQL
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def tag_present_predicate
|
|
99
|
+
"#{tag_expression} IS NOT NULL AND #{tag_expression} != ''"
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def tag_expression
|
|
103
|
+
@tag_expression ||= LlmCostTracker::LlmApiCall.tag_value_expression(key, table_name: "sub")
|
|
49
104
|
end
|
|
50
105
|
|
|
51
|
-
def
|
|
52
|
-
value
|
|
106
|
+
def normalized_limit(value)
|
|
107
|
+
value = value.to_i
|
|
108
|
+
value.positive? ? [value, DEFAULT_LIMIT].min : DEFAULT_LIMIT
|
|
53
109
|
end
|
|
54
110
|
end
|
|
55
111
|
end
|
|
@@ -5,15 +5,18 @@ module LlmCostTracker
|
|
|
5
5
|
TagKeyRow = Data.define(:key, :calls_count, :distinct_values)
|
|
6
6
|
|
|
7
7
|
class TagKeyExplorer
|
|
8
|
+
DEFAULT_LIMIT = 100
|
|
9
|
+
|
|
8
10
|
class << self
|
|
9
|
-
def call(scope: LlmCostTracker::LlmApiCall.all)
|
|
10
|
-
new(scope: scope).rows
|
|
11
|
+
def call(scope: LlmCostTracker::LlmApiCall.all, limit: DEFAULT_LIMIT)
|
|
12
|
+
new(scope: scope, limit: limit).rows
|
|
11
13
|
end
|
|
12
14
|
end
|
|
13
15
|
|
|
14
|
-
def initialize(scope:)
|
|
16
|
+
def initialize(scope:, limit:)
|
|
15
17
|
@scope = scope
|
|
16
18
|
@connection = LlmCostTracker::LlmApiCall.connection
|
|
19
|
+
@limit = normalized_limit(limit)
|
|
17
20
|
end
|
|
18
21
|
|
|
19
22
|
def rows
|
|
@@ -32,7 +35,7 @@ module LlmCostTracker
|
|
|
32
35
|
|
|
33
36
|
private
|
|
34
37
|
|
|
35
|
-
attr_reader :scope, :connection
|
|
38
|
+
attr_reader :scope, :connection, :limit
|
|
36
39
|
|
|
37
40
|
def subquery
|
|
38
41
|
scope.to_sql
|
|
@@ -62,6 +65,7 @@ module LlmCostTracker
|
|
|
62
65
|
AND sub.tags != ''
|
|
63
66
|
GROUP BY jt.key
|
|
64
67
|
ORDER BY calls_count DESC
|
|
68
|
+
LIMIT #{limit}
|
|
65
69
|
SQL
|
|
66
70
|
end
|
|
67
71
|
|
|
@@ -76,6 +80,7 @@ module LlmCostTracker
|
|
|
76
80
|
AND sub.tags::jsonb <> '{}'::jsonb
|
|
77
81
|
GROUP BY key
|
|
78
82
|
ORDER BY calls_count DESC
|
|
83
|
+
LIMIT #{limit}
|
|
79
84
|
SQL
|
|
80
85
|
end
|
|
81
86
|
|
|
@@ -91,8 +96,14 @@ module LlmCostTracker
|
|
|
91
96
|
AND sub.tags != ''
|
|
92
97
|
GROUP BY je.key
|
|
93
98
|
ORDER BY calls_count DESC
|
|
99
|
+
LIMIT #{limit}
|
|
94
100
|
SQL
|
|
95
101
|
end
|
|
102
|
+
|
|
103
|
+
def normalized_limit(value)
|
|
104
|
+
value = value.to_i
|
|
105
|
+
value.positive? ? [value, DEFAULT_LIMIT].min : DEFAULT_LIMIT
|
|
106
|
+
end
|
|
96
107
|
end
|
|
97
108
|
end
|
|
98
109
|
end
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
<% if entries.empty? %>
|
|
3
3
|
<span class="lct-tag-empty">(untagged)</span>
|
|
4
4
|
<% else %>
|
|
5
|
-
<span class="lct-tag-chips" title="<%=
|
|
5
|
+
<span class="lct-tag-chips" title="<%= tag_chips_title(tags) %>">
|
|
6
6
|
<% entries.each do |entry| %>
|
|
7
7
|
<% if entry[:more] %>
|
|
8
8
|
<span class="lct-tag-chip lct-tag-chip-more">+<%= entry[:more] %></span>
|
|
@@ -50,6 +50,10 @@
|
|
|
50
50
|
<span><strong><%= percent(coverage_percent(@tagged_calls, @total_calls)) %></strong> coverage</span>
|
|
51
51
|
<span><strong><%= number(@distinct_values) %></strong> distinct values</span>
|
|
52
52
|
</p>
|
|
53
|
+
|
|
54
|
+
<% if @tag_values_limited %>
|
|
55
|
+
<p class="lct-toolbar-note">Showing top <%= number(@tag_value_limit) %> values by spend.</p>
|
|
56
|
+
<% end %>
|
|
53
57
|
</section>
|
|
54
58
|
|
|
55
59
|
<% if @rows.empty? %>
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "errors"
|
|
4
|
+
require_relative "tag_key"
|
|
4
5
|
require_relative "value_helpers"
|
|
5
6
|
require_relative "configuration/instrumentation"
|
|
6
7
|
|
|
@@ -8,31 +9,21 @@ module LlmCostTracker
|
|
|
8
9
|
class Configuration
|
|
9
10
|
include ConfigurationInstrumentation
|
|
10
11
|
|
|
11
|
-
OPENAI_COMPATIBLE_PROVIDERS = {
|
|
12
|
-
"openrouter.ai" => "openrouter",
|
|
13
|
-
"api.deepseek.com" => "deepseek"
|
|
14
|
-
}.freeze
|
|
12
|
+
OPENAI_COMPATIBLE_PROVIDERS = { "openrouter.ai" => "openrouter", "api.deepseek.com" => "deepseek" }.freeze
|
|
15
13
|
|
|
16
14
|
BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
|
|
17
15
|
STORAGE_ERROR_BEHAVIORS = %i[ignore warn raise].freeze
|
|
18
16
|
STORAGE_BACKENDS = %i[log active_record custom].freeze
|
|
19
17
|
UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
|
|
20
|
-
SHARED_SCALAR_ATTRIBUTES = %i[
|
|
21
|
-
|
|
22
|
-
custom_storage
|
|
23
|
-
on_budget_exceeded
|
|
24
|
-
monthly_budget
|
|
25
|
-
daily_budget
|
|
26
|
-
per_call_budget
|
|
27
|
-
log_level
|
|
28
|
-
prices_file
|
|
29
|
-
].freeze
|
|
18
|
+
SHARED_SCALAR_ATTRIBUTES = %i[enabled custom_storage on_budget_exceeded monthly_budget daily_budget per_call_budget
|
|
19
|
+
log_level prices_file max_tag_count max_tag_value_bytesize].freeze
|
|
30
20
|
SHARED_ENUM_ATTRIBUTES = {
|
|
31
21
|
storage_backend: [STORAGE_BACKENDS, :log],
|
|
32
22
|
budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
|
|
33
23
|
storage_error_behavior: [STORAGE_ERROR_BEHAVIORS, :warn],
|
|
34
24
|
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
|
|
35
25
|
}.freeze
|
|
26
|
+
DEFAULT_REDACTED_TAG_KEYS = %w[api_key access_token authorization credential password refresh_token secret].freeze
|
|
36
27
|
|
|
37
28
|
attr_reader(
|
|
38
29
|
*SHARED_SCALAR_ATTRIBUTES,
|
|
@@ -41,6 +32,7 @@ module LlmCostTracker
|
|
|
41
32
|
:pricing_overrides,
|
|
42
33
|
:instrumented_integrations,
|
|
43
34
|
:report_tag_breakdowns,
|
|
35
|
+
:redacted_tag_keys,
|
|
44
36
|
:storage_backend,
|
|
45
37
|
:storage_error_behavior,
|
|
46
38
|
:unknown_pricing_behavior,
|
|
@@ -61,9 +53,12 @@ module LlmCostTracker
|
|
|
61
53
|
self.unknown_pricing_behavior = :warn
|
|
62
54
|
@log_level = :info
|
|
63
55
|
@prices_file = nil
|
|
64
|
-
@
|
|
56
|
+
@max_tag_count = 50
|
|
57
|
+
@max_tag_value_bytesize = 1024
|
|
58
|
+
@pricing_overrides = {}
|
|
65
59
|
@instrumented_integrations = []
|
|
66
60
|
@report_tag_breakdowns = []
|
|
61
|
+
@redacted_tag_keys = DEFAULT_REDACTED_TAG_KEYS.dup
|
|
67
62
|
self.openai_compatible_providers = OPENAI_COMPATIBLE_PROVIDERS
|
|
68
63
|
@finalized = false
|
|
69
64
|
end
|
|
@@ -85,7 +80,12 @@ module LlmCostTracker
|
|
|
85
80
|
|
|
86
81
|
def report_tag_breakdowns=(value)
|
|
87
82
|
ensure_shared_configuration_mutable!
|
|
88
|
-
@report_tag_breakdowns = value
|
|
83
|
+
@report_tag_breakdowns = normalize_report_tag_breakdowns(value)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def redacted_tag_keys=(value)
|
|
87
|
+
ensure_shared_configuration_mutable!
|
|
88
|
+
@redacted_tag_keys = Array(value).map(&:to_s)
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
SHARED_SCALAR_ATTRIBUTES.each do |name|
|
|
@@ -107,6 +107,7 @@ module LlmCostTracker
|
|
|
107
107
|
@pricing_overrides = ValueHelpers.deep_freeze(@pricing_overrides || {})
|
|
108
108
|
@instrumented_integrations = ValueHelpers.deep_freeze(@instrumented_integrations || [])
|
|
109
109
|
@report_tag_breakdowns = ValueHelpers.deep_freeze(Array(@report_tag_breakdowns))
|
|
110
|
+
@redacted_tag_keys = ValueHelpers.deep_freeze(Array(@redacted_tag_keys))
|
|
110
111
|
@openai_compatible_providers = ValueHelpers.deep_freeze(@openai_compatible_providers || {})
|
|
111
112
|
@finalized = true
|
|
112
113
|
self
|
|
@@ -123,6 +124,7 @@ module LlmCostTracker
|
|
|
123
124
|
ValueHelpers.deep_dup(@instrumented_integrations || [])
|
|
124
125
|
)
|
|
125
126
|
copy.instance_variable_set(:@report_tag_breakdowns, ValueHelpers.deep_dup(@report_tag_breakdowns || []))
|
|
127
|
+
copy.instance_variable_set(:@redacted_tag_keys, ValueHelpers.deep_dup(@redacted_tag_keys || []))
|
|
126
128
|
copy.instance_variable_set(
|
|
127
129
|
:@openai_compatible_providers,
|
|
128
130
|
ValueHelpers.deep_dup(@openai_compatible_providers || {})
|
|
@@ -149,6 +151,10 @@ module LlmCostTracker
|
|
|
149
151
|
end
|
|
150
152
|
end
|
|
151
153
|
|
|
154
|
+
def normalize_report_tag_breakdowns(value)
|
|
155
|
+
Array(value).map { |key| TagKey.validate!(key, error_class: Error) }
|
|
156
|
+
end
|
|
157
|
+
|
|
152
158
|
def ensure_shared_configuration_mutable!
|
|
153
159
|
return unless finalized?
|
|
154
160
|
|