llm_cost_tracker 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +43 -0
- data/README.md +18 -9
- data/app/controllers/llm_cost_tracker/calls_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +3 -15
- data/app/controllers/llm_cost_tracker/tags_controller.rb +7 -6
- data/app/helpers/llm_cost_tracker/application_helper.rb +21 -6
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +3 -1
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +42 -0
- data/app/services/llm_cost_tracker/dashboard/filter.rb +6 -8
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +6 -5
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +74 -18
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +15 -4
- data/app/views/llm_cost_tracker/shared/_tag_chips.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +4 -0
- data/docs/architecture.md +28 -0
- data/docs/budgets.md +45 -0
- data/docs/configuration.md +65 -0
- data/docs/cookbook.md +185 -0
- data/docs/dashboard-overview.png +0 -0
- data/docs/dashboard.md +38 -0
- data/docs/extending.md +32 -0
- data/docs/operations.md +44 -0
- data/docs/pricing.md +94 -0
- data/docs/querying.md +36 -0
- data/docs/streaming.md +70 -0
- data/docs/technical/README.md +10 -0
- data/docs/technical/data-flow.md +67 -0
- data/docs/technical/extension-points.md +111 -0
- data/docs/technical/module-map.md +197 -0
- data/docs/technical/operational-notes.md +77 -0
- data/docs/upgrading.md +46 -0
- data/lib/llm_cost_tracker/capture_verifier.rb +71 -0
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +1 -1
- data/lib/llm_cost_tracker/configuration/storage_backend.rb +26 -0
- data/lib/llm_cost_tracker/configuration.rb +24 -17
- data/lib/llm_cost_tracker/doctor/capture_check.rb +39 -0
- data/lib/llm_cost_tracker/doctor.rb +6 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +1 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +7 -1
- data/lib/llm_cost_tracker/integrations/anthropic.rb +51 -3
- data/lib/llm_cost_tracker/integrations/base.rb +77 -6
- data/lib/llm_cost_tracker/integrations/object_reader.rb +1 -1
- data/lib/llm_cost_tracker/integrations/openai.rb +78 -5
- data/lib/llm_cost_tracker/integrations/registry.rb +36 -4
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +171 -0
- data/lib/llm_cost_tracker/integrations/stream_tracker.rb +166 -0
- data/lib/llm_cost_tracker/llm_api_call.rb +2 -77
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +63 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +8 -4
- data/lib/llm_cost_tracker/parsers/gemini.rb +8 -1
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +12 -3
- data/lib/llm_cost_tracker/price_registry.rb +3 -0
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +41 -12
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +6 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +75 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +77 -0
- data/lib/llm_cost_tracker/pricing/lookup.rb +110 -0
- data/lib/llm_cost_tracker/pricing.rb +25 -108
- data/lib/llm_cost_tracker/report.rb +8 -1
- data/lib/llm_cost_tracker/report_data.rb +25 -9
- data/lib/llm_cost_tracker/retention.rb +33 -16
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +115 -0
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +42 -0
- data/lib/llm_cost_tracker/storage/active_record_store.rb +26 -0
- data/lib/llm_cost_tracker/storage/custom_backend.rb +32 -0
- data/lib/llm_cost_tracker/storage/dispatcher.rb +11 -34
- data/lib/llm_cost_tracker/storage/log_backend.rb +38 -0
- data/lib/llm_cost_tracker/storage/registry.rb +63 -0
- data/lib/llm_cost_tracker/stream_capture.rb +7 -0
- data/lib/llm_cost_tracker/stream_collector.rb +25 -1
- data/lib/llm_cost_tracker/tag_sanitizer.rb +81 -0
- data/lib/llm_cost_tracker/tag_sql.rb +34 -0
- data/lib/llm_cost_tracker/tracker.rb +6 -2
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +4 -0
- data/lib/tasks/llm_cost_tracker.rake +49 -0
- metadata +40 -6
|
@@ -54,8 +54,34 @@ module LlmCostTracker
|
|
|
54
54
|
ActiveRecordRollups.period_totals(periods, time: time)
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
def prune(cutoff:, batch_size:)
|
|
58
|
+
deleted = 0
|
|
59
|
+
loop do
|
|
60
|
+
batch = prune_batch(cutoff, batch_size)
|
|
61
|
+
deleted += batch
|
|
62
|
+
break if batch < batch_size
|
|
63
|
+
end
|
|
64
|
+
deleted
|
|
65
|
+
end
|
|
66
|
+
|
|
57
67
|
private
|
|
58
68
|
|
|
69
|
+
def prune_batch(cutoff, batch_size)
|
|
70
|
+
LlmCostTracker::LlmApiCall.transaction do
|
|
71
|
+
rows = LlmCostTracker::LlmApiCall
|
|
72
|
+
.where(tracked_at: ...cutoff)
|
|
73
|
+
.order(:id)
|
|
74
|
+
.limit(batch_size)
|
|
75
|
+
.lock
|
|
76
|
+
.pluck(:id, :tracked_at, :total_cost)
|
|
77
|
+
next 0 if rows.empty?
|
|
78
|
+
|
|
79
|
+
deleted = LlmCostTracker::LlmApiCall.where(id: rows.map(&:first)).delete_all
|
|
80
|
+
ActiveRecordRollups.decrement!(rows) if deleted.positive?
|
|
81
|
+
deleted
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
59
85
|
def stringify_tags(tags)
|
|
60
86
|
tags.transform_keys(&:to_s).transform_values { |value| stringify_tag_value(value) }
|
|
61
87
|
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "registry"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
module Storage
|
|
7
|
+
class CustomBackend
|
|
8
|
+
class << self
|
|
9
|
+
def save(event)
|
|
10
|
+
result = LlmCostTracker.configuration.custom_storage&.call(event)
|
|
11
|
+
result == false ? false : event
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def verify
|
|
15
|
+
if LlmCostTracker.configuration.custom_storage.respond_to?(:call)
|
|
16
|
+
return [
|
|
17
|
+
VerificationResult.new(
|
|
18
|
+
:ok,
|
|
19
|
+
"storage",
|
|
20
|
+
"custom storage callable configured; external sink was not invoked"
|
|
21
|
+
)
|
|
22
|
+
]
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
[
|
|
26
|
+
VerificationResult.new(:error, "storage", "custom storage backend requires config.custom_storage")
|
|
27
|
+
]
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "../logging"
|
|
4
|
+
require_relative "registry"
|
|
5
|
+
require_relative "active_record_backend"
|
|
6
|
+
require_relative "custom_backend"
|
|
7
|
+
require_relative "log_backend"
|
|
4
8
|
|
|
5
9
|
module LlmCostTracker
|
|
6
10
|
module Storage
|
|
7
11
|
class Dispatcher
|
|
8
12
|
class << self
|
|
9
13
|
def save(event)
|
|
10
|
-
|
|
11
|
-
case config.storage_backend
|
|
12
|
-
when :log then log_event(event, config)
|
|
13
|
-
when :active_record then active_record_save(event)
|
|
14
|
-
when :custom then custom_save(event, config)
|
|
15
|
-
end
|
|
14
|
+
backend.save(event)
|
|
16
15
|
rescue LlmCostTracker::BudgetExceededError, LlmCostTracker::UnknownPricingError
|
|
17
16
|
raise
|
|
18
17
|
rescue StandardError => e
|
|
@@ -22,34 +21,8 @@ module LlmCostTracker
|
|
|
22
21
|
|
|
23
22
|
private
|
|
24
23
|
|
|
25
|
-
def
|
|
26
|
-
|
|
27
|
-
"tokens=#{event.total_tokens} " \
|
|
28
|
-
"cost=#{log_cost_label(event)}"
|
|
29
|
-
message += " latency=#{event.latency_ms}ms" if event.latency_ms
|
|
30
|
-
message += " stream=#{event.stream}" if event.stream
|
|
31
|
-
message += " source=#{event.usage_source}" if event.usage_source
|
|
32
|
-
message += " tags=#{event.tags}" unless event.tags.empty?
|
|
33
|
-
|
|
34
|
-
Logging.log(config.log_level, message)
|
|
35
|
-
event
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def log_cost_label(event) = event.cost ? "$#{format('%.6f', event.cost.total_cost)}" : "unknown"
|
|
39
|
-
|
|
40
|
-
def active_record_save(event)
|
|
41
|
-
require_relative "../llm_api_call" unless defined?(LlmCostTracker::LlmApiCall)
|
|
42
|
-
require_relative "active_record_store" unless defined?(LlmCostTracker::Storage::ActiveRecordStore)
|
|
43
|
-
|
|
44
|
-
ActiveRecordStore.save(event)
|
|
45
|
-
event
|
|
46
|
-
rescue LoadError => e
|
|
47
|
-
raise Error, "ActiveRecord storage requires the active_record gem: #{e.message}"
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
def custom_save(event, config)
|
|
51
|
-
result = config.custom_storage&.call(event)
|
|
52
|
-
result == false ? false : event
|
|
24
|
+
def backend
|
|
25
|
+
Registry.fetch(LlmCostTracker.configuration.storage_backend)
|
|
53
26
|
end
|
|
54
27
|
|
|
55
28
|
def handle_error(error)
|
|
@@ -64,5 +37,9 @@ module LlmCostTracker
|
|
|
64
37
|
end
|
|
65
38
|
end
|
|
66
39
|
end
|
|
40
|
+
|
|
41
|
+
Registry.register(:log, LogBackend)
|
|
42
|
+
Registry.register(:active_record, ActiveRecordBackend)
|
|
43
|
+
Registry.register(:custom, CustomBackend)
|
|
67
44
|
end
|
|
68
45
|
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../logging"
|
|
4
|
+
require_relative "registry"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Storage
|
|
8
|
+
class LogBackend
|
|
9
|
+
class << self
|
|
10
|
+
def save(event)
|
|
11
|
+
config = LlmCostTracker.configuration
|
|
12
|
+
message = "#{event.provider}/#{event.model} " \
|
|
13
|
+
"tokens=#{event.total_tokens} " \
|
|
14
|
+
"cost=#{cost_label(event)}"
|
|
15
|
+
message += " latency=#{event.latency_ms}ms" if event.latency_ms
|
|
16
|
+
message += " stream=#{event.stream}" if event.stream
|
|
17
|
+
message += " source=#{event.usage_source}" if event.usage_source
|
|
18
|
+
message += " tags=#{event.tags}" unless event.tags.empty?
|
|
19
|
+
|
|
20
|
+
Logging.log(config.log_level, message)
|
|
21
|
+
event
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def verify
|
|
25
|
+
[
|
|
26
|
+
VerificationResult.new(:ok, "storage", "log backend configured; capture writes to logs only")
|
|
27
|
+
]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def cost_label(event)
|
|
33
|
+
event.cost ? "$#{format('%.6f', event.cost.total_cost)}" : "unknown"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "monitor"
|
|
4
|
+
|
|
5
|
+
require_relative "../errors"
|
|
6
|
+
|
|
7
|
+
module LlmCostTracker
|
|
8
|
+
module Storage
|
|
9
|
+
VerificationResult = Data.define(:status, :name, :message)
|
|
10
|
+
|
|
11
|
+
module Registry
|
|
12
|
+
MUTEX = Monitor.new
|
|
13
|
+
|
|
14
|
+
class << self
|
|
15
|
+
def register(name, backend)
|
|
16
|
+
name = normalize_name(name)
|
|
17
|
+
validate_backend!(backend)
|
|
18
|
+
MUTEX.synchronize { @backends = backends.merge(name => backend).freeze }
|
|
19
|
+
backend
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def fetch(name)
|
|
23
|
+
key = normalize_name(name)
|
|
24
|
+
backends.fetch(key) do
|
|
25
|
+
raise Error, "Unknown storage_backend: #{key.inspect}. Use one of: #{names.join(', ')}"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def registered?(name)
|
|
30
|
+
backends.key?(normalize_name(name))
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def names
|
|
34
|
+
backends.keys
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def backends
|
|
40
|
+
@backends || MUTEX.synchronize { @backends ||= {}.freeze }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def normalize_name(name)
|
|
44
|
+
name.to_sym
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def validate_backend!(backend)
|
|
48
|
+
return if backend.respond_to?(:save)
|
|
49
|
+
|
|
50
|
+
raise ArgumentError, "storage backend must respond to save"
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def self.register(name, backend)
|
|
56
|
+
Registry.register(name, backend)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def self.backends
|
|
60
|
+
Registry.names
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "json"
|
|
3
4
|
require "monitor"
|
|
4
5
|
|
|
6
|
+
require_relative "stream_capture"
|
|
5
7
|
require_relative "value_helpers"
|
|
6
8
|
|
|
7
9
|
module LlmCostTracker
|
|
@@ -16,6 +18,8 @@ module LlmCostTracker
|
|
|
16
18
|
@pricing_mode = pricing_mode
|
|
17
19
|
@metadata = ValueHelpers.deep_dup(metadata || {})
|
|
18
20
|
@events = []
|
|
21
|
+
@captured_bytes = 0
|
|
22
|
+
@overflowed = false
|
|
19
23
|
@explicit_usage = nil
|
|
20
24
|
@started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
21
25
|
@finished = false
|
|
@@ -45,7 +49,7 @@ module LlmCostTracker
|
|
|
45
49
|
def event(data, type: nil)
|
|
46
50
|
@monitor.synchronize do
|
|
47
51
|
ensure_open!
|
|
48
|
-
|
|
52
|
+
capture_event(data, type: type) unless data.nil?
|
|
49
53
|
end
|
|
50
54
|
self
|
|
51
55
|
end
|
|
@@ -71,6 +75,7 @@ module LlmCostTracker
|
|
|
71
75
|
@finished = true
|
|
72
76
|
{
|
|
73
77
|
events: @events.dup,
|
|
78
|
+
overflowed: @overflowed,
|
|
74
79
|
explicit_usage: ValueHelpers.deep_dup(@explicit_usage),
|
|
75
80
|
model: @model,
|
|
76
81
|
latency_ms: @latency_ms,
|
|
@@ -105,6 +110,7 @@ module LlmCostTracker
|
|
|
105
110
|
|
|
106
111
|
def build_parsed_usage(snapshot)
|
|
107
112
|
return build_from_explicit_usage(snapshot) if snapshot[:explicit_usage]
|
|
113
|
+
return build_unknown_usage(snapshot) if snapshot[:overflowed]
|
|
108
114
|
|
|
109
115
|
parsed = Parsers::Registry.find_for_provider(@provider)&.parse_stream(nil, nil, 200, snapshot[:events])
|
|
110
116
|
return finalize(parsed, snapshot) if parsed
|
|
@@ -157,6 +163,24 @@ module LlmCostTracker
|
|
|
157
163
|
)
|
|
158
164
|
end
|
|
159
165
|
|
|
166
|
+
def capture_event(data, type:)
|
|
167
|
+
copied = ValueHelpers.deep_dup(data)
|
|
168
|
+
size = event_bytes(copied, type)
|
|
169
|
+
if @captured_bytes + size <= StreamCapture::LIMIT_BYTES
|
|
170
|
+
@events << { event: type, data: copied }
|
|
171
|
+
@captured_bytes += size
|
|
172
|
+
else
|
|
173
|
+
@overflowed = true
|
|
174
|
+
@events.clear
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def event_bytes(data, type)
|
|
179
|
+
JSON.generate(event: type, data: data).bytesize
|
|
180
|
+
rescue JSON::GeneratorError, TypeError
|
|
181
|
+
type.to_s.bytesize + data.to_s.bytesize
|
|
182
|
+
end
|
|
183
|
+
|
|
160
184
|
def error_metadata(errored) = errored ? { stream_errored: true } : {}
|
|
161
185
|
|
|
162
186
|
def elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at) * 1000).round
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
module TagSanitizer
|
|
7
|
+
REDACTED_VALUE = "[REDACTED]"
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def call(tags, config: LlmCostTracker.configuration)
|
|
11
|
+
tags = (tags || {}).to_h
|
|
12
|
+
tags.first(max_tag_count(config)).each_with_object({}) do |(key, value), sanitized|
|
|
13
|
+
sanitized[key] = sanitized_value(key, value, config)
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def sanitized_value(key, value, config)
|
|
20
|
+
return REDACTED_VALUE if redacted_key?(key, config)
|
|
21
|
+
|
|
22
|
+
string = value_string(value)
|
|
23
|
+
return value if string.bytesize <= max_tag_value_bytesize(config)
|
|
24
|
+
|
|
25
|
+
truncate_bytes(string, max_tag_value_bytesize(config))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def redacted_key?(key, config)
|
|
29
|
+
normalized = normalized_key(key)
|
|
30
|
+
redacted_keys(config).any? do |candidate|
|
|
31
|
+
redacted_key_component?(normalized, candidate)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def redacted_keys(config)
|
|
36
|
+
Array(config.redacted_tag_keys).map { |key| normalized_key(key) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def normalized_key(key)
|
|
40
|
+
key.to_s
|
|
41
|
+
.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
|
|
42
|
+
.gsub(/([a-z\d])([A-Z])/, '\1_\2')
|
|
43
|
+
.downcase
|
|
44
|
+
.gsub(/[^a-z0-9]+/, "_")
|
|
45
|
+
.gsub(/_+/, "_")
|
|
46
|
+
.delete_prefix("_")
|
|
47
|
+
.delete_suffix("_")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def redacted_key_component?(key, candidate)
|
|
51
|
+
key == candidate ||
|
|
52
|
+
key.start_with?("#{candidate}_") ||
|
|
53
|
+
key.end_with?("_#{candidate}") ||
|
|
54
|
+
key.include?("_#{candidate}_")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def value_string(value)
|
|
58
|
+
case value
|
|
59
|
+
when Hash, Array
|
|
60
|
+
JSON.generate(value)
|
|
61
|
+
else
|
|
62
|
+
value.to_s
|
|
63
|
+
end
|
|
64
|
+
rescue JSON::GeneratorError, TypeError
|
|
65
|
+
value.to_s
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def truncate_bytes(string, limit)
|
|
69
|
+
string.byteslice(0, limit).to_s.encode("UTF-8", invalid: :replace, undef: :replace)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def max_tag_count(config)
|
|
73
|
+
[config.max_tag_count.to_i, 0].max
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def max_tag_value_bytesize(config)
|
|
77
|
+
[config.max_tag_value_bytesize.to_i, 0].max
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "tag_key"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
module TagSql
|
|
7
|
+
class << self
|
|
8
|
+
def value_expression(model, key, table_name:)
|
|
9
|
+
key = TagKey.validate!(key)
|
|
10
|
+
column = "#{table_name}.#{model.connection.quote_column_name('tags')}"
|
|
11
|
+
|
|
12
|
+
case model.connection.adapter_name
|
|
13
|
+
when /postgres/i
|
|
14
|
+
json_column = model.tags_jsonb_column? ? column : "(#{column})::jsonb"
|
|
15
|
+
"#{json_column}->>#{model.connection.quote(key)}"
|
|
16
|
+
when /mysql/i
|
|
17
|
+
"JSON_UNQUOTE(JSON_EXTRACT(#{column}, #{model.connection.quote(json_path(key))}))"
|
|
18
|
+
else
|
|
19
|
+
"json_extract(#{column}, #{model.connection.quote(json_path(key))})"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def value_label(value)
|
|
24
|
+
value.nil? || value == "" ? "(untagged)" : value.to_s
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def json_path(key)
|
|
30
|
+
"$.\"#{key}\""
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -6,7 +6,7 @@ module LlmCostTracker
|
|
|
6
6
|
class Tracker
|
|
7
7
|
EVENT_NAME = "llm_request.llm_cost_tracker"
|
|
8
8
|
|
|
9
|
-
USAGE_SOURCES = %i[response stream_final sdk_response manual unknown].freeze
|
|
9
|
+
USAGE_SOURCES = %i[response stream_final sdk_response ruby_llm manual unknown].freeze
|
|
10
10
|
|
|
11
11
|
class << self
|
|
12
12
|
def enforce_budget!
|
|
@@ -84,7 +84,7 @@ module LlmCostTracker
|
|
|
84
84
|
hidden_output_tokens: usage[:hidden_output_tokens],
|
|
85
85
|
pricing_mode: usage[:pricing_mode],
|
|
86
86
|
cost: cost_data,
|
|
87
|
-
tags:
|
|
87
|
+
tags: sanitized_tags(metadata).freeze,
|
|
88
88
|
latency_ms: normalized_latency_ms(latency_ms),
|
|
89
89
|
stream: stream ? true : false,
|
|
90
90
|
usage_source: normalized_usage_source(usage_source),
|
|
@@ -95,6 +95,10 @@ module LlmCostTracker
|
|
|
95
95
|
|
|
96
96
|
def normalized_latency_ms(latency_ms) = latency_ms.nil? ? nil : [latency_ms.to_i, 0].max
|
|
97
97
|
|
|
98
|
+
def sanitized_tags(metadata)
|
|
99
|
+
LlmCostTracker::TagSanitizer.call(LlmCostTracker::TagContext.tags.merge(EventMetadata.tags(metadata)))
|
|
100
|
+
end
|
|
101
|
+
|
|
98
102
|
def normalized_usage_source(value)
|
|
99
103
|
return nil if value.nil?
|
|
100
104
|
|
data/lib/llm_cost_tracker.rb
CHANGED
|
@@ -30,16 +30,20 @@ require_relative "llm_cost_tracker/budget"
|
|
|
30
30
|
require_relative "llm_cost_tracker/unknown_pricing"
|
|
31
31
|
require_relative "llm_cost_tracker/event_metadata"
|
|
32
32
|
require_relative "llm_cost_tracker/tag_context"
|
|
33
|
+
require_relative "llm_cost_tracker/tag_sanitizer"
|
|
33
34
|
require_relative "llm_cost_tracker/tags_column"
|
|
34
35
|
require_relative "llm_cost_tracker/tag_key"
|
|
36
|
+
require_relative "llm_cost_tracker/tag_sql"
|
|
35
37
|
require_relative "llm_cost_tracker/tag_query"
|
|
36
38
|
require_relative "llm_cost_tracker/tag_accessors"
|
|
39
|
+
require_relative "llm_cost_tracker/llm_api_call_metrics"
|
|
37
40
|
require_relative "llm_cost_tracker/tracker"
|
|
38
41
|
require_relative "llm_cost_tracker/retention"
|
|
39
42
|
require_relative "llm_cost_tracker/report_data"
|
|
40
43
|
require_relative "llm_cost_tracker/report_formatter"
|
|
41
44
|
require_relative "llm_cost_tracker/report"
|
|
42
45
|
require_relative "llm_cost_tracker/doctor"
|
|
46
|
+
require_relative "llm_cost_tracker/capture_verifier"
|
|
43
47
|
|
|
44
48
|
module LlmCostTracker
|
|
45
49
|
CONFIGURATION_MUTEX = Monitor.new
|
|
@@ -7,11 +7,21 @@ namespace :llm_cost_tracker do
|
|
|
7
7
|
desc "Check LLM Cost Tracker setup"
|
|
8
8
|
task :doctor do
|
|
9
9
|
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
10
|
+
require_relative "../llm_cost_tracker"
|
|
10
11
|
checks = LlmCostTracker::Doctor.call
|
|
11
12
|
puts LlmCostTracker::Doctor.report(checks)
|
|
12
13
|
abort("llm_cost_tracker: doctor found setup errors") unless LlmCostTracker::Doctor.healthy?(checks)
|
|
13
14
|
end
|
|
14
15
|
|
|
16
|
+
desc "Verify that LLM Cost Tracker can capture and persist a synthetic event"
|
|
17
|
+
task :verify_capture do
|
|
18
|
+
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
19
|
+
require_relative "../llm_cost_tracker"
|
|
20
|
+
checks = LlmCostTracker::CaptureVerifier.call
|
|
21
|
+
puts LlmCostTracker::CaptureVerifier.report(checks)
|
|
22
|
+
abort("llm_cost_tracker: capture verification failed") unless LlmCostTracker::CaptureVerifier.healthy?(checks)
|
|
23
|
+
end
|
|
24
|
+
|
|
15
25
|
desc "Print an LLM cost report from ActiveRecord storage"
|
|
16
26
|
task report: :environment do
|
|
17
27
|
days = (ENV["DAYS"] || LlmCostTracker::Report::DEFAULT_DAYS).to_i
|
|
@@ -74,6 +84,17 @@ namespace :llm_cost_tracker do
|
|
|
74
84
|
puts " pricing is up to date" if result.up_to_date
|
|
75
85
|
abort("llm_cost_tracker: pricing check failed") unless result.up_to_date
|
|
76
86
|
end
|
|
87
|
+
|
|
88
|
+
desc "Explain how a provider/model price is matched. Use PROVIDER=... MODEL=..."
|
|
89
|
+
task :explain do
|
|
90
|
+
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
91
|
+
require_relative "../llm_cost_tracker"
|
|
92
|
+
|
|
93
|
+
explanation = price_explanation_from_env
|
|
94
|
+
puts "llm_cost_tracker: #{explanation.message}"
|
|
95
|
+
print_price_explanation(explanation)
|
|
96
|
+
abort("llm_cost_tracker: price is incomplete or unknown") unless explanation.complete?
|
|
97
|
+
end
|
|
77
98
|
end
|
|
78
99
|
end
|
|
79
100
|
# rubocop:enable Metrics/BlockLength
|
|
@@ -95,3 +116,31 @@ def price_refresh_output_path
|
|
|
95
116
|
FileUtils.mkdir_p(File.dirname(path))
|
|
96
117
|
path
|
|
97
118
|
end
|
|
119
|
+
|
|
120
|
+
def price_explanation_from_env
|
|
121
|
+
provider = ENV["PROVIDER"].to_s.strip
|
|
122
|
+
model = ENV["MODEL"].to_s.strip
|
|
123
|
+
abort("llm_cost_tracker: use PROVIDER=... MODEL=...") if provider.empty? || model.empty?
|
|
124
|
+
|
|
125
|
+
LlmCostTracker::Pricing.explain(
|
|
126
|
+
provider: provider,
|
|
127
|
+
model: model,
|
|
128
|
+
pricing_mode: ENV.fetch("PRICING_MODE", nil),
|
|
129
|
+
input_tokens: ENV.fetch("INPUT_TOKENS", 1).to_i,
|
|
130
|
+
output_tokens: ENV.fetch("OUTPUT_TOKENS", 1).to_i,
|
|
131
|
+
cache_read_input_tokens: ENV.fetch("CACHE_READ_INPUT_TOKENS", 0).to_i,
|
|
132
|
+
cache_write_input_tokens: ENV.fetch("CACHE_WRITE_INPUT_TOKENS", 0).to_i
|
|
133
|
+
)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def print_price_explanation(explanation)
|
|
137
|
+
return unless explanation.matched?
|
|
138
|
+
|
|
139
|
+
puts " source: #{explanation.source}"
|
|
140
|
+
puts " matched_key: #{explanation.matched_key}"
|
|
141
|
+
puts " matched_by: #{explanation.matched_by}"
|
|
142
|
+
puts " pricing_mode: #{explanation.pricing_mode || 'standard'}"
|
|
143
|
+
explanation.effective_prices.each do |key, value|
|
|
144
|
+
puts " #{key}: #{value.nil? ? 'missing' : value}"
|
|
145
|
+
end
|
|
146
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm_cost_tracker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sergii Khomenko
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-04-
|
|
11
|
+
date: 2026-04-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -222,10 +222,10 @@ dependencies:
|
|
|
222
222
|
- - "~>"
|
|
223
223
|
- !ruby/object:Gem::Version
|
|
224
224
|
version: '3.0'
|
|
225
|
-
description: Tracks token usage, latency, and estimated costs for
|
|
226
|
-
Google Gemini, OpenRouter, DeepSeek, and OpenAI-compatible APIs. Works
|
|
227
|
-
middleware or explicit track/track_stream helpers, with ActiveRecord
|
|
228
|
-
attribution, price sync tasks, and budget guardrails.
|
|
225
|
+
description: Tracks token usage, latency, and estimated costs for RubyLLM, OpenAI,
|
|
226
|
+
Anthropic, Google Gemini, OpenRouter, DeepSeek, and OpenAI-compatible APIs. Works
|
|
227
|
+
through Faraday middleware or explicit track/track_stream helpers, with ActiveRecord
|
|
228
|
+
storage, tag-based attribution, price sync tasks, and budget guardrails.
|
|
229
229
|
email:
|
|
230
230
|
- sergey@mm.st
|
|
231
231
|
executables: []
|
|
@@ -255,6 +255,7 @@ files:
|
|
|
255
255
|
- app/helpers/llm_cost_tracker/pagination_helper.rb
|
|
256
256
|
- app/services/llm_cost_tracker/dashboard/data_quality.rb
|
|
257
257
|
- app/services/llm_cost_tracker/dashboard/data_quality_aggregate.rb
|
|
258
|
+
- app/services/llm_cost_tracker/dashboard/date_range.rb
|
|
258
259
|
- app/services/llm_cost_tracker/dashboard/filter.rb
|
|
259
260
|
- app/services/llm_cost_tracker/dashboard/overview_stats.rb
|
|
260
261
|
- app/services/llm_cost_tracker/dashboard/provider_breakdown.rb
|
|
@@ -282,13 +283,33 @@ files:
|
|
|
282
283
|
- app/views/llm_cost_tracker/tags/index.html.erb
|
|
283
284
|
- app/views/llm_cost_tracker/tags/show.html.erb
|
|
284
285
|
- config/routes.rb
|
|
286
|
+
- docs/architecture.md
|
|
287
|
+
- docs/budgets.md
|
|
288
|
+
- docs/configuration.md
|
|
289
|
+
- docs/cookbook.md
|
|
290
|
+
- docs/dashboard-overview.png
|
|
291
|
+
- docs/dashboard.md
|
|
292
|
+
- docs/extending.md
|
|
293
|
+
- docs/operations.md
|
|
294
|
+
- docs/pricing.md
|
|
295
|
+
- docs/querying.md
|
|
296
|
+
- docs/streaming.md
|
|
297
|
+
- docs/technical/README.md
|
|
298
|
+
- docs/technical/data-flow.md
|
|
299
|
+
- docs/technical/extension-points.md
|
|
300
|
+
- docs/technical/module-map.md
|
|
301
|
+
- docs/technical/operational-notes.md
|
|
302
|
+
- docs/upgrading.md
|
|
285
303
|
- lib/llm_cost_tracker.rb
|
|
286
304
|
- lib/llm_cost_tracker/assets.rb
|
|
287
305
|
- lib/llm_cost_tracker/budget.rb
|
|
306
|
+
- lib/llm_cost_tracker/capture_verifier.rb
|
|
288
307
|
- lib/llm_cost_tracker/configuration.rb
|
|
289
308
|
- lib/llm_cost_tracker/configuration/instrumentation.rb
|
|
309
|
+
- lib/llm_cost_tracker/configuration/storage_backend.rb
|
|
290
310
|
- lib/llm_cost_tracker/cost.rb
|
|
291
311
|
- lib/llm_cost_tracker/doctor.rb
|
|
312
|
+
- lib/llm_cost_tracker/doctor/capture_check.rb
|
|
292
313
|
- lib/llm_cost_tracker/engine.rb
|
|
293
314
|
- lib/llm_cost_tracker/engine_compatibility.rb
|
|
294
315
|
- lib/llm_cost_tracker/errors.rb
|
|
@@ -317,7 +338,10 @@ files:
|
|
|
317
338
|
- lib/llm_cost_tracker/integrations/object_reader.rb
|
|
318
339
|
- lib/llm_cost_tracker/integrations/openai.rb
|
|
319
340
|
- lib/llm_cost_tracker/integrations/registry.rb
|
|
341
|
+
- lib/llm_cost_tracker/integrations/ruby_llm.rb
|
|
342
|
+
- lib/llm_cost_tracker/integrations/stream_tracker.rb
|
|
320
343
|
- lib/llm_cost_tracker/llm_api_call.rb
|
|
344
|
+
- lib/llm_cost_tracker/llm_api_call_metrics.rb
|
|
321
345
|
- lib/llm_cost_tracker/logging.rb
|
|
322
346
|
- lib/llm_cost_tracker/middleware/faraday.rb
|
|
323
347
|
- lib/llm_cost_tracker/parameter_hash.rb
|
|
@@ -341,20 +365,30 @@ files:
|
|
|
341
365
|
- lib/llm_cost_tracker/price_sync/registry_writer.rb
|
|
342
366
|
- lib/llm_cost_tracker/prices.json
|
|
343
367
|
- lib/llm_cost_tracker/pricing.rb
|
|
368
|
+
- lib/llm_cost_tracker/pricing/effective_prices.rb
|
|
369
|
+
- lib/llm_cost_tracker/pricing/explainer.rb
|
|
370
|
+
- lib/llm_cost_tracker/pricing/lookup.rb
|
|
344
371
|
- lib/llm_cost_tracker/railtie.rb
|
|
345
372
|
- lib/llm_cost_tracker/report.rb
|
|
346
373
|
- lib/llm_cost_tracker/report_data.rb
|
|
347
374
|
- lib/llm_cost_tracker/report_formatter.rb
|
|
348
375
|
- lib/llm_cost_tracker/request_url.rb
|
|
349
376
|
- lib/llm_cost_tracker/retention.rb
|
|
377
|
+
- lib/llm_cost_tracker/storage/active_record_backend.rb
|
|
350
378
|
- lib/llm_cost_tracker/storage/active_record_rollups.rb
|
|
351
379
|
- lib/llm_cost_tracker/storage/active_record_store.rb
|
|
380
|
+
- lib/llm_cost_tracker/storage/custom_backend.rb
|
|
352
381
|
- lib/llm_cost_tracker/storage/dispatcher.rb
|
|
382
|
+
- lib/llm_cost_tracker/storage/log_backend.rb
|
|
383
|
+
- lib/llm_cost_tracker/storage/registry.rb
|
|
384
|
+
- lib/llm_cost_tracker/stream_capture.rb
|
|
353
385
|
- lib/llm_cost_tracker/stream_collector.rb
|
|
354
386
|
- lib/llm_cost_tracker/tag_accessors.rb
|
|
355
387
|
- lib/llm_cost_tracker/tag_context.rb
|
|
356
388
|
- lib/llm_cost_tracker/tag_key.rb
|
|
357
389
|
- lib/llm_cost_tracker/tag_query.rb
|
|
390
|
+
- lib/llm_cost_tracker/tag_sanitizer.rb
|
|
391
|
+
- lib/llm_cost_tracker/tag_sql.rb
|
|
358
392
|
- lib/llm_cost_tracker/tags_column.rb
|
|
359
393
|
- lib/llm_cost_tracker/tracker.rb
|
|
360
394
|
- lib/llm_cost_tracker/unknown_pricing.rb
|