llm_cost_tracker 0.14.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +1 -1
- data/app/controllers/llm_cost_tracker/application_controller.rb +16 -3
- data/app/controllers/llm_cost_tracker/pricing_controller.rb +2 -2
- data/app/controllers/llm_cost_tracker/tags_controller.rb +5 -4
- data/app/helpers/llm_cost_tracker/application_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +5 -0
- data/app/models/llm_cost_tracker/call.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/filter.rb +24 -21
- data/app/services/llm_cost_tracker/dashboard/params.rb +10 -0
- data/app/views/llm_cost_tracker/tags/show.html.erb +3 -0
- data/config/routes.rb +6 -1
- data/lib/llm_cost_tracker/budget/per_tag.rb +8 -4
- data/lib/llm_cost_tracker/capture/event_window.rb +99 -0
- data/lib/llm_cost_tracker/capture/sse.rb +108 -32
- data/lib/llm_cost_tracker/capture/stream_collector.rb +17 -56
- data/lib/llm_cost_tracker/capture/stream_tap.rb +57 -0
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +5 -35
- data/lib/llm_cost_tracker/doctor/price_check.rb +1 -0
- data/lib/llm_cost_tracker/errors.rb +13 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +6 -6
- data/lib/llm_cost_tracker/ingestion/batch.rb +37 -8
- data/lib/llm_cost_tracker/integrations/anthropic.rb +7 -1
- data/lib/llm_cost_tracker/integrations/base.rb +17 -1
- data/lib/llm_cost_tracker/integrations/openai/batch_capture.rb +6 -1
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +53 -18
- data/lib/llm_cost_tracker/ledger/isolation.rb +30 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +2 -1
- data/lib/llm_cost_tracker/ledger/rollups.rb +12 -4
- data/lib/llm_cost_tracker/ledger/storable.rb +16 -0
- data/lib/llm_cost_tracker/ledger/store.rb +12 -11
- data/lib/llm_cost_tracker/ledger.rb +1 -0
- data/lib/llm_cost_tracker/logging.rb +3 -1
- data/lib/llm_cost_tracker/middleware/faraday.rb +32 -43
- data/lib/llm_cost_tracker/parsers.rb +5 -1
- data/lib/llm_cost_tracker/prices.json +878 -139
- data/lib/llm_cost_tracker/pricing/registry.rb +12 -1
- data/lib/llm_cost_tracker/pricing/sync/change_printer.rb +6 -1
- data/lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb +47 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +30 -13
- data/lib/llm_cost_tracker/providers/azure/parser.rb +19 -0
- data/lib/llm_cost_tracker/providers/gemini/parser.rb +4 -0
- data/lib/llm_cost_tracker/providers/openai/response_parser.rb +5 -1
- data/lib/llm_cost_tracker/providers/openai/usage_extractor.rb +10 -5
- data/lib/llm_cost_tracker/providers/openai_compatible/parser.rb +4 -0
- data/lib/llm_cost_tracker/redaction.rb +32 -0
- data/lib/llm_cost_tracker/tags/sanitizer.rb +10 -36
- data/lib/llm_cost_tracker/tracker.rb +3 -4
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +4 -2
- data/lib/tasks/llm_cost_tracker.rake +6 -4
- metadata +16 -4
|
@@ -33,6 +33,12 @@ module LlmCostTracker
|
|
|
33
33
|
@prices_file_mtime_iso = nil
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
+
def validate_file!(path)
|
|
37
|
+
file_prices(path)
|
|
38
|
+
file_rates(path)
|
|
39
|
+
file_metadata(path)
|
|
40
|
+
end
|
|
41
|
+
|
|
36
42
|
def builtin_prices
|
|
37
43
|
@builtin_prices ||= normalize_price_entries(
|
|
38
44
|
raw_registry.fetch("models", {}), context: "bundled prices"
|
|
@@ -157,10 +163,15 @@ module LlmCostTracker
|
|
|
157
163
|
def loading(path)
|
|
158
164
|
yield
|
|
159
165
|
rescue Errno::ENOENT, Psych::Exception, ArgumentError, TypeError => e
|
|
160
|
-
raise Error, "Unable to load prices_file #{path.inspect}: #{e.message}"
|
|
166
|
+
raise Error, "Unable to load prices_file #{path.inspect}: #{e.message}; fix or delete it"
|
|
161
167
|
end
|
|
162
168
|
|
|
163
169
|
def load_raw_file_registry(path)
|
|
170
|
+
unless File.exist?(path)
|
|
171
|
+
Logging.warn("pricing.file #{path} does not exist; using bundled prices until prices:refresh creates it")
|
|
172
|
+
return {}.freeze
|
|
173
|
+
end
|
|
174
|
+
|
|
164
175
|
loading(path) { (YAML.safe_load_file(path, aliases: false) || {}).freeze }
|
|
165
176
|
end
|
|
166
177
|
|
|
@@ -4,7 +4,12 @@ module LlmCostTracker
|
|
|
4
4
|
module Pricing
|
|
5
5
|
module Sync
|
|
6
6
|
module ChangePrinter
|
|
7
|
-
def self.call(changes, output: $stdout)
|
|
7
|
+
def self.call(changes, suspicious: [], output: $stdout)
|
|
8
|
+
if suspicious.any?
|
|
9
|
+
output.puts " suspicious changes (refresh writes them only with FORCE=1): #{suspicious.size}"
|
|
10
|
+
suspicious.each { |finding| output.puts " - #{finding}" }
|
|
11
|
+
end
|
|
12
|
+
|
|
8
13
|
service_changes = changes["service_charges"]
|
|
9
14
|
model_changes = changes.except("service_charges")
|
|
10
15
|
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../../currency"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
module Pricing
|
|
7
|
+
module Sync
|
|
8
|
+
module SnapshotGuard
|
|
9
|
+
class << self
|
|
10
|
+
def call(current:, remote:, changes:)
|
|
11
|
+
findings = changes.except("service_charges").flat_map do |model, fields|
|
|
12
|
+
next [] unless remote["models"].key?(model)
|
|
13
|
+
|
|
14
|
+
fields.map { |field, values| finding("#{model} #{field}", field, values) }
|
|
15
|
+
end
|
|
16
|
+
changes.fetch("service_charges", {}).each do |provider, components|
|
|
17
|
+
components.each { |component, values| findings << finding("#{provider}.#{component}", component, values) }
|
|
18
|
+
end
|
|
19
|
+
(findings << currency_finding(current, remote)).compact
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def finding(label, field, values)
|
|
25
|
+
from, to = values.values_at("from", "to").map { |value| value&.to_f }
|
|
26
|
+
"#{label}: #{from.inspect} -> #{to.inspect}" if suspicious?(field, from, to)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def suspicious?(field, from, to)
|
|
30
|
+
return %w[input output].include?(field) && from.positive? if to.nil?
|
|
31
|
+
return false if from.nil?
|
|
32
|
+
return true if from.zero? || to.zero?
|
|
33
|
+
|
|
34
|
+
[to / from, from / to].max >= 100
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def currency_finding(current, remote)
|
|
38
|
+
return if current.empty?
|
|
39
|
+
|
|
40
|
+
from, to = [current, remote].map { |registry| registry.dig("metadata", "currency") || DEFAULT_CURRENCY }
|
|
41
|
+
"currency: #{from} -> #{to}" unless from.upcase == to.upcase
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -9,6 +9,7 @@ require_relative "registry"
|
|
|
9
9
|
require_relative "sync/fetcher"
|
|
10
10
|
require_relative "sync/registry_diff"
|
|
11
11
|
require_relative "sync/registry_writer"
|
|
12
|
+
require_relative "sync/snapshot_guard"
|
|
12
13
|
|
|
13
14
|
module LlmCostTracker
|
|
14
15
|
module Pricing
|
|
@@ -18,8 +19,8 @@ module LlmCostTracker
|
|
|
18
19
|
"https://raw.githubusercontent.com/sergey-homenko/llm_cost_tracker/main/lib/llm_cost_tracker/prices.json"
|
|
19
20
|
SUPPORTED_SCHEMA_VERSION = 1
|
|
20
21
|
|
|
21
|
-
RefreshResult = Data.define(:path, :source_url, :source_version, :changes, :written, :not_modified)
|
|
22
|
-
CheckResult = Data.define(:path, :source_url, :source_version, :changes, :up_to_date)
|
|
22
|
+
RefreshResult = Data.define(:path, :source_url, :source_version, :changes, :suspicious, :written, :not_modified)
|
|
23
|
+
CheckResult = Data.define(:path, :source_url, :source_version, :changes, :suspicious, :up_to_date)
|
|
23
24
|
|
|
24
25
|
class << self
|
|
25
26
|
def configured_output_path(env: ENV, config: LlmCostTracker.configuration)
|
|
@@ -39,6 +40,7 @@ module LlmCostTracker
|
|
|
39
40
|
def refresh(path: DEFAULT_OUTPUT_PATH,
|
|
40
41
|
url: DEFAULT_REMOTE_URL,
|
|
41
42
|
preview: false,
|
|
43
|
+
force: false,
|
|
42
44
|
fetcher: Fetcher.new,
|
|
43
45
|
today: Date.today)
|
|
44
46
|
current = load_registry(path)
|
|
@@ -49,15 +51,18 @@ module LlmCostTracker
|
|
|
49
51
|
path: path,
|
|
50
52
|
url: url,
|
|
51
53
|
response: response,
|
|
52
|
-
|
|
53
|
-
|
|
54
|
+
changes: registry_changes(current, current),
|
|
55
|
+
suspicious: [],
|
|
54
56
|
written: false,
|
|
55
57
|
not_modified: true
|
|
56
58
|
)
|
|
57
59
|
end
|
|
58
60
|
|
|
59
61
|
remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
|
|
62
|
+
changes = registry_changes(current, remote)
|
|
63
|
+
suspicious = SnapshotGuard.call(current: current, remote: remote, changes: changes)
|
|
60
64
|
unless preview
|
|
65
|
+
refuse_suspicious_snapshot!(path, suspicious) unless force || suspicious.empty?
|
|
61
66
|
RegistryWriter.new.call(path: path, registry: remote)
|
|
62
67
|
Pricing::Registry.reset!
|
|
63
68
|
end
|
|
@@ -65,8 +70,8 @@ module LlmCostTracker
|
|
|
65
70
|
path: path,
|
|
66
71
|
url: url,
|
|
67
72
|
response: response,
|
|
68
|
-
|
|
69
|
-
|
|
73
|
+
changes: changes,
|
|
74
|
+
suspicious: suspicious,
|
|
70
75
|
written: !preview,
|
|
71
76
|
not_modified: false
|
|
72
77
|
)
|
|
@@ -79,22 +84,25 @@ module LlmCostTracker
|
|
|
79
84
|
if response.not_modified
|
|
80
85
|
return CheckResult.new(
|
|
81
86
|
path: path,
|
|
82
|
-
source_url: url,
|
|
87
|
+
source_url: Redaction.text(url),
|
|
83
88
|
source_version: response.source_version,
|
|
84
89
|
changes: {},
|
|
90
|
+
suspicious: [],
|
|
85
91
|
up_to_date: true
|
|
86
92
|
)
|
|
87
93
|
end
|
|
88
94
|
|
|
89
95
|
remote = normalize_remote_registry(response.body, url: url, response: response, today: today)
|
|
90
96
|
changes = registry_changes(current, remote)
|
|
97
|
+
suspicious = SnapshotGuard.call(current: current, remote: remote, changes: changes)
|
|
91
98
|
|
|
92
99
|
CheckResult.new(
|
|
93
100
|
path: path,
|
|
94
|
-
source_url: url,
|
|
101
|
+
source_url: Redaction.text(url),
|
|
95
102
|
source_version: response.source_version,
|
|
96
103
|
changes: changes,
|
|
97
|
-
|
|
104
|
+
suspicious: suspicious,
|
|
105
|
+
up_to_date: changes.empty? && suspicious.empty?
|
|
98
106
|
)
|
|
99
107
|
end
|
|
100
108
|
|
|
@@ -128,7 +136,7 @@ module LlmCostTracker
|
|
|
128
136
|
"metadata" => metadata.merge(
|
|
129
137
|
"schema_version" => schema_version,
|
|
130
138
|
"updated_at" => metadata["updated_at"] || today.iso8601,
|
|
131
|
-
"source_url" => url,
|
|
139
|
+
"source_url" => Redaction.text(url),
|
|
132
140
|
"source_version" => response.source_version
|
|
133
141
|
),
|
|
134
142
|
"models" => models
|
|
@@ -156,17 +164,26 @@ module LlmCostTracker
|
|
|
156
164
|
raise Error, "Unable to parse remote pricing snapshot: #{e.message}"
|
|
157
165
|
end
|
|
158
166
|
|
|
159
|
-
def refresh_result(path:, url:, response:,
|
|
167
|
+
def refresh_result(path:, url:, response:, changes:, suspicious:, written:, not_modified:)
|
|
160
168
|
RefreshResult.new(
|
|
161
169
|
path: path,
|
|
162
|
-
source_url: url,
|
|
170
|
+
source_url: Redaction.text(url),
|
|
163
171
|
source_version: response.source_version,
|
|
164
|
-
changes:
|
|
172
|
+
changes: changes,
|
|
173
|
+
suspicious: suspicious,
|
|
165
174
|
written: written,
|
|
166
175
|
not_modified: not_modified
|
|
167
176
|
)
|
|
168
177
|
end
|
|
169
178
|
|
|
179
|
+
def refuse_suspicious_snapshot!(path, suspicious)
|
|
180
|
+
listed = suspicious.first(20).map { |finding| "\n - #{finding}" }.join
|
|
181
|
+
raise Error,
|
|
182
|
+
"Refusing to write pricing file #{path}: the remote snapshot has #{suspicious.size} " \
|
|
183
|
+
"suspicious price change(s):#{listed}\n" \
|
|
184
|
+
"Review them with PREVIEW=1, then refresh with FORCE=1 (force: true) to accept them."
|
|
185
|
+
end
|
|
186
|
+
|
|
170
187
|
def registry_changes(current, remote)
|
|
171
188
|
model_changes = RegistryDiff.call(current.fetch("models", {}), remote.fetch("models", {}))
|
|
172
189
|
charge_changes = service_charges_diff(
|
|
@@ -38,6 +38,25 @@ module LlmCostTracker
|
|
|
38
38
|
match = uri&.path&.match(%r{/openai/deployments/([^/]+)/})
|
|
39
39
|
match && match[1]
|
|
40
40
|
end
|
|
41
|
+
|
|
42
|
+
def auto_enable_stream_usage?(request_url, request_parsed)
|
|
43
|
+
super && stream_options_api?(parsed_uri(request_url)) &&
|
|
44
|
+
!request_parsed.key?("data_sources") && !image_input?(request_parsed)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def stream_options_api?(uri)
|
|
50
|
+
uri.path.start_with?("/openai/v1/") ||
|
|
51
|
+
uri.query.to_s[/api-version=(\d{4}-\d{2}-\d{2})/, 1].to_s >= "2024-06-01"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def image_input?(request)
|
|
55
|
+
Array(request["messages"]).any? do |message|
|
|
56
|
+
message.is_a?(Hash) &&
|
|
57
|
+
Array(message["content"]).any? { |part| part.is_a?(Hash) && part["type"] == "image_url" }
|
|
58
|
+
end
|
|
59
|
+
end
|
|
41
60
|
end
|
|
42
61
|
end
|
|
43
62
|
end
|
|
@@ -70,10 +70,14 @@ module LlmCostTracker
|
|
|
70
70
|
build_unknown_stream_usage(**context)
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
-
def auto_enable_stream_usage?(request_url)
|
|
73
|
+
def auto_enable_stream_usage?(request_url, _request_parsed)
|
|
74
74
|
openai_chat_completions_url?(request_url)
|
|
75
75
|
end
|
|
76
76
|
|
|
77
|
+
def retain_stream_event?(data)
|
|
78
|
+
data.is_a?(Hash) && (data["item"].is_a?(Hash) || data["response"].is_a?(Hash))
|
|
79
|
+
end
|
|
80
|
+
|
|
77
81
|
private
|
|
78
82
|
|
|
79
83
|
def stream_capture_context(events:, request:, request_url:)
|
|
@@ -13,9 +13,10 @@ module LlmCostTracker
|
|
|
13
13
|
output_tokens = (usage[:output_tokens] || usage[:completion_tokens]).to_i
|
|
14
14
|
cache_read = cache_read_input_tokens(usage)
|
|
15
15
|
cache_write = cache_write_input_tokens(usage)
|
|
16
|
-
|
|
16
|
+
uncached_input = [input_tokens - cache_read - cache_write, 0].max
|
|
17
|
+
image_input = uncached_input_tokens(usage, :image_tokens).clamp(0, uncached_input)
|
|
18
|
+
audio_input = uncached_input_tokens(usage, :audio_tokens).clamp(0, uncached_input - image_input)
|
|
17
19
|
audio_output = audio_output_tokens(usage)
|
|
18
|
-
image_input = image_input_tokens(usage)
|
|
19
20
|
image_output, regular_output = split_output(
|
|
20
21
|
output_tokens: output_tokens,
|
|
21
22
|
image_output_details: image_output_tokens(usage),
|
|
@@ -25,7 +26,7 @@ module LlmCostTracker
|
|
|
25
26
|
)
|
|
26
27
|
|
|
27
28
|
Usage::TokenUsage.build(
|
|
28
|
-
input_tokens:
|
|
29
|
+
input_tokens: uncached_input - audio_input - image_input,
|
|
29
30
|
output_tokens: regular_output,
|
|
30
31
|
total_tokens: usage[:total_tokens],
|
|
31
32
|
cache_read_input_tokens: cache_read,
|
|
@@ -60,9 +61,13 @@ module LlmCostTracker
|
|
|
60
61
|
def self.image_output_tokens(usage) = detail(usage, OUTPUT_DETAIL_KEYS, :image_tokens)
|
|
61
62
|
def self.text_output_tokens(usage) = detail(usage, OUTPUT_DETAIL_KEYS, :text_tokens)
|
|
62
63
|
|
|
63
|
-
def self.
|
|
64
|
+
def self.uncached_input_tokens(usage, key)
|
|
65
|
+
detail(usage, INPUT_DETAIL_KEYS, key) - detail(usage, INPUT_DETAIL_KEYS, :cached_tokens_details, key)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def self.detail(usage, containers, *path)
|
|
64
69
|
containers.each do |container|
|
|
65
|
-
value = usage.dig(container,
|
|
70
|
+
value = usage.dig(container, *path)
|
|
66
71
|
return value.to_i if value
|
|
67
72
|
end
|
|
68
73
|
0
|
|
@@ -30,6 +30,10 @@ module LlmCostTracker
|
|
|
30
30
|
def provider_for(request_url)
|
|
31
31
|
self.class.provider_for_uri(parsed_uri(request_url)) || "openai_compatible"
|
|
32
32
|
end
|
|
33
|
+
|
|
34
|
+
def auto_enable_stream_usage?(request_url, request_parsed)
|
|
35
|
+
super && Configuration::Capture::OPENAI_COMPATIBLE_PROVIDERS.key?(parsed_uri(request_url).host.to_s.downcase)
|
|
36
|
+
end
|
|
33
37
|
end
|
|
34
38
|
end
|
|
35
39
|
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module Redaction
|
|
5
|
+
REDACTED = "[REDACTED]"
|
|
6
|
+
|
|
7
|
+
TOKEN = Regexp.union(
|
|
8
|
+
/sk-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}/,
|
|
9
|
+
/AKIA[0-9A-Z]{16}/,
|
|
10
|
+
/gh[opsur]_[A-Za-z0-9]{16,}/,
|
|
11
|
+
/github_pat_[A-Za-z0-9_]{20,}/,
|
|
12
|
+
/eyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/i,
|
|
13
|
+
/bearer\s+[A-Za-z0-9_.-]{20,}/i,
|
|
14
|
+
/xox[abprs]-[A-Za-z0-9-]{10,}/,
|
|
15
|
+
/(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}/,
|
|
16
|
+
/AIza[0-9A-Za-z_-]{35}/
|
|
17
|
+
)
|
|
18
|
+
EMBEDDED_TOKEN = /(?<![A-Za-z0-9_-])(?:#{TOKEN})(?![A-Za-z0-9_-])/
|
|
19
|
+
PARAM = /(?<![A-Za-z0-9])((?:api)?key|token|secret|password|sig(?:nature)?)=[^&\s"']+/i
|
|
20
|
+
HEADER = /((?:authorization|api-key)["']?\s*(?:=>|:)\s*["']?)(?:[A-Za-z]+\s+)?[^\s"',;]+/i
|
|
21
|
+
USERINFO = %r{://[^/\s@?#]+@}
|
|
22
|
+
private_constant :TOKEN, :EMBEDDED_TOKEN, :PARAM, :HEADER, :USERINFO
|
|
23
|
+
|
|
24
|
+
def self.text(value)
|
|
25
|
+
value.to_s.scrub
|
|
26
|
+
.gsub(USERINFO, "://#{REDACTED}@")
|
|
27
|
+
.gsub(HEADER, "\\1#{REDACTED}")
|
|
28
|
+
.gsub(PARAM, "\\1=#{REDACTED}")
|
|
29
|
+
.gsub(EMBEDDED_TOKEN, REDACTED)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -2,23 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
require "active_support/core_ext/string/inflections"
|
|
4
4
|
|
|
5
|
+
require_relative "../ledger/storable"
|
|
6
|
+
require_relative "../redaction"
|
|
7
|
+
|
|
5
8
|
module LlmCostTracker
|
|
6
9
|
module Tags
|
|
7
10
|
module Sanitizer
|
|
8
|
-
REDACTED_VALUE =
|
|
9
|
-
|
|
10
|
-
SECRET_VALUE_PATTERNS = [
|
|
11
|
-
/\Ask-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}\z/,
|
|
12
|
-
/\AAKIA[0-9A-Z]{16}\z/,
|
|
13
|
-
/\Agh[opsur]_[A-Za-z0-9]{16,}\z/,
|
|
14
|
-
/\Agithub_pat_[A-Za-z0-9_]{20,}\z/,
|
|
15
|
-
/\Aeyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\z/i,
|
|
16
|
-
/\Abearer\s+[A-Za-z0-9_.-]{20,}\z/i,
|
|
17
|
-
/\Axox[abprs]-[A-Za-z0-9-]{10,}\z/,
|
|
18
|
-
/\A(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\z/,
|
|
19
|
-
/\AAIza[0-9A-Za-z_-]{35}\z/
|
|
20
|
-
].freeze
|
|
21
|
-
private_constant :SECRET_VALUE_PATTERNS
|
|
11
|
+
REDACTED_VALUE = Redaction::REDACTED
|
|
22
12
|
|
|
23
13
|
class << self
|
|
24
14
|
def call(tags, config: LlmCostTracker.configuration)
|
|
@@ -58,15 +48,9 @@ module LlmCostTracker
|
|
|
58
48
|
def sanitized_value(key, value, redacted, limit)
|
|
59
49
|
return REDACTED_VALUE if redacted_key?(key, redacted)
|
|
60
50
|
|
|
61
|
-
|
|
62
|
-
return REDACTED_VALUE if scrubbed.equal?(REDACTED_SENTINEL)
|
|
63
|
-
|
|
64
|
-
scalar_truncate(scrubbed, limit)
|
|
51
|
+
scalar_truncate(scrub_secrets(Ledger::Storable.clean(value)), limit)
|
|
65
52
|
end
|
|
66
53
|
|
|
67
|
-
REDACTED_SENTINEL = Object.new.freeze
|
|
68
|
-
private_constant :REDACTED_SENTINEL
|
|
69
|
-
|
|
70
54
|
def scalar_truncate(value, limit)
|
|
71
55
|
case value
|
|
72
56
|
when Hash
|
|
@@ -86,26 +70,16 @@ module LlmCostTracker
|
|
|
86
70
|
def scrub_secrets(value)
|
|
87
71
|
case value
|
|
88
72
|
when Hash
|
|
89
|
-
value.
|
|
90
|
-
scrubbed = scrub_secrets(nested)
|
|
91
|
-
out[key] = scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
|
|
92
|
-
end
|
|
73
|
+
value.transform_values { |nested| scrub_secrets(nested) }
|
|
93
74
|
when Array
|
|
94
|
-
value.map
|
|
95
|
-
scrubbed = scrub_secrets(nested)
|
|
96
|
-
scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
|
|
97
|
-
end
|
|
75
|
+
value.map { |nested| scrub_secrets(nested) }
|
|
98
76
|
else
|
|
99
|
-
|
|
77
|
+
string = value.to_s
|
|
78
|
+
scrubbed = Redaction.text(string)
|
|
79
|
+
scrubbed == string ? value : scrubbed
|
|
100
80
|
end
|
|
101
81
|
end
|
|
102
82
|
|
|
103
|
-
def secret_shaped?(string)
|
|
104
|
-
return false if string.bytesize < 16
|
|
105
|
-
|
|
106
|
-
SECRET_VALUE_PATTERNS.any? { |pattern| pattern.match?(string) }
|
|
107
|
-
end
|
|
108
|
-
|
|
109
83
|
def redacted_key?(key, redacted)
|
|
110
84
|
return false if redacted.empty?
|
|
111
85
|
|
|
@@ -27,10 +27,8 @@ module LlmCostTracker
|
|
|
27
27
|
|
|
28
28
|
tags = build_tags(context_tags: context_tags, metadata: metadata)
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
Pricing::Unknown.process(event.model, pricing_mode: calculation.mode)
|
|
33
|
-
end
|
|
30
|
+
unpriced = calculation.token_cost.nil? && event.token_usage.total_tokens.positive? &&
|
|
31
|
+
calculation.priced_line_items.none?(&:priced?)
|
|
34
32
|
|
|
35
33
|
event = build_event(event: event, calculation: calculation, tags: tags, latency_ms: latency_ms)
|
|
36
34
|
|
|
@@ -43,6 +41,7 @@ module LlmCostTracker
|
|
|
43
41
|
|
|
44
42
|
yield if block_given?
|
|
45
43
|
notify_subscribers(event)
|
|
44
|
+
Pricing::Unknown.process(event.model, pricing_mode: calculation.mode) if unpriced
|
|
46
45
|
behavior_override = :raise if enforce_budget
|
|
47
46
|
Budget.check!(event, behavior_override: behavior_override)
|
|
48
47
|
Budget.check_persisted!([event], behavior_override: behavior_override) unless Ingestion.async?
|
data/lib/llm_cost_tracker.rb
CHANGED
|
@@ -57,6 +57,7 @@ module LlmCostTracker
|
|
|
57
57
|
yield(config)
|
|
58
58
|
config.finalize!
|
|
59
59
|
Pricing::Registry.reset!
|
|
60
|
+
Pricing::Registry.validate_file!(config.pricing.file)
|
|
60
61
|
Integrations.install!
|
|
61
62
|
config
|
|
62
63
|
end
|
|
@@ -128,10 +129,11 @@ module LlmCostTracker
|
|
|
128
129
|
metadata: tags
|
|
129
130
|
)
|
|
130
131
|
yield collector
|
|
131
|
-
|
|
132
|
-
rescue StandardError
|
|
132
|
+
rescue Exception # rubocop:disable Lint/RescueException -- record the spend and keep the caller's exception
|
|
133
133
|
collector&.finish!(errored: true)
|
|
134
134
|
raise
|
|
135
|
+
else
|
|
136
|
+
collector.finish!
|
|
135
137
|
end
|
|
136
138
|
end
|
|
137
139
|
end
|
|
@@ -98,7 +98,8 @@ namespace :llm_cost_tracker do
|
|
|
98
98
|
namespace :prices do
|
|
99
99
|
desc(
|
|
100
100
|
"Refresh the configured pricing file from the maintained LLM Cost Tracker price snapshot. " \
|
|
101
|
-
"Use PREVIEW=1 to preview, URL=... to override the source,
|
|
101
|
+
"Use PREVIEW=1 to preview, FORCE=1 to accept suspicious price changes, URL=... to override the source, " \
|
|
102
|
+
"or OUTPUT=path/to/file.json."
|
|
102
103
|
)
|
|
103
104
|
task :refresh do
|
|
104
105
|
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
@@ -110,7 +111,8 @@ namespace :llm_cost_tracker do
|
|
|
110
111
|
result = LlmCostTracker::Pricing::Sync.refresh(
|
|
111
112
|
path: output_path,
|
|
112
113
|
url: source_url,
|
|
113
|
-
preview: preview
|
|
114
|
+
preview: preview,
|
|
115
|
+
force: ENV["FORCE"] == "1"
|
|
114
116
|
)
|
|
115
117
|
|
|
116
118
|
action = if preview
|
|
@@ -124,7 +126,7 @@ namespace :llm_cost_tracker do
|
|
|
124
126
|
puts "llm_cost_tracker: #{action} pricing file #{result.path}"
|
|
125
127
|
puts " source: #{result.source_url}"
|
|
126
128
|
puts " version: #{result.source_version.inspect}" if result.source_version
|
|
127
|
-
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes)
|
|
129
|
+
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
|
|
128
130
|
end
|
|
129
131
|
|
|
130
132
|
desc "Compare the current pricing file with the maintained LLM Cost Tracker price snapshot."
|
|
@@ -139,7 +141,7 @@ namespace :llm_cost_tracker do
|
|
|
139
141
|
puts "llm_cost_tracker: checked pricing file #{result.path}"
|
|
140
142
|
puts " source: #{result.source_url}"
|
|
141
143
|
puts " version: #{result.source_version.inspect}" if result.source_version
|
|
142
|
-
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes)
|
|
144
|
+
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
|
|
143
145
|
puts " pricing is up to date" if result.up_to_date
|
|
144
146
|
abort("llm_cost_tracker: pricing check failed") unless result.up_to_date
|
|
145
147
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm_cost_tracker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.14.
|
|
4
|
+
version: 0.14.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sergii Khomenko
|
|
@@ -205,16 +205,22 @@ dependencies:
|
|
|
205
205
|
name: ruby_llm
|
|
206
206
|
requirement: !ruby/object:Gem::Requirement
|
|
207
207
|
requirements:
|
|
208
|
-
- - "
|
|
208
|
+
- - ">="
|
|
209
209
|
- !ruby/object:Gem::Version
|
|
210
210
|
version: '1.16'
|
|
211
|
+
- - "<"
|
|
212
|
+
- !ruby/object:Gem::Version
|
|
213
|
+
version: '3'
|
|
211
214
|
type: :development
|
|
212
215
|
prerelease: false
|
|
213
216
|
version_requirements: !ruby/object:Gem::Requirement
|
|
214
217
|
requirements:
|
|
215
|
-
- - "
|
|
218
|
+
- - ">="
|
|
216
219
|
- !ruby/object:Gem::Version
|
|
217
220
|
version: '1.16'
|
|
221
|
+
- - "<"
|
|
222
|
+
- !ruby/object:Gem::Version
|
|
223
|
+
version: '3'
|
|
218
224
|
- !ruby/object:Gem::Dependency
|
|
219
225
|
name: simplecov
|
|
220
226
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -340,9 +346,11 @@ files:
|
|
|
340
346
|
- lib/llm_cost_tracker/assets.rb
|
|
341
347
|
- lib/llm_cost_tracker/budget.rb
|
|
342
348
|
- lib/llm_cost_tracker/budget/per_tag.rb
|
|
349
|
+
- lib/llm_cost_tracker/capture/event_window.rb
|
|
343
350
|
- lib/llm_cost_tracker/capture/sdk_payload.rb
|
|
344
351
|
- lib/llm_cost_tracker/capture/sse.rb
|
|
345
352
|
- lib/llm_cost_tracker/capture/stream_collector.rb
|
|
353
|
+
- lib/llm_cost_tracker/capture/stream_tap.rb
|
|
346
354
|
- lib/llm_cost_tracker/capture/stream_tracker.rb
|
|
347
355
|
- lib/llm_cost_tracker/capture_verifier.rb
|
|
348
356
|
- lib/llm_cost_tracker/charges/cost.rb
|
|
@@ -399,6 +407,7 @@ files:
|
|
|
399
407
|
- lib/llm_cost_tracker/integrations/openai/patches.rb
|
|
400
408
|
- lib/llm_cost_tracker/integrations/ruby_llm.rb
|
|
401
409
|
- lib/llm_cost_tracker/ledger.rb
|
|
410
|
+
- lib/llm_cost_tracker/ledger/isolation.rb
|
|
402
411
|
- lib/llm_cost_tracker/ledger/period.rb
|
|
403
412
|
- lib/llm_cost_tracker/ledger/period/totals.rb
|
|
404
413
|
- lib/llm_cost_tracker/ledger/rollups.rb
|
|
@@ -411,6 +420,7 @@ files:
|
|
|
411
420
|
- lib/llm_cost_tracker/ledger/schema/calls.rb
|
|
412
421
|
- lib/llm_cost_tracker/ledger/schema/ingestion/inbox_entries.rb
|
|
413
422
|
- lib/llm_cost_tracker/ledger/schema/ingestion/leases.rb
|
|
423
|
+
- lib/llm_cost_tracker/ledger/storable.rb
|
|
414
424
|
- lib/llm_cost_tracker/ledger/store.rb
|
|
415
425
|
- lib/llm_cost_tracker/ledger/tags/breakdown.rb
|
|
416
426
|
- lib/llm_cost_tracker/ledger/tags/encoding.rb
|
|
@@ -436,6 +446,7 @@ files:
|
|
|
436
446
|
- lib/llm_cost_tracker/pricing/sync/fetcher.rb
|
|
437
447
|
- lib/llm_cost_tracker/pricing/sync/registry_diff.rb
|
|
438
448
|
- lib/llm_cost_tracker/pricing/sync/registry_writer.rb
|
|
449
|
+
- lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb
|
|
439
450
|
- lib/llm_cost_tracker/pricing/unknown.rb
|
|
440
451
|
- lib/llm_cost_tracker/providers.rb
|
|
441
452
|
- lib/llm_cost_tracker/providers/anthropic/parser.rb
|
|
@@ -454,6 +465,7 @@ files:
|
|
|
454
465
|
- lib/llm_cost_tracker/providers/openai/usage_extractor.rb
|
|
455
466
|
- lib/llm_cost_tracker/providers/openai_compatible/parser.rb
|
|
456
467
|
- lib/llm_cost_tracker/railtie.rb
|
|
468
|
+
- lib/llm_cost_tracker/redaction.rb
|
|
457
469
|
- lib/llm_cost_tracker/report.rb
|
|
458
470
|
- lib/llm_cost_tracker/report/data.rb
|
|
459
471
|
- lib/llm_cost_tracker/report/formatter.rb
|
|
@@ -486,7 +498,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
486
498
|
requirements:
|
|
487
499
|
- - ">="
|
|
488
500
|
- !ruby/object:Gem::Version
|
|
489
|
-
version: 3.
|
|
501
|
+
version: 3.3.0
|
|
490
502
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
491
503
|
requirements:
|
|
492
504
|
- - ">="
|