llm_cost_tracker 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +99 -16
- data/README.md +17 -29
- data/app/controllers/llm_cost_tracker/application_controller.rb +19 -5
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +1 -0
- data/app/controllers/llm_cost_tracker/models_controller.rb +3 -1
- data/app/controllers/llm_cost_tracker/pricing_controller.rb +2 -2
- data/app/controllers/llm_cost_tracker/tags_controller.rb +5 -4
- data/app/helpers/llm_cost_tracker/application_helper.rb +1 -1
- data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +16 -0
- data/app/models/llm_cost_tracker/call.rb +10 -4
- data/app/models/llm_cost_tracker/call_rollup.rb +19 -3
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +13 -0
- data/app/services/llm_cost_tracker/dashboard/filter.rb +24 -21
- data/app/services/llm_cost_tracker/dashboard/monthly_budget.rb +1 -1
- data/app/services/llm_cost_tracker/dashboard/pagination.rb +2 -1
- data/app/services/llm_cost_tracker/dashboard/params.rb +10 -0
- data/app/services/llm_cost_tracker/dashboard/pricing_overview.rb +3 -3
- data/app/services/llm_cost_tracker/dashboard/setup_state.rb +5 -3
- data/app/views/llm_cost_tracker/calls/show.html.erb +6 -8
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +11 -0
- data/app/views/llm_cost_tracker/shared/_filter_pill_date.html.erb +1 -3
- data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb +1 -3
- data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb +1 -3
- data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +1 -3
- data/app/views/llm_cost_tracker/tags/show.html.erb +3 -0
- data/config/routes.rb +6 -1
- data/lib/llm_cost_tracker/budget/per_tag.rb +163 -0
- data/lib/llm_cost_tracker/budget.rb +120 -32
- data/lib/llm_cost_tracker/capture/event_window.rb +99 -0
- data/lib/llm_cost_tracker/capture/sse.rb +108 -32
- data/lib/llm_cost_tracker/capture/stream_collector.rb +17 -56
- data/lib/llm_cost_tracker/capture/stream_tap.rb +57 -0
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +5 -35
- data/lib/llm_cost_tracker/charges/cost_status.rb +4 -3
- data/lib/llm_cost_tracker/configuration/budgets.rb +93 -0
- data/lib/llm_cost_tracker/configuration/capture.rb +42 -0
- data/lib/llm_cost_tracker/configuration/ingestion.rb +20 -0
- data/lib/llm_cost_tracker/configuration/mutability.rb +33 -0
- data/lib/llm_cost_tracker/configuration/pricing.rb +36 -0
- data/lib/llm_cost_tracker/configuration/section.rb +59 -0
- data/lib/llm_cost_tracker/configuration/tags.rb +52 -0
- data/lib/llm_cost_tracker/configuration.rb +69 -125
- data/lib/llm_cost_tracker/deprecator.rb +9 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +17 -8
- data/lib/llm_cost_tracker/doctor/price_check.rb +2 -1
- data/lib/llm_cost_tracker/doctor.rb +4 -4
- data/lib/llm_cost_tracker/engine.rb +4 -0
- data/lib/llm_cost_tracker/errors.rb +25 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb +2 -2
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +2 -2
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb +0 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +8 -4
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +50 -30
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb +40 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb +41 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_indexes_generator.rb +30 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_per_tag_budgets_generator.rb +30 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +38 -7
- data/lib/llm_cost_tracker/ingestion/pool.rb +9 -2
- data/lib/llm_cost_tracker/ingestion.rb +3 -7
- data/lib/llm_cost_tracker/integrations/anthropic.rb +7 -1
- data/lib/llm_cost_tracker/integrations/base.rb +17 -1
- data/lib/llm_cost_tracker/integrations/openai/batch_capture.rb +16 -13
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +53 -18
- data/lib/llm_cost_tracker/ledger/isolation.rb +30 -0
- data/lib/llm_cost_tracker/ledger/period/totals.rb +3 -2
- data/lib/llm_cost_tracker/ledger/rollups.rb +45 -7
- data/lib/llm_cost_tracker/ledger/storable.rb +16 -0
- data/lib/llm_cost_tracker/ledger/store.rb +21 -13
- data/lib/llm_cost_tracker/ledger/tags/encoding.rb +15 -5
- data/lib/llm_cost_tracker/ledger.rb +1 -0
- data/lib/llm_cost_tracker/logging.rb +5 -5
- data/lib/llm_cost_tracker/middleware/faraday.rb +33 -44
- data/lib/llm_cost_tracker/parsers.rb +5 -1
- data/lib/llm_cost_tracker/prices.json +2089 -372
- data/lib/llm_cost_tracker/pricing/backfill.rb +11 -1
- data/lib/llm_cost_tracker/pricing/calculation.rb +4 -4
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +26 -15
- data/lib/llm_cost_tracker/pricing/matcher.rb +7 -0
- data/lib/llm_cost_tracker/pricing/rate.rb +1 -2
- data/lib/llm_cost_tracker/pricing/registry.rb +17 -6
- data/lib/llm_cost_tracker/pricing/sync/change_printer.rb +6 -1
- data/lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb +47 -0
- data/lib/llm_cost_tracker/pricing/sync.rb +31 -14
- data/lib/llm_cost_tracker/pricing/unknown.rb +11 -8
- data/lib/llm_cost_tracker/providers/anthropic/usage_extractor.rb +3 -2
- data/lib/llm_cost_tracker/providers/azure/parser.rb +19 -0
- data/lib/llm_cost_tracker/providers/gemini/parser.rb +4 -0
- data/lib/llm_cost_tracker/providers/openai/model_families.rb +0 -7
- data/lib/llm_cost_tracker/providers/openai/response_parser.rb +10 -3
- data/lib/llm_cost_tracker/providers/openai/usage_extractor.rb +14 -8
- data/lib/llm_cost_tracker/providers/openai_compatible/parser.rb +6 -2
- data/lib/llm_cost_tracker/railtie.rb +3 -7
- data/lib/llm_cost_tracker/redaction.rb +32 -0
- data/lib/llm_cost_tracker/report/data.rb +2 -2
- data/lib/llm_cost_tracker/retention.rb +22 -10
- data/lib/llm_cost_tracker/tags/context.rb +3 -3
- data/lib/llm_cost_tracker/tags/sanitizer.rb +14 -40
- data/lib/llm_cost_tracker/tracker.rb +16 -26
- data/lib/llm_cost_tracker/usage/catalog.rb +1 -2
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +10 -3
- data/lib/tasks/llm_cost_tracker.rake +29 -14
- metadata +37 -12
|
@@ -8,14 +8,10 @@ module LlmCostTracker
|
|
|
8
8
|
app.config.eager_load_paths << models_path unless app.config.eager_load_paths.include?(models_path)
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
+
GENERATOR_FILES = File.expand_path("generators/llm_cost_tracker/*_generator.rb", __dir__)
|
|
12
|
+
|
|
11
13
|
generators do
|
|
12
|
-
|
|
13
|
-
require_relative "generators/llm_cost_tracker/prices_generator"
|
|
14
|
-
require_relative "generators/llm_cost_tracker/call_rollups_generator"
|
|
15
|
-
require_relative "generators/llm_cost_tracker/async_ingestion_generator"
|
|
16
|
-
require_relative "generators/llm_cost_tracker/upgrade_call_rollups_provider_generator"
|
|
17
|
-
require_relative "generators/llm_cost_tracker/upgrade_image_tokens_generator"
|
|
18
|
-
require_relative "generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator"
|
|
14
|
+
Dir[GENERATOR_FILES].each { |path| require path }
|
|
19
15
|
end
|
|
20
16
|
end
|
|
21
17
|
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module Redaction
|
|
5
|
+
REDACTED = "[REDACTED]"
|
|
6
|
+
|
|
7
|
+
TOKEN = Regexp.union(
|
|
8
|
+
/sk-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}/,
|
|
9
|
+
/AKIA[0-9A-Z]{16}/,
|
|
10
|
+
/gh[opsur]_[A-Za-z0-9]{16,}/,
|
|
11
|
+
/github_pat_[A-Za-z0-9_]{20,}/,
|
|
12
|
+
/eyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/i,
|
|
13
|
+
/bearer\s+[A-Za-z0-9_.-]{20,}/i,
|
|
14
|
+
/xox[abprs]-[A-Za-z0-9-]{10,}/,
|
|
15
|
+
/(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}/,
|
|
16
|
+
/AIza[0-9A-Za-z_-]{35}/
|
|
17
|
+
)
|
|
18
|
+
EMBEDDED_TOKEN = /(?<![A-Za-z0-9_-])(?:#{TOKEN})(?![A-Za-z0-9_-])/
|
|
19
|
+
PARAM = /(?<![A-Za-z0-9])((?:api)?key|token|secret|password|sig(?:nature)?)=[^&\s"']+/i
|
|
20
|
+
HEADER = /((?:authorization|api-key)["']?\s*(?:=>|:)\s*["']?)(?:[A-Za-z]+\s+)?[^\s"',;]+/i
|
|
21
|
+
USERINFO = %r{://[^/\s@?#]+@}
|
|
22
|
+
private_constant :TOKEN, :EMBEDDED_TOKEN, :PARAM, :HEADER, :USERINFO
|
|
23
|
+
|
|
24
|
+
def self.text(value)
|
|
25
|
+
value.to_s.scrub
|
|
26
|
+
.gsub(USERINFO, "://#{REDACTED}@")
|
|
27
|
+
.gsub(HEADER, "\\1#{REDACTED}")
|
|
28
|
+
.gsub(PARAM, "\\1=#{REDACTED}")
|
|
29
|
+
.gsub(EMBEDDED_TOKEN, REDACTED)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -34,7 +34,7 @@ module LlmCostTracker
|
|
|
34
34
|
end
|
|
35
35
|
from = now - days.days
|
|
36
36
|
scope = LlmCostTracker::Call.where(tracked_at: from..now)
|
|
37
|
-
tag_breakdowns ||= LlmCostTracker.configuration.
|
|
37
|
+
tag_breakdowns ||= LlmCostTracker.configuration.tags.report_breakdown_keys || []
|
|
38
38
|
aggregate = totals(scope)
|
|
39
39
|
|
|
40
40
|
new(
|
|
@@ -55,7 +55,7 @@ module LlmCostTracker
|
|
|
55
55
|
def self.totals(scope)
|
|
56
56
|
scope
|
|
57
57
|
.select(
|
|
58
|
-
"COALESCE(SUM(total_cost), 0) AS total_cost, " \
|
|
58
|
+
"COALESCE(SUM(#{LlmCostTracker::Call.qualified(:total_cost)}), 0) AS total_cost, " \
|
|
59
59
|
"COUNT(*) AS requests_count, " \
|
|
60
60
|
"AVG(latency_ms) AS average_latency_ms, " \
|
|
61
61
|
"COALESCE(SUM(CASE WHEN #{Charges::CostStatus.unknown_pricing_sql} " \
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
module LlmCostTracker
|
|
4
4
|
module Retention
|
|
5
5
|
DEFAULT_BATCH_SIZE = 5_000
|
|
6
|
+
ROLLUP_COLUMNS = %i[tracked_at total_cost pricing_snapshot provider].freeze
|
|
7
|
+
private_constant :ROLLUP_COLUMNS
|
|
6
8
|
|
|
7
9
|
class << self
|
|
8
10
|
def prune(older_than:, batch_size: DEFAULT_BATCH_SIZE, now: Time.now.utc)
|
|
@@ -26,11 +28,23 @@ module LlmCostTracker
|
|
|
26
28
|
require_relative "ingestion"
|
|
27
29
|
return 0 unless LlmCostTracker::Ingestion::InboxEntry.table_exists?
|
|
28
30
|
|
|
29
|
-
LlmCostTracker::Ingestion::InboxEntry.where(tracked_at: ...cutoff)
|
|
31
|
+
scope = LlmCostTracker::Ingestion::InboxEntry.where(tracked_at: ...cutoff)
|
|
32
|
+
warn_undrained(scope.pending)
|
|
33
|
+
scope.delete_all
|
|
30
34
|
end
|
|
31
35
|
|
|
32
36
|
private
|
|
33
37
|
|
|
38
|
+
def warn_undrained(scope)
|
|
39
|
+
count, cost = scope.pick(Arel.sql("COUNT(*)"), Arel.sql("COALESCE(SUM(total_cost), 0)"))
|
|
40
|
+
return if count.to_i.zero?
|
|
41
|
+
|
|
42
|
+
Logging.warn(
|
|
43
|
+
"Retention.prune_inbox is deleting #{count} inbox row(s) worth #{cost} that were never drained " \
|
|
44
|
+
"into the ledger. Their spend is lost. Drain the inbox before pruning, or raise the retention window."
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
|
|
34
48
|
def resolve_cutoff(older_than, now)
|
|
35
49
|
cutoff = case older_than
|
|
36
50
|
when Time, DateTime then older_than.utc
|
|
@@ -58,22 +72,20 @@ module LlmCostTracker
|
|
|
58
72
|
|
|
59
73
|
def prune_batch(cutoff, batch_size)
|
|
60
74
|
LlmCostTracker::Call.transaction do
|
|
61
|
-
|
|
62
|
-
rows = prunable_rows(cutoff, batch_size, with_rollup_columns: cache_rollups)
|
|
75
|
+
rows = prunable_rows(cutoff, batch_size)
|
|
63
76
|
next 0 if rows.empty?
|
|
64
77
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
LlmCostTracker::Ledger::Rollups.decrement!(rows) if cache_rollups && deleted.positive?
|
|
78
|
+
deleted = LlmCostTracker::Call.where(id: rows.map(&:id)).delete_all
|
|
79
|
+
LlmCostTracker::Ledger::Rollups.decrement!(rows) if deleted.positive?
|
|
68
80
|
deleted
|
|
69
81
|
end
|
|
70
82
|
end
|
|
71
83
|
|
|
72
|
-
def prunable_rows(cutoff, batch_size
|
|
84
|
+
def prunable_rows(cutoff, batch_size)
|
|
73
85
|
relation = LlmCostTracker::Call.where(tracked_at: ...cutoff).order(:id).limit(batch_size).lock
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
relation.select(
|
|
86
|
+
columns = [:id]
|
|
87
|
+
columns += ROLLUP_COLUMNS if LlmCostTracker::Ledger::Rollups.cache_active?
|
|
88
|
+
relation.select(*columns).to_a
|
|
77
89
|
end
|
|
78
90
|
end
|
|
79
91
|
end
|
|
@@ -18,15 +18,15 @@ module LlmCostTracker
|
|
|
18
18
|
|
|
19
19
|
def tags
|
|
20
20
|
config = LlmCostTracker.configuration
|
|
21
|
-
base = config.
|
|
22
|
-
Sanitizer.call(call_default_tags(config.
|
|
21
|
+
base = config.tags.static_sanitized_default ||
|
|
22
|
+
Sanitizer.call(call_default_tags(config.tags.default).to_h)
|
|
23
23
|
base.merge(*Array(ActiveSupport::IsolatedExecutionState[KEY]))
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
def call_default_tags(proc_or_lambda)
|
|
27
27
|
proc_or_lambda.call
|
|
28
28
|
rescue StandardError => e
|
|
29
|
-
Logging.warn("LlmCostTracker
|
|
29
|
+
Logging.warn("LlmCostTracker tags.default proc raised: #{e.class}: #{e.message}; using empty default tags")
|
|
30
30
|
{}
|
|
31
31
|
end
|
|
32
32
|
end
|
|
@@ -2,30 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
require "active_support/core_ext/string/inflections"
|
|
4
4
|
|
|
5
|
+
require_relative "../ledger/storable"
|
|
6
|
+
require_relative "../redaction"
|
|
7
|
+
|
|
5
8
|
module LlmCostTracker
|
|
6
9
|
module Tags
|
|
7
10
|
module Sanitizer
|
|
8
|
-
REDACTED_VALUE =
|
|
9
|
-
|
|
10
|
-
SECRET_VALUE_PATTERNS = [
|
|
11
|
-
/\Ask-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}\z/,
|
|
12
|
-
/\AAKIA[0-9A-Z]{16}\z/,
|
|
13
|
-
/\Agh[opsur]_[A-Za-z0-9]{16,}\z/,
|
|
14
|
-
/\Agithub_pat_[A-Za-z0-9_]{20,}\z/,
|
|
15
|
-
/\Aeyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\z/i,
|
|
16
|
-
/\Abearer\s+[A-Za-z0-9_.-]{20,}\z/i,
|
|
17
|
-
/\Axox[abprs]-[A-Za-z0-9-]{10,}\z/,
|
|
18
|
-
/\A(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\z/,
|
|
19
|
-
/\AAIza[0-9A-Za-z_-]{35}\z/
|
|
20
|
-
].freeze
|
|
21
|
-
private_constant :SECRET_VALUE_PATTERNS
|
|
11
|
+
REDACTED_VALUE = Redaction::REDACTED
|
|
22
12
|
|
|
23
13
|
class << self
|
|
24
14
|
def call(tags, config: LlmCostTracker.configuration)
|
|
25
15
|
tags = (tags || {}).to_h
|
|
26
|
-
redacted = config.
|
|
27
|
-
limit = [config.
|
|
28
|
-
max_count = [config.
|
|
16
|
+
redacted = config.tags.normalized_redacted_keys
|
|
17
|
+
limit = [config.tags.max_value_bytesize.to_i, 0].max
|
|
18
|
+
max_count = [config.tags.max_count.to_i, 0].max
|
|
29
19
|
tags.to_a.last(max_count).each_with_object({}) do |(key, value), sanitized|
|
|
30
20
|
next unless valid_key?(key)
|
|
31
21
|
|
|
@@ -43,7 +33,7 @@ module LlmCostTracker
|
|
|
43
33
|
|
|
44
34
|
def cap(tags, config: LlmCostTracker.configuration)
|
|
45
35
|
tags = (tags || {}).to_h
|
|
46
|
-
max_count = [config.
|
|
36
|
+
max_count = [config.tags.max_count.to_i, 0].max
|
|
47
37
|
return tags if tags.size <= max_count
|
|
48
38
|
|
|
49
39
|
tags.to_a.last(max_count).to_h
|
|
@@ -58,15 +48,9 @@ module LlmCostTracker
|
|
|
58
48
|
def sanitized_value(key, value, redacted, limit)
|
|
59
49
|
return REDACTED_VALUE if redacted_key?(key, redacted)
|
|
60
50
|
|
|
61
|
-
|
|
62
|
-
return REDACTED_VALUE if scrubbed.equal?(REDACTED_SENTINEL)
|
|
63
|
-
|
|
64
|
-
scalar_truncate(scrubbed, limit)
|
|
51
|
+
scalar_truncate(scrub_secrets(Ledger::Storable.clean(value)), limit)
|
|
65
52
|
end
|
|
66
53
|
|
|
67
|
-
REDACTED_SENTINEL = Object.new.freeze
|
|
68
|
-
private_constant :REDACTED_SENTINEL
|
|
69
|
-
|
|
70
54
|
def scalar_truncate(value, limit)
|
|
71
55
|
case value
|
|
72
56
|
when Hash
|
|
@@ -86,26 +70,16 @@ module LlmCostTracker
|
|
|
86
70
|
def scrub_secrets(value)
|
|
87
71
|
case value
|
|
88
72
|
when Hash
|
|
89
|
-
value.
|
|
90
|
-
scrubbed = scrub_secrets(nested)
|
|
91
|
-
out[key] = scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
|
|
92
|
-
end
|
|
73
|
+
value.transform_values { |nested| scrub_secrets(nested) }
|
|
93
74
|
when Array
|
|
94
|
-
value.map
|
|
95
|
-
scrubbed = scrub_secrets(nested)
|
|
96
|
-
scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
|
|
97
|
-
end
|
|
75
|
+
value.map { |nested| scrub_secrets(nested) }
|
|
98
76
|
else
|
|
99
|
-
|
|
77
|
+
string = value.to_s
|
|
78
|
+
scrubbed = Redaction.text(string)
|
|
79
|
+
scrubbed == string ? value : scrubbed
|
|
100
80
|
end
|
|
101
81
|
end
|
|
102
82
|
|
|
103
|
-
def secret_shaped?(string)
|
|
104
|
-
return false if string.bytesize < 16
|
|
105
|
-
|
|
106
|
-
SECRET_VALUE_PATTERNS.any? { |pattern| pattern.match?(string) }
|
|
107
|
-
end
|
|
108
|
-
|
|
109
83
|
def redacted_key?(key, redacted)
|
|
110
84
|
return false if redacted.empty?
|
|
111
85
|
|
|
@@ -25,25 +25,12 @@ module LlmCostTracker
|
|
|
25
25
|
usage_source: event.usage_source
|
|
26
26
|
)
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
Budget.enforce!(provider: event.provider,
|
|
30
|
-
model: event.model,
|
|
31
|
-
estimate: calculation.cost&.total,
|
|
32
|
-
force: true)
|
|
33
|
-
end
|
|
28
|
+
tags = build_tags(context_tags: context_tags, metadata: metadata)
|
|
34
29
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
Pricing::Unknown.process(event.model)
|
|
38
|
-
end
|
|
30
|
+
unpriced = calculation.token_cost.nil? && event.token_usage.total_tokens.positive? &&
|
|
31
|
+
calculation.priced_line_items.none?(&:priced?)
|
|
39
32
|
|
|
40
|
-
event = build_event(
|
|
41
|
-
event: event,
|
|
42
|
-
calculation: calculation,
|
|
43
|
-
metadata: metadata,
|
|
44
|
-
latency_ms: latency_ms,
|
|
45
|
-
context_tags: context_tags
|
|
46
|
-
)
|
|
33
|
+
event = build_event(event: event, calculation: calculation, tags: tags, latency_ms: latency_ms)
|
|
47
34
|
|
|
48
35
|
if Ingestion.async?
|
|
49
36
|
Ingestion::Inbox.save(event)
|
|
@@ -54,11 +41,20 @@ module LlmCostTracker
|
|
|
54
41
|
|
|
55
42
|
yield if block_given?
|
|
56
43
|
notify_subscribers(event)
|
|
57
|
-
|
|
44
|
+
Pricing::Unknown.process(event.model, pricing_mode: calculation.mode) if unpriced
|
|
45
|
+
behavior_override = :raise if enforce_budget
|
|
46
|
+
Budget.check!(event, behavior_override: behavior_override)
|
|
47
|
+
Budget.check_persisted!([event], behavior_override: behavior_override) unless Ingestion.async?
|
|
58
48
|
|
|
59
49
|
event
|
|
60
50
|
end
|
|
61
51
|
|
|
52
|
+
def build_tags(context_tags:, metadata:)
|
|
53
|
+
resolved = (context_tags || LlmCostTracker::Tags::Context.tags).to_h
|
|
54
|
+
sanitized_metadata = LlmCostTracker::Tags::Sanitizer.call(metadata.to_h)
|
|
55
|
+
LlmCostTracker::Tags::Sanitizer.cap(resolved.merge(sanitized_metadata)).freeze
|
|
56
|
+
end
|
|
57
|
+
|
|
62
58
|
private
|
|
63
59
|
|
|
64
60
|
def notify_subscribers(event)
|
|
@@ -69,13 +65,12 @@ module LlmCostTracker
|
|
|
69
65
|
Logging.warn("Subscriber raised on #{EVENT_NAME}: #{e.class}: #{e.message}")
|
|
70
66
|
end
|
|
71
67
|
|
|
72
|
-
def build_event(event:, calculation:,
|
|
73
|
-
context_tags = (context_tags || LlmCostTracker::Tags::Context.tags).to_h
|
|
68
|
+
def build_event(event:, calculation:, tags:, latency_ms:)
|
|
74
69
|
event.with(
|
|
75
70
|
event_id: SecureRandom.uuid,
|
|
76
71
|
pricing_mode: calculation.mode,
|
|
77
72
|
cost: calculation.cost,
|
|
78
|
-
tags:
|
|
73
|
+
tags: tags,
|
|
79
74
|
latency_ms: finite_latency_ms(latency_ms),
|
|
80
75
|
tracked_at: Time.now.utc,
|
|
81
76
|
cost_status: calculation.cost_status,
|
|
@@ -84,11 +79,6 @@ module LlmCostTracker
|
|
|
84
79
|
)
|
|
85
80
|
end
|
|
86
81
|
|
|
87
|
-
def build_tags(context_tags:, metadata:)
|
|
88
|
-
sanitized_metadata = LlmCostTracker::Tags::Sanitizer.call(metadata.to_h)
|
|
89
|
-
LlmCostTracker::Tags::Sanitizer.cap(context_tags.merge(sanitized_metadata)).freeze
|
|
90
|
-
end
|
|
91
|
-
|
|
92
82
|
def finite_latency_ms(latency_ms)
|
|
93
83
|
return nil if latency_ms.nil?
|
|
94
84
|
|
data/lib/llm_cost_tracker.rb
CHANGED
|
@@ -57,6 +57,7 @@ module LlmCostTracker
|
|
|
57
57
|
yield(config)
|
|
58
58
|
config.finalize!
|
|
59
59
|
Pricing::Registry.reset!
|
|
60
|
+
Pricing::Registry.validate_file!(config.pricing.file)
|
|
60
61
|
Integrations.install!
|
|
61
62
|
config
|
|
62
63
|
end
|
|
@@ -110,7 +111,12 @@ module LlmCostTracker
|
|
|
110
111
|
provider_workspace_id: nil,
|
|
111
112
|
pricing_mode: nil)
|
|
112
113
|
require_relative "llm_cost_tracker/capture/stream_collector"
|
|
113
|
-
|
|
114
|
+
if enforce_budget
|
|
115
|
+
Budget.enforce!(provider: provider,
|
|
116
|
+
model: model,
|
|
117
|
+
force: true,
|
|
118
|
+
tags: Tracker.build_tags(context_tags: nil, metadata: tags))
|
|
119
|
+
end
|
|
114
120
|
collector = Capture::StreamCollector.new(
|
|
115
121
|
provider: provider.to_s,
|
|
116
122
|
model: model,
|
|
@@ -123,10 +129,11 @@ module LlmCostTracker
|
|
|
123
129
|
metadata: tags
|
|
124
130
|
)
|
|
125
131
|
yield collector
|
|
126
|
-
|
|
127
|
-
rescue StandardError
|
|
132
|
+
rescue Exception # rubocop:disable Lint/RescueException -- record the spend and keep the caller's exception
|
|
128
133
|
collector&.finish!(errored: true)
|
|
129
134
|
raise
|
|
135
|
+
else
|
|
136
|
+
collector.finish!
|
|
130
137
|
end
|
|
131
138
|
end
|
|
132
139
|
end
|
|
@@ -6,7 +6,15 @@ require "rails/generators"
|
|
|
6
6
|
require_relative "../llm_cost_tracker/generators/llm_cost_tracker/install_generator"
|
|
7
7
|
require_relative "../llm_cost_tracker/pricing/sync/change_printer"
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
module LlmCostTrackerTasks
|
|
10
|
+
def self.price_refresh_output_path
|
|
11
|
+
path = LlmCostTracker::Pricing::Sync.configured_output_path
|
|
12
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
13
|
+
path
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# rubocop:disable-next Metrics/BlockLength
|
|
10
18
|
namespace :llm_cost_tracker do
|
|
11
19
|
desc "Install LLM Cost Tracker with dashboard and prices, migrate, and run doctor"
|
|
12
20
|
task :setup do
|
|
@@ -75,22 +83,36 @@ namespace :llm_cost_tracker do
|
|
|
75
83
|
puts "llm_cost_tracker: rebuilt #{rows} rollup rows from the calls ledger"
|
|
76
84
|
end
|
|
77
85
|
|
|
86
|
+
desc "Copy call cost and time onto existing llm_cost_tracker_call_tags rows so per-tag " \
|
|
87
|
+
"budgets count history recorded before the upgrade. Use BATCH_SIZE=N to tune."
|
|
88
|
+
task backfill_tag_costs: :environment do
|
|
89
|
+
unless LlmCostTracker::Budget::PerTag.columns?
|
|
90
|
+
abort("llm_cost_tracker: llm_cost_tracker_call_tags is missing the cost columns; " \
|
|
91
|
+
"run the upgrade_per_tag_budgets generator and migrate first")
|
|
92
|
+
end
|
|
93
|
+
batch_size = (ENV["BATCH_SIZE"] || LlmCostTracker::Budget::PerTag::DEFAULT_BACKFILL_BATCH).to_i
|
|
94
|
+
filled = LlmCostTracker::Budget::PerTag.backfill(batch_size: batch_size)
|
|
95
|
+
puts "llm_cost_tracker: filled cost and time on #{filled} tag rows"
|
|
96
|
+
end
|
|
97
|
+
|
|
78
98
|
namespace :prices do
|
|
79
99
|
desc(
|
|
80
100
|
"Refresh the configured pricing file from the maintained LLM Cost Tracker price snapshot. " \
|
|
81
|
-
"Use PREVIEW=1 to preview, URL=... to override the source,
|
|
101
|
+
"Use PREVIEW=1 to preview, FORCE=1 to accept suspicious price changes, URL=... to override the source, " \
|
|
102
|
+
"or OUTPUT=path/to/file.json."
|
|
82
103
|
)
|
|
83
104
|
task :refresh do
|
|
84
105
|
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
85
106
|
require_relative "../llm_cost_tracker"
|
|
86
107
|
|
|
87
|
-
output_path = price_refresh_output_path
|
|
108
|
+
output_path = LlmCostTrackerTasks.price_refresh_output_path
|
|
88
109
|
source_url = LlmCostTracker::Pricing::Sync.configured_remote_url
|
|
89
110
|
preview = ENV["PREVIEW"] == "1"
|
|
90
111
|
result = LlmCostTracker::Pricing::Sync.refresh(
|
|
91
112
|
path: output_path,
|
|
92
113
|
url: source_url,
|
|
93
|
-
preview: preview
|
|
114
|
+
preview: preview,
|
|
115
|
+
force: ENV["FORCE"] == "1"
|
|
94
116
|
)
|
|
95
117
|
|
|
96
118
|
action = if preview
|
|
@@ -104,7 +126,7 @@ namespace :llm_cost_tracker do
|
|
|
104
126
|
puts "llm_cost_tracker: #{action} pricing file #{result.path}"
|
|
105
127
|
puts " source: #{result.source_url}"
|
|
106
128
|
puts " version: #{result.source_version.inspect}" if result.source_version
|
|
107
|
-
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes)
|
|
129
|
+
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
|
|
108
130
|
end
|
|
109
131
|
|
|
110
132
|
desc "Compare the current pricing file with the maintained LLM Cost Tracker price snapshot."
|
|
@@ -112,23 +134,16 @@ namespace :llm_cost_tracker do
|
|
|
112
134
|
Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
|
|
113
135
|
require_relative "../llm_cost_tracker"
|
|
114
136
|
|
|
115
|
-
output_path = price_refresh_output_path
|
|
137
|
+
output_path = LlmCostTrackerTasks.price_refresh_output_path
|
|
116
138
|
source_url = LlmCostTracker::Pricing::Sync.configured_remote_url
|
|
117
139
|
result = LlmCostTracker::Pricing::Sync.check(path: output_path, url: source_url)
|
|
118
140
|
|
|
119
141
|
puts "llm_cost_tracker: checked pricing file #{result.path}"
|
|
120
142
|
puts " source: #{result.source_url}"
|
|
121
143
|
puts " version: #{result.source_version.inspect}" if result.source_version
|
|
122
|
-
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes)
|
|
144
|
+
LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
|
|
123
145
|
puts " pricing is up to date" if result.up_to_date
|
|
124
146
|
abort("llm_cost_tracker: pricing check failed") unless result.up_to_date
|
|
125
147
|
end
|
|
126
148
|
end
|
|
127
149
|
end
|
|
128
|
-
# rubocop:enable Metrics/BlockLength
|
|
129
|
-
|
|
130
|
-
def price_refresh_output_path
|
|
131
|
-
path = LlmCostTracker::Pricing::Sync.configured_output_path
|
|
132
|
-
FileUtils.mkdir_p(File.dirname(path))
|
|
133
|
-
path
|
|
134
|
-
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm_cost_tracker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.14.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sergii Khomenko
|
|
@@ -109,42 +109,42 @@ dependencies:
|
|
|
109
109
|
requirements:
|
|
110
110
|
- - "~>"
|
|
111
111
|
- !ruby/object:Gem::Version
|
|
112
|
-
version: '1.
|
|
112
|
+
version: '1.61'
|
|
113
113
|
type: :development
|
|
114
114
|
prerelease: false
|
|
115
115
|
version_requirements: !ruby/object:Gem::Requirement
|
|
116
116
|
requirements:
|
|
117
117
|
- - "~>"
|
|
118
118
|
- !ruby/object:Gem::Version
|
|
119
|
-
version: '1.
|
|
119
|
+
version: '1.61'
|
|
120
120
|
- !ruby/object:Gem::Dependency
|
|
121
121
|
name: nokogiri
|
|
122
122
|
requirement: !ruby/object:Gem::Requirement
|
|
123
123
|
requirements:
|
|
124
124
|
- - "~>"
|
|
125
125
|
- !ruby/object:Gem::Version
|
|
126
|
-
version: '1.
|
|
126
|
+
version: '1.19'
|
|
127
127
|
type: :development
|
|
128
128
|
prerelease: false
|
|
129
129
|
version_requirements: !ruby/object:Gem::Requirement
|
|
130
130
|
requirements:
|
|
131
131
|
- - "~>"
|
|
132
132
|
- !ruby/object:Gem::Version
|
|
133
|
-
version: '1.
|
|
133
|
+
version: '1.19'
|
|
134
134
|
- !ruby/object:Gem::Dependency
|
|
135
135
|
name: openai
|
|
136
136
|
requirement: !ruby/object:Gem::Requirement
|
|
137
137
|
requirements:
|
|
138
138
|
- - "~>"
|
|
139
139
|
- !ruby/object:Gem::Version
|
|
140
|
-
version: '0.
|
|
140
|
+
version: '0.80'
|
|
141
141
|
type: :development
|
|
142
142
|
prerelease: false
|
|
143
143
|
version_requirements: !ruby/object:Gem::Requirement
|
|
144
144
|
requirements:
|
|
145
145
|
- - "~>"
|
|
146
146
|
- !ruby/object:Gem::Version
|
|
147
|
-
version: '0.
|
|
147
|
+
version: '0.80'
|
|
148
148
|
- !ruby/object:Gem::Dependency
|
|
149
149
|
name: pg
|
|
150
150
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -205,16 +205,22 @@ dependencies:
|
|
|
205
205
|
name: ruby_llm
|
|
206
206
|
requirement: !ruby/object:Gem::Requirement
|
|
207
207
|
requirements:
|
|
208
|
-
- - "
|
|
208
|
+
- - ">="
|
|
209
|
+
- !ruby/object:Gem::Version
|
|
210
|
+
version: '1.16'
|
|
211
|
+
- - "<"
|
|
209
212
|
- !ruby/object:Gem::Version
|
|
210
|
-
version: '
|
|
213
|
+
version: '3'
|
|
211
214
|
type: :development
|
|
212
215
|
prerelease: false
|
|
213
216
|
version_requirements: !ruby/object:Gem::Requirement
|
|
214
217
|
requirements:
|
|
215
|
-
- - "
|
|
218
|
+
- - ">="
|
|
219
|
+
- !ruby/object:Gem::Version
|
|
220
|
+
version: '1.16'
|
|
221
|
+
- - "<"
|
|
216
222
|
- !ruby/object:Gem::Version
|
|
217
|
-
version: '
|
|
223
|
+
version: '3'
|
|
218
224
|
- !ruby/object:Gem::Dependency
|
|
219
225
|
name: simplecov
|
|
220
226
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -339,9 +345,12 @@ files:
|
|
|
339
345
|
- lib/llm_cost_tracker.rb
|
|
340
346
|
- lib/llm_cost_tracker/assets.rb
|
|
341
347
|
- lib/llm_cost_tracker/budget.rb
|
|
348
|
+
- lib/llm_cost_tracker/budget/per_tag.rb
|
|
349
|
+
- lib/llm_cost_tracker/capture/event_window.rb
|
|
342
350
|
- lib/llm_cost_tracker/capture/sdk_payload.rb
|
|
343
351
|
- lib/llm_cost_tracker/capture/sse.rb
|
|
344
352
|
- lib/llm_cost_tracker/capture/stream_collector.rb
|
|
353
|
+
- lib/llm_cost_tracker/capture/stream_tap.rb
|
|
345
354
|
- lib/llm_cost_tracker/capture/stream_tracker.rb
|
|
346
355
|
- lib/llm_cost_tracker/capture_verifier.rb
|
|
347
356
|
- lib/llm_cost_tracker/charges/cost.rb
|
|
@@ -349,7 +358,15 @@ files:
|
|
|
349
358
|
- lib/llm_cost_tracker/charges/line_item.rb
|
|
350
359
|
- lib/llm_cost_tracker/check.rb
|
|
351
360
|
- lib/llm_cost_tracker/configuration.rb
|
|
361
|
+
- lib/llm_cost_tracker/configuration/budgets.rb
|
|
362
|
+
- lib/llm_cost_tracker/configuration/capture.rb
|
|
363
|
+
- lib/llm_cost_tracker/configuration/ingestion.rb
|
|
364
|
+
- lib/llm_cost_tracker/configuration/mutability.rb
|
|
365
|
+
- lib/llm_cost_tracker/configuration/pricing.rb
|
|
366
|
+
- lib/llm_cost_tracker/configuration/section.rb
|
|
367
|
+
- lib/llm_cost_tracker/configuration/tags.rb
|
|
352
368
|
- lib/llm_cost_tracker/currency.rb
|
|
369
|
+
- lib/llm_cost_tracker/deprecator.rb
|
|
353
370
|
- lib/llm_cost_tracker/doctor.rb
|
|
354
371
|
- lib/llm_cost_tracker/doctor/ingestion_check.rb
|
|
355
372
|
- lib/llm_cost_tracker/doctor/price_check.rb
|
|
@@ -369,9 +386,13 @@ files:
|
|
|
369
386
|
- lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb
|
|
370
387
|
- lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb
|
|
371
388
|
- lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb
|
|
389
|
+
- lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb
|
|
390
|
+
- lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb
|
|
372
391
|
- lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb
|
|
373
392
|
- lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb
|
|
374
393
|
- lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb
|
|
394
|
+
- lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_indexes_generator.rb
|
|
395
|
+
- lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_per_tag_budgets_generator.rb
|
|
375
396
|
- lib/llm_cost_tracker/ingestion.rb
|
|
376
397
|
- lib/llm_cost_tracker/ingestion/batch.rb
|
|
377
398
|
- lib/llm_cost_tracker/ingestion/inbox.rb
|
|
@@ -386,6 +407,7 @@ files:
|
|
|
386
407
|
- lib/llm_cost_tracker/integrations/openai/patches.rb
|
|
387
408
|
- lib/llm_cost_tracker/integrations/ruby_llm.rb
|
|
388
409
|
- lib/llm_cost_tracker/ledger.rb
|
|
410
|
+
- lib/llm_cost_tracker/ledger/isolation.rb
|
|
389
411
|
- lib/llm_cost_tracker/ledger/period.rb
|
|
390
412
|
- lib/llm_cost_tracker/ledger/period/totals.rb
|
|
391
413
|
- lib/llm_cost_tracker/ledger/rollups.rb
|
|
@@ -398,6 +420,7 @@ files:
|
|
|
398
420
|
- lib/llm_cost_tracker/ledger/schema/calls.rb
|
|
399
421
|
- lib/llm_cost_tracker/ledger/schema/ingestion/inbox_entries.rb
|
|
400
422
|
- lib/llm_cost_tracker/ledger/schema/ingestion/leases.rb
|
|
423
|
+
- lib/llm_cost_tracker/ledger/storable.rb
|
|
401
424
|
- lib/llm_cost_tracker/ledger/store.rb
|
|
402
425
|
- lib/llm_cost_tracker/ledger/tags/breakdown.rb
|
|
403
426
|
- lib/llm_cost_tracker/ledger/tags/encoding.rb
|
|
@@ -423,6 +446,7 @@ files:
|
|
|
423
446
|
- lib/llm_cost_tracker/pricing/sync/fetcher.rb
|
|
424
447
|
- lib/llm_cost_tracker/pricing/sync/registry_diff.rb
|
|
425
448
|
- lib/llm_cost_tracker/pricing/sync/registry_writer.rb
|
|
449
|
+
- lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb
|
|
426
450
|
- lib/llm_cost_tracker/pricing/unknown.rb
|
|
427
451
|
- lib/llm_cost_tracker/providers.rb
|
|
428
452
|
- lib/llm_cost_tracker/providers/anthropic/parser.rb
|
|
@@ -441,6 +465,7 @@ files:
|
|
|
441
465
|
- lib/llm_cost_tracker/providers/openai/usage_extractor.rb
|
|
442
466
|
- lib/llm_cost_tracker/providers/openai_compatible/parser.rb
|
|
443
467
|
- lib/llm_cost_tracker/railtie.rb
|
|
468
|
+
- lib/llm_cost_tracker/redaction.rb
|
|
444
469
|
- lib/llm_cost_tracker/report.rb
|
|
445
470
|
- lib/llm_cost_tracker/report/data.rb
|
|
446
471
|
- lib/llm_cost_tracker/report/formatter.rb
|
|
@@ -473,7 +498,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
473
498
|
requirements:
|
|
474
499
|
- - ">="
|
|
475
500
|
- !ruby/object:Gem::Version
|
|
476
|
-
version: 3.
|
|
501
|
+
version: 3.3.0
|
|
477
502
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
478
503
|
requirements:
|
|
479
504
|
- - ">="
|