llm_cost_tracker 0.13.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +99 -16
  3. data/README.md +17 -29
  4. data/app/controllers/llm_cost_tracker/application_controller.rb +19 -5
  5. data/app/controllers/llm_cost_tracker/data_quality_controller.rb +1 -0
  6. data/app/controllers/llm_cost_tracker/models_controller.rb +3 -1
  7. data/app/controllers/llm_cost_tracker/pricing_controller.rb +2 -2
  8. data/app/controllers/llm_cost_tracker/tags_controller.rb +5 -4
  9. data/app/helpers/llm_cost_tracker/application_helper.rb +1 -1
  10. data/app/helpers/llm_cost_tracker/dashboard_query_helper.rb +16 -0
  11. data/app/models/llm_cost_tracker/call.rb +10 -4
  12. data/app/models/llm_cost_tracker/call_rollup.rb +19 -3
  13. data/app/services/llm_cost_tracker/dashboard/data_quality.rb +13 -0
  14. data/app/services/llm_cost_tracker/dashboard/filter.rb +24 -21
  15. data/app/services/llm_cost_tracker/dashboard/monthly_budget.rb +1 -1
  16. data/app/services/llm_cost_tracker/dashboard/pagination.rb +2 -1
  17. data/app/services/llm_cost_tracker/dashboard/params.rb +10 -0
  18. data/app/services/llm_cost_tracker/dashboard/pricing_overview.rb +3 -3
  19. data/app/services/llm_cost_tracker/dashboard/setup_state.rb +5 -3
  20. data/app/views/llm_cost_tracker/calls/show.html.erb +6 -8
  21. data/app/views/llm_cost_tracker/data_quality/index.html.erb +11 -0
  22. data/app/views/llm_cost_tracker/shared/_filter_pill_date.html.erb +1 -3
  23. data/app/views/llm_cost_tracker/shared/_filter_pill_model.html.erb +1 -3
  24. data/app/views/llm_cost_tracker/shared/_filter_pill_provider.html.erb +1 -3
  25. data/app/views/llm_cost_tracker/shared/_filter_pill_stream.html.erb +1 -3
  26. data/app/views/llm_cost_tracker/tags/show.html.erb +3 -0
  27. data/config/routes.rb +6 -1
  28. data/lib/llm_cost_tracker/budget/per_tag.rb +163 -0
  29. data/lib/llm_cost_tracker/budget.rb +120 -32
  30. data/lib/llm_cost_tracker/capture/event_window.rb +99 -0
  31. data/lib/llm_cost_tracker/capture/sse.rb +108 -32
  32. data/lib/llm_cost_tracker/capture/stream_collector.rb +17 -56
  33. data/lib/llm_cost_tracker/capture/stream_tap.rb +57 -0
  34. data/lib/llm_cost_tracker/capture/stream_tracker.rb +5 -35
  35. data/lib/llm_cost_tracker/charges/cost_status.rb +4 -3
  36. data/lib/llm_cost_tracker/configuration/budgets.rb +93 -0
  37. data/lib/llm_cost_tracker/configuration/capture.rb +42 -0
  38. data/lib/llm_cost_tracker/configuration/ingestion.rb +20 -0
  39. data/lib/llm_cost_tracker/configuration/mutability.rb +33 -0
  40. data/lib/llm_cost_tracker/configuration/pricing.rb +36 -0
  41. data/lib/llm_cost_tracker/configuration/section.rb +59 -0
  42. data/lib/llm_cost_tracker/configuration/tags.rb +52 -0
  43. data/lib/llm_cost_tracker/configuration.rb +69 -125
  44. data/lib/llm_cost_tracker/deprecator.rb +9 -0
  45. data/lib/llm_cost_tracker/doctor/ingestion_check.rb +17 -8
  46. data/lib/llm_cost_tracker/doctor/price_check.rb +2 -1
  47. data/lib/llm_cost_tracker/doctor.rb +4 -4
  48. data/lib/llm_cost_tracker/engine.rb +4 -0
  49. data/lib/llm_cost_tracker/errors.rb +25 -3
  50. data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb +2 -2
  51. data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +2 -2
  52. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb +0 -1
  53. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +8 -4
  54. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +50 -30
  55. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb +40 -0
  56. data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb +41 -0
  57. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_indexes_generator.rb +30 -0
  58. data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_per_tag_budgets_generator.rb +30 -0
  59. data/lib/llm_cost_tracker/ingestion/batch.rb +38 -7
  60. data/lib/llm_cost_tracker/ingestion/pool.rb +9 -2
  61. data/lib/llm_cost_tracker/ingestion.rb +3 -7
  62. data/lib/llm_cost_tracker/integrations/anthropic.rb +7 -1
  63. data/lib/llm_cost_tracker/integrations/base.rb +17 -1
  64. data/lib/llm_cost_tracker/integrations/openai/batch_capture.rb +16 -13
  65. data/lib/llm_cost_tracker/integrations/ruby_llm.rb +53 -18
  66. data/lib/llm_cost_tracker/ledger/isolation.rb +30 -0
  67. data/lib/llm_cost_tracker/ledger/period/totals.rb +3 -2
  68. data/lib/llm_cost_tracker/ledger/rollups.rb +45 -7
  69. data/lib/llm_cost_tracker/ledger/storable.rb +16 -0
  70. data/lib/llm_cost_tracker/ledger/store.rb +21 -13
  71. data/lib/llm_cost_tracker/ledger/tags/encoding.rb +15 -5
  72. data/lib/llm_cost_tracker/ledger.rb +1 -0
  73. data/lib/llm_cost_tracker/logging.rb +5 -5
  74. data/lib/llm_cost_tracker/middleware/faraday.rb +33 -44
  75. data/lib/llm_cost_tracker/parsers.rb +5 -1
  76. data/lib/llm_cost_tracker/prices.json +2089 -372
  77. data/lib/llm_cost_tracker/pricing/backfill.rb +11 -1
  78. data/lib/llm_cost_tracker/pricing/calculation.rb +4 -4
  79. data/lib/llm_cost_tracker/pricing/effective_prices.rb +26 -15
  80. data/lib/llm_cost_tracker/pricing/matcher.rb +7 -0
  81. data/lib/llm_cost_tracker/pricing/rate.rb +1 -2
  82. data/lib/llm_cost_tracker/pricing/registry.rb +17 -6
  83. data/lib/llm_cost_tracker/pricing/sync/change_printer.rb +6 -1
  84. data/lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb +47 -0
  85. data/lib/llm_cost_tracker/pricing/sync.rb +31 -14
  86. data/lib/llm_cost_tracker/pricing/unknown.rb +11 -8
  87. data/lib/llm_cost_tracker/providers/anthropic/usage_extractor.rb +3 -2
  88. data/lib/llm_cost_tracker/providers/azure/parser.rb +19 -0
  89. data/lib/llm_cost_tracker/providers/gemini/parser.rb +4 -0
  90. data/lib/llm_cost_tracker/providers/openai/model_families.rb +0 -7
  91. data/lib/llm_cost_tracker/providers/openai/response_parser.rb +10 -3
  92. data/lib/llm_cost_tracker/providers/openai/usage_extractor.rb +14 -8
  93. data/lib/llm_cost_tracker/providers/openai_compatible/parser.rb +6 -2
  94. data/lib/llm_cost_tracker/railtie.rb +3 -7
  95. data/lib/llm_cost_tracker/redaction.rb +32 -0
  96. data/lib/llm_cost_tracker/report/data.rb +2 -2
  97. data/lib/llm_cost_tracker/retention.rb +22 -10
  98. data/lib/llm_cost_tracker/tags/context.rb +3 -3
  99. data/lib/llm_cost_tracker/tags/sanitizer.rb +14 -40
  100. data/lib/llm_cost_tracker/tracker.rb +16 -26
  101. data/lib/llm_cost_tracker/usage/catalog.rb +1 -2
  102. data/lib/llm_cost_tracker/version.rb +1 -1
  103. data/lib/llm_cost_tracker.rb +10 -3
  104. data/lib/tasks/llm_cost_tracker.rake +29 -14
  105. metadata +37 -12
@@ -8,14 +8,10 @@ module LlmCostTracker
8
8
  app.config.eager_load_paths << models_path unless app.config.eager_load_paths.include?(models_path)
9
9
  end
10
10
 
11
+ GENERATOR_FILES = File.expand_path("generators/llm_cost_tracker/*_generator.rb", __dir__)
12
+
11
13
  generators do
12
- require_relative "generators/llm_cost_tracker/install_generator"
13
- require_relative "generators/llm_cost_tracker/prices_generator"
14
- require_relative "generators/llm_cost_tracker/call_rollups_generator"
15
- require_relative "generators/llm_cost_tracker/async_ingestion_generator"
16
- require_relative "generators/llm_cost_tracker/upgrade_call_rollups_provider_generator"
17
- require_relative "generators/llm_cost_tracker/upgrade_image_tokens_generator"
18
- require_relative "generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator"
14
+ Dir[GENERATOR_FILES].each { |path| require path }
19
15
  end
20
16
  end
21
17
  end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module LlmCostTracker
4
+ module Redaction
5
+ REDACTED = "[REDACTED]"
6
+
7
+ TOKEN = Regexp.union(
8
+ /sk-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}/,
9
+ /AKIA[0-9A-Z]{16}/,
10
+ /gh[opsur]_[A-Za-z0-9]{16,}/,
11
+ /github_pat_[A-Za-z0-9_]{20,}/,
12
+ /eyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/i,
13
+ /bearer\s+[A-Za-z0-9_.-]{20,}/i,
14
+ /xox[abprs]-[A-Za-z0-9-]{10,}/,
15
+ /(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}/,
16
+ /AIza[0-9A-Za-z_-]{35}/
17
+ )
18
+ EMBEDDED_TOKEN = /(?<![A-Za-z0-9_-])(?:#{TOKEN})(?![A-Za-z0-9_-])/
19
+ PARAM = /(?<![A-Za-z0-9])((?:api)?key|token|secret|password|sig(?:nature)?)=[^&\s"']+/i
20
+ HEADER = /((?:authorization|api-key)["']?\s*(?:=>|:)\s*["']?)(?:[A-Za-z]+\s+)?[^\s"',;]+/i
21
+ USERINFO = %r{://[^/\s@?#]+@}
22
+ private_constant :TOKEN, :EMBEDDED_TOKEN, :PARAM, :HEADER, :USERINFO
23
+
24
+ def self.text(value)
25
+ value.to_s.scrub
26
+ .gsub(USERINFO, "://#{REDACTED}@")
27
+ .gsub(HEADER, "\\1#{REDACTED}")
28
+ .gsub(PARAM, "\\1=#{REDACTED}")
29
+ .gsub(EMBEDDED_TOKEN, REDACTED)
30
+ end
31
+ end
32
+ end
@@ -34,7 +34,7 @@ module LlmCostTracker
34
34
  end
35
35
  from = now - days.days
36
36
  scope = LlmCostTracker::Call.where(tracked_at: from..now)
37
- tag_breakdowns ||= LlmCostTracker.configuration.report_tag_breakdowns || []
37
+ tag_breakdowns ||= LlmCostTracker.configuration.tags.report_breakdown_keys || []
38
38
  aggregate = totals(scope)
39
39
 
40
40
  new(
@@ -55,7 +55,7 @@ module LlmCostTracker
55
55
  def self.totals(scope)
56
56
  scope
57
57
  .select(
58
- "COALESCE(SUM(total_cost), 0) AS total_cost, " \
58
+ "COALESCE(SUM(#{LlmCostTracker::Call.qualified(:total_cost)}), 0) AS total_cost, " \
59
59
  "COUNT(*) AS requests_count, " \
60
60
  "AVG(latency_ms) AS average_latency_ms, " \
61
61
  "COALESCE(SUM(CASE WHEN #{Charges::CostStatus.unknown_pricing_sql} " \
@@ -3,6 +3,8 @@
3
3
  module LlmCostTracker
4
4
  module Retention
5
5
  DEFAULT_BATCH_SIZE = 5_000
6
+ ROLLUP_COLUMNS = %i[tracked_at total_cost pricing_snapshot provider].freeze
7
+ private_constant :ROLLUP_COLUMNS
6
8
 
7
9
  class << self
8
10
  def prune(older_than:, batch_size: DEFAULT_BATCH_SIZE, now: Time.now.utc)
@@ -26,11 +28,23 @@ module LlmCostTracker
26
28
  require_relative "ingestion"
27
29
  return 0 unless LlmCostTracker::Ingestion::InboxEntry.table_exists?
28
30
 
29
- LlmCostTracker::Ingestion::InboxEntry.where(tracked_at: ...cutoff).delete_all
31
+ scope = LlmCostTracker::Ingestion::InboxEntry.where(tracked_at: ...cutoff)
32
+ warn_undrained(scope.pending)
33
+ scope.delete_all
30
34
  end
31
35
 
32
36
  private
33
37
 
38
+ def warn_undrained(scope)
39
+ count, cost = scope.pick(Arel.sql("COUNT(*)"), Arel.sql("COALESCE(SUM(total_cost), 0)"))
40
+ return if count.to_i.zero?
41
+
42
+ Logging.warn(
43
+ "Retention.prune_inbox is deleting #{count} inbox row(s) worth #{cost} that were never drained " \
44
+ "into the ledger. Their spend is lost. Drain the inbox before pruning, or raise the retention window."
45
+ )
46
+ end
47
+
34
48
  def resolve_cutoff(older_than, now)
35
49
  cutoff = case older_than
36
50
  when Time, DateTime then older_than.utc
@@ -58,22 +72,20 @@ module LlmCostTracker
58
72
 
59
73
  def prune_batch(cutoff, batch_size)
60
74
  LlmCostTracker::Call.transaction do
61
- cache_rollups = LlmCostTracker.configuration.cache_rollups
62
- rows = prunable_rows(cutoff, batch_size, with_rollup_columns: cache_rollups)
75
+ rows = prunable_rows(cutoff, batch_size)
63
76
  next 0 if rows.empty?
64
77
 
65
- ids = cache_rollups ? rows.map(&:id) : rows
66
- deleted = LlmCostTracker::Call.where(id: ids).delete_all
67
- LlmCostTracker::Ledger::Rollups.decrement!(rows) if cache_rollups && deleted.positive?
78
+ deleted = LlmCostTracker::Call.where(id: rows.map(&:id)).delete_all
79
+ LlmCostTracker::Ledger::Rollups.decrement!(rows) if deleted.positive?
68
80
  deleted
69
81
  end
70
82
  end
71
83
 
72
- def prunable_rows(cutoff, batch_size, with_rollup_columns:)
84
+ def prunable_rows(cutoff, batch_size)
73
85
  relation = LlmCostTracker::Call.where(tracked_at: ...cutoff).order(:id).limit(batch_size).lock
74
- return relation.pluck(:id) unless with_rollup_columns
75
-
76
- relation.select(:id, :tracked_at, :total_cost, :pricing_snapshot, :provider).to_a
86
+ columns = [:id]
87
+ columns += ROLLUP_COLUMNS if LlmCostTracker::Ledger::Rollups.cache_active?
88
+ relation.select(*columns).to_a
77
89
  end
78
90
  end
79
91
  end
@@ -18,15 +18,15 @@ module LlmCostTracker
18
18
 
19
19
  def tags
20
20
  config = LlmCostTracker.configuration
21
- base = config.static_sanitized_default_tags ||
22
- Sanitizer.call(call_default_tags(config.default_tags).to_h)
21
+ base = config.tags.static_sanitized_default ||
22
+ Sanitizer.call(call_default_tags(config.tags.default).to_h)
23
23
  base.merge(*Array(ActiveSupport::IsolatedExecutionState[KEY]))
24
24
  end
25
25
 
26
26
  def call_default_tags(proc_or_lambda)
27
27
  proc_or_lambda.call
28
28
  rescue StandardError => e
29
- Logging.warn("LlmCostTracker default_tags proc raised: #{e.class}: #{e.message}; using empty default tags")
29
+ Logging.warn("LlmCostTracker tags.default proc raised: #{e.class}: #{e.message}; using empty default tags")
30
30
  {}
31
31
  end
32
32
  end
@@ -2,30 +2,20 @@
2
2
 
3
3
  require "active_support/core_ext/string/inflections"
4
4
 
5
+ require_relative "../ledger/storable"
6
+ require_relative "../redaction"
7
+
5
8
  module LlmCostTracker
6
9
  module Tags
7
10
  module Sanitizer
8
- REDACTED_VALUE = "[REDACTED]"
9
-
10
- SECRET_VALUE_PATTERNS = [
11
- /\Ask-(?:ant-|admin-|proj-|svcacct-|live-|test-)?[A-Za-z0-9_-]{16,}\z/,
12
- /\AAKIA[0-9A-Z]{16}\z/,
13
- /\Agh[opsur]_[A-Za-z0-9]{16,}\z/,
14
- /\Agithub_pat_[A-Za-z0-9_]{20,}\z/,
15
- /\Aeyj[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\z/i,
16
- /\Abearer\s+[A-Za-z0-9_.-]{20,}\z/i,
17
- /\Axox[abprs]-[A-Za-z0-9-]{10,}\z/,
18
- /\A(?:sk|rk|pk)_(?:live|test)_[A-Za-z0-9]{20,}\z/,
19
- /\AAIza[0-9A-Za-z_-]{35}\z/
20
- ].freeze
21
- private_constant :SECRET_VALUE_PATTERNS
11
+ REDACTED_VALUE = Redaction::REDACTED
22
12
 
23
13
  class << self
24
14
  def call(tags, config: LlmCostTracker.configuration)
25
15
  tags = (tags || {}).to_h
26
- redacted = config.normalized_redacted_tag_keys
27
- limit = [config.max_tag_value_bytesize.to_i, 0].max
28
- max_count = [config.max_tag_count.to_i, 0].max
16
+ redacted = config.tags.normalized_redacted_keys
17
+ limit = [config.tags.max_value_bytesize.to_i, 0].max
18
+ max_count = [config.tags.max_count.to_i, 0].max
29
19
  tags.to_a.last(max_count).each_with_object({}) do |(key, value), sanitized|
30
20
  next unless valid_key?(key)
31
21
 
@@ -43,7 +33,7 @@ module LlmCostTracker
43
33
 
44
34
  def cap(tags, config: LlmCostTracker.configuration)
45
35
  tags = (tags || {}).to_h
46
- max_count = [config.max_tag_count.to_i, 0].max
36
+ max_count = [config.tags.max_count.to_i, 0].max
47
37
  return tags if tags.size <= max_count
48
38
 
49
39
  tags.to_a.last(max_count).to_h
@@ -58,15 +48,9 @@ module LlmCostTracker
58
48
  def sanitized_value(key, value, redacted, limit)
59
49
  return REDACTED_VALUE if redacted_key?(key, redacted)
60
50
 
61
- scrubbed = scrub_secrets(value)
62
- return REDACTED_VALUE if scrubbed.equal?(REDACTED_SENTINEL)
63
-
64
- scalar_truncate(scrubbed, limit)
51
+ scalar_truncate(scrub_secrets(Ledger::Storable.clean(value)), limit)
65
52
  end
66
53
 
67
- REDACTED_SENTINEL = Object.new.freeze
68
- private_constant :REDACTED_SENTINEL
69
-
70
54
  def scalar_truncate(value, limit)
71
55
  case value
72
56
  when Hash
@@ -86,26 +70,16 @@ module LlmCostTracker
86
70
  def scrub_secrets(value)
87
71
  case value
88
72
  when Hash
89
- value.each_with_object({}) do |(key, nested), out|
90
- scrubbed = scrub_secrets(nested)
91
- out[key] = scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
92
- end
73
+ value.transform_values { |nested| scrub_secrets(nested) }
93
74
  when Array
94
- value.map do |nested|
95
- scrubbed = scrub_secrets(nested)
96
- scrubbed.equal?(REDACTED_SENTINEL) ? REDACTED_VALUE : scrubbed
97
- end
75
+ value.map { |nested| scrub_secrets(nested) }
98
76
  else
99
- secret_shaped?(value.to_s) ? REDACTED_SENTINEL : value
77
+ string = value.to_s
78
+ scrubbed = Redaction.text(string)
79
+ scrubbed == string ? value : scrubbed
100
80
  end
101
81
  end
102
82
 
103
- def secret_shaped?(string)
104
- return false if string.bytesize < 16
105
-
106
- SECRET_VALUE_PATTERNS.any? { |pattern| pattern.match?(string) }
107
- end
108
-
109
83
  def redacted_key?(key, redacted)
110
84
  return false if redacted.empty?
111
85
 
@@ -25,25 +25,12 @@ module LlmCostTracker
25
25
  usage_source: event.usage_source
26
26
  )
27
27
 
28
- if enforce_budget
29
- Budget.enforce!(provider: event.provider,
30
- model: event.model,
31
- estimate: calculation.cost&.total,
32
- force: true)
33
- end
28
+ tags = build_tags(context_tags: context_tags, metadata: metadata)
34
29
 
35
- if calculation.token_cost.nil? && event.token_usage.total_tokens.positive? &&
36
- calculation.priced_line_items.none?(&:priced?)
37
- Pricing::Unknown.process(event.model)
38
- end
30
+ unpriced = calculation.token_cost.nil? && event.token_usage.total_tokens.positive? &&
31
+ calculation.priced_line_items.none?(&:priced?)
39
32
 
40
- event = build_event(
41
- event: event,
42
- calculation: calculation,
43
- metadata: metadata,
44
- latency_ms: latency_ms,
45
- context_tags: context_tags
46
- )
33
+ event = build_event(event: event, calculation: calculation, tags: tags, latency_ms: latency_ms)
47
34
 
48
35
  if Ingestion.async?
49
36
  Ingestion::Inbox.save(event)
@@ -54,11 +41,20 @@ module LlmCostTracker
54
41
 
55
42
  yield if block_given?
56
43
  notify_subscribers(event)
57
- Budget.check!(event)
44
+ Pricing::Unknown.process(event.model, pricing_mode: calculation.mode) if unpriced
45
+ behavior_override = :raise if enforce_budget
46
+ Budget.check!(event, behavior_override: behavior_override)
47
+ Budget.check_persisted!([event], behavior_override: behavior_override) unless Ingestion.async?
58
48
 
59
49
  event
60
50
  end
61
51
 
52
+ def build_tags(context_tags:, metadata:)
53
+ resolved = (context_tags || LlmCostTracker::Tags::Context.tags).to_h
54
+ sanitized_metadata = LlmCostTracker::Tags::Sanitizer.call(metadata.to_h)
55
+ LlmCostTracker::Tags::Sanitizer.cap(resolved.merge(sanitized_metadata)).freeze
56
+ end
57
+
62
58
  private
63
59
 
64
60
  def notify_subscribers(event)
@@ -69,13 +65,12 @@ module LlmCostTracker
69
65
  Logging.warn("Subscriber raised on #{EVENT_NAME}: #{e.class}: #{e.message}")
70
66
  end
71
67
 
72
- def build_event(event:, calculation:, metadata:, latency_ms:, context_tags:)
73
- context_tags = (context_tags || LlmCostTracker::Tags::Context.tags).to_h
68
+ def build_event(event:, calculation:, tags:, latency_ms:)
74
69
  event.with(
75
70
  event_id: SecureRandom.uuid,
76
71
  pricing_mode: calculation.mode,
77
72
  cost: calculation.cost,
78
- tags: build_tags(context_tags: context_tags, metadata: metadata),
73
+ tags: tags,
79
74
  latency_ms: finite_latency_ms(latency_ms),
80
75
  tracked_at: Time.now.utc,
81
76
  cost_status: calculation.cost_status,
@@ -84,11 +79,6 @@ module LlmCostTracker
84
79
  )
85
80
  end
86
81
 
87
- def build_tags(context_tags:, metadata:)
88
- sanitized_metadata = LlmCostTracker::Tags::Sanitizer.call(metadata.to_h)
89
- LlmCostTracker::Tags::Sanitizer.cap(context_tags.merge(sanitized_metadata)).freeze
90
- end
91
-
92
82
  def finite_latency_ms(latency_ms)
93
83
  return nil if latency_ms.nil?
94
84
 
@@ -16,8 +16,7 @@ module LlmCostTracker
16
16
  "request" => "per_request",
17
17
  "session" => "per_session",
18
18
  "hour" => "per_hour",
19
- "minute" => "per_minute",
20
- "image" => "per_image"
19
+ "minute" => "per_minute"
21
20
  }.freeze
22
21
 
23
22
  class << self
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module LlmCostTracker
4
- VERSION = "0.13.0"
4
+ VERSION = "0.14.1"
5
5
  end
@@ -57,6 +57,7 @@ module LlmCostTracker
57
57
  yield(config)
58
58
  config.finalize!
59
59
  Pricing::Registry.reset!
60
+ Pricing::Registry.validate_file!(config.pricing.file)
60
61
  Integrations.install!
61
62
  config
62
63
  end
@@ -110,7 +111,12 @@ module LlmCostTracker
110
111
  provider_workspace_id: nil,
111
112
  pricing_mode: nil)
112
113
  require_relative "llm_cost_tracker/capture/stream_collector"
113
- Budget.enforce!(provider: provider, model: model, force: true) if enforce_budget
114
+ if enforce_budget
115
+ Budget.enforce!(provider: provider,
116
+ model: model,
117
+ force: true,
118
+ tags: Tracker.build_tags(context_tags: nil, metadata: tags))
119
+ end
114
120
  collector = Capture::StreamCollector.new(
115
121
  provider: provider.to_s,
116
122
  model: model,
@@ -123,10 +129,11 @@ module LlmCostTracker
123
129
  metadata: tags
124
130
  )
125
131
  yield collector
126
- collector.finish!
127
- rescue StandardError
132
+ rescue Exception # rubocop:disable Lint/RescueException -- record the spend and keep the caller's exception
128
133
  collector&.finish!(errored: true)
129
134
  raise
135
+ else
136
+ collector.finish!
130
137
  end
131
138
  end
132
139
  end
@@ -6,7 +6,15 @@ require "rails/generators"
6
6
  require_relative "../llm_cost_tracker/generators/llm_cost_tracker/install_generator"
7
7
  require_relative "../llm_cost_tracker/pricing/sync/change_printer"
8
8
 
9
- # rubocop:disable Metrics/BlockLength
9
+ module LlmCostTrackerTasks
10
+ def self.price_refresh_output_path
11
+ path = LlmCostTracker::Pricing::Sync.configured_output_path
12
+ FileUtils.mkdir_p(File.dirname(path))
13
+ path
14
+ end
15
+ end
16
+
17
+ # rubocop:disable-next Metrics/BlockLength
10
18
  namespace :llm_cost_tracker do
11
19
  desc "Install LLM Cost Tracker with dashboard and prices, migrate, and run doctor"
12
20
  task :setup do
@@ -75,22 +83,36 @@ namespace :llm_cost_tracker do
75
83
  puts "llm_cost_tracker: rebuilt #{rows} rollup rows from the calls ledger"
76
84
  end
77
85
 
86
+ desc "Copy call cost and time onto existing llm_cost_tracker_call_tags rows so per-tag " \
87
+ "budgets count history recorded before the upgrade. Use BATCH_SIZE=N to tune."
88
+ task backfill_tag_costs: :environment do
89
+ unless LlmCostTracker::Budget::PerTag.columns?
90
+ abort("llm_cost_tracker: llm_cost_tracker_call_tags is missing the cost columns; " \
91
+ "run the upgrade_per_tag_budgets generator and migrate first")
92
+ end
93
+ batch_size = (ENV["BATCH_SIZE"] || LlmCostTracker::Budget::PerTag::DEFAULT_BACKFILL_BATCH).to_i
94
+ filled = LlmCostTracker::Budget::PerTag.backfill(batch_size: batch_size)
95
+ puts "llm_cost_tracker: filled cost and time on #{filled} tag rows"
96
+ end
97
+
78
98
  namespace :prices do
79
99
  desc(
80
100
  "Refresh the configured pricing file from the maintained LLM Cost Tracker price snapshot. " \
81
- "Use PREVIEW=1 to preview, URL=... to override the source, or OUTPUT=path/to/file.json."
101
+ "Use PREVIEW=1 to preview, FORCE=1 to accept suspicious price changes, URL=... to override the source, " \
102
+ "or OUTPUT=path/to/file.json."
82
103
  )
83
104
  task :refresh do
84
105
  Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
85
106
  require_relative "../llm_cost_tracker"
86
107
 
87
- output_path = price_refresh_output_path
108
+ output_path = LlmCostTrackerTasks.price_refresh_output_path
88
109
  source_url = LlmCostTracker::Pricing::Sync.configured_remote_url
89
110
  preview = ENV["PREVIEW"] == "1"
90
111
  result = LlmCostTracker::Pricing::Sync.refresh(
91
112
  path: output_path,
92
113
  url: source_url,
93
- preview: preview
114
+ preview: preview,
115
+ force: ENV["FORCE"] == "1"
94
116
  )
95
117
 
96
118
  action = if preview
@@ -104,7 +126,7 @@ namespace :llm_cost_tracker do
104
126
  puts "llm_cost_tracker: #{action} pricing file #{result.path}"
105
127
  puts " source: #{result.source_url}"
106
128
  puts " version: #{result.source_version.inspect}" if result.source_version
107
- LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes)
129
+ LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
108
130
  end
109
131
 
110
132
  desc "Compare the current pricing file with the maintained LLM Cost Tracker price snapshot."
@@ -112,23 +134,16 @@ namespace :llm_cost_tracker do
112
134
  Rake::Task["environment"].invoke if Rake::Task.task_defined?("environment")
113
135
  require_relative "../llm_cost_tracker"
114
136
 
115
- output_path = price_refresh_output_path
137
+ output_path = LlmCostTrackerTasks.price_refresh_output_path
116
138
  source_url = LlmCostTracker::Pricing::Sync.configured_remote_url
117
139
  result = LlmCostTracker::Pricing::Sync.check(path: output_path, url: source_url)
118
140
 
119
141
  puts "llm_cost_tracker: checked pricing file #{result.path}"
120
142
  puts " source: #{result.source_url}"
121
143
  puts " version: #{result.source_version.inspect}" if result.source_version
122
- LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes)
144
+ LlmCostTracker::Pricing::Sync::ChangePrinter.call(result.changes, suspicious: result.suspicious)
123
145
  puts " pricing is up to date" if result.up_to_date
124
146
  abort("llm_cost_tracker: pricing check failed") unless result.up_to_date
125
147
  end
126
148
  end
127
149
  end
128
- # rubocop:enable Metrics/BlockLength
129
-
130
- def price_refresh_output_path
131
- path = LlmCostTracker::Pricing::Sync.configured_output_path
132
- FileUtils.mkdir_p(File.dirname(path))
133
- path
134
- end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: llm_cost_tracker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.14.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sergii Khomenko
@@ -109,42 +109,42 @@ dependencies:
109
109
  requirements:
110
110
  - - "~>"
111
111
  - !ruby/object:Gem::Version
112
- version: '1.42'
112
+ version: '1.61'
113
113
  type: :development
114
114
  prerelease: false
115
115
  version_requirements: !ruby/object:Gem::Requirement
116
116
  requirements:
117
117
  - - "~>"
118
118
  - !ruby/object:Gem::Version
119
- version: '1.42'
119
+ version: '1.61'
120
120
  - !ruby/object:Gem::Dependency
121
121
  name: nokogiri
122
122
  requirement: !ruby/object:Gem::Requirement
123
123
  requirements:
124
124
  - - "~>"
125
125
  - !ruby/object:Gem::Version
126
- version: '1.16'
126
+ version: '1.19'
127
127
  type: :development
128
128
  prerelease: false
129
129
  version_requirements: !ruby/object:Gem::Requirement
130
130
  requirements:
131
131
  - - "~>"
132
132
  - !ruby/object:Gem::Version
133
- version: '1.16'
133
+ version: '1.19'
134
134
  - !ruby/object:Gem::Dependency
135
135
  name: openai
136
136
  requirement: !ruby/object:Gem::Requirement
137
137
  requirements:
138
138
  - - "~>"
139
139
  - !ruby/object:Gem::Version
140
- version: '0.63'
140
+ version: '0.80'
141
141
  type: :development
142
142
  prerelease: false
143
143
  version_requirements: !ruby/object:Gem::Requirement
144
144
  requirements:
145
145
  - - "~>"
146
146
  - !ruby/object:Gem::Version
147
- version: '0.63'
147
+ version: '0.80'
148
148
  - !ruby/object:Gem::Dependency
149
149
  name: pg
150
150
  requirement: !ruby/object:Gem::Requirement
@@ -205,16 +205,22 @@ dependencies:
205
205
  name: ruby_llm
206
206
  requirement: !ruby/object:Gem::Requirement
207
207
  requirements:
208
- - - "~>"
208
+ - - ">="
209
+ - !ruby/object:Gem::Version
210
+ version: '1.16'
211
+ - - "<"
209
212
  - !ruby/object:Gem::Version
210
- version: '1.15'
213
+ version: '3'
211
214
  type: :development
212
215
  prerelease: false
213
216
  version_requirements: !ruby/object:Gem::Requirement
214
217
  requirements:
215
- - - "~>"
218
+ - - ">="
219
+ - !ruby/object:Gem::Version
220
+ version: '1.16'
221
+ - - "<"
216
222
  - !ruby/object:Gem::Version
217
- version: '1.15'
223
+ version: '3'
218
224
  - !ruby/object:Gem::Dependency
219
225
  name: simplecov
220
226
  requirement: !ruby/object:Gem::Requirement
@@ -339,9 +345,12 @@ files:
339
345
  - lib/llm_cost_tracker.rb
340
346
  - lib/llm_cost_tracker/assets.rb
341
347
  - lib/llm_cost_tracker/budget.rb
348
+ - lib/llm_cost_tracker/budget/per_tag.rb
349
+ - lib/llm_cost_tracker/capture/event_window.rb
342
350
  - lib/llm_cost_tracker/capture/sdk_payload.rb
343
351
  - lib/llm_cost_tracker/capture/sse.rb
344
352
  - lib/llm_cost_tracker/capture/stream_collector.rb
353
+ - lib/llm_cost_tracker/capture/stream_tap.rb
345
354
  - lib/llm_cost_tracker/capture/stream_tracker.rb
346
355
  - lib/llm_cost_tracker/capture_verifier.rb
347
356
  - lib/llm_cost_tracker/charges/cost.rb
@@ -349,7 +358,15 @@ files:
349
358
  - lib/llm_cost_tracker/charges/line_item.rb
350
359
  - lib/llm_cost_tracker/check.rb
351
360
  - lib/llm_cost_tracker/configuration.rb
361
+ - lib/llm_cost_tracker/configuration/budgets.rb
362
+ - lib/llm_cost_tracker/configuration/capture.rb
363
+ - lib/llm_cost_tracker/configuration/ingestion.rb
364
+ - lib/llm_cost_tracker/configuration/mutability.rb
365
+ - lib/llm_cost_tracker/configuration/pricing.rb
366
+ - lib/llm_cost_tracker/configuration/section.rb
367
+ - lib/llm_cost_tracker/configuration/tags.rb
352
368
  - lib/llm_cost_tracker/currency.rb
369
+ - lib/llm_cost_tracker/deprecator.rb
353
370
  - lib/llm_cost_tracker/doctor.rb
354
371
  - lib/llm_cost_tracker/doctor/ingestion_check.rb
355
372
  - lib/llm_cost_tracker/doctor/price_check.rb
@@ -369,9 +386,13 @@ files:
369
386
  - lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb
370
387
  - lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb
371
388
  - lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb
389
+ - lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_indexes.rb.erb
390
+ - lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_per_tag_budgets.rb.erb
372
391
  - lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb
373
392
  - lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb
374
393
  - lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb
394
+ - lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_indexes_generator.rb
395
+ - lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_per_tag_budgets_generator.rb
375
396
  - lib/llm_cost_tracker/ingestion.rb
376
397
  - lib/llm_cost_tracker/ingestion/batch.rb
377
398
  - lib/llm_cost_tracker/ingestion/inbox.rb
@@ -386,6 +407,7 @@ files:
386
407
  - lib/llm_cost_tracker/integrations/openai/patches.rb
387
408
  - lib/llm_cost_tracker/integrations/ruby_llm.rb
388
409
  - lib/llm_cost_tracker/ledger.rb
410
+ - lib/llm_cost_tracker/ledger/isolation.rb
389
411
  - lib/llm_cost_tracker/ledger/period.rb
390
412
  - lib/llm_cost_tracker/ledger/period/totals.rb
391
413
  - lib/llm_cost_tracker/ledger/rollups.rb
@@ -398,6 +420,7 @@ files:
398
420
  - lib/llm_cost_tracker/ledger/schema/calls.rb
399
421
  - lib/llm_cost_tracker/ledger/schema/ingestion/inbox_entries.rb
400
422
  - lib/llm_cost_tracker/ledger/schema/ingestion/leases.rb
423
+ - lib/llm_cost_tracker/ledger/storable.rb
401
424
  - lib/llm_cost_tracker/ledger/store.rb
402
425
  - lib/llm_cost_tracker/ledger/tags/breakdown.rb
403
426
  - lib/llm_cost_tracker/ledger/tags/encoding.rb
@@ -423,6 +446,7 @@ files:
423
446
  - lib/llm_cost_tracker/pricing/sync/fetcher.rb
424
447
  - lib/llm_cost_tracker/pricing/sync/registry_diff.rb
425
448
  - lib/llm_cost_tracker/pricing/sync/registry_writer.rb
449
+ - lib/llm_cost_tracker/pricing/sync/snapshot_guard.rb
426
450
  - lib/llm_cost_tracker/pricing/unknown.rb
427
451
  - lib/llm_cost_tracker/providers.rb
428
452
  - lib/llm_cost_tracker/providers/anthropic/parser.rb
@@ -441,6 +465,7 @@ files:
441
465
  - lib/llm_cost_tracker/providers/openai/usage_extractor.rb
442
466
  - lib/llm_cost_tracker/providers/openai_compatible/parser.rb
443
467
  - lib/llm_cost_tracker/railtie.rb
468
+ - lib/llm_cost_tracker/redaction.rb
444
469
  - lib/llm_cost_tracker/report.rb
445
470
  - lib/llm_cost_tracker/report/data.rb
446
471
  - lib/llm_cost_tracker/report/formatter.rb
@@ -473,7 +498,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
473
498
  requirements:
474
499
  - - ">="
475
500
  - !ruby/object:Gem::Version
476
- version: 3.4.0
501
+ version: 3.3.0
477
502
  required_rubygems_version: !ruby/object:Gem::Requirement
478
503
  requirements:
479
504
  - - ">="